From 11e4b13986a58e2ca7a0eceb4c5f98e84cf66874 Mon Sep 17 00:00:00 2001 From: Nate O'Farrell Date: Sun, 19 May 2024 10:37:56 -0400 Subject: [PATCH] IDEA 3.1.0 Release See CHANGELOG --- .editorconfig | 13 + .gitattributes | 3 + .github/ISSUE_TEMPLATE/bug_report.md | 42 + .github/ISSUE_TEMPLATE/feature_request.md | 17 + .github/PULL_REQUEST_TEMPLATE.md | 5 + .gitignore | 193 + CHANGELOG.md | 202 + CODE_OF_CONDUCT.md | 4 + CONTRIBUTING.md | 61 + IDEA_VERSION.txt | 1 + LICENSE.txt | 175 + NOTICE.txt | 7 + README.md | 27 + THIRD_PARTY_LICENSES.txt | 346 + deployment/ecr/idea-administrator/Dockerfile | 47 + ...grated-digital-engineering-on-aws.template | 234 + idea-admin-windows.ps1 | 109 + idea-admin.sh | 125 + requirements/dev.in | 12 + requirements/dev.txt | 130 + requirements/doc.in | 5 + requirements/doc.txt | 20 + requirements/idea-administrator.in | 8 + requirements/idea-administrator.txt | 85 + requirements/idea-cluster-manager.in | 5 + requirements/idea-cluster-manager.txt | 78 + requirements/idea-dev-lambda.in | 2 + requirements/idea-dev-lambda.txt | 9 + requirements/idea-scheduler.in | 5 + requirements/idea-scheduler.txt | 75 + requirements/idea-sdk.in | 34 + .../idea-virtual-desktop-controller.in | 3 + .../idea-virtual-desktop-controller.txt | 73 + requirements/tests.in | 5 + requirements/tests.txt | 14 + software_versions.yml | 4 + .../idea-administrator/install/install.sh | 60 + .../idea-administrator/resources/cdk/cdk.json | 11 + .../resources/cdk/cdk_toolkit_stack.yml | 602 + .../resources/config/region_ami_config.yml | 85 + .../config/region_elb_account_id.yml | 31 + .../config/region_timezone_config.yml | 24 + .../config/templates/analytics/settings.yml | 27 + .../templates/bastion-host/settings.yml | 17 + .../templates/cluster-manager/settings.yml | 102 + .../config/templates/cluster/logging.yml | 119 + .../config/templates/cluster/settings.yml | 243 + .../_templates/activedirectory.yml | 126 + .../aws_managed_activedirectory.yml | 79 + .../directoryservice/_templates/openldap.yml | 24 + .../templates/directoryservice/settings.yml | 68 + .../templates/global-settings/settings.yml | 688 + .../resources/config/templates/idea.yml | 76 + .../templates/identity-provider/settings.yml | 34 + .../config/templates/metrics/settings.yml | 93 + .../config/templates/scheduler/settings.yml | 169 + .../shared-storage/_templates/efs.yml | 13 + .../shared-storage/_templates/fsx_lustre.yml | 12 + .../templates/shared-storage/settings.yml | 147 + .../virtual-desktop-controller/settings.yml | 269 + .../resources/config/values.yml | 111 + .../resources/input_params/install_params.yml | 665 + .../input_params/shared_storage_params.yml | 578 + .../idea-admin-cdk-toolkit-policy.yml | 31 + .../idea-admin-create-policy.yml | 141 + .../idea-admin-delete-policy.yml | 68 + .../integration_tests/job_test_cases.yml | 476 + .../integration_tests/session_test_cases.yml | 98 + .../resources/lambda_functions/__init__.py | 14 + .../idea_analytics_sink/__init__.py | 10 + .../idea_analytics_sink/handler.py | 75 + .../idea_analytics_sink/requirements.txt | 1 + .../__init__.py | 10 + .../handler.py | 50 + .../__init__.py | 10 + .../handler.py | 205 + .../__init__.py | 10 + .../handler.py | 53 + .../__init__.py | 10 + .../handler.py | 92 + .../__init__.py | 10 + .../handler.py | 91 + .../__init__.py | 10 + .../handler.py | 96 + .../__init__.py | 10 + .../handler.py | 314 + .../requirements.txt | 1 + .../__init__.py | 10 + .../handler.py | 101 + .../__init__.py | 10 + .../handler.py | 152 + .../__init__.py | 10 + .../handler.py | 81 + .../idea_efs_throughput/__init__.py | 10 + .../idea_efs_throughput/handler.py | 58 + .../idea_lambda_commons/__init__.py | 14 + .../idea_lambda_commons/cfn_response.py | 64 + .../cfn_response_status.py | 22 + .../idea_lambda_commons/http_client.py | 48 + .../idea_solution_metrics/__init__.py | 10 + .../idea_solution_metrics/handler.py | 93 + .../policies/_templates/activedirectory.yml | 13 + .../policies/_templates/aws-managed-ad.yml | 13 + .../policies/_templates/custom-kms-key.yml | 9 + .../_templates/lambda-basic-execution.yml | 6 + .../policies/_templates/openldap.yml | 10 + .../amazon-prometheus-remote-write-access.yml | 6 + .../amazon-ssm-managed-instance-core.yml | 36 + .../policies/analytics-sink-lambda.yml | 31 + .../resources/policies/backup-create.yml | 266 + .../resources/policies/backup-restore.yml | 264 + .../resources/policies/backup-s3-create.yml | 64 + .../resources/policies/backup-s3-restore.yml | 43 + .../resources/policies/bastion-host.yml | 33 + .../cloud-watch-agent-server-policy.yml | 17 + .../resources/policies/cluster-manager.yml | 128 + .../resources/policies/compute-node.yml | 56 + ...ler-scheduled-event-transformer-lambda.yml | 22 + .../controller-ssm-command-pass-role.yml | 29 + .../custom-resource-cluster-endpoints.yml | 21 + .../custom-resource-ec2-create-tags.yml | 8 + .../custom-resource-get-ad-security-group.yml | 16 + ...m-resource-get-user-pool-client-secret.yml | 16 + ...custom-resource-opensearch-private-ips.yml | 19 + ...ustom-resource-self-signed-certificate.yml | 42 + ...om-resource-update-cluster-prefix-list.yml | 17 + ...ustom-resource-update-cluster-settings.yml | 17 + .../policies/ec2state-event-transformer.yml | 20 + .../policies/efs-throughput-lambda.yml | 22 + .../resources/policies/log-retention.yml | 9 + .../resources/policies/openldap-server.yml | 38 + .../resources/policies/scheduler.yml | 284 + .../solution-metrics-lambda-function.yml | 15 + .../resources/policies/spot-fleet-request.yml | 43 + .../policies/virtual-desktop-controller.yml | 155 + .../policies/virtual-desktop-dcv-broker.yml | 89 + ...virtual-desktop-dcv-connection-gateway.yml | 48 + .../policies/virtual-desktop-dcv-host.yml | 67 + .../src/ideaadministrator/__init__.py | 20 + .../src/ideaadministrator/app/__init__.py | 10 + .../app/aws_service_availability_helper.py | 309 + .../src/ideaadministrator/app/cdk/__init__.py | 10 + .../src/ideaadministrator/app/cdk/cdk_app.py | 225 + .../ideaadministrator/app/cdk/cdk_invoker.py | 1005 + .../app/cdk/constructs/__init__.py | 20 + .../app/cdk/constructs/analytics.py | 177 + .../app/cdk/constructs/backup.py | 124 + .../app/cdk/constructs/base.py | 178 + .../app/cdk/constructs/common.py | 604 + .../app/cdk/constructs/directory_service.py | 452 + .../app/cdk/constructs/dns.py | 118 + .../app/cdk/constructs/existing_resources.py | 181 + .../app/cdk/constructs/network.py | 683 + .../app/cdk/constructs/storage.py | 319 + .../app/cdk/idea_code_asset.py | 168 + .../app/cdk/stacks/__init__.py | 24 + .../app/cdk/stacks/analytics_stack.py | 358 + .../app/cdk/stacks/base_stack.py | 157 + .../app/cdk/stacks/bastion_host_stack.py | 241 + .../app/cdk/stacks/bootstrap_stack.py | 25 + .../app/cdk/stacks/cluster_manager_stack.py | 474 + .../app/cdk/stacks/cluster_stack.py | 1176 + .../app/cdk/stacks/directoryservice_stack.py | 416 + .../app/cdk/stacks/identity_provider_stack.py | 203 + .../app/cdk/stacks/metrics_stack.py | 127 + .../app/cdk/stacks/scheduler_stack.py | 516 + .../app/cdk/stacks/shared_storage_stack.py | 238 + .../virtual_desktop_controller_stack.py | 1058 + .../app/cluster_prefix_list_helper.py | 103 + .../ideaadministrator/app/config_generator.py | 575 + .../ideaadministrator/app/delete_cluster.py | 889 + .../app/deployment_helper.py | 207 + .../app/directory_service_helper.py | 91 + .../ideaadministrator/app/installer_params.py | 1227 + .../src/ideaadministrator/app/patch_helper.py | 277 + .../app/shared_storage_helper.py | 553 + .../app/single_sign_on_helper.py | 506 + .../ideaadministrator/app/support_helper.py | 206 + .../src/ideaadministrator/app/values_diff.py | 44 + .../app/vpc_endpoints_helper.py | 333 + .../src/ideaadministrator/app_constants.py | 26 + .../src/ideaadministrator/app_context.py | 44 + .../src/ideaadministrator/app_main.py | 1954 ++ .../src/ideaadministrator/app_messages.py | 29 + .../src/ideaadministrator/app_props.py | 311 + .../src/ideaadministrator/app_protocols.py | 23 + .../src/ideaadministrator/app_utils.py | 219 + .../integration_tests/__init__.py | 10 + .../cluster_manager_tests.py | 364 + .../integration_tests/scheduler/__init__.py | 10 + .../scheduler/job_test_case.py | 329 + .../integration_tests/scheduler_tests.py | 342 + .../integration_tests/test_constants.py | 77 + .../integration_tests/test_context.py | 226 + .../integration_tests/test_invoker.py | 86 + .../virtual_desktop_controller_tests.py | 1397 + .../virtual_desktop_tests_util.py | 1222 + .../src/ideaadministrator_meta/__init__.py | 15 + source/idea/idea-administrator/src/setup.py | 30 + .../idea/idea-administrator/tests/conftest.py | 10 + .../idea-administrator/tests/test_example.py | 16 + .../_templates/linux/aws_cli.jinja2 | 11 + .../_templates/linux/aws_ssm.jinja2 | 17 + .../_templates/linux/chronyd.jinja2 | 34 + .../_templates/linux/cloudwatch_agent.jinja2 | 55 + .../linux/create_idea_app_certs.jinja2 | 16 + .../_templates/linux/dcv_server.jinja2 | 99 + .../linux/dcv_session_manager_agent.jinja2 | 42 + .../linux/disable_motd_update.jinja2 | 7 + .../linux/disable_nouveau_drivers.jinja2 | 23 + .../_templates/linux/disable_se_linux.jinja2 | 12 + .../linux/disable_strict_host_check.jinja2 | 6 + .../_templates/linux/disable_ulimit.jinja2 | 8 + .../_templates/linux/efs_mount_helper.jinja2 | 27 + .../_templates/linux/epel_repo.jinja2 | 20 + .../_templates/linux/fsx_lustre_client.jinja2 | 56 + .../fsx_lustre_client_tuning_postmount.jinja2 | 20 + .../fsx_lustre_client_tuning_prereboot.jinja2 | 14 + .../_templates/linux/gpu_drivers.jinja2 | 246 + .../linux/idea_service_account.jinja2 | 8 + .../linux/join_activedirectory.jinja2 | 182 + .../linux/join_directoryservice.jinja2 | 8 + .../_templates/linux/join_openldap.jinja2 | 108 + .../idea-bootstrap/_templates/linux/jq.jinja2 | 11 + .../_templates/linux/motd.jinja2 | 15 + .../linux/mount_shared_storage.jinja2 | 62 + .../_templates/linux/nfs_utils.jinja2 | 12 + .../_templates/linux/nodejs.jinja2 | 28 + .../_templates/linux/openmpi.jinja2 | 29 + .../_templates/linux/openpbs.jinja2 | 90 + .../_templates/linux/openpbs_client.jinja2 | 23 + .../_templates/linux/prometheus.jinja2 | 58 + .../linux/prometheus_node_exporter.jinja2 | 46 + .../_templates/linux/python.jinja2 | 59 + .../_templates/linux/restrict_ssh.jinja2 | 11 + .../_templates/linux/stunnel.jinja2 | 33 + .../linux/sudoer_secure_path.jinja2 | 7 + .../_templates/linux/supervisord.jinja2 | 52 + .../_templates/linux/system_packages.jinja2 | 15 + .../_templates/linux/tag_ebs_volumes.jinja2 | 34 + .../linux/tag_network_interface.jinja2 | 31 + .../windows/join_activedirectory.jinja2 | 140 + .../windows/mount_shared_storage.jinja2 | 46 + .../bastion-host/setup.sh.jinja2 | 116 + .../cluster-manager/install_app.sh.jinja2 | 69 + .../cluster-manager/setup.sh.jinja2 | 128 + .../idea-bootstrap/common/bootstrap_common.sh | 280 + .../compute_node_ami_builder.sh.jinja2 | 69 + ...ute_node_ami_builder_post_reboot.sh.jinja2 | 74 + .../compute-node-ami-builder/setup.sh.jinja2 | 57 + .../configure_hyperthreading.jinja2 | 12 + .../configure_openpbs_compute_node.jinja2 | 26 + .../compute-node/_templates/efa.jinja2 | 28 + .../_templates/scheduler_start.jinja2 | 7 + .../_templates/scheduler_stop.jinja2 | 7 + .../_templates/scratch_storage.jinja2 | 147 + .../compute-node/compute_node.sh.jinja2 | 93 + .../compute_node_post_reboot.sh.jinja2 | 31 + .../compute-node/setup.sh.jinja2 | 62 + .../dcv-broker/install_app.sh.jinja2 | 184 + .../idea-bootstrap/dcv-broker/setup.sh.jinja2 | 100 + .../install_app.sh.jinja2 | 199 + .../dcv-connection-gateway/setup.sh.jinja2 | 100 + .../_templates/install_openldap.jinja2 | 242 + .../openldap-server/setup.sh.jinja2 | 108 + .../configure_openpbs_server.jinja2 | 146 + .../scheduler/install_app.sh.jinja2 | 99 + .../scheduler/scheduler_post_reboot.sh.jinja2 | 34 + .../idea-bootstrap/scheduler/setup.sh.jinja2 | 129 + .../install_app.sh.jinja2 | 62 + .../setup.sh.jinja2 | 118 + .../configure_dcv_host.sh.jinja2 | 333 + .../configure_dcv_host_post_reboot.sh.jinja2 | 51 + .../setup.sh.jinja2 | 134 + .../ConfigureDCVHost.ps1.jinja2 | 150 + .../SetUp.ps1.jinja2 | 174 + source/idea/idea-cli/src/ideacli/__init__.py | 90 + source/idea/idea-cli/src/ideacli/idea.py | 16 + .../idea-cli/src/ideacli_meta/__init__.py | 4 + source/idea/idea-cli/src/setup.py | 17 + .../resources/api/api_doc.yml | 142 + .../resources/defaults/email_templates.yml | 186 + .../src/ideaclustermanager/__init__.py | 19 + .../src/ideaclustermanager/app/__init__.py | 10 + .../app/accounts/__init__.py | 10 + .../app/accounts/account_tasks.py | 219 + .../app/accounts/accounts_service.py | 1287 + .../app/accounts/ad_automation_agent.py | 259 + .../app/accounts/auth_constants.py | 14 + .../app/accounts/auth_utils.py | 65 + .../app/accounts/cognito_user_pool.py | 515 + .../app/accounts/db/__init__.py | 10 + .../app/accounts/db/ad_automation_dao.py | 90 + .../app/accounts/db/group_dao.py | 209 + .../app/accounts/db/group_members_dao.py | 186 + .../app/accounts/db/sequence_config_dao.py | 96 + .../accounts/db/single_sign_on_state_dao.py | 138 + .../app/accounts/db/user_dao.py | 229 + .../app/accounts/helpers/__init__.py | 10 + .../helpers/preset_computer_helper.py | 443 + .../app/accounts/ldapclient/__init__.py | 14 + .../ldapclient/abstract_ldap_client.py | 789 + .../ldapclient/active_directory_client.py | 388 + .../ldapclient/ldap_client_factory.py | 72 + .../app/accounts/ldapclient/ldap_utils.py | 79 + .../accounts/ldapclient/openldap_client.py | 159 + .../app/accounts/user_home_directory.py | 202 + .../ideaclustermanager/app/api/__init__.py | 10 + .../app/api/accounts_api.py | 344 + .../app/api/analytics_api.py | 61 + .../ideaclustermanager/app/api/api_invoker.py | 168 + .../ideaclustermanager/app/api/auth_api.py | 204 + .../app/api/cluster_settings_api.py | 124 + .../app/api/email_templates_api.py | 95 + .../app/api/projects_api.py | 139 + .../src/ideaclustermanager/app/app_context.py | 44 + .../src/ideaclustermanager/app/app_main.py | 70 + .../ideaclustermanager/app/app_messages.py | 88 + .../src/ideaclustermanager/app/app_utils.py | 25 + .../app/cluster_manager_app.py | 202 + .../app/email_templates/__init__.py | 10 + .../email_templates/email_templates_dao.py | 215 + .../email_templates_service.py | 147 + .../app/notifications/__init__.py | 10 + .../notifications/notifications_service.py | 169 + .../app/projects/__init__.py | 10 + .../app/projects/db/__init__.py | 10 + .../app/projects/db/projects_dao.py | 285 + .../app/projects/db/user_projects_dao.py | 257 + .../app/projects/project_tasks.py | 89 + .../app/projects/projects_service.py | 352 + .../ideaclustermanager/app/tasks/__init__.py | 12 + .../ideaclustermanager/app/tasks/base_task.py | 24 + .../app/tasks/task_manager.py | 121 + .../src/ideaclustermanager/app/web_portal.py | 270 + .../src/ideaclustermanager/cli/__init__.py | 34 + .../src/ideaclustermanager/cli/accounts.py | 339 + .../src/ideaclustermanager/cli/cli_main.py | 42 + .../src/ideaclustermanager/cli/cli_utils.py | 205 + .../src/ideaclustermanager/cli/groups.py | 215 + .../ideaclustermanager/cli/ldap_commands.py | 182 + .../src/ideaclustermanager/cli/logs.py | 91 + .../src/ideaclustermanager/cli/module.py | 22 + .../src/ideaclustermanager_meta/__init__.py | 13 + source/idea/idea-cluster-manager/src/setup.py | 31 + .../idea-cluster-manager/tests/conftest.py | 178 + .../tests/ideaclustermanagertests/__init__.py | 10 + .../mock_ldap_client.py | 63 + .../ideaclustermanagertests/test_accounts.py | 224 + .../ideaclustermanagertests/test_projects.py | 476 + source/idea/idea-cluster-manager/webapp/.env | 4 + .../idea-cluster-manager/webapp/.gitignore | 23 + .../idea-cluster-manager/webapp/README.md | 46 + .../idea-cluster-manager/webapp/package.json | 104 + .../webapp/public/android-chrome-192x192.png | Bin 0 -> 14357 bytes .../webapp/public/android-chrome-512x512.png | Bin 0 -> 49084 bytes .../webapp/public/apple-touch-icon.png | Bin 0 -> 13118 bytes .../webapp/public/browserconfig.xml | 9 + .../webapp/public/favicon-16x16.png | Bin 0 -> 1256 bytes .../webapp/public/favicon-32x32.png | Bin 0 -> 1896 bytes .../webapp/public/favicon.ico | Bin 0 -> 15086 bytes .../webapp/public/index.html | 129 + .../webapp/public/logo.png | Bin 0 -> 179683 bytes .../webapp/public/manifest.json | 20 + .../webapp/public/mstile-150x150.png | Bin 0 -> 8101 bytes .../webapp/public/robots.txt | 3 + .../webapp/public/safari-pinned-tab.svg | 80 + .../idea-cluster-manager/webapp/src/App.scss | 194 + .../webapp/src/App.test.tsx | 22 + .../idea-cluster-manager/webapp/src/App.tsx | 917 + .../webapp/src/client/accounts-client.ts | 220 + .../webapp/src/client/analytics-client.ts | 34 + .../webapp/src/client/auth-client.ts | 184 + .../webapp/src/client/base-client.ts | 57 + .../webapp/src/client/clients.ts | 236 + .../src/client/cluster-settings-client.ts | 69 + .../webapp/src/client/data-model.ts | 1981 ++ .../src/client/email-templates-client.ts | 70 + .../webapp/src/client/file-browser-client.ts | 88 + .../webapp/src/client/idea-api-invoker.ts | 227 + .../webapp/src/client/index.ts | 40 + .../webapp/src/client/projects-client.ts | 97 + .../src/client/scheduler-admin-client.ts | 258 + .../webapp/src/client/scheduler-client.ts | 87 + .../client/virtual-desktop-admin-client.ts | 246 + .../src/client/virtual-desktop-client.ts | 183 + .../src/client/virtual-desktop-dcv-client.ts | 42 + .../client/virtual-desktop-utils-client.ts | 97 + .../webapp/src/common/app-context.ts | 252 + .../webapp/src/common/app-logger.ts | 147 + .../src/common/authentication-context.ts | 513 + .../webapp/src/common/config-utils.ts | 121 + .../webapp/src/common/constants.ts | 72 + .../webapp/src/common/error-codes.ts | 21 + .../webapp/src/common/exceptions.ts | 37 + .../webapp/src/common/index.ts | 18 + .../webapp/src/common/shared-storage-utils.ts | 240 + .../webapp/src/common/token-utils.ts | 137 + .../webapp/src/common/utils.ts | 919 + .../src/components/app-layout/app-layout.tsx | 150 + .../webapp/src/components/app-layout/index.ts | 17 + .../components/charts/pie-or-donut-chart.tsx | 112 + .../webapp/src/components/common/index.tsx | 44 + .../components/form-builder/form-builder.tsx | 880 + .../src/components/form-builder/index.ts | 17 + .../src/components/form-field/form-field.tsx | 1715 ++ .../webapp/src/components/form-field/index.ts | 29 + .../form-review-field/form-review-field.tsx | 169 + .../src/components/form-review-field/index.ts | 19 + .../webapp/src/components/form/form.tsx | 380 + .../webapp/src/components/form/index.ts | 19 + .../webapp/src/components/key-value/index.ts | 23 + .../src/components/key-value/key-value.tsx | 265 + .../webapp/src/components/list-view/index.ts | 17 + .../src/components/list-view/list-view.tsx | 607 + .../webapp/src/components/modals/confirm.tsx | 99 + .../webapp/src/components/modals/index.ts | 18 + .../webapp/src/components/modals/view.tsx | 93 + .../webapp/src/components/navbar/index.ts | 21 + .../webapp/src/components/navbar/navbar.tsx | 317 + .../password-strength-check/index.ts | 22 + .../password-strength-check.tsx | 81 + .../src/components/side-navigation/index.ts | 19 + .../side-navigation/side-navigation.scss | 27 + .../side-navigation/side-navigation.tsx | 80 + .../src/components/split-panel/index.ts | 17 + .../components/split-panel/split-panel.tsx | 95 + .../webapp/src/components/table/index.ts | 17 + .../webapp/src/components/table/table.tsx | 493 + .../webapp/src/components/tabs/index.ts | 17 + .../webapp/src/components/tabs/tabs.tsx | 42 + .../src/components/time-range-slider/index.ts | 19 + .../time-range-slider/time-range-slider.tsx | 181 + .../webapp/src/components/wizard/index.ts | 27 + .../webapp/src/components/wizard/wizard.tsx | 696 + .../webapp/src/docs/_footer.md | 6 + .../webapp/src/docs/account-settings.md | 2 + .../webapp/src/docs/active-jobs.md | 26 + .../webapp/src/docs/cluster-settings.md | 2 + .../webapp/src/docs/cluster-status.md | 9 + .../webapp/src/docs/completed-jobs.md | 4 + .../webapp/src/docs/create-hpc-application.md | 130 + .../webapp/src/docs/create-queue-profile.md | 21 + .../webapp/src/docs/dashboard.md | 2 + .../webapp/src/docs/email-templates.md | 7 + .../webapp/src/docs/file-browser.md | 23 + .../webapp/src/docs/groups.md | 16 + .../webapp/src/docs/home.md | 5 + .../webapp/src/docs/hpc-applications.md | 19 + .../webapp/src/docs/hpc-license-create.md | 3 + .../webapp/src/docs/hpc-license-update.md | 3 + .../webapp/src/docs/hpc-licenses.md | 30 + .../webapp/src/docs/hpc-settings.md | 2 + .../my-shared-virtual-desktop-sessions.md | 16 + .../src/docs/my-virtual-desktop-sessions.md | 64 + .../webapp/src/docs/projects.md | 26 + .../webapp/src/docs/queue-profiles.md | 29 + .../webapp/src/docs/ssh-access.md | 14 + .../webapp/src/docs/submit-job.md | 2 + .../webapp/src/docs/update-hpc-application.md | 130 + .../webapp/src/docs/update-queue-profile.md | 21 + .../webapp/src/docs/users.md | 33 + .../src/docs/virtual-desktop-dashboard.md | 3 + .../webapp/src/docs/virtual-desktop-debug.md | 3 + .../virtual-desktop-permission-profiles.md | 16 + .../docs/virtual-desktop-session-detail.md | 3 + .../src/docs/virtual-desktop-sessions.md | 22 + .../src/docs/virtual-desktop-settings.md | 3 + .../virtual-desktop-software-stack-detail.md | 3 + .../docs/virtual-desktop-software-stacks.md | 10 + .../webapp/src/index.scss | 16 + .../idea-cluster-manager/webapp/src/index.tsx | 174 + .../src/navigation/navigation-utils.tsx | 53 + .../webapp/src/navigation/side-nav-items.tsx | 218 + .../src/pages/account/account-settings.tsx | 350 + .../webapp/src/pages/auth/auth-challenge.tsx | 118 + .../auth/auth-confirm-forgot-password.tsx | 125 + .../webapp/src/pages/auth/auth-context.ts | 18 + .../src/pages/auth/auth-forgot-password.tsx | 107 + .../webapp/src/pages/auth/auth-interfaces.ts | 72 + .../webapp/src/pages/auth/auth-layout.tsx | 80 + .../webapp/src/pages/auth/auth-login.tsx | 126 + .../webapp/src/pages/auth/auth-route.tsx | 46 + .../webapp/src/pages/auth/auth.scss | 65 + .../webapp/src/pages/auth/index.ts | 32 + .../pages/cluster-admin/cluster-settings.tsx | 688 + .../pages/cluster-admin/cluster-status.tsx | 552 + .../pages/cluster-admin/email-templates.tsx | 341 + .../src/pages/cluster-admin/projects.tsx | 625 + .../src/pages/dashboard/dashboard-main.tsx | 69 + .../webapp/src/pages/dashboard/index.ts | 19 + .../dashboard/job-submissions-widget.tsx | 110 + .../webapp/src/pages/home.tsx | 304 + .../webapp/src/pages/home/file-browser.tsx | 1180 + .../webapp/src/pages/home/log-tail.tsx | 253 + .../webapp/src/pages/home/ssh-access.tsx | 246 + .../webapp/src/pages/hpc/hpc-applications.tsx | 251 + .../webapp/src/pages/hpc/hpc-custom-amis.tsx | 64 + .../webapp/src/pages/hpc/hpc-licenses.tsx | 294 + .../webapp/src/pages/hpc/hpc-nodes.tsx | 219 + .../src/pages/hpc/hpc-notifications.tsx | 64 + .../src/pages/hpc/hpc-scheduler-settings.tsx | 151 + .../webapp/src/pages/hpc/hpc-utils.ts | 306 + .../webapp/src/pages/hpc/job-templates.tsx | 63 + .../webapp/src/pages/hpc/jobs.tsx | 647 + .../webapp/src/pages/hpc/queues.tsx | 444 + .../src/pages/hpc/sample-job-params.json | 267 + .../pages/hpc/sample-job-script-jinja2.txt | 36 + .../pages/hpc/sample-job-script-simple.txt | 33 + .../webapp/src/pages/hpc/submit-job.tsx | 1396 + .../src/pages/hpc/update-hpc-application.tsx | 775 + .../src/pages/hpc/update-hpc-license.tsx | 426 + .../src/pages/hpc/update-queue-profile.tsx | 984 + .../src/pages/user-management/groups.tsx | 473 + .../src/pages/user-management/users.tsx | 868 + .../virtual-desktop-az-distribution.tsx | 137 + .../charts/virtual-desktop-base-chart.tsx | 23 + .../charts/virtual-desktop-baseos-chart.tsx | 137 + .../virtual-desktop-instance-types-chart.tsx | 137 + .../charts/virtual-desktop-project-chart.tsx | 136 + .../virtual-desktop-software-stack-chart.tsx | 134 + .../charts/virtual-desktop-state-chart.tsx | 136 + .../components/dcv-client-help-modal.tsx | 215 + .../virtual-desktop-schedule-modal.tsx | 331 + .../virtual-desktop-session-card.tsx | 448 + ...rtual-desktop-session-status-indicator.tsx | 57 + .../virtual-desktop-create-session-form.tsx | 673 + ...irtual-desktop-permission-profile-form.tsx | 265 + ...rtual-desktop-software-stack-edit-form.tsx | 184 + ...esktop-update-session-permissions-form.tsx | 647 + .../my-shared-virtual-desktop-sessions.tsx | 314 + .../my-virtual-desktop-sessions.tsx | 1149 + .../virtual-desktop-dashboard.tsx | 146 + .../virtual-desktop-debug.tsx | 186 + ...tual-desktop-permission-profile-detail.tsx | 244 + .../virtual-desktop-permission-profiles.tsx | 336 + .../virtual-desktop-session-detail.tsx | 310 + .../virtual-desktop-sessions.tsx | 848 + .../virtual-desktop-settings.tsx | 407 + .../virtual-desktop-software-stack-detail.tsx | 219 + .../virtual-desktop-software-stacks.tsx | 535 + .../webapp/src/react-app-env.d.ts | 1 + .../webapp/src/service-worker-registration.ts | 141 + .../webapp/src/service-worker.ts | 139 + .../webapp/src/service/auth-service.ts | 400 + .../src/service/cluster-settings-service.ts | 268 + .../webapp/src/service/index.ts | 22 + .../src/service/job-templates-service.ts | 167 + .../src/service/local-storage-service.ts | 44 + .../webapp/src/setupTests.ts | 5 + .../webapp/src/styles/home.scss | 63 + .../idea-cluster-manager/webapp/tsconfig.json | 26 + .../idea-cluster-manager/webapp/yarn.lock | 12628 ++++++++ .../src/ideadatamodel/__init__.py | 27 + .../src/ideadatamodel/analytics/__init__.py | 12 + .../ideadatamodel/analytics/analytics_api.py | 27 + .../analytics/analytics_model.py | 27 + .../src/ideadatamodel/api/__init__.py | 13 + .../src/ideadatamodel/api/api_model.py | 136 + .../src/ideadatamodel/api/logging.py | 18 + .../src/ideadatamodel/app/__init__.py | 13 + .../src/ideadatamodel/app/app_api.py | 28 + .../src/ideadatamodel/app/app_model.py | 24 + .../src/ideadatamodel/auth/__init__.py | 13 + .../src/ideadatamodel/auth/auth_api.py | 638 + .../src/ideadatamodel/auth/auth_model.py | 67 + .../src/ideadatamodel/aws/__init__.py | 19 + .../ideadatamodel/aws/autoscaling_group.py | 76 + .../ideadatamodel/aws/cloudformation_stack.py | 85 + .../aws/cloudformation_stack_resources.py | 47 + .../src/ideadatamodel/aws/ec2_instance.py | 652 + .../ideadatamodel/aws/ec2_instance_type.py | 261 + .../aws/ec2_spot_fleet_request_config.py | 69 + .../src/ideadatamodel/aws/model.py | 246 + .../idea-data-model/src/ideadatamodel/base.py | 32 + .../cluster_resources/__init__.py | 12 + .../cluster_resources_model.py | 398 + .../cluster_settings/__init__.py | 13 + .../cluster_settings/cluster_settings_api.py | 96 + .../cluster_settings_model.py | 13 + .../src/ideadatamodel/common/__init__.py | 12 + .../src/ideadatamodel/common/common_model.py | 616 + .../src/ideadatamodel/constants.py | 374 + .../ideadatamodel/email_templates/__init__.py | 13 + .../email_templates/email_templates_api.py | 112 + .../email_templates/email_templates_model.py | 27 + .../src/ideadatamodel/errorcodes.py | 142 + .../src/ideadatamodel/exceptions/__init__.py | 13 + .../exceptions/exception_utils.py | 135 + .../ideadatamodel/exceptions/exceptions.py | 58 + .../src/ideadatamodel/filesystem/__init__.py | 13 + .../filesystem/filesystem_api.py | 164 + .../filesystem/filesystem_model.py | 44 + .../src/ideadatamodel/locale/__init__.py | 92 + .../src/ideadatamodel/model_utils.py | 456 + .../ideadatamodel/notifications/__init__.py | 13 + .../notifications/notifications_api.py | 28 + .../notifications/notifications_model.py | 24 + .../src/ideadatamodel/projects/__init__.py | 13 + .../ideadatamodel/projects/projects_api.py | 163 + .../ideadatamodel/projects/projects_model.py | 44 + .../src/ideadatamodel/scheduler/__init__.py | 13 + .../ideadatamodel/scheduler/scheduler_api.py | 614 + .../scheduler/scheduler_model.py | 1798 ++ .../src/ideadatamodel/user_input/__init__.py | 13 + .../user_input/user_input_api.py | 87 + .../user_input/user_input_model.py | 403 + .../ideadatamodel/virtual_desktop/__init__.py | 13 + .../virtual_desktop/virtual_desktop_api.py | 791 + .../virtual_desktop/virtual_desktop_model.py | 263 + .../src/ideadatamodel_meta/__init__.py | 13 + source/idea/idea-data-model/src/setup.py | 15 + .../images/favicon.ico | Bin 0 -> 15086 bytes .../idea-dcv-connection-gateway/index.html | 16 + .../idea-scheduler/resources/api/api_doc.yml | 15 + .../openpbs/hooks/openpbs_hook_handler.py | 1583 + .../resources/opensearch/template_jobs.json | 668 + .../resources/opensearch/template_nodes.json | 267 + .../resources/scripts/license_check.py | 57 + .../resources/scripts/send_logs_s3.sh | 39 + .../src/ideascheduler/__init__.py | 18 + .../src/ideascheduler/app/__init__.py | 10 + .../src/ideascheduler/app/api/__init__.py | 14 + .../src/ideascheduler/app/api/opepbs_api.py | 202 + .../app/api/scheduler_admin_api.py | 501 + .../ideascheduler/app/api/scheduler_api.py | 343 + .../app/api/scheduler_api_invoker.py | 46 + .../src/ideascheduler/app/app_context.py | 117 + .../src/ideascheduler/app/app_main.py | 70 + .../src/ideascheduler/app/app_protocols.py | 503 + .../app/applications/__init__.py | 10 + .../app/applications/hpc_applications_dao.py | 241 + .../applications/hpc_applications_service.py | 167 + .../src/ideascheduler/app/aws/__init__.py | 16 + .../app/aws/aws_bugets_helper.py | 115 + .../app/aws/ec2_service_quota_helper.py | 296 + .../ideascheduler/app/aws/instance_cache.py | 291 + .../ideascheduler/app/aws/instance_monitor.py | 86 + .../ideascheduler/app/aws/pricing_helper.py | 266 + .../ideascheduler/app/documents/__init__.py | 12 + .../app/documents/document_store.py | 402 + .../ideascheduler/app/licenses/__init__.py | 10 + .../app/licenses/license_resources_dao.py | 207 + .../app/licenses/license_service.py | 294 + .../src/ideascheduler/app/metrics/__init__.py | 290 + .../app/notifications/__init__.py | 10 + .../app/notifications/job_notifications.py | 67 + .../app/provisioning/__init__.py | 24 + .../app/provisioning/job_monitor/__init__.py | 10 + .../job_monitor/finished_job_processor.py | 305 + .../app/provisioning/job_monitor/job_cache.py | 428 + .../provisioning/job_monitor/job_monitor.py | 495 + .../job_monitor/job_submission_tracker.py | 48 + .../provisioning/job_provisioner/__init__.py | 10 + .../job_provisioner/batch_capacity_helper.py | 159 + .../cloudformation_stack_builder.py | 590 + .../job_provisioner/job_provisioner.py | 601 + .../job_provisioner/job_provisioning_util.py | 565 + .../job_provisioning_queue/__init__.py | 10 + .../hpc_queue_profiles_dao.py | 417 + .../hpc_queue_profiles_service.py | 473 + .../job_provisioning_queue.py | 483 + .../app/provisioning/node_monitor/__init__.py | 10 + .../node_monitor/node_house_keeper.py | 927 + .../provisioning/node_monitor/node_monitor.py | 251 + .../ideascheduler/app/scheduler/__init__.py | 13 + .../app/scheduler/job_param_builder.py | 2967 ++ .../app/scheduler/lsf/__init__.py | 0 .../app/scheduler/openpbs/__init__.py | 16 + .../app/scheduler/openpbs/openpbs_api.py | 28 + .../openpbs/openpbs_api_invocation_context.py | 514 + .../scheduler/openpbs/openpbs_constants.py | 28 + .../scheduler/openpbs/openpbs_converter.py | 375 + .../app/scheduler/openpbs/openpbs_model.py | 624 + .../app/scheduler/openpbs/openpbs_qselect.py | 135 + .../app/scheduler/openpbs/openpbs_qstat.py | 358 + .../scheduler/openpbs/openpbs_scheduler.py | 501 + .../app/scheduler/slurm/__init__.py | 0 .../app/scheduler/soca_scheduler.py | 134 + .../src/ideascheduler/app/scheduler_app.py | 183 + .../app/scheduler_default_settings.py | 302 + .../src/ideascheduler/cli/__init__.py | 35 + .../src/ideascheduler/cli/ami_builder.py | 612 + .../src/ideascheduler/cli/cli_main.py | 62 + .../src/ideascheduler/cli/jobs.py | 602 + .../src/ideascheduler/cli/logs.py | 117 + .../src/ideascheduler/cli/module.py | 22 + .../src/ideascheduler/cli/nodes.py | 48 + .../src/ideascheduler_meta/__init__.py | 15 + source/idea/idea-scheduler/src/setup.py | 20 + source/idea/idea-scheduler/tests/conftest.py | 94 + .../test_cloudformation_stack_builder.py | 353 + .../tests/test_job_param_builder.py | 1411 + source/idea/idea-sdk/src/MANIFEST.in | 5 + source/idea/idea-sdk/src/ideasdk/__init__.py | 14 + .../src/ideasdk/analytics/__init__.py | 11 + .../ideasdk/analytics/analytics_service.py | 144 + .../idea/idea-sdk/src/ideasdk/api/__init__.py | 13 + .../src/ideasdk/api/api_invocation_context.py | 489 + .../idea/idea-sdk/src/ideasdk/api/base_api.py | 20 + .../idea/idea-sdk/src/ideasdk/app/__init__.py | 14 + .../idea/idea-sdk/src/ideasdk/app/soca_app.py | 268 + .../idea-sdk/src/ideasdk/app/soca_app_api.py | 38 + .../src/ideasdk/app/soca_app_commands.py | 68 + .../idea-sdk/src/ideasdk/artwork/__init__.py | 71 + .../idea-sdk/src/ideasdk/auth/__init__.py | 12 + .../src/ideasdk/auth/token_service.py | 495 + .../idea/idea-sdk/src/ideasdk/aws/__init__.py | 18 + .../src/ideasdk/aws/aws_client_provider.py | 351 + .../src/ideasdk/aws/aws_endpoints.json | 24840 ++++++++++++++++ .../idea-sdk/src/ideasdk/aws/aws_endpoints.py | 76 + .../idea-sdk/src/ideasdk/aws/aws_resources.py | 768 + .../idea/idea-sdk/src/ideasdk/aws/aws_util.py | 1005 + .../src/ideasdk/aws/ec2_instance_types_db.py | 68 + .../src/ideasdk/aws/iam_permission_util.py | 95 + .../src/ideasdk/aws/instance_metadata_util.py | 82 + .../src/ideasdk/aws/opensearch/__init__.py | 10 + .../aws/opensearch/aws_opensearch_client.py | 162 + .../aws/opensearch/opensearch_filters.py | 93 + .../aws/opensearch/opensearchable_db.py | 99 + .../src/ideasdk/bootstrap/__init__.py | 14 + .../bootstrap/bootstrap_package_builder.py | 135 + .../bootstrap/bootstrap_userdata_builder.py | 270 + .../src/ideasdk/bootstrap/bootstrap_utils.py | 82 + .../idea-sdk/src/ideasdk/cache/__init__.py | 12 + .../idea-sdk/src/ideasdk/cache/soca_cache.py | 193 + .../idea-sdk/src/ideasdk/client/__init__.py | 16 + .../src/ideasdk/client/accounts_client.py | 78 + .../src/ideasdk/client/evdi_client.py | 45 + .../client/notifications_async_client.py | 34 + .../src/ideasdk/client/projects_client.py | 121 + .../src/ideasdk/client/soca_client.py | 213 + .../src/ideasdk/clustering/__init__.py | 13 + .../src/ideasdk/clustering/leader_election.py | 91 + .../clustering/leader_election_constants.py | 18 + .../idea-sdk/src/ideasdk/common/__init__.py | 0 .../src/ideasdk/common/threading/__init__.py | 54 + .../idea-sdk/src/ideasdk/compat/__init__.py | 10 + .../idea-sdk/src/ideasdk/compat/windows.py | 12 + .../idea-sdk/src/ideasdk/config/__init__.py | 0 .../src/ideasdk/config/cluster_config.py | 188 + .../src/ideasdk/config/cluster_config_db.py | 533 + .../src/ideasdk/config/soca_config.py | 183 + .../idea-sdk/src/ideasdk/context/__init__.py | 15 + .../src/ideasdk/context/arn_builder.py | 258 + .../src/ideasdk/context/bootstrap_context.py | 233 + .../src/ideasdk/context/soca_cli_context.py | 360 + .../src/ideasdk/context/soca_context.py | 330 + .../ideasdk/context/validators/__init__.py | 10 + .../src/ideasdk/distributed_lock/__init__.py | 13 + .../distributed_lock/distributed_lock.py | 148 + .../idea-sdk/src/ideasdk/dynamodb/__init__.py | 10 + .../dynamodb/dynamodb_stream_subscriber.py | 27 + .../dynamodb/dynamodb_stream_subscription.py | 221 + .../src/ideasdk/filesystem/__init__.py | 10 + .../src/ideasdk/filesystem/filebrowser_api.py | 99 + .../ideasdk/filesystem/filesystem_helper.py | 451 + .../idea-sdk/src/ideasdk/logging/__init__.py | 14 + .../src/ideasdk/logging/console_logger.py | 63 + .../src/ideasdk/logging/soca_logging.py | 282 + .../idea-sdk/src/ideasdk/metrics/__init__.py | 18 + .../src/ideasdk/metrics/base_accumulator.py | 31 + .../src/ideasdk/metrics/base_metrics.py | 235 + .../ideasdk/metrics/cloudwatch/__init__.py | 13 + .../cloudwatch/cloudwatch_agent_config.py | 258 + .../metrics/cloudwatch/cloudwatch_metrics.py | 118 + .../amazon-cloudwatch-agent-linux.yml | 337 + .../amazon-cloudwatch-agent-windows.yml | 0 .../src/ideasdk/metrics/fast_write_counter.py | 56 + .../src/ideasdk/metrics/metric_timer.py | 27 + .../metrics/metrics_provider_factory.py | 61 + .../src/ideasdk/metrics/metrics_service.py | 141 + .../ideasdk/metrics/null_metrics_provider.py | 26 + .../ideasdk/metrics/prometheus/__init__.py | 10 + .../metrics/prometheus/prometheus_config.py | 169 + .../metrics/prometheus/prometheus_metrics.py | 151 + .../prometheus/templates/prometheus-linux.yml | 18 + .../templates/prometheus-windows.yml | 0 source/idea/idea-sdk/src/ideasdk/notice.py | 23 + .../src/ideasdk/protocols/__init__.py | 730 + .../idea-sdk/src/ideasdk/protocols/empty.py | 74 + .../idea-sdk/src/ideasdk/pubsub/__init__.py | 12 + .../src/ideasdk/pubsub/soca_pubsub.py | 42 + .../idea-sdk/src/ideasdk/server/__init__.py | 12 + .../idea/idea-sdk/src/ideasdk/server/cors.py | 37 + .../idea-sdk/src/ideasdk/server/options.py | 60 + .../src/ideasdk/server/sanic_config.py | 91 + .../src/ideasdk/server/soca_server.py | 992 + .../idea-sdk/src/ideasdk/service/__init__.py | 13 + .../src/ideasdk/service/soca_service.py | 40 + .../ideasdk/service/soca_service_registry.py | 44 + .../idea-sdk/src/ideasdk/shell/__init__.py | 12 + .../idea-sdk/src/ideasdk/shell/log_tail.py | 154 + .../src/ideasdk/shell/shell_invoker.py | 257 + .../src/ideasdk/thread_pool/__init__.py | 10 + .../src/ideasdk/thread_pool/idea_thread.py | 19 + .../thread_pool/idea_threadpool_service.py | 115 + .../src/ideasdk/user_input/__init__.py | 10 + .../src/ideasdk/user_input/framework.py | 1454 + .../idea-sdk/src/ideasdk/utils/__init__.py | 19 + .../src/ideasdk/utils/datetime_utils.py | 114 + .../src/ideasdk/utils/environment_utils.py | 88 + .../src/ideasdk/utils/group_name_helper.py | 73 + .../src/ideasdk/utils/jinja2_utils.py | 42 + .../src/ideasdk/utils/module_metadata.py | 64 + .../idea/idea-sdk/src/ideasdk/utils/utils.py | 721 + .../idea-sdk/src/ideasdk_meta/__init__.py | 15 + source/idea/idea-sdk/src/setup.py | 27 + source/idea/idea-sdk/tests/conftest.py | 29 + .../tests/test_api_invocation_context.py | 94 + .../idea-sdk/tests/test_file_system_helper.py | 123 + source/idea/idea-sdk/tests/test_server.py | 294 + source/idea/idea-sdk/tests/test_utils.py | 377 + .../src/ideatestutils/__init__.py | 15 + .../src/ideatestutils/aws/__init__.py | 16 + .../ideatestutils/aws/mock_instance_types.py | 60 + .../ideatestutils/aws/templates/c5.large.json | 86 + .../aws/templates/c5.xlarge.json | 86 + .../aws/templates/c5n.18xlarge.json | 105 + .../ideatestutils/aws/templates/t3.micro.json | 85 + .../src/ideatestutils/config/__init__.py | 16 + .../src/ideatestutils/config/mock_config.py | 45 + .../config/templates/default.yml | 602 + .../src/ideatestutils/dynamodb/__init__.py | 10 + .../ideatestutils/dynamodb/dynamodb_local.py | 183 + .../src/ideatestutils/idea_test_props.py | 32 + .../src/ideatestutils/projects/__init__.py | 16 + .../ideatestutils/projects/mock_projects.py | 40 + .../projects/templates/default.json | 19 + .../resources/api/api_doc.yml | 449 + .../base-permission-profile-config.yaml | 103 + .../resources/base-software-stack-config.yaml | 534 + .../resources/dcv_broker_swagger_client.yaml | 928 + .../opensearch/session_entry_template.yml | 390 + .../session_permission_entry_template.yml | 74 + .../software_stack_entry_template.yml | 81 + .../resources/permission-config.yaml | 63 + .../ideavirtualdesktopcontroller/__init__.py | 18 + .../app/__init__.py | 10 + .../app/api/__init__.py | 17 + .../app/api/virtual_desktop_admin_api.py | 667 + .../app/api/virtual_desktop_api.py | 767 + .../app/api/virtual_desktop_api_invoker.py | 43 + .../app/api/virtual_desktop_dcv_api.py | 53 + .../app/api/virtual_desktop_user_api.py | 508 + .../app/api/virtual_desktop_utils_api.py | 175 + .../app/app_context.py | 64 + .../app/app_main.py | 73 + .../app/app_protocols.py | 80 + .../app/clients/__init__.py | 11 + .../app/clients/dcv_broker_client/__init__.py | 11 + .../dcv_broker_client/dcv_broker_client.py | 338 + .../dcv_broker_client_utils.py | 163 + .../clients/dcvssmswaggerclient/__init__.py | 83 + .../dcvssmswaggerclient/api/__init__.py | 9 + .../api/get_session_connection_data_api.py | 133 + .../dcvssmswaggerclient/api/servers_api.py | 325 + .../api/session_permissions_api.py | 131 + .../dcvssmswaggerclient/api/sessions_api.py | 424 + .../clients/dcvssmswaggerclient/api_client.py | 631 + .../dcvssmswaggerclient/configuration.py | 244 + .../dcvssmswaggerclient/models/__init__.py | 74 + .../clients/dcvssmswaggerclient/models/aws.py | 197 + .../models/close_server_request_data.py | 141 + .../close_server_successful_response.py | 113 + .../close_server_unsuccessful_response.py | 167 + .../models/close_servers_response.py | 169 + .../dcvssmswaggerclient/models/cpu_info.py | 225 + .../models/cpu_load_average.py | 169 + .../models/create_session_request_data.py | 477 + .../models/create_sessions_response.py | 169 + .../models/delete_session_request_data.py | 169 + .../delete_session_successful_response.py | 141 + .../delete_session_unsuccessful_response.py | 141 + .../models/delete_sessions_response.py | 169 + .../models/describe_servers_request_data.py | 169 + .../models/describe_servers_response.py | 169 + .../models/describe_sessions_request_data.py | 197 + .../models/describe_sessions_response.py | 169 + .../dcvssmswaggerclient/models/endpoint.py | 175 + .../get_session_connection_data_response.py | 139 + .../get_session_screenshot_request_data.py | 113 + ..._session_screenshot_successful_response.py | 111 + ...ession_screenshot_unsuccessful_response.py | 139 + .../get_session_screenshots_response.py | 169 + .../clients/dcvssmswaggerclient/models/gpu.py | 141 + .../dcvssmswaggerclient/models/host.py | 297 + .../models/key_value_pair.py | 141 + .../models/logged_in_user.py | 113 + .../dcvssmswaggerclient/models/memory.py | 141 + .../models/open_server_request_data.py | 113 + .../models/open_server_successful_response.py | 113 + .../open_server_unsuccessful_response.py | 167 + .../models/open_servers_response.py | 169 + .../clients/dcvssmswaggerclient/models/os.py | 225 + .../dcvssmswaggerclient/models/server.py | 515 + .../dcvssmswaggerclient/models/session.py | 419 + .../models/session_screenshot.py | 141 + .../models/session_screenshot_image.py | 197 + .../dcvssmswaggerclient/models/swap.py | 141 + ...nsuccessful_create_session_request_data.py | 139 + ...update_session_permissions_request_data.py | 169 + .../update_session_permissions_response.py | 169 + ...session_permissions_successful_response.py | 113 + ...ssion_permissions_unsuccessful_response.py | 141 + .../app/clients/dcvssmswaggerclient/rest.py | 317 + .../app/clients/events_client/__init__.py | 10 + .../clients/events_client/events_client.py | 72 + .../app/events/__init__.py | 10 + .../app/events/events_utils.py | 256 + .../app/events/handlers/__init__.py | 10 + .../app/events/handlers/base_event_handler.py | 152 + .../db_entry_event_handlers/__init__.py | 10 + .../base_db_event_handler.py | 81 + .../db_entry_created_event_handler.py | 80 + .../db_entry_deleted_event_handler.py | 79 + .../db_entry_updated_event_handler.py | 152 + ...erdata_execution_complete_event_handler.py | 140 + .../dcv_host_event_handlers/__init__.py | 10 + .../dcv_host_ready_event_handler.py | 101 + .../dcv_host_reboot_complete_event_handler.py | 97 + .../ec2_state_change_event_handler.py | 97 + .../__init__.py | 10 + ...ssion_permissions_enforce_event_handler.py | 45 + ...ession_permissions_update_event_handler.py | 53 + ...ion_software_stack_update_event_handler.py | 45 + .../__init__.py | 10 + ..._session_scheduled_resume_event_handler.py | 48 + ...ea_session_scheduled_stop_event_handler.py | 59 + .../idea_session_terminate_event_handler.py | 43 + .../handlers/scheduled_event_handler.py | 63 + .../__init__.py | 10 + ..._windows_command_progress_event_handler.py | 49 + ..._windows_command_progress_event_handler.py | 64 + ..._session_command_progress_event_handler.py | 53 + ...lization_command_progress_event_handler.py | 56 + .../__init__.py | 10 + .../user_created_event_handler.py | 33 + .../user_disabled_event_handler.py | 43 + .../__init__.py | 10 + ...date_dcv_session_creation_event_handler.py | 88 + ...date_dcv_session_deletion_event_handler.py | 79 + .../validate_software_stack_event_handler.py | 68 + .../app/events/service/__init__.py | 10 + .../controller_queue_monitor_service.py | 187 + .../service/event_queue_monitoring_service.py | 49 + .../events/service/events_handler_thread.py | 176 + .../app/permission_profiles/__init__.py | 10 + .../app/permission_profiles/constants.py | 5 + .../virtual_desktop_permission_profile_db.py | 259 + .../app/schedules/__init__.py | 10 + .../app/schedules/constants.py | 8 + .../schedules/virtual_desktop_schedule_db.py | 215 + .../virtual_desktop_schedule_utils.py | 267 + .../app/servers/__init__.py | 10 + .../app/servers/constants.py | 8 + .../app/servers/virtual_desktop_server_db.py | 165 + .../servers/virtual_desktop_server_utils.py | 136 + .../app/session_permissions/__init__.py | 10 + .../app/session_permissions/constants.py | 25 + .../virtual_desktop_session_permission_db.py | 295 + ...irtual_desktop_session_permission_utils.py | 181 + .../app/sessions/__init__.py | 10 + .../app/sessions/constants.py | 44 + .../virtual_desktop_session_counters_db.py | 166 + .../sessions/virtual_desktop_session_db.py | 441 + .../sessions/virtual_desktop_session_utils.py | 294 + .../app/software_stacks/__init__.py | 10 + .../app/software_stacks/constants.py | 26 + .../virtual_desktop_software_stack_db.py | 463 + .../virtual_desktop_software_stack_utils.py | 70 + .../app/ssm_commands/__init__.py | 10 + .../app/ssm_commands/constants.py | 3 + .../virtual_desktop_ssm_commands_db.py | 129 + .../virtual_desktop_ssm_commands_utils.py | 172 + .../app/virtual_desktop_controller_app.py | 216 + .../app/virtual_desktop_controller_utils.py | 424 + .../app/virtual_desktop_notifiable_db.py | 66 + .../cli/__init__.py | 35 + .../cli/cli_main.py | 43 + .../ideavirtualdesktopcontroller/cli/logs.py | 91 + .../cli/module.py | 43 + .../cli/sessions.py | 135 + .../cli/software_stacks.py | 37 + .../__init__.py | 13 + .../src/setup.py | 20 + .../tests/conftest.py | 10 + .../tests/test_example.py | 16 + tasks/__init__.py | 64 + tasks/admin.py | 228 + tasks/apispec.py | 142 + tasks/build.py | 118 + tasks/clean.py | 95 + tasks/cli.py | 70 + tasks/devtool.py | 369 + tasks/docker.py | 66 + tasks/idea.py | 455 + tasks/package.py | 150 + tasks/release.py | 223 + tasks/requirements.py | 88 + tasks/tests.py | 199 + tasks/tools/__init__.py | 11 + tasks/tools/build_tool.py | 400 + tasks/tools/clean_tool.py | 74 + tasks/tools/open_api_tool.py | 388 + tasks/tools/package_tool.py | 134 + tasks/tools/typings_generator.py | 364 + tasks/web_portal.py | 75 + 1008 files changed, 204011 insertions(+) create mode 100644 .editorconfig create mode 100644 .gitattributes create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md create mode 100644 .github/ISSUE_TEMPLATE/feature_request.md create mode 100644 .github/PULL_REQUEST_TEMPLATE.md create mode 100644 .gitignore create mode 100644 CHANGELOG.md create mode 100644 CODE_OF_CONDUCT.md create mode 100644 CONTRIBUTING.md create mode 100644 IDEA_VERSION.txt create mode 100644 LICENSE.txt create mode 100644 NOTICE.txt create mode 100644 README.md create mode 100644 THIRD_PARTY_LICENSES.txt create mode 100644 deployment/ecr/idea-administrator/Dockerfile create mode 100644 deployment/integrated-digital-engineering-on-aws.template create mode 100755 idea-admin-windows.ps1 create mode 100755 idea-admin.sh create mode 100644 requirements/dev.in create mode 100644 requirements/dev.txt create mode 100644 requirements/doc.in create mode 100644 requirements/doc.txt create mode 100644 requirements/idea-administrator.in create mode 100644 requirements/idea-administrator.txt create mode 100644 requirements/idea-cluster-manager.in create mode 100644 requirements/idea-cluster-manager.txt create mode 100644 requirements/idea-dev-lambda.in create mode 100644 requirements/idea-dev-lambda.txt create mode 100644 requirements/idea-scheduler.in create mode 100644 requirements/idea-scheduler.txt create mode 100644 requirements/idea-sdk.in create mode 100644 requirements/idea-virtual-desktop-controller.in create mode 100644 requirements/idea-virtual-desktop-controller.txt create mode 100644 requirements/tests.in create mode 100644 requirements/tests.txt create mode 100644 software_versions.yml create mode 100644 source/idea/idea-administrator/install/install.sh create mode 100644 source/idea/idea-administrator/resources/cdk/cdk.json create mode 100644 source/idea/idea-administrator/resources/cdk/cdk_toolkit_stack.yml create mode 100644 source/idea/idea-administrator/resources/config/region_ami_config.yml create mode 100644 source/idea/idea-administrator/resources/config/region_elb_account_id.yml create mode 100644 source/idea/idea-administrator/resources/config/region_timezone_config.yml create mode 100644 source/idea/idea-administrator/resources/config/templates/analytics/settings.yml create mode 100644 source/idea/idea-administrator/resources/config/templates/bastion-host/settings.yml create mode 100644 source/idea/idea-administrator/resources/config/templates/cluster-manager/settings.yml create mode 100644 source/idea/idea-administrator/resources/config/templates/cluster/logging.yml create mode 100644 source/idea/idea-administrator/resources/config/templates/cluster/settings.yml create mode 100644 source/idea/idea-administrator/resources/config/templates/directoryservice/_templates/activedirectory.yml create mode 100644 source/idea/idea-administrator/resources/config/templates/directoryservice/_templates/aws_managed_activedirectory.yml create mode 100644 source/idea/idea-administrator/resources/config/templates/directoryservice/_templates/openldap.yml create mode 100644 source/idea/idea-administrator/resources/config/templates/directoryservice/settings.yml create mode 100644 source/idea/idea-administrator/resources/config/templates/global-settings/settings.yml create mode 100644 source/idea/idea-administrator/resources/config/templates/idea.yml create mode 100644 source/idea/idea-administrator/resources/config/templates/identity-provider/settings.yml create mode 100644 source/idea/idea-administrator/resources/config/templates/metrics/settings.yml create mode 100644 source/idea/idea-administrator/resources/config/templates/scheduler/settings.yml create mode 100644 source/idea/idea-administrator/resources/config/templates/shared-storage/_templates/efs.yml create mode 100644 source/idea/idea-administrator/resources/config/templates/shared-storage/_templates/fsx_lustre.yml create mode 100644 source/idea/idea-administrator/resources/config/templates/shared-storage/settings.yml create mode 100644 source/idea/idea-administrator/resources/config/templates/virtual-desktop-controller/settings.yml create mode 100644 source/idea/idea-administrator/resources/config/values.yml create mode 100644 source/idea/idea-administrator/resources/input_params/install_params.yml create mode 100644 source/idea/idea-administrator/resources/input_params/shared_storage_params.yml create mode 100644 source/idea/idea-administrator/resources/installer_policies/idea-admin-cdk-toolkit-policy.yml create mode 100644 source/idea/idea-administrator/resources/installer_policies/idea-admin-create-policy.yml create mode 100644 source/idea/idea-administrator/resources/installer_policies/idea-admin-delete-policy.yml create mode 100644 source/idea/idea-administrator/resources/integration_tests/job_test_cases.yml create mode 100644 source/idea/idea-administrator/resources/integration_tests/session_test_cases.yml create mode 100644 source/idea/idea-administrator/resources/lambda_functions/__init__.py create mode 100644 source/idea/idea-administrator/resources/lambda_functions/idea_analytics_sink/__init__.py create mode 100644 source/idea/idea-administrator/resources/lambda_functions/idea_analytics_sink/handler.py create mode 100644 source/idea/idea-administrator/resources/lambda_functions/idea_analytics_sink/requirements.txt create mode 100644 source/idea/idea-administrator/resources/lambda_functions/idea_controller_scheduled_event_transformer/__init__.py create mode 100644 source/idea/idea-administrator/resources/lambda_functions/idea_controller_scheduled_event_transformer/handler.py create mode 100644 source/idea/idea-administrator/resources/lambda_functions/idea_custom_resource_cluster_endpoints/__init__.py create mode 100644 source/idea/idea-administrator/resources/lambda_functions/idea_custom_resource_cluster_endpoints/handler.py create mode 100644 source/idea/idea-administrator/resources/lambda_functions/idea_custom_resource_create_tags/__init__.py create mode 100644 source/idea/idea-administrator/resources/lambda_functions/idea_custom_resource_create_tags/handler.py create mode 100644 source/idea/idea-administrator/resources/lambda_functions/idea_custom_resource_get_ad_security_group/__init__.py create mode 100644 source/idea/idea-administrator/resources/lambda_functions/idea_custom_resource_get_ad_security_group/handler.py create mode 100644 source/idea/idea-administrator/resources/lambda_functions/idea_custom_resource_get_user_pool_client_secret/__init__.py create mode 100644 source/idea/idea-administrator/resources/lambda_functions/idea_custom_resource_get_user_pool_client_secret/handler.py create mode 100644 source/idea/idea-administrator/resources/lambda_functions/idea_custom_resource_opensearch_private_ips/__init__.py create mode 100644 source/idea/idea-administrator/resources/lambda_functions/idea_custom_resource_opensearch_private_ips/handler.py create mode 100644 source/idea/idea-administrator/resources/lambda_functions/idea_custom_resource_self_signed_certificate/__init__.py create mode 100644 source/idea/idea-administrator/resources/lambda_functions/idea_custom_resource_self_signed_certificate/handler.py create mode 100644 source/idea/idea-administrator/resources/lambda_functions/idea_custom_resource_self_signed_certificate/requirements.txt create mode 100644 source/idea/idea-administrator/resources/lambda_functions/idea_custom_resource_update_cluster_prefix_list/__init__.py create mode 100644 source/idea/idea-administrator/resources/lambda_functions/idea_custom_resource_update_cluster_prefix_list/handler.py create mode 100644 source/idea/idea-administrator/resources/lambda_functions/idea_custom_resource_update_cluster_settings/__init__.py create mode 100644 source/idea/idea-administrator/resources/lambda_functions/idea_custom_resource_update_cluster_settings/handler.py create mode 100644 source/idea/idea-administrator/resources/lambda_functions/idea_ec2_state_event_transformation_lambda/__init__.py create mode 100644 source/idea/idea-administrator/resources/lambda_functions/idea_ec2_state_event_transformation_lambda/handler.py create mode 100644 source/idea/idea-administrator/resources/lambda_functions/idea_efs_throughput/__init__.py create mode 100644 source/idea/idea-administrator/resources/lambda_functions/idea_efs_throughput/handler.py create mode 100644 source/idea/idea-administrator/resources/lambda_functions/idea_lambda_commons/__init__.py create mode 100644 source/idea/idea-administrator/resources/lambda_functions/idea_lambda_commons/cfn_response.py create mode 100644 source/idea/idea-administrator/resources/lambda_functions/idea_lambda_commons/cfn_response_status.py create mode 100644 source/idea/idea-administrator/resources/lambda_functions/idea_lambda_commons/http_client.py create mode 100644 source/idea/idea-administrator/resources/lambda_functions/idea_solution_metrics/__init__.py create mode 100644 source/idea/idea-administrator/resources/lambda_functions/idea_solution_metrics/handler.py create mode 100644 source/idea/idea-administrator/resources/policies/_templates/activedirectory.yml create mode 100644 source/idea/idea-administrator/resources/policies/_templates/aws-managed-ad.yml create mode 100644 source/idea/idea-administrator/resources/policies/_templates/custom-kms-key.yml create mode 100644 source/idea/idea-administrator/resources/policies/_templates/lambda-basic-execution.yml create mode 100644 source/idea/idea-administrator/resources/policies/_templates/openldap.yml create mode 100644 source/idea/idea-administrator/resources/policies/amazon-prometheus-remote-write-access.yml create mode 100644 source/idea/idea-administrator/resources/policies/amazon-ssm-managed-instance-core.yml create mode 100644 source/idea/idea-administrator/resources/policies/analytics-sink-lambda.yml create mode 100644 source/idea/idea-administrator/resources/policies/backup-create.yml create mode 100644 source/idea/idea-administrator/resources/policies/backup-restore.yml create mode 100644 source/idea/idea-administrator/resources/policies/backup-s3-create.yml create mode 100644 source/idea/idea-administrator/resources/policies/backup-s3-restore.yml create mode 100644 source/idea/idea-administrator/resources/policies/bastion-host.yml create mode 100644 source/idea/idea-administrator/resources/policies/cloud-watch-agent-server-policy.yml create mode 100644 source/idea/idea-administrator/resources/policies/cluster-manager.yml create mode 100644 source/idea/idea-administrator/resources/policies/compute-node.yml create mode 100644 source/idea/idea-administrator/resources/policies/controller-scheduled-event-transformer-lambda.yml create mode 100644 source/idea/idea-administrator/resources/policies/controller-ssm-command-pass-role.yml create mode 100644 source/idea/idea-administrator/resources/policies/custom-resource-cluster-endpoints.yml create mode 100644 source/idea/idea-administrator/resources/policies/custom-resource-ec2-create-tags.yml create mode 100644 source/idea/idea-administrator/resources/policies/custom-resource-get-ad-security-group.yml create mode 100644 source/idea/idea-administrator/resources/policies/custom-resource-get-user-pool-client-secret.yml create mode 100644 source/idea/idea-administrator/resources/policies/custom-resource-opensearch-private-ips.yml create mode 100644 source/idea/idea-administrator/resources/policies/custom-resource-self-signed-certificate.yml create mode 100644 source/idea/idea-administrator/resources/policies/custom-resource-update-cluster-prefix-list.yml create mode 100644 source/idea/idea-administrator/resources/policies/custom-resource-update-cluster-settings.yml create mode 100644 source/idea/idea-administrator/resources/policies/ec2state-event-transformer.yml create mode 100644 source/idea/idea-administrator/resources/policies/efs-throughput-lambda.yml create mode 100644 source/idea/idea-administrator/resources/policies/log-retention.yml create mode 100644 source/idea/idea-administrator/resources/policies/openldap-server.yml create mode 100644 source/idea/idea-administrator/resources/policies/scheduler.yml create mode 100644 source/idea/idea-administrator/resources/policies/solution-metrics-lambda-function.yml create mode 100644 source/idea/idea-administrator/resources/policies/spot-fleet-request.yml create mode 100644 source/idea/idea-administrator/resources/policies/virtual-desktop-controller.yml create mode 100644 source/idea/idea-administrator/resources/policies/virtual-desktop-dcv-broker.yml create mode 100644 source/idea/idea-administrator/resources/policies/virtual-desktop-dcv-connection-gateway.yml create mode 100644 source/idea/idea-administrator/resources/policies/virtual-desktop-dcv-host.yml create mode 100644 source/idea/idea-administrator/src/ideaadministrator/__init__.py create mode 100644 source/idea/idea-administrator/src/ideaadministrator/app/__init__.py create mode 100644 source/idea/idea-administrator/src/ideaadministrator/app/aws_service_availability_helper.py create mode 100644 source/idea/idea-administrator/src/ideaadministrator/app/cdk/__init__.py create mode 100644 source/idea/idea-administrator/src/ideaadministrator/app/cdk/cdk_app.py create mode 100644 source/idea/idea-administrator/src/ideaadministrator/app/cdk/cdk_invoker.py create mode 100644 source/idea/idea-administrator/src/ideaadministrator/app/cdk/constructs/__init__.py create mode 100644 source/idea/idea-administrator/src/ideaadministrator/app/cdk/constructs/analytics.py create mode 100644 source/idea/idea-administrator/src/ideaadministrator/app/cdk/constructs/backup.py create mode 100644 source/idea/idea-administrator/src/ideaadministrator/app/cdk/constructs/base.py create mode 100644 source/idea/idea-administrator/src/ideaadministrator/app/cdk/constructs/common.py create mode 100644 source/idea/idea-administrator/src/ideaadministrator/app/cdk/constructs/directory_service.py create mode 100644 source/idea/idea-administrator/src/ideaadministrator/app/cdk/constructs/dns.py create mode 100644 source/idea/idea-administrator/src/ideaadministrator/app/cdk/constructs/existing_resources.py create mode 100644 source/idea/idea-administrator/src/ideaadministrator/app/cdk/constructs/network.py create mode 100644 source/idea/idea-administrator/src/ideaadministrator/app/cdk/constructs/storage.py create mode 100644 source/idea/idea-administrator/src/ideaadministrator/app/cdk/idea_code_asset.py create mode 100644 source/idea/idea-administrator/src/ideaadministrator/app/cdk/stacks/__init__.py create mode 100644 source/idea/idea-administrator/src/ideaadministrator/app/cdk/stacks/analytics_stack.py create mode 100644 source/idea/idea-administrator/src/ideaadministrator/app/cdk/stacks/base_stack.py create mode 100644 source/idea/idea-administrator/src/ideaadministrator/app/cdk/stacks/bastion_host_stack.py create mode 100644 source/idea/idea-administrator/src/ideaadministrator/app/cdk/stacks/bootstrap_stack.py create mode 100644 source/idea/idea-administrator/src/ideaadministrator/app/cdk/stacks/cluster_manager_stack.py create mode 100644 source/idea/idea-administrator/src/ideaadministrator/app/cdk/stacks/cluster_stack.py create mode 100644 source/idea/idea-administrator/src/ideaadministrator/app/cdk/stacks/directoryservice_stack.py create mode 100644 source/idea/idea-administrator/src/ideaadministrator/app/cdk/stacks/identity_provider_stack.py create mode 100644 source/idea/idea-administrator/src/ideaadministrator/app/cdk/stacks/metrics_stack.py create mode 100644 source/idea/idea-administrator/src/ideaadministrator/app/cdk/stacks/scheduler_stack.py create mode 100644 source/idea/idea-administrator/src/ideaadministrator/app/cdk/stacks/shared_storage_stack.py create mode 100644 source/idea/idea-administrator/src/ideaadministrator/app/cdk/stacks/virtual_desktop_controller_stack.py create mode 100644 source/idea/idea-administrator/src/ideaadministrator/app/cluster_prefix_list_helper.py create mode 100644 source/idea/idea-administrator/src/ideaadministrator/app/config_generator.py create mode 100644 source/idea/idea-administrator/src/ideaadministrator/app/delete_cluster.py create mode 100644 source/idea/idea-administrator/src/ideaadministrator/app/deployment_helper.py create mode 100644 source/idea/idea-administrator/src/ideaadministrator/app/directory_service_helper.py create mode 100644 source/idea/idea-administrator/src/ideaadministrator/app/installer_params.py create mode 100644 source/idea/idea-administrator/src/ideaadministrator/app/patch_helper.py create mode 100644 source/idea/idea-administrator/src/ideaadministrator/app/shared_storage_helper.py create mode 100644 source/idea/idea-administrator/src/ideaadministrator/app/single_sign_on_helper.py create mode 100644 source/idea/idea-administrator/src/ideaadministrator/app/support_helper.py create mode 100644 source/idea/idea-administrator/src/ideaadministrator/app/values_diff.py create mode 100644 source/idea/idea-administrator/src/ideaadministrator/app/vpc_endpoints_helper.py create mode 100644 source/idea/idea-administrator/src/ideaadministrator/app_constants.py create mode 100644 source/idea/idea-administrator/src/ideaadministrator/app_context.py create mode 100644 source/idea/idea-administrator/src/ideaadministrator/app_main.py create mode 100644 source/idea/idea-administrator/src/ideaadministrator/app_messages.py create mode 100644 source/idea/idea-administrator/src/ideaadministrator/app_props.py create mode 100644 source/idea/idea-administrator/src/ideaadministrator/app_protocols.py create mode 100644 source/idea/idea-administrator/src/ideaadministrator/app_utils.py create mode 100644 source/idea/idea-administrator/src/ideaadministrator/integration_tests/__init__.py create mode 100644 source/idea/idea-administrator/src/ideaadministrator/integration_tests/cluster_manager_tests.py create mode 100644 source/idea/idea-administrator/src/ideaadministrator/integration_tests/scheduler/__init__.py create mode 100644 source/idea/idea-administrator/src/ideaadministrator/integration_tests/scheduler/job_test_case.py create mode 100644 source/idea/idea-administrator/src/ideaadministrator/integration_tests/scheduler_tests.py create mode 100644 source/idea/idea-administrator/src/ideaadministrator/integration_tests/test_constants.py create mode 100644 source/idea/idea-administrator/src/ideaadministrator/integration_tests/test_context.py create mode 100644 source/idea/idea-administrator/src/ideaadministrator/integration_tests/test_invoker.py create mode 100644 source/idea/idea-administrator/src/ideaadministrator/integration_tests/virtual_desktop_controller_tests.py create mode 100644 source/idea/idea-administrator/src/ideaadministrator/integration_tests/virtual_desktop_tests_util.py create mode 100644 source/idea/idea-administrator/src/ideaadministrator_meta/__init__.py create mode 100644 source/idea/idea-administrator/src/setup.py create mode 100644 source/idea/idea-administrator/tests/conftest.py create mode 100644 source/idea/idea-administrator/tests/test_example.py create mode 100644 source/idea/idea-bootstrap/_templates/linux/aws_cli.jinja2 create mode 100644 source/idea/idea-bootstrap/_templates/linux/aws_ssm.jinja2 create mode 100644 source/idea/idea-bootstrap/_templates/linux/chronyd.jinja2 create mode 100644 source/idea/idea-bootstrap/_templates/linux/cloudwatch_agent.jinja2 create mode 100644 source/idea/idea-bootstrap/_templates/linux/create_idea_app_certs.jinja2 create mode 100644 source/idea/idea-bootstrap/_templates/linux/dcv_server.jinja2 create mode 100644 source/idea/idea-bootstrap/_templates/linux/dcv_session_manager_agent.jinja2 create mode 100644 source/idea/idea-bootstrap/_templates/linux/disable_motd_update.jinja2 create mode 100644 source/idea/idea-bootstrap/_templates/linux/disable_nouveau_drivers.jinja2 create mode 100644 source/idea/idea-bootstrap/_templates/linux/disable_se_linux.jinja2 create mode 100644 source/idea/idea-bootstrap/_templates/linux/disable_strict_host_check.jinja2 create mode 100644 source/idea/idea-bootstrap/_templates/linux/disable_ulimit.jinja2 create mode 100644 source/idea/idea-bootstrap/_templates/linux/efs_mount_helper.jinja2 create mode 100644 source/idea/idea-bootstrap/_templates/linux/epel_repo.jinja2 create mode 100644 source/idea/idea-bootstrap/_templates/linux/fsx_lustre_client.jinja2 create mode 100644 source/idea/idea-bootstrap/_templates/linux/fsx_lustre_client_tuning_postmount.jinja2 create mode 100644 source/idea/idea-bootstrap/_templates/linux/fsx_lustre_client_tuning_prereboot.jinja2 create mode 100644 source/idea/idea-bootstrap/_templates/linux/gpu_drivers.jinja2 create mode 100644 source/idea/idea-bootstrap/_templates/linux/idea_service_account.jinja2 create mode 100644 source/idea/idea-bootstrap/_templates/linux/join_activedirectory.jinja2 create mode 100644 source/idea/idea-bootstrap/_templates/linux/join_directoryservice.jinja2 create mode 100644 source/idea/idea-bootstrap/_templates/linux/join_openldap.jinja2 create mode 100644 source/idea/idea-bootstrap/_templates/linux/jq.jinja2 create mode 100644 source/idea/idea-bootstrap/_templates/linux/motd.jinja2 create mode 100644 source/idea/idea-bootstrap/_templates/linux/mount_shared_storage.jinja2 create mode 100644 source/idea/idea-bootstrap/_templates/linux/nfs_utils.jinja2 create mode 100644 source/idea/idea-bootstrap/_templates/linux/nodejs.jinja2 create mode 100644 source/idea/idea-bootstrap/_templates/linux/openmpi.jinja2 create mode 100644 source/idea/idea-bootstrap/_templates/linux/openpbs.jinja2 create mode 100644 source/idea/idea-bootstrap/_templates/linux/openpbs_client.jinja2 create mode 100644 source/idea/idea-bootstrap/_templates/linux/prometheus.jinja2 create mode 100644 source/idea/idea-bootstrap/_templates/linux/prometheus_node_exporter.jinja2 create mode 100644 source/idea/idea-bootstrap/_templates/linux/python.jinja2 create mode 100644 source/idea/idea-bootstrap/_templates/linux/restrict_ssh.jinja2 create mode 100644 source/idea/idea-bootstrap/_templates/linux/stunnel.jinja2 create mode 100644 source/idea/idea-bootstrap/_templates/linux/sudoer_secure_path.jinja2 create mode 100644 source/idea/idea-bootstrap/_templates/linux/supervisord.jinja2 create mode 100644 source/idea/idea-bootstrap/_templates/linux/system_packages.jinja2 create mode 100644 source/idea/idea-bootstrap/_templates/linux/tag_ebs_volumes.jinja2 create mode 100644 source/idea/idea-bootstrap/_templates/linux/tag_network_interface.jinja2 create mode 100644 source/idea/idea-bootstrap/_templates/windows/join_activedirectory.jinja2 create mode 100644 source/idea/idea-bootstrap/_templates/windows/mount_shared_storage.jinja2 create mode 100644 source/idea/idea-bootstrap/bastion-host/setup.sh.jinja2 create mode 100644 source/idea/idea-bootstrap/cluster-manager/install_app.sh.jinja2 create mode 100644 source/idea/idea-bootstrap/cluster-manager/setup.sh.jinja2 create mode 100644 source/idea/idea-bootstrap/common/bootstrap_common.sh create mode 100644 source/idea/idea-bootstrap/compute-node-ami-builder/compute_node_ami_builder.sh.jinja2 create mode 100644 source/idea/idea-bootstrap/compute-node-ami-builder/compute_node_ami_builder_post_reboot.sh.jinja2 create mode 100644 source/idea/idea-bootstrap/compute-node-ami-builder/setup.sh.jinja2 create mode 100644 source/idea/idea-bootstrap/compute-node/_templates/configure_hyperthreading.jinja2 create mode 100644 source/idea/idea-bootstrap/compute-node/_templates/configure_openpbs_compute_node.jinja2 create mode 100644 source/idea/idea-bootstrap/compute-node/_templates/efa.jinja2 create mode 100644 source/idea/idea-bootstrap/compute-node/_templates/scheduler_start.jinja2 create mode 100644 source/idea/idea-bootstrap/compute-node/_templates/scheduler_stop.jinja2 create mode 100644 source/idea/idea-bootstrap/compute-node/_templates/scratch_storage.jinja2 create mode 100644 source/idea/idea-bootstrap/compute-node/compute_node.sh.jinja2 create mode 100644 source/idea/idea-bootstrap/compute-node/compute_node_post_reboot.sh.jinja2 create mode 100644 source/idea/idea-bootstrap/compute-node/setup.sh.jinja2 create mode 100644 source/idea/idea-bootstrap/dcv-broker/install_app.sh.jinja2 create mode 100644 source/idea/idea-bootstrap/dcv-broker/setup.sh.jinja2 create mode 100644 source/idea/idea-bootstrap/dcv-connection-gateway/install_app.sh.jinja2 create mode 100644 source/idea/idea-bootstrap/dcv-connection-gateway/setup.sh.jinja2 create mode 100644 source/idea/idea-bootstrap/openldap-server/_templates/install_openldap.jinja2 create mode 100644 source/idea/idea-bootstrap/openldap-server/setup.sh.jinja2 create mode 100644 source/idea/idea-bootstrap/scheduler/_templates/configure_openpbs_server.jinja2 create mode 100644 source/idea/idea-bootstrap/scheduler/install_app.sh.jinja2 create mode 100644 source/idea/idea-bootstrap/scheduler/scheduler_post_reboot.sh.jinja2 create mode 100644 source/idea/idea-bootstrap/scheduler/setup.sh.jinja2 create mode 100644 source/idea/idea-bootstrap/virtual-desktop-controller/install_app.sh.jinja2 create mode 100644 source/idea/idea-bootstrap/virtual-desktop-controller/setup.sh.jinja2 create mode 100644 source/idea/idea-bootstrap/virtual-desktop-host-linux/configure_dcv_host.sh.jinja2 create mode 100644 source/idea/idea-bootstrap/virtual-desktop-host-linux/configure_dcv_host_post_reboot.sh.jinja2 create mode 100644 source/idea/idea-bootstrap/virtual-desktop-host-linux/setup.sh.jinja2 create mode 100644 source/idea/idea-bootstrap/virtual-desktop-host-windows/ConfigureDCVHost.ps1.jinja2 create mode 100644 source/idea/idea-bootstrap/virtual-desktop-host-windows/SetUp.ps1.jinja2 create mode 100644 source/idea/idea-cli/src/ideacli/__init__.py create mode 100644 source/idea/idea-cli/src/ideacli/idea.py create mode 100644 source/idea/idea-cli/src/ideacli_meta/__init__.py create mode 100644 source/idea/idea-cli/src/setup.py create mode 100644 source/idea/idea-cluster-manager/resources/api/api_doc.yml create mode 100644 source/idea/idea-cluster-manager/resources/defaults/email_templates.yml create mode 100644 source/idea/idea-cluster-manager/src/ideaclustermanager/__init__.py create mode 100644 source/idea/idea-cluster-manager/src/ideaclustermanager/app/__init__.py create mode 100644 source/idea/idea-cluster-manager/src/ideaclustermanager/app/accounts/__init__.py create mode 100644 source/idea/idea-cluster-manager/src/ideaclustermanager/app/accounts/account_tasks.py create mode 100644 source/idea/idea-cluster-manager/src/ideaclustermanager/app/accounts/accounts_service.py create mode 100644 source/idea/idea-cluster-manager/src/ideaclustermanager/app/accounts/ad_automation_agent.py create mode 100644 source/idea/idea-cluster-manager/src/ideaclustermanager/app/accounts/auth_constants.py create mode 100644 source/idea/idea-cluster-manager/src/ideaclustermanager/app/accounts/auth_utils.py create mode 100644 source/idea/idea-cluster-manager/src/ideaclustermanager/app/accounts/cognito_user_pool.py create mode 100644 source/idea/idea-cluster-manager/src/ideaclustermanager/app/accounts/db/__init__.py create mode 100644 source/idea/idea-cluster-manager/src/ideaclustermanager/app/accounts/db/ad_automation_dao.py create mode 100644 source/idea/idea-cluster-manager/src/ideaclustermanager/app/accounts/db/group_dao.py create mode 100644 source/idea/idea-cluster-manager/src/ideaclustermanager/app/accounts/db/group_members_dao.py create mode 100644 source/idea/idea-cluster-manager/src/ideaclustermanager/app/accounts/db/sequence_config_dao.py create mode 100644 source/idea/idea-cluster-manager/src/ideaclustermanager/app/accounts/db/single_sign_on_state_dao.py create mode 100644 source/idea/idea-cluster-manager/src/ideaclustermanager/app/accounts/db/user_dao.py create mode 100644 source/idea/idea-cluster-manager/src/ideaclustermanager/app/accounts/helpers/__init__.py create mode 100644 source/idea/idea-cluster-manager/src/ideaclustermanager/app/accounts/helpers/preset_computer_helper.py create mode 100644 source/idea/idea-cluster-manager/src/ideaclustermanager/app/accounts/ldapclient/__init__.py create mode 100644 source/idea/idea-cluster-manager/src/ideaclustermanager/app/accounts/ldapclient/abstract_ldap_client.py create mode 100644 source/idea/idea-cluster-manager/src/ideaclustermanager/app/accounts/ldapclient/active_directory_client.py create mode 100644 source/idea/idea-cluster-manager/src/ideaclustermanager/app/accounts/ldapclient/ldap_client_factory.py create mode 100644 source/idea/idea-cluster-manager/src/ideaclustermanager/app/accounts/ldapclient/ldap_utils.py create mode 100644 source/idea/idea-cluster-manager/src/ideaclustermanager/app/accounts/ldapclient/openldap_client.py create mode 100644 source/idea/idea-cluster-manager/src/ideaclustermanager/app/accounts/user_home_directory.py create mode 100644 source/idea/idea-cluster-manager/src/ideaclustermanager/app/api/__init__.py create mode 100644 source/idea/idea-cluster-manager/src/ideaclustermanager/app/api/accounts_api.py create mode 100644 source/idea/idea-cluster-manager/src/ideaclustermanager/app/api/analytics_api.py create mode 100644 source/idea/idea-cluster-manager/src/ideaclustermanager/app/api/api_invoker.py create mode 100644 source/idea/idea-cluster-manager/src/ideaclustermanager/app/api/auth_api.py create mode 100644 source/idea/idea-cluster-manager/src/ideaclustermanager/app/api/cluster_settings_api.py create mode 100644 source/idea/idea-cluster-manager/src/ideaclustermanager/app/api/email_templates_api.py create mode 100644 source/idea/idea-cluster-manager/src/ideaclustermanager/app/api/projects_api.py create mode 100644 source/idea/idea-cluster-manager/src/ideaclustermanager/app/app_context.py create mode 100644 source/idea/idea-cluster-manager/src/ideaclustermanager/app/app_main.py create mode 100644 source/idea/idea-cluster-manager/src/ideaclustermanager/app/app_messages.py create mode 100644 source/idea/idea-cluster-manager/src/ideaclustermanager/app/app_utils.py create mode 100644 source/idea/idea-cluster-manager/src/ideaclustermanager/app/cluster_manager_app.py create mode 100644 source/idea/idea-cluster-manager/src/ideaclustermanager/app/email_templates/__init__.py create mode 100644 source/idea/idea-cluster-manager/src/ideaclustermanager/app/email_templates/email_templates_dao.py create mode 100644 source/idea/idea-cluster-manager/src/ideaclustermanager/app/email_templates/email_templates_service.py create mode 100644 source/idea/idea-cluster-manager/src/ideaclustermanager/app/notifications/__init__.py create mode 100644 source/idea/idea-cluster-manager/src/ideaclustermanager/app/notifications/notifications_service.py create mode 100644 source/idea/idea-cluster-manager/src/ideaclustermanager/app/projects/__init__.py create mode 100644 source/idea/idea-cluster-manager/src/ideaclustermanager/app/projects/db/__init__.py create mode 100644 source/idea/idea-cluster-manager/src/ideaclustermanager/app/projects/db/projects_dao.py create mode 100644 source/idea/idea-cluster-manager/src/ideaclustermanager/app/projects/db/user_projects_dao.py create mode 100644 source/idea/idea-cluster-manager/src/ideaclustermanager/app/projects/project_tasks.py create mode 100644 source/idea/idea-cluster-manager/src/ideaclustermanager/app/projects/projects_service.py create mode 100644 source/idea/idea-cluster-manager/src/ideaclustermanager/app/tasks/__init__.py create mode 100644 source/idea/idea-cluster-manager/src/ideaclustermanager/app/tasks/base_task.py create mode 100644 source/idea/idea-cluster-manager/src/ideaclustermanager/app/tasks/task_manager.py create mode 100644 source/idea/idea-cluster-manager/src/ideaclustermanager/app/web_portal.py create mode 100644 source/idea/idea-cluster-manager/src/ideaclustermanager/cli/__init__.py create mode 100644 source/idea/idea-cluster-manager/src/ideaclustermanager/cli/accounts.py create mode 100644 source/idea/idea-cluster-manager/src/ideaclustermanager/cli/cli_main.py create mode 100644 source/idea/idea-cluster-manager/src/ideaclustermanager/cli/cli_utils.py create mode 100644 source/idea/idea-cluster-manager/src/ideaclustermanager/cli/groups.py create mode 100644 source/idea/idea-cluster-manager/src/ideaclustermanager/cli/ldap_commands.py create mode 100644 source/idea/idea-cluster-manager/src/ideaclustermanager/cli/logs.py create mode 100644 source/idea/idea-cluster-manager/src/ideaclustermanager/cli/module.py create mode 100644 source/idea/idea-cluster-manager/src/ideaclustermanager_meta/__init__.py create mode 100644 source/idea/idea-cluster-manager/src/setup.py create mode 100644 source/idea/idea-cluster-manager/tests/conftest.py create mode 100644 source/idea/idea-cluster-manager/tests/ideaclustermanagertests/__init__.py create mode 100644 source/idea/idea-cluster-manager/tests/ideaclustermanagertests/mock_ldap_client.py create mode 100644 source/idea/idea-cluster-manager/tests/ideaclustermanagertests/test_accounts.py create mode 100644 source/idea/idea-cluster-manager/tests/ideaclustermanagertests/test_projects.py create mode 100644 source/idea/idea-cluster-manager/webapp/.env create mode 100644 source/idea/idea-cluster-manager/webapp/.gitignore create mode 100644 source/idea/idea-cluster-manager/webapp/README.md create mode 100644 source/idea/idea-cluster-manager/webapp/package.json create mode 100644 source/idea/idea-cluster-manager/webapp/public/android-chrome-192x192.png create mode 100644 source/idea/idea-cluster-manager/webapp/public/android-chrome-512x512.png create mode 100644 source/idea/idea-cluster-manager/webapp/public/apple-touch-icon.png create mode 100644 source/idea/idea-cluster-manager/webapp/public/browserconfig.xml create mode 100644 source/idea/idea-cluster-manager/webapp/public/favicon-16x16.png create mode 100644 source/idea/idea-cluster-manager/webapp/public/favicon-32x32.png create mode 100644 source/idea/idea-cluster-manager/webapp/public/favicon.ico create mode 100644 source/idea/idea-cluster-manager/webapp/public/index.html create mode 100644 source/idea/idea-cluster-manager/webapp/public/logo.png create mode 100644 source/idea/idea-cluster-manager/webapp/public/manifest.json create mode 100644 source/idea/idea-cluster-manager/webapp/public/mstile-150x150.png create mode 100644 source/idea/idea-cluster-manager/webapp/public/robots.txt create mode 100644 source/idea/idea-cluster-manager/webapp/public/safari-pinned-tab.svg create mode 100644 source/idea/idea-cluster-manager/webapp/src/App.scss create mode 100644 source/idea/idea-cluster-manager/webapp/src/App.test.tsx create mode 100644 source/idea/idea-cluster-manager/webapp/src/App.tsx create mode 100644 source/idea/idea-cluster-manager/webapp/src/client/accounts-client.ts create mode 100644 source/idea/idea-cluster-manager/webapp/src/client/analytics-client.ts create mode 100644 source/idea/idea-cluster-manager/webapp/src/client/auth-client.ts create mode 100644 source/idea/idea-cluster-manager/webapp/src/client/base-client.ts create mode 100644 source/idea/idea-cluster-manager/webapp/src/client/clients.ts create mode 100644 source/idea/idea-cluster-manager/webapp/src/client/cluster-settings-client.ts create mode 100644 source/idea/idea-cluster-manager/webapp/src/client/data-model.ts create mode 100644 source/idea/idea-cluster-manager/webapp/src/client/email-templates-client.ts create mode 100644 source/idea/idea-cluster-manager/webapp/src/client/file-browser-client.ts create mode 100644 source/idea/idea-cluster-manager/webapp/src/client/idea-api-invoker.ts create mode 100644 source/idea/idea-cluster-manager/webapp/src/client/index.ts create mode 100644 source/idea/idea-cluster-manager/webapp/src/client/projects-client.ts create mode 100644 source/idea/idea-cluster-manager/webapp/src/client/scheduler-admin-client.ts create mode 100644 source/idea/idea-cluster-manager/webapp/src/client/scheduler-client.ts create mode 100644 source/idea/idea-cluster-manager/webapp/src/client/virtual-desktop-admin-client.ts create mode 100644 source/idea/idea-cluster-manager/webapp/src/client/virtual-desktop-client.ts create mode 100644 source/idea/idea-cluster-manager/webapp/src/client/virtual-desktop-dcv-client.ts create mode 100644 source/idea/idea-cluster-manager/webapp/src/client/virtual-desktop-utils-client.ts create mode 100644 source/idea/idea-cluster-manager/webapp/src/common/app-context.ts create mode 100644 source/idea/idea-cluster-manager/webapp/src/common/app-logger.ts create mode 100644 source/idea/idea-cluster-manager/webapp/src/common/authentication-context.ts create mode 100644 source/idea/idea-cluster-manager/webapp/src/common/config-utils.ts create mode 100644 source/idea/idea-cluster-manager/webapp/src/common/constants.ts create mode 100644 source/idea/idea-cluster-manager/webapp/src/common/error-codes.ts create mode 100644 source/idea/idea-cluster-manager/webapp/src/common/exceptions.ts create mode 100644 source/idea/idea-cluster-manager/webapp/src/common/index.ts create mode 100644 source/idea/idea-cluster-manager/webapp/src/common/shared-storage-utils.ts create mode 100644 source/idea/idea-cluster-manager/webapp/src/common/token-utils.ts create mode 100644 source/idea/idea-cluster-manager/webapp/src/common/utils.ts create mode 100644 source/idea/idea-cluster-manager/webapp/src/components/app-layout/app-layout.tsx create mode 100644 source/idea/idea-cluster-manager/webapp/src/components/app-layout/index.ts create mode 100644 source/idea/idea-cluster-manager/webapp/src/components/charts/pie-or-donut-chart.tsx create mode 100644 source/idea/idea-cluster-manager/webapp/src/components/common/index.tsx create mode 100644 source/idea/idea-cluster-manager/webapp/src/components/form-builder/form-builder.tsx create mode 100644 source/idea/idea-cluster-manager/webapp/src/components/form-builder/index.ts create mode 100644 source/idea/idea-cluster-manager/webapp/src/components/form-field/form-field.tsx create mode 100644 source/idea/idea-cluster-manager/webapp/src/components/form-field/index.ts create mode 100644 source/idea/idea-cluster-manager/webapp/src/components/form-review-field/form-review-field.tsx create mode 100644 source/idea/idea-cluster-manager/webapp/src/components/form-review-field/index.ts create mode 100644 source/idea/idea-cluster-manager/webapp/src/components/form/form.tsx create mode 100644 source/idea/idea-cluster-manager/webapp/src/components/form/index.ts create mode 100644 source/idea/idea-cluster-manager/webapp/src/components/key-value/index.ts create mode 100644 source/idea/idea-cluster-manager/webapp/src/components/key-value/key-value.tsx create mode 100644 source/idea/idea-cluster-manager/webapp/src/components/list-view/index.ts create mode 100644 source/idea/idea-cluster-manager/webapp/src/components/list-view/list-view.tsx create mode 100644 source/idea/idea-cluster-manager/webapp/src/components/modals/confirm.tsx create mode 100644 source/idea/idea-cluster-manager/webapp/src/components/modals/index.ts create mode 100644 source/idea/idea-cluster-manager/webapp/src/components/modals/view.tsx create mode 100644 source/idea/idea-cluster-manager/webapp/src/components/navbar/index.ts create mode 100644 source/idea/idea-cluster-manager/webapp/src/components/navbar/navbar.tsx create mode 100644 source/idea/idea-cluster-manager/webapp/src/components/password-strength-check/index.ts create mode 100644 source/idea/idea-cluster-manager/webapp/src/components/password-strength-check/password-strength-check.tsx create mode 100644 source/idea/idea-cluster-manager/webapp/src/components/side-navigation/index.ts create mode 100644 source/idea/idea-cluster-manager/webapp/src/components/side-navigation/side-navigation.scss create mode 100644 source/idea/idea-cluster-manager/webapp/src/components/side-navigation/side-navigation.tsx create mode 100644 source/idea/idea-cluster-manager/webapp/src/components/split-panel/index.ts create mode 100644 source/idea/idea-cluster-manager/webapp/src/components/split-panel/split-panel.tsx create mode 100644 source/idea/idea-cluster-manager/webapp/src/components/table/index.ts create mode 100644 source/idea/idea-cluster-manager/webapp/src/components/table/table.tsx create mode 100644 source/idea/idea-cluster-manager/webapp/src/components/tabs/index.ts create mode 100644 source/idea/idea-cluster-manager/webapp/src/components/tabs/tabs.tsx create mode 100644 source/idea/idea-cluster-manager/webapp/src/components/time-range-slider/index.ts create mode 100644 source/idea/idea-cluster-manager/webapp/src/components/time-range-slider/time-range-slider.tsx create mode 100644 source/idea/idea-cluster-manager/webapp/src/components/wizard/index.ts create mode 100644 source/idea/idea-cluster-manager/webapp/src/components/wizard/wizard.tsx create mode 100644 source/idea/idea-cluster-manager/webapp/src/docs/_footer.md create mode 100644 source/idea/idea-cluster-manager/webapp/src/docs/account-settings.md create mode 100644 source/idea/idea-cluster-manager/webapp/src/docs/active-jobs.md create mode 100644 source/idea/idea-cluster-manager/webapp/src/docs/cluster-settings.md create mode 100644 source/idea/idea-cluster-manager/webapp/src/docs/cluster-status.md create mode 100644 source/idea/idea-cluster-manager/webapp/src/docs/completed-jobs.md create mode 100644 source/idea/idea-cluster-manager/webapp/src/docs/create-hpc-application.md create mode 100644 source/idea/idea-cluster-manager/webapp/src/docs/create-queue-profile.md create mode 100644 source/idea/idea-cluster-manager/webapp/src/docs/dashboard.md create mode 100644 source/idea/idea-cluster-manager/webapp/src/docs/email-templates.md create mode 100644 source/idea/idea-cluster-manager/webapp/src/docs/file-browser.md create mode 100644 source/idea/idea-cluster-manager/webapp/src/docs/groups.md create mode 100644 source/idea/idea-cluster-manager/webapp/src/docs/home.md create mode 100644 source/idea/idea-cluster-manager/webapp/src/docs/hpc-applications.md create mode 100644 source/idea/idea-cluster-manager/webapp/src/docs/hpc-license-create.md create mode 100644 source/idea/idea-cluster-manager/webapp/src/docs/hpc-license-update.md create mode 100644 source/idea/idea-cluster-manager/webapp/src/docs/hpc-licenses.md create mode 100644 source/idea/idea-cluster-manager/webapp/src/docs/hpc-settings.md create mode 100644 source/idea/idea-cluster-manager/webapp/src/docs/my-shared-virtual-desktop-sessions.md create mode 100644 source/idea/idea-cluster-manager/webapp/src/docs/my-virtual-desktop-sessions.md create mode 100644 source/idea/idea-cluster-manager/webapp/src/docs/projects.md create mode 100644 source/idea/idea-cluster-manager/webapp/src/docs/queue-profiles.md create mode 100644 source/idea/idea-cluster-manager/webapp/src/docs/ssh-access.md create mode 100644 source/idea/idea-cluster-manager/webapp/src/docs/submit-job.md create mode 100644 source/idea/idea-cluster-manager/webapp/src/docs/update-hpc-application.md create mode 100644 source/idea/idea-cluster-manager/webapp/src/docs/update-queue-profile.md create mode 100644 source/idea/idea-cluster-manager/webapp/src/docs/users.md create mode 100644 source/idea/idea-cluster-manager/webapp/src/docs/virtual-desktop-dashboard.md create mode 100644 source/idea/idea-cluster-manager/webapp/src/docs/virtual-desktop-debug.md create mode 100644 source/idea/idea-cluster-manager/webapp/src/docs/virtual-desktop-permission-profiles.md create mode 100644 source/idea/idea-cluster-manager/webapp/src/docs/virtual-desktop-session-detail.md create mode 100644 source/idea/idea-cluster-manager/webapp/src/docs/virtual-desktop-sessions.md create mode 100644 source/idea/idea-cluster-manager/webapp/src/docs/virtual-desktop-settings.md create mode 100644 source/idea/idea-cluster-manager/webapp/src/docs/virtual-desktop-software-stack-detail.md create mode 100644 source/idea/idea-cluster-manager/webapp/src/docs/virtual-desktop-software-stacks.md create mode 100644 source/idea/idea-cluster-manager/webapp/src/index.scss create mode 100644 source/idea/idea-cluster-manager/webapp/src/index.tsx create mode 100644 source/idea/idea-cluster-manager/webapp/src/navigation/navigation-utils.tsx create mode 100644 source/idea/idea-cluster-manager/webapp/src/navigation/side-nav-items.tsx create mode 100644 source/idea/idea-cluster-manager/webapp/src/pages/account/account-settings.tsx create mode 100644 source/idea/idea-cluster-manager/webapp/src/pages/auth/auth-challenge.tsx create mode 100644 source/idea/idea-cluster-manager/webapp/src/pages/auth/auth-confirm-forgot-password.tsx create mode 100644 source/idea/idea-cluster-manager/webapp/src/pages/auth/auth-context.ts create mode 100644 source/idea/idea-cluster-manager/webapp/src/pages/auth/auth-forgot-password.tsx create mode 100644 source/idea/idea-cluster-manager/webapp/src/pages/auth/auth-interfaces.ts create mode 100644 source/idea/idea-cluster-manager/webapp/src/pages/auth/auth-layout.tsx create mode 100644 source/idea/idea-cluster-manager/webapp/src/pages/auth/auth-login.tsx create mode 100644 source/idea/idea-cluster-manager/webapp/src/pages/auth/auth-route.tsx create mode 100644 source/idea/idea-cluster-manager/webapp/src/pages/auth/auth.scss create mode 100644 source/idea/idea-cluster-manager/webapp/src/pages/auth/index.ts create mode 100644 source/idea/idea-cluster-manager/webapp/src/pages/cluster-admin/cluster-settings.tsx create mode 100644 source/idea/idea-cluster-manager/webapp/src/pages/cluster-admin/cluster-status.tsx create mode 100644 source/idea/idea-cluster-manager/webapp/src/pages/cluster-admin/email-templates.tsx create mode 100644 source/idea/idea-cluster-manager/webapp/src/pages/cluster-admin/projects.tsx create mode 100644 source/idea/idea-cluster-manager/webapp/src/pages/dashboard/dashboard-main.tsx create mode 100644 source/idea/idea-cluster-manager/webapp/src/pages/dashboard/index.ts create mode 100644 source/idea/idea-cluster-manager/webapp/src/pages/dashboard/job-submissions-widget.tsx create mode 100644 source/idea/idea-cluster-manager/webapp/src/pages/home.tsx create mode 100644 source/idea/idea-cluster-manager/webapp/src/pages/home/file-browser.tsx create mode 100644 source/idea/idea-cluster-manager/webapp/src/pages/home/log-tail.tsx create mode 100644 source/idea/idea-cluster-manager/webapp/src/pages/home/ssh-access.tsx create mode 100644 source/idea/idea-cluster-manager/webapp/src/pages/hpc/hpc-applications.tsx create mode 100644 source/idea/idea-cluster-manager/webapp/src/pages/hpc/hpc-custom-amis.tsx create mode 100644 source/idea/idea-cluster-manager/webapp/src/pages/hpc/hpc-licenses.tsx create mode 100644 source/idea/idea-cluster-manager/webapp/src/pages/hpc/hpc-nodes.tsx create mode 100644 source/idea/idea-cluster-manager/webapp/src/pages/hpc/hpc-notifications.tsx create mode 100644 source/idea/idea-cluster-manager/webapp/src/pages/hpc/hpc-scheduler-settings.tsx create mode 100644 source/idea/idea-cluster-manager/webapp/src/pages/hpc/hpc-utils.ts create mode 100644 source/idea/idea-cluster-manager/webapp/src/pages/hpc/job-templates.tsx create mode 100644 source/idea/idea-cluster-manager/webapp/src/pages/hpc/jobs.tsx create mode 100644 source/idea/idea-cluster-manager/webapp/src/pages/hpc/queues.tsx create mode 100644 source/idea/idea-cluster-manager/webapp/src/pages/hpc/sample-job-params.json create mode 100644 source/idea/idea-cluster-manager/webapp/src/pages/hpc/sample-job-script-jinja2.txt create mode 100644 source/idea/idea-cluster-manager/webapp/src/pages/hpc/sample-job-script-simple.txt create mode 100644 source/idea/idea-cluster-manager/webapp/src/pages/hpc/submit-job.tsx create mode 100644 source/idea/idea-cluster-manager/webapp/src/pages/hpc/update-hpc-application.tsx create mode 100644 source/idea/idea-cluster-manager/webapp/src/pages/hpc/update-hpc-license.tsx create mode 100644 source/idea/idea-cluster-manager/webapp/src/pages/hpc/update-queue-profile.tsx create mode 100644 source/idea/idea-cluster-manager/webapp/src/pages/user-management/groups.tsx create mode 100644 source/idea/idea-cluster-manager/webapp/src/pages/user-management/users.tsx create mode 100644 source/idea/idea-cluster-manager/webapp/src/pages/virtual-desktops/charts/virtual-desktop-az-distribution.tsx create mode 100644 source/idea/idea-cluster-manager/webapp/src/pages/virtual-desktops/charts/virtual-desktop-base-chart.tsx create mode 100644 source/idea/idea-cluster-manager/webapp/src/pages/virtual-desktops/charts/virtual-desktop-baseos-chart.tsx create mode 100644 source/idea/idea-cluster-manager/webapp/src/pages/virtual-desktops/charts/virtual-desktop-instance-types-chart.tsx create mode 100644 source/idea/idea-cluster-manager/webapp/src/pages/virtual-desktops/charts/virtual-desktop-project-chart.tsx create mode 100644 source/idea/idea-cluster-manager/webapp/src/pages/virtual-desktops/charts/virtual-desktop-software-stack-chart.tsx create mode 100644 source/idea/idea-cluster-manager/webapp/src/pages/virtual-desktops/charts/virtual-desktop-state-chart.tsx create mode 100644 source/idea/idea-cluster-manager/webapp/src/pages/virtual-desktops/components/dcv-client-help-modal.tsx create mode 100644 source/idea/idea-cluster-manager/webapp/src/pages/virtual-desktops/components/virtual-desktop-schedule-modal.tsx create mode 100644 source/idea/idea-cluster-manager/webapp/src/pages/virtual-desktops/components/virtual-desktop-session-card.tsx create mode 100644 source/idea/idea-cluster-manager/webapp/src/pages/virtual-desktops/components/virtual-desktop-session-status-indicator.tsx create mode 100644 source/idea/idea-cluster-manager/webapp/src/pages/virtual-desktops/forms/virtual-desktop-create-session-form.tsx create mode 100644 source/idea/idea-cluster-manager/webapp/src/pages/virtual-desktops/forms/virtual-desktop-permission-profile-form.tsx create mode 100644 source/idea/idea-cluster-manager/webapp/src/pages/virtual-desktops/forms/virtual-desktop-software-stack-edit-form.tsx create mode 100644 source/idea/idea-cluster-manager/webapp/src/pages/virtual-desktops/forms/virtual-desktop-update-session-permissions-form.tsx create mode 100644 source/idea/idea-cluster-manager/webapp/src/pages/virtual-desktops/my-shared-virtual-desktop-sessions.tsx create mode 100644 source/idea/idea-cluster-manager/webapp/src/pages/virtual-desktops/my-virtual-desktop-sessions.tsx create mode 100644 source/idea/idea-cluster-manager/webapp/src/pages/virtual-desktops/virtual-desktop-dashboard.tsx create mode 100644 source/idea/idea-cluster-manager/webapp/src/pages/virtual-desktops/virtual-desktop-debug.tsx create mode 100644 source/idea/idea-cluster-manager/webapp/src/pages/virtual-desktops/virtual-desktop-permission-profile-detail.tsx create mode 100644 source/idea/idea-cluster-manager/webapp/src/pages/virtual-desktops/virtual-desktop-permission-profiles.tsx create mode 100644 source/idea/idea-cluster-manager/webapp/src/pages/virtual-desktops/virtual-desktop-session-detail.tsx create mode 100644 source/idea/idea-cluster-manager/webapp/src/pages/virtual-desktops/virtual-desktop-sessions.tsx create mode 100644 source/idea/idea-cluster-manager/webapp/src/pages/virtual-desktops/virtual-desktop-settings.tsx create mode 100644 source/idea/idea-cluster-manager/webapp/src/pages/virtual-desktops/virtual-desktop-software-stack-detail.tsx create mode 100644 source/idea/idea-cluster-manager/webapp/src/pages/virtual-desktops/virtual-desktop-software-stacks.tsx create mode 100644 source/idea/idea-cluster-manager/webapp/src/react-app-env.d.ts create mode 100644 source/idea/idea-cluster-manager/webapp/src/service-worker-registration.ts create mode 100644 source/idea/idea-cluster-manager/webapp/src/service-worker.ts create mode 100644 source/idea/idea-cluster-manager/webapp/src/service/auth-service.ts create mode 100644 source/idea/idea-cluster-manager/webapp/src/service/cluster-settings-service.ts create mode 100644 source/idea/idea-cluster-manager/webapp/src/service/index.ts create mode 100644 source/idea/idea-cluster-manager/webapp/src/service/job-templates-service.ts create mode 100644 source/idea/idea-cluster-manager/webapp/src/service/local-storage-service.ts create mode 100644 source/idea/idea-cluster-manager/webapp/src/setupTests.ts create mode 100644 source/idea/idea-cluster-manager/webapp/src/styles/home.scss create mode 100644 source/idea/idea-cluster-manager/webapp/tsconfig.json create mode 100644 source/idea/idea-cluster-manager/webapp/yarn.lock create mode 100644 source/idea/idea-data-model/src/ideadatamodel/__init__.py create mode 100644 source/idea/idea-data-model/src/ideadatamodel/analytics/__init__.py create mode 100644 source/idea/idea-data-model/src/ideadatamodel/analytics/analytics_api.py create mode 100644 source/idea/idea-data-model/src/ideadatamodel/analytics/analytics_model.py create mode 100644 source/idea/idea-data-model/src/ideadatamodel/api/__init__.py create mode 100644 source/idea/idea-data-model/src/ideadatamodel/api/api_model.py create mode 100644 source/idea/idea-data-model/src/ideadatamodel/api/logging.py create mode 100644 source/idea/idea-data-model/src/ideadatamodel/app/__init__.py create mode 100644 source/idea/idea-data-model/src/ideadatamodel/app/app_api.py create mode 100644 source/idea/idea-data-model/src/ideadatamodel/app/app_model.py create mode 100644 source/idea/idea-data-model/src/ideadatamodel/auth/__init__.py create mode 100644 source/idea/idea-data-model/src/ideadatamodel/auth/auth_api.py create mode 100644 source/idea/idea-data-model/src/ideadatamodel/auth/auth_model.py create mode 100644 source/idea/idea-data-model/src/ideadatamodel/aws/__init__.py create mode 100644 source/idea/idea-data-model/src/ideadatamodel/aws/autoscaling_group.py create mode 100644 source/idea/idea-data-model/src/ideadatamodel/aws/cloudformation_stack.py create mode 100644 source/idea/idea-data-model/src/ideadatamodel/aws/cloudformation_stack_resources.py create mode 100644 source/idea/idea-data-model/src/ideadatamodel/aws/ec2_instance.py create mode 100644 source/idea/idea-data-model/src/ideadatamodel/aws/ec2_instance_type.py create mode 100644 source/idea/idea-data-model/src/ideadatamodel/aws/ec2_spot_fleet_request_config.py create mode 100644 source/idea/idea-data-model/src/ideadatamodel/aws/model.py create mode 100644 source/idea/idea-data-model/src/ideadatamodel/base.py create mode 100644 source/idea/idea-data-model/src/ideadatamodel/cluster_resources/__init__.py create mode 100644 source/idea/idea-data-model/src/ideadatamodel/cluster_resources/cluster_resources_model.py create mode 100644 source/idea/idea-data-model/src/ideadatamodel/cluster_settings/__init__.py create mode 100644 source/idea/idea-data-model/src/ideadatamodel/cluster_settings/cluster_settings_api.py create mode 100644 source/idea/idea-data-model/src/ideadatamodel/cluster_settings/cluster_settings_model.py create mode 100644 source/idea/idea-data-model/src/ideadatamodel/common/__init__.py create mode 100644 source/idea/idea-data-model/src/ideadatamodel/common/common_model.py create mode 100644 source/idea/idea-data-model/src/ideadatamodel/constants.py create mode 100644 source/idea/idea-data-model/src/ideadatamodel/email_templates/__init__.py create mode 100644 source/idea/idea-data-model/src/ideadatamodel/email_templates/email_templates_api.py create mode 100644 source/idea/idea-data-model/src/ideadatamodel/email_templates/email_templates_model.py create mode 100644 source/idea/idea-data-model/src/ideadatamodel/errorcodes.py create mode 100644 source/idea/idea-data-model/src/ideadatamodel/exceptions/__init__.py create mode 100644 source/idea/idea-data-model/src/ideadatamodel/exceptions/exception_utils.py create mode 100644 source/idea/idea-data-model/src/ideadatamodel/exceptions/exceptions.py create mode 100644 source/idea/idea-data-model/src/ideadatamodel/filesystem/__init__.py create mode 100644 source/idea/idea-data-model/src/ideadatamodel/filesystem/filesystem_api.py create mode 100644 source/idea/idea-data-model/src/ideadatamodel/filesystem/filesystem_model.py create mode 100644 source/idea/idea-data-model/src/ideadatamodel/locale/__init__.py create mode 100644 source/idea/idea-data-model/src/ideadatamodel/model_utils.py create mode 100644 source/idea/idea-data-model/src/ideadatamodel/notifications/__init__.py create mode 100644 source/idea/idea-data-model/src/ideadatamodel/notifications/notifications_api.py create mode 100644 source/idea/idea-data-model/src/ideadatamodel/notifications/notifications_model.py create mode 100644 source/idea/idea-data-model/src/ideadatamodel/projects/__init__.py create mode 100644 source/idea/idea-data-model/src/ideadatamodel/projects/projects_api.py create mode 100644 source/idea/idea-data-model/src/ideadatamodel/projects/projects_model.py create mode 100644 source/idea/idea-data-model/src/ideadatamodel/scheduler/__init__.py create mode 100644 source/idea/idea-data-model/src/ideadatamodel/scheduler/scheduler_api.py create mode 100644 source/idea/idea-data-model/src/ideadatamodel/scheduler/scheduler_model.py create mode 100644 source/idea/idea-data-model/src/ideadatamodel/user_input/__init__.py create mode 100644 source/idea/idea-data-model/src/ideadatamodel/user_input/user_input_api.py create mode 100644 source/idea/idea-data-model/src/ideadatamodel/user_input/user_input_model.py create mode 100644 source/idea/idea-data-model/src/ideadatamodel/virtual_desktop/__init__.py create mode 100644 source/idea/idea-data-model/src/ideadatamodel/virtual_desktop/virtual_desktop_api.py create mode 100644 source/idea/idea-data-model/src/ideadatamodel/virtual_desktop/virtual_desktop_model.py create mode 100644 source/idea/idea-data-model/src/ideadatamodel_meta/__init__.py create mode 100644 source/idea/idea-data-model/src/setup.py create mode 100644 source/idea/idea-dcv-connection-gateway/images/favicon.ico create mode 100644 source/idea/idea-dcv-connection-gateway/index.html create mode 100644 source/idea/idea-scheduler/resources/api/api_doc.yml create mode 100644 source/idea/idea-scheduler/resources/openpbs/hooks/openpbs_hook_handler.py create mode 100644 source/idea/idea-scheduler/resources/opensearch/template_jobs.json create mode 100644 source/idea/idea-scheduler/resources/opensearch/template_nodes.json create mode 100644 source/idea/idea-scheduler/resources/scripts/license_check.py create mode 100644 source/idea/idea-scheduler/resources/scripts/send_logs_s3.sh create mode 100644 source/idea/idea-scheduler/src/ideascheduler/__init__.py create mode 100644 source/idea/idea-scheduler/src/ideascheduler/app/__init__.py create mode 100644 source/idea/idea-scheduler/src/ideascheduler/app/api/__init__.py create mode 100644 source/idea/idea-scheduler/src/ideascheduler/app/api/opepbs_api.py create mode 100644 source/idea/idea-scheduler/src/ideascheduler/app/api/scheduler_admin_api.py create mode 100644 source/idea/idea-scheduler/src/ideascheduler/app/api/scheduler_api.py create mode 100644 source/idea/idea-scheduler/src/ideascheduler/app/api/scheduler_api_invoker.py create mode 100644 source/idea/idea-scheduler/src/ideascheduler/app/app_context.py create mode 100644 source/idea/idea-scheduler/src/ideascheduler/app/app_main.py create mode 100644 source/idea/idea-scheduler/src/ideascheduler/app/app_protocols.py create mode 100644 source/idea/idea-scheduler/src/ideascheduler/app/applications/__init__.py create mode 100644 source/idea/idea-scheduler/src/ideascheduler/app/applications/hpc_applications_dao.py create mode 100644 source/idea/idea-scheduler/src/ideascheduler/app/applications/hpc_applications_service.py create mode 100644 source/idea/idea-scheduler/src/ideascheduler/app/aws/__init__.py create mode 100644 source/idea/idea-scheduler/src/ideascheduler/app/aws/aws_bugets_helper.py create mode 100644 source/idea/idea-scheduler/src/ideascheduler/app/aws/ec2_service_quota_helper.py create mode 100644 source/idea/idea-scheduler/src/ideascheduler/app/aws/instance_cache.py create mode 100644 source/idea/idea-scheduler/src/ideascheduler/app/aws/instance_monitor.py create mode 100644 source/idea/idea-scheduler/src/ideascheduler/app/aws/pricing_helper.py create mode 100644 source/idea/idea-scheduler/src/ideascheduler/app/documents/__init__.py create mode 100644 source/idea/idea-scheduler/src/ideascheduler/app/documents/document_store.py create mode 100644 source/idea/idea-scheduler/src/ideascheduler/app/licenses/__init__.py create mode 100644 source/idea/idea-scheduler/src/ideascheduler/app/licenses/license_resources_dao.py create mode 100644 source/idea/idea-scheduler/src/ideascheduler/app/licenses/license_service.py create mode 100644 source/idea/idea-scheduler/src/ideascheduler/app/metrics/__init__.py create mode 100644 source/idea/idea-scheduler/src/ideascheduler/app/notifications/__init__.py create mode 100644 source/idea/idea-scheduler/src/ideascheduler/app/notifications/job_notifications.py create mode 100644 source/idea/idea-scheduler/src/ideascheduler/app/provisioning/__init__.py create mode 100644 source/idea/idea-scheduler/src/ideascheduler/app/provisioning/job_monitor/__init__.py create mode 100644 source/idea/idea-scheduler/src/ideascheduler/app/provisioning/job_monitor/finished_job_processor.py create mode 100644 source/idea/idea-scheduler/src/ideascheduler/app/provisioning/job_monitor/job_cache.py create mode 100644 source/idea/idea-scheduler/src/ideascheduler/app/provisioning/job_monitor/job_monitor.py create mode 100644 source/idea/idea-scheduler/src/ideascheduler/app/provisioning/job_monitor/job_submission_tracker.py create mode 100644 source/idea/idea-scheduler/src/ideascheduler/app/provisioning/job_provisioner/__init__.py create mode 100644 source/idea/idea-scheduler/src/ideascheduler/app/provisioning/job_provisioner/batch_capacity_helper.py create mode 100644 source/idea/idea-scheduler/src/ideascheduler/app/provisioning/job_provisioner/cloudformation_stack_builder.py create mode 100644 source/idea/idea-scheduler/src/ideascheduler/app/provisioning/job_provisioner/job_provisioner.py create mode 100644 source/idea/idea-scheduler/src/ideascheduler/app/provisioning/job_provisioner/job_provisioning_util.py create mode 100644 source/idea/idea-scheduler/src/ideascheduler/app/provisioning/job_provisioning_queue/__init__.py create mode 100644 source/idea/idea-scheduler/src/ideascheduler/app/provisioning/job_provisioning_queue/hpc_queue_profiles_dao.py create mode 100644 source/idea/idea-scheduler/src/ideascheduler/app/provisioning/job_provisioning_queue/hpc_queue_profiles_service.py create mode 100644 source/idea/idea-scheduler/src/ideascheduler/app/provisioning/job_provisioning_queue/job_provisioning_queue.py create mode 100644 source/idea/idea-scheduler/src/ideascheduler/app/provisioning/node_monitor/__init__.py create mode 100644 source/idea/idea-scheduler/src/ideascheduler/app/provisioning/node_monitor/node_house_keeper.py create mode 100644 source/idea/idea-scheduler/src/ideascheduler/app/provisioning/node_monitor/node_monitor.py create mode 100644 source/idea/idea-scheduler/src/ideascheduler/app/scheduler/__init__.py create mode 100644 source/idea/idea-scheduler/src/ideascheduler/app/scheduler/job_param_builder.py create mode 100644 source/idea/idea-scheduler/src/ideascheduler/app/scheduler/lsf/__init__.py create mode 100644 source/idea/idea-scheduler/src/ideascheduler/app/scheduler/openpbs/__init__.py create mode 100644 source/idea/idea-scheduler/src/ideascheduler/app/scheduler/openpbs/openpbs_api.py create mode 100644 source/idea/idea-scheduler/src/ideascheduler/app/scheduler/openpbs/openpbs_api_invocation_context.py create mode 100644 source/idea/idea-scheduler/src/ideascheduler/app/scheduler/openpbs/openpbs_constants.py create mode 100644 source/idea/idea-scheduler/src/ideascheduler/app/scheduler/openpbs/openpbs_converter.py create mode 100644 source/idea/idea-scheduler/src/ideascheduler/app/scheduler/openpbs/openpbs_model.py create mode 100644 source/idea/idea-scheduler/src/ideascheduler/app/scheduler/openpbs/openpbs_qselect.py create mode 100644 source/idea/idea-scheduler/src/ideascheduler/app/scheduler/openpbs/openpbs_qstat.py create mode 100644 source/idea/idea-scheduler/src/ideascheduler/app/scheduler/openpbs/openpbs_scheduler.py create mode 100644 source/idea/idea-scheduler/src/ideascheduler/app/scheduler/slurm/__init__.py create mode 100644 source/idea/idea-scheduler/src/ideascheduler/app/scheduler/soca_scheduler.py create mode 100644 source/idea/idea-scheduler/src/ideascheduler/app/scheduler_app.py create mode 100644 source/idea/idea-scheduler/src/ideascheduler/app/scheduler_default_settings.py create mode 100644 source/idea/idea-scheduler/src/ideascheduler/cli/__init__.py create mode 100644 source/idea/idea-scheduler/src/ideascheduler/cli/ami_builder.py create mode 100644 source/idea/idea-scheduler/src/ideascheduler/cli/cli_main.py create mode 100644 source/idea/idea-scheduler/src/ideascheduler/cli/jobs.py create mode 100644 source/idea/idea-scheduler/src/ideascheduler/cli/logs.py create mode 100644 source/idea/idea-scheduler/src/ideascheduler/cli/module.py create mode 100644 source/idea/idea-scheduler/src/ideascheduler/cli/nodes.py create mode 100644 source/idea/idea-scheduler/src/ideascheduler_meta/__init__.py create mode 100644 source/idea/idea-scheduler/src/setup.py create mode 100644 source/idea/idea-scheduler/tests/conftest.py create mode 100644 source/idea/idea-scheduler/tests/test_cloudformation_stack_builder.py create mode 100644 source/idea/idea-scheduler/tests/test_job_param_builder.py create mode 100644 source/idea/idea-sdk/src/MANIFEST.in create mode 100644 source/idea/idea-sdk/src/ideasdk/__init__.py create mode 100644 source/idea/idea-sdk/src/ideasdk/analytics/__init__.py create mode 100644 source/idea/idea-sdk/src/ideasdk/analytics/analytics_service.py create mode 100644 source/idea/idea-sdk/src/ideasdk/api/__init__.py create mode 100644 source/idea/idea-sdk/src/ideasdk/api/api_invocation_context.py create mode 100644 source/idea/idea-sdk/src/ideasdk/api/base_api.py create mode 100644 source/idea/idea-sdk/src/ideasdk/app/__init__.py create mode 100644 source/idea/idea-sdk/src/ideasdk/app/soca_app.py create mode 100644 source/idea/idea-sdk/src/ideasdk/app/soca_app_api.py create mode 100644 source/idea/idea-sdk/src/ideasdk/app/soca_app_commands.py create mode 100644 source/idea/idea-sdk/src/ideasdk/artwork/__init__.py create mode 100644 source/idea/idea-sdk/src/ideasdk/auth/__init__.py create mode 100644 source/idea/idea-sdk/src/ideasdk/auth/token_service.py create mode 100644 source/idea/idea-sdk/src/ideasdk/aws/__init__.py create mode 100644 source/idea/idea-sdk/src/ideasdk/aws/aws_client_provider.py create mode 100644 source/idea/idea-sdk/src/ideasdk/aws/aws_endpoints.json create mode 100644 source/idea/idea-sdk/src/ideasdk/aws/aws_endpoints.py create mode 100644 source/idea/idea-sdk/src/ideasdk/aws/aws_resources.py create mode 100644 source/idea/idea-sdk/src/ideasdk/aws/aws_util.py create mode 100644 source/idea/idea-sdk/src/ideasdk/aws/ec2_instance_types_db.py create mode 100644 source/idea/idea-sdk/src/ideasdk/aws/iam_permission_util.py create mode 100644 source/idea/idea-sdk/src/ideasdk/aws/instance_metadata_util.py create mode 100644 source/idea/idea-sdk/src/ideasdk/aws/opensearch/__init__.py create mode 100644 source/idea/idea-sdk/src/ideasdk/aws/opensearch/aws_opensearch_client.py create mode 100644 source/idea/idea-sdk/src/ideasdk/aws/opensearch/opensearch_filters.py create mode 100644 source/idea/idea-sdk/src/ideasdk/aws/opensearch/opensearchable_db.py create mode 100644 source/idea/idea-sdk/src/ideasdk/bootstrap/__init__.py create mode 100644 source/idea/idea-sdk/src/ideasdk/bootstrap/bootstrap_package_builder.py create mode 100644 source/idea/idea-sdk/src/ideasdk/bootstrap/bootstrap_userdata_builder.py create mode 100644 source/idea/idea-sdk/src/ideasdk/bootstrap/bootstrap_utils.py create mode 100644 source/idea/idea-sdk/src/ideasdk/cache/__init__.py create mode 100644 source/idea/idea-sdk/src/ideasdk/cache/soca_cache.py create mode 100644 source/idea/idea-sdk/src/ideasdk/client/__init__.py create mode 100644 source/idea/idea-sdk/src/ideasdk/client/accounts_client.py create mode 100644 source/idea/idea-sdk/src/ideasdk/client/evdi_client.py create mode 100644 source/idea/idea-sdk/src/ideasdk/client/notifications_async_client.py create mode 100644 source/idea/idea-sdk/src/ideasdk/client/projects_client.py create mode 100644 source/idea/idea-sdk/src/ideasdk/client/soca_client.py create mode 100644 source/idea/idea-sdk/src/ideasdk/clustering/__init__.py create mode 100644 source/idea/idea-sdk/src/ideasdk/clustering/leader_election.py create mode 100644 source/idea/idea-sdk/src/ideasdk/clustering/leader_election_constants.py create mode 100644 source/idea/idea-sdk/src/ideasdk/common/__init__.py create mode 100644 source/idea/idea-sdk/src/ideasdk/common/threading/__init__.py create mode 100644 source/idea/idea-sdk/src/ideasdk/compat/__init__.py create mode 100644 source/idea/idea-sdk/src/ideasdk/compat/windows.py create mode 100644 source/idea/idea-sdk/src/ideasdk/config/__init__.py create mode 100644 source/idea/idea-sdk/src/ideasdk/config/cluster_config.py create mode 100644 source/idea/idea-sdk/src/ideasdk/config/cluster_config_db.py create mode 100644 source/idea/idea-sdk/src/ideasdk/config/soca_config.py create mode 100644 source/idea/idea-sdk/src/ideasdk/context/__init__.py create mode 100644 source/idea/idea-sdk/src/ideasdk/context/arn_builder.py create mode 100644 source/idea/idea-sdk/src/ideasdk/context/bootstrap_context.py create mode 100644 source/idea/idea-sdk/src/ideasdk/context/soca_cli_context.py create mode 100644 source/idea/idea-sdk/src/ideasdk/context/soca_context.py create mode 100644 source/idea/idea-sdk/src/ideasdk/context/validators/__init__.py create mode 100644 source/idea/idea-sdk/src/ideasdk/distributed_lock/__init__.py create mode 100644 source/idea/idea-sdk/src/ideasdk/distributed_lock/distributed_lock.py create mode 100644 source/idea/idea-sdk/src/ideasdk/dynamodb/__init__.py create mode 100644 source/idea/idea-sdk/src/ideasdk/dynamodb/dynamodb_stream_subscriber.py create mode 100644 source/idea/idea-sdk/src/ideasdk/dynamodb/dynamodb_stream_subscription.py create mode 100644 source/idea/idea-sdk/src/ideasdk/filesystem/__init__.py create mode 100644 source/idea/idea-sdk/src/ideasdk/filesystem/filebrowser_api.py create mode 100644 source/idea/idea-sdk/src/ideasdk/filesystem/filesystem_helper.py create mode 100644 source/idea/idea-sdk/src/ideasdk/logging/__init__.py create mode 100644 source/idea/idea-sdk/src/ideasdk/logging/console_logger.py create mode 100644 source/idea/idea-sdk/src/ideasdk/logging/soca_logging.py create mode 100644 source/idea/idea-sdk/src/ideasdk/metrics/__init__.py create mode 100644 source/idea/idea-sdk/src/ideasdk/metrics/base_accumulator.py create mode 100644 source/idea/idea-sdk/src/ideasdk/metrics/base_metrics.py create mode 100644 source/idea/idea-sdk/src/ideasdk/metrics/cloudwatch/__init__.py create mode 100644 source/idea/idea-sdk/src/ideasdk/metrics/cloudwatch/cloudwatch_agent_config.py create mode 100644 source/idea/idea-sdk/src/ideasdk/metrics/cloudwatch/cloudwatch_metrics.py create mode 100644 source/idea/idea-sdk/src/ideasdk/metrics/cloudwatch/templates/amazon-cloudwatch-agent-linux.yml create mode 100644 source/idea/idea-sdk/src/ideasdk/metrics/cloudwatch/templates/amazon-cloudwatch-agent-windows.yml create mode 100644 source/idea/idea-sdk/src/ideasdk/metrics/fast_write_counter.py create mode 100644 source/idea/idea-sdk/src/ideasdk/metrics/metric_timer.py create mode 100644 source/idea/idea-sdk/src/ideasdk/metrics/metrics_provider_factory.py create mode 100644 source/idea/idea-sdk/src/ideasdk/metrics/metrics_service.py create mode 100644 source/idea/idea-sdk/src/ideasdk/metrics/null_metrics_provider.py create mode 100644 source/idea/idea-sdk/src/ideasdk/metrics/prometheus/__init__.py create mode 100644 source/idea/idea-sdk/src/ideasdk/metrics/prometheus/prometheus_config.py create mode 100644 source/idea/idea-sdk/src/ideasdk/metrics/prometheus/prometheus_metrics.py create mode 100644 source/idea/idea-sdk/src/ideasdk/metrics/prometheus/templates/prometheus-linux.yml create mode 100644 source/idea/idea-sdk/src/ideasdk/metrics/prometheus/templates/prometheus-windows.yml create mode 100644 source/idea/idea-sdk/src/ideasdk/notice.py create mode 100644 source/idea/idea-sdk/src/ideasdk/protocols/__init__.py create mode 100644 source/idea/idea-sdk/src/ideasdk/protocols/empty.py create mode 100644 source/idea/idea-sdk/src/ideasdk/pubsub/__init__.py create mode 100644 source/idea/idea-sdk/src/ideasdk/pubsub/soca_pubsub.py create mode 100644 source/idea/idea-sdk/src/ideasdk/server/__init__.py create mode 100644 source/idea/idea-sdk/src/ideasdk/server/cors.py create mode 100644 source/idea/idea-sdk/src/ideasdk/server/options.py create mode 100644 source/idea/idea-sdk/src/ideasdk/server/sanic_config.py create mode 100644 source/idea/idea-sdk/src/ideasdk/server/soca_server.py create mode 100644 source/idea/idea-sdk/src/ideasdk/service/__init__.py create mode 100644 source/idea/idea-sdk/src/ideasdk/service/soca_service.py create mode 100644 source/idea/idea-sdk/src/ideasdk/service/soca_service_registry.py create mode 100644 source/idea/idea-sdk/src/ideasdk/shell/__init__.py create mode 100644 source/idea/idea-sdk/src/ideasdk/shell/log_tail.py create mode 100644 source/idea/idea-sdk/src/ideasdk/shell/shell_invoker.py create mode 100644 source/idea/idea-sdk/src/ideasdk/thread_pool/__init__.py create mode 100644 source/idea/idea-sdk/src/ideasdk/thread_pool/idea_thread.py create mode 100644 source/idea/idea-sdk/src/ideasdk/thread_pool/idea_threadpool_service.py create mode 100644 source/idea/idea-sdk/src/ideasdk/user_input/__init__.py create mode 100644 source/idea/idea-sdk/src/ideasdk/user_input/framework.py create mode 100644 source/idea/idea-sdk/src/ideasdk/utils/__init__.py create mode 100644 source/idea/idea-sdk/src/ideasdk/utils/datetime_utils.py create mode 100644 source/idea/idea-sdk/src/ideasdk/utils/environment_utils.py create mode 100644 source/idea/idea-sdk/src/ideasdk/utils/group_name_helper.py create mode 100644 source/idea/idea-sdk/src/ideasdk/utils/jinja2_utils.py create mode 100644 source/idea/idea-sdk/src/ideasdk/utils/module_metadata.py create mode 100644 source/idea/idea-sdk/src/ideasdk/utils/utils.py create mode 100644 source/idea/idea-sdk/src/ideasdk_meta/__init__.py create mode 100644 source/idea/idea-sdk/src/setup.py create mode 100644 source/idea/idea-sdk/tests/conftest.py create mode 100644 source/idea/idea-sdk/tests/test_api_invocation_context.py create mode 100644 source/idea/idea-sdk/tests/test_file_system_helper.py create mode 100644 source/idea/idea-sdk/tests/test_server.py create mode 100644 source/idea/idea-sdk/tests/test_utils.py create mode 100644 source/idea/idea-test-utils/src/ideatestutils/__init__.py create mode 100644 source/idea/idea-test-utils/src/ideatestutils/aws/__init__.py create mode 100644 source/idea/idea-test-utils/src/ideatestutils/aws/mock_instance_types.py create mode 100644 source/idea/idea-test-utils/src/ideatestutils/aws/templates/c5.large.json create mode 100644 source/idea/idea-test-utils/src/ideatestutils/aws/templates/c5.xlarge.json create mode 100644 source/idea/idea-test-utils/src/ideatestutils/aws/templates/c5n.18xlarge.json create mode 100644 source/idea/idea-test-utils/src/ideatestutils/aws/templates/t3.micro.json create mode 100644 source/idea/idea-test-utils/src/ideatestutils/config/__init__.py create mode 100644 source/idea/idea-test-utils/src/ideatestutils/config/mock_config.py create mode 100644 source/idea/idea-test-utils/src/ideatestutils/config/templates/default.yml create mode 100644 source/idea/idea-test-utils/src/ideatestutils/dynamodb/__init__.py create mode 100644 source/idea/idea-test-utils/src/ideatestutils/dynamodb/dynamodb_local.py create mode 100644 source/idea/idea-test-utils/src/ideatestutils/idea_test_props.py create mode 100644 source/idea/idea-test-utils/src/ideatestutils/projects/__init__.py create mode 100644 source/idea/idea-test-utils/src/ideatestutils/projects/mock_projects.py create mode 100644 source/idea/idea-test-utils/src/ideatestutils/projects/templates/default.json create mode 100644 source/idea/idea-virtual-desktop-controller/resources/api/api_doc.yml create mode 100644 source/idea/idea-virtual-desktop-controller/resources/base-permission-profile-config.yaml create mode 100644 source/idea/idea-virtual-desktop-controller/resources/base-software-stack-config.yaml create mode 100644 source/idea/idea-virtual-desktop-controller/resources/dcv_broker_swagger_client.yaml create mode 100644 source/idea/idea-virtual-desktop-controller/resources/opensearch/session_entry_template.yml create mode 100644 source/idea/idea-virtual-desktop-controller/resources/opensearch/session_permission_entry_template.yml create mode 100644 source/idea/idea-virtual-desktop-controller/resources/opensearch/software_stack_entry_template.yml create mode 100644 source/idea/idea-virtual-desktop-controller/resources/permission-config.yaml create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/__init__.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/__init__.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/api/__init__.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/api/virtual_desktop_admin_api.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/api/virtual_desktop_api.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/api/virtual_desktop_api_invoker.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/api/virtual_desktop_dcv_api.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/api/virtual_desktop_user_api.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/api/virtual_desktop_utils_api.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/app_context.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/app_main.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/app_protocols.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/clients/__init__.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/clients/dcv_broker_client/__init__.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/clients/dcv_broker_client/dcv_broker_client.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/clients/dcv_broker_client/dcv_broker_client_utils.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/clients/dcvssmswaggerclient/__init__.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/clients/dcvssmswaggerclient/api/__init__.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/clients/dcvssmswaggerclient/api/get_session_connection_data_api.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/clients/dcvssmswaggerclient/api/servers_api.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/clients/dcvssmswaggerclient/api/session_permissions_api.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/clients/dcvssmswaggerclient/api/sessions_api.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/clients/dcvssmswaggerclient/api_client.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/clients/dcvssmswaggerclient/configuration.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/clients/dcvssmswaggerclient/models/__init__.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/clients/dcvssmswaggerclient/models/aws.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/clients/dcvssmswaggerclient/models/close_server_request_data.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/clients/dcvssmswaggerclient/models/close_server_successful_response.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/clients/dcvssmswaggerclient/models/close_server_unsuccessful_response.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/clients/dcvssmswaggerclient/models/close_servers_response.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/clients/dcvssmswaggerclient/models/cpu_info.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/clients/dcvssmswaggerclient/models/cpu_load_average.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/clients/dcvssmswaggerclient/models/create_session_request_data.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/clients/dcvssmswaggerclient/models/create_sessions_response.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/clients/dcvssmswaggerclient/models/delete_session_request_data.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/clients/dcvssmswaggerclient/models/delete_session_successful_response.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/clients/dcvssmswaggerclient/models/delete_session_unsuccessful_response.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/clients/dcvssmswaggerclient/models/delete_sessions_response.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/clients/dcvssmswaggerclient/models/describe_servers_request_data.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/clients/dcvssmswaggerclient/models/describe_servers_response.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/clients/dcvssmswaggerclient/models/describe_sessions_request_data.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/clients/dcvssmswaggerclient/models/describe_sessions_response.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/clients/dcvssmswaggerclient/models/endpoint.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/clients/dcvssmswaggerclient/models/get_session_connection_data_response.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/clients/dcvssmswaggerclient/models/get_session_screenshot_request_data.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/clients/dcvssmswaggerclient/models/get_session_screenshot_successful_response.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/clients/dcvssmswaggerclient/models/get_session_screenshot_unsuccessful_response.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/clients/dcvssmswaggerclient/models/get_session_screenshots_response.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/clients/dcvssmswaggerclient/models/gpu.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/clients/dcvssmswaggerclient/models/host.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/clients/dcvssmswaggerclient/models/key_value_pair.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/clients/dcvssmswaggerclient/models/logged_in_user.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/clients/dcvssmswaggerclient/models/memory.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/clients/dcvssmswaggerclient/models/open_server_request_data.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/clients/dcvssmswaggerclient/models/open_server_successful_response.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/clients/dcvssmswaggerclient/models/open_server_unsuccessful_response.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/clients/dcvssmswaggerclient/models/open_servers_response.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/clients/dcvssmswaggerclient/models/os.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/clients/dcvssmswaggerclient/models/server.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/clients/dcvssmswaggerclient/models/session.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/clients/dcvssmswaggerclient/models/session_screenshot.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/clients/dcvssmswaggerclient/models/session_screenshot_image.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/clients/dcvssmswaggerclient/models/swap.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/clients/dcvssmswaggerclient/models/unsuccessful_create_session_request_data.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/clients/dcvssmswaggerclient/models/update_session_permissions_request_data.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/clients/dcvssmswaggerclient/models/update_session_permissions_response.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/clients/dcvssmswaggerclient/models/update_session_permissions_successful_response.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/clients/dcvssmswaggerclient/models/update_session_permissions_unsuccessful_response.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/clients/dcvssmswaggerclient/rest.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/clients/events_client/__init__.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/clients/events_client/events_client.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/events/__init__.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/events/events_utils.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/events/handlers/__init__.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/events/handlers/base_event_handler.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/events/handlers/db_entry_event_handlers/__init__.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/events/handlers/db_entry_event_handlers/base_db_event_handler.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/events/handlers/db_entry_event_handlers/db_entry_created_event_handler.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/events/handlers/db_entry_event_handlers/db_entry_deleted_event_handler.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/events/handlers/db_entry_event_handlers/db_entry_updated_event_handler.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/events/handlers/dcv_broker_userdata_execution_complete_event_handler.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/events/handlers/dcv_host_event_handlers/__init__.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/events/handlers/dcv_host_event_handlers/dcv_host_ready_event_handler.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/events/handlers/dcv_host_event_handlers/dcv_host_reboot_complete_event_handler.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/events/handlers/ec2_state_change_event_handler.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/events/handlers/idea_session_permissions_event_handlers/__init__.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/events/handlers/idea_session_permissions_event_handlers/idea_session_permissions_enforce_event_handler.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/events/handlers/idea_session_permissions_event_handlers/idea_session_permissions_update_event_handler.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/events/handlers/idea_session_software_stack_update_event_handler.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/events/handlers/idea_session_state_event_handlers/__init__.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/events/handlers/idea_session_state_event_handlers/idea_session_scheduled_resume_event_handler.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/events/handlers/idea_session_state_event_handlers/idea_session_scheduled_stop_event_handler.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/events/handlers/idea_session_state_event_handlers/idea_session_terminate_event_handler.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/events/handlers/scheduled_event_handler.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/events/handlers/ssm_commands_progress_event_handlers/__init__.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/events/handlers/ssm_commands_progress_event_handlers/disable_userdata_windows_command_progress_event_handler.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/events/handlers/ssm_commands_progress_event_handlers/enable_userdata_windows_command_progress_event_handler.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/events/handlers/ssm_commands_progress_event_handlers/idea_resume_session_command_progress_event_handler.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/events/handlers/ssm_commands_progress_event_handlers/idea_session_cpu_utilization_command_progress_event_handler.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/events/handlers/user_management_event_handlers/__init__.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/events/handlers/user_management_event_handlers/user_created_event_handler.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/events/handlers/user_management_event_handlers/user_disabled_event_handler.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/events/handlers/validate_dcv_session_event_handlers/__init__.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/events/handlers/validate_dcv_session_event_handlers/validate_dcv_session_creation_event_handler.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/events/handlers/validate_dcv_session_event_handlers/validate_dcv_session_deletion_event_handler.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/events/handlers/validate_software_stack_event_handler.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/events/service/__init__.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/events/service/controller_queue_monitor_service.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/events/service/event_queue_monitoring_service.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/events/service/events_handler_thread.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/permission_profiles/__init__.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/permission_profiles/constants.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/permission_profiles/virtual_desktop_permission_profile_db.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/schedules/__init__.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/schedules/constants.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/schedules/virtual_desktop_schedule_db.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/schedules/virtual_desktop_schedule_utils.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/servers/__init__.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/servers/constants.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/servers/virtual_desktop_server_db.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/servers/virtual_desktop_server_utils.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/session_permissions/__init__.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/session_permissions/constants.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/session_permissions/virtual_desktop_session_permission_db.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/session_permissions/virtual_desktop_session_permission_utils.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/sessions/__init__.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/sessions/constants.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/sessions/virtual_desktop_session_counters_db.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/sessions/virtual_desktop_session_db.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/sessions/virtual_desktop_session_utils.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/software_stacks/__init__.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/software_stacks/constants.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/software_stacks/virtual_desktop_software_stack_db.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/software_stacks/virtual_desktop_software_stack_utils.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/ssm_commands/__init__.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/ssm_commands/constants.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/ssm_commands/virtual_desktop_ssm_commands_db.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/ssm_commands/virtual_desktop_ssm_commands_utils.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/virtual_desktop_controller_app.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/virtual_desktop_controller_utils.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/app/virtual_desktop_notifiable_db.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/cli/__init__.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/cli/cli_main.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/cli/logs.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/cli/module.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/cli/sessions.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller/cli/software_stacks.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/ideavirtualdesktopcontroller_meta/__init__.py create mode 100644 source/idea/idea-virtual-desktop-controller/src/setup.py create mode 100644 source/idea/idea-virtual-desktop-controller/tests/conftest.py create mode 100644 source/idea/idea-virtual-desktop-controller/tests/test_example.py create mode 100644 tasks/__init__.py create mode 100644 tasks/admin.py create mode 100644 tasks/apispec.py create mode 100644 tasks/build.py create mode 100644 tasks/clean.py create mode 100644 tasks/cli.py create mode 100644 tasks/devtool.py create mode 100644 tasks/docker.py create mode 100644 tasks/idea.py create mode 100644 tasks/package.py create mode 100644 tasks/release.py create mode 100644 tasks/requirements.py create mode 100644 tasks/tests.py create mode 100644 tasks/tools/__init__.py create mode 100644 tasks/tools/build_tool.py create mode 100644 tasks/tools/clean_tool.py create mode 100644 tasks/tools/open_api_tool.py create mode 100644 tasks/tools/package_tool.py create mode 100644 tasks/tools/typings_generator.py create mode 100644 tasks/web_portal.py diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 00000000..3056f60e --- /dev/null +++ b/.editorconfig @@ -0,0 +1,13 @@ +root = true + +[*] +indent_style = space +indent_size = 4 +insert_final_newline = true +trim_trailing_whitespace = true +end_of_line = lf +charset = utf-8 +max_line_length = 340 + +[*.{yml,yaml,json,js,css,html,jinja,jinja2,bat,sh,ps1,template}] +indent_size = 2 diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..eb656fc0 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,3 @@ +*.css linguist-vendored +*.scss linguist-vendored +*.js linguist-vendored \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 00000000..faee85a9 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,42 @@ +--- +name: Bug report +about: Create a report to help us improve +title: '' +labels: bug +assignees: '' + +--- + +**Describe the bug** +A clear and concise description of what the bug is. + +**To Reproduce** +Steps to reproduce the behavior. + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Please complete the following information about the solution:** +- [ ] Version: [e.g. v1.0.0] + +To get the version of the solution, you can look at the description of the created CloudFormation stack. For example, "_(SO0021) - Video On Demand workflow with AWS Step Functions, MediaConvert, MediaPackage, S3, CloudFront and DynamoDB. Version **v5.0.0**_". If the description does not contain the version information, you can look at the mappings section of the template: + +```yaml +Mappings: + SourceCode: + General: + S3Bucket: "solutions" + KeyPrefix: "scale-out-computing-on-aws/v2.0.0" +``` + +- [ ] Region: [e.g. us-east-1] +- [ ] Was the solution modified from the version published on this repository? +- [ ] If the answer to the previous question was yes, are the changes available on GitHub? +- [ ] Have you checked your [service quotas](https://docs.aws.amazon.com/general/latest/gr/aws_service_limits.html) for the sevices this solution uses? +- [ ] Were there any errors in the CloudWatch Logs? + +**Screenshots** +If applicable, add screenshots to help explain your problem (please **DO NOT include sensitive information**). + +**Additional context** +Add any other context about the problem here. \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 00000000..d7954026 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,17 @@ +--- +name: Feature request +about: Suggest an idea for this solution +title: '' +labels: enhancement +assignees: '' + +--- + +**Is your feature request related to a problem? Please describe.** +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +**Describe the feature you'd like** +A clear and concise description of what you want to happen. + +**Additional context** +Add any other context or screenshots about the feature request here. \ No newline at end of file diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 00000000..283f6872 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,5 @@ +*Issue #, if available:* + +*Description of changes:* + +By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice. \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..834f046f --- /dev/null +++ b/.gitignore @@ -0,0 +1,193 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# IDEA Specific +deployment/ecr/idea-administrator/*.tar.gz +open-source/ +deployment/global-s3-assets/ +deployment/regional-s3-assets/ + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +tests/modules/vdc/outputs/ + +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env/ +.venv/ +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ + +# IDE and Editor artifacts # +*.bbprojectd +.idea/* +.idea +*.iml + +# Temporary Files # +tmp_* +cfg.tmp.json + +# OS generated files # +.DS_Store +.DS_Store? + +# VScode +.vscode/ + +# Node +**/node_modules/** + + +## Local Development + +scratch/ +tasks/local_dev.py +build-dev-mode/ +codebuild_build.sh + +# Locally stored "Eclipse launch configurations" +*.launch + +# PyDev specific (Python IDE for Eclipse) +*.pydevproject + +# CDT-specific (C/C++ Development Tooling) +.cproject + +# CDT- autotools +.autotools + +# Java annotation processor (APT) +.factorypath + +# PDT-specific (PHP Development Tools) +.buildpath + +# sbteclipse plugin +.target + +# Tern plugin +.tern-project + +# TeXlipse plugin +.texlipse + +# STS (Spring Tool Suite) +.springBeans + +# Code Recommenders +.recommenders/ + +# Annotation Processing +.apt_generated/ +.apt_generated_test/ + +# Scala IDE specific (Scala & Java development for Eclipse) +.cache-main +.scala_dependencies +.worksheet + +# Uncomment this line if you wish to ignore the project description file. +# Typically, this file would be tracked if it contains build/dependency configurations: +.project + +# Below path has been safelisted to allow NICE DCV Connection Gateway to work. +# It contains the js script that validates certificates and enables connection establishing. +!source/idea/idea-dcv-connection-gateway/js/lib/ + +# Test outputs +integration-test-results/ diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 00000000..1ed82796 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,202 @@ +# Change Log +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [3.0.0] - 2022-11-22 +### Added +- Modular architecture with a major rewrite of all components. +- Administrator App for managing and deploying IDEA clusters. +- New Module: Cluster +- New Module: Metrics & Monitoring +- New Module: Analytics +- New Module: Directory Service +- New Module: Identity Provider +- New Module: Shared Storage +- New Module: eVDI +- New Module: Scale-Out Computing +- New Module: Bastion Host + +## [2.7.3] - 2022-08-20 +### Changed +- Bumped Lambda Python Runtime to 3.7 + +## [2.7.2] - 2022-04-25 +### Changed +- Fix node version to v8.7.0 (later versions need updated versions of GLIBC that are not available for AL2/CentOS7/RHEL7) +- Update RHEL7 AMI IDs to RHEL7.9 +- Update AL2 AMI IDs + +## [2.7.1] - 2022-02-15 +### Changed +- NodeJS/npm is now managed via NVM (#64: Contributor @cfsnate) +- Fixed IAM policies required to install SOCA and added support for cdk boostrap (#64: Contributor @cfsnate) +- More consistent way to install EPEL repository across distros +- Better way to install SSM on the Scheduler host (similar to what we are already doing with ComputeNodes) +- Updated remote job submission to fix error with group ownership when using a remote input file +- DCV desktops now honor correct subnet when specified +- Fix issue causing installer to crash when using IPv6-only VPC subnets +- Fix logger issue on DCV instance lifecycle (#67, contributor @tammy-ruby-cherry) + +## [2.7.0] - 2021-11-18 +### Added +- SOCA installer is managed by CDK (https://aws.amazon.com/cdk/) +- Enabled full WSGI debug mode for SOCA Web UI +- Added support for WeightedCapacity enabling add_nodes.py to launch capacity based on vCPUs or cores +- CDK: Added support for Active Directory via AWS Directory Service +- CDK: Users can now re-use their existing AWS resources (VPC, subnets, security groups, FSxL, EFS, Directory Services ...) when installing SOCA +- CDK: Users can extend the base installer with their own code (see cdk_construct_user_customization) +- CDK: /apps & /data partition can now be configured to use EFS or FSxL as storage provider +- CDK: Users can now use your own CMK (Customer Managed Key) to encrypt your EFS, FSxL, EBS or SecretsManager +- CDK: Users can configure the number of NAT gateways to be deployed when installing a new cluster +- CDK: Users can customize your OpenSearch (formerly Elasticsearch) domain (number of nodes, type of instance) +- CDK: Users can configure the backup retention time (default to 7 days) +- CDK: Users can now deploy SOCA in private subnets only +- CDK: Added support for VPC endpoints creation +- Users can now specify up to 4 additional security groups for compute nodes assigned to their simulations +- Users can now specific a custom IAM instance profile for compute nodes assigned to their simulations +- Deprecated ldap_manager.py in favor of the native REST API +- Added a custom path for Windows DCV logs +- Name of the SOCA cluster is now accessible on the Web interface +- DCV session management is now available via REST API +- Customer EC2 AMI management is now available via REST API +- Added job-shared queue enabling multiple jobs to run on the same EC2 instance for jobs with similar requirements +- Desktops sessions are now tracked on OpenSearch (formerly Elasticsearch) via "soca_desktops" index + +### Changed +- Upgraded DCV to 2021.2 +- Upgraded EFA to 1.13.0 +- Upgraded OpenMPI to 4.1.1 +- Auto-Terminate stopped DCV instances now delete the associated cloudformation stack +- Fixed #55 (bug and bug fix: automatic hibernation (Linux desktops)) +- Prevent system accounts (ec2-user/centos) to submit jobs +- OpenMPI is now installed under /apps/openmpi +- Changed default OpenSearch (formerly Elasticsearch) indexes to "soca_jobs" and "soca_nodes" (previously "jobs" and "pbsnodes") + +## [2.6.1] - 2021-03-22 +### Added +- Added Name tag to EIPNat in Network.template +- Added support for Milan and Cape Town +- EBS volumes provisioned for DCV sessions (Windows/Linux) are now tagged properly +- Support for Graviton2 instances +- Ability to disable web APIs via @disabled decorator + +### Changed +- Updated EFA to 1.11.1 +- Updated Python 3.7.1 to Python 3.7.9 +- Update DCV version to 2020.2 +- Updated awscli, boto3, and botocore to support instances announced at Re:Invent 2020 +- Use new gp3 volumes instead of gp2 since they're more cost effective and provide 3000 IOPS baseline +- Removed SchedulerPublicIPAllocation from Scheduler.template as it's no longer used +- Updated CentOS, ALI2 and RHEL76 AMI IDs +- Instances with NVME instance store don't become unresponsive post-restart due to filesystem checks enforcement +- OpenSearch (formerly Elasticsearch) is now deployed in private subnets + +## [2.6.0] - 2020-10-29 +### Added +- Users can now launch Windows instances with DCV +- Users can now configure their DCV sessions based on their own schedule +- Users can stop/hibernate DCV sessions +- Users can change the hardware of their DCV sessions after the initial launch +- Admins can create DCV AMI with pre-configured applications +- Added support for DCV session storage. Upload/download data to SOCA directly from your DCV desktop (C:\storage-root for windows and $HOME/storage-root for linux) +- Admins can now prevent users to download the files via the web ui +- SOCA automatically enable/disable EFS provisioned throughput based on current I/O activity + +### Changed +- Removed deprecated `soca_aws_infos` hook +- Fixed an issue that caused the web interface to become unresponsive after an API reset +- Users can now easily import/export application profiles +- Fixed an issue that caused Nvidia Tesla drivers to be incorrectly installed on P3 instances +- Manual_build.py now automatically upload the installer to your S3 bucket +- Upgraded to PBS v20 +- Upgraded DCV to 2020.1-9012 + +## [2.5.0] - 2020-07-17 +### Added +- Support for Elastic MetricBeat +- Added HTTP REST API to interact with SOCA +- Users can now decide to restrict a job to Reserved Instances +- Revamped Web Interface + - Added filesystem explorer + - Users can upload files/folders via drag & drop interface + - Users can edit files directly on SOCA using a cloud text editor + - Users can now manage membership of their own LDAP group via web + - Users can now understand why they job is not started (eg: instance issue, misconfiguration, AWS limit, license limit) directly on the UI + - Users can kill their job via the web + - Admins can manage SOCA LDAP via web (create group, user, manage ownership and permissions) + - Admins can creates application profiles and let user submit job via web interface + - Ability to trigger Linux commands via HTML form +- Admins can now limit the number of running jobs per queue +- Admins can now limit the number of running instances per queue +- Admins can now specify the idle timeout value for any DCV sessions. Inactive DCV sessions will be automatically terminated after this period +- Job selection can now configured at queue level (FIFO or fair share) +- Dry run now supports vCpus limit +- Support for custom shells + +### Changed +- Updated Troposphere to 2.6.1 +- Updated EFA to 1.9.3 +- Updated Nice DCV to 2020.0-8428 +- Updated OpenSearch (formerly Elasticsearch) to 7.4 +- You can specify a name for your DCV sessions +- You can now specify custom AMI, base OS or storage options for your DCV sessions +- Project assigned to DCV jobs has been renamed to "remotedesktop" (previously "gui") +- Dispatcher script is now running every minute +- SOCA now deploys 2 instances for OpenSearch (formerly Elasticsearch) for high availability +- Users can now specify DEPLOYMENT_TYPE for their FSX for Lustre filesystems +- Users can specify PerUnitThroughput when FSx for Lustre deployment type is set to PERSISTENT +- DCV now supports G4 instance type (#24) +- X11 is now configured correctly for ALI 3D DCV session (#23) + +## [2.0.1] - 2020-04-20 +### Added +- Support for SpotFleet + +### Changed +- NVIDIA drivers are now automatically installed when a GPU instance is provisioned +- Deployed MATE Desktop for DCV for Amazon Linux 2 + +## [2.0.0] - 2020-03-18 +### Added +- Support for MixedInstancePolicy and InstanceDistribution +- Support for non-EBS optimized instances such as t2 +- Integration of AWS Session Manager +- Integration of AWS Backup +- Integration of AWS Cognito +- Integration of Troposphere +- Admins can now manage ACL (individual/LDAP groups) at queue level +- Admins can now restrict specific type/family of instance at queue level +- Admins can now prevent users to change specific EC2 parameters +- Users can now install SOCA using existing resources such as VPC, Security Groups ... +- Users now have the ability to retain EBS disks associated to a simulation for debugging purposes +- SOCA now prevent jobs to be submitted if .yaml configuration files are malformed +- Scheduler Root EBS is now tagged with cluster ID +- Scheduler Network Interface is now tagged with cluster ID +- Scheduler and Compute hosts are now sync with Chrony (Amazon Time Sync) +- Support for FSx for Lustre new Scratch2/Scratch1 and Persistent mode +- Added Compute nodes logs on EFS (/apps/soca//cluster_node_bootstrap/logs///*.log) for easy debugging + +### Changed +- Ignore installation if PBSPro is already configured on the AMI +- Fixed bug when stack name only use uppercase +- ComputeNode bootstrap scripts are now loaded from EFS +- Users can now open a SSH session using SSM Session Manager +- Processes are now automatically launched upon scheduler reboot +- Max Spot price now default to the OD price +- Default admin password now supports special characters +- Ulimit is now disabled by default on all compute nodes +- Dispatcher automatically append "s3://" if not present when using FSx For Lustre +- Updated default OpenSearch (formerly Elasticsearch) instance to m5.large to support encryption at rest +- SOCA libraries are now installed under /apps/soca/ location to support multi SOCA environments +- Web UI now display the reason when a DCV job can't be submitted +- Customers can now provision large number of EC2 hosts across multiple subnets using a single API call +- Smart detection of Placement Group requirement when using more than 1 subnet +- Added retry mechanism for some AWS API calls which throttled when provisioning > 1000 nodes in a single API call +- ALB Target Groups are now correctly deleted once the DCV sessions is terminated +- SOCA version is now displayed on the web interface +- Updated EFA version to 1.8.3 + +## [1.0.0] - 2019-11-20 +- Release Candidate diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 00000000..3b644668 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,4 @@ +## Code of Conduct +This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). +For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact +opensource-codeofconduct@amazon.com with any additional questions or comments. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000..5f3bcfa7 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,61 @@ +# Contributing Guidelines + +Thank you for your interest in contributing to our project. Whether it's a bug report, new feature, correction, or additional +documentation, we greatly value feedback and contributions from our community. + +Please read through this document before submitting any issues or pull requests to ensure we have all the necessary +information to effectively respond to your bug report or contribution. + + +## Reporting Bugs/Feature Requests + +We welcome you to use the GitHub issue tracker to report bugs or suggest features. + +When filing an issue, please check [existing open](https://github.com/awslabs/solution-for-scale-out-computing-on-aws/issues), or [recently closed](https://github.com/awslabs/solution-for-scale-out-computing-on-aws/issues?utf8=%E2%9C%93&q=is%3Aissue%20is%3Aclosed%20), issues to make sure somebody else hasn't already +reported the issue. Please try to include as much information as you can. Details like these are incredibly useful: + +* A reproducible test case or series of steps +* The version of our code being used +* Any modifications you've made relevant to the bug +* Anything unusual about your environment or deployment + + +## Contributing via Pull Requests +Contributions via pull requests are much appreciated. Before sending us a pull request, please ensure that: + +1. You are working against the latest source on the *main* branch. +2. You check existing open, and recently merged, pull requests to make sure someone else hasn't addressed the problem already. +3. You open an issue to discuss any significant work - we would hate for your time to be wasted. + +To send us a pull request, please: + +1. Fork the repository. +2. Modify the source; please focus on the specific change you are contributing. If you also reformat all the code, it will be hard for us to focus on your change. +3. Ensure local tests pass. +4. Commit to your fork using clear commit messages. +5. Send us a pull request, answering any default questions in the pull request interface. +6. Pay attention to any automated CI failures reported in the pull request, and stay involved in the conversation. + +GitHub provides additional document on [forking a repository](https://help.github.com/articles/fork-a-repo/) and +[creating a pull request](https://help.github.com/articles/creating-a-pull-request/). + + +## Finding contributions to work on +Looking at the existing issues is a great way to find something to contribute on. As our projects, by default, use the default GitHub issue labels ((enhancement/bug/duplicate/help wanted/invalid/question/wontfix), looking at any ['help wanted'](https://github.com/awslabs/solution-for-scale-out-computing-on-aws/labels/help%20wanted) issues is a great place to start. + + +## Code of Conduct +This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). +For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact +opensource-codeofconduct@amazon.com with any additional questions or comments. + + +## Security issue notifications +If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/). Please do **not** create a public github issue. + + +## Licensing + +See the [LICENSE](https://github.com/awslabs/solution-for-scale-out-computing-on-aws/blob/main/LICENSE) file for our project's licensing. We will ask you to confirm the licensing of your contribution. + +We may ask you to sign a [Contributor License Agreement (CLA)](http://en.wikipedia.org/wiki/Contributor_License_Agreement) for larger changes. diff --git a/IDEA_VERSION.txt b/IDEA_VERSION.txt new file mode 100644 index 00000000..a0cd9f0c --- /dev/null +++ b/IDEA_VERSION.txt @@ -0,0 +1 @@ +3.1.0 \ No newline at end of file diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 00000000..19dc35b2 --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,175 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. \ No newline at end of file diff --git a/NOTICE.txt b/NOTICE.txt new file mode 100644 index 00000000..2dc5c0c6 --- /dev/null +++ b/NOTICE.txt @@ -0,0 +1,7 @@ +Integrated Digital Engineering on AWS +Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +Licensed under the Apache License Version 2.0 (the "License"). You may not use this file except +in compliance with the License. A copy of the License is located at http://www.apache.org/licenses/ +or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied. See the License for the +specific language governing permissions and limitations under the License. diff --git a/README.md b/README.md new file mode 100644 index 00000000..d832199d --- /dev/null +++ b/README.md @@ -0,0 +1,27 @@ +# Integrated Digital Engineering on AWS + +## Documentation + +https://docs.ide-on-aws.com/ + +## Installation + +Refer to [IDEA Installation](https://docs.ide-on-aws.com/idea/first-time-users/install-idea) for installation instructions. + +This solution collects anonymous operational metrics to help AWS improve the quality of the solution. + +*** + +Copyright 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/THIRD_PARTY_LICENSES.txt b/THIRD_PARTY_LICENSES.txt new file mode 100644 index 00000000..475a53c2 --- /dev/null +++ b/THIRD_PARTY_LICENSES.txt @@ -0,0 +1,346 @@ +** cloudwatch-fluent-metrics; version 0.5.2 -- https://github.com/awslabs/cloudwatch-fluent-metrics +** botocore; version 1.27 -- https://github.com/boto/botocore + +Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + +"License" shall mean the terms and conditions for use, reproduction, and +distribution as defined by Sections 1 through 9 of this document. + +"Licensor" shall mean the copyright owner or entity authorized by the copyright +owner that is granting the License. + +"Legal Entity" shall mean the union of the acting entity and all other entities +that control, are controlled by, or are under common control with that entity. +For the purposes of this definition, "control" means (i) the power, direct or +indirect, to cause the direction or management of such entity, whether by +contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the +outstanding shares, or (iii) beneficial ownership of such entity. + +"You" (or "Your") shall mean an individual or Legal Entity exercising +permissions granted by this License. + +"Source" form shall mean the preferred form for making modifications, including +but not limited to software source code, documentation source, and configuration +files. + +"Object" form shall mean any form resulting from mechanical transformation or +translation of a Source form, including but not limited to compiled object code, +generated documentation, and conversions to other media types. + +"Work" shall mean the work of authorship, whether in Source or Object form, made +available under the License, as indicated by a copyright notice that is included +in or attached to the work (an example is provided in the Appendix below). + +"Derivative Works" shall mean any work, whether in Source or Object form, that +is based on (or derived from) the Work and for which the editorial revisions, +annotations, elaborations, or other modifications represent, as a whole, an +original work of authorship. For the purposes of this License, Derivative Works +shall not include works that remain separable from, or merely link (or bind by +name) to the interfaces of, the Work and Derivative Works thereof. + +"Contribution" shall mean any work of authorship, including the original version +of the Work and any modifications or additions to that Work or Derivative Works +thereof, that is intentionally submitted to Licensor for inclusion in the Work +by the copyright owner or by an individual or Legal Entity authorized to submit +on behalf of the copyright owner. For the purposes of this definition, +"submitted" means any form of electronic, verbal, or written communication sent +to the Licensor or its representatives, including but not limited to +communication on electronic mailing lists, source code control systems, and +issue tracking systems that are managed by, or on behalf of, the Licensor for +the purpose of discussing and improving the Work, but excluding communication +that is conspicuously marked or otherwise designated in writing by the copyright +owner as "Not a Contribution." + +"Contributor" shall mean Licensor and any individual or Legal Entity on behalf +of whom a Contribution has been received by Licensor and subsequently +incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of this +License, each Contributor hereby grants to You a perpetual, worldwide, non- +exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, +prepare Derivative Works of, publicly display, publicly perform, sublicense, and +distribute the Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of this License, +each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no- +charge, royalty-free, irrevocable (except as stated in this section) patent +license to make, have made, use, offer to sell, sell, import, and otherwise +transfer the Work, where such license applies only to those patent claims +licensable by such Contributor that are necessarily infringed by their +Contribution(s) alone or by combination of their Contribution(s) with the Work +to which such Contribution(s) was submitted. If You institute patent litigation +against any entity (including a cross-claim or counterclaim in a lawsuit) +alleging that the Work or a Contribution incorporated within the Work +constitutes direct or contributory patent infringement, then any patent licenses +granted to You under this License for that Work shall terminate as of the date +such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the Work or +Derivative Works thereof in any medium, with or without modifications, and in +Source or Object form, provided that You meet the following conditions: + + (a) You must give any other recipients of the Work or Derivative Works a +copy of this License; and + + (b) You must cause any modified files to carry prominent notices stating +that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works that You +distribute, all copyright, patent, trademark, and attribution notices from the +Source form of the Work, excluding those notices that do not pertain to any part +of the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its distribution, +then any Derivative Works that You distribute must include a readable copy of +the attribution notices contained within such NOTICE file, excluding those +notices that do not pertain to any part of the Derivative Works, in at least one +of the following places: within a NOTICE text file distributed as part of the +Derivative Works; within the Source form or documentation, if provided along +with the Derivative Works; or, within a display generated by the Derivative +Works, if and wherever such third-party notices normally appear. The contents of +the NOTICE file are for informational purposes only and do not modify the +License. You may add Your own attribution notices within Derivative Works that +You distribute, alongside or as an addendum to the NOTICE text from the Work, +provided that such additional attribution notices cannot be construed as +modifying the License. + + You may add Your own copyright statement to Your modifications and may +provide additional or different license terms and conditions for use, +reproduction, or distribution of Your modifications, or for any such Derivative +Works as a whole, provided Your use, reproduction, and distribution of the Work +otherwise complies with the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, any +Contribution intentionally submitted for inclusion in the Work by You to the +Licensor shall be under the terms and conditions of this License, without any +additional terms or conditions. Notwithstanding the above, nothing herein shall +supersede or modify the terms of any separate license agreement you may have +executed with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade names, +trademarks, service marks, or product names of the Licensor, except as required +for reasonable and customary use in describing the origin of the Work and +reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or agreed to in +writing, Licensor provides the Work (and each Contributor provides its +Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied, including, without limitation, any warranties +or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A +PARTICULAR PURPOSE. You are solely responsible for determining the +appropriateness of using or redistributing the Work and assume any risks +associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, whether in +tort (including negligence), contract, or otherwise, unless required by +applicable law (such as deliberate and grossly negligent acts) or agreed to in +writing, shall any Contributor be liable to You for damages, including any +direct, indirect, special, incidental, or consequential damages of any character +arising as a result of this License or out of the use or inability to use the +Work (including but not limited to damages for loss of goodwill, work stoppage, +computer failure or malfunction, or any and all other commercial damages or +losses), even if such Contributor has been advised of the possibility of such +damages. + +9. Accepting Warranty or Additional Liability. While redistributing the Work or +Derivative Works thereof, You may choose to offer, and charge a fee for, +acceptance of support, warranty, indemnity, or other liability obligations +and/or rights consistent with this License. However, in accepting such +obligations, You may act only on Your own behalf and on Your sole +responsibility, not on behalf of any other Contributor, and only if You agree to +indemnify, defend, and hold each Contributor harmless for any liability incurred +by, or claims asserted against, such Contributor by reason of your accepting any +such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + +To apply the Apache License to your work, attach the following boilerplate +notice, with the fields enclosed by brackets "[]" replaced with your own +identifying information. (Don't include the brackets!) The text should be +enclosed in the appropriate comment syntax for the file format. We also +recommend that a file or class name and description of purpose be included on +the same "printed page" as the copyright notice for easier identification within +third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +* For cloudwatch-fluent-metrics see also this required NOTICE: + FluentMetrics + Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. +* For botocore see also this required NOTICE: + + Botocore + Copyright 2012-2022 Amazon.com, Inc. or its affiliates. All Rights Reserved. + + ---- + + Botocore includes vendorized parts of the requests python library for +backwards compatibility. + + Requests License + ================ + + Copyright 2013 Kenneth Reitz + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Botocore includes vendorized parts of the urllib3 library for backwards +compatibility. + + Urllib3 License + =============== + + This is the MIT license: http://www.opensource.org/licenses/mit-license.php + + Copyright 2008-2011 Andrey Petrov and contributors (see CONTRIBUTORS.txt), + Modifications copyright 2012 Kenneth Reitz. + + Permission is hereby granted, free of charge, to any person obtaining a copy +of this + software and associated documentation files (the "Software"), to deal in the +Software + without restriction, including without limitation the rights to use, copy, +modify, merge, + publish, distribute, sublicense, and/or sell copies of the Software, and to +permit persons + to whom the Software is furnished to do so, subject to the following +conditions: + + The above copyright notice and this permission notice shall be included in +all copies or + substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, + INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR +A PARTICULAR + PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +HOLDERS BE LIABLE + FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR + OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE +USE OR OTHER + DEALINGS IN THE SOFTWARE. + + Bundle of CA Root Certificates + ============================== + + ***** BEGIN LICENSE BLOCK ***** + This Source Code Form is subject to the terms of the + Mozilla Public License, v. 2.0. If a copy of the MPL + was not distributed with this file, You can obtain + one at http://mozilla.org/MPL/2.0/. + + ***** END LICENSE BLOCK ***** + +------ + +** O'Reilly Python Cookbook; version 3rd Edition -- https://www.oreilly.com/library/view/python-cookbook-3rd/9781449357337/ +https://www.oreilly.com/library/view/python-cookbook/0596001673/pr02s03.html + +Copyright (c) 2001, Sami Hangaslammi +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + * Neither the name of the Sami Hangaslammi nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS +OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; +OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR +OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF +ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +------ + +** pydantic-to-typescript; version 1.0.10 -- https://github.com/phillipdupuis/pydantic-to-typescript +Copyright (c) 2020 Phillip Dupuis + +Copyright (c) 2020 Phillip Dupuis + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +------ + +** sanic; version 22.3.2 -- https://sanicframework.org/ +Copyright (c) 2016-present Sanic Community + +MIT License + +Copyright (c) 2016-present Sanic Community + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/deployment/ecr/idea-administrator/Dockerfile b/deployment/ecr/idea-administrator/Dockerfile new file mode 100644 index 00000000..6b613d1a --- /dev/null +++ b/deployment/ecr/idea-administrator/Dockerfile @@ -0,0 +1,47 @@ +FROM public.ecr.aws/docker/library/python:3.9.16-slim + +WORKDIR /root + +RUN apt-get update && \ + apt-get -y install \ + curl \ + tar \ + unzip \ + locales \ + && apt-get clean + + +ENV DEBIAN_FRONTEND=noninteractive +ENV LC_ALL="en_US.UTF-8" \ + LC_CTYPE="en_US.UTF-8" \ + LANG="en_US.UTF-8" + +RUN sed -i -e "s/# $LANG.*/$LANG UTF-8/" /etc/locale.gen \ + && locale-gen "en_US.UTF-8" \ + && dpkg-reconfigure locales + +# install aws cli +RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" && \ + unzip -qq awscliv2.zip && \ + ./aws/install && \ + rm -rf ./aws awscliv2.zip + +# install nvm and node +RUN curl -sL https://deb.nodesource.com/setup_16.x | bash - \ + && apt-get install -y nodejs \ + && apt-get clean all + +# add all packaged artifacts to container +ARG PUBLIC_ECR_TAG +ENV PUBLIC_ECR_TAG=${PUBLIC_ECR_TAG} +ADD all-*.tar.gz /root/.idea/downloads/ + +# install administrator app +RUN mkdir -p /root/.idea/downloads/idea-administrator-${PUBLIC_ECR_TAG} && \ + tar -xvf /root/.idea/downloads/idea-administrator-*.tar.gz -C /root/.idea/downloads/idea-administrator-${PUBLIC_ECR_TAG} && \ + /bin/bash /root/.idea/downloads/idea-administrator-${PUBLIC_ECR_TAG}/install.sh && \ + rm -rf /root/.idea/downloads/idea-administrator-${PUBLIC_ECR_TAG} + +CMD ["bash"] + + diff --git a/deployment/integrated-digital-engineering-on-aws.template b/deployment/integrated-digital-engineering-on-aws.template new file mode 100644 index 00000000..a92f97bf --- /dev/null +++ b/deployment/integrated-digital-engineering-on-aws.template @@ -0,0 +1,234 @@ +AWSTemplateFormatVersion: 2010-09-09 +Description: (SO0072) Integrated Digital Engineering on AWS (IDEA) + +Metadata: + + AWS::CloudFormation::Interface: + ParameterGroups: + + - Label: + default: Linux Distribution + Parameters: + - ClusterName + - BaseOS + + - Label: + default: Network and Security + Parameters: + - VpcCidr + - ClientIp + - SSHKeyPair + + - Label: + default: Cluster Administrator + Parameters: + - AdministratorEmail + + - Label: + default: Installer EC2 Instance + Parameters: + - InstallerAmiId + - InstallerCdkToolkitPolicyArn + - InstallerCreateAccessPolicyArn + - InstallerDeleteAccessPolicyArn + + ParameterLabels: + InstallerAmiId: + default: The AMI ID for the installer EC2 Instance + InstallerCdkToolkitPolicyArn: + default: IDEA Administrator CDK ToolKit IAM Policy ARN + InstallerCreateAccessPolicyArn: + default: IDEA Administrator Create Access IAM Policy ARN + InstallerDeleteAccessPolicyArn: + default: IDEA Administrator Delete Access IAM Policy ARN + +Parameters: + + InstallerAmiId: + Type: 'AWS::SSM::Parameter::Value' + Description: Do not change this value. We will use the latest Amazon Linux 2 instance AMI ID based on your region. + Default: /aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-gp2 + + InstallerCdkToolkitPolicyArn: + Type: String + Description: | + The ARN of the IAM Policy to be attached to the Installer EC2 Instance. This policy should contain all the permissions + listed under "deployment/idea/aws/idea-admin-cdk-toolkit-policy.json" + + InstallerCreateAccessPolicyArn: + Type: String + Description: | + The ARN of the IAM Policy to be attached to the Installer EC2 Instance. This policy should contain all the permissions + listed under "deployment/idea/aws/idea-admin-create-policy.json" + + InstallerDeleteAccessPolicyArn: + Type: String + Description: | + The ARN of the IAM Policy to be attached to the Installer EC2 Instance. This policy should contain all the permissions + listed under "deployment/idea/aws/idea-admin-delete-policy.json" + + ClusterName: + Type: String + Description: Name of your cluster. + AllowedPattern: 'idea-.+' + ConstraintDescription: The name of the cluster must start with "idea-". + + BaseOS: + Type: String + "AllowedValues": [ + "amazonlinux2", + "centos7", + "rhel7" + ] + "Description": IMPORTANT CENTOS USERS > You MUST subscribe to https://aws.amazon.com/marketplace/pp/B00O7WM7QW first if using CentOS + "Default": amazonlinux2 + + VpcCidr: + Type: String + Description: Choose the Cidr block (/16 down to /24) you want to use for your VPC (eg 10.0.0.0/16 down to 10.0.0.0/24) + AllowedPattern: '((\d{1,3})\.){3}\d{1,3}/(1[6-9]|2[0-4])' + ConstraintDescription: Your VPC must use x.x.x.x/16 - x.x.x.x/24 CIDR range + Default: 10.0.0.0/16 + + ClientIp: + Type: String + Description: Default IP(s) allowed to directly SSH into the scheduler and access ElasticSearch. 0.0.0.0/0 means ALL INTERNET access. You probably want to change it with your own IP/subnet (x.x.x.x/32 for your own ip or x.x.x.x/24 for range. Replace x.x.x.x with your own PUBLIC IP. You can get your public IP using tools such as https://ifconfig.co/). Make sure to keep it restrictive! + AllowedPattern: '(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})/(\d{1,2})' + ConstraintDescription: ClientIP must be a valid IP or network range of the form x.x.x.x/x. If you want to add everyone (not recommended) use 0.0.0.0/0 otherwise specify your IP/NETMASK (e.g x.x.x/32 or x.x.x.x/24 for subnet range) + + SSHKeyPair: + Type: AWS::EC2::KeyPair::KeyName + Description: Default SSH pem keys used to SSH into cluster instances. + AllowedPattern: .+ + + AdministratorEmail: + Type: String + Description: | + Provide an Email Address for the cluster administrator account. You will receive an email with your temporary credentials during cluster installation. After the solution is deployed, you can use the temporary credentials to login + and reset the password. + MinLength: 3 + + +Resources: + + InstallerIamRole: + Type: AWS::IAM::Role + Properties: + Path: / + AssumeRolePolicyDocument: + Statement: + - Effect: Allow + Principal: + Service: + - !Sub ec2.${AWS::URLSuffix} + - !Sub ssm.${AWS::URLSuffix} + Action: + - sts:AssumeRole + ManagedPolicyArns: + - !Sub arn:${AWS::Partition}:iam::aws:policy/AmazonSSMManagedInstanceCore + - !Ref InstallerCdkToolkitPolicyArn + - !Ref InstallerCreateAccessPolicyArn + - !Ref InstallerDeleteAccessPolicyArn + + InstallerInstanceProfile: + Type: AWS::IAM::InstanceProfile + Properties: + Path: / + Roles: + - !Ref InstallerIamRole + + InstallerEc2: + Type: 'AWS::EC2::Instance' + CreationPolicy: + ResourceSignal: + Timeout: PT2H + Properties: + SecurityGroups: + - !Ref InstallerEc2SecurityGroup + KeyName: !Ref SSHKeyPair + ImageId: !Ref InstallerAmiId + IamInstanceProfile: !Ref InstallerInstanceProfile + InstanceType: t3.medium + Tags: + - Key: Name + Value: !Sub ${ClusterName}-installer + UserData: + "Fn::Base64": !Sub | + #!/bin/bash + + set -x + + IDEA_ECR_REPO="public.ecr.aws/g8j8s8q8/idea-administrator" + IDEA_REVISION="v3.1.0" + INSTANCE_PUBLIC_IP=$(TOKEN=$(curl --silent -X PUT 'http://169.254.169.254/latest/api/token' -H 'X-aws-ec2-metadata-token-ttl-seconds: 300') && curl --silent -H "X-aws-ec2-metadata-token: ${!TOKEN}" 'http://169.254.169.254/latest/meta-data/public-ipv4') + + mkdir -p /root/.idea/clusters/${ClusterName}/${AWS::Region} + + echo " + --- + aws_partition: ${AWS::Partition} + aws_region: ${AWS::Region} + aws_account_id: '${AWS::AccountId}' + aws_dns_suffix: ${AWS::URLSuffix} + cluster_name: ${ClusterName} + administrator_email: ${AdministratorEmail} + vpc_cidr_block: ${VpcCidr} + ssh_key_pair_name: ${SSHKeyPair} + cluster_access: client-ip + client_ip: + - ${ClientIp} + - ${!INSTANCE_PUBLIC_IP}/32 + alb_public: true + use_vpc_endpoints: false + directory_service_provider: openldap + kms_key_type: aws-managed + enabled_modules: + - metrics + - scheduler + - virtual-desktop-controller + - bastion-host + metrics_provider: cloudwatch + base_os: ${BaseOS} + instance_type: m5.large + volume_size: '200' + " > /root/.idea/clusters/${ClusterName}/${AWS::Region}/values.yml + + yum install -y docker + systemctl enable docker.service + systemctl start docker.service + + docker pull ${!IDEA_ECR_REPO}:${!IDEA_REVISION} + + docker run --rm -i \ + -v /root/.idea/clusters:/root/.idea/clusters \ + -e AWS_DEFAULT_REGION=${AWS::Region} \ + -e IDEA_ADMIN_AWS_CREDENTIAL_PROVIDER=Ec2InstanceMetadata \ + ${!IDEA_ECR_REPO}:${!IDEA_REVISION} \ + idea-admin quick-setup \ + --values-file /root/.idea/clusters/${ClusterName}/${AWS::Region}/values.yml \ + --force + + /opt/aws/bin/cfn-signal -e "$?" --stack "${AWS::StackName}" --resource InstallerEc2 --region "${AWS::Region}" + + InstallerEc2SecurityGroup: + Type: 'AWS::EC2::SecurityGroup' + Metadata: + cfn_nag: + rules_to_suppress: + - id: W5 + reason: "Allow all IP egress from the installer EC2 instance" + - id: W29 + reason: "Allow all TCP ports egress from the installer EC2 instance" + Properties: + GroupDescription: Enable SSH access via port 22 from ClientIP + SecurityGroupIngress: + - IpProtocol: tcp + FromPort: 22 + ToPort: 22 + Description: SSH Access from Client IP + CidrIp: !Ref ClientIp + SecurityGroupEgress: + - FromPort: 0 + ToPort: 65535 + Description: All egress TCP traffic + CidrIp: 0.0.0.0/0 diff --git a/idea-admin-windows.ps1 b/idea-admin-windows.ps1 new file mode 100755 index 00000000..72727f16 --- /dev/null +++ b/idea-admin-windows.ps1 @@ -0,0 +1,109 @@ +###################################################################################################################### +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # +# # +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance # +# with the License. A copy of the License is located at # +# # +# http://www.apache.org/licenses/LICENSE-2.0 # +# # +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES # +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # +# and limitations under the License. # +###################################################################################################################### + +function Exit-Fail($message,$command) { + Write-Host "[MESSAGE]: ${message} `n[COMMAND EXECUTED]: ${command}`n[HELP]: Refer to ${DocumentationError} for troubleshooting." -foregroundcolor "red" + Read-Host -Prompt "Installation was not successful, press Enter to exit" + Exit 1 +} + +function Verify-Command($type,$message,$command) { + if ($type -eq "Get-Command") { + if($Error) { + Exit-Fail $message $command + } + } + elseif ($type -eq "Invoke-Expression") { + if($LASTEXITCODE -ne 0) { + Exit-Fail $message $command + } + } + else { + Write-Output "type must be either Get-Command or Invoke-Expression" + Read-Host -Prompt "Installation was not successful, press Enter to exit" + Exit 1 + } +} + +$IDEADevMode = if ($Env:IDEA_DEV_MODE) {$Env:IDEA_DEV_MODE} else {""} +$VirtualEnv = if ($Env:VIRTUAL_ENV) {$Env:VIRTUAL_ENV} else {""} +$ScriptDir = $PSScriptRoot +$IDEARevision = if ($Env:IDEA_REVISION) {$Env:IDEA_REVISION} else {"v3.1.0"} +$IDEADockerRepo = "public.ecr.aws/g8j8s8q8" +$DocumentationError = "https://ide-on-aws.com" +$AWSProfile = if ($Env:AWS_PROFILE) {$Env:AWS_PROFILE} else {"default"} +$AWSRegion= if ($Env:AWS_REGION) {$Env:AWS_REGION} else {"us-east-1"} +Set-Location -Path "${ScriptDir}" + + +if ($IDEADevMode -ne "") { + if (Test-Path -Path "${ScriptDir}/IDEA_VERSION.txt") { + $IDEADevMode="true" + } + else { + $IDEADevMode="false" + } +} + +if ($IDEADevMode -eq "true") { + Write-Host "Development Mode is only supported on Linux/Mac" + <# + if ($VirtualEnv -eq "") { + if (Test-Path -Path "$ScriptDir/venv") { + . "$ScriptDir/venv/bin/activate" + } + else { + Verify-Command "Get-Command" "Python Virtual Environment not detected. Install virtual environment to execute idea-admin.sh in dev mode." "source ${ScriptDir}/venv/bin/activate" + } + } + IDEA_SKIP_WEB_BUILD=${IDEA_SKIP_WEB_BUILD:-'0'} + $IDESkipWebBuild = if (IDEA_SKIP_WEB_BUILD) {$Env:IDEA_DEV_MODE} else {""} + TOKENS=$(echo $(printf ",\"%s\"" "${@}")) + TOKENS=${TOKENS:1} + ARGS=$(echo "[${TOKENS}]" | base64) + CMD="invoke cli.admin --args=${ARGS}" + IDEA_SKIP_WEB_BUILD=${IDEA_SKIP_WEB_BUILD} eval $CMD + exit $? #> +} + +$DockerBin=$(Get-Command docker).source 2>$null +Verify-Command "Get-Command" "Docker not detected. Download and install it from https://docs.docker.com/get-docker/. Read the Docker Subscription Service Agreement first (https://www.docker.com/legal/docker-subscription-service-agreement/)." "Get-Command docker" + +$AWSCliBin=$(Get-Command aws).source 2>$null +Verify-Command "Get-Command" "awscli not detected. Download and install it from https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html" "Get-Command aws" + +if (-not (Test-Path "$HOME/.idea/clusters") ) { + New-Item -ItemType "directory" -Path "$HOME/.idea/clusters" | Out-Null +} + +Invoke-Expression "& '$DockerBin' version" | Out-Null 2>$null +Verify-Command "Invoke-Expression" "Docker is installed on the system but it does not seems to be running. Start Docker first." "docker info" + +[System.Net.Dns]::GetHostEntry("public.ecr.aws") 2>$null | Out-Null +Verify-Command "Get-Command" "Unable to query ECR. Are you connected to internet?" "[System.Net.Dns]::GetHostEntry($IDEADockerRepo)" + +# Select-String -Quiet does not work properly if the number of Docker images is 0, so we go old school and verify if the variable is empty +$ImageExist = Invoke-Expression "& '$DockerBin' images" | Select-String "$IDEADockerRepo/idea-administrator" | Select-String "$IDEARevision" +if ($ImageExist -eq $null) { + Invoke-Expression "& '$DockerBin' pull $IDEADockerRepo/idea-administrator:$IDEARevision" + Verify-Command "Invoke-Expression" "Unable to download IDEA container image. Refer to the error above. If your token has expired, run: docker logout public.ecr.aws" "$DockerBin pull $IDEADockerRepo/idea-administrator:$IDEARevision" +} + +if ($args.count -eq 0) { + $args = "quick-setup" + Write-Host "No arguments detected, defaulting to quick-setup. Use -h to see all options." +} + +Invoke-Expression "& '$DockerBin' run --rm -it -v $HOME/.idea/clusters:/root/.idea/clusters -v $HOME/.aws:/root/.aws $IDEADockerRepo/idea-administrator:$IDEARevision idea-admin $args" + +Read-Host -Prompt "Press Enter to exit" diff --git a/idea-admin.sh b/idea-admin.sh new file mode 100755 index 00000000..575a1890 --- /dev/null +++ b/idea-admin.sh @@ -0,0 +1,125 @@ +#!/bin/bash + +###################################################################################################################### +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # +# # +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance # +# with the License. A copy of the License is located at # +# # +# http://www.apache.org/licenses/LICENSE-2.0 # +# # +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES # +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions # +# and limitations under the License. # +###################################################################################################################### + +# Integrated Digital Engineering on AWS - Installation Script +# +# Usage: +# ./idea-admin.sh --help +# +# Environment Variables: +# * IDEA_REVISION - Use to override the default IDEA version. +# * IDEA_DOCKER_REPO - Use to override the default Docker/ECR repository. +# * IDEA_ECR_CREDS_RESET - Set to false, if you handle AWS ECR authentication manually. +# * IDEA_ADMIN_AWS_CREDENTIAL_PROVIDER - Set to "Ec2InstanceMetadata", if you want install IDEA from an EC2 Instance +# using Instance Profile credentials from EC2 Instance Metadata. +# * IDEA_ADMIN_ENABLE_CDK_NAG_SCAN - Set to "false", if you want to disable cdk-nag scan. Default: true +# * IDEA_DEV_MODE - Set to "true" if you are working with IDEA sources + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +IDEA_REVISION=${IDEA_REVISION:-"v3.1.0"} +IDEA_DOCKER_REPO=${IDEA_DOCKER_REPO:-"public.ecr.aws/g8j8s8q8/idea-administrator"} +IDEA_ECR_CREDS_RESET=${IDEA_ECR_CREDS_RESET:-"true"} +IDEA_ADMIN_AWS_CREDENTIAL_PROVIDER=${IDEA_ADMIN_AWS_CREDENTIAL_PROVIDER:=""} +IDEA_ADMIN_ENABLE_CDK_NAG_SCAN=${IDEA_ADMIN_ENABLE_CDK_NAG_SCAN:-"true"} + +DOCUMENTATION_ERROR="https://ide-on-aws.com" +NC="\033[0m" # No Color +RED="\033[1;31m" +GREEN="\033[1;32m" +YELLOW="\033[1;33m" + +verify_command() { + # shellcheck disable=SC2181 + if [[ "$?" -ne "0" ]]; then + echo -e "${RED}[MESSAGE]: ${1} \n[HELP]: Refer to ${DOCUMENTATION_ERROR} for troubleshooting.${NC}" + exit 1 + fi +} + +if [[ "${IDEA_DEV_MODE}" == "true" ]]; then + if [[ ! -f ${SCRIPT_DIR}/IDEA_VERSION.txt ]]; then + echo -e "${RED}idea-admin.sh must be executed from IDEA project root directory when using developer mode." + exit 1 + fi + if [[ -z "${VIRTUAL_ENV}" ]]; then + if [[ -d ${SCRIPT_DIR}/venv ]]; then + source ${SCRIPT_DIR}/venv/bin/activate + else + echo -e "${RED}Python Virtual Environment not detected. Install virtual environment to execute idea-admin.sh in developer mode." + exit 1 + fi + fi + IDEA_SKIP_WEB_BUILD=${IDEA_SKIP_WEB_BUILD:-'0'} + TOKENS=$(echo $(printf ",\"%s\"" "${@}")) + TOKENS=${TOKENS:1} + if [[ $(uname -s) == "Linux" ]]; then + ARGS=$(echo "[${TOKENS}]" | base64 -w0) + else + ARGS=$(echo "[${TOKENS}]" | base64) + fi + CMD="invoke cli.admin --args=${ARGS}" + + IDEA_SKIP_WEB_BUILD=${IDEA_SKIP_WEB_BUILD} \ + IDEA_ADMIN_AWS_CREDENTIAL_PROVIDER=${IDEA_ADMIN_AWS_CREDENTIAL_PROVIDER} \ + IDEA_ADMIN_ENABLE_CDK_NAG_SCAN=${IDEA_ADMIN_ENABLE_CDK_NAG_SCAN} \ + eval $CMD + + exit $? +fi + +cd "${SCRIPT_DIR}" + +# Check if Docker is installed +DOCKER_BIN=$(command -v docker) +verify_command "Docker not detected. Download and install it from https://docs.docker.com/get-docker/. Read the Docker Subscription Service Agreement first (https://www.docker.com/legal/docker-subscription-service-agreement/)." + +# Check if aws cli (https://aws.amazon.com/cli/) is installed +AWSCLI_BIN=$(command -v aws) +verify_command "awscli not detected. Download and install it from https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html" + +# Create folder hierarchy +MKDIR_BIN=$(command -v mkdir) +${MKDIR_BIN} -p ${HOME}/.idea/clusters +verify_command "Unable to create ${HOME}/.idea/clusters. Verify path and permissions." + +# Check if Docker is running +${DOCKER_BIN} info >> /dev/null 2>&1 +verify_command "Docker is installed on the system but it does not seems to be running. Start Docker first." + +# Reset ECR credentials +if [[ "${IDEA_ECR_CREDS_RESET}" == "true" ]]; then + # Check if user is connected to internet an can ping ECR repo + DIG_BIN=$(command -v dig) + ${DIG_BIN} +tries=1 +time=3 ${IDEA_DOCKER_REPO} >> /dev/null 2>&1 + verify_command "Unable to query ECR. Are you connected to internet?" + + ${DOCKER_BIN} logout public.ecr.aws >> /dev/null 2>&1 + verify_command "Failed to refresh ECR credentials. docker logout public.ecr.aws failed" +fi + +# Pull IDEA docker image if needed +${DOCKER_BIN} images | grep "${IDEA_DOCKER_REPO}" | grep -q "${IDEA_REVISION}" +if [[ $? -ne 0 ]]; then + ${DOCKER_BIN} pull ${IDEA_DOCKER_REPO}:${IDEA_REVISION} + verify_command "Unable to download IDEA container image. Refer to the error above." +fi + +# Launch installer +${DOCKER_BIN} run --rm -it -v ${HOME}/.idea/clusters:/root/.idea/clusters \ + -e IDEA_ADMIN_AWS_CREDENTIAL_PROVIDER=${IDEA_ADMIN_AWS_CREDENTIAL_PROVIDER} \ + -e IDEA_ADMIN_ENABLE_CDK_NAG_SCAN=${IDEA_ADMIN_ENABLE_CDK_NAG_SCAN} \ + -v ~/.aws:/root/.aws ${IDEA_DOCKER_REPO}:${IDEA_REVISION} \ + idea-admin ${@} + diff --git a/requirements/dev.in b/requirements/dev.in new file mode 100644 index 00000000..6be61df5 --- /dev/null +++ b/requirements/dev.in @@ -0,0 +1,12 @@ +-r idea-dev-lambda.in +-r idea-administrator.in +-r idea-scheduler.in +-r idea-virtual-desktop-controller.in +-r idea-cluster-manager.in +-r doc.in +-r tests.in +pip-tools +invoke +pyfiglet +openapi-schema-pydantic +pylint diff --git a/requirements/dev.txt b/requirements/dev.txt new file mode 100644 index 00000000..4e1a6754 --- /dev/null +++ b/requirements/dev.txt @@ -0,0 +1,130 @@ +aiofiles==0.8.0 +alembic==1.7.7 +arrow==1.2.1 +astroid==2.12.11 +attrs==21.4.0 +aws-cdk-asset-awscli-v1==2.2.52 +aws-cdk-asset-kubectl-v20==2.1.1 +aws-cdk-asset-node-proxy-agent-v5==2.0.42 +aws-cdk-lib==2.63.0 +banal==1.0.6 +blinker==1.4 +boto3==1.26.61 +botocore==1.29.61 +cacheout==0.13.1 +cachetools==5.1.0 +cattrs==22.1.0 +cdk-nag==2.18.17 +certifi==2022.9.14 +cffi==1.15.0 +cfn-flip==1.3.0 +charset-normalizer==2.0.12 +click==8.1.3 +colored==1.4.3 +commonmark==0.9.1 +constructs==10.1.10 +coverage[toml]==6.5.0 +cryptography==36.0.1 +dataset==1.5.2 +decorator==5.1.1 +defusedxml==0.7.1 +dill==0.3.5.1 +exceptiongroup==1.0.0rc6 +fastcounter==1.1.0 +ghp-import==2.1.0 +greenlet==1.1.2 +httptools==0.4.0 +idna==3.3 +importlib-metadata==4.11.3 +iniconfig==1.1.1 +invoke==1.7.1 +ipaddress==1.0.23 +isort==5.10.1 +jinja2==3.1.2 +jmespath==1.0.0 +jsii==1.74.0 +lazy-object-proxy==1.7.1 +ldappool==3.0.0 +mako==1.2.0 +markdown==3.3.7 +markupsafe==2.1.1 +mccabe==0.7.0 +memory-profiler==0.60.0 +mergedeep==1.3.4 +mkdocs==1.3.0 +mkdocs-material==8.2.15 +mkdocs-material-extensions==1.0.3 +multidict==6.0.2 +mypy==0.950 +mypy-extensions==0.4.3 +openapi-schema-pydantic==1.2.4 +opensearch-py==2.0.0 +orjson==3.6.5 +packaging==21.3 +pep517==0.12.0 +pip-tools==6.6.1 +platformdirs==2.5.2 +pluggy==1.0.0 +prettytable==3.3.0 +prometheus-client==0.14.1 +prompt-toolkit==3.0.29 +psutil==5.9.0 +publication==0.0.3 +py==1.11.0 +pyasn1==0.4.8 +pyasn1-modules==0.2.8 +pycparser==2.21 +pydantic==1.9.0 +pyfiglet==0.8.post1 +pygments==2.12.0 +pyhocon==0.3.59 +pyjwt==2.4.0 +pylint==2.15.4 +pymdown-extensions==9.4 +pyparsing==2.4.7 +pytest==7.1.2 +pytest-cov==4.0.0 +pytest-mock==3.10.0 +python-dateutil==2.8.2 +python-dynamodb-lock==0.9.1 +python-ldap==3.4.0 +pytz==2022.1 +pytz-deprecation-shim==0.1.0.post0 +pyyaml==6.0 +pyyaml-env-tag==0.1 +questionary==1.10.0 +random-password-generator==2.2.0 +requests==2.27.1 +requests-aws4auth==1.1.2 +requests-unixsocket==0.3.0 +rich==12.4.1 +s3transfer==0.6.0 +sanic==22.3.2 +sanic-routing==22.3.0 +semver==2.13.0 +sh==1.14.2 +shortuuid==1.0.9 +six==1.16.0 +sqlalchemy==1.4.36 +supervisor==4.2.4 +tomli==2.0.1 +tomlkit==0.11.5 +troposphere==4.3.0 +typeguard==2.13.3 +typing-extensions==4.2.0 +tzdata==2022.1 +tzlocal==4.2 +ujson==5.7.0 +urllib3==1.26.9 +uvloop==0.16.0 +validators==0.19.0 +watchdog==2.1.8 +wcwidth==0.2.5 +websockets==10.3 +wheel==0.37.1 +wrapt==1.14.1 +zipp==3.8.0 + +# The following packages are considered to be unsafe in a requirements file: +# pip +# setuptools diff --git a/requirements/doc.in b/requirements/doc.in new file mode 100644 index 00000000..f7157c86 --- /dev/null +++ b/requirements/doc.in @@ -0,0 +1,5 @@ +mkdocs +pygments +pymdown-extensions +mkdocs-material +mkdocs-material-extensions diff --git a/requirements/doc.txt b/requirements/doc.txt new file mode 100644 index 00000000..919a0b67 --- /dev/null +++ b/requirements/doc.txt @@ -0,0 +1,20 @@ +click==8.1.3 +ghp-import==2.1.0 +importlib-metadata==4.11.3 +jinja2==3.1.2 +markdown==3.3.7 +markupsafe==2.1.1 +mergedeep==1.3.4 +mkdocs==1.3.0 +mkdocs-material==8.2.15 +mkdocs-material-extensions==1.0.3 +packaging==21.3 +pygments==2.12.0 +pymdown-extensions==9.4 +pyparsing==3.0.9 +python-dateutil==2.8.2 +pyyaml==6.0 +pyyaml-env-tag==0.1 +six==1.16.0 +watchdog==2.1.8 +zipp==3.8.0 diff --git a/requirements/idea-administrator.in b/requirements/idea-administrator.in new file mode 100644 index 00000000..b94aab72 --- /dev/null +++ b/requirements/idea-administrator.in @@ -0,0 +1,8 @@ +-r idea-sdk.in +colored +ipaddress +sanic==22.3.2 +aws-cdk-lib==2.63.0 +cdk-nag +prettytable +defusedxml diff --git a/requirements/idea-administrator.txt b/requirements/idea-administrator.txt new file mode 100644 index 00000000..d5595361 --- /dev/null +++ b/requirements/idea-administrator.txt @@ -0,0 +1,85 @@ +aiofiles==0.8.0 +alembic==1.8.0 +arrow==1.2.1 +attrs==21.4.0 +aws-cdk-asset-awscli-v1==2.2.52 +aws-cdk-asset-kubectl-v20==2.1.1 +aws-cdk-asset-node-proxy-agent-v5==2.0.42 +aws-cdk-lib==2.63.0 +banal==1.0.6 +blinker==1.4 +boto3==1.26.61 +botocore==1.29.61 +cacheout==0.13.1 +cattrs==22.1.0 +cdk-nag==2.18.17 +certifi==2022.5.18 +cffi==1.15.0 +cfn-flip==1.3.0 +charset-normalizer==2.0.12 +click==8.1.3 +colored==1.4.3 +commonmark==0.9.1 +constructs==10.1.10 +cryptography==36.0.1 +dataset==1.5.2 +decorator==5.1.1 +defusedxml==0.7.1 +exceptiongroup==1.0.0rc6 +fastcounter==1.1.0 +greenlet==1.1.2 +httptools==0.4.0 +idna==3.3 +ipaddress==1.0.23 +jinja2==3.1.2 +jmespath==1.0.0 +jsii==1.74.0 +mako==1.2.1 +markupsafe==2.1.1 +multidict==6.0.2 +mypy==0.950 +mypy-extensions==0.4.3 +opensearch-py==2.0.0 +orjson==3.6.5 +prettytable==3.3.0 +prometheus-client==0.14.1 +prompt-toolkit==3.0.29 +psutil==5.9.0 +publication==0.0.3 +pycparser==2.21 +pydantic==1.9.0 +pygments==2.12.0 +pyhocon==0.3.59 +pyjwt==2.4.0 +pyparsing==2.4.7 +python-dateutil==2.8.2 +python-dynamodb-lock==0.9.1 +pytz==2022.1 +pytz-deprecation-shim==0.1.0.post0 +pyyaml==6.0 +questionary==1.10.0 +random-password-generator==2.2.0 +requests==2.27.1 +requests-aws4auth==1.1.2 +requests-unixsocket==0.3.0 +rich==12.4.1 +s3transfer==0.6.0 +sanic==22.3.2 +sanic-routing==22.3.0 +semver==2.13.0 +sh==1.14.2 +shortuuid==1.0.9 +six==1.16.0 +sqlalchemy==1.4.39 +tomli==2.0.1 +troposphere==4.3.0 +typeguard==2.13.3 +typing-extensions==4.2.0 +tzdata==2022.1 +tzlocal==4.2 +ujson==5.7.0 +urllib3==1.26.9 +uvloop==0.16.0 +validators==0.19.0 +wcwidth==0.2.5 +websockets==10.3 diff --git a/requirements/idea-cluster-manager.in b/requirements/idea-cluster-manager.in new file mode 100644 index 00000000..ffb48d48 --- /dev/null +++ b/requirements/idea-cluster-manager.in @@ -0,0 +1,5 @@ +-r idea-sdk.in +supervisor +sanic=22.3.2 +python-ldap +ldappool diff --git a/requirements/idea-cluster-manager.txt b/requirements/idea-cluster-manager.txt new file mode 100644 index 00000000..7d58da7b --- /dev/null +++ b/requirements/idea-cluster-manager.txt @@ -0,0 +1,78 @@ +aiofiles==0.8.0 +alembic==1.8.0 +arrow==1.2.1 +banal==1.0.6 +blinker==1.4 +boto3==1.26.61 +botocore==1.29.61 +cacheout==0.13.1 +certifi==2022.5.18 +cffi==1.15.0 +cfn-flip==1.3.0 +charset-normalizer==2.0.12 +click==8.1.3 +commonmark==0.9.1 +cryptography==36.0.1 +dataset==1.5.2 +decorator==5.1.1 +fastcounter==1.1.0 +greenlet==1.1.2 +httptools==0.4.0 +idna==3.3 +jinja2==3.1.2 +jmespath==1.0.0 +ldappool==3.0.0 +mako==1.2.1 +markupsafe==2.1.1 +multidict==6.0.2 +mypy==0.950 +mypy-extensions==0.4.3 +opensearch-py==2.0.0 +orjson==3.6.5 +prettytable==3.3.0 +prometheus-client==0.14.1 +prompt-toolkit==3.0.29 +psutil==5.9.0 +pyasn1==0.4.8 +pyasn1-modules==0.2.8 +pycparser==2.21 +pydantic==1.9.0 +pygments==2.12.0 +pyhocon==0.3.59 +pyjwt==2.4.0 +pyparsing==2.4.7 +python-dateutil==2.8.2 +python-dynamodb-lock==0.9.1 +python-ldap==3.4.0 +pytz==2022.1 +pytz-deprecation-shim==0.1.0.post0 +pyyaml==6.0 +questionary==1.10.0 +random-password-generator==2.2.0 +requests==2.27.1 +requests-aws4auth==1.1.2 +requests-unixsocket==0.3.0 +rich==12.4.1 +s3transfer==0.6.0 +sanic==22.3.2 +sanic-routing==22.3.0 +semver==2.13.0 +sh==1.14.2 +shortuuid==1.0.9 +six==1.16.0 +sqlalchemy==1.4.39 +supervisor==4.2.4 +tomli==2.0.1 +troposphere==4.3.0 +typing-extensions==4.2.0 +tzdata==2022.1 +tzlocal==4.2 +ujson==5.7.0 +urllib3==1.26.9 +uvloop==0.16.0 +validators==0.19.0 +wcwidth==0.2.5 +websockets==10.3 + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/requirements/idea-dev-lambda.in b/requirements/idea-dev-lambda.in new file mode 100644 index 00000000..98d751ad --- /dev/null +++ b/requirements/idea-dev-lambda.in @@ -0,0 +1,2 @@ +opensearch-py +cryptography diff --git a/requirements/idea-dev-lambda.txt b/requirements/idea-dev-lambda.txt new file mode 100644 index 00000000..97a01ea7 --- /dev/null +++ b/requirements/idea-dev-lambda.txt @@ -0,0 +1,9 @@ +certifi==2022.9.24 +cffi==1.15.1 +charset-normalizer==2.1.1 +cryptography==38.0.1 +idna==3.4 +opensearch-py==2.0.0 +pycparser==2.21 +requests==2.28.1 +urllib3==1.26.12 diff --git a/requirements/idea-scheduler.in b/requirements/idea-scheduler.in new file mode 100644 index 00000000..eb00d8cc --- /dev/null +++ b/requirements/idea-scheduler.in @@ -0,0 +1,5 @@ +-r idea-sdk.in +cachetools +prettytable +supervisor +sanic==22.3.2 diff --git a/requirements/idea-scheduler.txt b/requirements/idea-scheduler.txt new file mode 100644 index 00000000..ab63608d --- /dev/null +++ b/requirements/idea-scheduler.txt @@ -0,0 +1,75 @@ +aiofiles==0.8.0 +alembic==1.7.7 +arrow==1.2.1 +banal==1.0.6 +blinker==1.4 +boto3==1.26.61 +botocore==1.29.61 +cacheout==0.13.1 +cachetools==5.1.0 +certifi==2022.5.18 +cffi==1.15.0 +cfn-flip==1.3.0 +charset-normalizer==2.0.12 +click==8.1.3 +commonmark==0.9.1 +cryptography==36.0.1 +dataset==1.5.2 +decorator==5.1.1 +fastcounter==1.1.0 +greenlet==1.1.2 +httptools==0.4.0 +idna==3.3 +jinja2==3.1.2 +jmespath==1.0.0 +mako==1.2.0 +markupsafe==2.1.1 +multidict==6.0.2 +mypy==0.950 +mypy-extensions==0.4.3 +opensearch-py==2.0.0 +orjson==3.6.5 +prettytable==3.3.0 +prometheus-client==0.14.1 +prompt-toolkit==3.0.29 +psutil==5.9.0 +pycparser==2.21 +pydantic==1.9.0 +pygments==2.12.0 +pyhocon==0.3.59 +pyjwt==2.4.0 +pyparsing==2.4.7 +python-dateutil==2.8.2 +python-dynamodb-lock==0.9.1 +pytz==2022.1 +pytz-deprecation-shim==0.1.0.post0 +pyyaml==6.0 +questionary==1.10.0 +random-password-generator==2.2.0 +requests==2.27.1 +requests-aws4auth==1.1.2 +requests-unixsocket==0.3.0 +rich==12.4.1 +s3transfer==0.6.0 +sanic==22.3.2 +sanic-routing==22.3.0 +semver==2.13.0 +sh==1.14.2 +shortuuid==1.0.9 +six==1.16.0 +sqlalchemy==1.4.36 +supervisor==4.2.4 +tomli==2.0.1 +troposphere==4.3.0 +typing-extensions==4.2.0 +tzdata==2022.1 +tzlocal==4.2 +ujson==5.7.0 +urllib3==1.26.9 +uvloop==0.16.0 +validators==0.19.0 +wcwidth==0.2.5 +websockets==10.3 + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/requirements/idea-sdk.in b/requirements/idea-sdk.in new file mode 100644 index 00000000..8b493179 --- /dev/null +++ b/requirements/idea-sdk.in @@ -0,0 +1,34 @@ +semver +mypy +pyparsing +pytz +tzlocal +arrow +pydantic +PyYAML +orjson +Click +botocore +boto3==1.26.61 +requests +requests-aws4auth +requests-unixsocket +questionary +rich +pyhocon +sh +cacheout +validators +fastcounter +psutil +blinker +troposphere +opensearch-py +Jinja2 +cryptography +PyJWT +random-password-generator +shortuuid +dataset +prometheus-client +python-dynamodb-lock diff --git a/requirements/idea-virtual-desktop-controller.in b/requirements/idea-virtual-desktop-controller.in new file mode 100644 index 00000000..9363d358 --- /dev/null +++ b/requirements/idea-virtual-desktop-controller.in @@ -0,0 +1,3 @@ +-r idea-sdk.in +supervisor +sanic==22.3.2 diff --git a/requirements/idea-virtual-desktop-controller.txt b/requirements/idea-virtual-desktop-controller.txt new file mode 100644 index 00000000..45c8be13 --- /dev/null +++ b/requirements/idea-virtual-desktop-controller.txt @@ -0,0 +1,73 @@ +aiofiles==0.8.0 +alembic==1.8.0 +arrow==1.2.1 +banal==1.0.6 +blinker==1.4 +boto3==1.26.61 +botocore==1.29.61 +cacheout==0.13.1 +certifi==2022.5.18 +cffi==1.15.0 +cfn-flip==1.3.0 +charset-normalizer==2.0.12 +click==8.1.3 +commonmark==0.9.1 +cryptography==36.0.1 +dataset==1.5.2 +decorator==5.1.1 +fastcounter==1.1.0 +greenlet==1.1.2 +httptools==0.4.0 +idna==3.3 +jinja2==3.1.2 +jmespath==1.0.0 +mako==1.2.1 +markupsafe==2.1.1 +multidict==6.0.2 +mypy==0.950 +mypy-extensions==0.4.3 +opensearch-py==2.0.0 +orjson==3.6.5 +prometheus-client==0.14.1 +prompt-toolkit==3.0.29 +psutil==5.9.0 +pycparser==2.21 +pydantic==1.9.0 +pygments==2.12.0 +pyhocon==0.3.59 +pyjwt==2.4.0 +pyparsing==2.4.7 +python-dateutil==2.8.2 +python-dynamodb-lock==0.9.1 +pytz==2022.1 +pytz-deprecation-shim==0.1.0.post0 +pyyaml==6.0 +questionary==1.10.0 +random-password-generator==2.2.0 +requests==2.27.1 +requests-aws4auth==1.1.2 +requests-unixsocket==0.3.0 +rich==12.4.1 +s3transfer==0.6.0 +sanic==22.3.2 +sanic-routing==22.3.0 +semver==2.13.0 +sh==1.14.2 +shortuuid==1.0.9 +six==1.16.0 +sqlalchemy==1.4.39 +supervisor==4.2.4 +tomli==2.0.1 +troposphere==4.3.0 +typing-extensions==4.2.0 +tzdata==2022.1 +tzlocal==4.2 +ujson==5.7.0 +urllib3==1.26.9 +uvloop==0.16.0 +validators==0.19.0 +wcwidth==0.2.5 +websockets==10.3 + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/requirements/tests.in b/requirements/tests.in new file mode 100644 index 00000000..c13095c7 --- /dev/null +++ b/requirements/tests.in @@ -0,0 +1,5 @@ +pytest +pytest-mock +pytest-cov +memory_profiler +defusedxml diff --git a/requirements/tests.txt b/requirements/tests.txt new file mode 100644 index 00000000..53af7b57 --- /dev/null +++ b/requirements/tests.txt @@ -0,0 +1,14 @@ +attrs==21.4.0 +coverage[toml]==6.5.0 +defusedxml==0.7.1 +iniconfig==1.1.1 +memory-profiler==0.60.0 +packaging==21.3 +pluggy==1.0.0 +psutil==5.9.0 +py==1.11.0 +pyparsing==3.0.9 +pytest==7.1.2 +pytest-cov==4.0.0 +pytest-mock==3.10.0 +tomli==2.0.1 diff --git a/software_versions.yml b/software_versions.yml new file mode 100644 index 00000000..2f831656 --- /dev/null +++ b/software_versions.yml @@ -0,0 +1,4 @@ +aws_cdk_version: 2.63.0 +node_version: 16.10.0 +nvm_version: 0.39.0 +python_version: 3.9.16 diff --git a/source/idea/idea-administrator/install/install.sh b/source/idea/idea-administrator/install/install.sh new file mode 100644 index 00000000..ac4af09f --- /dev/null +++ b/source/idea/idea-administrator/install/install.sh @@ -0,0 +1,60 @@ +#!/bin/bash + +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +# IDEA Administrator Installation Script + +IDEA_APP_DEPLOY_DIR="/root/.idea" +IDEA_CDK_VERSION="2.63.0" + +SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" + +IDEA_LIB_DIR="${IDEA_APP_DEPLOY_DIR}/lib" +IDEA_BIN_DIR="${IDEA_APP_DEPLOY_DIR}/bin" +IDEA_OLD_DIR="${IDEA_APP_DEPLOY_DIR}/old" +IDEA_DOWNLOADS_DIR="${IDEA_APP_DEPLOY_DIR}/downloads" +IDEA_CDK_DIR="${IDEA_LIB_DIR}/idea-cdk" + +exit_fail () { + echo -e "Installation Failed: $1" + exit 1 +} + +setup_deploy_dir () { + mkdir -p "${IDEA_APP_DEPLOY_DIR}" + mkdir -p "${IDEA_LIB_DIR}" + mkdir -p "${IDEA_BIN_DIR}" + mkdir -p "${IDEA_DOWNLOADS_DIR}" + mkdir -p "${IDEA_LIB_DIR}" + mkdir -p "${IDEA_OLD_DIR}" +} + +check_and_install_cdk () { + echo "installing aws-cdk for idea ..." + mkdir -p "${IDEA_CDK_DIR}" + pushd "${IDEA_CDK_DIR}" + npm init --force --yes + npm install "aws-cdk@${IDEA_CDK_VERSION}" --save + popd +} + +function install () { + setup_deploy_dir + check_and_install_cdk + pip install --default-timeout=100 -r ${SCRIPT_DIR}/requirements.txt + pip install $(ls ${SCRIPT_DIR}/*-lib.tar.gz) + local APP_DIR="${IDEA_APP_DEPLOY_DIR}/idea-administrator" + mkdir -p "${APP_DIR}" + mv ${SCRIPT_DIR}/resources ${APP_DIR} +} + +install \ No newline at end of file diff --git a/source/idea/idea-administrator/resources/cdk/cdk.json b/source/idea/idea-administrator/resources/cdk/cdk.json new file mode 100644 index 00000000..ae06edbd --- /dev/null +++ b/source/idea/idea-administrator/resources/cdk/cdk.json @@ -0,0 +1,11 @@ +{ + "app": "soca-admin cdk app", + "context": { + "aws-cdk:enableDiffNoFail": "true", + "@aws-cdk/core:stackRelativeExports": "true", + "@aws-cdk/aws-ecr-assets:dockerIgnoreSupport": true, + "@aws-cdk/aws-secretsmanager:parseOwnedSecretName": true, + "@aws-cdk/aws-kms:defaultKeyPolicies": true, + "@aws-cdk/aws-ecs-patterns:removeDefaultDesiredCount": true + } +} diff --git a/source/idea/idea-administrator/resources/cdk/cdk_toolkit_stack.yml b/source/idea/idea-administrator/resources/cdk/cdk_toolkit_stack.yml new file mode 100644 index 00000000..5fe31313 --- /dev/null +++ b/source/idea/idea-administrator/resources/cdk/cdk_toolkit_stack.yml @@ -0,0 +1,602 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +# IDEA CDK Toolkit CloudFormation Template +# +# * The original CDK toolkit template has been modified for IDEA requirements. +# * Original template can be viewed and compared by running: +# ~/.idea/lib/idea-cdk/node_modules/aws-cdk/bin/cdk bootstrap --show-template +# * All customizations are annotated with an IDEA Customization ID [CUSTOMIZATION_ID] for reference and additional documentation + +# Developer Notes: +# * The template is loaded as a Jinja2 template. +# * Customizations to the rendered template are not supported as the generated file is overwritten each time ./idea-admin.sh bootstrap is run. +# * Refer to source/idea/idea-administrator/src/ideaadministrator/app/cdk/cdk_invoker.py -> bootstrap_cluster() for implementation details. + +Description: 'CDK ToolKit Stack, Cluster: {{ cluster_name }}' +Parameters: + TrustedAccounts: + Description: List of AWS accounts that are trusted to publish assets and deploy stacks to this environment + Default: "" + Type: CommaDelimitedList + TrustedAccountsForLookup: + Description: List of AWS accounts that are trusted to look up values in this environment + Default: "" + Type: CommaDelimitedList + CloudFormationExecutionPolicies: + Description: List of the ManagedPolicy ARN(s) to attach to the CloudFormation deployment role + Default: "" + Type: CommaDelimitedList + FileAssetsBucketName: + Description: The name of the S3 bucket used for file assets + Default: "" + Type: String + FileAssetsBucketKmsKeyId: + Description: Empty to create a new key (default), 'AWS_MANAGED_KEY' to use a managed S3 key, or the ID/ARN of an existing key. + Default: "" + Type: String + ContainerAssetsRepositoryName: + Description: A user-provided custom name to use for the container assets ECR repository + Default: "" + Type: String + Qualifier: + Description: An identifier to distinguish multiple bootstrap stacks in the same environment + Default: hnb659fds + Type: String + AllowedPattern: "[A-Za-z0-9_-]{1,10}" + ConstraintDescription: Qualifier must be an alphanumeric identifier of at most 10 characters + PublicAccessBlockConfiguration: + Description: Whether or not to enable S3 Staging Bucket Public Access Block Configuration + Default: "true" + Type: String + AllowedValues: + - "true" + - "false" +Conditions: + HasTrustedAccounts: + Fn::Not: + - Fn::Equals: + - "" + - Fn::Join: + - "" + - Ref: TrustedAccounts + HasTrustedAccountsForLookup: + Fn::Not: + - Fn::Equals: + - "" + - Fn::Join: + - "" + - Ref: TrustedAccountsForLookup + HasCloudFormationExecutionPolicies: + Fn::Not: + - Fn::Equals: + - "" + - Fn::Join: + - "" + - Ref: CloudFormationExecutionPolicies + HasCustomFileAssetsBucketName: + Fn::Not: + - Fn::Equals: + - "" + - Ref: FileAssetsBucketName + CreateNewKey: + Fn::Equals: + - "" + - Ref: FileAssetsBucketKmsKeyId + UseAwsManagedKey: + Fn::Equals: + - AWS_MANAGED_KEY + - Ref: FileAssetsBucketKmsKeyId + HasCustomContainerAssetsRepositoryName: + Fn::Not: + - Fn::Equals: + - "" + - Ref: ContainerAssetsRepositoryName + UsePublicAccessBlockConfiguration: + Fn::Equals: + - "true" + - Ref: PublicAccessBlockConfiguration +Resources: + FileAssetsBucketEncryptionKey: + Type: AWS::KMS::Key + Properties: + KeyPolicy: + Statement: + - Action: + - kms:Create* + - kms:Describe* + - kms:Enable* + - kms:List* + - kms:Put* + - kms:Update* + - kms:Revoke* + - kms:Disable* + - kms:Get* + - kms:Delete* + - kms:ScheduleKeyDeletion + - kms:CancelKeyDeletion + - kms:GenerateDataKey + Effect: Allow + Principal: + AWS: + Ref: AWS::AccountId + Resource: "*" + - Action: + - kms:Decrypt + - kms:DescribeKey + - kms:Encrypt + - kms:ReEncrypt* + - kms:GenerateDataKey* + Effect: Allow + Principal: + AWS: "*" + Resource: "*" + Condition: + StringEquals: + kms:CallerAccount: + Ref: AWS::AccountId + kms:ViaService: + - Fn::Sub: s3.${AWS::Region}.{{ aws_dns_suffix }} + - Action: + - kms:Decrypt + - kms:DescribeKey + - kms:Encrypt + - kms:ReEncrypt* + - kms:GenerateDataKey* + Effect: Allow + Principal: + AWS: + Fn::Sub: ${FilePublishingRole.Arn} + Resource: "*" + Condition: CreateNewKey + FileAssetsBucketEncryptionKeyAlias: + Condition: CreateNewKey + Type: AWS::KMS::Alias + Properties: + AliasName: + Fn::Sub: alias/cdk-${Qualifier}-assets-key + TargetKeyId: + Ref: FileAssetsBucketEncryptionKey + StagingBucket: + Type: AWS::S3::Bucket + Properties: + BucketName: + Fn::If: + - HasCustomFileAssetsBucketName + - Fn::Sub: ${FileAssetsBucketName} + - Fn::Sub: cdk-${Qualifier}-assets-${AWS::AccountId}-${AWS::Region} + AccessControl: Private + BucketEncryption: + ServerSideEncryptionConfiguration: + # [3] IDEA customization for bucket encryption to use SSE-S3 instead of SSE-KMS. + # * AWS ALB Access logs do not support SSE-KMS + # * NLB Access Logs needs additional policy configuration to CMK in AWS KMS so that NLB service can use the key to encrypt logs + # * the lowest common denominator is to enable SSE-S3 so that ALB + NLB access logs can work in default configuration + # * due to this change, the cdk bootstrap parameters: --bootstrap-kms-key-id and --bootstrap-customer-key are not supported in default configuration. + # * SIs and ISVs are expected to customize this template based on their requirements to deploy using Customer Managed KMS Keys. + # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-s3-bucket-serversideencryptionrule.html + - ServerSideEncryptionByDefault: + SSEAlgorithm: AES256 + # SSEAlgorithm: aws:kms + # KMSMasterKeyID: + # Fn::If: + # - CreateNewKey + # - Fn::Sub: ${FileAssetsBucketEncryptionKey.Arn} + # - Fn::If: + # - UseAwsManagedKey + # - Ref: AWS::NoValue + # - Fn::Sub: ${FileAssetsBucketKmsKeyId} + PublicAccessBlockConfiguration: + Fn::If: + - UsePublicAccessBlockConfiguration + - BlockPublicAcls: true + BlockPublicPolicy: true + IgnorePublicAcls: true + RestrictPublicBuckets: true + - Ref: AWS::NoValue + + VersioningConfiguration: + Status: Enabled + + # [5] Tags to enable AWS Backup for Cluster S3 Bucket + Tags: + - Key: "idea:ClusterName" + Value: "{{ cluster_name }}" + - Key: "idea:BackupPlan" + Value: "cluster" + UpdateReplacePolicy: Retain + DeletionPolicy: Retain + StagingBucketPolicy: + Type: AWS::S3::BucketPolicy + Properties: + Bucket: + Ref: StagingBucket + PolicyDocument: + Id: AccessControl + Version: "2012-10-17" + Statement: + - Sid: AllowSSLRequestsOnly + Action: s3:* + Effect: Deny + Resource: + - Fn::Sub: ${StagingBucket.Arn} + - Fn::Sub: ${StagingBucket.Arn}/* + Condition: + Bool: + aws:SecureTransport: "false" + Principal: "*" + + ### Begin: IDEA Cluster S3 Bucket Policy Customizations + + # [1] ALB Access Logs - AwsSolutions-ELB2 + # refer: https://docs.aws.amazon.com/elasticloadbalancing/latest/application/enable-access-logging.html + - Sid: IdeaAlbAccessLogs + Effect: Allow + Principal: + {% if aws_elb_account_id %} + AWS: + - Fn::Sub: arn:${AWS::Partition}:iam::{{ aws_elb_account_id }}:root + {% else %} + Service: logdelivery.elasticloadbalancing.{{ aws_dns_suffix }} + {% endif %} + Action: + - s3:PutObject + Resource: + - Fn::Sub: ${StagingBucket.Arn}/logs/* + + # [2] NLB Access Logs + # refer: https://docs.aws.amazon.com/elasticloadbalancing/latest/network/load-balancer-access-logs.html + - Sid: IdeaNlbAccessLogs-AWSLogDeliveryWrite + Effect: Allow + Principal: + Service: delivery.logs.{{ aws_dns_suffix }} + Action: + - s3:PutObject + Resource: + - Fn::Sub: ${StagingBucket.Arn}/logs/* + Condition: + StringEquals: + s3:x-amz-acl: bucket-owner-full-control + - Sid: IdeaNlbAccessLogs-AWSLogDeliveryAclCheck + Effect: Allow + Principal: + Service: delivery.logs.{{ aws_dns_suffix }} + Action: + - s3:GetBucketAcl + Resource: + - Fn::Sub: ${StagingBucket.Arn} + + ### End: IDEA Cluster S3 Bucket Policy Customizations + ContainerAssetsRepository: + Type: AWS::ECR::Repository + Properties: + ImageTagMutability: IMMUTABLE + RepositoryName: + Fn::If: + - HasCustomContainerAssetsRepositoryName + - Fn::Sub: ${ContainerAssetsRepositoryName} + - Fn::Sub: cdk-${Qualifier}-container-assets-${AWS::AccountId}-${AWS::Region} + RepositoryPolicyText: + Version: "2012-10-17" + Statement: + - Sid: LambdaECRImageRetrievalPolicy + Effect: Allow + Principal: + Service: lambda.{{ aws_dns_suffix }} + Action: + - ecr:BatchGetImage + - ecr:GetDownloadUrlForLayer + Condition: + StringLike: + aws:sourceArn: + Fn::Sub: arn:${AWS::Partition}:lambda:${AWS::Region}:${AWS::AccountId}:function:* + FilePublishingRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Statement: + - Action: sts:AssumeRole + Effect: Allow + Principal: + AWS: + Ref: AWS::AccountId + - Fn::If: + - HasTrustedAccounts + - Action: sts:AssumeRole + Effect: Allow + Principal: + AWS: + Ref: TrustedAccounts + - Ref: AWS::NoValue + RoleName: + Fn::Sub: cdk-${Qualifier}-file-publishing-role-${AWS::AccountId}-${AWS::Region} + Tags: + - Key: aws-cdk:bootstrap-role + Value: file-publishing + ImagePublishingRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Statement: + - Action: sts:AssumeRole + Effect: Allow + Principal: + AWS: + Ref: AWS::AccountId + - Fn::If: + - HasTrustedAccounts + - Action: sts:AssumeRole + Effect: Allow + Principal: + AWS: + Ref: TrustedAccounts + - Ref: AWS::NoValue + RoleName: + Fn::Sub: cdk-${Qualifier}-image-publishing-role-${AWS::AccountId}-${AWS::Region} + Tags: + - Key: aws-cdk:bootstrap-role + Value: image-publishing + LookupRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Statement: + - Action: sts:AssumeRole + Effect: Allow + Principal: + AWS: + Ref: AWS::AccountId + - Fn::If: + - HasTrustedAccountsForLookup + - Action: sts:AssumeRole + Effect: Allow + Principal: + AWS: + Ref: TrustedAccountsForLookup + - Ref: AWS::NoValue + - Fn::If: + - HasTrustedAccounts + - Action: sts:AssumeRole + Effect: Allow + Principal: + AWS: + Ref: TrustedAccounts + - Ref: AWS::NoValue + RoleName: + Fn::Sub: cdk-${Qualifier}-lookup-role-${AWS::AccountId}-${AWS::Region} + ManagedPolicyArns: + - Fn::Sub: arn:${AWS::Partition}:iam::aws:policy/ReadOnlyAccess + Policies: + - PolicyDocument: + Statement: + - Sid: DontReadSecrets + Effect: Deny + Action: + - kms:Decrypt + Resource: "*" + Version: "2012-10-17" + PolicyName: LookupRolePolicy + Tags: + - Key: aws-cdk:bootstrap-role + Value: lookup + FilePublishingRoleDefaultPolicy: + Type: AWS::IAM::Policy + Properties: + PolicyDocument: + Statement: + - Action: + - s3:GetObject* + - s3:GetBucket* + - s3:GetEncryptionConfiguration + - s3:List* + - s3:DeleteObject* + - s3:PutObject* + - s3:Abort* + Resource: + - Fn::Sub: ${StagingBucket.Arn} + - Fn::Sub: ${StagingBucket.Arn}/* + Effect: Allow + - Action: + - kms:Decrypt + - kms:DescribeKey + - kms:Encrypt + - kms:ReEncrypt* + - kms:GenerateDataKey* + Effect: Allow + Resource: + Fn::If: + - CreateNewKey + - Fn::Sub: ${FileAssetsBucketEncryptionKey.Arn} + - Fn::Sub: arn:${AWS::Partition}:kms:${AWS::Region}:${AWS::AccountId}:key/${FileAssetsBucketKmsKeyId} + Version: "2012-10-17" + Roles: + - Ref: FilePublishingRole + PolicyName: + Fn::Sub: cdk-${Qualifier}-file-publishing-role-default-policy-${AWS::AccountId}-${AWS::Region} + ImagePublishingRoleDefaultPolicy: + Type: AWS::IAM::Policy + Properties: + PolicyDocument: + Statement: + - Action: + - ecr:PutImage + - ecr:InitiateLayerUpload + - ecr:UploadLayerPart + - ecr:CompleteLayerUpload + - ecr:BatchCheckLayerAvailability + - ecr:DescribeRepositories + - ecr:DescribeImages + - ecr:BatchGetImage + - ecr:GetDownloadUrlForLayer + Resource: + Fn::Sub: ${ContainerAssetsRepository.Arn} + Effect: Allow + - Action: + - ecr:GetAuthorizationToken + Resource: "*" + Effect: Allow + Version: "2012-10-17" + Roles: + - Ref: ImagePublishingRole + PolicyName: + Fn::Sub: cdk-${Qualifier}-image-publishing-role-default-policy-${AWS::AccountId}-${AWS::Region} + DeploymentActionRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Statement: + - Action: sts:AssumeRole + Effect: Allow + Principal: + AWS: + Ref: AWS::AccountId + - Fn::If: + - HasTrustedAccounts + - Action: sts:AssumeRole + Effect: Allow + Principal: + AWS: + Ref: TrustedAccounts + - Ref: AWS::NoValue + Policies: + - PolicyDocument: + Statement: + - Sid: CloudFormationPermissions + Effect: Allow + Action: + - cloudformation:CreateChangeSet + - cloudformation:DeleteChangeSet + - cloudformation:DescribeChangeSet + - cloudformation:DescribeStacks + - cloudformation:ExecuteChangeSet + - cloudformation:CreateStack + - cloudformation:UpdateStack + Resource: "*" + - Sid: PipelineCrossAccountArtifactsBucket + Effect: Allow + Action: + - s3:GetObject* + - s3:GetBucket* + - s3:List* + - s3:Abort* + - s3:DeleteObject* + - s3:PutObject* + Resource: "*" + Condition: + StringNotEquals: + s3:ResourceAccount: + Ref: AWS::AccountId + - Sid: PipelineCrossAccountArtifactsKey + Effect: Allow + Action: + - kms:Decrypt + - kms:DescribeKey + - kms:Encrypt + - kms:ReEncrypt* + - kms:GenerateDataKey* + Resource: "*" + Condition: + StringEquals: + kms:ViaService: + Fn::Sub: s3.${AWS::Region}.{{ aws_dns_suffix }} + - Action: iam:PassRole + Resource: + Fn::Sub: ${CloudFormationExecutionRole.Arn} + Effect: Allow + - Sid: CliPermissions + Action: + - cloudformation:DescribeStackEvents + - cloudformation:GetTemplate + - cloudformation:DeleteStack + - cloudformation:UpdateTerminationProtection + - sts:GetCallerIdentity + - cloudformation:GetTemplateSummary + Resource: "*" + Effect: Allow + - Sid: CliStagingBucket + Effect: Allow + Action: + - s3:GetObject* + - s3:GetBucket* + - s3:List* + Resource: + - Fn::Sub: ${StagingBucket.Arn} + - Fn::Sub: ${StagingBucket.Arn}/* + - Sid: ReadVersion + Effect: Allow + Action: + - ssm:GetParameter + Resource: + - Fn::Sub: arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter${CdkBootstrapVersion} + Version: "2012-10-17" + PolicyName: default + RoleName: + Fn::Sub: cdk-${Qualifier}-deploy-role-${AWS::AccountId}-${AWS::Region} + Tags: + - Key: aws-cdk:bootstrap-role + Value: deploy + CloudFormationExecutionRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Statement: + - Action: sts:AssumeRole + Effect: Allow + Principal: + Service: cloudformation.{{ aws_dns_suffix }} + Version: "2012-10-17" + ManagedPolicyArns: + Fn::If: + - HasCloudFormationExecutionPolicies + - Ref: CloudFormationExecutionPolicies + - Fn::If: + - HasTrustedAccounts + - Ref: AWS::NoValue + - - Fn::Sub: arn:${AWS::Partition}:iam::aws:policy/AdministratorAccess + RoleName: + Fn::Sub: cdk-${Qualifier}-cfn-exec-role-${AWS::AccountId}-${AWS::Region} + CdkBootstrapVersion: + Type: AWS::SSM::Parameter + Properties: + Type: String + Name: + Fn::Sub: /cdk-bootstrap/${Qualifier}/version + Value: "14" +Outputs: + BucketName: + Description: The name of the S3 bucket owned by the CDK toolkit stack + Value: + Fn::Sub: ${StagingBucket} + BucketDomainName: + Description: The domain name of the S3 bucket owned by the CDK toolkit stack + Value: + Fn::Sub: ${StagingBucket.RegionalDomainName} + FileAssetKeyArn: + Description: The ARN of the KMS key used to encrypt the asset bucket (deprecated) + Value: + Fn::If: + - CreateNewKey + - Fn::Sub: ${FileAssetsBucketEncryptionKey.Arn} + - Fn::Sub: ${FileAssetsBucketKmsKeyId} + Export: + Name: + Fn::Sub: CdkBootstrap-${Qualifier}-FileAssetKeyArn + ImageRepositoryName: + Description: The name of the ECR repository which hosts docker image assets + Value: + Fn::Sub: ${ContainerAssetsRepository} + BootstrapVersion: + Description: The version of the bootstrap resources that are currently mastered in this stack + Value: + Fn::GetAtt: + - CdkBootstrapVersion + - Value + diff --git a/source/idea/idea-administrator/resources/config/region_ami_config.yml b/source/idea/idea-administrator/resources/config/region_ami_config.yml new file mode 100644 index 00000000..0d450d27 --- /dev/null +++ b/source/idea/idea-administrator/resources/config/region_ami_config.yml @@ -0,0 +1,85 @@ +af-south-1: + amazonlinux2: ami-073f101842e8ceae7 + centos7: ami-0b761332115c38669 + rhel7: ami-03bc1929bae8e8d10 +ap-east-1: + amazonlinux2: ami-0dff3691c13e5881d + centos7: ami-09611bd6fa5dd0e3d + rhel7: ami-09871f5062b616ac8 +ap-northeast-1: + amazonlinux2: ami-0387b0d09183a3a97 + centos7: ami-0ddea5e0f69c193a4 + rhel7: ami-00e3b125d72527ff6 +ap-northeast-2: + amazonlinux2: ami-030656986ed2d9f00 + centos7: ami-0e4214f08b51e23cc + rhel7: ami-0f878d6caa1fa98f0 +ap-south-1: + amazonlinux2: ami-08c6724c280604575 + centos7: ami-0ffc7af9c06de0077 + rhel7: ami-024685afee5678595 +ap-southeast-1: + amazonlinux2: ami-049f20cccc294bb90 + centos7: ami-0adfdaea54d40922b + rhel7: ami-057fd2d861be8fb5e +ap-southeast-2: + amazonlinux2: ami-0b2eed9bc374d87a9 + centos7: ami-03d56f451ca110e99 + rhel7: ami-0343583dce592b33a +ap-southeast-3: + amazonlinux2: ami-0280cd826703b26b9 +ca-central-1: + amazonlinux2: ami-029758462acdf767c + centos7: ami-0a7c5b189b6460115 + rhel7: ami-02cc622b97f7e8d45 +eu-central-1: + amazonlinux2: ami-0d10f2d3b9ab936a1 + centos7: ami-08b6d44b4f6f7b279 + rhel7: ami-0f0fc6bdd397422dd +eu-north-1: + amazonlinux2: ami-0854c75c979835223 + centos7: ami-0358414bac2039369 + rhel7: ami-0b844bbd294a8e075 +eu-south-1: + amazonlinux2: ami-02fb9ea0a3dc1e298 + centos7: ami-0fe3899b62205176a + rhel7: ami-0533b4c2e62e8bfb5 +eu-west-1: + amazonlinux2: ami-0b0bf695cabdc2ce8 + centos7: ami-04f5641b0d178a27a + rhel7: ami-002d3240b69a0ef4e +eu-west-2: + amazonlinux2: ami-0619177a5b68d29e3 + centos7: ami-0b22fcaf3564fb0c9 + rhel7: ami-045151e990cd5bf35 +eu-west-3: + amazonlinux2: ami-092c29a186204ba09 + centos7: ami-072ec828dae86abe5 + rhel7: ami-025295ed8743be8fd +me-south-1: + amazonlinux2: ami-0aa583da61ed90680 + centos7: ami-0ac17dcdd6f6f4eb6 + rhel7: ami-0ba9dbb5f12f9e19b +sa-east-1: + amazonlinux2: ami-0fbb0781e8c7f140a + centos7: ami-02334c45dd95ca1fc + rhel7: ami-08d0639f173f8d91c +us-east-1: + amazonlinux2: ami-00db75007d6c5c578 + centos7: ami-00e87074e52e6c9f9 + rhel7: ami-0051b1b2c5a166c8c +us-east-2: + amazonlinux2: ami-0489c6c0a2c0b6281 + centos7: ami-00f8e2c955f7ffa9b + rhel7: ami-0c1c3220d0b1716d2 +us-gov-west-1: + amazonlinux2: ami-0929c04d678a107cc + rhel7: ami-02b48c6842582f942 +us-west-1: + amazonlinux2: ami-01cf9b8078ea41f1b + centos7: ami-08d2d8b00f270d03b + rhel7: ami-05ca2e876e4d5669a +us-west-2: + amazonlinux2: ami-012363a297a261d65 + centos7: ami-0686851c4e7b1a8e1 + rhel7: ami-068fd644a9270e323 diff --git a/source/idea/idea-administrator/resources/config/region_elb_account_id.yml b/source/idea/idea-administrator/resources/config/region_elb_account_id.yml new file mode 100644 index 00000000..60e075be --- /dev/null +++ b/source/idea/idea-administrator/resources/config/region_elb_account_id.yml @@ -0,0 +1,31 @@ +# Configuration to enable ALB access logs policy for cluster s3 bucket +# * For regions, where aws elb account id is not available, logdelivery.elasticloadbalancing.amazonaws.com will be used as service principal. +# * AWS Outpost are not supported. +# * refer to implementation in CdkInvoker -> bootstrap_cluster() for implementation details +# * refer: https://docs.aws.amazon.com/elasticloadbalancing/latest/application/enable-access-logging.html for more details + +us-east-1: 127311923021 +us-east-2: 033677994240 +us-west-1: 027434742980 +us-west-2: 797873946194 +af-south-1: 098369216593 +ap-east-1: 754344448648 +ap-southeast-3: 589379963580 +ap-south-1: 718504428378 +ap-northeast-3: 383597477331 +ap-northeast-2: 600734575887 +ap-southeast-1: 114774131450 +ap-southeast-2: 783225319266 +ap-northeast-1: 582318560864 +ca-central-1: 985666609251 +eu-central-1: 054676820928 +eu-west-1: 156460612806 +eu-west-2: 652711504416 +eu-south-1: 635631232127 +eu-west-3: 009996457667 +eu-north-1: 897822967062 +me-south-1: 076674570225 +sa-east-1: 507241528517 +us-gov-east-1: 190560391635 +us-gov-west-1: 048591011584 + diff --git a/source/idea/idea-administrator/resources/config/region_timezone_config.yml b/source/idea/idea-administrator/resources/config/region_timezone_config.yml new file mode 100644 index 00000000..a26263cc --- /dev/null +++ b/source/idea/idea-administrator/resources/config/region_timezone_config.yml @@ -0,0 +1,24 @@ +af-south-1: Africa/Johannesburg +ap-east-1: Asia/Hong_Kong +ap-northeast-1: Asia/Tokyo +ap-northeast-2: Asia/Seoul +ap-south-1: Asia/Kolkata +ap-southeast-1: Asia/Singapore +ap-southeast-2: Australia/Sydney +ap-southeast-3: Asia/Jakarta +ca-central-1: America/Regina +eu-central-1: Europe/Berlin +eu-north-1: Europe/Stockholm +eu-south-1: Europe/Rome +eu-west-1: Europe/Dublin +eu-west-2: Europe/London +eu-west-3: Europe/Paris +me-south-1: Asia/Bahrain +sa-east-1: America/Sao_Paulo +us-east-1: America/New_York +us-east-2: America/New_York +us-west-1: America/Los_Angeles +us-west-2: America/Los_Angeles +us-gov-west-1: America/Los_Angeles +us-gov-east-1: America/New_York +default: America/Los_Angeles diff --git a/source/idea/idea-administrator/resources/config/templates/analytics/settings.yml b/source/idea/idea-administrator/resources/config/templates/analytics/settings.yml new file mode 100644 index 00000000..e174f00f --- /dev/null +++ b/source/idea/idea-administrator/resources/config/templates/analytics/settings.yml @@ -0,0 +1,27 @@ +# Configure your AWS OpenSearch/Kibana options below +opensearch: + use_existing: {{use_existing_opensearch_cluster | lower}} + {%- if use_existing_opensearch_cluster %} + domain_vpc_endpoint_url: "{{opensearch_domain_endpoint}}" + {%- else %} + data_node_instance_type: "m5.large.search" # instance type for opensearch data nodes + data_nodes: 2 # number of data nodes for elasticsearch + ebs_volume_size: 100 # ebs volume size attached to data nodes + removal_policy: "DESTROY" # RETAIN will preserve the cluster even if you delete the stack. + node_to_node_encryption: true + logging: + app_log_enabled: true # Specify if Amazon OpenSearch Service application logging should be set up. + slow_index_log_enabled: true # Log Amazon OpenSearch Service audit logs to this log group + slow_search_log_enabled: true # Specify if slow search logging should be set up. + {%- endif %} + default_number_of_shards: 2 + default_number_of_replicas: 1 + + endpoints: + external: + priority: 16 + path_patterns: ['/_dashboards*'] + +kinesis: + shard_count: 2 + stream_mode: PROVISIONED diff --git a/source/idea/idea-administrator/resources/config/templates/bastion-host/settings.yml b/source/idea/idea-administrator/resources/config/templates/bastion-host/settings.yml new file mode 100644 index 00000000..9bef7ce7 --- /dev/null +++ b/source/idea/idea-administrator/resources/config/templates/bastion-host/settings.yml @@ -0,0 +1,17 @@ +public: {{ alb_public | lower }} +instance_type: {{instance_type}} +base_os: {{base_os}} +instance_ami: {{instance_ami}} +volume_size: {{volume_size}} +hostname: "{{module_id}}.{{cluster_name}}.{{aws_region}}.local" + +cloudwatch_logs: + enabled: true + +ec2: + # enable detailed monitoring for bastion-host ec2 instance, disabled by default + enable_detailed_monitoring: false + # enable termination protection for bastion-host ec2 instance, enabled by default + enable_termination_protection: true + # instance metadata access method + metadata_http_tokens: "required" # supported values are "required" for IMDSv2 or "optional" for IMDSv1 \ No newline at end of file diff --git a/source/idea/idea-administrator/resources/config/templates/cluster-manager/settings.yml b/source/idea/idea-administrator/resources/config/templates/cluster-manager/settings.yml new file mode 100644 index 00000000..b6aad4b5 --- /dev/null +++ b/source/idea/idea-administrator/resources/config/templates/cluster-manager/settings.yml @@ -0,0 +1,102 @@ +# Cluster Manager Settings + +endpoints: + external: + priority: 12 + path_patterns: ['/{{ module_id }}/*'] + internal: + priority: 12 + path_patterns: ['/{{ module_id }}/*'] + +server: + enable_http: true + hostname: 0.0.0.0 + port: 8443 + enable_tls: true + tls_certificate_file: {{apps_mount_dir}}/{{cluster_name}}/certs/idea.crt + tls_key_file: {{apps_mount_dir}}/{{cluster_name}}/certs/idea.key + enable_unix_socket: true + unix_socket_file: /run/idea.sock + max_workers: 16 + enable_metrics: false + graceful_shutdown_timeout: 10 + api_context_path: /{{ module_id }} + web_resources_context_path: / + +web_portal: + title: 'Integrated Digital Engineering on AWS' + logo: ~ + subtitle: ~ + copyright_text: 'Copyright {year} Amazon Inc. or its affiliates. All Rights Reserved.' + + # session management refers to client side session management + # valid values are one of: [in-memory, local-storage] + # * in-memory: implies access tokens and refresh tokens will be saved in-memory within ServerWorker instance in WebBrowser. (recommended) + # * local-storage: implies access tokens and refresh tokens will be saved in browser local storage. + # Note: ServiceWorkers cannot be activated on Browsers when serving over insecure HTTPS/TLS context. Unless you are using valid TLS certs, ServiceWorkers will not be activated. + # Web Portal implementation will fall back to local-storage for session management when ServiceWorker cannot be installed. + # Change below value to local-storage only if you want to disable ServiceWorker based session management. + session_management: in-memory + + # front end default log level + # 0: OFF + # 1: ERROR + # 2: WARN + # 3: INFO + # 4: DEBUG + # 5: TRACE + # log level can be overriden by user by setting the `idea.log-level` local storage key using browser console + # eg. localStorage.setItem('idea.log-level', '4') + default_log_level: 3 + +oauth2_client: + # cluster manager OAuth 2.0 client is used for managing authentication for web-portal (when single sign-on is disabled) + # set the below to an appropriate value based on your requirements and security posture. + # recommended value is between 12 and 24 hours. web-portal session will expire and user will need to log in again after this interval. + refresh_token_validity_hours: 12 + +logging: + logs_directory: /opt/idea/app/logs + profile: production + default_log_file_name: application.log + +cloudwatch_logs: + enabled: true + +ec2: + autoscaling: + public: false + instance_type: {{instance_type}} + base_os: {{base_os}} + instance_ami: {{instance_ami}} + volume_size: {{volume_size}} + enable_detailed_monitoring: false + min_capacity: 1 + max_capacity: 3 + cooldown_minutes: 5 + new_instances_protected_from_scale_in: false + elb_healthcheck: + # Specifies the time in minutes Auto Scaling waits before checking the health status of an EC2 instance that has come into service. + grace_time_minutes: 15 + cpu_utilization_scaling_policy: + target_utilization_percent: 80 + estimated_instance_warmup_minutes: 15 + rolling_update_policy: + max_batch_size: 1 + min_instances_in_service: 1 + pause_time_minutes: 15 + # instance metadata access method + metadata_http_tokens: "required" # supported values are "required" for IMDSv2 or "optional" for IMDSv1 + +cache: + long_term: + max_size: 1000 + ttl_seconds: 86400 # 1 day + short_term: + max_size: 10000 + ttl_seconds: 600 # 10 minutes + +notifications: + # email notifications are supported at the moment. slack, sms and other channels will be supported in a future release. + email: + enabled: true diff --git a/source/idea/idea-administrator/resources/config/templates/cluster/logging.yml b/source/idea/idea-administrator/resources/config/templates/cluster/logging.yml new file mode 100644 index 00000000..5d173ea1 --- /dev/null +++ b/source/idea/idea-administrator/resources/config/templates/cluster/logging.yml @@ -0,0 +1,119 @@ +# logging configuration for IDEA supported application servers + +logging: + + # these are python supported log formatters. you can customize these as per your needs + # customize these and change the profile as per your needs + formatters: + default: + format: "[%(asctime)s] [%(levelname)s] [%(name)s] %(message)s" + + # these are python supported log handlers. + handlers: + + console: + class: logging.handlers.StreamHandler + + file: + + # see: https://docs.python.org/3/library/logging.handlers.html#timedrotatingfilehandler for more details. + # TimedRotatingFileHandler documentation is copied/sourced from above link for easy reference. + # Attributes: atTime, delay and errors are not supported at the moment. + + class: logging.handlers.TimedRotatingFileHandler + + # Rotating happens based on the product of "when" and "interval" + interval: 1 + + # You can use the when to specify the type of interval. + # The list of possible values is below: + # > S - Seconds + # > M - Minutes + # > H - Hours + # > D - Days + # > W0-W6 - Weekday (0=Monday) + # > midnight - Roll over at midnight. + when: midnight + + # If backupCount is nonzero, at most backupCount files will be kept, and if more would be created + # when rollover occurs, the oldest one is deleted. The deletion logic uses the interval to determine + # which files to delete, so changing the interval may leave old files lying around. + backupCount: 15 + + + # logging profiles enables managing presets for logging configurations + profiles: + + # * console profile: + # > use this profile during development or for cli apps. DEBUG logs are enabled and all logs are routed + # to console/stdout only + # > this profile should NOT be used in production application server deployments + console: + formatter: default + loggers: + app: + level: DEBUG + handlers: + - console + root: + level: WARNING + handlers: + - console + + # * production profile: + # > use this profile for production environments + # > INFO and file based logging is enabled for this profile + production: + formatter: default + loggers: + app: + level: INFO + handlers: + - file + root: + level: WARNING + handlers: + - file + + # * production profile: + # > use this profile for to enable DEBUG logging on production environments + # > limit usage of this profile in production environments and change the profile back to production once the issue is resolved. + # > DEBUG and file based logging is enabled for this profile + debug: + formatter: default + loggers: + app: + level: DEBUG + handlers: + - file + root: + level: WARNING + handlers: + - file + + # Audit log settings for IDEA application servers + # API invocation request/response logs include: + # Who (Actor), What (API Namespace) and When (Timestamp) and Why (Authorization Type) + # log entries are of the below format: + # API Request: + # (req) [actor:|auth_type:|request_id:|client_id:] [API Namespace | JSON Payload] + # API Response: + # (req) [actor:|auth_type:|request_id:|client_id:] [API Namespace | JSON Payload] ( ms) [OK|] + # auditing tags below can be enabled to log additional information. + # Developer Note: + # * audit logging framework does not cover the target entity. eg. User, Job, Session etc. + # * The underlying API implementation can/should log specifics about the target entity in the API + # using the logging methods in ApiInvocationContext to preserve and log with auditing context/tags + audit_logs: + # indicate if the request / response json payload should be logged instead of just the API Namespace. + # enabling this setting to true will significantly increase the amount of logs generated (and ingested into CloudWatch logs if enabled/or applicable) + # if the cluster has significantly high traffic, consider evaluating the EBS Volume size and log rotation settings to avoid disk utilization, performance degradation problems + enable_payload_tracing: false + + # enable/disable additional tags in audit log context. + # enabling client_id, request_id may impact log verbosity and cloudwatch log ingestion volume + tags: + - actor + - auth_type + # - client_id + # - request_id diff --git a/source/idea/idea-administrator/resources/config/templates/cluster/settings.yml b/source/idea/idea-administrator/resources/config/templates/cluster/settings.yml new file mode 100644 index 00000000..4ec4ee85 --- /dev/null +++ b/source/idea/idea-administrator/resources/config/templates/cluster/settings.yml @@ -0,0 +1,243 @@ +administrator_email: "{{administrator_email}}" +administrator_username: "{{administrator_username}}" +cluster_name: "{{cluster_name}}" +cluster_s3_bucket: "{{cluster_name}}-cluster-{{aws_region}}-{{aws_account_id}}" +encoding: utf-8 +home_dir: "{{apps_mount_dir}}/{{cluster_name}}" +locale: "{{ cluster_locale }}" +timezone: "{{ cluster_timezone }}" + +aws: + account_id: "{{ aws_account_id }}" + dns_suffix: "{{ aws_dns_suffix }}" + partition: "{{ aws_partition }}" + pricing_region: us-east-1 + region: "{{ aws_region }}" + +ses: + enabled: false + account_id: "{{aws_account_id}}" + region: "{{aws_region}}" + sender_email: "{{administrator_email}}" + # The maximum number of emails that Amazon SES can accept from your account each second. + # You can exceed this quota for short bursts, but not for sustained periods of time. + # For more information, see: https://docs.aws.amazon.com/ses/latest/DeveloperGuide/manage-sending-quotas.html + max_sending_rate: 1 + +route53: + private_hosted_zone_name: "{{cluster_name}}.{{aws_region}}.local" + +network: + # You must specify the maximum number of entries for the prefix list. The maximum number of entries cannot be changed later. + cluster_prefix_list_max_entries: 10 + prefix_list_ids: + {{ utils.to_yaml(prefix_list_ids) | indent(4) }} + client_ip: + {{ utils.to_yaml(client_ip) | indent(4) }} + ssh_key_pair: "{{ssh_key_pair_name}}" + + {%- if use_existing_vpc %} + use_existing_vpc: true + vpc_id: "{{vpc_id}}" + + private_subnets: + {{ utils.to_yaml(private_subnet_ids) | indent(4) }} + public_subnets: + {{ utils.to_yaml(public_subnet_ids) | indent(4) }} + {%- else %} + max_azs: 3 + nat_gateways: 1 + subnet_config: + public: + cidr_mask: 26 + private: + cidr_mask: 18 + vpc_cidr_block: "{{vpc_cidr_block}}" + vpc_flow_logs: true + {%- endif %} + + use_vpc_endpoints: {{use_vpc_endpoints | lower}} + {%- if use_vpc_endpoints %} + # provide VPC Interface and Gateway Endpoints below if use_vpc_endpoints = True. + + # enabled: true/false config entry in interface endpoint can be used to specify if the endpoint can be used in the boto3 client across all IDEA modules. + + # For new VPCs: + # the cluster CDK stack will provision all endpoints specified in vpc_gateway_endpoints and vpc_interface_endpoints. + # Applicable VPC Endpoints for the region are automatically generated using vpc_endpoints_helper.py + # During upgrades, if an endpoint entry is removed, the VPC endpoint entry will be deleted by the cluster stack. + # enabling/disabling the config entry has no impact on provisioning of the VPC Interface endpoint, but is used during boto3 client configuration + + # For existing VPCs: + # It is admin responsibility to provide and configure all VPC endpoints in the config. + # VPC endpoints will NOT be provisioned by the Cluster CDK Stack. + # Format for providing VPC endpoints is as below: + # vpc_gateway_endpoints: + # - s3 + # - dynamodb + # vpc_interface_endpoints: + # ec2: + # enabled: true + # endpoint_url: https://endpoint-url + vpc_gateway_endpoints: + {{ utils.to_yaml(vpc_gateway_endpoints) | indent(4) }} + vpc_interface_endpoints: + {{ utils.to_yaml(vpc_interface_endpoints) | indent(4) }} + {%- endif %} + +# AWS Key Management Service +kms: + # can be one of [customer-managed, aws-managed] + key_type: {{ kms_key_type if kms_key_type else '~' }} + +# Configure cluster-wide AWS Secrets Manager settings below +secretsmanager: + kms_key_id: {{ kms_key_id if kms_key_id else '~' }} # Specify your own CMK to encrypt your Secret manager. If set to ~ encryption will be managed by the default AWS key + +# Configure cluster-wide SQS settings below +sqs: + kms_key_id: {{ kms_key_id if kms_key_id else '~' }} # Specify your own CMK to encrypt SQS queues. If set to ~ encryption will be managed by the default AWS key + +# Configure cluster-wide SNS settings below +sns: + kms_key_id: {{ kms_key_id if kms_key_id else '~' }} # Specify your own CMK to encrypt SNS topic. If set to ~ encryption will be managed by the default AWS key + +# Configure cluster-wide DynamoDB settings below. +dynamodb: + # this configuration is not supported and used at the moment. + # customizations are required to enable DDB encryption at rest. + kms_key_id: {{ kms_key_id if kms_key_id else '~' }} # Specify your own CMK to encrypt DynamoDB tables. If set to ~ encryption will be managed by the default AWS key + +solution: + # Enable to disable IDEA Anonymous Metric Collection. + # Refer to def build_metrics() on source/idea/idea-scheduler/src/ideascheduler/app/provisioning/job_provisioner/cloudformation_stack_builder.py for a list of metric being sent + enable_solution_metrics: true + custom_anonymous_metric_entry: ~ # Add a string to be automatically sent as "Misc" entry for Anonymous Metrics. + +iam: + # IAM policy ARNs provided below will be attached to all IAM roles for EC2 Instances launched by IDEA. + ec2_managed_policy_arns: [] + +load_balancers: + external_alb: + public: {{alb_public | lower}} + access_logs: true + certificates: + # if alb_custom_certificate_provided = false, self signed certificates will be generated for external ALB + # if alb_custom_certificate_provided = true, import your own certificates to AWS ACM and provide alb_custom_certificate_acm_certificate_arn, alb_custom_dns_name + provided: {{alb_custom_certificate_provided | lower}} + acm_certificate_arn: {{alb_custom_certificate_acm_certificate_arn if alb_custom_certificate_acm_certificate_arn else '~'}} + custom_dns_name: {{alb_custom_dns_name if alb_custom_dns_name else '~'}} + # SSL/TLS Policy on External Load Balancer + # For a list of policies - consult the documentation at https://docs.aws.amazon.com/elasticloadbalancing/latest/application/create-https-listener.html#describe-ssl-policies + ssl_policy: ELBSecurityPolicy-FS-1-2-Res-2020-10 + internal_alb: + access_logs: true + # SSL/TLS Policy on Internal Load Balancer + # For a list of policies - consult the documentation at https://docs.aws.amazon.com/elasticloadbalancing/latest/application/create-https-listener.html#describe-ssl-policies + ssl_policy: ELBSecurityPolicy-FS-1-2-Res-2020-10 + +cloudwatch_logs: + # enable or disable publishing logs to cloudwatch across the cluster. + # individual modules will check if cluster.cloudwatch_logs.enabled before checking for their respective logging config. + enabled: true + # Specifies in seconds the maximum amount of time that logs remain in the memory buffer before being sent to the server. + # No matter the setting for this field, if the size of the logs in the buffer reaches 1 MB, the logs are immediately sent to the server. + force_flush_interval: 5 + # this is default value for retention. individual modules may choose to set a different log retention value + retention_in_days: 90 + + + +# AWS Backup Configuration +# Refer: https://docs.aws.amazon.com/aws-backup/latest/devguide/whatisbackup.html to know more about AWS Backup and which services are supported by AWS Backup. +# Refer to IDEA Documentation > Architecture > Backup for additional details on AWS Backup integration. +backups: + + # enable or disable back up plans. + # if disabled, AWS Backups infrastructure will not be provisioned. + # enabling backups after cluster deployment, needs cluster stack upgrade. + # WARNING: + # * disabling backups after cluster deployment is not recommended + # * needs cluster stack upgrade. + # * provisioned backup infrastructure will be DELETED! + # * deletion will fail for vaults containing recovery points! + enabled: {{ enable_aws_backup | lower }} + + # enable or disable restore permissions for AWS Backup + # if true, restore permission policies will be added to the Backup Role + enable_restore: true + + # Backup Vault configuration + # a new backup vault will be provisioned per IDEA cluster if backups is enabled. + # existing backup vault is not supported. + backup_vault: + + # Specify your own CMK to encrypt your Secret manager. If set to ~ encryption will be managed by the default AWS key + kms_key_id: {{ kms_key_id if kms_key_id else '~' }} + + # The removal policy to apply to the vault. + # Note that removing a vault that contains recovery points will fail. + removal_policy: "DESTROY" + + # backup plan configuration applicable for infrastructure nodes + (new) shared storage provisioned by IDEA across all modules + # for eVDI nodes, refer to backup-plan configuration in eVDI settings. + backup_plan: + + # Backup Selection (Resource assignments) + # Resource assignments specify which resources will be backed up by this Backup plan. + selection: + + # Resources having below tags will be assigned to the cluster's backup plan + tags: + - "Key=idea:BackupPlan,Value={{ cluster_name }}-{{ module_id }}" + + rules: + + # + # delete_after_days: + # Specifies the duration after creation that a recovery point is deleted. + # Must be greater than `move_to_cold_storage_after` + + # move_to_cold_storage_after: + # Specifies the duration after creation that a recovery point is moved to cold storage. + + # schedule_expression: + # A CRON expression specifying when AWS Backup initiates a backup job + # Refer: https://docs.aws.amazon.com/AmazonCloudWatch/latest/events/ScheduledEvents.html + + # start_window_minutes: + # The duration after a backup is scheduled before a job is canceled if it doesn't start successfully + # minimum duration: 60 minutes + + # completion_window_minutes: + # The duration after a backup job is successfully started before it must be completed or it is canceled by AWS Backup + # minimum duration: 60 minutes + + + # a default rule is created for daily backup at 5AM UTC, with below parameters. + # rules can be added / modified based on requirements. + # a cluster stack upgrade is required if rules are modified after cluster is deployed. + # NOTE: + # manual changes to rules in backup plan via AWS Console, after cluster has been deployed can result in conflicts or upgrade failures. + + default: + delete_after_days: 7 + move_to_cold_storage_after_days: ~ + schedule_expression: "cron(0 5 * * ? *)" + start_window_minutes: 60 # 1 hour + completion_window_minutes: 480 # 8 hours + +# weekly: +# delete_after_days: 30 +# move_to_cold_storage_after_days: ~ +# schedule_expression: "cron(0 5 * * ? *)" +# start_window_minutes: 240 # 4 hours +# completion_window_minutes: 1440 # 24 hours +# +# monthly: +# delete_after_days: 180 +# move_to_cold_storage_after_days: 30 +# schedule_expression: "cron(0 5 * * ? *)" +# start_window_minutes: 480 # 8 hour +# completion_window_minutes: 1440 # 24 hours diff --git a/source/idea/idea-administrator/resources/config/templates/directoryservice/_templates/activedirectory.yml b/source/idea/idea-administrator/resources/config/templates/directoryservice/_templates/activedirectory.yml new file mode 100644 index 00000000..027eda87 --- /dev/null +++ b/source/idea/idea-administrator/resources/config/templates/directoryservice/_templates/activedirectory.yml @@ -0,0 +1,126 @@ +# Begin: Microsoft AD Settings (activedirectory) + +# Below is a template configuration to connect to your existing "On-Prem" or "Self-Managed" AD on AWS +# customize below configuration based on your environment requirements - below configuration will not work out of the box. +# +# +# When using the 'activedirectory' Directory Service back-end - READ-ONLY access is activated in IDEA +# for User and Group management. All user and group management activities take place in Active Directory directly. +# READ-WRITE is still required for the creation of Computer objects. +# SSO is expected to be configured and linked to the same Active Directory back-end. +# + +# The NetBIOS name for your domain +ad_short_name: IDEA + +# Password Max Age in Days. Used by Cluster IDP such as Cognito UserPool or KeyCloak in JWT Claims +# Authenticated API requests will be rejected if the password has expired. +# see: https://docs.aws.amazon.com/directoryservice/latest/admin-guide/ms_ad_password_policies.html +password_max_age: 42 + +# for on-prem AD, using ldaps is strongly recommended. +ldap_connection_uri: "ldap://idea.local" + +sssd: + # By default, the 'activedirectory' provider will rely on POSIX attributes defined in Active Directory (uidNumber, gidNumber). + # If a user does not have these POSIX attributes - they will not be able to log into IDEA. + # It is up to the AD Administrator / IDEA Administrator to arrange for POSIX attributes to be properly added to the AD schema for the IDEA users. + # If you want to enable ID mapping from the object SID, set ldap_id_mapping = false + # Note that this can have consequences when using systems that do not understand ID mapping. + # For further details about ID mapping and the ldap_id_mapping parameter, see the sssd-ldap(8) man page. + ldap_id_mapping: false + +ad_automation: + # time to live - for the ad-automation DDB table entry containing OTP and any other attributes + entry_ttl_seconds: 1800 + + # enable or disable service account's password rotation + # when set to true, IDEA ADAutomationAgent running in Cluster Manager will try to reset the service account credentials, + # when nearing expiration. + # This should be left as 'false' unless you know the AD service account is allowed to update its password in 'activedirectory' mode. + enable_root_password_reset: false + + # the max amount of time it could take to process the AD automation request. + sqs_visibility_timeout_seconds: 30 + + # the hostname prefix + # this should ideally be 5chars or below to provide space for unique hostname generation. + # Unique hostnames of 15chars are generated for NetBIOS compatibility (e.g. IDEA-C2C2C429E1) + hostname_prefix: "IDEA-" + +users: + # The Organizational Unit (OU) in your domain, in which IDEA cluster Users can be found + # If just the name of the OU, e.g. "Users" is provided, the qualified OU path will be computed as below: + # OU=Users,OU=IDEA,dc=idea,dc=local + # Provide the fully qualified OU to avoid any ambiguity. + ou: OU=Users,OU=IDEA,DC=idea,DC=local + +groups: + # The Organizational Unit (OU) in your domain, in which IDEA cluster Groups can be found + # If just the name of the OU, e.g. "Users" is provided, the qualified OU path will be computed as below: + # OU=Users,OU=IDEA,dc=idea,dc=local + # Provide the fully qualified OU to avoid any ambiguity. + ou: OU=Users,OU=IDEA,DC=idea,DC=local + +computers: + # The Organizational Unit (OU) in your domain, in which IDEA Computer Accounts (Applicable Infra + SOCA Compute + eVDI) can be _added_ + # The IDEA service account _must_ be allowed to create Computer objects in this OU in order to join devices to AD. + # If just the name of the OU, e.g. "Computers" is provided, the qualified OU path will be computed as below: + # OU=Computers,OU=IDEA,DC=idea,DC=local + # Provide the fully qualified OU to avoid any ambiguity. + ou: OU=Computers,OU=IDEA,DC=idea,DC=local + +sudoers: + # specify the group name to be used to manage Sudo users. + # this group will be added to /etc/sudoers on all cluster nodes that join AD. + group_name: AWS Delegated Administrators + + # specify the OU to be used for managing sudoers group (if applicable) + ou: ~ + +clusteradmin: + # Specify the full DN of an existing entry that serves as IDEA clusteradmin. + # Note that this is _not_ the same as the service account that binds to the AD. + # This is used to bootstrap the IDEA installation so that the configuration can + # be reachable. + # This can be configured outside the IDEA Users OU for existing AD accounts. + # Failure to set this properly will result in a non-functional IDEA deployment. + # + # clusteradmin_username_secret_arn should contain the fully qualified DN of the + # entry. e.g. cn=clusteradmin,OU=Org123,DC=idea,DC=local + # + # clusteradmin_password_secret_arn should contain the plaintext password + # + clusteradmin_username_secret_arn: ~ + clusteradmin_password_secret_arn: ~ + + +# +# Provide a mapping of the base IDEA groups to Active Directory DN +# +# If just the name of the group is supplied, e.g. "cluster-manager-administrators-module-group" is provided, the qualified path will be computed as below: +# cn=, +# otherwise it is treated as a full DN +# e.g. cn=cluster-manager-administrators-module-group,OU=Org123,DC=domain,DC=local + + +group_mapping: + # idea-required-group is a special mapping + # When users SSO for the first time - they do not have cache entries in DDB. + # The users must be part of this AD group to be considered eligible for the IDEA cluster. + # This can be specified to a specific group. + # The IDEA / AD Administrator is expected to create this group and maintain the proper membership + # of the users. + # e.g. CN=IDEAUsers,OU=AADDC Users,DC=idea-admin,DC=cloud + idea-required-group: IDEAUsers + cluster-manager-administrators-module-group: cluster-manager-administrators-module-group + cluster-manager-users-module-group: cluster-manager-users-module-group + default-project-group: default-project-group + managers-cluster-group: managers-cluster-group + scheduler-administrators-module-group: scheduler-administrators-module-group + scheduler-users-module-group: scheduler-users-module-group + vdc-administrators-module-group: vdc-administrators-module-group + vdc-users-module-group: vdc-users-module-group + + +# End: Microsoft AD Settings diff --git a/source/idea/idea-administrator/resources/config/templates/directoryservice/_templates/aws_managed_activedirectory.yml b/source/idea/idea-administrator/resources/config/templates/directoryservice/_templates/aws_managed_activedirectory.yml new file mode 100644 index 00000000..b7b58664 --- /dev/null +++ b/source/idea/idea-administrator/resources/config/templates/directoryservice/_templates/aws_managed_activedirectory.yml @@ -0,0 +1,79 @@ +# Begin: AWS Managed Microsoft AD Settings (aws_managed_activedirectory) + +{%- if use_existing_directory_service %} +# Indicates if the AWS Managed Microsoft AD is already provisioned and a new AD should not be provisioned. +use_existing: true + +# The DirectoryId of the existing AWS Managed Microsoft AD. +directory_id: {{ directory_id or '~' }} +{%- endif %} + +# The NetBIOS name for your domain +ad_short_name: "IDEA" + +# AWS Managed Microsoft AD Edition. Must be one of: [Standard, Enterprise] +# Note: Enterprise edition is not tested/supported yet, and additional configurations may be required and/or cdk stack needs to be updated. +ad_edition: "Standard" + +# added for future use - not supported yet. +# primary_region: "{{aws_region}}" +# replica_region: "{{aws_region}}" + +# Password Max Age in Days. Used by Cluster IDP such as Cognito UserPool or KeyCloak in JWT Claims +# Authenticated API requests will be rejected if the password has expired. +# see: https://docs.aws.amazon.com/directoryservice/latest/admin-guide/ms_ad_password_policies.html +password_max_age: 42 + +ldap_connection_uri: "ldap://idea.local" + +sssd: + # By default, the AD provider will rely on POSIX attributes defined in Active Directory. + # By default, IDEA will populate these values during user/group creation (uidNumber, gidNumber). + # If you want to enable ID mapping from the object SID, set ldap_id_mapping = false + # For further details about ID mapping and the ldap_id_mapping parameter, see the sssd-ldap(8) man page. + ldap_id_mapping: false + +ad_automation: + # time to live - for the ad-automation DDB table entry containing OTP and any other attributes + entry_ttl_seconds: 1800 + + # enable or disable service account's password rotation + # when set to true, IDEA ADAutomationAgent running in Cluster Manager will try to reset the service account credentials, + # when nearing expiration. + enable_root_password_reset: true + + # the max amount of time it could take to process the ad automation request. + sqs_visibility_timeout_seconds: 30 + +users: + # The Organizational Unit (OU) in your domain, in which IDEA cluster Users can be managed + # If just the name of the OU, e.g. "Users" is provided, the qualified OU path will be computed as below: + # OU=Users,OU=IDEA,dc=idea,dc=local + # Provide the fully qualified OU to avoid any ambiguity. + ou: OU=Users,OU=IDEA,DC=idea,DC=local + +groups: + # The Organizational Unit (OU) in your domain, in which IDEA cluster Groups can be managed + # If just the name of the OU, e.g. "Users" is provided, the qualified OU path will be computed as below: + # OU=Users,OU=IDEA,dc=idea,dc=local + # Provide the fully qualified OU to avoid any ambiguity. + ou: OU=Users,OU=IDEA,DC=idea,DC=local + +computers: + # The Organizational Unit (OU) in your domain, in which IDEA Computer Accounts (Applicable Infra + SOCA Compute + eVDI) can be added + # If just the name of the OU, e.g. "Computers" is provided, the qualified OU path will be computed as below: + # OU=Computers,OU=IDEA,DC=idea,DC=local + # Provide the fully qualified OU to avoid any ambiguity. + ou: OU=Computers,OU=IDEA,DC=idea,DC=local + +sudoers: + # specify the group name to be used to manage Sudo users. + # this group will be added to /etc/sudoers on all cluster nodes that join AD. + group_name: AWS Delegated Administrators + + # The Organizational Unit (OU) in your domain, in which the Sudoers group is available. + # Provide the fully qualified OU to avoid any ambiguity. + ou: OU=AWS Delegated Groups,DC=idea,DC=local + + +# End: AWS Managed Microsoft AD Settings (aws_managed_activedirectory) diff --git a/source/idea/idea-administrator/resources/config/templates/directoryservice/_templates/openldap.yml b/source/idea/idea-administrator/resources/config/templates/directoryservice/_templates/openldap.yml new file mode 100644 index 00000000..b0a524dc --- /dev/null +++ b/source/idea/idea-administrator/resources/config/templates/directoryservice/_templates/openldap.yml @@ -0,0 +1,24 @@ +# Begin: OpenLDAP Settings (openldap) + +public: false +instance_type: {{instance_type}} +base_os: {{base_os}} +instance_ami: {{instance_ami}} +volume_size: {{volume_size}} +hostname: "openldap.{{cluster_name}}.{{aws_region}}.local" +ldap_connection_uri: "ldap://openldap.{{cluster_name}}.{{aws_region}}.local" +cloudwatch_logs: + enabled: true +ec2: + # enable detailed monitoring for openldap ec2 instance, disabled by default + enable_detailed_monitoring: false + # enable termination protection for openldap ec2 instance, enabled by default + enable_termination_protection: true + # instance metadata access method + metadata_http_tokens: "required" # supported values are "required" for IMDSv2 or "optional" for IMDSv1 + +# NOTE: OU configurations will not be supported for OpenLDAP server. + # refer to source/idea-bootstrap/openldap-server/_templates/install_openldap.jinja2 to customize the OUs based on your requirements. + +# End: OpenLDAP Settings (openldap) + diff --git a/source/idea/idea-administrator/resources/config/templates/directoryservice/settings.yml b/source/idea/idea-administrator/resources/config/templates/directoryservice/settings.yml new file mode 100644 index 00000000..f0c5dfe9 --- /dev/null +++ b/source/idea/idea-administrator/resources/config/templates/directoryservice/settings.yml @@ -0,0 +1,68 @@ +# Directory Service Settings +# +# IDEA Supports below Directory Service providers: +# +# 1) OpenLDAP (Provider: openldap) +# When using OpenLDAP as the DirectoryService Provider, IDEA will provision an EC2 Instance with OpenLDAP. +# OpenLDAP settings can be customized or configured using `source/idea/idea-bootstrap/openldap-server/_templates/install_openldap.jinja2` +# An existing OpenLDAP Server is not supported and you will need to manually export / import LDAP objects from your existing OpenLDAP server to the new OpenLDAP server. +# +# 2) AWS Managed Microsoft AD (Provider: aws_managed_activedirectory) +# Use AWS Managed Microsoft AD, when you can provide full control for specific OUs for the AWS Managed Microsoft AD. +# By full control, IDEA Cluster has access to create AD Objects including Users, Groups and Computers. +# This is applicable for use-cases where IDEA cluster's Directory Service acts independently than that of your On-Prem or CORP AD. +# For production deployments, we recommend provisioning an AWS Managed Microsoft outside the scope of IDEA Cluster Deployment +# and use the existing resources flow to re-use the existing AWS Managed Microsoft AD. +# If you are planning to run Scale-Out Workloads + eVDI Workloads, consider using the "Enterprise" edition, instead of Standard edition. +# All Infrastructure hosts (primarily Linux), and eVDI Linux + Windows hosts will join AD. +# Consider creating a separate OU for Users and Groups for IDEA Cluster Management. + +# 3) Microsoft AD (Provider: activedirectory) +# Use Microsoft AD as the provider to make the cluster work with your On-Prem or Self-Managed AD on AWS or AWS Managed AD. +# In this use case, IDEA does not have write access to create Users and Groups. +# Write access to create Computer accounts is required. +# A custom synchronization service is required to be implemented to sync Users, Groups and Memberships. + +provider: "{{directory_service_provider}}" +automation_dir: "{{apps_mount_dir}}/{{cluster_name}}/{{module_id}}/automation" + +# Start UID and GID Configuration +# modify these to start from max+1 of your existing cluster, if you are migrating from an existing SOCA 2.x cluster +start_uid: 5000 +start_gid: 5000 + +name: idea.local +ldap_base: dc=idea,dc=local + +# Directory Service Root (Service Account) Credentials. +# --------------------------------------------- +# By default, the directory service stack will provision 2 Secrets in Secrets Manager. +# 1. Root UserName - defaults to Admin +# 2. Root Password - random generated secret string +# set root_credentials_provided: true, if you want to specify your own credentials. +# the secrets you manually create must be tagged with: +# idea:ClusterName = {{cluster_name}} +# idea:ModuleName = directoryservice +# idea:ModuleId = {{module_id}} +# Refer to Architecture FAQs: FAQ 8 for additional details. +# IMPORTANT NOTE: If you are provisioning a new AWS Managed Microsoft AD for your cluster, +# it is strongly recommended your provide your own credentials via AWS Secrets Manager. +# The primary reason is, after 42 days, the AD Automation Agent will change the secret as admin credentials will expire. +# After this event, if you every upgrade the directory service stack, CDK will update the password secret to the original one +# as the changes applied by AD Automation Agent were not via CDK, and CDK state information does not know about this change. +root_credentials_provided: {{ use_existing_directory_service | lower }} +root_username_secret_arn: {{ directory_service_root_username_secret_arn or '~' }} +root_password_secret_arn: {{ directory_service_root_password_secret_arn or '~' }} + + +{% if directory_service_provider == 'openldap' %} +{% include 'directoryservice/_templates/openldap.yml' %} +{%- endif %} + +{%- if directory_service_provider == 'aws_managed_activedirectory' %} +{% include 'directoryservice/_templates/aws_managed_activedirectory.yml' %} +{%- endif %} + +{%- if directory_service_provider == 'activedirectory' %} +{% include 'directoryservice/_templates/activedirectory.yml' %} +{%- endif %} diff --git a/source/idea/idea-administrator/resources/config/templates/global-settings/settings.yml b/source/idea/idea-administrator/resources/config/templates/global-settings/settings.yml new file mode 100644 index 00000000..464630bb --- /dev/null +++ b/source/idea/idea-administrator/resources/config/templates/global-settings/settings.yml @@ -0,0 +1,688 @@ +module_sets: + default: + cluster: + module_id: cluster + analytics: + module_id: analytics + identity-provider: + module_id: identity-provider + directoryservice: + module_id: directoryservice + shared-storage: + module_id: shared-storage + cluster-manager: + module_id: cluster-manager + {% if 'bastion-host' in enabled_modules -%} + bastion-host: + module_id: bastion-host + {%- endif %} + {% if 'scheduler' in enabled_modules -%} + scheduler: + module_id: scheduler + {%- endif %} + {% if 'virtual-desktop-controller' in enabled_modules -%} + virtual-desktop-controller: + module_id: vdc + {%- endif %} + {% if 'metrics' in enabled_modules -%} + metrics: + module_id: metrics + {%- endif %} + +package_config: + amazon_cloudwatch_agent: + download_link: ~ + # since there are multiple variations and platforms, to avoid creating multiple configuration entries, below pattern is used. + # you can always override the downloading link by providing the download_link. + # if download_link is empty, download_link_pattern will be used to download cloudwatch agent + # refer to below files to perform additional customizations or implementation details: + # * idea-bootstrap/_templates/linux/cloudwatch_agent.jinja2 + # * idea-bootstrap/_templates/windows/cloudwatch_agent.jinja2 + # CN and GovCloud Partitions will need to change or adjust the download_url_pattern accordingly. + download_link_pattern: https://s3.amazonaws.com/amazoncloudwatch-agent/%os%/%architecture%/latest/amazon-cloudwatch-agent.%ext% + + aws_ssm: + x86_64: https://s3.amazonaws.com/ec2-downloads-windows/SSMAgent/latest/linux_amd64/amazon-ssm-agent.rpm + aarch64: https://s3.amazonaws.com/ec2-downloads-windows/SSMAgent/latest/linux_arm64/amazon-ssm-agent.rpm + + linux_packages: + application: + # Extra package to install on Scheduler host, including OpenPBS dependencies + - dejavu-fonts-common + - dejavu-sans-fonts + - fontconfig + - fontpackages-filesystem + - freetype + - htop + - hwloc + - hwloc-libs + - libICE + - libSM + - libX11 + - libX11-common + - libX11-devel + - libXau + - libXft + - libXrender + - libical + - libpng + - libtool-ltdl + - libxcb + - tcl + - tk + - rpm-build + - libtool + - hwloc-devel + - libXt-devel + - libedit-devel + - libical-devel + - ncurses-devel + - perl + - python3 + - python3-pip + - python3-devel + - tcl-devel + - tk-devel + - swig + - expat-devel + - openssl-devel + - libXext + - libXft + - autoconf + - automake + - hwloc-devel + - stress + + dcv_amazonlinux: + # List of packages to install when using Mate Desktop + - gdm + - gnome-session + - gnome-classic-session + - gnome-session-xsession + - gnome-terminal + - gnu-free-fonts-common + - gnu-free-mono-fonts + - gnu-free-sans-fonts + - gnu-free-serif-fonts + - xorg-x11-server-Xorg + - xorg-x11-server-utils + - xorg-x11-utils + - xorg-x11-fonts-Type1 + - xorg-x11-drivers + - gstreamer1-plugins-good + - pcsc-lite-libs + + openldap_client: + - openldap-clients + + openldap_server: + # OpenLDAP Server and dependencies + - compat-openldap + - cyrus-sasl + - cyrus-sasl-devel + - openldap + - openldap-devel + - openldap-servers + - unixODBC + - unixODBC-devel + + sssd: + # SSSD and dependencies + - adcli + - avahi-libs + - bind-libs + - bind-libs-lite + - bind-license + - bind-utils + - c-ares + - cups-libs + - cyrus-sasl-gssapi + - http-parser + - krb5-workstation + - libdhash + - libipa_hbac + - libldb + - libsmbclient + - libsss_autofs + - libsss_certmap + - libsss_idmap + - libsss_nss_idmap + - libsss_sudo + - libtalloc + - libtdb + - libtevent + - libwbclient + - oddjob + - oddjob-mkhomedir + - python-sssdconfig + - realmd + - samba-client-libs + - samba-common + - samba-common-libs + - samba-common-tools + - sssd + - sssd-ad + - sssd-client + - sssd-common + - sssd-common-pac + - sssd-ipa + - sssd-krb5 + - sssd-krb5-common + - sssd-ldap + - sssd-proxy + system: + # Default packages installed on all Linux systems + - chrony + - cpp + - e2fsprogs + - e2fsprogs-libs + - gcc + - gcc-c++ + - gcc-gfortran + - glibc + - glibc-common + - glibc-devel + - glibc-headers + - gssproxy + - htop + - kernel + - kernel-devel + - kernel-headers + - keyutils + - keyutils-libs-devel + - krb5-devel + - krb5-libs + - libbasicobjects + - libcollection + - libcom_err + - libcom_err-devel + - libevent + - libffi-devel + - libgcc + - libgfortran + - libgomp + - libini_config + - libkadm5 + - libmpc + - libnfsidmap + - libpath_utils + - libquadmath + - libquadmath-devel + - libref_array + - libselinux + - libselinux-devel + - libselinux-python + - libselinux-utils + - libsepol + - libsepol-devel + - libss + - libstdc++ + - libstdc++-devel + - libtalloc + - libtevent + - libtirpc + - libverto-devel + - libverto-tevent + - libglvnd-devel + - make + - mpfr + - mdadm + - nvme-cli + - elfutils-libelf-devel + - nfs-utils + - git + - htop + - jq + - openssl + - openssl-devel + - openssl-libs + - pcre + - pcre-devel + - perl + - perl-Carp + - perl-Encode + - perl-Env + - perl-Exporter + - perl-File-Path + - perl-File-Temp + - perl-Filter + - perl-Getopt-Long + - perl-HTTP-Tiny + - perl-PathTools + - perl-Pod-Escapes + - perl-Pod-Perldoc + - perl-Pod-Simple + - perl-Pod-Usage + - perl-Scalar-List-Utils + - perl-Socket + - perl-Storable + - perl-Switch + - perl-Text-ParseWords + - perl-Time-HiRes + - perl-Time-Local + - perl-constant + - perl-libs + - perl-macros + - perl-parent + - perl-podlators + - perl-threads + - perl-threads-shared + - quota + - quota-nls + - redhat-lsb + - rpcbind + - sqlite-devel + - system-lsb + - nss-pam-ldapd + - tcp_wrappers + - vim + - wget + - zlib + - zlib-devel + + # used by cluster-manager to convert .pem files to .ppk files + putty: + - putty + + nodejs: + version: "16.10.0" + nvm_version: "0.39.0" + npm_version: "7.15.1" + url: "https://raw.githubusercontent.com/nvm-sh/nvm/" + openmpi: + version: "4.1.4" + url: "https://download.open-mpi.org/release/open-mpi/v4.1/openmpi-4.1.4.tar.gz" + checksum: "61f78909fb582114afae0408e85caa22529ed511765065059593c97b5a4f73152cd56b96812f5d80047e4e6fa114397e" + checksum_method: sha384 + python: + version: "3.9.16" + url: "https://www.python.org/ftp/python/3.9.16/Python-3.9.16.tgz" + checksum: "87acee12323b63a2e0c368193c03fd57e008585c754b6bceec6d5ec4c0bc34b3bb1ff20f31b6f5aff6e02502e7f5b291" + checksum_method: sha384 + + {%- if metrics_provider and 'prometheus' in metrics_provider %} + prometheus: + installer: + linux: + x86_64: https://github.com/prometheus/prometheus/releases/download/v2.37.0/prometheus-2.37.0.linux-amd64.tar.gz + aarch64: https://github.com/prometheus/prometheus/releases/download/v2.37.0/prometheus-2.37.0.linux-arm64.tar.gz + windows: + x86_64: https://github.com/prometheus/prometheus/releases/download/v2.37.0/prometheus-2.37.0.windows-amd64.zip + aarch64: https://github.com/prometheus/prometheus/releases/download/v2.37.0/prometheus-2.37.0.windows-arm64.zip + exporters: + node_exporter: + linux: + x86_64: https://github.com/prometheus/node_exporter/releases/download/v1.3.1/node_exporter-1.3.1.linux-amd64.tar.gz + aarch64: https://github.com/prometheus/node_exporter/releases/download/v1.3.1/node_exporter-1.3.1.linux-arm64.tar.gz + {%- endif %} + + {%- if 'scheduler' in enabled_modules %} + efa: + version: "1.21.0" + url: "https://efa-installer.amazonaws.com/aws-efa-installer-1.21.0.tar.gz" + checksum: "036e60b28793ed64072a53c9b67ab58e4cc93bcb6732bf5b7be2b3d4a9e082affbd37f6b91007641adb2860e08037823" + checksum_method: sha384 + openpbs: + version: "22.05.11" + url: "https://github.com/openpbs/openpbs/archive/v22.05.11.tar.gz" + checksum: "b1516586af058b3b52074fbc8e7849243ff983c58baff9a39e4d80d8e8c960e4c4cef1ae063b9729aea87fe5c52ca193" + checksum_method: sha384 + # can be release or dev. if dev, sources will be cloned from github and installed for the configured release version. + # additional customizations can be implemented in idea-bootstrap/_templates/linux/openpbs.jinja2 + type: release + packages: + - postgresql + - postgresql-contrib + - postgresql-devel + - postgresql-libs + - postgresql-server + {%- endif %} + + {%- if 'virtual-desktop-controller' in enabled_modules %} + dcv: + gpg_key: https://d1uj6qtbmh3dt5.cloudfront.net/NICE-GPG-KEY + host: + x86_64: + {%- if 'windows' in supported_base_os %} + windows: + url: https://d1uj6qtbmh3dt5.cloudfront.net/2022.1/Servers/nice-dcv-server-x64-Release-2022.1-13300.msi + sha256sum: a0581180d56612eccfa4af6e4958d608cd0076a0d2e186b7e45cfe1c6ab49b61 + {%- endif %} + linux: + {%- if 'amazonlinux2' in supported_base_os or 'rhel7' in supported_base_os or 'centos7' in supported_base_os %} + al2_rhel_centos7: + version: 2022.1-13300-el7-x86_64 + tgz: nice-dcv-2022.1-13300-el7-x86_64.tgz + url: https://d1uj6qtbmh3dt5.cloudfront.net/2022.1/Servers/nice-dcv-2022.1-13300-el7-x86_64.tgz + sha256sum: fe782c3ff6a1fd9291f7dcca0eadeec7cce47f1ae13d2da97fbed714fe00cf4d + {%- endif %} + {%- if 'rhel8' in supported_base_os or 'centos8' in supported_base_os or 'rocky8' in supported_base_os %} + rhel_centos_rocky8: + version: 2022.1-13300-el8-x86_64 + tgz: nice-dcv-2022.1-13300-el8-x86_64.tgz + url: https://d1uj6qtbmh3dt5.cloudfront.net/2022.1/Servers/nice-dcv-2022.1-13300-el8-x86_64.tgz + sha256sum: d047aa01166e6b8315807bc0138680cfa4938325b83e638e7c717d7481b848e8 + {%- endif %} + {%- if 'suse12' in supported_base_os %} + suse12: + version: 2022.1-13300-sles12-x86_64 + tgz: nice-dcv-2022.1-13300-sles12-x86_64.tgz + url: https://d1uj6qtbmh3dt5.cloudfront.net/2022.1/Servers/nice-dcv-2022.1-13300-sles12-x86_64.tgz + sha256sum: a7139e108db9d74ace1ebfccf665b0cdbb487c946dee5f4dda000f14c189ec4f + {%- endif %} + {%- if 'suse15' in supported_base_os %} + suse15: + version: 2022.1-13300-sles15-x86_64 + tgz: nice-dcv-2022.1-13300-sles15-x86_64.tgz + url: https://d1uj6qtbmh3dt5.cloudfront.net/2022.1/Servers/nice-dcv-2022.1-13300-sles15-x86_64.tgz + sha256sum: 7e8d9bc22e674014fe442207889be80efcb144bdff44e6b5c5a1391871fa565e + {%- endif %} + ubuntu: + {%- if 'ubuntu1804' in supported_base_os %} + ubuntu1804: + version: 2022.1-13300-ubuntu1804-x86_64 + tgz: nice-dcv-2022.1-13300-ubuntu1804-x86_64.tgz + url: https://d1uj6qtbmh3dt5.cloudfront.net/2022.1/Servers/nice-dcv-2022.1-13300-ubuntu1804-x86_64.tgz + sha256sum: db15078bacfb01c0583b1edd541031f31085f07beeca66fc0131225da2925def + {%- endif %} + {%- if 'ubuntu2004' in supported_base_os %} + ubuntu2004: + version: 2022.1-13300-ubuntu2004-x86_64 + tgz: nice-dcv-2022.1-13300-ubuntu2004-x86_64.tgz + url: https://d1uj6qtbmh3dt5.cloudfront.net/2022.1/Servers/nice-dcv-2022.1-13300-ubuntu2004-x86_64.tgz + sha256sum: 2e8c4a58645f3f91987b2b1086de5d46cf1c686c34046d57ee0e6f1e526a3031 + {%- endif %} + {%- if 'ubuntu2204' in supported_base_os %} + ubuntu2204: + version: 2022.1-13300-ubuntu2204-x86_64 + tgz: nice-dcv-2022.1-13300-ubuntu2204-x86_64.tgz + url: https://d1uj6qtbmh3dt5.cloudfront.net/2022.1/Servers/nice-dcv-2022.1-13300-ubuntu2204-x86_64.tgz + sha256sum: 3e431255b30b2a69d145a09d18b308ed3f5fa7eb5a879ba241fb183c45795d40 + {%- endif %} + aarch64: + linux: + {%- if 'amazonlinux2' in supported_base_os or 'rhel7' in supported_base_os or 'centos7' in supported_base_os %} + al2_rhel_centos7: + version: 2022.1-13300-el7-aarch64 + tgz: nice-dcv-2022.1-13300-el7-aarch64.tgz + url: https://d1uj6qtbmh3dt5.cloudfront.net/2022.1/Servers/nice-dcv-2022.1-13300-el7-aarch64.tgz + sha256sum: 62bea89c151c3ad840a9ffd17271227b6a51909e5529a4ff3ec401b37fde1667 + {%- endif %} + {%- if 'rhel8' in supported_base_os or 'centos8' in supported_base_os or 'rocky8' in supported_base_os %} + rhel_centos_rocky8: + version: 2022.1-13300-el8-aarch64 + tgz: nice-dcv-2022.1-13300-el8-aarch64.tgz + url: https://d1uj6qtbmh3dt5.cloudfront.net/2022.1/Servers/nice-dcv-2022.1-13300-el8-aarch64.tgz + sha256sum: 9adaae8c52d594008dffc06361aa3848d2e5b37833445b007f15a79306fb672a + {%- endif %} + ubuntu: + {%- if 'ubuntu1804' in supported_base_os %} + ubuntu1804: + version: 2022.1-13300-ubuntu1804-aarch64 + tgz: nice-dcv-2022.1-13300-ubuntu1804-aarch64.tgz + url: https://d1uj6qtbmh3dt5.cloudfront.net/2022.1/Servers/nice-dcv-2022.1-13300-ubuntu1804-aarch64.tgz + sha256sum: cf0d5259254bed4f9777cc64b151e3faa1b4e2adffdf2be5082e64c744ba5b3b + {%- endif %} + {%- if 'ubuntu2204' in supported_base_os %} + ubuntu2204: + version: 2022.1-13300-ubuntu2204-aarch64 + tgz: nice-dcv-2022.1-13300-ubuntu2204-aarch64.tgz + url: https://d1uj6qtbmh3dt5.cloudfront.net/2022.1/Servers/nice-dcv-2022.1-13300-ubuntu2204-x86_64.tgz + sha256sum: 9aedd8b969c18c473c9b7d31e3d960f2d79968342f9ffdaef63d82cc96767088 + {%- endif %} + agent: + x86_64: + {%- if 'windows' in supported_base_os %} + windows: + url: https://d1uj6qtbmh3dt5.cloudfront.net/2022.1/Servers/nice-dcv-server-x64-Release-2022.1-13300.msi + sha256sum: e043eda4ed5421692b60e31ac83e23ce2bc3d3098165b0d50344361c6bb0f808 + {%- endif %} + linux: + {%- if 'amazonlinux2' in supported_base_os or 'rhel7' in supported_base_os or 'centos7' in supported_base_os %} + al2_rhel_centos7: + version: 2022.1.592-1.el7.x86_64 + url: https://d1uj6qtbmh3dt5.cloudfront.net/2022.1/SessionManagerAgents/nice-dcv-session-manager-agent-2022.1.592-1.el7.x86_64.rpm + sha256sum: 7825a389900fd89143f8c0deeff0bfb0336bbf59092249211503bbd7d2d12754 + {%- endif %} + {%- if 'rhel8' in supported_base_os or 'centos8' in supported_base_os or 'rocky8' in supported_base_os %} + rhel_centos_rocky8: + version: 2022.1.592-1.el8.x86_64 + url: https://d1uj6qtbmh3dt5.cloudfront.net/2022.1/SessionManagerAgents/nice-dcv-session-manager-agent-2022.1.592-1.el8.x86_64.rpm + sha256sum: b45558cfb7034f5f2cbef9f87cb2db7fc118c80a8c80da55fe8e73be320e4581 + {%- endif %} + {%- if 'suse12' in supported_base_os %} + suse12: + version: 2022.1.592-1.sles12.x86_64 + url: https://d1uj6qtbmh3dt5.cloudfront.net/2022.1/SessionManagerAgents/nice-dcv-session-manager-agent-2022.1.592-1.sles12.x86_64.rpm + sha256sum: a9fbf7e9e23a7b72f5f5f9e7c14504560f4aea3df13d5e3a93254c6efce2d4a7 + {%- endif %} + {%- if 'suse15' in supported_base_os %} + suse15: + version: 2022.1.592-1.sles15.x86_64 + url: https://d1uj6qtbmh3dt5.cloudfront.net/2022.1/SessionManagerAgents/nice-dcv-session-manager-agent-2022.1.592-1.sles15.x86_64.rpm + sha256sum: 93b2575b92b25b8c01e3a979ee005df19d752b326acd10293ee460b40946175b + {%- endif %} + ubuntu: + {%- if 'ubuntu1804' in supported_base_os %} + ubuntu1804: + version: 2022.1.592-1_amd64.ubuntu1804 + url: https://d1uj6qtbmh3dt5.cloudfront.net/2022.1/SessionManagerAgents/nice-dcv-session-manager-agent_2022.1.592-1_amd64.ubuntu1804.deb + sha256sum: bdce81b7541f90f86a8e4dbda2f013de344b90a6818c3ce1a285a1fc5ebb7d71 + {%- endif %} + {%- if 'ubuntu2004' in supported_base_os %} + ubuntu2004: + version: 2022.1.592-1_amd64.ubuntu2004 + url: https://d1uj6qtbmh3dt5.cloudfront.net/2022.1/SessionManagerAgents/nice-dcv-session-manager-agent_2022.1.592-1_amd64.ubuntu2004.deb + sha256sum: 37f354106f136453ea64b21ccec2f20855913dbd547abd9503424693243f3851 + {%- endif %} + {%- if 'ubuntu2204' in supported_base_os %} + ubuntu2204: + version: 2022.1.592-1_amd64.ubuntu2204 + url: https://d1uj6qtbmh3dt5.cloudfront.net/2022.1/SessionManagerAgents/nice-dcv-session-manager-agent_2022.1.592-1_amd64.ubuntu2204.deb + sha256sum: 44a389dc9fd9813831d605370d002c29c4d8ff1a3671d3b1cb4327965ea3198c + {%- endif %} + aarch64: + linux: + {%- if 'amazonlinux2' in supported_base_os or 'rhel7' in supported_base_os or 'centos7' in supported_base_os %} + al2_rhel_centos7: + version: 2022.1.592-1.el7.aarch64 + url: https://d1uj6qtbmh3dt5.cloudfront.net/2022.1/SessionManagerAgents/nice-dcv-session-manager-agent-2022.1.592-1.el7.aarch64.rpm + sha256sum: 81bba492f340ce0f9930d7a7c698f07eca57e9274ea6167da5d6c6103527465b + {%- endif %} + {%- if 'rhel8' in supported_base_os or 'centos8' in supported_base_os or 'rocky8' in supported_base_os %} + rhel_centos_rocky8: + version: 2022.1.592-1.el8.aarch64 + url: https://d1uj6qtbmh3dt5.cloudfront.net/2022.1/SessionManagerAgents/nice-dcv-session-manager-agent-2022.1.592-1.el8.aarch64.rpm + sha256sum: b70531548f4d648399df202529b20154d0cf62b58eb63ec53985faf9f0c1de42 + {%- endif %} + ubuntu: + {%- if 'ubuntu1804' in supported_base_os %} + ubuntu1804: + version: 2022.1.592-1_arm64.ubuntu1804 + url: https://d1uj6qtbmh3dt5.cloudfront.net/2022.1/SessionManagerAgents/nice-dcv-session-manager-agent_2022.1.592-1_arm64.ubuntu1804.deb + sha256sum: 92017952fb02a773f1c411202900f69ffed945c4778635103cb13aaf5ebacdf0 + {%- endif %} + {%- if 'ubuntu2204' in supported_base_os %} + ubuntu2204: + version: 2022.1.592-1_arm64.ubuntu2204 + url: https://d1uj6qtbmh3dt5.cloudfront.net/2022.1/SessionManagerAgents/nice-dcv-session-manager-agent_2022.1.592-1_arm64.ubuntu2204.deb + sha256sum: 2ee0829620f30855d76029b5de0b222bff262325c28f0b5b1e67cad3995d78b7 + {%- endif %} + connection_gateway: + x86_64: + linux: + {%- if 'amazonlinux2' in supported_base_os or 'rhel7' in supported_base_os or 'centos7' in supported_base_os %} + al2_rhel_centos7: + version: 2022.1.377-1.el7.x86_64 + url: https://d1uj6qtbmh3dt5.cloudfront.net/2022.1/Gateway/nice-dcv-connection-gateway-2022.1.377-1.el7.x86_64.rpm + sha256sum: 3cdd671482fd58670dc037118709fb7810e8dcfbe040799a8f991ffebd5dafa4 + {%- endif %} + {%- if 'rhel8' in supported_base_os or 'centos8' in supported_base_os or 'rocky8' in supported_base_os %} + rhel_centos_rocky8: + version: 2022.1.377-1.el8.x86_64 + url: https://d1uj6qtbmh3dt5.cloudfront.net/2022.1/Gateway/nice-dcv-connection-gateway-2022.1.377-1.el8.x86_64.rpm + sha256sum: 09de6b4debab90c14ba09c24ee5fa59b2e40b30d56ab4c09fcf93efef16883ef + {%- endif %} + ubuntu: + {%- if 'ubuntu1804' in supported_base_os %} + ubuntu1804: + version: 2022.1.377-1_amd64.ubuntu1804 + url: https://d1uj6qtbmh3dt5.cloudfront.net/2022.1/Gateway/nice-dcv-connection-gateway_2022.1.377-1_amd64.ubuntu1804.deb + sha256sum: 806f9bf6c4d367168796a4ad307a39c6386c983f6c3f8e2ee31339a7a3a5a71f + {%- endif %} + {%- if 'ubuntu2004' in supported_base_os %} + ubuntu2004: + version: 2022.1.377-1_amd64.ubuntu2004 + url: https://d1uj6qtbmh3dt5.cloudfront.net/2022.1/Gateway/nice-dcv-connection-gateway_2022.1.377-1_amd64.ubuntu2004.deb + sha256sum: d5c0ecc0b70ff51d0670c682c1a298300cf18a8d47268853b8d3532ed6f28115 + {%- endif %} + {%- if 'ubuntu2204' in supported_base_os %} + ubuntu2204: + version: 2022.1.377-1_amd64.ubuntu2204 + url: https://d1uj6qtbmh3dt5.cloudfront.net/2022.1/Gateway/nice-dcv-connection-gateway_2022.1.377-1_amd64.ubuntu2204.deb + sha256sum: d52ba3712df30b0452bb9c3618439cdc22d7527bb0fccc9ae445169c31a09c80 + {%- endif %} + aarch64: + linux: + {%- if 'amazonlinux2' in supported_base_os or 'rhel7' in supported_base_os or 'centos7' in supported_base_os %} + al2_rhel_centos7: + version: 2022.1.377-1.el7.aarch64 + url: https://d1uj6qtbmh3dt5.cloudfront.net/2022.1/Gateway/nice-dcv-connection-gateway-2022.1.377-1.el7.aarch64.rpm + sha256sum: dab04b1b20e91e664dd51d81190c6030e7104d8455de9f72bad946c591f57126 + {%- endif %} + {%- if 'rhel8' in supported_base_os or 'centos8' in supported_base_os or 'rocky8' in supported_base_os %} + rhel_centos_rocky8: + version: 2022.1.377-1.el8.aarch64 + url: https://d1uj6qtbmh3dt5.cloudfront.net/2022.1/Gateway/nice-dcv-connection-gateway-2022.1.377-1.el8.aarch64.rpm + sha256sum: 2babf08b13587ab0542185d5d9056dfe974b1926f7daea035900e2ed0d0399eb + {%- endif %} + ubuntu: + {%- if 'ubuntu1804' in supported_base_os %} + ubuntu1804: + version: 2022.1.377-1_arm64.ubuntu1804 + url: https://d1uj6qtbmh3dt5.cloudfront.net/2022.1/Gateway/nice-dcv-connection-gateway_2022.1.377-1_arm64.ubuntu1804.deb + sha256sum: 04fb476168a4a9affa02e3a70512206b44171409b280a63bdbdaa4fa4b2fb8bc + {%- endif %} + {%- if 'ubuntu2004' in supported_base_os %} + ubuntu2004: + version: 2022.1.377-1_arm64.ubuntu2004 + url: https://d1uj6qtbmh3dt5.cloudfront.net/2022.1/Gateway/nice-dcv-connection-gateway_2022.1.377-1_arm64.ubuntu2004.deb + sha256sum: c21090d5c0fbd988655695d55f2bc43b19556b3f4910915f23edabf8ac5829db + {%- endif %} + {%- if 'ubuntu2204' in supported_base_os %} + ubuntu2204: + version: 2022.1.377-1_arm64.ubuntu2204 + url: https://d1uj6qtbmh3dt5.cloudfront.net/2022.1/Gateway/nice-dcv-connection-gateway_2022.1.377-1_arm64.ubuntu2204.deb + sha256sum: c8004c51f026b90a2df8b560b381c0a725be531ff721e89182105376c35d568a + {%- endif %} + broker: + linux: + {%- if 'amazonlinux2' in supported_base_os or 'rhel7' in supported_base_os or 'centos7' in supported_base_os %} + al2_rhel_centos7: + version: 2022.1.355-1.el7.noarch + url: https://d1uj6qtbmh3dt5.cloudfront.net/2022.1/SessionManagerBrokers/nice-dcv-session-manager-broker-2022.1.355-1.el7.noarch.rpm + sha256sum: db3bad6cf7b295b3f2b63ab8ebb4dd3e41d3caaaceee1443213fc6472512d5a9 + {%- endif %} + {%- if 'rhel8' in supported_base_os or 'centos8' in supported_base_os or 'rocky8' in supported_base_os %} + rhel_centos_rocky8: + version: 2022.1.355-1.el8.noarch + url: https://d1uj6qtbmh3dt5.cloudfront.net/2022.1/SessionManagerBrokers/nice-dcv-session-manager-broker-2022.1.355-1.el8.noarch.rpm + sha256sum: 20fedb637e2a8e689a34a7b8c6af33c5fc026bf6a221d48afbd49c2e0dc8d8b9 + {%- endif %} + ubuntu: + {%- if 'ubuntu1804' in supported_base_os %} + ubuntu1804: + version: 2022.1.355-1_all.ubuntu1804 + url: https://d1uj6qtbmh3dt5.cloudfront.net/2022.1/SessionManagerBrokers/nice-dcv-session-manager-broker_2022.1.355-1_all.ubuntu1804.deb + sha256sum: 4c8ef05ee5f0abafc77d22db4d46e87180bf9c1a449e788bc8719a9a6e26821b + {%- endif %} + {%- if 'ubuntu2004' in supported_base_os %} + ubuntu2004: + version: 2022.1.355-1_all.ubuntu2004 + url: https://d1uj6qtbmh3dt5.cloudfront.net/2022.1/SessionManagerBrokers/nice-dcv-session-manager-broker_2022.1.355-1_all.ubuntu2004.deb + sha256sum: 64ade6670e609148a89dd807e6d15650d6113efc41b6f37fbac8e0761dfb78a4 + {%- endif %} + {%- if 'ubuntu2204' in supported_base_os %} + ubuntu2204: + version: 2022.1.355-1_all.ubuntu2204 + url: https://d1uj6qtbmh3dt5.cloudfront.net/2022.1/SessionManagerBrokers/nice-dcv-session-manager-broker_2022.1.355-1_all.ubuntu2204.deb + sha256sum: 4dda3777a205ee5ad67de9d87585914682c3b8cf9bccd21da9ad75f0ecda7be6 + {%- endif %} + clients: + windows: + msi: + label: MSI + url: https://d1uj6qtbmh3dt5.cloudfront.net/2022.1/Clients/nice-dcv-client-Release-2022.1-8261.msi + zip: + label: ZIP + url: https://d1uj6qtbmh3dt5.cloudfront.net/2022.1/Clients/nice-dcv-client-Release-portable-2022.1-8261.zip + macos: + m1: + label: M1 Chip + url: https://d1uj6qtbmh3dt5.cloudfront.net/2022.1/Clients/nice-dcv-viewer-2022.1.4279.arm64.dmg + intel: + label: Intel Chip + url: https://d1uj6qtbmh3dt5.cloudfront.net/2022.1/Clients/nice-dcv-viewer-2022.1.4279.x86_64.dmg + linux: + rhel_centos7: + label: RHEL 7 | Cent OS 7 + url: https://d1uj6qtbmh3dt5.cloudfront.net/2022.1/Clients/nice-dcv-viewer-2022.1.4251-1.el7.x86_64.rpm + rhel_centos_rocky8: + label: RHEL 8 | Cent OS 8 | Rocky Linux 8 + url: https://d1uj6qtbmh3dt5.cloudfront.net/2022.1/Clients/nice-dcv-viewer-2022.1.4251-1.el8.x86_64.rpm + suse15: + label: SUSE Enterprise Linux 15 + url: https://d1uj6qtbmh3dt5.cloudfront.net/2022.1/Clients/nice-dcv-viewer-2022.1.4251-1.sles15.x86_64.rpm + ubuntu: + ubuntu1804: + label: Ubuntu 18.04 + url: https://d1uj6qtbmh3dt5.cloudfront.net/2022.1/Clients/nice-dcv-viewer_2022.1.4251-1_amd64.ubuntu1804.deb + ubuntu2004: + label: Ubuntu 20.04 + url: https://d1uj6qtbmh3dt5.cloudfront.net/2022.1/Clients/nice-dcv-viewer_2022.1.4251-1_amd64.ubuntu2004.deb + ubuntu2204: + label: Ubuntu 22.04 + url: https://d1uj6qtbmh3dt5.cloudfront.net/2022.1/Clients/nice-dcv-viewer_2022.1.4251-1_amd64.ubuntu2204.deb + {%- endif %} + +gpu_settings: + nvidia: + s3_bucket_url: "ec2-linux-nvidia-drivers.s3.amazonaws.com" + s3_bucket_path: "s3://ec2-linux-nvidia-drivers/latest/" + amd: + s3_bucket_url: "ec2-amd-linux-drivers.s3.amazonaws.com" + s3_bucket_path: "s3://ec2-amd-linux-drivers/latest/" + + instance_families: + - p2 + - p3 + - p4d + - p4de + - g2 + - g3 + - g3s + - g5 + - g5g + - g4dn + - g4ad + nvidia_public_driver_versions: + ltsb_version: <sb_version 470.141.03 + production_version: &production_version 510.47.03 + p2: *ltsb_version + g2: *ltsb_version + g3: *production_version + g3s: *production_version + g4dn: *production_version + g5: *production_version + g5g: *production_version + p3: *production_version + p4d: *production_version + p4de: *production_version + +# provide custom tags for all resources created by IDEA +# for eg. to add custom tags, tags as below: +# custom_tags: +# - Key=custom:MyTagName,Value=MyTagValue +# - Key=AnotherExampleName,Value=Another Example Value +custom_tags: [] + diff --git a/source/idea/idea-administrator/resources/config/templates/idea.yml b/source/idea/idea-administrator/resources/config/templates/idea.yml new file mode 100644 index 00000000..8321a050 --- /dev/null +++ b/source/idea/idea-administrator/resources/config/templates/idea.yml @@ -0,0 +1,76 @@ +modules: + + - name: global-settings + id: global-settings + type: config + config_files: + - settings.yml + + - name: cluster + id: cluster + type: stack + config_files: + - settings.yml + - logging.yml + + - name: analytics + id: analytics + type: stack + config_files: + - settings.yml + + - name: identity-provider + id: identity-provider + type: stack + config_files: + - settings.yml + + - name: directoryservice + id: directoryservice + type: stack + config_files: + - settings.yml + + - name: shared-storage + id: shared-storage + type: stack + config_files: + - settings.yml + + - name: cluster-manager + id: cluster-manager + type: app + config_files: + - settings.yml + + {% if 'scheduler' in enabled_modules %} + - name: scheduler + id: scheduler + type: app + config_files: + - settings.yml + {% endif %} + + {% if 'virtual-desktop-controller' in enabled_modules %} + - name: virtual-desktop-controller + id: vdc + type: app + config_files: + - settings.yml + {% endif %} + + {% if 'metrics' in enabled_modules %} + - name: metrics + id: metrics + type: stack + config_files: + - settings.yml + {% endif %} + + {% if 'bastion-host' in enabled_modules %} + - name: bastion-host + id: bastion-host + type: stack + config_files: + - settings.yml + {% endif %} diff --git a/source/idea/idea-administrator/resources/config/templates/identity-provider/settings.yml b/source/idea/idea-administrator/resources/config/templates/identity-provider/settings.yml new file mode 100644 index 00000000..efd6d4df --- /dev/null +++ b/source/idea/idea-administrator/resources/config/templates/identity-provider/settings.yml @@ -0,0 +1,34 @@ +# Identity Provider Settings +# +# Supported identity providers are Amazon Cognito User Pool and Keycloak (Opensource IDP: https://www.keycloak.org/). +# +# Use Cognito User Pools, when your target AWS Region supports Cognito User Pools and you can allow exceptions for VPC Endpoints, as Cognito User Pools do not support VPC Endpoints. +# Use Keycloak, when your target AWS Region does not support Cognito IDP and/or your InfoSec policies do not allow any exceptions for VPC Endpoints. +# +# Note: Support for Keycloak is work in progress. + +provider: "{{identity_provider}}" + +{%- if identity_provider == 'cognito-idp' %} +cognito: + administrators_group_name: administrators-cluster-group + managers_group_name: managers-cluster-group + removal_policy: "DESTROY" # RETAIN will preserve the user pool even if you delete the stack. + advanced_security_mode: "AUDIT" # Allowed values: AUDIT, ENFORCED, OFF - Depending on regional ability. See https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-cognito-userpool-userpooladdons.html + email_provider: "cognito" # "cognito" or "ses" + # Options that control Cognito interactions with Amazon Simple Email Service (SES) + # NOTE: You must configure and verify email addresses prior to IDEA installation. + # NOTE2: These controls can be the same or independent of the cluster.ses settings which cover + # emails that are generated by IDEA for specific events. + # Only used if email_provider is set to "ses" + # + ses: + ses_region: ~ + from_email: ~ + configuration_set: ~ + from_name: ~ + reply_to_address: ~ + # to enable SSO, run ./idea-admin.sh sso configure. do not modify this configuration to true to enable SSO. + sso_enabled: false + +{%- endif %} diff --git a/source/idea/idea-administrator/resources/config/templates/metrics/settings.yml b/source/idea/idea-administrator/resources/config/templates/metrics/settings.yml new file mode 100644 index 00000000..6e041787 --- /dev/null +++ b/source/idea/idea-administrator/resources/config/templates/metrics/settings.yml @@ -0,0 +1,93 @@ +# metrics module configuration +# Note: metrics module must be enabled during initial cluster deployment. +# if module is enabled post initial deployment, metric daemons on existing hosts need to be manually installed and configured. +# same is true for any changes to metrics provider. make an informed decision and choose your metrics provider based on your requirements. + +# graphana, visualizations and dashboards is out of IDEA scope. +# you need to manually configure the applicable dashboards based on the metrics provider you've configured for the cluster. + +# provider can be one of [cloudwatch, amazon_managed_prometheus, prometheus] +provider: "{{metrics_provider}}" + +{%- if metrics_provider == 'cloudwatch' %} +cloudwatch: + # `dashboard_name` can only contain alphanumerics, dash (-) and underscore (_). + dashboard_name: "{{cluster_name}}_{{aws_region}}" + + # Specifies the default value for often all metrics are to be collected. + # Individual modules may choose to override this value for specific types of metrics. + metrics_collection_interval: 60 + + # Specifies in seconds the maximum amount of time that metrics remain in the memory buffer before being sent to the server. + # No matter the setting for this, if the size of the metrics in the buffer reaches 40 KB or 20 different metrics, the metrics are immediately sent to the server. + force_flush_interval: 60 +{%- endif %} + +{%- if metrics_provider == 'amazon_managed_prometheus' %} +amazon_managed_prometheus: + workspace_name: {{cluster_name}}-workspace +{%- endif %} + +{%- if metrics_provider in ['amazon_managed_prometheus', 'prometheus'] %} +prometheus: + + # default scrape_interval. individual modules may override this configuration. + # How frequently to scrape targets by default. + # a duration matching the regular expression ((([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?|0), e.g. 1d, 1h30m, 5m, 10s + scrape_interval: 60s + + # default scrape_timeout. individual modules may override this configuration. + # How long until a scrape request times out. + # a duration matching the regular expression ((([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?|0), e.g. 1d, 1h30m, 5m, 10s + scrape_timeout: 10s + + # The labels to add to any time series or alerts when communicating with + # external systems (federation, remote storage, Alertmanager). + # Note: If you have existing prometheus timeseries data, adding new labels to + # external_labels in this config will create new time series for the metrics!! + external_labels: + cluster_name: '{{cluster_name}}' + # enable aws_region, if you are deploying multiple IDEA clusters and collecting all metrics to a single metrics endpoint. + # aws_region: '{{aws_region}}' + + # if metrics_provider = prometheus, provide your custom prometheus remote write configuration. + # for more details, refer to: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_write + # for amazon_managed_prometheus, metrics stack will update these parameters via cluster settings custom resource + # Note: prometheus supports remote write as list. multiple remote writes is not supported in IDEA. + # You can add additional configuration as specified in the remote_write configuration + remote_write: + url: {{ prometheus_remote_write_url if prometheus_remote_write_url else '~' }} + queue_config: + max_samples_per_send: 1000 + max_shards: 200 + capacity: 2500 + + # applicable only for custom prometheus - TODO + # for custom prometheus, do NOT hardcode credentials using `password`. + # create a secret in AWS Secrets Manager and provide the ARN as password_file + # IDEA framework will handle the logic to read the secret at run time. + # basic_auth: + # [ username: ] + # [ password: ] + # [ password_file: ] + + # applicable only for custom prometheus - TODO + # to configure Authorization header, do NOT hardcode `credentials`. + # create a secret in AWS Secrets Manager and provide the ARN as credentials_file + # IDEA framework will handle the logic to read the secret at run time. + # authorization: + # Sets the authentication type. + # [ type: | default: Bearer ] + # Sets the credentials. It is mutually exclusive with `credentials_file`. + # [ credentials: ] + # Sets the credentials to the credentials read from the configured file. It is mutually exclusive with `credentials`. + # [ credentials_file: ] + + # NOTE: this is not used in current IDEA release. + # this config will not be provisioned in prometheus.yml, but will be used by IDEA to query remote prometheus service. + # plotting and charting prometheus metrics on IDEA WebPortal is on the roadmap. + # for Amazon Managed Prometheus, this will be updated via ClusterSettings custom resource after provisioning Prometheus Workspace. + # for custom prometheus, update this section in a similar fashion as that of remote_write configuration. + remote_read: + url: {{ prometheus_query_url if prometheus_query_url else '~' }} +{%- endif %} diff --git a/source/idea/idea-administrator/resources/config/templates/scheduler/settings.yml b/source/idea/idea-administrator/resources/config/templates/scheduler/settings.yml new file mode 100644 index 00000000..f2675dc2 --- /dev/null +++ b/source/idea/idea-administrator/resources/config/templates/scheduler/settings.yml @@ -0,0 +1,169 @@ +# Scale-Out Computing Settings + +# specify if the scheduler instance can be launched in public or private subnet +public: false +instance_type: {{instance_type}} +base_os: {{base_os}} +instance_ami: {{instance_ami}} +volume_size: {{volume_size}} +hostname: "{{module_id}}.{{cluster_name}}.{{aws_region}}.local" +provider: openpbs # only openpbs is supported as of now + +# provide additional policy arns to be attached to the scheduler IAM Role created by IDEA +scheduler_iam_policy_arns: [] + +compute_node_os: {{base_os}} +compute_node_ami: {{instance_ami}} + +# provide additional policy arns to be attached to the default compute node IAM Role created by IDEA +compute_node_iam_policy_arns: [] + +endpoints: + external: + priority: 15 + path_patterns: ['/{{module_id}}/*'] + internal: + priority: 15 + path_patterns: ['/{{module_id}}/*'] + +server: + enable_http: true + hostname: 0.0.0.0 + port: 8443 + enable_tls: true + tls_certificate_file: {{apps_mount_dir}}/{{cluster_name}}/certs/idea.crt + tls_key_file: {{apps_mount_dir}}/{{cluster_name}}/certs/idea.key + enable_unix_socket: true + unix_socket_file: /run/idea.sock + max_workers: 16 + enable_metrics: false + graceful_shutdown_timeout: 10 + api_context_path: /{{module_id}} + +logging: + logs_directory: /opt/idea/app/logs + profile: production + default_log_file_name: application.log + +cloudwatch_logs: + enabled: true + +ec2: + # enable detailed monitoring for scheduler ec2 instance, enabled by default + enable_detailed_monitoring: true + # enable termination protection for scheduler ec2 instance, enabled by default + enable_termination_protection: true + # instance metadata access method + metadata_http_tokens: "required" # supported values are "required" for IMDSv2 or "optional" for IMDSv1 + +cache: + long_term: + max_size: 1000 + ttl_seconds: 86400 # 1 day + short_term: + max_size: 10000 + ttl_seconds: 600 # 10 minutes + +cost_estimation: + ec2_boot_penalty_seconds: 300 + default_fsx_lustre_size: 1200 + # Update EBS rate for your region + # EBS Formulas: https://aws.amazon.com/ebs/pricing/ + ebs_gp3_storage: 0.08 # $ per gb per month + ebs_io1_storage: 0.125 # IOPS per month + provisioned_iops: 0.065 # IOPS per month + fsx_lustre: 0.000194 # GB per hour + +job_provisioning: + # specifies the duration after which the cloud formation stack should be terminated and job provisioning should be retried. + # this is used to ensure provisioning capacity failures, if any are retried + stack_provisioning_timeout_seconds: 1800 + + # the interval at which node housekeeping session is executed. + # housekeeping cycles ensure new ec2 instances (nodes) join the cluster and old nodes are deleted. + # additionally, custom metrics are published at below frequency for cluster monitoring + node_housekeeping_interval_seconds: 60 + + # the interval that job monitor waits before fetching new jobs queued from scheduler. + job_submission_queue_interval_seconds: 1 + + # the interval at which jobs are provisioned + # job provisioner will block/wait for this duration before proceeding to provision another queued job. + # for ephemeral capacity - this is applicable per job, for batch/job shared capacity, this interval is applicable per batch + # this ensures cloudformation stack creation requests are not throttled. + job_provisioning_interval_seconds: 1 + + # the interval to wait for batch job provisioning + # if more jobs are queued during this interval, provisioner will wait until all jobs in the batch have been queued and accepted + # applicable only for job-shared/batch scaling mode queues. + batch_provisioning_wait_interval_seconds: 3 + + # the interval at which idea scheduler checks and processes finished jobs + finished_job_processing_interval_seconds: 30 + + # SpotFleet Request configuration + # refer to: https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-ec2-spotfleet-spotfleetrequestconfigdata.html for additional documentation + spot_fleet_request: + excess_capacity_termination_policy: noTermination # can be one of [default, noTerminate] + instance_interruption_behavior: terminate # can be one of [hibernate, stop, terminate] + spot_maintenance_strategies: ~ # can be null (~) or CapacityRebalance + request_type: maintain # can be one of [maintain, request] + + # Mixed Instance Policy configuration for ASG + # Used when a mix of On-Demand + Spot instances are configured during job submission + mixed_instances_policy: + # refer to https://docs.aws.amazon.com/autoscaling/ec2/APIReference/API_InstancesDistribution.html for more info + on_demand_allocation_strategy: prioritized # can be one of [prioritized, lowest-price] + + # Placement Group config + placement_group: + # refer to: https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-ec2-placementgroup.html for more details + # AWS Outpost placements is not supported. + + # The placement strategy + strategy: cluster # can be one of [cluster, partition, spread] + +fair_share: + start_score: 100 + running_job_penalty: -60 + score_type: linear + c1: 1 + c2: 0 + +notifications: + enabled: true + job_started: + email_template: scheduler.job-started + job_completed: + email_template: scheduler.job-completed + +scratch_storage: + fsx_lustre: + mount_point: /fsx + ebs: + mount_point: /scratch + instance_store: + mount_point: /scratch + +openpbs: + server: + # OpenPBS Server Configuration + # - These parameters are configurable ONE TIME during scheduler installation. + # - Changes to openpbs server configurations post-deployment need to be manually applied. + + # Note: Documentation is sourced from: Altair PBS Professional 2022.1 Big Book PDF. Refer to the manual for more details. + + # This attribute specifies whether, for each user, the username at the submission host must be the same as the one at the server host. + # The username at the server host must always be the same as the username at the execution host. + # When flatuid is set to True, the server assumes that UserA@host1 is the same as UserA@host2. Therefore, if flatuid is True, UserA@host2 can operate on UserA@host1's job. + flatuid: 'true' + + # configure whether PBS preserves job history, and for how long, by setting values for the job_history_enable and job_history_duration server attributes. + job_history_enable: '1' + job_history_duration: '72:00:00' + + # time in seconds between scheduling iterations + scheduler_iteration: '30' + + # The maximum number of vnodes allowed to be in the process of being provisioned. + max_concurrent_provision: '5000' diff --git a/source/idea/idea-administrator/resources/config/templates/shared-storage/_templates/efs.yml b/source/idea/idea-administrator/resources/config/templates/shared-storage/_templates/efs.yml new file mode 100644 index 00000000..e7a4f83b --- /dev/null +++ b/source/idea/idea-administrator/resources/config/templates/shared-storage/_templates/efs.yml @@ -0,0 +1,13 @@ +efs: + {%- if kwargs.use_existing_fs %} + use_existing_fs: true + file_system_id: "{{ kwargs.file_system_id }}" # if an existing file system id is provided, provisioning of EFS will be skipped and above parameters will be applied as is. existing file system is supported only when using existing vpc. + {%- else %} + kms_key_id: {{ kwargs.kms_key_id or '~' }} # Specify your own CMK to encrypt EFS file system. If set to ~ encryption will be managed by the default AWS key + throughput_mode: "bursting" # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-efs-filesystem.html#cfn-efs-filesystem-throughputmode + performance_mode: "generalPurpose" # https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-efs-filesystem.html#cfn-efs-filesystem-performancemode + encrypted: true # Select whether you want to encrypt the filesystem or not. + removal_policy: "DESTROY" # RETAIN will preserve the EFS even if you delete the stack. + cloudwatch_monitoring: {{ ( kwargs.efs.cloudwatch_monitoring or False ) | lower }} + transition_to_ia: {{ kwargs.efs.transition_to_ia or '~' }} + {%- endif %} diff --git a/source/idea/idea-administrator/resources/config/templates/shared-storage/_templates/fsx_lustre.yml b/source/idea/idea-administrator/resources/config/templates/shared-storage/_templates/fsx_lustre.yml new file mode 100644 index 00000000..8dd26418 --- /dev/null +++ b/source/idea/idea-administrator/resources/config/templates/shared-storage/_templates/fsx_lustre.yml @@ -0,0 +1,12 @@ +fsx_lustre: + {%- if kwargs.use_existing_fs %} + use_existing_fs: true + file_system_id: "{{ kwargs.file_system_id }}" # if an existing file system id is provided, provisioning of new FSx lustre will be skipped. existing file system is supported only when using existing vpc. + {%- else %} + kms_key_id: {{ kwargs.kms_key_id or '~' }} # Specify your own CMK to encrypt EFS or FSx_Lustre file system. If set to ~ encryption will be managed by the default AWS key + deployment_type: {{ kwargs.fsx_lustre.deployment_type or 'PERSISTENT_1' }} # Allowed values: PERSISTENT_1 | SCRATCH_1 | SCRATCH_2. https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-fsx-filesystem-lustreconfiguration.html#cfn-fsx-filesystem-lustreconfiguration-deploymenttype + drive_cache_type: "NONE" # Allowed values: NONE | READ. Required when storage_type is HDD. https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-fsx-filesystem-lustreconfiguration.html#cfn-fsx-filesystem-lustreconfiguration-drivecachetype + per_unit_storage_throughput: 50 # Allowed values: 12, 40 for HDD, 50, 100, 200 for SSD. Required for the PERSISTENT_1 deployment_type. https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-fsx-filesystem-lustreconfiguration.html#cfn-fsx-filesystem-lustreconfiguration-perunitstoragethroughput + storage_capacity: 1200 # For SCRATCH_2 and PERSISTENT_1 types, valid values are 1,200, 2,400, then continuing in increments of 2,400 GiB. For SCRATCH_1 deployment types, valid values are 1,200, 2,400, 3,600, then continuing in increments of 3,600 GiB. https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-fsx-filesystem.html#cfn-fsx-filesystem-storagecapacity + storage_type: "SSD" # Allowed values: SSD or HDD. https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-fsx-filesystem.html#cfn-fsx-filesystem-storagetype + {%- endif %} diff --git a/source/idea/idea-administrator/resources/config/templates/shared-storage/settings.yml b/source/idea/idea-administrator/resources/config/templates/shared-storage/settings.yml new file mode 100644 index 00000000..f04f1ffd --- /dev/null +++ b/source/idea/idea-administrator/resources/config/templates/shared-storage/settings.yml @@ -0,0 +1,147 @@ +# Shared Storage Settings +# * apps and data shared storage are mandatory. +# * you can optionally provide additional "existing" file systems + +{%- macro storage_provider_config(provider, kwargs) %} +{%- if provider == 'efs' %} + {%- include 'shared-storage/_templates/efs.yml' %} +{%- elif provider == 'fsx_lustre' %} + {%- include 'shared-storage/_templates/fsx_lustre.yml' %} +{%- endif %} +{%- endmacro %} + +# application storage for cluster. +# used to store common applications, scripts, files and logs across the cluster +apps: + title: "Shared Storage - Apps" + provider: "{{storage_apps_provider}}" + mount_dir: "{{apps_mount_dir}}" + # Mount options default to using NFSv4. + # Adjust mount_options if encryption in transit is needed via the Amazon EFS Mount Helper + mount_options: "nfs4 nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2,noresvport 0 0" + #mount_options: "efs _netdev,noresvport,tls,iam 0 0" + scope: + - cluster + {{ storage_provider_config(provider=storage_apps_provider, kwargs={ + 'use_existing_fs': use_existing_apps_fs, + 'file_system_id': existing_apps_fs_id, + 'kms_key_id': kms_key_id, + 'efs': { + 'cloudwatch_monitoring': True + }, + 'fsx_lustre': { + 'deployment_type': 'PERSISTENT_1' + } + }) | indent(2) }} + +# data storage for cluster. +# used to store user home directories +data: + title: "Shared Storage - Data" + provider: "{{storage_data_provider}}" + mount_dir: "{{data_mount_dir}}" + # Mount options default to using NFSv4. + # Adjust mount_options if encryption in transit is needed via the Amazon EFS Mount Helper + mount_options: "nfs4 nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2,noresvport 0 0" + #mount_options: "efs _netdev,noresvport,tls,iam 0 0" + scope: + - cluster + {{ storage_provider_config(provider=storage_data_provider, kwargs={ + 'use_existing_fs': use_existing_data_fs, + 'file_system_id': existing_data_fs_id, + 'kms_key_id': kms_key_id, + 'efs': { + 'cloudwatch_monitoring': False, + 'transition_to_ia': 'AFTER_30_DAYS' + }, + 'fsx_lustre': { + 'deployment_type': 'PERSISTENT_1' + } + }) | indent(2) }} + + +# Example configurations for existing File Systems +# Below configurations are for quick reference. +# Configurations specific to your environment and cluster can be generated using below utility: +# ./idea-admin.sh shared-storage attach-file-system --help + +# Existing Amazon EFS +# demo: +# title: Demo FS +# provider: efs +# scope: +# - cluster +# mount_dir: /demo +# mount_options: nfs4 nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2,noresvport 0 0 +# efs: +# use_existing_fs: true +# file_system_id: fs-05b067d4a78e0fb29 +# dns: fs-05b067d4a78e0fb29.efs.us-east-1.amazonaws.com +# encrypted: true + +# Existing FSx for Lustre +# demo: +# title: Demo FS +# provider: fsx_lustre +# scope: +# - cluster +# mount_dir: /demo +# mount_options: lustre defaults,noatime,flock,_netdev 0 0 +# fsx_lustre: +# use_existing_fs: true +# file_system_id: fs-01a2ccc035f0f007c +# dns: fs-01a2ccc035f0f007c.fsx.us-east-1.amazonaws.com +# mount_name: drohpbev +# version: '2.10' + +# Existing FSx for NetApp ONTAP +# demo: +# title: Demo FS +# provider: fsx_netapp_ontap +# scope: +# - cluster +# mount_drive: Z +# mount_dir: /demo +# mount_options: nfs4 nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2,noresvport 0 0 +# fsx_netapp_ontap: +# use_existing_fs: true +# file_system_id: fs-09753a84872d3209b +# svm: +# svm_id: svm-064990494a2dbd4c2 +# smb_dns: IDEA-DEV-SVM1.IDEA.LOCAL +# nfs_dns: svm-064990494a2dbd4c2.fs-09753a84872d3209b.fsx.us-east-1.amazonaws.com +# management_dns: svm-064990494a2dbd4c2.fs-09753a84872d3209b.fsx.us-east-1.amazonaws.com +# iscsi_dns: iscsi.svm-064990494a2dbd4c2.fs-09753a84872d3209b.fsx.us-east-1.amazonaws.com +# volume: +# volume_id: fsvol-0f791716b33592fff +# volume_path: / +# security_style: MIXED +# cifs_share_name: share # Use same name reported by 'vserver cifs share show' command in ONTAP CLI + +# Existing FSx for OpenZFS +# demo: +# title: Demo FS +# provider: fsx_openzfs +# scope: +# - cluster +# mount_dir: /demo +# mount_options: nfs4 nfsvers=4.1,rsize=1048576,wsize=1048576,timeo=600 0 0 +# fsx_openzfs: +# use_existing_fs: true +# file_system_id: fs-09e1b30aab982aa34 +# dns: fs-09e1b30aab982aa34.fsx.us-east-1.amazonaws.com +# volume_id: fsvol-00d420eeb064ac36a +# volume_path: /fsx + +# Existing FSx for Windows File Server +# demo: +# title: Demo FS +# provider: fsx_windows_file_server +# scope: +# - cluster +# mount_drive: Z +# fsx_windows_file_server: +# use_existing_fs: true +# file_system_id: fs-0c1f74968df26462e +# dns: amznfsx0wallrm9.idea.local +# preferred_file_server_ip: 10.0.113.174 diff --git a/source/idea/idea-administrator/resources/config/templates/virtual-desktop-controller/settings.yml b/source/idea/idea-administrator/resources/config/templates/virtual-desktop-controller/settings.yml new file mode 100644 index 00000000..3d062207 --- /dev/null +++ b/source/idea/idea-administrator/resources/config/templates/virtual-desktop-controller/settings.yml @@ -0,0 +1,269 @@ +server: + enable_http: true + hostname: 0.0.0.0 + port: 8443 + enable_tls: true + tls_certificate_file: {{apps_mount_dir}}/{{cluster_name}}/certs/idea.crt + tls_key_file: {{apps_mount_dir}}/{{cluster_name}}/certs/idea.key + enable_unix_socket: true + unix_socket_file: /run/idea.sock + max_workers: 16 + enable_metrics: false + graceful_shutdown_timeout: 10 + api_context_path: /{{ module_id }} + +controller: + autoscaling: + public: false + instance_type: {{instance_type}} + base_os: {{base_os}} + instance_ami: {{instance_ami}} + volume_size: {{volume_size}} + enabled_detailed_monitoring: false + min_capacity: 1 + max_capacity: 3 + cooldown_minutes: 5 + new_instances_protected_from_scale_in: false + elb_healthcheck: + # Specifies the time in minutes Auto Scaling waits before checking the health status of an EC2 instance that has come into service. + grace_time_minutes: 15 + cpu_utilization_scaling_policy: + target_utilization_percent: 80 + estimated_instance_warmup_minutes: 15 + rolling_update_policy: + max_batch_size: 1 + min_instances_in_service: 1 + pause_time_minutes: 15 + # instance metadata access method + metadata_http_tokens: "required" # supported values are "required" for IMDSv2 or "optional" for IMDSv1 + request_handler_threads: + min: 1 + max: 8 + endpoints: + external: + priority: 13 + path_patterns: [ '/{{ module_id }}/*' ] + internal: + priority: 13 + path_patterns: [ '/{{ module_id }}/*' ] + +dcv_broker: + autoscaling: + public: false + instance_type: {{instance_type}} + base_os: {{base_os}} + instance_ami: {{instance_ami}} + volume_size: {{volume_size}} + enabled_detailed_monitoring: false + min_capacity: 1 + max_capacity: 3 + cooldown_minutes: 5 + new_instances_protected_from_scale_in: false + elb_healthcheck: + # Specifies the time in minutes Auto Scaling waits before checking the health status of an EC2 instance that has come into service. + grace_time_minutes: 15 + cpu_utilization_scaling_policy: + target_utilization_percent: 80 + estimated_instance_warmup_minutes: 15 + rolling_update_policy: + max_batch_size: 1 + min_instances_in_service: 1 + pause_time_minutes: 15 + # instance metadata access method + metadata_http_tokens: "required" # supported values are "required" for IMDSv2 or "optional" for IMDSv1 + client_communication_port: 8444 # DO NOT CHANGE + agent_communication_port: 8445 # DO NOT CHANGE + gateway_communication_port: 8446 # DO NOT CHANGE + # SSL/TLS Policy on VDC HTTPS listeners + # For a list of policies - consult the documentation at https://docs.aws.amazon.com/elasticloadbalancing/latest/application/create-https-listener.html#describe-ssl-policies + ssl_policy: ELBSecurityPolicy-FS-1-2-Res-2020-10 + session_token_validity: 1440 # in minutes + dynamodb_table: + autoscaling: + enabled: true + read_capacity: + min_units: 5 + max_units: 20 + target_utilization: 70 # in percentage + scale_in_cooldown: 60 # in seconds + scale_out_cooldown: 60 # in seconds + write_capacity: + min_units: 5 + max_units: 20 + target_utilization: 70 # in percentage + scale_in_cooldown: 60 # in seconds + scale_out_cooldown: 60 # in seconds + +dcv_connection_gateway: + autoscaling: + public: false + instance_type: {{instance_type}} + base_os: {{base_os}} + instance_ami: {{instance_ami}} + volume_size: {{volume_size}} + enabled_detailed_monitoring: false + min_capacity: 1 + max_capacity: 3 + cooldown_minutes: 5 + new_instances_protected_from_scale_in: false + elb_healthcheck: + # Specifies the time in minutes Auto Scaling waits before checking the health status of an EC2 instance that has come into service. + grace_time_minutes: 15 + cpu_utilization_scaling_policy: + target_utilization_percent: 80 + estimated_instance_warmup_minutes: 15 + rolling_update_policy: + max_batch_size: 1 + min_instances_in_service: 1 + pause_time_minutes: 15 + # instance metadata access method + metadata_http_tokens: "required" # supported values are "required" for IMDSv2 or "optional" for IMDSv1 + certificate: + provided: {{ dcv_connection_gateway_custom_certificate_provided | lower }} + custom_dns_name: {{ dcv_connection_gateway_custom_dns_hostname if dcv_connection_gateway_custom_dns_hostname else '~' }} + certificate_secret_arn: {{ dcv_connection_gateway_custom_certificate_certificate_secret_arn if dcv_connection_gateway_custom_certificate_certificate_secret_arn else '~' }} + private_key_secret_arn: {{ dcv_connection_gateway_custom_certificate_private_key_secret_arn if dcv_connection_gateway_custom_certificate_private_key_secret_arn else '~' }} + +external_nlb: + # external NLB access logs are enabled by default + access_logs: true + +opensearch: + dcv_session: + alias: {{cluster_name}}_{{module_id}}_user_sessions + software_stack: + alias: {{cluster_name}}_{{module_id}}_software_stacks + session_permission: + alias: {{cluster_name}}_{{module_id}}_session_permission + +dcv_session: + idle_timeout: 1440 # in minutes + idle_timeout_warning: 300 # in seconds + cpu_utilization_threshold: 30 # in percentage + max_root_volume_memory: 1000 # in GB + additional_security_groups: [] + allowed_sessions_per_user: 5 + instance_types: + allow: # Supports both instance families and types. E.g. specify t3 for family and t3.large for instance type + - t3 + - g4dn + - g4ad + - m6a + - m6g + deny: [] # Supports both instance families and types. E.g. specify t3 for family and t3.large for instance type + quic_support: {{ dcv_session_quic_support | lower }} + # instance metadata access method + metadata_http_tokens: "required" # supported values are "required" for IMDSv2 or "optional" for IMDSv1 + notifications: + provisioning: + enabled: false + email_template: virtual-desktop-controller.session-provisioning + creating: + enabled: false + email_template: virtual-desktop-controller.session-creating + initializing: + enabled: false + email_template: virtual-desktop-controller.session-initializing + resuming: + enabled: false + email_template: virtual-desktop-controller.session-resuming + ready: + enabled: true + email_template: virtual-desktop-controller.session-ready + stopping: + enabled: false + email_template: virtual-desktop-controller.session-stopping + stopped: + enabled: true + email_template: virtual-desktop-controller.session-stopped + deleting: + enabled: false + email_template: virtual-desktop-controller.session-deleting + error: + enabled: true + email_template: virtual-desktop-controller.session-error + deleted: + enabled: true + email_template: virtual-desktop-controller.session-deleted + session-shared: + enabled: true + email_template: virtual-desktop-controller.session-shared + session-permission-updated: + enabled: true + email_template: virtual-desktop-controller.session-permission-updated + session-permission-expired: + enabled: true + email_template: virtual-desktop-controller.session-permission-expired + + working_hours: + start_up_time: '09:00' + shut_down_time: '17:00' + + # You can provide default schedules of every day of the week in this section. + # Every schedule entry looks as follows + # type: NO_SCHEDULE, WORKING_HOURS (timings are defined in working_hours.start_up_time and working_hours.shut_down_time), STOP_ALL_DAY, START_ALL_DAY, CUSTOM_SCHEDULE + # start_up_time: To be provided IF type is CUSTOM_SCHEDULE + # shut_down_time: To be provided IF type is CUSTOM_SCHEDULE + schedule: + monday: + type: NO_SCHEDULE + start_up_time: ~ + shut_down_time: ~ + tuesday: + type: NO_SCHEDULE + start_up_time: ~ + shut_down_time: ~ + wednesday: + type: NO_SCHEDULE + start_up_time: ~ + shut_down_time: ~ + thursday: + type: NO_SCHEDULE + start_up_time: ~ + shut_down_time: ~ + friday: + type: NO_SCHEDULE + start_up_time: ~ + shut_down_time: ~ + saturday: + type: STOP_ALL_DAY + start_up_time: ~ + shut_down_time: ~ + sunday: + type: STOP_ALL_DAY + start_up_time: ~ + shut_down_time: ~ + default_profiles: + admin: admin_profile + owner: owner_profile + +logging: + logs_directory: /opt/idea/app/logs + profile: production + default_log_file_name: application.log + +cloudwatch_logs: + enabled: true + +cache: + long_term: + max_size: 1000 + ttl_seconds: 86400 # 1 day + short_term: + max_size: 10000 + ttl_seconds: 600 # 10 minutes + + +vdi_host_backup: + enabled: {{ enable_aws_backup | lower }} + backup_plan: + selection: + tags: + - "Key=idea:BackupPlan,Value={{ cluster_name }}-{{ module_id }}" + rules: + default: + delete_after_days: 7 + move_to_cold_storage_after_days: ~ + schedule_expression: "cron(0 5 * * ? *)" + start_window_minutes: 60 # 1 hour + completion_window_minutes: 480 # 8 hours diff --git a/source/idea/idea-administrator/resources/config/values.yml b/source/idea/idea-administrator/resources/config/values.yml new file mode 100644 index 00000000..374a3561 --- /dev/null +++ b/source/idea/idea-administrator/resources/config/values.yml @@ -0,0 +1,111 @@ +# Cluster Settings +cluster_name: idea-sample +cluster_timezone: ~ # leaving blank or ~ will result in using the TimezoneId for the city closest to the selected AWS Region +cluster_locale: en_US + +# Cluster Administrator +administrator_email: admin@example.com +administrator_username: clusteradmin # name cannot be administrator, admin, root, ideaserviceaccount, ec2-user, centos, ssm-user to avoid conflicts with DirectoryService or system accounts. + +# AWS Account and Region Settings +aws_account_id: 123456789012 +aws_dns_suffix: amazonaws.com +aws_partition: aws +aws_region: us-east-1 + +# Shared Storage Settings +storage_apps_provider: efs +apps_mount_dir: /apps +storage_data_provider: efs +data_mount_dir: /data + +# Network Settings +prefix_list_ids: [ ] +client_ip: [ ] +ssh_key_pair_name: my-ec2-key-pair +vpc_cidr_block: 10.0.0.0/16 +use_vpc_endpoints: false + +# Application LoadBalancer Settings +# if alb_public = true external ALB will be deployed in public subnet +# if alb_public = false external ALB will be deployed in private subnet +alb_public: true + +# if alb_custom_certificate_provided = false, self signed certificates will be generated for external ALB +# if alb_custom_certificate_provided = true, import your own certificates to AWS ACM and provide alb_custom_certificate_acm_certificate_arn, alb_custom_dns_name +alb_custom_certificate_provided: +alb_custom_certificate_acm_certificate_arn: ~ +alb_custom_dns_name: ~ + +# Identity Provider +identity_provider: cognito-idp + +# Directory Service +directory_service_provider: openldap # openldap or activedirectory + +# Provide a custom KMS Key ID for Secrets and Shared Storage Encryption/Decryption +kms_key_id: ~ + +# Provide the default Instance Types for all infrastructure EC2 instances +instance_type: ~ +base_os: ~ +instance_ami: ~ +volume_size: 200 + +# Provide a list of module names to be enabled. cluster, directoryservice, cluster-manager and analytics are mandatory modules for IDEA to operate and will always be deployed +enabled_modules: [ ] + +# metrics +metrics_provider: cloudwatch +# required when metrics_provider = prometheus +prometheus_remote_write_url: ~ + +# aws backup +enable_aws_backup: true + +# Virtual Desktop Controller - DCV Broker +dcv_broker_instance_ami: ~ +dcv_broker_instance_type: ~ +dcv_broker_volume_size: 200 + +# Virtual Desktop Controller - DCV Connection Gateway +dcv_connection_gateway_instance_ami: ~ +dcv_connection_gateway_instance_type: ~ +dcv_connection_gateway_volume_size: 200 +# Virtual Desktop Controller - DCV Gateway/Reverse Proxy Server +reverse_proxy_server_instance_ami: ~ +reverse_proxy_server_instance_type: ~ +reverse_proxy_server_volume_size: 200 + +# dcv_session_quic_support is the flag to control for QUIC support. +# if dcv_session_quic_support: true, then we can not use internal certificates, turn dcv_session_nlb_custom_certificate_provided to true +dcv_session_quic_support: false + +# if dcv_connection_gateway_custom_certificate_provided: true, generate your own self signed certificates import the certificate content and private key to 2 separate secrets and provide +# dcv_connection_gateway_custom_certificate_certificate_secret_arn, dcv_connection_gateway_custom_certificate_private_key_secret_arn, dcv_connection_gateway_custom_dns_hostname +# if dcv_connection_gateway_custom_certificate_provided: false, we will create our internal self signed certificates and use our internal hostname. +dcv_connection_gateway_custom_certificate_provided: false +dcv_connection_gateway_custom_certificate_certificate_secret_arn: ~ +dcv_connection_gateway_custom_certificate_private_key_secret_arn: ~ +dcv_connection_gateway_custom_dns_hostname: ~ + +# Build cluster using existing resources +use_existing_vpc: false +vpc_id: ~ # value is required when use_existing_vpc == true +private_subnet_ids: [ ] # value is required when use_existing_vpc == true +public_subnet_ids: [ ] # value is required when use_existing_vpc == true + +use_existing_apps_fs: false # use_existing_vpc should be true when use_existing_apps_fs == true +existing_apps_fs_id: ~ # value is required when use_existing_apps_fs == true + +use_existing_data_fs: false # use_existing_vpc should be true when use_existing_data_fs == true +existing_data_fs_id: ~ # value is required when use_existing_data_fs == true + +use_existing_opensearch_cluster: false # use_existing_vpc should be true when use_existing_opensearch_cluster == true +opensearch_domain_endpoint: ~ # value is required when use_existing_opensearch_cluster == true + +use_existing_directory_service: false # use_existing_vpc should be true when use_existing_directory_service == true +directory_id: ~ # value is required when use_existing_directory_service == true +directory_service_root_username_secret_arn: ~ # value is required when use_existing_directory_service == true +directory_service_root_password_secret_arn: ~ # value is required when use_existing_directory_service == true + diff --git a/source/idea/idea-administrator/resources/input_params/install_params.yml b/source/idea/idea-administrator/resources/input_params/install_params.yml new file mode 100644 index 00000000..038d5760 --- /dev/null +++ b/source/idea/idea-administrator/resources/input_params/install_params.yml @@ -0,0 +1,665 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +SocaInputParamSpec: + + name: idea-installation-params + title: "IDEA Installation User Input Params" + version: 1.0.0 + + modules: + - name: install-idea + title: "Install IDEA" + sections: + - name: aws-account + title: "AWS Credentials and Region" + required: yes + params: + - name: aws_profile + - name: aws_partition + - name: aws_profile2 + - name: aws_region + - name: cluster-settings + title: "Cluster Settings" + required: yes + params: + - name: cluster_name + - name: administrator_email + - name: vpc_cidr_block + - name: ssh_key_pair_name + - name: cluster_access + - name: client_ip + - name: prefix_list_ids + - name: alb_public + - name: use_vpc_endpoints + - name: directory_service_provider + - name: enable_aws_backup + - name: kms_key_type + - name: kms_key_id + - name: module-settings + title: "Module Settings" + required: yes + params: + - name: enabled_modules + # - name: identity_provider todo - enable after keycloak implementation + - name: metrics_provider + - name: prometheus_remote_write_url + - name: base_os + - name: instance_type + - name: volume_size + + - name: install-idea-using-existing-resources + title: "Install IDEA (using existing resources)" + sections: + - name: aws-account + title: "AWS Credentials and Region" + required: yes + params: + - name: aws_profile + - name: aws_partition + - name: aws_profile2 + - name: aws_region + - name: cluster-settings + title: "Cluster Settings" + required: yes + params: + - name: cluster_name + - name: administrator_email + - name: ssh_key_pair_name + - name: cluster_access + - name: client_ip + - name: prefix_list_ids + - name: alb_public + - name: use_vpc_endpoints + - name: directory_service_provider + - name: enable_aws_backup + - name: kms_key_type + - name: kms_key_id + - name: existing-resources + title: "Existing Resources" + required: yes + params: + - name: vpc_id + - name: existing_resources + - name: public_subnet_ids + - name: private_subnet_ids + - name: directory_id + - name: directory_service_root_username_secret_arn + - name: directory_service_root_password_secret_arn + - name: storage_apps_provider + - name: existing_apps_fs_id + - name: storage_data_provider + - name: existing_data_fs_id + - name: opensearch_domain_endpoint + - name: module-settings + title: "Module Settings" + required: yes + params: + - name: enabled_modules + # - name: identity_provider todo - enable after keycloak implementation + - name: metrics_provider + - name: prometheus_remote_write_url + - name: base_os + - name: instance_type + - name: volume_size + + #-------------------------------------------------------------------------------------------------- + # deployment input parameters + # these parameters can be included or customized in the deployment options configured above + #-------------------------------------------------------------------------------------------------- + params: + #-------------------------------------------------------------------------------------------------- + # aws credential selection params + #-------------------------------------------------------------------------------------------------- + - name: aws_profile + title: "AWS Profile" + description: "Select the AWS Profile you wish to use for IDEA Installation:" + param_type: select + data_type: str + help_text: ~ + default: default + validate: + required: yes + tag: default + + - name: aws_partition + title: "AWS Partition" + description: "Select the AWS Partition you wish to use for the IDEA cluster:" + param_type: select + data_type: str + help_text: ~ + default: aws + tag: default + markdown: | + A Partition is a group of AWS Region and Service objects. + We use the Partition to determine the applicable AWS Regions in which you want to install IDEA. + + # In some scenarios a secondary AWS Profile is required to bootstrap the installation. + # This is common when the selected partition/region lacks the SSM namespace to probe + # for service availability. This question is only asked if the 'when' clause is matched. + # Defaults to ask when the aws_partition is aws-us-gov (US GovCloud) + - name: aws_profile2 + title: "AWS Profile2" + description: "Select the AWS Commercial Profile you wish to use for IDEA Installation:" + param_type: select + data_type: str + help_text: ~ + default: default + validate: + required: yes + tag: default + when: + param: aws_partition + eq: aws-us-gov + + - name: aws_region + title: "AWS Region" + description: "Select the AWS Region you want to use for the IDEA cluster:" + param_type: select + data_type: str + help_text: ~ + default: ~ + validate: + required: yes + tag: default + custom: + defaults: + aws: us-east-1 + aws-cn: cn-north-1 + aws-us-gov: us-gov-east-1 + aws-iso: us-iso-east-1 + aws-iso-b: us-isob-east-1 + + #-------------------------------------------------------------------------------------------------- + # cluster params + #-------------------------------------------------------------------------------------------------- + - name: cluster_name + title: "Cluster Name" + description: "Enter the IDEA Cluster" + param_type: text + data_type: str + help_text: "eg. 'prod', 'beta' or 'dev'. ClusterName will be automatically prefixed 'idea-'" + default: ~ + validate: + required: yes + auto_prefix: 'idea-' + + - name: administrator_email + title: "Administrator Email Address" + description: "Enter the email address of the IDEA Cluster Administrator" + param_type: text + data_type: str + help_text: ~ + default: ~ + validate: + required: yes + regex: ^(\S+@\S+)$ + markdown: | + An administrator account will be created using this email address. + You'll receive an invitation email with the administrator username and a temporary password. + + #-------------------------------------------------------------------------------------------------- + # network params + #-------------------------------------------------------------------------------------------------- + - name: cluster_access + title: "Cluster Access" + description: "Select IP Address or a Prefix List to access cluster resources." + param_type: select + data_type: str + help_text: ~ + default: $first + choices: + - title: Use IP Address / CIDR Block + value: client-ip + - title: Use VPC Prefix List + value: prefix-list + tag: default + markdown: ~ + + - name: prefix_list_ids + title: "Prefix List" + description: "Enter existing VPC Prefix List Ids" + param_type: text + multiple: true + data_type: str + custom_type: vpc-prefix-list + help_text: "multiple prefix list ids can be separated by comma" + tag: default + validate: + required: yes + when: + param: cluster_access + eq: prefix-list + + - name: client_ip + title: "Client IP" + description: "Enter the IP Address or CIDR Block to allow access to Cluster:" + param_type: text + data_type: str + help_text: ~ + default: ~ + refreshable: yes + validate: + regex: ^([0-9]{1,3}\.){3}[0-9]{1,3}(\/([0-9]|[1-2][0-9]|3[0-2]))?$ + required: yes + when: + param: cluster_access + eq: client-ip + + - name: vpc_cidr_block + title: "VPC CIDR Block" + description: "Enter the CIDR Block for your VPC" + param_type: text + data_type: str + help_text: ~ + default: 10.0.0.0/16 + validate: + regex: ^([0-9]{1,3}\.){3}[0-9]{1,3}($|/(16|18|20|22|24|26))$ + required: yes + + - name: ssh_key_pair_name + title: "SSH KeyPair" + description: "Select the SSH KeyPair for accessing the EC2 Instance" + param_type: select + data_type: str + help_text: ~ + default: $first + validate: + required: yes + markdown: ~ + + - name: use_vpc_endpoints + title: "Use VPC Endpoints?" + description: "Do you want to use VPC Endpoints?" + param_type: confirm + data_type: bool + help_text: "VPC endpoints allow traffic to flow between a VPC and other services without ever leaving the AWS network" + default: no + + - name: confirm_vpc_endpoints + title: "VPC Endpoints" + description: "VPC Endpoint Configuration" + param_type: confirm + data_type: bool + help_text: "Verify the above VPC endpoint configuration." + default: ~ + required: true + when: + param: use_vpc_endpoints + eq: yes + + - name: enable_aws_backup + title: "Enable AWS Backup?" + description: "Do you want to enable integration with AWS Backup?" + param_type: confirm + data_type: bool + help_text: ~ + default: yes + + - name: kms_key_type + title: "KMS Encryption Key Type" + description: "Select the encryption key type for the cluster" + param_type: select + data_type: str + help_text: ~ + default: $first + choices: + - title: AWS Managed + value: aws-managed + - title: Customer Managed + value: customer-managed + validate: + required: yes + markdown: ~ + + - name: kms_key_id + title: "Customer Managed KMS Key ID" + description: "Enter the ID of the customer managed key from AWS Key Management Service" + param_type: select + data_type: str + help_text: ~ + default: $first + validate: + required: yes + markdown: ~ + when: + param: kms_key_type + eq: customer-managed + + #-------------------------------------------------------------------------------------------------- + # external application load balancer params + #-------------------------------------------------------------------------------------------------- + - name: alb_public + title: "Is ALB Public?" + description: "Deploy application load balancer in public subnets?" + param_type: confirm + data_type: bool + help_text: "Recommended: Yes" + default: yes + + #-------------------------------------------------------------------------------------------------- + # module settings + #-------------------------------------------------------------------------------------------------- + + - name: enabled_modules + title: "IDEA Modules" + description: "Select all applicable modules you want to deploy" + param_type: checkbox + data_type: str + multiple: true + help_text: ~ + tag: default + markdown: ~ + validate: + required: yes + + - name: identity_provider + title: "Identity Provider" + description: "Select the Identity Provider" + param_type: select + data_type: str + help_text: ~ + default: openldap + tag: default + choices: + - title: Amazon Cognito UserPool + value: cognito-idp + - title: Keycloak + value: keycloak + validate: + required: yes + + - name: directory_service_provider + title: "Directory Service" + description: "Select the DirectoryService Provider" + param_type: select + data_type: str + help_text: ~ + default: openldap + tag: default + choices: + - title: OpenLDAP + value: openldap + - title: AWS Managed Microsoft AD + value: aws_managed_activedirectory + - title: Microsoft Active Directory (On-Prem or Self-Managed) + value: activedirectory + validate: + required: yes + + - name: base_os + title: "Base OS" + description: "Select the Base Operating System:" + param_type: select + data_type: str + help_text: ~ + default: amazonlinux2 + choices: + - title: Amazon Linux 2 + value: amazonlinux2 + - title: CentOS 7 + value: centos7 + - title: Red Hat Enterprise Linux 7 + value: rhel7 + tag: default + markdown: ~ + + - name: instance_type + title: "Instance Type" + description: "Select the EC2 Instance Type:" + param_type: select + data_type: str + help_text: ~ + default: m5.large + tag: default + choices: + - value: t3.medium + - value: t3.large + - value: m5.large + - value: m5.xlarge + validate: + required: yes + + - name: volume_size + title: "Volume Size (GB)" + description: "Enter the storage volume size for node" + param_type: text + data_type: int + help_text: "Size of the EBS root disk in GBs" + default: 200 + validate: + required: yes + min: 20 + max: 1000 + + #-------------------------------------------------------------------------------------------------- + # metrics settings + #-------------------------------------------------------------------------------------------------- + + - name: metrics_provider + title: "Metrics Provider" + description: "Select a metrics provider" + param_type: select + data_type: str + help_text: ~ + default: $first + validate: + required: yes + when: + param: enabled_modules + contains: metrics + + - name: prometheus_remote_write_url + title: "Remote Write URL" + description: "Enter your custom prometheus remote write url" + param_type: text + data_type: str + help_text: ~ + validate: + required: yes + when: + param: metrics_provider + eq: prometheus + + #-------------------------------------------------------------------------------------------------- + # existing resources + #-------------------------------------------------------------------------------------------------- + - name: vpc_id + title: "VPC" + description: "Select an existing VPC to deploy IDEA" + param_type: select + data_type: str + help_text: ~ + default: $first + validate: + required: yes + + - name: existing_resources + title: "Existing Resources" + description: "Select existing resources from the VPC you want to use for the cluster" + param_type: checkbox + data_type: str + multiple: true + help_text: ~ + tag: default + markdown: ~ + validate: + required: yes + + - name: private_subnet_ids + title: "Existing Private Subnets" + description: "Select existing private subnets" + param_type: checkbox + data_type: str + multiple: true + help_text: ~ + tag: default + markdown: ~ + validate: + required: yes + when: + param: existing_resources + contains: 'subnets:private' + + - name: public_subnet_ids + title: "Existing Public Subnets" + description: "Select existing public subnets" + param_type: checkbox + data_type: str + multiple: true + help_text: ~ + tag: default + markdown: ~ + validate: + required: yes + when: + param: existing_resources + contains: 'subnets:public' + + - name: storage_apps_provider + title: "Storage Provider: Apps" + description: "Select apps storage provider" + param_type: select + data_type: str + help_text: ~ + tag: default + markdown: ~ + validate: + required: yes + choices: + - title: 'Amazon EFS' + value: efs + - title: 'Amazon FSx for Lustre' + value: fsx_lustre + when: + param: existing_resources + contains: 'shared-storage:apps' + + - name: existing_apps_fs_id + title: "Existing Apps File System" + description: "Select existing file system for Apps" + param_type: select + data_type: str + help_text: ~ + tag: default + markdown: ~ + validate: + required: yes + when: + param: existing_resources + contains: 'shared-storage:apps' + + - name: storage_data_provider + title: "Storage Provider: Data" + description: "Select data storage provider" + param_type: select + data_type: str + help_text: ~ + tag: default + markdown: ~ + validate: + required: yes + choices: + - title: 'Amazon EFS' + value: efs + - title: 'Amazon FSx for Lustre' + value: fsx_lustre + when: + param: existing_resources + contains: 'shared-storage:data' + + - name: existing_data_fs_id + title: "Existing Data File System" + description: "Select existing file system for Data" + param_type: select + data_type: str + help_text: ~ + tag: default + markdown: ~ + validate: + required: yes + when: + param: existing_resources + contains: 'shared-storage:data' + + - name: opensearch_domain_endpoint + title: "Existing OpenSearch Service Domain" + description: "Select existing OpenSearch Service Domain" + param_type: select + data_type: str + help_text: ~ + tag: default + markdown: ~ + validate: + required: yes + when: + param: existing_resources + contains: 'analytics:opensearch' + + - name: directory_id + title: "Existing Directory" + description: "Select existing AWS Managed Microsoft AD" + param_type: select + data_type: str + help_text: ~ + tag: default + markdown: ~ + validate: + required: yes + when: + and: + - param: existing_resources + contains: directoryservice:aws_managed_activedirectory + - param: directory_service_provider + eq: aws_managed_activedirectory + + - name: directory_service_root_username_secret_arn + title: "Microsoft AD Service Account Name" + description: "Enter the ARN of the secret containing service account username" + param_type: text + data_type: str + help_text: ~ + tag: default + markdown: ~ + validate: + required: yes + when: + and: + - param: existing_resources + contains: directoryservice:aws_managed_activedirectory + - param: directory_service_provider + eq: aws_managed_activedirectory + + - name: directory_service_root_password_secret_arn + title: "AD Service Account Password" + description: "Enter the ARN of the secret containing service account password" + param_type: text + data_type: str + help_text: ~ + tag: default + markdown: ~ + validate: + required: yes + when: + and: + - param: existing_resources + contains: directoryservice:aws_managed_activedirectory + - param: directory_service_provider + eq: aws_managed_activedirectory + + #-------------------------------------------------------------------------------------------------- + # tags: these indicate the icon or status of a parameter. + # developer note: currently these are not used, but do not ignore them, as they will be used + # in a future release. + #-------------------------------------------------------------------------------------------------- + tags: + - name: default + ascii: '?' diff --git a/source/idea/idea-administrator/resources/input_params/shared_storage_params.yml b/source/idea/idea-administrator/resources/input_params/shared_storage_params.yml new file mode 100644 index 00000000..19eb73a2 --- /dev/null +++ b/source/idea/idea-administrator/resources/input_params/shared_storage_params.yml @@ -0,0 +1,578 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +SocaInputParamSpec: + + name: shared-storage-params + title: "IDEA Shared Storage User Input Params" + version: 1.0.0 + + modules: + + - name: common-settings + title: "Add Shared Storage to an IDEA Cluster" + sections: + - name: shared-storage + title: "Shared Storage Settings" + required: yes + params: + - name: shared_storage_name + - name: shared_storage_title + - name: vpc_id + - name: shared_storage_provider + - name: shared_storage_mount_dir + - name: shared_storage_mount_drive + - name: shared_storage_scope + - name: shared_storage_scope_modules + - name: shared_storage_scope_projects + - name: shared_storage_scope_queue_profiles + + - name: efs_new + title: "Amazon EFS (New)" + sections: + - name: params + title: "New Amazon EFS Settings" + required: yes + params: + - name: efs.throughput_mode + - name: efs.performance_mode + - name: efs.cloudwatch_monitoring + - name: efs.transition_to_ia + - name: efs.mount_options + + - name: efs_existing + title: "Amazon EFS (Existing)" + sections: + - name: params + title: "Existing Amazon EFS Settings" + required: yes + params: + - name: efs.file_system_id + - name: efs.mount_options + + - name: fsx_cache_existing + title: "Amazon File Cache (Existing)" + sections: + - name: params + title: "Existing Amazon File Cache Settings" + required: yes + params: + - name: fsx_cache.file_system_id + - name: fsx_cache.mount_options + + - name: fsx_lustre_existing + title: "FSx for Lustre (Existing)" + sections: + - name: params + title: "Existing FSx for Lustre Settings" + required: yes + params: + - name: fsx_lustre.file_system_id + - name: fsx_lustre.mount_options + + - name: fsx_netapp_ontap_existing + title: "FSx for NetApp ONTAP (Existing)" + sections: + - name: params + title: "Existing FSx for NetApp ONTAP Settings" + required: yes + params: + - name: fsx_netapp_ontap.file_system_id + - name: fsx_netapp_ontap.svm_id + - name: fsx_netapp_ontap.volume_id + - name: fsx_netapp_ontap.mount_options + - name: fsx_netapp_ontap.cifs_share_name + + - name: fsx_openzfs_existing + title: "FSx for OpenZFS (Existing)" + sections: + - name: params + title: "Existing FSx for OpenZFS Settings" + required: yes + params: + - name: fsx_openzfs.file_system_id + - name: fsx_openzfs.volume_id + - name: fsx_openzfs.mount_options + + - name: fsx_windows_file_server_existing + title: "FSx for Windows File Server (Existing)" + sections: + - name: params + title: "Existing FSx for Windows File Server Settings" + required: yes + params: + - name: fsx_windows_file_server.file_system_id + + #-------------------------------------------------------------------------------------------------- + # shared storage input parameters + # these parameters can be included or customized in user input modules configured above + #-------------------------------------------------------------------------------------------------- + params: + + #-------------------------------------------------------------------------------------------------- + # Common Settings + #-------------------------------------------------------------------------------------------------- + - name: shared_storage_name + title: "Name" + description: "Enter the name of the shared storage file system" + param_type: text + data_type: str + help_text: "Must be all lower case, no spaces or special characters" + default: ~ + validate: + required: yes + regex: ^[a-z0-9_]{2,64}$ + tag: default + + - name: shared_storage_title + title: "Title" + description: "Enter a friendly title for the file system" + param_type: text + data_type: str + help_text: ~ + default: ~ + validate: + required: yes + tag: default + + - name: shared_storage_description + title: "Description" + description: "Enter a friendly description for the file system" + param_type: text + data_type: str + help_text: ~ + default: ~ + validate: + required: yes + tag: default + + - name: shared_storage_provider + title: "Shared Storage Provider" + description: "Select a provider for the shared storage file system" + param_type: select + data_type: str + help_text: new file system provisioning only supported for Amazon EFS + default: $first + validate: + required: yes + tag: default + + - name: shared_storage_mount_dir + title: "Mount Directory" + description: "Location of the mount directory. eg. /my-mount-dir" + param_type: text + data_type: str + help_text: ~ + default: ~ + validate: + required: yes + regex: ^(/)([^/\0]+(/)?)+$ + tag: default + when: + param: shared_storage_provider + not_eq: fsx_windows_file_server + + - name: shared_storage_mount_drive + title: "Mount Drive" + description: "The mount drive letter for Windows, eg. Z" + param_type: text + data_type: str + help_text: 'without colon (:)' + default: ~ + validate: + required: yes + regex: ^[A-Z]{1}$ + tag: default + when: + param: shared_storage_provider + in: + - fsx_windows_file_server + - fsx_netapp_ontap + + - name: shared_storage_scope + title: "Mount Scopes" + description: "Select the mount scope for file system" + param_type: select + data_type: str + help_text: ~ + default: $first + choices: + - title: Cluster + value: cluster + - title: Module + value: module + - title: Project + value: project + - title: Queue Profile (Scale-Out Computing) + value: scheduler:queue-profile + validate: + required: yes + tag: default + + - name: shared_storage_scope_modules + title: "Module" + description: "Select the modules for which file system must to be mounted" + param_type: checkbox + data_type: str + multiple: true + help_text: ~ + default: + - bastion-host + - cluster-manager + choices: + - title: Bastion Host + value: bastion-host + - title: Cluster Manager + value: cluster-manager + - title: Scale-Out Computing + value: scheduler + - title: eVDI + value: virtual-desktop-controller + validate: + required: yes + when: + param: shared_storage_scope + contains: module + tag: default + + - name: shared_storage_scope_projects + title: "Project" + description: "Enter project code names" + param_type: text + data_type: str + multiple: true + help_text: multiple project codes can be separated by comma + validate: + required: yes + when: + param: shared_storage_scope + contains: project + tag: default + + - name: shared_storage_scope_queue_profiles + title: "Queue Profiles" + description: "Enter queue profile names" + param_type: text + data_type: str + multiple: true + help_text: multiple queue profile names can be separated by comma + validate: + required: yes + when: + param: shared_storage_scope + contains: scheduler:queue-profile + tag: default + + - name: use_existing_fs + title: "Use Existing File System?" + description: "Do you want to use an existing file system?" + param_type: confirm + data_type: bool + help_text: + default: yes + validate: + required: yes + tag: default + + - name: vpc_id + title: "VPC" + description: "Select the VPC from which an existing file system can be used" + param_type: select + data_type: str + help_text: ~ + default: $first + validate: + required: yes + when: + and: + - param: use_existing_fs + eq: yes + - param: vpc_id + empty: true + + #-------------------------------------------------------------------------------------------------- + # Amazon EFS + #-------------------------------------------------------------------------------------------------- + + - name: efs.throughput_mode + title: "Throughput Mode" + description: "Select the throughput mode" + param_type: select + data_type: str + help_text: ~ + default: $first + choices: + - title: Bursting + value: bursting + - title: Provisioned + value: provisioned + tag: default + markdown: ~ + validate: + required: yes + + - name: efs.performance_mode + title: "Performance Mode" + description: "Select the performance mode" + param_type: select + data_type: str + help_text: ~ + default: $first + choices: + - title: General Purpose + value: generalPurpose + - title: MaxIO + value: maxIO + tag: default + markdown: ~ + validate: + required: yes + + - name: efs.cloudwatch_monitoring + title: "Enable CloudWatch Monitoring" + description: "Enable cloudwatch monitoring to manage throughput?" + param_type: confirm + data_type: bool + help_text: + default: false + validate: + required: yes + tag: default + + - name: efs.transition_to_ia + title: "Lifecycle Policy" + description: "Transition to infrequent access (IA) storage?" + param_type: select + data_type: str + help_text: ~ + default: $first + choices: + - title: Transition to IA Disabled + value: DISABLED + - title: After 7 Days + value: AFTER_7_DAYS + - title: After 14 Days + value: AFTER_14_DAYS + - title: After 30 Days + value: AFTER_30_DAYS + - title: After 60 Days + value: AFTER_60_DAYS + - title: After 90 Days + value: AFTER_90_DAYS + tag: default + markdown: ~ + validate: + required: no + + - name: efs.file_system_id + title: "Existing Amazon EFS" + description: "Select an existing Amazon EFS file system" + param_type: select + data_type: str + help_text: ~ + tag: default + markdown: ~ + validate: + required: yes + + - name: efs.mount_options + title: "EFS Mount Options" + description: "Select EFS mount method" + param_type: select + data_type: str + help_text: ~ + default: $first + choices: + - title: Use NFSv4 Mount Options + value: "nfs4 nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2,noresvport 0 0" + - title: Use Amazon EFS Mount Helper (required for TLS 1.2 / encryption of data in transit) + value: "efs _netdev,noresvport,tls,iam 0 0" + tag: default + markdown: ~ + validate: + required: yes + + #-------------------------------------------------------------------------------------------------- + # Amazon File Cache (uses Lustre client but different APIs) + #-------------------------------------------------------------------------------------------------- + + - name: fsx_cache.file_system_id + title: "Existing Amazon File Cache" + description: "Select an existing Amazon File Cache file system" + param_type: select + data_type: str + help_text: ~ + tag: default + markdown: ~ + validate: + required: yes + + - name: fsx_cache.mount_options + title: "Mount Options" + description: "Enter /etc/fstab mount options" + param_type: select + data_type: str + help_text: ~ + default: 'lustre defaults,noatime,flock,_netdev 0 0' + tag: default + markdown: ~ + validate: + required: yes + + #-------------------------------------------------------------------------------------------------- + # Amazon FSx for Lustre + #-------------------------------------------------------------------------------------------------- + + - name: fsx_lustre.file_system_id + title: "Existing FSx for Lustre" + description: "Select an existing Lustre file system" + param_type: select + data_type: str + help_text: ~ + tag: default + markdown: ~ + validate: + required: yes + + - name: fsx_lustre.mount_options + title: "Mount Options" + description: "Enter /etc/fstab mount options" + param_type: select + data_type: str + help_text: ~ + default: 'lustre defaults,noatime,flock,_netdev 0 0' + tag: default + markdown: ~ + validate: + required: yes + + #-------------------------------------------------------------------------------------------------- + # Amazon FSx for NetApp ONTAP + #-------------------------------------------------------------------------------------------------- + + - name: fsx_netapp_ontap.file_system_id + title: "Existing FSx for NetApp ONTAP" + description: "Select an existing ONTAP file system" + param_type: select + data_type: str + help_text: ~ + tag: default + markdown: ~ + validate: + required: yes + + - name: fsx_netapp_ontap.svm_id + title: "Storage Virtual Machine" + description: "Select an existing SVM to connect to ONTAP file system" + param_type: select + data_type: str + help_text: ~ + tag: default + markdown: ~ + validate: + required: yes + + - name: fsx_netapp_ontap.volume_id + title: "Existing FSx for NetApp ONTAP Volume" + description: "Select an existing NetApp ONTAP Volume" + param_type: select + data_type: str + help_text: ~ + tag: default + markdown: ~ + validate: + required: yes + + - name: fsx_netapp_ontap.mount_options + title: "Mount Options" + description: "Enter mount options" + param_type: select + data_type: str + help_text: ~ + default: 'nfs4 nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2,noresvport 0 0' + tag: default + markdown: ~ + validate: + required: yes + + - name: fsx_netapp_ontap.cifs_share_name + title: "Existing FSx for NetApp ONTAP CIFS share name" + description: "Enter an existing NetApp ONTAP CIFS share name" + param_type: text + data_type: str + help_text: ~ + tag: default + markdown: ~ + validate: + required: yes + regex: ^[\w`~!@#$%^&(){}'._-]{2,256}$ + + #-------------------------------------------------------------------------------------------------- + # Amazon FSx for OpenZFS + #-------------------------------------------------------------------------------------------------- + + - name: fsx_openzfs.file_system_id + title: "Existing FSx for OpenZFS" + description: "Select an existing OpenZFS file system" + param_type: select + data_type: str + help_text: ~ + tag: default + markdown: ~ + validate: + required: yes + + - name: fsx_openzfs.volume_id + title: "Existing FSx for OpenZFS Volume" + description: "Select an existing OpenZFS Volume" + param_type: select + data_type: str + help_text: ~ + tag: default + markdown: ~ + validate: + required: yes + + - name: fsx_openzfs.mount_options + title: "Mount Options" + description: "Enter mount options" + param_type: select + data_type: str + help_text: ~ + default: 'nfs4 nfsvers=4.1,rsize=1048576,wsize=1048576,timeo=600 0 0' + tag: default + markdown: ~ + validate: + required: yes + + #-------------------------------------------------------------------------------------------------- + # Amazon FSx for Windows File Server + #-------------------------------------------------------------------------------------------------- + + - name: fsx_windows_file_server.file_system_id + title: "Existing FSx for Windows File Server" + description: "Select an existing Windows File Server file system" + param_type: select + data_type: str + help_text: ~ + tag: default + markdown: ~ + validate: + required: yes + + #-------------------------------------------------------------------------------------------------- + # tags: these indicate the icon or status of a parameter. + # developer note: currently these are not used, but do not ignore them, as they will be used + # in a future release. + #-------------------------------------------------------------------------------------------------- + tags: + - name: default + ascii: '?' diff --git a/source/idea/idea-administrator/resources/installer_policies/idea-admin-cdk-toolkit-policy.yml b/source/idea/idea-administrator/resources/installer_policies/idea-admin-cdk-toolkit-policy.yml new file mode 100644 index 00000000..85e64a4e --- /dev/null +++ b/source/idea/idea-administrator/resources/installer_policies/idea-admin-cdk-toolkit-policy.yml @@ -0,0 +1,31 @@ +Version: '2012-10-17' +Statement: + - Sid: S3BucketPermissionsForCDK + Effect: Allow + Action: + - s3:PutEncryptionConfiguration + - s3:PutBucketVersioning + - s3:PutBucketPublicAccessBlock + - s3:PutBucketPolicy + - s3:GetBucketPolicy + - s3:CreateBucket + Resource: arn:aws:s3:::idea* + - Sid: CreateCDKToolkit + Effect: Allow + Action: + - ecr:CreateRepository + - ecr:DescribeRepositories + - ecr:SetRepositoryPolicy + - ec2:DescribeAccountAttributes + - ssm:GetParameter + - ssm:GetParameters + - ssm:PutParameter + - ssm:DeleteParameter + - ssm:GetParametersByPath + - iam:CreateRole + - iam:CreateServiceLinkedRole + - iam:GetRole + - iam:PutRolePolicy + - sts:AssumeRole + - ssm:AddTagsToResource + Resource: '*' diff --git a/source/idea/idea-administrator/resources/installer_policies/idea-admin-create-policy.yml b/source/idea/idea-administrator/resources/installer_policies/idea-admin-create-policy.yml new file mode 100644 index 00000000..30cd3da1 --- /dev/null +++ b/source/idea/idea-administrator/resources/installer_policies/idea-admin-create-policy.yml @@ -0,0 +1,141 @@ +Version: '2012-10-17' +Statement: + - Sid: CreatePermissionsToInstallIDEA + Effect: Allow + Action: + - sns:CreateTopic + - sns:GetTopicAttributes + - sns:ListSubscriptionsByTopic + - sns:SetTopicAttributes + - sns:Subscribe + - sns:TagResource + - backup-storage:MountCapsule + - backup:CreateBackupPlan + - backup:CreateBackupSelection + - backup:CreateBackupVault + - backup:DescribeBackupVault + - backup:GetBackupPlan + - backup:GetBackupSelection + - backup:TagResource + - cloudformation:CreateChangeSet + - cloudformation:CreateStack + - cloudformation:DeleteChangeSet + - cloudformation:DescribeChangeSet + - cloudformation:DescribeStackEvents + - cloudformation:DescribeStacks + - cloudformation:ExecuteChangeSet + - cloudformation:GetTemplate + - cloudformation:UpdateTerminationProtection + - cloudwatch:PutMetricAlarm + - ds:CreateMicrosoftAD + - ds:DescribeDirectories + - ec2:DescribeImages + - ec2:AllocateAddress + - ec2:AssociateAddress + - ec2:AssociateRouteTable + - ec2:AttachInternetGateway + - ec2:AuthorizeSecurityGroupEgress + - ec2:AuthorizeSecurityGroupIngress + - ec2:CreateFlowLogs + - ec2:CreateInternetGateway + - ec2:CreateNatGateway + - ec2:CreateNetworkInterface + - ec2:CreateNetworkInterfacePermission + - ec2:CreateRoute + - ec2:CreateRouteTable + - ec2:CreateSecurityGroup + - ec2:CreateSubnet + - ec2:CreateTags + - ec2:CreateVpc + - ec2:CreateVpcEndpoint + - ec2:DescribeAddresses + - ec2:DescribeAvailabilityZones + - ec2:DescribeFlowLogs + - ec2:DescribeInstances + - ec2:DescribeInternetGateways + - ec2:DescribeKeyPairs + - ec2:DescribeNatGateways + - ec2:DescribeNetworkInterfaces + - ec2:DescribeNetwork* + - ec2:DescribeRegions + - ec2:DescribeRouteTables + - ec2:DescribeSecurityGroups + - ec2:DescribeSubnets + - ec2:DescribeVpcAttribute + - ec2:DescribeVpcEndpoints + - ec2:DescribeVpcEndpointServices + - ec2:DescribeVpcs + - ec2:ModifySubnetAttribute + - ec2:ModifyVpcAttribute + - ec2:RevokeSecurityGroupEgress + - ec2:RunInstances + - elasticfilesystem:CreateFileSystem + - elasticfilesystem:CreateMountTarget + - elasticfilesystem:DescribeFileSystems + - elasticfilesystem:DescribeMountTargets + - elasticfilesystem:PutLifecycleConfiguration + - elasticloadbalancing:AddTags + - elasticloadbalancing:CreateListener + - elasticloadbalancing:CreateLoadBalancer + - elasticloadbalancing:CreateRule + - elasticloadbalancing:CreateTargetGroup + - elasticloadbalancing:DeleteTargetGroup + - elasticloadbalancing:DescribeListeners + - elasticloadbalancing:DescribeLoadBalancers + - elasticloadbalancing:DescribeRules + - elasticloadbalancing:DescribeTargetGroups + - elasticloadbalancing:DescribeTargetHealth + - elasticloadbalancing:ModifyLoadBalancerAttributes + - elasticloadbalancing:ModifyRule + - elasticloadbalancing:RegisterTargets + - es:AddTags + - es:CreateElasticsearchDomain + - es:DescribeElasticsearchDomain + - es:ListDomainNames + - fsx:CreateFileSystem + - fsx:DescribeFileSystems + - fsx:TagResource + - iam:AddRoleToInstanceProfile + - iam:AttachRolePolicy + - iam:CreateInstanceProfile + - iam:CreateRole + - iam:GetRole + - iam:GetRolePolicy + - iam:ListRoles + - iam:PassRole + - iam:PutRolePolicy + - iam:TagRole + - kms:CreateGrant + - kms:Decrypt + - kms:DescribeKey + - kms:GenerateDataKey + - kms:RetireGrant + - lambda:AddPermission + - lambda:CreateFunction + - lambda:GetFunction + - lambda:InvokeFunction + - logs:CreateLogGroup + - logs:DescribeLogGroups + - logs:PutRetentionPolicy + - route53resolver:AssociateResolverEndpointIpAddress + - route53resolver:AssociateResolverRule + - route53resolver:CreateResolverEndpoint + - route53resolver:CreateResolverRule + - route53resolver:GetResolverEndpoint + - route53resolver:GetResolverRule + - route53resolver:GetResolverRuleAssociation + - route53resolver:PutResolverRulePolicy + - route53resolver:TagResource + - s3:*Object + - s3:GetBucketLocation + - s3:ListBucket + - secretsmanager:CreateSecret + - secretsmanager:TagResource + - sts:DecodeAuthorizationMessage + - dynamodb:* + - events:* + - sqs:* + - route53:CreateHostedZone + - route53:CreateVPCAssociationAuthorization + - route53:GetHostedZone + Resource: '*' diff --git a/source/idea/idea-administrator/resources/installer_policies/idea-admin-delete-policy.yml b/source/idea/idea-administrator/resources/installer_policies/idea-admin-delete-policy.yml new file mode 100644 index 00000000..3b3ca235 --- /dev/null +++ b/source/idea/idea-administrator/resources/installer_policies/idea-admin-delete-policy.yml @@ -0,0 +1,68 @@ +Version: '2012-10-17' +Statement: + - Sid: DeletePermissionsIfIDEAInstallFails + Effect: Allow + Action: + - sns:DeleteTopic + - sns:Unsubscribe + - backup:DeleteBackupPlan + - backup:DeleteBackupSelection + - backup:DeleteBackupVault + - cloudformation:DeleteChangeSet + - cloudformation:DeleteStack + - cloudwatch:DeleteAlarms + - cloudwatch:DescribeAlarms + - ds:DeleteDirectory + - ec2:DeleteFlowLogs + - ec2:DeleteInternetGateway + - ec2:DeleteNatGateway + - ec2:DeleteNetworkInterface + - ec2:DeleteRoute + - ec2:DeleteRouteTable + - ec2:DeleteSecurityGroup + - ec2:DeleteSubnet + - ec2:DeleteVpc + - ec2:DeleteVpcEndpoints + - ec2:DescribeVpcEndpoints + - ec2:DescribeManagedPrefixLists + - ec2:DescribeInstanceAttribute + - ec2:DetachInternetGateway + - ec2:DisassociateAddress + - ec2:DisassociateRouteTable + - ec2:ReleaseAddress + - ec2:RevokeSecurityGroupIngress + - ec2:TerminateInstances + - ec2:ModifyInstanceAttribute + - ecr:DeleteRepository + - elasticfilesystem:DeleteFileSystem + - elasticfilesystem:DeleteMountTarget + - elasticloadbalancing:DeRegisterTargets + - elasticloadbalancing:DeleteListener + - elasticloadbalancing:DeleteLoadBalancer + - elasticloadbalancing:DeleteRule + - es:DeleteElasticsearchDomain + - fsx:DeleteFileSystem + - iam:DeleteInstanceProfile + - iam:DeleteRole + - iam:DeleteRolePolicy + - iam:DeleteServiceLinkedRole + - iam:DetachRolePolicy + - iam:RemoveRoleFromInstanceProfile + - lambda:DeleteFunction + - lambda:RemovePermission + - logs:DescribeLogGroups + - logs:DeleteLogGroup + - route53resolver:DeleteResolverEndpoint + - route53resolver:DeleteResolverRule + - route53resolver:DisassociateResolverRule + - secretsmanager:DeleteSecret + - s3:DeleteBucketPolicy + - s3:DeleteBucket + - s3:ListAllMyBuckets + - s3:ListBucketVersions + - route53:DeleteHostedZone + - acm:DescribeCertificate + - acm:ListCertificates + - acm:RequestCertificate + - tag:GetResources + Resource: '*' diff --git a/source/idea/idea-administrator/resources/integration_tests/job_test_cases.yml b/source/idea/idea-administrator/resources/integration_tests/job_test_cases.yml new file mode 100644 index 00000000..0ad1dd5c --- /dev/null +++ b/source/idea/idea-administrator/resources/integration_tests/job_test_cases.yml @@ -0,0 +1,476 @@ + +configuration: + instance_type_command: &instance_type_command >- + TOKEN=$(curl --silent -X PUT 'http://169.254.169.254/latest/api/token' -H 'X-aws-ec2-metadata-token-ttl-seconds: 300') && + curl --silent -H "X-aws-ec2-metadata-token: ${TOKEN}" 'http://169.254.169.254/latest/meta-data/instance-type' + +test_cases: + + - name: instance_type_t2 + resource: instance_type=t2.large + command: *instance_type_command + expected_output: t2.large + queue: normal + skip_regions: ['af-south-1', 'eu-north-1', 'eu-south-1', 'me-south-1', 'me-central-1', 'ap-east-1'] + + - name: instance_type_t3 + resource: instance_type=t3.large + command: *instance_type_command + expected_output: t3.large + queue: normal + skip_regions: [] + + - name: instance_type_t3a + resource: instance_type=t3a.large + command: *instance_type_command + expected_output: t3a.large + queue: normal + skip_regions: ['af-south-1', 'eu-north-1', 'ap-northeast-3', 'me-south-1', 'me-central-1', 'ap-east-1'] + + - name: instance_type_t4g + resource: instance_type=t4g.large + command: *instance_type_command + expected_output: t4g.large + queue: normal + skip_regions: ['all'] # todo: uncomment when we have graviton support on region_ami_config + #skip_regions: ['af-south-1', 'ap-northeast-3', 'me-south-1'] + + - name: instance_type_m5 + resource: instance_type=m5.large + command: *instance_type_command + expected_output: m5.large + queue: normal + skip_regions: [] + + - name: instance_type_m5a + resource: instance_type=m5a.large + command: *instance_type_command + expected_output: m5a.large + queue: normal + skip_regions: ['af-south-1', 'eu-north-1', 'ap-northeast-3', 'me-south-1', 'me-central-1', 'ap-east-1'] + + - name: instance_type_m5n + resource: instance_type=m5n.large + command: *instance_type_command + expected_output: m5n.large + queue: normal + skip_regions: ['af-south-1', 'eu-north-1', 'ap-south-1', 'eu-west-3', 'eu-west-2', 'eu-south-1', 'ap-northeast-3', 'ap-northeast-2', 'me-south-1', 'me-central-1', 'sa-east-1', 'ca-central-1', 'ap-east-1', 'ap-southeast-2', 'us-west-1'] + + - name: instance_type_m5zn + resource: instance_type=m5zn.large + command: *instance_type_command + expected_output: m5zn.large + queue: normal + skip_regions: ['us-gov-east-1', 'us-gov-west-1', 'af-south-1', 'eu-north-1', 'ap-south-1', 'eu-west-3', 'eu-west-2', 'eu-south-1', 'ap-northeast-3', 'me-south-1', 'me-central-1', 'ca-central-1', 'ap-east-1'] + + - name: instance_type_m6a + resource: instance_type=m6a.large + command: *instance_type_command + expected_output: m6a.large + queue: normal + skip_regions: ['us-gov-east-1', 'us-gov-west-1', 'af-south-1', 'eu-north-1', 'eu-west-3', 'eu-west-2', 'eu-south-1', 'ap-northeast-3', 'ap-northeast-2', 'me-south-1', 'ap-northeast-1', 'me-central-1', 'sa-east-1', 'ca-central-1', 'ap-east-1', 'ap-southeast-1', 'ap-southeast-2', 'us-west-1'] + + - name: instance_type_m6i + resource: instance_type=m6i.large + command: *instance_type_command + expected_output: m6i.large + queue: normal + skip_regions: ['af-south-1', 'ap-northeast-3', 'me-central-1'] + + - name: instance_type_m6g + resource: instance_type=m6g.large + command: *instance_type_command + expected_output: m6g.large + queue: normal + skip_regions: ['all'] # todo: uncomment when we have graviton support on region_ami_config + #skip_regions: ['af-south-1', 'ap-northeast-3'] + + - name: instance_type_c5 + resource: instance_type=c5.2xlarge + command: *instance_type_command + expected_output: c5.2xlarge + queue: normal + skip_regions: [] + + - name: instance_type_c5a + resource: instance_type=c5a.large + command: *instance_type_command + expected_output: c5a.large + queue: normal + skip_regions: ['ap-northeast-3', 'me-central-1'] + + - name: instance_type_c5n + resource: instance_type=c5n.large + command: *instance_type_command + expected_output: c5n.large + queue: normal + skip_regions: ['ap-northeast-3', 'me-central-1'] + + - name: instance_type_c6a + resource: instance_type=c6a.large + command: *instance_type_command + expected_output: c6a.large + queue: normal + skip_regions: ['us-gov-east-1', 'us-gov-west-1', 'af-south-1', 'eu-north-1', 'eu-west-3', 'eu-west-2', 'eu-south-1', 'ap-northeast-3', 'ap-northeast-2', 'me-south-1', 'ap-northeast-1', 'me-central-1', 'sa-east-1', 'ca-central-1', 'ap-east-1', 'ap-southeast-1', 'ap-southeast-2', 'us-west-1'] + + - name: instance_type_c6i + resource: instance_type=c6i.large + command: *instance_type_command + expected_output: c6i.large + queue: normal + skip_regions: ['af-south-1', 'ap-northeast-3', 'me-central-1'] + + - name: instance_type_c6gn + resource: instance_type=c6gn.medium + command: *instance_type_command + expected_output: c6gn.medium + queue: normal + skip_regions: ['all'] # todo: uncomment when we have graviton support on region_ami_config + #skip_regions: ['af-south-1', 'me-central-1'] + + - name: instance_type_c6g + resource: instance_type=c6g.medium + command: *instance_type_command + expected_output: c6g.medium + queue: normal + skip_regions: ['all'] # todo: uncomment when we have graviton support on region_ami_config + #skip_regions: ['af-south-1'] + + - name: instance_type_c7g + resource: instance_type=c7g.medium + command: *instance_type_command + expected_output: c7g.medium + queue: normal + skip_regions: ['all'] # todo: uncomment when we have graviton support on region_ami_config + #skip_regions: ['af-south-1', 'eu-north-1', 'ap-south-1', 'eu-west-3', 'eu-west-2', 'eu-south-1', 'ap-northeast-3', 'ap-northeast-2', 'me-south-1', 'ap-northeast-1', 'me-central-1', 'sa-east-1', 'ca-central-1', 'ap-east-1', 'ap-southeast-1', 'ap-southeast-2', 'eu-central-1', 'us-west-1'] + + - name: instance_type_hpc6a + resource: instance_type=hpc6a.48xlarge + command: *instance_type_command + expected_output: hpc6a.48xlarge + queue: normal + skip_regions: ['all'] + #skip_regions: ['af-south-1', 'ap-south-1', 'eu-west-3', 'eu-west-2', 'eu-south-1', 'eu-west-1', 'ap-northeast-3', 'ap-northeast-2', 'me-south-1', 'ap-northeast-1', 'me-central-1', 'sa-east-1', 'ca-central-1', 'ap-east-1', 'ap-southeast-1', 'ap-southeast-2', 'eu-central-1', 'us-east-1', 'us-west-1', 'us-west-2'] + + - name: instance_type_r4 + resource: instance_type=r4.large + command: *instance_type_command + expected_output: r4.large + queue: normal + skip_regions: ['af-south-1', 'eu-north-1', 'eu-south-1', 'me-south-1', 'me-central-1', 'ap-east-1'] + + - name: instance_type_r5 + resource: instance_type=r5.large + command: *instance_type_command + expected_output: r5.large + queue: normal + skip_regions: [] + + - name: instance_type_r5a + resource: instance_type=r5a.large + command: *instance_type_command + expected_output: r5a.large + queue: normal + skip_regions: ['af-south-1', 'eu-north-1', 'ap-northeast-3', 'me-south-1', 'me-central-1', 'ap-east-1'] + + - name: instance_type_r5b + resource: instance_type=r5b.large + command: *instance_type_command + expected_output: r5b.large + queue: normal + skip_regions: ['us-gov-east-1', 'us-gov-west-1', 'af-south-1', 'eu-north-1', 'ap-south-1', 'eu-west-3', 'eu-south-1', 'ap-northeast-3', 'me-south-1', 'me-central-1', 'ap-east-1', 'us-west-1'] + + - name: instance_type_r5n + resource: instance_type=r5n.large + command: *instance_type_command + expected_output: r5n.large + queue: normal + skip_regions: ['ap-northeast-3', 'me-south-1', 'me-central-1'] + + - name: instance_type_r6a + resource: instance_type=r6a.large + command: *instance_type_command + expected_output: r6a.large + queue: normal + skip_regions: ['us-gov-east-1', 'us-gov-west-1', 'af-south-1', 'eu-north-1', 'eu-west-3', 'eu-west-2', 'eu-south-1', 'ap-northeast-3', 'ap-northeast-2', 'me-south-1', 'ap-northeast-1', 'me-central-1', 'sa-east-1', 'ca-central-1', 'ap-east-1', 'ap-southeast-1', 'ap-southeast-2', 'us-west-1'] + + - name: instance_type_r6g + resource: instance_type=r6g.large + command: *instance_type_command + expected_output: r6g.large + queue: normal + skip_regions: ['all'] # todo: uncomment when we have graviton support on region_ami_config + #skip_regions: ['af-south-1', 'ap-northeast-3', 'me-south-1'] + + - name: instance_type_r6i + resource: instance_type=r6i.large + command: *instance_type_command + expected_output: r6i.large + queue: normal + skip_regions: ['af-south-1', 'ap-northeast-3', 'me-south-1', 'me-central-1'] + + - name: instance_type_x2gd + resource: instance_type=x2gd.large + command: *instance_type_command + expected_output: x2gd.large + queue: normal + skip_regions: ['all'] # todo: uncomment when we have graviton support on region_ami_config + #skip_regions: ['af-south-1', 'eu-north-1', 'ap-south-1', 'eu-west-3', 'eu-west-2', 'eu-south-1', 'ap-northeast-3', 'ap-northeast-2', 'me-south-1', 'ap-northeast-1', 'me-central-1', 'sa-east-1', 'ca-central-1', 'ap-east-1', 'ap-southeast-1', 'ap-southeast-2', 'eu-central-1', 'us-west-1'] + + - name: instance_type_x2iedn + resource: instance_type=x2iedn.xlarge + command: *instance_type_command + expected_output: x2iedn.xlarge + queue: normal + skip_regions: ['all'] + + - name: instance_type_x2iezn + resource: instance_type=x2iezn.2xlarge + command: *instance_type_command + expected_output: x2iezn.2xlarge + queue: normal + skip_regions: ['all'] + + - name: instance_type_p3 + resource: instance_type=p3.2xlarge + command: *instance_type_command + expected_output: p3.2xlarge + queue: normal + skip_regions: ['all'] + + - name: instance_type_p4d + resource: instance_type=p4d.24xlarge + command: *instance_type_command + expected_output: p4d.24xlarge + queue: normal + skip_regions: ['all'] + + - name: instance_type_trn1 + resource: instance_type=trn1.2xl + command: *instance_type_command + expected_output: trn1.2xl + queue: normal + skip_regions: ['all'] + + - name: instance_type_inf1 + resource: instance_type=inf1.xlarge + command: *instance_type_command + expected_output: inf1.xlarge + queue: normal + skip_regions: ['all'] + + - name: instance_type_g5 + resource: instance_type=g5.xlarge + command: *instance_type_command + expected_output: g5.xlarge + queue: normal + skip_regions: ['all'] + + - name: instance_type_g5g + resource: instance_type=g5g.xlarge + command: *instance_type_command + expected_output: g5g.xlarge + queue: normal + skip_regions: ['all'] + + - name: instance_type_g4dn + resource: instance_type=g4dn.xlarge + command: *instance_type_command + expected_output: g4dn.xlarge + queue: normal + skip_regions: ['all'] + + - name: instance_type_g4ad + resource: instance_type=g4ad.xlarge + command: *instance_type_command + expected_output: g4ad.xlarge + queue: normal + skip_regions: ['all'] + + - name: instance_type_f1 + resource: instance_type=f1.2xlarge + command: *instance_type_command + expected_output: f1.2xlarge + queue: normal + skip_regions: ['all'] + + # Other tests + - name: hello_world + resource: instance_type=t3.large + command: echo HelloWorld + expected_output: HelloWorld + queue: normal + skip_regions: [] + + - name: scratch_size + resource: instance_type=t2.medium,scratch_size=36 + command: /bin/df -h --output=size /scratch | tail -n1 | tr -d ' ' + expected_output: 36G + queue: normal + skip_regions: [] + + - name: root_size + resource: instance_type=t2.medium,root_size=44 + command: /bin/df -h --output=size / | tail -n1 | tr -d ' ' + expected_output: 44G + queue: normal + skip_regions: [] + + - name: efa + resource: instance_type=c5n.9xlarge,efa_support=True + command: /opt/amazon/efa/bin/fi_info -p efa -t FI_EP_RDM | head -n1 + expected_output: "provider: efa" + queue: normal + skip_regions: [] + + - name: instance_store_single_volume + resource: instance_type=m5ad.large + command: /bin/df -h --output=size /scratch | tail -n1 | tr -d ' ' + expected_output: 69G + queue: normal + skip_regions: [] + + - name: instance_store_multiple_volumes + resource: instance_type=m5ad.4xlarge + command: /bin/df -h --output=size /scratch | tail -n1 | tr -d ' ' + expected_output: 550G + queue: normal + skip_regions: [] + + - name: ht_enabled + resource: instance_type=m5.xlarge,ht_support=True + command: /bin/lscpu --extended | sed 1d | wc -l + expected_output: 4 + queue: normal + skip_regions: [] + + - name: ht_disabled + resource: instance_type=m5.xlarge,ht_support=False + command: /bin/lscpu --extended | sed 1d | wc -l + expected_output: 2 + queue: normal + skip_regions: [] + + - name: fsx_2tb + resource: fsx_lustre=True,fsx_lustre_size=2400 + command: /bin/lfs df -h /fsx | grep filesystem_summary | awk '{print $2}' + expected_output: 2.2T + queue: normal + skip_regions: [] + + - name: check_spot + resource: spot_price=auto + command: >- + TOKEN=$(curl --silent -X PUT 'http://169.254.169.254/latest/api/token' -H 'X-aws-ec2-metadata-token-ttl-seconds: 300') && + curl --silent -H "X-aws-ec2-metadata-token: ${TOKEN}" 'http://169.254.169.254/latest/meta-data/instance-life-cycle' + expected_output: spot + queue: normal + skip_regions: ['us-gov-east-1', 'us-gov-west-1'] + + - name: spotfleet + resource: instance_type=t3.large+t3.xlarge,spot_price=auto + command: >- + TOKEN=$(curl --silent -X PUT 'http://169.254.169.254/latest/api/token' -H 'X-aws-ec2-metadata-token-ttl-seconds: 300') && + curl --silent -H "X-aws-ec2-metadata-token: ${TOKEN}" 'http://169.254.169.254/latest/meta-data/instance-life-cycle' ; + echo -n "-"; pbsnodes -v `hostname -s` | grep spot_fleet_request | awk '{print $3}' | cut -f1 -d- ; + expected_output: spot-sfr + queue: normal + skip_regions: ['us-gov-east-1', 'us-gov-west-1'] + + - name: placement_group_enabled + resource: nodes=2,placement_group=True,instance_type=m5.large + command: >- + TOKEN=$(curl --silent -X PUT 'http://169.254.169.254/latest/api/token' -H 'X-aws-ec2-metadata-token-ttl-seconds: 300') && + curl --silent -o /dev/null -w '%{http_code}' -H "X-aws-ec2-metadata-token: ${TOKEN}" 'http://169.254.169.254/latest/meta-data/placement/group-name' + expected_output: 200 + queue: normal + skip_regions: [] + + - name: placement_group_disabled + resource: nodes=2,placement_group=False,instance_type=m5.large + command: >- + TOKEN=$(curl --silent -X PUT 'http://169.254.169.254/latest/api/token' -H 'X-aws-ec2-metadata-token-ttl-seconds: 300') && + curl --silent -o /dev/null -w '%{http_code}' -H "X-aws-ec2-metadata-token: ${TOKEN}" 'http://169.254.169.254/latest/meta-data/placement/group-name' + expected_output: 404 + queue: normal + skip_regions: [] + + # Tests for job-shared queue need to be named as job-shared_1_* and job-shared_2_* + # This is needed so that the output for 2nd job can be compared versus the output of the 1st job + - name: job_shared_ondemand_1 + resource: instance_type=t3.xlarge+t3.2xlarge,ht_support=True,placement_group=False + command: hostname -s + expected_output: ip-[0-9]*-[0-9]*-[0-9]*-[0-9]* + regex_match: true + queue: job-shared + skip_regions: [] + + - name: job_shared_ondemand_2 + resource: instance_type=t3.xlarge+t3.2xlarge,ht_support=True,placement_group=False + command: hostname -s + expected_output: ip-[0-9]*-[0-9]*-[0-9]*-[0-9]* + regex_match: true + queue: job-shared + skip_regions: [] + + - name: job_shared_spot_1 + resource: instance_type=t3.xlarge+t3.2xlarge,ht_support=True,placement_group=False,spot_price=auto + command: hostname -s + expected_output: ip-[0-9]*-[0-9]*-[0-9]*-[0-9]* + regex_match: true + queue: job-shared + skip_regions: ['us-gov-east-1', 'us-gov-west-1'] + + - name: job_shared_spot_2 + resource: instance_type=t3.xlarge+t3.2xlarge,ht_support=True,placement_group=False,spot_price=auto + command: hostname -s + expected_output: ip-[0-9]*-[0-9]*-[0-9]*-[0-9]* + regex_match: true + queue: job-shared + skip_regions: ['us-gov-east-1', 'us-gov-west-1'] + + # Following jobs are invalid. We are just confirming scheduler hooks works as expected + - name: invalid_instance_type + resource: instance_type=t4.donotexist + command: /bin/echo Test + expected_output: "ec2 instance_type is invalid" + queue: normal + error_code: JOB_SUBMISSION_FAILED + skip_regions: [] + + - name: service_quota_not_available + resource: instance_type=c5.24xlarge,nodes=5000 + command: /bin/echo Test + expected_output: "Following AWS Service Quota needs to be requested from AWS" + queue: normal + error_code: JOB_SUBMISSION_FAILED + skip_regions: [] + + - name: hook_security_groups + resource: security_groups=sg-fakeone + command: /bin/echo Test + expected_output: "Security groups not found or invalid" + queue: normal + error_code: JOB_SUBMISSION_FAILED + skip_regions: [] + + - name: hook_iam_role + resource: instance_profile=FakeProfile + command: /bin/echo Test + expected_output: "Instance profile not found" + queue: normal + error_code: JOB_SUBMISSION_FAILED + skip_regions: [] + + # Tests below are skipped by default as they require user inputs + # Update and change skip flag to false to run them + + - name: custom_subnet + # Please enter a test subnet id + resource: subnet_id=ENTER_USER_CUSTOM_VALUE + command: >- + TOKEN=$(curl --silent -X PUT 'http://169.254.169.254/latest/api/token' -H 'X-aws-ec2-metadata-token-ttl-seconds: 300') && + curl --silent -o /dev/null -w '%{http_code}' -H "X-aws-ec2-metadata-token: ${TOKEN}" 'http:/169.254.169.254/latest/meta-data/network/interfaces/macs/' + expected_output: ENTER_USER_CUSTOM_VALUE + queue: normal + skip_regions: ["all"] diff --git a/source/idea/idea-administrator/resources/integration_tests/session_test_cases.yml b/source/idea/idea-administrator/resources/integration_tests/session_test_cases.yml new file mode 100644 index 00000000..6eaf6312 --- /dev/null +++ b/source/idea/idea-administrator/resources/integration_tests/session_test_cases.yml @@ -0,0 +1,98 @@ +testcases: + - name: windows-x86_64 + base_os: windows + software_stack_id: ss-base-windows-x86-64-base + hibernation_enabled: false + instance_type: t3.xlarge + storage_size: 500 + + - name: windows-gpu-nvidia + base_os: windows + software_stack_id: ss-base-windows-x86-64-base-nvidia + hibernation_enabled: false + instance_type: g4dn.xlarge + storage_size: 500 + + - name: windows-gpu-amd + base_os: windows + software_stack_id: ss-base-windows-x86-64-base-amd + hibernation_enabled: false + instance_type: g4ad.xlarge + storage_size: 500 + + - name: rhel7-x86_64 + base_os: rhel7 + software_stack_id: ss-base-rhel7-x86-64-base + hibernation_enabled: false + instance_type: t3.xlarge + storage_size: 500 + + - name: rhel7-gpu-nvidia + base_os: rhel7 + software_stack_id: ss-base-rhel7-x86-64-base + hibernation_enabled: false + instance_type: g4dn.xlarge + storage_size: 500 + + - name: rhel7-gpu-amd + base_os: rhel7 + software_stack_id: ss-base-rhel7-x86-64-base + hibernation_enabled: false + instance_type: g4ad.xlarge + storage_size: 500 + + - name: centos7-x86_64 + base_os: centos7 + software_stack_id: ss-base-centos7-x86-64-base + hibernation_enabled: False + instance_type: t3.xlarge + storage_size: 500 + + - name: centos7-arm + base_os: centos7 + software_stack_id: ss-base-centos7-arm64-base + hibernation_enabled: False + instance_type: m6g.2xlarge + storage_size: 500 + + - name: centos7-gpu-nvidia + base_os: centos7 + software_stack_id: ss-base-centos7-x86-64-base + hibernation_enabled: False + instance_type: g4dn.xlarge + storage_size: 500 + + - name: centos7-gpu-amd + base_os: centos7 + software_stack_id: ss-base-centos7-x86-64-base + hibernation_enabled: False + instance_type: g4ad.xlarge + storage_size: 500 + + - name: amazon-linux2-x86_64 + base_os: amazonlinux2 + software_stack_id: ss-base-amazonlinux2-x86-64-base + hibernation_enabled: False + instance_type: t3.xlarge + storage_size: 500 + + - name: amazon-linux2-arm + base_os: amazonlinux2 + software_stack_id: ss-base-amazonlinux2-arm64-base + hibernation_enabled: False + instance_type: m6g.2xlarge + storage_size: 500 + + - name: amazon-linux2-gpu-nvidia + base_os: amazonlinux2 + software_stack_id: ss-base-amazonlinux2-x86-64-base + hibernation_enabled: False + instance_type: g4dn.xlarge + storage_size: 500 + + - name: amazon-linux2-gpu-amd + base_os: amazonlinux2 + software_stack_id: ss-base-amazonlinux2-x86-64-base + hibernation_enabled: False + instance_type: g4ad.xlarge + storage_size: 500 diff --git a/source/idea/idea-administrator/resources/lambda_functions/__init__.py b/source/idea/idea-administrator/resources/lambda_functions/__init__.py new file mode 100644 index 00000000..368a432f --- /dev/null +++ b/source/idea/idea-administrator/resources/lambda_functions/__init__.py @@ -0,0 +1,14 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +# lambda functions to be deployed. should not contain any SDK dependencies. + + diff --git a/source/idea/idea-administrator/resources/lambda_functions/idea_analytics_sink/__init__.py b/source/idea/idea-administrator/resources/lambda_functions/idea_analytics_sink/__init__.py new file mode 100644 index 00000000..6d8d18a3 --- /dev/null +++ b/source/idea/idea-administrator/resources/lambda_functions/idea_analytics_sink/__init__.py @@ -0,0 +1,10 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. diff --git a/source/idea/idea-administrator/resources/lambda_functions/idea_analytics_sink/handler.py b/source/idea/idea-administrator/resources/lambda_functions/idea_analytics_sink/handler.py new file mode 100644 index 00000000..085e31ff --- /dev/null +++ b/source/idea/idea-administrator/resources/lambda_functions/idea_analytics_sink/handler.py @@ -0,0 +1,75 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. +""" +Analytics Sink Lambda: +This function is triggered by the analytics kinesis stream when any module submits an analytics event +""" +import base64 +import os +import json +import logging +from opensearchpy import OpenSearch, helpers + +opensearch_endpoint = os.environ.get('opensearch_endpoint') +if not opensearch_endpoint.startswith('https://'): + opensearch_endpoint = f'https://{opensearch_endpoint}' + +os_client = OpenSearch( + hosts=[opensearch_endpoint], + port=443, + use_ssl=True, + verify_certs=True +) + +logger = logging.getLogger() +logger.setLevel(logging.INFO) + + +def handler(event, _): + try: + bulk_request_with_timestamp = [] + for record in event['Records']: + analytics_entry = base64.b64decode(record["kinesis"]["data"]) + analytics_entry = json.loads(analytics_entry.decode()) + request = { + "_id": analytics_entry["document_id"], + "_index": analytics_entry["index_id"] + } + + if analytics_entry["action"] == 'CREATE_ENTRY': + # create new + request["_op_type"] = "create" + request["_source"] = analytics_entry["entry"] + + elif analytics_entry["action"] == 'UPDATE_ENTRY': + # update existing + request["_op_type"] = "update" + request["doc"] = analytics_entry["entry"] + else: + # delete + request["_op_type"] = "delete" + + bulk_request_with_timestamp.append((analytics_entry["timestamp"], request)) + + bulk_request_with_timestamp_sorted = sorted(bulk_request_with_timestamp, key=lambda x: x[0]) + bulk_request = [] + for entry in bulk_request_with_timestamp_sorted: + request = entry[1] + logger.info(f'Submitting request for action: {request["_op_type"]} for document_id: {request["_id"]} to index {request["_index"]}') + bulk_request.append(request) + + response = helpers.bulk( + client=os_client, + actions=bulk_request + ) + logger.info(response) + except Exception as e: + logger.exception(f'Error while processing analytics request for event: {json.dumps(event)}, error: {e}') diff --git a/source/idea/idea-administrator/resources/lambda_functions/idea_analytics_sink/requirements.txt b/source/idea/idea-administrator/resources/lambda_functions/idea_analytics_sink/requirements.txt new file mode 100644 index 00000000..63c4ec34 --- /dev/null +++ b/source/idea/idea-administrator/resources/lambda_functions/idea_analytics_sink/requirements.txt @@ -0,0 +1 @@ +opensearch-py diff --git a/source/idea/idea-administrator/resources/lambda_functions/idea_controller_scheduled_event_transformer/__init__.py b/source/idea/idea-administrator/resources/lambda_functions/idea_controller_scheduled_event_transformer/__init__.py new file mode 100644 index 00000000..6d8d18a3 --- /dev/null +++ b/source/idea/idea-administrator/resources/lambda_functions/idea_controller_scheduled_event_transformer/__init__.py @@ -0,0 +1,10 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. diff --git a/source/idea/idea-administrator/resources/lambda_functions/idea_controller_scheduled_event_transformer/handler.py b/source/idea/idea-administrator/resources/lambda_functions/idea_controller_scheduled_event_transformer/handler.py new file mode 100644 index 00000000..1f6f9e8d --- /dev/null +++ b/source/idea/idea-administrator/resources/lambda_functions/idea_controller_scheduled_event_transformer/handler.py @@ -0,0 +1,50 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. +""" +Scheduled event transformer +This function is triggered by an event-bridge event rule periodically. It repackages the event and forwards it to the +Controller Events Queue. +""" +import json +import boto3 +import os +import logging + +logger = logging.getLogger() +logger.setLevel(logging.INFO) + +sqs_client = boto3.client('sqs') +ec2_resource = boto3.resource('ec2') + + +def handler(event, _): + try: + detail_type = event['detail-type'] + if detail_type != 'Scheduled Event': + return + + forwarding_event = { + 'event_group_id': 'SCHEDULED_EVENT', + 'event_type': 'SCHEDULED_EVENT', + 'detail': { + 'time': event['time'] + } + } + + logger.info('Forwarding scheduled event to Controller') + response = sqs_client.send_message( + QueueUrl=os.environ.get('IDEA_CONTROLLER_EVENTS_QUEUE_URL'), + MessageBody=json.dumps(forwarding_event), + MessageGroupId='SCHEDULED_EVENT' + ) + logger.info(response) + except Exception as e: + logger.exception(f'error in handling scheduled event: {event}, error: {e}') diff --git a/source/idea/idea-administrator/resources/lambda_functions/idea_custom_resource_cluster_endpoints/__init__.py b/source/idea/idea-administrator/resources/lambda_functions/idea_custom_resource_cluster_endpoints/__init__.py new file mode 100644 index 00000000..6d8d18a3 --- /dev/null +++ b/source/idea/idea-administrator/resources/lambda_functions/idea_custom_resource_cluster_endpoints/__init__.py @@ -0,0 +1,10 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. diff --git a/source/idea/idea-administrator/resources/lambda_functions/idea_custom_resource_cluster_endpoints/handler.py b/source/idea/idea-administrator/resources/lambda_functions/idea_custom_resource_cluster_endpoints/handler.py new file mode 100644 index 00000000..8eda1a1c --- /dev/null +++ b/source/idea/idea-administrator/resources/lambda_functions/idea_custom_resource_cluster_endpoints/handler.py @@ -0,0 +1,205 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +""" +Cluster Endpoints + +This function is used by individual module stacks to expose module API or web endpoints via External and Internal ALBs. +""" +import time + +import botocore.exceptions + +from idea_lambda_commons import HttpClient, CfnResponse, CfnResponseStatus +import boto3 +import logging +import json + +logger = logging.getLogger() +logger.setLevel(logging.INFO) + + +def find_rule_arn(elbv2_client, listener_arn: str, endpoint_name: str): + describe_rules_result = elbv2_client.describe_rules( + ListenerArn=listener_arn, + PageSize=100 # ALB no. of rules limit + ) + + rules = describe_rules_result.get('Rules', []) + rule_arns = [] + for rule in rules: + rule_arn = rule.get('RuleArn') + rule_arns.append(rule_arn) + + # a batch based implementation can caused race conditions during describe_tags operation, as the rule was deleted + # from another invocation during delete vdc stack operation. switching back to one by one query of tags + # added sleep to ensure requests are not being throttled + for rule_arn in rule_arns: + try: + describe_tag_results = elbv2_client.describe_tags( + ResourceArns=[rule_arn] + ) + tag_descriptions = describe_tag_results.get('TagDescriptions', []) + for tag_description in tag_descriptions: + rule_arn = tag_description.get('ResourceArn') + tags = tag_description.get('Tags', []) + for tag in tags: + if tag.get('Key') == 'idea:EndpointName': + if endpoint_name == tag.get('Value'): + return rule_arn + time.sleep(1) + except botocore.exceptions.ClientError as e: + logger.warning(f'failed to fetch tags for rule arn: {rule_arn} - {e}') + time.sleep(2) + continue + + return None + + +def handler(event: dict, context): + logger.info(f'ReceivedEvent: {json.dumps(event)}') + request_type = event.get('RequestType', None) + resource_properties = event.get('ResourceProperties', {}) + + # a unique name identifying the endpoint + endpoint_name = resource_properties.get('endpoint_name', '__NOT_PROVIDED__') + + client = HttpClient() + + try: + + if endpoint_name is None or endpoint_name == '__NOT_PROVIDED__': + raise ValueError('endpoint_name is required and cannot be empty') + + # listener arn + listener_arn = resource_properties.get('listener_arn') + if listener_arn is None: + raise ValueError('listener_arn is required and cannot be empty') + + # the module that hosts the web ui will send default_action = True. + # in the default setup, this will be cluster manager + default_action = resource_properties.get('default_action', False) + + # a json array structure as per: + # https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/elbv2.html#ElasticLoadBalancingv2.Client.create_rule + conditions = resource_properties.get('conditions', []) + if not default_action: + if conditions is None or len(conditions) == 0: + raise ValueError('conditions[] is required and cannot be empty') + + # a json array structure as per: + # https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/elbv2.html#ElasticLoadBalancingv2.Client.create_rule + actions = resource_properties.get('actions', []) + if not default_action: + if actions is None or len(actions) == 0: + raise ValueError('actions[] is required and cannot be empty') + + priority_value = resource_properties.get('priority') + priority = -1 + if not default_action: + priority = int(priority_value) + if priority <= 0: + raise ValueError('priority must be greater than 0') + + # any applicable tags need to added to the listener rules + tags = resource_properties.get('tags', {}) + tags['idea:EndpointName'] = endpoint_name + + resource_tags = [] + for key, value in tags.items(): + resource_tags.append({ + 'Key': key, + 'Value': value + }) + + elbv2_client = boto3.client('elbv2') + + if default_action: + if request_type in ('Create', 'Update'): + elbv2_client.modify_listener( + ListenerArn=listener_arn, + DefaultActions=actions + ) + logger.info(f'default action modified') + else: + elbv2_client.modify_listener( + ListenerArn=listener_arn, + DefaultActions=[ + { + 'Type': 'fixed-response', + 'FixedResponseConfig': { + 'MessageBody': json.dumps({ + 'success': True, + 'message': 'OK' + }), + 'StatusCode': '200', + 'ContentType': 'application/json' + } + } + ] + ) + logger.info(f'default action reset to fixed response') + + elif request_type == 'Create': + result = elbv2_client.create_rule( + ListenerArn=listener_arn, + Conditions=conditions, + Priority=priority, + Actions=actions, + Tags=resource_tags + ) + rules = result.get('Rules', []) + rule_arn = rules[0].get('RuleArn') + logger.info(f'rule created. rule arn: {rule_arn}') + + elif request_type == 'Update': + rule_arn = find_rule_arn(elbv2_client, listener_arn=listener_arn, endpoint_name=endpoint_name) + if rule_arn is not None: + elbv2_client.modify_rule( + RuleArn=rule_arn, + Conditions=conditions, + Actions=actions + ) + logger.info(f'rule modified. rule arn: {rule_arn}') + else: + logger.warning(f'rule not found for target group. rule update skipped.') + + elif request_type == 'Delete': + rule_arn = find_rule_arn(elbv2_client, listener_arn=listener_arn, endpoint_name=endpoint_name) + if rule_arn is not None: + elbv2_client.delete_rule( + RuleArn=rule_arn + ) + logger.info(f'rule deleted. rule arn: {rule_arn}') + else: + logger.warning('rule could not be deleted. rule arn not found for target group') + + client.send_cfn_response(CfnResponse( + context=context, + event=event, + status=CfnResponseStatus.SUCCESS, + data={}, + physical_resource_id=endpoint_name + )) + + except Exception as e: + error_message = f'failed to {request_type} endpoint: {endpoint_name} - {e}' + logger.exception(error_message) + client.send_cfn_response(CfnResponse( + context=context, + event=event, + status=CfnResponseStatus.FAILED, + data={}, + physical_resource_id=endpoint_name, + reason=error_message + )) + finally: + client.destroy() diff --git a/source/idea/idea-administrator/resources/lambda_functions/idea_custom_resource_create_tags/__init__.py b/source/idea/idea-administrator/resources/lambda_functions/idea_custom_resource_create_tags/__init__.py new file mode 100644 index 00000000..6d8d18a3 --- /dev/null +++ b/source/idea/idea-administrator/resources/lambda_functions/idea_custom_resource_create_tags/__init__.py @@ -0,0 +1,10 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. diff --git a/source/idea/idea-administrator/resources/lambda_functions/idea_custom_resource_create_tags/handler.py b/source/idea/idea-administrator/resources/lambda_functions/idea_custom_resource_create_tags/handler.py new file mode 100644 index 00000000..bcf6d5e9 --- /dev/null +++ b/source/idea/idea-administrator/resources/lambda_functions/idea_custom_resource_create_tags/handler.py @@ -0,0 +1,53 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +from idea_lambda_commons import HttpClient, CfnResponse, CfnResponseStatus +import boto3 +import logging + +logging.getLogger().setLevel(logging.INFO) + + +def handler(event, context): + """ + Tag EC2 Resource + """ + http_client = HttpClient() + try: + logging.info(f'ReceivedEvent: {event}') + + resource_id = event['ResourceProperties']['ResourceId'] + tags = event['ResourceProperties']['Tags'] + + ec2_client = boto3.client('ec2') + ec2_client.create_tags( + Resources=[resource_id], + Tags=tags + ) + + http_client.send_cfn_response(CfnResponse( + context=context, + event=event, + status=CfnResponseStatus.SUCCESS, + data={}, + physical_resource_id=resource_id + )) + except Exception as e: + logging.exception(f'Failed to Tag EC2 Resource: {e}') + http_client.send_cfn_response(CfnResponse( + context=context, + event=event, + status=CfnResponseStatus.FAILED, + data={ + 'error': str(e) + }, + physical_resource_id=str(e) + )) diff --git a/source/idea/idea-administrator/resources/lambda_functions/idea_custom_resource_get_ad_security_group/__init__.py b/source/idea/idea-administrator/resources/lambda_functions/idea_custom_resource_get_ad_security_group/__init__.py new file mode 100644 index 00000000..6d8d18a3 --- /dev/null +++ b/source/idea/idea-administrator/resources/lambda_functions/idea_custom_resource_get_ad_security_group/__init__.py @@ -0,0 +1,10 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. diff --git a/source/idea/idea-administrator/resources/lambda_functions/idea_custom_resource_get_ad_security_group/handler.py b/source/idea/idea-administrator/resources/lambda_functions/idea_custom_resource_get_ad_security_group/handler.py new file mode 100644 index 00000000..8a468a4f --- /dev/null +++ b/source/idea/idea-administrator/resources/lambda_functions/idea_custom_resource_get_ad_security_group/handler.py @@ -0,0 +1,92 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +from idea_lambda_commons import HttpClient, CfnResponse, CfnResponseStatus +import boto3 +import logging +import json + +logger = logging.getLogger() +logger.setLevel(logging.INFO) + +PHYSICAL_RESOURCE_ID = 'ad-controller-security-group-id' + + +def handler(event, context): + + http_client = HttpClient() + try: + logger.info(f'ReceivedEvent: {json.dumps(event)}') + request_type = event.get('RequestType') + + if request_type == 'Delete': + http_client.send_cfn_response(CfnResponse( + context=context, + event=event, + status=CfnResponseStatus.SUCCESS, + data={}, + physical_resource_id=PHYSICAL_RESOURCE_ID + )) + return + + resource_properties = event.get('ResourceProperties', {}) + directory_id = resource_properties.get('DirectoryId') + logger.info('AD DirectoryId: ' + directory_id) + + ds_client = boto3.client('ds') + response = ds_client.describe_directories( + DirectoryIds=[directory_id] + ) + + directories = response.get('DirectoryDescriptions', []) + + security_group_id = None + if len(directories) > 0: + directory = directories[0] + vpc_settings = directory.get('VpcSettings', {}) + security_group_id = vpc_settings.get('SecurityGroupId', None) + + if security_group_id is None: + msg = f'Could not find SecurityGroupId for DirectoryId: {directory_id}' + logger.error(msg) + http_client.send_cfn_response(CfnResponse( + context=context, + event=event, + status=CfnResponseStatus.FAILED, + data={ + 'error': msg + }, + physical_resource_id=PHYSICAL_RESOURCE_ID + )) + else: + http_client.send_cfn_response(CfnResponse( + context=context, + event=event, + status=CfnResponseStatus.SUCCESS, + data={ + 'SecurityGroupId': security_group_id + }, + physical_resource_id=PHYSICAL_RESOURCE_ID + )) + except Exception as e: + error_message = f'Failed to get SecurityGroupId for Directory: {e}' + logger.exception(error_message) + http_client.send_cfn_response(CfnResponse( + context=context, + event=event, + status=CfnResponseStatus.FAILED, + data={ + 'error': error_message + }, + physical_resource_id=PHYSICAL_RESOURCE_ID + )) + finally: + http_client.destroy() diff --git a/source/idea/idea-administrator/resources/lambda_functions/idea_custom_resource_get_user_pool_client_secret/__init__.py b/source/idea/idea-administrator/resources/lambda_functions/idea_custom_resource_get_user_pool_client_secret/__init__.py new file mode 100644 index 00000000..6d8d18a3 --- /dev/null +++ b/source/idea/idea-administrator/resources/lambda_functions/idea_custom_resource_get_user_pool_client_secret/__init__.py @@ -0,0 +1,10 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. diff --git a/source/idea/idea-administrator/resources/lambda_functions/idea_custom_resource_get_user_pool_client_secret/handler.py b/source/idea/idea-administrator/resources/lambda_functions/idea_custom_resource_get_user_pool_client_secret/handler.py new file mode 100644 index 00000000..ce3ac0dd --- /dev/null +++ b/source/idea/idea-administrator/resources/lambda_functions/idea_custom_resource_get_user_pool_client_secret/handler.py @@ -0,0 +1,91 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +from idea_lambda_commons import HttpClient, CfnResponse, CfnResponseStatus +import boto3 +import logging +import json + +logger = logging.getLogger() +logger.setLevel(logging.INFO) + +PHYSICAL_RESOURCE_ID = 'user-pool-client-secret' + + +def handler(event, context): + http_client = HttpClient() + client_id = None + try: + logger.info(f'ReceivedEvent: {json.dumps(event)}') + request_type = event.get('RequestType') + + if request_type == 'Delete': + http_client.send_cfn_response(CfnResponse( + context=context, + event=event, + status=CfnResponseStatus.SUCCESS, + data={}, + physical_resource_id=PHYSICAL_RESOURCE_ID + )) + return + + resource_properties = event.get('ResourceProperties', {}) + user_pool_id = resource_properties.get('UserPoolId') + client_id = resource_properties.get('ClientId') + logger.info(f'UserPoolId: {user_pool_id}, ClientId: {client_id}') + + cognito_idp_client = boto3.client('cognito-idp') + response = cognito_idp_client.describe_user_pool_client( + UserPoolId=user_pool_id, + ClientId=client_id + ) + + client = response.get('UserPoolClient', {}) + client_secret = client.get('ClientSecret', None) + + if client_secret is None: + msg = f'Could not find ClientSecret for ClientId: {client_id}' + logger.error(msg) + http_client.send_cfn_response(CfnResponse( + context=context, + event=event, + status=CfnResponseStatus.FAILED, + data={ + 'error': msg + }, + physical_resource_id=f'{PHYSICAL_RESOURCE_ID}-{client_id}' + )) + else: + http_client.send_cfn_response(CfnResponse( + context=context, + event=event, + status=CfnResponseStatus.SUCCESS, + data={ + 'ClientSecret': client_secret + }, + physical_resource_id=f'{PHYSICAL_RESOURCE_ID}-{client_id}' + ), log_response=False) # disable response logging to prevent exposing client secret in logs + except Exception as e: + error_message = f'Failed to get ClientSecret for UserPool Client. - {e}' + logger.exception(error_message) + if client_id is None: + client_id = 'failed' + http_client.send_cfn_response(CfnResponse( + context=context, + event=event, + status=CfnResponseStatus.FAILED, + data={ + 'error': error_message + }, + physical_resource_id=f'{PHYSICAL_RESOURCE_ID}-{client_id}' + )) + finally: + http_client.destroy() diff --git a/source/idea/idea-administrator/resources/lambda_functions/idea_custom_resource_opensearch_private_ips/__init__.py b/source/idea/idea-administrator/resources/lambda_functions/idea_custom_resource_opensearch_private_ips/__init__.py new file mode 100644 index 00000000..6d8d18a3 --- /dev/null +++ b/source/idea/idea-administrator/resources/lambda_functions/idea_custom_resource_opensearch_private_ips/__init__.py @@ -0,0 +1,10 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. diff --git a/source/idea/idea-administrator/resources/lambda_functions/idea_custom_resource_opensearch_private_ips/handler.py b/source/idea/idea-administrator/resources/lambda_functions/idea_custom_resource_opensearch_private_ips/handler.py new file mode 100644 index 00000000..e3db36ac --- /dev/null +++ b/source/idea/idea-administrator/resources/lambda_functions/idea_custom_resource_opensearch_private_ips/handler.py @@ -0,0 +1,96 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +from idea_lambda_commons import HttpClient, CfnResponse, CfnResponseStatus +import boto3 +import logging +import json + +logger = logging.getLogger() +logger.setLevel(logging.INFO) + +PHYSICAL_RESOURCE_ID = 'opensearch-private-ip-addresses' + + +def handler(event, context): + domain_name = None + http_client = HttpClient() + try: + logger.info(f'ReceivedEvent: {json.dumps(event)}') + request_type = event.get('RequestType') + + if request_type == 'Delete': + http_client.send_cfn_response(CfnResponse( + context=context, + event=event, + status=CfnResponseStatus.SUCCESS, + data={}, + physical_resource_id=PHYSICAL_RESOURCE_ID + )) + return + + resource_properties = event.get('ResourceProperties', {}) + domain_name = resource_properties.get('DomainName') + logger.info('OpenSearch DomainName: ' + domain_name) + + ec2_client = boto3.client('ec2') + response = ec2_client.describe_network_interfaces(Filters=[ + {'Name': 'description', 'Values': [f'ES {domain_name}']}, + {'Name': 'requester-id', 'Values': ['amazon-elasticsearch']} + ]) + + network_interfaces = response.get('NetworkInterfaces', []) + result = [] + for network_interface in network_interfaces: + logger.debug(network_interface) + private_ip_addresses = network_interface.get('PrivateIpAddresses', []) + for private_ip_address in private_ip_addresses: + ip_address = private_ip_address.get('PrivateIpAddress', None) + if ip_address is None: + continue + result.append(ip_address) + + if len(result) == 0: + msg = 'No IP addresses found' + logger.error(msg) + http_client.send_cfn_response(CfnResponse( + context=context, + event=event, + status=CfnResponseStatus.FAILED, + data={ + 'error': msg + }, + physical_resource_id=PHYSICAL_RESOURCE_ID + )) + else: + http_client.send_cfn_response(CfnResponse( + context=context, + event=event, + status=CfnResponseStatus.SUCCESS, + data={ + 'IpAddresses': ','.join(result) + }, + physical_resource_id=PHYSICAL_RESOURCE_ID + )) + except Exception as e: + logger.exception(f'Failed to get ES Private IP Address: {e}') + error_message = f'Exception getting private IP addresses for ES soca-{domain_name}' + http_client.send_cfn_response(CfnResponse( + context=context, + event=event, + status=CfnResponseStatus.FAILED, + data={ + 'error': error_message + }, + physical_resource_id=PHYSICAL_RESOURCE_ID + )) + finally: + http_client.destroy() diff --git a/source/idea/idea-administrator/resources/lambda_functions/idea_custom_resource_self_signed_certificate/__init__.py b/source/idea/idea-administrator/resources/lambda_functions/idea_custom_resource_self_signed_certificate/__init__.py new file mode 100644 index 00000000..6d8d18a3 --- /dev/null +++ b/source/idea/idea-administrator/resources/lambda_functions/idea_custom_resource_self_signed_certificate/__init__.py @@ -0,0 +1,10 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. diff --git a/source/idea/idea-administrator/resources/lambda_functions/idea_custom_resource_self_signed_certificate/handler.py b/source/idea/idea-administrator/resources/lambda_functions/idea_custom_resource_self_signed_certificate/handler.py new file mode 100644 index 00000000..2e06c063 --- /dev/null +++ b/source/idea/idea-administrator/resources/lambda_functions/idea_custom_resource_self_signed_certificate/handler.py @@ -0,0 +1,314 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +""" +IDEA Self-Signed Certificate. + +Self-signed certificates are generated for below scenarios: +-------------------------------------------------------------------------------- +1. Create a Self-signed certificate for the external ALB +This is executed only during the first installation of the IDEA cluster. If the self-signed certificate already exists in ACM for the + cluster's default domain name: cluster-name.idea.default, a new certificate will not be created. +It is STRONGLY RECOMMENDED for you to upload your own certificate on ACM and update the Load balancer with your +personal/corporate certificate + +2. Create a self-signed certificate for the internal ALB +This certificate is used by IDEA to configure the internal load-balancer using ACM. + + +Developer Notes: +-------------------------------------------------------------------------------- + +GetSecretValue does not support IAM condition keys based on Name of the secret. Since secrets are not deleted immediately, and are +scheduled for deletion at a later date, add additional idea:SecretName tag is added to individual secrets. This allows for searching +for applicable secrets using the idea:SecretName tag and also, individual permissions can be granted based on this tag. +""" +import datetime + +from idea_lambda_commons import HttpClient, CfnResponse, CfnResponseStatus +import boto3 +import botocore.exceptions +import logging +import json +import time + +from cryptography import x509 +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives import hashes +from cryptography.hazmat.primitives.asymmetric import rsa +from cryptography.hazmat.primitives import serialization +from cryptography.x509.oid import NameOID + +logger = logging.getLogger() +logger.setLevel(logging.INFO) + + +def handle_delete(event: dict, context): + resource_properties = event.get('ResourceProperties', {}) + certificate_name = resource_properties.get('certificate_name') + domain_name = resource_properties.get('domain_name') + + client = HttpClient() + try: + + certificate_secret_name = f'{certificate_name}-certificate' + private_key_secret_name = f'{certificate_name}-private-key' + + secretsmanager_client = boto3.client('secretsmanager') + list_secrets_result = secretsmanager_client.list_secrets( + Filters=[ + { + 'Key': 'tag-key', + 'Values': ['idea:SecretName'] + }, + { + 'Key': 'tag-value', + 'Values': [certificate_secret_name, private_key_secret_name] + } + ] + ) + secret_list = list_secrets_result.get('SecretList', []) + for secret in secret_list: + secret_arn = secret.get('ARN') + logger.info(f'deleting secret: {secret_arn} ...') + secretsmanager_client.delete_secret( + SecretId=secret_arn, + ForceDeleteWithoutRecovery=True + ) + + acm_client = boto3.client('acm') + result = acm_client.list_certificates(CertificateStatuses=['ISSUED']) + + certificate_summary_list = result.get('CertificateSummaryList', []) + + for cert in certificate_summary_list: + if domain_name == cert.get('DomainName'): + acm_certificate_arn = cert.get('CertificateArn') + logger.info(f'deleting acm certificate: {acm_certificate_arn} ...') + retry_count = 10 + current = 0 + sleep_interval = 5 + while current < retry_count: + try: + current += 1 + logger.info(f'deleting certificate - attempt: {current}') + acm_client.delete_certificate( + CertificateArn=acm_certificate_arn + ) + break + except botocore.exceptions.ClientError as e: + if e.response['Error']['Code'] == 'ResourceInUseException': + logger.warning(f'cannot delete certificate - {e}. wait till the applicable resource releases the certificate.') + time.sleep(sleep_interval) + else: + raise e + + client.send_cfn_response(CfnResponse( + context=context, + event=event, + status=CfnResponseStatus.SUCCESS, + data={}, + physical_resource_id=certificate_name + )) + except Exception as e: + error_message = f'failed to delete certificate: {certificate_name} - {e}' + logger.exception(error_message) + client.send_cfn_response(CfnResponse( + context=context, + event=event, + status=CfnResponseStatus.FAILED, + data={}, + physical_resource_id=certificate_name, + reason=error_message + )) + finally: + client.destroy() + + +def handle_create_or_update(event: dict, context): + resource_properties = event.get('ResourceProperties', {}) + domain_name = resource_properties.get('domain_name') + certificate_name = resource_properties.get('certificate_name') + create_acm_certificate = resource_properties.get('create_acm_certificate', False) + kms_key_id = resource_properties.get('kms_key_id', None) + tags = resource_properties.get('tags', {}) + + client = HttpClient() + + try: + + common_tags = [] + for key, value in tags.items(): + common_tags.append({ + 'Key': key, + 'Value': value + }) + + certificate_secret_name = f'{certificate_name}-certificate' + private_key_secret_name = f'{certificate_name}-private-key' + certificate_content = None + private_key_content = None + certificate_secret_arn = None + private_key_secret_arn = None + + secretsmanager_client = boto3.client('secretsmanager') + list_secrets_result = secretsmanager_client.list_secrets( + Filters=[ + { + 'Key': 'tag-key', + 'Values': ['idea:SecretName'] + }, + { + 'Key': 'tag-value', + 'Values': [certificate_secret_name, private_key_secret_name] + } + ] + ) + secret_list = list_secrets_result.get('SecretList', []) + for secret in secret_list: + name = secret.get('Name') + arn = secret.get('ARN') + get_secret_result = secretsmanager_client.get_secret_value( + SecretId=arn + ) + secret_string = get_secret_result.get('SecretString') + if name == certificate_secret_name: + certificate_content = secret_string + certificate_secret_arn = arn + logger.info(f'found: {name}') + elif name == private_key_secret_name: + logger.info(f'found: {name}') + private_key_content = secret_string + private_key_secret_arn = arn + + if certificate_content is None and private_key_content is None: + one_day = datetime.timedelta(1, 0, 0) + private_key = rsa.generate_private_key( + public_exponent=65537, + key_size=2048, + backend=default_backend()) + public_key = private_key.public_key() + subject = x509.Name([ + x509.NameAttribute(NameOID.COUNTRY_NAME, 'US'), + x509.NameAttribute(NameOID.STATE_OR_PROVINCE_NAME, 'California'), + x509.NameAttribute(NameOID.LOCALITY_NAME, 'Sunnyvale'), + x509.NameAttribute(NameOID.ORGANIZATION_NAME, certificate_name), + x509.NameAttribute(NameOID.COMMON_NAME, domain_name) + ]) + + certificate = x509.CertificateBuilder() \ + .subject_name(subject) \ + .issuer_name(subject) \ + .not_valid_before(datetime.datetime.today() - one_day) \ + .not_valid_after(datetime.datetime.today() + (one_day * 3650)) \ + .serial_number(x509.random_serial_number()) \ + .public_key(public_key) \ + .add_extension( + x509.SubjectAlternativeName([ + x509.DNSName(domain_name) + ]), critical=False) \ + .add_extension(x509.BasicConstraints(ca=False, path_length=None), critical=True)\ + .sign(private_key=private_key, algorithm=hashes.SHA256(), backend=default_backend()) + + certificate_content = certificate.public_bytes(serialization.Encoding.PEM).decode("utf-8") + private_key_content = private_key.private_bytes( + serialization.Encoding.PEM, + serialization.PrivateFormat.TraditionalOpenSSL, + serialization.NoEncryption() + ).decode("utf-8") + + # create certificate secret + certificate_secret_tags = list(common_tags) + certificate_secret_tags.append({ + 'Key': 'idea:SecretName', + 'Value': certificate_secret_name + }) + create_secret_request = { + 'Name': f'{certificate_secret_name}', + 'Description': f'Self-Signed certificate for domain name: {domain_name}', + 'SecretString': certificate_content, + 'Tags': certificate_secret_tags + } + if kms_key_id is not None: + create_secret_request['KmsKeyId'] = kms_key_id + create_certificate_secret_result = secretsmanager_client.create_secret(**create_secret_request) + certificate_secret_arn = create_certificate_secret_result.get('ARN') + + # create private key secret + private_key_secret_tags = list(common_tags) + private_key_secret_tags.append({ + 'Key': 'idea:SecretName', + 'Value': private_key_secret_name + }) + create_secret_request = { + 'Name': f'{private_key_secret_name}', + 'Description': f'Self-Signed certificate private key for domain name: {domain_name}', + 'SecretString': private_key_content, + 'Tags': private_key_secret_tags + } + if kms_key_id is not None: + create_secret_request['KmsKeyId'] = kms_key_id + create_private_key_secret_result = secretsmanager_client.create_secret(**create_secret_request) + private_key_secret_arn = create_private_key_secret_result.get('ARN') + + acm_certificate_arn = None + if create_acm_certificate: + acm_client = boto3.client('acm') + result = acm_client.list_certificates(CertificateStatuses=['ISSUED']) + + certificate_summary_list = result.get('CertificateSummaryList', []) + + for cert in certificate_summary_list: + if domain_name == cert.get('DomainName'): + acm_certificate_arn = cert.get('CertificateArn') + + if acm_certificate_arn is None: + response = acm_client.import_certificate( + Certificate=certificate_content, + PrivateKey=private_key_content, + Tags=common_tags + ) + acm_certificate_arn = response.get('CertificateArn') + + client.send_cfn_response(CfnResponse( + context=context, + event=event, + status=CfnResponseStatus.SUCCESS, + data={ + 'certificate_secret_arn': certificate_secret_arn, + 'private_key_secret_arn': private_key_secret_arn, + 'acm_certificate_arn': acm_certificate_arn + }, + physical_resource_id=certificate_name + )) + + except Exception as e: + error_message = f'failed to create certificate: {certificate_name} - {e}' + logger.exception(error_message) + client.send_cfn_response(CfnResponse( + context=context, + event=event, + status=CfnResponseStatus.FAILED, + data={}, + physical_resource_id=certificate_name, + reason=error_message + )) + finally: + client.destroy() + + +def handler(event: dict, context): + logger.info(f'ReceivedEvent: {json.dumps(event)}') + request_type = event.get('RequestType', None) + if request_type == 'Delete': + handle_delete(event, context) + else: + handle_create_or_update(event, context) diff --git a/source/idea/idea-administrator/resources/lambda_functions/idea_custom_resource_self_signed_certificate/requirements.txt b/source/idea/idea-administrator/resources/lambda_functions/idea_custom_resource_self_signed_certificate/requirements.txt new file mode 100644 index 00000000..0d38bc5e --- /dev/null +++ b/source/idea/idea-administrator/resources/lambda_functions/idea_custom_resource_self_signed_certificate/requirements.txt @@ -0,0 +1 @@ +cryptography diff --git a/source/idea/idea-administrator/resources/lambda_functions/idea_custom_resource_update_cluster_prefix_list/__init__.py b/source/idea/idea-administrator/resources/lambda_functions/idea_custom_resource_update_cluster_prefix_list/__init__.py new file mode 100644 index 00000000..6d8d18a3 --- /dev/null +++ b/source/idea/idea-administrator/resources/lambda_functions/idea_custom_resource_update_cluster_prefix_list/__init__.py @@ -0,0 +1,10 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. diff --git a/source/idea/idea-administrator/resources/lambda_functions/idea_custom_resource_update_cluster_prefix_list/handler.py b/source/idea/idea-administrator/resources/lambda_functions/idea_custom_resource_update_cluster_prefix_list/handler.py new file mode 100644 index 00000000..a0b8ea1d --- /dev/null +++ b/source/idea/idea-administrator/resources/lambda_functions/idea_custom_resource_update_cluster_prefix_list/handler.py @@ -0,0 +1,101 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +from idea_lambda_commons import HttpClient, CfnResponse, CfnResponseStatus +import boto3 +import logging + +logger = logging.getLogger() +logger.setLevel(logging.INFO) + +PHYSICAL_RESOURCE_ID = 'cluster-prefix-list' + + +def handler(event: dict, context): + """ + Check and add new IP addresses to Cluster Prefix List + Will not remove IP addresses from the cluster prefix list. + """ + client = HttpClient() + try: + logging.info(f'ReceivedEvent: {event}') + + request_type = event.get('RequestType', None) + if request_type == 'Delete': + client.send_cfn_response(CfnResponse( + context=context, + event=event, + status=CfnResponseStatus.SUCCESS, + data={}, + physical_resource_id=PHYSICAL_RESOURCE_ID + )) + return + + resource_properties = event.get('ResourceProperties', {}) + prefix_list_id = resource_properties['prefix_list_id'] + add_entries = resource_properties.get('add_entries', []) + + add_entries_map = {} + for entry in add_entries: + cidr = entry['Cidr'] + add_entries_map[cidr] = entry + + ec2_client = boto3.client('ec2') + + # find all existing entries and check if any entries already exist. + prefix_list_paginator = ec2_client.get_paginator('get_managed_prefix_list_entries') + prefix_list_iterator = prefix_list_paginator.paginate(PrefixListId=prefix_list_id) + for prefix_list in prefix_list_iterator: + cidr_entries = prefix_list.get('Entries', []) + for cidr_entry in cidr_entries: + cidr = cidr_entry['Cidr'] + if cidr in add_entries_map: + del add_entries_map[cidr] + + add_entries = list(add_entries_map.values()) + + if len(add_entries) > 0: + for entry in add_entries: + logger.info(f'adding new entry to cluster prefix list: {entry}') + describe_result = ec2_client.describe_managed_prefix_lists( + PrefixListIds=[prefix_list_id] + ) + prefix_list_info = describe_result['PrefixLists'][0] + version = prefix_list_info['Version'] + ec2_client.modify_managed_prefix_list( + AddEntries=add_entries, + PrefixListId=prefix_list_id, + CurrentVersion=version + ) + else: + logger.info('no new entries to add. skip.') + + client.send_cfn_response(CfnResponse( + context=context, + event=event, + status=CfnResponseStatus.SUCCESS, + data={}, + physical_resource_id=PHYSICAL_RESOURCE_ID + )) + + except Exception as e: + error_message = f'failed to update cluster prefix list: {e}' + logging.exception(error_message) + client.send_cfn_response(CfnResponse( + context=context, + event=event, + status=CfnResponseStatus.FAILED, + data={}, + physical_resource_id=PHYSICAL_RESOURCE_ID, + reason=error_message + )) + finally: + client.destroy() diff --git a/source/idea/idea-administrator/resources/lambda_functions/idea_custom_resource_update_cluster_settings/__init__.py b/source/idea/idea-administrator/resources/lambda_functions/idea_custom_resource_update_cluster_settings/__init__.py new file mode 100644 index 00000000..6d8d18a3 --- /dev/null +++ b/source/idea/idea-administrator/resources/lambda_functions/idea_custom_resource_update_cluster_settings/__init__.py @@ -0,0 +1,10 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. diff --git a/source/idea/idea-administrator/resources/lambda_functions/idea_custom_resource_update_cluster_settings/handler.py b/source/idea/idea-administrator/resources/lambda_functions/idea_custom_resource_update_cluster_settings/handler.py new file mode 100644 index 00000000..86bf1b28 --- /dev/null +++ b/source/idea/idea-administrator/resources/lambda_functions/idea_custom_resource_update_cluster_settings/handler.py @@ -0,0 +1,152 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +from idea_lambda_commons import HttpClient, CfnResponse, CfnResponseStatus +import boto3 +import logging +import json + +logger = logging.getLogger() +logger.setLevel(logging.INFO) + + +def handler(event: dict, context): + """ + Create or Update Cluster Configuration based on key received key value pairs + """ + logger.info(f'ReceivedEvent: {json.dumps(event)}') + request_type = event.get('RequestType', None) + resource_properties = event.get('ResourceProperties', {}) + old_resource_properties = event.get('OldResourceProperties', None) + cluster_name = resource_properties.get('cluster_name') + module_id = resource_properties.get('module_id') + version = resource_properties.get('version') + stack_name = f'{cluster_name}-{module_id}' + physical_resource_id = f'{cluster_name}-{module_id}-settings' + client = HttpClient() + try: + + dynamodb = boto3.resource('dynamodb') + + cluster_settings_table_name = f'{cluster_name}.cluster-settings' + cluster_settings_table = dynamodb.Table(cluster_settings_table_name) + + settings = resource_properties.get('settings') + modules_table_name = f'{cluster_name}.modules' + modules_table = dynamodb.Table(modules_table_name) + + if request_type == 'Delete': + + for key in settings: + config_key = f'{module_id}.{key}' + logger.info(f'deleting config: {config_key}') + cluster_settings_table.delete_item( + Key={ + 'key': config_key + } + ) + + modules_table.update_item( + Key={ + 'module_id': module_id + }, + UpdateExpression='SET #status=:status, #stack_name=:stack_name, #version=:version', + ExpressionAttributeNames={ + '#status': 'status', + '#stack_name': 'stack_name', + '#version': 'version', + }, + ExpressionAttributeValues={ + ':status': 'not-deployed', + ':stack_name': None, + ':version': None + } + ) + + else: + + for key in settings: + value = settings[key] + + config_key = f'{module_id}.{key}' + logger.info(f'updating config: {config_key} = {value}') + cluster_settings_table.update_item( + Key={ + 'key': config_key + }, + UpdateExpression='SET #value=:value, #source=:source ADD #version :version', + ExpressionAttributeNames={ + '#value': 'value', + '#version': 'version', + '#source': 'source' + }, + ExpressionAttributeValues={ + ':value': value, + ':source': 'stack', + ':version': 1 + } + ) + + modules_table.update_item( + Key={ + 'module_id': module_id + }, + UpdateExpression='SET #status=:status, #stack_name=:stack_name, #version=:version', + ExpressionAttributeNames={ + '#status': 'status', + '#stack_name': 'stack_name', + '#version': 'version', + }, + ExpressionAttributeValues={ + ':status': 'deployed', + ':stack_name': stack_name, + ':version': version + } + ) + + # in case of a stack Update event, compute a delta using OldResourceProperties + # and clean up any values that are no longer applicable + # this takes care of scenarios where a resource was deleted from the updated stack + if old_resource_properties is not None: + old_settings = old_resource_properties.get('settings', {}) + settings_to_delete = [] + for old_key in old_settings: + if old_key not in settings: + settings_to_delete.append(old_key) + if len(settings_to_delete) > 0: + for key in settings_to_delete: + config_key = f'{module_id}.{key}' + logger.info(f'deleting config: {config_key}') + cluster_settings_table.delete_item( + Key={ + 'key': config_key + } + ) + + client.send_cfn_response(CfnResponse( + context=context, + event=event, + status=CfnResponseStatus.SUCCESS, + data={}, + physical_resource_id=physical_resource_id + )) + + except Exception as e: + logger.exception(f'failed to update cluster settings for module: {module_id} - {e}') + client.send_cfn_response(CfnResponse( + context=context, + event=event, + status=CfnResponseStatus.FAILED, + data={}, + physical_resource_id=physical_resource_id + )) + finally: + client.destroy() diff --git a/source/idea/idea-administrator/resources/lambda_functions/idea_ec2_state_event_transformation_lambda/__init__.py b/source/idea/idea-administrator/resources/lambda_functions/idea_ec2_state_event_transformation_lambda/__init__.py new file mode 100644 index 00000000..6d8d18a3 --- /dev/null +++ b/source/idea/idea-administrator/resources/lambda_functions/idea_ec2_state_event_transformation_lambda/__init__.py @@ -0,0 +1,10 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. diff --git a/source/idea/idea-administrator/resources/lambda_functions/idea_ec2_state_event_transformation_lambda/handler.py b/source/idea/idea-administrator/resources/lambda_functions/idea_ec2_state_event_transformation_lambda/handler.py new file mode 100644 index 00000000..617f5b72 --- /dev/null +++ b/source/idea/idea-administrator/resources/lambda_functions/idea_ec2_state_event_transformation_lambda/handler.py @@ -0,0 +1,81 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. +""" +EC2 State event transformation +This function is triggered every time a state change event is triggered by an EC2 instance tagged +with IDEA_CLUSTER. It repackages the event, appends tag information and then publishes an 'Ec2.StateChangeEvent' +intended for the different modules that have subscribed to this event within the cluster +""" +import boto3 +import os +import json +import logging + +logger = logging.getLogger() +logger.setLevel(logging.INFO) + +sns_client = boto3.client('sns') +ec2_resource = boto3.resource('ec2') + + +def handler(event, _): + try: + detail_type = event['detail-type'] + if detail_type != 'EC2 Instance State-change Notification': + logger.error(f'ERROR. Invalid detail type {detail_type}') + return + + cluster_name_tag_key = os.environ.get('IDEA_CLUSTER_NAME_TAG_KEY') + cluster_name_tag_value = os.environ.get('IDEA_CLUSTER_NAME_TAG_VALUE') + + instance_id = event['detail']['instance-id'] + state = event['detail']['state'] + ec2instance = ec2_resource.Instance(instance_id) + event['detail']['tags'] = {} + cluster_match = False + + message_attributes = {} + for tags in ec2instance.tags: + event['detail']['tags'][tags["Key"]] = tags["Value"] + message_attributes[tags["Key"].replace(':', '_')] = { + 'DataType': 'String', + 'StringValue': tags["Value"] + } + + if tags["Key"] == cluster_name_tag_key and tags["Value"] == cluster_name_tag_value: + cluster_match = True + + if not cluster_match: + logger.info(f'tag_key(s): {cluster_name_tag_key} and tag_value(s): {cluster_name_tag_value} on instance-id: {instance_id} not found. NO=OP.') + return + + forwarding_event = { + 'header': { + 'namespace': 'Ec2.StateChangeEvent', + 'request_id': instance_id + }, + 'payload': event['detail'] + } + + logger.info(f'forwarding ec2-state-event for {instance_id} for state {state}') + forwarding_topic_arn = os.environ.get('IDEA_EC2_STATE_SNS_TOPIC_ARN') + response = sns_client.publish( + TopicArn=forwarding_topic_arn, + MessageStructure='json', + MessageAttributes=message_attributes, + Message=json.dumps({ + 'default': json.dumps(forwarding_event), + 'sqs': forwarding_event, + })) + + logger.info(response) + except Exception as e: + logger.exception(f'Error in Handling ec2 state change event: {event}, error: {e}') diff --git a/source/idea/idea-administrator/resources/lambda_functions/idea_efs_throughput/__init__.py b/source/idea/idea-administrator/resources/lambda_functions/idea_efs_throughput/__init__.py new file mode 100644 index 00000000..6d8d18a3 --- /dev/null +++ b/source/idea/idea-administrator/resources/lambda_functions/idea_efs_throughput/__init__.py @@ -0,0 +1,10 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. diff --git a/source/idea/idea-administrator/resources/lambda_functions/idea_efs_throughput/handler.py b/source/idea/idea-administrator/resources/lambda_functions/idea_efs_throughput/handler.py new file mode 100644 index 00000000..7cb48081 --- /dev/null +++ b/source/idea/idea-administrator/resources/lambda_functions/idea_efs_throughput/handler.py @@ -0,0 +1,58 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +import json +import boto3 +import datetime +import os +import logging + +logger = logging.getLogger() +logger.setLevel(logging.INFO) + + +def handler(event, _): + cw_client = boto3.client('cloudwatch') + efs_client = boto3.client('efs') + message = json.loads(event['Records'][0]['Sns']['Message']) + file_system_id = message['Trigger']['Dimensions'][0]['value'] + logger.info('FilesystemID: ' + file_system_id) + + now = datetime.datetime.now() + start_time = now - datetime.timedelta(seconds=300) + end_time = min(now, start_time + datetime.timedelta(seconds=300)) + response = cw_client.get_metric_statistics(Namespace='AWS/EFS', MetricName='BurstCreditBalance', + Dimensions=[{'Name': 'FileSystemId', 'Value': file_system_id}], + Period=60, StartTime=start_time, EndTime=end_time, Statistics=['Average']) + efs_average_burst_credit_balance = response['Datapoints'][0]['Average'] + logger.info(f'EFS AverageBurstCreditBalance: {efs_average_burst_credit_balance}') + + response = efs_client.describe_file_systems(FileSystemId=file_system_id) + throughput_mode = response['FileSystems'][0]['ThroughputMode'] + logger.info('EFS ThroughputMode: ' + str(throughput_mode)) + + if efs_average_burst_credit_balance < int(os.environ['EFSBurstCreditLowThreshold']): + # CreditBalance is less than LowThreshold --> Change to ProvisionedThroughput + if throughput_mode == 'bursting': + # Update filesystem to Provisioned + efs_client.update_file_system( + FileSystemId=file_system_id, + ThroughputMode='provisioned', + ProvisionedThroughputInMibps=5.0) + logger.info('Updating EFS: ' + file_system_id + ' to Provisioned ThroughputMode with 5 MiB/sec') + elif efs_average_burst_credit_balance > int(os.environ['EFSBurstCreditHighThreshold']): + # CreditBalance is greater than HighThreshold --> Change to Bursting + if throughput_mode == 'provisioned': + # Update filesystem to Bursting + efs_client.update_file_system( + FileSystemId=file_system_id, + ThroughputMode='bursting') + logger.info('Updating EFS: ' + file_system_id + ' to Bursting ThroughputMode') diff --git a/source/idea/idea-administrator/resources/lambda_functions/idea_lambda_commons/__init__.py b/source/idea/idea-administrator/resources/lambda_functions/idea_lambda_commons/__init__.py new file mode 100644 index 00000000..673e20a1 --- /dev/null +++ b/source/idea/idea-administrator/resources/lambda_functions/idea_lambda_commons/__init__.py @@ -0,0 +1,14 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +from idea_lambda_commons.cfn_response_status import CfnResponseStatus +from idea_lambda_commons.cfn_response import CfnResponse +from idea_lambda_commons.http_client import HttpClient diff --git a/source/idea/idea-administrator/resources/lambda_functions/idea_lambda_commons/cfn_response.py b/source/idea/idea-administrator/resources/lambda_functions/idea_lambda_commons/cfn_response.py new file mode 100644 index 00000000..a9432b45 --- /dev/null +++ b/source/idea/idea-administrator/resources/lambda_functions/idea_lambda_commons/cfn_response.py @@ -0,0 +1,64 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +from __future__ import print_function +import json +from dataclasses import dataclass +from typing import Optional, Any +import logging + +from idea_lambda_commons import CfnResponseStatus + +logger = logging.getLogger() +logger.setLevel(logging.INFO) + + +@dataclass +class CfnResponse: + context: Optional[Any] + event: Optional[dict] + status: Optional[CfnResponseStatus] + physical_resource_id: Optional[str] = None + no_echo: Optional[bool] = False + reason: Optional[str] = None + data: Optional[dict] = None + + def build_response_payload(self) -> str: + + context = self.context + + reason = self.reason + if reason is None: + reason = f'See the details in CloudWatch Log Stream: {context.log_stream_name}' + + physical_resource_id = self.physical_resource_id + if physical_resource_id is None: + physical_resource_id = context.log_stream_name + + stack_id = self.event.get('StackId', None) + request_id = self.event.get('RequestId', None) + logical_resource_id = self.event.get('LogicalResourceId', None) + + payload = { + 'Status': self.status, + 'Reason': reason, + 'PhysicalResourceId': physical_resource_id, + 'StackId': stack_id, + 'RequestId': request_id, + 'LogicalResourceId': logical_resource_id, + 'NoEcho': self.no_echo, + 'Data': self.data + } + return json.dumps(payload) + + @property + def response_url(self) -> str: + return self.event.get('ResponseURL', None) diff --git a/source/idea/idea-administrator/resources/lambda_functions/idea_lambda_commons/cfn_response_status.py b/source/idea/idea-administrator/resources/lambda_functions/idea_lambda_commons/cfn_response_status.py new file mode 100644 index 00000000..2e722696 --- /dev/null +++ b/source/idea/idea-administrator/resources/lambda_functions/idea_lambda_commons/cfn_response_status.py @@ -0,0 +1,22 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +from __future__ import print_function +from enum import Enum +import logging + +logger = logging.getLogger() +logger.setLevel(logging.INFO) + + +class CfnResponseStatus(str, Enum): + SUCCESS = 'SUCCESS' + FAILED = 'FAILED' diff --git a/source/idea/idea-administrator/resources/lambda_functions/idea_lambda_commons/http_client.py b/source/idea/idea-administrator/resources/lambda_functions/idea_lambda_commons/http_client.py new file mode 100644 index 00000000..e7f4edf7 --- /dev/null +++ b/source/idea/idea-administrator/resources/lambda_functions/idea_lambda_commons/http_client.py @@ -0,0 +1,48 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. +from __future__ import print_function +import urllib3 +import logging + +logger = logging.getLogger() +logger.setLevel(logging.INFO) + + +class HttpClient: + + def __init__(self): + self._http_client = urllib3.PoolManager() + + def http_post(self, *args, **kwargs): + return self._http_client.request(*args, **kwargs) + + def send_cfn_response(self, response, log_response=True): + + json_response = response.build_response_payload() + + if log_response: + logger.info(f'SendingResponse: {json_response}') + else: + logger.info(f'SendingResponse: REDACTED') + + try: + response_url = response.response_url + response = self._http_client.request('PUT', response_url, headers={ + 'content-type': '', + 'content-length': str(len(json_response)) + }, body=json_response) + logger.info(f'StatusCode: {response.status}') + except Exception as e: + logger.exception(f'SendResponseFailed, Error: {e}') + + def destroy(self): + self._http_client.clear() + self._http_client = None diff --git a/source/idea/idea-administrator/resources/lambda_functions/idea_solution_metrics/__init__.py b/source/idea/idea-administrator/resources/lambda_functions/idea_solution_metrics/__init__.py new file mode 100644 index 00000000..6d8d18a3 --- /dev/null +++ b/source/idea/idea-administrator/resources/lambda_functions/idea_solution_metrics/__init__.py @@ -0,0 +1,10 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. diff --git a/source/idea/idea-administrator/resources/lambda_functions/idea_solution_metrics/handler.py b/source/idea/idea-administrator/resources/lambda_functions/idea_solution_metrics/handler.py new file mode 100644 index 00000000..b824d867 --- /dev/null +++ b/source/idea/idea-administrator/resources/lambda_functions/idea_solution_metrics/handler.py @@ -0,0 +1,93 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +from idea_lambda_commons import HttpClient, CfnResponse, CfnResponseStatus + +import json +import datetime +import os +import logging + +logger = logging.getLogger() +logger.setLevel(logging.INFO) + +http_client = HttpClient() + +PHYSICAL_RESOURCE_ID = 'SolutionMetricsSO0072' + +# Metric Keys that come in to the Lambda that are not relayed to the IDEA solution metrics +# ServiceToken contains the AWS account number which would remove the anonymous nature of the metrics +METRIC_DENYLIST_KEYS = ['ServiceToken'] + + +def post_metrics(event): + try: + request_timestamp = str(datetime.datetime.utcnow().isoformat()) + solution_id = 'SO0072' + uuid = event['RequestId'] + data = { + 'RequestType': event['RequestType'], + 'RequestTimeStamp': request_timestamp + } + + # Data is being sent by source/idea/idea-scheduler/src/ideascheduler/app/provisioning/job_provisioner/cloudformation_stack_builder.py + for k, v in event['ResourceProperties'].items(): + if ( + k not in data.keys() and + k not in METRIC_DENYLIST_KEYS + ): + data[k] = v + # Metrics Account (Production) + metrics_url = os.environ.get('AWS_METRICS_URL', 'https://metrics.awssolutionsbuilder.com/generic') + + time_stamp = {'TimeStamp': request_timestamp} + params = { + 'Solution': solution_id, + 'UUID': uuid, + 'Data': data + } + + metrics = dict(time_stamp, **params) + json_data = json.dumps(metrics, indent=4) + logger.info(params) + headers = {'content-type': 'application/json'} + req = http_client.http_post('POST', + metrics_url, + body=json_data.encode('utf-8'), + headers=headers) + rsp_code = req.status + logger.info(f'ResponseCode: {rsp_code}') + except Exception as e: + logger.exception(f'failed to post metrics: {e}') + + +def handler(event, context): + """ + To improve performance and usability, IDEA sends anonymous metrics to AWS. + You can disable this by setting 'cluster.solution.enable_solution_metrics' to False with idea-admin.sh + Data tracked: + - SOCA Instance information + - SOCA Instance Count + - SOCA Launch/Delete time + """ + try: + # Send Anonymous Metrics + post_metrics(event) + except Exception as e: + logger.exception(f'failed to post metrics: {e}') + finally: + http_client.send_cfn_response(CfnResponse( + context=context, + event=event, + status=CfnResponseStatus.SUCCESS, + data={}, + physical_resource_id=PHYSICAL_RESOURCE_ID + )) diff --git a/source/idea/idea-administrator/resources/policies/_templates/activedirectory.yml b/source/idea/idea-administrator/resources/policies/_templates/activedirectory.yml new file mode 100644 index 00000000..c033ed85 --- /dev/null +++ b/source/idea/idea-administrator/resources/policies/_templates/activedirectory.yml @@ -0,0 +1,13 @@ + {%- if context.config.get_string('directoryservice.provider') == 'activedirectory' %} + - Action: + - sqs:SendMessage + Resource: + - '{{ context.arns.get_ad_automation_sqs_queue_arn() }}' + Effect: Allow + Sid: ADAutomationSQS + - Action: + - dynamodb:GetItem + Resource: '{{ context.arns.get_ad_automation_ddb_table_arn() }}' + Effect: Allow + Sid: ADAutomationDDB + {%- endif %} diff --git a/source/idea/idea-administrator/resources/policies/_templates/aws-managed-ad.yml b/source/idea/idea-administrator/resources/policies/_templates/aws-managed-ad.yml new file mode 100644 index 00000000..9fd5717f --- /dev/null +++ b/source/idea/idea-administrator/resources/policies/_templates/aws-managed-ad.yml @@ -0,0 +1,13 @@ + {%- if context.config.get_string('directoryservice.provider') == 'aws_managed_activedirectory' %} + - Action: + - sqs:SendMessage + Resource: + - '{{ context.arns.get_ad_automation_sqs_queue_arn() }}' + Effect: Allow + Sid: ADAutomationSQS + - Action: + - dynamodb:GetItem + Resource: '{{ context.arns.get_ad_automation_ddb_table_arn() }}' + Effect: Allow + Sid: ADAutomationDDB + {%- endif %} diff --git a/source/idea/idea-administrator/resources/policies/_templates/custom-kms-key.yml b/source/idea/idea-administrator/resources/policies/_templates/custom-kms-key.yml new file mode 100644 index 00000000..9d0efba6 --- /dev/null +++ b/source/idea/idea-administrator/resources/policies/_templates/custom-kms-key.yml @@ -0,0 +1,9 @@ + {%- if context.config.get_string('cluster.kms.key_type') == 'customer-managed' %} + - Action: + - kms:Encrypt + - kms:Decrypt + - kms:GenerateDataKey + Resource: + - '{{ context.arns.kms_key_arn }}' + Effect: Allow + {%- endif %} diff --git a/source/idea/idea-administrator/resources/policies/_templates/lambda-basic-execution.yml b/source/idea/idea-administrator/resources/policies/_templates/lambda-basic-execution.yml new file mode 100644 index 00000000..9e1bb3bc --- /dev/null +++ b/source/idea/idea-administrator/resources/policies/_templates/lambda-basic-execution.yml @@ -0,0 +1,6 @@ + - Action: + - logs:CreateLogGroup + - logs:CreateLogStream + - logs:PutLogEvents + Resource: {{ context.arns.get_lambda_log_group_arn() }} + Effect: Allow diff --git a/source/idea/idea-administrator/resources/policies/_templates/openldap.yml b/source/idea/idea-administrator/resources/policies/_templates/openldap.yml new file mode 100644 index 00000000..8fab82df --- /dev/null +++ b/source/idea/idea-administrator/resources/policies/_templates/openldap.yml @@ -0,0 +1,10 @@ + {%- if context.config.get_string('directoryservice.provider') == 'openldap' %} + - Action: + - secretsmanager:GetSecretValue + Condition: + StringEquals: + secretsmanager:ResourceTag/idea:ClusterName: '{{ context.cluster_name }}' + secretsmanager:ResourceTag/idea:SecretName: '{{ context.cluster_name }}-{{ context.config.get_module_id("directoryservice") }}-certificate' + Resource: '*' + Effect: Allow + {%- endif %} diff --git a/source/idea/idea-administrator/resources/policies/amazon-prometheus-remote-write-access.yml b/source/idea/idea-administrator/resources/policies/amazon-prometheus-remote-write-access.yml new file mode 100644 index 00000000..748f4cab --- /dev/null +++ b/source/idea/idea-administrator/resources/policies/amazon-prometheus-remote-write-access.yml @@ -0,0 +1,6 @@ +Version: '2012-10-17' +Statement: + - Action: + - aps:RemoteWrite + Effect: Allow + Resource: '*' diff --git a/source/idea/idea-administrator/resources/policies/amazon-ssm-managed-instance-core.yml b/source/idea/idea-administrator/resources/policies/amazon-ssm-managed-instance-core.yml new file mode 100644 index 00000000..63b9d467 --- /dev/null +++ b/source/idea/idea-administrator/resources/policies/amazon-ssm-managed-instance-core.yml @@ -0,0 +1,36 @@ +Version: '2012-10-17' +Statement: + - Effect: Allow + Action: + - ssm:DescribeAssociation + - ssm:GetDeployablePatchSnapshotForInstance + - ssm:GetDocument + - ssm:DescribeDocument + - ssm:GetManifest + - ssm:GetParameter + - ssm:GetParameters + - ssm:ListAssociations + - ssm:ListInstanceAssociations + - ssm:PutInventory + - ssm:PutComplianceItems + - ssm:PutConfigurePackageResult + - ssm:UpdateAssociationStatus + - ssm:UpdateInstanceAssociationStatus + - ssm:UpdateInstanceInformation + Resource: '*' + - Effect: Allow + Action: + - ssmmessages:CreateControlChannel + - ssmmessages:CreateDataChannel + - ssmmessages:OpenControlChannel + - ssmmessages:OpenDataChannel + Resource: '*' + - Effect: Allow + Action: + - ec2messages:AcknowledgeMessage + - ec2messages:DeleteMessage + - ec2messages:FailMessage + - ec2messages:GetEndpoint + - ec2messages:GetMessages + - ec2messages:SendReply + Resource: '*' diff --git a/source/idea/idea-administrator/resources/policies/analytics-sink-lambda.yml b/source/idea/idea-administrator/resources/policies/analytics-sink-lambda.yml new file mode 100644 index 00000000..4b9c20ba --- /dev/null +++ b/source/idea/idea-administrator/resources/policies/analytics-sink-lambda.yml @@ -0,0 +1,31 @@ +Version: '2012-10-17' +Statement: + - Action: + - kinesis:DescribeStream + - kinesis:DescribeStreamSummary + - kinesis:GetRecords + - kinesis:GetShardIterator + - kinesis:ListShards + - kinesis:ListStreams + - kinesis:SubscribeToShard + Resource: + - {{ context.arns.get_kinesis_arn() }} + Effect: Allow + + - Action: + - logs:CreateLogGroup + - logs:CreateLogStream + - logs:PutLogEvents + Resource: '*' + Effect: Allow + + - Effect: Allow + Action: + - es:ESHttpPost + - es:ESHttpPut + Resource: '*' + + - Effect: Allow + Action: + - ec2:* + Resource: '*' diff --git a/source/idea/idea-administrator/resources/policies/backup-create.yml b/source/idea/idea-administrator/resources/policies/backup-create.yml new file mode 100644 index 00000000..67403937 --- /dev/null +++ b/source/idea/idea-administrator/resources/policies/backup-create.yml @@ -0,0 +1,266 @@ +# AWSBackupServiceRolePolicyForBackup (Copy of AWS Managed Policy) +# +# Customize and/or scope down based on specific requirements for your cluster and desired infrastructure components. + +Version: '2012-10-17' +Statement: + +- Action: + - dynamodb:DescribeTable + - dynamodb:CreateBackup + Resource: arn:{{ context.aws_partition }}:dynamodb:*:*:table/* + Effect: Allow + +- Action: + - dynamodb:DescribeBackup + - dynamodb:DeleteBackup + Resource: arn:{{ context.aws_partition }}:dynamodb:*:*:table/*/backup/* + Effect: Allow + +- Effect: Allow + Action: + - rds:AddTagsToResource + - rds:ListTagsForResource + - rds:DescribeDBSnapshots + - rds:CreateDBSnapshot + - rds:CopyDBSnapshot + - rds:DescribeDBInstances + - rds:CreateDBClusterSnapshot + - rds:DescribeDBClusters + - rds:DescribeDBClusterSnapshots + - rds:CopyDBClusterSnapshot + Resource: '*' + +- Effect: Allow + Action: + - rds:ModifyDBInstance + Resource: + - arn:{{ context.aws_partition }}:rds:*:*:db:* + +- Effect: Allow + Action: + - rds:DeleteDBSnapshot + - rds:ModifyDBSnapshotAttribute + Resource: + - arn:{{ context.aws_partition }}:rds:*:*:snapshot:awsbackup:* + +- Effect: Allow + Action: + - rds:DeleteDBClusterSnapshot + - rds:ModifyDBClusterSnapshotAttribute + Resource: + - arn:{{ context.aws_partition }}:rds:*:*:cluster-snapshot:awsbackup:* + +- Effect: Allow + Action: + - storagegateway:CreateSnapshot + - storagegateway:ListTagsForResource + Resource: arn:{{ context.aws_partition }}:storagegateway:*:*:gateway/*/volume/* + +- Effect: Allow + Action: + - ec2:CopySnapshot + Resource: arn:{{ context.aws_partition }}:ec2:*::snapshot/* + +- Effect: Allow + Action: + - ec2:CopyImage + Resource: '*' + +- Effect: Allow + Action: + - ec2:CreateTags + - ec2:DeleteSnapshot + Resource: arn:{{ context.aws_partition }}:ec2:*::snapshot/* + +- Effect: Allow + Action: + - ec2:CreateImage + - ec2:DeregisterImage + Resource: '*' + +- Effect: Allow + Action: + - ec2:CreateTags + Resource: arn:{{ context.aws_partition }}:ec2:*:*:image/* + +- Effect: Allow + Action: + - ec2:DescribeSnapshots + - ec2:DescribeTags + - ec2:DescribeImages + - ec2:DescribeInstances + - ec2:DescribeInstanceAttribute + - ec2:DescribeInstanceCreditSpecifications + - ec2:DescribeNetworkInterfaces + - ec2:DescribeElasticGpus + - ec2:DescribeSpotInstanceRequests + Resource: '*' + +- Effect: Allow + Action: + - ec2:ModifySnapshotAttribute + - ec2:ModifyImageAttribute + Resource: '*' + Condition: + 'Null': + aws:ResourceTag/aws:backup:source-resource: 'false' + +- Effect: Allow + Action: + - backup:DescribeBackupVault + - backup:CopyIntoBackupVault + Resource: arn:{{ context.aws_partition }}:backup:*:*:backup-vault:* + +- Effect: Allow + Action: + - backup:CopyFromBackupVault + Resource: '*' + +- Action: + - elasticfilesystem:Backup + - elasticfilesystem:DescribeTags + Resource: arn:{{ context.aws_partition }}:elasticfilesystem:*:*:file-system/* + Effect: Allow + +- Effect: Allow + Action: + - ec2:CreateSnapshot + - ec2:DeleteSnapshot + - ec2:DescribeVolumes + - ec2:DescribeSnapshots + Resource: + - arn:{{ context.aws_partition }}:ec2:*::snapshot/* + - arn:{{ context.aws_partition }}:ec2:*:*:volume/* + +- Action: + - kms:Decrypt + - kms:GenerateDataKey + Effect: Allow + Resource: '*' + Condition: + StringLike: + kms:ViaService: + - dynamodb.*.{{ context.aws_dns_suffix }} + +- Action: kms:DescribeKey + Effect: Allow + Resource: '*' + +- Action: kms:CreateGrant + Effect: Allow + Resource: '*' + Condition: + Bool: + kms:GrantIsForAWSResource: 'true' + +- Action: + - kms:GenerateDataKeyWithoutPlaintext + Effect: Allow + Resource: arn:{{ context.aws_partition }}:kms:*:*:key/* + Condition: + StringLike: + kms:ViaService: + - ec2.*.{{ context.aws_dns_suffix }} + +- Action: + - tag:GetResources + Resource: '*' + Effect: Allow + +- Effect: Allow + Action: + - ssm:CancelCommand + - ssm:GetCommandInvocation + Resource: '*' + +- Effect: Allow + Action: ssm:SendCommand + Resource: + - arn:{{ context.aws_partition }}:ssm:*:*:document/AWSEC2-CreateVssSnapshot + - arn:{{ context.aws_partition }}:ec2:*:*:instance/* + +- Action: fsx:DescribeBackups + Effect: Allow + Resource: arn:{{ context.aws_partition }}:fsx:*:*:backup/* + +- Effect: Allow + Action: fsx:CreateBackup + Resource: + - arn:{{ context.aws_partition }}:fsx:*:*:file-system/* + - arn:{{ context.aws_partition }}:fsx:*:*:backup/* + - arn:{{ context.aws_partition }}:fsx:*:*:volume/* + +- Action: fsx:DescribeFileSystems + Effect: Allow + Resource: arn:{{ context.aws_partition }}:fsx:*:*:file-system/* + +- Effect: Allow + Action: fsx:DescribeVolumes + Resource: arn:{{ context.aws_partition }}:fsx:*:*:volume/* + +- Effect: Allow + Action: fsx:ListTagsForResource + Resource: + - arn:{{ context.aws_partition }}:fsx:*:*:file-system/* + - arn:{{ context.aws_partition }}:fsx:*:*:volume/* + +- Action: fsx:DeleteBackup + Effect: Allow + Resource: arn:{{ context.aws_partition }}:fsx:*:*:backup/* + +- Effect: Allow + Action: + - fsx:ListTagsForResource + - fsx:ManageBackupPrincipalAssociations + - fsx:CopyBackup + - fsx:TagResource + Resource: arn:{{ context.aws_partition }}:fsx:*:*:backup/* + +- Sid: DynamodbBackupPermissions + Effect: Allow + Action: + - dynamodb:StartAwsBackupJob + - dynamodb:ListTagsOfResource + Resource: arn:{{ context.aws_partition }}:dynamodb:*:*:table/* + +- Sid: BackupGatewayBackupPermissions + Effect: Allow + Action: + - backup-gateway:Backup + - backup-gateway:ListTagsForResource + Resource: arn:{{ context.aws_partition }}:backup-gateway:*:*:vm/* + +- Effect: Allow + Action: + - cloudformation:GetTemplate + - cloudformation:DescribeStacks + - cloudformation:ListStackResources + Resource: arn:{{ context.aws_partition }}:cloudformation:*:*:stack/*/* + +- Effect: Allow + Action: + - redshift:CreateClusterSnapshot + - redshift:DescribeClusterSnapshots + - redshift:DescribeTags + Resource: + - arn:{{ context.aws_partition }}:redshift:*:*:snapshot:*/* + - arn:{{ context.aws_partition }}:redshift:*:*:cluster:* + +- Effect: Allow + Action: + - redshift:DeleteClusterSnapshot + Resource: + - arn:{{ context.aws_partition }}:redshift:*:*:snapshot:*/* + +- Effect: Allow + Action: + - redshift:DescribeClusters + Resource: + - arn:{{ context.aws_partition }}:redshift:*:*:cluster:* + +- Effect: Allow + Action: + - redshift:CreateTags + Resource: + - arn:{{ context.aws_partition }}:redshift:*:*:snapshot:*/* diff --git a/source/idea/idea-administrator/resources/policies/backup-restore.yml b/source/idea/idea-administrator/resources/policies/backup-restore.yml new file mode 100644 index 00000000..ad86e21b --- /dev/null +++ b/source/idea/idea-administrator/resources/policies/backup-restore.yml @@ -0,0 +1,264 @@ +# AWSBackupServiceRolePolicyForRestores (Copy of AWS Managed Policy) +# +# Restore policy will be added to the backup role only if cluster.backups.enable_restore == True (default: True) +# Customize and/or scope down based on specific requirements for your cluster and desired infrastructure components. + +Version: '2012-10-17' +Statement: + +- Effect: Allow + Action: + - dynamodb:Scan + - dynamodb:Query + - dynamodb:UpdateItem + - dynamodb:PutItem + - dynamodb:GetItem + - dynamodb:DeleteItem + - dynamodb:BatchWriteItem + - dynamodb:DescribeTable + Resource: arn:{{ context.aws_partition }}:dynamodb:*:*:table/* + +- Effect: Allow + Action: + - dynamodb:RestoreTableFromBackup + Resource: arn:{{ context.aws_partition }}:dynamodb:*:*:table/*/backup/* + +- Effect: Allow + Action: + - ec2:CreateVolume + - ec2:DeleteVolume + Resource: + - arn:{{ context.aws_partition }}:ec2:*::snapshot/* + - arn:{{ context.aws_partition }}:ec2:*:*:volume/* + +- Effect: Allow + Action: + - ec2:DescribeImages + - ec2:DescribeInstances + - ec2:DescribeSnapshots + - ec2:DescribeVolumes + Resource: '*' + +- Effect: Allow + Action: + - storagegateway:DeleteVolume + - storagegateway:DescribeCachediSCSIVolumes + - storagegateway:DescribeStorediSCSIVolumes + Resource: arn:{{ context.aws_partition }}:storagegateway:*:*:gateway/*/volume/* + +- Effect: Allow + Action: + - storagegateway:DescribeGatewayInformation + - storagegateway:CreateStorediSCSIVolume + - storagegateway:CreateCachediSCSIVolume + Resource: arn:{{ context.aws_partition }}:storagegateway:*:*:gateway/* + +- Effect: Allow + Action: + - storagegateway:ListVolumes + Resource: arn:{{ context.aws_partition }}:storagegateway:*:*:* + +- Effect: Allow + Action: + - rds:DescribeDBInstances + - rds:DescribeDBSnapshots + - rds:ListTagsForResource + - rds:RestoreDBInstanceFromDBSnapshot + - rds:DeleteDBInstance + - rds:AddTagsToResource + - rds:DescribeDBClusters + - rds:RestoreDBClusterFromSnapshot + - rds:DeleteDBCluster + - rds:RestoreDBInstanceToPointInTime + Resource: '*' + +- Effect: Allow + Action: + - elasticfilesystem:Restore + - elasticfilesystem:CreateFilesystem + - elasticfilesystem:DescribeFilesystems + - elasticfilesystem:DeleteFilesystem + Resource: arn:{{ context.aws_partition }}:elasticfilesystem:*:*:file-system/* + +- Effect: Allow + Action: kms:DescribeKey + Resource: '*' + +- Effect: Allow + Action: + - kms:Decrypt + - kms:Encrypt + - kms:GenerateDataKey + - kms:ReEncryptTo + - kms:ReEncryptFrom + Resource: '*' + Condition: + StringLike: + kms:ViaService: + - dynamodb.*.{{ context.aws_dns_suffix }} + - ec2.*.{{ context.aws_dns_suffix }} + - elasticfilesystem.*.{{ context.aws_dns_suffix }} + - rds.*.{{ context.aws_dns_suffix }} + - redshift.*.{{ context.aws_dns_suffix }} + +- Effect: Allow + Action: kms:CreateGrant + Resource: '*' + Condition: + Bool: + kms:GrantIsForAWSResource: 'true' + +- Effect: Allow + Action: + - ebs:CompleteSnapshot + - ebs:StartSnapshot + - ebs:PutSnapshotBlock + Resource: arn:{{ context.aws_partition }}:ec2:*::snapshot/* + +- Effect: Allow + Action: + - rds:CreateDBInstance + Resource: arn:{{ context.aws_partition }}:rds:*:*:db:* + +- Effect: Allow + Action: + - ec2:DeleteSnapshot + - ec2:DeleteTags + Resource: arn:{{ context.aws_partition }}:ec2:*::snapshot/* + Condition: + 'Null': + aws:ResourceTag/aws:backup:source-resource: 'false' + +- Effect: Allow + Action: ec2:CreateTags + Resource: + - arn:{{ context.aws_partition }}:ec2:*::snapshot/* + - arn:{{ context.aws_partition }}:ec2:*:*:instance/* + Condition: + ForAllValues:StringEquals: + aws:TagKeys: + - aws:backup:source-resource + +- Effect: Allow + Action: + - ec2:RunInstances + Resource: '*' + +- Effect: Allow + Action: + - ec2:TerminateInstances + Resource: arn:{{ context.aws_partition }}:ec2:*:*:instance/* + +- Effect: Allow + Action: + - fsx:CreateFileSystemFromBackup + Resource: + - arn:{{ context.aws_partition }}:fsx:*:*:file-system/* + - arn:{{ context.aws_partition }}:fsx:*:*:backup/* + +- Effect: Allow + Action: + - fsx:DescribeFileSystems + - fsx:TagResource + Resource: arn:{{ context.aws_partition }}:fsx:*:*:file-system/* + +- Effect: Allow + Action: fsx:DescribeBackups + Resource: arn:{{ context.aws_partition }}:fsx:*:*:backup/* + +- Effect: Allow + Action: + - fsx:DeleteFileSystem + - fsx:UntagResource + Resource: arn:{{ context.aws_partition }}:fsx:*:*:file-system/* + Condition: + 'Null': + aws:ResourceTag/aws:backup:source-resource: 'false' + +- Effect: Allow + Action: + - fsx:DescribeVolumes + Resource: arn:{{ context.aws_partition }}:fsx:*:*:volume/* + +- Effect: Allow + Action: + - fsx:CreateVolumeFromBackup + - fsx:TagResource + Resource: + - arn:{{ context.aws_partition }}:fsx:*:*:volume/* + Condition: + ForAllValues:StringEquals: + aws:TagKeys: + - aws:backup:source-resource + +- Effect: Allow + Action: + - fsx:CreateVolumeFromBackup + Resource: + - arn:{{ context.aws_partition }}:fsx:*:*:storage-virtual-machine/* + - arn:{{ context.aws_partition }}:fsx:*:*:backup/* +- Effect: Allow + Action: + - fsx:DeleteVolume + - fsx:UntagResource + Resource: arn:{{ context.aws_partition }}:fsx:*:*:volume/* + Condition: + 'Null': + aws:ResourceTag/aws:backup:source-resource: 'false' + +- Effect: Allow + Action: ds:DescribeDirectories + Resource: '*' + +- Sid: DynamoDBRestorePermissions + Effect: Allow + Action: + - dynamodb:RestoreTableFromAwsBackup + Resource: arn:{{ context.aws_partition }}:dynamodb:*:*:table/* + +- Sid: GatewayRestorePermissions + Effect: Allow + Action: + - backup-gateway:Restore + Resource: arn:{{ context.aws_partition }}:backup-gateway:*:*:hypervisor/* + +- Effect: Allow + Action: + - cloudformation:CreateChangeSet + - cloudformation:DescribeChangeSet + Resource: arn:{{ context.aws_partition }}:cloudformation:*:*:stack/*/* + +- Effect: Allow + Action: + - redshift:RestoreFromClusterSnapshot + - redshift:RestoreTableFromClusterSnapshot + Resource: + - arn:{{ context.aws_partition }}:redshift:*:*:snapshot:*/* + - arn:{{ context.aws_partition }}:redshift:*:*:cluster:* + +- Effect: Allow + Action: + - redshift:DescribeClusters + Resource: + - arn:{{ context.aws_partition }}:redshift:*:*:cluster:* + +- Effect: Allow + Action: + - redshift:DescribeTableRestoreStatus + Resource: '*' + +- Effect: Allow + Action: + - ec2:DescribeAccountAttributes + - ec2:DescribeAddresses + - ec2:DescribeAvailabilityZones + - ec2:DescribeSecurityGroups + - ec2:DescribeSubnets + - ec2:DescribeVpcs + - ec2:DescribeInternetGateways + Resource: '*' + +- Effect: Allow + Action: + - iam:PassRole + Resource: '*' diff --git a/source/idea/idea-administrator/resources/policies/backup-s3-create.yml b/source/idea/idea-administrator/resources/policies/backup-s3-create.yml new file mode 100644 index 00000000..9c3fd6e5 --- /dev/null +++ b/source/idea/idea-administrator/resources/policies/backup-s3-create.yml @@ -0,0 +1,64 @@ +# AWSBackupServiceRolePolicyForS3Backup (Copy of AWS Managed Policy) +# +# Customize and/or scope down based on specific requirements for your cluster and desired infrastructure components. + +Version: '2012-10-17' +Statement: + +- Effect: Allow + Action: cloudwatch:GetMetricData + Resource: '*' + +- Effect: Allow + Action: + - events:DeleteRule + - events:PutTargets + - events:DescribeRule + - events:EnableRule + - events:PutRule + - events:RemoveTargets + - events:ListTargetsByRule + - events:DisableRule + Resource: + - arn:{{ context.aws_partition }}:events:*:*:rule/AwsBackupManagedRule* + +- Effect: Allow + Action: events:ListRules + Resource: '*' + +- Effect: Allow + Action: + - kms:Decrypt + - kms:DescribeKey + Resource: '*' + Condition: + StringLike: + kms:ViaService: s3.*.{{ context.aws_dns_suffix }} + +- Effect: Allow + Action: + - s3:GetBucketTagging + - s3:GetInventoryConfiguration + - s3:ListBucketVersions + - s3:ListBucket + - s3:GetBucketVersioning + - s3:GetBucketLocation + - s3:GetBucketAcl + - s3:PutInventoryConfiguration + - s3:GetBucketNotification + - s3:PutBucketNotification + Resource: arn:{{ context.aws_partition }}:s3:::* + +- Effect: Allow + Action: + - s3:GetObjectAcl + - s3:GetObject + - s3:GetObjectVersionTagging + - s3:GetObjectVersionAcl + - s3:GetObjectTagging + - s3:GetObjectVersion + Resource: arn:{{ context.aws_partition }}:s3:::*/* + +- Effect: Allow + Action: s3:ListAllMyBuckets + Resource: '*' diff --git a/source/idea/idea-administrator/resources/policies/backup-s3-restore.yml b/source/idea/idea-administrator/resources/policies/backup-s3-restore.yml new file mode 100644 index 00000000..0311d363 --- /dev/null +++ b/source/idea/idea-administrator/resources/policies/backup-s3-restore.yml @@ -0,0 +1,43 @@ +# AWSBackupServiceRolePolicyForS3Restore (Copy of AWS Managed Policy) +# +# Restore policy will be added to the backup role only if cluster.backups.enable_restore == True (default: True) +# Customize and/or scope down based on specific requirements for your cluster and desired infrastructure components. + +Version: '2012-10-17' +Statement: + +- Effect: Allow + Action: + - s3:CreateBucket + - s3:ListBucketVersions + - s3:ListBucket + - s3:GetBucketVersioning + - s3:GetBucketLocation + - s3:PutBucketVersioning + Resource: + - arn:{{ context.aws_partition }}:s3:::* + +- Effect: Allow + Action: + - s3:GetObject + - s3:GetObjectVersion + - s3:DeleteObject + - s3:PutObjectVersionAcl + - s3:GetObjectVersionAcl + - s3:GetObjectTagging + - s3:PutObjectTagging + - s3:GetObjectAcl + - s3:PutObjectAcl + - s3:ListMultipartUploadParts + - s3:PutObject + Resource: + - arn:{{ context.aws_partition }}:s3:::*/* + +- Effect: Allow + Action: + - kms:DescribeKey + - kms:GenerateDataKey + Resource: '*' + Condition: + StringLike: + kms:ViaService: s3.*.{{ context.aws_dns_suffix }} diff --git a/source/idea/idea-administrator/resources/policies/bastion-host.yml b/source/idea/idea-administrator/resources/policies/bastion-host.yml new file mode 100644 index 00000000..40dc4332 --- /dev/null +++ b/source/idea/idea-administrator/resources/policies/bastion-host.yml @@ -0,0 +1,33 @@ +Version: '2012-10-17' +Statement: + - Action: + - ec2:DescribeVolumes + - ec2:DescribeNetworkInterfaces + Resource: '*' + Effect: Allow + - Action: + - ec2:CreateTags + Resource: + - '{{ context.arns.get_arn("ec2", "volume/*", aws_region="*") }}' + - '{{ context.arns.get_arn("ec2", "network-interface/*", aws_region="*") }}' + Effect: Allow + - Action: + - s3:GetObject + - s3:ListBucket + - s3:GetBucketAcl + Resource: + {{ context.utils.to_yaml(context.arns.s3_bucket_arns) | indent(6) }} + Effect: Allow + + - Action: + - logs:PutRetentionPolicy + Resource: '*' + Effect: Allow + +{% include '_templates/aws-managed-ad.yml' %} + +{% include '_templates/activedirectory.yml' %} + +{% include '_templates/openldap.yml' %} + +{% include '_templates/custom-kms-key.yml' %} diff --git a/source/idea/idea-administrator/resources/policies/cloud-watch-agent-server-policy.yml b/source/idea/idea-administrator/resources/policies/cloud-watch-agent-server-policy.yml new file mode 100644 index 00000000..0ad09962 --- /dev/null +++ b/source/idea/idea-administrator/resources/policies/cloud-watch-agent-server-policy.yml @@ -0,0 +1,17 @@ +Version: '2012-10-17' +Statement: + - Effect: Allow + Action: + - cloudwatch:PutMetricData + - ec2:DescribeVolumes + - ec2:DescribeTags + - logs:PutLogEvents + - logs:DescribeLogStreams + - logs:DescribeLogGroups + - logs:CreateLogStream + - logs:CreateLogGroup + Resource: '*' + - Effect: Allow + Action: + - ssm:GetParameter + Resource: '{{ context.arns.get_arn("ssm", "parameter/AmazonCloudWatch-*", aws_region="*") }}' diff --git a/source/idea/idea-administrator/resources/policies/cluster-manager.yml b/source/idea/idea-administrator/resources/policies/cluster-manager.yml new file mode 100644 index 00000000..701fca0c --- /dev/null +++ b/source/idea/idea-administrator/resources/policies/cluster-manager.yml @@ -0,0 +1,128 @@ +Version: '2012-10-17' +Statement: + - Action: sqs:SendMessage + Resource: + - '{{ context.arns.get_sqs_arn(context.config.get_module_id("virtual-desktop-controller") + "-controller") }}' + Effect: Allow + - Action: + - ec2:DescribeVolumes + - ec2:DescribeNetworkInterfaces + - ec2:DescribeInstances + - ec2:DescribeInstanceTypes + - budgets:ViewBudget + Resource: '*' + Effect: Allow + + - Action: + - ec2:CreateTags + Resource: + - '{{ context.arns.get_arn("ec2", "volume/*", aws_region="*") }}' + - '{{ context.arns.get_arn("ec2", "network-interface/*", aws_region="*") }}' + Effect: Allow + + - Action: + - ses:SendEmail + Resource: + - '{{ context.arns.ses_arn }}' + Effect: Allow + + - Action: + - s3:GetObject + - s3:ListBucket + - s3:PutObject + - s3:GetBucketAcl + Resource: + {{ context.utils.to_yaml(context.arns.s3_bucket_arns) | indent(6) }} + Effect: Allow + + - Action: + - dynamodb:GetItem + - dynamodb:Query + - dynamodb:Scan + - dynamodb:DescribeTable + - dynamodb:DescribeStream + - dynamodb:GetRecords + - dynamodb:GetShardIterator + - dynamodb:ListStreams + - dynamodb:UpdateItem + - dynamodb:PutItem + - dynamodb:DeleteItem + - dynamodb:TagResource + - dynamodb:CreateTable + - dynamodb:UpdateTable + - dynamodb:UpdateTimeToLive + Resource: + - '{{ context.arns.get_ddb_table_arn("cluster-settings") }}' + - '{{ context.arns.get_ddb_table_arn("cluster-settings/stream/*") }}' + - '{{ context.arns.get_ddb_table_arn("modules") }}' + - '{{ context.arns.get_ddb_table_arn("email-templates") }}' + - '{{ context.arns.get_ddb_table_arn("accounts.sequence-config") }}' + - '{{ context.arns.get_ddb_table_arn("accounts.users") }}' + - '{{ context.arns.get_ddb_table_arn("accounts.groups") }}' + - '{{ context.arns.get_ddb_table_arn("accounts.group-members") }}' + - '{{ context.arns.get_ddb_table_arn("accounts.sso-state") }}' + - '{{ context.arns.get_ddb_table_arn("accounts.group-members/stream/*") }}' + - '{{ context.arns.get_ddb_table_arn("projects") }}' + - '{{ context.arns.get_ddb_table_arn("projects/index/*") }}' + - '{{ context.arns.get_ddb_table_arn("projects.user-projects") }}' + - '{{ context.arns.get_ddb_table_arn("projects.project-groups") }}' + - '{{ context.arns.get_ddb_table_arn("ad-automation") }}' + - '{{ context.arns.get_ddb_table_arn(context.module_id + ".distributed-lock") }}' + Effect: Allow + + - Action: + - cloudwatch:PutMetricData + Resource: '*' + Effect: Allow + Condition: + StringLike: + cloudwatch:namespace: IDEA/* + + {%- if context.config.get_string('directoryservice.provider') == 'aws_managed_activedirectory' %} + - Action: + - ds:ResetUserPassword + Resource: '{{ context.arns.get_directory_service_arn() }}' + Effect: Allow + {%- endif %} + + + # PutSecret value is required so that cluster-manager can refresh directory service credentials when nearing expiration + - Action: + - secretsmanager:GetSecretValue + - secretsmanager:PutSecretValue + Condition: + StringEquals: + secretsmanager:ResourceTag/idea:ClusterName: '{{ context.cluster_name }}' + secretsmanager:ResourceTag/idea:ModuleName: + - cluster-manager + - directoryservice + Resource: '*' + Effect: Allow + + - Action: + - cognito-idp:* + Resource: '{{ context.arns.user_pool_arn }}' + Effect: Allow + + - Action: + - sqs:* + Resource: + - '{{ context.arns.get_sqs_arn(context.module_id + "-tasks.fifo") }}' + - '{{ context.arns.get_sqs_arn(context.module_id + "-notifications.fifo") }}' + - '{{ context.arns.get_sqs_arn(context.config.get_module_id("directoryservice") + "-ad-automation.fifo") }}' + Effect: Allow + Sid: ClusterManagerSQSQueues + + - Action: + - logs:PutRetentionPolicy + Resource: '*' + Effect: Allow + + - Action: + - kinesis:PutRecord + - kinesis:PutRecords + Resource: + - '{{ context.arns.get_kinesis_arn() }}' + Effect: Allow + +{% include '_templates/custom-kms-key.yml' %} diff --git a/source/idea/idea-administrator/resources/policies/compute-node.yml b/source/idea/idea-administrator/resources/policies/compute-node.yml new file mode 100644 index 00000000..b4716c46 --- /dev/null +++ b/source/idea/idea-administrator/resources/policies/compute-node.yml @@ -0,0 +1,56 @@ +Version: '2012-10-17' +Statement: + + - Sid: AllowReadAccessForClusterS3Bucket + Action: + - s3:GetObject + - s3:ListBucket + Resource: + {{ context.utils.to_yaml(context.arns.s3_bucket_arns) | indent(6) }} + Effect: Allow + + - Action: + - s3:GetObject + - s3:ListBucket + Resource: + {{ context.utils.to_yaml(context.arns.s3_global_arns) | indent(6) }} + Effect: Allow + + - Action: + - ec2:CreateTags + Resource: + - '{{ context.arns.get_arn("ec2", "volume/*", aws_region="*") }}' + - '{{ context.arns.get_arn("ec2", "network-interface/*", aws_region="*") }}' + - '{{ context.arns.get_arn("ec2", "instance/*", aws_region="*") }}' + Effect: Allow + + - Action: + - ec2:DescribeVolumes + - ec2:DescribeNetworkInterfaces + - fsx:CreateDataRepositoryTask + - fsx:DescribeFileSystems + - tag:GetResources + - tag:GetTagValues + - tag:GetTagKeys + Resource: '*' + Effect: Allow + + - Action: + - sqs:SendMessage + Resource: + - '{{ context.arns.get_sqs_arn(context.module_id + "-job-status-events") }}' + Effect: Allow + Sid: JobStatusEvents + + - Action: + - logs:PutRetentionPolicy + Resource: '*' + Effect: Allow + +{% include '_templates/aws-managed-ad.yml' %} + +{% include '_templates/activedirectory.yml' %} + +{% include '_templates/openldap.yml' %} + +{% include '_templates/custom-kms-key.yml' %} diff --git a/source/idea/idea-administrator/resources/policies/controller-scheduled-event-transformer-lambda.yml b/source/idea/idea-administrator/resources/policies/controller-scheduled-event-transformer-lambda.yml new file mode 100644 index 00000000..3dbfd2f1 --- /dev/null +++ b/source/idea/idea-administrator/resources/policies/controller-scheduled-event-transformer-lambda.yml @@ -0,0 +1,22 @@ +Version: '2012-10-17' +Statement: + - Action: + - logs:CreateLogGroup + Resource: "{{ context.arns.get_lambda_log_group_arn() }}" + Effect: Allow + Sid: CloudWatchLogsPermissions + - Action: + - logs:CreateLogStream + - logs:PutLogEvents + - logs:DeleteLogStream + Resource: "{{ context.arns.lambda_log_stream_arn }}" + Effect: Allow + Sid: CloudWatchLogStreamPermissions + - Action: + - ec2:DescribeInstances + Resource: "*" + Effect: Allow + - Effect: Allow + Action: sqs:SendMessage + Resource: + - '{{ context.arns.get_sqs_arn(context.config.get_module_id("virtual-desktop-controller") + "-events.fifo") }}' diff --git a/source/idea/idea-administrator/resources/policies/controller-ssm-command-pass-role.yml b/source/idea/idea-administrator/resources/policies/controller-ssm-command-pass-role.yml new file mode 100644 index 00000000..c52113ed --- /dev/null +++ b/source/idea/idea-administrator/resources/policies/controller-ssm-command-pass-role.yml @@ -0,0 +1,29 @@ +Version: '2012-10-17' +Statement: + - Action: + - sns:Publish + Resource: + - '{{ context.arns.get_sns_arn(context.config.get_module_id("virtual-desktop-controller") + "-ssm-commands-sns-topic") }}' + Effect: Allow + + - Action: + - s3:GetObject + - s3:ListBucket + - s3:PutObject + - s3:GetBucketAcl + Resource: + {{ context.utils.to_yaml(context.arns.s3_bucket_arns) | indent(6) }} + Effect: Allow + + - Action: + - logs:PutRetentionPolicy + Resource: '*' + Effect: Allow + + - Action: + - logs:CreateLogGroup + - logs:CreateLogStream + - logs:PutLogEvents + Resource: + - '{{ context.arns.get_log_group_arn(context.config.get_module_id("virtual-desktop-controller") + "/dcv-session/*") }}' + Effect: Allow diff --git a/source/idea/idea-administrator/resources/policies/custom-resource-cluster-endpoints.yml b/source/idea/idea-administrator/resources/policies/custom-resource-cluster-endpoints.yml new file mode 100644 index 00000000..0d4da099 --- /dev/null +++ b/source/idea/idea-administrator/resources/policies/custom-resource-cluster-endpoints.yml @@ -0,0 +1,21 @@ +Version: '2012-10-17' +Statement: + - Action: + - logs:CreateLogGroup + - logs:CreateLogStream + - logs:DeleteLogStream + - logs:PutLogEvents + Resource: '{{ context.arns.get_lambda_log_group_arn() }}' + Effect: Allow + Sid: CloudWatchLogsPermissions + + - Action: + - elasticloadbalancing:ModifyListener + - elasticloadbalancing:CreateRule + - elasticloadbalancing:DeleteRule + - elasticloadbalancing:ModifyRule + - elasticloadbalancing:DescribeRules + - elasticloadbalancing:DescribeTags + Resource: '*' + Effect: Allow + Sid: ClusterEndpointManagementRules diff --git a/source/idea/idea-administrator/resources/policies/custom-resource-ec2-create-tags.yml b/source/idea/idea-administrator/resources/policies/custom-resource-ec2-create-tags.yml new file mode 100644 index 00000000..2317ebfa --- /dev/null +++ b/source/idea/idea-administrator/resources/policies/custom-resource-ec2-create-tags.yml @@ -0,0 +1,8 @@ +Version: '2012-10-17' +Statement: + - Action: + - ec2:CreateTags + Resource: '*' + Effect: Allow + +{% include '_templates/lambda-basic-execution.yml' %} diff --git a/source/idea/idea-administrator/resources/policies/custom-resource-get-ad-security-group.yml b/source/idea/idea-administrator/resources/policies/custom-resource-get-ad-security-group.yml new file mode 100644 index 00000000..1c0aee57 --- /dev/null +++ b/source/idea/idea-administrator/resources/policies/custom-resource-get-ad-security-group.yml @@ -0,0 +1,16 @@ +Version: '2012-10-17' +Statement: + - Action: + - logs:CreateLogGroup + - logs:CreateLogStream + - logs:DeleteLogStream + - logs:PutLogEvents + Resource: '{{ context.arns.get_lambda_log_group_arn() }}' + Effect: Allow + Sid: CloudWatchLogsPermissions + + - Action: + - ds:DescribeDirectories + Resource: '*' + Effect: Allow + Sid: DescribeDirectories diff --git a/source/idea/idea-administrator/resources/policies/custom-resource-get-user-pool-client-secret.yml b/source/idea/idea-administrator/resources/policies/custom-resource-get-user-pool-client-secret.yml new file mode 100644 index 00000000..3dbbcf28 --- /dev/null +++ b/source/idea/idea-administrator/resources/policies/custom-resource-get-user-pool-client-secret.yml @@ -0,0 +1,16 @@ +Version: '2012-10-17' +Statement: + - Action: + - logs:CreateLogGroup + - logs:CreateLogStream + - logs:DeleteLogStream + - logs:PutLogEvents + Resource: '{{ context.arns.get_lambda_log_group_arn() }}' + Effect: Allow + Sid: CloudWatchLogsPermissions + + - Action: + - cognito-idp:DescribeUserPoolClient + Resource: '*' + Effect: Allow + Sid: DescribeUserPoolClient diff --git a/source/idea/idea-administrator/resources/policies/custom-resource-opensearch-private-ips.yml b/source/idea/idea-administrator/resources/policies/custom-resource-opensearch-private-ips.yml new file mode 100644 index 00000000..aa4bc07d --- /dev/null +++ b/source/idea/idea-administrator/resources/policies/custom-resource-opensearch-private-ips.yml @@ -0,0 +1,19 @@ +Version: '2012-10-17' +Statement: + - Action: + - logs:CreateLogGroup + - logs:CreateLogStream + - logs:DeleteLogStream + - logs:PutLogEvents + Resource: '{{ context.arns.get_lambda_log_group_arn() }}' + Effect: Allow + Sid: CloudWatchLogsPermissions + + - Action: + - ec2:DescribeNetworkInterfaces + Resource: '*' + Effect: Allow + Condition: + ForAllValues:ArnEqualsIfExists: + ec2:Vpc: '{{ context.arns.vpc_arn }}' + Sid: DescribeNetworkInterfaces diff --git a/source/idea/idea-administrator/resources/policies/custom-resource-self-signed-certificate.yml b/source/idea/idea-administrator/resources/policies/custom-resource-self-signed-certificate.yml new file mode 100644 index 00000000..553a1745 --- /dev/null +++ b/source/idea/idea-administrator/resources/policies/custom-resource-self-signed-certificate.yml @@ -0,0 +1,42 @@ +Version: '2012-10-17' +Statement: + - Action: + - logs:CreateLogGroup + Resource: '{{ context.arns.get_lambda_log_group_arn() }}' + Effect: Allow + Sid: CloudWatchLogsPermissions + + - Action: + - logs:CreateLogStream + - logs:PutLogEvents + Resource: '{{ context.arns.lambda_log_stream_arn }}' + Effect: Allow + Sid: CloudWatchLogStreamPermissions + + - Action: + - acm:ImportCertificate + - acm:ListCertificates + - acm:DeleteCertificate + - acm:AddTagsToCertificate + Resource: '*' + Effect: Allow + Sid: ACMPermissions + + {%- if context.config.get_string('cluster.secretsmanager.kms_key_id') %} + - Action: + - kms:GenerateDataKey + - kms:Decrypt + Resource: + - '{{ context.arns.kms_key_arn }}' + Effect: Allow + {%- endif %} + + - Action: + - secretsmanager:ListSecrets + - secretsmanager:DeleteSecret + - secretsmanager:GetSecretValue + - secretsmanager:CreateSecret + - secretsmanager:TagResource + Resource: '*' + Effect: Allow + Sid: SecretManagerPermissions diff --git a/source/idea/idea-administrator/resources/policies/custom-resource-update-cluster-prefix-list.yml b/source/idea/idea-administrator/resources/policies/custom-resource-update-cluster-prefix-list.yml new file mode 100644 index 00000000..9e41cdcf --- /dev/null +++ b/source/idea/idea-administrator/resources/policies/custom-resource-update-cluster-prefix-list.yml @@ -0,0 +1,17 @@ +Version: '2012-10-17' +Statement: + - Action: + - logs:CreateLogGroup + - logs:CreateLogStream + - logs:DeleteLogStream + - logs:PutLogEvents + Resource: '{{ context.arns.get_lambda_log_group_arn() }}' + Effect: Allow + Sid: CloudWatchLogsPermissions + + - Action: + - ec2:GetManagedPrefixListEntries + - ec2:ModifyManagedPrefixList + - ec2:DescribeManagedPrefixLists + Resource: '*' + Effect: Allow diff --git a/source/idea/idea-administrator/resources/policies/custom-resource-update-cluster-settings.yml b/source/idea/idea-administrator/resources/policies/custom-resource-update-cluster-settings.yml new file mode 100644 index 00000000..234fd7f3 --- /dev/null +++ b/source/idea/idea-administrator/resources/policies/custom-resource-update-cluster-settings.yml @@ -0,0 +1,17 @@ +Version: '2012-10-17' +Statement: + - Action: + - logs:CreateLogGroup + - logs:CreateLogStream + - logs:DeleteLogStream + - logs:PutLogEvents + Resource: {{ context.arns.get_lambda_log_group_arn() }} + Effect: Allow + Sid: CloudWatchLogsPermissions + + - Action: + - dynamodb:UpdateItem + - dynamodb:DeleteItem + Resource: + {{ context.utils.to_yaml(context.arns.cluster_config_ddb_arn) | indent(6) }} + Effect: Allow diff --git a/source/idea/idea-administrator/resources/policies/ec2state-event-transformer.yml b/source/idea/idea-administrator/resources/policies/ec2state-event-transformer.yml new file mode 100644 index 00000000..56c0e5c3 --- /dev/null +++ b/source/idea/idea-administrator/resources/policies/ec2state-event-transformer.yml @@ -0,0 +1,20 @@ +Version: '2012-10-17' +Statement: +- Action: + - logs:CreateLogGroup + Resource: "{{ context.arns.get_lambda_log_group_arn() }}" + Effect: Allow + Sid: CloudWatchLogsPermissions +- Action: + - logs:CreateLogStream + - logs:PutLogEvents + Resource: "{{ context.arns.lambda_log_stream_arn }}" + Effect: Allow + Sid: CloudWatchLogStreamPermissions +- Action: + - ec2:DescribeInstances + Resource: "*" + Effect: Allow +- Effect: Allow + Action: sns:Publish + Resource: "{{ context.arns.get_sns_arn('*ec2-state-change*') }}" diff --git a/source/idea/idea-administrator/resources/policies/efs-throughput-lambda.yml b/source/idea/idea-administrator/resources/policies/efs-throughput-lambda.yml new file mode 100644 index 00000000..2c584431 --- /dev/null +++ b/source/idea/idea-administrator/resources/policies/efs-throughput-lambda.yml @@ -0,0 +1,22 @@ +Version: '2012-10-17' +Statement: + - Action: + - logs:CreateLogGroup + - logs:CreateLogStream + - logs:PutLogEvents + Resource: '{{ context.arns.get_log_group_arn("*EFSAppsLambda*") }}' + Effect: Allow + Sid: CloudWatchLogsPermissions + + - Action: + - cloudwatch:GetMetricStatistics + Resource: '*' + Effect: Allow + Sid: CloudWatchMetricsPermissions + + - Action: + - elasticfilesystem:DescribeFileSystems + - elasticfilesystem:UpdateFileSystem + Resource: '{{ context.arns.get_arn("elasticfilesystem", "file-system/*") }}' + Effect: Allow + Sid: EFSPermissions diff --git a/source/idea/idea-administrator/resources/policies/log-retention.yml b/source/idea/idea-administrator/resources/policies/log-retention.yml new file mode 100644 index 00000000..6471825d --- /dev/null +++ b/source/idea/idea-administrator/resources/policies/log-retention.yml @@ -0,0 +1,9 @@ +Version: '2012-10-17' +Statement: + - Action: + - logs:PutRetentionPolicy + - logs:DeleteRetentionPolicy + Resource: '*' + Effect: Allow + +{% include '_templates/lambda-basic-execution.yml' %} diff --git a/source/idea/idea-administrator/resources/policies/openldap-server.yml b/source/idea/idea-administrator/resources/policies/openldap-server.yml new file mode 100644 index 00000000..c0d60b42 --- /dev/null +++ b/source/idea/idea-administrator/resources/policies/openldap-server.yml @@ -0,0 +1,38 @@ +Version: '2012-10-17' +Statement: + - Action: + - ec2:DescribeVolumes + - ec2:DescribeNetworkInterfaces + Resource: '*' + Effect: Allow + + - Action: + - ec2:CreateTags + Resource: + - '{{ context.arns.get_arn("ec2", "volume/*", aws_region="*") }}' + - '{{ context.arns.get_arn("ec2", "network-interface/*", aws_region="*") }}' + Effect: Allow + + - Action: + - s3:GetObject + - s3:ListBucket + - s3:GetBucketAcl + Resource: + {{ context.utils.to_yaml(context.arns.s3_bucket_arns) | indent(6) }} + Effect: Allow + + - Action: + - secretsmanager:GetSecretValue + Condition: + StringEquals: + secretsmanager:ResourceTag/idea:ClusterName: '{{ context.cluster_name }}' + secretsmanager:ResourceTag/idea:ModuleName: directoryservice + Resource: '*' + Effect: Allow + + - Action: + - logs:PutRetentionPolicy + Resource: '*' + Effect: Allow + +{% include '_templates/custom-kms-key.yml' %} diff --git a/source/idea/idea-administrator/resources/policies/scheduler.yml b/source/idea/idea-administrator/resources/policies/scheduler.yml new file mode 100644 index 00000000..97be10f9 --- /dev/null +++ b/source/idea/idea-administrator/resources/policies/scheduler.yml @@ -0,0 +1,284 @@ +Version: '2012-10-17' +Statement: + - Action: + - pricing:GetProducts + - budgets:ViewBudget + - ec2:DescribeInstances + - ec2:DescribeSubnets + - ec2:DescribeSecurityGroups + - ec2:DescribeImages + - ec2:DescribeInstanceAttribute + - ec2:DescribeInstanceTypes + - ec2:DescribeInstanceStatus + - ec2:DescribeReservedInstances + - ec2:DescribeSpotInstanceRequests + - ec2:DescribeVpcClassicLink + - ec2:DescribeVolumes + - ec2:DescribePlacementGroups + - ec2:DescribeKeyPairs + - ec2:DescribeLaunchTemplates + - ec2:DescribeLaunchTemplateVersions + - ec2:DescribeNetworkInterfaces + - ec2:DescribeSpotFleetRequests + - ec2:DescribeSpotFleetInstances + - ec2:DescribeSpotFleetRequestHistory + - ec2:DescribeTags + - fsx:DescribeFileSystems + - iam:GetInstanceProfile + - autoscaling:DescribeAutoScalingGroups + - autoscaling:DescribeScalingActivities + - autoscaling:DescribeLaunchConfigurations + - elasticloadbalancing:DescribeRules + - elasticloadbalancing:DescribeListeners + - elasticloadbalancing:DescribeTargetGroups + - savingsplans:DescribeSavingsPlans + - servicequotas:ListServiceQuotas + - ssm:ListDocuments + - ssm:ListDocumentVersions + - ssm:DescribeDocument + - ssm:GetDocument + - ssm:DescribeInstanceInformation + - ssm:DescribeDocumentParameters + - ssm:DescribeInstanceProperties + - ssm:ListCommands + - ssm:GetCommandInvocation + - ssm:DescribeAutomationExecutions + Resource: '*' + Effect: Allow + + - Condition: + StringLikeIfExists: + autoscaling:LaunchConfigurationName: '{{ context.cluster_name }}*' + Action: + - autoscaling:UpdateAutoScalingGroup + - autoscaling:DeleteAutoScalingGroup + - autoscaling:CreateAutoScalingGroup + - autoscaling:DetachInstances + - ec2:DeleteLaunchTemplate + - ec2:CreateLaunchTemplate + - fsx:CreateDataRepositoryTask + Resource: '*' + Effect: Allow + + - Action: + - ec2:CreateTags + Resource: + - '{{ context.arns.get_arn("ec2", "volume/*", aws_region="*") }}' + - '{{ context.arns.get_arn("ec2", "network-interface/*", aws_region="*") }}' + - '{{ context.arns.get_arn("ec2", "instance/*", aws_region="*") }}' + - '{{ context.arns.get_arn("ec2", "spot-instances-request/*", aws_region="*") }}' + Effect: Allow + + - Action: + - cloudformation:CreateStack + - cloudformation:DeleteStack + - cloudformation:DescribeStackResources + - cloudformation:DescribeStacks + - cloudformation:GetTemplate + Resource: '*' + Effect: Allow + + - Condition: + ForAllValues:ArnEqualsIfExists: + ec2:Vpc: '{{ context.arns.vpc_arn }}' + Action: + - ec2:RunInstances + - ec2:StopInstances + - ec2:StartInstances + - ec2:TerminateInstances + - ec2:CreatePlacementGroup + - ec2:DeletePlacementGroup + - ec2:ModifyInstanceAttribute + Resource: + {{ context.utils.to_yaml(context.arns.ec2_common_arns) | indent(6) }} + Effect: Allow + + - Action: + - ssm:SendCommand + Resource: + - '{{ context.arns.get_arn("ec2", "instance/*") }}' + - '{{ context.arns.get_arn("ssm", "document/AWS-RunPowerShellScript", aws_account_id="") }}' + - '{{ context.arns.get_arn("ssm", "document/AWS-RunShellScript", aws_account_id="") }}' + Effect: Allow + + - Action: + - ssm:StartAutomationExecution + Resource: + - '{{ context.arns.get_ssm_arn("automation-definition/") }}' + Effect: Allow + + - Action: + - ssm:StopAutomationExecution + - ssm:GetAutomationExecution + Resource: + - '{{ context.arns.get_ssm_arn("automation-execution/") }}' + Effect: Allow + + - Action: + - lambda:InvokeFunction + Resource: + - '{{ context.arns.get_lambda_arn() }}' + Effect: Allow + + - Action: + - fsx:CreateFileSystem + - fsx:TagResource + Resource: + - '{{ context.arns.get_arn("fsx", "file-system/*") }}' + Effect: Allow + + - Condition: + StringLike: + aws:ResourceTag/idea:ClusterName: '{{ context.cluster_name }}' + Action: + - fsx:DeleteFileSystem + Resource: + - '{{ context.arns.get_arn("fsx", "file-system/*") }}' + Effect: Allow + + - Action: + - iam:CreateServiceLinkedRole + - iam:AttachRolePolicy + - iam:PutRolePolicy + Resource: + {{ context.utils.to_yaml(context.arns.service_role_arns) | indent(6) }} + Effect: Allow + + - Condition: + ForAllValues:ArnEqualsIfExists: + ec2:Vpc: '{{ context.arns.vpc_arn }}' + Action: + - ec2:CreatePlacementGroup + - ec2:DeletePlacementGroup + - ec2:RequestSpotFleet + - ec2:ModifySpotFleetRequest + - ec2:CancelSpotFleetRequests + Resource: '*' + Effect: Allow + + - Action: + - s3:GetObject + - s3:ListBucket + - s3:PutObject + - s3:GetBucketAcl + Resource: + {{ context.utils.to_yaml(context.arns.s3_bucket_arns) | indent(6) }} + Effect: Allow + + - Action: + - dynamodb:GetItem + - dynamodb:Query + - dynamodb:Scan + - dynamodb:DescribeTable + - dynamodb:DescribeStream + - dynamodb:GetRecords + - dynamodb:GetShardIterator + - dynamodb:ListStreams + Resource: + {{ context.utils.to_yaml(context.arns.cluster_config_ddb_arn) | indent(6) }} + Effect: Allow + + - Action: + - dynamodb:TagResource + - dynamodb:BatchGet* + - dynamodb:DescribeStream + - dynamodb:DescribeTable + - dynamodb:Get* + - dynamodb:Query + - dynamodb:Scan + - dynamodb:BatchWrite* + - dynamodb:CreateTable + - dynamodb:Delete* + - dynamodb:Update* + - dynamodb:PutItem + Resource: + - '{{ context.arns.get_ddb_table_arn(context.module_id + ".queue-profiles") }}' + - '{{ context.arns.get_ddb_table_arn(context.module_id + ".applications") }}' + - '{{ context.arns.get_ddb_table_arn(context.module_id + ".license-resources") }}' + Effect: Allow + + - Condition: + ForAllValues:ArnEqualsIfExists: + ec2:Vpc: '{{ context.arns.vpc_arn }}' + Action: + - iam:PassRole + - iam:CreateServiceLinkedRole + Resource: + - '{{ context.vars.compute_node_role_arn }}' + - '{{ context.vars.spot_fleet_request_role_arn }}' + Effect: Allow + + - Action: + - cloudwatch:PutMetricData + Resource: '*' + Effect: Allow + Condition: + StringLike: + cloudwatch:namespace: IDEA/* + - Action: + - iam:SimulatePrincipalPolicy + Resource: + - '{{ context.vars.scheduler_role_arn }}' + Effect: Allow + + - Action: + - secretsmanager:GetSecretValue + Condition: + StringEquals: + secretsmanager:ResourceTag/idea:ClusterName: '{{ context.cluster_name }}' + secretsmanager:ResourceTag/idea:ModuleName: scheduler + Resource: '*' + Effect: Allow + + - Action: + - sqs:* + Resource: + - '{{ context.arns.get_sqs_arn(context.module_id + "-job-status-events") }}' + Effect: Allow + Sid: JobStatusEventsQueue + + - Action: + - sqs:SendMessage + Resource: + - '{{ context.arns.get_sqs_arn(context.config.get_module_id("cluster-manager") + "-notifications.fifo") }}' + Effect: Allow + Sid: SendUserNotifications + + - Action: + - logs:PutRetentionPolicy + Resource: '*' + Effect: Allow + + - Action: + - kinesis:PutRecord + - kinesis:PutRecords + Resource: + - '{{ context.arns.get_kinesis_arn() }}' + Effect: Allow + + - Effect: Allow + Action: + - ec2:CreateSnapshot + - ec2:CreateImage + Resource: + - '*' + Sid: ComputeNodeAmiBuilderPermissions1 + + - Effect: Allow + Action: + - ec2:CreateTags + Condition: + StringEquals: + ec2:CreateAction: + - CreateImage + Resource: + - '*' + Sid: ComputeNodeAmiBuilderPermissions2 + +{% include '_templates/aws-managed-ad.yml' %} + +{% include '_templates/openldap.yml' %} + +{% include '_templates/activedirectory.yml' %} + +{% include '_templates/custom-kms-key.yml' %} diff --git a/source/idea/idea-administrator/resources/policies/solution-metrics-lambda-function.yml b/source/idea/idea-administrator/resources/policies/solution-metrics-lambda-function.yml new file mode 100644 index 00000000..83f7c116 --- /dev/null +++ b/source/idea/idea-administrator/resources/policies/solution-metrics-lambda-function.yml @@ -0,0 +1,15 @@ +Version: '2012-10-17' +Statement: + - Action: + - logs:CreateLogGroup + Resource: '{{ context.arns.get_lambda_log_group_arn() }}' + Effect: Allow + Sid: CloudWatchLogsPermissions + + - Action: + - logs:CreateLogStream + - logs:PutLogEvents + - logs:DeleteLogStream + Resource: '{{ context.arns.lambda_log_stream_arn }}' + Effect: Allow + Sid: CloudWatchLogStreamPermissions diff --git a/source/idea/idea-administrator/resources/policies/spot-fleet-request.yml b/source/idea/idea-administrator/resources/policies/spot-fleet-request.yml new file mode 100644 index 00000000..e21b4abc --- /dev/null +++ b/source/idea/idea-administrator/resources/policies/spot-fleet-request.yml @@ -0,0 +1,43 @@ +Version: '2012-10-17' +Statement: + - Action: + - ec2:DescribeImages + - ec2:DescribeSubnets + - ec2:DescribeInstanceStatus + Resource: '*' + Effect: Allow + Sid: ReadOnlyPermissions + + - Action: + - ec2:RequestSpotInstances + Resource: '*' + Effect: Allow + Sid: RequestSpotInstances + + - Action: + - ec2:CreateTags + Resource: '*' + Effect: Allow + Sid: CreateTagsForAllSpotFleetResources + + - Condition: + StringEquals: + ec2:Vpc: '{{ context.arns.vpc_arn }}' + Action: + - ec2:TerminateInstances + - ec2:RunInstances + - iam:CreateServiceLinkedRole + Resource: + - '{{ context.arns.get_arn("ec2", "instance/*", aws_region="*") }}' + Effect: Allow + Sid: SpotFleetPermissions + + - Condition: + StringEquals: + iam:PassedToService: + - ec2.{{ context.aws_dns_suffix }} + Action: iam:PassRole + Resource: + - '{{ context.vars.spot_fleet_request_role_arn }}' + Effect: Allow + Sid: PassRole diff --git a/source/idea/idea-administrator/resources/policies/virtual-desktop-controller.yml b/source/idea/idea-administrator/resources/policies/virtual-desktop-controller.yml new file mode 100644 index 00000000..91806e58 --- /dev/null +++ b/source/idea/idea-administrator/resources/policies/virtual-desktop-controller.yml @@ -0,0 +1,155 @@ +Version: '2012-10-17' +Statement: + - Action: + - events:PutTargets + - events:PutRule + - events:PutEvents + - events:DeleteRule + - events:RemoveTargets + - ec2:DescribeVolumes + - ec2:DescribeNetworkInterfaces + - ec2:DescribeImageAttribute + - ec2:DescribeImages + - ec2:ModifyInstanceAttribute + - ec2:CreateImage + - ec2:StartInstances + - ec2:TerminateInstances + - ec2:StopInstances + - ec2:RebootInstances + - ec2:DescribeInstances + - ec2:DescribeInstanceTypes + - ec2:CreateTags + - ec2:RegisterImage + - ec2:DeregisterImage + - ec2:RunInstances + - fsx:CreateDataRepositoryTask + - fsx:DescribeFileSystems + - tag:GetResources + - tag:GetTagValues + - tag:GetTagKeys + - ssm:ListDocuments + - ssm:ListDocumentVersions + - ssm:DescribeDocument + - ssm:GetDocument + - ssm:DescribeInstanceInformation + - ssm:DescribeDocumentParameters + - ssm:DescribeInstanceProperties + - ssm:ListCommands + - ssm:SendCommand + - ssm:GetCommandInvocation + - ssm:DescribeAutomationExecutions + - dynamodb:ListTables + - application-autoscaling:RegisterScalableTarget + - application-autoscaling:PutScalingPolicy + - application-autoscaling:DescribeScalingPolicies + Resource: '*' + Effect: Allow + + - Action: + - s3:GetObject + - s3:ListBucket + - s3:PutObject + - s3:GetBucketAcl + Resource: + {{ context.utils.to_yaml(context.arns.s3_bucket_arns) | indent(6) }} + Effect: Allow + + - Effect: Allow + Action: + - sqs:DeleteMessage + - sqs:ReceiveMessage + - sqs:SendMessage + - sqs:GetQueueAttributes + Resource: + - '{{ context.arns.get_sqs_arn(context.config.get_module_id("virtual-desktop-controller") + "-events.fifo") }}' + - '{{ context.arns.get_sqs_arn(context.config.get_module_id("virtual-desktop-controller") + "-controller") }}' + + - Effect: Allow + Action: route53:ChangeResourceRecordSets + Resource: '{{ context.arns.get_route53_hostedzone_arn() }}' + + - Action: + - dynamodb:GetItem + - dynamodb:Query + - dynamodb:Scan + - dynamodb:DescribeTable + - dynamodb:DescribeStream + - dynamodb:GetRecords + - dynamodb:GetShardIterator + - dynamodb:ListStreams + Resource: + {{ context.utils.to_yaml(context.arns.cluster_config_ddb_arn) | indent(6) }} + Effect: Allow + + - Action: + - dynamodb:BatchGet* + - dynamodb:DescribeStream + - dynamodb:DescribeTable + - dynamodb:Get* + - dynamodb:Query + - dynamodb:Scan + - dynamodb:BatchWrite* + - dynamodb:CreateTable + - dynamodb:Delete* + - dynamodb:Update* + - dynamodb:PutItem + - dynamodb:TagResource + Resource: + - '{{ context.arns.get_ddb_table_arn(context.config.get_module_id("virtual-desktop-controller") + ".*") }}' + Effect: Allow + + - Action: + - cloudwatch:PutMetricData + Resource: '*' + Effect: Allow + + - Action: + - secretsmanager:GetSecretValue + Condition: + StringEquals: + secretsmanager:ResourceTag/idea:ClusterName: '{{ context.cluster_name }}' + secretsmanager:ResourceTag/idea:ModuleName: virtual-desktop-controller + Resource: '*' + Effect: Allow + + - Action: + - sqs:SendMessage + Resource: + - '{{ context.arns.get_sqs_arn(context.config.get_module_id("cluster-manager") + "-notifications.fifo") }}' + Effect: Allow + Sid: SendUserNotifications + + - Action: + - logs:PutRetentionPolicy + Resource: '*' + Effect: Allow + + - Action: + - kinesis:PutRecord + - kinesis:PutRecords + Resource: + - '{{ context.arns.get_kinesis_arn() }}' + Effect: Allow + + - Action: + - iam:CreateServiceLinkedRole + Resource: '{{ context.arns.get_ddb_application_autoscaling_service_role_arn() }}' + Effect: Allow + Condition: + StringLike: + iam:AWSServiceName: 'dynamodb.application-autoscaling.amazonaws.com' + + - Action: + - iam:AttachRolePolicy + - iam:PutRolePolicy + Resource: '{{ context.arns.get_ddb_application_autoscaling_service_role_arn() }}' + Effect: Allow + + + {% include '_templates/aws-managed-ad.yml' %} + + {% include '_templates/activedirectory.yml' %} + + {% include '_templates/openldap.yml' %} + + {% include '_templates/custom-kms-key.yml' %} diff --git a/source/idea/idea-administrator/resources/policies/virtual-desktop-dcv-broker.yml b/source/idea/idea-administrator/resources/policies/virtual-desktop-dcv-broker.yml new file mode 100644 index 00000000..13a5bac6 --- /dev/null +++ b/source/idea/idea-administrator/resources/policies/virtual-desktop-dcv-broker.yml @@ -0,0 +1,89 @@ +Version: '2012-10-17' +Statement: + - Action: sqs:SendMessage + Resource: + - '{{ context.arns.get_sqs_arn(context.config.get_module_id("virtual-desktop-controller") + "-events.fifo") }}' + Effect: Allow + - Action: + - s3:GetObject + - s3:ListBucket + - s3:PutObject + - s3:GetBucketAcl + Resource: + {{ context.utils.to_yaml(context.arns.s3_bucket_arns) | indent(6) }} + Effect: Allow + - Action: + - s3:GetObject + - s3:ListBucket + - s3:PutObject + - s3:GetBucketAcl + Resource: + {{ context.utils.to_yaml(context.arns.s3_bucket_arns) | indent(6) }} + Effect: Allow + - Action: + - dynamodb:BatchGet* + - dynamodb:DescribeStream + - dynamodb:DescribeTable + - dynamodb:Get* + - dynamodb:Query + - dynamodb:Scan + - dynamodb:BatchWrite* + - dynamodb:CreateTable + - dynamodb:Delete* + - dynamodb:Update* + - dynamodb:PutItem + Resource: + - '{{ context.arns.get_ddb_table_arn(context.config.get_module_id("virtual-desktop-controller") + ".*") }}' + Effect: Allow + - Action: + - cloudwatch:PutMetricData + Resource: '*' + Effect: Allow + - Action: + - sns:* + - events:PutTargets + - events:PutRule + - events:PutEvents + - events:DeleteRule + - events:RemoveTargets + - ec2:DescribeVolumes + - ec2:DescribeNetworkInterfaces + - ec2:DescribeImageAttribute + - ec2:DescribeImages + - ec2:DescribeInstances + - ec2:ModifyInstanceAttribute + - ec2:CreateImage + - ec2:StartInstances + - ec2:TerminateInstances + - ec2:StopInstances + - ec2:RebootInstances + - ec2:CreateTags + - ec2:RegisterImage + - ec2:DeregisterImage + - ec2:RunInstances + - fsx:CreateDataRepositoryTask + - fsx:DescribeFileSystems + - tag:GetResources + - tag:GetTagValues + - tag:GetTagKeys + - iam:PassRole + - ssm:ListDocuments + - ssm:ListDocumentVersions + - ssm:DescribeDocument + - ssm:GetDocument + - ssm:DescribeInstanceInformation + - ssm:DescribeDocumentParameters + - ssm:DescribeInstanceProperties + - ssm:ListCommands + - ssm:SendCommand + - ssm:GetCommandInvocation + - ssm:DescribeAutomationExecutions + - elasticloadbalancing:DescribeTargetHealth + Resource: '*' + Effect: Allow + - Action: + - logs:PutRetentionPolicy + Resource: '*' + Effect: Allow + +{% include '_templates/custom-kms-key.yml' %} diff --git a/source/idea/idea-administrator/resources/policies/virtual-desktop-dcv-connection-gateway.yml b/source/idea/idea-administrator/resources/policies/virtual-desktop-dcv-connection-gateway.yml new file mode 100644 index 00000000..31de64e2 --- /dev/null +++ b/source/idea/idea-administrator/resources/policies/virtual-desktop-dcv-connection-gateway.yml @@ -0,0 +1,48 @@ +Version: '2012-10-17' +Statement: + - Action: + - ec2:DescribeVolumes + - ec2:DescribeNetworkInterfaces + - ec2:DescribeImageAttribute + - ec2:DescribeImages + - ec2:CreateImage + - ec2:StartInstances + - ec2:TerminateInstances + - ec2:StopInstances + - ec2:CreateTags + - ec2:RegisterImage + - ec2:DeregisterImage + - ec2:RunInstances + - fsx:CreateDataRepositoryTask + - fsx:DescribeFileSystems + - tag:GetResources + - tag:GetTagValues + - tag:GetTagKeys + - iam:PassRole + Resource: '*' + Effect: Allow + + - Action: + - s3:GetObject + - s3:ListBucket + - s3:PutObject + - s3:GetBucketAcl + Resource: + {{ context.utils.to_yaml(context.arns.s3_bucket_arns) | indent(6) }} + Effect: Allow + + - Action: + - secretsmanager:GetSecretValue + Condition: + StringEquals: + secretsmanager:ResourceTag/idea:ClusterName: '{{ context.cluster_name }}' + secretsmanager:ResourceTag/idea:ModuleName: virtual-desktop-controller + Resource: '*' + Effect: Allow + + - Action: + - logs:PutRetentionPolicy + Resource: '*' + Effect: Allow + +{% include '_templates/custom-kms-key.yml' %} diff --git a/source/idea/idea-administrator/resources/policies/virtual-desktop-dcv-host.yml b/source/idea/idea-administrator/resources/policies/virtual-desktop-dcv-host.yml new file mode 100644 index 00000000..f4404eae --- /dev/null +++ b/source/idea/idea-administrator/resources/policies/virtual-desktop-dcv-host.yml @@ -0,0 +1,67 @@ +Version: '2012-10-17' +Statement: + + - Action: + - logs:CreateLogGroup + - logs:CreateLogStream + - logs:PutLogEvents + Resource: '*' + Effect: Allow + + - Action: + - s3:GetObject + - s3:ListBucket + Resource: + {{ context.utils.to_yaml(context.arns.s3_global_arns) | indent(6) }} + Effect: Allow + + - Action: + - s3:GetObject + - s3:ListBucket + Resource: + {{ context.utils.to_yaml(context.arns.s3_bucket_arns) | indent(6) }} + Effect: Allow + + - Action: + - s3:GetObject + - s3:ListBucket + Resource: + {{ context.utils.to_yaml(context.arns.dcv_license_s3_bucket_arns) | indent(6) }} + Effect: Allow + + - Effect: Allow + Action: sqs:* + Resource: + - '{{ context.arns.get_sqs_arn(context.config.get_module_id("virtual-desktop-controller") + "-events.fifo") }}' + + - Action: + - ec2:CreateTags + Resource: + - '{{ context.arns.get_arn("ec2", "volume/*", aws_region="*") }}' + - '{{ context.arns.get_arn("ec2", "network-interface/*", aws_region="*") }}' + - '{{ context.arns.get_arn("ec2", "instance/*", aws_region="*") }}' + Effect: Allow + + - Action: + - ec2:DescribeVolumes + - ec2:DescribeNetworkInterfaces + - fsx:CreateDataRepositoryTask + - fsx:DescribeFileSystems + - tag:GetResources + - tag:GetTagValues + - tag:GetTagKeys + Resource: '*' + Effect: Allow + + - Action: + - logs:PutRetentionPolicy + Resource: '*' + Effect: Allow + +{% include '_templates/aws-managed-ad.yml' %} + +{% include '_templates/activedirectory.yml' %} + +{% include '_templates/openldap.yml' %} + +{% include '_templates/custom-kms-key.yml' %} diff --git a/source/idea/idea-administrator/src/ideaadministrator/__init__.py b/source/idea/idea-administrator/src/ideaadministrator/__init__.py new file mode 100644 index 00000000..2601793a --- /dev/null +++ b/source/idea/idea-administrator/src/ideaadministrator/__init__.py @@ -0,0 +1,20 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +import ideaadministrator_meta +from ideaadministrator.app_protocols import AdministratorContextProtocol +from ideaadministrator.app_props import AdministratorProps + +__name__ = ideaadministrator_meta.__name__ +__version__ = ideaadministrator_meta.__version__ + +props = AdministratorProps() +Context = AdministratorContextProtocol diff --git a/source/idea/idea-administrator/src/ideaadministrator/app/__init__.py b/source/idea/idea-administrator/src/ideaadministrator/app/__init__.py new file mode 100644 index 00000000..6d8d18a3 --- /dev/null +++ b/source/idea/idea-administrator/src/ideaadministrator/app/__init__.py @@ -0,0 +1,10 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. diff --git a/source/idea/idea-administrator/src/ideaadministrator/app/aws_service_availability_helper.py b/source/idea/idea-administrator/src/ideaadministrator/app/aws_service_availability_helper.py new file mode 100644 index 00000000..0a216716 --- /dev/null +++ b/source/idea/idea-administrator/src/ideaadministrator/app/aws_service_availability_helper.py @@ -0,0 +1,309 @@ + +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +from ideadatamodel import constants +from ideasdk.utils import Utils +from ideasdk.context import SocaCliContext + +from typing import List, Dict +from prettytable import PrettyTable + + +class AwsServiceAvailabilityHelper: + + def __init__(self, aws_region: str = None, aws_profile: str = None, aws_secondary_profile: str = None): + if Utils.is_empty(aws_region): + aws_region = 'us-east-1' + + self.session = Utils.create_boto_session(aws_region=aws_region, aws_profile=aws_profile) + if aws_region in Utils.get_value_as_list('SSM_DISCOVERY_RESTRICTED_REGION_LIST', constants.CAVEATS, default=[]): + aws_ssm_region = Utils.get_value_as_string('SSM_DISCOVERY_FALLBACK_REGION', constants.CAVEATS, default='us-east-1') + self.ssm_session = Utils.create_boto_session(aws_region=aws_ssm_region, aws_profile=aws_secondary_profile) + else: + aws_ssm_region = aws_region + self.ssm_session = self.session + + self.ssm_client = self.ssm_session.client( + service_name='ssm', + region_name=aws_ssm_region + ) + + self.context = SocaCliContext() + + self.idea_services = { + 'acm': { + 'title': 'AWS Certificate Manager (ACM)', + 'required': True + }, + 'acm-pca': { + 'title': 'ACM Private CA', + 'required': False + }, + 'aps': { + 'title': 'Amazon Managed Service for Prometheus', + 'required': False + }, + 'backup': { + 'title': 'AWS Backup', + 'required': True + }, + 'budgets': { + 'title': 'AWS Budgets', + 'required': False + }, + 'cloudformation': { + 'title': 'AWS CloudFormation', + 'required': True + }, + 'cloudwatch': { + 'title': 'Amazon CloudWatch', + 'required': True + }, + 'cognito-idp': { + 'title': 'Amazon Cognito - User Pools', + 'required': True + }, + 'ds': { + 'title': 'AWS Directory Service for Microsoft Active Directory', + 'required': False + }, + 'dynamodb': { + 'title': 'Amazon DynamoDB', + 'required': True + }, + 'dynamodbstreams': { + 'title': 'Amazon DynamoDB Streams', + 'required': True + }, + 'ebs': { + 'title': 'Amazon Elastic Block Store (EBS)', + 'required': True + }, + 'ec2': { + 'title': 'Amazon Elastic Compute Cloud (EC2)', + 'required': True + }, + 'efs': { + 'title': 'Amazon Elastic File System (EFS)', + 'required': True + }, + 'elb': { + 'title': 'Amazon Elastic Load Balancing (ELB)', + 'required': True + }, + 'es': { + 'title': 'Amazon OpenSearch Service', + 'required': True + }, + 'eventbridge': { + 'title': 'Amazon EventBridge', + 'required': True + }, + 'events': { + 'title': 'Amazon Events', + 'required': True + }, + 'filecache': { + 'title': 'Amazon File Cache', + 'required': False + }, + 'fsx': { + 'title': 'Amazon FSx', + 'required': False + }, + 'fsx-lustre': { + 'title': 'Amazon FSx for Lustre', + 'required': False + }, + 'fsx-ontap': { + 'title': 'Amazon FSx for NetApp ONTAP', + 'required': False + }, + 'fsx-openzfs': { + 'title': 'Amazon FSx for OpenZFS', + 'required': False + }, + 'fsx-windows': { + 'title': 'Amazon FSx for Windows File Server', + 'required': False + }, + 'grafana': { + 'title': 'Amazon Managed Grafana', + 'required': False + }, + 'iam': { + 'title': 'AWS Identity and Access Management (IAM)', + 'required': True + }, + 'kinesis': { + 'title': 'Amazon Kinesis', + 'required': True + }, + 'kms': { + 'title': 'AWS Key Management Service (KMS)', + 'required': True + }, + 'lambda': { + 'title': 'AWS Lambda', + 'required': True + }, + 'logs': { + 'title': 'Amazon CloudWatch Logs', + 'required': True + }, + 'pricing': { + 'title': 'AWS Pricing API', + 'required': False + }, + 'route53': { + 'title': 'Amazon Route 53', + 'required': True + }, + 'route53resolver': { + 'title': 'Amazon Route 53 Resolver', + 'required': False + }, + 's3': { + 'title': 'Amazon Simple Storage Service (S3)', + 'required': True + }, + 'secretsmanager': { + 'title': 'AWS Secrets Manager', + 'required': True + }, + 'service-quotas': { + 'title': 'AWS Service Quotas', + 'required': True + }, + 'ses': { + 'title': 'Amazon Simple Email Service (SES)', + 'required': False + }, + 'sns': { + 'title': 'Amazon Simple Notification Service (SNS)', + 'required': True + }, + 'sqs': { + 'title': 'Amazon Simple Queue Service (SQS)', + 'required': True + }, + 'ssm': { + 'title': 'AWS Systems Manager (SSM)', + 'required': True + }, + 'sts': { + 'title': 'AWS Security Token Service (STS)', + 'required': True + }, + 'vpc': { + 'title': 'Amazon Virtual Private Cloud (VPC)', + 'required': True + } + } + self.idea_service_names = list(self.idea_services.keys()) + self.idea_service_names.sort() + + def get_available_services(self, aws_region: str) -> Dict: + result = {} + + next_token = None + all_services = set() + while True: + + if next_token is None: + get_parameters_result = self.ssm_client.get_parameters_by_path( + Path=f'/aws/service/global-infrastructure/regions/{aws_region}/services' + ) + else: + get_parameters_result = self.ssm_client.get_parameters_by_path( + Path=f'/aws/service/global-infrastructure/regions/{aws_region}/services', + NextToken=next_token + ) + + parameters = Utils.get_value_as_list('Parameters', get_parameters_result, []) + for parameter in parameters: + all_services.add(parameter['Value']) + + next_token = Utils.get_value_as_string('NextToken', get_parameters_result) + if next_token is None: + break + + for service in self.idea_services: + result[service] = service in all_services + + return result + + def get_availability_matrix(self, aws_regions: List[str]) -> List[Dict]: + matrix = {} + for aws_region in aws_regions: + service_map = self.get_available_services(aws_region=aws_region) + for service, available in service_map.items(): + if service in matrix: + region_info = matrix[service] + else: + region_info = {} + matrix[service] = region_info + region_info[aws_region] = available + + result = [] + for service, region_info in matrix.items(): + result.append({ + 'service': service, + 'regions': region_info + }) + + result.sort(key=lambda d: d['service']) + return result + + def print_availability_matrix(self, aws_regions: List[str]): + with self.context.spinner('building service availability matrix ...'): + entries = self.get_availability_matrix(aws_regions=aws_regions) + + table = PrettyTable([ + 'Service', + 'Required', + *aws_regions + ]) + table.align = 'l' + for entry in entries: + service_name = entry['service'] + service_info = self.idea_services[service_name] + service_title = service_info['title'] + required = service_info['required'] + service = f'{service_title} [{service_name}]' + row = [ + service, + 'Yes' if required else 'No' + ] + for region in aws_regions: + available = entry['regions'][region] + row.append('Yes' if available else 'No') + table.add_row(row) + print(table) + + def print_idea_services(self): + table = PrettyTable([ + 'AWS Service', + 'Name', + 'Required' + ]) + table.align = 'l' + for service_name in self.idea_service_names: + service_info = self.idea_services[service_name] + table.add_row([ + service_info['title'], + service_name, + 'Yes' if service_info['required'] else 'No' + ]) + print(table) + + diff --git a/source/idea/idea-administrator/src/ideaadministrator/app/cdk/__init__.py b/source/idea/idea-administrator/src/ideaadministrator/app/cdk/__init__.py new file mode 100644 index 00000000..6d8d18a3 --- /dev/null +++ b/source/idea/idea-administrator/src/ideaadministrator/app/cdk/__init__.py @@ -0,0 +1,10 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. diff --git a/source/idea/idea-administrator/src/ideaadministrator/app/cdk/cdk_app.py b/source/idea/idea-administrator/src/ideaadministrator/app/cdk/cdk_app.py new file mode 100644 index 00000000..f6745527 --- /dev/null +++ b/source/idea/idea-administrator/src/ideaadministrator/app/cdk/cdk_app.py @@ -0,0 +1,225 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +from ideadatamodel import constants +from ideasdk.utils import Utils, EnvironmentUtils +from ideaadministrator.app.cdk.stacks import ( + SocaBootstrapStack, + ClusterStack, + IdentityProviderStack, + DirectoryServiceStack, + SharedStorageStack, + ClusterManagerStack, + SchedulerStack, + BastionHostStack, + AnalyticsStack, + VirtualDesktopControllerStack, + MetricsStack +) + +import aws_cdk as cdk +from aws_cdk import Aspects +from cdk_nag import AwsSolutionsChecks + + +class CdkApp: + """ + IDEA CDK App + """ + + def __init__(self, cluster_name: str, + module_name: str, + module_id: str, + aws_region: str, + deployment_id: str, + termination_protection: bool, + aws_profile: str = None): + + self.cluster_name = cluster_name + self.module_name = module_name + self.module_id = module_id + self.aws_region = aws_region + self.aws_profile = aws_profile + self.deployment_id = deployment_id + self.termination_protection = termination_protection + + session = Utils.create_boto_session(aws_region=aws_region, aws_profile=aws_profile) + + sts = session.client('sts') + result = sts.get_caller_identity() + account_id = Utils.get_value_as_string('Account', result) + + self.cdk_app = cdk.App() + + # add cdk nag scan + enable_nag_scan = Utils.get_as_bool(EnvironmentUtils.get_environment_variable('IDEA_ADMIN_ENABLE_CDK_NAG_SCAN'), True) + if enable_nag_scan: + Aspects.of(self.cdk_app).add(AwsSolutionsChecks()) + + self.cdk_env = cdk.Environment( + account=account_id, + region=aws_region + ) + + def bootstrap_stack(self): + SocaBootstrapStack( + scope=self.cdk_app, + env=self.cdk_env, + stack_name=f'{self.cluster_name}-bootstrap' + ) + + def cluster_stack(self): + ClusterStack( + scope=self.cdk_app, + cluster_name=self.cluster_name, + aws_region=self.aws_region, + aws_profile=self.aws_profile, + module_id=self.module_id, + deployment_id=self.deployment_id, + termination_protection=self.termination_protection, + env=self.cdk_env + ) + + def identity_provider_stack(self): + IdentityProviderStack( + scope=self.cdk_app, + cluster_name=self.cluster_name, + aws_region=self.aws_region, + aws_profile=self.aws_profile, + module_id=self.module_id, + deployment_id=self.deployment_id, + termination_protection=self.termination_protection, + env=self.cdk_env + ) + + def directoryservice_stack(self): + DirectoryServiceStack( + scope=self.cdk_app, + cluster_name=self.cluster_name, + aws_region=self.aws_region, + aws_profile=self.aws_profile, + module_id=self.module_id, + deployment_id=self.deployment_id, + termination_protection=self.termination_protection, + env=self.cdk_env + ) + + def shared_storage_stack(self): + SharedStorageStack( + scope=self.cdk_app, + cluster_name=self.cluster_name, + aws_region=self.aws_region, + aws_profile=self.aws_profile, + module_id=self.module_id, + deployment_id=self.deployment_id, + termination_protection=self.termination_protection, + env=self.cdk_env + ) + + def cluster_manager_stack(self): + ClusterManagerStack( + scope=self.cdk_app, + cluster_name=self.cluster_name, + aws_region=self.aws_region, + aws_profile=self.aws_profile, + module_id=self.module_id, + deployment_id=self.deployment_id, + termination_protection=self.termination_protection, + env=self.cdk_env + ) + + def scheduler_stack(self): + SchedulerStack( + scope=self.cdk_app, + cluster_name=self.cluster_name, + aws_region=self.aws_region, + aws_profile=self.aws_profile, + module_id=self.module_id, + deployment_id=self.deployment_id, + termination_protection=self.termination_protection, + env=self.cdk_env + ) + + def bastion_host_stack(self): + BastionHostStack( + scope=self.cdk_app, + cluster_name=self.cluster_name, + aws_region=self.aws_region, + aws_profile=self.aws_profile, + module_id=self.module_id, + deployment_id=self.deployment_id, + termination_protection=self.termination_protection, + env=self.cdk_env + ) + + def analytics_stack(self): + AnalyticsStack( + scope=self.cdk_app, + cluster_name=self.cluster_name, + aws_region=self.aws_region, + aws_profile=self.aws_profile, + module_id=self.module_id, + deployment_id=self.deployment_id, + termination_protection=self.termination_protection, + env=self.cdk_env + ) + + def virtual_desktop_controller_stack(self): + VirtualDesktopControllerStack( + scope=self.cdk_app, + cluster_name=self.cluster_name, + aws_region=self.aws_region, + aws_profile=self.aws_profile, + module_id=self.module_id, + deployment_id=self.deployment_id, + termination_protection=self.termination_protection, + env=self.cdk_env + ) + + def metrics_stack(self): + MetricsStack( + scope=self.cdk_app, + cluster_name=self.cluster_name, + aws_region=self.aws_region, + aws_profile=self.aws_profile, + module_id=self.module_id, + deployment_id=self.deployment_id, + termination_protection=self.termination_protection, + env=self.cdk_env + ) + + def build_stack(self): + if self.module_name == constants.MODULE_BOOTSTRAP: + self.bootstrap_stack() + elif self.module_name == constants.MODULE_CLUSTER: + self.cluster_stack() + elif self.module_name == constants.MODULE_IDENTITY_PROVIDER: + self.identity_provider_stack() + elif self.module_name == constants.MODULE_DIRECTORYSERVICE: + self.directoryservice_stack() + elif self.module_name == constants.MODULE_SHARED_STORAGE: + self.shared_storage_stack() + elif self.module_name == constants.MODULE_CLUSTER_MANAGER: + self.cluster_manager_stack() + elif self.module_name == constants.MODULE_SCHEDULER: + self.scheduler_stack() + elif self.module_name == constants.MODULE_BASTION_HOST: + self.bastion_host_stack() + elif self.module_name == constants.MODULE_ANALYTICS: + self.analytics_stack() + elif self.module_name == constants.MODULE_VIRTUAL_DESKTOP_CONTROLLER: + self.virtual_desktop_controller_stack() + elif self.module_name == constants.MODULE_METRICS: + self.metrics_stack() + + def invoke(self): + self.build_stack() + self.cdk_app.synth() diff --git a/source/idea/idea-administrator/src/ideaadministrator/app/cdk/cdk_invoker.py b/source/idea/idea-administrator/src/ideaadministrator/app/cdk/cdk_invoker.py new file mode 100644 index 00000000..7bcf36f1 --- /dev/null +++ b/source/idea/idea-administrator/src/ideaadministrator/app/cdk/cdk_invoker.py @@ -0,0 +1,1005 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +import ideaadministrator + +from ideadatamodel import ( + constants, + exceptions, + CustomFileLoggerParams +) +from ideasdk.utils import Utils, Jinja2Utils +from ideasdk.shell import ShellInvoker +from ideasdk.logging import SocaLogging +from ideasdk.bootstrap import BootstrapPackageBuilder, BootstrapUtils +from ideasdk.config.cluster_config_db import ClusterConfigDB +from ideasdk.config.cluster_config import ClusterConfig +from ideasdk.context import BootstrapContext +from ideasdk.metrics.cloudwatch.cloudwatch_agent_config import CloudWatchAgentLogFileOptions +from ideasdk.aws import AwsClientProvider, AWSClientProviderOptions + +from typing import Optional, List, Dict +import os +import shutil +import logging +import arrow + + +class CdkInvoker: + """ + CDK Invoker is responsible for: + + 1. invoke the CDK App using cdk nodejs binary. + 2. manage and update cluster configurations after a stack is deployed + + CDK is natively written in TypeScript (and ultimately runs in nodejs), and provides ports for other languages. + For IDEA, CDK stacks and constructs in Python, which are synthesized to a cloud formation template that are processed by CDK nodejs binary. + + The CDK Invoker class supports CDK invocation for 2 primary use cases: + + 1. as an installed app (currently used primarily in Docker Container flows) + - in this scenario all packages are installed to python's site-packages + - this makes invocation of cdk easier, where the --app in cdk becomes:: + + cdk --app 'idea-admin cdk cdk-app --cluster-name idea-dev2 --aws-region us-east-1 --module-id bootstrap --module-name bootstrap --deployment-id 700f4b2c-110b-4395-a208-b13874d20ec5 --termination-protection true' + + - `idea-admin cdk cdk-app .. ..` synthesizes the cloudformation template and cdk uses the template to deploy the stack + + 2. from sources during development (Dev Mode) + - this flow is a bit tricky as we need to ensure all the source paths of all modules are available when `--app` parameter is passed to CDK. + - this is addressed using `invoke`, as invoke is used for all dev automation flows. + - the command to invoke cdk becomes:: + + cdk --app 'invoke cli.admin --args base64("cdk cdk-app --cluster-name idea-dev2 --aws-region us-east-1 --module-id bootstrap --module-name bootstrap --deployment-id 700f4b2c-110b-4395-a208-b13874d20ec5 --termination-protection true") + + but there are complications in this flow: + 1. invoke is only available in the dev virtual environment. to address this, the invocation os.environ is passed to the + shell invocation so that the `invoke` is available in PATH. + 2. all CDK related artifacts, logs, outputs should be generated in ~/.idea/clusters///_cdk folder. + this by design and not a requirement. primarily so that each cluster can retain it's deployment information in once place + for easier debugging and future reference. + to achieve this, the implementation changes directory (cd) to _cdk before executing: cdk --app "..." + when the directory is changed, `invoke` has no clue where to find `invoke` dev automation tasks. to address this, + a --search-root parameter is added to tell invoke where to find the project sources. + + after 1 and 2, the final invocation command becomes:: + + cdk --app 'invoke --search-root "" cli.admin --args "base64data"' + + CDK Binary: + The installation process does not want to rely on the user to install a specific version of the CDK version. rather, the admin-app + installation takes care of installing the specific CDK version to ~/.idea/lib/idea-cdk + + Developer must install the CDK binary manually to ~/.idea/lib/idea-cdk using:: + + mkdir -p ~/.idea/lib/idea-cdk && pushd ~/.idea/lib/idea-cdk + npm init --force --yes + npm install aws-cdk@{LATEST_SUPPORTED_VERSION} --save + popd + + """ + + def __init__(self, + cluster_name: str, + aws_region: str, + module_id: str, + module_set: str, + aws_profile: str = None, + deployment_id: str = None, + termination_protection: bool = True, + rollback: bool = True): + + self.cluster_name = cluster_name + self.aws_region = aws_region + self.module_id = module_id + self.aws_profile = aws_profile + self.termination_protection = termination_protection + self.rollback = rollback + + if Utils.is_empty(module_set): + module_set = constants.DEFAULT_MODULE_SET + self.module_set = module_set + + if Utils.is_empty(deployment_id): + deployment_id = Utils.uuid() + self.deployment_id = deployment_id + + # custom file logging to enable exporting deployment logs for debugging and support + self.logging = SocaLogging() + cluster_logs_dir = ideaadministrator.props.cluster_logs_dir(self.cluster_name, self.aws_region) + now = arrow.utcnow() + log_file_name = now.format('YYYY-MM-DD') + self.file_logger = self.logging.get_custom_file_logger( + params=CustomFileLoggerParams( + logger_name='cdk-invoker', + log_dir_name=cluster_logs_dir, + log_file_name=f'cdk_{log_file_name}.log', + when='midnight', + interval=1, + backupCount=10 + ), + log_level=logging.INFO, + fmt='[%(asctime)s] %(message)s' + ) + + cluster_deployments_dir = ideaadministrator.props.cluster_deployments_dir(self.cluster_name, self.aws_region) + self.deployment_dir = os.path.join(cluster_deployments_dir, self.deployment_id) + os.makedirs(self.deployment_dir, exist_ok=True) + + self.cluster_config_db = ClusterConfigDB( + cluster_name=self.cluster_name, + aws_region=self.aws_region, + aws_profile=self.aws_profile + ) + + if self.module_id == 'bootstrap': + self.module_name = 'bootstrap' + else: + module_info = self.cluster_config_db.get_module_info(self.module_id) + self.module_name = module_info['name'] + + self.cdk_home = ideaadministrator.props.cluster_cdk_dir( + cluster_name=self.cluster_name, + aws_region=self.aws_region + ) + + cdk_json = os.path.join(self.cdk_home, 'cdk.json') + if not os.path.isfile(cdk_json): + cdk_json_template = os.path.join(ideaadministrator.props.resources_dir, 'cdk', 'cdk.json') + shutil.copy(cdk_json_template, cdk_json) + + self.MODULE_MAPPING_INVOKE_MAPPING: Dict[str, ()] = { + constants.MODULE_CLUSTER: self.invoke_cluster, + constants.MODULE_SHARED_STORAGE: self.invoke_shared_storage, + constants.MODULE_IDENTITY_PROVIDER: self.invoke_identity_provider, + constants.MODULE_DIRECTORYSERVICE: self.invoke_directoryservice, + constants.MODULE_CLUSTER_MANAGER: self.invoke_cluster_manager, + constants.MODULE_SCHEDULER: self.invoke_scheduler, + constants.MODULE_BASTION_HOST: self.invoke_bastion_host, + constants.MODULE_VIRTUAL_DESKTOP_CONTROLLER: self.invoke_virtual_desktop_controller, + constants.MODULE_ANALYTICS: self.invoke_analytics, + constants.MODULE_METRICS: self.invoke_metrics + } + + def log(self, message: str): + """ + use this method to log additional information for debugging to file + do NOT use or update this method to print() to console to keep the console UX as simple and clean as possible. + :param message: + """ + self.file_logger.info(f'(DeploymentId: {self.deployment_id}) {message}') + + def log_invocation_context(self): + self.log(f'[InvocationContext] ClusterName: {self.cluster_name}, ' + f'Region: {self.aws_region}, ' + f'Profile: {self.aws_profile}, ' + f'Rollback: {self.rollback}, ' + f'ModuleName: {self.module_name}, ' + f'ModuleId: {self.module_id}') + + def exec_shell(self, shell_cmd: str, silent=False, **_): + + def callback(line): + + line = str(line).rstrip() + if Utils.is_not_empty(line): + self.log(message=line) + + if silent: + return + + print(line, flush=True) + + shell = ShellInvoker(cwd=self.cdk_home) + + env = ideaadministrator.props.get_env() + # to ensure cdk nodejs lib does not use some other aws profile, + # override AWS_DEFAULT_PROFILE to the profile given in params + # this resolves the below installation error: + # Environment aws:///us-east-1 failed bootstrapping: Error: Need to perform AWS calls for account target-account, but the current credentials are for + if Utils.is_not_empty(self.aws_profile): + env['AWS_DEFAULT_PROFILE'] = self.aws_profile + + env_trace = {} + for env_key, env_value in env.items(): + # skip sensitive values + if not env_key.startswith(('IDEA', 'AWS')): + continue + env_trace[env_key] = env_value + + self.log(f'Env: {Utils.to_json(env_trace)}') + self.log(f'shell> {shell_cmd}') + + process = shell.invoke_stream( + cmd=shell_cmd, + callback=callback, + shell=True, + env=env, + start_new_session=True + ) + + try: + return_code = process.start_streaming() + except KeyboardInterrupt: + self.log('KeyboardInterrupt') + process.send_stop_signal() + return_code = process.wait() + + self.log(f'ExitCode: {return_code}') + + if return_code != 0: + raise SystemExit(return_code) + + def get_cdk_app_cmd(self) -> str: + """ + Returns the command to build the CDK app + when dev_mode is true, invoke automation is used to ensure sources instead of calling idea-admin + :return: + """ + cdk_app_args = [ + '--cluster-name', + self.cluster_name, + '--aws-region', + self.aws_region, + '--module-id', + self.module_id, + '--module-name', + self.module_name, + '--deployment-id', + self.deployment_id, + '--termination-protection', + str(self.termination_protection).lower() + ] + + # bug fix for below error + # Environment aws:///us-east-1 failed bootstrapping: Error: Need to perform AWS calls for account target-account, but the current credentials are for + if Utils.is_not_empty(self.aws_profile): + cdk_app_args += [ + '--aws-profile', + self.aws_profile + ] + + if ideaadministrator.props.is_dev_mode(): + args = ['cdk', 'cdk-app'] + cdk_app_args + args_encoded = Utils.base64_encode(Utils.to_json(args)) + + cdk_app_cmd = f'invoke ' \ + f'--search-root "{ideaadministrator.props.dev_mode_project_root_dir}" ' \ + f'cli.admin --args {args_encoded}' + + args_trace = ' '.join(args) + cdk_app_cmd_trace = f'invoke ' \ + f'--search-root "{ideaadministrator.props.dev_mode_project_root_dir}" ' \ + f'cli.admin --args base64([{args_trace}])' + self.log(f'CDKApp (DevMode): {cdk_app_cmd_trace}') + else: + cdk_app_args_s = ' '.join(cdk_app_args) + cdk_app_cmd = f'idea-admin cdk cdk-app {cdk_app_args_s}' + self.log(f'CDKApp: {cdk_app_cmd}') + + return cdk_app_cmd + + def setup_cluster_cdk_dir(self): + cluster_cdk_dir = ideaadministrator.props.cluster_cdk_dir( + cluster_name=self.cluster_name, + aws_region=self.aws_region + ) + cdk_json = os.path.join(cluster_cdk_dir, 'cdk.json') + if not os.path.isfile(cdk_json): + cdk_json_template = os.path.join(ideaadministrator.props.resources_dir, 'cdk', 'cdk.json') + shutil.copy(cdk_json_template, cdk_json) + + def get_cdk_command(self, name: str, params: Optional[List[str]] = None, context_params: Dict[str, str] = None) -> str: + """ + build the cdk command. eg: + $ cdk deploy --app "idea-admin cdk-app -c user-config.json [--params] [-c [context_param_key]=[context_param_value]] + """ + + self.setup_cluster_cdk_dir() + + if params is None: + params = [] + cmd = [ideaadministrator.props.cdk_bin] + name.split(' ') + params + if name == 'deploy': + cmd.append(f'--rollback {str(self.rollback).lower()}') + if Utils.is_not_empty(self.aws_profile): + cmd.append(f'--profile {self.aws_profile}') + if context_params is not None: + for key, value in context_params.items(): + cmd.append(f'-c {key}={value}') + cmd.append(f'--change-set-name idea-{Utils.uuid()}') + return ' '.join(cmd) + + def cdk_synth(self): + try: + self.log('CdkInvoker: Begin Synth') + self.log_invocation_context() + cdk_app_cmd = self.get_cdk_app_cmd() + cdk_cmd = self.get_cdk_command( + 'synth', + params=[ + f"--app '{cdk_app_cmd}'" + ]) + self.exec_shell(cdk_cmd, print_cmd=False) + finally: + self.log('CdkInvoker: End Synth') + + def cdk_diff(self): + try: + self.log('CdkInvoker: Begin Diff') + self.log_invocation_context() + cdk_app_cmd = self.get_cdk_app_cmd() + cdk_cmd = self.get_cdk_command( + 'diff', + params=[ + f"--app '{cdk_app_cmd}'" + ]) + self.exec_shell(cdk_cmd, print_cmd=False) + finally: + self.log('CdkInvoker: End Diff') + + def cdk_destroy(self): + try: + self.log('CdkInvoker: Begin Destroy') + self.log_invocation_context() + cdk_app_cmd = self.get_cdk_app_cmd() + cdk_cmd = self.get_cdk_command( + 'destroy', + params=[ + f"--app '{cdk_app_cmd}'" + ]) + self.exec_shell(cdk_cmd, print_cmd=True) + finally: + self.log('CdkInvoker: End Destroy') + + def build_and_upload_bootstrap_package(self, bootstrap_context: BootstrapContext, + bootstrap_package_basename: str, + bootstrap_components: List[str], + force_build=False, + upload=True) -> str: + """ + render the bootstrap package and upload to the cluster's s3 bucket. + returns the S3 uri of the uploaded bootstrap package + """ + + if ideaadministrator.props.is_dev_mode(): + bootstrap_source_dir = ideaadministrator.props.dev_mode_bootstrap_source_dir + else: + bootstrap_source_dir = os.path.join(ideaadministrator.props.resources_dir, 'bootstrap') + + builder = BootstrapPackageBuilder( + bootstrap_context=bootstrap_context, + source_directory=bootstrap_source_dir, + target_package_basename=bootstrap_package_basename, + components=bootstrap_components, + tmp_dir=self.deployment_dir, + force_build=force_build + ) + bootstrap_package_archive_file = builder.build() + session = Utils.create_boto_session(self.aws_region, self.aws_profile) + s3_client = session.client('s3') + + cluster_s3_bucket = bootstrap_context.config.get_string('cluster.cluster_s3_bucket', required=True) + bootstrap_package_uri = f's3://{cluster_s3_bucket}/idea/bootstrap/{os.path.basename(bootstrap_package_archive_file)}' + + if upload: + print(f'uploading bootstrap package {bootstrap_package_uri} ...') + s3_client.upload_file( + Bucket=cluster_s3_bucket, + Filename=bootstrap_package_archive_file, + Key=f'idea/bootstrap/{os.path.basename(bootstrap_package_archive_file)}' + ) + return bootstrap_package_uri + + def upload_release_package(self, bootstrap_context: BootstrapContext, package_name: str, upload=True) -> str: + if ideaadministrator.props.is_dev_mode(): + package_dist_dir = ideaadministrator.props.dev_mode_project_dist_dir + else: + package_dist_dir = ideaadministrator.props.soca_downloads_dir + + app_package = os.path.join(package_dist_dir, package_name) + if not Utils.is_file(app_package): + raise exceptions.general_exception(f'package not found: {app_package}') + + cluster_s3_bucket = bootstrap_context.config.get_string('cluster.cluster_s3_bucket', required=True) + app_package_uri = f's3://{cluster_s3_bucket}/idea/releases/{package_name}' + + if upload: + aws_client = AwsClientProvider(options=AWSClientProviderOptions(profile=self.aws_profile, region=self.aws_region)) + print(f'uploading release package: {app_package_uri} ...') + aws_client.s3().upload_file( + Bucket=cluster_s3_bucket, + Filename=app_package, + Key=f'idea/releases/{os.path.basename(app_package)}' + ) + + return app_package_uri + + def bootstrap_cluster(self, cluster_bucket: str): + try: + self.log('CdkInvoker: Bootstrap Begin') + self.log_invocation_context() + + # render the cdk toolkit stack in cdk home + toolkit_stack_target_file = os.path.join(self.cdk_home, 'cdk_toolkit_stack.yml') + aws_client = AwsClientProvider(options=AWSClientProviderOptions(profile=self.aws_profile, region=self.aws_region)) + cdk_resources_dir = os.path.join(ideaadministrator.props.resources_dir, 'cdk') + env = Jinja2Utils.env_using_file_system_loader(search_path=cdk_resources_dir) + toolkit_stack_template = env.get_template('cdk_toolkit_stack.yml') + cluster_config = ClusterConfig( + cluster_name=self.cluster_name, + aws_region=self.aws_region, + aws_profile=self.aws_profile, + module_id=constants.MODULE_CLUSTER, + module_set=self.module_set + ) + + # find the elb account id for the current region + with open(ideaadministrator.props.region_elb_account_id_file(), 'r') as f: + region_elb_account_id_config = Utils.from_yaml(f.read()) + + elb_account_id = Utils.get_value_as_string(self.aws_region, region_elb_account_id_config) + + toolkit_stack_content = toolkit_stack_template.render(**{ + 'cluster_name': self.cluster_name, + 'aws_dns_suffix': aws_client.aws_dns_suffix(), + 'cluster_s3_bucket': cluster_config.get_string('cluster.cluster_s3_bucket', required=True), + 'config': cluster_config, + 'aws_elb_account_id': elb_account_id + }) + with open(toolkit_stack_target_file, 'w') as f: + f.write(toolkit_stack_content) + print(f'rendered cdk toolkit stack template for cluster: {self.cluster_name}, template: {toolkit_stack_target_file}') + + stack_name = f'{self.cluster_name}-bootstrap' + cdk_app_command = self.get_cdk_app_cmd() + cdk_toolkit_qualifier = Utils.shake_256(self.cluster_name, 5) + cmd = [ + ideaadministrator.props.cdk_bin, + 'bootstrap', + f"--app '{cdk_app_command}' ", + f'--bootstrap-bucket-name {cluster_bucket} ' + f'--toolkit-stack-name {stack_name} ' + f'--termination-protection {str(self.termination_protection).lower()} ' + f'--qualifier {cdk_toolkit_qualifier}', + f'--template {toolkit_stack_target_file}' + ] + + # bootstrap stack tags + tags = { + constants.IDEA_TAG_CLUSTER_NAME: self.cluster_name + } + custom_tags = cluster_config.get_list('global-settings.custom_tags', []) + custom_tags_dict = Utils.convert_custom_tags_to_key_value_pairs(custom_tags) + tags = {**custom_tags_dict, **tags} + for tag_key, tag_value in tags.items(): + cmd.append(f'--tags "{tag_key}={tag_value}"') + + if Utils.is_not_empty(self.aws_profile): + cmd.append(f'--profile {self.aws_profile}') + + bootstrap_cmd = ' '.join(cmd) + self.exec_shell(bootstrap_cmd) + finally: + self.log('CdkInvoker: Bootstrap End') + + def invoke_cluster(self, **_): + outputs_file = os.path.join(self.deployment_dir, 'cluster-outputs.json') + cdk_app_cmd = self.get_cdk_app_cmd() + cdk_cmd = self.get_cdk_command('deploy', [ + f"--app '{cdk_app_cmd}' ", + f'--outputs-file {outputs_file}', + f'--require-approval never' + ]) + self.exec_shell(cdk_cmd) + + def invoke_shared_storage(self, **_): + outputs_file = os.path.join(self.deployment_dir, 'shared-storage-outputs.json') + cdk_app_cmd = self.get_cdk_app_cmd() + cdk_cmd = self.get_cdk_command('deploy', [ + f"--app '{cdk_app_cmd}' ", + f'--outputs-file {outputs_file}', + f'--require-approval never' + ]) + self.exec_shell(cdk_cmd) + + def invoke_analytics(self, **_): + outputs_file = os.path.join(self.deployment_dir, 'analytics-outputs.json') + cdk_app_cmd = self.get_cdk_app_cmd() + cdk_cmd = self.get_cdk_command(f'deploy', [ + f"--app '{cdk_app_cmd}' ", + f'--outputs-file {outputs_file} ', + f'--require-approval never' + ]) + self.exec_shell(cdk_cmd) + + def invoke_metrics(self, **_): + outputs_file = os.path.join(self.deployment_dir, 'metrics-outputs.json') + cdk_app_cmd = self.get_cdk_app_cmd() + cdk_cmd = self.get_cdk_command('deploy', [ + f"--app '{cdk_app_cmd}' ", + f'--outputs-file {outputs_file}', + f'--require-approval never' + ]) + self.exec_shell(cdk_cmd) + + def invoke_identity_provider(self, **_): + outputs_file = os.path.join(self.deployment_dir, 'identity-provider-outputs.json') + cdk_app_cmd = self.get_cdk_app_cmd() + cdk_cmd = self.get_cdk_command('deploy', [ + f"--app '{cdk_app_cmd}' ", + f'--outputs-file {outputs_file}', + f'--require-approval never' + ]) + self.exec_shell(cdk_cmd) + + def invoke_directoryservice(self, **kwargs): + modules = self.cluster_config_db.get_cluster_modules() + for module in modules: + module_name = module['name'] + module_id = module['module_id'] + status = module['status'] + if module_name == constants.MODULE_CLUSTER and status == 'not-deployed': + raise exceptions.general_exception(f'cannot deploy {self.module_id}. module: {module_id} is not yet deployed.') + + cluster_config = ClusterConfig( + cluster_name=self.cluster_name, + aws_region=self.aws_region, + aws_profile=self.aws_profile, + module_id=self.module_id, + module_set=self.module_set + ) + + deploy_stack = Utils.get_value_as_bool('deploy_stack', kwargs, True) + provider = cluster_config.get_string('directoryservice.provider', required=True) + + if provider == constants.DIRECTORYSERVICE_OPENLDAP: + + render_bootstrap_package = Utils.get_value_as_bool('render_bootstrap_package', kwargs, True) + force_build_bootstrap = Utils.get_value_as_bool('force_build_bootstrap', kwargs, True) + upload_bootstrap_package = Utils.get_value_as_bool('upload_bootstrap_package', kwargs, True) + + base_os = cluster_config.get_string('directoryservice.base_os', required=True) + instance_type = cluster_config.get_string('directoryservice.instance_type', required=True) + bootstrap_context = BootstrapContext( + config=cluster_config, + module_name=self.module_name, + module_id=self.module_id, + module_set=self.module_set, + base_os=base_os, + instance_type=instance_type + ) + + BootstrapUtils.check_and_attach_cloudwatch_logging_and_metrics( + bootstrap_context=bootstrap_context, + metrics_namespace=f'{self.cluster_name}/{self.module_id}/openldap-server', + node_type=constants.NODE_TYPE_INFRA, + enable_logging=False, + log_files=[] + ) + + bootstrap_package_uri = None + if render_bootstrap_package or upload_bootstrap_package: + bootstrap_package_uri = self.build_and_upload_bootstrap_package( + bootstrap_context=bootstrap_context, + bootstrap_package_basename=f'bootstrap-{self.module_id}-{self.deployment_id}', + bootstrap_components=[ + 'common', + 'openldap-server' + ], + upload=upload_bootstrap_package, + force_build=force_build_bootstrap + ) + if upload_bootstrap_package and deploy_stack: + outputs_file = os.path.join(self.deployment_dir, f'directoryservice-outputs.json') + cdk_app_cmd = self.get_cdk_app_cmd() + cdk_cmd = self.get_cdk_command(f'deploy', params=[ + f"--app '{cdk_app_cmd}' ", + f'--outputs-file {outputs_file} ', + f'--require-approval never' + ], context_params={ + 'bootstrap_package_uri': bootstrap_package_uri + }) + self.exec_shell(cdk_cmd) + + else: + if deploy_stack: + outputs_file = os.path.join(self.deployment_dir, f'directoryservice-outputs.json') + cdk_app_cmd = self.get_cdk_app_cmd() + cdk_cmd = self.get_cdk_command(f'deploy', params=[ + f"--app '{cdk_app_cmd}' ", + f'--outputs-file {outputs_file} ', + f'--require-approval never' + ]) + self.exec_shell(cdk_cmd) + + def invoke_cluster_manager(self, **kwargs): + upload_release_package = Utils.get_value_as_bool('upload_release_package', kwargs, True) + render_bootstrap_package = Utils.get_value_as_bool('render_bootstrap_package', kwargs, True) + force_build_bootstrap = Utils.get_value_as_bool('force_build_bootstrap', kwargs, True) + upload_bootstrap_package = Utils.get_value_as_bool('upload_bootstrap_package', kwargs, True) + deploy_stack = Utils.get_value_as_bool('deploy_stack', kwargs, True) + + cluster_config = ClusterConfig( + cluster_name=self.cluster_name, + aws_region=self.aws_region, + aws_profile=self.aws_profile, + module_id=self.module_id, + module_set=self.module_set + ) + + base_os = cluster_config.get_string('cluster-manager.ec2.autoscaling.base_os', required=True) + instance_type = cluster_config.get_string('cluster-manager.ec2.autoscaling.instance_type', required=True) + bootstrap_context = BootstrapContext( + config=cluster_config, + module_name=self.module_name, + module_id=self.module_id, + module_set=self.module_set, + base_os=base_os, + instance_type=instance_type + ) + + app_package_uri = self.upload_release_package( + bootstrap_context=bootstrap_context, + package_name=f'idea-cluster-manager-{ideaadministrator.props.current_release_version}.tar.gz', + upload=upload_release_package + ) + bootstrap_context.vars.app_package_uri = app_package_uri + + BootstrapUtils.check_and_attach_cloudwatch_logging_and_metrics( + bootstrap_context=bootstrap_context, + metrics_namespace=f'{self.cluster_name}/{self.module_id}', + node_type=constants.NODE_TYPE_APP, + enable_logging=cluster_config.get_bool('cluster-manager.cloudwatch_logs.enabled', False), + log_files=[ + CloudWatchAgentLogFileOptions( + file_path='/opt/idea/app/logs/**.log', + log_group_name=f'/{self.cluster_name}/{self.module_id}', + log_stream_name='application_{ip_address}' + ) + ] + ) + + bootstrap_package_uri = None + if render_bootstrap_package or upload_bootstrap_package: + bootstrap_package_uri = self.build_and_upload_bootstrap_package( + bootstrap_context=bootstrap_context, + bootstrap_package_basename=f'bootstrap-{self.module_id}-{self.deployment_id}', + bootstrap_components=[ + 'cluster-manager' + ], + upload=upload_bootstrap_package, + force_build=force_build_bootstrap + ) + + if upload_release_package and upload_bootstrap_package and deploy_stack: + outputs_file = os.path.join(self.deployment_dir, f'cluster-manager-outputs.json') + cdk_app_cmd = self.get_cdk_app_cmd() + cdk_cmd = self.get_cdk_command(f'deploy', params=[ + f"--app '{cdk_app_cmd}' ", + f'--outputs-file {outputs_file} ', + f'--require-approval never' + ], context_params={ + 'bootstrap_package_uri': bootstrap_package_uri + }) + self.exec_shell(cdk_cmd) + + def invoke_scheduler(self, **kwargs): + upload_release_package = Utils.get_value_as_bool('upload_release_package', kwargs, True) + render_bootstrap_package = Utils.get_value_as_bool('render_bootstrap_package', kwargs, True) + force_build_bootstrap = Utils.get_value_as_bool('force_build_bootstrap', kwargs, True) + upload_bootstrap_package = Utils.get_value_as_bool('upload_bootstrap_package', kwargs, True) + deploy_stack = Utils.get_value_as_bool('deploy_stack', kwargs, True) + + cluster_config = ClusterConfig( + cluster_name=self.cluster_name, + aws_region=self.aws_region, + aws_profile=self.aws_profile, + module_id=self.module_id, + module_set=self.module_set + ) + + base_os = cluster_config.get_string('scheduler.base_os', required=True) + instance_type = cluster_config.get_string('scheduler.instance_type', required=True) + bootstrap_context = BootstrapContext( + config=cluster_config, + module_name=self.module_name, + module_id=self.module_id, + module_set=self.module_set, + base_os=base_os, + instance_type=instance_type + ) + + app_package_uri = self.upload_release_package( + bootstrap_context=bootstrap_context, + package_name=f'idea-scheduler-{ideaadministrator.props.current_release_version}.tar.gz', + upload=upload_release_package + ) + bootstrap_context.vars.app_package_uri = app_package_uri + + log_files = [ + CloudWatchAgentLogFileOptions( + file_path='/opt/idea/app/logs/**.log', + log_group_name=f'/{self.cluster_name}/{self.module_id}', + log_stream_name='application_{ip_address}' + ) + ] + if cluster_config.get_string('scheduler.provider', required=True) == constants.SCHEDULER_OPENPBS: + log_files += [ + CloudWatchAgentLogFileOptions( + file_path='/var/spool/pbs/server_logs/**.log', + log_group_name=f'/{self.cluster_name}/{self.module_id}/openpbs', + log_stream_name='server_logs_{ip_address}' + ), + CloudWatchAgentLogFileOptions( + file_path='/var/spool/pbs/sched_logs/**.log', + log_group_name=f'/{self.cluster_name}/{self.module_id}/openpbs', + log_stream_name='sched_logs_{ip_address}' + ), + CloudWatchAgentLogFileOptions( + file_path='/var/spool/pbs/server_priv/accounting/**.log', + log_group_name=f'/{self.cluster_name}/{self.module_id}/openpbs', + log_stream_name='accounting_logs_{ip_address}' + ) + ] + BootstrapUtils.check_and_attach_cloudwatch_logging_and_metrics( + bootstrap_context=bootstrap_context, + metrics_namespace=f'{self.cluster_name}/{self.module_id}', + node_type=constants.NODE_TYPE_APP, + enable_logging=cluster_config.get_bool('scheduler.cloudwatch_logs.enabled', False), + log_files=log_files + ) + + bootstrap_package_uri = None + if render_bootstrap_package or upload_bootstrap_package: + bootstrap_package_uri = self.build_and_upload_bootstrap_package( + bootstrap_context=bootstrap_context, + bootstrap_package_basename=f'bootstrap-{self.module_id}-{self.deployment_id}', + bootstrap_components=[ + 'scheduler' + ], + upload=upload_bootstrap_package, + force_build=force_build_bootstrap + ) + + if upload_release_package and upload_bootstrap_package and deploy_stack: + outputs_file = os.path.join(self.deployment_dir, f'scheduler-outputs.json') + cdk_app_cmd = self.get_cdk_app_cmd() + cdk_cmd = self.get_cdk_command(f'deploy', params=[ + f"--app '{cdk_app_cmd}' ", + f'--outputs-file {outputs_file} ', + f'--require-approval never' + ], context_params={ + 'bootstrap_package_uri': bootstrap_package_uri + }) + self.exec_shell(cdk_cmd) + + def invoke_virtual_desktop_controller(self, **kwargs): + upload_release_package = Utils.get_value_as_bool('upload_release_package', kwargs, True) + render_bootstrap_package = Utils.get_value_as_bool('render_bootstrap_package', kwargs, True) + force_build_bootstrap = Utils.get_value_as_bool('force_build_bootstrap', kwargs, True) + upload_bootstrap_package = Utils.get_value_as_bool('upload_bootstrap_package', kwargs, True) + deploy_stack = Utils.get_value_as_bool('deploy_stack', kwargs, True) + + cluster_config = ClusterConfig( + cluster_name=self.cluster_name, + aws_region=self.aws_region, + aws_profile=self.aws_profile, + module_id=self.module_id, + module_set=self.module_set + ) + + # controller + controller_bootstrap_context = BootstrapContext( + config=cluster_config, + module_name=self.module_name, + module_id=self.module_id, + module_set=self.module_set, + base_os=cluster_config.get_string('virtual-desktop-controller.controller.autoscaling.base_os', required=True), + instance_type=cluster_config.get_string('virtual-desktop-controller.controller.autoscaling.instance_type', required=True) + ) + app_package_uri = self.upload_release_package( + bootstrap_context=controller_bootstrap_context, + package_name=f'idea-virtual-desktop-controller-{ideaadministrator.props.current_release_version}.tar.gz', + upload=upload_release_package + ) + controller_bootstrap_context.vars.controller_package_uri = app_package_uri + BootstrapUtils.check_and_attach_cloudwatch_logging_and_metrics( + bootstrap_context=controller_bootstrap_context, + metrics_namespace=f'{self.cluster_name}/{self.module_id}/controller', + node_type=constants.NODE_TYPE_APP, + enable_logging=cluster_config.get_bool('virtual-desktop-controller.cloudwatch_logs.enabled', False), + log_files=[ + CloudWatchAgentLogFileOptions( + file_path='/opt/idea/app/logs/**.log', + log_group_name=f'/{self.cluster_name}/{self.module_id}/controller', + log_stream_name='application_{ip_address}' + ) + ] + ) + + # dcv broker + broker_bootstrap_context = BootstrapContext( + config=cluster_config, + module_name=self.module_name, + module_id=self.module_id, + module_set=self.module_set, + base_os=cluster_config.get_string('virtual-desktop-controller.dcv_broker.autoscaling.base_os', required=True), + instance_type=cluster_config.get_string('virtual-desktop-controller.dcv_broker.autoscaling.instance_type', required=True) + ) + + BootstrapUtils.check_and_attach_cloudwatch_logging_and_metrics( + bootstrap_context=broker_bootstrap_context, + metrics_namespace=f'{self.cluster_name}/{self.module_id}/dcv-broker', + node_type=constants.NODE_TYPE_INFRA, + enable_logging=cluster_config.get_bool('virtual-desktop-controller.cloudwatch_logs.enabled', False), + log_files=[ + CloudWatchAgentLogFileOptions( + file_path='/var/log/dcv-session-manager-broker/**.log', + log_group_name=f'/{self.cluster_name}/{self.module_id}/dcv-broker', + log_stream_name='dcv-session-manager-broker_{ip_address}' + ) + ] + ) + + # dcv connection gateway + dcv_connection_gateway_bootstrap_context = BootstrapContext( + config=cluster_config, + module_name=self.module_name, + module_id=self.module_id, + module_set=self.module_set, + base_os=cluster_config.get_string('virtual-desktop-controller.dcv_connection_gateway.autoscaling.base_os', required=True), + instance_type=cluster_config.get_string('virtual-desktop-controller.dcv_connection_gateway.autoscaling.instance_type', required=True) + ) + dcv_connection_gateway_uri = self.upload_release_package( + bootstrap_context=dcv_connection_gateway_bootstrap_context, + package_name=f'idea-dcv-connection-gateway-{ideaadministrator.props.current_release_version}.tar.gz', + upload=upload_release_package + ) + dcv_connection_gateway_bootstrap_context.vars.dcv_connection_gateway_package_uri = dcv_connection_gateway_uri + BootstrapUtils.check_and_attach_cloudwatch_logging_and_metrics( + bootstrap_context=dcv_connection_gateway_bootstrap_context, + metrics_namespace=f'{self.cluster_name}/{self.module_id}/dcv-connection-gateway', + node_type=constants.NODE_TYPE_INFRA, + enable_logging=cluster_config.get_bool('virtual-desktop-controller.cloudwatch_logs.enabled', False), + log_files=[ + CloudWatchAgentLogFileOptions( + file_path='/var/log/dcv-connection-gateway/**.log', + log_group_name=f'/{self.cluster_name}/{self.module_id}/dcv-connection-gateway', + log_stream_name='dcv-connection-gateway_{ip_address}' + ) + ] + ) + + controller_bootstrap_package_uri = None + dcv_broker_package_uri = None + dcv_connection_gateway_package_uri = None + if render_bootstrap_package or upload_bootstrap_package: + controller_bootstrap_package_uri = self.build_and_upload_bootstrap_package( + bootstrap_context=controller_bootstrap_context, + bootstrap_package_basename=f'bootstrap-{self.module_id}-controller-{self.deployment_id}', + bootstrap_components=[ + 'virtual-desktop-controller' + ], + upload=upload_bootstrap_package, + force_build=force_build_bootstrap + ) + dcv_broker_package_uri = self.build_and_upload_bootstrap_package( + bootstrap_context=broker_bootstrap_context, + bootstrap_package_basename=f'bootstrap-{self.module_id}-dcv-broker-{self.deployment_id}', + bootstrap_components=[ + 'dcv-broker' + ], + upload=upload_bootstrap_package, + force_build=force_build_bootstrap + ) + dcv_connection_gateway_package_uri = self.build_and_upload_bootstrap_package( + bootstrap_context=dcv_connection_gateway_bootstrap_context, + bootstrap_package_basename=f'bootstrap-{self.module_id}-dcv-connection-gateway-{self.deployment_id}', + bootstrap_components=[ + 'dcv-connection-gateway' + ], + upload=upload_bootstrap_package, + force_build=force_build_bootstrap + ) + + if upload_release_package and upload_bootstrap_package and deploy_stack: + outputs_file = os.path.join(self.deployment_dir, f'virtual-desktop-controller-outputs.json') + cdk_app_cmd = self.get_cdk_app_cmd() + cdk_cmd = self.get_cdk_command(f'deploy', params=[ + f"--app '{cdk_app_cmd}' ", + f'--outputs-file {outputs_file} ', + f'--require-approval never' + ], context_params={ + 'controller_bootstrap_package_uri': controller_bootstrap_package_uri, + 'dcv_broker_bootstrap_package_uri': dcv_broker_package_uri, + 'dcv_connection_gateway_package_uri': dcv_connection_gateway_package_uri + }) + self.exec_shell(cdk_cmd) + + def invoke_bastion_host(self, **kwargs): + modules = self.cluster_config_db.get_cluster_modules() + for module in modules: + module_name = module['name'] + module_id = module['module_id'] + status = module['status'] + if module_name == constants.MODULE_SCHEDULER and status == 'not-deployed': + raise exceptions.general_exception(f'cannot deploy {self.module_id}. module: {module_id} is not yet deployed.') + + render_bootstrap_package = Utils.get_value_as_bool('render_bootstrap_package', kwargs, True) + force_build_bootstrap = Utils.get_value_as_bool('force_build_bootstrap', kwargs, True) + upload_bootstrap_package = Utils.get_value_as_bool('upload_bootstrap_package', kwargs, True) + deploy_stack = Utils.get_value_as_bool('deploy_stack', kwargs, True) + + cluster_config = ClusterConfig( + cluster_name=self.cluster_name, + aws_region=self.aws_region, + aws_profile=self.aws_profile, + module_id=self.module_id, + module_set=self.module_set + ) + + base_os = cluster_config.get_string('bastion-host.base_os', required=True) + instance_type = cluster_config.get_string('bastion-host.instance_type', required=True) + bootstrap_context = BootstrapContext( + config=cluster_config, + module_name=self.module_name, + module_id=self.module_id, + module_set=self.module_set, + base_os=base_os, + instance_type=instance_type + ) + BootstrapUtils.check_and_attach_cloudwatch_logging_and_metrics( + bootstrap_context=bootstrap_context, + metrics_namespace=f'{self.cluster_name}/{self.module_id}', + node_type=constants.NODE_TYPE_INFRA, + enable_logging=False, + log_files=[] + ) + + bootstrap_package_uri = None + if render_bootstrap_package or upload_bootstrap_package: + bootstrap_package_uri = self.build_and_upload_bootstrap_package( + bootstrap_context=bootstrap_context, + bootstrap_package_basename=f'bootstrap-{self.module_id}-{self.deployment_id}', + bootstrap_components=[ + 'common', + 'bastion-host' + ], + upload=upload_bootstrap_package, + force_build=force_build_bootstrap + ) + + if upload_bootstrap_package and deploy_stack: + outputs_file = os.path.join(self.deployment_dir, f'bastion-host-outputs.json') + cdk_app_cmd = self.get_cdk_app_cmd() + cdk_cmd = self.get_cdk_command(f'deploy', params=[ + f"--app '{cdk_app_cmd}' ", + f'--outputs-file {outputs_file} ', + f'--require-approval never' + ], context_params={ + 'bootstrap_package_uri': bootstrap_package_uri + }) + self.exec_shell(cdk_cmd) + + def invoke(self, **kwargs): + + try: + self.log('CdkInvoker: Begin') + self.log_invocation_context() + + if self.module_name not in self.MODULE_MAPPING_INVOKE_MAPPING: + self.log(f'module name not found: {self.module_name}') + return + + self.MODULE_MAPPING_INVOKE_MAPPING[self.module_name](**kwargs) + finally: + self.log('CdkInvoker: End') diff --git a/source/idea/idea-administrator/src/ideaadministrator/app/cdk/constructs/__init__.py b/source/idea/idea-administrator/src/ideaadministrator/app/cdk/constructs/__init__.py new file mode 100644 index 00000000..b3e1fc37 --- /dev/null +++ b/source/idea/idea-administrator/src/ideaadministrator/app/cdk/constructs/__init__.py @@ -0,0 +1,20 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +from .base import * +from .common import * +from .backup import BackupPlan +from .existing_resources import * +from .network import * +from .dns import * +from .directory_service import * +from .storage import * +from .analytics import OpenSearch diff --git a/source/idea/idea-administrator/src/ideaadministrator/app/cdk/constructs/analytics.py b/source/idea/idea-administrator/src/ideaadministrator/app/cdk/constructs/analytics.py new file mode 100644 index 00000000..4fea0b91 --- /dev/null +++ b/source/idea/idea-administrator/src/ideaadministrator/app/cdk/constructs/analytics.py @@ -0,0 +1,177 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +__all__ = ('OpenSearch') # noqa + +from ideaadministrator.app.cdk.constructs import SocaBaseConstruct, ExistingSocaCluster, IdeaNagSuppression +from ideasdk.context import ArnBuilder +from ideaadministrator.app_context import AdministratorContext +from ideasdk.utils import Utils + +from typing import List, Dict, Optional + +import aws_cdk as cdk +import constructs +from aws_cdk import ( + aws_ec2 as ec2, + aws_iam as iam, + aws_opensearchservice as opensearch +) + + +class OpenSearch(SocaBaseConstruct, opensearch.Domain): + + def __init__( + self, context: AdministratorContext, name: str, scope: constructs.Construct, + cluster: ExistingSocaCluster, + security_groups: List[ec2.ISecurityGroup], + data_nodes: int, + data_node_instance_type: str, + ebs_volume_size: int, + removal_policy: cdk.RemovalPolicy, + version: Optional[opensearch.EngineVersion] = None, + create_service_linked_role: bool = True, + access_policies: Optional[List[iam.PolicyStatement]] = None, + advanced_options: Optional[Dict[str, str]] = None, + automated_snapshot_start_hour: Optional[int] = None, + capacity: Optional[opensearch.CapacityConfig] = None, + cognito_dashboards_auth: Optional[opensearch.CognitoOptions] = None, + custom_endpoint: Optional[opensearch.CustomEndpointOptions] = None, + domain_name: Optional[str] = None, + ebs: Optional[opensearch.EbsOptions] = None, + enable_version_upgrade: Optional[bool] = None, + encryption_at_rest: Optional[opensearch.EncryptionAtRestOptions] = None, + enforce_https: Optional[bool] = None, + fine_grained_access_control: Optional[opensearch.AdvancedSecurityOptions] = None, + logging: Optional[opensearch.LoggingOptions] = None, + node_to_node_encryption: Optional[bool] = None, + tls_security_policy: Optional[opensearch.TLSSecurityPolicy] = None, + use_unsigned_basic_auth: Optional[bool] = None, + vpc_subnets: Optional[List[ec2.SubnetSelection]] = None, + zone_awareness: Optional[opensearch.ZoneAwarenessConfig] = None + ): + + self.context = context + + if version is None: + version = opensearch.EngineVersion.OPENSEARCH_2_3 + + if vpc_subnets is None: + vpc_subnets = [ec2.SubnetSelection( + subnets=cluster.private_subnets[0:data_nodes] + )] + + if zone_awareness is None: + if data_nodes > 1: + zone_awareness = opensearch.ZoneAwarenessConfig( + enabled=True, + availability_zone_count=min(3, data_nodes) + ) + else: + zone_awareness = opensearch.ZoneAwarenessConfig( + enabled=False + ) + + if domain_name is None: + domain_name = self.build_resource_name(name).lower() + + if enforce_https is None: + enforce_https = True + + if encryption_at_rest is None: + encryption_at_rest = opensearch.EncryptionAtRestOptions( + enabled=True + ) + + if ebs is None: + ebs = opensearch.EbsOptions( + volume_size=ebs_volume_size, + volume_type=ec2.EbsDeviceVolumeType.GP3 + ) + + if capacity is None: + capacity = opensearch.CapacityConfig( + data_node_instance_type=data_node_instance_type, + data_nodes=data_nodes + ) + + if automated_snapshot_start_hour is None: + automated_snapshot_start_hour = 0 + + if removal_policy is None: + removal_policy = cdk.RemovalPolicy.DESTROY + + if access_policies is None: + arn_builder = ArnBuilder(self.context.config()) + access_policies = [ + iam.PolicyStatement( + principals=[iam.AnyPrincipal()], + actions=['es:ESHttp*'], + resources=[ + arn_builder.get_arn( + 'es', + f'domain/{domain_name}/*' + ) + ] + ) + ] + + if advanced_options is None: + advanced_options = { + 'rest.action.multi.allow_explicit_index': 'true' + } + + super().__init__( + context, name, scope, + version=version, + access_policies=access_policies, + advanced_options=advanced_options, + automated_snapshot_start_hour=automated_snapshot_start_hour, + capacity=capacity, + cognito_dashboards_auth=cognito_dashboards_auth, + custom_endpoint=custom_endpoint, + domain_name=domain_name, + ebs=ebs, + enable_version_upgrade=enable_version_upgrade, + encryption_at_rest=encryption_at_rest, + enforce_https=enforce_https, + fine_grained_access_control=fine_grained_access_control, + logging=logging, + node_to_node_encryption=node_to_node_encryption, + removal_policy=removal_policy, + security_groups=security_groups, + tls_security_policy=tls_security_policy, + use_unsigned_basic_auth=use_unsigned_basic_auth, + vpc=cluster.vpc, + vpc_subnets=vpc_subnets, + zone_awareness=zone_awareness) + + if create_service_linked_role: + + aws_service_name = self.context.config().get_string('global-settings.opensearch.aws_service_name') + if Utils.is_empty(aws_service_name): + dns_suffix = self.context.config().get_string('cluster.aws.dns_suffix', required=True) + aws_service_name = f'es.{dns_suffix}' + + # DO NOT CHANGE THE DESCRIPTION OF THE ROLE. + service_linked_role = iam.CfnServiceLinkedRole( + self, + self.build_resource_name('es-service-linked-role'), + aws_service_name=aws_service_name, + description='Role for ES to access resources in the VPC' + ) + self.node.add_dependency(service_linked_role) + + self.add_nag_suppression(suppressions=[ + IdeaNagSuppression(rule_id='AwsSolutions-OS3', reason='Access to OpenSearch cluster is restricted within a VPC'), + IdeaNagSuppression(rule_id='AwsSolutions-OS4', reason='Use existing resources flow to provision an even more scalable OpenSearch cluster with dedicated master nodes'), + IdeaNagSuppression(rule_id='AwsSolutions-OS5', reason='Access to OpenSearch cluster is restricted within a VPC') + ]) diff --git a/source/idea/idea-administrator/src/ideaadministrator/app/cdk/constructs/backup.py b/source/idea/idea-administrator/src/ideaadministrator/app/cdk/constructs/backup.py new file mode 100644 index 00000000..dd7133c0 --- /dev/null +++ b/source/idea/idea-administrator/src/ideaadministrator/app/cdk/constructs/backup.py @@ -0,0 +1,124 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +from ideaadministrator.app_context import AdministratorContext +from ideaadministrator.app.cdk.constructs import SocaBaseConstruct +from ideasdk.utils import Utils + +from typing import Dict, Optional, List +import constructs +import aws_cdk as cdk +from aws_cdk import ( + aws_iam as iam, + aws_backup as backup, + aws_events as events +) + + +class BackupPlan(SocaBaseConstruct): + """ + Backup Plan + """ + + def __init__(self, context: AdministratorContext, name: str, scope: constructs.Construct, + backup_plan_name: str, + backup_plan_config: Dict, + backup_vault: backup.IBackupVault, + backup_role: iam.IRole): + self.scope = scope + self.backup_vault = backup_vault + self.backup_plan_name = backup_plan_name + self.backup_plan_config = backup_plan_config + self.backup_role = backup_role + + super().__init__(context, name) + + self.backup_plan: Optional[backup.BackupPlan] = None + self.backup_selection: Optional[backup.BackupSelection] = None + + self._build_backup_plan() + + def get_backup_plan_arn(self) -> str: + return self.backup_plan.backup_plan_arn + + def _build_backup_plan_rules(self) -> List[backup.BackupPlanRule]: + + result = [] + rules = Utils.get_value_as_dict('rules', self.backup_plan_config, default={}) + + for rule_name, rule in rules.items(): + delete_after_days = Utils.get_value_as_int('delete_after_days', rule) + start_window_minutes = Utils.get_value_as_int('start_window_minutes', rule) + completion_window_minutes = Utils.get_value_as_int('completion_window_minutes', rule) + schedule_expression = Utils.get_value_as_string('schedule_expression', rule) + move_to_cold_storage_after_days = Utils.get_value_as_int('move_to_cold_storage_after_days', rule, default=None) + + assert delete_after_days is not None + assert start_window_minutes is not None + assert completion_window_minutes is not None + assert schedule_expression is not None + + move_to_cold_storage_after = None + if move_to_cold_storage_after_days is not None: + move_to_cold_storage_after = cdk.Duration.days(move_to_cold_storage_after_days) + + result.append(backup.BackupPlanRule( + rule_name=rule_name, + backup_vault=self.backup_vault, + start_window=cdk.Duration.minutes(start_window_minutes), + completion_window=cdk.Duration.minutes(completion_window_minutes), + delete_after=cdk.Duration.days(delete_after_days), + move_to_cold_storage_after=move_to_cold_storage_after, + schedule_expression=events.Schedule.expression(schedule_expression) + )) + + return result + + def _build_backup_plan(self): + # build rules + backup_plan_rules = self._build_backup_plan_rules() + + # build backup plan + backup_plan_enable_windows_vss = Utils.get_value_as_bool('enable_windows_vss', self.backup_plan_config, default=False) + backup_plan = backup.BackupPlan( + self.scope, self.backup_plan_name, + backup_plan_name=self.backup_plan_name, + backup_plan_rules=backup_plan_rules, + backup_vault=self.backup_vault, + windows_vss=backup_plan_enable_windows_vss + ) + + # build backup selection + selection = Utils.get_value_as_dict('selection', self.backup_plan_config, default={}) + + tags = Utils.get_value_as_list('tags', selection, []) + selection_tags = Utils.convert_custom_tags_to_key_value_pairs(tags) + + resources = [] + for key, value in selection_tags.items(): + resources.append(backup.BackupResource( + tag_condition=backup.TagCondition( + key=key, + value=value, + operation=backup.TagOperation.STRING_EQUALS + ) + )) + + backup_selection = backup.BackupSelection( + self.scope, f'{self.backup_plan_name}-selection', + backup_plan=backup_plan, + resources=resources, + backup_selection_name=f'{self.backup_plan_name}-selection', + role=self.backup_role + ) + + self.backup_plan = backup_plan + self.backup_selection = backup_selection diff --git a/source/idea/idea-administrator/src/ideaadministrator/app/cdk/constructs/base.py b/source/idea/idea-administrator/src/ideaadministrator/app/cdk/constructs/base.py new file mode 100644 index 00000000..cb143dd4 --- /dev/null +++ b/source/idea/idea-administrator/src/ideaadministrator/app/cdk/constructs/base.py @@ -0,0 +1,178 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +__all__ = ( + 'IdeaNagSuppression', + 'SocaBaseConstruct' +) + +from ideadatamodel import exceptions, errorcodes, constants, SocaBaseModel +from ideasdk.utils import Utils +import ideaadministrator +from ideaadministrator.app_context import AdministratorContext + +from typing import Optional, List +import aws_cdk as cdk +import constructs +from aws_cdk import ( + aws_iam as iam +) +from cdk_nag import NagSuppressions + + +class IdeaNagSuppression(SocaBaseModel): + rule_id: str + reason: str + + +class SocaBaseConstruct: + + def __init__(self, context: AdministratorContext, name: str, scope: constructs.IConstruct = None, **kwargs): + self.context = context + self._name = name + self._construct_id: Optional[str] = self.get_construct_id() + self.kwargs = kwargs + if scope: + super().__init__(scope, self.construct_id, **kwargs) + self.add_common_tags() + + @property + def has_tags(self) -> bool: + return self.kwargs is not None and 'tags' in self.kwargs + + @property + def cluster_name(self) -> str: + return self.context.config().get_string('cluster.cluster_name', required=True) + + @property + def release_version(self) -> str: + return ideaadministrator.__version__ + + @property + def construct_id_prefix(self) -> str: + return f'{self.cluster_name}' + + @property + def resource_name_prefix(self) -> str: + return f'{self.cluster_name}' + + @property + def construct_id(self) -> str: + return self._construct_id + + @property + def name(self) -> str: + if Utils.is_empty(self._name): + raise exceptions.SocaException( + error_code=errorcodes.GENERAL_ERROR, + message='Invalid CDK Construct. "name" is required.' + ) + return self._name + + @property + def resource_name(self) -> str: + return self.build_resource_name(self.name) + + def name_title_case(self): + return Utils.to_title_case(self.name) + + @property + def aws_account_arn(self) -> str: + aws_partition = self.context.config().get_string('cluster.aws.partition', required=True) + aws_account_id = self.context.config().get_string('cluster.aws.account_id', required=True) + return f'arn:{aws_partition}:*:*:{aws_account_id}:*' + + @staticmethod + def build_service_principal(service_name) -> iam.ServicePrincipal: + service_fqdn = f'{service_name}.{cdk.Aws.URL_SUFFIX}' + return iam.ServicePrincipal(service_fqdn) + + # override if required + def get_construct_id(self) -> Optional[str]: + return self.name + + # trimmed format - {prefix}-{region}-{name[:10]}-{hash} + def build_trimmed_resource_name(self, name: str, region_suffix=False, trim_length=64) -> str: + prefix = self.resource_name_prefix + suffix = '' + if region_suffix: + suffix = f'-{self.context.config().get_string("cluster.aws.region", required=True)}' + + resource_name = f'{prefix}-{name}{suffix}' + trimmed_resource_name = f'{prefix}{suffix}-{name[:10]}-{Utils.shake_256(data=resource_name, num_bytes=int((trim_length - 12 - len(prefix) - len(suffix)) / 2))}' + return trimmed_resource_name + + def build_resource_name(self, name: str, region_suffix=False) -> str: + prefix = self.resource_name_prefix + resource_name = f'{prefix}-{name}' + if region_suffix: + aws_region = self.context.config().get_string('cluster.aws.region', required=True) + resource_name = f'{resource_name}-{aws_region}' + return resource_name + + def add_common_tags(self, construct: Optional[constructs.IConstruct] = None): + if construct is None and isinstance(self, constructs.Construct): + construct = self + cdk.Tags.of(construct).add(constants.IDEA_TAG_NAME, self.resource_name) + cdk.Tags.of(construct).add(constants.IDEA_TAG_CLUSTER_NAME, self.cluster_name) + + def add_backup_tags(self, construct: Optional[constructs.IConstruct] = None): + """ + add AWS Backup integration tags. + backup tags cannot be added as a blanket tag on CFN template or common tags. + only specific resources that need backup integration must be tagged. + """ + if construct is None and isinstance(self, constructs.Construct): + construct = self + cdk.Tags.of(construct).add(constants.IDEA_TAG_BACKUP_PLAN, f'{self.cluster_name}-{constants.MODULE_CLUSTER}') + + def build_instance_profile_arn(self, instance_profile_ref: str): + aws_partition = self.context.config().get_string('cluster.aws.partition', required=True) + aws_account_id = self.context.config().get_string('cluster.aws.account_id', required=True) + return f'arn:{aws_partition}:iam::{aws_account_id}:instance-profile/{instance_profile_ref}' + + def is_ds_activedirectory(self) -> bool: + return self.context.config().get_string('directoryservice.provider') in (constants.DIRECTORYSERVICE_AWS_MANAGED_ACTIVE_DIRECTORY, + constants.DIRECTORYSERVICE_ACTIVE_DIRECTORY) + + def is_ds_openldap(self) -> bool: + return self.context.config().get_string('directoryservice.provider') == constants.DIRECTORYSERVICE_OPENLDAP + + def add_nag_suppression(self, suppressions: List[IdeaNagSuppression], construct: constructs.IConstruct = None, apply_to_children: bool = False): + if construct is None: + construct = self + cdk_nag_suppressions = [] + for suppression in suppressions: + cdk_nag_suppressions.append({ + 'id': suppression.rule_id, + 'reason': suppression.reason + }) + if isinstance(construct, cdk.Stack): + NagSuppressions.add_stack_suppressions( + stack=construct, + suppressions=cdk_nag_suppressions, + apply_to_nested_stacks=apply_to_children + ) + else: + NagSuppressions.add_resource_suppressions( + construct=construct, + suppressions=cdk_nag_suppressions, + apply_to_children=apply_to_children + ) + + def get_kms_key_arn(self, key_id: str) -> str: + # arn:AWS_PARTITION:kms:AWS_REGION:ACCOUNT_ID:key/UUID + if key_id.startswith('arn:'): + return key_id + aws_partition = self.context.config().get_string('cluster.aws.partition', required=True) + aws_region = self.context.config().get_string('cluster.aws.region', required=True) + aws_account_id = self.context.config().get_string('cluster.aws.account_id', required=True) + return f'arn:{aws_partition}:kms:{aws_region}:{aws_account_id}:key/{key_id}' diff --git a/source/idea/idea-administrator/src/ideaadministrator/app/cdk/constructs/common.py b/source/idea/idea-administrator/src/ideaadministrator/app/cdk/constructs/common.py new file mode 100644 index 00000000..0992d6e6 --- /dev/null +++ b/source/idea/idea-administrator/src/ideaadministrator/app/cdk/constructs/common.py @@ -0,0 +1,604 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +__all__ = ( + 'LambdaFunction', + 'Policy', + 'ManagedPolicy', + 'Role', + 'InstanceProfile', + 'CustomResource', + 'CreateTagsCustomResource', + 'SQSQueue', + 'SNSTopic', + 'SNSSubscription', + 'CloudWatchAlarm', + 'Output', + 'DynamoDBTable', + 'KinesisStream' +) + +from aws_cdk.aws_ec2 import IVpc, SubnetSelection, ISecurityGroup + +from ideaadministrator.app.cdk.idea_code_asset import IdeaCodeAsset, SupportedLambdaPlatforms +from ideaadministrator.app_context import AdministratorContext +from ideaadministrator.app.cdk.constructs import SocaBaseConstruct, IdeaNagSuppression +from ideaadministrator.app_utils import AdministratorUtils +from ideasdk.utils import Utils +from ideadatamodel import SocaAnyPayload, exceptions + +from typing import List, Dict, Optional, Any, Union + +import aws_cdk as cdk +import constructs +from aws_cdk import ( + aws_lambda as lambda_, + aws_logs as logs, + aws_iam as iam, + aws_cloudwatch as cloudwatch, + aws_sns as sns, + aws_sqs as sqs, + aws_dynamodb as dynamodb, + aws_kinesis as kinesis, + aws_kms as kms +) + + +class LambdaFunction(SocaBaseConstruct, lambda_.Function): + MAX_NAME_LENGTH = 64 + + def __init__(self, context: AdministratorContext, + name: str, + scope: constructs.IConstruct, + idea_code_asset: IdeaCodeAsset = None, + code: lambda_.AssetCode = None, + handler: str = None, + description: str = None, + memory_size: int = 128, + runtime: lambda_.Runtime = lambda_.Runtime.PYTHON_3_9, + timeout_seconds: int = 60, + vpc: Optional[IVpc] = None, + security_groups: Optional[List[ISecurityGroup]] = None, + vpc_subnets: Optional[SubnetSelection] = None, + log_retention: logs.RetentionDays = None, + log_retention_role: Optional[iam.IRole] = None, + environment: dict = None, + role: iam.IRole = None): + self.context = context + + if Utils.is_empty(idea_code_asset): + if Utils.are_empty(code, handler): + raise exceptions.invalid_params('Provide either idea_code_asset or (code and handler)') + else: + lambda_build_dir = idea_code_asset.build_lambda() + code = lambda_.Code.from_asset( + path=lambda_build_dir, + asset_hash_type=cdk.AssetHashType.CUSTOM, + asset_hash=idea_code_asset.asset_hash + ) + handler = idea_code_asset.lambda_handler + + timeout = cdk.Duration.seconds(timeout_seconds) + function_name = self.build_resource_name(name), + if isinstance(function_name, tuple): + function_name = ' '.join(function_name) + + if len(function_name) > self.MAX_NAME_LENGTH: + function_name = self.build_trimmed_resource_name(name, trim_length=self.MAX_NAME_LENGTH) + + super().__init__( + context, name, scope, + function_name=function_name, + description=description, + memory_size=memory_size, + runtime=runtime, + timeout=timeout, + log_retention=log_retention, + handler=handler, + environment=environment, + code=code, + role=role, + vpc=vpc, + security_groups=security_groups, + vpc_subnets=vpc_subnets, + log_retention_role=log_retention_role) + + +class Policy(SocaBaseConstruct, iam.Policy): + + def __init__(self, context: AdministratorContext, + name: str, + scope: constructs.Construct, + policy_template_name: str, + vars: SocaAnyPayload = None, # noqa + module_id: str = None): + self.context = context + self.policy_template_name = policy_template_name + self.vars = vars + self.module_id = module_id + super().__init__(context, name, scope, document=self.build_policy_json()) + + self.add_nag_suppression(suppressions=[ + IdeaNagSuppression(rule_id='AwsSolutions-IAM5', reason='Wild-card policies are scoped with conditions and/or applicable prefixes.') + ]) + + def build_policy_json(self) -> iam.PolicyDocument: + policy = AdministratorUtils.render_policy( + policy_template_name=self.policy_template_name, + cluster_name=self.context.cluster_name(), + module_id=self.module_id, + config=self.context.config(), + vars=self.vars + ) + return iam.PolicyDocument.from_json(policy) + + +class ManagedPolicy(SocaBaseConstruct, iam.ManagedPolicy): + + def __init__(self, context: AdministratorContext, + name: str, + scope: constructs.Construct, + description: str, + policy_template_name: str, + managed_policy_name: str, + vars: SocaAnyPayload = None, # noqa + module_id: str = None): + self.context = context + self.policy_template_name = policy_template_name + self.vars = vars + self.module_id = module_id + super().__init__(context, name, scope, managed_policy_name=managed_policy_name, description=description, document=self.build_policy_json()) + + self.add_nag_suppression(suppressions=[ + IdeaNagSuppression(rule_id='AwsSolutions-IAM5', reason='AWS Managed Policies are expected to be customized and scoped down.' + 'AWS Managed policies are copied over to enable these customizations.') + ]) + + def build_policy_json(self) -> iam.PolicyDocument: + policy = AdministratorUtils.render_policy( + policy_template_name=self.policy_template_name, + cluster_name=self.context.cluster_name(), + module_id=self.module_id, + config=self.context.config(), + vars=self.vars + ) + return iam.PolicyDocument.from_json(policy) + + +class Role(SocaBaseConstruct, iam.Role): + MAX_NAME_LENGTH = 64 + + def __init__(self, context: AdministratorContext, name: str, scope: constructs.Construct, + description: str, + assumed_by: List[str], + inline_policies: List[iam.Policy] = None, + managed_policies: List[str] = None): + + self.context = context + role_name = self.build_resource_name(name, region_suffix=True) + if isinstance(role_name, tuple): + role_name = ' '.join(role_name) + + if len(role_name) > self.MAX_NAME_LENGTH: + role_name = self.build_trimmed_resource_name(name, region_suffix=True, trim_length=self.MAX_NAME_LENGTH) + + super().__init__(context, name, scope, + role_name=role_name, + description=description, + assumed_by=self.build_assumed_by(assumed_by)) + if inline_policies is not None: + for policy in inline_policies: + self.attach_inline_policy(policy) + if managed_policies is not None: + for policy in managed_policies: + if policy.startswith('arn:'): + name = policy.split('/')[1] + self.add_managed_policy(iam.ManagedPolicy.from_managed_policy_arn(self, name, policy)) + else: + self.add_managed_policy(iam.ManagedPolicy.from_aws_managed_policy_name(policy)) + + def build_assumed_by(self, assumed_by: List[str]) -> iam.IPrincipal: + principals = [] + for service in assumed_by: + principals.append(self.build_service_principal(service)) + return iam.CompositePrincipal(*principals) + + +class InstanceProfile(SocaBaseConstruct, iam.CfnInstanceProfile): + def __init__(self, context: AdministratorContext, name: str, scope: constructs.Construct, + roles: List[iam.Role]): + self.context = context + role_names = [] + for role in roles: + role_names.append(role.role_name) + super().__init__(context, name, scope, instance_profile_name=self.build_resource_name(name, region_suffix=True), roles=role_names) + + +class CustomResource(SocaBaseConstruct): + def __init__(self, context: AdministratorContext, + name: str, + scope: constructs.Construct, + idea_code_asset: IdeaCodeAsset, + policy_statements: Optional[List[iam.PolicyStatement]] = None, + policy_template_name: Optional[str] = None, + removal_policy: Optional[cdk.RemovalPolicy] = None, + resource_type: Optional[str] = None, + runtime: lambda_.Runtime = lambda_.Runtime.PYTHON_3_9, + lambda_timeout_seconds: int = 60, + lambda_log_retention_role: Optional[iam.IRole] = None): + + super().__init__(context, name) + self.scope = scope + self.idea_code_asset = idea_code_asset + self.policy_template_name = policy_template_name + self.policy_statements = policy_statements + self.removal_policy = removal_policy + self.lambda_timeout_seconds = lambda_timeout_seconds + self.lambda_log_retention_role = lambda_log_retention_role + self.runtime = runtime + + prefix = 'Custom::' + if Utils.is_empty(resource_type): + self.resource_type = f'{prefix}{self.name_title_case()}' + else: + if resource_type.startswith(prefix): + self.resource_type = resource_type + else: + self.resource_type = f'{prefix}{resource_type}' + + self.lambda_role: Optional[Role] = None + self.lambda_policy: Optional[Policy] = None + self.lambda_function: Optional[LambdaFunction] = None + + self.build_lambda_function() + + def build_lambda_function(self): + + if Utils.is_not_empty(self.policy_template_name): + self.lambda_policy = Policy( + context=self.context, + name=f'{self.name}-lambda-policy', + scope=self.scope, + policy_template_name=self.policy_template_name + ) + + self.lambda_role = Role( + self.context, + f'{self.name}-role', + self.scope, + f'Role for {self.resource_type} for Cluster: {self.cluster_name}', + assumed_by=['lambda'] + ) + if self.lambda_policy is not None: + self.lambda_role.attach_inline_policy(self.lambda_policy) + + self.lambda_function = LambdaFunction( + self.context, + f'{self.name}-lambda', + self.scope, + idea_code_asset=self.idea_code_asset, + description=f'{self.resource_type} Lambda Function for Cluster: {self.cluster_name}', + timeout_seconds=self.lambda_timeout_seconds, + role=self.lambda_role, + log_retention_role=self.lambda_log_retention_role, + runtime=self.runtime + ) + + if self.policy_statements is not None: + for statement in self.policy_statements: + self.lambda_function.add_to_role_policy(statement=statement) + + if self.lambda_role is not None: + self.lambda_function.node.add_dependency(self.lambda_role) + if self.lambda_policy is not None: + self.lambda_function.node.add_dependency(self.lambda_policy) + + def invoke(self, name: str, properties: Dict[str, Any]) -> cdk.CustomResource: + custom_resource = cdk.CustomResource( + self.scope, + name, + service_token=self.lambda_function.function_arn, + properties=properties, + removal_policy=self.removal_policy, + resource_type=self.resource_type + ) + custom_resource.node.add_dependency(self.lambda_function) + return custom_resource + + +class CreateTagsCustomResource(CustomResource): + def __init__(self, context: AdministratorContext, + scope: constructs.Construct, + lambda_log_retention_role: Optional[iam.IRole] = None): + super().__init__( + context, 'ec2-create-tags', scope, + idea_code_asset=IdeaCodeAsset( + lambda_package_name='idea_custom_resource_create_tags', + lambda_platform=SupportedLambdaPlatforms.PYTHON + ), + policy_template_name='custom-resource-ec2-create-tags.yml', + resource_type='EC2CreateTags', + lambda_log_retention_role=lambda_log_retention_role) + + def apply(self, name: str, resource_id: str, tags: Dict[str, Any]) -> cdk.CustomResource: + aws_tags = [] + for key, value in tags.items(): + aws_tags.append({ + 'Key': key, + 'Value': str(value) + }) + return super().invoke(name, properties={ + 'ResourceId': resource_id, + 'Tags': aws_tags + }) + + +class SQSQueue(SocaBaseConstruct, sqs.Queue): + + def __init__(self, context: AdministratorContext, id_: str, scope: constructs.Construct, *, + content_based_deduplication: Optional[bool] = None, + data_key_reuse: Optional[cdk.Duration] = None, + dead_letter_queue: Optional[Union[sqs.DeadLetterQueue, Dict[str, Any]]] = None, + deduplication_scope: Optional[sqs.DeduplicationScope] = None, + delivery_delay: Optional[cdk.Duration] = None, + encrypt_at_rest: Optional[bool] = True, + encryption: Optional[sqs.QueueEncryption] = None, + encryption_master_key: Optional[str] = None, + fifo: Optional[bool] = None, + fifo_throughput_limit: Optional[sqs.FifoThroughputLimit] = None, + max_message_size_bytes: Optional[int] = None, + queue_name: Optional[str] = None, + receive_message_wait_time: Optional[cdk.Duration] = None, + removal_policy: Optional[cdk.RemovalPolicy] = None, + retention_period: Optional[cdk.Duration] = None, + visibility_timeout: Optional[cdk.Duration] = None, + is_dead_letter_queue: bool = False): + self.context = context + + if encrypt_at_rest: + if encryption is None: + encryption = sqs.QueueEncryption.KMS_MANAGED + + encryption_master_key_ = None + if Utils.is_not_empty(encryption_master_key): + key_arn = self.get_kms_key_arn(encryption_master_key) + encryption_master_key_ = kms.Key.from_key_arn(scope=scope, id=f'{id_}-kms-key', key_arn=key_arn) + encryption = sqs.QueueEncryption.KMS + else: + encryption = sqs.QueueEncryption.UNENCRYPTED + encryption_master_key_ = None + + super().__init__( + context=context, + name=id_, + scope=scope, + content_based_deduplication=content_based_deduplication, + data_key_reuse=data_key_reuse, + dead_letter_queue=dead_letter_queue, + deduplication_scope=deduplication_scope, + delivery_delay=delivery_delay, + encryption=encryption, + encryption_master_key=encryption_master_key_, + fifo=fifo, + fifo_throughput_limit=fifo_throughput_limit, + max_message_size_bytes=max_message_size_bytes, + queue_name=queue_name, + receive_message_wait_time=receive_message_wait_time, + removal_policy=removal_policy, + retention_period=retention_period, + visibility_timeout=visibility_timeout + ) + + if encrypt_at_rest: + self.add_to_resource_policy(iam.PolicyStatement( + sid='AlwaysEncrypted', + effect=iam.Effect.DENY, + actions=['sqs:*'], + conditions={ + 'Bool': { + 'aws:SecureTransport': 'false' + } + }, + resources=[self.queue_arn], + principals=[iam.AnyPrincipal()] + )) + else: + self.add_nag_suppression(suppressions=[ + IdeaNagSuppression(rule_id='AwsSolutions-SQS2', reason='SQS encryption key is configurable, but is not provided in cluster config.') + ]) + self.add_nag_suppression(suppressions=[ + IdeaNagSuppression(rule_id='AwsSolutions-SQS4', reason='SQS encryption key is configurable, but is not provided in cluster config.') + ]) + + if is_dead_letter_queue: + self.add_nag_suppression(suppressions=[ + IdeaNagSuppression(rule_id='AwsSolutions-SQS3', reason='Dead letter queue') + ]) + + +class SNSTopic(SocaBaseConstruct, sns.Topic): + + def __init__(self, context: AdministratorContext, id_: str, scope: constructs.Construct, *, + fifo: bool = None, + master_key: str = None, + display_name: str = None, + topic_name: str = None, + policy_statements: List[iam.PolicyStatement] = None): + + self.context = context + + if Utils.is_empty(topic_name): + topic_name = self.build_resource_name(id_) + + if Utils.is_empty(display_name): + display_name = topic_name + + if Utils.is_not_empty(master_key): + kms_key_arn = self.get_kms_key_arn(master_key) + master_key_ = kms.Key.from_key_arn(scope=scope, id=f'{id_}-kms-key', key_arn=kms_key_arn) + else: + master_key_ = kms.Alias.from_alias_name(scope=scope, id=f'{id_}-kms-key-default', alias_name='alias/aws/sns') + + super().__init__( + context, id_, scope, + display_name=display_name, + fifo=fifo, + topic_name=topic_name, + master_key=master_key_) + + if policy_statements is not None: + for statement in policy_statements: + self.add_to_resource_policy(statement) + + self.add_to_resource_policy(iam.PolicyStatement( + sid='AlwaysEncrypted', + effect=iam.Effect.DENY, + actions=['SNS:Publish'], + conditions={ + 'Bool': { + 'aws:SecureTransport': 'false' + } + }, + resources=[self.topic_arn], + principals=[iam.AnyPrincipal()] + )) + + +class SNSSubscription(SocaBaseConstruct, sns.Subscription): + + def __init__(self, context: AdministratorContext, name: str, scope: constructs.Construct, + protocol: sns.SubscriptionProtocol, + endpoint: str, + topic: sns.ITopic): + super().__init__(context, name, scope, + protocol=protocol, + endpoint=endpoint, + topic=topic) + + +class CloudWatchAlarm(SocaBaseConstruct, cloudwatch.Alarm): + + def __init__(self, context: AdministratorContext, name: str, scope: constructs.Construct, + metric_name: str, + metric_namespace: str, + threshold: Union[int, float], + metric_dimensions: Dict = None, + period: cdk.Duration = None, + period_seconds=60, + statistic: str = 'Average', + actions_enabled: bool = None, + alarm_description: str = None, + comparison_operator: cloudwatch.ComparisonOperator = None, + evaluation_periods=10, + datapoints_to_alarm: Union[int, float, None] = None, + evaluate_low_sample_count_percentile: str = None, + treat_missing_data: cloudwatch.TreatMissingData = None, + actions: List[cloudwatch.IAlarmAction] = None): + + self.context = context + + if period is None: + period = cdk.Duration.seconds(period_seconds) + + if alarm_description is None: + alarm_description = f'{metric_name} Alarm' + + super().__init__(context, name, scope, + metric=cloudwatch.Metric( + metric_name=metric_name, + namespace=metric_namespace, + dimensions_map=metric_dimensions, + period=period, + statistic=statistic + ), + threshold=threshold, + evaluation_periods=evaluation_periods, + actions_enabled=actions_enabled, + alarm_description=alarm_description, + alarm_name=self.build_resource_name(name), + comparison_operator=comparison_operator, + datapoints_to_alarm=datapoints_to_alarm, + evaluate_low_sample_count_percentile=evaluate_low_sample_count_percentile, + treat_missing_data=treat_missing_data) + + if actions is not None: + for action in actions: + self.add_alarm_action(action) + + +class Output(SocaBaseConstruct): + + def __init__(self, context: AdministratorContext, name: str, scope: constructs.Construct, + value_ref: Any, description: str = None, export_name=None): + super().__init__(context, name) + self.output = cdk.CfnOutput( + scope, + name, + value=value_ref, + description=description, + export_name=export_name + ) + + @property + def description(self) -> str: + return self.output.description + + @property + def value(self) -> Any: + return self.output.value + + @property + def string_value(self) -> str: + return str(self.output.value) + + +class DynamoDBTable(SocaBaseConstruct, dynamodb.Table): + + def __init__(self, context: AdministratorContext, name: str, scope: constructs.Construct, + table_name: str, + partition_key: dynamodb.Attribute, + billing_mode: Optional[dynamodb.BillingMode] = None, + read_capacity: Optional[int] = None, + write_capacity: Optional[int] = None, + sort_key: Optional[dynamodb.Attribute] = None, + removal_policy: Optional[cdk.RemovalPolicy] = None, + replication_regions: Optional[List[str]] = None, + replication_timeout: Optional[cdk.Duration] = None, + stream: Optional[dynamodb.StreamViewType] = None, + time_to_live_attribute: Optional[str] = None, + wait_for_replication_to_finish: Optional[bool] = None): + self.context = context + + super().__init__(context, name, scope, + table_name=table_name, + billing_mode=billing_mode, + contributor_insights_enabled=False, + encryption=None, + encryption_key=None, + point_in_time_recovery=None, + read_capacity=read_capacity, + removal_policy=removal_policy, + replication_regions=replication_regions, + replication_timeout=replication_timeout, + stream=stream, + time_to_live_attribute=time_to_live_attribute, + wait_for_replication_to_finish=wait_for_replication_to_finish, + write_capacity=write_capacity, + partition_key=partition_key, + sort_key=sort_key) + + +class KinesisStream(SocaBaseConstruct, kinesis.Stream): + def __init__(self, context: AdministratorContext, name: str, scope: constructs.Construct, stream_name: str, stream_mode: kinesis.StreamMode, shard_count: Optional[int]): + super().__init__(context, name, scope, + stream_name=f'{context.cluster_name()}-{stream_name}', + stream_mode=stream_mode, + shard_count=shard_count) diff --git a/source/idea/idea-administrator/src/ideaadministrator/app/cdk/constructs/directory_service.py b/source/idea/idea-administrator/src/ideaadministrator/app/cdk/constructs/directory_service.py new file mode 100644 index 00000000..d771a05f --- /dev/null +++ b/source/idea/idea-administrator/src/ideaadministrator/app/cdk/constructs/directory_service.py @@ -0,0 +1,452 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +__all__ = ( + 'DirectoryServiceCredentials', + 'ActiveDirectory', + 'UserPool', + 'OAuthClientIdAndSecret' +) + +from ideaadministrator.app.cdk.idea_code_asset import IdeaCodeAsset, SupportedLambdaPlatforms +from ideadatamodel import constants +from ideasdk.utils import Utils, GroupNameHelper + +from ideaadministrator.app.cdk.constructs import SocaBaseConstruct, ExistingSocaCluster, DNSResolverEndpoint, DNSResolverRule, CustomResource, IdeaNagSuppression +from ideaadministrator.app_context import AdministratorContext + +from typing import Optional, List + +import constructs +import aws_cdk as cdk +from aws_cdk import ( + aws_ec2 as ec2, + aws_directoryservice as ds, + aws_secretsmanager as secretsmanager, + aws_cognito as cognito +) + + +class DirectoryServiceCredentials(SocaBaseConstruct): + """ + Directory Service Root User Credentials + + These credentials are initialized once during the initial cluster creation. + """ + + def __init__(self, context: AdministratorContext, name: str, scope: constructs.Construct, + admin_username: str, + admin_password: Optional[str] = None): + super().__init__(context, name) + + self.credentials_provided = self.context.config().get_bool('directoryservice.root_credentials_provided', default=False) + + if self.credentials_provided: + return + + kms_key_id = self.context.config().get_string('cluster.secretsmanager.kms_key_id') + + ds_provider = self.context.config().get_string('directoryservice.provider', required=True) + + admin_username_key = f'{ds_provider}-admin-username' + self.admin_username = secretsmanager.CfnSecret( + scope, + admin_username_key, + description=f'{ds_provider} Root Username, Cluster: {self.cluster_name}', + kms_key_id=kms_key_id, + name=self.build_resource_name(admin_username_key), + secret_string=admin_username + ) + # access to the secret is restricted using tags. + cdk.Tags.of(self.admin_username).add(constants.IDEA_TAG_MODULE_NAME, constants.MODULE_DIRECTORYSERVICE) + + admin_password_key = f'{ds_provider}-admin-password' + if Utils.is_empty(admin_password): + self.admin_password = secretsmanager.CfnSecret( + scope, + admin_password_key, + description=f'{ds_provider} Root Password, Cluster: {self.cluster_name}', + kms_key_id=kms_key_id, + name=self.build_resource_name(admin_password_key), + generate_secret_string=secretsmanager.CfnSecret.GenerateSecretStringProperty( + exclude_characters='$@;"\\\'', + password_length=16 + ) + ) + else: + self.admin_password = secretsmanager.CfnSecret( + scope, + admin_password_key, + description=f'{ds_provider} Root Password, Cluster: {self.cluster_name}', + kms_key_id=kms_key_id, + name=self.build_resource_name(admin_password_key), + secret_string=admin_password + ) + # access to the secret is restricted using tags. + cdk.Tags.of(self.admin_password).add(constants.IDEA_TAG_MODULE_NAME, constants.MODULE_DIRECTORYSERVICE) + + suppressions = [ + IdeaNagSuppression(rule_id='AwsSolutions-SMG4', reason='Secret rotation not applicable for DirectoryService credentials.') + ] + self.add_nag_suppression(construct=self.admin_username, suppressions=suppressions) + self.add_nag_suppression(construct=self.admin_password, suppressions=suppressions) + + def get_username_secret_arn(self) -> str: + if self.credentials_provided: + return self.context.config().get_string('directoryservice.root_username_secret_arn', required=True) + else: + return self.admin_username.ref + + def get_password_secret_arn(self) -> str: + if self.credentials_provided: + return self.context.config().get_string('directoryservice.root_password_secret_arn', required=True) + else: + return self.admin_password.ref + + +class OAuthClientIdAndSecret(SocaBaseConstruct): + """ + Create ClientId and ClientSecret in Secrets Manager + """ + + def __init__(self, context: AdministratorContext, + secret_name_prefix: str, + module_name: str, + scope: constructs.Construct, + client_id: str, client_secret: str): + """ + :param context: + :param secret_name_prefix is used to create the secret name. + * -client-id + * -client-secret + :param module_name is used to tag the secret values. + access to secrets is restricted by tag name in IAM roles + :param scope: constructs.Construct + :param client_id: the client_id + :param client_secret: the client secret + """ + super().__init__(context, secret_name_prefix) + + kms_key_id = self.context.config().get_string('cluster.secretsmanager.kms_key_id') + + client_id_key = f'{secret_name_prefix}-client-id' + self.client_id = secretsmanager.CfnSecret( + scope, + client_id_key, + description=f'{secret_name_prefix} ClientId, Cluster: {self.cluster_name}', + kms_key_id=kms_key_id, + name=self.build_resource_name(client_id_key), + secret_string=client_id + ) + self.client_id.apply_removal_policy(cdk.RemovalPolicy.DESTROY) + cdk.Tags.of(self.client_id).add(constants.IDEA_TAG_MODULE_NAME, module_name) + + client_secret_key = f'{secret_name_prefix}-client-secret' + self.client_secret = secretsmanager.CfnSecret( + scope, + client_secret_key, + description=f'{secret_name_prefix} ClientSecret, Cluster: {self.cluster_name}', + kms_key_id=kms_key_id, + name=self.build_resource_name(client_secret_key), + secret_string=client_secret + ) + self.client_secret.apply_removal_policy(cdk.RemovalPolicy.DESTROY) + cdk.Tags.of(self.client_secret).add(constants.IDEA_TAG_MODULE_NAME, module_name) + + suppressions = [ + IdeaNagSuppression(rule_id='AwsSolutions-SMG4', reason='Secret rotation not applicable for OAuth 2.0 ClientId/Secret') + ] + self.add_nag_suppression(construct=self.client_id, suppressions=suppressions) + self.add_nag_suppression(construct=self.client_secret, suppressions=suppressions) + + +class ActiveDirectory(SocaBaseConstruct): + + def __init__(self, context: AdministratorContext, name: str, scope: constructs.Construct, + cluster: ExistingSocaCluster, + subnets: Optional[List[ec2.ISubnet]] = None, + enable_sso: Optional[bool] = False): + super().__init__(context, name) + + self.scope = scope + self.cluster = cluster + self.enable_sso = enable_sso + + self.ad_admin_username = 'Admin' + + self.credentials = self.build_credentials(self.ad_admin_username) + + self.ad_name = self.context.config().get_string('directoryservice.name', required=True) + self.ad_short_name = self.context.config().get_string('directoryservice.ad_short_name', required=True) + self.ad_edition = self.context.config().get_string('directoryservice.ad_edition', required=True) + + self.launch_subnets = [] + if subnets is None: + subnets = self.cluster.private_subnets + + for subnet in subnets: + self.launch_subnets.append(subnet.subnet_id) + if len(self.launch_subnets) == 2: + break + + self.ad = self.build_ad() + + self.build_dns_resolver() + + def build_credentials(self, username: str) -> DirectoryServiceCredentials: + return DirectoryServiceCredentials( + self.context, + 'ds-activedirectory-credentials', + self.scope, + admin_username=username + ) + + def build_ad(self) -> ds.CfnMicrosoftAD: + + vpc_settings = ds.CfnMicrosoftAD.VpcSettingsProperty( + subnet_ids=self.launch_subnets, + vpc_id=self.cluster.vpc.vpc_id) + + ad = ds.CfnMicrosoftAD( + self.scope, + self.construct_id, + name=self.ad_name, + password=cdk.SecretValue.secrets_manager(self.credentials.get_password_secret_arn()).to_string(), + vpc_settings=vpc_settings, + edition=self.ad_edition, + enable_sso=self.enable_sso, + short_name=self.ad_short_name) + self.add_common_tags(ad) + return ad + + def build_dns_resolver(self): + + get_ad_security_group_result = CustomResource( + context=self.context, + name='get-ad-security-group-id', + scope=self.scope, + idea_code_asset=IdeaCodeAsset( + lambda_package_name='idea_custom_resource_get_ad_security_group', + lambda_platform=SupportedLambdaPlatforms.PYTHON + ), + lambda_timeout_seconds=15, + policy_template_name='custom-resource-get-ad-security-group.yml', + resource_type='ADSecurityGroupId' + ).invoke( + name=self.resource_name, + properties={ + 'DirectoryId': self.ad.ref + } + ) + + endpoint = DNSResolverEndpoint( + context=self.context, + name=f'{self.cluster_name}', + scope=self.scope, + vpc=self.cluster.vpc, + subnet_ids=self.launch_subnets, + security_group_ids=[get_ad_security_group_result.get_att_string('SecurityGroupId')] + ) + + DNSResolverRule( + context=self.context, + name=self.name, + scope=self.scope, + domain_name=self.ad_name, + vpc=self.cluster.vpc, + resolver_endpoint_id=endpoint.resolver_endpoint.attr_resolver_endpoint_id, + ip_addresses=[ + cdk.Fn.select(0, self.ad.attr_dns_ip_addresses), + cdk.Fn.select(1, self.ad.attr_dns_ip_addresses) + ], + port='53' + ) + + +class UserPool(SocaBaseConstruct): + + def __init__(self, context: AdministratorContext, name: str, scope: constructs.Construct, + props: cognito.UserPoolProps = None): + super().__init__(context, name) + self.scope = scope + + self.user_pool: Optional[cognito.UserPool] = None + self.domain: Optional[cognito.UserPoolDomain] = None + self.secrets: Optional[List[OAuthClientIdAndSecret]] = [] + self.props = props + if self.props is None: + self.props = cognito.UserPoolProps() + + self.build_user_pool(self.props) + + def build_user_pool(self, props: cognito.UserPoolProps): + account_recovery = props.account_recovery + if account_recovery is None: + account_recovery = cognito.AccountRecovery.EMAIL_ONLY + auto_verify = props.auto_verify + if auto_verify is None: + auto_verify = cognito.AutoVerifiedAttrs(email=True, phone=False) + custom_attributes = props.custom_attributes + if custom_attributes is None: + custom_attributes = { + 'cluster_name': cognito.StringAttribute(mutable=True), + 'aws_region': cognito.StringAttribute(mutable=True), + 'password_last_set': cognito.NumberAttribute(mutable=True), + 'password_max_age': cognito.NumberAttribute(mutable=True) + } + mfa = props.mfa + if mfa is None: + mfa = cognito.Mfa.OPTIONAL + mfa_second_factor = props.mfa_second_factor + if mfa_second_factor is None: + mfa_second_factor = cognito.MfaSecondFactor(otp=True, sms=False) + + password_policy = props.password_policy + if password_policy is None: + password_policy = cognito.PasswordPolicy( + min_length=8, + require_digits=True, + require_lowercase=True, + require_symbols=True, + require_uppercase=True, + temp_password_validity=cdk.Duration.days(7) + ) + + removal_policy = props.removal_policy + if removal_policy is None: + removal_policy = cdk.RemovalPolicy.DESTROY + + self_sign_up_enabled = props.self_sign_up_enabled + if self_sign_up_enabled is None: + self_sign_up_enabled = False + + sign_in_aliases = props.sign_in_aliases + if sign_in_aliases is None: + sign_in_aliases = cognito.SignInAliases( + username=True, + preferred_username=False, + phone=False, + email=True + ) + + sign_in_case_sensitive = props.sign_in_case_sensitive + if sign_in_case_sensitive is None: + sign_in_case_sensitive = False + + standard_attributes = props.standard_attributes + if standard_attributes is None: + standard_attributes = cognito.StandardAttributes( + email=cognito.StandardAttribute(mutable=True, required=True) + ) + user_invitation = props.user_invitation + if user_invitation is None: + user_invitation = cognito.UserInvitationConfig( + email_subject=f'({self.cluster_name}) Your IDEA Account', + email_body=f''' + Hello {{username}}, +

+ You have been invited to join the {self.cluster_name} cluster. +
+ Your temporary password is {{####}} + ''' + ) + + user_pool_name = props.user_pool_name + if user_pool_name is None: + user_pool_name = f'{self.cluster_name}-user-pool' + + advanced_security_mode = None + + if self.context.aws().aws_region() in Utils.get_value_as_list('COGNITO_ADVANCED_SECURITY_UNAVAIL_REGION_LIST', constants.CAVEATS): + self.context.warning(f'Cognito Advanced security NOT SET - Not available in this region ({self.context.aws().aws_region()})') + advanced_security_mode = None + else: + advanced_security_mode_cfg = self.context.config().get_string('identity-provider.cognito.advanced_security_mode', default='AUDIT') + + if advanced_security_mode_cfg.upper() == 'AUDIT': + advanced_security_mode = cognito.AdvancedSecurityMode.AUDIT + elif advanced_security_mode_cfg.upper() == 'ENFORCED': + advanced_security_mode = cognito.AdvancedSecurityMode.ENFORCED + else: + advanced_security_mode = cognito.AdvancedSecurityMode.OFF + + self.user_pool = cognito.UserPool( + scope=self.scope, + id=user_pool_name, + account_recovery=account_recovery, + advanced_security_mode=advanced_security_mode, + auto_verify=auto_verify, + custom_attributes=custom_attributes, + custom_sender_kms_key=props.custom_sender_kms_key, + deletion_protection=True, + device_tracking=props.device_tracking, + email=props.email, + enable_sms_role=props.enable_sms_role, + lambda_triggers=props.lambda_triggers, + mfa=mfa, + mfa_message=props.mfa_message, + mfa_second_factor=mfa_second_factor, + password_policy=password_policy, + removal_policy=removal_policy, + self_sign_up_enabled=self_sign_up_enabled, + sign_in_aliases=sign_in_aliases, + sign_in_case_sensitive=sign_in_case_sensitive, + sms_role=props.sms_role, + sms_role_external_id=props.sms_role_external_id, + standard_attributes=standard_attributes, + user_invitation=user_invitation, + user_pool_name=user_pool_name, + user_verification=props.user_verification + ) + self.add_common_tags(self.user_pool) + + # MFA + self.add_nag_suppression(construct=self.user_pool, suppressions=[ + IdeaNagSuppression(rule_id='AwsSolutions-COG2', reason='Suppress MFA warning. MFA provided by customer IdP/SSO methods.') + ]) + + # advanced security mode suppression + self.add_nag_suppression(construct=self.user_pool, suppressions=[ + IdeaNagSuppression(rule_id='AwsSolutions-COG3', reason='suppress advanced security rule 1/to save cost, 2/Not supported in GovCloud') + ]) + + group_name_helper = GroupNameHelper(self.context) + + cognito.CfnUserPoolGroup( + scope=self.scope, + id=f'{user_pool_name}-administrators-group', + description='Administrators group (Sudo Users)', + group_name=group_name_helper.get_cluster_administrators_group(), + precedence=1, + user_pool_id=self.user_pool.user_pool_id + ) + + cognito.CfnUserPoolGroup( + scope=self.scope, + id=f'{user_pool_name}-managers-group', + description='Managers group with limited administration access.', + group_name=group_name_helper.get_cluster_managers_group(), + precedence=2, + user_pool_id=self.user_pool.user_pool_id + ) + + domain_url = self.context.config().get_string('identity-provider.cognito.domain_url') + if Utils.is_not_empty(domain_url): + domain_prefix = domain_url.replace('https://', '').split('.')[0] + else: + domain_prefix = f'{self.cluster_name}-{Utils.uuid()}' + + self.domain = self.user_pool.add_domain( + id='domain', + cognito_domain=cognito.CognitoDomainOptions( + domain_prefix=domain_prefix + ) + ) diff --git a/source/idea/idea-administrator/src/ideaadministrator/app/cdk/constructs/dns.py b/source/idea/idea-administrator/src/ideaadministrator/app/cdk/constructs/dns.py new file mode 100644 index 00000000..02fbf1c1 --- /dev/null +++ b/source/idea/idea-administrator/src/ideaadministrator/app/cdk/constructs/dns.py @@ -0,0 +1,118 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +__all__ = ( + 'DNSResolverEndpoint', + 'DNSResolverRule', + 'PrivateHostedZone' +) + +from ideaadministrator.app.cdk.constructs import ( + SocaBaseConstruct +) +from ideaadministrator.app_context import AdministratorContext + +from typing import List, Optional +import constructs +from aws_cdk import ( + aws_ec2 as ec2, + aws_route53 as route53, + aws_route53resolver as route53resolver +) + + +class DNSResolverEndpoint(SocaBaseConstruct): + + def __init__(self, context: AdministratorContext, name: str, scope: constructs.Construct, + vpc: ec2.IVpc, + security_group_ids: List[str], + subnet_ids: Optional[List[str]], + direction: str = 'OUTBOUND'): + super().__init__(context, name) + self.scope = scope + self.vpc = vpc + self.subnet_ids = subnet_ids + self.security_group_ids = security_group_ids + self.direction = direction + + self.resolver_endpoint = self.build_resolver_endpoint() + + def build_resolver_endpoint(self) -> route53resolver.CfnResolverEndpoint: + ip_addresses = [] + for subnet_id in self.subnet_ids: + ip_addresses.append(route53resolver.CfnResolverEndpoint.IpAddressRequestProperty(subnet_id=subnet_id)) + + resolver_endpoint = route53resolver.CfnResolverEndpoint( + scope=self.scope, + id=f'{self.name}-dns-resolver-endpoint', + direction=self.direction, + name=self.name, + ip_addresses=ip_addresses, + security_group_ids=self.security_group_ids + ) + self.add_common_tags(resolver_endpoint) + + return resolver_endpoint + + +class DNSResolverRule(SocaBaseConstruct): + + def __init__(self, context: AdministratorContext, name: str, scope: constructs.Construct, + domain_name: str, + vpc: ec2.IVpc, + resolver_endpoint_id: str, + ip_addresses: List[str], + rule_type: str = 'FORWARD', + port: str = None): + super().__init__(context, name) + self.scope = scope + self.domain_name = domain_name + self.vpc = vpc + self.resolver_endpoint_id = resolver_endpoint_id + + target_ips = [] + for ip_address in ip_addresses: + target_ips.append(route53resolver.CfnResolverRule.TargetAddressProperty( + ip=ip_address, + port=port + )) + + self.resolver_rule = route53resolver.CfnResolverRule( + scope=self.scope, + id=f'{self.name}-dns-resolver-rule', + name=f'{self.name}-dns-resolver-rule', + domain_name=self.domain_name, + rule_type=rule_type, + resolver_endpoint_id=self.resolver_endpoint_id, + target_ips=target_ips + ) + self.add_common_tags(self.resolver_rule) + + self.resolver_rule_assoc = route53resolver.CfnResolverRuleAssociation( + scope=self.scope, + id=f'{self.name}-dns-resolver-rule-association', + resolver_rule_id=self.resolver_rule.attr_resolver_rule_id, + vpc_id=self.vpc.vpc_id + ) + self.add_common_tags(self.resolver_rule_assoc) + + +class PrivateHostedZone(SocaBaseConstruct, route53.PrivateHostedZone): + + def __init__(self, context: AdministratorContext, scope: constructs.Construct, vpc: ec2.IVpc): + self.context = context + zone_name = self.context.config().get_string('cluster.route53.private_hosted_zone_name', required=True) + super().__init__(context=context, + name=f'{self.cluster_name}-private-hosted-zone', + scope=scope, + vpc=vpc, + comment=f'Private Hosted Zone for IDEA Cluster: {self.cluster_name}', + zone_name=zone_name) diff --git a/source/idea/idea-administrator/src/ideaadministrator/app/cdk/constructs/existing_resources.py b/source/idea/idea-administrator/src/ideaadministrator/app/cdk/constructs/existing_resources.py new file mode 100644 index 00000000..6c5fdcbb --- /dev/null +++ b/source/idea/idea-administrator/src/ideaadministrator/app/cdk/constructs/existing_resources.py @@ -0,0 +1,181 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +__all__ = ( + 'ExistingVpc', + 'ExistingSocaCluster' +) + +from ideasdk.utils import Utils + +from ideaadministrator.app_context import AdministratorContext + +from ideaadministrator.app.cdk.constructs import ( + SocaBaseConstruct +) + +from typing import List, Optional, Dict + +import constructs +from aws_cdk import ( + aws_ec2 as ec2, + aws_iam as iam +) + + +class ExistingVpc(SocaBaseConstruct): + """ + Class encapsulating Existing VPC + Acts as an intermediate entity to retrieve subnets that are configured in cluster config, instead of returning all subnets in Vpc. + Downstream stacks should use ExistingVpc instead of ec2.Vpc to retrieve subnet information. + """ + + def __init__(self, context: AdministratorContext, name: str, scope: constructs.Construct): + super().__init__(context, name) + self.scope = scope + self.vpc_id = self.context.config().get_string('cluster.network.vpc_id', required=True) + self.vpc = ec2.Vpc.from_lookup(self.scope, 'vpc', vpc_id=self.vpc_id) + self._private_subnets: Optional[List[ec2.ISubnet]] = None + self._public_subnets: Optional[List[ec2.ISubnet]] = None + + def lookup_vpc(self): + self.vpc = ec2.Vpc.from_lookup(self.scope, 'vpc', vpc_id=self.vpc_id) + + def get_public_subnet_ids(self) -> List[str]: + return self.context.config().get_list('cluster.network.public_subnets', []) + + def get_private_subnet_ids(self) -> List[str]: + return self.context.config().get_list('cluster.network.private_subnets', []) + + def get_public_subnets(self) -> List[ec2.ISubnet]: + """ + filter subnets from Vpc based on subnet ids configured in `cluster.network.public_subnets` + the result is sorted based on the order of subnet ids provided in the configuration. + """ + if self._public_subnets is not None: + return self._public_subnets + + public_subnet_ids = self.get_public_subnet_ids() + if Utils.is_empty(public_subnet_ids): + self._public_subnets = [] + return self._public_subnets + + result = [] + if self.vpc.public_subnets is not None: + for subnet in self.vpc.public_subnets: + if subnet.subnet_id in public_subnet_ids: + result.append(subnet) + + # sort based on index in public_subnets[] configuration + result.sort(key=lambda x: public_subnet_ids.index(x.subnet_id)) + + self._public_subnets = result + return self._public_subnets + + def get_private_subnets(self) -> List[ec2.ISubnet]: + """ + filter subnets from Vpc based on subnet ids configured in `cluster.network.private_subnets` + the result is sorted based on the order of subnet ids provided in the configuration. + + The order of subnets is important in below use cases: + * private_subnets[0] is anchored for single-zone file systems. + * Amazon EFS mount points are provisioned based on the order of subnets in configuration. if the order is changed, that could result in upgrade failures for net-new shared-storage module configurations. + + Note for Isolated Subnets: + * IDEA does not support pure isolated subnets. The expectation is internet access is available for all nodes either via TransitGateway or by some other means in the associated RouteTable for the subnet. + * CDK automagically buckets the subnets under private and isolated based on NAT Gateway configuration. This scenario is applicable when admin uses existing resources flow, where + Isolated Subnets (subnets without NAT gateway attached to RouteTable) are configured as IDEA "private subnets". + * After lookup, ec2.IVpc buckets the subnets under public_subnets, private_subnets and isolated_subnets. + * To ensure all configured subnets are selected, both vpc.private_subnets and vpc.isolated_subnets are checked to resolve ec2.ISubnet + """ + if self._private_subnets is not None: + return self._private_subnets + + private_subnet_ids = self.get_private_subnet_ids() + if Utils.is_empty(private_subnet_ids): + self._private_subnets = [] + return self._public_subnets + + result = [] + if self.vpc.private_subnets is not None: + for subnet in self.vpc.private_subnets: + if subnet.subnet_id in private_subnet_ids: + result.append(subnet) + if self.vpc.isolated_subnets is not None: + for subnet in self.vpc.isolated_subnets: + if subnet.subnet_id in private_subnet_ids: + result.append(subnet) + + # sort based on index in private_subnets[] configuration + result.sort(key=lambda x: private_subnet_ids.index(x.subnet_id)) + + self._private_subnets = result + return self._private_subnets + + +class ExistingSocaCluster(SocaBaseConstruct): + + def __init__(self, context: AdministratorContext, scope: constructs.Construct): + super().__init__(context, 'existing-cluster') + + self.scope = scope + + self.existing_vpc = ExistingVpc( + context=self.context, + name='existing-vpc', + scope=self.scope + ) + self.security_groups: Dict[str, ec2.ISecurityGroup] = {} + self.roles: Dict[str, iam.IRole] = {} + + # IAM roles + self.lookup_roles() + + # security groups + self.lookup_security_groups() + + @property + def vpc(self) -> ec2.IVpc: + return self.existing_vpc.vpc + + @property + def public_subnets(self) -> List[ec2.ISubnet]: + return self.existing_vpc.get_public_subnets() + + @property + def private_subnets(self) -> List[ec2.ISubnet]: + return self.existing_vpc.get_private_subnets() + + def lookup_roles(self): + roles = self.context.config().get_config('cluster.iam.roles', required=True) + for name in roles: + self.roles[name] = iam.Role.from_role_arn( + self.scope, + f'{name}-role', + role_arn=roles[name] + ) + + def get_role(self, name: str) -> iam.IRole: + if not Utils.value_exists(name, self.roles): + pass + return Utils.get_any_value(name, self.roles) + + def lookup_security_groups(self): + security_groups = self.context.config().get_config('cluster.network.security_groups', required=True) + for name in security_groups: + self.security_groups[name] = ec2.SecurityGroup.from_security_group_id( + self.scope, + f'{name}-security-group', + security_group_id=security_groups[name] + ) + + def get_security_group(self, name: str) -> Optional[ec2.ISecurityGroup]: + return Utils.get_any_value(name, self.security_groups) diff --git a/source/idea/idea-administrator/src/ideaadministrator/app/cdk/constructs/network.py b/source/idea/idea-administrator/src/ideaadministrator/app/cdk/constructs/network.py new file mode 100644 index 00000000..f5ab3c2b --- /dev/null +++ b/source/idea/idea-administrator/src/ideaadministrator/app/cdk/constructs/network.py @@ -0,0 +1,683 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +__all__ = ( + 'ElasticIP', + 'Vpc', + 'SecurityGroup', + 'BastionHostSecurityGroup', + 'ExternalLoadBalancerSecurityGroup', + 'InternalLoadBalancerSecurityGroup', + 'SharedStorageSecurityGroup', + 'OpenLDAPServerSecurityGroup', + 'WebPortalSecurityGroup', + 'SchedulerSecurityGroup', + 'ComputeNodeSecurityGroup', + 'VpcEndpointSecurityGroup', + 'VpcGatewayEndpoint', + 'VpcInterfaceEndpoint', + 'OpenSearchSecurityGroup', + 'DefaultClusterSecurityGroup', + 'VirtualDesktopPublicLoadBalancerAccessSecurityGroup', + 'VirtualDesktopBastionAccessSecurityGroup' +) + +from typing import List, Optional, Dict + +import aws_cdk as cdk +import constructs +from aws_cdk import ( + aws_ec2 as ec2, + aws_logs as logs, + aws_iam as iam +) + +from ideaadministrator.app.cdk.constructs import ( + SocaBaseConstruct, + CreateTagsCustomResource, + IdeaNagSuppression +) +from ideaadministrator.app_context import AdministratorContext +from ideadatamodel import constants +from ideasdk.utils import Utils + + +class ElasticIP(SocaBaseConstruct, ec2.CfnEIP): + + def __init__(self, context: AdministratorContext, name: str, scope: constructs.Construct): + self.context = context + super().__init__(context, name, scope) + + +class Vpc(SocaBaseConstruct, ec2.Vpc): + + def __init__(self, context: AdministratorContext, name: str, scope: constructs.Construct): + self.context = context + self.scope = scope + super().__init__(context, name, scope, + cidr=context.config().get_string('cluster.network.vpc_cidr_block'), + nat_gateways=context.config().get_int('cluster.network.nat_gateways'), + enable_dns_support=True, + enable_dns_hostnames=True, + max_azs=context.config().get_int('cluster.network.max_azs'), + subnet_configuration=self.build_subnet_configuration(), + flow_logs=self.build_flow_logs()) + + def build_flow_logs(self) -> Optional[Dict[str, ec2.FlowLogOptions]]: + vpc_flow_logs = self.context.config().get_bool('cluster.network.vpc_flow_logs', False) + if not vpc_flow_logs: + return None + + vpc_flow_logs_removal_policy = self.context.config().get_string('cluster.network.vpc_flow_logs_removal_policy', 'DESTROY') + log_group_name = self.context.config().get_string('cluster.network.vpc_flow_logs_group_name', f'{self.cluster_name}-vpc-flow-logs') + log_group = logs.LogGroup(self.scope, 'vpc-flow-logs-group', + log_group_name=log_group_name, + removal_policy=cdk.RemovalPolicy(vpc_flow_logs_removal_policy)) + iam_role = iam.Role(self.scope, 'vpc-flow-logs-role', + assumed_by=self.build_service_principal('vpc-flow-logs'), + description=f'IAM Role for VPC Flow Logs, Cluster: {self.cluster_name}', + role_name=f'{self.cluster_name}-vpc-flow-logs-{self.context.aws().aws_region()}') + return { + 'cloud-watch': ec2.FlowLogOptions( + destination=ec2.FlowLogDestination.to_cloud_watch_logs( + log_group=log_group, + iam_role=iam_role + ), + traffic_type=ec2.FlowLogTrafficType.ALL + ) + } + + @property + def nat_gateway_ips(self) -> List[constructs.IConstruct]: + result = [] + if self.public_subnets is None: + return result + for subnet in self.public_subnets: + eip = subnet.node.try_find_child('EIP') + if eip is None: + continue + result.append(eip) + return result + + def build_subnet_configuration(self) -> List[ec2.SubnetConfiguration]: + result = [] + # a public subnet always is required to support cognito access as Cognito does support VPC endpoints. + public_subnet_config = self.build_public_subnet_config() + result.append(public_subnet_config) + + # private can be optional + private_subnet_config = self.build_private_subnet_config() + if private_subnet_config is not None: + result.append(private_subnet_config) + + # isolated can be optional + isolated_subnet_config = self.build_isolated_subnet_config() + if isolated_subnet_config is not None: + result.append(isolated_subnet_config) + + return result + + def build_public_subnet_config(self) -> ec2.SubnetConfiguration: + cidr_mask = self.context.config().get_int('cluster.network.subnet_config.public.cidr_mask', 26) + return ec2.SubnetConfiguration( + name='public', + cidr_mask=cidr_mask, + subnet_type=ec2.SubnetType.PUBLIC + ) + + def build_private_subnet_config(self) -> Optional[ec2.SubnetConfiguration]: + cidr_mask = self.context.config().get_int('cluster.network.subnet_config.private.cidr_mask', 18) + return ec2.SubnetConfiguration( + name='private', + cidr_mask=cidr_mask, + subnet_type=ec2.SubnetType.PRIVATE_WITH_NAT + ) + + def build_isolated_subnet_config(self) -> Optional[ec2.SubnetConfiguration]: + """ + build isolated subnet only if subnet config is provided. + do not return any default value. + :return: isolated subnet configuration + """ + + cidr_mask = self.context.config().get_int('cluster.network.subnet_config.isolated.cidr_mask', None) + if cidr_mask is None: + return None + + return ec2.SubnetConfiguration( + name='isolated', + cidr_mask=cidr_mask, + subnet_type=ec2.SubnetType.PRIVATE_ISOLATED + ) + + @property + def public_subnet_ids(self) -> List[str]: + result = [] + for subnet in self.public_subnets: + result.append(subnet.subnet_id) + return result + + @property + def private_subnet_ids(self) -> List[str]: + result = [] + for subnet in self.private_subnets: + result.append(subnet.subnet_id) + return result + + +class SecurityGroup(SocaBaseConstruct, ec2.SecurityGroup): + + def __init__(self, context: AdministratorContext, name: str, scope: constructs.Construct, + vpc: ec2.IVpc, + allow_all_outbound=False, + description=Optional[str]): + self.context = context + super().__init__(context, name, scope, + security_group_name=self.build_resource_name(name), + vpc=vpc, + allow_all_outbound=allow_all_outbound, + description=description) + self.vpc = vpc + self.add_nag_suppression(suppressions=[]) + + def add_nag_suppression(self, suppressions: List[IdeaNagSuppression], construct: constructs.IConstruct = None, apply_to_children: bool = True): + updated_suppressions = [ + # [Warning at /idea-test1-cluster/external-load-balancer-security-group/Resource] CdkNagValidationFailure: 'AwsSolutions-EC23' threw an error during validation. This is generally caused by a parameter referencing an intrinsic function. For more details enable verbose logging.' + IdeaNagSuppression(rule_id='AwsSolutions-EC23', reason='suppress warning: parameter referencing intrinsic function') + ] + if suppressions: + updated_suppressions += suppressions + super().add_nag_suppression(updated_suppressions, construct, apply_to_children) + + def add_outbound_traffic_rule(self): + self.add_egress_rule( + ec2.Peer.ipv4('0.0.0.0/0'), + ec2.Port.tcp_range(0, 65535), + description='Allow all egress for TCP' + ) + + def add_api_ingress_rule(self): + self.add_ingress_rule( + ec2.Peer.ipv4(self.vpc.vpc_cidr_block), + ec2.Port.tcp(8443), + description='Allow HTTP traffic from all VPC nodes for API access' + ) + + def add_loadbalancer_ingress_rule(self, loadbalancer_security_group: ec2.ISecurityGroup): + self.add_ingress_rule( + loadbalancer_security_group, + ec2.Port.tcp(8443), + description='Allow HTTPs traffic from Load Balancer' + ) + + def add_bastion_host_ingress_rule(self, bastion_host_security_group: ec2.ISecurityGroup): + self.add_ingress_rule( + bastion_host_security_group, + ec2.Port.tcp(22), + description='Allow SSH from Bastion Host') + + def add_active_directory_rules(self): + self.add_ingress_rule( + ec2.Peer.ipv4(self.vpc.vpc_cidr_block), + ec2.Port.udp_range(0, 1024), + description='Allow UDP Traffic from VPC. Required for Directory Service' + ) + self.add_egress_rule( + ec2.Peer.ipv4('0.0.0.0/0'), + ec2.Port.udp_range(0, 1024), + description='Allow UDP Traffic. Required for Directory Service' + ) + + +class BastionHostSecurityGroup(SecurityGroup): + """ + Security Group for Bastion Host + Only instance that will be in Public Subnet. All other instances will be launched in private subnets. + """ + + def __init__(self, context: AdministratorContext, name: str, scope: constructs.Construct, vpc: ec2.IVpc, cluster_prefix_list_id: str): + super().__init__(context, name, scope, vpc, description='Bastion host security group') + self.cluster_prefix_list_id = cluster_prefix_list_id + self.setup_ingress() + self.setup_egress() + + if self.is_ds_activedirectory(): + self.add_active_directory_rules() + + def setup_ingress(self): + cluster_prefix_list = ec2.Peer.prefix_list(self.cluster_prefix_list_id) + self.add_ingress_rule( + cluster_prefix_list, + ec2.Port.tcp(22), + description='Allow SSH access from Cluster Prefix List to Bastion Host' + ) + + prefix_list_ids = self.context.config().get_list('cluster.network.prefix_list_ids') + if Utils.is_not_empty(prefix_list_ids): + for prefix_list_id in prefix_list_ids: + prefix_list = ec2.Peer.prefix_list(prefix_list_id) + self.add_ingress_rule( + prefix_list, + ec2.Port.tcp(22), + description='Allow SSH access from Prefix List to Bastion Host' + ) + + self.add_ingress_rule( + ec2.Peer.ipv4(self.vpc.vpc_cidr_block), + ec2.Port.tcp(22), + description='Allow SSH traffic from all VPC nodes' + ) + + def setup_egress(self): + self.add_outbound_traffic_rule() + + +class ExternalLoadBalancerSecurityGroup(SecurityGroup): + """ + External Load Balancer Security Group + """ + + def __init__(self, context: AdministratorContext, name: str, scope: constructs.Construct, vpc: ec2.IVpc, + cluster_prefix_list_id: str, + bastion_host_security_group: ec2.ISecurityGroup): + super().__init__(context, name, scope, vpc, description='External Application Load Balancer security group') + self.cluster_prefix_list_id = cluster_prefix_list_id + self.bastion_host_security_group = bastion_host_security_group + self.setup_ingress() + self.setup_egress() + + def add_peer_ingress_rule(self, peer: ec2.IPeer, peer_type: str): + self.add_ingress_rule( + peer, + ec2.Port.tcp(443), + description=f'Allow HTTPS access from {peer_type} to ALB' + ) + + self.add_ingress_rule( + peer, + ec2.Port.tcp(80), + description=f'Allow HTTP access from {peer_type} to ALB' + ) + + def setup_ingress(self): + cluster_prefix_list = ec2.Peer.prefix_list(self.cluster_prefix_list_id) + self.add_peer_ingress_rule(cluster_prefix_list, 'Cluster Prefix List') + + prefix_list_ids = self.context.config().get_list('cluster.network.prefix_list_ids') + if Utils.is_not_empty(prefix_list_ids): + for prefix_list_id in prefix_list_ids: + if Utils.is_not_empty(prefix_list_id): + prefix_list = ec2.Peer.prefix_list(prefix_list_id) + self.add_peer_ingress_rule(prefix_list, 'Prefix List') + + self.add_ingress_rule( + self.bastion_host_security_group, + ec2.Port.tcp(80), + description='Allow HTTP from Bastion Host') + + self.add_ingress_rule( + self.bastion_host_security_group, + ec2.Port.tcp(443), + description='Allow HTTPs from Bastion Host') + + def setup_egress(self): + self.add_outbound_traffic_rule() + + def add_nat_gateway_ips_ingress_rule(self, nat_gateway_ips: List[constructs.IConstruct]): + # allow NAT EIP to communicate with ALB. This so that virtual desktop instances or instances + # in private subnets can open the Web Portal or Access the APIs using the ALB endpoint + for eip in nat_gateway_ips: + self.add_ingress_rule( + ec2.Peer.ipv4(f'{eip.ref}/32'), + ec2.Port.tcp(443), + description=f'Allow NAT EIP to communicate to ALB.' + ) + + +class SharedStorageSecurityGroup(SecurityGroup): + """ + Shared Storage Security Group + Attached to EFS and FSx File Systems. Access is open to all nodes from VPC. + + Note for Lustre rules: + Although Lustre is optional and may not be used for /apps or /data, + compute nodes can mount FSx Lustre on-demand to /scratch. For this reason, Lustre rules are provisioned during cluster creation. + """ + + def __init__(self, context: AdministratorContext, name: str, scope: constructs.Construct, vpc: ec2.IVpc): + super().__init__(context, name, scope, vpc, description='Shared Storage security group for EFS/FSx file systems') + self.setup_ingress() + self.setup_egress() + + def setup_ingress(self): + # NFS + self.add_ingress_rule( + ec2.Peer.ipv4(self.vpc.vpc_cidr_block), + ec2.Port.tcp(2049), + description='Allow NFS traffic from all VPC nodes to EFS') + + # FSx for Lustre + self.add_ingress_rule( + ec2.Peer.ipv4(self.vpc.vpc_cidr_block), + ec2.Port.tcp(988), + description='Allow FSx Lustre traffic from all VPC nodes') + self.add_ingress_rule( + ec2.Peer.ipv4(self.vpc.vpc_cidr_block), + ec2.Port.tcp_range(1021, 1023), + description='Allow FSx Lustre traffic from all VPC nodes') + + def setup_egress(self): + self.add_outbound_traffic_rule() + + +class OpenLDAPServerSecurityGroup(SecurityGroup): + """ + OpenLDAP Server Security Group + """ + + def __init__(self, context: AdministratorContext, name: str, scope: constructs.Construct, + vpc: ec2.IVpc, + bastion_host_security_group: ec2.ISecurityGroup): + super().__init__(context, name, scope, vpc, description='OpenLDAP server security group') + self.bastion_host_security_group = bastion_host_security_group + self.setup_ingress() + self.setup_egress() + + def setup_ingress(self): + self.add_ingress_rule( + ec2.Peer.ipv4(self.vpc.vpc_cidr_block), + ec2.Port.tcp(389), + description='Allow LDAP traffic from all VPC nodes' + ) + self.add_api_ingress_rule() + self.add_bastion_host_ingress_rule(self.bastion_host_security_group) + + def setup_egress(self): + self.add_outbound_traffic_rule() + + +class WebPortalSecurityGroup(SecurityGroup): + """ + Cluster Manager Security Group + """ + + def __init__(self, context: AdministratorContext, name: str, scope: constructs.Construct, + vpc: ec2.IVpc, + bastion_host_security_group: ec2.ISecurityGroup, + loadbalancer_security_group: ec2.ISecurityGroup): + super().__init__(context, name, scope, vpc, description='Web Portal security group') + self.bastion_host_security_group = bastion_host_security_group + self.loadbalancer_security_group = loadbalancer_security_group + self.setup_ingress() + self.setup_egress() + if self.is_ds_activedirectory(): + self.add_active_directory_rules() + + def setup_ingress(self): + self.add_api_ingress_rule() + self.add_bastion_host_ingress_rule(self.bastion_host_security_group) + self.add_loadbalancer_ingress_rule(self.loadbalancer_security_group) + + def setup_egress(self): + self.add_outbound_traffic_rule() + + +class SchedulerSecurityGroup(SecurityGroup): + """ + HPC Scheduler Security Group + """ + + def __init__(self, context: AdministratorContext, name: str, scope: constructs.Construct, + vpc: ec2.IVpc, + bastion_host_security_group: ec2.ISecurityGroup, + loadbalancer_security_group: ec2.ISecurityGroup): + super().__init__(context, name, scope, vpc, description='Scheduler security group') + self.bastion_host_security_group = bastion_host_security_group + self.loadbalancer_security_group = loadbalancer_security_group + self.setup_ingress() + self.setup_egress() + if self.is_ds_activedirectory(): + self.add_active_directory_rules() + + def setup_ingress(self): + self.add_api_ingress_rule() + + self.add_ingress_rule( + ec2.Peer.ipv4(self.vpc.vpc_cidr_block), + ec2.Port.tcp_range(0, 65535), + description='Allow all TCP traffic from VPC to scheduler' + ) + + self.add_bastion_host_ingress_rule(self.bastion_host_security_group) + + self.add_loadbalancer_ingress_rule(self.loadbalancer_security_group) + + def setup_egress(self): + self.add_outbound_traffic_rule() + + +class ComputeNodeSecurityGroup(SecurityGroup): + + def __init__(self, context: AdministratorContext, name: str, scope: constructs.Construct, + vpc: ec2.IVpc): + super().__init__(context, name, scope, vpc, description='Compute Node security group') + self.setup_ingress() + self.setup_egress() + if self.is_ds_activedirectory(): + self.add_active_directory_rules() + + def setup_ingress(self): + self.add_ingress_rule( + ec2.Peer.ipv4(self.vpc.vpc_cidr_block), + ec2.Port.tcp_range(0, 65535), + description='All TCP traffic from all VPC nodes to compute node' + ) + + self.add_ingress_rule( + self, + ec2.Port.all_traffic(), + description='Allow all traffic between compute nodes and EFA' + ) + + def setup_egress(self): + self.add_outbound_traffic_rule() + + self.add_egress_rule( + self, + ec2.Port.all_traffic(), + description='Allow all traffic between compute nodes and EFA' + ) + + +class VirtualDesktopBastionAccessSecurityGroup(SecurityGroup): + """ + Virtual Desktop Security Group with Bastion Access + """ + component_name: str + + def __init__(self, context: AdministratorContext, name: str, scope: constructs.Construct, vpc: ec2.IVpc, + bastion_host_security_group: ec2.ISecurityGroup, description: str, directory_service_access: bool, component_name: str): + super().__init__(context, name, scope, vpc, description=description) + self.component_name = component_name + self.bastion_host_security_group = bastion_host_security_group + self.setup_ingress() + self.setup_egress() + if directory_service_access and self.is_ds_activedirectory(): + self.add_active_directory_rules() + + def setup_ingress(self): + self.add_api_ingress_rule() + self.add_ingress_rule( + ec2.Peer.ipv4(self.vpc.vpc_cidr_block), + ec2.Port.all_traffic(), + description=f'Allow all Internal traffic TO {self.component_name}' + ) + self.add_bastion_host_ingress_rule(self.bastion_host_security_group) + + def setup_egress(self): + self.add_outbound_traffic_rule() + + +class VirtualDesktopPublicLoadBalancerAccessSecurityGroup(SecurityGroup): + """ + Virtual Desktop Security Group with Bastion and Public Loadbalancer Access + """ + component_name: str + + def __init__(self, context: AdministratorContext, name: str, scope: constructs.Construct, vpc: ec2.IVpc, + bastion_host_security_group: ec2.ISecurityGroup, description: str, directory_service_access: bool, component_name: str, + public_loadbalancer_security_group: ec2.ISecurityGroup): + super().__init__(context, name, scope, vpc, description=description) + self.component_name = component_name + self.public_loadbalancer_security_group = public_loadbalancer_security_group + self.bastion_host_security_group = bastion_host_security_group + self.setup_ingress() + self.setup_egress() + if directory_service_access and self.is_ds_activedirectory(): + self.add_active_directory_rules() + + def setup_ingress(self): + self.add_api_ingress_rule() + self.add_ingress_rule( + ec2.Peer.ipv4(self.vpc.vpc_cidr_block), + ec2.Port.all_traffic(), + description=f'Allow all Internal traffic TO {self.component_name}' + ) + self.add_bastion_host_ingress_rule(self.bastion_host_security_group) + self.add_loadbalancer_ingress_rule(self.public_loadbalancer_security_group) + + def setup_egress(self): + self.add_outbound_traffic_rule() + + +class VpcEndpointSecurityGroup(SecurityGroup): + + def __init__(self, context: AdministratorContext, name: str, scope: constructs.Construct, vpc: ec2.IVpc): + super().__init__(context, name, scope, vpc, allow_all_outbound=True, description='VPC Endpoints Security Group') + self.add_ingress_rule( + ec2.Peer.ipv4(self.vpc.vpc_cidr_block), + ec2.Port.tcp(443), + description='Allow HTTPS traffic from VPC' + ) + + +class VpcGatewayEndpoint(SocaBaseConstruct): + + def __init__(self, context: AdministratorContext, scope: constructs.Construct, + service: str, + vpc: ec2.IVpc, + create_tags: CreateTagsCustomResource): + super().__init__(context, f'{service}-gateway-endpoint') + self.scope = scope + + self.endpoint = vpc.add_gateway_endpoint( + self.construct_id, + service=ec2.GatewayVpcEndpointAwsService( + name=service + ) + ) + + create_tags.apply( + name=self.name, + resource_id=self.endpoint.vpc_endpoint_id, + tags={ + constants.IDEA_TAG_NAME: self.name, + constants.IDEA_TAG_CLUSTER_NAME: self.cluster_name + } + ) + + +class VpcInterfaceEndpoint(SocaBaseConstruct): + + def __init__(self, context: AdministratorContext, scope: constructs.Construct, + service: str, + vpc: ec2.IVpc, + vpc_endpoint_security_group: ec2.ISecurityGroup, + create_tags: CreateTagsCustomResource): + super().__init__(context, f'{service}-vpc-endpoint') + self.scope = scope + + # this is a change from 2.x behaviour, where access to VPC endpoints was restricted by security group. + # in 3.x, since security groups for individual components do not exist during cluster/network creation, + # all VPC traffic can communicate with VPC Interface endpoints. + self.endpoint = vpc.add_interface_endpoint(self.construct_id, + service=ec2.InterfaceVpcEndpointAwsService( + name=service + ), + open=True, + # setting private_dns_enabled = True can be problem in GovCloud where Route53 and in turn Private Hosted Zones is not supported. + private_dns_enabled=False, + lookup_supported_azs=True, + security_groups=[vpc_endpoint_security_group]) + + create_tags.apply( + name=self.name, + resource_id=self.endpoint.vpc_endpoint_id, + tags={ + constants.IDEA_TAG_NAME: self.name, + constants.IDEA_TAG_CLUSTER_NAME: self.cluster_name + } + ) + + def get_endpoint_url(self) -> str: + dns = cdk.Fn.select(1, cdk.Fn.split(':', cdk.Fn.select(0, self.endpoint.vpc_endpoint_dns_entries))) + return f'https://{dns}' + + +class OpenSearchSecurityGroup(SecurityGroup): + + def __init__(self, context: AdministratorContext, name: str, scope: constructs.Construct, + vpc: ec2.IVpc): + super().__init__(context, name, scope, vpc, description='OpenSearch security group') + self.setup_ingress() + self.setup_egress() + + def setup_ingress(self): + self.add_ingress_rule( + ec2.Peer.ipv4(self.vpc.vpc_cidr_block), + ec2.Port.tcp(443), + description='Allow HTTPS traffic from all VPC nodes to OpenSearch' + ) + + def setup_egress(self): + self.add_outbound_traffic_rule() + + +class DefaultClusterSecurityGroup(SecurityGroup): + """ + Default Cluster Security Group with no inbound or outbound rules. + """ + + def __init__(self, context: AdministratorContext, name: str, scope: constructs.Construct, + vpc: ec2.IVpc): + super().__init__(context, name, scope, vpc, description='Default Cluster Security') + + +class InternalLoadBalancerSecurityGroup(SecurityGroup): + """ + Internal Load Balancer Security Group + """ + + def __init__(self, context: AdministratorContext, name: str, scope: constructs.Construct, + vpc: ec2.IVpc): + super().__init__(context, name, scope, vpc, description='Internal load balancer security group') + self.setup_ingress() + self.setup_egress() + + def setup_ingress(self): + self.add_ingress_rule( + ec2.Peer.ipv4(self.vpc.vpc_cidr_block), + ec2.Port.tcp(443), + description='Allow HTTPS traffic from all VPC nodes to OpenSearch' + ) + + def setup_egress(self): + self.add_outbound_traffic_rule() diff --git a/source/idea/idea-administrator/src/ideaadministrator/app/cdk/constructs/storage.py b/source/idea/idea-administrator/src/ideaadministrator/app/cdk/constructs/storage.py new file mode 100644 index 00000000..6d62b5ed --- /dev/null +++ b/source/idea/idea-administrator/src/ideaadministrator/app/cdk/constructs/storage.py @@ -0,0 +1,319 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +__all__ = ( + 'AmazonEFS', + 'FSxForLustre' +) + +from ideaadministrator.app.cdk.idea_code_asset import IdeaCodeAsset, SupportedLambdaPlatforms +from ideasdk.utils import Utils + +from ideaadministrator.app.cdk.constructs import SocaBaseConstruct +from ideaadministrator.app.cdk.constructs.common import ( + SNSTopic, + SNSSubscription, + CloudWatchAlarm, + Role, + Policy, + LambdaFunction +) +from ideaadministrator.app_context import AdministratorContext + +from typing import List, Optional, Tuple, Dict + +import aws_cdk as cdk +import constructs +from aws_cdk import ( + aws_ec2 as ec2, + aws_lambda as lambda_, + aws_iam as iam, + aws_efs as efs, + aws_fsx as fsx, + aws_cloudwatch as cloudwatch, + aws_cloudwatch_actions as cw_actions, + aws_sns as sns +) + + +class AmazonEFS(SocaBaseConstruct): + + def __init__(self, context: AdministratorContext, name: str, scope: constructs.Construct, + vpc: ec2.IVpc, + security_group: ec2.SecurityGroup, + efs_config: Dict, + subnets: List[ec2.ISubnet] = None, + log_retention_role: Optional[iam.IRole] = None, + ): + super().__init__(context, name) + + self.scope = scope + self.vpc = vpc + self.security_group = security_group + self.kms_key_id = Utils.get_value_as_string('kms_key_id', efs_config) + self.subnets = subnets + self.log_retention_role = log_retention_role + self.cloud_watch_monitoring = Utils.get_value_as_bool('cloudwatch_monitoring', efs_config, False) + removal_policy = Utils.get_value_as_string('removal_policy', efs_config) + if removal_policy == 'DESTROY': + removal_policy = 'DELETE' + self.deletion_policy = cdk.CfnDeletionPolicy(removal_policy) + self.transition_to_ia = Utils.get_value_as_string('transition_to_ia', efs_config) + self.encrypted = Utils.get_value_as_bool('encrypted', efs_config, True) + self.throughput_mode = Utils.get_value_as_string('throughput_mode', efs_config, 'bursting') + self.performance_mode = Utils.get_value_as_string('performance_mode', efs_config, 'generalPurpose') + + file_system, mount_targets = self.build_file_system() + self.file_system = file_system + self.mount_targets = mount_targets + + self.cloud_watch_monitoring: Optional[Tuple[sns.Topic, + cloudwatch.Alarm, + cloudwatch.Alarm, + lambda_.Function, + sns.Subscription]] = None + + if self.cloud_watch_monitoring: + self.cloud_watch_monitoring = self.build_cloudwatch_monitoring() + + def get_subnets(self) -> List[ec2.ISubnet]: + if self.subnets is not None: + return self.subnets + else: + return self.vpc.private_subnets + + def build_file_system(self) -> Tuple[efs.CfnFileSystem, List[efs.CfnMountTarget]]: + + lifecycle_policies = None + if Utils.is_not_empty(self.transition_to_ia): + lifecycle_policies = [efs.CfnFileSystem.LifecyclePolicyProperty(transition_to_ia=self.transition_to_ia)] + + file_system = efs.CfnFileSystem( + scope=self.scope, + id=self.name, + encrypted=self.encrypted, + file_system_tags=[ + efs.CfnFileSystem.ElasticFileSystemTagProperty( + key='Name', + value=self.build_resource_name(self.name) + ) + ], + kms_key_id=self.kms_key_id, + throughput_mode=self.throughput_mode, + performance_mode=self.performance_mode, + lifecycle_policies=lifecycle_policies, + file_system_policy={ + "Version": "2012-10-17", + "Id": "efs-prevent-anonymous-access-policy", + "Statement": [ + { + "Sid": "efs-statement", + "Effect": "Allow", + "Principal": { + "AWS": "*" + }, + "Action": [ + "elasticfilesystem:ClientRootAccess", + "elasticfilesystem:ClientWrite", + "elasticfilesystem:ClientMount" + ], + "Condition": { + "Bool": { + "elasticfilesystem:AccessedViaMountTarget": "true" + } + } + } + ] + } + ) + self.add_common_tags(file_system) + self.add_backup_tags(file_system) + file_system.cfn_options.deletion_policy = self.deletion_policy + + security_group_ids = [self.security_group.security_group_id] + + mount_targets = [] + for index, subnet in enumerate(self.get_subnets()): + mount_target_construct_id = f'{self.construct_id}-mount-target-{index + 1}' + mount_target = efs.CfnMountTarget( + scope=file_system, + id=mount_target_construct_id, + file_system_id=file_system.ref, + security_groups=security_group_ids, + subnet_id=subnet.subnet_id + ) + self.add_common_tags(mount_target) + mount_targets.append(mount_target) + + return file_system, mount_targets + + def build_cloudwatch_monitoring(self) -> Tuple[sns.Topic, + cloudwatch.Alarm, + cloudwatch.Alarm, + lambda_.Function, + sns.Subscription]: + + sns_topic_name = f'{self.name}-alarms' + sns_topic = SNSTopic( + self.context, sns_topic_name, self.file_system, + topic_name=self.build_resource_name(sns_topic_name), + display_name=f'({self.cluster_name}) {sns_topic_name}', + master_key=self.context.config().get_string('cluster.sns.kms_key_id') + ) + + sns_topic_policy = iam.PolicyStatement( + effect=iam.Effect.ALLOW, + actions=['sns:Publish'], + resources=[sns_topic.topic_arn], + principals=[self.build_service_principal('cloudwatch')], + conditions={ + 'ArnLike': { + 'aws:SourceArn': self.aws_account_arn + } + } + ) + sns_topic.add_to_resource_policy(sns_topic_policy) + + alarm_props = { + 'context': self.context, + 'scope': self.file_system, + 'metric_name': 'BurstCreditBalance', + 'metric_namespace': 'AWS/EFS', + 'metric_dimensions': { + 'FileSystemId': self.file_system.ref + }, + 'evaluation_periods': 10, + 'period_seconds': 60, + 'statistic': 'Average', + 'actions': [cw_actions.SnsAction(sns_topic)] + } + + low_alarm = CloudWatchAlarm( + **alarm_props, + name=f'{self.name}-burst-credit-balance-low', + comparison_operator=cloudwatch.ComparisonOperator.LESS_THAN_OR_EQUAL_TO_THRESHOLD, + threshold=10000000 + ) + + high_alarm = CloudWatchAlarm( + **alarm_props, + name=f'{self.name}-burst-credit-balance-high', + comparison_operator=cloudwatch.ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD, + threshold=2000000000000 + ) + + efs_throughput_lambda_name = f'{self.name}-throughput' + efs_throughput_lambda_role = Role( + self.context, + f'{efs_throughput_lambda_name}-role', + self.scope, + f'IAM role to monitor {self.name} throughput for Cluster: {self.cluster_name}', + assumed_by=['lambda'], + inline_policies=[ + Policy( + context=self.context, + name=f'{efs_throughput_lambda_name}-policy', + scope=self.scope, + policy_template_name='efs-throughput-lambda.yml' + ) + ] + ) + + efs_throughput_lambda = LambdaFunction( + context=self.context, + name=efs_throughput_lambda_name, + scope=self.file_system, + idea_code_asset=IdeaCodeAsset( + lambda_package_name='idea_efs_throughput', + lambda_platform=SupportedLambdaPlatforms.PYTHON + ), + role=efs_throughput_lambda_role, + log_retention_role=self.log_retention_role + ) + efs_throughput_lambda.add_environment('EFSBurstCreditLowThreshold', '10000000') + efs_throughput_lambda.add_environment('EFSBurstCreditHighThreshold', '2000000000000') + efs_throughput_lambda.add_permission('InvokePermissions', + principal=self.build_service_principal('sns'), + action='lambda:InvokeFunction') + + sns_topic_subscription = SNSSubscription( + context=self.context, + name=f'{sns_topic_name}-subscription', + scope=sns_topic, + protocol=sns.SubscriptionProtocol.LAMBDA, + endpoint=efs_throughput_lambda.function_arn, + topic=sns_topic + ) + + return sns_topic, low_alarm, high_alarm, efs_throughput_lambda, sns_topic_subscription + + +class FSxForLustre(SocaBaseConstruct): + + def __init__(self, context: AdministratorContext, name: str, scope: constructs.Construct, + vpc: ec2.IVpc, + fsx_lustre_config: Dict, + security_group: ec2.SecurityGroup, + subnets: Optional[List[ec2.ISubnet]] = None): + super().__init__(context, name) + + self.scope = scope + self.vpc = vpc + self.security_group = security_group + self.subnets = subnets + + self.kms_key_id = Utils.get_value_as_string('kms_key_id', fsx_lustre_config) + self.deployment_type = Utils.get_value_as_string('deployment_type', fsx_lustre_config) + self.storage_type = Utils.get_value_as_string('storage_type', fsx_lustre_config) + self.per_unit_storage_throughput = Utils.get_value_as_int('per_unit_storage_throughput', fsx_lustre_config) + self.storage_capacity = Utils.get_value_as_int('storage_capacity', fsx_lustre_config) + self.drive_cache_type = Utils.get_value_as_int('drive_cache_type', fsx_lustre_config) + + self.file_system = self.build_file_system() + + def get_subnets(self) -> List[ec2.ISubnet]: + if self.subnets is not None: + return self.subnets + else: + return self.vpc.private_subnets + + def build_file_system(self) -> fsx.CfnFileSystem: + + deployment_type = self.deployment_type + storage_type = self.storage_type + per_unit_storage_throughput = None + drive_cache_type = None + if storage_type == 'SSD': + if deployment_type == 'PERSISTENT_1': + per_unit_storage_throughput = self.per_unit_storage_throughput + else: + drive_cache_type = self.drive_cache_type + + security_group_ids = [self.security_group.security_group_id] + + first_subnet = self.get_subnets()[0] + file_system = fsx.CfnFileSystem( + scope=self.scope, + id=self.name, + file_system_type='LUSTRE', + subnet_ids=[first_subnet.subnet_id], + lustre_configuration=fsx.CfnFileSystem.LustreConfigurationProperty( + deployment_type=deployment_type, + per_unit_storage_throughput=per_unit_storage_throughput, + drive_cache_type=drive_cache_type + ), + security_group_ids=security_group_ids, + kms_key_id=self.kms_key_id, + storage_capacity=self.storage_capacity + ) + self.add_common_tags(file_system) + self.add_backup_tags(file_system) + return file_system diff --git a/source/idea/idea-administrator/src/ideaadministrator/app/cdk/idea_code_asset.py b/source/idea/idea-administrator/src/ideaadministrator/app/cdk/idea_code_asset.py new file mode 100644 index 00000000..742490ce --- /dev/null +++ b/source/idea/idea-administrator/src/ideaadministrator/app/cdk/idea_code_asset.py @@ -0,0 +1,168 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +import ideaadministrator +from ideaadministrator.app_utils import AdministratorUtils +from ideadatamodel import exceptions + +from ideasdk.shell import ShellInvoker +from ideasdk.utils import Utils + + +from enum import Enum +from typing import Dict +import os +import pathlib +import shutil + + +class SupportedLambdaPlatforms(str, Enum): + PYTHON = 'PYTHON' + + +class IdeaCodeAsset: + """ + Used to build lambda code assets with applicable package dependencies + """ + SOURCE_CODE_CHECKSUM_FILE = 'source.checksum.sha' + PKG_DIR = 'pkg' + PKG_DIR_CHECKSUM_FILE = f'pkg.checksum.sha' + + def __init__(self, lambda_platform: SupportedLambdaPlatforms, lambda_package_name: str): + self._lambda_platform = lambda_platform + self._lambda_package_name = lambda_package_name + self._build_location = os.path.join(AdministratorUtils.get_package_build_dir(), self.lambda_package_name) + self._common_code_location = ideaadministrator.props.lambda_function_commons_dir + + self.PLATFORM_MANAGE_REQUIREMENTS_MAP: Dict[SupportedLambdaPlatforms, ()] = { + SupportedLambdaPlatforms.PYTHON: self._manage_python_requirements + } + + self.PLATFORM_BUILD_MAP: Dict[SupportedLambdaPlatforms, ()] = { + SupportedLambdaPlatforms.PYTHON: self._build_python_lambda + } + + @property + def asset_hash(self) -> str: + return Utils.compute_checksum_for_dir(os.path.join(self._build_location, self.PKG_DIR)) + + @property + def lambda_handler(self) -> str: + return f'{self.lambda_package_name}.handler.handler' + + @property + def lambda_platform(self) -> SupportedLambdaPlatforms: + return self._lambda_platform + + @property + def source_code_location(self) -> str: + return os.path.join(ideaadministrator.props.lambda_functions_dir, self.lambda_package_name) + + @property + def lambda_package_name(self) -> str: + return self._lambda_package_name + + @staticmethod + def _manage_python_requirements(build_src: str, lambda_package_name: str): + requirements_file = os.path.join(build_src, lambda_package_name, 'requirements.txt') + if Utils.is_file(requirements_file): + shutil.move(requirements_file, build_src) + + @staticmethod + def _build_python_lambda(build_location: str): + if not Utils.is_file(os.path.join(build_location, 'requirements.txt')): + return + + shell = ShellInvoker(cwd=build_location) + response = shell.invoke( + shell=True, + cmd=[f'pip install -r requirements.txt --platform manylinux2014_x86_64 --only-binary=:all: --target . --upgrade'], + env=ideaadministrator.props.get_env() + ) + if response.returncode != 0: + raise exceptions.general_exception(f'Issue building the lambda: {response}') + + def _validate_checksum_for_source_code(self) -> bool: + if not Utils.is_dir(self.source_code_location): + return False + + if not Utils.is_dir(self._common_code_location): + return False + + if not Utils.is_file(os.path.join(self._build_location, self.SOURCE_CODE_CHECKSUM_FILE)): + return False + + checksum = Utils.compute_checksum_for_dirs([self.source_code_location, self._common_code_location]) + with open(os.path.join(self._build_location, self.SOURCE_CODE_CHECKSUM_FILE)) as f: + existing_checksum = str(f.read().strip()) + + return existing_checksum == checksum + + def _validate_checksum_for_package(self) -> bool: + if not Utils.is_dir(os.path.join(self._build_location, self.PKG_DIR)): + return False + + if not Utils.is_file(os.path.join(self._build_location, self.PKG_DIR_CHECKSUM_FILE)): + return False + + with open(os.path.join(self._build_location, self.PKG_DIR_CHECKSUM_FILE)) as f: + existing_checksum = str(f.read().strip()) + + return existing_checksum == self.asset_hash + + def build_lambda(self): + if self.lambda_platform not in self.PLATFORM_BUILD_MAP.keys(): + raise exceptions.general_exception(f'Invalid lambda_platform: {self.lambda_platform}. Supported only {self.PLATFORM_BUILD_MAP.keys()}') + + if Utils.is_dir(self._build_location): + if not self._validate_checksum_for_source_code(): + print(f'source updated for {self.lambda_package_name}...') + shutil.rmtree(self._build_location) + elif not self._validate_checksum_for_package(): + print(f'existing package corrupted for lambda: {self.lambda_package_name} ...') + shutil.rmtree(self._build_location) + else: + print(f're-use code assets for lambda: {self.lambda_package_name} ...') + return os.path.join(self._build_location, self.PKG_DIR) + + print(f'building code assets for lambda: {self.lambda_package_name} ...') + build_src = os.path.join(self._build_location, self.PKG_DIR) + parent = pathlib.Path(build_src).parent + if not parent.is_dir(): + os.makedirs(parent) + + # if we have reached here, then we need to build the code again + shutil.copytree(self._common_code_location, os.path.join(build_src, ideaadministrator.props.lambda_function_commons_package_name)) + shutil.copytree(self.source_code_location, os.path.join(build_src, self.lambda_package_name)) + + self.PLATFORM_MANAGE_REQUIREMENTS_MAP[self.lambda_platform]( + build_src=build_src, + lambda_package_name=self.lambda_package_name + ) + + self.PLATFORM_BUILD_MAP[self.lambda_platform]( + build_location=build_src + ) + + shell = ShellInvoker(cwd=self._build_location) + shell.invoke( + shell=True, + cmd=[f'echo {self.asset_hash} > {self.PKG_DIR_CHECKSUM_FILE}'], + env=ideaadministrator.props.get_env(), + ) + + checksum = Utils.compute_checksum_for_dirs([self.source_code_location, self._common_code_location]) + shell.invoke( + shell=True, + cmd=[f'echo {checksum} > {self.SOURCE_CODE_CHECKSUM_FILE}'], + env=ideaadministrator.props.get_env(), + ) + return os.path.join(self._build_location, self.PKG_DIR) diff --git a/source/idea/idea-administrator/src/ideaadministrator/app/cdk/stacks/__init__.py b/source/idea/idea-administrator/src/ideaadministrator/app/cdk/stacks/__init__.py new file mode 100644 index 00000000..e455fa72 --- /dev/null +++ b/source/idea/idea-administrator/src/ideaadministrator/app/cdk/stacks/__init__.py @@ -0,0 +1,24 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + + +from .base_stack import IdeaBaseStack +from .bootstrap_stack import SocaBootstrapStack +from .cluster_stack import ClusterStack +from .identity_provider_stack import IdentityProviderStack +from .directoryservice_stack import DirectoryServiceStack +from .shared_storage_stack import SharedStorageStack +from .cluster_manager_stack import ClusterManagerStack +from .scheduler_stack import SchedulerStack +from .bastion_host_stack import BastionHostStack +from .analytics_stack import AnalyticsStack +from .virtual_desktop_controller_stack import VirtualDesktopControllerStack +from .metrics_stack import MetricsStack diff --git a/source/idea/idea-administrator/src/ideaadministrator/app/cdk/stacks/analytics_stack.py b/source/idea/idea-administrator/src/ideaadministrator/app/cdk/stacks/analytics_stack.py new file mode 100644 index 00000000..604b7a54 --- /dev/null +++ b/source/idea/idea-administrator/src/ideaadministrator/app/cdk/stacks/analytics_stack.py @@ -0,0 +1,358 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. +from ideaadministrator.app.cdk.idea_code_asset import IdeaCodeAsset, SupportedLambdaPlatforms +from ideadatamodel import ( + constants +) + +import ideaadministrator +from ideaadministrator.app.cdk.stacks import IdeaBaseStack + +from ideaadministrator.app.cdk.constructs import ( + ExistingSocaCluster, + KinesisStream, + IdeaNagSuppression, + Role, + Policy, + LambdaFunction, + OpenSearchSecurityGroup, + OpenSearch, + CustomResource +) +from ideadatamodel import exceptions + +from ideasdk.utils import Utils + +from typing import Optional +import aws_cdk as cdk +import constructs +from aws_cdk import ( + aws_ec2 as ec2, + aws_opensearchservice as opensearch, + aws_elasticloadbalancingv2 as elbv2, + aws_kinesis as kinesis, + aws_lambda as lambda_, + aws_lambda_event_sources as lambda_event_sources, + aws_logs as logs +) + + +class AnalyticsStack(IdeaBaseStack): + + def __init__(self, scope: constructs.Construct, + cluster_name: str, + aws_region: str, + aws_profile: str, + module_id: str, + deployment_id: str, + termination_protection: bool = True, + env: cdk.Environment = None): + + super().__init__( + scope=scope, + cluster_name=cluster_name, + aws_region=aws_region, + aws_profile=aws_profile, + module_id=module_id, + deployment_id=deployment_id, + termination_protection=termination_protection, + description=f'ModuleId: {module_id}, Cluster: {cluster_name}, Version: {ideaadministrator.props.current_release_version}', + tags={ + constants.IDEA_TAG_MODULE_ID: module_id, + constants.IDEA_TAG_MODULE_NAME: constants.MODULE_ANALYTICS, + constants.IDEA_TAG_MODULE_VERSION: ideaadministrator.props.current_release_version + }, + env=env + ) + + self.cluster = ExistingSocaCluster(self.context, self.stack) + + self.security_group: Optional[ec2.ISecurityGroup] = None + self.opensearch: Optional[opensearch.Domain] = None + self.kinesis_stream: Optional[KinesisStream] = None + + self.build_security_group() + is_existing = self.context.config().get_bool('analytics.opensearch.use_existing', default=False) + if is_existing: + domain_vpc_endpoint_url = self.context.config().get_string('analytics.opensearch.domain_vpc_endpoint_url', required=True) + self.opensearch = opensearch.Domain.from_domain_endpoint( + scope=self.stack, + id='existing-opensearch', + domain_endpoint=f'https://{domain_vpc_endpoint_url}' + ) + self.build_dashboard_endpoints() + else: + self.build_opensearch() + self.build_dashboard_endpoints() + + self.add_nag_suppression( + construct=self.stack, + suppressions=[ + IdeaNagSuppression(rule_id='AwsSolutions-IAM5', reason='CDK L2 construct does not support custom LogGroup permissions'), + IdeaNagSuppression(rule_id='AwsSolutions-IAM4', reason='Usage is required for Service Linked Role'), + IdeaNagSuppression(rule_id='AwsSolutions-L1', reason='CDK L2 construct does not offer options to customize the Lambda runtime'), + ] + ) + + self.build_analytics_input_stream() + self.build_cluster_settings() + + def build_analytics_input_stream(self): + + stream_config = self.context.config().get_string('analytics.kinesis.stream_mode', required=True) + if stream_config not in {'PROVISIONED', 'ON_DEMAND'}: + raise exceptions.invalid_params('analytics.kinesis.stream_mode needs to be one of PROVISIONED or ON_DEMAND only') + + if stream_config == 'PROVISIONED': + stream_mode = kinesis.StreamMode.PROVISIONED + shard_count = self.context.config().get_int('analytics.kinesis.shard_count', required=True) + else: + stream_mode = kinesis.StreamMode.ON_DEMAND + shard_count = None + + self.kinesis_stream = KinesisStream( + context=self.context, + name=f'{self.module_id}-kinesis-stream', + scope=self.stack, + stream_name=f'{self.module_id}-kinesis-stream', + stream_mode=stream_mode, + shard_count=shard_count + ) + if self.aws_region in Utils.get_value_as_list('KINESIS_STREAMS_CLOUDFORMATION_UNSUPPORTED_STREAMMODEDETAILS_REGION_LIST', constants.CAVEATS, []): + self.kinesis_stream.node.default_child.add_property_deletion_override('StreamModeDetails') + + lambda_name = f'{self.module_id}-sink-lambda' + stream_processing_lambda_role = Role( + context=self.context, + name=f'{lambda_name}-role', + scope=self.stack, + description=f'Role for {lambda_name} function for Cluster: {self.cluster_name}', + assumed_by=['lambda']) + + stream_processing_lambda_role.attach_inline_policy(Policy( + context=self.context, + name=f'{lambda_name}-policy', + scope=self.stack, + policy_template_name='analytics-sink-lambda.yml' + )) + + stream_processing_lambda = LambdaFunction( + self.context, + lambda_name, + self.stack, + idea_code_asset=IdeaCodeAsset( + lambda_package_name='idea_analytics_sink', + lambda_platform=SupportedLambdaPlatforms.PYTHON + ), + description=f'Lambda to process analytics-kinesis-stream data', + timeout_seconds=900, + security_groups=[self.security_group], + role=stream_processing_lambda_role, + environment={ + 'opensearch_endpoint': self.opensearch.domain_endpoint + }, + vpc=self.cluster.vpc, + vpc_subnets=ec2.SubnetSelection( + subnets=self.cluster.private_subnets + ) + ) + + stream_processing_lambda.add_event_source(lambda_event_sources.KinesisEventSource( + self.kinesis_stream, + batch_size=100, + starting_position=lambda_.StartingPosition.LATEST + )) + + def build_security_group(self): + self.security_group = OpenSearchSecurityGroup( + context=self.context, + name=f'{self.module_id}-opensearch-security-group', + scope=self.stack, + vpc=self.cluster.vpc + ) + + def check_service_linked_role_exists(self) -> bool: + try: + aws_dns_suffix = self.context.config().get_string('cluster.aws.dns_suffix', required=True) + list_roles_result = self.context.aws().iam().list_roles( + PathPrefix=f'/aws-service-role/es.{aws_dns_suffix}') + roles = Utils.get_value_as_list('Roles', list_roles_result, default=[]) + + list_roles_result = self.context.aws().iam().list_roles( + PathPrefix=f'/aws-service-role/opensearchservice.{aws_dns_suffix}') + roles.extend(Utils.get_value_as_list('Roles', list_roles_result, default=[])) + + return Utils.is_not_empty(roles) + except Exception as e: + self.context.aws_util().handle_aws_exception(e) + + def build_opensearch(self): + create_service_linked_role = not self.check_service_linked_role_exists() + + data_nodes = self.context.config().get_int('analytics.opensearch.data_nodes', required=True) + data_node_instance_type = self.context.config().get_string('analytics.opensearch.data_node_instance_type', required=True) + ebs_volume_size = self.context.config().get_int('analytics.opensearch.ebs_volume_size', required=True) + node_to_node_encryption = self.context.config().get_bool('analytics.opensearch.node_to_node_encryption', required=True) + removal_policy = self.context.config().get_string('analytics.opensearch.removal_policy', required=True) + app_log_removal_policy = self.context.config().get_string('analytics.opensearch.logging.app_log_removal_policy', default='DESTROY') + search_log_removal_policy = self.context.config().get_string('analytics.opensearch.logging.search_log_removal_policy', default='DESTROY') + slow_index_log_removal_policy = self.context.config().get_string('analytics.opensearch.logging.slow_index_log_removal_policy', default='DESTROY') + + self.opensearch = OpenSearch( + context=self.context, + name='analytics', + scope=self.stack, + cluster=self.cluster, + security_groups=[self.security_group], + data_nodes=data_nodes, + data_node_instance_type=data_node_instance_type, + ebs_volume_size=ebs_volume_size, + removal_policy=cdk.RemovalPolicy(removal_policy), + node_to_node_encryption=node_to_node_encryption, + create_service_linked_role=create_service_linked_role, + logging=opensearch.LoggingOptions( + slow_search_log_enabled=self.context.config().get_bool('analytics.opensearch.logging.slow_search_log_enabled', required=True), + slow_search_log_group=logs.LogGroup( + scope=self.stack, + id='analytics-search-log-group', + log_group_name=f'/{self.cluster_name}/{self.module_id}/search-log', + removal_policy=cdk.RemovalPolicy(search_log_removal_policy) + ), + app_log_enabled=self.context.config().get_bool('analytics.opensearch.logging.app_log_enabled', required=True), + app_log_group=logs.LogGroup( + scope=self.stack, + id='analytics-app-log-group', + log_group_name=f'/{self.cluster_name}/{self.module_id}/app-log', + removal_policy=cdk.RemovalPolicy(app_log_removal_policy) + ), + slow_index_log_enabled=self.context.config().get_bool('analytics.opensearch.logging.slow_index_log_enabled', required=True), + slow_index_log_group=logs.LogGroup( + scope=self.stack, + id='analytics-slow-index-log-group', + log_group_name=f'/{self.cluster_name}/{self.module_id}/slow-index-log', + removal_policy=cdk.RemovalPolicy(slow_index_log_removal_policy) + ), + # Audit logs are not enabled by default and not supported as this setting require fine-grained access permissions. + # Manually provision OpenSearch cluster to enable audit logs and use existing resources flow to use the OpenSearch cluster + audit_log_enabled=False + )) + + def build_dashboard_endpoints(self): + + cluster_endpoints_lambda_arn = self.context.config().get_string('cluster.cluster_endpoints_lambda_arn', required=True) + external_https_listener_arn = self.context.config().get_string('cluster.load_balancers.external_alb.https_listener_arn', required=True) + dashboard_endpoint_path_patterns = self.context.config().get_list('analytics.opensearch.endpoints.external.path_patterns', required=True) + dashboard_endpoint_priority = self.context.config().get_int('analytics.opensearch.endpoints.external.priority', required=True) + + is_existing = self.context.config().get_bool('analytics.opensearch.use_existing', default=False) + if is_existing: + domain_name = self.opensearch.domain_name + if domain_name.startswith('vpc-'): + # existing lookup returns the domain name as vpc-idea-dev1-analytics + # when used in describe_domain, service returns error - Domain not found: vpc-idea-dev1-analytics + # hack - fix to replace vpc- and then perform look up + domain_name = domain_name.replace('vpc-', '', 1) + describe_domain_result = self.context.aws().opensearch().describe_domain(DomainName=domain_name) + domain_status = describe_domain_result['DomainStatus'] + domain_cluster_config = domain_status['ClusterConfig'] + data_nodes = domain_cluster_config['InstanceCount'] + else: + domain_name = self.opensearch.domain_name + data_nodes = self.context.config().get_int('analytics.opensearch.data_nodes', required=True) + + opensearch_private_ips = CustomResource( + context=self.context, + name='opensearch-private-ips', + scope=self.stack, + idea_code_asset=IdeaCodeAsset( + lambda_package_name='idea_custom_resource_opensearch_private_ips', + lambda_platform=SupportedLambdaPlatforms.PYTHON + ), + lambda_timeout_seconds=180, + policy_template_name='custom-resource-opensearch-private-ips.yml', + resource_type='OpenSearchPrivateIPAddresses' + ).invoke( + name='opensearch-private-ips', + properties={ + 'DomainName': domain_name + } + ) + + targets = [] + for i in range(data_nodes * 3): + targets.append(elbv2.CfnTargetGroup.TargetDescriptionProperty( + id=cdk.Fn.select( + index=i, + array=cdk.Fn.split( + delimiter=',', + source=opensearch_private_ips.get_att_string('IpAddresses') + ) + ) + )) + + dashboard_target_group = elbv2.CfnTargetGroup( + self.stack, + f'{self.cluster_name}-dashboard-target-group', + port=443, + protocol='HTTPS', + target_type='ip', + vpc_id=self.cluster.vpc.vpc_id, + name=self.get_target_group_name('dashboard'), + targets=targets, + health_check_path='/' + ) + + cdk.CustomResource( + self.stack, + 'dashboard-endpoint', + service_token=cluster_endpoints_lambda_arn, + properties={ + 'endpoint_name': f'{self.module_id}-dashboard-endpoint', + 'listener_arn': external_https_listener_arn, + 'priority': dashboard_endpoint_priority, + 'target_group_arn': dashboard_target_group.ref, + 'conditions': [ + { + 'Field': 'path-pattern', + 'Values': dashboard_endpoint_path_patterns + } + ], + 'actions': [ + { + 'Type': 'forward', + 'TargetGroupArn': dashboard_target_group.ref + } + ], + 'tags': { + constants.IDEA_TAG_CLUSTER_NAME: self.cluster_name, + constants.IDEA_TAG_MODULE_ID: self.module_id, + constants.IDEA_TAG_MODULE_NAME: constants.MODULE_ANALYTICS + } + }, + resource_type='Custom::DashboardEndpointExternal' + ) + + def build_cluster_settings(self): + + cluster_settings = { + 'deployment_id': self.deployment_id, + 'opensearch.domain_name': self.opensearch.domain_name, + 'opensearch.domain_arn': self.opensearch.domain_arn, + 'opensearch.domain_endpoint': self.opensearch.domain_endpoint, + 'opensearch.dashboard_endpoint': f'{self.opensearch.domain_endpoint}/_dashboards', + 'kinesis.stream_name': self.kinesis_stream.stream_name, + 'kinesis.stream_arn': self.kinesis_stream.stream_arn + } + + if self.security_group is not None: + cluster_settings['opensearch.security_group_id'] = self.security_group.security_group_id + + self.update_cluster_settings(cluster_settings) diff --git a/source/idea/idea-administrator/src/ideaadministrator/app/cdk/stacks/base_stack.py b/source/idea/idea-administrator/src/ideaadministrator/app/cdk/stacks/base_stack.py new file mode 100644 index 00000000..24b13b55 --- /dev/null +++ b/source/idea/idea-administrator/src/ideaadministrator/app/cdk/stacks/base_stack.py @@ -0,0 +1,157 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +from ideaadministrator.app.cdk.constructs import ( + SocaBaseConstruct +) +from ideadatamodel import constants, exceptions +from ideaadministrator.app_context import AdministratorContext +from ideasdk.utils import Utils, GroupNameHelper + +import constructs +import aws_cdk as cdk +from aws_cdk import ( + aws_cognito as cognito +) + +from typing import Optional, Dict, List + + +class IdeaBaseStack(SocaBaseConstruct): + + def __init__(self, + scope: constructs.Construct, + cluster_name: str, + aws_region: str, + aws_profile: str, + module_id: str, + deployment_id: str, + description: Optional[str] = None, + tags: Optional[Dict] = None, + env: cdk.Environment = None, + termination_protection=True + ): + + self.context = AdministratorContext( + cluster_name=cluster_name, + aws_region=aws_region, + aws_profile=aws_profile, + module_id=module_id + ) + + self.stack_name = self.context.get_stack_name(module_id) + self.module_id = module_id + self.aws_region = aws_region + self.deployment_id = deployment_id + + super().__init__(self.context, module_id) + + if tags is None: + tags = {} + + tags = {**tags, **{ + constants.IDEA_TAG_CLUSTER_NAME: self.cluster_name + }} + + custom_tags = self.context.config().get_list('global-settings.custom_tags', []) + custom_tags_dict = Utils.convert_custom_tags_to_key_value_pairs(custom_tags) + tags = {**custom_tags_dict, **tags} + + cdk_toolkit_qualifier = Utils.shake_256(cluster_name, 5) + cluster_s3_bucket = self.context.config().get_string('cluster.cluster_s3_bucket', required=True) + + self.stack = cdk.Stack( + scope, + self.stack_name, + description=description, + env=env, + stack_name=self.stack_name, + tags=tags, + termination_protection=termination_protection, + synthesizer=cdk.DefaultStackSynthesizer( + qualifier=cdk_toolkit_qualifier, + bucket_prefix=f'cdk/', + file_assets_bucket_name=cluster_s3_bucket + ) + ) + + def get_target_group_name(self, identifier: str) -> str: + # target group name cannot be more than 32 characters + # max calculation - cluster name (max: 11) - (1) identifier (max: 11) - (1) suffix/hash (cluster_name + module_id) (8) = 32 + suffix = Utils.shake_256(f'{self.cluster_name}.{self.module_id}', 4) + target_group_name = f'{self.cluster_name}-{identifier}-{suffix}' + if len(target_group_name) > 32: + raise exceptions.invalid_params(f'target group name: {target_group_name} cannot more more than 32 characters.') + return target_group_name + + def update_cluster_settings(self, cluster_settings: Dict): + cluster_settings_lambda_arn = self.context.config().get_string('cluster.cluster_settings_lambda_arn', required=True) + cdk.CustomResource( + self.stack, + f'{self.cluster_name}-{self.module_id}-settings', + service_token=cluster_settings_lambda_arn, + properties={ + 'cluster_name': self.cluster_name, + 'module_id': self.module_id, + 'version': self.release_version, + 'settings': cluster_settings + }, + resource_type='Custom::ClusterSettings' + ) + + def is_metrics_provider_amazon_managed_prometheus(self) -> bool: + metrics_provider = self.context.config().get_string('metrics.provider') + if Utils.is_empty(metrics_provider): + return False + return metrics_provider == constants.METRICS_PROVIDER_AMAZON_MANAGED_PROMETHEUS + + def get_ec2_instance_managed_policies(self) -> List[str]: + ec2_managed_policies = [ + self.context.config().get_string('cluster.iam.policies.amazon_ssm_managed_instance_core_arn', required=True), + # since we need logs to be pushed to cloud watch by default, we don't check if metrics provider is cloud watch. + # additionally, some modules and services might not support prometheus metrics and in that case, metrics will be available via cloudwatch + self.context.config().get_string('cluster.iam.policies.cloud_watch_agent_server_arn', required=True), + ] + + if self.is_metrics_provider_amazon_managed_prometheus(): + ec2_managed_policies.append(self.context.config().get_string('cluster.iam.policies.amazon_prometheus_remote_write_arn', required=True)) + + ec2_managed_policy_arns = self.context.config().get_list('cluster.iam.ec2_managed_policy_arns', []) + ec2_managed_policies += ec2_managed_policy_arns + return ec2_managed_policies + + def lookup_user_pool(self) -> cognito.IUserPool: + return cognito.UserPool.from_user_pool_id( + self.stack, + f'{self.cluster_name}-user-pool', + self.context.config().get_string('identity-provider.cognito.user_pool_id', required=True) + ) + + def build_access_control_groups(self, user_pool: cognito.IUserPool): + group_name_helper = GroupNameHelper(self.context) + # module administrators group + cognito.CfnUserPoolGroup( + scope=self.stack, + id=f'{self.module_id}-administrators-group', + description=f'Module administrators group for module id: {self.module_id}, cluster: {self.cluster_name}', + group_name=group_name_helper.get_module_administrators_group(self.module_id), + precedence=3, + user_pool_id=user_pool.user_pool_id + ) + # module users group + cognito.CfnUserPoolGroup( + scope=self.stack, + id=f'{self.module_id}-users-group', + description=f'Module user group for module id: {self.module_id}, cluster: {self.cluster_name}', + group_name=group_name_helper.get_module_users_group(self.module_id), + precedence=4, + user_pool_id=user_pool.user_pool_id + ) diff --git a/source/idea/idea-administrator/src/ideaadministrator/app/cdk/stacks/bastion_host_stack.py b/source/idea/idea-administrator/src/ideaadministrator/app/cdk/stacks/bastion_host_stack.py new file mode 100644 index 00000000..bf9573d4 --- /dev/null +++ b/source/idea/idea-administrator/src/ideaadministrator/app/cdk/stacks/bastion_host_stack.py @@ -0,0 +1,241 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +from ideadatamodel import ( + constants +) +from ideasdk.bootstrap import BootstrapUserDataBuilder +from ideasdk.utils import Utils + +import ideaadministrator +from ideaadministrator.app.cdk.stacks import IdeaBaseStack +from ideaadministrator.app.cdk.constructs import ( + ExistingSocaCluster, + InstanceProfile, + IdeaNagSuppression, + Role, + Policy +) + +from typing import Optional +import aws_cdk as cdk +from aws_cdk import ( + aws_ec2 as ec2, + aws_route53 as route53 +) +import constructs + + +class BastionHostStack(IdeaBaseStack): + """ + Bastion Host Stack + """ + + def __init__(self, scope: constructs.Construct, + cluster_name: str, + aws_region: str, + aws_profile: str, + module_id: str, + deployment_id: str, + termination_protection: bool = True, + env: cdk.Environment = None): + + super().__init__( + scope=scope, + cluster_name=cluster_name, + aws_region=aws_region, + aws_profile=aws_profile, + module_id=module_id, + deployment_id=deployment_id, + termination_protection=termination_protection, + description=f'ModuleId: {module_id}, Cluster: {cluster_name}, Version: {ideaadministrator.props.current_release_version}', + tags={ + constants.IDEA_TAG_MODULE_ID: module_id, + constants.IDEA_TAG_MODULE_NAME: constants.MODULE_BASTION_HOST, + constants.IDEA_TAG_MODULE_VERSION: ideaadministrator.props.current_release_version + }, + env=env + ) + + self.bootstrap_package_uri = self.stack.node.try_get_context('bootstrap_package_uri') + + self.cluster = ExistingSocaCluster(self.context, self.stack) + self.bastion_host_role: Optional[Role] = None + self.bastion_host_instance_profile: Optional[InstanceProfile] = None + self.ec2_instance: Optional[ec2.Instance] = None + self.cluster_dns_record_set: Optional[route53.RecordSet] = None + + self.build_iam_roles() + self.build_ec2_instance() + self.build_route53_record_set() + self.build_cluster_settings() + + def build_iam_roles(self): + + ec2_managed_policies = self.get_ec2_instance_managed_policies() + + self.bastion_host_role = Role( + context=self.context, + name=f'{self.module_id}-role', + scope=self.stack, + description='IAM role assigned to the bastion-host', + assumed_by=['ssm', 'ec2'], + managed_policies=ec2_managed_policies + ) + self.bastion_host_role.attach_inline_policy( + Policy( + context=self.context, + name='bastion-host-policy', + scope=self.stack, + policy_template_name='bastion-host.yml' + ) + ) + self.bastion_host_instance_profile = InstanceProfile( + context=self.context, + name=f'{self.module_id}-instance-profile', + scope=self.stack, + roles=[self.bastion_host_role] + ) + # Make sure the role exists before trying to create the instance profile + self.bastion_host_instance_profile.node.add_dependency(self.bastion_host_role) + + def build_ec2_instance(self): + + is_public = self.context.config().get_bool('bastion-host.public', default=False) + base_os = self.context.config().get_string('bastion-host.base_os', required=True) + instance_ami = self.context.config().get_string('bastion-host.instance_ami', required=True) + instance_type = self.context.config().get_string('bastion-host.instance_type', required=True) + volume_size = self.context.config().get_int('bastion-host.volume_size', default=200) + key_pair_name = self.context.config().get_string('cluster.network.ssh_key_pair', required=True) + enable_detailed_monitoring = self.context.config().get_bool('bastion-host.ec2.enable_detailed_monitoring', default=False) + enable_termination_protection = self.context.config().get_bool('bastion-host.ec2.enable_termination_protection', default=False) + metadata_http_tokens = self.context.config().get_string('bastion-host.ec2.metadata_http_tokens', required=True) + + instance_profile_name = self.bastion_host_instance_profile.instance_profile_name + security_group = self.cluster.get_security_group(constants.MODULE_BASTION_HOST) + + if is_public and len(self.cluster.public_subnets) > 0: + subnet_ids = self.cluster.existing_vpc.get_public_subnet_ids() + else: + subnet_ids = self.cluster.existing_vpc.get_private_subnet_ids() + + block_device_name = Utils.get_ec2_block_device_name(base_os) + + user_data = BootstrapUserDataBuilder( + aws_region=self.aws_region, + bootstrap_package_uri=self.bootstrap_package_uri, + install_commands=[ + '/bin/bash bastion-host/setup.sh' + ], + base_os=base_os + ).build() + + launch_template = ec2.LaunchTemplate( + self.stack, f'{self.module_id}-lt', + instance_type=ec2.InstanceType(instance_type), + machine_image=ec2.MachineImage.generic_linux({ + self.aws_region: instance_ami + }), + user_data=ec2.UserData.custom(cdk.Fn.sub(user_data)), + key_name=key_pair_name, + block_devices=[ec2.BlockDevice( + device_name=block_device_name, + volume=ec2.BlockDeviceVolume(ebs_device=ec2.EbsDeviceProps( + volume_size=volume_size, + volume_type=ec2.EbsDeviceVolumeType.GP3 + ) + ) + )], + require_imdsv2=True if metadata_http_tokens == "required" else False + ) + + self.ec2_instance = ec2.CfnInstance( + self.stack, + f'{self.module_id}-instance', + block_device_mappings=[ + ec2.CfnInstance.BlockDeviceMappingProperty( + device_name=block_device_name, + ebs=ec2.CfnInstance.EbsProperty( + volume_size=volume_size, + volume_type='gp3' + ) + ) + ], + disable_api_termination=enable_termination_protection, + iam_instance_profile=instance_profile_name, + instance_type=instance_type, + image_id=instance_ami, + key_name=key_pair_name, + launch_template=ec2.CfnInstance.LaunchTemplateSpecificationProperty( + version=launch_template.latest_version_number, + launch_template_id=launch_template.launch_template_id), + network_interfaces=[ + ec2.CfnInstance.NetworkInterfaceProperty( + device_index='0', + associate_public_ip_address=is_public, + group_set=[security_group.security_group_id], + subnet_id=subnet_ids[0], + ) + ], + user_data=cdk.Fn.base64(cdk.Fn.sub(user_data)), + monitoring=enable_detailed_monitoring + ) + cdk.Tags.of(self.ec2_instance).add('Name', self.build_resource_name(self.module_id)) + cdk.Tags.of(self.ec2_instance).add(constants.IDEA_TAG_NODE_TYPE, constants.NODE_TYPE_INFRA) + self.add_backup_tags(self.ec2_instance) + self.ec2_instance.node.add_dependency(self.bastion_host_instance_profile) + + if not enable_detailed_monitoring: + self.add_nag_suppression( + construct=self.ec2_instance, + suppressions=[IdeaNagSuppression(rule_id='AwsSolutions-EC28', reason='Detailed monitoring is a configurable option to save costs.')] + ) + + if not enable_termination_protection: + self.add_nag_suppression( + construct=self.ec2_instance, + suppressions=[IdeaNagSuppression(rule_id='AwsSolutions-EC29', reason='termination protection is a configurable option. Enable termination protection via AWS EC2 console after deploying the cluster if required.')] + ) + + def build_route53_record_set(self): + hostname = self.context.config().get_string('bastion-host.hostname', required=True) + private_hosted_zone_id = self.context.config().get_string('cluster.route53.private_hosted_zone_id', required=True) + private_hosted_zone_name = self.context.config().get_string('cluster.route53.private_hosted_zone_name', required=True) + self.cluster_dns_record_set = route53.RecordSet( + self.stack, + f'{self.module_id}-dns-record', + record_type=route53.RecordType.A, + target=route53.RecordTarget.from_ip_addresses(self.ec2_instance.attr_private_ip), + ttl=cdk.Duration.minutes(5), + record_name=hostname, + zone=route53.HostedZone.from_hosted_zone_attributes( + scope=self.stack, + id='cluster-dns', + hosted_zone_id=private_hosted_zone_id, + zone_name=private_hosted_zone_name + ) + ) + + def build_cluster_settings(self): + + cluster_settings = {} # noqa + cluster_settings['deployment_id'] = self.deployment_id + cluster_settings['private_ip'] = self.ec2_instance.attr_private_ip + cluster_settings['private_dns_name'] = self.ec2_instance.attr_private_dns_name + + is_public = self.context.config().get_bool('bastion-host.public', False) + if is_public: + cluster_settings['public_ip'] = self.ec2_instance.attr_public_ip + cluster_settings['instance_id'] = self.ec2_instance.ref + cluster_settings['iam_role_arn'] = self.bastion_host_role.role_arn + cluster_settings['instance_profile_arn'] = self.bastion_host_instance_profile.ref + + self.update_cluster_settings(cluster_settings) diff --git a/source/idea/idea-administrator/src/ideaadministrator/app/cdk/stacks/bootstrap_stack.py b/source/idea/idea-administrator/src/ideaadministrator/app/cdk/stacks/bootstrap_stack.py new file mode 100644 index 00000000..2becf773 --- /dev/null +++ b/source/idea/idea-administrator/src/ideaadministrator/app/cdk/stacks/bootstrap_stack.py @@ -0,0 +1,25 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +import aws_cdk as cdk +import constructs + + +class SocaBootstrapStack(cdk.Stack): + """ + IDEA Bootstrap Stack + Empty stack as CDK bootstrap needs a stack to bootstrap. + + CloudFormation Stack will not be created. + """ + + def __init__(self, scope: constructs.Construct, env: cdk.Environment, stack_name: str): + super().__init__(scope, env=env, stack_name=stack_name) diff --git a/source/idea/idea-administrator/src/ideaadministrator/app/cdk/stacks/cluster_manager_stack.py b/source/idea/idea-administrator/src/ideaadministrator/app/cdk/stacks/cluster_manager_stack.py new file mode 100644 index 00000000..1f0bc149 --- /dev/null +++ b/source/idea/idea-administrator/src/ideaadministrator/app/cdk/stacks/cluster_manager_stack.py @@ -0,0 +1,474 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +from ideadatamodel import ( + constants +) +from ideasdk.utils import Utils +from ideasdk.bootstrap import BootstrapUserDataBuilder + +import ideaadministrator +from ideaadministrator.app.cdk.stacks import IdeaBaseStack +from ideaadministrator.app.cdk.constructs import ( + ExistingSocaCluster, + OAuthClientIdAndSecret, + SQSQueue, + Policy, + Role, + WebPortalSecurityGroup, + IdeaNagSuppression +) +from typing import Optional +import aws_cdk as cdk +from aws_cdk import ( + aws_ec2 as ec2, + aws_cognito as cognito, + aws_sqs as sqs, + aws_elasticloadbalancingv2 as elbv2, + aws_autoscaling as asg +) +import constructs + + +class ClusterManagerStack(IdeaBaseStack): + """ + Setup infrastructure for IDEA Cluster Manager Module + """ + + def __init__(self, scope: constructs.Construct, + cluster_name: str, + aws_region: str, + aws_profile: str, + module_id: str, + deployment_id: str, + termination_protection: bool = True, + env: cdk.Environment = None): + + super().__init__( + scope=scope, + cluster_name=cluster_name, + aws_region=aws_region, + aws_profile=aws_profile, + module_id=module_id, + deployment_id=deployment_id, + termination_protection=termination_protection, + description=f'ModuleId: {module_id}, Cluster: {cluster_name}, Version: {ideaadministrator.props.current_release_version}', + tags={ + constants.IDEA_TAG_MODULE_ID: module_id, + constants.IDEA_TAG_MODULE_NAME: constants.MODULE_CLUSTER_MANAGER, + constants.IDEA_TAG_MODULE_VERSION: ideaadministrator.props.current_release_version + }, + env=env + ) + + self.bootstrap_package_uri = self.stack.node.try_get_context('bootstrap_package_uri') + self.cluster = ExistingSocaCluster(self.context, self.stack) + + self.oauth2_client_secret: Optional[OAuthClientIdAndSecret] = None + self.cluster_tasks_sqs_queue: Optional[SQSQueue] = None + self.notifications_sqs_queue: Optional[SQSQueue] = None + self.cluster_manager_role: Optional[Role] = None + self.cluster_manager_security_group: Optional[WebPortalSecurityGroup] = None + self.auto_scaling_group: Optional[asg.AutoScalingGroup] = None + self.web_portal_endpoint: Optional[cdk.CustomResource] = None + self.external_endpoint: Optional[cdk.CustomResource] = None + self.internal_endpoint: Optional[cdk.CustomResource] = None + + self.user_pool = self.lookup_user_pool() + + self.build_oauth2_client() + self.build_access_control_groups(user_pool=self.user_pool) + self.build_sqs_queues() + self.build_iam_roles() + self.build_security_groups() + self.build_auto_scaling_group() + self.build_endpoints() + self.build_cluster_settings() + + def build_oauth2_client(self): + + # add resource server + resource_server = self.user_pool.add_resource_server( + id='resource-server', + identifier=self.module_id, + scopes=[ + cognito.ResourceServerScope(scope_name='read', scope_description='Allow Read Access'), + cognito.ResourceServerScope(scope_name='write', scope_description='Allow Write Access') + ] + ) + + # add new client to user pool + refresh_token_validity_hours = self.context.config().get_int('cluster-manager.oauth2_client.refresh_token_validity_hours', default=24) + client = self.user_pool.add_client( + id=f'{self.module_id}-client', + access_token_validity=cdk.Duration.hours(1), + auth_flows=cognito.AuthFlow( + admin_user_password=True + ), + generate_secret=True, + id_token_validity=cdk.Duration.hours(1), + o_auth=cognito.OAuthSettings( + flows=cognito.OAuthFlows(client_credentials=True), + scopes=[ + cognito.OAuthScope.custom(f'{self.module_id}/read'), + cognito.OAuthScope.custom(f'{self.module_id}/write') + ] + ), + refresh_token_validity=cdk.Duration.hours(refresh_token_validity_hours), + user_pool_client_name=self.module_id + ) + client.node.add_dependency(resource_server) + + # read secret value by invoking custom resource + oauth_credentials_lambda_arn = self.context.config().get_string('identity-provider.cognito.oauth_credentials_lambda_arn', required=True) + client_secret = cdk.CustomResource( + scope=self.stack, + id=f'{self.module_id}-creds', + service_token=oauth_credentials_lambda_arn, + properties={ + 'UserPoolId': self.user_pool.user_pool_id, + 'ClientId': client.user_pool_client_id + }, + resource_type='Custom::GetOAuthCredentials' + ) + + # save client id and client secret to AWS Secrets Manager + self.oauth2_client_secret = OAuthClientIdAndSecret( + context=self.context, + secret_name_prefix=self.module_id, + module_name=constants.MODULE_CLUSTER_MANAGER, + scope=self.stack, + client_id=client.user_pool_client_id, + client_secret=client_secret.get_att_string('ClientSecret') + ) + + def build_iam_roles(self): + ec2_managed_policies = self.get_ec2_instance_managed_policies() + + self.cluster_manager_role = Role( + context=self.context, + name=f'{self.module_id}-role', + scope=self.stack, + description='IAM role assigned to the cluster-manager', + assumed_by=['ssm', 'ec2'], + managed_policies=ec2_managed_policies + ) + self.cluster_manager_role.attach_inline_policy( + Policy( + context=self.context, + name='cluster-manager-policy', + scope=self.stack, + policy_template_name='cluster-manager.yml', + module_id=self.module_id + ) + ) + + def build_security_groups(self): + self.cluster_manager_security_group = WebPortalSecurityGroup( + context=self.context, + name=f'{self.module_id}-security-group', + scope=self.stack, + vpc=self.cluster.vpc, + bastion_host_security_group=self.cluster.get_security_group('bastion-host'), + loadbalancer_security_group=self.cluster.get_security_group('external-load-balancer') + ) + + def build_sqs_queues(self): + + kms_key_id = self.context.config().get_string('cluster.sqs.kms_key_id') + + self.cluster_tasks_sqs_queue = SQSQueue( + self.context, 'cluster-tasks-sqs-queue', self.stack, + queue_name=f'{self.cluster_name}-{self.module_id}-tasks.fifo', + fifo=True, + content_based_deduplication=True, + encryption_master_key=kms_key_id, + dead_letter_queue=sqs.DeadLetterQueue( + max_receive_count=30, + queue=SQSQueue( + self.context, 'cluster-tasks-sqs-queue-dlq', self.stack, + queue_name=f'{self.cluster_name}-{self.module_id}-tasks-dlq.fifo', + fifo=True, + content_based_deduplication=True, + encryption_master_key=kms_key_id, + is_dead_letter_queue=True + ) + ) + ) + self.add_common_tags(self.cluster_tasks_sqs_queue) + self.add_common_tags(self.cluster_tasks_sqs_queue.dead_letter_queue.queue) + + self.notifications_sqs_queue = SQSQueue( + self.context, 'notifications-sqs-queue', self.stack, + queue_name=f'{self.cluster_name}-{self.module_id}-notifications.fifo', + fifo=True, + content_based_deduplication=True, + encryption_master_key=kms_key_id, + dead_letter_queue=sqs.DeadLetterQueue( + max_receive_count=3, + queue=SQSQueue( + self.context, 'notifications-sqs-queue-dlq', self.stack, + queue_name=f'{self.cluster_name}-{self.module_id}-notifications-dlq.fifo', + fifo=True, + content_based_deduplication=True, + encryption_master_key=kms_key_id, + is_dead_letter_queue=True + ) + ) + ) + self.add_common_tags(self.notifications_sqs_queue) + self.add_common_tags(self.notifications_sqs_queue.dead_letter_queue.queue) + + def build_auto_scaling_group(self): + + key_pair_name = self.context.config().get_string('cluster.network.ssh_key_pair', required=True) + is_public = self.context.config().get_bool('cluster-manager.ec2.autoscaling.public', False) and len(self.cluster.public_subnets) > 0 + base_os = self.context.config().get_string('cluster-manager.ec2.autoscaling.base_os', required=True) + instance_ami = self.context.config().get_string('cluster-manager.ec2.autoscaling.instance_ami', required=True) + instance_type = self.context.config().get_string('cluster-manager.ec2.autoscaling.instance_type', required=True) + volume_size = self.context.config().get_int('cluster-manager.ec2.autoscaling.volume_size', default=200) + enable_detailed_monitoring = self.context.config().get_bool('cluster-manager.ec2.autoscaling.enable_detailed_monitoring', default=False) + min_capacity = self.context.config().get_int('cluster-manager.ec2.autoscaling.min_capacity', default=1) + max_capacity = self.context.config().get_int('cluster-manager.ec2.autoscaling.max_capacity', default=3) + cooldown_minutes = self.context.config().get_int('cluster-manager.ec2.autoscaling.cooldown_minutes', default=5) + new_instances_protected_from_scale_in = self.context.config().get_bool('cluster-manager.ec2.autoscaling.new_instances_protected_from_scale_in', default=True) + elb_healthcheck_grace_time_minutes = self.context.config().get_int('cluster-manager.ec2.autoscaling.elb_healthcheck.grace_time_minutes', default=15) + scaling_policy_target_utilization_percent = self.context.config().get_int('cluster-manager.ec2.autoscaling.cpu_utilization_scaling_policy.target_utilization_percent', default=80) + scaling_policy_estimated_instance_warmup_minutes = self.context.config().get_int('cluster-manager.ec2.autoscaling.cpu_utilization_scaling_policy.estimated_instance_warmup_minutes', default=15) + rolling_update_max_batch_size = self.context.config().get_int('cluster-manager.ec2.autoscaling.rolling_update_policy.max_batch_size', default=1) + rolling_update_min_instances_in_service = self.context.config().get_int('cluster-manager.ec2.autoscaling.rolling_update_policy.min_instances_in_service', default=1) + rolling_update_pause_time_minutes = self.context.config().get_int('cluster-manager.ec2.autoscaling.rolling_update_policy.pause_time_minutes', default=15) + metadata_http_tokens = self.context.config().get_string('cluster-manager.ec2.autoscaling.metadata_http_tokens', required=True) + + if is_public: + vpc_subnets = ec2.SubnetSelection( + subnets=self.cluster.public_subnets + ) + else: + vpc_subnets = ec2.SubnetSelection( + subnets=self.cluster.private_subnets + ) + + block_device_name = Utils.get_ec2_block_device_name(base_os) + + user_data = BootstrapUserDataBuilder( + aws_region=self.aws_region, + bootstrap_package_uri=self.bootstrap_package_uri, + install_commands=[ + '/bin/bash cluster-manager/setup.sh' + ], + base_os=base_os + ).build() + + launch_template = ec2.LaunchTemplate( + self.stack, f'{self.module_id}-lt', + instance_type=ec2.InstanceType(instance_type), + machine_image=ec2.MachineImage.generic_linux({ + self.aws_region: instance_ami + }), + security_group=self.cluster_manager_security_group, + user_data=ec2.UserData.custom(cdk.Fn.sub(user_data)), + key_name=key_pair_name, + block_devices=[ec2.BlockDevice( + device_name=block_device_name, + volume=ec2.BlockDeviceVolume(ebs_device=ec2.EbsDeviceProps( + volume_size=volume_size, + volume_type=ec2.EbsDeviceVolumeType.GP3 + )) + )], + role=self.cluster_manager_role, + require_imdsv2=True if metadata_http_tokens == "required" else False + ) + + self.auto_scaling_group = asg.AutoScalingGroup( + self.stack, 'cluster-manager-asg', + vpc=self.cluster.vpc, + vpc_subnets=vpc_subnets, + auto_scaling_group_name=f'{self.cluster_name}-{self.module_id}-asg', + launch_template=launch_template, + instance_monitoring=asg.Monitoring.DETAILED if enable_detailed_monitoring else asg.Monitoring.BASIC, + group_metrics=[asg.GroupMetrics.all()], + min_capacity=min_capacity, + max_capacity=max_capacity, + new_instances_protected_from_scale_in=new_instances_protected_from_scale_in, + cooldown=cdk.Duration.minutes(cooldown_minutes), + health_check=asg.HealthCheck.elb( + grace=cdk.Duration.minutes(elb_healthcheck_grace_time_minutes) + ), + update_policy=asg.UpdatePolicy.rolling_update( + max_batch_size=rolling_update_max_batch_size, + min_instances_in_service=rolling_update_min_instances_in_service, + pause_time=cdk.Duration.minutes(rolling_update_pause_time_minutes) + ), + termination_policies=[ + asg.TerminationPolicy.DEFAULT + ] + ) + + self.auto_scaling_group.scale_on_cpu_utilization( + 'cpu-utilization-scaling-policy', + target_utilization_percent=scaling_policy_target_utilization_percent, + estimated_instance_warmup=cdk.Duration.minutes(scaling_policy_estimated_instance_warmup_minutes) + ) + + cdk.Tags.of(self.auto_scaling_group).add(constants.IDEA_TAG_NODE_TYPE, constants.NODE_TYPE_APP) + cdk.Tags.of(self.auto_scaling_group).add(constants.IDEA_TAG_NAME, f'{self.cluster_name}-{self.module_id}') + self.auto_scaling_group.node.add_dependency(self.cluster_tasks_sqs_queue) + self.auto_scaling_group.node.add_dependency(self.notifications_sqs_queue) + + if not enable_detailed_monitoring: + self.add_nag_suppression( + construct=self.auto_scaling_group, + suppressions=[IdeaNagSuppression(rule_id='AwsSolutions-EC28', reason='detailed monitoring is a configurable option to save costs')], + apply_to_children=True + ) + + self.add_nag_suppression( + construct=self.auto_scaling_group, + suppressions=[ + IdeaNagSuppression(rule_id='AwsSolutions-AS3', reason='ASG notifications scaling notifications can be managed via AWS Console') + ] + ) + + def build_endpoints(self): + + cluster_endpoints_lambda_arn = self.context.config().get_string('cluster.cluster_endpoints_lambda_arn', required=True) + + external_https_listener_arn = self.context.config().get_string('cluster.load_balancers.external_alb.https_listener_arn', required=True) + # web portal endpoint + default_target_group = elbv2.CfnTargetGroup( + self.stack, + 'web-portal-target-group', + port=8443, + protocol='HTTPS', + target_type='instance', + vpc_id=self.cluster.vpc.vpc_id, + name=self.get_target_group_name('web-portal'), + health_check_path='/healthcheck' + ) + + self.web_portal_endpoint = cdk.CustomResource( + self.stack, + 'web-portal-endpoint', + service_token=cluster_endpoints_lambda_arn, + properties={ + 'endpoint_name': f'{self.module_id}-web-portal-endpoint', + 'listener_arn': external_https_listener_arn, + 'priority': 0, + 'default_action': True, + 'actions': [ + { + 'Type': 'forward', + 'TargetGroupArn': default_target_group.ref + } + ] + }, + resource_type='Custom::WebPortalEndpoint' + ) + + # cluster manager api external endpoint + external_endpoint_priority = self.context.config().get_int('cluster-manager.endpoints.external.priority', required=True) + external_endpoint_path_patterns = self.context.config().get_list('cluster-manager.endpoints.external.path_patterns', required=True) + external_target_group = elbv2.CfnTargetGroup( + self.stack, + f'{self.module_id}-external-target-group', + port=8443, + protocol='HTTPS', + target_type='instance', + vpc_id=self.cluster.vpc.vpc_id, + name=self.get_target_group_name('cm-ext'), + health_check_path='/healthcheck' + ) + self.external_endpoint = cdk.CustomResource( + self.stack, + 'external-endpoint', + service_token=cluster_endpoints_lambda_arn, + properties={ + 'endpoint_name': f'{self.module_id}-external-endpoint', + 'listener_arn': external_https_listener_arn, + 'priority': external_endpoint_priority, + 'conditions': [ + { + 'Field': 'path-pattern', + 'Values': external_endpoint_path_patterns + } + ], + 'actions': [ + { + 'Type': 'forward', + 'TargetGroupArn': external_target_group.ref + } + ] + }, + resource_type='Custom::ClusterManagerEndpointExternal' + ) + + # cluster manager api internal endpoint + internal_https_listener_arn = self.context.config().get_string('cluster.load_balancers.internal_alb.https_listener_arn', required=True) + internal_endpoint_priority = self.context.config().get_int('cluster-manager.endpoints.internal.priority', required=True) + internal_endpoint_path_patterns = self.context.config().get_list('cluster-manager.endpoints.internal.path_patterns', required=True) + internal_target_group = elbv2.CfnTargetGroup( + self.stack, + f'{self.module_id}-internal-target-group', + port=8443, + protocol='HTTPS', + target_type='instance', + vpc_id=self.cluster.vpc.vpc_id, + name=self.get_target_group_name('cm-int'), + health_check_path='/healthcheck' + ) + self.internal_endpoint = cdk.CustomResource( + self.stack, + 'internal-endpoint', + service_token=cluster_endpoints_lambda_arn, + properties={ + 'endpoint_name': f'{self.module_id}-internal-endpoint', + 'listener_arn': internal_https_listener_arn, + 'priority': internal_endpoint_priority, + 'conditions': [ + { + 'Field': 'path-pattern', + 'Values': internal_endpoint_path_patterns + } + ], + 'actions': [ + { + 'Type': 'forward', + 'TargetGroupArn': internal_target_group.ref + } + ] + }, + resource_type='Custom::ClusterManagerEndpointInternal' + ) + + # register target groups with ASG + self.auto_scaling_group.node.default_child.target_group_arns = [ + default_target_group.ref, + internal_target_group.ref, + external_target_group.ref + ] + + def build_cluster_settings(self): + cluster_settings = { + 'deployment_id': self.deployment_id, + 'client_id': self.oauth2_client_secret.client_id.ref, + 'client_secret': self.oauth2_client_secret.client_secret.ref, + 'security_group_id': self.cluster_manager_security_group.security_group_id, + 'iam_role_arn': self.cluster_manager_role.role_arn, + 'task_queue_url': self.cluster_tasks_sqs_queue.queue_url, + 'task_queue_arn': self.cluster_tasks_sqs_queue.queue_arn, + 'notifications_queue_url': self.notifications_sqs_queue.queue_url, + 'notifications_queue_arn': self.notifications_sqs_queue.queue_arn, + 'asg_name': self.auto_scaling_group.auto_scaling_group_name, + 'asg_arn': self.auto_scaling_group.auto_scaling_group_arn + } + + self.update_cluster_settings(cluster_settings) diff --git a/source/idea/idea-administrator/src/ideaadministrator/app/cdk/stacks/cluster_stack.py b/source/idea/idea-administrator/src/ideaadministrator/app/cdk/stacks/cluster_stack.py new file mode 100644 index 00000000..cd79bfa1 --- /dev/null +++ b/source/idea/idea-administrator/src/ideaadministrator/app/cdk/stacks/cluster_stack.py @@ -0,0 +1,1176 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +import ideaadministrator +from ideaadministrator.app.cdk.idea_code_asset import IdeaCodeAsset, SupportedLambdaPlatforms +from ideaadministrator.app.cdk.stacks import IdeaBaseStack +from ideadatamodel import ( + constants +) +from ideasdk.utils import Utils + +from ideaadministrator.app.cdk.constructs import ( + Vpc, + ExistingVpc, + VpcGatewayEndpoint, + VpcInterfaceEndpoint, + PrivateHostedZone, + SecurityGroup, + DefaultClusterSecurityGroup, + BastionHostSecurityGroup, + ExternalLoadBalancerSecurityGroup, + InternalLoadBalancerSecurityGroup, + VpcEndpointSecurityGroup, + CreateTagsCustomResource, + Policy, + ManagedPolicy, + Role, + LambdaFunction, + SNSTopic, + BackupPlan +) +from ideaadministrator import app_constants + +from typing import List, Dict, Optional + +import aws_cdk as cdk +import constructs +from aws_cdk import ( + aws_ec2 as ec2, + aws_events as events, + aws_elasticloadbalancingv2 as elbv2, + aws_route53 as route53, + aws_route53_targets as route53_targets, + aws_events_targets as events_targets, + aws_s3 as s3, + aws_backup as backup, + aws_kms as kms +) + + +class ClusterStack(IdeaBaseStack): + """ + Cluster Stack + + Provisions base infrastructure components for IDEA Cluster: + * VPC, VPC Endpoints (if applicable) + * Internal and External Load Balancers + * Common IAM Roles and Security Groups + * Self-signed Certificates (if applicable) + * Route53 Private Hosted Zone + * Common custom resource Lambda Functions + * Cluster Prefix List + * AWS Backup Vault and Backup Plan + * Cluster Settings + """ + + def __init__(self, scope: constructs.Construct, + cluster_name: str, + aws_region: str, + aws_profile: str, + module_id: str, + deployment_id: str, + termination_protection: bool = True, + env: cdk.Environment = None): + + super().__init__(scope=scope, + cluster_name=cluster_name, + aws_region=aws_region, + aws_profile=aws_profile, + module_id=module_id, + deployment_id=deployment_id, + termination_protection=termination_protection, + description=f'ModuleId: {module_id}, Cluster: {cluster_name}, Version: {ideaadministrator.props.current_release_version}', + tags={ + constants.IDEA_TAG_MODULE_ID: module_id, + constants.IDEA_TAG_MODULE_NAME: constants.MODULE_CLUSTER, + constants.IDEA_TAG_MODULE_VERSION: ideaadministrator.props.current_release_version + }, + env=env) + + # if user wants to use an existing VPC, user may provide and existing VPC ID. + # in that case, lookup the VPC and optionally, VpcInterface endpoints if specified, using ExistingVpc construct. + # else, create a new Vpc + self._vpc: Optional[Vpc] = None + self._existing_vpc: Optional[ExistingVpc] = None + + # provisioned only for new VPC + self.vpc_gateway_endpoints: Optional[Dict[str, VpcGatewayEndpoint]] = None + self.vpc_interface_endpoints: Optional[Dict[str, VpcInterfaceEndpoint]] = None + + self.self_signed_certificate_lambda: Optional[LambdaFunction] = None + self.external_certificate: Optional[cdk.CustomResource] = None + self.internal_certificate: Optional[cdk.CustomResource] = None + + self.backup_policies: Optional[Dict[str, ManagedPolicy]] = None + self.backup_role: Optional[Role] = None + self.backup_vault: Optional[backup.BackupVault] = None + self.backup_plan: Optional[BackupPlan] = None + + self.cluster_endpoints_lambda: Optional[LambdaFunction] = None + self.external_alb: Optional[elbv2.ApplicationLoadBalancer] = None + self.external_alb_https_listener: Optional[elbv2.CfnListener] = None + self.internal_alb_dns_record_set: Optional[route53.RecordSet] = None + self.internal_alb: Optional[elbv2.ApplicationLoadBalancer] = None + self.internal_alb_https_listener: Optional[elbv2.CfnListener] = None + self.internal_alb_dcv_broker_client_listener: Optional[elbv2.CfnListener] = None + self.internal_alb_dcv_broker_agent_listener: Optional[elbv2.CfnListener] = None + self.internal_alb_dcv_broker_gateway_listener: Optional[elbv2.CfnListener] = None + + self.private_hosted_zone: Optional[PrivateHostedZone] = None + self.cluster_prefix_list: Optional[ec2.CfnPrefixList] = None + self.security_groups: Dict[str, SecurityGroup] = {} + self.roles: Dict[str, Role] = {} + self.amazon_ssm_managed_instance_core_policy: Optional[ManagedPolicy] = None + self.cloud_watch_agent_server_policy: Optional[ManagedPolicy] = None + self.amazon_prometheus_remote_write_policy: Optional[ManagedPolicy] = None + + self.oauth_credentials_lambda: Optional[LambdaFunction] = None + self.solution_metrics_lambda: Optional[LambdaFunction] = None + self.cluster_settings_lambda: Optional[LambdaFunction] = None + + self.ec2_events_sns_topic: Optional[SNSTopic] = None + + # build backups + self.build_backups() + + # build common policies + # these policies are essential available in form of managed policies, but AWS managed policies are too broad + # from a security perspective and need to scope them down + self.build_policies() + + # build roles + self.build_roles() + + # build self-signed certificates lambda function + self.build_self_signed_certificates_lambda() + + # self-signed certificates + self.build_self_signed_certificates() + + # cluster settings lambda function + self.build_cluster_settings_lambda() + + # vpc + self.build_vpc() + + # cluster prefix list + self.build_cluster_prefix_list() + + # security groups + self.build_security_groups() + + # setup private hosted zone + self.build_private_hosted_zone() + + # ec2-notification module + self.build_ec2_notification_module() + + # cluster endpoints + self.build_cluster_endpoints() + + # vpc endpoints + self.build_vpc_endpoints() + + # solution metrics lambda function + self.build_solution_metrics_lambda() + + # build outputs + self.build_cluster_settings() + + @property + def vpc(self) -> ec2.IVpc: + if self.context.config().get_bool('cluster.network.use_existing_vpc', False): + return self._existing_vpc.vpc + else: + return self._vpc + + def private_subnets(self) -> List[ec2.ISubnet]: + if self.context.config().get_bool('cluster.network.use_existing_vpc', False): + return self._existing_vpc.get_private_subnets() + else: + return self.vpc.private_subnets + + def public_subnets(self) -> List[ec2.ISubnet]: + if self.context.config().get_bool('cluster.network.use_existing_vpc', False): + return self._existing_vpc.get_public_subnets() + else: + return self.vpc.public_subnets + + def build_backups(self): + + enable_backup = self.context.config().get_bool('cluster.backups.enabled', default=False) + if not enable_backup: + return + + enable_restore = self.context.config().get_bool('cluster.backups.enable_restore', default=True) + + # role and policies + # all backup polices must be managed policies and not inline policies. + # if added as inline policies - the maximum policy size of 10240 bytes exceeded error is raised. + backup_create_policy = ManagedPolicy( + self.context, 'backup-create-policy', self.stack, + managed_policy_name=f'{self.cluster_name}-{self.aws_region}-backup-create', + description='Provides AWS Backup permission to create backups on your behalf across AWS services', + policy_template_name='backup-create.yml' + ) + backup_s3_create_policy = ManagedPolicy( + self.context, 'backup-s3-create-policy', self.stack, + managed_policy_name=f'{self.cluster_name}-{self.aws_region}-backup-s3-create', + description='Policy containing permissions necessary for AWS Backup to backup data in any S3 bucket. ' + 'This includes read access to all S3 objects and any decrypt access for all KMS keys.', + policy_template_name='backup-s3-create.yml' + ) + + backup_restore_policy = None + backup_s3_restore_policy = None + if enable_restore: + backup_restore_policy = ManagedPolicy( + self.context, 'backup-restore-policy', self.stack, + managed_policy_name=f'{self.cluster_name}-{self.aws_region}-backup-restore', + description='Provides AWS Backup permission to perform restores on your behalf across AWS services. ' + 'This policy includes permissions to create and delete AWS resources, such as EBS volumes, RDS instances, and EFS file systems, which are part of the restore process.', + policy_template_name='backup-restore.yml' + ) + backup_s3_restore_policy = ManagedPolicy( + self.context, 'backup-s3-restore-policy', self.stack, + managed_policy_name=f'{self.cluster_name}-{self.aws_region}-backup-s3-restore', + description='Policy containing permissions necessary for AWS Backup to restore a S3 backup to a bucket. ' + 'This includes read/write permissions to all S3 buckets, and permissions to GenerateDataKey and DescribeKey for all KMS keys.', + policy_template_name='backup-s3-restore.yml' + ) + + backup_role = Role( + context=self.context, + name=f'{self.module_id}-backup-role', + scope=self.stack, + description='Role used by AWS Backup to authenticate when backing or restoring the resources', + assumed_by=['backup'] + ) + backup_role.add_managed_policy(backup_create_policy) + backup_role.add_managed_policy(backup_s3_create_policy) + if enable_restore: + backup_role.add_managed_policy(backup_restore_policy) + backup_role.add_managed_policy(backup_s3_restore_policy) + + # backup vault + backup_vault_removal_policy = self.context.config().get_string('cluster.backups.backup_vault.removal_policy', default='DESTROY') + backup_vault_kms_key_id = self.context.config().get_string('cluster.backups.backup_vault.kms_key_id', default=None) + backup_vault_encryption_key = None + if Utils.is_not_empty(backup_vault_kms_key_id): + backup_vault_encryption_key = kms.Key.from_lookup(self.stack, 'backup-vault-kms-key', self.get_kms_key_arn(key_id=backup_vault_kms_key_id)) + backup_vault = backup.BackupVault( + self.stack, 'backup-vault', + backup_vault_name=f'{self.cluster_name}-{self.module_id}-backup-vault', + encryption_key=backup_vault_encryption_key, + removal_policy=cdk.RemovalPolicy(backup_vault_removal_policy) + ) + + # backup plan + backup_plan_config = self.context.config().get_config('cluster.backups.backup_plan') + # create immutable reference to prevent BackupSelection from automatically adding the AWS Backup related managed policies to the role. + immutable_backup_role = backup_role.without_policy_updates() + backup_plan = BackupPlan( + self.context, 'cluster-backup-plan', self.stack, + backup_plan_name=f'{self.cluster_name}-{self.module_id}', + backup_plan_config=backup_plan_config, + backup_vault=backup_vault, + backup_role=immutable_backup_role + ) + + backup_plan.backup_selection.node.add_dependency(backup_create_policy) + backup_plan.backup_selection.node.add_dependency(backup_s3_create_policy) + if enable_restore: + backup_plan.backup_selection.node.add_dependency(backup_restore_policy) + backup_plan.backup_selection.node.add_dependency(backup_s3_restore_policy) + backup_plan.backup_selection.node.add_dependency(backup_role) + + self.backup_role = backup_role + self.backup_vault = backup_vault + self.backup_plan = backup_plan + + def build_policies(self): + self.amazon_ssm_managed_instance_core_policy = ManagedPolicy( + self.context, 'amazon-ssm-managed-instance-core', self.stack, + managed_policy_name=f'{self.cluster_name}-{self.aws_region}-amazon-ssm-managed-instance-core', + description='The policy for Amazon EC2 Role to enable AWS Systems Manager service core functionality.', + policy_template_name='amazon-ssm-managed-instance-core.yml' + ) + + self.cloud_watch_agent_server_policy = ManagedPolicy( + self.context, 'cloud-watch-agent-server-policy', self.stack, + managed_policy_name=f'{self.cluster_name}-{self.aws_region}-cloud-watch-agent-server-policy', + description='Permissions required to use AmazonCloudWatchAgent on servers', + policy_template_name='cloud-watch-agent-server-policy.yml' + ) + + if self.is_metrics_provider_amazon_managed_prometheus(): + self.amazon_prometheus_remote_write_policy = ManagedPolicy( + self.context, 'amazon-prometheus-remote-write-access', self.stack, + managed_policy_name=f'{self.cluster_name}-{self.aws_region}-amazon-prometheus-remote-write-access', + description='Grants write only access to AWS Managed Prometheus workspaces', + policy_template_name='amazon-prometheus-remote-write-access.yml' + ) + + def build_roles(self): + lambda_log_retention_role = Role( + context=self.context, + name=app_constants.LOG_RETENTION_ROLE_NAME, + scope=self.stack, + description='log retention role for CDK custom resources', + assumed_by=['lambda'], + inline_policies=[ + Policy( + context=self.context, + name='LogRetention', + scope=self.stack, + policy_template_name='log-retention.yml' + ) + ] + ) + self.roles[app_constants.LOG_RETENTION_ROLE_NAME] = lambda_log_retention_role + + def build_vpc(self): + # if the user specifies an existing vpc id in the user config, then do not create any vpc resources + # it is upto the administrator to manage the user config file for future upgrades and provide the same configuration file + # each time + + if self.context.config().get_bool('cluster.network.use_existing_vpc', False): + self._existing_vpc = ExistingVpc( + context=self.context, + name='existing-vpc', + scope=self.stack + ) + else: + self._vpc = Vpc(self.context, 'vpc', self.stack) + + def build_private_hosted_zone(self): + self.private_hosted_zone = PrivateHostedZone( + context=self.context, + scope=self.stack, + vpc=self.vpc + ) + + def build_cluster_prefix_list(self): + """ + cluster admins need an easy way to allow or deny additional IP addresses. + + instead of managing individual IP addresses in BastionHostSecurityGroup, ExternalALBSecurityGroup and DCVConnectionGatewaySecurityGroup, the `cluster` prefix list provides + a central place to manage access to cluster from the external world. + + for mature infrastructure deployments, where prefix lists already exist - cluster admins will provide prefix lists to access the cluster. config key: `cluster.network.prefix_list_ids` + + for cluster deployments that provide IP address to access the cluster, the IP addresses will be automatically added to the new cluster prefix list. config key: `cluster.network.cluster_prefix_list_id` + + All applicable security groups will allow access from `cluster.network.prefix_list_ids[] + cluster.network.cluster_prefix_list_id` + + Note for existing resources: + `cluster.network.cluster_prefix_list_id` will NOT be reused across clusters and each IDEA cluster will create a new cluster prefix list. + cluster admins can provide their custom prefix lists as part of `cluster.network.prefix_list_ids` to be reused across clusters + + Managing IP Addresses in cluster.network.cluster_prefix_list_id: + We need to enable updating the cluster prefix list, outside the CDK stack. If cluster admins need to deploy the cluster stack each time to add a new IP address, the entire point of `easy way` becomes moot. + The CDK stack will simply create the prefix list and only 'ADD' any new IP address if provided in config. + Use `idea-admin.sh utils cluster-prefix-list --help` to manage external access to the cluster. + """ + cluster_prefix_list_max_entries = self.context.config().get_int('cluster.network.cluster_prefix_list_max_entries', 10) + self.cluster_prefix_list = ec2.CfnPrefixList( + scope=self.stack, + id='cluster-prefix-list', + address_family='IPv4', + max_entries=cluster_prefix_list_max_entries, + prefix_list_name=f'{self.cluster_name}-prefix-list' + ) + self.add_common_tags(self.cluster_prefix_list) + + # do not create the custom resource if client_ip is not provided. + client_ips = self.context.config().get_list('cluster.network.client_ip') + if Utils.is_empty(client_ips): + return + + lambda_name = 'update-cluster-prefix-list' + lambda_policy = Policy( + context=self.context, + name=f'{lambda_name}-policy', + scope=self.stack, + policy_template_name='custom-resource-update-cluster-prefix-list.yml' + ) + lambda_role = Role( + context=self.context, + name=f'{lambda_name}-role', + scope=self.stack, + description=f'Role to manage cluster prefix list Lambda function for Cluster: {self.cluster_name}', + assumed_by=['lambda']) + lambda_role.attach_inline_policy(lambda_policy) + + lambda_function = LambdaFunction( + context=self.context, + name=lambda_name, + scope=self.stack, + idea_code_asset=IdeaCodeAsset( + lambda_package_name='idea_custom_resource_update_cluster_prefix_list', + lambda_platform=SupportedLambdaPlatforms.PYTHON + ), + description='Manage Cluster Prefix List', + timeout_seconds=180, + role=lambda_role, + log_retention_role=self.roles[app_constants.LOG_RETENTION_ROLE_NAME] + ) + lambda_function.node.add_dependency(lambda_policy) + lambda_function.node.add_dependency(lambda_role) + lambda_function.node.add_dependency(self.cluster_prefix_list) + + entries = [] + for client_ip in client_ips: + if '/' not in client_ip: + client_ip = f'{client_ip}/32' + entries.append({ + 'Cidr': client_ip, + 'Description': 'Allow access to cluster from Client IP' + }) + + cluster_prefix_list_custom_resource = cdk.CustomResource( + self.stack, + f'{self.cluster_name}-{self.module_id}-cluster-prefix-list', + service_token=lambda_function.function_arn, + properties={ + 'prefix_list_id': self.cluster_prefix_list.attr_prefix_list_id, + 'add_entries': entries + }, + resource_type='Custom::ClusterPrefixList' + ) + cluster_prefix_list_custom_resource.node.add_dependency(lambda_function) + + def build_security_groups(self): + # default cluster security group + cluster_security_group = DefaultClusterSecurityGroup( + context=self.context, + name='default-security-group', + scope=self.stack, + vpc=self.vpc + ) + self.security_groups['cluster'] = cluster_security_group + + # bastion host + bastion_host_security_group = BastionHostSecurityGroup( + context=self.context, + name='bastion-host-security-group', + scope=self.stack, + vpc=self.vpc, + cluster_prefix_list_id=self.cluster_prefix_list.attr_prefix_list_id + ) + self.security_groups['bastion-host'] = bastion_host_security_group + + # external load balancer + external_loadbalancer_security_group = ExternalLoadBalancerSecurityGroup( + context=self.context, + name='external-load-balancer-security-group', + scope=self.stack, + vpc=self.vpc, + bastion_host_security_group=bastion_host_security_group, + cluster_prefix_list_id=self.cluster_prefix_list.attr_prefix_list_id + ) + self.security_groups['external-load-balancer'] = external_loadbalancer_security_group + + # internal load balancer + internal_loadbalancer_security_group = InternalLoadBalancerSecurityGroup( + context=self.context, + name='internal-load-balancer-security-group', + scope=self.stack, + vpc=self.vpc + ) + self.security_groups['internal-load-balancer'] = internal_loadbalancer_security_group + + # add nat eips to load balancer security group + nat_eips = [] + for subnet_info in self.vpc.public_subnets: + nat_eip_for_subnet = subnet_info.node.try_find_child("EIP") + if nat_eip_for_subnet is not None: + nat_eips.append(nat_eip_for_subnet) + if len(nat_eips) > 0: + external_loadbalancer_security_group.add_nat_gateway_ips_ingress_rule(nat_eips) + + # vpc endpoint + use_existing_vpc = self.context.config().get_bool('cluster.network.use_existing_vpc') + use_vpc_endpoints = self.context.config().get_bool('cluster.network.use_vpc_endpoints') + if not use_existing_vpc and use_vpc_endpoints: + vpc_endpoint_security_group = VpcEndpointSecurityGroup( + context=self.context, + name='vpc-endpoint-security-group', + scope=self.stack, + vpc=self.vpc + ) + self.security_groups['vpc-endpoint'] = vpc_endpoint_security_group + + def build_vpc_endpoints(self): + use_vpc_endpoints = self.context.config().get_bool('cluster.network.use_vpc_endpoints', False) + if not use_vpc_endpoints: + return + + use_existing_vpc = self.context.config().get_bool('cluster.network.use_existing_vpc', False) + if use_existing_vpc: + return + + vpc_gateway_endpoints = {} + vpc_interface_endpoints = {} + + create_tags = CreateTagsCustomResource( + context=self.context, + scope=self.stack, + lambda_log_retention_role=self.roles[app_constants.LOG_RETENTION_ROLE_NAME] + ) + + gateway_endpoints = self.context.config().get_list('cluster.network.vpc_gateway_endpoints', []) + for service in gateway_endpoints: + vpc_gateway_endpoints[service] = VpcGatewayEndpoint( + context=self.context, + scope=self.stack, + service=service, + vpc=self.vpc, + create_tags=create_tags + ) + + interface_endpoints = self.context.config().get_config('cluster.network.vpc_interface_endpoints', default={}) + for service in interface_endpoints: + enabled = Utils.get_value_as_bool('enabled', interface_endpoints[service], default=False) + if not enabled: + continue + vpc_interface_endpoints[service] = VpcInterfaceEndpoint( + context=self.context, + scope=self.stack, + service=service, + vpc=self.vpc, + vpc_endpoint_security_group=self.security_groups['vpc-endpoint'], + create_tags=create_tags + ) + + self.vpc_gateway_endpoints = vpc_gateway_endpoints + self.vpc_interface_endpoints = vpc_interface_endpoints + + def build_cluster_settings_lambda(self): + lambda_name = 'cluster-settings' + + cluster_settings_lambda_role = Role( + context=self.context, + name=f'{lambda_name}-role', + scope=self.stack, + description=f'Role for cluster-settings lambda function for Cluster: {self.cluster_name}', + assumed_by=['lambda']) + + cluster_settings_lambda_role.attach_inline_policy(Policy( + context=self.context, + name=f'{lambda_name}-policy', + scope=self.stack, + policy_template_name='custom-resource-update-cluster-settings.yml' + )) + + self.cluster_settings_lambda = LambdaFunction( + context=self.context, + name=lambda_name, + scope=self.stack, + idea_code_asset=IdeaCodeAsset( + lambda_package_name='idea_custom_resource_update_cluster_settings', + lambda_platform=SupportedLambdaPlatforms.PYTHON + ), + description='Update cluster settings during cluster module deployment', + timeout_seconds=180, + role=cluster_settings_lambda_role, + log_retention_role=self.roles[app_constants.LOG_RETENTION_ROLE_NAME] + ) + self.cluster_settings_lambda.node.add_dependency(cluster_settings_lambda_role) + + def build_solution_metrics_lambda(self): + lambda_name = 'solution-metrics' + + solution_metrics_lambda_role = Role( + context=self.context, + name=f'{lambda_name}-role', + scope=self.stack, + description=f'Role for solution-metrics metrics Lambda function for Cluster: {self.cluster_name}', + assumed_by=['lambda']) + + solution_metrics_lambda_role.attach_inline_policy(Policy( + context=self.context, + name=f'{lambda_name}-policy', + scope=self.stack, + policy_template_name='solution-metrics-lambda-function.yml' + )) + + self.solution_metrics_lambda = LambdaFunction( + context=self.context, + name=lambda_name, + scope=self.stack, + idea_code_asset=IdeaCodeAsset( + lambda_package_name='idea_solution_metrics', + lambda_platform=SupportedLambdaPlatforms.PYTHON + ), + description='Send anonymous Metrics to AWS', + timeout_seconds=180, + role=solution_metrics_lambda_role, + log_retention_role=self.roles[app_constants.LOG_RETENTION_ROLE_NAME] + ) + + def build_self_signed_certificates_lambda(self): + """ + build self-signed certificates lambda function to be used as a custom resource in downstream modules + """ + lambda_name = 'self-signed-certificate' + + self_signed_certificate_policy = Policy( + context=self.context, + name=f'{lambda_name}-policy', + scope=self.stack, + policy_template_name='custom-resource-self-signed-certificate.yml' + ) + self_signed_certificate_role = Role( + context=self.context, + name=f'{lambda_name}-role', + scope=self.stack, + description=f'Role for generating self-signed certificates Lambda function for Cluster: {self.cluster_name}', + assumed_by=['lambda']) + self_signed_certificate_role.attach_inline_policy(self_signed_certificate_policy) + + self.self_signed_certificate_lambda = LambdaFunction( + context=self.context, + name=lambda_name, + scope=self.stack, + idea_code_asset=IdeaCodeAsset( + lambda_package_name='idea_custom_resource_self_signed_certificate', + lambda_platform=SupportedLambdaPlatforms.PYTHON + ), + description='Manage self-signed certificates for IDEA cluster infrastructure', + timeout_seconds=180, + role=self_signed_certificate_role, + log_retention_role=self.roles[app_constants.LOG_RETENTION_ROLE_NAME] + ) + # if dependency is not explicitly added, stack deletion fails due to race condition, where policy is deleted before lambda function + # and deletion fails. + self.self_signed_certificate_lambda.node.add_dependency(self_signed_certificate_policy) + self.self_signed_certificate_lambda.node.add_dependency(self_signed_certificate_role) + + def build_self_signed_certificates(self): + external_certificate_provided = self.context.config().get_bool('cluster.load_balancers.external_alb.certificates.provided', False) + if not external_certificate_provided: + # create self-signed certificate for external ALB and NLB + # all virtual desktop streaming traffic is routed via NLB + # all API and Web Portal traffic is routed via ALB + self.external_certificate = cdk.CustomResource( + self.stack, + f'{self.cluster_name}-{self.module_id}-external-cert', + service_token=self.self_signed_certificate_lambda.function_arn, + properties={ + 'domain_name': f'{self.cluster_name}.idea.default', + 'certificate_name': f'{self.cluster_name}-external', + 'create_acm_certificate': True, + 'kms_key_id': self.context.config().get_string('cluster.secretsmanager.kms_key_id'), + 'tags': { + 'Name': f'{self.cluster_name} external alb certs', + 'idea:ClusterName': self.cluster_name + } + }, + resource_type='Custom::SelfSignedCertificateExternal' + ) + self.external_certificate.node.add_dependency(self.self_signed_certificate_lambda) + + # create self-signed certificate for internal ALB + private_hosted_zone_name = self.context.config().get_string('cluster.route53.private_hosted_zone_name', required=True) + self.internal_certificate = cdk.CustomResource( + self.stack, + f'{self.cluster_name}-{self.module_id}-internal-cert', + service_token=self.self_signed_certificate_lambda.function_arn, + properties={ + 'domain_name': f'*.{private_hosted_zone_name}', + 'certificate_name': f'{self.cluster_name}-internal', + 'create_acm_certificate': True, + 'kms_key_id': self.context.config().get_string('cluster.secretsmanager.kms_key_id'), + 'tags': { + 'Name': f'{self.cluster_name} internal alb certs', + 'idea:ClusterName': self.cluster_name + } + }, + resource_type='Custom::SelfSignedCertificateInternal' + ) + self.internal_certificate.node.add_dependency(self.self_signed_certificate_lambda) + + def get_alb_listener_default_actions(self, listener_arn: str = None) -> List[elbv2.CfnListener.ActionProperty]: + """ + ALB listener must be created with a default action. + + after the cluster stack is deployed, the virtual desktop controller stack will update the default action to point to dcv broker + to avoid replacing the default action set for dcv broker, fetch the default actions if the listener exists. + + Similar is applicable for cluster manager, when cluster manager stack is deployed, it will update the default listener on + external ALB to point to web portal. + + When cluster stack is updated or re-rerun, the listener exists, + fetch the existing listener configuration and apply the config to the listener. + + ** Note ** + Currently, only target group forwarding is supported. Additional implementation is required to support + other types of existing listener configurations. + + :param listener_arn: + :return: + """ + default_actions = None + if Utils.is_not_empty(listener_arn): + describe_listeners_result = self.context.aws().elbv2().describe_listeners( + ListenerArns=[listener_arn] + ) + existing_action = describe_listeners_result['Listeners'][0]['DefaultActions'][0] + if existing_action['Type'] == 'forward': + default_actions = [ + elbv2.CfnListener.ActionProperty( + type='forward', + forward_config=elbv2.CfnListener.ForwardConfigProperty( + target_groups=[ + elbv2.CfnListener.TargetGroupTupleProperty( + target_group_arn=existing_action['TargetGroupArn'] + ) + ] + ) + ) + ] + if default_actions is None: + default_actions = [ + elbv2.CfnListener.ActionProperty( + type='fixed-response', + fixed_response_config=elbv2.CfnListener.FixedResponseConfigProperty( + status_code='200', + content_type='application/json', + message_body=Utils.to_json({ + 'success': True, + 'message': 'OK' + }) + ) + ) + ] + return default_actions + + def build_ec2_notification_module(self): + self.ec2_events_sns_topic = SNSTopic( + self.context, 'cluster-ec2-state-change-sns-topic', self.stack, + display_name=f'{self.cluster_name}-{self.module_id}-ec2-state-change-sns-topic', + topic_name=f'{self.cluster_name}-{self.module_id}-ec2-state-change-sns-topic', + master_key=self.context.config().get_string('cluster.sns.kms_key_id') + ) + self.add_common_tags(self.ec2_events_sns_topic) + + lambda_name = f'{self.module_id}-ec2state-event-transformer' + ec2_state_event_transformation_lambda_role = Role( + context=self.context, + name=f'{lambda_name}-role', + scope=self.stack, + assumed_by=['lambda'], + description=f'{lambda_name}-role' + ) + + ec2_state_event_transformation_lambda_role.attach_inline_policy(Policy( + context=self.context, + name=f'{lambda_name}-policy', + scope=self.stack, + policy_template_name='ec2state-event-transformer.yml' + )) + + ec2_state_event_transformation_lambda = LambdaFunction( + context=self.context, + name=lambda_name, + description=f'{self.module_id} lambda to intercept all ec2 state change events and transform to the required event object', + scope=self.stack, + environment={ + 'IDEA_EC2_STATE_SNS_TOPIC_ARN': self.ec2_events_sns_topic.topic_arn, + 'IDEA_CLUSTER_NAME_TAG_KEY': constants.IDEA_TAG_CLUSTER_NAME, + 'IDEA_CLUSTER_NAME_TAG_VALUE': self.context.cluster_name() + }, + timeout_seconds=180, + role=ec2_state_event_transformation_lambda_role, + idea_code_asset=IdeaCodeAsset( + lambda_package_name='idea_ec2_state_event_transformation_lambda', + lambda_platform=SupportedLambdaPlatforms.PYTHON + ) + ) + + ec2_monitoring_rule = events.Rule( + scope=self.stack, + id=f'{self.cluster_name}-ec2-state-monitoring-rule', + enabled=True, + rule_name=f'{self.cluster_name}-{self.module_id}-ec2-state-monitoring-rule', + description='Event Rule to monitor state changes on EC2 Instances', + event_pattern=events.EventPattern( + source=['aws.ec2'], + detail_type=['EC2 Instance State-change Notification'], + region=[self.aws_region] + ) + ) + ec2_monitoring_rule.add_target(events_targets.LambdaFunction( + ec2_state_event_transformation_lambda + )) + + def build_cluster_endpoints(self): + lambda_name = 'cluster-endpoints' + + cluster_endpoints_policy = Policy( + context=self.context, + name=f'{lambda_name}-policy', + scope=self.stack, + policy_template_name='custom-resource-cluster-endpoints.yml' + ) + cluster_endpoints_role = Role( + context=self.context, + name=f'{lambda_name}-role', + scope=self.stack, + description=f'Role for cluster endpoints lambda function for Cluster: {self.cluster_name}', + assumed_by=['lambda']) + + cluster_endpoints_role.attach_inline_policy(cluster_endpoints_policy) + + self.cluster_endpoints_lambda = LambdaFunction( + context=self.context, + name=lambda_name, + scope=self.stack, + idea_code_asset=IdeaCodeAsset( + lambda_package_name='idea_custom_resource_cluster_endpoints', + lambda_platform=SupportedLambdaPlatforms.PYTHON + ), + description='Manage cluster endpoints exposed via internal and external ALB', + timeout_seconds=600, + role=cluster_endpoints_role, + log_retention_role=self.roles[app_constants.LOG_RETENTION_ROLE_NAME] + ) + self.cluster_endpoints_lambda.node.add_dependency(cluster_endpoints_policy) + self.cluster_endpoints_lambda.node.add_dependency(cluster_endpoints_role) + + # external ALB - can be deployed in public or private subnets + is_public = self.context.config().get_bool('cluster.load_balancers.external_alb.public', default=True) + external_alb_subnets = self.public_subnets() if is_public is True else self.private_subnets() + self.external_alb = elbv2.ApplicationLoadBalancer( + self.stack, + f'{self.cluster_name}-external-alb', + load_balancer_name=f'{self.cluster_name}-external-alb', + security_group=self.security_groups['external-load-balancer'], + http2_enabled=True, + vpc=self.vpc, + vpc_subnets=ec2.SubnetSelection(subnets=external_alb_subnets), + internet_facing=is_public + ) + if self.external_certificate is not None: + self.external_alb.node.add_dependency(self.external_certificate) + + # internal ALB - will always be deployed in private subnets + self.internal_alb = elbv2.ApplicationLoadBalancer( + self.stack, + f'{self.cluster_name}-internal-alb', + load_balancer_name=f'{self.cluster_name}-internal-alb', + security_group=self.security_groups['internal-load-balancer'], + http2_enabled=True, + vpc=self.vpc, + vpc_subnets=ec2.SubnetSelection(subnets=self.private_subnets()), + internet_facing=False + ) + + # Manage Access Logs for external/internal Application Load Balancer + # https://docs.aws.amazon.com/elasticloadbalancing/latest/application/load-balancer-access-logs.html + external_alb_enable_access_log = self.context.config().get_bool('cluster.load_balancers.external_alb.access_logs', default=False) + internal_alb_enable_access_log = self.context.config().get_bool('cluster.load_balancers.internal_alb.access_logs', default=False) + + access_log_destination = None + if external_alb_enable_access_log or internal_alb_enable_access_log: + access_log_destination = s3.Bucket.from_bucket_name(scope=self.stack, id='cluster-s3-bucket', bucket_name=self.context.config().get_string('cluster.cluster_s3_bucket', required=True)) + + if external_alb_enable_access_log: + self.external_alb.log_access_logs(access_log_destination, f'logs/{self.module_id}/alb-access-logs/external-alb') + if internal_alb_enable_access_log: + self.internal_alb.log_access_logs(access_log_destination, f'logs/{self.module_id}/alb-access-logs/internal-alb') + + elbv2.CfnListener( + self.external_alb, + 'http-listener', + port=80, + load_balancer_arn=self.external_alb.load_balancer_arn, + protocol='HTTP', + default_actions=[ + elbv2.CfnListener.ActionProperty( + type='redirect', + redirect_config=elbv2.CfnListener.RedirectConfigProperty( + host='#{host}', + path='/#{path}', + port='443', + protocol='HTTPS', + query='#{query}', + status_code='HTTP_301' + ) + ) + ] + ) + + # ALB listener must be created with a default action. + # after the cluster stack is deployed, the cluster manager stack can update the default action to point to web portal + # to avoid replacing the default action set for web-portal, fetch the default actions if the listener exists + external_alb_listener_arn = self.context.config().get_string('cluster.load_balancers.external_alb.https_listener_arn') + external_alb_default_actions = self.get_alb_listener_default_actions(external_alb_listener_arn) + + if self.external_certificate is None: + external_acm_certificate_arn = self.context.config().get_string('cluster.load_balancers.external_alb.certificates.acm_certificate_arn', required=True) + else: + external_acm_certificate_arn = self.external_certificate.get_att_string('acm_certificate_arn') + + self.external_alb_https_listener = elbv2.CfnListener( + self.external_alb, + 'https-listener', + port=443, + ssl_policy=self.context.config().get_string('cluster.load_balancers.external_alb.ssl_policy', default='ELBSecurityPolicy-FS-1-2-Res-2020-10'), + load_balancer_arn=self.external_alb.load_balancer_arn, + protocol='HTTPS', + certificates=[ + elbv2.CfnListener.CertificateProperty( + certificate_arn=external_acm_certificate_arn + ) + ], + default_actions=external_alb_default_actions + ) + if self.external_certificate is not None: + self.external_alb_https_listener.node.add_dependency(self.external_certificate) + + self.internal_alb.node.add_dependency(self.internal_certificate) + + internal_acm_certificate_arn = self.internal_certificate.get_att_string('acm_certificate_arn') + self.internal_alb_https_listener = elbv2.CfnListener( + self.internal_alb, + 'https-listener', + port=443, + ssl_policy=self.context.config().get_string('cluster.load_balancers.internal_alb.ssl_policy', default='ELBSecurityPolicy-FS-1-2-Res-2020-10'), + load_balancer_arn=self.internal_alb.load_balancer_arn, + protocol='HTTPS', + certificates=[ + elbv2.CfnListener.CertificateProperty( + certificate_arn=internal_acm_certificate_arn + ) + ], + default_actions=[ + elbv2.CfnListener.ActionProperty( + type='fixed-response', + fixed_response_config=elbv2.CfnListener.FixedResponseConfigProperty( + status_code='200', + content_type='application/json', + message_body=Utils.to_json({ + 'success': True, + 'message': 'OK' + }) + ) + ) + ] + ) + self.internal_alb_https_listener.node.add_dependency(self.internal_certificate) + if self.aws_region in Utils.get_value_as_list('ROUTE53_CROSS_ZONE_ALIAS_RESTRICTED_REGION_LIST', constants.CAVEATS, []): + self.internal_alb_dns_record_set = route53.CnameRecord( + self.stack, + 'internal-alb-dns-record', + record_name=f'internal-alb.{self.private_hosted_zone.zone_name}', + zone=self.private_hosted_zone, + domain_name=self.internal_alb.load_balancer_dns_name, + ttl=cdk.Duration.minutes(5) + ) + else: + self.internal_alb_dns_record_set = route53.RecordSet( + self.stack, + 'internal-alb-dns-record', + record_type=route53.RecordType.A, + target=route53.RecordTarget.from_alias(route53_targets.LoadBalancerTarget(self.internal_alb)), + ttl=cdk.Duration.minutes(5), + record_name=f'internal-alb.{self.private_hosted_zone.zone_name}', + zone=self.private_hosted_zone + ) + + if self.context.config().is_module_enabled(constants.MODULE_VIRTUAL_DESKTOP_CONTROLLER): + dcv_broker_client_listener_arn = self.context.config().get_string('cluster.external_alb.dcv_broker_client_listener_arn') + dcv_broker_client_listener_default_actions = self.get_alb_listener_default_actions(dcv_broker_client_listener_arn) + dcv_broker_client_communication_port = self.context.config().get_int('virtual-desktop-controller.dcv_broker.client_communication_port', required=True) + self.internal_alb_dcv_broker_client_listener = elbv2.CfnListener( + self.internal_alb, + 'dcv-broker-client-listener', + port=dcv_broker_client_communication_port, + ssl_policy=self.context.config().get_string('virtual-desktop-controller.dcv_broker.ssl_policy', default='ELBSecurityPolicy-FS-1-2-Res-2020-10'), + load_balancer_arn=self.internal_alb.load_balancer_arn, + protocol='HTTPS', + certificates=[ + elbv2.CfnListener.CertificateProperty( + certificate_arn=internal_acm_certificate_arn + ) + ], + default_actions=dcv_broker_client_listener_default_actions + ) + self.internal_alb_dcv_broker_client_listener.node.add_dependency(self.internal_certificate) + self.security_groups['internal-load-balancer'].add_ingress_rule( + ec2.Peer.ipv4(self.vpc.vpc_cidr_block), + ec2.Port.tcp(dcv_broker_client_communication_port), + description='Allow HTTPS traffic from DCV Clients to DCV Broker' + ) + + dcv_broker_agent_listener_arn = self.context.config().get_string('cluster.external_alb.dcv_broker_agent_listener_arn') + dcv_broker_agent_listener_default_actions = self.get_alb_listener_default_actions(dcv_broker_agent_listener_arn) + dcv_broker_agent_communication_port = self.context.config().get_int('virtual-desktop-controller.dcv_broker.agent_communication_port', required=True) + self.internal_alb_dcv_broker_agent_listener = elbv2.CfnListener( + self.internal_alb, + 'dcv-broker-agent-listener', + port=dcv_broker_agent_communication_port, + ssl_policy=self.context.config().get_string('virtual-desktop-controller.dcv_broker.ssl_policy', default='ELBSecurityPolicy-FS-1-2-Res-2020-10'), + load_balancer_arn=self.internal_alb.load_balancer_arn, + protocol='HTTPS', + certificates=[ + elbv2.CfnListener.CertificateProperty( + certificate_arn=internal_acm_certificate_arn + ) + ], + default_actions=dcv_broker_agent_listener_default_actions + ) + self.internal_alb_dcv_broker_agent_listener.node.add_dependency(self.internal_certificate) + self.security_groups['internal-load-balancer'].add_ingress_rule( + ec2.Peer.ipv4(self.vpc.vpc_cidr_block), + ec2.Port.tcp(dcv_broker_agent_communication_port), + description='Allow HTTPS traffic from DCV Agents to DCV Broker' + ) + + dcv_broker_gateway_listener_arn = self.context.config().get_string('cluster.external_alb.dcv_broker_gateway_listener_arn') + dcv_broker_gateway_listener_default_actions = self.get_alb_listener_default_actions(dcv_broker_gateway_listener_arn) + dcv_broker_gateway_communication_port = self.context.config().get_int('virtual-desktop-controller.dcv_broker.gateway_communication_port', required=True) + self.internal_alb_dcv_broker_gateway_listener = elbv2.CfnListener( + self.internal_alb, + 'dcv-broker-gateway-listener', + port=dcv_broker_gateway_communication_port, + ssl_policy=self.context.config().get_string('virtual-desktop-controller.dcv_broker.ssl_policy', default='ELBSecurityPolicy-FS-1-2-Res-2020-10'), + load_balancer_arn=self.internal_alb.load_balancer_arn, + protocol='HTTPS', + certificates=[ + elbv2.CfnListener.CertificateProperty( + certificate_arn=internal_acm_certificate_arn + ) + ], + + default_actions=dcv_broker_gateway_listener_default_actions + ) + self.internal_alb_dcv_broker_gateway_listener.node.add_dependency(self.internal_certificate) + self.security_groups['internal-load-balancer'].add_ingress_rule( + ec2.Peer.ipv4(self.vpc.vpc_cidr_block), + ec2.Port.tcp(dcv_broker_gateway_communication_port), + description='Allow HTTPS traffic from DCV Connection Gateway to DCV Broker' + ) + + def build_cluster_settings(self): + # cluster settings are applied in the current module_id scope. module_id should not be provided in the key for settings. + cluster_settings = { + 'deployment_id': self.deployment_id, + 'network.vpc_id': self.vpc.vpc_id, + 'network.cluster_prefix_list_id': self.cluster_prefix_list.attr_prefix_list_id + } + + public_subnets = self.context.config().get_list('cluster.network.public_subnets', []) + is_external_alb_public = self.context.config().get_bool('cluster.load_balancers.external_alb.public', default=True) + if Utils.is_empty(public_subnets): + if is_external_alb_public: + for subnet in self.vpc.public_subnets: + public_subnets.append(subnet.subnet_id) + cluster_settings['network.public_subnets'] = public_subnets + + private_subnets = self.context.config().get_list('cluster.network.private_subnets', []) + if Utils.is_empty(private_subnets): + for subnet in self.vpc.private_subnets: + private_subnets.append(subnet.subnet_id) + cluster_settings['network.private_subnets'] = private_subnets + + if not self.context.config().get_bool('cluster.network.use_existing_vpc', False): + cluster_settings['network.nat_gateway_ips'] = [] + for eip in self.vpc.nat_gateway_ips: + cluster_settings['network.nat_gateway_ips'].append(f'{eip.ref}') + + # SecurityGroupIds + for name, security_group in self.security_groups.items(): + cluster_settings[f'network.security_groups.{name}'] = security_group.security_group_id + + # RoleArns + for name, role in self.roles.items(): + cluster_settings[f'iam.roles.{name}'] = role.role_arn + # Policy Arns + cluster_settings[f'iam.policies.amazon_ssm_managed_instance_core_arn'] = self.amazon_ssm_managed_instance_core_policy.managed_policy_arn + cluster_settings[f'iam.policies.cloud_watch_agent_server_arn'] = self.cloud_watch_agent_server_policy.managed_policy_arn + if self.amazon_prometheus_remote_write_policy is not None: + cluster_settings[f'iam.policies.amazon_prometheus_remote_write_arn'] = self.amazon_prometheus_remote_write_policy.managed_policy_arn + + cluster_settings['solution.solution_metrics_lambda_arn'] = self.solution_metrics_lambda.function_arn + cluster_settings['cluster_settings_lambda_arn'] = self.cluster_settings_lambda.function_arn + cluster_settings['self_signed_certificate_lambda_arn'] = self.self_signed_certificate_lambda.function_arn + + # route53 - private hosted zone settings + cluster_settings['route53.private_hosted_zone_id'] = self.private_hosted_zone.hosted_zone_id + cluster_settings['route53.private_hosted_zone_arn'] = self.private_hosted_zone.hosted_zone_arn + + # certificates + if not self.context.config().get_bool('cluster.load_balancers.external_alb.certificates.provided', required=True): + cluster_settings['load_balancers.external_alb.certificates.certificate_secret_arn'] = self.external_certificate.get_att_string('certificate_secret_arn') + cluster_settings['load_balancers.external_alb.certificates.private_key_secret_arn'] = self.external_certificate.get_att_string('private_key_secret_arn') + cluster_settings['load_balancers.external_alb.certificates.acm_certificate_arn'] = self.external_certificate.get_att_string('acm_certificate_arn') + + cluster_settings['load_balancers.internal_alb.certificates.certificate_secret_arn'] = self.internal_certificate.get_att_string('certificate_secret_arn') + cluster_settings['load_balancers.internal_alb.certificates.private_key_secret_arn'] = self.internal_certificate.get_att_string('private_key_secret_arn') + cluster_settings['load_balancers.internal_alb.certificates.acm_certificate_arn'] = self.internal_certificate.get_att_string('acm_certificate_arn') + cluster_settings['load_balancers.internal_alb.certificates.custom_dns_name'] = f'internal-alb.{self.private_hosted_zone.zone_name}' + + # cluster endpoints + cluster_settings['cluster_endpoints_lambda_arn'] = self.cluster_endpoints_lambda.function_arn + cluster_settings['load_balancers.external_alb.load_balancer_arn'] = self.external_alb.load_balancer_arn + cluster_settings['load_balancers.external_alb.load_balancer_dns_name'] = self.external_alb.load_balancer_dns_name + cluster_settings['load_balancers.external_alb.https_listener_arn'] = self.external_alb_https_listener.attr_listener_arn + + cluster_settings['load_balancers.internal_alb.load_balancer_arn'] = self.internal_alb.load_balancer_arn + cluster_settings['load_balancers.internal_alb.load_balancer_dns_name'] = self.internal_alb.load_balancer_dns_name + cluster_settings['load_balancers.internal_alb.https_listener_arn'] = self.internal_alb_https_listener.attr_listener_arn + + cluster_settings['ec2.state_change_notifications_sns_topic_arn'] = self.ec2_events_sns_topic.topic_arn + cluster_settings['ec2.state_change_notifications_sns_topic_name'] = self.ec2_events_sns_topic.topic_name + + if self.internal_alb_dcv_broker_client_listener: + cluster_settings['load_balancers.internal_alb.dcv_broker_client_listener_arn'] = self.internal_alb_dcv_broker_client_listener.attr_listener_arn + if self.internal_alb_dcv_broker_agent_listener: + cluster_settings['load_balancers.internal_alb.dcv_broker_agent_listener_arn'] = self.internal_alb_dcv_broker_agent_listener.attr_listener_arn + if self.internal_alb_dcv_broker_gateway_listener: + cluster_settings['load_balancers.internal_alb.dcv_broker_gateway_listener_arn'] = self.internal_alb_dcv_broker_gateway_listener.attr_listener_arn + + # vpc interface endpoints endpoint_url configuration + # vpc interface endpoint url will be updated only once during provisioning. + # if admin has updated the configuration, the endpoint_url for the endpoint will not be updated. + # gateway endpoints do not need any additional configuration as traffic to applicable services will be routed automatically once provisioned + if self.vpc_interface_endpoints is not None: + for service in self.vpc_interface_endpoints: + endpoint_config_key = f'network.vpc_interface_endpoints.{service}.endpoint_url' + existing_endpoint_url = self.context.config().get_string(f'cluster.{endpoint_config_key}') + if Utils.is_empty(existing_endpoint_url): + endpoint = self.vpc_interface_endpoints[service] + cluster_settings[endpoint_config_key] = endpoint.get_endpoint_url() + else: + cluster_settings[endpoint_config_key] = existing_endpoint_url + + # backups + if self.context.config().get_bool('cluster.backups.enabled', default=False): + cluster_settings['backups.role_arn'] = self.backup_role.role_arn + cluster_settings['backups.backup_vault.arn'] = self.backup_vault.backup_vault_arn + cluster_settings['backups.backup_plan.arn'] = self.backup_plan.get_backup_plan_arn() + + cdk.CustomResource( + self.stack, + f'{self.cluster_name}-{self.module_id}-settings', + service_token=self.cluster_settings_lambda.function_arn, + properties={ + 'cluster_name': self.cluster_name, + 'module_id': self.module_id, + 'version': self.release_version, + 'settings': cluster_settings + }, + resource_type='Custom::ClusterSettings' + ) diff --git a/source/idea/idea-administrator/src/ideaadministrator/app/cdk/stacks/directoryservice_stack.py b/source/idea/idea-administrator/src/ideaadministrator/app/cdk/stacks/directoryservice_stack.py new file mode 100644 index 00000000..e4597c04 --- /dev/null +++ b/source/idea/idea-administrator/src/ideaadministrator/app/cdk/stacks/directoryservice_stack.py @@ -0,0 +1,416 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +from ideadatamodel import ( + constants +) +from ideasdk.bootstrap import BootstrapUserDataBuilder +from ideasdk.utils import Utils + +import ideaadministrator +from ideaadministrator.app.cdk.stacks import IdeaBaseStack + +from ideaadministrator.app.cdk.constructs import ( + ExistingSocaCluster, + Role, + Policy, + InstanceProfile, + OpenLDAPServerSecurityGroup, + DirectoryServiceCredentials, + ActiveDirectory, + SQSQueue, + IdeaNagSuppression +) +from typing import Optional +import aws_cdk as cdk +from aws_cdk import ( + aws_ec2 as ec2, + aws_route53 as route53, + aws_sqs as sqs +) +import constructs + + +class DirectoryServiceStack(IdeaBaseStack): + """ + Directory Service Stack + + Based on directory service provider, provisions one of: + 1. OpenLDAP Server + 2. Microsoft AD + a) AWS Managed Microsoft AD (if new) + b) Infrastructure for AD Automation (SQS Queue) + 3. Secrets for OpenLDAP Root Account or AD Service Account (if not secret ARNs are not provided) + 4. Applicable cluster settings + """ + + def __init__(self, scope: constructs.Construct, + cluster_name: str, + aws_region: str, + aws_profile: str, + module_id: str, + deployment_id: str, + termination_protection: bool = True, + env: cdk.Environment = None): + + super().__init__( + scope=scope, + cluster_name=cluster_name, + aws_region=aws_region, + aws_profile=aws_profile, + module_id=module_id, + deployment_id=deployment_id, + termination_protection=termination_protection, + description=f'ModuleId: {module_id}, Cluster: {cluster_name}, Version: {ideaadministrator.props.current_release_version}', + tags={ + constants.IDEA_TAG_MODULE_ID: module_id, + constants.IDEA_TAG_MODULE_NAME: constants.MODULE_DIRECTORYSERVICE, + constants.IDEA_TAG_MODULE_VERSION: ideaadministrator.props.current_release_version + }, + env=env + ) + + self.cluster = ExistingSocaCluster(self.context, self.stack) + + self.bootstrap_package_uri: Optional[str] = None + self.openldap_role: Optional[Role] = None + self.openldap_instance_profile: Optional[InstanceProfile] = None + self.openldap_security_group: Optional[OpenLDAPServerSecurityGroup] = None + self.openldap_certs: Optional[cdk.CustomResource] = None + self.openldap_ec2_instance: Optional[ec2.CfnInstance] = None + self.openldap_cluster_dns_record_set: Optional[route53.RecordSet] = None + self.openldap_credentials: Optional[DirectoryServiceCredentials] = None + + self.activedirectory: Optional[ActiveDirectory] = None + + self.ad_automation_sqs_queue: Optional[SQSQueue] = None + + provider = self.context.config().get_string('directoryservice.provider', required=True) + if provider == constants.DIRECTORYSERVICE_OPENLDAP: + + root_credentials_provided = self.context.config().get_bool('directoryservice.root_credentials_provided', default=False) + if root_credentials_provided: + root_username_secret_arn = self.context.config().get_string('directoryservice.root_username_secret_arn') + assert Utils.is_not_empty(root_username_secret_arn) + root_password_secret_arn = self.context.config().get_string('directoryservice.root_password_secret_arn') + assert Utils.is_not_empty(root_password_secret_arn) + + self.bootstrap_package_uri = self.stack.node.try_get_context('bootstrap_package_uri') + self.openldap_credentials = DirectoryServiceCredentials( + context=self.context, + name=f'{self.module_id}-openldap-credentials', + scope=self.stack, + admin_username='Admin' + ) + self.build_iam_roles() + self.build_security_groups() + self.build_openldap_certs() + self.build_ec2_instance() + self.build_route53_record_set() + self.build_openldap_cluster_settings() + + elif provider == constants.DIRECTORYSERVICE_AWS_MANAGED_ACTIVE_DIRECTORY: + + root_credentials_provided = self.context.config().get_bool('directoryservice.root_credentials_provided', default=False) + if root_credentials_provided: + root_username_secret_arn = self.context.config().get_string('directoryservice.root_username_secret_arn') + assert Utils.is_not_empty(root_username_secret_arn) + root_password_secret_arn = self.context.config().get_string('directoryservice.root_password_secret_arn') + assert Utils.is_not_empty(root_password_secret_arn) + + use_existing = self.context.config().get_bool('directoryservice.use_existing', default=False) + if use_existing: + directory_id = self.context.config().get_string('directoryservice.directory_id') + assert Utils.is_not_empty(directory_id) + else: + self.activedirectory = ActiveDirectory( + context=self.context, + name='active-directory', + scope=self.stack, + cluster=self.cluster, + enable_sso=False + ) + self.build_ad_automation_sqs_queue() + self.build_aws_managed_ad_cluster_settings() + + # ActiveDirectory that is self-managed + # Additional configuration is required to bootstrap a clusteradmin + # as we do not have write-access to create users after installation + # This is done with the directoryservice helper + elif provider == constants.DIRECTORYSERVICE_ACTIVE_DIRECTORY: + + root_credentials_provided = self.context.config().get_bool('directoryservice.root_credentials_provided', default=False) + assert root_credentials_provided is True + root_username_secret_arn = self.context.config().get_string('directoryservice.root_username_secret_arn') + assert Utils.is_not_empty(root_username_secret_arn) + root_password_secret_arn = self.context.config().get_string('directoryservice.root_password_secret_arn') + assert Utils.is_not_empty(root_password_secret_arn) + # + # todo - make these better / exceptions / errors / logging + clusteradmin_username_secret_arn = self.context.config().get_string('directoryservice.clusteradmin.clusteradmin_username_secret_arn') + assert Utils.is_not_empty(clusteradmin_username_secret_arn) + clusteradmin_password_secret_arn = self.context.config().get_string('directoryservice.clusteradmin.clusteradmin_password_secret_arn') + assert Utils.is_not_empty(clusteradmin_password_secret_arn) + + + self.build_ad_automation_sqs_queue() + self.build_activedirectory_cluster_settings() + + def build_iam_roles(self): + ec2_managed_policies = self.get_ec2_instance_managed_policies() + + self.openldap_role = Role( + context=self.context, + name=f'{self.module_id}-openldap-role', + scope=self.stack, + description='IAM role assigned to the OpenLDAP server', + assumed_by=['ssm', 'ec2'], + managed_policies=ec2_managed_policies + ) + self.openldap_role.attach_inline_policy( + Policy( + context=self.context, + name='openldap-server-policy', + scope=self.stack, + policy_template_name='openldap-server.yml' + ) + ) + self.openldap_instance_profile = InstanceProfile( + context=self.context, + name=f'{self.module_id}-openldap-instance-profile', + scope=self.stack, + roles=[self.openldap_role] + ) + + def build_security_groups(self): + self.openldap_security_group = OpenLDAPServerSecurityGroup( + context=self.context, + name=f'{self.module_id}-security-group', + scope=self.stack, + vpc=self.cluster.vpc, + bastion_host_security_group=self.cluster.get_security_group('bastion-host') + ) + + def build_openldap_certs(self): + """ + build openldap tls certs and save to secrets manager + * only openldap server will have access to private key + * all cluster nodes will have access to certificate in order to join directory service + """ + hostname = self.context.config().get_string('directoryservice.hostname', required=True) + self_signed_certificate_lambda_arn = self.context.config().get_string('cluster.self_signed_certificate_lambda_arn', required=True) + self.openldap_certs = cdk.CustomResource( + self.stack, + 'openldap-server-certs', + service_token=self_signed_certificate_lambda_arn, + properties={ + 'domain_name': hostname, + 'certificate_name': f'{self.cluster_name}-{self.module_id}', + 'create_acm_certificate': False, + 'kms_key_id': self.context.config().get_string('cluster.secretsmanager.kms_key_id'), + 'tags': { + 'Name': f'{self.cluster_name}-{self.module_id}', + 'idea:ClusterName': self.cluster_name, + 'idea:ModuleName': constants.MODULE_DIRECTORYSERVICE + } + }, + resource_type=f'Custom::SelfSignedCertificateOpenLDAPServer' + ) + + def build_ec2_instance(self): + + is_public = self.context.config().get_bool('directoryservice.public', False) + base_os = self.context.config().get_string('directoryservice.base_os', required=True) + instance_ami = self.context.config().get_string('directoryservice.instance_ami', required=True) + instance_type = self.context.config().get_string('directoryservice.instance_type', required=True) + volume_size = self.context.config().get_int('directoryservice.volume_size', 200) + key_pair_name = self.context.config().get_string('cluster.network.ssh_key_pair', required=True) + enable_detailed_monitoring = self.context.config().get_bool('directoryservice.ec2.enable_detailed_monitoring', default=False) + enable_termination_protection = self.context.config().get_bool('directoryservice.ec2.enable_termination_protection', default=False) + metadata_http_tokens = self.context.config().get_string('directoryservice.ec2.metadata_http_tokens', required=True) + + if is_public and len(self.cluster.public_subnets) > 0: + subnet_ids = self.cluster.existing_vpc.get_public_subnet_ids() + else: + subnet_ids = self.cluster.existing_vpc.get_private_subnet_ids() + + block_device_name = Utils.get_ec2_block_device_name(base_os) + + user_data = BootstrapUserDataBuilder( + aws_region=self.aws_region, + bootstrap_package_uri=self.bootstrap_package_uri, + install_commands=[ + '/bin/bash openldap-server/setup.sh' + ], + base_os=base_os, + infra_config={ + 'LDAP_ROOT_USERNAME_SECRET_ARN': '${__LDAP_ROOT_USERNAME_SECRET_ARN__}', + 'LDAP_ROOT_PASSWORD_SECRET_ARN': '${__LDAP_ROOT_PASSWORD_SECRET_ARN__}', + 'LDAP_TLS_CERTIFICATE_SECRET_ARN': '${__LDAP_TLS_CERTIFICATE_SECRET_ARN__}', + 'LDAP_TLS_PRIVATE_KEY_SECRET_ARN': '${__LDAP_TLS_PRIVATE_KEY_SECRET_ARN__}' + } + ).build() + + substituted_userdata = cdk.Fn.sub(user_data, { + '__LDAP_ROOT_USERNAME_SECRET_ARN__': self.openldap_credentials.get_username_secret_arn(), + '__LDAP_ROOT_PASSWORD_SECRET_ARN__': self.openldap_credentials.get_password_secret_arn(), + '__LDAP_TLS_CERTIFICATE_SECRET_ARN__': self.openldap_certs.get_att_string('certificate_secret_arn'), + '__LDAP_TLS_PRIVATE_KEY_SECRET_ARN__': self.openldap_certs.get_att_string('private_key_secret_arn') + }) + + launch_template = ec2.LaunchTemplate( + self.stack, f'{self.module_id}-lt', + instance_type=ec2.InstanceType(instance_type), + machine_image=ec2.MachineImage.generic_linux({ + self.aws_region: instance_ami + } + ), + user_data=ec2.UserData.custom(substituted_userdata), + key_name=key_pair_name, + block_devices=[ec2.BlockDevice( + device_name=block_device_name, + volume=ec2.BlockDeviceVolume(ebs_device=ec2.EbsDeviceProps( + volume_size=volume_size, + volume_type=ec2.EbsDeviceVolumeType.GP3 + ) + ) + )], + require_imdsv2=True if metadata_http_tokens == "required" else False + ) + + self.openldap_ec2_instance = ec2.CfnInstance( + self.stack, + f'{self.module_id}-instance', + block_device_mappings=[ + ec2.CfnInstance.BlockDeviceMappingProperty( + device_name=block_device_name, + ebs=ec2.CfnInstance.EbsProperty( + volume_size=volume_size, + volume_type='gp3' + ) + ) + ], + disable_api_termination=enable_termination_protection, + iam_instance_profile=self.openldap_instance_profile.instance_profile_name, + instance_type=instance_type, + image_id=instance_ami, + key_name=key_pair_name, + launch_template=ec2.CfnInstance.LaunchTemplateSpecificationProperty( + version=launch_template.latest_version_number, + launch_template_id=launch_template.launch_template_id), + network_interfaces=[ + ec2.CfnInstance.NetworkInterfaceProperty( + device_index='0', + associate_public_ip_address=is_public, + group_set=[self.openldap_security_group.security_group_id], + subnet_id=subnet_ids[0], + ) + ], + user_data=cdk.Fn.base64(substituted_userdata), + monitoring=enable_detailed_monitoring + ) + cdk.Tags.of(self.openldap_ec2_instance).add('Name', self.build_resource_name(self.module_id)) + cdk.Tags.of(self.openldap_ec2_instance).add(constants.IDEA_TAG_NODE_TYPE, constants.NODE_TYPE_INFRA) + self.add_backup_tags(self.openldap_ec2_instance) + + if not enable_detailed_monitoring: + self.add_nag_suppression( + construct=self.openldap_ec2_instance, + suppressions=[IdeaNagSuppression(rule_id='AwsSolutions-EC28', reason='detailed monitoring is a configurable option to save costs')] + ) + + if not enable_termination_protection: + self.add_nag_suppression( + construct=self.openldap_ec2_instance, + suppressions=[IdeaNagSuppression(rule_id='AwsSolutions-EC29', reason='termination protection not supported in CDK L2 construct. enable termination protection via AWS EC2 console after deploying the cluster.')] + ) + + def build_route53_record_set(self): + hostname = self.context.config().get_string('directoryservice.hostname', required=True) + private_hosted_zone_id = self.context.config().get_string('cluster.route53.private_hosted_zone_id', required=True) + private_hosted_zone_name = self.context.config().get_string('cluster.route53.private_hosted_zone_name', required=True) + self.openldap_cluster_dns_record_set = route53.RecordSet( + self.stack, + f'{self.module_id}-dns-record', + record_type=route53.RecordType.A, + target=route53.RecordTarget.from_ip_addresses(self.openldap_ec2_instance.attr_private_ip), + ttl=cdk.Duration.minutes(5), + record_name=hostname, + zone=route53.HostedZone.from_hosted_zone_attributes( + scope=self.stack, + id='cluster-dns', + hosted_zone_id=private_hosted_zone_id, + zone_name=private_hosted_zone_name + ) + ) + + def build_ad_automation_sqs_queue(self): + self.ad_automation_sqs_queue = SQSQueue( + self.context, 'ad-automation-sqs-queue', self.stack, + queue_name=f'{self.cluster_name}-{self.module_id}-ad-automation.fifo', + fifo=True, + content_based_deduplication=True, + dead_letter_queue=sqs.DeadLetterQueue( + max_receive_count=30, + queue=SQSQueue( + self.context, 'ad-automation-sqs-queue-dlq', self.stack, + queue_name=f'{self.cluster_name}-{self.module_id}-ad-automation-dlq.fifo', + fifo=True, + content_based_deduplication=True, + is_dead_letter_queue=True + ) + ) + ) + self.add_common_tags(self.ad_automation_sqs_queue) + self.add_common_tags(self.ad_automation_sqs_queue.dead_letter_queue.queue) + + def build_openldap_cluster_settings(self): + + cluster_settings = { + 'deployment_id': self.deployment_id, + 'private_ip': self.openldap_ec2_instance.attr_private_ip, + 'private_dns_name': self.openldap_ec2_instance.attr_private_dns_name, + 'instance_id': self.openldap_ec2_instance.ref, + 'security_group_id': self.openldap_security_group.security_group_id, + 'iam_role_arn': self.openldap_role.role_arn, + 'instance_profile_arn': self.openldap_instance_profile.ref, + 'root_username_secret_arn': self.openldap_credentials.admin_username.ref, + 'root_password_secret_arn': self.openldap_credentials.admin_password.ref, + 'tls_certificate_secret_arn': self.openldap_certs.get_att_string('certificate_secret_arn'), + 'tls_private_key_secret_arn': self.openldap_certs.get_att_string('private_key_secret_arn') + } + + is_public = self.context.config().get_bool('directoryservice.public', False) + if is_public: + cluster_settings['public_ip'] = self.openldap_ec2_instance.attr_public_ip + + self.update_cluster_settings(cluster_settings) + + def build_aws_managed_ad_cluster_settings(self): + cluster_settings = { + 'deployment_id': self.deployment_id, + 'ad_automation.sqs_queue_url': self.ad_automation_sqs_queue.queue_url, + 'ad_automation.sqs_queue_arn': self.ad_automation_sqs_queue.queue_arn + } + if self.activedirectory is not None: + cluster_settings['directory_id'] = self.activedirectory.ad.ref + cluster_settings['root_username_secret_arn'] = self.activedirectory.credentials.get_username_secret_arn() + cluster_settings['root_password_secret_arn'] = self.activedirectory.credentials.get_password_secret_arn() + + self.update_cluster_settings(cluster_settings) + + def build_activedirectory_cluster_settings(self): + cluster_settings = { + 'deployment_id': self.deployment_id, + 'ad_automation.sqs_queue_url': self.ad_automation_sqs_queue.queue_url, + 'ad_automation.sqs_queue_arn': self.ad_automation_sqs_queue.queue_arn + } + self.update_cluster_settings(cluster_settings) diff --git a/source/idea/idea-administrator/src/ideaadministrator/app/cdk/stacks/identity_provider_stack.py b/source/idea/idea-administrator/src/ideaadministrator/app/cdk/stacks/identity_provider_stack.py new file mode 100644 index 00000000..caf702c4 --- /dev/null +++ b/source/idea/idea-administrator/src/ideaadministrator/app/cdk/stacks/identity_provider_stack.py @@ -0,0 +1,203 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. +from ideaadministrator.app.cdk.idea_code_asset import IdeaCodeAsset, SupportedLambdaPlatforms +from ideadatamodel import ( + constants, exceptions +) +from ideasdk.utils import Utils + +import ideaadministrator +from ideaadministrator.app.cdk.stacks import IdeaBaseStack +from ideaadministrator import app_constants + +from ideaadministrator.app.cdk.constructs import ( + ExistingSocaCluster, + UserPool, + LambdaFunction, + Role, + Policy +) +from typing import Optional +import aws_cdk as cdk +from aws_cdk import ( + aws_cognito as cognito +) +import constructs + +import os + + +class IdentityProviderStack(IdeaBaseStack): + """ + Identity Provider Stack + + Based on selected identity provider, this stack provisions one of: + * Cognito User Pool + * Keycloak + + For Cognito IDP + --------------- + + Below resources are provisioned: + * User Pool + * Custom Resource Lambda Function so that downstream modules can fetch OAuth2 ClientId/ClientSecret and save to AWS SecretsManager + + For Keycloak (TODO) + ------------ + + Below resources are provisioned: + * ASG + LaunchTemplate + EC2 Instances + * External ALB Endpoint + * Internal ALB Endpoint + * Multi-AZ RDS (Aurora) + * SecretManager Secrets + * CDK Custom Resource as a shim for down stream modules to register OAuth2 clients, Resource Servers + """ + + def __init__(self, scope: constructs.Construct, + cluster_name: str, + aws_region: str, + aws_profile: str, + module_id: str, + deployment_id: str, + termination_protection: bool = True, + env: cdk.Environment = None): + + super().__init__( + scope=scope, + cluster_name=cluster_name, + aws_region=aws_region, + aws_profile=aws_profile, + module_id=module_id, + deployment_id=deployment_id, + termination_protection=termination_protection, + description=f'ModuleId: {module_id}, Cluster: {cluster_name}, Version: {ideaadministrator.props.current_release_version}', + tags={ + constants.IDEA_TAG_MODULE_ID: module_id, + constants.IDEA_TAG_MODULE_NAME: constants.MODULE_IDENTITY_PROVIDER, + constants.IDEA_TAG_MODULE_VERSION: ideaadministrator.props.current_release_version + }, + env=env + ) + + self.cluster = ExistingSocaCluster(self.context, self.stack) + + self.user_pool: Optional[UserPool] = None + self.oauth_credentials_lambda: Optional[LambdaFunction] = None + + provider = self.context.config().get_string('identity-provider.provider', required=True) + if provider == constants.IDENTITY_PROVIDER_KEYCLOAK: + + raise exceptions.general_exception(f'identity provider: {provider} not supported (yet).') + + elif provider == constants.IDENTITY_PROVIDER_COGNITO_IDP: + + self.build_cognito_idp() + self.build_cognito_cluster_settings() + + else: + + raise exceptions.general_exception(f'identity provider: {provider} not supported') + + def build_cognito_idp(self): + + removal_policy_value = self.context.config().get_string('identity-provider.cognito.removal_policy', required=True) + removal_policy = cdk.RemovalPolicy(removal_policy_value) + + # Note: do not use the custom dns name, as the dns name might not have been configured during initial deployment. + # admins must manually update the email invitation template via AWS Console after the stack is deployed. + external_alb_dns = self.context.config().get_string('cluster.load_balancers.external_alb.load_balancer_dns_name', required=True) + external_endpoint = f'https://{external_alb_dns}' + + # do not touch the user pool invitation emails after the initial cluster creation. + # if the user pool is already created, read the existing values and set them again to avoid replacing the values + # during cdk stack update + user_pool_id = self.context.config().get_string('identity-provider.cognito.user_pool_id') + if Utils.is_empty(user_pool_id): + user_invitation_email_subject = f'Invitation to Join IDEA Cluster: {self.cluster_name}' + email_message = [ + '

Hello {username},

', + f'

You have been invited to join the {self.cluster_name} cluster.

', + f'

Your temporary password is:

', + '

{####}

', + '

You can sign in to your account using the link below:
', + f'{external_endpoint}

', + f'

---
', + f'IDEA Cluster Admin

' + ] + user_invitation_email_body = os.linesep.join(email_message) + else: + describe_user_pool_result = self.context.aws().cognito_idp().describe_user_pool( + UserPoolId=user_pool_id + ) + invite_message_template = describe_user_pool_result['UserPool']['AdminCreateUserConfig']['InviteMessageTemplate'] + user_invitation_email_subject = invite_message_template['EmailSubject'] + user_invitation_email_body = invite_message_template['EmailMessage'] + + self.user_pool = UserPool( + context=self.context, + name=f'{self.cluster_name}-user-pool', + scope=self.stack, + props=cognito.UserPoolProps( + removal_policy=removal_policy, + user_invitation=cognito.UserInvitationConfig( + email_subject=user_invitation_email_subject, + email_body=user_invitation_email_body + ) + ) + ) + + # build lambda function in cluster stack to retrieve the client secret + # the clientId and client secret is saved in secrets manager for each module + # since we do not want to create a lambda function for each module, we create this lambda function during + # cluster stack creation, save the lambda function arn in cluster config, and + # then invoke it as a custom resource during individual module stack creation + + lambda_name = 'oauth-credentials' + oauth_credentials_lambda_role = Role( + context=self.context, + name=f'{lambda_name}-role', + scope=self.stack, + description=f'Role for auth credentials Lambda function for Cluster: {self.cluster_name}', + assumed_by=['lambda']) + + oauth_credentials_lambda_role.attach_inline_policy(Policy( + context=self.context, + name=f'{lambda_name}-policy', + scope=self.stack, + policy_template_name='custom-resource-get-user-pool-client-secret.yml' + )) + self.oauth_credentials_lambda = LambdaFunction( + context=self.context, + name=lambda_name, + scope=self.stack, + idea_code_asset=IdeaCodeAsset( + lambda_package_name='idea_custom_resource_get_user_pool_client_secret', + lambda_platform=SupportedLambdaPlatforms.PYTHON + ), + description='Get OAuth Credentials for a ClientId in UserPool', + timeout_seconds=180, + role=oauth_credentials_lambda_role, + log_retention_role=self.cluster.get_role(app_constants.LOG_RETENTION_ROLE_NAME) + ) + self.oauth_credentials_lambda.node.add_dependency(oauth_credentials_lambda_role) + + def build_cognito_cluster_settings(self): + + cluster_settings = { + 'deployment_id': self.deployment_id, + 'cognito.user_pool_id': self.user_pool.user_pool.user_pool_id, + 'cognito.provider_url': self.user_pool.user_pool.user_pool_provider_url, + 'cognito.domain_url': self.user_pool.domain.base_url(fips=True if self.aws_region in Utils.get_value_as_list('COGNITO_REQUIRE_FIPS_ENDPOINT_REGION_LIST', constants.CAVEATS, []) else False), + 'cognito.oauth_credentials_lambda_arn': self.oauth_credentials_lambda.function_arn + } + + self.update_cluster_settings(cluster_settings) diff --git a/source/idea/idea-administrator/src/ideaadministrator/app/cdk/stacks/metrics_stack.py b/source/idea/idea-administrator/src/ideaadministrator/app/cdk/stacks/metrics_stack.py new file mode 100644 index 00000000..91dbc78c --- /dev/null +++ b/source/idea/idea-administrator/src/ideaadministrator/app/cdk/stacks/metrics_stack.py @@ -0,0 +1,127 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +from ideadatamodel import ( + constants, + exceptions +) + +import ideaadministrator +from ideaadministrator.app.cdk.stacks import IdeaBaseStack + +from ideaadministrator.app.cdk.constructs import ( + ExistingSocaCluster +) + +from typing import Optional +import aws_cdk as cdk +import constructs +from aws_cdk import ( + aws_aps as aps, + aws_cloudwatch as cloudwatch +) + + +class MetricsStack(IdeaBaseStack): + + def __init__(self, scope: constructs.Construct, + cluster_name: str, + aws_region: str, + aws_profile: str, + module_id: str, + deployment_id: str, + termination_protection: bool = True, + env: cdk.Environment = None): + super().__init__( + scope=scope, + cluster_name=cluster_name, + aws_region=aws_region, + aws_profile=aws_profile, + module_id=module_id, + deployment_id=deployment_id, + termination_protection=termination_protection, + description=f'ModuleId: {module_id}, Cluster: {cluster_name}, Version: {ideaadministrator.props.current_release_version}', + tags={ + constants.IDEA_TAG_MODULE_ID: module_id, + constants.IDEA_TAG_MODULE_NAME: constants.MODULE_METRICS, + constants.IDEA_TAG_MODULE_VERSION: ideaadministrator.props.current_release_version + }, + env=env + ) + + self.cluster = ExistingSocaCluster(self.context, self.stack) + + self.cloudwatch_dashboard: Optional[cloudwatch.Dashboard] = None + self.amazon_prometheus_workspace: Optional[aps.CfnWorkspace] = None + + if self.is_cloudwatch(): + self.build_cloudwatch() + elif self.is_amazon_managed_prometheus(): + self.build_amazon_managed_prometheus() + elif self.is_prometheus(): + self.build_prometheus() + else: + raise exceptions.general_exception(f'metrics provider: {self.get_metrics_provider()} not supported') + + self.build_cluster_settings() + + def get_metrics_provider(self) -> str: + return self.context.config().get_string('metrics.provider', required=True) + + def is_cloudwatch(self) -> bool: + return self.get_metrics_provider() == constants.METRICS_PROVIDER_CLOUDWATCH + + def is_amazon_managed_prometheus(self) -> bool: + return self.get_metrics_provider() == constants.METRICS_PROVIDER_AMAZON_MANAGED_PROMETHEUS + + def is_prometheus(self) -> bool: + return self.get_metrics_provider() == constants.METRICS_PROVIDER_PROMETHEUS + + def build_cloudwatch(self): + dashboard_name = self.context.config().get_string('metrics.cloudwatch.dashboard_name', required=True) + # todo - add widgets based on modules deployed + self.cloudwatch_dashboard = cloudwatch.Dashboard( + scope=self.stack, + id='cloudwatch-dashboard', + dashboard_name=dashboard_name + ) + + def build_amazon_managed_prometheus(self): + workspace_name = self.context.config().get_string('metrics.amazon_managed_prometheus.workspace_name', required=True) + self.amazon_prometheus_workspace = aps.CfnWorkspace( + scope=self.stack, + id='prometheus-workspace', + alias=workspace_name + ) + self.add_common_tags(self.amazon_prometheus_workspace) + + def build_prometheus(self): + # validate and do nothing as of now. + # might need additional configurations to provision secrets to authenticate with remote write url + self.context.config().get_string('metrics.prometheus.remote_write.url', required=True) + self.context.config().get_string('metrics.prometheus.query.url', required=True) + + def build_cluster_settings(self): + cluster_settings = { + 'deployment_id': self.deployment_id + } + + if self.is_cloudwatch(): + cluster_settings['cloudwatch.dashboard_arn'] = self.cloudwatch_dashboard.dashboard_arn + elif self.is_amazon_managed_prometheus(): + cluster_settings['amazon_managed_prometheus.workspace_id'] = self.amazon_prometheus_workspace.attr_workspace_id + cluster_settings['amazon_managed_prometheus.workspace_arn'] = self.amazon_prometheus_workspace.attr_arn + cluster_settings['prometheus.remote_write.url'] = f'{self.amazon_prometheus_workspace.attr_prometheus_endpoint}api/v1/remote_write' + cluster_settings['prometheus.remote_read.url'] = f'{self.amazon_prometheus_workspace.attr_prometheus_endpoint}api/v1/query' + elif self.is_prometheus(): + pass + + self.update_cluster_settings(cluster_settings) diff --git a/source/idea/idea-administrator/src/ideaadministrator/app/cdk/stacks/scheduler_stack.py b/source/idea/idea-administrator/src/ideaadministrator/app/cdk/stacks/scheduler_stack.py new file mode 100644 index 00000000..a329f600 --- /dev/null +++ b/source/idea/idea-administrator/src/ideaadministrator/app/cdk/stacks/scheduler_stack.py @@ -0,0 +1,516 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +from ideadatamodel import ( + constants, + SocaAnyPayload +) +from ideasdk.bootstrap import BootstrapUserDataBuilder +from ideasdk.utils import Utils + +import ideaadministrator +from ideaadministrator.app.cdk.constructs import ( + ExistingSocaCluster, + OAuthClientIdAndSecret, + SQSQueue, + Policy, + Role, + InstanceProfile, + ComputeNodeSecurityGroup, + SchedulerSecurityGroup, + IdeaNagSuppression +) +from ideaadministrator.app.cdk.stacks import IdeaBaseStack + +from typing import Optional + +import aws_cdk as cdk +import constructs +from aws_cdk import ( + aws_ec2 as ec2, + aws_cognito as cognito, + aws_sqs as sqs, + aws_route53 as route53, + aws_elasticloadbalancingv2 as elbv2 +) + + +class SchedulerStack(IdeaBaseStack): + """ + Scheduler Stack + + Provisions infrastructure for Scale-Out Computing on AWS / HPC Scheduler module. + """ + + def __init__(self, scope: constructs.Construct, + cluster_name: str, + aws_region: str, + aws_profile: str, + module_id: str, + deployment_id: str, + termination_protection: bool = True, + env: cdk.Environment = None): + + super().__init__( + scope=scope, + cluster_name=cluster_name, + aws_region=aws_region, + aws_profile=aws_profile, + module_id=module_id, + deployment_id=deployment_id, + termination_protection=termination_protection, + description=f'ModuleId: {module_id}, Cluster: {cluster_name}, Version: {ideaadministrator.props.current_release_version}', + tags={ + constants.IDEA_TAG_MODULE_ID: module_id, + constants.IDEA_TAG_MODULE_NAME: constants.MODULE_SCHEDULER, + constants.IDEA_TAG_MODULE_VERSION: ideaadministrator.props.current_release_version + }, + env=env + ) + + self.bootstrap_package_uri = self.stack.node.try_get_context('bootstrap_package_uri') + self.cluster = ExistingSocaCluster(self.context, self.stack) + + self.oauth2_client_secret: Optional[OAuthClientIdAndSecret] = None + self.scheduler_role: Optional[Role] = None + self.scheduler_instance_profile: Optional[InstanceProfile] = None + self.compute_node_role: Optional[Role] = None + self.compute_node_instance_profile: Optional[InstanceProfile] = None + self.spot_fleet_request_role: Optional[Role] = None + self.scheduler_security_group: Optional[SchedulerSecurityGroup] = None + self.compute_node_security_group: Optional[ComputeNodeSecurityGroup] = None + self.job_status_sqs_queue: Optional[SQSQueue] = None + self.ec2_instance: Optional[ec2.CfnInstance] = None + self.cluster_dns_record_set: Optional[route53.RecordSet] = None + self.external_endpoint: Optional[cdk.CustomResource] = None + self.internal_endpoint: Optional[cdk.CustomResource] = None + + self.user_pool = self.lookup_user_pool() + + self.build_oauth2_client() + self.build_access_control_groups(user_pool=self.user_pool) + self.build_sqs_queue() + self.build_iam_roles() + self.build_security_groups() + self.build_ec2_instance() + self.build_route53_record_set() + self.build_endpoints() + self.build_cluster_settings() + + def build_oauth2_client(self): + # add resource server + resource_server = self.user_pool.add_resource_server( + id='resource-server', + identifier=self.module_id, + scopes=[ + cognito.ResourceServerScope(scope_name='read', scope_description='Allow Read Access'), + cognito.ResourceServerScope(scope_name='write', scope_description='Allow Write Access') + ] + ) + + # add new client to user pool + client = self.user_pool.add_client( + id=f'{self.module_id}-client', + access_token_validity=cdk.Duration.hours(1), + generate_secret=True, + id_token_validity=cdk.Duration.hours(1), + o_auth=cognito.OAuthSettings( + flows=cognito.OAuthFlows(client_credentials=True), + scopes=[ + cognito.OAuthScope.custom(f'{self.module_id}/read'), + cognito.OAuthScope.custom(f'{self.module_id}/write'), + cognito.OAuthScope.custom(f'{self.context.config().get_module_id(constants.MODULE_CLUSTER_MANAGER)}/read') + ] + ), + refresh_token_validity=cdk.Duration.days(30), + user_pool_client_name=self.module_id + ) + client.node.add_dependency(resource_server) + + # read secret value by invoking custom resource + oauth_credentials_lambda_arn = self.context.config().get_string('identity-provider.cognito.oauth_credentials_lambda_arn', required=True) + client_secret = cdk.CustomResource( + scope=self.stack, + id=f'{self.module_id}-creds', + service_token=oauth_credentials_lambda_arn, + properties={ + 'UserPoolId': self.user_pool.user_pool_id, + 'ClientId': client.user_pool_client_id + }, + resource_type='Custom::GetOAuthCredentials' + ) + + # save client id and client secret to AWS Secrets Manager + self.oauth2_client_secret = OAuthClientIdAndSecret( + context=self.context, + secret_name_prefix=self.module_id, + module_name=constants.MODULE_SCHEDULER, + scope=self.stack, + client_id=client.user_pool_client_id, + client_secret=client_secret.get_att_string('ClientSecret') + ) + + def build_iam_roles(self): + + ec2_managed_policies = self.get_ec2_instance_managed_policies() + + # scheduler + scheduler_policy_arns = self.context.config().get_list('cluster.iam.scheduler_iam_policy_arns', []) + scheduler_policy_arns += ec2_managed_policies + scheduler_policy_arns = list(set(scheduler_policy_arns)) + self.scheduler_role = Role( + context=self.context, + name=f'{self.module_id}-role', + scope=self.stack, + description='IAM role assigned to the scheduler', + assumed_by=['ssm', 'ec2'], + managed_policies=scheduler_policy_arns + ) + self.scheduler_instance_profile = InstanceProfile( + context=self.context, + name=f'{self.module_id}-scheduler-instance-profile', + scope=self.stack, + roles=[self.scheduler_role] + ) + + # compute node + compute_node_policy_arns = self.context.config().get_list('cluster.iam.compute_node_iam_policy_arns', []) + compute_node_policy_arns += ec2_managed_policies + compute_node_policy_arns = list(set(compute_node_policy_arns)) + self.compute_node_role = Role( + context=self.context, + name=f'{self.module_id}-compute-node-role', + scope=self.stack, + description='IAM role assigned to the compute nodes', + assumed_by=['ssm', 'ec2'], + managed_policies=compute_node_policy_arns + ) + self.compute_node_instance_profile = InstanceProfile( + context=self.context, + name=f'{self.module_id}-compute-node-instance-profile', + scope=self.stack, + roles=[self.compute_node_role] + ) + + # spot fleet request + self.spot_fleet_request_role = Role( + context=self.context, + name=f'{self.module_id}-spot-fleet-request-role', + scope=self.stack, + description='IAM role to manage SpotFleet requests', + assumed_by=['spotfleet'] + ) + + variables = SocaAnyPayload() + variables.scheduler_role_arn = self.scheduler_role.role_arn + variables.compute_node_role_arn = self.compute_node_role.role_arn + variables.spot_fleet_request_role_arn = self.spot_fleet_request_role.role_arn + + self.scheduler_role.attach_inline_policy( + Policy( + context=self.context, + name='scheduler-policy', + scope=self.stack, + policy_template_name='scheduler.yml', + vars=variables, + module_id=self.module_id + ) + ) + self.compute_node_role.attach_inline_policy( + Policy( + context=self.context, + name='compute-node-policy', + scope=self.stack, + policy_template_name='compute-node.yml', + vars=variables, + module_id=self.module_id + ) + ) + self.spot_fleet_request_role.attach_inline_policy( + Policy( + context=self.context, + name='spot-fleet-policy', + scope=self.stack, + policy_template_name='spot-fleet-request.yml', + vars=variables, + module_id=self.module_id + ) + ) + + def build_sqs_queue(self): + kms_key_id = self.context.config().get_string('cluster.sqs.kms_key_id') + + self.job_status_sqs_queue = SQSQueue( + self.context, 'job-status-events', self.stack, + queue_name=f'{self.cluster_name}-{self.module_id}-job-status-events', + encryption_master_key=kms_key_id, + dead_letter_queue=sqs.DeadLetterQueue( + max_receive_count=10, + queue=SQSQueue( + self.context, 'job-status-events-dlq', self.stack, + queue_name=f'{self.cluster_name}-{self.module_id}-job-status-events-dlq', + encryption_master_key=kms_key_id, + is_dead_letter_queue=True + ) + ) + ) + self.add_common_tags(self.job_status_sqs_queue) + self.add_common_tags(self.job_status_sqs_queue.dead_letter_queue.queue) + + def build_security_groups(self): + self.scheduler_security_group = SchedulerSecurityGroup( + context=self.context, + name=f'{self.module_id}-security-group', + scope=self.stack, + vpc=self.cluster.vpc, + bastion_host_security_group=self.cluster.get_security_group('bastion-host'), + loadbalancer_security_group=self.cluster.get_security_group('external-load-balancer') + ) + + self.compute_node_security_group = ComputeNodeSecurityGroup( + context=self.context, + name=f'{self.module_id}-compute-node-security-group', + scope=self.stack, + vpc=self.cluster.vpc + ) + + def build_ec2_instance(self): + + is_public = self.context.config().get_bool('scheduler.public', False) + base_os = self.context.config().get_string('scheduler.base_os', required=True) + instance_ami = self.context.config().get_string('scheduler.instance_ami', required=True) + instance_type = self.context.config().get_string('scheduler.instance_type', required=True) + volume_size = self.context.config().get_int('scheduler.volume_size', 200) + key_pair_name = self.context.config().get_string('cluster.network.ssh_key_pair', required=True) + enable_detailed_monitoring = self.context.config().get_bool('scheduler.ec2.enable_detailed_monitoring', default=False) + enable_termination_protection = self.context.config().get_bool('scheduler.ec2.enable_termination_protection', default=False) + metadata_http_tokens = self.context.config().get_string('scheduler.ec2.metadata_http_tokens', required=True) + + if is_public and len(self.cluster.public_subnets) > 0: + subnet_ids = self.cluster.existing_vpc.get_public_subnet_ids() + else: + subnet_ids = self.cluster.existing_vpc.get_private_subnet_ids() + + block_device_name = Utils.get_ec2_block_device_name(base_os) + + user_data = BootstrapUserDataBuilder( + aws_region=self.aws_region, + bootstrap_package_uri=self.bootstrap_package_uri, + install_commands=[ + '/bin/bash scheduler/setup.sh' + ], + base_os=base_os + ).build() + + launch_template = ec2.LaunchTemplate( + self.stack, f'{self.module_id}-lt', + instance_type=ec2.InstanceType(instance_type), + machine_image=ec2.MachineImage.generic_linux({ + self.aws_region: instance_ami + }), + user_data=ec2.UserData.custom(cdk.Fn.sub(user_data)), + key_name=key_pair_name, + block_devices=[ec2.BlockDevice( + device_name=block_device_name, + volume=ec2.BlockDeviceVolume(ebs_device=ec2.EbsDeviceProps( + volume_size=volume_size, + volume_type=ec2.EbsDeviceVolumeType.GP3 + ) + ) + )], + require_imdsv2=True if metadata_http_tokens == "required" else False + ) + + self.ec2_instance = ec2.CfnInstance( + self.stack, + f'{self.module_id}-instance', + block_device_mappings=[ + ec2.CfnInstance.BlockDeviceMappingProperty( + device_name=block_device_name, + ebs=ec2.CfnInstance.EbsProperty( + volume_size=volume_size, + volume_type='gp3' + ) + ) + ], + disable_api_termination=enable_termination_protection, + iam_instance_profile=self.scheduler_instance_profile.instance_profile_name, + instance_type=instance_type, + image_id=instance_ami, + key_name=key_pair_name, + launch_template=ec2.CfnInstance.LaunchTemplateSpecificationProperty( + version=launch_template.latest_version_number, + launch_template_id=launch_template.launch_template_id), + network_interfaces=[ + ec2.CfnInstance.NetworkInterfaceProperty( + device_index='0', + associate_public_ip_address=is_public, + group_set=[self.scheduler_security_group.security_group_id], + subnet_id=subnet_ids[0], + ) + ], + user_data=cdk.Fn.base64(cdk.Fn.sub(user_data)), + monitoring=enable_detailed_monitoring + ) + cdk.Tags.of(self.ec2_instance).add('Name', self.build_resource_name(self.module_id)) + cdk.Tags.of(self.ec2_instance).add(constants.IDEA_TAG_NODE_TYPE, constants.NODE_TYPE_APP) + self.add_backup_tags(self.ec2_instance) + + if not enable_detailed_monitoring: + self.add_nag_suppression( + construct=self.ec2_instance, + suppressions=[IdeaNagSuppression(rule_id='AwsSolutions-EC28', reason='detailed monitoring is a configurable option to save costs')] + ) + + if not enable_termination_protection: + self.add_nag_suppression( + construct=self.ec2_instance, + suppressions=[IdeaNagSuppression(rule_id='AwsSolutions-EC29', reason='termination protection not supported in CDK L2 construct. enable termination protection via AWS EC2 console after deploying the cluster.')] + ) + + def build_route53_record_set(self): + hostname = self.context.config().get_string('scheduler.hostname', required=True) + private_hosted_zone_id = self.context.config().get_string('cluster.route53.private_hosted_zone_id', required=True) + private_hosted_zone_name = self.context.config().get_string('cluster.route53.private_hosted_zone_name', required=True) + self.cluster_dns_record_set = route53.RecordSet( + self.stack, + f'{self.module_id}-dns-record', + record_type=route53.RecordType.A, + target=route53.RecordTarget.from_ip_addresses(self.ec2_instance.attr_private_ip), + ttl=cdk.Duration.minutes(5), + record_name=hostname, + zone=route53.HostedZone.from_hosted_zone_attributes( + scope=self.stack, + id='cluster-dns', + hosted_zone_id=private_hosted_zone_id, + zone_name=private_hosted_zone_name + ) + ) + + def build_endpoints(self): + external_target_group = elbv2.CfnTargetGroup( + self.stack, + f'{self.module_id}-external-target-group', + port=8443, + protocol='HTTPS', + target_type='ip', + vpc_id=self.cluster.vpc.vpc_id, + name=self.get_target_group_name('sched-ext'), + targets=[elbv2.CfnTargetGroup.TargetDescriptionProperty( + id=self.ec2_instance.attr_private_ip + )], + health_check_path='/healthcheck' + ) + cluster_endpoints_lambda_arn = self.context.config().get_string('cluster.cluster_endpoints_lambda_arn', required=True) + external_https_listener_arn = self.context.config().get_string('cluster.load_balancers.external_alb.https_listener_arn', required=True) + external_endpoint_priority = self.context.config().get_int('scheduler.endpoints.external.priority', required=True) + external_endpoint_path_patterns = self.context.config().get_list('scheduler.endpoints.external.path_patterns', required=True) + + self.external_endpoint = cdk.CustomResource( + self.stack, + 'external-endpoint', + service_token=cluster_endpoints_lambda_arn, + properties={ + 'endpoint_name': f'{self.module_id}-external-endpoint', + 'listener_arn': external_https_listener_arn, + 'priority': external_endpoint_priority, + 'target_group_arn': external_target_group.ref, + 'conditions': [ + { + 'Field': 'path-pattern', + 'Values': external_endpoint_path_patterns + } + ], + 'actions': [ + { + 'Type': 'forward', + 'TargetGroupArn': external_target_group.ref + } + ], + 'tags': { + constants.IDEA_TAG_CLUSTER_NAME: self.cluster_name, + constants.IDEA_TAG_MODULE_ID: self.module_id, + constants.IDEA_TAG_MODULE_NAME: constants.MODULE_SCHEDULER + } + }, + resource_type='Custom::SchedulerEndpointExternal' + ) + + internal_https_listener_arn = self.context.config().get_string('cluster.load_balancers.internal_alb.https_listener_arn', + required=True) + internal_endpoint_priority = self.context.config().get_int('scheduler.endpoints.internal.priority', required=True) + internal_endpoint_path_patterns = self.context.config().get_list('scheduler.endpoints.internal.path_patterns', required=True) + + internal_target_group = elbv2.CfnTargetGroup( + self.stack, + f'{self.module_id}-internal-target-group', + port=8443, + protocol='HTTPS', + target_type='ip', + vpc_id=self.cluster.vpc.vpc_id, + name=self.get_target_group_name('sched-int'), + targets=[elbv2.CfnTargetGroup.TargetDescriptionProperty( + id=self.ec2_instance.attr_private_ip + )], + health_check_path='/healthcheck' + ) + + self.internal_endpoint = cdk.CustomResource( + self.stack, + 'internal-endpoint', + service_token=cluster_endpoints_lambda_arn, + properties={ + 'endpoint_name': f'{self.module_id}-internal-endpoint', + 'listener_arn': internal_https_listener_arn, + 'priority': internal_endpoint_priority, + 'target_group_arn': internal_target_group.ref, + 'conditions': [ + { + 'Field': 'path-pattern', + 'Values': internal_endpoint_path_patterns + } + ], + 'actions': [ + { + 'Type': 'forward', + 'TargetGroupArn': internal_target_group.ref + } + ], + 'tags': { + constants.IDEA_TAG_CLUSTER_NAME: self.cluster_name, + constants.IDEA_TAG_MODULE_ID: self.module_id, + constants.IDEA_TAG_MODULE_NAME: constants.MODULE_SCHEDULER + } + }, + resource_type='Custom::SchedulerEndpointInternal' + ) + + def build_cluster_settings(self): + cluster_settings = {} + cluster_settings['deployment_id'] = self.deployment_id + cluster_settings['private_ip'] = self.ec2_instance.attr_private_ip + cluster_settings['private_dns_name'] = self.ec2_instance.attr_private_dns_name + + is_public = self.context.config().get_bool('scheduler.public', False) + if is_public: + cluster_settings['public_ip'] = self.ec2_instance.attr_public_ip + cluster_settings['instance_id'] = self.ec2_instance.ref + cluster_settings['client_id'] = self.oauth2_client_secret.client_id.ref + cluster_settings['client_secret'] = self.oauth2_client_secret.client_secret.ref + cluster_settings['security_group_id'] = self.scheduler_security_group.security_group_id + cluster_settings['iam_role_arn'] = self.scheduler_role.role_arn + cluster_settings['compute_node_security_group_ids'] = [self.compute_node_security_group.security_group_id] + cluster_settings['compute_node_iam_role_arn'] = self.compute_node_role.role_arn + cluster_settings['compute_node_instance_profile_arn'] = self.compute_node_instance_profile.ref + cluster_settings['spot_fleet_request_iam_role_arn'] = self.spot_fleet_request_role.role_arn + cluster_settings['job_status_sqs_queue_url'] = self.job_status_sqs_queue.queue_url + + self.update_cluster_settings(cluster_settings) diff --git a/source/idea/idea-administrator/src/ideaadministrator/app/cdk/stacks/shared_storage_stack.py b/source/idea/idea-administrator/src/ideaadministrator/app/cdk/stacks/shared_storage_stack.py new file mode 100644 index 00000000..e94a4ceb --- /dev/null +++ b/source/idea/idea-administrator/src/ideaadministrator/app/cdk/stacks/shared_storage_stack.py @@ -0,0 +1,238 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +from ideadatamodel import ( + constants, exceptions +) +from ideasdk.utils import Utils + +import ideaadministrator +from ideaadministrator.app.cdk.stacks import IdeaBaseStack +from ideaadministrator import app_constants +from ideaadministrator.app_context import AdministratorContext + +from ideaadministrator.app.cdk.constructs import ( + AmazonEFS, + FSxForLustre, + SharedStorageSecurityGroup, + ExistingSocaCluster +) +from typing import Optional, Union, Dict, List +import aws_cdk as cdk +import constructs + + +class FileSystemHolder: + """ + Holder object to build cluster settings + """ + + def __init__(self, context: AdministratorContext, + name: str, + storage_config: Dict, + file_system_id: str, + file_system: Optional[Union[AmazonEFS, FSxForLustre]] = None): + + self._context = context + self.name = name + self.storage_config = storage_config + self.file_system_id = file_system_id + self.file_system = file_system + + @property + def provider(self) -> str: + return self.storage_config.get('provider') + + def get_fs_type(self) -> str: + if self.provider == 'efs': + return 'efs' + else: + return 'fsx' + + @property + def file_system_dns(self) -> str: + aws_region = self._context.config().get_string('cluster.aws.region', required=True) + aws_dns_suffix = self._context.config().get_string('cluster.aws.dns_suffix', required=True) + return f'{self.file_system_id}.{self.get_fs_type()}.{aws_region}.{aws_dns_suffix}' + + +class SharedStorageStack(IdeaBaseStack): + """ + Shared Storage Stack + * Provisions applicable file systems configured in shared-storage settings. + * Apps and Data file systems are mandatory and must exist in configuration. + """ + + def __init__(self, scope: constructs.Construct, + cluster_name: str, + aws_region: str, + aws_profile: str, + module_id: str, + deployment_id: str, + termination_protection: bool = True, + env: cdk.Environment = None): + + super().__init__( + scope=scope, + cluster_name=cluster_name, + aws_region=aws_region, + aws_profile=aws_profile, + module_id=module_id, + deployment_id=deployment_id, + termination_protection=termination_protection, + description=f'ModuleId: {module_id}, Cluster: {cluster_name}, Version: {ideaadministrator.props.current_release_version}', + tags={ + constants.IDEA_TAG_MODULE_ID: module_id, + constants.IDEA_TAG_MODULE_NAME: constants.MODULE_SHARED_STORAGE, + constants.IDEA_TAG_MODULE_VERSION: ideaadministrator.props.current_release_version + }, + env=env + ) + + self.security_group: Optional[SharedStorageSecurityGroup] = None + self.file_systems: List[FileSystemHolder] = [] + + self.cluster = ExistingSocaCluster(self.context, self.stack) + + # verify if `apps` and `data` storage is configured + # simple check to ensure apps and data config keys exist + self.context.config().get_string('shared-storage.apps.provider', required=True) + self.context.config().get_string('shared-storage.data.provider', required=True) + + # build security group + self.build_security_group() + + # build applicable file systems + self.build_shared_storage() + + # build cluster settings + self.build_cluster_settings() + + def build_security_group(self): + self.security_group = SharedStorageSecurityGroup( + context=self.context, + name='shared-storage-security-group', + scope=self.stack, + vpc=self.cluster.vpc + ) + + def build_shared_storage(self): + """ + loop over all file system settings and provisions new file systems for applicable provider + provisioning of new file systems is not limited to Apps and Data. + """ + storage_configs = self.context.config().get_config('shared-storage', required=True) + for key in storage_configs: + storage_config = storage_configs.get(key) + + # future-proof - so that if/when new non storage config keys are added, implementation should not bomb + if not isinstance(storage_config, Dict): + continue + + # if the dict/config object does not contain 'provider' and 'mount_dir' key, assume it is not a storage config + provider = storage_config.get('provider', default=None) + mount_dir = storage_config.get('mount_dir', default=None) + if Utils.is_empty(provider) or Utils.is_empty(mount_dir): + continue + + if provider not in constants.SUPPORTED_STORAGE_PROVIDERS: + raise exceptions.cluster_config_error(f'file system provider: {provider} not supported') + + # if existing file system, add to registry and skip provisioning. + existing_fs = storage_config.get(f'{provider}.use_existing_fs', default=False) + if existing_fs: + file_system_id = storage_config.get(f'{provider}.file_system_id') + if Utils.is_empty(file_system_id): + raise exceptions.cluster_config_error(f'shared-storage.{key}.{provider}.file_system_id is required') + + self.file_systems.append(FileSystemHolder( + context=self.context, + name=key, + storage_config=storage_config, + file_system_id=file_system_id + )) + continue + + # provision new file systems + if provider == constants.STORAGE_PROVIDER_EFS: + self.build_efs(name=key, storage_config=storage_config) + elif provider == constants.STORAGE_PROVIDER_FSX_LUSTRE: + self.build_fsx_lustre(name=key, storage_config=storage_config) + + def build_efs(self, name: str, storage_config: Dict): + """ + provision new EFS + """ + efs = AmazonEFS( + context=self.context, + name=f'{name}-storage-efs', + scope=self.stack, + vpc=self.cluster.vpc, + efs_config=storage_config.get('efs'), + security_group=self.security_group, + log_retention_role=self.cluster.get_role(app_constants.LOG_RETENTION_ROLE_NAME), + subnets=self.cluster.private_subnets + ) + self.file_systems.append(FileSystemHolder( + context=self.context, + name=name, + storage_config=storage_config, + file_system_id=efs.file_system.ref, + file_system=efs + )) + + def build_fsx_lustre(self, name: str, storage_config: Dict): + """ + provision new FSx for Lustre file system + """ + fsx_lustre = FSxForLustre( + context=self.context, + name=f'{name}-storage-fsx-lustre', + scope=self.stack, + vpc=self.cluster.vpc, + fsx_lustre_config=storage_config.get('fsx_lustre'), + security_group=self.security_group, + subnets=self.cluster.private_subnets + ) + self.file_systems.append(FileSystemHolder( + context=self.context, + name=name, + storage_config=storage_config, + file_system_id=fsx_lustre.file_system.ref, + file_system=fsx_lustre + )) + + def build_cluster_settings(self): + + cluster_settings = { + 'deployment_id': self.deployment_id, + 'security_group_id': self.security_group.security_group_id + } + + for file_system in self.file_systems: + + # cluster settings for file system dns need to be updated only if new file systems are provisioned for efs and fsx_lustre + # if existing file systems configurations are added/updated via ./idea-admin.sh shared-storage utility, dns property will already exist. + if file_system.provider in [ + constants.STORAGE_PROVIDER_EFS, + constants.STORAGE_PROVIDER_FSX_LUSTRE + ]: + file_system_dns = self.context.config().get_string(f'shared-storage.{self.name}.dns') + if Utils.is_empty(file_system_dns): + cluster_settings[f'{file_system.name}.{file_system.provider}.dns'] = file_system.file_system_dns + + # skip settings update for existing file systems + if file_system.file_system is None: + continue + + cluster_settings[f'{file_system.name}.{file_system.provider}.file_system_id'] = file_system.file_system_id + + self.update_cluster_settings(cluster_settings) diff --git a/source/idea/idea-administrator/src/ideaadministrator/app/cdk/stacks/virtual_desktop_controller_stack.py b/source/idea/idea-administrator/src/ideaadministrator/app/cdk/stacks/virtual_desktop_controller_stack.py new file mode 100644 index 00000000..ec421ecc --- /dev/null +++ b/source/idea/idea-administrator/src/ideaadministrator/app/cdk/stacks/virtual_desktop_controller_stack.py @@ -0,0 +1,1058 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +import ideaadministrator +from ideaadministrator.app.cdk.idea_code_asset import IdeaCodeAsset, SupportedLambdaPlatforms +from ideaadministrator.app.cdk.constructs import ( + ExistingSocaCluster, + OAuthClientIdAndSecret, + Role, + InstanceProfile, + VirtualDesktopBastionAccessSecurityGroup, + VirtualDesktopPublicLoadBalancerAccessSecurityGroup, + SQSQueue, + SNSTopic, + Policy, + LambdaFunction, + SecurityGroup, + IdeaNagSuppression, + BackupPlan +) +from ideaadministrator.app.cdk.stacks import IdeaBaseStack +from ideadatamodel import ( + constants, + SocaAnyPayload +) +from ideadatamodel.constants import IDEA_TAG_MODULE_ID +from ideasdk.bootstrap import BootstrapUserDataBuilder +from ideasdk.context import ArnBuilder +from ideasdk.utils import Utils + +import aws_cdk as cdk +import constructs +from aws_cdk import ( + aws_ec2 as ec2, + aws_cognito as cognito, + aws_sqs as sqs, + aws_events as events, + aws_sns as sns, + aws_sns_subscriptions as sns_subscription, + aws_autoscaling as asg, + aws_elasticloadbalancingv2 as elbv2, + aws_events_targets as events_targets, + aws_s3 as s3, + aws_backup as backup, + aws_iam as iam +) + +from aws_cdk.aws_events import Schedule +from typing import Optional + + +class VirtualDesktopControllerStack(IdeaBaseStack): + """ + Virtual Desktop Controller (eVDI) Stack + + Provisions infrastructure for eVDI Module: + * ASG for Controller, NICE DCV Broker, NICE DCV Connection Gateway + * Network Load Balancer + * Security Groups + * AWS Backups for eVDI Hosts + * Additional components for eVDI functionality. + """ + + def __init__(self, scope: constructs.Construct, cluster_name: str, aws_region: str, aws_profile: str, module_id: str, deployment_id: str, termination_protection: bool = True, env: cdk.Environment = None): + + super().__init__( + scope=scope, + cluster_name=cluster_name, + aws_region=aws_region, + aws_profile=aws_profile, + module_id=module_id, + deployment_id=deployment_id, + termination_protection=termination_protection, + description=f'ModuleId: {module_id}, Cluster: {cluster_name}, Version: {ideaadministrator.props.current_release_version}', + tags={ + constants.IDEA_TAG_MODULE_ID: module_id, + constants.IDEA_TAG_MODULE_NAME: constants.MODULE_VIRTUAL_DESKTOP_CONTROLLER, + constants.IDEA_TAG_MODULE_VERSION: ideaadministrator.props.current_release_version + }, + env=env + ) + + self.COMPONENT_DCV_BROKER = 'broker' + self.COMPONENT_DCV_CONNECTION_GATEWAY = 'gateway' + self.COMPONENT_CONTROLLER = 'controller' + self.CONFIG_MAPPING = { + self.COMPONENT_CONTROLLER: 'controller', + self.COMPONENT_DCV_CONNECTION_GATEWAY: 'dcv_connection_gateway', + self.COMPONENT_DCV_BROKER: 'dcv_broker' + } + self.COMPONENT_DCV_HOST = 'host' + + self.BROKER_CLIENT_COMMUNICATION_PORT = self.context.config().get_int('virtual-desktop-controller.dcv_broker.client_communication_port', required=True) + self.BROKER_AGENT_COMMUNICATION_PORT = self.context.config().get_int('virtual-desktop-controller.dcv_broker.agent_communication_port', required=True) + self.BROKER_GATEWAY_COMMUNICATION_PORT = self.context.config().get_int('virtual-desktop-controller.dcv_broker.gateway_communication_port', required=True) + + self.CLUSTER_ENDPOINTS_LAMBDA_ARN = self.context.config().get_string('cluster.cluster_endpoints_lambda_arn', required=True) + + self.cluster = ExistingSocaCluster(self.context, self.stack) + self.arn_builder = ArnBuilder(self.context.config()) + + self.oauth2_client_secret: Optional[OAuthClientIdAndSecret] = None + + self.dcv_host_role: Optional[Role] = None + self.controller_role: Optional[Role] = None + self.dcv_broker_role: Optional[Role] = None + self.scheduled_event_transformer_lambda_role: Optional[Role] = None + self.dcv_host_instance_profile: Optional[InstanceProfile] = None + + self.dcv_host_security_group: Optional[VirtualDesktopBastionAccessSecurityGroup] = None + self.controller_security_group: Optional[VirtualDesktopPublicLoadBalancerAccessSecurityGroup] = None + self.dcv_connection_gateway_security_group: Optional[VirtualDesktopPublicLoadBalancerAccessSecurityGroup] = None + self.dcv_broker_security_group: Optional[VirtualDesktopBastionAccessSecurityGroup] = None + self.dcv_broker_alb_security_group: Optional[VirtualDesktopBastionAccessSecurityGroup] = None + + self.dcv_connection_gateway_self_signed_cert: Optional[cdk.CustomResource] = None + self.external_nlb: Optional[elbv2.NetworkLoadBalancer] = None + + self.event_sqs_queue: Optional[SQSQueue] = None + self.controller_sqs_queue: Optional[SQSQueue] = None + self.ssm_commands_sns_topic: Optional[SNSTopic] = None + self.ssm_command_pass_role: Optional[Role] = None + + self.controller_auto_scaling_group: Optional[asg.AutoScalingGroup] = None + self.dcv_broker_autoscaling_group: Optional[asg.AutoScalingGroup] = None + self.dcv_connection_gateway_autoscaling_group: Optional[asg.AutoScalingGroup] = None + + self.backup_plan: Optional[BackupPlan] = None + + self.user_pool = self.lookup_user_pool() + + self.build_oauth2_client() + self.build_access_control_groups(user_pool=self.user_pool) + + self.build_sqs_queues() + self.build_scheduled_event_notification_infra() + self.subscribe_to_ec2_notification_events() + + self.build_virtual_desktop_controller() + self.build_dcv_broker() + self.build_dcv_connection_gateway() + self.build_dcv_host_infra() + + self.build_controller_ssm_commands_notification_infra() + self.build_backups() + + self.setup_egress_rules_for_quic() + self.build_cluster_settings() + + def setup_egress_rules_for_quic(self): + quic_supported = self.context.config().get_bool('virtual-desktop-controller.dcv_session.quic_support', required=True) + if not quic_supported: + return + + self.dcv_host_security_group.add_egress_rule( + ec2.Peer.ipv4('0.0.0.0/0'), + ec2.Port.udp_range(0, 65535), + description='Allow all egress for UDP for QUIC Support on DCV Host' + ) + + self.dcv_connection_gateway_security_group.add_egress_rule( + ec2.Peer.ipv4('0.0.0.0/0'), + ec2.Port.udp_range(0, 65535), + description='Allow all egress for UDP for QUIC Support on DCV Connection Gateway' + ) + + def build_sqs_queues(self): + self.event_sqs_queue = SQSQueue( + self.context, 'virtual-desktop-controller-events-queue', self.stack, + queue_name=f'{self.cluster_name}-{self.module_id}-events.fifo', + fifo=True, + fifo_throughput_limit=sqs.FifoThroughputLimit.PER_MESSAGE_GROUP_ID, + deduplication_scope=sqs.DeduplicationScope.MESSAGE_GROUP, + content_based_deduplication=True, + encryption_master_key=self.context.config().get_string('cluster.sqs.kms_key_id'), + dead_letter_queue=sqs.DeadLetterQueue( + max_receive_count=60, + queue=SQSQueue( + self.context, 'virtual-desktop-controller-events-queue-dlq', self.stack, + queue_name=f'{self.cluster_name}-{self.module_id}-events-dlq.fifo', + fifo=True, + fifo_throughput_limit=sqs.FifoThroughputLimit.PER_MESSAGE_GROUP_ID, + deduplication_scope=sqs.DeduplicationScope.MESSAGE_GROUP, + content_based_deduplication=True, + encryption_master_key=self.context.config().get_string('cluster.sqs.kms_key_id'), + is_dead_letter_queue=True + ) + ) + ) + self.add_common_tags(self.event_sqs_queue) + self.add_common_tags(self.event_sqs_queue.dead_letter_queue.queue) + + kms_key = self.context.config().get_string('cluster.sqs.kms_key_id', default=None) + encrypt_at_rest = Utils.is_not_empty(kms_key) + + self.controller_sqs_queue = SQSQueue( + self.context, 'virtual-desktop-controller-queue', self.stack, + queue_name=f'{self.cluster_name}-{self.module_id}-controller', + encrypt_at_rest=encrypt_at_rest, + encryption_master_key=kms_key, + dead_letter_queue=sqs.DeadLetterQueue( + max_receive_count=30, + queue=SQSQueue( + self.context, 'virtual-desktop-controller-queue-dlq', self.stack, + queue_name=f'{self.cluster_name}-{self.module_id}-controller-dlq', + encrypt_at_rest=encrypt_at_rest, + encryption_master_key=kms_key, + is_dead_letter_queue=True + ) + ) + ) + self.add_common_tags(self.controller_sqs_queue) + self.add_common_tags(self.controller_sqs_queue.dead_letter_queue.queue) + + def build_controller_ssm_commands_notification_infra(self): + self.ssm_command_pass_role = Role( + context=self.context, + name=f'{self.module_id}-ssm-commands-sns-topic-role', + scope=self.stack, + assumed_by=['ssm'], + description=f'IAM role for SSM Commands to send notifications via SNS' + ) + + self.ssm_command_pass_role.attach_inline_policy(Policy( + context=self.context, + name=f'{self.cluster_name}-{self.module_id}-ssm-commands-sns-topic-role-policy', + scope=self.stack, + policy_template_name='controller-ssm-command-pass-role.yml' + )) + self.ssm_command_pass_role.grant_pass_role(self.controller_role) + + self.ssm_commands_sns_topic = SNSTopic( + self.context, 'virtual-desktop-controller-sns-topic', self.stack, + topic_name=f'{self.cluster_name}-{self.module_id}-ssm-commands-sns-topic', + display_name=f'{self.cluster_name}-{self.module_id}-ssm-commands-topic', + master_key=self.context.config().get_string('cluster.sns.kms_key_id') + ) + self.add_common_tags(self.ssm_commands_sns_topic) + self.ssm_commands_sns_topic.add_subscription(sns_subscription.SqsSubscription( + queue=self.controller_sqs_queue, + dead_letter_queue=self.controller_sqs_queue.dead_letter_queue.queue + )) + + def subscribe_to_ec2_notification_events(self): + ec2_event_sns_topic = sns.Topic.from_topic_arn( + self.stack, f'{self.cluster_name}-{self.module_id}-ec2-state-change-topic', + self.context.config().get_string('cluster.ec2.state_change_notifications_sns_topic_arn', required=True) + ) + + ec2_event_sns_topic.add_subscription(sns_subscription.SqsSubscription( + queue=self.controller_sqs_queue, + dead_letter_queue=self.controller_sqs_queue.dead_letter_queue.queue, + filter_policy={ + IDEA_TAG_MODULE_ID.replace(':', '_'): sns.SubscriptionFilter.string_filter( + allowlist=[self.module_id] + ) + } + )) + + def build_scheduled_event_notification_infra(self): + lambda_name = f'{self.module_id}-scheduled-event-transformer' + self.scheduled_event_transformer_lambda_role = Role( + context=self.context, + name=f'{lambda_name}-role', + scope=self.stack, + assumed_by=['lambda'], + description=f'{lambda_name}-role' + ) + + self.scheduled_event_transformer_lambda_role.attach_inline_policy(Policy( + context=self.context, + name=f'{lambda_name}-policy', + scope=self.stack, + policy_template_name='controller-scheduled-event-transformer-lambda.yml' + )) + + scheduled_event_transformer_lambda = LambdaFunction( + context=self.context, + name=lambda_name, + description=f'{self.module_id} lambda to intercept all scheduled events and transform to the required event object.', + scope=self.stack, + environment={ + 'IDEA_CONTROLLER_EVENTS_QUEUE_URL': self.event_sqs_queue.queue_url + }, + timeout_seconds=180, + role=self.scheduled_event_transformer_lambda_role, + idea_code_asset=IdeaCodeAsset( + lambda_package_name='idea_controller_scheduled_event_transformer', + lambda_platform=SupportedLambdaPlatforms.PYTHON + ) + ) + + schedule_trigger_rule = events.Rule( + scope=self.stack, + id=f'{self.cluster_name}-{self.module_id}-schedule-rule', + enabled=True, + rule_name=f'{self.cluster_name}-{self.module_id}-schedule-rule', + description='Event Rule to Trigger schedule check EVERY 30 minutes on VDC Controller', + schedule=Schedule.cron( + minute='0/30' + ) + ) + + schedule_trigger_rule.add_target(events_targets.LambdaFunction( + scheduled_event_transformer_lambda + )) + self.add_common_tags(schedule_trigger_rule) + + def build_oauth2_client(self): + + # add resource server + resource_server = self.user_pool.add_resource_server( + id='resource-server', + identifier=self.module_id, + scopes=[ + cognito.ResourceServerScope(scope_name='read', scope_description='Allow Read Access'), + cognito.ResourceServerScope(scope_name='write', scope_description='Allow Write Access') + ] + ) + + # dcv session manager / external auth + # refer: https://docs.aws.amazon.com/dcv/latest/sm-admin/ext-auth.html + # todo - make an api call to check if "dcv-session-manager" resource server already exists in user pool + # this is to cover cases for blue/green deployments when an additional instance of eVDI module is deployed + session_manager_resource_server = self.user_pool.add_resource_server( + id='dcv-session-manager-resource-server', + identifier='dcv-session-manager', + scopes=[ + cognito.ResourceServerScope(scope_name='sm_scope', scope_description='sm_scope') + ] + ) + + # add new client to user pool + client = self.user_pool.add_client( + id=f'{self.module_id}-client', + access_token_validity=cdk.Duration.hours(1), + generate_secret=True, + id_token_validity=cdk.Duration.hours(1), + o_auth=cognito.OAuthSettings( + flows=cognito.OAuthFlows(client_credentials=True), + scopes=[ + cognito.OAuthScope.custom(f'{self.module_id}/read'), + cognito.OAuthScope.custom(f'{self.module_id}/write'), + cognito.OAuthScope.custom(f'{self.context.config().get_module_id(constants.MODULE_CLUSTER_MANAGER)}/read'), + cognito.OAuthScope.custom('dcv-session-manager/sm_scope') + ] + ), + refresh_token_validity=cdk.Duration.days(30), + user_pool_client_name=self.module_id + ) + client.node.add_dependency(session_manager_resource_server) + client.node.add_dependency(resource_server) + + # read secret value by invoking custom resource + oauth_credentials_lambda_arn = self.context.config().get_string('identity-provider.cognito.oauth_credentials_lambda_arn', required=True) + client_secret = cdk.CustomResource( + scope=self.stack, + id=f'{self.module_id}-creds', + service_token=oauth_credentials_lambda_arn, + properties={ + 'UserPoolId': self.user_pool.user_pool_id, + 'ClientId': client.user_pool_client_id + }, + resource_type='Custom::GetOAuthCredentials' + ) + + # save client id and client secret to AWS Secrets Manager + self.oauth2_client_secret = OAuthClientIdAndSecret( + context=self.context, + secret_name_prefix=self.module_id, + module_name=constants.MODULE_VIRTUAL_DESKTOP_CONTROLLER, + scope=self.stack, + client_id=client.user_pool_client_id, + client_secret=client_secret.get_att_string('ClientSecret') + ) + + def build_dcv_host_infra(self): + self.dcv_host_role = self._build_iam_role( + role_description=f'IAM role assigned to virtual-desktop-{self.COMPONENT_DCV_HOST}', + component_name=self.COMPONENT_DCV_HOST, + component_jinja=f'virtual-desktop-dcv-host.yml' + ) + self.dcv_host_role.grant_pass_role(self.controller_role) + + self.dcv_host_instance_profile = InstanceProfile( + context=self.context, + name=f'{self.module_id}-{self.COMPONENT_DCV_HOST}-instance-profile', + scope=self.stack, + roles=[self.dcv_host_role] + ) + + self.dcv_host_security_group = VirtualDesktopBastionAccessSecurityGroup( + context=self.context, + name=f'{self.module_id}-dcv-host-security-group', + scope=self.stack, + vpc=self.cluster.vpc, + bastion_host_security_group=self.cluster.get_security_group('bastion-host'), + description='Security Group for DCV Host', + directory_service_access=True, + component_name='DCV Host' + ) + + def _build_alb(self, component_name: str, internet_facing: bool, security_group: SecurityGroup) -> elbv2.ApplicationLoadBalancer: + return elbv2.ApplicationLoadBalancer( + self.stack, + f'{self.cluster_name}-{self.module_id}-{component_name}-alb', + load_balancer_name=f'{self.cluster_name}-{self.module_id}-alb', + security_group=security_group, + vpc=self.cluster.vpc, + internet_facing=internet_facing + ) + + def build_dcv_broker(self): + # client target group and registration + client_target_group = elbv2.ApplicationTargetGroup( + self.stack, + f'{self.COMPONENT_DCV_BROKER}-client-target-group', + port=self.BROKER_CLIENT_COMMUNICATION_PORT, + target_type=elbv2.TargetType.INSTANCE, + protocol=elbv2.ApplicationProtocol.HTTPS, + vpc=self.cluster.vpc, + target_group_name=self.get_target_group_name(f'{self.COMPONENT_DCV_BROKER}-c') + ) + client_target_group.configure_health_check( + enabled=True, + path='/health' + ) + + cdk.CustomResource( + self.stack, + 'dcv-broker-client-endpoint', + service_token=self.CLUSTER_ENDPOINTS_LAMBDA_ARN, + properties={ + 'endpoint_name': 'broker-client-endpoint', + 'listener_arn': self.context.config().get_string('cluster.load_balancers.internal_alb.dcv_broker_client_listener_arn', required=True), + 'priority': 0, + 'default_action': True, + 'actions': [ + { + 'Type': 'forward', + 'TargetGroupArn': client_target_group.target_group_arn + } + ] + }, + resource_type='Custom::DcvBrokerClientEndpointInternal' + ) + + # agent target group and registration + agent_target_group = elbv2.ApplicationTargetGroup( + self.stack, + f'{self.COMPONENT_DCV_BROKER}-agent-target-group', + port=self.BROKER_AGENT_COMMUNICATION_PORT, + target_type=elbv2.TargetType.INSTANCE, + protocol=elbv2.ApplicationProtocol.HTTPS, + vpc=self.cluster.vpc, + target_group_name=self.get_target_group_name(f'{self.COMPONENT_DCV_BROKER}-a') + ) + agent_target_group.configure_health_check( + enabled=True, + path='/health' + ) + + cdk.CustomResource( + self.stack, + 'dcv-broker-agent-endpoint', + service_token=self.CLUSTER_ENDPOINTS_LAMBDA_ARN, + properties={ + 'endpoint_name': 'broker-client-endpoint', + 'listener_arn': self.context.config().get_string('cluster.load_balancers.internal_alb.dcv_broker_agent_listener_arn', required=True), + 'priority': 0, + 'default_action': True, + 'actions': [ + { + 'Type': 'forward', + 'TargetGroupArn': agent_target_group.target_group_arn + } + ] + }, + resource_type='Custom::DcvBrokerAgentEndpointInternal' + ) + + # gateway target group and registration + gateway_target_group = elbv2.ApplicationTargetGroup( + self.stack, + f'{self.COMPONENT_DCV_BROKER}-gateway-target-group', + port=self.BROKER_GATEWAY_COMMUNICATION_PORT, + target_type=elbv2.TargetType.INSTANCE, + protocol=elbv2.ApplicationProtocol.HTTPS, + vpc=self.cluster.vpc, + target_group_name=self.get_target_group_name(f'{self.COMPONENT_DCV_BROKER}-g') + ) + gateway_target_group.configure_health_check( + enabled=True, + path='/health' + ) + + cdk.CustomResource( + self.stack, + 'dcv-broker-gateway-endpoint', + service_token=self.CLUSTER_ENDPOINTS_LAMBDA_ARN, + properties={ + 'endpoint_name': 'broker-gateway-endpoint', + 'listener_arn': self.context.config().get_string('cluster.load_balancers.internal_alb.dcv_broker_gateway_listener_arn', required=True), + 'priority': 0, + 'default_action': True, + 'actions': [ + { + 'Type': 'forward', + 'TargetGroupArn': gateway_target_group.target_group_arn + } + ] + }, + resource_type='Custom::DcvBrokerGatewayEndpointInternal' + ) + + # security group + self.dcv_broker_security_group = VirtualDesktopBastionAccessSecurityGroup( + context=self.context, + name=f'{self.module_id}-{self.COMPONENT_DCV_BROKER}-security-group', + scope=self.stack, + vpc=self.cluster.vpc, + bastion_host_security_group=self.cluster.get_security_group('bastion-host'), + description='Security Group for Virtual Desktop DCV Broker', + directory_service_access=False, + component_name='DCV Broker' + ) + + # autoscaling group + dcv_broker_package_uri = self.stack.node.try_get_context('dcv_broker_bootstrap_package_uri') + if Utils.is_empty(dcv_broker_package_uri): + dcv_broker_package_uri = 'not-provided' + + broker_userdata = BootstrapUserDataBuilder( + aws_region=self.aws_region, + bootstrap_package_uri=dcv_broker_package_uri, + install_commands=[ + f'/bin/bash dcv-broker/setup.sh' + ], + infra_config={ + 'BROKER_CLIENT_TARGET_GROUP_ARN': '${__BROKER_CLIENT_TARGET_GROUP_ARN__}', + 'CONTROLLER_EVENTS_QUEUE_URL': '${__CONTROLLER_EVENTS_QUEUE_URL__}' + }, + base_os=self.context.config().get_string('virtual-desktop-controller.dcv_broker.autoscaling.base_os', required=True) + ).build() + substituted_userdata = cdk.Fn.sub(broker_userdata, { + '__BROKER_CLIENT_TARGET_GROUP_ARN__': client_target_group.target_group_arn, + '__CONTROLLER_EVENTS_QUEUE_URL__': self.event_sqs_queue.queue_url + }) + + self.dcv_broker_role = self._build_iam_role( + role_description=f'IAM role assigned to virtual-desktop-{self.COMPONENT_DCV_BROKER}', + component_name=self.COMPONENT_DCV_BROKER, + component_jinja=f'virtual-desktop-dcv-broker.yml' + ) + + self.dcv_broker_autoscaling_group = self._build_auto_scaling_group( + component_name=self.COMPONENT_DCV_BROKER, + security_group=self.dcv_broker_security_group, + iam_role=self.dcv_broker_role, + substituted_userdata=substituted_userdata, + node_type=constants.NODE_TYPE_INFRA + ) + self.dcv_broker_autoscaling_group.node.add_dependency(self.event_sqs_queue) + + # receiving error jsii.errors.JSIIError: Cannot add AutoScalingGroup to 2nd Target Group + # if same ASG is added to both internal and external target groups. + # workaround below - reference to https://github.com/aws/aws-cdk/issues/5667#issuecomment-827549394 + self.dcv_broker_autoscaling_group.node.default_child.target_group_arns = [ + agent_target_group.target_group_arn, + client_target_group.target_group_arn, + gateway_target_group.target_group_arn + ] + + def _build_iam_role(self, role_description: str, component_name: str, component_jinja: str) -> Role: + + ec2_managed_policies = self.get_ec2_instance_managed_policies() + + role = Role( + context=self.context, + name=f'{self.module_id}-{component_name}-role', + scope=self.stack, + description=role_description, + assumed_by=['ssm', 'ec2'], + managed_policies=ec2_managed_policies + ) + variables = SocaAnyPayload() + variables.role_arn = role.role_arn + role.attach_inline_policy( + Policy( + context=self.context, + name=f'{self.cluster_name}-{self.module_id}-{component_name}-policy', + scope=self.stack, + policy_template_name=component_jinja, + vars=variables + ) + ) + return role + + def build_virtual_desktop_controller(self): + self.controller_security_group = VirtualDesktopPublicLoadBalancerAccessSecurityGroup( + context=self.context, + name=f'{self.module_id}-{self.COMPONENT_CONTROLLER}-security-group', + scope=self.stack, + vpc=self.cluster.vpc, + bastion_host_security_group=self.cluster.get_security_group('bastion-host'), + public_loadbalancer_security_group=self.cluster.get_security_group('external-load-balancer'), + description='Security Group for Virtual Desktop Controller', + directory_service_access=True, + component_name='Virtual Desktop Controller' + ) + + controller_bootstrap_package_uri = self.stack.node.try_get_context('controller_bootstrap_package_uri') + if Utils.is_empty(controller_bootstrap_package_uri): + controller_bootstrap_package_uri = 'not-provided' + + self.controller_role = self._build_iam_role( + role_description=f'IAM role assigned to virtual-desktop-{self.COMPONENT_CONTROLLER}', + component_name=self.COMPONENT_CONTROLLER, + component_jinja=f'virtual-desktop-controller.yml' + ) + + self.controller_auto_scaling_group = self._build_auto_scaling_group( + component_name=self.COMPONENT_CONTROLLER, + security_group=self.controller_security_group, + iam_role=self.controller_role, + substituted_userdata=cdk.Fn.sub(BootstrapUserDataBuilder( + aws_region=self.aws_region, + bootstrap_package_uri=controller_bootstrap_package_uri, + install_commands=[ + '/bin/bash virtual-desktop-controller/setup.sh' + ], + base_os=self.context.config().get_string('virtual-desktop-controller.controller.autoscaling.base_os', required=True) + ).build()), + node_type=constants.NODE_TYPE_APP + ) + + self.controller_auto_scaling_group.node.add_dependency(self.event_sqs_queue) + self.controller_auto_scaling_group.node.add_dependency(self.controller_sqs_queue) + + # external target group + external_target_group = elbv2.ApplicationTargetGroup( + self.stack, + 'controller-target-group-ext', + port=8443, + protocol=elbv2.ApplicationProtocol.HTTPS, + protocol_version=elbv2.ApplicationProtocolVersion.HTTP1, + target_type=elbv2.TargetType.INSTANCE, + vpc=self.cluster.vpc, + target_group_name=self.get_target_group_name('vdc-ext') + ) + external_target_group.configure_health_check( + enabled=True, + path='/healthcheck' + ) + + external_https_listener_arn = self.context.config().get_string('cluster.load_balancers.external_alb.https_listener_arn', required=True) + path_patterns = self.context.config().get_list('virtual-desktop-controller.controller.endpoints.external.path_patterns', required=True) + priority = self.context.config().get_int('virtual-desktop-controller.controller.endpoints.external.priority', required=True) + cdk.CustomResource( + self.stack, + 'controller-endpoint-ext', + service_token=self.CLUSTER_ENDPOINTS_LAMBDA_ARN, + properties={ + 'endpoint_name': f'{self.module_id}-controller-endpoint-ext', + 'listener_arn': external_https_listener_arn, + 'priority': priority, + 'conditions': [ + { + 'Field': 'path-pattern', + 'Values': path_patterns + } + ], + 'actions': [ + { + 'Type': 'forward', + 'TargetGroupArn': external_target_group.target_group_arn + } + ] + }, + resource_type='Custom::ControllerEndpointExternal' + ) + + # internal target group + internal_target_group = elbv2.ApplicationTargetGroup( + self.stack, + 'controller-target-group-int', + port=8443, + protocol=elbv2.ApplicationProtocol.HTTPS, + protocol_version=elbv2.ApplicationProtocolVersion.HTTP1, + target_type=elbv2.TargetType.INSTANCE, + vpc=self.cluster.vpc, + target_group_name=self.get_target_group_name('vdc-int') + ) + internal_target_group.configure_health_check( + enabled=True, + path='/healthcheck' + ) + + internal_https_listener_arn = self.context.config().get_string('cluster.load_balancers.internal_alb.https_listener_arn', required=True) + path_patterns = self.context.config().get_list('virtual-desktop-controller.controller.endpoints.internal.path_patterns', required=True) + priority = self.context.config().get_int('virtual-desktop-controller.controller.endpoints.internal.priority', required=True) + cdk.CustomResource( + self.stack, + 'controller-endpoint-int', + service_token=self.CLUSTER_ENDPOINTS_LAMBDA_ARN, + properties={ + 'endpoint_name': f'{self.module_id}-controller-endpoint-int', + 'listener_arn': internal_https_listener_arn, + 'priority': priority, + 'conditions': [ + { + 'Field': 'path-pattern', + 'Values': path_patterns + } + ], + 'actions': [ + { + 'Type': 'forward', + 'TargetGroupArn': internal_target_group.target_group_arn + } + ] + }, + resource_type='Custom::ControllerEndpointInternal' + ) + + # receiving error jsii.errors.JSIIError: Cannot add AutoScalingGroup to 2nd Target Group + # if same ASG is added to both internal and external target groups. + # workaround below - reference to https://github.com/aws/aws-cdk/issues/5667#issuecomment-827549394 + self.controller_auto_scaling_group.node.default_child.target_group_arns = [ + internal_target_group.target_group_arn, + external_target_group.target_group_arn + ] + + def _build_auto_scaling_group(self, component_name: str, security_group: SecurityGroup, iam_role: Role, substituted_userdata: str, node_type: str) -> asg.AutoScalingGroup: + is_public = self.context.config().get_bool(f'virtual-desktop-controller.{self.CONFIG_MAPPING[component_name]}.autoscaling.public', False) and len(self.cluster.public_subnets) > 0 + if is_public: + vpc_subnets = ec2.SubnetSelection( + subnets=self.cluster.public_subnets + ) + else: + vpc_subnets = ec2.SubnetSelection( + subnets=self.cluster.private_subnets + ) + + base_os = self.context.config().get_string(f'virtual-desktop-controller.{self.CONFIG_MAPPING[component_name]}.autoscaling.base_os', required=True) + block_device_name = Utils.get_ec2_block_device_name(base_os) + enable_detailed_monitoring = self.context.config().get_bool(f'virtual-desktop-controller.{self.CONFIG_MAPPING[component_name]}.autoscaling.enable_detailed_monitoring', default=False) + metadata_http_tokens = self.context.config().get_string(f'virtual-desktop-controller.{self.CONFIG_MAPPING[component_name]}.autoscaling.metadata_http_tokens', required=True) + + launch_template = ec2.LaunchTemplate( + self.stack, f'{component_name}-lt', + instance_type=ec2.InstanceType(self.context.config().get_string(f'virtual-desktop-controller.{self.CONFIG_MAPPING[component_name]}.autoscaling.instance_type', required=True)), + machine_image=ec2.MachineImage.generic_linux({ + self.aws_region: self.context.config().get_string(f'virtual-desktop-controller.{self.CONFIG_MAPPING[component_name]}.autoscaling.instance_ami', required=True) + }), + security_group=security_group, + user_data=ec2.UserData.custom(substituted_userdata), + key_name=self.context.config().get_string('cluster.network.ssh_key_pair', required=True), + block_devices=[ec2.BlockDevice( + device_name=block_device_name, + volume=ec2.BlockDeviceVolume(ebs_device=ec2.EbsDeviceProps( + volume_size=self.context.config().get_int(f'virtual-desktop-controller.{self.CONFIG_MAPPING[component_name]}.autoscaling.volume_size', default=200), + volume_type=ec2.EbsDeviceVolumeType.GP3 + )) + )], + role=iam_role, + require_imdsv2=True if metadata_http_tokens == "required" else False + ) + + auto_scaling_group = asg.AutoScalingGroup( + self.stack, f'{component_name}-asg', + vpc=self.cluster.vpc, + vpc_subnets=vpc_subnets, + auto_scaling_group_name=f'{self.cluster_name}-{self.module_id}-{component_name}-asg', + launch_template=launch_template, + instance_monitoring=asg.Monitoring.DETAILED if enable_detailed_monitoring else asg.Monitoring.BASIC, + group_metrics=[asg.GroupMetrics.all()], + min_capacity=self.context.config().get_int(f'virtual-desktop-controller.{self.CONFIG_MAPPING[component_name]}.autoscaling.min_capacity', default=1), + max_capacity=self.context.config().get_int(f'virtual-desktop-controller.{self.CONFIG_MAPPING[component_name]}.autoscaling.max_capacity', default=3), + new_instances_protected_from_scale_in=self.context.config().get_bool(f'virtual-desktop-controller.{self.CONFIG_MAPPING[component_name]}.autoscaling.new_instances_protected_from_scale_in', default=True), + cooldown=cdk.Duration.minutes(self.context.config().get_int(f'virtual-desktop-controller.{self.CONFIG_MAPPING[component_name]}.autoscaling.cooldown_minutes', default=5)), + health_check=asg.HealthCheck.elb( + grace=cdk.Duration.minutes(self.context.config().get_int(f'virtual-desktop-controller.{self.CONFIG_MAPPING[component_name]}.autoscaling.elb_healthcheck.grace_time_minutes', default=15)) + ), + update_policy=asg.UpdatePolicy.rolling_update( + max_batch_size=self.context.config().get_int(f'virtual-desktop-controller.{self.CONFIG_MAPPING[component_name]}.autoscaling.rolling_update_policy.max_batch_size', default=1), + min_instances_in_service=self.context.config().get_int(f'virtual-desktop-controller.{self.CONFIG_MAPPING[component_name]}.autoscaling.rolling_update_policy.min_instances_in_service', default=1), + pause_time=cdk.Duration.minutes(self.context.config().get_int(f'virtual-desktop-controller.{self.CONFIG_MAPPING[component_name]}.autoscaling.rolling_update_policy.pause_time_minutes', default=15)) + ), + termination_policies=[ + asg.TerminationPolicy.DEFAULT + ] + ) + + auto_scaling_group.scale_on_cpu_utilization( + 'cpu-utilization-scaling-policy', + target_utilization_percent=self.context.config().get_int(f'virtual-desktop-controller.{self.CONFIG_MAPPING[component_name]}.autoscaling.cpu_utilization_scaling_policy.target_utilization_percent', default=80), + estimated_instance_warmup=cdk.Duration.minutes(self.context.config().get_int(f'virtual-desktop-controller.{self.CONFIG_MAPPING[component_name]}.autoscaling.cpu_utilization_scaling_policy.estimated_instance_warmup_minutes', default=15)) + ) + + cdk.Tags.of(auto_scaling_group).add(constants.IDEA_TAG_NODE_TYPE, node_type) + cdk.Tags.of(auto_scaling_group).add(constants.IDEA_TAG_NAME, f'{self.cluster_name}-{self.module_id}-{component_name}') + + if not enable_detailed_monitoring: + self.add_nag_suppression( + construct=auto_scaling_group, + suppressions=[IdeaNagSuppression(rule_id='AwsSolutions-EC28', reason='detailed monitoring is a configurable option to save costs')], + apply_to_children=True + ) + + self.add_nag_suppression( + construct=auto_scaling_group, + suppressions=[ + IdeaNagSuppression(rule_id='AwsSolutions-AS3', reason='ASG notifications scaling notifications can be managed via AWS Console') + ] + ) + return auto_scaling_group + + def _build_dcv_connection_gateway_instance_infrastructure(self): + self.dcv_connection_gateway_security_group = VirtualDesktopPublicLoadBalancerAccessSecurityGroup( + context=self.context, + name=f'{self.module_id}-{self.COMPONENT_DCV_CONNECTION_GATEWAY}-security-group', + scope=self.stack, + vpc=self.cluster.vpc, + bastion_host_security_group=self.cluster.get_security_group('bastion-host'), + public_loadbalancer_security_group=self.cluster.get_security_group('external-load-balancer'), + description='Security Group for Virtual Desktop DCV Connection Gateway', + directory_service_access=False, + component_name='DCV Connection Gateway' + ) + + dcv_connection_gateway_bootstrap_package_uri = self.stack.node.try_get_context('dcv_connection_gateway_package_uri') + if Utils.is_empty(dcv_connection_gateway_bootstrap_package_uri): + dcv_connection_gateway_bootstrap_package_uri = 'not-provided' + + connection_gateway_userdata = BootstrapUserDataBuilder( + aws_region=self.aws_region, + bootstrap_package_uri=dcv_connection_gateway_bootstrap_package_uri, + install_commands=[ + f'/bin/bash dcv-connection-gateway/setup.sh' + ], + infra_config={ + 'CERTIFICATE_SECRET_ARN': '${__CERTIFICATE_SECRET_ARN__}', + 'PRIVATE_KEY_SECRET_ARN': '${__PRIVATE_KEY_SECRET_ARN__}', + }, + base_os=self.context.config().get_string('virtual-desktop-controller.dcv_connection_gateway.autoscaling.base_os', required=True) + ).build() + + external_certificate_provided = self.context.config().get_bool('virtual-desktop-controller.dcv_connection_gateway.certificate.provided', required=True) + if not external_certificate_provided: + substituted_userdata = cdk.Fn.sub(connection_gateway_userdata, { + '__CERTIFICATE_SECRET_ARN__': self.dcv_connection_gateway_self_signed_cert.get_att_string('certificate_secret_arn'), + '__PRIVATE_KEY_SECRET_ARN__': self.dcv_connection_gateway_self_signed_cert.get_att_string('private_key_secret_arn') + }) + else: + substituted_userdata = cdk.Fn.sub(connection_gateway_userdata, { + '__CERTIFICATE_SECRET_ARN__': self.context.config().get_string('virtual-desktop-controller.dcv_connection_gateway.certificate.certificate_secret_arn', required=True), + '__PRIVATE_KEY_SECRET_ARN__': self.context.config().get_string('virtual-desktop-controller.dcv_connection_gateway.certificate.private_key_secret_arn', required=True) + }) + + self.dcv_connection_gateway_autoscaling_group = self._build_auto_scaling_group( + component_name=self.COMPONENT_DCV_CONNECTION_GATEWAY, + security_group=self.dcv_connection_gateway_security_group, + iam_role=self._build_iam_role( + role_description=f'IAM role assigned to virtual-desktop-{self.COMPONENT_DCV_CONNECTION_GATEWAY}', + component_name=self.COMPONENT_DCV_CONNECTION_GATEWAY, + component_jinja=f'virtual-desktop-dcv-connection-gateway.yml' + ), + substituted_userdata=substituted_userdata, + node_type=constants.NODE_TYPE_INFRA + ) + + def _build_dcv_connection_gateway_network_infrastructure(self): + + is_public = self.context.config().get_bool('cluster.load_balancers.external_alb.public', default=True) + external_nlb_subnets = self.cluster.public_subnets if is_public is True else self.cluster.private_subnets + self.external_nlb = elbv2.NetworkLoadBalancer( + self.stack, + f'{self.cluster_name}-{self.module_id}-external-nlb', + load_balancer_name=f'{self.cluster_name}-{self.module_id}-external-nlb', + vpc=self.cluster.vpc, + internet_facing=is_public, + vpc_subnets=ec2.SubnetSelection(subnets=external_nlb_subnets) + ) + + # enable access logs + external_alb_enable_access_log = self.context.config().get_bool('virtual-desktop-controller.external_nlb.access_logs', default=False) + if external_alb_enable_access_log: + access_log_destination = s3.Bucket.from_bucket_name(scope=self.stack, id='cluster-s3-bucket', bucket_name=self.context.config().get_string('cluster.cluster_s3_bucket', required=True)) + self.external_nlb.log_access_logs(access_log_destination, f'logs/{self.module_id}/external-nlb-access-logs') + + quic_supported = self.context.config().get_bool('virtual-desktop-controller.dcv_session.quic_support', required=True) + protocol = elbv2.Protocol.TCP + tg_suffix = 'TN' # TCP Network + if quic_supported: + protocol = elbv2.Protocol.TCP_UDP + tg_suffix = 'TUN' # TCP - UDP Network + + dcv_connection_gateway_target_group = elbv2.NetworkTargetGroup( + self.stack, + 'dcv-connection-gateway-target-group-nlb', + port=8443, + protocol=protocol, + target_type=elbv2.TargetType.INSTANCE, + vpc=self.cluster.vpc, + targets=[self.dcv_connection_gateway_autoscaling_group], + target_group_name=self.get_target_group_name(f'{self.COMPONENT_DCV_CONNECTION_GATEWAY}-{tg_suffix}'), + health_check=elbv2.HealthCheck( + port='8989', + protocol=elbv2.Protocol.TCP + ), + connection_termination=True + ) + # Can not provide stickiness attributes directly in CDK Construct. Refer - https://github.com/aws/aws-cdk/issues/17491 + # Documentation on attributes. Refer - https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-elasticloadbalancingv2-targetgroup-targetgroupattribute.html + dcv_connection_gateway_target_group.set_attribute('stickiness.enabled', 'true') + dcv_connection_gateway_target_group.set_attribute('stickiness.type', 'source_ip') + + elbv2.NetworkListener( + self.external_nlb, + 'dcv-connection-gateway-nlb-listener', + load_balancer=self.external_nlb, + protocol=protocol, + port=443, + default_action=elbv2.NetworkListenerAction.forward( + target_groups=[ + dcv_connection_gateway_target_group + ] + ) + ) + + self.dcv_connection_gateway_security_group.add_ingress_rule( + ec2.Peer.ipv4(self.cluster.vpc.vpc_cidr_block), + ec2.Port.tcp(8989), + description='Allow TCP traffic access for HealthCheck to DCV Connection Gateway' + ) + + cluster_prefix_list_id = self.context.config().get_string('cluster.network.cluster_prefix_list_id', required=True) + self.dcv_connection_gateway_security_group.add_ingress_rule( + ec2.Peer.prefix_list(cluster_prefix_list_id), + ec2.Port.all_traffic(), + description='Allow all Traffic access from Cluster Prefix List to DCV Connection Gateway' + ) + + prefix_list_ids = self.context.config().get_list('cluster.network.prefix_list_ids', default=[]) + for prefix_list_id in prefix_list_ids: + self.dcv_connection_gateway_security_group.add_ingress_rule( + ec2.Peer.prefix_list(prefix_list_id), + ec2.Port.all_traffic(), + description='Allow all traffic access from Prefix List to DCV Connection Gateway' + ) + + def _build_self_signed_cert_for_dcv_connection_gateway(self): + self_signed_certificate_lambda_arn = self.context.config().get_string('cluster.self_signed_certificate_lambda_arn', required=True) + self.dcv_connection_gateway_self_signed_cert = cdk.CustomResource( + self.stack, + f'{self.cluster_name}-{self.module_id}-external-cert-{self.COMPONENT_DCV_CONNECTION_GATEWAY}', + service_token=self_signed_certificate_lambda_arn, + properties={ + 'domain_name': f'{self.module_id}.{self.cluster_name}.idea.default', + 'certificate_name': f'{self.cluster_name}-{self.module_id}-{self.COMPONENT_DCV_CONNECTION_GATEWAY}-certificate', + 'create_acm_certificate': False, + 'kms_key_id': self.context.config().get_string('cluster.secretsmanager.kms_key_id'), + 'tags': { + 'Name': f'{self.cluster_name}-{self.module_id}-{self.COMPONENT_DCV_CONNECTION_GATEWAY} Self Signed Certificate', + 'idea:ClusterName': self.cluster_name, + 'idea:ModuleName': 'virtual-desktop-controller' + } + }, + resource_type=f'Custom::SelfSignedCertificateConnectionGateway' + ) + + def build_dcv_connection_gateway(self): + external_certificate_provided = self.context.config().get_bool('virtual-desktop-controller.dcv_connection_gateway.certificate.provided', required=True) + if not external_certificate_provided: + self._build_self_signed_cert_for_dcv_connection_gateway() + + self._build_dcv_connection_gateway_instance_infrastructure() + self._build_dcv_connection_gateway_network_infrastructure() + + def build_backups(self): + cluster_backups_enabled = self.context.config().get_string('cluster.backups.enabled', default=False) + if not cluster_backups_enabled: + return + + vdi_host_backup_enabled = self.context.config().get_bool('virtual-desktop-controller.vdi_host_backup.enabled', default=False) + if not vdi_host_backup_enabled: + return + + backup_vault_arn = self.context.config().get_string('cluster.backups.backup_vault.arn', required=True) + backup_role_arn = self.context.config().get_string('cluster.backups.role_arn', required=True) + + backup_role = iam.Role.from_role_arn(self.stack, 'backup-role', backup_role_arn) + backup_vault = backup.BackupVault.from_backup_vault_arn(self.stack, 'cluster-backup-vault', backup_vault_arn) + + backup_plan_config = self.context.config().get_config('virtual-desktop-controller.vdi_host_backup.backup_plan') + + self.backup_plan = BackupPlan( + self.context, 'vdi-host-backup-plan', self.stack, + backup_plan_name=f'{self.cluster_name}-{self.module_id}', + backup_plan_config=backup_plan_config, + backup_vault=backup_vault, + backup_role=backup_role + ) + + def build_cluster_settings(self): + cluster_settings = { + 'deployment_id': self.deployment_id, + 'client_id': self.oauth2_client_secret.client_id.ref, + 'client_secret': self.oauth2_client_secret.client_secret.ref, + 'dcv_host_security_group_id': self.dcv_host_security_group.security_group_id, + 'dcv_host_role_arn': self.dcv_host_role.role_arn, + 'dcv_host_role_name': self.dcv_host_role.role_name, + 'dcv_host_role_id': self.dcv_host_role.role_id, + 'dcv_broker_role_arn': self.dcv_broker_role.role_arn, + 'dcv_broker_role_name': self.dcv_broker_role.role_name, + 'dcv_broker_role_id': self.dcv_broker_role.role_id, + 'scheduled_event_transformer_lambda_role_arn': self.scheduled_event_transformer_lambda_role.role_arn, + 'scheduled_event_transformer_lambda_role_name': self.scheduled_event_transformer_lambda_role.role_name, + 'scheduled_event_transformer_lambda_role_id': self.scheduled_event_transformer_lambda_role.role_id, + 'dcv_host_instance_profile_arn': self.build_instance_profile_arn(self.dcv_host_instance_profile.ref), + 'ssm_commands_sns_topic_arn': self.ssm_commands_sns_topic.topic_arn, + 'ssm_commands_sns_topic_name': self.ssm_commands_sns_topic.topic_name, + 'ssm_commands_pass_role_arn': self.ssm_command_pass_role.role_arn, + 'ssm_commands_pass_role_id': self.ssm_command_pass_role.role_id, + 'ssm_commands_pass_role_name': self.ssm_command_pass_role.role_name, + 'controller_iam_role_arn': self.controller_role.role_arn, + 'controller_iam_role_name': self.controller_role.role_name, + 'controller_iam_role_id': self.controller_role.role_id, + 'events_sqs_queue_url': self.event_sqs_queue.queue_url, + 'events_sqs_queue_arn': self.event_sqs_queue.queue_arn, + 'controller_sqs_queue_url': self.controller_sqs_queue.queue_url, + 'controller_sqs_queue_arn': self.controller_sqs_queue.queue_arn, + 'external_nlb.load_balancer_dns_name': self.external_nlb.load_balancer_dns_name, + 'controller.asg_name': self.controller_auto_scaling_group.auto_scaling_group_name, + 'controller.asg_arn': self.controller_auto_scaling_group.auto_scaling_group_arn, + 'dcv_broker.asg_name': self.dcv_broker_autoscaling_group.auto_scaling_group_name, + 'dcv_broker.asg_arn': self.dcv_broker_autoscaling_group.auto_scaling_group_arn, + 'dcv_connection_gateway.asg_name': self.dcv_connection_gateway_autoscaling_group.auto_scaling_group_name, + 'dcv_connection_gateway.asg_arn': self.dcv_connection_gateway_autoscaling_group.auto_scaling_group_arn + } + + if not self.context.config().get_bool('virtual-desktop-controller.dcv_connection_gateway.certificate.provided', default=False): + cluster_settings['dcv_connection_gateway.certificate.certificate_secret_arn'] = self.dcv_connection_gateway_self_signed_cert.get_att_string('certificate_secret_arn') + cluster_settings['dcv_connection_gateway.certificate.private_key_secret_arn'] = self.dcv_connection_gateway_self_signed_cert.get_att_string('private_key_secret_arn') + + if self.backup_plan is not None: + cluster_settings['vdi_host_backup.backup_plan.arn'] = self.backup_plan.get_backup_plan_arn() + + self.update_cluster_settings(cluster_settings) diff --git a/source/idea/idea-administrator/src/ideaadministrator/app/cluster_prefix_list_helper.py b/source/idea/idea-administrator/src/ideaadministrator/app/cluster_prefix_list_helper.py new file mode 100644 index 00000000..ac02152d --- /dev/null +++ b/source/idea/idea-administrator/src/ideaadministrator/app/cluster_prefix_list_helper.py @@ -0,0 +1,103 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +from ideasdk.context import SocaCliContext, SocaContextOptions +from ideasdk.utils import Utils + +from typing import List, Dict + + +class ClusterPrefixListHelper: + + def __init__(self, cluster_name: str, aws_region: str, aws_profile: str = None): + self.cluster_name = cluster_name + self.aws_region = aws_region + self.aws_profile = aws_profile + + self.context = SocaCliContext(options=SocaContextOptions( + cluster_name=cluster_name, + aws_region=aws_region, + aws_profile=aws_profile, + enable_aws_client_provider=True + )) + + self.cluster_prefix_list_id = self.context.config().get_string('cluster.network.cluster_prefix_list_id', required=True) + + def _get_current_version(self) -> int: + describe_result = self.context.aws().ec2().describe_managed_prefix_lists( + PrefixListIds=[self.cluster_prefix_list_id] + ) + prefix_list_info = describe_result['PrefixLists'][0] + version = prefix_list_info['Version'] + return int(version) + + def list_entries(self) -> List[Dict]: + result = [] + prefix_list_paginator = self.context.aws().ec2().get_paginator('get_managed_prefix_list_entries') + prefix_list_iterator = prefix_list_paginator.paginate(PrefixListId=self.cluster_prefix_list_id) + for prefix_list in prefix_list_iterator: + cidr_entries = prefix_list.get('Entries', []) + for cidr_entry in cidr_entries: + result.append({ + 'cidr': cidr_entry['Cidr'], + 'description': Utils.get_value_as_string('Description', cidr_entry) + }) + return result + + def add_entry(self, cidr: str, description: str): + if Utils.is_empty(cidr): + self.context.error('cidr is required') + raise SystemExit(1) + + result = self.list_entries() + for entry in result: + if entry['cidr'] == cidr: + self.context.warning(f'CIDR: {cidr} already exists in cluster prefix list: {self.cluster_prefix_list_id}') + raise SystemExit(1) + + cidr_entry = { + 'Cidr': cidr + } + if Utils.is_not_empty(description): + cidr_entry['Description'] = description + + self.context.aws().ec2().modify_managed_prefix_list( + AddEntries=[cidr_entry], + PrefixListId=self.cluster_prefix_list_id, + CurrentVersion=self._get_current_version() + ) + self.context.success(f'CIDR: {cidr} added to cluster prefix list: {self.cluster_prefix_list_id}.') + + def remove_entry(self, cidr: str): + if Utils.is_empty(cidr): + self.context.error('cidr is required') + raise SystemExit(1) + + result = self.list_entries() + found = False + for entry in result: + if entry['cidr'] == cidr: + found = True + break + if not found: + self.context.warning(f'CIDR: {cidr} not found in cluster prefix list: {self.cluster_prefix_list_id}') + raise SystemExit(1) + + cidr_entry = { + 'Cidr': cidr + } + + self.context.aws().ec2().modify_managed_prefix_list( + RemoveEntries=[cidr_entry], + PrefixListId=self.cluster_prefix_list_id, + CurrentVersion=self._get_current_version() + ) + self.context.success(f'CIDR: {cidr} was removed from cluster prefix list: {self.cluster_prefix_list_id}.') diff --git a/source/idea/idea-administrator/src/ideaadministrator/app/config_generator.py b/source/idea/idea-administrator/src/ideaadministrator/app/config_generator.py new file mode 100644 index 00000000..d70800b2 --- /dev/null +++ b/source/idea/idea-administrator/src/ideaadministrator/app/config_generator.py @@ -0,0 +1,575 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +from ideasdk.utils import Utils, Jinja2Utils +from ideadatamodel import exceptions, constants + +from ideaadministrator.app_props import AdministratorProps +from ideaadministrator.app.vpc_endpoints_helper import VpcEndpointsHelper + +import os +from typing import Dict, List, Optional +from pathlib import Path + + +class ConfigGenerator: + + def __init__(self, values: Dict): + self.props = AdministratorProps() + + self.user_values = values + + # Begin: User Values Getters and Validations + + def get_cluster_name(self) -> str: + cluster_name = Utils.get_value_as_string('cluster_name', self.user_values) + if Utils.is_empty(cluster_name): + raise exceptions.invalid_params('cluster_name is required') + return cluster_name + + def get_cluster_timezone(self) -> Optional[str]: + aws_region = self.get_aws_region() + region_timezone_file = self.props.region_timezone_config_file() + with open(region_timezone_file, 'r') as f: + region_timezone_config = Utils.from_yaml(f.read()) + + default_timezone_id = Utils.get_value_as_string('default', region_timezone_config, 'America/Los_Angeles') + region_timezone_id = Utils.get_value_as_string(aws_region, region_timezone_config, default_timezone_id) + return region_timezone_id + + def get_cluster_locale(self) -> Optional[str]: + return Utils.get_value_as_string('cluster_locale', self.user_values, 'en_US') + + def get_administrator_email(self) -> str: + value = Utils.get_value_as_string('administrator_email', self.user_values, None) + if Utils.is_empty(value): + raise exceptions.invalid_params('administrator_email is required') + return value + + def get_administrator_username(self) -> str: + return Utils.get_value_as_string('administrator_username', self.user_values, 'clusteradmin') + + def get_aws_account_id(self) -> str: + value = Utils.get_value_as_string('aws_account_id', self.user_values, None) + if Utils.is_empty(value): + raise exceptions.invalid_params('aws_account_id is required') + return value + + def get_aws_dns_suffix(self) -> str: + value = Utils.get_value_as_string('aws_dns_suffix', self.user_values, None) + if Utils.is_empty(value): + raise exceptions.invalid_params('aws_dns_suffix is required') + return value + + def get_aws_partition(self) -> str: + value = Utils.get_value_as_string('aws_partition', self.user_values, None) + if Utils.is_empty(value): + raise exceptions.invalid_params('aws_partition is required') + return value + + def get_aws_region(self) -> str: + value = Utils.get_value_as_string('aws_region', self.user_values) + if Utils.is_empty(value): + raise exceptions.invalid_params('aws_region is required') + return value + + def get_aws_profile(self) -> Optional[str]: + return Utils.get_value_as_string('aws_profile', self.user_values, None) + + def get_storage_apps_provider(self) -> str: + return Utils.get_value_as_string('storage_apps_provider', self.user_values, 'efs') + + def get_apps_mount_dir(self) -> str: + return Utils.get_value_as_string('apps_mount_dir', self.user_values, '/apps') + + def get_storage_data_provider(self) -> str: + return Utils.get_value_as_string('storage_data_provider', self.user_values, 'efs') + + def get_data_mount_dir(self) -> str: + return Utils.get_value_as_string('data_mount_dir', self.user_values, '/data') + + def get_prefix_list_ids(self) -> List[str]: + return Utils.get_value_as_list('prefix_list_ids', self.user_values, []) + + def get_client_ip(self) -> List[str]: + return Utils.get_value_as_list('client_ip', self.user_values, []) + + def get_ssh_key_pair_name(self) -> str: + value = Utils.get_value_as_string('ssh_key_pair_name', self.user_values) + if Utils.is_empty(value): + raise exceptions.invalid_params('ssh_key_pair_name is required') + return value + + def get_vpc_cidr_block(self) -> Optional[str]: + if self.get_use_existing_vpc(): + return None + value = Utils.get_value_as_string('vpc_cidr_block', self.user_values) + if Utils.is_empty(value): + raise exceptions.invalid_params('vpc_cidr_block is required') + return value + + def get_use_vpc_endpoints(self) -> bool: + return Utils.get_value_as_bool('use_vpc_endpoints', self.user_values, False) + + def get_alb_public(self) -> bool: + return Utils.get_value_as_bool('alb_public', self.user_values, True) + + def get_directory_service_provider(self) -> str: + return Utils.get_value_as_string('directory_service_provider', self.user_values, 'openldap') + + def get_use_existing_directory_service(self) -> Optional[bool]: + value = Utils.get_value_as_bool('use_existing_directory_service', self.user_values, False) + if value and not self.get_use_existing_vpc(): + raise exceptions.invalid_params('use_existing_directory_service cannot be True if use_existing_vpc = False') + return value + + def get_directory_id(self) -> Optional[str]: + value = Utils.get_value_as_string('directory_id', self.user_values) + if Utils.is_empty(value) and self.get_use_existing_directory_service(): + raise exceptions.invalid_params('directory_id is required when use_existing_directory_service = True') + return value + + def get_directory_service_root_username_secret_arn(self) -> Optional[str]: + value = Utils.get_value_as_string('directory_service_root_username_secret_arn', self.user_values) + if Utils.is_empty(value) and self.get_use_existing_directory_service(): + raise exceptions.invalid_params('directory_service_root_password_secret_arn is required when use_existing_directory_service = True') + return value + + def get_directory_service_root_password_secret_arn(self) -> Optional[str]: + value = Utils.get_value_as_string('directory_service_root_password_secret_arn', self.user_values) + if Utils.is_empty(value) and self.get_use_existing_directory_service(): + raise exceptions.invalid_params('directory_service_root_password_secret_arn is required when use_existing_directory_service = True') + return value + + def get_identity_provider(self) -> str: + return Utils.get_value_as_string('identity_provider', self.user_values, 'cognito-idp') + + def get_enable_aws_backup(self) -> bool: + return Utils.get_value_as_bool('enable_aws_backup', self.user_values, True) + + def get_kms_key_type(self) -> str: + return Utils.get_value_as_string('kms_key_type', self.user_values, None) + + def get_kms_key_id(self) -> str: + return Utils.get_value_as_string('kms_key_id', self.user_values, None) + + def get_instance_type(self) -> str: + return Utils.get_value_as_string('instance_type', self.user_values, 'm5.large') + + def get_base_os(self) -> str: + return Utils.get_value_as_string('base_os', self.user_values, 'amazonlinux2') + + def get_dcv_connection_gateway_instance_type(self) -> str: + return Utils.get_value_as_string('dcv_connection_gateway_instance_type', self.user_values, 'm5.xlarge') + + def get_dcv_connection_gateway_volume_size(self) -> int: + return Utils.get_value_as_int('dcv_connection_gateway_volume_size', self.user_values, 200) + + def get_dcv_connection_gateway_instance_ami(self) -> str: + dcv_connection_gateway_instance_ami = Utils.get_value_as_string('dcv_connection_gateway_instance_ami', self.user_values) + if Utils.is_not_empty(dcv_connection_gateway_instance_ami): + return dcv_connection_gateway_instance_ami + + aws_region = self.get_aws_region() + regions_config_file = self.props.region_ami_config_file() + with open(regions_config_file, 'r') as f: + regions_config = Utils.from_yaml(f.read()) + ami_config = Utils.get_value_as_dict(aws_region, regions_config) + if ami_config is None: + raise exceptions.general_exception(f'aws_region: {aws_region} not found in region_ami_config.yml') + base_os = self.get_base_os() + ami_id = Utils.get_value_as_string(base_os, ami_config) + if Utils.is_empty(ami_id): + raise exceptions.general_exception(f'instance_ami not found for base_os: {base_os}, region: {aws_region}') + return ami_id + + def get_dcv_broker_instance_type(self) -> str: + return Utils.get_value_as_string('dcv_broker_instance_type', self.user_values, 'm5.xlarge') + + def get_dcv_broker_volume_size(self) -> int: + return Utils.get_value_as_int('dcv_broker_volume_size', self.user_values, 200) + + def get_dcv_broker_instance_ami(self) -> str: + dcv_broker_instance_ami = Utils.get_value_as_string('dcv_broker_instance_ami', self.user_values) + if Utils.is_not_empty(dcv_broker_instance_ami): + return dcv_broker_instance_ami + aws_region = self.get_aws_region() + regions_config_file = self.props.region_ami_config_file() + with open(regions_config_file, 'r') as f: + regions_config = Utils.from_yaml(f.read()) + ami_config = Utils.get_value_as_dict(aws_region, regions_config) + if ami_config is None: + raise exceptions.general_exception(f'aws_region: {aws_region} not found in region_ami_config.yml') + base_os = self.get_base_os() + ami_id = Utils.get_value_as_string(base_os, ami_config) + if Utils.is_empty(ami_id): + raise exceptions.general_exception(f'instance_ami not found for base_os: {base_os}, region: {aws_region}') + return ami_id + + def get_instance_ami(self) -> str: + instance_ami = Utils.get_value_as_string('instance_ami', self.user_values) + if Utils.is_not_empty(instance_ami): + return instance_ami + aws_region = self.get_aws_region() + regions_config_file = self.props.region_ami_config_file() + with open(regions_config_file, 'r') as f: + regions_config = Utils.from_yaml(f.read()) + ami_config = Utils.get_value_as_dict(aws_region, regions_config) + if ami_config is None: + raise exceptions.general_exception(f'aws_region: {aws_region} not found in region_ami_config.yml') + base_os = self.get_base_os() + ami_id = Utils.get_value_as_string(base_os, ami_config) + if Utils.is_empty(ami_id): + raise exceptions.general_exception(f'instance_ami not found for base_os: {base_os}, region: {aws_region}') + return ami_id + + def get_volume_size(self) -> int: + return Utils.get_value_as_int('volume_size', self.user_values, 200) + + def get_enabled_modules(self) -> List[str]: + enabled_modules = Utils.get_value_as_list('enabled_modules', self.user_values) + if Utils.is_not_empty(enabled_modules): + return enabled_modules + return [] + + def get_metrics_provider(self) -> Optional[str]: + return Utils.get_value_as_string('metrics_provider', self.user_values, 'cloudwatch') + + def get_prometheus_remote_write_url(self) -> Optional[str]: + metrics_provider = self.get_metrics_provider() + if metrics_provider != 'prometheus': + return None + remote_write_url = Utils.get_value_as_string('prometheus_remote_write_url', self.user_values) + if Utils.is_empty(remote_write_url): + raise exceptions.general_exception(f'prometheus_remote_write_url is required when metrics_provider = prometheus') + return remote_write_url + + def get_use_existing_vpc(self) -> bool: + return Utils.get_value_as_bool('use_existing_vpc', self.user_values, False) + + def get_vpc_id(self) -> Optional[str]: + value = Utils.get_value_as_string('vpc_id', self.user_values) + if Utils.is_empty(value) and self.get_use_existing_vpc(): + raise exceptions.invalid_params('vpc_id is required when use_existing_vpc = True') + return value + + def get_private_subnet_ids(self) -> Optional[str]: + private_subnet_ids = Utils.get_value_as_list('private_subnet_ids', self.user_values, []) + public_subnet_ids = Utils.get_value_as_list('public_subnet_ids', self.user_values, []) + if self.get_use_existing_vpc() and Utils.is_empty(private_subnet_ids) and Utils.is_empty(public_subnet_ids): + raise exceptions.invalid_params('private_subnet_ids is required when use_existing_vpc = True') + return private_subnet_ids + + def get_public_subnet_ids(self) -> Optional[str]: + private_subnet_ids = Utils.get_value_as_list('private_subnet_ids', self.user_values, []) + public_subnet_ids = Utils.get_value_as_list('public_subnet_ids', self.user_values, []) + if self.get_use_existing_vpc() and Utils.is_empty(private_subnet_ids) and Utils.is_empty(public_subnet_ids): + raise exceptions.invalid_params('public_subnet_ids is required when use_existing_vpc = True') + return public_subnet_ids + + def get_use_existing_apps_fs(self) -> Optional[bool]: + value = Utils.get_value_as_bool('use_existing_apps_fs', self.user_values, False) + if value and not self.get_use_existing_vpc(): + raise exceptions.invalid_params('use_existing_apps_fs cannot be True if use_existing_vpc = False') + return value + + def get_existing_apps_fs_id(self) -> Optional[str]: + value = Utils.get_value_as_string('existing_apps_fs_id', self.user_values) + if Utils.is_empty(value) and self.get_use_existing_apps_fs(): + raise exceptions.invalid_params('existing_apps_fs_id is required when existing_apps_fs_id = True') + return value + + def get_use_existing_data_fs(self) -> Optional[bool]: + value = Utils.get_value_as_bool('use_existing_data_fs', self.user_values, False) + if value and not self.get_use_existing_vpc(): + raise exceptions.invalid_params('use_existing_data_fs cannot be True if use_existing_vpc = False') + return value + + def get_existing_data_fs_id(self) -> Optional[str]: + value = Utils.get_value_as_string('existing_data_fs_id', self.user_values) + if Utils.is_empty(value) and self.get_use_existing_apps_fs(): + raise exceptions.invalid_params('existing_data_fs_id is required when use_existing_data_fs = True') + return value + + def get_use_existing_opensearch_cluster(self) -> Optional[str]: + value = Utils.get_value_as_bool('use_existing_opensearch_cluster', self.user_values, False) + if value and not self.get_use_existing_vpc(): + raise exceptions.invalid_params('use_existing_opensearch_cluster cannot be True if use_existing_vpc = False') + return value + + def get_opensearch_domain_arn(self) -> Optional[str]: + value = Utils.get_value_as_string('opensearch_domain_arn', self.user_values) + if Utils.is_empty(value) and self.get_use_existing_opensearch_cluster(): + raise exceptions.invalid_params('opensearch_domain_arn is required when use_existing_opensearch_cluster = True') + return value + + def get_opensearch_domain_endpoint(self) -> Optional[str]: + value = Utils.get_value_as_string('opensearch_domain_endpoint', self.user_values) + if Utils.is_empty(value) and self.get_use_existing_opensearch_cluster(): + raise exceptions.invalid_params('opensearch_domain_endpoint is required when use_existing_opensearch_cluster = True') + return value + + def get_alb_custom_certificate_provided(self) -> Optional[bool]: + return Utils.get_value_as_bool('alb_custom_certificate_provided', self.user_values, default=False) + + def get_alb_custom_certificate_acm_certificate_arn(self) -> Optional[str]: + alb_custom_certificate_provided = self.get_alb_custom_certificate_provided() + alb_custom_certificate_acm_certificate_arn = Utils.get_value_as_string('alb_custom_certificate_acm_certificate_arn', self.user_values) + if alb_custom_certificate_provided and Utils.is_empty(alb_custom_certificate_acm_certificate_arn): + raise exceptions.cluster_config_error('Need to provide alb_custom_certificate_acm_certificate_arn if alb_custom_certificate_provided is true') + return alb_custom_certificate_acm_certificate_arn + + def get_alb_get_custom_dns_name(self) -> Optional[str]: + alb_custom_certificate_provided = self.get_alb_custom_certificate_provided() + alb_custom_dns_name = Utils.get_value_as_string('alb_custom_dns_name', self.user_values) + if alb_custom_certificate_provided and Utils.is_empty(alb_custom_dns_name): + raise exceptions.cluster_config_error('Need to provide alb_custom_dns_name if alb_custom_certificate_provided is true') + return alb_custom_dns_name + + def get_dcv_session_traffic_routing(self) -> str: + value = Utils.get_value_as_string('dcv_session_traffic_routing', self.user_values, default='ALB') + if value not in {'ALB', 'NLB'}: + raise exceptions.cluster_config_error(f'Invalid value - "{value}" provided for dcv_session_traffic_routing. Can be one of ALB or NLB only.') + return value + + def get_dcv_session_quic_support(self) -> Optional[bool]: + return Utils.get_value_as_bool('dcv_session_quic_support', self.user_values, default=False) + + def get_dcv_connection_gateway_custom_certificate_provided(self) -> Optional[bool]: + return Utils.get_value_as_bool('dcv_connection_gateway_custom_certificate_provided', self.user_values, default=False) + + def get_dcv_connection_gateway_custom_dns_hostname(self) -> Optional[str]: + value = Utils.get_value_as_string('dcv_connection_gateway_custom_dns_hostname', self.user_values) + custom_certificate_provided = self.get_dcv_connection_gateway_custom_certificate_provided() + if custom_certificate_provided and Utils.is_empty(value): + raise exceptions.cluster_config_error('Need to provide dcv_connection_gateway_custom_dns_hostname if dcv_connection_gateway_custom_certificate_provided is true') + return value + + def get_dcv_connection_gateway_custom_certificate_certificate_secret_arn(self) -> Optional[str]: + value = Utils.get_value_as_string('dcv_connection_gateway_custom_certificate_certificate_secret_arn', self.user_values) + custom_certificate_provided = self.get_dcv_connection_gateway_custom_certificate_provided() + if custom_certificate_provided and Utils.is_empty(value): + raise exceptions.cluster_config_error('Need to provide dcv_connection_gateway_custom_certificate_certificate_secret_arn if dcv_connection_gateway_custom_certificate_provided is true') + return value + + def get_dcv_connection_gateway_custom_certificate_private_key_secret_arn(self) -> Optional[str]: + value = Utils.get_value_as_string('dcv_connection_gateway_custom_certificate_private_key_secret_arn', self.user_values) + custom_certificate_provided = self.get_dcv_connection_gateway_custom_certificate_provided() + if custom_certificate_provided and Utils.is_empty(value): + raise exceptions.cluster_config_error('Need to provide dcv_connection_gateway_custom_certificate_private_key_secret_arn if dcv_connection_gateway_custom_certificate_provided is true') + return value + + # End: User Values Getters and Validations + + def get_cluster_config_dir(self): + return self.props.cluster_config_dir(self.get_cluster_name(), self.get_aws_region()) + + def get_vpc_gateway_endpoints(self) -> Optional[List[str]]: + if not self.get_use_vpc_endpoints(): + return None + if self.get_use_existing_vpc(): + return [] + helper = VpcEndpointsHelper(aws_region=self.get_aws_region(), aws_profile=self.get_aws_profile()) + return helper.get_supported_gateway_endpoint_services() + + def get_vpc_interface_endpoints(self) -> Optional[Dict]: + if not self.get_use_vpc_endpoints(): + return None + if self.get_use_existing_vpc(): + return {} + helper = VpcEndpointsHelper(aws_region=self.get_aws_region(), aws_profile=self.get_aws_profile()) + return helper.get_supported_interface_endpoint_services() + + def generate_config_from_templates(self, temp=False, path=None): + values = { + 'utils': Utils, + 'cluster_name': self.get_cluster_name(), + 'cluster_timezone': self.get_cluster_timezone(), + 'cluster_locale': self.get_cluster_locale(), + 'administrator_email': self.get_administrator_email(), + 'administrator_username': self.get_administrator_username(), + 'aws_account_id': self.get_aws_account_id(), + 'aws_dns_suffix': self.get_aws_dns_suffix(), + 'aws_partition': self.get_aws_partition(), + 'aws_region': self.get_aws_region(), + 'storage_apps_provider': self.get_storage_apps_provider(), + 'storage_data_provider': self.get_storage_data_provider(), + 'apps_mount_dir': self.get_apps_mount_dir(), + 'data_mount_dir': self.get_data_mount_dir(), + 'prefix_list_ids': self.get_prefix_list_ids(), + 'client_ip': self.get_client_ip(), + 'ssh_key_pair_name': self.get_ssh_key_pair_name(), + 'vpc_cidr_block': self.get_vpc_cidr_block(), + 'use_vpc_endpoints': self.get_use_vpc_endpoints(), + 'vpc_gateway_endpoints': self.get_vpc_gateway_endpoints(), + 'vpc_interface_endpoints': self.get_vpc_interface_endpoints(), + 'identity_provider': self.get_identity_provider(), + 'directory_service_provider': self.get_directory_service_provider(), + 'use_existing_directory_service': self.get_use_existing_directory_service(), + 'directory_id': self.get_directory_id(), + 'directory_service_root_username_secret_arn': self.get_directory_service_root_username_secret_arn(), + 'directory_service_root_password_secret_arn': self.get_directory_service_root_password_secret_arn(), + 'enable_aws_backup': self.get_enable_aws_backup(), + 'kms_key_type': self.get_kms_key_type(), + 'kms_key_id': self.get_kms_key_id(), + 'instance_type': self.get_instance_type(), + 'base_os': self.get_base_os(), + 'instance_ami': self.get_instance_ami(), + 'volume_size': self.get_volume_size(), + 'enabled_modules': self.get_enabled_modules(), + 'metrics_provider': self.get_metrics_provider(), + 'prometheus_remote_write_url': self.get_prometheus_remote_write_url(), + 'use_existing_vpc': self.get_use_existing_vpc(), + 'vpc_id': self.get_vpc_id(), + 'private_subnet_ids': self.get_private_subnet_ids(), + 'public_subnet_ids': self.get_public_subnet_ids(), + 'use_existing_apps_fs': self.get_use_existing_apps_fs(), + 'existing_apps_fs_id': self.get_existing_apps_fs_id(), + 'use_existing_data_fs': self.get_use_existing_data_fs(), + 'existing_data_fs_id': self.get_existing_data_fs_id(), + 'use_existing_opensearch_cluster': self.get_use_existing_opensearch_cluster(), + 'opensearch_domain_endpoint': self.get_opensearch_domain_endpoint(), + 'alb_public': self.get_alb_public(), + 'alb_custom_certificate_provided': self.get_alb_custom_certificate_provided(), + 'alb_custom_certificate_acm_certificate_arn': self.get_alb_custom_certificate_acm_certificate_arn(), + 'alb_custom_dns_name': self.get_alb_get_custom_dns_name(), + 'dcv_session_quic_support': self.get_dcv_session_quic_support(), + 'dcv_connection_gateway_custom_certificate_provided': self.get_dcv_connection_gateway_custom_certificate_provided(), + 'dcv_connection_gateway_custom_dns_hostname': self.get_dcv_connection_gateway_custom_dns_hostname(), + 'dcv_connection_gateway_custom_certificate_certificate_secret_arn': self.get_dcv_connection_gateway_custom_certificate_certificate_secret_arn(), + 'dcv_connection_gateway_custom_certificate_private_key_secret_arn': self.get_dcv_connection_gateway_custom_certificate_private_key_secret_arn(), + 'dcv_connection_gateway_instance_ami': self.get_dcv_connection_gateway_instance_ami(), + 'dcv_connection_gateway_instance_type': self.get_dcv_connection_gateway_instance_type(), + 'dcv_connection_gateway_volume_size': self.get_dcv_connection_gateway_volume_size(), + 'dcv_broker_instance_ami': self.get_dcv_broker_instance_ami(), + 'dcv_broker_instance_type': self.get_dcv_broker_instance_type(), + 'dcv_broker_volume_size': self.get_dcv_broker_volume_size(), + } + + env = Jinja2Utils.env_using_file_system_loader(search_path=self.props.cluster_config_templates_dir) + + if temp: + cluster_config_dir = os.path.join(path, 'config') + os.makedirs(cluster_config_dir, exist_ok=True) + else: + cluster_config_dir = self.get_cluster_config_dir() + + idea_config_template = env.get_template('idea.yml') + idea_config_content = idea_config_template.render(**values) + idea_config_file = Path(os.path.join(cluster_config_dir, 'idea.yml')) + + idea_config = Utils.from_yaml(idea_config_content) + + modules = idea_config['modules'] + for module in modules: + module_name = module['name'] + module_id = module['id'] + print(f'processing module: {module_name}') + config_files = module['config_files'] + for file in config_files: + # templates should contain settings with module name, + # target file generation will create directory with module_id + template = env.get_template(os.path.join(module_name, file)) + settings = template.render(**values, module_id=module_id, module_name=module_name, supported_base_os=constants.SUPPORTED_OS) + settings_file = Path(os.path.join(cluster_config_dir, module_id, file)) + settings_file.parent.mkdir(exist_ok=True) + + with open(settings_file, 'w') as f: + f.write(settings) + print(f'generated config file: {settings_file}') + + with open(idea_config_file, 'w') as f: + f.write(Utils.to_yaml(idea_config)) + + def read_modules_from_files(self, directory: str = None) -> List[Dict]: + if directory is not None: + cluster_config_dir = directory + else: + cluster_config_dir = self.get_cluster_config_dir() + idea_config_file = os.path.join(cluster_config_dir, 'idea.yml') + with open(idea_config_file, 'r') as f: + idea_config = Utils.from_yaml(f.read()) + modules = idea_config['modules'] + return modules + + def read_config_from_files(self, config_dir: Optional[str] = None) -> Dict: + """ + read config files (settings.yml) from local file system and return as Dict + :param config_dir: path to the `config` directory that contains `idea.yml` + :return: cluster configuration as Dict + """ + + if Utils.is_not_empty(config_dir): + cluster_config_dir = config_dir + else: + cluster_config_dir = self.get_cluster_config_dir() + + config = {} + + idea_config_file = os.path.join(cluster_config_dir, 'idea.yml') + with open(idea_config_file, 'r') as f: + idea_config = Utils.from_yaml(f.read()) + + modules = idea_config['modules'] + for module in modules: + module_id = module['id'] + + config_files = module['config_files'] + module_settings = {} + for file in config_files: + settings_file = os.path.join(cluster_config_dir, module_id, file) + with open(settings_file, 'r') as f: + settings = Utils.from_yaml(f.read()) + module_settings = {**module_settings, **settings} + + config[module_id] = module_settings + + return config + + def traverse_config(self, config_entries: List[Dict], prefix: str, config: Dict, filter_key_prefix: str = None): + for key in config: + + if '.' in key or ':' in key: + raise exceptions.general_exception(f'Config key name: {key} under: {prefix} cannot contain a dot(.), colon(:) or comma(,)') + + value = Utils.get_any_value(key, config) + + if Utils.is_not_empty(prefix): + path_prefix = f'{prefix}.{key}' + else: + path_prefix = key + + if isinstance(value, dict): + + self.traverse_config(config_entries, path_prefix, value, filter_key_prefix) + + else: + if Utils.is_not_empty(filter_key_prefix): + if not path_prefix.startswith(filter_key_prefix): + continue + + config_entries.append({ + 'key': path_prefix, + 'value': value + }) + + def convert_config_to_key_value_pairs(self, key_prefix: str, path: str = None) -> List[Dict]: + config = self.read_config_from_files(path) + config_entries = [] + self.traverse_config(config_entries, '', config, filter_key_prefix=key_prefix) + return config_entries + + def get_config_local(self, path: str = None): + if path is not None: + config_entries = self.convert_config_to_key_value_pairs('', path) + else: + config_entries = self.convert_config_to_key_value_pairs('') + return config_entries diff --git a/source/idea/idea-administrator/src/ideaadministrator/app/delete_cluster.py b/source/idea/idea-administrator/src/ideaadministrator/app/delete_cluster.py new file mode 100644 index 00000000..45a47733 --- /dev/null +++ b/source/idea/idea-administrator/src/ideaadministrator/app/delete_cluster.py @@ -0,0 +1,889 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +from ideasdk.context import SocaCliContext, SocaContextOptions +from ideasdk.config.cluster_config_db import ClusterConfigDB +from ideasdk.utils import Utils +from ideadatamodel import constants, exceptions, errorcodes, EC2Instance, SocaMemory, SocaMemoryUnit + +from typing import Optional, List +from prettytable import PrettyTable +import time +import botocore.exceptions + + +class DeleteCluster: + + def __init__(self, cluster_name: str, aws_region: str, aws_profile: str, delete_bootstrap: bool, delete_databases: bool, delete_backups: bool, delete_cloudwatch_logs: bool, delete_all: bool, force: bool): + self.cluster_name = cluster_name + self.aws_region = aws_region + self.aws_profile = aws_profile + self.delete_bootstrap = delete_bootstrap + self.delete_databases = delete_databases + self.delete_backups = delete_backups + self.delete_cloudwatch_logs = delete_cloudwatch_logs + self.delete_all = delete_all + self.force = force + self.delete_failed_attempt = 0 + self.delete_failed_max_attempts = 3 + + self.context = SocaCliContext( + options=SocaContextOptions( + aws_region=aws_region, + aws_profile=aws_profile, + enable_aws_client_provider=True, + enable_aws_util=True + ) + ) + + try: + self.cluster_config_db = ClusterConfigDB( + cluster_name=cluster_name, + aws_region=aws_region, + aws_profile=aws_profile + ) + except exceptions.SocaException as e: + if e.error_code == errorcodes.CLUSTER_CONFIG_NOT_INITIALIZED: + self.cluster_config_db: Optional[ClusterConfigDB] = None + else: + raise e + + if self.cluster_config_db is not None: + self.cluster_modules = self.cluster_config_db.get_cluster_modules() + else: + self.cluster_modules = [] + + self.app_modules = [] + for cluster_module in self.cluster_modules: + module_type = Utils.get_value_as_string('type', cluster_module, None) + if module_type == 'app': + self.app_modules.append(cluster_module) + + self.ec2_instances = [] + self.termination_protected_ec2_instances = [] + + # all stacks that belong to the cluster, but not the actual cluster stack + self.cloud_formation_stacks = [] + # cluster stack. there will most likely be only one stack, but the modular data structures do support multiple stack for clusters. + self.cluster_stacks = [] + # Identity Provider stacks - requires disable of the UserPool protection + self.identity_provider_stacks = [] + + self.dynamodb_tables = [] + + self.cloudwatch_logs = [] + + def get_bootstrap_stack_name(self) -> str: + return f'{self.cluster_name}-bootstrap' + + def find_ec2_instances(self): + self.context.info('Searching for EC2 instances to be terminated ...') + ec2_instances_to_delete = [] + termination_protected_instances = [] + ec2_instances = self.context.aws_util().ec2_describe_instances( + filters=[ + { + 'Name': f'tag:{constants.IDEA_TAG_CLUSTER_NAME}', + 'Values': [self.cluster_name] + } + ] + ) + for ec2_instance in ec2_instances: + if ec2_instance.state == 'terminated': + continue + + # check termination protection instances + describe_instance_attribute_result = self.context.aws().ec2().describe_instance_attribute( + Attribute='disableApiTermination', + InstanceId=ec2_instance.instance_id + ) + disable_api_termination = Utils.get_value_as_dict('DisableApiTermination', describe_instance_attribute_result) + disable_api_termination_enabled = Utils.get_value_as_bool('Value', disable_api_termination, False) + if disable_api_termination_enabled: + termination_protected_instances.append(ec2_instance) + time.sleep(0.1) # 10 tps - might need to be adjusted in-future. allowed 100 TPS - https://docs.aws.amazon.com/AWSEC2/latest/APIReference/throttling.html + + # app and infra node type instances will be terminated by their respective cloudformation stacks + # we are primarily interested in the instances launched without CloudFormation stack + node_type = ec2_instance.soca_node_type + if node_type in (constants.NODE_TYPE_APP, constants.NODE_TYPE_INFRA): + continue + ec2_instances_to_delete.append(ec2_instance) + + self.termination_protected_ec2_instances = termination_protected_instances + self.ec2_instances = ec2_instances_to_delete + + @staticmethod + def print_ec2_instances(ec2_instances: List[EC2Instance]): + instance_table = PrettyTable(['Name', 'Instance Id', 'Private IP', 'Instance Type', 'Status']) + instance_table.align = 'l' + for ec2_instance in ec2_instances: + instance_table.add_row([ + ec2_instance.get_tag('Name'), + ec2_instance.instance_id, + ec2_instance.private_ip_address, + ec2_instance.instance_type, + ec2_instance.state + ]) + print(instance_table) + + def delete_ec2_instances(self): + if len(self.termination_protected_ec2_instances) > 0: + for ec2_instance in self.termination_protected_ec2_instances: + self.context.info(f'disabling termination protection for EC2 instance: {ec2_instance.instance_id} ...') + self.context.aws().ec2().modify_instance_attribute( + InstanceId=ec2_instance.instance_id, + DisableApiTermination={ + 'Value': False + } + ) + self.context.success(f'termination protection disabled for EC2 instance: {ec2_instance.instance_id}') + time.sleep(1) + + if len(self.ec2_instances) > 0: + for ec2_instance in self.ec2_instances: + self.context.info(f'terminating EC2 instance: {ec2_instance.instance_id}') + self.context.aws().ec2().terminate_instances( + InstanceIds=[ec2_instance.instance_id] + ) + self.context.success(f'terminated EC2 instance: {ec2_instance.instance_id}') + time.sleep(1) + + def _get_app_instance(self, module_id: str) -> Optional[EC2Instance]: + describe_instances_result = self.context.aws().ec2().describe_instances( + Filters=[ + { + 'Name': 'instance-state-name', + 'Values': ['pending', 'stopped', 'running'] + }, + { + 'Name': f'tag:{constants.IDEA_TAG_CLUSTER_NAME}', + 'Values': [self.cluster_name] + }, + { + 'Name': f'tag:{constants.IDEA_TAG_MODULE_ID}', + 'Values': [module_id] + }, + { + 'Name': f'tag:{constants.IDEA_TAG_NODE_TYPE}', + 'Values': ['app'] + } + ] + ) + + reservations = Utils.get_value_as_list('Reservations', describe_instances_result, []) + for reservation in reservations: + instances = Utils.get_value_as_list('Instances', reservation) + for instance in instances: + ec2_instance = EC2Instance(instance) + if ec2_instance.state == 'running': + return ec2_instance + return None + + def invoke_app_app_module_clean_up(self): + instance_ids = [] + for module in self.app_modules: + module_id = Utils.get_value_as_string('module_id', module, None) + if Utils.is_empty(module_id): + continue + + app_instance = self._get_app_instance(module_id) + if Utils.is_empty(app_instance): + continue + + print(f'executing app-module-clean-up commands for app: {module_id}') + instance_ids.append(app_instance.instance_id) + + command_to_execute = f'sudo ideactl app-module-clean-up' + if self.delete_databases: + command_to_execute = f'{command_to_execute} --delete-databases' + + send_command_result = self.context.aws().ssm().send_command( + InstanceIds=instance_ids, + DocumentName='AWS-RunShellScript', + Parameters={ + 'commands': [command_to_execute] + } + ) + + command_id = send_command_result['Command']['CommandId'] + while True: + list_command_invocations_result = self.context.aws().ssm().list_command_invocations( + CommandId=command_id, + Details=False + ) + command_invocations = list_command_invocations_result['CommandInvocations'] + + completed_count = 0 + failed_count = 0 + + for command_invocation in command_invocations: + status = command_invocation['Status'] + + if status in ('Success', 'TimedOut', 'Cancelled', 'Failed'): + completed_count += 1 + + if status in ('TimedOut', 'Cancelled', 'Failed'): + failed_count += 1 + + if completed_count == len(command_invocations): + break + + time.sleep(10) + + def find_cloud_formation_stacks(self): + self.context.info('Searching for CloudFormation stacks to be terminated ...') + stacks_to_delete = [] + cluster_stacks = [] + identity_provider_stacks = [] + pagination_token = None + while True: + request = { + 'TagFilters': [ + { + 'Key': constants.IDEA_TAG_CLUSTER_NAME, + 'Values': [self.cluster_name] + } + ], + 'ResourceTypeFilters': ['cloudformation'] + } + if pagination_token is None: + get_resources_result = self.context.aws().resource_groups_tagging_api().get_resources(**request) + else: + get_resources_result = self.context.aws().resource_groups_tagging_api().get_resources(**{ + **request, + 'PaginationToken': pagination_token + }) + + pagination_token = Utils.get_value_as_string('PaginationToken', get_resources_result) + + resources = Utils.get_value_as_list('ResourceTagMappingList', get_resources_result, []) + for resource in resources: + try: + + stack_id = Utils.get_value_as_string('ResourceARN', resource) + + stack = self.describe_cloud_formation_stack(stack_id) + stack_name = Utils.get_value_as_string('StackName', stack) + + if Utils.is_empty(stack_name): + continue + + if self.is_bootstrap_stack(stack_name): + continue + + if self.is_cluster_stack(stack_name): + cluster_stacks.append(stack) + elif self.is_identity_provider_stack(stack_name): + identity_provider_stacks.append(stack) + else: + stacks_to_delete.append(stack) + + # sleep for a while to ensure we don't flood describe_stack() API + time.sleep(0.5) + + except botocore.exceptions.ClientError as e: + # race condition, where resources tagging API returns a cfn stack, but the stack could be deleted. + # * this scenario occurs when scheduler launches a job stack and deletes it by the time delete cluster calls describe cfn stack + # * to address this scenario, skip all validation error cases + if e.response['Error']['Code'] != 'ValidationError': + raise e + + if Utils.is_empty(pagination_token): + break + + self.cloud_formation_stacks = stacks_to_delete + self.cluster_stacks = cluster_stacks + self.identity_provider_stacks = identity_provider_stacks + + def print_cloud_formation_stacks(self): + stacks_table = PrettyTable(['Stack Name', 'Status', 'Termination Protection']) + stacks_table.align = 'l' + stacks = self.cloud_formation_stacks + self.cluster_stacks + for stack in stacks: + stacks_table.add_row([ + Utils.get_value_as_string('StackName', stack), + Utils.get_value_as_string('StackStatus', stack), + Utils.get_value_as_bool('EnableTerminationProtection', stack, False) + ]) + + if len(stacks) > 0: + print(stacks_table) + print(f'{len(stacks)} stacks will be terminated.') + + def describe_cloud_formation_stack(self, stack_name: str): + describe_stack_result = self.context.aws().cloudformation().describe_stacks( + StackName=stack_name + ) + stacks = Utils.get_value_as_list('Stacks', describe_stack_result) + return stacks[0] + + def delete_cloud_formation_stack(self, stack_name: str): + + try: + stack = self.describe_cloud_formation_stack(stack_name) + except botocore.exceptions.ClientError as e: + if e.response['Error']['Code'] == 'ValidationError': + return + else: + raise e + + enable_termination_protection = Utils.get_value_as_bool('EnableTerminationProtection', stack, False) + if not self.force and enable_termination_protection: + confirm = self.context.prompt(f'Termination protection is enabled for stack: {stack_name}. Disable and terminate?') + if not confirm: + self.context.error('Abort cluster deletion') + raise SystemExit + + if enable_termination_protection: + print(f'disabling termination protection for stack: {stack_name}') + self.context.aws().cloudformation().update_termination_protection( + EnableTerminationProtection=False, + StackName=stack_name + ) + + print(f'terminating CloudFormation stack: {stack_name}') + self.context.aws().cloudformation().delete_stack( + StackName=stack_name + ) + + def delete_dynamo_table(self, table_name: str): + try: + print(f'deleting table: {table_name} ...') + self.context.aws().dynamodb().delete_table(TableName=table_name) + self.context.success(f'deleted dynamodb table: {table_name}') + except botocore.exceptions.BotoCoreError as e: + raise e + + def is_bootstrap_stack(self, stack_name: str) -> bool: + return stack_name == self.get_bootstrap_stack_name() + + def is_cluster_stack(self, stack_name: str) -> bool: + for module in self.cluster_modules: + module_name = module['name'] + if module_name == constants.MODULE_CLUSTER: + cluster_stack_name = module['stack_name'] + if cluster_stack_name == stack_name: + return True + if stack_name == f'{self.cluster_name}-cluster': + return True + return False + + def is_analytics_stack(self, stack_name: str) -> bool: + for module in self.cluster_modules: + module_name = module['name'] + if module_name == constants.MODULE_ANALYTICS: + analytics_stack_name = module['stack_name'] + if analytics_stack_name == stack_name: + return True + if stack_name == f'{self.cluster_name}-analytics': + return True + return False + + def is_identity_provider_stack(self, stack_name: str) -> bool: + for module in self.cluster_modules: + module_name = module['name'] + if module_name == constants.MODULE_IDENTITY_PROVIDER: + cluster_stack_name = module['stack_name'] + if cluster_stack_name == stack_name: + return True + if stack_name == f'{self.cluster_name}-identity-provider': + return True + return False + + def try_delete_vpc_lambda_enis(self): + # fix to address scenario where deleting lambda function in VPC takes a very long time + # and in-turn causes cluster deletion to either fail + describe_network_interfaces_result = self.context.aws().ec2().describe_network_interfaces( + Filters=[ + { + 'Name': 'description', + 'Values': [ + f'AWS Lambda VPC ENI-{self.cluster_name}-*' + ] + }, + { + 'Name': 'status', + 'Values': [ + 'available' + ] + } + ] + ) + network_interfaces = Utils.get_value_as_list('NetworkInterfaces', describe_network_interfaces_result, []) + if len(network_interfaces) > 0: + print('found VPC lambda network interfaces for the cluster in "available" state. deleting ...') + for network_interface in network_interfaces: + network_interface_id = network_interface['NetworkInterfaceId'] + description = network_interface['Description'] + print(f'deleting VPC Lambda ENI - NetworkInterfaceId: {network_interface_id}, Description: {description}') + self.context.aws().ec2().delete_network_interface( + NetworkInterfaceId=network_interface_id + ) + + def check_stack_deletion_status(self, stack_names: List[str]) -> bool: + delete_failed = 0 + stacks_pending = list(stack_names) + + while len(stacks_pending) > 0: + + stacks_deleted = [] + for stack_name in stacks_pending: + try: + describe_stack_result = self.context.aws().cloudformation().describe_stacks( + StackName=stack_name + ) + stacks = Utils.get_value_as_list('Stacks', describe_stack_result) + stack = stacks[0] + stack_status = Utils.get_value_as_string('StackStatus', stack) + + if stack_status == 'DELETE_COMPLETE': + self.context.success(f'stack: {stack_name}, status: {stack_status}') + stacks_deleted.append(stack_name) + elif stack_status == 'DELETE_FAILED': + if self.delete_failed_attempt < self.delete_failed_max_attempts: + if self.is_analytics_stack(stack_name): + self.try_delete_vpc_lambda_enis() + self.context.warning(f'stack: {stack_name}, status: {stack_status}, submitting a new delete_cloud_formation_stack request. [Loop {self.delete_failed_attempt}/{self.delete_failed_max_attempts}]') + self.delete_cloud_formation_stack(stack_name) + self.delete_failed_attempt += 1 + else: + self.context.error(f'stack: {stack_name}, status: {stack_status}') + stacks_deleted.append(stack_name) + delete_failed += 1 + else: + print(f'stack: {stack_name}, status: {stack_status}') + if self.is_analytics_stack(stack_name): + self.try_delete_vpc_lambda_enis() + + except botocore.exceptions.ClientError as e: + if e.response['Error']['Code'] == 'ValidationError': + self.context.success(f'stack: {stack_name}, status: DELETE_COMPLETE') + stacks_deleted.append(stack_name) + else: + raise e + + for stack_name in stacks_deleted: + stacks_pending.remove(stack_name) + + if len(stacks_pending) > 0: + num_stacks_pending = len(stacks_pending) + if num_stacks_pending == 1: + print(f'waiting for 1 stack to be deleted: {stacks_pending} ...') + else: + print(f'waiting for {num_stacks_pending} stacks to be deleted: {stacks_pending} ...') + time.sleep(15) + + return delete_failed == 0 + + def delete_identity_provider_stacks(self): + stack_names = [] + user_pool_ids_to_unprotect = [] + + describe_user_pool_paginator = self.context.aws().cognito_idp().get_paginator('list_user_pools') + user_pool_iter = describe_user_pool_paginator.paginate(MaxResults=50) + + # Walk the list of user pools in the account looking for our matching pool + # The pool must match the expected name, as well as having the proper + # idea:ClusterName tag for us to consider it as valid. + for page in user_pool_iter: + user_pools = Utils.get_value_as_list('UserPools', page, default=[]) + + confirm_delete_pools = self.force + if not self.force: + confirm_delete_pools = self.context.prompt(f'Are you sure you want to delete the User Pools associated with the cluster: 'f'{self.cluster_name}? This action is not reversible.') + + if not confirm_delete_pools: + self.context.error('Aborting Delete Operation - User Pools remain intact!') + raise SystemExit + + if user_pools: + for pool in user_pools: + pool_name = Utils.get_value_as_string('Name', pool, default='') + pool_id = Utils.get_value_as_string('Id', pool, default=None) + if pool_name == f'{self.cluster_name}-user-pool' and pool_id is not None: + user_pool_ids_to_unprotect.append(pool_id) + + # Unprotect the discovered user pools if they have matching Tags + for pool_id in user_pool_ids_to_unprotect: + describe_user_pool_result = self.context.aws().cognito_idp().describe_user_pool( + UserPoolId=pool_id + ) + delete_protection = Utils.get_value_as_string('DeletionProtection', describe_user_pool_result, default='ACTIVE') + + if delete_protection.upper() == 'ACTIVE': + pool_tags = Utils.get_value_as_dict('UserPoolTags', Utils.get_value_as_dict('UserPool', describe_user_pool_result, default={}), default={}) + for tag_name, tag_value in pool_tags.items(): + if tag_name == constants.IDEA_TAG_CLUSTER_NAME and tag_value == self.cluster_name: + self.context.info(f'Cognito User Pool {pool_id} - Deletion Protection is {delete_protection} - Removing') + self.context.aws().cognito_idp().update_user_pool( + UserPoolId=pool_id, + DeletionProtection='INACTIVE' + ) + time.sleep(0.5) + + # For non-Cognito deployments - we should just land here to cleanup the stacks + # If there are any other cleanups needed - add them here + for stack in self.identity_provider_stacks: + stack_name = Utils.get_value_as_string('StackName', stack) + stack_names.append(stack_name) + self.delete_cloud_formation_stack(stack_name) + + deletion_status = self.check_stack_deletion_status(stack_names) + if not deletion_status: + self.context.error('failed to delete CloudFormation stacks. abort!') + raise SystemExit + + def delete_cloud_formation_stacks(self): + stack_names = [] + for stack in self.cloud_formation_stacks: + stack_name = Utils.get_value_as_string('StackName', stack) + stack_names.append(stack_name) + self.delete_cloud_formation_stack(stack_name) + + deletion_status = self.check_stack_deletion_status(stack_names) + if not deletion_status: + self.context.error('failed to delete CloudFormation stacks. abort!') + raise SystemExit + + def find_dynamodb_tables(self): + last_evaluated_table_name = None + while True: + if Utils.is_empty(last_evaluated_table_name): + list_tables_result = self.context.aws().dynamodb().list_tables() + else: + list_tables_result = self.context.aws().dynamodb().list_tables(ExclusiveStartTableName=last_evaluated_table_name) + tables = Utils.get_value_as_list('TableNames', list_tables_result, []) + for table_name in tables: + if table_name.startswith(f'{self.cluster_name}.'): + self.dynamodb_tables.append(table_name) + + last_evaluated_table_name = Utils.get_value_as_string('LastEvaluatedTableName', list_tables_result, None) + if Utils.is_empty(last_evaluated_table_name): + break + + def print_dynamodb_tables(self): + dynamodb_table = PrettyTable(['Table Name']) + dynamodb_table.align = 'l' + tables = self.dynamodb_tables + for table in tables: + dynamodb_table.add_row([table]) + + if len(tables) > 0: + print(dynamodb_table) + print(f'{len(tables)} tables will be deleted.') + + def delete_dynamodb_tables(self): + for table in self.dynamodb_tables: + self.delete_dynamo_table(table) + + # Cleanup Cloudwatch Alarms for all tables + self.delete_cloudwatch_alarms() + + + def delete_cloudwatch_alarms(self): + alarms_to_delete = [] + # Generate our list of Cloudwatch alarms that pertain to us + # This will grab all alarms for the present cluster as well + # as any previous clusters with the same name. This can clean up + # previous versions that didn't have alarm cleanup + try: + paginator = self.context.aws().cloudwatch().get_paginator('describe_alarms') + page_iterator = paginator.paginate(AlarmNamePrefix=f'TargetTracking-table/{self.cluster_name}') + for page in page_iterator: + for metric in page.get('MetricAlarms', []): + alarm_name = metric.get('AlarmName', '') + alarm_namespace = metric.get('Namespace', 'unknown-namespace') + if alarm_namespace != 'AWS/DynamoDB': + continue + + for dim in metric.get('Dimensions', []): + dim_name = dim.get('Name', '') + dim_value = dim.get('Value', '') + + # Only concerned for the DDB alarms + if dim_name != 'TableName': + continue + # Make sure it is a DDB table we expect as part of our schema + if dim_value not in self.dynamodb_tables: + self.context.warning(f'Found a mismatched CloudWatch alarm / DynamoDB table name: {alarm_name}') + continue + # + if alarm_name in alarms_to_delete: + continue + # If we make it this far - it is the alarm we are looking for, append. + alarms_to_delete.append(alarm_name) + except Exception as e: + self.context.warning(f'Exception ({e}) trying to get CloudWatch Alarms for {self.cluster_name}') + raise e + + # Now delete the list + if len(alarms_to_delete) > 0: + self.context.info(f'Found {len(alarms_to_delete):,} CloudWatch alarms to delete.') + # delete_alarms() operates on 100 at a time, so we chunk them + # in case we have many alarms to delete. + try: + for chunk in range(0, len(alarms_to_delete), 100): + delete_resp = self.context.aws().cloudwatch().delete_alarms(AlarmNames=alarms_to_delete[chunk:chunk+100]) + if delete_resp and delete_resp.get('ResponseMetadata', {}).get('HTTPStatusCode', 400) == 200: + self.context.info(f'Successfully deleted CloudWatch alarms batch #{chunk} for {self.cluster_name}') + else: + self.context.warning(f'Error during delete of CloudWatch Alarms batch #{chunk} for {self.cluster_name}: {delete_resp}') + except Exception as e: + self.context.warning(f'Exception ({e}) during delete of CloudWatch Alarms for {self.cluster_name}') + raise e + self.context.success(f'Deleted CloudWatch alarms for: {self.cluster_name}') + else: + self.context.info(f'Did not find any Cloudwatch alarms to delete...') + + def find_cloudwatch_logs(self): + total_bytes = 0 + self.context.info('Searching for CloudWatch log groups to be deleted ...') + paginator = self.context.aws().logs().get_paginator('describe_log_groups') + + for prefix in {f'/{self.cluster_name}', f'/aws/lambda/{self.cluster_name}'}: + self.context.info(f'Looking for Cloudwatch logs in {prefix}') + page_iterator = paginator.paginate(logGroupNamePrefix=prefix) + for page in page_iterator: + for log_group in page["logGroups"]: + log_group_name = Utils.get_value_as_string('logGroupName', log_group) + log_group_bytes = Utils.get_value_as_int('storedBytes', log_group, default=0) + if Utils.is_empty(log_group_name): + continue + self.cloudwatch_logs.append({'name': log_group_name, 'size': log_group_bytes}) + + def print_cloudwatch_logs(self): + total_size = 0 + cloudwatch_logs_table = PrettyTable(['Log Group Name', 'Size']) + cloudwatch_logs_table.align = 'l' + logs = self.cloudwatch_logs + for log in logs: + log_group_size = SocaMemory(value=log.get('size', 0), unit=SocaMemoryUnit.BYTES) + total_size += log.get('size', 0) + cloudwatch_logs_table.add_row([log.get('name'), log_group_size.as_unit(SocaMemoryUnit.MB)]) + + if len(logs) > 0: + print(cloudwatch_logs_table) + + total_size_mb = SocaMemory(value=total_size, unit=SocaMemoryUnit.BYTES) + print(f'{len(logs)} log groups will be deleted. Total Size: {total_size_mb.as_unit(SocaMemoryUnit.MB)}') + + def delete_cloudwatch_log_groups(self): + for log_group in self.cloudwatch_logs: + try: + log_group_name = log_group.get('name') + print(f'deleting cloudwatch log group: {log_group_name} ...') + self.context.aws().logs().delete_log_group(logGroupName=log_group_name) + self.context.success(f'deleted log group: {log_group_name}') + time.sleep(.1) + except botocore.exceptions.BotoCoreError as e: + raise e + + def delete_bootstrap_and_s3_bucket(self): + stack_name = self.get_bootstrap_stack_name() + self.delete_cloud_formation_stack(stack_name) + self.delete_s3_bucket() + + def delete_cluster_stack(self): + stack_names = [] + for cluster_stack in self.cluster_stacks: + stack_name = Utils.get_value_as_string('StackName', cluster_stack) + stack_names.append(stack_name) + self.delete_cloud_formation_stack(stack_name) + + deletion_status = self.check_stack_deletion_status(stack_names) + if not deletion_status: + self.context.error('failed to delete cluster stack. abort!') + raise SystemExit + + def delete_s3_bucket(self): + try: + + # get s3 bucket name + bucket_name = None + if self.cluster_config_db is not None: + config_entry = self.cluster_config_db.get_config_entry('cluster.cluster_s3_bucket') + + if Utils.is_not_empty(config_entry): + bucket_name = config_entry['value'] + + if Utils.is_empty(bucket_name): + account_id = self.context.aws().aws_account_id() + bucket_name = str(self.cluster_name) + '-cluster-' + self.aws_region + '-' + account_id + + s3_bucket = self.context.aws().s3_bucket() + bucket = s3_bucket.Bucket(bucket_name) + if bucket.creation_date: + print(f'found cluster S3 bucket: {bucket_name}') + else: + self.context.warning('cluster bucket not found. skip.') + return + + print(f'deleting S3 bucket: {bucket_name} for cluster ...') + + # create s3 bucket resource and delete all versions in the bucket + # This includes current versions - no explicit object list needed + bucket.object_versions.all().delete() + time.sleep(5) + + # delete the bucket itself + self.context.aws().s3().delete_bucket(Bucket=bucket_name) + + self.context.success(f'bucket {bucket_name} deleted successfully') + + except botocore.exceptions.BotoCoreError as e: + raise e + + def delete_backup_vault_recovery_points(self): + """ + delete all recovery points for the backup vault configured for the cluster. + the implementation assumes the backup vault will be named as: CLUSTER_NAME-cluster-backup-vault. + """ + # backup vault must be of below name format + backup_vault_name = f'{self.cluster_name}-cluster-backup-vault' + try: + + # basic check to find if backup vault exists + self.context.aws().backup().describe_backup_vault( + BackupVaultName=backup_vault_name + ) + + total_recovery_points = 0 + total_deleted = 0 + + next_token = None + while True: + + if next_token is None: + list_recovery_points_by_backup_vault_result = self.context.aws().backup().list_recovery_points_by_backup_vault( + BackupVaultName=backup_vault_name + ) + else: + list_recovery_points_by_backup_vault_result = self.context.aws().backup().list_recovery_points_by_backup_vault( + BackupVaultName=backup_vault_name, + NextToken=next_token + ) + + next_token = Utils.get_value_as_string('NextToken', list_recovery_points_by_backup_vault_result) + + recovery_points = Utils.get_value_as_list('RecoveryPoints', list_recovery_points_by_backup_vault_result, []) + + total_recovery_points += len(recovery_points) + + for recovery_point in recovery_points: + recovery_point_arn = Utils.get_value_as_string('RecoveryPointArn', recovery_point) + + # can be one of: 'COMPLETED'|'PARTIAL'|'DELETING'|'EXPIRED' + # if status is not COMPLETED/EXPIRED, do not attempt to delete, but wait for deletion or backup completion. + recovery_point_status = Utils.get_value_as_string('Status', recovery_point) + if recovery_point_status not in ('COMPLETED', 'EXPIRED'): + continue + + self.context.info(f'deleting recovery point: {recovery_point_arn} ...') + self.context.aws().backup().delete_recovery_point( + BackupVaultName=backup_vault_name, + RecoveryPointArn=recovery_point_arn + ) + total_deleted += 1 + time.sleep(.1) + + if next_token is None: + if total_recovery_points == total_deleted: + self.context.info(f'deleted {total_recovery_points} recovery points.') + break + else: + total_pending = total_recovery_points - total_deleted + with self.context.spinner(f'waiting for {total_pending} recovery points to be deleted or ready for deleting ...'): + time.sleep(10) + + except botocore.exceptions.ClientError as e: + if e.response['Error']['Code'] != 'ResourceNotFoundException': + # possibly backups are not enabled for the cluster. skip + pass + else: + raise e + + def invoke(self): + + # Finding ec2 instances + self.find_ec2_instances() + + if Utils.is_not_empty(self.ec2_instances): + self.print_ec2_instances(self.ec2_instances) + print(f'{len(self.ec2_instances)} ec2 instances will be terminated.') + + # Finding CloudFormation stacks + self.find_cloud_formation_stacks() + self.print_cloud_formation_stacks() + + if not self.force: + confirm = self.context.prompt(f'Are you sure you want to delete cluster: {self.cluster_name}, region: {self.aws_region} ?') + if not confirm: + return + + self.invoke_app_app_module_clean_up() + + if Utils.is_not_empty(self.termination_protected_ec2_instances): + self.print_ec2_instances(self.termination_protected_ec2_instances) + print(f'found {len(self.termination_protected_ec2_instances)} EC2 instances with termination protection enabled.') + if not self.force: + confirm = self.context.prompt(f'Are you sure you want to disable termination protection for above instances ?') + if not confirm: + return + + self.delete_ec2_instances() + self.delete_cloud_formation_stacks() + + # Delete identity-provider stack - removing UserPool protection + self.delete_identity_provider_stacks() + + # delete backups if applicable + if self.delete_backups or self.delete_all: + confirm_delete_backups = self.force + if not self.force: + confirm_delete_backups = self.context.prompt(f'Are you sure you want to delete all the backup recovery points associated with the cluster: 'f'{self.cluster_name}?') + + if confirm_delete_backups: + self.delete_backup_vault_recovery_points() + + self.delete_cluster_stack() + + if self.delete_bootstrap or self.delete_all: + confirm_delete_bootstrap = self.force + if not self.force: + confirm_delete_bootstrap = self.context.prompt(f'Are you sure you want to delete the bootstrap stack and S3 Bucket associated with the cluster: 'f'{self.cluster_name}? This action is not reversible.') + + if confirm_delete_bootstrap: + self.delete_bootstrap_and_s3_bucket() + + if self.delete_databases or self.delete_all: + self.find_dynamodb_tables() + if Utils.is_not_empty(self.dynamodb_tables): + self.print_dynamodb_tables() + + confirm_delete_databases = self.force + if not self.force: + confirm_delete_databases = self.context.prompt(f'Are you sure you want to delete all dynamodb tables associated with the cluster: 'f'{self.cluster_name}?') + + if confirm_delete_databases: + self.delete_dynamodb_tables() + + if self.delete_cloudwatch_logs or self.delete_all: + self.find_cloudwatch_logs() + if Utils.is_not_empty(self.cloudwatch_logs): + self.print_cloudwatch_logs() + + confirm_delete_cloudwatch_logs = self.force + if not self.force: + confirm_delete_cloudwatch_logs = self.context.prompt(f'Are you sure you want to delete all cloudwatch logs associated with the cluster: 'f'{self.cluster_name}?') + + if confirm_delete_cloudwatch_logs: + self.delete_cloudwatch_log_groups() diff --git a/source/idea/idea-administrator/src/ideaadministrator/app/deployment_helper.py b/source/idea/idea-administrator/src/ideaadministrator/app/deployment_helper.py new file mode 100644 index 00000000..2fcc4a70 --- /dev/null +++ b/source/idea/idea-administrator/src/ideaadministrator/app/deployment_helper.py @@ -0,0 +1,207 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +from ideadatamodel import constants, exceptions +from ideasdk.config.cluster_config_db import ClusterConfigDB +from ideaadministrator.app.cdk.cdk_invoker import CdkInvoker +from ideasdk.utils import Utils, ModuleMetadataHelper + +from typing import List +from collections import OrderedDict +import threading +import botocore.exceptions +import time + + +class DeploymentHelper: + + def __init__(self, cluster_name: str, aws_region: str, + termination_protection: bool = True, + deployment_id: str = None, + upgrade: bool = False, + all_modules: bool = False, + force_build_bootstrap: bool = False, + optimize_deployment: bool = False, + module_set: str = constants.DEFAULT_MODULE_SET, + module_ids: List[str] = None, + aws_profile: str = None, + rollback: bool = True): + + self.cluster_name = cluster_name + self.aws_region = aws_region + self.termination_protection = termination_protection + self.upgrade = upgrade + self.rollback = rollback + self.all_modules = all_modules + self.module_set = module_set + + if Utils.is_empty(deployment_id): + deployment_id = Utils.uuid() + self.deployment_id = deployment_id + + self.force_build_bootstrap = force_build_bootstrap + self.optimize_deployment = optimize_deployment + self.aws_profile = aws_profile + + self.module_metadata_helper = ModuleMetadataHelper() + + self.cluster_config_db = ClusterConfigDB( + cluster_name=cluster_name, + aws_region=aws_region, + aws_profile=aws_profile + ) + + self.cluster_modules = {} + self.initialize_cluster_modules() + + if self.all_modules: + self.module_ids = self.cluster_modules.keys() + else: + self.module_ids = module_ids + + def initialize_cluster_modules(self): + modules = self.cluster_config_db.get_cluster_modules() + for module in modules: + self.cluster_modules[module['module_id']] = module + + def get_all_module_ids(self) -> List[str]: + pass + + def deploy_module(self, module_id: str): + module_info = self.cluster_modules[module_id] + module_name = module_info['name'] + print(f'deploying module: {module_name}, module id: {module_id}') + CdkInvoker( + module_id=module_id, + cluster_name=self.cluster_name, + aws_region=self.aws_region, + aws_profile=self.aws_profile, + rollback=self.rollback, + termination_protection=self.termination_protection, + deployment_id=self.deployment_id, + module_set=self.module_set + ).invoke(force_build_bootstrap=self.force_build_bootstrap) + + def get_deployment_order(self) -> List[str]: + + module_deployment_order = [] + + for module_id in self.module_ids: + module_info = self.cluster_modules[module_id] + if module_info is None: + continue + if module_info['type'] == constants.MODULE_TYPE_CONFIG: + continue + if module_info['status'] == 'deployed': + if not self.upgrade: + continue + + module_deployment_order.append({ + 'module_id': module_id, + 'priority': self.module_metadata_helper.get_module_deployment_priority(module_name=module_info['name']) + }) + + module_deployment_order.sort(key=lambda m: m['priority']) + + return [m['module_id'] for m in module_deployment_order] + + def get_optimized_deployment_order(self) -> List[List[str]]: + + deployment_order = self.get_deployment_order() + module_deployments = OrderedDict() + + for module_id in deployment_order: + module_info = self.cluster_modules[module_id] + if module_info is None: + continue + if module_info['type'] == constants.MODULE_TYPE_CONFIG: + continue + if module_info['status'] == 'deployed': + if not self.upgrade: + continue + + priority = self.module_metadata_helper.get_module_deployment_priority(module_name=module_info['name']) + + if priority in module_deployments: + module_deployments[priority].append(module_id) + else: + module_deployments[priority] = [module_id] + + result = [] + for module_set in module_deployments.values(): + result.append(module_set) + return result + + def print_no_op_message(self): + if self.upgrade: + # will this ever be the case ? + print('could not find any modules to upgrade.') + else: + if len(self.module_ids) == 1: + print(f'{self.module_ids[0]} is already deployed. use the --upgrade flag to upgrade or re-deploy the module.') + else: + module_s = ', '.join(self.module_ids) + print(f'[{module_s}] are already deployed. use the --upgrade flag to re-deploy these modules.') + + def invoke(self): + if self.optimize_deployment and len(self.module_ids) > 1: + optimized_deployment_order = self.get_optimized_deployment_order() + if len(optimized_deployment_order) == 0: + self.print_no_op_message() + return + + print(f'optimized deployment order: {optimized_deployment_order}') + for modules in optimized_deployment_order: + threads = [] + for module_id in modules: + thread = threading.Thread( + name=f'Thread: {module_id}', + target=self.deploy_module, + kwargs={'module_id': module_id} + ) + threads.append(thread) + thread.start() + # process the next entry after 10 seconds. + time.sleep(10) + + for thread in threads: + thread.join() + + # check for deployment status of previous modules + # if any of them are not deployed, skip deployment of the next module set + try: + self.initialize_cluster_modules() + except botocore.exceptions.ClientError as e: + if e.response['Error']['Code'] == 'ExpiredTokenException': + # deployment can take sometimes more than an hour and + # credentials can expire for those generated hourly using STS. re-init ClusterConfigDB and boto session + self.cluster_config_db = ClusterConfigDB( + cluster_name=self.cluster_name, + aws_region=self.aws_region, + aws_profile=self.aws_profile + ) + self.initialize_cluster_modules() + else: + raise e + + for module_id in modules: + module_info = self.cluster_modules[module_id] + if module_info['status'] != 'deployed': + raise exceptions.general_exception(f'deployment failed. could not deploy module: {module_id}') + + else: + deployment_order = self.get_deployment_order() + if len(deployment_order) == 0: + self.print_no_op_message() + return + + for module_id in deployment_order: + self.deploy_module(module_id) diff --git a/source/idea/idea-administrator/src/ideaadministrator/app/directory_service_helper.py b/source/idea/idea-administrator/src/ideaadministrator/app/directory_service_helper.py new file mode 100644 index 00000000..cb372772 --- /dev/null +++ b/source/idea/idea-administrator/src/ideaadministrator/app/directory_service_helper.py @@ -0,0 +1,91 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +from ideasdk.utils import Utils +from ideasdk.context import SocaCliContext, SocaContextOptions +from ideadatamodel import constants + +from typing import Dict + + +class DirectoryServiceHelper: + """ + Helper class for Directory Service related functionality. + """ + + def __init__(self, cluster_name: str, aws_region: str, aws_profile: str = None): + self.cluster_name = cluster_name + self.aws_region = aws_region + self.aws_profile = aws_profile + + def create_service_account_secrets(self, username: str, password: str, purpose: str, kms_key_id: str = None) -> Dict: + """ + In order to use an existing AWS Managed Active Directory (or even in case of an On-Prem or Self-Managed AD), + cluster manager needs to have access to the service account credentials. + + this method helps admins to create the service account credentials as secrets in AWS Secrets manager, with + appropriate Tags such that cluster manager is authorized to access these credentials via IAM policies + + Note: + * This method is expected to be invoked prior to creation of the cluster or any configuration updates to cluster config db. + * For that reason, context is not initialized in the constructor and cluster_name is not passed to the context options. + + :param str username: username of the account + :param str password: password of the account + :param str purpose: the purpose/title of the account + :param str kms_key_id: the customer managed kms encryption key that you plan to use for secrets manager + """ + + context = SocaCliContext( + options=SocaContextOptions( + aws_region=self.aws_region, + aws_profile=self.aws_profile, + enable_aws_client_provider=True, + enable_iam_permission_util=True + ) + ) + + tags = Utils.convert_tags_dict_to_aws_tags({ + constants.IDEA_TAG_CLUSTER_NAME: self.cluster_name, + constants.IDEA_TAG_MODULE_NAME: constants.MODULE_DIRECTORYSERVICE, + constants.IDEA_TAG_MODULE_ID: constants.MODULE_DIRECTORYSERVICE + }) + + # create username secret + create_username_secret_request = { + 'Name': f'{self.cluster_name}-{constants.MODULE_DIRECTORYSERVICE}-{purpose}-username', + 'Description': f'DirectoryService {purpose} username, Cluster: {self.cluster_name}', + 'SecretString': username, + 'Tags': tags + } + if Utils.is_not_empty(kms_key_id): + create_username_secret_request['KmsKeyId'] = kms_key_id + # This can produce exception from AWS API for duplicate secret + create_username_secret_result = context.aws().secretsmanager().create_secret(**create_username_secret_request) + username_secret_arn = Utils.get_value_as_string('ARN', create_username_secret_result) + + # create password secret + create_password_secret_request = { + 'Name': f'{self.cluster_name}-{constants.MODULE_DIRECTORYSERVICE}-{purpose}-password', + 'Description': f'DirectoryService {purpose} password, Cluster: {self.cluster_name}', + 'SecretString': password, + 'Tags': tags + } + if Utils.is_not_empty(kms_key_id): + create_password_secret_request['KmsKeyId'] = kms_key_id + # This can produce exception from AWS API for duplicate secret + create_password_secret_result = context.aws().secretsmanager().create_secret(**create_password_secret_request) + password_secret_arn = Utils.get_value_as_string('ARN', create_password_secret_result) + + return { + 'username_secret_arn': username_secret_arn, + 'password_secret_arn': password_secret_arn + } diff --git a/source/idea/idea-administrator/src/ideaadministrator/app/installer_params.py b/source/idea/idea-administrator/src/ideaadministrator/app/installer_params.py new file mode 100644 index 00000000..67749aa4 --- /dev/null +++ b/source/idea/idea-administrator/src/ideaadministrator/app/installer_params.py @@ -0,0 +1,1227 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +__all__ = ( + 'DefaultPrompt', + 'ClusterNamePrompt', + 'AwsProfilePrompt', + 'AwsPartitionPrompt', + 'AwsRegionPrompt', + 'ClientIPPrompt', + 'SSHKeyPairPrompt', + 'VpcCidrBlockPrompt', + 'EmailPrompt', + 'VpcIdPrompt', + 'ExistingResourcesPrompt', + 'SubnetIdsPrompt', + 'FileSystemIdPrompt', + 'OpenSearchDomainEndpointPrompt', + 'DirectoryServiceIdPrompt', + 'PrefixListIdPrompt', + 'EnabledModulesPrompt', + 'MetricsProviderPrompt', + 'SharedStorageProviderPrompt', + 'SharedStorageNamePrompt', + 'SharedStorageVirtualMachineIdPrompt', + 'SharedStorageVolumeIdPrompt', + 'InstallerPromptFactory', + 'QuickSetupPromptFactory', + 'SharedStoragePromptFactory' +) + +from ideasdk.utils import Utils +from ideadatamodel import exceptions, errorcodes, constants +from ideasdk.user_input.framework import ( + SocaPrompt, + SocaUserInputSection, + SocaUserInputModule, + SocaUserInputDefaultType, + SocaUserInputChoicesType, + SocaUserInputChoice, + SocaPromptRegistry, + SocaUserInputParamRegistry, + SocaUserInputArgs +) +from ideadatamodel.user_input import SocaUserInputParamMetadata +from ideasdk.context import SocaCliContext + +import ideaadministrator +from ideaadministrator import app_constants +from ideaadministrator.app_utils import AdministratorUtils +from ideaadministrator.app.aws_service_availability_helper import AwsServiceAvailabilityHelper + +from typing import Optional, List, TypeVar +import validators + +T = TypeVar('T') +SELECT_CHOICE_OTHER = constants.SELECT_CHOICE_OTHER + + +class DefaultPrompt(SocaPrompt[T]): + + def __init__(self, factory: 'InstallerPromptFactory', + param: SocaUserInputParamMetadata, + default: SocaUserInputDefaultType = None, + choices: SocaUserInputChoicesType = None): + if default is None: + default = self.get_default + self.factory = factory + super().__init__(context=factory.context, + args=factory.args, + param=param, + default=default, + choices=choices, + registry=factory.install_prompts) + self._args = factory.args + self.context = factory.context + + @property + def args(self) -> SocaUserInputArgs: + return self._args + + def get_default(self, reset: bool = False) -> T: + if reset: + return None + else: + return self.args.get(self.name) + + def validate(self, value: T): + super().validate(value) + + +class ClusterNamePrompt(DefaultPrompt[str]): + + def __init__(self, factory: 'InstallerPromptFactory'): + super().__init__(factory=factory, + param=factory.args.get_meta('cluster_name')) + + def validate(self, value: str): + + super().validate(value) + + # validate is not called for choice. so we are good here ... + token = AdministratorUtils.sanitize_cluster_name(value) + if Utils.is_empty(token): + return + + if token == f'{app_constants.CLUSTER_NAME_PREFIX}-{SELECT_CHOICE_OTHER}': + raise self.validation_error(message=f'Invalid ClusterName: {token}. ' + f'"{SELECT_CHOICE_OTHER}" is a reserved keyword and cannot be used in ClusterName.') + + if 'idea' in token.replace('idea-', '', 1): + raise self.validation_error(message=f'Invalid ClusterName: {token}. ' + f'"{token}" contains value "idea" and is not allowed.') + + min_length, max_length = app_constants.CLUSTER_NAME_MIN_MAX_LENGTH + if not min_length <= len(token) <= max_length: + raise self.validation_error(message=f'ClusterName: ({token}) length must be between {min_length} and {max_length} characters. ' + f'Current: {len(token)}') + + regenerate = Utils.get_as_bool(self.args.get('_regenerate'), False) + if regenerate: + return + + vpcs = Utils.get_as_list(self.context.get_aws_resources().get_vpcs(), []) + for vpc in vpcs: + if vpc.cluster_name is None: + continue + if vpc.cluster_name == token: + raise self.validation_error(message=f'Cluster: ({token}) already exists and is in use by Vpc: {vpc.vpc_id}. ' + f'Please enter a different cluster name.') + + bootstrap_stack_name = f'{token}-bootstrap' + existing_bootstrap_stack = self.context.aws_util().cloudformation_describe_stack(bootstrap_stack_name) + if existing_bootstrap_stack is not None: + raise self.validation_error(message=f'Cluster: ({token}) already exists and is in use by Bootstrap Stack: {bootstrap_stack_name}. ' + f'Please enter a different cluster name.') + + def filter(self, value: str) -> Optional[str]: + token = AdministratorUtils.sanitize_cluster_name(value) + cluster_name = super().filter(token) + return cluster_name + + def get_choices(self, refresh: bool = False) -> List[SocaUserInputChoice]: + installed_clusters = self.context.get_aws_resources().list_available_cluster_names() + choices = [] + for cluster_name in installed_clusters: + choices.append(SocaUserInputChoice(title=cluster_name, value=cluster_name)) + return choices + + +class AwsProfilePrompt(DefaultPrompt[str]): + + def __init__(self, factory: 'InstallerPromptFactory'): + super().__init__(factory=factory, + param=factory.args.get_meta('aws_profile')) + + def get_choices(self, refresh: bool = False) -> List[SocaUserInputChoice]: + profiles = self.context.get_aws_resources().get_aws_profiles(refresh) + choices = [] + for profile in profiles: + choices.append(SocaUserInputChoice(title=profile.title, value=profile.name)) + return choices + + +# In some situations a secondary AWS profile may be needed. +# This can include partitions (GovCloud) that do not support all +# services or functions. +class AwsProfilePrompt2(DefaultPrompt[str]): + def __init__(self, factory: 'InstallerPromptFactory'): + super().__init__(factory=factory, + param=factory.args.get_meta('aws_profile2')) + + def get_choices(self, refresh: bool = False) -> List[SocaUserInputChoice]: + profiles = self.context.get_aws_resources().get_aws_profiles(refresh) + choices = [] + for profile in profiles: + choices.append(SocaUserInputChoice(title=profile.title, value=profile.name)) + return choices + + +class AwsPartitionPrompt(DefaultPrompt[str]): + + def __init__(self, factory: 'InstallerPromptFactory'): + super().__init__(factory=factory, + param=factory.args.get_meta('aws_partition')) + + def get_choices(self, refresh: bool = False) -> List[SocaUserInputChoice]: + partitions = self.context.aws().aws_endpoints().list_partitions() + choices = [] + for partition in partitions: + title = f'{partition.name} [{partition.partition}]' + choices.append(SocaUserInputChoice(title=title, value=partition.partition)) + return choices + + +class AwsRegionPrompt(DefaultPrompt[str]): + + def __init__(self, factory: 'InstallerPromptFactory'): + super().__init__(factory=factory, + param=factory.args.get_meta('aws_region'), + default=self.get_default) + + def get_partition(self) -> Optional[str]: + partition_prompt = self.factory.install_prompts.get('aws_partition') + return partition_prompt.default() + + def get_default(self, reset: bool = False) -> Optional[str]: + region = self.args.get('aws_region') + + partition = self.context.aws().aws_endpoints().get_partition(partition=self.get_partition()) + regions = partition.regions + for region_ in regions: + if region_.region == region: + return region + + partition = self.get_partition() + + custom = self.param.custom + defaults = Utils.get_value_as_dict('defaults', custom) + + return Utils.get_value_as_string(partition, defaults) + + def get_choices(self, refresh: bool = False) -> List[SocaUserInputChoice]: + partition = self.context.aws().aws_endpoints().get_partition(partition=self.get_partition()) + regions = partition.regions + choices = [] + for region in regions: + title = f'{region.name} [{region.region}]' + choices.append(SocaUserInputChoice(title=title, value=region.region)) + return choices + + +class ClientIPPrompt(DefaultPrompt[str]): + + def __init__(self, factory: 'InstallerPromptFactory'): + super().__init__(factory, + param=factory.args.get_meta('client_ip'), + default=self.get_default) + + def get_default(self, reset: bool = False) -> Optional[str]: + if not reset: + client_ip = self.args.get('client_ip') + if client_ip: + return client_ip + return AdministratorUtils.detect_client_ip() + + def filter(self, value: str) -> List[str]: + tokens = value.split(',') + result = [] + for token in tokens: + if Utils.is_empty(token): + continue + if '/' not in token: + token = f'{token}/32' + result.append(token.strip()) + super().filter(result) + return result + + +class SSHKeyPairPrompt(DefaultPrompt[str]): + + def __init__(self, factory: 'InstallerPromptFactory', param: SocaUserInputParamMetadata): + super().__init__(factory=factory, param=param) + + def get_choices(self, refresh: bool = False) -> List[SocaUserInputChoice]: + result = [] + + key_pairs = self.context.get_aws_resources().get_ec2_key_pairs(refresh) + if key_pairs is None: + key_pairs = [] + + if len(key_pairs) == 0: + raise exceptions.soca_exception(error_code=errorcodes.USER_INPUT_FLOW_INTERRUPT, + message='Unable to find any existing EC2 SSH Key Pairs in this region.' + 'Create a new EC2 Key Pair from the AWS Console and re-run idea-admin.') + + for key_pair in key_pairs: + result.append(SocaUserInputChoice( + title=key_pair.key_name, + value=key_pair.key_name + )) + + return result + + +class VpcCidrBlockPrompt(DefaultPrompt[str]): + + def __init__(self, factory: 'InstallerPromptFactory'): + super().__init__(factory=factory, + param=factory.args.get_meta('vpc_cidr_block'), + default=self.get_default) + + def get_default(self, reset: bool = False) -> Optional[str]: + vpcs = self.context.get_aws_resources().get_vpcs() + if len(vpcs) == 0: + return self.param.get_default() + + cidr_blocks = {} + for vpc in vpcs: + cidr_blocks[vpc.cidr_block] = True + + for i in range(0, 255): + candidate = f'10.{i}.0.0/16' + if candidate in cidr_blocks: + continue + return candidate + + def validate(self, value: str): + + super().validate(value) + + vpcs = self.context.get_aws_resources().get_vpcs() + if len(vpcs) == 0: + return + + regenerate = Utils.get_as_bool(self.args.get('_regenerate'), False) + if regenerate: + return + + for vpc in vpcs: + if value == vpc.cidr_block: + raise self.validation_error( + message=f'VPC CIDR Block: {value} is already used by VPC: {vpc.vpc_id}. Please enter a different CIDR block ' + f'to avoid IP Address conflicts between VPCs.' + ) + + +class EmailPrompt(DefaultPrompt[str]): + + def __init__(self, factory: 'InstallerPromptFactory', param: SocaUserInputParamMetadata): + super().__init__(factory=factory, + param=param) + + def validate(self, value: str): + super().validate(value) + if not validators.email(value): + raise self.validation_error('Invalid Email Address') + + +class VpcIdPrompt(DefaultPrompt[str]): + + def __init__(self, factory: 'InstallerPromptFactory'): + super().__init__(factory=factory, + param=factory.args.get_meta('vpc_id')) + + def get_choices(self, refresh: bool = False) -> List[SocaUserInputChoice]: + vpcs = self.context.get_aws_resources().get_vpcs(refresh) + + if len(vpcs) == 0: + raise exceptions.general_exception('Unable to find any existing VPC in this region.') + + choices = [] + + for vpc in vpcs: + choices.append(SocaUserInputChoice( + title=f'{vpc.title}', + value=vpc.vpc_id + )) + + return choices + + def filter(self, value) -> Optional[T]: + self.args.set('use_existing_vpc', True) + return super().filter(value) + + +class ExistingResourcesPrompt(DefaultPrompt[str]): + + def __init__(self, factory: 'InstallerPromptFactory'): + super().__init__(factory=factory, + param=factory.args.get_meta('existing_resources'), + default=self.get_default) + self.existing_file_systems = False + self.existing_opensearch = False + self.existing_subnets = False + self.existing_directories = False + + self.vpc_id = None + + def get_default(self, reset: bool = False) -> Optional[List[str]]: + result = [] + if self.existing_subnets: + result.append('subnets:public') + result.append('subnets:private') + if self.existing_file_systems: + result.append('shared-storage:apps') + result.append('shared-storage:data') + if self.existing_opensearch: + result.append('analytics:opensearch') + if self.existing_directories: + result.append('directoryservice:aws_managed_activedirectory') + return result + + def validate(self, value: List[str]): + if Utils.is_empty(value): + raise self.validation_error( + message='Existing resource selection is required' + ) + if 'subnets:public' not in value and 'subnets:private' not in value: + raise self.validation_error( + message='Either one of [Subnets: Public, Subnets: Private] is required' + ) + + def get_choices(self, refresh: bool = False) -> List[SocaUserInputChoice]: + + vpc_id = self.args.get('vpc_id') + if Utils.is_empty(vpc_id): + raise exceptions.general_exception('vpc_id is required to find existing resources') + + alb_public = Utils.get_as_bool(self.args.get('alb_public'), default=True) + + if Utils.is_not_empty(self.vpc_id): + if self.vpc_id != vpc_id: + refresh = True + + self.vpc_id = vpc_id + + with self.context.spinner('search for existing subnets ...'): + subnets = self.context.get_aws_resources().get_subnets(vpc_id=vpc_id, refresh=refresh) + self.existing_subnets = len(subnets) > 0 + with self.context.spinner('search for existing file systems ...'): + file_systems = self.context.get_aws_resources().get_file_systems(vpc_id=vpc_id, refresh=refresh) + self.existing_file_systems = len(file_systems) > 0 + with self.context.spinner('search for existing opensearch clusters ...'): + opensearch_clusters = self.context.get_aws_resources().get_opensearch_clusters(vpc_id=vpc_id, refresh=refresh) + self.existing_opensearch = len(opensearch_clusters) > 0 + with self.context.spinner('search for existing directories ...'): + directories = self.context.get_aws_resources().get_directories(vpc_id=vpc_id, refresh=refresh) + directory_service_provider = self.args.get('directory_service_provider') + self.existing_directories = len(directories) > 0 and directory_service_provider == constants.DIRECTORYSERVICE_AWS_MANAGED_ACTIVE_DIRECTORY + + choices = [] + if self.existing_subnets: + if alb_public: + choices.append(SocaUserInputChoice( + title='Subnets: Public', + value='subnets:public' + )) + choices.append(SocaUserInputChoice( + title='Subnets: Private', + value='subnets:private' + )) + if self.existing_file_systems: + choices.append(SocaUserInputChoice( + title='Shared Storage: Apps', + value='shared-storage:apps' + )) + choices.append(SocaUserInputChoice( + title='Shared Storage: Data', + value='shared-storage:data' + )) + if self.existing_opensearch: + choices.append(SocaUserInputChoice( + title='Analytics: OpenSearch Clusters', + value='analytics:opensearch' + )) + if self.existing_directories: + choices.append(SocaUserInputChoice( + title='Directory: AWS Managed Microsoft AD', + value='directoryservice:aws_managed_activedirectory' + )) + return choices + + +class SubnetIdsPrompt(DefaultPrompt[str]): + + def __init__(self, factory: 'InstallerPromptFactory', param: SocaUserInputParamMetadata): + super().__init__(factory=factory, + param=param) + + def validate(self, value: List[str]): + + vpc_id = self.args.get('vpc_id') + if Utils.is_empty(vpc_id): + raise exceptions.general_exception('vpc_id is required to find existing resources') + subnets = self.context.get_aws_resources().get_subnets(vpc_id=vpc_id) + selected_subnets = [] + for subnet in subnets: + if subnet.subnet_id in value: + selected_subnets.append(subnet) + + if self.param.name == 'private_subnet_ids': + subnet_ids_alt = self.args.get('public_subnet_ids') + subnet_ids_alt_title = 'public subnets' + else: + subnet_ids_alt = self.args.get('private_subnet_ids') + subnet_ids_alt_title = 'private subnets' + + selected_azs = [] + for subnet in selected_subnets: + if subnet.availability_zone in selected_azs: + raise self.validation_error( + message='Multiple subnet selection from the same Availability Zone is not supported.' + ) + selected_azs.append(subnet.availability_zone) + + if Utils.is_not_empty(subnet_ids_alt): + if subnet.subnet_id in subnet_ids_alt: + raise self.validation_error( + message=f'SubnetId: {subnet.subnet_id} is already selected as part of {subnet_ids_alt_title} selection.' + ) + + if self.param.name == 'private_subnet_ids' and len(selected_subnets) < 2: + raise self.validation_error( + message=f'Minimum 2 subnet selections are required to ensure high availability.' + ) + + def get_choices(self, refresh: bool = False) -> List[SocaUserInputChoice]: + + vpc_id = self.args.get('vpc_id') + if Utils.is_empty(vpc_id): + raise exceptions.general_exception('vpc_id is required to find existing resources') + + subnets = self.context.get_aws_resources().get_subnets(vpc_id=vpc_id, refresh=refresh) + + if len(subnets) == 0: + raise exceptions.general_exception(f'Unable to find any existing subnets for vpc: {vpc_id}') + + choices = [] + + for subnet in subnets: + choices.append(SocaUserInputChoice( + title=subnet.title, + value=subnet.subnet_id + )) + + return choices + + +class FileSystemIdPrompt(DefaultPrompt[str]): + + def __init__(self, factory: 'InstallerPromptFactory', param: SocaUserInputParamMetadata, storage_provider_key: str, existing_storage_flag_key: Optional[str] = None): + super().__init__(factory=factory, + param=param) + self.existing_storage_flag_key = existing_storage_flag_key + self.storage_provider_key = storage_provider_key + + def get_choices(self, refresh: bool = False) -> List[SocaUserInputChoice]: + + vpc_id = self.args.get('vpc_id') + if Utils.is_empty(vpc_id): + raise exceptions.general_exception('vpc_id is required to find existing resources') + + with self.context.spinner(f'checking available file systems in Vpc: {vpc_id} ...'): + file_systems = self.context.get_aws_resources().get_file_systems(vpc_id=vpc_id, refresh=refresh) + + storage_provider = self.args.get(self.storage_provider_key) + if Utils.is_empty(storage_provider): + raise exceptions.general_exception(f'{self.storage_provider_key} is required to find existing file systems') + + filtered_file_systems = [] + for file_system in file_systems: + if file_system.provider == storage_provider: + filtered_file_systems.append(file_system) + + if len(filtered_file_systems) == 0: + raise exceptions.soca_exception( + error_code=errorcodes.USER_INPUT_FLOW_INTERRUPT, + message=f'Unable to find any existing file systems for provider: {storage_provider}' + ) + + choices = [] + + for file_system in filtered_file_systems: + choices.append(SocaUserInputChoice( + title=file_system.title, + value=file_system.file_system_id + )) + + return choices + + def filter(self, value) -> Optional[T]: + if self.existing_storage_flag_key is not None: + self.args.set(self.existing_storage_flag_key, True) + return super().filter(value) + + +class SharedStorageVirtualMachineIdPrompt(DefaultPrompt[str]): + + def __init__(self, factory: 'InstallerPromptFactory', param: SocaUserInputParamMetadata, file_system_id_key: str): + super().__init__(factory=factory, + param=param) + self.file_system_id_key = file_system_id_key + + def get_choices(self, refresh: bool = False) -> List[SocaUserInputChoice]: + + vpc_id = self.args.get('vpc_id') + if Utils.is_empty(vpc_id): + raise exceptions.general_exception('vpc_id is required to find existing resources') + + file_system_id = self.args.get(self.file_system_id_key) + if Utils.is_empty(file_system_id): + raise exceptions.general_exception('file_system_id is required to find storage virtual machines') + + file_systems = self.context.get_aws_resources().get_file_systems(vpc_id=vpc_id, refresh=refresh) + + target_file_system = None + + for file_system in file_systems: + if file_system.file_system_id == file_system_id: + target_file_system = file_system + break + + if target_file_system is None: + raise exceptions.general_exception(f'Unable to find any existing file systems for file_system_id: {file_system_id}') + + choices = [] + + storage_virtual_machines = target_file_system.get_storage_virtual_machines() + for svm_entry in storage_virtual_machines: + svm_id = target_file_system.get_svm_id(svm_entry) + title = target_file_system.get_storage_virtual_machine_title(svm_entry) + choices.append(SocaUserInputChoice( + title=title, + value=svm_id + )) + + return choices + + +class SharedStorageVolumeIdPrompt(DefaultPrompt[str]): + + def __init__(self, factory: 'InstallerPromptFactory', param: SocaUserInputParamMetadata, file_system_id_key: str, svm_id_key: str = None): + super().__init__(factory=factory, + param=param) + self.file_system_id_key = file_system_id_key + self.svm_id_key = svm_id_key + + def get_choices(self, refresh: bool = False) -> List[SocaUserInputChoice]: + + vpc_id = self.args.get('vpc_id') + if Utils.is_empty(vpc_id): + raise exceptions.general_exception('vpc_id is required to find existing resources') + + file_system_id = self.args.get(self.file_system_id_key) + if Utils.is_empty(file_system_id): + raise exceptions.general_exception('file_system_id is required to find storage volumes') + + svm_id = None + if Utils.is_not_empty(self.svm_id_key): + svm_id = self.args.get(self.svm_id_key) + if Utils.is_empty(svm_id): + raise exceptions.general_exception('svm_id is required to find storage volumes') + + file_systems = self.context.get_aws_resources().get_file_systems(vpc_id=vpc_id, refresh=refresh) + + target_file_system = None + + for file_system in file_systems: + if file_system.file_system_id == file_system_id: + target_file_system = file_system + break + + if target_file_system is None: + raise exceptions.general_exception(f'Unable to find any existing file systems for file_system_id: {file_system_id}') + + choices = [] + + volumes = target_file_system.get_volumes() + for volume_entry in volumes: + volume_id = target_file_system.get_volume_id(volume_entry) + title = target_file_system.get_volume_title(volume_entry) + if Utils.is_not_empty(svm_id): + volume_svm_id = target_file_system.get_volume_svm_id(volume_entry) + if Utils.is_empty(volume_svm_id): + continue + if volume_svm_id != svm_id: + continue + choices.append(SocaUserInputChoice( + title=title, + value=volume_id + )) + + return choices + + +class OpenSearchDomainEndpointPrompt(DefaultPrompt[str]): + + def __init__(self, factory: 'InstallerPromptFactory'): + super().__init__(factory=factory, + param=factory.args.get_meta('opensearch_domain_endpoint')) + + def get_choices(self, refresh: bool = False) -> List[SocaUserInputChoice]: + + vpc_id = self.args.get('vpc_id') + if Utils.is_empty(vpc_id): + raise exceptions.general_exception('vpc_id is required to find existing resources') + + opensearch_clusters = self.context.get_aws_resources().get_opensearch_clusters(vpc_id=vpc_id, refresh=refresh) + + if len(opensearch_clusters) == 0: + raise exceptions.general_exception(f'Unable to find any existing opensearch clusters') + + choices = [] + + for opensearch_cluster in opensearch_clusters: + choices.append(SocaUserInputChoice( + title=opensearch_cluster.title, + value=opensearch_cluster.vpc_endpoint + )) + + return choices + + def filter(self, value) -> Optional[T]: + self.args.set('use_existing_opensearch_cluster', True) + return super().filter(value) + + +class DirectoryServiceIdPrompt(DefaultPrompt[str]): + + def __init__(self, factory: 'InstallerPromptFactory'): + super().__init__(factory=factory, + param=factory.args.get_meta('directory_id')) + + def get_choices(self, refresh: bool = False) -> List[SocaUserInputChoice]: + + vpc_id = self.args.get('vpc_id') + if Utils.is_empty(vpc_id): + raise exceptions.general_exception('vpc_id is required to find existing resources') + + directories = self.context.get_aws_resources().get_directories(vpc_id=vpc_id, refresh=refresh) + + if len(directories) == 0: + raise exceptions.general_exception(f'Unable to find any existing directories') + + choices = [] + + for directory in directories: + choices.append(SocaUserInputChoice( + title=directory.title, + value=directory.directory_id + )) + + return choices + + def filter(self, value) -> Optional[T]: + self.args.set('use_existing_directory_service', True) + return super().filter(value) + + +class PrefixListIdPrompt(DefaultPrompt[str]): + + def __init__(self, factory: 'InstallerPromptFactory'): + super().__init__(factory=factory, + param=factory.args.get_meta('prefix_list_ids')) + + def get_choices(self, refresh: bool = False) -> List[SocaUserInputChoice]: + prefix_lists = self.context.get_aws_resources().get_ec2_prefix_lists(refresh) + + choices = [] + + for prefix_list in prefix_lists: + choices.append(SocaUserInputChoice( + title=prefix_list.title, + value=prefix_list.prefix_list_id + )) + + return choices + + def filter(self, value: str) -> Optional[List[str]]: + tokens = value.split(',') + result = [] + for token in tokens: + if Utils.is_empty(token): + continue + result.append(token.strip()) + return super().filter(result) + + +class EnabledModulesPrompt(DefaultPrompt[str]): + + def __init__(self, factory: 'InstallerPromptFactory'): + super().__init__(factory=factory, + param=factory.args.get_meta('enabled_modules')) + + def get_choices(self, refresh: bool = False) -> List[SocaUserInputChoice]: + result = [ + SocaUserInputChoice( + title='Global Settings (required)', + value=constants.MODULE_GLOBAL_SETTINGS, + checked=True, + disabled=True + ), + SocaUserInputChoice( + title='Cluster (required)', + value=constants.MODULE_CLUSTER, + checked=True, + disabled=True + ), + SocaUserInputChoice( + title='Analytics (required)', + value=constants.MODULE_ANALYTICS, + checked=True, + disabled=True + ), + SocaUserInputChoice( + title='Identity Provider (required)', + value=constants.MODULE_IDENTITY_PROVIDER, + checked=True, + disabled=True + ), + SocaUserInputChoice( + title='Directory Service (required)', + value=constants.MODULE_DIRECTORYSERVICE, + checked=True, + disabled=True + ), + SocaUserInputChoice( + title='Shared Storage (required)', + value=constants.MODULE_SHARED_STORAGE, + checked=True, + disabled=True + ), + SocaUserInputChoice( + title='Cluster Manager (required)', + value=constants.MODULE_CLUSTER_MANAGER, + checked=True, + disabled=True + ), + SocaUserInputChoice( + title='Metrics and Monitoring', + value=constants.MODULE_METRICS + ), + SocaUserInputChoice( + title='Scale-out Computing on AWS (SOCA) for HPC', + value=constants.MODULE_SCHEDULER + ), + SocaUserInputChoice( + title='Enterprise Virtual Desktop Infrastructure (eVDI)', + value=constants.MODULE_VIRTUAL_DESKTOP_CONTROLLER + ), + SocaUserInputChoice( + title='Bastion Host', + value=constants.MODULE_BASTION_HOST + ) + ] + return result + + +class MetricsProviderPrompt(DefaultPrompt[str]): + + def __init__(self, factory: 'InstallerPromptFactory'): + super().__init__(factory=factory, + param=factory.args.get_meta('metrics_provider')) + + def get_choices(self, refresh: bool = False) -> List[SocaUserInputChoice]: + aws_region = self.args.get('aws_region') + if Utils.is_empty(aws_region): + raise exceptions.general_exception('aws_region is required to identify available metrics providers.') + + aws_profile = self.args.get('aws_profile') + aws_secondary_profile = self.args.get('aws_profile2') + + profile_string = f" {aws_profile}" + if aws_secondary_profile: + profile_string = f"s (primary: {aws_profile}, secondary: {aws_secondary_profile})" + + with self.context.spinner(f'checking available services in region: {aws_region} using AWS Profile{profile_string} ...'): + availability_helper = AwsServiceAvailabilityHelper(aws_region=aws_region, aws_profile=aws_profile, aws_secondary_profile=aws_secondary_profile) + available_services = availability_helper.get_available_services(aws_region) + + result = [ + SocaUserInputChoice( + title='AWS CloudWatch', + value=constants.METRICS_PROVIDER_CLOUDWATCH, + disabled='cloudwatch' not in available_services + ), + SocaUserInputChoice( + title='Amazon Managed Service for Prometheus', + value=constants.METRICS_PROVIDER_AMAZON_MANAGED_PROMETHEUS, + disabled='aps' not in available_services + ), + SocaUserInputChoice( + title='Custom Prometheus Server', + value=constants.METRICS_PROVIDER_PROMETHEUS + ) + ] + + return result + + +class SharedStorageNamePrompt(DefaultPrompt[str]): + + def __init__(self, factory: 'InstallerPromptFactory'): + super().__init__(factory=factory, + param=factory.args.get_meta('shared_storage_name')) + + def validate(self, value: str): + is_upgrade = Utils.get_as_bool(self.args.get('_upgrade'), default=False) + if is_upgrade: + existing_config = self.context.config().get_config(f'shared-storage.{value}', default=None) + if existing_config is not None: + raise self.validation_error(f'A file system named: {value} already exists in shared storage settings') + + super().validate(value) + + +class SharedStorageProviderPrompt(DefaultPrompt[str]): + + def __init__(self, factory: 'InstallerPromptFactory'): + super().__init__(factory=factory, + param=factory.args.get_meta('shared_storage_provider')) + + def get_choices(self, refresh: bool = False) -> List[SocaUserInputChoice]: + use_existing_fs = Utils.get_as_bool(self.args.get('use_existing_fs'), False) + + if use_existing_fs: + + vpc_id = self.args.get('vpc_id') + if Utils.is_empty(vpc_id): + raise exceptions.general_exception('vpc_id is required to find existing resources') + + with self.context.spinner(f'checking available file systems in Vpc: {vpc_id} ...'): + file_systems = self.context.get_aws_resources().get_file_systems(vpc_id=vpc_id, refresh=refresh) + + if len(file_systems) == 0: + raise exceptions.soca_exception( + error_code=errorcodes.USER_INPUT_FLOW_INTERRUPT, + message=f'Unable to find any existing file systems in VPC: {vpc_id}' + ) + + providers = set() + for file_system in file_systems: + providers.add(file_system.provider) + + return [ + SocaUserInputChoice( + title='Amazon EFS', + value=constants.STORAGE_PROVIDER_EFS, + disabled=constants.STORAGE_PROVIDER_EFS not in providers + ), + SocaUserInputChoice( + title='Amazon File Cache', + value=constants.STORAGE_PROVIDER_FSX_CACHE, + disabled=constants.STORAGE_PROVIDER_FSX_CACHE not in providers + ), + SocaUserInputChoice( + title='Amazon FSx for Lustre', + value=constants.STORAGE_PROVIDER_FSX_LUSTRE, + disabled=constants.STORAGE_PROVIDER_FSX_LUSTRE not in providers + ), + SocaUserInputChoice( + title='Amazon FSx for NetApp ONTAP', + value=constants.STORAGE_PROVIDER_FSX_NETAPP_ONTAP, + disabled=constants.STORAGE_PROVIDER_FSX_NETAPP_ONTAP not in providers + ), + SocaUserInputChoice( + title='Amazon FSx for OpenZFS', + value=constants.STORAGE_PROVIDER_FSX_OPENZFS, + disabled=constants.STORAGE_PROVIDER_FSX_OPENZFS not in providers + ), + SocaUserInputChoice( + title='Amazon FSx for Windows File Server', + value=constants.STORAGE_PROVIDER_FSX_WINDOWS_FILE_SERVER, + disabled=constants.STORAGE_PROVIDER_FSX_WINDOWS_FILE_SERVER not in providers + ) + ] + + else: + + return [ + SocaUserInputChoice( + title='Amazon EFS', + value=constants.STORAGE_PROVIDER_EFS + ) + ] + + +class InstallerPromptFactory: + """ + Base class for prompt initialization + """ + + def __init__(self, context: SocaCliContext, + param_registry: SocaUserInputParamRegistry, + args: SocaUserInputArgs): + + self.context = context + self.args = args + self.param_registry = param_registry + + self._install_prompts = SocaPromptRegistry( + context=self.context, + param_spec=self.param_registry + ) + + @property + def install_prompts(self) -> SocaPromptRegistry: + return self._install_prompts + + def register(self, prompt: SocaPrompt): + self.install_prompts.add(prompt) + + def initialize_prompts(self, user_input_module: str): + params = self.param_registry.get_params(module=user_input_module) + for param_meta in params: + self.build_prompt(param_meta) + + def build_prompt(self, param_meta: SocaUserInputParamMetadata): + prompt = None + try: + prompt = self.install_prompts.get(param_meta.name) + except exceptions.SocaException as e: + if e.error_code == errorcodes.INPUT_PROMPT_NOT_FOUND: + pass + else: + raise e + + if prompt is None: + prompt = DefaultPrompt( + factory=self, + param=param_meta + ) + return prompt + + def build_section(self, module: str, section: str) -> SocaUserInputSection: + + section_meta = self.param_registry.get_section(module, section) + section_params = self.param_registry.get_params(module, section) + section_prompts = [] + + for param_meta in section_params: + + if section == 'aws-account' and ideaadministrator.props.use_ec2_instance_metadata_credentials(): + # if administrator app is running inside EC2 Instance with InstanceMetadata credentials, + # skip profile selection prompt + if param_meta.name == 'aws_profile': + continue + + prompt = self.install_prompts.get(param_meta.name) + prompt.param = param_meta + section_prompts.append(prompt) + + return SocaUserInputSection( + context=self.context, + section=section_meta, + prompts=section_prompts + ) + + @staticmethod + def section_callback(context: ideaadministrator.Context, section: SocaUserInputSection, factory: 'InstallerPromptFactory') -> bool: + """ + the section call back is called by SocaUserInputModule after each section is completed. + + upon completion of the aws "aws-account" section, we re-initialize the aws clients and dependencies, + so that additional API calls to AWS will use the updated profile and region. + + :param context: + :param section: + :param factory: + :return: + """ + + if section.name == 'aws-account': + + aws_region = factory.args.get('aws_region') + + if ideaadministrator.props.use_ec2_instance_metadata_credentials(): + factory.args.set('aws_profile', None) + context.aws_init(aws_profile=None, aws_region=aws_region) + else: + aws_profile = factory.args.get('aws_profile') + context.aws_init(aws_profile=aws_profile, aws_region=aws_region) + + factory.args.set('aws_partition', context.aws().aws_partition()) + factory.args.set('aws_account_id', context.aws().aws_account_id()) + factory.args.set('aws_dns_suffix', context.aws().aws_dns_suffix()) + + return True + + def build_module(self, user_input_module: str, start_section: int = None, display_info: bool = True, display_steps: bool = True) -> SocaUserInputModule: + + self.initialize_prompts(user_input_module) + + module_meta = self.param_registry.get_module(user_input_module) + + sections = [] + if Utils.is_not_empty(module_meta.sections): + for section_meta in module_meta.sections: + section = self.build_section(user_input_module, section_meta.name) + sections.append(section) + + return SocaUserInputModule( + context=self.context, + module=module_meta, + section_callback=self.section_callback, + section_callback_kwargs={ + 'factory': self + }, + sections=sections, + restart_errorcodes=[errorcodes.INSTALLER_MISSING_PERMISSIONS], + display_info=display_info, + display_steps=display_steps, + start_section=start_section + ) + + +class QuickSetupPromptFactory(InstallerPromptFactory): + """ + Initialize Prompts for Quick Setup Installation Flows + """ + + def __init__(self, context: SocaCliContext, + param_registry: SocaUserInputParamRegistry, + args: SocaUserInputArgs): + super().__init__(context, param_registry, args) + + def initialize_prompts(self, user_input_module: str): + self.initialize_overrides() + super().initialize_prompts(user_input_module) + + def initialize_overrides(self): + self.register(ClusterNamePrompt(factory=self)) + self.register(EmailPrompt(factory=self, param=self.args.get_meta('administrator_email'))) + self.register(AwsProfilePrompt(factory=self)) + self.register(AwsPartitionPrompt(factory=self)) + self.register(AwsRegionPrompt(factory=self)) + self.register(ClientIPPrompt(factory=self)) + self.register(PrefixListIdPrompt(factory=self)) + self.register(SSHKeyPairPrompt(factory=self, param=self.args.get_meta('ssh_key_pair_name'))) + self.register(EnabledModulesPrompt(factory=self)) + self.register(MetricsProviderPrompt(factory=self)) + self.register(VpcCidrBlockPrompt(factory=self)) + self.register(VpcIdPrompt(factory=self)) + self.register(ExistingResourcesPrompt(factory=self)) + self.register(SubnetIdsPrompt(factory=self, param=self.args.get_meta('private_subnet_ids'))) + self.register(SubnetIdsPrompt(factory=self, param=self.args.get_meta('public_subnet_ids'))) + self.register(FileSystemIdPrompt( + factory=self, + param=self.args.get_meta('existing_apps_fs_id'), + existing_storage_flag_key='use_existing_apps_fs', + storage_provider_key='storage_apps_provider' + )) + self.register(FileSystemIdPrompt( + factory=self, + param=self.args.get_meta('existing_data_fs_id'), + existing_storage_flag_key='use_existing_data_fs', + storage_provider_key='storage_data_provider' + )) + self.register(OpenSearchDomainEndpointPrompt(factory=self)) + self.register(DirectoryServiceIdPrompt(factory=self)) + + +class SharedStoragePromptFactory(InstallerPromptFactory): + """ + Initialize Prompts for Shared Storage Configuration + """ + + def __init__(self, context: SocaCliContext, + param_registry: SocaUserInputParamRegistry, + args: SocaUserInputArgs): + super().__init__(context, param_registry, args) + + def initialize_prompts(self, user_input_module: str): + self.initialize_overrides() + super().initialize_prompts(user_input_module) + + def initialize_overrides(self): + self.register(VpcIdPrompt(factory=self)) + self.register(SharedStorageNamePrompt(factory=self)) + self.register(SharedStorageProviderPrompt(factory=self)) + + # efs + self.register(FileSystemIdPrompt( + factory=self, + param=self.args.get_meta('efs.file_system_id'), + storage_provider_key='shared_storage_provider' + )) + + # Amazon File Cache + self.register(FileSystemIdPrompt( + factory=self, + param=self.args.get_meta('fsx_cache.file_system_id'), + storage_provider_key='shared_storage_provider' + )) + + # lustre + self.register(FileSystemIdPrompt( + factory=self, + param=self.args.get_meta('fsx_lustre.file_system_id'), + storage_provider_key='shared_storage_provider' + )) + + # ontap + self.register(FileSystemIdPrompt( + factory=self, + param=self.args.get_meta('fsx_netapp_ontap.file_system_id'), + storage_provider_key='shared_storage_provider' + )) + self.register(SharedStorageVirtualMachineIdPrompt( + factory=self, + param=self.args.get_meta('fsx_netapp_ontap.svm_id'), + file_system_id_key='fsx_netapp_ontap.file_system_id' + )) + self.register(SharedStorageVolumeIdPrompt( + factory=self, + param=self.args.get_meta('fsx_netapp_ontap.volume_id'), + file_system_id_key='fsx_netapp_ontap.file_system_id', + svm_id_key='fsx_netapp_ontap.svm_id' + )) + + # openzfs + self.register(FileSystemIdPrompt( + factory=self, + param=self.args.get_meta('fsx_openzfs.file_system_id'), + storage_provider_key='shared_storage_provider' + )) + self.register(SharedStorageVolumeIdPrompt( + factory=self, + param=self.args.get_meta('fsx_openzfs.volume_id'), + file_system_id_key='fsx_openzfs.file_system_id' + )) + + # Windows file server + self.register(FileSystemIdPrompt( + factory=self, + param=self.args.get_meta('fsx_windows_file_server.file_system_id'), + storage_provider_key='shared_storage_provider' + )) diff --git a/source/idea/idea-administrator/src/ideaadministrator/app/patch_helper.py b/source/idea/idea-administrator/src/ideaadministrator/app/patch_helper.py new file mode 100644 index 00000000..ea6d1b74 --- /dev/null +++ b/source/idea/idea-administrator/src/ideaadministrator/app/patch_helper.py @@ -0,0 +1,277 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + + +import ideaadministrator +from ideadatamodel import exceptions, constants, EC2Instance +from ideasdk.utils import Utils + +from ideasdk.context import SocaCliContext, SocaContextOptions + +from typing import List +import os +import time +from prettytable import PrettyTable + +PATCH_LOG = '/root/bootstrap/logs/patch.log' + + +class PatchHelper: + + def __init__(self, cluster_name: str, aws_region: str, aws_profile: str, component: str, + instance_selector: str, + module_id: str, package_uri: str, force: bool, + patch_command: str): + + if Utils.is_empty(cluster_name): + raise exceptions.invalid_params('cluster_name is required') + if Utils.is_empty(aws_region): + raise exceptions.invalid_params('aws_region is required') + if Utils.is_empty(module_id): + raise exceptions.invalid_params('module_id is required') + + self.cluster_name = cluster_name + self.aws_region = aws_region + self.aws_profile = aws_profile + self.component = component + self.instance_selector = instance_selector + self.module_id = module_id + self.user_package_uri = package_uri + self.force = force + self.patch_command = patch_command + + self.context = SocaCliContext(options=SocaContextOptions( + cluster_name=cluster_name, + aws_region=aws_region, + aws_profile=aws_profile, + enable_aws_client_provider=True + )) + + module_info = self.context.get_cluster_module_info(module_id) + if module_info is None: + raise exceptions.general_exception(f'module not found: {module_id}') + module_type = module_info['type'] + if module_type != 'app': + raise exceptions.general_exception(f'patching is not supported for module: {module_id}, type: {module_type}') + self.module_name = module_info['name'] + + status = module_info['status'] + if status != 'deployed': + raise exceptions.general_exception(f'cannot patch module. module: {module_id} is not yet deployed.') + + def try_get_s3_package_uri(self) -> str: + """ + if package uri is provided by the user, check if the package uri is local or s3 path. + if package is local, upload to cluster's s3 bucket and return uri + + if package uri is not provided, find the local package uri for current release, upload to s3 and return the s3 path. + if running in dev mode from sources, package uri is: /dist/.tar.gz + if running in docker container, package uri is: /root/.idea/downloads/.tar.gz + + :return: s3 path + """ + if Utils.is_not_empty(self.user_package_uri): + package_uri = self.user_package_uri + else: + + if ideaadministrator.props.is_dev_mode(): + package_dist_dir = ideaadministrator.props.dev_mode_project_dist_dir + else: + package_dist_dir = ideaadministrator.props.soca_downloads_dir + + package_uri = os.path.join(package_dist_dir, + f'idea-{self.module_name}-{ideaadministrator.props.current_release_version}.tar.gz') + + if package_uri.startswith('s3://'): + return package_uri + + if not Utils.is_file(package_uri): + raise exceptions.file_not_found(f'release package not found: {package_uri}') + + cluster_s3_bucket = self.context.config().get_string('cluster.cluster_s3_bucket', required=True) + + s3_path = f'idea/patches/{os.path.basename(package_uri)}' + s3_package_uri = f's3://{cluster_s3_bucket}/{s3_path}' + self.context.info(f'uploading package: {package_uri} to {s3_package_uri} ...') + self.context.aws().s3().upload_file( + Bucket=cluster_s3_bucket, + Filename=package_uri, + Key=s3_path + ) + return s3_package_uri + + def get_patch_run_command(self, package_uri: str) -> str: + if self.module_name in ( + constants.MODULE_DIRECTORYSERVICE, + constants.MODULE_CLUSTER_MANAGER, + constants.MODULE_SCHEDULER + ): + return f'sudo /bin/bash /root/bootstrap/latest/{self.module_name}/install_app.sh {package_uri} >> {PATCH_LOG}' + elif self.module_name == constants.MODULE_VIRTUAL_DESKTOP_CONTROLLER: + if Utils.is_empty(self.component): + return f'sudo /bin/bash /root/bootstrap/latest/{self.module_name}/install_app.sh {package_uri} >> {PATCH_LOG}' + else: + # TODO: Deprecate + return f'sudo /bin/bash /root/bootstrap/latest/reverse-proxy-server/install_app.sh {package_uri} >> {PATCH_LOG}' + + def print_ec2_instance_table(self, instances: List[EC2Instance]): + table = PrettyTable(['Instance Id', 'Instance Name', 'Host Name', 'Private IP', 'State']) + table.align = 'l' + + for instance in instances: + table.add_row([instance.instance_id, instance.get_tag('Name'), instance.private_dns_name_fqdn, instance.private_ip_address, instance.state]) + + print(table) + + def patch_app(self): + + self.context.info('searching for applicable ec2 instances ...') + describe_instances_result = self.context.aws().ec2().describe_instances( + Filters=[ + { + 'Name': 'instance-state-name', + 'Values': ['pending', 'stopped', 'running'] + }, + { + 'Name': f'tag:{constants.IDEA_TAG_CLUSTER_NAME}', + 'Values': [self.cluster_name] + }, + { + 'Name': f'tag:{constants.IDEA_TAG_MODULE_ID}', + 'Values': [self.module_id] + }, + { + 'Name': f'tag:{constants.IDEA_TAG_NODE_TYPE}', + 'Values': ['app'] + } + ] + ) + + instances_to_patch = [] + instances_cannot_be_patched = [] + + reservations = Utils.get_value_as_list('Reservations', describe_instances_result, []) + for reservation in reservations: + instances = Utils.get_value_as_list('Instances', reservation) + for instance in instances: + ec2_instance = EC2Instance(instance) + if ec2_instance.state == 'running': + if Utils.is_empty(self.instance_selector) or self.instance_selector == 'all': + instances_to_patch.append(ec2_instance) + else: + if self.instance_selector == 'any': + instances_to_patch.append(ec2_instance) + break + else: + instances_cannot_be_patched = ec2_instance + + if len(instances_cannot_be_patched) > 0: + self.context.warning('Below instances cannot be patched as the instances are not running: ') + self.print_ec2_instance_table(instances_cannot_be_patched) + + if len(instances_to_patch) == 0: + self.context.warning('No instances found to be patched. Abort.') + return + + self.print_ec2_instance_table(instances_to_patch) + if not self.force: + confirm = self.context.prompt(f'Are you sure you want to patch the above running ec2 instances for module: {self.module_name}?') + if not confirm: + self.context.info('Patch aborted!') + return + + with self.context.spinner('patching ec2 instances via AWS Systems Manager (Run Command) ... '): + + if Utils.is_empty(self.patch_command): + package_uri = self.try_get_s3_package_uri() + patch_command = self.get_patch_run_command(package_uri) + else: + patch_command = self.patch_command + print(f'patch command: {patch_command}') + + instance_ids = [] + for ec2_instance in instances_to_patch: + instance_ids.append(ec2_instance.instance_id) + + send_command_result = self.context.aws().ssm().send_command( + InstanceIds=instance_ids, + DocumentName='AWS-RunShellScript', + Parameters={ + 'commands': [ + f'sudo echo "# $(date) executing patch ..." >> {PATCH_LOG}', + patch_command, + f'sudo tail -10 {PATCH_LOG}' + ] + } + ) + + command_id = send_command_result['Command']['CommandId'] + while True: + list_command_invocations_result = self.context.aws().ssm().list_command_invocations( + CommandId=command_id, + Details=False + ) + command_invocations = list_command_invocations_result['CommandInvocations'] + + completed_count = 0 + failed_count = 0 + + for command_invocation in command_invocations: + status = command_invocation['Status'] + + if status in ('Success', 'TimedOut', 'Cancelled', 'Failed'): + completed_count += 1 + + if status in ('TimedOut', 'Cancelled', 'Failed'): + failed_count += 1 + + if len(command_invocations) > 0: + self.context.info(f'Patching completed on {completed_count} out of {len(command_invocations)} instances') + if completed_count == len(command_invocations) and len(command_invocations) > 0: + break + + time.sleep(10) + + list_command_invocations_result = self.context.aws().ssm().list_command_invocations( + CommandId=command_id, + Details=True + ) + command_invocations = list_command_invocations_result['CommandInvocations'] + + instance_id_to_command_invocations = {} + for command_invocation in command_invocations: + instance_id = command_invocation['InstanceId'] + instance_id_to_command_invocations[instance_id] = command_invocation + + self.context.info(f'Patch execution status for SSM Command Id: {command_id}') + table = PrettyTable(['Instance Id', 'Instance Name', 'Host Name', 'Private IP', 'State', 'Patch Status']) + table.align = 'l' + for ec2_instance in instances_to_patch: + command_invocation = instance_id_to_command_invocations[ec2_instance.instance_id] + patch_status = command_invocation['Status'] + table.add_row([ + ec2_instance.instance_id, + ec2_instance.get_tag('Name'), + ec2_instance.private_dns_name_fqdn, + ec2_instance.private_ip_address, + ec2_instance.state, + patch_status + ]) + + print(table) + + if failed_count > 0: + self.context.error(f'Patch failed. Please check the patch logs for the instances at {PATCH_LOG}') + else: + self.context.success('Patch executed successfully. Please verify the patch functionality as per release notes / change log.') + + def apply(self): + self.patch_app() diff --git a/source/idea/idea-administrator/src/ideaadministrator/app/shared_storage_helper.py b/source/idea/idea-administrator/src/ideaadministrator/app/shared_storage_helper.py new file mode 100644 index 00000000..53a6168d --- /dev/null +++ b/source/idea/idea-administrator/src/ideaadministrator/app/shared_storage_helper.py @@ -0,0 +1,553 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +import ideaadministrator +from ideadatamodel import ( + exceptions, + errorcodes, + constants +) + +from ideasdk.utils import Utils, EnvironmentUtils +from ideasdk.user_input.framework import ( + SocaUserInputParamRegistry, + SocaUserInputArgs +) + +from ideasdk.context import SocaCliContext, SocaContextOptions +from ideasdk.config.cluster_config import ClusterConfig + +from ideaadministrator.app.installer_params import SharedStoragePromptFactory +from ideaadministrator.app.deployment_helper import DeploymentHelper +from ideaadministrator.app.config_generator import ConfigGenerator + +from typing import Dict, List, Optional + +NEXT_STEP_UPDATE_SETTINGS = 'Update Cluster Settings and Exit' +NEXT_STEP_UPGRADE_MODULE = 'Deploy Module: Shared Storage' +NEXT_STEP_DEPLOY_MODULE = 'Upgrade Module: Shared Storage' +NEXT_STEP_EXIT = 'Exit' + + +class SharedStorageHelper: + """ + Shared Storage Helper + Utility class to manage, update and/or generate shared storage file systems cluster configurations. + + Supported File Systems: + * Amazon EFS + * Amazon File Cache + * Amazon FSx for Lustre + * Amazon FSx for NetApp ONTAP + * Amazon FSx for OpenZFS + * Amazon FSx for Windows File Server + + primary use-case for this class is to enable admins generate the shared storage configurations for various file systems supported by IDEA. + class can be used before or after updating cluster configuration DB. + + if the cluster configuration DB is updated with shared storage file systems, after any cluster nodes are launched, + the existing node's fstab must be manually updated and mounted to reflect new file system config changes + + any new cluster nodes launched will mount the file systems based applicable scope and filters. + + What's supported? + ----------------- + * Generate configurations for mounting new Amazon EFS file system + * Generate configurations for mounting existing Amazon EFS + FSx file systems + This works only if you have an existing file system provisioned via AWS Console or some other means and is associated to an existing VPC. + The implementation does not perform any checks if cluster nodes can access the file system endpoints and the responsibility to check for + network access/connectivity is delegated to the cluster administrator. + + Roadmap: + ------- + * Generate configurations for provisioning new FSx file systems + """ + + def __init__(self, cluster_name: str, aws_region: str, aws_profile: str, kms_key_id: str): + + self.cluster_name = cluster_name + self.aws_region = aws_region + self.aws_profile = aws_profile + self.kms_key_id = kms_key_id + + self.config_initialized = False + self.existing_cluster = False + self.shared_storage_deployed = False + self.context: Optional[SocaCliContext] = None + + if Utils.is_not_empty(self.cluster_name): + try: + + self.context = SocaCliContext(options=SocaContextOptions( + cluster_name=cluster_name, + aws_region=aws_region, + aws_profile=aws_profile, + enable_aws_client_provider=True, + enable_aws_util=True, + locale=EnvironmentUtils.get_environment_variable('LC_CTYPE', default='en_US') + )) + self.config_initialized = True + + cluster_module_id = self.context.config().get_module_id(constants.MODULE_CLUSTER) + cluster_module_info = self.context.get_cluster_module_info(module_id=cluster_module_id) + self.existing_cluster = cluster_module_info['status'] == 'deployed' + + shared_storage_module_id = self.context.config().get_module_id(constants.MODULE_SHARED_STORAGE) + shared_storage_module_info = self.context.get_cluster_module_info(module_id=shared_storage_module_id) + self.shared_storage_deployed = shared_storage_module_info['status'] == 'deployed' + + except exceptions.SocaException as e: + if e.error_code == errorcodes.CLUSTER_CONFIG_NOT_INITIALIZED: + pass + else: + raise e + + if self.context is None: + self.context = SocaCliContext(options=SocaContextOptions( + aws_region=aws_region, + aws_profile=aws_profile, + enable_aws_client_provider=True, + enable_aws_util=True + )) + + def prompt(self, use_existing_fs: bool) -> Dict: + # user input framework init + param_registry = SocaUserInputParamRegistry( + context=self.context, + file=ideaadministrator.props.shared_storage_params_file + ) + shared_storage_args = SocaUserInputArgs( + context=self.context, + param_registry=param_registry + ) + prompt_factory = SharedStoragePromptFactory( + context=self.context, + args=shared_storage_args, + param_registry=param_registry + ) + + shared_storage_args.set('use_existing_fs', use_existing_fs) + + # skip vpc selection prompt if cluster name is provided and cluster module is deployed + if self.existing_cluster: + vpc_id = self.context.config().get_string('cluster.network.vpc_id', required=True) + shared_storage_args.set('vpc_id', vpc_id) + + # prompt common settings + shared_storage_module = prompt_factory.build_module( + user_input_module='common-settings', + display_steps=False + ) + shared_storage_result = shared_storage_module.safe_ask() + + provider = Utils.get_value_as_string('shared_storage_provider', shared_storage_result) + + if use_existing_fs: + target_user_input_module = f'{provider}_existing' + else: + target_user_input_module = f'{provider}_new' + + # prompt provider specific settings + provider_module = prompt_factory.build_module( + user_input_module=target_user_input_module, + display_info=False, + display_steps=False + ) + provider_result = provider_module.safe_ask() + + # merge and return result + return { + **shared_storage_result, + **provider_result, + 'use_existing_fs': use_existing_fs + } + + @staticmethod + def get_delimited_tokens(s: str, delimiter: str = ',', default=None) -> List[str]: + tokens = s.split(delimiter) + result = [] + for token in tokens: + if Utils.is_empty(token): + continue + token = Utils.get_as_string(token).strip().lower() + result.append(token) + if Utils.is_empty(result): + return default + return result + + def build_scope_config(self, params: Dict) -> Dict: + + scope = self.get_delimited_tokens(Utils.get_value_as_string('shared_storage_scope', params), default=['cluster']) + + scope_config = { + 'scope': scope + } + + if 'module' in scope: + modules = Utils.get_value_as_list('shared_storage_scope_modules', params, []) + scope_config['modules'] = modules + + if 'project' in scope: + projects = self.get_delimited_tokens(Utils.get_value_as_string('shared_storage_scope_projects', params), default=[constants.DEFAULT_PROJECT]) + scope_config['projects'] = projects + + if 'scheduler:queue-profile' in scope: + queue_profiles = self.get_delimited_tokens(Utils.get_value_as_string('shared_storage_scope_queue_profiles', params), default=['compute']) + scope_config['queue_profiles'] = queue_profiles + + return scope_config + + def build_common_config(self, params: Dict) -> Dict: + provider = Utils.get_value_as_string('shared_storage_provider', params) + config = { + 'title': Utils.get_value_as_string('shared_storage_title', params), + 'provider': provider, + **self.build_scope_config(params) + } + + if provider in (constants.STORAGE_PROVIDER_FSX_NETAPP_ONTAP, + constants.STORAGE_PROVIDER_FSX_WINDOWS_FILE_SERVER): + config['mount_drive'] = Utils.get_value_as_string('shared_storage_mount_drive', params, 'Z:') + + if provider != constants.STORAGE_PROVIDER_FSX_WINDOWS_FILE_SERVER: + config['mount_dir'] = Utils.get_value_as_string('shared_storage_mount_dir', params) + config['mount_options'] = Utils.get_value_as_string(f'{provider}.mount_options', params) + + return config + + @staticmethod + def use_existing_file_system(params: Dict) -> bool: + return Utils.get_value_as_bool('use_existing_fs', params, False) + + def build_efs_config(self, params: Dict) -> Dict: + + name = Utils.get_value_as_string('shared_storage_name', params) + + if self.use_existing_file_system(params=params): + + file_system_id = Utils.get_value_as_string('efs.file_system_id', params) + + # describe efs does not return dns name. construct fs dns using below format + # ensuring it is aws partition aware + aws_region = self.aws_region + aws_dns_suffix = self.context.aws().aws_dns_suffix() + file_system_dns = f'{file_system_id}.efs.{aws_region}.{aws_dns_suffix}' + + describe_fs_result = self.context.aws().efs().describe_file_systems( + FileSystemId=file_system_id + ) + + file_system = describe_fs_result['FileSystems'][0] + encrypted = Utils.get_value_as_bool('Encrypted', file_system) + + config = { + name: { + **self.build_common_config(params), + constants.STORAGE_PROVIDER_EFS: { + 'use_existing_fs': True, + 'file_system_id': Utils.get_value_as_string('efs.file_system_id', params), + 'dns': file_system_dns, + 'encrypted': encrypted + } + } + } + else: + transition_to_ia = Utils.get_value_as_string('efs.transition_to_ia', params) + if transition_to_ia == 'DISABLED': + transition_to_ia = None + + config = { + name: { + **self.build_common_config(params), + 'efs': { + 'kms_key_id': self.kms_key_id, + 'encrypted': True, + 'throughput_mode': Utils.get_value_as_string('efs.throughput_mode', params), + 'performance_mode': Utils.get_value_as_string('efs.performance_mode', params), + 'removal_policy': Utils.get_value_as_string('efs.removal_policy', params, 'DESTROY'), + 'cloudwatch_monitoring': Utils.get_value_as_bool('efs.cloudwatch_monitoring', params, False), + 'transition_to_ia': transition_to_ia + } + } + } + + return config + + def build_fsx_cache_config(self, params: Dict) -> Dict: + name = Utils.get_value_as_string('shared_storage_name', params) + file_system_id = Utils.get_value_as_string('fsx_cache.file_system_id', params) + describe_fs_result = self.context.aws().fsx().describe_file_caches( + FileCacheIds=[file_system_id] + ) + + file_system = describe_fs_result['FileCaches'][0] + file_system_dns = Utils.get_value_as_string('DNSName', file_system) + lustre_config = Utils.get_value_as_dict('LustreConfiguration', file_system, {}) + mount_name = Utils.get_value_as_string('MountName', lustre_config) + lustre_version = Utils.get_value_as_string('FileCacheTypeVersion', file_system) + return { + name: { + **self.build_common_config(params), + 'fsx_cache': { + 'use_existing_fs': True, + 'file_system_id': file_system_id, + 'dns': file_system_dns, + 'mount_name': mount_name, + 'version': lustre_version + } + } + } + + def build_fsx_lustre_config(self, params: Dict) -> Dict: + name = Utils.get_value_as_string('shared_storage_name', params) + + file_system_id = Utils.get_value_as_string('fsx_lustre.file_system_id', params) + describe_fs_result = self.context.aws().fsx().describe_file_systems( + FileSystemIds=[file_system_id] + ) + + file_system = describe_fs_result['FileSystems'][0] + file_system_dns = Utils.get_value_as_string('DNSName', file_system) + + lustre_config = Utils.get_value_as_dict('LustreConfiguration', file_system, {}) + + mount_name = Utils.get_value_as_string('MountName', lustre_config) + lustre_version = Utils.get_value_as_string('FileSystemTypeVersion', file_system) + + return { + name: { + **self.build_common_config(params), + 'fsx_lustre': { + 'use_existing_fs': True, + 'file_system_id': file_system_id, + 'dns': file_system_dns, + 'mount_name': mount_name, + 'version': lustre_version + } + } + } + + def build_fsx_netapp_ontap_config(self, params: Dict) -> Dict: + name = Utils.get_value_as_string('shared_storage_name', params) + file_system_id = Utils.get_value_as_string('fsx_netapp_ontap.file_system_id', params) + + # fetch svm info + svm_id = Utils.get_value_as_string('fsx_netapp_ontap.svm_id', params) + describe_svm_result = self.context.aws().fsx().describe_storage_virtual_machines( + StorageVirtualMachineIds=[svm_id] + ) + svm = describe_svm_result['StorageVirtualMachines'][0] + endpoints = Utils.get_value_as_dict('Endpoints', svm, {}) + + nfs_endpoint = Utils.get_value_as_dict('Nfs', endpoints, {}) + nfs_dns = Utils.get_value_as_string('DNSName', nfs_endpoint) + + smb_endpoint = Utils.get_value_as_dict('Smb', endpoints, {}) + smb_dns = Utils.get_value_as_string('DNSName', smb_endpoint) + + management_endpoint = Utils.get_value_as_dict('Management', endpoints, {}) + management_dns = Utils.get_value_as_string('DNSName', management_endpoint) + + iscsi_endpoint = Utils.get_value_as_dict('Iscsi', endpoints, {}) + iscsi_dns = Utils.get_value_as_string('DNSName', iscsi_endpoint) + + # fetch volume info + volume_id = Utils.get_value_as_string('fsx_netapp_ontap.volume_id', params) + + describe_volume_result = self.context.aws().fsx().describe_volumes( + VolumeIds=[volume_id] + ) + volume = describe_volume_result['Volumes'][0] + volume_ontap_config = Utils.get_value_as_dict('OntapConfiguration', volume, {}) + + volume_path = Utils.get_value_as_string('JunctionPath', volume_ontap_config) + security_style = Utils.get_value_as_string('SecurityStyle', volume_ontap_config) + cifs_share_name = Utils.get_value_as_string('fsx_netapp_ontap.cifs_share_name', params, default='') + + return { + name: { + **self.build_common_config(params), + 'fsx_netapp_ontap': { + 'use_existing_fs': True, + 'file_system_id': file_system_id, + 'svm': { + 'svm_id': svm_id, + 'smb_dns': smb_dns, + 'nfs_dns': nfs_dns, + 'management_dns': management_dns, + 'iscsi_dns': iscsi_dns + }, + 'volume': { + 'volume_id': volume_id, + 'volume_path': volume_path, + 'security_style': security_style, + 'cifs_share_name': cifs_share_name + } + } + } + } + + def build_fsx_openzfs_config(self, params: Dict) -> Dict: + name = Utils.get_value_as_string('shared_storage_name', params) + + file_system_id = Utils.get_value_as_string('fsx_openzfs.file_system_id', params) + describe_fs_result = self.context.aws().fsx().describe_file_systems( + FileSystemIds=[file_system_id] + ) + + file_system = describe_fs_result['FileSystems'][0] + file_system_dns = Utils.get_value_as_string('DNSName', file_system) + + volume_id = Utils.get_value_as_string('fsx_openzfs.volume_id', params) + + describe_volume_result = self.context.aws().fsx().describe_volumes( + VolumeIds=[volume_id] + ) + volume = describe_volume_result['Volumes'][0] + volume_ontap_config = Utils.get_value_as_dict('OpenZFSConfiguration', volume, {}) + volume_path = Utils.get_value_as_string('VolumePath', volume_ontap_config) + + return { + name: { + **self.build_common_config(params), + 'fsx_openzfs': { + 'use_existing_fs': True, + 'file_system_id': file_system_id, + 'dns': file_system_dns, + 'volume_id': volume_id, + 'volume_path': volume_path + } + } + } + + def build_fsx_windows_file_server_config(self, params: Dict) -> Dict: + name = Utils.get_value_as_string('shared_storage_name', params) + + file_system_id = Utils.get_value_as_string('fsx_windows_file_server.file_system_id', params) + describe_fs_result = self.context.aws().fsx().describe_file_systems( + FileSystemIds=[file_system_id] + ) + + file_system = describe_fs_result['FileSystems'][0] + file_system_dns = Utils.get_value_as_string('DNSName', file_system) + + windows_config = Utils.get_value_as_dict('WindowsConfiguration', file_system, {}) + preferred_file_server_ip = Utils.get_value_as_string('PreferredFileServerIp', windows_config) + + return { + name: { + **self.build_common_config(params), + 'fsx_windows_file_server': { + 'use_existing_fs': True, + 'file_system_id': file_system_id, + 'dns': file_system_dns, + 'preferred_file_server_ip': preferred_file_server_ip + } + } + } + + def build_shared_storage_config(self, params: Dict) -> Dict: + provider = Utils.get_value_as_string('shared_storage_provider', params) + + if provider == constants.STORAGE_PROVIDER_EFS: + return self.build_efs_config(params=params) + elif provider == constants.STORAGE_PROVIDER_FSX_CACHE: + return self.build_fsx_cache_config(params=params) + elif provider == constants.STORAGE_PROVIDER_FSX_LUSTRE: + return self.build_fsx_lustre_config(params=params) + elif provider == constants.STORAGE_PROVIDER_FSX_NETAPP_ONTAP: + return self.build_fsx_netapp_ontap_config(params=params) + elif provider == constants.STORAGE_PROVIDER_FSX_OPENZFS: + return self.build_fsx_openzfs_config(params=params) + elif provider == constants.STORAGE_PROVIDER_FSX_WINDOWS_FILE_SERVER: + return self.build_fsx_windows_file_server_config(params=params) + + raise exceptions.general_exception(f'shared storage provider: {provider} not supported') + + def print_config(self, config: Dict): + self.context.print_rule('Shared Storage Config') + self.context.new_line() + config_yaml = Utils.to_yaml(config) + config_yaml = config_yaml.replace(': null', ': ~') + print(config_yaml) + self.context.print_rule() + + def prompt_next_step(self, params: Dict) -> str: + next_step_choices = [NEXT_STEP_EXIT] + + if self.config_initialized: + next_step_choices.append(NEXT_STEP_UPDATE_SETTINGS) + + if not self.use_existing_file_system(params=params) and self.existing_cluster: + if self.shared_storage_deployed: + next_step_choices.append(NEXT_STEP_UPGRADE_MODULE) + else: + next_step_choices.append(NEXT_STEP_DEPLOY_MODULE) + + if len(next_step_choices) == 1: + return NEXT_STEP_EXIT + + return self.context.prompt( + message='How do you want to proceed further?', + default=NEXT_STEP_EXIT, + choices=next_step_choices + ) + + def update_cluster_settings(self, module_id: str, config: Dict): + config_generator = ConfigGenerator( + values={} + ) + config_entries = [] + config_generator.traverse_config( + config_entries=config_entries, + prefix=module_id, + config=config + ) + + cluster_config: ClusterConfig = self.context.config() + cluster_config.db.sync_cluster_settings_in_db( + config_entries=config_entries, + overwrite=True + ) + + def add_file_system(self, use_existing_fs: bool): + + # ask for input + params = self.prompt(use_existing_fs=use_existing_fs) + + # build config + config = self.build_shared_storage_config(params=params) + + # print config + self.print_config(config=config) + + # what to do next? + next_step = self.prompt_next_step(params=params) + + if next_step == NEXT_STEP_EXIT: + raise SystemExit + + if next_step == NEXT_STEP_UPDATE_SETTINGS: + module_id = self.context.config().get_module_id(constants.MODULE_SHARED_STORAGE) + self.update_cluster_settings(module_id=module_id, config=config) + raise SystemExit + + if next_step in (NEXT_STEP_DEPLOY_MODULE, NEXT_STEP_UPGRADE_MODULE): + module_id = self.context.config().get_module_id(constants.MODULE_SHARED_STORAGE) + self.update_cluster_settings(module_id=module_id, config=config) + DeploymentHelper( + cluster_name=self.cluster_name, + aws_region=self.aws_region, + upgrade=next_step == NEXT_STEP_UPGRADE_MODULE, + all_modules=False, + module_ids=[module_id], + aws_profile=self.aws_profile + ).deploy_module(module_id) diff --git a/source/idea/idea-administrator/src/ideaadministrator/app/single_sign_on_helper.py b/source/idea/idea-administrator/src/ideaadministrator/app/single_sign_on_helper.py new file mode 100644 index 00000000..1227151a --- /dev/null +++ b/source/idea/idea-administrator/src/ideaadministrator/app/single_sign_on_helper.py @@ -0,0 +1,506 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +from ideasdk.utils import Utils, EnvironmentUtils +from ideadatamodel import exceptions, constants +from ideasdk.config.cluster_config import ClusterConfig +from ideasdk.context import SocaCliContext, SocaContextOptions + +from typing import Optional, Dict, List, Any +import botocore.exceptions +import time + +DEFAULT_USER_POOL_CLIENT_NAME = 'single-sign-on-client' +DEFAULT_IDENTITY_PROVIDER_IDENTIFIER = 'single-sign-on-identity-provider' + + +class SingleSignOnHelper: + """ + Configures Single Sign-On for the Cluster + + Multiple Identity Providers for the cluster is NOT supported. Identity Provider must support SAMLv2 or OIDC. + Disabling Single-Sign On is NOT supported. + + Helps with: + 1. Initializing the Identity Provider in the Cluster's Cognito User Pool + 2. Initializing the Cognito User Pool Client to be used for Communicating with IDP + 3. Linking existing non-system users with IDP + """ + + def __init__(self, cluster_name: str, aws_region: str, aws_profile: str = None): + self.cluster_name = cluster_name + self.aws_region = aws_region + self.aws_profile = aws_profile + + self.context = SocaCliContext( + options=SocaContextOptions( + cluster_name=cluster_name, + aws_region=aws_region, + aws_profile=aws_profile, + enable_aws_client_provider=True, + enable_iam_permission_util=True, + locale=EnvironmentUtils.get_environment_variable('LC_CTYPE', default='en_US') + ) + ) + self.config: ClusterConfig = self.context.config() + + def _update_config_entry(self, key: str, value: Any): + module_id = self.config.get_module_id(constants.MODULE_IDENTITY_PROVIDER) + self.config.db.set_config_entry(f'{module_id}.{key}', value) + self.config.put(f'{module_id}.{key}', value) + + def get_callback_urls(self) -> List[str]: + """ + Build the callback URLs to be configured in the User Pool's OAuth 2.0 Client used for communicating with IDP. + Since custom domain name can be added at a later point in time, callback urls for both the default load balancer domain name + and the custom domain name (if available) are returned. + + The `create_or_update_user_pool_client` method can be called multiple times and will update the callback urls returned by this method. + """ + load_balancer_dns_name = self.context.config().get_string('cluster.load_balancers.external_alb.load_balancer_dns_name', required=True) + + custom_dns_name = self.context.config().get_string('cluster.load_balancers.external_alb.certificates.custom_dns_name') + if Utils.is_empty(custom_dns_name): + custom_dns_name = self.context.config().get_string('cluster.load_balancers.external_alb.custom_dns_name') + + cluster_manager_web_context_path = self.context.config().get_string('cluster-manager.server.web_resources_context_path', required=True) + if cluster_manager_web_context_path == '/': + sso_auth_callback_path = '/sso/oauth2/callback' + else: + sso_auth_callback_path = f'{cluster_manager_web_context_path}/oauth2/callback' + + if not sso_auth_callback_path.startswith('/'): + sso_auth_callback_path = f'/{sso_auth_callback_path}' + + callback_urls = [ + f'https://{load_balancer_dns_name}{sso_auth_callback_path}' + ] + if Utils.is_not_empty(custom_dns_name): + callback_urls.append(f'https://{custom_dns_name}{sso_auth_callback_path}') + return callback_urls + + def get_idp_redirect_uri(self, provider_type: str) -> str: + """ + The redirect URL that must be configured with the IDP. + """ + domain_url = self.config.get_string('identity-provider.cognito.domain_url', required=True) + if provider_type == constants.SSO_IDP_PROVIDER_OIDC: + return f'{domain_url}/oauth2/idpresponse' + else: + return f'{domain_url}/saml2/idpresponse' + + def get_entity_id(self) -> str: + """ + The entity id required for Azure based SAML providers. + Refer: https://aws.amazon.com/blogs/security/how-to-set-up-amazon-cognito-for-federated-authentication-using-azure-ad/ + """ + user_pool_id = self.context.config().get_string('identity-provider.cognito.user_pool_id', required=True) + return f'urn:amazon:cognito:sp:{user_pool_id}' + + def create_or_update_user_pool_client(self, provider_name: str, refresh_token_validity_hours: int = None) -> Dict: + """ + setup the user pool client used for communicating with the IDP. + + the ClientId and ClientSecret are saved to cluster configuration and secrets manager. + cluster configuration with clientId and secret arn is updated only once during creation. + + method can be invoked multiple times to update the user pool client if it already exists to support updating + the callback urls + """ + + user_pool_id = self.context.config().get_string('identity-provider.cognito.user_pool_id', required=True) + sso_client_id = self.context.config().get_string('identity-provider.cognito.sso_client_id') + if refresh_token_validity_hours is None or refresh_token_validity_hours <= 0: + refresh_token_validity_hours = 12 + + callback_urls = self.get_callback_urls() + user_pool_client_request = { + 'UserPoolId': user_pool_id, + 'ClientName': DEFAULT_USER_POOL_CLIENT_NAME, + 'AccessTokenValidity': 1, + 'IdTokenValidity': 1, + 'RefreshTokenValidity': refresh_token_validity_hours, + 'TokenValidityUnits': { + 'AccessToken': 'hours', + 'IdToken': 'hours', + 'RefreshToken': 'hours' + }, + 'ReadAttributes': [ + 'address', + 'birthdate', + 'custom:aws_region', + 'custom:cluster_name', + 'custom:password_last_set', + 'custom:password_max_age', + 'email', + 'email_verified', + 'family_name', + 'gender', + 'given_name', + 'locale', + 'middle_name', + 'name', + 'nickname', + 'phone_number', + 'phone_number_verified', + 'picture', + 'preferred_username', + 'profile', + 'updated_at', + 'website', + 'zoneinfo' + ], + 'AllowedOAuthFlows': [ + 'code' + ], + 'AllowedOAuthScopes': [ + 'email', + 'openid', + 'aws.cognito.signin.user.admin' + ], + 'CallbackURLs': callback_urls, + 'SupportedIdentityProviders': [provider_name], + 'AllowedOAuthFlowsUserPoolClient': True + } + + if Utils.is_not_empty(sso_client_id): + user_pool_client_request['ClientId'] = sso_client_id + update_user_pool_client_result = self.context.aws().cognito_idp().update_user_pool_client(**user_pool_client_request) + return update_user_pool_client_result['UserPoolClient'] + + user_pool_client_request['GenerateSecret'] = True + create_user_pool_client_result = self.context.aws().cognito_idp().create_user_pool_client(**user_pool_client_request) + user_pool_client = create_user_pool_client_result['UserPoolClient'] + + # get custom kms key id for secrets manager if configured + # and add kms key id to request if available. else boto client throws validation exception for None + kms_key_id = self.config.get_string('cluster.secretsmanager.kms_key_id') + + tags = [ + { + 'Key': constants.IDEA_TAG_CLUSTER_NAME, + 'Value': self.cluster_name + }, + { + 'Key': constants.IDEA_TAG_MODULE_NAME, + 'Value': constants.MODULE_CLUSTER_MANAGER + } + ] + + secret_name = f'{self.cluster_name}-sso-client-secret' + try: + describe_secret_result = self.context.aws().secretsmanager().describe_secret( + SecretId=secret_name + ) + except botocore.exceptions.ClientError as e: + if e.response['Error']['Code'] == 'ResourceNotFoundException': + describe_secret_result = None + else: + raise e + + if describe_secret_result is None: + create_secret_client_secret_request = { + 'Name': f'{self.cluster_name}-sso-client-secret', + 'Description': f'Single Sign-On OAuth2 Client Secret for Cluster: {self.cluster_name}', + 'Tags': tags, + 'SecretString': user_pool_client['ClientSecret'] + } + if Utils.is_not_empty(kms_key_id): + create_secret_client_secret_request['KmsKeyId'] = kms_key_id + create_secret_client_secret_result = self.context.aws().secretsmanager().create_secret(**create_secret_client_secret_request) + secret_arn = create_secret_client_secret_result['ARN'] + else: + update_secret_client_secret_request = { + 'SecretId': describe_secret_result['ARN'], + 'SecretString': user_pool_client['ClientSecret'] + } + if Utils.is_not_empty(kms_key_id): + update_secret_client_secret_request['KmsKeyId'] = kms_key_id + update_secret_client_secret_result = self.context.aws().secretsmanager().update_secret(**update_secret_client_secret_request) + secret_arn = update_secret_client_secret_result['ARN'] + + self._update_config_entry('cognito.sso_client_id', user_pool_client['ClientId']) + self._update_config_entry('cognito.sso_client_secret', secret_arn) + return user_pool_client + + @staticmethod + def get_saml_provider_details(**kwargs) -> Dict: + """ + build the SAML provider details based on user input + the `saml_metadata_file` must be a path to the file on local file system, which is read and contents are sent to boto API. + + refer to below link for more details: + https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/cognito-idp.html#CognitoIdentityProvider.Client.create_identity_provider + """ + provider_details = {} + saml_metadata_url = Utils.get_value_as_string('saml_metadata_url', kwargs) + saml_metadata_file = Utils.get_value_as_string('saml_metadata_file', kwargs) + if Utils.are_empty(saml_metadata_url, saml_metadata_file): + raise exceptions.invalid_params('Either one of [saml_metadata_url, saml_metadata_file] is required, when provider_type = SAML') + + if Utils.is_not_empty(saml_metadata_file): + if not Utils.is_file(saml_metadata_file): + raise exceptions.invalid_params(f'file not found: {saml_metadata_file}') + + with open(saml_metadata_file, 'r') as f: + saml_metadata_file_contents = f.read() + + provider_details['MetadataFile'] = saml_metadata_file_contents + else: + provider_details['MetadataURL'] = saml_metadata_url + return provider_details + + @staticmethod + def get_oidc_provider_details(**kwargs) -> Dict: + """ + build the OIDC provider details based on user input. + + refer to below link for more details: + https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/cognito-idp.html#CognitoIdentityProvider.Client.create_identity_provider + """ + + provider_details = {} + oidc_client_id = Utils.get_value_as_string('oidc_client_id', kwargs) + oidc_client_secret = Utils.get_value_as_string('oidc_client_secret', kwargs) + oidc_issuer = Utils.get_value_as_string('oidc_issuer', kwargs) + oidc_attributes_request_method = Utils.get_value_as_string('oidc_attributes_request_method', kwargs, 'GET') + oidc_authorize_scopes = Utils.get_value_as_string('oidc_authorize_scopes', kwargs, 'openid') + oidc_authorize_url = Utils.get_value_as_string('oidc_authorize_url', kwargs) + oidc_token_url = Utils.get_value_as_string('oidc_token_url', kwargs) + oidc_attributes_url = Utils.get_value_as_string('oidc_attributes_url', kwargs) + oidc_jwks_uri = Utils.get_value_as_string('oidc_jwks_uri', kwargs) + + if Utils.is_empty(oidc_client_id): + raise exceptions.invalid_params('oidc_client_id is required') + if Utils.is_empty(oidc_client_secret): + raise exceptions.invalid_params('oidc_client_secret is required') + if Utils.is_empty(oidc_issuer): + raise exceptions.invalid_params('oidc_issuer is required') + + provider_details['client_id'] = oidc_client_id + provider_details['client_secret'] = oidc_client_secret + provider_details['attributes_request_method'] = oidc_attributes_request_method + provider_details['authorize_scopes'] = oidc_authorize_scopes + provider_details['oidc_issuer'] = oidc_issuer + if Utils.is_not_empty(oidc_authorize_url): + provider_details['authorize_url'] = oidc_authorize_url + if Utils.is_not_empty(oidc_token_url): + provider_details['token_url'] = oidc_token_url + if Utils.is_not_empty(oidc_attributes_url): + provider_details['attributes_url'] = oidc_attributes_url + if Utils.is_not_empty(oidc_jwks_uri): + provider_details['jwks_uri'] = oidc_jwks_uri + + return provider_details + + def get_identity_provider(self) -> Optional[Dict]: + """ + method to check if SSO identity provider is already created to ensure only a single IDP is created for the cluster. + The `identity-provider.cognito.sso_idp_identifier` config entry value is used as the identifier. if the entry does not exist, + value of `DEFAULT_IDENTITY_PROVIDER_IDENTIFIER` is used. + """ + user_pool_id = self.context.config().get_string('identity-provider.cognito.user_pool_id', required=True) + sso_idp_identifier = self.context.config().get_string('identity-provider.cognito.sso_idp_identifier', DEFAULT_IDENTITY_PROVIDER_IDENTIFIER) + try: + result = self.context.aws().cognito_idp().get_identity_provider_by_identifier( + UserPoolId=user_pool_id, + IdpIdentifier=sso_idp_identifier + ) + return result['IdentityProvider'] + except botocore.exceptions.ClientError as e: + if e.response['Error']['Code'] == 'ResourceNotFoundException': + return None + else: + raise e + + def create_or_update_identity_provider(self, provider_name: str, provider_type: str, provider_email_attribute: str, **kwargs): + """ + setup the Identity Provider for the Cognito User Pool. + at the moment, only OIDC and SAML provider types are supported. + + provider details are built using the `get_saml_provider_details` for SAML and `get_oidc_provider_details` for OIDC. + + Important: + `identity-provider.cognito.sso_idp_provider_name` config entry is updated with the name of the IDP provider. + changing this value will result in adding multiple IDPs for the cluster and will result in broken SSO functionality. + as part of AccountsService during user creation, if SSO is enabled, the new user will be linked with the IDP Provider name + specified in `identity-provider.cognito.sso_idp_provider_name` + """ + + if Utils.is_empty(provider_name): + raise exceptions.invalid_params('provider_name is required') + if Utils.is_empty(provider_type): + raise exceptions.invalid_params('provider_type is required') + if Utils.is_empty(provider_email_attribute): + raise exceptions.invalid_params('provider_email_attribute is required') + + if provider_type == constants.SSO_IDP_PROVIDER_SAML: + provider_details = self.get_saml_provider_details(**kwargs) + elif provider_type == constants.SSO_IDP_PROVIDER_OIDC: + provider_details = self.get_oidc_provider_details(**kwargs) + else: + raise exceptions.invalid_params('provider type must be one of: SAML or OIDC') + + user_pool_id = self.context.config().get_string('identity-provider.cognito.user_pool_id', required=True) + + existing_identity_provider = self.get_identity_provider() + sso_idp_identifier = self.context.config().get_string('identity-provider.cognito.sso_idp_identifier', DEFAULT_IDENTITY_PROVIDER_IDENTIFIER) + if existing_identity_provider is not None: + self.context.aws().cognito_idp().update_identity_provider( + UserPoolId=user_pool_id, + ProviderName=provider_name, + ProviderDetails=provider_details, + AttributeMapping={ + 'email': provider_email_attribute + }, + IdpIdentifiers=[ + sso_idp_identifier + ] + ) + else: + self.context.aws().cognito_idp().create_identity_provider( + UserPoolId=user_pool_id, + ProviderName=provider_name, + ProviderType=provider_type, + ProviderDetails=provider_details, + AttributeMapping={ + 'email': provider_email_attribute + }, + IdpIdentifiers=[ + sso_idp_identifier + ] + ) + + self._update_config_entry('cognito.sso_idp_provider_name', provider_name) + self._update_config_entry('cognito.sso_idp_provider_type', provider_type) + self._update_config_entry('cognito.sso_idp_identifier', sso_idp_identifier) + self._update_config_entry('cognito.sso_idp_provider_email_attribute', provider_email_attribute) + + def link_existing_users(self): + """ + if there are existing users in the user pool, added prior to enabling SSO, link all the existing users with IDP. + this ensures SSO works for existing users that were created before enabling IDP. + + system administration users such as clusteradmin are not linked with IDP with an assumption that no such users will exist + in corporate directory service. + + when SSO is enabled, accessing the cluster endpoint will result in signing-in automatically. + to ensure clusteradmin user can sign-in, a special query parameter can be provided: + `sso=False` + sending this parameter as part of query string will not trigger the sso flow from the web portal. + """ + user_pool_id = self.context.config().get_string('identity-provider.cognito.user_pool_id', required=True) + provider_name = self.context.config().get_string('identity-provider.cognito.sso_idp_provider_name', required=True) + provider_type = self.context.config().get_string('identity-provider.cognito.sso_idp_provider_type', required=True) + cluster_admin_username = self.context.config().get_string('cluster.administrator_username', required=True) + + if provider_type == constants.SSO_IDP_PROVIDER_OIDC: + provider_email_attribute = 'email' + else: + provider_email_attribute = self.context.config().get_string('identity-provider.cognito.sso_idp_provider_email_attribute', required=True) + + while True: + list_users_result = self.context.aws().cognito_idp().list_users( + UserPoolId=user_pool_id + ) + users = list_users_result['Users'] + for user in users: + try: + if user['UserStatus'] == 'EXTERNAL_PROVIDER': + continue + + username = user['Username'] + + # exclude system administration users + if username in cluster_admin_username or username.startswith('clusteradmin'): + print(f'system administration user found: {username}. skip linking with IDP.') + continue + + email = None + already_linked = False + user_attributes = Utils.get_value_as_list('Attributes', user, []) + for user_attribute in user_attributes: + name = user_attribute['Name'] + if name == 'email': + email = Utils.get_value_as_string('Value', user_attribute) + elif name == 'identities': + identities = Utils.get_value_as_list('Value', user_attribute, []) + for identity in identities: + if identity['providerName'] == provider_name: + already_linked = True + + if Utils.is_empty(email): + continue + if already_linked: + print(f'user: {username}, email: {email} already linked. skip.') + continue + + print(f'linking user: {username}, email: {email} ...') + + def admin_link_provider_for_user(**kwargs): + print(f'link request: {Utils.to_json(kwargs)}') + self.context.aws().cognito_idp().admin_link_provider_for_user(**kwargs) + + admin_link_provider_for_user( + UserPoolId=user_pool_id, + DestinationUser={ + 'ProviderName': 'Cognito', + 'ProviderAttributeName': 'cognito:username', + 'ProviderAttributeValue': user['Username'] + }, + SourceUser={ + 'ProviderName': provider_name, + 'ProviderAttributeName': provider_email_attribute, + 'ProviderAttributeValue': email + } + ) + # sleep for a while to avoid flooding aws with these requests. + time.sleep(0.2) + except Exception as e: + print(f'failed to link user: {user} with IDP: {provider_name} - {e}') + + pagination_token = Utils.get_value_as_string('PaginationToken', list_users_result) + if Utils.is_empty(pagination_token): + break + + def configure_sso(self, provider_name: str, provider_type: str, **kwargs): + """ + execute series of steps to configure Single Sign-On for the cluster. + + `identity-provider.cognito.sso_enabled` boolean config entry is set to True at the end of the flow. + this config entry is the single place in the cluster to indicate SSO is enabled. + """ + # identity provider + with self.context.spinner('creating identity provider ...'): + self.create_or_update_identity_provider( + provider_name=provider_name, + provider_type=provider_type, + **kwargs + ) + self.context.success('✓ identity provider created') + + # user pool client + with self.context.spinner('creating user pool client ...'): + self.create_or_update_user_pool_client( + provider_name=provider_name, + refresh_token_validity_hours=Utils.get_value_as_int('refresh_token_validity_hours', kwargs, default=None) + ) + self.context.success('✓ user pool client created') + + # link existing users + with self.context.spinner('linking existing users with IDP ...'): + self.link_existing_users() + self.context.success('✓ existing users linked with IDP') + + # update cluster settings - this must be last step in the pipeline. + with self.context.spinner('enabling Single Sign-On ...'): + self._update_config_entry('cognito.sso_enabled', True) + self.context.success('✓ Single Sign-On enabled for cluster') diff --git a/source/idea/idea-administrator/src/ideaadministrator/app/support_helper.py b/source/idea/idea-administrator/src/ideaadministrator/app/support_helper.py new file mode 100644 index 00000000..e99eab01 --- /dev/null +++ b/source/idea/idea-administrator/src/ideaadministrator/app/support_helper.py @@ -0,0 +1,206 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +import ideaadministrator +from ideasdk.context import SocaCliContext +from ideasdk.utils import Utils +from ideasdk.config.cluster_config import ClusterConfig +from ideadatamodel import ( + exceptions, errorcodes, + SocaUserInputParamMetadata, + SocaUserInputParamType, + SocaUserInputValidate, + SocaUserInputChoice +) + +from typing import Dict +import os +import shutil +import arrow + +PKG_CONTENTS_DEPLOYMENT_LOGS = 'deployment-logs' +PKG_CONTENTS_DEPLOYMENTS_DIR = 'deployments-dir' +PKG_CONTENTS_CDK_CONFIG = 'cdk-config' +PKG_CONTENTS_CONFIG_VALUES_FILE = 'config-values-file' +PKG_CONTENTS_CLUSTER_CONFIG_DB = 'cluster-config-db' +PKG_CONTENTS_CLUSTER_CONFIG_LOCAL = 'cluster-config-local' + + +class SupportHelper: + """ + Support Helper + helper class to orchestrate support related commands, starting with support package for deployment. + will be extended further to implement/export support packages for individual modules using SSM + S3 + """ + + def __init__(self, cluster_name: str, aws_region: str, aws_profile: str, module_set: str): + self.cluster_name = cluster_name + self.aws_region = aws_region + self.aws_profile = aws_profile + self.module_set = module_set + self.context = SocaCliContext() + + def get_deployment_debug_user_input(self) -> Dict: + + result = self.context.ask( + title='IDEA Deployment Support', + description='Use deployment support to gather all applicable artifacts to build a support package (zip file) to be sent to IDEA Support team.', + questions=[ + SocaUserInputParamMetadata( + name='package_contents', + title='Package Contents', + description='Select applicable artifacts to be added to support package', + data_type='str', + param_type=SocaUserInputParamType.CHECKBOX, + multiple=True, + default=[ + PKG_CONTENTS_DEPLOYMENT_LOGS, + PKG_CONTENTS_CONFIG_VALUES_FILE, + PKG_CONTENTS_CLUSTER_CONFIG_DB, + PKG_CONTENTS_CLUSTER_CONFIG_LOCAL + # PKG_CONTENTS_DEPLOYMENTS_DIR is not added by design. contents can get a bit large. + # after @madbajaj completes P70264990, this can be added. + ], + choices=[ + SocaUserInputChoice(title='Deployment Logs', value=PKG_CONTENTS_DEPLOYMENT_LOGS), + SocaUserInputChoice(title='Configuration (values.yml)', value=PKG_CONTENTS_CONFIG_VALUES_FILE), + SocaUserInputChoice(title='Cluster Configuration (DB)', value=PKG_CONTENTS_CLUSTER_CONFIG_DB), + SocaUserInputChoice(title='Cluster Configuration (Local)', value=PKG_CONTENTS_CLUSTER_CONFIG_LOCAL), + SocaUserInputChoice(title='Bootstrap Packages', value=PKG_CONTENTS_DEPLOYMENTS_DIR) + ], + validate=SocaUserInputValidate( + required=True + ) + ) + ]) + return result + + def get_deployment_debug_package_dir(self) -> str: + support_dir = ideaadministrator.props.cluster_support_dir(cluster_name=self.cluster_name, aws_region=self.aws_region) + support_package_dir = os.path.join(support_dir, f'idea-deployment-debug-pkg-{Utils.file_system_friendly_timestamp()}') + os.makedirs(support_package_dir, exist_ok=True) + return support_package_dir + + def add_deployment_logs(self, target_dir: str): + logs_dir = ideaadministrator.props.cluster_logs_dir(self.cluster_name, self.aws_region) + self.context.info(f'copying deployment logs: {logs_dir} ...') + shutil.copytree(logs_dir, os.path.join(target_dir, 'logs')) + + def add_cdk_config_dir(self, target_dir: str): + cdk_dir = ideaadministrator.props.cluster_cdk_dir(self.cluster_name, self.aws_region) + self.context.info(f'copying cdk config: {cdk_dir} ...') + shutil.copytree(cdk_dir, os.path.join(target_dir, '_cdk')) + + def add_deployments_dir(self, target_dir: str): + deployments_dir = ideaadministrator.props.cluster_deployments_dir(self.cluster_name, self.aws_region) + self.context.info(f'copying deployments: {deployments_dir} ...') + shutil.copytree(deployments_dir, os.path.join(target_dir, 'deployments')) + + def add_values_file(self, target_dir: str): + values_file = ideaadministrator.props.values_file(self.cluster_name, self.aws_region) + if not Utils.is_file(values_file): + return + self.context.info(f'copying values.yml: {values_file} ...') + shutil.copy(values_file, target_dir) + + def add_cluster_config_local(self, target_dir: str): + config_dir = ideaadministrator.props.cluster_config_dir(self.cluster_name, self.aws_region) + if not Utils.is_dir(config_dir): + return + self.context.info(f'copying cluster config (local): {config_dir} ...') + shutil.copytree(config_dir, os.path.join(target_dir, 'config_local')) + + def add_cluster_config_db(self, target_dir: str): + try: + cluster_config = ClusterConfig( + cluster_name=self.cluster_name, + aws_region=self.aws_region, + aws_profile=self.aws_profile, + module_set=self.module_set + ) + except exceptions.SocaException as e: + if e.error_code == errorcodes.CLUSTER_CONFIG_NOT_INITIALIZED: + return + else: + raise e + + config_db_dir = os.path.join(target_dir, 'config_db') + os.makedirs(config_db_dir, exist_ok=True) + + config_db_file = os.path.join(config_db_dir, 'config.yml') + self.context.info(f'copying cluster config from db: {config_db_file} ...') + with open(config_db_file, 'w') as f: + f.write(cluster_config.as_yaml()) + + modules_db_file = os.path.join(config_db_dir, 'modules.yml') + self.context.info(f'copying modules from db: {modules_db_file} ...') + cluster_modules = cluster_config.db.get_cluster_modules() + with open(modules_db_file, 'w') as f: + f.write(Utils.to_yaml(cluster_modules)) + + def build_deployment_debug_package(self, user_input: Dict) -> str: + + package_dir = self.get_deployment_debug_package_dir() + + with open(os.path.join(package_dir, 'package.yml'), 'w') as f: + package_data = { + 'type': 'deployment-debug', + 'created_on': arrow.utcnow().format(), + 'options': user_input + } + f.write(Utils.to_yaml(package_data)) + + self.context.info(f'building debug package: {package_dir} ...') + package_contents = Utils.get_value_as_list('package_contents', user_input, []) + + if PKG_CONTENTS_DEPLOYMENT_LOGS in package_contents: + self.add_deployment_logs(package_dir) + if PKG_CONTENTS_CDK_CONFIG in package_contents: + self.add_cdk_config_dir(package_dir) + if PKG_CONTENTS_DEPLOYMENTS_DIR in package_contents: + self.add_deployments_dir(package_dir) + if PKG_CONTENTS_CONFIG_VALUES_FILE in package_contents: + self.add_values_file(package_dir) + if PKG_CONTENTS_CLUSTER_CONFIG_LOCAL in package_contents: + self.add_cluster_config_local(package_dir) + if PKG_CONTENTS_CLUSTER_CONFIG_DB in package_contents: + self.add_cluster_config_db(package_dir) + + shutil.make_archive(package_dir, 'zip', package_dir) + return f'{package_dir}.zip' + + def invoke(self): + # ask for user input for applicable package components + user_input = self.get_deployment_debug_user_input() + + # build package based on user input + with self.context.spinner('building debug package ...'): + package_file = self.build_deployment_debug_package(user_input=user_input) + + # print support information template + self.context.new_line() + self.context.print_rule('IDEA Support') + self.context.new_line() + self.context.print_title('Step 1) Upload the below package (zip file) to a secured location accessible by IDEA support team.') + self.context.info(f'Debug Package: {package_file}') + self.context.new_line() + self.context.print_title('Step 2) Send an email to idea-support@amazon.com with below details') + self.context.info('* Subject: [Deployment] {short description of your problem}') + self.context.info('* Description: {Describe your use-case and the problem}') + self.context.info('* Debug Package Download Link: {package download link}') + self.context.info('* Full Name: ') + self.context.info('* Email: ') + self.context.info('* Company Name: ') + self.context.info('* City: ') + self.context.info('* Country: ') + self.context.info('* AWS Business Development or AWS Partner Network SA Contact: ') + self.context.new_line() + self.context.print_rule() diff --git a/source/idea/idea-administrator/src/ideaadministrator/app/values_diff.py b/source/idea/idea-administrator/src/ideaadministrator/app/values_diff.py new file mode 100644 index 00000000..8151abc0 --- /dev/null +++ b/source/idea/idea-administrator/src/ideaadministrator/app/values_diff.py @@ -0,0 +1,44 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +from ideaadministrator.app_props import AdministratorProps +import os + + +class ValuesDiff: + + def __init__(self, cluster_name: str, aws_region: str): + self.props = AdministratorProps() + self.cluster_name = cluster_name + self.aws_region = aws_region + self.values_file = 'values.yml' + self.values_file_dir = 'values/' + + def get_cluster_name(self) -> str: + return self.cluster_name + + def get_aws_region(self) -> str: + return self.aws_region + + def get_cluster_region_dir(self) -> str: + cluster_home = self.props.cluster_dir(self.get_cluster_name()) + cluster_region_dir = self.props.cluster_region_dir(cluster_home, self.get_aws_region()) + os.makedirs(cluster_region_dir, exist_ok=True) + return cluster_region_dir + + def get_values_file_path(self) -> str: + return os.path.join(self.get_cluster_region_dir(), self.values_file) + + def get_values_file_s3_key(self) -> str: + return self.values_file_dir + self.values_file + + + diff --git a/source/idea/idea-administrator/src/ideaadministrator/app/vpc_endpoints_helper.py b/source/idea/idea-administrator/src/ideaadministrator/app/vpc_endpoints_helper.py new file mode 100644 index 00000000..b4bdff3e --- /dev/null +++ b/source/idea/idea-administrator/src/ideaadministrator/app/vpc_endpoints_helper.py @@ -0,0 +1,333 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +from ideasdk.utils import Utils +from ideasdk.context import SocaCliContext, SocaContextOptions + +from typing import List, Dict, Set + +from prettytable import PrettyTable + + +class VpcEndpointsHelper: + + def __init__(self, aws_region: str, aws_profile: str = None): + self.aws_region = aws_region + self.aws_profile = aws_profile + + self.context = SocaCliContext( + options=SocaContextOptions( + aws_region=aws_region, + aws_profile=aws_profile, + enable_aws_client_provider=True, + enable_iam_permission_util=True + ) + ) + + aws_dns_suffix = self.context.aws().aws_dns_suffix() + tokens = aws_dns_suffix.split('.') + tokens.reverse() + self.service_name_domain = '.'.join(tokens) + + self.gateway_endpoints = { + 's3': { + 'enabled': True, + 'endpoint_url': None + }, + 'dynamodb': { + 'enabled': True, + 'endpoint_url': None + } + } + + self.interface_endpoints = { + 'cloudformation': { + 'enabled': True, + 'endpoint_url': None + }, + 'ec2': { + 'enabled': True, + 'endpoint_url': None + }, + 'ec2messages': { + 'enabled': True, + 'endpoint_url': None + }, + 'ebs': { + 'enabled': True, + 'endpoint_url': None + }, + 'elasticfilesystem': { + 'enabled': True, + 'endpoint_url': None + }, + 'elasticfilesystem-fips': { + 'enabled': False, + 'endpoint_url': None + }, + 'elasticloadbalancing': { + 'enabled': True, + 'endpoint_url': None + }, + 'logs': { + 'enabled': True, + 'endpoint_url': None + }, + 'monitoring': { + 'enabled': True, + 'endpoint_url': None + }, + 'secretsmanager': { + 'enabled': True, + 'endpoint_url': None + }, + 'sns': { + 'enabled': True, + 'endpoint_url': None + }, + 'sqs': { + 'enabled': True, + 'endpoint_url': None + }, + 'events': { + 'enabled': True, + 'endpoint_url': None + }, + 'ssm': { + 'enabled': True, + 'endpoint_url': None + }, + 'ssmmessages': { + 'enabled': True, + 'endpoint_url': None + }, + 'fsx': { + 'enabled': True, + 'endpoint_url': None + }, + 'fsx-fips': { + 'enabled': False, + 'endpoint_url': None + }, + 'backup': { + 'enabled': True, + 'endpoint_url': None + }, + 'grafana': { + 'enabled': True, + 'endpoint_url': None + }, + 'acm-pca': { + 'enabled': False, + 'endpoint_url': None + }, + 'kinesis-streams': { + 'enabled': True, + 'endpoint_url': None + } + } + + def get_service_name(self, short_name: str): + return f'{self.service_name_domain}.{self.aws_region}.{short_name}' + + def get_idea_service_names(self) -> Set[str]: + service_names = set() + for service_name in self.gateway_endpoints: + service_names.add(self.get_service_name(service_name)) + for service_name in self.interface_endpoints: + service_names.add(self.get_service_name(service_name)) + return service_names + + def get_vpc_endpoint_services(self) -> List[Dict]: + describe_result = self.context.aws().ec2().describe_vpc_endpoint_services() + + service_details = Utils.get_value_as_list('ServiceDetails', describe_result, []) + service_details_map = {} + for service_detail in service_details: + service_name = service_detail['ServiceName'] + if service_name in service_details_map: + services = service_details_map[service_name] + else: + services = [] + service_details_map[service_name] = services + services.append(service_detail) + + idea_service_names = self.get_idea_service_names() + result = [] + for service_name in idea_service_names: + if service_name in service_details_map: + service_details = service_details_map[service_name] + for service_detail in service_details: + for service_type in service_detail['ServiceType']: + result.append({ + 'service_name': service_name, + 'available': True, + 'service_type': service_type['ServiceType'], + 'availability_zones': service_detail['AvailabilityZones'] + }) + else: + result.append({ + 'service_name': service_name, + 'available': False + }) + return result + + def get_supported_gateway_endpoint_services(self) -> List[str]: + result = [] + describe_result = self.context.aws().ec2().describe_vpc_endpoint_services( + Filters=[ + { + 'Name': 'service-type', + 'Values': ['Gateway'] + } + ] + ) + service_names = Utils.get_value_as_list('ServiceNames', describe_result, []) + for service_name in self.gateway_endpoints: + if self.get_service_name(service_name) in service_names: + result.append(service_name) + return result + + def get_supported_interface_endpoint_services(self) -> Dict: + result = {} + describe_result = self.context.aws().ec2().describe_vpc_endpoint_services( + Filters=[ + { + 'Name': 'service-type', + 'Values': ['Interface'] + } + ] + ) + service_names = Utils.get_value_as_list('ServiceNames', describe_result, []) + for service_name in self.interface_endpoints: + if self.get_service_name(service_name) in service_names: + result[service_name] = { + 'enabled': self.interface_endpoints[service_name]['enabled'], + 'endpoint_url': None + } + return result + + def print_vpc_endpoint_services(self): + service_details = self.get_vpc_endpoint_services() + table = PrettyTable([ + 'Service Name', + f'Is Available in {self.aws_region}', + 'Service Type', + 'Availability Zones' + ]) + table.align = 'l' + for service_detail in service_details: + + service_name = service_detail['service_name'] + is_available = service_detail['available'] + + if is_available: + service_type = service_detail['service_type'] + availability_zones = ', '.join(service_detail['availability_zones']) + else: + service_type = '-' + availability_zones = '-' + + table.add_row([ + service_name, + 'Yes' if is_available else 'No', + service_type, + availability_zones + ]) + + print(table) + + def find_existing_vpc_endpoints(self, vpc_id: str, vpc_endpoint_type: str) -> List[Dict]: + vpc_endpoints = [] + next_token = None + while True: + + describe_request = { + 'Filters': [ + { + 'Name': 'vpc-id', + 'Values': [vpc_id] + }, + { + 'Name': 'vpc-endpoint-type', + 'Values': [vpc_endpoint_type] + } + ] + } + if next_token is not None: + describe_request['NextToken'] = next_token + + describe_result = self.context.aws().ec2().describe_vpc_endpoints(**describe_request) + + endpoints = Utils.get_value_as_list('VpcEndpoints', describe_result, []) + vpc_endpoints += endpoints + + next_token = Utils.get_value_as_string('NextToken', describe_result) + if next_token is None: + break + + return vpc_endpoints + + def find_existing_gateway_endpoints(self, vpc_id: str) -> Dict: + endpoints = self.find_existing_vpc_endpoints(vpc_id=vpc_id, vpc_endpoint_type='Gateway') + result = {} + for endpoint in endpoints: + service_name = endpoint['ServiceName'] + short_name = service_name.split('.')[-1] + result[short_name] = endpoint + return result + + def is_interface_endpoint_enabled(self, service_name: str): + if service_name in self.interface_endpoints: + return self.interface_endpoints[service_name]['enabled'] + return False + + def is_gateway_endpoint_enabled(self, service_name: str): + if service_name in self.gateway_endpoints: + return self.gateway_endpoints[service_name]['enabled'] + return False + + def find_missing_gateway_endpoint_services(self, vpc_id: str) -> List[str]: + existing_endpoints = self.find_existing_gateway_endpoints(vpc_id) + supported_endpoints = self.get_supported_gateway_endpoint_services() + missing_endpoints = [] + for service_name in supported_endpoints: + if service_name in existing_endpoints: + existing_endpoint = existing_endpoints[service_name] + if existing_endpoint['State'] not in ('PendingAcceptance', 'Pending', 'Available'): + continue + if not self.is_gateway_endpoint_enabled(service_name): + continue + missing_endpoints.append(service_name) + return missing_endpoints + + def find_existing_interface_endpoints(self, vpc_id: str) -> Dict: + endpoints = self.find_existing_vpc_endpoints(vpc_id=vpc_id, vpc_endpoint_type='Interface') + result = {} + for endpoint in endpoints: + service_name = endpoint['ServiceName'] + short_name = service_name.split('.')[-1] + result[short_name] = endpoint + return result + + def find_missing_interface_endpoint_services(self, vpc_id: str) -> List[str]: + existing_endpoints = self.find_existing_interface_endpoints(vpc_id) + supported_endpoints = self.get_supported_interface_endpoint_services() + missing_endpoints = [] + for service_name in supported_endpoints: + if service_name in existing_endpoints: + existing_endpoint = existing_endpoints[service_name] + if existing_endpoint['State'] not in ('PendingAcceptance', 'Pending', 'Available'): + continue + if not self.is_interface_endpoint_enabled(service_name): + continue + missing_endpoints.append(service_name) + return missing_endpoints diff --git a/source/idea/idea-administrator/src/ideaadministrator/app_constants.py b/source/idea/idea-administrator/src/ideaadministrator/app_constants.py new file mode 100644 index 00000000..0e94954d --- /dev/null +++ b/source/idea/idea-administrator/src/ideaadministrator/app_constants.py @@ -0,0 +1,26 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + + +INSTALLER_CONFIG_DEFAULTS_FILE = 'installer_config_defaults.yml' +INSTALLER_CONFIG_FILE = 'installer_config.yml' +AWS_ENDPOINTS_FILE = 'aws_endpoints.json' + +CLUSTER_NAME_PREFIX = 'idea-' +CLUSTER_NAME_MIN_MAX_LENGTH = (8, 11) + +ACM_CERTIFICATE_SELF_SIGNED_DOMAIN = 'IDEA.DEFAULT.CREATE.YOUR.OWN.CERT' +ACM_CERTIFICATE_SELF_SIGNED_DOMAIN_OLD = 'SOCA.DEFAULT.CREATE.YOUR.OWN.CERT' + +LOG_RETENTION_ROLE_NAME = 'log-retention' + +DEPLOYMENT_OPTION_INSTALL_IDEA = 'install-idea' +DEPLOYMENT_OPTION_INSTALL_IDEA_USING_EXISTING_RESOURCES = 'install-idea-using-existing-resources' diff --git a/source/idea/idea-administrator/src/ideaadministrator/app_context.py b/source/idea/idea-administrator/src/ideaadministrator/app_context.py new file mode 100644 index 00000000..3cc62a6e --- /dev/null +++ b/source/idea/idea-administrator/src/ideaadministrator/app_context.py @@ -0,0 +1,44 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +from ideadatamodel import exceptions +from ideasdk.utils import Utils, EnvironmentUtils +from ideasdk.context import SocaContextOptions +from ideaadministrator.app_protocols import ( + AdministratorContextProtocol +) +from ideaadministrator.app_utils import AdministratorUtils + +from typing import Optional + + +class AdministratorContext(AdministratorContextProtocol): + + def __init__(self, cluster_name: str, aws_region: str, aws_profile: Optional[str] = None, module_id: Optional[str] = None): + super().__init__( + options=SocaContextOptions( + cluster_name=cluster_name, + aws_region=aws_region, + aws_profile=aws_profile, + module_id=module_id, + enable_aws_client_provider=True, + enable_aws_util=True, + locale=EnvironmentUtils.get_environment_variable('LC_CTYPE', default='en_US') + ) + ) + self._admin_utils = AdministratorUtils() + + def get_stack_name(self, module_id: str = None) -> str: + if Utils.is_empty(module_id): + raise exceptions.invalid_params('get_stack_name: cannot create stack name without a module_id') + + cluster_name = self.config().get_string('cluster.cluster_name') + return f'{cluster_name}-{module_id}' diff --git a/source/idea/idea-administrator/src/ideaadministrator/app_main.py b/source/idea/idea-administrator/src/ideaadministrator/app_main.py new file mode 100644 index 00000000..b15c5386 --- /dev/null +++ b/source/idea/idea-administrator/src/ideaadministrator/app_main.py @@ -0,0 +1,1954 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +from ideadatamodel import ( + exceptions, + errorcodes, + constants, + SocaKeyValue, + SocaUserInputParamMetadata, + SocaUserInputParamType, + SocaUserInputValidate, + SocaUserInputChoice +) +from ideadatamodel.constants import CLICK_SETTINGS +from ideasdk.utils import Utils, ModuleMetadataHelper +from ideasdk.user_input.framework import ( + SocaUserInputParamRegistry, + SocaUserInputArgs +) +from ideasdk.config.cluster_config_db import ClusterConfigDB +from ideasdk.config.cluster_config import ClusterConfig +from ideasdk.config.soca_config import SocaConfig +from ideasdk.context import SocaCliContext, SocaContextOptions + +import ideaadministrator +from ideaadministrator import app_constants +from ideaadministrator.app_utils import AdministratorUtils +from ideaadministrator.app.installer_params import QuickSetupPromptFactory +from ideaadministrator.app_props import AdministratorProps +from ideaadministrator.app.cdk.cdk_invoker import CdkInvoker +from ideaadministrator.app.config_generator import ConfigGenerator +from ideaadministrator.app.delete_cluster import DeleteCluster +from ideaadministrator.app.patch_helper import PatchHelper +from ideaadministrator.app.deployment_helper import DeploymentHelper +from ideaadministrator.app.single_sign_on_helper import SingleSignOnHelper +from ideaadministrator.integration_tests.test_context import TestContext +from ideaadministrator.integration_tests.test_invoker import TestInvoker +from ideaadministrator.app.values_diff import ValuesDiff +from ideaadministrator.app.vpc_endpoints_helper import VpcEndpointsHelper +from ideaadministrator.app.aws_service_availability_helper import AwsServiceAvailabilityHelper +from ideaadministrator.app.cluster_prefix_list_helper import ClusterPrefixListHelper +from ideaadministrator.app.support_helper import SupportHelper +from ideaadministrator.app.directory_service_helper import DirectoryServiceHelper +from ideaadministrator.app.shared_storage_helper import SharedStorageHelper + +from prettytable import PrettyTable +import os +import sys +import click +import requests +import warnings +from rich.table import Table +from rich.console import Console +import time +import botocore.exceptions + + +@click.group(context_settings=CLICK_SETTINGS) +@click.version_option(version=ideaadministrator.__version__) +def main(): + """ + IDEA Administrator - install, deploy and manage your clusters + + \b + * Deploy/Install IDEA from scratch + ./idea-admin.sh quick-setup + + \b + * Deploy/Install IDEA using existing resources running on your AWS account + ./idea-admin.sh quick-setup --existing-resources + + * For all other operations (update/delete/check config...) please refer to the help section below + """ + pass + + +@click.group() +def config(): + """ + configuration management options + """ + pass + + +@click.group() +def cdk(): + """ + cdk app + """ + pass + + +@click.group() +def sso(): + """ + single sign-on configuration options + """ + pass + + +@click.group() +def directoryservice(): + """ + directory service utilities + """ + pass + + +@click.group() +def utils(): + """ + cluster configuration utilities + """ + pass + + +@utils.group() +def vpc_endpoints(): + """ + vpc endpoint utilities + """ + pass + + +@utils.group() +def cluster_prefix_list(): + """ + cluster prefix list utilities + """ + pass + + +@click.group() +def support(): + """ + support options + """ + pass + + +@click.group() +def shared_storage(): + """ + shared-storage options + """ + pass + + +@config.command('generate', context_settings=CLICK_SETTINGS) +@click.option('--values-file', help='path to values.yml file') +@click.option('--config-dir', help='path to where to create config directory') +@click.option('--force', is_flag=True, help='Skip all confirmation prompts.') +@click.option('--existing-resources', is_flag=True, help='Generate configuration using existing resources') +@click.option('--regenerate', is_flag=True, help='Regenerate configuration for an existing cluster. Enables skipping validations such as existing cluster name and CIDR block.') +def config_generate(values_file: str, config_dir: str, force: bool, existing_resources: bool = False, regenerate: bool = False): + """ + generate configuration + """ + + context = SocaCliContext( + options=SocaContextOptions(enable_aws_client_provider=True) + ) + + if config_dir is not None: + if not os.path.isdir(config_dir): + context.error(f'{config_dir} not found or is not a valid directory.') + raise SystemExit + + if Utils.is_empty(values_file): + + param_registry = SocaUserInputParamRegistry( + context=context, + file=ideaadministrator.props.install_params_file + ) + + installer_args = SocaUserInputArgs(context, param_registry=param_registry) + installer_args.set('_regenerate', regenerate) + + if existing_resources: + deployment_option = app_constants.DEPLOYMENT_OPTION_INSTALL_IDEA_USING_EXISTING_RESOURCES + else: + deployment_option = app_constants.DEPLOYMENT_OPTION_INSTALL_IDEA + + prompt_factory = QuickSetupPromptFactory( + context=context, + args=installer_args, + param_registry=param_registry + ) + + user_input_module = prompt_factory.build_module(user_input_module=deployment_option) + user_input_module.safe_ask() + values = installer_args.build() + + cluster_name = Utils.get_value_as_string('cluster_name', values) + aws_region = Utils.get_value_as_string('aws_region', values) + + if not Utils.are_empty(cluster_name, aws_region): + if not Utils.is_empty(config_dir): + cluster_region_dir = config_dir + else: + props = AdministratorProps() + cluster_dir = props.cluster_dir(cluster_name) + cluster_region_dir = props.cluster_region_dir(cluster_dir, aws_region) + + if not force: + if AdministratorUtils.cluster_region_dir_exists_and_has_config(cluster_region_dir): + confirm = context.prompt(f'Config directory: {cluster_region_dir} is not empty, would you like to overwrite it?', default=True) + if not confirm: + context.info('Aborted!') + raise SystemExit + AdministratorUtils.cleanup_cluster_region_dir(cluster_region_dir) + + os.makedirs(cluster_region_dir, exist_ok=True) + values_file = os.path.join(cluster_region_dir, 'values.yml') + + context.info(f'saving values to: {values_file}') + with open(values_file, 'w') as f: + f.write(Utils.to_yaml(values)) + else: + context.warning('Cluster name and AWS region are required') + raise SystemExit + + # values file is given + else: + if not Utils.is_file(values_file): + raise exceptions.invalid_params(f'file not found: {values_file}') + + with open(values_file, 'r') as f: + values = Utils.from_yaml(f.read()) + + props = AdministratorProps() + # config dir is not provided + if Utils.is_empty(config_dir): + cluster_dir = props.cluster_dir(Utils.get_value_as_string('cluster_name', values)) + cluster_region_dir = props.cluster_region_dir(cluster_dir, Utils.get_value_as_string('aws_region', values)) + else: + cluster_region_dir = config_dir + + values_file_copy = os.path.join(cluster_region_dir, 'values.yml') + + preserve_values_file = False + if Utils.are_equal(values_file, values_file_copy): + preserve_values_file = True + + if not force: + if AdministratorUtils.cluster_region_dir_exists_and_has_config(cluster_region_dir): + confirm = context.prompt(f'Config directory: {cluster_region_dir} is not empty, would you like to overwrite it?', default=True) + if not confirm: + context.info('Aborted!') + raise SystemExit + AdministratorUtils.cleanup_cluster_region_dir(cluster_region_dir, preserve_values_file) + + os.makedirs(cluster_region_dir, exist_ok=True) + + context.info(f'saving values to: {values_file_copy}') + with open(values_file_copy, 'w') as f: + f.write(Utils.to_yaml(values)) + + config_generator = ConfigGenerator(values) + + context.print_title('generating config from templates ...') + + if Utils.is_empty(config_dir): + config_generator.generate_config_from_templates() + else: + config_generator.generate_config_from_templates(True, cluster_region_dir) + + # values returned by generate_config are used by quick setup flow. do not remove the return statement. + return values + + +@config.command('save-values', context_settings=CLICK_SETTINGS) +@click.option('--cluster-name', required=True, help="Cluster Name") +@click.option('--aws-profile', help='AWS Profile') +@click.option('--aws-region', required=True, help='AWS Region') +@click.option('--values-file', help='path to values.yml file') +def save_values(cluster_name: str, aws_profile: str, aws_region: str, values_file: str): + """ + save values file in s3 bucket + """ + + context = SocaCliContext( + options=SocaContextOptions(enable_aws_client_provider=True) + ) + + cluster_config_db = ClusterConfigDB( + cluster_name=cluster_name, + aws_region=aws_region, + aws_profile=aws_profile + ) + + bucket_name = get_bucket_name(cluster_name, aws_region, cluster_config_db, context) + + values_diff = ValuesDiff(cluster_name, aws_region) + + if Utils.is_empty(values_file): + values_file = values_diff.get_values_file_path() + print_using_default_warning('Values file', values_file, context) + + context.info(f'Saving in bucket: {bucket_name} at location: values/value.yml') + context.aws().s3().upload_file( + Bucket=bucket_name, + Filename=values_file, + Key='values/values.yml' + ) + + +@config.command('update', context_settings=CLICK_SETTINGS) +@click.option('--cluster-name', required=True, help='Cluster Name') +@click.option('--aws-profile', help='AWS Profile Name') +@click.option('--aws-region', required=True, help='AWS Region') +@click.option('--force', is_flag=True, help='Skip all confirmation prompts.') +@click.option('--overwrite', is_flag=True, help='Overwrite existing db config entries. ' + 'Default behavior is to skip if the config entry exists.') +@click.option('--key-prefix', help='Update configuration for the keys matching the given key prefix.') +@click.option('--config-dir', help='Path to Config Directory; Uses default location if not provided') +@click.option('--module-set', help='Name of the ModuleSet. Default: default') +def config_update(cluster_name: str, aws_profile: str, aws_region: str, force: bool, overwrite: bool, key_prefix: str, config_dir: str, module_set: str): + """ + update configuration from local file system to cluster settings db + """ + + context = SocaCliContext() + + if Utils.is_empty(module_set): + module_set = constants.DEFAULT_MODULE_SET + + values = { + 'cluster_name': cluster_name, + 'aws_profile': aws_profile, + 'aws_region': aws_region + } + config_generator = ConfigGenerator(values) + + if Utils.is_not_empty(config_dir): + cluster_config_dir = os.path.join(config_dir, 'config') + if not os.path.isdir(cluster_config_dir): + context.error(f'{cluster_config_dir} does not exist') + raise SystemExit + else: + cluster_config_dir = config_generator.get_cluster_config_dir() + + local_config_dict = config_generator.read_config_from_files(config_dir=cluster_config_dir) + local_config = SocaConfig(config=local_config_dict) + + # perform a basic sanity check to see if the config in config dir is indeed the configuration for the expected cluster/aws region + # this scenario is more likely to happen when using the --config-dir option + # where the configurations in the provided config dir does not match given --cluster-name and --aws-region + cluster_module_id = local_config.get_string(f'global-settings.module_sets.{module_set}.cluster.module_id', required=True) + local_config_cluster_name = local_config.get_string(f'{cluster_module_id}.cluster_name', required=True) + if local_config_cluster_name != cluster_name: + raise exceptions.cluster_config_error(f'local configuration in {cluster_config_dir} does not match the given cluster name: {cluster_name}') + local_config_aws_region = local_config.get_string(f'{cluster_module_id}.aws.region', required=True) + if local_config_aws_region != aws_region: + raise exceptions.cluster_config_error(f'local configuration in {cluster_config_dir} does not match the given aws region: {aws_region}') + + def read_local_config_entries(): + context.info(f'reading cluster settings from {cluster_config_dir} ...') + config_entries_ = config_generator.convert_config_to_key_value_pairs(key_prefix=key_prefix, path=cluster_config_dir) + table = PrettyTable(['Key', 'Value']) + table.align = 'l' + for entry in config_entries_: + key = Utils.get_value_as_string('key', entry, '-') + value = Utils.get_any_value('value', entry, '-') + if isinstance(value, list): + value = Utils.to_yaml(value) + table.add_row([ + key, + value + ]) + print(table) + return config_entries_ + + config_entries = read_local_config_entries() + if not force: + while True: + result = context.prompt( + message='Are you sure you want to update cluster settings db with above configuration from local file system?', + default='Yes', + choices=['Yes', 'Reload Changes', 'Exit'] + ) + if result == 'Exit': + context.info('Aborted!') + raise SystemExit + elif result == 'Reload Changes': + config_entries = read_local_config_entries() + else: + break + + cluster_config_db = ClusterConfigDB( + cluster_name=config_generator.get_cluster_name(), + aws_region=config_generator.get_aws_region(), + aws_profile=config_generator.get_aws_profile(), + create_database=True + ) + if config_dir: + modules = config_generator.read_modules_from_files(cluster_config_dir) + else: + modules = config_generator.read_modules_from_files() + cluster_config_db.sync_modules_in_db(modules) + cluster_config_db.sync_cluster_settings_in_db(config_entries, overwrite) + + +@config.command('set', context_settings=CLICK_SETTINGS) +@click.option('--cluster-name', required=True, help='Cluster Name') +@click.option('--aws-profile', help='AWS Profile Name') +@click.option('--aws-region', required=True, help='AWS Region') +@click.option('--force', is_flag=True, help='Skip confirmation prompts') +@click.argument('entries', required=True, nargs=-1) +def set_config(cluster_name: str, aws_profile: str, aws_region: str, force: bool, entries): + """ + set config entries + + \b + entry must be of below format: Key=KEY_NAME,Type=[str|int|float|bool|list|list|list|list],Value=[VALUE|[VALUE1,VALUE2,...]] ... + config key names cannot contain: comma(,), colon(:) + + Examples: + + \b + 1) To set a string config type: + ./idea-admin.sh config set Key=global-settings.string_val,Type=string,Value=stringcontent --cluster-name YOUR_CLUSTER_NAME --aws-region YOUR_AWS_REGION + + \b + 2) To set an integer config type: + ./idea-admin.sh config set Key=global-settings.int_val,Type=int,Value=12 --cluster-name YOUR_CLUSTER_NAME --aws-region YOUR_AWS_REGION + + \b + 3) To set a config with list of strings: + ./idea-admin.sh config set "Key=my_config.string_list,Type=list,Value=value1,value2" --cluster-name YOUR_CLUSTER_NAME --aws-region YOUR_AWS_REGION + + \b + 4) Update multiple config entries: + ./idea-admin.sh config set Key=global-settings.string_val,Type=string,Value=stringcontent \\ + "Key=global-settings.integer_list,Type=list,Value=1,2" \\ + "Key=global-settings.string_list,Type=list,Value=str1,str2" \\ + --cluster-name YOUR_CLUSTER_NAME \\ + --aws-region YOUR_AWS_REGION + """ + + config_entries = [] + for index, entry in enumerate(entries): + tokens = entry.split(',', 2) + key = tokens[0].split('Key=')[1].strip() + data_type = tokens[1].split('Type=')[1].strip() + value = tokens[2].split('Value=')[1].strip() + + if Utils.is_empty(key): + raise exceptions.cluster_config_error(f'[{index}] Key is required') + if ',' in key or ':' in key: + raise exceptions.cluster_config_error(f'[{index}] Invalid Key: {key}. comma(,) and colon(:) are not allowed in key names.') + if Utils.is_empty(data_type): + raise exceptions.cluster_config_error(f'[{index}] Type is required') + if Utils.is_empty(value): + raise exceptions.cluster_config_error(f'[{index}] Value is required') + + is_list = False + if data_type in ('str', 'string'): + data_type = 'str' + elif data_type in ('int', 'integer'): + data_type = 'int' + elif data_type in ('bool', 'boolean'): + data_type = 'bool' + elif data_type in ('float', 'decimal'): + data_type = 'float' + elif data_type in ('list', 'list'): + data_type = 'str' + is_list = True + elif data_type in ('list', 'list'): + data_type = 'int' + is_list = True + elif data_type in ('list', 'list'): + data_type = 'bool' + is_list = True + elif data_type in ('list', 'list'): + data_type = 'float' + is_list = True + else: + raise exceptions.cluster_config_error(f'[{index}] Type: {data_type} not supported') + + if is_list: + tokens = value.split(',') + value = [] + for token in tokens: + if Utils.is_empty(token): + continue + value.append(token.strip()) + if data_type == 'int': + for val in value: + if not Utils.is_int(val): + raise exceptions.cluster_config_error(f'[{index}] Value: {value} is not a valid list<{data_type}>') + value = Utils.get_as_int_list(value) + elif data_type == 'float': + for val in value: + if not Utils.is_float(val): + raise exceptions.cluster_config_error(f'[{index}] Value: {value} is not a valid list<{data_type}>') + value = Utils.get_as_float_list(value) + elif data_type == 'int': + value = Utils.get_as_bool_list(value) + else: + value = Utils.get_as_string_list(value) + else: + if data_type == 'int': + if not Utils.is_int(value): + raise exceptions.cluster_config_error(f'[{index}] Value: {value} is not a valid {data_type}') + value = Utils.get_as_int(value) + elif data_type == 'float': + if not Utils.is_float(value): + raise exceptions.cluster_config_error(f'[{index}] Value: {value} is not a valid {data_type}') + value = Utils.get_as_float(value) + elif data_type == 'bool': + value = Utils.get_as_bool(value) + else: + value = Utils.get_as_string(value) + + config_entries.append({ + 'key': key, + 'value': value + }) + + context = SocaCliContext() + + table = PrettyTable(['Key', 'Value']) + table.align = 'l' + for config_entry in config_entries: + key = Utils.get_value_as_string('key', config_entry, '-') + value = Utils.get_any_value('value', config_entry, '-') + if isinstance(value, list): + value = Utils.to_yaml(value) + table.add_row([key, value]) + + print(table) + if not force: + confirm = context.prompt('Are you sure you want to update above config entries?') + if not confirm: + context.info('Abort!') + raise SystemExit + + db = ClusterConfigDB( + cluster_name=cluster_name, + aws_region=aws_region, + aws_profile=aws_profile + ) + for config_entry in config_entries: + db.set_config_entry(config_entry['key'], config_entry['value']) + + +@config.command('export', context_settings=CLICK_SETTINGS) +@click.option('--cluster-name', required=True, help='Cluster Name') +@click.option('--aws-profile', help='AWS Profile Name') +@click.option('--aws-region', required=True, help='AWS Region') +@click.option('--export-dir', help='Export Directory. Defaults to: ~/.idea/clusters///config') +def export_config(cluster_name: str, aws_profile: str, aws_region: str, export_dir: str): + """ + export configuration + """ + + if Utils.is_empty(export_dir): + props = AdministratorProps() + cluster_dir = props.cluster_dir(cluster_name) + cluster_region_dir = props.cluster_region_dir(cluster_dir, aws_region) + export_dir = os.path.join(cluster_region_dir, 'config') + + if Utils.is_dir(export_dir): + files = os.listdir(export_dir) + applicable_files = [] + for file in files: + if file == '.DS_Store': + continue + applicable_files.append(file) + + if len(applicable_files) > 0: + raise exceptions.general_exception(f'export directory: {export_dir} already exists and can cause merge conflicts. ' + f'backup your existing configuration to another directory and try again.') + + db = ClusterConfigDB( + cluster_name=cluster_name, + aws_region=aws_region, + aws_profile=aws_profile + ) + print(f'exporting config from db to {export_dir} ...') + os.makedirs(export_dir, exist_ok=True) + cluster_config = db.build_config_from_db() + config_dict = cluster_config.as_dict() + + modules = db.get_cluster_modules() + + idea_config = { + 'modules': [] + } + for module in modules: + module_id = module['module_id'] + module_name = module['name'] + module_type = module['type'] + module_export_dir = os.path.join(export_dir, module_id) + os.makedirs(module_export_dir, exist_ok=True) + module_settings = Utils.get_value_as_dict(module_id, config_dict) + module_settings_file = os.path.join(module_export_dir, 'settings.yml') + with open(module_settings_file, 'w') as f: + f.write(Utils.to_yaml(module_settings)) + idea_config['modules'].append({ + 'name': module_name, + 'id': module_id, + 'type': module_type, + 'config_files': ['settings.yml'] + }) + + with open(os.path.join(export_dir, 'idea.yml'), 'w') as f: + f.write(Utils.to_yaml(idea_config)) + + +@config.command('download-values', context_settings=CLICK_SETTINGS) +@click.option('--cluster-name', required=True, help='Cluster Name') +@click.option('--aws-profile', help='AWS Profile') +@click.option('--aws-region', required=True, help='AWS Region') +@click.option('--values-dir', help='Path to folder to save values.yml file') +def download_values(cluster_name: str, aws_profile: str, aws_region: str, values_dir: str): + """ + download values.yml from s3 bucket to default or provided location + """ + + context = SocaCliContext( + options=SocaContextOptions(enable_aws_client_provider=True) + ) + + cluster_config_db = ClusterConfigDB( + cluster_name=cluster_name, + aws_region=aws_region, + aws_profile=aws_profile + ) + + values_diff = ValuesDiff(cluster_name, aws_region) + if Utils.is_empty(values_dir): + values_file = values_diff.get_values_file_path() + print_using_default_warning('Values file directory', values_diff.get_cluster_region_dir(), context) + else: + os.makedirs(values_dir, exist_ok=True) + values_file = os.path.join(values_dir, 'values.yml') + + bucket_name = get_bucket_name(cluster_name, aws_region, cluster_config_db, context) + + response = context.aws().s3().get_object(Bucket=bucket_name, Key=values_diff.get_values_file_s3_key()) + + values = Utils.from_yaml(response['Body']) + + print(f'saving values to: {values_file}') + with open(values_file, 'w') as f: + f.write(Utils.to_yaml(values)) + + +@config.command('diff', context_settings=CLICK_SETTINGS) +@click.option('--cluster-name', required=True, help='Cluster Name') +@click.option('--aws-profile', help='AWS Profile Name') +@click.option('--aws-region', required=True, help='AWS Region') +@click.option('--config-dir', help='Path to local config folder; default location will be used if none provided') +def diff_config(cluster_name: str, aws_profile: str, aws_region: str, config_dir: str): + """ + diff configuration files between the latest config and the config in the db + """ + + props = AdministratorProps() + if Utils.is_empty(config_dir): + cluster_home = props.cluster_dir(cluster_name) + cluster_region_dir = props.cluster_region_dir(cluster_home, aws_region) + cluster_config_dir = os.path.join(cluster_region_dir, 'config') + print_using_default_warning('Configuration Directory', cluster_config_dir) + + db = ClusterConfigDB( + cluster_name=cluster_name, + aws_region=aws_region, + aws_profile=aws_profile + ) + + db_config_entries = {} + config_entries = db.get_config_entries() + + for entry in config_entries: + db_config_entries[Utils.get_value_as_string('key', entry)] = Utils.get_value_as_string('value', entry, '-') + + local_config_entries = {} + values = { + 'cluster_name': cluster_name, + 'aws_profile': aws_profile, + 'aws_region': aws_region + } + config_generator = ConfigGenerator(values) + local_config = config_generator.get_config_local(config_dir) + + for entry in local_config: + local_config_entries[Utils.get_value_as_string('key', entry)] = Utils.get_value_as_string('value', entry, '-') + + set1 = set(local_config_entries.items()) + set2 = set(db_config_entries.items()) + + set_x = set2 - set1 + set_y = set1 - set2 + + table = Table() + table.add_column("Key", justify="left", style="cyan", no_wrap=False) + table.add_column("Old Value", justify="left", style="red", no_wrap=False) + table.add_column("New Value", justify="left", style="green", no_wrap=False) + table.add_column("Status", justify="left", style="magenta", no_wrap=False) + + rows = [] + + for item in set_x: + if item[0] in local_config_entries: + rows.append((item[0], item[1], local_config_entries.get(item[0]), 'MODIFIED')) + else: + rows.append((item[0], item[1], 'n/a', 'DELETED')) + for item in set_y: + if item[0] in db_config_entries: + continue + rows.append((item[0], 'n/a', item[1], 'ADDED')) + + for row in sorted(rows): + table.add_row(row[0], row[1], row[2], row[3]) + + console = Console() + console.print(table) + + +@config.command('show', context_settings=CLICK_SETTINGS) +@click.option('--cluster-name', required=True, help='Cluster Name') +@click.option('--aws-profile', help='AWS Profile Name') +@click.option('--aws-region', required=True, help='AWS Region') +@click.option('-q', '--query', help='Search Query for configuration entries. Accepts a regular expression.') +@click.option('--format', 'output_format', help='Output format. One of [table, yaml, raw]. Default: table') +def show_config(cluster_name: str, aws_profile: str, aws_region: str, query: str, output_format: str): + """ + show configuration for a cluster as yaml + """ + cluster_config_db = ClusterConfigDB( + cluster_name=cluster_name, + aws_region=aws_region, + aws_profile=aws_profile + ) + if output_format == 'yaml': + cluster_config = cluster_config_db.build_config_from_db(query=query) + print(cluster_config.as_yaml()) + elif output_format == 'raw': + entries = cluster_config_db.get_config_entries(query=query) + for entry in entries: + value = entry.get('value') + if value is not None: + print(str(value)) + else: + config_entries = cluster_config_db.get_config_entries(query=query) + table = PrettyTable(['Key', 'Value', 'Version']) + table.align = 'l' + for entry in config_entries: + key = Utils.get_value_as_string('key', entry, '-') + value = Utils.get_any_value('value', entry, '-') + if isinstance(value, list): + value = Utils.to_yaml(value) + version = Utils.get_value_as_int('version', entry, 0) + table.add_row([ + key, + value, + version + ]) + print(table) + + +@config.command('delete', context_settings=CLICK_SETTINGS) +@click.option('--cluster-name', required=True, help='Cluster Name') +@click.option('--aws-profile', help='AWS Profile Name') +@click.option('--aws-region', required=True, help='AWS Region') +@click.argument('config-key-prefixes', nargs=-1, required=True) +def delete_config(cluster_name: str, aws_profile: str, aws_region: str, config_key_prefixes): + """ + delete all configuration entries for a given config key prefix. + + to delete all configuration entries for module id: analytics, run: + idea-admin config delete analytics. + + to delete all configuration entries for alb.listener_rules.*, run: + idea-admin config delete alb.listener_rules. + """ + db = ClusterConfigDB( + cluster_name=cluster_name, + aws_region=aws_region, + aws_profile=aws_profile + ) + for config_key_prefix in config_key_prefixes: + config_key_prefix = config_key_prefix.strip() + db.delete_config_entries(config_key_prefix) + + +@click.command('bootstrap') +@click.option('--cluster-name', required=True, help='Cluster Name') +@click.option('--aws-profile', help='AWS Profile Name') +@click.option('--aws-region', required=True, help='AWS Region') +@click.option('--termination-protection', default=True, help='Set termination protection to true or false. Default: true') +@click.option('--module-set', help='Name of the ModuleSet. Default: default') +def bootstrap_cluster(cluster_name: str, aws_profile: str, aws_region: str, termination_protection: bool, module_set: str): + """ + bootstrap cluster + """ + + context = SocaCliContext() + + db = ClusterConfigDB( + cluster_name=cluster_name, + aws_region=aws_region, + aws_profile=aws_profile + ) + + cluster_s3_bucket = db.get_cluster_s3_bucket() + with context.spinner(f'bootstrapping cluster CDK stack and S3 bucket: {cluster_s3_bucket} ...'): + CdkInvoker( + module_id='bootstrap', + module_set=module_set, + cluster_name=cluster_name, + aws_region=aws_region, + aws_profile=aws_profile, + termination_protection=termination_protection + ).bootstrap_cluster(cluster_bucket=cluster_s3_bucket) + + +@click.command() +@click.option('--cluster-name', required=True, help='Cluster Name') +@click.option('--aws-region', required=True, help='AWS Region') +@click.option('--aws-profile', help='AWS Profile Name') +@click.option('--termination-protection', default=True, help='Set termination protection to true or false. Default: true') +@click.option('--deployment-id', help='A UUID to identify the deployment.') +@click.option('--upgrade', is_flag=True, help='Upgrade the module by re-running the CDK stack if the module has already been deployed.') +@click.option('--force-build-bootstrap', is_flag=True, help='If the bootstrap package directory for a given DeploymentId already exists, ' + 'the directory will be deleted and rendered again.') +@click.option('--rollback/--no-rollback', default=True, help='Rollback stack to stable state on failure. Defaults to "true", iterate more rapidly with --no-rollback.') +@click.option('--optimize-deployment', is_flag=True, help='If flag is provided, deployment will be optimized and applicable stacks will be deployed in parallel.') +@click.option('--module-set', help='Name of the ModuleSet. Default: default') +@click.argument('MODULES', required=True, nargs=-1) +def deploy(cluster_name: str, aws_region: str, aws_profile: str, termination_protection: bool, deployment_id: str, + upgrade: bool, force_build_bootstrap: bool, rollback: bool, optimize_deployment: bool, module_set: str, modules): + """ + deploy modules + + deploy an IDEA module using the Module Id. You can provide multiple module ids and each module will be deployed sequentially. + The order of module deployment will be handled automatically to ensure correct module dependencies. + + Use `all` as the module id to deploy all modules + The default behavior of deploy is to skip deployment of a module if it's already deployed. + Use --upgrade to re-run the cdk stack for the module. + + Experimental: + The --optimize-deployment flag can be provided to optimize deployment time and deploy applicable modules in parallel. + """ + + # dedupe and convert to list + module_ids_to_deploy = [] + all_modules = False + for module_id in modules: + if module_id == 'all': + all_modules = True + if module_id in module_ids_to_deploy: + continue + module_ids_to_deploy.append(module_id) + + if all_modules: + if len(module_ids_to_deploy) > 1: + raise exceptions.invalid_params(f'fatal error - use of "all" deployment must be the only requested module') + module_ids_to_deploy = None + + DeploymentHelper( + cluster_name=cluster_name, + aws_region=aws_region, + termination_protection=termination_protection, + deployment_id=deployment_id, + upgrade=upgrade, + module_set=module_set, + all_modules=all_modules, + force_build_bootstrap=force_build_bootstrap, + optimize_deployment=optimize_deployment, + module_ids=module_ids_to_deploy, + aws_profile=aws_profile, + rollback=rollback + ).invoke() + + +@cdk.command('synth') +@click.option('--cluster-name', required=True, help='Cluster Name') +@click.option('--aws-region', required=True, help='AWS Region') +@click.option('--aws-profile', help='AWS Profile Name') +@click.option('--deployment-id', help='A UUID to identify the deployment.') +@click.option('--module-set', help='Name of the ModuleSet. Default: default') +@click.argument('module', required=True) +def cdk_synth(cluster_name: str, aws_region: str, aws_profile: str, deployment_id: str, module_set: str, module: str): + """ + synthesize cloudformation template for a module + """ + CdkInvoker( + module_id=module, + module_set=module_set, + cluster_name=cluster_name, + aws_region=aws_region, + deployment_id=deployment_id, + aws_profile=aws_profile + ).cdk_synth() + + +@cdk.command() +@click.option('--cluster-name', required=True, help='Cluster Name') +@click.option('--aws-region', required=True, help='AWS Region') +@click.option('--aws-profile', help='AWS Profile Name') +@click.option('--deployment-id', help='A UUID to identify the deployment.') +@click.option('--module-set', help='Name of the ModuleSet. Default: default') +@click.argument('module', required=True) +def diff(cluster_name: str, aws_region: str, aws_profile: str, deployment_id: str, module_set: str, module: str): + """ + compares the specified module with the deployed module + """ + CdkInvoker( + module_id=module, + module_set=module_set, + cluster_name=cluster_name, + aws_region=aws_region, + deployment_id=deployment_id, + aws_profile=aws_profile + ).cdk_diff() + + +@cdk.command(context_settings=CLICK_SETTINGS) +@click.option('--cluster-name', required=True, help='Cluster Name') +@click.option('--aws-profile', help='AWS Profile Name') +@click.option('--aws-region', required=True, help='AWS Region') +@click.option('--module-name', required=True, help='module name') +@click.option('--module-id', required=True, help='module id') +@click.option('--deployment-id', help='A UUID to identify the deployment.') +@click.option('--termination-protection', default=True, help='Toggle termination protection for the cloud formation stack. Default: true') +def cdk_app(cluster_name, aws_profile, aws_region, module_name, module_id, deployment_id, termination_protection): + """ + cdk app + """ + from ideaadministrator.app.cdk.cdk_app import CdkApp + CdkApp( + cluster_name=cluster_name, + aws_profile=aws_profile, + aws_region=aws_region, + module_name=module_name, + module_id=module_id, + deployment_id=deployment_id, + termination_protection=termination_protection + ).invoke() + + +@click.command() +@click.option('--cluster-name', required=True, help='Cluster Name') +@click.option('--aws-region', required=True, help='AWS Region') +@click.option('--aws-profile', help='AWS Profile Name') +@click.option('--deployment-id', help='Deployment Id') +@click.option('--module-set', help='Name of the ModuleSet. Default: default') +@click.argument('module', required=True) +def upload_packages(cluster_name: str, aws_region: str, aws_profile: str, deployment_id: str, module_set: str, module: str): + """ + upload applicable packages for a module + """ + CdkInvoker( + module_id=module, + module_set=module_set, + cluster_name=cluster_name, + aws_region=aws_region, + aws_profile=aws_profile, + deployment_id=deployment_id + ).invoke( + upload_bootstrap_package=True, + upload_release_package=True, + deploy_stack=False + ) + + +@click.command('patch') +@click.option('--cluster-name', required=True, help='Cluster Name') +@click.option('--aws-region', required=True, help='AWS Region') +@click.option('--aws-profile', help='AWS Profile Name') +@click.option('--package-uri', help='S3 package URI or package file path on local file system') +@click.option('--component', help='Component name') +@click.option('--instance-selector', help='Can be one of: [all, one]') +@click.option('--patch-command', help='Patch Command') +@click.option('--force', is_flag=True, help='Skip all confirmation prompts') +@click.argument('module', required=True) +def patch_module(cluster_name: str, aws_region: str, aws_profile: str, package_uri: str, component: str, instance_selector: str, force: bool, patch_command: str, module: str): + """ + patch application module with the current release + + only supported for modules with type = 'app' + """ + + PatchHelper( + cluster_name=cluster_name, + aws_region=aws_region, + aws_profile=aws_profile, + package_uri=package_uri, + component=component, + instance_selector=instance_selector, + module_id=module, + force=force, + patch_command=patch_command + ).apply() + + +@click.command('check-cluster-status') +@click.option('--cluster-name', required=True, help='Cluster Name') +@click.option('--aws-region', required=True, help='AWS Region') +@click.option('--aws-profile', help='AWS Profile Name') +@click.option('--wait', is_flag=True, help='Wait until all cluster endpoints are healthy.') +@click.option('--wait-timeout', default=900, help='Wait timeout in seconds. Default: 900 (15 mins)') +@click.option('--debug', is_flag=True, help='Print debug messages') +@click.option('--module-set', help='Name of the ModuleSet. Default: default') +def check_cluster_status(cluster_name: str, aws_region: str, aws_profile: str, wait: bool, wait_timeout: int, debug: bool, module_set: str): + """ + check status for all applicable cluster endpoints + """ + + def check_status(endpoint_url): + def get_status() -> bool: + try: + with warnings.catch_warnings(): + warnings.simplefilter('ignore') + result = requests.get(url=endpoint_url, verify=False) + if debug: + print(f'{endpoint_url} - {result.status_code} - {Utils.to_json(result.text)}') + if result.status_code == 200: + return True + else: + return False + except Exception as e: + if debug: + print(f'{e}') + return False + + return get_status + + cluster_config = ClusterConfig( + cluster_name=cluster_name, + aws_region=aws_region, + aws_profile=aws_profile, + module_set=module_set + ) + + context = SocaCliContext() + module_metadata = ModuleMetadataHelper() + + cluster_endpoint = cluster_config.get_cluster_external_endpoint() + cluster_modules = cluster_config.db.get_cluster_modules() + + endpoints = [] + + for cluster_module in cluster_modules: + module_id = cluster_module['module_id'] + module_name = cluster_module['name'] + module_type = cluster_module['type'] + if module_name == constants.MODULE_ANALYTICS: + url = f'{cluster_endpoint}/_dashboards/' + endpoints.append({ + 'name': 'OpenSearch Service Dashboard', + 'endpoint': url, + 'check_status': check_status(url) + }) + elif module_type == constants.MODULE_TYPE_APP: + url = f'{cluster_endpoint}/{module_id}/healthcheck' + endpoints.append({ + 'name': module_metadata.get_module_title(module_name), + 'endpoint': url, + 'check_status': check_status(url) + }) + + current_time = Utils.current_time_ms() + end_time = current_time + (wait_timeout * 1000) + fail_count = 0 + keyboard_interrupt = False + + while current_time < end_time: + + context.info(f'checking endpoint status for cluster: {cluster_name}, url: {cluster_endpoint} ...') + + table = PrettyTable(['Module', 'Endpoint', 'Status']) + table.align = 'l' + + fail_count = 0 + for endpoint in endpoints: + success = endpoint['check_status']() + if success: + status = 'SUCCESS' + else: + status = 'FAIL' + fail_count += 1 + + table.add_row([ + endpoint['name'], + endpoint['endpoint'], + status + ]) + + print(table) + + if not wait: + break + + if fail_count == 0: + break + + print('failed to verify all cluster endpoints. wait ... (Press Ctrl + C to exit) ') + try: + time.sleep(60) + except KeyboardInterrupt: + keyboard_interrupt = True + print('Check-endpoint status aborted.') + break + current_time = Utils.current_time_ms() + + if not keyboard_interrupt: + if wait and current_time >= end_time: + context.warning('check endpoint status timed-out. please verify your cluster\'s External ALB Security Group configuration and check correct ingress rules have been configured.') + + if fail_count > 0: + raise SystemExit(1) + + +@click.command() +@click.option('--cluster-name', required=True, help='Cluster Name') +@click.option('--aws-region', required=True, help='AWS Region') +@click.option('--aws-profile', help='AWS Profile Name') +@click.option('--deployment-id', help='Deployment Id') +@click.option('--force-build', is_flag=True, help='Delete and re-build bootstrap package if directory already exists.') +@click.option('--module-set', help='Name of the ModuleSet. Default: default') +@click.argument('module', required=True) +def build_bootstrap_package(cluster_name: str, aws_region: str, aws_profile: str, deployment_id: str, + force_build: bool, module_set: str, module: str): + """ + build bootstrap package for a module + """ + CdkInvoker( + module_id=module, + module_set=module_set, + cluster_name=cluster_name, + aws_region=aws_region, + aws_profile=aws_profile, + deployment_id=deployment_id + ).invoke( + render_bootstrap_package=True, + force_build_bootstrap=force_build, + upload_bootstrap_package=False, + upload_release_package=False, + deploy_stack=False + ) + + +@click.command() +@click.option('--cluster-name', required=True, help='Cluster Name') +@click.option('--aws-region', required=True, help='AWS Region') +@click.option('--aws-profile', help='AWS Profile Name') +def list_modules(cluster_name: str, aws_region: str, aws_profile: str): + """ + list all modules for a cluster + """ + db = ClusterConfigDB( + cluster_name=cluster_name, + aws_region=aws_region, + aws_profile=aws_profile + ) + modules = db.get_cluster_modules() + table = PrettyTable(['Title', 'Name', 'Module ID', 'Type', 'Stack Name', 'Version', 'Status']) + table.align = 'l' + for module in modules: + table.add_row([ + Utils.get_value_as_string('title', module), + Utils.get_value_as_string('name', module), + Utils.get_value_as_string('module_id', module), + Utils.get_value_as_string('type', module), + Utils.get_value_as_string('stack_name', module, '-'), + Utils.get_value_as_string('version', module, '-'), + Utils.get_value_as_string('status', module) + ]) + print(table) + + +@click.command() +@click.option('--cluster-name', required=True, help='Cluster Name') +@click.option('--aws-region', required=True, help='AWS Region') +@click.option('--aws-profile', help='AWS Profile Name') +@click.option('--module-set', help='Name of the ModuleSet. Default: default') +def show_connection_info(cluster_name: str, aws_region: str, aws_profile: str, module_set: str): + """ + print cluster connection information + """ + cluster_config = ClusterConfig( + cluster_name=cluster_name, + aws_region=aws_region, + aws_profile=aws_profile, + module_set=module_set + ) + + connection_info_entries = [] + + cluster_modules = cluster_config.db.get_cluster_modules() + + cluster_endpoint = cluster_config.get_cluster_external_endpoint() + + if Utils.is_not_empty(cluster_endpoint): + for cluster_module in cluster_modules: + module_name = cluster_module['name'] + module_id = cluster_module['module_id'] + module_status = cluster_module['status'] + if module_status != 'deployed': + continue + if module_name == constants.MODULE_CLUSTER_MANAGER: + connection_info_entries.append({ + 'key': 'Web Portal', + 'value': cluster_endpoint, + 'weight': 0 + }) + elif module_name == constants.MODULE_ANALYTICS: + connection_info_entries.append({ + 'key': 'Analytics Dashboard', + 'value': f'{cluster_endpoint}/_dashboards', + 'weight': 3 + }) + elif module_name == constants.MODULE_BASTION_HOST: + key_pair_name = cluster_config.get_string('cluster.network.ssh_key_pair') + ip_address = cluster_config.get_string(f'{module_id}.public_ip') + if Utils.is_empty(ip_address): + ip_address = cluster_config.get_string(f'{module_id}.private_ip') + if Utils.is_not_empty(ip_address): + base_os = cluster_config.get_string(f'{module_id}.base_os', required=True) + ec2_username = AdministratorUtils.get_ec2_username(base_os) + connection_info_entries.append({ + 'key': f'Bastion Host (SSH Access)', + 'value': f'ssh -i ~/.ssh/{key_pair_name}.pem {ec2_username}@{ip_address}', + 'weight': 1 + }) + + instance_id = cluster_config.get_string(f'{module_id}.instance_id') + if Utils.is_not_empty(instance_id): + aws_partition = cluster_config.get_string('cluster.aws.partition', required=True) + connection_manager_url = AdministratorUtils.get_session_manager_url(aws_partition, aws_region, instance_id) + connection_info_entries.append({ + 'key': f'Bastion Host (Session Manager URL)', + 'value': connection_manager_url, + 'weight': 2 + }) + + context = SocaCliContext() + + if len(connection_info_entries) > 0: + + connection_info_entries.sort(key=lambda x: x['weight']) + for entry in connection_info_entries: + key = entry['key'] + value = entry['value'] + context.print(f'{key}: {value}') + else: + context.error(f'No connection information found for cluster: {cluster_name}. Is the cluster deployed?') + + +@click.command() +def quick_setup_help(): + """ + display quick-setup help + """ + values_file = ideaadministrator.props.default_values_file + with open(values_file, 'r') as f: + content = f.read() + print(content) + + +@click.command() +@click.option('--values-file', help='path to values.yml file') +@click.option('--existing-resources', is_flag=True, help='Install IDEA using existing resources') +@click.option('--termination-protection', default=True, help='enable/disable termination protection for all stacks') +@click.option('--deployment-id', help='Deployment Id') +@click.option('--optimize-deployment', is_flag=True, help='If flag is provided, deployment will be optimized and applicable stacks will be deployed in parallel.') +@click.option('--force', is_flag=True, help='Skip all confirmation prompts') +@click.option('--skip-config', is_flag=True, help='Skip config generation and update steps. Assumes configuration tables are already created and configuration has already been synced. --values-file is required, when --skip-config flag is provided.') +@click.option('--rollback/--no-rollback', default=True, help='Rollback stack to stable state on failure. Defaults to "true", iterate more rapidly with --no-rollback.') +@click.option('--module-set', help='Name of the ModuleSet. Default: default') +@click.pass_context +def quick_setup(ctx, values_file: str, existing_resources: bool, termination_protection: bool, deployment_id: str, optimize_deployment: bool, force: bool, skip_config: bool, rollback: bool, module_set: str): + """ + Install a new cluster + """ + + cli = SocaCliContext() + if skip_config: + + if Utils.is_empty(values_file): + cli.error('--values-file is required when --skip-config flag is provided.') + raise SystemExit(1) + if not Utils.is_file(values_file): + cli.error(f'File not found: {values_file}') + raise SystemExit(1) + with open(values_file, 'r') as f: + values = Utils.from_yaml(f.read()) + + cluster_name = Utils.get_value_as_string('cluster_name', values) + aws_region = Utils.get_value_as_string('aws_region', values) + aws_profile = Utils.get_value_as_string('aws_profile', values) + + else: + + ctx.invoke(about) + + # generate config + values = ctx.invoke(config_generate, values_file=values_file, existing_resources=existing_resources, force=force) + + cluster_name = Utils.get_value_as_string('cluster_name', values) + aws_region = Utils.get_value_as_string('aws_region', values) + aws_profile = Utils.get_value_as_string('aws_profile', values) + + # update config + ctx.invoke(config_update, cluster_name=cluster_name, aws_region=aws_region, aws_profile=aws_profile, force=force) + + # print cluster configuration + ctx.invoke(show_config, cluster_name=cluster_name, aws_region=aws_region, aws_profile=aws_profile) + + # print module info + ctx.invoke(list_modules, cluster_name=cluster_name, aws_region=aws_region, aws_profile=aws_profile) + + if not force: + continue_deployment = cli.prompt(f'Are you sure you want to deploy above IDEA modules with applicable configuration settings?', default=True) + if not continue_deployment: + cli.info('Deployment aborted!') + raise SystemExit + + # todo - check required services using AwsServiceAvailabilityHelper before proceeding ahead with deployment + + # bootstrap cluster + ctx.invoke(bootstrap_cluster, cluster_name=cluster_name, aws_region=aws_region, aws_profile=aws_profile, + termination_protection=termination_protection) + + # deploy stacks + deployment_helper = DeploymentHelper( + cluster_name=cluster_name, + aws_region=aws_region, + termination_protection=termination_protection, + deployment_id=deployment_id, + module_set=module_set, + force_build_bootstrap=True, + optimize_deployment=optimize_deployment, + aws_profile=aws_profile, + all_modules=True, + upgrade=False + ) + module_ids = deployment_helper.get_deployment_order() + if len(module_ids) > 0: + if optimize_deployment: + deployment_order = deployment_helper.get_optimized_deployment_order() + else: + deployment_order = module_ids + with cli.spinner(f'deploying modules: {deployment_order}'): + ctx.invoke(deploy, + cluster_name=cluster_name, + aws_profile=aws_profile, + aws_region=aws_region, + termination_protection=termination_protection, + deployment_id=deployment_id, + rollback=rollback, + optimize_deployment=optimize_deployment, + modules=tuple(module_ids)) + else: + cli.info('all modules are already deployed. skipping deployment.') + + # wait for cluster endpoints to be healthy + ctx.invoke(check_cluster_status, cluster_name=cluster_name, aws_profile=aws_profile, aws_region=aws_region, wait=True, wait_timeout=30 * 60) + + # print module info + ctx.invoke(list_modules, cluster_name=cluster_name, aws_profile=aws_profile, aws_region=aws_region) + + # print connection information + cli.print_rule('Cluster Connection Info') + ctx.invoke(show_connection_info, cluster_name=cluster_name, aws_profile=aws_profile, aws_region=aws_region) + cli.print_rule() + + +@click.command() +@click.option('--cluster-name', required=True, help='Cluster Name') +@click.option('--aws-region', required=True, help='AWS Region') +@click.option('--aws-profile', help='AWS Profile Name') +@click.option('--delete-bootstrap', is_flag=True, help='Delete Bootstrap and S3 bucket') +@click.option('--delete-databases', is_flag=True, help='Delete Databases') +@click.option('--delete-backups', is_flag=True, help='Delete Backups') +@click.option('--delete-cloudwatch-logs', is_flag=True, help='Delete CloudWatch Logs') +@click.option('--delete-all', is_flag=True, help='Delete all') +@click.option('--force', is_flag=True, help='Skip confirmation prompts') +def delete_cluster(cluster_name: str, aws_region: str, aws_profile: str, delete_bootstrap: bool, delete_databases: bool, delete_backups: bool, delete_cloudwatch_logs: bool, delete_all: bool, force: bool): + """ + delete cluster + """ + + DeleteCluster( + cluster_name=cluster_name, + aws_region=aws_region, + aws_profile=aws_profile, + delete_bootstrap=delete_bootstrap, + delete_databases=delete_databases, + delete_backups=delete_backups, + delete_cloudwatch_logs=delete_cloudwatch_logs, + delete_all=delete_all, + force=force + ).invoke() + + +@click.command() +@click.option('--cluster-name', required=True, help='Cluster Name') +@click.option('--aws-region', required=True, help='AWS Region') +@click.option('--aws-profile', help='AWS Profile Name') +@click.option('--force', is_flag=True, help='Skip confirmation prompts') +def delete_backups(cluster_name: str, aws_region: str, aws_profile: str, force: bool): + """ + delete all recovery points in the cluster's backup vault + """ + + context = SocaCliContext() + confirm_delete_backups = force + if not force: + confirm_delete_backups = context.prompt(f'Are you sure you want to delete all the backup recovery points?') + + if not confirm_delete_backups: + return + + DeleteCluster( + cluster_name=cluster_name, + aws_region=aws_region, + aws_profile=aws_profile, + delete_bootstrap=False, + delete_databases=False, + delete_backups=True, + force=force + ).delete_backup_vault_recovery_points() + + +@click.command() +@click.option('--no-banner', is_flag=True, help='Do not print graphics and additional information') +def about(no_banner: bool): + """ + print IDEA release version info + """ + props = AdministratorProps() + dev_mode = props.is_dev_mode() + if no_banner: + tokens = [ + 'Integrated Digital Engineering on AWS (IDEA)', + f'Version: v{ideaadministrator.__version__}' + ] + if dev_mode: + tokens.append('(Developer Mode)') + print(', '.join(tokens)) + else: + context = SocaCliContext() + meta_info = [ + SocaKeyValue(key='Version', value=ideaadministrator.__version__) + ] + if dev_mode: + meta_info.append(SocaKeyValue(key='(Developer Mode)')) + context.print_banner(meta_info=meta_info) + + +@click.command() +@click.option('--cluster-name', required=True, help='Cluster Name') +@click.option('--aws-region', required=True, help='AWS Region') +@click.option('--aws-profile', help='AWS Profile Name') +@click.option('--admin-username', required=True, help='Cluster Administrator Username') +@click.option('--admin-password', required=True, help='Cluster Administrator Password') +@click.option('--test-case-id', help='Provide specific Test Case Id to execute. Multiple test case ids can be provided as a comma separated string.') +@click.option('--debug', is_flag=True, help='Enable debug logging') +@click.option('--param', '-p', help='Additional test case parameters used by individual test cases. Format: Key=param1,Value=value1', multiple=True) +@click.option('--module-set', help='Name of the ModuleSet. Default: default') +@click.argument('MODULES', required=True, nargs=-1) +def run_integration_tests(cluster_name: str, aws_region: str, aws_profile: str, admin_username: str, admin_password: str, + test_case_id: str, debug: bool, param: tuple, module_set: str, modules): + """ + run integration tests for a module + + examples: + + \b + * Run all scheduler test cases. All job test cases will be executed for the configured Compute Node OS and Compute Node AMI for the cluster + ./idea-admin.sh run-integration-tests scheduler \\ + --cluster-name idea-test1 \\ + --aws-region eu-west-1 \\ + --admin-username YOUR_ADMIN_USER \\ + --admin-password YOUR_ADMIN_PASSWORD + + \b + * Run all scheduler job submission test cases for all multiple base os + ./idea-admin.sh run-integration-tests scheduler \\ + --cluster-name idea-test1 \\ + --aws-region eu-west-1 \\ + --admin-username YOUR_ADMIN_USER \\ + --admin-password YOUR_ADMIN_PASSWORD \\ + --param base_os=amazonlinux2,centos7,rhel7 \\ + --test-case-id SCHEDULER_JOB_TEST_CASES + + \b + * Run scheduler job submission test cases: hello_world and custom_instance_type for base_os: amazonlinux2 with a custom ami + ./idea-admin.sh run-integration-tests scheduler \\ + --cluster-name idea-test1 \\ + --aws-region eu-west-1 \\ + --admin-username YOUR_ADMIN_USER \\ + --admin-password YOUR_ADMIN_PASSWORD \\ + --param job_test_cases=hello_world,custom_instance_type \\ + --param base_os=amazonlinux2:YOUR_CUSTOM_AMI \\ + --test-case-id SCHEDULER_JOB_TEST_CASES + """ + + # dedupe and convert to list + module_ids_to_test = [] + for module_id in modules: + if module_id in module_ids_to_test: + continue + module_ids_to_test.append(module_id) + + extra_params = {} + if Utils.is_not_empty(param): + for token in param: + kv = token.split('=', 1) + if len(kv) == 2: + key = kv[0] + value = kv[1] + extra_params[key] = value + + if not Utils.is_empty(test_case_id): + test_case_ids = test_case_id.split(',') + else: + test_case_ids = [] + + test_context = TestContext( + cluster_name=cluster_name, + aws_region=aws_region, + aws_profile=aws_profile, + admin_username=admin_username, + admin_password=admin_password, + debug=debug, + extra_params=extra_params, + test_case_ids=test_case_ids, + module_set=module_set, + module_ids=module_ids_to_test + ) + + test_invoker = TestInvoker( + test_context=test_context, + module_ids=module_ids_to_test + ) + + test_invoker.invoke() + + +@sso.command('show-idp-info', context_settings=CLICK_SETTINGS) +@click.option('--cluster-name', required=True, help='Cluster Name') +@click.option('--aws-region', required=True, help='AWS Region') +@click.option('--aws-profile', help='AWS Profile Name') +@click.option('--provider-type', required=True, help='Identity Provider Type. Can be one of [OIDC, SAML].') +def sso_show_idp_info(cluster_name: str, aws_region: str, aws_profile: str, provider_type: str): + """ + print single sign-on IDP redirect uri and related information for the cluster + + \b + When provider_type = OIDC: + Identity Provider Redirect URI = [cognito_domain_url]/oauth2/idpresponse + + \b + When provider_type = SAML: + Identity Provider Redirect URI = [cognito_domain_url]/saml2/idpresponse + Entity ID = urn:amazon:cognito:sp:[cognito_user_pool_id] + + Use this URI to configure your Identity Provider before you enable SSO for the cluster. + """ + sso_helper = SingleSignOnHelper( + cluster_name=cluster_name, + aws_region=aws_region, + aws_profile=aws_profile + ) + context = SocaCliContext() + + if provider_type.strip().upper() not in (constants.SSO_IDP_PROVIDER_SAML, constants.SSO_IDP_PROVIDER_OIDC): + context.error('Invalid provider type. Must be one of: [SAML, OIDC]') + raise SystemExit(1) + + context.print_title('Redirect URL') + context.print(sso_helper.get_idp_redirect_uri(provider_type=provider_type)) + + if provider_type == constants.SSO_IDP_PROVIDER_SAML: + context.print_title('Entity ID') + context.print(sso_helper.get_entity_id()) + + context.new_line() + context.info('Configure the above information in your IDP prior to running ./idea-admin.sh sso configure ...') + + +@sso.command('configure', context_settings=CLICK_SETTINGS) +@click.option('--cluster-name', required=True, help='Cluster Name') +@click.option('--aws-region', required=True, help='AWS Region') +@click.option('--aws-profile', help='AWS Profile Name') +@click.option('--provider-name', required=True, help='Identity Provider Name') +@click.option('--provider-type', required=True, help='Identity Provider Type. Can be one of [OIDC, SAML].') +@click.option('--provider-email-attribute', required=True, help='The name of the email attribute from Provider.') +@click.option('--refresh-token-validity-hours', type=int, help='Refresh token validity in hours. Default: 12') +@click.option('--oidc-client-id', help='OIDC Client Id.') +@click.option('--oidc-client-secret', help='OIDC Client Secret') +@click.option('--oidc-issuer', help='OIDC Issuer. The issuer URL you received from the OIDC provider.') +@click.option('--oidc-attributes-request-method', help='OIDC Attributes Request Method') +@click.option('--oidc-authorize-scopes', help='OIDC Authorize Scopes') +@click.option('--oidc-authorize-url', help='OIDC Authorize URL. The endpoint a user is redirected to at sign-in.') +@click.option('--oidc-token-url', help='OIDC Token URL. The endpoint Amazon Cognito uses to exchange the code received in a user\'s request for an ID token.') +@click.option('--oidc-attributes-url', help='OIDC Attributes URL. Also called as UserInfo endpoint. The userInfo endpoint is used by Cognito to retrieve information about the authenticated user. ') +@click.option('--oidc-jwks-uri', help='OIDC JWKS URI. The endpoint used to decode and verify tokens issued by the identity provider.') +@click.option('--saml-metadata-url', help='SAML Metadata URL') +@click.option('--saml-metadata-file', help='SAML Metadata File') +def sso_configure(cluster_name: str, aws_region: str, aws_profile: str, **kwargs): + """ + configure single sign-on for the cluster + """ + sso_helper = None + try: + sso_helper = SingleSignOnHelper( + cluster_name=cluster_name, + aws_region=aws_region, + aws_profile=aws_profile + ) + sso_helper.configure_sso(**kwargs) + except exceptions.SocaException as e: + if e.error_code == errorcodes.INVALID_PARAMS: + sso_helper.context.error(f'{e}') + else: + raise e + + +def print_using_default_warning(arg: str, path: str, cli: SocaCliContext = None): + if cli is None: + cli = SocaCliContext() + cli.warning(f'WARNING: {arg} was not specified; Using default location: {path}') + + +def get_bucket_name(cluster_name: str, aws_region: str, + cluster_config_db: ClusterConfigDB, context: SocaCliContext): + config_entry = cluster_config_db.get_config_entry('cluster.cluster_s3_bucket') + bucket_name = None + if Utils.is_not_empty(config_entry): + bucket_name = config_entry['value'] + if Utils.is_empty(bucket_name): + aws_account_id = context.aws().aws_account_id() + bucket_name = f'{cluster_name}-cluster-{aws_region}-{aws_account_id}' + return bucket_name + + +@vpc_endpoints.command('service-info', context_settings=CLICK_SETTINGS) +@click.option('--aws-region', required=True, help='AWS Region') +@click.option('--aws-profile', help='AWS Profile Name') +def vpc_endpoints_service_info(aws_region: str, aws_profile: str): + """ + print available vpc endpoint services in the region + """ + vpc_endpoint_helper = VpcEndpointsHelper( + aws_region=aws_region, + aws_profile=aws_profile + ) + vpc_endpoint_helper.print_vpc_endpoint_services() + + +@utils.command(context_settings=CLICK_SETTINGS) +def aws_services(): + """ + print all AWS services (required and optional) used by IDEA + """ + helper = AwsServiceAvailabilityHelper() + helper.print_idea_services() + + +@utils.command('check-aws-services', context_settings=CLICK_SETTINGS) +@click.option('--aws-profile', help='AWS Profile Name') +@click.argument('AWS_REGION', nargs=-1, required=True) +def check_aws_services(aws_profile: str, aws_region): + """ + check and print availability of AWS services required by IDEA for a given AWS region + """ + helper = AwsServiceAvailabilityHelper(aws_profile=aws_profile) + helper.print_availability_matrix(aws_regions=list(aws_region)) + + +@cluster_prefix_list.command('show', context_settings=CLICK_SETTINGS) +@click.option('--cluster-name', required=True, help='Cluster Name') +@click.option('--aws-region', required=True, help='AWS Region') +@click.option('--aws-profile', help='AWS Profile Name') +def cluster_prefix_list_show(cluster_name: str, aws_region: str, aws_profile: str): + """ + print all CIDR entries in the cluster prefix list + """ + entries = ClusterPrefixListHelper( + cluster_name=cluster_name, + aws_region=aws_region, + aws_profile=aws_profile + ).list_entries() + + table = PrettyTable(['CIDR', 'Description']) + table.align = 'l' + for entry in entries: + cidr = Utils.get_value_as_string('cidr', entry) + description = Utils.get_value_as_string('description', entry, '-') + table.add_row([cidr, description]) + print(table) + + +@cluster_prefix_list.command('add-entry', context_settings=CLICK_SETTINGS) +@click.option('--cluster-name', required=True, help='Cluster Name') +@click.option('--aws-region', required=True, help='AWS Region') +@click.option('--aws-profile', help='AWS Profile Name') +@click.option('--cidr', required=True, help='CIDR Entry') +@click.option('--description', required=True, help='CIDR Entry Description') +def cluster_prefix_list_add_entry(cluster_name: str, aws_region: str, aws_profile: str, cidr: str, description: str): + """ + add CIDR entry to cluster prefix list + """ + ClusterPrefixListHelper( + cluster_name=cluster_name, + aws_region=aws_region, + aws_profile=aws_profile + ).add_entry(cidr=cidr, description=description) + + +@cluster_prefix_list.command('remove-entry', context_settings=CLICK_SETTINGS) +@click.option('--cluster-name', required=True, help='Cluster Name') +@click.option('--aws-region', required=True, help='AWS Region') +@click.option('--aws-profile', help='AWS Profile Name') +@click.option('--cidr', required=True, help='CIDR Entry') +def cluster_prefix_list_remove_entry(cluster_name: str, aws_region: str, aws_profile: str, cidr: str): + """ + remove CIDR entry from cluster prefix list + """ + ClusterPrefixListHelper( + cluster_name=cluster_name, + aws_region=aws_region, + aws_profile=aws_profile + ).remove_entry(cidr=cidr) + + +@support.command('deployment', context_settings=CLICK_SETTINGS) +@click.option('--cluster-name', required=True, help='Cluster Name') +@click.option('--aws-region', required=True, help='AWS Region') +@click.option('--aws-profile', help='AWS Profile Name') +@click.option('--module-set', help='Name of the ModuleSet. Default: default') +def support_deployment(cluster_name: str, aws_region: str, aws_profile: str, module_set: str): + """ + build deployment support debug package + """ + SupportHelper( + cluster_name=cluster_name, + aws_region=aws_region, + aws_profile=aws_profile, + module_set=module_set + ).invoke() + + +@directoryservice.command('create-service-account-secrets', context_settings=CLICK_SETTINGS) +@click.option('--cluster-name', required=True, help='Cluster Name') +@click.option('--aws-region', required=True, help='AWS Region') +@click.option('--aws-profile', help='AWS Profile Name') +@click.option('--username', help='Service Account Username') +@click.option('--password', help='Service Account Password') +@click.option('--kms-key-id', help='KMS Key ID') +@click.option('--purpose', help='Account Purpose (e.g. service-account, clusteradmin)') +def ds_create_service_secrets(cluster_name: str, aws_region: str, aws_profile: str, + username: str, password: str, kms_key_id: str, purpose: str): + """ + create service account secrets for directory service + """ + + context = SocaCliContext() + + helper = DirectoryServiceHelper( + cluster_name=cluster_name, + aws_region=aws_region, + aws_profile=aws_profile + ) + + if Utils.is_any_empty(username, password): + ask_result = context.ask( + title='Directory Service - Service Account Credentials', + description='Enter the account credentials to create secrets in AWS Secrets Manager with applicable tags', + questions=[ + SocaUserInputParamMetadata( + name='purpose', + title='Purpose', + description=f"Enter account purpose", + data_type='str', + param_type=SocaUserInputParamType.SELECT, + multiple=False, + choices=[ + SocaUserInputChoice(title='clusteradmin - Used for initial login/configuration of the IDEA cluster', value='clusteradmin'), + SocaUserInputChoice(title='service-account - Used for binding to the Active Directory/LDAP', value='service-account') + ], + default=purpose, + validate=SocaUserInputValidate(required=True) + ), + SocaUserInputParamMetadata( + name='username', + title='Username', + description='Enter account username', + data_type='str', + param_type=SocaUserInputParamType.TEXT, + default=username, + validate=SocaUserInputValidate(required=True) + ), + SocaUserInputParamMetadata( + name='password', + title='Password', + description='Enter account password', + data_type='str', + param_type=SocaUserInputParamType.PASSWORD, + default=username, + validate=SocaUserInputValidate(required=True) + ) + ]) + purpose = ask_result['purpose'] + username = ask_result['username'] + password = ask_result['password'] + + with context.spinner(f'creating {purpose} secrets ...'): + secret_arns = helper.create_service_account_secrets( + purpose=purpose, + username=username, + password=password, + kms_key_id=kms_key_id + ) + + username_secret_arn = secret_arns['username_secret_arn'] + password_secret_arn = secret_arns['password_secret_arn'] + + context.success(f'directory service {purpose} secrets created successfully: ') + print(f'Account Purpose: {purpose}') + print(f'Username Secret ARN: {username_secret_arn}') + print(f'Password Secret ARN: {password_secret_arn}') + + + + + + +@shared_storage.command('add-file-system', context_settings=CLICK_SETTINGS) +@click.option('--cluster-name', help='Cluster Name') +@click.option('--aws-region', required=True, help='AWS Region') +@click.option('--aws-profile', help='AWS Profile Name') +@click.option('--kms-key-id', help='KMS Key ID') +def add_file_system(cluster_name: str, aws_region: str, aws_profile: str, kms_key_id: str): + """ + add new shared-storage file-system + """ + SharedStorageHelper( + cluster_name=cluster_name, + aws_region=aws_region, + aws_profile=aws_profile, + kms_key_id=kms_key_id + ).add_file_system(use_existing_fs=False) + + +@shared_storage.command('attach-file-system', context_settings=CLICK_SETTINGS) +@click.option('--cluster-name', help='Cluster Name') +@click.option('--aws-region', required=True, help='AWS Region') +@click.option('--aws-profile', help='AWS Profile Name') +@click.option('--kms-key-id', help='KMS Key ID') +def attach_file_system(cluster_name: str, aws_region: str, aws_profile: str, kms_key_id: str): + """ + attach existing shared-storage file-system + """ + SharedStorageHelper( + cluster_name=cluster_name, + aws_region=aws_region, + aws_profile=aws_profile, + kms_key_id=kms_key_id + ).add_file_system(use_existing_fs=True) + + +main.add_command(deploy) +main.add_command(cdk) +main.add_command(config) +main.add_command(bootstrap_cluster) +main.add_command(upload_packages) +main.add_command(list_modules) +main.add_command(build_bootstrap_package) +main.add_command(show_connection_info) +main.add_command(quick_setup_help) +main.add_command(quick_setup) +main.add_command(delete_cluster) +main.add_command(patch_module) +main.add_command(check_cluster_status) +main.add_command(about) +main.add_command(run_integration_tests) +main.add_command(sso) +main.add_command(utils) +main.add_command(support) +main.add_command(directoryservice) +main.add_command(shared_storage) + + +def main_wrapper(): + success = True + exit_code = 0 + + try: + + args = sys.argv[1:] + + has_params = False + if len(args) > 0: + if len(args) == 1: + param = args[0] + has_params = Utils.is_not_empty(param) + elif len(args) > 1: + has_params = True + + if has_params: + main(args) + else: + main(['-h']) + + except exceptions.SocaException as e: + if e.error_code == errorcodes.USER_INPUT_FLOW_INTERRUPT: + pass + elif e.error_code == errorcodes.CLUSTER_CONFIG_NOT_INITIALIZED: + click.secho(f'{e}. Is the cluster configuration synced?', fg='red', bold=True) + success = False + elif e.error_code in (errorcodes.CONFIG_ERROR, + errorcodes.USER_INPUT_FLOW_ERROR, + errorcodes.INTEGRATION_TEST_FAILED): + click.secho(f'{e}', fg='red', bold=True) + success = False + else: + raise e + except botocore.exceptions.ProfileNotFound as e: + click.secho(f'{e}', fg='red', bold=True) + success = False + except SystemExit as e: + success = e.code == 0 + exit_code = e.code + except Exception as e: + click.secho(f'Command failed with error: {e}', fg='red', bold=True) + raise e + + if not success: + if exit_code == 0: + exit_code = 1 + sys.exit(exit_code) + + +# used only for local testing +if __name__ == '__main__': + main_wrapper() diff --git a/source/idea/idea-administrator/src/ideaadministrator/app_messages.py b/source/idea/idea-administrator/src/ideaadministrator/app_messages.py new file mode 100644 index 00000000..6b8e6168 --- /dev/null +++ b/source/idea/idea-administrator/src/ideaadministrator/app_messages.py @@ -0,0 +1,29 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +AWS_CLI_NOT_INSTALLED = """ +# AWS Profile not found! + +We could not find any AWS profiles to use for SOCA installation. Please ensure AWS cli is installed and configured. + +To install AWS CLI, run: +```bash +$ pip3 install awscli +``` + +If AWS CLI is installed, please ensure it is configured to use an AWS Account. To configure aws cli, run: +```bash +$ aws configure +``` + +Refer to AWS CLI Docs (https://awscli.amazonaws.com/v2/documentation/api/latest/reference/index.html) for more information. + +""" diff --git a/source/idea/idea-administrator/src/ideaadministrator/app_props.py b/source/idea/idea-administrator/src/ideaadministrator/app_props.py new file mode 100644 index 00000000..c45b65e2 --- /dev/null +++ b/source/idea/idea-administrator/src/ideaadministrator/app_props.py @@ -0,0 +1,311 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +from ideasdk.utils import Utils, EnvironmentUtils +from ideadatamodel import exceptions + +from ideaadministrator import app_constants +import ideaadministrator_meta + +from typing import Dict, Optional, List +from pathlib import Path +import os + + +class AdministratorProps: + + @staticmethod + def is_dev_mode() -> bool: + return Utils.get_as_bool(EnvironmentUtils.idea_dev_mode(), False) + + @property + def current_release_version(self) -> str: + return ideaadministrator_meta.__version__ + + @property + def dev_mode_admin_root_dir(self) -> str: + script_dir = Path(os.path.abspath(__file__)) + return str(script_dir.parent.parent.parent) + + @property + def dev_mode_admin_app_source_dir(self) -> str: + return os.path.join(self.dev_mode_admin_root_dir, 'src') + + @property + def dev_mode_resources_dir(self) -> str: + return os.path.join(self.dev_mode_admin_root_dir, 'resources') + + @property + def dev_mode_project_root_dir(self) -> str: + script_dir = Path(os.path.abspath(__file__)) + return str(script_dir.parent.parent.parent.parent.parent.parent) + + @property + def dev_mode_project_build_dir(self) -> str: + return os.path.join(self.dev_mode_project_root_dir, 'build') + + @property + def dev_mode_project_dist_dir(self) -> str: + return os.path.join(self.dev_mode_project_root_dir, 'dist') + + @property + def dev_mode_sdk_source_root(self) -> str: + return os.path.join(self.dev_mode_project_root_dir, 'source', 'idea', 'idea-sdk', 'src') + + @property + def dev_mode_bootstrap_source_dir(self) -> str: + return os.path.join(self.dev_mode_project_root_dir, 'source', 'idea', 'idea-bootstrap') + + @property + def dev_mode_data_model_src(self) -> str: + return os.path.join(self.dev_mode_project_root_dir, 'source', 'idea', 'idea-data-model', 'src') + + @property + def dev_mode_site_packages(self) -> str: + virtual_env = EnvironmentUtils.get_environment_variable('VIRTUAL_ENV', required=True) + return os.path.join(virtual_env, 'lib', 'python3.7', 'site-packages') + + @property + def dev_mode_venv_bin_dir(self) -> str: + virtual_env = EnvironmentUtils.get_environment_variable('VIRTUAL_ENV', required=True) + return os.path.join(virtual_env, 'bin') + + @property + def resources_dir(self) -> str: + if self.is_dev_mode(): + return self.dev_mode_resources_dir + else: + return os.path.join(self.soca_admin_dir, 'resources') + + @property + def lambda_function_commons_package_name(self) -> str: + return 'idea_lambda_commons' + + @property + def lambda_function_commons_dir(self) -> str: + return os.path.join(self.lambda_functions_dir, self.lambda_function_commons_package_name) + + @property + def lambda_functions_dir(self) -> str: + return os.path.join(self.resources_dir, 'lambda_functions') + + @property + def policy_templates_dir(self) -> str: + return os.path.join(self.resources_dir, 'policies') + + @property + def user_data_template_dir(self) -> str: + return os.path.join(self.resources_dir, 'userdata') + + @property + def installer_config_defaults_file(self) -> str: + return os.path.join(self.resources_dir, app_constants.INSTALLER_CONFIG_DEFAULTS_FILE) + + @property + def default_values_file(self) -> str: + return os.path.join(self.resources_dir, 'config', 'values.yml') + + @property + def install_params_file(self) -> str: + return os.path.join(self.resources_dir, 'input_params', 'install_params.yml') + + @property + def shared_storage_params_file(self) -> str: + return os.path.join(self.resources_dir, 'input_params', 'shared_storage_params.yml') + + @property + def aws_endpoints_file(self) -> str: + return os.path.join(self.resources_dir, app_constants.AWS_ENDPOINTS_FILE) + + @property + def cluster_default_config_dir(self) -> str: + return os.path.join(self.resources_dir, 'config') + + @property + def cluster_config_templates_dir(self) -> str: + return os.path.join(self.cluster_default_config_dir, 'templates') + + @property + def cluster_config_schema_dir(self) -> str: + return os.path.join(self.cluster_default_config_dir, 'schema') + + @property + def idea_user_home(self) -> str: + return os.path.expanduser(os.path.join('~', '.idea')) + + @property + def soca_build_dir(self) -> str: + build_dir = os.path.join(self.idea_user_home, 'build') + os.makedirs(build_dir, exist_ok=True) + return build_dir + + @property + def soca_downloads_dir(self) -> str: + downloads_dir = os.path.join(self.idea_user_home, 'downloads') + os.makedirs(downloads_dir, exist_ok=True) + return downloads_dir + + @property + def app_name(self) -> str: + return ideaadministrator_meta.__name__ + + @property + def app_version(self) -> str: + return ideaadministrator_meta.__version__ + + @property + def soca_admin_dir(self) -> str: + home_dir = self.idea_user_home + admin_dir = os.path.join(home_dir, self.app_name) + os.makedirs(admin_dir, exist_ok=True) + return admin_dir + + @property + def soca_admin_logs_dir(self) -> str: + logs_dir = os.path.join(self.idea_user_home, 'logs', self.app_name) + if not Utils.is_dir(logs_dir): + os.makedirs(logs_dir, exist_ok=True) + return logs_dir + + def cluster_dir(self, cluster_name: str, create=True) -> str: + idea_user_home = self.idea_user_home + cluster_home = os.path.join(idea_user_home, 'clusters', cluster_name) + if create: + os.makedirs(cluster_home, exist_ok=True) + return cluster_home + + @staticmethod + def cluster_region_dir(cluster_home: str, aws_region: str, create=True) -> str: + region_dir = os.path.join(cluster_home, aws_region) + if create and not os.path.isdir(region_dir): + os.makedirs(region_dir, exist_ok=True) + return region_dir + + def values_file(self, cluster_name: str, aws_region: str, create=False) -> str: + cluster_home = self.cluster_dir(cluster_name, create=create) + region_dir = self.cluster_region_dir(cluster_home, aws_region, create=create) + return os.path.join(region_dir, 'values.yml') + + def region_ami_config_file(self) -> str: + return os.path.join(self.resources_dir, 'config', 'region_ami_config.yml') + + def region_timezone_config_file(self) -> str: + return os.path.join(self.resources_dir, 'config', 'region_timezone_config.yml') + + def region_elb_account_id_file(self) -> str: + return os.path.join(self.resources_dir, 'config', 'region_elb_account_id.yml') + + def cluster_cdk_dir(self, cluster_name: str, aws_region: str) -> str: + cluster_home = self.cluster_dir(cluster_name) + region_dir = self.cluster_region_dir(cluster_home, aws_region) + cdk_home = os.path.join(region_dir, '_cdk') + os.makedirs(cdk_home, exist_ok=True) + return cdk_home + + def cluster_logs_dir(self, cluster_name: str, aws_region: str) -> str: + cluster_home = self.cluster_dir(cluster_name) + region_dir = self.cluster_region_dir(cluster_home, aws_region) + logs_dir = os.path.join(region_dir, 'logs') + os.makedirs(logs_dir, exist_ok=True) + return logs_dir + + def cluster_deployments_dir(self, cluster_name: str, aws_region: str) -> str: + cluster_home = self.cluster_dir(cluster_name) + region_dir = self.cluster_region_dir(cluster_home, aws_region) + deployments_dir = os.path.join(region_dir, 'deployments') + os.makedirs(deployments_dir, exist_ok=True) + return deployments_dir + + def cluster_support_dir(self, cluster_name: str, aws_region: str) -> str: + cluster_home = self.cluster_dir(cluster_name) + region_dir = self.cluster_region_dir(cluster_home, aws_region) + support_dir = os.path.join(region_dir, 'support') + os.makedirs(support_dir, exist_ok=True) + return support_dir + + def cluster_config_dir(self, cluster_name: str, aws_region: str) -> str: + cluster_home = self.cluster_dir(cluster_name) + region_dir = self.cluster_region_dir(cluster_home, aws_region) + config_dir = os.path.join(region_dir, 'config') + os.makedirs(config_dir, exist_ok=True) + return config_dir + + @property + def cdk_bin(self) -> str: + if os.name == 'nt': + # for windows, cdk.cmd needs to be invoked, which is located at a different place. + cdk_bin = os.path.join(self.idea_user_home, 'lib', 'idea-cdk', 'node_modules', '.bin', 'cdk') + else: + cdk_bin = os.path.join(self.idea_user_home, 'lib', 'idea-cdk', 'node_modules', 'aws-cdk', 'bin', 'cdk') + if not os.path.isfile(cdk_bin): + raise exceptions.general_exception(f'Unable to find cdk binary at: {cdk_bin}.' + f' Please ensure {self.app_name} is installed correctly or re-run installation.') + return cdk_bin + + def get_env(self) -> Dict[str, str]: + env = os.environ.copy() + if not self.is_dev_mode(): + return env + + path = Utils.get_value_as_string('PATH', env) + paths = path.split(os.pathsep) + + admin_app_sources = [ + self.dev_mode_venv_bin_dir, + self.dev_mode_site_packages, + self.dev_mode_data_model_src, + self.dev_mode_sdk_source_root, + self.dev_mode_admin_app_source_dir, + self.dev_mode_project_root_dir + ] + + for admin_path in admin_app_sources: + paths.insert(0, admin_path) + + env['PATH'] = os.pathsep.join(paths) + return env + + def get_get_cdk_app_cmd(self, config_file: str) -> str: + """ + Returns the command to build the CDK app + when dev_mode is true, invoke automation is used to ensure sources instead of calling idea-admin + :return: + """ + if self.is_dev_mode(): + return f'invoke ' \ + f'--search-root "{self.dev_mode_project_root_dir}" ' \ + f'cli.admin --args "cdk cdk-app --config-file {config_file}"' + else: + return f'idea-admin cdk cdk-app --config-file "{config_file}"' + + def get_cdk_command(self, name: str, stacks: str = None, params: Optional[List[str]] = None, rollback=True) -> str: + """ + build the cdk command. eg: + $ cdk deploy --app "idea-admin cdk-app -c user-config.json [--additional_cdk_params] + """ + if params is None: + params = [] + cmd = [self.cdk_bin] + name.split(' ') + params + if name == 'deploy': + cmd.append(f'--rollback {str(rollback).lower()}') + if stacks is not None: + cmd.append(stacks) + return ' '.join(cmd) + + @staticmethod + def use_ec2_instance_metadata_credentials() -> bool: + """ + read an IDEA specific environment variable, indicating if administrator app is running within an ec2 instance. + """ + aws_credential_provider = EnvironmentUtils.get_environment_variable('IDEA_ADMIN_AWS_CREDENTIAL_PROVIDER', default=None) + if Utils.is_empty(aws_credential_provider): + return False + return aws_credential_provider == 'Ec2InstanceMetadata' diff --git a/source/idea/idea-administrator/src/ideaadministrator/app_protocols.py b/source/idea/idea-administrator/src/ideaadministrator/app_protocols.py new file mode 100644 index 00000000..fb3da209 --- /dev/null +++ b/source/idea/idea-administrator/src/ideaadministrator/app_protocols.py @@ -0,0 +1,23 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +from ideasdk.context import SocaCliContext, SocaContextOptions +from abc import abstractmethod + + +class AdministratorContextProtocol(SocaCliContext): + + def __init__(self, options: SocaContextOptions = None): + super().__init__(options=options) + + @abstractmethod + def get_stack_name(self, module: str) -> str: + ... diff --git a/source/idea/idea-administrator/src/ideaadministrator/app_utils.py b/source/idea/idea-administrator/src/ideaadministrator/app_utils.py new file mode 100644 index 00000000..bb3e9f9a --- /dev/null +++ b/source/idea/idea-administrator/src/ideaadministrator/app_utils.py @@ -0,0 +1,219 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +from ideasdk.utils import Utils, Jinja2Utils +from ideadatamodel import constants, exceptions, SocaAnyPayload +from ideasdk.config.cluster_config import ClusterConfig +from ideasdk.context.arn_builder import ArnBuilder + +import ideaadministrator +from ideaadministrator import app_constants + +from typing import Optional, List, Tuple, Dict +import requests +import requests.exceptions +import os +import shutil +import yaml.parser + + +class AdministratorUtils: + + @staticmethod + def detect_client_ip() -> Optional[str]: + """ + Try to determine the IPv4 of the customer. + If IP cannot be determined we will prompt the user to enter the value manually + :return: IP Address or None + """ + try: + requests.packages.urllib3.util.connection.HAS_IPV6 = False # noqa + http_result = requests.get('https://checkip.amazonaws.com/', timeout=15) + if http_result.status_code != 200: + return None + + ip_address = str(http_result.text).strip() + return f'{ip_address}/32' + + except requests.exceptions.RequestException as e: + raise e + + @staticmethod + def sanitize_cluster_name(value: Optional[str]) -> str: + if Utils.is_empty(value): + return '' + token = value.strip().lower() + if token.startswith(app_constants.CLUSTER_NAME_PREFIX): + token = token.replace(app_constants.CLUSTER_NAME_PREFIX, '') + if Utils.is_empty(token): + return '' + return f'{app_constants.CLUSTER_NAME_PREFIX}{token}' + + @staticmethod + def get_ssh_connection_string(os_: str, ip_address: str, keypair_name: str) -> Optional[str]: + # todo - need to check if on windows, .exe or something else needs to be handled + user = AdministratorUtils.get_ec2_username(os_) + keypair_path = f'~/.ssh/{keypair_name}.pem' + return str(f'ssh ' + f'-i {keypair_path} ' + f'{user}@{ip_address}') + + @staticmethod + def get_ec2_username(os_: str) -> str: + if os_ == constants.OS_CENTOS7: + return 'centos7' + else: + return 'ec2-user' + + @staticmethod + def get_session_manager_url(aws_partition: str, aws_region: str, instance_id: str) -> str: + # todo - implement this for other partitions (aws-iso, aws-iso-b) + + # console_prefix - comes before 'console' in the URI - including the '.' + # console_suffix - comes after 'console' in the URI - including '.' + console_prefix = '' + console_suffix = '.aws.amazon.com' + + if aws_partition == 'aws-cn': + console_prefix = '' + console_suffix = '.amazonaws.cn' + elif aws_partition == 'aws-us-gov': + console_prefix = '' + console_suffix = '.amazonaws-us-gov.com' + else: + console_prefix = f'{aws_region}.' + console_suffix = '.aws.amazon.com' + + return str(f'https://{console_prefix}console{console_suffix}' + f'/systems-manager/session-manager' + f'/{instance_id}?region={aws_region}') + + @staticmethod + def get_package_build_dir() -> str: + if ideaadministrator.props.is_dev_mode(): + return ideaadministrator.props.dev_mode_project_build_dir + else: + return ideaadministrator.props.soca_build_dir + + @staticmethod + def get_package_dist_dir() -> str: + if ideaadministrator.props.is_dev_mode(): + return ideaadministrator.props.dev_mode_project_dist_dir + else: + return ideaadministrator.props.soca_downloads_dir + + @staticmethod + def get_packages_to_upload() -> List[str]: + """ + if SOCA_DEV_MODE=true, find packages from dist dir + else, find packages in ~/.idea/downloads to be uploaded + :return: + """ + + package_dist_dir = AdministratorUtils.get_package_dist_dir() + + packages_to_upload = [] + + files = os.listdir(package_dist_dir) + for file in files: + if not file.endswith('tar.gz'): + continue + if 'docs' in file: + continue + if file.startswith('idea-administrator'): + continue + if file.startswith('all-'): + continue + if ideaadministrator.props.current_release_version not in file: + continue + packages_to_upload.append(os.path.join(package_dist_dir, file)) + + return packages_to_upload + + @staticmethod + def cluster_region_dir_exists_and_has_config(directory: str) -> bool: + """ + check if cluster region dir has applicable configuration files/folders + :param directory: path to cluster region dir + :return: + """ + if not os.path.isdir(directory): + return False + + config_dir = os.path.join(directory, 'config') + if Utils.is_dir(config_dir): + return True + + values_file = os.path.join(directory, 'values.yml') + if Utils.is_file(values_file): + return True + + return False + + @staticmethod + def cleanup_cluster_region_dir(directory: str, preserve_values_file: bool = False, scope: Tuple[str] = ('config', 'values.yml')): + """ + Delete config generation related files and directories if the directory already exists and is not empty + :param directory: + :param preserve_values_file: + :param scope: defaults to config/ and values.yml + :return: + """ + if not os.path.isdir(directory): + raise exceptions.file_not_found(f'{directory} not found') + + files = os.listdir(directory) + for file_name in files: + if file_name not in scope: + continue + + file_path = os.path.join(directory, file_name) + if os.path.isdir(file_path): + shutil.rmtree(file_path) + elif os.path.isfile(file_path): + if preserve_values_file: + continue + os.remove(file_path) + + @staticmethod + def render_policy(policy_template_name: str, + cluster_name: str, + module_id: str, + config: ClusterConfig, + vars: SocaAnyPayload = None) -> Dict: # noqa + env = Jinja2Utils.env_using_file_system_loader(search_path=ideaadministrator.props.policy_templates_dir) + policy_template = env.get_template(policy_template_name) + + if vars is None: + vars = SocaAnyPayload() # noqa + + policy_content = policy_template.render(context={ + 'cluster_name': cluster_name, + 'module_id': module_id, + 'aws_region': config.get_string('cluster.aws.region', required=True), + 'aws_dns_suffix': config.get_string('cluster.aws.dns_suffix', required=True), + 'aws_partition': config.get_string('cluster.aws.partition', required=True), + 'aws_account_id': config.get_string('cluster.aws.account_id', required=True), + 'config': config, + 'arns': ArnBuilder(config=config), + 'vars': vars, + 'utils': Utils + }) + + try: + return Utils.from_yaml(policy_content) + except yaml.parser.ParserError as e: + lines = policy_content.splitlines() + numbered_lines = [] + for index, line in enumerate(lines): + numbered_lines.append('{:>5}: {}'.format(str(index+1), line)) + print(f'failed to decode policy json: {policy_template_name} - {e}. Content: {os.linesep}{os.linesep.join(numbered_lines)}') + raise e diff --git a/source/idea/idea-administrator/src/ideaadministrator/integration_tests/__init__.py b/source/idea/idea-administrator/src/ideaadministrator/integration_tests/__init__.py new file mode 100644 index 00000000..6d8d18a3 --- /dev/null +++ b/source/idea/idea-administrator/src/ideaadministrator/integration_tests/__init__.py @@ -0,0 +1,10 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. diff --git a/source/idea/idea-administrator/src/ideaadministrator/integration_tests/cluster_manager_tests.py b/source/idea/idea-administrator/src/ideaadministrator/integration_tests/cluster_manager_tests.py new file mode 100644 index 00000000..9f0d3b1d --- /dev/null +++ b/source/idea/idea-administrator/src/ideaadministrator/integration_tests/cluster_manager_tests.py @@ -0,0 +1,364 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +from ideadatamodel import ( + AuthResult, + User, + CreateUserRequest, + CreateUserResult, + GetUserRequest, + GetUserResult, + GetGroupRequest, + GetGroupResult, + InitiateAuthRequest, + InitiateAuthResult, + ChangePasswordRequest, + ChangePasswordResult, + ListUsersRequest, + ListUsersResult, + CreateProjectRequest, + CreateProjectResult, + Project, + SocaKeyValue, + EnableProjectRequest, + GetProjectRequest, + GetProjectResult, + UpdateProjectRequest, + ListProjectsRequest, + ListProjectsResult, + DeleteProjectRequest, + DeleteProjectResult, + DeleteUserRequest, + DeleteUserResult +) +from ideasdk.utils import Utils, GroupNameHelper +from ideadatamodel import exceptions + +from ideaadministrator.integration_tests.test_context import TestContext +from ideaadministrator.integration_tests import test_constants + +from typing import Optional + +user_auth: Optional[AuthResult] = None +test_username = None +test_password = None +user: Optional[User] = None + + +def test_admin_create_user(context: TestContext): + global test_username + global test_password + test_username = Utils.generate_password(8, 0, 8, 0, 0) + test_password = Utils.generate_password(8, 1, 1, 1, 1) + email = f'{test_username}@samplesocauser.local' + result = context.get_cluster_manager_client().invoke_alt( + namespace='Accounts.CreateUser', + payload=CreateUserRequest( + user=User( + username=test_username, + password=test_password, + email=email, + additional_groups=["managers-cluster-group"], + ), + email_verified=True + ), + result_as=CreateUserResult, + access_token=context.get_admin_access_token() + ) + global user + user = result.user + assert Utils.is_not_empty(user) + assert Utils.is_not_empty(user.username) + assert Utils.are_equal(test_username, user.username) + assert Utils.are_equal(email, user.email) + assert Utils.are_equal(user.status, 'CONFIRMED') + assert Utils.is_false(user.sudo) + assert Utils.is_true(user.enabled) + assert Utils.is_not_empty(user.uid) + assert Utils.is_not_empty(user.gid) + assert Utils.is_not_empty(user.group_name) + assert Utils.is_not_empty(user.login_shell) + assert Utils.are_equal(user.login_shell, '/bin/bash') + assert Utils.are_equal(user.home_dir, f'/data/home/{test_username}') + + +def test_admin_get_user(context: TestContext): + result = context.get_cluster_manager_client().invoke_alt( + namespace='Accounts.GetUser', + payload=GetUserRequest( + username=test_username + ), + result_as=GetUserResult, + access_token=context.get_admin_access_token() + ) + + assert result.user is not None + assert result.user.username == test_username + + +def test_get_user_group(context: TestContext): + assert test_username is not None + group_name = GroupNameHelper(context.idea_context).get_user_group(str(test_username)) + result = context.get_cluster_manager_client().invoke_alt( + namespace='Accounts.GetGroup', + payload=GetGroupRequest( + group_name=group_name + ), + result_as=GetGroupResult, + access_token=context.get_admin_access_token() + ) + group = result.group + assert group is not None + assert group.name == group_name + + +def test_user_initiate_auth(context: TestContext): + result = context.get_cluster_manager_client().invoke_alt( + namespace='Auth.InitiateAuth', + payload=InitiateAuthRequest( + auth_flow='USER_PASSWORD_AUTH', + username=test_username, + password=test_password + ), + result_as=InitiateAuthResult + ) + assert result.auth is not None + assert Utils.is_not_empty(result.auth.access_token) + global user_auth + user_auth = result.auth + + +def test_user_change_password(context: TestContext): + global test_password + old_password = test_password + test_password = Utils.generate_password(8, 1, 1, 1, 1) + + context.get_cluster_manager_client().invoke_alt( + namespace='Auth.ChangePassword', + payload=ChangePasswordRequest( + username=test_username, + old_password=old_password, + new_password=test_password + ), + result_as=ChangePasswordResult, + access_token=user_auth.access_token + ) + + +def test_admin_list_users(context: TestContext): + result = context.get_cluster_manager_client().invoke_alt( + namespace='Accounts.ListUsers', + payload=ListUsersRequest(), + result_as=ListUsersResult, + access_token=context.get_admin_access_token() + ) + assert len(result.listing) > 0 + + +def test_admin_create_project(context: TestContext): + project_name = Utils.generate_password(8, 0, 8, 0, 0) + + create_project_result = context.get_cluster_manager_client().invoke_alt( + namespace='Projects.CreateProject', + payload=CreateProjectRequest( + project=Project( + name=project_name, + title=f'{project_name} title', + description=f'{project_name} description', + ldap_groups=[GroupNameHelper(context.idea_context).get_default_project_group()], + tags=[ + SocaKeyValue(key='key', value='value') + ] + ) + ), + result_as=CreateProjectResult, + access_token=context.get_admin_access_token() + ) + + project = create_project_result.project + assert project is not None + assert project.project_id is not None and len(project.project_id) > 0 + assert project.enabled is None or project.enabled is False + + global TEST_PROJECT_ID + TEST_PROJECT_ID = project.project_id + + context.get_cluster_manager_client().invoke_alt( + namespace='Projects.EnableProject', + payload=EnableProjectRequest( + project_id=project.project_id + ), + access_token=context.get_admin_access_token() + ) + + +def test_admin_get_project(context: TestContext): + assert context.is_test_case_passed(test_constants.PROJECTS_CREATE_PROJECT) + + get_project_result = context.get_cluster_manager_client().invoke_alt( + namespace='Projects.GetProject', + payload=GetProjectRequest( + project_id=TEST_PROJECT_ID + ), + result_as=GetProjectResult, + access_token=context.get_admin_access_token() + ) + + assert get_project_result.project is not None + assert get_project_result.project.enabled is True + + +def test_admin_update_project(context: TestContext): + assert context.is_test_case_passed(test_constants.PROJECTS_CREATE_PROJECT) + + get_project_result = context.get_cluster_manager_client().invoke_alt( + namespace='Projects.GetProject', + payload=GetProjectRequest( + project_id=TEST_PROJECT_ID + ), + result_as=GetProjectResult, + access_token=context.get_admin_access_token() + ) + + project = get_project_result.project + updated_title = f'{project.title} - updated' + project.title = updated_title + + context.get_cluster_manager_client().invoke_alt( + namespace='Projects.UpdateProject', + payload=UpdateProjectRequest( + project=project + ), + access_token=context.get_admin_access_token() + ) + + get_updated_project_result = context.get_cluster_manager_client().invoke_alt( + namespace='Projects.GetProject', + payload=GetProjectRequest( + project_id=TEST_PROJECT_ID + ), + result_as=GetProjectResult, + access_token=context.get_admin_access_token() + ) + + assert get_updated_project_result.project.title == updated_title + + +def test_admin_list_projects(context: TestContext): + assert context.is_test_case_passed(test_constants.PROJECTS_CREATE_PROJECT) + + list_projects_result = context.get_cluster_manager_client().invoke_alt( + namespace='Projects.ListProjects', + payload=ListProjectsRequest(), + result_as=ListProjectsResult, + access_token=context.get_admin_access_token() + ) + + assert list_projects_result.listing is not None + assert len(list_projects_result.listing) > 0 + + found = False + for project in list_projects_result.listing: + if project.project_id == TEST_PROJECT_ID: + found = True + assert found is True + + +def test_admin_disable_project(context: TestContext): + assert context.is_test_case_passed(test_constants.PROJECTS_CREATE_PROJECT) + + context.get_cluster_manager_client().invoke_alt( + namespace='Projects.DisableProject', + payload=DeleteProjectRequest( + project_id=TEST_PROJECT_ID + ), + result_as=DeleteProjectResult, + access_token=context.get_admin_access_token() + ) + + try: + context.get_cluster_manager_client().invoke_alt( + namespace='Projects.GetProject', + payload=GetProjectRequest( + project_id=TEST_PROJECT_ID + ), + result_as=GetProjectResult, + access_token=context.get_admin_access_token() + ) + except exceptions.SocaException as e: + assert e.error_code == 'SCHEDULER_HPC_PROJECT_NOT_FOUND' + + +def test_admin_disable_user(context: TestContext): + context.get_cluster_manager_client().invoke_alt( + namespace='Accounts.DisableUser', + payload=DeleteUserRequest( + username=test_username + ), + result_as=DeleteUserResult, + access_token=context.get_admin_access_token() + ) + + +TEST_CASES = [ + { + 'test_case_id': test_constants.ACCOUNTS_CREATE_USER, + 'test_case': test_admin_create_user + }, + { + 'test_case_id': test_constants.ACCOUNTS_GET_USER, + 'test_case': test_admin_get_user + }, + { + 'test_case_id': test_constants.ACCOUNTS_GET_USER_GROUP, + 'test_case': test_get_user_group + }, + { + 'test_case_id': test_constants.AUTH_INITIATE_AUTH, + 'test_case': test_user_initiate_auth + }, + { + 'test_case_id': test_constants.AUTH_CHANGE_PASSWORD, + 'test_case': test_user_change_password + }, + { + 'test_case_id': test_constants.ACCOUNTS_LIST_USERS, + 'test_case': test_admin_list_users + }, + { + 'test_case_id': test_constants.ACCOUNTS_DISABLE_USER, + 'test_case': test_admin_disable_user + }, + { + 'test_case_id': test_constants.PROJECTS_CREATE_PROJECT, + 'test_case': test_admin_create_project + }, + { + 'test_case_id': test_constants.PROJECTS_GET_PROJECT, + 'test_case': test_admin_get_project + }, + { + 'test_case_id': test_constants.PROJECTS_GET_PROJECT, + 'test_case': test_admin_get_project + }, + { + 'test_case_id': test_constants.PROJECTS_UPDATE_PROJECT, + 'test_case': test_admin_update_project + }, + { + 'test_case_id': test_constants.PROJECTS_LIST_PROJECTS, + 'test_case': test_admin_list_projects + }, + { + 'test_case_id': test_constants.PROJECTS_DISABLE_PROJECT, + 'test_case': test_admin_disable_project + } +] diff --git a/source/idea/idea-administrator/src/ideaadministrator/integration_tests/scheduler/__init__.py b/source/idea/idea-administrator/src/ideaadministrator/integration_tests/scheduler/__init__.py new file mode 100644 index 00000000..6d8d18a3 --- /dev/null +++ b/source/idea/idea-administrator/src/ideaadministrator/integration_tests/scheduler/__init__.py @@ -0,0 +1,10 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. diff --git a/source/idea/idea-administrator/src/ideaadministrator/integration_tests/scheduler/job_test_case.py b/source/idea/idea-administrator/src/ideaadministrator/integration_tests/scheduler/job_test_case.py new file mode 100644 index 00000000..a65c6faa --- /dev/null +++ b/source/idea/idea-administrator/src/ideaadministrator/integration_tests/scheduler/job_test_case.py @@ -0,0 +1,329 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +from ideasdk.utils import Utils +from ideadatamodel import ( + exceptions, errorcodes +) +from ideadatamodel import ( + SubmitJobResult, + SubmitJobRequest, + GetJobRequest, + GetJobResult, + ReadFileRequest, + ReadFileResult +) + +import ideaadministrator +from ideaadministrator.integration_tests.test_context import TestContext + +import os +from typing import Dict, Optional +import arrow +import re +import time +import random + + +class JobTestCase: + + def __init__(self, helper: 'JobSubmissionHelper', test_case_id: str, base_os: str, ami_id: str, test_case_config: Dict): + self.helper = helper + self.context = helper.context + self.test_case_id = test_case_id + self.test_case_config = test_case_config + self.base_os = base_os + resources = Utils.get_value_as_string('resource', test_case_config) + self.resources = f'{resources},instance_ami={ami_id},base_os={base_os}' + self.command = Utils.get_value_as_string('command', test_case_config) + if ';' in self.command: + self.command = f'{{ {self.command} }}' + + self.expected_output = Utils.get_value_as_string('expected_output', test_case_config) + if self.expected_output: + self.expected_output = self.expected_output.strip() + + self.regex_match = Utils.get_value_as_bool('regex_match', test_case_config, False) + self.expected_error_code = Utils.get_value_as_string('error_code', test_case_config) + self.queue = Utils.get_value_as_string('queue', test_case_config) + self.job_group = Utils.get_value_as_string('job_group', test_case_config) + + self.status = 'PENDING' + self.exception = None + self.actual_output = None + self.job_submit_result: Optional[SubmitJobResult] = None + self.output_folder = f'/data/home/{self.context.admin_username}/integration-tests/{helper.context.test_run_id}' + self.output_file = f'{self.output_folder}/{self.test_case_id}.log' + self.message: Optional[str] = None + self.submission_time: Optional[arrow.Arrow] = None + self.max_dry_run_attempts = 10 + self.dry_run_attempt = 0 + self.subnets = self.context.cluster_config.get_list('cluster.network.private_subnets', required=True) + # Subnets that we have previously attempted/failed (EC2 DryRun failure) + self.invalid_subnets = set() + + def submit_job(self): + _potential_subnets = [x for x in self.subnets if x not in self.invalid_subnets] + if self.dry_run_attempt > 0: + self.context.info(f'submit_job() - {self.test_case_id} - Subnets: {self.subnets} Invalid_Subnets: {self.invalid_subnets} _potential_subnets = {_potential_subnets} ReAttempt: {self.dry_run_attempt}') + # No subnets passed the self.invalid_subnets filter - error this TestCaseId + if not _potential_subnets: + self.status = 'FAIL' + self.context.error(f'Failed to Submit Job for TestCaseId: {self.test_case_id} - ERROR: Exhausted all potential subnets.') + + subnet_id = random.choice(_potential_subnets) + job_script = f"""#!/bin/bash +#PBS -N {self.test_case_id} +#PBS -q {self.queue} +#PBS -l {self.resources} +#PBS -l subnet_id={subnet_id} +#PBS -P default +mkdir -p {self.output_folder} +{self.command} > {self.output_file} + """ + try: + result = self.context.get_scheduler_client().invoke_alt( + namespace='Scheduler.SubmitJob', + payload=SubmitJobRequest( + job_script_interpreter='pbs', + job_script=Utils.base64_encode(job_script) + ), + result_as=SubmitJobResult, + access_token=self.context.get_admin_access_token() + ) + self.job_submit_result = result + self.status = 'IN_PROGRESS' + self.submission_time = arrow.get() + self.context.info(f'submit_job() - Job submitted for TestCaseId: {self.test_case_id} on subnet_id {subnet_id} Retry: {self.dry_run_attempt}') + + except exceptions.SocaException as e: + self.context.info(f'submit_job() - SOCA Exception: {e}') + if Utils.is_not_empty(self.expected_error_code) and e.error_code == self.expected_error_code: + if Utils.is_not_empty(self.expected_output): + if self.expected_output in e.message: + self.status = 'PASS' + else: + self.status = 'FAIL' + else: + self.status = 'PASS' + else: + self.exception = e + self.status = 'AUTO_RETRY' + if 'EC2DryRunFailed' in e.message: + self.context.info(f'submit_job() - Encountered DryRun failure for {self.test_case_id}') + while self.dry_run_attempt < self.max_dry_run_attempts and self.status != 'IN_PROGRESS': + self.dry_run_attempt += 1 + if subnet_id not in self.invalid_subnets: + self.context.warning(f'submit_job() - Adding subnet_id {subnet_id} as an invalid_subnet') + self.invalid_subnets.add(subnet_id) + else: + self.context.warning(f'submit_job() - Subnet_id {subnet_id} already in invalid_subnets set. This Should not happen.') + + self.context.warning(f'submit_job() - TestCaseId: {self.test_case_id} / {subnet_id}. Retry in 15 secs using a different subnet_id Invalid_subnets: {self.invalid_subnets} [Loop: {self.dry_run_attempt}/{self.max_dry_run_attempts}]') + + time.sleep(15) + # Clear the old exception and try again + self.exception = None + self.status = 'PENDING' + self.submit_job() + else: + self.status = 'FAIL' + self.context.error(f'Failed to Submit Job for TestCaseId: {self.test_case_id} - Unknown exception: {e}') + except Exception as e: + self.status = 'FAIL' + self.exception = e + self.context.error(f'Failed to Submit Job for TestCaseId: {self.test_case_id} - Exception: {e}') + + def get_job_id(self) -> str: + if self.job_submit_result and self.job_submit_result.job: + return self.job_submit_result.job.job_id + return 'UNKNOWN' + + def check_progress(self): + try: + self.context.get_scheduler_client().invoke_alt( + namespace='Scheduler.GetCompletedJob', + payload=GetJobRequest( + job_id=self.job_submit_result.job.job_id + ), + result_as=GetJobResult, + access_token=self.context.get_admin_access_token() + ) + self.status = 'COMPLETE' + except exceptions.SocaException as e: + if e.error_code == errorcodes.JOB_NOT_FOUND: + self.status = 'IN_PROGRESS' + else: + raise e + + def get_output(self) -> str: + read_file_result = self.context.get_cluster_manager_client().invoke_alt( + namespace='FileBrowser.ReadFile', + payload=ReadFileRequest( + file=self.output_file + ), + result_as=ReadFileResult, + access_token=self.context.get_admin_access_token() + ) + return Utils.base64_decode(read_file_result.content).strip() + + def verify_output(self): + try: + self.actual_output = self.get_output() + if self.regex_match: + success = re.match(self.expected_output, self.actual_output) + else: + success = self.expected_output in self.actual_output + + if success: + self.status = 'PASS' + self.context.success(f'Job TestCaseId: {self.test_case_id} {self.expected_output} == {self.actual_output} [PASS]') + else: + self.status = 'FAIL' + self.context.error(f'Job TestCaseId: {self.test_case_id} {self.expected_output} != {self.actual_output} [FAIL]') + + except Exception as e: + self.status = 'FAIL' + self.exception = e + + +class JobSubmissionHelper: + + def __init__(self, context: TestContext): + + self.context = context + self.job_test_case_config = self.read_job_test_case_config() + self.job_test_cases: Dict[str, JobTestCase] = {} + + @staticmethod + def read_job_test_case_config(): + resources_dir = ideaadministrator.props.resources_dir + test_cases_file = os.path.join(resources_dir, 'integration_tests', 'job_test_cases.yml') + with open(test_cases_file, 'r') as f: + return Utils.from_yaml(f.read()) + + def get_ami_id(self, base_os: str) -> Optional[str]: + base_os_param = self.context.extra_params.get('base_os') + if Utils.is_not_empty(base_os_param): + tokens = base_os_param.split(',') + for token in tokens: + kv = token.split(':') + if len(kv) == 2: + provided_base_os = kv[0].strip() + ami_id = kv[1].strip() + if Utils.is_not_empty(ami_id) and base_os == provided_base_os: + return ami_id + + compute_node_os = self.context.cluster_config.get_string('scheduler.compute_node_os', required=True) + if base_os == compute_node_os: + return self.context.cluster_config.get_string('scheduler.compute_node_ami', required=True) + + regions_config_file = ideaadministrator.props.region_ami_config_file() + with open(regions_config_file, 'r') as f: + regions_config = Utils.from_yaml(f.read()) + ami_config = Utils.get_value_as_dict(self.context.aws_region, regions_config) + if ami_config is None: + raise exceptions.general_exception(f'aws_region: {self.context.aws_region} not found in region_ami_config.yml') + ami_id = Utils.get_value_as_string(base_os, ami_config) + if Utils.is_empty(ami_id): + raise exceptions.general_exception(f'instance_ami not found for base_os: {base_os}, region: {self.context.aws_region}') + return ami_id + + def get_test_case_config(self, name: str) -> Dict: + all_test_case_configs = Utils.get_value_as_list('test_cases', self.job_test_case_config) + + for test_case_config in all_test_case_configs: + + current_test_case_name = Utils.get_value_as_string('name', test_case_config) + + if current_test_case_name != name: + continue + + return test_case_config + + def get_or_create_test_case(self, name: str, base_os: str) -> JobTestCase: + test_case_id = f'{base_os}_{name}' + if test_case_id in self.job_test_cases: + return self.job_test_cases[test_case_id] + + ami_id = self.get_ami_id(base_os) + test_case_config = self.get_test_case_config(name) + + job_test_case = JobTestCase( + helper=self, + test_case_id=test_case_id, + base_os=base_os, + ami_id=ami_id, + test_case_config=test_case_config + ) + self.job_test_cases[test_case_id] = job_test_case + return job_test_case + + def submit_job(self, name: str, base_os: str) -> JobTestCase: + """ + fetches the test case config and submits the job if not already submitted + :param name: name of the test case + :param base_os: one of the supported base os name + :return: + """ + test_case = self.get_or_create_test_case(name, base_os) + if test_case.status == 'PENDING': + test_case.submit_job() + return test_case + + def get_test_case_count(self) -> int: + return len(self.job_test_cases) + + def get_success_count(self) -> int: + result = 0 + for test_case_id in self.job_test_cases: + test_case = self.job_test_cases[test_case_id] + if test_case.status == 'PASS': + result += 1 + return result + + def get_failed_count(self) -> int: + result = 0 + for test_case_id in self.job_test_cases: + test_case = self.job_test_cases[test_case_id] + if test_case.status == 'FAIL': + result += 1 + return result + + def get_completed_count(self) -> int: + result = 0 + for test_case_id in self.job_test_cases: + test_case = self.job_test_cases[test_case_id] + if test_case.status in ('PASS', 'FAIL'): + result += 1 + return result + + def get_in_progress_count(self) -> int: + result = 0 + for test_case_id in self.job_test_cases: + test_case = self.job_test_cases[test_case_id] + if test_case.status == 'IN_PROGRESS': + result += 1 + return result + + def print_summary(self, header: str): + self.context.idea_context.print_title( + f'[RunId: {self.context.test_run_id}] {header}: ' + f'Total: {self.get_test_case_count()}, ' + f'Completed: {self.get_completed_count()}, ' + f'Success: {self.get_success_count()}, ' + f'Failed: {self.get_failed_count()}') + print('{:100s} {:10s} {:20s}'.format('Job Test Case Id', 'Job Id', 'Status')) + for test_case_id in self.job_test_cases: + test_case = self.job_test_cases[test_case_id] + print('{:100s} {:10s} {:20s}'.format(test_case_id, test_case.get_job_id(), test_case.status)) + + diff --git a/source/idea/idea-administrator/src/ideaadministrator/integration_tests/scheduler_tests.py b/source/idea/idea-administrator/src/ideaadministrator/integration_tests/scheduler_tests.py new file mode 100644 index 00000000..5c047d83 --- /dev/null +++ b/source/idea/idea-administrator/src/ideaadministrator/integration_tests/scheduler_tests.py @@ -0,0 +1,342 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +from ideasdk.utils import Utils, GroupNameHelper +from ideadatamodel import ( + exceptions, errorcodes +) +from ideadatamodel import ( + HpcQueueProfile, + CreateProjectRequest, + CreateProjectResult, + Project, + SocaKeyValue, + CreateQueueProfileRequest, + CreateQueueProfileResult, + SocaQueueMode, + SocaJobParams, + SocaScalingMode, + EnableQueueProfileRequest, + ListQueueProfilesRequest, + ListQueueProfilesResult, + SubmitJobRequest, + SubmitJobResult, + DisableQueueProfileRequest, + DeleteQueueProfileRequest +) + +from ideaadministrator.integration_tests.test_context import TestContext +from ideaadministrator.integration_tests import test_constants +from ideaadministrator.integration_tests.scheduler.job_test_case import JobSubmissionHelper + +from typing import Optional, List +import time + + +TEST_PROJECT_ID: Optional[str] = None + + +def _submit_all_jobs(context: TestContext, helper: JobSubmissionHelper, base_os_names: List[str], test_case_filter: List[str]): + all_test_case_configs = Utils.get_value_as_list('test_cases', helper.job_test_case_config) + job_region = context.cluster_config.get_string('cluster.aws.region') + job_submission_count = 0 + for base_os in base_os_names: + for test_case_config in all_test_case_configs: + + # check if test case can be skipped + skip_regions = Utils.get_value_as_list('skip_regions', test_case_config) + if skip_regions: + if job_region in skip_regions or "all" in skip_regions: + continue + + name = Utils.get_value_as_string('name', test_case_config) + if Utils.is_not_empty(test_case_filter): + if name in test_case_filter: + helper.submit_job(name, base_os) + job_submission_count += 1 + else: + helper.submit_job(name, base_os) + job_submission_count += 1 + + helper.context.info(f'Job Test Cases - Submitted {job_submission_count} Jobs.') + + +def _verify_all_jobs(helper: JobSubmissionHelper): + try: + helper.context.info(f'Job Test Cases - Start Job Validations ...') + + iteration_interval = 60 + max_iterations = 30 + current_iteration = 0 + + while current_iteration <= max_iterations: + + current_iteration += 1 + + for test_case_id in helper.job_test_cases: + test_case = helper.job_test_cases[test_case_id] + try: + + if test_case.status in ('PASS', 'FAIL'): + continue + + test_case.check_progress() + if test_case.status == 'IN_PROGRESS': + continue + + if test_case.status == 'COMPLETE': + test_case.verify_output() + + except Exception as e: + helper.context.error(f'failed to verify status for TestCaseId: {test_case.test_case_id} - {e}') + + helper.print_summary('Job Test Case Progress') + + if helper.get_completed_count() >= helper.get_test_case_count(): + break + + helper.context.info(f'[{current_iteration} of {max_iterations}] Sleeping for {iteration_interval} seconds') + time.sleep(iteration_interval) + except KeyboardInterrupt: + helper.context.error('Job test case verification aborted!') + + +def test_jobs(context: TestContext): + base_os_param = context.extra_params.get('base_os') + if Utils.is_empty(base_os_param): + base_os_names = [context.cluster_config.get_string('scheduler.compute_node_os')] + else: + tokens = base_os_param.split(',') + base_os_names = [] + for token in tokens: + base_os_names.append(token.split(':')[0].strip()) + + job_filter_string = context.extra_params.get('job_test_cases') + if Utils.is_empty(job_filter_string): + job_filters = [] + else: + job_filters = job_filter_string.split(',') + + helper = JobSubmissionHelper( + context=context + ) + _submit_all_jobs(context, helper, base_os_names, job_filters) + _verify_all_jobs(helper) + + # shared job verifications. ensure that test cases with same job group execute on the same host + job_groups = {} + for job_test_case_id in helper.job_test_cases: + test_case = helper.job_test_cases[job_test_case_id] + if test_case.job_group is None: + continue + + job_shared_test_case = helper.job_test_cases[job_test_case_id] + job_group_key = f'{job_shared_test_case.base_os}.{job_shared_test_case.job_group}' + + if job_group_key in job_groups: + jobs = job_groups[job_group_key] + else: + jobs = [] + job_groups[job_group_key] = jobs + + jobs.append(test_case) + + result = helper.get_success_count() == helper.get_test_case_count() + if len(job_groups) > 0: + for job_group_key in job_groups: + jobs = job_groups[job_group_key] + if len(jobs) < 2: + continue + if jobs[0].actual_output != jobs[1].actual_output: + jobs[0].status = 'FAIL' + jobs[1].status = 'FAIL' + result = False + + helper.print_summary('Job Test Case Summary') + assert result + + +def test_admin_queue_profiles(context: TestContext): + created_queue_profile: Optional[HpcQueueProfile] = None + + try: + + test_queue_profile_name = Utils.generate_password(8, 0, 8, 0, 0) + + # Create project to associate to the queue + queue_profile_project = context.get_cluster_manager_client().invoke_alt( + namespace='Projects.CreateProject', + payload=CreateProjectRequest( + project=Project( + name=test_queue_profile_name, + title=f'{test_queue_profile_name} title', + description=f'{test_queue_profile_name} description', + ldap_groups=[GroupNameHelper(context.idea_context).get_default_project_group()], + tags=[ + SocaKeyValue(key='key', value='value') + ] + ) + ), + result_as=CreateProjectResult, + access_token=context.get_admin_access_token() + ) + + # create queue profile + context.info(f'Create Queue Profile: {test_queue_profile_name}') + create_queue_profile_result = context.get_scheduler_client().invoke_alt( + namespace='SchedulerAdmin.CreateQueueProfile', + payload=CreateQueueProfileRequest( + queue_profile=HpcQueueProfile( + title='Integration Test Queue Profile', + name=test_queue_profile_name, + description='Test Queue Profile - Description', + queues=[test_queue_profile_name], + queue_mode=SocaQueueMode.FIFO, + projects=[queue_profile_project.project], + scaling_mode=SocaScalingMode.SINGLE_JOB, + terminate_when_idle=0, + keep_forever=False, + default_job_params=SocaJobParams( + instance_types=['c5.large'], + base_os=context.cluster_config.get_string('scheduler.compute_node_os', required=True), + instance_ami=context.cluster_config.get_string('scheduler.compute_node_ami', required=True) + ) + ) + ), + result_as=CreateQueueProfileResult, + access_token=context.get_admin_access_token() + ) + + queue_profile = create_queue_profile_result.queue_profile + assert queue_profile is not None + assert queue_profile.queue_profile_id is not None + assert queue_profile.enabled is False + + created_queue_profile = queue_profile + + # enable queue profile + context.info(f'Enable Queue Profile: {test_queue_profile_name}') + context.get_scheduler_client().invoke_alt( + namespace='SchedulerAdmin.EnableQueueProfile', + payload=EnableQueueProfileRequest( + queue_profile_name=queue_profile.name + ), + access_token=context.get_admin_access_token() + ) + + # list queue profiles + context.info('Listing Queue Profiles') + list_queue_profiles_result = context.get_scheduler_client().invoke_alt( + namespace='SchedulerAdmin.ListQueueProfiles', + payload=ListQueueProfilesRequest(), + result_as=ListQueueProfilesResult, + access_token=context.get_admin_access_token() + ) + assert list_queue_profiles_result.listing is not None + assert len(list_queue_profiles_result.listing) > 0 + found = False + for current_queue_profile in list_queue_profiles_result.listing: + if current_queue_profile.queue_profile_id == created_queue_profile.queue_profile_id: + context.info(f'Found Queue Profile: {test_queue_profile_name} in listing response') + found = True + assert found is True + + # dry run submit job + context.info(f'Submit DryRun Job on Queue Profile: {test_queue_profile_name}') + job_script_valid = f"""#!/bin/bash +#PBS -N queue_profile_test +#PBS -q {test_queue_profile_name} +#PBS -P default +/bin/echo test + """ + submit_job_result = context.get_scheduler_client().invoke_alt( + namespace='Scheduler.SubmitJob', + payload=SubmitJobRequest( + job_script_interpreter='pbs', + job_script=Utils.base64_encode(job_script_valid), + dry_run=True + ), + result_as=SubmitJobResult, + access_token=context.get_admin_access_token() + ) + + assert len(submit_job_result.validations.results) == 0 + + job_script_invalid_project = f"""#!/bin/bash +#PBS -N queue_profile_test +#PBS -q {test_queue_profile_name} +#PBS -P PROJECT_DOES_NOT_EXIST +/bin/echo test + """ + try: + context.get_scheduler_client().invoke_alt( + namespace='Scheduler.SubmitJob', + payload=SubmitJobRequest( + job_script_interpreter='pbs', + job_script=Utils.base64_encode(job_script_invalid_project) + ), + result_as=SubmitJobResult, + access_token=context.get_admin_access_token() + ) + assert False + except exceptions.SocaException as e: + assert e.error_code == errorcodes.JOB_SUBMISSION_FAILED + assert 'PROJECT_DOES_NOT_EXIST' in e.message + + # disable queue profile + context.info(f'Disable Queue Profile: {test_queue_profile_name}') + context.get_scheduler_client().invoke_alt( + namespace='SchedulerAdmin.DisableQueueProfile', + payload=DisableQueueProfileRequest( + queue_profile_name=queue_profile.name + ), + access_token=context.get_admin_access_token() + ) + + # try submit job on disabled queue profile + try: + context.get_scheduler_client().invoke_alt( + namespace='Scheduler.SubmitJob', + payload=SubmitJobRequest( + job_script_interpreter='pbs', + job_script=Utils.base64_encode(job_script_valid), + dry_run=True + ), + result_as=SubmitJobResult, + access_token=context.get_admin_access_token() + ) + assert False + except exceptions.SocaException as e: + assert e.error_code == 'SCHEDULER_QUEUE_PROFILE_DISABLED' + + finally: + + if created_queue_profile is not None: + context.info(f'Delete Queue Profile: {created_queue_profile.name}') + context.get_scheduler_client().invoke_alt( + namespace='SchedulerAdmin.DeleteQueueProfile', + payload=DeleteQueueProfileRequest( + queue_profile_name=created_queue_profile.name + ), + access_token=context.get_admin_access_token() + ) + + +TEST_CASES = [ + { + 'test_case_id': test_constants.SCHEDULER_ADMIN_QUEUE_PROFILES, + 'test_case': test_admin_queue_profiles + }, + { + 'test_case_id': test_constants.SCHEDULER_JOB_TEST_CASES, + 'test_case': test_jobs + } +] diff --git a/source/idea/idea-administrator/src/ideaadministrator/integration_tests/test_constants.py b/source/idea/idea-administrator/src/ideaadministrator/integration_tests/test_constants.py new file mode 100644 index 00000000..c5cf2ec6 --- /dev/null +++ b/source/idea/idea-administrator/src/ideaadministrator/integration_tests/test_constants.py @@ -0,0 +1,77 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +ACCOUNTS_CREATE_USER = 'ACCOUNTS_CREATE_USER' +ACCOUNTS_GET_USER = 'ACCOUNTS_GET_USER' +ACCOUNTS_GET_USER_GROUP = 'ACCOUNTS_GET_USER_GROUP' +ACCOUNTS_LIST_USERS = 'ACCOUNTS_LIST_USERS' +ACCOUNTS_DISABLE_USER = 'ACCOUNTS_DISABLE_USER' +ACCOUNTS_ENABLE_USER = 'ACCOUNTS_ENABLE_USER' +AUTH_INITIATE_AUTH = 'AUTH_INITIATE_AUTH' +AUTH_REFRESH_TOKEN = 'AUTH_REFRESH_TOKEN' +AUTH_CHANGE_PASSWORD = 'AUTH_CHANGE_PASSWORD' +PROJECTS_CREATE_PROJECT = 'PROJECTS_CREATE_PROJECT' +PROJECTS_GET_PROJECT = 'PROJECTS_GET_PROJECT' +PROJECTS_UPDATE_PROJECT = 'PROJECTS_UPDATE_PROJECT' +PROJECTS_LIST_PROJECTS = 'PROJECTS_LIST_PROJECTS' +PROJECTS_DISABLE_PROJECT = 'PROJECTS_DISABLE_PROJECT' + +SCHEDULER_ADMIN_QUEUE_PROFILES = 'SCHEDULER_ADMIN_QUEUE_PROFILES' +SCHEDULER_JOB_TEST_CASES = 'SCHEDULER_JOB_TEST_CASES' + +# VDC Admin +VIRTUAL_DESKTOP_TEST_ADMIN_BATCH_SESSION_WORKFLOW = 'VIRTUAL_DESKTOP_TEST_ADMIN_BATCH_SESSION_WORKFLOW' +VIRTUAL_DESKTOP_TEST_ADMIN_CREATE_SESSION = 'VIRTUAL_DESKTOP_TEST_ADMIN_CREATE_SESSION' +VIRTUAL_DESKTOP_TEST_ADMIN_GET_SESSION_SCREENSHOT = 'VIRTUAL_DESKTOP_TEST_ADMIN_GET_SESSION_SCREENSHOT' +VIRTUAL_DESKTOP_TEST_ADMIN_UPDATE_SESSION = 'VIRTUAL_DESKTOP_TEST_ADMIN_UPDATE_SESSION' +VIRTUAL_DESKTOP_TEST_ADMIN_GET_SESSION_INFO = 'VIRTUAL_DESKTOP_TEST_ADMIN_GET_SESSION_INFO' +VIRTUAL_DESKTOP_TEST_ADMIN_GET_SOFTWARE_STACK_INFO = 'VIRTUAL_DESKTOP_TEST_ADMIN_GET_SOFTWARE_STACK_INFO' +VIRTUAL_DESKTOP_TEST_ADMIN_LIST_SESSIONS = 'VIRTUAL_DESKTOP_TEST_ADMIN_LIST_SESSIONS' +VIRTUAL_DESKTOP_TEST_ADMIN_LIST_SOFTWARE_STACKS = 'VIRTUAL_DESKTOP_TEST_ADMIN_LIST_SOFTWARE_STACKS' +VIRTUAL_DESKTOP_TEST_ADMIN_CREATE_SOFTWARE_STACK = 'VIRTUAL_DESKTOP_TEST_ADMIN_CREATE_SOFTWARE_STACK' +VIRTUAL_DESKTOP_TEST_ADMIN_UPDATE_SOFTWARE_STACK = 'VIRTUAL_DESKTOP_TEST_ADMIN_UPDATE_SOFTWARE_STACK' +VIRTUAL_DESKTOP_TEST_ADMIN_UPDATE_SESSION_PERMISSIONS = 'VIRTUAL_DESKTOP_TEST_ADMIN_UPDATE_SESSION_PERMISSIONS' +VIRTUAL_DESKTOP_TEST_ADMIN_CREATE_SOFTWARE_STACK_FROM_SESSION = 'VIRTUAL_DESKTOP_TEST_ADMIN_CREATE_SOFTWARE_STACK_FROM_SESSION' +VIRTUAL_DESKTOP_TEST_ADMIN_GET_SESSION_CONNECTION_INFO = 'VIRTUAL_DESKTOP_TEST_ADMIN_GET_SESSION_CONNECTION_INFO' +VIRTUAL_DESKTOP_TEST_ADMIN_REINDEX_USER_SESSIONS = 'VIRTUAL_DESKTOP_TEST_ADMIN_REINDEX_USER_SESSIONS' +VIRTUAL_DESKTOP_TEST_ADMIN_REINDEX_SOFTWARE_STACKS = 'VIRTUAL_DESKTOP_TEST_ADMIN_REINDEX_SOFTWARE_STACKS' +VIRTUAL_DESKTOP_TEST_ADMIN_CREATE_PERMISSION_PROFILE = 'VIRTUAL_DESKTOP_TEST_ADMIN_CREATE_PERMISSION_PROFILE' +VIRTUAL_DESKTOP_TEST_ADMIN_UPDATE_PERMISSION_PROFILE = 'VIRTUAL_DESKTOP_TEST_ADMIN_UPDATE_PERMISSION_PROFILE' +VIRTUAL_DESKTOP_TEST_ADMIN_LIST_SESSION_PERMISSIONS = 'VIRTUAL_DESKTOP_TEST_ADMIN_LIST_SESSION_PERMISSIONS' +VIRTUAL_DESKTOP_TEST_ADMIN_LIST_SHARED_PERMISSIONS = 'VIRTUAL_DESKTOP_TEST_ADMIN_LIST_SHARED_PERMISSIONS' +VIRTUAL_DESKTOP_TEST_ADMIN_DELETE_SESSIONS = 'VIRTUAL_DESKTOP_TEST_ADMIN_DELETE_SESSIONS' + +# VDC Utils +VIRTUAL_DESKTOP_TEST_LIST_SUPPORTED_OS = 'VIRTUAL_DESKTOP_TEST_LIST_SUPPORTED_OS' +VIRTUAL_DESKTOP_TEST_LIST_SUPPORTED_GPU = 'VIRTUAL_DESKTOP_TEST_LIST_SUPPORTED_GPU' +VIRTUAL_DESKTOP_TEST_LIST_SCHEDULE_TYPES = 'VIRTUAL_DESKTOP_TEST_LIST_SCHEDULE_TYPES' +VIRTUAL_DESKTOP_TEST_LIST_ALLOWED_INSTANCE_TYPES = 'VIRTUAL_DESKTOP_TEST_LIST_ALLOWED_INSTANCE_TYPES' +VIRTUAL_DESKTOP_TEST_LIST_ALLOWED_INSTANCE_TYPES_FOR_SESSION = 'VIRTUAL_DESKTOP_TEST_LIST_ALLOWED_INSTANCE_TYPES_FOR_SESSION' +VIRTUAL_DESKTOP_TEST_GET_BASE_PERMISSIONS = 'VIRTUAL_DESKTOP_TEST_GET_BASE_PERMISSIONS' +VIRTUAL_DESKTOP_TEST_LIST_PERMISSION_PROFILES = 'VIRTUAL_DESKTOP_TEST_LIST_PERMISSION_PROFILES' +VIRTUAL_DESKTOP_TEST_GET_PERMISSION_PROFILE = 'VIRTUAL_DESKTOP_TEST_GET_PERMISSION_PROFILE' + +# VDC DCV +VIRTUAL_DESKTOP_TEST_DESCRIBE_SERVERS = 'VIRTUAL_DESKTOP_TEST_DESCRIBE_SERVERS' +VIRTUAL_DESKTOP_TEST_DESCRIBE_SESSIONS = 'VIRTUAL_DESKTOP_TEST_DESCRIBE_SESSIONS' + +# VDC User +VIRTUAL_DESKTOP_TEST_USER_CREATE_SESSION = 'VIRTUAL_DESKTOP_TEST_USER_CREATE_SESSION' +VIRTUAL_DESKTOP_TEST_USER_GET_SESSION_SCREENSHOT = 'VIRTUAL_DESKTOP_TEST_USER_GET_SESSION_SCREENSHOT' +VIRTUAL_DESKTOP_TEST_USER_UPDATE_SESSION = 'VIRTUAL_DESKTOP_TEST_USER_UPDATE_SESSION' +VIRTUAL_DESKTOP_TEST_USER_GET_SESSION_CONNECTION_INFO = 'VIRTUAL_DESKTOP_TEST_USER_GET_SESSION_CONNECTION_INFO' +VIRTUAL_DESKTOP_TEST_USER_GET_SESSION_INFO = 'VIRTUAL_DESKTOP_TEST_USER_GET_SESSION_INFO' +VIRTUAL_DESKTOP_TEST_USER_SESSION_WORKFLOW = 'VIRTUAL_DESKTOP_TEST_USER_SESSION_WORKFLOW' +VIRTUAL_DESKTOP_TEST_USER_LIST_SESSIONS = 'VIRTUAL_DESKTOP_TEST_USER_LIST_SESSIONS' +VIRTUAL_DESKTOP_TEST_USER_LIST_SOFTWARE_STACKS = 'VIRTUAL_DESKTOP_TEST_USER_LIST_SOFTWARE_STACKS' +VIRTUAL_DESKTOP_TEST_USER_UPDATE_SESSION_PERMISSIONS = 'VIRTUAL_DESKTOP_TEST_USER_UPDATE_SESSION_PERMISSIONS' +VIRTUAL_DESKTOP_TEST_USER_LIST_SESSION_PERMISSIONS = 'VIRTUAL_DESKTOP_TEST_USER_LIST_SESSION_PERMISSIONS' +VIRTUAL_DESKTOP_TEST_USER_LIST_SHARED_PERMISSIONS = 'VIRTUAL_DESKTOP_TEST_USER_LIST_SHARED_PERMISSIONS' diff --git a/source/idea/idea-administrator/src/ideaadministrator/integration_tests/test_context.py b/source/idea/idea-administrator/src/ideaadministrator/integration_tests/test_context.py new file mode 100644 index 00000000..8614dbc0 --- /dev/null +++ b/source/idea/idea-administrator/src/ideaadministrator/integration_tests/test_context.py @@ -0,0 +1,226 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +from ideasdk.client import SocaClient, SocaClientOptions +from ideasdk.context import SocaCliContext +from ideasdk.utils import Utils +from ideadatamodel import ( + AuthResult, + InitiateAuthRequest, + InitiateAuthResult +) +from ideadatamodel import exceptions, constants +from ideasdk.config.cluster_config import ClusterConfig + +from typing import Optional, Dict, List +import arrow + + +class TestContext: + + def __init__(self, cluster_name: str, + aws_region: str, + admin_username: str, + admin_password: str, + module_set: str, + aws_profile: Optional[str] = None, + debug: bool = False, + extra_params: Optional[Dict] = None, + test_case_ids: Optional[List[str]] = None, + module_ids: Optional[List] = None): + + self.aws_region = aws_region + self.aws_profile = aws_profile + self.admin_username = admin_username + self.admin_password = admin_password + self.module_ids = module_ids + self.non_admin_username = None + self.non_admin_password = None + + self.debug = debug + self.extra_params = extra_params if extra_params else {} + self.filter_test_case_ids = test_case_ids if Utils.is_not_empty(test_case_ids) else None + self.module_set = module_set + + self.cluster_config = ClusterConfig( + cluster_name=cluster_name, + aws_region=aws_region, + aws_profile=aws_profile, + module_set=module_set + ) + self.virtual_desktop_controller_active_module_id = self.cluster_config.get_module_id(constants.MODULE_VIRTUAL_DESKTOP_CONTROLLER) + + self.cluster_endpoint = self.cluster_config.get_cluster_external_endpoint() + + self.idea_context = SocaCliContext() + self.test_run_id = Utils.file_system_friendly_timestamp() + + # initialize admin authentication + # create internal http client to disable admin password exposed in integration test logs + cluster_manager_module_id = self.cluster_config.get_module_id(constants.MODULE_CLUSTER_MANAGER) + self._admin_http_client = SocaClient(context=self.idea_context, + options=SocaClientOptions( + enable_logging=False, + endpoint=f'{self.cluster_endpoint}/{cluster_manager_module_id}/api/v1', + verify_ssl=False + )) + self.admin_auth_expires_on: Optional[arrow.arrow] = None + self.non_admin_auth_expires_on: Optional[arrow.arrow] = None + self.admin_auth: Optional[AuthResult] = None + self.non_admin_auth: Optional[AuthResult] = None + self.initialize_admin_auth() + if self.set_non_admin_user_account(extra_params): + self.initialize_non_admin_auth() + self.module_ids = module_ids + + # test clients map module_id => SocaClient + self.clients: Dict[str, SocaClient] = {} + + self.test_cases = {} + + def get_client(self, module_name: str, timeout: Optional[int] = 10) -> SocaClient: + + module_id = self.cluster_config.get_module_id(module_name=module_name) + + if module_id in self.clients: + return self.clients[module_id] + + module_info = self.cluster_config.db.get_module_info(module_id) + if module_info is None: + raise exceptions.general_exception(f'module not found for module id: {module_id}') + + api_path = f'/{module_id}/api/v1' + + if Utils.is_empty(api_path): + raise exceptions.general_exception(f'could not find api context path for module: {module_id}') + + client = SocaClient(context=self.idea_context, options=SocaClientOptions(enable_logging=self.debug, + endpoint=f'{self.cluster_endpoint}{api_path}', + verify_ssl=False, + timeout=timeout)) + self.clients[module_id] = client + return client + + def set_non_admin_user_account(self, extra_params) -> bool: + if self.virtual_desktop_controller_active_module_id in self.module_ids: + if 'test_username' in extra_params and 'test_password' in extra_params: + if Utils.is_not_empty(extra_params.get('test_username')) and Utils.is_not_empty(extra_params.get('test_password')): + self.non_admin_username = extra_params.get('test_username') + self.non_admin_password = extra_params.get('test_password') + return True + else: + raise exceptions.general_exception(f'Invalid \'test_username\' or \'test_password\' value. Required parameter to execute {self.virtual_desktop_controller_active_module_id} tests') + else: + raise exceptions.general_exception(f'Missing Parameter \'test_username\' or \'test_password\'. Required parameter to execute {self.virtual_desktop_controller_active_module_id} tests.') + + def get_virtual_desktop_controller_client(self, timeout: Optional[int] = 10) -> SocaClient: + return self.get_client(constants.MODULE_VIRTUAL_DESKTOP_CONTROLLER, timeout) + + def get_scheduler_client(self) -> SocaClient: + return self.get_client(constants.MODULE_SCHEDULER) + + def get_cluster_manager_client(self) -> SocaClient: + return self.get_client(constants.MODULE_CLUSTER_MANAGER) + + def initialize_admin_auth(self): + if self.admin_auth is None: + self.idea_context.info('Initializing Admin Authentication ...') + result = self._admin_http_client.invoke_alt('Auth.InitiateAuth', InitiateAuthRequest( + auth_flow='USER_PASSWORD_AUTH', + username=self.admin_username, + password=self.admin_password + ), result_as=InitiateAuthResult) + else: + self.idea_context.info('Renewing Admin Authentication Access Token ...') + result = self._admin_http_client.invoke_alt('Auth.InitiateAuth', InitiateAuthRequest( + auth_flow='REFRESH_TOKEN_AUTH', + username=self.admin_username, + password=self.admin_auth.refresh_token + ), result_as=InitiateAuthResult) + + admin_auth = result.auth + if Utils.is_empty(admin_auth.access_token): + raise exceptions.general_exception('access_token not found') + if Utils.is_empty(admin_auth.refresh_token): + raise exceptions.general_exception('refresh_token not found') + + self.admin_auth = admin_auth + self.admin_auth_expires_on = arrow.get().shift(seconds=admin_auth.expires_in) + + def initialize_non_admin_auth(self): + if self.non_admin_auth is None: + self.idea_context.info('Initializing Non-Admin Authentication ...') + result = self._admin_http_client.invoke_alt('Auth.InitiateAuth', InitiateAuthRequest( + auth_flow='USER_PASSWORD_AUTH', + username=self.non_admin_username, + password=self.non_admin_password + ), result_as=InitiateAuthResult) + else: + self.idea_context.info('Renewing Non-Admin Authentication Access Token ...') + result = self._admin_http_client.invoke_alt('Auth.InitiateAuth', InitiateAuthRequest( + auth_flow='REFRESH_TOKEN_AUTH', + username=self.non_admin_username, + password=self.non_admin_auth.refresh_token + ), result_as=InitiateAuthResult) + + non_admin_auth = result.auth + if Utils.is_empty(non_admin_auth.access_token): + raise exceptions.general_exception('access_token not found') + if Utils.is_empty(non_admin_auth.refresh_token): + raise exceptions.general_exception('refresh_token not found') + + self.non_admin_auth = non_admin_auth + self.non_admin_auth_expires_on = arrow.get().shift(seconds=non_admin_auth.expires_in) + + def get_admin_access_token(self) -> str: + if self.admin_auth is None: + raise exceptions.general_exception('admin authentication not initialized') + if self.admin_auth_expires_on > arrow.get().shift(minutes=-5): + return self.admin_auth.access_token + self.initialize_admin_auth() + return self.admin_auth.access_token + + def get_non_admin_access_token(self): + if self.non_admin_auth is None: + raise exceptions.general_exception('non admin authentication not initialized') + if self.non_admin_auth_expires_on > arrow.get().shift(minutes=-5): + return self.non_admin_auth.access_token + self.initialize_non_admin_auth() + return self.non_admin_auth.access_token + + def begin_test_case(self, test_case_id: str): + self.idea_context.print(f'{test_case_id} [STARTED]') + self.test_cases[test_case_id] = 'IN_PROGRESS' + + def end_test_case(self, test_case_id: str, success: bool): + if success: + self.idea_context.success(f'{test_case_id} [PASS]') + self.test_cases[test_case_id] = 'PASS' + else: + self.idea_context.error(f'{test_case_id} [FAIL]') + self.test_cases[test_case_id] = 'FAIL' + + def is_test_case_passed(self, test_case_id: str) -> bool: + if test_case_id in self.test_cases: + return self.test_cases.get(test_case_id) == 'PASS' + return False + + def info(self, message: str): + self.idea_context.info(message) + + def warning(self, message: str): + self.idea_context.warning(message) + + def success(self, message: str): + self.idea_context.success(message) + + def error(self, message: str): + self.idea_context.error(message) diff --git a/source/idea/idea-administrator/src/ideaadministrator/integration_tests/test_invoker.py b/source/idea/idea-administrator/src/ideaadministrator/integration_tests/test_invoker.py new file mode 100644 index 00000000..a64e8124 --- /dev/null +++ b/source/idea/idea-administrator/src/ideaadministrator/integration_tests/test_invoker.py @@ -0,0 +1,86 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +from ideadatamodel import constants, exceptions, errorcodes + +from ideaadministrator.integration_tests.test_context import TestContext +from ideaadministrator.integration_tests import cluster_manager_tests +from ideaadministrator.integration_tests import scheduler_tests, virtual_desktop_controller_tests + +from typing import List +import traceback + + +class TestInvoker: + def __init__(self, test_context: TestContext, module_ids: List[str]): + self.test_context = test_context + self.module_ids = module_ids + + self.module_test_cases = { + constants.MODULE_CLUSTER_MANAGER: cluster_manager_tests.TEST_CASES, + constants.MODULE_SCHEDULER: scheduler_tests.TEST_CASES, + constants.MODULE_VIRTUAL_DESKTOP_CONTROLLER: virtual_desktop_controller_tests.TEST_CASES + } + + def invoke(self): + for module_id in self.module_ids: + + # set current module_id in cluster config so that module setting keys for + # module test cases are resolved correctly + self.test_context.cluster_config.set_module_id(module_id) + + module_info = self.test_context.cluster_config.module_info + module_status = module_info['status'] + if module_status != 'deployed': + raise exceptions.general_exception(f'module id: {module_id} is not deployed yet.') + + module_name = module_info['name'] + + test_cases = self.module_test_cases.get(module_name) + if test_cases is None: + self.test_context.warning(f'no test cases found for module: {module_name}') + continue + + total_count = 0 + fail_count = 0 + success_count = 0 + for test_case in test_cases: + test_case_id = test_case['test_case_id'] + test_case_fn = test_case['test_case'] + + if self.test_context.filter_test_case_ids is not None: + if test_case_id not in self.test_context.filter_test_case_ids: + continue + + try: + + total_count += 1 + + self.test_context.begin_test_case(test_case_id) + test_case_fn(context=self.test_context) + self.test_context.end_test_case(test_case_id, True) + + success_count += 1 + + except Exception as e: + print(e) + if self.test_context.debug: + traceback.print_exc() + self.test_context.end_test_case(test_case_id, False) + + fail_count += 1 + + if fail_count > 0: + success_percent = (success_count / total_count) * 100 + raise exceptions.soca_exception( + error_code=errorcodes.INTEGRATION_TEST_FAILED, + message=f'{fail_count} of {total_count} test cases failed. success rate: {round(success_percent, 2)}%' + ) diff --git a/source/idea/idea-administrator/src/ideaadministrator/integration_tests/virtual_desktop_controller_tests.py b/source/idea/idea-administrator/src/ideaadministrator/integration_tests/virtual_desktop_controller_tests.py new file mode 100644 index 00000000..6ecec48b --- /dev/null +++ b/source/idea/idea-administrator/src/ideaadministrator/integration_tests/virtual_desktop_controller_tests.py @@ -0,0 +1,1397 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +from ideaadministrator.integration_tests.virtual_desktop_tests_util import ( + get_sessions_test_cases_list, + create_batch_sessions, + create_session, + SessionsTestResultMap, + VirtualDesktopSessionTestResults, + SessionsTestHelper, + VirtualDesktopTestHelper, + VirtualDesktopApiHelper, + SessionWorkflow +) +from ideaadministrator.integration_tests import test_constants +from ideaadministrator.integration_tests.test_context import TestContext +import time +from ideadatamodel import ( + exceptions, + VirtualDesktopSessionState +) + + +# VDC Admin Tests +def test_admin_batch_session_workflow(context: TestContext): + try: + testcase_id = test_constants.VIRTUAL_DESKTOP_TEST_ADMIN_BATCH_SESSION_WORKFLOW + admin_access_token = context.get_admin_access_token() + # 1. Get Sessions list from the testdata file + sessions = get_sessions_test_cases_list(context, context.admin_username, admin_access_token) + # 2. Create Batch Sessions + batch_sessions = create_batch_sessions(context, sessions) + # 3. Execute Tests + for session in batch_sessions: + session_workflow = SessionWorkflow(context, session, testcase_id, context.admin_username, admin_access_token, 'VirtualDesktopAdmin.GetSessionInfo') + session_workflow.test_session_workflow() + + except exceptions.SocaException as e: + context.error(f'Failed to execute test admin session workflow. Error Code : {e.error_code} Error: {e.message}') + + +def test_admin_create_session(context: TestContext): + test_case_id = test_constants.VIRTUAL_DESKTOP_TEST_ADMIN_CREATE_SESSION + test_case_name = 'Test Admin Create Session' + sleep_timer = 30 + admin_access_token = context.get_admin_access_token() + test_results_map = SessionsTestResultMap(test_case_name) + vdc_test_helper = VirtualDesktopTestHelper(context) + + try: + vdc_test_helper.before_test(test_case_name) + + sessions = get_sessions_test_cases_list(context, context.admin_username, admin_access_token) + + new_session = create_session(context, sessions[6], admin_access_token, 'VirtualDesktop.CreateSession') + sessionHelper = SessionsTestHelper(context, new_session, context.admin_username, admin_access_token) + time.sleep(sleep_timer) + session_status = sessionHelper.wait_and_verify_session_state_matches(VirtualDesktopSessionState.READY, 'VirtualDesktopAdmin.GetSessionInfo') + + if session_status.get('session_state_matches'): + vdc_test_helper.set_new_session(new_session) + vdc_test_helper.on_test_pass(test_case_name, test_results_map) + + else: + session = sessionHelper.get_session_info('VirtualDesktopAdmin.GetSessionInfo') + testcase_error_message = f'Failed to execute {test_case_name}.Session Name : {session.name}. Session is in invalid State : {session.state}. Session ID : {session.idea_session_id}' + session_status.get('error_log') + test_results_map.update_test_result_map(VirtualDesktopSessionTestResults.FAILED, testcase_error_message) + assert False + + except (exceptions.SocaException, Exception) as error: + vdc_test_helper.on_test_exception(test_case_name, error, test_results_map) + + finally: + vdc_test_helper.after_test(test_case_name, test_results_map, test_case_id) + + +def test_admin_get_session_screenshot(context: TestContext): + test_case_name = 'Test Admin Get Session Screenshot' + test_results_map = SessionsTestResultMap(test_case_name) + test_case_id = test_constants.VIRTUAL_DESKTOP_TEST_ADMIN_GET_SESSION_SCREENSHOT + admin_access_token = context.get_admin_access_token() + vdc_test_helper = VirtualDesktopTestHelper(context) + + try: + vdc_test_helper.before_test(test_case_name) + + if vdc_test_helper.is_new_session_created(): + session_helper = SessionsTestHelper(context, vdc_test_helper.get_new_session(), context.admin_username, admin_access_token) + response = session_helper.get_session_screenshot('VirtualDesktopAdmin.GetSessionScreenshot') + + if response.success is not None: + vdc_test_helper.on_test_pass(test_case_name, test_results_map) + + else: + vdc_test_helper.on_test_fail(test_case_name, response, test_results_map) + + else: + testcase_error_message = f'Created session is None. Skipping {test_case_name}.' + test_results_map.update_test_result_map(VirtualDesktopSessionTestResults.FAILED, testcase_error_message) + context.error(testcase_error_message) + + except exceptions.SocaException as error: + vdc_test_helper.on_test_exception(test_case_name, error, test_results_map) + + finally: + vdc_test_helper.after_test(test_case_name, test_results_map, test_case_id) + + +def test_admin_update_session(context: TestContext): + test_case_name = 'Test Admin Update Session' + test_case_id = test_constants.VIRTUAL_DESKTOP_TEST_ADMIN_UPDATE_SESSION + admin_access_token = context.get_admin_access_token() + test_results_map = SessionsTestResultMap(test_case_name) + vdc_test_helper = VirtualDesktopTestHelper(context) + + try: + vdc_test_helper.before_test(test_case_name) + + if vdc_test_helper.is_new_session_created(): + session_helper = SessionsTestHelper(context, vdc_test_helper.get_new_session(), context.admin_username, admin_access_token) + response = session_helper.update_session('VirtualDesktopAdmin.UpdateSession') + + if response.session is not None: + vdc_test_helper.on_test_pass(test_case_name, test_results_map) + + else: + vdc_test_helper.on_test_fail(test_case_name, response, test_results_map) + + else: + testcase_error_message = f'Created session is None. Skipping {test_case_name}. ' + test_results_map.update_test_result_map(VirtualDesktopSessionTestResults.FAILED, testcase_error_message) + context.error(testcase_error_message) + + except exceptions.SocaException as error: + vdc_test_helper.on_test_exception(test_case_name, error, test_results_map) + + finally: + vdc_test_helper.after_test(test_case_name, test_results_map, test_case_id) + + +def test_admin_get_session_info(context: TestContext): + test_case_name = 'Test Admin Get Session Info' + test_case_id = test_constants.VIRTUAL_DESKTOP_TEST_ADMIN_GET_SESSION_INFO + admin_access_token = context.get_admin_access_token() + test_results_map = SessionsTestResultMap(test_case_name) + vdc_test_helper = VirtualDesktopTestHelper(context) + + try: + vdc_test_helper.before_test(test_case_name) + + if vdc_test_helper.is_new_session_created(): + session_helper = SessionsTestHelper(context, vdc_test_helper.get_new_session(), context.admin_username, admin_access_token) + response = session_helper.get_session_info('VirtualDesktopAdmin.GetSessionInfo') + + if response.base_os is not None: + vdc_test_helper.on_test_pass(test_case_name, test_results_map) + + else: + vdc_test_helper.on_test_fail(test_case_name, response, test_results_map) + + else: + testcase_error_message = f'Created session is None. Skipping {test_case_name}. ' + test_results_map.update_test_result_map(VirtualDesktopSessionTestResults.FAILED, testcase_error_message) + context.error(testcase_error_message) + + except exceptions.SocaException as error: + vdc_test_helper.on_test_exception(test_case_name, error, test_results_map) + + finally: + vdc_test_helper.after_test(test_case_name, test_results_map, test_case_id) + + +def test_admin_get_software_stack_info(context: TestContext): + test_case_name = 'Test Admin Get Software Stack Info' + test_case_id = test_constants.VIRTUAL_DESKTOP_TEST_ADMIN_GET_SOFTWARE_STACK_INFO + admin_access_token = context.get_admin_access_token() + test_results_map = SessionsTestResultMap(test_case_name) + vdc_test_helper = VirtualDesktopTestHelper(context) + + try: + vdc_test_helper.before_test(test_case_name) + + if vdc_test_helper.is_new_session_created(): + session_helper = SessionsTestHelper(context, vdc_test_helper.get_new_session(), context.admin_username, admin_access_token) + response = session_helper.get_software_stack_info() + + if response is not None: + vdc_test_helper.on_test_pass(test_case_name, test_results_map) + + else: + vdc_test_helper.on_test_fail(test_case_name, response, test_results_map) + + else: + testcase_error_message = f'Created session is None. Skipping {test_case_name}. ' + test_results_map.update_test_result_map(VirtualDesktopSessionTestResults.FAILED, testcase_error_message) + context.error(testcase_error_message) + + except exceptions.SocaException as error: + vdc_test_helper.on_test_exception(test_case_name, error, test_results_map) + + finally: + vdc_test_helper.after_test(test_case_name, test_results_map, test_case_id) + + +def test_admin_get_session_connection_info(context: TestContext): + test_case_name = 'Test Admin Get Session Connection Info' + test_case_id = test_constants.VIRTUAL_DESKTOP_TEST_ADMIN_GET_SESSION_CONNECTION_INFO + admin_access_token = context.get_admin_access_token() + test_results_map = SessionsTestResultMap(test_case_name) + vdc_test_helper = VirtualDesktopTestHelper(context) + + try: + vdc_test_helper.before_test(test_case_name) + + if vdc_test_helper.is_new_session_created(): + session_helper = SessionsTestHelper(context, vdc_test_helper.get_new_session(), context.admin_username, admin_access_token) + response = session_helper.get_session_connection_info('VirtualDesktopAdmin.GetSessionConnectionInfo') + + if response is not None: + vdc_test_helper.on_test_pass(test_case_name, test_results_map) + + else: + vdc_test_helper.on_test_fail(test_case_name, response, test_results_map) + + else: + testcase_error_message = f'Created session is None. Skipping {test_case_name}. ' + test_results_map.update_test_result_map(VirtualDesktopSessionTestResults.FAILED, testcase_error_message) + context.error(testcase_error_message) + + except exceptions.SocaException as error: + vdc_test_helper.on_test_exception(test_case_name, error, test_results_map) + + finally: + vdc_test_helper.after_test(test_case_name, test_results_map, test_case_id) + + +def test_admin_reindex_user_sessions(context: TestContext): + test_case_name = 'Test Admin Reindex User Sessions' + test_case_id = test_constants.VIRTUAL_DESKTOP_TEST_ADMIN_REINDEX_USER_SESSIONS + admin_access_token = context.get_admin_access_token() + test_results_map = SessionsTestResultMap(test_case_name) + vdc_api_helper = VirtualDesktopApiHelper(context, admin_access_token, context.admin_username) + vdc_test_helper = VirtualDesktopTestHelper(context) + + try: + vdc_test_helper.before_test(test_case_name) + + response = vdc_api_helper.reindex_user_session() + + if response is not None: + vdc_test_helper.on_test_pass(test_case_name, test_results_map) + + else: + vdc_test_helper.on_test_fail(test_case_name, response, test_results_map) + + except exceptions.SocaException as error: + vdc_test_helper.on_test_exception(test_case_name, error, test_results_map) + + finally: + vdc_test_helper.after_test(test_case_name, test_results_map, test_case_id) + + +def test_admin_reindex_software_stacks(context: TestContext): + test_case_name = 'Test Admin Reindex Software Stacks' + test_case_id = test_constants.VIRTUAL_DESKTOP_TEST_ADMIN_REINDEX_SOFTWARE_STACKS + admin_access_token = context.get_admin_access_token() + test_results_map = SessionsTestResultMap(test_case_name) + vdc_api_helper = VirtualDesktopApiHelper(context, admin_access_token, context.admin_username) + vdc_test_helper = VirtualDesktopTestHelper(context) + try: + vdc_test_helper.before_test(test_case_name) + + response = vdc_api_helper.reindex_software_stacks() + + if response is not None: + vdc_test_helper.on_test_pass(test_case_name, test_results_map) + + else: + vdc_test_helper.on_test_fail(test_case_name, response, test_results_map) + + except exceptions.SocaException as error: + vdc_test_helper.on_test_exception(test_case_name, error, test_results_map) + + finally: + vdc_test_helper.after_test(test_case_name, test_results_map, test_case_id) + + +def test_admin_create_permission_profile(context: TestContext): + test_case_name = 'Test Admin Create Permission Profile' + test_case_id = test_constants.VIRTUAL_DESKTOP_TEST_ADMIN_CREATE_PERMISSION_PROFILE + admin_access_token = context.get_admin_access_token() + test_results_map = SessionsTestResultMap(test_case_name) + vdc_api_helper = VirtualDesktopApiHelper(context, admin_access_token, context.admin_username) + vdc_test_helper = VirtualDesktopTestHelper(context) + + try: + vdc_test_helper.before_test(test_case_name) + + response = vdc_api_helper.create_permissions_profile() + + if response is not None: + vdc_test_helper.on_test_pass(test_case_name, test_results_map) + + else: + vdc_test_helper.on_test_fail(test_case_name, response, test_results_map) + + except exceptions.SocaException as error: + vdc_test_helper.on_test_exception(test_case_name, error, test_results_map) + + finally: + vdc_test_helper.after_test(test_case_name, test_results_map, test_case_id) + + +def test_admin_update_permission_profile(context: TestContext): + test_case_name = 'Test Admin Update Permission Profile' + test_case_id = test_constants.VIRTUAL_DESKTOP_TEST_ADMIN_UPDATE_PERMISSION_PROFILE + admin_access_token = context.get_admin_access_token() + test_results_map = SessionsTestResultMap(test_case_name) + vdc_api_helper = VirtualDesktopApiHelper(context, admin_access_token, context.admin_username) + vdc_test_helper = VirtualDesktopTestHelper(context) + try: + vdc_test_helper.before_test(test_case_name) + + response = vdc_api_helper.update_permission_profile('VirtualDesktopAdmin.UpdatePermissionProfile') + + if response is not None: + vdc_test_helper.on_test_pass(test_case_name, test_results_map) + + else: + vdc_test_helper.on_test_fail(test_case_name, response, test_results_map) + + except exceptions.SocaException as error: + vdc_test_helper.on_test_exception(test_case_name, error, test_results_map) + + finally: + vdc_test_helper.after_test(test_case_name, test_results_map, test_case_id) + + +def test_admin_list_session_permissions(context: TestContext): + test_case_name = 'Test Admin List Session Permissions' + test_case_id = test_constants.VIRTUAL_DESKTOP_TEST_ADMIN_LIST_SESSION_PERMISSIONS + admin_access_token = context.get_admin_access_token() + test_results_map = SessionsTestResultMap(test_case_name) + vdc_test_helper = VirtualDesktopTestHelper(context) + + try: + vdc_test_helper.before_test(test_case_name) + + if vdc_test_helper.is_new_session_created(): + session_helper = SessionsTestHelper(context, vdc_test_helper.get_new_session(), context.admin_username, admin_access_token) + response = session_helper.list_session_permissions('VirtualDesktopAdmin.ListSessionPermissions') + + if response is not None: + vdc_test_helper.on_test_pass(test_case_name, test_results_map) + + else: + vdc_test_helper.on_test_fail(test_case_name, response, test_results_map) + + else: + testcase_error_message = f'Created session is None. Skipping {test_case_name}. ' + test_results_map.update_test_result_map(VirtualDesktopSessionTestResults.FAILED, testcase_error_message) + context.error(testcase_error_message) + + except exceptions.SocaException as error: + vdc_test_helper.on_test_exception(test_case_name, error, test_results_map) + + finally: + vdc_test_helper.after_test(test_case_name, test_results_map, test_case_id) + + +def test_admin_list_shared_permissions(context: TestContext): + test_case_name = 'Test Admin List Shared Permissions' + test_case_id = test_constants.VIRTUAL_DESKTOP_TEST_ADMIN_LIST_SHARED_PERMISSIONS + admin_access_token = context.get_admin_access_token() + test_results_map = SessionsTestResultMap(test_case_name) + vdc_test_helper = VirtualDesktopTestHelper(context) + try: + vdc_test_helper.before_test(test_case_name) + + if vdc_test_helper.is_new_session_created(): + session_helper = SessionsTestHelper(context, vdc_test_helper.get_new_session(), context.admin_username, admin_access_token) + response = session_helper.list_shared_permissions('VirtualDesktopAdmin.ListSharedPermissions') + + if response is not None: + vdc_test_helper.on_test_pass(test_case_name, test_results_map) + + else: + vdc_test_helper.on_test_fail(test_case_name, response, test_results_map) + + else: + testcase_error_message = f'Created session is None. Skipping {test_case_name}. ' + test_results_map.update_test_result_map(VirtualDesktopSessionTestResults.FAILED, testcase_error_message) + context.error(testcase_error_message) + + except exceptions.SocaException as error: + vdc_test_helper.on_test_exception(test_case_name, error, test_results_map) + + finally: + vdc_test_helper.after_test(test_case_name, test_results_map, test_case_id) + + +def test_admin_list_sessions(context: TestContext): + test_case_name = 'Test Admin List Sessions' + test_case_id = test_constants.VIRTUAL_DESKTOP_TEST_ADMIN_LIST_SESSIONS + admin_access_token = context.get_admin_access_token() + test_results_map = SessionsTestResultMap(test_case_name) + vdc_test_helper = VirtualDesktopTestHelper(context) + + vdc_api_helper = VirtualDesktopApiHelper(context, admin_access_token, context.admin_username) + + try: + vdc_test_helper.before_test(test_case_name) + + response = vdc_api_helper.list_sessions('VirtualDesktopAdmin.ListSessions') + + if response.listing is not None: + vdc_test_helper.on_test_pass(test_case_name, test_results_map) + + else: + vdc_test_helper.on_test_fail(test_case_name, response, test_results_map) + + except exceptions.SocaException as error: + vdc_test_helper.on_test_exception(test_case_name, error, test_results_map) + + finally: + vdc_test_helper.after_test(test_case_name, test_results_map, test_case_id) + + +def test_admin_list_software_stacks(context: TestContext): + test_case_name = 'Test Admin List Software Stacks' + test_case_id = test_constants.VIRTUAL_DESKTOP_TEST_ADMIN_LIST_SOFTWARE_STACKS + admin_access_token = context.get_admin_access_token() + test_results_map = SessionsTestResultMap(test_case_name) + vdc_test_helper = VirtualDesktopTestHelper(context) + + vdc_api_helper = VirtualDesktopApiHelper(context, admin_access_token, context.admin_username) + + try: + vdc_test_helper.before_test(test_case_name) + + response = vdc_api_helper.list_software_stacks('VirtualDesktopAdmin.ListSoftwareStacks') + + if response.listing is not None: + vdc_test_helper.on_test_pass(test_case_name, test_results_map) + + else: + vdc_test_helper.on_test_fail(test_case_name, response, test_results_map) + + except exceptions.SocaException as error: + vdc_test_helper.on_test_exception(test_case_name, error, test_results_map) + + finally: + vdc_test_helper.after_test(test_case_name, test_results_map, test_case_id) + + +def test_admin_create_software_stack(context: TestContext): + test_case_name = 'Test Admin Create Software Stack' + test_case_id = test_constants.VIRTUAL_DESKTOP_TEST_ADMIN_CREATE_SOFTWARE_STACK + admin_access_token = context.get_admin_access_token() + test_results_map = SessionsTestResultMap(test_case_name) + vdc_test_helper = VirtualDesktopTestHelper(context) + + try: + vdc_test_helper.before_test(test_case_name) + + if vdc_test_helper.is_new_session_created(): + session_helper = SessionsTestHelper(context, vdc_test_helper.get_new_session(), context.admin_username, admin_access_token) + response = session_helper.create_software_stack() + + if response.software_stack is not None: + vdc_test_helper.on_test_pass(test_case_name, test_results_map) + vdc_test_helper.set_new_software_stack(response.software_stack) + else: + vdc_test_helper.on_test_fail(test_case_name, response, test_results_map) + + else: + testcase_error_message = f'Created session is None. Skipping {test_case_name}.' + test_results_map.update_test_result_map(VirtualDesktopSessionTestResults.FAILED, testcase_error_message) + context.error(testcase_error_message) + + except exceptions.SocaException as error: + vdc_test_helper.on_test_exception(test_case_name, error, test_results_map) + + finally: + vdc_test_helper.after_test(test_case_name, test_results_map, test_case_id) + + +def test_admin_create_software_stack_from_session(context: TestContext): + test_case_name = 'Test Admin Create Software Stack from Session' + test_case_id = test_constants.VIRTUAL_DESKTOP_TEST_ADMIN_CREATE_SOFTWARE_STACK_FROM_SESSION + admin_access_token = context.get_admin_access_token() + test_results_map = SessionsTestResultMap(test_case_name) + vdc_test_helper = VirtualDesktopTestHelper(context) + + try: + vdc_test_helper.before_test(test_case_name) + + if vdc_test_helper.is_new_session_created(): + session_helper = SessionsTestHelper(context, vdc_test_helper.get_new_session(), context.admin_username, admin_access_token) + response = session_helper.create_software_stack_from_session() + + if response.software_stack is not None: + vdc_test_helper.on_test_pass(test_case_name, test_results_map) + vdc_test_helper.set_new_software_stack(response.software_stack) + else: + vdc_test_helper.on_test_fail(test_case_name, response, test_results_map) + + else: + testcase_error_message = f'Created session is None. Skipping {test_case_name}.' + test_results_map.update_test_result_map(VirtualDesktopSessionTestResults.FAILED, testcase_error_message) + context.error(testcase_error_message) + + except exceptions.SocaException as error: + vdc_test_helper.on_test_exception(test_case_name, error, test_results_map) + + finally: + vdc_test_helper.after_test(test_case_name, test_results_map, test_case_id) + + +def test_admin_update_software_stack(context: TestContext): + test_case_name = 'Test Admin Update Software Stack' + test_case_id = test_constants.VIRTUAL_DESKTOP_TEST_ADMIN_UPDATE_SOFTWARE_STACK + admin_access_token = context.get_admin_access_token() + test_results_map = SessionsTestResultMap(test_case_name) + vdc_test_helper = VirtualDesktopTestHelper(context) + try: + vdc_test_helper.before_test(test_case_name) + + if vdc_test_helper.is_new_session_created(): + session_helper = SessionsTestHelper(context, vdc_test_helper.get_new_session(), context.admin_username, admin_access_token) + response = session_helper.update_software_stack(vdc_test_helper.get_new_software_stack()) + + if response.software_stack is not None: + vdc_test_helper.on_test_pass(test_case_name, test_results_map) + else: + vdc_test_helper.on_test_fail(test_case_name, response, test_results_map) + + else: + testcase_error_message = f'Created session is None. Skipping {test_case_name}.' + test_results_map.update_test_result_map(VirtualDesktopSessionTestResults.FAILED, testcase_error_message) + context.error(testcase_error_message) + + except exceptions.SocaException as error: + vdc_test_helper.on_test_exception(test_case_name, error, test_results_map) + + finally: + vdc_test_helper.after_test(test_case_name, test_results_map, test_case_id) + + +def test_admin_update_session_permissions(context: TestContext): + test_case_name = 'Test Admin Update Session Permission' + test_case_id = test_constants.VIRTUAL_DESKTOP_TEST_ADMIN_UPDATE_SESSION_PERMISSIONS + admin_access_token = context.get_admin_access_token() + test_results_map = SessionsTestResultMap(test_case_name) + vdc_test_helper = VirtualDesktopTestHelper(context) + try: + vdc_test_helper.before_test(test_case_name) + + if vdc_test_helper.is_new_session_created(): + session_helper = SessionsTestHelper(context, vdc_test_helper.get_new_session(), context.admin_username, admin_access_token) + response = session_helper.update_session_permissions('VirtualDesktopAdmin.UpdateSessionPermissions') + + if response.permissions is not None: + vdc_test_helper.on_test_pass(test_case_name, test_results_map) + else: + vdc_test_helper.on_test_fail(test_case_name, response, test_results_map) + + else: + testcase_error_message = f'Created session is None. Skipping {test_case_name}.' + test_results_map.update_test_result_map(VirtualDesktopSessionTestResults.FAILED, testcase_error_message) + context.error(testcase_error_message) + + except exceptions.SocaException as error: + vdc_test_helper.on_test_exception(test_case_name, error, test_results_map) + + finally: + vdc_test_helper.after_test(test_case_name, test_results_map, test_case_id) + + +# VDC Utils Tests +def test_list_supported_os(context: TestContext): + test_case_name = 'Test Supported OS' + test_case_id = test_constants.VIRTUAL_DESKTOP_TEST_LIST_SUPPORTED_OS + admin_access_token = context.get_admin_access_token() + test_results_map = SessionsTestResultMap(test_case_name) + vdc_test_helper = VirtualDesktopTestHelper(context) + + vdc_api_helper = VirtualDesktopApiHelper(context, admin_access_token, context.admin_username) + + try: + vdc_test_helper.before_test(test_case_name) + + response = vdc_api_helper.list_supported_os() + + if response.listing is not None: + vdc_test_helper.on_test_pass(test_case_name, test_results_map) + + else: + vdc_test_helper.on_test_fail(test_case_name, response, test_results_map) + + except exceptions.SocaException as error: + vdc_test_helper.on_test_exception(test_case_name, error, test_results_map) + + finally: + vdc_test_helper.after_test(test_case_name, test_results_map, test_case_id) + + +def test_list_supported_gpu(context: TestContext): + test_case_name = 'Test List Supported GPU' + test_case_id = test_constants.VIRTUAL_DESKTOP_TEST_LIST_SUPPORTED_GPU + admin_access_token = context.get_admin_access_token() + test_results_map = SessionsTestResultMap(test_case_name) + vdc_test_helper = VirtualDesktopTestHelper(context) + + vdc_api_helper = VirtualDesktopApiHelper(context, admin_access_token, context.admin_username) + + try: + vdc_test_helper.before_test(test_case_name) + + response = vdc_api_helper.list_supported_gpu() + + if response.listing is not None: + vdc_test_helper.on_test_pass(test_case_name, test_results_map) + else: + vdc_test_helper.on_test_fail(test_case_name, response, test_results_map) + + except exceptions.SocaException as error: + vdc_test_helper.on_test_exception(test_case_name, error, test_results_map) + + finally: + vdc_test_helper.after_test(test_case_name, test_results_map, test_case_id) + + +def test_list_schedule_types(context: TestContext): + test_case_name = 'Test List Schedule Types' + test_case_id = test_constants.VIRTUAL_DESKTOP_TEST_LIST_SCHEDULE_TYPES + admin_access_token = context.get_admin_access_token() + test_results_map = SessionsTestResultMap(test_case_name) + vdc_test_helper = VirtualDesktopTestHelper(context) + + vdc_api_helper = VirtualDesktopApiHelper(context, admin_access_token, context.admin_username) + + try: + vdc_test_helper.before_test(test_case_name) + + response = vdc_api_helper.list_schedule_types() + + if response.listing is not None: + vdc_test_helper.on_test_pass(test_case_name, test_results_map) + else: + vdc_test_helper.on_test_fail(test_case_name, response, test_results_map) + + except exceptions.SocaException as error: + vdc_test_helper.on_test_exception(test_case_name, error, test_results_map) + + finally: + vdc_test_helper.after_test(test_case_name, test_results_map, test_case_id) + + +def test_list_allowed_instance_types(context: TestContext): + test_case_name = 'Test List Allowed Instance Types' + test_case_id = test_constants.VIRTUAL_DESKTOP_TEST_LIST_ALLOWED_INSTANCE_TYPES + admin_access_token = context.get_admin_access_token() + test_results_map = SessionsTestResultMap(test_case_name) + vdc_test_helper = VirtualDesktopTestHelper(context) + + vdc_api_helper = VirtualDesktopApiHelper(context, admin_access_token, context.admin_username) + + try: + vdc_test_helper.before_test(test_case_name) + + response = vdc_api_helper.list_allowed_instance_types() + + if response.listing is not None: + vdc_test_helper.on_test_pass(test_case_name, test_results_map) + else: + vdc_test_helper.on_test_fail(test_case_name, response, test_results_map) + + except exceptions.SocaException as error: + vdc_test_helper.on_test_exception(test_case_name, error, test_results_map) + + finally: + vdc_test_helper.after_test(test_case_name, test_results_map, test_case_id) + + +def test_list_allowed_instance_types_for_session(context: TestContext): + test_case_name = 'Test List Allowed Instance Types for Session' + test_case_id = test_constants.VIRTUAL_DESKTOP_TEST_LIST_ALLOWED_INSTANCE_TYPES_FOR_SESSION + admin_access_token = context.get_admin_access_token() + test_results_map = SessionsTestResultMap(test_case_name) + vdc_test_helper = VirtualDesktopTestHelper(context) + try: + vdc_test_helper.before_test(test_case_name) + + if vdc_test_helper.is_new_session_created(): + session_helper = SessionsTestHelper(context, vdc_test_helper.get_new_session(), context.admin_username, admin_access_token) + response = session_helper.list_allowed_instances_type_for_session() + + if response.listing is not None: + vdc_test_helper.on_test_pass(test_case_name, test_results_map) + else: + vdc_test_helper.on_test_fail(test_case_name, response, test_results_map) + + else: + testcase_error_message = f'Created session is None. Skipping {test_case_name}.' + test_results_map.update_test_result_map(VirtualDesktopSessionTestResults.FAILED, testcase_error_message) + context.error(testcase_error_message) + + except exceptions.SocaException as error: + vdc_test_helper.on_test_exception(test_case_name, error, test_results_map) + + finally: + vdc_test_helper.after_test(test_case_name, test_results_map, test_case_id) + + +def test_list_permission_profiles(context: TestContext): + test_case_name = 'Test List Permission Profiles' + test_case_id = test_constants.VIRTUAL_DESKTOP_TEST_LIST_PERMISSION_PROFILES + admin_access_token = context.get_admin_access_token() + test_results_map = SessionsTestResultMap(test_case_name) + vdc_test_helper = VirtualDesktopTestHelper(context) + + vdc_api_helper = VirtualDesktopApiHelper(context, admin_access_token, context.admin_username) + + try: + vdc_test_helper.before_test(test_case_name) + + response = vdc_api_helper.list_permission_profiles() + + if response.listing is not None: + vdc_test_helper.on_test_pass(test_case_name, test_results_map) + else: + vdc_test_helper.on_test_fail(test_case_name, response, test_results_map) + + except exceptions.SocaException as error: + vdc_test_helper.on_test_exception(test_case_name, error, test_results_map) + + finally: + vdc_test_helper.after_test(test_case_name, test_results_map, test_case_id) + + +def test_get_permission_profile(context: TestContext): + test_case_name = 'Test Get Permission Profile' + test_case_id = test_constants.VIRTUAL_DESKTOP_TEST_GET_PERMISSION_PROFILE + admin_access_token = context.get_admin_access_token() + test_results_map = SessionsTestResultMap(test_case_name) + vdc_api_helper = VirtualDesktopApiHelper(context, admin_access_token, context.admin_username) + vdc_test_helper = VirtualDesktopTestHelper(context) + + try: + vdc_test_helper.before_test(test_case_name) + + response = vdc_api_helper.get_permission_profile() + + if response is not None: + vdc_test_helper.on_test_pass(test_case_name, test_results_map) + else: + vdc_test_helper.on_test_fail(test_case_name, response, test_results_map) + + except exceptions.SocaException as error: + vdc_test_helper.on_test_exception(test_case_name, error, test_results_map) + + finally: + vdc_test_helper.after_test(test_case_name, test_results_map, test_case_id) + + +def test_get_base_permissions(context: TestContext): + test_case_name = 'Test Get Base Permissions' + test_case_id = test_constants.VIRTUAL_DESKTOP_TEST_GET_BASE_PERMISSIONS + admin_access_token = context.get_admin_access_token() + test_results_map = SessionsTestResultMap(test_case_name) + vdc_test_helper = VirtualDesktopTestHelper(context) + + vdc_api_helper = VirtualDesktopApiHelper(context, admin_access_token, context.admin_username) + + try: + vdc_test_helper.before_test(test_case_name) + + response = vdc_api_helper.get_base_permissions() + + if response is not None: + vdc_test_helper.on_test_pass(test_case_name, test_results_map) + else: + vdc_test_helper.on_test_fail(test_case_name, response, test_results_map) + + except exceptions.SocaException as error: + vdc_test_helper.on_test_exception(test_case_name, error, test_results_map) + + finally: + vdc_test_helper.after_test(test_case_name, test_results_map, test_case_id) + + +# DCV Tests +def test_describe_servers(context: TestContext): + test_case_name = 'Test Describe Servers' + test_case_id = test_constants.VIRTUAL_DESKTOP_TEST_DESCRIBE_SERVERS + admin_access_token = context.get_admin_access_token() + test_results_map = SessionsTestResultMap(test_case_name) + vdc_api_helper = VirtualDesktopApiHelper(context, admin_access_token, context.admin_username) + vdc_test_helper = VirtualDesktopTestHelper(context) + + try: + vdc_test_helper.before_test(test_case_name) + + response = vdc_api_helper.describe_servers() + + if response is not None: + vdc_test_helper.on_test_pass(test_case_name, test_results_map) + else: + vdc_test_helper.on_test_fail(test_case_name, response, test_results_map) + + except exceptions.SocaException as error: + vdc_test_helper.on_test_exception(test_case_name, error, test_results_map) + + finally: + vdc_test_helper.after_test(test_case_name, test_results_map, test_case_id) + + +def test_describe_sessions(context: TestContext): + test_case_name = 'Test Describe Sessions' + test_case_id = test_constants.VIRTUAL_DESKTOP_TEST_DESCRIBE_SESSIONS + admin_access_token = context.get_admin_access_token() + test_results_map = SessionsTestResultMap(test_case_name) + vdc_api_helper = VirtualDesktopApiHelper(context, admin_access_token, context.admin_username) + vdc_test_helper = VirtualDesktopTestHelper(context) + + try: + vdc_test_helper.before_test(test_case_name) + + response = vdc_api_helper.describe_sessions() + + if response is not None: + vdc_test_helper.on_test_pass(test_case_name, test_results_map) + else: + vdc_test_helper.on_test_fail(test_case_name, response, test_results_map) + + except exceptions.SocaException as error: + vdc_test_helper.on_test_exception(test_case_name, error, test_results_map) + + finally: + vdc_test_helper.after_test(test_case_name, test_results_map, test_case_id) + + +def test_admin_delete_sessions(context: TestContext): + test_case_name = 'Test Admin Delete Sessions' + test_case_id = test_constants.VIRTUAL_DESKTOP_TEST_ADMIN_DELETE_SESSIONS + admin_access_token = context.get_admin_access_token() + test_results_map = SessionsTestResultMap(test_case_name) + vdc_test_helper = VirtualDesktopTestHelper(context) + try: + vdc_test_helper.before_test(test_case_name) + + if vdc_test_helper.is_new_session_created(): + session_helper = SessionsTestHelper(context, vdc_test_helper.get_new_session(), context.admin_username, admin_access_token) + response = session_helper.delete_session('VirtualDesktopAdmin.DeleteSessions') + + if response.success: + vdc_test_helper.on_test_pass(test_case_name, test_results_map) + + else: + vdc_test_helper.on_test_fail(test_case_name, response, test_results_map) + + else: + testcase_error_message = f'Created session is None. Skipping {test_case_name}. ' + test_results_map.update_test_result_map(VirtualDesktopSessionTestResults.FAILED, testcase_error_message) + context.error(testcase_error_message) + + except exceptions.SocaException as error: + vdc_test_helper.on_test_exception(test_case_name, error, test_results_map) + + finally: + vdc_test_helper.after_test(test_case_name, test_results_map, test_case_id) + + +# User tests + +def test_user_create_session(context: TestContext): + namespace = 'VirtualDesktop.CreateSession' + test_case_id = test_constants.VIRTUAL_DESKTOP_TEST_USER_CREATE_SESSION + test_case_name = 'Test User Create Session' + sleep_timer = 30 + user_access_token = context.get_non_admin_access_token() + test_results_map = SessionsTestResultMap(test_case_name) + vdc_test_helper = VirtualDesktopTestHelper(context) + + try: + vdc_test_helper.before_test(test_case_name) + + sessions = get_sessions_test_cases_list(context, context.non_admin_username, user_access_token) + + new_session = create_session(context, sessions[7], user_access_token, namespace) + + sessionHelper = SessionsTestHelper(context, new_session, context.non_admin_username, user_access_token) + time.sleep(sleep_timer) + session_status = sessionHelper.wait_and_verify_session_state_matches(VirtualDesktopSessionState.READY, 'VirtualDesktop.GetSessionInfo') + + if session_status.get('session_state_matches'): + vdc_test_helper.set_new_session(new_session) + vdc_test_helper.on_test_pass(test_case_name, test_results_map) + + else: + session = sessionHelper.get_session_info('VirtualDesktop.GetSessionInfo') + testcase_error_message = f'Failed to execute {test_case_name}.Session Name : {session.name}. Session is in invalid State : {session.state}. Session ID : {session.idea_session_id}' + session_status.get('error_log') + test_results_map.update_test_result_map(VirtualDesktopSessionTestResults.FAILED, testcase_error_message) + + except (exceptions.SocaException, Exception) as error: + vdc_test_helper.on_test_exception(test_case_name, error, test_results_map) + + finally: + vdc_test_helper.after_test(test_case_name, test_results_map, test_case_id) + + +def test_user_get_session_screenshot(context: TestContext): + test_case_name = 'Test User Get Session Screenshot' + test_results_map = SessionsTestResultMap(test_case_name) + test_case_id = test_constants.VIRTUAL_DESKTOP_TEST_USER_GET_SESSION_SCREENSHOT + user_access_token = context.get_non_admin_access_token() + vdc_test_helper = VirtualDesktopTestHelper(context) + + try: + vdc_test_helper.before_test(test_case_name) + + if vdc_test_helper.is_new_session_created(): + session_helper = SessionsTestHelper(context, vdc_test_helper.get_new_session(), context.non_admin_username, user_access_token) + response = session_helper.get_session_screenshot('VirtualDesktop.GetSessionScreenshot') + + if response.success is not None: + vdc_test_helper.on_test_pass(test_case_name, test_results_map) + + else: + vdc_test_helper.on_test_fail(test_case_name, response, test_results_map) + + else: + testcase_error_message = f'Created session is None. Skipping {test_case_name}' + test_results_map.update_test_result_map(VirtualDesktopSessionTestResults.FAILED, testcase_error_message) + context.error(testcase_error_message) + assert False + + except exceptions.SocaException as error: + vdc_test_helper.on_test_exception(test_case_name, error, test_results_map) + + finally: + vdc_test_helper.after_test(test_case_name, test_results_map, test_case_id) + + +def test_user_update_session(context: TestContext): + test_case_name = 'Test User Update Session' + test_case_id = test_constants.VIRTUAL_DESKTOP_TEST_USER_UPDATE_SESSION + user_access_token = context.get_non_admin_access_token() + test_results_map = SessionsTestResultMap(test_case_name) + vdc_test_helper = VirtualDesktopTestHelper(context) + + try: + vdc_test_helper.before_test(test_case_name) + + if vdc_test_helper.is_new_session_created(): + session_helper = SessionsTestHelper(context, vdc_test_helper.get_new_session(), context.non_admin_username, user_access_token) + response = session_helper.update_session('VirtualDesktop.UpdateSession') + + if response.session is not None: + vdc_test_helper.on_test_pass(test_case_name, test_results_map) + + else: + vdc_test_helper.on_test_fail(test_case_name, response, test_results_map) + + else: + testcase_error_message = f'Created session is None. Skipping {test_case_name}' + test_results_map.update_test_result_map(VirtualDesktopSessionTestResults.FAILED, testcase_error_message) + context.error(testcase_error_message) + assert False + + except exceptions.SocaException as error: + vdc_test_helper.on_test_exception(test_case_name, error, test_results_map) + + finally: + vdc_test_helper.after_test(test_case_name, test_results_map, test_case_id) + + +def test_user_get_session_connection_info(context: TestContext): + test_case_name = 'Test User Get Session Connection Info' + test_case_id = test_constants.VIRTUAL_DESKTOP_TEST_USER_GET_SESSION_CONNECTION_INFO + user_access_token = context.get_non_admin_access_token() + test_results_map = SessionsTestResultMap(test_case_name) + vdc_test_helper = VirtualDesktopTestHelper(context) + + try: + vdc_test_helper.before_test(test_case_name) + + if vdc_test_helper.is_new_session_created(): + session_helper = SessionsTestHelper(context, vdc_test_helper.get_new_session(), context.non_admin_username, user_access_token) + response = session_helper.get_session_connection_info('VirtualDesktop.GetSessionConnectionInfo') + + if response is not None: + vdc_test_helper.on_test_pass(test_case_name, test_results_map) + + else: + vdc_test_helper.on_test_fail(test_case_name, response, test_results_map) + + else: + testcase_error_message = f'Created session is None. Skipping {test_case_name}.' + test_results_map.update_test_result_map(VirtualDesktopSessionTestResults.FAILED, testcase_error_message) + context.error(testcase_error_message) + assert False + + except exceptions.SocaException as error: + vdc_test_helper.on_test_exception(test_case_name, error, test_results_map) + + finally: + vdc_test_helper.after_test(test_case_name, test_results_map, test_case_id) + + +def test_user_get_session_info(context: TestContext): + test_case_name = 'Test User Get Session Info' + test_case_id = test_constants.VIRTUAL_DESKTOP_TEST_USER_GET_SESSION_INFO + user_access_token = context.get_non_admin_access_token() + test_results_map = SessionsTestResultMap(test_case_name) + vdc_test_helper = VirtualDesktopTestHelper(context) + + try: + vdc_test_helper.before_test(test_case_name) + + if vdc_test_helper.is_new_session_created(): + session_helper = SessionsTestHelper(context, vdc_test_helper.get_new_session(), context.non_admin_username, user_access_token) + response = session_helper.get_session_info('VirtualDesktop.GetSessionInfo') + + if response.base_os is not None: + vdc_test_helper.on_test_pass(test_case_name, test_results_map) + + else: + vdc_test_helper.on_test_fail(test_case_name, response, test_results_map) + + else: + testcase_error_message = f'Created session is None. Skipping {test_case_name}.' + test_results_map.update_test_result_map(VirtualDesktopSessionTestResults.FAILED, testcase_error_message) + context.error(testcase_error_message) + assert False + + except exceptions.SocaException as error: + vdc_test_helper.on_test_exception(test_case_name, error, test_results_map) + + finally: + vdc_test_helper.after_test(test_case_name, test_results_map, test_case_id) + + +def test_user_session_workflow(context: TestContext): + testcase_id = test_constants.VIRTUAL_DESKTOP_TEST_USER_SESSION_WORKFLOW + vdc_test_helper = VirtualDesktopTestHelper(context) + try: + if vdc_test_helper.is_new_session_created(): + user_access_token = context.get_non_admin_access_token() + session_workflow = SessionWorkflow(context, vdc_test_helper.get_new_session(), testcase_id, context.non_admin_username, user_access_token, 'VirtualDesktop.GetSessionInfo') + session_workflow.test_session_workflow('user') + else: + testcase_error_message = f'Created session is None. Skipping {testcase_id}.' + context.error(testcase_error_message) + except exceptions.SocaException as e: + context.error(f'Failed to execute test user session workflow. Error Code : {e.error_code} Error: {e.message}') + + +def test_user_list_sessions(context: TestContext): + test_case_name = 'Test User List Sessions' + test_case_id = test_constants.VIRTUAL_DESKTOP_TEST_USER_LIST_SESSIONS + non_admin_access_token = context.get_non_admin_access_token() + test_results_map = SessionsTestResultMap(test_case_name) + vdc_api_helper = VirtualDesktopApiHelper(context, non_admin_access_token, context.non_admin_username) + vdc_test_helper = VirtualDesktopTestHelper(context) + + try: + vdc_test_helper.before_test(test_case_name) + + response = vdc_api_helper.list_sessions('VirtualDesktop.ListSessions') + + if response.listing is not None: + vdc_test_helper.on_test_pass(test_case_name, test_results_map) + + else: + vdc_test_helper.on_test_fail(test_case_name, response, test_results_map) + + except exceptions.SocaException as error: + vdc_test_helper.on_test_exception(test_case_name, error, test_results_map) + + finally: + vdc_test_helper.after_test(test_case_name, test_results_map, test_case_id) + + +def test_user_list_software_stacks(context: TestContext): + test_case_name = 'Test User List Software Stacks' + test_case_id = test_constants.VIRTUAL_DESKTOP_TEST_USER_LIST_SOFTWARE_STACKS + user_access_token = context.get_non_admin_access_token() + test_results_map = SessionsTestResultMap(test_case_name) + vdc_api_helper = VirtualDesktopApiHelper(context, user_access_token, context.non_admin_username) + vdc_test_helper = VirtualDesktopTestHelper(context) + + try: + vdc_test_helper.before_test(test_case_name) + + response = vdc_api_helper.list_software_stacks('VirtualDesktop.ListSoftwareStacks') + + if response is not None: + vdc_test_helper.on_test_pass(test_case_name, test_results_map) + + else: + vdc_test_helper.on_test_fail(test_case_name, response, test_results_map) + + except exceptions.SocaException as error: + vdc_test_helper.on_test_exception(test_case_name, error, test_results_map) + + finally: + vdc_test_helper.after_test(test_case_name, test_results_map, test_case_id) + + +def test_user_update_session_permissions(context: TestContext): + test_case_name = 'Test User Update Session Permission' + test_case_id = test_constants.VIRTUAL_DESKTOP_TEST_USER_UPDATE_SESSION_PERMISSIONS + user_access_token = context.get_non_admin_access_token() + test_results_map = SessionsTestResultMap(test_case_name) + vdc_test_helper = VirtualDesktopTestHelper(context) + try: + vdc_test_helper.before_test(test_case_name) + + if vdc_test_helper.is_new_session_created(): + session_helper = SessionsTestHelper(context, vdc_test_helper.get_new_session(), context.non_admin_username, user_access_token) + response = session_helper.update_session_permissions('VirtualDesktop.UpdateSessionPermissions') + + if response is not None: + vdc_test_helper.on_test_pass(test_case_name, test_results_map) + else: + vdc_test_helper.on_test_fail(test_case_name, response, test_results_map) + + else: + testcase_error_message = f'Created session is None. Skipping {test_case_name}.' + test_results_map.update_test_result_map(VirtualDesktopSessionTestResults.FAILED, testcase_error_message) + context.error(testcase_error_message) + assert False + + except exceptions.SocaException as error: + vdc_test_helper.on_test_exception(test_case_name, error, test_results_map) + + finally: + vdc_test_helper.after_test(test_case_name, test_results_map, test_case_id) + + +def test_user_list_session_permissions(context: TestContext): + test_case_name = 'Test User List Session Permissions' + test_case_id = test_constants.VIRTUAL_DESKTOP_TEST_USER_LIST_SESSION_PERMISSIONS + user_access_token = context.get_non_admin_access_token() + test_results_map = SessionsTestResultMap(test_case_name) + vdc_test_helper = VirtualDesktopTestHelper(context) + + try: + vdc_test_helper.before_test(test_case_name) + + if vdc_test_helper.is_new_session_created(): + session_helper = SessionsTestHelper(context, vdc_test_helper.get_new_session(), context.non_admin_username, user_access_token) + response = session_helper.list_session_permissions('VirtualDesktop.ListSessionPermissions') + + if response is not None: + vdc_test_helper.on_test_pass(test_case_name, test_results_map) + + else: + vdc_test_helper.on_test_fail(test_case_name, response, test_results_map) + + else: + testcase_error_message = f'Created session is None. Skipping {test_case_name}. ' + test_results_map.update_test_result_map(VirtualDesktopSessionTestResults.FAILED, testcase_error_message) + context.error(testcase_error_message) + assert False + + except exceptions.SocaException as error: + vdc_test_helper.on_test_exception(test_case_name, error, test_results_map) + + finally: + vdc_test_helper.after_test(test_case_name, test_results_map, test_case_id) + + +def test_user_list_shared_permissions(context: TestContext): + test_case_name = 'Test User List Shared Permissions' + test_case_id = test_constants.VIRTUAL_DESKTOP_TEST_USER_LIST_SHARED_PERMISSIONS + user_access_token = context.get_non_admin_access_token() + test_results_map = SessionsTestResultMap(test_case_name) + vdc_test_helper = VirtualDesktopTestHelper(context) + try: + vdc_test_helper.before_test(test_case_name) + + if vdc_test_helper.is_new_session_created(): + session_helper = SessionsTestHelper(context, vdc_test_helper.get_new_session(), context.non_admin_username, user_access_token) + response = session_helper.list_shared_permissions('VirtualDesktop.ListSharedPermissions') + + if response is not None: + vdc_test_helper.on_test_pass(test_case_name, test_results_map) + + else: + vdc_test_helper.on_test_fail(test_case_name, response, test_results_map) + + else: + testcase_error_message = f'Created session is None. Skipping {test_case_name}. ' + test_results_map.update_test_result_map(VirtualDesktopSessionTestResults.FAILED, testcase_error_message) + context.error(testcase_error_message) + assert False + + except exceptions.SocaException as error: + vdc_test_helper.on_test_exception(test_case_name, error, test_results_map) + + finally: + vdc_test_helper.after_test(test_case_name, test_results_map, test_case_id) + + +TEST_CASES = [ + { + 'test_case_id': test_constants.VIRTUAL_DESKTOP_TEST_ADMIN_BATCH_SESSION_WORKFLOW, + 'test_case': test_admin_batch_session_workflow + }, + + { + 'test_case_id': test_constants.VIRTUAL_DESKTOP_TEST_ADMIN_CREATE_SESSION, + 'test_case': test_admin_create_session + }, + + { + 'test_case_id': test_constants.VIRTUAL_DESKTOP_TEST_ADMIN_GET_SESSION_SCREENSHOT, + 'test_case': test_admin_get_session_screenshot + }, + + { + 'test_case_id': test_constants.VIRTUAL_DESKTOP_TEST_ADMIN_UPDATE_SESSION, + 'test_case': test_admin_update_session + }, + + { + 'test_case_id': test_constants.VIRTUAL_DESKTOP_TEST_ADMIN_GET_SESSION_INFO, + 'test_case': test_admin_get_session_info + }, + + { + 'test_case_id': test_constants.VIRTUAL_DESKTOP_TEST_ADMIN_LIST_SESSIONS, + 'test_case': test_admin_list_sessions + }, + + { + 'test_case_id': test_constants.VIRTUAL_DESKTOP_TEST_ADMIN_LIST_SOFTWARE_STACKS, + 'test_case': test_admin_list_software_stacks + }, + { + 'test_case_id': test_constants.VIRTUAL_DESKTOP_TEST_ADMIN_CREATE_SOFTWARE_STACK, + 'test_case': test_admin_create_software_stack + }, + { + 'test_case_id': test_constants.VIRTUAL_DESKTOP_TEST_ADMIN_UPDATE_SOFTWARE_STACK, + 'test_case': test_admin_update_software_stack + }, + + { + 'test_case_id': test_constants.VIRTUAL_DESKTOP_TEST_ADMIN_CREATE_SOFTWARE_STACK_FROM_SESSION, + 'test_case': test_admin_create_software_stack_from_session + }, + { + 'test_case_id': test_constants.VIRTUAL_DESKTOP_TEST_ADMIN_GET_SESSION_CONNECTION_INFO, + 'test_case': test_admin_get_session_connection_info + }, + { + 'test_case_id': test_constants.VIRTUAL_DESKTOP_TEST_ADMIN_REINDEX_USER_SESSIONS, + 'test_case': test_admin_reindex_user_sessions + }, + { + 'test_case_id': test_constants.VIRTUAL_DESKTOP_TEST_ADMIN_REINDEX_SOFTWARE_STACKS, + 'test_case': test_admin_reindex_software_stacks + }, + { + 'test_case_id': test_constants.VIRTUAL_DESKTOP_TEST_ADMIN_CREATE_PERMISSION_PROFILE, + 'test_case': test_admin_create_permission_profile + }, + + { + 'test_case_id': test_constants.VIRTUAL_DESKTOP_TEST_ADMIN_LIST_SESSION_PERMISSIONS, + 'test_case': test_admin_list_session_permissions + }, + { + 'test_case_id': test_constants.VIRTUAL_DESKTOP_TEST_ADMIN_LIST_SHARED_PERMISSIONS, + 'test_case': test_admin_list_shared_permissions + }, + { + 'test_case_id': test_constants.VIRTUAL_DESKTOP_TEST_ADMIN_UPDATE_PERMISSION_PROFILE, + 'test_case': test_admin_update_permission_profile + }, + { + 'test_case_id': test_constants.VIRTUAL_DESKTOP_TEST_ADMIN_UPDATE_SESSION_PERMISSIONS, + 'test_case': test_admin_update_session_permissions + }, + { + 'test_case_id': test_constants.VIRTUAL_DESKTOP_TEST_ADMIN_GET_SOFTWARE_STACK_INFO, + 'test_case': test_admin_get_software_stack_info + }, + + { + 'test_case_id': test_constants.VIRTUAL_DESKTOP_TEST_LIST_SUPPORTED_OS, + 'test_case': test_list_supported_os + }, + + { + 'test_case_id': test_constants.VIRTUAL_DESKTOP_TEST_LIST_SUPPORTED_GPU, + 'test_case': test_list_supported_gpu + }, + + { + 'test_case_id': test_constants.VIRTUAL_DESKTOP_TEST_LIST_SCHEDULE_TYPES, + 'test_case': test_list_schedule_types + }, + { + 'test_case_id': test_constants.VIRTUAL_DESKTOP_TEST_LIST_ALLOWED_INSTANCE_TYPES, + 'test_case': test_list_allowed_instance_types + }, + { + 'test_case_id': test_constants.VIRTUAL_DESKTOP_TEST_LIST_ALLOWED_INSTANCE_TYPES_FOR_SESSION, + 'test_case': test_list_allowed_instance_types_for_session + }, + + { + 'test_case_id': test_constants.VIRTUAL_DESKTOP_TEST_GET_BASE_PERMISSIONS, + 'test_case': test_get_base_permissions + }, + { + 'test_case_id': test_constants.VIRTUAL_DESKTOP_TEST_LIST_PERMISSION_PROFILES, + 'test_case': test_list_permission_profiles + }, + { + 'test_case_id': test_constants.VIRTUAL_DESKTOP_TEST_GET_PERMISSION_PROFILE, + 'test_case': test_get_permission_profile + }, + + { + 'test_case_id': test_constants.VIRTUAL_DESKTOP_TEST_DESCRIBE_SERVERS, + 'test_case': test_describe_servers + }, + + { + 'test_case_id': test_constants.VIRTUAL_DESKTOP_TEST_DESCRIBE_SERVERS, + 'test_case': test_describe_sessions + }, + + { + 'test_case_id': test_constants.VIRTUAL_DESKTOP_TEST_USER_CREATE_SESSION, + 'test_case': test_user_create_session + }, + { + 'test_case_id': test_constants.VIRTUAL_DESKTOP_TEST_USER_GET_SESSION_SCREENSHOT, + 'test_case': test_user_get_session_screenshot + }, + { + 'test_case_id': test_constants.VIRTUAL_DESKTOP_TEST_USER_UPDATE_SESSION, + 'test_case': test_user_update_session + }, + { + 'test_case_id': test_constants.VIRTUAL_DESKTOP_TEST_USER_GET_SESSION_CONNECTION_INFO, + 'test_case': test_user_get_session_connection_info + }, + { + 'test_case_id': test_constants.VIRTUAL_DESKTOP_TEST_USER_GET_SESSION_INFO, + 'test_case': test_user_get_session_info + }, + { + 'test_case_id': test_constants.VIRTUAL_DESKTOP_TEST_USER_SESSION_WORKFLOW, + 'test_case': test_user_session_workflow + }, + + { + 'test_case_id': test_constants.VIRTUAL_DESKTOP_TEST_USER_LIST_SESSIONS, + 'test_case': test_user_list_sessions + }, + { + 'test_case_id': test_constants.VIRTUAL_DESKTOP_TEST_USER_LIST_SOFTWARE_STACKS, + 'test_case': test_user_list_software_stacks + }, + { + 'test_case_id': test_constants.VIRTUAL_DESKTOP_TEST_USER_UPDATE_SESSION_PERMISSIONS, + 'test_case': test_user_update_session_permissions + }, + { + 'test_case_id': test_constants.VIRTUAL_DESKTOP_TEST_USER_LIST_SESSION_PERMISSIONS, + 'test_case': test_user_list_session_permissions + }, + { + 'test_case_id': test_constants.VIRTUAL_DESKTOP_TEST_USER_LIST_SHARED_PERMISSIONS, + 'test_case': test_user_list_shared_permissions + }, + { + 'test_case_id': test_constants.VIRTUAL_DESKTOP_TEST_ADMIN_DELETE_SESSIONS, + 'test_case': test_admin_delete_sessions + } +] diff --git a/source/idea/idea-administrator/src/ideaadministrator/integration_tests/virtual_desktop_tests_util.py b/source/idea/idea-administrator/src/ideaadministrator/integration_tests/virtual_desktop_tests_util.py new file mode 100644 index 00000000..5eb34f5f --- /dev/null +++ b/source/idea/idea-administrator/src/ideaadministrator/integration_tests/virtual_desktop_tests_util.py @@ -0,0 +1,1222 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. +from enum import Enum +import ideaadministrator +from ideasdk.utils import Utils +from ideadatamodel import ( + VirtualDesktopBaseOS, + Project, + VirtualDesktopSession, + VirtualDesktopSoftwareStack, + VirtualDesktopServer, + SocaMemory, + SocaMemoryUnit, + VirtualDesktopSessionScreenshot, + VirtualDesktopGPU, + VirtualDesktopSessionConnectionInfo, + VirtualDesktopSessionPermission, + GetSessionScreenshotRequest, + GetSessionScreenshotResponse, + CreateSoftwareStackResponse, + CreateSoftwareStackRequest, + UpdateSoftwareStackRequest, + UpdateSoftwareStackResponse, + UpdateSessionPermissionRequest, + UpdateSessionPermissionResponse, + CreateSoftwareStackFromSessionRequest, + CreateSoftwareStackFromSessionResponse, + UpdateSessionRequest, + UpdateSessionResponse, + StopSessionRequest, + StopSessionResponse, + ResumeSessionsRequest, + ResumeSessionsResponse, + RebootSessionRequest, + RebootSessionResponse, + DeleteSessionRequest, + DeleteSessionResponse, + GetSoftwareStackInfoRequest, + GetSoftwareStackInfoResponse, + GetSessionConnectionInfoRequest, + GetSessionConnectionInfoResponse, + ListPermissionsRequest, + ListPermissionsResponse, + ListAllowedInstanceTypesForSessionRequest, + ListAllowedInstanceTypesForSessionResponse, + VirtualDesktopSessionState, + VirtualDesktopPermissionProfile, + VirtualDesktopPermission, + CreatePermissionProfileRequest, + CreatePermissionProfileResponse, + UpdatePermissionProfileRequest, + UpdatePermissionProfileResponse, + ListSoftwareStackRequest, + ListSoftwareStackResponse, + ListSessionsRequest, + ListSessionsResponse, + ReIndexUserSessionsRequest, + ReIndexUserSessionsResponse, + ReIndexSoftwareStacksRequest, + ReIndexSoftwareStacksResponse, + ListSupportedOSRequest, + ListSupportedOSResponse, + ListSupportedGPURequest, + ListSupportedGPUResponse, + ListScheduleTypesRequest, + ListScheduleTypesResponse, + ListAllowedInstanceTypesRequest, + ListAllowedInstanceTypesResponse, + ListPermissionProfilesRequest, + ListPermissionProfilesResponse, + GetPermissionProfileRequest, + GetPermissionProfileResponse, + GetBasePermissionsRequest, + GetBasePermissionsResponse, + DescribeServersRequest, + DescribeServersResponse, + DescribeSessionsRequest, + DescribeSessionsResponse, + GetUserProjectsRequest, + GetUserProjectsResult, + BatchCreateSessionRequest, + CreateSessionRequest, + GetSessionInfoRequest +) +from ideaadministrator.integration_tests.test_context import TestContext +import time +from defusedxml import cElementTree +from defusedxml.ElementTree import parse +from ideadatamodel import ( + exceptions +) + +import os +from typing import Dict, List, Optional + +__new_created_session__: VirtualDesktopSession = None +__new_software_stack__: VirtualDesktopSoftwareStack = None +__is_test_results_report_created__ = False +__vdc_test_results__ = {} + + +class VirtualDesktopSessionTestcases(str, Enum): + CREATE_SESSION = 'CREATE' + STOP_SESSION = 'STOP' + RESUME_SESSION = 'RESUME' + REBOOT_SESSION = 'REBOOT' + DELETE_SESSION = 'DELETE' + + +class VirtualDesktopSessionTestResults(str, Enum): + PASS = 'PASS' + FAILED = 'FAIL' + SKIP = 'SKIP' + + +class SessionsTestResultMap: + def __init__(self, test_case_name: str): + self.__test_case_name = test_case_name + self.test_results: List = [] + self.__test_results_map = {} + + def update_test_result_map(self, test_case_status: str, test_case_error_message: str = ''): + self.__test_results_map['test_case_name'] = self.__test_case_name + self.__test_results_map['test_case_status'] = test_case_status + self.__test_results_map['test_case_error_message'] = test_case_error_message + self.test_results.append(self.__test_results_map) + + def get_test_results(self) -> [Dict]: + return self.test_results + + +class TestReportHelper: + def __init__(self, context: TestContext, test_case_id: str, test_results_file): + self.context = context + self.test_case_id = test_case_id + self.test_results_file = test_results_file + + def create_test_suites_element(self) -> cElementTree: + """Create a element in test results file""" + test_suites = cElementTree.Element('testsuites', name='Test_Run' + self.context.test_run_id) + return test_suites + + def create_test_suite_element(self, test_suites) -> cElementTree: + """Create a element in test results file """ + test_results_summary = _get_test_summary_for_testsuite(self.test_case_id) + test_suite = cElementTree.SubElement(test_suites, 'testsuite', name=self.test_case_id, tests=test_results_summary['total_tests'], failures=test_results_summary['total_tests_failed'], skips=test_results_summary['total_tests_skipped']) + return test_suite + + def create_test_cases_in_a_test_suite(self, test_cases_results: Dict, test_suite: cElementTree): + """Create a element in test results file """ + for test_info in test_cases_results[self.test_case_id]: + test_case_name = test_info['test_case_name'] + test_case_status = test_info['test_case_status'] + test_case_error_message = test_info['test_case_error_message'] + test_case = cElementTree.SubElement(test_suite, 'testcase', name=test_case_name) + if test_case_status == VirtualDesktopSessionTestResults.FAILED: + cElementTree.SubElement(test_case, 'failure', message=test_case_error_message) + + def write_to_xml_file(self, test_report_tree: cElementTree): + test_report_tree.write(self.test_results_file, + xml_declaration=True, encoding='utf-8', + method='xml') + + def parse_file_and_return_tree_element(self) -> cElementTree: + test_result_tree = parse(self.test_results_file) + return test_result_tree + + @staticmethod + def append_test_suites_to_test_report_tree(test_result_tree: cElementTree): + test_suites = test_result_tree.getroot() + return test_suites + + +class SessionPayload: + + def __init__(self, context: TestContext, session_data: Dict, username: str, access_token: str): + self.name = session_data.get('name') + self.base_os = VirtualDesktopBaseOS(session_data.get('base_os')) + self.software_stack_id = session_data.get('software_stack_id') + self.hibernation_enabled = session_data.get('hibernation_enabled') + self.instance_type = session_data.get('instance_type') + self.value = session_data.get('storage_size') + self.context = context + self.project: Project + self.username = username + self.access_token = access_token + + projects_list = _get_user_projects_list(self.context, self.username, self.access_token) + for project in projects_list: + if 'default' in project.name: + self.project = project + + def get_session_payload(self) -> List[VirtualDesktopSession]: + try: + session_payload = [VirtualDesktopSession( + name=self.name, owner=self.username, + software_stack=VirtualDesktopSoftwareStack( + base_os=self.base_os, + stack_id=self.software_stack_id, + ), hibernation_enabled=self.hibernation_enabled, + project=self.project, + server=VirtualDesktopServer( + instance_type=self.instance_type, + root_volume_size=SocaMemory( + value=self.value, + unit=SocaMemoryUnit.GB + ) + ))] + return session_payload + except exceptions.SocaException as e: + self.context.info(f'Failed to get Session payload.Error : {e}') + + +class SessionsTestHelper: + + def __init__(self, context: TestContext, session: VirtualDesktopSession, username: str, access_token: str): + self.context = context + self.session = session + self.session_list: List[VirtualDesktopSession] = [self.session] + self.prefix_text = 'TEST STATUS : ' + self.session_id = self.session.idea_session_id + self.access_token = access_token + self.ami_id = self.session.software_stack.ami_id + self.stack_id = self.session.software_stack.stack_id + self.image_type = self.session.type + self.dcv_session_id = self.session.dcv_session_id + self.idea_session_id = self.session.idea_session_id + self.idea_session_owner = self.session.owner + self.create_time = self.session.created_on.ctime() + self.failure_reason = self.session.failure_reason + self.projects_list: [Project] + self.username = username + + self.projects_list = _get_user_projects_list(self.context, self.username, self.access_token) + + def _get_session_screenshot_payload(self) -> List[VirtualDesktopSessionScreenshot]: + try: + session_payload = [VirtualDesktopSessionScreenshot( + image_type=self.image_type, + dcv_session_id=self.dcv_session_id, + idea_session_id=self.idea_session_id, + idea_session_owner=self.idea_session_owner, + create_time=self.create_time, + failure_reason=self.failure_reason, + )] + return session_payload + + except exceptions.SocaException as e: + self.context.info(f'Failed to get Session Screenshot payload. Error : {e}') + + def _get_session_software_stack_payload(self) -> VirtualDesktopSoftwareStack: + software_stack_name = 'VDC_Tests' + try: + session_payload = VirtualDesktopSoftwareStack(base_os=VirtualDesktopBaseOS.WINDOWS, + ami_id=self.ami_id, + description="Integration Tests", + name=software_stack_name, + min_storage=SocaMemory( + value=10, + unit=SocaMemoryUnit.GB + ), min_ram=SocaMemory(value=2, + unit=SocaMemoryUnit.GB), + gpu=VirtualDesktopGPU.NO_GPU, + failure_reason=self.failure_reason, + projects=self.projects_list + ) + return session_payload + + except exceptions.SocaException as e: + self.context.info(f'Failed to get Software Stack Payload. Error : {e}') + + def _get_session_connection_info_payload(self) -> List[VirtualDesktopSessionConnectionInfo]: + + try: + session_payload = [VirtualDesktopSessionConnectionInfo( + dcv_session_id=self.dcv_session_id, + idea_session_id=self.idea_session_id, + idea_session_owner=self.idea_session_owner, + failure_reason=self.failure_reason, + )] + return session_payload + + except exceptions.SocaException as e: + self.context.info(f'Failed to get Session payload. Error : {e}') + + def _get_session_permission_payload(self) -> [VirtualDesktopSessionPermission]: + permission_profile = VirtualDesktopPermissionProfile() + try: + session_permission_payload = [VirtualDesktopSessionPermission( + actor_name='clusteradmin', + idea_session_id=self.idea_session_id, + idea_session_owner=self.session.owner, + idea_session_name=self.session.name, + idea_session_state=self.session.state, + idea_session_base_os=self.session.base_os, + idea_session_created_on=self.session.created_on, + idea_session_hibernation_enabled=self.session.hibernation_enabled, + permission_profile=permission_profile.get_permission('admin_profile'))] + return session_permission_payload + + except exceptions.SocaException as e: + self.context.error(f'Failed to get Session Permission payload. Error : {e}') + + def get_session_screenshot(self, namespace: str) -> GetSessionScreenshotResponse: + try: + response = self.context.get_virtual_desktop_controller_client(timeout=7200).invoke_alt( + namespace=namespace, + payload=GetSessionScreenshotRequest(screenshots=self._get_session_screenshot_payload()), + result_as=GetSessionScreenshotResponse, + access_token=self.access_token) + return response + except (exceptions.SocaException, Exception) as e: + self.context.error(f'Failed to Get Session Screenshot. Error : {e}') + + def create_software_stack(self) -> CreateSoftwareStackResponse: + try: + response = self.context.get_virtual_desktop_controller_client(timeout=7200).invoke_alt( + namespace='VirtualDesktopAdmin.CreateSoftwareStack', + payload=CreateSoftwareStackRequest(software_stack=self._get_session_software_stack_payload()), + result_as=CreateSoftwareStackResponse, + access_token=self.access_token) + return response + except (exceptions.SocaException, Exception) as e: + self.context.error(f'Failed to Create Software Stack. Error : {e}') + + def update_software_stack(self, software_stack: VirtualDesktopSoftwareStack) -> UpdateSoftwareStackResponse: + try: + + response = self.context.get_virtual_desktop_controller_client(timeout=7200).invoke_alt( + namespace='VirtualDesktopAdmin.UpdateSoftwareStack', + payload=UpdateSoftwareStackRequest(software_stack=software_stack), + result_as=UpdateSoftwareStackResponse, + access_token=self.access_token) + return response + + except (exceptions.SocaException, Exception) as e: + self.context.error(f'Failed to Update Software Stack. Error : {e}') + + def update_session_permissions(self, namespace: str) -> UpdateSessionPermissionResponse: + try: + + response = self.context.get_virtual_desktop_controller_client(timeout=7200).invoke_alt( + namespace=namespace, + payload=UpdateSessionPermissionRequest(create=self._get_session_permission_payload()), + result_as=UpdateSessionPermissionResponse, + access_token=self.access_token) + return response + + except (exceptions.SocaException, Exception) as e: + self.context.error(f'Failed to Update Session Permissions. Error : {e}') + + def create_software_stack_from_session(self) -> CreateSoftwareStackFromSessionResponse: + try: + software_stack_payload = self._get_session_software_stack_payload() + response = self.context.get_virtual_desktop_controller_client(timeout=7200).invoke_alt( + namespace='VirtualDesktopAdmin.CreateSoftwareStackFromSession', + payload=CreateSoftwareStackFromSessionRequest(session=self.session, new_software_stack=software_stack_payload), + result_as=CreateSoftwareStackFromSessionResponse, + access_token=self.access_token) + return response + + except (exceptions.SocaException, Exception) as e: + self.context.error(f'Failed to Create Software Stack from Session. Error : {e}') + + def update_session(self, namespace: str) -> UpdateSessionResponse: + try: + response = self.context.get_virtual_desktop_controller_client(timeout=7200).invoke_alt( + namespace=namespace, + payload=UpdateSessionRequest(session=self.session), + result_as=UpdateSessionResponse, + access_token=self.access_token) + return response + + except (exceptions.SocaException, Exception) as e: + self.context.error(f'Failed to Update Session.Session Name: {self.session.name} Error : {e}') + + def stop_sessions(self, namespace: str) -> StopSessionResponse: + try: + response = self.context.get_virtual_desktop_controller_client(timeout=7200).invoke_alt( + namespace=namespace, + payload=StopSessionRequest(sessions=self.session_list), + access_token=self.access_token) + time.sleep(20) + return response + except (exceptions.SocaException, Exception) as e: + self.context.error(f'Failed to Stop Session.Session Name: {self.session.name} Error : {e}') + + def resume_sessions(self, namespace: str) -> ResumeSessionsResponse: + try: + response = self.context.get_virtual_desktop_controller_client(timeout=7200).invoke_alt( + namespace=namespace, + payload=ResumeSessionsRequest(sessions=self.session_list), + access_token=self.access_token) + return response + except(exceptions.SocaException, Exception) as e: + self.context.error(f'Failed to Resume Sessions.Session Name: {self.session.name} Error : {e}') + + def reboot_sessions(self, namespace: str) -> RebootSessionResponse: + try: + response = self.context.get_virtual_desktop_controller_client(timeout=7200).invoke_alt( + namespace=namespace, + payload=RebootSessionRequest(sessions=self.session_list), + access_token=self.access_token) + return response + except(exceptions.SocaException, Exception) as e: + self.context.error(f'Failed to Reboot Sessions.Session Name: {self.session.name} Error : {e}') + + def delete_session(self, namespace: str) -> DeleteSessionResponse: + try: + response = self.context.get_virtual_desktop_controller_client(timeout=7200).invoke_alt( + namespace=namespace, + payload=DeleteSessionRequest(sessions=self.session_list), + access_token=self.access_token) + return response + except (exceptions.SocaException, Exception) as e: + self.context.error(f'Failed to Delete Session.Session Name: {self.session.name} Error : {e}') + + def get_session_info(self, namespace: str) -> VirtualDesktopSession: + try: + time.sleep(10) + response = self.context.get_virtual_desktop_controller_client(timeout=7200).invoke_alt(namespace=namespace, + payload=GetSessionInfoRequest(session=self.session), + access_token=self.access_token) + session_info: VirtualDesktopSession = VirtualDesktopSession(**response.session) + return session_info + except Exception as e: + self.context.error(f'Failed to get Session Info. Session Name: {self.session.name} Error : {e}') + + def get_software_stack_info(self) -> GetSoftwareStackInfoResponse: + try: + time.sleep(10) + response = self.context.get_virtual_desktop_controller_client(timeout=7200).invoke_alt(namespace='VirtualDesktopAdmin.GetSoftwareStackInfo', + payload=GetSoftwareStackInfoRequest(stack_id=self.stack_id), + access_token=self.access_token) + return response + except Exception as e: + self.context.error(f'Failed to Get Software Stack Info. Error : {e}') + + def get_session_connection_info(self, namespace: str) -> GetSessionConnectionInfoResponse: + try: + connection_info = self._get_session_connection_info_payload() + response = self.context.get_virtual_desktop_controller_client(timeout=7200).invoke_alt(namespace=namespace, + payload=GetSessionConnectionInfoRequest(connection_info=connection_info[0]), + result_as=GetSessionConnectionInfoResponse, + access_token=self.access_token) + return response + except Exception as e: + self.context.error(f'Failed to Get Session Connection Info. Error : {e}') + + def update_user_session(self) -> UpdateSessionResponse: + try: + response = self.context.get_virtual_desktop_controller_client(timeout=7200).invoke_alt( + namespace='VirtualDesktopAdmin.UpdateUserSession', + payload=UpdateSessionRequest(session=self.session), + result_as=UpdateSessionResponse, + access_token=self.access_token) + return response + except (exceptions.SocaException, Exception) as e: + self.context.error(f'Failed to Update User Session. Error : {e}') + + def list_session_permissions(self, namespace: str) -> ListPermissionsResponse: + try: + response = self.context.get_virtual_desktop_controller_client(timeout=7200).invoke_alt( + namespace=namespace, + payload=ListPermissionsRequest(idea_session_id=self.idea_session_id, username=self.context.admin_username), + result_as=ListPermissionsResponse, + access_token=self.access_token) + return response + except (exceptions.SocaException, Exception) as e: + self.context.error(f'Failed to List Session Permissions. Error : {e}') + + def list_shared_permissions(self, namespace: str) -> ListPermissionsResponse: + try: + response = self.context.get_virtual_desktop_controller_client(timeout=7200).invoke_alt( + namespace=namespace, + payload=ListPermissionsRequest(idea_session_id=self.idea_session_id, username=self.context.admin_username), + result_as=ListPermissionsResponse, + access_token=self.access_token) + return response + except (exceptions.SocaException, Exception) as e: + self.context.error(f'Failed to List Shared Permissions. Error : {e}') + + def list_allowed_instances_type_for_session(self) -> ListAllowedInstanceTypesForSessionResponse: + try: + response = self.context.get_virtual_desktop_controller_client(timeout=7200).invoke_alt( + namespace='VirtualDesktopUtils.ListAllowedInstanceTypesForSession', + payload=ListAllowedInstanceTypesForSessionRequest(session=self.session), + result_as=ListAllowedInstanceTypesForSessionResponse, + access_token=self.access_token) + return response + except (exceptions.SocaException, Exception) as e: + self.context.error(f'Failed to List Allowed Instances Type for Session. Session Name: {self.session.name} Error : {e}') + + def wait_and_verify_session_state_matches(self, expected_state: VirtualDesktopSessionState, get_session_info_namespace: str) -> Dict: + session_state = {'session_state_matches': bool, 'error_log': ''} + sleep_timer = 30 + try: + wait_counter = 0 + current_session = self.get_session_info(get_session_info_namespace) + + while current_session.state != expected_state and current_session.state != VirtualDesktopSessionState.ERROR: + self.context.info(f'Session Status: Session Name {current_session.name} is in {current_session.state} State') + time.sleep(sleep_timer) + current_session = self.get_session_info(get_session_info_namespace) + wait_counter += 1 + if wait_counter >= 40: + break + current_session = self.get_session_info(get_session_info_namespace) + + if current_session.state == expected_state: + session_state_log = f'Session Status: Session Name {current_session.name} is in {current_session.state} State' + self.context.info(session_state_log) + session_state.update({'session_state_matches': True, 'error_log': ''}) + return session_state + + session_state_log = f'Exceeded maximum wait time for State to change.Expected State {expected_state} and Current State is {current_session.state}' + self.context.error(session_state_log) + session_state.update({'session_state_matches': False, 'error_log': session_state_log}) + return session_state + + except Exception as e: + session_state_log = f'Failed to verify session state. Error : {e}' + self.context.error(session_state_log) + session_state.update({'session_state_matches': False, 'error_log': session_state_log}) + + +class VirtualDesktopApiHelper: + + def __init__(self, context: TestContext, access_token: str, username: str): + self.context = context + self.access_token = access_token + projects_list = _get_user_projects_list(context, username, access_token) + + if projects_list is not None: + for project in projects_list: + if 'default' in project.name: + self.project = project + + self.project_id = self.project.project_id + + def _get_virtual_desktop_permission_payload(self) -> List[VirtualDesktopPermission]: + try: + permission_payload = [VirtualDesktopPermission( + key='builtin', + name='Built In', + description='All features', + enabled=False)] + return permission_payload + + except exceptions.SocaException as e: + self.context.error(f'Failed to get Virtual Desktop Permission Payload. Error : {e}') + + def _get_permission_profile_payload(self) -> VirtualDesktopPermissionProfile: + test_profile = 'VDC_Test_Profile' + + try: + session_payload = VirtualDesktopPermissionProfile( + profile_id=test_profile, + title=test_profile, + description=test_profile, + permissions=self._get_virtual_desktop_permission_payload() + ) + return session_payload + + except exceptions.SocaException as e: + self.context.error(f'Failed to Get Permission Profile Payload. Error : {e}') + + def create_permissions_profile(self) -> CreatePermissionProfileResponse: + try: + response = self.context.get_virtual_desktop_controller_client(timeout=7200).invoke_alt( + namespace='VirtualDesktopAdmin.CreatePermissionProfile', + payload=CreatePermissionProfileRequest(profile=self._get_permission_profile_payload()), + result_as=CreatePermissionProfileResponse, + access_token=self.access_token) + return response + except (exceptions.SocaException, Exception) as e: + self.context.error(f'Failed to Create Permissions Profile. Error : {e}') + + def update_permission_profile(self, namespace: str) -> UpdatePermissionProfileResponse: + try: + response = self.context.get_virtual_desktop_controller_client(timeout=7200).invoke_alt( + namespace=namespace, + payload=UpdatePermissionProfileRequest(profile=self._get_permission_profile_payload()), + result_as=UpdatePermissionProfileResponse, + access_token=self.access_token) + return response + except (exceptions.SocaException, Exception) as e: + self.context.error(f'Failed to Update Permissions Profile. Error : {e}') + + def list_software_stacks(self, namespace: str) -> ListSoftwareStackResponse: + try: + response = self.context.get_virtual_desktop_controller_client(timeout=7200).invoke_alt( + namespace=namespace, + payload=ListSoftwareStackRequest( + disabled_also=True, + project_id=self.project_id + ), + result_as=ListSoftwareStackResponse, + access_token=self.access_token) + return response + except (exceptions.SocaException, Exception) as e: + self.context.error(f'Failed to List Software Stack. Error : {e}') + + def list_sessions(self, namespace: str) -> ListSessionsResponse: + try: + response = self.context.get_virtual_desktop_controller_client(timeout=7200).invoke_alt( + namespace=namespace, + payload=ListSessionsRequest(), + result_as=ListSessionsResponse, + access_token=self.access_token) + return response + except (exceptions.SocaException, Exception) as e: + self.context.error(f'Failed to List Sessions. Error : {e}') + + def reindex_user_session(self) -> ReIndexUserSessionsResponse: + try: + response = self.context.get_virtual_desktop_controller_client(timeout=7200).invoke_alt( + namespace='VirtualDesktopAdmin.ReIndexUserSessions', + payload=ReIndexUserSessionsRequest(), + result_as=ReIndexUserSessionsResponse, + access_token=self.access_token) + return response + except (exceptions.SocaException, Exception) as e: + self.context.error(f'Failed to Reindex User Session. Error : {e}') + + def reindex_software_stacks(self) -> ReIndexSoftwareStacksResponse: + try: + response = self.context.get_virtual_desktop_controller_client(timeout=7200).invoke_alt( + namespace='VirtualDesktopAdmin.ReIndexSoftwareStacks', + payload=ReIndexSoftwareStacksRequest(), + result_as=ReIndexSoftwareStacksResponse, + access_token=self.access_token) + return response + except (exceptions.SocaException, Exception) as e: + self.context.error(f'Failed to Reindex Software Stacks. Error : {e}') + + # VDC Utils + def list_supported_os(self) -> ListSupportedOSResponse: + try: + response = self.context.get_virtual_desktop_controller_client(timeout=7200).invoke_alt( + namespace='VirtualDesktopUtils.ListSupportedOS', + payload=ListSupportedOSRequest(), + result_as=ListSupportedOSResponse, + access_token=self.access_token) + return response + except (exceptions.SocaException, Exception) as e: + self.context.error(f'Failed to List Supported OS. Error : {e}') + + def list_supported_gpu(self) -> ListSupportedGPUResponse: + try: + response = self.context.get_virtual_desktop_controller_client(timeout=7200).invoke_alt( + namespace='VirtualDesktopUtils.ListSupportedGPU', + payload=ListSupportedGPURequest(), + result_as=ListSupportedGPUResponse, + access_token=self.access_token) + return response + except (exceptions.SocaException, Exception) as e: + self.context.error(f'Failed to List Supported GPU. Error : {e}') + + def list_schedule_types(self) -> ListScheduleTypesResponse: + try: + response = self.context.get_virtual_desktop_controller_client(timeout=7200).invoke_alt( + namespace='VirtualDesktopUtils.ListScheduleTypes', + payload=ListScheduleTypesRequest(), + result_as=ListScheduleTypesResponse, + access_token=self.access_token) + return response + except (exceptions.SocaException, Exception) as e: + self.context.error(f'Failed to List Schedule Types. Error : {e}') + + def list_allowed_instance_types(self) -> ListAllowedInstanceTypesResponse: + try: + response = self.context.get_virtual_desktop_controller_client(timeout=7200).invoke_alt( + namespace='VirtualDesktopUtils.ListAllowedInstanceTypes', + payload=ListAllowedInstanceTypesRequest(), + result_as=ListAllowedInstanceTypesResponse, + access_token=self.access_token) + return response + except (exceptions.SocaException, Exception) as e: + self.context.error(f'Failed to List Allowed Instances Types. Error : {e}') + + def list_permission_profiles(self) -> ListPermissionProfilesResponse: + try: + response = self.context.get_virtual_desktop_controller_client(timeout=7200).invoke_alt( + namespace='VirtualDesktopUtils.ListPermissionProfiles', + payload=ListPermissionProfilesRequest(), + result_as=ListPermissionProfilesResponse, + access_token=self.access_token + ) + return response + except (exceptions.SocaException, Exception) as e: + self.context.error(f'Failed to List Permission Profiles. Error : {e}') + + def get_permission_profile(self) -> GetPermissionProfileResponse: + try: + response = self.context.get_virtual_desktop_controller_client(timeout=7200).invoke_alt( + namespace='VirtualDesktopUtils.GetPermissionProfile', + payload=GetPermissionProfileRequest(profile_id='owner_profile'), + result_as=GetPermissionProfileResponse, + access_token=self.access_token) + return response + except (exceptions.SocaException, Exception) as e: + self.context.error(f'Failed to Get Permission Profile. Error : {e}') + + def get_base_permissions(self) -> GetBasePermissionsResponse: + try: + response = self.context.get_virtual_desktop_controller_client(timeout=7200).invoke_alt( + namespace='VirtualDesktopUtils.GetBasePermissions', + payload=GetBasePermissionsRequest(), + result_as=GetBasePermissionsResponse, + access_token=self.access_token) + return response + except (exceptions.SocaException, Exception) as e: + self.context.error(f'Failed to Get Base Permissions. Error : {e}') + + # Virtual Desktop DCV + def describe_servers(self) -> DescribeServersResponse: + try: + response = self.context.get_virtual_desktop_controller_client(timeout=7200).invoke_alt( + namespace='VirtualDesktopDCV.DescribeServers', + payload=DescribeServersRequest(), + result_as=DescribeServersResponse, + access_token=self.access_token) + return response + except (exceptions.SocaException, Exception) as e: + self.context.error(f'Failed to Describe Servers. Error : {e}') + + def describe_sessions(self) -> DescribeSessionsResponse: + try: + response = self.context.get_virtual_desktop_controller_client(timeout=7200).invoke_alt( + namespace='VirtualDesktopDCV.DescribeSessions', + payload=DescribeSessionsRequest(), + result_as=DescribeSessionsResponse, + access_token=self.access_token) + return response + except (exceptions.SocaException, Exception) as e: + self.context.error(f'Failed to Describe Sessions. Error : {e}') + + +class SessionWorkflow: + def __init__(self, context: TestContext, session: VirtualDesktopSession, test_case_id: str, username: str, access_token: str, get_session_info_namespace: str): + self.context = context + self.session = session + self.test_case_id = test_case_id + self.access_token = access_token + self.get_session_info_namespace = get_session_info_namespace + self.username = username + self.session_helper = SessionsTestHelper(self.context, self.session, self.username, self.access_token) + + def _test_stop_session(self, test_case_name: str, stop_session_namespace: str) -> [VirtualDesktopSessionTestResults, str]: + test_results_list = [] + + try: + self._on_session_test_start(VirtualDesktopSessionTestcases.STOP_SESSION) + session_status = self.session_helper.wait_and_verify_session_state_matches(VirtualDesktopSessionState.READY, self.get_session_info_namespace) + if session_status.get('session_state_matches'): + stop_session_response = self.session_helper.stop_sessions(stop_session_namespace) + if stop_session_response.success: + time.sleep(20) + self.session = self.session_helper.get_session_info(self.get_session_info_namespace) + session_status = self.session_helper.wait_and_verify_session_state_matches(VirtualDesktopSessionState.STOPPED, self.get_session_info_namespace) + if session_status.get('session_state_matches'): + return self._on_session_test_pass(test_case_name, test_results_list, VirtualDesktopSessionTestcases.STOP_SESSION) + else: + return self._on_session_response_failed(test_case_name, test_results_list, VirtualDesktopSessionTestcases.STOP_SESSION) + + return self._on_session_state_mismatch(test_case_name, test_results_list, session_status, VirtualDesktopSessionTestcases.STOP_SESSION) + + except (exceptions.SocaException, Exception) as error: + return self._on_session_in_exception(test_case_name, test_results_list, error, VirtualDesktopSessionTestcases.STOP_SESSION) + + def _test_resume_session(self, test_case_name: str, stop_session_namespace: str, resume_session_namespace: str) -> [VirtualDesktopSessionTestResults, str]: + test_results_list = [] + sleep_timer = 30 + try: + self._on_session_test_start(VirtualDesktopSessionTestcases.RESUME_SESSION) + self.session = self.session_helper.get_session_info(self.get_session_info_namespace) + + if self.session.state != VirtualDesktopSessionState.STOPPED: + self.session_helper.stop_sessions(stop_session_namespace) + + self.session = self.session_helper.get_session_info(self.get_session_info_namespace) + session_status = self.session_helper.wait_and_verify_session_state_matches(VirtualDesktopSessionState.STOPPED, self.get_session_info_namespace) + + if session_status.get('session_state_matches'): + self.context.info(self.session_helper.prefix_text + f'Initiating {VirtualDesktopSessionTestcases.RESUME_SESSION} test for {self.session.name}') + + resume_session_response = self.session_helper.resume_sessions(resume_session_namespace) + time.sleep(sleep_timer) + + if resume_session_response.success: + self.session = self.session_helper.get_session_info(self.get_session_info_namespace) + session_status = self.session_helper.wait_and_verify_session_state_matches(VirtualDesktopSessionState.READY, self.get_session_info_namespace) + + if session_status.get('session_state_matches'): + return self._on_session_test_pass(test_case_name, test_results_list, VirtualDesktopSessionTestcases.RESUME_SESSION) + + else: + return self._on_session_state_mismatch(test_case_name, test_results_list, session_status, VirtualDesktopSessionTestcases.RESUME_SESSION) + + return self._on_session_response_failed(test_case_name, test_results_list, VirtualDesktopSessionTestcases.RESUME_SESSION) + + except exceptions.SocaException as error: + return self._on_session_in_exception(test_case_name, test_results_list, error, VirtualDesktopSessionTestcases.RESUME_SESSION) + + def _test_reboot_session(self, test_case_name: str, reboot_session_namespace: str) -> [VirtualDesktopSessionTestResults, str]: + test_results_list = [] + sleep_timer = 10 + try: + self._on_session_test_start(VirtualDesktopSessionTestcases.REBOOT_SESSION) + session_status = self.session_helper.wait_and_verify_session_state_matches(VirtualDesktopSessionState.READY, self.get_session_info_namespace) + + if session_status.get('session_state_matches'): + reboot_session_response = self.session_helper.reboot_sessions(reboot_session_namespace) + if reboot_session_response.success: + time.sleep(sleep_timer) + + self.session = self.session_helper.get_session_info(self.get_session_info_namespace) + session_status = self.session_helper.wait_and_verify_session_state_matches(VirtualDesktopSessionState.READY, self.get_session_info_namespace) + + if session_status.get('session_state_matches'): + return self._on_session_test_pass(test_case_name, test_results_list, VirtualDesktopSessionTestcases.REBOOT_SESSION) + else: + return self._on_session_response_failed(test_case_name, test_results_list, VirtualDesktopSessionTestcases.REBOOT_SESSION) + + return self._on_session_state_mismatch(test_case_name, test_results_list, session_status, VirtualDesktopSessionTestcases.REBOOT_SESSION) + + except exceptions.SocaException as error: + return self._on_session_in_exception(test_case_name, test_results_list, error, VirtualDesktopSessionTestcases.REBOOT_SESSION) + + def _test_delete_session(self, test_case_name: str, delete_session_namespace: str) -> [VirtualDesktopSessionTestResults, str]: + test_results_list = [] + try: + self._on_session_test_start(VirtualDesktopSessionTestcases.DELETE_SESSION) + session_status = self.session_helper.wait_and_verify_session_state_matches(VirtualDesktopSessionState.READY, self.get_session_info_namespace) + + if session_status.get('session_state_matches'): + delete_session_response = self.session_helper.delete_session(delete_session_namespace) + if delete_session_response is not None: + return self._on_session_test_pass(test_case_name, test_results_list, VirtualDesktopSessionTestcases.DELETE_SESSION) + else: + return self._on_session_response_failed(test_case_name, test_results_list, VirtualDesktopSessionTestcases.DELETE_SESSION) + else: + return self._on_session_state_mismatch(test_case_name, test_results_list, session_status, VirtualDesktopSessionTestcases.DELETE_SESSION) + + except exceptions.SocaException as error: + return self._on_session_in_exception(test_case_name, test_results_list, error, VirtualDesktopSessionTestcases.DELETE_SESSION) + + def test_session_workflow(self, user_type: Optional = 'admin'): + stop_session_namespace = 'VirtualDesktopAdmin.StopSessions' + resume_session_namespace = 'VirtualDesktopAdmin.ResumeSessions' + reboot_session_namespace = 'VirtualDesktopAdmin.RebootSessions' + delete_session_namespace = 'VirtualDesktopAdmin.DeleteSessions' + if user_type == 'user': + stop_session_namespace = 'VirtualDesktop.StopSessions' + resume_session_namespace = 'VirtualDesktop.ResumeSessions' + reboot_session_namespace = 'VirtualDesktop.RebootSessions' + delete_session_namespace = 'VirtualDesktop.DeleteSessions' + + session_tests_result: List = [] + + test_case_results = {'test_case_name': '', 'test_case_status': '', 'test_case_error_message': ''} + + session_helper = SessionsTestHelper(self.context, self.session, self.username, self.access_token) + session_response = session_helper.get_session_info(self.get_session_info_namespace) + wait_counter = 0 + sleep_timer = 30 + + try: + while session_response.state not in VirtualDesktopSessionState.ERROR: + session_response = session_helper.get_session_info(self.get_session_info_namespace) + time.sleep(sleep_timer) + self.context.info(f'SESSION STATUS : SESSION NAME {session_response.name} is in {session_response.state} STATE') + self.context.info('-' * 80) + session_response = session_helper.get_session_info(self.get_session_info_namespace) + + if session_response.state == VirtualDesktopSessionState.READY: + # Test 1 : Stop Session + time.sleep(sleep_timer) + test_case_results['test_case_name'] = f'Test {VirtualDesktopSessionTestcases.STOP_SESSION} Session for {self.session.name}' + + stop_session_test_case_results = self._test_stop_session(test_case_results['test_case_name'], stop_session_namespace) + session_tests_result.append(stop_session_test_case_results) + + # Test 2 : Resume Session + time.sleep(sleep_timer) + test_case_results['test_case_name'] = f'Test {VirtualDesktopSessionTestcases.RESUME_SESSION} Session for {self.session.name}' + + resume_session_test_case_results = self._test_resume_session(test_case_results['test_case_name'], stop_session_namespace, resume_session_namespace) + session_tests_result.append(resume_session_test_case_results) + + # Test 3 : Reboot Session + time.sleep(sleep_timer) + test_case_results['test_case_name'] = f'Test {VirtualDesktopSessionTestcases.REBOOT_SESSION} Session for {self.session.name}' + + reboot_session_test_case_results = self._test_reboot_session(test_case_results['test_case_name'], reboot_session_namespace) + session_tests_result.append(reboot_session_test_case_results) + + # Test 4 : Delete Session + time.sleep(sleep_timer) + test_case_results['test_case_name'] = f'Test {VirtualDesktopSessionTestcases.DELETE_SESSION} Session for {self.session.name}' + + terminate_session_test_case_results = self._test_delete_session(test_case_results['test_case_name'], delete_session_namespace) + session_tests_result.append(terminate_session_test_case_results) + break + wait_counter += 1 + time.sleep(sleep_timer) + session_response = session_helper.get_session_info(self.get_session_info_namespace) + + if wait_counter >= 60: + testcase_error_message = f'TEST STATUS: Exceeded maximum wait time for State to change. Session Name {session_response.name} is in {session_response.state} State.Marking tests as Skip status' + self.context.error(testcase_error_message) + test_case_results['test_case_status'] = VirtualDesktopSessionTestResults.SKIP.value + test_case_results['test_case_error_message'] = testcase_error_message + session_tests_result.append(test_case_results) + break + session_response = session_helper.get_session_info(self.get_session_info_namespace) + + if session_response.state == VirtualDesktopSessionState.ERROR: + testcase_error_message = f'TEST STATUS: Failed to execute tests. Session Name {session_response.name} is in {session_response.state} State.Marking tests as Skip status' + self.context.error(testcase_error_message) + test_case_results['test_case_status'] = VirtualDesktopSessionTestResults.SKIP.value + test_case_results['test_case_error_message'] = testcase_error_message + session_tests_result.append(test_case_results) + + except (exceptions.SocaException, Exception) as e: + testcase_error_message = f'Failed to execute tests for {self.session.name}, Error:{e}' + self.context.error(testcase_error_message) + test_case_results['test_case_status'] = VirtualDesktopSessionTestResults.FAILED.value + test_case_results['test_case_error_message'] = testcase_error_message + session_tests_result.append(test_case_results) + + finally: + _update_test_results(self.test_case_id, session_tests_result) + _create_or_update_test_report_xml(self.context, self.test_case_id) + + def _on_session_test_start(self, test_type: VirtualDesktopSessionTestcases): + self.context.info(self.session_helper.prefix_text + f'Initiating {test_type} test for {self.session.name}') + + def _on_session_test_pass(self, test_case_name: str, test_results_list: List, test_type: VirtualDesktopSessionTestcases): + test_failure_reason = '' + self.session = self.session_helper.get_session_info(self.get_session_info_namespace) + self.context.info(self.session_helper.prefix_text + f'Completed {test_type} test for {self.session.name}') + test_results_list.extend([VirtualDesktopSessionTestResults.PASS, test_failure_reason]) + return self._get_session_test_result(test_case_name, test_results_list, test_type) + + def _on_session_response_failed(self, test_case_name: str, test_results_list: List, test_type: VirtualDesktopSessionTestcases): + test_failure_reason = f'{test_type} Response Status : Failed.' + self.context.error(test_failure_reason) + test_results_list.extend([VirtualDesktopSessionTestResults.FAILED, test_failure_reason]) + return self._get_session_test_result(test_case_name, test_results_list, test_type) + + def _on_session_state_mismatch(self, test_case_name: str, test_results_list: List, session_status: Dict, test_type: VirtualDesktopSessionTestcases): + self.session = self.session_helper.get_session_info(self.get_session_info_namespace) + test_failure_reason = self.session_helper.prefix_text + f'Failed to execute {test_type} test.Session Name : {self.session.name}, Session ID : {self.session.idea_session_id} is in invalid State : {self.session.state}. ' + session_status.get('error_log') + self.context.error(test_failure_reason) + test_results_list.extend([VirtualDesktopSessionTestResults.FAILED, test_failure_reason]) + return self._get_session_test_result(test_case_name, test_results_list, test_type) + + def _on_session_in_exception(self, test_case_name, test_results_list, error: exceptions.SocaException, test_type: VirtualDesktopSessionTestcases): + self.session = self.session_helper.get_session_info(self.get_session_info_namespace) + test_failure_reason = self.session_helper.prefix_text + f'Failed to execute Test {test_type}.Session Name : {self.session.name}, Session ID : {self.session.idea_session_id} - Error: {error}' + self.context.error(test_failure_reason) + test_results_list.extend([VirtualDesktopSessionTestResults.FAILED, test_failure_reason]) + return self._get_session_test_result(test_case_name, test_results_list, test_type) + + def _get_session_test_result(self, test_case_name: str, test_result_list: List, test_type: VirtualDesktopSessionTestcases) -> Dict: + time.sleep(30) + session_test_case_item = {'test_case_name': test_case_name, 'test_case_status': '', 'test_case_error_message': ''} + + if not test_result_list: + test_case_error_message = self.context.error(f'Test Status : Test Results list is empty for session name {self.session.name}, testcase {test_type}. Marking test as Skip.') + session_test_case_item['test_case_status'] = VirtualDesktopSessionTestResults.SKIP.value + session_test_case_item['test_cases_error_message'] = test_case_error_message + + else: + session_test_case_item['test_case_status'] = test_result_list[0] + session_test_case_item['test_cases_error_message'] = test_result_list[1] + + return session_test_case_item + + +class VirtualDesktopTestHelper: + + def __init__(self, context: TestContext): + self.context = context + + @staticmethod + def set_new_session(session: VirtualDesktopSession): + global __new_created_session__ + __new_created_session__ = session + + @staticmethod + def set_new_software_stack(software_stack: VirtualDesktopSoftwareStack): + global __new_software_stack__ + __new_software_stack__ = software_stack + + @staticmethod + def is_new_session_created() -> bool: + global __new_created_session__ + if __new_created_session__ is not None: + return True + return False + + @staticmethod + def get_new_session() -> VirtualDesktopSession: + global __new_created_session__ + return __new_created_session__ + + @staticmethod + def get_new_software_stack() -> VirtualDesktopSoftwareStack: + global __new_software_stack__ + return __new_software_stack__ + + def before_test(self, test_case_name): + self.context.info(f'Test Status : Starting {test_case_name}') + + def on_test_pass(self, test_case_name, test_results_map: SessionsTestResultMap): + self.context.info(f'{test_case_name} : PASS') + testcase_error_message = '' + test_results_map.update_test_result_map(VirtualDesktopSessionTestResults.PASS, testcase_error_message) + + def on_test_fail(self, test_case_name, response, test_results_map: SessionsTestResultMap): + self.context.error(f'{test_case_name} : FAILED') + testcase_error_message = f'Failed : {test_case_name}. Response data: {response}' + test_results_map.update_test_result_map(VirtualDesktopSessionTestResults.FAILED, testcase_error_message) + assert False + + def on_test_exception(self, test_case_name, error: exceptions.SocaException, test_results_map: SessionsTestResultMap): + self.context.error(f'{test_case_name} : FAILED') + test_case_error_message = f'Failed to Execute {test_case_name}.Error : {error}' + test_results_map.update_test_result_map(VirtualDesktopSessionTestResults.FAILED, test_case_error_message) + assert False + + def after_test(self, test_case_name, test_results_map: SessionsTestResultMap, test_case_id): + _update_test_results(test_case_id, test_results_map.get_test_results()) + _create_or_update_test_report_xml(self.context, test_case_id) + self.context.info(f'Test Status : Completed {test_case_name}') + + +def _read_session_test_case_config(context: TestContext): + input_yml_file_name = 'session_test_cases.yml' + try: + resources_dir = ideaadministrator.props.resources_dir + test_cases_file = os.path.join(resources_dir, 'integration_tests', input_yml_file_name) + with open(test_cases_file, 'r') as f: + return Utils.from_yaml(f.read()) + except (FileNotFoundError, Exception) as e: + context.error(f'Failed to read file {input_yml_file_name}. Error : {e}') + + +def _get_user_projects_list(context: TestContext, username: str, access_token: str) -> List[Project]: + try: + list_projects_result = context.get_cluster_manager_client().invoke_alt( + namespace='Projects.GetUserProjects', + payload=GetUserProjectsRequest(username=username), + result_as=GetUserProjectsResult, + access_token=access_token + ) + return list_projects_result.projects + except exceptions.SocaException as e: + context.error(f'Failed to Get User Projects List. Error : {e}') + + +def _update_test_results(test_case_id: str, test_cases_result: List[Dict]): + global __vdc_test_results__ + for testcase_element in test_cases_result: + if test_case_id not in testcase_element: + __vdc_test_results__[test_case_id] = test_cases_result + else: + __vdc_test_results__[test_case_id].extend(test_cases_result) + + +def _get_test_summary_for_testsuite(test_case_id: str) -> Dict: + """returns test run summary for each test suite""" + + global __vdc_test_results__ + test_results_summary = {'total_tests': '', 'total_tests_failed': '', 'total_tests_skipped': ''} + total_tests = len(__vdc_test_results__[test_case_id]) + test_results_summary['total_tests'] = str(total_tests) + total_tests_failed = 0 + total_tests_skipped = 0 + for test_info in __vdc_test_results__[test_case_id]: + test_case_status = test_info['test_case_status'] + if test_case_status == VirtualDesktopSessionTestResults.FAILED: + total_tests_failed += 1 + elif test_case_status == VirtualDesktopSessionTestResults.SKIP: + total_tests_skipped += 1 + test_results_summary['total_tests_failed'] = str(total_tests_failed) + test_results_summary['total_tests_skipped'] = str(total_tests_skipped) + return test_results_summary + + +def _create_or_update_test_report_xml(context: TestContext, test_case_id: str): + global __is_test_results_report_created__ + global __vdc_test_results__ + test_results_path = os.path.join(ideaadministrator.props.dev_mode_project_root_dir, 'integration-test-results', context.test_run_id) + if not os.path.exists(test_results_path): + os.makedirs(test_results_path) + try: + sessions_test_report = 'vdc_test_report.xml' + test_results_file = os.path.join(test_results_path, sessions_test_report) + + test_report_helper = TestReportHelper(context, test_case_id, test_results_file) + + # Create New Test report + if not __is_test_results_report_created__: + # 1. Create + test_suites = test_report_helper.create_test_suites_element() + + # 2. Create + test_suite = test_report_helper.create_test_suite_element(test_suites) + + # 3. Create in element + test_report_helper.create_test_cases_in_a_test_suite(__vdc_test_results__, test_suite) + test_results_tree = cElementTree.ElementTree(test_suites) + + # 4. Write to Test report + test_report_helper.write_to_xml_file(test_results_tree) + __is_test_results_report_created__ = True + + # Update Existing Test Report + else: + # 1. Parse test results file + test_results_tree = test_report_helper.parse_file_and_return_tree_element() + + # 2. Append to existing + test_suites = test_report_helper.append_test_suites_to_test_report_tree(test_results_tree) + + # 3. Create + test_suite = test_report_helper.create_test_suite_element(test_suites) + + # 4. Create in element + test_report_helper.create_test_cases_in_a_test_suite(__vdc_test_results__, test_suite) + + # 5. Write to Test report + test_report_helper.write_to_xml_file(test_results_tree) + + except Exception as e: + context.error(f'Failed to create Test Results XML file {e}') + + +def get_sessions_test_cases_list(context: TestContext, username: str, access_token: str) -> List[VirtualDesktopSession]: + try: + session_test_data = _read_session_test_case_config(context) + param_test_case_name = context.extra_params.get('session_test_cases') + param_base_os = context.extra_params.get('base_os') + sessions_test_cases_list = [] + + # Create testcases list based on input parameter as base_os + if param_base_os is not None: + context.info(f'Creating testcases list for test case name : {param_base_os}') + param_base_os_list = param_base_os.split(',') + for session_values in session_test_data.values(): + for session_data in session_values: + session_data_base_os = session_data.get('base_os') + if session_data_base_os in param_base_os_list: + new_session_payload = SessionPayload(context, session_data, username, access_token) + sessions_test_cases_list.extend(new_session_payload.get_session_payload()) + + # Create testcases list based on input parameter as testcase name + if param_test_case_name is not None: + context.info(f'Creating testcases list for test case name : {param_test_case_name}') + param_test_case_name_list = param_test_case_name.split(',') + for session_values in session_test_data.values(): + for session_data in session_values: + if session_data.get('name') in param_test_case_name_list: + new_session_payload = SessionPayload(context, session_data, username, access_token) + sessions_test_cases_list.extend(new_session_payload.get_session_payload()) + + # Create all the default testcases + else: + context.info(f'Creating default testcases list') + for session_values in session_test_data.values(): + for session_data in session_values: + new_session_payload = SessionPayload(context, session_data, username, access_token) + sessions_test_cases_list.extend(new_session_payload.get_session_payload()) + return sessions_test_cases_list + except exceptions.SocaException as e: + context.error(f'Failed to Get Sessions Testcases List.Error : {e}') + + +def create_batch_sessions(context: TestContext, sessions: List[VirtualDesktopSession]) -> List[VirtualDesktopSession]: + try: + created_sessions: List[VirtualDesktopSession] = [] + batch_create_session_response = context.get_virtual_desktop_controller_client(timeout=7200).invoke_alt( + namespace='VirtualDesktopAdmin.BatchCreateSessions', + payload=BatchCreateSessionRequest(sessions=sessions), + access_token=context.get_admin_access_token() + ) + context.info(f'Test Status : Submitted {len(sessions)} session request(s).') + + if not batch_create_session_response.success: + context.error(f'Failed to Create Batch Sessions. API response log :{batch_create_session_response}') + assert False + for session_response in batch_create_session_response.success: + created_sessions.append(VirtualDesktopSession(**session_response)) + return created_sessions + except exceptions.SocaException as e: + context.error(f'Failed to Create Batch Sessions. Error : {e}') + + +def create_session(context: TestContext, session: VirtualDesktopSession, access_token: str, namespace: str) -> VirtualDesktopSession: + try: + context.info(f'Create Session Status : Initiating Create Session for {session.name}') + session_response = context.get_virtual_desktop_controller_client(timeout=7200).invoke_alt( + namespace=namespace, + payload=CreateSessionRequest(session=session), + access_token=access_token) + session: VirtualDesktopSession = VirtualDesktopSession(**session_response.session) + context.info(f'Create Session Status : Completed Create Session for {session.name}') + return session + except (exceptions.SocaException, Exception) as e: + context.error(f'Failed to Create Session for {session.name}. Error : {e}') diff --git a/source/idea/idea-administrator/src/ideaadministrator_meta/__init__.py b/source/idea/idea-administrator/src/ideaadministrator_meta/__init__.py new file mode 100644 index 00000000..42f08e5b --- /dev/null +++ b/source/idea/idea-administrator/src/ideaadministrator_meta/__init__.py @@ -0,0 +1,15 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +# pkg config for soca-admin. no dependencies. + +__name__ = 'idea-administrator' +__version__ = '3.1.0' diff --git a/source/idea/idea-administrator/src/setup.py b/source/idea/idea-administrator/src/setup.py new file mode 100644 index 00000000..65c1bc5a --- /dev/null +++ b/source/idea/idea-administrator/src/setup.py @@ -0,0 +1,30 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +from setuptools import setup, find_packages +import ideaadministrator_meta + +setup( + name=ideaadministrator_meta.__name__, + version=ideaadministrator_meta.__version__, + description='Administrator App', + url='https://awslabs.github.io/scale-out-computing-on-aws/', + author='Amazon', + license='Apache License, Version 2.0', + packages=find_packages(), + package_dir={ + 'ideaadministrator': 'ideaadministrator' + }, + entry_points=''' + [console_scripts] + idea-admin=ideaadministrator.app_main:main_wrapper + ''' +) diff --git a/source/idea/idea-administrator/tests/conftest.py b/source/idea/idea-administrator/tests/conftest.py new file mode 100644 index 00000000..6d8d18a3 --- /dev/null +++ b/source/idea/idea-administrator/tests/conftest.py @@ -0,0 +1,10 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. diff --git a/source/idea/idea-administrator/tests/test_example.py b/source/idea/idea-administrator/tests/test_example.py new file mode 100644 index 00000000..f67589e8 --- /dev/null +++ b/source/idea/idea-administrator/tests/test_example.py @@ -0,0 +1,16 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +def test_example(): + """ + example test case + """ + assert 1 + 1 is 2 diff --git a/source/idea/idea-bootstrap/_templates/linux/aws_cli.jinja2 b/source/idea/idea-bootstrap/_templates/linux/aws_cli.jinja2 new file mode 100644 index 00000000..60236430 --- /dev/null +++ b/source/idea/idea-bootstrap/_templates/linux/aws_cli.jinja2 @@ -0,0 +1,11 @@ +# Begin: AWS CLI +{%- if context.base_os in ('centos7', 'rhel7') %} +which aws > /dev/null 2>&1 +if [[ "$?" != "0" ]]; then + log_info "# installing aws-cli" + yum install -y python3-pip + PIP=$(which pip3) + $PIP install awscli +fi +{%- endif %} +# End: AWS CLI diff --git a/source/idea/idea-bootstrap/_templates/linux/aws_ssm.jinja2 b/source/idea/idea-bootstrap/_templates/linux/aws_ssm.jinja2 new file mode 100644 index 00000000..abe45218 --- /dev/null +++ b/source/idea/idea-bootstrap/_templates/linux/aws_ssm.jinja2 @@ -0,0 +1,17 @@ +# Begin: AWS Systems Manager Agent +{%- if context.base_os in ('amazonlinux2', 'centos7', 'rhel7') %} +systemctl status amazon-ssm-agent +if [[ "$?" != "0" ]]; then + machine=$(uname -m) + if [[ $machine == "x86_64" ]]; then + yum install -y {{ context.config.get_string('global-settings.package_config.aws_ssm.x86_64', required=True) }} + elif [[ $machine == "aarch64" ]]; then + yum install -y {{ context.config.get_string('global-settings.package_config.aws_ssm.aarch64', required=True) }} + fi + systemctl enable amazon-ssm-agent || true + systemctl restart amazon-ssm-agent +fi +{%- endif %} +# End: AWS Systems Manager Agent + + diff --git a/source/idea/idea-bootstrap/_templates/linux/chronyd.jinja2 b/source/idea/idea-bootstrap/_templates/linux/chronyd.jinja2 new file mode 100644 index 00000000..f4d29c26 --- /dev/null +++ b/source/idea/idea-bootstrap/_templates/linux/chronyd.jinja2 @@ -0,0 +1,34 @@ +# Begin: Configure chronyd +{%- if context.base_os in ('amazonlinux2', 'centos7', 'rhel7') %} +yum remove -y ntp +mv /etc/chrony.conf /etc/chrony.conf.original +echo -e " +# use the local instance NTP service, if available +server 169.254.169.123 prefer iburst minpoll 4 maxpoll 4 + +# Use public servers from the pool.ntp.org project. +# Please consider joining the pool (http://www.pool.ntp.org/join.html). +# !!! [BEGIN] IDEA REQUIREMENT +# You will need to open UDP egress traffic on your security group if you want to enable public pool +#pool 2.amazon.pool.ntp.org iburst +# !!! [END] IDEA REQUIREMENT +# Record the rate at which the system clock gains/losses time. +driftfile /var/lib/chrony/drift + +# Allow the system clock to be stepped in the first three updates +# if its offset is larger than 1 second. +makestep 1.0 3 + +# Specify file containing keys for NTP authentication. +keyfile /etc/chrony.keys + +# Specify directory for log files. +logdir /var/log/chrony + +# save data between restarts for fast re-load +dumponexit +dumpdir /var/run/chrony +" > /etc/chrony.conf +systemctl enable chronyd +{%- endif %} +# End: Configure chronyd diff --git a/source/idea/idea-bootstrap/_templates/linux/cloudwatch_agent.jinja2 b/source/idea/idea-bootstrap/_templates/linux/cloudwatch_agent.jinja2 new file mode 100644 index 00000000..03ce22c7 --- /dev/null +++ b/source/idea/idea-bootstrap/_templates/linux/cloudwatch_agent.jinja2 @@ -0,0 +1,55 @@ +# Begin: Install CloudWatch Agent + +CLOUDWATCH_AGENT_BOOTSTRAP_DIR="/root/bootstrap/amazon-cloudwatch-agent" +mkdir -p ${CLOUDWATCH_AGENT_BOOTSTRAP_DIR} +function get_cloudwatch_agent_download_link() { + local DOWNLOAD_LINK="{{ context.config.get_string('global-settings.package_config.amazon_cloudwatch_agent.download_link', default='') }}" + if [[ ! -z "${DOWNLOAD_LINK}" ]]; then + echo -n "${DOWNLOAD_LINK}" + return 0 + fi + local DOWNLOAD_LINK_PATTERN="{{ context.config.get_string('global-settings.package_config.amazon_cloudwatch_agent.download_link_pattern', required=True) }}" + echo -n ${DOWNLOAD_LINK_PATTERN} > ${CLOUDWATCH_AGENT_BOOTSTRAP_DIR}/cloudwatch_download_link.txt + local BASE_OS="{{ context.base_os }}" + case $BASE_OS in + amazonlinux2) + sed -i 's/%os%/amazon_linux/g' ${CLOUDWATCH_AGENT_BOOTSTRAP_DIR}/cloudwatch_download_link.txt + sed -i 's/%ext%/rpm/g' ${CLOUDWATCH_AGENT_BOOTSTRAP_DIR}/cloudwatch_download_link.txt + ;; + rhel7) + sed -i 's/%os%/redhat/g' ${CLOUDWATCH_AGENT_BOOTSTRAP_DIR}/cloudwatch_download_link.txt + sed -i 's/%ext%/rpm/g' ${CLOUDWATCH_AGENT_BOOTSTRAP_DIR}/cloudwatch_download_link.txt + ;; + centos7) + sed -i 's/%os%/centos/g' ${CLOUDWATCH_AGENT_BOOTSTRAP_DIR}/cloudwatch_download_link.txt + sed -i 's/%ext%/rpm/g' ${CLOUDWATCH_AGENT_BOOTSTRAP_DIR}/cloudwatch_download_link.txt + ;; + esac + local MACHINE=$(uname -m) + case $MACHINE in + aarch64) + sed -i 's/%architecture%/arm64/g' ${CLOUDWATCH_AGENT_BOOTSTRAP_DIR}/cloudwatch_download_link.txt + ;; + x86_64) + sed -i 's/%architecture%/amd64/g' ${CLOUDWATCH_AGENT_BOOTSTRAP_DIR}/cloudwatch_download_link.txt + ;; + esac + cat ${CLOUDWATCH_AGENT_BOOTSTRAP_DIR}/cloudwatch_download_link.txt +} +CLOUDWATCH_AGENT_DOWNLOAD_LINK="$(get_cloudwatch_agent_download_link)" +CLOUDWATCH_AGENT_PACKAGE_NAME="$(basename ${CLOUDWATCH_AGENT_DOWNLOAD_LINK})" +pushd ${CLOUDWATCH_AGENT_BOOTSTRAP_DIR} +wget "${CLOUDWATCH_AGENT_DOWNLOAD_LINK}" +rpm -U ./${CLOUDWATCH_AGENT_PACKAGE_NAME} +popd + +{%- set cloudwatch_agent_config = context.get_cloudwatch_agent_config(additional_log_files=additional_log_files) %} +{%- if cloudwatch_agent_config %} +echo '{{ context.utils.to_json(cloudwatch_agent_config, indent=True) }}' > /opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json +/opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a fetch-config -m ec2 -s -c file:/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json +{%- else %} +log_warning "Install CloudWatch Agent: cloudwatch_agent_config not provided." +{%- endif %} +# End: Install CloudWatch Agent + + diff --git a/source/idea/idea-bootstrap/_templates/linux/create_idea_app_certs.jinja2 b/source/idea/idea-bootstrap/_templates/linux/create_idea_app_certs.jinja2 new file mode 100644 index 00000000..d42d6288 --- /dev/null +++ b/source/idea/idea-bootstrap/_templates/linux/create_idea_app_certs.jinja2 @@ -0,0 +1,16 @@ +# Begin: Generate IDEA App Certs +function generate_idea_app_certs () { + # Generate 10 years internal SSL certificate for IDEA App APIs + local CERTS_DIR="${IDEA_CLUSTER_HOME}/certs" + mkdir -p "${CERTS_DIR}" + local CERT_KEY="${CERTS_DIR}/idea.key" + local CERT_FILE="${CERTS_DIR}/idea.crt" + if [[ ! -f "${CERT_KEY}" ]]; then + openssl req -new -newkey rsa:4096 -days 3650 -nodes -x509 \ + -subj "/C=US/ST=California/L=Sunnyvale/CN=*.{{ context.config.get_string('cluster.route53.private_hosted_zone_name') }}" \ + -keyout "${CERT_KEY}" -out "${CERT_FILE}" + fi + chmod 600 ${CERTS_DIR} +} +generate_idea_app_certs +# End: Generate IDEA App Certs diff --git a/source/idea/idea-bootstrap/_templates/linux/dcv_server.jinja2 b/source/idea/idea-bootstrap/_templates/linux/dcv_server.jinja2 new file mode 100644 index 00000000..5fab37e4 --- /dev/null +++ b/source/idea/idea-bootstrap/_templates/linux/dcv_server.jinja2 @@ -0,0 +1,99 @@ +# Begin: DCV Server + +DCV_GPG_KEY_DCV_SERVER="{{ context.config.get_string('global-settings.package_config.dcv.gpg_key', required=True) }}" +DCV_SERVER_X86_64_URL="{{ context.config.get_string('global-settings.package_config.dcv.host.x86_64.linux.al2_rhel_centos7.url', required=True) }}" +DCV_SERVER_X86_64_TGZ="{{ context.config.get_string('global-settings.package_config.dcv.host.x86_64.linux.al2_rhel_centos7.tgz', required=True) }}" +DCV_SERVER_X86_64_VERSION="{{ context.config.get_string('global-settings.package_config.dcv.host.x86_64.linux.al2_rhel_centos7.version', required=True) }}" +DCV_SERVER_X86_64_SHA256_HASH="{{ context.config.get_string('global-settings.package_config.dcv.host.x86_64.linux.al2_rhel_centos7.sha256sum', required=True) }}" + +DCV_SERVER_AARCH64_URL="{{ context.config.get_string('global-settings.package_config.dcv.host.aarch64.linux.al2_rhel_centos7.url', required=True) }}" +DCV_SERVER_AARCH64_TGZ="{{ context.config.get_string('global-settings.package_config.dcv.host.aarch64.linux.al2_rhel_centos7.tgz', required=True) }}" +DCV_SERVER_AARCH64_VERSION="{{ context.config.get_string('global-settings.package_config.dcv.host.aarch64.linux.al2_rhel_centos7.version', required=True) }}" +DCV_SERVER_AARCH64_SHA256_HASH="{{ context.config.get_string('global-settings.package_config.dcv.host.aarch64.linux.al2_rhel_centos7.sha256sum', required=True) }}" + +log_info "Installing GPU drivers" +{% if context.is_gpu_instance_type() -%} + sudo rm -rf /etc/X11/XF86Config* + {%- include '_templates/linux/gpu_drivers.jinja2' %} +{% else -%} + log_info "GPU InstanceType not detected. Skipping GPU driver installation." +{% endif -%} + +if [[ -z "$(rpm -qa gnome-terminal)" ]]; then +{% if context.base_os == 'amazonlinux2' -%} + DCV_AMAZONLINUX_PKGS=({{ ' '.join(context.config.get_list('global-settings.package_config.linux_packages.dcv_amazonlinux', required=True)) }}) + yum install -y $(echo ${DCV_AMAZONLINUX_PKGS[*]}) +{% elif context.base_os == 'rhel7' -%} + # RHEL 7.x/8.x and CentOS 8.x + yum groups mark convert + yum groupinstall "Server with GUI" -y --skip-broken +{% elif context.base_os == 'centos7' -%} + # CentOS 7.x + yum groups mark convert + yum groupinstall "GNOME Desktop" -y --skip-broken +{% endif -%} +else + log_info "Found gnome-terminal pre-installed... skipping dcv prereq installation..." +fi + +rpm --import ${DCV_GPG_KEY_DCV_SERVER} +machine=$(uname -m) #x86_64 or aarch64 +DCV_SERVER_URL="" +DCV_SERVER_TGZ="" +DCV_SERVER_VERSION="" +DCV_SERVER_SHA256_HASH="" +if [[ $machine == "x86_64" ]]; then + # x86_64 + DCV_SERVER_URL=${DCV_SERVER_X86_64_URL} + DCV_SERVER_TGZ=${DCV_SERVER_X86_64_TGZ} + DCV_SERVER_VERSION=${DCV_SERVER_X86_64_VERSION} + DCV_SERVER_SHA256_HASH=${DCV_SERVER_X86_64_SHA256_HASH} +else + # aarch64 + DCV_SERVER_URL=${DCV_SERVER_AARCH64_URL} + DCV_SERVER_TGZ=${DCV_SERVER_AARCH64_TGZ} + DCV_SERVER_VERSION=${DCV_SERVER_AARCH64_VERSION} + DCV_SERVER_SHA256_HASH=${DCV_SERVER_AARCH64_SHA256_HASH} +fi + +if [[ -z "$(rpm -qa nice-dcv-server)" ]]; then + wget ${DCV_SERVER_URL} + if [[ $(sha256sum ${DCV_SERVER_TGZ} | awk '{print $1}') != ${DCV_SERVER_SHA256_HASH} ]]; then + echo -e "FATAL ERROR: Checksum for DCV Server failed. File may be compromised." > /etc/motd + exit 1 + fi + tar zxvf ${DCV_SERVER_TGZ} + + pushd nice-dcv-${DCV_SERVER_VERSION} + {% if context.base_os == 'amazonlinux2' -%} + rpm -ivh nice-xdcv-*.${machine}.rpm + rpm -ivh nice-dcv-server-*.${machine}.rpm + rpm -ivh nice-dcv-web-viewer-*.${machine}.rpm + {% elif context.base_os in ('rhel7', 'centos7') -%} + rpm -ivh nice-xdcv-*.${machine}.rpm --nodeps + rpm -ivh nice-dcv-server-*.${machine}.rpm --nodeps + rpm -ivh nice-dcv-web-viewer-*.${machine}.rpm --nodeps + {% endif -%} + + {% if context.is_gpu_instance_type() -%} + if [[ $machine == "x86_64" ]]; then + echo "Detected GPU instance, adding support for nice-dcv-gl" + rpm -ivh nice-dcv-gl*.x86_64.rpm + fi + {% endif -%} + popd + rm -rf nice-dcv-${DCV_SERVER_VERSION} + rm -rf ${DCV_SERVER_TGZ} +else + log_info "Found nice-dcv-server pre-installed... skipping installation..." +fi + +{% if context.base_os == 'amazonlinux2' %} + echo "Base os is {{ context.base_os }}. No need for firewall disabling" +{% else %} + # RHEL 7.x/8.x and CentOS 7.x/8.x + systemctl stop firewalld + systemctl disable firewalld +{% endif -%} + +# End: DCV Server diff --git a/source/idea/idea-bootstrap/_templates/linux/dcv_session_manager_agent.jinja2 b/source/idea/idea-bootstrap/_templates/linux/dcv_session_manager_agent.jinja2 new file mode 100644 index 00000000..15a5ed3f --- /dev/null +++ b/source/idea/idea-bootstrap/_templates/linux/dcv_session_manager_agent.jinja2 @@ -0,0 +1,42 @@ +# BEGIN: DCV Session Manager Agent + +DCV_GPG_KEY_DCV_AGENT="{{ context.config.get_string('global-settings.package_config.dcv.gpg_key', required=True) }}" +DCV_SESSION_MANAGER_AGENT_X86_64_URL="{{ context.config.get_string('global-settings.package_config.dcv.agent.x86_64.linux.al2_rhel_centos7.url', required=True) }}" +DCV_SESSION_MANAGER_AGENT_X86_64_VERSION="{{ context.config.get_string('global-settings.package_config.dcv.agent.x86_64.linux.al2_rhel_centos7.version', required=True) }}" +DCV_SESSION_MANAGER_AGENT_X86_64_SHA256_HASH="{{ context.config.get_string('global-settings.package_config.dcv.agent.x86_64.linux.al2_rhel_centos7.sha256sum', required=True) }}" +DCV_SESSION_MANAGER_AGENT_AARCH64_URL="{{ context.config.get_string('global-settings.package_config.dcv.agent.aarch64.linux.al2_rhel_centos7.url', required=True) }}" +DCV_SESSION_MANAGER_AGENT_AARCH64_VERSION="{{ context.config.get_string('global-settings.package_config.dcv.agent.aarch64.linux.al2_rhel_centos7.version', required=True) }}" +DCV_SESSION_MANAGER_AGENT_AARCH64_SHA256_HASH="{{ context.config.get_string('global-settings.package_config.dcv.agent.aarch64.linux.al2_rhel_centos7.sha256sum', required=True) }}" + +rpm --import ${DCV_GPG_KEY_DCV_AGENT} +machine=$(uname -m) #x86_64 or aarch64 +AGENT_URL="" +AGENT_VERSION="" +AGENT_SHA256_HASH="" +if [[ $machine == "x86_64" ]]; then + # x86_64 + AGENT_URL=${DCV_SESSION_MANAGER_AGENT_X86_64_URL} + AGENT_VERSION=${DCV_SESSION_MANAGER_AGENT_X86_64_VERSION} + AGENT_SHA256_HASH=${DCV_SESSION_MANAGER_AGENT_X86_64_SHA256_HASH} +else + # aarch64 + AGENT_URL=${DCV_SESSION_MANAGER_AGENT_AARCH64_URL} + AGENT_VERSION=${DCV_SESSION_MANAGER_AGENT_AARCH64_VERSION} + AGENT_SHA256_HASH=${DCV_SESSION_MANAGER_AGENT_AARCH64_SHA256_HASH} +fi + +if [[ -z "$(rpm -qa nice-dcv-session-manager-agent)" ]]; then + wget ${AGENT_URL} + if [[ $(sha256sum nice-dcv-session-manager-agent-${AGENT_VERSION}.rpm | awk '{print $1}') != ${AGENT_SHA256_HASH} ]]; then + echo -e "FATAL ERROR: Checksum for DCV Session Manager Agent failed. File may be compromised." > /etc/motd + exit 1 + fi + yum install -y nice-dcv-session-manager-agent-${AGENT_VERSION}.rpm +else + log_info "Found nice-dcv-session-manager-agent pre-installed... skipping installation..." +fi +log_info "# installing dcv agent complete ..." + +rm -rf nice-dcv-session-manager-agent-${AGENT_VERSION}.rpm + +# END: DCV Session Manager Agent diff --git a/source/idea/idea-bootstrap/_templates/linux/disable_motd_update.jinja2 b/source/idea/idea-bootstrap/_templates/linux/disable_motd_update.jinja2 new file mode 100644 index 00000000..9f1c61a3 --- /dev/null +++ b/source/idea/idea-bootstrap/_templates/linux/disable_motd_update.jinja2 @@ -0,0 +1,7 @@ +# Begin: Disable motd update - applicable only for amazonlinux2 +{%- if context.base_os == 'amazonlinux2' %} +/usr/sbin/update-motd --disable +rm /etc/cron.d/update-motd +rm -f /etc/update-motd.d/* +{%- endif %} +# End: Disable motd update diff --git a/source/idea/idea-bootstrap/_templates/linux/disable_nouveau_drivers.jinja2 b/source/idea/idea-bootstrap/_templates/linux/disable_nouveau_drivers.jinja2 new file mode 100644 index 00000000..4a95fd53 --- /dev/null +++ b/source/idea/idea-bootstrap/_templates/linux/disable_nouveau_drivers.jinja2 @@ -0,0 +1,23 @@ +{%- if context.is_nvidia_gpu() %} +# Begin: Disable NVIDIA Nouveau Drivers - Is GPU Instance Type: {{ context.is_gpu_instance_type() }}, Is NVIDIA GPU: {{ context.is_nvidia_gpu() }} +{%- if context.base_os in ('centos7', 'rhel7') %} +grep -q "rdblacklist=nouveau" /etc/default/grub +if [[ "$?" != "0" ]]; then + log_info "Disabling the nouveau open source driver for NVIDIA graphics cards" + cat << EOF | tee --append /etc/modprobe.d/blacklist.conf +blacklist vga16fb +blacklist nouveau +blacklist rivafb +blacklist nvidiafb +blacklist rivatv +EOF + echo GRUB_CMDLINE_LINUX="rdblacklist=nouveau" >> /etc/default/grub + grub2-mkconfig -o /boot/grub2/grub.cfg + + set_reboot_required "Disable NVIDIA Nouveau Drivers" +fi +{%- else %} +log_info "Not required for amazonlinux2" +{%- endif %} +# End: Disable NVIDIA Nouveau Drivers +{%- endif %} diff --git a/source/idea/idea-bootstrap/_templates/linux/disable_se_linux.jinja2 b/source/idea/idea-bootstrap/_templates/linux/disable_se_linux.jinja2 new file mode 100644 index 00000000..831f075e --- /dev/null +++ b/source/idea/idea-bootstrap/_templates/linux/disable_se_linux.jinja2 @@ -0,0 +1,12 @@ +{%- if context.base_os in ('amazonlinux2', 'centos7', 'rhel7') %} +# Begin: Disable SE Linux +sestatus | grep -q "disabled" +if [[ "$?" != "0" ]]; then + # disables selinux for current session + sestatus 0 + # reboot is required to apply this change permanently. ensure reboot is the last line called from userdata. + sed -i 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/selinux/config + set_reboot_required "Disable SE Linux" +fi +# End: Disable SE Linux +{%- endif %} diff --git a/source/idea/idea-bootstrap/_templates/linux/disable_strict_host_check.jinja2 b/source/idea/idea-bootstrap/_templates/linux/disable_strict_host_check.jinja2 new file mode 100644 index 00000000..4d7bb772 --- /dev/null +++ b/source/idea/idea-bootstrap/_templates/linux/disable_strict_host_check.jinja2 @@ -0,0 +1,6 @@ +# Begin: Disable Strict Host Check +{%- if context.base_os in ('amazonlinux2', 'centos7', 'rhel7') %} +echo "StrictHostKeyChecking no" >> /etc/ssh/ssh_config +echo "UserKnownHostsFile /dev/null" >> /etc/ssh/ssh_config +{%- endif %} +# End: Disable Strict Host Check diff --git a/source/idea/idea-bootstrap/_templates/linux/disable_ulimit.jinja2 b/source/idea/idea-bootstrap/_templates/linux/disable_ulimit.jinja2 new file mode 100644 index 00000000..277c871c --- /dev/null +++ b/source/idea/idea-bootstrap/_templates/linux/disable_ulimit.jinja2 @@ -0,0 +1,8 @@ +# Begin: Disable ulimit +{%- if context.base_os in ('amazonlinux2', 'centos7', 'rhel7') %} +echo -e " +* hard memlock unlimited +* soft memlock unlimited +" >> /etc/security/limits.conf +{%- endif %} +# End: Disable ulimit diff --git a/source/idea/idea-bootstrap/_templates/linux/efs_mount_helper.jinja2 b/source/idea/idea-bootstrap/_templates/linux/efs_mount_helper.jinja2 new file mode 100644 index 00000000..5fb80c6c --- /dev/null +++ b/source/idea/idea-bootstrap/_templates/linux/efs_mount_helper.jinja2 @@ -0,0 +1,27 @@ +# Begin: AWS EFS Mount Helper installation + +{%- if context.base_os in ('amazonlinux2', 'centos7', 'rhel7') %} + + {% include '_templates/linux/stunnel.jinja2' %} + +function install_efs_mount_helper () { + which amazon-efs-mount-watchdog > /dev/null 2>&1 + if [[ "$?" != "0" ]]; then + log_info "Installing Amazon EFS Mount Helper for {{ context.base_os }}" + {%- if context.base_os in ('amazonlinux2') %} + yum install -y amazon-efs-utils + {%- elif context.base_os in ('centos7','rhel7') %} + log_info "Installing Amazon EFS Mount Helper from Github" + git clone https://github.com/aws/efs-utils + cd efs-utils + make rpm + yum -y install build/amazon-efs-utils*rpm + {%- endif %} + else + log_info "Found existing Amazon EFS Mount Helper on system" + fi +} +install_efs_mount_helper +{%- endif %} + +# End: AWS EFS Mount Helper diff --git a/source/idea/idea-bootstrap/_templates/linux/epel_repo.jinja2 b/source/idea/idea-bootstrap/_templates/linux/epel_repo.jinja2 new file mode 100644 index 00000000..cad5c4b0 --- /dev/null +++ b/source/idea/idea-bootstrap/_templates/linux/epel_repo.jinja2 @@ -0,0 +1,20 @@ +# Begin: Install EPEL Repo +{%- if context.base_os == 'amazonlinux2' %} +if [[ ! -f "/etc/yum.repos.d/epel.repo" ]]; then + amazon-linux-extras install -y epel + yum update --security -y +fi +{%- endif %} +{%- if context.base_os == 'centos7' %} +if [[ ! -f "/etc/yum.repos.d/epel.repo" ]]; then + yum -y install epel-release +fi +{%- endif %} +{%- if context.base_os == 'rhel7' %} +if [[ ! -f "/etc/yum.repos.d/epel.repo" ]]; then + yum -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm +fi +{%- endif %} +# End: Install EPEL Repo + + diff --git a/source/idea/idea-bootstrap/_templates/linux/fsx_lustre_client.jinja2 b/source/idea/idea-bootstrap/_templates/linux/fsx_lustre_client.jinja2 new file mode 100644 index 00000000..9c23fda8 --- /dev/null +++ b/source/idea/idea-bootstrap/_templates/linux/fsx_lustre_client.jinja2 @@ -0,0 +1,56 @@ +# Begin: FSx Lustre Client +{%- if context.base_os == 'amazonlinux2' %} +if [[ -z "$(rpm -qa lustre-client)" ]]; then + amazon-linux-extras install -y lustre +fi +{%- elif context.base_os in ('centos7', 'rhel7') %} +if [[ -z "$(rpm -qa lustre-client)" ]]; then + kernel=$(uname -r) + machine=$(uname -m) + log_info "Found kernel version: $kernel running on: $machine" + if [[ $kernel == *"3.10.0-957"*$machine ]]; then + yum -y install https://downloads.whamcloud.com/public/lustre/lustre-2.12.9/el7/client/RPMS/x86_64/kmod-lustre-client-2.12.9-1.el7.x86_64.rpm + yum -y install https://downloads.whamcloud.com/public/lustre/lustre-2.12.9/el7/client/RPMS/x86_64/lustre-client-2.12.9-1.el7.x86_64.rpm + set_reboot_required "FSx for Lustre client installed" + elif [[ $kernel == *"3.10.0-1062"*$machine ]]; then + wget https://fsx-lustre-client-repo-public-keys.s3.amazonaws.com/fsx-rpm-public-key.asc -O /tmp/fsx-rpm-public-key.asc + rpm --import /tmp/fsx-rpm-public-key.asc + wget https://fsx-lustre-client-repo.s3.amazonaws.com/el/7/fsx-lustre-client.repo -O /etc/yum.repos.d/aws-fsx.repo + sed -i 's#7#7.7#' /etc/yum.repos.d/aws-fsx.repo + yum clean all + yum install -y kmod-lustre-client lustre-client + set_reboot_required "FSx for Lustre client installed" + elif [[ $kernel == *"3.10.0-1127"*$machine ]]; then + wget https://fsx-lustre-client-repo-public-keys.s3.amazonaws.com/fsx-rpm-public-key.asc -O /tmp/fsx-rpm-public-key.asc + rpm --import /tmp/fsx-rpm-public-key.asc + wget https://fsx-lustre-client-repo.s3.amazonaws.com/el/7/fsx-lustre-client.repo -O /etc/yum.repos.d/aws-fsx.repo + sed -i 's#7#7.8#' /etc/yum.repos.d/aws-fsx.repo + yum clean all + yum install -y kmod-lustre-client lustre-client + set_reboot_required "FSx for Lustre client installed" + elif [[ $kernel == *"3.10.0-1160"*$machine ]]; then + wget https://fsx-lustre-client-repo-public-keys.s3.amazonaws.com/fsx-rpm-public-key.asc -O /tmp/fsx-rpm-public-key.asc + rpm --import /tmp/fsx-rpm-public-key.asc + wget https://fsx-lustre-client-repo.s3.amazonaws.com/el/7/fsx-lustre-client.repo -O /etc/yum.repos.d/aws-fsx.repo + yum clean all + yum install -y kmod-lustre-client lustre-client + set_reboot_required "FSx for Lustre client installed" + elif [[ $kernel == *"4.18.0-193"*$machine ]]; then + # FSX for Lustre on aarch64 is supported only on 4.18.0-193 + wget https://fsx-lustre-client-repo-public-keys.s3.amazonaws.com/fsx-rpm-public-key.asc -O /tmp/fsx-rpm-public-key.asc + rpm --import /tmp/fsx-rpm-public-key.asc + wget https://fsx-lustre-client-repo.s3.amazonaws.com/centos/7/fsx-lustre-client.repo -O /etc/yum.repos.d/aws-fsx.repo + yum clean all + yum install -y kmod-lustre-client lustre-client + set_reboot_required "FSx for Lustre client installed" + else + log_error "Can't install FSx for Lustre client as kernel version: $kernel isn't matching expected versions: (x86_64: 3.10.0-957, -1062, -1127, -1160, aarch64: 4.18.0-193)!" + fi +fi +{%- endif %} + +# Performance tuning +{% include '_templates/linux/fsx_lustre_client_tuning_prereboot.jinja2' %} + + +# End: FSx Lustre Client diff --git a/source/idea/idea-bootstrap/_templates/linux/fsx_lustre_client_tuning_postmount.jinja2 b/source/idea/idea-bootstrap/_templates/linux/fsx_lustre_client_tuning_postmount.jinja2 new file mode 100644 index 00000000..251f1096 --- /dev/null +++ b/source/idea/idea-bootstrap/_templates/linux/fsx_lustre_client_tuning_postmount.jinja2 @@ -0,0 +1,20 @@ +# Begin: FSx Lustre Client Tuning - Post-Mount +# https://docs.aws.amazon.com/fsx/latest/LustreGuide/performance.html#performance-tips +{%- if context.base_os in ('amazonlinux2', 'centos7', 'rhel7') %} + NPROCS=$(nproc) + GB_MEM=$(free --si -g | egrep '^Mem:' | awk '{print $2}') + log_info "Detected ${NPROCS} CPUs / ${GB_MEM} GiB memory for Lustre performance tuning" + if [[ "${NPROCS}" -ge 64 ]]; then + log_info "Applying CPU count Lustre performance tuning" + lctl set_param osc.*OST*.max_rpcs_in_flight=32 + lctl set_param mdc.*.max_rpcs_in_flight=64 + lctl set_param mdc.*.max_mod_rpcs_in_flight=50 + fi + + if [[ "${GB_MEM}" -ge 64 ]]; then + log_info "Applying memory size Lustre performance tuning" + lctl set_param ldlm.namespaces.*.lru_max_age=600000 + fi + lctl lustre_build_version +{% endif %} +# End: FSx Lustre Client Tuning - Post-Mount diff --git a/source/idea/idea-bootstrap/_templates/linux/fsx_lustre_client_tuning_prereboot.jinja2 b/source/idea/idea-bootstrap/_templates/linux/fsx_lustre_client_tuning_prereboot.jinja2 new file mode 100644 index 00000000..4c5baed8 --- /dev/null +++ b/source/idea/idea-bootstrap/_templates/linux/fsx_lustre_client_tuning_prereboot.jinja2 @@ -0,0 +1,14 @@ +# Begin: FSx Lustre Client Tuning - Pre-Reboot +# https://docs.aws.amazon.com/fsx/latest/LustreGuide/performance.html#performance-tips +{%- if context.base_os in ('amazonlinux2', 'centos7', 'rhel7') %} + NPROCS=$(nproc) + GB_MEM=$(free --si -g | egrep '^Mem:' | awk '{print $2}') + log_info "Detected ${NPROCS} CPUs / ${GB_MEM} GiB memory for Lustre performance tuning" + if [[ "${NPROCS}" -ge 64 ]]; then + log_info "Applying CPU count Lustre performance tuning" + echo "options ptlrpc ptlrpcd_per_cpt_max=32" >> /etc/modprobe.d/modprobe.conf + echo "options ksocklnd credits=2560" >> /etc/modprobe.d/modprobe.conf + fi + set_reboot_required "Lustre client tuning applied" +{% endif %} +# End: FSx Lustre Client Tuning - Pre-Reboot diff --git a/source/idea/idea-bootstrap/_templates/linux/gpu_drivers.jinja2 b/source/idea/idea-bootstrap/_templates/linux/gpu_drivers.jinja2 new file mode 100644 index 00000000..ee5f6d62 --- /dev/null +++ b/source/idea/idea-bootstrap/_templates/linux/gpu_drivers.jinja2 @@ -0,0 +1,246 @@ +# Begin: Install GPU Drivers - Is GPU Instance Type: {{ context.is_gpu_instance_type() }} +{%- if context.is_gpu_instance_type() %} +{%- if context.is_nvidia_gpu() %} +function install_nvidia_grid_drivers () { + which nvidia-smi > /dev/null 2>&1 + if [[ "$?" == "0" ]]; then + log_info "GPU driver already installed. Skip." + return 0 + fi + + log_info "Installing NVIDIA GRID Drivers" + mkdir -p /root/bootstrap/gpu_drivers + pushd /root/bootstrap/gpu_drivers + + local AWS=$(command -v aws) + local DRIVER_BUCKET_REGION=$(curl -s --head {{ context.config.get_string('global-settings.gpu_settings.nvidia.s3_bucket_url', required=True) }} | grep bucket-region | awk '{print $2}' | tr -d '\r\n') + $AWS --region ${DRIVER_BUCKET_REGION} s3 cp --quiet --recursive {{ context.config.get_string('global-settings.gpu_settings.nvidia.s3_bucket_path', required=True) }} . + local x_server_pid=$(cat /tmp/.X0-lock) + if [[ ! -z "${x_server_pid}" ]]; then + kill $x_server_pid + fi + + /bin/sh NVIDIA-Linux-x86_64*.run --no-precompiled-interface --run-nvidia-xconfig --no-questions --accept-license --silent + log_info "X server configuration for GPU start..." + local NVIDIAXCONFIG=$(which nvidia-xconfig) + $NVIDIAXCONFIG --preserve-busid --enable-all-gpus + log_info "X server configuration for GPU end..." + set_reboot_required "Installed NVIDIA Grid Driver" + popd +} + +function install_nvidia_public_drivers() { + which nvidia-smi > /dev/null 2>&1 + if [[ "$?" == "0" ]]; then + log_info "GPU driver already installed. Skip." + return 0 + fi + + # Instance Product Type Product Series Product + # G2 GRID GRID Series GRID K520 + # G3/G3s Tesla M-Class M60 + # G4dn Tesla T-Series T4 + # G5 Tesla A-Series A10 - G5 (instances require driver version 470.00 or later) + # G5g Tesla T-Series NVIDIA T4G (G5g instances require driver version 470.82.01 or later. The operating systems is Linux aarch64) + # P2 Tesla K-Series K80 + # P3 Tesla V-Series V100 + # P4d Tesla A-Series A100 (320 GB HBM2 GPU memory) + # P4de Tesla A-Series A100 (640 GB HBM2e GPU memory) + + local DRIVER_VERSION="{{ context.get_nvidia_gpu_driver_version() }}" + + mkdir -p /root/bootstrap/gpu_drivers + pushd /root/bootstrap/gpu_drivers + + local MACHINE=$(uname -m) + curl -fSsl -O https://us.download.nvidia.com/tesla/${DRIVER_VERSION}/NVIDIA-Linux-${MACHINE}-${DRIVER_VERSION}.run + + local x_server_pid=$(cat /tmp/.X0-lock) + if [[ ! -z "${x_server_pid}" ]]; then + kill $x_server_pid + fi + + /bin/sh NVIDIA-Linux-${MACHINE}-${DRIVER_VERSION}.run -q -a -n -s + log_info "X server configuration for GPU start..." + local NVIDIAXCONFIG=$(which nvidia-xconfig) + $NVIDIAXCONFIG --preserve-busid --enable-all-gpus + log_info "X server configuration for GPU end..." + set_reboot_required "Installed NVIDIA Public Driver" + + popd +} +{%- elif context.is_amd_gpu() %} +function install_amd_gpu_drivers() { + which -s /opt/amdgpu-pro/bin/clinfo + if [[ "$?" == "0" ]]; then + log_info "GPU driver already installed. Skip." + return 0 + fi + # + # Instance GPU + # G4ad Radeon Pro V520 + # + mkdir -p /root/bootstrap/gpu_drivers + pushd /root/bootstrap/gpu_drivers + + local AWS=$(command -v aws) + local DRIVER_BUCKET_REGION=$(curl -s --head {{ context.config.get_string('global-settings.gpu_settings.amd.s3_bucket_url', required=True) }} | grep bucket-region | awk '{print $2}' | tr -d '\r\n') + $AWS --region ${DRIVER_BUCKET_REGION} s3 cp --quiet --recursive {{ context.config.get_string('global-settings.gpu_settings.amd.s3_bucket_path', required=True) }} . + tar -xf amdgpu-pro-*rhel*.tar.xz + cd amdgpu-pro* + rpm --import RPM-GPG-KEY-amdgpu + /bin/sh ./amdgpu-pro-install -y --opencl=pal,legacy + + set_reboot_required "Installed AMD GPU Driver" + + mkdir -p /etc/X11/ +echo """Section \"ServerLayout\" + Identifier \"Layout0\" + Screen 0 \"Screen0\" + InputDevice \"Keyboard0\" \"CoreKeyboard\" + InputDevice \"Mouse0\" \"CorePointer\" +EndSection +Section \"Files\" + ModulePath \"/opt/amdgpu/lib64/xorg/modules/drivers\" + ModulePath \"/opt/amdgpu/lib/xorg/modules\" + ModulePath \"/opt/amdgpu-pro/lib/xorg/modules/extensions\" + ModulePath \"/opt/amdgpu-pro/lib64/xorg/modules/extensions\" + ModulePath \"/usr/lib64/xorg/modules\" + ModulePath \"/usr/lib/xorg/modules\" +EndSection +Section \"InputDevice\" + # generated from default + Identifier \"Mouse0\" + Driver \"mouse\" + Option \"Protocol\" \"auto\" + Option \"Device\" \"/dev/psaux\" + Option \"Emulate3Buttons\" \"no\" + Option \"ZAxisMapping\" \"4 5\" +EndSection +Section \"InputDevice\" + # generated from default + Identifier \"Keyboard0\" + Driver \"kbd\" +EndSection +Section \"Monitor\" + Identifier \"Monitor0\" + VendorName \"Unknown\" + ModelName \"Unknown\" +EndSection +Section \"Device\" + Identifier \"Device0\" + Driver \"amdgpu\" + VendorName \"AMD\" + BoardName \"Radeon MxGPU V520\" + BusID \"PCI:0:30:0\" +EndSection +Section \"Extensions\" + Option \"DPMS\" \"Disable\" +EndSection +Section \"Screen\" + Identifier \"Screen0\" + Device \"Device0\" + Monitor \"Monitor0\" + DefaultDepth 24 + Option \"AllowEmptyInitialConfiguration\" \"True\" + SubSection \"Display\" + Virtual 3840 2160 + Depth 32 + EndSubSection +EndSection +"""> /etc/X11/xorg.conf + popd +} +{%- endif %} +function install_gpu_drivers () { + + # Identify Instance Type and Instance Family and install applicable GPU Drivers + # + # Types of Drivers: + # * Tesla drivers + # These drivers are intended primarily for compute workloads, which use GPUs for computational tasks such as parallelized floating-point + # calculations for machine learning and fast Fourier transforms for high performance computing applications. + # * GRID drivers + # These drivers are certified to provide optimal performance for professional visualization applications that render content such + # as 3D models or high-resolution videos. + + local NODE_TYPE="{{ node_type }}" + + local INSTANCE_TYPE=$(instance_type) + local INSTANCE_FAMILY=$(instance_family) + + local MACHINE=$(uname -m) + log_info "Detected GPU instance type: ${INSTANCE_TYPE}. Installing GPU Drivers ..." + + # refer to: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/install-nvidia-driver.html + # Available drivers by instance type for Tesla vs Grid mapping. + # we'll use NODE_TYPE="dcv" to install Grid drivers and NODE_TYPE="compute" to install Tesla drivers. + case ${INSTANCE_FAMILY} in + p4d|p4de) + log_info "Intel / NVIDIA A100" + # Tesla driver: Yes, GRID driver: No + install_nvidia_public_drivers + ;; + p3) + log_info "Intel / NVIDIA Tesla V100" + # Tesla driver: Yes, GRID driver: Yes (Using Marketplace AMIs only) + if [[ ${NODE_TYPE} == "dcv" ]]; then + install_nvidia_grid_drivers + else + install_nvidia_public_drivers + fi + ;; + p2) + log_info "Intel / NVIDIA K80" + # Tesla driver: Yes, GRID driver: No + install_nvidia_public_drivers + ;; + g5) + log_info "AMD / NVIDIA A10G" + # Tesla driver: Yes, GRID driver: Yes + if [[ ${NODE_TYPE} == "dcv" ]]; then + install_nvidia_grid_drivers + else + install_nvidia_public_drivers + fi + ;; + g5g) + log_info "Arm / NVIDIA T4G" + # Tesla driver: Yes, GRID driver: No + # This Tesla driver also supports optimized graphics applications specific to the ARM64 platform + install_nvidia_public_drivers + ;; + g4dn) + log_info "Intel / NVIDIA T4" + # Tesla driver: Yes, GRID driver: Yes + if [[ ${NODE_TYPE} == "dcv" ]]; then + install_nvidia_grid_drivers + else + install_nvidia_public_drivers + fi + ;; + g4ad) + log_info "AMD / AMD Radeon Pro V520" + install_amd_gpu_drivers + ;; + g3|g3s) + log_info "Intel / NVIDIA Tesla M60" + # Tesla driver: Yes, GRID driver: Yes + if [[ ${NODE_TYPE} == "dcv" ]]; then + install_nvidia_grid_drivers + else + install_nvidia_public_drivers + fi + ;; + g2) + log_info "Intel / NVIDIA Tesla M60" + # Tesla driver: Yes, GRID driver: No + install_nvidia_public_drivers + ;; + esac +} +install_gpu_drivers +{%- else %} +log_info "GPU InstanceType not detected. Skip." +{%- endif %} +# End: Install GPU Drivers diff --git a/source/idea/idea-bootstrap/_templates/linux/idea_service_account.jinja2 b/source/idea/idea-bootstrap/_templates/linux/idea_service_account.jinja2 new file mode 100644 index 00000000..9561945f --- /dev/null +++ b/source/idea/idea-bootstrap/_templates/linux/idea_service_account.jinja2 @@ -0,0 +1,8 @@ +# Begin: Create ideaserviceaccount +id ideaserviceaccount +if [[ "$?" != "0" ]]; then + useradd --system --shell /bin/false ideaserviceaccount +fi +# End: Create ideaserviceaccount + + diff --git a/source/idea/idea-bootstrap/_templates/linux/join_activedirectory.jinja2 b/source/idea/idea-bootstrap/_templates/linux/join_activedirectory.jinja2 new file mode 100644 index 00000000..92a26727 --- /dev/null +++ b/source/idea/idea-bootstrap/_templates/linux/join_activedirectory.jinja2 @@ -0,0 +1,182 @@ +# Begin: Join ActiveDirectory + +IDEA_CLUSTER_DATA_DIR="{{ context.config.get_string('shared-storage.data.mount_dir', required=True) }}" +AD_AUTHORIZATION_NONCE=${RANDOM} +AD_AUTHORIZATION_INSTANCE_ID=$(instance_id) +AD_AUTOMATION_SQS_QUEUE_URL="{{ context.config.get_string('directoryservice.ad_automation.sqs_queue_url', required=True) }}" +AD_AUTOMATION_DDB_TABLE_NAME="${IDEA_CLUSTER_NAME}.ad-automation" +AD_DOMAIN_NAME="{{ context.config.get_string('directoryservice.name', required=True) }}" +AD_REALM_NAME="{{ context.config.get_string('directoryservice.name', required=True).upper() }}" +AD_SUDOERS_GROUP_NAME="{{ context.config.get_string('directoryservice.sudoers.group_name', required=True) }}" +AD_SUDOERS_GROUP_NAME_ESCAPED="{{ context.config.get_string('directoryservice.sudoers.group_name', required=True).replace(' ', '\ ') }}" +SSSD_LDAP_ID_MAPPING="{{ context.config.get_bool('directoryservice.sssd.ldap_id_mapping', default=False) | lower }}" + +AWS=$(command -v aws) +JQ=$(command -v jq) +REALM=$(command -v realm) + + +function ad_automation_sqs_send_message () { + local PAYLOAD=$($JQ -nc \ + --arg instance_id "${AD_AUTHORIZATION_INSTANCE_ID}" \ + --arg nonce "${AD_AUTHORIZATION_NONCE}" \ + '{ + "header": { + "namespace": "ADAutomation.PresetComputer" + }, + "payload": { + "nonce": $nonce, + "instance_id": $instance_id + } + }') + + $AWS sqs send-message \ + --queue-url "${AD_AUTOMATION_SQS_QUEUE_URL}" \ + --message-body "${PAYLOAD}" \ + --message-group-id ${AD_AUTHORIZATION_INSTANCE_ID} \ + --message-deduplication-id "ADAutomation.PresetComputer.${AD_AUTHORIZATION_INSTANCE_ID}.${AD_AUTHORIZATION_NONCE}" \ + --region ${AWS_REGION} + + return $? +} + +function ad_automation_request_authorization () { + local ATTEMPT_COUNT=0 + local MAX_ATTEMPTS=5 + ad_automation_sqs_send_message + while [[ $? -ne 0 ]] && [[ ${ATTEMPT_COUNT} -le ${MAX_ATTEMPTS} ]] + do + local SLEEP_TIME=$(( RANDOM % 33 + 8 )) # Minimum of 8 seconds sleep + log_info "(${ATTEMPT_COUNT} of ${MAX_ATTEMPTS}) failed to request authorization to join AD, retrying in ${SLEEP_TIME} seconds ..." + sleep ${SLEEP_TIME} + ((ATTEMPT_COUNT++)) + ad_automation_sqs_send_message + done +} + +function ad_automation_get_authorization () { + if [[ ! -f "/root/.convert_from_dynamodb_object.jq" ]]; then + create_jq_ddb_filter + fi + local AUTOMATION_ENTRY_KEY=$($JQ -nc \ + --arg instance_id "${AD_AUTHORIZATION_INSTANCE_ID}" \ + --arg nonce "${AD_AUTHORIZATION_NONCE}" \ + '{"instance_id": {"S": $instance_id}, "nonce": {"S": $nonce }}') + + $AWS dynamodb get-item \ + --table-name ${AD_AUTOMATION_DDB_TABLE_NAME} \ + --key ${AUTOMATION_ENTRY_KEY} \ + --region "${AWS_REGION}" | jq -f /root/.convert_from_dynamodb_object.jq | jq '.Item' +} + +function ad_automation_wait_for_authorization_and_join () { + local ATTEMPT_COUNT=0 + local MAX_ATTEMPTS=180 # wait for no more than 30 minutes ( as max SLEEP_TIME: 0 <= SLEEP_TIME <= 10 ) + + local AD_AUTHORIZATION_ENTRY=$(ad_automation_get_authorization) + while [[ -z "${AD_AUTHORIZATION_ENTRY}" ]] && [[ ${ATTEMPT_COUNT} -le ${MAX_ATTEMPTS} ]] + do + local SLEEP_TIME=$(( RANDOM % 33 + 8 )) # 8-40sec of sleep time + log_info "(${ATTEMPT_COUNT} of ${MAX_ATTEMPTS}) waiting for AD authorization, retrying in ${SLEEP_TIME} seconds ..." + sleep ${SLEEP_TIME} + ((ATTEMPT_COUNT++)) + AD_AUTHORIZATION_ENTRY=$(ad_automation_get_authorization) + done + + local AUTHORIZATION_STATUS=$(echo "${AD_AUTHORIZATION_ENTRY}" | jq -r '.status') + if [[ "${AUTHORIZATION_STATUS}" == 'success' ]]; then + log_info "[Join AD] authorization successful. joining realm using OTP ..." + local ONE_TIME_PASSWORD=$(echo "${AD_AUTHORIZATION_ENTRY}" | jq -r '.otp') + # ensure we join to the domain controller where the computer account was created. + local DOMAIN_CONTROLLER=$(echo "${AD_AUTHORIZATION_ENTRY}" | jq -r '.domain_controller') + export IDEA_HOSTNAME=$(echo "${AD_AUTHORIZATION_ENTRY}" | jq -r '.hostname') + local LOCAL_HOSTNAME=$(hostname -s) + log_info "[Join AD] Using a local hostname of ${LOCAL_HOSTNAME^^} / IDEA Hostname: ${IDEA_HOSTNAME^^} for AD Join operation" + $REALM join \ + --one-time-password="${ONE_TIME_PASSWORD}" \ + --computer-name="${IDEA_HOSTNAME^^}" \ + --client-software=sssd \ + --server-software=active-directory \ + --membership-software=adcli \ + --verbose \ + ${DOMAIN_CONTROLLER} + else + local ERROR_CODE=$(echo "${AD_AUTHORIZATION_ENTRY}" | jq -r '.error_code') + local ERROR_MESSAGE=$(echo "${AD_AUTHORIZATION_ENTRY}" | jq -r '.message') + log_error "[Join AD] authorization failed: (${ERROR_CODE}) ${ERROR_MESSAGE}" + fi +} + +ad_automation_request_authorization +ad_automation_wait_for_authorization_and_join +# ad_automation_wait_for_authorization_and_join exports IDEA_HOSTNAME for our Kerberos info + +grep -q "## Add the \"${AD_SUDOERS_GROUP_NAME}\"" /etc/sudoers +if [[ "$?" != "0" ]]; then + echo -e " +## Add the \"${AD_SUDOERS_GROUP_NAME}\" group from the ${AD_DOMAIN_NAME} domain. +%${AD_SUDOERS_GROUP_NAME_ESCAPED} ALL=(ALL:ALL) ALL +">> /etc/sudoers +fi + +if [[ -f /etc/sssd/sssd.conf ]]; then + cp /etc/sssd/sssd.conf /etc/sssd/sssd.conf.orig +fi + +echo -e "[sssd] +domains = ${AD_DOMAIN_NAME} +config_file_version = 2 +services = nss, pam + +[domain/${AD_DOMAIN_NAME}] +ad_domain = ${AD_DOMAIN_NAME} +krb5_realm = ${AD_REALM_NAME} +realmd_tags = manages-system joined-with-adcli +cache_credentials = true +id_provider = ad +access_provider = ad +auth_provider = ad +chpass_provider = ad +krb5_store_password_if_offline = true +default_shell = /bin/bash + +# posix uidNumber and gidNumber will be ignored when ldap_id_mapping = true +ldap_id_mapping = ${SSSD_LDAP_ID_MAPPING} + +use_fully_qualified_names = false +fallback_homedir = ${IDEA_CLUSTER_DATA_DIR}/home/%u + +# disable or set to false for very large environments +enumerate = true + +sudo_provider = none + +# Use our AD-created IDEA hostname +ldap_sasl_authid = ${IDEA_HOSTNAME}\$ + +[nss] +homedir_substring = ${IDEA_CLUSTER_DATA_DIR}/home + +[pam] + +[autofs] + +[ssh] + +[secrets] +" > /etc/sssd/sssd.conf + +chmod 600 /etc/sssd/sssd.conf + +systemctl enable sssd +systemctl restart sssd + +# note: sss is removed for nsswitch to compared to openldap, to avoid mail spams. +grep -q "sudoers: files" /etc/nsswitch.conf +if [[ "$?" != "0" ]]; then + echo "sudoers: files" >> /etc/nsswitch.conf +fi + +# End: Join ActiveDirectory + + diff --git a/source/idea/idea-bootstrap/_templates/linux/join_directoryservice.jinja2 b/source/idea/idea-bootstrap/_templates/linux/join_directoryservice.jinja2 new file mode 100644 index 00000000..79cc39d6 --- /dev/null +++ b/source/idea/idea-bootstrap/_templates/linux/join_directoryservice.jinja2 @@ -0,0 +1,8 @@ +# Begin: Join Directory Service +{% if context.config.get_string('directoryservice.provider') == 'openldap' %} + {%- include '_templates/linux/join_openldap.jinja2' %} +{% endif -%} +{%- if context.config.get_string('directoryservice.provider') in ['activedirectory', 'aws_managed_activedirectory'] %} + {%- include '_templates/linux/join_activedirectory.jinja2' %} +{% endif -%} +# End: Join Directory Service diff --git a/source/idea/idea-bootstrap/_templates/linux/join_openldap.jinja2 b/source/idea/idea-bootstrap/_templates/linux/join_openldap.jinja2 new file mode 100644 index 00000000..5dd43c5f --- /dev/null +++ b/source/idea/idea-bootstrap/_templates/linux/join_openldap.jinja2 @@ -0,0 +1,108 @@ +# Begin: Join OpenLDAP + +IDEA_DS_TLS_CERTIFICATE_SECRET_ARN="{{context.config.get_string('directoryservice.tls_certificate_secret_arn', required=True)}}" +IDEA_CLUSTER_DATA_DIR="{{context.config.get_string('shared-storage.data.mount_dir', required=True)}}" +IDEA_DS_LDAP_BASE="{{context.config.get_string('directoryservice.ldap_base', required=True)}}" +IDEA_DS_LDAP_HOST="{{context.config.get_string('directoryservice.hostname', required=True)}}" + +if [[ -f "/etc/openldap/ldap.conf" ]]; then + cp /etc/openldap/ldap.conf /etc/openldap/ldap.conf.orig +fi +echo -e " +TLS_CACERTDIR /etc/openldap/cacerts/ + +# Turning this off breaks GSSAPI used with krb5 when rdns = false +SASL_NOCANON on + +URI ldap://${IDEA_DS_LDAP_HOST} + +BASE ${IDEA_DS_LDAP_BASE} +" > /etc/openldap/ldap.conf + +if [ -e /etc/sssd/sssd.conf ]; then + cp /etc/sssd/sssd.conf /etc/sssd/sssd.conf.orig +fi + +echo -e "[domain/default] +enumerate = True +autofs_provider = ldap +cache_credentials = True +ldap_search_base = ${IDEA_DS_LDAP_BASE} +id_provider = ldap +auth_provider = ldap +chpass_provider = ldap +sudo_provider = ldap +ldap_sudo_search_base = ou=Sudoers,${IDEA_DS_LDAP_BASE} +ldap_uri = ldap://${IDEA_DS_LDAP_HOST} +ldap_id_use_start_tls = True +use_fully_qualified_names = False +ldap_tls_cacertdir = /etc/openldap/cacerts +ldap_sudo_full_refresh_interval=86400 +ldap_sudo_smart_refresh_interval=3600 + +[sssd] +services = nss, pam, autofs, sudo +full_name_format = %2\$s\%1\$s +domains = default + +[nss] +homedir_substring = ${IDEA_CLUSTER_DATA_DIR}/home + +[pam] + +[sudo] + +[autofs] + +[ssh] + +[pac] + +[ifp] + +[secrets] +" > /etc/sssd/sssd.conf +chmod 600 /etc/sssd/sssd.conf + +mkdir -p /etc/openldap/cacerts/ +DS_TLS_CERTIFICATE=$(get_secret "${IDEA_DS_TLS_CERTIFICATE_SECRET_ARN}") +echo -n "${DS_TLS_CERTIFICATE}" > /etc/openldap/cacerts/openldap-server.pem + +authconfig --disablesssd \ + --disablesssdauth \ + --disableldap \ + --disableldapauth \ + --disablekrb5 \ + --disablekrb5kdcdns \ + --disablekrb5realmdns \ + --disablewinbind \ + --disablewinbindauth \ + --disablewinbindkrb5 \ + --disableldaptls \ + --disablerfc2307bis \ + --updateall + +sss_cache -E + +authconfig --enablesssd \ + --enablesssdauth \ + --enableldap \ + --enableldaptls \ + --enableldapauth \ + --ldapserver="ldap://${IDEA_DS_LDAP_HOST}" \ + --ldapbasedn="${IDEA_DS_LDAP_BASE}" \ + --enablelocauthorize \ + --enablemkhomedir \ + --enablecachecreds \ + --updateall + +systemctl enable sssd +systemctl restart sssd + +grep -q "sudoers: files sss" /etc/nsswitch.conf +if [[ "$?" != "0" ]]; then + echo "sudoers: files sss" >> /etc/nsswitch.conf +fi + +# End: Join OpenLDAP + diff --git a/source/idea/idea-bootstrap/_templates/linux/jq.jinja2 b/source/idea/idea-bootstrap/_templates/linux/jq.jinja2 new file mode 100644 index 00000000..d14f7d6d --- /dev/null +++ b/source/idea/idea-bootstrap/_templates/linux/jq.jinja2 @@ -0,0 +1,11 @@ +# Begin: Install jq +{%- if context.base_os in ('amazonlinux2', 'centos7', 'rhel7') %} +which jq > /dev/null 2>&1 +if [[ "$?" != "0" ]]; then + log_info "installing jq" + yum install -y jq +fi +{%- endif %} +# End: Install jq + + diff --git a/source/idea/idea-bootstrap/_templates/linux/motd.jinja2 b/source/idea/idea-bootstrap/_templates/linux/motd.jinja2 new file mode 100644 index 00000000..fcf37312 --- /dev/null +++ b/source/idea/idea-bootstrap/_templates/linux/motd.jinja2 @@ -0,0 +1,15 @@ +# Begin: motd update +{%- if context.base_os in ('amazonlinux2', 'centos7', 'rhel7') %} +echo -e " + ________ _________ + / _/ __ \/ ____/ | + / // / / / __/ / /| | + _/ // /_/ / /___/ ___ | + /___/_____/_____/_/ |_| +{% for message in messages %} +{{ message }} +{% endfor %} +> source /etc/environment to load IDEA paths +" > /etc/motd +{%- endif %} +# End: motd update diff --git a/source/idea/idea-bootstrap/_templates/linux/mount_shared_storage.jinja2 b/source/idea/idea-bootstrap/_templates/linux/mount_shared_storage.jinja2 new file mode 100644 index 00000000..f95aa38c --- /dev/null +++ b/source/idea/idea-bootstrap/_templates/linux/mount_shared_storage.jinja2 @@ -0,0 +1,62 @@ +# Begin: Mount Shared Storage +{%- if context.base_os in ('amazonlinux2', 'centos7', 'rhel7') %} + {%- if context.has_storage_provider('fsx_lustre') or context.has_storage_provider('fsx_cache') %} + {% include '_templates/linux/fsx_lustre_client.jinja2' %} + {%- endif %} + function mount_shared_storage () { + {%- for name, storage in context.config.get_config('shared-storage').items() %} + {%- if context.eval_shared_storage_scope(shared_storage=storage) %} + {%- if storage['provider'] == 'efs' %} + echo "# Using Provider {{storage['provider']}} for {{storage['mount_dir']}} using options {{storage['mount_options']}}" + mkdir -p "{{storage['mount_dir']}}" + add_efs_to_fstab "{{storage['efs']['dns']}}" \ + "{{storage['mount_dir']}}" \ + "{{storage['mount_options']}}" + {%- elif storage['provider'] == 'fsx_cache' %} + mkdir -p "{{storage['mount_dir']}}" + add_fsx_lustre_to_fstab "{{storage['fsx_cache']['dns']}}" \ + "{{storage['mount_dir']}}" \ + "{{storage['mount_options']}}" \ + "{{storage['fsx_cache']['mount_name']}}" + {%- elif storage['provider'] == 'fsx_lustre' %} + mkdir -p "{{storage['mount_dir']}}" + add_fsx_lustre_to_fstab "{{storage['fsx_lustre']['dns']}}" \ + "{{storage['mount_dir']}}" \ + "{{storage['mount_options']}}" \ + "{{storage['fsx_lustre']['mount_name']}}" + {%- elif storage['provider'] == 'fsx_netapp_ontap' and storage['fsx_netapp_ontap']['volume']['security_style'] %} + mkdir -p "{{storage['mount_dir']}}" + add_fsx_netapp_ontap_to_fstab "{{storage['fsx_netapp_ontap']['svm']['nfs_dns']}}" \ + "{{storage['mount_dir']}}" \ + "{{storage['mount_options']}}" \ + "{{storage['fsx_netapp_ontap']['volume']['volume_path']}}" + {%- elif storage['provider'] == 'fsx_openzfs' %} + mkdir -p "{{storage['mount_dir']}}" + add_fsx_openzfs_to_fstab "{{storage['fsx_openzfs']['dns']}}" \ + "{{storage['mount_dir']}}" \ + "{{storage['mount_options']}}" \ + "{{storage['fsx_openzfs']['volume_path']}}" + {%- endif %} + {%- endif %} + {%- endfor %} + + local AWS=$(command -v aws) + local FS_MOUNT_ATTEMPT=0 + local FS_MOUNT_MAX_ATTEMPTS=5 + mount -a + while [[ $? -ne 0 ]] && [[ ${FS_MOUNT_ATTEMPT} -le 5 ]] + do + local SLEEP_TIME=$(( RANDOM % 33 + 8 )) # Minimum of 8 seconds sleep + log_info "(${FS_MOUNT_ATTEMPT} of ${FS_MOUNT_MAX_ATTEMPTS}) Failed to mount FS, retrying in ${SLEEP_TIME} seconds ..." + sleep ${SLEEP_TIME} + ((FS_MOUNT_ATTEMPT++)) + mount -a + done + } + mount_shared_storage + {%- if context.has_storage_provider('fsx_lustre') or context.has_storage_provider('fsx_cache') %} + # Lustre client tuning for some adjustments takes place _after_ the client mounts have taken place + {% include '_templates/linux/fsx_lustre_client_tuning_postmount.jinja2' %} + {%- endif %} +{%- endif %} +# End: Mount Shared Storage diff --git a/source/idea/idea-bootstrap/_templates/linux/nfs_utils.jinja2 b/source/idea/idea-bootstrap/_templates/linux/nfs_utils.jinja2 new file mode 100644 index 00000000..76c6170a --- /dev/null +++ b/source/idea/idea-bootstrap/_templates/linux/nfs_utils.jinja2 @@ -0,0 +1,12 @@ +# Begin: NFS Utils and dependency items + +{%- if context.base_os in ('centos7', 'rhel7') %} +if [[ -z "$(rpm -qa nfs-utils)" ]]; then + log_info "# installing nfs-utils" + yum install -y nfs-utils +fi +{%- endif %} + +{% include '_templates/linux/efs_mount_helper.jinja2' %} + +# End: NFS Utils diff --git a/source/idea/idea-bootstrap/_templates/linux/nodejs.jinja2 b/source/idea/idea-bootstrap/_templates/linux/nodejs.jinja2 new file mode 100644 index 00000000..3a427807 --- /dev/null +++ b/source/idea/idea-bootstrap/_templates/linux/nodejs.jinja2 @@ -0,0 +1,28 @@ +# Begin: Install NodeJS +# Usage: Ensure NVM_DIR environment variable is added to /etc/environment when nodejs is installed. + +IDEA_NODE_VERSION="{{ context.config.get_string('global-settings.package_config.nodejs.version', required=True) }}" +IDEA_NVM_VERSION="{{ context.config.get_string('global-settings.package_config.nodejs.nvm_version', required=True) }}" +IDEA_NPM_VERSION="{{ context.config.get_string('global-settings.package_config.nodejs.npm_version', required=True) }}" + +function install_nodejs () { + local NVM_DIR=${NVM_DIR:-"/root/.nvm"} + mkdir -p ${NVM_DIR} + + if [[ ! -f "${NVM_DIR}/nvm.sh" ]]; then + curl -o- "{{ context.config.get_string('global-settings.package_config.nodejs.url', required=True) }}v${IDEA_NVM_VERSION}/install.sh" | bash + fi + source "${NVM_DIR}/nvm.sh" + nvm which --silent "${IDEA_NODE_VERSION}" > /dev/null 2>&1 + if [[ $? == 0 ]] ; then + log_info "node v${IDEA_NODE_VERSION} is installed." + nvm use "${IDEA_NODE_VERSION}" + return 0 + fi + + log_info "installing node v${IDEA_NODE_VERSION} ..." + nvm install "${IDEA_NODE_VERSION}" + nvm use "${IDEA_NODE_VERSION}" +} +install_nodejs +# End: Install NodeJS diff --git a/source/idea/idea-bootstrap/_templates/linux/openmpi.jinja2 b/source/idea/idea-bootstrap/_templates/linux/openmpi.jinja2 new file mode 100644 index 00000000..490330bf --- /dev/null +++ b/source/idea/idea-bootstrap/_templates/linux/openmpi.jinja2 @@ -0,0 +1,29 @@ +# Begin: Install OpenMPI + +OPENMPI_VERSION="{{ context.config.get_string('global-settings.package_config.openmpi.version') }}" +OPENMPI_URL="{{ context.config.get_string('global-settings.package_config.openmpi.url') }}" +OPENMPI_HASH="{{ context.config.get_string('global-settings.package_config.openmpi.checksum').lower().strip() }}" +OPENMPI_HASH_METHOD="{{ context.config.get_string('global-settings.package_config.openmpi.checksum_method').lower().strip() }}" +OPENMPI_TGZ="{{ context.config.get_string('global-settings.package_config.openmpi.url').split('/')[-1] }}" + +OPENMPI_WORK_DIR="/root/bootstrap/openmpi" +OPENMPI_INSTALL_DIR="{{ context.config.get_string('shared-storage.apps.mount_dir', required=True) }}/openmpi" +mkdir -p "${OPENMPI_WORK_DIR}" +mkdir -p "${OPENMPI_INSTALL_DIR}" + +if [[ ! -d "${OPENMPI_INSTALL_DIR}/openmpi-${OPENMPI_VERSION}" ]]; then + pushd "${OPENMPI_WORK_DIR}" + wget "${OPENMPI_URL}" + if [[ $(openssl ${OPENMPI_HASH_METHOD} "${OPENMPI_TGZ}" | awk '{print $2}') != "${OPENMPI_HASH}" ]]; then + echo -e "FATAL ERROR: ${OPENMPI_HASH_METHOD} Checksum for OpenMPI failed. File may be compromised." > /etc/motd + else + tar xvf "${OPENMPI_TGZ}" + cd openmpi-"${OPENMPI_VERSION}" + ./configure --prefix="${OPENMPI_INSTALL_DIR}/${OPENMPI_VERSION}" + NUM_PROCS=`nproc --all` + MAKE_FLAGS="-j${NUM_PROCS}" + make ${MAKE_FLAGS} + make install ${MAKE_FLAGS} + fi +fi +# End: Install OpenMPI diff --git a/source/idea/idea-bootstrap/_templates/linux/openpbs.jinja2 b/source/idea/idea-bootstrap/_templates/linux/openpbs.jinja2 new file mode 100644 index 00000000..fade880a --- /dev/null +++ b/source/idea/idea-bootstrap/_templates/linux/openpbs.jinja2 @@ -0,0 +1,90 @@ +# Begin: Install OpenPBS +{%- if context.base_os in ('amazonlinux2', 'centos7', 'rhel7') %} +{%- if context.config.get_string('global-settings.package_config.openpbs.type') == 'release' %} +function install_openpbs () { + local OPENPBS_VERSION="{{ context.config.get_string('global-settings.package_config.openpbs.version', required=True) }}" + local OPENPBS_URL="{{ context.config.get_string('global-settings.package_config.openpbs.url', required=True) }}" + local OPENPBS_HASH="{{ context.config.get_string('global-settings.package_config.openpbs.checksum', required=True).lower().strip() }}" + local OPENPBS_HASH_METHOD="{{ context.config.get_string('global-settings.package_config.openpbs.checksum_method', required=True).lower().strip() }}" + local OPENPBS_TGZ="$(basename ${OPENPBS_URL})" + local OPENPBS_WORK_DIR="/root/bootstrap/openpbs" + # check if already installed and is the correct version + local OPENPBS_INSTALLED_VERSION=$(/opt/pbs/bin/qstat --version | awk '{print $NF}') + if [[ "${OPENPBS_INSTALLED_VERSION}" == "${OPENPBS_VERSION}" ]]; then + log_info "OpenPBS is already installed. Version: ${OPENPBS_VERSION}" + return 0 + fi + + log_info "OpenPBS not detected or incorrect version, Installing OpenPBS version: ${OPENPBS_VERSION}..." + + OPENPBS_PKGS="{{ ' '.join(context.config.get_list('global-settings.package_config.openpbs.packages')) }}" + {%- if context.base_os == 'rhel7' %} + yum install -y ${OPENPBS_PKGS} --enablerepo rhel-7-server-rhui-optional-rpms + {%- endif %} + {%- if context.base_os in ('amazonlinux2', 'centos7') %} + yum install -y $(echo ${OPENPBS_PKGS[*]}) + {%- endif %} + mkdir -p "${OPENPBS_WORK_DIR}" + pushd ${OPENPBS_WORK_DIR} + # ADD A WAITER SO THAT IF CONNECTION IS THROTTLED WE CAN CONTINUE AFTER WAITING. + wget ${OPENPBS_URL} + if [[ $(openssl ${OPENPBS_HASH_METHOD} ${OPENPBS_TGZ} | awk '{print $2}') != ${OPENPBS_HASH} ]]; then + echo -e "FATAL ERROR: ${OPENPBS_HASH_METHOD} Checksum for OpenPBS failed. File may be compromised." > /etc/motd + exit 1 + fi + tar zxvf ${OPENPBS_TGZ} + local OPENPBS_WORK_DIR="${OPENPBS_WORK_DIR}/openpbs-${OPENPBS_VERSION}" + pushd ${OPENPBS_WORK_DIR} + ./autogen.sh + ./configure PBS_VERSION=${OPENPBS_VERSION} --prefix=/opt/pbs + local NUM_PROCS=`nproc --all` + local MAKE_FLAGS="-j${NUM_PROCS}" + make ${MAKE_FLAGS} + make install ${MAKE_FLAGS} + /opt/pbs/libexec/pbs_postinstall + chmod 4755 /opt/pbs/sbin/pbs_iff /opt/pbs/sbin/pbs_rcp + popd + popd +} +install_openpbs +{%- endif %} +{%- if context.config.get_string('global-settings.package_config.openpbs.type') == 'dev' %} +function install_openpbs () { + local OPENPBS_VERSION="{{ context.config.get_string('global-settings.package_config.openpbs.version', required=True) }}" + local OPENPBS_WORK_DIR="/root/bootstrap/openpbs" + # check if already installed and is the correct version + local OPENPBS_INSTALLED_VERSION=$(/opt/pbs/bin/qstat --version | awk '{print $NF}') + if [[ "${OPENPBS_INSTALLED_VERSION}" == "${OPENPBS_VERSION}" ]]; then + log_info "OpenPBS is already installed. Version: ${OPENPBS_VERSION}" + return 0 + fi + + log_info "OpenPBS Not Detected, Installing OpenPBS ..." + + OPENPBS_PKGS="{{ ' '.join(context.config.get_list('global-settings.package_config.openpbs.packages')) }}" + {%- if context.base_os == 'rhel7' %} + yum install -y ${OPENPBS_PKGS} --enablerepo rhel-7-server-rhui-optional-rpms + {%- endif %} + {%- if context.base_os in ('amazonlinux2', 'centos7') %} + yum install -y $(echo ${OPENPBS_PKGS[*]}) + {%- endif %} + mkdir -p "${OPENPBS_WORK_DIR}" + pushd ${OPENPBS_WORK_DIR} + git clone https://github.com/openpbs/openpbs.git + cd openpbs + sh ./autogen.sh + ./configure PBS_VERSION=${OPENPBS_VERSION} --prefix=/opt/pbs + ./autogen.sh + ./configure --prefix=/opt/pbs + local NUM_PROCS=`nproc --all` + local MAKE_FLAGS="-j${NUM_PROCS}" + make ${MAKE_FLAGS} + make install ${MAKE_FLAGS} + /opt/pbs/libexec/pbs_postinstall + chmod 4755 /opt/pbs/sbin/pbs_iff /opt/pbs/sbin/pbs_rcp + popd +} +install_openpbs +{%- endif %} +{%- endif %} +# End: Install OpenPBS diff --git a/source/idea/idea-bootstrap/_templates/linux/openpbs_client.jinja2 b/source/idea/idea-bootstrap/_templates/linux/openpbs_client.jinja2 new file mode 100644 index 00000000..30b51560 --- /dev/null +++ b/source/idea/idea-bootstrap/_templates/linux/openpbs_client.jinja2 @@ -0,0 +1,23 @@ +# Begin: OpenPBS Client +{% include '_templates/linux/openpbs.jinja2' %} + +echo -e "PBS_SERVER={{ context.config.get_string('scheduler.private_dns_name', required=True).split('.')[0] }} +PBS_START_SERVER=0 +PBS_START_SCHED=0 +PBS_START_COMM=0 +PBS_START_MOM=1 +PBS_EXEC=/opt/pbs +PBS_HOME=/var/spool/pbs +PBS_CORE_LIMIT=unlimited +PBS_SCP=/usr/bin/scp +" > /etc/pbs.conf + +echo -e " +\$clienthost {{ context.config.get_string('scheduler.private_dns_name').split('.')[0] }} +\$usecp *:/dev/null /dev/null +\$usecp *:{{ context.config.get_string('shared-storage.data.mount_dir') }} {{ context.config.get_string('shared-storage.data.mount_dir') }} +" > /var/spool/pbs/mom_priv/config + +systemctl start pbs +systemctl enable pbs +# End: OpenPBS Client diff --git a/source/idea/idea-bootstrap/_templates/linux/prometheus.jinja2 b/source/idea/idea-bootstrap/_templates/linux/prometheus.jinja2 new file mode 100644 index 00000000..3d80f6b9 --- /dev/null +++ b/source/idea/idea-bootstrap/_templates/linux/prometheus.jinja2 @@ -0,0 +1,58 @@ +# Begin: Install Prometheus +{%- if context.base_os in ('amazonlinux2', 'centos7', 'rhel7') %} +PROMETHEUS_AMD64_URL="{{ context.config.get_string('global-settings.package_config.prometheus.installer.linux.x86_64', required=True) }}" +PROMETHEUS_ARM64_URL="{{ context.config.get_string('global-settings.package_config.prometheus.installer.linux.aarch64', required=True) }}" +function install_prometheus () { + local MACHINE=$(uname -m) + local DOWNLOAD_URL="" + if [[ ${MACHINE} == "aarch64" ]]; then + DOWNLOAD_URL="${PROMETHEUS_ARM64_URL}" + else + DOWNLOAD_URL="${PROMETHEUS_AMD64_URL}" + fi + local PACKAGE_ARCHIVE=$(basename ${DOWNLOAD_URL}) + local PACKAGE_NAME="${PACKAGE_ARCHIVE%.tar.gz*}" + PROMETHEUS_DIR="/root/bootstrap/prometheus" + mkdir -p ${PROMETHEUS_DIR} + pushd ${PROMETHEUS_DIR} + wget ${DOWNLOAD_URL} + tar -xvf ${PACKAGE_ARCHIVE} + cp ${PACKAGE_NAME}/prometheus /usr/local/bin/ +} +function setup_prometheus_service () { + echo "[Unit] +Description=Prometheus +Wants=network-online.target +After=network-online.target + +[Service] +User=root +Group=root +Type=simple +ExecStart=/usr/local/bin/prometheus --config.file /etc/prometheus/prometheus.yml --storage.tsdb.path /opt/prometheus/data + +[Install] +WantedBy=multi-user.target +" > /etc/systemd/system/prometheus.service +} +install_prometheus +mkdir -p /opt/prometheus/data +{%- if context.is_prometheus_exporter_enabled('app_exporter') %} +if [[ ! -f /root/metrics_api_token.txt ]]; then + echo -n "${RANDOM}-${RANDOM}${RANDOM}${RANDOM}-${RANDOM}" > /root/metrics_api_token.txt +fi +{%- endif %} +{%- set prometheus_config = context.get_prometheus_config(additional_scrape_configs=additional_scrape_configs) %} +{%- if prometheus_config %} +mkdir -p /etc/prometheus +echo '{{ context.utils.to_yaml(prometheus_config) }}' > /etc/prometheus/prometheus.yml +setup_prometheus_service +systemctl enable prometheus +systemctl start prometheus +{%- else %} +log_warning "Install Prometheus: prometheus_config not provided." +{%- endif %} +{%- endif %} +# End: Install Prometheus + + diff --git a/source/idea/idea-bootstrap/_templates/linux/prometheus_node_exporter.jinja2 b/source/idea/idea-bootstrap/_templates/linux/prometheus_node_exporter.jinja2 new file mode 100644 index 00000000..9a0da895 --- /dev/null +++ b/source/idea/idea-bootstrap/_templates/linux/prometheus_node_exporter.jinja2 @@ -0,0 +1,46 @@ +# Begin: Install Prometheus Node Exporter +{%- if context.base_os in ('amazonlinux2', 'centos7', 'rhel7') and context.is_prometheus_exporter_enabled('node_exporter') %} +PROMETHEUS_NODE_EXPORTER_AMD64_URL="{{ context.config.get_string('global-settings.package_config.prometheus.exporters.node_exporter.linux.x86_64', required=True) }}" +PROMETHEUS_NODE_EXPORTER_ARM64_URL="{{ context.config.get_string('global-settings.package_config.prometheus.exporters.node_exporter.linux.aarch64', required=True) }}" +function install_prometheus_node_exporter () { + local MACHINE=$(uname -m) + local DOWNLOAD_URL="" + if [[ ${MACHINE} == "aarch64" ]]; then + DOWNLOAD_URL="${PROMETHEUS_NODE_EXPORTER_ARM64_URL}" + else + DOWNLOAD_URL="${PROMETHEUS_NODE_EXPORTER_AMD64_URL}" + fi + local PACKAGE_ARCHIVE=$(basename ${DOWNLOAD_URL}) + local PACKAGE_NAME="${PACKAGE_ARCHIVE%.tar.gz*}" + PROMETHEUS_DIR="/root/bootstrap/prometheus" + mkdir -p ${PROMETHEUS_DIR} + pushd ${PROMETHEUS_DIR} + wget ${DOWNLOAD_URL} + tar -xvf ${PACKAGE_ARCHIVE} + cp ${PACKAGE_NAME}/node_exporter /usr/local/bin/ +} +function setup_prometheus_node_exporter_service () { + echo "[Unit] +Description=Prometheus Node Exporter Service +After=network.target + +[Service] +User=root +Group=root +Type=simple +ExecStart=/usr/local/bin/node_exporter + +[Install] +WantedBy=multi-user.target +" > /etc/systemd/system/node-exporter.service +} +install_prometheus_node_exporter +setup_prometheus_node_exporter_service +systemctl enable node-exporter +systemctl start node-exporter +{%- else %} +log_info "Prometheus Node Exporter is disabled" +{%- endif %} +# End: Install Prometheus Node Exporter + + diff --git a/source/idea/idea-bootstrap/_templates/linux/python.jinja2 b/source/idea/idea-bootstrap/_templates/linux/python.jinja2 new file mode 100644 index 00000000..268d1a79 --- /dev/null +++ b/source/idea/idea-bootstrap/_templates/linux/python.jinja2 @@ -0,0 +1,59 @@ +# Begin: Install Python + +PYTHON_VERSION="{{ context.config.get_string('global-settings.package_config.python.version', required=True) }}" +PYTHON_URL="{{ context.config.get_string('global-settings.package_config.python.url', required=True) }}" +PYTHON_HASH="{{ context.config.get_string('global-settings.package_config.python.checksum', required=True).lower().strip() }}" +PYTHON_HASH_METHOD="{{ context.config.get_string('global-settings.package_config.python.checksum_method', required=True).lower().strip() }}" +PYTHON_TGZ="{{ context.config.get_string('global-settings.package_config.python.url', required=True).split('/')[-1] }}" + +function install_python () { + # - ALIAS_PREFIX: Will generate symlinks for python3 and pip3 for the alias: + # eg. if ALIAS_PREFIX == 'idea', idea_python and idea_pip will be available for use. + # - INSTALL_DIR: the location where python will be installed. + local ALIAS_PREFIX="{{ alias_prefix }}" + local INSTALL_DIR="{{ install_dir }}" + + local PYTHON3_BIN="${INSTALL_DIR}/latest/bin/python3" + local CURRENT_VERSION="$(${PYTHON3_BIN} --version | awk {'print $NF'})" + if [[ "${CURRENT_VERSION}" == "${PYTHON_VERSION}" ]]; then + log_info "Python already installed and at correct version." + else + + log_info "Python not detected, installing" + + local TIMESTAMP=$(date +%s) + local TMP_DIR="/root/bootstrap/python_installer/${ALIAS_PREFIX}-${TIMESTAMP}" + mkdir -p "${TMP_DIR}" + pushd ${TMP_DIR} + + wget ${PYTHON_URL} + if [[ $(openssl ${PYTHON_HASH_METHOD} ${PYTHON_TGZ} | awk '{print $2}') != ${PYTHON_HASH} ]]; then + echo -e "FATAL ERROR: ${PYTHON_HASH_METHOD} Checksum for Python failed. File may be compromised." > /etc/motd + exit 1 + fi + + tar xvf ${PYTHON_TGZ} + pushd "Python-${PYTHON_VERSION}" + local PYTHON_DIR="${INSTALL_DIR}/${PYTHON_VERSION}" + ./configure LDFLAGS="-L/usr/lib64/openssl" \ + CPPFLAGS="-I/usr/include/openssl" \ + -enable-loadable-sqlite-extensions \ + --prefix="${PYTHON_DIR}" + + local NUM_PROCS=`nproc --all` + local MAKE_FLAGS="-j${NUM_PROCS}" + make ${MAKE_FLAGS} + make ${MAKE_FLAGS} install + + popd + popd + + # create symlinks + local PYTHON_LATEST="${INSTALL_DIR}/latest" + ln -sf "${PYTHON_DIR}" "${PYTHON_LATEST}" + ln -sf "${PYTHON_LATEST}/bin/python3" "${PYTHON_LATEST}/bin/${ALIAS_PREFIX}_python" + ln -sf "${PYTHON_LATEST}/bin/pip3" "${PYTHON_LATEST}/bin/${ALIAS_PREFIX}_pip" + fi +} +install_python +# End Install Python diff --git a/source/idea/idea-bootstrap/_templates/linux/restrict_ssh.jinja2 b/source/idea/idea-bootstrap/_templates/linux/restrict_ssh.jinja2 new file mode 100644 index 00000000..f00470c9 --- /dev/null +++ b/source/idea/idea-bootstrap/_templates/linux/restrict_ssh.jinja2 @@ -0,0 +1,11 @@ +# Begin: Restrict SSH access +function restrict_ssh () { + local ADMIN_USERNAME="{{ context.config.get_string('cluster.administrator_username', required=True) }}" + grep -q "AllowGroups {{ context.default_system_user }} ssm-user ${ADMIN_USERNAME}-user-group" /etc/ssh/sshd_config + if [[ "$?" != "0" ]]; then + echo "AllowGroups {{ context.default_system_user }} ssm-user ${ADMIN_USERNAME}-user-group" >> /etc/ssh/sshd_config + fi + systemctl restart sshd +} +restrict_ssh +# End: Restrict SSH access diff --git a/source/idea/idea-bootstrap/_templates/linux/stunnel.jinja2 b/source/idea/idea-bootstrap/_templates/linux/stunnel.jinja2 new file mode 100644 index 00000000..97592056 --- /dev/null +++ b/source/idea/idea-bootstrap/_templates/linux/stunnel.jinja2 @@ -0,0 +1,33 @@ +# +# Begin: Install stunnel package. +# Stunnel is used to provide data in transit encryption via TLS for EFS/NFS traffic. +# https://docs.aws.amazon.com/efs/latest/ug/encryption-in-transit.html +# Mounts will appear connecting to localhost / '127.0.0.1' as they are tunneled in TLS. +# +{%- if context.base_os in ('amazonlinux2', 'centos7', 'rhel7') %} + function install_stunnel () { + {%- if context.base_os in ('amazonlinux2') %} + STUNNEL_CMD='stunnel5' + {%- elif context.base_os in ('centos7','rhel7') %} + STUNNEL_CMD='stunnel' + {%- endif %} + which ${STUNNEL_CMD} > /dev/null 2>&1 + if [[ "$?" != "0" ]]; then + log_info "Installing stunnel for {{ context.base_os }}" + yum install -y ${STUNNEL_CMD} + else + log_info "Found existing stunnel on system" + fi + # If needed - fixup configuration for older stunnel 4 packages that lack support for cert hostname enforcement + STUNNEL_VERSION=$(${STUNNEL_CMD} -version 2>&1 | egrep '^stunnel' | awk '{print $2}' | cut -f1 -d.) + log_info "Detected stunnel version ${STUNNEL_VERSION}" + if [[ "${STUNNEL_VERSION}" == '4' ]]; then + log_info "Stunnel 4 detected - Disabling stunnel_check_cert_hostname in /etc/amazon/efs/efs-utils.conf " + sed -i.backup 's/stunnel_check_cert_hostname = true/stunnel_check_cert_hostname = false/g' /etc/amazon/efs/efs-utils.conf + log_info "Configuration now:" + log_info "$(grep stunnel_check_cert_hostname /etc/amazon/efs/efs-utils.conf)" + fi +} +install_stunnel +{%- endif %} +# End: Install stunnel diff --git a/source/idea/idea-bootstrap/_templates/linux/sudoer_secure_path.jinja2 b/source/idea/idea-bootstrap/_templates/linux/sudoer_secure_path.jinja2 new file mode 100644 index 00000000..5cc6ee81 --- /dev/null +++ b/source/idea/idea-bootstrap/_templates/linux/sudoer_secure_path.jinja2 @@ -0,0 +1,7 @@ +# Begin: Update Sudoer Secure Path +# Enables sudo users to execute commands on this path. +{%- if context.base_os in ('amazonlinux2', 'centos7', 'rhel7') %} +echo 'Defaults secure_path="{{ secure_path }}"' > /etc/sudoers.d/secure_path +{%- endif %} +# End: Update Sudoer Secure Path + diff --git a/source/idea/idea-bootstrap/_templates/linux/supervisord.jinja2 b/source/idea/idea-bootstrap/_templates/linux/supervisord.jinja2 new file mode 100644 index 00000000..cdb1b80c --- /dev/null +++ b/source/idea/idea-bootstrap/_templates/linux/supervisord.jinja2 @@ -0,0 +1,52 @@ +# Begin: supervisord config +echo -e "; idea app supervisord config file + +[unix_http_server] +file=/run/supervisor.sock +chmod=0700 +chown=root:root + +[supervisord] +logfile=/var/log/supervisord.log +logfile_maxbytes=50MB +logfile_backups=10 +loglevel=info +pidfile=/run/supervisord.pid +nodaemon=false +minfds=1024 +minprocs=200 +user=root + +[rpcinterface:supervisor] +supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface + +[supervisorctl] +serverurl=unix:///run/supervisor.sock + +[include] +files = supervisord.d/*.ini +" > /etc/supervisord.conf + +mkdir -p /etc/supervisord.d + +echo "[Unit] +Description=supervisord - Supervisor process control system for UNIX +Documentation=http://supervisord.org +After=network.target + +[Service] +Type=forking +EnvironmentFile=/etc/environment +ExecStart=/opt/idea/python/latest/bin/supervisord -c /etc/supervisord.conf +ExecReload=/opt/idea/python/latest/bin/supervisorctl -c /etc/supervisord.conf reload +ExecStop=/opt/idea/python/latest/bin/supervisorctl -c /etc/supervisord.conf shutdown +User=root + +[Install] +WantedBy=multi-user.target +" > /etc/systemd/system/supervisord.service + +systemctl enable supervisord +systemctl restart supervisord + +# End: supervisord config diff --git a/source/idea/idea-bootstrap/_templates/linux/system_packages.jinja2 b/source/idea/idea-bootstrap/_templates/linux/system_packages.jinja2 new file mode 100644 index 00000000..dd4060bd --- /dev/null +++ b/source/idea/idea-bootstrap/_templates/linux/system_packages.jinja2 @@ -0,0 +1,15 @@ +# Begin: System Packages Install + +SYSTEM_PKGS=({{ ' '.join(context.config.get_list('global-settings.package_config.linux_packages.system', required=True)) }}) +APPLICATION_PKGS=({{ ' '.join(context.config.get_list('global-settings.package_config.linux_packages.application', required=True)) }}) +SSSD_PKGS=({{ ' '.join(context.config.get_list('global-settings.package_config.linux_packages.sssd', required=True)) }}) +OPENLDAP_CLIENT_PKGS=({{ ' '.join(context.config.get_list('global-settings.package_config.linux_packages.openldap_client', required=True)) }}) + +{%- if context.base_os == 'rhel7' %} + yum install -y $(echo ${SYSTEM_PKGS[*]} ${APPLICATION_PKGS[*]} ${SSSD_PKGS[*]} ${OPENLDAP_CLIENT_PKGS[*]}) --enablerepo rhel-7-server-rhui-optional-rpms +{%- elif context.base_os in ('amazonlinux2', 'centos7') %} + yum install -y $(echo ${SYSTEM_PKGS[*]} ${APPLICATION_PKGS[*]} ${SSSD_PKGS[*]} ${OPENLDAP_CLIENT_PKGS[*]}) +{%- endif %} + +# End: System Packages Install + diff --git a/source/idea/idea-bootstrap/_templates/linux/tag_ebs_volumes.jinja2 b/source/idea/idea-bootstrap/_templates/linux/tag_ebs_volumes.jinja2 new file mode 100644 index 00000000..45239541 --- /dev/null +++ b/source/idea/idea-bootstrap/_templates/linux/tag_ebs_volumes.jinja2 @@ -0,0 +1,34 @@ +# Begin: Tag EBS Volumes +function tag_ebs_volumes () { + local TAGS='{{ context.utils.to_json(ebs_volume_tags + context.get_custom_aws_tags()) }}' + local AWS=$(command -v aws) + local AWS_INSTANCE_ID=$(instance_id) + local VOLUMES=$($AWS ec2 describe-volumes \ + --filters "Name=attachment.instance-id,Values=${AWS_INSTANCE_ID}" \ + --region "{{ context.aws_region }}" \ + --query "Volumes[*].[VolumeId]" \ + --out text) + local EBS_IDS=$(echo "${VOLUMES}" | tr "\n" " ") + $AWS ec2 create-tags \ + --resources "${EBS_IDS}" \ + --region "{{ context.aws_region }}" \ + --tags "${TAGS}" + + local MAX_RETRIES=5 + local RETRY_COUNT=0 + while [[ $? -ne 0 ]] && [[ ${RETRY_COUNT} -lt ${MAX_RETRIES} ]] + do + local SLEEP_TIME=$(( RANDOM % 33 + 8 )) # Minimum of 8 seconds sleeping + log_info "(${RETRY_COUNT}/${MAX_RETRIES}) ec2 tag failed due to EC2 API error, retrying in ${SLEEP_TIME} seconds ..." + sleep ${SLEEP_TIME} + ((RETRY_COUNT++)) + $AWS ec2 create-tags \ + --resources "${EBS_IDS}" \ + --region "{{ context.aws_region }}" \ + --tags "${TAGS}" + done +} +tag_ebs_volumes +# End: Tag EBS Volumes + + diff --git a/source/idea/idea-bootstrap/_templates/linux/tag_network_interface.jinja2 b/source/idea/idea-bootstrap/_templates/linux/tag_network_interface.jinja2 new file mode 100644 index 00000000..25378b81 --- /dev/null +++ b/source/idea/idea-bootstrap/_templates/linux/tag_network_interface.jinja2 @@ -0,0 +1,31 @@ +# Begin: Tag Network Interface +function tag_network_interface () { + local TAGS='{{ context.utils.to_json(network_interface_tags + context.get_custom_aws_tags()) }}' + local AWS=$(command -v aws) + local AWS_INSTANCE_ID=$(instance_id) + local INTERFACES=$($AWS ec2 describe-network-interfaces \ + --filters "Name=attachment.instance-id,Values=${AWS_INSTANCE_ID}" \ + --region "{{ context.aws_region }}" \ + --query "NetworkInterfaces[*].[NetworkInterfaceId]" \ + --out text) + local ENI_IDS=$(echo "${INTERFACES}" | tr "\n" " ") + $AWS ec2 create-tags --resources "${ENI_IDS}" \ + --region "{{ context.aws_region }}" \ + --tags "${TAGS}" + + local MAX_RETRIES=5 + local RETRY_COUNT=0 + while [[ $? -ne 0 ]] && [[ ${RETRY_COUNT} -lt ${MAX_RETRIES} ]] + do + local SLEEP_TIME=$(( RANDOM % 33 + 8 )) # Sleep for 8-40 seconds + log_info "(${RETRY_COUNT}/${MAX_RETRIES}) ec2 tag failed due to EC2 API error, retrying in ${SLEEP_TIME} seconds ..." + sleep ${SLEEP_TIME} + ((RETRY_COUNT++)) + $AWS ec2 create-tags --resources "${ENI_IDS}" \ + --region "{{ context.aws_region }}" \ + --tags "${TAGS}" + done +} +tag_network_interface +# End: Tag Network Interface + diff --git a/source/idea/idea-bootstrap/_templates/windows/join_activedirectory.jinja2 b/source/idea/idea-bootstrap/_templates/windows/join_activedirectory.jinja2 new file mode 100644 index 00000000..303045a0 --- /dev/null +++ b/source/idea/idea-bootstrap/_templates/windows/join_activedirectory.jinja2 @@ -0,0 +1,140 @@ +# Begin: Join ActiveDirectory (PowerShell/Windows) + +# Add the AmazonDynamoDB .NET framework class. +Add-Type -Path (${env:ProgramFiles(x86)} + "\AWS SDK for .NET\bin\Net45\AWSSDK.DynamoDBv2.dll") + +$AWS_REGION = "{{ context.config.get_string('cluster.aws.region', required=True) }}" +$AD_AUTHORIZATION_NONCE = $( Get-Random -Maximum 32767 ) +$AD_AUTHORIZATION_INSTANCE_ID = $( Get-EC2InstanceMetadata -Category InstanceId ) +$AD_AUTOMATION_SQS_QUEUE_URL = "{{ context.config.get_string('directoryservice.ad_automation.sqs_queue_url', required=True) }}" +$AD_AUTOMATION_DDB_TABLE_NAME = "{{ context.config.get_string('cluster.cluster_name', required=True) }}.ad-automation" +$AD_DOMAIN_NAME = "{{ context.config.get_string('directoryservice.name', required=True) }}" +$WINDOWS_HOSTNAME = $env:COMPUTERNAME + +function Send-ADAutomationSQSMessage +{ + <# + .SYNOPSIS + Send the authorization payload to IDEA AD Automation SQS Queue + #> + [OutputType([Boolean])] + [CmdletBinding()] + + $payload = @{ header = @{ namespace = 'ADAutomation.PresetComputer' }; payload = @{ nonce = $AD_AUTHORIZATION_NONCE; instance_id = $AD_AUTHORIZATION_INSTANCE_ID; hostname = $WINDOWS_HOSTNAME } } | ConvertTo-Json -Compress + # todo - support vpc endpoint + Send-SQSMessage -QueueUrl $AD_AUTOMATION_SQS_QUEUE_URL ` + -MessageBody $payload ` + -MessageGroupId $AD_AUTHORIZATION_INSTANCE_ID ` + -MessageDeduplicationId "ADAutomation.PresetComputer.$AD_AUTHORIZATION_INSTANCE_ID.$AD_AUTHORIZATION_NONCE" ` + -Region $AWS_REGION + $? +} + +function Start-ADAutomationAuthorization +{ + <# + .SYNOPSIS + Invoke Send-ADAutomationSQSMessage until message is posted successfully. + #> + [CmdletBinding()] + $attemptCount = 0 + $maxAttempts = 5 + $success = Send-ADAutomationSQSMessage + while (($success -ne $true) -and ($attemptCount -le $maxAttempts)) + { + $sleepTime = $( Get-Random -Maximum 10 ) + Write-Host "($attemptCount of $maxAttempts) failed to request authorization to join AD, retrying in $sleepTime seconds ..." + Start-Sleep -Seconds $sleepTime + $attemptCount++ + $success = Send-ADAutomationSQSMessage + } +} + +function Get-ADAutomationAuthorizationEntry +{ + <# + .SYNOPSIS + Retrieve the AD Automation Authorization entry from IDEA AD Automation DynamoDB Table + #> + [OutputType([Hashtable])] + [OutputType([System.Void])] + [CmdletBinding()] + Param( + [Parameter(Mandatory = $true)] + [Amazon.DynamoDBv2.AmazonDynamoDBClient] $DDBClient + ) + + $request = New-Object Amazon.DynamoDBv2.Model.GetItemRequest + $request.TableName = $AD_AUTOMATION_DDB_TABLE_NAME + + # key - hash: instance_id, range: nonce + $request.Key = New-Object 'system.collections.generic.dictionary[string,Amazon.DynamoDBv2.Model.AttributeValue]' + $hashKey = New-Object Amazon.DynamoDBv2.Model.AttributeValue + $hashKey.S = $AD_AUTHORIZATION_INSTANCE_ID + $request.Key.Add("instance_id", $hashKey) + $rangeKey = New-Object Amazon.DynamoDBv2.Model.AttributeValue + $rangeKey.S = $AD_AUTHORIZATION_NONCE + $request.Key.Add("nonce", $rangeKey) + + $response = $DDBClient.GetItem($request) + + if ($response.IsItemSet -eq $false) + { + return + } + + if ($response.Item.status.S -eq "success") + { + @{ status = $($response.Item.status.S); domain_controller = $($response.Item.domain_controller.S); otp = $($response.Item.otp.S) } + } + else + { + @{ status = $($response.Item.status.S); error_code = $($response.Item.error_code.S); message = $($response.Item.message.S) } + } +} + +function Start-ADAutomationWaitForAuthorizationAndJoin +{ + <# + .SYNOPSIS + Wait for cluster manager to create a computer account. Keep polling IDEA AD Automation DDB table for authorization entry. + If status = 'success', Join AD using the Domain Controller and One-Time Password available in response. + If status = 'fail', log error code and message + #> + [CmdletBinding()] + + # Initialize DynamoDB Client + $regionEndpoint = [Amazon.RegionEndPoint]::GetBySystemName($AWS_REGION) + $ddbClient = New-Object Amazon.DynamoDBv2.AmazonDynamoDBClient($regionEndpoint) + + $attemptCount = 0 + $maxAttempts = 180 # wait for no more than 30 minutes ( as max $sleepTime: 0 <= $sleepTime <= 10 ) + $authorizationEntry = Get-ADAutomationAuthorizationEntry -DDBClient $ddbClient + while (($authorizationEntry -eq $null) -and ($attemptCount -le $maxAttempts)) + { + $sleepTime = $( Get-Random -Maximum 10 ) + Write-Host "($attemptCount of $maxAttempts) waiting for AD authorization, retrying in $sleepTime seconds ..." + Start-Sleep -Seconds $sleepTime + $attemptCount++ + $authorizationEntry = Get-ADAutomationAuthorizationEntry -DDBClient $ddbClient + } + + if ($authorizationEntry.status -eq "success") + { + Write-Host "[Join AD] authorization successful. joining AD, Domain: $AD_DOMAIN_NAME, OTP: $($authorizationEntry.otp) ..." + $joinCred = New-Object pscredential -ArgumentList ([pscustomobject]@{ + UserName = $null + Password = (ConvertTo-SecureString -String $($authorizationEntry.otp) -AsPlainText -Force)[0] + }) + Add-Computer -Domain $AD_DOMAIN_NAME ` + -Options UnsecuredJoin, PasswordPass ` + -Credential $joinCred + } + else + { + Write-Host "[Join AD] authorization failed: ($($authorizationEntry.error_code)) $($authorizationEntry.message)" + } + +} + +# End: Join ActiveDirectory (PowerShell/Windows) diff --git a/source/idea/idea-bootstrap/_templates/windows/mount_shared_storage.jinja2 b/source/idea/idea-bootstrap/_templates/windows/mount_shared_storage.jinja2 new file mode 100644 index 00000000..458ae6e9 --- /dev/null +++ b/source/idea/idea-bootstrap/_templates/windows/mount_shared_storage.jinja2 @@ -0,0 +1,46 @@ +# Begin: Mount Shared Storage +function Mount-SharedStorage +{ + <# + .SYNOPSIS + Mount applicable Windows File Shares as Network Drives + #> + [CmdletBinding()] + Param( + [Parameter(Mandatory = $true)] + [string] $DomainUserName + ) + + $shares = [System.Collections.ArrayList]::new() + {%- for name, storage in context.config.get_config('shared-storage').items() %} + {%- if context.eval_shared_storage_scope(shared_storage=storage) %} + {%- if storage['provider'] == 'fsx_netapp_ontap' and 'mount_drive' in storage and 'cifs_share_name' in storage['fsx_netapp_ontap']['volume'] %} + $shares.Add(@{MountDrive='{{ storage['mount_drive'] }}:'; Path='\\{{ storage['fsx_netapp_ontap']['svm']['smb_dns'] }}\{{ storage['fsx_netapp_ontap']['volume']['cifs_share_name']}}'}) | Out-Null + {%- endif %} + {%- if storage['provider'] == 'fsx_windows_file_server' and 'mount_drive' in storage %} + $shares.Add(@{MountDrive='{{ storage['mount_drive'] }}:'; Path='\\{{ storage['fsx_windows_file_server']['dns'] }}\share'}) | Out-Null + {%- endif %} + {%- endif %} + {%- endfor %} + + # create batch file and scheduled task only if any shared storage mounts are applicable + if ($shares.Count -gt 0) + { + + $batchFileCommands = [System.Collections.ArrayList]::new() + for($i=0; $i -lt $shares.Count; $i++) { + $batchFileCommands.Add("if not exist $($shares[$i].MountDrive) (net use $($shares[$i].MountDrive) $($shares[$i].Path) /persistent:yes)") | Out-Null + } + $batchFileContent = $batchFileCommands -join "`n" + $batchFile = "C:\IDEA\LocalScripts\MountSharedStorage.bat" + New-Item $batchFile -ItemType File -Value $batchFileContent -Force + + # create a scheduled task to execute after the domain user logs in + $action = New-ScheduledTaskAction -Execute $batchFile + $trigger = New-ScheduledTaskTrigger -AtLogOn -User $DomainUserName + Register-ScheduledTask -Action $action -Trigger $trigger -TaskName "MountSharedStorage" -Description "Mount Shared Storage" -User $DomainUserName + + } + +} +# End: Mount Shared Storage diff --git a/source/idea/idea-bootstrap/bastion-host/setup.sh.jinja2 b/source/idea/idea-bootstrap/bastion-host/setup.sh.jinja2 new file mode 100644 index 00000000..bf5d3aa7 --- /dev/null +++ b/source/idea/idea-bootstrap/bastion-host/setup.sh.jinja2 @@ -0,0 +1,116 @@ +#!/bin/bash + +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +set -x + +echo -e " +## [BEGIN] IDEA Environment Configuration - Do Not Delete +AWS_DEFAULT_REGION={{ context.aws_region }} +AWS_REGION={{ context.aws_region }} +IDEA_BASE_OS={{ context.base_os }} +IDEA_MODULE_NAME={{ context.module_name }} +IDEA_MODULE_ID={{ context.module_id }} +IDEA_MODULE_SET={{ context.module_set }} +IDEA_MODULE_VERSION={{ context.module_version }} +IDEA_CLUSTER_S3_BUCKET={{ context.cluster_s3_bucket }} +IDEA_CLUSTER_NAME={{ context.cluster_name }} +IDEA_CLUSTER_HOME={{ context.cluster_home_dir }} +IDEA_APP_DEPLOY_DIR={{ context.app_deploy_dir }} +BOOTSTRAP_DIR=/root/bootstrap +## [END] IDEA Environment Configuration + +PATH=/bin:/usr/bin:/sbin:/usr/sbin:/usr/local/bin:/opt/idea/python/latest/bin +" > /etc/environment + +source /etc/environment + +echo -n "no" > ${BOOTSTRAP_DIR}/reboot_required.txt + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +source "${SCRIPT_DIR}/../common/bootstrap_common.sh" + +{% include '_templates/linux/idea_service_account.jinja2' %} + +{% include '_templates/linux/aws_cli.jinja2' %} + +{% include '_templates/linux/aws_ssm.jinja2' %} + +{% include '_templates/linux/epel_repo.jinja2' %} + +{% include '_templates/linux/system_packages.jinja2' %} + +{%- include '_templates/linux/cloudwatch_agent.jinja2' %} + +{%- if context.is_metrics_provider_prometheus() %} + {%- include '_templates/linux/prometheus.jinja2' %} + {%- include '_templates/linux/prometheus_node_exporter.jinja2' %} +{%- endif %} + +{% include '_templates/linux/nfs_utils.jinja2' %} + +{% include '_templates/linux/jq.jinja2' %} + +{%- with ebs_volume_tags = [ + {'Key':'idea:ClusterName', 'Value': context.cluster_name }, + {'Key':'idea:ModuleName', 'Value': context.module_name }, + {'Key':'idea:ModuleId', 'Value': context.module_id }, + {'Key':'Name', 'Value': context.cluster_name + '/' + context.module_id + ' Root Volume' } +] %} + {% include '_templates/linux/tag_ebs_volumes.jinja2' %} +{%- endwith %} + +{%- with network_interface_tags = [ + {'Key':'idea:ClusterName', 'Value': context.cluster_name }, + {'Key':'idea:ModuleName', 'Value': context.module_name }, + {'Key':'idea:ModuleId', 'Value': context.module_id }, + {'Key':'Name', 'Value': context.cluster_name + '/' + context.module_id + ' Network Interface' } +] %} + {% include '_templates/linux/tag_network_interface.jinja2' %} +{%- endwith %} + +{% include '_templates/linux/disable_se_linux.jinja2' %} + +{%- with alias_prefix = 'idea', install_dir = '/opt/idea/python' %} + {% include '_templates/linux/python.jinja2' %} +{%- endwith %} + +{% include '_templates/linux/chronyd.jinja2' %} + +{% include '_templates/linux/disable_ulimit.jinja2' %} + +{% include '_templates/linux/disable_strict_host_check.jinja2' %} + +{% include '_templates/linux/disable_motd_update.jinja2' %} + +{%- with secure_path = '/sbin:/bin:/usr/sbin:/usr/bin:/opt/idea/python/latest/bin' %} + {% include '_templates/linux/sudoer_secure_path.jinja2' %} +{%- endwith %} + +{%- with messages = [ + context.module_name + ' (v'+context.module_version+'), Cluster: ' + context.cluster_name +] %} + {% include '_templates/linux/motd.jinja2' %} +{%- endwith %} + +{% include '_templates/linux/mount_shared_storage.jinja2' %} + +{% include '_templates/linux/join_directoryservice.jinja2' %} + +{% if context.config.get_string('scheduler.private_dns_name') %} + {% include '_templates/linux/openpbs_client.jinja2' %} +{% endif %} + +REBOOT_REQUIRED=$(cat /root/bootstrap/reboot_required.txt) +if [[ "${REBOOT_REQUIRED}" == "yes" ]]; then + reboot +fi diff --git a/source/idea/idea-bootstrap/cluster-manager/install_app.sh.jinja2 b/source/idea/idea-bootstrap/cluster-manager/install_app.sh.jinja2 new file mode 100644 index 00000000..b94ce435 --- /dev/null +++ b/source/idea/idea-bootstrap/cluster-manager/install_app.sh.jinja2 @@ -0,0 +1,69 @@ +#!/bin/bash + +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +set -x + +source /etc/environment + +APP_PACKAGE_DOWNLOAD_URI="${1}" +APP_NAME="cluster-manager" + +AWS=$(command -v aws) +# AWS_REGION comes from sourcing /etc/environment +INSTANCE_REGION=${AWS_REGION} +S3_BUCKET=$(echo ${APP_PACKAGE_DOWNLOAD_URI} | cut -f3 -d/) +if [[ ${INSTANCE_REGION} =~ ^us-gov-[a-z]+-[0-9]+$ ]]; then + S3_BUCKET_REGION=$(curl -s --head ${S3_BUCKET}.s3.us-gov-west-1.amazonaws.com | grep bucket-region | awk '{print $2}' | tr -d '\r\n') +else + S3_BUCKET_REGION=$(curl -s --head ${S3_BUCKET}.s3.us-east-1.amazonaws.com | grep bucket-region | awk '{print $2}' | tr -d '\r\n') +fi +echo "Instance Region: [${INSTANCE_REGION}] S3 Bucket Region: [${S3_BUCKET_REGION}]" +$AWS --region ${S3_BUCKET_REGION} s3 cp "${APP_PACKAGE_DOWNLOAD_URI}" "${BOOTSTRAP_DIR}/" +PACKAGE_ARCHIVE=$(basename "${APP_PACKAGE_DOWNLOAD_URI}") +PACKAGE_NAME="${PACKAGE_ARCHIVE%.tar.gz*}" +PACKAGE_DIR="${BOOTSTRAP_DIR}/${PACKAGE_NAME}" +mkdir -p ${PACKAGE_DIR} +tar -xvf ${BOOTSTRAP_DIR}/${PACKAGE_ARCHIVE} -C ${PACKAGE_DIR} +idea_pip install -r ${PACKAGE_DIR}/requirements.txt +idea_pip install $(ls ${PACKAGE_DIR}/*-lib.tar.gz) +mkdir -p ${IDEA_APP_DEPLOY_DIR}/${APP_NAME} +mkdir -p ${IDEA_APP_DEPLOY_DIR}/logs + +# copy webapp +if [[ -d "${IDEA_APP_DEPLOY_DIR}/${APP_NAME}/webapp" ]]; then + rm -rf "${IDEA_APP_DEPLOY_DIR}/${APP_NAME}/webapp" +fi +cp -r ${PACKAGE_DIR}/webapp ${IDEA_APP_DEPLOY_DIR}/${APP_NAME} + +# copy resources +if [[ -d "${IDEA_APP_DEPLOY_DIR}/${APP_NAME}/resources" ]]; then + rm -rf "${IDEA_APP_DEPLOY_DIR}/${APP_NAME}/resources" +fi +cp -r ${PACKAGE_DIR}/resources ${IDEA_APP_DEPLOY_DIR}/${APP_NAME} + +{% include '_templates/linux/create_idea_app_certs.jinja2' %} + +{% include '_templates/linux/supervisord.jinja2' %} + +echo "[program:${APP_NAME}] +command=/opt/idea/python/latest/bin/ideaserver +process_name=${APP_NAME} +redirect_stderr=true +stdout_logfile = /opt/idea/app/logs/stdout.log +stdout_logfile_maxbytes=50MB +stdout_logfile_backups=10 +startsecs=30 +startretries=3 +" > /etc/supervisord.d/${APP_NAME}.ini + +systemctl restart supervisord diff --git a/source/idea/idea-bootstrap/cluster-manager/setup.sh.jinja2 b/source/idea/idea-bootstrap/cluster-manager/setup.sh.jinja2 new file mode 100644 index 00000000..30a5f6f4 --- /dev/null +++ b/source/idea/idea-bootstrap/cluster-manager/setup.sh.jinja2 @@ -0,0 +1,128 @@ +#!/bin/bash + +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +set -x + +{%- set PATH = '/bin:/usr/bin:/sbin:/usr/sbin:/usr/local/bin:/opt/idea/python/latest/bin' %} +echo -e " +## [BEGIN] IDEA Environment Configuration - Do Not Delete +AWS_DEFAULT_REGION={{ context.aws_region }} +AWS_REGION={{ context.aws_region }} +IDEA_BASE_OS={{ context.base_os }} +IDEA_MODULE_NAME={{ context.module_name }} +IDEA_MODULE_ID={{ context.module_id }} +IDEA_MODULE_SET={{ context.module_set }} +IDEA_MODULE_VERSION={{ context.module_version }} +IDEA_CLUSTER_S3_BUCKET={{ context.cluster_s3_bucket }} +IDEA_CLUSTER_NAME={{ context.cluster_name }} +IDEA_CLUSTER_HOME={{ context.cluster_home_dir }} +IDEA_APP_DEPLOY_DIR={{ context.app_deploy_dir }} +BOOTSTRAP_DIR=/root/bootstrap +## [END] IDEA Environment Configuration + +PATH={{ PATH }} +" > /etc/environment + +source /etc/environment + +echo -n "no" > ${BOOTSTRAP_DIR}/reboot_required.txt + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +source "${SCRIPT_DIR}/../common/bootstrap_common.sh" + +{% include '_templates/linux/idea_service_account.jinja2' %} + +{% include '_templates/linux/aws_cli.jinja2' %} + +{% include '_templates/linux/aws_ssm.jinja2' %} + +{% include '_templates/linux/epel_repo.jinja2' %} + +{% include '_templates/linux/system_packages.jinja2' %} + +{%- include '_templates/linux/cloudwatch_agent.jinja2' %} + +{%- include '_templates/linux/restrict_ssh.jinja2' %} + +{%- if context.is_metrics_provider_prometheus() %} + {%- include '_templates/linux/prometheus.jinja2' %} + {%- include '_templates/linux/prometheus_node_exporter.jinja2' %} +{%- endif %} + +{% include '_templates/linux/nfs_utils.jinja2' %} + +{% include '_templates/linux/jq.jinja2' %} + +{%- with ebs_volume_tags = [ + {'Key':'idea:ClusterName', 'Value': context.cluster_name }, + {'Key':'idea:ModuleName', 'Value': context.module_name }, + {'Key':'idea:ModuleId', 'Value': context.module_id }, + {'Key':'Name', 'Value': context.cluster_name + '/' + context.module_id + ' Root Volume' } +] %} + {%- include '_templates/linux/tag_ebs_volumes.jinja2' %} +{%- endwith %} + +{%- with network_interface_tags = [ + {'Key':'idea:ClusterName', 'Value': context.cluster_name }, + {'Key':'idea:ModuleName', 'Value': context.module_name }, + {'Key':'idea:ModuleId', 'Value': context.module_id }, + {'Key':'Name', 'Value': context.cluster_name + '/' + context.module_id + ' Network Interface' } +] %} + {%- include '_templates/linux/tag_network_interface.jinja2' %} +{%- endwith %} + +{% include '_templates/linux/disable_se_linux.jinja2' %} + +log_info "Installing LDAP + Putty packages ..." +OPENLDAP_SERVER_PKGS=({{ ' '.join(context.config.get_list('global-settings.package_config.linux_packages.openldap_server', required=True)) }}) +PUTTY_PKGS=({{ ' '.join(context.config.get_list('global-settings.package_config.linux_packages.putty', required=True)) }}) +yum install -y ${OPENLDAP_SERVER_PKGS[*]} ${PUTTY_PKGS[*]} + +{% with alias_prefix = 'idea', install_dir = '/opt/idea/python' %} + {%- include '_templates/linux/python.jinja2' %} +{%- endwith %} + +{% include '_templates/linux/chronyd.jinja2' %} + +{% include '_templates/linux/disable_ulimit.jinja2' %} + +{% include '_templates/linux/disable_strict_host_check.jinja2' %} + +{% include '_templates/linux/disable_motd_update.jinja2' %} + +{%- with secure_path = PATH %} + {%- include '_templates/linux/sudoer_secure_path.jinja2' %} +{%- endwith %} + +{%- with messages = [ + context.module_name + ' (v'+context.module_version+'), Cluster: ' + context.cluster_name +] %} + {%- include '_templates/linux/motd.jinja2' %} +{%- endwith %} + +{% include '_templates/linux/mount_shared_storage.jinja2' %} + +/bin/bash ${SCRIPT_DIR}/install_app.sh "{{context.vars.app_package_uri}}" + +# cluster manager nodes should join directory service after app has started. +# this is to ensure AD Automation Agent has started, when directory service provider is activedirectory +{% include '_templates/linux/join_directoryservice.jinja2' %} + +# lock down access to IDEA_CLUSTER_HOME for root user +mkdir -p "${IDEA_CLUSTER_HOME}" +chmod 700 "${IDEA_CLUSTER_HOME}" + +REBOOT_REQUIRED=$(cat /root/bootstrap/reboot_required.txt) +if [[ "${REBOOT_REQUIRED}" == "yes" ]]; then + reboot +fi diff --git a/source/idea/idea-bootstrap/common/bootstrap_common.sh b/source/idea/idea-bootstrap/common/bootstrap_common.sh new file mode 100644 index 00000000..458a794e --- /dev/null +++ b/source/idea/idea-bootstrap/common/bootstrap_common.sh @@ -0,0 +1,280 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +BOOTSTRAP_DIR="/root/bootstrap" + +function exit_fail () { + echo "Failed: ${1}" + exit 1 +} + +function log_info() { + echo "[$(date +"%Y-%m-%d %H:%M:%S,%3N")] [INFO] ${1}" +} + +function log_warning() { + echo "[$(date +"%Y-%m-%d %H:%M:%S,%3N")] [WARNING] ${1}" +} + +function log_error() { + echo "[$(date +"%Y-%m-%d %H:%M:%S,%3N")] [ERROR] ${1}" +} + +function log_debug() { + echo "[$(date +"%Y-%m-%d %H:%M:%S,%3N")] [DEBUG] ${1}" +} + +function set_reboot_required () { + log_info "Reboot Required: ${1}" + echo -n "yes" > ${BOOTSTRAP_DIR}/reboot_required.txt +} + +function get_reboot_required () { + if [[ -f ${BOOTSTRAP_DIR}/reboot_required.txt ]]; then + cat ${BOOTSTRAP_DIR}/reboot_required.txt + fi + echo -n "no" +} + +function imds_get () { + local SLASH='' + local IMDS_HOST="http://169.254.169.254" + local IMDS_TTL="300" + # prepend a slash if needed + if [[ "${1:0:1}" == '/' ]]; then + SLASH='' + else + SLASH='/' + fi + local URL="${IMDS_HOST}${SLASH}${1}" + + # Get an Auth token + local TOKEN=$(curl --silent -X PUT "${IMDS_HOST}/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: ${IMDS_TTL}") + + # Get the requested value and echo it back + local OUTPUT=$(curl --silent -H "X-aws-ec2-metadata-token: ${TOKEN}" "${URL}") + echo -n "${OUTPUT}" +} + +function instance_type () { + local INSTANCE_TYPE=$(imds_get /latest/meta-data/instance-type) + echo -n "${INSTANCE_TYPE}" +} + +function instance_family () { + local INSTANCE_FAMILY=$(imds_get /latest/meta-data/instance-type | cut -d. -f1) + echo -n "${INSTANCE_FAMILY}" +} + +function instance_id () { + local INSTANCE_ID=$(imds_get /latest/meta-data/instance-id) + echo -n "${INSTANCE_ID}" +} + +function instance_region () { + local INSTANCE_REGION=$(imds_get /latest/meta-data/placement/region) + echo -n "${INSTANCE_REGION}" +} + +function get_secret() { + local SECRET_ID="${1}" + local MAX_ATTEMPT=10 + local CURRENT_ATTEMPT=0 + local SLEEP_INTERVAL=180 + local AWS=$(which aws) + local command="${AWS} secretsmanager get-secret-value --secret-id ${SECRET_ID} --query SecretString --region ${AWS_DEFAULT_REGION} --output text" + while ! secret=$($command); do + ((CURRENT_ATTEMPT=CURRENT_ATTEMPT+1)) + if [[ ${CURRENT_ATTEMPT} -ge ${MAX_ATTEMPT} ]]; then + echo "error: Timed out waiting for secret from secrets manager" + return 1 + fi + echo "Secret Manager is not ready yet ... Waiting ${SLEEP_INTERVAL} s... Loop count is: ${CURRENT_ATTEMPT}/${MAX_ATTEMPT}" + sleep ${SLEEP_INTERVAL} + done + echo -n "${secret}" +} + +function get_server_ip() { + # Get server IP - based on CR-69639334 - @jasackle + # This may return multiple IP addresses + # So we store an array (SERVER_IP_ARRAY) for them and set the SERVER_IP to the first encountered IPv4 address + # for any future needs in the script for the canonical IPv4 host IP + local SERVER_IP_ARRAY=($(hostname -I)) + local SERVER_IP="" + for ip in ${!SERVER_IP_ARRAY[@]}; do + # Only consider IPv4 as SERVER_IP for now due to PBS interactions with IPv6 addresses. + # We don't have to worry about the regex false matching on an IPv6 address + # with an embedded IPv4 address since this is not user input validation. + # This is just validation from the hostname command outputs. + # Example 2001:db8::10.0.0.1 becomes 2001:db::a00:1 in-kernel and from hostname command output + IPV4_COUNT=$(echo ${SERVER_IP_ARRAY[$ip]} | grep -Ec "(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)") + if [[ -z "${SERVER_IP}" && "${IPV4_COUNT}" -eq '1' ]]; then + SERVER_IP=${SERVER_IP_ARRAY[$ip]} + fi + done + echo "${SERVER_IP}" +} + +# Generate a Shake256 value from the input +function get_shake256() { + local STR="${1}" + local SHAKE_LEN="${2}" + local IDEA_PYTHON=$(command -v idea_python) + local SHAKE_VAL=$(${IDEA_PYTHON} -c "import sys; import hashlib; print(hashlib.shake_256(''.join(sys.argv[1]).encode('utf-8')).hexdigest(int(sys.argv[2])))" "${STR}" "${SHAKE_LEN}") + echo -n "${SHAKE_VAL}" +} + +function get_shake256_hostname() { + local HOSTNAME=$(hostname -s) + local HOSTNAME_PREFIX="IDEA-" + # This is the overall max len in chars. Generally this will be 15 + # for NetBIOS legacy reasons + local MAX_LENGTH=15 + # /2 as get_shake256 takes number of hex bytes - which are 2 ASCII char to display + local ALLOWED_LEN_BYTES=$(((${MAX_LENGTH} - ${#HOSTNAME_PREFIX})/2)) + SHAKE_HOSTNAME=$(get_shake256 "${HOSTNAME}" "${ALLOWED_LEN_BYTES}") + echo -n "${HOSTNAME_PREFIX}${SHAKE_HOSTNAME}" +} + +# fsx for lustre +function add_fsx_lustre_to_fstab () { + local FS_DOMAIN="${1}" + local MOUNT_DIR="${2}" + local MOUNT_OPTIONS=${3} + local FS_MOUNT_NAME="${4}" + + if [[ -z "${MOUNT_OPTIONS}" ]]; then + MOUNT_OPTIONS="lustre defaults,noatime,flock,_netdev 0 0" + fi + + grep -q " ${MOUNT_DIR}/" /etc/fstab + if [[ "$?" == "0" ]]; then + log_info "skip add_fsx_lustre_to_fstab: existing entry found for mount dir: ${MOUNT_DIR}" + return 0 + fi + + # handle cases for scratch file systems during SOCA job submission + if [[ -z "${FS_MOUNT_NAME}" ]]; then + local FSX_ID=$(echo "${FS_DOMAIN}" | cut -d. -f1) + local AWS=$(command -v aws) + FS_MOUNT_NAME=$($AWS fsx describe-file-systems \ + --file-system-ids ${FSX_ID} \ + --query FileSystems[].LustreConfiguration.MountName \ + --region ${AWS_DEFAULT_REGION} \ + --output text) + fi + echo "${FS_DOMAIN}@tcp:/${FS_MOUNT_NAME} ${MOUNT_DIR}/ ${MOUNT_OPTIONS}" >> /etc/fstab +} + +function remove_fsx_lustre_from_fstab () { + local MOUNT_DIR="${1}" + sed -i.bak "\@ ${MOUNT_DIR}/@d" /etc/fstab +} + +# efs +function add_efs_to_fstab () { + local FS_DOMAIN="${1}" + local MOUNT_DIR="${2}" + local MOUNT_OPTIONS=${3} + + # Fallback to NFS options for EFS + if [[ -z "${MOUNT_OPTIONS}" ]]; then + log_info "# INFO - Using Fallback NFS options for EFS due to no mount_options" + MOUNT_OPTIONS="nfs4 nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2,noresvport 0 0" + fi + + grep -q " ${MOUNT_DIR}/" /etc/fstab + if [[ "$?" == "0" ]]; then + log_info "skip add_efs_to_fstab: existing entry found for mount dir: ${MOUNT_DIR}" + return 0 + fi + + echo "${FS_DOMAIN}:/ ${MOUNT_DIR}/ ${MOUNT_OPTIONS}" >> /etc/fstab +} + +function remove_efs_from_fstab () { + local MOUNT_DIR="${1}" + sed -i.bak "\@ ${MOUNT_DIR}/@d" /etc/fstab +} + +# openzfs +function add_fsx_openzfs_to_fstab () { + local FS_DOMAIN="${1}" + local MOUNT_DIR="${2}" + local MOUNT_OPTIONS=${3} + local FS_VOLUME_PATH="${4}" + + if [[ -z "${MOUNT_OPTIONS}" ]]; then + MOUNT_OPTIONS="nfs4 nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2,noresvport 0 0" + fi + + grep -q " ${MOUNT_DIR}/" /etc/fstab + if [[ "$?" == "0" ]]; then + log_info "skip add_openzfs_to_fstab: existing entry found for mount dir: ${MOUNT_DIR}" + return 0 + fi + + # eg. filesystem-dns-name:volume-path /localpath nfs nfsver=version defaults 0 0 + echo "${FS_DOMAIN}:${FS_VOLUME_PATH} ${MOUNT_DIR}/ ${MOUNT_OPTIONS}" >> /etc/fstab +} + +function remove_fsx_openzfs_from_fstab () { + local MOUNT_DIR="${1}" + sed -i.bak "\@ ${MOUNT_DIR}/@d" /etc/fstab +} + +# netapp ontap +function add_fsx_netapp_ontap_to_fstab () { + local FS_DOMAIN="${1}" + local MOUNT_DIR="${2}" + local MOUNT_OPTIONS=${3} + local FS_VOLUME_PATH="${4}" + + if [[ -z "${MOUNT_OPTIONS}" ]]; then + MOUNT_OPTIONS="nfs4 nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2,noresvport 0 0" + fi + + grep -q " ${MOUNT_DIR}/" /etc/fstab + if [[ "$?" == "0" ]]; then + log_info "skip add_netapp_ontap_to_fstab: existing entry found for mount dir: ${MOUNT_DIR}" + return 0 + fi + + # eg. svm-dns-name:volume-junction-path /fsx nfs nfsvers=version,defaults 0 0 + echo "${FS_DOMAIN}:${FS_VOLUME_PATH} ${MOUNT_DIR}/ ${MOUNT_OPTIONS}" >> /etc/fstab +} + +function remove_fsx_netapp_ontap_from_fstab () { + local MOUNT_DIR="${1}" + sed -i.bak "\@ ${MOUNT_DIR}/@d" /etc/fstab +} + +function create_jq_ddb_filter () { + echo ' +def convert_from_dynamodb_object: + def get_property($key): select(keys == [$key])[$key]; + ((objects | { value: get_property("S") }) + // (objects | { value: get_property("N") | tonumber }) + // (objects | { value: get_property("B") }) + // (objects | { value: get_property("M") | map_values(convert_from_dynamodb_object) }) + // (objects | { value: get_property("L") | map(convert_from_dynamodb_object) }) + // (objects | { value: get_property("SS") }) + // (objects | { value: get_property("NS") | map(tonumber) }) + // (objects | { value: get_property("BOOL") }) + // (objects | { value: get_property("BS") }) + // (objects | { value: map_values(convert_from_dynamodb_object) }) + // (arrays | { value: map(convert_from_dynamodb_object) }) + // { value: . }).value + ; +convert_from_dynamodb_object +' > /root/.convert_from_dynamodb_object.jq +} diff --git a/source/idea/idea-bootstrap/compute-node-ami-builder/compute_node_ami_builder.sh.jinja2 b/source/idea/idea-bootstrap/compute-node-ami-builder/compute_node_ami_builder.sh.jinja2 new file mode 100644 index 00000000..216ea839 --- /dev/null +++ b/source/idea/idea-bootstrap/compute-node-ami-builder/compute_node_ami_builder.sh.jinja2 @@ -0,0 +1,69 @@ +#!/bin/bash + +set -x + +source /etc/environment + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +source "${SCRIPT_DIR}/../common/bootstrap_common.sh" + +{% set TAGS = [ + {'Key':'idea:ClusterName', 'Value': context.cluster_name }, + {'Key':'idea:ModuleName', 'Value': context.module_name }, + {'Key':'idea:ModuleId', 'Value': context.module_id }, + {'Key':'Name', 'Value': context.vars.ami_name } +] %} + +{%- with ebs_volume_tags = TAGS %} + {% include '_templates/linux/tag_ebs_volumes.jinja2' %} +{%- endwith %} + +{%- with network_interface_tags = TAGS %} + {% include '_templates/linux/tag_network_interface.jinja2' %} +{%- endwith %} + +{% include '_templates/linux/disable_se_linux.jinja2' %} + +{% include '_templates/linux/system_packages.jinja2' %} + +{% include '_templates/linux/cloudwatch_agent.jinja2' %} + +{%- if context.is_metrics_provider_prometheus() %} + {%- include '_templates/linux/prometheus.jinja2' %} + {%- include '_templates/linux/prometheus_node_exporter.jinja2' %} +{%- endif %} + +{% include '_templates/linux/chronyd.jinja2' %} + +{% include '_templates/linux/disable_motd_update.jinja2' %} + +{%- with messages = [ + 'Compute Node AMI, AMI Name: ' + context.vars.ami_name + ', ModuleId: ' + context.module_id + ', Cluster: ' + context.cluster_name +] %} + {% include '_templates/linux/motd.jinja2' %} +{%- endwith %} + +{% if context.config.get_string('scheduler.provider') == 'openpbs' %} + {% include 'compute-node/_templates/configure_openpbs_compute_node.jinja2' %} +{% endif %} + +{%- if 'efa' in context.vars.enabled_drivers %} + {% include 'compute-node/_templates/efa.jinja2' %} +{%- endif %} + +{%- if 'fsx_lustre' in context.vars.enabled_drivers %} + {% include '_templates/linux/fsx_lustre_client.jinja2' %} +{%- endif %} + +{% include '_templates/linux/disable_nouveau_drivers.jinja2' %} + +REBOOT_REQUIRED=$(cat /root/bootstrap/reboot_required.txt) +if [[ "${REBOOT_REQUIRED}" == "yes" ]]; then + log_info "reboot required. compute_node_ami_builder_post_reboot.sh will be executed after reboot ..." + (crontab -l; echo "@reboot /bin/bash ${SCRIPT_DIR}/compute_node_ami_builder_post_reboot.sh >> ${IDEA_COMPUTE_NODE_AMI_BUILDER_LOGS_DIR}/compute_node_ami_builder_bootstrap.log 2>&1") | crontab - + reboot +else + mount -a + log_info "reboot not required. executing compute_node_ami_builder_post_reboot.sh ..." + /bin/bash ${SCRIPT_DIR}/compute_node_ami_builder_post_reboot.sh >> "${IDEA_COMPUTE_NODE_AMI_BUILDER_LOGS_DIR}/compute_node_ami_builder_bootstrap.log" 2>&1 +fi diff --git a/source/idea/idea-bootstrap/compute-node-ami-builder/compute_node_ami_builder_post_reboot.sh.jinja2 b/source/idea/idea-bootstrap/compute-node-ami-builder/compute_node_ami_builder_post_reboot.sh.jinja2 new file mode 100644 index 00000000..89cf7770 --- /dev/null +++ b/source/idea/idea-bootstrap/compute-node-ami-builder/compute_node_ami_builder_post_reboot.sh.jinja2 @@ -0,0 +1,74 @@ +#!/bin/bash + +set -x + +source /etc/environment + +# reset reboot_required.txt +echo -n "no" > ${BOOTSTRAP_DIR}/reboot_required.txt + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +source "${SCRIPT_DIR}/../common/bootstrap_common.sh" + +{% include 'compute-node/_templates/scheduler_stop.jinja2' %} + +{%- with node_type = 'compute' %} + {%- include '_templates/linux/gpu_drivers.jinja2' %} +{%- endwith %} + +# a reboot may be required if GPU Drivers are installed. +REBOOT_REQUIRED=$(cat /root/bootstrap/reboot_required.txt) +if [[ "${REBOOT_REQUIRED}" == "yes" ]]; then + reboot +else + + # user data customizations + if [[ -f ${IDEA_CLUSTER_HOME}/${IDEA_MODULE_ID}/ami_builder/userdata_customizations.sh ]]; then + /bin/bash ${IDEA_CLUSTER_HOME}/${IDEA_MODULE_ID}/ami_builder/userdata_customizations.sh >> ${IDEA_COMPUTE_NODE_AMI_BUILDER_LOGS_DIR}/userdata_customizations.log 2>&1 + fi + + # clean-up: shared storage mounts + {% for name, storage in context.config.get_config('shared-storage').items() %} + {%- if context.eval_shared_storage_scope(shared_storage=storage) %} + {%- if storage.provider == 'efs' %} + remove_efs_from_fstab "{{ storage.mount_dir }}" + {%- endif %} + {%- if storage.provider == 'fsx_lustre' %} + remove_fsx_lustre_from_fstab "{{ storage.mount_dir }}" + {%- endif %} + {%- if storage.provider == 'fsx_netapp_ontap' %} + remove_fsx_netapp_ontap_from_fstab "{{ storage.mount_dir }}" + {%- endif %} + {%- if storage.provider == 'fsx_openzfs' %} + remove_fsx_openzfs_from_fstab "{{ storage.mount_dir }}" + {%- endif %} + {%- endif %} + {% endfor %} + + # clean-up + rm -rf /var/tmp/* /tmp/* /var/crash/* + rm -rf /etc/ssh/ssh_host_* + rm -f /etc/udev/rules.d/70-persistent-net.rules + grep -l "Created by cloud-init on instance boot automatically" /etc/sysconfig/network-scripts/ifcfg-* | xargs rm -f + rm -rf /root/bootstrap/logs + rm -f /root/bootstrap/reboot_required.txt + {% if context.config.get_string('scheduler.provider') == 'openpbs' %} + systemctl disable pbs + {% endif %} + + # clear crontab to ensure new image starts from clean slate + crontab -l | grep -v 'compute_node_ami_builder_post_reboot.sh' | crontab - + + # create idea_preinstalled_packages.log file indicating packages have been preinstalled on this ami. + echo "$(date)" >> /root/bootstrap/idea_preinstalled_packages.log + + # flush filesystem buffers before signaling ami creation + /bin/sync + + # create tag indicating ec2 instance is ready for creating AMI + aws ec2 create-tags \ + --resources "${AWS_INSTANCE_ID}" \ + --region "${AWS_REGION}" \ + --tags Key=idea:AmiBuilderStatus,Value=complete + +fi diff --git a/source/idea/idea-bootstrap/compute-node-ami-builder/setup.sh.jinja2 b/source/idea/idea-bootstrap/compute-node-ami-builder/setup.sh.jinja2 new file mode 100644 index 00000000..786675a2 --- /dev/null +++ b/source/idea/idea-bootstrap/compute-node-ami-builder/setup.sh.jinja2 @@ -0,0 +1,57 @@ +#!/bin/bash + +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +set -x + +{% set PATH = '/bin:/usr/bin:/sbin:/usr/sbin:/usr/local/bin:/opt/pbs/bin:/opt/pbs/sbin:/opt/pbs/bin' %} +echo -n "no" > ${BOOTSTRAP_DIR}/reboot_required.txt +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +source "${SCRIPT_DIR}/../common/bootstrap_common.sh" + +echo -e " +## [BEGIN] IDEA Environment Configuration - Do Not Delete +AWS_DEFAULT_REGION={{ context.aws_region }} +AWS_REGION={{ context.aws_region }} +AWS_INSTANCE_ID=$(instance_id) +IDEA_BASE_OS={{ context.base_os }} +IDEA_MODULE_NAME={{ context.module_name }} +IDEA_MODULE_ID={{ context.module_id }} +IDEA_MODULE_SET={{ context.module_set }} +IDEA_MODULE_VERSION={{ context.module_version }} +IDEA_CLUSTER_S3_BUCKET={{ context.cluster_s3_bucket }} +IDEA_CLUSTER_NAME={{ context.cluster_name }} +IDEA_CLUSTER_HOME={{ context.cluster_home_dir }} +IDEA_COMPUTE_NODE_AMI_BUILDER_LOGS_DIR={{ context.vars.ami_dir }}/logs/$(hostname -s) +IDEA_AMI_NAME={{ context.vars.ami_name }} +BOOTSTRAP_DIR=/root/bootstrap +## [END] IDEA Environment Configuration + +PATH={{ PATH }} +" > /etc/environment + +source /etc/environment + +{% include 'compute-node/_templates/scheduler_stop.jinja2' %} + +{% include '_templates/linux/idea_service_account.jinja2' %} + +{% include '_templates/linux/epel_repo.jinja2' %} + +{% include '_templates/linux/nfs_utils.jinja2' %} + +{% include '_templates/linux/mount_shared_storage.jinja2' %} + +mkdir -p ${IDEA_COMPUTE_NODE_AMI_BUILDER_LOGS_DIR} + +log_info "executing compute_node_ami_builder.sh ..." +/bin/bash ${SCRIPT_DIR}/compute_node_ami_builder.sh >> "${IDEA_COMPUTE_NODE_AMI_BUILDER_LOGS_DIR}/compute_node_ami_builder_bootstrap.log" 2>&1 diff --git a/source/idea/idea-bootstrap/compute-node/_templates/configure_hyperthreading.jinja2 b/source/idea/idea-bootstrap/compute-node/_templates/configure_hyperthreading.jinja2 new file mode 100644 index 00000000..dfb5cef5 --- /dev/null +++ b/source/idea/idea-bootstrap/compute-node/_templates/configure_hyperthreading.jinja2 @@ -0,0 +1,12 @@ +# Begin: Configure Hyper-threading +{% if context.vars.job.params.enable_ht_support %} +# Hyper-threading is enabled. No configuration required. +{% else %} +for cpunum in $(awk -F'[,-]' '{print $2}' /sys/devices/system/cpu/cpu*/topology/thread_siblings_list | sort -un); +do + echo 0 > /sys/devices/system/cpu/cpu${cpunum}/online; +done +{% endif %} +# End: Configure Hyper-threading + + diff --git a/source/idea/idea-bootstrap/compute-node/_templates/configure_openpbs_compute_node.jinja2 b/source/idea/idea-bootstrap/compute-node/_templates/configure_openpbs_compute_node.jinja2 new file mode 100644 index 00000000..cc8a52bf --- /dev/null +++ b/source/idea/idea-bootstrap/compute-node/_templates/configure_openpbs_compute_node.jinja2 @@ -0,0 +1,26 @@ +# Begin: Configure OpenPBS Compute Node +{% include '_templates/linux/openpbs.jinja2' %} + +echo -e "PBS_SERVER={{ context.config.get_string('scheduler.private_dns_name', required=True).split('.')[0] }} +PBS_START_SERVER=0 +PBS_START_SCHED=0 +PBS_START_COMM=0 +PBS_START_MOM=1 +PBS_EXEC=/opt/pbs +PBS_HOME=/var/spool/pbs +PBS_CORE_LIMIT=unlimited +PBS_SCP=/usr/bin/scp +" > /etc/pbs.conf + +echo -e " +\$clienthost {{ context.config.get_string('scheduler.private_dns_name').split('.')[0] }} +\$usecp *:/dev/null /dev/null +\$usecp *:{{ context.config.get_string('shared-storage.data.mount_dir') }} {{ context.config.get_string('shared-storage.data.mount_dir') }} +" > /var/spool/pbs/mom_priv/config + +echo -e "PATH=/bin:/usr/bin +IDEA_CLUSTER_NAME=${IDEA_CLUSTER_NAME} +AWS_DEFAULT_REGION=${AWS_DEFAULT_REGION} +IDEA_JOB_STATUS_SQS_QUEUE_URL={{context.config.get_string('scheduler.job_status_sqs_queue_url')}} +" > /var/spool/pbs/pbs_environment +# End: Configure OpenPBS Compute Node diff --git a/source/idea/idea-bootstrap/compute-node/_templates/efa.jinja2 b/source/idea/idea-bootstrap/compute-node/_templates/efa.jinja2 new file mode 100644 index 00000000..0535caa7 --- /dev/null +++ b/source/idea/idea-bootstrap/compute-node/_templates/efa.jinja2 @@ -0,0 +1,28 @@ +# Begin: Amazon EFA (Elastic Fabric Adapter) +function install_efa() { + local EFA_URL="{{ context.config.get_string('global-settings.package_config.efa.url', required=True) }}" + local EFA_HASH="{{ context.config.get_string('global-settings.package_config.efa.checksum', required=True).lower().strip() }}" + local EFA_HASH_METHOD="{{ context.config.get_string('global-settings.package_config.efa.checksum_method', required=True).lower().strip() }}" + local EFA_TGZ="$(basename ${EFA_URL})" + local EFA_BOOTSTRAP_DIR="/root/bootstrap/efa" + if [[ -d "${EFA_BOOTSTRAP_DIR}" ]]; then + rm -rf "${EFA_BOOTSTRAP_DIR}" + fi + mkdir -p ${EFA_BOOTSTRAP_DIR} + pushd ${EFA_BOOTSTRAP_DIR} + curl --silent -O ${EFA_URL} + if [[ $(openssl ${EFA_HASH_METHOD} ${EFA_TGZ} | awk '{print $2}') != ${EFA_HASH} ]]; then + echo -e "FATAL ERROR: ${EFA_HASH_METHOD} Checksum for EFA failed. File may be compromised." > /etc/motd + exit 1 + fi + tar -xf ${EFA_TGZ} + pushd ${EFA_BOOTSTRAP_DIR}/aws-efa-installer + /bin/bash efa_installer.sh -y + set_reboot_required "EFA Driver Installed" + popd + popd +} +install_efa +# End: Amazon EFA (Elastic Fabric Adapter) + + diff --git a/source/idea/idea-bootstrap/compute-node/_templates/scheduler_start.jinja2 b/source/idea/idea-bootstrap/compute-node/_templates/scheduler_start.jinja2 new file mode 100644 index 00000000..9931bcdb --- /dev/null +++ b/source/idea/idea-bootstrap/compute-node/_templates/scheduler_start.jinja2 @@ -0,0 +1,7 @@ +# Begin: Scheduler Start +{% if context.config.get_string('scheduler.provider') == 'openpbs' %} +systemctl start pbs +{% endif %} +# Begin: Scheduler Start + + diff --git a/source/idea/idea-bootstrap/compute-node/_templates/scheduler_stop.jinja2 b/source/idea/idea-bootstrap/compute-node/_templates/scheduler_stop.jinja2 new file mode 100644 index 00000000..5446f118 --- /dev/null +++ b/source/idea/idea-bootstrap/compute-node/_templates/scheduler_stop.jinja2 @@ -0,0 +1,7 @@ +# Begin: Scheduler Stop +{% if context.config.get_string('scheduler.provider') == 'openpbs' %} +systemctl stop pbs +{% endif %} +# End: Scheduler Stop + + diff --git a/source/idea/idea-bootstrap/compute-node/_templates/scratch_storage.jinja2 b/source/idea/idea-bootstrap/compute-node/_templates/scratch_storage.jinja2 new file mode 100644 index 00000000..ba4879f3 --- /dev/null +++ b/source/idea/idea-bootstrap/compute-node/_templates/scratch_storage.jinja2 @@ -0,0 +1,147 @@ +# Begin: Scratch Storage +{% if context.vars.job.params.fsx_lustre.enabled %} +{% include '_templates/linux/fsx_lustre_client.jinja2' %} +function setup_scratch_storage_fsx_for_lustre () { + FSX_MOUNTPOINT="{{ context.config.get_string('scheduler.scratch_storage.fsx_lustre.mount_point', '/fsx') }}" + AWS_DNS_SUFFIX="{{ context.config.get_string('cluster.aws.dns_suffix', required=True) }}" + mkdir -p ${FSX_MOUNTPOINT} + chmod 777 ${FSX_MOUNTPOINT} + {% if context.vars.job.params.fsx_lustre.existing_fsx %} + add_fsx_lustre_to_fstab "{{ context.vars.job.params.fsx_lustre.existing_fsx }}" "${FSX_MOUNTPOINT}" + {% else %} + # wait for new scratch FSx to be ready and then add to fstab + local COMPUTE_STACK="{{ context.vars.job.get_compute_stack() }}" + local AWS_REGION="{{ context.aws_region }}" + local AWS=$(command -v aws) + local FSX_ARN=$($AWS resourcegroupstaggingapi get-resources \ + --tag-filters "Key=idea:FSx,Values=true" "Key=idea:StackId,Values=${COMPUTE_STACK}" \ + --query ResourceTagMappingList[].ResourceARN \ + --region "${AWS_REGION}" \ + --output text) + local FSX_ID=$(echo ${FSX_ARN} | cut -d/ -f2) + + local FSX_DNS="${FSX_ID}.fsx.${AWS_REGION}.${AWS_DNS_SUFFIX}" + + local CHECK_FSX_STATUS=$($AWS fsx describe-file-systems \ + --file-system-ids "${FSX_ID}" \ + --query FileSystems[].Lifecycle \ + --region "${AWS_REGION}" \ + --output text) + + local LOOP_COUNT=1 + while [[ "${CHECK_FSX_STATUS}" != "AVAILABLE" ]] && [[ ${LOOP_COUNT} -lt 10 ]] + do + echo "FSX does not seem to be in AVAILABLE status yet ... waiting 60 secs" + sleep 60 + CHECK_FSX_STATUS=$($AWS fsx describe-file-systems \ + --file-system-ids "${FSX_ID}" \ + --query FileSystems[].Lifecycle \ + --region "${AWS_REGION}" \ + --output text) + ((LOOP_COUNT++)) + done + + if [[ "${CHECK_FSX_STATUS}" == "AVAILABLE" ]]; then + echo "FSx is AVAILABLE" + add_fsx_lustre_to_fstab "${FSX_DNS}" "${FSX_MOUNTPOINT}" + else + echo "FSx is not available even after 10 minutes timeout, ignoring FSx mount." + fi + {% endif %} +} +setup_scratch_storage_fsx_for_lustre +{% elif context.vars.job.params.scratch_storage_size.value > 0 %} +function setup_scratch_storage_ebs () { + local SCRATCH_SIZE={{ context.vars.job.params.scratch_storage_size.int_val() }} + local SCRATCH_MOUNTPOINT="{{ context.config.get_string('scheduler.scratch_storage.ebs.mount_point', '/scratch') }}" + + local LIST_ALL_DISKS=$(lsblk --list | grep disk | awk '{print $1}') + for disk in ${LIST_ALL_DISKS}; + do + local CHECK_IF_PARTITION_EXIST=$(lsblk -b /dev/${disk} | grep part | wc -l) + local CHECK_PARTITION_SIZE=$(lsblk -lnb /dev/${disk} -o SIZE) + let SCRATCH_SIZE_IN_BYTES=${SCRATCH_SIZE}*1024*1024*1024 + if [[ ${CHECK_IF_PARTITION_EXIST} -eq 0 ]] && [[ ${CHECK_PARTITION_SIZE} -eq ${SCRATCH_SIZE_IN_BYTES} ]]; then + echo "Detected /dev/${disk} with no partition as scratch device" + mkfs -t ext4 /dev/${disk} + mkdir -p ${SCRATCH_MOUNTPOINT} + chmod 777 ${SCRATCH_MOUNTPOINT} + echo "/dev/${disk} ${SCRATCH_MOUNTPOINT} ext4 defaults 0 0" >> /etc/fstab + fi + done +} +setup_scratch_storage_ebs +{% else %} +function setup_scratch_storage_instance_store () { + local SCRATCH_MOUNTPOINT="{{ context.config.get_string('scheduler.scratch_storage.instance_store.mount_point', '/scratch') }}" + {% raw %} + # Use Instance Store if possible. + local DEVICES=() + if [[ ! -z $(ls /dev/nvme[0-9]n1) ]]; then + echo 'Detected Instance Store: NVME' + DEVICES=$(ls /dev/nvme[0-9]n1) + elif [[ ! -z $(ls /dev/xvdc[a-z]) ]]; then + echo 'Detected Instance Store: SSD' + DEVICES=$(ls /dev/xvdc[a-z]) + fi + + if [[ -z ${DEVICES} ]]; then + echo 'No instance store detected on this machine.' + return 0 + fi + + echo "Detected Instance Store with NVME: ${DEVICES}" + + # Clear Devices which are already mounted (eg: when customer import their own AMI) + local VOLUME_LIST=() + for device in ${DEVICES}; + do + local CHECK_IF_PARTITION_EXIST=$(lsblk -b ${device} | grep part | wc -l) + if [[ ${CHECK_IF_PARTITION_EXIST} -eq 0 ]]; then + echo "${device} is free and can be used" + VOLUME_LIST+=(${device}) + fi + done + + local VOLUME_COUNT=${#VOLUME_LIST[@]} + if [[ ${VOLUME_COUNT} -eq 0 ]]; then + echo "All volumes detected already have a partition or mount point and can't be used as scratch devices" + return 0 + fi + + if [[ ${VOLUME_COUNT} -eq 1 ]]; then + + # If only 1 instance store, mfks as ext4 + echo "Detected 1 NVMe device available, formatting as ext4 .." + mkfs -t ext4 ${VOLUME_LIST} + mkdir -p ${SCRATCH_MOUNTPOINT} + chmod 777 ${SCRATCH_MOUNTPOINT} + echo "${VOLUME_LIST} ${SCRATCH_MOUNTPOINT} ext4 defaults,nofail 0 0" >> /etc/fstab + + elif [[ ${VOLUME_COUNT} -gt 1 ]]; then + + # When instance has more than 1 instance store, raid + mount them as /scratch + echo "Detected more than 1 NVMe device available, creating XFS fs ..." + local DEVICE_NAME="md0" + for dev in ${VOLUME_LIST[@]}; + do + dd if=/dev/zero of=${dev} bs=1M count=1 + done + echo yes | mdadm --create \ + --force \ + --verbose \ + --level=0 \ + --raid-devices=${VOLUME_COUNT} /dev/${DEVICE_NAME} ${VOLUME_LIST[@]} + mkfs -t ext4 /dev/${DEVICE_NAME} + mdadm --detail --scan | tee -a /etc/mdadm.conf + mkdir -p ${SCRATCH_MOUNTPOINT} + chmod 777 ${SCRATCH_MOUNTPOINT} + echo "/dev/${DEVICE_NAME} ${SCRATCH_MOUNTPOINT} ext4 defaults,nofail 0 0" >> /etc/fstab + fi + {% endraw %} +} +setup_scratch_storage_instance_store +{% endif %} +# End: Scratch Storage + + diff --git a/source/idea/idea-bootstrap/compute-node/compute_node.sh.jinja2 b/source/idea/idea-bootstrap/compute-node/compute_node.sh.jinja2 new file mode 100644 index 00000000..48e3b90f --- /dev/null +++ b/source/idea/idea-bootstrap/compute-node/compute_node.sh.jinja2 @@ -0,0 +1,93 @@ +#!/bin/bash + +set -x + +source /etc/environment + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +source "${SCRIPT_DIR}/../common/bootstrap_common.sh" + +{% set TAGS = [ + {'Key':'idea:ClusterName', 'Value': context.cluster_name }, + {'Key':'idea:ModuleName', 'Value': context.module_name }, + {'Key':'idea:ModuleId', 'Value': context.module_id }, + {'Key':'Name', 'Value': 'JobId: ' + context.vars.job.job_id + ', Cluster: ' + context.cluster_name }, + {'Key':'idea:JobOwner', 'Value': context.vars.job.owner }, + {'Key':'idea:Project', 'Value': context.vars.job.project }, + {'Key':'idea:JobQueue', 'Value': context.vars.job.queue } +] %} + +{%- with ebs_volume_tags = TAGS %} + {% include '_templates/linux/tag_ebs_volumes.jinja2' %} +{%- endwith %} + +{%- with network_interface_tags = TAGS %} + {% include '_templates/linux/tag_network_interface.jinja2' %} +{%- endwith %} + +if [[ ! -f ${BOOTSTRAP_DIR}/idea_preinstalled_packages.log ]]; then + + {% include '_templates/linux/disable_se_linux.jinja2' %} + + {% include '_templates/linux/system_packages.jinja2' %} + + {% include '_templates/linux/cloudwatch_agent.jinja2' %} + + {%- if context.is_metrics_provider_prometheus() %} + {%- include '_templates/linux/prometheus.jinja2' %} + {%- include '_templates/linux/prometheus_node_exporter.jinja2' %} + {%- endif %} + +else + log_info "Found ${BOOTSTRAP_DIR}/idea_preinstalled_packages.log... skipping package installation..." +fi + +{% include '_templates/linux/chronyd.jinja2' %} + +{% include '_templates/linux/disable_ulimit.jinja2' %} + +{% include '_templates/linux/disable_strict_host_check.jinja2' %} + +{% include '_templates/linux/disable_motd_update.jinja2' %} + +{%- if context.vars.job.is_persistent_capacity() %} + {% set motd_messages = [ + 'Compute Node, Queue: ' + context.vars.job.queue + ', Keep Forever - Stack UUID: ' + context.vars.job.provisioning_options.stack_uuid + ', Cluster: ' + context.cluster_name + ] %} +{%- elif context.vars.job.is_shared_capacity() %} + {% set motd_messages = [ + 'Compute Node, Queue: ' + context.vars.job.queue + ', JobGroup: ' + context.vars.job.job_group + ', Cluster: ' + context.cluster_name + ] %} +{%- else %} + {% set motd_messages = [ + 'Compute Node, Queue: ' + context.vars.job.queue + ', JobId: ' + context.vars.job.job_id + ', Cluster: ' + context.cluster_name + ] %} +{%- endif %} +{%- with messages = motd_messages %} + {% include '_templates/linux/motd.jinja2' %} +{%- endwith %} + +{% include '_templates/linux/join_directoryservice.jinja2' %} + +{% if context.config.get_string('scheduler.provider') == 'openpbs' %} + {% include 'compute-node/_templates/configure_openpbs_compute_node.jinja2' %} +{% endif %} + +{% include 'compute-node/_templates/scratch_storage.jinja2' %} + +{%- if context.vars.job.params.enable_efa_support %} + {% include 'compute-node/_templates/efa.jinja2' %} +{%- endif %} + +{% include '_templates/linux/disable_nouveau_drivers.jinja2' %} + +REBOOT_REQUIRED=$(cat /root/bootstrap/reboot_required.txt) +if [[ "${REBOOT_REQUIRED}" == "yes" ]]; then + log_info "reboot required. compute_node_post_reboot.sh will be executed after reboot ..." + (crontab -l; echo "@reboot /bin/bash ${SCRIPT_DIR}/compute_node_post_reboot.sh >> ${IDEA_COMPUTE_NODE_LOGS_DIR}/compute_node_bootstrap.log" 2>&1) | crontab - + reboot +else + mount -a + log_info "reboot not required. executing compute_node_post_reboot.sh ..." + /bin/bash ${SCRIPT_DIR}/compute_node_post_reboot.sh >> "${IDEA_COMPUTE_NODE_LOGS_DIR}/compute_node_bootstrap.log" 2>&1 +fi diff --git a/source/idea/idea-bootstrap/compute-node/compute_node_post_reboot.sh.jinja2 b/source/idea/idea-bootstrap/compute-node/compute_node_post_reboot.sh.jinja2 new file mode 100644 index 00000000..b910d0a1 --- /dev/null +++ b/source/idea/idea-bootstrap/compute-node/compute_node_post_reboot.sh.jinja2 @@ -0,0 +1,31 @@ +#!/bin/bash + +set -x + +source /etc/environment + +# reset reboot_required.txt +echo -n "no" > ${BOOTSTRAP_DIR}/reboot_required.txt + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +source "${SCRIPT_DIR}/../common/bootstrap_common.sh" + +{% include 'compute-node/_templates/scheduler_stop.jinja2' %} + +{%- with node_type = 'compute' %} + {% include '_templates/linux/gpu_drivers.jinja2' %} +{%- endwith %} + +# a reboot may be required if GPU Drivers are installed. +REBOOT_REQUIRED=$(cat /root/bootstrap/reboot_required.txt) +if [[ "${REBOOT_REQUIRED}" == "yes" ]]; then + reboot +else + {% include 'compute-node/_templates/configure_hyperthreading.jinja2' %} + + if [[ -f ${IDEA_CLUSTER_HOME}/${IDEA_MODULE_ID}/compute_node/userdata_customizations.sh ]]; then + /bin/bash ${IDEA_CLUSTER_HOME}/${IDEA_MODULE_ID}/compute_node/userdata_customizations.sh >> ${IDEA_COMPUTE_NODE_LOGS_DIR}/userdata_customizations.log 2>&1 + fi + + {% include 'compute-node/_templates/scheduler_start.jinja2' %} +fi diff --git a/source/idea/idea-bootstrap/compute-node/setup.sh.jinja2 b/source/idea/idea-bootstrap/compute-node/setup.sh.jinja2 new file mode 100644 index 00000000..6a862a73 --- /dev/null +++ b/source/idea/idea-bootstrap/compute-node/setup.sh.jinja2 @@ -0,0 +1,62 @@ +#!/bin/bash + +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +set -x + +{% set PATH = '/bin:/usr/bin:/sbin:/usr/sbin:/usr/local/bin:/opt/pbs/bin:/opt/pbs/sbin:/opt/pbs/bin' %} +echo -e " +## [BEGIN] IDEA Environment Configuration - Do Not Delete +AWS_DEFAULT_REGION={{ context.aws_region }} +AWS_REGION={{ context.aws_region }} +IDEA_BASE_OS={{ context.base_os }} +IDEA_MODULE_NAME={{ context.module_name }} +IDEA_MODULE_ID={{ context.module_id }} +IDEA_MODULE_SET={{ context.module_set }} +IDEA_MODULE_VERSION={{ context.module_version }} +IDEA_CLUSTER_S3_BUCKET={{ context.cluster_s3_bucket }} +IDEA_CLUSTER_NAME={{ context.cluster_name }} +IDEA_CLUSTER_HOME={{ context.cluster_home_dir }} +IDEA_JOB_ID={{ context.vars.job.job_id }} +IDEA_JOB_UID={{ context.vars.job.job_uid }} +IDEA_JOB_OWNER={{ context.vars.job.owner }} +IDEA_JOB_GROUP={{ context.vars.job.job_group }} +IDEA_JOB_NAME={{ context.vars.job.job_name }} +IDEA_JOB_QUEUE={{ context.vars.job.queue }} +IDEA_JOB_SCALING_MODE={{ context.vars.job.scaling_mode }} +IDEA_COMPUTE_NODE_LOGS_DIR={{ context.vars.job_directory }}/logs/$(hostname -s) +BOOTSTRAP_DIR=/root/bootstrap +## [END] IDEA Environment Configuration + +PATH={{ PATH }} +" > /etc/environment + +source /etc/environment + +echo -n "no" > ${BOOTSTRAP_DIR}/reboot_required.txt + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +source "${SCRIPT_DIR}/../common/bootstrap_common.sh" +{% include 'compute-node/_templates/scheduler_stop.jinja2' %} + +{% include '_templates/linux/idea_service_account.jinja2' %} + +{% include '_templates/linux/epel_repo.jinja2' %} + +{% include '_templates/linux/nfs_utils.jinja2' %} + +{% include '_templates/linux/mount_shared_storage.jinja2' %} + +mkdir -p ${IDEA_COMPUTE_NODE_LOGS_DIR} + +log_info "executing compute_node.sh ..." +/bin/bash ${SCRIPT_DIR}/compute_node.sh >> "${IDEA_COMPUTE_NODE_LOGS_DIR}/compute_node_bootstrap.log" 2>&1 diff --git a/source/idea/idea-bootstrap/dcv-broker/install_app.sh.jinja2 b/source/idea/idea-bootstrap/dcv-broker/install_app.sh.jinja2 new file mode 100644 index 00000000..f8738b70 --- /dev/null +++ b/source/idea/idea-bootstrap/dcv-broker/install_app.sh.jinja2 @@ -0,0 +1,184 @@ +#!/bin/bash + +# source environment. at this point, PATH should have been already updated +# and must contain soca python installation +source /etc/environment +source /root/bootstrap/infra.cfg + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +source "${SCRIPT_DIR}/../common/bootstrap_common.sh" + +# DCV Broker +DCV_GPG_KEY="{{ context.config.get_string('global-settings.package_config.dcv.gpg_key', required=True) }}" +DCV_SESSION_MANAGER_BROKER_NOARCH_VERSION="{{ context.config.get_string('global-settings.package_config.dcv.broker.linux.al2_rhel_centos7.version', required=True) }}" +DCV_SESSION_MANAGER_BROKER_URL="{{ context.config.get_string('global-settings.package_config.dcv.broker.linux.al2_rhel_centos7.url', required=True) }}" +DCV_SESSION_MANAGER_BROKER_SHA256_HASH="{{ context.config.get_string('global-settings.package_config.dcv.broker.linux.al2_rhel_centos7.sha256sum', required=True) }}" + +timestamp=$(date +%s) +clean_hostname=$(hostname -s) +AWS_REGION={{ context.aws_region }} +CLIENT_TO_BROKER_PORT="{{ context.config.get_string('virtual-desktop-controller.dcv_broker.client_communication_port', required=True) }}" +AGENT_TO_BROKER_PORT="{{ context.config.get_string('virtual-desktop-controller.dcv_broker.agent_communication_port', required=True) }}" +GATEWAY_TO_BROKER_PORT="{{ context.config.get_string('virtual-desktop-controller.dcv_broker.gateway_communication_port', required=True) }}" +SESSION_TOKEN_DURATION="{{ context.config.get_string('virtual-desktop-controller.dcv_broker.session_token_validity', required=True) }}" +DYNAMODB_TABLE_RCU="{{ context.config.get_string('virtual-desktop-controller.dcv_broker.dynamodb_table.read_capacity.min_units', default=5) }}" +DYNAMODB_TABLE_WCU="{{ context.config.get_string('virtual-desktop-controller.dcv_broker.dynamodb_table.write_capacity.min_units', default=5) }}" + +MODULE_ID="{{ context.module_id }}" +STAGING_AREA_RELATIVE_PATH='staging_area' + +function restart_dcv_broker() { + log_info "dcv session manager broker started" + systemctl restart dcv-session-manager-broker + log_info "dcv session manager broker complete" +} + +function clean_staging_area() { + rm -rf '${STAGING_AREA_RELATIVE_PATH}' +} + +function setup_staging_area() { + echo "setup staging area start" + mkdir -p ${STAGING_AREA_RELATIVE_PATH} + rm -rf '${STAGING_AREA_RELATIVE_PATH}/*' + echo "setup staging area end" +} + +function install_dcv_broker_package() { + echo "dcv session manager broker package installation start." + setup_staging_area + + rpm --import ${DCV_GPG_KEY} + pushd ${STAGING_AREA_RELATIVE_PATH} + wget ${DCV_SESSION_MANAGER_BROKER_URL} + if [[ $(sha256sum nice-dcv-session-manager-broker-${DCV_SESSION_MANAGER_BROKER_NOARCH_VERSION}.rpm | awk '{print $1}') != ${DCV_SESSION_MANAGER_BROKER_SHA256_HASH} ]]; then + echo -e "FATAL ERROR: Checksum for DCV Broker failed. File may be compromised." > /etc/motd + exit 1 + fi + yum install -y nice-dcv-session-manager-broker-${DCV_SESSION_MANAGER_BROKER_NOARCH_VERSION}.rpm + popd + echo "dcv session manager broker package installation complete." +} + +function configure_dcv_broker_properties() { + echo "dcv session manager broker properties configuration started." + + if [[ -f /etc/dcv-session-manager-broker/session-manager-broker.properties ]]; then + mv /etc/dcv-session-manager-broker/session-manager-broker.properties /etc/dcv-session-manager-broker/session-manager-broker.properties.${timestamp} + fi + + echo -e " +# session-manager-working-path = /tmp +enable-authorization-server = false +enable-authorization = true +enable-agent-authorization = false + +enable-persistence = true +persistence-db = dynamodb +dynamodb-region = ${AWS_REGION} +dynamodb-table-rcu = ${DYNAMODB_TABLE_RCU} +dynamodb-table-wcu = ${DYNAMODB_TABLE_WCU} +dynamodb-table-name-prefix = ${IDEA_CLUSTER_NAME}.${MODULE_ID}.dcv-broker. +# jdbc-connection-url = jdbc:mysql://database-mysql.rds.amazonaws.com:3306/database-mysql +# jdbc-user = admin +# jdbc-password = password +# enable-api-yaml = true + +connect-session-token-duration-minutes = ${SESSION_TOKEN_DURATION} + +# Dont want to see deleted session info for now. +delete-session-duration-hours = 0 + +# Dont want to see unreachable dcv servers for now. Setting fails for values less than 60 +seconds-before-deleting-unreachable-dcv-server = 60 + +# create-sessions-number-of-retries-on-failure = 2 +# autorun-file-arguments-max-size = 50 +# autorun-file-arguments-max-argument-length = 150 + +client-to-broker-connector-https-port = ${CLIENT_TO_BROKER_PORT} +client-to-broker-connector-bind-host = 0.0.0.0 +# client-to-broker-connector-key-store-file = test_security/KeyStore.jks +# client-to-broker-connector-key-store-pass = dcvsm1 +agent-to-broker-connector-https-port = ${AGENT_TO_BROKER_PORT} +agent-to-broker-connector-bind-host = 0.0.0.0 +# agent-to-broker-connector-key-store-file = test_security/KeyStore.jks +# agent-to-broker-connector-key-store-pass = dcvsm1 + +enable-gateway = true +gateway-to-broker-connector-https-port = ${GATEWAY_TO_BROKER_PORT} +gateway-to-broker-connector-bind-host = 0.0.0.0 +# gateway-to-broker-connector-key-store-file = test_security/KeyStore.jks +# gateway-to-broker-connector-key-store-pass = dcvsm1 +# enable-tls-client-auth-gateway = true +# gateway-to-broker-connector-trust-store-file = test_security/TrustStore.jks +# gateway-to-broker-connector-trust-store-pass = dcvsm1 + +# Broker To Broker +broker-to-broker-port = 47100 +cli-to-broker-port = 47200 +broker-to-broker-bind-host = 0.0.0.0 +broker-to-broker-discovery-port = 47500 +# broker-to-broker-discovery-addresses = 127.0.0.1:47500 +# broker-to-broker-discovery-multicast-group = 127.0.0.1 +# broker-to-broker-discovery-multicast-port = 47400 +broker-to-broker-discovery-aws-region = ${AWS_REGION} +broker-to-broker-discovery-aws-alb-target-group-arn = ${BROKER_CLIENT_TARGET_GROUP_ARN} +broker-to-broker-distributed-memory-max-size-mb = 4096 +# broker-to-broker-key-store-file = test_security/KeyStore.jks +# broker-to-broker-key-store-pass = dcvsm1 +broker-to-broker-connection-login = dcvsm-user +broker-to-broker-connection-pass = dcvsm-pass + +# Metrics +metrics-fleet-name-dimension = ${IDEA_CLUSTER_NAME} +enable-cloud-watch-metrics = true +# if cloud-watch-region is not provided, the region is taken from EC2 IMDS +# cloud-watch-region = ${AWS_REGION} +session-manager-working-path = /var/lib/dcvsmbroker + +# GetSessionScreenshot API +session-screenshot-max-height = 600 +session-screenshot-max-width = 800 +" > /etc/dcv-session-manager-broker/session-manager-broker.properties + + chown root:dcvsmbroker /etc/dcv-session-manager-broker/session-manager-broker.properties + chmod 640 /etc/dcv-session-manager-broker/session-manager-broker.properties + + systemctl enable dcv-session-manager-broker + restart_dcv_broker + echo "dcv session manager broker properties configuration complete." +} + +function configure_dcv_broker() { + echo "Configure dcv broker started" + provider_url="{{ context.config.get_string('identity-provider.cognito.provider_url', required=True) }}" + dcv-session-manager-broker register-auth-server --url "${provider_url}/.well-known/jwks.json" + echo "Configure dcv broker complete" +} + +function verify_broker_installation() { + local output=$(curl -k -s "https://localhost:${CLIENT_TO_BROKER_PORT}/sessionConnectionData/aSession/aOwner" | jq -r '.error') + local count=0 + while [[ ! "$output" == "No authorization header" ]] + do + echo -ne "DCV Session Manager broker verification failed.. sleeping for 30 seconds; ${count} seconds already slept \033[0K\r" + count=count+30 + sleep 30 + output=$(curl -k "https://localhost:${CLIENT_TO_BROKER_PORT}/sessionConnectionData/aSession/aOwner" | jq -r '.error') + done + echo "DCV Session Manager broker verification success" +} + +function notify_controller() { + MESSAGE="{\"event_group_id\":\"BROKER-USERDATA-COMPLETE\", \"event_type\":\"DCV_BROKER_USERDATA_EXECUTION_COMPLETE_EVENT\"}" + AWS=$(command -v aws) + $AWS sqs send-message --queue-url ${CONTROLLER_EVENTS_QUEUE_URL} --message-body "${MESSAGE}" --region ${AWS_REGION} --message-group-id "BROKER-USERDATA-COMPLETE" +} + +install_dcv_broker_package +configure_dcv_broker_properties +verify_broker_installation +configure_dcv_broker +clean_staging_area +notify_controller diff --git a/source/idea/idea-bootstrap/dcv-broker/setup.sh.jinja2 b/source/idea/idea-bootstrap/dcv-broker/setup.sh.jinja2 new file mode 100644 index 00000000..b4075beb --- /dev/null +++ b/source/idea/idea-bootstrap/dcv-broker/setup.sh.jinja2 @@ -0,0 +1,100 @@ +#!/bin/bash + +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +set -x + +{% set PATH = '/bin:/usr/bin:/sbin:/usr/sbin:/usr/local/bin' %} +echo -e " +## [BEGIN] IDEA Environment Configuration - Do Not Delete +AWS_DEFAULT_REGION={{ context.aws_region }} +AWS_REGION={{ context.aws_region }} +IDEA_BASE_OS={{ context.base_os }} +IDEA_MODULE_NAME={{ context.module_name }} +IDEA_MODULE_ID={{ context.module_id }} +IDEA_MODULE_SET={{ context.module_set }} +IDEA_MODULE_VERSION={{ context.module_version }} +IDEA_CLUSTER_S3_BUCKET={{ context.cluster_s3_bucket }} +IDEA_CLUSTER_NAME={{ context.cluster_name }} +IDEA_CLUSTER_HOME={{ context.cluster_home_dir }} +IDEA_APP_DEPLOY_DIR={{ context.app_deploy_dir }} +BOOTSTRAP_DIR=/root/bootstrap +## [END] IDEA Environment Configuration + +PATH=/bin:/usr/bin:/sbin:/usr/sbin:/usr/local/bin +" > /etc/environment + +source /etc/environment + +echo -n "no" > ${BOOTSTRAP_DIR}/reboot_required.txt + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +source "${SCRIPT_DIR}/../common/bootstrap_common.sh" + +{% include '_templates/linux/idea_service_account.jinja2' %} + +{% include '_templates/linux/aws_cli.jinja2' %} + +{% include '_templates/linux/aws_ssm.jinja2' %} + +{% include '_templates/linux/epel_repo.jinja2' %} + +{% include '_templates/linux/system_packages.jinja2' %} + +{%- include '_templates/linux/cloudwatch_agent.jinja2' %} + +{%- include '_templates/linux/restrict_ssh.jinja2' %} + +{%- if context.is_metrics_provider_prometheus() %} + {%- include '_templates/linux/prometheus.jinja2' %} + {%- include '_templates/linux/prometheus_node_exporter.jinja2' %} +{%- endif %} + +{% include '_templates/linux/jq.jinja2' %} + +{%- with ebs_volume_tags = [ + {'Key':'idea:ClusterName', 'Value': context.cluster_name }, + {'Key':'idea:ModuleName', 'Value': context.module_name }, + {'Key':'idea:ModuleId', 'Value': context.module_id }, + {'Key':'Name', 'Value': context.cluster_name + '/' + context.module_id + ' Root Volume' } +] %} + {% include '_templates/linux/tag_ebs_volumes.jinja2' %} +{%- endwith %} + +{%- with network_interface_tags = [ + {'Key':'idea:ClusterName', 'Value': context.cluster_name }, + {'Key':'idea:ModuleName', 'Value': context.module_name }, + {'Key':'idea:ModuleId', 'Value': context.module_id }, + {'Key':'Name', 'Value': context.cluster_name + '/' + context.module_id + ' Network Interface' } +] %} + {% include '_templates/linux/tag_network_interface.jinja2' %} +{%- endwith %} + +{% include '_templates/linux/chronyd.jinja2' %} + +{% include '_templates/linux/disable_ulimit.jinja2' %} + +{% include '_templates/linux/disable_strict_host_check.jinja2' %} + +{% include '_templates/linux/disable_motd_update.jinja2' %} + +{%- with secure_path = PATH %} + {% include '_templates/linux/sudoer_secure_path.jinja2' %} +{%- endwith %} + +{%- with messages = [ + context.module_name + ' (v'+context.module_version+'), Cluster: ' + context.cluster_name +] %} + {% include '_templates/linux/motd.jinja2' %} +{%- endwith %} + +/bin/bash ${SCRIPT_DIR}/install_app.sh diff --git a/source/idea/idea-bootstrap/dcv-connection-gateway/install_app.sh.jinja2 b/source/idea/idea-bootstrap/dcv-connection-gateway/install_app.sh.jinja2 new file mode 100644 index 00000000..0b81f8b2 --- /dev/null +++ b/source/idea/idea-bootstrap/dcv-connection-gateway/install_app.sh.jinja2 @@ -0,0 +1,199 @@ +#!/bin/bash + +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +set -x + +source /etc/environment +source /root/bootstrap/infra.cfg +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +source "${SCRIPT_DIR}/../common/bootstrap_common.sh" + +APP_PACKAGE_DOWNLOAD_URI="${1}" +APP_NAME="dcv-connection-gateway" + +AWS=$(command -v aws) +INSTANCE_REGION=$(instance_region) +S3_BUCKET=$(echo ${APP_PACKAGE_DOWNLOAD_URI} | cut -f3 -d/) + +if [[ ${INSTANCE_REGION} =~ ^us-gov-[a-z]+-[0-9]+$ ]]; then + S3_BUCKET_REGION=$(curl -s --head ${S3_BUCKET}.s3.us-gov-west-1.amazonaws.com | grep bucket-region | awk '{print $2}' | tr -d '\r\n') +else + S3_BUCKET_REGION=$(curl -s --head ${S3_BUCKET}.s3.us-east-1.amazonaws.com | grep bucket-region | awk '{print $2}' | tr -d '\r\n') +fi + +$AWS --region ${S3_BUCKET_REGION} s3 cp "${APP_PACKAGE_DOWNLOAD_URI}" "${BOOTSTRAP_DIR}/" +PACKAGE_ARCHIVE=$(basename "${APP_PACKAGE_DOWNLOAD_URI}") +PACKAGE_NAME="${PACKAGE_ARCHIVE%.tar.gz*}" +APP_DIR=${IDEA_APP_DEPLOY_DIR}/${APP_NAME} +if [[ -d "${APP_DIR}" ]]; then + rm -rf "${APP_DIR}" +fi +mkdir -p ${APP_DIR} +tar -xvf ${BOOTSTRAP_DIR}/${PACKAGE_ARCHIVE} -C ${APP_DIR} + +DCV_GPG_KEY="{{ context.config.get_string('global-settings.package_config.dcv.gpg_key', required=True) }}" +DCV_CONNECTION_GATEWAY_VERSION="{{ context.config.get_string('global-settings.package_config.dcv.connection_gateway.x86_64.linux.al2_rhel_centos7.version', required=True) }}" +DCV_CONNECTION_GATEWAY_URL="{{ context.config.get_string('global-settings.package_config.dcv.connection_gateway.x86_64.linux.al2_rhel_centos7.url', required=True) }}" +DCV_CONNECTION_GATEWAY_SHA256_HASH="{{ context.config.get_string('global-settings.package_config.dcv.connection_gateway.x86_64.linux.al2_rhel_centos7.sha256sum', required=True) }}" + +INTERNAL_ALB_ENDPOINT="{{ context.config.get_cluster_internal_endpoint() }}" +GATEWAY_TO_BROKER_PORT="{{ context.config.get_string("virtual-desktop-controller.dcv_broker.gateway_communication_port", required=True) }}" + +DCV_SERVER_X86_64_URL="{{ context.config.get_string('global-settings.package_config.dcv.host.x86_64.linux.al2_rhel_centos7.url', required=True) }}" +DCV_SERVER_X86_64_TGZ="{{ context.config.get_string('global-settings.package_config.dcv.host.x86_64.linux.al2_rhel_centos7.tgz', required=True) }}" +DCV_SERVER_X86_64_VERSION="{{ context.config.get_string('global-settings.package_config.dcv.host.x86_64.linux.al2_rhel_centos7.version', required=True) }}" +DCV_SERVER_X86_64_SHA256_HASH="{{ context.config.get_string('global-settings.package_config.dcv.host.x86_64.linux.al2_rhel_centos7.sha256sum', required=True) }}" + +DCV_SERVER_AARCH64_URL="{{ context.config.get_string('global-settings.package_config.dcv.host.aarch64.linux.al2_rhel_centos7.url', required=True) }}" +DCV_SERVER_AARCH64_TGZ="{{ context.config.get_string('global-settings.package_config.dcv.host.aarch64.linux.al2_rhel_centos7.tgz', required=True) }}" +DCV_SERVER_AARCH64_VERSION="{{ context.config.get_string('global-settings.package_config.dcv.host.aarch64.linux.al2_rhel_centos7.version', required=True) }}" +DCV_SERVER_AARCH64_SHA256_HASH="{{ context.config.get_string('global-settings.package_config.dcv.host.aarch64.linux.al2_rhel_centos7.sha256sum', required=True) }}" + +DCV_WEB_VIEWER_INSTALL_LOCATION="/usr/share/dcv/www" + +timestamp=$(date +%s) + +function setup_nginx() { + yum install nginx1 -y + yum install nginx -y +echo """ +server { + listen 80; + listen [::]:80; + root ${DCV_WEB_VIEWER_INSTALL_LOCATION}; +} +""" > /etc/nginx/conf.d/default.conf + systemctl enable nginx + systemctl start nginx +} + +function install_dcv_connection_gateway() { + yum install -y nc + rpm --import ${DCV_GPG_KEY} + wget ${DCV_CONNECTION_GATEWAY_URL} + if [[ $(sha256sum nice-dcv-connection-gateway-${DCV_CONNECTION_GATEWAY_VERSION}.rpm | awk '{print $1}') != ${DCV_CONNECTION_GATEWAY_SHA256_HASH} ]]; then + echo -e "FATAL ERROR: Checksum for DCV Connection Gateway failed. File may be compromised." > /etc/motd + exit 1 + fi + yum install -y nice-dcv-connection-gateway-${DCV_CONNECTION_GATEWAY_VERSION}.rpm + rm -rf nice-dcv-connection-gateway-${DCV_CONNECTION_GATEWAY_VERSION}.rpm +} + +function install_dcv_web_viewer() { + echo "# installing dcv web viewer ..." + + local machine=$(uname -m) #x86_64 or aarch64 + local DCV_SERVER_URL="" + local DCV_SERVER_TGZ="" + local DCV_SERVER_VERSION="" + local DCV_SERVER_SHA256_HASH="" + if [[ $machine == "x86_64" ]]; then + # x86_64 + DCV_SERVER_URL=${DCV_SERVER_X86_64_URL} + DCV_SERVER_TGZ=${DCV_SERVER_X86_64_TGZ} + DCV_SERVER_VERSION=${DCV_SERVER_X86_64_VERSION} + DCV_SERVER_SHA256_HASH=${DCV_SERVER_X86_64_SHA256_HASH} + else + # aarch64 + DCV_SERVER_URL=${DCV_SERVER_AARCH64_URL} + DCV_SERVER_TGZ=${DCV_SERVER_AARCH64_TGZ} + DCV_SERVER_VERSION=${DCV_SERVER_AARCH64_VERSION} + DCV_SERVER_SHA256_HASH=${DCV_SERVER_AARCH64_SHA256_HASH} + fi + + wget ${DCV_SERVER_URL} + if [[ $(sha256sum ${DCV_SERVER_TGZ} | awk '{print $1}') != ${DCV_SERVER_SHA256_HASH} ]]; then + echo -e "FATAL ERROR: Checksum for DCV Server failed. File may be compromised." > /etc/motd + exit 1 + fi + tar zxvf ${DCV_SERVER_TGZ} + + pushd nice-dcv-${DCV_SERVER_VERSION} + {% if context.base_os == 'amazonlinux2' -%} + rpm -ivh nice-dcv-web-viewer-*.${machine}.rpm + {% elif context.base_os in ('rhel7', 'centos7') -%} + rpm -ivh nice-dcv-web-viewer-*.${machine}.rpm --nodeps + {% endif -%} + popd + rm -rf nice-dcv-${DCV_SERVER_VERSION} + rm -rf ${DCV_SERVER_TGZ} + + yes | cp -a ${APP_DIR}/static_resources/. ${DCV_WEB_VIEWER_INSTALL_LOCATION} +} + +function configure_certificates() { + local CERT_CONTENT=$(get_secret ${CERTIFICATE_SECRET_ARN}) + local PRIVATE_KEY_CONTENT=$(get_secret ${PRIVATE_KEY_SECRET_ARN}) + mkdir -p /etc/dcv-connection-gateway/certs/ + if [[ -f /etc/dcv-connection-gateway/certs/default_cert.pem ]]; then + mv /etc/dcv-connection-gateway/certs/default_cert.pem /etc/dcv-connection-gateway/certs/default_cert.pem.${timestamp} + fi + if [[ -f /etc/dcv-connection-gateway/certs/default_key_pkcs1.pem ]]; then + mv /etc/dcv-connection-gateway/certs/default_key_pkcs1.pem /etc/dcv-connection-gateway/certs/default_key_pkcs1.pem.${timestamp} + fi + if [[ -f /etc/dcv-connection-gateway/certs/default_key_pkcs8.pem ]]; then + mv /etc/dcv-connection-gateway/certs/default_key_pkcs8.pem /etc/dcv-connection-gateway/certs/default_key_pkcs8.pem.${timestamp} + fi + touch /etc/dcv-connection-gateway/certs/default_cert.pem + touch /etc/dcv-connection-gateway/certs/default_key_pkcs1.pem + echo -e "${CERT_CONTENT}" > /etc/dcv-connection-gateway/certs/default_cert.pem + echo -e "${PRIVATE_KEY_CONTENT}" > /etc/dcv-connection-gateway/certs/default_key_pkcs1.pem + openssl pkcs8 -topk8 -inform PEM -outform PEM -nocrypt -in /etc/dcv-connection-gateway/certs/default_key_pkcs1.pem -out /etc/dcv-connection-gateway/certs/default_key_pkcs8.pem + chmod 600 /etc/dcv-connection-gateway/certs/default_cert.pem + chmod 600 /etc/dcv-connection-gateway/certs/default_key_pkcs1.pem + chmod 600 /etc/dcv-connection-gateway/certs/default_key_pkcs8.pem + + chown -R dcvcgw:dcvcgw /etc/dcv-connection-gateway/certs/default_cert.pem + chown -R dcvcgw:dcvcgw /etc/dcv-connection-gateway/certs/default_key_pkcs1.pem + chown -R dcvcgw:dcvcgw /etc/dcv-connection-gateway/certs/default_key_pkcs8.pem +} + +function configure_dcv_connection_gateway() { + if [[ -f /etc/dcv-connection-gateway/dcv-connection-gateway.conf ]]; then + mv /etc/dcv-connection-gateway/dcv-connection-gateway.conf /etc/dcv-connection-gateway/dcv-connection-gateway.conf.${timestamp} + fi + touch /etc/dcv-connection-gateway/dcv-connection-gateway.conf + echo -e "[log] +level = \"trace\" + +[gateway] +quic-listen-endpoints = [\"0.0.0.0:8443\"] +web-listen-endpoints = [\"0.0.0.0:8443\", \"[::]:8445\"] +cert-file = \"/etc/dcv-connection-gateway/certs/default_cert.pem\" +cert-key-file = \"/etc/dcv-connection-gateway/certs/default_key_pkcs8.pem\" + +[health-check] +bind-addr = \"::\" +port = 8989 + +[dcv] +tls-strict = false + +[resolver] +url = \"${INTERNAL_ALB_ENDPOINT}:${GATEWAY_TO_BROKER_PORT}\" +tls-strict = false + +[web-resources] +url = \"http://localhost:80\" +tls-strict = false +" > /etc/dcv-connection-gateway/dcv-connection-gateway.conf + + systemctl enable dcv-connection-gateway + systemctl start dcv-connection-gateway +} + +setup_nginx +install_dcv_connection_gateway +install_dcv_web_viewer +configure_certificates +configure_dcv_connection_gateway diff --git a/source/idea/idea-bootstrap/dcv-connection-gateway/setup.sh.jinja2 b/source/idea/idea-bootstrap/dcv-connection-gateway/setup.sh.jinja2 new file mode 100644 index 00000000..985bfbdc --- /dev/null +++ b/source/idea/idea-bootstrap/dcv-connection-gateway/setup.sh.jinja2 @@ -0,0 +1,100 @@ +#!/bin/bash + +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +set -x + +{% set PATH = '/bin:/usr/bin:/sbin:/usr/sbin:/usr/local/bin' %} +echo -e " +## [BEGIN] IDEA Environment Configuration - Do Not Delete +AWS_DEFAULT_REGION={{ context.aws_region }} +AWS_REGION={{ context.aws_region }} +IDEA_BASE_OS={{ context.base_os }} +IDEA_MODULE_NAME={{ context.module_name }} +IDEA_MODULE_ID={{ context.module_id }} +IDEA_MODULE_SET={{ context.module_set }} +IDEA_MODULE_VERSION={{ context.module_version }} +IDEA_CLUSTER_S3_BUCKET={{ context.cluster_s3_bucket }} +IDEA_CLUSTER_NAME={{ context.cluster_name }} +IDEA_CLUSTER_HOME={{ context.cluster_home_dir }} +IDEA_APP_DEPLOY_DIR={{ context.app_deploy_dir }} +BOOTSTRAP_DIR=/root/bootstrap +## [END] IDEA Environment Configuration + +PATH=/bin:/usr/bin:/sbin:/usr/sbin:/usr/local/bin +" > /etc/environment + +source /etc/environment + +echo -n "no" > ${BOOTSTRAP_DIR}/reboot_required.txt + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +source "${SCRIPT_DIR}/../common/bootstrap_common.sh" + +{% include '_templates/linux/idea_service_account.jinja2' %} + +{% include '_templates/linux/aws_cli.jinja2' %} + +{% include '_templates/linux/aws_ssm.jinja2' %} + +{% include '_templates/linux/epel_repo.jinja2' %} + +{% include '_templates/linux/system_packages.jinja2' %} + +{%- include '_templates/linux/cloudwatch_agent.jinja2' %} + +{%- include '_templates/linux/restrict_ssh.jinja2' %} + +{%- if context.is_metrics_provider_prometheus() %} + {%- include '_templates/linux/prometheus.jinja2' %} + {%- include '_templates/linux/prometheus_node_exporter.jinja2' %} +{%- endif %} + +{% include '_templates/linux/jq.jinja2' %} + +{%- with ebs_volume_tags = [ + {'Key':'idea:ClusterName', 'Value': context.cluster_name }, + {'Key':'idea:ModuleName', 'Value': context.module_name }, + {'Key':'idea:ModuleId', 'Value': context.module_id }, + {'Key':'Name', 'Value': context.cluster_name + '/' + context.module_id + ' Root Volume' } +] %} + {% include '_templates/linux/tag_ebs_volumes.jinja2' %} +{%- endwith %} + +{%- with network_interface_tags = [ + {'Key':'idea:ClusterName', 'Value': context.cluster_name }, + {'Key':'idea:ModuleName', 'Value': context.module_name }, + {'Key':'idea:ModuleId', 'Value': context.module_id }, + {'Key':'Name', 'Value': context.cluster_name + '/' + context.module_id + ' Network Interface' } +] %} + {% include '_templates/linux/tag_network_interface.jinja2' %} +{%- endwith %} + +{% include '_templates/linux/chronyd.jinja2' %} + +{% include '_templates/linux/disable_ulimit.jinja2' %} + +{% include '_templates/linux/disable_strict_host_check.jinja2' %} + +{% include '_templates/linux/disable_motd_update.jinja2' %} + +{%- with secure_path = PATH %} + {% include '_templates/linux/sudoer_secure_path.jinja2' %} +{%- endwith %} + +{%- with messages = [ + context.module_name + ' (v'+context.module_version+'), Cluster: ' + context.cluster_name +] %} + {% include '_templates/linux/motd.jinja2' %} +{%- endwith %} + +/bin/bash ${SCRIPT_DIR}/install_app.sh "{{context.vars.dcv_connection_gateway_package_uri}}" diff --git a/source/idea/idea-bootstrap/openldap-server/_templates/install_openldap.jinja2 b/source/idea/idea-bootstrap/openldap-server/_templates/install_openldap.jinja2 new file mode 100644 index 00000000..567c622d --- /dev/null +++ b/source/idea/idea-bootstrap/openldap-server/_templates/install_openldap.jinja2 @@ -0,0 +1,242 @@ +# Begin: OpenLDAP Server Configuration +SERVER_HOSTNAME="{{ context.config.get_string('directoryservice.hostname', required=True) }}" +CLUSTER_DATA_DIR="{{ context.config.get_string('shared-storage.data.mount_dir', required=True) }}" +IDEA_DS_PROVIDER="{{ context.config.get_string('directoryservice.provider', required=True) }}" +IDEA_DS_LDAP_BASE="{{ context.config.get_string('directoryservice.ldap_base', required=True) }}" +IDEA_DS_LDAP_NAME="{{ context.config.get_string('directoryservice.name', required=True) }}" + +source ${BOOTSTRAP_DIR}/infra.cfg + +DS_ROOT_USERNAME=$(get_secret "${LDAP_ROOT_USERNAME_SECRET_ARN}") +DS_ROOT_PASSWORD=$(get_secret "${LDAP_ROOT_PASSWORD_SECRET_ARN}") +DS_TLS_CERTIFICATE=$(get_secret "${LDAP_TLS_CERTIFICATE_SECRET_ARN}") +DS_TLS_PRIVATE_KEY=$(get_secret "${LDAP_TLS_PRIVATE_KEY_SECRET_ARN}") + +log_info "Installing LDAP packages ..." +OPENLDAP_SERVER_PKGS=({{ ' '.join(context.config.get_list('global-settings.package_config.linux_packages.openldap_server', required=True)) }}) + +yum install -y ${OPENLDAP_SERVER_PKGS[*]} + +if [[ -f "/etc/openldap/ldap.conf" ]]; then + cp /etc/openldap/ldap.conf /etc/openldap/ldap.conf.orig +fi + +echo -e " +TLS_CACERTDIR /etc/openldap/cacerts + +# Turning this off breaks GSSAPI used with krb5 when rdns = false +SASL_NOCANON on + +URI ldap://${SERVER_HOSTNAME} + +BASE ${IDEA_DS_LDAP_BASE} +" > /etc/openldap/ldap.conf + +systemctl enable slapd +systemctl restart slapd + +if [[ ! -f "/etc/openldap/certs/idea-openldap.key" ]]; then + echo -n "${DS_TLS_PRIVATE_KEY}" > /etc/openldap/certs/idea-openldap.key + echo -n "${DS_TLS_CERTIFICATE}" > /etc/openldap/certs/idea-openldap.crt + chown ldap:ldap /etc/openldap/certs/idea-openldap.key /etc/openldap/certs/idea-openldap.crt + chmod 600 /etc/openldap/certs/idea-openldap.key /etc/openldap/certs/idea-openldap.crt +fi + +# Be mindful with quotes in this section. Double-quote will clobber the crypt format strings as shell arguments +echo -e ' +dn: olcDatabase={-1}frontend,cn=config +changetype: modify +replace: olcPasswordHash +olcPasswordHash: {CRYPT} + +dn: cn=config +changetype: modify +replace: olcPasswordCryptSaltFormat +olcPasswordCryptSaltFormat: $6$%.16s +' > "${BOOTSTRAP_DIR}/password-hash-format.ldif" + +/bin/ldapmodify -Y EXTERNAL -H ldapi:/// -f "${BOOTSTRAP_DIR}/password-hash-format.ldif" + +ENCRYPTED_DS_ROOT_PASSWORD=$(/sbin/slappasswd -s "${DS_ROOT_PASSWORD}" -h '{CRYPT}' -c '$6$%.16s') + +echo -e " +dn: olcDatabase={2}hdb,cn=config +changetype: modify +replace: olcSuffix +olcSuffix: ${IDEA_DS_LDAP_BASE} + +dn: olcDatabase={2}hdb,cn=config +changetype: modify +replace: olcRootDN +olcRootDN: cn=${DS_ROOT_USERNAME},${IDEA_DS_LDAP_BASE} + +dn: olcDatabase={2}hdb,cn=config +changetype: modify +replace: olcRootPW +olcRootPW: ${ENCRYPTED_DS_ROOT_PASSWORD} +" > "${BOOTSTRAP_DIR}/db.ldif" +/bin/ldapmodify -Y EXTERNAL -H ldapi:/// -f "${BOOTSTRAP_DIR}/db.ldif" + +echo -e " +dn: cn=config +changetype: modify +replace: olcTLSCertificateFile +olcTLSCertificateFile: /etc/openldap/certs/idea-openldap.crt +- +replace: olcTLSCertificateKeyFile +olcTLSCertificateKeyFile: /etc/openldap/certs/idea-openldap.key +" > "${BOOTSTRAP_DIR}/update_ssl_cert.ldif" +/bin/ldapmodify -Y EXTERNAL -H ldapi:/// -f "${BOOTSTRAP_DIR}/update_ssl_cert.ldif" + +echo -e " +dn: olcDatabase={2}hdb,cn=config +changetype: modify +replace: olcAccess +olcAccess: {0}to attrs=userPassword by self write by anonymous auth by group.exact=\"ou=admins,${IDEA_DS_LDAP_BASE}\" write by * none +- +add: olcAccess +olcAccess: {1}to * by dn.base=\"gidNumber=0+uidNumber=0,cn=peercred,cn=external,cn=auth\" write by dn.base=\"ou=admins,${IDEA_DS_LDAP_BASE}\" write by * read +" > "${BOOTSTRAP_DIR}/change_user_password.ldif" +/bin/ldapmodify -Y EXTERNAL -H ldapi:/// -f "${BOOTSTRAP_DIR}/change_user_password.ldif" + +# configure SSSVLV overlay (server side sorting and virtual list view) +echo -e " +dn: cn=module,cn=config +objectClass: olcModuleList +cn: module +olcModulepath: /usr/lib64/openldap +olcModuleload: sssvlv.la + +dn: olcOverlay=sssvlv,olcDatabase={2}hdb,cn=config +objectClass: olcSssVlvConfig +olcOverlay: sssvlv +olcSssVlvMax: 10 +olcSssVlvMaxKeys: 5 +" > "${BOOTSTRAP_DIR}/sssvlv_overlay.ldif" +/bin/ldapadd -Y EXTERNAL -H ldapi:/// -f "${BOOTSTRAP_DIR}/sssvlv_overlay.ldif" + +# configure sudoers +echo -e " +dn: cn=sudo,cn=schema,cn=config +objectClass: olcSchemaConfig +cn: sudo +olcAttributeTypes: ( 1.3.6.1.4.1.15953.9.1.1 NAME 'sudoUser' DESC 'User(s) who may run sudo' EQUALITY caseExactIA5Match SUBSTR caseExactIA5SubstringsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.26 ) +olcAttributeTypes: ( 1.3.6.1.4.1.15953.9.1.2 NAME 'sudoHost' DESC 'Host(s) who may run sudo' EQUALITY caseExactIA5Match SUBSTR caseExactIA5SubstringsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.26 ) +olcAttributeTypes: ( 1.3.6.1.4.1.15953.9.1.3 NAME 'sudoCommand' DESC 'Command(s) to be executed by sudo' EQUALITY caseExactIA5Match SYNTAX 1.3.6.1.4.1.1466.115.121.1.26 ) +olcAttributeTypes: ( 1.3.6.1.4.1.15953.9.1.4 NAME 'sudoRunAs' DESC 'User(s) impersonated by sudo (deprecated)' EQUALITY caseExactIA5Match SYNTAX 1.3.6.1.4.1.1466.115.121.1.26 ) +olcAttributeTypes: ( 1.3.6.1.4.1.15953.9.1.5 NAME 'sudoOption' DESC 'Options(s) followed by sudo' EQUALITY caseExactIA5Match SYNTAX 1.3.6.1.4.1.1466.115.121.1.26 ) +olcAttributeTypes: ( 1.3.6.1.4.1.15953.9.1.6 NAME 'sudoRunAsUser' DESC 'User(s) impersonated by sudo' EQUALITY caseExactIA5Match SYNTAX 1.3.6.1.4.1.1466.115.121.1.26 ) +olcAttributeTypes: ( 1.3.6.1.4.1.15953.9.1.7 NAME 'sudoRunAsGroup' DESC 'Group(s) impersonated by sudo' EQUALITY caseExactIA5Match SYNTAX 1.3.6.1.4.1.1466.115.121.1.26 ) +olcObjectClasses: ( 1.3.6.1.4.1.15953.9.2.1 NAME 'sudoRole' SUP top STRUCTURAL DESC 'Sudoer Entries' MUST ( cn ) MAY ( sudoUser $ sudoHost $ sudoCommand $ sudoRunAs $ sudoRunAsUser $ sudoRunAsGroup $ sudoOption $ description ) ) +" > "${BOOTSTRAP_DIR}/sudoers.ldif" +/bin/ldapadd -Y EXTERNAL -H ldapi:/// -f "${BOOTSTRAP_DIR}/sudoers.ldif" + + +domain_component=$(echo "${IDEA_DS_LDAP_NAME}" | cut -d "." -f1) +echo -e " +dn: ${IDEA_DS_LDAP_BASE} +dc: ${domain_component} +objectClass: top +objectClass: domain + +dn: cn=${DS_ROOT_USERNAME},${IDEA_DS_LDAP_BASE} +objectClass: organizationalRole +cn: ${DS_ROOT_USERNAME} +description: LDAP Manager + +dn: ou=People,${IDEA_DS_LDAP_BASE} +objectClass: organizationalUnit +ou: People + +dn: ou=Group,${IDEA_DS_LDAP_BASE} +objectClass: organizationalUnit +ou: Group + +dn: ou=Sudoers,${IDEA_DS_LDAP_BASE} +objectClass: organizationalUnit + +dn: ou=admins,${IDEA_DS_LDAP_BASE} +objectClass: organizationalUnit +ou: Group +" > "${BOOTSTRAP_DIR}/base.ldif" + +/bin/ldapadd -Y EXTERNAL -H ldapi:/// -f "/etc/openldap/schema/cosine.ldif" +/bin/ldapadd -Y EXTERNAL -H ldapi:/// -f "/etc/openldap/schema/nis.ldif" +/bin/ldapadd -Y EXTERNAL -H ldapi:/// -f "/etc/openldap/schema/inetorgperson.ldif" + +echo -n "${DS_ROOT_PASSWORD}" | /bin/ldapadd \ + -x -W \ + -y /dev/stdin \ + -D "cn=${DS_ROOT_USERNAME},${IDEA_DS_LDAP_BASE}" \ + -f "${BOOTSTRAP_DIR}/base.ldif" + + +authconfig \ + --enablesssd \ + --enablesssdauth \ + --enableldap \ + --enableldapauth \ + --ldapserver="ldap://${SERVER_HOSTNAME}" \ + --ldapbasedn="${IDEA_DS_LDAP_BASE}" \ + --enablelocauthorize \ + --enablemkhomedir \ + --enablecachecreds \ + --updateall + + +if [[ -f /etc/sssd/sssd.conf ]]; then + cp /etc/sssd/sssd.conf /etc/sssd/sssd.conf.orig +fi + +echo -e " +[domain/default] +enumerate = True +autofs_provider = ldap +cache_credentials = True +ldap_search_base = ${IDEA_DS_LDAP_BASE} +id_provider = ldap +auth_provider = ldap +chpass_provider = ldap +sudo_provider = ldap +ldap_tls_cacert = /etc/openldap/certs/idea-openldap.crt +ldap_sudo_search_base = ou=Sudoers,${IDEA_DS_LDAP_BASE} +ldap_uri = ldap://${SERVER_HOSTNAME} +ldap_id_use_start_tls = True +use_fully_qualified_names = False +ldap_tls_cacertdir = /etc/openldap/certs/ +ldap_sudo_full_refresh_interval=86400 +ldap_sudo_smart_refresh_interval=3600 + +[sssd] +services = nss, pam, autofs, sudo +full_name_format = %2\$s\%1\$s +domains = default + +[nss] +homedir_substring = ${CLUSTER_DATA_DIR}/home + +[pam] + +[sudo] + +[autofs] + +[ssh] + +[pac] + +[ifp] + +[secrets] +" > /etc/sssd/sssd.conf +chmod 600 /etc/sssd/sssd.conf + +systemctl enable sssd +systemctl restart sssd + +grep -q "sudoers: files sss" /etc/nsswitch.conf +if [[ "$?" == "1" ]]; then + echo "sudoers: files sss" >> /etc/nsswitch.conf +fi + +# End: OpenLDAP Server Configuration diff --git a/source/idea/idea-bootstrap/openldap-server/setup.sh.jinja2 b/source/idea/idea-bootstrap/openldap-server/setup.sh.jinja2 new file mode 100644 index 00000000..c522f6d8 --- /dev/null +++ b/source/idea/idea-bootstrap/openldap-server/setup.sh.jinja2 @@ -0,0 +1,108 @@ +#!/bin/bash + +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +set -x + +{%- set PATH = '/bin:/usr/bin:/sbin:/usr/sbin:/usr/local/bin' %} +echo -e " +## [BEGIN] IDEA Environment Configuration - Do Not Delete +AWS_DEFAULT_REGION={{ context.aws_region }} +AWS_REGION={{ context.aws_region }} +IDEA_BASE_OS={{ context.base_os }} +IDEA_MODULE_NAME={{ context.module_name }} +IDEA_MODULE_ID={{ context.module_id }} +IDEA_MODULE_SET={{ context.module_set }} +IDEA_MODULE_VERSION={{ context.module_version }} +IDEA_CLUSTER_S3_BUCKET={{ context.cluster_s3_bucket }} +IDEA_CLUSTER_NAME={{ context.cluster_name }} +IDEA_CLUSTER_HOME={{ context.cluster_home_dir }} +BOOTSTRAP_DIR=/root/bootstrap +## [END] IDEA Environment Configuration + +PATH={{ PATH }} +" > /etc/environment + +source /etc/environment + +echo -n "no" > ${BOOTSTRAP_DIR}/reboot_required.txt + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +source "${SCRIPT_DIR}/../common/bootstrap_common.sh" + +{% include '_templates/linux/idea_service_account.jinja2' %} + +{% include '_templates/linux/aws_cli.jinja2' %} + +{% include '_templates/linux/aws_ssm.jinja2' %} + +{% include '_templates/linux/epel_repo.jinja2' %} + +{% include '_templates/linux/system_packages.jinja2' %} + +{%- include '_templates/linux/cloudwatch_agent.jinja2' %} + +{%- include '_templates/linux/restrict_ssh.jinja2' %} + +{%- if context.is_metrics_provider_prometheus() %} + {%- include '_templates/linux/prometheus.jinja2' %} + {%- include '_templates/linux/prometheus_node_exporter.jinja2' %} +{%- endif %} + +{% include '_templates/linux/nfs_utils.jinja2' %} + +{% include '_templates/linux/jq.jinja2' %} + +{%- with ebs_volume_tags = [ + {'Key':'idea:ClusterName', 'Value': context.cluster_name }, + {'Key':'idea:ModuleName', 'Value': context.module_name }, + {'Key':'idea:ModuleId', 'Value': context.module_id }, + {'Key':'Name', 'Value': context.cluster_name + '/' + context.module_id + ' Root Volume' } +] %} + {%- include '_templates/linux/tag_ebs_volumes.jinja2' %} +{%- endwith %} + +{%- with network_interface_tags = [ + {'Key':'idea:ClusterName', 'Value': context.cluster_name }, + {'Key':'idea:ModuleName', 'Value': context.module_name }, + {'Key':'idea:ModuleId', 'Value': context.module_id }, + {'Key':'Name', 'Value': context.cluster_name + '/' + context.module_id + ' Network Interface' } +] %} + {%- include '_templates/linux/tag_network_interface.jinja2' %} +{%- endwith %} + +{% include '_templates/linux/disable_se_linux.jinja2' %} + +{% include '_templates/linux/chronyd.jinja2' %} + +{% include '_templates/linux/disable_ulimit.jinja2' %} + +{% include '_templates/linux/disable_strict_host_check.jinja2' %} + +{% include '_templates/linux/disable_motd_update.jinja2' %} + +{%- with secure_path = PATH %} + {%- include '_templates/linux/sudoer_secure_path.jinja2' %} +{%- endwith %} + +{%- with messages = [ + 'OpenLDAP Server (v'+context.module_version+'), Cluster: ' + context.cluster_name +] %} + {%- include '_templates/linux/motd.jinja2' %} +{%- endwith %} + +{%- include 'openldap-server/_templates/install_openldap.jinja2' %} + +REBOOT_REQUIRED=$(cat /root/bootstrap/reboot_required.txt) +if [[ "${REBOOT_REQUIRED}" == "yes" ]]; then + reboot +fi diff --git a/source/idea/idea-bootstrap/scheduler/_templates/configure_openpbs_server.jinja2 b/source/idea/idea-bootstrap/scheduler/_templates/configure_openpbs_server.jinja2 new file mode 100644 index 00000000..41525bce --- /dev/null +++ b/source/idea/idea-bootstrap/scheduler/_templates/configure_openpbs_server.jinja2 @@ -0,0 +1,146 @@ +# Begin: OpenPBS Server Configuration + +{% include '_templates/linux/openpbs.jinja2' %} + +SCHEDULER_PRIVATE_IP=$(get_server_ip) +SCHEDULER_HOSTNAME=$(hostname) +SCHEDULER_HOSTNAME_ALT=$(hostname -s) + +log_info "configure: /etc/hosts" +echo ${SCHEDULER_PRIVATE_IP} ${SCHEDULER_HOSTNAME} ${SCHEDULER_HOSTNAME_ALT} >> /etc/hosts + +log_info "configure: /etc/pbs.conf" +if [[ -f "/etc/pbs.conf" ]]; then + mv /etc/pbs.conf /etc/pbs.conf.$(date +%s) +fi +echo "PBS_SERVER=${SCHEDULER_HOSTNAME_ALT} +PBS_START_SERVER=1 +PBS_START_SCHED=1 +PBS_START_COMM=1 +PBS_START_MOM=0 +PBS_EXEC=/opt/pbs +PBS_HOME=/var/spool/pbs +PBS_CORE_LIMIT=unlimited +PBS_SCP=/usr/bin/scp +" > /etc/pbs.conf + +log_info "configure: /var/spool/pbs/mom_priv/config" +if [[ -f /var/spool/pbs/mom_priv/config ]]; then + mv /var/spool/pbs/mom_priv/config /var/spool/pbs/mom_priv/config.$(date +%s) +fi +echo "\$clienthost ${SCHEDULER_HOSTNAME_ALT}" > /var/spool/pbs/mom_priv/config + +log_info "configure: /var/spool/pbs/server_priv/resourcedef" +if [[ -f /var/spool/pbs/server_priv/resourcedef ]]; then + mv /var/spool/pbs/server_priv/resourcedef /var/spool/pbs/server_priv/resourcedef.$(date +%s) +fi +echo -e " +anonymous_metrics type=string +availability_zone type=string +availability_zone_id type=string +base_os type=string +compute_node type=string flag=h +efa_support type=string +error_message type=string +force_ri type=string +fsx_lustre type=string +fsx_lustre_deployment_type type=string +fsx_lustre_per_unit_throughput type=string +fsx_lustre_size type=string +ht_support type=string +instance_profile type=string +instance_ami type=string +instance_id type=string +instance_type type=string +instance_type_used type=string +keep_ebs type=string +placement_group type=string +root_size type=string +scratch_iops type=string +scratch_size type=string +security_groups type=string +spot_allocation_count type=string +spot_allocation_strategy type=string +spot_price type=string +stack_id type=string +subnet_id type=string +system_metrics type=string +queue_type type=string +job_id type=string +job_group type=string +job_uid type=string +provisioning_time type=string +dry_run type=string +cluster_name type=string +cluster_version type=string +scaling_mode type=string +lifecyle type=string +tenancy type=string +spot_fleet_request type=string +auto_scaling_group type=string +keep_forever type=string +terminate_when_idle type=string +launch_time type=string +capacity_added type=string +job_started_email_template type=string +job_completed_email_template type=string +" > /var/spool/pbs/server_priv/resourcedef + +log_info "configure: /var/spool/pbs/sched_priv/sched_config" +# add compute_node to list of required resource if not already added. +grep -q "compute_node" /var/spool/pbs/sched_priv/sched_config +if [[ "$?" != "0" ]]; then + sed -i 's/resources: "ncpus, mem, arch, host, vnode, aoe, eoe"/resources: "ncpus, mem, arch, host, vnode, aoe, eoe, compute_node"/g' /var/spool/pbs/sched_priv/sched_config +fi + +log_info "configure: /var/spool/pbs/pbs_environment" +# setup openpbs environment variables +if [[ -f "/var/spool/pbs/pbs_environment" ]]; then + mv /var/spool/pbs/pbs_environment /var/spool/pbs/pbs_environment.$(date +%s) +fi +echo "PATH=/bin:/usr/bin +IDEA_SCHEDULER_UNIX_SOCKET=/run/idea.sock +" > /var/spool/pbs/pbs_environment + +log_info "configure: disable system account job submission" +# Disable job submission using system account +grep -q "alias qsub" "/home/{{ context.default_system_user }}/.bash_profile" +if [[ "$?" != "0" ]]; then + echo "alias qsub='echo -e \" !!!! Do not submit job with system account. \n\n Please use LDAP account instead.\"'" >> /home/{{ context.default_system_user }}/.bash_profile +fi + +log_info "enable and restart pbs" +systemctl enable pbs + +systemctl restart pbs +MAX_RETRIES=5 +RETRY_COUNT=0 +while [[ $? -ne 0 ]] && [[ ${RETRY_COUNT} -lt ${MAX_RETRIES} ]] +do + SLEEP_TIME=$(( RANDOM % 33 + 8 )) # Sleep for 8-40 seconds + log_info "(${RETRY_COUNT}/${MAX_RETRIES}) failed to start pbs, retrying in ${SLEEP_TIME} seconds ..." + sleep ${SLEEP_TIME} + ((RETRY_COUNT++)) + systemctl restart pbs +done + +if [[ $? -ne 0 ]]; then + log_error "failed to start pbs server. scheduler pbs configuration failed !!" + exit 1 +fi + +log_info "configure: openpbs server" +/opt/pbs/bin/qmgr -c "set server flatuid = {{ context.config.get_string('scheduler.openpbs.server.flatuid', default='true') }}" +/opt/pbs/bin/qmgr -c "set server job_history_enable = {{ context.config.get_string('scheduler.openpbs.server.job_history_enable', default='1') }}" +/opt/pbs/bin/qmgr -c "set server job_history_duration = {{ context.config.get_string('scheduler.openpbs.server.job_history_duration', default='72:00:00') }}" +/opt/pbs/bin/qmgr -c "set server scheduler_iteration = {{ context.config.get_string('scheduler.openpbs.server.scheduler_iteration', default='30') }}" +/opt/pbs/bin/qmgr -c "set server max_concurrent_provision = {{ context.config.get_string('scheduler.openpbs.server.max_concurrent_provision', default='5000') }}" + +log_info "configure: create openpbs default queue" +/opt/pbs/bin/qmgr -c "create queue normal" +/opt/pbs/bin/qmgr -c "set queue normal queue_type = Execution" +/opt/pbs/bin/qmgr -c "set queue normal started = True" +/opt/pbs/bin/qmgr -c "set queue normal enabled = True" +/opt/pbs/bin/qmgr -c "set server default_queue = normal" + +# End: OpenPBS Server Configuration diff --git a/source/idea/idea-bootstrap/scheduler/install_app.sh.jinja2 b/source/idea/idea-bootstrap/scheduler/install_app.sh.jinja2 new file mode 100644 index 00000000..a7f22c3a --- /dev/null +++ b/source/idea/idea-bootstrap/scheduler/install_app.sh.jinja2 @@ -0,0 +1,99 @@ +#!/bin/bash + +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +set -x + +source /etc/environment + +APP_PACKAGE_DOWNLOAD_URI="${1}" +APP_NAME="scheduler" + +AWS=$(command -v aws) +INSTANCE_REGION=$(instance_region) +S3_BUCKET=$(echo ${APP_PACKAGE_DOWNLOAD_URI} | cut -f3 -d/) + +if [[ ${INSTANCE_REGION} =~ ^us-gov-[a-z]+-[0-9]+$ ]]; then + S3_BUCKET_REGION=$(curl -s --head ${S3_BUCKET}.s3.us-gov-west-1.amazonaws.com | grep bucket-region | awk '{print $2}' | tr -d '\r\n') +else + S3_BUCKET_REGION=$(curl -s --head ${S3_BUCKET}.s3.us-east-1.amazonaws.com | grep bucket-region | awk '{print $2}' | tr -d '\r\n') +fi +$AWS s3 --region ${S3_BUCKET_REGION} cp "${APP_PACKAGE_DOWNLOAD_URI}" "${BOOTSTRAP_DIR}/" + +PACKAGE_ARCHIVE=$(basename "${APP_PACKAGE_DOWNLOAD_URI}") +PACKAGE_NAME="${PACKAGE_ARCHIVE%.tar.gz*}" +PACKAGE_DIR="${BOOTSTRAP_DIR}/${PACKAGE_NAME}" +mkdir -p ${PACKAGE_DIR} +tar -xvf ${BOOTSTRAP_DIR}/${PACKAGE_ARCHIVE} -C ${PACKAGE_DIR} +idea_pip install -r ${PACKAGE_DIR}/requirements.txt +idea_pip install $(ls ${PACKAGE_DIR}/*-lib.tar.gz) +mkdir -p ${IDEA_APP_DEPLOY_DIR}/${APP_NAME} +mkdir -p ${IDEA_APP_DEPLOY_DIR}/logs +if [[ -d "${IDEA_APP_DEPLOY_DIR}/${APP_NAME}/resources" ]]; then + rm -rf "${IDEA_APP_DEPLOY_DIR}/${APP_NAME}/resources" +fi +for directory in ${PACKAGE_DIR}/resources/bootstrap/*/ ; do + candidate="$(basename $directory)" + [ "$candidate" = "_templates" ] && continue + [ "$candidate" = "common" ] && continue + [ "$candidate" = "compute-node" ] && continue + [ "$candidate" = "compute-node-ami-builder" ] && continue + echo "rm -rf ${directory}" +done +cp -r ${PACKAGE_DIR}/resources ${IDEA_APP_DEPLOY_DIR}/${APP_NAME} + +{% if context.config.get_string('scheduler.provider') == 'openpbs' %} +# configure openpbs hooks +/opt/pbs/bin/qmgr -c "create hook validate_job event='queuejob,modifyjob,movejob'" +/opt/pbs/bin/qmgr -c "import hook validate_job application/x-python default ${IDEA_APP_DEPLOY_DIR}/${APP_NAME}/resources/openpbs/hooks/openpbs_hook_handler.py" +/opt/pbs/bin/qmgr -c "create hook job_status event='runjob,execjob_begin,execjob_end'" +/opt/pbs/bin/qmgr -c "import hook job_status application/x-python default ${IDEA_APP_DEPLOY_DIR}/${APP_NAME}/resources/openpbs/hooks/openpbs_hook_handler.py" +{% endif %} + +# since floating license resource checks will be performed by scheduler on compute nodes, +# the license check script is exported to scripts dir on apps shared storage +SCHEDULER_SCRIPTS_DIR="${IDEA_CLUSTER_HOME}/{{ context.module_id }}/scripts" +mkdir -p ${SCHEDULER_SCRIPTS_DIR} +if [[ ! -f "${SCHEDULER_SCRIPTS_DIR}/license_check.py" ]]; then + cp ${IDEA_APP_DEPLOY_DIR}/${APP_NAME}/resources/scripts/license_check.py ${SCHEDULER_SCRIPTS_DIR} +fi + +# compute node userdata customizations script +COMPUTE_NODE_DIR="${IDEA_CLUSTER_HOME}/{{ context.module_id }}/compute_node" +mkdir -p ${COMPUTE_NODE_DIR} +if [[ ! -f "${COMPUTE_NODE_DIR}/userdata_customizations.sh" ]]; then + touch ${COMPUTE_NODE_DIR}/userdata_customizations.sh +fi + +# compute node ami builder userdata customizations script +COMPUTE_NODE_AMI_BUILDER_DIR="${IDEA_CLUSTER_HOME}/{{ context.module_id }}/ami_builder" +mkdir -p ${COMPUTE_NODE_AMI_BUILDER_DIR} +if [[ ! -f "${COMPUTE_NODE_AMI_BUILDER_DIR}/userdata_customizations.sh" ]]; then + touch ${COMPUTE_NODE_AMI_BUILDER_DIR}/userdata_customizations.sh +fi + +{% include '_templates/linux/create_idea_app_certs.jinja2' %} + +{% include '_templates/linux/supervisord.jinja2' %} + +echo "[program:${APP_NAME}] +command=/opt/idea/python/latest/bin/ideaserver +process_name=${APP_NAME} +redirect_stderr=true +stdout_logfile = /opt/idea/app/logs/stdout.log +stdout_logfile_maxbytes=50MB +stdout_logfile_backups=10 +startsecs=30 +startretries=3 +" > /etc/supervisord.d/${APP_NAME}.ini + +systemctl restart supervisord diff --git a/source/idea/idea-bootstrap/scheduler/scheduler_post_reboot.sh.jinja2 b/source/idea/idea-bootstrap/scheduler/scheduler_post_reboot.sh.jinja2 new file mode 100644 index 00000000..c942fb24 --- /dev/null +++ b/source/idea/idea-bootstrap/scheduler/scheduler_post_reboot.sh.jinja2 @@ -0,0 +1,34 @@ +#!/bin/bash + +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +set -x + +source /etc/environment + +SOURCE="${1}" + +# if SOURCE == crontab, remove scheduler_post_reboot.sh entry from current crontab to prevent this script to run on the next reboot +if [[ "${SOURCE}" == "crontab" ]]; then + crontab -l | grep -v 'scheduler_post_reboot.sh' | crontab - +fi + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +source "${SCRIPT_DIR}/../common/bootstrap_common.sh" + +{% include '_templates/linux/openmpi.jinja2' %} + +# Install Python /apps/python for compute nodes +# this will not re-install python if it already exists. +{%- with alias_prefix = 'compute', install_dir = context.config.get_string('shared-storage.apps.mount_dir') + '/python' %} + {% include '_templates/linux/python.jinja2' %} +{%- endwith %} diff --git a/source/idea/idea-bootstrap/scheduler/setup.sh.jinja2 b/source/idea/idea-bootstrap/scheduler/setup.sh.jinja2 new file mode 100644 index 00000000..8d351961 --- /dev/null +++ b/source/idea/idea-bootstrap/scheduler/setup.sh.jinja2 @@ -0,0 +1,129 @@ +#!/bin/bash + +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +set -x + +{% set PATH = '/bin:/usr/bin:/sbin:/usr/sbin:/usr/local/bin:/opt/pbs/bin:/opt/pbs/sbin:/opt/pbs/bin:/opt/idea/python/latest/bin' %} +echo -e " +## [BEGIN] IDEA Environment Configuration - Do Not Delete +AWS_DEFAULT_REGION={{ context.aws_region }} +AWS_REGION={{ context.aws_region }} +IDEA_BASE_OS={{ context.base_os }} +IDEA_MODULE_NAME={{ context.module_name }} +IDEA_MODULE_ID={{ context.module_id }} +IDEA_MODULE_SET={{ context.module_set }} +IDEA_MODULE_VERSION={{ context.module_version }} +IDEA_CLUSTER_S3_BUCKET={{ context.cluster_s3_bucket }} +IDEA_CLUSTER_NAME={{ context.cluster_name }} +IDEA_CLUSTER_HOME={{ context.cluster_home_dir }} +IDEA_APP_DEPLOY_DIR={{ context.app_deploy_dir }} +BOOTSTRAP_DIR=/root/bootstrap +## [END] IDEA Environment Configuration + +PATH={{ PATH }} + +" > /etc/environment + +source /etc/environment + +echo -n "no" > ${BOOTSTRAP_DIR}/reboot_required.txt + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +source "${SCRIPT_DIR}/../common/bootstrap_common.sh" + +{% include '_templates/linux/idea_service_account.jinja2' %} + +{% include '_templates/linux/aws_cli.jinja2' %} + +{% include '_templates/linux/aws_ssm.jinja2' %} + +{% include '_templates/linux/epel_repo.jinja2' %} + +{% include '_templates/linux/system_packages.jinja2' %} + +{%- include '_templates/linux/cloudwatch_agent.jinja2' %} + +{%- include '_templates/linux/restrict_ssh.jinja2' %} + +{%- if context.is_metrics_provider_prometheus() %} + {%- include '_templates/linux/prometheus.jinja2' %} + {%- include '_templates/linux/prometheus_node_exporter.jinja2' %} +{%- endif %} + +{% include '_templates/linux/nfs_utils.jinja2' %} + +{% include '_templates/linux/jq.jinja2' %} + +{%- with ebs_volume_tags = [ + {'Key':'idea:ClusterName', 'Value': context.cluster_name }, + {'Key':'idea:ModuleName', 'Value': context.module_name }, + {'Key':'idea:ModuleId', 'Value': context.module_id }, + {'Key':'Name', 'Value': context.cluster_name + '/' + context.module_id + ' Root Volume' } +] %} + {% include '_templates/linux/tag_ebs_volumes.jinja2' %} +{%- endwith %} + +{%- with network_interface_tags = [ + {'Key':'idea:ClusterName', 'Value': context.cluster_name }, + {'Key':'idea:ModuleName', 'Value': context.module_name }, + {'Key':'idea:ModuleId', 'Value': context.module_id }, + {'Key':'Name', 'Value': context.cluster_name + '/' + context.module_id + ' Network Interface' } +] %} + {% include '_templates/linux/tag_network_interface.jinja2' %} +{%- endwith %} + +{% include '_templates/linux/disable_se_linux.jinja2' %} + +{%- with alias_prefix = 'idea', install_dir = '/opt/idea/python' %} + {% include '_templates/linux/python.jinja2' %} +{%- endwith %} + +{% include '_templates/linux/chronyd.jinja2' %} + +{% include '_templates/linux/disable_ulimit.jinja2' %} + +{% include '_templates/linux/disable_strict_host_check.jinja2' %} + +{% include '_templates/linux/disable_motd_update.jinja2' %} + +{%- with secure_path = PATH %} + {% include '_templates/linux/sudoer_secure_path.jinja2' %} +{%- endwith %} + +{%- with messages = [ + context.module_name + ' (v'+context.module_version+'), Cluster: ' + context.cluster_name +] %} + {% include '_templates/linux/motd.jinja2' %} +{%- endwith %} + +{% include '_templates/linux/mount_shared_storage.jinja2' %} + +{% include '_templates/linux/join_directoryservice.jinja2' %} + +{% if context.config.get_string('scheduler.provider') == 'openpbs' %} + {% include 'scheduler/_templates/configure_openpbs_server.jinja2' %} +{% endif %} + +/bin/bash ${SCRIPT_DIR}/install_app.sh "{{context.vars.app_package_uri}}" + +# upload scheduler logs to S3 +mkdir -p ${IDEA_APP_DEPLOY_DIR}/logs +echo "@daily source /etc/environment; /bin/bash ${IDEA_APP_DEPLOY_DIR}/scheduler/resources/scripts/send_logs_s3.sh >> ${IDEA_APP_DEPLOY_DIR}/logs/send_logs_s3.log 2>&1" | crontab - + +REBOOT_REQUIRED=$(cat /root/bootstrap/reboot_required.txt) +if [[ "${REBOOT_REQUIRED}" == "yes" ]]; then + (crontab -l; echo "@reboot /bin/bash ${SCRIPT_DIR}/scheduler_post_reboot.sh crontab >> ${BOOTSTRAP_DIR}/logs/scheduler_post_reboot.log 2>&1") | crontab - + reboot +else + /bin/bash ${SCRIPT_DIR}/scheduler_post_reboot.sh >> ${BOOTSTRAP_DIR}/logs/scheduler_post_reboot.log 2>&1 +fi diff --git a/source/idea/idea-bootstrap/virtual-desktop-controller/install_app.sh.jinja2 b/source/idea/idea-bootstrap/virtual-desktop-controller/install_app.sh.jinja2 new file mode 100644 index 00000000..2f91cbe4 --- /dev/null +++ b/source/idea/idea-bootstrap/virtual-desktop-controller/install_app.sh.jinja2 @@ -0,0 +1,62 @@ +#!/bin/bash + +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +set -x + +source /etc/environment + +APP_PACKAGE_DOWNLOAD_URI="${1}" +APP_NAME="virtual-desktop-controller" + +AWS=$(command -v aws) +INSTANCE_REGION=$(instance_region) +S3_BUCKET=$(echo ${APP_PACKAGE_DOWNLOAD_URI} | cut -f3 -d/) + +if [[ ${INSTANCE_REGION} =~ ^us-gov-[a-z]+-[0-9]+$ ]]; then + S3_BUCKET_REGION=$(curl -s --head ${S3_BUCKET}.s3.us-gov-west-1.amazonaws.com | grep bucket-region | awk '{print $2}' | tr -d '\r\n') +else + S3_BUCKET_REGION=$(curl -s --head ${S3_BUCKET}.s3.us-east-1.amazonaws.com | grep bucket-region | awk '{print $2}' | tr -d '\r\n') +fi + +$AWS --region ${S3_BUCKET_REGION} s3 cp "${APP_PACKAGE_DOWNLOAD_URI}" "${BOOTSTRAP_DIR}/" + +PACKAGE_ARCHIVE=$(basename "${APP_PACKAGE_DOWNLOAD_URI}") +PACKAGE_NAME="${PACKAGE_ARCHIVE%.tar.gz*}" +PACKAGE_DIR="${BOOTSTRAP_DIR}/${PACKAGE_NAME}" +mkdir -p ${PACKAGE_DIR} +tar -xvf ${BOOTSTRAP_DIR}/${PACKAGE_ARCHIVE} -C ${PACKAGE_DIR} +idea_pip install -r ${PACKAGE_DIR}/requirements.txt +idea_pip install $(ls ${PACKAGE_DIR}/*-lib.tar.gz) +mkdir -p ${IDEA_APP_DEPLOY_DIR}/${APP_NAME} +mkdir -p ${IDEA_APP_DEPLOY_DIR}/logs +if [[ -d "${IDEA_APP_DEPLOY_DIR}/${APP_NAME}/resources" ]]; then +rm -rf "${IDEA_APP_DEPLOY_DIR}/${APP_NAME}/resources" +fi +cp -r ${PACKAGE_DIR}/resources ${IDEA_APP_DEPLOY_DIR}/${APP_NAME} + +{% include '_templates/linux/create_idea_app_certs.jinja2' %} + +{% include '_templates/linux/supervisord.jinja2' %} + +echo "[program:${APP_NAME}] +command=/opt/idea/python/latest/bin/ideaserver +process_name=${APP_NAME} +redirect_stderr=true +stdout_logfile = /opt/idea/app/logs/stdout.log +stdout_logfile_maxbytes=50MB +stdout_logfile_backups=10 +startsecs=30 +startretries=3 +" > /etc/supervisord.d/${APP_NAME}.ini + +systemctl restart supervisord diff --git a/source/idea/idea-bootstrap/virtual-desktop-controller/setup.sh.jinja2 b/source/idea/idea-bootstrap/virtual-desktop-controller/setup.sh.jinja2 new file mode 100644 index 00000000..d5eae982 --- /dev/null +++ b/source/idea/idea-bootstrap/virtual-desktop-controller/setup.sh.jinja2 @@ -0,0 +1,118 @@ +#!/bin/bash + +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +set -x + +{% set PATH = '/bin:/usr/bin:/sbin:/usr/sbin:/usr/local/bin:/opt/idea/python/latest/bin' %} + +echo -e " +## [BEGIN] IDEA Environment Configuration - Do Not Delete +AWS_DEFAULT_REGION={{ context.aws_region }} +AWS_REGION={{ context.aws_region }} +IDEA_BASE_OS={{ context.base_os }} +IDEA_MODULE_NAME={{ context.module_name }} +IDEA_MODULE_ID={{ context.module_id }} +IDEA_MODULE_SET={{ context.module_set }} +IDEA_MODULE_VERSION={{ context.module_version }} +IDEA_CLUSTER_S3_BUCKET={{ context.cluster_s3_bucket }} +IDEA_CLUSTER_NAME={{ context.cluster_name }} +IDEA_CLUSTER_HOME={{ context.cluster_home_dir }} +IDEA_APP_DEPLOY_DIR={{ context.app_deploy_dir }} +BOOTSTRAP_DIR=/root/bootstrap +## [END] IDEA Environment Configuration + +PATH={{ PATH }} +" > /etc/environment + +source /etc/environment + +echo -n "no" > ${BOOTSTRAP_DIR}/reboot_required.txt + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +source "${SCRIPT_DIR}/../common/bootstrap_common.sh" + +{% include '_templates/linux/idea_service_account.jinja2' %} + +{% include '_templates/linux/aws_cli.jinja2' %} + +{% include '_templates/linux/aws_ssm.jinja2' %} + +{% include '_templates/linux/epel_repo.jinja2' %} + +{% include '_templates/linux/system_packages.jinja2' %} + +{%- include '_templates/linux/cloudwatch_agent.jinja2' %} + +{%- include '_templates/linux/restrict_ssh.jinja2' %} + +{%- if context.is_metrics_provider_prometheus() %} + {%- include '_templates/linux/prometheus.jinja2' %} + {%- include '_templates/linux/prometheus_node_exporter.jinja2' %} +{%- endif %} + +{% include '_templates/linux/nfs_utils.jinja2' %} + +{% include '_templates/linux/jq.jinja2' %} + +{%- with ebs_volume_tags = [ + {'Key':'idea:ClusterName', 'Value': context.cluster_name }, + {'Key':'idea:ModuleName', 'Value': context.module_name }, + {'Key':'idea:ModuleId', 'Value': context.module_id }, + {'Key':'Name', 'Value': context.cluster_name + '/' + context.module_id + ' Root Volume' } +] %} + {% include '_templates/linux/tag_ebs_volumes.jinja2' %} +{%- endwith %} + +{%- with network_interface_tags = [ + {'Key':'idea:ClusterName', 'Value': context.cluster_name }, + {'Key':'idea:ModuleName', 'Value': context.module_name }, + {'Key':'idea:ModuleId', 'Value': context.module_id }, + {'Key':'Name', 'Value': context.cluster_name + '/' + context.module_id + ' Network Interface' } +] %} + {% include '_templates/linux/tag_network_interface.jinja2' %} +{%- endwith %} + +{% include '_templates/linux/disable_se_linux.jinja2' %} + +{%- with alias_prefix = 'idea', install_dir = '/opt/idea/python' %} + {% include '_templates/linux/python.jinja2' %} +{%- endwith %} + +{% include '_templates/linux/chronyd.jinja2' %} + +{% include '_templates/linux/disable_ulimit.jinja2' %} + +{% include '_templates/linux/disable_strict_host_check.jinja2' %} + +{% include '_templates/linux/disable_motd_update.jinja2' %} + +{%- with secure_path = PATH %} + {% include '_templates/linux/sudoer_secure_path.jinja2' %} +{%- endwith %} + +{%- with messages = [ + context.module_name + ' (v'+context.module_version+'), Cluster: ' + context.cluster_name +] %} + {% include '_templates/linux/motd.jinja2' %} +{%- endwith %} + +{% include '_templates/linux/mount_shared_storage.jinja2' %} + +{% include '_templates/linux/join_directoryservice.jinja2' %} + +/bin/bash ${SCRIPT_DIR}/install_app.sh "{{context.vars.controller_package_uri}}" + +REBOOT_REQUIRED=$(cat /root/bootstrap/reboot_required.txt) +if [[ "${REBOOT_REQUIRED}" == "yes" ]]; then + reboot +fi diff --git a/source/idea/idea-bootstrap/virtual-desktop-host-linux/configure_dcv_host.sh.jinja2 b/source/idea/idea-bootstrap/virtual-desktop-host-linux/configure_dcv_host.sh.jinja2 new file mode 100644 index 00000000..64a0aa17 --- /dev/null +++ b/source/idea/idea-bootstrap/virtual-desktop-host-linux/configure_dcv_host.sh.jinja2 @@ -0,0 +1,333 @@ +#!/bin/bash + +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +set -x +source /etc/environment +timestamp=$(date +%s) +if [[ -f ${BOOTSTRAP_DIR}/logs/configure_dcv_host.log ]]; then + mv ${BOOTSTRAP_DIR}/logs/configure_dcv_host.log ${BOOTSTRAP_DIR}/logs/configure_dcv_host.log.${timestamp} +fi +exec >> ${BOOTSTRAP_DIR}/logs/configure_dcv_host.log 2>&1 + +SOURCE="${1}" +if [[ "${SOURCE}" == "crontab" ]]; then + # clean crontab, remove current file from reboot commands + crontab -l | grep -v 'configure_dcv_host.sh' | crontab - +fi +echo -n "no" > ${BOOTSTRAP_DIR}/reboot_required.txt +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +source "${SCRIPT_DIR}/../common/bootstrap_common.sh" +timestamp=$(date '+%s') + +BROKER_CERTIFICATE_LOCATION_LOCAL='/etc/dcv/dcv_broker/dcvsmbroker_ca.pem' +BROKER_HOSTNAME="{{ context.config.get_cluster_internal_endpoint().replace('https://', '') }}" +INTERNAL_ALB_ENDPOINT="{{ context.config.get_cluster_internal_endpoint() }}" +BROKER_AGENT_CONNECTION_PORT="{{ context.config.get_int('virtual-desktop-controller.dcv_broker.agent_communication_port', required=True) }}" +IDEA_SESSION_ID="{{ context.vars.idea_session_id }}" +IDEA_SESSION_OWNER="{{ context.vars.session_owner }}" + +function install_microphone_redirect() { + if [[ -z "$(rpm -qa pulseaudio-utils)" ]]; then + echo "Installing microphone redirect..." + yum install -y pulseaudio-utils + else + log_info "Found pulseaudio-utils pre-installed... skipping installation..." + fi +} + +function install_usb_support() { + if [[ -z "$(lsmod | grep eveusb)" ]]; then + echo "Installing usb support..." + DCV_USB_DRIVER_INSTALLER=$(which dcvusbdriverinstaller) + $DCV_USB_DRIVER_INSTALLER --quiet + else + log_info "Found eveusb kernel module pre-installed... skipping installation..." + fi + + echo "# disable x11 display power management system" + echo -e ' +Section "Extensions" + Option "DPMS" "Disable" +EndSection' > /etc/X11/xorg.conf.d/99-disable-dpms.conf +} + +function download_broker_certificate() { + # local BROKER_CERTIFICATE_URL="{{ context.vars.broker_cert_url }}" + # if [[ "${BROKER_CERTIFICATE_URL}" == s3://* ]]; then + # local AWS=$(command -v aws) + # $AWS s3 cp "${BROKER_CERTIFICATE_URL}" "${BROKER_CERTIFICATE_LOCATION_LOCAL}" + # else + # cp "${BROKER_CERTIFICATE_URL}" "${BROKER_CERTIFICATE_LOCATION_LOCAL}" + # fi + echo "Downloading broker certificate logic goes here.. NO-OP" +} + +function configure_dcv_host() { + local HOSTNAME=$(hostname -s) + local IDLE_TIMEOUT="{{ context.config.get_string('virtual-desktop-controller.dcv_session.idle_timeout', required=True) }}" + local IDLE_TIMEOUT_WARNING="{{ context.config.get_string('virtual-desktop-controller.dcv_session.idle_timeout_warning', required=True) }}" + echo "# configuring dcv host ..." + + if [[ -f /etc/dcv/dcv.conf ]]; then + mv /etc/dcv/dcv.conf /etc/dcv/dcv.conf.${timestamp} + fi + + local DCV_STORAGE_ROOT="/data/home/${IDEA_SESSION_OWNER}/storage-root" + if [[ -L ${DCV_STORAGE_ROOT} ]]; then + echo "something fishy is going on here. a sym-link should not exist. check with the session owner for bad actor or unwarranted usage of system." + exit 1 + fi + if [[ ! -d ${DCV_STORAGE_ROOT} ]]; then + mkdir -p ${DCV_STORAGE_ROOT} # Create the storage root location if needed + chown ${IDEA_SESSION_OWNER} ${DCV_STORAGE_ROOT} + fi + local GL_DISPLAYS_VALUE=":0.0" + echo -e "[license] +[log] +level=debug +[session-management] +virtual-session-xdcv-args=\"-listen tcp\" +[session-management/defaults] +[session-management/automatic-console-session] +[display] +# add more if using an instance with more GPU +cuda-devices=[\"0\"] +[display/linux] +gl-displays = [\"${GL_DISPLAYS_VALUE}\"] +[display/linux] +use-glx-fallback-provider=true +[connectivity] +enable-quic-frontend=true +idle-timeout=${IDLE_TIMEOUT} +idle-timeout-warning=${IDLE_TIMEOUT_WARNING} +[security] +supervision-control=\"enforced\" +# ca-file=\"${BROKER_CERTIFICATE_LOCATION_LOCAL}\" +auth-token-verifier=\"${INTERNAL_ALB_ENDPOINT}:${BROKER_AGENT_CONNECTION_PORT}/agent/validate-authentication-token\" +no-tls-strict=true +os-auto-lock=false +administrators=[\"dcvsmagent\"] +[windows] +disable-display-sleep=true +" > /etc/dcv/dcv.conf +} + +function configure_dcv_agent() { + if [[ -f "/etc/dcv-session-manager-agent/agent.conf" ]]; then + mv /etc/dcv-session-manager-agent/agent.conf /etc/dcv-session-manager-agent/agent.conf.${timestamp} + fi + + echo -e "version = '0.1' +[agent] +# hostname or IP of the broker. This parameter is mandatory. +broker_host = '${BROKER_HOSTNAME}' +# The port of the broker. Default: 8445 +broker_port = ${BROKER_AGENT_CONNECTION_PORT} +# CA used to validate the certificate of the broker. +# ca_file = '${BROKER_CERTIFICATE_LOCATION_LOCAL}' +tls_strict = false +# Folder on the file system from which the tag files are read. +tags_folder = '/etc/dcv-session-manager-agent/tags/' +broker_update_interval = 15 +[log] +level = 'debug' +rotation = 'daily' +" > /etc/dcv-session-manager-agent/agent.conf + + mkdir -p /etc/dcv-session-manager-agent/tags/ + if [[ -f "/etc/dcv-session-manager-agent/tags/idea_tags.toml" ]]; then + mkdir -p /etc/dcv-session-manager-agent/archive-tags/ + mv /etc/dcv-session-manager-agent/tags/idea_tags.toml /etc/dcv-session-manager-agent/archive-tags/idea_tags.toml.${timestamp} + fi + + echo -e "idea_session_id=\"${IDEA_SESSION_ID}\"" > /etc/dcv-session-manager-agent/tags/idea_tags.toml +} + +function restart_x_server() { + echo "# restart x server ..." + sudo systemctl isolate multi-user.target + sudo systemctl isolate graphical.target + sudo systemctl set-default graphical.target + echo "# wait for x server to start ..." +} + +function configure_gl() { + {% if context.base_os == 'amazonlinux2' -%} + echo "OS is {{ context.base_os }}, no need for this configuration. NO-OP..." + {% elif context.is_gpu_instance_type() -%} + if [[ $machine == "x86_64" ]]; then + echo "Detected GPU instance, configuring GL..." + local IDEA_SERVICES_PATH="/opt/idea/.services" + local IDEA_SERVICES_LOGS_PATH="${IDEA_SERVICES_PATH}/logs" + mkdir -p "${IDEA_SERVICES_LOGS_PATH}" + DCVGLADMIN=$(which dcvgladmin) + +echo """ +echo \"GLADMIN START\" >> ${IDEA_SERVICES_LOGS_PATH}/idea-reboot-do-not-edit-or-delete-idea-gl.log 2>&1 +echo \$(date) >> ${IDEA_SERVICES_LOGS_PATH}/idea-reboot-do-not-edit-or-delete-idea-gl.log 2>&1 +${DCVGLADMIN} disable >> ${IDEA_SERVICES_LOGS_PATH}/idea-reboot-do-not-edit-or-delete-idea-gl.log 2>&1 +${DCVGLADMIN} enable >> ${IDEA_SERVICES_LOGS_PATH}/idea-reboot-do-not-edit-or-delete-idea-gl.log 2>&1 +echo \$(date) >> ${IDEA_SERVICES_LOGS_PATH}/idea-reboot-do-not-edit-or-delete-idea-gl.log 2>&1 +echo \"GLADMIN END\" >> ${IDEA_SERVICES_LOGS_PATH}/idea-reboot-do-not-edit-or-delete-idea-gl.log 2>&1 +""" >> /etc/rc.d/rc.local + + chmod +x /etc/rc.d/rc.local + systemctl enable rc-local + fi + {% else %} + echo "OS is {{ context.base_os }}, BUT not a GL Machine. No need for this configuration. NO-OP..." + {% endif %} +} + +function start_and_validate_x_server() { + restart_x_server + verify_x_server_is_up +} + +function start_and_configure_dcv_service() { + echo "# start dcv server ..." + sudo systemctl enable dcvserver + echo """ +# This file is part of systemd. +# AGENT +# systemd is free software; you can redistribute it and/or modify it +# under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation; either version 2.1 of the License, or +# (at your option) any later version. + +[Unit] +Description=NICE DCV server daemon +DefaultDependencies=no +Conflicts=umount.target +After=network-online.target remote-fs.target +Before=umount.target + +[Service] +PermissionsStartOnly=true +ExecStartPre=-/sbin/modprobe eveusb +ExecStart=/usr/bin/dcvserver -d --service +Restart=always +BusName=com.nicesoftware.DcvServer +User=dcv +Group=dcv + +[Install] +WantedBy=multi-user.target +""" > /usr/lib/systemd/system/dcvserver.service + + # Because we have modified the .service file we need to tell systemctl to reload and recreate the dependency tree again. + # This is necessary because we are introducing a dependency on network.targets. + # Refer - https://serverfault.com/questions/700862/do-systemd-unit-files-have-to-be-reloaded-when-modified + sudo systemctl daemon-reload + sudo systemctl restart dcvserver +} + +function start_and_configure_dcv_agent_service() { + echo "# start dcv session manager agent ..." + sudo systemctl enable dcv-session-manager-agent + echo """ +# This file is part of systemd. +# AGENT +# systemd is free software; you can redistribute it and/or modify it +# under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation; either version 2.1 of the License, or +# (at your option) any later version. + +[Unit] +Description=Agent component of DCV Session Manager +DefaultDependencies=no +Conflicts=umount.target +After=network-online.target remote-fs.target +Before=umount.target + +[Service] +Type=simple +ExecStart=/usr/libexec/dcv-session-manager-agent/dcvsessionmanageragent +User=dcvsmagent +Group=dcvsmagent + +[Install] +WantedBy=multi-user.target +""" > /usr/lib/systemd/system/dcv-session-manager-agent.service + + # Because we have modified the .service file we need to tell systemctl to reload and recreate the dependency tree again. + # This is necessary because we are introducing a dependency on network.targets. + # Refer - https://serverfault.com/questions/700862/do-systemd-unit-files-have-to-be-reloaded-when-modified + sudo systemctl daemon-reload + sudo systemctl restart dcv-session-manager-agent +} + +function x_server_validation_command() { + echo $(DISPLAY=:0 XAUTHORITY=$(ps aux | grep "X.*\-auth" | grep -v grep | sed -n 's/.*-auth \([^ ]\+\).*/\1/p') xhost | grep "SI:localuser:dcv$") +} + +function verify_x_server_is_up() { + echo "# validating if x server is running ..." + sleep 10 + local output=$(x_server_validation_command) + local count=0 + while [[ ! "$output" == "SI:localuser:dcv" ]] + do + echo -ne "Waiting for X Server to come up.. sleeping for 10 more seconds; ${count} seconds already slept \033[0K\r" + count=$(($count+10)) + sleep 10 + output=$(x_server_validation_command) + if [[ $(expr $count % 50) == 0 ]] + then + echo "Waited 5 times in a row. Was unsuccessful. trying to restart x server again..." + restart_x_server + fi + done + echo "x server is up an running...." +} + +## -- DCV RELATED EXECUTION BEGINS HERE -- ## + +{% include '_templates/linux/dcv_server.jinja2' %} +{% include '_templates/linux/dcv_session_manager_agent.jinja2' %} +install_usb_support +install_microphone_redirect + +download_broker_certificate +configure_dcv_host +configure_dcv_agent + +configure_gl +machine=$(uname -m) #x86_64 or aarch64 +if [[ $machine == "x86_64" ]]; then + start_and_validate_x_server +fi +start_and_configure_dcv_service +start_and_configure_dcv_agent_service + +## -- DCV RELATED EXECUTION ENDS HERE -- ## + +# run user customizations if available +if [[ -f ${IDEA_CLUSTER_HOME}/dcv_host/userdata_customizations.sh ]]; then + /bin/bash ${IDEA_CLUSTER_HOME}/dcv_host/userdata_customizations.sh >> ${BOOTSTRAP_DIR}/logs/userdata_customizations.log 2>&1 +fi + +# notify controller +CONTROLLER_EVENTS_QUEUE_URL="{{ context.config.get_string('virtual-desktop-controller.events_sqs_queue_url', required=True) }}" +MESSAGE="{{ context.vars.dcv_host_ready_message }}" +AWS=$(command -v aws) +$AWS sqs send-message --queue-url ${CONTROLLER_EVENTS_QUEUE_URL} --message-body ${MESSAGE} --region ${AWS_REGION} --message-group-id ${IDEA_SESSION_ID} + +# set up crontab to notify controller on reboot +REBOOT_REQUIRED=$(cat /root/bootstrap/reboot_required.txt) +if [[ "${REBOOT_REQUIRED}" == "yes" ]]; then + (crontab -l; echo "@reboot /bin/bash ${SCRIPT_DIR}/configure_dcv_host_post_reboot.sh crontab") | crontab - + reboot +else + /bin/bash ${SCRIPT_DIR}/configure_dcv_host_post_reboot.sh +fi diff --git a/source/idea/idea-bootstrap/virtual-desktop-host-linux/configure_dcv_host_post_reboot.sh.jinja2 b/source/idea/idea-bootstrap/virtual-desktop-host-linux/configure_dcv_host_post_reboot.sh.jinja2 new file mode 100644 index 00000000..763b2185 --- /dev/null +++ b/source/idea/idea-bootstrap/virtual-desktop-host-linux/configure_dcv_host_post_reboot.sh.jinja2 @@ -0,0 +1,51 @@ +#!/bin/bash + +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +set -x +source /etc/environment +timestamp=$(date +%s) +if [[ -f ${BOOTSTRAP_DIR}/logs/configure_dcv_host_post_reboot.log ]]; then + mv ${BOOTSTRAP_DIR}/logs/configure_dcv_host_post_reboot.log ${BOOTSTRAP_DIR}/logs/configure_dcv_host_post_reboot.log.${timestamp} +fi +exec >> ${BOOTSTRAP_DIR}/logs/configure_dcv_host_post_reboot.log 2>&1 +SOURCE="${1}" + +if [[ "${SOURCE}" == "crontab" ]]; then + # clean crontab, remove current file from reboot commands + crontab -l | grep -v 'configure_dcv_host_post_reboot.sh' | crontab - +fi +echo -n "no" > ${BOOTSTRAP_DIR}/reboot_required.txt +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +source "${SCRIPT_DIR}/../common/bootstrap_common.sh" + +CONTROLLER_EVENTS_QUEUE_URL="{{ context.config.get_string('virtual-desktop-controller.events_sqs_queue_url', required=True) }}" +IDEA_SESSION_ID="{{ context.vars.idea_session_id }}" +IDEA_SESSION_OWNER="{{ context.vars.session_owner }}" + +IDEA_SERVICES_PATH="/opt/idea/.services" +IDEA_SERVICES_LOGS_PATH="${IDEA_SERVICES_PATH}/logs" +mkdir -p "${IDEA_SERVICES_LOGS_PATH}" +AWS=$(command -v aws) +echo """#!/bin/bash +timestamp=\$(date) +echo \"START\" >> ${IDEA_SERVICES_LOGS_PATH}/idea-reboot-do-not-edit-or-delete-idea-notif.log 2>&1 +echo \$(date) >> ${IDEA_SERVICES_LOGS_PATH}/idea-reboot-do-not-edit-or-delete-idea-notif.log 2>&1 +${AWS} sqs send-message --queue-url ${CONTROLLER_EVENTS_QUEUE_URL} --message-body \"{\\\"event_group_id\\\":\\\"${IDEA_SESSION_ID}\\\",\\\"event_type\\\":\\\"DCV_HOST_REBOOT_COMPLETE_EVENT\\\",\\\"detail\\\":{\\\"idea_session_id\\\":\\\"${IDEA_SESSION_ID}\\\",\\\"idea_session_owner\\\":\\\"${IDEA_SESSION_OWNER}\\\",\\\"timestamp\\\":\\\"\${timestamp}\\\"}}\" --region ${AWS_REGION} --message-group-id ${IDEA_SESSION_ID} >> ${IDEA_SERVICES_LOGS_PATH}/idea-reboot-do-not-edit-or-delete-idea-notif.log 2>&1 +echo \$(date) >> ${IDEA_SERVICES_LOGS_PATH}/idea-reboot-do-not-edit-or-delete-idea-notif.log 2>&1 +echo \"END\" >> ${IDEA_SERVICES_LOGS_PATH}/idea-reboot-do-not-edit-or-delete-idea-notif.log 2>&1 +""" > "${IDEA_SERVICES_PATH}/idea-reboot-do-not-edit-or-delete-idea-notif.sh" + +chmod +x "${IDEA_SERVICES_PATH}/idea-reboot-do-not-edit-or-delete-idea-notif.sh" + +crontab -l | grep -v "idea-reboot-do-not-edit-or-delete-idea-notif.sh" | crontab - +(crontab -l; echo "@reboot /bin/bash ${IDEA_SERVICES_PATH}/idea-reboot-do-not-edit-or-delete-idea-notif.sh") | crontab - diff --git a/source/idea/idea-bootstrap/virtual-desktop-host-linux/setup.sh.jinja2 b/source/idea/idea-bootstrap/virtual-desktop-host-linux/setup.sh.jinja2 new file mode 100644 index 00000000..82b2d04f --- /dev/null +++ b/source/idea/idea-bootstrap/virtual-desktop-host-linux/setup.sh.jinja2 @@ -0,0 +1,134 @@ +#!/bin/bash + +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +set -x + +{% set PATH = '/bin:/usr/bin:/sbin:/usr/sbin:/usr/local/bin' %} + +echo -e " +## [BEGIN] IDEA Environment Configuration - Do Not Delete +AWS_DEFAULT_REGION={{ context.aws_region }} +AWS_REGION={{ context.aws_region }} +IDEA_BASE_OS={{ context.base_os }} +IDEA_MODULE_NAME={{ context.module_name }} +IDEA_MODULE_ID={{ context.module_id }} +IDEA_MODULE_SET={{ context.module_set }} +IDEA_MODULE_VERSION={{ context.module_version }} +IDEA_CLUSTER_S3_BUCKET={{ context.cluster_s3_bucket }} +IDEA_CLUSTER_NAME={{ context.cluster_name }} +IDEA_CLUSTER_HOME={{ context.cluster_home_dir }} +IDEA_APP_DEPLOY_DIR={{ context.app_deploy_dir }} +BOOTSTRAP_DIR=/root/bootstrap +## [END] IDEA Environment Configuration + +PATH={{ PATH }} +" > /etc/environment + +source /etc/environment + +echo -n "no" > ${BOOTSTRAP_DIR}/reboot_required.txt + + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +source "${SCRIPT_DIR}/../common/bootstrap_common.sh" + +{% include '_templates/linux/idea_service_account.jinja2' %} + +{% include '_templates/linux/aws_cli.jinja2' %} + +{% include '_templates/linux/aws_ssm.jinja2' %} + +{% include '_templates/linux/nfs_utils.jinja2' %} + +{% include '_templates/linux/mount_shared_storage.jinja2' %} + +if [[ ! -f ${BOOTSTRAP_DIR}/idea_preinstalled_packages.log ]]; then + yum install -y deltarpm + yum -y upgrade + + {% include '_templates/linux/epel_repo.jinja2' %} + + {% include '_templates/linux/system_packages.jinja2' %} + + {%- include '_templates/linux/cloudwatch_agent.jinja2' %} + + {%- if context.is_metrics_provider_prometheus() %} + {%- include '_templates/linux/prometheus.jinja2' %} + {%- include '_templates/linux/prometheus_node_exporter.jinja2' %} + {%- endif %} + + {% include '_templates/linux/jq.jinja2' %} + + {% include '_templates/linux/disable_se_linux.jinja2' %} +else + log_info "Found ${BOOTSTRAP_DIR}/idea_preinstalled_packages.log... skipping package installation..." +fi + +{%- with ebs_volume_tags = [ + {'Key':'idea:ClusterName', 'Value': context.cluster_name }, + {'Key':'idea:ModuleName', 'Value': context.module_name }, + {'Key':'idea:ModuleId', 'Value': context.module_id }, + {'Key':'Name', 'Value': context.cluster_name + '/' + context.module_id + ' Root Volume' } +] %} + {% include '_templates/linux/tag_ebs_volumes.jinja2' %} +{%- endwith %} + +{%- with network_interface_tags = [ + {'Key':'idea:ClusterName', 'Value': context.cluster_name }, + {'Key':'idea:ModuleName', 'Value': context.module_name }, + {'Key':'idea:ModuleId', 'Value': context.module_id }, + {'Key':'Name', 'Value': context.cluster_name + '/' + context.module_id + ' Network Interface' } +] %} + {% include '_templates/linux/tag_network_interface.jinja2' %} +{%- endwith %} + +{% include '_templates/linux/chronyd.jinja2' %} + +{% include '_templates/linux/disable_ulimit.jinja2' %} + +{% include '_templates/linux/disable_strict_host_check.jinja2' %} + +{% include '_templates/linux/disable_motd_update.jinja2' %} + +{%- with secure_path = PATH %} + {% include '_templates/linux/sudoer_secure_path.jinja2' %} +{%- endwith %} + +{%- with messages = [ + context.module_name + ' (v'+context.module_version+'), Cluster: ' + context.cluster_name +] %} + {% include '_templates/linux/motd.jinja2' %} +{%- endwith %} + +{% include '_templates/linux/join_directoryservice.jinja2' %} + +{% if context.config.get_string('scheduler.provider') == 'openpbs' %} + {% include '_templates/linux/openpbs_client.jinja2' %} +{% endif %} + +{% if context.is_gpu_instance_type() -%} + {% include '_templates/linux/disable_nouveau_drivers.jinja2' %} +{% else %} + log_info "GPU InstanceType not detected. Skipping disabling of Nouveau Drivers..." +{% endif %} + +# Cleaning up earlier reboot notifications added by our code. +crontab -l | grep -v "idea-reboot-do-not-edit-or-delete-idea-notif.sh" | crontab - + +if [[ ! -f ${BOOTSTRAP_DIR}/idea_preinstalled_packages.log ]]; then + # This reboot is not optional. We need to reboot since we have done sudo yum upgrade. + (crontab -l; echo "@reboot /bin/bash ${SCRIPT_DIR}/configure_dcv_host.sh crontab") | crontab - + reboot +else + /bin/bash ${SCRIPT_DIR}/configure_dcv_host.sh +fi diff --git a/source/idea/idea-bootstrap/virtual-desktop-host-windows/ConfigureDCVHost.ps1.jinja2 b/source/idea/idea-bootstrap/virtual-desktop-host-windows/ConfigureDCVHost.ps1.jinja2 new file mode 100644 index 00000000..f1bff813 --- /dev/null +++ b/source/idea/idea-bootstrap/virtual-desktop-host-windows/ConfigureDCVHost.ps1.jinja2 @@ -0,0 +1,150 @@ +function Write-ToLog { + # LOG: IDEA Bootstrap Log: Get-Content C:\ProgramData\Amazon\EC2-Windows\Launch\Log\UserdataExecutionIDEA.log + # LOG: Default User Data: Get-Content C:\ProgramData\Amazon\EC2-Windows\Launch\Log\UserdataExecution.log + + Param ( + [ValidateNotNullOrEmpty()] + [Parameter(Mandatory=$true)] + [String] $Message, + [String] $LogFile = ('{0}\ProgramData\Amazon\EC2-Windows\Launch\Log\UserdataExecutionIDEA.log' -f $env:SystemDrive), + [ValidateSet('Error','Warn','Info')] + [string] $Level = 'Info' + ) + + if (-not(Test-Path -Path $LogFile)) { + $null = New-Item -Path $LogFile -ItemType File -Force + } + + + $FormattedDate = Get-Date -Format 'yyyy-MM-dd HH:mm:ss' + switch ($Level) { + 'Error' { + $LevelText = 'ERROR:' + } + 'Warn' { + $LevelText = 'WARNING:' + } + 'Info' { + $LevelText = 'INFO:' + } + } + # If Level == Error send ses message ? + "$FormattedDate $LevelText $Message" | Out-File -FilePath $LogFile -Append +} + +function Bootstrap-DCV-WindowsHost { + Param () + + $IDEASessionID = "{{ context.vars.idea_session_id }}" + $LocalUser = "{{ context.vars.session_owner }}" + $BrokerHostname = "{{ context.config.get_cluster_internal_endpoint().replace('https://', '') }}" + $InternalAlbEndpoint = "{{ context.config.get_cluster_internal_endpoint() }}" + $BrokerAgentConnectionPort = "{{ context.config.get_int('virtual-desktop-controller.dcv_broker.agent_communication_port', required=True) }}" + $IdleTimeout = "{{ context.config.get_string('virtual-desktop-controller.dcv_session.idle_timeout', required=True) }}" + $IdleTimeoutWarning = "{{ context.config.get_string('virtual-desktop-controller.dcv_session.idle_timeout_warning', required=True) }}" + + # On Windows Machines a Console Session with session-id = console IS ALWAYS CREATED by default. That we need to close. + $CurrentLocation = Get-Location + cd "C:\Program Files\NICE\DCV\Server\bin" + .\dcv.exe close-session console + cd $CurrentLocation + + Stop-Service dcvserver + Stop-Service DcvSessionManagerAgentService + + $DCVRegistryPath = "Microsoft.PowerShell.Core\Registry::\HKEY_USERS\S-1-5-18\Software\GSettings\com\nicesoftware\dcv" + [string]$IMDS_Token = Invoke-RestMethod -Headers @{"X-aws-ec2-metadata-token-ttl-seconds" = "600"} -Method PUT -Uri http://169.254.169.254/latest/api/token + $Hostname = Invoke-RestMethod -Headers @{"X-aws-ec2-metadata-token" = $IMDS_Token} -Method GET -Uri http://169.254.169.254/latest/meta-data/hostname + $InstanceId = Invoke-RestMethod -Headers @{"X-aws-ec2-metadata-token" = $IMDS_Token} -Method GET -Uri http://169.254.169.254/latest/meta-data/instance-id + + New-Item -Path "$DCVRegistryPath\" -Name "connectivity" -Force + New-ItemProperty -Path "$DCVRegistryPath\connectivity" -Name "idle-timeout" -PropertyType "DWord" -Value $IdleTimeout -Force + New-ItemProperty -Path "$DCVRegistryPath\connectivity" -Name "idle-timeout-warning" -PropertyType "DWord" -Value $IdleTimeoutWarning -Force + New-ItemProperty -Path "$DCVRegistryPath\connectivity" -Name "enable-quic-frontend" -PropertyType "DWord" -Value 1 -Force + + # Session Authentication + New-Item -Path "$DCVRegistryPath\" -Name "security" -Force + New-ItemProperty -Path "$DCVRegistryPath\security" -Name "auth-token-verifier" -PropertyType "String" -Value "$InternalAlbEndpoint`:$BrokerAgentConnectionPort/agent/validate-authentication-token" -Force + New-ItemProperty -Path "$DCVRegistryPath\security" -Name "no-tls-strict" -PropertyType "DWord" -Value 1 -Force + New-ItemProperty -Path "$DCVRegistryPath\security" -Name "os-auto-lock" -PropertyType "DWord" -Value 0 -Force + + # Disable sleep + New-Item -Path "$DCVRegistryPath\" -Name "windows" -Force + New-ItemProperty -Path "$DCVRegistryPath\windows" -Name "disable-display-sleep" -PropertyType "DWord" -Value 1 -Force + + if (Test-Path -Path "$DCVRegistryPath\session-management") { + # Disabling the original settings to start a CONSOLE session on start up + + Remove-Item -Path "$DCVRegistryPath\session-management" -Recurse -Force -Confirm:$false + # This will remove the following + #Remove-ItemProperty -Path "$DCVRegistryPath\session-management\automatic-console-session" -Name "owner" -Force + #Remove-ItemProperty -Path "$DCVRegistryPath\session-management\automatic-console-session" -Name "storage-root" -Force + #Remove-Item -Path "$DCVRegistryPath\session-management\automatic-console-session" -Force + #Remove-ItemProperty -Path "$DCVRegistryPath\session-management" -Name "create-session" -Force + #Remove-Item -Path "$DCVRegistryPath\session-management\defaults" -Force + #Remove-Item -Path "$DCVRegistryPath\session-management" -Force + } + + $IdeaTagsFolder = "C:\Program Files\NICE\DCVSessionManagerAgent\conf\tags" + if (-not(Test-Path -Path "$IdeaTagsFolder")) { + New-Item -Path $IdeaTagsFolder -ItemType Directory + } + New-Item -Path $IdeaTagsFolder -Name "idea_tags.toml" -ItemType File -Force -Value "idea_session_id=`"$IDEASessionID`"" + + # SET AGENT.CONF + $AgentConfContent = "version = '0.1' +[agent] +broker_host = '$BrokerHostname' +broker_port = $BrokerAgentConnectionPort +tls_strict = false +tags_folder = '$IdeaTagsFolder' +broker_update_interval = 15 +[log] +level = 'debug' +rotation = 'daily' +# directory = 'C:\ProgramData\NICE\DCVSessionManagerAgent\log' +" + + $AgentConfFolder = "C:\Program Files\NICE\DCVSessionManagerAgent\conf" + $CurrentTimeStamp = [DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds() + + if (Test-Path -Path "$AgentConfFolder\agent.conf") { + Move-Item -Path $AgentConfFolder -Destination "$AgentConfFolder\agent.conf.$CurrentTimeStamp" -Force + } + + New-Item -Path $AgentConfFolder -Name "agent.conf" -ItemType File -Force -Value "$AgentConfContent" + + if (-Not (Test-Path -Path "C:\session-storage\$LocalUser")) { + New-Item -Path "C:\session-storage\$LocalUser" -ItemType Directory + } + + # DCVSessionManagerAgentService is usually disabled on the system. Need to enable it. + Set-Service -Name DcvSessionManagerAgentService -StartupType Automatic + +{%- if context.config.get_string('directoryservice.provider', required=True) in ['aws_managed_activedirectory', 'activedirectory'] %} + # if ActiveDirectory, add the user to local Administrators group. + # todo: need to have a broader discussion with the group on this. this will work well as long as no shared storage is mounted automatically. + # once support for NETAPP OnTAP is added and if NETAPP OnTAP is used for /apps, this potentially introduces a data security risk. + $IdeaScriptsDirectory = "C:\IDEA\LocalScripts" + $ActiveDirectoryNetbiosName = "{{ context.config.get_string('directoryservice.ad_short_name', required=True) }}" + $PostBootstrapRebootExecuteOnceContent = " +# Add user to local Administrators group (commenting this for now, until we have decision. may be make this configurable via eVDI settings ...) +# Add-LocalGroupMember -Group Administrators -Member `"$ActiveDirectoryNetbiosName\$LocalUser`" + +# Set session owner to the one in Domain. (Does this need to be here? Need to check with @madbajaj if this can be moved outside.) +New-Item -Path `"$DCVRegistryPath`" -Name `"session-management/automatic-console-session`" -Force +New-ItemProperty -Path `"$DCVRegistryPath\session-management\automatic-console-session`" -Name `"owner`" -PropertyType `"String`" -Value `"$ActiveDirectoryNetbiosName\$LocalUser`" -Force +New-ItemProperty -Path `"$DCVRegistryPath\session-management\automatic-console-session`" -Name `"storage-root`" -PropertyType `"String`" -Value `"C:\session-storage\$LocalUser`" -Force + +# remove scheduled task after executing once +Unregister-ScheduledTask -TaskName PostBootstrapRebootExecuteOnce -Confirm:`$false +" + + New-Item -Path $IdeaScriptsDirectory -Name "PostBootstrapRebootExecuteOnce.ps1" -ItemType File -Force -Value "$PostBootstrapRebootExecuteOnceContent" + Register-ScheduledTask -TaskName PostBootstrapRebootExecuteOnce -AtStartup -User Administrator +{%- endif %} + + # Restart Services + Start-Service dcvserver + Start-Service DcvSessionManagerAgentService +} diff --git a/source/idea/idea-bootstrap/virtual-desktop-host-windows/SetUp.ps1.jinja2 b/source/idea/idea-bootstrap/virtual-desktop-host-windows/SetUp.ps1.jinja2 new file mode 100644 index 00000000..9149fbeb --- /dev/null +++ b/source/idea/idea-bootstrap/virtual-desktop-host-windows/SetUp.ps1.jinja2 @@ -0,0 +1,174 @@ +{%- if context.config.get_string('directoryservice.provider', required=True) in ['aws_managed_activedirectory', 'activedirectory'] %} +{% include '_templates/windows/join_activedirectory.jinja2' %} +{%- endif %} + +{% include '_templates/windows/mount_shared_storage.jinja2' %} + +function Write-ToLog { + # LOG: IDEA Bootstrap Log: Get-Content C:\ProgramData\Amazon\EC2-Windows\Launch\Log\UserdataExecutionIDEA.log + # LOG: Default User Data: Get-Content C:\ProgramData\Amazon\EC2-Windows\Launch\Log\UserdataExecution.log + + Param ( + [ValidateNotNullOrEmpty()] + [Parameter(Mandatory=$true)] + [String] $Message, + [String] $LogFile = ('{0}\ProgramData\Amazon\EC2-Windows\Launch\Log\UserdataExecutionIDEA.log' -f $env:SystemDrive), + [ValidateSet('Error','Warn','Info')] + [string] $Level = 'Info' + ) + + if (-not(Test-Path -Path $LogFile)) { + $null = New-Item -Path $LogFile -ItemType File -Force + } + + + $FormattedDate = Get-Date -Format 'yyyy-MM-dd HH:mm:ss' + switch ($Level) { + 'Error' { + $LevelText = 'ERROR:' + } + 'Warn' { + $LevelText = 'WARNING:' + } + 'Info' { + $LevelText = 'INFO:' + } + } + # If Level == Error send ses message ? + "$FormattedDate $LevelText $Message" | Out-File -FilePath $LogFile -Append +} + +function Setup-WindowsEC2Instance { + Param() + # Creating unique hostname to avoid NETBIOS name conflict when using an existing AMI + [string]$IMDS_Token = Invoke-RestMethod -Headers @{"X-aws-ec2-metadata-token-ttl-seconds" = "600"} -Method PUT -Uri http://169.254.169.254/latest/api/token + $InstanceId = Invoke-RestMethod -Headers @{"X-aws-ec2-metadata-token" = $IMDS_Token} -Method GET -Uri http://169.254.169.254/latest/meta-data/instance-id + # flip below lines if you want to enable/disable computer renaming + $DesktopHostname = $env:COMPUTERNAME + # $DesktopHostname = $InstanceId.substring($InstanceId.length - 15, 15).ToUpper() + if ($env:COMPUTERNAME -ne $DesktopHostname) { + Write-ToLog -Message ("Hostname detected $env:COMPUTERNAME . Renaming Computer to $DesktopHostname ...") + Rename-Computer -NewName $DesktopHostname -Force + Write-ToLog -Message ("Name has been changed, re-enabling user data as we are about to restart the system ...") + & C:\ProgramData\Amazon\EC2-Windows\Launch\Scripts\InitializeInstance.ps1 -Schedule + Write-ToLog -Message "Restarting Computer ..." + Restart-Computer -Force + } + + $IDEASessionID = "{{ context.vars.idea_session_id }}" + $LocalUser = "{{ context.vars.session_owner }}" + $VdcRequestQueueURL = "{{ context.config.get_string('virtual-desktop-controller.events_sqs_queue_url', required=True) }}" + $IdeaWebPortalURL = "{{ context.config.get_cluster_external_endpoint() }}" + + {%- if context.config.get_string('directoryservice.provider', required=True) in ['aws_managed_activedirectory', 'activedirectory'] %} + $ServerInAD = (Get-WmiObject -Class Win32_ComputerSystem).PartOfDomain + $DomainUserName = "{{ context.config.get_string('directoryservice.ad_short_name', required=True) }}\$LocalUser" + if ($ServerInAD -eq $false) + { + Start-ADAutomationAuthorization + Start-ADAutomationWaitForAuthorizationAndJoin + + New-ItemProperty -Path "HKLM:\Software\Microsoft\Windows NT\CurrentVersion\Winlogon" -Name "AutoAdminLogon" -PropertyType "DWord" -Value 1 -Force + New-ItemProperty -Path "HKLM:\Software\Microsoft\Windows NT\CurrentVersion\Winlogon" -Name "DefaultUserName" -PropertyType "String" -Value $DomainUserName -Force + } + {%- else %} + # Avoid to hit race condition after the Restart-Computer + Start-Sleep -Seconds 5 + Write-ToLog -Message "Creating new local IDEA user: $LocalUser" + $DomainUserName = $LocalUser + $LocalPasswordInput = "{{ context.utils.generate_password() }}" + $LocalPassword = ConvertTo-SecureString -String $LocalPasswordInput -AsPlainText -Force + $LocalUserExist = Get-LocalUser $LocalUser + if ($LocalUserExist) { + Set-LocalUser $LocalUser -Password $LocalPassword + } else { + New-LocalUser $LocalUser -Password $LocalPassword -PasswordNeverExpires -AccountNeverExpires + } + + $isInGroup = (Get-LocalGroupMember -Group "Administrators").Name -Contains "$env:COMPUTERNAME\$LocalUser" + Write-ToLog -Message "Checking if $LocalUser is in Administrators Group: $isInGroup" + + if ($isInGroup -eq $false) { + Write-ToLog -Message "Adding $LocalUser to the Administrators Group" + Add-LocalGroupMember -Group "Administrators" -Member $LocalUser + } + + # Set OS Properties, AutoLogin + New-ItemProperty -Path "HKLM:\Software\Microsoft\Windows NT\CurrentVersion\Winlogon" -Name "AutoAdminLogon" -PropertyType "DWord" -Value 1 -Force + New-ItemProperty -Path "HKLM:\Software\Microsoft\Windows NT\CurrentVersion\Winlogon" -Name "DefaultUserName" -PropertyType "String" -Value $LocalUser -Force + New-ItemProperty -Path "HKLM:\Software\Microsoft\Windows NT\CurrentVersion\Winlogon" -Name "DefaultPassword" -PropertyType "String" -Value $LocalPasswordInput -Force + + Write-ToLog -Message "Creating unique password for local Admin user" + $AdminPasswordInput = "{{ context.utils.short_uuid() }}" + $AdminPassword = ConvertTo-SecureString -String $AdminPasswordInput -AsPlainText -Force + Get-LocalUser -Name "Administrator" | Set-LocalUser -Password $AdminPassword + + Write-ToLog -Message "Write LOCAL users password and make it readable only by the user. This file will be overriden if an AMI is created." + $LocalPasswordFile="C:\LOCAL_ADMINS_PASSWORD.txt" + + if (Test-Path $LocalPasswordFile ) { + Write-ToLog -Message "$LocalPasswordFile exists. Removing it first." + Remove-Item $LocalPasswordFile + } + New-Item $LocalPasswordFile -Value "Local User(s) credentials for this machine $env:COMPUTERNAME only `r`n=========`r`r$LocalUser : $LocalPasswordInput `r`nAdministrator : $AdminPasswordInput" -Force + Write-ToLog -Message "Disable inheritance (required to remove all users read-only access) for $LocalPasswordFile" + $DisableInheritance = Get-Acl -Path $LocalPasswordFile + $DisableInheritance.SetAccessRuleProtection($true, $true) + Set-Acl -Path $LocalPasswordFile -AclObject $DisableInheritance + + Write-ToLog -Message "Restrict this file to local user and admin groups only" + + $DeleteAcl = (Get-Item $LocalPasswordFile).GetAccessControl('Access') + $UserAccessToRemove = $DeleteAcl.Access | ?{ $_.IsInherited -eq $false -and $_.IdentityReference -eq 'BUILTIN\Users' } + $DeleteAcl.RemoveAccessRuleAll($UserAccessToRemove) + Set-Acl -AclObject $DeleteAcl $LocalPasswordFile + + #$AdminAccessToRemove = $DeleteAcl.Access | ?{ $_.IsInherited -eq $false -and $_.IdentityReference -eq 'BUILTIN\Administrators' } + #$DeleteAcl.RemoveAccessRuleAll($AdminAccessToRemove) + #Set-Acl -AclObject $DeleteAcl $LocalPasswordFile + + Write-ToLog -Message "Restrict Viewing to local user & Admin group only for $LocalPasswordFile" + $NewAcl = Get-Acl -Path $LocalPasswordFile + $GrantAccessToLocalUser = New-Object -TypeName System.Security.AccessControl.FileSystemAccessRule("$env:COMPUTERNAME\$LocalUser", "FullControl", "Allow") + $NewAcl.SetAccessRule($GrantAccessToLocalUser) + Set-Acl -AclObject $NewAcl $LocalPasswordFile + {%- endif %} + + # Disable User Access Control (UAC)" + Set-ItemProperty "HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Policies\System" -Name "ConsentPromptBehaviorAdmin" -Value 00000000 -Force + + # "Disable Internet Explorer Enhanced Security Configuration" + $AdminKey = "HKLM:\SOFTWARE\Microsoft\Active Setup\Installed Components\{A509B1A7-37EF-4b3f-8CFC-4F3A74704073}" + $UserKey = "HKLM:\SOFTWARE\Microsoft\Active Setup\Installed Components\{A509B1A8-37EF-4b3f-8CFC-4F3A74704073}" + Set-ItemProperty -Path $AdminKey -Name "IsInstalled" -Value 0 -Force + Set-ItemProperty -Path $UserKey -Name "IsInstalled" -Value 0 -Force + + Write-ToLog -Message "Create Shortcut to IDEA web interface" + $WshShell = New-Object -comObject WScript.Shell + $IdeaShortcutAdmin = $WshShell.CreateShortcut("C:\Users\Default\Desktop\IDEA_Interface.url") + $IdeaShortcutAdmin.TargetPath = "$IdeaWebPortalURL" + $IdeaShortcutAdmin.Save() + + Write-ToLog -Message "mount any applicable shared storage file systems" + Mount-SharedStorage -DomainUserName $DomainUserName + + Write-ToLog -Message "Install and Configure NICE DCV" + Import-Module .\ConfigureDCVHost.ps1 + Bootstrap-DCV-WindowsHost + + $RebootNotificationContent = " +`$CurrentTimeStamp = [DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds() +`$RebootMessage = `"{```"event_group_id```":```"$IDEASessionID```",```"event_type```":```"DCV_HOST_REBOOT_COMPLETE_EVENT```",```"detail```":{```"idea_session_id```":```"$IDEASessionID```",```"idea_session_owner```":```"$LocalUser```",```"timestamp```":```"`$CurrentTimeStamp```"}}`" +# todo - support vpc endpoint +Send-SQSMessage -QueueUrl $VdcRequestQueueURL -MessageBody `$RebootMessage -MessageGroupId $IDEASessionID +" + $IdeaScriptsDirectory = "C:\IDEA\LocalScripts" + New-Item -Path $IdeaScriptsDirectory -Name "IdeaRebootNotification.ps1" -ItemType File -Force -Value "$RebootNotificationContent" + schtasks /create /sc onstart /tn IdeaRebootNotification /tr "powershell -File $IdeaScriptsDirectory\IdeaRebootNotification.ps1" /ru system /f + + $Message = "{{ context.vars.dcv_host_ready_message }}" + # todo - support vpc endpoint + Send-SQSMessage -QueueUrl $VdcRequestQueueURL -MessageBody $Message -MessageGroupId $IDEASessionID + + Restart-Computer -Force +} diff --git a/source/idea/idea-cli/src/ideacli/__init__.py b/source/idea/idea-cli/src/ideacli/__init__.py new file mode 100644 index 00000000..5b867905 --- /dev/null +++ b/source/idea/idea-cli/src/ideacli/__init__.py @@ -0,0 +1,90 @@ +import click +import ideacli_meta + +__version__ = ideacli_meta.__version__ + +CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help'], max_content_width=1200) + + +class CliContext(object): + + def __init__(self): + self.verbosity = 0 + self.debug = False + self.format = 'table' + + def as_dict(self): + return { + 'verbosity': self.verbosity, + 'debug': self.debug, + 'format': self.format + } + + +context = click.make_pass_decorator(CliContext, ensure=True) + + +def verbosity_option(f): + def callback(ctx, param, value): + state = ctx.ensure_object(CliContext) + state.verbosity = value + return value + + return click.option('-v', '--verbose', count=True, + expose_value=False, + help='Enables verbosity.', + callback=callback)(f) + + +def debug_option(f): + def callback(ctx, param, value): + state = ctx.ensure_object(CliContext) + state.debug = value + return value + + return click.option('--debug', + is_flag=True, + expose_value=False, + help='Enables mode', + callback=callback)(f) + + +def format_option(f): + def callback(ctx, param, value): + state = ctx.ensure_object(CliContext) + if value: + state.format = value + return value + + return click.option('-f', '--format', + expose_value=False, + help='Output Format. One of [json, table]', + callback=callback)(f) + + +def common_options(include: list = None, exclude: list = None): + def add_common_options(func): + + mandatory = { + 'format': format_option + } + + optional = { + 'debug': debug_option, + 'verbosity': verbosity_option + } + + for option in mandatory: + if exclude: + if option not in exclude: + func = mandatory[option](func) + else: + func = mandatory[option](func) + + if include: + for option in include: + func = optional[option](func) + + return func + + return add_common_options diff --git a/source/idea/idea-cli/src/ideacli/idea.py b/source/idea/idea-cli/src/ideacli/idea.py new file mode 100644 index 00000000..761ed4b8 --- /dev/null +++ b/source/idea/idea-cli/src/ideacli/idea.py @@ -0,0 +1,16 @@ +import sys +import click +import ideacli + + +@click.group(context_settings=ideacli.CONTEXT_SETTINGS) +@click.version_option(version=ideacli.__version__) +def main(): + """ + IDEA CLI + """ + pass + + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/source/idea/idea-cli/src/ideacli_meta/__init__.py b/source/idea/idea-cli/src/ideacli_meta/__init__.py new file mode 100644 index 00000000..bff439bb --- /dev/null +++ b/source/idea/idea-cli/src/ideacli_meta/__init__.py @@ -0,0 +1,4 @@ +# pkgconfig for soca-cli. no dependencies # noqa + +__version__ = '0.0.1' +__name__ = 'idea-cli' diff --git a/source/idea/idea-cli/src/setup.py b/source/idea/idea-cli/src/setup.py new file mode 100644 index 00000000..81169dcf --- /dev/null +++ b/source/idea/idea-cli/src/setup.py @@ -0,0 +1,17 @@ +from setuptools import setup, find_packages +import ideacli_meta + +setup( + name=ideacli_meta.__name__, + version=ideacli_meta.__version__, + description='CLIs', + url='https://awslabs.github.io/scale-out-computing-on-aws/', + author='Amazon', + license='Apache License, Version 2.0', + packages=find_packages(), + include_package_data=True, + entry_points=''' + [console_scripts] + soca=ideacli.soca:main + ''' +) diff --git a/source/idea/idea-cluster-manager/resources/api/api_doc.yml b/source/idea/idea-cluster-manager/resources/api/api_doc.yml new file mode 100644 index 00000000..62a511fd --- /dev/null +++ b/source/idea/idea-cluster-manager/resources/api/api_doc.yml @@ -0,0 +1,142 @@ +spec: + + title: 'IDEA / Cluster Manager API' + + description: 'All APIs served by Cluster Manager' + + tags: + - name: Accounts + description: Account Management APIs **(Elevated Access)** + - name: Auth + description: Authentication APIs **(Public + Authenticated Access)** + - name: EmailTemplates + description: Email Template APIs **(Elevated Access)** + - name: Projects + description: Project Management APIs **(Elevated Access)** + - name: ClusterSettings + description: Cluster Settings APIs **(Authenticated + Elevated Access)** + - name: FileBrowser + description: File Browser APIs **(Authenticated Access)** + + entries: + - namespace: Auth.InitiateAuth + request: + examples: + - name: Username/Password Auth + value: | + { + "header": { + "namespace": "Auth.InitiateAuth", + "request_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "version": 1 + }, + "payload": { + "auth_flow": "USER_PASSWORD_AUTH", + "username": "demouser", + "password": "MySecretPassword_123" + } + } + - name: Refresh Token Auth + value: | + { + "header": { + "namespace": "Auth.InitiateAuth", + "request_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "version": 1 + }, + "payload": { + "auth_flow": "REFRESH_TOKEN_AUTH", + "username": "demouser", + "refresh_token": "eyJjd..." + } + } + response: + examples: + - name: Username/Password Auth + value: | + { + "header": { + "namespace": "Auth.InitiateAuth", + "request_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "version": 1 + }, + "success": true, + "payload": { + "auth": { + "access_token": "eyJra...", + "id_token": "eyJra...", + "refresh_token": "eyJjd...", + "expires_in": 3600, + "token_type": "Bearer" + } + } + } + - name: Refresh Token Auth + value: | + { + "header": { + "namespace": "Auth.InitiateAuth", + "request_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "version": 1 + }, + "success": true, + "payload": { + "auth": { + "access_token": "eyJra...", + "id_token": "eyJra...", + "expires_in": 3600, + "token_type": "Bearer" + } + } + } + + - namespace: Accounts.CreateUser + request: + examples: + - name: Create User (Invitation Email) + value: | + { + "header": { + "namespace": "Accounts.CreateUser" + }, + "payload": { + "user": { + "username": "demouser", + "email": "demouser@example.com" + } + } + } + response: + examples: + - name: Create User (Invited, Not Verified) + value: | + { + "header": { + "namespace": "Accounts.CreateUser", + "request_id": "fa61c1c8-5f24-442a-91a1-45f486df0939" + }, + "success": true, + "payload": { + "user": { + "username": "demouser", + "email": "demouser@amazon.com", + "uid": 5010, + "gid": 5011, + "group_name": "demouserusergroup", + "additional_groups": [ + "demouserusergroup", + "defaultclustergroup" + ], + "login_shell": "/bin/bash", + "home_dir": "/data/home/demouser", + "sudo": false, + "status": "FORCE_CHANGE_PASSWORD", + "enabled": true, + "created_on": "2022-07-27T02:12:43.478000+00:00", + "updated_on": "2022-07-27T02:12:43.611000+00:00" + } + } + } + + + diff --git a/source/idea/idea-cluster-manager/resources/defaults/email_templates.yml b/source/idea/idea-cluster-manager/resources/defaults/email_templates.yml new file mode 100644 index 00000000..679ce829 --- /dev/null +++ b/source/idea/idea-cluster-manager/resources/defaults/email_templates.yml @@ -0,0 +1,186 @@ +templates: + - name: "scheduler.job-started" + title: "HPC Job Started" + template_type: jinja2 + subject: "[IDEA Job Started] {{job.name}} ({{job.job_id}}) has started" + body: | + Hello {{job.owner}},

+ This email is to notify you that your job {{job.job_id}} has started.
+ You will receive an email once your job is complete. + +

Job Information

+
    +
  • Job Id: {{job.job_id}}
    • +
  • Job Name: {{job.name}}
    • +
  • Job Queue: {{job.queue_type}}
    • +
+
+ This is an automated email, please do not respond. + - name: "scheduler.job-completed" + title: "HPC Job Completed" + template_type: jinja2 + subject: "[IDEA Job Completed] {{job.name}} ({{job.job_id}}) has completed" + body: | + Hello {{job.owner}},

+ This email is to notify you that your job {{job.job_id}} has completed.
+ +

Job Information

+
    +
  • Job Id: {{job.job_id}}
    • +
  • Job Name: {{job.name}}
    • +
  • Job Queue: {{job.queue_type}}
    • +
+
+ This is an automated email, please do not respond. + - name: "virtual-desktop-controller.session-provisioning" + title: "Virtual Desktop Provisioning" + template_type: jinja2 + subject: "[{{cluster_name}}] Virtual Desktop Provisioning {{session.name}}" + body: | + Hello {{session.owner}},

+ This email is to notify you that the infrastructure for your session {{session.name}} is being provisioned. +
+
+
+ This is an automated email, please do not respond. + - name: "virtual-desktop-controller.session-creating" + title: "Virtual Desktop Creating" + template_type: jinja2 + subject: "[{{cluster_name}}] Virtual Desktop Creating {{session.name}}" + body: | + Hello {{session.owner}},

+ This email is to notify you that the infrastructure provisioning for your session {{session.name}} is now complete, and your session is being created. +
+
+
+ This is an automated email, please do not respond. + - name: "virtual-desktop-controller.session-initializing" + title: "Virtual Desktop Initializing" + template_type: jinja2 + subject: "[{{cluster_name}}] Virtual Desktop Initializing {{session.name}}" + body: | + Hello {{session.owner}},

+ This email is to notify you that your Virtual Desktop Session {{session.name}} has been created and is now being initialized. +
+
+
+ This is an automated email, please do not respond. + - name: "virtual-desktop-controller.session-resuming" + title: "Virtual Desktop Resuming" + template_type: jinja2 + subject: "[{{cluster_name}}] Virtual Desktop Resuming {{session.name}}" + body: | + Hello {{session.owner}},

+ This email is to notify you that your Virtual Desktop Session {{session.name}} is resuming. +
+
+
+ This is an automated email, please do not respond. + - name: "virtual-desktop-controller.session-ready" + title: "Virtual Desktop Ready" + template_type: jinja2 + subject: "[{{cluster_name}}] Virtual Desktop Ready {{session.name}}" + body: | + Hello {{session.owner}},

+ This email is to notify you that your Virtual Desktop Session {{session.name}} is now Ready. +
+
+
+ This is an automated email, please do not respond. + - name: "virtual-desktop-controller.session-stopping" + title: "Virtual Desktop Stopping" + template_type: jinja2 + subject: "[{{cluster_name}}] Virtual Desktop Stopping {{session.name}}" + body: | + Hello {{session.owner}},

+ This email is to notify you that your Virtual Desktop Session {{session.name}} is being stopped. +
+
+
+ This is an automated email, please do not respond. + - name: "virtual-desktop-controller.session-stopped" + title: "Virtual Desktop Stopped" + template_type: jinja2 + subject: "[{{cluster_name}}] Virtual Desktop Stopped {{session.name}}" + body: | + Hello {{session.owner}},

+ This email is to notify you that your Virtual Desktop Session {{session.name}} has stopped. +
+
+
+ This is an automated email, please do not respond. + - name: "virtual-desktop-controller.session-deleting" + title: "Virtual Desktop Deleting" + template_type: jinja2 + subject: "[{{cluster_name}}] Virtual Desktop Deleting {{session.name}}" + body: | + Hello {{session.owner}},

+ This email is to notify you that your Virtual Desktop Session {{session.name}} is being deleted. +
+
+
+ This is an automated email, please do not respond. + - name: "virtual-desktop-controller.session-error" + title: "Virtual Desktop Error" + template_type: jinja2 + subject: "[{{cluster_name}}] Virtual Desktop Error {{session.name}}" + body: | + Hello {{session.owner}},

+ This email is to notify you that your Virtual Desktop Session {{session.name}} has entered an error state. +
+ Please reach out to your system administrator for further help. +
+
+
+ This is an automated email, please do not respond. + - name: "virtual-desktop-controller.session-deleted" + title: "Virtual Desktop Deleted" + template_type: jinja2 + subject: "[{{cluster_name}}] Virtual Desktop Deleted {{session.name}}" + body: | + Hello {{session.owner}},

+ This email is to notify you that your Virtual Desktop Session {{session.name}} has been deleted. +
+
+
+ This is an automated email, please do not respond. + - name: "virtual-desktop-controller.session-shared" + title: "Virtual Desktop Shared" + template_type: jinja2 + subject: "[{{cluster_name}}] Virtual Desktop shared with you" + body: | + Hello {{session_permission.actor_name}},

+ This email is to notify you that Virtual Desktop {{session_permission.idea_session_name}} owned by {{session_permission.idea_session_owner}} has been shared with you. +
You can access the session via the following url: +
{{url}} +

Access to this virtual desktop expires on {{formatted_expiry_date}} +
+
+ This is an automated email, please do not respond. + - name: "virtual-desktop-controller.session-permission-updated" + title: "Virtual Desktop Permission Updated" + template_type: jinja2 + subject: "[{{cluster_name}}] Permissions updated for your shared session: {{session_permission.idea_session_name}} owned by {{session_permission.idea_session_owner}}" + body: | + Hello {{session_permission.actor_name}},

+ Permissions for the shared session: {{session_permission.idea_session_name}} owned by {{session_permission.idea_session_owner}} has been updated +
You can access the session using the below url: +
{{url}} +

Access to this virtual desktop expires on {{formatted_expiry_date}} +
+
+
+ This is an automated email, please do not respond. + - name: "virtual-desktop-controller.session-permission-expired" + title: "Virtual Desktop Permission Expired" + template_type: jinja2 + subject: "[{{cluster_name}}] Access for shared session: {{session_permission.idea_session_name}} owned by {{session_permission.idea_session_owner}} has expired" + body: | + Hello {{session_permission.actor_name}},

+ Access to the shared session: {{session_permission.idea_session_name}} owned by {{session_permission.idea_session_owner}} has expired +
You will no longer be able to access this shared virtual desktop session. +

Contact the session owner: {{session_permission.idea_session_owner}} if you want to continue accessing this session. +
+
+
+ This is an automated email, please do not respond. diff --git a/source/idea/idea-cluster-manager/src/ideaclustermanager/__init__.py b/source/idea/idea-cluster-manager/src/ideaclustermanager/__init__.py new file mode 100644 index 00000000..43144480 --- /dev/null +++ b/source/idea/idea-cluster-manager/src/ideaclustermanager/__init__.py @@ -0,0 +1,19 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +import ideaclustermanager_meta + +__name__ = ideaclustermanager_meta.__name__ +__version__ = ideaclustermanager_meta.__version__ + +from ideaclustermanager.app.app_context import ClusterManagerAppContext + +AppContext = ClusterManagerAppContext diff --git a/source/idea/idea-cluster-manager/src/ideaclustermanager/app/__init__.py b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/__init__.py new file mode 100644 index 00000000..6d8d18a3 --- /dev/null +++ b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/__init__.py @@ -0,0 +1,10 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. diff --git a/source/idea/idea-cluster-manager/src/ideaclustermanager/app/accounts/__init__.py b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/accounts/__init__.py new file mode 100644 index 00000000..6d8d18a3 --- /dev/null +++ b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/accounts/__init__.py @@ -0,0 +1,10 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. diff --git a/source/idea/idea-cluster-manager/src/ideaclustermanager/app/accounts/account_tasks.py b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/accounts/account_tasks.py new file mode 100644 index 00000000..0c54817e --- /dev/null +++ b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/accounts/account_tasks.py @@ -0,0 +1,219 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +__all__ = ( + 'SyncUserInDirectoryServiceTask', + 'CreateUserHomeDirectoryTask', + 'SyncGroupInDirectoryServiceTask', + 'SyncPasswordInDirectoryServiceTask', + 'GroupMembershipUpdatedTask' +) + +from ideasdk.utils import Utils +from ideadatamodel import exceptions, errorcodes + +import ideaclustermanager +from ideaclustermanager.app.tasks.base_task import BaseTask +from ideaclustermanager.app.accounts.user_home_directory import UserHomeDirectory + +from typing import Dict +import os +import pathlib +import shutil + + +class SyncGroupInDirectoryServiceTask(BaseTask): + + def __init__(self, context: ideaclustermanager.AppContext): + self.context = context + self.logger = context.logger(self.get_name()) + + def get_name(self) -> str: + return 'accounts.sync-group' + + def invoke(self, payload: Dict): + group_name = payload['group_name'] + group = self.context.accounts.group_dao.get_group(group_name) + + if self.context.ldap_client.is_readonly(): + self.logger.info(f'Read-only Directory service - NOOP for {self.get_name()}') + return + + delete_group = False + if group is not None: + if group['enabled']: + self.logger.info(f'sync group: {group_name} directory service ...') + self.context.ldap_client.sync_group( + group_name=group['group_name'], + gid=group['gid'] + ) + else: + delete_group = True + else: + delete_group = True + + if delete_group: + self.logger.info(f'deleting group: {group_name} from directory service ...') + if self.context.ldap_client.is_existing_group(group_name): + self.context.ldap_client.delete_group(group_name) + + +class GroupMembershipUpdatedTask(BaseTask): + + def __init__(self, context: ideaclustermanager.AppContext): + self.context = context + self.logger = context.logger(self.get_name()) + + def get_name(self) -> str: + return 'accounts.group-membership-updated' + + def invoke(self, payload: Dict): + group_name = payload['group_name'] + username = payload['username'] + operation = payload['operation'] + group = self.context.accounts.group_dao.get_group(group_name) + ds_readonly = self.context.ldap_client.is_readonly() + + if group is None: + raise exceptions.soca_exception( + error_code=errorcodes.AUTH_GROUP_NOT_FOUND, + message=f'group not found: {group_name}' + ) + if group['enabled']: + if operation == 'add': + + if ds_readonly: + self.logger.info(f'add member: {username} to group: {group_name} in READ-ONLY directory service ...') + else: + self.logger.info(f'add member: {username} to group: {group_name} in directory service ...') + self.context.ldap_client.add_user_to_group([username], group_name) + + # update membership in user projects (DynamoDB) + self.logger.info(f'add member: {username} to group: {group_name} - DAO ...') + self.context.projects.user_projects_dao.group_member_added( + group_name=group_name, + username=username + ) + + else: + + if ds_readonly: + self.logger.info(f'NOOP - remove member: {username} from group: {group_name} in readonly directory service ...') + else: + self.logger.info(f'remove member: {username} from group: {group_name} in directory service ...') + self.context.ldap_client.remove_user_from_group([username], group_name) + + # update membership in user projects + self.context.projects.user_projects_dao.group_member_removed( + group_name=group_name, + username=username + ) + + +class SyncUserInDirectoryServiceTask(BaseTask): + + def __init__(self, context: ideaclustermanager.AppContext): + self.context = context + self.logger = context.logger(self.get_name()) + + def get_name(self) -> str: + return 'accounts.sync-user' + + def invoke(self, payload: Dict): + username = payload['username'] + user = self.context.accounts.user_dao.get_user(username) + group_name = user['group_name'] + enabled = user['enabled'] + sudo = Utils.get_value_as_bool('sudo', user, False) + readonly = self.context.ldap_client.is_readonly() + + if readonly: + self.logger.info(f'sync user: Read-only Directory service - sync {username} NOOP ...') + self.logger.info(f'sync user: returning ...') + return + + if enabled: + self.logger.info(f'sync user: {username} TO directory service ...') + self.context.ldap_client.sync_group( + group_name=user['group_name'], + gid=user['gid'] + ) + self.context.ldap_client.sync_user( + uid=user['uid'], + gid=user['gid'], + username=user['username'], + email=user['email'], + login_shell=user['login_shell'], + home_dir=user['home_dir'] + ) + + if sudo: + if not self.context.ldap_client.is_sudo_user(username): + self.context.ldap_client.add_sudo_user(username) + else: + if self.context.ldap_client.is_sudo_user(username): + self.context.ldap_client.remove_sudo_user(username) + + else: + self.logger.info(f'deleting user: {username} from directory service ...') + if self.context.ldap_client.is_existing_group(group_name): + self.context.ldap_client.delete_group(group_name) + if self.context.ldap_client.is_existing_user(username): + self.context.ldap_client.delete_user(username) + if self.context.ldap_client.is_sudo_user(username): + self.context.ldap_client.remove_sudo_user(username) + + +class SyncPasswordInDirectoryServiceTask(BaseTask): + + def __init__(self, context: ideaclustermanager.AppContext): + self.context = context + self.logger = context.logger(self.get_name()) + + def get_name(self) -> str: + return 'accounts.sync-password' + + def invoke(self, payload: Dict): + username = payload['username'] + password_file = payload['password_file'] + + self.logger.info(f'sync password for user: {username} in directory service') + + with open(password_file, 'r') as f: + password = f.read() + + self.context.ldap_client.change_password(username, password) + + # clean up password file + password_file_path = pathlib.Path(password_file) + ds_automation_dir = self.context.config().get_string('directoryservice.automation_dir', required=True) + if str(password_file_path.parent.parent) == ds_automation_dir: + shutil.rmtree(password_file_path.parent) + else: + os.remove(password_file) + + +class CreateUserHomeDirectoryTask(BaseTask): + + def __init__(self, context: ideaclustermanager.AppContext): + self.context = context + self.logger = context.logger(self.get_name()) + + def get_name(self) -> str: + return 'accounts.create-home-directory' + + def invoke(self, payload: Dict): + username = payload['username'] + user = self.context.accounts.get_user(username) + UserHomeDirectory( + context=self.context, + user=user + ).initialize() diff --git a/source/idea/idea-cluster-manager/src/ideaclustermanager/app/accounts/accounts_service.py b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/accounts/accounts_service.py new file mode 100644 index 00000000..c3a706ce --- /dev/null +++ b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/accounts/accounts_service.py @@ -0,0 +1,1287 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. +from ideasdk.client.evdi_client import EvdiClient +from ideasdk.context import SocaContext +from ideadatamodel.auth import ( + User, + Group, + InitiateAuthRequest, + InitiateAuthResult, + RespondToAuthChallengeRequest, + RespondToAuthChallengeResult, + ListUsersRequest, + ListUsersResult, + ListGroupsRequest, + ListGroupsResult, + ListUsersInGroupRequest, + ListUsersInGroupResult +) +from ideadatamodel import exceptions, errorcodes, constants +from ideasdk.utils import Utils, GroupNameHelper +from ideasdk.auth import TokenService + +from ideaclustermanager.app.accounts.ldapclient.abstract_ldap_client import AbstractLDAPClient +from ideaclustermanager.app.accounts.cognito_user_pool import CognitoUserPool +from ideaclustermanager.app.accounts import auth_constants +from ideaclustermanager.app.accounts.auth_utils import AuthUtils +from ideaclustermanager.app.accounts.db.group_dao import GroupDAO +from ideaclustermanager.app.accounts.db.user_dao import UserDAO +from ideaclustermanager.app.accounts.db.sequence_config_dao import SequenceConfigDAO +from ideaclustermanager.app.accounts.db.group_members_dao import GroupMembersDAO +from ideaclustermanager.app.accounts.db.single_sign_on_state_dao import SingleSignOnStateDAO + +from ideaclustermanager.app.tasks.task_manager import TaskManager + +from typing import Optional, List +import os +import re +import tempfile +import time + + +def nonce() -> str: + return Utils.short_uuid() + + +class AccountsService: + """ + Account Management Service + + Integrates with OpenLDAP/AD, Cognito User Pools and exposes functionality around: + 1. User Management + 2. Groups + 3. User Onboarding + 4. Single Sign-On + + The service is primarily invoked via AuthAPI and AccountsAPI + """ + + def __init__(self, context: SocaContext, + ldap_client: Optional[AbstractLDAPClient], + user_pool: Optional[CognitoUserPool], + evdi_client: Optional[EvdiClient], + task_manager: Optional[TaskManager], + token_service: Optional[TokenService]): + + self.context = context + self.logger = context.logger('accounts-service') + self.user_pool = user_pool + self.ldap_client = ldap_client + self.evdi_client = evdi_client + self.task_manager = task_manager + self.token_service = token_service + + self.group_name_helper = GroupNameHelper(context) + self.user_dao = UserDAO(context, user_pool=user_pool) + self.group_dao = GroupDAO(context) + self.sequence_config_dao = SequenceConfigDAO(context) + self.group_members_dao = GroupMembersDAO(context, self.user_dao) + self.sso_state_dao = SingleSignOnStateDAO(context) + + self.user_dao.initialize() + self.group_dao.initialize() + self.sequence_config_dao.initialize() + self.group_members_dao.initialize() + self.sso_state_dao.initialize() + + self.ds_automation_dir = self.context.config().get_string('directoryservice.automation_dir', required=True) + + def is_cluster_administrator(self, username: str) -> bool: + cluster_administrator = self.context.config().get_string('cluster.administrator_username', required=True) + return username == cluster_administrator + + def is_sso_enabled(self) -> bool: + return self.context.config().get_bool('identity-provider.cognito.sso_enabled', False) + + # user group management methods + + def get_group(self, group_name: str) -> Optional[Group]: + if Utils.is_empty(group_name): + raise exceptions.invalid_params('group_name is required') + + group = self.group_dao.get_group(group_name) + + if group is None: + raise exceptions.SocaException( + error_code=errorcodes.AUTH_GROUP_NOT_FOUND, + message=f'Group not found: {group_name}' + ) + + return self.group_dao.convert_from_db(group) + + def create_group(self, group: Group) -> Group: + """ + create a new group + :param group: Group + """ + + ds_readonly = self.ldap_client.is_readonly() + + group_name = group.name + if Utils.is_empty(group_name): + raise exceptions.invalid_params('group.name is required') + if Utils.is_empty(group.group_type): + raise exceptions.invalid_params('group.group_type is required') + if group.group_type not in constants.ALL_GROUP_TYPES: + raise exceptions.invalid_params(f'invalid group type: {group.group_type}. must be one of: {constants.ALL_GROUP_TYPES}') + + ds_name = group.ds_name + if Utils.is_empty(ds_name) and ds_readonly: + raise exceptions.invalid_params(f'group.ds_name is required when using Read-Only Directory Service: {constants.DIRECTORYSERVICE_ACTIVE_DIRECTORY}') + + db_existing_group = self.group_dao.get_group(group_name) + if db_existing_group is not None: + raise exceptions.invalid_params(f'group: {group_name} already exists') + + # Make sure we have a supplied GID when RO DS + gid = group.gid + if gid is None: + if ds_readonly: + raise exceptions.invalid_params(f'group.gid is required when using Read-Only Directory Service: {constants.DIRECTORYSERVICE_ACTIVE_DIRECTORY}') + else: + gid = self.sequence_config_dao.next_gid() + group.gid = gid + + group.enabled = True + + db_group = self.group_dao.convert_to_db(group) + + self.group_dao.create_group(db_group) + + self.task_manager.send( + task_name='accounts.sync-group', + payload={ + 'group_name': group_name + }, + message_group_id=group_name, + message_dedupe_id=f'{group_name}.create-group.{nonce()}' + ) + + return self.group_dao.convert_from_db(db_group) + + def modify_group(self, group: Group) -> Group: + """ + modify an existing group + :param group: Group + """ + + group_name = group.name + if Utils.is_empty(group_name): + raise exceptions.invalid_params('group_name is required') + + db_group = self.group_dao.get_group(group_name) + if db_group is None: + raise exceptions.soca_exception( + error_code=errorcodes.AUTH_GROUP_NOT_FOUND, + message=f'group not found for group name: {group_name}' + ) + + if not db_group['enabled']: + raise exceptions.soca_exception( + error_code=errorcodes.AUTH_GROUP_IS_DISABLED, + message=f'cannot modify a disabled group' + ) + + # do not support modification of group name or GID + # only title updates are supported + update_group = { + 'group_name': db_group['group_name'], + 'title': db_group['title'] + } + + # only update db, sync with DS not required. + updated_group = self.group_dao.update_group(update_group) + + return self.group_dao.convert_from_db(updated_group) + + def delete_group(self, group_name: str, force: bool = False): + """ + delete a group + :param group_name: + :param force: force delete a group even if group has existing members + :return: + """ + + if Utils.is_empty(group_name): + raise exceptions.invalid_params('group_name is required') + + group = self.group_dao.get_group(group_name) + + if group is None: + raise exceptions.soca_exception( + error_code=errorcodes.AUTH_GROUP_NOT_FOUND, + message=f'group: {group_name} not found.' + ) + + if self.group_members_dao.has_users_in_group(group_name): + if force: + usernames = self.group_members_dao.get_usernames_in_group(group_name=group_name) + self.remove_users_from_group(usernames=usernames, group_name=group_name, force=force) + else: + raise exceptions.soca_exception( + error_code=errorcodes.AUTH_INVALID_OPERATION, + message='group must be empty before it can be deleted.' + ) + + self.group_dao.delete_group(group_name=group_name) + + self.task_manager.send( + task_name='accounts.sync-group', + payload={ + 'group_name': group_name + }, + message_group_id=group_name, + message_dedupe_id=f'{group_name}.update-group.{nonce()}' + ) + + def enable_group(self, group_name: str): + if Utils.is_empty(group_name): + raise exceptions.invalid_params('group_name is required') + + group = self.group_dao.get_group(group_name) + + if group is None: + raise exceptions.soca_exception( + error_code=errorcodes.AUTH_GROUP_NOT_FOUND, + message=f'group: {group_name} not found.' + ) + + self.group_dao.update_group({ + 'group_name': group_name, + 'enabled': True + }) + + self.task_manager.send( + task_name='accounts.sync-group', + payload={ + 'group_name': group_name + }, + message_group_id=group_name, + message_dedupe_id=f'{group_name}.update-group.{nonce()}' + ) + + def disable_group(self, group_name: str): + if Utils.is_empty(group_name): + raise exceptions.invalid_params('group_name is required') + + group = self.group_dao.get_group(group_name) + + if group is None: + raise exceptions.soca_exception( + error_code=errorcodes.AUTH_GROUP_NOT_FOUND, + message=f'group: {group_name} not found.' + ) + + self.group_dao.update_group({ + 'group_name': group_name, + 'enabled': False + }) + + self.task_manager.send( + task_name='accounts.sync-group', + payload={ + 'group_name': group_name + }, + message_group_id=group_name, + message_dedupe_id=f'{group_name}.update-group.{nonce()}' + ) + + def list_groups(self, request: ListGroupsRequest) -> ListGroupsResult: + return self.group_dao.list_groups(request) + + def list_users_in_group(self, request: ListUsersInGroupRequest) -> ListUsersInGroupResult: + return self.group_members_dao.list_users_in_group(request) + + def add_users_to_group(self, usernames: List[str], group_name: str): + if Utils.is_empty(usernames): + raise exceptions.invalid_params('usernames is required') + + if Utils.is_empty(group_name): + raise exceptions.invalid_params('group_name is required') + + group = self.group_dao.get_group(group_name) + if group is None: + raise exceptions.soca_exception( + error_code=errorcodes.AUTH_GROUP_NOT_FOUND, + message=f'group: {group_name} not found.' + ) + if not group['enabled']: + raise exceptions.soca_exception( + error_code=errorcodes.AUTH_GROUP_IS_DISABLED, + message=f'cannot add users to a disabled user group' + ) + + users = [] + for username in usernames: + user = self.user_dao.get_user(username) + if user is None: + raise exceptions.soca_exception(error_code=errorcodes.AUTH_USER_NOT_FOUND, message=f'user not found: {username}') + if not user['enabled']: + raise exceptions.soca_exception(error_code=errorcodes.AUTH_USER_IS_DISABLED, message=f'user is disabled: {username}') + users.append(user) + + for user in users: + + username = user['username'] + additional_groups = Utils.get_value_as_list('additional_groups', user, []) + + if group_name not in additional_groups: + additional_groups.append(group_name) + + self.user_dao.update_user({ + 'username': username, + 'additional_groups': additional_groups + }) + self.group_members_dao.create_membership(group_name, username) + + if group['group_type'] not in (constants.GROUP_TYPE_USER, constants.GROUP_TYPE_PROJECT): + self.user_pool.admin_add_user_to_group(username=username, group_name=group_name) + + self.task_manager.send( + task_name='accounts.group-membership-updated', + payload={ + 'group_name': group_name, + 'username': username, + 'operation': 'add' + }, + message_group_id=username + ) + + def remove_user_from_groups(self, username: str, group_names: List[str]): + """ + remove a user from multiple groups. + useful for operations such as delete user. + :param username: + :param group_names: + :return: + """ + if Utils.is_empty(username): + raise exceptions.invalid_params('usernames is required') + if Utils.is_empty(group_names): + raise exceptions.invalid_params('group_names is required') + + self.logger.info(f'removing user: {username} from groups: {group_names}') + + # dedupe and sanitize + groups_to_remove = [] + for group_name in group_names: + if group_name in groups_to_remove: + continue + groups_to_remove.append(group_name) + + user = self.user_dao.get_user(username) + if user is None: + raise exceptions.soca_exception(error_code=errorcodes.AUTH_USER_NOT_FOUND, message=f'user not found: {username}') + if not user['enabled']: + raise exceptions.soca_exception(error_code=errorcodes.AUTH_USER_IS_DISABLED, message=f'user is disabled: {username}') + + additional_groups = Utils.get_value_as_list('additional_groups', user, []) + + for group_name in groups_to_remove: + group = self.group_dao.get_group(group_name) + if group is None: + self.logger.warning(f'user: {username} cannot be removed from group: {group_name}. group not found') + continue + if not group['enabled']: + self.logger.warning(f'user: {username} cannot be removed from group: {group_name}. group is disabled') + continue + + if group_name in additional_groups: + additional_groups.remove(group_name) + + self.group_members_dao.delete_membership(group_name, username) + + if group['group_type'] not in (constants.GROUP_TYPE_USER, constants.GROUP_TYPE_PROJECT): + self.user_pool.admin_remove_user_from_group(username=username, group_name=group_name) + + self.task_manager.send( + task_name='accounts.group-membership-updated', + payload={ + 'group_name': group_name, + 'username': username, + 'operation': 'remove' + }, + message_group_id=username + ) + + self.user_dao.update_user({ + 'username': username, + 'additional_groups': additional_groups + }) + + def remove_users_from_group(self, usernames: List[str], group_name: str, force: bool = False): + """ + remove multiple users from a group + useful for bulk operations from front end + :param usernames: + :param group_name: + :param force: force delete even if group is disabled. if user is not found, skip user. + :return: + """ + if Utils.is_empty(usernames): + raise exceptions.invalid_params('usernames is required') + + if Utils.is_empty(group_name): + raise exceptions.invalid_params('group_name is required') + + group = self.group_dao.get_group(group_name) + if group is None: + raise exceptions.soca_exception( + error_code=errorcodes.AUTH_GROUP_NOT_FOUND, + message=f'group: {group_name} not found.' + ) + if not group['enabled'] and not force: + raise exceptions.soca_exception( + error_code=errorcodes.AUTH_GROUP_IS_DISABLED, + message=f'cannot remove users from a disabled user group' + ) + + users = [] + for username in usernames: + user = self.user_dao.get_user(username) + if user is None: + if force: + continue + raise exceptions.soca_exception(error_code=errorcodes.AUTH_USER_NOT_FOUND, message=f'user not found: {username}') + if not user['enabled'] and not force: + raise exceptions.soca_exception(error_code=errorcodes.AUTH_USER_IS_DISABLED, message=f'user is disabled: {username}') + users.append(user) + + for user in users: + + username = user['username'] + additional_groups = Utils.get_value_as_list('additional_groups', user, []) + if group_name in additional_groups: + additional_groups.remove(group_name) + + self.user_dao.update_user({ + 'username': username, + 'additional_groups': additional_groups + }) + self.group_members_dao.delete_membership(group_name, username) + + if group['group_type'] not in (constants.GROUP_TYPE_USER, constants.GROUP_TYPE_PROJECT): + self.user_pool.admin_remove_user_from_group(username=username, group_name=group_name) + + self.task_manager.send( + task_name='accounts.group-membership-updated', + payload={ + 'group_name': group_name, + 'username': username, + 'operation': 'remove' + }, + message_group_id=username + ) + + # sudo user management methods + + def add_sudo_user(self, username: str): + + if Utils.is_empty(username): + raise exceptions.invalid_params('username is required') + + user = self.user_dao.get_user(username) + if user is None: + raise exceptions.SocaException( + error_code=errorcodes.AUTH_USER_NOT_FOUND, + message=f'User not found: {username}' + ) + + if Utils.get_value_as_bool('sudo', user, False): + return + + self.user_dao.update_user({ + 'username': username, + 'sudo': True + }) + + self.user_pool.admin_add_sudo_user(username) + + self.task_manager.send( + task_name='accounts.sync-user', + payload={ + 'username': username + }, + message_group_id=username, + message_dedupe_id=f'{username}.add-sudo-user.{nonce()}' + ) + + def remove_sudo_user(self, username: str): + + if Utils.is_empty(username): + raise exceptions.invalid_params('username is required') + + if self.is_cluster_administrator(username): + raise AuthUtils.invalid_operation(f'Admin rights cannot be revoked from IDEA Cluster Administrator: {username}.') + + user = self.user_dao.get_user(username) + if user is None: + raise exceptions.SocaException( + error_code=errorcodes.AUTH_USER_NOT_FOUND, + message=f'User not found: {username}' + ) + + if not Utils.get_value_as_bool('sudo', user, False): + return + + self.user_dao.update_user({ + 'username': username, + 'sudo': False + }) + + self.user_pool.admin_remove_sudo_user(username) + + self.task_manager.send( + task_name='accounts.sync-user', + payload={ + 'username': username + }, + message_group_id=username, + message_dedupe_id=f'{username}.remove-sudo-user.{nonce()}' + ) + + # user management methods + + def get_user(self, username: str) -> User: + + username = AuthUtils.sanitize_username(username) + if Utils.is_empty(username): + raise exceptions.invalid_params('username is required.') + + user = self.user_dao.get_user(username) + if user is None: + raise exceptions.SocaException( + error_code=errorcodes.AUTH_USER_NOT_FOUND, + message=f'User not found: {username}' + ) + + return self.user_dao.convert_from_db(user) + + def create_user(self, user: User, email_verified: bool = False) -> User: + """ + create a new user + """ + + # username + username = user.username + username = AuthUtils.sanitize_username(username) + if Utils.is_empty(username): + raise exceptions.invalid_params('user.username is required') + if not re.match(auth_constants.USERNAME_REGEX, username): + raise exceptions.invalid_params(f'user.username must match regex: {auth_constants.USERNAME_REGEX}') + AuthUtils.check_allowed_username(username) + + existing_user = self.user_dao.get_user(username) + if existing_user is not None: + raise exceptions.soca_exception( + error_code=errorcodes.AUTH_USER_ALREADY_EXISTS, + message=f'username: {username} already exists.' + ) + + # email + email = user.email + email = AuthUtils.sanitize_email(email) + + # password + password = user.password + if email_verified: + if Utils.is_empty(password): + raise exceptions.invalid_params('user.password is required') + else: + self.logger.debug(f'create_user() - setting password to random value') + password = Utils.generate_password(8,2,2,2,2) + #password = None + + # login_shell + login_shell = user.login_shell + if Utils.is_empty(login_shell): + login_shell = auth_constants.DEFAULT_LOGIN_SHELL + + # home_dir + home_dir = os.path.join(auth_constants.USER_HOME_DIR_BASE, username) + + # note: no validations on uid / gid if existing uid/gid is provided. + # ensuring uid/gid uniqueness is administrator's responsibility. + + # uid + uid = user.uid + if uid is None: + uid = self.sequence_config_dao.next_uid() + + # gid, group name + if self.ldap_client.is_readonly(): + #group_name = self.group_name_helper.get_default_project_group() + group_name = self.group_name_helper.get_user_group(username) + else: + group_name = self.group_name_helper.get_user_group(username) + + gid = user.gid + if gid is None: + gid = self.sequence_config_dao.next_gid() + + # sudo + sudo = Utils.get_as_bool(user.sudo, False) + + self.logger.info(f'creating Cognito user pool entry: {username} , uid: {uid}, gid: {gid}, Group Name: {group_name}, Email: {email} , email_verified: {email_verified}') + + self.user_pool.admin_create_user( + username=username, + email=email, + password=password, + email_verified=email_verified + ) + if self.is_sso_enabled(): + self.logger.debug(f'Performing IDP Link for {username} / {email}') + self.user_pool.admin_link_idp_for_user(username, email) + if sudo: + self.logger.debug(f'Performing SUDO for {username}') + self.user_pool.admin_add_sudo_user(username) + + # additional groups + additional_groups = Utils.get_as_list(user.additional_groups, []) + self.logger.debug(f'Additional groups for {username}: {additional_groups}') + if group_name in additional_groups: + additional_groups.remove(group_name) + + # We may have an existing group that we are getting mapped to + existing_group = self.group_dao.get_group(group_name=group_name) + if existing_group is None: + self.logger.debug(f'Creating new group for {username}: GroupName: {group_name}') + self.group_dao.create_group({ + 'title': f'{username}\'s Personal User Group', + 'group_name': group_name, + 'gid': gid, + 'group_type': constants.GROUP_TYPE_USER, + 'ref': username, + 'enabled': True + }) + else: + self.logger.debug(f'No need to create group for username: {username}: Group ({group_name}) already exists.') + + + created_user = self.user_dao.create_user({ + 'username': username, + 'email': email, + 'uid': uid, + 'gid': gid, + 'group_name': group_name, + 'additional_groups': additional_groups, + 'login_shell': login_shell, + 'home_dir': home_dir, + 'sudo': sudo, + 'enabled': True + }) + self.task_manager.send( + task_name='accounts.sync-user', + payload={ + 'username': username + }, + message_group_id=username, + message_dedupe_id=f'{username}.create-user.{nonce()}' + ) + self.logger.debug(f'Adding user {username} to specific group {group_name}') + self.add_users_to_group([username], group_name) + + self.logger.debug(f'Adding user {username} to default group: {self.group_name_helper.get_default_project_group()}') + self.add_users_to_group([username], self.group_name_helper.get_default_project_group()) + + for additional_group in additional_groups: + self.logger.debug(f'Adding username {username} to additional group: {additional_group}') + self.add_users_to_group([username], additional_group) + + self.task_manager.send( + task_name='accounts.create-home-directory', + payload={ + 'username': username + }, + message_group_id=username + ) + + if Utils.is_not_empty(password): + self.change_ldap_password(username, password) + + return self.user_dao.convert_from_db(created_user) + + def modify_user(self, user: User, email_verified: bool = False) -> User: + """ + Modify User + + Only ``email`` updates are supported at the moment. + + :param user: + :param email_verified: + :return: + """ + + username = user.username + username = AuthUtils.sanitize_username(username) + + existing_user = self.user_dao.get_user(username) + if existing_user is None: + raise exceptions.soca_exception(error_code=errorcodes.AUTH_USER_NOT_FOUND, + message=f'User not found: {username}') + if not existing_user['enabled']: + raise exceptions.soca_exception(error_code=errorcodes.AUTH_USER_IS_DISABLED, + message=f'User is disabled and cannot be modified.') + + user_updates = { + 'username': username + } + + if Utils.is_not_empty(user.email): + new_email = AuthUtils.sanitize_email(user.email) + existing_email = Utils.get_value_as_string('email', existing_user) + if existing_email != new_email: + user_updates['email'] = new_email + self.user_pool.admin_update_email(username, new_email, email_verified=email_verified) + if self.is_sso_enabled(): + self.user_pool.admin_link_idp_for_user(username, new_email) + + if Utils.is_not_empty(user.login_shell): + user_updates['login_shell'] = user.login_shell + + if user.sudo is not None: + user_updates['sudo'] = Utils.get_as_bool(user.sudo, False) + + if user.uid is not None: + user_updates['uid'] = user.uid + + if user.gid is not None: + user_updates['gid'] = user.gid + + updated_user = self.user_dao.update_user(user_updates) + + self.task_manager.send( + task_name='accounts.sync-user', + payload={ + 'username': username + }, + message_group_id=username, + message_dedupe_id=f'{username}.modify-user.{nonce()}' + ) + + return self.user_dao.convert_from_db(updated_user) + + def enable_user(self, username: str): + if Utils.is_empty(username): + raise exceptions.invalid_params('username is required') + + existing_user = self.user_dao.get_user(username) + if existing_user is None: + raise exceptions.soca_exception(error_code=errorcodes.AUTH_USER_NOT_FOUND, + message=f'User not found: {username}') + + is_enabled = Utils.get_value_as_bool('enabled', existing_user, False) + if is_enabled: + return + + self.user_pool.admin_enable_user(username) + self.user_dao.update_user({ + 'username': username, + 'enabled': True + }) + self.group_dao.update_group({ + 'group_name': existing_user['group_name'], + 'enabled': True + }) + + self.task_manager.send( + task_name='accounts.sync-user', + payload={ + 'username': username + }, + message_group_id=username, + message_dedupe_id=f'{username}.enable-user.{nonce()}' + ) + + def disable_user(self, username: str): + if Utils.is_empty(username): + raise exceptions.invalid_params('username is required') + + if self.is_cluster_administrator(username): + raise AuthUtils.invalid_operation(f'Cluster Administrator cannot be disabled.') + + existing_user = self.user_dao.get_user(username) + if existing_user is None: + raise exceptions.soca_exception(error_code=errorcodes.AUTH_USER_NOT_FOUND, + message=f'User not found: {username}') + + is_enabled = Utils.get_value_as_bool('enabled', existing_user, False) + if not is_enabled: + return + + self.user_pool.admin_disable_user(username) + self.user_dao.update_user({ + 'username': username, + 'enabled': False + }) + self.group_dao.update_group({ + 'group_name': existing_user['group_name'], + 'enabled': False + }) + + self.task_manager.send( + task_name='accounts.sync-user', + payload={ + 'username': username + }, + message_group_id=username, + message_dedupe_id=f'{username}.disable-user.{nonce()}' + ) + self.evdi_client.publish_user_disabled_event(username=username) + + def delete_user(self, username: str): + log_tag = f'(DeleteUser: {username})' + + if self.is_cluster_administrator(username): + raise AuthUtils.invalid_operation(f'Cluster Administrator cannot be deleted.') + + user = self.user_dao.get_user(username=username) + if user is None: + self.logger.info(f'{log_tag} user not found. skip.') + return + + # remove user from all group memberships + group_name = Utils.get_value_as_string('group_name', user) + + groups = Utils.get_value_as_list('additional_groups', user, []) + if group_name in groups: + groups.remove(group_name) + if Utils.is_not_empty(groups): + self.logger.info(f'{log_tag} clean-up group memberships') + try: + self.remove_user_from_groups(username=username, group_names=groups) + except exceptions.SocaException as e: + if e.error_code == errorcodes.AUTH_USER_IS_DISABLED: + pass + else: + raise e + + # disable user from db, user pool and delete from directory service + self.logger.info(f'{log_tag} disabling user') + self.disable_user(username=username) + + # delete user in user pool + self.logger.info(f'{log_tag} delete user from user pool') + self.user_pool.admin_delete_user(username=username) + + # delete user's group from db + self.logger.info(f'{log_tag} deleting group: {group_name}') + self.delete_group(group_name=group_name, force=True) + + # delete user from db + self.logger.info(f'{log_tag} delete user in ddb') + self.user_dao.delete_user(username=username) + + def reset_password(self, username: str): + username = AuthUtils.sanitize_username(username) + if Utils.is_empty(username): + raise exceptions.invalid_params('username is required') + + # trigger reset password email + self.user_pool.admin_reset_password(username) + + def list_users(self, request: ListUsersRequest) -> ListUsersResult: + return self.user_dao.list_users(request) + + def change_ldap_password(self, username: str, password: str): + """ + Change password for given username in ldap or ad + :param username: + :param password: + :return: + """ + if not Utils.is_dir(self.ds_automation_dir): + os.makedirs(self.ds_automation_dir) + os.chmod(self.ds_automation_dir, 0o700) + + temp_dir = tempfile.mkdtemp(dir=self.ds_automation_dir) + password_file = os.path.join(temp_dir, 'password.txt') + with open(password_file, 'w') as f: + f.write(password) + + is_user_synced = self.ldap_client.is_existing_user(username) + if not is_user_synced: + self.task_manager.send( + task_name='accounts.sync-user', + payload={ + 'username': username + }, + message_group_id=username, + message_dedupe_id=f'{username}.change-password.{nonce()}' + ) + + self.task_manager.send( + task_name='accounts.sync-password', + payload={ + 'username': username, + 'password_file': password_file + }, + message_group_id=username + ) + + def change_password(self, access_token: str, username: str, old_password: str, new_password: str): + """ + change password for given username in user pool and ldap + this method expects an access token from an already logged-in user, who is trying to change their password. + :return: + """ + + # change password in user pool before changing in ldap + self.user_pool.change_password( + username=username, + access_token=access_token, + old_password=old_password, + new_password=new_password + ) + + # sync password change in ldap + self.change_ldap_password( + username=username, + password=new_password + ) + + # public API methods for user onboarding, login, forgot password flows. + + def initiate_auth(self, request: InitiateAuthRequest) -> InitiateAuthResult: + + auth_flow = request.auth_flow + if Utils.is_empty(auth_flow): + raise exceptions.invalid_params('auth_flow is required.') + + if auth_flow == 'USER_PASSWORD_AUTH': + + username = request.username + if Utils.is_empty(username): + raise exceptions.invalid_params('username is required.') + + password = request.password + if Utils.is_empty(password): + raise exceptions.invalid_params('password is required.') + + return self.user_pool.initiate_username_password_auth(request) + + elif auth_flow == 'REFRESH_TOKEN_AUTH': + + username = request.username + if Utils.is_empty(username): + raise exceptions.invalid_params('username is required.') + + refresh_token = request.refresh_token + if Utils.is_empty(refresh_token): + raise exceptions.invalid_params('refresh_token is required.') + + return self.user_pool.initiate_refresh_token_auth(username, refresh_token) + + elif auth_flow == 'SSO_AUTH': + + if not self.is_sso_enabled(): + raise exceptions.unauthorized_access() + + authorization_code = request.authorization_code + if Utils.is_empty(authorization_code): + raise exceptions.invalid_params('authorization_code is required.') + + db_sso_state = self.sso_state_dao.get_sso_state(authorization_code) + if db_sso_state is None: + raise exceptions.unauthorized_access() + + auth_result = self.sso_state_dao.convert_from_db(db_sso_state) + + self.sso_state_dao.delete_sso_state(authorization_code) + + return InitiateAuthResult( + auth=auth_result + ) + + elif auth_flow == 'SSO_REFRESH_TOKEN_AUTH': + + if not self.is_sso_enabled(): + raise exceptions.unauthorized_access() + + username = request.username + if Utils.is_empty(username): + raise exceptions.invalid_params('username is required.') + + refresh_token = request.refresh_token + if Utils.is_empty(refresh_token): + raise exceptions.invalid_params('refresh_token is required.') + + return self.user_pool.initiate_refresh_token_auth(username, refresh_token, sso=True) + + def respond_to_auth_challenge(self, request: RespondToAuthChallengeRequest) -> RespondToAuthChallengeResult: + + if Utils.is_empty(request.username): + raise exceptions.invalid_params('username is required.') + + challenge_name = request.challenge_name + if Utils.is_empty(challenge_name): + raise exceptions.invalid_params('challenge_name is required.') + if challenge_name != 'NEW_PASSWORD_REQUIRED': + raise exceptions.invalid_params(f'challenge_name: {challenge_name} is not supported.') + + if Utils.is_empty(request.session): + raise exceptions.invalid_params('session is required.') + + if Utils.is_empty(request.new_password): + raise exceptions.invalid_params('new_password is required.') + + self.logger.debug(f'respond_to_auth_challenge() - Request: {request}') + + result = self.user_pool.respond_to_auth_challenge(request) + + if result.auth is not None: + self.change_ldap_password(request.username, request.new_password) + + return result + + def forgot_password(self, username: str): + """ + invoke user pool's forgot password API + introduce mandatory timing delays to ensure valid / invalid user invocations are processed in approximately the same time + """ + if Utils.is_empty(username): + raise exceptions.invalid_params('username is required') + + wait_time_seconds = 5 + start = Utils.current_time_ms() + self.user_pool.forgot_password(username) + end = Utils.current_time_ms() + total_secs = (end - start) / 1000 + + if total_secs <= wait_time_seconds: + time.sleep(wait_time_seconds - total_secs) + + def confirm_forgot_password(self, username: str, password: str, confirmation_code: str): + if Utils.is_empty(username): + raise exceptions.invalid_params('username is required') + if Utils.is_empty(password): + raise exceptions.invalid_params('password is required') + if Utils.is_empty(confirmation_code): + raise exceptions.invalid_params('confirmation_code is required') + + # update user-pool first to verify confirmation code. + self.user_pool.confirm_forgot_password(username, password, confirmation_code) + + # sync with LDAP/AD + self.change_ldap_password(username, password) + + def sign_out(self, refresh_token: str, sso_auth: bool): + """ + revokes the refresh token issued by InitiateAuth API. + """ + self.token_service.revoke_authorization( + refresh_token=refresh_token, + sso_auth=sso_auth + ) + + def global_sign_out(self, username: str): + """ + Signs out a user from all devices. + It also invalidates all refresh tokens that Amazon Cognito has issued to a user. + The user's current access and ID tokens remain valid until they expire. + """ + + if Utils.is_empty(username): + raise exceptions.invalid_params('username is required') + + self.user_pool.admin_global_sign_out(username=username) + + + def _get_ds_group_name(self, groupname: str) -> str: + ds_group_name = self.context.config().get_string(f'directoryservice.group_mapping.{groupname}', default=groupname) + return ds_group_name + + def _get_gid_from_existing_ldap_group(self, groupname: str): + existing_gid = None + existing_group_from_ds = self.ldap_client.get_group(group_name=groupname) + self.logger.debug(f'READ-ONLY DS lookup results for group {groupname}: {existing_group_from_ds}') + + if existing_group_from_ds is None: + raise exceptions.soca_exception( + error_code=errorcodes.GENERAL_ERROR, + message=f'Unable to Resolve a required IDEA group from directory services: IDEA group {groupname}' + ) + + existing_gid = Utils.get_value_as_int('gid', existing_group_from_ds, default=None) + + if existing_gid is None: + raise exceptions.soca_exception( + error_code=errorcodes.GENERAL_ERROR, + message=f'Found group without POSIX gidNumber attribute - UNABLE to use this group. Update group with gidNumber attribute within Directory Service: {groupname}' + ) + + return existing_gid + + def create_defaults(self): + ds_provider = self.context.config().get_string('directoryservice.provider', required=True) + ds_readonly = self.ldap_client.is_readonly() + + self.logger.info(f'Creating defaults for Directory Service provider ({ds_provider}). Read-Only status: {ds_readonly}') + + # create default project group + #if ds_readonly: + # # todo - these may be DNs ? Clean them up to just names? + # default_project_group_name = self.context.config().get_string('directoryservice.group_mapping.default-project-group', required=True) + #else: + + default_project_group_name = self.group_name_helper.get_default_project_group() + default_project_group_name_ds = None + if ds_readonly: + default_project_group_name_ds = self._get_ds_group_name(groupname=default_project_group_name) + + self.logger.info(f'Default group name: ({default_project_group_name}) Directory Service group name: ({default_project_group_name_ds})') + + default_group = self.group_dao.get_group(default_project_group_name) + if default_group is None: + self.logger.info(f'Default group not found: {default_project_group_name}') + + gid = None + if ds_readonly: + gid = self._get_gid_from_existing_ldap_group(groupname=default_project_group_name_ds) + + self.logger.info(f'creating default group: ({default_project_group_name}) / Directory Service: ({default_project_group_name_ds})') + if gid is not None: + self.logger.info(f'Group will use Directory Service Discovered GID: {gid}') + else: + self.logger.info(f'Group will use AUTO GID') + + self.create_group( + group=Group( + title=f'IDEA Default Project Group (DS: {default_project_group_name_ds})', + name=default_project_group_name, + ds_name=default_project_group_name_ds, + gid=gid, + group_type=constants.GROUP_TYPE_PROJECT, + ref=constants.DEFAULT_PROJECT + ) + ) + + # cluster admin user + admin_username = self.context.config().get_string('cluster.administrator_username', required=True) + admin_email = self.context.config().get_string('cluster.administrator_email', required=True) + admin_user = self.user_dao.get_user(username=admin_username) + if admin_user is None: + self.logger.info(f'creating cluster admin user: {admin_username}') + self.create_user( + user=User( + username=admin_username, + email=admin_email, + sudo=True + ), + email_verified=False + ) + + # create managers group + cluster_managers_group_name = self.group_name_helper.get_cluster_managers_group() + cluster_managers_group_name_ds = None + if ds_readonly: + cluster_managers_group_name_ds = self._get_ds_group_name(groupname=cluster_managers_group_name) + + cluster_managers_group = self.group_dao.get_group(cluster_managers_group_name) + + if cluster_managers_group is None: + self.logger.info(f'Cluster-managers group not found: {cluster_managers_group_name} / DS Translation: {cluster_managers_group_name_ds}') + + gid = None + if ds_readonly: + gid = self._get_gid_from_existing_ldap_group(groupname=cluster_managers_group_name_ds) + + self.logger.info(f'creating managers group: {cluster_managers_group_name} from DS group name {cluster_managers_group_name_ds}') + if gid is not None: + self.logger.info(f'Group will use Directory Service Discovered GID: {gid}') + else: + self.logger.info(f'Group will use AUTO GID') + + self.create_group( + group=Group( + title='Managers (cluster administrators without sudo access)', + name=cluster_managers_group_name, + ds_name=cluster_managers_group_name_ds, + gid=gid, + group_type=constants.GROUP_TYPE_CLUSTER + ) + ) + + # for all "app" modules in the cluster, create the module users and module administrators group to enable fine-grained access + # if an application module is added at a later point in time, a cluster-manager restart should fix the issue. + # ideally, an 'ideactl initialize-defaults' command is warranted to address this scenario and will be taken up in a future release. + modules = self.context.get_cluster_modules() + for module in modules: + if module['type'] != constants.MODULE_TYPE_APP: + continue + module_id = module['module_id'] + module_name = module['name'] + + module_administrators_group_name = self.group_name_helper.get_module_administrators_group(module_id=module_id) + module_administrators_group_name_ds = None + + if ds_readonly: + module_administrators_group_name_ds = self._get_ds_group_name(groupname=module_administrators_group_name) + + self.logger.info(f'Looking up module group info for ModuleID: ({module_id}) Name: ({module_name}) / IDEA: {module_administrators_group_name} DS translation: {module_administrators_group_name_ds}') + + module_administrators_group = self.group_dao.get_group(module_administrators_group_name) + + if module_administrators_group is None: + self.logger.info(f'Module administrators group not found: {module_administrators_group_name}') + + gid = None + if ds_readonly: + gid = self._get_gid_from_existing_ldap_group(groupname=module_administrators_group_name_ds) + + if gid is not None: + self.logger.info(f'Group will use Directory Service Discovered GID: {gid}') + else: + self.logger.info(f'Group will use AUTO GID') + + self.logger.info(f'creating module administrators group: {module_administrators_group_name}') + self.create_group( + group=Group( + title=f'Administrators for Module: {module_name}, ModuleId: {module_id}, DS: {module_administrators_group_name_ds}', + name=module_administrators_group_name, + ds_name=module_administrators_group_name_ds, + gid=gid, + group_type=constants.GROUP_TYPE_MODULE, + ref=module_id + ) + ) + + + module_users_group_name = self.group_name_helper.get_module_users_group(module_id=module_id) + module_users_group_name_ds = None + + if ds_readonly: + module_users_group_name_ds = self._get_ds_group_name(groupname=module_users_group_name) + + module_users_group = self.group_dao.get_group(module_users_group_name) + if module_users_group is None: + self.logger.info(f'creating module administrators group: {module_users_group_name} / DS Translation: {module_users_group_name_ds}') + + gid = None + if ds_readonly: + gid = self._get_gid_from_existing_ldap_group(groupname=module_users_group_name_ds) + + if gid is not None: + self.logger.info(f'Group will use Directory Service Discovered GID: {gid}') + else: + self.logger.info(f'Group will use AUTO GID') + + self.create_group( + group=Group( + title=f'Users for Module: {module_name}, ModuleId: {module_id}', + name=module_users_group_name, + ds_name=module_users_group_name_ds, + gid=gid, + group_type=constants.GROUP_TYPE_MODULE, + ref=module_id + ) + ) diff --git a/source/idea/idea-cluster-manager/src/ideaclustermanager/app/accounts/ad_automation_agent.py b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/accounts/ad_automation_agent.py new file mode 100644 index 00000000..a73a29b6 --- /dev/null +++ b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/accounts/ad_automation_agent.py @@ -0,0 +1,259 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + + +from ideasdk.service import SocaService +from ideasdk.context import SocaContext +from ideasdk.utils import Utils +from ideadatamodel import exceptions, errorcodes + +from ideaclustermanager.app.accounts.ldapclient.active_directory_client import ActiveDirectoryClient +from ideaclustermanager.app.accounts.db.ad_automation_dao import ADAutomationDAO +from ideaclustermanager.app.accounts.helpers.preset_computer_helper import PresetComputeHelper + +from typing import Dict +from threading import Thread, Event +import arrow +import ldap # noqa +import time +import random + +DEFAULT_MAX_MESSAGES = 1 +DEFAULT_WAIT_INTERVAL_SECONDS = 20 + +AD_RESET_PASSWORD_LOCK_KEY = 'activedirectory.reset-password' + + +class ADAutomationAgent(SocaService): + """ + IDEA - Active Directory Automation Agent + + * Manages password rotation of AD Admin credentials + * Manages automation for creating preset-computers using adcli for cluster nodes + + Developer Note: + * Expect admins to heavily customize this implementation + """ + + def __init__(self, context: SocaContext, ldap_client: ActiveDirectoryClient): + super().__init__(context) + self.context = context + self.logger = context.logger('ad-automation-agent') + self.ldap_client = ldap_client + + self.ad_automation_sqs_queue_url = self.context.config().get_string('directoryservice.ad_automation.sqs_queue_url', required=True) + + self.ad_automation_dao = ADAutomationDAO(context=self.context) + self.ad_automation_dao.initialize() + + self._stop_event = Event() + self._automation_thread = Thread(name='ad-automation-thread', target=self.automation_loop) + + def is_password_expired(self) -> bool: + + root_username = self.ldap_client.ldap_root_username + user = self.ldap_client.get_user(root_username, trace=False) + if user is None: + raise exceptions.soca_exception( + error_code=errorcodes.GENERAL_ERROR, + message='ActiveDirectory admin user not found' + ) + + password_last_set = Utils.get_value_as_int('password_last_set', user, 0) + password_max_age = Utils.get_value_as_int('password_max_age', user) + + password_expiry_date = None + if password_last_set != 0: + password_last_set_date = arrow.get(password_last_set) + password_expiry_date = password_last_set_date.shift(days=password_max_age) + + if password_expiry_date is not None: + + now = arrow.utcnow() + + # check for expiration 2 days before expiration + now_minus_2days = now.shift(days=-2) + + if now_minus_2days < password_expiry_date: + return False + + return True + + def check_and_reset_admin_password(self): + """ + check and reset password for AD admin user + """ + + if not self.is_password_expired(): + return + + try: + + # sleep for a random interval to prevent race condition + time.sleep(random.randint(1, 30)) + + self.logger.info('acquiring lock to reset ds credentials ...') + self.context.distributed_lock().acquire(key=AD_RESET_PASSWORD_LOCK_KEY) + + self.logger.info('reset ds credentials lock acquired.') + + # check again, where other node may have acquired the lock and already performed the update. + if not self.is_password_expired(): + self.logger.info('ds credentials already reset. re-sync ds credentials from secrets manager ...') + self.ldap_client.refresh_root_username_password() + return + + username = self.ldap_client.ldap_root_username + new_password = Utils.generate_password(16, 2, 2, 2, 2) + + # change password in Managed AD (calls ds.reset_password) + success = False + current_retry = 0 + max_retries = 4 + backoff_in_seconds = 5 + while not success and current_retry < max_retries: + try: + self.ldap_client.change_password(username, new_password) + + # wait for password change to sync across domain controllers + time.sleep(30) + + success = True + except exceptions.SocaException as e: + if e.error_code == errorcodes.AUTH_USER_NOT_FOUND: + self.logger.info(f'reset password for {username}: waiting for user to be synced with AD domain controllers ...') + interval = Utils.get_retry_backoff_interval(current_retry, max_retries, backoff_in_seconds) + current_retry += 1 + time.sleep(interval) + else: + raise e + + # update root password secret + self.ldap_client.update_root_password(new_password) + self.logger.info('ds credentials updated.') + + finally: + + self.context.distributed_lock().release(key=AD_RESET_PASSWORD_LOCK_KEY) + self.logger.info('reset ds credentials lock released.') + + def automation_loop(self): + + while not self._stop_event.is_set(): + + admin_user_ok = False + + try: + + enable_root_password_reset = self.context.config().get_bool('directoryservice.ad_automation.enable_root_password_reset', default=False) + if enable_root_password_reset: + self.check_and_reset_admin_password() + admin_user_ok = True + + visibility_timeout = self.context.config().get_int('directoryservice.ad_automation.sqs_visibility_timeout_seconds', default=30) + result = self.context.aws().sqs().receive_message( + QueueUrl=self.ad_automation_sqs_queue_url, + MaxNumberOfMessages=DEFAULT_MAX_MESSAGES, + WaitTimeSeconds=DEFAULT_WAIT_INTERVAL_SECONDS, + AttributeNames=['SenderId'], + VisibilityTimeout=visibility_timeout + ) + + sqs_messages = Utils.get_value_as_list('Messages', result, []) + + delete_messages = [] + + def add_to_delete(sqs_message_: Dict): + delete_messages.append({ + 'Id': sqs_message_['MessageId'], + 'ReceiptHandle': sqs_message_['ReceiptHandle'] + }) + + for sqs_message in sqs_messages: + try: + + message_body = Utils.get_value_as_string('Body', sqs_message) + + request = Utils.from_json(message_body) + header = Utils.get_value_as_dict('header', request) + namespace = Utils.get_value_as_string('namespace', header) + + # todo - constants for the namespaces supported + ad_automation_namespaces = {'ADAutomation.PresetComputer', 'ADAutomation.UpdateComputerDescription', 'ADAutomation.DeleteComputer'} + if namespace not in ad_automation_namespaces: + self.logger.error(f'Invalid request: namespace {namespace} not supported. Supported namespaces: {ad_automation_namespaces}') + add_to_delete(sqs_message) + continue + + attributes = Utils.get_value_as_dict('Attributes', sqs_message, {}) + sender_id = Utils.get_value_as_string('SenderId', attributes) + + self.logger.info(f'Processing AD automation event: {namespace}') + + try: + + if namespace == 'ADAutomation.PresetComputer': + PresetComputeHelper( + context=self.context, + ldap_client=self.ldap_client, + ad_automation_dao=self.ad_automation_dao, + sender_id=sender_id, + request=request + ).invoke() + elif namespace == 'ADAutomation.DeleteComputer': + self.logger.debug(f'Processing AD automation event: DeleteComputer') + elif namespace == 'ADAutomation.UpdateComputerDescription': + self.logger.debug(f'Processing AD automation event: UpdateComputerDescription') + + # no exception, AD automation succeeded. delete from queue. + add_to_delete(sqs_message) + + except exceptions.SocaException as e: + if e.error_code == errorcodes.AD_AUTOMATION_PRESET_COMPUTER_FAILED: + self.logger.error(f'{e}') + add_to_delete(sqs_message) + elif e.error_code == errorcodes.AD_AUTOMATION_PRESET_COMPUTER_RETRY: + # do nothing. request will be retried after visibility timeout interval. + self.logger.warning(f'{e} - request will be retried in {visibility_timeout} seconds') + else: + # retry on any unhandled exception. + raise e + + except Exception as e: + self.logger.exception(f'failed to process sqs message: {e}. payload: {Utils.to_json(sqs_message)}. processing will be retried in {visibility_timeout} seconds ...') + + if len(delete_messages) > 0: + delete_message_result = self.context.aws().sqs().delete_message_batch( + QueueUrl=self.ad_automation_sqs_queue_url, + Entries=delete_messages + ) + failed = Utils.get_value_as_list('Failed', delete_message_result, []) + if len(failed) > 0: + self.logger.error(f'Failed to delete AD automation entries. This could result in an infinite loop. Consider increasing the directoryservice.ad_automation.sqs_visibility_timeout_seconds. failed messages: {failed}') + + except KeyboardInterrupt: + pass + except Exception as e: + self.logger.exception(f'ad automation failure: {e}') + finally: + # wait only if admin user is not OK and keep retrying. + # if admin user and/or credentials are ok, wait will be handled by sqs receive message long polling + if not admin_user_ok: + self._stop_event.wait(DEFAULT_WAIT_INTERVAL_SECONDS) + + def start(self): + self.logger.info('starting ad automation agent ...') + self._automation_thread.start() + + def stop(self): + self.logger.info('stopping ad automation agent ...') + self._stop_event.set() + self._automation_thread.join() diff --git a/source/idea/idea-cluster-manager/src/ideaclustermanager/app/accounts/auth_constants.py b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/accounts/auth_constants.py new file mode 100644 index 00000000..7edd87ac --- /dev/null +++ b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/accounts/auth_constants.py @@ -0,0 +1,14 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +DEFAULT_LOGIN_SHELL = '/bin/bash' +USER_HOME_DIR_BASE = '/data/home' +USERNAME_REGEX = r'^(?=.{3,20}$)(?![_.])(?!.*[_.]{2})[a-z0-9._]+(? str: + if Utils.is_empty(username): + raise exceptions.invalid_params('username is required') + return username.strip().lower() + + @staticmethod + def sanitize_email(email: str) -> str: + if Utils.is_empty(email): + raise exceptions.invalid_params('email is required') + + email = email.strip().lower() + + if not validators.email(email): + raise exceptions.invalid_params(f'invalid email: {email}') + + return email + @staticmethod + def sanitize_sub(sub: str) -> str: + if Utils.is_empty(sub): + raise exceptions.invalid_params('sub is required') + + sub = sub.strip().lower() + + if not validators.uuid(sub): + raise exceptions.invalid_params(f'invalid sub(expected UUID): {sub}') + + return sub + + @staticmethod + def invalid_operation(message: str) -> exceptions.SocaException: + return exceptions.SocaException( + error_code=errorcodes.AUTH_INVALID_OPERATION, + message=message + ) + + @staticmethod + def check_allowed_username(username: str): + if username.strip().lower() in ('root', + 'admin', + 'administrator', + constants.IDEA_SERVICE_ACCOUNT, + 'ec2-user', + 'centos', + 'ssm-user'): + raise exceptions.invalid_params(f'invalid username: {username}. Change username to prevent conflicts with local or directory system users.') diff --git a/source/idea/idea-cluster-manager/src/ideaclustermanager/app/accounts/cognito_user_pool.py b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/accounts/cognito_user_pool.py new file mode 100644 index 00000000..6849276a --- /dev/null +++ b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/accounts/cognito_user_pool.py @@ -0,0 +1,515 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +from ideadatamodel import ( + exceptions, + errorcodes, + constants, + SocaBaseModel, + InitiateAuthRequest, + InitiateAuthResult, + RespondToAuthChallengeRequest, + RespondToAuthChallengeResult, + AuthResult, + CognitoUser +) +from ideasdk.utils import Utils +from ideasdk.context import SocaContext + +from typing import Optional, Dict +import botocore.exceptions +import hmac +import hashlib +import base64 + + +class CognitoUserPoolOptions(SocaBaseModel): + user_pool_id: Optional[str] + admin_group_name: Optional[str] + client_id: Optional[str] + client_secret: Optional[str] + + +class CognitoUserPool: + + def __init__(self, context: SocaContext, options: CognitoUserPoolOptions): + self._context = context + self._logger = context.logger('user-pool') + + if Utils.is_empty(options.user_pool_id): + raise exceptions.invalid_params('options.user_pool_id is required') + if Utils.is_empty(options.admin_group_name): + raise exceptions.invalid_params('options.admin_group_name is required') + if Utils.is_empty(options.client_id): + raise exceptions.invalid_params('options.client_id is required') + if Utils.is_empty(options.client_secret): + raise exceptions.invalid_params('options.client_secret is required') + + self.options = options + + self._sso_client_id: Optional[str] = None + self._sso_client_secret: Optional[str] = None + + @property + def user_pool_id(self) -> str: + return self.options.user_pool_id + + @property + def admin_group_name(self) -> str: + return self.options.admin_group_name + + def get_client_id(self, sso: bool = False) -> str: + if sso: + return self._context.config().get_string('identity-provider.cognito.sso_client_id', required=True) + else: + return self.options.client_id + + def get_client_secret(self, sso: bool = False) -> str: + if sso: + if self._sso_client_secret is None: + self._sso_client_secret = self._context.config().get_secret('identity-provider.cognito.sso_client_secret', required=True) + return self._sso_client_secret + else: + return self.options.client_secret + + def is_activedirectory(self) -> bool: + provider = self._context.config().get_string('directoryservice.provider', required=True) + return provider in (constants.DIRECTORYSERVICE_ACTIVE_DIRECTORY, + constants.DIRECTORYSERVICE_AWS_MANAGED_ACTIVE_DIRECTORY) + + def get_secret_hash(self, username: str, sso: bool = False): + + client_id = self.get_client_id(sso) + client_secret = self.get_client_secret(sso) + + # A keyed-hash message authentication code (HMAC) calculated using + # the secret key of a user pool client and username plus the client + # ID in the message. + dig = hmac.new( + key=Utils.to_bytes(client_secret), + msg=Utils.to_bytes(f'{username}{client_id}'), + digestmod=hashlib.sha256 + ).digest() + return base64.b64encode(dig).decode() + + @staticmethod + def build_user_cache_key(username: str) -> str: + return f'cognito.user-pool.user.{username}' + + @staticmethod + def build_auth_result(cognito_auth_result: Dict) -> AuthResult: + access_token = Utils.get_value_as_string('AccessToken', cognito_auth_result) + expires_in = Utils.get_value_as_int('ExpiresIn', cognito_auth_result) + token_type = Utils.get_value_as_string('TokenType', cognito_auth_result) + refresh_token = Utils.get_value_as_string('RefreshToken', cognito_auth_result) + id_token = Utils.get_value_as_string('IdToken', cognito_auth_result) + return AuthResult( + access_token=access_token, + expires_in=expires_in, + token_type=token_type, + refresh_token=refresh_token, + id_token=id_token + ) + + def admin_get_user(self, username: str) -> Optional[CognitoUser]: + if Utils.is_empty(username): + raise exceptions.invalid_params('username is required') + + cache_key = self.build_user_cache_key(username) + + user = self._context.cache().short_term().get(cache_key) + if user is not None: + return user + + try: + result = self._context.aws().cognito_idp().admin_get_user( + UserPoolId=self.user_pool_id, + Username=username + ) + except botocore.exceptions.ClientError as e: + if e.response['Error']['Code'] == 'UserNotFoundException': + return None + else: + raise e + + user = CognitoUser(**result) + self._context.cache().short_term().set(cache_key, user) + return user + + def admin_create_user(self, username: str, email: str, password: Optional[str] = None, email_verified=False): + + if email_verified and Utils.is_empty(password): + raise exceptions.invalid_params('password is required when email_verified=True') + + create_user_params = { + 'UserPoolId': self.user_pool_id, + 'Username': username, + 'UserAttributes': [ + { + 'Name': 'email', + 'Value': email + }, + { + 'Name': 'email_verified', + 'Value': str(email_verified) + }, + { + 'Name': 'custom:cluster_name', + 'Value': str(self._context.cluster_name()) + }, + { + 'Name': 'custom:aws_region', + 'Value': str(self._context.aws().aws_region()) + } + ], + 'DesiredDeliveryMediums': ['EMAIL'] + } + + if email_verified: + create_user_params['MessageAction'] = 'SUPPRESS' + else: + if Utils.is_not_empty(password): + create_user_params['TemporaryPassword'] = password + + create_result = self._context.aws().cognito_idp().admin_create_user(**create_user_params) + + created_user = Utils.get_value_as_dict('User', create_result) + status = Utils.get_value_as_string('UserStatus', created_user) + self._logger.info(f'CreateUser: {username}, Status: {status}, EmailVerified: {email_verified}') + + if email_verified: + self.admin_set_password(username, password, permanent=True) + + def admin_delete_user(self, username: str): + if Utils.is_empty(username): + raise exceptions.invalid_params('username is required') + + try: + self._context.aws().cognito_idp().admin_delete_user( + UserPoolId=self.user_pool_id, + Username=username + ) + except botocore.exceptions.ClientError as e: + if e.response['Error']['Code'] == 'UserNotFoundException': + pass + else: + raise e + self._context.cache().short_term().delete(self.build_user_cache_key(username)) + + def admin_enable_user(self, username: str): + if Utils.is_empty(username): + raise exceptions.invalid_params('username is required') + self._context.aws().cognito_idp().admin_enable_user( + UserPoolId=self.user_pool_id, + Username=username + ) + self._context.cache().short_term().delete(self.build_user_cache_key(username)) + + def admin_disable_user(self, username: str): + if Utils.is_empty(username): + raise exceptions.invalid_params('username is required') + + self._context.aws().cognito_idp().admin_disable_user( + UserPoolId=self.user_pool_id, + Username=username + ) + self._context.cache().short_term().delete(self.build_user_cache_key(username)) + + def admin_add_sudo_user(self, username: str): + self.admin_add_user_to_group(username=username, group_name=self.admin_group_name) + + def admin_add_user_to_group(self, username: str, group_name: str): + if Utils.is_empty(username): + raise exceptions.invalid_params('username is required') + if Utils.is_empty(group_name): + raise exceptions.invalid_params('username is required') + + self._context.aws().cognito_idp().admin_add_user_to_group( + UserPoolId=self.user_pool_id, + Username=username, + GroupName=group_name + ) + + def admin_link_idp_for_user(self, username: str, email: str): + + if Utils.is_empty(username): + raise exceptions.invalid_params('username is required') + + cluster_administrator = self._context.config().get_string('cluster.administrator_username', required=True) + if username in cluster_administrator or username.startswith('clusteradmin'): + self._logger.info(f'system administration user found: {username}. skip linking with IDP.') + return + + provider_name = self._context.config().get_string('identity-provider.cognito.sso_idp_provider_name', required=True) + provider_type = self._context.config().get_string('identity-provider.cognito.sso_idp_provider_type', required=True) + if provider_type == constants.SSO_IDP_PROVIDER_OIDC: + provider_email_attribute = 'email' + else: + provider_email_attribute = self._context.config().get_string('identity-provider.cognito.sso_idp_provider_email_attribute', required=True) + + self._context.aws().cognito_idp().admin_link_provider_for_user( + UserPoolId=self.user_pool_id, + DestinationUser={ + 'ProviderName': 'Cognito', + 'ProviderAttributeName': 'cognito:username', + 'ProviderAttributeValue': username + }, + SourceUser={ + 'ProviderName': provider_name, + 'ProviderAttributeName': provider_email_attribute, + 'ProviderAttributeValue': email + } + ) + + def admin_remove_sudo_user(self, username: str): + self.admin_remove_user_from_group(username=username, group_name=self.admin_group_name) + + def admin_remove_user_from_group(self, username: str, group_name: str): + if Utils.is_empty(username): + raise exceptions.invalid_params('username is required') + if Utils.is_empty(group_name): + raise exceptions.invalid_params('group_name is required') + + self._context.aws().cognito_idp().admin_remove_user_from_group( + UserPoolId=self.user_pool_id, + Username=username, + GroupName=group_name + ) + + def password_updated(self, username: str): + if not self.is_activedirectory(): + return + + password_max_age = self._context.config().get_int('directoryservice.password_max_age', required=True) + self._context.aws().cognito_idp().admin_update_user_attributes( + UserPoolId=self.user_pool_id, + Username=username, + UserAttributes=[ + { + 'Name': 'custom:password_last_set', + 'Value': str(Utils.current_time_ms()) + }, + { + 'Name': 'custom:password_max_age', + 'Value': str(password_max_age) + } + ] + ) + + def admin_set_password(self, username: str, password: str, permanent: bool = False): + self._context.aws().cognito_idp().admin_set_user_password( + UserPoolId=self.user_pool_id, + Username=username, + Password=password, + Permanent=permanent + ) + + self.password_updated(username) + + self._context.cache().short_term().delete(self.build_user_cache_key(username)) + self._logger.info(f'SetPassword: {username}, Permanent: {permanent}') + + def admin_reset_password(self, username: str): + self._context.aws().cognito_idp().admin_reset_user_password( + UserPoolId=self.user_pool_id, + Username=username, + ) + self._context.cache().short_term().delete(self.build_user_cache_key(username)) + self._logger.info(f'ResetPassword: {username}') + + def admin_update_email(self, username: str, email: str, email_verified: bool = False): + self._context.aws().cognito_idp().admin_update_user_attributes( + UserPoolId=self.user_pool_id, + Username=username, + UserAttributes=[ + { + 'Name': 'email', + 'Value': email + }, + { + 'Name': 'email_verified', + 'Value': str(email_verified) + } + ] + ) + self._context.cache().short_term().delete(self.build_user_cache_key(username)) + + def admin_set_email_verified(self, username: str): + self._context.aws().cognito_idp().admin_update_user_attributes( + UserPoolId=self.user_pool_id, + Username=username, + UserAttributes=[ + { + 'Name': 'email_verified', + 'Value': 'true' + } + ] + ) + self._context.cache().short_term().delete(self.build_user_cache_key(username)) + + def admin_global_sign_out(self, username: str): + self._context.aws().cognito_idp().admin_user_global_sign_out( + UserPoolId=self.user_pool_id, + Username=username + ) + + def initiate_username_password_auth(self, request: InitiateAuthRequest) -> InitiateAuthResult: + + username = request.username + if Utils.is_empty(username): + raise exceptions.invalid_params('username is required.') + + password = request.password + if Utils.is_empty(password): + raise exceptions.invalid_params('password is required.') + + try: + cognito_result = self._context.aws().cognito_idp().admin_initiate_auth( + AuthFlow='ADMIN_USER_PASSWORD_AUTH', + AuthParameters={ + 'USERNAME': username, + 'PASSWORD': password, + 'SECRET_HASH': self.get_secret_hash(username) + }, + UserPoolId=self.user_pool_id, + ClientId=self.get_client_id() + ) + except botocore.exceptions.ClientError as e: + error_code = e.response['Error']['Code'] + if error_code in ('NotAuthorizedException', 'UserNotFoundException'): + raise exceptions.unauthorized_access('Invalid username or password') + elif error_code == 'PasswordResetRequiredException': + raise exceptions.soca_exception( + error_code=errorcodes.AUTH_PASSWORD_RESET_REQUIRED, + message='Password reset required for the user' + ) + else: + raise e + + auth_result = None + challenge_name = None + challenge_params = None + session = None + + cognito_auth_result = Utils.get_value_as_dict('AuthenticationResult', cognito_result) + if cognito_auth_result is None: + challenge_name = Utils.get_value_as_string('ChallengeName', cognito_result) + challenge_params = Utils.get_value_as_dict('ChallengeParameters', cognito_result) + session = Utils.get_value_as_string('Session', cognito_result) + else: + auth_result = self.build_auth_result(cognito_auth_result) + + return InitiateAuthResult( + challenge_name=challenge_name, + challenge_params=challenge_params, + session=session, + auth=auth_result + ) + + def respond_to_auth_challenge(self, request: RespondToAuthChallengeRequest) -> RespondToAuthChallengeResult: + cognito_result = self._context.aws().cognito_idp().admin_respond_to_auth_challenge( + UserPoolId=self.user_pool_id, + ClientId=self.options.client_id, + ChallengeName=request.challenge_name, + Session=request.session, + ChallengeResponses={ + 'USERNAME': request.username, + 'NEW_PASSWORD': request.new_password, + 'SECRET_HASH': self.get_secret_hash(request.username) + } + ) + + auth_result = None + challenge_name = None + challenge_params = None + session = None + + cognito_auth_result = Utils.get_value_as_dict('AuthenticationResult', cognito_result) + if cognito_auth_result is None: + challenge_name = Utils.get_value_as_string('ChallengeName', cognito_result) + challenge_params = Utils.get_value_as_dict('ChallengeParameters', cognito_result) + session = Utils.get_value_as_string('Session', cognito_result) + else: + self.admin_set_email_verified(request.username) + self.password_updated(request.username) + auth_result = self.build_auth_result(cognito_auth_result) + self._context.cache().short_term().delete(self.build_user_cache_key(request.username)) + + return RespondToAuthChallengeResult( + challenge_name=challenge_name, + challenge_params=challenge_params, + session=session, + auth=auth_result + ) + + def initiate_refresh_token_auth(self, username: str, refresh_token: str, sso: bool = False): + + if Utils.is_empty(username): + raise exceptions.invalid_params('username is required.') + + if Utils.is_empty(refresh_token): + raise exceptions.invalid_params('refresh_token is required.') + + try: + cognito_result = self._context.aws().cognito_idp().admin_initiate_auth( + AuthFlow='REFRESH_TOKEN_AUTH', + AuthParameters={ + 'REFRESH_TOKEN': refresh_token, + 'SECRET_HASH': self.get_secret_hash(username, sso) + }, + UserPoolId=self.user_pool_id, + ClientId=self.get_client_id(sso) + ) + except botocore.exceptions.ClientError as e: + error_code = e.response['Error']['Code'] + if error_code == 'NotAuthorizedException': + raise exceptions.unauthorized_access('Refresh token invalid or expired') + else: + raise e + + cognito_auth_result = Utils.get_value_as_dict('AuthenticationResult', cognito_result) + auth_result = self.build_auth_result(cognito_auth_result) + return RespondToAuthChallengeResult( + auth=auth_result + ) + + def forgot_password(self, username: str): + try: + self._context.aws().cognito_idp().forgot_password( + ClientId=self.options.client_id, + SecretHash=self.get_secret_hash(username), + Username=username + ) + except botocore.exceptions.ClientError as e: + if e.response['Error']['Code'] == 'UserNotFoundException': + # return success response even for user not found failures to mitigate user enumeration risks + pass + else: + raise e + + def confirm_forgot_password(self, username: str, password: str, confirmation_code: str): + self._context.aws().cognito_idp().confirm_forgot_password( + ClientId=self.options.client_id, + SecretHash=self.get_secret_hash(username), + Username=username, + Password=password, + ConfirmationCode=confirmation_code + ) + self.password_updated(username) + self._context.cache().short_term().delete(self.build_user_cache_key(username)) + + def change_password(self, username: str, access_token: str, old_password: str, new_password: str): + self._context.aws().cognito_idp().change_password( + AccessToken=access_token, + PreviousPassword=old_password, + ProposedPassword=new_password + ) + self.password_updated(username) diff --git a/source/idea/idea-cluster-manager/src/ideaclustermanager/app/accounts/db/__init__.py b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/accounts/db/__init__.py new file mode 100644 index 00000000..6d8d18a3 --- /dev/null +++ b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/accounts/db/__init__.py @@ -0,0 +1,10 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. diff --git a/source/idea/idea-cluster-manager/src/ideaclustermanager/app/accounts/db/ad_automation_dao.py b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/accounts/db/ad_automation_dao.py new file mode 100644 index 00000000..a4aa4f1b --- /dev/null +++ b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/accounts/db/ad_automation_dao.py @@ -0,0 +1,90 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +from ideasdk.utils import Utils +from ideadatamodel import exceptions +from ideasdk.context import SocaContext + +from typing import Dict + + +class ADAutomationDAO: + + def __init__(self, context: SocaContext): + self.context = context + self.ad_automation_entry_ttl_seconds = context.config().get_int('directoryservice.ad_automation.entry_ttl_seconds', default=30 * 60) + self.table = None + + def get_table_name(self) -> str: + return f'{self.context.cluster_name()}.ad-automation' + + def initialize(self): + self.context.aws_util().dynamodb_create_table( + create_table_request={ + 'TableName': self.get_table_name(), + 'AttributeDefinitions': [ + { + 'AttributeName': 'instance_id', + 'AttributeType': 'S' + }, + { + 'AttributeName': 'nonce', + 'AttributeType': 'S' + } + ], + 'KeySchema': [ + { + 'AttributeName': 'instance_id', + 'KeyType': 'HASH' + }, + { + 'AttributeName': 'nonce', + 'KeyType': 'RANGE' + } + ], + 'BillingMode': 'PAY_PER_REQUEST' + }, + wait=True, + ttl=True, + ttl_attribute_name='ttl' + ) + self.table = self.context.aws().dynamodb_table().Table(self.get_table_name()) + + def create_ad_automation_entry(self, entry: Dict, ttl=None) -> Dict: + + instance_id = Utils.get_value_as_string('instance_id', entry) + if Utils.is_empty(instance_id): + raise exceptions.invalid_params('instance_id is required') + + nonce = Utils.get_value_as_string('nonce', entry) + if Utils.is_empty(nonce): + raise exceptions.invalid_params('nonce is required') + + status = Utils.get_value_as_string('status', entry) + if Utils.is_empty(status): + raise exceptions.invalid_params('status is required.') + + if status not in ('success', 'fail'): + raise exceptions.invalid_params('status must be one of [success, fail]') + + if ttl is None: + ttl = Utils.current_time_ms() + (self.ad_automation_entry_ttl_seconds * 60 * 1000) + + created_entry = { + **entry, + 'ttl': ttl, + 'created_on': Utils.current_time_ms(), + 'updated_on': Utils.current_time_ms() + } + self.table.put_item( + Item=created_entry + ) + return created_entry diff --git a/source/idea/idea-cluster-manager/src/ideaclustermanager/app/accounts/db/group_dao.py b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/accounts/db/group_dao.py new file mode 100644 index 00000000..9d47a1f8 --- /dev/null +++ b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/accounts/db/group_dao.py @@ -0,0 +1,209 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +from ideasdk.utils import Utils +from ideadatamodel import exceptions, ListGroupsRequest, ListGroupsResult, Group, SocaPaginator +from ideasdk.context import SocaContext + +from typing import Optional, Dict +from boto3.dynamodb.conditions import Attr + + +class GroupDAO: + + def __init__(self, context: SocaContext, logger=None): + self.context = context + if logger is not None: + self.logger = logger + else: + self.logger = context.logger('group-dao') + self.table = None + + def get_table_name(self) -> str: + return f'{self.context.cluster_name()}.accounts.groups' + + def initialize(self): + self.context.aws_util().dynamodb_create_table( + create_table_request={ + 'TableName': self.get_table_name(), + 'AttributeDefinitions': [ + { + 'AttributeName': 'group_name', + 'AttributeType': 'S' + } + ], + 'KeySchema': [ + { + 'AttributeName': 'group_name', + 'KeyType': 'HASH' + } + ], + 'BillingMode': 'PAY_PER_REQUEST' + }, + wait=True + ) + self.table = self.context.aws().dynamodb_table().Table(self.get_table_name()) + + @staticmethod + def convert_from_db(group: Dict) -> Group: + return Group( + **{ + 'name': Utils.get_value_as_string('group_name', group), + 'ds_name': Utils.get_value_as_string('ds_name', group), + 'gid': Utils.get_value_as_int('gid', group), + 'title': Utils.get_value_as_string('title', group), + 'description': Utils.get_value_as_string('description', group), + 'enabled': Utils.get_value_as_bool('enabled', group), + 'group_type': Utils.get_value_as_string('group_type', group), + 'created_on': Utils.get_value_as_int('created_on', group), + 'updated_on': Utils.get_value_as_int('updated_on', group) + } + ) + + @staticmethod + def convert_to_db(group: Group) -> Dict: + db_group = { + 'group_name': group.name, + } + if group.name is not None: + db_group['group_name'] = group.name + if group.gid is not None: + db_group['gid'] = group.gid + if group.title is not None: + db_group['title'] = group.title + if group.description is not None: + db_group['description'] = group.description + if group.enabled is not None: + db_group['enabled'] = group.enabled + if group.group_type is not None: + db_group['group_type'] = group.group_type + if group.ref is not None: + db_group['ref'] = group.ref + if group.ds_name is not None: + db_group['ds_name'] = group.ds_name + return db_group + + def create_group(self, group: Dict) -> Dict: + + group_name = Utils.get_value_as_string('group_name', group) + if Utils.is_empty(group_name): + raise exceptions.invalid_params('group_name is required') + + created_group = { + **group, + 'created_on': Utils.current_time_ms(), + 'updated_on': Utils.current_time_ms() + } + self.table.put_item( + Item=created_group, + ConditionExpression=Attr('group_name').not_exists() + ) + return created_group + + def get_group(self, group_name: str) -> Optional[Dict]: + if Utils.is_empty(group_name): + raise exceptions.invalid_params('group_name is required') + result = self.table.get_item( + Key={ + 'group_name': group_name + } + ) + return Utils.get_value_as_dict('Item', result) + + def update_group(self, group: Dict) -> Optional[Dict]: + group_name = Utils.get_value_as_string('group_name', group) + if Utils.is_empty(group_name): + raise exceptions.invalid_params('group_name is required') + + group['updated_on'] = Utils.current_time_ms() + + update_expression_tokens = [] + expression_attr_names = {} + expression_attr_values = {} + + for key, value in group.items(): + if key in ('group_name', 'created_on'): + continue + update_expression_tokens.append(f'#{key} = :{key}') + expression_attr_names[f'#{key}'] = key + expression_attr_values[f':{key}'] = value + + result = self.table.update_item( + Key={ + 'group_name': group_name + }, + ConditionExpression=Attr('group_name').eq(group_name), + UpdateExpression='SET ' + ', '.join(update_expression_tokens), + ExpressionAttributeNames=expression_attr_names, + ExpressionAttributeValues=expression_attr_values, + ReturnValues='ALL_NEW' + ) + + updated_user = result['Attributes'] + updated_user['group_name'] = group_name + return updated_user + + def delete_group(self, group_name: str): + if Utils.is_empty(group_name): + raise exceptions.invalid_params('group_name is required') + self.table.delete_item( + Key={ + 'group_name': group_name + } + ) + + def list_groups(self, request: ListGroupsRequest) -> ListGroupsResult: + + scan_request = {} + + cursor = request.cursor + last_evaluated_key = None + if Utils.is_not_empty(cursor): + last_evaluated_key = Utils.from_json(Utils.base64_decode(cursor)) + if last_evaluated_key is not None: + scan_request['LastEvaluatedKey'] = last_evaluated_key + + scan_filter = None + if Utils.is_not_empty(request.filters): + scan_filter = {} + for filter_ in request.filters: + if filter_.eq is not None: + scan_filter[filter_.key] = { + 'AttributeValueList': [filter_.eq], + 'ComparisonOperator': 'EQ' + } + if filter_.like is not None: + scan_filter[filter_.key] = { + 'AttributeValueList': [filter_.like], + 'ComparisonOperator': 'CONTAINS' + } + if scan_filter is not None: + scan_request['ScanFilter'] = scan_filter + + scan_result = self.table.scan(**scan_request) + + db_groups = Utils.get_value_as_list('Items', scan_result, []) + groups = [] + for db_group in db_groups: + group = self.convert_from_db(db_group) + groups.append(group) + + response_cursor = None + last_evaluated_key = Utils.get_any_value('LastEvaluatedKey', scan_result) + if last_evaluated_key is not None: + response_cursor = Utils.base64_encode(Utils.to_json(last_evaluated_key)) + + return ListGroupsResult( + listing=groups, + paginator=SocaPaginator( + cursor=response_cursor + ) + ) diff --git a/source/idea/idea-cluster-manager/src/ideaclustermanager/app/accounts/db/group_members_dao.py b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/accounts/db/group_members_dao.py new file mode 100644 index 00000000..cae94971 --- /dev/null +++ b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/accounts/db/group_members_dao.py @@ -0,0 +1,186 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +from ideasdk.utils import Utils +from ideadatamodel import exceptions, ListUsersInGroupRequest, ListUsersInGroupResult, SocaPaginator +from ideasdk.context import SocaContext + +from ideaclustermanager.app.accounts.auth_utils import AuthUtils +from ideaclustermanager.app.accounts.db.user_dao import UserDAO + +from typing import List +from boto3.dynamodb.conditions import Key + + +class GroupMembersDAO: + + def __init__(self, context: SocaContext, user_dao: UserDAO, logger=None): + self.context = context + self.user_dao = user_dao + if logger is not None: + self.logger = logger + else: + self.logger = context.logger('group-members-dao') + self.table = None + + def get_table_name(self) -> str: + return f'{self.context.cluster_name()}.accounts.group-members' + + def initialize(self): + self.context.aws_util().dynamodb_create_table( + create_table_request={ + 'TableName': self.get_table_name(), + 'AttributeDefinitions': [ + { + 'AttributeName': 'group_name', + 'AttributeType': 'S' + }, + { + 'AttributeName': 'username', + 'AttributeType': 'S' + } + ], + 'KeySchema': [ + { + 'AttributeName': 'group_name', + 'KeyType': 'HASH' + }, + { + 'AttributeName': 'username', + 'KeyType': 'RANGE' + } + ], + 'BillingMode': 'PAY_PER_REQUEST' + }, + wait=True + ) + self.table = self.context.aws().dynamodb_table().Table(self.get_table_name()) + + def create_membership(self, group_name: str, username: str): + username = AuthUtils.sanitize_username(username) + if Utils.is_empty(group_name): + raise exceptions.invalid_params('group_name is required') + + self.table.put_item( + Item={ + 'group_name': group_name, + 'username': username + } + ) + + def delete_membership(self, group_name: str, username: str): + username = AuthUtils.sanitize_username(username) + if Utils.is_empty(group_name): + raise exceptions.invalid_params('group_name is required') + + self.table.delete_item( + Key={ + 'group_name': group_name, + 'username': username + } + ) + + def has_users_in_group(self, group_name: str) -> bool: + query_result = self.table.query( + Limit=1, + KeyConditions={ + 'group_name': { + 'AttributeValueList': [group_name], + 'ComparisonOperator': 'EQ' + } + } + ) + memberships = Utils.get_value_as_list('Items', query_result, []) + return len(memberships) > 0 + + def get_usernames_in_group(self, group_name: str) -> List[str]: + if Utils.is_empty(group_name): + raise exceptions.invalid_params('group_name is required') + + usernames = [] + + exclusive_start_key = None + while True: + if exclusive_start_key is not None: + query_result = self.table.query( + ExclusiveStartKey=exclusive_start_key, + KeyConditionExpression=Key('group_name').eq(group_name) + ) + else: + query_result = self.table.query( + KeyConditionExpression=Key('group_name').eq(group_name) + ) + + db_user_groups = Utils.get_value_as_list('Items', query_result, []) + for db_user_group in db_user_groups: + usernames.append(db_user_group['username']) + + exclusive_start_key = Utils.get_any_value('LastEvaluatedKey', query_result) + if exclusive_start_key is None: + break + + return usernames + + def list_users_in_group(self, request: ListUsersInGroupRequest) -> ListUsersInGroupResult: + group_names = request.group_names + if Utils.is_empty(group_names): + raise exceptions.invalid_params('group_names are required') + + cursor = request.cursor + exclusive_start_keys = None + last_evaluated_keys = {} + username_set = set() + users = [] + + if Utils.is_not_empty(cursor): + exclusive_start_keys = Utils.from_json(Utils.base64_decode(cursor)) + + for group_name in group_names: + exclusive_start_key = Utils.get_value_as_dict(group_name, exclusive_start_keys, {}) + if Utils.is_not_empty(exclusive_start_key): + query_result = self.table.query( + Limit=request.page_size, + ExclusiveStartKey=exclusive_start_key, + KeyConditionExpression=Key('group_name').eq(group_name) + ) + else: + query_result = self.table.query( + Limit=request.page_size, + KeyConditionExpression=Key('group_name').eq(group_name) + ) + + db_user_groups = Utils.get_value_as_list('Items', query_result, []) + for db_user_group in db_user_groups: + db_username = db_user_group['username'] + if db_username in username_set: + continue + username_set.add(db_username) + db_user = self.user_dao.get_user(db_username) + if db_user is None: + continue + user = self.user_dao.convert_from_db(db_user) + users.append(user) + + last_evaluated_key = Utils.get_any_value('LastEvaluatedKey', query_result) + if Utils.is_not_empty(last_evaluated_key): + last_evaluated_keys[group_name] = last_evaluated_key + + response_cursor = None + if Utils.is_not_empty(last_evaluated_keys): + response_cursor = Utils.base64_encode(Utils.to_json(last_evaluated_keys)) + + return ListUsersInGroupResult( + listing=users, + paginator=SocaPaginator( + page_size=request.page_size, + cursor=response_cursor + ) + ) diff --git a/source/idea/idea-cluster-manager/src/ideaclustermanager/app/accounts/db/sequence_config_dao.py b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/accounts/db/sequence_config_dao.py new file mode 100644 index 00000000..a93313fe --- /dev/null +++ b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/accounts/db/sequence_config_dao.py @@ -0,0 +1,96 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +from ideasdk.utils import Utils +from ideasdk.context import SocaContext +from boto3.dynamodb.conditions import Attr +import botocore.exceptions + +DEFAULT_START_ID = 5000 +KEY_USERS = 'users' +KEY_GROUPS = 'groups' + + +class SequenceConfigDAO: + + def __init__(self, context: SocaContext, logger=None): + self.context = context + if logger is not None: + self.logger = logger + else: + self.logger = context.logger('sequence-config-dao') + self.table = None + + def get_table_name(self) -> str: + return f'{self.context.cluster_name()}.accounts.sequence-config' + + def initialize(self): + self.context.aws_util().dynamodb_create_table( + create_table_request={ + 'TableName': self.get_table_name(), + 'AttributeDefinitions': [ + { + 'AttributeName': 'key', + 'AttributeType': 'S' + } + ], + 'KeySchema': [ + { + 'AttributeName': 'key', + 'KeyType': 'HASH' + } + ], + 'BillingMode': 'PAY_PER_REQUEST' + }, + wait=True + ) + self.table = self.context.aws().dynamodb_table().Table(self.get_table_name()) + + self._init_key(KEY_USERS, self.context.config().get_int('directoryservice.start_uid', default=DEFAULT_START_ID)) + self._init_key(KEY_GROUPS, self.context.config().get_int('directoryservice.start_gid', default=DEFAULT_START_ID)) + + def _init_key(self, key: str, value: int): + try: + self.table.put_item( + Item={ + 'key': key, + 'value': value + }, + ConditionExpression=Attr('key').not_exists() + ) + except botocore.exceptions.ClientError as e: + if e.response['Error']['Code'] == 'ConditionalCheckFailedException': + pass + else: + raise e + + def _next(self, key: str) -> int: + result = self.table.update_item( + Key={ + 'key': key + }, + UpdateExpression='ADD #value :value', + ExpressionAttributeNames={ + '#value': 'value' + }, + ExpressionAttributeValues={ + ':value': 1 + }, + ReturnValues='ALL_OLD' + ) + attributes = result['Attributes'] + return Utils.get_value_as_int('value', attributes) + + def next_uid(self) -> int: + return self._next(KEY_USERS) + + def next_gid(self) -> int: + return self._next(KEY_GROUPS) diff --git a/source/idea/idea-cluster-manager/src/ideaclustermanager/app/accounts/db/single_sign_on_state_dao.py b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/accounts/db/single_sign_on_state_dao.py new file mode 100644 index 00000000..049d436e --- /dev/null +++ b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/accounts/db/single_sign_on_state_dao.py @@ -0,0 +1,138 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +from ideasdk.utils import Utils +from ideadatamodel import exceptions, AuthResult +from ideasdk.context import SocaContext + +from typing import Optional, Dict +from boto3.dynamodb.conditions import Attr + + +class SingleSignOnStateDAO: + + def __init__(self, context: SocaContext, logger=None): + self.context = context + if logger is not None: + self.logger = logger + else: + self.logger = context.logger('single-sign-on-state-dao') + self.table = None + + def get_table_name(self) -> str: + return f'{self.context.cluster_name()}.accounts.sso-state' + + def initialize(self): + self.context.aws_util().dynamodb_create_table( + create_table_request={ + 'TableName': self.get_table_name(), + 'AttributeDefinitions': [ + { + 'AttributeName': 'state', + 'AttributeType': 'S' + } + ], + 'KeySchema': [ + { + 'AttributeName': 'state', + 'KeyType': 'HASH' + } + ], + 'BillingMode': 'PAY_PER_REQUEST' + }, + wait=True, + ttl=True, + ttl_attribute_name='ttl' + ) + self.table = self.context.aws().dynamodb_table().Table(self.get_table_name()) + + @staticmethod + def convert_from_db(sso_state: Dict) -> AuthResult: + return AuthResult( + **{ + 'access_token': Utils.get_value_as_string('access_token', sso_state), + 'refresh_token': Utils.get_value_as_string('refresh_token', sso_state), + 'id_token': Utils.get_value_as_string('id_token', sso_state), + 'expires_in': Utils.get_value_as_int('expires_in', sso_state), + 'token_type': Utils.get_value_as_string('token_type', sso_state) + } + ) + + def create_sso_state(self, sso_state: Dict) -> Dict: + + state = Utils.get_value_as_string('state', sso_state) + if Utils.is_empty(state): + raise exceptions.invalid_params('state is required') + + created_state = { + **sso_state, + 'ttl': Utils.current_time_ms() + (10 * 60 * 1000), # 10 minutes + 'created_on': Utils.current_time_ms(), + 'updated_on': Utils.current_time_ms() + } + self.table.put_item( + Item=created_state, + ConditionExpression=Attr('state').not_exists() + ) + return created_state + + def update_sso_state(self, sso_state: Dict) -> Dict: + + state = Utils.get_value_as_string('state', sso_state) + if Utils.is_empty(state): + raise exceptions.invalid_params('state is required') + + sso_state['updated_on'] = Utils.current_time_ms() + + update_expression_tokens = [] + expression_attr_names = {} + expression_attr_values = {} + + for key, value in sso_state.items(): + if key in ('state', 'created_on'): + continue + update_expression_tokens.append(f'#{key} = :{key}') + expression_attr_names[f'#{key}'] = key + expression_attr_values[f':{key}'] = value + + result = self.table.update_item( + Key={ + 'state': state + }, + ConditionExpression=Attr('state').eq(state), + UpdateExpression='SET ' + ', '.join(update_expression_tokens), + ExpressionAttributeNames=expression_attr_names, + ExpressionAttributeValues=expression_attr_values, + ReturnValues='ALL_NEW' + ) + + updated_sso_state = result['Attributes'] + updated_sso_state['state'] = state + return updated_sso_state + + def get_sso_state(self, state: str) -> Optional[Dict]: + if Utils.is_empty(state): + raise exceptions.invalid_params('state is required') + result = self.table.get_item( + Key={ + 'state': state + } + ) + return Utils.get_value_as_dict('Item', result) + + def delete_sso_state(self, state: str): + if Utils.is_empty(state): + raise exceptions.invalid_params('state is required') + self.table.delete_item( + Key={ + 'state': state + } + ) diff --git a/source/idea/idea-cluster-manager/src/ideaclustermanager/app/accounts/db/user_dao.py b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/accounts/db/user_dao.py new file mode 100644 index 00000000..2cb9635b --- /dev/null +++ b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/accounts/db/user_dao.py @@ -0,0 +1,229 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +from ideasdk.utils import Utils +from ideadatamodel import ListUsersRequest, ListUsersResult, SocaPaginator, User +from ideasdk.context import SocaContext + +from ideaclustermanager.app.accounts.auth_utils import AuthUtils +from ideaclustermanager.app.accounts.cognito_user_pool import CognitoUserPool + +from typing import Optional, Dict +from boto3.dynamodb.conditions import Attr + + +class UserDAO: + + def __init__(self, context: SocaContext, user_pool: CognitoUserPool, logger=None): + self.context = context + if logger is not None: + self.logger = logger + else: + self.logger = context.logger('user-dao') + self.user_pool = user_pool + self.table = None + + def get_table_name(self) -> str: + return f'{self.context.cluster_name()}.accounts.users' + + def initialize(self): + self.context.aws_util().dynamodb_create_table( + create_table_request={ + 'TableName': self.get_table_name(), + 'AttributeDefinitions': [ + { + 'AttributeName': 'username', + 'AttributeType': 'S' + } + ], + 'KeySchema': [ + { + 'AttributeName': 'username', + 'KeyType': 'HASH' + } + ], + 'BillingMode': 'PAY_PER_REQUEST' + }, + wait=True + ) + self.table = self.context.aws().dynamodb_table().Table(self.get_table_name()) + + def convert_from_db(self, user: Dict) -> User: + user = User( + **{ + 'username': Utils.get_value_as_string('username', user), + 'email': Utils.get_value_as_string('email', user), + 'uid': Utils.get_value_as_int('uid', user), + 'gid': Utils.get_value_as_int('gid', user), + 'group_name': Utils.get_value_as_string('group_name', user), + 'login_shell': Utils.get_value_as_string('login_shell', user), + 'home_dir': Utils.get_value_as_string('home_dir', user), + 'sudo': Utils.get_value_as_bool('sudo', user), + 'enabled': Utils.get_value_as_bool('enabled', user), + 'password_last_set': Utils.get_value_as_int('password_last_set', user), + 'password_max_age': Utils.get_value_as_int('password_max_age', user), + 'additional_groups': Utils.get_value_as_list('additional_groups', user), + 'created_on': Utils.get_value_as_int('created_on', user), + 'updated_on': Utils.get_value_as_int('updated_on', user) + } + ) + cognito_user = self.user_pool.admin_get_user(user.username) + if cognito_user is not None: + user.status = cognito_user.UserStatus + else: + user.status = 'DELETED' + return user + + @staticmethod + def convert_to_db(user: User) -> Dict: + db_user = { + 'username': user.username + } + if user.email is not None: + db_user['email'] = user.email + if user.uid is not None: + db_user['uid'] = user.uid + if user.gid is not None: + db_user['gid'] = user.gid + if user.group_name is not None: + db_user['group_name'] = user.group_name + if user.login_shell is not None: + db_user['login_shell'] = user.login_shell + if user.home_dir is not None: + db_user['home_dir'] = user.home_dir + if user.sudo is not None: + db_user['sudo'] = user.sudo + if user.enabled is not None: + db_user['enabled'] = user.enabled + if user.password_last_set is not None: + db_user['password_last_set'] = int(user.password_last_set.timestamp() * 1000) + if user.password_max_age is not None: + db_user['password_max_age'] = int(user.password_max_age) + if user.additional_groups is not None: + db_user['additional_groups'] = user.additional_groups + + return db_user + + def create_user(self, user: Dict) -> Dict: + + username = Utils.get_value_as_string('username', user) + username = AuthUtils.sanitize_username(username) + + created_user = { + **user, + 'username': username, + 'created_on': Utils.current_time_ms(), + 'updated_on': Utils.current_time_ms() + } + + self.table.put_item( + Item=created_user, + ConditionExpression=Attr('username').not_exists() + ) + return created_user + + def get_user(self, username: str) -> Optional[Dict]: + username = AuthUtils.sanitize_username(username) + result = self.table.get_item( + Key={ + 'username': username + } + ) + return Utils.get_value_as_dict('Item', result) + + + def update_user(self, user: Dict) -> Dict: + username = Utils.get_value_as_string('username', user) + username = AuthUtils.sanitize_username(username) + user['updated_on'] = Utils.current_time_ms() + + update_expression_tokens = [] + expression_attr_names = {} + expression_attr_values = {} + + for key, value in user.items(): + if key in ('username', 'created_on'): + continue + update_expression_tokens.append(f'#{key} = :{key}') + expression_attr_names[f'#{key}'] = key + expression_attr_values[f':{key}'] = value + + result = self.table.update_item( + Key={ + 'username': username + }, + ConditionExpression=Attr('username').eq(username), + UpdateExpression='SET ' + ', '.join(update_expression_tokens), + ExpressionAttributeNames=expression_attr_names, + ExpressionAttributeValues=expression_attr_values, + ReturnValues='ALL_NEW' + ) + + updated_user = result['Attributes'] + updated_user['username'] = username + return updated_user + + def delete_user(self, username: str): + + username = AuthUtils.sanitize_username(username) + self.table.delete_item( + Key={ + 'username': username + } + ) + + def list_users(self, request: ListUsersRequest) -> ListUsersResult: + + scan_request = {} + + cursor = request.cursor + last_evaluated_key = None + if Utils.is_not_empty(cursor): + last_evaluated_key = Utils.from_json(Utils.base64_decode(cursor)) + if last_evaluated_key is not None: + scan_request['LastEvaluatedKey'] = last_evaluated_key + + scan_filter = None + if Utils.is_not_empty(request.filters): + scan_filter = {} + for filter_ in request.filters: + if filter_.eq is not None: + scan_filter[filter_.key] = { + 'AttributeValueList': [filter_.eq], + 'ComparisonOperator': 'EQ' + } + if filter_.like is not None: + scan_filter[filter_.key] = { + 'AttributeValueList': [filter_.like], + 'ComparisonOperator': 'CONTAINS' + } + if scan_filter is not None: + scan_request['ScanFilter'] = scan_filter + + scan_result = self.table.scan(**scan_request) + + db_users = Utils.get_value_as_list('Items', scan_result, []) + users = [] + for db_user in db_users: + user = self.convert_from_db(db_user) + users.append(user) + + response_cursor = None + last_evaluated_key = Utils.get_any_value('LastEvaluatedKey', scan_result) + if last_evaluated_key is not None: + response_cursor = Utils.base64_encode(Utils.to_json(last_evaluated_key)) + + return ListUsersResult( + listing=users, + paginator=SocaPaginator( + cursor=response_cursor + ) + ) diff --git a/source/idea/idea-cluster-manager/src/ideaclustermanager/app/accounts/helpers/__init__.py b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/accounts/helpers/__init__.py new file mode 100644 index 00000000..6d8d18a3 --- /dev/null +++ b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/accounts/helpers/__init__.py @@ -0,0 +1,10 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. diff --git a/source/idea/idea-cluster-manager/src/ideaclustermanager/app/accounts/helpers/preset_computer_helper.py b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/accounts/helpers/preset_computer_helper.py new file mode 100644 index 00000000..9c1a5de1 --- /dev/null +++ b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/accounts/helpers/preset_computer_helper.py @@ -0,0 +1,443 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +from ideasdk.context import SocaContext +from ideasdk.utils import Utils +from ideasdk.shell import ShellInvoker +from ideadatamodel import exceptions, errorcodes, EC2Instance + +from ideaclustermanager.app.accounts.ldapclient.active_directory_client import ActiveDirectoryClient +from ideaclustermanager.app.accounts.db.ad_automation_dao import ADAutomationDAO + +from typing import Dict, List, Optional +import botocore.exceptions +import secrets +import string + + + +class PresetComputeHelper: + """ + Helper to manage creating preset Computer Accounts in AD using adcli. + + Errors: + * AD_AUTOMATION_PRESET_COMPUTER_FAILED - when the request is bad, invalid or cannot be retried. + * AD_AUTOMATION_PRESET_COMPUTER_RETRY - when request is valid, but preset-computer operation fails due to intermittent or timing errors. + operation will be retried based on SQS visibility timeout settings. + """ + + def __init__(self, context: SocaContext, ldap_client: ActiveDirectoryClient, ad_automation_dao: ADAutomationDAO, sender_id: str, request: Dict): + """ + :param context: + :param ldap_client: + :param ad_automation_dao: + :param sender_id: SenderId attribute from SQS Message + :param request: the original request payload envelope + """ + self.context = context + self.ldap_client = ldap_client + self.ad_automation_dao = ad_automation_dao + self.sender_id = sender_id + self.request = request + + self.logger = context.logger('preset-computer') + + self.nonce: Optional[str] = None + self.instance_id: Optional[str] = None + self.ec2_instance: Optional[EC2Instance] = None + self.hostname: Optional[str] = None + + # Metadata for AD joins + self.aws_account = self.context.config().get_string('cluster.aws.account_id', required=True) + self.cluster_name = self.context.config().get_string('cluster.cluster_name', required=True) + self.aws_region = self.context.config().get_string('cluster.aws.region', required=True) + # parse and validate sender_id and request + payload = Utils.get_value_as_dict('payload', request, {}) + + # SenderId attribute from SQS message to protect against spoofing. + if Utils.is_empty(sender_id): + raise exceptions.soca_exception( + error_code=errorcodes.AD_AUTOMATION_PRESET_COMPUTER_FAILED, + message='Unable to verify cluster node identity: SenderId is required' + ) + + # enforce a nonce for an additional layer of protection against spoofing and help tracing + nonce = Utils.get_value_as_string('nonce', payload) + if Utils.is_empty(nonce): + raise exceptions.soca_exception( + error_code=errorcodes.AD_AUTOMATION_PRESET_COMPUTER_FAILED, + message='nonce is required' + ) + self.nonce = nonce + + # when sent from an EC2 Instance with an IAM Role attached, SenderId is of below format (IAM role ID): + # AROAZKN2GIY65I74VE5YH:i-035b89c7f49714a3e + sender_id_tokens = sender_id.split(':') + if len(sender_id_tokens) != 2: + raise exceptions.soca_exception( + error_code=errorcodes.AD_AUTOMATION_PRESET_COMPUTER_FAILED, + message='Unable to verify cluster node identity: Invalid SenderId' + ) + + instance_id = sender_id_tokens[1] + try: + ec2_instances = self.context.aws_util().ec2_describe_instances(filters=[ + { + 'Name': 'instance-id', + 'Values': [instance_id] + }, + { + 'Name': 'instance-state-name', + 'Values': ['running'] + } + ]) + except botocore.exceptions.ClientError as e: + error_code = str(e.response['Error']['Code']) + if error_code.startswith('InvalidInstanceID'): + raise exceptions.soca_exception( + error_code=errorcodes.AD_AUTOMATION_PRESET_COMPUTER_FAILED, + message=f'Unable to verify cluster node identity: Invalid InstanceId - {instance_id}' + ) + else: + # for all other errors, retry + raise e + + if len(ec2_instances) == 0: + raise exceptions.soca_exception( + error_code=errorcodes.AD_AUTOMATION_PRESET_COMPUTER_FAILED, + message=f'Unable to verify cluster node identity: InstanceId = {instance_id} not found' + ) + + self.instance_id = instance_id + + self.ec2_instance = ec2_instances[0] + + # for Windows instances, there is no way to fetch the hostname from describe instances API. + # request payload from windows instances will contain hostname. eg. EC2AMAZ-6S29U5P + hostname = Utils.get_value_as_string('hostname', payload) + if Utils.is_empty(hostname): + # Generate and make use of an IDEA hostname + hostname_data = f"{self.aws_region}|{self.aws_account}|{self.cluster_name}|{self.instance_id}" + hostname_prefix = self.context.config().get_string('directoryservice.ad_automation.hostname_prefix', default='IDEA-') + + # todo - move to constants + # todo - change this to produce the shake output and take this many chars vs. bytes (hex) + # check the configured hostname_prefix length and how much it leaves us for generating the random portion. + avail_chars = (15 - len(hostname_prefix)) + if avail_chars < 4: + raise exceptions.soca_exception( + error_code=errorcodes.AD_AUTOMATION_PRESET_COMPUTER_FAILED, + message=f'{self.log_tag}configured hostname_prefix is too large. Required at least 4 char of random data. Decrease the size of directoryservice.ad_automation.hostname_prefix: {len(hostname_prefix)}' + ) + + self.logger.info(f'Using hostname information {hostname_data} (configured hostname prefix: [{hostname_prefix}] / len {len(hostname_prefix)} / {avail_chars} chars available for random portion)') + # Take the last n-chars from the resulting shake256 bucket of 256 + shake_value = Utils.shake_256(hostname_data, 256)[(avail_chars * -1):] + hostname = f'{hostname_prefix}{shake_value}'.upper() + self.logger.info(f'Generated IDEA hostname / AD hostname of: {hostname} / len {len(hostname)}') + + self.hostname = hostname + self.logger.info(f'Using hostname for AD join: {self.hostname}') + + self._shell = ShellInvoker(logger=self.logger) + + # verify if adcli is installed and available on the system. + which_adcli = self._shell.invoke('command -v adcli', shell=True) + if which_adcli.returncode != 0: + raise exceptions.general_exception('unable to locate adcli on system to initialize PresetComputerHelper') + self.ADCLI = which_adcli.stdout + + # initialize domain controller IP addresses + self._domain_controller_ips = self.get_domain_controller_ip_addresses() + + @property + def log_tag(self) -> str: + return f'(Host: {self.hostname}, InstanceId: {self.instance_id}, Nonce: {self.nonce})' + + def get_ldap_computers_base(self) -> str: + ou_computers = self.context.config().get_string('directoryservice.computers.ou', required=True) + if '=' in ou_computers: + return ou_computers + return f'ou={ou_computers},ou={self.ldap_client.ad_netbios},{self.ldap_client.ldap_base}' + + @staticmethod + def get_ldap_computer_filterstr(hostname: str) -> str: + return f'(&(objectClass=computer)(cn={hostname}))' + + def is_existing_computer_account(self, trace=False) -> bool: + search_result = self.ldap_client.search_s( + base=self.get_ldap_computers_base(), + filterstr=self.get_ldap_computer_filterstr(self.hostname), + attrlist=['dn'], + trace=trace + ) + return len(search_result) > 0 + + def get_domain_controller_ip_addresses(self) -> List[str]: + """ + perform adcli discovery on the AD domain name and return all the domain controller hostnames. + :return: hostnames all available domain controllers + """ + result = self._shell.invoke( + cmd=[ + self.ADCLI, + 'info', + self.ldap_client.domain_name.upper() + ] + ) + if result.returncode != 0: + raise exceptions.soca_exception( + error_code=errorcodes.AD_AUTOMATION_PRESET_COMPUTER_RETRY, + message=f'{self.log_tag} failed to perform adcli information discovery on AD domain: {self.ldap_client.domain_name}' + ) + + # self.logger.debug(f'ADCLI Domain info: {result.stdout}') + # Example output for a domain with 6 domain controllers: + # [domain] + # domain-name = idea.local + # domain-short = IDEA + # domain-forest = idea.local + # domain-controller = IP-C6130254.idea.local + # domain-controller-site = us-east-1 + # domain-controller-flags = gc ldap ds kdc timeserv closest writable full-secret ads-web + # domain-controller-usable = yes + # domain-controllers = IP-C6130254.idea.local IP-C6120243.idea.local ip-c61301a7.idea.local ip-c61202c6.idea.local ip-c6120053.idea.local ip-c612008c.idea.local + # [computer] + # computer-site = us-east-1 + + # store the output in domain_query for quick review of params + domain_query = {} + lines = str(result.stdout).splitlines() + for line in lines: + line = line.strip() + if line.startswith('['): + continue + try: + result_key = line.split(' =')[0] + result_value = line.split('= ')[1] + except IndexError as e: + self.logger.warning(f'Error parsing AD discovery output: {e}. Line skipped: {line}') + continue + + self.logger.debug(f'Key: [{result_key:25}] Value: [{result_value:25}]') + + if ( + not Utils.get_as_string(result_key, default='') or + not Utils.get_as_string(result_value, default='') + ): + self.logger.warning(f'Error parsing AD discovery output. Unable to parse Key/Value Pair. Check adcli version/output. Line skipped: {line}') + continue + + # Save for later + domain_query[result_key] = result_value + + # Sanity check our query results + # todo - should domain-controller-flags be evaluated for writeable or other health flags? + + # domain-name must be present and match our configuration + domain_name = Utils.get_value_as_string('domain-name', domain_query, default=None) + if domain_name is None: + raise exceptions.soca_exception( + error_code=errorcodes.AD_AUTOMATION_PRESET_COMPUTER_RETRY, + message=f'{self.log_tag} Unable to validate AD domain discovery for domain-name: {self.ldap_client.domain_name}' + ) + + if domain_name.upper() != self.ldap_client.domain_name.upper(): + raise exceptions.soca_exception( + error_code=errorcodes.AD_AUTOMATION_PRESET_COMPUTER_RETRY, + message=f"{self.log_tag} AD domain discovery mismatch for domain-name: Got: {domain_name.upper()} Expected: {self.ldap_client.domain_name.upper()}" + ) + + # domain-short must be present and match our configuration + domain_shortname = Utils.get_value_as_string('domain-short', domain_query, default=None) + if domain_shortname is None: + raise exceptions.soca_exception( + error_code=errorcodes.AD_AUTOMATION_PRESET_COMPUTER_RETRY, + message=f'{self.log_tag} Unable to validate AD domain discovery for domain shortname: {self.ldap_client.domain_name}' + ) + if domain_shortname.upper() != self.ldap_client.ad_netbios.upper(): + raise exceptions.soca_exception( + error_code=errorcodes.AD_AUTOMATION_PRESET_COMPUTER_RETRY, + message=f'{self.log_tag} AD domain discovery mismatch for shortname: Got: {domain_shortname.upper()} Expected: {self.ldap_client.ad_netbios.upper()}' + ) + + # domain_controllers must be a list of domain controllers + # split() vs. split(' ') - we don't want empty entries in the list + # else our len() check would be incorrect + domain_controllers = domain_query.get('domain-controllers', '').strip().split() + if len(domain_controllers) == 0: + raise exceptions.soca_exception( + error_code=errorcodes.AD_AUTOMATION_PRESET_COMPUTER_RETRY, + message=f'{self.log_tag} no domain controllers found for AD domain: {self.ldap_client.domain_name}. check your firewall settings and verify if traffic is allowed on port 53.' + ) + + return domain_controllers + + def get_any_domain_controller_ip(self) -> str: + """ + Return the next domain controller in the list as discovered from adcli + :return: Domain Controller IP Address + """ + if len(self._domain_controller_ips) == 0: + raise exceptions.soca_exception( + error_code=errorcodes.AD_AUTOMATION_PRESET_COMPUTER_RETRY, + message=f'{self.log_tag} all existing AD domain controllers have been tried to create computer account, but failed. request will be retried.' + ) + + # We just take the first remaining domain controller as adcli discovery organizes the list for us + # .pop(0) should be safe as we just checked for len() + selected_dc = self._domain_controller_ips.pop(0) + self.logger.info(f'Selecting AD domain controller for operation: {selected_dc}') + + return selected_dc + + def delete_computer(self, domain_controller_ip: str): + delete_computer_result = self._shell.invoke( + cmd_input=self.ldap_client.ldap_root_password, + cmd=[ + self.ADCLI, + 'delete-computer', + f'--domain-controller={domain_controller_ip}', + f'--login-user={self.ldap_client.ldap_root_username}', + '--stdin-password', + f'--domain={self.ldap_client.domain_name}', + f'--domain-realm={self.ldap_client.domain_name.upper()}', + self.hostname + ] + ) + if delete_computer_result.returncode != 0: + raise exceptions.soca_exception( + error_code=errorcodes.AD_AUTOMATION_PRESET_COMPUTER_FAILED, + message=f'{self.log_tag} failed to delete existing computer account: {delete_computer_result}' + ) + + def preset_computer(self, domain_controller_ip: str) -> str: + + # generate one-time-password. password cannot be more than 120 characters and cannot start with numbers. + # OTP = + 119-characters from letters(mixed case) and digits == 120 chars + # prefix_letter is always a letter (mixed case) to avoid a digit landing as the first character. + # Expanding the pool to include printable/punctuation can be considered but would introduce + # escaping and quoting considerations as it is passed to the shell/adcli. + one_time_password = secrets.choice(string.ascii_letters) + one_time_password += ''.join(secrets.choice(string.ascii_letters + string.digits) for i in range(119)) + + # sanity check to make sure we never create a computer with a weak domain password + if len(one_time_password) != 120: + raise exceptions.soca_exception( + error_code=errorcodes.AD_AUTOMATION_PRESET_COMPUTER_FAILED, + message=f'{self.log_tag} Internal error - Failed to generate a strong domain password' + ) + + preset_computer_result = self._shell.invoke( + cmd_input=self.ldap_client.ldap_root_password, + cmd=[ + self.ADCLI, + 'preset-computer', + f'--domain-controller={domain_controller_ip}', + f'--login-user={self.ldap_client.ldap_root_username}', + '--stdin-password', + f'--one-time-password={one_time_password}', + f'--domain={self.ldap_client.domain_name}', + f'--domain-ou={self.get_ldap_computers_base()}', + '--verbose', + self.hostname + ], + skip_error_logging=True + ) + + if preset_computer_result.returncode != 0: + raise exceptions.soca_exception( + error_code=errorcodes.AD_AUTOMATION_PRESET_COMPUTER_FAILED, + message=f'{self.log_tag} failed to preset-computer. details: {preset_computer_result}' + ) + + # We cannot issue an immediate update as the modify_s aims at the domain + # and replicaton may have not taken place yet. So this should be queued for an update + # when the object appears within the AD connection? + # todo - jobIDs / other info that is useful to the AD admin? + # should the incoming node provide a description field to cluster-manager? + # infra nodes wouldn't have a jobID + #self.ldap_client.update_computer_description( + # computer=self.hostname, + # description=f'IDEA|{self.cluster_name}|{self.instance_id}' + #) + # We cannot include this in the preset-computer or other adcli commands as this + # is not inlucded in all adcli implementations for our baseOSs. So we manually update LDAP. + # eg. + # # f"--description='IDEA {self.aws_region} / {self.cluster_name} / {self.instance_id}'", + + return one_time_password + + def invoke(self): + """ + call adcli preset-computer and update the ad-automation dynamodb table with applicable status + """ + + try: + + # fetch a domain controller IP to "pin" all adcli operations to ensure we don't run into synchronization problems + domain_controller_ip = self.get_any_domain_controller_ip() + + if self.is_existing_computer_account(): + # if computer already exists in AD + # it is likely the case where the private IP is being reused in the VPC where an old cluster node was deleted without removing entry from AD. + # delete and create preset-computer + self.logger.warning(f'{self.log_tag} found existing computer account. deleting using DC: {domain_controller_ip} ...') + self.delete_computer(domain_controller_ip=domain_controller_ip) + + self.logger.info(f'{self.log_tag} creating new computer account using DC: {domain_controller_ip} ...') + + try: + one_time_password = self.preset_computer(domain_controller_ip=domain_controller_ip) + except exceptions.SocaException as e: + if e.error_code == errorcodes.AD_AUTOMATION_PRESET_COMPUTER_FAILED: + # ad is finicky. returns below error even for valid request: + # ! Cannot set computer password: Authentication error + if 'Cannot set computer password: Authentication error' in e.message: + self.delete_computer(domain_controller_ip=domain_controller_ip) + return self.invoke() + else: + # if failed due to some other reason, re-raise as a retry exception + raise exceptions.soca_exception( + error_code=errorcodes.AD_AUTOMATION_PRESET_COMPUTER_RETRY, + message=e.message + ) + else: + raise e + + self.ad_automation_dao.create_ad_automation_entry(entry={ + 'instance_id': self.instance_id, + 'nonce': self.nonce, + 'hostname': self.hostname, + 'otp': one_time_password, + 'domain_controller': domain_controller_ip, + 'node_type': self.ec2_instance.soca_node_type, + 'module_id': self.ec2_instance.idea_module_id, + 'status': 'success' + }) + + self.logger.info(f'{self.log_tag} computer account created successfully.') + except exceptions.SocaException as e: + if e.error_code == errorcodes.AD_AUTOMATION_PRESET_COMPUTER_FAILED: + # add feedback entry for the host indicating failure status, and stop polling ddb + self.ad_automation_dao.create_ad_automation_entry(entry={ + 'instance_id': self.instance_id, + 'nonce': self.nonce, + 'hostname': self.hostname, + 'status': 'fail', + 'error_code': e.error_code, + 'message': e.message, + 'node_type': self.ec2_instance.soca_node_type, + 'module_id': self.ec2_instance.idea_module_id + }) + + # raise exception in all cases + raise e diff --git a/source/idea/idea-cluster-manager/src/ideaclustermanager/app/accounts/ldapclient/__init__.py b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/accounts/ldapclient/__init__.py new file mode 100644 index 00000000..0e10cfed --- /dev/null +++ b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/accounts/ldapclient/__init__.py @@ -0,0 +1,14 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +from ideaclustermanager.app.accounts.ldapclient.abstract_ldap_client import AbstractLDAPClient, LdapClientOptions +from ideaclustermanager.app.accounts.ldapclient.openldap_client import OpenLDAPClient +from ideaclustermanager.app.accounts.ldapclient.active_directory_client import ActiveDirectoryClient diff --git a/source/idea/idea-cluster-manager/src/ideaclustermanager/app/accounts/ldapclient/abstract_ldap_client.py b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/accounts/ldapclient/abstract_ldap_client.py new file mode 100644 index 00000000..f16a8069 --- /dev/null +++ b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/accounts/ldapclient/abstract_ldap_client.py @@ -0,0 +1,789 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +from ideadatamodel import ( + exceptions, errorcodes, constants, + SocaBaseModel, + SocaFilter, + SocaPaginator +) +from ideasdk.context import SocaContext +from ideasdk.utils import Utils +from ideaclustermanager.app.accounts.ldapclient.ldap_utils import LdapUtils + +from typing import Optional, Dict, List, Tuple +import ldap # noqa +from ldap.ldapobject import LDAPObject # noqa +from ldap.controls import SimplePagedResultsControl # noqa +from ldap.controls.vlv import VLVRequestControl, VLVResponseControl # noqa +from ldap.controls.sss import SSSRequestControl, SSSResponseControl # noqa +from ldappool import ConnectionManager # noqa +from abc import abstractmethod +import base64 + + +class LdapClientOptions(SocaBaseModel): + uri: Optional[str] + domain_name: Optional[str] + root_username: Optional[str] + root_password: Optional[str] + root_username_secret_arn: Optional[str] + root_password_secret_arn: Optional[str] + root_username_file: Optional[str] + root_password_file: Optional[str] + connection_pool_enabled: Optional[bool] + connection_pool_size: Optional[int] + connection_retry_max: Optional[int] + connection_retry_delay: Optional[float] + connection_timeout: Optional[float] + ad_netbios: Optional[str] + directory_id: Optional[str] + password_max_age: Optional[float] + + +DEFAULT_LDAP_CONNECTION_POOL_SIZE = 10 +DEFAULT_LDAP_CONNECTION_RETRY_MAX = 60 +DEFAULT_LDAP_CONNECTION_RETRY_DELAY = 10 +DEFAULT_LDAP_CONNECTION_TIMEOUT = 10 +DEFAULT_LDAP_ENABLE_CONNECTION_POOL = True +DEFAULT_LDAP_PAGE_SIZE = 20 +DEFAULT_LDAP_PAGE_START = 0 +DEFAULT_LDAP_COOKIE = '' + +UID_MIN = 5000 +UID_MAX = 65533 # 65534 is for "nobody" and 65535 is reserved +GID_MIN = 5000 +GID_MAX = 65533 + + +class AbstractLDAPClient: + + def __init__(self, context: SocaContext, options: LdapClientOptions, logger=None): + self.context = context + + if logger is not None: + self.logger = logger + else: + self.logger = context.logger('ldap-client') + + self.options = options + + if Utils.is_empty(self.domain_name): + raise exceptions.general_exception('options.domain_name is required') + + self._root_username: Optional[str] = None + self._root_password: Optional[str] = None + + self.refresh_root_username_password() + + # initialize pooled connection manager for LDAP to conserve resources + self.connection_manager = ConnectionManager( + uri=self.ldap_uri, + size=Utils.get_as_int(options.connection_pool_size, DEFAULT_LDAP_CONNECTION_POOL_SIZE), + retry_max=Utils.get_as_int(options.connection_retry_max, DEFAULT_LDAP_CONNECTION_RETRY_MAX), + retry_delay=Utils.get_as_float(options.connection_retry_delay, DEFAULT_LDAP_CONNECTION_RETRY_DELAY), + timeout=Utils.get_as_float(options.connection_timeout, DEFAULT_LDAP_CONNECTION_TIMEOUT), + use_pool=Utils.get_as_bool(options.connection_pool_enabled, DEFAULT_LDAP_ENABLE_CONNECTION_POOL) + ) + + # ldap wrapper methods + def add_s(self, dn, modlist): + trace_message = f'ldapadd -x -D "{self.ldap_root_bind}" -H {self.ldap_uri} "{dn}"' + attributes = [] + for mod in modlist: + key = mod[0] + values = [] + for value in mod[1]: + values.append(Utils.from_bytes(value)) + attributes.append(f'{key}={",".join(values)}') + self.logger.info(f'> {trace_message}, attributes: ({" ".join(attributes)})') + with self.get_ldap_root_connection() as conn: + conn.add_s(dn, modlist) + + def modify_s(self, dn, modlist): + trace_message = f'ldapmodify -x -D "{self.ldap_root_bind}" -H {self.ldap_uri} "{dn}"' + attributes = [] + for mod in modlist: + key = mod[1] + values = [] + for value in mod[2]: + values.append(Utils.from_bytes(value)) + attributes.append(f'{key}={",".join(values)}') + self.logger.info(f'> {trace_message}, attributes: ({" ".join(attributes)})') + with self.get_ldap_root_connection() as conn: + conn.modify_s(dn, modlist) + + def delete_s(self, dn): + trace_message = f'ldapdelete -x -D "{self.ldap_root_bind}" -H {self.ldap_uri} "{dn}"' + self.logger.info(f'> {trace_message}') + with self.get_ldap_root_connection() as conn: + conn.delete_s(dn) + + def search_s(self, base, scope=ldap.SCOPE_SUBTREE, filterstr=None, attrlist=None, attrsonly=0, trace=True): + if trace: + trace_message = f'ldapsearch -x -b "{base}" -D "{self.ldap_root_bind}" -H {self.ldap_uri} "{filterstr}"' + if attrlist is not None: + trace_message = f'{trace_message} {" ".join(attrlist)}' + self.logger.info(f'> {trace_message}') + with self.get_ldap_root_connection() as conn: + return conn.search_s(base, scope, filterstr, attrlist, attrsonly) + + def simple_paginated_search(self, base, scope=ldap.SCOPE_SUBTREE, + filterstr=None, + attrlist=None, + attrsonly=0, + timeout=-1, + page_size: int = None, + cookie: str = None) -> Dict: + trace_message = f'ldapsearch -x -b "{base}" -D "{self.ldap_root_bind}" -H {self.ldap_uri} "{filterstr}"' + if attrlist is not None: + trace_message = f'{trace_message} {" ".join(attrlist)}' + self.logger.info(f'> {trace_message}') + result = [] + with self.get_ldap_root_connection() as conn: + page_size = Utils.get_as_int(page_size, DEFAULT_LDAP_PAGE_SIZE) + if cookie is None: + cookie_b = DEFAULT_LDAP_COOKIE + else: + cookie_b = base64.b64decode(cookie) + serverctrls = [SimplePagedResultsControl(True, size=page_size, cookie=cookie_b)] + + message_id = conn.search_ext(base, scope, filterstr, attrlist, attrsonly, serverctrls, None, timeout) + rtype, rdata, rmsgid, res_serverctrls = conn.result3(message_id) + + result.extend(rdata) + controls = [control for control in res_serverctrls if control.controlType == SimplePagedResultsControl.controlType] + if controls and len(controls) > 0: + return { + 'result': result, + 'total': None, + 'cookie': Utils.from_bytes(base64.b64encode(controls[0].cookie)) + } + else: + return { + 'result': result, + 'total': None, + 'cookie': None + } + + def sssvlv_paginated_search(self, + base, + sort_by: str, + scope=ldap.SCOPE_SUBTREE, + filterstr=None, + attrlist=None, + attrsonly=0, + timeout=-1, + start=0, + page_size: int = None) -> Dict: + """ + Server Side Sorting and Virtual List View (SSSVLV) based search and paging for LDAP + Refer to https://ldapwiki.com/wiki/Virtual%20List%20View%20Control for more details + + start: 0 based offset + ldap offsets are 1 based, but to normalize pagination across all soca services, + all paging offsets must start with 0 for SOCA services + + sort_by: : + eg. + 1. gidNumber:integerOrderingMatch + 2. uidNumber:integerOrderingMatch + 3. cn:caseIgnoreOrderingMatch + Refer to https://ldapwiki.com/wiki/MatchingRule for more details. + + :param base: + :param sort_by: + :param scope: + :param filterstr: + :param attrlist: + :param attrsonly: + :param timeout: float + :param start: int, default 0 + :param page_size: int, default: 20 + :return: + """ + trace_message = f'ldapsearch -x -b "{base}" -D "{self.ldap_root_bind}" -H {self.ldap_uri} "{filterstr}"' + if attrlist is not None: + trace_message = f'{trace_message} {" ".join(attrlist)}' + self.logger.info(f'> {trace_message}') + result = [] + + # sssvlv has a limit on no. of concurrent paginated result sets per connection to manage memory. + # and does not offer a clean way to manage or clean up paginated results. + # to overcome this, create a new connection each time instead of using a pooled connection + # this comes at a significant performance hit and needs to be assessed based on no. of users and size of directory + conn = None + try: + conn = ldap.initialize(self.ldap_uri) + conn.bind_s(who=self.ldap_root_bind, cred=self.ldap_root_password, method=ldap.AUTH_SIMPLE) + + page_size = Utils.get_as_int(page_size, DEFAULT_LDAP_PAGE_SIZE) + page_start = Utils.get_as_int(start, DEFAULT_LDAP_PAGE_START) + + serverctrls = [ + VLVRequestControl( + criticality=True, + before_count=0, + after_count=page_size - 1, + offset=page_start + 1, + content_count=0 + ), + SSSRequestControl( + criticality=True, + ordering_rules=[sort_by] + ) + ] + + message_id = conn.search_ext(base, scope, filterstr, attrlist, attrsonly, serverctrls, None, timeout) + rtype, rdata, rmsgid, res_serverctrls = conn.result3(message_id) + + total = 0 + for res_control in res_serverctrls: + if res_control.controlType == VLVResponseControl.controlType: + total = res_control.content_count + + result.extend(rdata) + return { + 'result': result, + 'total': total + } + + except ldap.VLV_ERROR: + return { + 'result': result, + 'total': 0 + } + finally: + try: + if conn: + conn.unbind_s() + except AttributeError: + pass + + @property + def ds_provider(self) -> str: + return self.context.config().get_string('directoryservice.provider', required=True) + + def is_openldap(self) -> bool: + return self.ds_provider == constants.DIRECTORYSERVICE_OPENLDAP + + def is_activedirectory(self) -> bool: + return self.ds_provider in ( + constants.DIRECTORYSERVICE_AWS_MANAGED_ACTIVE_DIRECTORY, + constants.DIRECTORYSERVICE_ACTIVE_DIRECTORY + ) + def is_readonly(self) -> bool: + return self.ds_provider in ( + constants.DIRECTORYSERVICE_ACTIVE_DIRECTORY + ) + + @property + def domain_name(self) -> str: + return self.options.domain_name + + @property + def ldap_base(self) -> str: + tokens = self.domain_name.split('.') + return f'dc={",dc=".join(tokens)}' + + @property + def ldap_uri(self) -> str: + if Utils.is_not_empty(self.options.uri): + return self.options.uri + return 'ldap://localhost' + + @property + def password_max_age(self) -> Optional[float]: + return self.options.password_max_age + + def fetch_root_username(self) -> str: + if Utils.is_not_empty(self.options.root_username): + return self.options.root_username + + file = self.options.root_username_file + if Utils.is_not_empty(file) and Utils.is_file(file): + with open(file, 'r') as f: + return f.read().strip() + + secret_arn = self.options.root_username_secret_arn + if Utils.is_not_empty(secret_arn): + result = self.context.aws().secretsmanager().get_secret_value( + SecretId=secret_arn + ) + return Utils.get_value_as_string('SecretString', result) + + raise exceptions.soca_exception( + error_code=errorcodes.GENERAL_ERROR, + message='DS root username/password not configured' + ) + + def fetch_root_password(self) -> str: + if Utils.is_not_empty(self.options.root_password): + return self.options.root_password + + file = self.options.root_password_file + if Utils.is_not_empty(file) and Utils.is_file(file): + with open(file, 'r') as f: + return f.read().strip() + + secret_arn = self.options.root_password_secret_arn + if Utils.is_not_empty(secret_arn): + result = self.context.aws().secretsmanager().get_secret_value( + SecretId=secret_arn + ) + return Utils.get_value_as_string('SecretString', result) + + raise exceptions.soca_exception( + error_code=errorcodes.GENERAL_ERROR, + message='DS root username/password not configured' + ) + + def update_root_password(self, password: str): + if Utils.is_not_empty(self.options.root_password): + self.options.root_password = password + + file = self.options.root_password_file + if Utils.is_not_empty(file) and Utils.is_file(file): + with open(file, 'w') as f: + f.write(password) + + secret_arn = self.options.root_password_secret_arn + if Utils.is_not_empty(secret_arn): + self.context.aws().secretsmanager().put_secret_value( + SecretId=secret_arn, + SecretString=password + ) + + self._root_password = password + + def refresh_root_username_password(self): + self._root_username = self.fetch_root_username() + self._root_password = self.fetch_root_password() + + @property + def ldap_root_username(self) -> str: + return self._root_username + + @property + def ldap_root_password(self) -> str: + return self._root_password + + def get_ldap_root_connection(self) -> LDAPObject: + """ + returns an LDAP connection object bound to ROOT user from the connection pool + """ + return self.connection_manager.connection( + bind=self.ldap_root_bind, + passwd=self.ldap_root_password + ) + + @staticmethod + def convert_ldap_group(ldap_group: Dict) -> Optional[Dict]: + if Utils.is_empty(ldap_group): + return None + name = LdapUtils.get_string_value('cn', ldap_group) + gid = LdapUtils.get_int_value('gidNumber', ldap_group) + users = LdapUtils.get_string_list('memberUid', ldap_group) + + return { + 'name': name, + 'gid': gid, + 'users': users + } + + def is_existing_group(self, group_name: str) -> bool: + if Utils.is_empty(group_name): + raise exceptions.invalid_params('group_name is required') + + result = self.search_s( + base=self.ldap_group_base, + filterstr=self.build_group_filterstr(group_name), + attrlist=['cn'] + ) + + return len(result) > 0 + + def is_existing_user(self, username: str) -> bool: + + if Utils.is_empty(username): + raise exceptions.invalid_params('username is required') + + results = self.search_s( + base=self.ldap_user_base, + filterstr=self.build_user_filterstr(username), + attrlist=['cn'] + ) + + return len(results) > 0 + + def is_sudo_user(self, username: str) -> bool: + results = self.search_s( + base=self.ldap_sudoers_base, + filterstr=self.build_sudoer_filterstr(username), + attrlist=['cn'] + ) + + return len(results) > 0 + + + @property + @abstractmethod + def ldap_computers_base(self) -> str: + ... + + @property + @abstractmethod + def ldap_user_base(self) -> str: + ... + + @property + @abstractmethod + def ldap_user_filterstr(self) -> str: + ... + + def build_computer_dn(self, computer: str) -> str: + return f'cn={computer},{self.ldap_computers_base}' + + def build_user_dn(self, username: str) -> str: + return f'cn={username},{self.ldap_user_base}' + + def build_user_filterstr(self, username: str) -> str: + return f'(&{self.ldap_user_filterstr}(cn={username}))' + + @property + @abstractmethod + def ldap_group_base(self) -> str: + ... + + @property + @abstractmethod + def ldap_group_filterstr(self) -> str: + ... + + def build_group_dn(self, group_name: str) -> str: + return f'cn={group_name},{self.ldap_group_base}' + + def build_group_filterstr(self, group_name: str = None, username: str = None) -> str: + filterstr = self.ldap_group_filterstr + if group_name is not None: + filterstr = f'{filterstr}(cn={group_name})' + if username is not None: + filterstr = f'{filterstr}(memberUid={username})' + return f'(&{filterstr})' + + @property + @abstractmethod + def ldap_sudoers_base(self) -> str: + ... + + @property + @abstractmethod + def ldap_sudoer_filterstr(self) -> str: + ... + + def build_sudoer_filterstr(self, username: str) -> str: + return f'(&{self.ldap_sudoer_filterstr}(cn={username}))' + + @property + @abstractmethod + def ldap_root_bind(self) -> str: + ... + + @abstractmethod + def change_password(self, username: str, password: str): + ... + + def convert_ldap_user(self, ldap_user: Dict) -> Optional[Dict]: + if Utils.is_empty(ldap_user): + return None + + self.logger.debug(f'convert_ldap_user() - Converting from full lookup results: {ldap_user}') + + cn = LdapUtils.get_string_value('cn', ldap_user) + username = LdapUtils.get_string_value('uid', ldap_user) + if Utils.is_empty(username): + username = LdapUtils.get_string_value('sAMAccountName', ldap_user) + email = LdapUtils.get_string_value('mail', ldap_user) + uid = LdapUtils.get_int_value('uidNumber', ldap_user) + gid = LdapUtils.get_int_value('gidNumber', ldap_user) + login_shell = LdapUtils.get_string_value('loginShell', ldap_user) + home_dir = LdapUtils.get_string_value('homeDirectory', ldap_user) + + # UserAccountControl from ActiveDirectory + if self.is_activedirectory(): + user_account_control = LdapUtils.get_int_value('userAccountControl', ldap_user) + else: + user_account_control = None + + password_last_set = None + password_last_set_win = LdapUtils.get_int_value('pwdLastSet', ldap_user) + if password_last_set_win is not None and password_last_set_win != 0: + password_last_set = Utils.from_win_timestamp(password_last_set_win) + + result = { + 'cn': cn, + 'username': username, + 'email': email, + 'uid': uid, + 'gid': gid, + 'login_shell': login_shell, + 'home_dir': home_dir, + 'user_account_control': user_account_control + } + + if password_last_set: + result['password_last_set'] = password_last_set + result['password_max_age'] = self.password_max_age + + return result + + def get_group(self, group_name: str) -> Optional[Dict]: + result = self.search_s( + base=self.ldap_group_base, + filterstr=self.build_group_filterstr(group_name) + ) + + if len(result) == 0: + return None + + return self.convert_ldap_group(result[0][1]) + + @abstractmethod + def sync_group(self, group_name: str, gid: int) -> Dict: + ... + + def get_all_groups_with_user(self, username: str) -> List[str]: + ldap_result = self.search_s( + base=self.ldap_group_base, + filterstr=self.build_group_filterstr(username=username), + attrlist=['cn'] + ) + + result = [] + for ldap_group in ldap_result: + name = LdapUtils.get_string_value('cn', ldap_group[1]) + result.append(name) + + return result + + def search_groups(self, group_name_filter: SocaFilter = None, + username_filter: SocaFilter = None, + page_size: int = None, + start: int = 0) -> Tuple[List[Dict], SocaPaginator]: + result = [] + + group_name_token = None + if group_name_filter is not None: + if Utils.is_not_empty(group_name_filter.eq): + group_name_token = group_name_filter.eq + elif Utils.is_not_empty(group_name_filter.starts_with): + group_name_token = f'{group_name_filter.starts_with}*' + elif Utils.is_not_empty(group_name_filter.ends_with): + group_name_token = f'*{group_name_filter.ends_with}' + elif Utils.is_not_empty(group_name_filter.like): + group_name_token = f'*{group_name_filter.like}*' + + username_token = None + if username_filter is not None: + if Utils.is_not_empty(username_filter.eq): + username_token = username_filter.eq + elif Utils.is_not_empty(username_filter.starts_with): + username_token = f'{username_filter.starts_with}*' + elif Utils.is_not_empty(username_filter.ends_with): + username_token = f'*{username_filter.ends_with}' + elif Utils.is_not_empty(username_filter.like): + username_token = f'*{username_filter.like}*' + + if Utils.are_empty(group_name_token, username_token): + filterstr = self.ldap_group_filterstr + else: + filterstr = self.build_group_filterstr(group_name=group_name_token, username=username_token) + + if self.is_activedirectory(): + + search_result = self.simple_paginated_search( + base=self.ldap_group_base, + filterstr=filterstr + ) + + ldap_result = Utils.get_value_as_list('result', search_result, default=[]) + total = len(ldap_result) + + else: + + search_result = self.sssvlv_paginated_search( + base=self.ldap_group_base, + sort_by='gidNumber:integerOrderingMatch', + filterstr=filterstr, + page_size=page_size, + start=start + ) + + ldap_result = Utils.get_value_as_list('result', search_result) + total = Utils.get_value_as_int('total', search_result) + + for ldap_group in ldap_result: + user_group = self.convert_ldap_group(ldap_group[1]) + if Utils.is_empty(user_group): + continue + result.append(user_group) + + return result, SocaPaginator(page_size=page_size, start=start, total=total) + + def add_user_to_group(self, usernames: List[str], group_name: str): + try: + group_dn = self.build_group_dn(group_name) + group_attrs = [] + for username in usernames: + group_attrs.append((ldap.MOD_ADD, 'memberUid', [Utils.to_bytes(username)])) + self.modify_s(group_dn, group_attrs) + except ldap.TYPE_OR_VALUE_EXISTS: + pass + + def remove_user_from_group(self, usernames: List[str], group_name: str): + group_dn = self.build_group_dn(group_name) + mod_attrs = [] + for username in usernames: + mod_attrs.append((ldap.MOD_DELETE, 'memberUid', [Utils.to_bytes(username)])) + self.modify_s(group_dn, mod_attrs) + + def delete_group(self, group_name: str): + if not self.is_existing_group(group_name): + return + + group_dn = self.build_group_dn(group_name) + self.delete_s(group_dn) + + @abstractmethod + def add_sudo_user(self, username: str): + ... + + @abstractmethod + def remove_sudo_user(self, username: str): + ... + + def list_sudo_users(self) -> List[str]: + result = [] + + ldap_result = self.search_s( + base=self.ldap_sudoers_base, + filterstr=self.ldap_sudoer_filterstr, + attrlist=['cn'] + ) + + for entry in ldap_result: + sudo_user = entry[1] + username = LdapUtils.get_string_value('cn', sudo_user) + result.append(username) + + return result + + def get_user(self, username: str, trace=True) -> Optional[Dict]: + + results = self.search_s( + base=self.ldap_user_base, + filterstr=self.build_user_filterstr(username), + trace=trace + ) + + if len(results) == 0: + return None + + return self.convert_ldap_user(results[0][1]) + + @abstractmethod + def create_service_account(self, username: str, password: str): + ... + + @abstractmethod + def sync_user(self, *, + uid: int, + gid: int, + username: str, + email: str, + login_shell: str, + home_dir: str) -> Dict: + ... + + def update_computer_description(self, computer: str, description: str): + computer_dn = self.build_computer_dn(computer) + computer_attrs = [ + (ldap.MOD_REPLACE, 'description', [Utils.to_bytes(description)]) + ] + self.modify_s(computer_dn, computer_attrs) + + def update_email(self, username: str, email: str): + user_dn = self.build_user_dn(username) + user_attrs = [ + (ldap.MOD_REPLACE, 'mail', [Utils.to_bytes(email)]) + ] + self.modify_s(user_dn, user_attrs) + + def delete_user(self, username: str): + user_dn = self.build_user_dn(username) + self.delete_s(user_dn) + + def search_users(self, username_filter: SocaFilter, page_size: int = None, start: int = 0) -> Tuple[List[Dict], SocaPaginator]: + result = [] + + filterstr = self.ldap_user_filterstr + if username_filter is not None: + if Utils.is_not_empty(username_filter.eq): + filterstr = self.build_user_filterstr(username=username_filter.eq) + elif Utils.is_not_empty(username_filter.starts_with): + filterstr = self.build_user_filterstr(username=f'{username_filter.starts_with}*') + elif Utils.is_not_empty(username_filter.ends_with): + filterstr = self.build_user_filterstr(username=f'*{username_filter.ends_with}') + elif Utils.is_not_empty(username_filter.like): + filterstr = self.build_user_filterstr(username=f'*{username_filter.like}*') + + if self.is_activedirectory(): + + search_result = self.simple_paginated_search( + base=self.ldap_user_base, + filterstr=filterstr + ) + + ldap_result = Utils.get_value_as_list('result', search_result) + total = len(ldap_result) + + else: + + search_result = self.sssvlv_paginated_search( + base=self.ldap_user_base, + sort_by='uidNumber:integerOrderingMatch', + filterstr=filterstr, + page_size=page_size, + start=start + ) + + ldap_result = Utils.get_value_as_list('result', search_result) + total = Utils.get_value_as_int('total', search_result) + + for ldap_user in ldap_result: + user = self.convert_ldap_user(ldap_user[1]) + if Utils.is_empty(user): + continue + result.append(user) + + return result, SocaPaginator(page_size=page_size, start=start, total=total) + + def authenticate_user(self, username: str, password: str) -> bool: + try: + user_dn = self.build_user_dn(username) + conn = ldap.initialize(self.ldap_uri) + # try bind + conn.bind_s(who=user_dn, cred=password, method=ldap.AUTH_SIMPLE) + # free resources + conn.unbind_ext_s() + return True + except ldap.INVALID_CREDENTIALS: + return False diff --git a/source/idea/idea-cluster-manager/src/ideaclustermanager/app/accounts/ldapclient/active_directory_client.py b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/accounts/ldapclient/active_directory_client.py new file mode 100644 index 00000000..e7f07a1f --- /dev/null +++ b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/accounts/ldapclient/active_directory_client.py @@ -0,0 +1,388 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +from ideasdk.context import SocaContext +from ideasdk.utils import Utils +from ideadatamodel import exceptions, errorcodes, constants + +from ideaclustermanager.app.accounts.ldapclient.abstract_ldap_client import AbstractLDAPClient, LdapClientOptions +from ideaclustermanager.app.accounts.ldapclient.ldap_utils import LdapUtils + +from typing import Dict, Optional +import ldap # noqa +import time + + +class ActiveDirectoryClient(AbstractLDAPClient): + + def __init__(self, context: SocaContext, options: LdapClientOptions, logger=None): + if logger is None: + logger = context.logger('active-directory-client') + super().__init__(context, options, logger=logger) + + if Utils.is_empty(self.directory_id) and (self.ds_provider == constants.DIRECTORYSERVICE_AWS_MANAGED_ACTIVE_DIRECTORY): + raise exceptions.general_exception(f'options.directory_id is required when using {constants.DIRECTORYSERVICE_AWS_MANAGED_ACTIVE_DIRECTORY}') + if Utils.is_empty(self.ad_netbios): + raise exceptions.general_exception('options.ad_netbios is required') + + @property + def ds_provider(self) -> str: + return self.context.config().get_string('directoryservice.provider', required=True) + + @property + def directory_id(self) -> str: + return self.options.directory_id + + @property + def ad_netbios(self) -> str: + return self.options.ad_netbios + + @property + def ldap_uri(self) -> str: + if Utils.is_not_empty(self.options.uri): + return self.options.uri + return f'ldap://{self.domain_name}' + + @property + def ldap_root_bind(self) -> str: + return f'{self.ldap_root_username}@{self.domain_name}' + @property + def ldap_computers_base(self) -> str: + ou_computers = self.context.config().get_string('directoryservice.computers.ou', required=True) + if '=' in ou_computers: + return ou_computers + return f'ou={ou_computers},ou={self.ad_netbios},{self.ldap_base}' + @property + def ldap_user_base(self) -> str: + ou_users = self.context.config().get_string('directoryservice.users.ou', required=True) + if '=' in ou_users: + return ou_users + return f'ou={ou_users},ou={self.ad_netbios},{self.ldap_base}' + + @property + def ldap_user_filterstr(self) -> str: + return f'(objectClass=user)' + + @property + def ldap_group_base(self) -> str: + ou_groups = self.context.config().get_string('directoryservice.groups.ou', required=True) + if '=' in ou_groups: + return ou_groups + return f'ou={ou_groups},ou={self.ad_netbios},{self.ldap_base}' + + @property + def ldap_group_filterstr(self) -> str: + return f'(objectClass=group)' + + @property + def ldap_sudoers_group_dn(self) -> str: + sudoers_group_name = self.context.config().get_string('directoryservice.sudoers.group_name', required=True) + tokens = [ + f'cn={sudoers_group_name}' + ] + ou_sudoers = self.context.config().get_string('directoryservice.sudoers.ou') + if Utils.is_not_empty(ou_sudoers): + if '=' in ou_sudoers: + tokens.append(ou_sudoers) + else: + tokens.append(f'ou={ou_sudoers}') + tokens.append(self.ldap_base) + return ','.join(tokens) + + @property + def ldap_sudoers_base(self) -> str: + return self.ldap_user_base + + @property + def ldap_sudoer_filterstr(self) -> str: + return f'(&(objectClass=user)(memberOf={self.ldap_sudoers_group_dn}))' + + def build_sudoer_filterstr(self, username: str) -> str: + return f'(&{self.ldap_sudoer_filterstr[2:-1]}(cn={username}))' + + def build_samaccountname_filterstr(self, username: str) -> str: + return f'(&{self.ldap_user_filterstr}(sAMAccountName={username}))' + + def build_nestedgroup_membership_filterstr(self, username: str, groupname: str) -> str: + return f"(&(memberOf:1.2.840.113556.1.4.1941:={self.build_group_dn(group_name=groupname)})(objectCategory=person)(objectClass=user)(sAMAccountName={username}))" + + def build_email_filterstr(self, email: str) -> str: + return f'(&{self.ldap_user_filterstr}(mail={email}))' + + def build_nestedgroup_email_membership_filterstr(self, email: str, groupname: str) -> str: + return f"(&(memberOf:1.2.840.113556.1.4.1941:={self.build_group_dn(group_name=groupname)})(objectCategory=person)(objectClass=user)(mail={email}))" + + def get_user(self, username: str, required_group=None, trace=True) -> Optional[Dict]: + + # required_group will recursively validate user in a group + # only works in MS-AD LDAP environments + results = self.search_s( + base=self.ldap_user_base, + filterstr=self.build_nestedgroup_membership_filterstr(username=username, groupname=required_group) if required_group else self.build_samaccountname_filterstr(username=username), + trace=trace + ) + + if len(results) == 0: + return None + + return self.convert_ldap_user(results[0][1]) + + def get_user_by_email(self, email: str, required_group=None, trace=True) -> Optional[Dict]: + + # required_group will recursively validate user in a group + # only works in MS-AD LDAP environments + results = self.search_s( + base=self.ldap_user_base, + filterstr=self.build_nestedgroup_email_membership_filterstr(email=email, groupname=required_group) if required_group else self.build_email_filterstr(email=email), + trace=trace + ) + + if len(results) == 0: + return None + + return self.convert_ldap_user(results[0][1]) + + + def change_password(self, username: str, password: str): + + ad_provider = self.context.config().get_string('directoryservice.provider', required=True) + + if ad_provider == constants.DIRECTORYSERVICE_ACTIVE_DIRECTORY: + self.logger.info(f'change_passwd() for Active Directory provider {ad_provider} - NOOP') + return + + elif ad_provider == constants.DIRECTORYSERVICE_AWS_MANAGED_ACTIVE_DIRECTORY: + try: + self.context.aws().ds().reset_user_password( + DirectoryId=self.directory_id, + UserName=username, + NewPassword=password + ) + except self.context.aws().ds().exceptions.UserDoesNotExistException: + raise exceptions.soca_exception( + error_code=errorcodes.AUTH_USER_NOT_FOUND, + message=f'{username} is not yet synced with AD Domain Controllers. Please wait for a few minutes and try again.' + ) + else: + raise exceptions.soca_exception( + error_code=errorcodes.VALIDATION_FAILED, + message=f'Unable to update password. Active Directory provider is unsupported: {ad_provider}' + ) + + def create_service_account(self, username: str, password: str) -> Dict: + user_principal_name = f'{username}@{self.domain_name}' + + user_dn = self.build_user_dn(username) + existing = self.is_existing_user(username) + + readonly = self.is_readonly() + + if existing: + if readonly: + return self.get_user(username) + else: + raise exceptions.invalid_params(f'username already exists: {username}') + + user_attrs = [ + ('objectClass', [ + Utils.to_bytes('top'), + Utils.to_bytes('person'), + Utils.to_bytes('user'), + Utils.to_bytes('organizationalPerson') + ]), + ('displayName', [Utils.to_bytes(username)]), + ('sAMAccountName', [Utils.to_bytes(username)]), + ('userPrincipalName', [Utils.to_bytes(user_principal_name)]), + ('cn', [Utils.to_bytes(username)]), + ('uid', [Utils.to_bytes(username)]) + ] + self.add_s(user_dn, user_attrs) + self.wait_for_user_creation(username, interval=5, min_success=5) + self.change_password(username=username, password=password) + self.add_sudo_user(username=username) + return self.get_user(username) + + def sync_user(self, *, + uid: int, + gid: int, + username: str, + email: str, + login_shell: str, + home_dir: str) -> Dict: + + user_principal_name = f'{username}@{self.domain_name}' + + user_dn = self.build_user_dn(username) + existing = self.is_existing_user(username) + + readonly = self.is_readonly() + + if readonly: + if existing: + self.logger.info(f'Working with existing user / read-only DS: {username}') + # TODO - validate the fetched user has the required attributes + # TODO - update the DynamoDB with the fetched uid/gid/info + return self.get_user(username) + else: + # todo - how to notify caller? + pass + + if existing: + user_attrs = [ + ('mail', [Utils.to_bytes(email)]), + ('uidNumber', [Utils.to_bytes(str(uid))]), + ('gidNumber', [Utils.to_bytes(str(gid))]), + ('loginShell', [Utils.to_bytes(login_shell)]), + ('homeDirectory', [Utils.to_bytes(home_dir)]) + ] + else: + user_attrs = [ + ('objectClass', [ + Utils.to_bytes('top'), + Utils.to_bytes('person'), + Utils.to_bytes('user'), + Utils.to_bytes('organizationalPerson') + ]), + ('displayName', [Utils.to_bytes(username)]), + ('mail', [Utils.to_bytes(email)]), + ('sAMAccountName', [Utils.to_bytes(username)]), + ('userPrincipalName', [Utils.to_bytes(user_principal_name)]), + ('cn', [Utils.to_bytes(username)]), + ('uid', [Utils.to_bytes(username)]), + ('uidNumber', [Utils.to_bytes(str(uid))]), + ('gidNumber', [Utils.to_bytes(str(gid))]), + ('loginShell', [Utils.to_bytes(login_shell)]), + ('homeDirectory', [Utils.to_bytes(home_dir)]) + ] + + if existing: + modify_user_attrs = [] + for user_attr in user_attrs: + modify_user_attr = list(user_attr) + modify_user_attr.insert(0, ldap.MOD_REPLACE) + modify_user_attrs.append(tuple(modify_user_attr)) + self.modify_s(user_dn, modify_user_attrs) + else: + self.add_s(user_dn, user_attrs) + # todo + self.logger.info('Applying user updates to enable user') + # Make sure the user account is enabled in AD. + # This is a two-step process + enable_attrs = [ + (ldap.MOD_REPLACE, 'userAccountControl', [Utils.to_bytes('512')]) + ] + self.modify_s(user_dn, enable_attrs) + self.wait_for_user_creation(username) + + return self.get_user(username) + + def sync_group(self, group_name: str, gid: int) -> Dict: + group_dn = self.build_group_dn(group_name) + + existing = self.is_existing_group(group_name) + + readonly = self.is_readonly() + if readonly: + if existing: + self.logger.info(f'Read-only DS - returning {group_name} lookup') + return self.get_group(group_name) + else: + self.logger.error(f'Unable to find existing group [{group_name}] in read-only Directory Service. Has it been created?') + # fall back to default group? + # todo how to notify the caller? + + if existing: + group_attrs = [ + ('gidNumber', [Utils.to_bytes(str(gid))]) + ] + else: + group_attrs = [ + ('objectClass', [ + Utils.to_bytes('top'), + Utils.to_bytes('group') + ]), + ('gidNumber', [Utils.to_bytes(str(gid))]), + ('sAMAccountName', [Utils.to_bytes(group_name)]), + ('cn', [Utils.to_bytes(group_name)]) + ] + + if existing: + modify_group_attrs = [] + for group_attr in group_attrs: + modify_group_attr = list(group_attr) + modify_group_attr.insert(0, ldap.MOD_REPLACE) + modify_group_attrs.append(tuple(modify_group_attr)) + self.modify_s(group_dn, modify_group_attrs) + else: + self.add_s(group_dn, group_attrs) + self.wait_for_group_creation(group_name) + + return self.get_group(group_name) + + def add_sudo_user(self, username: str): + user_dn = self.build_user_dn(username) + sudoer_group_dn = self.ldap_sudoers_group_dn + + group_attrs = [ + (ldap.MOD_ADD, 'member', [Utils.to_bytes(user_dn)]) + ] + + self.modify_s(sudoer_group_dn, group_attrs) + + def remove_sudo_user(self, username: str): + user_dn = self.build_user_dn(username) + sudoer_group_dn = self.ldap_sudoers_group_dn + + group_attrs = [ + (ldap.MOD_DELETE, 'member', [Utils.to_bytes(user_dn)]) + ] + + self.modify_s(sudoer_group_dn, group_attrs) + + def wait_for_group_creation(self, group_name: str, interval: float = 2, max_attempts: int = 15, min_success: int = 3): + """ + wait for group to be replicated across domain controllers. + :param group_name: + :param interval: + :param max_attempts: + :param min_success: + :return: + """ + current_attempt = 0 + current_success = 0 + while current_attempt < max_attempts: + if not self.is_existing_group(group_name): + current_attempt += 1 + else: + current_success += 1 + if current_success == min_success: + break + time.sleep(interval) + + def wait_for_user_creation(self, username: str, interval: float = 2, max_attempts: int = 15, min_success: int = 3): + """ + wait for user to be replicated across domain controllers. + :param username: + :param interval: + :param max_attempts: + :param min_success: + :return: + """ + current_attempt = 0 + current_success = 0 + while current_attempt < max_attempts: + if not self.is_existing_user(username): + current_attempt += 1 + else: + current_success += 1 + if current_success == min_success: + break + time.sleep(interval) diff --git a/source/idea/idea-cluster-manager/src/ideaclustermanager/app/accounts/ldapclient/ldap_client_factory.py b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/accounts/ldapclient/ldap_client_factory.py new file mode 100644 index 00000000..db7b087a --- /dev/null +++ b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/accounts/ldapclient/ldap_client_factory.py @@ -0,0 +1,72 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +from ideasdk.context import SocaContext +from ideadatamodel import constants, exceptions + +from ideaclustermanager.app.accounts.ldapclient import AbstractLDAPClient, LdapClientOptions, OpenLDAPClient, ActiveDirectoryClient + +from typing import TypeVar + +AbstractLDAPClientType = TypeVar('AbstractLDAPClientType', bound=AbstractLDAPClient) + + +def build_ldap_client(context: SocaContext, logger=None) -> AbstractLDAPClientType: + ds_provider = context.config().get_string('directoryservice.provider', required=True) + if ds_provider == constants.DIRECTORYSERVICE_OPENLDAP: + + ldap_connection_uri = context.config().get_string('directoryservice.ldap_connection_uri', required=True) + domain_name = context.config().get_string('directoryservice.name', required=True) + ds_root_username = context.config().get_secret('directoryservice.root_username_secret_arn', required=True) + ds_root_password = context.config().get_secret('directoryservice.root_password_secret_arn', required=True) + + return OpenLDAPClient( + context=context, + options=LdapClientOptions( + uri=ldap_connection_uri, + domain_name=domain_name, + root_username=ds_root_username, + root_password=ds_root_password + ), + logger=logger + ) + + elif ds_provider in {constants.DIRECTORYSERVICE_AWS_MANAGED_ACTIVE_DIRECTORY, constants.DIRECTORYSERVICE_ACTIVE_DIRECTORY}: + + domain_name = context.config().get_string('directoryservice.name', required=True) + if ds_provider == constants.DIRECTORYSERVICE_AWS_MANAGED_ACTIVE_DIRECTORY: + directory_id = context.config().get_string('directoryservice.directory_id', required=True) + else: + # Self-managed AD does not have a directory_id + directory_id = None + ad_netbios = context.config().get_string('directoryservice.ad_short_name', required=True) + ldap_connection_uri = context.config().get_string('directoryservice.ldap_connection_uri', required=True) + ds_root_username = context.config().get_secret('directoryservice.root_username_secret_arn', required=True) + ds_root_password = context.config().get_secret('directoryservice.root_password_secret_arn', required=True) + password_max_age = context.config().get_int('directoryservice.password_max_age', required=True) + + return ActiveDirectoryClient( + context=context, + options=LdapClientOptions( + uri=ldap_connection_uri, + domain_name=domain_name, + root_username=ds_root_username, + root_password=ds_root_password, + ad_netbios=ad_netbios, + directory_id=directory_id, + password_max_age=password_max_age + ), + logger=logger + ) + + else: + + raise exceptions.general_exception(f'directory service provider: {ds_provider} not supported') diff --git a/source/idea/idea-cluster-manager/src/ideaclustermanager/app/accounts/ldapclient/ldap_utils.py b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/accounts/ldapclient/ldap_utils.py new file mode 100644 index 00000000..bf6c8b5c --- /dev/null +++ b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/accounts/ldapclient/ldap_utils.py @@ -0,0 +1,79 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +from ideasdk.utils import Utils + +from typing import Optional, Dict, List +import crypt + +class LdapUtils: + + @staticmethod + def get_string_value(key: str, ldap_object: Dict) -> Optional[str]: + """ + utility method to extract string value from ldap object + """ + value_list = Utils.get_value_as_list(key, ldap_object) + if Utils.is_empty(value_list): + return None + value_bytes = value_list[0] + return Utils.from_bytes(value_bytes) + + @staticmethod + def get_string_list(key: str, ldap_object: Dict) -> Optional[List[str]]: + """ + utility method to extract string list value from ldap object + """ + value_list = Utils.get_value_as_list(key, ldap_object) + if Utils.is_empty(value_list): + return None + result = [] + for value_bytes in value_list: + result.append(Utils.from_bytes(value_bytes)) + return result + + @staticmethod + def get_int_value(key: str, ldap_object: Dict) -> Optional[int]: + """ + utility method to extract string list value from ldap object + """ + value_list = Utils.get_value_as_list(key, ldap_object) + if Utils.is_empty(value_list): + return None + value_bytes = value_list[0] + return Utils.get_as_int(Utils.from_bytes(value_bytes)) + + @staticmethod + def get_int_list(key: str, ldap_object: Dict) -> Optional[List[int]]: + """ + utility method to extract int list value from ldap object + """ + value_list = Utils.get_value_as_list(key, ldap_object) + if Utils.is_empty(value_list): + return None + result = [] + for value_bytes in value_list: + value = Utils.from_bytes(value_bytes) + int_value = Utils.get_as_int(value) + if int_value is None: + continue + result.append(int_value) + return result + + @staticmethod + def encrypt_password(password: str) -> str: + """ + utility method to encrypt user password + used during: create user, change password, respond to auth challenge flows + :return: encrypted user password suitable for OpenLDAP userPassword attribute + """ + crypt_string = crypt.crypt(word=password, salt=crypt.mksalt(method=crypt.METHOD_SHA512, rounds=10_000)) + return f'{{CRYPT}}{crypt_string}' diff --git a/source/idea/idea-cluster-manager/src/ideaclustermanager/app/accounts/ldapclient/openldap_client.py b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/accounts/ldapclient/openldap_client.py new file mode 100644 index 00000000..e1b60de0 --- /dev/null +++ b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/accounts/ldapclient/openldap_client.py @@ -0,0 +1,159 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +from ideasdk.context import SocaContext +from ideasdk.utils import Utils +from ideadatamodel import exceptions + +from ideaclustermanager.app.accounts.ldapclient.abstract_ldap_client import AbstractLDAPClient, LdapClientOptions +from ideaclustermanager.app.accounts.ldapclient.ldap_utils import LdapUtils + +from typing import Dict +import ldap # noqa + + +class OpenLDAPClient(AbstractLDAPClient): + + def __init__(self, context: SocaContext, options: LdapClientOptions, logger=None): + if logger is None: + logger = context.logger('openldap-client') + super().__init__(context, options, logger=logger) + + @property + def ldap_root_bind(self) -> str: + return f'cn={self.ldap_root_username},{self.ldap_base}' + + @property + def ldap_user_base(self) -> str: + return f'ou=People,{self.ldap_base}' + + @property + def ldap_user_filterstr(self) -> str: + return f'(objectClass=posixAccount)' + + @property + def ldap_group_base(self) -> str: + return f'ou=Group,{self.ldap_base}' + + @property + def ldap_group_filterstr(self) -> str: + return f'(objectClass=posixGroup)' + + def build_group_filterstr(self, group_name: str = None, username: str = None) -> str: + filterstr = self.ldap_group_filterstr + if group_name is not None: + filterstr = f'{filterstr}(cn={group_name})' + if username is not None: + filterstr = f'{filterstr}(memberUid={username})' + return f'(&{filterstr})' + + @property + def ldap_sudoers_base(self) -> str: + return f'ou=Sudoers,{self.ldap_base}' + + @property + def ldap_sudoer_filterstr(self) -> str: + return f'(objectClass=sudoRole)' + + def build_sudoer_dn(self, username: str) -> str: + return f'cn={username},{self.ldap_sudoers_base}' + + def change_password(self, username: str, password: str): + encrypted_password = LdapUtils.encrypt_password(password) + user_dn = self.build_user_dn(username) + user_attrs = [ + (ldap.MOD_REPLACE, 'userPassword', [Utils.to_bytes(encrypted_password)]) + ] + + self.modify_s(user_dn, user_attrs) + + def create_service_account(self, username: str, password: str): + raise exceptions.general_exception('not supported for OpenLDAP') + + def sync_user(self, *, + uid: int, + gid: int, + username: str, + email: str, + login_shell: str, + home_dir: str) -> Dict: + user_dn = self.build_user_dn(username) + user_attrs = [ + ('objectClass', [ + Utils.to_bytes('top'), + Utils.to_bytes('person'), + Utils.to_bytes('posixAccount'), + Utils.to_bytes('shadowAccount'), + Utils.to_bytes('inetOrgPerson'), + Utils.to_bytes('organizationalPerson') + ]), + ('uid', [Utils.to_bytes(username)]), + ('uidNumber', [Utils.to_bytes(str(uid))]), + ('gidNumber', [Utils.to_bytes(str(gid))]), + ('mail', [Utils.to_bytes(email)]), + ('cn', [Utils.to_bytes(username)]), + ('sn', [Utils.to_bytes(username)]), + ('loginShell', [Utils.to_bytes(login_shell)]), + ('homeDirectory', [Utils.to_bytes(home_dir)]) + ] + + if self.is_existing_user(username): + modify_user_attrs = [] + for user_attr in user_attrs: + modify_user_attr = list(user_attr) + modify_user_attr.insert(0, ldap.MOD_REPLACE) + modify_user_attrs.append(tuple(modify_user_attr)) + self.modify_s(user_dn, modify_user_attrs) + else: + self.add_s(user_dn, user_attrs) + + return self.get_user(username) + + def sync_group(self, group_name: str, gid: int) -> Dict: + group_dn = self.build_group_dn(group_name) + group_attrs = [ + ('objectClass', [ + Utils.to_bytes('top'), + Utils.to_bytes('posixGroup') + ]), + ('gidNumber', [Utils.to_bytes(str(gid))]), + ('cn', [Utils.to_bytes(group_name)]) + ] + + if self.is_existing_group(group_name): + modify_group_attrs = [] + for group_attr in group_attrs: + modify_group_attr = list(group_attr) + modify_group_attr.insert(0, ldap.MOD_REPLACE) + modify_group_attrs.append(tuple(modify_group_attr)) + self.modify_s(group_dn, modify_group_attrs) + else: + self.add_s(group_dn, group_attrs) + + return self.get_group(group_name) + + def add_sudo_user(self, username: str): + user_dn = self.build_sudoer_dn(username) + user_attrs = [ + ('objectClass', [ + Utils.to_bytes('top'), + Utils.to_bytes('sudoRole') + ]), + ('sudoHost', [Utils.to_bytes('ALL')]), + ('sudoUser', [Utils.to_bytes(username)]), + ('sudoCommand', [Utils.to_bytes('ALL')]) + ] + + self.add_s(user_dn, user_attrs) + + def remove_sudo_user(self, username: str): + user_dn = self.build_sudoer_dn(username) + self.delete_s(user_dn) diff --git a/source/idea/idea-cluster-manager/src/ideaclustermanager/app/accounts/user_home_directory.py b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/accounts/user_home_directory.py new file mode 100644 index 00000000..39ad334d --- /dev/null +++ b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/accounts/user_home_directory.py @@ -0,0 +1,202 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +from ideasdk.context import SocaContext +from ideadatamodel.auth import User +from ideasdk.utils import Utils +from ideadatamodel import exceptions + +import time +import os +import shutil +import pwd +from cryptography.hazmat.primitives import serialization as crypto_serialization +from cryptography.hazmat.primitives.asymmetric import rsa +from cryptography.hazmat.backends import default_backend as crypto_default_backend + +from ideaclustermanager.app.accounts.auth_constants import USER_HOME_DIR_BASE +from ideasdk.shell import ShellInvoker + + +class UserHomeDirectory: + + def __init__(self, context: SocaContext, user: User): + self._context = context + self._logger = context.logger('user-home-init') + self.user = user + self._shell = ShellInvoker(self._logger) + + @property + def home_dir(self) -> str: + return self.user.home_dir + + @property + def ssh_dir(self) -> str: + return os.path.join(self.user.home_dir, '.ssh') + + def own_path(self, path: str): + shutil.chown(path, user=self.user.username, group=Utils.get_as_int(pwd.getpwnam(self.user.username).pw_gid, default=0)) + + def initialize_ssh_dir(self): + os.makedirs(self.ssh_dir, exist_ok=True) + os.chmod(self.ssh_dir, 0o700) + self.own_path(self.ssh_dir) + + id_rsa_file = os.path.join(self.ssh_dir, 'id_rsa') + + # if an existing id_rsa file already exists, return + if Utils.is_file(id_rsa_file): + return + + key = rsa.generate_private_key( + backend=crypto_default_backend(), + public_exponent=65537, + key_size=2048 + ) + private_key = key.private_bytes( + crypto_serialization.Encoding.PEM, + crypto_serialization.PrivateFormat.TraditionalOpenSSL, + crypto_serialization.NoEncryption()) + public_key = key.public_key().public_bytes( + crypto_serialization.Encoding.OpenSSH, + crypto_serialization.PublicFormat.OpenSSH + ) + + with open(id_rsa_file, 'w') as f: + f.write(Utils.from_bytes(private_key)) + os.chmod(id_rsa_file, 0o600) + self.own_path(id_rsa_file) + + id_rsa_pub_file = os.path.join(self.ssh_dir, 'id_rsa.pub') + with open(id_rsa_pub_file, 'w') as f: + f.write(Utils.from_bytes(public_key)) + self.own_path(id_rsa_pub_file) + + authorized_keys_file = os.path.join(self.ssh_dir, 'authorized_keys') + with open(authorized_keys_file, 'w') as f: + f.write(Utils.from_bytes(public_key)) + os.chmod(id_rsa_file, 0o600) + self.own_path(authorized_keys_file) + + def initialize_home_dir(self): + os.makedirs(self.home_dir, exist_ok=True) + os.chmod(self.home_dir, 0o700) + self.own_path(self.home_dir) + + skeleton_files = [ + '/etc/skel/.bashrc', + '/etc/skel/.bash_profile', + '/etc/skel/.bash_logout' + ] + + for src_file in skeleton_files: + dest_file = os.path.join(self.home_dir, os.path.basename(src_file)) + if Utils.is_file(dest_file): + continue + shutil.copy(src_file, dest_file) + self.own_path(dest_file) + + def initialize(self): + # wait for system to sync the newly created user by using system libraries to resolve the user + # this happens on a fresh installation of auth-server, where all system services have just started + # and a new clusteradmin user is created. + # although the user is created in directory services, it's not yet synced with the local system + # If you continue to see this log message it may indicate that the underlying cluster-manager + # host is not probably linked to the back-end directory service in some fashion. + while True: + try: + pwd.getpwnam(self.user.username) + break + except KeyError: + self._logger.info(f'{self.user.username} not available yet. waiting for user to be synced ...') + time.sleep(5) + + self.initialize_home_dir() + self.initialize_ssh_dir() + + def archive(self): + if not Utils.is_dir(self.home_dir): + return + + archive_dir = os.path.join(USER_HOME_DIR_BASE, f'{self.user.username}_{Utils.file_system_friendly_timestamp()}') + shutil.move(self.home_dir, archive_dir) + os.chown(archive_dir, 0, 0) + os.chmod(archive_dir, 0o700) + + @staticmethod + def validate_and_sanitize_key_format(key_format: str) -> str: + if Utils.is_empty(key_format): + raise exceptions.invalid_params('key_format is required') + key_format = key_format.strip().lower() + if key_format not in ('ppk', 'pem'): + raise exceptions.invalid_params('key_format must be one of [pem, ppk]') + return key_format + + def get_key_name(self, key_format: str) -> str: + key_format = self.validate_and_sanitize_key_format(key_format) + cluster_name = self._context.cluster_name() + username = self.user.username + return f'{username}_{cluster_name}_privatekey.{key_format}' + + def get_key_material(self, key_format: str, platform: str = 'linux') -> str: + """ + read the user's private key from user's ~/.ssh directory + add \r\n (for platform = 'windows) or \n (for platform = 'linux' or 'osx') + :param key_format: one of [ppk, pem] + :param platform: one of [windows, linux, osx] + :return: private key content + """ + + if Utils.is_empty(platform): + platform = 'linux' + + id_rsa_file = os.path.join(self.ssh_dir, 'id_rsa') + if not Utils.is_empty(id_rsa_file): + exceptions.general_exception(f'private key not found in home directory for user: {self.user.username}') + + key_format = self.validate_and_sanitize_key_format(key_format) + + def read_private_key_content(file: str) -> str: + with open(file, 'r') as f: + private_key_content = f.read() + + lines = private_key_content.splitlines() + if platform in ('linux', 'osx'): + return '\n'.join(lines) + elif platform == 'windows': + return '\r\n'.join(lines) + + # default + return '\n'.join(lines) + + if key_format == 'pem': + return read_private_key_content(id_rsa_file) + + elif key_format == 'ppk': + result = self._shell.invoke('command -v puttygen', shell=True) + if result.returncode != 0: + raise exceptions.general_exception('puttygen binary not found in PATH') + putty_gen_bin = result.stdout + if not os.access(putty_gen_bin, os.X_OK): + os.chmod(putty_gen_bin, 0o700) + + id_rsa_ppk_file = os.path.join(self.ssh_dir, 'id_rsa.ppk') + if not Utils.is_file(id_rsa_ppk_file): + result = self._shell.invoke([ + 'su', + self.user.username, + '-c', + f'{putty_gen_bin} {id_rsa_file} -o {id_rsa_ppk_file}' + ]) + if result.returncode != 0: + raise exceptions.general_exception(f'failed to generate .ppk file: {result}') + + return read_private_key_content(id_rsa_ppk_file) diff --git a/source/idea/idea-cluster-manager/src/ideaclustermanager/app/api/__init__.py b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/api/__init__.py new file mode 100644 index 00000000..6d8d18a3 --- /dev/null +++ b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/api/__init__.py @@ -0,0 +1,10 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. diff --git a/source/idea/idea-cluster-manager/src/ideaclustermanager/app/api/accounts_api.py b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/api/accounts_api.py new file mode 100644 index 00000000..153f3d4e --- /dev/null +++ b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/api/accounts_api.py @@ -0,0 +1,344 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +from ideasdk.api import BaseAPI, ApiInvocationContext +from ideadatamodel.auth import ( + CreateUserRequest, + CreateUserResult, + GetUserRequest, + GetUserResult, + ModifyUserRequest, + ModifyUserResult, + EnableUserRequest, + EnableUserResult, + DisableUserRequest, + DisableUserResult, + DeleteUserRequest, + DeleteUserResult, + ListUsersRequest, + ResetPasswordRequest, + ResetPasswordResult, + CreateGroupRequest, + CreateGroupResult, + ModifyGroupRequest, + ModifyGroupResult, + DeleteGroupRequest, + DeleteGroupResult, + EnableGroupRequest, + EnableGroupResult, + DisableGroupRequest, + DisableGroupResult, + GetGroupRequest, + GetGroupResult, + ListGroupsRequest, + AddUserToGroupRequest, + AddUserToGroupResult, + RemoveUserFromGroupRequest, + RemoveUserFromGroupResult, + ListUsersInGroupRequest, + AddSudoUserRequest, + AddSudoUserResult, + RemoveSudoUserRequest, + GlobalSignOutRequest, + GlobalSignOutResult +) +from ideadatamodel import exceptions +from ideasdk.utils import Utils + +import ideaclustermanager + + +class AccountsAPI(BaseAPI): + + def __init__(self, context: ideaclustermanager.AppContext): + self.context = context + + self.SCOPE_WRITE = f'{self.context.module_id()}/write' + self.SCOPE_READ = f'{self.context.module_id()}/read' + + self.acl = { + 'Accounts.CreateUser': { + 'scope': self.SCOPE_WRITE, + 'method': self.create_user + }, + 'Accounts.GetUser': { + 'scope': self.SCOPE_READ, + 'method': self.get_user + }, + 'Accounts.ModifyUser': { + 'scope': self.SCOPE_WRITE, + 'method': self.modify_user + }, + 'Accounts.ListUsers': { + 'scope': self.SCOPE_READ, + 'method': self.list_users + }, + 'Accounts.EnableUser': { + 'scope': self.SCOPE_WRITE, + 'method': self.enable_user + }, + 'Accounts.DisableUser': { + 'scope': self.SCOPE_WRITE, + 'method': self.disable_user + }, + 'Accounts.DeleteUser': { + 'scope': self.SCOPE_WRITE, + 'method': self.delete_user + }, + 'Accounts.CreateGroup': { + 'scope': self.SCOPE_WRITE, + 'method': self.create_group + }, + 'Accounts.GetGroup': { + 'scope': self.SCOPE_READ, + 'method': self.get_group + }, + 'Accounts.ModifyGroup': { + 'scope': self.SCOPE_WRITE, + 'method': self.modify_group + }, + 'Accounts.EnableGroup': { + 'scope': self.SCOPE_WRITE, + 'method': self.enable_group + }, + 'Accounts.DisableGroup': { + 'scope': self.SCOPE_WRITE, + 'method': self.disable_group + }, + 'Accounts.DeleteGroup': { + 'scope': self.SCOPE_WRITE, + 'method': self.delete_group + }, + 'Accounts.AddUserToGroup': { + 'scope': self.SCOPE_WRITE, + 'method': self.add_user_to_group + }, + 'Accounts.RemoveUserFromGroup': { + 'scope': self.SCOPE_WRITE, + 'method': self.remove_user_from_group + }, + 'Accounts.ListGroups': { + 'scope': self.SCOPE_READ, + 'method': self.list_groups + }, + 'Accounts.ListUsersInGroup': { + 'scope': self.SCOPE_READ, + 'method': self.list_users_in_group + }, + 'Accounts.AddSudoUser': { + 'scope': self.SCOPE_WRITE, + 'method': self.add_sudo_user + }, + 'Accounts.RemoveSudoUser': { + 'scope': self.SCOPE_WRITE, + 'method': self.remove_sudo_user + }, + 'Accounts.GlobalSignOut': { + 'scope': self.SCOPE_WRITE, + 'method': self.global_sign_out + }, + 'Accounts.ResetPassword': { + 'scope': self.SCOPE_WRITE, + 'method': self.reset_password + } + } + + @property + def token_service(self): + return self.context.token_service + + def is_applicable(self, context: ApiInvocationContext, scope: str) -> bool: + access_token = context.access_token + decoded_token = self.token_service.decode_token(access_token) + + groups = Utils.get_value_as_list('cognito:groups', decoded_token) + if Utils.is_not_empty(groups) and self.context.user_pool.admin_group_name in groups: + return True + + token_scope = Utils.get_value_as_string('scope', decoded_token) + if Utils.is_empty(token_scope): + return False + return scope in token_scope.split(' ') + + def create_user(self, context: ApiInvocationContext): + request = context.get_request_payload_as(CreateUserRequest) + if Utils.is_empty(request.user): + raise exceptions.invalid_params('user is required') + + email_verified = Utils.get_as_bool(request.email_verified, False) + + created_user = self.context.accounts.create_user(request.user, email_verified) + context.success(CreateUserResult( + user=created_user + )) + + def get_user(self, context: ApiInvocationContext): + request = context.get_request_payload_as(GetUserRequest) + user = self.context.accounts.get_user(request.username) + context.success(GetUserResult(user=user)) + + def modify_user(self, context: ApiInvocationContext): + request = context.get_request_payload_as(ModifyUserRequest) + email_verified = Utils.get_as_bool(request.email_verified, False) + user = self.context.accounts.modify_user(request.user, email_verified) + context.success(ModifyUserResult( + user=user + )) + + def enable_user(self, context: ApiInvocationContext): + request = context.get_request_payload_as(EnableUserRequest) + self.context.accounts.enable_user(request.username) + context.success(EnableUserResult()) + + def disable_user(self, context: ApiInvocationContext): + request = context.get_request_payload_as(DisableUserRequest) + self.context.accounts.disable_user(request.username) + context.success(DisableUserResult()) + + def delete_user(self, context: ApiInvocationContext): + request = context.get_request_payload_as(DeleteUserRequest) + self.context.accounts.delete_user(request.username) + context.success(DeleteUserResult()) + + def list_users(self, context: ApiInvocationContext): + request = context.get_request_payload_as(ListUsersRequest) + result = self.context.accounts.list_users(request) + context.success(result) + + def global_sign_out(self, context: ApiInvocationContext): + request = context.get_request_payload_as(GlobalSignOutRequest) + + self.context.accounts.global_sign_out(username=request.username) + + context.success(GlobalSignOutResult()) + + def reset_password(self, context: ApiInvocationContext): + request = context.get_request_payload_as(ResetPasswordRequest) + self.context.accounts.reset_password(request.username) + context.success(ResetPasswordResult()) + + def create_group(self, context: ApiInvocationContext): + request = context.get_request_payload_as(CreateGroupRequest) + created_group = self.context.accounts.create_group(request.group) + context.success(CreateGroupResult( + group=created_group + )) + + def modify_group(self, context: ApiInvocationContext): + request = context.get_request_payload_as(ModifyGroupRequest) + modified_group = self.context.accounts.modify_group(request.group) + context.success(ModifyGroupResult( + group=modified_group + )) + + def enable_group(self, context: ApiInvocationContext): + request = context.get_request_payload_as(EnableGroupRequest) + self.context.accounts.enable_group(request.group_name) + context.success(EnableGroupResult()) + + def disable_group(self, context: ApiInvocationContext): + request = context.get_request_payload_as(DisableGroupRequest) + self.context.accounts.disable_group(request.group_name) + context.success(DisableGroupResult()) + + def delete_group(self, context: ApiInvocationContext): + request = context.get_request_payload_as(DeleteGroupRequest) + self.context.accounts.delete_group(request.group_name) + context.success(DeleteGroupResult()) + + def get_group(self, context: ApiInvocationContext): + request = context.get_request_payload_as(GetGroupRequest) + group = self.context.accounts.get_group(request.group_name) + context.success(GetGroupResult( + group=group + )) + + def list_groups(self, context: ApiInvocationContext): + request = context.get_request_payload_as(ListGroupsRequest) + result = self.context.accounts.list_groups(request) + context.success(result) + + def add_user_to_group(self, context: ApiInvocationContext): + request = context.get_request_payload_as(AddUserToGroupRequest) + self.context.accounts.add_users_to_group(request.usernames, request.group_name) + context.success(AddUserToGroupResult()) + + def remove_user_from_group(self, context: ApiInvocationContext): + request = context.get_request_payload_as(RemoveUserFromGroupRequest) + self.context.accounts.remove_users_from_group(request.usernames, request.group_name) + context.success(RemoveUserFromGroupResult()) + + def list_users_in_group(self, context: ApiInvocationContext): + request = context.get_request_payload_as(ListUsersInGroupRequest) + result = self.context.accounts.list_users_in_group(request) + context.success(result) + + def add_sudo_user(self, context: ApiInvocationContext): + request = context.get_request_payload_as(AddSudoUserRequest) + self.context.accounts.add_sudo_user(request.username) + user = self.context.accounts.get_user(request.username) + context.success(AddSudoUserResult( + user=user + )) + + def remove_sudo_user(self, context: ApiInvocationContext): + request = context.get_request_payload_as(RemoveSudoUserRequest) + self.context.accounts.remove_sudo_user(request.username) + user = self.context.accounts.get_user(request.username) + context.success(AddSudoUserResult( + user=user + )) + + @staticmethod + def check_sudo_authorization_services(context: ApiInvocationContext): + """ + only a sudo user should be able to create additional sudo users or perform + sudo user related operations + a "manager" user should be able to add new users, but without sudo access + a sudo user cannot be created using client credentials grant scope. needs a sudo "user" to create a sudo user. + """ + + namespace = context.namespace + if namespace == 'Accounts.CreateUser': + payload = context.get_request_payload_as(CreateUserRequest) + is_sudo_authorization_required = payload.user is not None and Utils.is_true(payload.user.sudo) + else: + is_sudo_authorization_required = namespace in ( + 'Accounts.AddSudoUser', + 'Accounts.RemoveSudoUser', + 'Accounts.ModifyUser' + ) + + if not is_sudo_authorization_required: + return + + if context.is_administrator(): + return + + raise exceptions.unauthorized_access() + + def invoke(self, context: ApiInvocationContext): + namespace = context.namespace + + acl_entry = Utils.get_value_as_dict(namespace, self.acl) + if acl_entry is None: + raise exceptions.unauthorized_access() + + # special check only applicable for sudo users related functionality + self.check_sudo_authorization_services(context) + + acl_entry_scope = Utils.get_value_as_string('scope', acl_entry) + is_authorized = context.is_authorized(elevated_access=True, scopes=[acl_entry_scope]) + + if is_authorized: + acl_entry['method'](context) + else: + raise exceptions.unauthorized_access() diff --git a/source/idea/idea-cluster-manager/src/ideaclustermanager/app/api/analytics_api.py b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/api/analytics_api.py new file mode 100644 index 00000000..fe22d358 --- /dev/null +++ b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/api/analytics_api.py @@ -0,0 +1,61 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +import ideaclustermanager + +from ideasdk.api import ApiInvocationContext, BaseAPI +from ideadatamodel.analytics import ( + OpenSearchQueryRequest, + OpenSearchQueryResult +) +from ideadatamodel import exceptions +from ideasdk.utils import Utils + + +class AnalyticsAPI(BaseAPI): + """ + Analytics API to query opensearch cluster. + + this is a stop-gap API and ideally, each module should implement their own version of Analytics API, + scoped to the indices or aliases exposed by a particular module. + + any write or index settings apis must not be exposed via this class. + an AnalyticsAdminAPI should be exposed with elevated access to enable such functionality. + + invocation simply checks if the invocation is authenticated (valid token) + """ + + def __init__(self, context: ideaclustermanager.AppContext): + self.context = context + + def opensearch_query(self, context: ApiInvocationContext): + """ + Send Raw ElasticSearch Query and Search Request + """ + request = context.get_request_payload_as(OpenSearchQueryRequest) + if Utils.is_empty(request.data): + raise exceptions.invalid_params('data is required') + + result = self.context.analytics_service().os_client.os_client.search( + **request.data + ) + + context.success(OpenSearchQueryResult( + data=result + )) + + def invoke(self, context: ApiInvocationContext): + if not context.is_authenticated(): + raise exceptions.unauthorized_access() + + namespace = context.namespace + if namespace == 'Analytics.OpenSearchQuery': + self.opensearch_query(context) diff --git a/source/idea/idea-cluster-manager/src/ideaclustermanager/app/api/api_invoker.py b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/api/api_invoker.py new file mode 100644 index 00000000..2804a01f --- /dev/null +++ b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/api/api_invoker.py @@ -0,0 +1,168 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +import ideaclustermanager +from ideasdk.api import ApiInvocationContext +from ideasdk.protocols import ApiInvokerProtocol +from ideasdk.auth import TokenService +from ideasdk.utils import Utils +from ideadatamodel.auth import ( + CreateUserRequest, + InitiateAuthRequest, + InitiateAuthResult, + RespondToAuthChallengeRequest, + RespondToAuthChallengeResult, + ChangePasswordRequest, + ConfirmForgotPasswordRequest, + GetUserPrivateKeyResult, + SignOutRequest +) +from ideadatamodel.filesystem import ( + ReadFileResult, + TailFileResult +) + +from ideasdk.app import SocaAppAPI +from ideasdk.filesystem.filebrowser_api import FileBrowserAPI +from ideaclustermanager.app.api.cluster_settings_api import ClusterSettingsAPI +from ideaclustermanager.app.api.analytics_api import AnalyticsAPI +from ideaclustermanager.app.api.projects_api import ProjectsAPI +from ideaclustermanager.app.api.accounts_api import AccountsAPI +from ideaclustermanager.app.api.auth_api import AuthAPI +from ideaclustermanager.app.api.email_templates_api import EmailTemplatesAPI + +from typing import Optional, Dict + + +class ClusterManagerApiInvoker(ApiInvokerProtocol): + + def __init__(self, context: ideaclustermanager.AppContext): + self._context = context + self.app_api = SocaAppAPI(context) + self.file_browser_api = FileBrowserAPI(context) + self.cluster_settings_api = ClusterSettingsAPI(context) + self.analytics_api = AnalyticsAPI(context) + self.projects_api = ProjectsAPI(context) + self.auth_api = AuthAPI(context) + self.accounts_api = AccountsAPI(context) + self.email_templates_api = EmailTemplatesAPI(context) + + def get_token_service(self) -> Optional[TokenService]: + return self._context.token_service + + def get_request_logging_payload(self, context: ApiInvocationContext) -> Optional[Dict]: + namespace = context.namespace + + if namespace == 'Auth.InitiateAuth': + request = context.get_request(deep_copy=True) + payload = context.get_request_payload_as(InitiateAuthRequest) + if payload.password is not None: + payload.password = '*****' + if payload.authorization_code is not None: + payload.authorization_code = '*****' + request['payload'] = Utils.to_dict(payload) + return request + elif namespace == 'Auth.SignOut': + request = context.get_request(deep_copy=True) + payload = context.get_request_payload_as(SignOutRequest) + payload.refresh_token = '*****' + request['payload'] = Utils.to_dict(payload) + return request + elif namespace == 'Auth.RespondToAuthChallenge': + request = context.get_request(deep_copy=True) + payload = context.get_request_payload_as(RespondToAuthChallengeRequest) + payload.new_password = '*****' + request['payload'] = Utils.to_dict(payload) + return request + elif namespace == 'Auth.ConfirmForgotPassword': + request = context.get_request(deep_copy=True) + payload = context.get_request_payload_as(ConfirmForgotPasswordRequest) + payload.password = '*****' + request['payload'] = Utils.to_dict(payload) + return request + elif namespace == 'Auth.ChangePassword': + request = context.get_request(deep_copy=True) + payload = context.get_request_payload_as(ChangePasswordRequest) + payload.old_password = '*****' + payload.new_password = '*****' + request['payload'] = Utils.to_dict(payload) + return request + elif namespace == 'Accounts.CreateUser': + request = context.get_request(deep_copy=True) + payload = context.get_request_payload_as(CreateUserRequest) + if payload.user is not None and Utils.is_not_empty(payload.user.password): + payload.user.password = '*****' + request['payload'] = Utils.to_dict(payload) + return request + + def get_response_logging_payload(self, context: ApiInvocationContext) -> Optional[Dict]: + if not context.is_success(): + return None + + namespace = context.namespace + if namespace == 'Auth.InitiateAuth': + response = context.get_response(deep_copy=True) + payload = context.get_response_payload_as(InitiateAuthResult) + if payload.auth is not None: + payload.auth.access_token = '*****' + if payload.auth.id_token is not None: + payload.auth.id_token = '*****' + if payload.auth.refresh_token is not None: + payload.auth.refresh_token = '*****' + response['payload'] = Utils.to_dict(payload) + return response + elif namespace == 'Auth.RespondToAuthChallenge': + response = context.get_response(deep_copy=True) + payload = context.get_response_payload_as(RespondToAuthChallengeResult) + if payload.auth is not None: + payload.auth.access_token = '*****' + if payload.auth.refresh_token is not None: + payload.auth.refresh_token = '*****' + response['payload'] = Utils.to_dict(payload) + return response + elif namespace == 'Auth.GetUserPrivateKey': + response = context.get_response(deep_copy=True) + payload = context.get_response_payload_as(GetUserPrivateKeyResult) + key_size = Utils.get_as_int(payload.key_material, default=0) + payload.key_material = f'**key_material_length=={key_size}**' + response['payload'] = Utils.to_dict(payload) + return response + elif namespace == 'FileBrowser.ReadFile': + response = context.get_response(deep_copy=True) + payload = context.get_response_payload_as(ReadFileResult) + payload.content = '****' + response['payload'] = Utils.to_dict(payload) + return response + elif namespace == 'FileBrowser.TailFile': + response = context.get_response(deep_copy=True) + payload = context.get_response_payload_as(TailFileResult) + payload.lines = ['****'] + response['payload'] = Utils.to_dict(payload) + return response + + def invoke(self, context: ApiInvocationContext): + namespace = context.namespace + if namespace.startswith('Auth.'): + self.auth_api.invoke(context) + elif namespace.startswith('App.'): + return self.app_api.invoke(context) + elif namespace.startswith('FileBrowser.'): + self.file_browser_api.invoke(context) + elif namespace.startswith('ClusterSettings.'): + self.cluster_settings_api.invoke(context) + elif namespace.startswith('Analytics.'): + self.analytics_api.invoke(context) + elif namespace.startswith('Projects.'): + self.projects_api.invoke(context) + elif namespace.startswith('Accounts.'): + self.accounts_api.invoke(context) + elif namespace.startswith('EmailTemplates.'): + self.email_templates_api.invoke(context) diff --git a/source/idea/idea-cluster-manager/src/ideaclustermanager/app/api/auth_api.py b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/api/auth_api.py new file mode 100644 index 00000000..d6265029 --- /dev/null +++ b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/api/auth_api.py @@ -0,0 +1,204 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +from ideasdk.api import BaseAPI, ApiInvocationContext +from ideadatamodel.auth import ( + GetUserResult, + InitiateAuthRequest, + RespondToAuthChallengeRequest, + ForgotPasswordRequest, + ForgotPasswordResult, + ChangePasswordRequest, + ChangePasswordResult, + ConfirmForgotPasswordRequest, + ConfirmForgotPasswordResult, + GetGroupResult, + AddUserToGroupRequest, + AddUserToGroupResult, + RemoveUserFromGroupRequest, + RemoveUserFromGroupResult, + ListUsersInGroupRequest, + GetUserPrivateKeyRequest, + GetUserPrivateKeyResult, + SignOutRequest, + SignOutResult, + GlobalSignOutRequest, + GlobalSignOutResult +) +from ideadatamodel import exceptions +from ideasdk.utils import Utils + +import ideaclustermanager +from ideaclustermanager.app.accounts.user_home_directory import UserHomeDirectory + + +class AuthAPI(BaseAPI): + + def __init__(self, context: ideaclustermanager.AppContext): + self.context = context + + def initiate_auth(self, context: ApiInvocationContext): + request = context.get_request_payload_as(InitiateAuthRequest) + result = self.context.accounts.initiate_auth(request) + context.success(result) + + def respond_to_auth_challenge(self, context: ApiInvocationContext): + request = context.get_request_payload_as(RespondToAuthChallengeRequest) + result = self.context.accounts.respond_to_auth_challenge(request) + context.success(result) + + def forgot_password(self, context: ApiInvocationContext): + request = context.get_request_payload_as(ForgotPasswordRequest) + self.context.accounts.forgot_password(request.username) + context.success(ForgotPasswordResult()) + + def confirm_forgot_password(self, context: ApiInvocationContext): + request = context.get_request_payload_as(ConfirmForgotPasswordRequest) + self.context.accounts.confirm_forgot_password( + request.username, + request.password, + request.confirmation_code + ) + context.success(ConfirmForgotPasswordResult()) + + def change_password(self, context: ApiInvocationContext): + username = context.get_username() + if Utils.is_empty(username): + raise exceptions.unauthorized_access() + + request = context.get_request_payload_as(ChangePasswordRequest) + self.context.accounts.change_password(context.access_token, username, request.old_password, request.new_password) + + context.success(ChangePasswordResult()) + + def get_user(self, context: ApiInvocationContext): + username = context.get_username() + if Utils.is_empty(username): + raise exceptions.unauthorized_access() + + user = self.context.accounts.get_user(username) + context.success(GetUserResult( + user=user + )) + + def get_user_group(self, context: ApiInvocationContext): + username = context.get_username() + if Utils.is_empty(username): + raise exceptions.unauthorized_access() + + group_name = self.context.accounts.group_name_helper.get_user_group(username) + group = self.context.accounts.get_group(group_name) + context.success(GetGroupResult( + group=group + )) + + def add_user_to_group(self, context: ApiInvocationContext): + username = context.get_username() + if Utils.is_empty(username): + raise exceptions.unauthorized_access() + + group_name = self.context.accounts.group_name_helper.get_user_group(username) + + request = context.get_request_payload_as(AddUserToGroupRequest) + self.context.accounts.add_users_to_group(request.usernames, group_name) + + context.success(AddUserToGroupResult()) + + def remove_user_from_group(self, context: ApiInvocationContext): + username = context.get_username() + if Utils.is_empty(username): + raise exceptions.unauthorized_access() + + group_name = self.context.accounts.group_name_helper.get_user_group(username) + + request = context.get_request_payload_as(RemoveUserFromGroupRequest) + self.context.accounts.remove_users_from_group(request.usernames, group_name) + + context.success(RemoveUserFromGroupResult()) + + def sign_out(self, context: ApiInvocationContext): + if not context.is_authenticated(): + raise exceptions.unauthorized_access() + + request = context.get_request_payload_as(SignOutRequest) + self.context.accounts.sign_out( + refresh_token=request.refresh_token, + sso_auth=Utils.get_as_bool(request.sso_auth, False) + ) + + context.success(SignOutResult()) + + def global_sign_out(self, context: ApiInvocationContext): + if not context.is_authenticated_user(): + raise exceptions.unauthorized_access() + + request = context.get_request_payload_as(GlobalSignOutRequest) + if Utils.is_not_empty(request.username) and request.username != context.get_username(): + raise exceptions.unauthorized_access() + + self.context.accounts.global_sign_out(username=context.get_username()) + + context.success(GlobalSignOutResult()) + + def get_user_private_key(self, context: ApiInvocationContext): + if not context.is_authorized_user(): + raise exceptions.unauthorized_access() + + request = context.get_request_payload_as(GetUserPrivateKeyRequest) + key_format = request.key_format + platform = request.platform + + user = self.context.accounts.get_user(context.get_username()) + home_dir = UserHomeDirectory(self.context, user) + + key_name = home_dir.get_key_name(key_format) + key_material = home_dir.get_key_material(key_format, platform) + + context.success(GetUserPrivateKeyResult( + name=key_name, + key_material=key_material + )) + + def list_users_in_group(self, context: ApiInvocationContext): + if not context.is_authenticated(): + raise exceptions.unauthorized_access() + request = context.get_request_payload_as(ListUsersInGroupRequest) + result = self.context.accounts.list_users_in_group(request) + context.success(result) + + def invoke(self, context: ApiInvocationContext): + namespace = context.namespace + if namespace == 'Auth.GlobalSignOut': + self.global_sign_out(context) + elif namespace == 'Auth.SignOut': + self.sign_out(context) + elif namespace == 'Auth.InitiateAuth': + self.initiate_auth(context) + elif namespace == 'Auth.RespondToAuthChallenge': + self.respond_to_auth_challenge(context) + elif namespace == 'Auth.ForgotPassword': + self.forgot_password(context) + elif namespace == 'Auth.ConfirmForgotPassword': + self.confirm_forgot_password(context) + elif namespace == 'Auth.ChangePassword': + self.change_password(context) + elif namespace == 'Auth.GetUser': + self.get_user(context) + elif namespace == 'Auth.GetGroup': + self.get_user_group(context) + elif namespace == 'Auth.AddUserToGroup': + self.add_user_to_group(context) + elif namespace == 'Auth.RemoveUserFromGroup': + self.remove_user_from_group(context) + elif namespace == 'Auth.GetUserPrivateKey': + self.get_user_private_key(context) + elif namespace == 'Auth.ListUsersInGroup': + self.list_users_in_group(context) diff --git a/source/idea/idea-cluster-manager/src/ideaclustermanager/app/api/cluster_settings_api.py b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/api/cluster_settings_api.py new file mode 100644 index 00000000..b16e793e --- /dev/null +++ b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/api/cluster_settings_api.py @@ -0,0 +1,124 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +import ideaclustermanager + +from ideasdk.api import ApiInvocationContext, BaseAPI +from ideadatamodel.cluster_settings import ( + ListClusterModulesResult, + ListClusterHostsRequest, + ListClusterHostsResult, + GetModuleSettingsRequest, + GetModuleSettingsResult, + DescribeInstanceTypesResult +) +from ideadatamodel import exceptions, constants +from ideasdk.utils import Utils + +from threading import RLock + + +class ClusterSettingsAPI(BaseAPI): + + def __init__(self, context: ideaclustermanager.AppContext): + self.context = context + self.instance_types_lock = RLock() + + def list_cluster_modules(self, context: ApiInvocationContext): + cluster_modules = self.context.get_cluster_modules() + context.success(ListClusterModulesResult( + listing=cluster_modules + )) + + def get_module_settings(self, context: ApiInvocationContext): + request = context.get_request_payload_as(GetModuleSettingsRequest) + + module_id = request.module_id + if Utils.is_empty(module_id): + raise exceptions.invalid_params('module_id is required') + + module_config = self.context.config().get_config(module_id, module_id=module_id) + + context.success(GetModuleSettingsResult( + settings=module_config.as_plain_ordered_dict() + )) + + def list_cluster_hosts(self, context: ApiInvocationContext): + # returns all infrastructure instances + request = context.get_request_payload_as(ListClusterHostsRequest) + ec2_instances = self.context.aws_util().ec2_describe_instances( + filters=[ + { + 'Name': 'instance-state-name', + 'Values': ['pending', 'stopped', 'running'] + }, + { + 'Name': f'tag:{constants.IDEA_TAG_CLUSTER_NAME}', + 'Values': [self.context.cluster_name()] + }, + { + 'Name': f'tag:{constants.IDEA_TAG_NODE_TYPE}', + 'Values': [constants.NODE_TYPE_INFRA, constants.NODE_TYPE_APP, constants.NODE_TYPE_AMI_BUILDER] + } + ], + page_size=request.page_size + ) + result = [] + for instance in ec2_instances: + result.append(instance.instance_data()) + + context.success(ListClusterHostsResult( + listing=result + )) + + def describe_instance_types(self, context: ApiInvocationContext): + + instance_types = self.context.cache().long_term().get('aws.ec2.all-instance-types') + if instance_types is None: + with self.instance_types_lock: + + instance_types = self.context.cache().long_term().get('aws.ec2.all-instance-types') + if instance_types is None: + instance_types = [] + has_more = True + next_token = None + + while has_more: + if next_token is None: + result = self.context.aws().ec2().describe_instance_types(MaxResults=100) + else: + result = self.context.aws().ec2().describe_instance_types(MaxResults=100, NextToken=next_token) + + next_token = Utils.get_value_as_string('NextToken', result) + has_more = Utils.is_not_empty(next_token) + current_instance_types = Utils.get_value_as_list('InstanceTypes', result) + if len(current_instance_types) > 0: + instance_types += current_instance_types + + self.context.cache().long_term().set('aws.ec2.all-instance-types', instance_types) + + context.success(DescribeInstanceTypesResult( + instance_types=instance_types + )) + + def invoke(self, context: ApiInvocationContext): + if not context.is_authenticated(): + raise exceptions.unauthorized_access() + + namespace = context.namespace + if namespace == 'ClusterSettings.ListClusterModules': + self.list_cluster_modules(context) + elif namespace == 'ClusterSettings.GetModuleSettings': + self.get_module_settings(context) + elif namespace == 'ClusterSettings.ListClusterHosts': + self.list_cluster_hosts(context) + elif namespace == 'ClusterSettings.DescribeInstanceTypes': + self.describe_instance_types(context) diff --git a/source/idea/idea-cluster-manager/src/ideaclustermanager/app/api/email_templates_api.py b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/api/email_templates_api.py new file mode 100644 index 00000000..cfdc3b91 --- /dev/null +++ b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/api/email_templates_api.py @@ -0,0 +1,95 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +import ideaclustermanager + +from ideasdk.api import ApiInvocationContext, BaseAPI +from ideadatamodel.email_templates import ( + CreateEmailTemplateRequest, + GetEmailTemplateRequest, + UpdateEmailTemplateRequest, + DeleteEmailTemplateRequest, + ListEmailTemplatesRequest +) +from ideadatamodel import exceptions +from ideasdk.utils import Utils + + +class EmailTemplatesAPI(BaseAPI): + + def __init__(self, context: ideaclustermanager.AppContext): + self.context = context + + self.SCOPE_WRITE = f'{self.context.module_id()}/write' + self.SCOPE_READ = f'{self.context.module_id()}/read' + + self.acl = { + 'EmailTemplates.CreateEmailTemplate': { + 'scope': self.SCOPE_WRITE, + 'method': self.create_email_template + }, + 'EmailTemplates.GetEmailTemplate': { + 'scope': self.SCOPE_READ, + 'method': self.get_email_template + }, + 'EmailTemplates.UpdateEmailTemplate': { + 'scope': self.SCOPE_WRITE, + 'method': self.update_email_template + }, + 'EmailTemplates.ListEmailTemplates': { + 'scope': self.SCOPE_READ, + 'method': self.list_email_templates + }, + 'EmailTemplates.DeleteEmailTemplate': { + 'scope': self.SCOPE_WRITE, + 'method': self.delete_email_template + } + } + + def create_email_template(self, context: ApiInvocationContext): + request = context.get_request_payload_as(CreateEmailTemplateRequest) + result = self.context.email_templates.create_email_template(request) + context.success(result) + + def get_email_template(self, context: ApiInvocationContext): + request = context.get_request_payload_as(GetEmailTemplateRequest) + result = self.context.email_templates.get_email_template(request) + context.success(result) + + def update_email_template(self, context: ApiInvocationContext): + request = context.get_request_payload_as(UpdateEmailTemplateRequest) + result = self.context.email_templates.update_email_template(request) + context.success(result) + + def list_email_templates(self, context: ApiInvocationContext): + request = context.get_request_payload_as(ListEmailTemplatesRequest) + result = self.context.email_templates.list_email_templates(request) + context.success(result) + + def delete_email_template(self, context: ApiInvocationContext): + request = context.get_request_payload_as(DeleteEmailTemplateRequest) + result = self.context.email_templates.delete_email_template(request) + context.success(result) + + def invoke(self, context: ApiInvocationContext): + namespace = context.namespace + + acl_entry = Utils.get_value_as_dict(namespace, self.acl) + if acl_entry is None: + raise exceptions.unauthorized_access() + + acl_entry_scope = Utils.get_value_as_string('scope', acl_entry) + is_authorized = context.is_authorized(elevated_access=True, scopes=[acl_entry_scope]) + + if is_authorized: + acl_entry['method'](context) + else: + raise exceptions.unauthorized_access() diff --git a/source/idea/idea-cluster-manager/src/ideaclustermanager/app/api/projects_api.py b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/api/projects_api.py new file mode 100644 index 00000000..8f90fee4 --- /dev/null +++ b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/api/projects_api.py @@ -0,0 +1,139 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +import ideaclustermanager + +from ideasdk.api import ApiInvocationContext, BaseAPI +from ideadatamodel.projects import ( + CreateProjectRequest, + GetProjectRequest, + UpdateProjectRequest, + ListProjectsRequest, + EnableProjectRequest, + DisableProjectRequest, + GetUserProjectsRequest +) +from ideadatamodel import exceptions +from ideasdk.utils import Utils + + +class ProjectsAPI(BaseAPI): + + def __init__(self, context: ideaclustermanager.AppContext): + self.context = context + + self.SCOPE_WRITE = f'{self.context.module_id()}/write' + self.SCOPE_READ = f'{self.context.module_id()}/read' + + self.acl = { + 'Projects.CreateProject': { + 'scope': self.SCOPE_WRITE, + 'method': self.create_project + }, + 'Projects.GetProject': { + 'scope': self.SCOPE_READ, + 'method': self.get_project + }, + 'Projects.UpdateProject': { + 'scope': self.SCOPE_WRITE, + 'method': self.update_project + }, + 'Projects.ListProjects': { + 'scope': self.SCOPE_READ, + 'method': self.list_projects + }, + 'Projects.GetUserProjects': { + 'scope': self.SCOPE_READ, + 'method': self.admin_get_user_projects + }, + 'Projects.EnableProject': { + 'scope': self.SCOPE_WRITE, + 'method': self.enable_project + }, + 'Projects.DisableProject': { + 'scope': self.SCOPE_WRITE, + 'method': self.disable_project + } + } + + def create_project(self, context: ApiInvocationContext): + request = context.get_request_payload_as(CreateProjectRequest) + result = self.context.projects.create_project(request) + context.success(result) + + def get_project(self, context: ApiInvocationContext): + request = context.get_request_payload_as(GetProjectRequest) + result = self.context.projects.get_project(request) + project = result.project + if project.is_budgets_enabled(): + budget = self.context.aws_util().budgets_get_budget(budget_name=project.budget.budget_name) + project.budget = budget + context.success(result) + + def update_project(self, context: ApiInvocationContext): + request = context.get_request_payload_as(UpdateProjectRequest) + result = self.context.projects.update_project(request) + context.success(result) + + def list_projects(self, context: ApiInvocationContext): + request = context.get_request_payload_as(ListProjectsRequest) + result = self.context.projects.list_projects(request) + for project in result.listing: + if project.is_budgets_enabled(): + # this call could possibly make some performance degradations, if the configured budget is not available. + # need to optimize this further. + budget = self.context.aws_util().budgets_get_budget(budget_name=project.budget.budget_name) + project.budget = budget + context.success(result) + + def get_user_projects(self, context: ApiInvocationContext): + request = context.get_request_payload_as(GetUserProjectsRequest) + request.username = context.get_username() + result = self.context.projects.get_user_projects(request) + context.success(result) + + def admin_get_user_projects(self, context: ApiInvocationContext): + request = context.get_request_payload_as(GetUserProjectsRequest) + if Utils.is_empty(request.username): + request.username = context.get_username() + result = self.context.projects.get_user_projects(request) + context.success(result) + + def enable_project(self, context: ApiInvocationContext): + request = context.get_request_payload_as(EnableProjectRequest) + result = self.context.projects.enable_project(request) + context.success(result) + + def disable_project(self, context: ApiInvocationContext): + request = context.get_request_payload_as(DisableProjectRequest) + result = self.context.projects.disable_project(request) + context.success(result) + + def invoke(self, context: ApiInvocationContext): + namespace = context.namespace + + acl_entry = Utils.get_value_as_dict(namespace, self.acl) + if acl_entry is None: + raise exceptions.unauthorized_access() + + acl_entry_scope = Utils.get_value_as_string('scope', acl_entry) + is_authorized = context.is_authorized(elevated_access=True, scopes=[acl_entry_scope]) + is_authenticated_user = context.is_authenticated_user() + + if is_authorized: + acl_entry['method'](context) + return + + if is_authenticated_user and namespace in ('Projects.GetUserProjects', 'Projects.GetProject'): + acl_entry['method'](context) + return + + raise exceptions.unauthorized_access() diff --git a/source/idea/idea-cluster-manager/src/ideaclustermanager/app/app_context.py b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/app_context.py new file mode 100644 index 00000000..51cffb95 --- /dev/null +++ b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/app_context.py @@ -0,0 +1,44 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +from ideasdk.context import SocaContext, SocaContextOptions +from ideasdk.auth import TokenService +from ideasdk.utils import GroupNameHelper + +from ideaclustermanager.app.projects.projects_service import ProjectsService +from ideaclustermanager.app.accounts.accounts_service import AccountsService +from ideaclustermanager.app.accounts.cognito_user_pool import CognitoUserPool +from ideaclustermanager.app.accounts.ldapclient import OpenLDAPClient, ActiveDirectoryClient +from ideaclustermanager.app.accounts.ad_automation_agent import ADAutomationAgent +from ideaclustermanager.app.email_templates.email_templates_service import EmailTemplatesService +from ideaclustermanager.app.notifications.notifications_service import NotificationsService +from ideaclustermanager.app.tasks.task_manager import TaskManager + +from typing import Optional, Union + + +class ClusterManagerAppContext(SocaContext): + + def __init__(self, options: SocaContextOptions): + super().__init__( + options=options + ) + + self.token_service: Optional[TokenService] = None + self.projects: Optional[ProjectsService] = None + self.user_pool: Optional[CognitoUserPool] = None + self.ldap_client: Optional[Union[OpenLDAPClient, ActiveDirectoryClient]] = None + self.accounts: Optional[AccountsService] = None + self.task_manager: Optional[TaskManager] = None + self.ad_automation_agent: Optional[ADAutomationAgent] = None + self.email_templates: Optional[EmailTemplatesService] = None + self.notifications: Optional[NotificationsService] = None + self.group_name_helper: Optional[GroupNameHelper] = None diff --git a/source/idea/idea-cluster-manager/src/ideaclustermanager/app/app_main.py b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/app_main.py new file mode 100644 index 00000000..71d920a8 --- /dev/null +++ b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/app_main.py @@ -0,0 +1,70 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +import ideaclustermanager + +from ideadatamodel import constants +from ideasdk.context import SocaContextOptions +from ideasdk.app.soca_app_commands import launch_decorator +from ideasdk.utils import EnvironmentUtils + +from ideaclustermanager.app.cluster_manager_app import ClusterManagerApp + +import sys +import click +import traceback + + +@click.version_option(version=ideaclustermanager.__version__) +@launch_decorator() +def main(**kwargs): + """ + start cluster-manager + """ + try: + + cluster_name = EnvironmentUtils.idea_cluster_name(required=True) + module_id = EnvironmentUtils.idea_module_id(required=True) + module_set = EnvironmentUtils.idea_module_set(required=True) + aws_region = EnvironmentUtils.aws_default_region(required=True) + + ClusterManagerApp( + context=ideaclustermanager.AppContext( + options=SocaContextOptions( + cluster_name=cluster_name, + module_name=constants.MODULE_CLUSTER_MANAGER, + module_id=module_id, + module_set=module_set, + aws_region=aws_region, + is_app_server=True, + enable_aws_client_provider=True, + enable_aws_util=True, + enable_instance_metadata_util=True, + use_vpc_endpoints=True, + enable_distributed_lock=True, + enable_leader_election=True, + enable_metrics=True, + enable_analytics=True + ) + ), + **kwargs + ).launch() + + except Exception as e: + print(f'failed to initialize application context: {e}') + traceback.print_exc() + print('exit code: 1') + sys.exit(1) + + +# used only for local testing +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/source/idea/idea-cluster-manager/src/ideaclustermanager/app/app_messages.py b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/app_messages.py new file mode 100644 index 00000000..06991ddf --- /dev/null +++ b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/app_messages.py @@ -0,0 +1,88 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + + +DEV_MODE_INDEX_PAGE = """ + + + + + + + IDEA Web Portal (Dev Mode) + + + +
+
+

SOCA Web Portal (Dev Mode)

+

When using dev-mode, do not use the Web App served by the backend server.

+

Start local front-end server using invoke as shown below:

+ 
$ invoke web-portal.serve-frontend
+ +
+

If you've accidentally landed on this page, click below to open the dev-mode front end app:

+

http://localhost:3000/

+ +
+
+

Happy Coding!

+
+
+ Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +
+
+ + +""" diff --git a/source/idea/idea-cluster-manager/src/ideaclustermanager/app/app_utils.py b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/app_utils.py new file mode 100644 index 00000000..02259c2a --- /dev/null +++ b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/app_utils.py @@ -0,0 +1,25 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +from ideasdk.utils import EnvironmentUtils +import os + + +class ClusterManagerUtils: + + @staticmethod + def get_app_deploy_dir() -> str: + app_deploy_dir = EnvironmentUtils.idea_app_deploy_dir(required=True) + return os.path.join(app_deploy_dir, 'cluster-manager') + + @staticmethod + def get_email_template_defaults_file() -> str: + return os.path.join(ClusterManagerUtils.get_app_deploy_dir(), 'resources', 'defaults', 'email_templates.yml') diff --git a/source/idea/idea-cluster-manager/src/ideaclustermanager/app/cluster_manager_app.py b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/cluster_manager_app.py new file mode 100644 index 00000000..84ea848e --- /dev/null +++ b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/cluster_manager_app.py @@ -0,0 +1,202 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +import ideasdk.app +from ideasdk.auth import TokenService, TokenServiceOptions +from ideadatamodel import constants +from ideasdk.client.evdi_client import EvdiClient +from ideasdk.server import SocaServerOptions +from ideasdk.utils import GroupNameHelper + +import ideaclustermanager +from ideaclustermanager.app.api.api_invoker import ClusterManagerApiInvoker +from ideaclustermanager.app.projects.projects_service import ProjectsService +from ideaclustermanager.app.projects.project_tasks import ( + ProjectEnabledTask, + ProjectDisabledTask, + ProjectGroupsUpdatedTask +) +from ideaclustermanager.app.accounts.accounts_service import AccountsService +from ideaclustermanager.app.accounts.cognito_user_pool import CognitoUserPool, CognitoUserPoolOptions + +from ideaclustermanager.app.accounts.ldapclient.ldap_client_factory import build_ldap_client + +from ideaclustermanager.app.accounts.ad_automation_agent import ADAutomationAgent +from ideaclustermanager.app.accounts.account_tasks import ( + SyncUserInDirectoryServiceTask, + CreateUserHomeDirectoryTask, + SyncGroupInDirectoryServiceTask, + SyncPasswordInDirectoryServiceTask, + GroupMembershipUpdatedTask +) +from ideaclustermanager.app.tasks.task_manager import TaskManager +from ideaclustermanager.app.web_portal import WebPortal +from ideaclustermanager.app.email_templates.email_templates_service import EmailTemplatesService +from ideaclustermanager.app.notifications.notifications_service import NotificationsService + +from typing import Optional + + +class ClusterManagerApp(ideasdk.app.SocaApp): + """ + cluster manager app + """ + + def __init__(self, context: ideaclustermanager.AppContext, + config_file: str, + env_file: str = None, + config_overrides_file: str = None, + validation_level: int = constants.CONFIG_LEVEL_CRITICAL, + **kwargs): + + api_path_prefix = context.config().get_string('cluster-manager.server.api_context_path', f'/{context.module_id()}') + super().__init__( + context=context, + config_file=config_file, + api_invoker=ClusterManagerApiInvoker(context=context), + env_file=env_file, + config_overrides_file=config_overrides_file, + validation_level=validation_level, + server_options=SocaServerOptions( + api_path_prefixes=[ + api_path_prefix + ], + enable_http_file_upload=True, + enable_metrics=True + ), + **kwargs + ) + self.context = context + self.web_portal: Optional[WebPortal] = None + + def app_initialize(self): + + # group name helper + self.context.group_name_helper = GroupNameHelper(self.context) + administrators_group_name = self.context.group_name_helper.get_cluster_administrators_group() + managers_group_name = self.context.group_name_helper.get_cluster_managers_group() + + # token service + domain_url = self.context.config().get_string('identity-provider.cognito.domain_url', required=True) + provider_url = self.context.config().get_string('identity-provider.cognito.provider_url', required=True) + client_id = self.context.config().get_secret('cluster-manager.client_id', required=True) + client_secret = self.context.config().get_secret('cluster-manager.client_secret', required=True) + self.context.token_service = TokenService( + context=self.context, + options=TokenServiceOptions( + cognito_user_pool_provider_url=provider_url, + cognito_user_pool_domain_url=domain_url, + client_id=client_id, + client_secret=client_secret, + administrators_group_name=administrators_group_name, + managers_group_name=managers_group_name + ) + ) + + # cognito user pool + user_pool_id = self.context.config().get_string('identity-provider.cognito.user_pool_id', required=True) + self.context.user_pool = CognitoUserPool( + context=self.context, + options=CognitoUserPoolOptions( + user_pool_id=user_pool_id, + admin_group_name=administrators_group_name, + client_id=client_id, + client_secret=client_secret + ) + ) + + # accounts service + ds_provider = self.context.config().get_string('directoryservice.provider', required=True) + self.context.ldap_client = build_ldap_client(self.context) + + if ds_provider in {constants.DIRECTORYSERVICE_AWS_MANAGED_ACTIVE_DIRECTORY, constants.DIRECTORYSERVICE_ACTIVE_DIRECTORY}: + self.context.ad_automation_agent = ADAutomationAgent( + context=self.context, + ldap_client=self.context.ldap_client + ) + + # task manager + self.context.task_manager = TaskManager( + context=self.context, + tasks=[ + SyncUserInDirectoryServiceTask(self.context), + SyncGroupInDirectoryServiceTask(self.context), + CreateUserHomeDirectoryTask(self.context), + SyncPasswordInDirectoryServiceTask(self.context), + GroupMembershipUpdatedTask(self.context), + ProjectEnabledTask(self.context), + ProjectDisabledTask(self.context), + ProjectGroupsUpdatedTask(self.context) + ] + ) + + # account service + evdi_client = EvdiClient(self.context) + self.context.accounts = AccountsService( + context=self.context, + ldap_client=self.context.ldap_client, + user_pool=self.context.user_pool, + task_manager=self.context.task_manager, + evdi_client=evdi_client, + token_service=self.context.token_service + ) + + # projects service + self.context.projects = ProjectsService( + context=self.context, + accounts_service=self.context.accounts, + task_manager=self.context.task_manager + ) + + # email templates + self.context.email_templates = EmailTemplatesService( + context=self.context + ) + + # notifications + self.context.notifications = NotificationsService( + context=self.context, + accounts=self.context.accounts, + email_templates=self.context.email_templates + ) + + # web portal + self.web_portal = WebPortal( + context=self.context, + server=self.server + ) + self.web_portal.initialize() + + def app_start(self): + if self.context.ad_automation_agent is not None: + self.context.ad_automation_agent.start() + + self.context.task_manager.start() + self.context.notifications.start() + + try: + self.context.distributed_lock().acquire(key='initialize-defaults') + self.context.accounts.create_defaults() + self.context.projects.create_defaults() + self.context.email_templates.create_defaults() + finally: + self.context.distributed_lock().release(key='initialize-defaults') + + def app_stop(self): + + if self.context.ad_automation_agent is not None: + self.context.ad_automation_agent.stop() + + if self.context.task_manager is not None: + self.context.task_manager.stop() + + if self.context.notifications is not None: + self.context.notifications.stop() diff --git a/source/idea/idea-cluster-manager/src/ideaclustermanager/app/email_templates/__init__.py b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/email_templates/__init__.py new file mode 100644 index 00000000..6d8d18a3 --- /dev/null +++ b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/email_templates/__init__.py @@ -0,0 +1,10 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. diff --git a/source/idea/idea-cluster-manager/src/ideaclustermanager/app/email_templates/email_templates_dao.py b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/email_templates/email_templates_dao.py new file mode 100644 index 00000000..9ee7af3c --- /dev/null +++ b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/email_templates/email_templates_dao.py @@ -0,0 +1,215 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +from ideasdk.utils import Utils +from ideadatamodel import (exceptions, EmailTemplate, ListEmailTemplatesRequest, ListEmailTemplatesResult, SocaPaginator) +from ideasdk.context import SocaContext + +from typing import Dict, Optional +from boto3.dynamodb.conditions import Attr +import arrow + + +class EmailTemplatesDAO: + + def __init__(self, context: SocaContext, logger=None): + self.context = context + if logger is not None: + self.logger = logger + else: + self.logger = context.logger('email-templates-dao') + self.table = None + + def get_table_name(self) -> str: + return f'{self.context.cluster_name()}.email-templates' + + def initialize(self): + self.context.aws_util().dynamodb_create_table( + create_table_request={ + 'TableName': self.get_table_name(), + 'AttributeDefinitions': [ + { + 'AttributeName': 'name', + 'AttributeType': 'S' + } + ], + 'KeySchema': [ + { + 'AttributeName': 'name', + 'KeyType': 'HASH' + } + ], + 'BillingMode': 'PAY_PER_REQUEST' + }, + wait=True + ) + self.table = self.context.aws().dynamodb_table().Table(self.get_table_name()) + + @staticmethod + def convert_from_db(email_template: Dict) -> EmailTemplate: + name = Utils.get_value_as_string('name', email_template) + title = Utils.get_value_as_string('title', email_template) + template_type = Utils.get_value_as_string('template_type', email_template) + subject = Utils.get_value_as_string('subject', email_template) + body = Utils.get_value_as_string('body', email_template) + created_on = Utils.get_value_as_int('created_on', email_template) + updated_on = Utils.get_value_as_int('updated_on', email_template) + return EmailTemplate( + name=name, + title=title, + template_type=template_type, + subject=subject, + body=body, + created_on=arrow.get(created_on).datetime, + updated_on=arrow.get(updated_on).datetime + ) + + @staticmethod + def convert_to_db(email_template: EmailTemplate) -> Dict: + db_email_template = { + 'name': email_template.name + } + + if email_template.title is not None: + db_email_template['title'] = email_template.title + + if email_template.template_type is not None: + db_email_template['template_type'] = email_template.template_type + + if email_template.subject is not None: + db_email_template['subject'] = email_template.subject + + if email_template.body is not None: + db_email_template['body'] = email_template.body + + return db_email_template + + def create_email_template(self, email_template: Dict) -> Dict: + + name = Utils.get_value_as_string('name', email_template) + if Utils.is_empty(name): + raise exceptions.invalid_params('name is required') + + created_email_template = { + **email_template, + 'created_on': Utils.current_time_ms(), + 'updated_on': Utils.current_time_ms() + } + self.table.put_item( + Item=created_email_template + ) + + return created_email_template + + def get_email_template(self, name: str) -> Optional[Dict]: + + if Utils.is_empty(name): + raise exceptions.invalid_params('name is required') + + result = self.table.get_item( + Key={ + 'name': name + } + ) + return Utils.get_value_as_dict('Item', result) + + def update_email_template(self, email_template: Dict): + + name = Utils.get_value_as_string('name', email_template) + if Utils.is_empty(name): + raise exceptions.invalid_params('name is required') + + email_template['updated_on'] = Utils.current_time_ms() + + update_expression_tokens = [] + expression_attr_names = {} + expression_attr_values = {} + + for key, value in email_template.items(): + if key in ('name', 'created_on'): + continue + update_expression_tokens.append(f'#{key} = :{key}') + expression_attr_names[f'#{key}'] = key + expression_attr_values[f':{key}'] = value + + result = self.table.update_item( + Key={ + 'name': name + }, + ConditionExpression=Attr('name').eq(name), + UpdateExpression='SET ' + ', '.join(update_expression_tokens), + ExpressionAttributeNames=expression_attr_names, + ExpressionAttributeValues=expression_attr_values, + ReturnValues='ALL_NEW' + ) + + updated_email_template = result['Attributes'] + updated_email_template['name'] = name + + return updated_email_template + + def delete_email_template(self, name: str): + + if Utils.is_empty(name): + raise exceptions.invalid_params('name is required') + + self.table.delete_item( + Key={ + 'name': name + } + ) + + def list_email_templates(self, request: ListEmailTemplatesRequest) -> ListEmailTemplatesResult: + scan_request = {} + + cursor = request.cursor + last_evaluated_key = None + if Utils.is_not_empty(cursor): + last_evaluated_key = Utils.from_json(Utils.base64_decode(cursor)) + if last_evaluated_key is not None: + scan_request['LastEvaluatedKey'] = last_evaluated_key + + scan_filter = None + if Utils.is_not_empty(request.filters): + scan_filter = {} + for filter_ in request.filters: + if filter_.eq is not None: + scan_filter[filter_.key] = { + 'AttributeValueList': [filter_.eq], + 'ComparisonOperator': 'EQ' + } + if filter_.like is not None: + scan_filter[filter_.key] = { + 'AttributeValueList': [filter_.like], + 'ComparisonOperator': 'CONTAINS' + } + if scan_filter is not None: + scan_request['ScanFilter'] = scan_filter + + scan_result = self.table.scan(**scan_request) + + db_email_templates = Utils.get_value_as_list('Items', scan_result, []) + email_templates = [] + for db_email_template in db_email_templates: + email_template = self.convert_from_db(db_email_template) + email_templates.append(email_template) + + response_cursor = None + last_evaluated_key = Utils.get_any_value('LastEvaluatedKey', scan_result) + if last_evaluated_key is not None: + response_cursor = Utils.base64_encode(Utils.to_json(last_evaluated_key)) + + return ListEmailTemplatesResult( + listing=email_templates, + paginator=SocaPaginator( + cursor=response_cursor + ) + ) diff --git a/source/idea/idea-cluster-manager/src/ideaclustermanager/app/email_templates/email_templates_service.py b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/email_templates/email_templates_service.py new file mode 100644 index 00000000..df8b1bf5 --- /dev/null +++ b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/email_templates/email_templates_service.py @@ -0,0 +1,147 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +from ideadatamodel import exceptions, errorcodes +from ideadatamodel.email_templates import ( + CreateEmailTemplateRequest, + CreateEmailTemplateResult, + GetEmailTemplateRequest, + GetEmailTemplateResult, + UpdateEmailTemplateRequest, + UpdateEmailTemplateResult, + DeleteEmailTemplateRequest, + DeleteEmailTemplateResult, + ListEmailTemplatesRequest, + ListEmailTemplatesResult, + EmailTemplate +) +from ideasdk.utils import Utils +from ideasdk.context import SocaContext + +from ideaclustermanager.app.email_templates.email_templates_dao import EmailTemplatesDAO +from ideaclustermanager.app.app_utils import ClusterManagerUtils + + +class EmailTemplatesService: + + def __init__(self, context: SocaContext): + self.context = context + self.logger = context.logger('email-templates') + + self.email_templates_dao = EmailTemplatesDAO(context) + self.email_templates_dao.initialize() + + def create_email_template(self, request: CreateEmailTemplateRequest) -> CreateEmailTemplateResult: + """ + Create a new EmailTemplate + validate required fields, add the email_template to DynamoDB and Cache. + :param request: + :return: the created email_template (with email_template_id) + """ + + if Utils.is_empty(request): + raise exceptions.invalid_params('request is required') + email_template = request.template + if Utils.is_empty(email_template): + raise exceptions.invalid_params('template is required') + if Utils.are_empty(request.template.name): + raise exceptions.invalid_params('template.name is required') + if Utils.are_empty(request.template.template_type): + raise exceptions.invalid_params('template.template_type is required') + + existing = self.email_templates_dao.get_email_template(email_template.name) + if existing is not None: + raise exceptions.invalid_params(f'email_template with name: {email_template.name} already exists') + + db_email_template = self.email_templates_dao.convert_to_db(email_template) + db_created_email_template = self.email_templates_dao.create_email_template(db_email_template) + + created_email_template = self.email_templates_dao.convert_from_db(db_created_email_template) + + return CreateEmailTemplateResult( + template=created_email_template + ) + + def get_email_template(self, request: GetEmailTemplateRequest) -> GetEmailTemplateResult: + """ + Retrieve the EmailTemplate + :param request.email_template_name name of the email_template + :param request.email_template_id uuid of the email_template + :return: the EmailTemplate + """ + if Utils.is_empty(request): + raise exceptions.invalid_params('request is required') + if Utils.are_empty(request.name): + raise exceptions.invalid_params('name is required') + + db_email_template = self.email_templates_dao.get_email_template(request.name) + + if db_email_template is None: + raise exceptions.soca_exception( + error_code=errorcodes.EMAIL_TEMPLATE_NOT_FOUND, + message=f'email_template not found for name: {request.name}' + ) + + return GetEmailTemplateResult( + template=self.email_templates_dao.convert_from_db(db_email_template) + ) + + def update_email_template(self, request: UpdateEmailTemplateRequest) -> UpdateEmailTemplateResult: + """ + Update an EmailTemplate + :param request: + :return: + """ + if Utils.is_empty(request): + raise exceptions.invalid_params('request is required') + if Utils.are_empty(request.template): + raise exceptions.invalid_params('template is required') + if Utils.are_empty(request.template.name): + raise exceptions.invalid_params('template.name is required') + if Utils.are_empty(request.template.template_type): + raise exceptions.invalid_params('template.template_type is required') + + db_email_template = self.email_templates_dao.get_email_template(request.template.name) + + if db_email_template is None: + raise exceptions.soca_exception( + error_code=errorcodes.EMAIL_TEMPLATE_NOT_FOUND, + message=f'email_template not found for name: {request.template.name}' + ) + + db_updated = self.email_templates_dao.update_email_template(self.email_templates_dao.convert_to_db(request.template)) + updated_email_template = self.email_templates_dao.convert_from_db(db_updated) + + return UpdateEmailTemplateResult( + template=updated_email_template + ) + + def delete_email_template(self, request: DeleteEmailTemplateRequest) -> DeleteEmailTemplateResult: + if Utils.is_empty(request.name): + raise exceptions.invalid_params('name is required') + self.email_templates_dao.delete_email_template(request.name) + return DeleteEmailTemplateResult() + + def list_email_templates(self, request: ListEmailTemplatesRequest) -> ListEmailTemplatesResult: + return self.email_templates_dao.list_email_templates(request) + + def create_defaults(self): + email_templates_file = ClusterManagerUtils.get_email_template_defaults_file() + with open(email_templates_file, 'r') as f: + templates = Utils.from_yaml(f.read())['templates'] + + for template in templates: + name = template['name'] + existing = self.email_templates_dao.get_email_template(name) + if existing is not None: + continue + self.logger.info(f'creating default email template: {name} ...') + self.create_email_template(CreateEmailTemplateRequest(template=EmailTemplate(**template))) diff --git a/source/idea/idea-cluster-manager/src/ideaclustermanager/app/notifications/__init__.py b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/notifications/__init__.py new file mode 100644 index 00000000..6d8d18a3 --- /dev/null +++ b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/notifications/__init__.py @@ -0,0 +1,10 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. diff --git a/source/idea/idea-cluster-manager/src/ideaclustermanager/app/notifications/notifications_service.py b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/notifications/notifications_service.py new file mode 100644 index 00000000..2db3866d --- /dev/null +++ b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/notifications/notifications_service.py @@ -0,0 +1,169 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +from ideasdk.context import SocaContext +from ideasdk.service import SocaService +from ideasdk.utils import Utils, Jinja2Utils +from ideadatamodel import GetEmailTemplateRequest, Notification + +from ideaclustermanager.app.accounts.accounts_service import AccountsService +from ideaclustermanager.app.email_templates.email_templates_service import EmailTemplatesService + +from typing import Dict +from concurrent.futures import ThreadPoolExecutor +import threading +import time + +MAX_WORKERS = 1 # must be between 1 and 10 + + +class NotificationsService(SocaService): + + def __init__(self, context: SocaContext, accounts: AccountsService, email_templates: EmailTemplatesService): + super().__init__(context) + + self.context = context + self.logger = context.logger('notifications') + + self.accounts = accounts + self.email_templates = email_templates + + self.exit = threading.Event() + self.notifications_queue_url = self.context.config().get_string('cluster-manager.notifications_queue_url', required=True) + + self.notifications_monitor_thread = threading.Thread( + target=self.notifications_queue_listener, + name='notifications-monitor' + ) + self.notifications_executors = ThreadPoolExecutor( + max_workers=MAX_WORKERS, + thread_name_prefix='notifications-executor' + ) + self.jinja2_env = Jinja2Utils.env_using_base_loader() + + def send_email(self, notification: Notification): + """ + send email + + payload must contain below fields: + + username: str + template_name: str - the name of the email template + params: dict - containing all parameters required to render the template + + """ + try: + + ses_enabled = self.context.config().get_bool('cluster.ses.enabled', False) + if not ses_enabled: + self.logger.debug('ses is disabled. skip.') + return + + sender_email = self.context.config().get_string('cluster.ses.sender_email', required=True) + max_sending_rate = self.context.config().get_int('cluster.ses.max_sending_rate', 1) + max_sending_rate = max(1, max_sending_rate) + + email_notifications_enabled = self.context.config().get_bool('cluster-manager.notifications.email.enabled', False) + if not email_notifications_enabled: + self.logger.debug('email notifications are disabled. skip.') + return + + ses_region = self.context.config().get_string('cluster.ses.region', required=True) + + ses = self.context.aws().ses(region_name=ses_region) + + username = notification.username + user = self.accounts.get_user(username=username) + if Utils.is_false(user.enabled): + self.logger.info(f'user: {username} has been disabled. skip email notification') + return + + email = user.email + if Utils.is_empty(email): + self.logger.warning(f'email address not found for user: {username}. skip email notification') + return + + template_name = notification.template_name + get_template_result = self.email_templates.get_email_template(GetEmailTemplateRequest(name=template_name)) + template = get_template_result.template + subject_template = template.subject + body_template = template.body + + params = Utils.get_as_dict(notification.params, {}) + + subject_template = self.jinja2_env.from_string(subject_template) + message_template = self.jinja2_env.from_string(body_template) + subject = subject_template.render(**params) + body = message_template.render(**params) + + ses.send_email( + Source=sender_email, + Destination={ + 'ToAddresses': [email] + }, + Message={ + 'Subject': { + 'Data': subject + }, + 'Body': { + 'Html': { + 'Data': body + } + } + } + ) + + delay = float(1 / max_sending_rate) + time.sleep(delay) + + except Exception as e: + self.logger.exception(f'failed to send email notification: {e}') + + def execute_notifications(self, sqs_message: Dict): + notifications_name = None + try: + message_body = Utils.get_value_as_string('Body', sqs_message) + receipt_handle = Utils.get_value_as_string('ReceiptHandle', sqs_message) + payload = Utils.from_json(message_body) + self.send_email(Notification(**payload)) + self.context.aws().sqs().delete_message( + QueueUrl=self.notifications_queue_url, + ReceiptHandle=receipt_handle + ) + except Exception as e: + self.logger.exception(f'failed to execute notifications: {notifications_name} - {e}') + + def notifications_queue_listener(self): + while not self.exit.is_set(): + try: + result = self.context.aws().sqs().receive_message( + QueueUrl=self.notifications_queue_url, + MaxNumberOfMessages=MAX_WORKERS, + WaitTimeSeconds=20 + ) + messages = Utils.get_value_as_list('Messages', result, []) + if len(messages) == 0: + continue + self.logger.info(f'received {len(messages)} messages') + for message in messages: + self.notifications_executors.submit(lambda message_: self.execute_notifications(message_), message) + + except Exception as e: + self.logger.exception(f'failed to poll queue: {e}') + + def start(self): + self.notifications_monitor_thread.start() + + def stop(self): + self.exit.set() + if self.notifications_monitor_thread.is_alive(): + self.notifications_monitor_thread.join() + self.notifications_executors.shutdown(wait=True) diff --git a/source/idea/idea-cluster-manager/src/ideaclustermanager/app/projects/__init__.py b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/projects/__init__.py new file mode 100644 index 00000000..6d8d18a3 --- /dev/null +++ b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/projects/__init__.py @@ -0,0 +1,10 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. diff --git a/source/idea/idea-cluster-manager/src/ideaclustermanager/app/projects/db/__init__.py b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/projects/db/__init__.py new file mode 100644 index 00000000..6d8d18a3 --- /dev/null +++ b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/projects/db/__init__.py @@ -0,0 +1,10 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. diff --git a/source/idea/idea-cluster-manager/src/ideaclustermanager/app/projects/db/projects_dao.py b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/projects/db/projects_dao.py new file mode 100644 index 00000000..6a1028d3 --- /dev/null +++ b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/projects/db/projects_dao.py @@ -0,0 +1,285 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +from ideasdk.utils import Utils +from ideadatamodel import (exceptions, Project, AwsProjectBudget, ListProjectsRequest, ListProjectsResult, + SocaPaginator, SocaKeyValue) +from ideasdk.context import SocaContext + +from typing import Dict, Optional +from boto3.dynamodb.conditions import Attr, Key +import arrow + +GSI_PROJECT_NAME = 'project-name-index' + + +class ProjectsDAO: + + def __init__(self, context: SocaContext, logger=None): + self.context = context + if logger is not None: + self.logger = logger + else: + self.logger = context.logger('projects-dao') + self.table = None + + def get_table_name(self) -> str: + return f'{self.context.cluster_name()}.projects' + + def initialize(self): + self.context.aws_util().dynamodb_create_table( + create_table_request={ + 'TableName': self.get_table_name(), + 'AttributeDefinitions': [ + { + 'AttributeName': 'project_id', + 'AttributeType': 'S' + }, + { + 'AttributeName': 'name', + 'AttributeType': 'S' + } + ], + 'KeySchema': [ + { + 'AttributeName': 'project_id', + 'KeyType': 'HASH' + } + ], + 'GlobalSecondaryIndexes': [ + { + 'IndexName': GSI_PROJECT_NAME, + 'KeySchema': [ + { + 'AttributeName': 'name', + 'KeyType': 'HASH' + } + ], + 'Projection': { + 'ProjectionType': 'ALL' + } + } + ], + 'BillingMode': 'PAY_PER_REQUEST' + }, + wait=True + ) + self.table = self.context.aws().dynamodb_table().Table(self.get_table_name()) + + @staticmethod + def convert_from_db(project: Dict) -> Project: + project_id = Utils.get_value_as_string('project_id', project) + name = Utils.get_value_as_string('name', project) + title = Utils.get_value_as_string('title', project) + description = Utils.get_value_as_string('description', project) + ldap_groups = Utils.get_value_as_list('ldap_groups', project, []) + enabled = Utils.get_value_as_bool('enabled', project, False) + enable_budgets = Utils.get_value_as_bool('enable_budgets', project, False) + budget_name = Utils.get_value_as_string('budget_name', project) + budget = None + if Utils.is_not_empty(budget_name): + budget = AwsProjectBudget( + budget_name=budget_name + ) + db_tags = Utils.get_value_as_dict('tags', project) + tags = None + if db_tags is not None: + tags = [] + for key, value in db_tags.items(): + tags.append(SocaKeyValue(key=key, value=value)) + + created_on = Utils.get_value_as_int('created_on', project) + updated_on = Utils.get_value_as_int('updated_on', project) + + return Project( + project_id=project_id, + title=title, + name=name, + description=description, + ldap_groups=ldap_groups, + tags=tags, + enabled=enabled, + enable_budgets=enable_budgets, + budget=budget, + created_on=arrow.get(created_on).datetime, + updated_on=arrow.get(updated_on).datetime + ) + + @staticmethod + def convert_to_db(project: Project) -> Dict: + """ + convert project to db object. + None values will not replace existing values + :param project: + :return: Dict + """ + db_project = { + 'project_id': project.project_id + } + + if project.name is not None: + db_project['name'] = project.name + + if project.title is not None: + db_project['title'] = project.title + + if project.description is not None: + db_project['description'] = project.description + + if project.ldap_groups is not None: + db_project['ldap_groups'] = project.ldap_groups + + if project.tags is not None: + tags = {} + for tag in project.tags: + tags[tag.key] = tag.value + db_project['tags'] = tags + + if project.enable_budgets is not None: + db_project['enable_budgets'] = project.enable_budgets + + if project.budget is not None and project.budget.budget_name is not None: + db_project['budget_name'] = project.budget.budget_name + + return db_project + + def create_project(self, project: Dict) -> Dict: + created_project = { + **project, + 'project_id': Utils.uuid(), + 'created_on': Utils.current_time_ms(), + 'updated_on': Utils.current_time_ms() + } + self.table.put_item( + Item=created_project + ) + + return created_project + + def get_project_by_id(self, project_id: str) -> Optional[Dict]: + + if Utils.is_empty(project_id): + raise exceptions.invalid_params('project_id is required') + + result = self.table.get_item( + Key={ + 'project_id': project_id + } + ) + return Utils.get_value_as_dict('Item', result) + + def get_project_by_name(self, name: str) -> Optional[Dict]: + + if Utils.is_empty(name): + raise exceptions.invalid_params('name is required') + + result = self.table.query( + IndexName=GSI_PROJECT_NAME, + KeyConditionExpression=Key('name').eq(name) + ) + items = Utils.get_value_as_list('Items', result, []) + if len(items) == 0: + return None + + return items[0] + + def update_project(self, project: Dict): + + project_id = Utils.get_value_as_string('project_id', project) + if Utils.is_empty(project_id): + raise exceptions.invalid_params('project_id is required') + + project['updated_on'] = Utils.current_time_ms() + + update_expression_tokens = [] + expression_attr_names = {} + expression_attr_values = {} + + for key, value in project.items(): + if key in ('project_id', 'created_on'): + continue + update_expression_tokens.append(f'#{key} = :{key}') + expression_attr_names[f'#{key}'] = key + expression_attr_values[f':{key}'] = value + + result = self.table.update_item( + Key={ + 'project_id': project_id + }, + ConditionExpression=Attr('project_id').eq(project_id), + UpdateExpression='SET ' + ', '.join(update_expression_tokens), + ExpressionAttributeNames=expression_attr_names, + ExpressionAttributeValues=expression_attr_values, + ReturnValues='ALL_NEW' + ) + + updated_project = result['Attributes'] + updated_project['project_id'] = project_id + + return updated_project + + def delete_project(self, project_id: str): + + if Utils.is_empty(project_id): + raise exceptions.invalid_params('project_id is required') + + self.table.delete_item( + Key={ + 'project_id': project_id + } + ) + + def list_projects(self, request: ListProjectsRequest) -> ListProjectsResult: + scan_request = {} + + cursor = request.cursor + last_evaluated_key = None + if Utils.is_not_empty(cursor): + last_evaluated_key = Utils.from_json(Utils.base64_decode(cursor)) + if last_evaluated_key is not None: + scan_request['LastEvaluatedKey'] = last_evaluated_key + + scan_filter = None + if Utils.is_not_empty(request.filters): + scan_filter = {} + for filter_ in request.filters: + if filter_.eq is not None: + scan_filter[filter_.key] = { + 'AttributeValueList': [filter_.eq], + 'ComparisonOperator': 'EQ' + } + if filter_.like is not None: + scan_filter[filter_.key] = { + 'AttributeValueList': [filter_.like], + 'ComparisonOperator': 'CONTAINS' + } + if scan_filter is not None: + scan_request['ScanFilter'] = scan_filter + + scan_result = self.table.scan(**scan_request) + + db_projects = Utils.get_value_as_list('Items', scan_result, []) + projects = [] + for db_project in db_projects: + project = self.convert_from_db(db_project) + projects.append(project) + + response_cursor = None + last_evaluated_key = Utils.get_any_value('LastEvaluatedKey', scan_result) + if last_evaluated_key is not None: + response_cursor = Utils.base64_encode(Utils.to_json(last_evaluated_key)) + + return ListProjectsResult( + listing=projects, + paginator=SocaPaginator( + cursor=response_cursor + ) + ) diff --git a/source/idea/idea-cluster-manager/src/ideaclustermanager/app/projects/db/user_projects_dao.py b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/projects/db/user_projects_dao.py new file mode 100644 index 00000000..c73815d1 --- /dev/null +++ b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/projects/db/user_projects_dao.py @@ -0,0 +1,257 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +from ideasdk.utils import Utils +from ideadatamodel import exceptions +from ideasdk.context import SocaContext + +from ideaclustermanager.app.accounts.accounts_service import AccountsService +from ideaclustermanager.app.projects.db.projects_dao import ProjectsDAO + +from typing import List +from boto3.dynamodb.conditions import Key + + +class UserProjectsDAO: + + def __init__(self, context: SocaContext, projects_dao: ProjectsDAO, accounts_service: AccountsService, logger=None): + self.context = context + self.projects_dao = projects_dao + self.accounts_service = accounts_service + if logger is not None: + self.logger = logger + else: + self.logger = context.logger('user-projects-dao') + + self.user_projects_table = None + self.project_groups_table = None + + def get_user_projects_table_name(self) -> str: + return f'{self.context.cluster_name()}.projects.user-projects' + + def get_project_groups_table_name(self) -> str: + return f'{self.context.cluster_name()}.projects.project-groups' + + def initialize(self): + self.context.aws_util().dynamodb_create_table( + create_table_request={ + 'TableName': self.get_user_projects_table_name(), + 'AttributeDefinitions': [ + { + 'AttributeName': 'username', + 'AttributeType': 'S' + }, + { + 'AttributeName': 'project_id', + 'AttributeType': 'S' + } + ], + 'KeySchema': [ + { + 'AttributeName': 'username', + 'KeyType': 'HASH' + }, + { + 'AttributeName': 'project_id', + 'KeyType': 'RANGE' + } + ], + 'BillingMode': 'PAY_PER_REQUEST' + }, + wait=True + ) + self.context.aws_util().dynamodb_create_table( + create_table_request={ + 'TableName': self.get_project_groups_table_name(), + 'AttributeDefinitions': [ + { + 'AttributeName': 'group_name', + 'AttributeType': 'S' + }, + { + 'AttributeName': 'project_id', + 'AttributeType': 'S' + } + ], + 'KeySchema': [ + { + 'AttributeName': 'group_name', + 'KeyType': 'HASH' + }, + { + 'AttributeName': 'project_id', + 'KeyType': 'RANGE' + } + ], + 'BillingMode': 'PAY_PER_REQUEST' + }, + wait=True + ) + self.user_projects_table = self.context.aws().dynamodb_table().Table(self.get_user_projects_table_name()) + self.project_groups_table = self.context.aws().dynamodb_table().Table(self.get_project_groups_table_name()) + + def create_user_project(self, project_id: str, username: str): + if Utils.is_empty(project_id): + raise exceptions.invalid_params('project_id is required') + if Utils.is_empty(username): + raise exceptions.invalid_params('username is required') + + self.logger.info(f'added user project: {project_id}, username: {username}') + self.user_projects_table.put_item( + Item={ + 'project_id': project_id, + 'username': username + } + ) + + def delete_user_project(self, project_id: str, username: str): + if Utils.is_empty(project_id): + raise exceptions.invalid_params('project_id is required') + if Utils.is_empty(username): + raise exceptions.invalid_params('username is required') + + self.logger.info(f'deleted user project: {project_id}, username: {username}') + self.user_projects_table.delete_item( + Key={ + 'project_id': project_id, + 'username': username + } + ) + + def ldap_group_added(self, project_id: str, group_name: str): + if Utils.is_empty(project_id): + raise exceptions.invalid_params('project_id is required') + if Utils.is_empty(group_name): + raise exceptions.invalid_params('username is required') + + self.project_groups_table.put_item( + Item={ + 'group_name': group_name, + 'project_id': project_id + } + ) + + usernames = self.accounts_service.group_members_dao.get_usernames_in_group(group_name) + for username in usernames: + self.create_user_project( + project_id=project_id, + username=username + ) + + def ldap_group_removed(self, project_id: str, group_name: str): + if Utils.is_empty(project_id): + raise exceptions.invalid_params('project_id is required') + if Utils.is_empty(group_name): + raise exceptions.invalid_params('username is required') + + self.project_groups_table.delete_item( + Key={ + 'group_name': group_name, + 'project_id': project_id + } + ) + + usernames = self.accounts_service.group_members_dao.get_usernames_in_group(group_name) + for username in usernames: + self.delete_user_project( + project_id=project_id, + username=username + ) + + def get_projects_by_username(self, username: str) -> List[str]: + if Utils.is_empty(username): + raise exceptions.invalid_params('username is required') + + result = self.user_projects_table.query( + KeyConditionExpression=Key('username').eq(username) + ) + user_projects = Utils.get_value_as_list('Items', result, []) + + project_ids = [] + for user_project in user_projects: + project_ids.append(user_project['project_id']) + return project_ids + + def get_projects_by_group_name(self, group_name: str) -> List[str]: + if Utils.is_empty(group_name): + raise exceptions.invalid_params('group_name is required') + self.logger.debug(f'get_projects_by_group_name() - Looking for {group_name} in UserProjectsDAO/DDB') + result = self.project_groups_table.query( + KeyConditionExpression=Key('group_name').eq(group_name) + ) + self.logger.debug(f'get_projects_by_group_name() - Group: {group_name} Result: {result}') + group_projects = Utils.get_value_as_list('Items', result, []) + self.logger.debug(f'get_projects_by_group_name() - Group: {group_name} group_projects: {group_projects}') + project_ids = [] + for group_project in group_projects: + project_ids.append(group_project['project_id']) + self.logger.debug(f'get_projects_by_group_name() - Group: {group_name} group_projects: {group_projects} - Returning IDs: {project_ids}') + return project_ids + + def group_member_added(self, group_name: str, username: str): + if Utils.is_empty(group_name): + raise exceptions.invalid_params('group_name is required') + if Utils.is_empty(username): + raise exceptions.invalid_params('username is required') + self.logger.info(f'group_member_added() - Adding {username} to {group_name} in DDB. Fetch project_id for {group_name}') + + project_ids = self.get_projects_by_group_name(group_name) + for project_id in project_ids: + self.create_user_project( + project_id=project_id, + username=username + ) + + def group_member_removed(self, group_name: str, username: str): + if Utils.is_empty(group_name): + raise exceptions.invalid_params('group_name is required') + if Utils.is_empty(username): + raise exceptions.invalid_params('username is required') + project_ids = self.get_projects_by_group_name(group_name) + for project_id in project_ids: + self.delete_user_project( + project_id=project_id, + username=username + ) + + def project_disabled(self, project_id: str): + if Utils.are_empty(project_id): + raise exceptions.invalid_params('project_id is required') + + project = self.projects_dao.get_project_by_id(project_id) + ldap_groups = project['ldap_groups'] + + for ldap_group in ldap_groups: + self.ldap_group_removed(project_id=project_id, group_name=ldap_group) + + def project_enabled(self, project_id: str): + if Utils.are_empty(project_id): + raise exceptions.invalid_params('project_id is required') + + project = self.projects_dao.get_project_by_id(project_id) + ldap_groups = project['ldap_groups'] + + for ldap_group in ldap_groups: + self.ldap_group_added(project_id=project_id, group_name=ldap_group) + + def is_user_in_project(self, username: str, project_id: str): + if Utils.is_empty(username): + raise exceptions.invalid_params('username is required') + if Utils.is_empty(project_id): + raise exceptions.invalid_params('project_id is required') + + result = self.user_projects_table.get_item( + Key={ + 'username': username, + 'project_id': project_id + } + ) + return Utils.get_value_as_dict('Item', result) is not None diff --git a/source/idea/idea-cluster-manager/src/ideaclustermanager/app/projects/project_tasks.py b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/projects/project_tasks.py new file mode 100644 index 00000000..8d81bb9a --- /dev/null +++ b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/projects/project_tasks.py @@ -0,0 +1,89 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +__all__ = ( + 'ProjectEnabledTask', + 'ProjectDisabledTask', + 'ProjectGroupsUpdatedTask' +) + +from ideasdk.utils import Utils + +import ideaclustermanager +from ideaclustermanager.app.tasks.base_task import BaseTask + +from typing import Dict + + +class ProjectEnabledTask(BaseTask): + + def __init__(self, context: ideaclustermanager.AppContext): + self.context = context + self.logger = context.logger(self.get_name()) + + def get_name(self) -> str: + return 'projects.project-enabled' + + def invoke(self, payload: Dict): + project_id = payload['project_id'] + self.context.projects.user_projects_dao.project_enabled( + project_id=project_id + ) + + +class ProjectDisabledTask(BaseTask): + + def __init__(self, context: ideaclustermanager.AppContext): + self.context = context + self.logger = context.logger(self.get_name()) + + def get_name(self) -> str: + return 'projects.project-disabled' + + def invoke(self, payload: Dict): + project_id = payload['project_id'] + self.context.projects.user_projects_dao.project_disabled( + project_id=project_id + ) + + +class ProjectGroupsUpdatedTask(BaseTask): + + def __init__(self, context: ideaclustermanager.AppContext): + self.context = context + self.logger = context.logger(self.get_name()) + + def get_name(self) -> str: + return 'projects.project-groups-updated' + + def invoke(self, payload: Dict): + project_id = payload['project_id'] + project = self.context.projects.projects_dao.get_project_by_id(project_id) + if project['enabled']: + groups_added = Utils.get_value_as_list('groups_added', payload, []) + groups_removed = Utils.get_value_as_list('groups_removed', payload, []) + + for ldap_group_name in groups_removed: + self.context.projects.user_projects_dao.ldap_group_added( + project_id=project_id, + group_name=ldap_group_name + ) + # perform full refresh to ensure if users existed in multiple groups, and one of the group + # was deleted, those users get added back again + self.context.projects.user_projects_dao.project_enabled( + project_id=project_id + ) + + for ldap_group_name in groups_added: + self.context.projects.user_projects_dao.ldap_group_added( + project_id=project_id, + group_name=ldap_group_name + ) diff --git a/source/idea/idea-cluster-manager/src/ideaclustermanager/app/projects/projects_service.py b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/projects/projects_service.py new file mode 100644 index 00000000..63cc254c --- /dev/null +++ b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/projects/projects_service.py @@ -0,0 +1,352 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + + +from ideadatamodel import exceptions, errorcodes, constants +from ideadatamodel.projects import ( + CreateProjectRequest, + CreateProjectResult, + GetProjectRequest, + GetProjectResult, + UpdateProjectRequest, + UpdateProjectResult, + ListProjectsRequest, + ListProjectsResult, + EnableProjectRequest, + EnableProjectResult, + DisableProjectRequest, + DisableProjectResult, + GetUserProjectsRequest, + GetUserProjectsResult, + Project +) +from ideasdk.utils import Utils, GroupNameHelper +from ideasdk.context import SocaContext + +from ideaclustermanager.app.projects.db.projects_dao import ProjectsDAO +from ideaclustermanager.app.projects.db.user_projects_dao import UserProjectsDAO +from ideaclustermanager.app.accounts.accounts_service import AccountsService +from ideaclustermanager.app.tasks.task_manager import TaskManager + + +class ProjectsService: + + def __init__(self, context: SocaContext, accounts_service: AccountsService, task_manager: TaskManager): + self.context = context + self.accounts_service = accounts_service + self.task_manager = task_manager + self.logger = context.logger('projects') + + self.projects_dao = ProjectsDAO(context) + self.projects_dao.initialize() + + self.user_projects_dao = UserProjectsDAO( + context=context, + projects_dao=self.projects_dao, + accounts_service=self.accounts_service + ) + self.user_projects_dao.initialize() + + def create_project(self, request: CreateProjectRequest) -> CreateProjectResult: + """ + Create a new Project + validate required fields, add the project to DynamoDB and Cache. + :param request: + :return: the created project (with project_id) + """ + + ds_provider = self.context.config().get_string('directoryservice.provider', required=True) + + if Utils.is_empty(request): + raise exceptions.invalid_params('request is required') + + project = request.project + if Utils.is_empty(project): + raise exceptions.invalid_params('project is required') + + if Utils.is_empty(project.name): + raise exceptions.invalid_params('project.name is required') + + existing = self.projects_dao.get_project_by_name(project.name) + if existing is not None: + raise exceptions.invalid_params(f'project with name: {project.name} already exists') + + if Utils.is_empty(project.ldap_groups): + raise exceptions.invalid_params('ldap_groups[] is required') + for ldap_group_name in project.ldap_groups: + # check if group exists + # Active Directory mode checks the back-end LDAP + if ds_provider in {constants.DIRECTORYSERVICE_ACTIVE_DIRECTORY}: + self.logger.debug(f'Performing DS lookup for group: {ldap_group_name}') + self.accounts_service.ldap_client.get_group(ldap_group_name) + else: + self.accounts_service.get_group(ldap_group_name) + + enable_budgets = Utils.get_as_bool(project.enable_budgets, False) + if enable_budgets: + if project.budget is None or Utils.is_empty(project.budget.budget_name): + raise exceptions.invalid_params('budget.budget_name is required when budgets are enabled') + budget_name = project.budget.budget_name + self.context.aws_util().budgets_get_budget(budget_name) + + # ensure project is always disabled during creation + project.enabled = False + + db_project = self.projects_dao.convert_to_db(project) + db_created_project = self.projects_dao.create_project(db_project) + + created_project = self.projects_dao.convert_from_db(db_created_project) + + return CreateProjectResult( + project=created_project + ) + + def get_project(self, request: GetProjectRequest) -> GetProjectResult: + """ + Retrieve the Project from the cache + :param request.project_name name of the project you are getting + :param request.project_id UUID of the project being searched + :return: Project from cache + """ + if Utils.is_empty(request): + raise exceptions.invalid_params('request is required') + if Utils.are_empty(request.project_id, request.project_name): + raise exceptions.invalid_params('Either project_id or project_name is required') + + self.logger.debug(f'get_project(): running with request: {request}') + + project = None + if Utils.is_not_empty(request.project_id): + project = self.projects_dao.get_project_by_id(request.project_id) + elif Utils.is_not_empty(request.project_name): + project = self.projects_dao.get_project_by_name(request.project_name) + + if project is None: + if Utils.is_not_empty(request.project_id): + raise exceptions.soca_exception( + error_code=errorcodes.PROJECT_NOT_FOUND, + message=f'project not found for project id: {request.project_id}' + ) + if Utils.is_not_empty(request.project_name): + raise exceptions.soca_exception( + error_code=errorcodes.PROJECT_NOT_FOUND, + message=f'project not found for project name: {request.project_name}' + ) + + return GetProjectResult( + project=self.projects_dao.convert_from_db(project) + ) + + def update_project(self, request: UpdateProjectRequest) -> UpdateProjectResult: + """ + Update a Project + :param request: + :return: + """ + if Utils.is_empty(request): + raise exceptions.invalid_params('request is required') + + project = request.project + if Utils.is_empty(project): + raise exceptions.invalid_params('project is required') + if Utils.is_empty(project.project_id): + raise exceptions.invalid_params('project.project_id is required') + + existing = self.projects_dao.get_project_by_id(project_id=project.project_id) + + if existing is None: + raise exceptions.soca_exception( + error_code=errorcodes.PROJECT_NOT_FOUND, + message=f'project not found for id: {project.project_id}' + ) + + if Utils.is_not_empty(project.name) and existing['name'] != project.name: + same_name_project = self.projects_dao.get_project_by_name(project.name) + if same_name_project is not None and same_name_project['project_id'] != project.project_id: + raise exceptions.invalid_params(f'project with name: {project.name} already exists') + + enable_budgets = Utils.get_as_bool(project.enable_budgets, False) + if enable_budgets: + if project.budget is None or Utils.is_empty(project.budget.budget_name): + raise exceptions.invalid_params('budget.budget_name is required when budgets are enabled') + budget_name = project.budget.budget_name + self.context.aws_util().budgets_get_budget(budget_name) + + groups_added = None + groups_removed = None + if Utils.is_not_empty(project.ldap_groups): + existing_ldap_groups = set(Utils.get_value_as_list('ldap_groups', existing, [])) + updated_ldap_groups = set(project.ldap_groups) + + groups_added = updated_ldap_groups - existing_ldap_groups + groups_removed = existing_ldap_groups - updated_ldap_groups + + if len(groups_added) > 0: + for ldap_group_name in groups_added: + # check if group exists + self.accounts_service.get_group(ldap_group_name) + + # none values will be skipped by db update. ensure enabled/disabled cannot be called via update project. + project.enabled = None + + db_updated = self.projects_dao.update_project(self.projects_dao.convert_to_db(project)) + updated_project = self.projects_dao.convert_from_db(db_updated) + + if updated_project.enabled: + if groups_added is not None or groups_removed is not None: + self.task_manager.send( + task_name='projects.project-groups-updated', + payload={ + 'project_id': updated_project.project_id, + 'groups_added': groups_added, + 'groups_removed': groups_removed + }, + message_group_id=updated_project.project_id + ) + + return UpdateProjectResult( + project=updated_project + ) + + def enable_project(self, request: EnableProjectRequest) -> EnableProjectResult: + + if Utils.is_empty(request): + raise exceptions.invalid_params('request is required') + if Utils.are_empty(request.project_id, request.project_name): + raise exceptions.invalid_params('Either project_id or project_name is required') + + project = None + if Utils.is_not_empty(request.project_id): + project = self.projects_dao.get_project_by_id(request.project_id) + elif Utils.is_not_empty(request.project_name): + project = self.projects_dao.get_project_by_name(request.project_name) + + if project is None: + raise exceptions.soca_exception( + error_code=errorcodes.PROJECT_NOT_FOUND, + message='project not found' + ) + + self.projects_dao.update_project({ + 'project_id': project['project_id'], + 'enabled': True + }) + + self.task_manager.send( + task_name='projects.project-enabled', + payload={ + 'project_id': project['project_id'] + }, + message_group_id=project['project_id'], + message_dedupe_id=Utils.short_uuid() + ) + + return EnableProjectResult() + + def disable_project(self, request: DisableProjectRequest) -> DisableProjectResult: + + if Utils.is_empty(request): + raise exceptions.invalid_params('request is required') + if Utils.are_empty(request.project_id, request.project_name): + raise exceptions.invalid_params('Either project_id or project_name is required') + + project = None + if Utils.is_not_empty(request.project_id): + project = self.projects_dao.get_project_by_id(request.project_id) + elif Utils.is_not_empty(request.project_name): + project = self.projects_dao.get_project_by_name(request.project_name) + + if project is None: + raise exceptions.soca_exception( + error_code=errorcodes.PROJECT_NOT_FOUND, + message='project not found' + ) + self.projects_dao.update_project({ + 'project_id': project['project_id'], + 'enabled': False + }) + + self.task_manager.send( + task_name='projects.project-disabled', + payload={ + 'project_id': project['project_id'] + }, + message_group_id=project['project_id'], + message_dedupe_id=Utils.short_uuid() + ) + + return DisableProjectResult() + + def list_projects(self, request: ListProjectsRequest) -> ListProjectsResult: + return self.projects_dao.list_projects(request) + + def get_user_projects(self, request: GetUserProjectsRequest) -> GetUserProjectsResult: + if Utils.is_empty(request): + raise exceptions.invalid_params('request is required') + if Utils.is_empty(request.username): + raise exceptions.invalid_params('username is required') + + self.logger.debug(f'get_user_projects() - request: {request}') + + # Probe directory service + ds_provider = self.context.config().get_string('directoryservice.provider', required=True) + self.logger.debug(f'ProjectsService.get_user_projects() - DS Provider is {ds_provider} ...') + if ds_provider in {constants.DIRECTORYSERVICE_ACTIVE_DIRECTORY}: + self.logger.debug(f'get_user_projects() - Running in AD mode - performing AD query for {request.username} group memberships...') + user_result = self.accounts_service.ldap_client.get_user(username=request.username) + self.logger.debug(f'get_user_projects() - User Result: {user_result}') + + user_projects = self.user_projects_dao.get_projects_by_username(request.username) + + result = [] + # todo - batch get + for project_id in user_projects: + db_project = self.projects_dao.get_project_by_id(project_id) + if db_project is None: + continue + if not db_project['enabled']: + continue + result.append(self.projects_dao.convert_from_db(db_project)) + result.sort(key=lambda p: p.name) + + return GetUserProjectsResult( + projects=result + ) + + def create_defaults(self): + + ds_provider = self.context.config().get_string('directoryservice.provider', required=True) + + self.logger.debug(f'ProjectsService.create_defaults() - DS Provider is {ds_provider} ...') + + default_project_group_name = GroupNameHelper(self.context).get_default_project_group() + + if ds_provider in {constants.DIRECTORYSERVICE_ACTIVE_DIRECTORY}: + default_project_group_name_ds = self.context.config().get_string('directoryservice.group_mapping.default-project-group', required=True) + else: + default_project_group_name_ds = default_project_group_name + + default_project = self.projects_dao.get_project_by_name(constants.DEFAULT_PROJECT) + self.logger.debug(f'Default project group name: {default_project_group_name} Project: {default_project}') + + if default_project is None: + self.logger.info('creating and enabling default project ...') + result = self.create_project(CreateProjectRequest( + project=Project( + name=constants.DEFAULT_PROJECT, + title='Default Project', + description='Default Project', + ldap_groups=[GroupNameHelper(self.context).get_default_project_group()] + ) + )) + self.enable_project(EnableProjectRequest( + project_id=result.project.project_id + )) diff --git a/source/idea/idea-cluster-manager/src/ideaclustermanager/app/tasks/__init__.py b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/tasks/__init__.py new file mode 100644 index 00000000..c9059a8c --- /dev/null +++ b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/tasks/__init__.py @@ -0,0 +1,12 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +from ideaclustermanager.app.tasks.base_task import BaseTask diff --git a/source/idea/idea-cluster-manager/src/ideaclustermanager/app/tasks/base_task.py b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/tasks/base_task.py new file mode 100644 index 00000000..f8c86014 --- /dev/null +++ b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/tasks/base_task.py @@ -0,0 +1,24 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +from abc import abstractmethod +from typing import Dict + + +class BaseTask: + + @abstractmethod + def get_name(self) -> str: + ... + + @abstractmethod + def invoke(self, payload: Dict): + ... diff --git a/source/idea/idea-cluster-manager/src/ideaclustermanager/app/tasks/task_manager.py b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/tasks/task_manager.py new file mode 100644 index 00000000..b015dfc4 --- /dev/null +++ b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/tasks/task_manager.py @@ -0,0 +1,121 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +from ideasdk.context import SocaContext +from ideasdk.service import SocaService +from ideasdk.utils import Utils + +from ideaclustermanager.app.tasks.base_task import BaseTask + +from typing import Dict, List +from concurrent.futures import ThreadPoolExecutor +import threading + +MAX_WORKERS = 1 # must be between 1 and 10 + + +class TaskManager(SocaService): + + def __init__(self, context: SocaContext, tasks: List[BaseTask]): + super().__init__(context) + + self.context = context + self.logger = context.logger('task-manager') + + self.tasks: Dict[str, BaseTask] = {} + for task in tasks: + self.tasks[task.get_name()] = task + + self.exit = threading.Event() + + self.task_monitor_thread = threading.Thread( + target=self.task_queue_listener, + name='task-monitor' + ) + self.task_executors = ThreadPoolExecutor( + max_workers=MAX_WORKERS, + thread_name_prefix='task-executor' + ) + + def get_task_queue_url(self) -> str: + return self.context.config().get_string('cluster-manager.task_queue_url', required=True) + + def execute_task(self, sqs_message: Dict): + task_name = None + try: + message_body = Utils.get_value_as_string('Body', sqs_message) + receipt_handle = Utils.get_value_as_string('ReceiptHandle', sqs_message) + task_message = Utils.from_json(message_body) + task_name = task_message['name'] + task_payload = task_message['payload'] + self.logger.info(f'executing task: {task_name}, payload: {Utils.to_json(task_payload)}') + + if task_name not in self.tasks: + self.logger.warning(f'not task registered for task name: {task_name}') + return + + task = self.tasks[task_name] + + task.invoke(task_payload) + + self.context.aws().sqs().delete_message( + QueueUrl=self.get_task_queue_url(), + ReceiptHandle=receipt_handle + ) + + except Exception as e: + self.logger.exception(f'failed to execute task: {task_name} - {e}') + + def task_queue_listener(self): + while not self.exit.is_set(): + try: + result = self.context.aws().sqs().receive_message( + QueueUrl=self.get_task_queue_url(), + MaxNumberOfMessages=MAX_WORKERS, + WaitTimeSeconds=20 + ) + messages = Utils.get_value_as_list('Messages', result, []) + if len(messages) == 0: + continue + self.logger.info(f'received {len(messages)} messages') + for message in messages: + self.task_executors.submit(lambda message_: self.execute_task(message_), message) + + except Exception as e: + self.logger.exception(f'failed to poll queue: {e}') + + def send(self, task_name: str, payload: Dict, message_group_id: str = None, message_dedupe_id: str = None): + task_message = { + 'name': task_name, + 'payload': payload + } + + if Utils.is_empty(message_group_id): + message_group_id = task_name + if Utils.is_empty(message_dedupe_id): + message_dedupe_id = Utils.sha256(Utils.to_json(task_message)) + + self.logger.debug(f'send task: {task_name}, message group id: {message_group_id}, DedupeId: {message_dedupe_id}') + self.context.aws().sqs().send_message( + QueueUrl=self.get_task_queue_url(), + MessageBody=Utils.to_json(task_message), + MessageDeduplicationId=message_dedupe_id, + MessageGroupId=message_group_id + ) + + def start(self): + self.task_monitor_thread.start() + + def stop(self): + self.exit.set() + if self.task_monitor_thread.is_alive(): + self.task_monitor_thread.join() + self.task_executors.shutdown(wait=True) diff --git a/source/idea/idea-cluster-manager/src/ideaclustermanager/app/web_portal.py b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/web_portal.py new file mode 100644 index 00000000..c0257b0d --- /dev/null +++ b/source/idea/idea-cluster-manager/src/ideaclustermanager/app/web_portal.py @@ -0,0 +1,270 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +import ideaclustermanager +import ideaclustermanager.app.app_messages as app_messages + +from ideasdk.utils import Utils, EnvironmentUtils, Jinja2Utils, ModuleMetadataHelper +from ideasdk.server import SocaServer +from ideadatamodel import exceptions, constants +from ideadatamodel.auth import ( + User, + Group, +) + +import os +from pathlib import Path +import sanic +from typing import Dict +from datetime import datetime + + +class WebPortal: + + def __init__(self, context: ideaclustermanager.AppContext, server: SocaServer): + self.context = context + self.server = server + self.logger = context.logger('web-portal') + self.module_metadata_helper = ModuleMetadataHelper() + self.web_app_dir = self.get_web_app_dir() + self.web_resources_context_path = self.context.config().get_string('cluster-manager.web_resources_context_path', '/') + self.web_template_env = Jinja2Utils.env_using_file_system_loader(search_path=self.web_app_dir, auto_escape=True) + + @staticmethod + def is_dev_mode() -> bool: + return Utils.get_as_bool(EnvironmentUtils.idea_dev_mode(), False) + + @property + def dev_mode_web_portal_root_dir(self) -> str: + script_dir = Path(os.path.abspath(__file__)) + return str(script_dir.parent.parent.parent.parent) + + def get_web_app_dir(self) -> str: + if self.is_dev_mode(): + web_sources_dir = os.path.join(self.dev_mode_web_portal_root_dir, 'webapp') + html_build_dir = os.path.join(web_sources_dir, 'build-dev-mode') + os.makedirs(html_build_dir, exist_ok=True) + index_page = os.path.join(html_build_dir, 'index.html') + with open(index_page, 'w') as f: + f.write(app_messages.DEV_MODE_INDEX_PAGE) + return html_build_dir + else: + return os.path.join(Utils.app_deploy_dir(), 'cluster-manager', 'webapp') + + def get_logo_url(self) -> str: + logo = self.context.config().get_string('cluster-manager.web_portal.logo') + if Utils.is_not_empty(logo): + # logo is provided as an external url or another file in public folder. + if logo.startswith('https://') or logo.startswith('/'): + return logo + + # assume logo is Key of the logo image file in cluster's s3 bucket. + # create a pre-signed url and return + return self.context.aws_util().create_s3_presigned_url( + key=logo, + expires_in=12 * 60 * 60 # 12 hours + ) + + return self.make_route_path('/logo.png') # default IDEA logo in public directory + + def get_copyright_text(self) -> str: + copyright_text = self.context.config().get_string('cluster-manager.web_portal.copyright_text') + + if Utils.is_empty(copyright_text): + copyright_text = constants.DEFAULT_COPYRIGHT_TEXT + + copyright_text = copyright_text.replace('{year}', str(datetime.now().year)) + return copyright_text + + def build_app_init_data(self, http_request) -> Dict: + sso_enabled = self.context.config().get_bool('identity-provider.cognito.sso_enabled', False) + + # build module metadata and applicable api context paths + module_set_id = Utils.get_as_string(self.server.get_query_param_as_string('module_set', http_request), self.context.module_set()) + modules = [] + module_set = self.context.config().get_config(f'global-settings.module_sets.{module_set_id}') + for module_name in module_set: + module_metadata = self.module_metadata_helper.get_module_metadata(module_name) + api_context_path = None + module_id = module_set.get_string(f'{module_name}.module_id') + # send from server side, instead of having path info hard-coded clients + # scope for further enhancements to read context path from applicable module's configuration + if module_metadata.type == constants.MODULE_TYPE_APP: + api_context_path = f'/{module_id}/api/v1' + modules.append({ + **Utils.to_dict(module_metadata), + 'module_id': module_id, + 'api_context_path': api_context_path + }) + + app_init_data = { + 'version': ideaclustermanager.__version__, + 'sso': sso_enabled, + 'cluster_name': self.context.cluster_name(), + 'aws_region': self.context.aws().aws_region(), + 'title': self.context.config().get_string('cluster-manager.web_portal.title', 'Integrated Digital Engineering on AWS'), + 'subtitle': self.context.config().get_string('cluster-manager.web_portal.subtitle', f'{self.context.cluster_name()} ({self.context.aws().aws_region()})'), + 'logo': self.get_logo_url(), + 'copyright_text': self.get_copyright_text(), + 'default_log_level': self.context.config().get_int('cluster-manager.web_portal.default_log_level', 3), + 'module_set': module_set_id, + 'modules': modules, + 'session_management': self.context.config().get_string('cluster-manager.web_portal.session_management', 'in-memory') + } + + if sso_enabled: + sso_auth_status = self.server.get_query_param_as_string('sso_auth_status', http_request) + if Utils.is_not_empty(sso_auth_status): + app_init_data['sso_auth_status'] = sso_auth_status + + sso_auth_code = self.server.get_query_param_as_string('sso_auth_code', http_request) + if Utils.is_not_empty(sso_auth_code): + app_init_data['sso_auth_code'] = sso_auth_code + + return app_init_data + + async def index_route(self, http_request): + template = self.web_template_env.get_template('index.html') + app_init_data = self.build_app_init_data(http_request) + result = template.render(app_init_data=Utils.base64_encode(Utils.to_json(app_init_data))) + + # add X-Frame-Options header to mitigate ClickJacking (embedding WebPortal within an iframe) + return sanic.response.html( + body=result, + status=200, + headers={ + 'X-Frame-Options': 'SAMEORIGIN' + } + ) + + async def sso_initiate_route(self, _): + sso_enabled = self.context.config().get_bool('identity-provider.cognito.sso_enabled', False) + if not sso_enabled: + return sanic.response.redirect(self.web_resources_context_path) + + try: + domain_url = self.context.config().get_string('identity-provider.cognito.domain_url', required=True) + sso_client_id = self.context.config().get_string('identity-provider.cognito.sso_client_id', required=True) + sso_idp_provider_name = self.context.config().get_string('identity-provider.cognito.sso_idp_provider_name', required=True) + external_endpoint_url = self.context.config().get_cluster_external_endpoint() + state = Utils.uuid() + redirect_uri = f'{external_endpoint_url}{self.make_route_path("/sso/oauth2/callback")}' + query_params = [ + 'response_type=code', + f'redirect_uri={Utils.url_encode(redirect_uri)}', + f'state={state}', + f'client_id={sso_client_id}', + f'identity_provider={sso_idp_provider_name}' + ] + authorization_url = f'{domain_url}/authorize?{"&".join(query_params)}' + + # todo - add throttling to prevent ddb flooding from malicious actors and code + self.context.accounts.sso_state_dao.create_sso_state({ + 'state': state, + 'redirect_uri': redirect_uri + }) + + self.logger.debug(f'redirecting: {authorization_url}') + return sanic.response.redirect(authorization_url) + except Exception as e: + self.logger.exception(f'failed to initiate sso: {e}') + return sanic.response.redirect(f'{self.web_resources_context_path}?sso_auth_status=FAIL') + + async def sso_oauth2_callback_route(self, http_request): + sso_enabled = self.context.config().get_bool('identity-provider.cognito.sso_enabled', False) + if not sso_enabled: + return sanic.response.redirect(self.web_resources_context_path) + + try: + + authorization_code = self.server.get_query_param_as_string('code', http_request) + state = self.server.get_query_param_as_string('state', http_request) + error = self.server.get_query_param_as_string('error', http_request) + error_description = self.server.get_query_param_as_string('error_description', http_request) + + if Utils.is_not_empty(error): + self.logger.warning(f'sso auth failure: error={error}, error_description={error_description}') + return sanic.response.redirect(f'{self.web_resources_context_path}?sso_auth_status=FAIL') + + if Utils.is_empty(authorization_code): + raise exceptions.invalid_params('code is required') + if Utils.is_empty(state): + raise exceptions.invalid_params('state is required') + + self.logger.debug(f'sso callback: authorization_code={authorization_code}, state={state}') + + # todo - add throttling to prevent ddb flooding from malicious actors and code + sso_state = self.context.accounts.sso_state_dao.get_sso_state(state=state) + if sso_state is None: + raise exceptions.invalid_params(f'sso state not found for state: {state}') + + auth_result = self.context.token_service.get_access_token_using_sso_auth_code( + authorization_code=authorization_code, + redirect_uri=sso_state['redirect_uri'] + ) + + # decode the access token and check if the username matches to the user that exists in DB + claims = self.context.token_service.decode_token(auth_result.access_token) + self.logger.info(f'sso token claims: {Utils.to_json(claims)}') + + username = Utils.get_value_as_string('username', claims) + self.logger.debug(f'Cognito SSO claims - Username: {username}') + + if Utils.is_empty(username): + self.logger.exception(f'Error - Unable to read IdP username from claims: {claims}') + return sanic.response.redirect(f'{self.web_resources_context_path}?sso_auth_status=FAIL') + + self.logger.debug(f'SSO auth: Looking up user: {username}') + existing_user = self.context.accounts.user_dao.get_user(username=username) + + if existing_user is None: + self.logger.warning(f'SSO auth: {username} is not previously known. User must be created before first SSO attempt') + # TODO auto-enrollment would go here + self.logger.info(f'SSO auth: Unable to process {username}') + self.logger.info(f'disabling federated user in user pool: {username}') + self.context.user_pool.admin_disable_user(username=username) + self.logger.info(f'deleting federated user in user pool: {username}') + self.context.user_pool.admin_delete_user(username=username) + return sanic.response.redirect(f'{self.web_resources_context_path}?sso_auth_status=FAIL') + + self.logger.debug(f'Updating SSO State for user {username}: {state}') + self.context.accounts.sso_state_dao.update_sso_state({ + 'state': state, + 'access_token': auth_result.access_token, + 'refresh_token': auth_result.refresh_token, + 'expires_in': auth_result.expires_in, + 'id_token': auth_result.id_token, + 'token_type': auth_result.token_type + }) + return sanic.response.redirect(f'{self.web_resources_context_path}?sso_auth_status=SUCCESS&sso_auth_code={state}') + + except Exception as e: + self.logger.exception(f'failed to process sso oauth2 callback: {e}') + return sanic.response.redirect(f'{self.web_resources_context_path}?sso_auth_status=FAIL') + + def make_route_path(self, path: str) -> str: + if not path.startswith('/'): + path = f'/{path}' + + if self.web_resources_context_path == '/': + return path + else: + return f'{self.web_resources_context_path}{path}' + + def initialize(self): + # index route + self.server.http_app.add_route(self.index_route, self.web_resources_context_path) + # sso route + self.server.http_app.add_route(self.sso_initiate_route, self.make_route_path('/sso')) + # sso oauth 2 callback route + self.server.http_app.add_route(self.sso_oauth2_callback_route, self.make_route_path('/sso/oauth2/callback')) + # add static resources at the end + self.server.http_app.static(self.web_resources_context_path, self.web_app_dir) diff --git a/source/idea/idea-cluster-manager/src/ideaclustermanager/cli/__init__.py b/source/idea/idea-cluster-manager/src/ideaclustermanager/cli/__init__.py new file mode 100644 index 00000000..7da95b1d --- /dev/null +++ b/source/idea/idea-cluster-manager/src/ideaclustermanager/cli/__init__.py @@ -0,0 +1,34 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +from ideadatamodel import constants +from ideasdk.context import SocaContextOptions, SocaCliContext +from ideasdk.utils import EnvironmentUtils + + +def build_cli_context(cluster_config: bool = False, enable_aws_client_provider: bool = False, enable_aws_util: bool = False): + cluster_name = EnvironmentUtils.idea_cluster_name(required=True) + module_id = EnvironmentUtils.idea_module_id(required=True) + module_set = EnvironmentUtils.idea_module_set(required=True) + aws_region = EnvironmentUtils.aws_default_region(required=True) + return SocaCliContext( + api_context_path=f'/{module_id}/api/v1', + options=SocaContextOptions( + module_name=constants.MODULE_CLUSTER_MANAGER, + module_id=module_id, + module_set=module_set, + cluster_name=cluster_name if cluster_config else None, + aws_region=aws_region, + enable_aws_client_provider=enable_aws_client_provider, + enable_aws_util=enable_aws_util, + default_logging_profile='console' + ) + ) diff --git a/source/idea/idea-cluster-manager/src/ideaclustermanager/cli/accounts.py b/source/idea/idea-cluster-manager/src/ideaclustermanager/cli/accounts.py new file mode 100644 index 00000000..d66630a2 --- /dev/null +++ b/source/idea/idea-cluster-manager/src/ideaclustermanager/cli/accounts.py @@ -0,0 +1,339 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +from ideadatamodel import ( + exceptions, + errorcodes, + constants, + GetUserResult, + GetUserRequest, + ListUsersRequest, + ListUsersResult, + CreateUserRequest, + CreateUserResult, + EnableUserRequest, + EnableUserResult, + DisableUserRequest, + DisableUserResult, + User, + ModifyUserRequest, + ModifyUserResult, + DeleteUserRequest, + DeleteUserResult +) +from ideaclustermanager.cli import build_cli_context +from ideasdk.utils import Utils +from ideaclustermanager.app.accounts.auth_utils import AuthUtils +from ideaclustermanager.cli.cli_utils import ClusterManagerUtils + +from rich.table import Table +import click +import csv + + +@click.group() +def accounts(): + """ + account management options + """ + + +@accounts.command(context_settings=constants.CLICK_SETTINGS) +@click.option('--username', required=True, help='Username') +def get_user(username: str): + """ + get user + """ + context = ClusterManagerUtils.get_soca_cli_context_cluster_manager() + try: + result = context.unix_socket_client.invoke_alt( + namespace='Accounts.GetUser', + payload=GetUserRequest( + username=username + ), + result_as=GetUserResult + ) + except exceptions.SocaException as e: + context.error(e.message) + return + context.print(Utils.to_yaml(result.user)) + + +@accounts.command(context_settings=constants.CLICK_SETTINGS) +@click.option('--username', required=True, help='Username') +def enable_user(username: str): + """ + enable user + """ + context = ClusterManagerUtils.get_soca_cli_context_cluster_manager() + try: + context.unix_socket_client.invoke_alt( + namespace='Accounts.EnableUser', + payload=EnableUserRequest( + username=username + ), + result_as=EnableUserResult + ) + except exceptions.SocaException as e: + context.error(e.message) + + +@accounts.command(context_settings=constants.CLICK_SETTINGS) +@click.option('--username', required=True, help='Username') +def disable_user(username: str): + """ + disable user + """ + context = ClusterManagerUtils.get_soca_cli_context_cluster_manager() + try: + context.unix_socket_client.invoke_alt( + namespace='Accounts.DisableUser', + payload=DisableUserRequest( + username=username + ), + result_as=DisableUserResult + ) + except exceptions.SocaException as e: + context.error(e.message) + + +@accounts.command(context_settings=constants.CLICK_SETTINGS) +@click.option('--username', required=True, help='Username') +def delete_user(username: str): + """ + delete user + """ + context = ClusterManagerUtils.get_soca_cli_context_cluster_manager() + try: + context.unix_socket_client.invoke_alt( + namespace='Accounts.DeleteUser', + payload=DeleteUserRequest( + username=username + ), + result_as=DeleteUserResult + ) + except exceptions.SocaException as e: + context.error(e.message) + + +@accounts.command(context_settings=constants.CLICK_SETTINGS) +def list_users(): + """ + list existing users + """ + + context = ClusterManagerUtils.get_soca_cli_context_cluster_manager() + result = context.unix_socket_client.invoke_alt( + namespace='Accounts.ListUsers', + payload=ListUsersRequest(), + result_as=ListUsersResult + ) + user_table = Table() + user_table.add_column("Username", justify="left", no_wrap=False) + user_table.add_column("UID", justify="left", no_wrap=False) + user_table.add_column("GID", justify="left", no_wrap=False) + user_table.add_column("Email", justify="left", no_wrap=False) + user_table.add_column("Is Admin?", justify="left", no_wrap=False) + user_table.add_column("Status", justify="left", no_wrap=False) + user_table.add_column("Group", justify="left", style="green", no_wrap=False) + user_table.add_column("Created On", justify="left", style="red", no_wrap=False) + for listing in result.listing: + user_table.add_row(listing.username, Utils.get_as_string(listing.uid), Utils.get_as_string(listing.gid), + listing.email, Utils.get_as_string(listing.sudo), Utils.get_as_string(listing.status), + listing.group_name, listing.created_on.strftime("%m/%d/%Y, %H:%M:%S")) + context.print(user_table) + + +@accounts.command(context_settings=constants.CLICK_SETTINGS) +@click.option('--username', required=True, help='The Username') +@click.option('--email', required=True, help='Email Address') +@click.option('--password', help='Password') +@click.option('--uid', help='UID') +@click.option('--gid', help='GID') +@click.option('--email-verified', is_flag=True, help='Indicate if the email address is verified. Invitation email will not be sent.') +@click.option('--sudo', is_flag=True, help='Indicate if user is an admin user with sudo access') +def create_user(**kwargs): + """ + create new user account + """ + request = { + 'user': { + 'username': Utils.get_value_as_string('username', kwargs), + 'email': Utils.get_value_as_string('email', kwargs), + 'sudo': Utils.get_value_as_bool('sudo', kwargs, False), + 'password': Utils.get_value_as_string('password', kwargs), + 'uid': Utils.get_value_as_int('uid', kwargs), + 'gid': Utils.get_value_as_int('gid', kwargs) + }, + 'email_verified': Utils.get_value_as_bool('email_verified', kwargs, False) + } + + context = build_cli_context() + result = context.unix_socket_client.invoke_alt( + namespace='Accounts.CreateUser', + payload=CreateUserRequest(**request), + result_as=CreateUserResult + ) + context.print_json(Utils.to_json(result.user)) + + +@accounts.command(context_settings=constants.CLICK_SETTINGS) +@click.option('--username', required=True, help='The Username') +@click.option('--email', help='Email Address') +@click.option('--uid', help='UID') +@click.option('--gid', help='GID') +@click.option('--sudo', help='Add or remove sudo access. Allowed values [yes/no/true/false/0/1]') +@click.option('--login-shell', help='Login shell for the user') +def modify_user(**kwargs): + """ + update an existing user account + + \b + supported fields: + * uid + * gid + * email + * sudo + """ + request = { + 'user': { + 'username': Utils.get_value_as_string('username', kwargs), + 'email': Utils.get_value_as_string('email', kwargs), + 'sudo': Utils.get_value_as_bool('sudo', kwargs), + 'uid': Utils.get_value_as_int('uid', kwargs), + 'gid': Utils.get_value_as_int('gid', kwargs), + 'login_shell': Utils.get_value_as_string('login_shell', kwargs) + } + } + + context = build_cli_context() + result = context.unix_socket_client.invoke_alt( + namespace='Accounts.ModifyUser', + payload=ModifyUserRequest(**request), + result_as=ModifyUserResult + ) + context.print_json(Utils.to_json(result.user)) + + +@accounts.command(context_settings=constants.CLICK_SETTINGS) +@click.option('--path-to-csv', help='path to the csv file') +@click.option('--force', is_flag=True, help='skips confirmation prompts') +@click.option('--generate-template', is_flag=True, help='generates a csv template compatible with this command') +@click.option('--template-path', help='location to generate template.csv file') +def batch_create_users(path_to_csv: str, force: bool, generate_template: bool, template_path: str): + """ + creates users from csv file + """ + + context = ClusterManagerUtils.get_soca_cli_context_cluster_manager() + + if generate_template: + # generate template + if template_path and Utils.is_dir(template_path): + csv_file = ClusterManagerUtils.generate_template(template_path) + context.info(f'The generated template file is located at {csv_file}') + else: + context.error('Path missing or incorrect; please provide a path to where to generate the "users_file_template.csv"') + return + + ClusterManagerUtils.check_if_csv_file(path_to_csv, context) + + user_table = ClusterManagerUtils.build_confirm_user_table() + status_table = ClusterManagerUtils.build_create_user_status_table() + + new_users = [] + new_usernames = [] + csv_validations = [] + csv_contains_error = False + csv_contains_duplicate = False + with open(path_to_csv, 'r', encoding="utf-8") as csv_file: + csv_reader = csv.DictReader(csv_file) + fieldnames = csv_reader.fieldnames + + # check for valid csv file + ClusterManagerUtils.check_has_valid_csv_headers(fieldnames, context) + + for row in csv_reader: + new_username = AuthUtils.sanitize_username(Utils.get_value_as_string('Username', row)) + valid_username_response = '' + try: + ClusterManagerUtils.check_valid_username(new_username) + except exceptions.SocaException as e: + if e.error_code == errorcodes.INVALID_PARAMS: + valid_username_response = e.message + else: + raise e + if not Utils.is_empty(valid_username_response): + csv_contains_error = True + new_email = Utils.get_value_as_string('Email', row).strip().lower() + + valid_email_response = '' + try: + ClusterManagerUtils.check_valid_email(new_email) + except exceptions.SocaException as e: + if e.error_code == errorcodes.INVALID_PARAMS: + valid_email_response = e.message + else: + raise e + if not Utils.is_empty(valid_email_response): + csv_contains_error = True + new_is_admin = Utils.get_value_as_bool('Is Admin?', row, default=False) + + new_usernames.append(new_username) + csv_validations.append((new_username, new_email, Utils.get_as_string(new_is_admin, default='False'), valid_username_response, valid_email_response)) + + user_table.add_row(new_username, new_email, Utils.get_as_string(new_is_admin)) + new_users.append(User(username=new_username, email=new_email, sudo=new_is_admin)) + + duplicate_users_in_csv = ClusterManagerUtils.check_duplicate_users_in_csv(new_usernames) + if len(duplicate_users_in_csv) > 0: + table = Table() + table.add_column('Username', justify='left', no_wrap=False) + for user in duplicate_users_in_csv: + table.add_row(user) + context.print(table) + context.error('These users are repeated in the csv file; please ensure all usernames are unique in the csv file and try again.') + return + + duplicate_users = ClusterManagerUtils.check_duplicate_users(new_usernames, context) + if len(duplicate_users) > 0: + csv_contains_duplicate = True + + if csv_contains_error or csv_contains_duplicate: + for item in csv_validations: + is_duplicate = item[0] in duplicate_users + + if not Utils.is_empty(item[3]) and not Utils.is_empty(item[4]): + status_table.add_row(item[0], item[1], item[2], "FAILED", f'{item[3]} AND {item[4]}', style='red') + elif not Utils.is_empty(item[3]): + status_table.add_row(item[0], item[1], item[2], "FAILED", f'{item[3]}', style='red') + elif not Utils.is_empty(item[4]) and is_duplicate: + status_table.add_row(item[0], item[1], item[2], "FAILED", f'user already exists; please remove them from the csv and try again AND {item[4]}', style='red') + elif not Utils.is_empty(item[4]): + status_table.add_row(item[0], item[1], item[2], "FAILED", f'{item[4]}', style='red') + elif is_duplicate: + status_table.add_row(item[0], item[1], item[2], "FAILED", 'user already exists; please remove them from the csv and try again', style='red') + else: + status_table.add_row(item[0], item[1], item[2], "ABORTED", 'csv file contains errors', style='bright_black') + context.print(status_table) + raise SystemExit(1) + + if not force: + context.print(user_table) + continue_deployment = context.prompt(f'Are you sure you want to add these users?') + if not continue_deployment: + for new_user in new_users: + status_table.add_row(new_user.username, new_user.email, Utils.get_as_string(new_user.sudo, default='False'), 'ABORTED', 'Adding user Aborted!') + context.print(status_table) + raise SystemExit(1) + + create_users_response = ClusterManagerUtils.create_users(new_users, context) + ClusterManagerUtils.print_create_user_status(create_users_response, context) diff --git a/source/idea/idea-cluster-manager/src/ideaclustermanager/cli/cli_main.py b/source/idea/idea-cluster-manager/src/ideaclustermanager/cli/cli_main.py new file mode 100644 index 00000000..6b33662a --- /dev/null +++ b/source/idea/idea-cluster-manager/src/ideaclustermanager/cli/cli_main.py @@ -0,0 +1,42 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +import ideaclustermanager +from ideadatamodel.constants import CLICK_SETTINGS + +from ideaclustermanager.cli.logs import logs +from ideaclustermanager.cli.accounts import accounts +from ideaclustermanager.cli.groups import groups +from ideaclustermanager.cli.ldap_commands import ldap_commands +from ideaclustermanager.cli.module import app_module_clean_up + +import sys +import click + + +@click.group(CLICK_SETTINGS) +@click.version_option(version=ideaclustermanager.__version__) +def main(): + """ + cluster manager + """ + pass + + +main.add_command(logs) +main.add_command(accounts) +main.add_command(groups) +main.add_command(ldap_commands) +main.add_command(app_module_clean_up) + +# used only for local testing +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/source/idea/idea-cluster-manager/src/ideaclustermanager/cli/cli_utils.py b/source/idea/idea-cluster-manager/src/ideaclustermanager/cli/cli_utils.py new file mode 100644 index 00000000..32cbf7a2 --- /dev/null +++ b/source/idea/idea-cluster-manager/src/ideaclustermanager/cli/cli_utils.py @@ -0,0 +1,205 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. +import csv +import os.path + +from ideasdk.context import SocaCliContext +from ideasdk.utils import Utils +from ideadatamodel import ( + User, + CreateUserRequest, + CreateUserResult, + GetUserRequest, + GetUserResult, + errorcodes, + exceptions +) +from ideaclustermanager.app.accounts import auth_constants +from ideaclustermanager.app.accounts.auth_utils import AuthUtils +from rich.table import Table + +from typing import List +import time +import re + +API_CONTEXT_PATH_CLUSTER_MANAGER = '/cluster-manager/api/v1' + + +class ClusterManagerUtils: + + @staticmethod + def get_soca_cli_context_cluster_manager() -> SocaCliContext: + return SocaCliContext(api_context_path=API_CONTEXT_PATH_CLUSTER_MANAGER) + + @staticmethod + def check_duplicate_users_in_csv(usernames: List[str]) -> List[str]: + seen = set() + duplicates = [] + + for username in usernames: + if username in seen: + duplicates.append(username) + else: + seen.add(username) + + return duplicates + + @staticmethod + def check_duplicate_users(usernames: List[str], context: SocaCliContext) -> List[str]: + + duplicates = [] + + for username in usernames: + result = None + try: + # calling get user to see if username already exists + result = context.unix_socket_client.invoke_alt( + namespace='Accounts.GetUser', + payload=GetUserRequest( + username=username + ), + result_as=GetUserResult + ) + except exceptions.SocaException as e: + # If error code == AUTH_USER_NOT_FOUND, we have a new user, continue flow + if e.error_code != errorcodes.AUTH_USER_NOT_FOUND: + raise e + except Exception as e: + raise e + + # username already exists + if result is not None: + duplicates.append(username) + + return duplicates + + @staticmethod + def create_users(users: List[User], context: SocaCliContext) -> [(User, str)]: + + create_users_response = [] + + if users is None or len(users) == 0: + return create_users_response + + for entry in users: + try: + context.unix_socket_client.invoke_alt( + namespace='Accounts.CreateUser', + payload=CreateUserRequest( + user=entry + ), + result_as=CreateUserResult + ) + response_entry = (entry, 'n/a') + except Exception as e: + response_entry = (entry, f'{e}') + + create_users_response.append(response_entry) + + return create_users_response + + @staticmethod + def print_create_user_status(users: [(User, str)], context: SocaCliContext): + + num_tries = 0 + while True: + all_done = True + table = ClusterManagerUtils.build_create_user_status_table() + for entry in users: + if Utils.are_equal(entry[1], 'n/a'): + response = context.shell.invoke(cmd=f'id {entry[0].username}', shell=True) + if response.returncode == 0: + table.add_row(entry[0].username, entry[0].email, Utils.get_as_string(entry[0].sudo, default='False'), 'SUCCESS', 'n/a', style='green') + else: + table.add_row(entry[0].username, entry[0].email, Utils.get_value_as_string(entry[0].sudo, default='False'), 'IN PROGRESS', 'n/a') + all_done = False + else: + table.add_row(entry[0].username, entry[0].email, Utils.get_value_as_string(entry[0].sudo, default='False'), 'FAILED', entry[1], style='red') + context.print(table) + + if all_done or num_tries == 10: + break + else: + num_tries += 1 + time.sleep(30) + + @staticmethod + def check_has_valid_csv_headers(headers, context: SocaCliContext): + if len(headers) < 2 or len(headers) > 3: + context.error('Incorrect number of columns provided; run with --generate-template to see an example') + raise SystemExit(1) + elif len(headers) == 2: + if 'Username' not in headers or 'Email' not in headers: + context.error('"Username" and "Email" fields must be provided in the header; "Is Admin?" is optional; run with --generate-template to see an example') + raise SystemExit(1) + elif len(headers) == 3: + if 'Username' not in headers or 'Email' not in headers or 'Is Admin?' not in headers: + context.error('"Username" and "Email" fields must be provided in the header; "Is Admin?" is optional; run with --generate-template to see an example') + raise SystemExit(1) + + @staticmethod + def check_if_csv_file(filepath: str, context: SocaCliContext): + if not Utils.is_file(filepath): + context.error(f'This path {filepath} is not a file; Please provide a csv file') + raise SystemExit(1) + elif not filepath.endswith('.csv'): + context.error(f'This file: {filepath} is not a csv file; Please provide a csv file') + raise SystemExit(1) + + @staticmethod + def generate_template(path: str) -> str: + + csv_file = os.path.join(path, 'users_file_template.csv') + header = ['Username', 'Email', 'Is Admin?'] + + with open(csv_file, 'w', encoding='UTF8') as f: + writer = csv.writer(f) + writer.writerow(header) + + return csv_file + + @staticmethod + def check_valid_username(username: str): + if Utils.is_empty(username): + raise exceptions.invalid_params("username is required") + elif not re.match(auth_constants.USERNAME_REGEX, username): + raise exceptions.invalid_params(f'user.username must match regex: {auth_constants.USERNAME_REGEX}') + elif username.strip().lower() in ('admin', 'administrator'): + raise exceptions.invalid_params(f'invalid username: {username}. Change username to prevent conflicts with local or directory system users.') + + @staticmethod + def check_valid_email(email: str): + try: + AuthUtils.sanitize_email(email) + except exceptions.SocaException as e: + raise e + + @staticmethod + def build_create_user_status_table() -> Table: + + status_table = Table() + status_table.add_column('Username', justify='left', no_wrap=False) + status_table.add_column('Email', justify='left', no_wrap=False) + status_table.add_column('Is Admin?', justify='left', no_wrap=False) + status_table.add_column('Status', justify='left', no_wrap=False) + status_table.add_column('Reason', justify='left', no_wrap=False) + + return status_table + + @staticmethod + def build_confirm_user_table() -> Table: + + confirm_table = Table() + confirm_table.add_column('Username', justify='left', no_wrap=False) + confirm_table.add_column('Email', justify='left', no_wrap=False) + confirm_table.add_column('Is Admin?', justify='left', no_wrap=False) + + return confirm_table diff --git a/source/idea/idea-cluster-manager/src/ideaclustermanager/cli/groups.py b/source/idea/idea-cluster-manager/src/ideaclustermanager/cli/groups.py new file mode 100644 index 00000000..60155d32 --- /dev/null +++ b/source/idea/idea-cluster-manager/src/ideaclustermanager/cli/groups.py @@ -0,0 +1,215 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +from ideadatamodel import ( + exceptions, + errorcodes, + constants, + GetGroupRequest, + GetGroupResult, + ListGroupsRequest, + ListGroupsResult, + ListUsersInGroupRequest, + ListUsersInGroupResult, + EnableGroupRequest, + EnableGroupResult, + DisableGroupRequest, + DisableGroupResult, + CreateGroupRequest, + CreateGroupResult, + DeleteGroupRequest, + DeleteGroupResult + +) +from ideaclustermanager.cli import build_cli_context +from ideasdk.utils import Utils +from ideaclustermanager.app.accounts.auth_utils import AuthUtils +from ideaclustermanager.cli.cli_utils import ClusterManagerUtils + +from rich.table import Table +import click +import csv + + +@click.group() +def groups(): + """ + group management options + """ + + +@groups.command(context_settings=constants.CLICK_SETTINGS) +@click.option('--groupname', required=True, help='Group Name') +def get_group(groupname: str): + """ + get group + """ + context = ClusterManagerUtils.get_soca_cli_context_cluster_manager() + try: + result = context.unix_socket_client.invoke_alt( + namespace='Accounts.GetGroup', + payload=GetGroupRequest( + group_name=groupname + ), + result_as=GetGroupResult + ) + except exceptions.SocaException as e: + context.error(e.message) + return + context.print(Utils.to_yaml(result.group)) + + +@groups.command(context_settings=constants.CLICK_SETTINGS) +def list_groups(): + """ + list existing groups + """ + + context = ClusterManagerUtils.get_soca_cli_context_cluster_manager() + result = context.unix_socket_client.invoke_alt( + namespace='Accounts.ListGroups', + payload=ListGroupsRequest(), + result_as=ListGroupsResult + ) + group_table = Table() + group_table.add_column("Group Name", justify="left", no_wrap=False) + group_table.add_column("Group Type", justify="left", no_wrap=False) + group_table.add_column("Directory Group Name", justify="left", no_wrap=False) + group_table.add_column("GID", justify="left", no_wrap=False) + group_table.add_column("Created On", justify="left", no_wrap=False) + group_table.add_column("Is Enabled?", justify="left", no_wrap=False) + for listing in result.listing: + group_table.add_row(listing.name, + Utils.get_as_string(listing.group_type), + Utils.get_as_string(listing.ds_name), + Utils.get_as_string(listing.gid), + listing.created_on.strftime("%m/%d/%Y, %H:%M:%S"), + Utils.get_as_string(listing.enabled), + ) + context.print(group_table) + +@groups.command(context_settings=constants.CLICK_SETTINGS) +@click.option('--groupname', required=True, help='Group Name to fetch users') +def list_users(groupname: str): + """ + list users in a group + """ + + context = ClusterManagerUtils.get_soca_cli_context_cluster_manager() + result = context.unix_socket_client.invoke_alt( + namespace='Accounts.ListUsersInGroup', + payload=ListUsersInGroupRequest( + group_names=[groupname] + ), + result_as=ListUsersInGroupResult + ) + group_table = Table() + group_table.add_column("User Name", justify="left", no_wrap=False) + group_table.add_column("User Email", justify="left", no_wrap=False) + group_table.add_column("User UID", justify="left", no_wrap=False) + group_table.add_column("User GID", justify="left", no_wrap=False) + group_table.add_column("Created On", justify="left", no_wrap=False) + group_table.add_column("Is Enabled?", justify="left", no_wrap=False) + for listing in result.listing: + group_table.add_row(listing.username, + Utils.get_as_string(listing.email), + Utils.get_as_string(listing.uid), + Utils.get_as_string(listing.gid), + listing.created_on.strftime("%m/%d/%Y, %H:%M:%S"), + Utils.get_as_string(listing.enabled), + ) + context.print(group_table) + + +@groups.command(context_settings=constants.CLICK_SETTINGS) +@click.option('--groupname', required=True, help='Group Name') +def enable_group(groupname: str): + """ + enable a group + """ + context = ClusterManagerUtils.get_soca_cli_context_cluster_manager() + try: + context.unix_socket_client.invoke_alt( + namespace='Accounts.EnableGroup', + payload=EnableGroupRequest( + group_name=groupname + ), + result_as=EnableGroupResult + ) + except exceptions.SocaException as e: + context.error(e.message) + + +@groups.command(context_settings=constants.CLICK_SETTINGS) +@click.option('--groupname', required=True, help='Group Name') +def disable_group(groupname: str): + """ + disable a group + """ + context = ClusterManagerUtils.get_soca_cli_context_cluster_manager() + try: + context.unix_socket_client.invoke_alt( + namespace='Accounts.DisableGroup', + payload=DisableGroupRequest( + group_name=groupname + ), + result_as=DisableGroupResult + ) + except exceptions.SocaException as e: + context.error(e.message) + +@groups.command(context_settings=constants.CLICK_SETTINGS) +@click.option('--groupname', required=True, help='Group Name') +@click.option('--title', required=True, help='Group Title') +@click.option('--description', required=True, help='Group Description') +@click.option('--ds-name', required=True, help='Directory Service Name') +@click.option('--group-type', required=True, help='Group Type') +@click.option('--gid', required=True, help='GID') +def create_group(**kwargs): + """ + create a new group + """ + request = { + 'group': { + 'name': Utils.get_value_as_string('groupname', kwargs), + 'title': Utils.get_value_as_string('title', kwargs), + 'description': Utils.get_value_as_string('description', kwargs), + 'ds_name': Utils.get_value_as_string('ds_name', kwargs), + 'gid': Utils.get_value_as_int('gid', kwargs), + 'group_type': Utils.get_value_as_string('group_type', kwargs), + 'enabled': True, + } + } + + context = build_cli_context() + result = context.unix_socket_client.invoke_alt( + namespace='Accounts.CreateGroup', + payload=CreateGroupRequest(**request), + result_as=CreateGroupResult + ) + context.print_json(Utils.to_json(result.group)) +@groups.command(context_settings=constants.CLICK_SETTINGS) +@click.option('--groupname', required=True, help='Groupname to DELETE') +def delete_group(groupname: str): + """ + delete a group + """ + context = ClusterManagerUtils.get_soca_cli_context_cluster_manager() + try: + context.unix_socket_client.invoke_alt( + namespace='Accounts.DeleteGroup', + payload=DeleteGroupRequest( + group_name=groupname + ), + result_as=DeleteGroupResult + ) + except exceptions.SocaException as e: + context.error(e.message) diff --git a/source/idea/idea-cluster-manager/src/ideaclustermanager/cli/ldap_commands.py b/source/idea/idea-cluster-manager/src/ideaclustermanager/cli/ldap_commands.py new file mode 100644 index 00000000..5587f925 --- /dev/null +++ b/source/idea/idea-cluster-manager/src/ideaclustermanager/cli/ldap_commands.py @@ -0,0 +1,182 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + + +from ideadatamodel import constants, SocaFilter + +from ideaclustermanager.app.accounts.ldapclient.ldap_client_factory import build_ldap_client +from ideaclustermanager.cli import build_cli_context + +from ideasdk.utils import Utils +from ideasdk.context import SocaCliContext + +from typing import Dict +import click + + +@click.group('ldap') +def ldap_commands(): + """ + ldap utilities + """ + + +@ldap_commands.command('search-users', context_settings=constants.CLICK_SETTINGS) +@click.option('-q', '--query', help="search query (substring of username)") +def search_users(query: str): + """ + search for users in directory service using ldap + """ + + context = build_cli_context(cluster_config=True, enable_aws_client_provider=True, enable_aws_util=True) + ldap_client = build_ldap_client(context) + + users, _ = ldap_client.search_users(username_filter=SocaFilter( + like=query + )) + context.print_json(users) + + +@ldap_commands.command('search-groups', context_settings=constants.CLICK_SETTINGS) +@click.option('-q', '--query', help="search query (substring of group name)") +def search_groups(query: str): + """ + search for groups in directory service using ldap + """ + + context = build_cli_context(cluster_config=True, enable_aws_client_provider=True, enable_aws_util=True) + ldap_client = build_ldap_client(context) + + groups, _ = ldap_client.search_groups(group_name_filter=SocaFilter( + like=query + )) + context.print_json(groups) + + +@ldap_commands.command('delete-user', context_settings=constants.CLICK_SETTINGS) +@click.option('-u', '--username', required=True, multiple=True, help="username of the user to be deleted. accepts multiple inputs eg. -u user1 -u user2") +def delete_user(username): + """ + delete user from directory service + """ + + context = build_cli_context(cluster_config=True, enable_aws_client_provider=True, enable_aws_util=True) + ldap_client = build_ldap_client(context) + + for user in username: + ldap_client.delete_user(username=user) + + +@ldap_commands.command('delete-group', context_settings=constants.CLICK_SETTINGS) +@click.option('-g', '--group', required=True, multiple=True, help="name of the group to be deleted. accepts multiple inputs eg. -g group1 -g group2") +def delete_group(group): + """ + delete group from directory service + """ + + context = build_cli_context(cluster_config=True, enable_aws_client_provider=True, enable_aws_util=True) + ldap_client = build_ldap_client(context) + + for group_name in group: + ldap_client.delete_group(group_name=group_name) + + +@ldap_commands.command('show-credentials', context_settings=constants.CLICK_SETTINGS) +@click.option('--username', is_flag=True, help="Print username") +@click.option('--password', is_flag=True, help="Print password") +def show_credentials(username: bool, password: bool): + """ + print service account credentials + """ + + context = build_cli_context(cluster_config=True, enable_aws_client_provider=True, enable_aws_util=True) + + if username: + print(context.config().get_secret('directoryservice.root_username_secret_arn'), end='') + if password: + print(context.config().get_secret('directoryservice.root_password_secret_arn'), end='') + + +def _send_task(context: SocaCliContext, task_name: str, payload: Dict, message_group_id: str, message_dedupe_id: str = None): + task_queue_url = context.config().get_string('cluster-manager.task_queue_url', required=True) + task_message = { + 'name': task_name, + 'payload': payload + } + + if Utils.is_empty(message_dedupe_id): + message_dedupe_id = Utils.uuid() + + context.info(f'send task: {task_name}, message group id: {message_group_id}, DedupeId: {message_dedupe_id}') + context.aws().sqs().send_message( + QueueUrl=task_queue_url, + MessageBody=Utils.to_json(task_message), + MessageDeduplicationId=Utils.uuid(), + MessageGroupId=message_group_id + ) + + +@ldap_commands.command('sync-user', context_settings=constants.CLICK_SETTINGS) +@click.option('-u', '--username', required=True, multiple=True, help="username of the user to be synced. accepts multiple inputs eg. -u user1 -u user2") +def sync_user(username): + """ + sync user account from db to directory service + """ + context = build_cli_context(cluster_config=True, enable_aws_client_provider=True, enable_aws_util=True) + for user in username: + _send_task( + context=context, + task_name='accounts.sync-user', + payload={ + 'username': user + }, + message_group_id=user + ) + + +@ldap_commands.command('sync-group', context_settings=constants.CLICK_SETTINGS) +@click.option('-g', '--group', required=True, multiple=True, help="name of the group to be synced. accepts multiple inputs eg. -g group1 -g group2") +def sync_group(group): + """ + sync group from db to directory service + """ + context = build_cli_context(cluster_config=True, enable_aws_client_provider=True, enable_aws_util=True) + for group_name in group: + _send_task( + context=context, + task_name='accounts.sync-group', + payload={ + 'group_name': group_name + }, + message_group_id=group_name + ) + + +@ldap_commands.command('create-service-account', context_settings=constants.CLICK_SETTINGS) +@click.option('-u', '--username', required=True, help='username of the service account') +@click.option('-p', '--password', required=True, help='password of the service account') +def create_service_account(username: str, password: str): + """ + create a service account with administrator access + + only supported for active directory + """ + context = build_cli_context(cluster_config=True, enable_aws_client_provider=True, enable_aws_util=True) + ds_provider = context.config().get_string('directoryservice.provider', required=True) + if ds_provider == constants.DIRECTORYSERVICE_OPENLDAP: + context.error('service account creation is not supported for OpenLDAP') + raise SystemExit(1) + + ldap_client = build_ldap_client(context) + with context.spinner(f'creating service account for username: {username} ...'): + account = ldap_client.create_service_account(username=username, password=password) + context.success('service account created successfully: ') + context.print_json(account) diff --git a/source/idea/idea-cluster-manager/src/ideaclustermanager/cli/logs.py b/source/idea/idea-cluster-manager/src/ideaclustermanager/cli/logs.py new file mode 100644 index 00000000..fc172f4b --- /dev/null +++ b/source/idea/idea-cluster-manager/src/ideaclustermanager/cli/logs.py @@ -0,0 +1,91 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +import ideaclustermanager +from ideadatamodel import constants, exceptions +from ideasdk.utils import EnvironmentUtils +from ideasdk.shell.log_tail import LogTail +from ideasdk.context import SocaCliContext + +from typing import Optional, List, Dict, Union +import os.path +import click + + +class LogFinder: + def __init__(self, kwargs: Dict, tokens: Optional[Union[str, List[str]]]): + self.tokens = tokens + self.kwargs = kwargs + self._config = None + self.files = [] + + def add_log_files(self, files: list = None): + if files is None: + return + for file in files: + if file in self.files: + continue + if not os.path.isfile(file): + click.echo(f'{file} not found. Ignoring') + continue + self.files.append(file) + + def find_applicable_logs(self): + self.add_log_files(files=[ + os.path.join(EnvironmentUtils.idea_app_deploy_dir(required=True), 'logs', 'application.log') + ]) + + def print_logs(self): + self.find_applicable_logs() + + if len(self.files) == 0: + raise exceptions.general_exception( + message='No log files files found.' + ) + + LogTail( + context=SocaCliContext(), + files=self.files, + search_tokens=self.tokens, + **self.kwargs + ).invoke() + + +@click.command(context_settings=constants.CLICK_SETTINGS, short_help='print logs for a specified resource(s)') +@click.argument('tokens', nargs=-1) +@click.option('--tail', '-t', default=100, help='Lines of recent log file to display. Default: 100') +@click.option('--follow', '-f', is_flag=True, help='Specify if the logs should be streamed') +@click.option('--command', '-c', is_flag=True, help='Print the command, instead of printing logs.') +@click.option('--and', '-a', is_flag=True, help='Specify if the TOKENS should be ANDed instead of ORed.') +def logs(tokens, **kwargs): + """ + idea logs + + \b + Examples: + + \b + # print last 100 lines of application logs + $ ideactl logs + + \b + # print last 100 lines of application logs + $ ideactl logs -t 100 + + \b + # follow application logs + $ ideactl logs -f + + """ + LogFinder( + tokens=tokens, + kwargs=kwargs + ).print_logs() diff --git a/source/idea/idea-cluster-manager/src/ideaclustermanager/cli/module.py b/source/idea/idea-cluster-manager/src/ideaclustermanager/cli/module.py new file mode 100644 index 00000000..9154e0e2 --- /dev/null +++ b/source/idea/idea-cluster-manager/src/ideaclustermanager/cli/module.py @@ -0,0 +1,22 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +from ideadatamodel import constants +import click + + +@click.command(context_settings=constants.CLICK_SETTINGS, short_help='Execute commands to clean up before deleting module') +@click.option('--delete-databases', is_flag=True) +def app_module_clean_up(delete_databases: bool): + """ + Utility hook to do any clean-up before the module is being deleted + """ + pass diff --git a/source/idea/idea-cluster-manager/src/ideaclustermanager_meta/__init__.py b/source/idea/idea-cluster-manager/src/ideaclustermanager_meta/__init__.py new file mode 100644 index 00000000..8ac7c101 --- /dev/null +++ b/source/idea/idea-cluster-manager/src/ideaclustermanager_meta/__init__.py @@ -0,0 +1,13 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +__name__ = 'idea-cluster-manager' +__version__ = '3.1.0' diff --git a/source/idea/idea-cluster-manager/src/setup.py b/source/idea/idea-cluster-manager/src/setup.py new file mode 100644 index 00000000..6bf18fba --- /dev/null +++ b/source/idea/idea-cluster-manager/src/setup.py @@ -0,0 +1,31 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +from setuptools import setup, find_packages +import ideaclustermanager_meta + +setup( + name=ideaclustermanager_meta.__name__, + version=ideaclustermanager_meta.__version__, + description='Cluster Manager', + url='https://awslabs.github.io/scale-out-computing-on-aws/', + author='Amazon', + license='Apache License, Version 2.0', + packages=find_packages(), + package_dir={ + 'ideaclustermanager': 'ideaclustermanager' + }, + entry_points=''' + [console_scripts] + ideactl=ideaclustermanager.cli.cli_main:main + ideaserver=ideaclustermanager.app.app_main:main + ''' +) diff --git a/source/idea/idea-cluster-manager/tests/conftest.py b/source/idea/idea-cluster-manager/tests/conftest.py new file mode 100644 index 00000000..be981567 --- /dev/null +++ b/source/idea/idea-cluster-manager/tests/conftest.py @@ -0,0 +1,178 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +from ideadatamodel import ( + SocaAnyPayload, +) + +from ideasdk.context import SocaContextOptions +from ideasdk.utils import Utils +from ideasdk.aws import AWSUtil, EC2InstanceTypesDB, AwsClientProvider +from ideasdk.auth import TokenService, TokenServiceOptions +from ideasdk.client.evdi_client import EvdiClient + +from ideaclustermanager import AppContext +from ideaclustermanager.app.accounts.accounts_service import AccountsService +from ideaclustermanager.app.projects.projects_service import ProjectsService +from ideaclustermanager.app.accounts.cognito_user_pool import CognitoUserPool, CognitoUserPoolOptions +from ideaclustermanager.app.tasks.task_manager import TaskManager + +from ideaclustermanagertests.mock_ldap_client import MockLdapClient + +from ideatestutils import MockInstanceTypes, MockConfig +from ideatestutils import IdeaTestProps +from ideatestutils.dynamodb.dynamodb_local import DynamoDBLocal + +import pytest +import boto3 +import time + +from _pytest.monkeypatch import MonkeyPatch + +# initialize monkey patch globally, so that it can be used inside session scoped context fixtures +# this allows session scoped monkey patches to be applicable across all unit tests +# monkeypatch.undo() is called at the end of context fixture +monkeypatch = MonkeyPatch() + + +@pytest.fixture(scope='session') +def ddb_local(): + ddb_local = DynamoDBLocal( + db_name='cluster-manager', + reset=True + ) + ddb_local.start() + + # wait for ddb local server to start ... + time.sleep(1) + + yield ddb_local + + ddb_local.stop() + + +@pytest.fixture(scope='session') +def context(ddb_local): + """ + fixture to initialize context with mock config and aws clients + goal is to ensure no network request are executed while executing unit tests + """ + + print('initializing cluster-manager context ...') + + def mock_function(*_, **__): + return {} + + mock_boto_session = SocaAnyPayload() + mock_boto_session.region_name = 'us-east-1' + mock_boto_session.client = mock_function + + mock_ec2_client = SocaAnyPayload() + mock_ec2_client.describe_security_groups = mock_function + + mock_s3_client = SocaAnyPayload() + mock_s3_client.upload_file = mock_function + mock_s3_client.get_bucket_acl = mock_function + + mock_cognito_idp = SocaAnyPayload() + mock_cognito_idp.admin_create_user = mock_function + mock_cognito_idp.admin_add_user_to_group = mock_function + mock_cognito_idp.admin_get_user = mock_function + mock_cognito_idp.admin_set_user_password = mock_function + mock_cognito_idp.admin_update_user_attributes = mock_function + mock_cognito_idp.admin_enable_user = mock_function + mock_cognito_idp.admin_disable_user = mock_function + + monkeypatch.setattr(EC2InstanceTypesDB, '_instance_type_names_from_botocore', MockInstanceTypes.get_instance_type_names) + monkeypatch.setattr(EvdiClient, 'publish_user_disabled_event', mock_function) + + def create_mock_boto_session(**_): + return boto3.Session( + aws_access_key_id='mock_access_key', + aws_secret_access_key='mock_secret_access_key', + region_name='us-east-1' + ) + + monkeypatch.setattr(Utils, 'create_boto_session', create_mock_boto_session) + monkeypatch.setattr(AWSUtil, 'get_ec2_instance_type', MockInstanceTypes.get_instance_type) + monkeypatch.setattr(AwsClientProvider, 's3', lambda *_: mock_s3_client) + monkeypatch.setattr(AwsClientProvider, 'ec2', lambda *_: mock_ec2_client) + monkeypatch.setattr(AwsClientProvider, 'cognito_idp', lambda *_: mock_cognito_idp) + + mock_config = MockConfig() + + test_props = IdeaTestProps() + + monkeypatch.setenv('IDEA_DEV_MODE', 'true') + + test_dir = test_props.get_test_dir('cluster-manager-tests') + monkeypatch.setenv('IDEA_APP_DEPLOY_DIR', test_dir) + + context = AppContext( + options=SocaContextOptions( + cluster_name='idea-mock', + module_id='cluster-manager', + module_name='cluster-manager', + module_set='default', + enable_aws_client_provider=True, + enable_aws_util=True, + use_vpc_endpoints=True, + config=mock_config.get_config() + ) + ) + context.task_manager = TaskManager( + context=context, + tasks=[] + ) + monkeypatch.setattr(context.task_manager, 'send', lambda *args, **kwargs: print(f'[TaskManager.send()] args: {args}, kwargs: {kwargs}')) + + context.ldap_client = MockLdapClient(context=context) + user_pool = CognitoUserPool( + context=context, + options=CognitoUserPoolOptions( + user_pool_id=context.config().get_string('identity-provider.cognito.user_pool_id', required=True), + admin_group_name=context.config().get_string('identity-provider.cognito.administrators_group_name', required=True), + client_id='mock-client-id', + client_secret='mock-client-secret' + ) + ) + + context.accounts = AccountsService( + context=context, + ldap_client=context.ldap_client, + user_pool=user_pool, + evdi_client=EvdiClient(context=context), + task_manager=context.task_manager, + token_service=None + ) + context.accounts.create_defaults() + + context.token_service = TokenService( + context=context, + options=TokenServiceOptions( + cognito_user_pool_provider_url=context.config().get_string('identity-provider.cognito.provider_url', required=True), + cognito_user_pool_domain_url=context.config().get_string('identity-provider.cognito.domain_url', required=True), + client_id='mock-client-id', + client_secret='mock-client-secret', + client_credentials_scope=[] + ) + ) + + context.projects = ProjectsService( + context=context, + accounts_service=context.accounts, + task_manager=context.task_manager + ) + + yield context + + print('cluster manager context clean-up ...') + monkeypatch.undo() diff --git a/source/idea/idea-cluster-manager/tests/ideaclustermanagertests/__init__.py b/source/idea/idea-cluster-manager/tests/ideaclustermanagertests/__init__.py new file mode 100644 index 00000000..6d8d18a3 --- /dev/null +++ b/source/idea/idea-cluster-manager/tests/ideaclustermanagertests/__init__.py @@ -0,0 +1,10 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. diff --git a/source/idea/idea-cluster-manager/tests/ideaclustermanagertests/mock_ldap_client.py b/source/idea/idea-cluster-manager/tests/ideaclustermanagertests/mock_ldap_client.py new file mode 100644 index 00000000..73b19e28 --- /dev/null +++ b/source/idea/idea-cluster-manager/tests/ideaclustermanagertests/mock_ldap_client.py @@ -0,0 +1,63 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +from ideaclustermanager import AppContext +from ideaclustermanager.app.accounts.ldapclient import OpenLDAPClient, LdapClientOptions +from ideasdk.utils import Utils + +import ldap # noqa + + +class MockLdapClient(OpenLDAPClient): + + def __init__(self, context: AppContext): + super().__init__( + context=context, + options=LdapClientOptions( + domain_name='idea.local', + uri='ldap://localhost', + root_username='mock', + root_password='mock' + ) + ) + + def add_s(self, dn, modlist): + trace_message = f'ldapadd -x -D "{self.ldap_root_bind}" -H {self.ldap_uri} "{dn}"' + attributes = [] + for mod in modlist: + key = mod[0] + values = [] + for value in mod[1]: + values.append(Utils.from_bytes(value)) + attributes.append(f'{key}={",".join(values)}') + self.logger.info(f'> {trace_message}, attributes: ({" ".join(attributes)})') + + def modify_s(self, dn, modlist): + trace_message = f'ldapmodify -x -D "{self.ldap_root_bind}" -H {self.ldap_uri} "{dn}"' + attributes = [] + for mod in modlist: + key = mod[1] + values = [] + for value in mod[2]: + values.append(Utils.from_bytes(value)) + attributes.append(f'{key}={",".join(values)}') + self.logger.info(f'> {trace_message}, attributes: ({" ".join(attributes)})') + + def delete_s(self, dn): + trace_message = f'ldapdelete -x -D "{self.ldap_root_bind}" -H {self.ldap_uri} "{dn}"' + self.logger.info(f'> {trace_message}') + + def search_s(self, base, scope=ldap.SCOPE_SUBTREE, filterstr=None, attrlist=None, attrsonly=0, trace=True): + trace_message = f'ldapsearch -x -b "{base}" -D "{self.ldap_root_bind}" -H {self.ldap_uri} "{filterstr}"' + if attrlist is not None: + trace_message = f'{trace_message} {" ".join(attrlist)}' + self.logger.info(f'> {trace_message}') + return () diff --git a/source/idea/idea-cluster-manager/tests/ideaclustermanagertests/test_accounts.py b/source/idea/idea-cluster-manager/tests/ideaclustermanagertests/test_accounts.py new file mode 100644 index 00000000..c00ff210 --- /dev/null +++ b/source/idea/idea-cluster-manager/tests/ideaclustermanagertests/test_accounts.py @@ -0,0 +1,224 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +""" +Test Cases for AccountsService +""" + +from ideaclustermanager import AppContext +from ideadatamodel import ( + exceptions, + errorcodes, + User, + ListUsersRequest +) + +import pytest +from typing import Optional + + +class AccountsTestContext: + crud_user: Optional[User] + + +def test_accounts_create_user_missing_username_should_fail(context: AppContext): + """ + create user with missing username + """ + with pytest.raises(exceptions.SocaException) as exc_info: + context.accounts.create_user(user=User( + username='' + )) + assert exc_info.value.error_code == errorcodes.INVALID_PARAMS + assert 'username is required' in exc_info.value.message + + +def test_accounts_create_user_invalid_username_should_fail(context: AppContext): + """ + create user with invalid username + """ + with pytest.raises(exceptions.SocaException) as exc_info: + context.accounts.create_user(user=User( + username='Invalid Username' + )) + assert exc_info.value.error_code == errorcodes.INVALID_PARAMS + assert 'user.username must match regex' in exc_info.value.message + + +def test_accounts_create_user_system_account_username_should_fail(context: AppContext): + """ + create user with username as system accounts + """ + with pytest.raises(exceptions.SocaException) as exc_info: + context.accounts.create_user(user=User( + username='root' + )) + assert exc_info.value.error_code == errorcodes.INVALID_PARAMS + assert 'invalid username:' in exc_info.value.message + + +def test_accounts_create_user_missing_email_should_fail(context: AppContext): + """ + create user with missing email + """ + with pytest.raises(exceptions.SocaException) as exc_info: + context.accounts.create_user(user=User( + username='mockuser1' + )) + assert exc_info.value.error_code == errorcodes.INVALID_PARAMS + assert 'email is required' in exc_info.value.message + + +def test_accounts_create_user_invalid_email_should_fail(context: AppContext): + """ + create user with invalid email + """ + with pytest.raises(exceptions.SocaException) as exc_info: + context.accounts.create_user(user=User( + username='mockuser1', + email='invalid-email' + )) + assert exc_info.value.error_code == errorcodes.INVALID_PARAMS + assert 'invalid email:' in exc_info.value.message + + +def test_accounts_create_user_with_verified_email_missing_password_should_fail(context: AppContext): + """ + create valid account with email verified and no password + """ + with pytest.raises(exceptions.SocaException) as exc_info: + context.accounts.create_user(user=User( + username='mockuser1', + email='mockuser1@example.com' + ), email_verified=True) + assert exc_info.value.error_code == errorcodes.INVALID_PARAMS + assert 'user.password is required' in exc_info.value.message + + +def test_accounts_crud_create_user(context: AppContext): + """ + create user + """ + created_user = context.accounts.create_user(user=User( + username='accounts_user1', + email='accounts_user1@example.com', + password='MockPassword_123' + ), email_verified=True) + + expected_group_name = f'{created_user.username}-user-group' + + assert created_user.username is not None + assert created_user.email is not None + + user = context.accounts.get_user(username=created_user.username) + + assert user is not None + assert user.username is not None + assert user.email is not None + assert user.group_name is not None + assert user.group_name == expected_group_name + assert user.additional_groups is not None + assert len(user.additional_groups) == 2 # default project group and personal user group + assert 'default-project-group' in user.additional_groups + assert expected_group_name in user.additional_groups + assert user.enabled is not None + assert user.enabled is True + assert user.uid is not None + assert user.gid is not None + assert user.sudo is not None or user.sudo is False + assert user.home_dir is not None + assert user.login_shell is not None + assert user.password is None + assert user.created_on is not None + assert user.updated_on is not None + + AccountsTestContext.crud_user = user + + +def test_accounts_crud_get_user(context: AppContext): + """ + get user + """ + assert AccountsTestContext.crud_user is not None + crud_user = AccountsTestContext.crud_user + + user = context.accounts.get_user(username=crud_user.username) + + assert user is not None + assert user.username == crud_user.username + assert user.uid == crud_user.uid + assert user.gid == crud_user.gid + + +def test_accounts_crud_modify_user(context: AppContext): + """ + modify user + """ + assert AccountsTestContext.crud_user is not None + crud_user = AccountsTestContext.crud_user + + modify_user = User( + username=crud_user.username, + email='accounts_user1_modified@example.com', + uid=6000, + gid=6000, + login_shell='/bin/csh' + ) + context.accounts.modify_user(user=modify_user, email_verified=True) + + user = context.accounts.get_user(username=crud_user.username) + assert user.username == modify_user.username + assert user.email == modify_user.email + assert user.uid == modify_user.uid + assert user.gid == modify_user.gid + assert user.login_shell == modify_user.login_shell + + +def test_accounts_crud_disable_user(context: AppContext): + """ + disable user + """ + assert AccountsTestContext.crud_user is not None + crud_user = AccountsTestContext.crud_user + + context.accounts.disable_user(crud_user.username) + user = context.accounts.get_user(username=crud_user.username) + assert user.enabled is False + + +def test_accounts_crud_enable_user(context: AppContext): + """ + enable user + """ + assert AccountsTestContext.crud_user is not None + crud_user = AccountsTestContext.crud_user + + context.accounts.enable_user(crud_user.username) + user = context.accounts.get_user(username=crud_user.username) + assert user.enabled is True + + +def test_accounts_crud_list_users(context: AppContext): + """ + list users + """ + assert AccountsTestContext.crud_user is not None + crud_user = AccountsTestContext.crud_user + + result = context.accounts.list_users(ListUsersRequest()) + assert result.listing is not None + + found = None + for user in result.listing: + if user.username == crud_user.username: + found = user + break + assert found is not None diff --git a/source/idea/idea-cluster-manager/tests/ideaclustermanagertests/test_projects.py b/source/idea/idea-cluster-manager/tests/ideaclustermanagertests/test_projects.py new file mode 100644 index 00000000..10fab54c --- /dev/null +++ b/source/idea/idea-cluster-manager/tests/ideaclustermanagertests/test_projects.py @@ -0,0 +1,476 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. + +""" +Test Cases for ProjectsService +""" + +from ideaclustermanager import AppContext +from ideaclustermanager.app.accounts.account_tasks import GroupMembershipUpdatedTask +from ideaclustermanager.app.projects.project_tasks import ( + ProjectEnabledTask, + ProjectDisabledTask +) + +from ideadatamodel import ( + exceptions, + errorcodes, + constants, + GetProjectRequest, + CreateProjectRequest, + Project, + SocaAnyPayload, + SocaKeyValue, + EnableProjectRequest, + DisableProjectRequest, + ListProjectsRequest, + Group, + User, + GetUserProjectsRequest +) + +import pytest +from typing import List, Optional + + +class ProjectsTestContext: + crud_project: Optional[Project] + + +def enable_project(context: AppContext, project: Project): + context.projects.enable_project(EnableProjectRequest( + project_id=project.project_id + )) + task = ProjectEnabledTask(context=context) + task.invoke({ + 'project_id': project.project_id + }) + + +def disable_project(context: AppContext, project: Project): + context.projects.enable_project(EnableProjectRequest( + project_id=project.project_id + )) + task = ProjectDisabledTask(context=context) + task.invoke({ + 'project_id': project.project_id + }) + + +def is_memberof(context: AppContext, user: User, project: Project) -> bool: + result = context.projects.get_user_projects(GetUserProjectsRequest( + username=user.username + )) + + if result.projects is None: + return False + + for user_project in result.projects: + if user_project.project_id == project.project_id: + return True + + return False + + +def add_member(context: AppContext, user: User, project: Project): + username = user.username + group_name = project.ldap_groups[0] + context.accounts.add_users_to_group( + usernames=[username], + group_name=group_name + ) + task = GroupMembershipUpdatedTask( + context=context + ) + task.invoke(payload={ + 'group_name': group_name, + 'username': username, + 'operation': 'add' + }) + + +def remove_member(context: AppContext, user: User, project: Project): + username = user.username + group_name = project.ldap_groups[0] + context.accounts.remove_users_from_group( + usernames=[username], + group_name=group_name + ) + task = GroupMembershipUpdatedTask( + context=context + ) + task.invoke(payload={ + 'group_name': group_name, + 'username': username, + 'operation': 'remove' + }) + + +@pytest.fixture(scope='module') +def membership(context: AppContext): + def create_project(project_name: str, project_title: str) -> Project: + # create project group + group = context.accounts.create_group(Group( + title=f'{project_title} Project Group', + name=f'{project_name}-project-group', + group_type=constants.GROUP_TYPE_PROJECT + )) + assert group is not None + + # create project + result = context.projects.create_project(CreateProjectRequest( + project=Project( + name=project_name, + title=project_title, + ldap_groups=[group.name] + ) + )) + assert result is not None + assert result.project is not None + + # enable project + enable_project(context=context, project=result.project) + + # get enabled project + result = context.projects.get_project(GetProjectRequest( + project_id=result.project.project_id + )) + + assert result.project is not None + assert result.project.enabled is True + + return result.project + + def create_user(username: str) -> User: + return context.accounts.create_user(user=User( + username=username, + password='MockPassword_123', + email=f'{username}@example.com' + ), email_verified=True) + + project_a = create_project('project-a', 'Project A') + project_b = create_project('project-b', 'Project B') + user_1 = create_user(username='project_user_1') + user_2 = create_user(username='project_user_2') + user_3 = create_user(username='project_user_3') + + return SocaAnyPayload( + project_a=project_a, + project_b=project_b, + user_1=user_1, + user_2=user_2, + user_3=user_3 + ) + + +def test_projects_create_defaults(context): + """ + defaults creation for projects + """ + + context.projects.create_defaults() + + result = context.projects.get_project(GetProjectRequest( + project_name='default' + )) + assert result is not None + assert result.project is not None + assert result.project.name == 'default' + + +def test_projects_crud_create_project(context): + """ + create project + """ + result = context.projects.create_project(CreateProjectRequest( + project=Project( + name='sampleproject', + title='Sample Project', + description='Sample Project Description', + ldap_groups=['default-project-group'], + tags=[ + SocaKeyValue(key='k1', value='v1'), + SocaKeyValue(key='k2', value='v2') + ] + ) + )) + assert result is not None + assert result.project is not None + assert result.project.name == 'sampleproject' + ProjectsTestContext.crud_project = result.project + + result = context.projects.get_project(GetProjectRequest( + project_name=ProjectsTestContext.crud_project.name + )) + assert result is not None + assert result.project is not None + assert result.project.enabled is False + assert result.project.ldap_groups is not None + assert result.project.ldap_groups[0] == ProjectsTestContext.crud_project.ldap_groups[0] + assert result.project.description is not None + assert result.project.description == 'Sample Project Description' + assert result.project.tags is not None + assert len(result.project.tags) == 2 + assert result.project.tags[0].key == 'k1' + assert result.project.tags[0].value == 'v1' + assert result.project.tags[1].key == 'k2' + assert result.project.tags[1].value == 'v2' + assert result.project.created_on is not None + assert result.project.updated_on is not None + + +def test_projects_crud_get_project_by_name(context): + """ + get project by name + """ + assert ProjectsTestContext.crud_project is not None + + result = context.projects.get_project(GetProjectRequest( + project_name=ProjectsTestContext.crud_project.name + )) + assert result is not None + assert result.project is not None + assert result.project.name == ProjectsTestContext.crud_project.name + assert result.project.project_id == ProjectsTestContext.crud_project.project_id + + +def test_projects_crud_get_project_by_id(context): + """ + get project by id + """ + assert ProjectsTestContext.crud_project is not None + + result = context.projects.get_project(GetProjectRequest( + project_id=ProjectsTestContext.crud_project.project_id + )) + assert result is not None + assert result.project is not None + assert result.project.name == ProjectsTestContext.crud_project.name + assert result.project.project_id == ProjectsTestContext.crud_project.project_id + + +def test_projects_crud_get_project_invalid_should_fail(context): + """ + get project - invalid project id or name + """ + + # by project id + with pytest.raises(exceptions.SocaException) as exc_info: + context.projects.get_project(GetProjectRequest( + project_id='unknown-project-id' + )) + assert exc_info.value.error_code == errorcodes.PROJECT_NOT_FOUND + + # by project name + with pytest.raises(exceptions.SocaException) as exc_info: + context.projects.get_project(GetProjectRequest( + project_name='unknown-project-name' + )) + assert exc_info.value.error_code == errorcodes.PROJECT_NOT_FOUND + + +def test_projects_crud_enable_project(context): + """ + enable project + """ + assert ProjectsTestContext.crud_project is not None + + context.projects.enable_project(EnableProjectRequest( + project_id=ProjectsTestContext.crud_project.project_id + )) + + result = context.projects.get_project(GetProjectRequest( + project_id=ProjectsTestContext.crud_project.project_id + )) + assert result is not None + assert result.project is not None + assert result.project.enabled is True + + +def test_projects_crud_disable_project(context): + """ + disable project + """ + assert ProjectsTestContext.crud_project is not None + + context.projects.disable_project(DisableProjectRequest( + project_id=ProjectsTestContext.crud_project.project_id + )) + + result = context.projects.get_project(GetProjectRequest( + project_id=ProjectsTestContext.crud_project.project_id + )) + assert result is not None + assert result.project is not None + assert result.project.enabled is False + + +def test_projects_crud_list_projects(context): + """ + list projects + """ + assert ProjectsTestContext.crud_project is not None + + result = context.projects.list_projects(ListProjectsRequest()) + + assert result is not None + assert result.listing is not None + assert len(result.listing) > 0 + + found = None + for project in result.listing: + if project.project_id == ProjectsTestContext.crud_project.project_id: + found = project + break + + assert found is not None + + +def test_projects_membership_setup(context, membership): + """ + check if membership setup data is valid and tests are starting with a clean slate. + """ + + assert membership.project_a is not None + assert membership.project_b is not None + assert membership.user_1 is not None + assert membership.user_2 is not None + assert membership.user_3 is not None + + assert is_memberof(context, membership.user_1, membership.project_a) is False + assert is_memberof(context, membership.user_2, membership.project_a) is False + assert is_memberof(context, membership.user_3, membership.project_a) is False + + assert is_memberof(context, membership.user_1, membership.project_b) is False + assert is_memberof(context, membership.user_2, membership.project_b) is False + assert is_memberof(context, membership.user_3, membership.project_b) is False + + +def test_projects_membership_member_added(context, membership): + """ + add user to a group and check if user is member of projects + """ + + add_member(context, membership.user_1, membership.project_a) + assert is_memberof(context, membership.user_1, membership.project_a) is True + + add_member(context, membership.user_2, membership.project_b) + assert is_memberof(context, membership.user_2, membership.project_b) is True + + +def test_projects_membership_member_removed(context, membership): + """ + add user to a group and check if user is member of projects + """ + + remove_member(context, membership.user_1, membership.project_a) + assert is_memberof(context, membership.user_1, membership.project_a) is False + + remove_member(context, membership.user_2, membership.project_b) + assert is_memberof(context, membership.user_2, membership.project_b) is False + + +def test_projects_membership_project_disabled(context, membership): + """ + disable project and add member. user should not be member of project. + """ + + disable_project(context, membership.project_a) + + add_member(context, membership.user_1, membership.project_a) + assert is_memberof(context, membership.user_1, membership.project_a) is False + + +def test_projects_membership_project_enabled(context, membership): + """ + add member, enable project and check if existing group members are part of project + """ + + add_member(context, membership.user_1, membership.project_a) + add_member(context, membership.user_2, membership.project_a) + add_member(context, membership.user_3, membership.project_a) + + enable_project(context, membership.project_a) + + assert is_memberof(context, membership.user_1, membership.project_a) is True + assert is_memberof(context, membership.user_2, membership.project_a) is True + assert is_memberof(context, membership.user_3, membership.project_a) is True + + +def test_projects_membership_multiple_projects(context, membership): + """ + add user to multiple projects. check for membership for all + """ + + # pre-requisites + remove_member(context, membership.user_1, membership.project_a) + remove_member(context, membership.user_1, membership.project_b) + enable_project(context, membership.project_a) + + add_member(context, membership.user_1, membership.project_a) + add_member(context, membership.user_1, membership.project_b) + + assert is_memberof(context, membership.user_1, membership.project_a) is True + assert is_memberof(context, membership.user_1, membership.project_b) is True + + +def test_projects_get_user_projects(context, membership): + """ + get user projects + """ + + # setup + def clear_memberships(user: User): + remove_member(context, user, membership.project_a) + remove_member(context, user, membership.project_b) + + clear_memberships(membership.user_1) + clear_memberships(membership.user_2) + clear_memberships(membership.user_3) + + enable_project(context, membership.project_a) + enable_project(context, membership.project_b) + + add_member(context, membership.user_1, membership.project_a) + + add_member(context, membership.user_2, membership.project_b) + + add_member(context, membership.user_3, membership.project_a) + add_member(context, membership.user_3, membership.project_b) + + # verify + + def check_user_projects(username: str, project_ids: List[str]): + result = context.projects.get_user_projects(GetUserProjectsRequest( + username=username + )) + assert result.projects is not None + count = 0 + for project in result.projects: + if project.project_id in project_ids: + count += 1 + + assert len(project_ids) == count + + check_user_projects(username=membership.user_1.username, project_ids=[ + membership.project_a.project_id + ]) + assert is_memberof(context, membership.user_1, membership.project_b) is False + + check_user_projects(username=membership.user_2.username, project_ids=[ + membership.project_b.project_id + ]) + assert is_memberof(context, membership.user_2, membership.project_a) is False + + check_user_projects(username=membership.user_3.username, project_ids=[ + membership.project_a.project_id, + membership.project_b.project_id + ]) diff --git a/source/idea/idea-cluster-manager/webapp/.env b/source/idea/idea-cluster-manager/webapp/.env new file mode 100644 index 00000000..c7979e90 --- /dev/null +++ b/source/idea/idea-cluster-manager/webapp/.env @@ -0,0 +1,4 @@ +REACT_APP_IDEA_HTTP_ENDPOINT="http://localhost:8080" +REACT_APP_IDEA_ALB_ENDPOINT="http://localhost:8080" +REACT_APP_IDEA_HTTP_API_SUFFIX="/api/v1" +REACT_APP_IDEA_RELEASE_VERSION="3.1.0" diff --git a/source/idea/idea-cluster-manager/webapp/.gitignore b/source/idea/idea-cluster-manager/webapp/.gitignore new file mode 100644 index 00000000..4d29575d --- /dev/null +++ b/source/idea/idea-cluster-manager/webapp/.gitignore @@ -0,0 +1,23 @@ +# See https://help.github.com/articles/ignoring-files/ for more about ignoring files. + +# dependencies +/node_modules +/.pnp +.pnp.js + +# testing +/coverage + +# production +/build + +# misc +.DS_Store +.env.local +.env.development.local +.env.test.local +.env.production.local + +npm-debug.log* +yarn-debug.log* +yarn-error.log* diff --git a/source/idea/idea-cluster-manager/webapp/README.md b/source/idea/idea-cluster-manager/webapp/README.md new file mode 100644 index 00000000..b58e0af8 --- /dev/null +++ b/source/idea/idea-cluster-manager/webapp/README.md @@ -0,0 +1,46 @@ +# Getting Started with Create React App + +This project was bootstrapped with [Create React App](https://github.com/facebook/create-react-app). + +## Available Scripts + +In the project directory, you can run: + +### `yarn start` + +Runs the app in the development mode.\ +Open [http://localhost:3000](http://localhost:3000) to view it in the browser. + +The page will reload if you make edits.\ +You will also see any lint errors in the console. + +### `yarn test` + +Launches the test runner in the interactive watch mode.\ +See the section about [running tests](https://facebook.github.io/create-react-app/docs/running-tests) for more information. + +### `yarn build` + +Builds the app for production to the `build` folder.\ +It correctly bundles React in production mode and optimizes the build for the best performance. + +The build is minified and the filenames include the hashes.\ +Your app is ready to be deployed! + +See the section about [deployment](https://facebook.github.io/create-react-app/docs/deployment) for more information. + +### `yarn eject` + +**Note: this is a one-way operation. Once you `eject`, you can’t go back!** + +If you aren’t satisfied with the build tool and configuration choices, you can `eject` at any time. This command will remove the single build dependency from your project. + +Instead, it will copy all the configuration files and the transitive dependencies (webpack, Babel, ESLint, etc) right into your project so you have full control over them. All of the commands except `eject` will still work, but they will point to the copied scripts so you can tweak them. At this point you’re on your own. + +You don’t have to ever use `eject`. The curated feature set is suitable for small and middle deployments, and you shouldn’t feel obligated to use this feature. However we understand that this tool wouldn’t be useful if you couldn’t customize it when you are ready for it. + +## Learn More + +You can learn more in the [Create React App documentation](https://facebook.github.io/create-react-app/docs/getting-started). + +To learn React, check out the [React documentation](https://reactjs.org/). diff --git a/source/idea/idea-cluster-manager/webapp/package.json b/source/idea/idea-cluster-manager/webapp/package.json new file mode 100644 index 00000000..0c0f4573 --- /dev/null +++ b/source/idea/idea-cluster-manager/webapp/package.json @@ -0,0 +1,104 @@ +{ + "name": "web-portal", + "version": "3.1.0", + "private": true, + "dependencies": { + "@cloudscape-design/components": "^3.0.82", + "@cloudscape-design/design-tokens": "^3.0.4", + "@cloudscape-design/global-styles": "^1.0.1", + "@fortawesome/fontawesome-svg-core": "^6.2.0", + "@fortawesome/free-brands-svg-icons": "^6.2.0", + "@fortawesome/free-solid-svg-icons": "^6.2.0", + "@fortawesome/react-fontawesome": "^0.2.0", + "@types/dot-object": "^2.1.2", + "@types/react": "^18.0.0", + "@types/react-beautiful-dnd": "^13.1.2", + "@types/react-dom": "^18.0.0", + "@types/uuid": "^8.3.4", + "@uppy/core": "^3.0.3", + "@uppy/dashboard": "^3.1.0", + "@uppy/xhr-upload": "^3.0.3", + "ace-builds": "^1.12.3", + "browser-image-compression": "^2.0.0", + "chonky": "^2.3.2", + "chonky-icon-fontawesome": "^2.3.2", + "dot-object": "^2.1.4", + "idb": "^7.0.1", + "moment": "^2.29.4", + "moment-timezone": "^0.5.38", + "nunjucks": "^3.2.3", + "react": "^18.2.0", + "react-beautiful-dnd": "^13.1.1", + "react-dom": "^18.2.0", + "react-json-view": "^1.21.3", + "react-markdown": "^8.0.3", + "react-range": "^1.8.14", + "react-router-dom": "^6.4.2", + "react-toastify": "^9.0.8", + "rehype-raw": "^6.1.1", + "remark-gfm": "^3.0.1", + "uuid": "^9.0.0", + "xterm-addon-fit": "^0.6.0", + "xterm-for-react": "^1.0.4" + }, + "devDependencies": { + "@testing-library/jest-dom": "^5.14.1", + "@testing-library/react": "^13.0.0", + "@testing-library/user-event": "^13.2.1", + "@types/jest": "^27.0.1", + "@types/node": "^16.7.13", + "@types/sass": "^1.43.1", + "react-scripts": "5.0.1", + "sass": "^1.44.0", + "source-map-explorer": "^2.5.3", + "stream-http": "^3.2.0", + "typescript": "^4.4.2", + "workbox-background-sync": "^6.4.2", + "workbox-broadcast-update": "^6.4.2", + "workbox-cacheable-response": "^6.4.2", + "workbox-core": "^6.4.2", + "workbox-expiration": "^6.4.2", + "workbox-google-analytics": "^6.4.2", + "workbox-navigation-preload": "^6.4.2", + "workbox-precaching": "^6.4.2", + "workbox-range-requests": "^6.4.2", + "workbox-routing": "^6.4.2", + "workbox-strategies": "^6.4.2", + "workbox-streams": "^6.4.2" + }, + "scripts": { + "serve": "react-scripts start", + "start": "react-scripts start", + "build": "react-scripts build", + "analyze": "source-map-explorer 'build/static/js/*.js'", + "test": "react-scripts test", + "eject": "react-scripts eject", + "sass": "sass src/Sass:src/Css --watch --no-source-map" + }, + "resolutions": { + "minimatch": ">=3.0.5", + "nth-check": ">=2.0.1", + "d3-color": ">=3.1.0", + "minimist": ">=1.2.6", + "moment": ">=2.29.4" + }, + "eslintConfig": { + "extends": [ + "react-app", + "react-app/jest" + ] + }, + "browserslist": { + "production": [ + ">0.2%", + "not dead", + "not op_mini all" + ], + "development": [ + "last 1 chrome version", + "last 1 firefox version", + "last 1 safari version" + ] + }, + "homepage": "." +} diff --git a/source/idea/idea-cluster-manager/webapp/public/android-chrome-192x192.png b/source/idea/idea-cluster-manager/webapp/public/android-chrome-192x192.png new file mode 100644 index 0000000000000000000000000000000000000000..e2109340f2a092321ea33196363330d531054d5c GIT binary patch literal 14357 zcmZ{LWmp_dug9i5i!QB@R?jB%q4J^U^@ZJ0C+~>JH z(lb3>Q#C!+HQjI3M60XHp`(zX0001V1$k-B_ZPF{inM1?qr18|i*MajKzr%?f?u&^hRUgye|0~_66lf9_Xd8)^QKl{a%@X@+P2bkI zXSYq2NiH){|JM{PNAtwO`q!5pZCDN~chen7@gnm3^?otDvyWZT&Wxi7+iLLaz%A;_ zlEQ;SxbpOlBVv|l_IQAia09zIzPIV>SKvi>R|H^tm@v|CPnvvE) zLR_ufvw)zt8b>*sAC+kqpwcGh{BZ_Z9Oyh-e)Bqxp%`0HKYRLuf_b_~0$eA!Wf@AT?wCj=Wu%9(WIxe|_P>V4omfoJx;MCCL9+8jk1_Ns*&`Q!FMxCo;}aRLL( z0fkvm3igl$_H05WD^Am9JDLI zS7C6U|60&*nekLL3ubBVzxN(yhtkNdrE4o!kLwGRX2=MYU%ar-sd8{ls5rH}F(7|^ zg*8r!YIxQbP*r>nZQv%@Qp&~KVd@4YCiTRxpEcH1`ETtDIyO3M zaez27Q9L0kcP8&Ao(BzxiNZCxaFG@9xl7;_hW$_TTyaXkm6GxF{(gG}64o-o&&J-l ziVX%aj9X-!E!Z{wi7>qaMC><`I?@B}y>i!NhPIl6Lf!In0;i3a3QgVeMH_BD@uDAc zf6>Hf^tYnozacM??HhbnA+1}cvV~V5c`i0XiD6bUUL?4>E`f`()DajX|K?Uc`Nd3E z`DqWg$Xs&f=1t4)ShSu9@qO@j6G z#ZlZL_QAjoYXV=G9uoJ8`+jGu+L5n}R~h!AG_&o>j7)@|7RXss z`3QQ$?hPwWS8(!jTshQCt~i+tOfhi9M`)HwB{`3DAWJ~8kKR&q4uwRL_;oOK?9L=d zhLA4;Sbx#rq=0XL*Xs)`o-%dw?auFsEuJJ$!tu-ZM2jfAA!-MCok&XJd}N2} z1z*4xs-Ree8jC?>}r%8Bq&1S=KWvI1#v!I^K7FxUs$?+9|CZT>6!C^U-D;cf z`Ix3bY9nT48thBAjnxAPv~S{WWA45dnc#t1rsT^_L{(44bT^yn_AP6Ce>m?21%A6S zk!%A|GnvGC!G(zfNX!)w6TLVE$ui%E~6)j&2cc^J7@$YCY z<#7Lvm6YU;>Wq$V*$7yIPXHUizv6RX2q3J4NN=&5Je-+n*J<;`Q&Q;_82J?k#HMV) z=cNwTY+M-38oI;i5Uc_yLk}{Zl2iC-0f2!B3v(N?Ny-L8^wt%-@Y^%&u!CogLdRFr2ht z0UYi}9ARQ3Gr|SwUyLERGj0V+NgXhjFy-C)6aow}3q3iTj?6Qpq)U~A zo{(Hgvd{i+DMfYzc{0czVD98WK9Mm$OQY<1m>6O%cOpV`%ASIPj&&J#N^9D^cu>Oe zuitXNFK}$jBpu8IkCb;+=0AFlyke;Hk#5i5^fm`qevt$NP~(6ue^K_88hV*?p@a~A z@#*xAscMOAJOC(IDq`Wy0InCFH50v2`nmTnicdfB{{(V6>?cx zGsR`DO|AZ7C-BqWk-98k|6lh3A>&2727VYfsM*Gy)@`|EJR)XiO3Jic z7*>cG2}kGepF$<-x+`)2ejMg?&P+_CFXFc+n18~%AuKmX_1k1F?0*aa-C&J&6?FuvKuixQ%>RF>Spp*_c^}&??3a z-%*eZx3MvLZ&blcJ|jho@~W_yttm6gRlS=ZQ6PeY3D4CP;Aoj$<$+|~i=Fh!Ci)93 z;F>}hk)CDr4$f(21=M4IAE>W^S!-EOZwCuuh<8W5^)=*pCdEG~l6URl{Hafjc0cbjQRlbp}f;8mg2CVhM1 zLy9VziQx=4`ir1~L>mFELBu3%GC$KjlcKZ4aNsU4+ZC9YupV!)z$vgcazK(j&Y4Xh zlEaU+5n7y)bbs4!)r;CuZ(Yg;J{b(degWJ9w&+QO<$|bO3byyJkkptwurq#3;U+4p zHR4AEBO%MlY}T+ockCg-Pa~@lVx%nNY|7SkLcPL;_<{kKnV@}; zyPjAyQq6_A8dEofq@rbSRKCc$IFK}q2aokHyK4ELWjQNIny~L-`j%`H^&$m`-K6MT z&h0ulZXtuk&aO3XhS14=!_tHR)DXS4$}DlJ8?h@o6Wkt^`(YzCUzU~4{Xa;$O?p+l zj-K@1KFQJ>AwC!TRcjlTT^>J-sNIzDTskkg@;OZwSIlIL>)7!ebhN8Zahns2gB`_B z0X|Uwl;+SNhEu9%k^K4c`uFePT}FpsRhR?3O@uq%3W9$v)WBBllhls-zQ0tnmq_Oc&tQy!ux@Co$iYk|`;boz^U5$A~<9HoaKG=^K z$Mk26;$n*Ud%Wm!jvcQ*)YM28Bd-vvMeSxN**{?ro`CuTL$1&t+_$F6;2kS=fI*>O7zT&0&rRq>Nu~fy%93qB->+s%FPl1_&voqAY*eXk}ybotD7) z!DOaR%U~*EiwVEAXcBrf)8T@wk3`MByb$k-&12Du15A-HnkM%&sz%JS5G2GGrmQ(^ z%4w=^zm<%ipkO@Rjgh?V{DwEL+bMd7J5%eezBUL!aFRV->`q{tXw1BesI;l zXX5C=`YF_Q$66ZbZzaJ1P%1bC&z6Y1S%Sr4*c{Qw2w;_aY!=UPy50KKfYgXj@&j4l z>U?h~jzYwFL<~3eJ-J}?NN@yRQwMcu(CZtwe=}J)M(XgR{Na0r-@y`paGYn#%$?&} z1Xz|$e-Nf0EDYf44$>cmH~a8zRU#H@r#2$)NSEQ8MC$k>ltyZPe9$l?^58TAI_UEV zdB7c=cnE!Vw#g?|digyTqYOw79wbV5MG(#@6L+22)03%;L1hWT%j7$!3t^_sZgjw? z61xR=?74hS+m8PQz&0y_iii$M`9MJ6A&{Yl1drY0iSPzza+#?;PuAZqZ`E!6&w zCHY%)QB?7hiPnjn=IuL$ru2zy@?p$EfhstW#Z_a0WEe_0oQD|04BCtch=KphIn!pg zNHKr6rYRQ9_pUmubTKq!{U3jHXuu29VMRLq`pNkm=~9jxf_(fr@-Y+{{8WG%a#QHK z>K|mEj4Y1&Dab68^I2{t8@;=6b-~$&L%G}Lau~{+@fH&4B$Ru~kXLoeaHsstxhaSP z;?MVv6Ch$~WnJP(@$MqBiywaV&W>TEiMI58YgtGvoJ>-o{)qHGV3d>5DYZLXR))P} zauv8~f~cFxdOSsW2eQihoOy)zGCZSi*D9LGwFUUv1SCM^Qnv8An4c@4^bp$EKN&_W zE(9ERZ;!@0Cnv2R-c?(bA>jzM?(2?hGMlZXF_l0M-xB#^yEpf!AL<2r7j2#S+)?$= z{XQ~IemQKN{!aa*d>57fN9ZH+OXKj?+Ly7teu9k%zUc zqqIrX#2DSv+gWJ*gAjrOp(}_T)X&6Jd_h}7Sq08W4#yM8TTOml_+0Z403Ay4`fyUs z>aFvSH^esCdj!iD?MK^!e{Fjkl6jv$o=k2XCgLOTVkK_YKR}R{=juLCH{;PIRpfoj z52~j{l8{L=;(k-_+jDOz>v0LB@lV?`_53x_%iYRm!gd$kOXph zdT~feh8m$O9T%q}z=j+0UnjEqG7;4`W2+*3+E^S`j#9fda~k_K97!&jqM7`TO!!;) zh1cKAc^Hp=3dD1^_97muNGcnCYwvpsRYgEf|r52nnv^WYEnD937Ox|7f0lb07 zV-!aRcaftNvzO5rgVIa5oP{@j&{x_BSY3V+B3M1r&J5}D9|%@heXhI=xS-PikPAtU z7-rArEO>_`BMnv?l(|nZFVoT$q-jf*y8*X~pJRr?Ww2c8J6O-uaAZeT4XxT|tG<2?ZT_DaU_kn8h{L>R|9y*Fuu~$iU1xnWPEB zS`CgrUZ{B?@z=+9NjX9&4E4C=_a?DMV{Zen@md4ZF(-V~e z1Qx%;X@O(==;bh`3Ys^>93S}H&NTUWV7d_eHLtzoCVaEVyZpbv)?ultwmYGc%_T9M zr(DO;u@MRW8l+!9n;4NXsboPjRxKEuTPVr!wrCge;ETC#w8WA7yg)P8I z>WSxkX(WP?j)=?Kxgs{x^Kpu_S&iHkW)M7~nNe+In}%IrxMe7jLcWIVg|inV0=4DQ%0fj>=>QQWfhgJh!Bi!I95VUmQ%t>kW|HKH_&#K zzj&KNusOmjuFd%fq)C{#760J#g4pILK~92Nqs!ttt88`f$F+V!&2ekZm{;K~2T z#=S_Rv#F5u9`;%gN}^D!Ezib#KV?KH+)}x=RXK}y8qZxnFXVIJ_~`%E#n zqH)XcKfu_bcqids#3AT&NQhLlg=dDy_!qdd`9l*tltH^z%==|YQ#Dfsv@T!=u{YX7 zKmo=P${PB8`=yud^>b4e)F)Y>8f#}wQ`>t6d^u4Q^^+?ZpE+D;0sjxe{!$!pefe~$ zzqyT*%A`WUA+-*>?)w}UEH~ElZ)eH3t<;Vvt5Hpa_x8Edat_vO&;4S`z{~ThCjLE` z#oki1a!lqA|G6^&gOua1CCX)Q^fiYy7qpm07lgFt9~JYxWjY4SO;UBGh^DpCKIgAYF<>MRKkRdGvumCzt{#8~3W5E5Q z%v)C&OkK>t?LX5_?<3&?)YA}J6~jF@(#|-f+cYBAZgRE0q!s62Fo5%COA_^dVGFUv z|8zfpFO`0bSP}S0aVrG&CHm-y*+V*ya1?SJ^^+zy%}#w_(Wtmg;YYTD5@|=UL7)GO z1H=tl4t7roFUCv~Q+_>|cb^V2b;e|rq&Hi~Qf8W9A02=AyK+7lPE;brw3(N?aDmjP zjP<(cQ9SL<3sNIViic+{)rVOrUWQwN8HCB)_|ErhIe6Sc>+WbU)XgkG%Q%TcMoNjLaXVU zkeR^y0rQ$!()tRxriN1gW2Ec8Q#NC>=vmhpDy#uJMdY@JY38mwd_s7GJzzjpL0s9i zMujqajlEUH`kHA6?{0$G44!T7bw(&KuFG9(nCovU#q~b^%s^ zE${(ii$z~S=ao~s(4#OoLr;Uh%EU^cLiOeHei*kh)nT9*4zWz_*hkw}T+4 zm!MRI>3hzBhvBkPFzXf!)RdIbN_}@m{09r4+WiMFySXLw$YO!u^KQz1YO{OMI@R}X zmj2q5sU4s{;yn<-Yi=6?RRMo2)K583SXf$I3ZKdqbepWB{rnIrKbHOkM45&$m>^ibtz1!RTFUvngQ-6pkr!w=s-!K$KtJno^}>+`<$KBJ)tko@LEy3 zBej%I^FQ9kyX5)DX;_##?|h?jT$#fE{a?^fC) zXYFU6#xM&D!ZkQkwO5=7uioyZe>|^-8rs+}NWsinSAX*h-hL%iWgI6DjDRvuF>?ft z>88#*WQXn?xUc)uc2QhTe&(rnf(rha=;;Cs@zTz;%+wuTP-$F!Sd#+ycWfXI16#rB zO&67~e#W5hoL;MO_^$`myQ)i*mTGJ_#U_q=(tw=j<>W62&3WzP+*=OHz{{l;3_Ib$ zYCn`<{g}@K`%1`>_p}hfAF{z)Kw$=)Y`o9?2T>o2ea(xEez%H|TnP<)u7H~JEISt# z9ew+-6Bn%YsdRcjM-aL!Jx8b^d@)AHS04dzBGh3BKr+*tE-T;e3E9Bx>Wkl1g`_KE z5$FVao<0D;!(dCW^>Jhbk*u)CCCqrvC3S$)2vg#2Yqt-h)O?G24k}lH*u@$Ol+Y09$K76%FM!*4mcfIXp$q<9;mnOdR$z75GOBC4-MTpjPoL z(dTWnG!AwH@sENrHDZ&s1rsLs4{x#* zgniY06ThM+W(uX!e_Y(7J{2*+Y6QQqZfk`nVk#i*1aofysWxPDr zf5jq$v%?e>zc+&3g|iTFC7qsfsr{ow(4v2ZT_B%5#4HU&sEV^j5Z9CeZZi}FiZ)Ok zVM8*KA*0cgsC~gN=ff^ACDmYE;6T&f+|^V`1vsQ>F5l&uz{yvfsA(s z*)s5ETQG?nK@#XGk8nfgY9<*1eYb88MhPRs7t4EoPdDwTQLb2T;0em;`<& zppQ2t$WQ~jTmL6_WV5-1&1d4R>O(kSk7ZjTB2qoobHkun-#~rVG8_Ae9bwY+Aq%ww zwSNOtL1B7{a2Q%0vuLDrfqAHncRkZQlAb8zaU^`(w_bi3WiX`S%1^Ly~WZ zE%wj-bEXYYe|jOj$mFx(yADOQW6q?x{F;rhNbY*(w`BKs z$2v>z7vOC6lzIn?*47E)YF4#}I*FcjjFaK89VEwL@(&&Sj5I=Naw6Em3>TRIE1>O+ z+9pVjP*xuC`a^Nf_1NF98fvtiNcUz4&4HSRnQFvaM|#c#LC4S=TItGEoERe=t?EUFZ^--Vsu&&W$%nF z;HrlcX3RBy_=v+z-U;!vR`1<671JaB!?(@2S0$v*?{sS&l8>B^w&{FXERrwZqXDhT z@n$8WoOxRM%rY%jcCKIqVj|p+TOa?%PNuLxzlLxXT&I%AsC#dsWSqR_^oM&QUlg~* z?u;DN-Djw9{xOsNxdnQKGkq|G#24t@&y`2O!i|VVj$ZfSo_f2sJbv3T`UR~vk?gZC zQU^=_-(592&eUC};Aw18%^UK(n1!O@-Yo(n`sdD2UoY>wV5Nh916NL_HGaJUR$_K` zhu^1lJqb1`y0Qmb!+V+k&^Nc+%}u5{p@^}2EAE)`H9y$h9}oxGUIrL8(fJ+KAw5iY zdia;xi{eNplesd(8wxl4lA{|>4IUWA!Z{RXM13R3Q6e2^ejqDH#7!hJ*o0M|Cf`kD zT(}$fR$d$5}es+fCP|-+8 zmU_zQ$gCbV##IH35P9I7rxT66e%m$pP2*`M0gc|wMK$h~`}lFB+M|R22*5E>;N;;bs1Yrn&tKIr_=t+^ zi$|f?@wN3riTiW)9?y0x7|rLJ2obt_9v4sKUhmFfdt7cCm8!Sw+8BLqiZAM7zpLyl z1xDP&wlIqcyV{gvuDOc2>eZ5dYPy(Y+OgE#V@D*1RpwA;_vFx$7J_h?{U8sdf6g6Y zJ5Tq!-Wd#^a{G=+Gl(^#i{8tb<8-v(Z&Ha6AlWeaxjL)%UTv4rjSYT{#&Jzzsx;UhX1S zXlbX+J33YIJ1oMrqeL-fyv!c!GQo7|12e4j5{Nou?~)MG^)k1a-d8L@a&t{CO1>D5 z=KV%OZNW{5-T!w}5ShIeAFq` zSUe6|NqspnK?wWA;oyZcW1#27pbG0?v9_K?B;!rU3rZJlmfs6DVgB;cf zBrk@G5qu&Bza;l`#Z#Kqx|O@AU#$P$)`!?*ZHR}8JMqMu&lD6NqLael%N%A{&ZL2f zVP5QeMKRf%FX0^KUp}OM--iO7NubliRp>8|qv0z;-pC_*+%S456)r8PbsmFjHgXoa3gOV0+x1}Okon< z5U-W~81fxB>QP>OzkSfFHSN%9^m#}2;C(T&fEp%zk!uP^B~ARD{Crob7VVvv6QunN4B&@a=*J zmv6g6t9Ji6=t#*TRnVTqLuohfBrs7qt>w2g30<1MY?*j9IMoVSY=0Fm4^j2n@o!tP z8izHrAoGUXUwQbozop%(bJq(I7>ue4UI^BgI&31gW`uJxZ8@dcntX>)bNKgC9`o$j zXAWdAuRkcfO)lh@HagXIg6aU?`3&3%u-+)gYzk+Ao82sd+{k#{h`uG8x%!W<%xGJ& zp)Pm?4+ih9J;`=Ob$+T0jOzMXk@|}7?+r|nsQjw&^6N-cJcA}wfan`Wu)gLgkAwo2 zB$}||fWW+&lY@S8Ec33nh6vL;0&GUQzQuF4biZ=V02KhYZFegf4=(0m!D-Z>tPH8d zZygtmB(J(VAtX9cfwpDKN1_fEvGW0SJ<*8&dFM{kNeE(Col`ka`(kov-uJ zkk0-pQEPQF?Mmf58L|pSRlzC55m43aTq1&#z4@fQa$7A)Jq{aoMO5Vf1 zd!*e5Zk|4Wf#j`xROfqHwQ!9oXh4CefkVPcwqaT@QTP&7i~=u$*;O`Cadk)^iCPS8 zN*?|km8}sut-c&v_vRyu$Dp_LBA?}z03kuLiJ(j_VXYv8v7SRUDOe@7P!ofnE}rm+ zau%_f;xB~{>Cn3mezl^4&!6)JW0+D#=TkKlT zUl%N{M&UtUP$u^TiO3DywQDil)%x(d1ZA5(;@+P67?K30XN#+`yIY_R1h)h+)^k%Y zS!R6np}SvC4e<8e$s1hK4b6%o4nOXinNW+L4Qu!Cb+K(RJiv#QAEc|*v9 zZF9I^EoC#)$ECK2ObnML4ouV+S~XmO44Jb-t6ivjH18uA**@K?7>0Y=6RaGkB6$an zy@r|5?eAjoO&v0K5Wk_V5RkYjUU41SL3REzo2{1$%)HDc{w>FR)oF0joppvH$e5s& z)wIgWBqn~vZ}XRC0I-aO;wHiu0Z~M(WaTUDAIeN<(E@evZTq%p9)ch~{p8{N1pbEJ zgrla8J4N*s{88ES>MfK`Tf3K~>8DC!uq&W{_7Vad$LR1Jm~D3h>?qH^+DrF_q5s+6 z_YfVRYzB=N?=@^?O4YHv8C^Yc9kTN;m5;zEqlWCp-{d(lJsHoWs2fp7j!(Nzxt-U; zm|5saQ;Ob%zZ_!;dQDoqMM)_QGhTc*RKo~?@pxzm0X=xN*q?9+nP(;aq6tLJ-Vo-9 z20ryEW3md=UX~Jo^3i)>bv~n>s!Mei4b?vFg5#8>90CGYPBFk&8sA`@xuC+f>bEl8 zvOHzHDJb$OdH@k*L4}xztfN@+sem|gU*V34{Rd4{aUjFOl7rF34YkkDN3u3koL&l_ z&lRyN3=ihDLQ!NEQ5T39NHga4i76K zNQWC5Z>MsJDkq-hv$cp2{SF#~L_^bf9xs`lAg$k=g3kQd33E`8~|C77(VS8T^$V zW+MhxlwAa}CKuw85@gHs)>=IE(IbuPGnb=vh_x;YNwwr*A5Xn7nepZT(^?o{(ZXte{%+mU#x-b{8pt)IT12i&`P<7DkEN zqqHNltu|+W3cR56kApm=u-H_RbhEqQL4WSxja7uuY-{HT9b>hXt=F?Il z3j6H;G)?X3_)h|S(7==005zgJ-o4jxAu?SE6hHaMiTi4){;hSPD^T!AQ9acyDl!xa zShZacF5FXt_4rxmpxr{6k7Z$rX?>$0nzqD-eBeQRq-Z15D-;pQK8r*QyjUCo-I3jDRzqt5u~N5;AMcCtU5w{81leB^Q_NZrOfm3zVHjg#1p-y3j2 zQjeRmK!mtkBtg8~jLvR0>+l`bQc_y$?_oC%%KXNFM~u#%g~e$bt1(qTNbt`FRNL!e zZJjLJYcn<+uY{LN*TZW0r151K?%3&)l))dRHI5&1zT#Yw<2ZQ`qdbgy_&ei1B~tDfz{4^=LEo-#V^DO5PZadugEAv--?6IhhK7P~eXfT- zV+g%fm{sklGTA$5PAHq~yT-J{N(Ei;1%1>+D*fL=mSWIR?NG*ZJ&b_j3m5`)ykYaR zvOyIeY^*(%+Z;8mVX=~LpBoAOWwh$cl#J~+2dni!*0eLbVrMDhb$Kwonerp68m<72 zDGfNFrx6N7U^b@hj6Zx`vw8PEK1h<PJ%rWD=CgNbWtXu3t zfr{g8-FB43dz^O_E|N+18EZpg!>3<1mn;DO!2`Eu@Vc|Ck4B&q->2OtM5syf{DQEv zD_zIJ5^JYa{qB>g+!Wnjt5kL?+l{|l zhpM2Ol~&}7+&s9NyCZ1Dp!9Z?=)L#w2x)aMhF5|pj=VpTLJD~g^TU@_u?#&YYB_y& zc~Ok}3rg9q@x9ero_@hp(c%=5{fF{eoLed(Z2Uy<_Lj0#32P7*;|out`_t{0Z;#4t zG>~|!Ak#XrrRCr#hN}u+)b_7+ucR_otNdS@k{SZW-+l48()70i&WU?eevKM%_Thv; zx&FU&hfJ@^yN8=NqMc<}Z}kCI2tjQ}jHNXe!^{N^2mlC3)C=$nu16WVTUM$3QfA$V z)+6!8@W*Y*pTe{_w2j*S*_32#3dqKwtAl)xTSx(R`VQPy!DzsnrF%3BSR;e97FbHo z>>3LmG!DPf7ypt2`C^>835g>?>q98|*ZJTzzu8fJhcC9W4$H@T)uE%{``BhRfz0wb zDz2eK1qc93yqr_joJ)V(oInq|!`+ngRvP?K@hYZ!AA}QxA1Dw)7%IZQ%xSTfv@ax` zukqSM4D<7FCK(evv5NH}azmpHP5|d))Zp2zt4cL(Sv7cdJMRBU4ZCkJ!~Vx9CWOSt z??1KCyUXage=&Er60~%+dd~n{99&$ioSdv2pR_qR1m9O)E|&jukfMU{_P+=moxa#w z`~3e2cCosp-U)Plwe{RJ&Aq6cU7f6L9jvI_eVnbRY@OXL0RZou-G696m(NU0V`}3Q z3j3M>I9gOTOH^E1T&3@LaJW>`AfACQaI}ruRO2fPzXvA<$p(?eLGlHPY=9F&Z1;bX SylL+&017gy(lwH1A^!u}WPAd1qkNOyNgcP&yPNbd?rH!LVhNJvRYNW*V= zf6wPn_#HTBiQW5}nLG2$Gjr$08R)4K;?dy&002Tw4UiE407U%=1mIwyUe5gIuTd`; z&PuvU06=pJ{;fSG>US0g4I^CuAb<-12#o{)9^VqW0|5940|0w=0D$y+0D#)Ppwmzm z^$)D)+Ug*{!{b*;Pjv?BFWBCix~kZ}@o@nFG49H5q^J)AG(k#W|D`{Leg#&La|C?5 zH}`uwlU6tdk5R?7aVndZdVo3xsG|H+s*NUl4@p28SJ|1M204j=5(Om*MrCxia)7Vj zXxH||(D6FixueUVMn#KFhT$SpYL+aidjnGM98meO%)OjjWifm z5-XhwF3124Q#amKg~JHhRPrn0c=(?E+itftpS4@RaH4MvEw1@c=V5~J>W}S1 zjheSgV<%%%eaR0%es}}7vTX^Sfs%?{93ltdT=F*In0TOvn)GV4?)p@1yw1Dw#gFux-~Ze8e)zq( zpx#f8IjQ8M&XOllvt4&)NdlNH&YuBpW@q5&<5^`}SUgN=6u&8u_&E-Yw)bY#0SGTh znd;ERD<+u|nQXBsU0}gZoDVG202)WHKzXD9ErRPx`3LLB1m&b#&Qy-=r0%xaWe1w* zW<6mvgqibrkz}}wHF5fKj$7>tF+h%Rhw#Pxm6X1-HteDM^dq*oGuAK<{sT~IwY(7& z)}n4>yxSl`&~LWiu4RhYp&QQ=1G16ndRqo0UfeT9;VT7fGed-}_;#DEEklZb|GtZo z4lQ8p3f056!?)c{e*C05Mpi@U%vJ7F4~+OTlOrL_D!?kst95B+7%P-%aX3*lR4q(v2;djN; z%O)m|8398^;s7=SWI7(aHZgw3)2;!rj|dTEy0r}zPh~nQ4}XKv$vOHDr>k^{nRPZ1 z-7fBi2Zs-H@+HoBgAF8kE+&r$A)Gj={pi!u+;iNCinye42W7Ap!+&oXQXzR9`jrNF zlp+x~U2L+rCm*~{+G_X%bwhG4iwcIJ<%8RW9tT_Kq(Ux=0%ewH*iQ;Hq@f1<7iOyQ z48WEP$U*e5>NCTO=WO#+yvs4s_B5}^=KwTrljEA@G-4;7Rkre;Zw(6x|6obAyye;N zu_eCM1Rau`b+Gj1$a8c%&L^6z-sIJyYYxC_;ukN5_IVIr5sQ_tHx|S{xg3v;fl+}KpWCugNVNa zC!mS4B^PvbbzRGYfS+P70Id+cY-G$kJ{{6an>o8i%w7N{Fzi@)ZPUMzU)nD+7mc8Q zr@)LNbjC?1Uozdl&^bc@c40JyMEvWxX?=M+0qG0DvJza^qmvA{E-b}LP)nhQt3@av zI@o_V@fU#!BXTgL^nXPe4Q1g9e?z`ouB+`s8KaPx9FQn(|GQ$wAL4$(2q2@2`TPs?(pFR`49SpG;%NN=l8eS8x)_TeVOfC zB<)}@TUgiJZH@SMEqNcjn(DsnSB3I2X#@tqAt_hdLo9qhVGjiPC*a%oiZ`JNBkeDa zO84GRQg&{Sk@pKSSM@}2ngGnDE-G$W-edPpdDW2!FA}lBI=dE#wz4Bl^o+-2*s)jX z&=B$(Hr^Yl>X%6k8QHV;X>2w$R;xXqEF<{eQnb^*cj2KkG9JjobCI|;^!BI2^`!|=-F(cQzT4RLiOUw zas8}1E*`EOk=`3TTw@hpu}kyBUJ&U@>o<9iycjvvnq0=&4K19 zODI2DA{R_URvZ5YOIooMJ5|*8_xITcOi$>}3lC1``>2|aWS5ekd!x2=?ig8q$3V26^c&zf$uVwGAdk z!BU!si*Vs=v~5*jV?fG3=tiD*^JXbwgFiq%%&9yAK0Uj%-L9~|Y ze^s|?^A)+R$KAuUaK5(){MRHkVp(M_ZC5V)(D|E%ji^@trC;MUVfPR0+Y19<;5FZp zHOnxiVC&O6aM(a?Q*O!XXr@YLWeKzY(F5@IzP_&a0Y+`kAAjiGjEg>)XOGf7KJo`z zZlpq##~B_85FX!g_>g>)Rp)WnniQ&0qUGaIk3RzC4uSSp;MXR#ag#^S*j)Z7h*0EA z*SHcOcYm@&tXciVh8ZU>s0}=;94h9eV<9t5y_2xlfY{}|?XQ*J0Wk$^cizdL%g-|k ziYXz?bvz|PIker?pdUunp{mvT5z+t z+vJ=RPKigV5NK^PEGxxEc!{{t+DBM1>7tYVa>I}aeVIWMu0KY6+i}l%RVvD>I%OMo zx*NAA&--U9t61ZG_xCRmjk-U)?2Bzkdp+*8vkvLXBT$;p@XE5vg`Z#7?KMw}d5p~2 z#XoUB`uL(YCE(n(I>lrOWEinGSZVV~3cCSgR}^bo+EO82!)ksDA@Ju6psC^WMj^IDE_Sy+i@@V%YU8r-P*=u zO}lRi-Fsn^q%*oJy`DnyL4Oche&sWhPYFY3q%)T(#UloxT66FUiD0yo_*mgU#Ie?5 zn>YFVT+`*MS^OU$$e@?Ls?7mv*krX z#BF&$2=!>j4@7v1-rjn8ZEv~!;;>AK?c?!(E|{8-9RgNpoDF*U=`G5Q^In_`{5{+M zVJvUkzS)&ZRH$uSvJMWyw-2t|{~?PvEQpX8+c(8rQ($?>hJ5@@$5^=7ARC@#Pv)Lq zcRf?WT|WOCKYj~x8Si*&myire#c+{J#rV7HMwc3ly;uUS)PE9m_wiy<7A}?2{Dk4C zXOi2UeWmyJyRpyk>2Wl*myaTFZe>kT3GS;}xDjPSY2&)n601!HI$JBu8wnP5Xd^*o zVO(^{7y+fZzL^UM`Oh0wJ-q=pbCp>QZOzI4(0`>Q)|&ICA(qG~(D$4VEy3TLy32xK zEaBMM0A}D@yc6bWebOrivvm-?0d;sIMt+xRf8R0y7j+`O57B`n~touK^g0(`M?vW)h<}>(vTxDRPS4nXDHCvLW|p zI#Z2&z-q62puP%N?{OFWVRP#}a{3T-s`SVGpXlwzUQk`Z$FROPK=twfk|x8eAI=46 zr3Uz+%B_g6Ob@1wvKOm$7moW0_eQ%7;!CustR_%W@*4Myer0K+no`5=^d3ab9Vsmb`Fq*2_GAWNTS ze&CK8jDQ$;ab)l`1W=BHG8`2NMK6(94zUCp|C=85)6TCO<`xfMB*0Qx-T}zhKZctz zpDeMk5mzX~KRUZzXqo;n8|(JovUf>QG!ZoS;MmD>EU?NS4w1)GxuTG#Onaw#30d=3 z_ZV3g3~)`_Z#`8N0t~En2dpk)Ttt(}!$xVMj_}wf@-X7R>-Rj2WD$o1Erc|fw22CY zrbR@d(Y?r(4Ctnlu=0Xa$! zv}()iRlNyquVgNZ8^YW4`CoaWm)i}uX3{^2L$g>eVux`_f4Wzn3^q{9EptVL2+CNW z0oR}P`j>a4$Z%KV{KTy|+0WQ0H6syCNoTCzZ5--b%bIW z67Wya3a7J%Fa7jRO{Y338nt$e-#sZPeNl9w)4G8;81a8z3KIAS7d<_pjM?a?7};XJ zRh;1qofu=ZWNUjYvq&cm{e2eE^tt@#r`ym`g4mn@ZwfA1VS%B;l{gX=?0BOPaZ|(u z-o6Yk1Ke?i?A~kQ!lJu+#s@_VpCs@`ma2t*lDDFZ3S|>eEHPMK5d+ZqX$;iIh+uq^ zd~4g5S$}3bFtVY2f2L1 z*K&%Cu;N~yG$MRhY5I^XCm0Y|xENHOV51Dl!4Y9ft1jWe_3L6woCrV;ke<=u4} z9pw;;s_?U=`v#8=DrcO>@k)NA67Vj8F_`MnhH+$<^2XG7K21gL<^)K^538^p#<@Tu zTd>W4!zV`*Oc{ujTTL0O-2M6VE#rqlq-s#_dz&2j{zGa7aI^R&j2GNL6&i=bOwDKz z9C(CvEBrfrxNGqbr1_fu03Q_*AD`frl`5;?pGM;}EtkGKn#># zZaYPMxLgV4=36H4l{zxFB4IVMK8LYLXb$p_aBzFJHVK&jy1U^yk`U$#or!()JrKjB zs&Z=dc=lN_Sug;vOa8oiG2q=1pUJANjZ6_(B{pH0s$?##F#jcE1+{P|Z+!2{A84;tS{E@E%7{Ofe? z655`;{xehm<5e$_!!nAb8fI&d2#4I?ax?)ftQrH=AnlCC3oR8lQ z#GQOiUvr3j2Pk$k_NpUP5lXcNtH^iXU1~{ualw8Mu*-Zg5KDS?uuN|!_9_3% zOf&r%hcYD+ojhUR!i7^9PrrG zp7mj$P~~3|$=cWRg8Q>`{*zlOF0n(T9x|{!!Y4!NrzS$D zeGkC2v2<;^mF!sPou893ahhUp^4?cR>TfTJu0+|$DXc$URO*8}uO}8M+>aH)O!L~r z{KTGgYN~wf; zu#F!c0JB}$^P4Qg-htC$uB-&DL}{klpQ2`gVXq@yTUF8N>>kPKruL{bedP$WxS(Qm zDgf)*o6$&*SC!(TYAMhERg^{+tq|630=if5t`EAeHwCvA&e(K-iyew@Aj%=Xp5R~Y zkJp$6zW=6{W$e|W_*rcFcht#rWyL+F9#kYH7@iEp2qh9RRZ$+)@ zK+#EG`e}A0jy?*&KPk0N_;-sX-x zmZa6CsVL69EMBl%o*( z2URBYEvnCaD2JD|XdsIdB3m=KlFc^)#=8S0Tewo(<-dx{?{e38GQ0B|SnF-&vKL3P zz`5MZdp=&WvCep=C=u+-3u6Fdb}Ps`l&A+MI`K)~tE^a#I1r z>2XbVv@%$*CE;C4C^_abmiv|ndG%*!HvmuSDodh4Bfb#r3-O1^No`YR{pHbsvgV!A z_}7-fS+Fxv{$o#)_Mul+&)yN$SY^YJtPXt|sefliRyC8O%XyACH87syB2Wp(1(Pm8 zNkxTm@{UfDW+!VoR}{ESG@R>6oO!BrVVM@Y%6UAUiR-X(VQ9-;D(rc-rFxO)^OeTr z_J+^3ZHJQFG3e<0M>rE+z?0F%H064TuiHPPK1K9lAx?L418e^g4a?1HzEq8>SQFKE zb&KU1LSV?srhr)JPs7DjtM1@N6Aau>qO;)*{$II0KWDd0@%XEfi{H|Rc7)T^JKSFW zWihIQNt6eOY;rJU-ENjAEnJ4b-I_<`7G z`-&HT&N)_>Y^^3&x@;nMCox;VK3AOORi>TzNj?M2PK=ntVI>!6J=(Gfx-Wh;>isl? zNxv-)OM!Im)s54eJ^&(_{ClX8DzgXZ)7}2>3&cZM`YytzKErpW>H-zYz3k}#<&d+z zg(OZs(}lQUB&?IW3F=!_>i7rHaT@<~3h!k|hAxFTDbj(2j$#j_82}-s(+HR;qMN2e zo9Pbg*$v#EH2=?n z67{g5BBx?2SBSdr!5hUHE_*v3jgPZP@g?lv7%rdU(IT+f+1go7@PzSeggiwASxmVN znk3^?YB3kE#ew2nJuXuLC9S%qSx$l$;{CQ89(R;PunQlnpsQmozvAX(9F-Tpy8UgV zdd7zOofU*IpnI=Y?*$dwH-BlF2DOrkVV&{BN4|XR+M6+H8vh;_0YD*{Nmz4^b+kSw z1oVdbmpIFKgOZIA-IJqW2pdrs_<5PN1a8!KL~a6 zDux@4DYAD99KYdiiu5(zvFtXVg*WO{J1*TB4vc8M`%90)istx4Mj0wQ%wb8=VD$C5 zmc%NDPoP>!qI{sxmS?79W9~lnpZn~KYxHwr=g|@^zryBh=*tD%%TJoF#3gTArfdU# z@ZAsuzV;yT{<&CE)pOkj2*`AE{&&F>+HG5`Q*{mYlZ{a?KiF;8?=^fnn9P{q#g7A& z(22A(pwrSnw;b0p0l$%L{OW*D#m?*VF3Mt>yr4X9n(2H}&q%pUkPdx0jf}1qC<61~;wdg%Nl-^;%)?Om57KJfTF+Gn2p8(c~K zaf0#fYYT{xxbT8VVS0Z`Gr+){y2So*4uFcM8_lyimOyFKef!doMZ`EYgn;&$R1vhW z2d_KZ`hCqVUFGPHT6Y7H-Ht05Y1W~j!@);ZEy~O}*%!res%vD75XSpK$PF1T8=5=K zbop1`Cu2(Yv*pr*ZPdNs-97pP3~da@PqgKOiEjLYoq_A^+@u!-D;HSb8N4l(Buu}N zRG7}iJI(_5$!9R1c&ToNumzUvJckFv+xg80)_z^>V}sa8$q4^)DslL0WCcNDDSGeo z(t)_~g9dWFyi`9wdqQ2Il{X_U@9w>hXai41o`b%T#8Pg;)xFMCa-G>(!;fmX0GP_m zZyMa~^zuEIa#n*$S3D>0bX&{YzM0XZh}8#5wG>+Hmz4LWxepKWZr7l0FSZd!u!>Jv zydq5kvZ7mmw}iCb^4an1uZ;IkJ!oJjVmmi1KiYO)mi(bz_d_qL^IoNKn<|U)5#unL z2diEK!~}hfgPUP$+%w75XqStSo3vOVcDj8Jf+nBf{1X*sdQ9dhZtC$uxLw4=HKg-e zO$OC{bSQlKdWS{>UP|`LpQpg9UE|pPy@q^cliS9>kF{Z?#O|*n*S_)zF{gJ9%%c18 zrt?(xmL})HiVQJ9qWv~ke|rY`l=!T`KN#c5>h?NnO+!RT#B!wZ0rtH(`sd63L2dB0 zl&pr{nPg#RK)H&D%REINNCX2)^P)Ms?ao=jIS`Ww7x(iC<8kF%u@vH8<^ta{zzu@uFbYq$rG*{8GoWI327KSqWK>xH}15@h8{~q z!ZW{_?(@5wR+DFD2MGf zza}ozB!qd@s=i-`0!zHd#hSp2APvsGt#DaTvUzDP0|uJAj?d>e`;~pq+LUPy;`R;+ z_CuZ^M*FNQZ&xS+>h`i3HEGXM_%6^rT1%*Fe=ps8jSwbarYZo1aQnom;byyBGzSDI zIzu!T6(Pt?Mo99Dc1_OBK04zmdZ4<>GCkx?p5%vFUE!tJJE2QTK_kMx^%M22Ewmln zKktpUj!c=YANkt8>4Cm$hJ4N52&{mh@{SrG{4hq+F~K)hK528V_Vgc^hLH}J0kZ76 z4b3;QVIr#ViE)}99ufOswyWxoVUIQ;jqcP6xT%|C zq*M8tI2nPkO6}}Bs2o?u>w0IOkS(U3#ibLiu_kVc z3Vu+dMHyOJo(<+Ami=o-5lv+bXd{qACx_77r5S-z5B(UlK5n z%|~|Y@ej3~0h;&0TlfBH=M;rFe~K3=MD7W5AH3S$t+grsT&*a?mr#3wIxX2b_TFDf z0P{kyPguTZ_6uyPQYTv-Yq>gf_pr#|r%#4_#<^^A*U$Bs<|$~0qUE*$-!41-=j5ViWJxmt1`(B1Dk8_lTJ_p>+K&bZ-=2oqO=BGJ1`n@`5Vm${;r@C z_0xOn8`Y7$H%6oxiFH+XuikE{6&>qCM6-gF{s-5q*0co={ z0_jPf!|z2<&6zm5=wi)*NXv5Qqg623v{skqoe>V4%B5O`MwL5%mBxMQESo<^fwib?qVaCB(uQVT2Np~gb?QZ@r;|eO{Stpq(NL56Wqdansb$L** z^1ygIez|MWc*N&Lw4R%*zyXGvk5&)X?&-Qn4~3la1fPm9!O;j#Vay4iO*>}-{=9rD zO&AfQgUKD6K^-P>azQm&^3fu+-aw9HH|rG_ajiFbqWBlI!$yR*>gv1TcB@HYUO-rx zf?=bCq>oI!q>JFxl$2-v{BhQ+QgVlid~)^={nt{LV{oMkLi8cK@)uFxON7gcmF>8#?mr$M!K@S~*A zBD)I!oZ!8%z8P46^KHjrb?oe2)OR;-jP8JQ*6yn{&D6vTobu&l1Efs1XA5svhD}{r zvnw=ecJjjs>+n~liN)gN^Sv(**2h+=H_JQzW(Bb`yYHb_!WiFGAtBhExS4qm)e6qR zfH74`^O>QLNpo<@N-qke7E+y^m}6q+{f`>e&%K9J2uVkHj9~=f4J5xj)~HCKFXr$C zE$(-P-fm)3%QlUkU!4}H0N8MB$&u&=aztl0z@EJN7_m4W%rZpGdb?vr@62jY2S=rh ztxE*FdG9gKq@8r>!eYT!!ri2jg|kA_;Ub?gCkq>4aoTEAHIH^*+ch6z9d3Z#+e{0T zti8)A@chldQz&59j3b0EOLRLD7sy8T0fX>@=~!dr9b9DF00UQfh6xevzf5^{gSb%o zOck3&BIzmr4421EQXO!Z=oxtds1@9x;kFVnfRQ*Y| zFiI?U*}cxb^o23O8C84H)nZf`D&~7!KIGQ_0PxHmK2yDxfW4ORUW=E|~8L z$<$Vx{O~`+gc~G?Fd~b!7mj#r@uw4MS>Oe<38LFj%|_Yx4K(&33iBL@A#mW`dmyXSYvKr|Ub zY~lc%RB2EZ%SjEDOt^v8h~CJQ(dC+?E_Sgmb^z}gOT8Pf-imP(jbX=47WAwan9CsP z7i`pb(U#cY908xLz-kJ@HFZmorFpztW^x}S%<%eb&jlz^~niB!{n)#jlUB;lO{xU zMGL99HP-I2WsO%bg;QK1$$smu#+up$^XA{&j{6xzc33gwl^sQzSkyo3>1hCT=`B^f zzF);>5K;1yXGl1^4R#dX!^-zmfO`3KsfH~7da%!saRdBKN(bj3|#mTJ~7 zB>KlI4GzUlpS6qYy{YL6Nl>IfPuD5@g7?^Bqm7En^gBS3tER!ziz}bDMD%$4&qiGJ z9&V67s)OuUUYrMkJI}D?$mNY=Pq^Z1S=o-shv2S3Qng8>qHJck=Wn8qY)-(HT~j*) zW;7);wQQ^=lP+1q|U`6bT;XS!^fJ(pM@=n0b47YNMJFlfnuB~yKp z5DQ*>nG=lU;!`|4H7FW12ralV*pqU_TOI3qbFaG>CV|m`KGaY0p!^K>uHY}IvTJe& z`1qXNTqzHl@YnS_M~a`rs4{?@s+(ot_wCJ=5dC2Jz~`0ub|1L(r=*!kcQ<JF8!wO~uv1t=o0c~a`Rb~dTgg^{A^Xhk6DQ`Li>%DHBU#)Qh7yt$7LZ?ZM}Lup zyvJb`M#kk6^a_poDO~{Wh>nvIuS&3De{=8hbTi>j2ZVh^gsA#kt`UZrdIbKlj{>ZA z4PXV8e|&rzs+POeg**Tm7TW8-8S>q9pym-V4m+eWqB*b=Kc1sWli9y#!GV1?I(G=W z@3lrZB4ateMlg7o)b-8RfFKc|5{1|sfrq;h|}JCr0XhJuJKqh5Ek?zN$$S@$Z| zrC;U#+fPN@&c(6nopHo5z%G^EC3)S$3qh7!Vw%PxsC*t#^3u_43`?P^=gZ#@7ijms zbF3h%_)d^+Dna==39u}9>hkj)cUsETb?83tGG_D)?>L5?H?5N{1J>|wT4w}fiK?@r zP+#nek?7r^=dD&^t^p`-`Gh!fE)E5&s_tiOhfiYFyM19L4$H#H`RCzzYSb8jD2YLJ z?EB5Q&6oh?`z*>Fbb&xehYuFRxNBgkD$@_X{nJ!9E*G*=r$cL$dl~OSiBx&q*Z4@M zinH+!-NSAs$Of=N%C#T4#0wg+0NxSD(F_|F7uLp z+pg(xov1Ftll~*;I~-iLr_Lx-w@x}@6EAG!2Xdo_b^;zZfTz>ckDZpwG~I@o zg-9R|P0_~|?Dytt=yPyrb%Sg!rLEvVOsxauBzpKQdj(i3Y&eT|X4}0_DM#LF*9kT4 z^hg=BJyzD9VhDz_OoesNeX8!JRZI)UQ@@A@k0EpZV=4^tL2ID&IEvQ0g&&#_^}5+E zk7BBdy$KcnSj9KtVE^M1Y+jU2Yj9BbX-!0`esO3Sge6A)zEI{BIFvm*iZDOG82uC9 z{dwsc=o(NS#(3&0F8IxO4N-f{B|KAon;xMj^;p0p#y^^M3uRUjK9gHwB__CQv^As;)V= zsc?q^sTtMhDPSLJB9X?B@&we;$qXlMxGOkN?xi#0dRCpXe!I8&qBFk|qej-ASBa#4 z#RxKlMxQ0myPcp_2P~rGHjT0u`P(0M0qMkzHuQKDg9BYHS!IAU_8sY_?vpLR`y-kf zSWnmnoAT!97`ACAu66d1^F59GGMKii>D@gC9Q0=BEQXk5Fv$RVd$0xAV#2WGD<-Ja0!z9wSGj*YRBLn;FoIrVLzc=D9% z)b-zt?Qtlot|!!6LN@h)e$L-gr&V;%-_n>rDkR3H%yLrfZ4>I0jp+))4RwIQ%J*+F zU!pB=j#O~FzhQ?k;=eYBT_p61{-RW>BL6sx71BTapR3IcU2xXgTSZ}qWeF6smIPWt zF8=Lxytv-fk|@z_2lgEPX2z^vqwZl*6s_v5^^pZ zAvsun$DKK|S#gaj)Fv=-QY5)0s*LEEM5YVw3CbxR*C^6f*I_pi0O$jXKkadBy{chq z^+<7T$a7>3dzJ7tDXz$w|xKB z$AW3ifAM2<@Q_^CK~fGGJ{huOtA?~4_KLNlg>exv2kN6>m*#+p96-PxkpP-@bZy2VMz1x=L;GifJoMJX|0iLZ5_MvHdci0>5l2}lGQp&Q?y?u&UaYe!UpX%o{v{*#NkqIC`5yn;1Pmc~K z0OXpNfj$BA3nXKLps7k$_D-yYOrr6L#dElNR3&unC=6g{*5u`?-@_K5n4JZEC zlGJbLM7T1B@M#y^{u9-s4lzpQ)g|^K9Bx=I9)01VVgplg`K+fd}MFSH6u zg##1{0I~eN&+QVYy$R6R#KuPa2UCcf_AT8Y?9me_*2F;(cGJEnv3&W)M`2 zCIYdDm}4+dM_&{>@Q#~2LsdU)Ir&)7b7r|4Pt;2G?XRvK!Nwqm9^devVEAlb?nzB- z7b_(hlhkFyqbEpnJZghm5RnI4BcI(3(>B^+E^#jjfcLz>dMyVzVJ-k&*NCV={6GMK z7AzGiD7}5zMX;fnLv1jjgAb~WBCO@UTk~?B(@4HWak`J{^-R+1KaiBm%e%!6*OO^f zxG#G>Bo?Q8l11)Kp(js9Lc|mLj7Yh@Zl4In;S5<{j{PSE|DzOA^Js_6ifYxivsAhp z@(>Kk>wU{0XZF@qgcH>N-}eeK533sNTGtUU{OV&Uz+WkN7kHs~?vz}Powv6tyns6X zB`7CYcI`~V%itr0;>1!j8!F?M{E6KQUg2BR{hW&MB<#-GP zGNnn`Yi+0#i&rc3n;AdaRHQ(`pCBv&a45A__`7v3w@10PY)D4kP2{CTsL@Y}J8zOj z!bX*mitCBFii(F+cwtJYhs~X{(2BhXb6TCDI-FW}S&jgj&W_VC7M&UgB zwVK*_0qZiH6jTR>?o!(YYr*vFn+E^yhEL<6=7+^zboMl4wjJ*JnWc(ub2dEx>W4=$ zJ#)yfAUy$c0X4Pvgyr>*3t1>)fHrl*s4abz12|;(vov16$wkXQ_K1-bXX8e`H9pGR zsXnV(B+1>g|Jtnb`dA{d4b`LXcL{T8?T_hY@=yIFP3OWz$721B?$Um=|J)c0mG&sN zS7@DC(p+@1(C~7>@U3i%>_LT7Oj_Fp;vQ)5&=Q>_|R5}}q!z|#K`*H3zDxK_W; zuC>&}9?wMWvm`@m(utY1b)W0p>^&10$m_%JE=ZC&m#~Rrq?uaBNhWJfe=unu!N0O9 zyhQi7xs+q0rThhk)Jg|Sz*a@WF2NIK zcmk*_DD){J`Kwb#dK%dgfL!?qzGqGO>Mv8Y{RgJUHC}lDCwthBtpdLBGXJjrF6k?W zbN-|4{YD$P=hP=r_aepqWkiudy84hzzq9v zjcFa&Dgmb4i8`qhpZ#rQDIE5goaRvM$NmElhF04Y%sO1IIAR6YZF75h+myh2^Thpb zAT8@npT+j!OXqb^XYHP_s^8S1OJ)k*$*r?*aQJY$!Ic=vScwbGK5Cecz=4uR%t>St z5^)^a$Gv5YKKIGG*6_@qv{qB2Y*yE-?&q^R{tB2$pEqpPc!YL$tV8L^Ji2H(@^B0B z1?71E-u`bHQU<;73k+!6{EEmQ^iKR)=mY?s3l-dxv+owC{AEmc|#yA}KB8oZe6y;Y-T6BXA2 z9$@>cBR$9X8h(YBHH7!k&4Q0`W`a(t1hOH$wDkHF+97He#h$&$Esif!D!OJpH_7VW z{GK-_RrA~umX&8~#=^F-pV3X-Dv@UY!RwE+Gfka}7=qh4ffP#fMB|_3^C-kwv>W8S z1T3;y^}U~WIb3|Bh<5ugvh1ot49c+6p=i)~1+~rub}ak~wq+zuW_;}%(r>niVn8%? zUl8^gX)YLRnrji+R5+u5l7U0&4oXwk*OSePh<$wmt~9Ng>LV2C;L9&uuM>gL5{`%lNY+w`VGWRgSU6KL>P?Xt zsRRs!&F>RpZ0=r0H1I+1Z1rV`ab3Swn-oO5qF7H=Qo64%g=UD}MHpZyboh@3|Hf1x3Xu)W5C4aKjlV>_gFBD6H|%mZD-s*P!rH{(-HdY@ zDuf}AwnF_YBsTc)SesH)l_Y&-BeKWf5M#pWW{ggf!}ci6oNSk9`==_b^UtRJXYh5M z2#OEBM8PxRFu&bJXJ7HK|A53xG|%bogBvK4(eQpD@12zb6rc!>JSQ77dHt?`6K&Pd zo7?W)^aMs{!~$>oppsesUbbnmkP0a_RBR$Mi;H(d=R1(y#$glQ5l1h4HV#g z!{6);RRcVc5uGeM)}2OwM)0;cxF5&0H~s?r4bsCzcq{DSobm#D@ee91^8*EzyGl+n z^Cohpi8ftC@s4sn-(Bc!<_T{ZSLZFGQY&W_nsB1H|l+b?>MlC02+YADT z4K@P6GWwR|v|QNA>`tBO9$k6|R?_0IB{cNJBGi*Elx#88(<18kq%n!SFB>=S^^*Q0 zckoU4&l^Vkd*Hu*r65vRR+);`odW8cgkUKEOPqPJkn>s52hWy&-L!#&?81IkD}P=q zT$s#u-n`f|s~=s7y;)1g79C04DYZu5up zu92)wMpg-)>%F^kx;C%-UWZzS@DX0y)UvJy;22D=f`x_Z9p2jDr;^V%cHBgobey@Y z6q9vnze1ZO!r1D@*80h2sUU8M9K$`gFuWS9-_YMaq^%+pPW%@tg+@1MVYe_~4_E~T z5qOike96_ba36jp^+`VrOUV@IgJ27*#2LfX1=#;?=PfZQk=W%ATb?U!#ljg~k>T#Q z-JKpxpA79-Gze2j5H3^KMO|qj)Vyns!lF}CeN1T!Kv8a@s58ZP-0rX6Fvw+Ukn>Lv zoK=nyvhf4usr-rk^+BuTTkLjW@e8==ATiGPoHG%O*cx=8VN#q1gv;%3RjydyD#o&s z9sTrqs7vjW@bvR)WO)ArM)W-B6|-)nN6757tqBSUP|XWe;+Pp6&yysGI%K;x&Ll-} zMWOripp`_?wqj?;9m^l%9XoS(3+@DRnvp&%Va4r~Ra!~9@3|tc7I>(EWHr%pB;w?^ z4G7}UqtZ^=qz;^lZh(OZkAdcwXdw(WEsl2M9{s}Sd4T_GL#fd}^_tZXf9EgamBMsS z2w_CKc9^vQIyqvaqe{hkX}Iau9u z`LleSKg0}#*K#HL`>lYjae4qhiM{a5PGN;|$f$uO>$LE~f7LR*@T*yO4z@7;oiFjp z)Zbd4h7#c!pDT;jhN0=A6~`ZAuNiJ%CUCuACF*hU{=rZG(;`@ID=;8>v6IP|%PT11K}s-w{>X(tEn zPC?PDQ_1|XFY9Av`Ss)#HLb)v)hhFG5C?OUsvy{Wi_GX%NesGNz7Ij>S)cmEC2=4> z8~pO`v}JL--vVuc*kzlR#GG*F<0$x`tBBju5e}IUJK2Y$XABLa&{*Fb9oOy!ne#M? zU#sEwYM{fMC(N(3&ezlSecM(of}foSVz@%ig;lY`gQ;;$tnpLt`Hz`NrL6ykP%>|A zlK!UoAMtzkUJ6Pu0xDpMeVw%DIPEZ8u`o#-#>aH#CI2emc*6;FIK;ip7Di+?-4R(8ZxgvErTA9lVB8BUGdqffv`Fb^ZcT8&Ns}etvTXkj*?)9s_){GF z-<6GmADBofL@(#T+QcigUgdV}$o(><*hXEw`|Wcr0x z_y@DingzFrRq5dn7Y6Ef@qc^((@BqMtWHR7wBa3Nr>WENsz6V5q_OP+MJhL70268m za!zR9R@y*U({1!_S%|gOmCD)tc?goQZ06emXWns6p@}SW3*Z0%e5y3_0=yE)YOp?? z>phAyi8q50(KA|3I<&gQNo9e~jr~MMTU=4gqfP@tJsV!6wCs9%hSvWo$aarWGtIl~ zZ`i&CN==TfDo&EMQy;5Zr{#3Eb1WNREpnbefS(dL2sqH83Mfot?xe~agvW>XQol*& zZ3zR*Tbu9FnFt;@`r0)(F3X97{C8`R+TZ3@uFgMsq3u?$=%dLcr3aI8Uh{6#vHgsj zJ2f;Zl0EyUB{3o_Eud7%hic>$zRVDExbHe53%V%AuJI&n^660Z*3QgHaBsnhsO-Ei zwpRgTC+!xXAU8R`nOn&|j@>3B?Lnd?B^3#D^uyPLl6e3|Au>Sqp0Tx;$0m2ZxBCS{JPPxfoK zOO!(l8;p?T9L@(DM&`uYnYQYiptXpMm|P(gpC=s0MhK&>u=TyDAS;rm-(`XYWI4(H zER`&2Y#)b%eoFO(@BIM!Si2r$OS)YOk#1?*N%ke=k>jQcZry7e=ALQAS7XV64vRcc zVdH?3?y-`@@T$`L@(K^9w^o?`g|mwb|!;wxgdV6j5LrY zn1!uhab@gGFZ%hG!xyR;&v@I?A8QlrC}6;POeMH`WImGA-$C}ByzgA`7e0=YWT8gn zK$HLWY%d6Lqiq&io$fmdq;~?sH6n`%dUheGJIRjb8@|M%Vxd>eje zKhKG0@3rN)-XmISZ+xOX-ud`_F0gHdks17! z|KYnsnu~JBI$N&`P-<)lwtC^5s;*P&fWPJ57=U7c>`a+rJH~j@-}`}-$o2{$(X!rB zY%7p3b<`6RSSyk#MAqGEMzD`?%jgf~#w5DJfCki4aEVNo%3E+hHhyq6<^i8w_INi* z+}bI!L?54V2N#*!{5#l_EvJ5-&3p_7ywuXM0T6OYR4zo(0exaxiJ;%z7Bas%oVdoS zF#Jr}z=Uf=Z0qDIF|Oey^%ePBH;H@?c8b90bL=^n?W_vlHSvl4bjcW2nHX(MRqaDa2e~U%Of=+zi?Uhs^)*d#6RZ?vus6TXmz+N zJ09kdz6*JFN%51sV$}2Q3Zi;4#_FdH3Fk`0i|81JF&OWlpbjSiXedHm7u-d!LWtXU ziTvu4|9hCXue35lJ(3+K)I(icdYFUCVl-+J3?E(YkL061cXm!`B9at~g?&$GLhnP# z_Gf1d;M!?72NE&E-~l1& zoNp__KHP_lbq^;ra2nA>m1FXM z7Vi7Ub`|gYppTVK9&{i2GI1n1DKcsMR1@*i2{cTBPlPHtE4y$z-)f;Mcn5f|i7J(e za`LRf`9Iqt^Cg7d=Nx{bccmti7m(2@Q^9c6z06BHH}{fxep;`^8#SYDeWF$@u6-kJ z6hA5+*6{5Y=ZmmM(jW3j0-Dn4pZqMt@?lGr(0?Wx2|bwYQ;yObk;e>v1a~u>HW;q- z+Dlj9!IF=Kr3r7irYP}s#!&dH(GgV$(b;}v^>2ktE8Fq1@seW>%Anmkh?JEVsXrcn zx#oI=coOq}Sg!mxe(~P(8gM5=rkI^p5U|Kib-gS*Kn$en&;G~$ux3g)VN@B!_`ksT z8%`X5zZI0)g4V`}7vw&ZiBwhwS13(A^Pu@*nwYbSHHM_@yCfTPS~~tKwEqo_hR;P4 z)As=ezQQXs-3WGmV9Ud2c=4`9tPNq-P%yaECvNz#J4*(aj8x~zpkMVO0Xv$IQ{eU5 zzX<4{%FD-2dG$v@8_|nO7M#@G0=lYWIQ{Bzyt>-Z_4%dbkEz*{nFnjc7ti0v9e@+Y zQZ;(8Jx01+!Xng-{#?WCe4`zQVWIy7UHoGJcPE4js- zBr$kB#6FA~S_p+H?b+ZU0~bUFE+M_==JA(7@}4j2up(JD-a{KB8H+h8vBq<@0}tq- zyRSyDuk;qk3`$g+U*wmNKyJ9a>|gz63BBN=+?t7wPXSp&RwCw5ZH+o<`g>h5xtQ$V+e7WIJ1o^Q+t^g7@qA?-VSi6K;uK;h!PLdfW5t6^Fxs z{a*L)y{e?O5ZOg6(^BtuWL5CL7vi9ss@Z;hf=Qda`qlNG@02^lNbyMrOlCDvScL3# z%#cURO(Hc;FZsgZWo`9ekwF#-a?pQ@)x57c{VU-{ebvXc^;UZ&EG;*_&YH9Di1fMY zoY3B^e_&KIJN9<&`q+!)G#LN)d+i&(^ne$}y&a=6T*HdTjv?Bdohtmh__G66C0SpI_(;aQU4 zUni|7c4-C^%+hurDMX9 zn;e^pg~&r%Gl%>6=l#{{Mxhn5y(}P(k|3Fmr|$(*FeV$Z?(y2bRZvQaww5PG*XCSMy`YnF2RARy<|{| zEREqMx;+L11xN_$w3JJzmR;36hmasaPm*OhqC&h*l>`{|dG`t8w;71s)j z6+We)rNjkAFx)>#^mlUZX}2=VeI1ZnZcDR~xdqh#Wf5zzrYE)Kl^ga}3c=cKO>i}x zXqD$Eeu={L23lxul=SCyp!SV0__2#~?Tr_~#35m2sj{ZL3Jrog7 z%>*(Qr~4O-nddI#kfDukZ`Shv;ym(ml4#PLE5|bl198C zEwnCGMizS}J`Ts3x46Xd8H7iZL8jnXkw?Rv;1={X8|l+nhL6Gf2xPYizl6|{IY7*F zumtd$?@6*UzY38IkJ3xU>ih}f+jfSF|5YS~DlO_yx|Yyx{M5`~IIq}@UZaqCFAu_$ z2QA>0xgHxToxej7tg?~Sa9|=UvnbLPuI!qp%Gqis`5$^ta^`-a}* zZz1+eiCimm$_I-$h;+VvvO*TZ9r=ic&0zDrJ8y@n_(v7(3Fq@`l?H6S$Fwn}Ex{2K zWI~?ID1m{IaQxGl8vzwYEf*3g}gd@bo>wT-tfH3*@;#22{- zUJs`MoGU!axSzAdaOq#^+NcY>2;Gjtij+{2srAgm2mM#=f>W#0$BuP=!r-Q^w6s$F zi(i8FXy3^5H0e!D;3*k3uD9WpXtjGJ=NgM(F-3(ZYsI@(Gqr@P(bC9KS&KasymJ{^ zjz1lAvJcBiY8?2}$#rV?vvmjWO2CRgn@o3Wf%|CS;P8EH>)tSdCiw%l$N{dJ>~{VW9B`aD zLOLTRB@-Lvh$4~ruHMA5#C@rt|2roxC}Y9>d)jvQ^Bo^?EIRxc)buIf$tOL7%B`)Z zJ6t8ziZ>hC{!gPEU15d=z{C&6o4GQm>jk%AFT8$^UTew^Xv+%OL{Ykr{CuWk_}F>b zWis*K{?*NR`snT(<`LQnUS0HA>UF1ENj2*-Q!NCyUP8_ zF99y6hLsyRX@`O(v|(5)241lAJQ=UDn9`Bt8oI)E)OuE&?B*rawvVB8@vVGE0fL`@ z6ny}~Q4|U`Z+$F&s#@dXKjN)ZCKbM;p5XT5!vPqE1)_8&A~1{3$$H#%*!E`EMSxu> z-+Ag$?8`@a@QB+s1|7UX+aHh!H5*=c$Hr3?In_j$j=`R00_uBnrYtFlwqa=L$JbfI zpYA^wkjzE5R{kbI%x2b=)G#D+_E->C83Z{y>->9}6@^Sdco#GMe|-nK6d#NB>oBBg zho4R%{l>>N=UphiZ<6fL%*iK3N#AbKdO?Y0WR8r-vae`c@Rka!l%+|r^(}oOG<6?Q zhCjhy;xr@Mv9luxhNda)z3cMVgKqeTH`x+kIrsfJgcHGpp{rTo?Q`}q&H)^XR{qO$ zv`W33RHn(o=s(x;91~+?CG{$bPsC_Fsu`>Qq)W)|{5)I$<`-Y=0^jXVCJ3+t>p~pK7E0CZtrut+&${)X{nVy{3~MRYn*!qXLQ^?An63u z=J($GZc^DF)vgWxsJq$z!R$H@h56Mxh{}2(WuhN`HBptf8eb&7907S*J6RjOVM54F z>`ZIPiz-i#c8T9bffajJ$3E$<7=&BWkZVpe%U*;=jRy*Y1`m4d$Itt;sl7_nn^$Vj zr-Cey?$CE^TKOgd&qu65cSnPp`iUQ}g6Mx)UrIXOJjTzot%=b;DS_Vrk?l=dtx0=S#Ru3+^s`S@2JXx%e^HOvR6^b} z*DY~vO6ry9ux9an%Sr?2@4jrJ|5oNu(O}r>br%_7F~YjwR-yp9YaT9mbEB_$avZ;( zF^AMSMNK*wR{-Hp3i+TU8R&*$Dl>;$FR#@z(&+CFv^Doq@}h4lq=&1e_UXJfQA-*n zv?O}v8X`tjIxJ*=VB zc9g4DDvgHzRcP)WBe*w`ZfGrteqw)ALJ$Niq`DN&mSW!8R|(Ubc<+o%WBq)w`T=+K z<~&}_M|J*ZG}@!+DhMVkbZ;i&ieZd#uCelm>~4CayTSYjbIl7?IlD9^4TSKALmFF)f`dRKwBp} z53ZEXza)^yA?0NNlOk);*)${`V*D4JE88)~fW6+rB=GW#o|Tv*KWow7E5ymqdMQ!N zb^*qvX#Jhj6!P+|eFCT{mBUZJ^Y|MAv*h^gaA}HvZM;hjJB4Nkq)-Hu{wh&W?r+Ak0=U^6Mn|IJ49r$LX$Ng=#1twFvT7=eY zI`G~6vcMHZjEaFm2-*Y>O!$#5Z4Q7{sLf}Q{{Xcc_wMkFwe(b(yW?|QD@r6Omn!;s zjiQX@8J(LEGB?S4Uu=&3j%Q5MXtr-$L)O1`uI>kC|9Mmty4JFFYMx4+`f7yjep%sH zI$tQ{1bs3);Q3;lt+#n4`XCJV)oMgML#%{F8Fcn*e%iI#_*`>{yB`$!w_u@CeL!>< zzmHKUoSpZdQ|aM;e4B*_wpsxjQkJ@D#IUW!Mb*UVoV=ugeX#qrjg-lYw!i)g=KVQF@otGBetZX)i zt!arxSw*cy&7fH zfg#IUEmt0$22pWf4O$i_lTl^G(_)sh9TI$suE_tlvdn9tu?n~N>Kt5!(OhH;T z%iqK} z8SlPM<*VViofLv>j-)B7$+1g~A(XfZf4F@-18;zP|7*JKh2Q+(LH`s`?qG=S>DO|{ zWT1}QW0SXnqXTA-snN#HkZc9-Ue%Y@O^EZ9DQ?&GpWRJOHYw(0_KJ6o@z0dXrj37v z09Aw(RZD8K1StL@B;{pH4|C#K-_%wb1?L{u36Nd_Bx3#u*h{3SN2Ab3H;*jou+Wf; zDQAzm{Z)bRlS^>}-1Aw*5n zxN_(GEw%WU?2WjP6Wus|Hd>Q@fdF<*{}}lDXw(SlLx|y|1X%DHOo>UU)V{RzpJ``SP~*2QW;#*vP~ zOfbb}`r^lf{mn;*m}}SEQQS+T1<>TuM|)c8p{pGBQe*V|4p@Utwjf|f>D`0`S?~r! zcC{LN{%8iw$NN)x`XlCf0>{{ZPH`>kzQvvLEc>I;3CC;0w44evP+hHm{T9rI4fuQa zgr^$&j?mY)+BtGMYW1HYRJo#b6g^a-&KHvAaUi!yB9?a8`nFP!9JI^io~3=JTk6>C zGXDcDEPnV8|FMO_BL1*%shv*K^&L&H<~})}7xFt)Z8E`0RHZw6CFuP(c*vK@$%jH^ z0ABr9w~~BRY@UPHV!BY%%s_GPiYQz1TbqW{sN_{MgIKDW!zVBM$kWNxjd9$c5DX=1 zG&7t)Rf(RhXVS1gQZM|*EqHp!D+L1QG&SfvkFyO|?J=(Ex4M7%Fx0+{Z#uud*HhF3 zuiUxcHx*noTF(D#*Ph(7_dt2SZFfbeBHH-`M9hyAsVaWMV@j4~VjS0!u4!&Db~12o zWW+xi!6mx&xob`{1dDvF<{iiD>fkeNeJ@pyviBdX8|s3!q%gG?Sq(8RS=VWWzpo-Y z$hQ5?TEe2H!*9y#kijQ?;Ck%emZ}Pqnfp88M!$>*@xwMr zT__N`2z%J|TO$FP0xTh{{Y6$4jDuRxJ8nc2>%H3f-H{TT`{$lN zLz{-O%}u)bITi21=2;jpf!(7lUx33>AzM;>n(`mR;RO&Hmh~YwrBHEG}hRe z{FPI4s9RZG%h{pXt0)haW`BJ3Yy2+0iD0VJx$Dxe?d39%FsvVfF$KNt!@AJPV&m72 z@T(kCu+EPeQy)KUe&O|LH_TUqDON{Xu-`)G?~@=Ojn3Zy{p83>UJQ|u3!|i0E9zs7 ztz>)rLw(f?y~z`isaJ5c{T#^LX|k_g6?)dY3qQ*^8SRgO|}76gyLICh2sZ8q+yv%5Jw) zamDR^>Msl3M$5{JSGLjXaIB^DTu00MK&;3fyP~5NZ?;zi#~!1NRP#P)g+(=(G-UyF z3RD=!ycDj43<5yGz>M7g4ww~P&h}IIMx>gij`(^iF#i)%3tIFs#7?~YFg$*FfjMXR zt_0l<{`H!Z!UJ1eUeWNqP6W#Y;HR7+5ErzY{G)Ar`Z6@Xp`*huj+eYiF4dQtSZG9( z`|>)-8PDC)M*aty9Eds@{h8eF=Xm;C!%zRHKi zjAfziq-XtENT~vt!;_PT_2-TrCkko33}O*V`xWaJA8~YT{%JK}sC2(7opXWTFjj;F zFvZm}~Tbv`U?HSWAImQkDLS}Oc;!biHx zSWFf#H$XCvSkA+ryG8pve>8{$KVDSFGq{g)6No$G)oIWy+uNTkqeyo%vMwl-B#V^Io3Fze~ zdirE|pYtfn`1LqP@5ko(%d1+GvTq3gvQf;k*}trIn)xH^FN7Gl(X-~o0<<%7Xb;fx z2)+|fM!{93ma0WIHaTFyTU2Ssm%<-TS?ykX9twr}gl>bz%VA}`oMb zJ(~Kw{oyX`SXM3Few@|wp zK~ZFq*y{U}=-I`mz`xGCwKyK{lqZ?$H$`D*nRxZ+wu5n8cgYUvzQT;zgEm{I+fY6piLB>nmxK z+k~vk?6%&vG>!$}UTtuMhrC6a#DDhgrN#w7*C(HFIuxiReea_0?RJ?80@i{_>hgs5 z!8>fS@=0}EGiumrL1TC<*zLrOyn_N$3D&nSyzG^E=tl;85=p1Z_txG7+3l$R)}Wp#?X<;VF`Z>Mi69@yGGj@NvK=4z5e1r< zKrI@rx`|=XPDK|*^UC6@_ua))1x|)a{)xu%NrnB~iJZ%epPD*7d8D`K&^IVB8iK)T zCN&Q|r7S7hsZ3J+A3C0-!wvacnhe(~!h21A)QaLZH??WU^1YRmPAq_j_k;8cNGsX% zNle!hbQ9B7*#Q){WQGt_-}5+Wu9cndzW5Ht4W$*j+XlN~cHje~>@q5$2VaoaX3{hs0nB1TC-F0_e}rqFt}iR@!cvx0xgO45jOi+La6ly=cM%8{I7aBNVVu zTut|0&Hpyogsq^`X29N~5067yXY8H-2Mv~;5p@4^i6qfiJbIfLYjUqSU@b?D^!&IB zt*4Yc-Im|rieMQdd~v56FEp8&>i9VsUZx5rca&4=q$yjzg`P*XlSIhz5jcb@>S^B6 zfZ!8;Rdz>(*CD@guQA=63IIf=JBRsW?Dsp`#)|}nbZ~pa=%=Z}3;|cHm9);z?a1)X z(7_xmT;~VZ@j(mpr@RhUSJmP_lf|OL=uF975lB;zwQp_SQvkgx4CsBZGaneH#+Y;X zM*Y!za4&$7*Ug4BNL6QyD$9K3#E#i@y_m5PU`DIrmV-4A%dfl+>yH5R{ZcQJrQl^v zZLv_t4g~7p(pC+-g104-q)!Sy*DGSQXhJ?sxmT-IQtB;tAqDr_MX zT&pYU`jS}1z_Os97O;4?n?ZJyk{Hb;JeAN()xkgZU(=`XT~zIsSDT6n1-DOXzV6RY zGy?t)GsstJ=Dw0DSNaDZwAHcShGR3(uTr2qoHjkEAm*bk4-EM`=oJ4qU;K}Mu*d^7 zNTyGzd{d0v^2K0Awb|Z?dga#6Wj3L#{?&CwX90Dwq?Fh_p3e#1G!|uEf$%h-ko^w7 zFhCHqEOZ=|2#Mqb_dXkt)dkYs$oZ=u<1(LYHxT;j7NZbY1k0CIJEy+iUns((5k*xD zW;<1?dh4KHyxtjvJO?H-EXIEKIo*FTtRgkiZYSN{L<7$TgqafrViqaJbqj=bon|v^ z!miat4t`iPUQ2lVyvh|nq~>dFK^LVRh>dwLOtaxZrxHrbxuE5~3kuOpQwLuaDe6W0 zjoT7v$f_Ub*;1pFTzp?bLe9(ucH43OiVK;a)A84(Es|B_UV@UdCJcMdst`mm46$}q~u#(#Mm%{?o=uDdXD!1Us;%pVuSa+!{jfEA)3x}9ce?X@lSuk zbYG#lU2Vyg_EbJRS^AVhSJ4i?uBJS54IT9b*2Z%yS%={kkAuOpZaWj;#3XT#8~hjy zE+?lQ8Kys4=*`5#g9kPq_-}L`6ME8C52TxEIzu{{4}%9H^nK6L zQOIszAnz8aUniM5x6uG~gd`=Hs=!Ip00kp*lsBdk{2b5O2MnT%IE)I4!m zBuj?r{(ewmAeA#Fp7u1jtIh%< zSU8m{_h2uM`f6I zBAfo_yoIY=U|qfxp`w4Zd35zY*+lV=IuIEjW6$WHFfKA;_>BF^@E zw|5hbAG1Qlg9Wxsw7M=i`Fe7QTmQq71sq==GO#0pJ)uq}2GgPQ>!;Zg*2ajU=pTw09|h-*Sb}Scw(Gv%?Lpv?2?> zHVWA889yu&EV6z-HezFz_re{g_q#Hwz0i(c8SGy>B+>5cm1W9}b&FyePcfJ8z~&f4 z*EQfx8z{8z0OX+blD^cM5ydmOvgq*X*KT#ab8u?TLmIvO+kI!%L{lcA-`NdHKF+MZ z10+!1@x!V`N$Z{r-akJ`P_U9ZfNs2`%G9A|_d+(5WpBHGZ^U@0u$&u3NXkhykygL( z^S0gwDAi#uzTTF$dWCGgYh23Zsvy`H##SI)_J>ye7q|Ki@lum+^1-7ZrRojj4I$O% z^FF>g_)a$`t=ksq7ODHU-@46_IL);3yI(_% z-ml<%G2(4+l@oABgSH2T8oi;=g(Lx( z4cbi8n8ERamgEF?N-H~Z$GD2ThO^o~zTIKSO4PYKapp=qWaJ86zVebgwIqzyBu(BV zY{Ry+Oerp&J-{SYBGOr=7cPtj_^>adwO^wOYQ;<2hlNfiymb!xyBeO4VXO_N$MkmH zI3-Zp0o1)B!E6@0IwZ3wep_Quf|e@CN>2h=8*nHXQek_ z+_L1^prQYz%%P=!t%bTgO#&)LsP#PWcoSPlvu|=F!~CmWH-|4j4X?bFC9RfP86);d zVn&qR1<)Lk-K;x<$}u&G0>jm-DABZ!9ZgsXU>XoaPA^UBb(xRwqm`qCD z;Zv$t?!!QxXty$4YeJO{+6Rj_X{2;uU$;?~Ecj8H`@Q?+hP5I7r5mO3I| zi8Pb}!DLd^L_VL%H1{c_Xr*ITM7>S!`0O+DjMe>Md) zj`9Q3Yg-CcUk$-^h`DBy&d?G72$IsS-3@Tb22uh3hjLE*b}+& zd8MqG*`J{LhCK;!dP>9=@5 z4_2`ZXRQ4P)T={We}Q@e`+~1M8dYm@{}pCpJ61 zz%8UxT3%C7ZCQ>Pm6+&4b-B68n=XJX?K!`eTqK=b9r#A>Le@kn$p&keHKnSf8fqma zvc2G?XJobV`on|JI|ezXQC!X+B^5_J2w%t_*Wya-wJ;P}B!&cM%KkE3q z93^hap$1H|balTA{V98h3MEmRM@M!z?&!9JBzBF-ZW^(e{jcLYJ0GfbLSBEIlW_PpoqmR2F`X9|Uk%{hI+u0)hX|`F0-Opl z9j?P<6dIj?)~FAG-N!M{Ey44gJh6SC|~q7+)`9ChsjLAKpVFG;3eM5?YUtRuzHd`xwcvynI( z3k}Y7WSKG!q%mUhN7(IU#cLE|fZ3H$d=K#NFVqR;cP$_Os?8C7(@N^d40e)tTex=yJ zehKR@=zzjRsEqtla;2~oYeE5vMVFMvQ5Nt8)nCbXrNfc2cj4ei&-WF9!Y@VG9=_WcdPxRA#tPr3%S#k(YAY)J5xJ(FN z+cx?0z(i5FiP_rTX8&dnR+VulPM{BXMJ0kx_c6V&gX<&T1Vv0u8-7QX8PEel1#Cg# z;(fW&VE4G)YBx82cDAk3)J`EIrI%I^evF}L>K`JM7B7gaauM{L1vkTLarCgHAq z4hXXtp@I-HzEPP{O=^#ci24z@m*4a$2jTFS(WYe zi5ecgD4F7iN?9@+raieIX=9<;7w*wucP3IzEFl2)gmr&l5`(i-VyLp}xJ*f@hv7`9 z?X+tKh5f}ZKS4c*NL6OD8`=$jHg{&rK!|dO;CsaZET4~D$NB!R_Dh(Oe~MJ*qvuYE z6&WkZaqceI3iES7LYUlB;NfqMeddi4WuL7QDBY2_AN3$hs0Ji5`=2%=(}7=IV&yq( zZq498X-hdbN15{ZYhikZ?crMXG}2xKVTJYwYxY{#d0`<~)=h0NC-tZyhGH>kbV6ziJEj_1O>t0! zUj89#%0l5)m{cUcBr#fu?p0O%8$;e|i-7emDmxTK*!7z3WuQSKdC@iJt$wN-^y5w^ z#<~rWiQA3qzZ5RVT zc;*q?-uILDfhG_W-!BXV%$PC=F(5E{*aLQi8Ip;7Q%U<>if&J6?aiQZuC_uV_;F_~ zjq0_FKu(?;X2G)0T>{P~C^2=!glM$V)u9%kGnYD1d|LwA={GQW8Id+OO1x_JpmRSX=JkblKNOnO<{e%JNIR^L&wKM9Qeeda8uAScI?V$ed`j`%4^SM_3MhlDoR zB7y$;>%xkpN{RwE=1}aKY;p!Oa`5{$xFozftc$DX-FvDeAXI5D9%0LFihD=*o@-(? zlyp1-c?oEipKy$u7vU$8*=qbJWg;Y9^5!Rck@oSIeZnbA%h^%r#V>mllO~F~1PP1m zwwF8)FQ&r6fdz4c;bF_XL_s8c8)eKDhV2KEXh=(uaJZ66ghM3x zd$JKs&QolJD_Wa0>1~1jEOYcS^14rc33epJ4-#QD^Oj#^B1y!bfL``n>Po^ypv8M9~<4PgHnt8h4=ze*tU z@Vs~wT(}_a+H9-SRD(|FOEF+i(HcE-f68(3SJW_Bce8?65vaP8^PC+N#qJ!>1?mHU?#UWp`h6g{T$XUDUK4WEfKwqX_-?(gZ zhSz>Ra(yKmgvCvKNe}m6lB(8Ped}jb{B3{=j7MdS`n>bQFB$xPH2GtUt%7 z#=&^>qjOyun+5@d9H4GwN6e$cKA;e2ihs!Ss-L{2ln%x@>AHGldQ#u(oBFHJX8cKb z8rVenyAFBNYZvLISA>z`mJP&bwr#s|zg{=PsQq_43&EHa-7XNjcg7a_B(!c<>B7Ol zWADN)fFAqf#(d=AQ9dGo$i=&1@He%82m(NY_W8qH9Ka0@&!|e-Mk8L-fE8fmst6{d zjMjYXhf3&ezhmY-4Z~{eYopN7l#*W2yQg*Uglm2xP zp7qdtO|68h2enkcB=wO9lMPJ5uVanD3JmTU+=(G%f2%eYhoWnL1q(~OMq-p0V!56_ z)qU)>^0cu&yF-@0%-8Cy;8=Q0;G@r>&M7pvfXE1w#vb{A_`qMPe6O0`O++#U7M+&C z2jG}YBR+%YNFq5(k!7gxQ#$K6*v0ziX+V|URY2$yP|QF;hGhch3rJ*KF&+ro+$G%5 z#Ox*=cx-T#m7QdGa4SUSgC^tvz~7e-LNNv#SNveQ7lAp%W02|+boQHuQo^7&NozRE zY;xzQR}dM=%}TGeYUK)>t=6iTcr@x)JE7(atuUnBy9c^B-eoQJWjwcKCfAhkM zD&-$4`$IFK5>vO_)Y{i$4md2VNqCARv%FavRaS%c`nx{X<{cXbGA8>mXr4xOvS=`X zC*PF(`vY23+z~)6A?+Wyg<`855_cpp;x^@LCtB;ioKy3O)|H!U2Oy7kcm=keJms{e zZg45I0_8XA$Q6sC-I}D0P(-jZ_D#2$NASf_=LzCWa6gVUEgpxr$x|Brat^OI{72NMfQa%>Zz8 zEFK<@Ebn%Pj2s@ON($y!r!20BTgblhamn&KW)-|%uE@1R2S01SAB~fp>nQ~-2b(ts z^;EHeJ8sG;vfQ6!W_X^MiPr+_ON%=={2A(hSicH@&YU$$4n<7WT>zsE*DWDjc_|fO zhqb?hlGQ;R(4AUY*93`NmcTs!s%cNQQBVH-F1V_$o*csvgO;9j`;wftVaI~BZ891l zFW7HOfgy%nR>#IksRcEdw;oMtIxGEfCWF)y<21jNc_}({ezT~6PGa&6Z#_--Td`Lf zQYpxs66qoWBd@GWDkxS^e4%(%C$kEtVxu!V(&vXwY~lX>*{2lP@eQk;pq>DMV_g7Y z(nqa$<~6(0w!V;?zYk&?B#o%f+q8hPwCoLxB8}o<=;0VRYhx1_x>Pv9e&Fu!@Hn_Y zp}M++&nRdD1FkB8>Kuu~Z8%-kAR!$aGM*gM_egLGGj1}c73OVe|&(d1RbWf1I z;{K@`9_3Gmc)8cSo!@hlh{a6yBs!V@q>29A>Spm1U5<11$?D3#u((@QmNqTtZT=MprtGTTKe{ij*I1w>e}9}W3dRG(a$Q#g;3PSzAV5^h%( zVP;16`Yfvx*8@!k)tq8Od|inx(1hOJsS+`aDQk2zeUc7E14N3*-l6it;MH10)681_-%EhiOIprrx(IVExKy`f~Cb2$|@b#-JA`S-Q@vjwn@-P<7^j; z%~c!#p1FEls?W=^Ve*NZ9@n4IUZ=>)u{!1u+kbp-Q~NNtb~w|e?F1UGGIB#bKOu%y z*DWt@JXwG{<0Wh3?{7Nk?kOUtQb8U}x8WGAq3YQwUh-|D;-m1l>+#p8Q%a_Pq>1B) zu60+)FGb-|g}vvz;J=-?j^Os!P$tf#I4s()@MlG&>urjXFO|Kiv&KmcQN0DLRr^M( z{+M!-9OskwL^3GWvG*fetx=wbWaYjQ$v}be(VrZd{e0vy75$05^z~Qe-DD|SiAlI8 zjhC)g=s61L&P#O1!~xR}=aTcMH0`V3rY(|xPYTbX82|toT~$RngBZM;zuWBR=Y6JS zPTR3O6*p#ND@GH|D%!M0MTXYFJJam*a?&T#2JfS=u2JUE#$Yw}A7&N&jTqjeguY&J zd$?(oTTld_$h|E#keECCKDc=G-#Hsg51;3ss)2tSKe1zXd97&wr+7CpA{Zwie*e4= z1u&7vw)p+6VdH>sMS=BgW9vFP_-MiL-HAQ9@9mT`16C4$bJKM@Nxdw>9~78hVKP>O zFg#M-eJzdxI=)ZKtUTsge?j!^3xT~q*CAp8{4^sefo}qMuYo?0=lx=6e3#Jl$DV<~ zfR7dJUv46Al$Qm*Q0d2IYq_p|XiRQ}L>Nk5H#OuE76 zRqX1=N-J}m4>b$V;9bNZs=8ivMfE2g9+x{PaZMPU+m1Zg z{t=Y+1-GH6TJ|+`OWd&~{TE$^M-Jcb|5^&o<){Gd(LG0B&Z5?SDt&l{9q!^j-lg&7 zy?!-?y*bXv%WHqVp#(3w#fCBz8=V<3eMddrr>e-;K`w1-{;SSO&zKV3M|YfVRB6!r z<}n@N@2Vj7v#qrwYk6!rbK)C8$OE%*V~}^?-YaF=<%^*R^HzIPMe7(NY0}G>;)TMxBhwjNjXF~wGlGOt+$i^DS~qu5yhrM_~^1SFUsnP=LG z<@q&Z^+lDsIcf4W)XYU+QkgDDvV+cxM(kRN+rC|gR+2H7Akh4rf%Sat)6-F{FXoX6 zxB|gP22)lDQU9p&s3M{FCDH8W;H6~UE-&5C1;xkkxEeyJIhsFlR>f1#??A;{94(!= zxh+_q(tGH(?(Q+xQ`zi#$j(^j}y#RTX*K)!K}M`B^jF_+(MQ zGDXT1oX6R4ngj7YH1$U(a@hSCJ-+=r%OHqs)qbU?_bl@En?}z6)7n=>#o07r4uiY9 z4GL2(`ah~InuS{BzC?cxw*4cfR=M+fKZDkt?TI|0l@1oKHSMm@ISw%pQX ziQl>7HA1V=q{Rtc%Kwr$Nq^h>N6FnFJS(vxDQ4);yS9hm?N@uu8TZRy?^H6Nh7cit zB&aYkwkvHixSVubP0qE49Y&_=3`Yg^BsvV;I3wv3UtTA?eKcSIHS~7_QF%eft()~y zBOjQ@M$6w6W7bzm<7UQlm{VW-)cTD@LGW<(ig&%m9Rfkrv`)&A8>|cgc0xz=q=WZs z%uJs51|y#%8dtNEbDP{|wsjPD5)7gNBn<(IN8qDMp-2&5U}sY*Vx)ni zVw#xqER6TXdxw`%s_9;fW>ZqAuxh#n2LUcJ0v63>-3Q}5;_b3m;X?}(Hi)+GG0I5x z;eCP#M|auvndRU0oc)%->q-5dWUhO`1iHo$BVqc+#u8(^vhTjqZ097ZVNM${hxX_N zm=X*-5sWQJXQ|HPR6Y5!5)T(E3gH76SIkTp+Amf+kwWoayD|r;LjA+e_)R})6dNXJ z>TSi-@`|G%oC4B|jT1Ox^c!SRIREW|w=g}T0}`VKuH$~r%?<5qm!GA(xfs$q++Ds< zdu}lmZuuixQM!JK56%s+7-X$upTW`KA~tqwWa5V{=`fx_DGEMN?V(rp=NK`eWj33m zcWM)D2dZj63U++(JZ50F`p;D{^$ymj>c>cC+<;&u_cHg>!D*p)Q zG=1x=2zVE-HbTSgimUYKw~;95{-{DoNlZU^Q*W+JUHYmE(gAQOBaQDsBg8h+34bTP zTfcL>qPc{sF^`Lahig;(fwGWJ>u`-)YZ`g`_}fcYV;9-cuK~1BboUB=Q@b^q9kp{A z%VklfX<{1R+I^k2v^^Sw19s< zUsQ&8Ur@FA&N&2ic*X2 zFg5Y4Y04T39`h5AmC0~vvDXnD*1`89Qp14Vw(R(f+Aagsd8;2VCs`!1hGW%&G=K8b z{>7qcBTaeLDabYIU(Aa7D%c6Nq@C@SPJS2^ob?OicYJ31istsBa!J~3Wr+Pl@RRRu zL8zF_!@vsu5DGXVxtXq;mY2y!la7?tJJ+YNG7bC+AI^(9$$|V0*XXqwHD|lkc4vEc zWAc{)lZlx~i8D3u=`Vii+fzxk#bY`1S0_{iZ=ym%hT891B*4K&CrbTWqk3Qw==Ww{ zC6?@?UXN%6*KMbeEiO6#Z!N)nSu{u2{XCaLV#-e*Im-`7({%U1dy1>P@RI;V2gbgq z4rL2&`7wQx4SvP;C7MnAfUZCwVLqJ(i3|WhD3Z=!0=+^9-zO&_WB;9nB>Q&Kd$Ck0 z>wQP}n%?|%Ou+}PI3=Tq1S6;#YU&%rqo1F(*~)0`tB$j9y4<~Et()8e?j5xA}q0TvekIE7X|#; zG^UVcRlg8@?n~DFvb1hdr$7dCqVe4eZqPF#V43Gap~WU$plc^yo$08t%CekzN^3h_ zjd-(exR|f`#U|$oYU*xU9KZb4j^)381Fo?GldFcn9N*OB8WaRR*uAj?@ib?K|Nj0A z=>)Oa4vp<@omq!Izp&OCbLD9IHZ}td^3G`RCCO+UXOLVVF^zfEv!Q{Dneb)M!ctec zg2Nqt!l$;>S@NtZU3E!VU$bde@6pxHtQP&++&xH~X`qM&63wPkU^j~Qdq}`ZI|2V`OxxH z!5@u$ZP`?Z>|rVb*r3S%QH}rJN>)maYV4LKd4fF!2zuqH@2u`}1cjR6ajc6G);xuF zh?pZMMARGe^bTr>*)<((`?`QRPJ!L$TOT1V@+O^n#PZIG|NDEs@4vTZtq56vafN1l z?Zbt0&zWgP%vcc9Z~*CI;%<3jH#a4!US&Q2%T2tMk&p%psaCucoPF7RD4PB>(cwA0oZUt0ClL{>V_UN?x_2q3EQLns40XB{5E2;c1(K~No-pdL8 zIf6YBU$2WTS5#KU)@&f%rQsDit?c(gJyc0b!Dcbvhy9u&k<_HteTdjJW?z3lb}#4N z`Tm200qBq(KQH89s#P0dKg3vd`7UAZZl`%N?1`HoVsND4VbO=D&nD4F%B(C8DspOMH}_i#ipG+oPHR1mbJ7Qy;-u~@)R3LYmm@r7kwIV70QD`4(?}C_#jo zBn-VYOr?gi4m=vx-phJzO@Bdx7kesXdNK5GxslY#=<_u*{M1r=!_oJMDR6twTz0oS zS*MEE9(EYx84LZL-;bU^;D#(p51Uz76S?YGWvITWfcp&^<#z%MZeDE|&*CV1^WI6~F!)ixk0O`@>kPSV{AViIH23@I9q?!F;!Hfi4BG zDO;ZlqwKrdl6Dn2HzE=0o8JckRIPk9^T;#deYt9;>woUvPXWw==Y!-JC~_^(i5wS3 zGSDj$SShbTPc5#m`V<3O!~lk zg}lwW+eV2>%vflNAu1Rz3yiCZW{r8R|L- zK<`2kf^S#qw7o@))*g?W>-LVbI=W$S2{fN{ch0^qevt*0#ygLrs9#VjK^>E2av#YW z^dtuq$8YP{ZsizXHwp8_2uz71s`vb1>`F1^-*HLwOD~LxE^EEfH?qnRr-QtY`@7QK z(_p+?6pwsoBRi-zgKEbahVqbq+9m*YD!5uGj|`RicREK&>6_)`++wS=RjJRB#XG0U zZkGE@g+ELF1jyvG-m&S$*||5ncVOXWjmFr_FJ@Di@l8daFe)NQU((T0j^XfVe5dDh zpe*qvj4b}P&T9OcU0b6LUOw4wkq1W?)0`K}mMU^PjoJlXsqsQ zx|#f()$^Klttdix2uUhZ31xOlnz5SFG*mb09#s*;99XGLxbtNM(n8`A>oztkpQ>CY z>9)vYP}LMy3va9Ojx1ff^)^ee&Y=1u%!Fj(85cf#QhsKsyqmMma?kjC~J6%14CF$gRAdJIR~2| z%hO7r$|0m{Vnq<+@UF7P2{lPZ)b9xb`E_hQl2PD9wD80?H9xvWsp-@qquJap^?^7$ zkZFN79>)Uv4^}Gm7?5g;IhH!RA9W7{%Qm}S@K^oQztaFxjFSu7RZ7~S`l5`F)+nAg=ay}8!B?aL z1<&sYNs@Fapu0i`otr|j>T*H!&!a9Rzs%RFzJg)N$ z$|LzSMYRjkgHVG=23>FxA7=89;ix5Pzk#&`G4(#D@?`M9P%3?drYv{q>gYTwnFxi@ zWujiZ;4MIRc_HgpwTlK){d+i^)wZh~%2PPMX_DC1;wh4&OF%X#fNY+qjFp!W@g;Fj zcvSTtonlXDQMjg2#Nrnx+f(W4x~J^UnAu}`KJp^OH&6bFloZp&0QIvME96ldJ2B0= z^n|6*7^0&*@?2x(`Aqa7p3+e3|M?eAIczxpy9a{GWH z9QQ(3%PF3D;U4%aXY-W-M3jeogG3(R?>_9=?Z4Z4kwzV`8D62>Wv{-(e#;(vcZJ#* z1;k?STK}ju242Z=lr-U1x(NH3#xpwk^WiL@;S^Q0)bkcZV95Bk>N1bjmTw>$mGOS( zB%v+lv8w#UG!MS8DRzOr5wrQoI7cNW{M`Ny*sCA+WNcHDOup9h=WUZY?q!8=DYgKf z)0?mVT=zQ?N8YxZYlCn_bn`Y{rEUpOs=g@t`{suTZLzuMYnkMj#l-qojohzSIJQ5Q zxttK6Kb`_}KB>!S&#on2q3RMqGs>|m^o458GL2_SwHw!_9VHIlW+@nA2uYtb$jcd#pR0#&ES>?qj$RJ zg-lq=57{o%(rLKd_5{e9_vjvL-wTk5U5?I=*9CQQpVgBXXD`Cpnh%jl+YCmf2L6$B z|Ad%ZR_EQ6DqQERerp=M!6``uJ4rT$JxA$7O+bfZKa`dlrw>xZttv|j+m+oxldb_% zOe}Z(3pFV%!&twS*)`Yo0iIh>x){P4Q1U_q*!t&)f*Vg+nhO0r6Bt=!!NP4jqvW6wIfMO613B;Ok92-5ApXTGOk-Kg z+6Q}2(`MT@<7?*TW9Sho=9wiex(ARd^26C?=Q%HFvi*AeazWutaUU^Ie ztdGZd#-ctd;L$;fcgv_U^!eD!c8HP3v_JnXv$!VMo^a~GM54y1{JV|?48O%?Y3Fm6 zQjFJq2`5Iv+ZOzK*1w{uNBEn>aC>N0bl7?bEK`5w46qRIGGx_SBJ<$v1`|<}?N|(b z$Yy+ky;Fj2+f0$X77rhfIUHp`odz8T55H0o@ci}^WNp`a(9A5=Z=HK>uSE3%jJ4Bz zo!i__cuQ2OA87L&)EBX`bofR~ss7*Ww8udrd^odp4hD|fR7e~AGIY*WBIH>Yz5js8 zPexv2dkMYm#ZO8|JsITVw+>Fyzm?L0;1<{0EgT~Yqn15u)7+o5j5hLTb{Di5?)g3< zg;GnSOcnsFUr1vWmE$*Y&C%73HCsq9b%uQ8f2N#(rDFvQxDaM?E~RNDAFRm*Wt-OUk{d~C-&mPjA-MX6A^gx@=G#8+R3$TIzw<% z!QRZ9K zlA&C0?l$}SH8GQ<*DH;mr&5pi+Kdp*6}KpK`>{WwO8+>sXL}g|IWnYG61uF|P`MzE zLpP0ni64{`l&oG^_1tsu8)&LSt(qSXP9RMapV^i|N-ja|E7IBwjdj4*plf#M{hXy) z-|E_(r*+E|!@ zQ`(ILH*Z&^m@DL;(6}9(M|YMDD2)CmVx1`o&*3v9bbk%w=;fmGIiGqYefopCTma86 zCD>A9F%IXCWx!hqZN@%_|GpF7YO&qVl5Me7Z|7~65?>{Y*8A;W*4LRS4d1mdiA?2= zu9Vn(1drGH_8FaQX1831c}Ffi^nt+W<7=gD0!1U@0YyGp+a|@Uiga$E!f;4Wx?!&c zvF(;Lj4Uatcj>LfR!6@qV+)xeqSa~I1ZnDBcqZFg(!sBDl!yKJ;ZMr!Yl|O@FR@Ii zi(DqSMUUE18#+SFVm2OQ0AFdBoX^>&LQOj`sz?pgXA|m;T>2zznS;IN82`(;9jy_q zui%=44&6kII5SXoPHV~p@^etqZxye#M0Dksa{b5P9#Vp1VV$Xcow@h5`$1>0^u;>= zs9C%`8zX@KPk5# zl(jD`FD4}CLG^fXRxHd%#(hVJUec88Rqn!A8st3h`LMP(gVy5KPxeBG`g!K6e+g?Q z_u-Y%#=v>39BzzCp+-^!$^0}`vxD6eaErK$F7X-yfUKw1gDd+@yf52HXsOLg_$i?4 zZ+a$!nwsD|#We(tI5&0HA?0qzN{`5LIts;mCgSO~#P}GQO$}nX$i__`D9+Dq=A{l5eZf2NY@=FSXh)>v|#zZe%iJi9UAB%jrxLr*6{RB*}6*?>F~ zMi;Knc0`4n?#K)IUeMPFH^f^RqSOnKX5yE51mzBEM-&YQ0AWuOurDP5M)#ekS8oiz z;bASZ!^Yt(`!F<8%M5uTa}?6G2qh~8Op0PQgXrdnM0I8qE;LU%EGMuWoesMOxki%C z)I`g5Ju}r}MaB%0x=Dkq@zzMD!1}idkEfri9*5Q9RE~E1I<4Xy;V(ilPjuwhM`J2AqBQ~jSP!@xsX=@k zES0wn{;wQTO`2Rd92oOliaNNV*PuGE5nlz3M$=Gy4UI1scUZ!+*70)$d>qG^-Si4aLuAlqAGge zloSHBE2LNozrr;Gd{(tZ=Roiw@81|La2}o5Lc)wUE88ozvE3SIF;ct~sYnN{_!5fp#HHMCz>)swov(zD_xp#;GTxHF$O3Oz zm$MPw?`Sb`;1@GoMyJcS-2+AixDC>zcU{$glFI*`l>P#Zr*Mp6rOits!%a+Gp!qY3 zW4<_2;)7a3KOtP{s##jm%;w#jmv>P`ZPCfegiF3#YjMY6d`8)_kO1o<#!$w<9@6EN zSQ%s2#?kw!|0>ie?|EiCx|zPAIhcGzqw`!k3ki6 z3H!!yjo7g^%czRs9^bbY1Qn=L39out*3}X_^BD%ylN>WxUk#>4HPF)%#Q4r7DFS!> zJrvn?icGZp2JHl_y$~^+@2lC?b==US>jvc~V)w24jB*UsG?v9j+i=cdO^}D&vp&x}=50@}jC%LaUPrZ@Rq7e^U_(E767wz{{WLHWm zweUyivF)bzZTu&C$mSm``A9v%fg`>09!%J>gANH{$}4Qu%gF0zd_I%?_s0T-!u(M4hw2SashI z5zncoZt96zIhv}`XXRLALeEg5w-RUV3*GBG;`G($Z&R-cu>7TeUPodGRU;|5 z1MSD8UeIbuaCJV(Qm^9=e^K5OgJtRk>56AP(xgdkXD2+rR!Bj4n8p}1Z;upZjNGd508Jj$Vc?YO){y2-;wlTPk*IjN;wo^jtZ# z_1-X_+)sw|2GSt;%+`hsO-u;KEhx7r1pCRwh52w?>zqK;k-VFm?#?kk^3RALgvooq z8~o=Uu%V|r(MCmZGIzM;h3vb;6_xm7>Ui*^D%N)G{puV{X)`TlVr61~YjRJ3Bg^_w>=r6`p6emegLJ0zD3 zhy3mOZ@&Ku7k)!`*4P27^5`ICT^RBQG9>^A{sEcFdHYuCIJOm46z>Kfb@}34y7FE3e)`$pyF;m6+7^&~^joy!>@9H{K z5lfn=*-kG?iCVMpdy?M|;TQNDE`@()6%^7k-n42qMY%L%1Q>x*>(a7op#(FrK(Nfo z#0PZ;bac(OjDY`B(G>hyZz7w~q4dqxh3*8=L=_1nej$5!9cS7_!-&>7cTR<8KbJ$D zy@tSgvNj|-dGq*}>bMtn(}syzbW6=8_qlrPrV~~N{%e95Wo0GT$Q-g^Sd9l=kQUZH z#nG5@(RKBIE?_~3gk!%Nyi4$+^y@O|Ea8+6T;@EMO2oeL<~rBq!}8F?lRqIHY05!* z%Ti|AMBP^WLe=cjda18x)9`1IN|wV)e}ZBVzatTjt{wxs({SF*P;t-tsh*)U_Mgwm zOB53cLejQa(6fGJzsDq$s(RmXll+@YRN8=nt!xek9Pr0;Azt;RV+Bh|jChnUXi_Mi zL~*_H9;7#9%!{VB+bB{ay7%Kl#6avLEB0(y1QV^OT9V+{Fza!g%m$TCk>%}Ucji;n zZt^#JH+(pHo8R8gch4l4blUV><&{l8ZIEmVp1GyLv>1~}m#OX8SxgeUmVbtx^C?}@ zIu<$>0$ZY->9LAap*J;u$o`i3V*d|86m=*2Kd%ULEvK^X(-L0)bbaD45VCda(uNbH z;HmiU{iMwzit7kjg|hO=1$WE4zBR=R>VCHtf3#ef8Wti3w@1y0?h_6il5>qFutB|~ zrga3uuvNW)o9opoo$e(0%Xy)=BElZ^6*9BeNjz{>E+4Bf{X8~r^1RP9_zY9woIeQl zW}qb)LOeVtd`>Q|uaW>2%qZte;9k1QzGZ#xM-=_dw`PQ=V&J#&9deeZorL`37nm0_ zKqmU^qU!QauQB~rEGm$kG4Xfd5~I(s$lYPUdTE{aV_ky&>DZJX`x0PwYrtfMtQ8xw z#nP!SV+y!wFqMCfi7{eg?5=wpV&GohfA~cH4*P_(j|tZm$-fSWh)1&Z1CE%DWPbQH zU4`tigf*n60_QIQ!_$tB*W*S}?SU|=;+$%ixtuFrg zM}7|E!WO`SGM4^UD4hT9)6y5Z0fg}snp69=6k*Iic%je=1JsN{l+-C15wEpMrAO#8 z$&?XPAr`Tc{CG_!$ZYj=*SYC}@Sj3ZwmEI@kioAVCB9T1gKS7$$a6txbJIXZ_≶-p#-p`->o!P>8()%XM>(^gYL-#HE zQs1|cyal>7ppI9SW4+5@9)c;_0Av|qFjby)qF>j;eCllzcIi{Z)(%3F%d5?f^adI` zn33>93)AvfjTZH~5T(K4AS{x%H69gY*>>pn@T&+LjHxA!iib{%D#mk%*;Yvg5e6x? zM|)Mr;Lrs1ff$?(Lm*m&J?iQ&mW}GlqiqnCgh&7h`oJK56F2Fj_b8MlH^?NkOaoYQ z62r-OK-vta7}uTT7O#Vce$YCsY8GZjv1h_Ur+27q34Ayl_;6;V)X{r1wCJtTim!WN z9TfrJG=V*v@qkm3RWtolThuoVUkZze!-?ORjpl zoCZ1ee7v86PC2M9Ha+t(-ue2l$xIm`#x}p?nod%|?UFjZj((~wvWH@V17Oi6i+Y&7Y^%UGGY?S7HWG)c9fu!#GLi6EIcISF9%6;zP9(~Z zp45S)g#-K*uBv_EuZ(N*{CCR#H$#P_r+qDVA#aa%h9FW(xSQma5ESjJG@8pRo+9&# zry=_M(vyg5`C95Pcbh-(m0}qCTF92ruW%(tgu7NtFwu4HtEpfrI&&d_efo3pd}ZSF=k>>b-r~(%R*pODB5^;clkWQmjiS#Xur_;e!swe)a)c6ZJ`Jix52A%vP6D z@NbZ#wgkml^~?tPRNFMbnw;;EP0tQo4h|t}a*y;fBWP%{!ae8P+@%;MvFC=7;SjFYlosihL20XrkI@Q8zj+%g_br9D@TD`k(rD(=?N== ziA=04vN&W^&l2ktQ+e=!8jw(1bQkRT3BYjOY8CjmjOa)402HnZFCGUB0y)d+giaiP z@F-Vjt%#_Stdyc4OXK73%+fRN&C9}q>imAQ~AD5 z@?NomQ?aa6&SLQgcKX)_exNDt15$a`jeplVGNs`$Ju(+WGN@9{KKOMYyN%b>>Jmpt zmaDf8;y(&l$$wxtRoyzKJA({*t0N*AT`Zd5Jg>N+Uem}uK{S@)(2_8+$8Mys+xMB-NXd}Yx%AUl>O~tuYs)y|8z$P zivvmCy*Kp7=8w?DOhu7`bpEFK!ck6t(Z}n^MF)8}t4vyR9Z57=AGS5vRP-`aIbPybtLbm1wX{@=PK^6_SdWmqU{ zjV~o`t?@6`Y~BJifWAy`XuZk5QhyArrsWZ66>{dEEUG#sVqBaaE6-9My|xSG3XQY2 z|AGBbXviP3We=pcQXf3oM;4=!fhI#Wf|KytgE}nuVaUImqDL;>syP1ek^=Wc1E=8f(jUTjCODdD zGWf2VGTZo2+$^$~@Cv4j>?YyH$N7b}XwSh^aSTtMb}jX5pYK~m)m_+^=U~=?0bwY* zCNDk+<1!m@2cm#|4nPG7LM}38qAq1g{Kgu&DrgPah93BFdu@V+ho#=8lqylUF3ImF zLSq~Bhyfn^M+0IMhe4!L_g1dO%%nz2sR@dPU76a5zr=|zh`z6wFp49ye~fwVNl7Cp zbwe8JOr|?3v_byUG>sLf50{${*xq9C@i=a572fu*$AG^Wvl}^IC@<;%Ouj$?BSS4k z+S#s?w&OEnM7iNrgcUP5hz|01_ftRS+?hO@d zh+D;Cno%@ND7ju(d?pX92A0SoX4Awc13ePmcK3~KWGA``S?Q&Ku>+i!rG+Z-6|Pgg zUv0o-=Q6Mmg)$s)mHZx!_N2_JzpN8b4WxD8~SOA1eGp?hri0H^zc%>rEbQc zue~vENGvUN8}lBBhf10U=A5+=}XUtIo4OYgkqbEe)%w=vU%AZfg7v$nN1?fleU0R+={Epuu>+4KuX8{P~TpuyP8 z+ zTV5<&EuVoBSpy*m@1VdUF1+dwM@>Z+=&7ol*%Kv3wg4Y${~OVw`}8=5?r7?8sw%B3 zbpL3Woof;qNUmWf$d^H11Jw0YSCf*xEn8sw3Yg0=z`{(3wFS$08mJGSAs(EW6h$MV zrVmlTtv)=99fxbtc2MZNnHA0v~O>rY&gUBXdPZM<}s?#$Id z!x`R2h#kbCHdg^$I07o-=g9Nyk8y#&c;%2br`k4 z*PMqJcbbo`xiNqon#($TSY#?X8R)`l7ta@qC~29kluoajdl`K~i@t`gZ-o6|DwRqE z>CB}lz6`%9fgR9-cj(xlyDFD&;%||h`8crZ$UkXX$m1Kj4|-2=JxMHV2kjTT6Ct}H zk#UI5Gf3~;D#;lG@8($a(4a7tLjZy4iMKJ(gjnE|o0_X`zk0(e&!k8G1{WV+w1PU2 z-&$}6zaz=8lDqfwnwn7CWU_e_?wJINe%|pCGKktlqg%Rg;DfRU_QYx2ud}t;%N~$g z`vB7qDx2YKeEX|2d=nwait!maHEp!AwW_LkzR)z1HzE7U*L_jIk5_c{i2^pnXa$II z^4r5_W**6AK50xfNAX5J#*4C|dGs_S2}pHhXI#l{n18s9&`*Ty5qX^d|KhP)oWLo1 z2ZsKJ@Zl=CAZ1%`hL_lqvWU-~=G~Y0f|(CJA5=q$w5GlWaGn+NzWg!0EiPw&x<@RW z@k#bMlD8JHN&FDqG&NR3)~LQ5tt`$7F)iYjx;^h6jb1HP=Bc}A-;*#Nx}TYR`gC2y zqxt=-Mqr@b)0h<+HQ>i?Vt&D5TaPMEzWm890IckrG|NAJu~+wk0` zv3fL%_?#6EcoRVNms%)@)r9HH>uCVdO{9((7}2?eCeB{osQvdEhGlQMLdQ`1A*mfOM{NjTZM8-$|a+R`UDX>SZ zKAE%KoLZ2MZ2Ri9>)o)`QN7}zJ>V(FO#up*@ITR8j5yYDo2Jj8-!ECA@1}0OvbRyx z-XB=2S1ruNa8)}U`*m{WeP~A|FUwgn!976k8W^@3)j2rm)D2B9Et$C22$Thwy$G=B z2!U;v@hAWx6y8=oobF|>t+uJQ1Ar>~|BUN8$vZl^yrt51f=$!@ zeRR;~HN9KAYq{$I@+bj@UeT8cPo!y$sdntL3{@OC7f$Hh=EG-C z(iG$&a2wHBMSvR;{s;V53TzEKWA4$c}Nd4?~@gm*^a0GT_{8SqJ1vpHHgSG%4 zzqkEwx${Dv$XUW|I?r8!rsiI>#4Gis?0L;Lo02X8PzCCoYnKy?ecV%vELbKG51gcX z=v7~Stwhfh|D4&Km27~=<`MI=npv(vtt*4h>r3Nd*>mX?vrtcpF%W_NJnPo0(j7J# zEG0oH60PFax>jhj;|1n^C$3R-3u)hiG#d}R-&XK9cI1aPXX|#$sa^A&+q<0uia6Ys z!W7qgW$_(QDsFA#<4XB~hB#-7&Ne$O|9hqCLguAh2Xp^!_rB*+RFUOIQ=ib;VIaub z^v}b)9&~<_lj9fChZ!mXN^C(_SnkMofQ=2Vi3N1l$MM}OO(JDfw9ARSPfy>~F|ZRK zoPPjZyJkQL9a$rwD+HJH;e9zkD%O%FFPp}x(-=w1=929AJ;z*n>)5fekD=v0HRBn& zK*NXD->TI=fPm@02sjRYCVzN#8=Y$Y6E+RjV&RH-eR2w<$U|RA`j+^VZq7pnYla*h zPxN?!g|VKoT0xJzSzZ3|(CS#{P}w??*EiY~ETGS%?n$sb9@({@B>RZj(Uz^Mk;6nU zI)gi! zI;PzW$ndpph8?kN#)@$d=7?cY;NEGOYHS2a&fm(Ju(vvxgGP~jUY$@lBoiA`%~J`~ z3wYQ`AJ%8?Z(_w z93e;{V%XK5r`9iH>Nb45Zxja#CjbG#NZ$WnzqAIPk)38dK#0%bMIaD}*+pK@#oWxr zLJa0)0sH~+@$m6+^73->h-&ljhyg!>d>sG#CmyIx@P7^XVsHM*()0g4V4a{<78s!G zrLE_pY32@baI&}jWNQI&@pQ0&d~$Gsfj}PV>t{F^j_+T-98ez|db_C!LT19|f?*Rg z5i2K7oc&|Z zmCVh}B$G@gC&_OTtF9`ChD?kM007Vwcqf+NDEy#f>b=^ zgJuAhw7O{qwTUpJW;xARSc-(-WuH2>1SVAGCboJCy-r?fOjro6q#v5S^~|j3zu=!mqO1qy_nfEG;!2#EfYj$tn@ z7!tAgmm-Mqcy4)LGCSQgZ*XM48)cq>b!9QU8#%x%0ZKbW8OS!=R}JH+tVc)`-yz;E z3Ke_40>;O06mYXl3ez1`??wS?2Dcf8t!Vua5w-~;T{8j(z-Uu#SoQG1G7s9@R*@p* z!ws1MHFGLtJ=TI(-*d+uzkAPb{h@&jcV5S5gWQ6heIUkCYNz=2A?U+6T>+Rmv2-UomO%hhWac4f{ zISs*f6m4klIXUO|jw43_5JB4Rj3g36)fi(l1w`UocIKUduhB%(D2xad*1S$MPSi~# z*5u`K&%WwtS!xqUL>WijNC4ddpn6Sca0N~N>kHzZs;`OjO8xtutZmBWd;|CRsD3GRd4Qc}F5bo<7xOZds-Gm9hkpru=&VG4pn z+{QrWc5W_$Mz)zWf6WQ;ia$f3aYq#-X|Qs#UyIj#I)V_(HYVYB~qXi?EQ zmPmacp)!B)lN2|dvK<9t{cM@fRz558%@`5^o!NzNx8^r_@#_X-NRtjd95yg?dJn4) zkRbIkZ|hEK^PpSgk7AbRf3efI8F%WLEwegnr9S}hRtv;}63^2+%@56v+nC5K`6$D| zfLnj5+GgxW6#2`tU8~gjk20!?;_c#_$Zpm$F0;<=6#L5fF2y z$xFk8_GE1KE&vut3?Gl!Xi05udkkpm=^4(b zD|X%$B!Fqa>m~AE-<*S`Po`MhIi05vPXYg+)?3~6%{K$|rOA1*9Hn&|?-gP6G=esa z=$oh*Yld&z+P*#g3PkxMuHR4^<67jYS9`Md_FXQPnHlf~2w1y~X^&AkY`^eslb%BM z`}0GioH}S&J-wUql%!Jr>~{a>mibO6R^mbte-P*5I7Vz(TjKfgAO~u`ushi$0z4cq zN8H*4+v90NvMU6|n@5l0CfbZ4VU^|*p`)CtR{~7KPrd5A}I=$l*6$hMf;Nf)~;(H z=2ySQ4r|mnB!5R&Pk@Jr*}A`b>`RLWNKNipvCvyL9G}&&U_dnD!(vG zFbu>orX%t+WF?eF0iRbVBssQx-Q%uguD;R#Y{yF9*eH5-JW-4ChpBo`^H+En>80^4 zy`3zmH^N)auy@hZZA%~jNCO5~G)^ylE9o+FU!hlnxZ_=Ts29B3@+&C&tBd3M%L!+i zwmyz^V|FARYMqL3gFzjMiOlLt3W7_e$wSFC63_s+B$}>}V+;|Y4u*XqE^SH{^oH7r z7n82}b%J!MLerSn!n`RTL_M!fDFZ;+;MG61e25E;=J@0gc@n=Hf_K+U6nC74&jBzY zLRf0whwL9}y2FhVZT=86TaRcAfMQVBd`sW*N#%( zI_ZA%+ZKbBGjSd#^5!5!-Hf=U#5NOs{$=U&t>TMMm@4MuV?!^}U!=q)5L;-ta}Hlu zqT%u0$i14m%h1`8BtHVeh}0p)TUSHy!4AyBOysk9Yv`5N$Q0T=%q{8{#l{n#Ba=MK zC~xMYHyZl%^KQIBCvz7~W{ada=@r2(ekosU%05KD8p<=!lCMX^ea%J6ZpjUX<5gR5 zzgw`!H#_Cl{B1)puR8s!dN0y#@|+_}^I{cS^E@iDUJk7y#AntjYD7z?Hfzs_yblcQaC9vD>zBzVy&4yT***U6A2n%y6u*GH zsa;4%s$cMf;hhwTlfevn9|f~FgI;XAz;`pHSweBbXu$7SaBXCZy=sz~9F`qu^bozs z^vsrhERZJ&_Q$YA6qt32S>6U8Wi#VNZM!3VU_Y*Ci$T=%?V#n0AIQAh|=; zUv%-eiDTCzC-yB(>!#yeZy&9idfE&boulpXQX_U>{EJ191AdzhZnt;24To3?xHDZ8 zU4=(Qxj2Dn&ynVww5*CNi$5{dLXq=7VI3t%-0sxBlIz=zqMjQAU!&Ayb6E$^2jvnm za%jrrNm|Eb?1skU{0PTG+ZEw@Vrc8%4$CQOpq60bLoq_Cu(CsvKL5Ih{-x4XHrDgq zKe(xD~rb!Gtx}-Ng@d3TAI)t zA!t+eBVHhBr|Rz|r)*~OZjJX8uFFpJ)W&*q969J{o_6~I*~aBkocMoc2+8jJOR0Nh z6!uvVjKaJIAix~*{_F#$gB75b&bBU-h|Nu#)(w@{+Ge2K{z2T!&DJqs7U(LVLFu$I zpMQ1vk2h7%0LCpJPNQ{FuQP=v4saD+_cakMK2vF;n5rQD|(I}li zCc{SrJE`8sjD7!fsBIdzSpa_uC$kDS!>E2k_HjstVSn1R)4{iXdV^U{o&7N`Gd* z)ss7@(qU+Qi;LJg3N=>dt`DLs}pJ4@O;cN*WY6?QExKm4w>+c+s^di@* zisWN%%qbiN`^@{UDH7RZ$o4UggFTxrb@R(vkD#A0iheiB-l&e`!-bE)?0|pv;oG9y zsfpxbCX~fBkjO&Y*KqVfp$Y_K*dt128+=qITOvl@53aMJgjRuMaGR>`amd1r67vo) zPVAgp+4Yl4`N!O;J+3<>K! zTI-%E?6t}J6sW!JAfGa8Mif7yTM{w2Lr;1G!GU6lFLK(}cVkOf!#x%!$FTVUuSQC$ z&I4d;2Y<=(Yok``jLYRiY6&%gax}nTz*wVrlmU+3oMN;DcN^NSea;#`ibUCNF+t2I z2K-wkNT8&g5#`BK9hRQA91Z1If(?e0=9jz9XWtcNv4Rx+J;^9h%^4@9Y=t}=nVp=E zwrXIskqkx;j8>tVa|q@*$_#7!-2*MsQ1`td_honV(?}D4m4Ij|LWJ$8?i)yQfZh9E zDb0HT&hZ-~w(I-uzJR~;z*0e!Gl0_sQ_haZ@3s`_^>d`9{5;tOwZG@~&xnqc-V(Pe z-o8P*v@M()yH~rufFn*8+Od!wg$4-lN?H8spxDMTuGRyDH)Ck6c zutC}$2U8Nf<~}lrPo~iJ7S+vY#^@uhOe8^-#)^Q6WI(>Hq?so{Ot3kaEkhF* z$o(W+{Oa=pbTw7JU!tBd282N{B33x$3uC}q+x?lVM*>|Sv3k}Myi(Sq1&irUsu1R; z1zd8vF1yeiv#icf!vc0ZxgERaei)7>Q59xtPe?JW|1O5h2K|w>Q`E0uhN=feu}KAv zPAs?L(n?s5Y&gID<&;%atxl6-4&!V?A2d!uLW^CKe%;#`{MJ!Q4L4wsw`hW{f4_{Q4?e_cYZqP~b%$ zQxDRLT*zRYy$=Ybh6jyKWHJ~U$_Pz2!QxCLj%x5DAFz&?6JvD!5yxVbBVGu~){I0cSG!&{XC|inH?|KZ~dw9R~9?2p2(= z+j;o(!^Z5}Oa*X|Rw<}tpjx~=V35nMV+{dU*Bw&l%jx>E?o@4$;G?8ck z@TA~W7+cB;6Rl=^z9MD;#-jvTU$PX!*BeuZK8AJ)ZhkcKvIf4FqMyDL!3ZO8+5YOe zxWDO6PX;SoHo?ehMcf|J;Co-f17%|OpccB`BxI|HDxfA3h}Y13p@q-OX*)}J7P3tX z2L8cc4yO;S?%pv~KqcY|O6-ur)~LMJ$*WSI1Z*8^(|wMK53F7LWFwhw_}QktS!kk? znrYFgywvb%6hQwCHsEypKJU3Fmh3EQVkoqiqo|8 z{gNX#V5>Qs;d3nv>`_tFJ&I;X;IAKctMR5{=cYiaoPdp;cRo_RM&F~8Z5lao7|Vyr zRGBZCVt`HgV1%QvV*_#m_9Ab#BMqzZT}&%4ue=Ksunv&VY9h&E^v5s9K9$e%xx~PA zG?p7wohX;FezOXR9(*rOOdTi0fPrf5maRgP0mHgccKp%DZu1+3AdjKn;tWP?lN`9xvgcV?SW6LXuGy;}E8ZNvn6=GXJGF zoAV@%AGx^DJLt{a(cn1e&rtYball~ea{8;ylo66EBkT4(?Hk>dFul0lbko?yp(Y6W zfno$-0jdpKP08{$W_!$$AmzL%5DuM$KhvS)ib1B&f^HZY<*7K-vyp1_B-E?(%qo$GhA=^Ati<%42N9kt0`&|bu6_3|@xO0N8O zeQ&JXPkNv)_R~Y?-+3?tf5BjxXtYE!v@7kmmR>YBSO}XGze2@6TPPJa z!&tqUaY7P0iAba|Db|#RliqYmbTJT_eNuW)>Iyw^zeYiPB#!#~7D~)-j1}rBwQ08< zuo+&1JKeAcT%XT-528BVCuJbX4@#o1@94ZzJ4E?UPG1ow**?2J^pzbkpS>0LgKJuA zE-k%z{szwiPrJVDT-CPG947WgIMFLP_w%#-hyp5f_CN?=NtX6{oAykHoTx!Dqbh6M zzY+k4!ge$qlXwfL#Ni_c-2Su|kx-`aqIyZ7wxk8TThrIe()t>n%r+D?geL%AOk}YF zc1xInwN0n%n#G){&*)(O0d?ZYg}opQj3xs_(p~H?$4_7H;>BoFSzHBvJnA}zzcN56 zUzjP9rAT0{*dOP#g8`5_1WsBY%@e-1q@o3wXTx7(oYE2&e=`=eU*1Ce^7NY6K6tD^ zc+?-TNuj6Rx{G*<+^8&hx%e(-dHCapNh=9whV{uE4-j!G8E{fewZg<}7DUmW(U1O38f~NV3W=9!t;?NJCn)NVdF@vZD4m)Cn6xkG?7GYc&c# z_ylGL-*M`o*?-P&k!EHWu)cGM5$ORFosj1vOPO$O##nD_{DBE5%^T-00{B>STZXF^ z#QmyEv&s#d7lF6wp0;V?O)7bSC6uH&DSu>mhPuHES)S8SaX3Ry zJmw|lQdL67uq4F7ny`{)T#uwNH~PZLkYKpL2ZkLuRD`#H@iiA+h2N#c_h&vFLtJ4r zsLxd}JrgM(lap7eMGOO%F^!6t)1B`?aa1{miZ$E-9$V33l1iM?xZy7|*u%=7rsyqd zz*}^%B)?^@;)hT|y9VWGf5qR2_@r(ry&YHqEy3A?LG~tnocZMqMzw;lbw3CmoYZY3 zx4ehHa|9gor3by#QGNfkm+jc|jZP+BDskg^0|uyafIhW_Wy&h}i<1E-7NLslPt7z2 zg)h!vGF-MmMK|+HNhCK-lXWiLAspW@O5~x!o8T1Z-aWOiz+B%;KV)Lq6Ap!rBZ+!x z|MTWXjWCbJLoyVhpab)sBc)a#(nDRVa}yXB++Ek4KqBh)vll8`?Gpm7jayKixxDaV z_gUHuuw?!GiG7;SoW|rk1oo!ebCDVa)5R@uUe==f9JyssS?{ zrX&h|^nJYMjN+o;OP*%D)6|(I($aCh&TbxllrlWRFBDvR{NESCL7jsZ54gT)FIW&U zNFmq+_&kUJR>QoJh==7fkog8PQ0RL?eB8TDMCW``h-`N$=t}~QUDG zD3sWhjynm>4&|tV*@PBXGO|*W6>F!Sp#yo==Lf6>ND3+VO@Hi3%ZTaqhyfdso%^rS z5y^8m{Z2_Ot!$d91KOr8@94s+(@rXWY{G7RvcDTIGg5Luen)%Dwz$_8%tuOkey+Vt zAQbk)#*xD?pjqh;`2B5^%DlB2@)*{@*NPmF3|s|o$bYU7sS+8m*Vm1Eq`%(hXFyy? z7^QH{c`3`=h0pc`v)OnSymT-@D!hpQjd(2{A?tic-RosKIz85Yu#?4@NW$Obf9&iw zVt561;3>>1EJTtW!1&O?B~F>BuG-I-ZVlQVA%S;dUnk*k(%Oi53PuY^{&Dvs z6O)y7Q16h?B}trMyyV-`m~UG{Iw{=x&THu|gFeQ57?djUz9L5ha2U?W32FLfwcp4C4kG#$q+y)x zJ-dEh;t|Gm6;s`X4k%$&p}o?@U`WY~MGjI8YM0#;uT1e4;!7|vRN2oV{gVNoyA*hP zc-{O-j}MHGA8v--fFZuo!X2eGjCO|BP|MeH8{zYk6I*6*kYXrP3{zej2tCcCU>c%o zms9j2<@XzQ=vtP+dBV{yk+>x@S_wtADvFn&DUDfOW!#yNvW0eA1J4MSy|YqZ<&~F& zjf89gP>}sn-ym?a2H>90<{4)kqTZ`afBAm~BKb3AUM%06%BMLC2wFEF|AXHQI(vc1 zGTI)3Il$C&mZ-dVg#s&tCG4$G2Q>bg87T7eYI^>g5K2=O0wVQK0-wTJjD81ePFbNA z$)*wtcz}>sf{!C%s!a$}{-6cqxruM9Ab<9ZrW}m! zzlmUs$3}M00*N2aR`^Re5uj~g)4y`xZMk*9H{U$FPhCvq)z)N@zs%|;_yu?li%{B) zanB>nK{iX+>EMFKA?NHS>WYhWCXg}?b%XqA2^s8^UkfK0uMJ z11a;$?D$!Gcd#VLt~9+Z5cXsHW;~_YqyxGwlI?Aj(_5!QhedzTBg@7GNo8SDSwf7b z(bQWk7&Z;ZDg6Vn#Go~#42T9>#tsne(7?AQnn>5iS^t$3eh$tsU1pQB_V3tN;K|{ zh;OZ>%sU}s6-;L~QDokyJ``c~?GxzUctk$WK8o``%esS|S#k#qzz#_zkIwrZ)@ z+9==yqYdX?^#&&owlTNEL;19sHKa4_bZ?vOXchleSweq#`#acrD`fdCs8Z^ws8RQ+gE+PQL8P!rImX)66aw8k3-IKfY6mBrWDniPv;Ki~a#KbRsOk{W<^Fb$?`L(Kk^KAY8 z-@fS~2f+n`Cfz@Ba8bFX=|JRDHmUgvQepgFEvx|1fh}#j@bi@JUBkylBjG0pWQ}{6 zSn zRitYiU4s)wzU>WHhQ8wvkW1H};=^+Lul-f#zBn;-!?15HnOoN670GT2Wwi|kkAD+N znzk+1W8bvtdNmZ-ELfUMNnMOV0!)N8VAab9&PwuYl_An6ksF@b)_Y&%1)<8Ojpz-D z6Oht$O>hrqd>u~yn95L-yA`){BhNk`ad@3S&W#MVL=5~1lc~oEl6n}E4>D5ESABD$ z>SgoAnbJUcdx;J!S~>7?j8<$}S`9X^6`u|>4TCDHgr44kI1VAf3ETp}s|r?YvU`>) zSJB6EFxnwLco#GQArQYAH#~fWzf7N$QVTU>?y>q6eHZEA^Wy2Y7y=T()$#$YR+k7K z3ntAR8X*$>hQ2(IP(@wZpr4UKNdmAFB;vVKG4DdPHX04*JTq&bjuo`tXADGqGMXg>TUuD`P&)zfR)zdeTR_I$2 zKC;M=iXtSlG{acRE3JH%QbhV>a^sJ-<+~7iXT&{}$h{j9S4K#BXDPAiq*jt;CCC4s zjuO-_MQ?fNd1-RN_s-`jfVtHCT&09jgK`oNcepwmN8MzDQZM&ST|Z_hP8bmV`@v$x zXi%y&FqQ~B*x%tX7$OqG{ZxJ(Z`KQYj0uDV7aS}&S*Y6;0$v667xdaRX8~sCuY$vE z@{6G?S)H=*f^2c192?R#eS9ydf!U!Wu`XJMSj)^lo`0tT(XUT)=wWuLLF`_*5P@P!)GENNp?)nx3h3^y zzwU`)+F%e#=P)M0+5Iduf-Sf8->^(}6k$649AzKiki-bdEjOeyl*wvY)uB<-d@IO@ zvWMI^jCKSM&Sk>0_;k4zyQj$KUeRoq#ZLT}`qsu^t&*`-73R^Z=FV<&asCb3$O>}5 zFa^l91>&+Q%%S3!+|IjJ-L+mK^zF1@3EMSA9k&n0&o8zPhGhNL!vM&(Xg3Py^)2X= zYQRWRHQfVh=jo;Zk1~l!F-XP9lh*JzI&F0_2{Hj7I01I+*nfrDtgOaQ^m7c1J^FCi z>Wr#-=W1tcbFqTU@7{A+r!?Ae`&Ryi@HfNud8He0q+D3yc5T!T^SBqbGCe)o=$XT- zeV%uDKi1nG+Wb`owqX1}Z55{(aA;@-h2A_)8`?j5ZbG|Uq+;FIkf~rvb)=)_vJqGN zXt;vWjGH$ZdF!~S6c&Ly<>o!X9rHx)SV_%9`6F&=Y{opRWR$;U=#^zD+c-f+72 zbkm1Q8ZKXLClv2`$l>hv1Y^tpwBEjWb9^Bo9eyY5U%XwKGD|Jw=pkWSG^Y#wLHRw4 zwv%fn??&IkG^fp93Vk5#h89kN{VI8eeFFQI=&L~X=BmloyU6^wo!k<8)X6QvGez&{)g$b{P%H5lP=>9jzAHj^LV zx0)?@ln88Z-q_6BZ?-GSmyf(-o%?Xs0LWRFGWs38=qOpFc9Zlusj8%@@pqFJ5pC!d z%r)K#DP8h7bzJ6Q=~j{)6o-LsPGyrra=3bZK0y3RjUE1XnA-zvvsbxMs(?j)43kEW z(zgj2f6EV56oq;lmc9*8W_4M$zak4i7QRHKuh!#=X3QP^{A#lW&XXygH5v?8>Leq9 zlA`U#(*1kq>wREqeHa`=mIVL$Nd@gMrcgCcH?uxUsSRnv20PsA;&!&?z|nx9&`F8b z$@LPn&+P!2{Ksn*?`E8 zcu92(Pb6O8yr7TrAAVZ4Bs=#&(dD!jac*k5dQ*r6m_ipq6Mks3fE#EV3%G?1T~AP} zI!5eH06XoTAdZXj?xx(g;2Hco3Y3dqdT~DJ7t8^9VO z-|@y7fQ@vg;=|r;7Mlbq`X`UohnDz_(X^d6BDu;~8*aJjeq%+?13ooMEtVIIZE>V` zr5&qnd0&`q=SZK--oTcPeU)cKN4VXxm}>s6hP$9!&-F!i6Ld&6nm_;R&tSwi(=**n z>gW6XxmTWkj5wF*&S(o{m8zY}1rNSPr6((8u|US!N+hRYAgme_ibXs;q`VOf} zxvcd<@4HvO4-pksh|j`FmJtxa81CkhLfuUsXIrl59;DJVuykY>W8YNmoS0KmfV75)s{<3wZ{xa4Q}#^jzS`NM`$&AA`TDf+kq2|n z2Bjc!j!FxjJ6pxGVJR!tTL5sW`0^6;oO24D`;ruB$YA@$j$N+1+QSIQu9)pk^-A;z zXWTwtI~^2w2pO#_s~!qY_fpKuzn~qd2N&}46wevQ`+?aMFrZhc3XydST6f?pHxQq0 zxAm}hNEwACrsg8|k~WFL9C&hds{52-dI~*J0LgQo95e#!z0rSfgt3LX0?g=yc6(_A z;l3$H|E{X$_e+H;$po;f?Tu?10uh)rmQxS22po?2W{) z(O+eO6KGvo%JfQNW&=z$pJxA+40=zV!emMiA5h-$-)2D;0eT6~3QhTDwtO1bQqMRO zn~EDd#1bZvYF}nk6!HZ9SLG=FAim35hRv;K;_WnZiL-nAOxIDoOzhY$E;T7<$FK!t zeRXLI1@>IOdtn0@F#3>LScE#d&fX@g_-sGGvE*dBU@scG>Zf-$t| zNpdI*hXHgHGZ+#yumfm6E-Domc>OnA84BUST2XH&U!}!|7zWMN^Vp`$0IG8_3{obm z_l!D<)6rr9KsGdywq*Du_8g2y#6LaFX=ZTRJ~MkbSq2lX^o76N_J(+Y=X2z|&u};U z*PoP*H8sP$J>V5Nq+F%=LNtE<0@WycDVYTB88J`{{Pk-WGROGv%q1WeW1n<>ktusO zLbIov#*IzGy0w6@6VN{_3uz!=75$FZT|+ff3vuYX z&`a|NsywTx&K-shTc?bXPeEKE6#F>%#kLXuaM`Hoi5ojY9izKwnC{57R++zsvEhE6-=}oo{TUWZzhh87f$#U$G=Z5sr^9%_$-wRd~c*A*&E^o<n*65k=6)gA_;G2Y%E`}p$Y6IZjR6up8H!=9lB}{kMUDG_-d<|tFUm{ww!giPL7XYmPgVxS%Mj~X zInca(*ZSIAxq!(nU!jJfzI)Xk&CE`u}cVVy59VX*&o6tGW2w>9!$@jdW1?L z%$aVlsS^8r4VSwzDSKy=_kBZ6JXFkPl2QB&tOkj}o}g4#`~0X+==PjbknrsXCp*ZC@CNcZ~S>`|5XLowb0(NIuvz@;TUKdjT7|(U#du?IuX!Q^7n{t zgaY%$Z-;2?!aX;$oENr$@rew)6b6>k182zB{8(6Ot;yPjHv#Pf9f%fI5FJ*n0QOtAx!ksk(%wl7T(%-Pbb$A3xmNtf7%07R>~pt}IRU!GTg-PX>7SB-Vi@_=>)i9LQ#Bj-^-S*!wN!qSq*Sj`*RAR0tt zbook?Qi!mE*iWaeBdDA~9JGQRERYpU$)#%;0LiFMIvXDBfP zw;V{;aWMxCX<9@=FUzCCwh{GJqlJbHrhuyRM${U2h0GLkcc>KnyaM!6QcrjX?k4Ni z&uY9N0R_16XNi8h#K-;F=9HQLJznuY#MiWUgrZm&S*}Ipj*rtHbY3$0Ue*>~Ho{gO zHXi|io0FTHjf;zoQ%IYWQ~2ZN<6`-5qO5|#^8YY6yIR}Ze*6C!em_)gelY0yY3qAw zTKG`9dAQozJK0creRH#+w0HBe0swr!?_Q%Ky6ZABPOHt#D(q_lU}#X-tx#}iaFo(< zVQ?s=LwHB5VQ5;kDQ7@`CP!ySNk04T_)O4olj5Bo0? CUpzwq literal 0 HcmV?d00001 diff --git a/source/idea/idea-cluster-manager/webapp/public/browserconfig.xml b/source/idea/idea-cluster-manager/webapp/public/browserconfig.xml new file mode 100644 index 00000000..b3930d0f --- /dev/null +++ b/source/idea/idea-cluster-manager/webapp/public/browserconfig.xml @@ -0,0 +1,9 @@ + + + + + + #da532c + + + diff --git a/source/idea/idea-cluster-manager/webapp/public/favicon-16x16.png b/source/idea/idea-cluster-manager/webapp/public/favicon-16x16.png new file mode 100644 index 0000000000000000000000000000000000000000..ae39e5792d48a8c52cf1bbbf738675349531ba9f GIT binary patch literal 1256 zcmeAS@N?(olHy`uVBq!ia0vp^0wB!63?wyl`GbKJOS+@4BLl<6e(pbstU$g(vPY0F z14ES>14Ba#1H&(%P{RubhEf9thF1v;3|2E37{m+a>ZdT%Tmo*jPWUHSbWL*VRc@jSUU`%_Vz#eXmTiZf&f+Hpy~Zckua% z_Px!e+qy%V8tXq!S6tSS=vchfuWV&QL&K7eq&aPwOFEKQcE)Y%3GpahUs2y~Q?$dj zc)NAcPR;yt1+~3t)l)0$+ctDX?&=9hsh*KjGjUI^e_;9Y)aq%wd;NXNRwq@>^ekN$ zUpYIva(-7+#pIU48LhczCpcc1Xa{spOJnWCmLi}F5B7PV?02~|(dODDtE&3e=El07 z=5nxW{SNebUz})nd6G5Ioj?!n=n2~2>vL*?GniX4p{005XZ-rEsC~V@z0GA)TJjgQ zrvQE7UAhM7i~5E}pcOzhKwm8FNCJAzvve)cd2y9^Nl81gqKn*&3pxUPFkcj(?;ze9bVz=*gp*&JviBz%B^=O)+#W5l^+b410$ z#Hv}q;M&y_2=pn?1fXYu;RbYMLe(r_=y{Z`^D12n488cuIqoIvfl?vmOVX;R{r~^} zAY1xDV8kyg3GxFaB65I)t2iqwv!jcepovJF!}WS#;m zldKJ!hNFa+lbTVijFP9TMYyrLhl@f;vJ1biD<4;eha{U>yse#|sH8o2Vz7Cng|cC= zn;)Y_8V{SjGq;Ran3A5Wmo!h9HREf+J8OZ?WK8mQcUiW5gY;A&hrPtp*OmPlHzxyw ziTctTfy4p{V-CaFBZv#Vr_v+c! z`t#eDPdB%ZXmD&ed3?I(h4c0f9ui6i6oe#1WQ3&DPBAbiJ$fiOF;GaLFQuesN=Z&k zjng6?7X?jARa0HtW=&)7Fg~3Lb0^Q99v=`95)%}qww^&mWBrPdkX5VJtz5f0KO-kA zFEJo2Ff=$n_wt1+m-61fc=L+w<)w>nUtVRsR8{*|SdA(3ujl0DKBf#tJj;FO8wz_! zpPgl!zRr-DX~vn^`x6#wPHK3x=&1L!SqCc^?zWj;Ii{*93-qRHiEBhjN@7W>RdP`( zkYX@0Ff!6LG}JY)3^6dU0wPl*Z7^e!FYA944Y~O#nQ4`{H9QvB>kQNoUKJ8i5|mi3 zP*9YgmYI{PP*Pcts*qVwlFYzRG3W6o9*)8=4UJR&r_Xpk4Pszc=GIH*7FHJao-D#F ztl-jMayW%qd2@)u=^Iy09657D<_P=g29E_^dJM0`1xr3TnS!E^!PC{xWt~$(6957w B*NOlD literal 0 HcmV?d00001 diff --git a/source/idea/idea-cluster-manager/webapp/public/favicon-32x32.png b/source/idea/idea-cluster-manager/webapp/public/favicon-32x32.png new file mode 100644 index 0000000000000000000000000000000000000000..9ac4183091844fc53af5818cda62a27f1fe12979 GIT binary patch literal 1896 zcmZ`)dpy$%8~-7fv7M5Tkco|QF!v~A?$*dSXt``zBIL4_TAfg)lv8WcToX1YCnLFE z=DH(u%@A^ZT^3$2`b=gIFaDDZbi$wkfD8wKxC8*$5w_yy0pJD#02XloV37s@ ziji5h?q`LB#ATGD9q{9)y=MQJBJ4<#oYD4@d>Ie`7;7>`ImF`+E&@>Vdhr)#Nm@gPzIVni%}{T}-}cOsM>@d>*- zzAKm5q|y21x%)}Y*8Uo=vJH$Rxaa`5wY+PkaXY|gbdor$!Uqu(VC zDVdWHA|O~U-C9IHlSx|6tmg9a)9wm|0Of>Sks^O9HOCg}h^Qmj^bYrK zXdf1+EUppPj}IeP(FIIQO9OH`Kqg5Uys#t4QvKm4hAmwO%P3O)vDFc~a=s#aE3#q47@u6;~wgjQg zv%97yFRF5-0nL+Z48mGQbA{yY=Zo7P{mQGa8zAVV#cSOrXszF#m_&Fosw z|IH{7_WEB+^JbF#OZ6P3`*-%YD@yCE%rC3uGbC-d#roMC%Ms)3uhPF_Yww4-Ad5l# z?HHzPo!(!^`LFDsQQg5el`v>NHvE>~-vKd`I+{f6wI8VzPVX&Vv6aqEvrx56@A0mFx=yJ@(jFs%4~;e+fiqGb-j*^ed6h|RTI zRkp$Uu=3QypY4~CWqX3csW09W&W2(~OD^}x)~3|pLDZz}{XLb`v;EI=l;*Bkx?nw= z@?PvW(vO`%fj+o^98sWb3TM=Q$$Y9x9+JHN0g=*2Nnlbq>;Ak-q~${P%NaTseBbACoEw84TPo)ItZ%6o)CdaH-9)gzLcdM&Q+Z?}Cz z9fxNSov}Ng6j>&W1N-?TGgO)BC)q?OmYb4v;fof z+K`xXQ|(-_o|&L~O*+``w8Oh89IiianZogL@$ntp|ISP(DV%jP=UGCO-K|2DW@t4G z+GNKApLX=7sp#613bVnpQ63I$_QX|*&Bl-@OF%zkz2;Z)JzLQgU%~01_+T zTUbfN4NEcQZSR%IM?P&-1OI6jGvTBwuX9AOfRyVks?NvDHu~NXV|(ElTb`;HvlzAN zz9uoFO;y4Ja@3HA%F3Ttr(TV}RL@`qsRXB~R`W+ovUZlf_g08pg{ITc#gF+<+3zBD zJp8BOR5OT@^_X%50;~OF75h0Im%x~HF z_!SgWoW8`}ihTpbr7xZSHrl*Rt2xOP6xODW7p^}XpdLkdbim|UYG#DeW=T1+@oWc3 zYptc#qcG`T znGLX1l$W-4hg)Zt^Be{c(~!{%l#$brL!}%MlY`pD8MhL|G|D`oox=m|oGy+EN3t`{ c>9vaW0FHmCez7S&$bGybIy1J6x>~40ewu)4xYE8Rr7Vf_xgSR ze)IZ!-}nC>GMR_WOEzSP4Cy1=`+J#exJ)MV@u|P>CX-#KtQ%6OU+yiFWzk>|ZFoes zh*K!HAJ5f^|L4DNCx`UhpFCwznS5&M9>t7_JC!p#JS^k!PSS)P;I9Dx_hs^FAMR2_ zTq{-1TDM&}x9h|34hNDZMuYz%_=oLPgm>7fnEC1s<(#u6s@F9S1^(oa;UI~952PTO*OCmX4S$&n^K}9q~#CdHS^}pee%%a?J4s- zM10#+^S!`czAeEODh zb^>!J68s0je|mjtOpgZ{w}=h=-ld9}kvo)8$N6sm0{&~@e@(1^SR^*T{T9{2Az-iG zkP`DkTj35fSNEBme=cyM<%8g0%SS%r3O@66m#&?vSsu)r;^Ne$3GV7s>}RgA(qo$W z^t4cO^#$<%dxGV|e~!0&a&>IM=ZZ0f7dwgVS}VaGwl;Nnr8VshxAiGt3OAkf1^;*8 z-!Uow+^7kb^FziJe5wp6yzmeI!Y?($imtikcK~}&Yud__g=ul4T00if=fh2>il>@S zp9KGHzRBpqi!pxIuda`@e)F=leQU14Zm_6V=~~l=^x5gA6YUxMGgHk~$i~!9Looe)G6Vli(H`I^A@#9oUaB_DV-gqpV*)3idO@i>|oM=gL>F z?rc&gROV@t+_Hu-_T9tGXAVrsf43?m|NYzM&q(XH{YMmCxiY-y&kG#itu^;xPt4WG z3tQ7ir15A6=FW8RpJVK=Ps)F<2D@V&U-0SEe7EhG_s@WT2ly`xE4uoOqpp*Qehq$& zV_j4CIU4!kY_0N?L7Vb)Q*9mV=jI*oo@G4b5B>`9{~i3BvHOFu`Tp4a_p$k3jVb(7 z75FdvTED%-+zFQUaU#Lbn=|TI!syb&iGliqNkKyR*`gyyt9TUbWtr*fOihL^A}zxZ zsM6*3a@EP=oZTL;=T$r(mG@3D`2Pm}1K$w$I!G9L~(<4ns?LA7n#aG^0E)RnhX!^*} zC?Dp&B8JV3@blJTXna0q>Ov}+V`l21H48?tfo|a~B@46(XGNv!{&8acf#iu^U@zJ$pSG}6IkOirVkhD(FR@-K^EAmF!G9S1 znslA+j_-%D@44ErZfm?@!(Z?6LXWyU`R{7*{zy4&mH38P0GOB}O-x!+Oj zgZNyDuX!C`^UQW-)G>U`GUAlA>r-RZMX4)d3ew`IVDn$i)g({M)+jy#f1~kZykULs zH?lUJ1pmLlZ&{MFbKtzZx9@mYQFlw_i3ES|BZ-09gGu8R2a-d&G4?(7%ER}AUyh$Y z2wyXBi)uj>zGh-^YOJX!ZAB^gkC@bJ&Vv8zEUmK2piOOfcW3%nWNn$gEPLCT*z6tM zoz=It>`>CU?v+WwdcNCA@Sg?$(cOv}ci31#PoX;A)1qG80sK9(wTe*ePNhCQt+Oi( z%d$&)33DeV`z?WfYm64+Z~xw!QUvxsdTrV}x^#7;^GE0d{N7;S$K09I zRG;^{UYD*hW~8StbES7__6~2bS1iuiHQ!ac`%#ykp8g6pzmhepQHB+p^90z>N9XJg zydM~w(y7kSFT>{VR%ROeIcp34HPJb{&S5tncUFJD%NYAd@E2CgGfgKF^#u#C`M--Z zZ0w57?1#-s6l^~DA83wCo@wrzl$rktYu0Jj+tY7kZL9+SC*c1N_;2Iw?)P{*)1RDa zo|BYmIi8@m_GHcK&zdz7{DZ+RXWnnc-*IVUo%MOI%J6mC;j5K_zrha?b@SbJUYNVL zWPWbB+j$C6M{6Z>XE<}`1Lpp-&iVrTKzy|mQF#Z(Ijg_lWz3yy@LOEr0DJgsCz(6%GI#u4^))ZA!VB!W2-8t(xcO8&S8eZQ9dl<9`1dn++Bc<_ z7^?>{)?Q-79OCn2;`8~zmh(?F^;}!k^WAn}?o=>$n&lVFG9C7Sn<^kiyaxUa#JIbH zEFYf;Ech&_tuVUkm$~B){wn6q6Rz5cb?}1yrkPKD1b(|85n}nk3tljMY{3_2;5FNf z2Nv}UQk$V0jK82u(dDSXU&h=~OY2=-Pcxl*9$xSi{7!FaJFpwzHTBZ6wz$Su>jnN} z;J2uBIjflaXNhr}ZP%IikAZ#vl>D=|`EmY*mlnV;l=~K4aT=$(>s_=N8AHLp8~pZo zi+S_Z8ohPLoBFkW&D8<+X>d~q;FMou?mPlF(#@~%QYqY^OZ(g0{#~`hPwofyEor)} z2F?IqZ6^Lso~!4ibz%9{o_x16;ifA2Zp-c5RN$$1%;1BY*UUJEGx$|B4ZhK%i4cJU0+)g}WUlnKAu-ctE z+PW|7PDj?OkT>;3D)2|)>$^4o($@GNl>Tk@LndpGKWe$;{zkq0p}OwAwqD+FkT(PB z>VMVNNt;B%+}mu`w%-(W^4FV!EV13P%bEsRS6*xTIZ>#uwSE7nO!kBA2b%>Pmu%nJ z0+6?CHU0vwTej*VL0<1#ecpcmV?eDS|7xqYOY}L&n)>_NdU>NpXm7Lq7*Hq0``Y?) zvE79mb@eqr)yrS%WcAMh_UEf_)(G;J_T~ja{-V8EE6AJGx9s;fYkuozZ%4O+{I$AH z^4ES}F9o99hjT1v!#=GB%Sd4}4|t;v@J2pxcK1BSf-mn{kvz#*E}vStOA)yqzP8Qn zuyk19&BEdROz7_oUl|HlY$Z4Tn6yn>UZXpf@t-d0fGgFquVAEj-_{bYKbYU1ucz*S zXI8@1P89FkTJDpFb!OgWu@`N{R@H*YjVaN4*w5nA)_SG=Gw*sc@8THyW9VNlt!wEu z`vuyEQP+$FE>*f0EiXL9;^ z5#%gO@M~M_7m$ySB|i}Eh^?u6;+jq?&KrNfEinp`2s1xJz_ zSPU=v82+l$ahR98-+NA8WdQn1i0QX7_I~86rw|jAGVjV6`)%kKmM9szJ96reCXTUG zCIw|+^BNr!qbAV{-OI>HoKt6HZi1K1!`B?U+EDB%^hGYh13qmq`t#@Hy|WumWGedY z?20yf7fheAFulziG2^b2unFNMxH3{KTZ zobme7?2_Lt$u3QS$1b0jyKfP>OT3h(4|I1YKb&wRacmdtofo-D3pU46qMR$o)-Por zPz3WnfPD}zqkk>qKUCT)@maSw3>yoUWotn-%X%`yjtKo*Y*k`v|9zAKJK6 z5jl*t&KG+hwk~z);)1leB%?YZ7v1HI{~dct;9`9l`&{_U7R4zd26BX+thFQ2ACLZF zV!2e3$M+_WoI)PoM_Mji3%Nhu-Ao>*$@%N}Y)kIR!G{y)M>#$N58Zun&dwsZ=sU*7 zv2APa)6+A$@ZF^`?|r5H@ZOWiK^Wn|?`r&|eKmDWjQ2D#(nh#=6|r3?zk6IvcIjU9 zzueTb?x=^;>cTg_7H(oTu~`iI<&1s)!rZcZ;+LU+0{T;=?}E*l%QqhbXXYU-clWh2 zGtUc7Ob<6RLfSVRdqrZ;|u+vg%< zzY_h8=Gs{6;#lhr!X2Af>*57_-`X)?---ve{z;`ScQE?R{Pwr;i(>yXOeehXGjdq# zhE245*q*&X(eN)z#rnII+VqU!taSzK`7!eDd8{=~*W4)frSk5IQ1jU#g09hpU-re% z$!D!=k)P*?6JBjD`*s53vo?70%`HZMqkJI#-HVKUI{Z@*`j^97X89IfZq+=%-n~rz zI2ya-4IeXx{GZD)W!^o8?*ERObA8fU@t6bf(-LRiqZMNT*Sbm1GxS_E9 zR9j-ZaY1M(Id4|m;3_ENfa$Q^fF7|ybqMr_dM=I^S2$ekosk!vW6pVuxSB7DWUUd@%tVFx6Qb;)%Za z)B%ol_|2V!c!~Yt5$wwxL7WtYpE?*nF&O{6!KUJmcEl%J&Nmvun$#EnuigCIeZyG` z783J~2KyLn&{TZ=UhKmQ#=rN)znjgTwO5HH|G>W35d6s?zHOgXhINtb0bIg59)$0H zhwqG!_yk$1 zx7`jOY7qDXrR|$@&6*g#TE8|5|8o`opHFk15!!X^$Bbz%Rz^P2~eZ<{>I`VDMr;|J^66XXbj$#2{igR^c5d83AXwqg%; N+$Mado%?LL{2yGc?i2t3 literal 0 HcmV?d00001 diff --git a/source/idea/idea-cluster-manager/webapp/public/index.html b/source/idea/idea-cluster-manager/webapp/public/index.html new file mode 100644 index 00000000..089aa9d3 --- /dev/null +++ b/source/idea/idea-cluster-manager/webapp/public/index.html @@ -0,0 +1,129 @@ + + + + + + + + + + + + + + + + + + + + + Integrated Digital Engineering on AWS + + + +
+ +
+
+ + + + diff --git a/source/idea/idea-cluster-manager/webapp/public/logo.png b/source/idea/idea-cluster-manager/webapp/public/logo.png new file mode 100644 index 0000000000000000000000000000000000000000..a5d1f493a16de8cdf3dcc623eb30b84e7e0cb13d GIT binary patch literal 179683 zcmeFZ1yfvI(>4mjU?Es=w*bK*xI+jKf+hrly9L+a43dQ4!QBZ23+^_!TX1(BTn3*x zL+@LdHo&<35Zw%kucB@ z-#}DsL63Kq(#q0ENEK07_r|D*dm(cz1q)?mBv!;40|`0!)8D^AM%eqafg75;WzNX2^N(_}#6(^APUZxtf`o!*gYyqX zu${XRag8Y|0OnkyAt=7N@hZdnsMH%&cQ+|qx9pv6F7jFm`2S!3{|@|rZU>AGF*a=q zm^kq9kn!;Xf)$O%g(uv1pT;-EYc!}Qe_SBasMWXll(4g4p8QiofMxWoa$w-)==mSB z`~=+Ok>k=ML2ZqP#=feR53)OgvKotgZap`Gl8Y6;(lNgK|K|c_!aXJE*~Njm3^xl;1^Qp*ng(L*xiOE#QByjc6p_ky9b3VeLoJsl&mtmg)Dq?MJKc z@?GM(#}{6gM@^!*74u+`%eijqo|7dpfF1O^%*W9vP&gHwS|IQFObC#CyLw*jSGC}; zwOq@goDBy+nx6L>6843Xsavw&d#IPC{l_K|$^?7Rb1#0RROOw~dlYmP8G~KS@|K`6 zXb!!yL~O|9kcB!|{vti_pgiq+hf`MC2d&?#`2Nh|FX)~Z`1QRK-i!QIEQqdic1;h4 zGWk2GWtH(IO_nC=Z1kNo>e?q$L7|_*jw0CD{@32p09PsY|2mtK>R)FgB`+B%e*??p zC_e&a)or8vIq|0i;5JZYpak`M(w(R-4X}bUFmVWFG zDAFP+JFNDoTTN!8c9+(vFGLQ5XvgrFtraWe%P$x=WO}->BD8hf^C+Ijy@CKdJ(;7> zPJnOU3#ihaQbYPT-wK*9ds&>K5yPluMkh2Pnkzu~Wp{~y0AURKo=e`anGdH1+8um< zrp-D^{=UrtwlZU*Vg7GO_euc2;i|A?E(&>cW+^-^?yEeK1eqmSV$NfH=_>S1?huc% z78>f^#HSbfe|{^z-Ro~c-#i41iTjpi4(aOhMR|zx7vQxEUnot7cYa{$i3MqT=(9?F z8QG8UCzU5}n~^DSdp9U}rU{p=aNO??i~LrL^ItvS1uQd;K~cTk6H<~N?T|F!D;9~$ z!SZ<%?}6$qRi;LRII;u43n^4eRCb-|)N3FcPa4{nMeSvwd>ZL*h2kbKitid+X!rZ0 zVKM(V9nIpX=GM!8MI&{iuA28moDg}4PR;Y4nE3@UJ*=&Rbp&$lVnFIXk?OPj+XQzp z0c#e}E;ByN$2Zd=Ko+~C+1I2Q9B@v=$l!pLqUz`!goX zm1;w0xGL!tyKgpK{~HecF)pZ)8+}>?IfmannnM~mh7pCp&wDU5fSs#AS8Xxmde$cQ z3tu{4ud}P-u+bAU{;&mrjNs&M@+g^)6fl z(&D*3n&8GB1(D@G=2upuR=uN(@^tj+59DpPkWN3PeqTf40kx^V3nKM89r152(j=6v z*zs01xR4PT|2M>ESz^p~Qd2#G;g~m&Gl_Sj6#5wsud4T$@%l`>N6+26rh1BhD_~sN zJ61UKW}zrIK=7@(*r8wx;o3RJZGz?9&! zeLIWgQFT(vMyKoD4_EhlP|u}GZ&%{|QUP*7Wt%doIwN6f1nnZ^ zTH*t4sOAR2waQd?SmHy-8e&Q;7u>t7-utU(*-^Sv6fuGS!Zf@9YFep8#Or1M^LhfH zk?_OdfqfTkjUyuD8uUKneReGsWNs8|0{P@?Qrjrtp7BeRPiBzca((LPPp3HoIV;yy z!txw=PY;R1SEl<14CA#0;&xD5gR{7}iX(m3T3gOoG^Up3MEq(a+(?QS0Q#P*_GG|K z(x3VyrjgjdL>b&vtb7!HtNhcc&g#&xumos{IE*jW`G#PcSz3mz@qqVIDrYri6 z!KM0REiY0Xih8p>adzeI^+qhdnNtnWkG~9!wz;=F0F}-bUh34oyc{YEg{H zLDJWqNX&_=@?KE$*4BV<#bXmkSI>7h8)X(an}sehGWG!BxLdefwqLP4@mg?8>T>6t z)evIFnPYHoj?JF?@jbZ<4e8sK%Q-R*=p&L@B>SFx8xvRPeO#wXt5sy(#NWi=|HX47 zc7iNi;RNQaiZDbv<^b3k&bZ++*Y+C&Eau>ta1gXR67(_j@yE`H?K27WH0FPalXe|o z=#y`t5+d=e{gVPJoJfQ(ykk~wU>`+4(0@qXzMkv|kY8@O$SBmVOEIRQI^o)tIT&S` z)8hF+;lS(AmlB2UA?|8okfBPIHMSp?UfB|osFF4b9|B3(!>B&|`L{b84L!>m6ta*n z3wV4x8^BJ6=NTFCSjM)KNQPq>teV=5+m)J>cW3DR$s&@iB%P`qz1GNi=0c{@jgjH_ z%|$ftHmv4T)t(DY%j&G1=5Kb@!p0l?T6}I~hVM7sda3UsFrfOp8uI!rVN$|^PwEv+ zTJndV4RT2M^q|9k?tK*gchISLgXEX{Hw=-g0~6S_|1>-#&=}Yv?;Dc}p{x>?F!-C7 z)9{`Re2*y{lwZn^g5&e`#-2l)kV?u7b&|v&n4_}~>gY5+j)IMp*n7dF5}#+vh4WzRHMR4Pu7@++l&|byTfc3X)N$Ex5FaBZ_C-=!rq|` zt=PQk4qN@fS~4%FN{gj$%f6{wqBHFqrRYlhuYf>W3?wgeC>fF3K>U8P`5b`Bj5c&_OF#ftz*PY-|3E;zIKP$l)u z8&^)!GvaT0^pA!25C?1?MU+b0R3n^lBg!B3=S{OPucS`GRMV29?3ndWkG%A9BL;BL zYCg_OO1i7+5QjqwVB3$K^xVjF&uJPx7u$5M43`7l!J_*-UoNSYeFj8g<1K!zeT~bB z3yw-sqi>mz&$_dUN;%t8P8?t70GceLby|-D+wMVQ+8+i1>9ZOlo$Ln%HRpNZ_JqF** zoPQ6%f7p@rGb|`t_B|-v@oQajDBG0un66eeH;1Ywc_{#>V1i`)CfgwdVTGJL zK^PB28kQ?pBYM}vR@oo^UG^Q|jfo=ljOcvIqvt-t&^iIBX=_+|k)~#tL}mGB5z31l zL@_UQE~EZ%{gHJ4trZ3%fm)y768lCiZ=dA!fxh`r{i!3BIts?BU>W0;EL|=h(TydI=pK(ZrOWz(uG;>T>HcJZxomVxS%x65(Z~x=J;*oc%Aw21}i#^p5!RzemKvWoE|_VH8m_w%}Gni!B`P_xRT?+ZeAp zojn=(1D`j`f61k_8KK6U^|A7O6ua^bU^+5zd#*Op(I6tWa@dz_Zfc|ACRc`lDmHy3fIQsW4?*? z3lqin$j^#Z&DC}3W~zaajg@$5 z$v_V*0WzOgM(DCbH<<66>Azf3{tWV*zS`d79ld?0=(}i{$!Ffn_O#;xh%fD{jJJ4C z3wrO?SV7hq5l?Ivdj;QZ%{<0CjQg)~A`%8{z&sKu1tjlXsMP2)A)9HIvcZ9U3>9wK z3zj2Qmmb||_3tzkfI>85YpT{)9LLQ&U(_uP6N=)W1JIRHYyW(H$BFeja!&Hem{gGh zQ@%mZ_Tc{gc?o+{l=VC-^Z&(8#NVzix}euq6+Vd4oHDT1BoU~lKZLjduU>ddWW1iq z)Tm>k9@0tK;Z+ecA7S<;oVrzI8*O`boQEXojOD@Z*$!dPj3c|(#a}?aum9TLAh(ol zHw{wTm;Nuzyf#vQHiiby&nvY-qykSFxT}4vq%|!T&PunbjbC0{HvH^oo`FfFv|AED zdzqwR*?R>;;!RVp=kS3{W;gMK(WX>4%pAd3ECSl~?J*uQ0rwTC{t-xAS$XbA&%NIM z51hR=D#1fMM@yR}LLKN84qntQIIxl4l%)Srp`#5)FN;l!&YOXW#b_njTL z?jAE!rsut#mRc2OwsT7j(5Jqq|KZ>-;%#Kts>Ne;my#!rM5rV@BZ@Q))-m~)*_S>V zhR#R1jb{PWOy!Pi9U+LI|HJ7ygGzJhB2%~rK`K%!mK0#9DM>X`T<9yObN7;LG)?8G z1d7au=}!^BGoLIzf5GV|n~51v3I*CLlu@`krjy?rjtyswn~Lz#+^_#~GQ5Ba=||%? zP3L=yzsmwpJieK@U@yy9YZniMUnIL8Ag#7BcW%8iex8EFFL`jSzJ1?VuxdNmzbRPm zjkS&KM6jrfW1eVYiux1Uc%4$dOjKAJ`|oHs`(K<2_XN1?;E2_foRLxApDR?Nr@=lW z$^HDbk&omKc>il3`{X3~mg(Unq+kSgpM+kC$^3(4itGFMDw&)VPJ-r}vbaTq>-#p4=F-Kr?-6vzY&~#@b4#+>s_n3<`Ofvu=g-7& zt@!ZJ+=N}w%^uX3SM_e~>qq-;<;`}ww(g`?_H+{L2A|&kqR^{gZR#zVk2-!Zg>GOO z)=pZ(tv{b6l)P{lDPCt%Z!COKnd&aSJ|HT1WINgPXhXJoh{w*>`%EG3r09)Q;a6ov z&Rn?c!)w+i$`k&AXtJU=cjFU5)*$Es4Z2b4ifiqx7gCDIbh^J&^NO(Oy%%Qv-m`}q zh?pu&i-g$Ml!>bb8L#UR8Etg{lCUm6^>0UH^SP{Si|Wu;zi~E0+vt0S*yy6Vm!7%` zGUvmiQ^!2540s#vq93=W2%AWhD=ri)AO;%dj7^*zH>Y2@YNFo9uhh8X_=t1ewpKu* zKVBym%aOeOo8!h2AQ0n)AuYBl(CeF&!Enw;3)n|{2bn0r6kh0;(Rct{WVSkl;n=y} z3A%4Kp(3;f^nRJUv4;0+z&5AO<#*OT1>nSP=`^ARX1H&AszP2F6!U{am? zWCB5kmXBj{ufi9`{IoTiCl5xKi;u|fMXO%>+D9yYZs#-(O8(SXv0B?PY(a7-G~wI5 zrD^<@+ev74=(U=ByNwl+^6D`m_d{uEKWwRFUh){nT(ZIOSXAkXg38-GmKlEh`}kpA z-|Dx_Bi_ofJtlP1@#FQcvw|^(BsQuA<27yo5)Z-hXoA=o#Ah`QsAR7l*YBm=Yyn!g z@v$n&=Rc`CD*Fpe?l@ylFCSKwVEu=v4vizgDAZGPh4;nhZaLI5T!T!61RzxLzdThr z4~AZy$4cp0{1n7?&rK1%#uegN^D%Q*boj}L8gljw$yf2> z!q8Zju{~pg^3e+WcUwPm`SbMfH}sNG4*GmuXgU1#nT2~!5*nx{13R1U)0xw*3WCM) z_z54Mgih-?AzS8iarP1SDCld>%L%Ec09k$?IoM^u5W=eIWni{LCT?`#?)&YXl)6l*TN|A(Sr-*E;sg$ z8FasQeAjqjuEWKMz^l|ue*q)`IJp&&ery6bB699}Hx1txPI|uhqF#mCp5D^)s(JU_ zslwAV(K~r`wi|t>+o)L8%u98FnY}fF7eyQg$SJ(NZ^xnq`Wyr8sO}X=ukWHy)Nx~e zEcgwT>B(3=B8(OYAgNV<9LBLLc?N4gpuJms0M6!>%h!{Q5ir@E zGC3FTVq-B}IYU?DoAk;Uly5_54w+@m!eCkB6mB#n-p zoC)(If^^Bm5KK+n+} zPhYi}0Y2}Nx{O1eYEVD%x=B3^ow?Dkv@r8aBt9ZNf0SWTQk;{ zuXkrKk@|C|tatkPFPI(YqVYbQJt*ZbMKl`+3|02z(N{!t{(SJYv~j474a3dsi2{AE zUpW@U$aLvBXY6{^?5l0F6^m#Ka22v@8vUFcs{$<3Xa@dTVU-u<1=?9oj-xbgFloJXnDLbAR3U8v$?Dbs)u7 zVQPt4aSVqd$G(KvHT3N=r~}66-D^{UH4|c#PBrV+al~Uz5k4VUolW4|g9=&Gb*Ie7 zyw63>Z-*o`m(;$b@S0xHyuaKL1UVkhT-*L-c>rH+wiT}#{;@ku`dw}(>&&9^GO zS5QCpr@zefTgZMwOZ`KqpYplT2Lm}ecn>12X=S{K`%Qza8(c8%{f!{}w+~NDfweik zgJ`{<_&$?)iFrX^*(LekLVC3wA)h1M4!t-#R?{u9ZRKF}xfKKP?_t%@U5Hd$WKOI> zQ>#WVyNHSE@30rCS5dc$@aCr+XsZWobCMWN^#P{n)Flj7xW+{)Bw?#svtWp)CB&3hgFy1~onScOt3yP5`fhL@uTLrU3 z%V|Y)IAWePf0UU~GpkL`*g1R1w8)&v8NEMfEJvL^9Qt-93e0$pUMnE)-sFXfLGDZL z%Pyq@EP0~&ycCDfj&QOX`{&+FonMn;4S z!_U)2NlCjkdx=01#Sc~oKNe^RQY2E;67hXq%=u2Y5BT(v-`jNdr8VsOJ9A038{|K+ zvj=1eG%8feO%`R5v*){mS63>>ad}IjyC~?r4lLdSRaYx5XTKp12Gui1C6j+SRe_g? zU}kFu-ZudDumv;^o!*RW?t-H`+n~rZ%)-9?bQd}kxg?43vo5mkpNGT)l0JYz&Bb#Q zBW1s$pGVb=454mse09;_yGhQzdXZkgjWXay`;&!1xS;3OEQo%~Ah5_fs~ik}U_0Nc zn_E~?tq9)J?YN%sd!CvJFvL(N3@H#k;#`yWh|Y~7UZ`w zK7mlzngluLAssz`S-gyJn%-Gb9-deG_%Kc01 z0|SmZd@j#h-PibMwDj)Ui&$x<1nxSIkh$5FFKmZ9J7_s;#X^m=xyQb<%JPh|z1 zKItdYJLyy#e;Yc~@%4lz>LgSDjsji!vdATVXx8^7!Xdb&eBwGs{h268>8!q0VZHH} ztoZNPd8jsQh=S|3Cq}oe5&pVt<}YV2Aoi$1kNo|2z#j2{=6&kdncq~RMBlnPhbtZ4 zb8o0h{z6`QF0)WHq5bkm+FlK5Nj!u!Jb=6~eRu!%nJ_5UlW5=Zn(QSdG!Bt(N4`RY zvw+vDjooD{>+@Jp(K^u&29``_QRP=p^OTtnf_KDr?acC}QYUAr=b2->?T{bn$}QU7 zj5C`xON{(To`ic-*G$%=G5r&4R5L;^rHKY-6svm64$)=5U9Q8Ex(*J%R_5mVKM-u|^42C*=E`ue5Jw zLTlr4gZ&$ZLq9^z(`sFLkhsO(`?xzE> zWKA8lvp!aaNkL0j4DV;&yd=m^5=3pYa5M6@%2Ps*@6+A)rAJAg;R$%CG@*QV`0_KD zwYDs^it#S*fnvO#I!K>1v3HH*hlx)FJ9cF<1|-`&XS#mea#p7{DqA{4-i3qjcjZ~b zXRem*ZQR4lp}efWp;rLo(Nyum_Yrrr7^{cyzB_`r=h*OXBZL6I>$jitDMwNVcQukr zxKgigCg1m?(Hcc=roKrFkVqSwzO6QECY;IBSZVb$@F~>)VDK13pIyT`9^CI1eAz&1 z8aE3I0sMAhbYvEA0{Wx`LSmU95l$CW+)yXWlg7#L#y;;8+h`v0xvxoD2?6|yPaCpz zW-5NDSOwU4xQG_`iE#`5w2CZBdeemRi}ri3M?H#kSd};-vV64N4hrWRo3K)5h z9k7uI=Es6cI&0M3+H0GOdY(MJTjiRKz;HI!F_P|p<&t{p5YHzj+(?s5_Z?G-$qW-^ zDc)_>em|TO-IwXXQCb~-kLxBrklpV$UWfbVrDl*{%u~!WTY?7}E6<`l-bqI(R9~4) z`1`g(OgE3x$&bEsdH%IVLBaxVzcx>kigbN16S}!TN*I4BH1O>s^K(0EcQU&(aZ!Kyc1n?|K=JSaeuBPK4kz zswX=R3Cnc>TVtrk&Ailt<~g*Gc^Llbj~YFKMi{}kcZ5;YwOX{>v>L4(we{KdzMjRM z$C!=w2vDe`FqXu_n+X6V5mveg+eQnjLNU+=Zl$~@PmP+a!lmZ&)KsUG#}1$A;lgJ# z3i3iK=0Ksd0fQT`Adwferw!Rp?P9M%)6$#1UR4 zjzO9v!(u_n*%IE?4bD@E_HCP6v132~CU06akCi`-m-S<_A{=aZG-UWb-dvK3T(rGC zfOiAu>4;ng*lNnQDrS5=sn-VN%8!VO%5CtY`c zI1kTCIl!6Uq`x-0$FH|ogBVnc>E1BhHN#Vdz2`&CxMLnpcBie4_oKnv&*@4es##u{ za|AD)f|D$3FLA_Z<#I17ahg>ff;dyyv~0-BC#S7b8uL+V>v9%EXxi!&UpgEmR=4Ni zm_|aT#eedcGEKis_w5YOSK}F;vxY#IwaM91JzbuDpQ&`JLg9NQe*5j|c_CEK4_*|d zz0CF9`fuPo;l?;w(4mnN<^PM9Qy5Rprv^KcbFNJ6pZJh^u{k=fa~qpS-O>f<5**Q- z$r|nysi`;m0XZAYjQ;#tS$W_@7Y}T68)nQl4H2W%7ixAxCGvT2zUzFtMKa^SXN~BB za}#6gHLd#BJ;lcM9kA_uK5xtWq?+mT8F)JAxSXb%vHYHoDTzndX+OipR^k|`P_mBS zYFpZUIP@X!4iyrU9vH}96lyCQjYu!HySTz~>W4cv7S3K)@wGL@S}LBm9w7!H4F$3t z*3UF#*q9S#<}}TD>NPYibmy*e#Y6^0M%hEHcPfiMR~%*9i!-gvqIk6x7`yKI4_Ak* zwi3C>DTimCtL(=a-@oghU7BaswNV)U^ChV{v46Cz7lK3?S#;Y=!8K0+qO(LB6nX5z zf;(vu4T)f6Wo1OQL7n7(!^u^1jF`!mhj#-p_Sa^>#h5fql&mB&DbAiwb+G#tlqz4L zKR-eB$F!ynIKM?lD**hU+ti+wxO#vV*WgaAaHnk5%P7{nEn{9{KgbDL81)GEq#OeN z!k^W9!H|31ijO=6e0b3q0PALwM!ahBO@U&E!HmWdR_cobC^BE_s}=jV)gsaI!_@^h9AcMCY#gIh)R4KKWP$?SW_o~JHBEu!w!EpK^=qpT&*p8sL0#{ zn(a1YK`7?l@4UincAM!iW-Z-TqWl(l9K5~|gu>U1UdB=$N57ohO222AmS@+Uh;~K< zO>u>+BDB!r9nkTsDmr5P!tOaIXb|{`E#9b;3>QC6#~32Z=-O7Sse4-FO zI-0F_7}{<=hW=$|1Kb{M!_A*j0|BA@1od}EO_>hM*lk%aGHj(%N2t;nh4_vRo`ZVR z>GvletWmP$D*TeecBtzX)arbk4&UgwZ#sMuT6E-g;`6h0=9&=6?ttNH`aUGq=Rf8W zEh_&cZu_V+e$zvp0oB<3Dqz3sXm*djC0o94T^HBY5}PcGmfi!_Sd+lsX{RC+pVSn+ zcbnZYm1XcSE|K+OcYB;8)l{!J@y|=6YnL0ccZW$kY`9fXB+X4jRlVIl-1b?8hgRNM zOi!C2g`cVd7EFJ6@dyCNw4dbwQ)cA`^UFa9BEqw3$H&b#I zrkNg1pjTYOhV@sjR`{25San9{JD11QMrw6GY~^B>Xi(Fyy99hxJ`31C!J?X)qxU@W_V!R9 zflQEH*oyXUBQvu6cs=#g(*p+0PdI4-qvMAqTd~qq)c4E`Hg5Jd<7h_X>P4o+p01IB zRQU(LO41*yBybzd#}yOA&VL>6-)X1)c+`fCf)ohVh(lUI`LRgHh@0bGOQ*fwi4E1E zGG`lWsQ7{!J9TNW#hQ-qaA?cI(liZ};M=+0A=iM{+S{ORZYm#-_umP-g}l-sF_i3E zGqccsrxEt<4}$F+TLAtx^cbt13MEQhSb!a>2G?$rd-7sX_toyRE8hL7q~R98INOj~ zzN7ebrH}4AFN$Wrzj@Q9G|#lM_O;hQzz#9i!9$YQM19|&F)F@abfc?s0Mf}I*-6UQ z!#_7S!ujcS`vnaX$;n^!mzWTJ z7q1R(0&X0kA!c{BitO^{8VAXP#ZjeW1xxY0NeW$h%nJgW_9$Iq@`Yjq>f#xWZ zF3O1liiHFPYJll#;BBC@I4Vg%G$NS_Sbl!edE>AsqO0xO`4nu=!3OTHJq%tr>hbds zo^xWdcQX>T(oAxZ)UGMI>#-tz}6}~NMNLQ&y zY@t{sNh%7y|74QZyknymP)2l8Hd2O~KZQQcinAbGAsPE4o!tEIO(XY%1Gx{Cz)KJ$~A+`hFAUlr(H`7FX&I6e$1cfKl1GhQ!tJIa;CW4 zw}109^d7%?t}PO=dd$gilNt^|h9wld)p_>fk5g%A6>gG%-4;1l{*y<1QeK_^3cu*P zD{3!G=I}tamAxHSbqOR2uygE`Y6?g(<290eAtV0b;A`cA=VU{>wD-m4*?J}B!;Oo9 z&xN4HA>ZMjjB1WiiS?pm)g-OvE`J{yr68I$e-t%TwzxhLOk3ymO|aI^Azj@!A$IjL zjXovY;|VXwFwHz9dGenTSn5DT$fXASD0&RJDN=P!XFL~bs~GdhnM{-Lvuu4H20w7UDggp2vtC4q-jj@ER||m@A{VZX>h-& zZD|4AC|?HX^#fdr6_rw790dX9+*A zwtRU|eSonX53aysOxvJEB$TH$hFwHn{Jz_hZ16vOYzD?+C+SZx!`J8}*;0mB_<`W$ zE9J>a%Ju}R^2qu8&0n$9@l{Wc0e#PWb(ArUR75x*SwFEo*~uLZyGf%4^*t3tWfhh!eYf6bi(dWQFu$F+unhBFwhT8} zG~>gYx3$rZBiX??CBMuE5Pon$7$Es{35L(x;qCGsKPkn@u^6dPH^gH;JSk2tJ^5EK z&!+?V-TK2iD0C%oS&{j($f#d`bs31fv}dpag8y-OWzN27X1*6Wp-VjSs*MmFJd}(! z*_FB5j%(`#SDK{fFL&Z%-1JX@=pzNZDtwgYu~!y!{;m|U*1;=34p|6Qy;>^-CwIY5 zKJ7y}^FsUg3+Te%ELD-Q4zScTzO+#`%Jw}qG$4|%Nta&TNvZYsHpr-(>`>WWeW>p+nbHmuaNN^w-t?Rm6>`g{CTf%C9xdhKog^o$Sn zRlBbUlZw-c6ILO!Z7T75R7j9Fuoy)&CUbKb%f_sHePK&9nlD#-ve_K}3pZ5tw<21k zA5)79;E~FfkIkx;6nGkV3FHW4n{{A;YOt}$NC6C=rf z8{H#c;|Ocmv}gQB^d-OkLFR6^e`;ZrM|?zi7p;WA?R1RU*s#)*i$GPum>@tw7geI8 zSBJ8bB|$6RW$Yb5K3_Lsp={#o&$K0Yu=fsGuRgeJIQBk*WAoY}Y^{3N>J>1%r6b$5 z?A%iHxZ$$91JO`N463)fv%#Z}zWfr=!hBy+#v{A#Aq>4zn=QcCmW@)Ko)m=*I|>CkA^y z$mxroCZ;9YRR*opO!L-Tl^SI-F8>gvOpiq_0a-r+E~dYN8CKKB$WEnVR1}3G*A~Iq z#>-y$xv`XKn{!F$N#dsZoFdC^B*Qmgn0YeQ9%uGAvufkUh@|j$M?zm0BtMa})hNwE za~v?y-iPJ%j|X3{ex;n9z?tS0LfOCG9xHv z0e9O-Ozjj2%D{FOrJQu1_uwuH4Re>Ld7BRAlsAAuiY7c`vmvkEI{>S-13?{wn)bhp zt3TD*z}&pwIP$SHvNcUDuuT6`bVwn3n>I)ch+}Kuc>-=r3k+D6XjH-}K-3Y=6sS(n z{_&HrFm=**;2X<O!e3J0_yRMGhERQfjUe z4_wUp*QGkrWC)bcpmpqw31WJ>J(YOi?sr!{Y+p7q$9(hNjKh=lg?huXg>YW2cLdN~L zeC5e5*cA#tls9a_7M=Jj54Hazq9cXAMr4=}B_lE)c_GdrICR^8zC?DjquH{-l9xkVLHIevnwljM`j5*zL6?602=lwPb4!Vc`<$Ft{2 zila}H|L_xdbTQnb9i%^##ErPk*h3TRQV}Kl^jrJofm5bi_k{`$@vQWBW_EqQ(RMe> zM_m05#y}VvDe`Ms#16VK8ySjKQRZ-J_da*oT!83e1-Pha6^~kYZ}ZM8B+6g?3^$DV*89C=+q0VTMB) z^F*e}<4M6hp;h|OMeBDHuMi{W0S2@uEn&+S zyG28i#;Wj7&c<9PqN73I+OfoDhzzK#6&9XViDYje@BMJ|>Qjt-7y$8$43uIA=ttF` zi4wy=PleKb-x^QS?E2^Ml-|r}1R5GR-h)~Wg!LI8* zHj)s_Q|`5Lt^QD6V09}@*8l#$=R(MS+I!F-7wppC?fS`*<|wK`2d zhf8i9TVG1r-?25VF|*tQgSJGrT%0rTe=tx4#NO`Anz%LqKZh!7pY7=6bt^!643r~5 zt$s`jHfY0ExR5hfZ%<4xo(et&q6z}u5(4Q0B0IU%azo;VJhdN(%6GI`t_NY(m*z$M|Yo%-XX@g@w zWmGA63ublUWx*d1W2aKbc-PZ6=+qVhm&(;d_Wqxu9JF# zpHoy8HqsK+a3Ps0vNatKcF8w@DOOEp=KriEI0n_SKi#zBv4&|C<#m-z{j6`(mb8}X z7sTKuP8hIu*!(B|e)H%b3a`x{jfqK(u)A#bdt|sMkE_vYA()_oAww-MroR+G9YOV` zd3hOGZOc1N&>I@7olStrE_TzeX z0w}SCm^7Dp!25O2DGFueAJdOnX2+?o4s(S=#i-Ouw?03D?Y?&k>cy7&V^(}rKmKf* zH2kKv>q2Gue7DSx>Gs|23FZaSTpInT;)T?7Ez;t5g|L7;iIOn`nwc=2Y6IVm8Db3z zq<}Y%|C;+zqHi*&2#JDajLtUu+xRSVp{l!S&x5012_?GP|LH|p9T;okDPXJ#bw`>d zI5LqHzFrRIjWn>WT20#w$ku0Zo+UFVG`fF-3N1Ooj6I`OBdzt>Jb=$qQV^MP$ zl|e3|@RTaUtG9*)IMlA(J#f?DZchwZqwtvITd@90JP2M$o;nP!X|z6zV^*OW*Xh1| zZUlO=8aw=eM?61^7+K@@dC(y-8Kc2U*!<+Hzi$b-8n5%)H6OLE`JnkY5%C7$GPlx? z;}^8Rv*voDfEPH=9SsnK+*!%XwlB^?Y;-}Lu{hOozRB^T_xJgkEj*hWJo=BvJ2i_3 z5J4c7*+0aELBJyfyz^2dyc5Y;=IgB)#HU>#8djI9(y)rP5nZ^=4ro#OOHSqJYMg&U z5_(q!t|{fmLPGJ+Gnaf({nFNLmQX;p9cQ5GfP{ca$!qXPT~h2#whBc!c++}NG2(3L zGQ}vsht31rP?Ds(L;BnBQuXWKmOFCWHo>7qtTo6|uO115YD_DXEM*zbMdL{Y%Q+Q18W*kRKY2NPHn4cxbv&dFj& z@t5rNelNb#SGULt7H8-|SQs8bECdnaFLDv*70abDlA4@kfVZwMhMn3a`*yo#HFgJ+ z=>Is{>Y>N;JT<%K^tbfg5#2JTqvG1TpJS#pQMGG*uQ`|S8X%7cK5(>nQKeS3ML_q_ zBK;#XY#SrSJ7Gn*ua^4*Xre3NEme{oJhkBvoag{^rfsd88%| zP6wv08D=s$~X7p)+nB*{}2|57*- zqqZ6Q<0XYwz2j;$;ng+fili!F&Q!qj~9$wnilK>zjiG zs8ZB0nxvZoR}|6jMD&^v8`tt09N3CgT$z$og7y@pK;C;WEuJpOU%M=r-S!Ef7tsXt z)Vx2=kO!H+<8mU8aG5iCyJM{zp`n*j&0c|5&p#bJ)y?F%(CQLV5t;w}G74n-(Ok7~ zf$jg$N}AVxp(Ibi7S}kUY&^{aWQ0xC2X{Lc6n9c6Pv zjO8N=wvR)g&Q@qD2+7vUj^Bvn&suxV)R<-x9+Xa_Nf{FYkv4nD0y^RnIj*c#mLJdl zgZCB2Lp(R&PieFS-|W6u_A#{vdl*rsd_U zkx-Vmkr?Pv-iakasfvjyCUWnv`9f&W@k#8H1zj4KD50Bs)m)0l5<~n=R*_2Qu>?%^ zh3Z1G?KkwmU+u;_326c%bGWnkn$nD;D|hRrt@pjudHin+J{Ux}o&9VF7!@`Xrt)2a z^@qHfW^jq>>8@WDsNBPh&kC}1{vHPTm)~E~8#(azlFZx~<0uGmx+=%IZ$z^Cq;6)Z zEr^i|@k4WHVjCTU38fv~*XArUZqo3sUNw50uEWkPm1#%^ijHS_YndB2a4=|B557O8 zrz}MB6N+(2I?JEj#)SSVtmL8=-ndWxdn6G-jp*^fBe(=pPp*umd?~>Bn4-uGhzN6t z7A|Ke*-)@C-8po3cXyr{{^wkL-{suQ^Q&EJuf1N?QsRw&HpQEj zp5H4qj7tEDLm==g%eqV3cv7yqsQ)Gju11w%ZW#xJDb<=1ZCk<0OP`j}(x)#Ho0y;% z{h-XxeRQQk5;Gog+RZdVS#Z|3Xmw(_J)Bgn`Za9x;(=4ib7@TE@$8;^bv#T$WZN9? zXJBy?yLg5KsuDRBS1Wp1kxzx0s#5pzlg?8Rz2F&+5!f%c5K3}Q&qMjTcs5}wZi7!m z8i4Z;zT3d>@nq%AR>vXo{^FOSgc&_jlA&0ecL>1Lhd}$BT3~zdc^VOdDvAKa!zVWK z4%oo}d{hYje12hrg@?`jc;30A1|oRd`Kf6h_{Ikqk8oZUfJnhC;tT<=7m-OJM_@041M1>z30O+t5r11 z{=kNqS?$+~p~%m2(mgRHSnIQMeHq(7Y?ytuC(M-ITjA*sr-00On!=t^`0Q@oDwX&* zN(AB-tsIETEM`@`hsTEX{pq<=0OwJ@>&W54-fuRIN9i(7`(gmbxqA4^5c1Pr#3i?O zH~FS7Fz${nO=NtX(4*47DLNDq!zqL*kaIQ`Dap8)hxkVx5i7*|wff1@w5lPaVbdnX zC_xsVkoVP~5CrTr=2BVkpzw`*xp!epOeRcm8FA2WxNg7~FtkizmsAo3gYDUYYaT|F<9gxhjk6 z?-X}@{7gHE&GPPF;BD#nSfGHyUv$sP)_tlKe?p`{I!ETnw6R0AXZHI+RY=eLo{ooO zDg?}F1a_;;)%&g{p+0|dy|Ocw(!qCrOOaCJt%h7D;Gl-0A0*C5MQ_%W5JwaFtPS&cD*4E@{pX6a&T1M!h=#$cH>pm z@j)uE7@Xzq#>IrF@ex= zH$7`2>HU0GqD!|z;k!-1<>hv13b)Y_I0K3kOQyS^YS-MP7Y_=P;+s+(Ae$b5C=M(J z0n*dw68~BiwRY9TsPH2*VUKR!S=W9;f>zi5P!?r ze$zV?Q5;ohy|?o68j)bHQBi;8k7~0~_~6?Uowamm-|&ObXuf?<_x@t+CB@8U#@le~ zKh(zmEDto|X5*nO@QKnC0kf4`m6i|VU1s2>$wp0jz}3E%mz~bk;ZvTDhYSBAH^j96 zcK~g#eFyb#!=e!k3i2g8hV2X1w}3>*E#=T;cj0BhEKvCgm+?miyv}*z&eA6}ORH z!R&yaJ1g5hmpF4Fn3PxCZp$MK`7ly64v1oekocu}QAUZ1@8*>$qw99l!o?VxCt+{uw&>-+5(=>q#5( z^m0KX<#s^x$*6D30{cxt!Cv!vTAPQiE%+4tRlpz&yZg3tOG-%H5iW1&&0t6!H{Ms( zK(t<3FP1=tpHs(XHOm(L4OqgiqFCgSJ2;?I(R7C8eeRrcfDN*W;y*-!&ZlWpf9?)T zoh_(cMqm#OL0dNOuy>?yNs;TsZC1wbx*p-Fx2fi$6GM4Wf)oNo!4+el2M>T!N_A%;g?eP=^!Mwk#UzoNwuGo)0J0C1XPSH%<%cxysPUz{!CUL!cJ6FN%5Wtbx4YTK26=J z5PQX|uliF~{xts)bKKqC?A_1bhsM^1@^$2b7a>-5(^<^|8FBw5{?bq$^+*H=T3Jd{ z>x=CT+GsZSr6da9eQ?fk{Q{4STrpQC=R}HO`2BvlBul~=0sBeFKUzSj?-JBu6v)Hs zQnp|aCvuS1f>F%BKd9=Z-_Mf-c2w=rHHm$$-#cbHJcb;x2FtJj+wqPLpYEn-r@Hk2 zIVCy)Ou}dWndk})2-8;=hPlN%m=4W!&Dju7SCy)rS)PW5+%0H?+K+r#|Ej#JNV1oX z-W4h4lPSN?Y4-NfowHyTuK`;jna{SlX+{yM83sBP)3^I(qx2k;V?K+m5m^Q>_^fYk zF7cT)k?b8gQbbvYVDQ>F+LKHVB)lnPh@k5T&%%O@;q{${NUz)a$-Qh5c$r#e;n@{( z@=xu2dX!xtOdJ|gxo}VJJI&ukbwjbX5mlsN-u2K|TScT{a&4L!)Mx_|^gW7{3YQ}t zs|-h|{uyYRe{}G+M;tKw_pJm@xG9k%?`k}GH5&Ez>|CGi8XwT8HGqNrt0f^d=FLy@;r%R;_ct>}54xMvC&J?tqf-4pL(M;#9Ft8B z=htz}&9~$+B?&DG|4N5?ZGA!9n{l#2-5;;@EoVsQ$mU3>R|1fU3(37|n-sqAsD1UJ z{ZR3MjP?|GK9@qP6=Pk_(uk;Ed>~sxx3w>|jqMYXpEd;+!I}yca&@qb-amgvjfDZE z0nj|~9YH^|Vz)v<2qbskLAw`kl_=VOi{ZLElc0r`=g(RQsx*|k%ac30WKW+P|44Ri zCAbsrehlYSE%;H+D^joZPBH#^zmm^5*MDDvvHE-nw6m=Gwazyn1=41cf89LFq8oX) znkr5{-R|E=I4ID4crKdk=s(fC`%0o38$illBo4!((&+U8TJAbC5oM|=jvPOr_xLE! zrLXc}7b5-*Z-l_<%sb+HPS&qhqx}Z)?<*Rt;Vef0wftlDe??E62G;~d-V5q$$TSe> zq$)$4nlBD$R0wjfIJ5JuVY- zVlp`X7rjVE#40?UM5qBzQZr?SrmtA&oAhrJL9=OQ3Lowig5}5^FyMx&3#Ps7t_JBd31Owyx}T_KQ;9V?o8=r) zC7_J7;{Xs<#yv}@fj3iemmgg4+_E3dk(}yN9M*(Lu}I9cUbuaY+<%z%<{ItwqlQW$ z3b7Z0>#=di*Z9pE+pFIeaM1tE3Wr{t^kTHRDAh?auN^;}HisZj%iI$+yAX%1CkcD)th}=VzeXO~nHP|=Zh{Q!|J&2E1HyLJq_lGO z3R!5JsdIb*t@bfpM_v{C9zHpW$$ydpxd9=E#-H-%X@su(f%hZ)M}BE!c&*$+&sfhe zL)u4Se|C4~V{5gu`*$3t6$(YAyHu8XYWJ#zlP{8L9WX975!PF1oxn{8I;>fK!g2Us zV&6l%K+-@Z_?9#-a~RQpksdqn2~$x}aQsK=#ttI9%NJuAd^bi$PKrQML*0rx79bCYgcfGsxuLdy)B<;Q>9;lNLbyw zVyvzIg;sa0-8Kyp64zjs&~WtLV-EHlF=%@)r5B|YGanYBLFW~N-%e3T>N~=^mOJ0E z!=#bilUgVKs>Xj#;@X8v-iQ#yl5WKgd`%uEw_2;gKUl|3iLl?j%>HWJWxF?-YoyWF z;BfRK8BN*)2i^MRZ0x0t*8P5OwgpQq$@ zpHL)Zw$V{;MwTk1x~`$XM>ev4zz8JPXH#K7X!^90gpQSz^6hphVVUhd{y@gMz}#m+ zaiYQ;(f3zl)8N(9Va~kL4f!knFf@e4f(lHy?@lalju9<{_U|K@$)_T&F{W}n2jMC- zTve-Bmz!usZX60TzrR$^NYMHc2^E$KCWp3$5_3KY+OK5;p`6#>AQm#h8y1xC#Xti# z(qRn0KAz0;j;JISf7$s!>GWse&?%TAl>&hyG)0~rb6#+FpACnl}(!(u9gDvOU zR6iXiS=oMeb(=Up5pKjfoPkfybQ?sC7bmOI2?%dQuG-8w`%(JJA-oh!`hQTHzR4so z?Q!_pNYMtJ$%wnvnsV&QYB6`jYs#>O%0T1*5~OVgGN>t0Gp)qSKN~!#T^3yvZuLMS zWM;N2;>34dKfMoMW~8hnJQ7g&=@FkG{A~LkCS@lG2crwX;s1sip)d$fN$}J~%T}jp z<{|kR#EF<^va$LUp1OK41Iixdl*ya4xg?LKBaPxE})ltu(K7U-_5pLnQNTv(Xv23L1O5Nx?{ zPRuBZt9hJ=xz4n~&q_$K(0yRQnggqXUtt3Ren`2WNoudF#udGNfs~V09z1YOL^oWx zE)81aC=|90GS$?^iFbWO)VA;GY*A9PnK+)RvUl;E@@;Ntb2`-=Io;M&?%NzK0&=P8Gj@ziqBjalQ*RUdRIKTbcNf~49M08$ zu<+6Cl`fppV{(U6BlFpasYgDI6j;f8YoKv{!q%}HzII&ZvA$dn{l}<;BdM6rq_yT! z!$E4v-!*90yqB%Hw8G!iy~~=d#^;r7%7OT}hS&ve|-bZv13{c;Uva{o#T+cw!Zj<=5_MVzw|9? zIxA958>zcSaBb3mQ7&*m`viby|Hrdq?fDk2nle2D-{`(i)m&&D)&a5lS_c|;V;10N ze={d?Or(AW@9JgMB&euSS9KWt9Ky6^xQG9uh~eR1dCf1}I*ht?h0SnmJ21IQB3P4@ z+9&@#w7$AUNpW3kJysF{t`uKI`KTw}pF4#5DPpGpYD@ckrPko<26hFj_dH*3A+&us z6YnoG8F2fgi-+fszhiuw`S;tpqZker5xA5lEOHfr6!~(eanXbwe^i9=%P^=q5Jf1#zV zcioSOal+wI+<>1&9~@7Y(QB1HUL$hV2)Ez@UnF=zQP5juiw^3CxzAJ|C}*%j#=3II z;c&d*s#EYe1RG^3{9cgxIf#Q_t|(Val34MxosIw3V~amASzR=j5&7>}1sa>SRh;er z2wgJfdYyUp3cK81HY=W{$q3u5;pKrjT(&ND5nVEw!`CAKWpx{1spg|s#Me7xIHS~l z7UR?(+@sO*tJ~6vv9hniAnt@d>m!;_MTA!6ng&k#7^^Qv@MHL&MFqOhDoH0K3~C?k ztS8i4l$c{f5fDUfHBY~2wG@vH`NxSIVc9Ghb;)GE|I)@`PhB=u{s5FZy;NZn!h}JO zf8Oknu|lkdC7$xu;X(6hgBj^*adIa13o^nM4a-a4s6a;|^}pis9@VbL z&HFr67rYksoJ#Cs9BAaHfei2@i&VWuGzlfZihea}mzfkfBac$B%Dd`D^!5OorQzJq zomb%^CBdi({Dp_PCozrauBJdx%Z9mh_)(^Zj4=Df<56tASbuBK5d3M(@o z;)Kqhr@|pSEk`T9$ZE9JPKsw7EirxK2p; z?wQTd|A~hXWW#fhTzRGHO8u$3saZWoY_$7acXENT_8Ex9;(XG|yJK4up8Y<>G`t2ptTFr3W2 zYP5a5GE1oWyXj9C9Esn*Xe#R~=2r8Tb*`jiyq0ufhu_T)V(=C`MA*V@Yk9n*M_)+mo~Q_YE>vZ&p@azyM55o&_}z^KQdQ*J#3DH~8w){f*@yz7H_8u+E2bxi0REE^6hmI3ZNt!U08{GY+I z3|2k|xH1?EB2XX~TvoEQtqT@#J$Sd2PIk%ycYUsWJgmP?k(9=$0%(JGJZN}xV&XFg zXUD)F(mF#IsjjqgEa=Yw@I?a^QqRPM;%HaPSJwEA&CVG8fe-X2BTg|nq!klg zuWmq@+gJ}SVG7OCl(mF>I&omosc2Je|7zLyiAccY0e)py^|lRMd7Lx)QhHg^LDQv= zl~DQ3pjINfNQB$v8T^xXK$+3o>7sd{9`EVVKWu34_pwbcPfNWa@CZvBP`%1szYfv8 z(sZ2agYf_8UPzW%&i+UReHYcVC+kX$M*V^2wVbP$`GzwEZZ8UDF67l463}l+vJELja{5`r;irPIQCmS2(r-|N1qvq8MiSn{_&Cn3b`)c`iq>n6gd0)FnQ_E`~;GX z_QT%cq1AGi84M%q%J)kijzo!WINVXl&`8|CX2E}Zn4unm%j!Mpcj zl)5-o2gsKT3N$%$URKE8$dc1qBM6Qfl8aXi8aXfW5@s$n+49cHrqt1;BT~qDFe3S(@yO5 z`}h>FC?&1JL;GGgJjD3Chcz^ym)Ktc@y-}O=3@fQ`gBXr_$iyP&b=lIF&0U3K{gos z+f-fc9T6zmlS`{_Y3dDhKB3a`L6F}-Cd9IgwrXGn?A01hoC}EVNDN5ocXwjQ^ygPX(6!zxhj*9qNIL*HHDu5-8 z=VT^~&+!sEi&|!n;&}cHSxSW4QTemm($JDf8cC6#?#hI=Rv!Y*8%y@@mV>mZ-cv=v zg;22I;_1$Xy!kPokX4Lfh(PLbO*L~uUiAvLH}Yms6mvdEoLd5=!B z@*T{wYfP_Fo;(ipkXc}FU%Dzzj-QLYgG@*PHBeg5nL;+vn@5k;A(R0oQqZZ__?NnD z%GZYgZVEA40cLnmWx$(;7Jyp_>T`?!&mcK6mvg(0AGsR-X^go0+`;M9yw^L0Ms;bL{UomoneOZ%?KOfh#?MV&ZeGGe^%gQOW;}QtKgn zhttseQUTM92YrSPK;JYw$wdK!IO~7%pzu_F&`tQb_QvJ6VY(7hlX3#iSeo-Fg2mZy zsnNXhC3fgoytRb>ag4}W*;{A0SK+^w7sYVC$g1hxK-DE3oGfxx%&B6@FqC3U$0iVb z@QCBi_j=+XA1~$wyuGDe)kv3^f35SDc&7P=aG4L5KI*NYBUp^dj-$WMi&TPnrYCOr z&uGuw)S533o{oNGhnZkDo2BO{vIX9!&m-YKD3*64@q&f3ztDnbhkALo|H`jg`C~bt^3x5psTKUiG z3e=!)+ge)xOq1|wEZCFes{oxB&a-gDwI-l_B^cTfKE*cJY{Zdnd(9HC{%4z>QXAHu z^uX<{=zOY5zTB#?RlTt%UC~g+3uiMR;`b(E^JHhIt7&U(ucQG=ly9&-W@zj+>X;p>4Gt^4i-wdo9ceF(F4 z2{XaATRG;>?ehZc#g9B{2oJZnGuV~|p7Pu4?yf16kp>p-advSZH=#X73c;bZ)~QH6+o#ra5GTY6@E%6=gE#FGKA?k^WjOPSYqn95M>%p2Iu zdwzNypYf!cV6S}M05JL>>qGi`oP|Gmt9g34+Ibq2j;~j?$M_2sHIZq^CKdI+n|nlN zg(2LKQ0Psk5%c6g!6O_HB|V$h!;NVYEqcaIov&W<%(|&PN6;e8@~Vt%g2b3VUb|r< zA7AOl#_5by$enbXIPAI`aAdu=XfM4EANj(u3Gk3;>wzQj1}J54ev<{iS80OfLIgV( z2r`ZGQ9WIjmTXzaE_{2Z-2w$}5ZRQ7z^f4uBnU3MbC<;F=_4uxVi*Zd)V?W*X*p|% zEM$3ct!+qEaAj@1f1oY~8qdijB>($t@ zLA|nrF_#@9IEjaw7M|;&hRvaetSjyQTc^GSfT*jNeM(6hsoZ zxzRg9Z2R)qROl(h7+%mCrOE~hKw~qEt_VnN@-u;WJ!Z3gW z1U`$8b}MJU8?V-O)oXrR>YQM}LLR~#I^^m@6y2dg#)=Vu3ij7Ago`l{>siMcqG&L+ zV86`i@v;cx`;X_%Ntgc0$1GG46>8p+`r|mVmE8!#sMg{dhkKNr+rhO|f{JKa=+k5U zq{64z=kfyszpfgsw!S{1vcXMIB3O&+AFB#qk}z&jA2jWp;{0~L;fH(331ncG9M8r7 zLxj#2r;(qBWihsvUaI4KvOKPl7e-~b-0u5NR=u==mNlACs(X{OGG$=lELmIGpvZz1gYrC}N@ zac)O*qDao)aP2Y^i|$K=&C7Q4%IG!LJ(`cB4rk)02t`OjYbgO4Cxd3?o@rJ+GRJ9G z;u|;Gs}9aq&U3RB!0d_G+?%yJjkQ`{w)bEz53WOEw*^j?>^D%RZjfjg)PjDnZM?`1 zs!{0UwYGbebRyw6h$|8Rhi%9Qae+S~ejBGSQN25tPn!Kb&E|U{g%DA7{OmOy$hzy{ z%s8QH0?k%5uo2u0(BTe7eeXVg$$S>Ez=fk^!$uSbKG=m3|9SsYJ|+t)7tj+jm4(mY zhK3*l_Gs-5B!4_K!?+Q~S_s8p1k1U?v0NF|ig0?^VVrCea!3;av>R_6);KM*>6`a9 z1TMS0;bUPYMIs}L4oA|X^!Md0#NZH3$`a0FE%FaiemLVJ^b0(fXF=;EYvw%-b@0Lu zvNz!9%O*GfJwD2)UhD~>`%uLJ!}i<}ES$0LYh9l6v4BiI0jz}Z?|1%E>d4oF4P$&o z2U~-K+S;X2NvNdh5QpRgn@3W?)Sb^~o{M>y>&*sVO>;m5>$cS>IMVta?;YEG3dp%v zZ`niBz7ruF_C`k>y7$N1I7SS=*#9Za{!cMtePIO$1|LXuIPErah|spxZ`37{P4gO+ zl0GWoVjN%^Zcq~RYa(R}J!X&t(A@$V3()-VWe0k;=HC#6vML0MNJ^!=&6We|dvN*P zi3phBR>`+jMzNZCbrbNAGuv;X2yX@Ab$*^aVN6>zp**^ zWD-a8j=vx+eZ2MOrKwovEm7ZE<>L&zs4jR^_O#!m!1&bs~*% zZO6l(VZtR##+`d5^ZfkL2UmN*BYF3A*_U_LxV{zzPGFGlJatRgwcATVTiXOG?p%h7 z5-Q-fi?&}oFydBMiLpwAjMxB3u@Ci0->~yh6*MR5U@YHr>l7Skj9aZd;D4e60koY~ zxC@_eddaT%iV>+zwr#CGTp1B`lgeyh=RL0B-mb6uu2+W8k#7&65G2oo9|hd=e!Ga6 z{_mnAVT+E;_$vLf=qjY#G>1_hYv&upqX&By3)OZux9s@JF5cY>6f`7!-`92%&QFmk zs}|AQ{>>OTu8u*XBsjg%7XS>QAA2Xt{@Rga#?#-9!m_PX%_PQo z8>=$HGq2xsTjI8e*e=JH?CU)`tzX)@zNYA1e!~RD>iE@V^xxBTR-K9G6}j}vmlBow(zk43$>AOMFg5>Sn6mgY093FFo$p zDVC+MrNMdm5jH_G_n>q$?`;jA;M!sxPi2(#!1;h+>G?=A&JP5@!)RY;)be^Hgym+d zW0%cG#DUPWe?G+i9-{BM4NiJg!diu&s89?jr&yZb`>vcU-~?<3Sw=m%4_IALHiYWc zW7Yjf3d?{6jtshjhnEx|1yw=NVakYG>%lz5&Twz%yq9ejBj)J`PXvL%ux-qMa-6$h zUPh2?%z7NtE(yC|*Ed)SnH(e<#eI{LKA1+9EHqnfva`#=#60%YrL)yf&LZi@$l8li zfJd(T??7oRIbP@Ps#qwy0n(!Nb7sebb>0J{Qf9lVBoi}5;f0%r7>drBzNJL(6XvrX zOWel5G}dZy#Ppe5Nrn#jP;mDOndm>M2rC3=?mvU19GmmvH9ml(&sKlrp~PIWd0?D7 zgB{nx309tYZ<)H_3~LcLW5zh#(roXy@6~r>mnnf7{R&9jm>_29u!l0>=>o%r<%QY) zo-@*_1)<|9)RZN^YiQpXJ8*HEqkY^dO~ReeBc%F=_v)o`5~`mzPznbd(;( zBE&g#UJg432WpgbZ3PN1etHK*L14fA`5(_!OB?BWXhmF82#Bo><8;(&geJCs`As_) zsT=2XJiIhfeXuAx-pzE3@Kewsu3iLHT!>djRu;HJljY98S#cH6A!N{WPmS~otiGj; zN$P=7qh3u-ZW7l@!!q!yE?+*87I3Rx6oaGj}lc%8a{s_5nk#5QT+L{ z4vN6s+27f*uOVKGF*wS9jV%Ap8I0RO9gZD0Zc{k*F@BNrIr32aYmeU3egC8G)#1QZ zU1j37dh*S3=0;$WFFi~5RL__@#RA*koi-(p0Lp0SRs?%VwloxEV=SS9VDVdi@BjC_ z-@wkhtMI>XCYqy!x*hE(Wour^rnssJd={p35;^L!$Vq+Vzd34l6=jqRT=qZQJX)gjEzXb(!TV2!;h7H zy;KKT8P+#Xo=5Q1EpD^I03ssurxDi!2^C#{M27~kCf_n!_m@tI2P4^H%pzAhNx%V9 z_^yG>sO{<%ByF@JYTnj`>hpin$_5lrGd1uj(t4>TyWb@6mfOY-k(aL1kS7}ai`b<# zOSS*ihmykfw?BiM)vCx*@5XQOus zoUt1ejg+dls}_cMMOflwWpoo^G6f}4V+EQtjEl}abXUk@dDR3uPTejITrF-#8Ll9! zL7o}TjPVP}6lpncPUU-tqw$GeOS3#$NZSn)a1+@J@uj`>q!a>+olT}$OOt2YKF|}u zkh_52Fc*nhW%DuxLn2RMZG6N1Ly-!YDTiIzl``%U@A*rva3|XzTziU7pMi3QdKg!T zoOE20*awpU%|yCNtGG~`mwzJ2cE|std^VBFRO#gTAhG<<*V(Zi97W!C1PEpGNvNzq z?fan1Y*PhfUvU~>5i)P9km?CFOyU*+o}WD~pA-*VJsMW_9uJDOIY9?A=IPB}Jjid*Ed*sXBotC+@x{uR|Xm-fi{*RuIfei~!*k+3_o}U{^ zfT=Wl8Z^z+lax--j&u0(+Legz-m4b<{Ij1VfUfw)z@13V!K>*P8acmYdKaq3GPAq; zyS&x@Ossb(1?}+&lSS!=2GU{Jc&_iIeT5bVdn!YzrGtvQBN`JDgrURlEuOP&$UU{r z4PNUN!0WrIocpDyjiJ(aS@u}J;CM-+;S}MM1P(D`;BeOS&czQ`ySqffSB3HZ zaL3Jdd3tiQux2(lau^N&l)NMW+DM+p?@AhWUvd~|!YHLb>}E!8s;5|ul&FvOZY$rw zX0Lt8R#}mcSbfpaj+YRz9qC`5?3M7Ph^Yf8qaq$EJ)Hg)b%A}G3?LW$U=HqGJpeY0 zoo?}98(F36^A@d0`y0M#P~&bB-g?zwSL;?A=%0wvEM$$J_t-Kne0^>{UNCLgXAoxl zXmP6*)5zPm*TTv`y1)KWx{q2U1XY|YLwGo#Ja)A#g!rp=q978rP9k{rj(0yp?)yT+ zWqumaImz|fh*FXeHgHU&hTx8>QuKzuneD;uya)awdyujh9aHeGI*3LT@q}y!HooC5cJ>55Y3bdB z$zMN0Y@h6JOG6rKXMdQjiLm^1C6fqidMXk{IbC&N6Y2CNTf;CV4NPQW^Pf-jc+Kuz zBljdQ|6?CW`y?j;W4;baQefC7?;h1d1(_M;F9Jxq1(BY;*8Oc;RB%VJKwcG zsSe-JJ0I|4rr10%$M1JcZd&slH{=xsw;x0_@?iow1oU>(2NDpbXiqvxQ=qsEGa4v( zWaL<&U6N#}WP+2d+O+1Z2J&gRDwp%Gzoq+X)_FcCe&$7*R@cyd~#gqFh zGj#P59bjabQ=rD?l~PD1?tsqQT!xmwt~F`8Sd&yhj}i}Y=lAP*e+6@?p~1?GAG4aj-=$*p@zn3(A-%@s- z?Pj@XI-XKYzB|3$Jh`{*S@?Ec-($#|l~F+rE7-t3>&Oes-@RNah=JPypXknoh)!o|UQf!~u`b>f@% z0l}uvO>)t;#PZ)!HTSm<_PB*7QcZ|V8fYEFA`Fl$g!pqaSWQ1`g$p>rhiYJn(YZ&# zKMq<+#S!SS33-)vd9v=yHQxuQ+TLc7SluKVKKCO&m#{}3j4~y8%nj*Uewzbsz-N8k=#YeAoadEE67x@To zYS52`xlsR+Vp#G5#ce`cQ_zRqErmxOnCRv&e>1#xS}qR}a3TM-uy5~G({`?+U}Ipk zeR@UE`lV(RmZVRjF7Rz1%Fs%=H@Tb@=*9nf`_>LyfUoCQt%e5PS&gP80}kP;|OZBD*YHhJEwC6WFxynt$i1L;B??)#QXJ_YP2 zbOgOS0-?TFsun0^agfAxbT0I?`pRP*zcyT&t{XEzK=Wv<6yrk_#oT7 zv=E8`?c))-95`8yEghu(eoR#U6X#w{iX$GF_p^BK3kw$8oRm|jlRYMY%7Qp^t#*yw z9`3X#Dlmf5g!fC}Ln7$!j|~Dx!BSa?n7LFOfW-RS-2un8wo`1}Udn}cZsiZ`f}L$z zzk$+sj{~mR7t0(#3~`JF!B2teoRY?xcjfFxJ&mT*<3|rFRTJM}kM4Vz=*M5x-}}Cj z|L06w-gz|}tb z>*Du#{V@H%Go)tT?MR*b5W~dFbL({Z0WUH^oOu?T(c^U@gGsdYz+^X4!z_L08f;dq zwjMMMWlo2LwmxS2lA~8VHn>Nc4e{Ill{sO3Q3PUXu~1br4@CFXm)q{k+X<~+keCRvSW3$;0Fi>K*uo-~ZZucdUmaFVEO)i1xoiF_@zJER<&mZFnx(;q0LF1!ym3`2)C-)tUx*hH;lMGl-cG z6$p3oE>R%(=O?Z6z$3s9LDz?Tw$}djPvwfIHoH-QsjnBYjnJc4(m^_WOZqa4hs5gP zio)WfDF&;$rh{j7?3KZrW&vrfwgv$~h0_2hV|RkrZe8MT0kIKkX~GNWx#dMcQo0sV z!}D|`r}aWg&(I0$#Fs2_WDmBTJtf)A7fho?yp^t&n~=HWLe4HBBT0y^U{j1IuW-|` zW)4$=H@7muZ+gt_b!n1E+l9ADwj$ICSWB%8#fUtH3Cb-c<<9=VZ)6Er-AXW66hU(e zG6+DPuc$D1XiZQFc6Bfc($^sU#_Qu+kf4-^d|q_q-+9xvx*pyTm9_ouu`I}j@Vx3I znGgFCv#Yl^q_>|13F{xoBZ|NA7yr;cPL+InD8Y zECPJO*X=f2zE`0Gl?bVJ+nSismzF!S zKD#L`fXG|kuJ3XY#`Jt&OG`tJnfY*LaK@x%83*t@Qe$=BJjz=F25#p<393+>h!9$O zR860Mv4Dl$6*(v!c;91$u9&AAwOMJ5y|8-uXKn~F$5y@YsQ-N43i%N_a)N$Cn1BKf zO`iaZ!1KX&n=Y)WbB|LdoJue>2`Jxp-c0aWYX~$#3`E({39?ZiAcE8XimG=_If4aX zUC=>2;{cY4HRBAEdKosS1!)ShlhQScoYgo!h|&3Cb*_T92Lyk~xjnll)svteo}>(^ zv;dm&MJn?!triLUP25Z@?))=H@Asw^oA(h}t6OSUxM8g?=n|lir0Bg6XRHRyr=>xM z#iN*4{E|5>B#iDubP5ua zzY+a&uBS;C`i1D1r`Xt{m-4~_-FZ|>Pf?FsPbf0xl| zMbi@>XH#SvuPz#v8mB+hJTh&RguL7cGwenVvsBw)H`g<03 z#s1C7cETr1jm?ysSCj25hxVukrKMFXt}li&NpTC0<7EC6>u8X?rizeD+V(uSiEi0R zB53NRgq=$hRhpj`eG5y?;c+S8lShUjA!`hZE7pqpF}lgRM$sX1c6`;BnrwMZ+#O+f zW=er0Y#*@kEJD0b0bcgYVg&Y;%KvrZOw`G_O&61%BEEYco5=ZU1nlL^NVjyuw~^k7 zG?KD14MLeFu@1n86AC55j;QW8?0`?l9IOBdLEFkUpn2$t9e`K4Bw-2TSIl&U)t@~| zvatk_a3d3)NbJgjZjS8t`yGOrVBA6aIlG6B$3tYdITL>qSE)Hp(<|LP&pS%asTU3y znH01GLUzgf4W_JSLk9(8Z*PL1to}X1w=l15Ow(1SpcqwXksYmPr=085TJ4d9^0M|A z&yc2A%?}C=-O|HX!zqjXkw^L-E&9B-^=-;XPZ~7%#{IclLw_&LFp(a``Tlo1Np4Q9 z4C!4C(w~E62Z!#(3n_?%tSDXEe=3 z^i|&W89z^nwjGSvHd_4N0Yc6d<{@8W$G|i8X-AW2WA&?0a{E(qJdt|}+yBIm9VtL; z4^H|+{|RjEh$A=*gUef+>%23Sd@$kY+l6Pt@a{O+PVDi(%J&b%NAB6TocAHqO}oor zXDAvFp;8aSmHcTS=GE3j;6Kt>?Z*AzE>Hfu|A|#&(WUdi@!zZ#{D209MHvcacj1&=L_C#q-nQ-R znTJgzkzvon;J<8G3RQ{i+Y@Y8zq#o9wuY^}ZLHlhi9dsyV5RI7RoOvwhweXaKtf0w zDNRXT5n=;_sX-Q=`n?TycN+n+qsRffqU#;SSr+`8QH1r^UI{)1wbaG(Dz^WmY}}G` zYCot_6LHsmfA_H7R=9~1Jiz8Kzs$~t2N6iBZ@q5d#Ms1?I&EBxK8gfyll%5KLt zk*l|xsH|zSpUx_+S@ecA)Tu_ukk(x1$g00Fzt}rE^vi87-8|OAjp`}pJ#O`o4&d1# z=P{cl9Uj;`o|_;*8nD7 zT?L(-o=fqoKY4nPuj33HB|%o|0PKaJ-46XTNfq+rS7Ohdo~ul6v*;Q6X-aQ6zR7X(S{3qT~_U(Ik6sAbmK~G$+X}wcDT%38T~@* zssA6c-U6t~@B1F6xpbFw2uPQJbPAH9G)PKGOLvDfh@>~r>6d+oLN3HM)u+s+LuD+acOGb%j6?itX-pHror+Zczybc%|= zL@B0oqxCCuC+MyD{tRsEkaXjQS~GXx_O76cDB<;*QrP-sHyo z`O5TaGQ{;+D;^PY{nHxSfF+ipUU*C_)q&)mA+U-*%SH-3)8U?Rc&{;S5YO>R;n2^h zt|HT%@mgmQjm%2tbwaatV-i*BUV*_r*55-3=hm0p-(q7A$j(mQsTh1PJT3hD@yP&< z*myNtvI&@jQo(4@R*vJ{BY1bvmjAv~@cS0wtcar;cxy^>n;fVo<#NRNWrZQ@9kV06 zHC|cWfBA|y2iYInR=8O64zJ$&GUx7(tZy2I@4oLe&hO>mn6sfhJtF$(-$paz)+;hieD zcd)z{X#NG=Uph#eUj^1DL^xIC%5t|0=T%&)Vk_Mp)Uwlh^|&k!Qugj7 zMjthm3W4|pyh&^1cY1VVJ-s2$^}ETI5X|wpc>Ib4uO zAM7DpN=cd@_CISEIXU1Q5{7h~Z1G3XT%Ye$oCgf&b>>shFrzb5h#;yAL}J>*1Bx1Y z-3XTPa->quZ*B%3w@vm2=5w*tUKdUUNV~b^VBvH*7cqD7Hr42Sc_0s5Knx-}cfL$* zDO6vh#SlB^fNGG%I4qD3aY<@FOgiJ|CSR%;BDtz?yGJff5_p_Z3!$v;rl7}KjT!{EJD`Nx{ z+5_>s9Kh@dJEgYN-r0-AyTZeXp~Ix1XBZV}LWNNd`9J+GD#w&ocZi!}sJpc(Roa0!u}>`trB zR%&e>rQTg;prE8Rk?35alY6=HvA2C*{2-^Nv9MG=Q+Sozj(6BCE1QTa@J*7ds!iC$ z4`szI=`uO#G+8I%o@e-TRzcp6I|IM9tw(=`f7apu3LO6#2Xh73U1=)`HVDhGUxO?Q zwqrZ_sa&gh5+HoF4?}Y+#SKA7lputdtE-IRnfu-~pA_Y8`^6SC2!8to1foXQpasX1 z94{RK(d5PCjJdmS8Gt~!<;EoI(lL8?p6i_Bz<41i%q4zPHi_cmwx68wF@CvCsqx0Q zX~k_!jnHRSfaBq0YM56(QW z)_}~vaF>=ANQKb07r{h{*%zSwEywG;6 zMl&Fh$L-USQxQKhuMhhPu5L$dGwnDnq$Htt=~vs+jsZA~C&|Y0HYEsF@xFN{X?{Y< zXt2Z(&`dxc2yM3edW#jH8f|}qZDr5$Di6{3Qr9F5{=<(a>(%YD;z;l=aHH}#)WK3p zhmZ9>mVRb%hA2gUmta9`tJ1VonkrL6w;mK|rth|p+` z&ZBf(K894F9LIiR?=Mi?ehnRY47KC~bYwV?(rxV7Apjg*Nl?YcdxMwVF7j|e)66Ie z&NcZsCY_Y7`~ts7`r*%jZdv%TfS-dYO7EYBwXm^Acr>FX3OCtq=Q+rAQ={WGu{Aqh zE%(B^8tLB9AyijqW!kLO_X|MM@Sv$IV8xx;0NCxkpQq6cWedP)d=t(A)D(LLj-1zK%4=c$yR|KRNo%*f9r=flcFTED*)UGn33jqWTrQOPtZZOt-S4_vov9&a9-4H=>3zwNmsfts6=M$BvzXU zP$6x*BCPku+NTj~OJJ+rON$_JeR=Kh-QVeFB&fv5>O%kr3)*`)!;=M*?xWtYQfGkf zA~iv%8>MP9j3+Mq@_>6Iy{!i^HS8wBrnunh(fr;_GCymxj*#_Y`jXERXZd1#Psf(F!Dlvji|ExO-u`^D{n`q#wzuh)bk_N!PMZ@E{_-*RnWe?6@sQOz zn{UzS&F5YhHBdbt)Vlo8aWBs_4xr=uBYpeo^hiCy*VI6W9S0`~0)q`P%0D4tlprB) z1X2@Xzb4@ZNin=K6mZ*2Z9*o1W_v^Z{m~w4+S>f8%r2ViyQy8;EbM(o5v~btO@GE& z5KpN-J1rSeNake7uVqeIY8*{WuOJP~Wx;?tciaGeI>5#7dup=)RT{wQ#tbNxCw4eh zt&(toiiYun-z%g z3-@vEo`I|(P~IAD=jty@Gs%i~uln%o_BagEDns}_$^dWQ7QkZra0vmta7e84+YAjD zXR)n-z(_dpf?DK^`xt`BGV4FKD-Ocgnxe1t5c(#7fud)ilU6=Z zM{(BC|M`w8urTd1r|DPk|vv!)qxp5`Vubd8YSvLlHHnFvPj`tV=c9{th=g`bMA z3rU$~`4sxDVT?8hD*%~RCI!BRc5_|DQ4@U0C`L}EuGDM#jGq1&qpre6q>oI}5 zVSW7+ZvCB-`x64f%TW^i2?ovx_9v`b6hVB@izf3$K*hVW%qpuN9(fmKOCOJ@GGctV z&ULmuKHSpWvd1(%EXkGttYid)R3m>!=@SG!ULQ8LEp)Tns_I_=Jo)wq1o{c*suoaX zIpH1d2~A*`HIOw#o7>;N1Sj%}<+dl94 z?5XT0Ba-E0Wo^dJ68RNT+!>xz%Ywn4lyFsWRXF^9J?%|wS-F*?p5#COT#*VUnBZE+pQ zhAlvEi^5eYVN0&tn|XVDn)$R$VD9&w?smy`Hs5K8|4d=rG$*c#6#DORUREby-<7EK|MxndE1 z8=tt2Ob@?BUr2`1j`-sC*jY`xYtM|zMeV*|hoxMgD{Kc{;lr(pIp_*M3t;z(h)q@3 zCQ|pCJSD{A;IVl@vri|b0N$}jt*^|+Qrb^W=nPa0cthCBsb4<^@zzN`)%YoE# z=5_CTS$LyDJ~=v8CxtS0>X0|8c8)4=OMhqh>~e1zam8_pb@~EusMKz!6GVG@RI~r? zt^ecMR-XOUTgk@zO%fun(=sq~(QJ{uls$5E4b>pLJ#zd?_2E~Ve`8-Hx41n~=P-gvB)u<231 zfKS-pP#n6?=(CaGo&$~dU3v|yllmGMU}&L&;0H+y=zy7d>Tr7fYPd!{p$2X|limFzgp=4CRU#`txuq8V7>!R9+uEnyRC?Wih$$0l8&UNYHL?TMF^2mFKck zugI2o7Xag6LuAGPVR>duR0oU}(F2{w)C;vLP;?_bM+rezIgdxtAMZK2wul)NK*Y5Y z{>&}zl(Fp!g&7N;C&8u%P7t|xB*XQqOoBT_26X_0)sXh+ZV7mW&N|XO)(-8EPbCUNbw!gG~%PW9K?0ubKg{j;b=StyG6)bD+CyOarN!E&XU0L7}5W z%v}pVOQ1Z4=Ehq10hf+A){uq5TR`HO_Bj(gtMf+}+5l?K9rF0|Edgx^X%Kc#tI5Mq z(~~6x2}lHGAN)-dJUsEO`fJD`reP1+?T3Vd$g8a8^IHy?5@>S_C>Wsuppgv?S1R6fO)N)6TtYybrbw+(0_uelg%6?)K zy8t7~-~h3IbORo*YBF`il%l4iLgf>^!@7-ssN8c;?B(rZ3%stF&`x2hX|BGjoAWGr z`=K8ufQK%1;ZwkE7s}>=U@n9T;WMMMu66E4Cn1sV$6@&R_D*EEMRHT`c(8X8{JLZH zGO68Mfm?Q)yM4{H22$fEa9Og_%yMIoRT1xuWbDR^#)K=pOSC

`QsknL!= zcEm@TmM6QnbhL^vg>wlV?TLu{fTm>6Ev{|?HD{`VWY{NJr zV4LkhN$tImS1=7XKtoQ@>4DQwzQ7N!HbIYD4@CXuL;R-BdyuYP+9L96k% z%^vRBj;V}|OEI#gqJE_`*^4DZHrokpf#Pn4wt-;yK06p95Fpx7vKsslpL+Q^;`j&KuKanmnz}ibpR|TzQE+WHk`p*l=hliqh zVpWW16wE4kWf^sFtS6VV(r^5ChhQ-PM;!}^5qKpLKnv&yI#o2g9=nN^6bc`&h?)fk&Ad+5!$*H&Xgai@N= zkZe+T;)Z-5$s6DEkffEC>AW(T?n`_&4pTM&bIwGA9&>^Z3NV!@rHaw+?2p4*ao$Ts z^?TQd2ht{Vhj#_(;C}#;Yd&>oaTry88o_k%y<)24Gh=kK35a6Q92=flhQ+l@+fIQ zagt2VTRGh8ZhK!L*3q*?HL@D7{kf*&u$fr)<*QR2<0m1HrYBVuMtJ@{=*CMWakCW@ zcr8bJpx4F0e{VH&CSb{jA3^GH&^5PIMrhHw#JgMf*~`WLe)Qh+)3ZBiJ~KrfyjX6| z8XQqI-1e^E*SZw74wsW#9RNQ*mkPGv5c=Rpuvb`xExGS?SJyLoX)hLW|7PNs>Tmmj zHey|q@P}T{0SR3`#%8mL-lXLCk5GpS{J<}vo9!j|Ezn;DF!y_e&p;uLMB1j}Af*xk zclj1A&@B(K{Db|k^|f;GZa*vZ)guge^XDHA=FaO=!piB!Vvy`wwo|r9tRCa@!v~fN zn(l^%4~-~kY)Gy$fK zi3hLT916;>hWT_A2CjpzI);kX=YGX?aLq{^+AErIwK*mM4VNJ*+enI&Us@ApYV~XZR8ewSY4-{GAZEgjp#tu)W*x#4Tif4V4b08IBdI zEqLMaOE} z*;Xbu@6MY>Nf5K|AAUSyx>SUuARh9JQOH&=Yw(UPa3cX(3wG5N-sxrwn;*>ks*itK zm`mNlW@;TTUQ&@X&up4D;W%o@fgTvT=;v!N{bikfg1e~emVDC36 zsB;A$Q}7g^~ntW1c{Tew413Iw^ROu18R%w*~7t}nAWw70)0%<$;a_jww&1&rWtQ>@`lA! zavhncP?-4tz!%&Ac|dmH6aJz5B5Ke#Jei4-s&Da;=7R#S(3(Y#Y0TaMDY`a%92+49c4tDth`&@?~NQ}@RmII}i~ldFXt9Xvvep&2a?IC37V zrMZ-fe_;q6*Cf=`lQLhjV5X)C_%+&iZ19%j>LrnMdxx1nvrk}ORU>`ZGTgD?S4Rg^ z!j3R(9G3ufk*yQ)atc-Y44G_MH1Eq?Q&)}zZ;p9^c`o!PdUYg?HdnM_t?ZrJ!Q60x z;^zzuIIL(o2^rPPHRQf)Fa8Ht(?bDt&X1+_2PUgR+PI%feVf@9 zwZ+f$!kx56oCrr2#~_wNoGxzTmIoYT&7JW!2MSrov(A^bH@ZnDBPuSv6gj8in$5zf zaBtXb%b}YUP>F?0o9x?4a4V$R~OR;VotkOwx{IJ=bz74g;XtSp|tjeIJm-TBMY#_iwydTdhTm7as3am&2$07SAK$L z^@8~{LWEz1s)kk{Hif2ogB71{=eSBqlvS(kOJ85p1;|Q4>QPKkey{l#*fTQyP8fWK zz%IG+>qo2@82w{f-Tat)nerWi7Vi0T>Uq&1X^MIAV*H7zCG1M^D*>Z6xkKI)>DZq# zW*RVHj2jRG_2e;rUB%f@gJV(#hzysT+l}!^jwZ%T99@w!#L)z@&2Z?m9%raSq$Jo7 zxHyy3mifN0CR`JSKGldJQ-3(w|AV^@%_maUAIPVS-QOp-eWI; z_iqIr!YEb4ebIgLk7d+fK7OcqarXYau^Ez*Qddq_`2tP5Nokr9F|rJE-h-faY7eV^ z%6mNSM2{VzYm;h_s=GBk+kPh2g%|@e5%AW~(YOYh6@memP-x$X(4gwWzvIi$D3f&U zzFxqbBMPDT=tBJz54rUz_{$qK^iT0WV^Oq4g3iumKou!%z^Dx%@7uLJb8Wqau!E`e z8qSZub(!YcI;v9i1xj5~KQpc;|9pN3Q@Wk!U5yP3Bczr;u6DC_8 zTNzb2L(V`pmF~EIR_@R$;ZdLeJ?PRQQKe>{lvv+!buf2kvqaY4h>RdzlAd+RX)LkZ zFZ|}Od=VI}6AI8fWp%G$R|z&Pm(@i6&uH26mnPC_x2J%!rw)dKaZPEt_E+(fR8Md{3_aVEU zb01@{`Hsll0)!oN@_*(|g)TQ)y&Wvynjw)bK<4O@rco_^cqkW7?e@@8t_BLpP(x>N zTW|5p|Dy+IKJ+}W#(nRL(-LG&WhGyzhB_di)Gy$9ALX-Y#|Yu8`nBX}!qt1Ze(L!) zPE>by=j5GKhKJRettgSahoYh`=-owwkRb)kH{+B+0|$DzNM9m+eq3+9`z+EzP#64? zCBzR`@x*1DCsXqg*%A~afXl1HV#q{RNQpiVG*bI|d1*y`m&Kz5P|L6z zWCj!JQOL6QlAwz)hKA&NmpopVW(1+%3#9aRdflZ}o#h$+5i}{PdGLt9x)siCDG zr)o^O;^Eio_a|paj|mTOOTIRd%2qrMc|TjZmqHz-*%VIj=kn{+YBm7&t4ynbb|gGj z?!sK+w7*Wd?PQm<{J(kIZ~)imjQ)m;A@qYRoe-cLa(?uN^rWyP-i^vL2Jt13zfm#E z&Iqo=MXcr7Th1FQmcUfh!1RVz%*qa?X-0U|GMfab^c3E12rj7gqHrf^$>QlY*n)BEF!J<^Wc=#5d)c8 z;c?y6Zq|6$aWRZHw7TBXG-#R-cCuAAGxn2jz7t&6NRkf&+3+AJT2T8)BoGyz3=`BS z@G03ngmp95zQXuX8&SJMjaPYz(d6Jh7-Ot~bo(-iNUrrFKUN&y_8&j~|9*neL{#hY zmWS@cE8bQ#qv|Xz%V!WvxZecbzUucH-3^3r!-NwwwhoLp{!0i&^8L?&c1}=$TIk`J z;d7a#9OHFACG)_Zc-%sGoaeE&X+|FXwCXh*E|*yC-NYO*+5|+%at_Nda_WDa(HW?AWV%+XvjKM{rqu|^8J%nt&(sKR<7f%UE4Cgq|NuuY+-(P`&PbF*I50T%(mN0 z$|c``!m0l#HeIM!)m?v4LqCSWg;y>BnNm z@*mbh%}?YfNL}6w`HF~Bt%pK6I_QxDZ^A+o2&Os$Z1UC!jG;<*e8k!KxRMK^e|}4^ znMgd16QOXWGJmA0tVcHr;KoMzR;MC^P6@@iN%|kpt;?)3$bSoyd@ao}#v6D%iSW9~ z^?$B{qU%2k7A=a_dp+yTp5SQZxiDYIaZ7nR2D?3gxt7U5gM)KEbz<1aZxOot;-{a= zHpX;^$yd`K=1OgPfe^?vG6OSnSI|2?ITq0HW`#i3n^S*1mMJj+8xJX9elxRCW6p5l zHK9j^fIP{8rJq_~QzyA~%e&Ls9IRhrH7*tN>(D+UWMY72rT`Wm}=)B2~%)SI`Ul_g}epHU$!W55;NElQ@+qeBXg(m^jp=i%BTdQ16v)_-os z@)Z;%_xI*XLGuQHSXQ2#OdRF)ZhoUgu|^o@={Ko2q=dzhjU<)6`>Ki?#qx9TgCpN^6z z4{P%sC@4wQ|BvUgl!Ol69+pc78(bJa<>Td0MVk7p;Q;Mhd5(0NBz+p>RKgD0Z$l`; z-v!wcI@&n;>v4_L^0 z`!Bh664@pVPtV(1E?qt+o!zKYX?v6#&9+qdkL~Jw3B621bp>F!afHce8P%Q0`qc>) zgC_2cBQ?1 z14T30qGSLkIh1g)WL5P^-h%Gx-^6Z`j?XukiC911?DK*0xcfY@;#tGXF#HcW7nlj(Lj2WBR%BUTX+W$1wfnLr%5(EKdOh8i1W|f&e;oei&Ubsj(XB;^o zwl=)8qV+Y=yoF7uqcIJi?3sxg`lksmY>$4)Egsk-UN8eILmN3MQzb^glHUh1;KXSJ zbY_qzakFQO(BVD(g1Gd%SM1xx*~3yU>|MDx#Bj374QCsRk(`;eW?Vr3hs*@vw8dr^ z+x{62KX)aX&qa2Sg^PSewP~Bco?twLE4A$FP^lam`l&X>Y$q?TT21cTdar<0_J15i z4V3e$=?{9t4NIP6fJEFTYXul(8=Gr0sK0VtX4RiXGeASi2B?PFes8pb966iX=!s5({!-VZlVt&lOXxNOKiZE zrf}AoeE%wQdk}8E#@1%nWSReJvh36a@tSVHfaJ_9$mH&%o5|Y8GNSy}65raMdG158E>#ZGq6(9lj*=?{!(tjxJ7iPegUB6{u`M9nO|bKxjm2XHL$(SkEG%SK?d6Ip8x)6 zk=0a;s~M-~TYubPQFh=kEL(TfLpOvF1Tqyn*Awh~(}E0VC!~NNFd}q0!@wCMq1RsF z=hRjwa661IR{NA+FgB7*#?V_hWJfyxXV*_r9nWh?0`VG_IcWVE3@{%+ecHigm%0IL zhtnfiC2@ogP&E~EG{1&lHoN!(=Z&XFcnb!k=reY<$41btK_Of1ygP zG&Hm(M<@$9k^i38gvO>bAwXw~yR zae+f3)vRu3-BcjkL`u2;+v42Nt#;D+7I@bL@TtGil+yUD$`0lY*Kfs&DvCNvk3l)x zEXZy1NaIAo`dshL*q0vFvbc?Ts+e!j{m5pWkDft(S6KYVfCPI;$6^V>_QmMN@{fm9VG&SJA#PirD5}(zFGt{CrD2kCNA&@8E`K|fVE9gmw zPbEzx{*ar|BErQgIUKQ03)c!&@Ot=a(lYkhh@m45qRTE|>vzsWdZlZHlE-)H^f3W} zVxh<`hCivT9jov{Wc}+aH`p;fz+e98#8Y0xyd^$kCmz+YfZ7%c&7s{1Etf3{6JSH@ z;ii0?3Tfl|>5@#s?~4OAN(~V-q$tARMG1|bsb`Js4$!Ul7%JgJL;`b&BL`E zNwDH^8Q1#fV$emy&aj@yO6b=@aGk{A_S|nb+r0N@4vf;cX9HL1UgPH(CSr}QsPNV# zR+*&@OS(g~xc5s2%^AlO-D1R_r1I9DGJD&LX~%suFw^4`pUb&vyZJiZT2moxSqzJW z0A)Wk99%rj*HDFi$xIY9GE&QWE_x|Pvd>YeR4to>{Lo`aVw`ABXglU`fsCjQzcv&? z;eDK*tCya&Etb&~0#^PJldiULliolY>}oU|8VR?V-E(D5I2ZfY@xUIh5-Yper%Hpn z>$6$x^Ox@lpqU_$`TsV#9Kc^o$cenvfR*Ozru$=j$6Q7R2Q@ty?-5OuaP;LylWPjs zIFbk)PV&5%Or!L8L(ycZw#`VDIQA!4s)c_hCIlQVHR_bAgN4 zV@u*RI@gC(=_E!1oK)|{iXEW$?q7!@6b?8Pp>5-07MMesK$_@zUW+&%l{@HWO53oH z8aRYL+6@=ZHEsMzEaC9>Ybq@tpO-7*{ z3+jWA3qt_81*KQ%rwcUoqoN!%Up}l_LXamPD+j{$^KIWy4T+&A(Zd`{y*4W>Qw7oQ zJA8jxRFvAlI7H}-N1*z=5zowGpvf~ZW4?gz7O|~B{gh#YL#Oqe=1s_Z#Lqt z9cysX((_V+(%bT*?pu@H|Erq-EG`6XfFpFAQ>*|pcQMjL&1h>AyB_Iz@}~-gxQ3>= zF9jcdM9_R~#QI_)1!`!=K;SOxZrkfjJ{N7$*sJ&`%8Q_wbROMzmoAVtv7BHc&QCLl z9hkM`5ZHHB2PY+Hr$ZKOPQ+7*aW*(H-;-5!eQKd8RPY}(yM$_=FFiC_VDGp%BSCu@ z>As(^dH3@z1~>6Cii4zPE18iA_Th&@;%~-VBptCPvHhaP22lo8w(W&+AG;V?K80*5D<5YO8V`Y);JoA);sxA0%+6=QvY(5PhkG7P zy|U0+2H-Y@=90j@972!mQ@AQ6;%C}q+g=nL*V0W@T&=tITERd{;8zMw&C?&OPjp7d z*O}No!A=wZ1;0Y+(82F+id|uYCm=m})d<%S)wtM|V&C&7_X~OL%&^PFW|a#!$vX4B zlvrs|>ZZqOHINdlwO7>C51FGu0E8652cOM-Gs?E|-8Z!nq^yg?AquNrcLDHoA$3^I z>}z#V*6`dTBq$L&VJdBxhgd9$QPArOdSgXHiN$E@%m|pG=RjfQ+{xVMsp14Ad>8RwCp2JpJ43;tbAOQL^S?JIh|tQ| zZ*o8EMP1HT$>xpA9l%6LqDu_ioKnp?>h|#NG)nAYuqIXux0l|s%Ed#{#wxg0iFM?*+K>uI*t%dQ?ou&wh_hv4np~)Tg@jmI5-Ko zS2B|74#arGIB`k2vF1vtvwG_UdKFVF@!69PiJpJRk$!F{jSoexA8)PemMyW28ZtS|FTAZANJpd|jm$+I zO;i;xJv5R8`PN|g-S)B8AU5?R;p11?a$!x-b8_J4QkFm z8f(x3!n7)0rq9F&E)GO78Tu%7i4gA$!u=-F1d-e9*>;E@#SZA%q&7};DaY7(*V%~a~?dgqR>Dr3r^1}g}wmi5$B@6rwCtNA9QQO~G zs-{HhcdEfE^720%^M7Qi;S!2?%+=>CAU+bS*S4G0iVb;+w3hIyxX$Oi@r5S>@Fp*3 z9tw`V9u_bx_UGOZX&M8uQMA+2-Kog4&DDPCAt#Mqw$3N2soy@Zau;3Rm#I}!(x8V5 zN}*UKeSPjbC-*py2z{{}3ZViHq4PfUJ}mwT^YWMh2QNjoLTi|l#NP$v#spP%N;+Cj zuapI6_9ivru%aY}A|f(zD-doQ_X7Sv3ft9C(m)3Gq*oI zNr_}I>CCa!iEZzrm$GTsy=W*j93$RGBl;2AY0Hd*l3gJilHAtRM>TEtc;E4(ruvi} z_pF-o7-*@Lb>T)Xo)=voxwFcz2egeazVyp`$-iWaA&2=v4T&WQ2Gs#(8w1#5m>A}Z zux9NtD;0S1>HSjjf)EO~F)PW%_TJ>1)PUX_DapQg6jz?{-gCb1GA8j`dgVLRfx(h-hVyXY-A{ z7oCKB4>CNn%)GI0cQcNP7Xk0D?@vL5vW7{ewg&brK6+wrm_58497r5G$*CAP^cRea zGOkN$fv(Ic?Zudi)%&d(nZLbUm2&^%)Wanb0DWrNKTrP@6g9JpoW(VM`Fi13NJ@}< z?`rZ&z%r82Gi)p3b+(_~lGJodXQ$^LX?yH&6E@Mmdd(kBqH^W0%7a@(`R>mYm55d zJ6GA`u$0ftICcJ5&xNDEG1`&)bfYh=n9U@%g1dDPgagSxd0T$dy;)Xp2)n?5&0Z!4 zZMd<-s|I}_O(xqKEx#o?Bz4CNaxcJ@l2StHDA%Tb$%t;$M9lLY()UdX%=J6l zSONt93;byih=8f9vu*rEH8Vp%eTT~`=YA91;UAa`&0&|DXIVDi|}|w(7pp$pIJ+392787IsF7kIc;LoS<2Z4#q|1?&X$Ow@0QN4xJ z3@=T{_r9}(cOa<_(`D*8t^tp;lae(>S$T4jAa_LYLGHJxRowXDU_X2ZE7nOD=D|uJ zaw&?(5fw9caXJWJVk4W~SADsuBg8X&`ZAM$s=#AzIsN%GQS_2ai$|^N-D0npke&WA z_;u21#UGc4QUZwBGnCDf7ookImzSm2_4|Z`t>R*SF&vjC&BJrl*27i-N+UH}6k-Rv zUJzX}LRe}M`20l`ntQFe15$NZ3Iu3gK0ojOU6V(byiggUo- znS3`cKXmTR2Q!w=n@-ky=NtQZ#qH9)cKcAZ${s?$cnFA4CnBR}p=ON7ZduqhU7um_ zb8Y^3U-ZKIGAa3w({wOz`Qm71n;cLH`OgW5gFl0|vHY26)&4Rl;7!KY45HL32`)zh z?aSHv!HmKD!81QPG$Ii|y(O!X=WiZY2|J`Hc011rljSqkoYo5Inl)l%lQUZ{8h>`P zo{20M#OF8heKsK>xNa(SZ+16GxHkMTm~Nn25qZKz>X6qe63C^Zgc?>HY83w76#u%x zF=69Xm4IG3xqQjhtv&($+6 zF{ED?FQ~MCnkSjnk!RBEBol>?c1x9^KT?SHYWr5aVHQV>XPlf?V*i6Ex>J!&R^ADsc_4K5tY0VFKrc{ z?RgTz6ZROa7H9tCp^u(d*T;=!I62-(DtuTMB@OUWw=yC)3bXuX(GdrtU=PAH5mxpy zVMV!`nI|+H^z=Hs0p=+BCu^|@&|+iP)Deq?-QP=4W!5cIUSa;}rCfgq#B2U-{7i}t zhZ9|o?ZxpiZ$(Q>b^L$qpUlb*)T)K_c@BRe#;^j|K+bS%C^G2Tj5W*H9~Nh$*Mp1- zdBTS@o%VEJa!KtdhH}+s`hiA6BEE^poJJdrVym1u_vhrXhgQOeUV1MsN2u>l_Dc2h zUXvuw;v|kC+m+(%8uak8l;eUY7r8T8QTrl4JRup4RRYs3Jq_*z8a=V1{aPKVjPmHT zvLg+nNI?-iyQgLw7bhHEyB49A7vphSqaNC*1vr07(b|)c6DGr-YtWmuwc(rg#*S!g zp|>xatM)b=0dI8o`DySil5k=O|0~zoRSN_96P%iVMZPjdq9An$Q$<)6tWFlNuTeep zj^y9E*J!J3*-UP{?()-#^YO)e#2M-wW;$3P`1w9>zKX*2EvF{otU*M(@!^^0 zGt`(}kRHJuAx}6+^j;ZAxg0v+pvOYaQj2S@{X`GGzAz+@h)88tsc3YVqq5=^ukVYf zo88(myqFB+XI`Rl@Hoo$KuxBf8r>sa7g4>=r2E?5HiY9A z)s|a}arOUQ`vGmr64db5^HJ~r<~J)-K~YpMZs&emYh_sL{+JFaRMa9#l(d^zr!Dx# z&$;kA<3birlOA2+EM*oja}lyX@(;$N@uAQ3<;^SSZnF;tMt-qW%k5jJ<+!DSxOODV zqHf|rgKw3(KB{rivlJ~$s+g0VUM>uG`f*HoP-B1?mBh}xl6`kHcoI@%Gv?~!_e>t!0r?ucwzWhSI8yBhS z2{I#JL8Ni>_(Jf5$c}xP>mIJp@>1vLTH@+_ z1iGT-M(WL_M41}?Z8P9)yjY-SURzwH`X6RdLYgS$7Jrs@RBWf3s_pTNe6{RABE2|D zcskr;Vhz6Q``|F%!$becW&@n7-+}hW{*7@zozGFKr{}>gd$)$MZzLdV+3vM$@vI1n zc$jUJ`rZyq-e%3y8{d&U`gp&QG1OSoiH7REELh^`)R)Oi^FAZnH?H zNYLzBg*!*r<-IBSAi|D_i+S6eR9g`AVk{6vq=B7BB{f-&cRH?3ye-9oSt9N6L;AR$ zQt9dc?+A^gL6eV)%W?0k|I>G@(sOM!lR@-0ZdiAvS`*4S#P&Lfo(4VU<8sQDhatD@ zg@oabR4?1Lomsn4i2HhdOPQ{l*j!(`czwp%kGh z6&S84`}K1Zc&5g7HWm@@0cd7qwKMb`)U=SGqDSkl$s)2(54 zg8j8Z%}{U&&+d4`*80slVGbO~luKr}_OkO+tszA~&n@wM8!D7lDJS zMa!%LHfxsmx;Y2rM02;lJbFJBNnF;@HfW`XJMpk!c-4-cZMw+Y=C5g%hpHcRl^Wb6 zCUb~hEmy~w(ic&-zBCS`ghO^O#l8YmHF&L zoTf%_U=ja7ny)<7K|lY6a)x^EM~ggh$%J*n(@)N&w1K|6kd50kY?^?n^kkgZK5ww- zCaa(BFmhCbKFz5oYooreQNA2tQaeOFSdZ~jeEjTDOkPkqT03!c9zDEu_@tkIF_Ro}$GlO!SL~ zbTNOI-erA(vsSInsH4H4^S+2#Xeb$;m|Dp8WaLO_r_5exFQ42`Tm7QpzF zP(jc3HE~&F6J1HtEC`}qj9?zGs?fNuJp39>l){Rn+nZI4;?LnPTHeAP($x9hV(~z~Fc{{;SG@+(0G&E`FtoeLP`j;QN^T_v+(rhP;j;v97l24i?vYDpj zb9{Fbq1&vdZzObf{TQ!y-ZTFGt&19?x8}CDM{=PP}adY24@_sIYK(@YNNtK*tlEOOQZ&#U*0S^IEIJa2xd{C`}1by!s2_w_Ig zB{{ToNlPl-jdY9B4HD8JDIL-vNH+q~N|!W9mmuBUUGvUheBSTxPkkP}_nfoW+H38- z=iK2WD{;AKaV9&UvN3knfP^;tBelTFK8~4ODn@JB!WwJTB8!}=t!Vzx>uh-y5{H|j z8fvXd@${S+DbWp*H0zlcc!*@$!p^DM2Y04BU-44F)dRYQo;&D~5h~x65ZBBEzPZip@;0Q`ZuW56W%acEjwR^(*LoqZUR|7P zYybN5SFOlcZrx!y&5>r1Y8yUqr)H+Q_=}h~JztW{XcI~OpCVgD-%OS^y&+XkNc<#_ zVm;2_p2$GUASFm|_9ObhqJa5HT%UOQyVRWz=z!@0^!GaYY6Tv+dn#NEh43RAw)zg_ z*-CND-ohj1ucwnXT?SdLLhre1^Q0^*zkYk!Hp#69tdZp^DV}O5D(UAYb2xp+nlbxk z<1ZQr{UGRYE%1vz;yr7>5y8fVeCp9u<&WQeG)bD1np;0%)#DI+mv0oKRjC`RCKwqB zOzi!%MHA(ct(c$FD4X@@Wln6lq@}8<(0+t;@@Ta=Tzq!w8K}T4oY87|oHf3R*@MVg zMd^IENzvD}U%cyjT*_A8{XFmqSQRt2L@j#8tWJzW_4SJlDME4MSj(QDQ>VY472UU| z)5gIw8jz+9Ry@@luI?3p6sk?UTM_2aak(#Qxo<7dY%rIwif zjw55=*3`LmuWGG$ZqxUKKiF%tWX#!Q!JYDn{yb6}{^rNX#NZdSNwm|eq^A_es#uhH z*y2B*Imf2DFk;*c$0|OfkptVW<{*Z~rq0Qajt6()&(txys!6HZy~(cg=!sj>e;vQ; z$f}ZAnHzmNO}2M3Epaf8dY<{EL5-DB(ZND3{Zr&gfJn=(PrPK>9{W_kizD(aANFJn zQ|3!c&9rUPsN!u6Zh|H|Z|k);8A&_#)_*v+75Ua1#3RW6cWFr+$cwrF2*E9$-5vp_ zJW0H3w<9ud|7(m-2kCUgcocI)fh?iyV(aND7>_lG4U<&V*6UcbyC{niz}BC6>lBl7 z@4tRkVcNi|`U{VvOojeQxvGnybDM?{Tp_T&4) z%cv2pU%-brfSRrj?BAFsGzS%Nc%f7D7}EBkr&%W%MJvV1+=n=B1HvU3@t%nHm;V%(#!E^U&4S408@n@o zoUCn>m(lYcX$||-aRJ<6+m_P)74M_9H~01fkINd!8+QMuzE)&wQfNe3lM=}NCRhV` zFwgO&#sVV4EdjVo0>2GM68mvD1;?N%O{nak82CkeFKuU*;&%oGuu>l(z1(vN!O+Oc z2=-XFSPjCV%xqLza1Q4Q>@X;9;9nLe z^C(9OPcjiRRJC-jaNEt^evL}Io>JPf;J5r2R!&%lh$2X6bbpo;Mxl%cVeO&5A(oF3 zU{j625#Q~}j;6_2Wyi7XQjTQ?zc7(-kYv_|!^>Xhk@6drF~cqn)7FSZI0{bORf&F< z{3ws2@o8v6n9SSgrCB)bT=;E&9+h>S57smCsW>-nwmUiwzdnyKirW{`WRrb;wpWgx z{yLR>C${3#NZs0r(GRH8oOv|I17Sn$r;)~Wrp{_rl~MMaQDs(%Uw4U@L;8wm`L*}_ zN^SCjX!2~q5NGj&u5-bE$qssDWgL14Or^k5-{<$O+U9A`UB^BMLl6pg1jo&>M9QV_ z-y=%R=F2eqQ-m3qJ+U++j$O~Jr^+7hrPQJilDo?XT2?#fpgDtL6uuN=;W^0@3H4tS zs%leXGDL=ANefR=`6y_gdn4Qt_Z2laZ{tmUd_HPg0k-SP5Ls!jETKE(5tu%0%+}f+ zFWyNj%C1q<)^zndoiAL`vpAfW)ILzr>fFsA`1RKIG2%3Z`1gTQZn(#3k5B{>R8x-nAG@iCeHLx~gH7 ztS%L^tzj%G5a(BtgE=BJhR$GWX$2zRP)D}#-#ZM{wEf#hXAZ*3y z&WqVOLvT5=~0vWD8BmC!_xN0n4j^26+s($XCoRYF<(CUST-&bP_rTp<9}= zfP23^>oE!O?+EkddL)Y&D?9Qw9tX9jKdFPs6<`&6eq41^& z^3!Xo!Sl@sF;eZA@XTRWf}LJ@9ohZk`yn!J_;lc_JD!dzj{05Q*4V-1j+9Ceno9xvnzwww^@9nIa&2Jh$@6J=}hF&|ZPPZsad;Jbf zIBGV3aF63YT2O0&?Vl4v_p4IWJ1$iW)Cr){2`CF8=s*5d>G9{8B za(L0Y&E3i)JCgk19Hiyh`V#8tSgN`?tF7psK&{|FbViGcB|q0G-L5`RqGcAV;9b(B zmDutnY&*<*q``B;VcuNd`3J*P-lu%0(<}`;H+sEiZ`lUVJ2f;Q_9rIdfGkHGAs+cI zI}XDIKHCbz))${Vh^zUgRM9hiZ;%!EUj%T)s&7T2>I^O5-7R0S>L1_x*c3Va7e(T} zCprFqDWXDpQLuo-tL6;ntj@q0$&`vLZLlM{PJS@CZ>v`ADF^)r4DdwSE=9OKf8CQz z3+V~h1CDU@qInrvO47pM#AaY||6yfw|3a%gAiwj&F~)tXDkEQ)Up<nUi9bga1w!mWWLC!$Igb|+9DLVlN(jU2k?Eg&2#dLQG9-G$L7)! z+WQAn>XPY}sSAAmqNk$wdg#;H_(xWujT?@`|6W`Snsd#Ed2iJJC75!vflv1n!k1Ai zM`HEHBWvCtDPrLeg-zbKWc%?|y;ianB{@+8*k3Ck7>=>`^|}QJRG-9{TDXR2o$E)| z{Af8?Qu=u|rnYu-dqwii{I<=t@ksQn<1NbNB~y6Or#2Y-w;#<3w7VWx3%`U|a|AZAw@u{(;qzq$~zf+ru z(V{OYk41bEbM*6V)R9!(#mBpQr1|R8aY$k;=4;8tc5^cO46CuHc5RBiJHA_`0qSlX zes8dpkUPF6H?p^GbGTX@Gs6Ffevcs2Pe$<;(V$@o0)a;b5CmX5r7b$226 z6UkqLlyr{#me%PXw`d}#diRa^Xe!|n_>ayh>0nhQ6uzc)vnMC;+K=t^d=w{ZdNU5_ z$8p~s^tKfD&7+Ev&&9HGMz6d+s^4tcI5gJu0&f-4F!A0uA$y*liXYUuXA6*JHkFoc zPro)INr8(i&QuimGBOPafdfbi$PMcV>-mJ@);LUd+qZh~O=%<($`uvRH}l%<>!#m- zuW39%e8uxmx$YxBQ`>YBR>{ct!{X>5MSjCA%FAw>gU*Ydjf(DiHRjmChK5iq7a4MG zJ^thhiojgXx`?`A6nA>lWtq|5Oix|)OFaltt`>>+5B}z2Ao)HUkeugi7%9H}IJdrp z0B#-R{#|&2?`j8d^=k1&-p(i&nK5NQBzUL5FQyEVs1&*yJ7iC*1tDYs4Y5fH0kqbNKzP2%!xcMIjacxPFo=vL1c9DZW1 zMyQXU)7v@jsUgt!Emx+}hi~6b-%Zms*-!V8uX&K#y#qDZUk^_&G&Q?1mG&LgHlM6q zZyvjp>g!aQRppO;_=zdvKF_{L*gF+7#b-nKEFrHV{x!13CIK8?0+wb z@3%k8ZKE_wXZAb8t%SE0;q{Zx76*S+m%O7;c+oaEyz~W2AYIB_T%-~6n!{2iJ>E@O zIgQt!&oRX8(A2+jHiGra=dEp-PakpOms?pD*Ty^WWe-Cc#6Q%*jfG`SsA8#|Yo%{> zvh&C}i*jUbh0<<=20KFmi}Q1|^>G!&>%OX2i7O0%-RGwlKGP!yH}Wiwe;t)UJ!SMp z)0T^#5c%(OPXZ#R)*D?_?HC!Mhs^1^PbIqEa4z*rD(}uIbI<&iG+0r@#+wtw67SkLW22;0|d3#!{v3|rgZ30d?xinivPn z{Coe5BU?Yyk=?{*mkObgar0MIo2eR!(U3X`6>!=sU9BJ4rPDc6nyV1Mzmg^Wo=?Uw zspWS|?%ly&;8j22Y88*{X)$3K)rQzAZT}R)cHOo!RX3JV=Cf_0qx3nqB6iIFOUgiQ zNyc)B!yLO56~U9ji(=Qbmj1X4!P_}vx@4X|Zy{wsL@}I#xUs1AM`JStwHDz-d5LRi zkfR?jYzh4QNS}-$dM~@$GmR&%L6hMTf5N)3tT3^95FyRAJw^*~GW=FS|Bh~U&$~h3 z$Eq+K7ww2Xv+mu*Rl}eMZs}p*)eQT-&;(^2qDTai?i(E~bOPM`>_{S-&(SMScJkoz zJbPxaHwD+b&fW~EKN0#zD`v<}8-128RzjqI45lv4Wg1}<8qe`DE#NSqdB^7})%)OW%Oa}*$;L{sX4C%9da}}1(=k&z z=OL|EcV@I7{MQBEHaRATjh#iQ;F}tD)vF?~ZRQ(ObamlG1~)ow-3X749|ZhW7M8vM zbCG{=_(ckbb$mRI*PV*Mc<#e-7bZ&|$GLjQv|KtbLME?YcdH1PK8)`1&LLl43=}DSV0oKqS_x8?*1m2ZDVig&gbDm^JJiI?y?mI8EtKzu*5dKqE}Za}Pn z1If-1_K3^!KdR15MP#6Vj3@G5Lc1(wf|b0%;>dkrfbU;bP~I8Dr?#LjjCZl{JLh%b z`ArmS4K*)uYx$Iy7JoFQ>TXs{4;YmaX&=v|0HdTOX|E=r9TS+a>>k(ADa;(FDQ--H(A@gQrl-Bz;$0x3MSIeVwoLTCn7S-^bu0*MKo&rQo9VboE)YveV* zLhF`E-?w%vy3gsQgfDG~-jnydnGYZY%#h3++X_1yIT|pfNJH!5q4pm$m^W>KCO6*N z%~wz4hWLxw?f(f!T&c`7_<}>(+MPeFn~t37GGob)$vyiPA?Y^mwZlYg>FgUtDbZT0 zb=_Z5L?n0`o!w-}ve-rTO# zAqGd} zchduRLSp@m&w0gBxTQi@f?bIw>;X>a@ilM=y@eyjlukVF`OL6_s4}<=iBjZPYZt4v z$Yq@W0Z;9*-!nYLHMO@3e-vmc)*>pz`(h{K?rH599M!Rr`DT_3(L2Vj1m46#&EXRD zF8V-EbM8Xj8(cv6321WXVqWmw)D1K5?O{8&8Gko4`$naEZwhiNjq5}GW7JHQbvN`5 z5au^sN;H)sX1HJdhyim1z;$%L$P#8Bp*J6|Fm+G6E=p|nQaK*V5>WT|6$@GnQ`_;V zt9htXR7e>Ne0qMjaJO!CdvMCp<5N(R91&U{?u7vVrdK$D0Uob^HzEZwmNH0w0aAcC zdU<0r!i3?rQ&Hf1lX8pGl#HdG{pLhe>i6-W|ho4F0a2<{NeH+BS?Z8;n?uXKHWSdNJ4SYw#V?z(M8LdUp*rR z?GscZNknq%VoHl`{^#{s?LC(wTnS(&c}XfrVK@g#i*Dkb^i%ETZB>SAwj3=rN$AUmZ&oLkWKiQ{iLy^K4bPQsBkc$x%=_v2SX$YcS`(sxPiv4_iFlg zg(my|M0mV-D=|QAKen#@VULc)0josWKotBYpVQSx$OHmv*?O7>n2nA41Dx5K7E#>?L+2>#ppKa%9!9V>jdvODApJn;e zEp$>0C8>M`-(C(ZCI`+rXY?0|p_-9(q*&w3&A|gC;#6Pq)q!lS>U`u&Abg-C z3^^L|YA04kH&Xna?ON4ETa^Iv2~2C=M6iBH7p`BN63>1}5q(q-x3d3iRaK~-^FH(| ztNdS*#~tG+LNxQWpUJswd1{k;7iDX)@NQowlr5NNv`H{Jn1I5>6HA21FV>X8zyUGe zkH*l-pQRorjb0kt)335K5vtM32us^VwC}s+2PCdZc)ic!{u{-3;E-t>#`9UPKl>6x zU2fHR>`z;?Oy!hIH;;h*q!b|$*WHDEe>Gg`iMx}I`ir$6ac{q-z5Q%dG>m z+^Y$tmbVjc&Q-epqdpL{RxKZcx@P-3Oc0=9B6LNT60Y#kpJ1<4CAr3L9Vug5dctbIoPZ+(2y&K=#*!o$Kvxp079c#wrgrU{x$FjePDZXKh-1n3>G^P=s+z! z-Xskr>q|EkJRkb^o$2gws*(yPW z?xMdKL7sI1oqOJ<(;1X_>EgFtQioFa`pBK<47zwyDS3^0pl_>}(swyhZzScL=?brX zAJaSeF?>4m*V$pYNWNp%o%C|kw5M$Q*v5*P^Bk}eV~ z31Z^)@tAWIuW^-CT-9V7n+1Go>V|adowf#4&!Bsv2)fj#@@QHv^Dwm&gO5EYj3bG4 zSME{+{4m@-BJfd}-+rskRcDr}#=Wh|yI164uj*{_TgyXZ6MTr+`ct3Ebr|+$Y-Gr( zPT%c){Ax;|<96iZp%P?|*|WF?S6?8EKamw{%wrkIE>+iz*JoItJ>v}~7pe_?$^&if zhP5(9dK_?vRqAyv6&!trBeX=<#`eH=H(9Q@FOfpkmgS8a!cs}M$!6(0x+#fDyZZP} zbopPR{y*>F0aD~$s+C?5{_-Uy_dSkcUDpC%2y?U-(m2aXD zbhc)-T|f2f=Q|s<^?x#%T?$I`#-8@G&tU?lm50LjEpqSN679uj1KXubcjZ$YTuO;CW^1TBYLhZ{+^2`>|8GO zh%bIToQnfU#;Pdv!<$yhT|bN8GFTC+3cz70!6`=qKL>fw8V)#9u=LvvPP?CT zE|gnOpOJo!lQKx5I1J0%(9u5i$dJ!ba&{5IT#Szel~iaC-_3*3feE}|Ug5Qf z6C42Z*qo1s1Oo0^pWVGTuwlP7Eo7cNA<5%ml`lHlx1s{u(QqG#T(Q{&7zW z=-{KnDn9L>7bwa~TIQ>=eDGO6;%IG;Eu#LdA=!Y&;lA8}uJh4aZkLiKj^*TOv- zR!Uy-PrChhzE1wTTzPw>c++Kx7tv}JG^h#D@*QEOP4hl`;|o`h!wE3GiSvYr{Pz4? z<)ZjDx}B%_iz%;3YDgL%(cDp-^f>r4bvdypxn2+t3};n(tCCjWDdTJC3hXR%>epzg zy8eR+Gwe8C>fyWgJsig?&}kFI!%c`Db3HV%&wQq1K8E|b5`xJ(4HvE}L@m}jEX*FB z>$Frn?}pU-2m4gyIrH4Lyr_Y#N_q&pTT_q&QMkaxxvVe68A+A>g>d(t8h|+mAHLfw z0&{NIenZv$<^Q+n%9!-a+d2 z%JyN`VX)j2gT6owDhty?tv{a|NSDszhIapSfnk|987t=Tli2nuJD%vwOknwU61R%V+jXdr-m3JyoL5ATQ7BSE*>-?8#ZW=-;S;9Q=zcV#F=$$){hncMJG!o7Ex}TI=U-L zLn9Ii`u+Q{L?8Nkw`qGi63$p0#~djN%JTk3-T7&zcifh59V6cS=n9#5sag5tjAMmX zl;+vDlrngVDO>DkrL9N%=)M6-e=tBO3`<$wQyqd2qCEoS_r_bZ1i_gVrYv1vx9Iz z{P_os$f@ekybpc?uPDQ#S7zVLw1I@x2W~(iCa~|7UyU(S+ z0MJ*-$M%^2pE=S*c7r!4VF~7cC^`K+Le@E{%1Be!IP>p4kPHxc&v5}h6Qw2FFkf(x zEyfY-y2!fnf064aUbcC+rqYek+`KZ>^OVSMAqp@L|5kI7Vk7G4Dbg_3L8NWo^TLyL zt(Rt`GskpMK8czdR{?TL!l&tV=>G;4pg*#KBECQ2t?+Nei!65FD0x!V4uB0<2V9fd zmPc@8IFkujvzg!TyM$qxqO4Fz(}>SNI|+^-oAv_2>xmSJ{p>{-P2~pfnGv@&Ncz!c zYL^v7h%PAABJ++B`Us!htC6giP}^XXV`yk!Olsl`6*T_`bQsQ7oSx4TJOCX-!r*Ls zqKb}|b{_TnXn&)$tC(bfsxZqS_r8dBNVc;EFh7~ z9X;(-a$-%wO2S`Ia9e#kiaL^aFVb#qGrBd=E|z=jCH|3if$^K zhe@*qKdUP|1f2hsbt6!L^8mt+OiSJmUd{yzXyjK`;9qut2iZ|_z(X&vr&`Wd8 zctWLlRIF*)Wcg_2{)6rLRIS7{00cPK2D3RHjyq8wJ>aEIKduNtVzy3-SsdVT3vRAo ztmy*k5uFJ+whEYU9&Mi^`VRkXBL1Ji<_Gw34)PcaeRAiP6-Gfdj`8}ImaG`d9YK2* zex1dH044|1#9zE?nqtv!z9o>Ihyf>IBRAfLIQr;nb+*K{zq#yZ-6SSii8jrTsv3Do zP!;8{o)zn4lp`WFo+l`-;Q>z(efURpcfou}L~r0*?gvcq1O2|fnPPzpT}Tn2y=-+(hUnG|iJROR@=3$QJYfW-<>7uGhBFxky0fg1|r5k>l5%WM9zS9-~Iy%WzK8 zYWHQ6@C z4tFCdjSoSBjlGtB6enF*tc|faAV6xDmN3dhIb^`Ea&rQDZLa>kX$OgtBuKA*b@#wP zME^-*Mu^~04k=^_n0u~Ze@-s5%1>`RYw=c`f+2^@2OZG!r=`~FMgg-lwHW(>4-0Le zV&l154fdy@Tva|UlF zbT0I=^K=}2tbLcN%z59AZgfkBE@8hR+}(IGQ~;bIDw<&hrq)xp3qH7i`G@RySNeJA zAvM2!1L9Ih4eEy6Fo!f;WMNxYBuS4$>2bx*qTC`F>7^-TcoQ12%b;cVj*eNE@AArJua=1$$p6T%^3sS$ zyqm6LbtO_L!&fv6Dexyv_yI&sF$1|p=23iJny+42ha5MOp50#In=O}z7?oPQpjt%P zy7;~#_CUnrQUIFVlHdtTfpChaCbw8{)zV4A#8BLSLij#loHkJ{^6nAP;iRo7_9T`V% zit%`_590XKFxX}ueCJ9mWq?F=m`4aX#SW@QP4x5=>q|lB>|MijK)+J*#-Lz~79R2O zz*&RW$isvsc~}KWDguTTPq^8}hY)&43E)t}Z(p|AA~UrLdWNtIQpx6~dPXC($_d3P zkIIY3Ra8>dC8#ZzcJFlArubDs9+C%4?Dwx141#&i*RT(XJ4qeN9Ne&*0+$=L@vq}M zgNW1H-2u*|zc&|J0w97>!u6wCuZQh*5%YD06lX&}x}`*g4aX3BEe}anky(|JmwIIM ztO7kcRH5!$>hqf(lf@onc_vWM-SUpnvG_oI@E+IRNagPw*o;*mJJwE_)*#3Q-4@jX`Jz(_-7wCAdR~cE-YOUdJ00VFV zudb6;?s;{rZyj}CuL#Q#>u0X__@0J^ZRz~}e&nuaar(mq%9FEOQK+jcG?lFq*jG;bC{M-&$Q^$WqGlsmZgWO(553J3!{){3HKIM9w?Iw$M^a~ zmu`Po0sz2&ic5rou&3QUe~bBWqQ7wc+QRUL5Uz$@A+i$3P5l&%ub)XSE2*q=P(2-{ znFe1;i)!C*QewLQl>S!kG>I(F59+JD@QQr*!6%iJ8z^&d5|d1E@rAjWLmVl4D8THz zjy^}FX(z9M+(YRV??a42A4K@SYb3=Y#0an1#>n4ZC%rYecxrDbiiMx(mo6cLQl!Sh zAD!i`l3r^ul5MC7?fT|30SQlpt-HLJPnVFEaI!DrH#lfc%tX&~bVvwQUdXDYGcTsXM3dAdc`}mxy(F;8cJl!RN}&v+cQh z?0>_H5kMFufX+1!b;F;E(A!?M2IhQX@Tzf40!{-K^lgbJ)4coWjo@%*&==<*PH|Pt zipO&he9i)>ALCb;U6ZCdw!9r9-z2*FoMdL~tJJd@ymm|h7~@f9jN_Cq4>HR*3T83xvFrRF@c1i8MAzC|O+}-Ma|erDHa|IabKgu&7Yi4g z6gyeoP92Q}{wD5x+MI2qklzqw1_YWOp{D11(z1JAs|uiq#74EEd2_B6Aaw?bGD^#` zpu$^!(-0Jf!!o%1Q`4b_d1q?Sg2IEb`i2KIPqpv5HB=#&G7_VrA{sS4jW6h`Cx*{u z+glf*kN(wqPp9OyttQ;~<$A*-0yg`>?ESn~dJ~JGr$#*~L?}Dn>NX@j(Uc>O*Ca0k znuhuQl)pzPp?P@3@F4MFnZMBdQYPme*xb72+qFWM!~*sF;qANWpQV82MB5#X&0M@q0%P09fz;;Goq4 z_9TlV*N2{`AOD9|gSVsY;q@Ly%e0}PA=AbCPVkkK&qS{{4K0mFaHEMDZx5Q9Z+r?^ z!i{6Zt#2?!GMh(?~XAmE@YzRj2gsa*&K?%)S{g}C4h)m&e7_;ZQ;(l@%+@(BEcB&u06Ee9uCI4F zmlT)eoEZ8}q(o}fEvS*8Yyxdb-rNtJ&{{|2*|#S_k`&|(FvTzd|zRhH8}|MVfm_n!f4Zy=1|`^6hkc_;YTXXJjz<1eBU z8yPZh?gr8n7!yqoidLe8?;Tz1)k>i}U zPXD~VM<%Fj8|{||bgO^s1vVlWaS+YU6~U6)A~Z=7mejLuc8$GHF0ZF}f_{O;Ynfu1 z0i^hB*uy^NcBQYK3cYwzo8FNP$O+Xf`a3ATZ^_*rr6HIIdzx{O9hIZ&NLih>C}Vz! zOx3`JVk!wO!|~6ffUrjahvyy|-UhEJ%IkjKsw!Vzk|?=SJ4<6XinGh2u30c2AguBy z)@Dri{3O`4*}u5iunG6B!=PqM04Hfo5N~m!(VW&75BYQ8x|kTdm)ciiLc4XgIjj3H!OjPtgI-mA<)?`QooYG)~$Qa>@DRoF* zUB`8kz1uBQ%c7=c%K`UzQ`Pjbj7jFMs8iicLtCZx=#lXsQxXk3|3*Tdi7%~>4USM^#=7Vt$#c* z^A^aK?~4g}5xa$9;#ecReXu>s2FW~?wtL-t+c$hXg8ai86x8mWcCZ-U-i@D}p1x&^ z);jSD;xBB^`SkOPbnewkv`#1snPEQ7XnIgH%rMBl)IoH87?F5;gH;%IoLApZKR8nE zJ?H(y^}0a`uH2=M*r>hn{t_f|RTIq#l{OIouy2(~aLdui^u=yR=0HA+cFDAUGJC=g87CC!wD6-F6^obd4991k^ z>>e@GQH0l7jZ;iQ3|TN?i#wsy_>xnQQk=(Vr37BSiaPohQ=sLn<&d+U=&r2lgYT5{ zBr*S2o$LRnRju#q{{|u9k9MgT1itJ?=f7_#`$l(hT$$1t~Tf z5I(%WYm+hvfITl@vEgBIqm@(?(6I`rKqZXJjgE>o@6f%YppFgoERR47YynGaSEIc+ zVZb7oT!d{6R-wy&EaYv@A*n}9YM>$r%sHK51odolSJIFB@hngqDr>pq{wpNn!ITW% zvF#Krcf3WyB1BXC1y4R3)xRs|O4=#*-mSdv$9nILe`a(Z4X;tJ&q29xk9n z=I&kzJ?{&-Lk5KdID=%*&xTi`w+-I3Hie&oxTj$Ym5BekjsJUC&};c|*4FRZ=nkjR z#r#BUV-9$_NM*2b9#KOFNI}?n;w(&djYDMbblmwH44kRi2Ec_CaM5HJ-N|NQGKWjRip`MXJ$Bxs1Arp_m?O3qFvz|*o%{9xSr-Z@*(HxLZJQ&4Wp{z+G$*fx z5TXyR<2+ZLRMD1!4c;w{moCrjG^p9Au?n|6T(5EAy}lfg+g6COK*Yru1=v0aU%z7U z$9jC%7vwyUvWE^>emGO;|DGdhFlmwUJJmuD>NWVM&h6?E1tEW^PHPA^QLpeZsZ?wn z1(_S0w&iM%cpv{^7QZ9H+N^~xHd zHNZh_aw4JM4}vOpQdkUS=Tk}08%c{i73Vrg`jONkaM7Xi;P zD^U2ZX-M*O6coiyVBdw^HqT4Ug)e^5`^N>jFIFlbho4vDX9z90ku0Q9CGYQjME;Iy zL=eLG`x02|{(xZ{(Z%X9#g&xaqXUR)&u;p(X9~IO+#KBs-N->7!;AQ!+}s7G`zI5t z&?<&sFY+deuUmNr!jh8i$DHuI*e(c-n5e*gl#VNPDT_BA{(p;k3iL+$PZa$MX#KE5 zn+_CXPwAr}YJ2MDs>+K}#jFRwa%D+rh_A+>YlC1h_n&pfrJtftu0cbv)dX~yJCGhf zT=U&L+Rt%h40lypUNjMpuX0%4bo6`VYu#I!Z~r+Fb|92?I|y0_9%#@O71*BLwE__p z9aZZzlv-|6r$!s4G6Fhe7XZG?U(aCC_xX_TRt-t0FXlia;<2Bc7J0pz_SD57i#Vj$ zat~3t;^=F+)ooPRA@4p)4(2b;VsTe%&NS!4DYyMI0XZmjB(08S{;p(6QP9wcxfP^d4u z=YDEaPV*f+h4}WLAwQ`w-xeU7>Lui&>w;XTUd`vOqn)mAiMqeuQ$$7m!gpeBr?T-~ z8Owg8Gi0t}HYqBwi!XOd)SPtk#;uC`*!fD1S^6G__2wF*`QRoagXc=J<#yq9br5vX zE3kRNVsrhw{vEPT?-=z>3~E{7I^_EWa+I)Gs#~nM4<^e>Ik+_0cO}}8R8!dSh6Q*N z565RX`by$9=h?UERUh2eg_XPvf5<*=}y(0(xU+hL>9K9`oii>X%e>%| z8OP1t04e8xfZGP0d_*UcEZrDnglHtd(n>20h|S*$p5og?ZH3gqqk`}cT={D{Oa44u zCi;>NeWyei@x6`A`&{Ptm~_uqBcKm}sS-ah%(sByGE(MT50l?rOxk&p;a}X`-AW64 z;jg-msKUFRxe-Ung}<~mP>I1mV(bYDVWlyt6u8(y zz}&Q0;1;_WJKZ$@WG!P{zY*%dDf?iRA3{#3r1d6A-}n2~7dOJs+u-qeKy-Zto)zM%ij`Y|^=?4SI^_N%|3*!?6vGwfmcmN8)}PvTsnNljb`A20MU zC)(6p1t({sM`tF~)N$eG-ZZ@0RTS+UbgxmO7{UiuK-(f8sp< zX?cG$^mw-={Tc(haS=$sLr8ZZq?=ac#LsU(klR(5S2aT=^J&LzmT1ezS6khX?v=`V z0^<-7UCpRiwcdB%O@9tFUeSt03Cx9gI51ri2F8z2YvexLu#_LTzi(0gSE`4Dve<5B zv9F|f-NL9`x;8<(_>MC%M#p!VOOF$@3r^z4H7HQ_4kV(Ed8$fPJP*L#1~-2F{y~wmG}15 zB>v=JFvoM0mi*c}rrse`FIu*BI>dgf762RNMh}#MUsjnZKoT=<=ctV}6>&^*u&NmI z96sk0-Us&z=)>dzpXP4oe0!GnA)$HS*#SJ{BrMqOGjMt+z1Rr|8f!6WM!qv zq*7K0c=L#K0v^xv$s(DXd8>W9UfpV-94n9W(9PW1sw;W(fz;P67-slM{Mr_O%yFsj zkmHggh|-_dMGK)GnNv?+-qr8+s`_TP;mfO_eZIVcw zaNbp%qieckRmMpDk+2Q0Lq~pL#V$QcrGOhiE54Wd8#EDEa1i3wJ-)|$v!6OK0|`=OV(AwL-R=#cYD zUq0i5?R>PBA!Ck$kLJ~#$=1Iq%O2RzvVUD-lQf>5xaz(x;XhvgT0`O|y?qc{JVm98 z`kvyKJvSvQt5Gjl&XszsBTb<@`K=RyCa2P9MGU<#H(#6VR>Oc zBPXC@Jl<2BiXQ3!HG7^Jlv}73jLSlo&jEA}MF$M}&(*PU0arCGo7X_Es0K=AW44ix zTV$m07z9%%AXfPN|6fxNIt0j74@AN&on7OV6LSACFi~GzWauoQ38a9$A}S0F$_#hk zK#+_#|MF4*^!X!NefEZq3EPJWsl+#H;OE8ia+nhXo!;+MWAZ6#us=82V|}uC{B2-a zviMWCDqbYuSks`XnjrI$x1r;_^ZY5#KIgLkqgByP7 z6d{=K5T?pK@3z_2amh??r%R>dr}Am1pPvdU@EvzL<)`Y!Y{(Ov#%Nh;-|i;Z@os#4 zaRcS@hwD3j!-%UEd5C{|nP9eYKj^@4T^J=iu)(x){>!U32NFN(G7V&=X-OTJpFoJG4`wOQV8DVVO6a{2Zt#nWI$;~6&eQ2b z+73r_#r2JZj*#?Z`Zw+(*qB+GAZu|{e_LZ6k<~Tv;UgZSthPXqx4M$?sP@ldp~!g5}>)*ChXE}}5;IVd5VFU5AWIX%jU)Qxf;wOn9o1*$9j_JRSk zN=kre9eN53eH)T0hG>c4l2q9jo7-;60Hr$8fO3@tdLvsBx}^s(^1I(1FybScr*h-&#*F zWGy+#yXX6%@Kr8uCDo}IrBk2wJfP?0-2A;KB}nCd>(LraSF82VN3 zWSoO-;{yezD=E%C?8z#H&SIEuBA8mSav>EKWio1rz=!EUKV9Y>;z~IYoT>CVU-=9w zr~LS%MC#nz{d4enGdzWlKg78Z`C<56IVgP=r@MW&I=*{x2em9libQ^hetLJCN$(oe z9*>^p@hm3wv-!M%i+({9`B;~;!CF~!DrMhs=;l^>EC}a!_(3P50dT@_Z~0liS4GvY zmW3(cKrp`F5SAht*^r|8Q5?vIbOQP%?(1Z**f(_jESR|vYQ>Nx^Btl|SI3ty#q}&C z5JdaL6SqwetPx{&O}gr<1YQ|D7RN5Gx1jFRHXmb2$c^HRYwTr=aW1a+z6$^7wJWc& z3(G>kaN%l~^p2l<69wD^!MFd~YFHXuYOzk@(@vithCB^hDp{J>vKhmu=b&a&z-sN9 zjyt)PUFdD@tkV?=G$!VA%rn!AZ$)pH-un3Aj{Cy>%s~9jM*LsJHUb-B{q+|8G6-VM zdNqD^N#3^c9(`!ap+Ot9V>tyCG;rPIQa5N>@vBOK?^B>(} zAe3IJ^_W0rEfM1JI0gKC9@ppfINxVAxy+g0b&N>U#+GjHRU@EJOvit;6SCOqjx;2N zAsYHC#-hqu{#)$kq}#gCWSPp{l@Sg>9`B@~407eO7Qn2_$FAoJ)F7e%$J944X2LAZ z#@N_)Hnwfsww;ZwjcwbuZES4Y*p02{X3u%gd%wRhGu=~NU0wY;&AwAfQ6c^-H$nU8 zMcc{CwTbNb7x{@WDZn}lv1gQPe|Hp6I>&KS{vR}5@>dh_tY35Jl%n@KhO@kV&q`CE z-w&-HX({tN>*CH^-go8~->s#hKdcLm57k_V3D;UhlnWcH@gZD-$p?@- zTnfi;N>e2%3V95TQ{eSTawn6D7kNJ09KrbmP5k||zcT8V^1BQ8+KX=`@6q4mMh9n;@P=A4P-q)gurV|H8lKzt}as;VM zlI|okNk`VO_&^>L%WV_GWmDdVuYQqpfa`xcU|+%Xw?Sod#B@yyAOu_u814ScA}fcQIa#P6oLbzx=!UcoHlk-Y{cN*u`6;fAL-{r#xXOHmQNOXb9Q~D>f0@>ODJQQUBFrJN)5tSl z59d z>`)vpN)rptTM5N!L@V~gAQO%#4{jnsOIx(40eAQu=567W5a}bf!$}KHiW0jx>tVK~ zA|jZ5t6{IZYUWFF8o(U`6fQY;@kjCsj=xyAfe&wh@~HODG5!N_5InynX`yfUAeA@uVHW+y3xZt=Nb!*==gGh0bR2(cyy+$P z$K6IR8x3`xaJ0JsR42LwC*nlhFr_#;?X#XB>8Xw=&M#9FZL%(~om)!kdwhkABZ=9dt zXR?XX*wfN9ZI$>9yL}Si5X&7|qRr|w@^p)DA{u`6W{t*GDuP8YE1+I9Nb{1uAs(+6 z=BNm!ndsrQH}p;PP3-I$>8jq3>l6DpY5v46n_G0%E2o&Y&6jf_6aB6HO=SWm9amiO zzCw4lFJy_OMUZ^ax$Hcv5Z=yNg!VaAs7=?sg0|V z{ydYHqCP+QHjf231l;8%f)$jd!MR0}>#=L)o2zebLczq)=Q!Hp1h-~D3q{UfdZOKofGDJKVGu}7%%sk zN!X`c&X_}mz!DHGsW)EPA}jTFaDaq9$AEj(jQq_$pDP8=cc6c4-}RU}GeFh{qi_YRs=6ioahfII+=9!APU+8eJ(Eb}i9m!^hAcN6IcXh1GGWL$uJPzzm z*xBTqx`DD8{qdy)^UTJDgfA9zcLxACwf_x;wGlBCvIFIG|7MQGvCC)!VScVj>jjL? z#u;1uNHEX20G4~Nhuz)JIIc&vS4v>S=pf#9@qw|Sr}FxE;&uH51~fHuzZp7=#$e#R z3~3v9Km)d&mrb4gUL#%n!fWV8Fd*>9oT9iNbc$s7_F5|A8lirQmGC&#Gx`V3b6HmV z66We;vnVWPVlI~w-^oD#)s^nHkSKgbJnpG^c$`EK3;RX_hc~N=LKZFus>|VtO0D`U zG*JHDmlY1w;#-T8?tuW~iwj$0V{Y)^Tne+ruF}A`3tx|meD`DJ_c8)$yGO? z_ZP)fzLqa3cqFxB>gv`gOOBbN+@q{O#)3E5BYJ&b*zGIkOL^7GRsgcf|M)M!7CJJVzp9KUPt=w;bpHzIA_~E*3VRXQG6U z*w4r>hqq`Asv8fKo@^Y69Qa}1BR3!u6v^%$F94S^n>t*WK zj;}-9m5`*$3xe+3v8VU4WbS_~>%GWR7wE z3C&M>AZ^f{2CVSD_5I`20s7+w;Y0qgL7!6vN3y*FN0V~2&*G@@v~LK~M`L@EI6EI0 zT>n!Zv=K8LumOE~|3Zt!hG+M~M|siZ9A#f{GvA}J%vt$2d>u4BqhR9udd><~=| zw+O&1g2?54=#wYhVzsJrbKWl?!MM2qmPX(@KERNJtwh-41@;@Nu^#*xiG$iuIJYr& zd6^z9`^y~&cnC|-(%lJ{%%K<1Es}KbrJk*vx?KxaNXguR3qB&!__9|wCk3*0mokA6gLx5)P#(k zFgEt#ob_JM?DhD=c-hqk%!~`F7%DrQZ9&3ND3mc%#ZRR1f zT)(+2#f79L`NLpxc5_ZltMgL_t|+cTF~;+w;eG5q`vlM9W|y;YeG8;Z8RR{!38yh%g2(`rV%QIe4Ni2Qxd#(O^YGCBfeLz2q zeO$s9F%m&o_xrH7eRL!|8s^&m1wCcZ6FBEaH!RhPuCGon%&>R&cu)3;>gU`+)UDq| z6sG5G>T+p9@AW44B}xo(!^miy?K}NhWt62Fj&Jgo+$VqyhH6eBAB3IA^0}jIpA-s#_skf^WUqTtPos%D>)_nCf6=LlAxafq<;Ljb{FC z%tJtbR#SV~F)2ahg~aInHeK7p$3qbY20ewNUL*DLYsO=D?(Q@oUp=b>uoHa|ysgzc z-wCYewmWC-3sc}|!^hqb+0~Y>8>uQ}iR?>bZ*31HT@hs}^AmEa)8pe+IPN`DZG)18 zK?_W+R*&}$a>cI4p0nQLDkAm-VFDbp+LMJ~&|5xgtDDVkl_OUqZ`@aLxbWrK3WjUP z*Vkf9I>r0v@DHG0E*bZgQ27(iqbr24PGi@=L-M#<3!x4AzgEhEhDbFl)N_?6mQOHD z+Id|~S1(?m{euz<(wX3-V53AAkX`}m zuZcmFebpyVzvwX>Ah1Eh(66R(mP;y5D9Y1lb|?7v9;X>h<76=Xmn(qLpO15FKrYFP zbCgYtxif-s_JLNUw<*!=Gs2%;I8;Ibc{0@FnNXb|p75@Ba#ci2%@S8cRw2xDcB(jg zqH?}D5!)gT7nTcB;b!D!_GDu1H)wyL;ZdxpK2`l5SALFz?heFR>FXE5r@ah9ovH_Z z=!8J4L4grxCCvgZThjOYWNCH4oj$IQT&vapmm=bUEck-%o<;KCvB^x=%2e9Zo@U|! zoH<#pj7L_8GngjN)-l4`c~gEX>=y{2@RCyO__qVXW^#S|&g(lX-13C|xcp*;dwV|D z-*#bdXyFo+*9JGv5kpRUNrE-P*##TA;$-0CgaoV&07%mgjmrdMc+RZzysvL0ef&F|akO<4TU6wvCsx;yFlgbgIid9axl*pkn8XzbsLq93U6Yz#X z*lRwcal(dQeS@%C+btx09|+RY;N%!+!4ird7B^P_c&=vq z&Vy2hSZE|fp9EMsZvKsB=?Ek%}#ro`H6=JPhhciA)b)P){{HM52OBO6!tA`y3qQ;y+plA z$an}WQbx8xddbtZ_>{bfFh05RZ^rLrqGOiwg)Dl?T#ghAp`aHsk5w>!6_t{M*bqtO z9;f5~n0I$@ zQqfc;&CH$5rU&KaA2>Upu<5 zX%h(OWe4jJKs>qT?xEh%S(405C#JVps1WHPd#WY8V-^UM4KOriJr z%Hjw`kFD)KhXCg{U}N^$fs2>{Q-|~7bszF+OGk!=f(-*EvSA&DWcL%39LAHkhqrgr zdhqyxJG@~ZQ8*OloW+-;hQ)#+9oI-ESig}5#VA_v$YD%OMNAio1FBGO=^G>aO9o?3 z)GB$j8X0nA%#HjVh3xOV4;usBhR9FxTE)imFe@b`tu4@v6^|3UV5Lj({ip^Czy9|yug(5`ElAP(JG~R(4@bpMLhPzi+)6)^DB|gxB}qMFj+oJ`+hhdgw=ycT~R9{}Qn>L@T8K zt%-jS=8_j=EHxOhv`|V}K=|A7Et)2kq{PREw6b*yw!150G?YemAIP9u#fwhM{=D>V?7;pmrS z^C|Z>@&^W$8 zk@*R%559W;vyfivYs%Lg$7sGkmPa^}f{6`+CPwQWM0{Ow8RB$=9*8yvs+}E3>&FOH zu&BMq(65urdWXBA0Z$P7DaiiOVI82K?dk}zXMxu{fRIeKO}zSDqjESo2}C-Gy+JUCDNJcOEMF8x zXkNOsHg~C8!v`=@Jh7m@%;^rB%aI+OMswAB^pvHcpA9S8Vcah;bzpj7kxeW46Ei*h z9c<(^w$H)yh{soZC(ZQo`Q4v?G{SD#=y~xm8Tf9W=i0tlhR;w57r}tT6ys_ve#3Ic zp`yd4vm(ua-*+0ySn#;9^F`SiN>K?TIY&j{R;lwQ3r&L|m<1H;>OH*T*-ov%>kJ4a zIp%gvUV2ppmWkz2;5bH_X%l=)NrzT&s;z$pyP8uSsg!kj*uBvP`~PaCr*$7j%uxr= z>-SXmFkbzE(QncNf^<;zKytrzU-Q3G!yGAbk@gm0=I>nX?ZEzvmD9MOWHRA^e6IXB zx}cPwyr<`o_UW)OR*^UcNx%zcWzIc8qa)5L8HT)xkQ!mgwf9WC6uV@1*tuLdgLFa<#V8?nl3xp;wGDi7>?@`(%9uwuB z6j3jSz?8Ju?mU?*2b@1B=Ys8NR3^s@X*0x)*TKTD*L}+#%Mw&8X`8W@j$1VJ?_2TQ z6vevZere%}d~-Q%QYqwuQdGg^{d1oi!sHJJQycc`YjDc z_c8WC0#~CfN|ul)E6Dbg)zpfA+&D47$B^>pcw8(5rRLa|Qaa5NDrxXlqXS(x5MQv= z`c`7roJC!-EGp^XF394$rWBJ@F6+wV+8Y)I#;{eyzgqaOag&lMh`th|wuYK>NcPAb z+TE3XY$nN7U7L6u1ms5{fg6^WrJc zM}zU|_S_rET)8Z`P<3EPL^PY%IIvW${*if6Dq7oAAc-VPmYAV7IXOU=I2#wP#BODn z9kZvS7z!cPBG0(E$owe%A;|ZlTSNZN8q{9QfkV4 z9TdMF8h#$)drzXJH0;n&CAhY@HG5zuid@(Gjm6Yo$Np5E?2OWg1obw~hOGFDF)Q+J z1vHh!T`7*84Aum(?9;~nCQw1QqjcMbl5qo@)_y1%m4_vNgl;XEAsW^vRi^87NHP`B zNE2ECXdv(d-E-wNd}fM9qQf=j{JWs#%dFP29{BpcIF-LWhhlqd*h??WIzdidAhGLP zAkUB6lMdc+Vj^9rVx6qp7dIXNp7I`u-4$PoDd2Ypz7eQ@7;;vU)bovHL{pAZI+QA1 zB592z;zWyqhRi!sL^p68!}#3(0$D~Z1dI#pE^(Qvpz!!IbAu;2o) zn!^twN(sT-h!>P>1=9MeW)x;!Q4hxD7Ij3 zeriY3tC;EM5#a%c{W@XY`6&q#)wByC2^y|a_qbBor}NQ^ju6Q!;;6UWZ|a3PbR5-- zvdBM!`1M+9lQD|mQgg7$$;XZOi-GR+zrH9-W~hLQj6OHO9_~Gm2Z+)=!G{6_Wn_G5 z?ak~*RvyiXQ`Wovhn$n^7px7M{>LPw1QkU2Z?zOieS&7qZh%~4lpQOXMKmVlMayBx zYyl#FR-z)R2p`O<&tcnZJh6~DkCnY@eAP|&G9~ZpNQ;lKlPv%iFQGw%JWc3Mj87Ar zoP@4;rW7O_$e(1P{zx$NYNphONuv*VM7ZiPgfh~<;eKi((JoE>y$8=Xj-luEQR0Po zR3OKm?4Nwe5^w|K5-&84>{$N=&qF!rT{jHzMH0}PVD!VjX^<;xvY#g_p~_Irz!*P( zeXo-~Y6oe~UWej~l^IvoKPs(Z z5g5C(A19pe5v35xERR3`SE^-QJeERP1sdBW&>|~GZs)&ddU&goM1CED7qop5F7h&iV(F+%L2 zxoD4<-ucOmE5GA(@VgwJZ{!wRf;VpN=};Z1$&?@x=dGx-z(v#1AHoe`)MdE+pbXB7 z0M9$AipfUrR#~jMRrb6yp^S?_F>NH1)X?b+6X@jctcQVLsA>x^zANO5S@y@DrlNUZ zoCQ>tE695AIcaAhEX_zW1c3>^>YYpd&N5meZAoC_@MlHVMiWb={WB63Ycm=QzeOX#cp0SvyM=0&lHkO8w8 zA^q)I77Fa2he5NfT0h1BqCyXuwLHPuB_FzlgthaFnU=OxQ0a8; zSBSoCa-FvocOD-rn;v5feC=nm&MMnxw$a}VoPIkdnYp#kEdcG2M+`{a#kIHix$W{n zeZaZUDo+kMl9BcE@G*4ODbUbEKUf3nryx71ui7v1EjWJ?Zj4Kf8amfe^3DV1E>?nS zyxP_ib&k^muV;4N+g+z0S7GEc9XFiYm8F4veRa_k?ug5RvUp3lDx!n_Dl$0uKjrRU z#0{1Zfa}JUfd`gFg>>shh%k7V5h@m|b~LWi5LAq`fTaqUNw5~5S3xC{?9Px`?Vm)u*Ds z@Pbp1cJBmb*nwcvowuTp+J5Y=Vz`8MTs5#p!r4~)S+&Y4WVT8e=U8MPc-K>cQBEHTvrE1%{h(mG2vVo<$? z9HVSu47yD+&58-=B0pPw96mEIndCZJklL*Ff(Y<mK2txZK+I*r?_DoeA8U;f%Uu!MXP2trhu@ja=8_2Wer2m|ET4836 zN1f5Hd&DW2uEiD}aqb9|Le)EJ=2}ZFK^jaaydXkb?0Qiqiwc`vQba^`;Dob(T+R1W z)D0JC%z((jgd2TV-=VoYre6187{Ys6+OsQG9sTemEQ+Ls2>%lZQD$IbU)}%Q9p@DX z4zMmA9K*X`=a6{ryAv)HprSN7zj6BlgYkq8@&=*>=2HDmUxV#mZ!9jyN;@t&61~Ag z;|tFN!5!`glc!~QnxIY|#Ue6G!~(Mo8h)o$08&(QeYljKc$QpBeLjEhf>z?McJ~GJ zWTtY=Vsxfza^geo-xyqd-l<|-g}gqD24Kbh37ot!A{vQpj{k#G*D#p-soF_BW;x7(w{4t{=Dvz%*h~| z$mpV~PNlqK%X)+ZC-qGc5T=SxuhjaQNbaNBw}HyqD0Ksiki9$Cv}aCPiDElm=d8(D zjyVOJrdr7otXrb%tL#@whjtUHE+zV5~dv0hGO#@b$!e1pq`SP$q( zg~n!I|6T||tx1e%wVJV{BQ$SVmU1~fnynG-DupjM2^AS6lNp3S`#ql@M{9{U%PCZu zP%YN1;>RZMk z$9F#Ao<6TQG$Uu;{gbQe$HEW_Bv)W&36LfZtr;4HG2ZQuo!MzZxCuCaw<)d9$({lI zno#-Q4v6b`!%^aR*XSQM?;o+76Adh=?h*rg`kc!mti$AQu=Tns>zN9%v{0k{lfi|Q zOt@3*kt}A>ErD5<`e(3rl8ZINM(3&RLHthE~dsoxOeMbOj#GdA+ z056QHT2(k&cz#qHRn`4`piNXvSz~j(7$5 zN-IJ=(Y50?ny-ADqOuI>-1@#A4upU4nWHS&0RAYdqjr1oK|VaF??5Dvd-VhAxb=}e z%XX@xpVQQ-hHcmbSczGeu)4ey#@&R?N>tGs)p`M2%*vWF{@+xF6x&zU1+2hVR}m;) zpG)R^))k*On3Btu3#WCE8%;{d>4Zl^2%DYRLY?#YiL@h4lK1mP&A4U;AA3Nr49rHP zsek2BT&yt~{cYfRK#%4yzdSI)O|4b?M&a{r)C#}%(&1`c+w}tHJceL^{23hndA48s zPI|~)h2X49tJ%eEY&3wc+urJQcnwpqJ9CQgFn@C~$)yS!TYl_iwIHWC@}Sf%H)=%) zxrI$i<549injNt)tYoo+&|GR~l`Qt>Mw>^lRnB5pykfS3dA6p}WGFtjmB#}4etmLT z<(YQo#cfn^l|mt+<@2Z`Pi4~mRM;WLyAk7vmAw`-Ph zDN;2d2XH&FDv?hCik<`}vkl@iuMf?z^cyA(lC)@Ypt0H`RkatXc_j%bn;NJrc0gLI zanGMYkG%R=&it2}@_r9oK@@eK8j=hSmA;^cv!e#Ng~fuY2Cm)xV1l;CI;<0(lBkDPKFBp7I{d?-FM7P583!7A$(;!&0Cj(KxaWn;2HevZ{tEPl44OQ}+FkVQ%*rOXED$YEcue9mNZrR@JGo)2FbLqcft@P1vI8 zRldyV)iuBU2GvyWAi(5Zt{=m{f?W4xit`h`QoDl;jEw+{uB)zbX%^4z;K=(3X8msD zxV$zhi=R;()7Wm&nmGT57H@a*378Z5{muL?o#P3bkLJm><;iyY3t^hp{~(1n2uR^2 zG!X7525$B1U)>myTAR7I!5WBAKco31kiXcNG}E(q|DdP z4w~NrV}vMqEDJQ;Au#|?-_v-Wvc`Fpb#o@~+}L0GtAsaL9`u=@QDsSAbCs*4rdq?olP^p2Kz(%M^9 z^U=YJnJixxe@2I8FtG5 z(tx&lYK+O(QZLJ=a0nh7CuZ#de7MO~mIr>UF-)R*dVbNJ znl6WC!V*w4?vKHo$a9&Vn46draJ+PdS}+HHeRuyL6K627B{=^TdBFVkk%CL!d((?C zhxrcf0_y_zf}t{yo;Hrlc>MBZI8U#~)1Q{lADKz>v7iA|gXGjN+F)i`ba}^Kg?e9# zVh~|%3bGX@1Uc0|QVsCz``QE*ev%lre`M3XnbbC1Q68Wf@HH&L=bXAKCE0AyvC_nz zOK*+9N!zaEYCaGl;88$)Lq;`J_!ul>I&A(OLNo@UTdwwT(*ur1%G?0wgQkp= z!UltueZ({<<(^&?)O2o%U4z(cs^(+ZVe3*HZLaST+}s!6qSDelp(T--qP|9hOWGVy zb5t$HYua*1So|sx{{SgpCou(9t=e?nGFA7>gOQ5H*`klCQbTPrDUr6&QNs_t!WCUr zPvuDow~8dO4P4aL-QVu8Y9c`x71i8)gb=sYh`Aqup)b&ekLmLSTPO|uvi2;4%oE*bN*IE{v{30@2rFt-@1StVH?UqjRj75kydbj zWr+Uc822px!==dh`tN!v+JSsOX09p^=(C+L!8+A(hCdiAA3gVBOK>Nv%6P350{(fUA{J$g4+S@ zDtqq96YU_1`dQec;T?*3dbET`fJ!d-t=pBC{?0?@6(`wlWwA5io-K2ShoBB?8L*rE zjSqgGEiS3539`BFVA2N6Op;-XOcXhus4Y#TzX4J&r@TT%M_nZ@YXvM|-GpYkKB5e^ z;KmzSWnzueB!DU^Ss9vqX8H@3dTG_1(`2*^pD4VN9K=u zoRqGYZtFDpwb6h+?h?P_2}T}dsfSHdk{u3>zfiD$ngr4Rm_$mOI5JU%M5E;2|VZou7%j|Kha>Z=y+t*zSX ztD-z)Q+-Kb^Kl0{&&ZxMROJ1z^LJSReJ87q4fsxvPXZ5pwzn|Li9>oz^8b_jP{G8$ zZ0k&e#orJoi!%e!!`4wZN;B>k>!I1?XFwAPO!72MUIU5QImnKRlItmQ)5B@FDfz?0hF$r~X*X6e)YA+T_E&R_n7tja8T2uvV ztIpM0s};j!YbdO&z23Fqf>SaGJO*cKWIL-QRz}4~SW~8`WqosA|uFs#s&# zD5hYIl=Z%*Jg_LqsxM#If0#c+JR(6Xg`y`5>R}#mCUvzjp9J8Cdc4zM=vArL^uat_ zL;LQGGy3?n(p`EREcT|s*w&WhGF?lv0A_-r<56k?B9W}3;+yx3+JtAW&4oBn97rdu zD$(3FpHd}0Q=z_i2Fzh771dG>zinM}$ACFcLm0@^+?)qtr$M#+jA=9WyVB-QMJzz{ zXi=?xL8xw_u!J|C!+~UyQg=RfRnNl>cp^Qn_EBk%`Eq-!AlLu7ASf`gVL1QpQk|zK zVa*qI$1&&X%b<%uko=&NY6NI5OO9x{&HRu+{I-(Dr>z{>ITR5C9k$Eu{Hl#zdYf>s z#_;+68CNZrT!zX%UbH}zD!^)`SfuX}k-Un?VY;U5w1W@e=6yo}8w zzk*t2t1bDcfr-vAfCOKpIy)paqAz&4K=*+RbI1ZF@`l)?|I2?ZyDB4yrj2l)|7^;e zMAoT+F8doQNjBFPx9#MjaBblmzY_|zQ$5<>Q>C^$g0jHzmC1fzO{nqFdi{!!Gq-;~ zJxD>U$ebJJk9AnTBsZ+ChtR;nDckm&b<7|nn?w>hy*&Y1fb;Jo9WF!snLct+aOQ5J0SB1(}=C$ z`L;vEb%=6DPJcELAsRZ#Wge&pMdkH>2qi-y1C>o6Oses84ybz@$F zV1#Th*HQS5KMl@sV&iXG9&;wHqqf=vMCR`PaR+V#(Vc}XseC)U49|G$VXc=0NI@1I7Yil)@3ZJw-zuWM z3#xp?N3Oz2gbJC(zSxC(kz_P&AiqKXOmv=k01F3P2!2_ZH16;`hilrixA{) zVjM;EG+ecmK7HMVGg)b?B?=W-)Ou$16UR(k1aKeUY&7aT<&#DKy-nK>d<7CeRyuRO z47L65U}9V#7&Py@HCy>MHPL=_X1E(r5-lqta+#Qb3zq zJZ-f_7ya9MQ?-5Qk@K`j^scL>n;FkbO)r8a$tyxFc8&0if2t`QVBnX|XoYDAjGK_N zl|P&;D!%f*-OVq>Cv4aE3v!edHmB zcSK_R8rUT@0p2WmI*l8v&YU zk?4zMFh>G|xoo~HN5PT_7|o&}Mw*tsXtC#0{gqRxo}F|C*uF^-Tq5y$Cu+tU!{WRW zFk7ot8Mqtt$JiM2{TeXzjmBKl;wEGn9rYPXp=~!w1OtMAR*AiZSJ>DCb84bcP`>#O zAL@vDV>Msr$Dz*UGKq1%Zy*cudhSqpJtnm8`GtH&ss32rSJNdk+Qy6{kdcF}Fuwe3 zUJs2eS>%+f=csJtV0F_+Co_QA<}{GEk>`_+qRi+t+@dwDwi>lJYlus(&> zEYr_ryk_nl|4}~8An9rZCJE{dT<`0x-P}L-t#5ub&Pwklh+_mm$MOg;0sQP*uwNMJ z_yZaUpT(Lly;@f}``$M0o-)?YP>RYdNt+S7F6-jnB##;zv!uEX=_nC1Q>gq7geYZH z;H8Y7->_K#m|jM5nbut|(;yJ{)mx>9yiCrzHJa&bBCFVPOf23?BD=n#&K8!Yf2f+K zSyM*5?z_0hL)y3QAqB{gxw%OAA@H%^(aF36>zqa zEjNwl^ldOmKz|OpXu^I@ERo#0;Z3L`+<>M$K%AuLYU2O>~gc!3wgF4 zC*>eyz(VxnbHI=4||=^vxDi@B4mmVlN;Z{3_GudGc-n7q;wQ$^ZvEU|rNUX+Fn* zs5bjHng3Tre(rvv<-^;E?I6voxO- zB#%E+x#zHgODhj1h^b|YnHZV;DF%;gz9rg_bc}-$HzXKrn&K%|&kreRO=Un6^%ea; zOnAy=siGmbU>dl*#E_^>k0|6bd;s`LwepTT-{!qZy|hNK@`JV)$W z{q9%UYrlP<+T()Fioe`X7wi-^y4rN48*{!Q@D9V9V^BZ#*e$52;yCf8k$&rjId(N8 z=yuRbkdlOgB>*|0<@hXCb$f`eYfF}gAON(ql#~7eHzLN|u-d(#f6m~pUu~%nuTG;D zWYbJ2Z_W7#rCn~o^hAG}?#*Kyk(TKVs}-3VK>-HLoSywJp+Egl(zz8G+vvIa45$Z7Om`h`HFC~nGAD{#UCOf6tVh8 zR4x$dGc+m?A`47({Tf6-(frK#jgVy0r4Y+rCdV-?p2^uH6O3$Zm1PP-^Q2Q*=!I-e zw{4m14)zLIx~S+K-bh-k}Yv9sN0(@&?wq&g95_%-erE98A94FWLr8Tra4_`{`W_}doR_dd?7fk-N6lx^|x0*)rQ zzgoO!m(mOg4$mh7F@jrK01FV%?kTeJ8}JbfxWs|uZ^qi0hI1B70+iGtC}TDaGKuB3 zRv+L<4vl~xqF*N3024$Ai%Oaa&@XlifC3Yv$40huuD{KEKvT6uNecE4<-A)iY%uiY zINkJngpvQ_Wh-=UZ#gH0gTh*Fd8@Nsapz_}|B+*->;3d7J9S^QixL(R2m$ z(lsdf6!Peua6p3iH-_puN-2u&*5IonipIJN)p1XtyLt>6$P9{~W{dqT!noNnh|L77%dMB~t<#UA5?oi!8*->+7(oGMEry$asO8b`CJFP~$pWs)izIY; zs9Haw4z*pJm}m3ak!x7r`V@Lz$09N0GJ1tu4zM7nFx>?voh84zkpDUDJ_Zs7BYU9t zXN)KOuok{V_++^a+;(nT&%F$@PfT?|6nK4h;EMOyjp(({O~VnJNVZUX%t3OduvR=W zBa&u_&vg#088)t##0XjWp-oat0(P=J_;JsZ2_g#~cJkYVwhQvjqrt z&8fEfXu?~$%25QHyO}7@4VAJ5Q%D=kn2vFDCOuQBGG8Vz-5vCbcd_N0z_qeMw7~(v zujg^Xcm2TB{jJw4_rT%3i%Q+ZUE4P%cj&@zdO|IS-D5_OvifVwcGs_qJ611NMDjD`~!V!$`!+F;wegjIgPswCt_@T>kj(f`C_<6>$}*qkYl#*Ye}{GgbE%s zvp>)3M`rs06x3E@oFatmWG-VXosK(>iTbHsE!5+|C_qP?>=_HQv>>#O*a4`|6`zrFgNx{2hF+ zqb4Bh9nP&c=FCDeX_074o4yNHb4%ioZ}_(dFPll}o|rTqVcDkBbkljxn9-;ZS~O5} zZab`9*Wp0XX{bj-e=0_P%b&@zeNT&Mm}D!q0-apFR<7`YwpnLc^TRx8(cjyKHo(1!t=_gv%-jU18cYZu4zj zc00hS9wqmmYwTGGG5QP!`c{$Zr+;Akv*liUT+i2DQ}!jPYC`m<<#T4H!q{(N^Y;xj zTWQMdf7x^Lz`ihVklkpf8o$f)P{^msr|RRqCArQ>_!W!RnQSugHJhJjsb{=zcQ84y z9FnaV0fEOl-7swQKGt>xRk=_cZo=sC=$hGDUA>-fv9}zrj>bbA@N|TgfHF|8KerYu z5eDN6i^5u{-{Tti6M<>RZAa(H>5)?s;OPrU%wFm(Be}GkZ0<_xSDD4~6HkPni?)4{ z=|?9XuASV$@#p9r>Ie=X-(gNlbB8MX<-#9`8D=>j)feQjo@qzc8jatrCfh?wBqwvx zX<;2LMlh=%I15_RrE>zY*V=tS@s+CTofb-$ z#1`N@XPwm@SIpKNaY2?cViCm~iF$XjhgBqXk-50$b>)5=h5Y)hj&cKJ--iyfm#w%_ zn1=>5cZQ$#lllQH@Xs&f4n7BJxYxHmJ2Ghu`5T29Owa*vB0$C${ESIE)#<@OBQ+xo zu#WGLaP<&;Vhtu~_Q9SXMRZY2yD+?8KNoG3lH(!KkL^uk08;EH!F-e-!Li;FCLqFA zBT$+A9KvC2MHvh$QDYDQmMrtFwIuHBX`25Pw(;!9Qcnu^WSK$zKv~-t{kQR_GK~Pe`f|5cBu=xKVbHGT_Cz@1e=?k8^})Ng6Z zP$qpyL9oTk(L^F9!gfRM=a{{$7+dlbeMOddP9>0D6?O78vO`%c{~uX*$^h%Pb=8ir z()q>S2+itGi836=zW8X#m;?*$bUVVxCA>i8V0ZNzwz&R-&|O{WZhD_EO^{1@K=brr zpL?rogUYP}IQA%I$P=Zfj~A5G{&u@;usq#@f(cDv`^2-d`Jq()G#p{7-ssq{H`7&S z;p0b!{|5wT)F0Mu0nLW<&Sk`h8f=(RAA4EN(Ra*6RPfAt=mtV4pjWj)N z^yFqfdkWF6qW@Xt?jB)5uSGOo*qwa8Qs0FF^`@ij+L!odBKpEluJuqUm9xuCQ7=6F zIbIbOe(=^^(mfJ8j?1$+&L$y(S@rv#k#+?sFM&_G`ioXIXlB>IIiZ_joJijYXR}sW zv4)RT2hEK*oSA9=CYA-!g(U`Slx*)_uN}t$xFfEj4rnCEUlT|Kn0(~z_!@wnDp<;T z>`8SiPnI125e@8toWLStxO2SUQBi`+yu*K+9CXN(7=EY1QE%(UZavfOm-^f$ zPdb^SzsFr*;VcveF|s1?Np?AZ+S?C@HEa&q*@hNuexSBrt=_t*I7yznM2BA68Wj`g zNW}$Rety4cZ`cpX&+gHuftG5LZKv3+Y^iA4+_gG5# zA^8wxE=CpgerZKf6~T`3)6#%)5(w{qZPn80Ump zp_rVS$Y1xv;`CAY3WK}M^yW+l5}@x-?p6-gUSC%R&$G8)ek==^ z7LT&pmj5DIFjwO1;TXzz_YT7$Jpwr|+b2s-$443QxR;qFz=!?g0M0nWZzXRh>d&~G zeSmAbQ+XPOr;X|z{rDY=+6N$W!W&R5US6MrFB+tnR)YEas5=k-?ZW*vc7j)Wc- z;EQ^2<*P2`^)>q_qP?{^!e_4K{x?h-pWPbtr)S#MNk%-^$|w&Q&RCOOXYD&;4(X72 zco0FqncYYV)JE=mo0SQ@8T%Tt_r&FJ2E#34j>%RhkaDH2tr>porMNy4l+aRIk1gY5Ud6eJgtlXd*+ z0l3ITvL}xLEYls5hwZi=%w30K*?SBD1d|??jvh*aDuXh2c?m$9f4pf!l7=28<7k|fqDpwXF zidqKfsI4+@EIWkxvrjnl;kD1eKzQjY8!N}O1R8e6^*KhAPd}8|W|gIX?d^YF=@xyt z@~c<=DonS{9mnu-*}Ju}3C)h8}rdKP8e4c&(X=*6nX?a$N_GQUc zly&Y%-KtDRYzm~gKJ8Vm6HE7F7Y1_yO{y&b5J8tG^Aa_s0T zyC!^^cy_pVU*TQ*s`x*6F3N^@EbS4R%6K!6Uz&~#X)LaLTpqIJTbMT;ACr#e9QHZC zdW+i3CHtO|5_y^X0As`}_n}u1%XI-#&C4ccZaGamk4@A2?DU3L@t}SXXuT+M1T(^;fUYtWY?&!VQnf41YTI6O%z4)01{<_+nCD=(=TcO(M*H%DU6H~lNoO$v5^3xN zOT!qNjkKi3=y;j=*;H+XcL$UheG@9BR!2eZVZkVw-!WRN7^;f$-VjiMxPxQ8?iWN; zEvNrJ6~a0#S-pf^%4QXB=w%(7XpU7_etG?4ltEb{i&5oA>wf_J+4##-Wt{R;;Sqi}q`%-)@u@?70Yi z8S3$D=xEjY5rqL+$?Q=sBTPt?pj&g>)$C{h3_naL6tHr;!ZKiIoMvlhdJ(Nq_WNwG zotPko%fFitL#R(+#Db^Qna6hQEPM4}Mev~5o1OP}c^8|$$D)&lFb5x%6T%8%XW{^E z@z?ptaqnSKr;!=_--uk*>|9yl87&4;RwsZ#e20J}Ar$292%Jhg_&#zdg=ypfhEAF-TxR8Aw%TzN2kfL z`;ggu+8L3|r1csinf`NJag=Ta4K6EQE3=%!>*OU8S>9cM=1Vg8hVh})tHB?(H>6$# zdLa-jMQl^|@5Dck&ApIR)M7@(QpYz9)2xe`>?BeXBJ>}91~rVM0HA59n*lf+aOfSY z8WwRJOnUju+8g(w=~^fk-HDcunyeRO;~Bz5)iuhBZpf$cI4rqrwY_qX3vD#0 zHysbz``N>VXJ+z}SpPmJeE7;#qeb%%;rp47=kLw%F{cl`R=gU^5Je;I3WPXw+11Dl z1UMx2&>0cFvyx%_qgSX{sTyBxwTPmA!R#i65*N#+LJYklY#)51$8M$Rdw{);7SU#p z(X0$q?52BVR3$bt3W7T!G9%A3Pvv3)xG>0X3GXJ|y)a)4W5Ut+5imX%)G|F>;VOJSM%jiv7#QcY2L02HaZK zaEh?`&omP!DOOu@ygy#5SB?ER)aafDk(GM96-4)A!SYkunFX#2livDp?T~6pXGUNn z^FBYW462_#aTz#$_n(xGrpC{3MHt+7&HTFW-$o5@PPVH-0XQ(%6|WQ1dj7{RkrqK7 zvK)SRq1IkTO0BBUW2J8AqT(wm2;e@wp}izdJRe1G5AQss(~{a6TpD#Sm;OVDi7h6}s;( z*rB)-%*nNeU&?YCGy_|`R~PaZigDxSNn{c3UlvePxpbJ)`z4T`=b)cA!qpPIZz->1 zzwp8ntuh|sKqA{o(?h-|5G%ytLvI1Jclmcp1vn=|H@}*pT^(rwr!fU*ljkFcG|#hb zU${*a;TpOwy+YEc_ertSaLM`6jW3lhnqX9t`y?QF7V<4vNTTUh_P}D93xUuG!%d}b znu}R3h2{@(x{FrA{1E2Ar#pJmtMcuTgc-Je2M$~0W}yR(mHW%zp)<@NugT2hp#RTf zEHD0^4#4O5WE!W@Q{7$Yqjtt{;xH@uo7fedHfYzJ3UXm#Q|8`FOZk^KJltr@ysBD< zKC|-f_s@{_*!95n;5X$q*Sv7#(`VCW( zsSF<){&AAdekyAG6q@%_&|#XNv&x}6vwgWBTztntJ@CrJxfslp5|~96;#|p*)5v4p z0A=^y7!P|Yy86YaSG88P)aCG97{8oDXi36+?RsXsXnFp~4ET8IPdjoyI4(Exi{$^s znyK7~wCgDCWXknELgZ6bGa+2wBorC^a&5-gZ_-nbGXj)HrZczpP}=WBy*MtLXO(a9 z?Tv;ME<;_2xs;^6QpPTcEQR84%}GIdCjatwFzx8xp>Uw3vv$jKU;GF#Gpb{Q{SCIb zfP($9)wAQ1BqGdtGRp&bw7N85Yh7t4_?fzZkLE)f+T+3F#&DPXe&j0d7V#D^QVv`G z1$~-QR5tq_b78a%7gh5&tCZkcl33WUu!fU#@9R3fCpjs6<=dI>_~z*kkfLlmWrDlR zpV|Avjq_>e9$6fXG%^U&@$O}Hvc)f$m>D@=XXiz{S8kF`c~@CUOKKBq2OFxL$Q>Px zDI9o7*uz-Vf3Gsm9#9l)`#kBNBk~x;39Js-{Qv$Ej}EeQ7nbjjLXY@f7+EiaGAf&l zjHaiAYsOa-|hG69jB2_`VF?AhJ9R$Rl@W12p*BeugD}7*i_$aHsWy1 zbfAiDqqu-PKAKZH+}#5vR(9vqWL;Q&dX#7(HWl4zyhB5Lr)IH2kTRUOWN96$ncrTk#$% zauKkp-=}n;R{}mfxXHAWoxgzy`lMVCsY1?jyv+C-KGT>L+bX=Tu2g+W`tkZfF$g!L z87#SMtfZg1?U%_&Z98k@%l$UF(4pki$Jdc8-sPN%@_u6|5sRGvtj9F2C-9Eozcq3D zc2N7bdW6xTA{qBbAb^*5dc!3BZLj90YaS5*MWd4_F(3d&1T2e2Jq~NIG;qBg_Q&JZ zNvN?2syMPr&DakfabC{w330n^q}QJJ$4i2oSDzRaLGWotA3M_~A1@~2%qUs86+>`0 zYqU5bxvp;I^2lBW42u68tW;ALvRQ>ir-_Rj@Zs`#9>=S(*KXU}_v_e4-`7ul*qlY~ z3n>oU)T*XoG%O3wgEGjUOPEY=H1$7^CPoQ=J!uh?DCIgFQ{^1KTO7zd(}YCc-HzNs zqFx%1Jpvbc1UMx1k~vdGfD+;y4r|TT4SP;H0w{#eLvKQF?S>Ky314J08Wg_+2CDl+ zrI3m0o%#5hCz!f)4BcQo?^gPSp2Yus`u}0K;hH5p2PA;f-c}T}P|(e-aXz_2`)oB7 z_}H)La$vk5`-;rBoyLZO;ew2LUA7Dr0=rMfFxA^OM-^tS$|;MW=YsD z)9mJ-;`;gPr1@L*t?vDM8^C_nR`Sbz>zY&3>aM%Z+{}4kaf1N(p)AJ-nNA6Z_LG9h z18(D#WdyLB$^D0JO{7V`k-|o*f2C5swdhqebF+qg-|-Hi5VqbM+7$k7IMNq~ddAhB zJg2d7-Cl;r(hXBDmO+-UY}J4*UvpPSo9otP7F?mJR6j+{!7T?QXW}@?0P#lVn2sRk zw=daT1ZnmSE3;aNL1Xpn<9@Vl4&B)~l)WKbH4X89ZAX7oI$mV^pLzrpbvCbNz9}+2 z#`d`h632@#TXQy1Gt#l1UTDoW*Mw|LMTcy+=m9*mUD{;DvkLYb2C##nm%PN;Uq}ww zEHvH6Am3HT|GJroUnAGDviYVz@OM^M%v6K7sb-{xq!h(QYc%E&1+^d_cRKXYhz z*Vlbfw@X_}2C81dKM$B1(R@hI)>7N0Af&83p!n`2%Yi&Zd1tnUU4e1;nGJiIymd29 zsg=U*mojPcon@REWOE;@_!#9Eh4UlHjbfVnar|@ZbK>eTl8^1^Le@vlql>7iB6-om zj3#iwS@w|?sA-IzS%+gS*uvi^I8#Tg>vM@xP6kc7nAaOww6mzK(2($P)9aKUmG{GW z7*Wle<9>2`mec_(diZ~FPy0`4gSeHy9VlrpXQ1cuhK#EI%k}MdzZf7Igmof5WOC0zb@W+ATRSQMENqw?B`W_EkF5PRm7P7Ch!euaJ2lN`=CazI>4X4~c zuwFzB3qOM~yEX=?xHgI?eaJKKqcD0ig3F<5<9&tBQ0-3HM(je;w)ptNwl=%zWC~NB zj;&Km^QAB{gp>Ym%uP|pYy_Z6lNe$%Me+!rLW2<_9(FlSU(r7HXnFh__Z^W0(if)?-F z-S~dAGWe(SD+-7?>3#K*(qF!k1k=sQ$h6xHl1qbY$VACgF*>P`*Q45S$9lxP#lyyu zf9E9aok3_~;aGW1+F&s`DKMb+RnPR&4%M6C6|?luJ=j=3rM+bhQt*=@Wm24$qLw2u!7-ooE)=eKszomtS9RBhVDNZnq- zx)*eHFe#4C$=EK{v!|kAXN@jiXj@n26KZjx2cZw)YNxeqE<6^cqosMfm^V>D4q8Xd z)!p5^wWi81CGv52q_66Y5sg_ljfqpgaOPXP#}95)W9gh;Jo;sq!H(k0S(_p0-{&Q} za?H}5?rJ>`?+YyEJ`c`3Xc-n&4FQW1hJe_1eZ=gh=|Am^afskJ%05 z*dHvB&;7nkcA6vyR1XyLCFC>{4l~)J*=o8Jy>(qyUy5KrjQm+ROur&8t)uooGei`$ zogP$O#IfK@25`|#i~Q&}T=9;|<8Yfkv^N}Zd`_jT&F=#Sfa44wFl;IJvBbdG^>%3R zVa2hDrl3~GV?jG2PPhsZ-Oo@a>)n0A=(`>AojjiFD=)6b?Xsbd-g^+kSkh{8DjyC)atQuv$SIMBwYi)V&s!ht+R^4S-|##qt!6&pP`Pf)FREaE+>73( zjYg^=oEH#SwEDU&K|G01KcsuVk+AzsPAO?r+k3OvDRFZvN%(Za&m2D94E%k5{m6KJ zC%ISS$>aG2W*0id&oq@h?01?Y+Q%jp30}%G3~!c~gL9zvlPB{}&4t6vZG#!%3kXDd zJoX1TB31V5%Z$wb{>@(pRu^q~pB-TJ$)4==6N#&pqt2!Hd|EyOAlR~=g72t*#yIX& zb3Ve}81b7oRfkD>C^XPUyxRN}ikzpwHt;Y^m2dTy^KVcSz}2!pM+dgkKciHF8}A=C zdNP`vk!@yet{F%8nY0xx&}Ri+8VS$DjFN|B?5QEweW}`4OCR=MGJOaCE!Ncb*N+=F z#JcJ}7!u$SVxZ%j_)fP8zx0i9RD&a+WKINytTl|bh4|;-2bBRt=vp-^ePk+$ix^pt*&H0a}hMDlmdq5t z^7FlD^O!c6Zm}B6mRk7806(M1c5MtLH@&*ljXYNeV-V6yT{4rJhf{}qzkHf46e|Zo zOw@9AO5?|r8=RtV7hWBk+UC=2U6?H~(imP-bq+tgz4teij#8ady_e&X*rD%nEYds_PG zGxUIOi^{&zixR;L2z~WJ?r2m%yYq|yg@0WxA!jsZA=CXDtrunH=(P^Lj~9J?zvc(k za%P-TUZxaAuowz)fHNKx5|_%6Qp@UL9UaP|Jk^RcdUaaJ!zrG>a|?BRUgYFo1-k)B zd)g5M$Uhv5U^1QV3eq7u+`ZmD@YBXod@CEpUfPkklh_KGmNARcSj9>88J@w(bU0pl zE#*K=TL0-n8ER>ZESY-HXBPMB(<(0C8hq(tZLmlwV&ZzR)yUd%arG(B#@Yx1PxzU^ zmK0U!pY`|I9EHWq4fLNwOrnbK@3t^-K;>R@xdlNkVgJ%S{EeAVm+#>Wxz{cQi>Z3j z{p{TPT6JkT8H_NOqplusvD(kgS{(yOj7-N~;lcM$mD~%$G=7eP0`iI~NFW75f5o{5 zwA3n>H;SFt2o4FP=@2b2i5M1uv%58%tPiiEUBb`A&iQ&l8ZXB z&Bjnp@q!M6zMJ{B<4vV45o#;GvVik5aB4GtHXE1Edbv2==duK-8)E1bLBZIGM2vYQ z7B>f9a8wp;+yS@{%cqfZMj5du52ItJipbI~ynnu4Ftzi(sG!m_{g?x5yirY;!OpAz z`z(W+(T+sw{#2FLSV8ZI>H+BDFWPZwJMQFlSuD9i@}O_!XoxJvBg(PcV=yul{i$QU!D*X4+}TTBK%rHDE#Qw# zpYWM)zs%&rIF>JiU}WL(q9=cC^)?NsNHSc>`$NjmR}L7uWXRn))C@scixcS_(JfVXkfhA1l1z*LBf`{lZX%u zw<@+&=OrSQ_J^%0(}sIgH~hzCax<4(Yj5!*(aT7FT1V!6d&{sZzRA;#(RGL!G2H<= zV^}<_nwZxGCK2`YPUQW{3=CRP+4R{wE|?et;>6Dxo}=U9yhqSe1=*rAyv^1J}FE|lOprKa^E`~5Q7pA{9+ z?_D=PqK*tilaKGC_Vd%Qg;9aHPxNN$wf+&>(7^UZ{DS4kJJa&q9LgveiqoKZd?;U>p&FX$+G=2^T7=-ZUNuRZQZo4m&`rzMOjta%bfc@D-4evd6KAf z=PD~DL$chb@?6+PNWZGBr0{r#739{nrJ3f_B`+nH_}@0;?9YDzo9r4Hpqg#Bfs(+A zf#tp|EoIBec&L|771dql$&w#Z(3!|aR12o#HBp+e1kk-SprU?sH|*g>?Kj&(-2H=@ z^iNSohUaHf`Rwk7*M4GbQw?n#(o|Bj+4oj*Vs(O)3OBO9mDN*p&73K2r9g4JE$1d8 z_|%8UcG7!w9tRY11>udGR!RCsJ#L&ruzG$2mYXN+4qCX?J413LB!2h2;3w7j#&5f* z`c0aD%qxQ1NBTFViiXt}S+29m^)hWL$RHV?1R-?P;Ob)Y&Eiv9Eu>xwW zisEys{*LpIL>zLTV|8=#CuZFL$J+240ms?^*AK^iHQ%v=hTkr1^v%sV)=+tz?r@$V zcgz1~M}2kGqxd{;&!c?uN=$s32X&U1V9o|obH$n%4E;x~!21+RJGT$zVnD?ZyRU39YmU(&by*n^*Dul#7B(|Mw(%KA0?F{x}+e23N6M2d=RPPTq=vHzme>M!vu z?Sst1SCpN{)^Pg>fV?Zt$BXKCJHalkbp_@gvHr>v3wfF6sgMXqV+rOb-wRFJ9`^Y* zr37k1+g9koCOp0UdcHa~=ae(5NzfS(CHcQu094IZYrr&$`}az4o`FZ?9~J$u3|l@| zjDK`>p(|$*v^G+kK3@ZPkuFfS+;ripExM(_ZhIA5zxjF|a3dd&!~0`jU?3^*)jT&_ z(J={U{;c`w)NS14SH#7k#RM z_h+iv1Af~JJv{<}eUXFC#we=^>4GJm5sngvQ35bCnP}U=K#{f`Tu-dLiBErvfve;H zKKZ};vrhkZdI%yw>0pgv;*|g~e{wN75I#y|&)h5nki=KMBca_85dv8lU7*Pj6j~!~ z3h2o+?lj0-$rIuPk9=vt-ExhORd2^b?U8CmnmBzlO`4{9SHRSjf zVfx0nqKh{K5`hQBfg-p$9QaU>q&gU8%q@oG@muXO5ql-YY;xTID{Z{aRff}Ne>T%l zJbU&I|CQbJ{@@$|GLD3|A)0{bqXTO*>X6-rL{kFEK|4B`qEs+amV8-j@t<+O9gCTn z62nw|I<5CUgeDxNCuCeBv5R~@ zU|cJH%|MC7PPVXLnrv?79uGhRIk-~x6>id)_J=CQi@TK7*0adHfm#2No21zal&;(m z1Mq}*2u_U|6pab$5uQk7-uyUnozzeEiY9SACaOJ%$fdRsHiMH9ICC)EUcNUWhJ$_$ zzU1r`6_(_ZF4l|nUriArBNQ^|LEEgajOo%M>0wur8a&<|d063dzmCm}`TqMJuim8N z0UHn;-d{NxeuXI#r5K-@!4!;Iv zQv&a+KL;!;;m+O~(FA&3{(W|@-Pw;QCPdC`@Zc5ce~hicFZ>jrkpcP^7r7Y#iyRfx zrJ34`zxMxSfg{q#VSBi1X?r(jv?qLcMo*}@%&W2L4U5kqBK2`1fa%uQFTg@$xUA^J=}w*X~1D4Ln;)J7#+FfSuY2ux%B4>-z%MxmIZ|9cLX8izp!$ zoJv*7o6gL~^$U#G%x7M)y-yyls33u@Y0%|EW2p1(MXM%;*VB$$1ybG( z5N#hq2EfAknD1Y7VUqaOIM6QzAjs59*nKd*2KolvexZ!y<#32`?d`5g(3_=vOi&(a zGdSVvksvnDvpqogPJv-@?Q9qSxy zG$rZP0UU2X!8Rm-FrKt4(3o+VFxqb5kkkMLhVG~69JOtFv5yJ)2twXx8AU{`v@MTO zAM)TQD26t8%zS_GqwrxRsn(d=Mzt=fwZhO`M5JUxx_NB@1%xWPANoVRV+JdK22(oD zqWt{P+!#J>M9F-5NbKX;AMw?QhN?_>DIq^Lb7DAD4C5n?UeBV2O48%$#6AbSM~MsN z+Pmr)xw9d~al7*4JPg*&SmAL2!^l>6ZO%%}*z_{d5H(rSTAz@nPRu2CGRIJLbY^CC zH9&>1>BG!-9S@}12W|OIZ@Onueap}Zg7wj&q17etKdN+EM10@KdtXG)&vJg1G*CDt z6$3F*XdPPBT$-?IQgCp94<$UoR@$*4C=0 zwj_A<*zNxelCcUoGLgvt53jlXMgyOd0Zw1u2(5i7lR+3ZzTq8gUb{OmWZ=Mc8@Mc5 z)<~GD*ftoVtxLD5cU?i9jO06C`yUPK~0B=1L@nwqN{*2GSio??a*H3;l0O!pzzl$AAvW7 z=@sZ3oZE_?2RN-)t5N;M0vLL=#t|=EsA1hdhOogJswyR*n&+EHl0NzNdm+p}33YLn z7o66&RId|vR^Ek`fIJ7$3(CHp5eTCrW2c|lwf%q;c<=DmjJ5|W`LU|W1W}^S`ZLxmqO6Sq&bk1^^*0ZPJ4CGq6qB<50$u}} z!U^*tmeYM(RpsJat6xwr;VGM$me*Lwsn4}afvhv%R=d88of>)C@hkzRv1Bct-ua9r zs)u|PU|61=mfM2{w8y^S)X%_Dmp?DOf5?|SVPfF&j{Ll2Cwt0erqHdxkpwE|Dx)74 zH2Y#T`7@7?HM+^MNzYYi^Jk7glnO9tO=eXMgMmWVZ0QByq1zdlowVFqscQ$SVBMAf z8h_7UxGla99JU^x$+aAhGdu`75=^S{N~PS7V7Z*fT;Pr}-MYhhs9X;WDDYlqDvi(n zZ(-nuiCnD@3?5YL_ZgaGqYA2WbH^vt>HIWTijIm!ZWY<&LhlC5fMio2IYrdJ#Ezd8`gpG%Cy!-C#@bm|JAyUwN zzSc5Ux`qPYwdo1;H?;0jIX!l$vEIjM zE6@D*__U}HE3nI=VDS?-Dfe3er&ZjUWW)YF*HL#9WGKtG7i-o0DK=+$5FDYe6PlHU zlY+0l_z(|q5LqGl9e)?&{ZuViC4orM8XjBnHHZkd#h}l5isK{)AASZ8dlYDceC_}? z6sD1KXoFpk4MI6oFy+Txh zZ*qsKBy2iNG^Ess&5KwtlGz}`{W(3*&Wgg{r#U!L0Kl5(jeV~h`s zo|Kd(l(}U>Q*6<5*rB7#Y~PkDO^t^)`5wW#vj*%}%TF?goIh2|2>VwNo33szEBpLS zLJPjI5_9ICbfPNVb(KLA{^wySr$-c)-$@yS#;jQHYw8CJj=?x}O)JFQV=}L=x*yVo zba`Z&(vk8Z$FCMmIv7M)WL{={QwN`R@>`1%2aLzJe_!dWg8oS3Nw-UY+0O`PwC%XZ z-H->W{F5}i6F2C2kHOl*B@(Y*t3Av=6hU=e(_veJ^M`B!C+HPQ&{rLTh$yBim(x0D zT3f0lpUx`(U&?~xO-5i4&qZ#W(tTb;-awr6YQ%(+*O`h47gJfk#F)Cb^NN-^*{SpU z+UXgN;hJN;9N4cMpNnTZC{FT3eHOwj7t$L~O&x&|s^PH1pEyGT!Y!g2-@qvY1&OV& zyDi!PjMa^mlG0Dy#cjzEA{$^^N5zj4zm}=AhNoYA^Mvv8qO{*s&@>$oQ!#&-#>w43 zS0_^YB)spyNZPq-^{5ul}7lGfLf_=M)!=B(O5Kh6AaG zBoXaCx;7oxE@7}{p6`e7T5d>-uAvUz9)1Qa$I4!FRL6 zbY^(eL6zpzb3?WlKOc|gWi{5Nsczbe_VhqnBx46EG`0a;UP2@a5)Nh!R#hC;c+9e4 z-Pb`~k|tV}Kh{&eMf9@wR`eS5$qJ^2(b1U2lKfeMu7rqOxhpSRFA6hcCW;i}vi{eG zZ(AGUUt zbGeiPL^zKz(`4W$3+ywjx@kLE=Z|Ja-eTZ;2TtlXDi9X&EF8bhahea4Z;w zoo}UAFxh#yAtpS$Z$D>iupny|K-1Q#=?UMrsjJpo!YPjn2*(bD`BHrRQ0*?w-llq! zdXsZzQm04Xs~qE91p%)JihQs+OSh$wqDAvlW=Y~hSmL7ceiC0IKlk=ik9k^&AW`lS z?VMlTXkSnpfzW?&F(@>DvM+?PvFnY=vp+Cbp-+a6&Z54J;FMO0`OxnC0>s%svT@uM zpnPlMF)z9MZ#ZLl9_^^UJTJ-XdW2JV=5afc$d^eHedVr$8X}$#OV{4-;k#wP&R#bTHoGMAXLiICenmD-J)r7eIGjU5gR>#9d>q$Bs0g)N2Ux z=i2_EU*I2Ydb7bWI6@+re-gp(wEb90Qve68m(}SOz#qN6B46jlq1Jk`P9y zy!XV+jQ2Q6^FmG)l=$nmL7B*APj)QCk@rLX;J$H6%Yyl=(?vLq}4vY`^lw417D|Gz-VbRO;-ryc=(%ZoF8FYD@ ziVF?BlG7Rhd;xtcT9q*O)M|&+tW#damVFf={X2(SDIrUeNE|{0F|rQ(-^Dtk`yRoa zc~yZzu#R9Gx{Jaxww0Zf7bmw?EE%u z|HI+13(j)YJ=J-!5Gl@WTfTlTlA7WAK}!J^*n}FN7kQQ?ruI4K!vEkXLX2RUGhXd~#^^-xoIx%PePFS`Rf~>}iCCou{zUij8}H$EQc{+N6`% zW8jff$d39oJNTtTPHG3a72nwGA<4)rR#|?(DG$HxzOPxv5bSrmpgHd)+@Awyu~3oC zGMZ0u!RKr}I_r0X%-aGuOPI6UIvVn!(=TlsbH-Y^lA<+2S7n32j49l_UC?^@Gh9`> zRk56>WrNoqf84KaWHUfQo{0kjaSV3s%5PY7rsuYCT+4i{wKYz9)bLONJtd*)#Vau= zx%=PRi@jIh49yY9d|MQ7K=H(C#B|FPt0N&}Ex@j>o- z2$AAI)s3X&q9Ig^I!eKWI&lLuayyDtu;zQbNW>8#cOB4{cP?+%A5g7L{}}}cCzT;%`N~tZ^0;7=c^0f> ze)41WpZI0g3pZ(5+l|>r{D9VBH$34AyzK6VN;6G-o|1Ky&?~OvnkE**Q=;vo(CRFH z8sJ~1Bg9WF>296yV)#i)U(b*2R{tzb&!xfVe6ufgNxtM-@Xgj&d!dUJd>IPEayyh zD`pQjl+Io|t+RRzfzM$u3yEd3^N)f zEMH@kypJKot`R*@IRVUBi3XTBN?~1u+np zF1qie+wpBD89!@P$*5)*3Us$Gb_kEbtypw32yToUcJ`0b6Ns8TJr!WU2QNgO0#3)v zLjcveQbciwM0QXLE0TuBY{eU9q9>ZiDPaEIebJZ%1G(q=`sE)8peeaa3;%rJb<0h8 zaj{b8g?N*b?eqqc&15~AxV661;cRqR!yvzc7&d6UEAPdpuWv{Eqxens$vzuKA&wJG zOvZ3ARR_e+NK`j6GjeEAoL1(pWMMN<@jb7GpQGb9_Q?MS*q6J%X}_|U3~b5QZ6Zib z-w1Bw&|17Wyk=jBTDSlTc&-RPUh|f82MF>W-i`)9>R(~o+PMp9gbZVMkY{yYmPp$Q zehHnbR4grts(U7SvU_;^9Z+U@`~EvT_rnK$!f|~E>S%3Mh0)~mkL6USszsCiw;@`A z{+B+P=lmW4AO6Nzqjf#G-R()oD=@Sj;&${h#{xcvgim zQivEUqZGozFiYz147REdRGtVpCa_an_S#U{{?pzxaFQ@Ms`-jvbLy;C=51UzSV}d# zp5riIZ8A3lJM0#OJ;6czBA60hh|P0D)Xlc?ZgY2~){qICLO6R-CHJBcP|j2;QRvCd zEoj+L-^=xF;0@=&T2Cp|f~!EWC2%xsU_$$$iWBliBjTchlE48?6Ivspwd9GN(g7}^ zQpDK!AQwh5&STHfaX6T9Nwl*{NC;sSRL-Jvhxd6p(Jx}p!2%j2&nRq^75pYvU z-g5_&mI>e-L_(3HYbA)BSeg#H%JdSd^EyB4VXv8GiHIwjH~txMzn=xquT8uzkogrf zybWn^9D8upRxcI6lCU=V{{4w3(_6Rv?uzeXQ=lgbN{3}dRaD~e68}f`*#XkNo zRX8(iPmU4c$Qc{9Qw>WAD63Z}C-Ce&W|I_rG}{>PDC<3u@z6g)1AY(n(JZvd(G~Z< z7OoFKZqMz?PerWV4vZtaY!Nq!R=oi-IAu>zJFG~i705kzuOVj>fGv=e36(z6)OONJ zvS4Rr=_`bOM`I%UfGlLTy6aGMJ+L7#PUm~qI6T%aAETw)LH;7 zq!^Li&+-8G@5eZ(tUtC8Pd&JlNWPKCHxz7Br!Vf#&j?oJ>bHAe_<8(}1av9~o;ZIM zGWFeJUAtn-(k@?9J zQ9i{Ml72yE^%+mzFdBM7 zAq7>w=axR^g6Si>z_k|Pa=3>jgy%Y!R~dH8rpGPyuYxmx&vW>K&Figmo#MD8Gf;fL zYCj_ho$QZf`3PDmE@PQ@?XXX*s5qpUZ7|!Bp)5Ydhis+VU<}93xMVNJTRKU*Ac-EW z4x-0}u8NFObM)$M&arg!7(XAU4Ev=n&CnN*{l{_j8j(`~LqEjkap%;=x1d*mmIC~< zA!#-PuhH_oEdCAkr@8%E@n@sQI-fY|@L;Q^20KZ3>qFu!dR%E~1$N>}UBN%*$%c^Q=^8v?RIQ%Pf5jDUuK01uXriDHCb? z4z_<&t>m|zWHOBCy0x9}RZ7v?&t-=`tV;QX_j7o@=#}#Xb9XZC`u6u`yOD<1`b;X< z;|_g^A;A)*|FS`9NkG7x+qL;3Tcg$7f9#9-^rWGxhJI0+5C}@-;KjAVn)xsdxu0w? zr2lVhs(zDIW1IV#5{Pht(Dc z7Zm;RtHb<_UD>6x3R08$i6Y*(uP&6knAQT8$My?KxNmN*uAjhC`lRHeDl_W<`t|j4w*(t4?AL~Nz%~lAaMJh0c)SDW4h(t8*r+^;e~pyMO@i;!;Zr3bL!xuEHze(X>4oVrQ?j3 zl6SUx7*6salg|FbvLT%&=0BJ5hJ>*qPZ{c6FLiTluu6MsIp(I{bzhxHofQ8Wp35&w ziW9lwEgpAXLep)s*ss0$?yB5r%%yf#gs?CzfiJ;m`_*A_PRqJSkFzfIjI9(I+T$gh zrgK?iL7ntOJWvW6L1p6Y0h*@5oqC^b1mPpd)MA~M1+FGCd(x~JC*S3NhI2!3W^oLT z%bjVn3OyBD5~3m%DTWKHiD0wQ_tLv7RTT^5S66psP1bw^WUM+5ER8M;$LU388x<43 z>=WD;30QvdhE5mSe^JdB;L>_CpFfnwf;JgK!Ovkjbe9scNd^ngfqAbU5y^7@M{Vj0 z_YyHvH8D2bT^0^Tjd_SQPU9~S_XaN~P=Pe3#YcE2b^nj1t6+$NYq~Tj-AH$LD%}W3 z!_wU?-K-$J(hbr`hjh1ecXzWiEFB;Eyx$MlxN~RDoH=*yoo{-{ltTPnS8gM`s}r`9 zgR19{i7DDG;%K{RGH`o&a{G(eJfLUdciog-m0(^ep}|uc_$=*VrU-qb;B=OEx%)FG z0DDP|gE0t&Y3|5YlGHHwZGH*8Rx^BjZ9PIXH#~>WFdrh6pX0Z1<&a&z-QahXETy;W z8V0Svp>Wv4UMtj(3=je&y}GYk73h|`#RQHr#>^L8c)=;?#9C>{T@#($AF5(%7xs|% z3EV!c;HYMLWPS5(DbRHyMX{rA|SIdKk zels^?>qQW_zjCaZ&opQ5mNB%{c{Ep%&iGR~Sliz+6F#& zC7UrQbLhje?=tBZh_dfQ;i(f6e8!(T9213H6az&+JCC<%3p8>0R5tG~`^+u*D7N^W z6b|>JNeQQijX!jVlL00eAIH|%+C;C+WIl9sC)`QCo3|TO{PA^Dn;~4`^Op|)KEx<2 ze`1&bQF06|1Q@k6wcDQLXn%MyQ|fKhyyAZK;(~?ln7nkkoMo}R%3iWQYMggcWm@Ud zQWX}UcC7@1X^u4ZET$I~JOp2f4USlj#GBc+o6%<u{j4j=FBMrf861WuQ!y!Ji@0eY62U4Ql$7zN`SG4VH9CGC6-I#ZCWv?roXJ=XAY`rVLbbxm#J3pmkKRI%z7qaHSPzHS} zM~4pDhe?o`JVZAMFOEQ15zGKm6MQ^lMt{v5A<=`QA?2bnsxV2|ZyMDpn9<2G)W82Cv9(h{Z2fCI&UOI5WxA(fs zyG2BXX?1vHQDt1DM6Yhlwz7e-KWJo2+g{s6v4!rBWt)Q5w9aWzw*mvtpUd)rwG*_G+3Gh9_N+zlWjN(G)!Q^ z(tH+VyBX@ei!x(|Ud~%serw7wD%&;K;yK^zed?TVG^RpN4y z!W`=g?LXI^1NJM;z0MLY`ZHfNC&MM)`YbRTT_5OfJMoW|`foqjtE8_vS5CU}^iK=o zlsN7(O!J<}{=R47X#q|>%@KKixLQAbcNBrz@>;gKh~A+g1>jc(8)ibT46ISWpNylO z*6(UkL2BfG4Xz*lt-rP6Xypcb-*Jsm*E4R2&B(y%mmD|J66v6Ed+z?Yk(aAMlQi#h z{j`UnIVfY5cK!ErobzCSR8q8*c*6MLp@ybxF$w&XEyz|@G803g--%R zOXUeTujI2XrbRqgt=-aZ1^?I4m1T@>ju)w3ZpNjiF-6gPVddZDjm@9uImyAG5s=3D z=A5SQ)G`Er@isEL`1HZtEEgJ2d&R@f?8D5+caTc$bm4KBEMm`dp`KCqpyg=6@|N}k z`tVF4Y>9@Hgw@hPy253={`7P^dEDD;H7)`tyt3uLubF5fY3}2aNT(o86k*LkKPctR zeSLM6GejLPceU8tFWr5wg*?+LsBi!zm}Gr-ZK}r}x2)ba{3^i3*VjXZU#{$e{`q(2au_~@WXLHmQ!2J& zYo}`X%*riJD1Y58y9J|YOVEDdbUW=LHJXOmP#(|$uJ_janJYur6raVQ` zH!A99#8S^8EmVD?!bcI*i>X=I4Pe|k42K@unThuKn)*=cnITA6+}}Tgt_cVjO1+-~ zsXLob+X|*f=&f-c&}JXsya@RF`z?uZFd!CcCZx4v%!-s|q+{QtbDwp93c=FwE006c zUORsnxGM1fVqIbwR6Ci`Nv{{!%D2o4W-Yu4;mq>ORQ+SioapR+<@|B`txx;|y>r%v z^6Gki39~J;$>4R|&y{aH!ftU&HKlfZ6HZx+&zfe@+?QKBJnQv`UU@Qnk9ss?e_<5K z6&41IswIai!*9W1s@21I)x(nRxwn>PL6<#QDyPCbP8M1MO`}uhBqx`-BLjl7u5CJc zNq3djB9exaX$c=rv+gwoA=-)b74XGAtxzWV?(wlz8sEf9yF2%jdokl@>Lhta{&(Iq zko^3>Y$Rk(Mi_0LiM?S@DXNx@hvKtWZqL16m&A2{{|a0t8_eilY95{ia!1%_=WqSQ zvvqfBT=|=!$WAjhRne4$9a_f7&3LJ*PkF6})?c`wqg^yM@}O~c<;!sTSq-mt70EN6 z!dPb&3?H^7Gr^z44{@q07Twg_7g*d3^&19C80zSRCFR(%)bHkny=n^$+Zb~|$Y3h_ z4{C1eh65(LxV@DNZt@96>sK8c%ZrNsCw-0rz~hA~{a5hECGbZyT)2ufw~Yt+2i-zb z`x}m?ww-Pp7b5$5S4ObkpT{2LL;eQH7wc*%Qp8NZ_%qA8&&b|}sR{QO$Y!}2C`BYw zf7lu45Uu$m<(R#*o(t%Q70%GDUwSS8?H&nK2=?o_0n%I|m#I=F2G zY2I=X+z0NHg`j%U`*tOsbT8i=HF$%Ww@BQOBLAv)^@HSb__D*uL`|E3YQxHRsxiL+ zGxS`E${x3aARS{%d7vRMVtzVzbu0UzFy-Jq`?psUWa=Z~bp0XgFF`mYbYh%EgPy6m zj^=##K|hNpEFBN44Hq54zRC@tOO`oU6o3R_!k1xS@$-*{9` z5UppVS$){YBZ`ICw9lTLw)=V=)3IymVM981r_p_NfpclW9yllTE}GuCSB93}{ZBWU z@6~p(!nxV%eDqF+Mspj4%+%Gb=O(6W7I7z!4009LmYtPAv`5(*?pFdn5hj;Dbo$OM z9t{751qu!f#)l;_O10nKUt956JB`9$vG6}^ z1)V1HM~mB}M&t1#YiG2p_nGjU3cPdvOo#2XOyatER2Ocam}FYryVI3R@*7-qd|!9i z2W@gF4sN%SK*i{HH`I7tMIFaU7S1kl%EFxxb+s)Lh5}l)I>hs^5(v~crP!#z1UQIo zEKdvH`?n5olCWjk9s6B=Ifn?xVlKX?@m5N8i#B3lX-> zeTICBcA)_Q1w5m6D=wdEt2lBr1_4dv`errSqz9N$zZvToP0OO9)_j6I8pF3)W8Vo^ z=iRJGxl3${u@cmR`e7e4a3`!XK059pDmwZ9g#E`3KOhnN9F5E;`+*cizX(EQot+pz z*|7bPJ=y)VZODQKqD7%5B2CAnYNO7aSePOh^rUyvthMDvuQ%y@*yq-0=oP%S?j|M} zPD=!)#e(sGCoi@~gv;Q002jT+rflJc#8tQ+$kSEco1}UZU*$w*zX1OEo@K3x*X6sy zd?P*YON2FX3wD66gZ0}rvr`dL)th9x4dc$?Mfo*!osvi9RaWns#&18TPizG{$OaO8 z;w_&JzjHQ7R^a_&$FnM*s zjnjySN&KTgrBDIeimB~Rr5%;wzb<4jqRzt^Sg;B`MZ-!uN?0z&VbgkbhCKDIwTh9Q zmjZcr8t*J*FO}ai>6iZlEIdzQB zW)Au+GvW<1@48f!E9kTQ&Lz`{$O-Kq$2F(wB$$Diw}fAnaHBm-viiy(}_41L6}L1nAk{!rG~9=OseiG^{LQ*_0Y*q1?apk zS4hEBd29G#@ALEkAQ;)4+ouB0OsuET`z1wD7Ep(jT9BvUGmqZy>rZj5g}2Dq(Zai# za$Gh6of{Gx)gQ9f*d19iqqJXgN^add=1(Voohn@Jw1?O!jFq-VPCpldiiQMj#{1MB zYF2<|a$j@XxocB@Sg*uF3^O;8KY|#}GJY@iUVN=UeX#nwFgbDZ;M_zrU~p=0B@FjP zDHL4=(s$BpH-RxHP z)oOE?B(#{5a%LxaQ`Rz|hg|APWaQ3uGpY9d*5WhJc={*Fe|rWkVI=4BuzXo|hT~iB zf$rXo-@o=I@VsKI9aU*v3%$sFbc5E{v;Ct!Oen&`>V53y7rndY0NTa_3c>gQ&9T|q zg*ergpLIkgNr0VT=#@vdhb`rqCvvBjYFPUY>LJ?4%(hz1e4i@TXS@(rf~0+X|Cfbn zGIxVFu*vz>xmzRL=$y%eyj28jh+SF{lfYPt(LQYx)Mf4XeSPB0F52Lqg?*7{Td=+Y4tWie>M#mmg zL6guw8S~9@)b8@Y7mcUs&kLkXwj9e_EsrJQb-m-}e+*A7S36w02O#Ye2tx4%mWex) zR^ofT>cljuBx>;v5rhZMn{yp<3O1e#k2uur8GO}O(LbI0>`9KT5MU(G0vyNh{%8q) z{u!Zb-g7Nle!>XJv|c`5E-!A#Sd*{FZTPtgJbZ!R;vTxIi3i-8Tr!xg4Hsz^N)<#?AeO6cy<2v3)Y#1vxnSvrom7 zq-LlNrO1S>e71P0ZZiqj-v+ajGrpN^Lh5g3DTCCs2Lr{KnYa_I;Y{(}E=C9gi3q!k zds}aU`EUjCxL!0a;_zm8ZD}Ra0m%`yIj~|4V31F&JOcBCs=4901G7cx zg6KlSXkyys2rXQ^h&*GV&zllyg_NLD{qjv?tejX|@qWeuj4 zR|_I}D6pNSK71E6>G4@Tv=fwoi;sU36Zyn1q%YQ){$wyKdnn<|FBodNb$4`PaWKu~ z6?k)D!0NdAMta}*q?qPfg$uQ9S-}cl6}7GP2HqPma)SK`I{SFUpASJE*}{2rgm6v3wZu%)oSOPj%LG?{}=->`%& zq{wVFJSAHLHik~=Fs?2`9+lvB47S^BJhEo5Gx5^cGvuGs*iWj9xI~Ik4)H{0_?Htj zmb)zkjd~-}XcdatAFT{_3LVayFIQOl#86>nTwf0PBk7y=G z!%2nMgnGrrV#~{dx=;1Lk^OZ=8@@3Q1ZW~}Wg5)@eAtx&PscMFp_`DVoig1b6bzA? z8ygFwXcG(1Z6-Fa3eWs*D6+Wo}hN&&*O_D+>ZFzC&!DkPL<< z_(jSTY!skvH)m+9=C!`ll2DXVw#3=Ojk-*_EQAKmCDug=?^$b6UFd(9BDLcz^1r+_26} z+d6h$@`T^f`-@yM-)8m0bQGTmLmodegt@mwEp_{qhy0N3C#MGXU87Ah9p#$xLh>(Q zMA-hK*5O4rwlGHxN#qAZGTm5kiIR$O;wBNeztYhR zFbR3foc=%xPrQJmQtJPR^nngs0r&WwIG{f298K;ienPk0QZ_zC7hd9Bg#AS&o~@Ak zqlNx1LE*ij&MBH@pWy`an)lOeL9fM;>v=r%gg)J5e(rN2AB$a2OG+SGxZrPHMQn*X zX5d;g{_wrP5M_&Lq}`uU>?na#Hk@dzx1Aaeg)$NIpB+*1MIpMK)-9iVILuVI%~}h! z(Ld8)yTT$8z2?bUt@eJ6_`NpY8apkd(43;YohzZ6KOBJD@+`0=(_1gx_;Ud<8D_$G zvN6CVb~UmuUx;?(K9haxpV>N*MYAIW*hGudB|XwSr@{E9HR3E6eYbn z;Y}!=q)wZnrP6s42xw~)ffHsNoa2rjOSEnjDUGyUb-~mLtF*UY@Rer)OhH?z zFJ;iDXCKawj+dBN;Y;CDhL3s2C*^VIcag$o7g0sYTTF>syTV%8gHp9OE9iqU4{^!#0ckzMl^mHV z2;S~97%blLrp7HvM620}AtFffm#^F0wq88v1RnT!LPhs(hxTJv72^f7k>uMCb;m9- zRxn1)IkhM?J_+MS{D*Plkx-_y*=z!et-H8>i*ZbtC=I%qpja~_u~`UDqxbpEx6*J? zlNLln`T-aomb(?0@ia-nloo}YJc=&e>kk7To(8Le^WI%i_h#%Zic?n=SL)i9TjTS{ z^y`YFW`2B=JUcf;HyZa2eQJk}`J>$kZ~3>$DuOvCQz0RfJpSUMFUd;y;2++j;h|*h z8~Md*hFK6u8v~QeQ=(({cu?DiIo{w6c8-XUka-=w%3fgejFFvKs{I*g#w|zb^(bAE z%>rBDPAhOOOXZoWo&7lZO?4%HZfYRlyw07C{_C)=G*5PqODlWDKvoAtdits?XHT{wE90Sr8)g@?h z`UOJsFeaE~E!i}X;f*_@oC#Ua_jEv%RY7w95NSN{TS%0pF&eFNjW*s@R;v?d;L#vb zz1BfnOm{X|$!8g&nhl{x<+Xlg%r0CNG`H4-X$C2y|4x(H?8cABEE+>p=RZk*Id zR`(Ve3J3H)Zih`pfA4qP@h3(nY)l0Tjz5x(*7RjvmA$_7o%cP?=HdF-&;l z6$i9$s@Z+jsIIv%iIbI5U-UyrYkPfsQOyqs(V<38dnkqShv+h9^W;97uZs`S&eBtE zCXf8;PZFJE!j1ZwztAaTP|h_aG6TD`dt`99q{;aA2b+Co{=rib6*)(~BEd1_gl-fl zuYCg~;kaXAJJw+)*QIVCh@2?x`Wc`XK?B?8Nhb?eNFbSnFmr92k@KVV!Mxpuc^uUO zshP(AoMF7;XwXM7gU|H#wRl)s=wBFj1&P*po$AO}1BHKN9qG8Ff^^Ae0*%&5RF zg%CYPLzzk0{Z2)pSF-=k3YGz?eD-T%9WU8KuyK!9j!c>XbJLi#J+S{I5qYwAZ2x+F zU}2~l{hz;th;q!tG}`a6iujB!Rr(;}B~N3(Wyx$@Z47pXi@;f_HjX8bqo>)ujh@Jj z$dBaDT6&M^s-w&?RTdD>>KaUm*dVVTkHa4yG?ijDgtvACw4RK2^qL-N6DCN1eg8IW z;cw_k8adtk3Dsd)ed2nMy{naN}UJ`=j$NF0A;YPni_ttkMs{w1=)JnLGd2 z;gpeDftT}pz>R^(H2_Yel4-_G3<7S&vnxwMDPQ=P{Lg$|2|O8P>$OrLLA2a7f!1HK zy!1t;GT(DV$v)`-p5(dT+g8OCx|=61Zh~O)x0%5*H#r*Z@11U#Krj3IBCn@qez#7# z{}F{wOrsrwYV(%7{f->HDSlE%$O;(;X4b8a`{a2Av?d=Ipcw*Z07S}0Rzmn%(_6%f znPz$4fkO8%(>SK=rhnXQe+gnXgcC^S_4XONJN?k|TvCAz`B!ftaKZF*k2k(6oykzW zlXSHQH9K|IX%2wX?4IBN04(xR_F5fUh4)Et9-baqzTi|)qZGON)Lmi0B$s{Qdz_&9SFwge zm}^6v2A=A^SW{X9@51M*f|BF9BtfKib0bC8NJU+5^F{W--KF=r?Gx1ytSo_3ch;9Jt^VlF}6Q-|>|M zOn|9U#`BW%U^gPh{rMOd@rB`*Su?*M!3MIO^yiQf#+Rk7Ws~wEF>*HVHLsy}{ZAiL z!~gTX;qcDc=@*wafa#>QInZ4+DLI&Rw{cGG`&3d%AJaQCtYUR(UP#}5v7drgFV}8( zHYqbHEADHj#uDcindcPCiP_~Y04#_*(#~V2b><`x-Ew`)ioO44c`YH#n7iN2dJhLw zj{e(mFYVu}ILJU+k7+kx0|0v^DS{`|+{f|H_5t&e>MHA7WnAiG-bpqx zZa3lR`p*?XhfhEl_b_PUWb8MiGob{OS`wcpy;K`xr;-2vTL6x}6Pa;J4o+ca|oDtW2sl?%2V$8a%etpzx!Q_fE9q^}@6GdDXpAv3_1! zpN%R&8r6DXy2nxko9`lJW+x5w<~ptxnBcj(I>lH zdWrjIE6(v@9lW>tChz>)p3iA~$Rxj#Nnw})sMe00aqPuHpb3oCOiJNRi;-(`B|5us z{K9BS*avzeXQGL~{f)tU_~n^I9$G#NB%^x=`xip+M@&)wN<$O?-1*I`&jpkbayZBt z^I<{*f8JYzP={yCJ7qVO|A85bUjueHb}Q)ZJy5~u&RJVLG^&WIU z3!6yhr&R);Z`PmfLZ}!}B)X}t8z=SKkh+MU3f!3xN@(ErIc+-gaU7}H8Z8H`r&o3L zk8kfe_O)t}X|A3hQgKD9eWoSZr{{=8V5*xE#;0LhUV$vW-vxG$k8_OsCpA?2oXcB? zFcG!gIyfDaUm)Z1P$QP;MBa-+jJ`d$-8ncW`d2}r8#J!gJHD}7n_g-G>|0%VrUuSwbeUqNk`m(*%+GibL7dXe+jVy z>>&~y3|ngD8tyr|5b$`6kpDrtpHIs?bS6X)~me%;iz@PWh-Z0&vFWnAS86+qhUFkhjbx_<;Ww zpKqF?=*Gd`hngC|r!{t@Ve)}jlr_nk}8|xZWOtlh|53{uzAG82Z;OtcWi|r%l zIhw+j*_2Z5GtdO1Yqx@f=tLx5PnX^UqvcfAIw}ceE z*cNUY9$d)Y9!nJ4dEZLdzqc~#Qz*ReD zPXwgkOX?+`X)nV+x~37)Ia_nP&S(o6x2e++nBg-ZtS`|NYf~c~^Gu>l2imm?kiyg+ z4)@9TE^kv>&oY&nsKPZKiuc;*rt(^cS0q`9X_Qr(sF-lJzm`9ptr_3H%j?&_upE*) zB!xS@pE`BatM|SEjLU>8QV#*JnBc>|5@ZroD=9|LhZKqKkl8$DL?yw7PzpF9=<^;cCF%$Nl~- zdOM;D5AfG<|1F5{jgA!h!w&k2u;QmIxGfsdaSOeh#ph7hls!^s(e2pkSGz1#vL=1> zH}3tf9SB--V=0x)4_KP0AcL~J(4x}IHj^KqwFf=2v|?N}G|Af5ts&u~x`%l=Y;^zPK74`Qp>b;_g7-H3#Bo_*tc@7Q&9_ zy7?oT?%hP5>4lJasEJ-fh2yo`OcJR`D$%jM-`0Y~+P*|H!E(`K4|L8nwW~%((*kNS zo~}mm6DTdL1Uln}7k{*1Y&1=%5F`fCFvRY>CGnzqRW@-eGp-wo`>>R^*D3WXQdDWesTXX76j@`>j8ul}j{N^%blL2-vZ8aM;dJX-W?kYc5ST6Sxh z!fShAg80rzl0B6!V_>f=r|Z|r!t4WbThy?M&IErBUi!bUIU#@U*JEa-C+zh~CwBJn zh5kw;%vQG9bKYGm&i~If-aNz;4KC`F@es)))e`SE3u;2--IGrkkg>>lAh94%?ZVz6Mvn}r$=56brz|j7eL#;I7+2)x zfAr<+rw8R|!NL&8r1i^&F@vE4^QZ)26w^(!j?y_QaU~f#h4u2vIV?Lai3V6Zv&MRCfrZ+ zt02$M7v7i__fdQ>!y0YqoX9LCg!RvntSPO&mk(C>(0j3s zeZz%)jd8Q_FJfzgNFoBCR`|xbS+|FCK5tu^+T6#P%4IsVng=Cr_)-)@XW-9-hM_sW zjY^Lnzhrh%nJvAKJywN(HB)pE&ZAcv7bkDq%X%8hob$Zi68!Z^q#qf2SG#B4!1O2t zaqXdTxbMCRv&i&uWr_mxZ*u&L8iMF>X`|L)mLVN<%Kq(nm7I;mK71_ao1imGW&)df zp5Az67V67Zj)$2&j)u8ExrF~ds7d+td-MUzlCkZuPDZ%*OkD)cm zC8y8{`)Jzh8mVyyVH^Wf0RCf(+6aBWow_dj(U-m~Z3$%HZZ|-l1 zb#^OeSg}Qll3e|268o!y{lm^ASZLq-_1Moh)8}4VvX|``zi~lXX1~ZmP=+bu6WH+u z!1_9brcm!{1~6Z`m+hx!oWBP<7)U5cF!N&%3!%e+^$$ocMAfh>(Xtnf^X`BoNufL! zs!*FL7q_Y4&!38B$tcyGGzf;U`c!@YUzCz?P(2MwO!1=~na+@q7^3U-+2n zFQxDOx0E_9T;B0RremETYu}%@9+U_bpG2@nVVpTBtF7;OymoI`Fi&O?Ql9C z!frLSe=`Bxkzg{PSTHmUmLWr!NMuI{hHOPpXNlXYV0h3$wFU2&QJVOQH^GVgxKmuI zRD(1QB68JoUnBChTgY(v?M2tfp&Cl%q|q%ILt>Xd*vOeLRQ_VQ@5l|sjXyYi5aH(L zh=}BUGJguPBb57^%56^lx)e4=3;Jm{U50boWj>RL&O~XO4ve4l*Oq`+`@V{^e%iek z*NhnJdK=BSTdJD%N^ASY>wtoH7plZC$DmSI&c8@=54=qiitO())_V4Ox?<1Esph1E z026h@sU`OQ|DDI+BaF=3%ZS5el}`*jZaD+$V^WB!Xw^L98I@M-A7-taizeK@)826s zCG9)f^G4&gu(fLD{ENWOPAc}-pRfim*WFTdloc1tt3HcV;XuLz*Ge;Jq6~e6Q*6gz-1bm_<|~R51)zRM=VzKn9`X#G9J>C+3kubQ0r{n zSMg2&+h^(;l_(QrEJS4OK{381Y>7FA_yokTRTh+e}Df-`WAs59+$Z!e?QY{B)QS7O}3Wbk*G92H21gwT>K)R zZP!8AC$i|dY*D!NCfedo@;XTWMom6jU)uU@zt?yXr;uE%P6Uf^Qj#C3T~5|~5h&bi zhu??&@(Kftf79(Dki_8G7Jxy$<90vrdEcya^^eSU$UHgMljy}ydgYuh3i8Hq#X#IV zp~pLw3eIeFTbqZCU!m`Deq|lyqfXA3xb^&*iJF=TEz!+*w12L2vDJ1_kfP|1P$Yw> zi#R*X3Z*kkE|E$9I-O@+g7%f%Esz#I?u!Xm3wbZw`jjof4b?3 z)B&U`)(a|dI3+ji^V&=r_^OPOC!M^Ae?N5Sv30U(bE#yykIU!sNMM2AkkMK!P7-k1v1pEJonLtl8>~3kJGK3;~6m4QfXrwTE>p^bPTjq2UrKt`IoX` zIqm+3#4v1x0eR`u9=+rEqtjnTHDJEX;Unw$k*jGAA`ZVZToA35TPqkK-8_M@z4@W_ z+PF>o=j<6P`dmRMelV1sEsUSQDkVlgW{^VWd5iWROa|!=D4>R)y`=s(PHvj>H|$cf+sB zT%UGIJ+f{W*It5wX)x?j@bBAj0WcI@mfmHT6h;YrrN}?|9ayueNq{!z`mW3oHJg>N z28(L%K9wL`FLrnFEQHp3<=JT!+VMo0NSgHqa&NAVc{`(>$aaV-&eNf3tQBN4b`^Mt zY9gHQT6di^b6JCgR2lEQ1R0^s?lH?jN~4|4l_S+Mvx(rS9s z4i4+0@V~pzKJ&9(=K?kpi3CmmX?QZdQcA&*&)vFv7mWDw;l|ry`Opk=Fs;1?TLI-; ziH$Hn=ZpCF_OBYEeVS<0f!jw@i1D$LTr7tmVRz+aYAa)W^$6}ZUgs}T!3V<+X)9?^ zZswWOeAdhWr;(q`%mQ~jSomoL5feLKh+MVHMU>Yc4=;;bUOaDj)2IF%>AAK7EaA46 z{+$TCHO?Txa9NBH+~2~pJmIK+ngYP4#l~7wFS394nY-9wMGj=tm9Lhbo6A>%Kf*!G zToc@*EbjlWf})7(Z2j<9*+Q-B%gM`?)%S~+fVZjYVl?L1PE?J*YeMUAQ=_{iCXzLs z+<{!yOlxj5yI)sL$+sa-SR=HbQLog$9JZD*o#nLKD%n27>-NyKmaLzzF6-eT6EeQ& zxv~hd@+eAdX>WX_(=mU0@^myO3xcQ|_^Uf`7;>T7YyhP5=T)3FqGm3=aFPE_r&ao0 zE_hqLZmCZS_kgzGO=6wN3v6MnoLs;ZOR#E6^Wztz(9wYq0WL z+Nsjx%4x?|vroo1>ClMj3(2F4vVndF=LLN}Ks%sPr}b?(GT~X{(m%EYAcTo#werIN z{ftf9*zD5;l7DW$T|rHv_O55?k~ffX-=iD^&0P9U%Fhi{y6XJbUrBI;bj*7Xjy~qO zpBm@6Tx%fL8kX(t0ewRrqG&u!)hacYbaUm-7LA}Z^xz})s>I2G)X1qzD>*DV? z6($oz`_-Zx)h1HE1SVw}Tz&i<8qHPtg#kBj4I7ENHtK-`fM4guSw2m&oF8=eX z*PerAy-qpkT;-$P3uFpMzE`|*@Gyc#mlF+c?R`}oM{F1QLByh&bhA6B|8TY?7s5lBI-_1xbN%nE>}^$MMAnBcAF9l5nHHvG*?}ZoJ(E zoq0Fi4!RZ!2?t!&TIgF-F2B!Uo_iaF^Yc_&XE3c8B-nvt_;QOGA@h%>8o5bs2v|m~ z(tyV;w{Uk^{J8QyvMV0B5Ada~AU%W>N7OVQ>fmKKX_-`n*rE^r9{$LJmtXPXIkRv~ ztV+OFd^c|aQ0XEhOLH=(Tn{UwOfQ%=#g_S z19nWkTc!s8F;#wi$58#AdksSf1;QsTbN(27h?(e$7s%3msF{YPKJir5MpM5i|AB*9 z?DFxCz_)vkxjwK1yRqBjqFnE&(U1R5AO12>L8LFAPDbXh*|aao4IozP@9-J=@B(X7 zvDz(3wFg|8DVMB1t^Gj+5qNb1)XP=`%u{jo5uH%+BvmVwJWz7FEoqb zqqMl17EZ$3gf+TNqIPc0WN4+?F3V9bPXGL>qM!|QVHv4VaT`@7KR3S*Z*?erc3K7K znJ+!j?U&nFB(Fawl=jSO;Wg_$O|%Yu_9Uub_-eD#gp?utadV_J>`dL(XtVyUA)V1})@M>HQgqvS%4TQ%(I{hQFMs-r;SgGRuPzg$wN?qN zG0J&`NY$DKjW6g&nnNw#wP2NsZHJ5ne>PoyQISDDq?HF@^(Rtk5Kcg$;W1iG0VAHe zL(I)nWibxR?M_r~>Q0-yZiEec@!{J6sjI-&ZBPr)$jNpPz3CqR^MrXo06v480a?xj z7r2ju7)6byep5!^sWgHYN0-QBTTS0D8-a#Iq;N7uC#PbdTE8@gwfzTxOk`Gr$P83Q zR9jP&b86Z2@i6}3SIV^#YG<;0r{82~3>unO3?FY>FDg z_C@b7tB1#H08Gn5K7uXxxBDlTs+*Mvp`K8`2~nK&O#THb6Hx#l^^8rJI^E||ddvJY z@yAF!t6&`Qp?FVIUQFQb03SO4K{gFZ*;}LHUvFKa{C0-uX>&H1%;OV^pKg@rLa}8g z-*q0Th)NjaO0`c!dHLJincwuc-S?$>86Maj8(%yX8xYa5t4cMM?>wJr^(QKkd+f8h zisA@VLRRGKTQyL%R7acnjd0SjcbKoA=T>whgqqDJPF?7u#3EBKYqjhBwF+yk)MnJEO{ z-UrnXJaN&Z2I;E=WDt}a*lWIUuV9{pd-RY0dI(h57cNB5>-D0M;dq?=%5 zLAwn`2UIshjm=R{3Y4AeMvWbeR|fTVvomD=`Ee&cYR8!VvTSY7ktM#BMI>=jdbBwC zh(NTnO=SdRy4=c-wQWCQ`m4cJcoF;)QjM{>63DJ_FNl$ag-4e)i>3hT-x;7d1u=TvsLD zkS|a93lFle26R}M6^<%KlRZqHl3p&U8$Os=pS66+sjZ zmVhakq>C|gRElr+(17J@nr$9!-z)rd!X&OS&Y%c#VTqD)|6y2(xRI>3e;|lwv>-{5 zC+oc|mT*(`fpXlgr-_V zmFtn4-kVs>oSuy3jKVE}j+xj7TeM`SL|-nA@SsVCoZ(|emhv8f&wk%pv#FMHyQsw1HQ#0C#Dte@YbPo z-}ixBFvl;sDr(jyt4jOIBeFK^|7o6ZFyp~cz>w1i z#LMvwwNX0bnwpAX`8)IPL=41?s4 z{@H{i3yNR(Qf-Fqk%$5z{*4NDP-h1W5mX)oz)^HPe~rwh``1nUvt|sDE01s2(Ihpt zPIh%7OTdomBOM(1^^FI&SvMRq=1L#?CRoIFOj_w z*z;?|R;?TYj@^pyUiF`bNJ8w2qcQG*YU_g6r9n&W!OH==x;LhTD^ z#1QSl=1=Rky}q_{KDv;3NuqqaCiIW~WQxEvWW8qDvUxrYnhpUzk&3yauBOeJ)SmbW zu5qF*kA(@4Eoz1H6>|BfhusDeA%#DZc4P$9*s)4}lcS_x!MWAPyOEu~V@ur8GgPq1 zYEyrf8Ed3;L=kuplF@4)a)u+TSh?6QWM!;*i5}iluNbb3o;UXTu?nyB;OLk~H|X-S z%1b<;9eQV8zC^EP(J>EM0X{+hYPl2&=H z^)-QQUxt~q)ova0)%PHluWPQOhu6QLdId|l75>y5m`fP1YmuM=8G>dqf za-^<{X&fVJ?DjiF`JY4&>~h0cejZk?(bCvoTGcQ=etmFWr8NpP$WpFd3w9b}SeHn_lc zt4>b)eaD}JPNl=ZaAS&(X^p187P>i(84EB91kuS|96pHg*fXE^g;v_RF6bjNI2{ms z)&0Mh!35y{cx%U6OH)dY7WpCAlvJQNSP&<@nnGu;UG&>{Fd6iLR@ZPPtaYNg%nBzo z!+&dXzd7>1mbf<~ionc|zC@Q_0yvPyn$bHFcBlxu3w&Z1?jco0P`*4yXBIqDS+A71$!N`2ABn0Pbk;p;sQy2u zzBw-M_xu0OwYX*5ZrQeNEPL6uxoo=@7gnv7ZEM-Ke)nqc&-eF#_oM4|opYY&oaezs zZSu9`DP4&rT*il^qd(j>Qrlx|Cf)lW4Pu}NpmOH+pdiOH)4{ch^c~$~Vim*N%f(kG zXP7n>nbj5!CkUzzVO)ONZi}1wV>6LKoNped;&CS;4u-`gsi=x-hP+yc*I+x0*obDpmE%~>SG8qb!k#D=$1Z(8_7!br-`WWGr3hRl7(z5 z>fXQ)x$7-i$$A^jXp6YMrz_X|`X@(Te7-KA(%mMehPvNvoUNuvL6+I{w=AAaqMX)T zEwy5>76-~D7nQ;|5gPL^qSSgq@UUuAgZ)^szsNSah1r-?SbHTP)MS~8#PJ2XaN=}y z@2=S?WU(C=e=PT=m5WM$WxBY{Cev8_-qiU`4oZP+yp*)w{H7T-B*ic^`U3r0{ zwH?t;9Y^AHO`5L!F{jk|^)B)y6Y0g|wC2wwE#iyK+|JnJ-Di)=mzXBt^X=L_>Dy_p zsopvz#YA%{nx%PX>^7*Q$DpDiH8fnpFwD2zWoKqkBAoiyM&i^dg%M}pf8(xxJwx=F z{)JU96Jju~%kRkjkKxlt>ICz}jumooIF+V2u116l#zov1z&_1jcvJNujv##l0-uz|LHs!1qg0(x&DR+i5 zm#D!GL~f&Oqbe{hX`z%ynz4lX@#4ytIQEs}bl>e!6l!gv0a0y{?uY{mblagH{BP&a z8+e%s7I!4~ibQWTtbj*WWiANOr~&855J`I`5ZS$`E;P zpXS#OBKkV?cdoDGbzt)>>NI3CQ?0uU9*vEQ2Cc&lRG^!Ut*mRhQP~okCjWs~YV6x+ zEvwDLg_Usv%Q=JrkB3kH#HcuWpvE@F1Tv&$p2O#hVIbh?J@Nq?9{TV-4aS7dtY7aa z(ryAWzyH+l@1LGx>Etu(7~dZNZA@u1Wd69Ra9gsnEk*9&7MJBHPMV!hM4X$)gwv zA3J$iBYYM@F@>OK1hz=-EFtBAhXuz{U<2qJ_tiX3?OhRB8+c(jxj!O6Z zQjRUx-Syb-&{cTAXGt>Wc8B&V7EY7?JejxKq1uytKk-~fbDP=lywhnx)w#H|n`hXY zdu>NHo`L>w7KlXY8?Fn5NRx>c+%fV|sdVA@H>?=%WxW-q!!zfd-jG)_e`Tn1fofwU zJ$}dceq4V28DfU}!2Q>Vt;f?W*9NFG*TO0dWEHLPoMRZ3y9VKMqWBg1H1|nHsIp4- zw+s$A(nJNmlhu66>gK~yKCHQ9rfoXBQ&f83r{=Y(Vn;K&R5j=%oXt!XPFpHYr*CsM zl35s5=;`!u9);?NTv~{c$atJ*+N1l%Vx&eR}^L2WluF35BdWAZSwRtYEY{Y^vVZOxdfH{l# z$9O48c;5W>6>>~VY2Lrd5q^CYA_m)8C_e3^9?U!Kn_x=b;uYjgwg}NncbWZb?i_47 zn^=+P-pO7W@rHm!C+qJJ8aDku&Obe;)n5%A&9;2`RJPl$tYcntL&0iWTzR?pao38` zdGt@3MCs~HA&*|Li_mgVHpX&_?%~{T_dJb!Tzuplyt{`pVX7RAC{y$Enk%On^AlK% zNROo-%^l)pzY+|7J>iX`1#$o6&vavdl3 zv1Pz+phbaUS~(8i9(xBXxa+sZ?c|7jXg&C+jby5}Z?>IFDNBXZ(I zK02*pl&Sl0Ha0z#VbiFOdggE!Do@2Un;ljyPfR7Y2P!Y}t<%zr<4Q}`iw6=3!8d|I zfwJiFw#$A^m2&eJl>uYdV-JsdSr6CNrYGkLHcX}6UA9kSXpKATb=MEDFp{bslhnr{ zjz4F8njvpy9t6~SRbfd;=-Zs_SNO~P@*wAZO1BrlL{{&{x%hlG>B^UOvO7US>?7=& zvS;>iUsTs`4MF1OLHUHLaiv7GdZpq>1n{4rrGowD{Vgv8$ji%U^-u5S`rw4?sxJ1r zhcqJw>ps~bsg_Y=1!l(3y_Ft-_M!>wAgFaM%dq2%-#H{{CX!;xwP4y%I21ITygK?2 zZEp#K$t}^yzDb^8k~P8Yeo^)_4cMmmhWBSy(8GMYzRC0&cUxg1Sw0BqS-QOry;f*Z zk61GcvtP&Vs_Utq4xMGL;(6-@(3Dg^(r$dV{(*loZnk_0e4;f5ZVZ1~=0!o`q29W1 z6#bKHKbMP;!^O>Jhr{8q>B)Ve>H=rVvzphOGPWRkc24?&Q9|D0IKu417+sAIWQ6YL zrFs!Pdo=%XiqM~@bF&9t3%Oc2oVyac6{EWsNw^-~Zm02vz&CCCz=m{t@{sxRq_JW+ z*^J^bzgjy^T177QYC^)sTW^{+{*HbqfJzdbnBRpDS#_&qhFtJ-75-m$K^8iwSQfM8 zuBy+g=MG<$OW7A{BH{ugYHRD6Ik9;D9$&H}qSJ+otDUWEm7BbOcQXzT?RgvV`rWRa zKHV>B;<29Z7@P1kzbG(T8<2NeUJtrI`Xm5?GVqbR&OkDahx6HODNrWcf+o$9ARkw= zsN=U%PP(eObqa5hnXzAxpWM(U&{d6YUxFd8(De4e`2 zIxw7>u9tvJs6dHNF_&0>WqLKXdO=Nl$KY{m=p*;v+Z`)jf3{EQ!+WFWYhD7MH>rzN zN)J@v6;`|2j6f20gOf$Ne9{?8mDe$QocG!_WM71J*rv878N`z7&bO9(E{Yw&YeN&20fdi^S({RW$xPQ+z$2`32>7i|&!|U2AjBE^ zGRW?xSdqs~TF4C+@nz1~tJ`EDgcW(n%?Z>xL6$?62P73vZWUin_M6CY1fE0#6bQ3X zkjfnTk`=5R*CzKT3G`cbY8`7LsH~S!+NPC_%1zt5gp%G3jgPLIzt~=lGZieYV}iO_ zP>70P45t*IekwFLKFNexA(cpR?uJHZN5d?zTC1aKxFW3)nNIx(S#ud8uVW8u z8C9isfsf#NHyFha59XZj7^jbX>pIyT=kuP9bz?t!?)8Yz4#3(Ur*}f4?g6W8j9Owt zek{X>2fV5n_|@^WLjFQn_$3E-D*1Aj(-Q8A=C>1F40M#qo(({5z`&_#MK=Ztu)-@i z(eb7TEf9XS__nX6rD#lPAMTm86-iy=Rb?cb7x`B-D%?eBFSO*iY$l(HkVD@79Mn(1 zud@+(hr8~ufLeA;)CAjL(-F^E?iucI7@{Sp>`#YnWKfq(<1VN+W%^976x*IrLKOsB zd2sA-4BY%_vfIGST zHk*n_Yb$2c!e!|p)J?BCQtbGiD0X)Q&6`yJi!O6?)lLSn?A_G*d@psgkY;v^ z=Hw>MO2$&wl3-K#`q@Ah!1D6 zCGlSXkC6d=#AGu{@0{tljTa?qD4m6VJIyw@tS#}oubGzKH;vVZ$RCb4ow~3%Dy4Kv zw3T14H;8*Gc%oy}0KI1aU34!$hW5>kkc($EjL8)G_x8o^iCEg;TmR zf5_cfJrAvW^=^mU)$-b(psnnFe!+!-1 z=0z}C$BOEVS6Q?6@l}2LFX6yA0)FJn#l0a}+qN|@6hUb(Ij$CmrBri$83*^JHkPd6 zyVx4LfW8FRP$XF}Tb-!PFd{IF&uf-ol9S|H9{=UPP{^28x{beAjy#i`p?oPu=jCsc zrEYn-?fBi=>ywZEIP?CrOP|L5R9v@5pVjldC7$yPdo>0)E7ucL!g(fOcUCN7vwBl4 zN*~L|e!j+U^1knTS+CT!lddHFKqAKm^MPd?;pkvK9;fXoOXlpzt}9@CBp=xe_HT%i z1B;Y0pV925yJMSBCEk^osBs+$gG@^onP@G6$~=(Vbr~>8p@1UN_KeT|?(p>~2s_72 zV}MPr5Y_HI{ZZ#Oz6oEgcP$T$_Aap!`9d5FHxa)7TzhrGZHtgE(gYo6d#IwHS$6M}CnGB7tOf%H6 zWV+HUOm)iEy*SIAJvvb02J9VdgpIzjhvN{?iI}?2Vth1W4+-{q)Rb~l`cBhnW#6OI zNHm7hG;Ze4th%vV)(`Sv-?>~+Y7jqK#JdM2;3(R}aB%f^n0AU|L93c`A*RNCe?Bkc zwjK49hl8>L*f<4VHjCxK&*^5)c05Cr(u}gT9~=^G{*-{{AtjQfKVwC4gv(GWmOd3$ z6iVZ$#5QIY-;X~c@QU;+<9F<1svgzO2+Y0cNdGn{B=BXVopYaY&!)RP!gGn;iHZ(L z*sdVK74VT3P8-Hd-o+#xA+}!cZ!X!a{?r91blM``J^6vAwW9q8gr6-&_UgS*1tJQN zpzmAro`m0$knmUE-=ijNyow8orW@uCI~atzs50A46plF38rhle0rPa*;QdGIkP_D% zZmhYR4Hr&7TQ+CuTE$4WM8k}t4N01O~6g~-YZI!TF zm_^Vd6u&b?eWKsRd-pG6F!iVH z0!iKHD8tkpy9bI9%Nm~^xE<+fqtmY9zL7oW@Q%0X8p0YVQ+&0!3%%enjeIQ?dt-Q5vNs-A+$Ow zFmZy|8W2pim;u2DoQRH066juamR6IKMYw3zbjqm0X%OP*!ZCHa|6V|(B5~3f%?gUKh)X#NCaL#nK+S2 z>1ew#HGorl7O0Fw##1D;D2G25XSq%za(RjT`D#gS%VnAbq1KF+A2l#QVHl^+PP?l^z>0OJ)J0!hcr#1T%M!*+i|=+ zV1y`vhCAs*v%H8W=N6MtD+acVCM+o7J4v!-Z2}cGTJq|BV1Cl5%A{=LBQ3ESS<5qG?gz`%_r%5uOIi`}-P6fjBp)VJ9i)Aa_fYMZ z@2J3pWhqQhTL6{GOvoVYmRY#3`w@1Ng+mZ`*HeJcFh!CZ98aSa@vf_XK}(vCxJlY16sR*AaecYuuuFye z>eMl=n`XW(ZDKc{j6bT%)n&Ccm-h*}G=nTyAiyUouefQq4uk8yoiI8lGCK;hl~g5F zm9vWJh4(kgI(S$1FhLyg#D~P5ta+9+=saDM)0Wo^7pslD73XMR1jtG>k^LS4g8Qjz z9F5WJz~VSdUF4AnvmT*h$t4wN>0FkZ2%46A-*4{s;a^M5=D*XR!0cFfod>zv`)hKT znonE5`)U~Xh1XKE)XRitS7Lxw7q=lc-O4zncRP9f(86R+bo(5=21{A3Fr>gkC83Y>9UCvtpe?WYr9o`Gg(BKx(I1UdQ)x(3c zRpNFn44<;-z88cC?hVL|#$R>%v6mNU45HZap=g{tRrhiHN2-(%Gq`tcyPPuZAKb1V zF7f&zPQI#MM4Zeijv6GL!*_fqIb{GDAd4|w`_T%>C@3Q-)vT(O!NUo~bwi7_PqVz& zxc-f9R2G%J^_x6+mj{3GtTO^_wwm$)EP{V?P1hc)_lAuLjKQRZb%KM)aO7klqZ7k{ zZa&OZXLo*4tO!s~X-S1*qO_YCYeyin)BUmy98GetV9We#V^N}4@8l1ifc|+`&!Yjz zY&Uobj6y=;HjjyjhQ3`~;p-w-*4A&|3;siV1tW(=()_n7{a^Xk6W0tClviP~*ssvW z`bT$?9pSg{E0*4PPLGx^NVIYlF`ATV8i4soB-hduoLk9Qm-59A$R}ZyX!WQqNzWgI zTH?z$vpd6rVxKa)D9i%6LStAcIsCNmj! zYVX^#=hJet6{QHEs(nVANO)H@7{6y@gKlBdzK;-`MB8JLsXi{E>UYODJ%B)6CDDz4ja^JMqDv zZ1~eStJ#|ljAqQE?pK;m)%6Z^97CZ1I?Nao?>?vn#E;1mr_-C^-kfzRNI}v{-|V8U zFtJu<9=U*22`gTayWwQO+_4}wi)H{fNXj7I z|EgxRN&Xr@zo~|;hUp^ z3JL(m3qJ84*0n%^w0(*1c5-mHGw7)7pHWgQwqG`xHklai^mRk9S9ZH!4^WwUB2PtYb!XI|Jd zr@PGPtt0*1@3QsTA)YnnOUh%9=Sgdy9=@K9rK{s}X^|Xw8LL&X^%?PKHMDI^a@M$zNj2n8~F%*#JsUMq&xTcFj6bZ zh*@3Bt1HFV5QXD_!NEO6&F1y(HgT6ifFt2}8M-W*dXZ!7(F5ye0+pr0p)>d$9pLKx z(~XkoL>clgjsJE5usz|pzg%*xUy+y@(oKmSGN}a8$0YvFngn&{_+LFk%0TINPIitW_~Q@8VN2bL$5azWn@~Gu3qKuMTS3 zdXDd3%dg*0P;>sH4*i(>&kEr)H!!Sb*U|{KqYpp|s%qA*ZOArl^j35#e3G z{xWS`x&uXxSJEM(Nva{1*R$fj#_AzNM;<%?+!b`->wk{Ul!|LIO=xbV78h|YaX*6Puav`Ch( z=O$e$)i(2x>B%yy>vTN0C`YpKOP)6EcL8mwIG%-s+l;*tc0@ezD;>bSCqh*uxy!Sn zBYbxQT%Yz%j5J{Lf&1(;9El1R*vf0F0GC2lM>#Jf5+Yzxj6(D6e^xX?bEon&%28A6 zQBh9RVqInwi<)psM6xH)y2TAA!rgavuGf7|Fqi-;Np)ia72SJTS0%7Bp0x7nnZ}GG)06e#ZQP$Xa5&J&36257;cmm0+_|T=fmd{ICNDxYhKnS26Y8}+( z%owlcwlO*`cP?PkYF#2e-ZcYK8+qf1J2!UF2e*(0=!~F9Cf>*d;2s=M@Q25tXBJGN z{F!)z11z(BT#&6H{2#;KTe(z-&?WqMAKK;C)s*B??m8wz|Mk!T+n(Z)_+ReLRvaJ6 zq}eXFq_Vz*e_4(Hy=@-pCbMr}WxXjbdrHwC(GiI;kpi!nC=kwnZcE?%VGa9SeX-1t z=q_G#juFR>Is@%)$_3mkz>@T(Zd;Dj>?*DCV(@7MG2{8kft z9`@7&-F_6iY`U-^u_HFYQ^{ms%06-ON2#Ii<#Lfzmxz}p(U`R#vqL$sOU@m-n`4XK z`tRs(SB|%mDt+zc+(_Fn)+ktX3j!zh%$*Xzk!OPfS==cBvM$ymzK z`OOu;& z|23}pb%u-O09i=LPvhzQAd^H-ql+< zQJSNXFJwnB0n#N~H`)5!r`z+3G_gZ*#s~;N3PfCO!NF+Ml5n%5i2OtEcQj@hAK$vp30CUKJ*O&UtM>hCNqOGALs+%BcJMKKj% z8blHR7`o#^0pAy@^`4Ld6Wa-dZWcksuEWKZjUOEm z&|-j+hxlgha0H;0vw2`C`;%QT91#*Ss)7}6Cdixv;-*hRrPep!OlsHZbj!`Od3}>n ztNu~)s$k%;&Nr8yJww-tg-$kIcg12}1Ozf8#;Dyr)4m1lLq27)FWXDFcE@04xP7_4 z0Nzqp;_B|*AjAefXvstTbfMa-jpcQc&xCH!zV2x*CH+Dm;i-6DiN@vU#e6SbpJ$<3 zfJu-ap5Gb%--=i#fXH_%SXtFGQs+##2fReCzM(e>C8I#Jmb9bsH~F6j65NM5)^ub% zm7f2~)xae*cz&&8k5=@P6HqfumxyaEBc3}!8bb-_W$jY=aDi8ZtW=XF6T;E%RS^!h zAJ?8vYY^fb&Vv=2HWjlb+zb-BieNOE_^uQeCi9fK7p)ONq$fF~vAZkJ6AMkM!*CRot~0fPJ^y5sjcO)9JkkNS4@JL zN(dC>p2e5@k+<)Mb@gaDn8Yr8-x?WQXzgzDHKMt!9u~qVffrKF5UC`J+kH_XsTvO{ z9-zW~xchzEoK66Le#h_RSH|3hFU7Q}8egQi#U%xWZ*_3M55hyY1aZRuoX=H$wz8Si zHgY8S$LNR?$=8QZQ2GpzMGb_7m0jA$6`#qm+Q-)ttWahn`BiD=N0tm?vTlO-5`6rR&<7D%uc#Zv zS{EIn{s072w^G6!giJfIz~9+vndEegX{ zPsKy|R4SAs4=Qq2NY};d7XTb0?nk1EkG8LR*8!M#Z#tkpg7o9G)+>b2$@slNq5oYZy3ftn>si^5 zJDoBLs31QyGzcNXCbYK4NRb%!KFx^luv6yi8S496a z@VbIHQkUuoU`XF9w*{Usu#<-H>#~+?2hjNR?sxhiKN{_J+x>B9zpx+IXVVNVUB@n~ zFE9IJ#f7w|pktaBkA(DgOutmY9I7EOCIT?A`vlF{1SlQ6pT@QW#%4A!72Awv=lk@F zM@wL=O_eUjHryazyHO@^Q|OvbzB_}9y>j?zPSI{tK%kAS{HVE6Ckfjm?zYH% zKp)=aME2FRN=67Yvy4;FssBazPlVYINy1GwH6RxaCItG&c8FV?^JI3Y8d>)IGmB6* zP7}+7;dM#-0R7n9KcAW2Ha2-6C~z3Bk7=WV+O6Sq#zSey&E3QhDH#$SaC?m*tT-_@Mj2uL8=iir@xz zyw*0c+q6=bc zsyF_O4Yb17sVm!NWs5ib5O(keH)*)0>W5lw+PB-tUX7tEazGvAPfoPIiam?~+Bm^6 z9X1TyFeS7_b%x3+4rWazg*P;UEGmPWvV<5qQkCs9gdN?CX8nI6B>|OD;`QYY>*y!Pews404|d4v-Saj8L<6JT_9+<$g@;O->cH{X0ZVX5moOBRP!0hb>@yzjsdg< z{Q*?%dLRlsi1riA5S@hQaVLRL5c(&LlKEE&x~m7Dh=dRSenb3`;lUjM#zgJg%H3Iu zmzf25M-BFB!;M$aU=L|ye#qTN-X@b<{EBOK*>2?W60FUyjy+Fgke~R71FA1c*Mx=j z)X>rKM-6ClJ|W1;k0X69v?hoHAzU z?-M>*)yA5|fR?Ga4cYX>$K;ifyz8l;z8JCo^d;a%;G|X&Qd>Z4RK+Z^8zjl>qkuXR zf&UL_g9s|%JdWi;fhP#T_&T7oiG`hL5G)UFD|HFW{FZM#?7AE0u+KXsl_{;(N>$Yv z;<6mf7osMES>*4DI^_u%c-x&7CK8A2XpRGekuAtc5RgQqB;0j}T5RLTAwLZT0MeLy z?l(8ZRx=(3gZ|nM@Uo65!0CL}26cvNo3Y(ONdm=(J|w||8?gc|F!zU2fv+JNWNGg_ z1)npSIjD(nOlrfVPJ#lb1-+Ltp?cE91(d(PoBuM$rvUFp_}h%q*#~3_ldNHdGt^? znBTK|Q?Fq`bpPbI`SEV%)TUWaDZu-QKaGd!!)1HIl>AV)1-NkznmkpxA1jRJUH77L)h9E^62YCygay7gW=s*}D zx;xPP>mY0Dkl0P0DtID_yvuG)1AE}(Cy*pQD7#p~U=zw!s!}tR*FJUL-PI0EE71K< z+lchFjb1?r{GUjwBkKvocD=F$`O!f%<^YvRGoC*Tkjq|=1$EHjK%HimhVH(cp-yV? zk?l-9Yx_1=*&knZ+Vh}qJp^bfv8KCQ)G1DbbXIg$$-Zd~7rLMO!OrfIuIWcM5efAi zMw>LICdO@(58Z~C$@;95Sz+d&mn5`T{mbO9egWmMoj0s8GdCR;|1LBQ9| zOSfy+t7BaV`L@=15%(C*3PdNrC>#Lvi_6sNvNmtC6uKGC7`C$09nx21k?LW`E zM{Liv_(H823ZQzg!?aW30_rJB->ebn2H-%5i@p(*FR#0b$$o|2`&=>>rQGV$B{~QV zODVOiovWMl`rOFY@sMscE%JV-z)o5I&L3uLN01q!x?lo?{h-+priBPbqk1|7$<|wZ z2b&B4&6BZkPJnnUnz)VM{p8Ck`J8kU*arfN&aQ|a|7&AdNIvZdTHTi&e(LWL$Q`F}MYTv_M(-l6e;H~Fn5a7$TTCKyz{hEt-eNBv)aoOdg}X~sQA#h) zg+blGfChM~TIFtOe&@z7_bZj& z33~iM>W*P*@t|F>I>_<*2dI0!=y#)7i&49h>ZwbTwg5_IbytTSv2xuog zm1maZ2xrOeX@b@o>_S=P!8 zJx}t*;9qXaYLWiMwl_DA=0`nC*ECjZe$%h};D?%B&x8c4D@D4+^OB}|-dpekdh4L} z$!kq}`#y$WDM_l84RE0#=F@OT^oKVbN@Ru85Ox;n!%cU0sc$52sXi#5{$w^__;tGg z^TS^SH9_>7*(~NB4c%MyemXMR-a;jxB%vNpXlh8O+~pl)eBgc-P34|Nc{tK~;A;1dsJO&?pe_oKCxSgj+(@BR5M<y z7pL`iiRI-MLS7j$W#U}(W~l=z#pR2}4}?9=2q;Dsp@+Uq{UMFuJIZoI{ZD)jNGWCH z&c@?2hofLsa4i*n!byIb%DJ`|TlEE>Lv@l0pz-Qs0gKi;l{f1pI<$?f zPh98`xql8JLcl(rBU94w(D6;%e^+3^%@Obc1)wCp2}4jH>fPI@pi##MF!hU#!XrNj z8xEv6&7#d3_;3g$@o^LL z_k221nhT$630Y{39@U*Y(D(;6B%mFY)n4vOY#xnXUq6F0b-}nu0^dgpMq^c{wC`^T z98p$^RCa&v=obE`T=N>x^C8K!4IMMlCm9LyA{jH`JQrBd!}^^B!-Eh;oz(pz@8d64*g!-oyE?1j?~hA`x}GZ4v%fmk5Zb7+6b znSH9TPAl{Wx|t+jwWfDNjN)HGp#~O(M_X{?70?D?nxlyND@d6yl5|q?7Z9Pu9aWq* zkIQ)Y8uI(-UjP2&uYML0Gic07`L}cDut5YGhW;clO&{QnX#(W&KF`?5YuDs|<9B<3 zFnp8)lcJn9&L#LNCp&$@Gb>V+uT8l8x@F-fTdq$H9RbZ3YX{#Ybhn-h7+le;?tgFIEV5q!W%X>=~wwXTHYZJ z2v}jz%1+;*R~QK_o8dq-&NL!}e)0MuL}z4JIfp?S!vg$TQd-E1 z%`HEa6XAR6L%1 zBkV#~_iUWtW$z0K=ttrujRA}Z5>cHs`EBpe;<>9CVAx=KKsFHb9@bc)f^M5NPXtgC z(|uJD)hvl`UDm;lO^plzBOBcTd69~@bWqDE`+y5zPlVmlT>VTQ?(_#rY#zrc%yO{o z_<^y<=H3YwsnLyKh7U*lF|O#&eAmfY5}4^O{L=aPh8WHiprXv@`>#B;7R_*5TUvxB z&D0-yCG8JGM}jR-KX*L*_Tk={#O3_T@gJmqwS=C&4$h&6G}kFGh7?t#^%3D8bN`CZ z^fg;43*Xh0wl#ovv7GWJNNfsC;PPyb7sHr3)lbS>Qg!S`PJ56vRjYLE6Ii5TDC;g> zsyUK0^Ad6@Stxl4J1I3da*7P6=Js)@|d`ujw)H($c&H%OjptKDWN@kn9U{Bqbg4x4YkRYaa z%%8u9W+ra%fs8tzR-HFA zyM*#9vO?&#B3f>Whq^c}j}a9#GtRU^+nr4mTDi{c$xKk|@S0r1_XryLKYYFXsLsu~ z0|gLJ!qxHX%=q0#2_5xe)aBT0LYU*nSbeuzWBy`d53hzAZ^y@yA~L+)a?)A2;=}EE zTsMAOd&E6ESCoh%E#LZsYI5FCzqDqHHDO9vTmTZ~`B#|_U^Yd`kYV6Dr~r?1rN%UWxluH0hU+(PyC-iuOIAxA}1ksQqn-fZJG3(oGm%V zFH<1C1W;~|*H^E%LGRBS5w8AhgVNdM0(Da1MY#1|ckC;jtSjD|!*Rly@y zQ|uk8-sDUtwYO1h-k$tWmwE{AyCAIR^+xfSVEC1ls_l>!Il2PCbJ9c7G^)+A$4e>)5_z zOLnf7F<|5rw&6K;*t-g-q1O`9CT7&DfUH8KE4JBX28dzYU3BBY!qz&(9>RS}9J1K5 z1iPtIo&f2j0jaim3nxBDb*8ac_C~s>vYX#KlRXlAZ$Kp{vH&mkYkMdNs^vpv`pt22;;=zmF)9w78&&<|8ia@& z-lz$IsgImk>Z&=e=0$L3

fuXtJDg$e$b z`7o42(`d2sYM{-KAtha+r@;GZfuECnhGTR^6u@#F4x@LRu%UFK-Tmb&3jGGeH{!GY80jy{x<7|h+v<|)TH2~eHwYib^9Ltcv@U)XKC}hwFvI z?Maw2kb0Wu-ECHf(gt-ND8AMvUq&);%K?rk1gYtHwfp!pA9s=5%GD3Bmc#W-`Cx~< zzykQ?Km84BhMmBp@qA8?qx!S_a(LK8Qxc|7C1W2YSo@(ZJh=$OM~v+xWzd~LC{<^~ zUbh>3T#&gb-bfgrm8V#PAB7#O1yZOmry2hakg=s_GI6!T5H!~XVdz7Ut z#;oiy(SI~cJ0DyHz6q7EiS->j#_QvpUuEesUoLzIz${>T4$7$H{@cyfChyaD=i+eE zjic&5pp!xI?TFBRn1cHD2R>>kKO#3~0 zXv7EcAgcYO`xJ2_%rdN%C!cVM$k4p^l&>FQ3rD{;;oIP$~RgW&+U49^9*ho_;s{+e1`Oluet%d4=$BhrzGO^0Vg)Ccp!P_}(AD|0a zssm}{;B8qmoVP<-3wPcyhj7^5OVr!i^Gp8H-bgN;Rs9lU zjT1lopVW_#ziychL8rg% zV1p)zJ?MrF_07uB5VW2@)#Bu_Ho$hRp_y;1uD^su^rOQ+71^RM@l{on#avsPv{r!m zF0_Lk=>5Hr8eC|c`=Tj)YZ50uE7uys4c=ficBdPA4aZNBtm3j}`C#M%%ISZsjQ&9= zkV=K|>tKIqJESRC>||{4ONFxM7-h}*3_3ht1;3}i`8};1%7@I9$mOQMNaI17mk;U3 z#j#wAuxZRI=`RktQ{R=dja&4 z=?48y00uZ00mC0NP`kc_6RPq5 zJ0r*@1#$UKA6$RGF70Win!n7u$(uRwQ=yX+59mGrMC%oWdqt2%NZ8yZL&k0`0ONh{ng zf*)JP{H%!2h38kk>e2bB^=F7f9g1b|fi}82K80cK*5W~RcH-`0Csl5X4#8#%EdJCH zpJWeV%N{^K)FwFi4e92wz+Wy8WaL-`TMTW!cusAC?`#QCeNL8DJX`sQzWBt+9Wj`D~b2&)X zsMj{3l@?R$5buAXAcH5ga)fAmqymRS&mG3ERGP48W>eIm&jk5opd1wE>4O^pjlN=r z$k~pE$=2)!4>^e*j!@r~%UZgKd@qgl(Z!!PJ+(axXQkft#aLF&bc&tsO~GwjYBXYx z^fKrtPFp=}O4gxtoWNyA>AAUYnU@VUX!Y|hiwCpFH5^|iYZ})4-H+tx?2K9hr$2|c zUq021uz?NlOub$`sA;dw$^XlY13MWAzpg7-cGzwQX4$aoMvla{bK+!psBG2g@8 z`SjY1Jozpmxeq6OOS6ZA1Nf_gZ8-WbS#ukruRV*~5k+Y`uFf=aU6oY%di?Ne<)I%M)8ZnW|_{CkZI>6eX5u0%L8&S5527ZLH#QV%w zLJlBq{2vZ6&-E+kFHPRZ%fi+6$QKi*MLU;HCQ`Q!^6;AqE>lel4> z$3!%6s}Y1wxNTSNhJ%)39+7p1$FbH9*~!~%{`PV-3%@lo&ERDdJg z>d9wYmJ7HmC_X5H;`x)pFXsBs^^O0JtgjAevTfg{L1{rkVjxN=-AGF-p&}q6B`w`C zU<_1{Mv!h05b5p>=?3W>(%o#pVEZOMKJWYfzJK?}z3aZN^E%@=j`O^;-k&bN~HERan7@J*d35+ZeDF6HqI-w8RdHSsY(Vcc?Ia%>(+ymF)csgka5gB#uqh1r;JwB=#Ve;tlwBjvP3gOlPbzuZ=pkvHPqrPTk9 zFSm$JD+`J{gV;31wffRkgv7lwbdM?rkIIV0#9E1?@$7|9Nh`L3liN+ zr^&|+2Sq2nFrTV>F~jJ1i>b0_ z1Zz0n&m(CZ%^YYaKRn?~YDeh|hyZoJ^MND(r4mh57WvR?J7^IZWTERFI?_2pqvQ8b z5*_lj=0L2{*d*p0F%ScRF%YKD+heA>v(B6$^{ApNVf%Rqe_@ zC>|!Iro>BjVWR5~5;!o5LU)HP#8;YzDEp8^;(0#;|D^Wuy=f=mCe&`*EZp(SSo(B$ z_N(k_TUY@8g@r`@p|r<-yo>&#Adj=O6Sg-4{7JdrQ8DGobR6=#6Gcg#Z(Z=a+q+co z?-3o@9abOKo>^jCSnq7M{Iwm{gv(6}KlJt?{qgwSr#^fZwwujz3rnLUeEJNUt!*TQ z>d{{S-O2wJk*?U_a-{)pnlk)j*3`<<6ztqQiSyeWV}KXSu!iZH?;3^#pm;c+SQ(4LA$p(H5$!gN23 zcanT7X)9YFh}NM+vv0R0Zey9y?T2<=BfV2NILphl5i<`%hBz!%1w_PyFYy8^;-wk} zeW6iYU6y${50XKMDzVZQUpbxIf6HElNNB}4puUr=46g~Gz+ zK37b84@e$F_k?F*JV>c{sPH7MkWmDKl_LOX^Vim$I6~P1ZM4O z=5p^T+AS|NG9OWj^008ia=jQ*%*5KX_iA=rm4G^N7YC4EEvI+V{=f)xH*skCixemb zBH4z(gbE2wA)>JY8&QHH8eA5-TZj4EZTLZL2Mi}S$iqS-20mHq-I2lDF-liZ z3SQd6_pSqPJY|92XG_5lEtk8d5;mqn%L93zaAtnc+ev%hI4b1c8o6)gb0wSgm*R50 z30CUJKE8=W)`@<`B^S)b<$HA7Cq<+0#^?6#>}q7IH|J+&%#bpzywZu_dVv%xBcG#6 zw2LNowH!g19(s3==Xi7J2IwBmPG){FkQh_8$dyFQsZ{%XbCj6Cc`3=kIYav)j{IsA zq^?4UB`1GPW$~{NF^Zu4eK&?HQwyNlYBo$Z;3i)-s#w-6@9CZ!qJwXCKjv}q$y@^s zuQ;iG*R7b4&7V&(h1LNy%`jj1j}zK6b~!p3(k{kasb*4efEq#xecAgw2;BBaCs&4i zEq8b<1VnT)E+a&Oqui4p42a_BrC#K!Oq&kR9;KT~y}F@V*Q~mW>cOk0uH(3gUy#*Z zW7Of-nR2iEy_Asc4kYvl#9VgPx?UE`3c+Mx-Hy>8IevRqEFA4vh_ORlRr^Ty=$h0g z0&V@L{PD2{-RkOrPvR8@l3mzFUq%?AJ)gP%p%K9Kp>jG!ZNox!$58G-5zzo*%3Ltk zZr7~}&esmNm(|Euibj~8MY-@*Ulb&l63)5xxnIaZp_Vn34+q6f_IoV1*x>qVVU5Uj zF+oQwH0qf7!ahnm28ITwkhIOovUYbRx8A9EFA{X?8VQ?Z!1aYbh1 zd%blEP?Rj}2|uX~p*g0A9*EGURAeWV)ITDrinE|md*SdDz|dPLHDvIgbWDu$m2gAf zqvaZ|b+u)AmwRV?bUy{(+phFj_qE~li=W5N@JA8-+!b0OZVn(4oj~PpIGC7o52BMZEf8q@(A2)_$jLxw)p>$8d0p`*HZ-+P<4*4H z#nXc|sHlSJcc)Ys@oS@*LJ~xciOYrcE*)l?pQu`L4%Zk=Obg zccvuP^5P=lD3zdjsX2L=2&O{gCy*&4cua1|j%d~`C?mtwTHF0h$awyB$lQ+?lU61Y zd#agXv|}cw7^zS&KA{AeeTOy4JuBIjhLs8yfdXgXEk#x7C)_7|h1-CN{VA(&+5b=+ zL|}GEi>**nFQ z%}FKK16obrHA=4MEv^)GmsJ)+)eK(ja7lmk(!vbshoWX}XMoUxrv%O8g~eA4bROTs zIIeVm2?mxRDjqMh(vQvD} ziR!yWj_4QP@eBpyspyi6NLMV!R+` zOJ`<~Hb?jX(h3IxDscbS+~@3ig$lM`ctcgQO_(?Q;cj2LJ_G<9rcCGnWjam2sl!n7 zymR*>o@3UJ9|m1%AA6#nuElu^Kl)bRoRe2i(%`4T-a&2Fs-o&(BAcq=ZFJ=Oe*`@I zS`^~eu&tcVaEx|lXGuIdTZ`h5A@I=)Qv%mGe5?|vTon66{JY^y7}1^Q$M0KqzB|xN zYIk(;NuTSJW0)gJBJh5?4^;l+Mex8Codd-lm!r!9cXzfU^NxAaYwOksQdV>JA6m(k z6o~ptPh7ataRsx%%Yhy=4hEXqTljt*#OWU3b$0RHvF}|vH~EH<4#ws_+B6POa9QcS)(}H+tl)TJF1S!IA9(78Rs4cSIkODNVbPa zo(Ld=Xa83^s@Kb^nWxzGU%~Dp^EMkQyBryv(V-2I2xTq7JRw>6aCj3XCWCCU_+v#=}V>nOi zF_~LiAYYUJl&P{lWb%sYLfc5R&i^1^t$>_pKI+(s0g&DLc{0t`Mw=S((2)znjB1lN zr`c{aG!ywE<1F}63sZ*1H7X?!J9onj5)tGGk=fxD1i2o>F-c)aV_vO;I3m6)B6E)V&2 zDB?DA#pBY(eoh<}Ue0{$g6stUQcL=iQxSo^gQ2{@IaG%I3y6BWI*$FO&{-_wHf7qk z6wwyorJ643TMh3m)_w6i6pv?HezrI;*PAYNhVcQBtcUCXx40U7uw#Y`l!03q1lhHr zp#nWRG?`m=U#OG6tjwtcG)3$(_1)QH@o$Q>U674BG2BClGRAl4lfS z%OTY)AZjX$`e-ljY`&GL=Zb1C5!6Z&ES@*|D3JJxPf%4ukBRIH-S2zNTR&ikJL~4p z39(#{YHWrvIZeM6UZhtWJ@U?drU15$)&V6wwwR8!Ky&2<*TC#4z7=5#TALDH8Q{z| z8a{bUFFBy`#|yp3qTuX1^fAwUbfG( zH(h;RH+OdzR_x)QPCd&rp7KeY>ub458=@(2V2>=O!Q1&;`^kggCLRx)HW%C}3lsst z71r7_^();-_R7O%irg|!K$B;a(k9e>`ld=fNDx{0U;&*uZhT2C)qk3c8&L00DB_Rt zL~Bequ}ba*&nY5P^9Oj8cnQT;TY?xOS;W@`8$U{)%B6jgxdPqEEW9Sn-$SL87oj<_ zF5jO2dZanhQ=Z1d`7=m~7$KK!hgjo%^`6f6X8psJUs$W`yMbzxV49ikW?cUi>7pEB zzDJu$rJ{g6680xNIC@2fEgLBufHOY!vnJCj@?U+;(YV_U<*-BKy1b-g)aHu;m#G>q zzZE|lhPjC-+{^gIh2^gO46cSDR#_phzF3tGze9)+q+FG2dA}rsmUl`p+pqX(;Kw(?Ieb6p{{RD2GU+6e(HR zB*yDM%g{+>Q>NaDF<%?27R{d`5(i2vkg>yQbYEv^%!hx`I4(^{Jqgz`_Srg;Jng6P zJv@^k)eAy(wcyK5}E_Tc3nON2Wd*EY^ zZ*w8hB~Gj(-mGGRJ${2>tKQ4Y-yJqD|D#Mrv6by4lB05qLa8XvB&?2l1(%iB`c_^Z zShYu!kr0jf(+R&Q4Nl-%BF5WcZpZ9K< zY*Nd}qrYA8oF{1WSER+#ngl|GfbvkY3@lsL540?70^vS;oQN}LK$nlpmnE^!=39N^ z9e=6v!ogxr%h|N94EasW;Sh>F*T!HnCd{n-BL|z1*f^>fs@S3O``=8e;&)VEYMKq- zYX)~0Ew*4(hR4=ltRMWTv&wE0?YszQbgVk}BfpnsF#BrAtAdz+o@@5AET#41xzQY5 zaXDLNUj7!l?TMv877MiG$%_5=`L9XbGM<}iLtfjf`v&#j5lv+!ut%F}T0Ekc8;u`f zp9m&++oxF$B;$|?yz72%bIdlamFN+~IBiWGbZ7aa(Vh#mra4(uo-a&sM zrG%Ex+j!Us*jGYFPwyw7zHWf28hjgbl7Gu?Txq~V=t;-)x594g0~}b7D`@&&-Aw{_ zr*^WqMXX2=y}%j#K1dJX+a9=j@(=lm$>CWJa`n( z;vtob=*6qZrZdZ}{z?qfw>C{W?)LlC+K9$hpE&8?(%&Oa-BWiUwSFpL*;~V+IF>e+ zNoUj#Qx!P!uj2ZPtG?Nbd#h$UtP!RC*>To!a+NF)G=IjtjOU9#+Eq$OAN+NiVwi02K0>Ddx zG|+L8S1+PfN~F;x*j<+?iI&%iaN<*hED=QNI8j=O6G5=%t>o=}CFt04i$ zpCrz$e2rx{GjZ)+m+}^_*}wr56vbUqei>%=scuV?OQvs^e%;jFd#C?6{i4`!jt%QZ z65v5%X*y+?T|TbSMj!7mfa|-MwAbm^F3;lQQgaYl&Og%3$_QwLn4?cg%rftyIK^!R zFTwaIl9hx~?#F?GL=SI}S=?dG54s0gvJ3ea3MB$(i*IbWg3jw}r4b$Q{WY16kXgB& zI)QU4Z(6C(rZ<@C>vYJ3vl#H^@C&6=7FV1o{EL|$%zI1AkY`RxBZC{Q;xw+p`O?o$ z`gZrvG#i%ZJtRCKx3@)=h7^+gwomfs{aoISx}-43j4lq%dcOv1I|S`x1)NDRGI3t< zL~Ncy;NMhUkXBg=+q;0p$+7kakP3t_386{tAc1xPV>zDl?3382F^aq zk&O*$O?u&M0KS1MjT|r*$9c*Pep0y zbZf}0mtD?+n~^7TuV+}qil&)qLX3YEa{4=d(3*icfy{e&$E}#lx2O~j_x;ez;rTnW z>eNG(YM0d-J0tU#cI~NpV%Ql3E79KRUR{hDJ7E8bPnZx%k=AEPBh3hZcp0F~ zf%WxQ@|Ny6F=wVB>_N#r5_%ys)Now)M^2Xpmv35sDF8}hYXfw&Hvtt@70&{dbaZLw z{r8*GTlPcUw`Y0RoiPGj{wT+iJPhapos-KI56lw{P0WrDc*DxkNpsY1&g+S_S=bO&`er$*(ZH zA#YCGWX1By!$}CDHf*xWYcV`H7oNJ!MzID&JIO75B!0OR-P&jO7EPz(1($ApQ3VmF_vy^3WAw?k9TpO!KXv#N=Vpx!7|jhK-S+p2{dE6aU8M z&&YPH7xCf^(v6DC0meSLBr-f-IxIXJu9mF!n&kpP8G=?Ne}$|xIPeQJ=rxR-z*NG} zTP{j4#HyG$u3WCnbX$^mPMgn1B!A87F6oy4?)HR z3B?+vB$&C}j>7Y{#ih0>4NWCQy|#Upy)sU9b>%f(3v0E4(#SgbI37MsVw)gqo9ts; zIX!u~XJJ;=-xcF$%D#|4)YDf?(N>9PS9E`!@>n5^aFw}V;l8%s&SeFka9sgURf%ao ze1qJbbCjYZu9mETk{8lqTfZ%jF!Tfes&aY}eEcJ+((<}Xh3*cX(Zz))%3E90k3n(O zoFKd67rvIeAnh7E78Ttk{vKY}01-nUIb`{aEg`4iALoflrRE;t(57F`1&?WK8w>9> zGZ7_TCR17_N{DetG)DI`Ue4e0+!6}*Kd6lvme2$T84MB9kig2?hxS29FPjw z4Kp)@_|EKA3<88-&?w9n@ogo(kgS<#9D-B_-#rMemrq}|?#igJd;d{nNj0eFdty9! z__4xMTRD+^Av?|gr(1XYvO6Mn(}BzJgW7cZ(o|=0ycWOy?iLmA-;W zu2GWRjUzUBa1!(l3uhj;U5odD2gTde+~5(_@r_dup~^o^TNF4T3UE-Q%zh)Aj(3Rj zW?p>o=v~gFR>^w$*6$z*M+{OL-$Fu?#%RB~;(6SMH9JFh^^tMfe4E)0EHV5qcTSGs z`32TNtvERl5hi|t(tDFNM4ufD(!R?t*t;kl@t^qI5L~yWx_c`Rz!}}d6XclC_)Y&6 zxYb88KAqlAq1BHi$Z%5ZsVN|$WM}zflBeP~&nTJsr?VeT<;MwezuUbX!5Vr!EwCi$wq7=4M*Yw&=YbQwLeYd`c%CvkP zwrGguGENf)5XD|_)4+%mz`x`ck74KY@|}vnpj_1uk6%3^=e@kDtvj7*L(Ui!V=Xbw

2u%af~JjRZD7eihCc&qa1bO1doJ&P2$M7Qy?Q%=AU-w9kSJw@ z?8`pB;i@2OdQ01$DOU;!rQ?@j?_-3}>Dh)8SVPPAJu4hn)U#5~8ew)}Qs484D^FsM zd%Sb|Y;-eVeuq=bfU2#^NwQvAT@6Ak;%$H18q9?vp~CsvjEag|Arf>oI(eS`)_fB$ z_+etK$io;V!zjHm`Hyg&%~3QE`nRq1@uDc3({MS!+dw;>uXd>GyG`61l8fs0MPXbW zCxMpOuAUzz@9^4#xH24S8`g#fq^gEIj=ZW)gJyjzNK+xPCQcGyc}h@CD+Ms;HK^gzfUw zns)LdhrPYl^-UTX|M+4`_^{3Cv)WJ_dwx4z&NI)kqe5A1)q8Q*3T_W}9QS6yp3?g= z{0Wq14g{~DiK&>cv{$hfmt!-8vPf3~80aN}L;vmapum~^5a7oSUFV6H8kV75^byC= z2+EOOjF{WYkr{4yu+1q7Xar8&w-T`i%Zy}D1UbuH7a zqw?%drXN3M%naKbolE;^rE?o{hbzp2+|chZ*i5(=2b&4Yn(O+kJOpt{=|Bh>Wj(E> z9;sJ-{!o2YrVj74&(?NYM>#fSnvc;Jo6Fz*qvQW=ekB6$Dl9_HWG?__7KgMn5k?Vm z)B;>Vx(A+p3n>Wfd2>Ykvf5FaZ-d0?#^hDj)M+piSi%HtIF$A%jhv(%sZz$s){QDo zceU=KiPwjmUImk5`IOA+LzM>Um;2AFue9l0gcz#c8+(;egVC1DDzSd`QfqShM%=+s z6=x3my#C?%XyfYSpTc+;1c#JD=ep{byaOgpVJ8>yuYf_OROxR) z&1GI4bg*Q;;if9Pul4C)eZ*rEOAN!DcnpD9_k8Y3sqOnEAfE7bWh1Zcc5y@t_M17Ml8j%Gpow zFqi|C5jPyqs|}Z=iVxBl+VWVjK-S{Pg2-0+{J)EjN&zzoI@!dZf8MDL8Hp2Hgcl^)*Qp|Mb25? zsi(k1s{2ewlgCK#f;C~|Z<8$sF1?3?1e~{iSAjQOnQ@=_0*4(BDhV$PmK@iu03T+( z%4VKG_f^9!*yJ?V<F7*r63g1)OHJe(q%zqrG>`As~g(P z@ZPY<=KU>VQDxfRngA$Evca=x)jyb^B^g-9(FdWxvE8#Dtv0}OKG+=*JT({>BHhi;gZwCo}V{^uV1yE5#eE9rP- zQ+nb7q)G=t75h&sZr5|1?avf-qnr;FZtZ6rO|<*1k-P}^ed^DZ52NWjrrB@UIJh!8 znufvcbruA1d~i+r>Qt|{eu|YyeS!*p0%4mWS@9Osr15AO7}OqIlioBqza@Y2K%M zNixo4kuo%m?82IQClLs$YW+m~E!OPAdi46`@Lt7HUUm@*{;RZU!}}%*cW5?x6Khd4 z;NGaHQ->uv*f&z{3sGZ$4E0)9m ztwy=_=O)2-%h1oaOB${mfYL2Ry|LW`9`YtbcC*#I6VV=VBM+oxc%f(l$3!4Qx81zkgOvJ@A!o*OewmU3kHgH)k#Arq}x zw*bqHcIA)sTNDaiec^~45|9yzG#pI+qOe7<#(HH44bk)e)$EqpXRi#6X&Tsi@n>iX z*j{PzJ_udHF&V3!!PZ0HDB+wU_Ap@XGykk{zpizkiNedrv}OSF!QgA4^(K3C=Zf*$GM{Abd)>% z1LTS~2(NP35K&KvhVvao-~61_zRbD`-WQo89T@T6lU*EbIbwktHrO5eMIc~s{8ja} zsBdC<6ON^P8LwOo$Aukzryi1fIqFAFZazM@SCMnXn)4{i<@&wLQ@nV|4f<+?(&?@! zp)Ha?GbUO`tS+D4kI~vw@eIu<06*8?@~o_q@)f-BShjw9Y7bGF|C2*kM?zp2bl~}P zx>*@-kX}mL?g`1y;vaHJ^8OAtekd4Q0`csqhL=2OL{)(6w1PgKtHYk3Y_4Lif>UP3 zTPjQEa^wR1Gnak3n-w8kUx^6uRnyoJ;%@H*8SX)RmOU$n`NBwb^iiWfMzzf_Db z9q`e3t|`3Ugf%^{l>Z>kPO}tVr;rxMN!UUeDgpk~bMoG5t1)!G$IF2`61}h{D zuAFFHnXygg+s(9J^R^@O+pOp#k77}pxQhX*92%Iy0h64iv|u#V?|p^E-mL4HqEoXpGHL;f0J(qeu-=AhhIlY? zpUt_%rK;>$@lUEh+tq@eOhj_OKA8z_z7&#GV_Kb!gl^QN^uGAxxiCUvNTc&IYaT-@ zWQ2cu;NCr{Xy%ZGlAezyEzI;409!Ws*V;1-Uo*ZhW^DZJSb1zRL3tf(9D}@iJ2+&J z6hPT4#T+=0|4dXu6bv4bCgsJp~D-+AxR&?RIwGy;? zVJ7f;^AtTB0D1X0OWRS$BNK<4ttSBY&gGU5K>W$Z`{1J!%2(ZC1+}!dC9ljfC@#r= zbVY5-t?fq3W5bxmwFk_*o#hukj2W4Y2E-b`Kv4O}X@HEE7n#h?6d8i z#&UCnKMR%?7))OY2nk(+?7$dGecMN9Wwv5&qL^ann=aB)<%3=B|8X%vM2x5l5zi#< z-tQ4J%H4-7^_e|IjY`p6YD|#yZ&NNgv3{2`d76h;o@q|)K?FeUzhqzl1syvc?<2GG zh1E_Cm1zcSJ`9#H)QZ*LwidUlA3)g6t`y|lLO6Sf)vBl;wflZNf5>aN6baVs-W0P( z9iW2DddSMqWpU$k-!miQ9q5d5n#BC~EUW>JB0rM8_M)^tM=VvsJy9IeZ%@}{*IyWV z;^F)r^aL*Sy~culh}3mEh<9vWyKxx2ScqXA`U$ib=VzG?@)~$f0ZB(8 zoOCggk>6eiBR{|Uo64V@u3K-hLzAm+Rl;-p$U28a9} z6O1sS*SX7C7YXkExFCYo>#3G8R zlv$bUv*RB}M8$<^`kZYMx2(&!&&Lp93Qd*lhu1?sFq2i{9?}ojTGNU6Fz)BCviyiY zy>f4w!XEyiHLjMDMVO#eS@6nDbNj`A1%U!euMJr06+{#|nqqm`AEtRZ$f~3dnK?rV zF!LLPsa$ZL$zD1tOu+3Fv8x3f!ghEDc(o8}^FH=O12 z`$l!Wfq@E(2V%wNm7jW+8d(NWd~f2^&XgnXJ^wDL<&lrtr9#UU!yE=%&PtYzQSJDI z1?=2^tn4?B=G4PA7VOyIDuDyX7 zDdB|WT2reulN0ZV1J!_)knVVME^b|U_&lz)oZ5Ops)EnF*2Zyb{l~3)6GmUe;@yuw zx%lqqEa2 z5m8oJxaHcF{XQ}U>tO_S$22kN(*eD|*5a#Rg4Xlj=@ZT8L;uEVjkVD$Jr;&CrFO|y z)4GdVzI?+2bdoph*eHC3yr~@=@s$r8AMa85vdK5kfmnnQE8qs?prxE`jv1N&(e?nI`((ws~3>FT};kxmr7bf|As4m z%CpYowk`*KbRWmKw)PN)lxWp>4jVxbF`HX)!}jQloG(T_RR$5H%sVOf#27bO7C+R@ zTGhSqj5e{CV&%W>Mr2U;HKw+Hu6MJR8)QEK;0e2EiusDT_`#;z?21$Tqzx>$(C~{c zLit)bpQu7~xyu0aex^;E;n~nnb%erOrcS>p|8r#S^P{b11uQ*#8J7uzNG}IiZ111w z_4`&{DrD+6`mR*iib;7C@-myDUz|Os@sjOb+G!M`am=5^UoF=PF=4vd0tsk(YA^%x zdWlwn1Lj!+B#1L&buUVqs}FR59cWwlKC%TKqGxmf^WfyKE8l#iXA@63j2W_VMEVSY z`MZ8&Jnl2&Pt%8|l1J|lK5qt-cc#*dB$JjLw+^^K>hzZe_tpH|s>brs90C7UX#Hk* zy|=gX?0k$v0fQ$MwSWq}X6K{l+S;c%c8&*}#5Dk$l=ss0ppe}lg}|%#y8t^raI-4J z<^mPeNHPl&HkblIY0(RiLebZmpla`nnr7z%oyzmqu9NdEr**ftCx9Sl=?2)ST-XDJ zfPlS0ia9(|c?=?j8Tj>UIb+JgWogFvPQo4md==I@EgTylE%xn-hW#6yr8^PejjooQ z5$+&;3U5IgY8qS#`#EWn3*G+9+;W@awfOyPt%ca+PUoj4&vl8?BIDfcy(^=npH1c0 z%+*i>F<+`0HVAP9X`5(m?|umEl!+pLE-U)rj0OC*uprEM2DN-gyFTw-sBP%T*1E?l zr-55Y@2Te*sEG8Pp(~v{F|0L|^cV>ys?j~h6V3R`pflTrSe&O*v&6nW8qc(mWXzUB^i z%F>}a?f2R+xX7X|h4|acW+1G>`REQXWe{axd@q%NKLo*n@?wH!Tp&wL-<-z>tn&V< zMDu@Q3o1O@Y+Sgk0B)T&;_3W{wE1nV6D*lxSTzJZo7Yju0T*YRBj3oRTqXz&w}ISf zpwdh5_`JuTIsS%*D_L#d6>wV4Y` zzB1G6Fq|_ge>4yjhvfR<-}l;^ntRa^jRIAD!weV$h<^Edyw}gGx%vA!x8J^KrC62q{{DptkS zIs0B~p$(c|UIS2nUvr8E11Xg86!ctc7|ZZ6>c?H!Ce1(=s>d&6V>cM=%gGj9n{7;( za>jw17cROnOaqVKk%enUMDj``-2P_@!9UYS-a|$2Vv2ZsJWb2rO9pQeUAi4L6OL&( z*ZK|sqV|tTu(Xb14txO}Qh1rsw6b@)J!CFCl{nGJq-DOAl2M(1#z6fwI!rs+BdjT( ze)DU;HFXz9!?08lIs`$JN07jmO%?IsX<;Ex|H8i4 z3o8@(9oub(KbQP9vbFZ=r#x;T2V^eb1JTOWDT?2273CwlfCpv8{tUToB7)9qW+Ipq z6!kzJ8~z>T_IKaPc7H!^wmkOh@5f~-EqYGEe4XaU8rjwmbh0<|1P&A-ehvGe{U~c# z6b>CiA;ppKz$765#=yB%WNBUfuIhi1hCl0z`29czaO|}VFf%FuAN>${TC#&)Ab0OP zwh!t%-UOl4wp!ZwUGKO})?IyEq76MD<|h z__YBnhz{P%reD;Oqq%Csl#r+y*%Iapq&(K0odC>)w$lXi>Eq*XCI9q*|GW3(zxST` z#Qa(s%8$qHsE-cJb5N$NVw-~fj75o)(%CJ;E|7#cYbjUp;& zr=wS|1^Hfwe1og-e=G>&sffWNyA|=}zmMD`lJy|_5W)LROqcv&;gYGgk}IOZ1~tY! zvv2UFBLiidwF=yuJ$Rjba|G)>hlS+j+d9?&C>5`*Y__|fSTB)hboMFS}V;zSms4` zU-R;8t?jM-+OxLEPPVNcR9`DSPWhN|%Zo;N*8o?FZjm!nl|fQbyo+i3KLFdcsaXyaAtBJVmU(B&4}tJ}teo zqagCzzy0HhG5=oX_xqY44`Qn^hoBGueV*^{b9g$5TH6$j3o4yE8_hT9Z{bMKWfSOj z!1W`p;OT`w=tW^v`m8unA=2rqY;=y~*k|oBTPI9g;b+qTx8jch%s1obJWniIDz!K> zBR=83PNpOMV1y>Oj{8s9jWODOM$BDEl;B>)_$P2$IMlRM5-CH&3qYk9UtV-foH9x8 zSW*wkHAegp;H9ekO4{^Xp&~O4Tt6xJU|M4_ndO0~lEFrAyH@{#081uDH6uS9v%Szw+s-h=P zqWiAX$Dhp43*zn;8vU3>irFnQM(mX%8u^X@y(@Qj50M?RmX zl<2#!ezY|*%Q}@I%3@oygwT6gmF)?5O3m*x;jql`?K1go91k6S+i(|uJI?FRzNob- zl;}7+0Ju;I$pF1wbe7FnT&8AW)F=Bo@Y!38sTQF;q0TS(!p}>{{l)0Dd`%?cQ12S~2eEqxs z2#}T5rm-XI={&_NV5Y+!eImKXvi)%Rt%sB$ z3OuAhm(b!tF>G!txt1hjS+W3q0ywv2G~h+eG7iv<#$ro(e0M3d4QGR;e5l|(qs)%i zV)n?pkwTF!$XRen!QVM4{~W`V44w;dgP+kqYmmo7U68IzH6|8n2_$$9=`GLZ!>PFeaFdtdLR51 zll^lYiXUSEpWeUDd5Z3q%^4y1CLMjNl^PtYSiEd#2@x52vlKvViNMu@&M9dxxbeGr z>;PH!p|U&jlDO|=vVvJpowJ0-jyI)W3)LIK6J!Jk?j@_F+nyJV@WN%Cy%Y${lGrpT)MR@pjrT$USx$V0KF*m&bqF+0oq7e=`RvWJ~&u|#h)PK+9XDA`4E7O6Y zAFccz{(qar3e9(@o#4E#6-@FdU-{2TJj`cjhuF9B2cbz;gannt8OrND2SR1fH6@l! zhy#y5i9bdW#hMQA-2vtf-nAR~v5fta!gp3{J64d0B`BSP&kB7d2TRMD?~0*G(^B2| zbmesr<4OL*O2l?zum(a?w0dSsvgcRJLb2w?&MD@o4U)dF)mZMy z%<=Wo?~Ibk*WUJGp?I(v)gTNHkY3%B4DuTqY0RgijoX~folr#U_8(d?plx2I`cmwFD$r*3Nd*&H>gb_>H&LS)VNY5LXMx2S9;Z zWXg<}TSOYFuqZF_U_tgKT%nwB_PiGafsxeA?ip0leIJjR`+{i2CG5_#0*K8w`q2Qr;j_#%8yW%3kN(#w+1$t}EgO4hl**LRft++zwm?R<715yoqFBo!boLSGAc$}4wYQ1t zL6`^Xq=7sGb;5*__GUJ#DEgv;?RxCoX3s3{RWE%Xid8^9?-q_j#?Wx6LcB^R3$!N< z1=>BxM_sL=-3Bw@Q=eJ!{lmiuYSw5pd@$8E~^2T$OfCi>N7XBhJpOkB7@LkF(oDefO~k`3u`%n5bM4G=7t^Ow9}caE^O#ijD#1y7K=jC?uT8ekAB-;Tc8qy zon-x1DV^}Vye@#R4eQ$Q<>XQni+|U#mekiGHISjILFMW~pR0T-7gu7n8csH3 zgL=vn4B!fgkz>tiw+~Q$XykK~LvL`r@d>aKNp>eBLb1z{5Dg|~qYLWQtZaVFMVvD;uq&;^NVmfOW8a?YK-B~;+zYN76Y z=GW?EN*9XF_8~9M2De)*i@Y*_hZJU&{$LE37}k>-kJ^(U^ZCF5u)R2u>dg4z zk;t6tsdXyx1rB`+)q{%=K^M<*JO_I?qNOXo78Q~z=}Z4A5`&ziG!y&_Y+au+4L{DD zB=6$6?bh!+8VjcnlS(De@K=|7OR-`tvy^%QA%e!dsi;qlC^3zghphvX;`cJO6^A3} zW>@S&S?Dchw7c)vH@s6EA0{U(xoyTM{w9-)QCT)kwx;Ung!*DLVQZsH8(jvLnI(9b zSPI6atYq>n+&{L1nU~2l2r}O9+#EX@Ak5bn)!Ub;!3;l%f}D98et+^mDIf7|@W_Wp z&I6cm4o;F^R9dFxsXvV09^~s?D?K{BV#itB6Y(d0K^a%kN90v$M@lp|LgW`OU9~?h zO_V2o^3y=M5pIK90wWHQAUSEXp7Lek>(kKD_Z_;49v?d$QYqYetzB1B(hJi)Rh=Hw zh95M&HHwUDOzDyCZ<7wqYYLVqYpTzPh=oWt%FSKAeg_lf$6~QL#3)E!Wz?OlhfjE* zzIv}c+nf++HSZ7bzwv%@^y5I?*|S&tQApH&`^Q@S)&0!k|Hv{iJ6~2txtQ*3s*pL2 z2`Od8MB!1!!anC0#{P3Z=JP$YV88|L&{GpyLGnv!Yxpx!g}2OPii@ExiuLixR%~gX zm01z^>}!1TAM@BBh`xg?WcchtFMgD}jEE)7UHQX6|BtP=jEger)`y1?6-4P!KuSWS zyBkDWy1S&MyE`PLbC4F0R2l}PL`u55yJuhqn0I*2Im+|@yoFcZhQi#&>I>qa}X91ghC2H-ubOsQ-uC{a?Np7#bnk{6jAyaViVxk4QS4 zb2jU|WaGFO&8~A0G;uD#^g$}d2JA}J^MjZCK;PsA(W6erV*n)<5qs{2szVm;`2(}* zG}HPw6{PeT&QaFqcw1S=Kl>HPg^uDw9t56-^+{wjU5c1b2=?`;JsV(nw4UATBg7Bl zCbN+P@DVwLHIjeOqF+B?L=k0CcK7nfgeVbxT3G4$PYI{EBJZvc@So}pGBULTCEB4W z7`CGscNM0&xOV4Q_#gB^wHvQT=idY{d|$n`B%Q_hEfir#@ak9^V3Scn`pNIZpH$DM zd!=gZU{ADoaF_g+m$w4KS6w;#EYM%rMML=Vu-n07xGhoPgoiK{@hzKgo&k%|hgnz% zBp1--+4bGm#I8J!LIg{g|5>laazF>Mu8-GaV7%*1r>-ZgSy9VXZR>y7@;`}&i5_MX z&9-nEA_2GVL13^UydzcD!^`3kJO(znWT8~$9msE@>LC4xoDGcLTs(vGODFSjg&@I4 zO8%f{Qe=YO36D`UGvX4DYNqlK?xWQVld6s>ESF;ue|95*>vTw&vU!~Rb)VRf!Z~c z|5FM4n2F;tv^Pnnf`1XdH@hmMb>Sb)y1ewc%qp<$>oPre%v`SUfDUMJKahn}eV!rC#{E6OX(vgX(wsWnA6!mm>(}atcx)Cvf9BU_w=o~O?v#` zeUODWFW`X@eounfw$}r`R(C(Eb!ldGzEhqf{*~j}|EEM57#bp4AFlVVc2e)ob*xb_ zUm$)N3^HZzfAy}$0xAnCzdeq0pe@6^6_2b#@qCkMpVnO6&*M!@Hq=_VN{bP;bWR8L zT27xkqrYXF)$TJm53ALf-NMgO=D9dzqr=)Y<-F-k5U$H6<>mY67xQNYohbHe-}~i2 z*>HyD;|C$+n*KUM%h_cbc4z+Lxh(%(Bi|+5ZDUu<4)9NwHZCjOpwCTgH9$F4@@kR zpTX+!44fbzxFGMngI5?L34&cgv?nf4ZF>TOoQ%dghgnxMQACGVOuTpBn>|}J%V-~d zbB>{@;8?l3yGrvd;mQB$^gjy5c!aW~&78Jlop+~K4B%rhM8e?*;FRk&WJZuRJ_b7x zg*Iq3-aS28@3F$WTMSF*7e6O%SzbXTZcbAnVqW2u@Zx0h;$TlFZJWn$K(|8*mx%$X zu1b5JlWcZjfhU~wrM!exZk$*@{og(Jfk6W zN9MoL`~mhK5`X|etq~u1h90;@23-k$aXnXnEx^8N)?v95SeW`ENQVTRi-%G|l9z$0 zN3NNQY?hIrjD4qP(P~*oI2`G>yOx#04}VMwYog_@(qufc`#wvZ%*{kR#;I^U`sTL+ zjqL^%(WmF25`}N`cBizzQ8*{w+O&z{J;WJt#-zyYlG9N+E8G;1naDV|`C485h!ZFi z;=G*II<~s-BqeB~W@_b~wV8*Dlq}K}7 z^egF2D%O5gdtRTR8)l|ly}`JT7WLh}Er31S-Aaa+YIfEQt6%Q12hzT<f|MF|Dly1HuC*72=I_3hP%VoCqsRww|0)p*P3 z0oBrJ7h=n7xE^EP4%%Sn-cb!AM3FEJjp#A*S>lJ=@aAoHZg#*Yejw zs(aaoSck0&L{*+dlM}lqGP;U<)m_l5=f{IG3Kx-!1ZaW7L50On$~tIEI%nnj9#8Jcq5(Y~@Z8-;`z0zFL7--q_X37I&MgB`q{axJ zELtCa((Z9qK;$p{g2mXt1eD{pv<}x2g_T;5PQ8rX7UWiZlnnx%u|48ipZz3j3#Knf;vB9_$@Ul0#de@FD$v8skOXL70-cvhuyB3F?K#pasdxq%;qz3emn`)p1 zE>Pu4BT~yY7!){QM-L-^eF-UeDrP-7o-cI)g&4Q1F?~j5F97Y#U{r$FliU^gl(VUh z_l`UkK*Z}9UONUMKBpL(=$_q$Eia9v+E)hwhnl#|rM(``j3;MxTnEmN4c6L7|LOpM z`c@$TwY{XXf0d#Nj(WIwdr+TY>l2GwDBET#V!_%yJx4>mK#57qdSMpeXEr4K>3stJ z0f>|pRAjx_crmKV9BB}XCA}Pca z*1t**Drh$8TdEeo`XmF?N_@Q&24eis2Vqg)Qd@UhvQp1F@mMt$V6G%XoFDwxv~OV2v!Y#INc0^d%t@PXM`>#ACcwXSbfrdUpmjLdE^`E$S14;h<1TlO9V2zLWk zw^}_mzMJ7^8{K{G&j}*~3=UE;ySpB!&gS^gKk8;IwNRf}Entv(@@4CDb6as&-;-8)O=?@N@n!aTF zq|ITK-#5D}?e8|G9smP0Lno7%v5*!EnLwN=Qh|Z z#!CO4g-UyUf;o>()O&JrT6d1-L1?PY-r*%vVz>8(f~N~=MOK!p|H6dGQI>CsM;Y!& zguJLS!h%5qO1f&P5k2K%$UVP#vPYOw?;tSr*y-Q#v7ghR-~#fWGDHWpAL8)dwlLM5 z8QxU2(4ZDFoS_4zLhR#uFD2x|Qo#?~Ngutmc7so9ed;Gao15XBh$L@4U$2}|OX_{M z9N}7d6>oe~$qac`amE9~VNfV-c&*rBNH}5cT~(PW7c(FoNlGBV#4_cBDf(v_?eF>D z${stiK0cF&tvjf+qoMLQ2I<@C^%@`-v7x! zo=~*mP5QUgmeW+O*qSX=;bTaxuoWBZh>S*z2ny6gPo?xTP_Iq$U>AKBXNc*9MdQ30 z8MIxA*<`x4Ae|DJbCCZ8@+eLU{;pwLcJnq@|!UBhQM4(Iy6q+?X3d&h`z z5@QxW3e89yH!D4^Mt>>h6 z3s7e;shKuA=?Z|pyOi7BT3atP$1J^i_wsx#m~g1YZ47b~kUC z69ivQLEd+B(V*e5V8kfb=Usj!Oc>QRTr}M*x3q1QaGbycz*NL34IWSffSCVC5$ivl zOVb{Ingc_u@w|qQ_1CD*^*Lo3keALjKZ>PMu{TMWmNv&ETd1jbW|m$}&ORFitMHgF zVQX!dLg54G3&bWz(s7ZW0dGEENGSJ1p+Yi7ZscaGOKU_sD9`F4&O>eIr=4L5lFQq~WgU(v|Di*B3;I??}8Rr1ETiyKR`6oPk%rXY^)^ ze9uz)Ru_oHW_VEje8=|M%v!*oQSjlo+#FIAdfQ5di=8}Oe{P)PKgAj+JT2L+I6V9Y z;t|T?|37TQ4ru2^Lk3rAQtgMRZqe;G{w_BRa}{1G4)}6d8EEHpC@MHG!jI{{)r|Ks z)#+60P!P76#Znn4=92Pxx`p6mb`3@w27z(cWB5uYV&sQ~fanGCTU`ff!6Kk6L zlS64=S#~!Cxl*srxB(qs{+9?N*=GU{(!y#2YDm!psRAp!OqW=aKfwMd(Oyfe$vsq@ z`$_-M^oi^sjuSgMr9)Gl_0Zl{QH>J17ODMt0KGkA!;>WRceEsU^|>#*`Omk028`2( z6;l|+}u^vzX;MJi4<*GUY`!dF&e5O@6p<@T{A@bXW?tHucB~Aw&}R! zH;i>+Y+eq+=(*H^{gUMLTZj2P26dMkeox-OliDl6Yd`!yAf7F==(Spwleu)qly>;;$dzEhD^ot4_B}YQ*v-^KAlV<`9Q_&#^%lq_M3Y`&Zj_zW_)8a9BdCW zAf)SZPohumyjvHb2bJL^eEIrI^8ikFB6TC}%sg3R|%Q2#&}&M-i>ijNqn(m=?@hu?H10-~Csmlza1b@~^a9IE%g_U#=a zLeMSt;-RtfIJKkdjoMvR!uPs&!SpRTfNLTt2K4y_03>W8dml ze)w7--FtuilTWzl88Ml3kIM7o8mFSmvITFwe1GB307CVJ)TcB^)bsxWD+*c+a!xGZ zr0a9CsL{hf7Qv5WacQyMQ8CP?s?@Rvg$7BN9Nr&Oq~=4ElNHUoPX=Yu zp8*MRy@BnA-4iY9__er8@1CZm)gQ_O&$I25FNfFv#2-5z70t4eFsw7*lO==@$q&7w zsn0}>de9hISXu=OLH11UFbT$tuEh6@vTVE7Vax7p+C2?}d%_lTvrSF28A&fFbL#<; zuW7CcL=xdD*BXM8WTuhu>7BT#5U*FaIiyvNBcFw2=iCHQ;#qkm{-=6!7c}l@$!F1; zr)c}K?ZTvo=8by-Ovg0m)IbSRS?Vy}B%n)X_J$#6fp@c3Qfq0QeWUG3rF$#k!2@X{ z&&EhP06zW0_a?$?mMGDyVn3{(hfSB(cP+t2?K?O6@c=(Xk$(2SCwoPO)R<(489nwp zAYB&pWGkcG+oDeYc`0SJ1Dz6D!>{OiKmlk@^Hr!iv^ddLYA!=iuWv>?(5l- z*Hk@Gn!kKQ^~IRn%FIpPt+Drvxq42yB3B6#|Nmb_pviph#(mI0Ug}BDq`gbX?=yY> z$o4-GUx?MQG6h24+3woyfz*7tD?2>7k$<;t);DG%HVmvKAz3YHewrY{1P?b0=$&;( z3~nZNeUw9R-Vs`2rGIN-KYxr=)LO)=vVis{K!}(xONtA*EZUX9@{RL&CBczI|+{qP0 zAUOE27gZ1BRDhnOYa?GR#nbw*`?z36$d7O};#;2W+aH4*T)t}O2LAOBHjVNdSK}>8 znQ_CQ`^ABGclFnr0u=X{BaYU*wYBM ziBYiyF;E0|uQt^m?!gDM)6hX*41H=KzAsi)^!0XJ-Qy{ucrs(<{;@}d$T!bBS0=tf zhemZ8iSBV@p;t)L7`|j9269N}?#+@Xf6$CHZS3|V(RwMh-q)G!!may8W~KlbZMfhH zg8wS)T~xJL<&;33Wah-WJkS~tM9+QO!@st_qW(Vr(%ygb!RQATR3(yW>J!d)zDs5* z^}aj=_m)#vLSj`Z79~bNyDZR7QyWMzkuB-F1FglRJlDtE#irGa!}V*;^e$a`xZEPzs}xM{YxkQ#J^EVd_B_v$ zrR1+}p@NG&%0m|Y#r!b60L}A)UbUcNMur^h_ys>?0uBipd@1%9F@)}r7Yl`*6v!cE z)Oa=Gm#R6}ph10}_inIEgY1k&BYR%j-aKYV`^ONA0rK&{qfDn_ehJk5vlp0WXmZo3 zm%CY&!Ppm3c8xp2RZgGnU>6j@DsOkoJTXMYdeHyUf|yVw=2lC7b`@<$|Bh%_+F|-> zXfpJp6{b&LQPME@5ailKmXzrhuwf$tk-R)Y{@(d5`uG0uqUku^H-0!0mG6#I8!A2Q zlEOZVM0q=&=*XZY0MbU{opt{@)^auOG6#T+fv-o*=n?ufydI=Gmsy`5Qr@qzXph!L zMX=DlkrOE@ho9ygB?IN@6xTevMTGDKTY5+L7$2NEMR-JOf;)~pvIut2Pmx@8EcsKH%NM^X4w=aG5nd?A@fe znEs+eE`6w>S@8gEpj)HRfGKg4Zv^NEfn0j>xbpSd#e33)Jg>_xjNBHAfl0TEm$i*1 zhfn@4Bzku!CO}*==^6#ADXM`}2a}R@^Pwa2(Qj_#B$eEC;XcWm>VS8cTKcmr_$E6g zWknTk!qDYT1mjau%g>^!z77%I=sp-G6GkM zmcG4@fmnAK7)isB?hOLo{uT>9XVY)#$=YS4;xE_NfzH#yC+3#JqF1Ct#sg_BDqE zjbGErscif-)H(}J9iW#)`e(1h5QANd%^z-y+`U-vtN#m9*w3FKKjfp+U?7j5B|mBx%Gq%Ep$AQcs#C^OVm`Y{MByyqmt4h9SRg|7mL#d&$G>k(Tf~{9=_MG z)BrFkzGfVFNT)jqMC^O5K1?mD=^ie)QdQZPeUBIZO)J>9P@5`z>)f0R_h`<;^nExA zera0L-&8o@gb=Si=tnib_E+*Bs+C?&5otW~o~zK=t5!Ki5*C!#C61$fQf<;pScSv< z1W^22FNKmhVcydrXp6ZceE(aTK+xa#i#?5pB!-X>GE+)cIn*W?U&W$JJmGlj#ZM8l z#q&hKNgZHDQ{0 z*0_G(xKCG1MFnCdF{$yCOP~*c_8n)*H-*U?4RLb!sl(sysUlr{}7p>Y##S$Lj-x9A`97Q#4q3lLM5dtqT|9Y;G z9AO8A|jl2D?U-F)gqF#!**8j~<`5MDpdLb$4w+K=^^^R|+^902fe8WPgbE*QfT` zkS5_B6OT^uaT(nODsqgvgpx9vm^7TdKy?o55^YpZr@{7JTJm-IGgQUTDNKsBSd~Ku z0CwAw3CIDGw6IQUbzLVaYr(|Vi_Z0i&xo#~TqB*;l1QUiIQ&N*JKnDh65Lte6T0Yh zO`uKclpv*BK(VAx;KA@2b$uH3e`pojt;L%XOlD$G3T9z9UE+mRC?_*T@m7+sm z3AU-Clcyfj3^DH~57P%K7+$@ewuc1F!F*yE)qnia03Q+j6CwbZf5*twR?OJWWQ{Ca z&oAShalKXCDA&cuq!wm8xaP*T@_=@FP2>9Ahlr=(+`~%u%QXa9rNQx@x{=`(aas*S z)r*rqh>Ck;&4biDOgoen|c<3xe9u7bFL$n zK?WI%wv=aiEIKFK{(}42W$~#wWXG+>#FC-b0OmTYI&#i7&)(pccl2^Oe5d52mY^I4$&FBJbZ)o?CQLyMyGEUW~P0J{z zt^<>>H5_#jBB6f*H_l}X{sHTi&7$h@eMbPtzGXhY_!*HZ-k{Wq>7-1fX{5k6DR>Jq zx=)yKma(oA9wu!J&<7Uz1!*u}Z3=)JH*_V9t=kC%dLF9(x_uw<^_j2M)Cv3xYsQbU zVR_;6^-^n4Wdy@wYa2^48dz%NfkO{k4UaF=q0x{1QEl#YYg_b8SN)Jyi zsQntJiVmPRkRBFI2f!7JXtLe8Sz&&zaQv05oBR>fEkb67OIZR)(J?sf2_GG0)7Rw5 zsj7xrU*^M`q~rXvZqVbYcF!Oi7C&hf9+co=}wmq{Pj-E zxSVC%spmUBpTi)73sl!Ch{j_M?y*%CGvrfTFP5#VT6pxha{y^GnyOP8y*rzNlY|{; z3u@N{6PK99r@(lB5JgRx$x$T^sNGQ5@M^jX8b1bQiy|12s5yjBKH2G4S^Q_T`?y(WO6Q4WuhPDrXw)3uwf-78CcJ>^#DL+?;_- z7IV5j=fR&ju@>zq&~C5l{pk1d&S5d=F*2lq`uZ%*LC##ygWi)yojV6=68kx^ZI7~{ z8e9LoeTI_oj#s{U@a&g!%9O3sjd^PXlw@MaA60nmH5>VbN`i4|16V*3+}m2=ZmR`v z7*=dl4*dt+&tD*yd4=)1ABhX!giws4?BDxgE#5HgzF%b_dyJ*bd-K7=hcy?~{Nwd3 zPYQ^=Qqt0K0dcyJ$)j^^&`4#Wx+;8tBizlWfrB1oEOl7i=EF1%h=w^JcFj%}JEH&L zcR>5}9sAdn)E?HuEh)2@k!r@A9iQkMH$)TRc!w@~yz^e|%MnaR*U~lg1EcWb=Oav9 zfH?cBjXC!PsHz)7c%vQ*6rVDV7m`kbWzl0u30>CBS}Xr@JNw6`Yj@Y7X&292_W!pR zbS9N+bKtxJ$E9+>KA&8f38aEWfx3(ipEe&wp{$tQEG;O>Z3N(Lr_6z3Gk;lf<^7;? zk=ic?ar#Hy)U{xaPtipT6jc^|l>wk~#{P>G-|r#?_jg9y9Ap_~!s+ldY?>pG=NX06 zmlx(r9nUQyW#FIRp9{at*RSVN0?0cH23&9~K{~*uVzA;Ub>Y?own@HyiO+NfSesTW z=S}MKhK@r-0gO+4^!+!cmQI>P1OockM*pq{0Kx;2y!8f=(RL5^!Xj?~5Lj4v_@n7l#CXMzDhBl2OZt>suVc4J zaF|(&v@6Sr$p?NnL?W8yU+ZSIxGM~j>Q8cupf>F?Bg0YNR9sSvBCd}{QK)Yn4R-BP z_WrWdA1t;ZpV$H6p~tc9W;LGK-McX@!4a-qTz1x;PDZH>yBoP}J6dHqZsdlEfakm_ z!G8^I?XE-DQf+STESYOx>L7dFST)w9fBq1#gbtAmcwx?CcU1hDG>uypQl=#GW>XwU z@t*Y0nx^}_cT+dC*;!M>^ktrigF@V;($OtLz%E1Ft?qkMA3j|}PxR}J5r0Jd@D^j% zJ=E@zLvlM0)BfL^k)jLp+T%IVqMB~Fu!?p4hs2)*b-4)II6f~tU1UP%^T;srfqhNX z#dPJtd>qogIhaYr7`wHzI@WyY$RRcF-AR?oHn6VPz*Q;e9>WMO7z?)4$P zzn(mDy|n!P+08IfT}^k{DXVZz!-{K~+X8g+`qHCY3)9)nqyeKL^Y6MoWVOCqf!V{Z z55gaaFDD|*=kji%!!4!Qd+j?IXf&9Q!{?%2`bB4fF@NU7!TUFr&_1-ZV$S!LV%SXj z7^MZ?_pP+Mw!5>7_S`(S*4)IBi~crlWr<&;FGB^iNG7CFYqVDtb<99G@oE2o;^L#K zpn+SPw0D)Ma}e>RZ^I4A4R76~p}283z50DJi;#(4!yu7smm#Y>1H>)ZAD(U)S^ZyU zR79R}`qcnzGpL=8MF>JR<}FZgs71@X%l$nTZ3P{Ej~7TIvN*c z^@lu;xc5e3Ii}wC*duy=>Fm;zvFK3rBzdsSOL<9G_Ig6Y61@mado5-1$d~73Gh!&_ zU))N=LfYl#tD$lyGBg*6SOD!V_2!wSmfO27WFMgDCP|eVqnyBmeH6eZE_@-9DbMex z`IHr^`^|1Ix`>5vwzb5wyU~Y0&hb-;g$i9am>+&wwm3eAso#8(Z;nm(*Zcv2d=%~E z7tYvbDZF&3yD-JEVCGFzsG-wh6x~C!xHg05Z&Tq<>rNL>gXB4;^}_v)w-gXnX}kw? zz+!!ujXt|6A(mC5xGT#bItQ_nPX(VC(`%IVJ0K2R1=lz0O1Muiz8z&|L3kEBQ0@gu zW+X7CMYGoax}x~wJE7i&A&BQQOQ~3G3CPQ!8X5XRrq5jzf8N!R6hDqxOFEZ*AOgY) ztF8X63Eq=K6k$m1BuGfshAZKZA$nyb5~p(6{p6>#DZXwivqqQUr>5Ia#~b&{ z|15XIJ8}iuTcp?@+RRETY~fA&V~4GoK0FN(?buggol-_d33^Fd@*U|_CWLqi3#ZYe zJ)TQ=AC9b8Sosk3RDMxkf8dKWpGRG$OR=3!UwJn0%p~K!2Sph?ajcWoT=1Ld7P)Dx z`WW)p@v+m0?o>!KTJC&~DnE%VjYNg`JbA0FH{DBEg@M3M_<~0BG@mx1MCVuEcPk&w zDG}HyMumu;>+d$2x?7eaxy}n?W3|x z&|+=s@I$j|Smf6O24H{xLG!a4ufd}x{py6(I0up994wHqEMaAdbP&br{!Sq4r51_v zu$T>2qqr&kJ(xa|LN4mlojDPk*3Lvkeg?Mn^#4X9dn7Z^8{76mo-hlbFTk3fo?`2_WukHl=dYaMo8Wq_-1OeX#zNXo6jl5p`ApYlomSjyg zyBR<{TRJoC?{OWBJqM{+n7Z_74{TnlZB*JuF zW1NHSkLv}b=-Xm*s(rfEJ^rn6#~sQTjpmtA$r42MefdK3QS*p6&f~5!9d&q{;;yd9 zs=1>6)Ek>mO+T&KQ0wpSw}fSpLihsDkIXw3#SXVj~)~@ghY4K zwNcsOMFzC+#ep`<$_l-1eu8BL{sl!q;MH9j@z!K0s#iX@LRf?eTjIJmh=VOxBT78G zwJ_=iU&u-w`f=@vuBOmVo511qi%)s`FM*bd6~1ASaHthG(K+z2RAh+$RI^7(Ci~+U zFIG-B<0vmsj@5-Xnl%N_XnyI4t)nKoxe~sfK9=EXmT414WjA zMyG$e&CwKyk~Ve%)DX<@LHZE0KdA=={q_P|ybGkQzRF z;jSU10;JtlkkbHjMM(5@sFc5YXrAH1T^rrI%}EYT91-yPveRZ!`vMHtojc^ywfmie zb9qVbfT*Y4M94hx8{AY@dB0!mzc0d{r{p??h>B~z;sfrabPRjrk8MURrM18yAOEXA zS$PLgHtX8Z59nHtmN{@EC~qYG*rf8h#o5j5vK~y&jelf%5yT;7nMEV*tjQN|1)Z56 zUZW2X@6oZ=Xg>V(@D@7}wut0wrxh+1&I-hkv^l94o5Let224o4WIMJ9Np;?r7gS=)&R3$^t8(0# z5PdK=-;M6b@V_FSs8J3wey441*FuzWFT1*XeLJbwKK64~DX=as6`0{(84#uTTJ9Vx#mEM%&+*u1fmFt(njd!E2E~0L$f0E?#ol{B2S7T%$1_o*5Y!5f8)X3n5X#u&#OF8XiG4Z}Jg zD@8rLM^FCQCLSxBzdBfSDn9sE3~9h;03|zzGIkKhXB1}D!Z)qcGi4(=DOEIl$3N%l zmd^TbE=qV&A`yk~W~EG7|277f+9;#bltf%$2Z!6K`I+RWH)Iq%9t>Dy#ZQA2QaC;s zkib9v2p2lZJKwlBAprsrrl_7Ja7vFb6Rt^VJ8YpUaW z^ELKK6~0i3FMY}FnXc|D2uoVNZe?)!Uj=;VS=bN9{B=GAl{w{L6A~yuPY;m$&xW^>o?sAD&XJ3ebWf|?_ z^_v-`Rw-Wk=>)pT1qD0`VnuwReuBXFS?fgXwbo-JZ{e4VBUx%q*1?U}n2_=dksR$aLctN&o7(ELhF-!Vmj} zFwjAz%su@hXRGhpgc#Tv*w66-br)R+vLTf);ROuuuQE$a+NRp9aMG2BwfV$~FfQ zmzKZS|LEeE^6Qc3dr)b4+MX3&%Jn--VXOu6ATbf~v$FRdTLu8J#E@c<5bZz>WW&+Y z(((XF7Iqb4xN63IdADF;k$!HfGumx&DP1Y3E@w>4#PN%e&ecnfiW?VwVJ+4D@cJ7b z^E9k;hkf&`kH~MGK6=aWMrQ3}`aw6(>PRh4dixq$&>HjghN)o<=1&oekGDw60^)C` zm2oyT!G2)r`}BPJuhBz5U^t~HaM`%sD6Cxz&1bj9RK%|46w8!IyX^D3)Ys2CzSa@d zCzq0{nMLcc$aj)9PZTnc4r)_PZaw?rHLarSyE*ObaM{;^d}aXoUfGIiYDy~DjKTXW zhA$Hv06IWwDOM`1PTsY2^|h1kPe;GRX+7Vn4q3m5=P4eRpB&8j-avdndna;G>-eKt zm4uNb>LeqlL#l~98~w>P`*>&pkDiBYiEP>~-S2pmc$$66^NAyIZqf-|#+|Jq3FI%h z#(J}xFQ`Zupd(3%i>;n(tD)q`pBinfSjbff&ZP5;=;q}|Y(QTBI310!zu1#zN`CiC zGU+Fw(t{+)PwDK;esy0I^o8CtmGf-~IcU$5Ip3%L$c^DefZvn1fymHAji^x#T!udR z+derZFW5|)hO!sFM;4xX_GLR};SegrJePNWWS-JhfT_M?;`<=-t> z&b<$;R=JX`tF3;&78c|7NnuZ(;fIH#UVx?)e19vXW?eS7j=d?(a_QJ4-)?$&#Ky?- z#LiGlzIwz-a@d%-n8FIP?~Skdv6shXJUcTJf3)lVHI@v~%m6IGhM8|~{J%yyQP2e0 zQEq%?kSuUBAb_kx1lE!Xp8s)xh1vfMg&}YU{Ped&2m$yxJD*Xo#&n@Uw8D=sEFq;N z;xy95(fUYtF%n&H(X7<{Fa-7+i*&!F=J4aSs<}eHcpyM)H__5@JJ>%M?dU~onxinES|d(mnvTOQ^RGtRcSBQlWeA%0AVqZ`9uyRk}J zjwp^?kpp=hhs7Z$(d~6(MdG*chQW|d z^|NwMs)N+{xbIx)merS0ZGAOy7Bo^)y6)m40$q5kT9n=ccI;^;;LTD-^TsFS_*m60 zI_b?MNu2H8Zk)=r>`yogP_u-VSJjyih4|+L$zm$fO`{h|E=d*X=+M;cj~1vy)`Sii z_9t1-gi@Yf8r8b3Xo}p%rXq%Kp25~JL{5{k?VL?Ce^lqH$6DptR#Cf3m& zYlJf7>929HExQgHEeTg%!&NSo1{tLF>6VD&2W+8;0i;Q;mHNHDH`0JK$tS>D_r3h% zUIHNYbCf_Lkn@^3t>Uoj7po{Wajs~Lar!U(`-*#0Qc>z#ui}=-F z9aI5ZTxVXVYG3W5=iC-W%!wK*w3`n(cnq#5tpZZ7fSYrOIJ~)x(458{O1q* zINd7f{Do5Br~O2%_P^mjV!zf&?97y~^mLU!097$KV2k^Ec}<+{_H~oN5mN9jL?rxv zqbs1@l!Ep8)7A4w9H>BSt!&iT%z2+bv3C82JF~klbi_UwpsU-2R4}h3tiU>lVCH1u2COkG_|Q|QG043%(~Gl$T|-;q$6BTRHD=V zJ@GsKIbK~$&YZp15bTv=mb^hj((V09F~zYKI%k|K#Ts&UZ9>UtWtq!?JN-(5sEF2u z#!Mn%c2>T%h}A9OFk=c6!W8*t>Lb|@dt;`&#@SUAQ+T-kn(8^`)OkPA1gXgkol}C4 zQ0-wZI0b8J-w|OTm$_7+?bR*v&Yh`!st~RUrd1nhx+!obZtcbDe0r!psT|hZ-a*K` z0c+x5*ho_Cjc-ZOQQpt7*9YH5Xj0wYuh0Vmh3-n|O6crlci_g9XnLqOrWQ>b>Ko42 zW%ovniE)>5-5uwAlE5ue1ODRsp38&95z$vvdYCj7<$SbRzmb~WltMe9o~7)uEh~&*U1ZTZR3CDR zPbFw>xOQqe>>gaCxqY+nY*-t-GR;8SYj;Nb_WwxP*Q@w=8_Ht`&3cUyZhq!G zHWN*M>ntk>b#5*mCV8x{3T&SDf3Zo`OoX6pJJYEqBwto-s5~qvSKdnA>;G}}bmNJm z>r2y?fmKTKPb`B@2)=qKVcY4L`@*;;cr~0m>j!e~**5roaS?mE40UpUVkYe=65D>D zPQGk8Zh8r2LI~mINjGgXQE91Ym>~J)2>Z*qGR^v5a;thY8z)gZ%0CxVcc-%L4xN~u z)hk`NkGzyYb3eDDJ`6XxV7I6Wb4ZiRqmcKL_PfezCl&@znsb9KAUm|b3n*`EUu_Ny z$xC{u{AAueTmPeH(->=!mLsQmkjN<9s-3ZUGiA?zp|DAWeOB{d-c*%}c1#%1iu&1# zqG}d>-Zn0aXK$+Y=lGUkV*~VIdO4H0E?s@Unxs?2VxH#4`y}V;%clwWXdQENttNRq z4{JdEA2aDH@@Ljg{UBcZr+b*ID!3cUr~E6KllTYbWMFB~Yh2>CmNe(mvH`8lgQ;X^ z*u=qU6Bz&0k$gzPJt;9cJqZh~_QL8LWP>~1+j{q`nUEBWIOLy{sL{g&cj)Nj|+5pw!#VNG3aVWA+HPrxua z(TU+(zb_rd(0av-evYA2nP^%JZ->(gr?XvEU7DljEZ?4%$7eIHL+`h&py)otFMog2 zu}j@sSNPpmke#R59G-Hdc19#HMnux=K;yfW8juhC|*AATOAezX7rc zLwkra-ra=IPXw!s7k_Y7&e@$hw_~yQ4Yf$aZNJ6nd05FWG3^cgu7LOM^;RRO{Tkj^ zys2=)C<0>q15r)c3;8h*y7t;)uN=e^I3Z4Nnty4kV1BqgvE6`JoRaP%)I4-px?#sL zX`MV!;@l=aB%Z48M;v*5;a@S$G`M*Piu%`(L5`L{*iaUM|1({$@hVUWd={dc8M?_k zRyL>F5+=z^<(4Y-?YLTOv=MfKo58n~fYcj(?2W|Dse!m9nvoJF94s=y1eBm%c%=u(-ty(*5z;kqOlPY;?uAwn!l! zn&I62&1&-Xh6?d|cV+$0v^I0-5k zm({4g<@-d)=?Y~QO@`e=RTKA1^SoG<@>vXbp3wLl!T~2u{EhD7 zdHL^dJ7ep@l^2i;CkJAq1y4kqGmGkryjob8=gq1gns=iHEE!G?Lr&Xg=uVX%DS|DW zW)Wm>RsVH`kr~-7O)gmZKTvmDiE7|ALI-NQZjarQ$40#3gRfnct~=mp{-O;lOe=O9 zjc!XRP9AmW@nHlvc!LCCZXymtUl*zBxMtlCT(25(ywshhjWBm^-=8%hqEhrGFZy~l&6``yLlk_hAmOY3)a3c zUWF^f_y4tTrC&*BVc3c$(#f@xQ#y)y%5%y|bFD}$bTl)w#YD?3H7!gq1(6Xnb<{Ss zQ#zVdRHIQjVjH*+{%TBZBC|0ulFhLZa>E6G7@G{voKyH0%$M$`_uD;pd7tOK?{lAy zOORO)Jo#Cm_cc->mplIe{ftCU_n4mnB9jNho0jW0!_#7mLO`Jcz5UX~z1-QNwWbc= z=L>Qe<%v5GBMpp@vxyUgVdr1ZldVLElROXw$7OvyF3ux>(ey)-3pA6&x3tMlE66FC zlWVu_Ww!<;4}Kri+KFx0tdG?SZWsz10Zi4SLSv@t-E0i>S-m9j?IP{? zzdFfe71TCYR#H56r~hows5dH=6<|ypa;aD=NLAdrNuygKg)8W{?474G_Tgx0X1*XX z{|5B|FBmTKV?J>#X-t~%@`+Vi7p^%X5aPFOenS|#s`anX`f|nYI=NSD=7`i|>4HAk zqg^*Ncj4z7JACwqR9E=)lvFrV0I#gnU1P5saEjVOdD&Om)T<~U46PIRrAbwT*88C; zXK}gb;#x9(qQxk}rjt;@hOpKRMQrNq2@&oj$ zIj}D{23$@(y#R2Ns&;6EK5gDF>gVS#ej41}{8ys-jyl$hsPh0)ta}`)j#Jmpe1V%&GR`?VZP-=`Y3^%%=hud0#TwchOj87 zQ6v-Nvvj!At7@DeNP^nM5uU{wlSl(d#&S}69Ph5n<)VU;F1GS2O8O*UVux^qy|M%k zYVY0qyO2XJ9|rAo-JYr1){#4fHcaPDn~L27zhJwb0!4`jGXWSMjoKBw2XxVU!LcMG!0-B{;T5B2#sbcraQuF`RajeHvkopn+N2bB0xc9TUZ50*VueESQrs=LQyM;s6)5i3QXGQ26?Z8v0g6lT z00BXj3d(XaI^GM1%RVWI zmUn$8SKHLE_eA>B1@DHJRECATv~hI4bv*GJ;TFORqo(>u1+)ABqwgu3c9D^{oan&= zj=Z248Jw=3emU}GT5~Blb^GjggDyEcakyhRBUNy97H?L_Xx1ld+bdgg;(f4yD47&t)0d7rI{cXEI`>aR_=&n z(i!OWc9Ro93HFcNq#dOda`q_SAi3rMDS~yxa&_=@=$}na5J|mtQ$-Yqy`*qT?I+X& znr8RQ2EN>zoN0^^+f06n(KfCAH*6BS@kS?7`Hdkp1BD4Onn)*luG{4YaL zYq}@&CNnss;FiLnGDFx_0aCUAbOUr0+%bbsVk)x0K3f%z2FWI}e_C&-CaY6ab+Bz{ zC~!E=$&;w)_m+qi4aQ7pCGjXwvhNr>Qy~>>Q*?<8QDPzta8lA zWhpf!@-JVDbOS!-o)lU0h~=o2e_V{{ zj&_vv9y=i~E@e8jQtBbgX^|<689wK^y?W-3yeApC?>?PKQ9i6cK&qriB{v0zHODn& z?GZ9TSIyAGC5Bfh065ht1kNDunFLQscI_F!6( zE@n38S;vD#hQ}bB=Gs{#_MVh_|7&Uh(D5s&UvQR46S9N(DiDDzzRfWWB18`(Pg>Jk zvogJ2qgNezJ{3(qQQ}>=2krn6>t9_hACxx#=J)E9;-m6^1#UOBA-e(6*IoC_oBbC6)eAoQw!{XXWI88*)1a1^Juc@Tx!Cbc4 zW%Ukw!{}D$z=rArdrNBOQOKNeQ7t0iW6$q=>**aF!&#C$Jo@M?+^4ha9TzYUDz-l% zlSVy^cI+TUZDwvJ_py&TCr?9<(}&ZrScgo^wFqkx=cOo_S48qxlcUIo@6Qn3u=|$Y z*pTSDN6=yPuxc|&)HPkSeiUo^KUIJKGGcE8gI~{5Nrw6KRN_*_Ofz~De>3Yi__|Ie zMhJ%^xaPATOww^kZ>j18ba2SGnJzVfn057LaHy{w@2Hr{G%(hMRx=b;S>W)FqT`$h zQZFHB{i))BbzXv2Kq_7MjpGZP;**E6)v;?Qzm2Kjr=ilcfy>`q!xu1X0XLdKkO$(< zyYayPf+E9uPKEEIy8MacXGIO?j-Q-2{U}yTe9hISAgk4<;iRn_z>0P^b>2b~znC>b zBHHqOsh!-Qm*;*ob{sHHA9;~pL31nk7W^L3Cy`r!^#yx%^_V`{qly_-={dQwT0sL4p=9RXQ?lrrO z@}&eKJ%#nZ&3Hu~Re#MS_{q#KJ+8wMW|@*s(*3%brmVY(0{V$9`UeE07}(T})X-#h z>B0o+52**j&x%g(sw_8U7`1;389U-7VmAaF0vtI7J6fzo~FRsNM@nZf~b{<{|0f3keEg^`(t|ZH%Mm zPut!XIO4)#-_;lmR%cu6Z(Jp;%6E^txWGQIWAh40+fCS~o2iO7D^{tpT(+8X$1sK+ z?wx(|Qm3swRT6eo$_C6f@MYMGt`yO{gv(=H{k@SS zyufS*CQ}0AsuoU44ARYcMRz~4&!m^74ttJiNXM3n7M2%r+ed3aQ17GK{>Mj=%n)K&gXP==C4jlOV)JTMVQ@u3C(aP#AmC5s0Mg=n< zdAtG*AS2gxM;O!)F%kv@QJaBUbX8X0BDcB@A*-ot+8WfZYQtoXy_k%Pej(68#0eNA zRVAVP*hCxf9G<3YPr8pqYY@^DXa(ON#k_UPh18S6sVMn%8>^@lycM@CMnkGvtk#cE zFdqpO;2iIQV3eSo4^3SpjBPKC?7El{KS}y!Eq`kBW1T@!P{0HI&*v!XU{8<^ZPugv zRD_c0&#!(drZrw1O|nKxH$>F#X_^&|pLb8%lXVo_mu* zRzaieA_h6DCgr7w*oPc*y$h>Q@4@3p8jkhg98tn#CO!Ug2WJE=B#5;$mM~jS!0I$1 z-EDOLXNJ2dY75uvaiA(HB|%|0)^>YDX^C#ce4{VX+4ALYBn~Lz?9#0V%&H`_sroJL0q`aA&z}BBOy5)L-7Pys}l&=;UrA8Yg->p5h87Cf%m36F}ob zFTTYk?A3T9ZCT52I*`o=%I&x}2IY(NH0r|}{j$P-!M-@}L%D_q`|e8u%L}%#qJSZ4 zoMcbZl6G7-nPsJAt=GsEuBI0~h8Klrm`EJmmXF3=^7`VeRfae`yN=?1sSO)%4ED9? z$0n-^Dt8n9eidWI22!No7ev>+LDY3A1wLF3f{QV^6P6j08BG3~eEPVyMyvs#LwfHe zTPL`w%N8TV9O3^UWvMbYLVh?AYj?&m;lq}o&xa)2g2+J+*6dUBUKQQ7y~08H`(dRIqD8% z_4wk2M_jgATTX{gkantl8Fu>;z!4ZNw?wy$tN7%h<#X=j4ro{%76cfIdK$`r^M}R& zXB!JggWFiMLL03*$Yn?!AY`b$q1Ll-)0|c>;4g72dP$u-$B;`M!O*85Ug|w>#hyu$ z!GTJiC=A8>CmHe{vBw*tj^Dd@^vv}#{CyK1YQNX$IIr4Afam*^aQq@iL%vco8)_)N z{_&ArGndOZf!54e14#J=OI=#^HQBp*huDn}ZACQHSUvDDGY5;SL)hnV?&PTp%Fxk7 zR5KzkwC8F`u=c^4L;tSXdB8Clu{WAi1#|2qKuUx9sPoH}JIdr1)Ei_&C6rg*6)duO zrx#BfgRh4=k_0keEC6c-ogn!K8f5&>zg=o_a8j2y z41pZN7BI?ec6=0@ekrBD4LWPsE>h&n3SSF5ng+><#J8b9@^~kW@PigVH34{MEN?3^ zpjr|eGHWsOW`p;_;POoWBMo98dTF4$qIl}pVXo`f9xYYjTYw_VJ#kvl)pVOFN~WrY z;kMfWv_k?FPUX&l6i2U6wm73>1ZrRF5w()F5C>9t(|^G8r9B}-GJHCOfbkz@nfsCa zi5ggaR865*IWgey5!=^{%Bfg!%fIuK#w&``O3Jg{Mm>H;H|*6KJ!HMP`&X-iB2GHa ztd!_R5v2&qw>kP_Hw0Yk}4Mj_A1 z%!J;U;sj-zWDUA=W0&U*iHP5C^QP+Y@L=>|93ccte1A)aKp&u||Jdu!ZXdI3sstMLn|DS;!lJ7}L8cS~4qdv;Nzu{Dtds^N0agNiCufr9M?<+6| zez52QAHBDv_M9TOOgJqsz_h;dG)mT~>{V-J+ad!E#)Adhvo-LNB)wg5jsvl zOzX2&_^VGU@+__nG8Bh@I|veY zo16Tj&>iIozreN8lEoAA@E;KDh&qUIz#i?nAK0m)XjV_*3;Yb%`>8>t#b@4ylyo!= zY3v#wv7z4)7%u`U7*}td^`TvQ5@)QV(7WbJ+bo%YrF5USB(N9@t1%wcvJ(vOnJmj) ziPV(N8}#9(dkOVX+_(*v3^65+39_-Oc=$lE;68U)CU3xIl(fc)fG4X-IGM@zinp{> zM;|E16A)9frIwMHC~fY4Iaca%k3gHur&R5JCo;GAiAf3i-m!AKS6}=%yzR|Eh0a78 zsy!`{D%P#03Hz&je-AFZ?&QV!I{lQ?K~a?u>GUaXbQT+|%L=-n__l$&1kd_^``{cd<-^|2xZ3DAeCVA#JWkP3F+N7J|OJ7%h*ZLz&@_mX%VWU?5y zo%ebMSDM-1i=~1}SGZ7JuVv+(k3xvj)nkSijT^o4zxeAKK6|N2Cr=i=le_xz*P ziZBJUa2H0M;k*Ac@RF{Z{+pX% zr0zUJzaP;qEvSO+)R1+M{m*{5=ft$O;b%WK?@A9eW_g#A~MW0X-A9S^Bmq7V>>XNAVJ8;;UBlT49c*$%7?_6K4~-X4jfxA ztZ>*d;<65^h(?C6ND6n}TB?%PD7$b@dV#OS5Pp!Slm>@rn==JFH*@tKQ(AX_I?=k zo%={WJM>{ zPuLzjnbdU%GD7Y~uR+UtG(bj|(IH&4c^T#_5j}C+utW{}kg0u;fBa@U;Z@2jr)m7= z`^IXAh2WI9nreo4cnLi&`O%5PLVdb}m@XXV&j}i8U9xdIj-%7U^IsL`7w3?`wC{t|a zu(X+6Lq;}Ue^wtIBAD3dcb7N+Vf^eB@2%tA=fM-!qm^^GaE9Mc#zn0HoG{++=n>p~ zX@-!Yc2%UJ=^_XpNEZ_S4uWs0|LmKpWcRQW#PzEi9=5^j?&kRyz5Tvu+v4PV}7r>r9s8!_FFK7ZBvX&*bjypzbCC7Mh z7z2Ulkq)sqAL}n>?6UVSg&-JsJ-t?~Z%9R-co!^bhB7ZVF9yPnsT`B;t{+FVJ8%BV zdZ}X_E@8L;?e|zri^tTXgOwtchFjL$>HL9nevL7|oNW3W(km~fJ7td+igr(6GaWY$ zRryIdx{Xfy+8_+XpWic|9!>qa>r-Vm86bE=xd6R~mN%>{q2Rx2wPfo0v@C6q)dXt# zm}&$%u_IQnF&5mffO#v9TeaCf-$6jhiP2o z^{>bfuD} z5|i?VGW2`A^(dSj|AK#axbc%&;kGQr^=;om_CWI@9WkTy~M#G7tFkZ;=;ZXkNrsJ`SE zW_>9J>iGK5=1a||hmT>mk6`qb>7B&azdnQ0$|BzWzE)*SnYZ5N6+b$e53lF&h%k%N zDmiG4UZpiYxd1Xo&DKS6$=cNAiZD^dwY9W}QEo(nz?@I%<`**d~;jP&tX7|Mk ztIMq9vg8}zkQNhlLqf0blXuZU7Es(DngTG9;E86^n~`31Ci_1a?;q#0c4mgH2%04A zd+(`^eN?R-k=jow8L>8Ad%Xl_-1NnO1 zAW<%5|BX?~dvVw^+UFp}tjhVJamib(#@nx7fqL8GKW_fy+w5isCxlcf@LFt@`L}G| zPZ62Jo_J62QA|6>cD5VTEp{l;q_bd@ca0O+aGWI-`nl6<$X=UF96Fb#%v6Nd+~BV@ z=iCKXgc#y5Qkj4ZY`rztkNrqZ;`B69-7u{$1{9(w+X?3g>*I&#Xq|F%JEd@P{WNS| z0PibN_}EyOAqxfI{WJac)!$x<$9XaGC-Lli-{a0iC|3mF)H=RJ$QN;SZ`Zm?I5NHT z%Qt}6+`e>b{lSlcY{gz6hL6nzdFd|p>AL?Uy(x))_`WYHdmqW52zFii2QPg<%JB3z z(XMCqvdd1wouUN?N~?1m^PDS}a&xk4E)0p`;xM3=DJQOv3Xd>C4e%Tnlz`{E=&NsM z1C6e4sUx(@)4cB)-RS-LHR42R8UrCB&RLB?2)KYDPQiovCZtQX`Ziab`K}2$5Pxee z%p}mI3U*!4;K<76NNj&&4!RPhscM6$T2{0^@`r_yXsB{x9ku!l?q(a~-ydKKvU>=M zs2f+tFeh11!ysw1iJtkP2e-0!c>hw61izN=MsTOp@MlJo1LFJCeZFT;ZNjJt*GqD! zvDG@*vGOKcHMTQSbii6H(#m#vRv}Dti`lED|GIi3@Z-3q|$@b9+gx zBGoh>68Havp)qydV>c@X1v5`z@IvBYeLcSC(9zaC_D-}>J;!D?Rq;!eVqsCivRT7F zTT1!e{Z2Y;-aBc^+r65Q|2+H+Y$oLjK2z$sFX^wrr?#u6q^~mOS>MwK z=eCCP-eeonlTrD|VyTb48 z9J5cdf?G;9rJdk!#cN)-qe)cFyuwalm;*BZnc33a*gyWnUyBYs!$} z5#IK;e~B0|RN?;@I`jXCRrS3H3UJ<_u@Hh1U`S>NRQ?0h!VGFDZtiA@IROOt1O&MG z`MLSTbolthF<&78uK!%btKmKT4*^GK3tKCn|6O3?QL8LQK+jj_1N5Dl7n6&dvz4ub zB@@)g#gfU^1!|7r%`-MmNeEqapFJDY7#ddo`woD^3gj^df?2_;2~TjqOme|5yDe~7 s;W|u1bJP7j!##981Vh0}IVwDWJ<7+>(>Fqi7!?3zd3Cw+Hy=a)3m|GuX8-^I literal 0 HcmV?d00001 diff --git a/source/idea/idea-cluster-manager/webapp/public/robots.txt b/source/idea/idea-cluster-manager/webapp/public/robots.txt new file mode 100644 index 00000000..e9e57dc4 --- /dev/null +++ b/source/idea/idea-cluster-manager/webapp/public/robots.txt @@ -0,0 +1,3 @@ +# https://www.robotstxt.org/robotstxt.html +User-agent: * +Disallow: diff --git a/source/idea/idea-cluster-manager/webapp/public/safari-pinned-tab.svg b/source/idea/idea-cluster-manager/webapp/public/safari-pinned-tab.svg new file mode 100644 index 00000000..a7934449 --- /dev/null +++ b/source/idea/idea-cluster-manager/webapp/public/safari-pinned-tab.svg @@ -0,0 +1,80 @@ + + + + +Created by potrace 1.14, written by Peter Selinger 2001-2017 + + + + + diff --git a/source/idea/idea-cluster-manager/webapp/src/App.scss b/source/idea/idea-cluster-manager/webapp/src/App.scss new file mode 100644 index 00000000..28c4216a --- /dev/null +++ b/source/idea/idea-cluster-manager/webapp/src/App.scss @@ -0,0 +1,194 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance + * with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES + * OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions + * and limitations under the License. + */ + +@use '@cloudscape-design/design-tokens/index.scss' as awsui; + +$idea-background-dark: #182836; + +.virtual-desktop-placeholder-image { + width: 100%; + height: 300px; + background-color: $idea-background-dark; + display: flex; + justify-content: center; + align-items: center; + color: white; + font-size: 18px; +} + +.application-placeholder-image { + width: 100px; + height: 40px; + background-color: #f1f1f1; + display: flex; + justify-content: center; + align-items: center; + font-size: 12px; +} + +.awsui-dark-mode { + .application-placeholder-image { + background-color: awsui.$color-background-container-content; + } +} + +.soca-key-value-group-section { + > .soca-key-value-group:nth-child(odd) { + border-right: 1px dashed #dddddd; + } +} + +.vdi-scheduler-cluster-timezone { + box-sizing: border-box; + padding: 10px 20px; + border: 1px dotted #dddddd; + background-color: #f1f1f1; + margin-bottom: 20px; +} + +.idea-list-view-text-filter { + width: 400px; +} + +code.idea-code-block { + font-family: awsui.$font-family-monospace; + background-color: awsui.$color-background-container-content; + color: awsui.$color-text-body-default; + display: block; +} + +.awsui-dark-mode { + .chonky-chonkyRoot { + background-color: awsui.$color-background-container-content; + } + + .chonky-dropdownList, .chonky-contextMenuList { + background-color: awsui.$color-background-container-header; + } +} + +// submit job +.hpc-submit-job { + .hpc-application-card { + padding: 4px; + border: 2px solid awsui.$color-border-control-default; + border-radius: 8px; + } + + .job-script { + background-color: #f1f1f1; + overflow: auto; + } + + .job-parameters { + background-color: #f1f1f1; + overflow: auto; + } + + .empty-tab { + display: flex; + align-items: center; + justify-items: center; + border: 2px solid awsui.$color-border-control-default; + border-radius: 8px; + padding: 10px 20px 20px 20px; + background-color: #f1f1f1; + } +} + +.awsui-dark-mode { + .hpc-submit-job { + .job-script, .job-parameters, .empty-tab { + background-color: awsui.$color-background-container-content; + } + } +} + +// form builder component +.idea-form-builder { + .form-parameters { + background-color: #f1f1f1; + overflow: auto; + } +} + +.awsui-dark-mode { + .idea-form-builder { + .form-parameters { + background-color: awsui.$color-background-container-content; + } + } +} + + +// file browser +.soca-file-browser { + .chonky-chonkyRoot { + border: none; + border-radius: 10px; + } +} + +.awsui-dark-mode { + .uppy-Dashboard-innerWrap { + background-color: awsui.$color-background-container-content; + } +} + +// log tail +.idea-log-tail-header { + padding: 10px; + font-size: small; + + > * { + margin-right: 10px; + } + + .title { + font-weight: bold; + } + + .file-name { + padding: 0 4px; + border-radius: 8px; + background: $idea-background-dark; + color: #ffffff; + border: 1px solid $idea-background-dark; + line-height: inherit; + } +} + +.idea-log-tail { + background: #111111; + + .idea-terminal { + width: 100vw; + height: calc(100vh - 42px); + } + + .idea-log-entries { + font-family: monospace, serif; + font-size: small; + padding: 10px; + + p { + padding: 0; + margin: 0; + + &.idea-log-loading { + padding: 10px; + font-size: smaller; + } + } + } +} diff --git a/source/idea/idea-cluster-manager/webapp/src/App.test.tsx b/source/idea/idea-cluster-manager/webapp/src/App.test.tsx new file mode 100644 index 00000000..b5c851d6 --- /dev/null +++ b/source/idea/idea-cluster-manager/webapp/src/App.test.tsx @@ -0,0 +1,22 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance + * with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES + * OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions + * and limitations under the License. + */ + +import React from 'react'; +import { render, screen } from '@testing-library/react'; +import App from './App'; + +test('renders learn react link', () => { + render(); + const linkElement = screen.getByText(/learn react/i); + expect(linkElement).toBeInTheDocument(); +}); diff --git a/source/idea/idea-cluster-manager/webapp/src/App.tsx b/source/idea/idea-cluster-manager/webapp/src/App.tsx new file mode 100644 index 00000000..56aeb90b --- /dev/null +++ b/source/idea/idea-cluster-manager/webapp/src/App.tsx @@ -0,0 +1,917 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance + * with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES + * OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions + * and limitations under the License. + */ + +import React, {Component} from 'react'; +import './App.scss'; +import {IdeaAuthChallenge, IdeaAuthConfirmForgotPassword, IdeaAuthenticatedRoute, IdeaAuthForgotPassword, IdeaAuthLogin} from "./pages/auth"; +import Home from "./pages/home"; +import {AppContext} from "./common"; +import Users from "./pages/user-management/users"; +import Groups from "./pages/user-management/groups"; +import Queues from "./pages/hpc/queues"; +import {ActiveJobs, CompletedJobs, AdminActiveJobs, AdminCompletedJobs} from "./pages/hpc/jobs"; +import HpcApplications from "./pages/hpc/hpc-applications"; +import HpcUpdateQueueProfile from "./pages/hpc/update-queue-profile"; +import SocaFileBrowser from "./pages/home/file-browser"; +import VirtualDesktopDashboard from "./pages/virtual-desktops/virtual-desktop-dashboard"; +import VirtualDesktopSessions from "./pages/virtual-desktops/virtual-desktop-sessions"; +import VirtualDesktopSoftwareStacks from "./pages/virtual-desktops/virtual-desktop-software-stacks"; +import MyVirtualDesktopSessions from "./pages/virtual-desktops/my-virtual-desktop-sessions"; +import VirtualDesktopSettings from "./pages/virtual-desktops/virtual-desktop-settings"; +import VirtualDesktopSessionDetail from "./pages/virtual-desktops/virtual-desktop-session-detail"; +import VirtualDesktopDebug from "./pages/virtual-desktops/virtual-desktop-debug"; +import {DashboardMain} from "./pages/dashboard"; +import UpdateHpcApplication from "./pages/hpc/update-hpc-application"; +import SubmitJob from "./pages/hpc/submit-job"; +import AccountSettings from "./pages/account/account-settings"; +import SSHAccess from "./pages/home/ssh-access"; +import ClusterSettings from "./pages/cluster-admin/cluster-settings"; +import ClusterStatus from "./pages/cluster-admin/cluster-status"; +import Projects from "./pages/cluster-admin/projects"; +import {Box, HelpPanel, SideNavigationProps, StatusIndicator} from "@cloudscape-design/components"; +import {NonCancelableCustomEvent} from "@cloudscape-design/components/internal/events"; +import {FlashbarProps} from "@cloudscape-design/components/flashbar/interfaces"; +import HpcLicenses from "./pages/hpc/hpc-licenses"; +import EmailTemplates from "./pages/cluster-admin/email-templates"; +import ReactMarkdown from "react-markdown"; +import remarkGfm from "remark-gfm"; +import rehypeRaw from "rehype-raw"; +import {applyDensity, applyMode, Density, Mode} from "@cloudscape-design/global-styles"; +import UpdateHpcLicense from "./pages/hpc/update-hpc-license"; +import HpcSchedulerSettings from "./pages/hpc/hpc-scheduler-settings"; +import VirtualDesktopPermissionProfiles from "./pages/virtual-desktops/virtual-desktop-permission-profiles"; +import VirtualDesktopPermissionProfileDetail from "./pages/virtual-desktops/virtual-desktop-permission-profile-detail"; +import MySharedVirtualDesktopSessions from "./pages/virtual-desktops/my-shared-virtual-desktop-sessions"; +import VirtualDesktopSoftwareStackDetail from "./pages/virtual-desktops/virtual-desktop-software-stack-detail"; +import {IdeaSideNavHeader, IdeaSideNavItems} from "./navigation/side-nav-items"; +import {IdeaAppNavigationProps, withRouter} from './navigation/navigation-utils'; +import {Routes, Route} from "react-router-dom"; +import IdeaLogTail from "./pages/home/log-tail"; +import Utils from './common/utils'; + + +export interface IdeaWebPortalAppProps extends IdeaAppNavigationProps { +} + +export interface IdeaWebPortalAppState { + sideNavHeader: SideNavigationProps.Header + sideNavItems: SideNavigationProps.Item[] + isLoggedIn: boolean + isInitialized: boolean + toolsOpen: boolean + tools: React.ReactNode + flashbarItems: FlashbarProps.MessageDefinition[] +} + +export interface OnToolsChangeEvent { + pageId: string + open: boolean +} + +export interface OnPageChangeEvent { + pageId: string +} + +export interface OnFlashbarChangeEvent { + items: FlashbarProps.MessageDefinition[] + append?: boolean +} + +class IdeaWebPortalApp extends Component { + + constructor(props: IdeaWebPortalAppProps) { + super(props); + this.state = { + isLoggedIn: false, + isInitialized: false, + sideNavHeader: { + text: '...', + href: '#/' + }, + sideNavItems: [], + toolsOpen: false, + tools: null, + flashbarItems: [] + } + } + + componentDidMount() { + AppContext.setOnRoute(this.onRoute) + const context = AppContext.get() + context.auth().isLoggedIn().then(loginStatus => { + + const init = () => { + this.setState({ + isInitialized: true, + isLoggedIn: loginStatus, + sideNavHeader: IdeaSideNavHeader(context), + sideNavItems: IdeaSideNavItems(context) + }) + context.setHooks(this.onLogin, this.onLogout) + } + + if (loginStatus) { + context.getClusterSettingsService().initialize().then(_ => { + init() + }) + } else { + init() + } + }).catch(error => { + console.error(error) + this.setState({ + isInitialized: true, + isLoggedIn: false + }) + }) + } + + onLogin = (): Promise => { + return new Promise((resolve, reject) => { + const context = AppContext.get() + context.getClusterSettingsService().initialize().then(_ => { + this.setState({ + isLoggedIn: true, + sideNavHeader: IdeaSideNavHeader(context), + sideNavItems: IdeaSideNavItems(context) + }, () => { + resolve(true) + }) + }).catch(error => { + console.error(error) + reject(false) + }) + }) + } + + onLogout = (): Promise => { + return new Promise((resolve, _) => { + this.setState({ + isLoggedIn: false + }, () => { + applyDensity(Density.Comfortable) + applyMode(Mode.Light) + resolve(true) + }) + }) + } + + onRoute = (path: string) => { + this.props.navigate(path) + } + + onSideNavChange = (event: NonCancelableCustomEvent) => { + const items = this.state.sideNavItems + items.forEach((item) => { + if (item.type === 'section') { + if (item.text === event.detail.item.text) { + item.defaultExpanded = event.detail.expanded + } + } + }) + this.setState({ + sideNavItems: [...items] + }) + } + + fetchContextHelp = (pageId: string) => { + let helpContent = require(`./docs/${pageId}.md`) + let footer = require(`./docs/_footer.md`) + fetch(helpContent).then(helpContentResponse => { + fetch(footer).then(footerResponse => { + helpContentResponse.text().then(content => { + footerResponse.text().then(footerContent => { + let lines = content.split('\n') + if (lines.length === 0) { + return + } + const header = lines[0] + const children = lines.splice(1).join('\n') + this.setState({ + tools: (} + children={} + footer={} + />) + }) + }) + }) + }) + }) + } + + onPageChange = (event: OnPageChangeEvent) => { + if (this.state.toolsOpen) { + this.setState({ + tools: () + }, () => { + this.fetchContextHelp(event.pageId) + }) + } + if (this.state.flashbarItems.length > 0) { + this.setState({ + flashbarItems: [] + }) + } + } + + onToolsChange = (event: OnToolsChangeEvent) => { + this.setState({ + toolsOpen: event.open, + tools: (event.open) ? () : null + }, () => { + this.fetchContextHelp(event.pageId) + }) + } + + onFlashbarChange = (event: OnFlashbarChangeEvent) => { + let items: FlashbarProps.MessageDefinition[] = [] + if (typeof event.append !== 'undefined' && event.append) { + this.state.flashbarItems.forEach(item => { + items.push(item) + }) + } + event.items.forEach((item, index) => { + item.id = Utils.getUUID() + if (item.dismissible) { + // create a closure to retain the index + const dismiss = (id: string) => { + return () => { + let updatedItems = [...this.state.flashbarItems] + updatedItems = updatedItems.filter(item => item.id !== id) + this.setState({ + flashbarItems: updatedItems + }) + } + } + item.onDismiss = dismiss(item.id!) + } + items.push(item) + }) + this.setState({ + flashbarItems: items + }) + } + + render() { + return (this.state.isInitialized && + + {/*authentication pages*/} + + + + }/> + + + + }/> + + + + }/> + + + + }/> + + {/*home*/} + + + + }/> + + + + + }/> + + {/*account settings*/} + + + + }/> + + {/*user home*/} + + + + }/> + + + + }/> + + + + }/> + + + + }/> + + + + }/> + + + + }/> + + + + }/> + + {/*hpc management pages*/} + + + + + }/> + + + + }/> + + + + }/> + + + + }/> + + + + }/> + + + + }/> + + + + }/> + + + + }/> + + + + }/> + + + + }/> + + + + }/> + + + + }/> + + + + }/> + + {/*virtual desktop*/} + + + + }/> + + + + }/> + + + + }/> + + + + }/> + + + + }/> + + + + }/> + + + + }/> + + + + }/> + + + + }/> + {/* cluster*/} + + + + }/> + + + + }/> + + + + }/> + + + + }/> + + + + }/> + + + + }/> + + ) + } +} + + +export default withRouter(IdeaWebPortalApp) diff --git a/source/idea/idea-cluster-manager/webapp/src/client/accounts-client.ts b/source/idea/idea-cluster-manager/webapp/src/client/accounts-client.ts new file mode 100644 index 00000000..ae271a1d --- /dev/null +++ b/source/idea/idea-cluster-manager/webapp/src/client/accounts-client.ts @@ -0,0 +1,220 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance + * with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES + * OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions + * and limitations under the License. + */ + +import { + CreateUserRequest, + CreateUserResult, + GetUserRequest, + GetUserResult, + ModifyUserRequest, + ModifyUserResult, + DeleteUserRequest, + DeleteUserResult, + EnableUserRequest, + EnableUserResult, + DisableUserRequest, + DisableUserResult, + ListUsersRequest, + ListUsersResult, + GlobalSignOutRequest, + GlobalSignOutResult, + CreateGroupRequest, + CreateGroupResult, + ModifyGroupRequest, + ModifyGroupResult, + DeleteGroupRequest, + DeleteGroupResult, + EnableGroupRequest, + EnableGroupResult, + DisableGroupRequest, + DisableGroupResult, + GetGroupRequest, + GetGroupResult, + ListGroupsRequest, + ListGroupsResult, + AddUserToGroupRequest, + AddUserToGroupResult, + RemoveUserFromGroupRequest, + RemoveUserFromGroupResult, + ListUsersInGroupRequest, + ListUsersInGroupResult, + AddSudoUserRequest, + AddSudoUserResult, + RemoveSudoUserRequest, + RemoveSudoUserResult, + ResetPasswordRequest, + ResetPasswordResult, GetModuleInfoRequest, GetModuleInfoResult +} from './data-model' +import IdeaBaseClient, {IdeaBaseClientProps} from "./base-client"; + +export interface AuthAdminClientProps extends IdeaBaseClientProps { +} + +class AccountsClient extends IdeaBaseClient{ + + getModuleInfo(): Promise { + return this.apiInvoker.invoke_alt( + 'App.GetModuleInfo', + {} + ) + } + + createUser(req: CreateUserRequest): Promise { + return this.apiInvoker.invoke_alt( + 'Accounts.CreateUser', + req + ) + } + + getUser(req: GetUserRequest): Promise { + return this.apiInvoker.invoke_alt( + 'Accounts.GetUser', + req + ) + } + + modifyUser(req: ModifyUserRequest): Promise { + return this.apiInvoker.invoke_alt( + 'Accounts.ModifyUser', + req + ) + } + + enableUser(req: EnableUserRequest): Promise { + return this.apiInvoker.invoke_alt( + 'Accounts.EnableUser', + req + ) + } + + disableUser(req: DisableUserRequest): Promise { + return this.apiInvoker.invoke_alt( + 'Accounts.DisableUser', + req + ) + } + + deleteUser(req: DeleteUserRequest): Promise { + return this.apiInvoker.invoke_alt( + 'Accounts.DeleteUser', + req + ) + } + + listUsers(req?: ListUsersRequest): Promise { + return this.apiInvoker.invoke_alt( + 'Accounts.ListUsers', + req + ) + } + + createGroup(req: CreateGroupRequest): Promise { + return this.apiInvoker.invoke_alt( + 'Accounts.CreateGroup', + req + ) + } + + getGroup(req: GetGroupRequest): Promise { + return this.apiInvoker.invoke_alt( + 'Accounts.GetGroup', + req + ) + } + + modifyGroup(req: ModifyGroupRequest): Promise { + return this.apiInvoker.invoke_alt( + 'Accounts.ModifyGroup', + req + ) + } + + enableGroup(req: EnableGroupRequest): Promise { + return this.apiInvoker.invoke_alt( + 'Accounts.EnableGroup', + req + ) + } + + disableGroup(req: DisableGroupRequest): Promise { + return this.apiInvoker.invoke_alt( + 'Accounts.DisableGroup', + req + ) + } + + deleteGroup(req: DeleteGroupRequest): Promise { + return this.apiInvoker.invoke_alt( + 'Accounts.DeleteGroup', + req + ) + } + + listGroups(req?: ListGroupsRequest): Promise { + return this.apiInvoker.invoke_alt( + 'Accounts.ListGroups', + req + ) + } + + listUsersInGroup(req?: ListUsersInGroupRequest): Promise { + return this.apiInvoker.invoke_alt( + 'Accounts.ListUsersInGroup', + req + ) + } + + addUserToGroup(req: AddUserToGroupRequest): Promise { + return this.apiInvoker.invoke_alt( + 'Accounts.AddUserToGroup', + req + ) + } + + removeUserFromGroup(req: RemoveUserFromGroupRequest): Promise { + return this.apiInvoker.invoke_alt( + 'Accounts.RemoveUserFromGroup', + req + ) + } + + addSudoUser(req: AddSudoUserRequest): Promise { + return this.apiInvoker.invoke_alt( + 'Accounts.AddSudoUser', + req + ) + } + + removeSudoUser(req: RemoveSudoUserRequest): Promise { + return this.apiInvoker.invoke_alt( + 'Accounts.RemoveSudoUser', + req + ) + } + + globalSignOut(req: GlobalSignOutRequest): Promise { + return this.apiInvoker.invoke_alt( + 'Accounts.GlobalSignOut', + req + ) + } + + resetPassword(req: ResetPasswordRequest): Promise { + return this.apiInvoker.invoke_alt( + 'Accounts.ResetPassword', + req + ) + } +} + +export default AccountsClient diff --git a/source/idea/idea-cluster-manager/webapp/src/client/analytics-client.ts b/source/idea/idea-cluster-manager/webapp/src/client/analytics-client.ts new file mode 100644 index 00000000..dd585eb2 --- /dev/null +++ b/source/idea/idea-cluster-manager/webapp/src/client/analytics-client.ts @@ -0,0 +1,34 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance + * with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES + * OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions + * and limitations under the License. + */ + +import { + OpenSearchQueryRequest, + OpenSearchQueryResult +} from './data-model' +import IdeaBaseClient, {IdeaBaseClientProps} from "./base-client"; + +export interface AnalyticsClientProps extends IdeaBaseClientProps { +} + +class AnalyticsClient extends IdeaBaseClient{ + + queryOpenSearch(req: OpenSearchQueryRequest): Promise { + return this.apiInvoker.invoke_alt( + 'Analytics.OpenSearchQuery', + req + ) + } + +} + +export default AnalyticsClient diff --git a/source/idea/idea-cluster-manager/webapp/src/client/auth-client.ts b/source/idea/idea-cluster-manager/webapp/src/client/auth-client.ts new file mode 100644 index 00000000..03a16f14 --- /dev/null +++ b/source/idea/idea-cluster-manager/webapp/src/client/auth-client.ts @@ -0,0 +1,184 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance + * with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES + * OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions + * and limitations under the License. + */ + +import { + InitiateAuthRequest, + InitiateAuthResult, + RespondToAuthChallengeRequest, + RespondToAuthChallengeResult, + ForgotPasswordRequest, + ForgotPasswordResult, + ChangePasswordRequest, + ChangePasswordResult, + ConfirmForgotPasswordRequest, + ConfirmForgotPasswordResult, + SignOutRequest, + SignOutResult, + GlobalSignOutRequest, + GlobalSignOutResult, + GetUserResult, + GetUserRequest, + GetUserPrivateKeyRequest, + GetUserPrivateKeyResult, + AddUserToGroupRequest, + AddUserToGroupResult, + RemoveUserFromGroupRequest, + RemoveUserFromGroupResult, + GetGroupRequest, + GetGroupResult, + GetModuleInfoRequest, + GetModuleInfoResult, + ListUsersInGroupRequest, + ListUsersInGroupResult +} from './data-model' + + +import {JwtTokenClaims} from "../common/token-utils"; +import IdeaBaseClient, {IdeaBaseClientProps} from "./base-client"; + +export interface AuthClientProps extends IdeaBaseClientProps { + baseUrl: string + apiContextPath: string + serviceWorkerRegistration?: ServiceWorkerRegistration +} + +/** + * Auth Client + */ +class AuthClient extends IdeaBaseClient{ + + getModuleInfo(): Promise { + return this.apiInvoker.invoke_alt( + 'App.GetModuleInfo', + {}, + true + ) + } + + initiateAuth(req: InitiateAuthRequest): Promise { + return this.apiInvoker.invoke_alt( + 'Auth.InitiateAuth', + req, + true + ) + } + + logout() { + return this.apiInvoker.logout() + } + + isLoggedIn(): Promise { + return this.apiInvoker.isLoggedIn() + } + + getAccessToken(): Promise { + return this.apiInvoker.getAccessToken() + } + + debug() { + return this.apiInvoker.debug() + } + + getClaims(): Promise { + return this.apiInvoker.getClaims() + } + + respondToAuthChallenge(req: RespondToAuthChallengeRequest): Promise { + return this.apiInvoker.invoke_alt( + 'Auth.RespondToAuthChallenge', + req, + true + ) + } + + forgotPassword(req: ForgotPasswordRequest): Promise { + return this.apiInvoker.invoke_alt( + 'Auth.ForgotPassword', + req, + true + ) + } + + confirmForgotPassword(req: ConfirmForgotPasswordRequest): Promise { + return this.apiInvoker.invoke_alt( + 'Auth.ConfirmForgotPassword', + req, + true + ) + } + + signOut(req: SignOutRequest): Promise { + return this.apiInvoker.invoke_alt( + 'Auth.SignOut', + req + ) + } + + globalSignOut(req: GlobalSignOutRequest): Promise { + return this.apiInvoker.invoke_alt( + 'Auth.GlobalSignOut', + req + ) + } + + changePassword(req: ChangePasswordRequest): Promise { + return this.apiInvoker.invoke_alt( + 'Auth.ChangePassword', + req + ) + } + + getUser(): Promise { + return this.apiInvoker.invoke_alt( + 'Auth.GetUser', + {} + ) + } + + getUserPrivateKey(request: GetUserPrivateKeyRequest): Promise { + return this.apiInvoker.invoke_alt( + 'Auth.GetUserPrivateKey', + request + ) + } + + getGroup(request: GetGroupRequest): Promise { + return this.apiInvoker.invoke_alt( + 'Auth.GetGroup', + request + ) + } + + addUserToGroup(request: AddUserToGroupRequest): Promise { + return this.apiInvoker.invoke_alt( + 'Auth.AddUserToGroup', + request + ) + } + + removeUserFromGroup(request: RemoveUserFromGroupRequest): Promise { + return this.apiInvoker.invoke_alt( + 'Auth.RemoveUserFromGroup', + request + ) + } + + listUsersInGroup(request: ListUsersInGroupRequest): Promise { + return this.apiInvoker.invoke_alt( + 'Auth.ListUsersInGroup', + request + ) + } +} + +export default AuthClient diff --git a/source/idea/idea-cluster-manager/webapp/src/client/base-client.ts b/source/idea/idea-cluster-manager/webapp/src/client/base-client.ts new file mode 100644 index 00000000..70d41a50 --- /dev/null +++ b/source/idea/idea-cluster-manager/webapp/src/client/base-client.ts @@ -0,0 +1,57 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance + * with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES + * OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions + * and limitations under the License. + */ + +import {IdeaAuthenticationContext} from "../common/authentication-context"; +import IdeaApiInvoker from "./idea-api-invoker"; + +export interface IdeaBaseClientProps { + name: string + baseUrl: string + apiContextPath: string + authContext?: IdeaAuthenticationContext + serviceWorkerRegistration?: ServiceWorkerRegistration +} + +class IdeaBaseClient

{ + readonly props: P + readonly apiInvoker: IdeaApiInvoker + + onLoginHook: (() => Promise) | null + onLogoutHook: (() => Promise) | null + + constructor(props: P) { + this.props = props + + this.apiInvoker = new IdeaApiInvoker({ + name: props.name, + url: this.getEndpointUrl(), + authContext: this.props.authContext, + serviceWorkerRegistration: this.props.serviceWorkerRegistration + }) + + this.onLoginHook = null + this.onLogoutHook = null + } + + getEndpointUrl(): string { + return `${this.props.baseUrl}${this.props.apiContextPath}` + } + + setHooks(onLogin: () => Promise, onLogout: () => Promise) { + this.onLoginHook = onLogin + this.onLogoutHook = onLogout + this.apiInvoker.setHooks(onLogin, onLogout) + } +} + +export default IdeaBaseClient diff --git a/source/idea/idea-cluster-manager/webapp/src/client/clients.ts b/source/idea/idea-cluster-manager/webapp/src/client/clients.ts new file mode 100644 index 00000000..3fc069a2 --- /dev/null +++ b/source/idea/idea-cluster-manager/webapp/src/client/clients.ts @@ -0,0 +1,236 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance + * with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES + * OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions + * and limitations under the License. + */ + +import AuthClient from "./auth-client"; +import AccountsClient from "./accounts-client"; +import SchedulerAdminClient from "./scheduler-admin-client"; +import SchedulerClient from "./scheduler-client"; +import FileBrowserClient from "./file-browser-client"; +import VirtualDesktopClient from "./virtual-desktop-client"; +import VirtualDesktopAdminClient from "./virtual-desktop-admin-client"; +import ClusterSettingsClient from "./cluster-settings-client"; +import AnalyticsClient from "./analytics-client"; +import ProjectsClient from "./projects-client"; +import EmailTemplatesClient from "./email-templates-client"; +import Utils from "../common/utils"; +import {Constants} from "../common/constants"; +import VirtualDesktopUtilsClient from "./virtual-desktop-utils-client"; +import VirtualDesktopDCVClient from "./virtual-desktop-dcv-client"; +import {IdeaAuthenticationContext} from "../common/authentication-context"; +import IdeaBaseClient, {IdeaBaseClientProps} from "./base-client"; + +export interface IdeaClientsProps { + appId: string + baseUrl: string + authContext?: IdeaAuthenticationContext + serviceWorkerRegistration?: ServiceWorkerRegistration +} + +class IdeaClients { + + private readonly authClient: AuthClient + private readonly authAdminClient: AccountsClient + private readonly schedulerAdminClient: SchedulerAdminClient + private readonly schedulerClient: SchedulerClient + private readonly fileBrowserClient: FileBrowserClient + private readonly virtualDesktopClient: VirtualDesktopClient + private readonly virtualDesktopAdminClient: VirtualDesktopAdminClient + private readonly virtualDesktopUtilsClient: VirtualDesktopUtilsClient + private readonly virtualDesktopDCVClient: VirtualDesktopDCVClient + private readonly clusterSettingsClient: ClusterSettingsClient + private readonly analyticsClient: AnalyticsClient + private readonly projectsClient: ProjectsClient + private readonly emailTemplatesClient: EmailTemplatesClient + + private readonly clients: IdeaBaseClient[] + + constructor(props: IdeaClientsProps) { + this.clients = [] + + this.authClient = new AuthClient({ + name: 'auth-client', + baseUrl: props.baseUrl, + authContext: props.authContext, + apiContextPath: Utils.getApiContextPath(Constants.MODULE_CLUSTER_MANAGER), + serviceWorkerRegistration: props.serviceWorkerRegistration + }) + this.clients.push(this.authClient) + + this.authAdminClient = new AccountsClient({ + name: 'accounts-client', + baseUrl: props.baseUrl, + authContext: props.authContext, + apiContextPath: Utils.getApiContextPath(Constants.MODULE_CLUSTER_MANAGER), + serviceWorkerRegistration: props.serviceWorkerRegistration + }) + this.clients.push(this.authAdminClient) + + this.schedulerAdminClient = new SchedulerAdminClient({ + name: 'scheduler-admin-client', + baseUrl: props.baseUrl, + authContext: props.authContext, + apiContextPath: Utils.getApiContextPath(Constants.MODULE_SCHEDULER), + serviceWorkerRegistration: props.serviceWorkerRegistration + }) + this.clients.push(this.schedulerAdminClient) + + this.schedulerClient = new SchedulerClient({ + name: 'scheduler-client', + baseUrl: props.baseUrl, + authContext: props.authContext, + apiContextPath: Utils.getApiContextPath(Constants.MODULE_SCHEDULER), + serviceWorkerRegistration: props.serviceWorkerRegistration + }) + this.clients.push(this.authClient) + + this.fileBrowserClient = new FileBrowserClient({ + name: 'file-browser-client', + baseUrl: props.baseUrl, + authContext: props.authContext, + apiContextPath: Utils.getApiContextPath(Constants.MODULE_CLUSTER_MANAGER), + serviceWorkerRegistration: props.serviceWorkerRegistration + }) + this.clients.push(this.fileBrowserClient) + + this.virtualDesktopAdminClient = new VirtualDesktopAdminClient({ + name: 'virtual-desktop-admin-client', + baseUrl: props.baseUrl, + authContext: props.authContext, + apiContextPath: Utils.getApiContextPath(Constants.MODULE_VIRTUAL_DESKTOP_CONTROLLER), + serviceWorkerRegistration: props.serviceWorkerRegistration + }) + this.clients.push(this.authClient) + + this.virtualDesktopClient = new VirtualDesktopClient({ + name: 'virtual-desktop-client', + baseUrl: props.baseUrl, + authContext: props.authContext, + apiContextPath: Utils.getApiContextPath(Constants.MODULE_VIRTUAL_DESKTOP_CONTROLLER), + serviceWorkerRegistration: props.serviceWorkerRegistration + }) + this.clients.push(this.virtualDesktopClient) + + this.virtualDesktopUtilsClient = new VirtualDesktopUtilsClient({ + name: 'virtual-desktop-utils-client', + baseUrl: props.baseUrl, + authContext: props.authContext, + apiContextPath: Utils.getApiContextPath(Constants.MODULE_VIRTUAL_DESKTOP_CONTROLLER), + serviceWorkerRegistration: props.serviceWorkerRegistration + }) + this.clients.push(this.virtualDesktopUtilsClient) + + this.virtualDesktopDCVClient = new VirtualDesktopDCVClient({ + name: 'virtual-desktop-dcv-client', + baseUrl: props.baseUrl, + authContext: props.authContext, + apiContextPath: Utils.getApiContextPath(Constants.MODULE_VIRTUAL_DESKTOP_CONTROLLER), + serviceWorkerRegistration: props.serviceWorkerRegistration + }) + this.clients.push(this.virtualDesktopDCVClient) + + this.clusterSettingsClient = new ClusterSettingsClient({ + name: 'cluster-settings-client', + baseUrl: props.baseUrl, + authContext: props.authContext, + apiContextPath: Utils.getApiContextPath(Constants.MODULE_CLUSTER_MANAGER), + serviceWorkerRegistration: props.serviceWorkerRegistration + }) + this.clients.push(this.clusterSettingsClient) + + this.analyticsClient = new AnalyticsClient({ + name: 'analytics-client', + baseUrl: props.baseUrl, + authContext: props.authContext, + apiContextPath: Utils.getApiContextPath(Constants.MODULE_CLUSTER_MANAGER), + serviceWorkerRegistration: props.serviceWorkerRegistration + }) + this.clients.push(this.analyticsClient) + + this.projectsClient = new ProjectsClient({ + name: 'projects-client', + baseUrl: props.baseUrl, + authContext: props.authContext, + apiContextPath: Utils.getApiContextPath(Constants.MODULE_CLUSTER_MANAGER), + serviceWorkerRegistration: props.serviceWorkerRegistration + }) + this.clients.push(this.projectsClient) + + this.emailTemplatesClient = new EmailTemplatesClient({ + name: 'email-templates-client', + baseUrl: props.baseUrl, + authContext: props.authContext, + apiContextPath: Utils.getApiContextPath(Constants.MODULE_CLUSTER_MANAGER), + serviceWorkerRegistration: props.serviceWorkerRegistration + }) + this.clients.push(this.emailTemplatesClient) + } + + getClients(): IdeaBaseClient[] { + return this.clients + } + + auth(): AuthClient { + return this.authClient + } + + accounts(): AccountsClient { + return this.authAdminClient + } + + schedulerAdmin(): SchedulerAdminClient { + return this.schedulerAdminClient + } + + scheduler(): SchedulerClient { + return this.schedulerClient + } + + fileBrowser(): FileBrowserClient { + return this.fileBrowserClient + } + + virtualDesktop(): VirtualDesktopClient { + return this.virtualDesktopClient + } + + virtualDesktopAdmin(): VirtualDesktopAdminClient { + return this.virtualDesktopAdminClient + } + + virtualDesktopUtils(): VirtualDesktopUtilsClient { + return this.virtualDesktopUtilsClient + } + + virtualDesktopDCV(): VirtualDesktopDCVClient { + return this.virtualDesktopDCVClient + } + + clusterSettings(): ClusterSettingsClient { + return this.clusterSettingsClient + } + + analytics(): AnalyticsClient { + return this.analyticsClient + } + + projects(): ProjectsClient { + return this.projectsClient + } + + emailTemplates(): EmailTemplatesClient { + return this.emailTemplatesClient + } + +} + +export default IdeaClients diff --git a/source/idea/idea-cluster-manager/webapp/src/client/cluster-settings-client.ts b/source/idea/idea-cluster-manager/webapp/src/client/cluster-settings-client.ts new file mode 100644 index 00000000..8ecd916c --- /dev/null +++ b/source/idea/idea-cluster-manager/webapp/src/client/cluster-settings-client.ts @@ -0,0 +1,69 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance + * with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES + * OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions + * and limitations under the License. + */ + +import { + ListClusterModulesRequest, + ListClusterModulesResult, + GetModuleSettingsRequest, + GetModuleSettingsResult, + ListClusterHostsRequest, + ListClusterHostsResult, + DescribeInstanceTypesRequest, + DescribeInstanceTypesResult, GetModuleInfoRequest, GetModuleInfoResult +} from './data-model' +import IdeaBaseClient, {IdeaBaseClientProps} from "./base-client"; + + +export interface ClusterSettingsClientProps extends IdeaBaseClientProps { +} + +class ClusterSettingsClient extends IdeaBaseClient { + + getModuleInfo(): Promise { + return this.apiInvoker.invoke_alt( + 'App.GetModuleInfo', + {} + ) + } + + listClusterModules(req: ListClusterModulesRequest): Promise { + return this.apiInvoker.invoke_alt( + 'ClusterSettings.ListClusterModules', + req + ) + } + + getModuleSettings(req: GetModuleSettingsRequest): Promise { + return this.apiInvoker.invoke_alt( + 'ClusterSettings.GetModuleSettings', + req + ) + } + + listClusterHosts(req: ListClusterHostsRequest): Promise { + return this.apiInvoker.invoke_alt( + 'ClusterSettings.ListClusterHosts', + req + ) + } + + describeInstanceTypes(req: DescribeInstanceTypesRequest): Promise { + return this.apiInvoker.invoke_alt( + 'ClusterSettings.DescribeInstanceTypes', + req + ) + } + +} + +export default ClusterSettingsClient diff --git a/source/idea/idea-cluster-manager/webapp/src/client/data-model.ts b/source/idea/idea-cluster-manager/webapp/src/client/data-model.ts new file mode 100644 index 00000000..2560a816 --- /dev/null +++ b/source/idea/idea-cluster-manager/webapp/src/client/data-model.ts @@ -0,0 +1,1981 @@ +/* tslint:disable */ +/* eslint-disable */ +/* This file is generated using IDEA invoke typings task. */ +/* Do not modify this file manually. */ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the License). You may not use this file except in compliance + * with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES + * OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions + * and limitations under the License. + */ + +export type SocaUserInputParamType = + | "text" + | "password" + | "new-password" + | "path" + | "confirm" + | "select" + | "raw_select" + | "checkbox" + | "autocomplete" + | "select_or_text" + | "choices" + | "auto" + | "image_upload" + | "heading1" + | "heading2" + | "heading3" + | "heading4" + | "heading5" + | "heading6" + | "paragraph" + | "code" + | "datepicker"; +export type VirtualDesktopBaseOS = "amazonlinux2" | "centos7" | "rhel7" | "windows"; +export type SocaMemoryUnit = "bytes" | "kib" | "mib" | "gib" | "tib" | "kb" | "mb" | "gb" | "tb"; +export type VirtualDesktopArchitecture = "x86_64" | "arm64"; +export type VirtualDesktopGPU = "NO_GPU" | "NVIDIA" | "AMD"; +export type SocaSortOrder = "asc" | "desc"; +export type SocaQueueMode = "fifo" | "fairshare" | "license-optimized"; +export type SocaScalingMode = "single-job" | "batch"; +export type SocaSpotAllocationStrategy = "capacity-optimized" | "lowest-price" | "diversified"; +export type SocaJobState = + | "transition" + | "queued" + | "held" + | "waiting" + | "running" + | "exit" + | "subjob_expired" + | "subjob_begun" + | "moved" + | "finished" + | "suspended"; +export type SocaCapacityType = "on-demand" | "spot" | "mixed"; +export type VirtualDesktopSessionType = "CONSOLE" | "VIRTUAL"; +export type VirtualDesktopSessionState = + | "PROVISIONING" + | "CREATING" + | "INITIALIZING" + | "READY" + | "RESUMING" + | "STOPPING" + | "STOPPED" + | "ERROR" + | "DELETING" + | "DELETED"; +export type DayOfWeek = "monday" | "tuesday" | "wednesday" | "thursday" | "friday" | "saturday" | "sunday"; +export type VirtualDesktopScheduleType = + | "WORKING_HOURS" + | "STOP_ALL_DAY" + | "START_ALL_DAY" + | "CUSTOM_SCHEDULE" + | "NO_SCHEDULE"; +export type VirtualDesktopSessionPermissionActorType = "USER" | "GROUP"; +export type DryRunOption = + | "true" + | "json:job" + | "json:bom" + | "json:budget" + | "json:quota" + | "json:queue" + | "notification:email" + | "debug"; +export type SocaComputeNodeState = + | "busy" + | "down" + | "free" + | "offline" + | "job-busy" + | "job-exclusive" + | "provisioning" + | "resv-exclusive" + | "stale" + | "stale-unknown" + | "unresolvable" + | "wait-provisioning" + | "initializing"; +export type SocaComputeNodeSharing = + | "default-excl" + | "default-exlchost" + | "default-shared" + | "force-excl" + | "force-exclhost" + | "ignore-excl"; +export type SocaJobPlacementArrangement = "free" | "pack" | "scatter" | "vscatter"; +export type SocaJobPlacementSharing = "excl" | "shared" | "exclhost" | "vscatter"; + +export interface FileList { + cwd?: string; + files?: FileData[]; +} +export interface FileData { + owner?: string; + group?: string; + file_id?: string; + name?: string; + ext?: string; + is_dir?: boolean; + is_hidden?: boolean; + is_sym_link?: boolean; + is_encrypted?: boolean; + size?: number; + mod_date?: string; + children_count?: number; + color?: string; + icon?: string; + folder_chain_icon?: string; + thumbnail_url?: string; +} +export interface RemoveSudoUserResult { + user?: User; +} +export interface User { + username?: string; + password?: string; + email?: string; + uid?: number; + gid?: number; + group_name?: string; + additional_groups?: string[]; + login_shell?: string; + home_dir?: string; + sudo?: boolean; + status?: string; + enabled?: boolean; + password_last_set?: string; + password_max_age?: number; + created_on?: string; + updated_on?: string; +} +export interface CreateFileResult {} +export interface DisableGroupResult {} +export interface DeleteHpcLicenseResourceRequest { + name?: string; +} +export interface GetModuleMetadataResult { + module?: SocaUserInputModuleMetadata; +} +export interface SocaUserInputModuleMetadata { + name?: string; + title?: string; + description?: string; + sections?: SocaUserInputSectionMetadata[]; + markdown?: string; +} +export interface SocaUserInputSectionMetadata { + name?: string; + module?: string; + title?: string; + description?: string; + required?: boolean; + review?: SocaUserInputSectionReview; + params?: SocaUserInputParamMetadata[]; + groups?: SocaUserInputGroupMetadata[]; + markdown?: string; +} +export interface SocaUserInputSectionReview { + prompt?: string; +} +export interface SocaUserInputParamMetadata { + name?: string; + template?: string; + title?: string; + prompt?: boolean; + description?: string; + description2?: string; + help_text?: string; + param_type?: SocaUserInputParamType; + data_type?: string; + custom_type?: string; + multiple?: boolean; + multiline?: boolean; + auto_enter?: boolean; + auto_focus?: boolean; + unique?: boolean; + default?: unknown; + readonly?: boolean; + validate?: SocaUserInputValidate; + choices?: SocaUserInputChoice[]; + choices_meta?: { + [k: string]: unknown; + }; + dynamic_choices?: boolean; + choices_empty_label?: string; + refreshable?: boolean; + ignore_case?: boolean; + match_middle?: boolean; + tag?: string; + export?: boolean; + when?: SocaUserInputParamCondition; + expose?: SocaUserInputParamExposeOptions; + markdown?: string; + developer_notes?: string; + custom?: { + [k: string]: unknown; + }; +} +export interface SocaUserInputValidate { + eq?: unknown; + not_eq?: unknown; + in?: string | unknown[]; + not_in?: string | unknown[]; + gt?: unknown; + gte?: unknown; + lt?: unknown; + lte?: unknown; + min?: unknown; + max?: unknown; + range?: SocaUserInputRange; + not_in_range?: SocaUserInputRange; + regex?: string; + not_regex?: string; + exact?: string; + starts_with?: string; + ends_with?: string; + empty?: boolean; + not_empty?: boolean; + contains?: unknown; + not_contains?: unknown; + required?: boolean; + auto_prefix?: string; +} +export interface SocaUserInputRange { + type?: string; + from?: unknown[]; + to?: unknown[]; +} +export interface SocaUserInputChoice { + title?: string; + value?: unknown; + disabled?: boolean; + checked?: boolean; + options?: SocaUserInputChoice[]; + description?: string; +} +export interface SocaUserInputParamCondition { + eq?: unknown; + not_eq?: unknown; + in?: string | unknown[]; + not_in?: string | unknown[]; + gt?: unknown; + gte?: unknown; + lt?: unknown; + lte?: unknown; + min?: unknown; + max?: unknown; + range?: SocaUserInputRange; + not_in_range?: SocaUserInputRange; + regex?: string; + not_regex?: string; + exact?: string; + starts_with?: string; + ends_with?: string; + empty?: boolean; + not_empty?: boolean; + contains?: unknown; + not_contains?: unknown; + param?: string; + and?: SocaUserInputParamCondition[]; + or?: SocaUserInputParamCondition[]; +} +export interface SocaUserInputParamExposeOptions { + cli?: SocaInputParamCliOptions; + web_app?: boolean; +} +export interface SocaInputParamCliOptions { + long_name?: string; + short_name?: string; + required?: string; + help_text?: string; +} +export interface SocaUserInputGroupMetadata { + name?: string; + module?: string; + section?: string; + title?: string; + description?: string; + params?: SocaUserInputParamMetadata[]; +} +export interface CreateGroupRequest { + group?: Group; +} +export interface Group { + title?: string; + description?: string; + name?: string; + gid?: number; + group_type?: string; + ref?: string; + enabled?: boolean; + created_on?: string; + updated_on?: string; +} +export interface SocaJobEstimatedBudgetUsage { + budget_name?: string; + budget_limit?: SocaAmount; + actual_spend?: SocaAmount; + forecasted_spend?: SocaAmount; + job_usage_percent?: number; + job_usage_percent_with_savings?: number; +} +export interface SocaAmount { + amount: number; + unit?: string; +} +export interface DeleteQueueProfileResult {} +export interface GetHpcApplicationResult { + application?: HpcApplication; +} +export interface HpcApplication { + application_id?: string; + title?: string; + description?: string; + thumbnail_url?: string; + thumbnail_data?: string; + form_template?: SocaUserInputModuleMetadata; + job_script_interpreter?: string; + job_script_type?: string; + job_script_template?: string; + projects?: Project[]; + created_on?: string; + updated_on?: string; +} +export interface Project { + project_id?: string; + name?: string; + title?: string; + description?: string; + enabled?: boolean; + ldap_groups?: string[]; + enable_budgets?: boolean; + budget?: AwsProjectBudget; + tags?: SocaKeyValue[]; + created_on?: string; + updated_on?: string; +} +export interface AwsProjectBudget { + budget_name?: string; + budget_limit?: SocaAmount; + actual_spend?: SocaAmount; + forecasted_spend?: SocaAmount; +} +export interface SocaKeyValue { + key?: string; + value?: string; +} +export interface DeleteUserRequest { + username?: string; +} +export interface GetSessionConnectionInfoRequest { + connection_info?: VirtualDesktopSessionConnectionInfo; +} +export interface VirtualDesktopSessionConnectionInfo { + dcv_session_id?: string; + idea_session_id?: string; + idea_session_owner?: string; + endpoint?: string; + username?: string; + web_url_path?: string; + access_token?: string; + failure_reason?: string; +} +export interface CreateSoftwareStackResponse { + software_stack?: VirtualDesktopSoftwareStack; +} +export interface VirtualDesktopSoftwareStack { + stack_id?: string; + base_os?: VirtualDesktopBaseOS; + name?: string; + description?: string; + created_on?: string; + updated_on?: string; + ami_id?: string; + failure_reason?: string; + enabled?: boolean; + min_storage?: SocaMemory; + min_ram?: SocaMemory; + architecture?: VirtualDesktopArchitecture; + gpu?: VirtualDesktopGPU; + projects?: Project[]; +} +export interface SocaMemory { + value: number; + unit: SocaMemoryUnit; +} +export interface ListFilesRequest { + paginator?: SocaPaginator; + sort_by?: SocaSortBy; + date_range?: SocaDateRange; + listing?: (SocaBaseModel | unknown)[]; + filters?: SocaFilter[]; + cwd?: string; +} +export interface SocaPaginator { + total?: number; + page_size?: number; + start?: number; + cursor?: string; +} +export interface SocaSortBy { + key?: string; + order?: SocaSortOrder; +} +export interface SocaDateRange { + key?: string; + start?: string; + end?: string; +} +export interface SocaBaseModel {} +export interface SocaFilter { + key?: string; + value?: unknown; + eq?: unknown; + in?: string | unknown[]; + like?: string; + starts_with?: string; + ends_with?: string; + and?: SocaFilter[]; + or?: SocaFilter[]; +} +export interface GetInstanceTypeOptionsResult { + instance_types?: SocaInstanceTypeOptions[]; +} +export interface SocaInstanceTypeOptions { + name?: string; + weighted_capacity?: number; + cpu_options_supported?: boolean; + default_core_count?: number; + default_vcpu_count?: number; + default_threads_per_core?: number; + threads_per_core?: number; + memory?: SocaMemory; + ebs_optimized?: boolean; +} +export interface ResetPasswordResult {} +export interface HpcQueueProfile { + queue_profile_id?: string; + title?: string; + description?: string; + name?: string; + projects?: Project[]; + queues?: string[]; + enabled?: boolean; + queue_mode?: SocaQueueMode; + scaling_mode?: SocaScalingMode; + terminate_when_idle?: number; + keep_forever?: boolean; + stack_uuid?: string; + queue_management_params?: SocaQueueManagementParams; + default_job_params?: SocaJobParams; + created_on?: string; + updated_on?: string; + status?: string; + limit_info?: LimitCheckResult; + queue_size?: number; +} +export interface SocaQueueManagementParams { + max_running_jobs?: number; + max_provisioned_instances?: number; + max_provisioned_capacity?: number; + wait_on_any_job_with_license?: boolean; + allowed_instance_types?: string[]; + excluded_instance_types?: string[]; + restricted_parameters?: string[]; + allowed_security_groups?: string[]; + allowed_instance_profiles?: string[]; +} +export interface SocaJobParams { + nodes?: number; + cpus?: number; + memory?: SocaMemory; + gpus?: number; + mpiprocs?: number; + walltime?: string; + base_os?: string; + instance_ami?: string; + instance_types?: string[]; + force_reserved_instances?: boolean; + spot?: boolean; + spot_price?: SocaAmount; + spot_allocation_count?: number; + spot_allocation_strategy?: SocaSpotAllocationStrategy; + subnet_ids?: string[]; + security_groups?: string[]; + instance_profile?: string; + keep_ebs_volumes?: boolean; + root_storage_size?: SocaMemory; + enable_scratch?: boolean; + scratch_provider?: string; + scratch_storage_size?: SocaMemory; + scratch_storage_iops?: number; + fsx_lustre?: SocaFSxLustreConfig; + enable_instance_store?: boolean; + enable_efa_support?: boolean; + enable_ht_support?: boolean; + enable_placement_group?: boolean; + enable_system_metrics?: boolean; + enable_anonymous_metrics?: boolean; + licenses?: SocaJobLicenseAsk[]; + compute_stack?: string; + stack_id?: string; + job_group?: string; + job_started_email_template?: string; + job_completed_email_template?: string; + custom_params?: { + [k: string]: string; + }; +} +export interface SocaFSxLustreConfig { + enabled?: boolean; + existing_fsx?: string; + s3_backend?: string; + import_path?: string; + export_path?: string; + deployment_type?: string; + per_unit_throughput?: number; + size?: SocaMemory; +} +export interface SocaJobLicenseAsk { + name?: string; + count?: number; +} +export interface LimitCheckResult { + success?: boolean; + limit_type?: string; + queue_threshold?: number; + queue_current?: number; + group_threshold?: number; + group_current?: number; +} +export interface ListPermissionProfilesRequest { + paginator?: SocaPaginator; + sort_by?: SocaSortBy; + date_range?: SocaDateRange; + listing?: (SocaBaseModel | unknown)[]; + filters?: SocaFilter[]; +} +export interface ProvisionAlwaysOnNodesResult { + stack_name?: string; + stack_id?: string; +} +export interface JobGroupMetrics { + total?: JobMetrics; + jobs?: { + [k: string]: JobMetrics; + }; +} +export interface JobMetrics { + active_jobs?: number; + desired_capacity?: number; +} +export interface RemoveUserFromGroupResult { + group?: Group; +} +export interface CreateProjectRequest { + project?: Project; +} +export interface GetSoftwareStackInfoResponse { + software_stack?: VirtualDesktopSoftwareStack; +} +export interface AuthResult { + access_token?: string; + id_token?: string; + refresh_token?: string; + expires_in?: number; + token_type?: string; +} +export interface GetBasePermissionsRequest {} +export interface DescribeSessionsResponse { + response?: { + [k: string]: unknown; + }; +} +export interface UpdateEmailTemplateResult { + template?: EmailTemplate; +} +export interface EmailTemplate { + name?: string; + title?: string; + template_type?: string; + subject?: string; + body?: string; + created_on?: string; + updated_on?: string; +} +export interface GetBasePermissionsResponse { + permissions?: VirtualDesktopPermission[]; +} +export interface VirtualDesktopPermission { + key?: string; + name?: string; + description?: string; + enabled?: boolean; +} +export interface ListJobsRequest { + paginator?: SocaPaginator; + sort_by?: SocaSortBy; + date_range?: SocaDateRange; + listing?: (SocaBaseModel | unknown)[]; + filters?: SocaFilter[]; + queue_type?: string; + queue?: string; + states?: SocaJobState[]; +} +export interface InitiateAuthRequest { + client_id?: string; + auth_flow?: string; + username?: string; + password?: string; + refresh_token?: string; + authorization_code?: string; +} +export interface ModuleInfo { + module_name?: string; + module_version?: string; + module_id?: string; +} +export interface SocaInputParamValidationEntry { + name?: string; + section?: string; + message?: string; + meta?: SocaUserInputParamMetadata; +} +export interface ListAllowedInstanceTypesRequest { + hibernation_support?: boolean; + software_stack?: VirtualDesktopSoftwareStack; +} +export interface AuthenticateUserRequest { + username?: string; + password?: string; +} +export interface ListHpcApplicationsResult { + paginator?: SocaPaginator; + sort_by?: SocaSortBy; + date_range?: SocaDateRange; + listing?: HpcApplication[]; + filters?: SocaFilter[]; +} +export interface ListProjectsResult { + paginator?: SocaPaginator; + sort_by?: SocaSortBy; + date_range?: SocaDateRange; + listing?: Project[]; + filters?: SocaFilter[]; +} +export interface ServiceQuota { + quota_name?: string; + available?: number; + consumed?: number; + desired?: number; +} +export interface GetGroupRequest { + group_name?: string; +} +export interface DeleteHpcLicenseResourceResult {} +export interface CreateQueuesRequest { + queue_profile_id?: string; + queue_profile_name?: string; + queue_names?: string; +} +export interface DeleteUserResult {} +export interface GetUserApplicationsRequest { + username?: string; + application_ids?: string[]; +} +export interface CreateGroupResult { + group?: Group; +} +export interface HpcLicenseResource { + name?: string; + title?: string; + availability_check_cmd?: string; + availability_check_status?: string; + reserved_count?: number; + available_count?: number; + created_on?: string; + updated_on?: string; +} +export interface SocaJob { + cluster_name?: string; + cluster_version?: string; + job_id?: string; + job_uid?: string; + job_group?: string; + project?: string; + name?: string; + queue?: string; + queue_type?: string; + scaling_mode?: SocaScalingMode; + owner?: string; + state?: SocaJobState; + exit_status?: string; + provisioned?: boolean; + error_message?: string; + queue_time?: string; + provisioning_time?: string; + start_time?: string; + end_time?: string; + total_time_secs?: number; + comment?: string; + debug?: boolean; + capacity_added?: boolean; + params?: SocaJobParams; + provisioning_options?: SocaJobProvisioningOptions; + estimated_budget_usage?: SocaJobEstimatedBudgetUsage; + estimated_bom_cost?: SocaJobEstimatedBOMCost; + execution_hosts?: SocaJobExecutionHost[]; + notifications?: SocaJobNotifications; +} +/** + * These are job provisioning parameters, that satisfy any of these cases: + * + * > computed dynamically based on JobParams provided by the user + * > are not defined as resources in the scheduler + * > are primarily used while provisioning capacity for the job + * > values from soca-configuration in AWS Secrets + * + * If any of these values can be potentially be submitted by the user during job submission, + * these values must be pulled up to JobParams. + */ +export interface SocaJobProvisioningOptions { + keep_forever?: boolean; + terminate_when_idle?: number; + ebs_optimized?: boolean; + spot_fleet_iam_role_arn?: string; + compute_fleet_instance_profile_arn?: string; + apps_fs_dns?: string; + apps_fs_provider?: string; + data_fs_dns?: string; + data_fs_provider?: string; + es_endpoint?: string; + stack_uuid?: string; + s3_bucket?: string; + s3_bucket_install_folder?: string; + scheduler_private_dns?: string; + scheduler_tcp_port?: number; + ssh_key_pair?: string; + auth_provider?: string; + tags?: { + [k: string]: string; + }; + anonymous_metrics_lambda_arn?: string; + instance_types?: SocaInstanceTypeOptions[]; +} +export interface SocaJobEstimatedBOMCost { + line_items?: SocaJobEstimatedBOMCostLineItem[]; + line_items_total?: SocaAmount; + savings?: SocaJobEstimatedBOMCostLineItem[]; + savings_total?: SocaAmount; + total?: SocaAmount; +} +export interface SocaJobEstimatedBOMCostLineItem { + title?: string; + service?: string; + product?: string; + quantity?: number; + unit?: string; + unit_price?: SocaAmount; + total_price?: SocaAmount; +} +export interface SocaJobExecutionHost { + host?: string; + instance_id?: string; + instance_type?: string; + capacity_type?: SocaCapacityType; + tenancy?: string; + reservation?: string; + execution?: SocaJobExecution; +} +export interface SocaJobExecution { + run_count?: number; + runs?: SocaJobExecutionRun[]; +} +export interface SocaJobExecutionRun { + run_id?: string; + start?: string; + end?: string; + exit_code?: number; + status?: string; + resources_used?: SocaJobExecutionResourcesUsed; +} +export interface SocaJobExecutionResourcesUsed { + cpu_time_secs?: number; + memory?: SocaMemory; + virtual_memory?: SocaMemory; + cpus?: number; + gpus?: number; + cpu_percent?: number; +} +export interface SocaJobNotifications { + started?: boolean; + completed?: boolean; + subjobs?: boolean; + job_started_email_template?: string; + job_completed_email_template?: string; +} +export interface ListFilesResult { + paginator?: SocaPaginator; + sort_by?: SocaSortBy; + date_range?: SocaDateRange; + listing?: FileData[]; + filters?: SocaFilter[]; + cwd?: string; +} +export interface DeleteSessionRequest { + sessions?: VirtualDesktopSession[]; +} +export interface VirtualDesktopSession { + dcv_session_id?: string; + idea_session_id?: string; + base_os?: VirtualDesktopBaseOS; + name?: string; + owner?: string; + type?: VirtualDesktopSessionType; + server?: VirtualDesktopServer; + created_on?: string; + updated_on?: string; + state?: VirtualDesktopSessionState; + description?: string; + software_stack?: VirtualDesktopSoftwareStack; + project?: Project; + schedule?: VirtualDesktopWeekSchedule; + connection_count?: number; + force?: boolean; + hibernation_enabled?: boolean; + is_launched_by_admin?: boolean; + locked?: boolean; + failure_reason?: string; +} +export interface VirtualDesktopServer { + server_id?: string; + idea_sesssion_id?: string; + idea_session_owner?: string; + instance_id?: string; + instance_type?: string; + private_ip?: string; + private_dns_name?: string; + public_ip?: string; + public_dns_name?: string; + availability?: string; + unavailability_reason?: string; + console_session_count?: number; + virtual_session_count?: number; + max_concurrent_sessions_per_user?: number; + max_virtual_sessions?: number; + state?: string; + locked?: boolean; + root_volume_size?: SocaMemory; + root_volume_iops?: number; + instance_profile_arn?: string; + security_groups?: string[]; + subnet_id?: string; + key_pair_name?: string; +} +export interface VirtualDesktopWeekSchedule { + monday?: VirtualDesktopSchedule; + tuesday?: VirtualDesktopSchedule; + wednesday?: VirtualDesktopSchedule; + thursday?: VirtualDesktopSchedule; + friday?: VirtualDesktopSchedule; + saturday?: VirtualDesktopSchedule; + sunday?: VirtualDesktopSchedule; +} +export interface VirtualDesktopSchedule { + schedule_id?: string; + idea_session_id?: string; + idea_session_owner?: string; + day_of_week?: DayOfWeek; + start_up_time?: string; + shut_down_time?: string; + schedule_type?: VirtualDesktopScheduleType; +} +export interface UpdateHpcApplicationRequest { + application?: HpcApplication; +} +export interface SubmitJobRequest { + job_owner?: string; + project?: string; + dry_run?: boolean; + job_script_interpreter?: string; + job_script?: string; +} +export interface ConfirmForgotPasswordRequest { + client_id?: string; + username?: string; + confirmation_code?: string; + password?: string; +} +export interface GetSessionConnectionInfoResponse { + connection_info?: VirtualDesktopSessionConnectionInfo; +} +export interface UpdateSoftwareStackRequest { + software_stack?: VirtualDesktopSoftwareStack; +} +export interface DeleteEmailTemplateRequest { + name?: string; +} +export interface GetModuleInfoRequest {} +export interface DecodedToken {} +export interface ListScheduleTypesRequest {} +export interface UpdateSessionPermissionRequest { + create?: VirtualDesktopSessionPermission[]; + delete?: VirtualDesktopSessionPermission[]; + update?: VirtualDesktopSessionPermission[]; +} +export interface VirtualDesktopSessionPermission { + idea_session_id?: string; + idea_session_owner?: string; + idea_session_name?: string; + idea_session_instance_type?: string; + idea_session_state?: VirtualDesktopSessionState; + idea_session_base_os?: VirtualDesktopBaseOS; + idea_session_created_on?: string; + idea_session_hibernation_enabled?: boolean; + idea_session_type?: VirtualDesktopSessionType; + permission_profile?: VirtualDesktopPermissionProfile; + actor_type?: VirtualDesktopSessionPermissionActorType; + actor_name?: string; + created_on?: string; + updated_on?: string; + expiry_date?: string; + failure_reason?: string; +} +export interface VirtualDesktopPermissionProfile { + profile_id?: string; + title?: string; + description?: string; + permissions?: VirtualDesktopPermission[]; + created_on?: string; + updated_on?: string; +} +export interface UpdateSessionResponse { + session?: VirtualDesktopSession; +} +export interface ProvisioningQueueMetrics { + total?: JobMetrics; + groups?: { + [k: string]: JobGroupMetrics; + }; +} +export interface CreateProjectResult { + project?: Project; +} +export interface SocaInputParamValidationResult { + entries?: SocaInputParamValidationEntry[]; +} +export interface SendNotificationResult {} +export interface GetParamChoicesRequest { + paginator?: SocaPaginator; + sort_by?: SocaSortBy; + date_range?: SocaDateRange; + listing?: (SocaBaseModel | unknown)[]; + filters?: SocaFilter[]; + module?: string; + param?: string; + refresh?: boolean; +} +export interface InitiateAuthResult { + challenge_name?: string; + session?: string; + challenge_params?: { + [k: string]: unknown; + }; + auth?: AuthResult; +} +export interface AuthenticateUserResult { + status?: boolean; +} +export interface ListAllowedInstanceTypesResponse { + paginator?: SocaPaginator; + sort_by?: SocaSortBy; + date_range?: SocaDateRange; + listing: unknown[]; + filters?: SocaFilter[]; +} +export interface EnableProjectRequest { + project_name?: string; + project_id?: string; +} +export interface ListPermissionProfilesResponse { + paginator?: SocaPaginator; + sort_by?: SocaSortBy; + date_range?: SocaDateRange; + listing?: VirtualDesktopPermissionProfile[]; + filters?: SocaFilter[]; +} +export interface ListHpcLicenseResourcesRequest { + paginator?: SocaPaginator; + sort_by?: SocaSortBy; + date_range?: SocaDateRange; + listing?: (SocaBaseModel | unknown)[]; + filters?: SocaFilter[]; +} +export interface GetUserApplicationsResult { + applications: HpcApplication[]; +} +export interface ListClusterModulesRequest { + paginator?: SocaPaginator; + sort_by?: SocaSortBy; + date_range?: SocaDateRange; + listing?: (SocaBaseModel | unknown)[]; + filters?: SocaFilter[]; +} +export interface CreateQueuesResult {} +export interface EnableUserRequest { + username?: string; +} +export interface GetGroupResult { + group?: Group; +} +export interface TailFileResult { + file?: string; + next_token?: string; + lines?: string[]; + line_count?: number; +} +export interface GetQueueProfileRequest { + queue_profile_name?: string; + queue_profile_id?: string; + queue_name?: string; +} +export interface DeleteEmailTemplateResult {} +export interface UpdateHpcApplicationResult { + application?: HpcApplication; +} +export interface ConfirmForgotPasswordResult {} +export interface ListSessionsResponse { + paginator?: SocaPaginator; + sort_by?: SocaSortBy; + date_range?: SocaDateRange; + listing?: VirtualDesktopSession[]; + filters?: SocaFilter[]; +} +export interface ModifyGroupRequest { + group?: Group; +} +export interface SubmitJobResult { + dry_run?: DryRunOption; + accepted?: boolean; + job?: SocaJob; + validations?: JobValidationResult; + incidentals?: JobValidationResult; + service_quotas?: ServiceQuota[]; + reserved_instances_unavailable?: boolean; + service_quota_unavailable?: boolean; + estimated_bom_cost?: SocaJobEstimatedBOMCost; + budget_usage?: SocaJobEstimatedBudgetUsage; +} +export interface JobValidationResult { + results?: JobValidationResultEntry[]; +} +export interface JobValidationResultEntry { + error_code?: string; + message?: string; +} +export interface GetSessionScreenshotRequest { + screenshots?: VirtualDesktopSessionScreenshot[]; +} +export interface VirtualDesktopSessionScreenshot { + image_type?: string; + image_data?: string; + dcv_session_id?: string; + idea_session_id?: string; + idea_session_owner?: string; + create_time?: string; + failure_reason?: string; +} +export interface UpdateSoftwareStackResponse { + software_stack?: VirtualDesktopSoftwareStack; +} +export interface GetProjectRequest { + project_name?: string; + project_id?: string; +} +export interface GetModuleInfoResult { + module?: ModuleInfo; +} +export interface ListJobsResult { + paginator?: SocaPaginator; + sort_by?: SocaSortBy; + date_range?: SocaDateRange; + listing?: SocaJob[]; + filters?: SocaFilter[]; +} +export interface UpdateSessionPermissionResponse { + permissions?: VirtualDesktopSessionPermission[]; +} +export interface SocaUserInputCondition { + eq?: unknown; + not_eq?: unknown; + in?: string | unknown[]; + not_in?: string | unknown[]; + gt?: unknown; + gte?: unknown; + lt?: unknown; + lte?: unknown; + min?: unknown; + max?: unknown; + range?: SocaUserInputRange; + not_in_range?: SocaUserInputRange; + regex?: string; + not_regex?: string; + exact?: string; + starts_with?: string; + ends_with?: string; + empty?: boolean; + not_empty?: boolean; + contains?: unknown; + not_contains?: unknown; +} +export interface RespondToAuthChallengeRequest { + client_id?: string; + session?: string; + challenge_name?: string; + challenge_params?: { + [k: string]: unknown; + }; + username?: string; + new_password?: string; +} +export interface GetUserPrivateKeyRequest { + key_format?: string; + platform?: string; +} +export interface SocaSchedulerInfo { + name?: string; + version?: string; +} +export interface ListClusterModulesResult { + paginator?: SocaPaginator; + sort_by?: SocaSortBy; + date_range?: SocaDateRange; + listing?: unknown[]; + filters?: SocaFilter[]; +} +export interface EnableProjectResult {} +export interface ListAllowedInstanceTypesForSessionRequest { + session?: VirtualDesktopSession; +} +export interface CreateHpcLicenseResourceRequest { + license_resource?: HpcLicenseResource; + dry_run?: boolean; +} +export interface SaveFileRequest { + file?: string; + content?: string; +} +export interface GetQueueProfileResult { + queue_profile?: HpcQueueProfile; +} +export interface DeleteQueuesRequest { + queue_profile_id?: string; + queue_profile_name?: string; + queue_names?: string; +} +export interface EnableUserResult { + user?: User; +} +export interface SocaQueue { + name?: string; + enabled?: boolean; + started?: boolean; + total_jobs?: number; + stats?: SocaQueueStats; +} +export interface SocaQueueStats { + transit?: number; + queued?: number; + held?: number; + waiting?: number; + running?: number; + exiting?: number; + begun?: number; +} +export interface CreateUserRequest { + user?: User; + email_verified?: boolean; +} +export interface SendNotificationRequest { + notification?: Notification; +} +export interface Notification { + username?: string; + template_name?: string; + params?: { + [k: string]: unknown; + }; + subject?: string; + body?: string; +} +export interface DeleteHpcApplicationRequest { + application_id?: string; +} +export interface GetParamChoicesResult { + paginator?: SocaPaginator; + sort_by?: SocaSortBy; + date_range?: SocaDateRange; + listing?: SocaUserInputChoice[]; + filters?: SocaFilter[]; +} +export interface SignOutRequest { + refresh_token?: string; + sso_auth?: boolean; +} +export interface JobUpdate { + queue?: string; + owner?: string; + job_id?: string; + timestamp?: string; +} +export interface DeleteJobRequest { + job_id?: string; +} +export interface GetPermissionProfileRequest { + paginator?: SocaPaginator; + sort_by?: SocaSortBy; + date_range?: SocaDateRange; + listing?: (SocaBaseModel | unknown)[]; + filters?: SocaFilter[]; + profile_id?: string; +} +export interface ModifyGroupResult { + group?: Group; +} +export interface GetProjectResult { + project?: Project; +} +export interface ListHpcLicenseResourcesResult { + paginator?: SocaPaginator; + sort_by?: SocaSortBy; + date_range?: SocaDateRange; + listing?: HpcLicenseResource[]; + filters?: SocaFilter[]; +} +export interface GetModuleSettingsRequest { + module_id?: string; +} +export interface CreateQueueProfileRequest { + queue_profile?: HpcQueueProfile; +} +export interface GetUserPrivateKeyResult { + name?: string; + key_material?: string; +} +export interface SaveFileResult {} +export interface UpdateQueueProfileRequest { + queue_profile?: HpcQueueProfile; +} +export interface DisableProjectRequest { + project_name?: string; + project_id?: string; +} +export interface SocaUserInputHandlers { + class?: string; + choices?: string; + default?: string; + validate?: string; + autocomplete?: string; + filter?: string; +} +export interface ListPermissionsRequest { + paginator?: SocaPaginator; + sort_by?: SocaSortBy; + date_range?: SocaDateRange; + listing?: (SocaBaseModel | unknown)[]; + filters?: SocaFilter[]; + idea_session_id?: string; + username?: string; +} +export interface ListScheduleTypesResponse { + paginator?: SocaPaginator; + sort_by?: SocaSortBy; + date_range?: SocaDateRange; + listing?: string[]; + filters?: SocaFilter[]; +} +export interface RespondToAuthChallengeResult { + challenge_name?: string; + session?: string; + challenge_params?: { + [k: string]: unknown; + }; + auth?: AuthResult; +} +export interface GetSessionInfoRequest { + session?: VirtualDesktopSession; +} +export interface ListEmailTemplatesRequest { + paginator?: SocaPaginator; + sort_by?: SocaSortBy; + date_range?: SocaDateRange; + listing?: (SocaBaseModel | unknown)[]; + filters?: SocaFilter[]; +} +export interface ReadFileResult { + file?: string; + content_type?: string; + content?: string; +} +export interface SetParamRequest { + module?: string; + param?: string; + value?: unknown; +} +export interface DeleteQueuesResult {} +export interface ListGroupsRequest { + paginator?: SocaPaginator; + sort_by?: SocaSortBy; + date_range?: SocaDateRange; + listing?: (SocaBaseModel | unknown)[]; + filters?: SocaFilter[]; + username?: string; +} +export interface DisableUserRequest { + username?: string; +} +export interface CreateHpcLicenseResourceResult { + license_resource?: HpcLicenseResource; +} +export interface DeleteHpcApplicationResult {} +export interface ListSoftwareStackRequest { + paginator?: SocaPaginator; + sort_by?: SocaSortBy; + date_range?: SocaDateRange; + listing?: (SocaBaseModel | unknown)[]; + filters?: SocaFilter[]; + disabled_also?: boolean; + project_id?: string; +} +export interface SignOutResult {} +export interface JobUpdates { + queued: JobUpdate[]; + modified: JobUpdate[]; + running: JobUpdate[]; +} +export interface CreateUserResult { + user?: User; +} +export interface GetModuleSettingsResult { + settings?: unknown; +} +export interface DeleteGroupRequest { + group_name?: string; +} +export interface ListAllowedInstanceTypesForSessionResponse { + paginator?: SocaPaginator; + sort_by?: SocaSortBy; + date_range?: SocaDateRange; + listing?: unknown[]; + filters?: SocaFilter[]; +} +export interface CheckHpcLicenseResourceAvailabilityRequest { + name?: string; +} +export interface ResumeSessionsResponse { + failed?: VirtualDesktopSession[]; + success?: VirtualDesktopSession[]; +} +export interface GetPermissionProfileResponse { + profile?: VirtualDesktopPermissionProfile; +} +export interface DownloadFilesRequest { + files?: string[]; +} +export interface UpdateQueueProfileResult { + queue_profile?: HpcQueueProfile; +} +export interface UpdateEmailTemplateRequest { + template?: EmailTemplate; +} +export interface UpdateProjectRequest { + project?: Project; +} +export interface ListSupportedOSRequest {} +export interface ListUsersInGroupResult { + paginator?: SocaPaginator; + sort_by?: SocaSortBy; + date_range?: SocaDateRange; + listing?: User[]; + filters?: SocaFilter[]; +} +export interface SetParamResult { + value?: unknown; + refresh?: boolean; +} +export interface DisableProjectResult {} +export interface CreateEmailTemplateRequest { + template?: EmailTemplate; +} +export interface ForgotPasswordRequest { + client_id?: string; + username?: string; +} +export interface GetSessionInfoResponse { + session?: VirtualDesktopSession; +} +export interface ListPermissionsResponse { + paginator?: SocaPaginator; + sort_by?: SocaSortBy; + date_range?: SocaDateRange; + listing?: VirtualDesktopSessionPermission[]; + filters?: SocaFilter[]; +} +export interface GetHpcLicenseResourceRequest { + name?: string; +} +export interface DisableUserResult { + user?: User; +} +export interface GlobalSignOutRequest { + username?: string; +} +export interface GetJobResult { + job?: SocaJob; +} +export interface GetUserRequest { + username?: string; +} +export interface ListHpcApplicationsRequest { + paginator?: SocaPaginator; + sort_by?: SocaSortBy; + date_range?: SocaDateRange; + listing?: (SocaBaseModel | unknown)[]; + filters?: SocaFilter[]; + lite?: boolean; +} +export interface OpenPBSInfo { + name?: string; + version?: string; + mom_private_dns?: string; + mom_port?: number; +} +export interface ListClusterHostsRequest { + paginator?: SocaPaginator; + sort_by?: SocaSortBy; + date_range?: SocaDateRange; + listing?: (SocaBaseModel | unknown)[]; + filters?: SocaFilter[]; + instance_ids?: string[]; +} +export interface CreateSessionRequest { + session?: VirtualDesktopSession; +} +export interface DownloadFilesResult { + download_url?: string; +} +export interface EnableQueueProfileRequest { + queue_profile_id?: string; + queue_profile_name?: string; +} +export interface DeleteGroupResult {} +export interface CreateQueueProfileResult { + queue_profile?: HpcQueueProfile; + validation_errors?: JobValidationResult; +} +export interface ReIndexUserSessionsRequest {} +export interface CheckHpcLicenseResourceAvailabilityResult { + available_count?: number; +} +export interface ListSessionsRequest { + paginator?: SocaPaginator; + sort_by?: SocaSortBy; + date_range?: SocaDateRange; + listing?: (SocaBaseModel | unknown)[]; + filters?: SocaFilter[]; +} +export interface OpenSearchQueryResult { + data?: { + [k: string]: unknown; + }; +} +export interface UpdatePermissionProfileRequest { + profile?: VirtualDesktopPermissionProfile; +} +export interface UpdateProjectResult { + project?: Project; +} +export interface AddSudoUserRequest { + username?: string; +} +export interface ListQueueProfilesRequest { + paginator?: SocaPaginator; + sort_by?: SocaSortBy; + date_range?: SocaDateRange; + listing?: (SocaBaseModel | unknown)[]; + filters?: SocaFilter[]; + lite?: boolean; +} +export interface ListGroupsResult { + paginator?: SocaPaginator; + sort_by?: SocaSortBy; + date_range?: SocaDateRange; + listing?: Group[]; + filters?: SocaFilter[]; +} +export interface ProvisioningCapacityInfo { + desired_capacity?: number; + group_capacity?: number; + target_capacity?: number; + existing_capacity?: number; + provisioned_capacity?: number; + idle_capacity?: number; + busy_capacity?: number; + pending_capacity?: number; + total_instances?: number; + idle_instances?: number; + busy_instances?: number; + pending_instances?: number; + max_provisioned_instances?: number; + max_provisioned_capacity?: number; + comment?: string; + error_code?: string; +} +export interface GetParamsRequest { + module?: string; + format?: string; +} +export interface UpdateSessionRequest { + session?: VirtualDesktopSession; +} +export interface GetUserProjectsRequest { + username?: string; +} +export interface CreateEmailTemplateResult { + template?: EmailTemplate; +} +export interface ForgotPasswordResult {} +export interface SocaPayload {} +export interface ListSoftwareStackResponse { + paginator?: SocaPaginator; + sort_by?: SocaSortBy; + date_range?: SocaDateRange; + listing?: VirtualDesktopSoftwareStack[]; + filters?: SocaFilter[]; +} +export interface ReadFileRequest { + file?: string; +} +export interface ListNodesRequest { + paginator?: SocaPaginator; + sort_by?: SocaSortBy; + date_range?: SocaDateRange; + listing?: (SocaBaseModel | unknown)[]; + filters?: SocaFilter[]; + instance_ids?: string[]; + instance_types?: string[]; + job_id?: string; + job_group?: string; + compute_stack?: string; + queue_type?: string; + states?: SocaComputeNodeState[]; +} +export interface ListClusterHostsResult { + paginator?: SocaPaginator; + sort_by?: SocaSortBy; + date_range?: SocaDateRange; + listing?: unknown[]; + filters?: SocaFilter[]; +} +export interface GetHpcLicenseResourceResult { + license_resource?: HpcLicenseResource; +} +export interface CreateSoftwareStackFromSessionRequest { + session?: VirtualDesktopSession; + new_software_stack?: VirtualDesktopSoftwareStack; +} +export interface CreateSessionResponse { + session?: VirtualDesktopSession; +} +export interface StopSessionRequest { + sessions?: VirtualDesktopSession[]; +} +export interface GlobalSignOutResult {} +export interface CreateHpcApplicationRequest { + application?: HpcApplication; +} +export interface SocaComputeNodeResources { + cpus?: number; + gpus?: number; + memory?: SocaMemory; +} +export interface EnableQueueProfileResult {} +export interface ListEmailTemplatesResult { + paginator?: SocaPaginator; + sort_by?: SocaSortBy; + date_range?: SocaDateRange; + listing?: EmailTemplate[]; + filters?: SocaFilter[]; +} +export interface DeleteJobResult {} +export interface GetUserResult { + user?: User; +} +export interface DeleteFilesRequest { + files?: string[]; +} +export interface SocaUserInputTag { + name?: string; + title?: string; + description?: string; + markdown?: string; + unicode?: string; + ascii?: string; + icon?: string; +} +export interface EnableGroupRequest { + group_name?: string; +} +export interface ReIndexUserSessionsResponse {} +export interface GetParamDefaultRequest { + module?: string; + param?: string; + reset?: boolean; +} +export interface ListUsersRequest { + paginator?: SocaPaginator; + sort_by?: SocaSortBy; + date_range?: SocaDateRange; + listing?: (SocaBaseModel | unknown)[]; + filters?: SocaFilter[]; +} +export interface QueuedJob { + priority?: number; + job_id: string; + job_group: string; + deleted?: boolean; + processed?: boolean; + capacity_added?: boolean; +} +export interface ListSupportedOSResponse { + paginator?: SocaPaginator; + sort_by?: SocaSortBy; + date_range?: SocaDateRange; + listing?: string[]; + filters?: SocaFilter[]; +} +export interface DeleteProjectRequest { + project_name?: string; + project_id?: string; +} +export interface SocaBatchResponsePayload { + failed?: unknown[]; + success?: unknown[]; +} +export interface AddUserToGroupRequest { + usernames?: string[]; + group_name?: string; +} +export interface UpdatePermissionProfileResponse { + profile?: VirtualDesktopPermissionProfile; +} +export interface GetParamsResult { + params?: { + [k: string]: unknown; + }; + yaml?: string; +} +export interface GetEmailTemplateRequest { + name?: string; +} +export interface ChangePasswordRequest { + username?: string; + old_password?: string; + new_password?: string; +} +export interface JobValidationDebugEntry { + title?: string; + name?: string; + description?: string; + valid?: boolean; + user_value?: unknown; + job_value?: unknown; + default_value?: unknown; +} +export interface DescribeInstanceTypesRequest {} +export interface DescribeServersRequest {} +export interface GetUserProjectsResult { + projects?: Project[]; +} +export interface VirtualDesktopSessionBatchResponsePayload { + failed?: VirtualDesktopSession[]; + success?: VirtualDesktopSession[]; +} +export interface DisableQueueProfileRequest { + queue_profile_id?: string; + queue_profile_name?: string; +} +export interface BatchCreateSessionRequest { + sessions?: VirtualDesktopSession[]; +} +export interface DeleteSessionResponse { + failed?: VirtualDesktopSession[]; + success?: VirtualDesktopSession[]; +} +export interface OpenSearchQueryRequest { + data?: { + [k: string]: unknown; + }; +} +export interface UpdateHpcLicenseResourceRequest { + license_resource?: HpcLicenseResource; + dry_run?: boolean; +} +export interface CreateSoftwareStackFromSessionResponse { + software_stack?: VirtualDesktopSoftwareStack; +} +export interface StopSessionResponse { + failed?: VirtualDesktopSession[]; + success?: VirtualDesktopSession[]; +} +export interface DeleteFilesResult {} +export interface CreateHpcApplicationResult { + application?: HpcApplication; +} +export interface ModifyUserRequest { + user?: User; + email_verified?: boolean; +} +export interface EnableGroupResult {} +export interface ReIndexSoftwareStacksRequest {} +export interface GetParamDefaultResult { + default?: unknown; +} +export interface DeleteProjectResult {} +export interface ListSupportedGPURequest {} +export interface SocaListingPayload { + paginator?: SocaPaginator; + sort_by?: SocaSortBy; + date_range?: SocaDateRange; + listing?: (SocaBaseModel | unknown)[]; + filters?: SocaFilter[]; +} +export interface RebootSessionResponse { + failed?: VirtualDesktopSession[]; + success?: VirtualDesktopSession[]; +} +export interface ListUsersInGroupRequest { + paginator?: SocaPaginator; + sort_by?: SocaSortBy; + date_range?: SocaDateRange; + listing?: (SocaBaseModel | unknown)[]; + filters?: SocaFilter[]; + group_names?: string[]; +} +export interface ListQueueProfilesResult { + paginator?: SocaPaginator; + sort_by?: SocaSortBy; + date_range?: SocaDateRange; + listing?: HpcQueueProfile[]; + filters?: SocaFilter[]; +} +export interface SocaInputParamSpec { + name?: string; + version?: string; + tags?: SocaUserInputTag[]; + modules?: SocaUserInputModuleMetadata[]; + params?: SocaUserInputParamMetadata[]; +} +export interface AddUserToGroupResult {} +export interface DescribeInstanceTypesResult { + instance_types: unknown[]; +} +export interface CreatePermissionProfileRequest { + profile?: VirtualDesktopPermissionProfile; +} +export interface ChangePasswordResult {} +export interface JobParameterInfo { + name?: string; + title?: string; + description?: string; + provider_names?: { + [k: string]: string; + }; +} +export interface SocaComputeNode { + host?: string; + cluster_name?: string; + cluster_version?: string; + states?: SocaComputeNodeState[]; + queue_type?: string; + queue?: string; + provisioning_time?: string; + last_used_time?: string; + last_state_changed_time?: string; + availability_zone?: string; + subnet_id?: string; + instance_id?: string; + instance_type?: string; + instance_ami?: string; + instance_profile?: string; + architecture?: string; + scheduler_info?: SocaSchedulerInfo; + sharing?: SocaComputeNodeSharing; + job_id?: string; + job_group?: string; + scaling_mode?: SocaScalingMode; + keep_forever?: boolean; + terminate_when_idle?: number; + compute_stack?: string; + stack_id?: string; + lifecyle?: string; + tenancy?: string; + spot_fleet_request?: string; + auto_scaling_group?: string; + spot?: boolean; + spot_price?: SocaAmount; + base_os?: string; + enable_placement_group?: boolean; + enable_ht_support?: boolean; + keep_ebs_volumes?: boolean; + root_storage_size?: SocaMemory; + scratch_storage_size?: SocaMemory; + scratch_storage_iops?: number; + enable_efa_support?: boolean; + force_reserved_instances?: boolean; + enable_system_metrics?: boolean; + enable_anonymous_metrics?: boolean; + fsx_lustre?: SocaFSxLustreConfig; + resources_available?: SocaComputeNodeResources; + resources_assigned?: SocaComputeNodeResources; + launch_time?: string; + termination_time?: string; + terminated?: boolean; + jobs?: string[]; +} +export interface DisableQueueProfileResult {} +export interface DescribeServersResponse { + response?: { + [k: string]: unknown; + }; +} +export interface VirtualDesktopApplicationProfile {} +export interface GetEmailTemplateResult { + template?: EmailTemplate; +} +export interface ListNodesResult { + paginator?: SocaPaginator; + sort_by?: SocaSortBy; + date_range?: SocaDateRange; + listing?: SocaComputeNode[]; + filters?: SocaFilter[]; +} +export interface RemoveSudoUserRequest { + username?: string; +} +export interface GetJobRequest { + job_id?: string; +} +export interface GetModuleMetadataRequest { + module?: string; +} +export interface ResumeSessionsRequest { + sessions?: VirtualDesktopSession[]; +} +export interface CreateFileRequest { + cwd?: string; + filename?: string; + is_folder?: boolean; +} +export interface GetHpcApplicationRequest { + application_id?: string; +} +export interface DisableGroupRequest { + group_name?: string; +} +export interface UpdateHpcLicenseResourceResult { + license_resource?: HpcLicenseResource; +} +export interface TailFileRequest { + file?: string; + line_count?: number; + next_token?: string; +} +export interface ReIndexSoftwareStacksResponse {} +export interface BatchCreateSessionResponse { + failed?: VirtualDesktopSession[]; + success?: VirtualDesktopSession[]; +} +export interface CreateSoftwareStackRequest { + software_stack?: VirtualDesktopSoftwareStack; +} +export interface RebootSessionRequest { + sessions?: VirtualDesktopSession[]; +} +export interface ModifyUserResult { + user?: User; +} +export interface GetInstanceTypeOptionsRequest { + enable_ht_support?: boolean; + instance_types?: string[]; + queue_name?: string; + queue_profile_name?: string; +} +export interface GetSessionScreenshotResponse { + failed?: VirtualDesktopSessionScreenshot[]; + success?: VirtualDesktopSessionScreenshot[]; +} +export interface RemoveUserFromGroupRequest { + usernames?: string[]; + group_name?: string; +} +export interface DeleteQueueProfileRequest { + queue_profile_id?: string; + queue_profile_name?: string; + delete_queues?: boolean; +} +export interface ResetPasswordRequest { + username?: string; +} +export interface ListProjectsRequest { + paginator?: SocaPaginator; + sort_by?: SocaSortBy; + date_range?: SocaDateRange; + listing?: (SocaBaseModel | unknown)[]; + filters?: SocaFilter[]; +} +export interface DescribeSessionsRequest { + sessions?: VirtualDesktopSession[]; +} +export interface CreatePermissionProfileResponse { + profile?: VirtualDesktopPermissionProfile; +} +export interface ProvisionAlwaysOnNodesRequest { + project_name?: string; + queue_profile_name?: string; + queue_name?: string; + owner?: string; + params?: SocaJobParams; +} +export interface ListUsersResult { + paginator?: SocaPaginator; + sort_by?: SocaSortBy; + date_range?: SocaDateRange; + listing?: User[]; + filters?: SocaFilter[]; +} +export interface ListSupportedGPUResponse { + paginator?: SocaPaginator; + sort_by?: SocaSortBy; + date_range?: SocaDateRange; + listing?: string[]; + filters?: SocaFilter[]; +} +export interface GetSoftwareStackInfoRequest { + stack_id?: string; +} +export interface AddSudoUserResult { + user?: User; +} +export interface SocaJobPlacement { + arrangement?: SocaJobPlacementArrangement; + sharing?: SocaJobPlacementSharing; + grouping?: string; +} diff --git a/source/idea/idea-cluster-manager/webapp/src/client/email-templates-client.ts b/source/idea/idea-cluster-manager/webapp/src/client/email-templates-client.ts new file mode 100644 index 00000000..fe1f833e --- /dev/null +++ b/source/idea/idea-cluster-manager/webapp/src/client/email-templates-client.ts @@ -0,0 +1,70 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance + * with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES + * OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions + * and limitations under the License. + */ + +import { + CreateEmailTemplateRequest, + CreateEmailTemplateResult, + GetEmailTemplateRequest, + GetEmailTemplateResult, + DeleteEmailTemplateRequest, + DeleteEmailTemplateResult, + UpdateEmailTemplateRequest, + UpdateEmailTemplateResult, + ListEmailTemplatesRequest, + ListEmailTemplatesResult +} from './data-model' +import IdeaBaseClient, {IdeaBaseClientProps} from "./base-client"; + +export interface EmailTemplatesClientProps extends IdeaBaseClientProps { +} + +class EmailTemplatesClient extends IdeaBaseClient { + + createEmailTemplate(req: CreateEmailTemplateRequest): Promise { + return this.apiInvoker.invoke_alt( + 'EmailTemplates.CreateEmailTemplate', + req + ) + } + + getEmailTemplate(req: GetEmailTemplateRequest): Promise { + return this.apiInvoker.invoke_alt( + 'EmailTemplates.GetEmailTemplate', + req + ) + } + + deleteEmailTemplate(req: DeleteEmailTemplateRequest): Promise { + return this.apiInvoker.invoke_alt( + 'EmailTemplates.DeleteEmailTemplate', + req + ) + } + + updateEmailTemplate(req: UpdateEmailTemplateRequest): Promise { + return this.apiInvoker.invoke_alt( + 'EmailTemplates.UpdateEmailTemplate', + req + ) + } + + listEmailTemplates(req: ListEmailTemplatesRequest): Promise { + return this.apiInvoker.invoke_alt( + 'EmailTemplates.ListEmailTemplates', + req + ) + } + +} + +export default EmailTemplatesClient diff --git a/source/idea/idea-cluster-manager/webapp/src/client/file-browser-client.ts b/source/idea/idea-cluster-manager/webapp/src/client/file-browser-client.ts new file mode 100644 index 00000000..a0fac231 --- /dev/null +++ b/source/idea/idea-cluster-manager/webapp/src/client/file-browser-client.ts @@ -0,0 +1,88 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance + * with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES + * OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions + * and limitations under the License. + */ + +import { + ListFilesRequest, + ListFilesResult, + ReadFileRequest, + ReadFileResult, + SaveFileRequest, + SaveFileResult, + DownloadFilesRequest, + DownloadFilesResult, + CreateFileRequest, + CreateFileResult, + DeleteFilesRequest, + DeleteFilesResult, + TailFileRequest, + TailFileResult +} from './data-model' +import IdeaBaseClient, {IdeaBaseClientProps} from "./base-client"; + +export interface FileBrowserClientProps extends IdeaBaseClientProps { +} + +class FileBrowserClient extends IdeaBaseClient { + + listFiles(req: ListFilesRequest): Promise { + return this.apiInvoker.invoke_alt( + 'FileBrowser.ListFiles', + req + ) + } + + downloadFiles(req: DownloadFilesRequest): Promise { + return this.apiInvoker.invoke_alt( + 'FileBrowser.DownloadFiles', + req + ) + } + + readFile(req: ReadFileRequest): Promise { + return this.apiInvoker.invoke_alt( + 'FileBrowser.ReadFile', + req + ) + } + + tailFile(req: TailFileRequest): Promise { + return this.apiInvoker.invoke_alt( + 'FileBrowser.TailFile', + req + ) + } + + saveFile(req: SaveFileRequest): Promise { + return this.apiInvoker.invoke_alt( + 'FileBrowser.SaveFile', + req + ) + } + + createFile(req: CreateFileRequest): Promise { + return this.apiInvoker.invoke_alt( + 'FileBrowser.CreateFile', + req + ) + } + + deleteFiles(req: DeleteFilesRequest): Promise { + return this.apiInvoker.invoke_alt( + 'FileBrowser.DeleteFiles', + req + ) + } + +} + +export default FileBrowserClient diff --git a/source/idea/idea-cluster-manager/webapp/src/client/idea-api-invoker.ts b/source/idea/idea-cluster-manager/webapp/src/client/idea-api-invoker.ts new file mode 100644 index 00000000..f171b280 --- /dev/null +++ b/source/idea/idea-cluster-manager/webapp/src/client/idea-api-invoker.ts @@ -0,0 +1,227 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance + * with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES + * OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions + * and limitations under the License. + */ + +import {v4 as uuid} from "uuid" +import IdeaException from "../common/exceptions"; +import {IdeaAuthenticationContext} from "../common/authentication-context"; +import {JwtTokenClaims} from "../common/token-utils"; +import {Constants} from "../common/constants"; +import AppLogger from "../common/app-logger"; +import {AUTH_TOKEN_EXPIRED} from "../common/error-codes"; + +export interface IdeaHeader { + namespace: string + request_id: string + version?: number +} + +export interface IdeaEnvelope { + header?: IdeaHeader; + payload?: T; + success?: boolean; + error_code?: string; + message?: string; +} + +export interface IdeaApiInvokerProps { + name: string + url: string + timeout?: number; + serviceWorkerRegistration?: ServiceWorkerRegistration + authContext?: IdeaAuthenticationContext +} + + +export class IdeaApiInvoker { + props: IdeaApiInvokerProps + logger: AppLogger + onLoginHook: (() => Promise) | null + onLogoutHook: (() => Promise) | null + + constructor(props: IdeaApiInvokerProps) { + this.props = props + this.logger = new AppLogger({ + name: props.name + }) + this.onLoginHook = null + this.onLogoutHook = null + } + + setHooks(onLogin: () => Promise, onLogout: () => Promise) { + this.onLoginHook = onLogin + this.onLogoutHook = onLogout + } + + private empty = () => { + const emptyPayload: any = {} + return emptyPayload + } + + async invoke_service_worker(message: any): Promise { + return new Promise((resolve, reject) => { + let messageChannel = new MessageChannel() + messageChannel.port1.onmessage = (event) => { + if(event.data.error) { + this.logger.error(event.data.error) + reject(event.data.error) + } + return resolve(event.data) + } + this.props.serviceWorkerRegistration!.active!.postMessage(message, [messageChannel.port2]) + }) + } + + async invoke(request: IdeaEnvelope, isPublic: boolean = false): Promise> { + + let url = `${this.props.url}/${request.header!.namespace}` + + if(this.logger.isTrace()) { + this.logger.trace(`(req) ${JSON.stringify(request, null, 2)}`) + } + + let response + if(this.props.serviceWorkerRegistration) { + const result = await this.invoke_service_worker({ + type: Constants.ServiceWorker.IDEA_API_INVOCATION, + options: { + url: url, + request: request, + isPublic: isPublic + } + }) + response = result.response + } else { + if(request.header?.namespace === 'Auth.InitiateAuth') { + response = await this.props.authContext?.initiateAuth(request) + } else { + response = await this.props.authContext?.invoke(url, request, isPublic) + } + } + + if(this.logger.isTrace()) { + this.logger.trace(`(res) ${JSON.stringify(response, null, 2)}`) + } + + if(typeof response.success !== 'undefined' && !response.success && response.error_code === AUTH_TOKEN_EXPIRED) { + this.logout().then(() => { + if(this.onLogoutHook) { + this.onLogoutHook() + } + }) + } + + return response + } + + async invoke_alt(namespace: string, payload?: REQ, isPublic: boolean = false): Promise { + const result = await this.invoke({ + header: { + namespace: namespace, + request_id: uuid() + }, + payload: (payload) ? payload : this.empty() + }, isPublic) + if (result.success) { + return result.payload! + } else { + console.error(result) + throw new IdeaException({ + errorCode: result.error_code!, + message: result.message, + payload: result.payload + }) + } + } + + + async isLoggedIn(): Promise { + if(this.props.serviceWorkerRegistration) { + const result = await this.invoke_service_worker({ + type: Constants.ServiceWorker.IDEA_AUTH_IS_LOGGED_IN + }) + return result.status + } else { + return this.props.authContext!.isLoggedIn() + } + } + + async logout(): Promise { + if(this.props.serviceWorkerRegistration) { + const result = await this.invoke_service_worker({ + type: Constants.ServiceWorker.IDEA_AUTH_LOGOUT + }) + return result.status + } else { + return this.props.authContext!.logout() + } + } + + async getAccessToken(): Promise { + if(this.props.serviceWorkerRegistration) { + const result = await this.invoke_service_worker({ + type: Constants.ServiceWorker.IDEA_AUTH_ACCESS_TOKEN + }) + return result.accessToken + } else { + return await this.props.authContext!.getAccessToken() + } + } + + debug() { + if(this.props.serviceWorkerRegistration) { + this.props.serviceWorkerRegistration!.active!.postMessage({ + type: Constants.ServiceWorker.IDEA_AUTH_DEBUG + }) + } else { + this.props.authContext!.printDebugInfo() + } + } + + async getClaims(): Promise { + if(this.props.serviceWorkerRegistration) { + const result = await this.invoke_service_worker({ + type: Constants.ServiceWorker.IDEA_AUTH_TOKEN_CLAIMS + }) + return result.claims + } else { + try { + return this.props.authContext!.getClaims() + } catch (error) { + return Promise.reject(error) + } + } + } + + async fetch(url: string, options: any, isPublic: boolean = false): Promise { + if(this.props.serviceWorkerRegistration) { + const result = await this.invoke_service_worker({ + type: Constants.ServiceWorker.IDEA_HTTP_FETCH, + options: { + url: url, + options: options, + isPublic: isPublic + } + }) + return result.response + } else { + try { + return await this.props.authContext!.fetch(url, options, isPublic) + } catch (error) { + return Promise.reject(error) + } + } + } + +} + +export default IdeaApiInvoker diff --git a/source/idea/idea-cluster-manager/webapp/src/client/index.ts b/source/idea/idea-cluster-manager/webapp/src/client/index.ts new file mode 100644 index 00000000..5fc059b7 --- /dev/null +++ b/source/idea/idea-cluster-manager/webapp/src/client/index.ts @@ -0,0 +1,40 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance + * with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES + * OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions + * and limitations under the License. + */ + +import IdeaApiInvoker from "./idea-api-invoker"; +import AuthClient from "./auth-client"; +import AccountsClient from "./accounts-client"; +import SchedulerAdminClient from "./scheduler-admin-client"; +import SchedulerClient from "./scheduler-client"; +import FileBrowserClient from "./file-browser-client"; +import VirtualDesktopClient from "./virtual-desktop-client"; +import VirtualDesktopAdminClient from "./virtual-desktop-admin-client"; +import ClusterSettingsClient from "./cluster-settings-client"; +import ProjectsClient from "./projects-client"; +import EmailTemplatesClient from "./email-templates-client"; +import IdeaClients from "./clients"; + +export { + IdeaApiInvoker, + AuthClient, + AccountsClient, + SchedulerAdminClient, + SchedulerClient, + FileBrowserClient, + VirtualDesktopClient, + VirtualDesktopAdminClient, + ClusterSettingsClient, + ProjectsClient, + EmailTemplatesClient, + IdeaClients +} diff --git a/source/idea/idea-cluster-manager/webapp/src/client/projects-client.ts b/source/idea/idea-cluster-manager/webapp/src/client/projects-client.ts new file mode 100644 index 00000000..fd598a65 --- /dev/null +++ b/source/idea/idea-cluster-manager/webapp/src/client/projects-client.ts @@ -0,0 +1,97 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance + * with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES + * OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions + * and limitations under the License. + */ + +import { + CreateProjectRequest, + CreateProjectResult, + GetProjectRequest, + GetProjectResult, + DeleteProjectRequest, + DeleteProjectResult, + UpdateProjectRequest, + UpdateProjectResult, + ListProjectsRequest, + ListProjectsResult, + EnableProjectRequest, + EnableProjectResult, + DisableProjectRequest, + DisableProjectResult, + GetUserProjectsRequest, + GetUserProjectsResult +} from './data-model' +import IdeaBaseClient, {IdeaBaseClientProps} from "./base-client"; + +export interface ProjectsClientProps extends IdeaBaseClientProps { +} + +class ProjectsClient extends IdeaBaseClient { + + createProject(req: CreateProjectRequest): Promise { + return this.apiInvoker.invoke_alt( + 'Projects.CreateProject', + req + ) + } + + getProject(req: GetProjectRequest): Promise { + return this.apiInvoker.invoke_alt( + 'Projects.GetProject', + req + ) + } + + deleteProject(req: DeleteProjectRequest): Promise { + return this.apiInvoker.invoke_alt( + 'Projects.DeleteProject', + req + ) + } + + updateProject(req: UpdateProjectRequest): Promise { + return this.apiInvoker.invoke_alt( + 'Projects.UpdateProject', + req + ) + } + + listProjects(req: ListProjectsRequest): Promise { + return this.apiInvoker.invoke_alt( + 'Projects.ListProjects', + req + ) + } + + getUserProjects(req: GetUserProjectsRequest): Promise { + return this.apiInvoker.invoke_alt( + 'Projects.GetUserProjects', + req + ) + } + + enableProject(req: EnableProjectRequest): Promise { + return this.apiInvoker.invoke_alt( + 'Projects.EnableProject', + req + ) + } + + disableProject(req: DisableProjectRequest): Promise { + return this.apiInvoker.invoke_alt( + 'Projects.DisableProject', + req + ) + } + +} + +export default ProjectsClient diff --git a/source/idea/idea-cluster-manager/webapp/src/client/scheduler-admin-client.ts b/source/idea/idea-cluster-manager/webapp/src/client/scheduler-admin-client.ts new file mode 100644 index 00000000..d43726d9 --- /dev/null +++ b/source/idea/idea-cluster-manager/webapp/src/client/scheduler-admin-client.ts @@ -0,0 +1,258 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance + * with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES + * OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions + * and limitations under the License. + */ + +import { + ListNodesRequest, + ListNodesResult, + ListJobsRequest, + ListJobsResult, + ListQueueProfilesRequest, + ListQueueProfilesResult, + EnableQueueProfileRequest, + EnableQueueProfileResult, + DisableQueueProfileRequest, + DisableQueueProfileResult, + CreateQueueProfileRequest, + CreateQueueProfileResult, + GetQueueProfileRequest, + GetQueueProfileResult, + UpdateQueueProfileRequest, + UpdateQueueProfileResult, + DeleteQueueProfileRequest, + DeleteQueueProfileResult, + CreateQueuesRequest, + CreateQueuesResult, + DeleteQueuesRequest, + DeleteQueuesResult, + ListHpcApplicationsRequest, + ListHpcApplicationsResult, + CreateHpcApplicationRequest, + CreateHpcApplicationResult, + GetHpcApplicationRequest, + GetHpcApplicationResult, + UpdateHpcApplicationRequest, + UpdateHpcApplicationResult, + DeleteHpcApplicationRequest, + DeleteHpcApplicationResult, + GetModuleInfoRequest, + GetModuleInfoResult, + GetUserApplicationsRequest, + GetUserApplicationsResult, + CreateHpcLicenseResourceRequest, + CreateHpcLicenseResourceResult, + GetHpcLicenseResourceRequest, + GetHpcLicenseResourceResult, + UpdateHpcLicenseResourceRequest, + UpdateHpcLicenseResourceResult, + DeleteHpcLicenseResourceRequest, + DeleteHpcLicenseResourceResult, + ListHpcLicenseResourcesRequest, + ListHpcLicenseResourcesResult, + CheckHpcLicenseResourceAvailabilityRequest, + CheckHpcLicenseResourceAvailabilityResult, + DeleteJobRequest, + DeleteJobResult +} from './data-model' +import IdeaBaseClient, {IdeaBaseClientProps} from "./base-client"; + +export interface SchedulerAdminClientProps extends IdeaBaseClientProps { +} + +class SchedulerAdminClient extends IdeaBaseClient { + + getModuleInfo(): Promise { + return this.apiInvoker.invoke_alt( + 'App.GetModuleInfo', + {} + ) + } + + listActiveJobs(req: ListJobsRequest): Promise { + return this.apiInvoker.invoke_alt( + 'SchedulerAdmin.ListActiveJobs', + req + ) + } + + listCompletedJobs(req: ListJobsRequest): Promise { + return this.apiInvoker.invoke_alt( + 'SchedulerAdmin.ListCompletedJobs', + req + ) + } + + listNodes(req: ListNodesRequest): Promise { + return this.apiInvoker.invoke_alt( + 'SchedulerAdmin.ListNodes', + req + ) + } + + createQueueProfile(req: CreateQueueProfileRequest): Promise { + return this.apiInvoker.invoke_alt( + 'SchedulerAdmin.CreateQueueProfile', + req + ) + } + + getQueueProfile(req: GetQueueProfileRequest): Promise { + return this.apiInvoker.invoke_alt( + 'SchedulerAdmin.GetQueueProfile', + req + ) + } + + updateQueueProfile(req: UpdateQueueProfileRequest): Promise { + return this.apiInvoker.invoke_alt( + 'SchedulerAdmin.UpdateQueueProfile', + req + ) + } + + deleteQueueProfile(req: DeleteQueueProfileRequest): Promise { + return this.apiInvoker.invoke_alt( + 'SchedulerAdmin.DeleteQueueProfile', + req + ) + } + + enableQueueProfile(req: EnableQueueProfileRequest): Promise { + return this.apiInvoker.invoke_alt( + 'SchedulerAdmin.EnableQueueProfile', + req + ) + } + + disableQueueProfile(req: DisableQueueProfileRequest): Promise { + return this.apiInvoker.invoke_alt( + 'SchedulerAdmin.DisableQueueProfile', + req + ) + } + + listQueueProfiles(req: ListQueueProfilesRequest): Promise { + return this.apiInvoker.invoke_alt( + 'SchedulerAdmin.ListQueueProfiles', + req + ) + } + + createQueues(req: CreateQueuesRequest): Promise { + return this.apiInvoker.invoke_alt( + 'SchedulerAdmin.CreateQueues', + req + ) + } + + deleteQueues(req: DeleteQueuesRequest): Promise { + return this.apiInvoker.invoke_alt( + 'SchedulerAdmin.DeleteQueues', + req + ) + } + + createHpcApplication(req: CreateHpcApplicationRequest): Promise { + return this.apiInvoker.invoke_alt( + 'SchedulerAdmin.CreateHpcApplication', + req + ) + } + + getHpcApplication(req: GetHpcApplicationRequest): Promise { + return this.apiInvoker.invoke_alt( + 'SchedulerAdmin.GetHpcApplication', + req + ) + } + + updateHpcApplication(req: UpdateHpcApplicationRequest): Promise { + return this.apiInvoker.invoke_alt( + 'SchedulerAdmin.UpdateHpcApplication', + req + ) + } + + deleteHpcApplication(req: DeleteHpcApplicationRequest): Promise { + return this.apiInvoker.invoke_alt( + 'SchedulerAdmin.DeleteHpcApplication', + req + ) + } + + listHpcApplications(req: ListHpcApplicationsRequest): Promise { + return this.apiInvoker.invoke_alt( + 'SchedulerAdmin.ListHpcApplications', + req + ) + } + + getUserApplications(req: GetUserApplicationsRequest): Promise { + return this.apiInvoker.invoke_alt( + 'SchedulerAdmin.GetUserApplications', + req + ) + } + + createHpcLicenseResource(req: CreateHpcLicenseResourceRequest): Promise { + return this.apiInvoker.invoke_alt( + 'SchedulerAdmin.CreateHpcLicenseResource', + req + ) + } + + getHpcLicenseResource(req: GetHpcLicenseResourceRequest): Promise { + return this.apiInvoker.invoke_alt( + 'SchedulerAdmin.GetHpcLicenseResource', + req + ) + } + + updateHpcLicenseResource(req: UpdateHpcLicenseResourceRequest): Promise { + return this.apiInvoker.invoke_alt( + 'SchedulerAdmin.UpdateHpcLicenseResource', + req + ) + } + + deleteHpcLicenseResource(req: DeleteHpcLicenseResourceRequest): Promise { + return this.apiInvoker.invoke_alt( + 'SchedulerAdmin.DeleteHpcLicenseResource', + req + ) + } + + listHpcLicenseResources(req: ListHpcLicenseResourcesRequest): Promise { + return this.apiInvoker.invoke_alt( + 'SchedulerAdmin.ListHpcLicenseResources', + req + ) + } + + checkHpcLicenseResourceAvailability(req: CheckHpcLicenseResourceAvailabilityRequest): Promise { + return this.apiInvoker.invoke_alt( + 'SchedulerAdmin.CheckHpcLicenseResourceAvailability', + req + ) + } + + deleteJob(req: DeleteJobRequest): Promise { + return this.apiInvoker.invoke_alt( + 'SchedulerAdmin.DeleteJob', + req + ) + } + + +} + +export default SchedulerAdminClient diff --git a/source/idea/idea-cluster-manager/webapp/src/client/scheduler-client.ts b/source/idea/idea-cluster-manager/webapp/src/client/scheduler-client.ts new file mode 100644 index 00000000..9f91a85e --- /dev/null +++ b/source/idea/idea-cluster-manager/webapp/src/client/scheduler-client.ts @@ -0,0 +1,87 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance + * with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES + * OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions + * and limitations under the License. + */ + +import { + GetInstanceTypeOptionsRequest, + GetInstanceTypeOptionsResult, + GetUserApplicationsRequest, + GetUserApplicationsResult, + ListJobsRequest, + ListJobsResult, + SubmitJobRequest, + SubmitJobResult, + DeleteJobRequest, + DeleteJobResult, + GetModuleInfoRequest, + GetModuleInfoResult, + +} from './data-model' +import IdeaBaseClient, {IdeaBaseClientProps} from "./base-client"; + +export interface SchedulerClientProps extends IdeaBaseClientProps { +} + +class SchedulerClient extends IdeaBaseClient { + + getModuleInfo(): Promise { + return this.apiInvoker.invoke_alt( + 'App.GetModuleInfo', + {} + ) + } + + listActiveJobs(req: ListJobsRequest): Promise { + return this.apiInvoker.invoke_alt( + 'Scheduler.ListActiveJobs', + req + ) + } + + listCompletedJobs(req: ListJobsRequest): Promise { + return this.apiInvoker.invoke_alt( + 'Scheduler.ListCompletedJobs', + req + ) + } + + getUserApplications(req: GetUserApplicationsRequest): Promise { + return this.apiInvoker.invoke_alt( + 'Scheduler.GetUserApplications', + req + ) + } + + submitJob(req: SubmitJobRequest): Promise { + return this.apiInvoker.invoke_alt( + 'Scheduler.SubmitJob', + req + ) + } + + getInstanceTypeOptions(req: GetInstanceTypeOptionsRequest): Promise { + return this.apiInvoker.invoke_alt( + 'Scheduler.GetInstanceTypeOptions', + req + ) + } + + deleteJob(req: DeleteJobRequest): Promise { + return this.apiInvoker.invoke_alt( + 'Scheduler.DeleteJob', + req + ) + } + +} + +export default SchedulerClient diff --git a/source/idea/idea-cluster-manager/webapp/src/client/virtual-desktop-admin-client.ts b/source/idea/idea-cluster-manager/webapp/src/client/virtual-desktop-admin-client.ts new file mode 100644 index 00000000..8d0dd16b --- /dev/null +++ b/source/idea/idea-cluster-manager/webapp/src/client/virtual-desktop-admin-client.ts @@ -0,0 +1,246 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance + * with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES + * OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions + * and limitations under the License. + */ + +import { + CreateSessionRequest, + CreateSessionResponse, + GetSessionInfoRequest, + GetSessionInfoResponse, + UpdateSessionRequest, + UpdateSessionResponse, + DeleteSessionRequest, + DeleteSessionResponse, + ListSessionsRequest, + ListSessionsResponse, + GetSessionScreenshotRequest, + GetSessionScreenshotResponse, + BatchCreateSessionRequest, + BatchCreateSessionResponse, + StopSessionRequest, + StopSessionResponse, + ResumeSessionsRequest, + ResumeSessionsResponse, + ListSoftwareStackRequest, + ListSoftwareStackResponse, + CreateSoftwareStackRequest, + CreateSoftwareStackResponse, + UpdateSoftwareStackRequest, + UpdateSoftwareStackResponse, + GetModuleInfoRequest, + GetModuleInfoResult, + RebootSessionRequest, + RebootSessionResponse, + CreateSoftwareStackFromSessionRequest, + CreateSoftwareStackFromSessionResponse, + VirtualDesktopSessionConnectionInfo, + GetSessionConnectionInfoRequest, + CreatePermissionProfileResponse, + CreatePermissionProfileRequest, + ListPermissionsRequest, + ListPermissionsResponse, + GetSoftwareStackInfoResponse, + GetSoftwareStackInfoRequest, + UpdatePermissionProfileResponse, + UpdatePermissionProfileRequest, + UpdateSessionPermissionRequest, + UpdateSessionPermissionResponse +} from './data-model' +import IdeaBaseClient, {IdeaBaseClientProps} from "./base-client"; + +export interface VirtualDesktopAdminClientProps extends IdeaBaseClientProps { +} + +class VirtualDesktopAdminClient extends IdeaBaseClient { + + getModuleInfo(): Promise { + return this.apiInvoker.invoke_alt( + 'App.GetModuleInfo', + {} + ) + } + + createSession(req: CreateSessionRequest): Promise { + return this.apiInvoker.invoke_alt( + 'VirtualDesktopAdmin.CreateSession', + req + ) + } + + batchCreateSessions(req: BatchCreateSessionRequest): Promise { + return this.apiInvoker.invoke_alt( + 'VirtualDesktopAdmin.BatchCreateSessions', + req + ) + } + + updateSession(req: UpdateSessionRequest): Promise { + return this.apiInvoker.invoke_alt( + 'VirtualDesktopAdmin.UpdateSession', + req + ) + } + + deleteSessions(req: DeleteSessionRequest): Promise { + return this.apiInvoker.invoke_alt( + 'VirtualDesktopAdmin.DeleteSessions', + req + ) + } + + getSessionInfo(req: GetSessionInfoRequest): Promise { + return this.apiInvoker.invoke_alt( + 'VirtualDesktopAdmin.GetSessionInfo', + req + ) + } + + listSessions(req: ListSessionsRequest): Promise { + return this.apiInvoker.invoke_alt( + 'VirtualDesktopAdmin.ListSessions', + req + ) + } + + stopSessions(req: StopSessionRequest): Promise { + return this.apiInvoker.invoke_alt( + 'VirtualDesktopAdmin.StopSessions', + req + ) + } + + rebootSessions(req: RebootSessionRequest): Promise { + return this.apiInvoker.invoke_alt( + 'VirtualDesktopAdmin.RebootSessions', + req + ) + } + + resumeSessions(req: ResumeSessionsRequest): Promise { + return this.apiInvoker.invoke_alt( + 'VirtualDesktopAdmin.ResumeSessions', + req + ) + } + + getSessionScreenshot(req: GetSessionScreenshotRequest): Promise { + return this.apiInvoker.invoke_alt( + 'VirtualDesktopAdmin.GetSessionScreenshot', + req + ) + } + + getSessionConnectionInfo(req: GetSessionConnectionInfoRequest): Promise { + return this.apiInvoker.invoke_alt( + 'VirtualDesktopAdmin.GetSessionConnectionInfo', + req + ) + } + + createSoftwareStack(req: CreateSoftwareStackRequest): Promise { + return this.apiInvoker.invoke_alt( + 'VirtualDesktopAdmin.CreateSoftwareStack', + req + ) + } + + updateSoftwareStack(req: UpdateSoftwareStackRequest): Promise { + return this.apiInvoker.invoke_alt( + 'VirtualDesktopAdmin.UpdateSoftwareStack', + req + ) + } + + getSoftwareStackInfo(req: GetSoftwareStackInfoRequest): Promise { + return this.apiInvoker.invoke_alt( + 'VirtualDesktopAdmin.GetSoftwareStackInfo', + req + ) + } + + listSoftwareStacks(req: ListSoftwareStackRequest): Promise { + return this.apiInvoker.invoke_alt( + 'VirtualDesktopAdmin.ListSoftwareStacks', + req + ) + } + + createSoftwareStackFromSession(req: CreateSoftwareStackFromSessionRequest): Promise { + return this.apiInvoker.invoke_alt( + 'VirtualDesktopAdmin.CreateSoftwareStackFromSession', + req + ) + } + + createPermissionProfile(req: CreatePermissionProfileRequest): Promise { + return this.apiInvoker.invoke_alt( + 'VirtualDesktopAdmin.CreatePermissionProfile', + req + ) + } + + updatePermissionProfile(req: UpdatePermissionProfileRequest): Promise { + return this.apiInvoker.invoke_alt( + 'VirtualDesktopAdmin.UpdatePermissionProfile', + req + ) + } + + listSessionPermissions(req: ListPermissionsRequest): Promise { + return this.apiInvoker.invoke_alt( + 'VirtualDesktopAdmin.ListSessionPermissions', + req + ) + } + + listSharedPermissions(req: ListPermissionsRequest): Promise { + return this.apiInvoker.invoke_alt( + 'VirtualDesktopAdmin.ListSharedPermissions', + req + ) + } + + updateSessionPermission(req: UpdateSessionPermissionRequest): Promise { + return this.apiInvoker.invoke_alt( + 'VirtualDesktopAdmin.UpdateSessionPermissions', + req + ) + } + + joinSession(idea_session_id: string, idea_session_owner: string, username?: string): Promise { + return new Promise(() => { + let connection_info: VirtualDesktopSessionConnectionInfo = { + idea_session_id: idea_session_id, + idea_session_owner: idea_session_owner + } + + if (username) { + connection_info.username = username + } + + this.getSessionConnectionInfo({ + connection_info: connection_info + }).then(result => { + return `${result.connection_info?.endpoint}${result.connection_info?.web_url_path}?authToken=${result.connection_info?.access_token}#${result.connection_info?.dcv_session_id}` + }).then(url => { + window.open(url); + return true + }).catch(error => { + console.error(error) + return false + }) + }) + } + +} + +export default VirtualDesktopAdminClient diff --git a/source/idea/idea-cluster-manager/webapp/src/client/virtual-desktop-client.ts b/source/idea/idea-cluster-manager/webapp/src/client/virtual-desktop-client.ts new file mode 100644 index 00000000..0d77af60 --- /dev/null +++ b/source/idea/idea-cluster-manager/webapp/src/client/virtual-desktop-client.ts @@ -0,0 +1,183 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance + * with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES + * OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions + * and limitations under the License. + */ + +import { + CreateSessionRequest, + CreateSessionResponse, + GetSessionInfoRequest, + GetSessionInfoResponse, + UpdateSessionRequest, + UpdateSessionResponse, + DeleteSessionRequest, + DeleteSessionResponse, + ListSessionsRequest, + ListSessionsResponse, + GetSessionScreenshotRequest, + GetSessionScreenshotResponse, + StopSessionRequest, + StopSessionResponse, + ResumeSessionsRequest, + ResumeSessionsResponse, + GetSessionConnectionInfoRequest, + GetSessionConnectionInfoResponse, + ListSoftwareStackRequest, + ListSoftwareStackResponse, + GetModuleInfoRequest, + GetModuleInfoResult, + RebootSessionResponse, + RebootSessionRequest, + UpdateSessionPermissionRequest, + UpdateSessionPermissionResponse, + ListPermissionsRequest, + ListPermissionsResponse, + VirtualDesktopSessionConnectionInfo +} from './data-model' +import IdeaBaseClient, {IdeaBaseClientProps} from "./base-client"; + +export interface VirtualDesktopClientProps extends IdeaBaseClientProps { +} + +class VirtualDesktopClient extends IdeaBaseClient { + + getModuleInfo(): Promise { + return this.apiInvoker.invoke_alt( + 'App.GetModuleInfo', + {} + ) + } + + createSession(req: CreateSessionRequest): Promise { + return this.apiInvoker.invoke_alt( + 'VirtualDesktop.CreateSession', + req + ) + } + + updateSession(req: UpdateSessionRequest): Promise { + return this.apiInvoker.invoke_alt( + 'VirtualDesktop.UpdateSession', + req + ) + } + + deleteSessions(req: DeleteSessionRequest): Promise { + return this.apiInvoker.invoke_alt( + 'VirtualDesktop.DeleteSessions', + req + ) + } + + getSessionInfo(req: GetSessionInfoRequest): Promise { + return this.apiInvoker.invoke_alt( + 'VirtualDesktop.GetSessionInfo', + req + ) + } + + getSessionScreenshot(req: GetSessionScreenshotRequest): Promise { + return this.apiInvoker.invoke_alt( + 'VirtualDesktop.GetSessionScreenshot', + req + ) + } + + getSessionConnectionInfo(req: GetSessionConnectionInfoRequest): Promise { + return this.apiInvoker.invoke_alt( + 'VirtualDesktop.GetSessionConnectionInfo', + req + ) + } + + listSessions(req: ListSessionsRequest): Promise { + return this.apiInvoker.invoke_alt( + 'VirtualDesktop.ListSessions', + req + ) + } + + stopSessions(req: StopSessionRequest): Promise { + return this.apiInvoker.invoke_alt( + 'VirtualDesktop.StopSessions', + req + ) + } + + resumeSessions(req: ResumeSessionsRequest): Promise { + return this.apiInvoker.invoke_alt( + 'VirtualDesktop.ResumeSessions', + req + ) + } + + rebootSessions(req: RebootSessionRequest): Promise { + return this.apiInvoker.invoke_alt( + 'VirtualDesktop.RebootSessions', + req + ) + } + + listSoftwareStacks(req: ListSoftwareStackRequest): Promise { + return this.apiInvoker.invoke_alt( + 'VirtualDesktop.ListSoftwareStacks', + req + ) + } + + listSharedPermissions(req: ListPermissionsRequest): Promise { + return this.apiInvoker.invoke_alt( + 'VirtualDesktop.ListSharedPermissions', + req + ) + } + + listSessionPermissions(req: ListPermissionsRequest): Promise { + return this.apiInvoker.invoke_alt( + 'VirtualDesktop.ListSessionPermissions', + req + ) + } + + updateSessionPermissions(req: UpdateSessionPermissionRequest): Promise { + return this.apiInvoker.invoke_alt( + 'VirtualDesktop.UpdateSessionPermissions', + req + ) + } + + joinSession(idea_session_id: string, idea_session_owner: string, username?: string): Promise { + return new Promise(() => { + let connection_info: VirtualDesktopSessionConnectionInfo = { + idea_session_id: idea_session_id, + idea_session_owner: idea_session_owner + } + + if (username) { + connection_info.username = username + } + + this.getSessionConnectionInfo({ + connection_info: connection_info + }).then(result => { + return `${result.connection_info?.endpoint}${result.connection_info?.web_url_path}?authToken=${result.connection_info?.access_token}#${result.connection_info?.dcv_session_id}` + }).then(url => { + window.open(url); + return true + }).catch(error => { + console.error(error) + return false + }) + }) + } +} + +export default VirtualDesktopClient diff --git a/source/idea/idea-cluster-manager/webapp/src/client/virtual-desktop-dcv-client.ts b/source/idea/idea-cluster-manager/webapp/src/client/virtual-desktop-dcv-client.ts new file mode 100644 index 00000000..910b21f8 --- /dev/null +++ b/source/idea/idea-cluster-manager/webapp/src/client/virtual-desktop-dcv-client.ts @@ -0,0 +1,42 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance + * with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES + * OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions + * and limitations under the License. + */ + +import { + DescribeServersRequest, DescribeServersResponse, + DescribeSessionsRequest, DescribeSessionsResponse + +} from './data-model' +import IdeaBaseClient, {IdeaBaseClientProps} from "./base-client"; + +export interface VirtualDesktopDCVClientProps extends IdeaBaseClientProps { +} + +class VirtualDesktopDCVClient extends IdeaBaseClient { + + describeSessions(req: DescribeSessionsRequest): Promise { + return this.apiInvoker.invoke_alt( + 'VirtualDesktopDCV.DescribeSessions', + req, + ) + } + + describeServers(req: DescribeServersRequest): Promise { + return this.apiInvoker.invoke_alt( + 'VirtualDesktopDCV.DescribeServers', + req + ) + } + +} + +export default VirtualDesktopDCVClient diff --git a/source/idea/idea-cluster-manager/webapp/src/client/virtual-desktop-utils-client.ts b/source/idea/idea-cluster-manager/webapp/src/client/virtual-desktop-utils-client.ts new file mode 100644 index 00000000..d6073811 --- /dev/null +++ b/source/idea/idea-cluster-manager/webapp/src/client/virtual-desktop-utils-client.ts @@ -0,0 +1,97 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance + * with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES + * OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions + * and limitations under the License. + */ + +import { + ListSupportedOSRequest, + ListSupportedOSResponse, + ListAllowedInstanceTypesRequest, + ListAllowedInstanceTypesResponse, + ListAllowedInstanceTypesForSessionResponse, + ListAllowedInstanceTypesForSessionRequest, + ListPermissionProfilesRequest, + ListPermissionProfilesResponse, + GetBasePermissionsRequest, + GetBasePermissionsResponse, + ListSupportedGPURequest, + ListSupportedGPUResponse, + ListScheduleTypesRequest, + ListScheduleTypesResponse, + GetPermissionProfileRequest, + GetPermissionProfileResponse +} from './data-model' +import IdeaBaseClient, {IdeaBaseClientProps} from "./base-client"; + +export interface VirtualDesktopUtilsClientProps extends IdeaBaseClientProps { +} + +class VirtualDesktopUtilsClient extends IdeaBaseClient { + + listSupportedOS(req: ListSupportedOSRequest): Promise { + return this.apiInvoker.invoke_alt( + 'VirtualDesktopUtils.ListSupportedOS', + req + ) + } + + listSupportedGPUs(req: ListSupportedGPURequest): Promise { + return this.apiInvoker.invoke_alt( + 'VirtualDesktopUtils.ListSupportedGPU', + req + ) + } + + listScheduleTypes(req: ListScheduleTypesRequest): Promise { + return this.apiInvoker.invoke_alt( + 'VirtualDesktopUtils.ListScheduleTypes', + req + ) + } + + listAllowedInstanceTypes(req: ListAllowedInstanceTypesRequest): Promise { + return this.apiInvoker.invoke_alt( + 'VirtualDesktopUtils.ListAllowedInstanceTypes', + req + ) + } + + listAllowedInstanceTypesForSession(req: ListAllowedInstanceTypesForSessionRequest): Promise { + return this.apiInvoker.invoke_alt( + 'VirtualDesktopUtils.ListAllowedInstanceTypesForSession', + req + ) + } + + getBasePermissions(req: GetBasePermissionsRequest): Promise { + return this.apiInvoker.invoke_alt( + 'VirtualDesktopUtils.GetBasePermissions', + req + ) + } + + listPermissionProfiles(req: ListPermissionProfilesRequest): Promise { + return this.apiInvoker.invoke_alt( + 'VirtualDesktopUtils.ListPermissionProfiles', + req + ) + } + + getPermissionProfile(req: GetPermissionProfileRequest): Promise { + return this.apiInvoker.invoke_alt( + 'VirtualDesktopUtils.GetPermissionProfile', + req + ) + } + +} + +export default VirtualDesktopUtilsClient diff --git a/source/idea/idea-cluster-manager/webapp/src/common/app-context.ts b/source/idea/idea-cluster-manager/webapp/src/common/app-context.ts new file mode 100644 index 00000000..9a85b430 --- /dev/null +++ b/source/idea/idea-cluster-manager/webapp/src/common/app-context.ts @@ -0,0 +1,252 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance + * with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES + * OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions + * and limitations under the License. + */ + +import {v4 as uuid} from "uuid" +import {AuthService, LocalStorageService} from "../service"; +import IdeaException from "../common/exceptions"; +import JobTemplatesService from "../service/job-templates-service"; +import ClusterSettingsService from "../service/cluster-settings-service"; +import Utils from "./utils"; +import {Constants} from "./constants"; +import {IdeaAuthenticationContext} from "./authentication-context"; +import {IdeaClients} from "../client"; + +export interface AppContextProps { + httpEndpoint: string + albEndpoint: string + releaseVersion: string + app: AppData + serviceWorkerRegistration?: ServiceWorkerRegistration +} + +export interface AppData { + sso: boolean + version: string + title: string + subtitle?: string + logo?: string + copyright_text?: string + module_set: string + modules: any + session_management: 'local-storage' | 'in-memory', + default_log_level: number +} + +const DARK_MODE_KEY = 'theme.dark-mode' +const COMPACT_MODE_KEY = 'theme.compact-mode' + +let IS_LOGGED_IN_INTERVAL: any = null + +class AppContext { + + private props: AppContextProps + + private readonly clients: IdeaClients + private readonly localStorageService: LocalStorageService + private readonly authContext?: IdeaAuthenticationContext + private readonly authService: AuthService + private readonly jobTemplatesService: JobTemplatesService + private readonly clusterSettingsService: ClusterSettingsService + + private static onRouteCallback?: any + + private isLoggedIn: boolean = false + private onLogin?: () => Promise + private onLogout?: () => Promise + + constructor(props: AppContextProps) { + this.props = props + + const authEndpoint = `${props.httpEndpoint}${Utils.getApiContextPath(Constants.MODULE_CLUSTER_MANAGER)}` + + const initializeServiceWorker = () => { + if (this.props.serviceWorkerRegistration) { + this.props.serviceWorkerRegistration.active!.postMessage({ + type: Constants.ServiceWorker.IDEA_AUTH_INIT, + options: { + authEndpoint: authEndpoint, + defaultLogLevel: props.app.default_log_level + } + }) + } + } + + if (this.props.serviceWorkerRegistration) { + initializeServiceWorker() + } else { + this.authContext = new IdeaAuthenticationContext({ + sessionManagement: 'local-storage', + authEndpoint: authEndpoint + }) + } + + this.localStorageService = new LocalStorageService({ + prefix: 'idea' + }) + + this.clients = new IdeaClients({ + appId: 'web-portal', + baseUrl: props.httpEndpoint, + authContext: this.authContext, + serviceWorkerRegistration: props.serviceWorkerRegistration + }) + + this.authService = new AuthService({ + localStorage: this.localStorageService, + clients: this.clients + }) + + this.clusterSettingsService = new ClusterSettingsService({ + clusterSettings: this.clients.clusterSettings() + }) + + this.jobTemplatesService = new JobTemplatesService({}) + + // the purpose of below interval is: + // 1. ensure we send a periodic heart-beat to service-worker so that service worker remains active. + // this is only applicable when service worker is initialized. on FireFox, service worker becomes inactive + // after 30-seconds of no activity to service worker and session expires prematurely. + // 2. check for login status changes and take respective actions + + if (IS_LOGGED_IN_INTERVAL != null) { + clearInterval(IS_LOGGED_IN_INTERVAL) + } + + IS_LOGGED_IN_INTERVAL = setInterval(() => { + + // initializing service worker in this interval ensures that in the event the service worker was stopped and + // started, the service worker knows the authentication endpoints + initializeServiceWorker() + + // check if the user is logged in. this may query the service worker or authentication context based on current session management mode. + this.authService.isLoggedIn().then((status) => { + if (this.isLoggedIn && !status) { + this.authService.logout().finally() + } + this.isLoggedIn = status + }) + }, 10000) + } + + static setOnRoute(onRoute: any) { + this.onRouteCallback = onRoute + } + + static get(): AppContext { + if (window.idea.context == null) { + throw new IdeaException({ + errorCode: 'APP_CONTEXT_NOT_INITIALIZED', + message: 'AppContext not initialized' + }) + } + return window.idea.context + } + + setHooks(onLogin: () => Promise, onLogout: () => Promise) { + this.onLogin = onLogin + this.onLogout = onLogout + this.authService.setHooks(onLogin, onLogout) + this.clients.getClients().forEach(client => { + client.setHooks(onLogin, onLogout) + }) + } + + setDarkMode(darkMode: boolean) { + return this.localStorage().setItem(DARK_MODE_KEY, `${darkMode}`) + } + + isDarkMode(): boolean { + return Utils.asBoolean(AppContext.get().localStorage().getItem(DARK_MODE_KEY), false) + } + + setCompactMode(compactMode: boolean) { + return this.localStorage().setItem(COMPACT_MODE_KEY, `${compactMode}`) + } + + isCompactMode(): boolean { + return Utils.asBoolean(AppContext.get().localStorage().getItem(COMPACT_MODE_KEY), false) + } + + getHttpEndpoint(): string { + return this.props.httpEndpoint + } + + getAlbEndpoint(): string { + return this.props.albEndpoint + } + + releaseVersion(): string { + return this.props.releaseVersion + } + + getTitle(): string { + return this.props.app.title + } + + getSubtitle(): string { + if (this.props.app.subtitle != null) { + return this.props.app.subtitle + } + const clusterName = this.auth().getClusterName() + if (Utils.isNotEmpty(clusterName)) { + return `${this.auth().getClusterName()} (${this.auth().getAwsRegion()})` + } else { + return '' + } + } + + getLogoUrl(): string | undefined { + return this.props.app.logo + } + + getCopyRightText(): string { + if (this.props.app.copyright_text) { + return this.props.app.copyright_text + } else { + return `Copyright ${new Date().getFullYear()} Amazon.com, Inc. or its affiliates. All Rights Reserved.` + } + } + + uuid(): string { + return uuid() + } + + client(): IdeaClients { + return this.clients + } + + auth(): AuthService { + return this.authService + } + + localStorage(): LocalStorageService { + return this.localStorageService + } + + jobTemplates(): JobTemplatesService { + return this.jobTemplatesService + } + + getClusterSettingsService(): ClusterSettingsService { + return this.clusterSettingsService + } + + routeTo(path: string) { + if (AppContext.onRouteCallback) { + AppContext.onRouteCallback(path) + } + } + +} + +export default AppContext diff --git a/source/idea/idea-cluster-manager/webapp/src/common/app-logger.ts b/source/idea/idea-cluster-manager/webapp/src/common/app-logger.ts new file mode 100644 index 00000000..2969fafb --- /dev/null +++ b/source/idea/idea-cluster-manager/webapp/src/common/app-logger.ts @@ -0,0 +1,147 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance + * with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES + * OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions + * and limitations under the License. + */ + +import Utils from "./utils"; + +const LogLevel = { + OFF: 0, + ERROR: 1, + WARN: 2, + INFO: 3, + DEBUG: 4, + TRACE: 5 +} + +const LogLevelName: { [k: number]: string } = { + 0: 'OFF', + 1: 'ERROR', + 2: 'WARN', + 3: 'INFO', + 4: 'DEBUG', + 5: 'TRACE' +} + +const KEY_LOG_LEVEL = 'idea.log-level' + +export interface AppLoggerProps { + name: string + default_log_level?: number +} + +/*** + * AppLoger class as a central logging function across IDEA frontend components + * + * To enable debug logging in production environments: + * 1. navigate to browser menu-> developer tools -> console + * 2. type: localStorage.setItem('idea.log-level', '4') to enable debug logging. + * 3. reload the page + * + * Roadmap: + * * Front end logs can be routed to cloud watch in a future release to track frontend activity and errors. + */ +class AppLogger { + + private readonly name: string + private readonly logLevel: number + + constructor(props: AppLoggerProps) { + this.name = props.name + + if(typeof props.default_log_level !== 'undefined') { + // window reference is not available in service worker, so default log level is passed to ServiceWorker during IDEA_APP_INIT + // which in turn is passed to AppLoggerProps by AuthenticationContext that runs within a service worker + this.logLevel = Utils.asNumber(props.default_log_level, LogLevel.INFO) + } else if(typeof window !== 'undefined' && typeof localStorage !== 'undefined') { + // for all other components (not initialized in service worker) window reference will be available and initialize default log level from app data + const defaultLogLevel = Utils.asNumber(window.idea.app.default_log_level, LogLevel.INFO) + // local storage reference is not available in service worker. + this.logLevel = Utils.asNumber(localStorage.getItem(KEY_LOG_LEVEL), defaultLogLevel) + } else { + this.logLevel = LogLevel.INFO + } + } + + private getLogLevel(logLevel: number): string { + let result = LogLevelName[logLevel] + if (result == null) { + return 'UNKNOWN' + } + return result + } + + private getLogTag(logLevel: number): string { + return `[${this.getLogLevel(logLevel)}] [${new Date().toISOString()}] [${this.name}]` + } + + isEnabled(): boolean { + return this.logLevel > LogLevel.OFF + } + + isError(): boolean { + return this.logLevel >= LogLevel.ERROR + } + + isWarn(): boolean { + return this.logLevel >= LogLevel.WARN + } + + isInfo(): boolean { + return this.logLevel >= LogLevel.INFO + } + + isDebug(): boolean { + return this.logLevel >= LogLevel.DEBUG + } + + isTrace(): boolean { + return this.logLevel >= LogLevel.TRACE + } + + error(message?: any, ...optionalParams: any[]) { + if (this.logLevel < LogLevel.ERROR) { + return + } + console.error(this.getLogTag(LogLevel.ERROR), message, ...optionalParams) + } + + warn(message?: any, ...optionalParams: any[]) { + if (this.logLevel < LogLevel.WARN) { + return + } + console.warn(`${this.getLogTag(LogLevel.WARN)} ${message}`, ...optionalParams) + } + + info(message?: any, ...optionalParams: any[]) { + if (this.logLevel < LogLevel.INFO) { + return + } + console.info(`${this.getLogTag(LogLevel.INFO)} ${message}`, ...optionalParams) + } + + debug(message?: any, ...optionalParams: any[]) { + if (this.logLevel < LogLevel.DEBUG) { + return + } + console.log(`${this.getLogTag(LogLevel.DEBUG)} ${message}`, ...optionalParams) + } + + trace(message?: any, ...optionalParams: any[]) { + if (this.logLevel < LogLevel.TRACE) { + return + } + console.log(`${this.getLogTag(LogLevel.TRACE)} ${message}`, ...optionalParams) + } + +} + +export default AppLogger diff --git a/source/idea/idea-cluster-manager/webapp/src/common/authentication-context.ts b/source/idea/idea-cluster-manager/webapp/src/common/authentication-context.ts new file mode 100644 index 00000000..e2ef7f87 --- /dev/null +++ b/source/idea/idea-cluster-manager/webapp/src/common/authentication-context.ts @@ -0,0 +1,513 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance + * with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES + * OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions + * and limitations under the License. + */ + +import {JwtTokenClaims, JwtTokenClaimsProvider, JwtTokenUtils} from "./token-utils"; +import {AUTH_TOKEN_EXPIRED, NETWORK_ERROR, REQUEST_TIMEOUT, SERVER_ERROR} from "./error-codes"; +import {LocalStorageService} from "../service"; +import IdeaException from "./exceptions"; +import Utils from "./utils"; +import AppLogger from "./app-logger"; + +export interface IdeaAuthenticationContextProps { + authEndpoint?: string + sessionManagement: 'local-storage' | 'in-memory' +} + +export interface InitializeAppOptions { + authEndpoint: string + defaultLogLevel: number +} + +const KEY_REFRESH_TOKEN = 'refresh-token' +const KEY_ACCESS_TOKEN = 'access-token' +const KEY_ID_TOKEN = 'id-token' +const KEY_SSO_AUTH = 'sso-auth' + +const HEADER_CONTENT_TYPE_JSON = 'application/json;charset=UTF-8' +const NETWORK_TIMEOUT = 10000 + +/** + * IDEA Authentication Context + * provides functionality authentication and managing "session" at client side. session can be managed via 2 modes: + * 1. LocalStorage + * 2. ServiceWorker (with tokens saved in-memory) + * + * ServiceWorker based session management is the ideal mechanism for production applications. + * LocalStorage mode, is an unsecure fallback mechanism when ServiceWorker cannot be initialized. + * + * ServiceWorker cannot be initialized in below scenarios: + * 1. Insecure SSL/TLS Context + * For service workers to be initialized, the origin must be served over HTTPS with valid certificates. + * Self-signed certificates do not work with Service Worker. + * Refer to: https://www.chromium.org/blink/serviceworker/service-worker-faq/ for additional details. + * + * 2. Not supported or disabled by browsers + * At the time of this writing, Service Workers are supported by most modern Web Browsers including Edge and Safari on iOS (11.3+) + * Refer to https://developer.mozilla.org/en-US/docs/Web/API/ServiceWorker for Browser Compatibility for Service Workers. + * + * ServiceWorkers can be disabled by browsers, for eg. FireFox automatically disables ServiceWorkers in incognito mode. + * Additionally, a user may disable service worker via browser preferences. In such scenarios, implementation will + * automatically fall back to local storage for session management. This behavior can be customized based on server side configuration. + * + */ + +export class IdeaAuthenticationContext { + + private readonly props: IdeaAuthenticationContextProps + + private authEndpoint: string | null + private ssoAuth: boolean + private readonly localStorage: LocalStorageService | null + + private refreshToken: string | null + private accessToken: string | null + private idToken: string | null + private claimsProvider: JwtTokenClaimsProvider | null + + private logger: AppLogger + + private authContextInitialized: boolean + private renewalInProgress: any + + constructor(props: IdeaAuthenticationContextProps) { + + this.logger = new AppLogger({ + name: 'authentication-context' + }) + + this.authContextInitialized = false + this.renewalInProgress = null + + this.authEndpoint = null + this.ssoAuth = false + this.localStorage = null + + this.refreshToken = null + this.accessToken = null + this.idToken = null + this.claimsProvider = null + + this.props = props + + // used in fallback mode when service-worker cannot be initialized + if (typeof this.props.authEndpoint !== 'undefined') { + this.authEndpoint = this.props.authEndpoint + } + + // session management will never be local-storage, when AuthenticationContext is initialized from ServiceWorker. + if (this.props.sessionManagement === 'local-storage') { + this.localStorage = new LocalStorageService({ + prefix: 'idea.auth' + }) + this.initializeFromLocalStorage() + } + } + + private initializeFromLocalStorage() { + if (this.localStorage == null) { + return + } + + this.accessToken = this.localStorage.getItem(KEY_ACCESS_TOKEN) + this.idToken = this.localStorage.getItem(KEY_ID_TOKEN) + this.refreshToken = this.localStorage.getItem(KEY_REFRESH_TOKEN) + let ssoAuth = this.localStorage.getItem(KEY_SSO_AUTH) + if (ssoAuth != null) { + this.ssoAuth = Utils.asBoolean(ssoAuth) + } + + if (this.accessToken != null && this.idToken != null) { + this.claimsProvider = new JwtTokenClaimsProvider(this.accessToken, this.idToken) + } + } + + /** + * Initialize app authentication context + * this is exposed primarily for the ServiceWorker flow, where the AuthenticationContext instance resides in ServiceWorker, and + * authEndpoint and ssoAuth must be initialized after the app is initialized. + * @param {string} options.authEndpoint + * @param {boolean} options.ssoAuth + */ + initializeAuthContext(options: InitializeAppOptions): Promise { + this.logger = new AppLogger({ + name: 'authentication-context', + default_log_level: options.defaultLogLevel + }) + return new Promise((resolve, _) => { + this.authEndpoint = options.authEndpoint + resolve(true) + }) + } + + private getAuthTokenExpiredError = () => { + this.logger.warn('session expired') + return { + success: false, + message: 'Session Expired', + error_code: AUTH_TOKEN_EXPIRED + } + } + + /** + * save the authentication result in-memory + * if session_management == 'local-storage', local storage is initialized and tokens are saved in local storage. + * @param authResult + * @param ssoAuth + * @private + */ + private saveAuthResult(authResult: any, ssoAuth: boolean) { + if (authResult.refresh_token) { + this.refreshToken = authResult.refresh_token + } + this.accessToken = authResult.access_token + this.idToken = authResult.id_token + this.claimsProvider = new JwtTokenClaimsProvider(this.accessToken!, this.idToken!) + this.ssoAuth = ssoAuth + + if (this.localStorage != null) { + if (authResult.refresh_token) { + this.localStorage.setItem(KEY_REFRESH_TOKEN, authResult.refresh_token!) + } + this.localStorage.setItem(KEY_SSO_AUTH, (ssoAuth) ? 'true' : 'false') + this.localStorage.setItem(KEY_ACCESS_TOKEN, authResult.access_token!) + this.localStorage.setItem(KEY_ID_TOKEN, authResult.id_token!) + } + } + + getClaims(): JwtTokenClaims { + if (this.claimsProvider == null) { + throw new IdeaException({ + errorCode: AUTH_TOKEN_EXPIRED, + message: 'Unauthorized Access' + }) + } + const claims = this.claimsProvider.getClaims() + this.logger.debug('jwt token claims', claims) + return claims + } + + isLoggedIn(): Promise { + if (this.localStorage != null) { + // this is primarily to allow force token renewal in local storage mode for testing, by deleting the access token from local storage + return this.renewAccessToken().then(() => { + if (this.accessToken != null && this.idToken != null) { + this.claimsProvider = new JwtTokenClaimsProvider(this.accessToken, this.idToken) + return true + } else { + return false + } + }) + } else { + const isLoggedIn = this.accessToken != null + this.logger.debug('is logged in: ', isLoggedIn) + return Promise.resolve(isLoggedIn) + } + } + + logout(): Promise { + + if (this.refreshToken == null) { + return Promise.resolve(true) + } + + const request = { + header: { + namespace: 'Auth.SignOut', + request_id: Utils.getUUID() + }, + payload: { + sso_auth: this.ssoAuth, + refresh_token: this.refreshToken + } + } + + const signOutEndpoint = `${this.authEndpoint}/Auth.SignOut` + this.logger.info('logging out ...') + + return this.invoke(signOutEndpoint, request, false) + .catch((error) => { + this.logger.warn('sign out failed: unable to invalidate refresh token', error) + return true + }).finally(() => { + this.refreshToken = null + this.accessToken = null + this.idToken = null + this.claimsProvider = null + if (this.localStorage != null) { + this.localStorage.removeItem(KEY_ACCESS_TOKEN) + this.localStorage.removeItem(KEY_REFRESH_TOKEN) + this.localStorage.removeItem(KEY_SSO_AUTH) + this.localStorage.removeItem(KEY_ID_TOKEN) + } + return true + }) + } + + getAccessToken(): Promise { + return this.renewAccessToken().then(success => { + if (success) { + return this.accessToken! + } else { + return Promise.reject(this.getAuthTokenExpiredError()) + } + }) + } + + isAccessTokenExpired(): boolean { + if (this.accessToken == null) { + return true + } + if (this.claimsProvider == null) { + return true + } + return new Date().getTime() >= (this.claimsProvider!.getExpiresAt() - (5 * 60 * 1000)) + } + + /** + * wrapper over the in-built fetch method + * @param url + * @param options + */ + private _fetch = (url: string, options: any): Promise => { + const abortController = new AbortController() + const timeout = setTimeout(() => abortController.abort(), NETWORK_TIMEOUT) + options = { + ...options, + signal: abortController.signal + } + return fetch(url, options).then(response => { + if (response.status === 200) { + return response.json() + } else { + this.logger.error('server error', response) + return { + success: false, + error_code: SERVER_ERROR, + message: 'Server error' + } + } + }).catch(error => { + this.logger.error('network error', error) + if (error.name === 'AbortError') { + return { + success: false, + error_code: REQUEST_TIMEOUT, + message: 'Request timed-out' + } + } else { + return { + success: false, + error_code: NETWORK_ERROR, + message: 'Network error' + } + } + }).finally(() => { + clearTimeout(timeout) + }) + } + + /** + * renew the access token using refresh token + * 1. upon concurrent invocation of renewAccessToken(), the Promise created by the first invocation will be returned to subsequent invocations. + * 2. if local storage is enabled, this function keeps the in-memory and local storage state in-sync by calling initializeFromLocalStorage() + * 3. if refresh token is saved in local storage, and idToken or accessToken is missing, implementation will still try to renew the token. + * + * returns true if token was renewed successfully, false if token cannot be renewed and session must expire. + * in case of network errors, throws exception with the IDEA response payload as error. token renewal invocations must handle this error appropriately. + * this ensures session is not invalidated due to network errors, where the session state is still valid and user does not need to re-login. + */ + private renewAccessToken(): Promise { + + // before renewing, check if the current in-memory tokens are stale. + // this can only happen when using local storage, as another tab may renew the access token and update local storage. + if (this.localStorage != null) { + // this may need some sort of lock in future as there will be concurrent renewal scenario when multiple tabs are active. + // since local storage is not recommended for production, this is safe to ignore. + let accessToken = this.localStorage.getItem(KEY_ACCESS_TOKEN) + if (accessToken !== this.accessToken) { + this.initializeFromLocalStorage() + if (!this.isAccessTokenExpired()) { + this.logger.info('✓ refreshed stale access token') + return Promise.resolve(true) + } + } + } + + if (!this.isAccessTokenExpired()) { + return Promise.resolve(true) + } + + if (this.refreshToken == null) { + return Promise.resolve(false) + } + + this.logger.info('renewing access token ...') + + let username + if (this.claimsProvider == null) { + if (this.accessToken != null) { + let claims = JwtTokenUtils.parseJwtToken(this.accessToken) + username = claims.username + } else if (this.idToken != null) { + let claims = JwtTokenUtils.parseJwtToken(this.idToken) + username = claims['cognito:username'] + } else { + console.info('✗ failed to renew token.') + return Promise.resolve(false) + } + } else { + username = this.claimsProvider.getUsername() + } + + if (this.renewalInProgress != null) { + return this.renewalInProgress + } + + let authFlow = 'REFRESH_TOKEN_AUTH' + if (this.ssoAuth) { + authFlow = 'SSO_REFRESH_TOKEN_AUTH' + } + + let request = { + header: { + namespace: 'Auth.InitiateAuth', + request_id: Utils.getUUID() + }, + payload: { + auth_flow: authFlow, + username: username, + refresh_token: this.refreshToken + } + } + + const authEndpoint = `${this.authEndpoint}/${request.header.namespace}` + this.renewalInProgress = this._fetch(authEndpoint, { + method: 'POST', + headers: { + 'Content-Type': HEADER_CONTENT_TYPE_JSON + }, + body: JSON.stringify(request) + }).then(result => { + if (result.success && result.payload.auth) { + this.logger.info('✓ access token renewed successfully') + this.saveAuthResult(result.payload.auth, this.ssoAuth) + return true + } else { + if (result.error_code === NETWORK_TIMEOUT || result.error_code === NETWORK_ERROR || result.error_code === SERVER_ERROR) { + throw result + } else { + this.logger.info('✗ failed to renew token.') + return false + } + } + }).finally(() => { + this.renewalInProgress = null + }) + + return this.renewalInProgress + } + + invoke(url: string, request: any, isPublic: boolean = false): Promise { + + const invokeApi = () => { + let headers: any = { + 'Content-Type': HEADER_CONTENT_TYPE_JSON + } + let fetchOptions = { + method: 'POST', + headers: headers, + body: JSON.stringify(request) + } + if (!isPublic) { + headers['Authorization'] = `Bearer ${this.accessToken}` + } + return this._fetch(url, fetchOptions) + } + + if (isPublic) { + return invokeApi() + } + + // if access token is expired, try to renew using refresh token and then invoke api. + if (this.isAccessTokenExpired()) { + return this.renewAccessToken().then(success => { + if (success) { + return invokeApi() + } else { + return Promise.resolve(this.getAuthTokenExpiredError()) + } + }).catch(error => { + return error + }) + } + + return invokeApi() + } + + initiateAuth(request: any): Promise { + const authEndpoint = `${this.authEndpoint}/${request.header.namespace}` + return this._fetch(authEndpoint, { + method: 'POST', + headers: { + 'Content-Type': HEADER_CONTENT_TYPE_JSON + }, + body: JSON.stringify(request) + }).then(result => { + if (result.success && result.payload.auth) { + // cache auth result in-memory and return success response. + // all subsequent API invocations will be attached with the Authorization header. + this.logger.debug('✓ initiate auth successful') + const isSsoAuth = request.payload.auth_flow === 'SSO_AUTH' + this.saveAuthResult(result.payload.auth, isSsoAuth) + + return { + success: true, + payload: {} + } + } + + // return response message in all the other scenarios, where refresh and access token is not exposed to + // the main thread. + return result + }) + } + + fetch(url: string, options: any, isPublic: boolean = false): Promise { + if (!isPublic) { + if (typeof options.headers === 'undefined') { + options.headers = {} + } + options.headers['Authorization'] = `Bearer ${this.accessToken}` + } + return fetch(url, options) + } + + /** + * print debug info using in-memory state. + * helpful to debug service worker implementation and check the in-memory state via browser console. + */ + printDebugInfo() { + this.isLoggedIn().then(status => { + if (status) { + console.log('Is Logged In: ', 'Yes') + console.log('Access Token: ', this.accessToken) + console.log('Refresh Token: ', this.refreshToken) + console.log('Id Token: ', this.idToken) + console.log('Is SSO Auth: ', this.ssoAuth) + console.log('Claims: ', this.claimsProvider?.getClaims()) + } else { + console.log('Is Logged In: ', 'No') + } + }) + } +} diff --git a/source/idea/idea-cluster-manager/webapp/src/common/config-utils.ts b/source/idea/idea-cluster-manager/webapp/src/common/config-utils.ts new file mode 100644 index 00000000..c3ab0132 --- /dev/null +++ b/source/idea/idea-cluster-manager/webapp/src/common/config-utils.ts @@ -0,0 +1,121 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance + * with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES + * OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions + * and limitations under the License. + */ + +import Utils from "./utils"; +import dot from "dot-object"; + +class ConfigUtils { + + static getInternalAlbDnsName(clusterSettings: any) { + return dot.pick('load_balancers.internal_alb.load_balancer_dns_name', clusterSettings) + } + + static getInternalAlbCustomDnsName(clusterSettings: any) { + let customDnsName = dot.pick('load_balancers.internal_alb.certificates.custom_dns_name', clusterSettings) + if(Utils.isEmpty(customDnsName)) { + customDnsName = dot.pick('load_balancers.internal_alb.custom_dns_name', clusterSettings) + } + return customDnsName + } + + static getInternalAlbUrl(clusterSettings: any) { + let dnsName = this.getInternalAlbCustomDnsName(clusterSettings) + if(Utils.isEmpty(dnsName)) { + dnsName = this.getInternalAlbDnsName(clusterSettings) + } + return `https://${Utils.asString(dnsName)}` + } + + static getInternalAlbArn(clusterSettings: any) { + return dot.pick('load_balancers.internal_alb.load_balancer_arn', clusterSettings) + } + + static getInternalAlbCertificateSecretArn(clusterSettings: any) { + let secretArn = dot.pick('load_balancers.internal_alb.certificates.certificate_secret_arn', clusterSettings) + if (Utils.isEmpty(secretArn)) { + secretArn = dot.pick('load_balancers.internal_alb.certificate_secret_arn', clusterSettings) + } + return secretArn + } + + static getInternalAlbPrivateKeySecretArn(clusterSettings: any) { + let secretArn = dot.pick('load_balancers.internal_alb.certificates.private_key_secret_arn', clusterSettings) + if (Utils.isEmpty(secretArn)) { + secretArn = dot.pick('load_balancers.internal_alb.private_key_secret_arn', clusterSettings) + } + return secretArn + } + + static getInternalAlbAcmCertificateArn(clusterSettings: any) { + let certificateArn = dot.pick('load_balancers.internal_alb.certificates.acm_certificate_arn', clusterSettings) + if (Utils.isEmpty(certificateArn)) { + certificateArn = dot.pick('load_balancers.internal_alb.acm_certificate_arn', clusterSettings) + } + return certificateArn + } + + static getExternalAlbDnsName(clusterSettings: any) { + return dot.pick('load_balancers.external_alb.load_balancer_dns_name', clusterSettings) + } + + static getExternalAlbCustomDnsName(clusterSettings: any) { + let customDnsName = dot.pick('load_balancers.external_alb.certificates.custom_dns_name', clusterSettings) + if(Utils.isEmpty(customDnsName)) { + customDnsName = dot.pick('load_balancers.external_alb.custom_dns_name', clusterSettings) + } + return customDnsName + } + + static getExternalAlbUrl(clusterSettings: any) { + let dnsName = this.getExternalAlbCustomDnsName(clusterSettings) + if (Utils.isEmpty(dnsName)) { + dnsName = this.getExternalAlbDnsName(clusterSettings) + } + return `https://${dnsName}` + } + + static getExternalAlbArn(clusterSettings: any) { + return dot.pick('load_balancers.external_alb.load_balancer_arn', clusterSettings) + } + + static isExternalAlbPublic(clusterSettings: any): boolean { + return Utils.asBoolean(dot.pick('load_balancers.external_alb.public', clusterSettings)) + } + + static getExternalAlbCertificateSecretArn(clusterSettings: any) { + let secretArn = dot.pick('load_balancers.external_alb.certificates.certificate_secret_arn', clusterSettings) + if (Utils.isEmpty(secretArn)) { + secretArn = dot.pick('load_balancers.external_alb.certificate_secret_arn', clusterSettings) + } + return secretArn + } + + static getExternalAlbPrivateKeySecretArn(clusterSettings: any) { + let secretArn = dot.pick('load_balancers.external_alb.certificates.private_key_secret_arn', clusterSettings) + if (Utils.isEmpty(secretArn)) { + secretArn = dot.pick('load_balancers.external_alb.private_key_secret_arn', clusterSettings) + } + return secretArn + } + + static getExternalAlbAcmCertificateArn(clusterSettings: any) { + let certificateArn = dot.pick('load_balancers.external_alb.certificates.acm_certificate_arn', clusterSettings) + if (Utils.isEmpty(certificateArn)) { + certificateArn = dot.pick('load_balancers.external_alb.acm_certificate_arn', clusterSettings) + } + return certificateArn + } + +} + +export default ConfigUtils diff --git a/source/idea/idea-cluster-manager/webapp/src/common/constants.ts b/source/idea/idea-cluster-manager/webapp/src/common/constants.ts new file mode 100644 index 00000000..85713c2d --- /dev/null +++ b/source/idea/idea-cluster-manager/webapp/src/common/constants.ts @@ -0,0 +1,72 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance + * with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES + * OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions + * and limitations under the License. + */ + +export const Constants = { + MODULE_VIRTUAL_DESKTOP_CONTROLLER: 'virtual-desktop-controller', + MODULE_SCHEDULER: 'scheduler', + MODULE_DIRECTORY_SERVICE: 'directoryservice', + MODULE_IDENTITY_PROVIDER: 'identity-provider', + MODULE_SHARED_STORAGE: 'shared-storage', + MODULE_METRICS: 'metrics', + MODULE_BASTION_HOST: 'bastion-host', + MODULE_ANALYTICS: 'analytics', + MODULE_CLUSTER: 'cluster', + MODULE_CLUSTER_MANAGER: 'cluster-manager', + MODULE_GLOBAL_SETTINGS: 'global-settings', + + NODE_TYPE_APP: 'app', + NODE_TYPE_INFRA: 'infra', + + MODULE_TYPE_APP: 'app', + MODULE_TYPE_STACK: 'stack', + MODULE_TYPE_CONFIG: 'config', + + ADMIN_ZONE_LINK_TEXT: 'ADMIN ZONE', + + SHARED_STORAGE_PROVIDER_EFS: 'efs', + SHARED_STORAGE_PROVIDER_FSX_CACHE: 'fsx_cache', + SHARED_STORAGE_PROVIDER_FSX_LUSTRE: 'fsx_lustre', + SHARED_STORAGE_PROVIDER_FSX_NETAPP_ONTAP: 'fsx_netapp_ontap', + SHARED_STORAGE_PROVIDER_FSX_OPENZFS: 'fsx_openzfs', + SHARED_STORAGE_PROVIDER_FSX_WINDOWS_FILE_SERVER: 'fsx_windows_file_server', + + SPLIT_PANEL_I18N_STRINGS: { + preferencesTitle: "Split panel preferences", + preferencesPositionLabel: "Split panel position", + preferencesPositionDescription: + "Choose the default split panel position for the service.", + preferencesPositionSide: "Side", + preferencesPositionBottom: "Bottom", + preferencesConfirm: "Confirm", + preferencesCancel: "Cancel", + closeButtonAriaLabel: "Close panel", + openButtonAriaLabel: "Open panel", + resizeHandleAriaLabel: "Resize split panel" + }, + + ServiceWorker: { + SKIP_WAITING: 'SKIP_WAITING', + IDEA_AUTH_INIT: 'IDEA.Auth.InitializeAuth', + IDEA_AUTH_TOKEN_CLAIMS: 'IDEA.Auth.GetTokenClaims', + IDEA_AUTH_IS_LOGGED_IN: 'IDEA.Auth.IsLoggedIn', + IDEA_AUTH_LOGOUT: 'IDEA.Auth.Logout', + IDEA_AUTH_DEBUG: 'IDEA.Auth.Debug', + IDEA_AUTH_ACCESS_TOKEN: 'IDEA.Auth.GetAccessToken', + IDEA_API_INVOCATION: 'IDEA.InvokeApi', + IDEA_HTTP_FETCH: 'IDEA.HttpFetch' + } +} + +export const ErrorCodes = { + MODULE_NOT_FOUND: 'MODULE_NOT_FOUND' +} diff --git a/source/idea/idea-cluster-manager/webapp/src/common/error-codes.ts b/source/idea/idea-cluster-manager/webapp/src/common/error-codes.ts new file mode 100644 index 00000000..ddc5fcc6 --- /dev/null +++ b/source/idea/idea-cluster-manager/webapp/src/common/error-codes.ts @@ -0,0 +1,21 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance + * with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES + * OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions + * and limitations under the License. + */ + +export const UNAUTHORIZED_ACCESS = 'UNAUTHORIZED_ACCESS' +export const AUTH_TOKEN_EXPIRED = 'AUTH_TOKEN_EXPIRED' +export const AUTH_LOGIN_CHALLENGE = 'AUTH_LOGIN_CHALLENGE' +export const AUTH_PASSWORD_RESET_REQUIRED = 'AUTH_PASSWORD_RESET_REQUIRED' + +export const SERVER_ERROR = 'SERVER_ERROR' +export const NETWORK_ERROR = 'NETWORK_ERROR' +export const REQUEST_TIMEOUT = 'REQUEST_TIMEOUT' diff --git a/source/idea/idea-cluster-manager/webapp/src/common/exceptions.ts b/source/idea/idea-cluster-manager/webapp/src/common/exceptions.ts new file mode 100644 index 00000000..ba1391ef --- /dev/null +++ b/source/idea/idea-cluster-manager/webapp/src/common/exceptions.ts @@ -0,0 +1,37 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance + * with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES + * OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions + * and limitations under the License. + */ + +export interface IdeaExceptionProps { + errorCode: string + message?: string + payload?: any +} + +class IdeaException { + errorCode: string + message?: string + payload?: any + + constructor(props: IdeaExceptionProps) { + this.errorCode = props.errorCode + this.message = props.message + this.payload = props.payload + } + + toString(): string { + return `[${this.errorCode}] ${this.message}` + } + +} + +export default IdeaException diff --git a/source/idea/idea-cluster-manager/webapp/src/common/index.ts b/source/idea/idea-cluster-manager/webapp/src/common/index.ts new file mode 100644 index 00000000..69889a26 --- /dev/null +++ b/source/idea/idea-cluster-manager/webapp/src/common/index.ts @@ -0,0 +1,18 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance + * with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES + * OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions + * and limitations under the License. + */ + +import AppContext from "./app-context"; + +export { + AppContext +} diff --git a/source/idea/idea-cluster-manager/webapp/src/common/shared-storage-utils.ts b/source/idea/idea-cluster-manager/webapp/src/common/shared-storage-utils.ts new file mode 100644 index 00000000..642ac0d1 --- /dev/null +++ b/source/idea/idea-cluster-manager/webapp/src/common/shared-storage-utils.ts @@ -0,0 +1,240 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance + * with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES + * OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions + * and limitations under the License. + */ + +import dot from "dot-object"; +import Utils from "./utils"; +import {Constants} from "./constants"; +import {AppContext} from "./index"; + +export class SharedStorageFileSystem { + + name: string + sharedStorage: any + + constructor(name: string, sharedStorage: any) { + this.name = name + this.sharedStorage = sharedStorage + } + + getName(): string { + return this.name + } + + getTitle(): string { + let title = dot.pick('title', this.sharedStorage) + if (Utils.isEmpty(title)) { + // backward compat + const name = this.getName() + switch (name) { + case 'apps': + title = 'Applications' + break + case 'data': + title = 'Data (User Home Directories)' + break + default: + title = name + break + } + } + return title + } + + getProvider(): string { + return dot.pick('provider', this.sharedStorage) + } + + isEfs(): boolean { + return this.getProvider() === Constants.SHARED_STORAGE_PROVIDER_EFS + } + + isFsxCache(): boolean { + return this.getProvider() === Constants.SHARED_STORAGE_PROVIDER_FSX_CACHE + } + + isFsxLustre(): boolean { + return this.getProvider() === Constants.SHARED_STORAGE_PROVIDER_FSX_LUSTRE + } + + isFsxOpenZfs(): boolean { + return this.getProvider() === Constants.SHARED_STORAGE_PROVIDER_FSX_OPENZFS + } + + isFsxNetAppOntap(): boolean { + return this.getProvider() === Constants.SHARED_STORAGE_PROVIDER_FSX_NETAPP_ONTAP + } + + isFsxWindowsFileServer(): boolean { + return this.getProvider() === Constants.SHARED_STORAGE_PROVIDER_FSX_WINDOWS_FILE_SERVER + } + + isVolumeApplicable(): boolean { + return this.isFsxNetAppOntap() || this.isFsxOpenZfs() + } + + getProviderTitle(): string { + return Utils.getFileSystemProviderTitle(this.getProvider()) + } + + isExistingFileSystem(): boolean { + const provider = this.getProvider() + return Utils.asBoolean(dot.pick(`${provider}.use_existing_fs`, this.sharedStorage)) + } + + getMountTarget(): string { + if(this.isFsxWindowsFileServer()) { + return `${this.getMountDrive()}:` + } + if(this.isFsxNetAppOntap()) { + const securityStyle = this.getVolumeSecurityStyle() + switch (securityStyle) { + case 'MIXED': + return `${this.getMountDirectory()} | ${this.getMountDrive()}:` + case 'NTFS': + return `${this.getMountDrive()}:` + case 'UNIX': + return this.getMountDirectory() + } + } + return this.getMountDirectory() + } + + getMountDirectory(): string { + return dot.pick('mount_dir', this.sharedStorage) + } + + getMountDrive(): string { + return dot.pick('mount_drive', this.sharedStorage) + } + + hasMountDrive(): boolean { + return Utils.isNotEmpty(this.getMountDrive()) + } + + getMountOptions(): string { + const mount_options = dot.pick('mount_options', this.sharedStorage) + if (Utils.isNotEmpty(mount_options)) { + return mount_options + } + if (this.isEfs()) { + return 'nfs4 nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2,noresvport 0 0' + } else if (this.isFsxLustre()) { + return 'lustre defaults,noatime,flock,_netdev 0 0' + } else if (this.isFsxNetAppOntap()) { + return 'nfs4 nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2,noresvport 0 0' + } else if (this.isFsxOpenZfs()) { + return 'nfs4 nfsvers=4.1,rsize=1048576,wsize=1048576,timeo=600 0 0' + } + return '' + } + + getMountName(): string { + const provider = this.getProvider() + return dot.pick(`${provider}.mount_name`, this.sharedStorage) + } + + getLustreVersion(): string { + const provider = this.getProvider() + return dot.pick(`${provider}.version`, this.sharedStorage) + } + + getScope(): string[] { + const scope = dot.pick('scope', this.sharedStorage) + if (Utils.isEmpty(scope)) { + return ['cluster'] + } + return scope + } + + isScopeProjects(): boolean { + return this.getScope().includes('project') + } + + getProjects(): string[] { + return dot.pick('projects', this.sharedStorage) + } + + isScopeModule(): boolean { + return this.getScope().includes('module') + } + + getModules(): string[] { + return dot.pick('modules', this.sharedStorage) + } + + isScopeQueueProfile(): boolean { + return AppContext.get().getClusterSettingsService().isSchedulerDeployed() && this.getScope().includes('scheduler:queue-profile') + } + + getQueueProfiles(): string[] { + return dot.pick('queue_profiles', this.sharedStorage) + } + + getFileSystemId(): string { + const provider = this.getProvider() + return dot.pick(`${provider}.file_system_id`, this.sharedStorage) + } + + getFileSystemDns(): string { + const provider = this.getProvider() + return dot.pick(`${provider}.dns`, this.sharedStorage) + } + + getSvmId(): string { + const provider = this.getProvider() + return dot.pick(`${provider}.svm.svm_id`, this.sharedStorage) + } + + getSvmSmbDns(): string { + const provider = this.getProvider() + return dot.pick(`${provider}.svm.smb_dns`, this.sharedStorage) + } + + getSvmNfsDns(): string { + const provider = this.getProvider() + return dot.pick(`${provider}.svm.nfs_dns`, this.sharedStorage) + } + + getSvmManagementDns(): string { + const provider = this.getProvider() + return dot.pick(`${provider}.svm.management_dns`, this.sharedStorage) + } + + getSvmIscsiDns(): string { + const provider = this.getProvider() + return dot.pick(`${provider}.svm.iscsi_dns`, this.sharedStorage) + } + + getVolumeId(): string { + const provider = this.getProvider() + if (this.isFsxNetAppOntap()) { + return dot.pick(`${provider}.volume.volume_id`, this.sharedStorage) + } else { + return dot.pick(`${provider}.volume_id`, this.sharedStorage) + } + } + + getVolumePath(): string { + const provider = this.getProvider() + if (this.isFsxNetAppOntap()) { + return dot.pick(`${provider}.volume.volume_path`, this.sharedStorage) + } else { + return dot.pick(`${provider}.volume_path`, this.sharedStorage) + } + } + + getVolumeSecurityStyle(): string { + const provider = this.getProvider() + return dot.pick(`${provider}.volume.security_style`, this.sharedStorage) + } +} diff --git a/source/idea/idea-cluster-manager/webapp/src/common/token-utils.ts b/source/idea/idea-cluster-manager/webapp/src/common/token-utils.ts new file mode 100644 index 00000000..cd7a92c6 --- /dev/null +++ b/source/idea/idea-cluster-manager/webapp/src/common/token-utils.ts @@ -0,0 +1,137 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance + * with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES + * OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions + * and limitations under the License. + */ + +export class JwtTokenUtils { + + static parseJwtToken(token: string): any { + let base64Url = token.split('.')[1]; + let base64 = base64Url.replace(/-/g, '+').replace(/_/g, '/'); + let jsonPayload = decodeURIComponent(atob(base64).split('').map(function (c) { + return '%' + ('00' + c.charCodeAt(0).toString(16)).slice(-2); + }).join('')); + return JSON.parse(jsonPayload); + } + +} + +export class JwtTokenClaimsProvider { + + private readonly accessToken: any + private readonly idToken: any + + constructor(accessToken: string, idToken: string) { + this.accessToken = JwtTokenUtils.parseJwtToken(accessToken) + this.idToken = JwtTokenUtils.parseJwtToken(idToken) + } + + getUsername(): string { + return this.accessToken.username + } + + getCognitoUsername(): string { + return this.idToken['cognito:username'] + } + + getGroups(): string[] { + let groups = this.accessToken['cognito:groups'] + if(groups == null) { + return [] + } + return groups + } + + getIssuedAt(): number { + return this.accessToken.iat * 1000 + } + + getExpiresAt(): number { + return this.accessToken.exp * 1000 + } + + getAuthTime(): number { + return this.accessToken.auth_time * 1000 + } + + getScope(): string[] { + let scope = this.accessToken.scope + if(scope == null) { + return [] + } + return scope.split(' ') + } + + getEmail(): string { + return this.idToken.email + } + + getClusterName(): string { + return this.idToken['custom:cluster_name'] + } + + getAwsRegion(): string { + return this.idToken['custom:aws_region'] + } + + getPasswordLastSet(): number { + let passwordLastSet = this.idToken['custom:password_last_set'] + if(passwordLastSet == null) { + return -1 + } + return parseInt(passwordLastSet, 10) + } + + getPasswordMaxAge(): number { + let passwordLastSet = this.idToken['custom:password_max_age'] + if(passwordLastSet == null) { + return -1 + } + return parseInt(passwordLastSet, 10) + } + + getEmailVerified(): boolean { + return this.idToken['email_verified'] + } + + getClaims(): JwtTokenClaims { + return { + username: this.getUsername(), + groups: this.getGroups(), + issued_at: this.getIssuedAt(), + expires_at: this.getExpiresAt(), + auth_time: this.getAuthTime(), + scope: this.getScope(), + email: this.getEmail(), + cluster_name: this.getClusterName(), + aws_region: this.getAwsRegion(), + password_last_set: this.getPasswordLastSet(), + password_max_age: this.getPasswordMaxAge(), + email_verified: this.getEmailVerified() + } + } + +} + +export interface JwtTokenClaims { + username: string + groups: string[] + issued_at: number + expires_at: number + auth_time: number + scope: string[] + email: string + cluster_name: string + aws_region: string + password_last_set?: number + password_max_age?: number + email_verified: boolean +} diff --git a/source/idea/idea-cluster-manager/webapp/src/common/utils.ts b/source/idea/idea-cluster-manager/webapp/src/common/utils.ts new file mode 100644 index 00000000..cf4ee859 --- /dev/null +++ b/source/idea/idea-cluster-manager/webapp/src/common/utils.ts @@ -0,0 +1,919 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance + * with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES + * OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions + * and limitations under the License. + */ + +import { + SocaListingPayload, + SocaUserInputParamCondition, + SocaUserInputParamMetadata, + SocaMemory, SocaAmount, SocaDateRange, + SocaUserInputChoice, VirtualDesktopGPU, VirtualDesktopSchedule, VirtualDesktopScheduleType, User +} from "../client/data-model"; +import {IdeaFormFieldRegistry} from "../components/form-field"; +import {v4 as uuid} from "uuid" +import {DateRangePickerProps} from "@cloudscape-design/components"; +import dot from "dot-object"; +import moment from "moment"; +import IdeaException from "./exceptions"; +import {Constants} from "./constants"; + +const TRUE_VALUES = ['true', 'yes', 'y'] +const FALSE_VALUES = ['false', 'no', 'n'] + +class Utils { + + static getUUID(): string { + return uuid() + } + + static getRandomInt(min: number, max: number): number { + min = Math.ceil(min); + max = Math.floor(max); + // The maximum is exclusive and the minimum is inclusive + return Math.floor(Math.random() * (max - min) + min); + } + + static asString(val?: any, def: string = ''): string { + if (val == null) { + return def + } + if (typeof val === 'string') { + return val + } + return `${val}` + } + + static asStringArray(val?: any): string[] { + if (val == null) { + return [] + } + if (!Array.isArray(val)) { + return [Utils.asString(val)] + } + + const array: any[] = val + const result: string[] = [] + array.forEach((entry) => { + result.push(Utils.asString(entry)) + }) + + return result + } + + static asBooleanArray(val?: any): boolean[] { + if (val == null) { + return [] + } + if (!Array.isArray(val)) { + return [Utils.asBoolean(val)] + } + + const array: any[] = val + const result: boolean[] = [] + array.forEach((entry) => { + result.push(Utils.asBoolean(entry)) + }) + + return result + } + + static asNumberArray(val?: any): number[] { + if (val == null) { + return [] + } + if (!Array.isArray(val)) { + return [Utils.asNumber(val)] + } + + const array: any[] = val + const result: number[] = [] + array.forEach((entry) => { + result.push(Utils.asNumber(entry)) + }) + + return result + } + + static isArrayEqual(array1: string[], array2: string[]): boolean { + return array1.length === array2.length && array1.every((value, index) => value === array2[index]) + } + + static asBoolean(value?: any, def: boolean = false): boolean { + if (value == null) { + return def + } + if (typeof value === 'boolean') { + return value + } + const stringVal = Utils.asString(value) + const token = stringVal.trim().toLowerCase() + if (['true', 'yes', 'y'].find((val) => val === token)) { + return true + } else if (['false', 'no', 'n'].find((val) => val === token)) { + return false + } + return def + } + + static asNumber(value?: any, def: number = 0, decimal: boolean = false): number { + if (value == null) { + return def + } + if (typeof value === 'number') { + return value + } + if (typeof value === 'string') { + try { + if (decimal) { + return parseFloat(value) + } else { + return parseInt(value, 10) + } + } catch (error) { + return def + } + } + return def + } + + static isEmpty(value?: any): boolean { + if (value == null) { + return true + } + if (typeof value === 'string') { + return value.trim().length === 0 + } + if (Array.isArray(value)) { + return value.length === 0 + } + if (typeof value === 'object') { + return Object.keys(value).length === 0 + } + return false + } + + static isNotEmpty(value?: any): boolean { + return !Utils.isEmpty(value) + } + + static areAllEmpty(...values: any): boolean { + if (values == null) { + return true + } + for (let i = 0; i < values.length; i++) { + let value = values[i] + if (Utils.isNotEmpty(value)) { + return false + } + } + return true + } + + static isAnyEmpty(...values: any): boolean { + if (values == null) { + return true + } + for (let i = 0; i < values.length; i++) { + let value = values[i] + if (Utils.isEmpty(value)) { + return true + } + } + return false + } + + static isBoolean(value?: any): boolean { + if (value == null) { + return false + } + return typeof value === 'boolean' + } + + static isPositiveInteger(value?: any): boolean { + if (value == null) { + return false + } + if (typeof value === 'number') { + return value >= 0 + } + if (typeof value === 'string') { + const s = value.trim() + const num = Number(s) + return Number.isInteger(num) && num >= 0 + } + return false + } + + static isTrue(value?: any): boolean { + if (typeof value === 'boolean') { + return value + } + if (typeof value === 'string') { + return TRUE_VALUES.includes(value.trim().toLowerCase()) + } + return false + } + + static isFalse(value?: any): boolean { + if (typeof value === 'boolean') { + return !value + } + if (typeof value === 'string') { + return FALSE_VALUES.includes(value.trim().toLowerCase()) + } + return false + } + + static getFilterValueAsString(key: string, payload?: SocaListingPayload): string { + if (payload == null) { + return '' + } + if (payload.filters == null || payload.filters.length === 0) { + return '' + } + for(let i=0; i { + fieldRegistry.list().forEach(field => { + if (field.getParamName() === param) { + return field.getParamMeta() + } + }) + return null + } + + if (Utils.isNotEmpty(when.param)) { + const paramName = when.param! + const paramValue = dot.pick(paramName, values) + + if (Utils.isTrue(when.empty) && Utils.isEmpty(paramValue)) { + return true + } else if (Utils.isTrue(when.not_empty) && Utils.isNotEmpty(paramValue)) { + return true + } else if (Utils.isNotEmpty(when.eq) && paramValue === when.eq) { + return true + } else if (Utils.isNotEmpty(when.not_eq) && paramValue !== when.not_eq) { + return true + } else if (Utils.isNotEmpty(when.in) && when.in?.includes(paramValue)) { + return true + } else if (Utils.isNotEmpty(when.not_in) && !when.not_in?.includes(paramValue)) { + return true + } + + const param = getParamMeta(paramName) + if (param != null && (param.data_type === 'int' || param.data_type === 'float')) { + const paramFloatVal = parseFloat(paramValue + '') + if (Utils.isNotEmpty(when.gt) && paramFloatVal > parseFloat(when.gt + '')) { + return true + } else if (Utils.isNotEmpty(when.gte) && paramFloatVal >= parseFloat(when.gte + '')) { + return true + } else if (Utils.isNotEmpty(when.lt) && paramFloatVal < parseFloat(when.lt + '')) { + return true + } else if (Utils.isNotEmpty(when.lte) && paramFloatVal <= parseFloat(when.lte + '')) { + return true + } + } + + return false + } + + if (Utils.isNotEmpty(when.and)) { + const and = when.and! + for (let i = 0; i < and.length; i++) { + const condition = and[i] + if (!Utils.canShowFormField(fieldRegistry, values, condition)) { + return false + } + } + return true + } + + if (Utils.isNotEmpty(when.or)) { + const or = when.or! + for (let i = 0; i < or.length; i++) { + const condition = or[i] + if (Utils.canShowFormField(fieldRegistry, values, condition)) { + return true + } + } + return false + } + + return true + } + + static getFormattedGPUManufacturer(gpu?: VirtualDesktopGPU): string { + if (gpu == null || gpu === 'NO_GPU') { + return 'N/A' + } + return gpu + } + + static getFormattedMemory(memory?: SocaMemory): string { + if (memory == null) { + return '-' + } + return `${memory.value}${memory.unit}`.toUpperCase() + } + + static getFormattedAmount(amount?: SocaAmount): string { + if (amount == null) { + return '-' + } + return `${amount.amount.toFixed(2)} ${amount.unit}` + } + + static isArray(value: any): boolean { + if (value == null) { + return false + } + return Array.isArray(value) + } + + static convertToDateRange(value?: DateRangePickerProps.Value | null): SocaDateRange | null { + if (value == null) { + return null + } + if (value.type === 'absolute') { + return { + start: value.startDate, + end: value.endDate + } + } else { + const amount = value.amount + const unit = value.unit + const end = new Date() + let start = new Date() + switch (unit) { + case 'second': + start.setTime(start.getTime() - (amount * 1000)) + break + case 'minute': + start.setTime(start.getTime() - (amount * 60 * 1000)) + break + case 'hour': + start.setTime(start.getTime() - (amount * 60 * 60 * 1000)) + break + case 'day': + start.setTime(start.getTime() - (amount * 24 * 60 * 60 * 1000)) + break + case 'week': + start.setTime(start.getTime() - (amount * 7 * 24 * 60 * 60 * 1000)) + break + case 'month': + start.setTime(start.getTime() - (amount * 30 * 24 * 60 * 60 * 1000)) + break + case 'year': + start.setTime(start.getTime() - (amount * 365 * 24 * 60 * 60 * 1000)) + break + } + return { + start: start.toISOString(), + end: end.toISOString() + } + } + } + + static getDCVSessionTypes(): SocaUserInputChoice[] { + const options: SocaUserInputChoice[] = [] + options.push({ + title: 'Virtual', + value: 'VIRTUAL' + }) + + options.push({ + title: 'Console', + value: 'CONSOLE' + }) + return options + } + + static getSupportedGPUChoices(gpuList: string[]): SocaUserInputChoice[] { + const options: SocaUserInputChoice[] = [] + gpuList.forEach(gpu => { + switch (gpu) { + case 'NO_GPU': + options.push({ + title: 'N/A', + value: 'NO_GPU' + }) + break + case 'AMD': + options.push({ + title: 'AMD', + value: 'AMD' + }) + break + case 'NVIDIA': + options.push({ + title: 'NVIDA', + value: 'NVIDIA' + }) + } + }) + return options + } + + static generateUserSelectionChoices(users: User[]): SocaUserInputChoice[] { + let choices: SocaUserInputChoice[] = [] + users?.forEach(user => { + choices.push({ + title: user.username, + description: user.email, + value: user.username + }) + }) + return choices + } + + static compare_instance_types = (a: any, b: any): number => { + let a_InstanceFamily = a.InstanceType.split('.')[0] + let b_InstanceFamily = b.InstanceType.split('.')[0] + if (a_InstanceFamily === b_InstanceFamily) { + // same instance family - sort in reverse memory order + return b.MemoryInfo.SizeInMiB - a.MemoryInfo.SizeInMiB + } else { + // diff instance family - return alphabetical + return a_InstanceFamily.toLowerCase().localeCompare(b_InstanceFamily.toLowerCase(), undefined, {numeric: true}); + } + } + + static generateInstanceTypeListing(instanceTypes: any[] | undefined): SocaUserInputChoice[] { + if (instanceTypes === undefined) + return [{ + title: 'No instance types available.', + disabled: true + }] + + let bareMetalChoice: SocaUserInputChoice[] = [] + let regularChoice: SocaUserInputChoice[] = [] + instanceTypes.sort(this.compare_instance_types) + + instanceTypes.forEach((instanceType) => { + let memory + if (instanceType.MemoryInfo.SizeInMiB < 1024) { + memory = `${instanceType.MemoryInfo.SizeInMiB}MiB` + } else { + memory = `${Utils.asNumber(instanceType.MemoryInfo.SizeInMiB / 1024)}GiB` + } + + let architectures = '' + instanceType.ProcessorInfo.SupportedArchitectures.forEach((arch: string) => { + architectures += arch + " | " + }) + architectures = architectures.slice(0, -3) + + let gpus = '' + instanceType?.GpuInfo?.Gpus?.forEach((gpuInfo: any) => { + gpus += gpuInfo.Manufacturer + ' | ' + }) + + if (gpus.length > 0) { + gpus = gpus.slice(0, -3) + gpus = ', GPU: ' + gpus + } + + let description_value = `vCPUs: ${instanceType.VCpuInfo.DefaultVCpus}${gpus}, Memory: ${memory}, Arch: ${architectures}` + + let instance_choice = { + title: instanceType.InstanceType, + value: instanceType.InstanceType, + description: description_value, + disabled: false + } + + if (instanceType.BareMetal) { + bareMetalChoice.push(instance_choice) + } else { + regularChoice.push(instance_choice) + } + }) + + let instanceTypeChoices: SocaUserInputChoice[] = [] + if (regularChoice.length > 0) { + instanceTypeChoices.push({ + 'title': 'Regular', + 'options': regularChoice + }) + } + + if (bareMetalChoice.length > 0) { + instanceTypeChoices.push({ + 'title': 'Bare Metal', + 'options': bareMetalChoice + }) + } + + if (instanceTypeChoices.length === 0) { + instanceTypeChoices.push({ + title: 'No instance types available.', + disabled: true + }) + } + return instanceTypeChoices + } + + static getSupportedOSChoices(osList: string[]): SocaUserInputChoice[] { + const options: SocaUserInputChoice[] = [] + osList.forEach(os => { + options.push({ + title: Utils.getOsTitle(os), + value: os + }) + }) + return options + } + + static getScheduleTypeDisplay(schedule_type: VirtualDesktopScheduleType | undefined, working_hours_start: string | undefined, working_hours_end: string | undefined, start_time: string | undefined, end_time: string | undefined): string { + if (schedule_type === 'NO_SCHEDULE') { + return 'No Schedule' + } + + if (schedule_type === 'CUSTOM_SCHEDULE') { + return `Custom Schedule (${start_time} - ${end_time})` + } + + if (schedule_type === 'WORKING_HOURS') { + return `Working Hours (${working_hours_start} - ${working_hours_end})` + } + + if (schedule_type === 'START_ALL_DAY') { + return 'Start All Day' + } + + return 'Stop All Day' + } + + static getScheduleDisplay(schedule: VirtualDesktopSchedule | undefined, working_hours_start: string | undefined, working_hours_end: string | undefined): string { + return Utils.getScheduleTypeDisplay(schedule?.schedule_type, working_hours_start, working_hours_end, schedule?.start_up_time, schedule?.shut_down_time) + } + + static getOsTitle(name?: string): string { + switch (name) { + case 'amazonlinux2': + return 'Amazon Linux 2' + case 'centos7': + return 'CentOS 7' + case 'rhel7': + return 'RedHat Enterprise Linux 7' + case 'windows': + return 'Windows' + } + return 'Unknown' + } + + static getAwsConsoleParts(awsRegion: string): [string, string, string] { + let consolePrefix = awsRegion + let consoleSuffix = '.aws.amazonaws.com' + let s3Prefix = 's3.' + + // Determine the consoleParts by AWS region/partition + switch (true) { + case /^cn-.*/i.test(awsRegion): + consolePrefix = '' + consoleSuffix = '.amazonaws.cn' + s3Prefix = '' + break; + case /^us-gov-.*/i.test(awsRegion): + consolePrefix = '' + consoleSuffix = '.amazonaws-us-gov.com' + s3Prefix = '' + break; + // TODO Add support for additional partitions as needed + default: + consolePrefix = `${awsRegion}.` + consoleSuffix = '.aws.amazon.com' + s3Prefix = 's3.' + break; + } + return [consolePrefix, consoleSuffix, s3Prefix] + } + + static getAwsConsoleUrl(awsRegion: string): string { + let [consolePrefix, consoleSuffix, s3Prefix] = Utils.getAwsConsoleParts(awsRegion) + // consolePrefix is what proceeds the plainword 'console', including the trailing dot + // consoleSuffix is what follows the plainword 'console', including the leading dot + return `https://${consolePrefix}console${consoleSuffix}` + } + + static getEc2InstanceUrl(awsRegion: string, instanceId: string): string { + const consoleUrl = Utils.getAwsConsoleUrl(awsRegion) + return `${consoleUrl}/ec2/v2/home?region=${awsRegion}#InstanceDetails:instanceId=${instanceId}` + } + + static getASGUrl(awsRegion: string, asgName: string): string { + const consoleUrl = Utils.getAwsConsoleUrl(awsRegion) + return `${consoleUrl}/ec2/v2/home?region=${awsRegion}#AutoScalingGroupDetails:id=${asgName}` + } + + static getSecurityGroupUrl(awsRegion: string, groupId: string): string { + const consoleUrl = Utils.getAwsConsoleUrl(awsRegion) + return `${consoleUrl}/ec2/v2/home?region=${awsRegion}#SecurityGroup:groupId=${groupId}` + } + + static getSessionManagerConnectionUrl(awsRegion: string, instanceId: string): string { + const consoleUrl = Utils.getAwsConsoleUrl(awsRegion) + return `${consoleUrl}/systems-manager/session-manager/${instanceId}?region=${awsRegion}` + } + + static getCognitoUserPoolUrl(awsRegion: string, userPoolId: string): string { + const consoleUrl = Utils.getAwsConsoleUrl(awsRegion) + return `${consoleUrl}/cognito/v2/idp/user-pools/${userPoolId}/users?region=${awsRegion}` + } + + static getS3BucketUrl(awsRegion: string, bucketName: string): string { + // getS3BucketUrl cannot use getAwsConsoleUrl() as it needs the s3Prefix + let [consolePrefix, consoleSuffix, s3Prefix] = Utils.getAwsConsoleParts(awsRegion) + return `https://${s3Prefix}console${consoleSuffix}/s3/buckets/${bucketName}?region=${awsRegion}` + } + + static copyToClipBoard(text: string): Promise { + if (!navigator.clipboard) { + return Promise.resolve(false) + } + return navigator.clipboard.writeText(text).then(() => { + return true + }).catch(error => { + console.error(error) + return false + }) + } + + static getDayOfWeek(): string | null { + const day = new Date().getDay() + switch (day) { + case 0: + return 'sunday' + case 1: + return 'monday' + case 2: + return 'tuesday' + case 3: + return 'wednesday' + case 4: + return 'thursday' + case 5: + return 'friday' + case 6: + return 'saturday' + } + return null + } + + static getDirectoryServiceTitle(provider: string): string { + switch (provider) { + case 'openldap': + return 'OpenLDAP' + case 'aws_managed_activedirectory': + return 'AWS Managed Microsoft AD' + case 'activedirectory': + return 'Microsoft AD (Self-Hosted or On-Prem)' + } + return 'Unknown' + } + + static getDaysBetween(date1: Date, date2: Date): number { + const m1 = moment(date1) + const m2 = moment(date2) + return m1.diff(m2, 'days') + } + + static getUserGroupName(moduleId: string) { + return `${moduleId}-users-module-group` + } + + static getAdministratorGroup(moduleId: string) { + return `${moduleId}-administrators-module-group` + } + + static getModuleId(moduleName: string): string { + const modules: any = window.idea.app.modules + let moduleId: string = '' + modules.forEach((module: any) => { + if (moduleName === module.name) { + moduleId = module.module_id + return true + } + }) + return moduleId + } + + static getApiContextPath(moduleName: string): string { + const modules: any = window.idea.app.modules + let apiContextPath: string = '' + modules.forEach((module: any) => { + if (moduleName === module.name) { + apiContextPath = module.api_context_path + return true + } + }) + + // default api context paths for local dev as index.html page is not rendered server side + if (Utils.isEmpty(apiContextPath)) { + switch (moduleName) { + case Constants.MODULE_CLUSTER_MANAGER: + apiContextPath = '/cluster-manager/api/v1' + break + case Constants.MODULE_SCHEDULER: + apiContextPath = '/scheduler/api/v1' + break + case Constants.MODULE_VIRTUAL_DESKTOP_CONTROLLER: + apiContextPath = '/vdc/api/v1' + break + } + } + + if (Utils.isEmpty(apiContextPath)) { + throw new IdeaException({ + errorCode: 'MODULE_NOT_FOUND', + message: `Module not found for name: ${moduleName}` + }) + } + return apiContextPath + } + + static getFileSystemProviderTitle(provider: string): string { + switch (provider) { + case Constants.SHARED_STORAGE_PROVIDER_EFS: + return 'Amazon EFS' + case Constants.SHARED_STORAGE_PROVIDER_FSX_CACHE: + return 'Amazon File Cache' + case Constants.SHARED_STORAGE_PROVIDER_FSX_LUSTRE: + return 'Amazon FSx for Lustre' + case Constants.SHARED_STORAGE_PROVIDER_FSX_NETAPP_ONTAP: + return 'Amazon FSx for NetApp ONTAP ' + case Constants.SHARED_STORAGE_PROVIDER_FSX_OPENZFS: + return 'Amazon FSx for OpenZFS' + case Constants.SHARED_STORAGE_PROVIDER_FSX_WINDOWS_FILE_SERVER: + return 'Amazon FSx for Windows File Server' + default: + return 'Unknown' + } + } + + /** + * try to find Platform based on UserAgent data. + * approximation based on information available. cannot be guaranteed. + * source: https://stackoverflow.com/questions/38241480/detect-macos-ios-windows-android-and-linux-os-with-js + */ + static getPlatform(): string { + const navigator: any = window.navigator + let userAgent = navigator.userAgent + let platform = navigator.platform + const OSX_TOKENS = ['Macintosh', 'MacIntel', 'MacPPC', 'Mac68K'] + const WINDOWS_TOKENS = ['Win32', 'Win64', 'Windows', 'WinCE'] + const IOS_TOKENS = ['iPhone', 'iPad', 'iPod'] + + let os = '' + + if (OSX_TOKENS.indexOf(platform) !== -1) { + os = 'osx'; + } else if (IOS_TOKENS.indexOf(platform) !== -1) { + os = 'ios'; + } else if (WINDOWS_TOKENS.indexOf(platform) !== -1) { + os = 'windows'; + } else if (/Android/.test(userAgent)) { + os = 'android'; + } else if (/Linux/.test(platform)) { + os = 'linux'; + } + + return os; + } + + static openNewTab(url: string) { + const element = document.createElement('a') + element.setAttribute('href', url) + element.setAttribute('target', '_blank') + element.style.display = 'none' + document.body.appendChild(element) + element.click() + document.body.removeChild(element) + } + + static getBanner(version: string): string { + return ` + '####:'########::'########::::'###:::: + . ##:: ##.... ##: ##.....::::'## ##::: + : ##:: ##:::: ##: ######:::'##:::. ##: + : ##:: ##:::: ##: ##...:::: #########: + '####: ########:: ########: ##:::: ##: + + Integrated Digital Engineering on AWS + Version ${version} + +` + } + + static getDefaultModuleSettings() { + return [ + { + 'deployment_priority': 3, + 'module_id': 'analytics', + 'name': 'analytics', + 'title': 'Analytics', + 'type': 'stack' + }, + { + 'deployment_priority': 7, + 'module_id': 'bastion-host', + 'name': 'bastion-host', + 'title': 'Bastion Host', + 'type': 'stack', + }, + { + 'api_context_path': "/cluster-manager/api/v1", + 'deployment_priority': 5, + 'module_id': 'cluster-manager', + 'name': 'cluster-manager', + 'title': 'Cluster Manager', + 'type': 'app' + }, + { + 'deployment_priority': 2, + 'module_id': 'cluster', + 'name': 'cluster', + 'title': 'Cluster', + 'type': 'stack' + }, + { + 'deployment_priority': 3, + 'module_id': 'directoryservice', + 'name': 'directoryservice', + 'title': 'Directory Service', + 'type': 'stack' + }, + { + 'deployment_priority': 3, + 'module_id': 'identity-provider', + 'name': 'identity-provider', + 'title': 'Identity Provider', + 'type': 'stack' + }, + { + 'deployment_priority': 3, + 'module_id': 'metrics', + 'name': 'metrics', + 'title': 'Metrics & Monitoring', + 'type': 'stack' + }, + { + 'api_context_path': "/scheduler/api/v1", + 'deployment_priority': 6, + 'module_id': 'scheduler', + 'name': 'scheduler', + 'title': 'Scale-Out Computing', + 'type': 'app' + }, + { + 'deployment_priority': 4, + 'module_id': 'shared-storage', + 'name': 'shared-storage', + 'title': 'Shared Storage', + 'type': 'stack' + }, + { + 'api_context_path': "/vdc/api/v1", + 'deployment_priority': 6, + 'module_id': 'vdc', + 'name': 'virtual-desktop-controller', + 'title': 'eVDI', + 'type': 'app' + } + ] + } + + static hideLoadingAnimation() { + document.getElementById('app-loading')!.style.display = 'none' + } + + static isSsoEnabled(): boolean { + return typeof window.idea.app.sso !== 'undefined' && window.idea.app.sso + } +} + +export default Utils diff --git a/source/idea/idea-cluster-manager/webapp/src/components/app-layout/app-layout.tsx b/source/idea/idea-cluster-manager/webapp/src/components/app-layout/app-layout.tsx new file mode 100644 index 00000000..f9d73944 --- /dev/null +++ b/source/idea/idea-cluster-manager/webapp/src/components/app-layout/app-layout.tsx @@ -0,0 +1,150 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance + * with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES + * OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions + * and limitations under the License. + */ + +import React, {Component} from "react"; +import {Box, BreadcrumbGroup, Flashbar, SpaceBetween} from "@cloudscape-design/components"; +import {AppContext} from "../../common"; +import AppLayout from "@cloudscape-design/components/app-layout"; +import {NonCancelableEventHandler} from "@cloudscape-design/components/internal/events"; +import {AppLayoutProps} from "@cloudscape-design/components/app-layout/interfaces"; +import {BreadcrumbGroupProps} from "@cloudscape-design/components/breadcrumb-group/interfaces"; +import {OnFlashbarChangeEvent, OnPageChangeEvent, OnToolsChangeEvent} from "../../App"; +import {FlashbarProps} from "@cloudscape-design/components/flashbar/interfaces"; +import {withRouter} from "../../navigation/navigation-utils"; +import IdeaSideNavigation, {IdeaSideNavigationProps} from "../side-navigation"; +import IdeaNavbar from "../navbar"; +import Utils from "../../common/utils"; + +export interface IdeaAppLayoutProps extends IdeaSideNavigationProps { + ideaPageId: string + contentType?: AppLayoutProps.ContentType + header?: React.ReactNode + content?: React.ReactNode + disableContentHeaderOverlap?: boolean + navigation?: React.ReactNode + breadcrumbItems?: BreadcrumbGroupProps.Item[] + splitPanel?: React.ReactNode + toolsOpen: boolean + tools: React.ReactNode + onToolsChange: (event: OnToolsChangeEvent) => void + onPageChange: (event: OnPageChangeEvent) => void + splitPanelSize?: number + splitPanelOpen?: boolean; + splitPanelPreferences?: AppLayoutProps.SplitPanelPreferences; + onSplitPanelResize?: NonCancelableEventHandler; + onSplitPanelToggle?: NonCancelableEventHandler; + onSplitPanelPreferencesChange?: NonCancelableEventHandler; + onFlashbarChange: (event: OnFlashbarChangeEvent) => void + flashbarItems: FlashbarProps.MessageDefinition[] + sideNavActivePath?: string +} + +export interface IdeaAppLayoutState { +} + +class IdeaAppLayout extends Component { + + componentDidMount() { + + Utils.hideLoadingAnimation() + + this.props.onPageChange({ + pageId: this.props.ideaPageId + }) + } + + buildBreadCrumbs() { + let items = this.props.breadcrumbItems + if (!items) { + return null + } + return + } + + buildNotifications() { + return + } + + buildFooter() { + return

+ } + + render() { + return ( +
+
+ +
+ + } + contentHeader={this.props.header} + breadcrumbs={this.buildBreadCrumbs()} + stickyNotifications={true} + notifications={this.buildNotifications()} + disableContentHeaderOverlap={(typeof this.props.disableContentHeaderOverlap !== 'undefined') ? this.props.disableContentHeaderOverlap : false} + content={ +
+
+ {this.props.content} +
+
+ } + contentType={(this.props.contentType) ? this.props.contentType : 'table'} + tools={this.props.tools} + toolsOpen={this.props.toolsOpen} + onToolsChange={(event) => { + if (this.props.onToolsChange) { + this.props.onToolsChange({ + open: event.detail.open, + pageId: this.props.ideaPageId + }) + } + }} + splitPanel={this.props.splitPanel} + splitPanelSize={this.props.splitPanelSize} + splitPanelOpen={this.props.splitPanelOpen} + splitPanelPreferences={this.props.splitPanelPreferences} + onSplitPanelResize={this.props.onSplitPanelResize} + onSplitPanelToggle={this.props.onSplitPanelToggle} + onSplitPanelPreferencesChange={this.props.onSplitPanelPreferencesChange} + /> +
+ ) + } +} + +export default withRouter(IdeaAppLayout) diff --git a/source/idea/idea-cluster-manager/webapp/src/components/app-layout/index.ts b/source/idea/idea-cluster-manager/webapp/src/components/app-layout/index.ts new file mode 100644 index 00000000..c32e64ba --- /dev/null +++ b/source/idea/idea-cluster-manager/webapp/src/components/app-layout/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance + * with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES + * OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions + * and limitations under the License. + */ + +import IdeaAppLayout, {IdeaAppLayoutProps} from "./app-layout"; + +export default IdeaAppLayout +export type {IdeaAppLayoutProps} diff --git a/source/idea/idea-cluster-manager/webapp/src/components/charts/pie-or-donut-chart.tsx b/source/idea/idea-cluster-manager/webapp/src/components/charts/pie-or-donut-chart.tsx new file mode 100644 index 00000000..b8b9147b --- /dev/null +++ b/source/idea/idea-cluster-manager/webapp/src/components/charts/pie-or-donut-chart.tsx @@ -0,0 +1,112 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance + * with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES + * OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions + * and limitations under the License. + */ + + +import {Container, Header, PieChart, PieChartProps, Select, SelectProps} from "@cloudscape-design/components"; +import React, {Component, RefObject} from "react"; + +export interface PieOrDonutChartProps extends PieChartProps { + headerDescription: string + headerText: string + enableSelection: boolean + defaultChartMode: 'piechart' | 'donutchart' +} + +interface PieOrDonutChartState { + chartMode: 'piechart' | 'donutchart' +} + +class PieOrDonutChart extends Component { + + chartSelector: RefObject + + constructor(props: PieOrDonutChartProps) { + super(props); + this.chartSelector = React.createRef() + this.state = { + chartMode: this.props.defaultChartMode + } + } + + buildHeaderActions() { + let donutSelectionOption = {label: "Donut", value: "donut"} + let pieSelectionOption = {label: "Pie", value: "pie"} + let selectionOption = donutSelectionOption + if (this.state.chartMode === 'piechart') { + selectionOption = pieSelectionOption + } + return ( + this.props.enableSelection && + { + const choices = this.state.choices + choices.forEach((choice) => { + if (item.value === choice.value) { + choice.title = event.detail.value + } + }) + this.setChoices(choices) + }}/> + ) + }, + { + label: 'Value', + control: (item: SocaUserInputChoice) => ( + { + const choices = this.state.choices + choices.forEach((choice) => { + if (item.title === choice.title) { + choice.value = event.detail.value + } + }) + this.setChoices(choices) + }}/> + ) + } + ]} + /> + } + + +
+

Preview

+
+ {this.state.showPreview && } +
+
+ + + + + + + + + + + } + {!this.state.edit && +
+ {this.state.debugMode &&

{this.props.id}

} + + + +
+ } + + + {!this.state.edit && } + {this.state.edit && + } + + + + + + ) + } +} + +class SocaFormBuilder extends Component { + + fields: { + [k: string]: IdeaFormBuilderField + } + + previewForm: RefObject + + constructor(props: IdeaFormBuilderProps) { + super(props); + this.fields = {} + this.previewForm = React.createRef() + this.state = { + metadata: [], + output: [], + ace: undefined, + preferences: undefined, + content: '', + language: 'json', + values: {} + } + } + + componentDidMount() { + import('ace-builds').then(ace => { + import('ace-builds/webpack-resolver').then(() => { + ace.config.set('useStrictCSP', true) + ace.config.set('loadWorkerFromBlob', false) + this.setState({ + ace: ace + }) + }) + }) + this.props.params.forEach((param) => { + this.addField(param) + }) + } + + getExampleParam(): SocaUserInputParamMetadata { + return { + name: `example_name_${Utils.getRandomInt(10000, 99999)}`, + title: 'Example Title', + description: 'Example Description', + help_text: 'Example Help Text', + data_type: 'str', + param_type: 'text', + validate: { + required: true + } + } + } + + addField(param?: SocaUserInputParamMetadata, position?: number) { + const metadata = this.state.metadata + const entry = { + id: Utils.getUUID(), + param: (param) ? param : this.getExampleParam() + } + if (position == null || position < 0) { + metadata.push(entry) + } else { + metadata.splice(position, 0, entry) + } + this.setState({ + metadata: metadata + }) + } + + getField(fieldId: string): IdeaFormBuilderField | null { + if (fieldId in this.fields) { + return this.fields[fieldId] + } + return null + } + + setFields(params: SocaUserInputParamMetadata[]) { + const metadata: IdeaFormBuilderFieldProps[] = [] + params.forEach(param => metadata.push({ + id: Utils.getUUID(), + param: param + })) + this.setState({ + metadata: metadata + }) + } + + moveField(fieldId: string, oldIndex: number, newIndex: number) { + const formField = this.getField(fieldId) + if (formField == null) { + return + } + const fields: any[] = this.state.metadata + if (newIndex >= fields.length) { + let k = newIndex - fields.length + 1; + while (k--) { + fields.push(null); + } + } + const fieldMetadata = this.state.metadata[oldIndex] + fieldMetadata.param = formField.build() + fields.splice(newIndex, 0, fields.splice(oldIndex, 1)[0]); + this.setState({ + metadata: fields + }) + } + + getFieldIndex(fieldId: string): number { + let found = -1 + this.state.metadata.forEach((entry, index) => { + if (entry.id === fieldId) { + found = index + return false + } + }) + if (found >= 0) { + return found + } + return -1 + } + + buildFormBuilder() { + return ( + { + if (event.reason === 'DROP') { + const sourceIndex = event.source.index + const destIndex = event.destination!.index + if (sourceIndex === destIndex) { + return + } + this.moveField(event.draggableId, sourceIndex, destIndex) + } + }}> + + {(provided: any) => ( +
+ { + this.state.metadata.map((entry, index) => { + return ( + + { + (provided: any) => ( +
+ { + const metadata = this.state.metadata + metadata.forEach((entry) => { + if (field.getId() === entry.id) { + entry.param = field.getParam() + } + }) + this.setState({ + metadata: metadata + }) + this.fields[field.getId()] = field + }} + onDelete={(field) => { + let found: number = -1 + this.state.metadata.forEach((entry, index) => { + if (entry.id === field.getId()) { + found = index + } + }) + const metadata = this.state.metadata + if (found >= 0) { + metadata.splice(found, 1) + } + this.setState({ + metadata: metadata + }) + delete this.fields[field.getId()] + }} + onUpdate={(field) => { + const metadata = this.state.metadata + metadata.forEach((entry) => { + if (field.getId() === entry.id) { + entry.param = field.getParam() + } + }) + this.setState({ + metadata: metadata + }) + }} + onCopy={(field) => { + const param = field.getParam() + this.addField({ + ...param, + name: `${param.name}_${Utils.getRandomInt(10000, 99999)}` + }, this.getFieldIndex(field.getId()) + 1) + }} + /> +
+ ) + } + + ) + }) + } + {provided.placeholder} +
+ )} + + + ) + } + + getFormParams() { + if (this.state.metadata.length === 0) { + return [] + } + const params: any[] = [] + this.state.metadata.forEach((entry) => { + params.push(entry.param) + }) + return params + } + + buildJsonEditor() { + return this.setState({ + preferences: e.detail + })} + onChange={(e) => { + this.setState({ + content: e.detail.value + }) + try { + this.setFields(JSON.parse(e.detail.value)) + } catch (e) { + console.error(e) + } + }} + loading={false} + i18nStrings={{ + loadingState: "Loading code editor", + errorState: + "There was an error loading the code editor.", + errorStateRecovery: "Retry", + editorGroupAriaLabel: "Code editor", + statusBarGroupAriaLabel: "Status bar", + cursorPosition: (row, column) => + `Ln ${row}, Col ${column}`, + errorsTab: "Errors", + warningsTab: "Warnings", + preferencesButtonAriaLabel: "Preferences", + paneCloseButtonAriaLabel: "Close", + preferencesModalHeader: "Preferences", + preferencesModalCancel: "Cancel", + preferencesModalConfirm: "Confirm", + preferencesModalWrapLines: "Wrap lines", + preferencesModalTheme: "Theme", + preferencesModalLightThemes: "Light themes", + preferencesModalDarkThemes: "Dark themes" + }} + /> + } + + buildInfoSection() { + if (this.props.infoSection) { + return this.props.infoSection + } else { + return
+

Design your form using Form Builder. You can preview the generated form using + the Preview Form tab.

+
+ } + } + + render() { + return ( +
+ + {this.buildInfoSection()} +
+ + + + + + +
+ + + { + if (event.detail.activeTabId === 'json-content') { + this.setState({ + content: JSON.stringify(this.getFormParams(), null, 4) + }) + } + }} + tabs={[ + { + id: 'preview-form', + label: 'Preview Form', + content: ( + + + { + if (!this.previewForm.current!.validate()) { + return + } + this.setState({ + values: this.previewForm.current!.getValues() + }) + }} + /> + + +

Click Check Form Parameters to verify the generated values below:

+
+ 
{JSON.stringify(this.state.values, null, 4)}
+
+ + + ) + }, + { + id: 'build-form', + label: 'Form Builder', + content: (this.buildFormBuilder()) + }, + { + id: 'json-content', + label: 'Form Builder (Advanced Mode using JSON)', + content: (this.buildJsonEditor()) + } + ]}/> +
+ ) + } +} + +export default SocaFormBuilder diff --git a/source/idea/idea-cluster-manager/webapp/src/components/form-builder/index.ts b/source/idea/idea-cluster-manager/webapp/src/components/form-builder/index.ts new file mode 100644 index 00000000..d9af966c --- /dev/null +++ b/source/idea/idea-cluster-manager/webapp/src/components/form-builder/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance + * with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES + * OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions + * and limitations under the License. + */ + +import IdeaFormBuilder, {IdeaFormBuilderProps} from "./form-builder"; + +export default IdeaFormBuilder +export type {IdeaFormBuilderProps} diff --git a/source/idea/idea-cluster-manager/webapp/src/components/form-field/form-field.tsx b/source/idea/idea-cluster-manager/webapp/src/components/form-field/form-field.tsx new file mode 100644 index 00000000..d92f42b0 --- /dev/null +++ b/source/idea/idea-cluster-manager/webapp/src/components/form-field/form-field.tsx @@ -0,0 +1,1715 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance + * with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES + * OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions + * and limitations under the License. + */ + +import React, {Component} from "react"; +import { + GetParamChoicesRequest, GetParamChoicesResult, + GetParamDefaultRequest, GetParamDefaultResult, GetParamsRequest, GetParamsResult, SetParamRequest, SetParamResult, + SocaUserInputGroupMetadata, + SocaUserInputParamMetadata, + SocaUserInputSectionMetadata +} from "../../client/data-model"; + +import Utils from '../../common/utils' +import Input, {InputProps} from "@cloudscape-design/components/input"; +import FormField, {FormFieldProps} from "@cloudscape-design/components/form-field"; +import { + Autosuggest, + Button, + ColumnLayout, DatePicker, Grid, HelpPanel, Link, + Multiselect, + RadioGroup, + Select, + SelectProps, SpaceBetween, + Textarea, + Toggle +} from "@cloudscape-design/components"; +import {BaseKeyDetail} from "@cloudscape-design/components/internal/events"; +import {faAdd, faRemove} from "@fortawesome/free-solid-svg-icons"; +import {FontAwesomeIcon} from "@fortawesome/react-fontawesome"; +import ReactMarkdown from "react-markdown"; + +export interface IdeaFormFieldSyncAPI { + getParamDefault(req: GetParamDefaultRequest): Promise + + setParam(req: SetParamRequest): Promise + + getParamChoices(req: GetParamChoicesRequest): Promise + + getParams(req: GetParamsRequest): Promise +} + +export interface IdeaFormFieldCustomActionProvider { + onCustomActionClick(customTypeName: string, formField: IdeaFormField, onSubmit: (value: any) => void): void + + getCustomActionLabel(customTypeName: string): string | null +} + + +export interface IdeaFormFieldProps { + module: string + param: SocaUserInputParamMetadata + section?: SocaUserInputSectionMetadata + group?: SocaUserInputGroupMetadata + onLifecycleEvent?: IdeaFormFieldLifecycleEventHandler + onStateChange?: IdeaFormFieldStateChangeEventHandler + visible?: boolean + updateTools?: any + syncApi?: IdeaFormFieldSyncAPI + value?: any + onKeyEnter?: IdeaFormFieldOnKeyEnterEventHandler + onFetchOptions?: (req: GetParamChoicesRequest) => Promise + customActionProvider?: IdeaFormFieldCustomActionProvider + stretch?: boolean +} + +export interface IdeaFormFieldState { + value?: any + value2?: any + + default?: any + + errorCode?: string | null + errorMessage?: string | null + + selectedOption?: any + selectedOptions?: any + options: any[] + + dynamicOptions: boolean + dynamicOptionsLoading: boolean + + defaultValueLoading: boolean + + visibility: boolean + + disabled: boolean + + stringVal(): string + + stringVal2(): string + + booleanVal(): boolean + + numberVal(): number + + stringArrayVal(): string[] + + booleanArrayVal(): boolean[] + + numberArrayVal(): number[] + + amountVal(): string + + memoryVal(): string + +} + +export interface IdeaFormFieldStateChangeEvent { + param: SocaUserInputParamMetadata, + value?: any | any[] + errorCode?: string | null + errorMessage?: string | null + refresh?: boolean, + ref: IdeaFormField +} + +export interface IdeaFormFieldLifecycleEvent { + type: LifecycleEventType + ref: IdeaFormField +} + +export interface IdeaFormFieldKeyEnterEvent { + ref: IdeaFormField +} + +type LifecycleEventType = 'mounted' | 'unmounted' +export type IdeaFormFieldLifecycleEventHandler = (event: IdeaFormFieldLifecycleEvent) => void; +export type IdeaFormFieldStateChangeEventHandler = (event: IdeaFormFieldStateChangeEvent) => void; +export type IdeaFormFieldOnKeyEnterEventHandler = (event: IdeaFormFieldKeyEnterEvent) => void; + +export interface IdeaFormFieldRegistryEntry { + field: IdeaFormField | null + lastKnownState: IdeaFormFieldState | null +} + +export class IdeaFormFieldRegistry { + fields: { + [k: string]: IdeaFormFieldRegistryEntry; + } + + constructor() { + this.fields = {} + } + + add(field: IdeaFormField) { + this.fields[field.getParamName()] = { + field: field, + lastKnownState: field.getState() + } + } + + delete(param: string) { + if (param in this.fields) { + this.fields[param].field = null + } + } + + getFormField(param: string): IdeaFormField | null { + if (!(param in this.fields)) { + return null + } + const entry = this.fields[param] + return entry.field + } + + getLastKnownState(param: string): IdeaFormFieldState | null { + if (!(param in this.fields)) { + return null + } + const entry = this.fields[param] + return entry.lastKnownState + } + + list(): IdeaFormField[] { + const fields: IdeaFormField[] = [] + for (const param in this.fields) { + const field = this.getFormField(param) + if (field == null) { + continue + } + fields.push(field) + } + return fields + } +} + + +class IdeaFormField extends Component { + + constructor(props: IdeaFormFieldProps) { + super(props) + this.state = { + + value: (this.props.value) ? this.props.value : this.props.param.default, + value2: undefined, + default: this.props.param.default, + + options: [], + selectedOption: {}, + selectedOptions: [], + + dynamicOptions: false, + dynamicOptionsLoading: false, + + defaultValueLoading: false, + + errorCode: null, + errorMessage: null, + + visibility: true, + disabled: (this.props.param.readonly) ? this.props.param.readonly : false, + + stringVal(): string { + if (Utils.isNotEmpty(this.value)) { + return Utils.asString(this.value) + } + return '' + }, + stringVal2(): string { + if (this.value2 != null) { + return Utils.asString(this.value2) + } + return '' + }, + stringArrayVal(): string[] { + if (this.value != null) { + return Utils.asStringArray(this.value) + } + return [] + }, + booleanVal(): boolean { + if (this.value != null) { + return Utils.asBoolean(this.value) + } + return false + }, + booleanArrayVal(): boolean[] { + if (this.value != null) { + return Utils.asBooleanArray(this.value) + } + return [] + }, + numberVal(decimal: boolean = false): number { + if (this.value != null) { + return Utils.asNumber(this.value, 0, decimal) + } + return 0 + }, + numberArrayVal(): number[] { + if (this.value != null) { + return Utils.asNumberArray(this.value) + } + return [] + }, + amountVal(): string { + if (typeof this.value === 'object') { + return Utils.asString(this.value.amount) + } + return '0.00' + }, + memoryVal(): string { + if (typeof this.value === 'object') { + return Utils.asString(this.value.value) + } + return '0' + } + } + } + + getParamName(): string { + return this.props.param.name! + } + + getParamMeta(): SocaUserInputParamMetadata { + return this.props.param + } + + isAutoComplete(): boolean { + return this.props.param.param_type === 'autocomplete' + } + + getState(): IdeaFormFieldState { + // shallow copy + return Object.assign({}, this.state) + } + + getValueAsString(): string { + if (this.isMultiple()) { + return this.state.stringArrayVal().join(', ') + } else { + return this.state.stringVal() + } + } + + getValueAsStringArray(): string[] { + return this.state.stringArrayVal() + } + + getSelectedOptionLabel(): string { + if (this.state.selectedOption && this.state.selectedOption.label) { + return this.state.selectedOption.label + } + return '' + } + + getSelectOptions(): any { + return this.state.options + } + + getDataType(): string { + if (this.props.param.data_type) { + return this.props.param.data_type + } + return 'str' + } + + getAnyValue(): any { + return this.state.value + } + + getTypedValue(): any { + if (this.isMultiple()) { + const dataType = this.getDataType() + switch (dataType) { + case 'int': + case 'float': + return this.state.numberArrayVal() + case 'bool': + return this.state.booleanArrayVal() + default: + return this.state.stringArrayVal() + } + } else { + const dataType = this.getDataType() + switch (dataType) { + case 'int': + case 'float': + return this.state.numberVal() + case 'bool': + return this.state.booleanVal() + case 'amount': + if (typeof this.state.value === 'object') { + return this.state.value + } else { + return { + amount: 0.0, + unit: 'USD' + } + } + case 'memory': + if (typeof this.state.value === 'object') { + return this.state.value + } else { + return { + value: 0, + unit: 'bytes' + } + } + default: + return this.state.stringVal() + } + } + } + + getErrorCode(): string | null { + if (Utils.isEmpty(this.state.errorCode)) { + return null + } + return this.state.errorCode! + } + + getErrorMessage(): string | null { + if (Utils.isEmpty(this.state.errorMessage)) { + return null + } + return this.state.errorMessage! + } + + fetchDefault(reset: boolean = false) { + if (!this.props.syncApi) { + return Promise.resolve(this.state) + } + return this.props.syncApi.getParamDefault({ + module: this.props.module, + param: this.props.param.name, + reset: reset + }).then(result => { + return new Promise(resolve => { + const state = { + default: result?.default + } + this.setState(state, () => { + resolve(state) + }) + }) + }) + } + + updateSelectedOptions(): boolean { + if (this.state.options == null || this.state.options.length === 0) { + return false + } + + const selectedOptions: any[] = [] + if (this.isMultiple()) { + const arrayVal = this.state.stringArrayVal() + this.state.options.forEach((option) => { + if (arrayVal.find((value) => value === option.value)) { + selectedOptions.push(option) + } + }) + } else { + const stringVal = this.state.stringVal() + this.state.options.forEach((option) => { + if (stringVal === option.value) { + selectedOptions.push(option) + } + }) + } + if (this.isMultiple()) { + this.setState({ + selectedOptions: selectedOptions + }) + } else { + this.setState({ + selectedOption: selectedOptions[0] + }) + } + return true + } + + setValue(value: any) { + this.setState({ + value: value + }, this.setStateCallback) + } + + setNull(): Promise { + const syncApi = this.props.syncApi + if (syncApi == null) { + return Promise.resolve({}) + } + return new Promise((resolve, reject) => { + syncApi.setParam({ + module: this.props.module, + param: this.getParamName(), + value: '' + }).then(result => { + resolve(result) + }, error => { + reject(error) + }) + }) + } + + reset(): Promise { + const setStateCallback = () => { + this.updateSelectedOptions() + this.setStateCallback() + } + return this.fetchDefault().then(_ => { + if (this.isMultiple()) { + this.setState({ + value: this.state.default, + errorCode: null, + errorMessage: null + }, setStateCallback) + } else { + this.setState({ + value: this.state.default, + errorCode: null, + errorMessage: null + }, setStateCallback) + } + return Promise.resolve(true) + }) + } + + public setOptions(result: GetParamChoicesResult, dynamicOptions: boolean = false) { + const choices = result?.listing! + const options = [] + + for (let i = 0; i < choices.length; i++) { + const choice = choices[i] + if (choice.options && choice.options.length > 0) { + const level2Options: any[] = [] + const option = { + label: choice.title, + options: level2Options + } + options.push(option) + const choicesLevel2 = choice.options + for (let j = 0; j < choicesLevel2.length; j++) { + const choiceLevel2 = choicesLevel2[j] + const value = Utils.asString(choiceLevel2.value) + level2Options.push({ + value: value, + label: choiceLevel2.title, + description: choiceLevel2.description, + disabled: choiceLevel2.disabled + }) + } + } else { + const value = Utils.asString(choice.value) + options.push({ + value: value, + label: choice.title, + description: choice.description, + disabled: choice.disabled + }) + } + } + const state = { + dynamicOptions: dynamicOptions, + options: options + } + this.setState(state, () => { + this.updateSelectedOptions() + }) + } + + fetchOptions(refresh: boolean = false): Promise { + + const paramType = this.props.param.param_type + const applicableParamTypes = ['select', 'raw_select', 'select_or_text', 'checkbox', 'autocomplete'] + const found = applicableParamTypes.find(value => value === paramType) + + if (Utils.isEmpty(found)) { + return Promise.resolve(this) + } + + let dynamicOptions = false + if (this.props.param.dynamic_choices != null) { + dynamicOptions = this.props.param.dynamic_choices + } else { + dynamicOptions = this.props.param.choices == null || this.props.param.choices.length === 0 + } + const syncApi = this.props.syncApi + + if (dynamicOptions) { + let onFetchOptions + if (syncApi != null) { + onFetchOptions = syncApi.getParamChoices + } else if (this.props.onFetchOptions != null) { + onFetchOptions = this.props.onFetchOptions + } + if (onFetchOptions == null) { + return Promise.resolve(this) + } + + return onFetchOptions({ + module: this.props.module, + param: this.props.param.name, + refresh: refresh + }).then((result) => { + + this.setOptions(result, dynamicOptions) + this.updateSelectedOptions() + return this + + }, (error) => { + this.setState({ + options: [] + }) + throw error + }) + } else { + this.setOptions({ + listing: this.props.param.choices! + }, dynamicOptions) + return Promise.resolve(this) + } + } + + componentDidMount() { + if (this.props.onLifecycleEvent) { + this.props.onLifecycleEvent({ + type: 'mounted', + ref: this + }) + } + this.initialize() + } + + initialize() { + Promise.all([this.fetchDefault(), this.fetchOptions()]) + .then(_ => { + const value = this.getTypedValue() + this.setState({ + value: value, + value2: value + }, () => { + if (Utils.isNotEmpty(this.state.value)) { + this.setStateCallback() + } + }) + }, error => { + console.error(error) + this.setState({ + value: undefined + }, this.setStateCallback) + }) + } + + componentWillUnmount() { + if (this.props.onLifecycleEvent) { + this.props.onLifecycleEvent({ + type: 'unmounted', + ref: this + }) + } + } + + publishStateChange(refresh: boolean = false) { + if (this.props.onStateChange != null) { + this.props.onStateChange({ + param: this.props.param, + ref: this, + value: this.getTypedValue(), + errorCode: this.state.errorCode, + errorMessage: this.state.errorMessage, + refresh: refresh + }) + } + } + + setStateCallback() { + const syncApi = this.props.syncApi + if (syncApi == null) { + this.publishStateChange(false) + return + } + syncApi.setParam({ + module: this.props.module, + param: this.props.param.name, + value: this.state.value + }).then((result) => { + // do not publish state change here + if (this.isMultiple()) { + const serverVal = Utils.asStringArray(result?.value) + const localVal = this.state.stringArrayVal() + if (!Utils.isArrayEqual(serverVal, localVal)) { + this.setState({ + value: serverVal + }) + } + } else { + const serverVal = Utils.asString(result?.value) + const localVal = this.state.stringVal() + if (serverVal !== localVal) { + this.setState({ + value: serverVal + }) + } + } + this.setState({ + errorCode: null, + errorMessage: null + }) + this.updateSelectedOptions() + this.publishStateChange(result?.refresh) + + }, (error) => { + console.error(error) + this.setState({ + errorCode: error.error_code, + errorMessage: error.message + }) + }) + } + + disable(should_disable: boolean) { + this.setState( + {disabled: should_disable}, this.setStateCallback + ) + } + + validate(): string { + const validate = this.props.param.validate + if (validate == null) { + return 'OK' + } + + if (!this.validateRegex()) { + return 'REGEX' + } + + if (validate.min != null || validate.max != null) { + const dataType = this.props.param.data_type! + const decimal = (dataType === 'float') + const numberVal = this.state.numberVal() + const min = Utils.asNumber(validate.min, 0, decimal) + const max = Utils.asNumber(validate.max, 0, decimal) + if (validate.min != null && validate.max != null) { + if (numberVal < min || numberVal > max) { + return 'NUMBER_RANGE' + } + } else if (validate.min != null) { + if (numberVal < min) { + return 'MIN_VALUE' + } + } else if (validate.max != null) { + if (numberVal > max) { + return 'MAX_VALUE' + } + } + } + + if (validate.required == null || Utils.isFalse(validate.required)) { + return 'OK' + } + + if (this.isMultiple()) { + if (this.state.stringArrayVal().length === 0) { + return 'REQUIRED' + } else { + let isEmpty = false + let values = this.state.stringArrayVal() + for (let i = 0; i < values.length; i++) { + if (Utils.isEmpty(values[i])) { + isEmpty = true + break + } + } + if (isEmpty) { + return 'REQUIRED' + } + return 'OK' + } + } else if (Utils.isEmpty(this.state.stringVal())) { + return 'REQUIRED' + } + + if(this.props.param.param_type === 'new-password') { + if(Utils.isEmpty(this.state.value2)) { + return 'REQUIRED' + } + } + + return 'OK' + } + + triggerValidate(): boolean { + const result = this.validate() + if (result === 'OK') { + this.setState({ + errorCode: null, + errorMessage: null + }) + return true + } else { + const errorCode = 'VALIDATION_FAILED' + let errorMessage + let displayTitle = this.props.param.title + if (displayTitle === undefined || displayTitle === null || displayTitle?.length === 0) { + displayTitle = this.props.param.name + } + switch (result) { + case 'REQUIRED': + errorMessage = `${displayTitle} is required.` + break + case 'NUMBER_RANGE': + errorMessage = `${displayTitle} must be between ${this.props.param.validate?.min} + and ${this.props.param.validate?.max}.` + break + case 'MIN_VALUE': + errorMessage = `${displayTitle} must be greater than or equal to ${this.props.param.validate?.min}` + break + case 'MAX_VALUE': + errorMessage = `${displayTitle} must be less than or equal to ${this.props.param.validate?.max}` + break + case 'REGEX': + errorMessage = `${displayTitle} must satisfy regex: ${this.props.param.validate?.regex}` + break + /*case 'CUSTOM_FAILED': + errorMessage = this.props.param.validate?.custom?.error_message + break*/ + default: + errorMessage = `${displayTitle} validation failed.` + } + this.setState({ + errorCode: errorCode, + errorMessage: errorMessage + }) + return false + } + } + + getNativeType(): string { + const dataType = this.props.param.data_type! + let result + if (dataType === 'int' || dataType === 'float') { + result = 'number' + } else if (dataType === 'str') { + result = 'string' + } else if (dataType === 'bool') { + result = 'boolean' + } else if (dataType === 'memory') { + result = 'memory' + } else if (dataType === 'amount') { + result = 'amount' + } else { + result = 'string' + } + return result + } + + getCustomType(): string | null { + if (this.props.param.custom_type) { + return this.props.param.custom_type + } + return null + } + + getInputMode(): InputProps.InputMode { + const type = this.getNativeType() + const dataType = this.props.param.data_type! + if (type === 'string') { + return 'text' + } else if (type === 'number') { + if (dataType === 'int') { + return 'numeric' + } else { + return 'decimal' + } + } + return 'text' + } + + isMultiple(): boolean { + if (this.props.param.multiple != null) { + return this.props.param.multiple + } + return false + } + + isReadOnly(): boolean { + if (this.props.param.readonly != null) { + return this.props.param.readonly + } + return false + } + + isAutoFocus(): boolean { + if (this.props.param.auto_focus != null) { + return this.props.param.auto_focus + } + return false + } + + isRefreshable(): boolean { + if (this.props.param.refreshable != null) { + return this.props.param.refreshable + } + return false + } + + getInputType(): InputProps.Type { + const type = this.getNativeType() + const paramType = this.props.param.param_type + if (type === 'string') { + if (paramType === 'password') { + return 'password' + } else { + return 'text' + } + } else if (type === 'number') { + return 'number' + } + return 'text' + } + + isMarkDownAvailable(): boolean { + return Utils.isNotEmpty(this.props.param.markdown) + } + + buildHelpPanel() { + return {this.props.param.title}}> + + + } + + buildFormField(field: React.ReactNode, props?: FormFieldProps, key?: string): React.ReactNode { + let label: React.ReactNode = this.props.param.title + let description: React.ReactNode = this.props.param.description + let constraintText: React.ReactNode = this.props.param.help_text + let stretch = false + let secondaryControl = null + if (props != null) { + if (props.label != null) { + label = props.label + } + if (props.description != null) { + description = props.description + } + if (props.constraintText != null) { + constraintText = props.constraintText + } + if (props.stretch != null) { + stretch = props.stretch + } + if (props.secondaryControl != null) { + secondaryControl = props.secondaryControl + } + } + + if (!key) { + key = `f-${this.getParamName()}` + } + + return ( + { + if (this.props.updateTools) { + this.props.updateTools(this.buildHelpPanel()) + } + }}>Info} + errorText={this.getErrorMessage()} + secondaryControl={secondaryControl}> + {field} + + ) + } + + validateRegex(value?: string): boolean { + if (this.props.param.validate == null) { + return true + } + const regex = this.props.param.validate.regex + if (regex == null || Utils.isEmpty(regex)) { + return true + } + const re = RegExp(regex) + let token = value + if (token == null) { + token = this.state.stringVal() + } + + return re.test(token); + + } + + triggerDynamicOptionsLoading(): Promise { + return new Promise((resolve, reject) => { + this.setState({ + dynamicOptionsLoading: true, + options: [], + selectedOption: {}, + selectedOptions: [] + }, () => { + this.fetchOptions(true) + .catch(error => reject(error)) + .finally(() => { + this.setState({ + dynamicOptionsLoading: false + }, () => { + resolve('OK') + }) + }) + }) + }) + } + + buildOptionsRefresh() { + if (this.isAutoComplete()) { + return + } + const loadDynamicOptions = (_: any) => { + this.triggerDynamicOptionsLoading() + .catch(error => { + console.error(error) + }) + } + return ( + + } + } + if (this.state.dynamicOptions) { + const optionsRefresh = this.buildOptionsRefresh() + if (customControl) { + return ( + + {optionsRefresh} + {customControl} + + ) + } else { + return optionsRefresh + } + } else { + return customControl + } + } + + onKeyDown = (event: CustomEvent) => { + if (event.detail.key === 'Enter' && this.props.onKeyEnter) { + this.props.onKeyEnter({ + ref: this + }) + } + } + + onInputStateChange(value: string) { + this.setState({ + value: value + }, () => { + if (this.triggerValidate()) { + this.setState({}, this.setStateCallback) + } + }) + } + + buildInput(props: FormFieldProps) { + let secondaryControl = null + if (this.isRefreshable()) { + secondaryControl = this.buildDefaultValueRefresh() + } + + return this.buildFormField( + { + this.onInputStateChange(event.detail.value) + }}/>, + { + ...props, + secondaryControl: secondaryControl + } + ) + } + + buildInputArray(props: FormFieldProps) { + return this.buildFormField( + + {this.getValueAsStringArray().length === 0 && } + {this.getValueAsStringArray().length > 0 && this.getValueAsStringArray().map((value, index) => { + return + { + const values: string[] = this.state.value + values[index] = event.detail.value + this.setState({ + value: values + }, this.setStateCallback) + }}/> + + + + + + })} + , + props + ) + } + + onPasswordStateChange(value: string, value2: boolean = false) { + + const stateCallback = () => { + if (this.props.param.param_type === 'new-password') { + const val1 = this.state.stringVal() + const val2 = this.state.stringVal2() + if (!Utils.isEmpty(val1) && !Utils.isEmpty(val2) && val1 === val2) { + this.setState({ + errorCode: null, + errorMessage: null + }) + this.setStateCallback() + } else if (!Utils.isEmpty(val2)) { + this.setState({ + errorCode: 'VERIFY_PASSWORD_DOES_NOT_MATCH', + errorMessage: 'Passwords do not match' + }) + } + } else if (this.props.param.param_type === 'password') { + this.setStateCallback() + } + } + + if (value2) { + this.setState({ + value2: value + }, stateCallback) + } else { + this.setState({ + value: value + }, stateCallback) + } + } + + buildPassword(value2: boolean = false, props: FormFieldProps) { + let label = this.props.param.title + let description = this.props.param.description + let value = () => this.state.stringVal() + if (value2) { + label = `Verify ${label}` + description = `Re-${description}` + value = () => this.state.stringVal2() + } + let key + if (value2) { + key = `f-${this.getParamName()}-2` + } else { + key = `f-${this.getParamName()}-1` + } + return this.buildFormField( + { + this.onPasswordStateChange(event.detail.value, value2) + }}/>, + { + ...props, + label: label, + description: description + }, + key + ) + } + + onAutoSuggestStateChange(value: string) { + this.setState({ + value: value + }, () => { + if (this.triggerValidate()) { + this.setState({}, this.setStateCallback) + } + }) + } + + /** + * AutoSuggest will preload all options and filtering is performed locally. + */ + buildAutoSuggest(props: FormFieldProps) { + let secondaryControl = this.buildFormFieldSecondaryControl() + return this.buildFormField( + `Use: ${value}`} + value={this.state.stringVal()} + options={this.state.options} + disabled={this.state.disabled} + autoFocus={this.isAutoFocus()} + onKeyDown={(event) => { + if (event.detail.key === 'Escape') { + // this prevents the modal getting dismissed + event.stopPropagation() + } + }} + onChange={(event) => this.onAutoSuggestStateChange(event.detail.value)}/>, + { + ...props, + secondaryControl: secondaryControl + } + ) + } + + onAutoCompleteStateChange(value: string) { + this.setState({ + value: value + }, () => { + if (this.triggerValidate()) { + this.setState({}, this.setStateCallback) + } + }) + } + + /** + * AutoComplete loads options based on text entered by user. + * onFetchOptions is called as user types the input + */ + buildAutoComplete(props: FormFieldProps) { + let secondaryControl = this.buildFormFieldSecondaryControl() + return this.buildFormField( + `Use: ${value}`} + value={this.state.stringVal()} + options={this.state.options} + statusType={(this.state.dynamicOptionsLoading) ? 'loading' : 'finished'} + filteringType="none" + disabled={this.state.disabled} + autoFocus={this.isAutoFocus()} + onKeyDown={(event) => { + // prevent the modal getting dismissed on Escape + if (event.detail.key === 'Escape') { + event.stopPropagation() + } + }} + onLoadItems={(event) => { + if (this.props.onFetchOptions) { + this.setState({ + options: [], + dynamicOptionsLoading: true + }, () => { + this.props.onFetchOptions!({ + param: this.getParamName(), + filters: [ + { + key: this.getParamName(), + value: event.detail.filteringText + } + ] + }).then(result => { + this.setOptions(result) + }).finally(() => { + this.setState({ + dynamicOptionsLoading: false + }) + }) + }) + } + }} + onChange={(event) => this.onAutoCompleteStateChange(event.detail.value)}/>, + { + ...props, + secondaryControl: secondaryControl + } + ) + } + + onDatePickerStateChange(selectedDate: string) { + this.setState({ + value: selectedDate + }, () => { + if (this.triggerValidate()) { + this.setState({}, this.setStateCallback) + } + }) + } + + buildDatePicker(props: FormFieldProps) { + return this.buildFormField( + this.onDatePickerStateChange(detail.value)} + readOnly={this.isReadOnly()} + autoFocus={this.isAutoFocus()} + value={this.state.stringVal()} + disabled={this.state.disabled} + openCalendarAriaLabel={selectedDate => + "Choose Date" + + (selectedDate + ? `, selected date is ${selectedDate}` + : "") + } + placeholder="YYYY/MM/DD" + nextMonthAriaLabel="Next month" + previousMonthAriaLabel="Previous month" + todayAriaLabel="Today"/>, props + ) + } + + onTextAreaStateChange(value: string) { + this.setState({ + value: value + }, () => { + if (this.triggerValidate()) { + this.setState({}, this.setStateCallback) + } + }) + } + + buildTextArea(props: FormFieldProps) { + return this.buildFormField( +