diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 93a9bdce..577d5bf4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -33,7 +33,7 @@ repos: - types-setuptools - types-tzlocal==4.2 - types-aiofiles==23.2.0.20240311 - exclude: tests + exclude: "(tests|s3experiment.py)" args: - --check-untyped-defs - --ignore-missing-imports diff --git a/poetry.lock b/poetry.lock index 22647f92..b988cd3e 100644 --- a/poetry.lock +++ b/poetry.lock @@ -177,6 +177,465 @@ files = [ docs = ["furo", "jaraco.packaging (>=9.3)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] testing = ["pytest (>=6,!=8.1.1)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)"] +[[package]] +name = "boto3" +version = "1.34.82" +description = "The AWS SDK for Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "boto3-1.34.82-py3-none-any.whl", hash = "sha256:6e0ee12e87b37fa81133e9308d0957fce4200c1ff37c96346538dba5e857da18"}, + {file = "boto3-1.34.82.tar.gz", hash = "sha256:fcdb84936b04d5f78c8c8667b65bf5b9803cf39fd25bb7fe57ba237074e36171"}, +] + +[package.dependencies] +botocore = ">=1.34.82,<1.35.0" +jmespath = ">=0.7.1,<2.0.0" +s3transfer = ">=0.10.0,<0.11.0" + +[package.extras] +crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] + +[[package]] +name = "boto3-stubs" +version = "1.34.82" +description = "Type annotations for boto3 1.34.82 generated with mypy-boto3-builder 7.23.2" +optional = false +python-versions = ">=3.8" +files = [ + {file = "boto3-stubs-1.34.82.tar.gz", hash = "sha256:be9e84af538fee69fc1c9cb0bb7cba254bb7b488cc87cea7038c336758d53253"}, + {file = "boto3_stubs-1.34.82-py3-none-any.whl", hash = "sha256:fdaf3c98d1c9a838ca19f0c4d52f1a7a4e5af27514199d5b9efb2d5320238309"}, +] + +[package.dependencies] +botocore-stubs = "*" +mypy-boto3-s3 = {version = ">=1.34.0,<1.35.0", optional = true, markers = "extra == \"s3\""} +types-s3transfer = "*" +typing-extensions = {version = ">=4.1.0", markers = "python_version < \"3.12\""} + +[package.extras] +accessanalyzer = ["mypy-boto3-accessanalyzer (>=1.34.0,<1.35.0)"] +account = ["mypy-boto3-account (>=1.34.0,<1.35.0)"] +acm = ["mypy-boto3-acm (>=1.34.0,<1.35.0)"] +acm-pca = ["mypy-boto3-acm-pca (>=1.34.0,<1.35.0)"] +alexaforbusiness = ["mypy-boto3-alexaforbusiness (>=1.34.0,<1.35.0)"] +all = ["mypy-boto3-accessanalyzer (>=1.34.0,<1.35.0)", "mypy-boto3-account (>=1.34.0,<1.35.0)", "mypy-boto3-acm (>=1.34.0,<1.35.0)", "mypy-boto3-acm-pca (>=1.34.0,<1.35.0)", "mypy-boto3-alexaforbusiness (>=1.34.0,<1.35.0)", "mypy-boto3-amp (>=1.34.0,<1.35.0)", "mypy-boto3-amplify (>=1.34.0,<1.35.0)", "mypy-boto3-amplifybackend (>=1.34.0,<1.35.0)", "mypy-boto3-amplifyuibuilder (>=1.34.0,<1.35.0)", "mypy-boto3-apigateway (>=1.34.0,<1.35.0)", "mypy-boto3-apigatewaymanagementapi (>=1.34.0,<1.35.0)", "mypy-boto3-apigatewayv2 (>=1.34.0,<1.35.0)", "mypy-boto3-appconfig (>=1.34.0,<1.35.0)", "mypy-boto3-appconfigdata (>=1.34.0,<1.35.0)", "mypy-boto3-appfabric (>=1.34.0,<1.35.0)", "mypy-boto3-appflow (>=1.34.0,<1.35.0)", "mypy-boto3-appintegrations (>=1.34.0,<1.35.0)", "mypy-boto3-application-autoscaling (>=1.34.0,<1.35.0)", "mypy-boto3-application-insights (>=1.34.0,<1.35.0)", "mypy-boto3-applicationcostprofiler (>=1.34.0,<1.35.0)", "mypy-boto3-appmesh (>=1.34.0,<1.35.0)", "mypy-boto3-apprunner (>=1.34.0,<1.35.0)", "mypy-boto3-appstream (>=1.34.0,<1.35.0)", "mypy-boto3-appsync (>=1.34.0,<1.35.0)", "mypy-boto3-arc-zonal-shift (>=1.34.0,<1.35.0)", "mypy-boto3-artifact (>=1.34.0,<1.35.0)", "mypy-boto3-athena (>=1.34.0,<1.35.0)", "mypy-boto3-auditmanager (>=1.34.0,<1.35.0)", "mypy-boto3-autoscaling (>=1.34.0,<1.35.0)", "mypy-boto3-autoscaling-plans (>=1.34.0,<1.35.0)", "mypy-boto3-b2bi (>=1.34.0,<1.35.0)", "mypy-boto3-backup (>=1.34.0,<1.35.0)", "mypy-boto3-backup-gateway (>=1.34.0,<1.35.0)", "mypy-boto3-backupstorage (>=1.34.0,<1.35.0)", "mypy-boto3-batch (>=1.34.0,<1.35.0)", "mypy-boto3-bcm-data-exports (>=1.34.0,<1.35.0)", "mypy-boto3-bedrock (>=1.34.0,<1.35.0)", "mypy-boto3-bedrock-agent (>=1.34.0,<1.35.0)", "mypy-boto3-bedrock-agent-runtime (>=1.34.0,<1.35.0)", "mypy-boto3-bedrock-runtime (>=1.34.0,<1.35.0)", "mypy-boto3-billingconductor (>=1.34.0,<1.35.0)", "mypy-boto3-braket (>=1.34.0,<1.35.0)", "mypy-boto3-budgets (>=1.34.0,<1.35.0)", "mypy-boto3-ce (>=1.34.0,<1.35.0)", "mypy-boto3-chatbot (>=1.34.0,<1.35.0)", "mypy-boto3-chime (>=1.34.0,<1.35.0)", "mypy-boto3-chime-sdk-identity (>=1.34.0,<1.35.0)", "mypy-boto3-chime-sdk-media-pipelines (>=1.34.0,<1.35.0)", "mypy-boto3-chime-sdk-meetings (>=1.34.0,<1.35.0)", "mypy-boto3-chime-sdk-messaging (>=1.34.0,<1.35.0)", "mypy-boto3-chime-sdk-voice (>=1.34.0,<1.35.0)", "mypy-boto3-cleanrooms (>=1.34.0,<1.35.0)", "mypy-boto3-cleanroomsml (>=1.34.0,<1.35.0)", "mypy-boto3-cloud9 (>=1.34.0,<1.35.0)", "mypy-boto3-cloudcontrol (>=1.34.0,<1.35.0)", "mypy-boto3-clouddirectory (>=1.34.0,<1.35.0)", "mypy-boto3-cloudformation (>=1.34.0,<1.35.0)", "mypy-boto3-cloudfront (>=1.34.0,<1.35.0)", "mypy-boto3-cloudfront-keyvaluestore (>=1.34.0,<1.35.0)", "mypy-boto3-cloudhsm (>=1.34.0,<1.35.0)", "mypy-boto3-cloudhsmv2 (>=1.34.0,<1.35.0)", "mypy-boto3-cloudsearch (>=1.34.0,<1.35.0)", "mypy-boto3-cloudsearchdomain (>=1.34.0,<1.35.0)", "mypy-boto3-cloudtrail (>=1.34.0,<1.35.0)", "mypy-boto3-cloudtrail-data (>=1.34.0,<1.35.0)", "mypy-boto3-cloudwatch (>=1.34.0,<1.35.0)", "mypy-boto3-codeartifact (>=1.34.0,<1.35.0)", "mypy-boto3-codebuild (>=1.34.0,<1.35.0)", "mypy-boto3-codecatalyst (>=1.34.0,<1.35.0)", "mypy-boto3-codecommit (>=1.34.0,<1.35.0)", "mypy-boto3-codeconnections (>=1.34.0,<1.35.0)", "mypy-boto3-codedeploy (>=1.34.0,<1.35.0)", "mypy-boto3-codeguru-reviewer (>=1.34.0,<1.35.0)", "mypy-boto3-codeguru-security (>=1.34.0,<1.35.0)", "mypy-boto3-codeguruprofiler (>=1.34.0,<1.35.0)", "mypy-boto3-codepipeline (>=1.34.0,<1.35.0)", "mypy-boto3-codestar (>=1.34.0,<1.35.0)", "mypy-boto3-codestar-connections (>=1.34.0,<1.35.0)", "mypy-boto3-codestar-notifications (>=1.34.0,<1.35.0)", "mypy-boto3-cognito-identity (>=1.34.0,<1.35.0)", "mypy-boto3-cognito-idp (>=1.34.0,<1.35.0)", "mypy-boto3-cognito-sync (>=1.34.0,<1.35.0)", "mypy-boto3-comprehend (>=1.34.0,<1.35.0)", "mypy-boto3-comprehendmedical (>=1.34.0,<1.35.0)", "mypy-boto3-compute-optimizer (>=1.34.0,<1.35.0)", "mypy-boto3-config (>=1.34.0,<1.35.0)", "mypy-boto3-connect (>=1.34.0,<1.35.0)", "mypy-boto3-connect-contact-lens (>=1.34.0,<1.35.0)", "mypy-boto3-connectcampaigns (>=1.34.0,<1.35.0)", "mypy-boto3-connectcases (>=1.34.0,<1.35.0)", "mypy-boto3-connectparticipant (>=1.34.0,<1.35.0)", "mypy-boto3-controlcatalog (>=1.34.0,<1.35.0)", "mypy-boto3-controltower (>=1.34.0,<1.35.0)", "mypy-boto3-cost-optimization-hub (>=1.34.0,<1.35.0)", "mypy-boto3-cur (>=1.34.0,<1.35.0)", "mypy-boto3-customer-profiles (>=1.34.0,<1.35.0)", "mypy-boto3-databrew (>=1.34.0,<1.35.0)", "mypy-boto3-dataexchange (>=1.34.0,<1.35.0)", "mypy-boto3-datapipeline (>=1.34.0,<1.35.0)", "mypy-boto3-datasync (>=1.34.0,<1.35.0)", "mypy-boto3-datazone (>=1.34.0,<1.35.0)", "mypy-boto3-dax (>=1.34.0,<1.35.0)", "mypy-boto3-deadline (>=1.34.0,<1.35.0)", "mypy-boto3-detective (>=1.34.0,<1.35.0)", "mypy-boto3-devicefarm (>=1.34.0,<1.35.0)", "mypy-boto3-devops-guru (>=1.34.0,<1.35.0)", "mypy-boto3-directconnect (>=1.34.0,<1.35.0)", "mypy-boto3-discovery (>=1.34.0,<1.35.0)", "mypy-boto3-dlm (>=1.34.0,<1.35.0)", "mypy-boto3-dms (>=1.34.0,<1.35.0)", "mypy-boto3-docdb (>=1.34.0,<1.35.0)", "mypy-boto3-docdb-elastic (>=1.34.0,<1.35.0)", "mypy-boto3-drs (>=1.34.0,<1.35.0)", "mypy-boto3-ds (>=1.34.0,<1.35.0)", "mypy-boto3-dynamodb (>=1.34.0,<1.35.0)", "mypy-boto3-dynamodbstreams (>=1.34.0,<1.35.0)", "mypy-boto3-ebs (>=1.34.0,<1.35.0)", "mypy-boto3-ec2 (>=1.34.0,<1.35.0)", "mypy-boto3-ec2-instance-connect (>=1.34.0,<1.35.0)", "mypy-boto3-ecr (>=1.34.0,<1.35.0)", "mypy-boto3-ecr-public (>=1.34.0,<1.35.0)", "mypy-boto3-ecs (>=1.34.0,<1.35.0)", "mypy-boto3-efs (>=1.34.0,<1.35.0)", "mypy-boto3-eks (>=1.34.0,<1.35.0)", "mypy-boto3-eks-auth (>=1.34.0,<1.35.0)", "mypy-boto3-elastic-inference (>=1.34.0,<1.35.0)", "mypy-boto3-elasticache (>=1.34.0,<1.35.0)", "mypy-boto3-elasticbeanstalk (>=1.34.0,<1.35.0)", "mypy-boto3-elastictranscoder (>=1.34.0,<1.35.0)", "mypy-boto3-elb (>=1.34.0,<1.35.0)", "mypy-boto3-elbv2 (>=1.34.0,<1.35.0)", "mypy-boto3-emr (>=1.34.0,<1.35.0)", "mypy-boto3-emr-containers (>=1.34.0,<1.35.0)", "mypy-boto3-emr-serverless (>=1.34.0,<1.35.0)", "mypy-boto3-entityresolution (>=1.34.0,<1.35.0)", "mypy-boto3-es (>=1.34.0,<1.35.0)", "mypy-boto3-events (>=1.34.0,<1.35.0)", "mypy-boto3-evidently (>=1.34.0,<1.35.0)", "mypy-boto3-finspace (>=1.34.0,<1.35.0)", "mypy-boto3-finspace-data (>=1.34.0,<1.35.0)", "mypy-boto3-firehose (>=1.34.0,<1.35.0)", "mypy-boto3-fis (>=1.34.0,<1.35.0)", "mypy-boto3-fms (>=1.34.0,<1.35.0)", "mypy-boto3-forecast (>=1.34.0,<1.35.0)", "mypy-boto3-forecastquery (>=1.34.0,<1.35.0)", "mypy-boto3-frauddetector (>=1.34.0,<1.35.0)", "mypy-boto3-freetier (>=1.34.0,<1.35.0)", "mypy-boto3-fsx (>=1.34.0,<1.35.0)", "mypy-boto3-gamelift (>=1.34.0,<1.35.0)", "mypy-boto3-glacier (>=1.34.0,<1.35.0)", "mypy-boto3-globalaccelerator (>=1.34.0,<1.35.0)", "mypy-boto3-glue (>=1.34.0,<1.35.0)", "mypy-boto3-grafana (>=1.34.0,<1.35.0)", "mypy-boto3-greengrass (>=1.34.0,<1.35.0)", "mypy-boto3-greengrassv2 (>=1.34.0,<1.35.0)", "mypy-boto3-groundstation (>=1.34.0,<1.35.0)", "mypy-boto3-guardduty (>=1.34.0,<1.35.0)", "mypy-boto3-health (>=1.34.0,<1.35.0)", "mypy-boto3-healthlake (>=1.34.0,<1.35.0)", "mypy-boto3-honeycode (>=1.34.0,<1.35.0)", "mypy-boto3-iam (>=1.34.0,<1.35.0)", "mypy-boto3-identitystore (>=1.34.0,<1.35.0)", "mypy-boto3-imagebuilder (>=1.34.0,<1.35.0)", "mypy-boto3-importexport (>=1.34.0,<1.35.0)", "mypy-boto3-inspector (>=1.34.0,<1.35.0)", "mypy-boto3-inspector-scan (>=1.34.0,<1.35.0)", "mypy-boto3-inspector2 (>=1.34.0,<1.35.0)", "mypy-boto3-internetmonitor (>=1.34.0,<1.35.0)", "mypy-boto3-iot (>=1.34.0,<1.35.0)", "mypy-boto3-iot-data (>=1.34.0,<1.35.0)", "mypy-boto3-iot-jobs-data (>=1.34.0,<1.35.0)", "mypy-boto3-iot1click-devices (>=1.34.0,<1.35.0)", "mypy-boto3-iot1click-projects (>=1.34.0,<1.35.0)", "mypy-boto3-iotanalytics (>=1.34.0,<1.35.0)", "mypy-boto3-iotdeviceadvisor (>=1.34.0,<1.35.0)", "mypy-boto3-iotevents (>=1.34.0,<1.35.0)", "mypy-boto3-iotevents-data (>=1.34.0,<1.35.0)", "mypy-boto3-iotfleethub (>=1.34.0,<1.35.0)", "mypy-boto3-iotfleetwise (>=1.34.0,<1.35.0)", "mypy-boto3-iotsecuretunneling (>=1.34.0,<1.35.0)", "mypy-boto3-iotsitewise (>=1.34.0,<1.35.0)", "mypy-boto3-iotthingsgraph (>=1.34.0,<1.35.0)", "mypy-boto3-iottwinmaker (>=1.34.0,<1.35.0)", "mypy-boto3-iotwireless (>=1.34.0,<1.35.0)", "mypy-boto3-ivs (>=1.34.0,<1.35.0)", "mypy-boto3-ivs-realtime (>=1.34.0,<1.35.0)", "mypy-boto3-ivschat (>=1.34.0,<1.35.0)", "mypy-boto3-kafka (>=1.34.0,<1.35.0)", "mypy-boto3-kafkaconnect (>=1.34.0,<1.35.0)", "mypy-boto3-kendra (>=1.34.0,<1.35.0)", "mypy-boto3-kendra-ranking (>=1.34.0,<1.35.0)", "mypy-boto3-keyspaces (>=1.34.0,<1.35.0)", "mypy-boto3-kinesis (>=1.34.0,<1.35.0)", "mypy-boto3-kinesis-video-archived-media (>=1.34.0,<1.35.0)", "mypy-boto3-kinesis-video-media (>=1.34.0,<1.35.0)", "mypy-boto3-kinesis-video-signaling (>=1.34.0,<1.35.0)", "mypy-boto3-kinesis-video-webrtc-storage (>=1.34.0,<1.35.0)", "mypy-boto3-kinesisanalytics (>=1.34.0,<1.35.0)", "mypy-boto3-kinesisanalyticsv2 (>=1.34.0,<1.35.0)", "mypy-boto3-kinesisvideo (>=1.34.0,<1.35.0)", "mypy-boto3-kms (>=1.34.0,<1.35.0)", "mypy-boto3-lakeformation (>=1.34.0,<1.35.0)", "mypy-boto3-lambda (>=1.34.0,<1.35.0)", "mypy-boto3-launch-wizard (>=1.34.0,<1.35.0)", "mypy-boto3-lex-models (>=1.34.0,<1.35.0)", "mypy-boto3-lex-runtime (>=1.34.0,<1.35.0)", "mypy-boto3-lexv2-models (>=1.34.0,<1.35.0)", "mypy-boto3-lexv2-runtime (>=1.34.0,<1.35.0)", "mypy-boto3-license-manager (>=1.34.0,<1.35.0)", "mypy-boto3-license-manager-linux-subscriptions (>=1.34.0,<1.35.0)", "mypy-boto3-license-manager-user-subscriptions (>=1.34.0,<1.35.0)", "mypy-boto3-lightsail (>=1.34.0,<1.35.0)", "mypy-boto3-location (>=1.34.0,<1.35.0)", "mypy-boto3-logs (>=1.34.0,<1.35.0)", "mypy-boto3-lookoutequipment (>=1.34.0,<1.35.0)", "mypy-boto3-lookoutmetrics (>=1.34.0,<1.35.0)", "mypy-boto3-lookoutvision (>=1.34.0,<1.35.0)", "mypy-boto3-m2 (>=1.34.0,<1.35.0)", "mypy-boto3-machinelearning (>=1.34.0,<1.35.0)", "mypy-boto3-macie2 (>=1.34.0,<1.35.0)", "mypy-boto3-managedblockchain (>=1.34.0,<1.35.0)", "mypy-boto3-managedblockchain-query (>=1.34.0,<1.35.0)", "mypy-boto3-marketplace-agreement (>=1.34.0,<1.35.0)", "mypy-boto3-marketplace-catalog (>=1.34.0,<1.35.0)", "mypy-boto3-marketplace-deployment (>=1.34.0,<1.35.0)", "mypy-boto3-marketplace-entitlement (>=1.34.0,<1.35.0)", "mypy-boto3-marketplacecommerceanalytics (>=1.34.0,<1.35.0)", "mypy-boto3-mediaconnect (>=1.34.0,<1.35.0)", "mypy-boto3-mediaconvert (>=1.34.0,<1.35.0)", "mypy-boto3-medialive (>=1.34.0,<1.35.0)", "mypy-boto3-mediapackage (>=1.34.0,<1.35.0)", "mypy-boto3-mediapackage-vod (>=1.34.0,<1.35.0)", "mypy-boto3-mediapackagev2 (>=1.34.0,<1.35.0)", "mypy-boto3-mediastore (>=1.34.0,<1.35.0)", "mypy-boto3-mediastore-data (>=1.34.0,<1.35.0)", "mypy-boto3-mediatailor (>=1.34.0,<1.35.0)", "mypy-boto3-medical-imaging (>=1.34.0,<1.35.0)", "mypy-boto3-memorydb (>=1.34.0,<1.35.0)", "mypy-boto3-meteringmarketplace (>=1.34.0,<1.35.0)", "mypy-boto3-mgh (>=1.34.0,<1.35.0)", "mypy-boto3-mgn (>=1.34.0,<1.35.0)", "mypy-boto3-migration-hub-refactor-spaces (>=1.34.0,<1.35.0)", "mypy-boto3-migrationhub-config (>=1.34.0,<1.35.0)", "mypy-boto3-migrationhuborchestrator (>=1.34.0,<1.35.0)", "mypy-boto3-migrationhubstrategy (>=1.34.0,<1.35.0)", "mypy-boto3-mobile (>=1.34.0,<1.35.0)", "mypy-boto3-mq (>=1.34.0,<1.35.0)", "mypy-boto3-mturk (>=1.34.0,<1.35.0)", "mypy-boto3-mwaa (>=1.34.0,<1.35.0)", "mypy-boto3-neptune (>=1.34.0,<1.35.0)", "mypy-boto3-neptune-graph (>=1.34.0,<1.35.0)", "mypy-boto3-neptunedata (>=1.34.0,<1.35.0)", "mypy-boto3-network-firewall (>=1.34.0,<1.35.0)", "mypy-boto3-networkmanager (>=1.34.0,<1.35.0)", "mypy-boto3-networkmonitor (>=1.34.0,<1.35.0)", "mypy-boto3-nimble (>=1.34.0,<1.35.0)", "mypy-boto3-oam (>=1.34.0,<1.35.0)", "mypy-boto3-omics (>=1.34.0,<1.35.0)", "mypy-boto3-opensearch (>=1.34.0,<1.35.0)", "mypy-boto3-opensearchserverless (>=1.34.0,<1.35.0)", "mypy-boto3-opsworks (>=1.34.0,<1.35.0)", "mypy-boto3-opsworkscm (>=1.34.0,<1.35.0)", "mypy-boto3-organizations (>=1.34.0,<1.35.0)", "mypy-boto3-osis (>=1.34.0,<1.35.0)", "mypy-boto3-outposts (>=1.34.0,<1.35.0)", "mypy-boto3-panorama (>=1.34.0,<1.35.0)", "mypy-boto3-payment-cryptography (>=1.34.0,<1.35.0)", "mypy-boto3-payment-cryptography-data (>=1.34.0,<1.35.0)", "mypy-boto3-pca-connector-ad (>=1.34.0,<1.35.0)", "mypy-boto3-personalize (>=1.34.0,<1.35.0)", "mypy-boto3-personalize-events (>=1.34.0,<1.35.0)", "mypy-boto3-personalize-runtime (>=1.34.0,<1.35.0)", "mypy-boto3-pi (>=1.34.0,<1.35.0)", "mypy-boto3-pinpoint (>=1.34.0,<1.35.0)", "mypy-boto3-pinpoint-email (>=1.34.0,<1.35.0)", "mypy-boto3-pinpoint-sms-voice (>=1.34.0,<1.35.0)", "mypy-boto3-pinpoint-sms-voice-v2 (>=1.34.0,<1.35.0)", "mypy-boto3-pipes (>=1.34.0,<1.35.0)", "mypy-boto3-polly (>=1.34.0,<1.35.0)", "mypy-boto3-pricing (>=1.34.0,<1.35.0)", "mypy-boto3-privatenetworks (>=1.34.0,<1.35.0)", "mypy-boto3-proton (>=1.34.0,<1.35.0)", "mypy-boto3-qbusiness (>=1.34.0,<1.35.0)", "mypy-boto3-qconnect (>=1.34.0,<1.35.0)", "mypy-boto3-qldb (>=1.34.0,<1.35.0)", "mypy-boto3-qldb-session (>=1.34.0,<1.35.0)", "mypy-boto3-quicksight (>=1.34.0,<1.35.0)", "mypy-boto3-ram (>=1.34.0,<1.35.0)", "mypy-boto3-rbin (>=1.34.0,<1.35.0)", "mypy-boto3-rds (>=1.34.0,<1.35.0)", "mypy-boto3-rds-data (>=1.34.0,<1.35.0)", "mypy-boto3-redshift (>=1.34.0,<1.35.0)", "mypy-boto3-redshift-data (>=1.34.0,<1.35.0)", "mypy-boto3-redshift-serverless (>=1.34.0,<1.35.0)", "mypy-boto3-rekognition (>=1.34.0,<1.35.0)", "mypy-boto3-repostspace (>=1.34.0,<1.35.0)", "mypy-boto3-resiliencehub (>=1.34.0,<1.35.0)", "mypy-boto3-resource-explorer-2 (>=1.34.0,<1.35.0)", "mypy-boto3-resource-groups (>=1.34.0,<1.35.0)", "mypy-boto3-resourcegroupstaggingapi (>=1.34.0,<1.35.0)", "mypy-boto3-robomaker (>=1.34.0,<1.35.0)", "mypy-boto3-rolesanywhere (>=1.34.0,<1.35.0)", "mypy-boto3-route53 (>=1.34.0,<1.35.0)", "mypy-boto3-route53-recovery-cluster (>=1.34.0,<1.35.0)", "mypy-boto3-route53-recovery-control-config (>=1.34.0,<1.35.0)", "mypy-boto3-route53-recovery-readiness (>=1.34.0,<1.35.0)", "mypy-boto3-route53domains (>=1.34.0,<1.35.0)", "mypy-boto3-route53resolver (>=1.34.0,<1.35.0)", "mypy-boto3-rum (>=1.34.0,<1.35.0)", "mypy-boto3-s3 (>=1.34.0,<1.35.0)", "mypy-boto3-s3control (>=1.34.0,<1.35.0)", "mypy-boto3-s3outposts (>=1.34.0,<1.35.0)", "mypy-boto3-sagemaker (>=1.34.0,<1.35.0)", "mypy-boto3-sagemaker-a2i-runtime (>=1.34.0,<1.35.0)", "mypy-boto3-sagemaker-edge (>=1.34.0,<1.35.0)", "mypy-boto3-sagemaker-featurestore-runtime (>=1.34.0,<1.35.0)", "mypy-boto3-sagemaker-geospatial (>=1.34.0,<1.35.0)", "mypy-boto3-sagemaker-metrics (>=1.34.0,<1.35.0)", "mypy-boto3-sagemaker-runtime (>=1.34.0,<1.35.0)", "mypy-boto3-savingsplans (>=1.34.0,<1.35.0)", "mypy-boto3-scheduler (>=1.34.0,<1.35.0)", "mypy-boto3-schemas (>=1.34.0,<1.35.0)", "mypy-boto3-sdb (>=1.34.0,<1.35.0)", "mypy-boto3-secretsmanager (>=1.34.0,<1.35.0)", "mypy-boto3-securityhub (>=1.34.0,<1.35.0)", "mypy-boto3-securitylake (>=1.34.0,<1.35.0)", "mypy-boto3-serverlessrepo (>=1.34.0,<1.35.0)", "mypy-boto3-service-quotas (>=1.34.0,<1.35.0)", "mypy-boto3-servicecatalog (>=1.34.0,<1.35.0)", "mypy-boto3-servicecatalog-appregistry (>=1.34.0,<1.35.0)", "mypy-boto3-servicediscovery (>=1.34.0,<1.35.0)", "mypy-boto3-ses (>=1.34.0,<1.35.0)", "mypy-boto3-sesv2 (>=1.34.0,<1.35.0)", "mypy-boto3-shield (>=1.34.0,<1.35.0)", "mypy-boto3-signer (>=1.34.0,<1.35.0)", "mypy-boto3-simspaceweaver (>=1.34.0,<1.35.0)", "mypy-boto3-sms (>=1.34.0,<1.35.0)", "mypy-boto3-sms-voice (>=1.34.0,<1.35.0)", "mypy-boto3-snow-device-management (>=1.34.0,<1.35.0)", "mypy-boto3-snowball (>=1.34.0,<1.35.0)", "mypy-boto3-sns (>=1.34.0,<1.35.0)", "mypy-boto3-sqs (>=1.34.0,<1.35.0)", "mypy-boto3-ssm (>=1.34.0,<1.35.0)", "mypy-boto3-ssm-contacts (>=1.34.0,<1.35.0)", "mypy-boto3-ssm-incidents (>=1.34.0,<1.35.0)", "mypy-boto3-ssm-sap (>=1.34.0,<1.35.0)", "mypy-boto3-sso (>=1.34.0,<1.35.0)", "mypy-boto3-sso-admin (>=1.34.0,<1.35.0)", "mypy-boto3-sso-oidc (>=1.34.0,<1.35.0)", "mypy-boto3-stepfunctions (>=1.34.0,<1.35.0)", "mypy-boto3-storagegateway (>=1.34.0,<1.35.0)", "mypy-boto3-sts (>=1.34.0,<1.35.0)", "mypy-boto3-supplychain (>=1.34.0,<1.35.0)", "mypy-boto3-support (>=1.34.0,<1.35.0)", "mypy-boto3-support-app (>=1.34.0,<1.35.0)", "mypy-boto3-swf (>=1.34.0,<1.35.0)", "mypy-boto3-synthetics (>=1.34.0,<1.35.0)", "mypy-boto3-textract (>=1.34.0,<1.35.0)", "mypy-boto3-timestream-influxdb (>=1.34.0,<1.35.0)", "mypy-boto3-timestream-query (>=1.34.0,<1.35.0)", "mypy-boto3-timestream-write (>=1.34.0,<1.35.0)", "mypy-boto3-tnb (>=1.34.0,<1.35.0)", "mypy-boto3-transcribe (>=1.34.0,<1.35.0)", "mypy-boto3-transfer (>=1.34.0,<1.35.0)", "mypy-boto3-translate (>=1.34.0,<1.35.0)", "mypy-boto3-trustedadvisor (>=1.34.0,<1.35.0)", "mypy-boto3-verifiedpermissions (>=1.34.0,<1.35.0)", "mypy-boto3-voice-id (>=1.34.0,<1.35.0)", "mypy-boto3-vpc-lattice (>=1.34.0,<1.35.0)", "mypy-boto3-waf (>=1.34.0,<1.35.0)", "mypy-boto3-waf-regional (>=1.34.0,<1.35.0)", "mypy-boto3-wafv2 (>=1.34.0,<1.35.0)", "mypy-boto3-wellarchitected (>=1.34.0,<1.35.0)", "mypy-boto3-wisdom (>=1.34.0,<1.35.0)", "mypy-boto3-workdocs (>=1.34.0,<1.35.0)", "mypy-boto3-worklink (>=1.34.0,<1.35.0)", "mypy-boto3-workmail (>=1.34.0,<1.35.0)", "mypy-boto3-workmailmessageflow (>=1.34.0,<1.35.0)", "mypy-boto3-workspaces (>=1.34.0,<1.35.0)", "mypy-boto3-workspaces-thin-client (>=1.34.0,<1.35.0)", "mypy-boto3-workspaces-web (>=1.34.0,<1.35.0)", "mypy-boto3-xray (>=1.34.0,<1.35.0)"] +amp = ["mypy-boto3-amp (>=1.34.0,<1.35.0)"] +amplify = ["mypy-boto3-amplify (>=1.34.0,<1.35.0)"] +amplifybackend = ["mypy-boto3-amplifybackend (>=1.34.0,<1.35.0)"] +amplifyuibuilder = ["mypy-boto3-amplifyuibuilder (>=1.34.0,<1.35.0)"] +apigateway = ["mypy-boto3-apigateway (>=1.34.0,<1.35.0)"] +apigatewaymanagementapi = ["mypy-boto3-apigatewaymanagementapi (>=1.34.0,<1.35.0)"] +apigatewayv2 = ["mypy-boto3-apigatewayv2 (>=1.34.0,<1.35.0)"] +appconfig = ["mypy-boto3-appconfig (>=1.34.0,<1.35.0)"] +appconfigdata = ["mypy-boto3-appconfigdata (>=1.34.0,<1.35.0)"] +appfabric = ["mypy-boto3-appfabric (>=1.34.0,<1.35.0)"] +appflow = ["mypy-boto3-appflow (>=1.34.0,<1.35.0)"] +appintegrations = ["mypy-boto3-appintegrations (>=1.34.0,<1.35.0)"] +application-autoscaling = ["mypy-boto3-application-autoscaling (>=1.34.0,<1.35.0)"] +application-insights = ["mypy-boto3-application-insights (>=1.34.0,<1.35.0)"] +applicationcostprofiler = ["mypy-boto3-applicationcostprofiler (>=1.34.0,<1.35.0)"] +appmesh = ["mypy-boto3-appmesh (>=1.34.0,<1.35.0)"] +apprunner = ["mypy-boto3-apprunner (>=1.34.0,<1.35.0)"] +appstream = ["mypy-boto3-appstream (>=1.34.0,<1.35.0)"] +appsync = ["mypy-boto3-appsync (>=1.34.0,<1.35.0)"] +arc-zonal-shift = ["mypy-boto3-arc-zonal-shift (>=1.34.0,<1.35.0)"] +artifact = ["mypy-boto3-artifact (>=1.34.0,<1.35.0)"] +athena = ["mypy-boto3-athena (>=1.34.0,<1.35.0)"] +auditmanager = ["mypy-boto3-auditmanager (>=1.34.0,<1.35.0)"] +autoscaling = ["mypy-boto3-autoscaling (>=1.34.0,<1.35.0)"] +autoscaling-plans = ["mypy-boto3-autoscaling-plans (>=1.34.0,<1.35.0)"] +b2bi = ["mypy-boto3-b2bi (>=1.34.0,<1.35.0)"] +backup = ["mypy-boto3-backup (>=1.34.0,<1.35.0)"] +backup-gateway = ["mypy-boto3-backup-gateway (>=1.34.0,<1.35.0)"] +backupstorage = ["mypy-boto3-backupstorage (>=1.34.0,<1.35.0)"] +batch = ["mypy-boto3-batch (>=1.34.0,<1.35.0)"] +bcm-data-exports = ["mypy-boto3-bcm-data-exports (>=1.34.0,<1.35.0)"] +bedrock = ["mypy-boto3-bedrock (>=1.34.0,<1.35.0)"] +bedrock-agent = ["mypy-boto3-bedrock-agent (>=1.34.0,<1.35.0)"] +bedrock-agent-runtime = ["mypy-boto3-bedrock-agent-runtime (>=1.34.0,<1.35.0)"] +bedrock-runtime = ["mypy-boto3-bedrock-runtime (>=1.34.0,<1.35.0)"] +billingconductor = ["mypy-boto3-billingconductor (>=1.34.0,<1.35.0)"] +boto3 = ["boto3 (==1.34.82)", "botocore (==1.34.82)"] +braket = ["mypy-boto3-braket (>=1.34.0,<1.35.0)"] +budgets = ["mypy-boto3-budgets (>=1.34.0,<1.35.0)"] +ce = ["mypy-boto3-ce (>=1.34.0,<1.35.0)"] +chatbot = ["mypy-boto3-chatbot (>=1.34.0,<1.35.0)"] +chime = ["mypy-boto3-chime (>=1.34.0,<1.35.0)"] +chime-sdk-identity = ["mypy-boto3-chime-sdk-identity (>=1.34.0,<1.35.0)"] +chime-sdk-media-pipelines = ["mypy-boto3-chime-sdk-media-pipelines (>=1.34.0,<1.35.0)"] +chime-sdk-meetings = ["mypy-boto3-chime-sdk-meetings (>=1.34.0,<1.35.0)"] +chime-sdk-messaging = ["mypy-boto3-chime-sdk-messaging (>=1.34.0,<1.35.0)"] +chime-sdk-voice = ["mypy-boto3-chime-sdk-voice (>=1.34.0,<1.35.0)"] +cleanrooms = ["mypy-boto3-cleanrooms (>=1.34.0,<1.35.0)"] +cleanroomsml = ["mypy-boto3-cleanroomsml (>=1.34.0,<1.35.0)"] +cloud9 = ["mypy-boto3-cloud9 (>=1.34.0,<1.35.0)"] +cloudcontrol = ["mypy-boto3-cloudcontrol (>=1.34.0,<1.35.0)"] +clouddirectory = ["mypy-boto3-clouddirectory (>=1.34.0,<1.35.0)"] +cloudformation = ["mypy-boto3-cloudformation (>=1.34.0,<1.35.0)"] +cloudfront = ["mypy-boto3-cloudfront (>=1.34.0,<1.35.0)"] +cloudfront-keyvaluestore = ["mypy-boto3-cloudfront-keyvaluestore (>=1.34.0,<1.35.0)"] +cloudhsm = ["mypy-boto3-cloudhsm (>=1.34.0,<1.35.0)"] +cloudhsmv2 = ["mypy-boto3-cloudhsmv2 (>=1.34.0,<1.35.0)"] +cloudsearch = ["mypy-boto3-cloudsearch (>=1.34.0,<1.35.0)"] +cloudsearchdomain = ["mypy-boto3-cloudsearchdomain (>=1.34.0,<1.35.0)"] +cloudtrail = ["mypy-boto3-cloudtrail (>=1.34.0,<1.35.0)"] +cloudtrail-data = ["mypy-boto3-cloudtrail-data (>=1.34.0,<1.35.0)"] +cloudwatch = ["mypy-boto3-cloudwatch (>=1.34.0,<1.35.0)"] +codeartifact = ["mypy-boto3-codeartifact (>=1.34.0,<1.35.0)"] +codebuild = ["mypy-boto3-codebuild (>=1.34.0,<1.35.0)"] +codecatalyst = ["mypy-boto3-codecatalyst (>=1.34.0,<1.35.0)"] +codecommit = ["mypy-boto3-codecommit (>=1.34.0,<1.35.0)"] +codeconnections = ["mypy-boto3-codeconnections (>=1.34.0,<1.35.0)"] +codedeploy = ["mypy-boto3-codedeploy (>=1.34.0,<1.35.0)"] +codeguru-reviewer = ["mypy-boto3-codeguru-reviewer (>=1.34.0,<1.35.0)"] +codeguru-security = ["mypy-boto3-codeguru-security (>=1.34.0,<1.35.0)"] +codeguruprofiler = ["mypy-boto3-codeguruprofiler (>=1.34.0,<1.35.0)"] +codepipeline = ["mypy-boto3-codepipeline (>=1.34.0,<1.35.0)"] +codestar = ["mypy-boto3-codestar (>=1.34.0,<1.35.0)"] +codestar-connections = ["mypy-boto3-codestar-connections (>=1.34.0,<1.35.0)"] +codestar-notifications = ["mypy-boto3-codestar-notifications (>=1.34.0,<1.35.0)"] +cognito-identity = ["mypy-boto3-cognito-identity (>=1.34.0,<1.35.0)"] +cognito-idp = ["mypy-boto3-cognito-idp (>=1.34.0,<1.35.0)"] +cognito-sync = ["mypy-boto3-cognito-sync (>=1.34.0,<1.35.0)"] +comprehend = ["mypy-boto3-comprehend (>=1.34.0,<1.35.0)"] +comprehendmedical = ["mypy-boto3-comprehendmedical (>=1.34.0,<1.35.0)"] +compute-optimizer = ["mypy-boto3-compute-optimizer (>=1.34.0,<1.35.0)"] +config = ["mypy-boto3-config (>=1.34.0,<1.35.0)"] +connect = ["mypy-boto3-connect (>=1.34.0,<1.35.0)"] +connect-contact-lens = ["mypy-boto3-connect-contact-lens (>=1.34.0,<1.35.0)"] +connectcampaigns = ["mypy-boto3-connectcampaigns (>=1.34.0,<1.35.0)"] +connectcases = ["mypy-boto3-connectcases (>=1.34.0,<1.35.0)"] +connectparticipant = ["mypy-boto3-connectparticipant (>=1.34.0,<1.35.0)"] +controlcatalog = ["mypy-boto3-controlcatalog (>=1.34.0,<1.35.0)"] +controltower = ["mypy-boto3-controltower (>=1.34.0,<1.35.0)"] +cost-optimization-hub = ["mypy-boto3-cost-optimization-hub (>=1.34.0,<1.35.0)"] +cur = ["mypy-boto3-cur (>=1.34.0,<1.35.0)"] +customer-profiles = ["mypy-boto3-customer-profiles (>=1.34.0,<1.35.0)"] +databrew = ["mypy-boto3-databrew (>=1.34.0,<1.35.0)"] +dataexchange = ["mypy-boto3-dataexchange (>=1.34.0,<1.35.0)"] +datapipeline = ["mypy-boto3-datapipeline (>=1.34.0,<1.35.0)"] +datasync = ["mypy-boto3-datasync (>=1.34.0,<1.35.0)"] +datazone = ["mypy-boto3-datazone (>=1.34.0,<1.35.0)"] +dax = ["mypy-boto3-dax (>=1.34.0,<1.35.0)"] +deadline = ["mypy-boto3-deadline (>=1.34.0,<1.35.0)"] +detective = ["mypy-boto3-detective (>=1.34.0,<1.35.0)"] +devicefarm = ["mypy-boto3-devicefarm (>=1.34.0,<1.35.0)"] +devops-guru = ["mypy-boto3-devops-guru (>=1.34.0,<1.35.0)"] +directconnect = ["mypy-boto3-directconnect (>=1.34.0,<1.35.0)"] +discovery = ["mypy-boto3-discovery (>=1.34.0,<1.35.0)"] +dlm = ["mypy-boto3-dlm (>=1.34.0,<1.35.0)"] +dms = ["mypy-boto3-dms (>=1.34.0,<1.35.0)"] +docdb = ["mypy-boto3-docdb (>=1.34.0,<1.35.0)"] +docdb-elastic = ["mypy-boto3-docdb-elastic (>=1.34.0,<1.35.0)"] +drs = ["mypy-boto3-drs (>=1.34.0,<1.35.0)"] +ds = ["mypy-boto3-ds (>=1.34.0,<1.35.0)"] +dynamodb = ["mypy-boto3-dynamodb (>=1.34.0,<1.35.0)"] +dynamodbstreams = ["mypy-boto3-dynamodbstreams (>=1.34.0,<1.35.0)"] +ebs = ["mypy-boto3-ebs (>=1.34.0,<1.35.0)"] +ec2 = ["mypy-boto3-ec2 (>=1.34.0,<1.35.0)"] +ec2-instance-connect = ["mypy-boto3-ec2-instance-connect (>=1.34.0,<1.35.0)"] +ecr = ["mypy-boto3-ecr (>=1.34.0,<1.35.0)"] +ecr-public = ["mypy-boto3-ecr-public (>=1.34.0,<1.35.0)"] +ecs = ["mypy-boto3-ecs (>=1.34.0,<1.35.0)"] +efs = ["mypy-boto3-efs (>=1.34.0,<1.35.0)"] +eks = ["mypy-boto3-eks (>=1.34.0,<1.35.0)"] +eks-auth = ["mypy-boto3-eks-auth (>=1.34.0,<1.35.0)"] +elastic-inference = ["mypy-boto3-elastic-inference (>=1.34.0,<1.35.0)"] +elasticache = ["mypy-boto3-elasticache (>=1.34.0,<1.35.0)"] +elasticbeanstalk = ["mypy-boto3-elasticbeanstalk (>=1.34.0,<1.35.0)"] +elastictranscoder = ["mypy-boto3-elastictranscoder (>=1.34.0,<1.35.0)"] +elb = ["mypy-boto3-elb (>=1.34.0,<1.35.0)"] +elbv2 = ["mypy-boto3-elbv2 (>=1.34.0,<1.35.0)"] +emr = ["mypy-boto3-emr (>=1.34.0,<1.35.0)"] +emr-containers = ["mypy-boto3-emr-containers (>=1.34.0,<1.35.0)"] +emr-serverless = ["mypy-boto3-emr-serverless (>=1.34.0,<1.35.0)"] +entityresolution = ["mypy-boto3-entityresolution (>=1.34.0,<1.35.0)"] +es = ["mypy-boto3-es (>=1.34.0,<1.35.0)"] +essential = ["mypy-boto3-cloudformation (>=1.34.0,<1.35.0)", "mypy-boto3-dynamodb (>=1.34.0,<1.35.0)", "mypy-boto3-ec2 (>=1.34.0,<1.35.0)", "mypy-boto3-lambda (>=1.34.0,<1.35.0)", "mypy-boto3-rds (>=1.34.0,<1.35.0)", "mypy-boto3-s3 (>=1.34.0,<1.35.0)", "mypy-boto3-sqs (>=1.34.0,<1.35.0)"] +events = ["mypy-boto3-events (>=1.34.0,<1.35.0)"] +evidently = ["mypy-boto3-evidently (>=1.34.0,<1.35.0)"] +finspace = ["mypy-boto3-finspace (>=1.34.0,<1.35.0)"] +finspace-data = ["mypy-boto3-finspace-data (>=1.34.0,<1.35.0)"] +firehose = ["mypy-boto3-firehose (>=1.34.0,<1.35.0)"] +fis = ["mypy-boto3-fis (>=1.34.0,<1.35.0)"] +fms = ["mypy-boto3-fms (>=1.34.0,<1.35.0)"] +forecast = ["mypy-boto3-forecast (>=1.34.0,<1.35.0)"] +forecastquery = ["mypy-boto3-forecastquery (>=1.34.0,<1.35.0)"] +frauddetector = ["mypy-boto3-frauddetector (>=1.34.0,<1.35.0)"] +freetier = ["mypy-boto3-freetier (>=1.34.0,<1.35.0)"] +fsx = ["mypy-boto3-fsx (>=1.34.0,<1.35.0)"] +gamelift = ["mypy-boto3-gamelift (>=1.34.0,<1.35.0)"] +glacier = ["mypy-boto3-glacier (>=1.34.0,<1.35.0)"] +globalaccelerator = ["mypy-boto3-globalaccelerator (>=1.34.0,<1.35.0)"] +glue = ["mypy-boto3-glue (>=1.34.0,<1.35.0)"] +grafana = ["mypy-boto3-grafana (>=1.34.0,<1.35.0)"] +greengrass = ["mypy-boto3-greengrass (>=1.34.0,<1.35.0)"] +greengrassv2 = ["mypy-boto3-greengrassv2 (>=1.34.0,<1.35.0)"] +groundstation = ["mypy-boto3-groundstation (>=1.34.0,<1.35.0)"] +guardduty = ["mypy-boto3-guardduty (>=1.34.0,<1.35.0)"] +health = ["mypy-boto3-health (>=1.34.0,<1.35.0)"] +healthlake = ["mypy-boto3-healthlake (>=1.34.0,<1.35.0)"] +honeycode = ["mypy-boto3-honeycode (>=1.34.0,<1.35.0)"] +iam = ["mypy-boto3-iam (>=1.34.0,<1.35.0)"] +identitystore = ["mypy-boto3-identitystore (>=1.34.0,<1.35.0)"] +imagebuilder = ["mypy-boto3-imagebuilder (>=1.34.0,<1.35.0)"] +importexport = ["mypy-boto3-importexport (>=1.34.0,<1.35.0)"] +inspector = ["mypy-boto3-inspector (>=1.34.0,<1.35.0)"] +inspector-scan = ["mypy-boto3-inspector-scan (>=1.34.0,<1.35.0)"] +inspector2 = ["mypy-boto3-inspector2 (>=1.34.0,<1.35.0)"] +internetmonitor = ["mypy-boto3-internetmonitor (>=1.34.0,<1.35.0)"] +iot = ["mypy-boto3-iot (>=1.34.0,<1.35.0)"] +iot-data = ["mypy-boto3-iot-data (>=1.34.0,<1.35.0)"] +iot-jobs-data = ["mypy-boto3-iot-jobs-data (>=1.34.0,<1.35.0)"] +iot1click-devices = ["mypy-boto3-iot1click-devices (>=1.34.0,<1.35.0)"] +iot1click-projects = ["mypy-boto3-iot1click-projects (>=1.34.0,<1.35.0)"] +iotanalytics = ["mypy-boto3-iotanalytics (>=1.34.0,<1.35.0)"] +iotdeviceadvisor = ["mypy-boto3-iotdeviceadvisor (>=1.34.0,<1.35.0)"] +iotevents = ["mypy-boto3-iotevents (>=1.34.0,<1.35.0)"] +iotevents-data = ["mypy-boto3-iotevents-data (>=1.34.0,<1.35.0)"] +iotfleethub = ["mypy-boto3-iotfleethub (>=1.34.0,<1.35.0)"] +iotfleetwise = ["mypy-boto3-iotfleetwise (>=1.34.0,<1.35.0)"] +iotsecuretunneling = ["mypy-boto3-iotsecuretunneling (>=1.34.0,<1.35.0)"] +iotsitewise = ["mypy-boto3-iotsitewise (>=1.34.0,<1.35.0)"] +iotthingsgraph = ["mypy-boto3-iotthingsgraph (>=1.34.0,<1.35.0)"] +iottwinmaker = ["mypy-boto3-iottwinmaker (>=1.34.0,<1.35.0)"] +iotwireless = ["mypy-boto3-iotwireless (>=1.34.0,<1.35.0)"] +ivs = ["mypy-boto3-ivs (>=1.34.0,<1.35.0)"] +ivs-realtime = ["mypy-boto3-ivs-realtime (>=1.34.0,<1.35.0)"] +ivschat = ["mypy-boto3-ivschat (>=1.34.0,<1.35.0)"] +kafka = ["mypy-boto3-kafka (>=1.34.0,<1.35.0)"] +kafkaconnect = ["mypy-boto3-kafkaconnect (>=1.34.0,<1.35.0)"] +kendra = ["mypy-boto3-kendra (>=1.34.0,<1.35.0)"] +kendra-ranking = ["mypy-boto3-kendra-ranking (>=1.34.0,<1.35.0)"] +keyspaces = ["mypy-boto3-keyspaces (>=1.34.0,<1.35.0)"] +kinesis = ["mypy-boto3-kinesis (>=1.34.0,<1.35.0)"] +kinesis-video-archived-media = ["mypy-boto3-kinesis-video-archived-media (>=1.34.0,<1.35.0)"] +kinesis-video-media = ["mypy-boto3-kinesis-video-media (>=1.34.0,<1.35.0)"] +kinesis-video-signaling = ["mypy-boto3-kinesis-video-signaling (>=1.34.0,<1.35.0)"] +kinesis-video-webrtc-storage = ["mypy-boto3-kinesis-video-webrtc-storage (>=1.34.0,<1.35.0)"] +kinesisanalytics = ["mypy-boto3-kinesisanalytics (>=1.34.0,<1.35.0)"] +kinesisanalyticsv2 = ["mypy-boto3-kinesisanalyticsv2 (>=1.34.0,<1.35.0)"] +kinesisvideo = ["mypy-boto3-kinesisvideo (>=1.34.0,<1.35.0)"] +kms = ["mypy-boto3-kms (>=1.34.0,<1.35.0)"] +lakeformation = ["mypy-boto3-lakeformation (>=1.34.0,<1.35.0)"] +lambda = ["mypy-boto3-lambda (>=1.34.0,<1.35.0)"] +launch-wizard = ["mypy-boto3-launch-wizard (>=1.34.0,<1.35.0)"] +lex-models = ["mypy-boto3-lex-models (>=1.34.0,<1.35.0)"] +lex-runtime = ["mypy-boto3-lex-runtime (>=1.34.0,<1.35.0)"] +lexv2-models = ["mypy-boto3-lexv2-models (>=1.34.0,<1.35.0)"] +lexv2-runtime = ["mypy-boto3-lexv2-runtime (>=1.34.0,<1.35.0)"] +license-manager = ["mypy-boto3-license-manager (>=1.34.0,<1.35.0)"] +license-manager-linux-subscriptions = ["mypy-boto3-license-manager-linux-subscriptions (>=1.34.0,<1.35.0)"] +license-manager-user-subscriptions = ["mypy-boto3-license-manager-user-subscriptions (>=1.34.0,<1.35.0)"] +lightsail = ["mypy-boto3-lightsail (>=1.34.0,<1.35.0)"] +location = ["mypy-boto3-location (>=1.34.0,<1.35.0)"] +logs = ["mypy-boto3-logs (>=1.34.0,<1.35.0)"] +lookoutequipment = ["mypy-boto3-lookoutequipment (>=1.34.0,<1.35.0)"] +lookoutmetrics = ["mypy-boto3-lookoutmetrics (>=1.34.0,<1.35.0)"] +lookoutvision = ["mypy-boto3-lookoutvision (>=1.34.0,<1.35.0)"] +m2 = ["mypy-boto3-m2 (>=1.34.0,<1.35.0)"] +machinelearning = ["mypy-boto3-machinelearning (>=1.34.0,<1.35.0)"] +macie2 = ["mypy-boto3-macie2 (>=1.34.0,<1.35.0)"] +managedblockchain = ["mypy-boto3-managedblockchain (>=1.34.0,<1.35.0)"] +managedblockchain-query = ["mypy-boto3-managedblockchain-query (>=1.34.0,<1.35.0)"] +marketplace-agreement = ["mypy-boto3-marketplace-agreement (>=1.34.0,<1.35.0)"] +marketplace-catalog = ["mypy-boto3-marketplace-catalog (>=1.34.0,<1.35.0)"] +marketplace-deployment = ["mypy-boto3-marketplace-deployment (>=1.34.0,<1.35.0)"] +marketplace-entitlement = ["mypy-boto3-marketplace-entitlement (>=1.34.0,<1.35.0)"] +marketplacecommerceanalytics = ["mypy-boto3-marketplacecommerceanalytics (>=1.34.0,<1.35.0)"] +mediaconnect = ["mypy-boto3-mediaconnect (>=1.34.0,<1.35.0)"] +mediaconvert = ["mypy-boto3-mediaconvert (>=1.34.0,<1.35.0)"] +medialive = ["mypy-boto3-medialive (>=1.34.0,<1.35.0)"] +mediapackage = ["mypy-boto3-mediapackage (>=1.34.0,<1.35.0)"] +mediapackage-vod = ["mypy-boto3-mediapackage-vod (>=1.34.0,<1.35.0)"] +mediapackagev2 = ["mypy-boto3-mediapackagev2 (>=1.34.0,<1.35.0)"] +mediastore = ["mypy-boto3-mediastore (>=1.34.0,<1.35.0)"] +mediastore-data = ["mypy-boto3-mediastore-data (>=1.34.0,<1.35.0)"] +mediatailor = ["mypy-boto3-mediatailor (>=1.34.0,<1.35.0)"] +medical-imaging = ["mypy-boto3-medical-imaging (>=1.34.0,<1.35.0)"] +memorydb = ["mypy-boto3-memorydb (>=1.34.0,<1.35.0)"] +meteringmarketplace = ["mypy-boto3-meteringmarketplace (>=1.34.0,<1.35.0)"] +mgh = ["mypy-boto3-mgh (>=1.34.0,<1.35.0)"] +mgn = ["mypy-boto3-mgn (>=1.34.0,<1.35.0)"] +migration-hub-refactor-spaces = ["mypy-boto3-migration-hub-refactor-spaces (>=1.34.0,<1.35.0)"] +migrationhub-config = ["mypy-boto3-migrationhub-config (>=1.34.0,<1.35.0)"] +migrationhuborchestrator = ["mypy-boto3-migrationhuborchestrator (>=1.34.0,<1.35.0)"] +migrationhubstrategy = ["mypy-boto3-migrationhubstrategy (>=1.34.0,<1.35.0)"] +mobile = ["mypy-boto3-mobile (>=1.34.0,<1.35.0)"] +mq = ["mypy-boto3-mq (>=1.34.0,<1.35.0)"] +mturk = ["mypy-boto3-mturk (>=1.34.0,<1.35.0)"] +mwaa = ["mypy-boto3-mwaa (>=1.34.0,<1.35.0)"] +neptune = ["mypy-boto3-neptune (>=1.34.0,<1.35.0)"] +neptune-graph = ["mypy-boto3-neptune-graph (>=1.34.0,<1.35.0)"] +neptunedata = ["mypy-boto3-neptunedata (>=1.34.0,<1.35.0)"] +network-firewall = ["mypy-boto3-network-firewall (>=1.34.0,<1.35.0)"] +networkmanager = ["mypy-boto3-networkmanager (>=1.34.0,<1.35.0)"] +networkmonitor = ["mypy-boto3-networkmonitor (>=1.34.0,<1.35.0)"] +nimble = ["mypy-boto3-nimble (>=1.34.0,<1.35.0)"] +oam = ["mypy-boto3-oam (>=1.34.0,<1.35.0)"] +omics = ["mypy-boto3-omics (>=1.34.0,<1.35.0)"] +opensearch = ["mypy-boto3-opensearch (>=1.34.0,<1.35.0)"] +opensearchserverless = ["mypy-boto3-opensearchserverless (>=1.34.0,<1.35.0)"] +opsworks = ["mypy-boto3-opsworks (>=1.34.0,<1.35.0)"] +opsworkscm = ["mypy-boto3-opsworkscm (>=1.34.0,<1.35.0)"] +organizations = ["mypy-boto3-organizations (>=1.34.0,<1.35.0)"] +osis = ["mypy-boto3-osis (>=1.34.0,<1.35.0)"] +outposts = ["mypy-boto3-outposts (>=1.34.0,<1.35.0)"] +panorama = ["mypy-boto3-panorama (>=1.34.0,<1.35.0)"] +payment-cryptography = ["mypy-boto3-payment-cryptography (>=1.34.0,<1.35.0)"] +payment-cryptography-data = ["mypy-boto3-payment-cryptography-data (>=1.34.0,<1.35.0)"] +pca-connector-ad = ["mypy-boto3-pca-connector-ad (>=1.34.0,<1.35.0)"] +personalize = ["mypy-boto3-personalize (>=1.34.0,<1.35.0)"] +personalize-events = ["mypy-boto3-personalize-events (>=1.34.0,<1.35.0)"] +personalize-runtime = ["mypy-boto3-personalize-runtime (>=1.34.0,<1.35.0)"] +pi = ["mypy-boto3-pi (>=1.34.0,<1.35.0)"] +pinpoint = ["mypy-boto3-pinpoint (>=1.34.0,<1.35.0)"] +pinpoint-email = ["mypy-boto3-pinpoint-email (>=1.34.0,<1.35.0)"] +pinpoint-sms-voice = ["mypy-boto3-pinpoint-sms-voice (>=1.34.0,<1.35.0)"] +pinpoint-sms-voice-v2 = ["mypy-boto3-pinpoint-sms-voice-v2 (>=1.34.0,<1.35.0)"] +pipes = ["mypy-boto3-pipes (>=1.34.0,<1.35.0)"] +polly = ["mypy-boto3-polly (>=1.34.0,<1.35.0)"] +pricing = ["mypy-boto3-pricing (>=1.34.0,<1.35.0)"] +privatenetworks = ["mypy-boto3-privatenetworks (>=1.34.0,<1.35.0)"] +proton = ["mypy-boto3-proton (>=1.34.0,<1.35.0)"] +qbusiness = ["mypy-boto3-qbusiness (>=1.34.0,<1.35.0)"] +qconnect = ["mypy-boto3-qconnect (>=1.34.0,<1.35.0)"] +qldb = ["mypy-boto3-qldb (>=1.34.0,<1.35.0)"] +qldb-session = ["mypy-boto3-qldb-session (>=1.34.0,<1.35.0)"] +quicksight = ["mypy-boto3-quicksight (>=1.34.0,<1.35.0)"] +ram = ["mypy-boto3-ram (>=1.34.0,<1.35.0)"] +rbin = ["mypy-boto3-rbin (>=1.34.0,<1.35.0)"] +rds = ["mypy-boto3-rds (>=1.34.0,<1.35.0)"] +rds-data = ["mypy-boto3-rds-data (>=1.34.0,<1.35.0)"] +redshift = ["mypy-boto3-redshift (>=1.34.0,<1.35.0)"] +redshift-data = ["mypy-boto3-redshift-data (>=1.34.0,<1.35.0)"] +redshift-serverless = ["mypy-boto3-redshift-serverless (>=1.34.0,<1.35.0)"] +rekognition = ["mypy-boto3-rekognition (>=1.34.0,<1.35.0)"] +repostspace = ["mypy-boto3-repostspace (>=1.34.0,<1.35.0)"] +resiliencehub = ["mypy-boto3-resiliencehub (>=1.34.0,<1.35.0)"] +resource-explorer-2 = ["mypy-boto3-resource-explorer-2 (>=1.34.0,<1.35.0)"] +resource-groups = ["mypy-boto3-resource-groups (>=1.34.0,<1.35.0)"] +resourcegroupstaggingapi = ["mypy-boto3-resourcegroupstaggingapi (>=1.34.0,<1.35.0)"] +robomaker = ["mypy-boto3-robomaker (>=1.34.0,<1.35.0)"] +rolesanywhere = ["mypy-boto3-rolesanywhere (>=1.34.0,<1.35.0)"] +route53 = ["mypy-boto3-route53 (>=1.34.0,<1.35.0)"] +route53-recovery-cluster = ["mypy-boto3-route53-recovery-cluster (>=1.34.0,<1.35.0)"] +route53-recovery-control-config = ["mypy-boto3-route53-recovery-control-config (>=1.34.0,<1.35.0)"] +route53-recovery-readiness = ["mypy-boto3-route53-recovery-readiness (>=1.34.0,<1.35.0)"] +route53domains = ["mypy-boto3-route53domains (>=1.34.0,<1.35.0)"] +route53resolver = ["mypy-boto3-route53resolver (>=1.34.0,<1.35.0)"] +rum = ["mypy-boto3-rum (>=1.34.0,<1.35.0)"] +s3 = ["mypy-boto3-s3 (>=1.34.0,<1.35.0)"] +s3control = ["mypy-boto3-s3control (>=1.34.0,<1.35.0)"] +s3outposts = ["mypy-boto3-s3outposts (>=1.34.0,<1.35.0)"] +sagemaker = ["mypy-boto3-sagemaker (>=1.34.0,<1.35.0)"] +sagemaker-a2i-runtime = ["mypy-boto3-sagemaker-a2i-runtime (>=1.34.0,<1.35.0)"] +sagemaker-edge = ["mypy-boto3-sagemaker-edge (>=1.34.0,<1.35.0)"] +sagemaker-featurestore-runtime = ["mypy-boto3-sagemaker-featurestore-runtime (>=1.34.0,<1.35.0)"] +sagemaker-geospatial = ["mypy-boto3-sagemaker-geospatial (>=1.34.0,<1.35.0)"] +sagemaker-metrics = ["mypy-boto3-sagemaker-metrics (>=1.34.0,<1.35.0)"] +sagemaker-runtime = ["mypy-boto3-sagemaker-runtime (>=1.34.0,<1.35.0)"] +savingsplans = ["mypy-boto3-savingsplans (>=1.34.0,<1.35.0)"] +scheduler = ["mypy-boto3-scheduler (>=1.34.0,<1.35.0)"] +schemas = ["mypy-boto3-schemas (>=1.34.0,<1.35.0)"] +sdb = ["mypy-boto3-sdb (>=1.34.0,<1.35.0)"] +secretsmanager = ["mypy-boto3-secretsmanager (>=1.34.0,<1.35.0)"] +securityhub = ["mypy-boto3-securityhub (>=1.34.0,<1.35.0)"] +securitylake = ["mypy-boto3-securitylake (>=1.34.0,<1.35.0)"] +serverlessrepo = ["mypy-boto3-serverlessrepo (>=1.34.0,<1.35.0)"] +service-quotas = ["mypy-boto3-service-quotas (>=1.34.0,<1.35.0)"] +servicecatalog = ["mypy-boto3-servicecatalog (>=1.34.0,<1.35.0)"] +servicecatalog-appregistry = ["mypy-boto3-servicecatalog-appregistry (>=1.34.0,<1.35.0)"] +servicediscovery = ["mypy-boto3-servicediscovery (>=1.34.0,<1.35.0)"] +ses = ["mypy-boto3-ses (>=1.34.0,<1.35.0)"] +sesv2 = ["mypy-boto3-sesv2 (>=1.34.0,<1.35.0)"] +shield = ["mypy-boto3-shield (>=1.34.0,<1.35.0)"] +signer = ["mypy-boto3-signer (>=1.34.0,<1.35.0)"] +simspaceweaver = ["mypy-boto3-simspaceweaver (>=1.34.0,<1.35.0)"] +sms = ["mypy-boto3-sms (>=1.34.0,<1.35.0)"] +sms-voice = ["mypy-boto3-sms-voice (>=1.34.0,<1.35.0)"] +snow-device-management = ["mypy-boto3-snow-device-management (>=1.34.0,<1.35.0)"] +snowball = ["mypy-boto3-snowball (>=1.34.0,<1.35.0)"] +sns = ["mypy-boto3-sns (>=1.34.0,<1.35.0)"] +sqs = ["mypy-boto3-sqs (>=1.34.0,<1.35.0)"] +ssm = ["mypy-boto3-ssm (>=1.34.0,<1.35.0)"] +ssm-contacts = ["mypy-boto3-ssm-contacts (>=1.34.0,<1.35.0)"] +ssm-incidents = ["mypy-boto3-ssm-incidents (>=1.34.0,<1.35.0)"] +ssm-sap = ["mypy-boto3-ssm-sap (>=1.34.0,<1.35.0)"] +sso = ["mypy-boto3-sso (>=1.34.0,<1.35.0)"] +sso-admin = ["mypy-boto3-sso-admin (>=1.34.0,<1.35.0)"] +sso-oidc = ["mypy-boto3-sso-oidc (>=1.34.0,<1.35.0)"] +stepfunctions = ["mypy-boto3-stepfunctions (>=1.34.0,<1.35.0)"] +storagegateway = ["mypy-boto3-storagegateway (>=1.34.0,<1.35.0)"] +sts = ["mypy-boto3-sts (>=1.34.0,<1.35.0)"] +supplychain = ["mypy-boto3-supplychain (>=1.34.0,<1.35.0)"] +support = ["mypy-boto3-support (>=1.34.0,<1.35.0)"] +support-app = ["mypy-boto3-support-app (>=1.34.0,<1.35.0)"] +swf = ["mypy-boto3-swf (>=1.34.0,<1.35.0)"] +synthetics = ["mypy-boto3-synthetics (>=1.34.0,<1.35.0)"] +textract = ["mypy-boto3-textract (>=1.34.0,<1.35.0)"] +timestream-influxdb = ["mypy-boto3-timestream-influxdb (>=1.34.0,<1.35.0)"] +timestream-query = ["mypy-boto3-timestream-query (>=1.34.0,<1.35.0)"] +timestream-write = ["mypy-boto3-timestream-write (>=1.34.0,<1.35.0)"] +tnb = ["mypy-boto3-tnb (>=1.34.0,<1.35.0)"] +transcribe = ["mypy-boto3-transcribe (>=1.34.0,<1.35.0)"] +transfer = ["mypy-boto3-transfer (>=1.34.0,<1.35.0)"] +translate = ["mypy-boto3-translate (>=1.34.0,<1.35.0)"] +trustedadvisor = ["mypy-boto3-trustedadvisor (>=1.34.0,<1.35.0)"] +verifiedpermissions = ["mypy-boto3-verifiedpermissions (>=1.34.0,<1.35.0)"] +voice-id = ["mypy-boto3-voice-id (>=1.34.0,<1.35.0)"] +vpc-lattice = ["mypy-boto3-vpc-lattice (>=1.34.0,<1.35.0)"] +waf = ["mypy-boto3-waf (>=1.34.0,<1.35.0)"] +waf-regional = ["mypy-boto3-waf-regional (>=1.34.0,<1.35.0)"] +wafv2 = ["mypy-boto3-wafv2 (>=1.34.0,<1.35.0)"] +wellarchitected = ["mypy-boto3-wellarchitected (>=1.34.0,<1.35.0)"] +wisdom = ["mypy-boto3-wisdom (>=1.34.0,<1.35.0)"] +workdocs = ["mypy-boto3-workdocs (>=1.34.0,<1.35.0)"] +worklink = ["mypy-boto3-worklink (>=1.34.0,<1.35.0)"] +workmail = ["mypy-boto3-workmail (>=1.34.0,<1.35.0)"] +workmailmessageflow = ["mypy-boto3-workmailmessageflow (>=1.34.0,<1.35.0)"] +workspaces = ["mypy-boto3-workspaces (>=1.34.0,<1.35.0)"] +workspaces-thin-client = ["mypy-boto3-workspaces-thin-client (>=1.34.0,<1.35.0)"] +workspaces-web = ["mypy-boto3-workspaces-web (>=1.34.0,<1.35.0)"] +xray = ["mypy-boto3-xray (>=1.34.0,<1.35.0)"] + +[[package]] +name = "botocore" +version = "1.34.82" +description = "Low-level, data-driven core of boto 3." +optional = false +python-versions = ">=3.8" +files = [ + {file = "botocore-1.34.82-py3-none-any.whl", hash = "sha256:8f839e9a88e7ac7185e406be4cf9926673374e8a6ecc295302f56f7e3c618692"}, + {file = "botocore-1.34.82.tar.gz", hash = "sha256:2fd14676152f9d64541099090cc64973fdf8232744256454de443583e35e497d"}, +] + +[package.dependencies] +jmespath = ">=0.7.1,<2.0.0" +python-dateutil = ">=2.1,<3.0.0" +urllib3 = {version = ">=1.25.4,<2.2.0 || >2.2.0,<3", markers = "python_version >= \"3.10\""} + +[package.extras] +crt = ["awscrt (==0.19.19)"] + +[[package]] +name = "botocore-stubs" +version = "1.34.69" +description = "Type annotations and code completion for botocore" +optional = false +python-versions = "<4.0,>=3.8" +files = [ + {file = "botocore_stubs-1.34.69-py3-none-any.whl", hash = "sha256:0c3835c775db1387246c1ba8063b197604462fba8603d9b36b5dc60297197b2f"}, + {file = "botocore_stubs-1.34.69.tar.gz", hash = "sha256:463248fd1d6e7b68a0c57bdd758d04c6bd0c5c2c3bfa81afdf9d64f0930b59bc"}, +] + +[package.dependencies] +types-awscrt = "*" + +[package.extras] +botocore = ["botocore"] + [[package]] name = "build" version = "1.2.1" @@ -951,6 +1410,17 @@ MarkupSafe = ">=2.0" [package.extras] i18n = ["Babel (>=2.7)"] +[[package]] +name = "jmespath" +version = "1.0.1" +description = "JSON Matching Expressions" +optional = false +python-versions = ">=3.7" +files = [ + {file = "jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980"}, + {file = "jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"}, +] + [[package]] name = "keyring" version = "25.1.0" @@ -1293,6 +1763,20 @@ files = [ {file = "multidict-6.0.5.tar.gz", hash = "sha256:f7e301075edaf50500f0b341543c41194d8df3ae5caf4702f2095f3ca73dd8da"}, ] +[[package]] +name = "mypy-boto3-s3" +version = "1.34.65" +description = "Type annotations for boto3.S3 1.34.65 service generated with mypy-boto3-builder 7.23.2" +optional = false +python-versions = ">=3.8" +files = [ + {file = "mypy-boto3-s3-1.34.65.tar.gz", hash = "sha256:2fcdf412ce2924b2f0b34db59abf06a9c0bbe4cd3361f14f0d2c1e211c0f7ddd"}, + {file = "mypy_boto3_s3-1.34.65-py3-none-any.whl", hash = "sha256:2aecfbe1c00654bc21f839068218d60123366954bf43a708baa50f9543e3f205"}, +] + +[package.dependencies] +typing-extensions = {version = ">=4.1.0", markers = "python_version < \"3.12\""} + [[package]] name = "nh3" version = "0.2.17" @@ -1587,6 +2071,20 @@ files = [ [package.dependencies] pytest = ">=7.0.0" +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +description = "Extensions to the standard Python datetime module" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"}, + {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"}, +] + +[package.dependencies] +six = ">=1.5" + [[package]] name = "python-lzo" version = "1.15" @@ -1762,6 +2260,23 @@ pygments = ">=2.13.0,<3.0.0" [package.extras] jupyter = ["ipywidgets (>=7.5.1,<9)"] +[[package]] +name = "s3transfer" +version = "0.10.1" +description = "An Amazon S3 Transfer Manager" +optional = false +python-versions = ">= 3.8" +files = [ + {file = "s3transfer-0.10.1-py3-none-any.whl", hash = "sha256:ceb252b11bcf87080fb7850a224fb6e05c8a776bab8f2b64b7f25b969464839d"}, + {file = "s3transfer-0.10.1.tar.gz", hash = "sha256:5683916b4c724f799e600f41dd9e10a9ff19871bf87623cc8f491cb4f5fa0a19"}, +] + +[package.dependencies] +botocore = ">=1.33.2,<2.0a.0" + +[package.extras] +crt = ["botocore[crt] (>=1.33.2,<2.0a.0)"] + [[package]] name = "scriv" version = "1.5.1" @@ -1827,6 +2342,17 @@ files = [ {file = "shortuuid-1.0.13.tar.gz", hash = "sha256:3bb9cf07f606260584b1df46399c0b87dd84773e7b25912b7e391e30797c5e72"}, ] +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] + [[package]] name = "structlog" version = "23.3.0" @@ -1877,6 +2403,39 @@ rfc3986 = ">=1.4.0" rich = ">=12.0.0" urllib3 = ">=1.26.0" +[[package]] +name = "types-awscrt" +version = "0.20.5" +description = "Type annotations and code completion for awscrt" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "types_awscrt-0.20.5-py3-none-any.whl", hash = "sha256:79d5bfb01f64701b6cf442e89a37d9c4dc6dbb79a46f2f611739b2418d30ecfd"}, + {file = "types_awscrt-0.20.5.tar.gz", hash = "sha256:61811bbf4de95248939f9276a434be93d2b95f6ccfe8aa94e56999e9778cfcc2"}, +] + +[[package]] +name = "types-s3transfer" +version = "0.10.0" +description = "Type annotations and code completion for s3transfer" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "types_s3transfer-0.10.0-py3-none-any.whl", hash = "sha256:44fcdf0097b924a9aab1ee4baa1179081a9559ca62a88c807e2b256893ce688f"}, + {file = "types_s3transfer-0.10.0.tar.gz", hash = "sha256:35e4998c25df7f8985ad69dedc8e4860e8af3b43b7615e940d53c00d413bdc69"}, +] + +[[package]] +name = "typing-extensions" +version = "4.11.0" +description = "Backported and Experimental Type Hints for Python 3.8+" +optional = false +python-versions = ">=3.8" +files = [ + {file = "typing_extensions-4.11.0-py3-none-any.whl", hash = "sha256:c1f94d72897edaf4ce775bb7558d5b79d8126906a14ea5ed1635921406c0387a"}, + {file = "typing_extensions-4.11.0.tar.gz", hash = "sha256:83f085bd5ca59c80295fc2a82ab5dac679cbe02b9f33f7d83af68e241bea51b0"}, +] + [[package]] name = "tzdata" version = "2024.1" @@ -1924,13 +2483,13 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "virtualenv" -version = "20.25.0" +version = "20.25.1" description = "Virtual Python Environment builder" optional = false python-versions = ">=3.7" files = [ - {file = "virtualenv-20.25.0-py3-none-any.whl", hash = "sha256:4238949c5ffe6876362d9c0180fc6c3a824a7b12b80604eeb8085f2ed7460de3"}, - {file = "virtualenv-20.25.0.tar.gz", hash = "sha256:bf51c0d9c7dd63ea8e44086fa1e4fb1093a31e963b86959257378aef020e1f1b"}, + {file = "virtualenv-20.25.1-py3-none-any.whl", hash = "sha256:961c026ac520bac5f69acb8ea063e8a4f071bcc9457b9c1f28f6b085c511583a"}, + {file = "virtualenv-20.25.1.tar.gz", hash = "sha256:e08e13ecdca7a0bd53798f356d5831434afa5b07b93f0abdf0797b7a06ffe197"}, ] [package.dependencies] @@ -2070,4 +2629,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = "~3.10" -content-hash = "8af2b0b4beb7499d0d3a541c58a6ece2c2f84cb6b482e9e88ec04fbfe97c8a02" +content-hash = "7d031b04e03ade4460ef0020099d7760ab937ea994e2342ca1bebd8812ff6c6d" diff --git a/pyproject.toml b/pyproject.toml index 3a9af314..8b59bfb1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -58,6 +58,8 @@ yarl = "1.9.2" frozenlist = "1.4.0" aiofiles = "^23.2.1" aioshutil = "^1.3" +boto3 = "^1.34.42" +boto3-stubs = {extras = ["s3"], version = "^1.34.42"} [tool.poetry.dev-dependencies] pre-commit = "^3.3.3" diff --git a/s3experiment.py b/s3experiment.py new file mode 100644 index 00000000..69c73198 --- /dev/null +++ b/s3experiment.py @@ -0,0 +1,362 @@ +import itertools +import json +import logging +import pathlib +import subprocess +from multiprocessing import Pool +from typing import Any + +import boto3 +import psycopg +import xxhash +from psycopg.rows import dict_row + +# FORMAT = '%(asctime)-15s %(clientip)s %(user)-8s %(message)s' +logging.basicConfig( + level=logging.INFO, format="%(asctime)s - %(process)d - %(message)s" +) +logger = logging.getLogger("backy") + + +worker_connection = None + + +def initialize_worker(): + global worker_connection + worker_connection = psycopg.connect( + "dbname=ctheune user=ctheune", row_factory=dict_row + ) + logging.info(worker_connection) + + +def _object_path(path, id: int, for_writing=False): + id_str = f"{id:016X}" + # Create directories for the first 3 (16 bit) words so we end up + # with 65k entries per directory level. + directory_str = id_str[:-4] + while directory_str: + segment, directory_str = directory_str[:4], directory_str[4:] + path = path.joinpath(segment) + path.mkdir(parents=True, exist_ok=True) + return path.joinpath(id_str).with_suffix(".object") + + +def backup_batch(arg): + conn = worker_connection + assert conn is not None + batch, bucket, path, revision = arg + try: + # XXX theoretically this could be passed into an initializer? + cur = conn.cursor() + + ACCESS_KEY = "UO16ZELUM61Z1E5VHKQ8" + SECRET_KEY = "Y2EaYIzoLgS2XpxaYJOlfpi43jdWw2hjdJm3tycg" + + # Detect all buckets we need to back up + s3_client = boto3.client( + "s3", + # XXX I'd prefer if this was doing proper roundtrips on different + # servers + # ... + endpoint_url="https://s3.whq.fcio.net", + aws_access_key_id=ACCESS_KEY, + aws_secret_access_key=SECRET_KEY, + ) + + known_objects = {} + for row in cur.execute( + """\ + SELECT + id, key, lastmodified, etag + FROM + object + WHERE + hash is not null AND + key IN + """ + + "(" + + ", ".join(("%s",) * len(batch)) + + ")", + [o["Key"] for o in batch], + ).fetchall(): + known_objects[(row["key"], row["lastmodified"], row["etag"])] = row[ + "id" + ] + + queued_revision_object_markers = [] + + for obj in batch: + # XXX Theoretically users can upload the same content, which should + # result in the same etag to modify the metadata. We're not sure + # whether this happens sufficiently often, to justify making this + # more complex and splitting the metadata + etag update. + + object_id = known_objects.get( + (obj["Key"], obj["LastModified"], obj["ETag"]) + ) + + if not object_id: + logger.info(f"Backing up object: {obj['Key']}") + object_data = s3_client.get_object( + Bucket=bucket, Key=obj["Key"] + ) + + # XXX consider upserting in the previous select instead? + object_id = cur.execute( + """\ + INSERT INTO + object (key, lastmodified, etag) + VALUES + (%s, %s, %s) + RETURNING + id + """, + (obj["Key"], obj["LastModified"], obj["ETag"]), + ).fetchone()["id"] + + f = _object_path(path, object_id).open("wb") + body = object_data.pop("Body") + hash = xxhash.xxh3_128() + + for chunk in body.iter_chunks(): + hash.update(chunk) + f.write(chunk) + + cur.execute( + """\ + UPDATE object + SET + hash = %s, + metadata = %s, + etag = %s + WHERE + id = %s + """, + # Note: between listing and retrieving, the ETag might have + # changed. Due to inaccuracies in the LastModified + # timestamp in get_object we don't now the millisecond + # accuracy of the new object. We record the new etag but + # we leave the higher accuracy but older timestamp in + # place. This might/will cause another backup with the + # correct timestamp and etag in the next backup, though. + ( + hash.hexdigest(), + json.dumps( + object_data["ResponseMetadata"]["HTTPHeaders"] + ), + object_data["ETag"], + object_id, + ), + ) + + queued_revision_object_markers.append((revision, object_id)) + + # Sync + # We experimented again with very fine grained fsyncing. However, + # fine-grained fsyncs cause at least 4% bandwidth overhead and + # - more importantly - about 5x IOPS overhead. Apparently syncing a + # full filesystem does have the benefit that journalling and ordering + # can be smarter. This makes the code much much simpler, however, + # this may be a bit latency sensitive in some situations where + # multiple jobs are running in parallel and buffers might be + # extensively used. + logger.info("Syncing") + subprocess.run(["sync", "-f", path], check=True) + # This also isn't thread-safe: when running _sync nobody should be + # modifying data in the database or on the disk in parallel. + insert_items = len(queued_revision_object_markers) + query = """\ + INSERT INTO + revision_object (revision, object) + VALUES + """ + ", ".join( + ["(%s, %s)"] * insert_items + ) + cur.execute( + query, list(itertools.chain(*queued_revision_object_markers)) + ) + conn.commit() + except Exception: + logger.exception("unknown error") + conn.rollback() + logger.info("Finished sync") + + +class BucketBackup: + revision: int + bucket: str + + def __init__(self, s3_client, path, revision, bucket): + self.path = path + self.s3_client = s3_client + self.revision = revision + self.bucket = bucket + + self.sync_attempts = 0 + + self.queued_revision_object_markers = [] + + def _iter_batches(self): + # 300ms for 1k objects within WHQ on the storage network which isn't + # the fastest so that's 5 minutes + ContinuationToken = "" + while True: + logger.info("Getting batch from S3") + res = self.s3_client.list_objects_v2( + Bucket=self.bucket, + ContinuationToken=ContinuationToken, + ) + logger.info("Got batch from S3") + yield res["Contents"], self.bucket, self.path, self.revision + + # This might be a bottleneck and we likely want to keep track of + # how much work was actually done and then commit every now and + # then. + if not res["IsTruncated"]: + break + ContinuationToken = res["NextContinuationToken"] + + def backup_(self): + batch = next(self._iter_batches()) + backup_batch(batch) + + def backup(self): + logger.info(f"Backing up bucket: {self.bucket}") + + p = Pool(5, initializer=initialize_worker) + # XXX create bucket as an object in the database for + # (id, owner, bucket) + # XXX also need to record the bucket in the object table + + with p: + for _ in p.imap_unordered( + backup_batch, self._iter_batches(), chunksize=1 + ): + pass + + +def prepare_db(): + conn = psycopg.connect("dbname=ctheune user=ctheune", row_factory=dict_row) + with conn.cursor() as cur: + # XXX table for buckets (per repository) (id, repo, bucket_name) + cur.execute( + # XXX bucket id + """ + CREATE TABLE IF NOT EXISTS object ( + id serial PRIMARY KEY, + hash VARCHAR, + key VARCHAR NOT NULL, + lastmodified varchar NOT NULL, + etag VARCHAR NOT NULL, + metadata VARCHAR + ) + """ + ) + cur.execute( + "CREATE INDEX IF NOT EXISTS " + "obj_key_idx ON object(key,lastmodified,etag)" + ) + cur.execute( + "CREATE INDEX IF NOT EXISTS " + "obj_key_lastmodified_etag_idx ON object(key,lastmodified,etag)" + ) + cur.execute( + """\ + CREATE TABLE IF NOT EXISTS revision ( + id serial PRIMARY KEY + ) + """ + ) + cur.execute( + """CREATE TABLE IF NOT EXISTS revision_object ( + revision INTEGER NOT NULL REFERENCES revision(id) + ON DELETE CASCADE, + object INTEGER NOT NULL REFERENCES object(id) + ON DELETE CASCADE, + PRIMARY KEY (revision, object)) + """ + ) + cur.execute( + "CREATE INDEX IF NOT EXISTS " + "revision_object_obj_idx ON revision_object(object)" + ) + cur.execute( + "CREATE INDEX IF NOT EXISTS " + "revision_object_rev_idx ON revision_object(revision)" + ) + + revision = cur.execute( + "insert into revision default values returning id" + ).fetchone()["id"] + + conn.commit() + + return revision + + +def main(): + ACCESS_KEY = "UO16ZELUM61Z1E5VHKQ8" + SECRET_KEY = "Y2EaYIzoLgS2XpxaYJOlfpi43jdWw2hjdJm3tycg" + + # Detect all buckets we need to back up + s3_client = boto3.client( + "s3", + # XXX I'd prefer if this was doing proper roundtrips on different + # servers + # ... + endpoint_url="https://s3.whq.fcio.net", + aws_access_key_id=ACCESS_KEY, + aws_secret_access_key=SECRET_KEY, + ) + + revision = prepare_db() + + for bucket in s3_client.list_buckets()["Buckets"]: + logging.info(f"Backup up bucket {bucket['Name']}") + backup = BucketBackup( + s3_client, pathlib.Path("."), revision, bucket["Name"] + ) + backup.backup() + + # bucket = target.open_multi("wb", self.revision.get_parent()) + # async with self.client as client: + # count = 0 + # async for obj in self.client.list_obj(): + # count += 1 + # if count > 100_000: + # break + # if count % 1000 == 0: + # print(obj.key, count) + # if bucket.create_shallow(obj): + # # TODO differentiate between modified and etag missmatch? + # continue + # print("not shallow") + # new = bucket.create_incoming_obj() + # fut = await client.submit_download_obj(obj, new) + # fut.add_done_callback( + # lambda fut: bucket.create_obj(*fut.result())) + # # complete all callbacks + # await asyncio.sleep(0) + # start = time.time() + # bucket.store.db.commit() + # print("commit time", time.time() - start) + + +if __name__ == "__main__": + import cProfile + import io + import pstats + from pstats import SortKey + + pr = cProfile.Profile() + pr.enable() + try: + main() + except Exception: + logger.exception("unknown error") + pr.disable() + s = io.StringIO() + sortby = SortKey.CUMULATIVE + ps = pstats.Stats(pr, stream=s).sort_stats(sortby) + ps.print_stats(50) + print(s.getvalue()) diff --git a/src/backy/backends/__init__.py b/src/backy/backends/__init__.py index a19d7584..383e8d0c 100644 --- a/src/backy/backends/__init__.py +++ b/src/backy/backends/__init__.py @@ -37,5 +37,9 @@ def select_backend(type_: str) -> Type[BackyBackend]: from backy.backends.cowfile import COWFileBackend return COWFileBackend + case "s3": + from backy.backends.s3 import S3Backend + + return S3Backend case _: raise ValueError(f"Invalid backend '{type_}'") diff --git a/src/backy/backends/chunked/__init__.py b/src/backy/backends/chunked/__init__.py index 0fbf74c5..72faef33 100644 --- a/src/backy/backends/chunked/__init__.py +++ b/src/backy/backends/chunked/__init__.py @@ -22,7 +22,7 @@ def __init__(self, revision: Revision, log: BoundLogger): self.revision = revision path = self.backup.path / "chunks" if path not in self.STORES: - self.STORES[path] = Store(self.backup.path / "chunks", log) + self.STORES[path] = Store(path, log) self.store = self.STORES[path] self.log = log.bind(subsystem="chunked") diff --git a/src/backy/backends/chunked/store.py b/src/backy/backends/chunked/store.py index d60151a0..467009ab 100644 --- a/src/backy/backends/chunked/store.py +++ b/src/backy/backends/chunked/store.py @@ -11,10 +11,6 @@ # lot more coordination about the users of the pool. -def rreplace(str, old, new): - return new.join(str.rsplit(old, 1)) - - class Store(object): # Signal that we should always override chunks that we want to write. # This can be used in the face of suspected inconsistencies while still diff --git a/src/backy/backends/s3/__init__.py b/src/backy/backends/s3/__init__.py new file mode 100644 index 00000000..cf680704 --- /dev/null +++ b/src/backy/backends/s3/__init__.py @@ -0,0 +1,51 @@ +from pathlib import Path +from typing import IO, Optional, Set + +from structlog.stdlib import BoundLogger + +from backy.revision import Revision, Trust +from backy.utils import END, report_status + +from ...backup import Backup +from .. import BackyBackend +from .bucketsnapshot import BucketSnapshot +from .store import Store + + +class S3Backend(BackyBackend): + # multiple Backends may share the same store + STORES: dict[Path, Store] = dict() + + backup: Backup + revision: Revision + store: Store + log: BoundLogger + + def __init__(self, revision: Revision, log: BoundLogger): + assert revision.backend_type == "s3" + self.backup = revision.backup + self.revision = revision + path = self.backup.path / "objects" + if path not in self.STORES: + self.STORES[path] = Store(path, log) + self.store = self.STORES[path] + self.log = log.bind(subsystem="s3") + + def open(self, mode: str = "rb", parent: Optional["Revision"] = None) -> IO: + raise NotImplementedError("s3 only supports open_multi") + + def open_multi( + self, + mode: str = "rb", + parent: Optional[Revision] = None, + ): + return BucketSnapshot(self.store, mode, self.revision, parent, self.log) + + def purge(self) -> None: + self.store.purge( + {r.uuid for r in self.backup.history if r.backend_type == "s3"} + ) + + @report_status + def verify(self): + self.store.verify(self.revision) diff --git a/src/backy/backends/s3/bucketsnapshot.py b/src/backy/backends/s3/bucketsnapshot.py new file mode 100644 index 00000000..153d1f07 --- /dev/null +++ b/src/backy/backends/s3/bucketsnapshot.py @@ -0,0 +1,115 @@ +import asyncio +import time +from typing import Dict, Iterable, Optional, Set + +from structlog.stdlib import BoundLogger + +import backy.backends.s3 +from backy.backends.s3.store import S3Obj, TemporaryS3Obj +from backy.revision import Revision +from backy.sources.obj_types import ObjectRestoreTarget, RemoteS3Obj + + +class BucketSnapshot: + id: int + store: "backy.backends.s3.Store" + mode: str # TODO used? + parent_objs: Dict[str, S3Obj] + log: BoundLogger + + def __init__( + self, + store: "backy.backends.s3.Store", + mode: str, + revision: Revision, + parent: Optional[Revision], + log: BoundLogger, + ): + self.store = store + self.mode = mode + with self.store.db: + self.id = store.get_or_create_rev(revision) + self.parent_objs = {} + start = time.time() + if parent: + parent_id = store.get_rev_id(parent) + if parent_id: + self.parent_objs = { + o.key: o for o in self.store.list_obj(parent_id) + } + self.parent_id = parent_id + self.log = log.bind(subsystem="s3-bucket-store") + self.log.debug( + "loaded-parent-objs", + num=len(self.parent_objs), + time=time.time() - start, + ) + + def create_shallow(self, obj: RemoteS3Obj) -> bool: + if obj.key in self.parent_objs: + # print("create shallow", obj.key, obj.lastmodified, obj.etag) + # o = self.store.get_object(obj.key, self.parent_id) + o = self.parent_objs[obj.key] + if o and o.lastmodified == obj.lastmodified and o.etag == obj.etag: + # with self.store.db: + self.store.add_obj_rev_relation(o.id, self.id) + return True + return False + + def create_obj( + self, remote_obj: RemoteS3Obj, tempobj: TemporaryS3Obj + ) -> S3Obj: + # TODO mode + # with self.store.db: + # print( + # "adding obj", + # remote_obj.key, + # remote_obj.lastmodified, + # remote_obj.etag, + # ) + obj = self.store.add_object(remote_obj) + self.store.add_obj_rev_relation(obj.id, self.id) + obj.path.parent.mkdir(parents=True, exist_ok=True) + tempobj.path.replace(obj.path) + tempobj.meta_path.replace(obj.meta_path) + return obj + + def list_obj(self) -> Iterable[S3Obj]: + yield from self.store.list_obj(self.id) + + def create_incoming_obj(self) -> TemporaryS3Obj: + return self.store.create_incoming_obj() + + def get_obj(self, key: str) -> Optional[S3Obj]: + return self.store.get_object(key, self.id) + + def restore(self, target: ObjectRestoreTarget) -> None: + # assumption: bucket exists and is empty + # self.client.create_bucket(Bucket=bucket) + # self.client.head_bucket(Bucket=bucket) + asyncio.run(self._restore(target)) + + async def _restore(self, target: "ObjectRestoreTarget") -> None: + remote_obj: Set[str] = set() + async with target: + async for obj in target.list_obj(): + remote_obj.add(obj.key) + print("restore found remote", obj.key) + + # todo: handle locking, versioning, errors + local_obj = self.get_obj(obj.key) + if not local_obj: + print("restore unknown remote", obj.key) + await target.submit_delete_obj(obj.key) + elif local_obj.lastmodified != obj.lastmodified: + # fixme: lastmodified will change when uploading + print("restore remote changed", obj.key) + print(local_obj.lastmodified, obj.lastmodified) + print(local_obj.etag, obj.etag) + await target.submit_upload_obj( + local_obj, meta_only=local_obj.etag == obj.etag + ) + for local_obj in self.list_obj(): + if local_obj.key not in remote_obj: + print("restore remote missing", local_obj.key) + await target.submit_upload_obj(local_obj) diff --git a/src/backy/backends/s3/store.py b/src/backy/backends/s3/store.py new file mode 100644 index 00000000..abeef699 --- /dev/null +++ b/src/backy/backends/s3/store.py @@ -0,0 +1,167 @@ +import datetime +import sqlite3 +from pathlib import Path +from sqlite3 import PARSE_DECLTYPES, Cursor +from typing import Iterable, Optional, Set + +from structlog.stdlib import BoundLogger + +from backy.revision import Revision +from backy.sources.obj_types import RemoteS3Obj, S3Obj, TemporaryS3Obj + +STORE_DEPTH = 3 + +sqlite3.register_adapter(datetime.datetime, lambda dt: dt.isoformat()) +sqlite3.register_converter( + "timestamp", lambda s: datetime.datetime.fromisoformat(s.decode()) +) + + +class Store(object): + path: Path + log: BoundLogger + db: sqlite3.Connection + incoming_path: Path + + def __init__(self, path: Path, log: BoundLogger): + self.path = path + self.path.mkdir(exist_ok=True) + self.incoming_path = self.path / "incoming" + self.incoming_path.mkdir(exist_ok=True) + self.log = log.bind(subsystem="s3-store") + self.db = sqlite3.connect( + path / "index.db", detect_types=PARSE_DECLTYPES + ) + self.db.row_factory = sqlite3.Row + with self.db: + self.db.execute( + """ + CREATE TABLE IF NOT EXISTS object ( + id INTEGER NOT NULL PRIMARY KEY, + key VARCHAR NOT NULL, + lastmodified timestamp NOT NULL, + etag VARCHAR NOT NULL + ) + """ + ) + self.db.execute( + "CREATE INDEX IF NOT EXISTS obj_key_idx ON object(key)" + ) + self.db.execute( + """\ + CREATE TABLE IF NOT EXISTS revision ( + id INTEGER NOT NULL PRIMARY KEY, + name VARCHAR NOT NULL UNIQUE + ) + """ + ) + self.db.execute( + """CREATE TABLE IF NOT EXISTS rev_obj ( + revision INTEGER NOT NULL REFERENCES revision(id) ON DELETE CASCADE, + object INTEGER NOT NULL REFERENCES object(id) ON DELETE CASCADE, + PRIMARY KEY (revision, object)) + """ + ) + self.db.execute( + "CREATE INDEX IF NOT EXISTS rev_obj_obj_idx ON rev_obj(object)" + ) + self.db.execute( + "CREATE INDEX IF NOT EXISTS rev_obj_rev_idx ON rev_obj(revision)" + ) + + def get_rev_id(self, revision: Revision) -> Optional[int]: + o = self.db.execute( + "SELECT id FROM revision WHERE name = ?", (revision.uuid,) + ).fetchone() + if o: + return o["id"] + return None + + def get_or_create_rev(self, revision: Revision) -> int: + self.db.execute( + "INSERT OR IGNORE INTO revision(name) VALUES(?)", + (revision.uuid,), + ) + id = self.get_rev_id(revision) + assert id is not None + return id + + def get_object(self, key: str, rev: int) -> Optional["S3Obj"]: + obj = self.db.execute( + "SELECT id, key, lastmodified, etag FROM object INNER JOIN rev_obj ON rev_obj.object = object.id WHERE object.key = ? AND rev_obj.revision = ?", + (key, rev), + ).fetchone() + if obj: + return S3Obj.from_db_row(self, obj) + return None + + def add_obj_rev_relation(self, obj_id: int, rev_id: int) -> None: + self.db.execute( + "INSERT INTO rev_obj(revision, object) VALUES(?, ?)", + (rev_id, obj_id), + ) + + def add_object(self, obj: RemoteS3Obj) -> "S3Obj": + res = self.db.execute( + "INSERT INTO object(key, lastmodified, etag) VALUES(?, ?, ?) RETURNING id, key, lastmodified, etag", + (obj.key, obj.lastmodified, obj.etag), + ).fetchone() + return S3Obj.from_db_row(self, res, "rb") + + def list_revisions(self) -> Cursor: + return self.db.execute("SELECT id, name FROM revision") + + def list_obj(self, rev: int) -> Iterable["S3Obj"]: + cur = self.db.execute( + "SELECT id, key, lastmodified, etag FROM object INNER JOIN rev_obj ON rev_obj.object = object.id WHERE rev_obj.revision = ?", + (rev,), + ) + for row in cur: + yield S3Obj.from_db_row(self, row) + + def ls(self) -> Iterable[int]: + # XXX this is fucking expensive + for file in self.path.rglob("*.object"): + yield int(file.name.removesuffix(".object")) + + def purge(self, existing_revs: Set[str]) -> None: + with self.db: + for id, name in self.list_revisions(): + if name not in existing_revs: + self.log.info("purge-rev", rev=name) + self.db.execute("DELETE FROM revision WHERE id = ?", (id,)) + with self.db: + # self.db.execute("DELETE FROM object LEFT OUTER JOIN rev_obj ON object.id = rev_obj.object WHERE rev_obj.revision IS NULL") + cursor = self.db.execute( + "DELETE FROM object WHERE NOT EXISTS (SELECT 1 from rev_obj WHERE rev_obj.object = object.id) RETURNING object.id" + ) + # it is ok to remove the files before commiting the transaction + # because the row entries will be removed by the next purge call + # and won't be used by a new revision because the parent no longer exists + num = 0 + for row in cursor: + num += 1 + self.object_path(row["id"]).unlink(missing_ok=True) + self.log.info("purge-obj", num=num) + + for id in self.ls(): + if not self.db.execute( + "SELECT 1 FROM object WHERE id = ?", (id,) + ).fetchone(): + self.object_path(id).unlink(missing_ok=True) + + def verify(self, revision: Revision): + rev_id = self.get_rev_id(revision) + assert rev_id + for obj in self.list_obj(rev_id): + pass + # TODO check integrity + + def object_path(self, id: int) -> Path: + path = self.path + for i in reversed(range(1, STORE_DEPTH + 1)): + path = path.joinpath(f"{(id>>(i*16)) & 0xffff:04x}") + return path.joinpath(f"{id:016x}").with_suffix(".object") + + def create_incoming_obj(self) -> "TemporaryS3Obj": + return TemporaryS3Obj.create_incoming(self.incoming_path) diff --git a/src/backy/backup.py b/src/backy/backup.py index f7283037..a3142d3c 100644 --- a/src/backy/backup.py +++ b/src/backy/backup.py @@ -34,6 +34,7 @@ from .revision import Revision, Trust, filter_schedule_tags from .schedule import Schedule from .sources import BackySourceFactory, select_source +from .sources.obj_types import ObjectRestoreTarget from .utils import CHUNK_SIZE, copy, posix_fadvise # Locking strategy: @@ -144,7 +145,7 @@ def __init__(self, path: Path, log: BoundLogger): # Initialize our source try: source_factory = select_source(self.config["source"]["type"]) - except IndexError: + except ValueError: self.log.error( "source-type-unavailable", _fmt_msg="No source type named `{type}` exists.", @@ -155,6 +156,7 @@ def __init__(self, path: Path, log: BoundLogger): # Initialize our backend self.default_backend_type = self.config["source"].get("backend", None) + # TODO: force s3 backend here? if self.default_backend_type is None: if not self.local_history: # Start fresh backups with our new default. @@ -414,6 +416,11 @@ def restore( restore_backend: RestoreBackend = RestoreBackend.AUTO, ) -> None: r = self.find(revision) + if r.backend_type == "s3": + r.backend.open_multi("rb").restore( + self.select_restore_target(target) + ) + return s = r.backend.open("rb") if restore_backend == RestoreBackend.AUTO: if self.backy_extract_supported(s): @@ -430,6 +437,32 @@ def restore( elif restore_backend == RestoreBackend.RUST: self.restore_backy_extract(r, target) + def select_restore_target(self, type_: str) -> ObjectRestoreTarget: + from backy.sources.s3.source import ( + S3, + AsyncS3Client, + S3LocalRestoreTarget, + ) + + if type_.startswith("s3://"): + + assert isinstance(self.source, S3) + return AsyncS3Client( + type_.removeprefix("s3://"), + self.config["source"]["endpoint_url"], + self.config["source"]["access_key"], + self.config["source"]["secret_key"], + self.log, + ) + elif type_.startswith("path://"): + return S3LocalRestoreTarget(Path(type_.removeprefix("path://"))) + elif type_.startswith("path-flat://"): + return S3LocalRestoreTarget( + Path(type_.removeprefix("path://")), separator=None + ) + else: + raise ValueError() + def backy_extract_supported(self, file: IO) -> bool: log = self.log.bind(subsystem="backy-extract") if not isinstance(file, backy.backends.chunked.File): diff --git a/src/backy/sources/__init__.py b/src/backy/sources/__init__.py index 23e63a3d..e4afbf7a 100644 --- a/src/backy/sources/__init__.py +++ b/src/backy/sources/__init__.py @@ -57,5 +57,9 @@ def select_source(type_: str) -> Type[BackySourceFactory]: from backy.sources.file import File return File + case "s3": + from backy.sources.s3.source import S3 + + return S3 case _: - raise ValueError(f"invalid backend: {type_}") + raise ValueError(f"invalid source: {type_}") diff --git a/src/backy/sources/ceph/source.py b/src/backy/sources/ceph/source.py index 8d6623fc..737b5d9c 100644 --- a/src/backy/sources/ceph/source.py +++ b/src/backy/sources/ceph/source.py @@ -24,6 +24,7 @@ class CephRBD(BackySource, BackySourceFactory, BackySourceContext): always_full: bool log: BoundLogger rbd: RBDClient + revision: Revision def __init__(self, config: dict, log: BoundLogger): self.pool = config["pool"] @@ -45,7 +46,7 @@ def ready(self) -> bool: self.log.exception("not-ready") return False - def __call__(self, revision): + def __call__(self, revision: Revision): self.revision = revision return self diff --git a/src/backy/sources/obj_types.py b/src/backy/sources/obj_types.py new file mode 100644 index 00000000..decb9813 --- /dev/null +++ b/src/backy/sources/obj_types.py @@ -0,0 +1,124 @@ +import datetime +import os +import tempfile +from abc import ABC, abstractmethod +from asyncio import Future +from contextlib import contextmanager +from pathlib import Path +from typing import IO, TYPE_CHECKING, AsyncIterable, Iterator + +import yaml +from mypy_boto3_s3.type_defs import ObjectTypeDef + +from backy.utils import posix_fadvise + +if TYPE_CHECKING: + from backy.backends.s3 import Store + + +class TemporaryS3Obj: + path: Path + mode: str + + @property + def meta_path(self) -> Path: + return self.path.with_suffix(".meta") + + @property + def writeable(self) -> bool: + return "w" in self.mode or "+" in self.mode + + def __init__(self, path: Path, mode: str = "rb"): + assert "b" in mode + self.path = path + self.mode = mode + + @classmethod + def create_incoming(cls, dir: Path) -> "TemporaryS3Obj": + fd, tmpfile_name = tempfile.mkstemp(dir=dir) + os.close(fd) + return cls(dir / tmpfile_name, "wb") + + def set_metadata(self, meta: dict) -> None: + assert self.writeable + with self.meta_path.open("w", encoding="utf-8") as f: + yaml.safe_dump({"Metadata": meta}, f) + f.flush() + os.fsync(f) + + def get_metadata(self) -> dict: + with self.meta_path.open(encoding="utf-8") as f: + return yaml.safe_load(f)["Metadata"] + + @contextmanager + def open(self) -> Iterator[IO]: + with self.path.open(self.mode) as f: + posix_fadvise(f.fileno(), 0, 0, os.POSIX_FADV_DONTNEED) # type: ignore + yield f + f.flush() + os.fsync(f) + + +class RemoteS3Obj: + key: str + lastmodified: datetime.datetime + etag: str + + def __init__( + self, + key: str, + lastmodified: datetime.datetime, + etag: str, + ): + self.key = key + self.lastmodified = lastmodified + self.etag = etag + + @classmethod + def from_api(cls, obj: ObjectTypeDef) -> "RemoteS3Obj": + return cls(obj["Key"], obj["LastModified"], obj["ETag"]) + + +class S3Obj(TemporaryS3Obj, RemoteS3Obj): + id: int + + def __init__( + self, + store: "Store", + id: int, + mode: str, + key: str, + lastmodified: datetime.datetime, + etag: str, + ): + self.id = id + TemporaryS3Obj.__init__(self, store.object_path(id), mode) + RemoteS3Obj.__init__(self, key, lastmodified, etag) + + @classmethod + def from_db_row(cls, store: "Store", row, mode="rb") -> "S3Obj": + return cls( + store, row["id"], mode, row["key"], row["lastmodified"], row["etag"] + ) + + +class ObjectRestoreTarget(ABC): + async def __aenter__(self) -> "ObjectRestoreTarget": + return self + + async def __aexit__(self, exc_type, exc_val, exc_tb) -> None: + pass + + @abstractmethod + async def submit_delete_obj(self, key: str) -> Future[None]: + ... + + @abstractmethod + async def submit_upload_obj( + self, obj: S3Obj, meta_only=False + ) -> Future[None]: + ... + + @abstractmethod + def list_obj(self, glob: str = "") -> AsyncIterable[RemoteS3Obj]: + ... diff --git a/src/backy/sources/s3/__init__.py b/src/backy/sources/s3/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/backy/sources/s3/source.py b/src/backy/sources/s3/source.py new file mode 100644 index 00000000..59a6c098 --- /dev/null +++ b/src/backy/sources/s3/source.py @@ -0,0 +1,269 @@ +import asyncio +import datetime +import time +from asyncio import Future, Task +from functools import partial +from pathlib import Path +from typing import AsyncIterable, Optional, Tuple +from urllib import parse + +import boto3 +import boto3.s3 +from botocore.exceptions import ClientError +from mypy_boto3_s3.client import S3Client +from mypy_boto3_s3.type_defs import DeleteObjectOutputTypeDef +from rich import print as rprint +from structlog.stdlib import BoundLogger + +from backy.backends import BackyBackend +from backy.backends.s3 import S3Backend +from backy.backends.s3.store import S3Obj, TemporaryS3Obj +from backy.revision import Revision +from backy.sources import BackySource, BackySourceContext, BackySourceFactory +from backy.sources.obj_types import ObjectRestoreTarget, RemoteS3Obj +from backy.utils import FuturePool, completed_future, copy + +# TODO: service-2.sdk-extras.json should be included in backy, see: https://github.com/ceph/ceph/tree/main/examples/rgw/boto3 + + +class S3(BackySource, BackySourceFactory, BackySourceContext): + revision: Revision + log: BoundLogger + client: "AsyncS3Client" + + def __init__(self, config: dict, log: BoundLogger) -> None: + self.client = AsyncS3Client( + config["bucket"], + config["endpoint_url"], + config["access_key"], + config["secret_key"], + log, + ) + self.log = log.bind(subsystem="s3") + + def ready(self) -> bool: + return self.client.head_bucket() + + def __call__(self, revision: Revision) -> BackySourceContext: + self.revision = revision + return self + + def __enter__(self) -> BackySource: + return self + + def backup(self, target: BackyBackend) -> None: + assert isinstance(target, S3Backend) + start = time.time() + asyncio.run(self._backup(target)) + print("time", time.time() - start) + + async def _backup(self, target: S3Backend): + bucket = target.open_multi("wb", self.revision.get_parent()) + async with self.client as client: + count = 0 + async for obj in self.client.list_obj(): + count += 1 + if count > 100_000: + break + if count % 1000 == 0: + print(obj.key, count) + if bucket.create_shallow(obj): + # TODO differentiate between modified and etag missmatch? + continue + print("not shallow") + new = bucket.create_incoming_obj() + fut = await client.submit_download_obj(obj, new) + fut.add_done_callback( + lambda fut: bucket.create_obj(*fut.result()) + ) + # complete all callbacks + await asyncio.sleep(0) + start = time.time() + bucket.store.db.commit() + print("commit time", time.time() - start) + + def verify(self, target: BackyBackend) -> bool: + return True + + +class S3LocalRestoreTarget(ObjectRestoreTarget): + path: Path + separator: Optional[str] + + def __init__(self, path: Path, separator: Optional[str] = "/"): + self.path = path + self.separator = separator + + def to_path(self, key: str) -> Path: + if self.separator is None: + p = self.path / key + else: + p = Path(self.path, *key.split(self.separator)) + assert p.resolve().is_relative_to(self.path.resolve()) + return p + + def to_key(self, path: Path) -> str: + if self.separator is None: + return path.name + return self.separator.join(path.relative_to(self.path).parts) + + async def submit_delete_obj(self, key: str) -> Future[None]: + p = self.to_path(key) + p.unlink() + return completed_future(None) + + async def submit_upload_obj( + self, obj: S3Obj, meta_only=False + ) -> Future[None]: + if meta_only: + return completed_future(None) + p = self.to_path(obj.key) + p.parent.mkdir(parents=True, exist_ok=True) + with obj.open() as source, p.open("wb") as target: + # TODO cp_reflink? + copy(source, target) + return completed_future(None) + + async def list_obj(self, glob: str = "") -> AsyncIterable[RemoteS3Obj]: + for p in self.path.rglob("*"): + if not p.is_file(): + continue + yield RemoteS3Obj(self.to_key(p), datetime.datetime.min, "") + + +class AsyncS3Client(ObjectRestoreTarget): + bucket: str + client: S3Client + future_pool: Optional[FuturePool] + pool_size: int + log: BoundLogger + + def __init__( + self, + bucket: str, + endpoint_url: str, + access_key: str, + secret_key: str, + log: BoundLogger, + pool_size: int = 30, + ): + self.bucket = bucket + self.client = boto3.client( + "s3", + endpoint_url=endpoint_url, + aws_access_key_id=access_key, + aws_secret_access_key=secret_key, + ) + self.log = log.bind(subsystem="s3-client") + self.pool_size = pool_size + + async def __aenter__(self) -> "AsyncS3Client": + self.future_pool = FuturePool( + asyncio.get_running_loop(), self.pool_size, thread_support=True + ) + await self.future_pool.__aenter__() + return self + + async def __aexit__(self, exc_type, exc_val, exc_tb) -> None: + assert self.future_pool + await self.future_pool.__aexit__(exc_type, exc_val, exc_tb) + self.future_pool = None + + async def submit_delete_obj( + self, key: str + ) -> Future[DeleteObjectOutputTypeDef]: + assert self.future_pool + return await self.future_pool.submit( + partial(self.client.delete_object, Bucket=self.bucket, Key=key), + ) + + async def submit_upload_obj( + self, obj: S3Obj, meta_only=False + ) -> Future[None]: + assert self.future_pool + meta = obj.get_metadata() + if meta_only: + tag_set = [{k: v} for k, v in meta.get("TagSet", {}).items()] + return await self.future_pool.submit( + partial( + self.client.put_object_tagging, + Bucket=self.bucket, + Key=obj.key, + Tagging={"TagSet": tag_set}, + ), + ) + else: + return await self.future_pool.submit( + partial( + self.client.upload_file, + Filename=str(obj.path), + Bucket=self.bucket, + Key=obj.key, + ExtraArgs={ + "Metadata": meta["Metadata"], + "Tagging": parse.urlencode(meta.get("TagSet", {})), + }, + ), + ) + + async def submit_download_obj( + self, spec: RemoteS3Obj, target: TemporaryS3Obj, meta_only=False + ) -> Future[Tuple[RemoteS3Obj, TemporaryS3Obj]]: + assert self.future_pool + return await self.future_pool.submit( + partial(self._download_obj, spec, target, meta_only) + ) + + def _download_obj( + self, spec: RemoteS3Obj, target: TemporaryS3Obj, meta_only=False + ) -> Tuple[RemoteS3Obj, TemporaryS3Obj]: + meta = {} + obj = self.client.get_object(Bucket=self.bucket, Key=spec.key) + if obj.get("TagCount", 0): + tags = self.client.get_object_tagging( + Bucket=self.bucket, Key=spec.key + ) + meta["TagSet"] = tags["TagSet"] + meta["Metadata"] = obj["Metadata"] + target.set_metadata(meta) + if not meta_only: + with target.open() as f: + for chunk in obj["Body"].iter_chunks(): + f.write(chunk) + # XXX LastModified from listobjects has milliseconds accuracy, getobject only has seconds accuracy + spec.etag = obj["ETag"] + return spec, target + + async def list_obj(self, glob: str = "") -> AsyncIterable[RemoteS3Obj]: + # TODO: ceph extension: unordered, might improve performance: not really + # MaxKeys is capped at 1k + inflight: Optional[Task] = asyncio.create_task( + asyncio.to_thread(self.client.list_objects_v2, Bucket=self.bucket) + ) + while inflight: + res = await inflight + if res["IsTruncated"]: + inflight = asyncio.create_task( + asyncio.to_thread( + self.client.list_objects_v2, + Bucket=self.bucket, + ContinuationToken=res["NextContinuationToken"], + ) + ) + else: + inflight = None + # rprint(res) + for o in res["Contents"]: + yield RemoteS3Obj.from_api(o) + + def head_bucket(self) -> bool: + try: + self.client.head_bucket(Bucket=self.bucket) + return True + except ClientError: + self.log.info("head_bucket", exc_style="short") + return False + + +class S3RemoteRestoreTarget(S3Client): + pass diff --git a/src/backy/sources/s3/tests/test_source.py b/src/backy/sources/s3/tests/test_source.py new file mode 100644 index 00000000..4550d019 --- /dev/null +++ b/src/backy/sources/s3/tests/test_source.py @@ -0,0 +1,82 @@ +from typing import Dict, Optional, Set + +from botocore.exceptions import ClientError +from mypy_boto3_s3.type_defs import ( + DeleteObjectOutputTypeDef, + GetObjectTaggingOutputTypeDef, + HeadBucketOutputTypeDef, + ListObjectsV2OutputTypeDef, + PutObjectTaggingOutputTypeDef, +) + + +class BotoObjectMock: + key:str + + def __init__(self): + pass + + +SUCCESSFUL_RESPONSE_META = { + "RequestId": "aaa", + "HTTPStatusCode": 200, + "HTTPHeaders": dict(), + "RetryAttempts": 0, + "HostId": "", +} + +def to_client_error(f): + def wrapped(self, *args, **kw): + try: + f(*args, **kw) + except Exception as e: + raise ClientError({}, "") from e + + return wrapped + + +class BotoClientMock: + buckets: Dict[str, Dict[str,BotoObjectMock]] + + def __init__(self, **buckets: Set[BotoObjectMock]): + self.buckets = buckets + + @to_client_error + def delete_object(self, Bucket: str, Key: str) -> DeleteObjectOutputTypeDef: + self.buckets[Bucket]. + + return { + "DeleteMarker": False, + "VersionId": "ahirtn", + "RequestCharged": "requester", + "ResponseMetadata": SUCCESSFUL_RESPONSE_META, + } + + @to_client_error + def put_object_tagging(self, Bucket: str, Key: str, Tagging: dict) -> PutObjectTaggingOutputTypeDef: + {"TagSet": tag_set} + + @to_client_error + def upload_file(self, Filename: str, Bucket: str, Key: str, ExtraArgs: dict) -> None: + {"Metadata": meta["Metadata"], + "Tagging": parse.urlencode(meta.get("TagSet", {})), + } + + @to_client_error + def get_object_tagging(self, Bucket: str, Key: str) -> GetObjectTaggingOutputTypeDef: + pass + + @to_client_error + def list_objects_v2(self, Bucket: str, ContinuationToken: Optional[str] = None) -> ListObjectsV2OutputTypeDef: + pass + + @to_client_error + def head_bucket(self, Bucket: str) -> HeadBucketOutputTypeDef: + assert Bucket in self.buckets + return { + "BucketLocationType": "AvailabilityZone", + "BucketLocationName": "test-location", + "BucketRegion": "test-region", + "AccessPointAlias": False, + "ResponseMetadata": SUCCESSFUL_RESPONSE_META, + } diff --git a/src/backy/utils.py b/src/backy/utils.py index 90357ac9..37b4efbb 100644 --- a/src/backy/utils.py +++ b/src/backy/utils.py @@ -11,9 +11,20 @@ import tempfile import time import typing -from asyncio import Event -from os import DirEntry -from typing import IO, Callable, Iterable, List, Literal, Optional, TypeVar +from asyncio import AbstractEventLoop, Event, Future, Task +from concurrent.futures import ThreadPoolExecutor +from typing import ( + IO, + Any, + Callable, + Coroutine, + Iterable, + List, + Literal, + Optional, + Set, + TypeVar, +) from zoneinfo import ZoneInfo import aiofiles.os as aos @@ -221,7 +232,7 @@ def fileno(self): os.POSIX_FADV_DONTNEED = None # type: ignore def posix_fadvise(*args, **kw): - log.debug("posix_fadivse-unavailable") + # log.debug("posix_fadivse-unavailable") return @@ -534,3 +545,62 @@ def list_split(l: List[_T], v: _T) -> List[List[_T]]: else: res[-1].append(i) return res + + +# can also be used as a threadpool with async close support +class FuturePool: + size: int + futures: Set[Future] + loop: AbstractEventLoop + thread_pool: Optional[ThreadPoolExecutor] + + def __init__( + self, loop: AbstractEventLoop, size: int, thread_support=False + ): + self.size = size + self.futures: Set[Future] = set() + self.thread_pool = ThreadPoolExecutor(size) if thread_support else None + self.loop = loop + + async def _wait(self, return_when: str = asyncio.ALL_COMPLETED): + _done, self.futures = await asyncio.wait( + self.futures, return_when=return_when + ) + for t in _done: + e: Optional[BaseException] = t.exception() + if e is not None: + raise e + + async def submit( + self, coro: Coroutine[Any, Any, _T] | Future[_T] | Callable[..., _T] + ) -> Future[_T]: + if len(self.futures) >= self.size: + await self._wait(asyncio.FIRST_COMPLETED) + + if asyncio.isfuture(coro): + f = coro + elif asyncio.iscoroutine(coro): + f = self.loop.create_task(coro) + elif self.thread_pool and callable(coro): + f = self.loop.run_in_executor(self.thread_pool, coro) + else: + raise ValueError() + + self.futures.add(f) + return f + + async def __aenter__(self): + return self + + async def __aexit__(self, exc_type, exc_val, exc_tb): + if self.futures: + await self._wait() + # all threads are part `self.futures` so this should not block + if self.thread_pool: + self.thread_pool.shutdown() + + +def completed_future(res: _T) -> Future[_T]: + future: Future[_T] = Future() + future.set_result(res) + return future