diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ffdd283 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +.ipynb_checkpoints +_build diff --git a/asg.ipynb b/asg.ipynb new file mode 100644 index 0000000..a00bf00 --- /dev/null +++ b/asg.ipynb @@ -0,0 +1,168 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "deletable": true, + "editable": true + }, + "outputs": [], + "source": [ + "!pip3 install --upgrade pip\n", + "!pip3 install boto3\n", + "!pip3 install pyyaml" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "deletable": true, + "editable": true + }, + "outputs": [], + "source": [ + "from functools import reduce\n", + "import os\n", + "import boto3\n", + "from botocore.client import Config\n", + "import yaml\n", + "from datetime import datetime, time, timedelta\n", + "import json\n", + "from pprint import pprint\n", + "from pprint import PrettyPrinter\n", + "from statistics import mean\n", + "import sys" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "deletable": true, + "editable": true + }, + "outputs": [], + "source": [ + "print(sys.version_info)\n", + "print(sys.version_info[:3])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "deletable": true, + "editable": true + }, + "outputs": [], + "source": [ + "config = yaml.load(open('config.yaml', 'r'))\n", + "#print(config)\n", + "aws_region = config['aws']['regions'][0]\n", + "account = config['aws']['accounts'][1]['name']\n", + "profile = config['aws']['accounts'][1]['profile']\n", + "account_number = config['aws']['accounts'][1]['account-number']\n", + "\n", + "print((\"%s - %s - %s - %s\" % (str(account_number),account,profile,aws_region)))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "deletable": true, + "editable": true + }, + "outputs": [], + "source": [ + "session = boto3.session.Session(profile_name=profile,region_name=aws_region)\n", + "print(session)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "deletable": true, + "editable": true + }, + "outputs": [], + "source": [ + "asgClient = session.client('autoscaling')\n", + "print(asgClient)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "deletable": true, + "editable": true + }, + "outputs": [], + "source": [ + "class Asg(object):\n", + " def __init__(self, name):\n", + " self.name = name\n", + " self.attributes = {}\n", + " \n", + " def __str__(self):\n", + " retrn = 'ASG: clusterName={0}'\n", + " return retrn" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "deletable": true, + "editable": true + }, + "outputs": [], + "source": [ + "# Create a reusable Paginator\n", + "paginator = asgClient.get_paginator('describe_auto_scaling_groups')\n", + "#print(paginator)\n", + "\n", + "# Create a PageIterator from the Paginator\n", + "page_iterator = paginator.paginate()\n", + "\n", + "asgs = []\n", + "\n", + "for page in page_iterator: \n", + " print(page)\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.5.2" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} \ No newline at end of file diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000..aff2e96 --- /dev/null +++ b/config.yaml @@ -0,0 +1,16 @@ +aws: + accounts: + - name: dev1 + short-name: "" + description: "" + account-number: + profile: + regions: + - us-west-2 + +aws-ec2-cw-stats: + metrics: + - metric: CPUUtilization + namespace: AWS/EC2 + statistics: Minimum,Maximum,Average + minutes: 60 \ No newline at end of file diff --git a/ebs.ipynb b/ebs.ipynb new file mode 100644 index 0000000..9780990 --- /dev/null +++ b/ebs.ipynb @@ -0,0 +1,158 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "deletable": true, + "editable": true + }, + "outputs": [], + "source": [ + "!pip3 install --upgrade pip\n", + "!pip3 install boto3\n", + "!pip3 install pyyaml" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "deletable": true, + "editable": true + }, + "outputs": [], + "source": [ + "from functools import reduce\n", + "import os\n", + "import boto3\n", + "import yaml\n", + "from datetime import datetime, time, timedelta\n", + "import json\n", + "from pprint import pprint\n", + "from pprint import PrettyPrinter\n", + "from statistics import mean\n", + "import sys" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "deletable": true, + "editable": true + }, + "outputs": [], + "source": [ + "print(sys.version_info)\n", + "print(sys.version_info[:3])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "deletable": true, + "editable": true + }, + "outputs": [], + "source": [ + "config = yaml.load(open('config.yaml', 'r'))\n", + "#print(config)\n", + "aws_region = config['aws']['regions'][0]\n", + "account = config['aws']['accounts'][1]['name']\n", + "profile = config['aws']['accounts'][1]['profile']\n", + "account_number = config['aws']['accounts'][1]['account-number']\n", + "\n", + "print((\"{} - {} - {} - {}\".format(str(account_number),account,profile,aws_region)))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "deletable": true, + "editable": true + }, + "outputs": [], + "source": [ + "session = boto3.session.Session(profile_name=profile,region_name=aws_region)\n", + "print(session)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "deletable": true, + "editable": true + }, + "outputs": [], + "source": [ + "ec2Client = session.client('ec2')\n", + "print(ec2Client)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "deletable": true, + "editable": true + }, + "outputs": [], + "source": [ + "response = ec2Client.describe_volumes(Filters = [{'Name':'tag:TAG_KEY', 'Values':['TAG_VALUE']}])\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "pp = PrettyPrinter() #indent=1)\n", + "pp.pprint(response)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.5.2" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/ec2.ipynb b/ec2.ipynb new file mode 100644 index 0000000..5e18f37 --- /dev/null +++ b/ec2.ipynb @@ -0,0 +1,340 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "!pip3 install --upgrade pip\n", + "!pip3 install boto3\n", + "!pip3 install pyyaml" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "from functools import reduce\n", + "import os\n", + "import boto3\n", + "import yaml\n", + "from datetime import datetime, time, timedelta\n", + "import json\n", + "from pprint import pprint\n", + "from pprint import PrettyPrinter\n", + "from statistics import mean\n", + "import sys" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "print(sys.version_info)\n", + "print(sys.version_info[:3])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "pp = PrettyPrinter() #indent=1)\n", + "print(pp)\n", + "section_delimiter = '\\n{}\\n'.format('--- '*20)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "config = yaml.safe_load(open('config.yaml', 'r'))\n", + "pp.pprint(config)\n", + "aws_region = config['aws']['regions'][0]\n", + "account = config['aws']['accounts'][0]['name']\n", + "profile = config['aws']['accounts'][0]['profile']\n", + "account_number = config['aws']['accounts'][0]['account-number']\n", + "\n", + "print(section_delimiter)\n", + "\n", + "print(('Selected Config: {} - {} - {} - {}'.format(str(account_number),account,profile,aws_region)))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "session = boto3.session.Session(profile_name=profile,region_name=aws_region)\n", + "print(('Session: {} - {}'.format(session.profile_name,session.region_name)))\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "ec2Client = session.client('ec2')\n", + "cwClient = session.client('cloudwatch',region_name=aws_region)\n", + "print(ec2Client)\n", + "print(cwClient)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "ec2s = ec2Client.describe_instances(\n", + " Filters=[\n", + " {\n", + " 'Name': 'tag:owner',\n", + " 'Values': ['jimmyray']\n", + " } \n", + " ]\n", + ")\n", + "\n", + "pp.pprint(ec2s)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [], + "source": [ + "def get_metric(instanceId, stat, metric):\n", + " data = cwClient.get_metric_statistics(\n", + " Period=900,\n", + " StartTime=(datetime.utcnow() - timedelta(days=14)),\n", + " EndTime=datetime.utcnow(),\n", + " MetricName=metric,\n", + " Namespace='AWS/EC2',\n", + " Statistics=[\n", + " stat\n", + " ],\n", + " Dimensions=[\n", + " {\n", + " 'Name':'InstanceId',\n", + " 'Value':instanceId\n", + " }\n", + " ]\n", + " )\n", + " \n", + " return data" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [], + "source": [ + "def get_images():\n", + " response = ec2Client.describe_images(\n", + " Owners=['self'],\n", + " Filters=[\n", + " {\n", + " 'Name': 'owner-id',\n", + " 'Values': [\n", + " str(account_number),\n", + " ]\n", + " },\n", + " ])['Images']\n", + " \n", + " data = {}\n", + " for image in response:\n", + " #print(image)\n", + " data[image['ImageId']] = image\n", + " \n", + " return data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "imageData = get_images()\n", + "pp.pprint(imageData)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "stat='Average'\n", + "metric='CPUUtilization'\n", + "wantedTags=['name','owner']\n", + "na='N/A'\n", + "\n", + "pp.pprint(ec2s['Reservations'])\n", + "\n", + "print(section_delimiter)\n", + "\n", + "print('# of EC2s: {}\\n'.format(str(len(ec2s['Reservations']))))\n", + "\n", + "for r in ec2s['Reservations']:\n", + " for i in r['Instances']:\n", + " tagStrings=[]\n", + " print('[EC2]')\n", + " print('Instanceid: {}'.format(i['InstanceId']))\n", + " print('InstanceType: {}'.format(i['InstanceType']))\n", + " print('Architecture: {}'.format(i['Architecture']))\n", + " print('State: {}'.format(i.get('State').get('Name')))\n", + " print()\n", + " print('ImageId: {}'.format(i['ImageId']))\n", + " \n", + " if i['ImageId'] in imageData:\n", + " print('ImageName: {}'.format(imageData[i['ImageId']].get('Name')))\n", + " print('ImageDescription: {}'.format(imageData[i['ImageId']].get('Description')))\n", + " \n", + " past = datetime.strptime(imageData[i['ImageId']].get('CreationDate'),'%Y-%m-%dT%H:%M:%S.000Z')\n", + " #print(past)\n", + " present = datetime.now()\n", + " #print(present)\n", + " delta = present-past\n", + " #print(delta.days)\n", + " \n", + " print('ImageAgeDays: {}'.format(str((present-past).days)))\n", + " \n", + " print('\\nVpcId: {}'.format(i.get('VpcId', na)))\n", + " print('SubnetId: {}\\n'.\n", + " format(i.get('SubnetId',na)))\n", + " \n", + " print('Tags:')\n", + " \n", + " for tag in i['Tags']:\n", + " if tag['Key'] in wantedTags:\n", + " tagStrings.append('Key: {}, Value: {}'.format(tag['Key'],tag['Value']))\n", + " \n", + " for tagString in sorted(tagStrings):\n", + " print(tagString)\n", + " \n", + " print()\n", + " \n", + " data = get_metric(i['InstanceId'], stat, metric)\n", + "\n", + " stats=[]\n", + " for dp in data['Datapoints']:\n", + " stats.append(dp[stat])\n", + "\n", + " print('Average CPU Utilization: {} %'.format(round(mean(stats),2))) \n", + " \n", + " print('\\nEBS:')\n", + " \n", + " ec2 = session.resource('ec2')\n", + " \n", + " for bdm in i['BlockDeviceMappings']:\n", + " volume = ec2.Volume(bdm['Ebs']['VolumeId'])\n", + " print('Volume ID: {}'.format(volume.volume_id))\n", + " print('Volume Size: {}GiBs'.format(volume.size))\n", + " print('Volume Type: {}'.format(volume.volume_type))\n", + " print('Device Mapping: {} delete on termination = {}'.format(bdm['DeviceName'],bdm['Ebs']['DeleteOnTermination']))\n", + "\n", + " \n", + " print(section_delimiter)\n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "sgs = ec2Client.describe_security_groups()\n", + "pp.pprint(sgs)\n", + "\n", + "print(section_delimiter)\n", + "\n", + "for group in sgs['SecurityGroups']:\n", + " #pp.pprint(group)\n", + " print(\"{} - {}\".format(group['GroupId'],group['Description']))\n", + "\n", + "print(section_delimiter)\n", + "\n", + "sgs = ec2Client.describe_security_groups(GroupIds=[''])\n", + "\n", + "pp.pprint(sgs)\n", + "#print(sgs)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "sqclient = boto3.client('service-quotas')\n", + "response = sqclient.list_service_quotas(\n", + " ServiceCode='ec2',\n", + " # NextToken='string',\n", + " # MaxResults=123\n", + ")\n", + "\n", + "pp.pprint(response)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3.7.3 64-bit", + "language": "python", + "name": "python37364bit80ce02abcfc949eb9541d0f033624b64" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/ecs.ipynb b/ecs.ipynb new file mode 100644 index 0000000..04675fb --- /dev/null +++ b/ecs.ipynb @@ -0,0 +1,200 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!pip3 install --upgrade pip\n", + "!pip3 install boto3\n", + "!pip3 install pyyaml" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "from functools import reduce\n", + "import os\n", + "import boto3\n", + "from botocore.client import Config\n", + "import yaml\n", + "from datetime import datetime, time, timedelta\n", + "import json\n", + "from pprint import pprint\n", + "from pprint import PrettyPrinter\n", + "from statistics import mean\n", + "import sys" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(sys.version_info)\n", + "print(sys.version_info[:3])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "config = yaml.load(open('config.yaml', 'r'))\n", + "#print(config)\n", + "aws_region = config['aws']['regions'][0]\n", + "account = config['aws']['accounts'][1]['name']\n", + "profile = config['aws']['accounts'][1]['profile']\n", + "account_number = config['aws']['accounts'][1]['account-number']\n", + "\n", + "print((\"%s - %s - %s - %s\" % (str(account_number),account,profile,aws_region)))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "session = boto3.session.Session(profile_name=profile,region_name=aws_region)\n", + "print(session)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ecsClient = session.client('ecs')\n", + "print(ecsClient)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "class EcsCluster(object):\n", + " def __init__(self, clusterName, clusterArn, status, registeredContainerInstancesCount, runningTasksCount, pendingTasksCount, activeServicesCount):\n", + " self.clusterName = clusterName\n", + " self.clusterArn = clusterArn\n", + " self.status = status\n", + " self.registeredContainerInstancesCount = registeredContainerInstancesCount\n", + " self.runningTasksCount = runningTasksCount\n", + " self.pendingTasksCount = pendingTasksCount\n", + " self.activeServicesCount = activeServicesCount\n", + " self.ec2s = []\n", + " \n", + " def set_ec2s(self, ec2s):\n", + " self.ec2s = ec2s\n", + " \n", + " def add_ec2s(self, ec2s):\n", + " self.ec2s.extend(ec2s)\n", + " \n", + " def __str__(self):\n", + " retrn = 'EcsCluster: clusterName={0}, clusterArn={1}, status={2}, registeredContainerInstancesCount={3}, runningTasksCount={4}, pendingTasksCount={5}, activeServicesCount={6}, ec2s={7}'.format(self.clusterName, self.clusterArn, self.status, str(self.registeredContainerInstancesCount), str(self.runningTasksCount), str(self.pendingTasksCount), str(self.activeServicesCount),tuple(self.ec2s))\n", + " return retrn" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Create a reusable Paginator\n", + "paginator = ecsClient.get_paginator('list_clusters')\n", + "#print(paginator)\n", + "\n", + "# Create a PageIterator from the Paginator\n", + "page_iterator = paginator.paginate()\n", + "\n", + "#clusterArns = []\n", + "clusters = []\n", + "#clusterEc2s = {}\n", + "\n", + "for page in page_iterator:\n", + " #print(page['clusterArns'])\n", + " #clusterArns.extend(page['clusterArns'])\n", + " \n", + " response = ecsClient.describe_clusters(clusters=page['clusterArns'])\n", + " #print(response)\n", + " for cluster in response.get('clusters'):\n", + " clusters.append(EcsCluster(cluster.get('clusterName'),cluster.get('clusterArn'),cluster.get('status'),cluster.get('registeredContainerInstancesCount'),cluster.get('runningTasksCount'),cluster.get('pendingTasksCount'),cluster.get('activeServicesCount')))\n", + " #print(response.get(clusters))\n", + " #print(page)\n", + "\n", + "#print(len(clusterArns))\n", + "#print(clusterArns)\n", + "#print(len(clusters))\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#Long Running\n", + "for cluster in clusters:\n", + " response = ecsClient.list_container_instances(cluster=cluster.clusterName)\n", + " cluster.set_ec2s(response.get('containerInstanceArns'))\n", + " #clusterEc2s[cluster.get('clusterName')] = response.get('containerInstanceArns')\n", + " \n", + "#print(clusterEc2s)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#pp = PrettyPrinter() #indent=1)\n", + "for cluster in clusters:\n", + " #print(cluster)\n", + " if cluster.registeredContainerInstancesCount > 10:\n", + " pp.pprint(str(cluster))\n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/helloworld.ipynb b/helloworld.ipynb new file mode 100644 index 0000000..a5998d5 --- /dev/null +++ b/helloworld.ipynb @@ -0,0 +1,49 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "hello world\n" + ] + } + ], + "source": [ + "print(\"hello world\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.3" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/iam.ipynb b/iam.ipynb new file mode 100644 index 0000000..470367d --- /dev/null +++ b/iam.ipynb @@ -0,0 +1,178 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!pip3 install --upgrade pip\n", + "!pip3 install boto3\n", + "!pip3 install pyyaml" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "from functools import reduce\n", + "import os\n", + "import boto3\n", + "import yaml\n", + "from datetime import datetime, time, timedelta\n", + "import json\n", + "from pprint import pprint\n", + "from pprint import PrettyPrinter\n", + "from statistics import mean\n", + "import sys" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "print(sys.version_info)\n", + "print(sys.version_info[:3])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "pp = PrettyPrinter() #indent=1)\n", + "print(pp)\n", + "section_delimiter = '\\n{}\\n'.format('--- '*20)\n", + "print(section_delimiter)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "config = yaml.safe_load(open('config.yaml', 'r'))\n", + "pp.pprint(config)\n", + "aws_region = config['aws']['regions'][0]\n", + "account = config['aws']['accounts'][0]['name']\n", + "profile = config['aws']['accounts'][0]['profile']\n", + "account_number = config['aws']['accounts'][0]['account-number']\n", + "\n", + "print(section_delimiter)\n", + "\n", + "print(('Selected Config: {} - {} - {} - {}'.format(str(account_number),account,profile,aws_region)))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "session = boto3.session.Session(profile_name=profile,region_name=aws_region)\n", + "print(('Session: {} - {}'.format(session.profile_name,session.region_name)))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "iam = session.client('iam')\n", + "print(iam)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "pols = iam.list_policies(OnlyAttached=True)\n", + "#pp.pprint(pols)\n", + "vers = dict()\n", + "\n", + "for pol in pols['Policies']:\n", + " vers[pol['PolicyName']] = pol['DefaultVersionId']\n", + "\n", + "pp.pprint(vers)\n", + "\n", + "for k,v in vers:\n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "pol = iam.get_policy(PolicyArn='arn:aws:iam::aws:policy/AmazonElasticTranscoder_FullAccess')\n", + "pp.pprint(pol['Policy']['Arn'])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "pol_ver = iam.get_policy_version(PolicyArn=pol['Policy']['Arn'],VersionId=pol['Policy']['DefaultVersionId'])\n", + "pp.pprint(pol_ver)\n", + "print(section_delimiter)\n", + "pp.pprint(pol_ver['PolicyVersion']['Document'])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3.7.3 64-bit", + "name": "python_defaultSpec_1595961918098" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": 3 + }, + "orig_nbformat": 2 + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/lambda-policy.ipynb b/lambda-policy.ipynb new file mode 100644 index 0000000..2cf0d19 --- /dev/null +++ b/lambda-policy.ipynb @@ -0,0 +1,146 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "!pip3 install --upgrade pip\n", + "!pip3 install boto3\n", + "!pip3 install pyyaml" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "import boto3\n", + "import json\n", + "session = boto3.session.Session()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "print session" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "awslambda = session.client('lambda')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "print awslambda" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "policy = awslambda.get_policy(FunctionName='CCT-Preprod')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "print policy" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "service = ['alexa-appkit.amazon.com','apigateway.amazonaws.com']\n", + "\n", + "#print len(policy)\n", + "\n", + "for key,value in policy.iteritems():\n", + " #print key\n", + " #print value\n", + " \n", + " if key == \"Policy\":\n", + " jsonPolicy = json.loads(value)\n", + " #print jsonPolicy\n", + " \n", + " if jsonPolicy['Statement']:\n", + " for statement in jsonPolicy['Statement']:\n", + " #print statement\n", + " if statement['Principal']['Service']:\n", + " for servs in service:\n", + " if servs in statement['Principal']['Service']:\n", + " print \"Found \" + statement['Principal']['Service']\n", + " print \"Sid \" + statement['Sid']\n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 2", + "language": "python", + "name": "python2" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.11" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/proxy-user.ipynb b/proxy-user.ipynb new file mode 100644 index 0000000..1d0a638 --- /dev/null +++ b/proxy-user.ipynb @@ -0,0 +1,54 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "import yaml\n", + "import sys\n", + "import os\n", + "\n", + "HTTP_PROXY_USER=os.environ.get('HTTP_PROXY_USER')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print('%s is the user.' %(HTTP_PROXY_USER))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.4" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/s3.ipynb b/s3.ipynb new file mode 100644 index 0000000..dd56bde --- /dev/null +++ b/s3.ipynb @@ -0,0 +1,376 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "deletable": true, + "editable": true + }, + "outputs": [], + "source": [ + "!pip3 install --upgrade pip\n", + "!pip3 install boto3\n", + "!pip3 install pyyaml" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": { + "collapsed": false, + "deletable": true, + "editable": true + }, + "outputs": [], + "source": [ + "from functools import reduce\n", + "import os\n", + "import boto3\n", + "from botocore.client import Config\n", + "from botocore.exceptions import ClientError\n", + "import yaml\n", + "from datetime import datetime, time, timedelta\n", + "import json\n", + "from pprint import pprint\n", + "from pprint import PrettyPrinter\n", + "from statistics import mean\n", + "import sys" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "deletable": true, + "editable": true, + "tags": [] + }, + "outputs": [], + "source": [ + "print(sys.version_info)\n", + "print(sys.version_info[:3])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "deletable": true, + "editable": true, + "tags": [] + }, + "outputs": [], + "source": [ + "config = yaml.load(open('config.yaml', 'r'))\n", + "#print(config)\n", + "aws_region = config['aws']['regions'][0]\n", + "account = config['aws']['accounts'][1]['name']\n", + "profile = config['aws']['accounts'][1]['profile']\n", + "account_number = config['aws']['accounts'][1]['account-number']\n", + "\n", + "print((\"%s - %s - %s - %s\" % (str(account_number),account,profile,aws_region)))" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [], + "source": [ + "def err_chk(fnCall, errormsg):\n", + " errStr = \"{} returned error: {}\".format(fnCall, e)\n", + " print(errStr)\n", + " return" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "deletable": true, + "editable": true, + "tags": [] + }, + "outputs": [], + "source": [ + "session = boto3.session.Session(profile_name=profile,region_name=aws_region)\n", + "print(session)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "deletable": true, + "editable": true, + "tags": [] + }, + "outputs": [], + "source": [ + "s3Client = session.client('s3', config=Config(signature_version='s3v4'))\n", + "print(s3Client)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "deletable": true, + "editable": true, + "tags": [] + }, + "outputs": [], + "source": [ + "buckets = s3Client.list_buckets()\n", + "print((\"%s buckets\" % (len(buckets.get('Buckets')))))\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "deletable": true, + "editable": true, + "tags": [] + }, + "outputs": [], + "source": [ + "pp = PrettyPrinter() #indent=1)\n", + "pp.pprint(buckets)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "tags": [] + }, + "outputs": [], + "source": [ + "location = s3Client.get_bucket_location(Bucket='')\n", + "pp.pprint(location)" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": { + "collapsed": false, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[]\n" + ] + } + ], + "source": [ + "denied = []\n", + "\n", + "for bucket in buckets.get('Buckets'):\n", + " bucketName = bucket.get('Name')\n", + " try:\n", + " head = s3Client.head_bucket(Bucket=bucketName)\n", + " except Exception as inst:\n", + " #print(type(inst)) # the exception instance\n", + " #print(inst.args[0]) # arguments stored in .args\n", + " if \"403\" in inst.args[0]:\n", + " denied.append(bucketName)\n", + " \n", + "print(denied)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "deletable": true, + "editable": true, + "tags": [] + }, + "outputs": [], + "source": [ + "try:\n", + " tags = s3Client.get_bucket_tagging(Bucket='')\n", + " print(tags)\n", + " pp.pprint(tags)\n", + "except ClientError as e:\n", + " err_chk(\"Tags could not be retrieved\", e)\n", + " #return" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "tags": [] + }, + "outputs": [], + "source": [ + "try:\n", + " policy = s3Client.get_bucket_policy(Bucket='')\n", + " pp.pprint(policy)\n", + "except ClientError as e:\n", + " err_chk(\"Policy could not be retrieved\", e)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "tags": [] + }, + "outputs": [], + "source": [ + "try:\n", + " acl = s3Client.get_bucket_acl(Bucket='')\n", + " pp.pprint(acl)\n", + "except ClientError as e:\n", + " err_chk(\"ACL could not be retrieved\",e)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "tags": [] + }, + "outputs": [], + "source": [ + "try:\n", + " notifications = s3Client.get_bucket_notification(Bucket='')\n", + " pp.pprint(notifications)\n", + "except ClientError as e:\n", + " err_chk(\"Notifications could not be retrieved\",e)" + ] + }, + { + "cell_type": "code", + "execution_count": 39, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "def tag_value_exists(tagSet, tagKey, tagValue):\n", + " for tag in tagSet.get('TagSet'):\n", + " if tag.get('Key') == tagKey:\n", + " if tag.get('Value') == tagValue:\n", + " return True\n", + " \n", + " return False" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "tags": [] + }, + "outputs": [], + "source": [ + "tagged = {}\n", + "failed = []\n", + "\n", + "for bucket in buckets.get('Buckets'):\n", + " #print(bucket)\n", + " bucketName = bucket.get('Name')\n", + " #print(bucketName)\n", + "\n", + " \n", + " #access = s3Client.head_bucket(Bucket=bucketName)\n", + " #print(access)\n", + " \n", + " if not bucketName in denied:\n", + " try:\n", + " tagSet = s3Client.get_bucket_tagging(Bucket=bucketName)\n", + " if tag_value_exists(tagSet, 'TAG_KEY', 'TAG_VALUE'):\n", + " print(bucketName)\n", + " tagged[bucketName] = tagSet\n", + " except:\n", + " failed.append(bucketName)\n", + "\n", + "print(tagged)\n", + "print(failed)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "tags": [] + }, + "outputs": [], + "source": [ + "pp.pprint(tagged)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "tags": [] + }, + "outputs": [], + "source": [ + "try:\n", + " objects = s3Client.list_objects(Bucket='')\n", + " print(objects)\n", + " contents = objects.get('Contents')\n", + " if contents != None:\n", + " print(contents)\n", + " print(len(contents))\n", + " pp.pprint(objects)\n", + "except ClientError as e:\n", + " err_chk(\"ACL could not be retrieved\",e)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.3-final" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/secrets.ipynb b/secrets.ipynb new file mode 100644 index 0000000..f049fb2 --- /dev/null +++ b/secrets.ipynb @@ -0,0 +1,140 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!pip3 install --upgrade pip\n", + "!pip3 install boto3\n", + "!pip3 install pyyaml" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "from functools import reduce\n", + "import os\n", + "import boto3\n", + "from botocore.client import Config\n", + "import yaml\n", + "from datetime import datetime, time, timedelta\n", + "import json\n", + "from pprint import pprint\n", + "from pprint import PrettyPrinter\n", + "from statistics import mean\n", + "import sys" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(sys.version_info)\n", + "print(sys.version_info[:3])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "config = yaml.load(open('config.yaml', 'r'))\n", + "#print(config)\n", + "aws_region = config['aws']['regions'][0]\n", + "account = config['aws']['accounts'][0]['name']\n", + "profile = config['aws']['accounts'][0]['profile']\n", + "account_number = config['aws']['accounts'][0]['account-number']\n", + "\n", + "print((\"%s - %s - %s - %s\" % (str(account_number),account,profile,aws_region)))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "session = boto3.session.Session(profile_name=profile,region_name=aws_region)\n", + "print(session)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "client = session.client('secretsmanager')\n", + "print(client)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Secret stored as K/V pair\n", + "name = \"my-secret\"\n", + "resp = client.get_secret_value(SecretId=name)\n", + "# print(resp)\n", + "print(type(resp['SecretString']))\n", + "print(resp['SecretString'])\n", + "# Known Key\n", + "print(json.loads(resp['SecretString']).get(\"my-secret-key\"))\n", + "# Unknown key\n", + "print(list(json.loads(resp['SecretString']).values())[0])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Secret stored as single value, no key\n", + "name = \"my-secret-also\"\n", + "resp = client.get_secret_value(SecretId=name)\n", + "# print(resp)\n", + "print(type(resp['SecretString']))\n", + "print(resp['SecretString'])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.2" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/service-catalog.ipynb b/service-catalog.ipynb new file mode 100644 index 0000000..2b4c51d --- /dev/null +++ b/service-catalog.ipynb @@ -0,0 +1,170 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!pip3 install --upgrade pip\n", + "!pip3 install boto3\n", + "!pip3 install pyyaml" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "from functools import reduce\n", + "import os\n", + "import boto3\n", + "import yaml\n", + "from datetime import datetime, time, timedelta\n", + "import json\n", + "from pprint import pprint\n", + "from pprint import PrettyPrinter\n", + "from statistics import mean\n", + "import sys" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "print(sys.version_info)\n", + "print(sys.version_info[:3])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "pp = PrettyPrinter() #indent=1)\n", + "print(pp)\n", + "section_delimiter = '\\n{}\\n'.format('--- '*20)\n", + "print(section_delimiter)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "config = yaml.safe_load(open('config.yaml', 'r'))\n", + "pp.pprint(config)\n", + "aws_region = config['aws']['regions'][0]\n", + "account = config['aws']['accounts'][1]['name']\n", + "profile = config['aws']['accounts'][1]['profile']\n", + "account_number = config['aws']['accounts'][1]['account-number']\n", + "\n", + "print(section_delimiter)\n", + "\n", + "print(('Selected Config: {} - {} - {} - {}'.format(str(account_number),account,profile,aws_region)))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "session = boto3.session.Session(profile_name=profile,region_name=aws_region)\n", + "print(('Session: {} - {}'.format(session.profile_name,session.region_name)))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "scClient = session.client('servicecatalog')\n", + "print(scClient)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "folios = scClient.list_portfolios()\n", + "pp.pprint(folios)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "tag_options = scClient.list_tag_options()\n", + "pp.pprint(tag_options)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "for tago in tag_options['TagOptionDetails']:\n", + " #pp.pprint(tago)\n", + " #print(tago['Id'])\n", + " pp.pprint(scClient.describe_tag_option(Id=tago['Id'])['TagOptionDetail'])\n", + " print(section_delimiter)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3.7.3 64-bit", + "language": "python", + "name": "python_defaultSpec_1595877268362" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.3" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +}