inf1-sentence-transformer.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "c98b2cb1-0871-4bd3-ac88-2fac276ded4b",
   "metadata": {},
   "source": [
    "# Sentence Transformers on Inf1"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "66b77e08-a3df-43c6-b101-805e05c6463b",
   "metadata": {
    "tags": []
   },
   "source": [
    "This notebook demonstrates the process of deploying sentence-transformers on Inf1 and utilizing all 4 neuron cores on inf1.xlarge instance type. The models have been traced using a separate script on bigger instance on EC2. Model can be traced within Studio Notebook as well."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1d707641-5ec0-42bb-b52f-42a4d47b325c",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Package installation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9caa4591-8a44-4b9c-b303-604d13eeed43",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!pip config set global.extra-index-url https://pip.repos.neuron.amazonaws.com"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a74697cc-1f04-4535-83e1-6a2766f40ff9",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!pip install torch-neuron neuron-cc[tensorflow] sagemaker transformers accelerate datasets --upgrade"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ca988d25-332e-4385-86f2-7de25417e353",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!pip install transformers -U "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "25c9706e-87f9-40be-bd89-89d57894d36e",
   "metadata": {},
   "source": [
    "## Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6fde25a3-361e-4e33-bc51-f86862ef5d96",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "import os\n",
    "import torch.neuron\n",
    "from transformers import AutoModel, AutoTokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "73c68e49-d57f-4aec-80bd-4b710bc765fc",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Based on the batch size select the corresponding directory\n",
    "batch_size = 50\n",
    "model_folder_name = f\"tmp{batch_size}\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f539d8d5-d026-41f2-a27a-438ed2ed448f",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%writefile code/inference.py\n",
    "\n",
    "print(\"Importing everything\")\n",
    "import os\n",
    "from transformers import AutoConfig, AutoTokenizer\n",
    "import torch\n",
    "import torch.neuron\n",
    "import torch.nn.functional as F\n",
    "import pathlib\n",
    "import json\n",
    "\n",
    "print(\"Imported everything\")\n",
    "\n",
    "# To use one neuron core per worker\n",
    "os.environ[\"NEURON_RT_NUM_CORES\"] = \"1\"\n",
    "batch_size = int(os.environ.get('ST_BATCH_SIZE', 10))\n",
    "max_length = os.environ.get('MAX_LENGTH', 256)\n",
    "\n",
    "# saved weights name\n",
    "AWS_NEURON_TRACED_WEIGHTS_NAME = f\"neuron_model_{batch_size}.pt\"\n",
    "print(f\"batch_size is {batch_size}\")\n",
    "\n",
    "#Mean Pooling - Take attention mask into account for correct averaging\n",
    "def mean_pooling(model_output, attention_mask):\n",
    "    token_embeddings = model_output[0] #First element of model_output contains all token embeddings\n",
    "    input_mask_expanded = attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float()\n",
    "    return torch.sum(token_embeddings * input_mask_expanded, 1) / torch.clamp(input_mask_expanded.sum(1), min=1e-9)\n",
    "\n",
    "def model_fn(model_dir):\n",
    "    # load tokenizer and neuron model from model_dir\n",
    "    tokenizer = AutoTokenizer.from_pretrained(model_dir)\n",
    "    print(f\"JIT Loading\")\n",
    "    model = torch.jit.load(os.path.join(model_dir, AWS_NEURON_TRACED_WEIGHTS_NAME))\n",
    "    print(f\"JIT Loading done\")\n",
    "    model_config = AutoConfig.from_pretrained(model_dir)\n",
    "    # encoded_input = tokenizer(sentences, padding=True, truncation=True, return_tensors='pt')    \n",
    "    return model, tokenizer, model_config\n",
    "\n",
    "def predict_fn(data, model_tokenizer_model_config):\n",
    "    # destruct model, tokenizer and model config\n",
    "    model, tokenizer, model_config = model_tokenizer_model_config\n",
    "\n",
    "    # create embeddings for inputs\n",
    "    print(\"Model input received\")\n",
    "    inputs = data.pop(\"inputs\", data)    \n",
    "    encoded_input = tokenizer(inputs, max_length=max_length, padding='max_length', truncation=True, return_tensors='pt')    # convert to tuple for neuron model\n",
    "    neuron_input = encoded_input['input_ids']    \n",
    "    print(f\"Neuron inputs created\")\n",
    "\n",
    "    # run prediciton\n",
    "    with torch.no_grad():\n",
    "        model_output = model(neuron_input)\n",
    "        sentence_embeddings = mean_pooling(model_output, encoded_input['attention_mask'])\n",
    "        sentence_embeddings = F.normalize(sentence_embeddings, p=2, dim=1)   \n",
    "\n",
    "    # return dictonary, which will be json serializable\n",
    "    return [{\"embeddings\": json.dumps(sentence_embeddings.tolist())}]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d545c6e9-f282-4b8f-be7c-eee190266017",
   "metadata": {},
   "source": [
    "## Sample inputs to trace the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9fad90ec-9928-4b8f-a520-bd39753a9b41",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "batches = [\"The movie had stunning visuals and a unique storyline, but the pacing felt off and some scenes were confusing\"] * batch_size\n",
    "max_length = 256"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7739409d-92f4-4ef1-bab8-cff491f2e25f",
   "metadata": {},
   "source": [
    "## Create a SageMaker session"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a80635b9-16a0-4ca2-84fa-11be4f3bff87",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import sagemaker\n",
    "import boto3\n",
    "sess = sagemaker.Session()\n",
    "# sagemaker session bucket -> used for uploading data, models and logs\n",
    "# sagemaker will automatically create this bucket if it not exists\n",
    "sagemaker_session_bucket=None\n",
    "if sagemaker_session_bucket is None and sess is not None:\n",
    "    # set to default bucket if a bucket name is not given\n",
    "    sagemaker_session_bucket = sess.default_bucket()\n",
    "\n",
    "try:\n",
    "    role = sagemaker.get_execution_role()\n",
    "except ValueError:\n",
    "    iam = boto3.client('iam')\n",
    "    role = iam.get_role(RoleName='sagemaker_execution_role')['Role']['Arn']\n",
    "\n",
    "sess = sagemaker.Session(default_bucket=sagemaker_session_bucket)\n",
    "\n",
    "print(f\"sagemaker role arn: {role}\")\n",
    "print(f\"sagemaker bucket: {sess.default_bucket()}\")\n",
    "print(f\"sagemaker session region: {sess.boto_region_name}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a293d2a9-6a8e-4464-a104-962c90b3e320",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!echo $model_folder_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3f167163-54ac-4986-8b80-31836320d257",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!cp -r ./code $model_folder_name\n",
    "\n",
    "# create a model.tar.gz archive with all the model artifacts and the inference.py script.\n",
    "%cd tmp50\n",
    "!tar zcvf model.tar.gz *\n",
    "%cd .."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "730f464a-2e54-4807-b749-d1efa820eca6",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# This is just for the s3 uploads and references\n",
    "model_id = \"sentence-transformer\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0fe2c133-9448-4871-9570-9e54f2aed5d0",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from sagemaker.s3 import S3Uploader\n",
    "\n",
    "# create s3 uri\n",
    "s3_model_path = f\"s3://{sess.default_bucket()}/{model_id}\"\n",
    "\n",
    "# upload model.tar.gz\n",
    "s3_model_uri = S3Uploader.upload(local_path=f\"tmp{batch_size}/model.tar.gz\", desired_s3_uri=s3_model_path)\n",
    "print(f\"model artifcats uploaded to {s3_model_uri}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5876b2d6-63a0-43bf-8301-8890fbd59ef6",
   "metadata": {},
   "source": [
    "## Model deployment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8457ff11-ad43-4bbb-9b82-3f995bc2b7c0",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from sagemaker.huggingface.model import HuggingFaceModel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "97accc35-863c-43cf-ac67-9cd8b62f3d16",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "image_uri_py37 = \"763104351884.dkr.ecr.us-west-2.amazonaws.com/huggingface-pytorch-inference-neuron:1.10.2-transformers4.20.1-neuron-py37-sdk1.19.1-ubuntu18.04\"\n",
    "image_uri_py310 = \"763104351884.dkr.ecr.us-west-2.amazonaws.com/pytorch-inference-neuron:1.13.1-neuron-py310-sdk2.14.1-ubuntu20.04\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8ace0c64-81a0-44f5-872d-66d97735fbf1",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "huggingface_model = HuggingFaceModel(\n",
    "   model_data=s3_model_uri,      # path to your model and script\n",
    "   role=role,                    # iam role with permissions to create an Endpoint\n",
    "   image_uri = image_uri_py37,\n",
    "   env={\n",
    "       \"SAGEMAKER_MODEL_SERVER_WORKERS\": \"4\", \n",
    "       \"ST_BATCH_SIZE\": \"50\",\n",
    "       \"NEURON_MONITOR_CW_REGION\": \"us-west-2\",\n",
    "       \"NEURON_MONITOR_CW_NAMESPACE\": \"/aws/sagemaker\",\n",
    "       \"NEURON_RT_LOG_LEVEL\": \"5\",   \n",
    "   }\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f000741e-4cbe-40f1-b255-dc31e43e3b7f",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "huggingface_model._is_compiled_model = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "303700d4-b150-43e4-8551-22330ba133cf",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "predictor = huggingface_model.deploy(\n",
    "    initial_instance_count=1,      # number of instances\n",
    "    instance_type=\"ml.inf1.xlarge\" # AWS Inferentia Instance\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b5b07b4e-1042-4886-bf19-cb0f22ea7df8",
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(3000):\n",
    "    for i in range(4):\n",
    "        result = predictor.predict({\"inputs\": batches})\n",
    "        # print(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "991ad339-154a-4b8d-8782-fec0fa1f7554",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8809ca79-838e-4aae-973e-74bf9ac80167",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "availableInstances": [
   {
    "_defaultOrder": 0,
    "_isFastLaunch": true,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 4,
    "name": "ml.t3.medium",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 1,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 8,
    "name": "ml.t3.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 2,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.t3.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 3,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.t3.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 4,
    "_isFastLaunch": true,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 8,
    "name": "ml.m5.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 5,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.m5.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 6,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.m5.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 7,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.m5.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 8,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.m5.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 9,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.m5.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 10,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.m5.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 11,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 384,
    "name": "ml.m5.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 12,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 8,
    "name": "ml.m5d.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 13,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.m5d.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 14,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.m5d.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 15,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.m5d.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 16,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.m5d.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 17,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.m5d.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 18,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.m5d.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 19,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 384,
    "name": "ml.m5d.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 20,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": true,
    "memoryGiB": 0,
    "name": "ml.geospatial.interactive",
    "supportedImageNames": [
     "sagemaker-geospatial-v1-0"
    ],
    "vcpuNum": 0
   },
   {
    "_defaultOrder": 21,
    "_isFastLaunch": true,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 4,
    "name": "ml.c5.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 22,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 8,
    "name": "ml.c5.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 23,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.c5.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 24,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.c5.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 25,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 72,
    "name": "ml.c5.9xlarge",
    "vcpuNum": 36
   },
   {
    "_defaultOrder": 26,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 96,
    "name": "ml.c5.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 27,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 144,
    "name": "ml.c5.18xlarge",
    "vcpuNum": 72
   },
   {
    "_defaultOrder": 28,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.c5.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 29,
    "_isFastLaunch": true,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.g4dn.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 30,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.g4dn.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 31,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.g4dn.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 32,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.g4dn.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 33,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 4,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.g4dn.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 34,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.g4dn.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 35,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 61,
    "name": "ml.p3.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 36,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 4,
    "hideHardwareSpecs": false,
    "memoryGiB": 244,
    "name": "ml.p3.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 37,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 488,
    "name": "ml.p3.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 38,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 768,
    "name": "ml.p3dn.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 39,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.r5.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 40,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.r5.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 41,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.r5.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 42,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.r5.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 43,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.r5.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 44,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 384,
    "name": "ml.r5.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 45,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 512,
    "name": "ml.r5.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 46,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 768,
    "name": "ml.r5.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 47,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.g5.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 48,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.g5.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 49,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.g5.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 50,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.g5.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 51,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.g5.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 52,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 4,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.g5.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 53,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 4,
    "hideHardwareSpecs": false,
    "memoryGiB": 384,
    "name": "ml.g5.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 54,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 768,
    "name": "ml.g5.48xlarge",
    "vcpuNum": 192
   },
   {
    "_defaultOrder": 55,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 1152,
    "name": "ml.p4d.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 56,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 1152,
    "name": "ml.p4de.24xlarge",
    "vcpuNum": 96
   }
  ],
  "instance_type": "ml.t3.medium",
  "kernelspec": {
   "display_name": "Python 3 (PyTorch 1.13 Python 3.9 CPU Optimized)",
   "language": "python",
   "name": "python3__SAGEMAKER_INTERNAL__arn:aws:sagemaker:us-west-2:236514542706:image/pytorch-1.13-cpu-py39"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}