combined-stack.yaml

## Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
#copies of the Software, and to permit persons to whom the Software is
#furnished to do so, subject to the following conditions:
#
#The above copyright notice and this permission notice shall be included in all
#copies or substantial portions of the Software.
#
#THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
#IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
#FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
#AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
#LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
#OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
#SOFTWARE.

AWSTemplateFormatVersion: '2010-09-09'
Description: This is a sample template. Please review security roles and policies and update template before using.
Parameters:
  EnvName:
    Type: String
    Description: Name of an environment. 'dev', 'staging', 'prod' and any name.
    AllowedPattern: ^.*[^0-9]$
    ConstraintDescription: Must end with non-numeric character.
    Default: poc

  LambdaCodepath:
    Type: String
    Default: "https://raw.githubusercontent.com/aws-samples/sagemaker-autopilot-sample-solution/main/src/"

  LambdaCodeZipFile:
    Type: String
    Default: "create-automl-pipeline.zip"

  LayerPckgPath:
    Type: String
    Default: "https://raw.githubusercontent.com/aws-samples/sagemaker-autopilot-sample-solution/main/src/"

  LayerPkgZipFile:
    Type: String
    Default: "sgmkr-sdk-layer.zip"

  SourceCodeBucket:
    Type: String
    Default: smpp-demo-bucket-sourcecode

  DataBucket:
    Type: String
    Default: smpp-demo-bucket-data
  ModelBucket:
    Type: String
    Default: smpp-demo-bucket-model
  LambdaHandlerPath:  
    Type: String
    Description: Path of a Lambda Handler. 
    AllowedPattern: ^.*[^0-9]$
    ConstraintDescription: Must end with non-numeric character.
    Default: lambda_function.lambda_handler

  LambdaFunctionName:
    Type: String
    AllowedPattern: '[a-zA-Z0-9]+[a-zA-Z0-9-]+[a-zA-Z0-9]+'
    Default: demo-sgmkr-pipeline-lambda

  TargetID:
    Type: String
    Default: targetid for event rule

  SNSARN:
    Type: String
    Default: ARN of the SNS Topic

  PipelineName:
    Type: String
    Default: SMTestPipeline

Outputs:
  SourceCodeBucket:
    Value: !Ref S3Bucket1
  
  DataBucket:
    Value: !Ref S3Bucket2

  ModelBucket:
    Value: !Ref S3Bucket3

  LambdaRoleARN:
    Description: Role for Lambda execution.
    Value:
      Fn::GetAtt:
        - LambdaRole
        - Arn
    Export:
      Name:
        Fn::Sub: LambdaRole-${EnvName}

  LambdaFunctionName:
    Value:
      Ref: LambdaFunction

  LambdaFunctionARN:
    Description: Lambda function ARN.
    Value:
      Fn::GetAtt:
        - LambdaFunction
        - Arn
    Export:
      Name:
        Fn::Sub: LambdaARN-${EnvName}
  InvokePipelineLambdaFunctionName:
    Value:
      Ref: InvokeSMPipelineLambda
  InvokePipelineLambdaFunctionARN:
    Description: Lambda function ARN.
    Value:
      Fn::GetAtt:
        - InvokeSMPipelineLambda
        - Arn
  DeployEndpointLambdaFunctionName:
    Value:
      Ref: DeploySMEndpointLambda
  DeployEndpointLambdaFunctionARN:
    Description: Lambda function ARN.
    Value:
      Fn::GetAtt:
        - DeploySMEndpointLambda
        - Arn

Resources:

  S3Bucket1:
    Type: AWS::S3::Bucket
    #Description: Bucket1
    Properties:
      BucketName: !Ref SourceCodeBucket

  S3Bucket2:
    Type: AWS::S3::Bucket
    #Description: Bucket2
    Properties:
      BucketName: !Ref DataBucket

  S3Bucket3:
    Type: AWS::S3::Bucket
    #Description: Bucket3
    Properties:
      BucketName: !Ref ModelBucket

  EventRule:
    Type: AWS::Events::Rule
    Properties:
      Description: rule to publish sagemaker pipeline status events
      EventBusName: default
      EventPattern:
        source:
          - aws.sagemaker
        detail-type:
          - SageMaker Model Building Pipeline Execution Status Change
      Name: !Sub Automl-sgmkr-pipeline-status-${EnvName}
      State: ENABLED
      Targets:
        - Id: !Ref TargetID
          Arn: !Ref SNSARN
          InputTransformer:
            InputPathsMap:
              event-type: $.detail-type
              id: $.id
              pipelinearn: $.detail.pipelineArn
              status: $.detail.currentPipelineExecutionStatus
            InputTemplate: |-
              {
                "id": "<id>",
                "event-type":"<event-type>",
                "pipelinearn": "<pipelinearn>",
                "status": "<status>"
              }

  LambdaRole:
    Type: AWS::IAM::Role
    Properties:
      RoleName:
        Fn::Sub: lambda-role-${EnvName}
      AssumeRolePolicyDocument:
        Statement:
          - Action:
            - sts:AssumeRole
            Effect: Allow
            Principal:
              Service:
              - lambda.amazonaws.com
              - sagemaker.amazonaws.com
        Version: 2012-10-17
      ManagedPolicyArns:
        - arn:aws:iam::aws:policy/AWSLambda_FullAccess
        - arn:aws:iam::aws:policy/AmazonS3FullAccess
        - arn:aws:iam::aws:policy/AmazonSageMakerFullAccess
        - arn:aws:iam::aws:policy/AmazonSQSFullAccess
      Path: /
      Policies:
        - PolicyName: iamgetrole
          PolicyDocument:
            Version: "2012-10-17"
            Statement:
              - Effect: Allow
                Action: 'iam:GetRole'
                Resource: '*'
  LambdaLayer:
    Type: AWS::Lambda::LayerVersion
    Properties:
      CompatibleRuntimes:
        - python3.9
      #update content property with location of code for lambda layer
      Content: src/sgmkr-sdk-layer
      Description: Sagemaker SDK layer
      LayerName: sgmkr-sdk-layer
      LicenseInfo: MIT
  
  LambdaFunction:
    Type: AWS::Lambda::Function
    Properties:
      FunctionName:
        Fn::Sub: create-sgmkr-pipeline-${EnvName}
      Description: LambdaFunction to create and execute sagemaker pipeline
      Runtime: python3.9
      #update code property with location of code for lambda function
      Code: src/create-automl-pipeline
      Handler: lambda_function.lambda_handler
      Layers:
          - !Ref LambdaLayer
      MemorySize: 256
      Timeout: 30
      Role:
        Fn::GetAtt:
          - LambdaRole
          - Arn
      Environment:
        Variables:
          ENV:
            Fn::Sub: ${EnvName}
          TZ: UTC

  lambdaLogGroup:
    Type: AWS::Logs::LogGroup
    Properties:
      LogGroupName: !Sub /aws/lambda/${LambdaFunctionName}/${EnvName}
      RetentionInDays: 90

  InvokeSMPipelineLambda:
    Type: AWS::Lambda::Function
    Properties:
      Code:
        ZipFile: |
          # Requirements:
          # sagemaker
          # boto3>=1.24.*
          # botocore>=1.27.*
          import json
          import boto3
          import pandas as pd
          from sagemaker import (
              AutoML,
              AutoMLInput,
              get_execution_role,
              MetricsSource,
              ModelMetrics,
              ModelPackage,
          )

          def lambda_handler(event, context):
              print("Log the received event")
              print("Received event: " + json.dumps(event, indent=2))
              # body=json.loads(event["body"])
              body=event
              
              pipeline_name = body['pipelinename']
              databucket= body['databucket']
              modelpackagename= body['modelpackagename']
              
              # Get a handle to the pipeline.
              from sagemaker.workflow.pipeline import Pipeline
              pipeline = Pipeline(name=pipeline_name)

              # Show the parameters.

              pipeline_description = pipeline.describe()
              pipeline_parameters = json.loads(pipeline_description['PipelineDefinition'])
              print(pd.DataFrame(pipeline_parameters['Parameters']))
              
              # Start the pipeline.
              pipeline_execution = pipeline.start(
                  parameters=dict(
                      InputUri=databucket,
                      ModelPackageName=modelpackagename,
                  )
              )
              print(pipeline_execution.describe())
              
              return {"statusCode": 200, "body": json.dumps("pipeline: " + pipeline_name + " execution started successfully.")} 

      FunctionName: !Sub invokesmpipeline-automl-${EnvName}
      Handler: index.lambda_handler
      Runtime: python3.9
      Layers:
          - !Ref LambdaLayer
      MemorySize: 256
      Timeout: 30
      Role:
        Fn::GetAtt:
          - LambdaRole
          - Arn
      Environment:
        Variables:
          ENV:
            Fn::Sub: ${EnvName}
          TZ: UTC
  DeploySMEndpointLambda:
    Type: AWS::Lambda::Function
    Properties:
      FunctionName: !Sub deploysmept-automl-${EnvName}
      Description: LambdaFunction to deloy model to sagemaker endpoint
      Runtime: python3.9
      Code:
        ZipFile: |
          """
          This Lambda function deploys the model to SageMaker Endpoint.
          If Endpoint exists, then Endpoint will be updated with new Endpoint Config.
          """

          import json
          import boto3
          import time
          import sagemaker

          sm_client = boto3.client("sagemaker")


          def lambda_handler(event, context):

              print(f"Received Event: {event}")
              body=event

              current_time = time.strftime("%m-%d-%H-%M-%S", time.gmtime())
              endpoint_instance_type = body["endpoint_instance_type"]
              base_model_name = body["model_name"]
              endpoint_config_name = "{}-{}".format(body["endpoint_config_name"], current_time)
              endpoint_name = body["endpoint_name"]

              model_package = sm_client.list_model_packages(
                  MaxResults=1,
                  ModelApprovalStatus="Approved",
                  ModelPackageGroupName=body["model_name"],
                  SortBy="CreationTime",
                  SortOrder="Descending",
              )
              print(model_package)
              model_version_arn = model_package["ModelPackageSummaryList"][0]["ModelPackageArn"]
              print(model_version_arn)

              model_name = f"{base_model_name}-{current_time}"
              print(f"Model name : {model_name}")
              container_list = [{'ModelPackageName': model_version_arn}]

              create_model_response = sm_client.create_model(
                  ModelName = model_name,
                  ExecutionRoleArn = sagemaker.get_execution_role(),
                  Containers = container_list
              )
              print(f"Model ARN: {create_model_response['ModelArn']}")

              endpoint_config_name = f"{base_model_name}-EndpointConfig-{current_time}"
              print(endpoint_config_name)
              create_endpoint_config_response = sm_client.create_endpoint_config(
                  EndpointConfigName = endpoint_config_name,
                  ProductionVariants=[{
                      'InstanceType': endpoint_instance_type,
                      'InitialVariantWeight': 1,
                      'InitialInstanceCount': 1,
                      'ModelName': model_name,
                      'VariantName': 'AllTraffic'}])

              # Check if an endpoint exists. If no - Create new endpoint, if yes - Update existing endpoint
              list_endpoints_response = sm_client.list_endpoints(
                  SortBy="CreationTime",
                  SortOrder="Descending",
                  NameContains=endpoint_name,
              )
              print(f"list_endpoints_response: {list_endpoints_response}")

              if len(list_endpoints_response["Endpoints"]) > 0:
                  print("Updating Endpoint with new Endpoint Configuration")
                  update_endpoint_response = sm_client.update_endpoint(
                      EndpointName=endpoint_name, EndpointConfigName=endpoint_config_name
                  )
                  print(f"update_endpoint_response: {update_endpoint_response}")
              else:
                  print("Creating Endpoint")
                  create_endpoint_response = sm_client.create_endpoint(
                      EndpointName=endpoint_name, EndpointConfigName=endpoint_config_name
                  )
                  print(f"create_endpoint_response: {create_endpoint_response}")

              return {"statusCode": 200, "body": json.dumps("Endpoint Created Successfully")}
              
      Handler: index.lambda_handler
      Runtime: python3.9
      Layers:
          - !Ref LambdaLayer
      MemorySize: 256
      Timeout: 30
      Role:
        Fn::GetAtt:
          - LambdaRole
          - Arn
      Environment:
        Variables:
          ENV:
            Fn::Sub: ${EnvName}
          TZ: UTC


  PrimerInvoke:
    Type: AWS::CloudFormation::CustomResource
    DependsOn: [LambdaFunction]
    Properties:
      ServiceToken: !GetAtt LambdaFunction.Arn
      pipelinename: !Ref PipelineName
      role: !Ref LambdaRole
      bucket: !Ref ModelBucket