diff --git a/.github/workflows/deploy_to_production.yml b/.github/workflows/deploy_to_production.yml index 10f1a2b..0f38ac1 100644 --- a/.github/workflows/deploy_to_production.yml +++ b/.github/workflows/deploy_to_production.yml @@ -14,12 +14,9 @@ jobs: steps: - uses: actions/checkout@v1 - name: Setup Node - uses: actions/setup-node@v2 + uses: actions/setup-node@v4 with: - node-version: 18 - # - name: Install AWS CDK - # shell: bash - # run: npm install -g aws-cdk + node-version: 20 - name: Install dependencies shell: bash run: npm install @@ -30,26 +27,9 @@ jobs: aws-secret-access-key: ${{ secrets.AWS_DEPLOY_SECRET_KEY }} aws-region: us-east-1 - name: Create .env file - uses: ozaytsev86/create-env-file@v1 + uses: aasmal97/create-env-file@v3.1.2 with: - ENV_AMAZON_REST_API_KEY: ${{ secrets.AMAZON_REST_API_KEY }} - ENV_GIT_HUB_PERSONAL_ACCESS_TOKEN: ${{ secrets.GIT_HUB_PERSONAL_ACCESS_TOKEN }} - ENV_GOOGLE_DRIVE_FOLDER_NAME: ${{ secrets.GOOGLE_DRIVE_FOLDER_NAME }} - ENV_GOOGLE_DRIVE_PARENT_FOLDER_NAME: ${{ secrets.GOOGLE_DRIVE_PARENT_FOLDER_NAME }} - ENV_GOOGLE_SERVICE_ACCOUNT_EMAIL: ${{ secrets.GOOGLE_SERVICE_ACCOUNT_EMAIL }} - ENV_GOOGLE_SERVICE_ACCOUNT_PRIVATE_KEY: ${{ secrets.GOOGLE_SERVICE_ACCOUNT_PRIVATE_KEY }} - ENV_STACKOVERFLOW_API_ID: ${{ secrets.STACKOVERFLOW_API_ID }} - ENV_STACKOVERFLOW_API_KEY: ${{ secrets.STACKOVERFLOW_API_KEY }} - ENV_STACKOVERFLOW_API_SECRET: ${{ secrets.STACKOVERFLOW_API_SECRET }} - ENV_WEBHOOKS_API_KEY: ${{ secrets.WEBHOOKS_API_KEY }} - ENV_WEBHOOKS_API_TOKEN: ${{ secrets.WEBHOOKS_API_TOKEN }} - ENV_WEBHOOKS_API_TOKEN_SECRET: ${{ secrets.WEBHOOKS_API_TOKEN_SECRET }} - ENV_AZURE_COMPUTER_VISION_API_ENDPOINT: ${{secrets.AZURE_COMPUTER_VISION_API_ENDPOINT}} - ENV_AZURE_COMPUTER_VISION_API_KEY: ${{secrets.AZURE_COMPUTER_VISION_API_KEY}} - ENV_LINKED_IN_PASSWORD: ${{secrets.LINKED_IN_PASSWORD}} - ENV_SES_EMAIL_ADDRESS: ${{secrets.SES_EMAIL_ADDRESS}} - ENV_SNS_PHONE_NUMBER: ${{secrets.SNS_PHONE_NUMBER}} - ENV_SEND_IN_BLUE_API_KEY: ${{secrets.SEND_IN_BLUE_API_KEY}} + APP_SECRETS: ${{toJSON(secrets)}} - name: Generate Cloud Template and Bootstrap shell: bash run: npm run bootstrap diff --git a/app/lib/restAPI/bundleApiFuncs.ts b/app/lib/restAPI/bundleApiFuncs.ts index f6d65db..1e0d099 100644 --- a/app/lib/restAPI/bundleApiFuncs.ts +++ b/app/lib/restAPI/bundleApiFuncs.ts @@ -1,20 +1,15 @@ import restApiMap from "./restApiMap"; import createFuncLocationMap from "../../../utils/createResources/createFuncLocationMap"; import { execShellCommand } from "../../../utils/buildFuncs/execShellCommand"; -import * as fs from "fs-extra"; -import path = require("path"); -async function copyDirectory(sourcePath: string, destPath: string) { - await fs.copy(sourcePath, destPath); - return `success, copied directory to ${destPath}`; -} const outPath = "../../../build/app/lib/restAPI/resources"; const locationFuncMap = createFuncLocationMap(restApiMap({})); const locationArr = Object.entries(locationFuncMap).map(([key, value]) => { const newPath = value.location.relative + "/index.ts"; return newPath; }); +//add skills cron job to compile +locationArr.push("./resources/skills/cronJob/index.ts"); const command = locationArr.reduce((a, b) => a + " " + b); - //bundle node api functions execShellCommand( `esbuild ${command} --bundle --platform=node --outdir=${outPath}`, @@ -24,20 +19,3 @@ execShellCommand( .catch((err) => { console.error(err); }); -//copy skill function to build folder -// Define the source and destination paths -const generalPath = path.join(__dirname, "./resources/skills/cronJob"); -const skillSourcePath = generalPath; -const skillDestPath = generalPath - //for windows - .replace("\\app\\lib\\restAPI\\", "\\build\\app\\lib\\restAPI\\") - //for linux - .replace("/app/lib/restAPI/", "/build/app/lib/restAPI/"); -console.log(skillSourcePath, skillDestPath, "paths"); -copyDirectory(skillSourcePath, skillDestPath) - .then((e) => { - console.log(e); - }) - .catch((err) => { - throw err; - }); diff --git a/app/lib/restAPI/resources/skills/cronJob/index.ts b/app/lib/restAPI/resources/skills/cronJob/index.ts new file mode 100644 index 0000000..3fb79da --- /dev/null +++ b/app/lib/restAPI/resources/skills/cronJob/index.ts @@ -0,0 +1,190 @@ +import { + DeleteItemCommand, + DynamoDBClient, + PutItemCommand, + QueryCommand, + QueryCommandInput, + UpdateItemCommand, +} from "@aws-sdk/client-dynamodb"; +import { marshall, unmarshall } from "@aws-sdk/util-dynamodb"; +import axios from "axios"; +type SkillsType = { + recordType: string; + name: string; + date_created: number | string; + order: number | string; +}; +type DataFuncsParams = Pick & { + skillName: SkillsType["name"]; +}; +const tableName = process.env.AMAZON_DYNAMO_DB_TABLE_NAME; + +const client = new DynamoDBClient({ + region: "us-east-1", +}); +const createSkill = async ({ skillName, order }: DataFuncsParams) => { + const currentTimestamp = Date.now(); + const item = { + recordType: "skill", + name: skillName, + date_created: currentTimestamp.toString(), + order, + }; + const putCommand = new PutItemCommand({ + TableName: tableName, + Item: marshall(item, { + convertClassInstanceToMap: true, + removeUndefinedValues: true, + }), + }); + return await client.send(putCommand); +}; +const deleteSkill = async (skillName: string) => { + const deleteCommand = new DeleteItemCommand({ + TableName: tableName, + Key: marshall( + { + recordType: "skill", + name: skillName, + }, + { + convertClassInstanceToMap: true, + removeUndefinedValues: true, + } + ), + }); + return await client.send(deleteCommand); +}; +const updateSkill = async ({ skillName, order }: DataFuncsParams) => { + const updateCommand = new UpdateItemCommand({ + TableName: tableName, + Key: marshall( + { + recordType: "skill", + name: skillName, + }, + { + convertClassInstanceToMap: true, + removeUndefinedValues: true, + } + ), + UpdateExpression: "SET #order = :order", + ExpressionAttributeNames: { + "#order": "order", + }, + ExpressionAttributeValues: marshall( + { + ":order": order, + }, + { + convertClassInstanceToMap: true, + removeUndefinedValues: true, + } + ), + }); + return await client.send(updateCommand); +}; + +const getSkillsInDynamoDbTable = async () => { + //create query command + const expValMap = { + ":skill": "skill", + }; + const expVal = marshall(expValMap, { + convertClassInstanceToMap: true, + removeUndefinedValues: true, + }); + const query: QueryCommandInput = { + TableName: tableName, + KeyConditionExpression: "#rt = :skill", + ExpressionAttributeNames: { + "#rt": "recordType", + }, + ExpressionAttributeValues: expVal, + }; + //send command + const command = new QueryCommand(query); + const response = await client.send(command); + const items = response.Items; + if (!items) return []; + const newItems = items.map((i) => unmarshall(i)) as SkillsType[]; + return newItems; +}; +const getLinkedInSkills = async () => { + try { + const req = await axios({ + method: "GET", + url: `https://nubela.co/proxycurl/api/v2/linkedin`, + headers: { + Authorization: `Bearer ${process.env.PROXYCURL_TOKEN}`, + }, + params: { + url: "https://linkedin.com/in/arky-asmal/", + fallback_to_cache: "on-error", + use_cache: "if-present", + skills: "include", + }, + }); + const data = req.data || {}; + if (!data.skills) return []; + const linkedInSkills = data.skills as string[]; + return linkedInSkills; + } catch (err) { + return []; + } +}; +const storeInDatabase = async (skills: string[]) => { + const skillsInDB = await getSkillsInDynamoDbTable(); + const skillsInDBMap: Record = Object.assign( + {}, + ...skillsInDB.map((val) => ({ [val.name]: val })) + ); + const skillsInLinkedInMap: Record< + string, + Pick + > = Object.assign( + {}, + ...skills.map((val, idx) => ({ + [val]: { + order: idx, + name: val, + }, + })) + ); + //loop through linkedin skills + const modifyPromiseArr = skills.map((val, idx) => { + //check if creation is needed + const skillName = val; + if (!(skillName in skillsInDBMap)) + return createSkill({ + skillName, + order: idx, + }); + //check if update needed + const storedIdx = skillsInDBMap[skillName].order; + const parsedIdx = + typeof storedIdx === "string" ? parseInt(storedIdx) : storedIdx; + if (parsedIdx === idx) return null; + return updateSkill({ + skillName, + order: idx, + }); + }); + //check if deletion is needed + //loop through skill in database already + const deletePromiseArr = skillsInDB.map((val) => { + const skillName = val.name; + if (!(skillName in skillsInLinkedInMap)) return deleteSkill(skillName); + return null; + }); + const results = await Promise.all([...modifyPromiseArr, ...deletePromiseArr]); + return results; +}; + +export async function handler() { + const linkedInSkills = await getLinkedInSkills(); + //this means failure, as LinkedIn skills will never be at zero + if (linkedInSkills.length <= 0) return; + const res = storeInDatabase(linkedInSkills); + return res; +} diff --git a/app/lib/restAPI/resources/skills/cronJob/main.py b/app/lib/restAPI/resources/skills/cronJob/main.py deleted file mode 100644 index 7575d37..0000000 --- a/app/lib/restAPI/resources/skills/cronJob/main.py +++ /dev/null @@ -1,105 +0,0 @@ -import os -import boto3 -import datetime -from boto3.dynamodb.types import TypeDeserializer -from boto3.dynamodb.conditions import Key -from dotenv import load_dotenv -from typing import List, Dict -from linkedin_api import Linkedin - -load_dotenv() -deserializer = TypeDeserializer() -# specify table name -table_name = os.getenv("AMAZON_DYNAMO_DB_TABLE_NAME") -table = boto3.resource("dynamodb", region_name="us-east-1").Table(table_name) - - -def unmarshall(x): - unmarshalled_item = {} - for key, value in x.items(): - unmarshalled_item[key] = deserializer.deserialize(value) - return unmarshalled_item - - -def get_skills_items() -> List[Dict[str, str]]: - # define query parameters - query_params = { - "TableName": table_name, - "KeyConditionExpression": Key("recordType").eq("skill"), - } - # make the query to DynamoDB and get results - response = table.query(**query_params) - # print all items returned - items = response["Items"] - return items - -def create_skill(name: str, order: int | None = None): - current_timestamp = datetime.datetime.now().timestamp() - item = { - "recordType": "skill", - "name": name, - "date_created": str(current_timestamp), - } - if order is not None: # if order is provided, add it to the item - item["order"] = (order) - response = table.put_item(Item=item) - return response - -def delete_skill(name: str): - response = table.delete_item( - Key={ - "recordType": "skill", - "name": name, - } - ) - return response -def update_skill(name: str, order: int | None = None): - response = table.update_item( - Key={ - "recordType": "skill", - "name": name, - }, - UpdateExpression="SET #order = :order", - ExpressionAttributeNames={ - "#order": "order" - }, - ExpressionAttributeValues={ - ":order": order - } - ) - return response - -def store_in_db(skills: List[Dict[str, str]]): - skills_in_db = get_skills_items() - skills_in_db_hashmap = {d["name"]: d for d in skills_in_db} - skills_hashmap = {d["name"]: d for d in skills} - for skillIdx in range(0, len(skills)): - skill = skills[skillIdx] - name = skill["name"] - if name not in skills_in_db_hashmap: - create_skill(name = name, order = skillIdx) - else: - stored_idx = int(skills_in_db_hashmap[name]["order"]) - if stored_idx != skillIdx: - update_skill(name=name, order = skillIdx) - for skill in skills_in_db: - name = skill["name"] - if name not in skills_hashmap: - delete_skill(name) - return skills - - -def get_skills() -> List[Dict[str, str]]: - # Authenticate using any Linkedin account credentials - linkedin_pw = os.getenv("LINKED_IN_PASSWORD") - api = Linkedin("arkyasmal97@gmail.com", linkedin_pw) - # GET a profile - profile_name = "arky-asmal" - profile_skills = api.get_profile_skills(profile_name) - return profile_skills - - -def lambda_handler(): - skills = get_skills() - store_res = store_in_db(skills) - return store_res diff --git a/app/lib/restAPI/resources/skills/cronJob/requirements.txt b/app/lib/restAPI/resources/skills/cronJob/requirements.txt deleted file mode 100644 index 5294346..0000000 --- a/app/lib/restAPI/resources/skills/cronJob/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -linkedin_api -python-dotenv -boto3 \ No newline at end of file diff --git a/app/lib/restAPI/resources/skills/cronJob/test.py b/app/lib/restAPI/resources/skills/cronJob/test.py deleted file mode 100644 index 5a4f127..0000000 --- a/app/lib/restAPI/resources/skills/cronJob/test.py +++ /dev/null @@ -1,5 +0,0 @@ -from main import lambda_handler -result = lambda_handler() -# result = get_skills_items() -print(result) -# update_skill(name="Autoclave", order=1) \ No newline at end of file diff --git a/app/lib/restAPI/resources/skills/cronJob/test.ts b/app/lib/restAPI/resources/skills/cronJob/test.ts new file mode 100644 index 0000000..bf4157d --- /dev/null +++ b/app/lib/restAPI/resources/skills/cronJob/test.ts @@ -0,0 +1,8 @@ +import { handler } from "./"; +import * as dotenv from "dotenv"; +dotenv.config(); +handler() + .then((res) => { + console.log(res); + }) + .catch((err) => console.error(err)); diff --git a/app/lib/restAPI/resources/utils/createResources/createSkillCronJob.ts b/app/lib/restAPI/resources/utils/createResources/createSkillCronJob.ts index 1e61beb..29bab84 100644 --- a/app/lib/restAPI/resources/utils/createResources/createSkillCronJob.ts +++ b/app/lib/restAPI/resources/utils/createResources/createSkillCronJob.ts @@ -1,18 +1,19 @@ import * as cdk from "aws-cdk-lib"; -import { PythonFunction } from "@aws-cdk/aws-lambda-python-alpha"; import { Runtime } from "aws-cdk-lib/aws-lambda"; import { createLambdaRole } from "../../../../../../utils/rolesFuncs/createLambdaRole"; import { createDynamoPolicy } from "../../../../../../utils/rolesFuncs/createDynamoPolicy"; import { createCronEvent } from "../../../../../../utils/createResources/createCronEvent"; import { LambdaFunction } from "aws-cdk-lib/aws-events-targets"; +import * as lambda from "aws-cdk-lib/aws-lambda"; + import path = require("path"); export const createSkillCronJob = ({ stack, skillsTableInfo, - secrets, - dirname + secrets, + dirname, }: { - dirname: string; + dirname: string; stack: cdk.Stack; skillsTableInfo: { name: string; @@ -21,12 +22,14 @@ export const createSkillCronJob = ({ }; secrets: { [key: string]: any }; }) => { - const skillsCronLambda = new PythonFunction(stack, "skillsCronJobLambda", { - entry: path.join(dirname, "./resources/skills/cronJob"), - runtime: Runtime.PYTHON_3_9, - index: "main.py", - handler: "lambda_handler", + const skillsCronLambda = new lambda.Function(stack, "skillsCronJobLambda", { + runtime: Runtime.NODEJS_20_X, + handler: `index.handler`, + code: lambda.Code.fromAsset( + path.join(dirname, "./resources/skills/cronJob").toString() + ), timeout: cdk.Duration.minutes(14), + memorySize: 512, role: createLambdaRole( "skillsCronJobLambdaRole", { @@ -42,16 +45,16 @@ export const createSkillCronJob = ({ ), environment: { AMAZON_DYNAMO_DB_TABLE_NAME: skillsTableInfo.name, - LINKED_IN_PASSWORD: secrets.LINKED_IN_PASSWORD, + PROXYCURL_TOKEN: secrets.PROXYCURL_TOKEN, }, }); const skillsCronJobTarget = new LambdaFunction(skillsCronLambda, { - retryAttempts: 2, + retryAttempts: 1, }); const skillsCronJobEvent = createCronEvent({ stack: stack, id: "skillsCronJobEvent", - hours: 23, + hours: 72, targets: [skillsCronJobTarget], }); diff --git a/app/lib/webhooks/resources/googleDrive/googleDrivePost/stepFunction/stateMachine.ts b/app/lib/webhooks/resources/googleDrive/googleDrivePost/stepFunction/stateMachine.ts index f03d4bf..02cc886 100644 --- a/app/lib/webhooks/resources/googleDrive/googleDrivePost/stepFunction/stateMachine.ts +++ b/app/lib/webhooks/resources/googleDrive/googleDrivePost/stepFunction/stateMachine.ts @@ -110,7 +110,9 @@ export const createGoogleDrivePostStateMachine = ({ const googleDrivePostStateMachine = stack && googleDrivePostDefintion ? new sfn.StateMachine(stack, `${googleDrivePostName}StateMachine`, { - definition: googleDrivePostDefintion, + definitionBody: sfn.DefinitionBody.fromChainable( + googleDrivePostDefintion + ), timeout: Duration.minutes(14), logs: { destination: new logs.LogGroup( diff --git a/package.json b/package.json index 17b6f3a..1d0819b 100644 --- a/package.json +++ b/package.json @@ -34,8 +34,6 @@ "typescript": "=4.9.4" }, "dependencies": { - "@aws-cdk/aws-lambda-python-alpha": "=2.115.0-alpha.0", - "@aws-cdk/aws-sns-subscriptions": "=1.198.1", "@aws-sdk/client-dynamodb": "=3.259.0", "@aws-sdk/client-s3": "=3.259.0", "@aws-sdk/client-ses": "=3.315.0", @@ -43,7 +41,7 @@ "@aws-sdk/util-dynamodb": "=3.262.0", "aws-cdk-lib": "=2.115.0", "aws-sdk": "=2.1303.0", - "axios": "=1.3.2", + "axios": "=1.7.9", "constructs": "=10.0.0", "date-fns": "=2.29.3", "dotenv": "=16.0.3", @@ -53,7 +51,6 @@ "html-to-text": "=9.0.4", "jimp": "=0.22.7", "jsonwebtoken": "=9.0.0", - "libphonenumber": "=0.0.10", "libphonenumber-js": "=1.10.26", "lodash": "=4.17.21", "source-map-support": "=0.5.21", diff --git a/utils/apiTemplates/getTemplate.ts b/utils/apiTemplates/getTemplate.ts index 35e7a83..8c947ed 100644 --- a/utils/apiTemplates/getTemplate.ts +++ b/utils/apiTemplates/getTemplate.ts @@ -156,38 +156,6 @@ export const queryUntilRequestPageNum = async ({ [partitionKey]: docKey[partitionKey], [sortKey]: docKey[sortKey], }; - // const docPartitionKey = docKey[partitionKey]; - // const docSortKey = docKey[sortKey]; - // const expression = `#partition = :partitionVal and #sort = :sortVal`; - // const docWithKeyResult = await queryOnce({ - // tableName, - // query: { - // TableName: tableName, - // KeyConditionExpression: expression, - // ExpressionAttributeNames: { - // "#partition": partitionKey, - // "#sort": sortKey, - // }, - // ExpressionAttributeValues: { - // ":partitionVal": docPartitionKey, - // ":sortVal": docSortKey, - // }, - // Limit: 1, - // }, - // }); - //error encountered retrieving document - //if (isAPIGatewayResult(docWithKeyResult)) return docWithKeyResult; - //return successResponse(results, successMessage); - //results.LastEvaluatedKey = docWithKeyResult.LastEvaluatedKey; - // const newKey = marshall( - // { ...docWithNewKey["pk"] }, - // { - // convertClassInstanceToMap: true, - // removeUndefinedValues: true, - // } - // ); - // marshall(docWithNewKey) - //results.LastEvaluatedKey = newKey } results.Count = results.Items.length; if (!results.LastEvaluatedKey || numLeft <= 0) { diff --git a/utils/createResources/createApiTree.ts b/utils/createResources/createApiTree.ts index e75bab1..9e2cb22 100644 --- a/utils/createResources/createApiTree.ts +++ b/utils/createResources/createApiTree.ts @@ -6,7 +6,7 @@ import createFuncLocationMap, { apiMethods, camelCase, } from "./createFuncLocationMap"; -import { aws_iam, Stack } from "aws-cdk-lib"; +import { aws_iam } from "aws-cdk-lib"; import { FunctionOptions } from "aws-cdk-lib/aws-lambda"; import { MethodLoggingLevel } from "aws-cdk-lib/aws-apigateway"; export const generateLocation = (