From e1c117971de3008fd38247f8c464c21a5ab9d8ac Mon Sep 17 00:00:00 2001 From: Dylan Kapnias Date: Wed, 5 Jul 2023 17:35:09 +0200 Subject: [PATCH 01/14] Backend has manual deployment --- backend/package.json | 8 +- backend/pm2.cfg.json | 11 ++ .../assets/configure_ubuntu_writetopdf.sh | 15 +++ infrastructure/bin/infrastructure.ts | 32 ++--- infrastructure/lib/amplify-stack.ts | 61 ++++++++++ infrastructure/lib/ec2-stack.ts | 113 ++++++++++++++++++ infrastructure/lib/infrastructure-stack.ts | 80 ------------- infrastructure/lib/s3-stack.ts | 36 ++++++ 8 files changed, 260 insertions(+), 96 deletions(-) create mode 100644 backend/pm2.cfg.json create mode 100644 infrastructure/assets/configure_ubuntu_writetopdf.sh create mode 100644 infrastructure/lib/amplify-stack.ts create mode 100644 infrastructure/lib/ec2-stack.ts delete mode 100644 infrastructure/lib/infrastructure-stack.ts create mode 100644 infrastructure/lib/s3-stack.ts diff --git a/backend/package.json b/backend/package.json index d2630855..2aec5a18 100644 --- a/backend/package.json +++ b/backend/package.json @@ -12,13 +12,17 @@ "start": "nest start", "start:dev": "nest start --watch", "start:debug": "nest start --debug --watch", - "start:prod": "node dist/main", + "start:prod": "node dist/src/main", "lint": "eslint \"{src,apps,libs,test}/**/*.ts\" --fix", "test": "jest --detectOpenHandles", "test:watch": "jest --watch", "test:cov": "jest --coverage", "test:debug": "node --inspect-brk -r tsconfig-paths/register -r ts-node/register node_modules/.bin/jest --runInBand", - "test:e2e": "jest --config ./test/jest-e2e.json --detectOpenHandles" + "test:e2e": "jest --config ./test/jest-e2e.json --detectOpenHandles", + "pm2:deploy:app": "npm run build && pm2 start pm2.cfg.json", + "pm2:start:app": "pm2 start pm2.cfg.json", + "pm2:stop:app": "pm2 stop pm2.cfg.json", + "pm2:destroy:app": "pm2 delete pm2.cfg.json" }, "dependencies": { "@aws-sdk/client-s3": "^3.350.0", diff --git a/backend/pm2.cfg.json b/backend/pm2.cfg.json new file mode 100644 index 00000000..0e94f661 --- /dev/null +++ b/backend/pm2.cfg.json @@ -0,0 +1,11 @@ + +{ + "apps": [ + { + "name": "api", + "script": "dist/src/main.js", + "instances": 1, + "exec_mode": "cluster" + } + ] + } \ No newline at end of file diff --git a/infrastructure/assets/configure_ubuntu_writetopdf.sh b/infrastructure/assets/configure_ubuntu_writetopdf.sh new file mode 100644 index 00000000..e6cc57af --- /dev/null +++ b/infrastructure/assets/configure_ubuntu_writetopdf.sh @@ -0,0 +1,15 @@ +#!/bin/bash -xe +# Install OS packages +sudo apt-get update +sudo apt-get upgrade -y +sudo apt-get install git -y +curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.34.0/install.sh | bash +. ~/.nvm/nvm.sh +nvm install 16.19.1 +sudo npm install pm2 -g + +git clone https://github.com/COS301-SE-2023/WriteToPdf +cd WriteToPdf +cd backend +git switch test +npm ci \ No newline at end of file diff --git a/infrastructure/bin/infrastructure.ts b/infrastructure/bin/infrastructure.ts index 6f5684b2..761c84df 100644 --- a/infrastructure/bin/infrastructure.ts +++ b/infrastructure/bin/infrastructure.ts @@ -1,21 +1,25 @@ #!/usr/bin/env node -import "source-map-support/register"; -import * as cdk from "aws-cdk-lib"; -import { WriteToPDFStack } from "../lib/infrastructure-stack"; +import { App } from "aws-cdk-lib"; +import { S3Stack } from "../lib/s3-stack"; +import { AmplifyStack } from "../lib/amplify-stack"; +import { EC2Stack } from "../lib/ec2-stack"; -const app = new cdk.App(); -new WriteToPDFStack(app, "WriteToPDFStack", { - /* If you don't specify 'env', this stack will be environment-agnostic. - * Account/Region-dependent features and context lookups will not work, - * but a single synthesized template can be deployed anywhere. */ +const app = new App(); - /* Uncomment the next line to specialize this stack for the AWS Account - * and Region that are implied by the current CLI configuration. */ - // env: { account: process.env.CDK_DEFAULT_ACCOUNT, region: process.env.CDK_DEFAULT_REGION }, +new S3Stack(app, "WriteToPDFS3Stack", { + env: { region: "us-east-1" }, + description: "Stack with resources needed to deploy WriteToPDF's S3 Bucket", +}); - /* Uncomment the next line if you know exactly what Account and Region you - * want to deploy the stack to. */ +new AmplifyStack(app, "WriteToPDFAmplifyStack", { env: { region: "us-east-1" }, + description: + "Stack with resources needed to deploy WriteToPDF's Amplify Frontend", +}); - /* For more information, see https://docs.aws.amazon.com/cdk/latest/guide/environments.html */ +new EC2Stack(app, "WriteToPDFEC2Stack", { + env: { region: "us-east-1" }, + description: "Stack with resources needed to deploy WriteToPDF's EC2 Backend", }); + +app.synth(); diff --git a/infrastructure/lib/amplify-stack.ts b/infrastructure/lib/amplify-stack.ts new file mode 100644 index 00000000..69228069 --- /dev/null +++ b/infrastructure/lib/amplify-stack.ts @@ -0,0 +1,61 @@ +import { + GitHubSourceCodeProvider, + App as Amp_App, +} from "@aws-cdk/aws-amplify-alpha"; +import { App, CfnOutput, SecretValue, Stack, StackProps } from "aws-cdk-lib"; +import { BuildSpec } from "aws-cdk-lib/aws-codebuild"; + +export class AmplifyStack extends Stack { + constructor(scope: App, id: string, props?: StackProps) { + super(scope, id, props); + + // ========================== AMPLIFY FRONTEND ========================== + + const sourceCodeProvider = new GitHubSourceCodeProvider({ + // oauthToken: cdk.SecretValue.secretsManager("GITHUB_TOKEN"), + oauthToken: SecretValue.unsafePlainText(""), + owner: "COS301-SE-2023", + repository: "WriteToPdf", + }); + + const environmentVariables = { + AMPLIFY_MONOREPO_APP_ROOT: "frontend", + }; + + const buildSpec = BuildSpec.fromObjectToYaml({ + version: "1.0", + applications: [ + { + appRoot: "frontend", + frontend: { + phases: { + preBuild: { + commands: ["npm ci"], + }, + build: { + commands: ["npm run build"], + }, + }, + artifacts: { + baseDirectory: "dist/frontend", + files: ["**/*"], + }, + cache: { + paths: ["node_modules/**/*"], + }, + }, + }, + ], + }); + + const amplifyFrontend = new Amp_App(this, "WriteToPDFAmplify", { + sourceCodeProvider, + environmentVariables, + buildSpec, + }); + + const testBranch = amplifyFrontend.addBranch("test", { + autoBuild: true, + }); + } +} diff --git a/infrastructure/lib/ec2-stack.ts b/infrastructure/lib/ec2-stack.ts new file mode 100644 index 00000000..ac0fd4d1 --- /dev/null +++ b/infrastructure/lib/ec2-stack.ts @@ -0,0 +1,113 @@ +import { App, CfnOutput, Stack, StackProps, Tags } from "aws-cdk-lib"; +import { + Vpc, + SubnetType, + Peer, + Port, + AmazonLinuxGeneration, + AmazonLinuxCpuType, + Instance, + SecurityGroup, + AmazonLinuxImage, + InstanceClass, + InstanceSize, + InstanceType, + MachineImage, + OperatingSystemType, +} from "aws-cdk-lib/aws-ec2"; +import { Role, ServicePrincipal, ManagedPolicy } from "aws-cdk-lib/aws-iam"; +import { readFileSync } from "fs"; + +export class EC2Stack extends Stack { + constructor(scope: App, id: string, props?: StackProps) { + super(scope, id, props); + + // IAM + // Policy for CodeDeploy bucket access + // Role that will be attached to the EC2 instance so it can be + // managed by AWS SSM + const webServerRole = new Role(this, "ec2Role", { + assumedBy: new ServicePrincipal("ec2.amazonaws.com"), + }); + + // IAM policy attachment to allow access to + webServerRole.addManagedPolicy( + ManagedPolicy.fromAwsManagedPolicyName("AmazonSSMManagedInstanceCore") + ); + + webServerRole.addManagedPolicy( + ManagedPolicy.fromAwsManagedPolicyName( + "service-role/AmazonEC2RoleforAWSCodeDeploy" + ) + ); + + // VPC + // This VPC has 3 public subnets, and that's it + const vpc = new Vpc(this, "WriteToPDFVPC", { + subnetConfiguration: [ + { + cidrMask: 24, + name: "pub01", + subnetType: SubnetType.PUBLIC, + }, + { + cidrMask: 24, + name: "pub02", + subnetType: SubnetType.PUBLIC, + }, + { + cidrMask: 24, + name: "pub03", + subnetType: SubnetType.PUBLIC, + }, + ], + }); + + // Security Groups + // This SG will only allow HTTP traffic to the Web server + const webSg = new SecurityGroup(this, "WriteToPDFWebSecurityGroup", { + vpc, + description: "Allows Inbound HTTP traffic to the web server.", + allowAllOutbound: true, + }); + + webSg.addIngressRule(Peer.anyIpv4(), Port.tcp(22)); + webSg.addIngressRule(Peer.anyIpv4(), Port.tcp(80)); + webSg.addIngressRule(Peer.anyIpv4(), Port.tcp(3000)); + + // EC2 Instance + // Get the latest AmazonLinux 2 AMI for the given region + const ami = new AmazonLinuxImage({ + generation: AmazonLinuxGeneration.AMAZON_LINUX_2, + cpuType: AmazonLinuxCpuType.X86_64, + }); + + const ubuntu = MachineImage.fromSsmParameter( + "/aws/service/canonical/ubuntu/server/focal/stable/current/amd64/hvm/ebs-gp2/ami-id", + { os: OperatingSystemType.LINUX } + ); + // The actual Web EC2 Instance for the web server + const webServer = new Instance(this, "WriteToPDFWebServer", { + vpc, + instanceType: InstanceType.of(InstanceClass.T2, InstanceSize.MICRO), + machineImage: ubuntu, + securityGroup: webSg, + role: webServerRole, + }); + + // User data - used for bootstrapping + const webSGUserData = readFileSync( + "./assets/configure_ubuntu_writetopdf.sh", + "utf-8" + ); + webServer.addUserData(webSGUserData); + // Tag the instance + Tags.of(webServer).add("application-name", "writetopdf-api"); + Tags.of(webServer).add("stage", "prod"); + + // Output the public IP address of the EC2 instance + new CfnOutput(this, "IP Address", { + value: webServer.instancePublicIp, + }); + } +} diff --git a/infrastructure/lib/infrastructure-stack.ts b/infrastructure/lib/infrastructure-stack.ts deleted file mode 100644 index d1263b81..00000000 --- a/infrastructure/lib/infrastructure-stack.ts +++ /dev/null @@ -1,80 +0,0 @@ -import * as cdk from "aws-cdk-lib"; -import * as s3 from "aws-cdk-lib/aws-s3"; -import * as iam from "aws-cdk-lib/aws-iam"; -import * as s3Deploy from "aws-cdk-lib/aws-s3-deployment"; -import * as cloudfront from "aws-cdk-lib/aws-cloudfront"; -import * as amplify from "@aws-cdk/aws-amplify-alpha"; -import { BuildSpec } from "aws-cdk-lib/aws-codebuild"; -// import { Role, ServicePrincipal, ManagedPolicy } from 'aws-cdk-lib/aws-iam'; - -export class WriteToPDFStack extends cdk.Stack { - constructor(scope: cdk.App, id: string, props?: cdk.StackProps) { - super(scope, id, props); - - // Create S3 bucket to hold user assets - const bucket = new s3.Bucket(this, "WriteToPDFAppBucket", { - publicReadAccess: false, - removalPolicy: cdk.RemovalPolicy.DESTROY, - // blockPublicAccess: s3.BlockPublicAccess.BLOCK_ACLS, - accessControl: s3.BucketAccessControl.BUCKET_OWNER_FULL_CONTROL, - }); - - const user = new iam.User(this, "MyIAMUser", { - userName: "writetopdf-webapp-test", - }); - - const policy = new iam.PolicyStatement({ - actions: ["s3:PutObject", "s3:GetObject", "s3:DeleteObject"], - resources: [bucket.bucketArn + "/*"], - }); - - user.addToPolicy(policy); - - const sourceCodeProvider = new amplify.GitHubSourceCodeProvider({ - // oauthToken: cdk.SecretValue.secretsManager("GITHUB_TOKEN"), - oauthToken: cdk.SecretValue.unsafePlainText(""), - owner: "COS301-SE-2023", - repository: "WriteToPdf", - }); - - const environmentVariables = { - AMPLIFY_MONOREPO_APP_ROOT: "frontend", - }; - - const buildSpec = BuildSpec.fromObjectToYaml({ - version: "1.0", - applications: [ - { - appRoot: "frontend", - frontend: { - phases: { - preBuild: { - commands: ["npm ci"], - }, - build: { - commands: ["npm run build"], - }, - }, - artifacts: { - baseDirectory: "dist/frontend", - files: ["**/*"], - }, - cache: { - paths: ["node_modules/**/*"], - }, - }, - }, - ], - }); - - const amplifyFrontend = new amplify.App(this, "WriteToPDFAmplify", { - sourceCodeProvider, - environmentVariables, - buildSpec, - }); - - const testBranch = amplifyFrontend.addBranch("test", { - autoBuild: true, - }); - } -} diff --git a/infrastructure/lib/s3-stack.ts b/infrastructure/lib/s3-stack.ts new file mode 100644 index 00000000..153abbd9 --- /dev/null +++ b/infrastructure/lib/s3-stack.ts @@ -0,0 +1,36 @@ +import { App, CfnOutput, RemovalPolicy, Stack, StackProps } from "aws-cdk-lib"; +import { PolicyStatement, User } from "aws-cdk-lib/aws-iam"; +import { Bucket, BucketAccessControl } from "aws-cdk-lib/aws-s3"; + +export class S3Stack extends Stack { + constructor(scope: App, id: string, props?: StackProps) { + super(scope, id, props); + + // Create S3 bucket to hold user assets + const bucket = new Bucket(this, "WriteToPDFAppBucket", { + publicReadAccess: false, + removalPolicy: RemovalPolicy.DESTROY, + // blockPublicAccess: s3.BlockPublicAccess.BLOCK_ACLS, + accessControl: BucketAccessControl.BUCKET_OWNER_FULL_CONTROL, + }); + + const user = new User(this, "MyIAMUser", { + userName: "writetopdf-webapp-test", + }); + + const policy = new PolicyStatement({ + actions: ["s3:PutObject", "s3:GetObject", "s3:DeleteObject"], + resources: [bucket.bucketArn + "/*"], + }); + + user.addToPolicy(policy); + + new CfnOutput(this, "S3BucketName", { + value: bucket.bucketName, + }); + + new CfnOutput(this, "S3BucketRegion", { + value: bucket.bucketRegionalDomainName, + }); + } +} From 4fe3b175bf86a5ad27508192f8128fb8cf6b4ec5 Mon Sep 17 00:00:00 2001 From: Dylan Kapnias Date: Fri, 7 Jul 2023 09:59:36 +0200 Subject: [PATCH 02/14] Added extra branched to Amplify. Added setup script for EC2 --- .../assets/configure_ubuntu_writetopdf.sh | 25 ++++++++++--------- infrastructure/lib/amplify-stack.ts | 8 ++++++ 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/infrastructure/assets/configure_ubuntu_writetopdf.sh b/infrastructure/assets/configure_ubuntu_writetopdf.sh index e6cc57af..58d6ce0a 100644 --- a/infrastructure/assets/configure_ubuntu_writetopdf.sh +++ b/infrastructure/assets/configure_ubuntu_writetopdf.sh @@ -1,15 +1,16 @@ #!/bin/bash -xe # Install OS packages -sudo apt-get update -sudo apt-get upgrade -y -sudo apt-get install git -y -curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.34.0/install.sh | bash -. ~/.nvm/nvm.sh -nvm install 16.19.1 -sudo npm install pm2 -g +sudo apt-get update && +sudo apt-get upgrade -y && +sudo apt-get install git -y && +curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.34.0/install.sh | bash && +. ~/.nvm/nvm.sh && +nvm install 16.19.1 && +sudo npm install pm2 -g && -git clone https://github.com/COS301-SE-2023/WriteToPdf -cd WriteToPdf -cd backend -git switch test -npm ci \ No newline at end of file +git clone https://github.com/COS301-SE-2023/WriteToPdf && +cd WriteToPdf && +cd backend && +git switch dev/devOps && +npm ci && +npm run pm2:deploy:app \ No newline at end of file diff --git a/infrastructure/lib/amplify-stack.ts b/infrastructure/lib/amplify-stack.ts index 69228069..99613a0d 100644 --- a/infrastructure/lib/amplify-stack.ts +++ b/infrastructure/lib/amplify-stack.ts @@ -54,8 +54,16 @@ export class AmplifyStack extends Stack { buildSpec, }); + const mainBranch = amplifyFrontend.addBranch("main", { + autoBuild: true, + }); + const testBranch = amplifyFrontend.addBranch("test", { autoBuild: true, }); + + const devBranch = amplifyFrontend.addBranch("dev/devOps", { + autoBuild: true, + }); } } From 5b1c2fe80942cbb59a99c1956e8bbca96bf9405c Mon Sep 17 00:00:00 2001 From: Dylan Kapnias Date: Fri, 7 Jul 2023 11:30:05 +0200 Subject: [PATCH 03/14] Testing Frontend deployment --- frontend/angular.json | 3 +++ frontend/package.json | 1 + frontend/src/environments/environment.staging.ts | 4 ++-- infrastructure/assets/configure_ubuntu_writetopdf.sh | 3 ++- infrastructure/lib/amplify-stack.ts | 2 +- 5 files changed, 9 insertions(+), 4 deletions(-) diff --git a/frontend/angular.json b/frontend/angular.json index 94ecf08a..86a9c653 100644 --- a/frontend/angular.json +++ b/frontend/angular.json @@ -96,6 +96,9 @@ }, "development": { "browserTarget": "frontend:build:development" + }, + "staging": { + "browserTarget": "frontend:build:staging" } }, "defaultConfiguration": "development" diff --git a/frontend/package.json b/frontend/package.json index 75b70efb..4bc8caf7 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -8,6 +8,7 @@ "build": "ng build", "watch": "ng build --watch --configuration development", "staging": "ng build --watch --configuration staging", + "prod": "ng build --configuration staging", "test": "cypress run", "test:unit": "jest", "format": "prettier --write \"src/**/*.ts\" \"src/**/*.html\"" diff --git a/frontend/src/environments/environment.staging.ts b/frontend/src/environments/environment.staging.ts index e9e5c3a9..c0d99eb6 100644 --- a/frontend/src/environments/environment.staging.ts +++ b/frontend/src/environments/environment.staging.ts @@ -1,4 +1,4 @@ export const environment = { - production: false, - apiURL: 'http://example.com/', // New URL must end in a slash + production: false, + apiURL: 'http://54.86.91.186:3000/', // New URL must end in a slash }; diff --git a/infrastructure/assets/configure_ubuntu_writetopdf.sh b/infrastructure/assets/configure_ubuntu_writetopdf.sh index 58d6ce0a..667f6674 100644 --- a/infrastructure/assets/configure_ubuntu_writetopdf.sh +++ b/infrastructure/assets/configure_ubuntu_writetopdf.sh @@ -3,10 +3,11 @@ sudo apt-get update && sudo apt-get upgrade -y && sudo apt-get install git -y && +cd /home/ubuntu/ && curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.34.0/install.sh | bash && . ~/.nvm/nvm.sh && nvm install 16.19.1 && -sudo npm install pm2 -g && +npm install pm2 -g && git clone https://github.com/COS301-SE-2023/WriteToPdf && cd WriteToPdf && diff --git a/infrastructure/lib/amplify-stack.ts b/infrastructure/lib/amplify-stack.ts index 99613a0d..f8ada75e 100644 --- a/infrastructure/lib/amplify-stack.ts +++ b/infrastructure/lib/amplify-stack.ts @@ -33,7 +33,7 @@ export class AmplifyStack extends Stack { commands: ["npm ci"], }, build: { - commands: ["npm run build"], + commands: ["npm run prod"], }, }, artifacts: { From a515b05c4bd53cf14aee8466cf26ba56738fda5d Mon Sep 17 00:00:00 2001 From: Dylan Kapnias Date: Sun, 9 Jul 2023 14:15:01 +0200 Subject: [PATCH 04/14] Added restart script --- backend/package.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/package.json b/backend/package.json index 2aec5a18..415ff4f3 100644 --- a/backend/package.json +++ b/backend/package.json @@ -22,7 +22,8 @@ "pm2:deploy:app": "npm run build && pm2 start pm2.cfg.json", "pm2:start:app": "pm2 start pm2.cfg.json", "pm2:stop:app": "pm2 stop pm2.cfg.json", - "pm2:destroy:app": "pm2 delete pm2.cfg.json" + "pm2:destroy:app": "pm2 delete pm2.cfg.json", + "pm2:restart:app": "npm run pm2:stop:app && npm run pm2:deploy:app" }, "dependencies": { "@aws-sdk/client-s3": "^3.350.0", From 6fb31ea62aa22fcf8f84a3d26d8ba95e7045d7b3 Mon Sep 17 00:00:00 2001 From: Dylan Kapnias Date: Sun, 23 Jul 2023 13:46:27 +0200 Subject: [PATCH 05/14] Create Textract Service and Client --- backend/package-lock.json | 929 +++++++++++++++++- backend/package.json | 1 + backend/src/app.module.ts | 4 + .../src/textract/textract.controller.spec.ts | 18 + backend/src/textract/textract.controller.ts | 4 + backend/src/textract/textract.module.ts | 7 + backend/src/textract/textract.service.spec.ts | 18 + backend/src/textract/textract.service.ts | 33 + 8 files changed, 1007 insertions(+), 7 deletions(-) create mode 100644 backend/src/textract/textract.controller.spec.ts create mode 100644 backend/src/textract/textract.controller.ts create mode 100644 backend/src/textract/textract.module.ts create mode 100644 backend/src/textract/textract.service.spec.ts create mode 100644 backend/src/textract/textract.service.ts diff --git a/backend/package-lock.json b/backend/package-lock.json index 9710bb01..ddac7b74 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -10,6 +10,7 @@ "license": "UNLICENSED", "dependencies": { "@aws-sdk/client-s3": "^3.350.0", + "@aws-sdk/client-textract": "^3.370.0", "@nestjs/common": "^9.0.0", "@nestjs/core": "^9.0.0", "@nestjs/jwt": "^10.0.3", @@ -536,6 +537,467 @@ "node": ">=14.0.0" } }, + "node_modules/@aws-sdk/client-textract": { + "version": "3.370.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-textract/-/client-textract-3.370.0.tgz", + "integrity": "sha512-xsGlxaljuaWT6oNlUM19e0kONBnGMpoIrnyDQO3RtTPrOzBJrfeKRJmKMODi2rP/IBUXRNJZAN3esn7vBgYDzA==", + "dependencies": { + "@aws-crypto/sha256-browser": "3.0.0", + "@aws-crypto/sha256-js": "3.0.0", + "@aws-sdk/client-sts": "3.370.0", + "@aws-sdk/credential-provider-node": "3.370.0", + "@aws-sdk/middleware-host-header": "3.370.0", + "@aws-sdk/middleware-logger": "3.370.0", + "@aws-sdk/middleware-recursion-detection": "3.370.0", + "@aws-sdk/middleware-signing": "3.370.0", + "@aws-sdk/middleware-user-agent": "3.370.0", + "@aws-sdk/types": "3.370.0", + "@aws-sdk/util-endpoints": "3.370.0", + "@aws-sdk/util-user-agent-browser": "3.370.0", + "@aws-sdk/util-user-agent-node": "3.370.0", + "@smithy/config-resolver": "^1.0.1", + "@smithy/fetch-http-handler": "^1.0.1", + "@smithy/hash-node": "^1.0.1", + "@smithy/invalid-dependency": "^1.0.1", + "@smithy/middleware-content-length": "^1.0.1", + "@smithy/middleware-endpoint": "^1.0.2", + "@smithy/middleware-retry": "^1.0.3", + "@smithy/middleware-serde": "^1.0.1", + "@smithy/middleware-stack": "^1.0.1", + "@smithy/node-config-provider": "^1.0.1", + "@smithy/node-http-handler": "^1.0.2", + "@smithy/protocol-http": "^1.1.0", + "@smithy/smithy-client": "^1.0.3", + "@smithy/types": "^1.1.0", + "@smithy/url-parser": "^1.0.1", + "@smithy/util-base64": "^1.0.1", + "@smithy/util-body-length-browser": "^1.0.1", + "@smithy/util-body-length-node": "^1.0.1", + "@smithy/util-defaults-mode-browser": "^1.0.1", + "@smithy/util-defaults-mode-node": "^1.0.1", + "@smithy/util-retry": "^1.0.3", + "@smithy/util-utf8": "^1.0.1", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/client-textract/node_modules/@aws-sdk/client-sso": { + "version": "3.370.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso/-/client-sso-3.370.0.tgz", + "integrity": "sha512-0Ty1iHuzNxMQtN7nahgkZr4Wcu1XvqGfrQniiGdKKif9jG/4elxsQPiydRuQpFqN6b+bg7wPP7crFP1uTxx2KQ==", + "dependencies": { + "@aws-crypto/sha256-browser": "3.0.0", + "@aws-crypto/sha256-js": "3.0.0", + "@aws-sdk/middleware-host-header": "3.370.0", + "@aws-sdk/middleware-logger": "3.370.0", + "@aws-sdk/middleware-recursion-detection": "3.370.0", + "@aws-sdk/middleware-user-agent": "3.370.0", + "@aws-sdk/types": "3.370.0", + "@aws-sdk/util-endpoints": "3.370.0", + "@aws-sdk/util-user-agent-browser": "3.370.0", + "@aws-sdk/util-user-agent-node": "3.370.0", + "@smithy/config-resolver": "^1.0.1", + "@smithy/fetch-http-handler": "^1.0.1", + "@smithy/hash-node": "^1.0.1", + "@smithy/invalid-dependency": "^1.0.1", + "@smithy/middleware-content-length": "^1.0.1", + "@smithy/middleware-endpoint": "^1.0.2", + "@smithy/middleware-retry": "^1.0.3", + "@smithy/middleware-serde": "^1.0.1", + "@smithy/middleware-stack": "^1.0.1", + "@smithy/node-config-provider": "^1.0.1", + "@smithy/node-http-handler": "^1.0.2", + "@smithy/protocol-http": "^1.1.0", + "@smithy/smithy-client": "^1.0.3", + "@smithy/types": "^1.1.0", + "@smithy/url-parser": "^1.0.1", + "@smithy/util-base64": "^1.0.1", + "@smithy/util-body-length-browser": "^1.0.1", + "@smithy/util-body-length-node": "^1.0.1", + "@smithy/util-defaults-mode-browser": "^1.0.1", + "@smithy/util-defaults-mode-node": "^1.0.1", + "@smithy/util-retry": "^1.0.3", + "@smithy/util-utf8": "^1.0.1", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/client-textract/node_modules/@aws-sdk/client-sso-oidc": { + "version": "3.370.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso-oidc/-/client-sso-oidc-3.370.0.tgz", + "integrity": "sha512-jAYOO74lmVXylQylqkPrjLzxvUnMKw476JCUTvCO6Q8nv3LzCWd76Ihgv/m9Q4M2Tbqi1iP2roVK5bstsXzEjA==", + "dependencies": { + "@aws-crypto/sha256-browser": "3.0.0", + "@aws-crypto/sha256-js": "3.0.0", + "@aws-sdk/middleware-host-header": "3.370.0", + "@aws-sdk/middleware-logger": "3.370.0", + "@aws-sdk/middleware-recursion-detection": "3.370.0", + "@aws-sdk/middleware-user-agent": "3.370.0", + "@aws-sdk/types": "3.370.0", + "@aws-sdk/util-endpoints": "3.370.0", + "@aws-sdk/util-user-agent-browser": "3.370.0", + "@aws-sdk/util-user-agent-node": "3.370.0", + "@smithy/config-resolver": "^1.0.1", + "@smithy/fetch-http-handler": "^1.0.1", + "@smithy/hash-node": "^1.0.1", + "@smithy/invalid-dependency": "^1.0.1", + "@smithy/middleware-content-length": "^1.0.1", + "@smithy/middleware-endpoint": "^1.0.2", + "@smithy/middleware-retry": "^1.0.3", + "@smithy/middleware-serde": "^1.0.1", + "@smithy/middleware-stack": "^1.0.1", + "@smithy/node-config-provider": "^1.0.1", + "@smithy/node-http-handler": "^1.0.2", + "@smithy/protocol-http": "^1.1.0", + "@smithy/smithy-client": "^1.0.3", + "@smithy/types": "^1.1.0", + "@smithy/url-parser": "^1.0.1", + "@smithy/util-base64": "^1.0.1", + "@smithy/util-body-length-browser": "^1.0.1", + "@smithy/util-body-length-node": "^1.0.1", + "@smithy/util-defaults-mode-browser": "^1.0.1", + "@smithy/util-defaults-mode-node": "^1.0.1", + "@smithy/util-retry": "^1.0.3", + "@smithy/util-utf8": "^1.0.1", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/client-textract/node_modules/@aws-sdk/client-sts": { + "version": "3.370.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-sts/-/client-sts-3.370.0.tgz", + "integrity": "sha512-utFxOPWIzbN+3kc415Je2o4J72hOLNhgR2Gt5EnRSggC3yOnkC4GzauxG8n7n5gZGBX45eyubHyPOXLOIyoqQA==", + "dependencies": { + "@aws-crypto/sha256-browser": "3.0.0", + "@aws-crypto/sha256-js": "3.0.0", + "@aws-sdk/credential-provider-node": "3.370.0", + "@aws-sdk/middleware-host-header": "3.370.0", + "@aws-sdk/middleware-logger": "3.370.0", + "@aws-sdk/middleware-recursion-detection": "3.370.0", + "@aws-sdk/middleware-sdk-sts": "3.370.0", + "@aws-sdk/middleware-signing": "3.370.0", + "@aws-sdk/middleware-user-agent": "3.370.0", + "@aws-sdk/types": "3.370.0", + "@aws-sdk/util-endpoints": "3.370.0", + "@aws-sdk/util-user-agent-browser": "3.370.0", + "@aws-sdk/util-user-agent-node": "3.370.0", + "@smithy/config-resolver": "^1.0.1", + "@smithy/fetch-http-handler": "^1.0.1", + "@smithy/hash-node": "^1.0.1", + "@smithy/invalid-dependency": "^1.0.1", + "@smithy/middleware-content-length": "^1.0.1", + "@smithy/middleware-endpoint": "^1.0.2", + "@smithy/middleware-retry": "^1.0.3", + "@smithy/middleware-serde": "^1.0.1", + "@smithy/middleware-stack": "^1.0.1", + "@smithy/node-config-provider": "^1.0.1", + "@smithy/node-http-handler": "^1.0.2", + "@smithy/protocol-http": "^1.1.0", + "@smithy/smithy-client": "^1.0.3", + "@smithy/types": "^1.1.0", + "@smithy/url-parser": "^1.0.1", + "@smithy/util-base64": "^1.0.1", + "@smithy/util-body-length-browser": "^1.0.1", + "@smithy/util-body-length-node": "^1.0.1", + "@smithy/util-defaults-mode-browser": "^1.0.1", + "@smithy/util-defaults-mode-node": "^1.0.1", + "@smithy/util-retry": "^1.0.3", + "@smithy/util-utf8": "^1.0.1", + "fast-xml-parser": "4.2.5", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/client-textract/node_modules/@aws-sdk/credential-provider-env": { + "version": "3.370.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.370.0.tgz", + "integrity": "sha512-raR3yP/4GGbKFRPP5hUBNkEmTnzxI9mEc2vJAJrcv4G4J4i/UP6ELiLInQ5eO2/VcV/CeKGZA3t7d1tsJ+jhCg==", + "dependencies": { + "@aws-sdk/types": "3.370.0", + "@smithy/property-provider": "^1.0.1", + "@smithy/types": "^1.1.0", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/client-textract/node_modules/@aws-sdk/credential-provider-ini": { + "version": "3.370.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.370.0.tgz", + "integrity": "sha512-eJyapFKa4NrC9RfTgxlXnXfS9InG/QMEUPPVL+VhG7YS6nKqetC1digOYgivnEeu+XSKE0DJ7uZuXujN2Y7VAQ==", + "dependencies": { + "@aws-sdk/credential-provider-env": "3.370.0", + "@aws-sdk/credential-provider-process": "3.370.0", + "@aws-sdk/credential-provider-sso": "3.370.0", + "@aws-sdk/credential-provider-web-identity": "3.370.0", + "@aws-sdk/types": "3.370.0", + "@smithy/credential-provider-imds": "^1.0.1", + "@smithy/property-provider": "^1.0.1", + "@smithy/shared-ini-file-loader": "^1.0.1", + "@smithy/types": "^1.1.0", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/client-textract/node_modules/@aws-sdk/credential-provider-node": { + "version": "3.370.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.370.0.tgz", + "integrity": "sha512-gkFiotBFKE4Fcn8CzQnMeab9TAR06FEAD02T4ZRYW1xGrBJOowmje9dKqdwQFHSPgnWAP+8HoTA8iwbhTLvjNA==", + "dependencies": { + "@aws-sdk/credential-provider-env": "3.370.0", + "@aws-sdk/credential-provider-ini": "3.370.0", + "@aws-sdk/credential-provider-process": "3.370.0", + "@aws-sdk/credential-provider-sso": "3.370.0", + "@aws-sdk/credential-provider-web-identity": "3.370.0", + "@aws-sdk/types": "3.370.0", + "@smithy/credential-provider-imds": "^1.0.1", + "@smithy/property-provider": "^1.0.1", + "@smithy/shared-ini-file-loader": "^1.0.1", + "@smithy/types": "^1.1.0", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/client-textract/node_modules/@aws-sdk/credential-provider-process": { + "version": "3.370.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.370.0.tgz", + "integrity": "sha512-0BKFFZmUO779Xdw3u7wWnoWhYA4zygxJbgGVSyjkOGBvdkbPSTTcdwT1KFkaQy2kOXYeZPl+usVVRXs+ph4ejg==", + "dependencies": { + "@aws-sdk/types": "3.370.0", + "@smithy/property-provider": "^1.0.1", + "@smithy/shared-ini-file-loader": "^1.0.1", + "@smithy/types": "^1.1.0", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/client-textract/node_modules/@aws-sdk/credential-provider-sso": { + "version": "3.370.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.370.0.tgz", + "integrity": "sha512-PFroYm5hcPSfC/jkZnCI34QFL3I7WVKveVk6/F3fud/cnP8hp6YjA9NiTNbqdFSzsyoiN/+e5fZgNKih8vVPTA==", + "dependencies": { + "@aws-sdk/client-sso": "3.370.0", + "@aws-sdk/token-providers": "3.370.0", + "@aws-sdk/types": "3.370.0", + "@smithy/property-provider": "^1.0.1", + "@smithy/shared-ini-file-loader": "^1.0.1", + "@smithy/types": "^1.1.0", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/client-textract/node_modules/@aws-sdk/credential-provider-web-identity": { + "version": "3.370.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.370.0.tgz", + "integrity": "sha512-CFaBMLRudwhjv1sDzybNV93IaT85IwS+L8Wq6VRMa0mro1q9rrWsIZO811eF+k0NEPfgU1dLH+8Vc2qhw4SARQ==", + "dependencies": { + "@aws-sdk/types": "3.370.0", + "@smithy/property-provider": "^1.0.1", + "@smithy/types": "^1.1.0", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/client-textract/node_modules/@aws-sdk/middleware-host-header": { + "version": "3.370.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-host-header/-/middleware-host-header-3.370.0.tgz", + "integrity": "sha512-CPXOm/TnOFC7KyXcJglICC7OiA7Kj6mT3ChvEijr56TFOueNHvJdV4aNIFEQy0vGHOWtY12qOWLNto/wYR1BAQ==", + "dependencies": { + "@aws-sdk/types": "3.370.0", + "@smithy/protocol-http": "^1.1.0", + "@smithy/types": "^1.1.0", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/client-textract/node_modules/@aws-sdk/middleware-logger": { + "version": "3.370.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-logger/-/middleware-logger-3.370.0.tgz", + "integrity": "sha512-cQMq9SaZ/ORmTJPCT6VzMML7OxFdQzNkhMAgKpTDl+tdPWynlHF29E5xGoSzROnThHlQPCjogU0NZ8AxI0SWPA==", + "dependencies": { + "@aws-sdk/types": "3.370.0", + "@smithy/types": "^1.1.0", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/client-textract/node_modules/@aws-sdk/middleware-recursion-detection": { + "version": "3.370.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-recursion-detection/-/middleware-recursion-detection-3.370.0.tgz", + "integrity": "sha512-L7ZF/w0lAAY/GK1khT8VdoU0XB7nWHk51rl/ecAg64J70dHnMOAg8n+5FZ9fBu/xH1FwUlHOkwlodJOgzLJjtg==", + "dependencies": { + "@aws-sdk/types": "3.370.0", + "@smithy/protocol-http": "^1.1.0", + "@smithy/types": "^1.1.0", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/client-textract/node_modules/@aws-sdk/middleware-sdk-sts": { + "version": "3.370.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-sdk-sts/-/middleware-sdk-sts-3.370.0.tgz", + "integrity": "sha512-ykbsoVy0AJtVbuhAlTAMcaz/tCE3pT8nAp0L7CQQxSoanRCvOux7au0KwMIQVhxgnYid4dWVF6d00SkqU5MXRA==", + "dependencies": { + "@aws-sdk/middleware-signing": "3.370.0", + "@aws-sdk/types": "3.370.0", + "@smithy/types": "^1.1.0", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/client-textract/node_modules/@aws-sdk/middleware-signing": { + "version": "3.370.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-signing/-/middleware-signing-3.370.0.tgz", + "integrity": "sha512-Dwr/RTCWOXdm394wCwICGT2VNOTMRe4IGPsBRJAsM24pm+EEqQzSS3Xu/U/zF4exuxqpMta4wec4QpSarPNTxA==", + "dependencies": { + "@aws-sdk/types": "3.370.0", + "@smithy/property-provider": "^1.0.1", + "@smithy/protocol-http": "^1.1.0", + "@smithy/signature-v4": "^1.0.1", + "@smithy/types": "^1.1.0", + "@smithy/util-middleware": "^1.0.1", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/client-textract/node_modules/@aws-sdk/middleware-user-agent": { + "version": "3.370.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-user-agent/-/middleware-user-agent-3.370.0.tgz", + "integrity": "sha512-2+3SB6MtMAq1+gVXhw0Y3ONXuljorh6ijnxgTpv+uQnBW5jHCUiAS8WDYiDEm7i9euJPbvJfM8WUrSMDMU6Cog==", + "dependencies": { + "@aws-sdk/types": "3.370.0", + "@aws-sdk/util-endpoints": "3.370.0", + "@smithy/protocol-http": "^1.1.0", + "@smithy/types": "^1.1.0", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/client-textract/node_modules/@aws-sdk/token-providers": { + "version": "3.370.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.370.0.tgz", + "integrity": "sha512-EyR2ZYr+lJeRiZU2/eLR+mlYU9RXLQvNyGFSAekJKgN13Rpq/h0syzXVFLP/RSod/oZenh/fhVZ2HwlZxuGBtQ==", + "dependencies": { + "@aws-sdk/client-sso-oidc": "3.370.0", + "@aws-sdk/types": "3.370.0", + "@smithy/property-provider": "^1.0.1", + "@smithy/shared-ini-file-loader": "^1.0.1", + "@smithy/types": "^1.1.0", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/client-textract/node_modules/@aws-sdk/types": { + "version": "3.370.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.370.0.tgz", + "integrity": "sha512-8PGMKklSkRKjunFhzM2y5Jm0H2TBu7YRNISdYzXLUHKSP9zlMEYagseKVdmox0zKHf1LXVNuSlUV2b6SRrieCQ==", + "dependencies": { + "@smithy/types": "^1.1.0", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/client-textract/node_modules/@aws-sdk/util-endpoints": { + "version": "3.370.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-endpoints/-/util-endpoints-3.370.0.tgz", + "integrity": "sha512-5ltVAnM79nRlywwzZN5i8Jp4tk245OCGkKwwXbnDU+gq7zT3CIOsct1wNZvmpfZEPGt/bv7/NyRcjP+7XNsX/g==", + "dependencies": { + "@aws-sdk/types": "3.370.0", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/client-textract/node_modules/@aws-sdk/util-user-agent-browser": { + "version": "3.370.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-browser/-/util-user-agent-browser-3.370.0.tgz", + "integrity": "sha512-028LxYZMQ0DANKhW+AKFQslkScZUeYlPmSphrCIXgdIItRZh6ZJHGzE7J/jDsEntZOrZJsjI4z0zZ5W2idj04w==", + "dependencies": { + "@aws-sdk/types": "3.370.0", + "@smithy/types": "^1.1.0", + "bowser": "^2.11.0", + "tslib": "^2.5.0" + } + }, + "node_modules/@aws-sdk/client-textract/node_modules/@aws-sdk/util-user-agent-node": { + "version": "3.370.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-node/-/util-user-agent-node-3.370.0.tgz", + "integrity": "sha512-33vxZUp8vxTT/DGYIR3PivQm07sSRGWI+4fCv63Rt7Q++fO24E0kQtmVAlikRY810I10poD6rwILVtITtFSzkg==", + "dependencies": { + "@aws-sdk/types": "3.370.0", + "@smithy/node-config-provider": "^1.0.1", + "@smithy/types": "^1.1.0", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + }, + "peerDependencies": { + "aws-crt": ">=1.0.0" + }, + "peerDependenciesMeta": { + "aws-crt": { + "optional": true + } + } + }, + "node_modules/@aws-sdk/client-textract/node_modules/fast-xml-parser": { + "version": "4.2.5", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.2.5.tgz", + "integrity": "sha512-B9/wizE4WngqQftFPmdaMYlXoJlJOYxGQOanC77fq9k8+Z0v5dDSVh+3glErdIROP//s/jgb7ZuxKfB8nVyo0g==", + "funding": [ + { + "type": "paypal", + "url": "https://paypal.me/naturalintelligence" + }, + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "dependencies": { + "strnum": "^1.0.5" + }, + "bin": { + "fxparser": "src/cli/cli.js" + } + }, "node_modules/@aws-sdk/config-resolver": { "version": "3.347.0", "resolved": "https://registry.npmjs.org/@aws-sdk/config-resolver/-/config-resolver-3.347.0.tgz", @@ -3087,12 +3549,296 @@ "@sinonjs/commons": "^3.0.0" } }, + "node_modules/@smithy/abort-controller": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@smithy/abort-controller/-/abort-controller-1.0.2.tgz", + "integrity": "sha512-tb2h0b+JvMee+eAxTmhnyqyNk51UXIK949HnE14lFeezKsVJTB30maan+CO2IMwnig2wVYQH84B5qk6ylmKCuA==", + "dependencies": { + "@smithy/types": "^1.1.1", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@smithy/config-resolver": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@smithy/config-resolver/-/config-resolver-1.0.2.tgz", + "integrity": "sha512-8Bk7CgnVKg1dn5TgnjwPz2ebhxeR7CjGs5yhVYH3S8x0q8yPZZVWwpRIglwXaf5AZBzJlNO1lh+lUhMf2e73zQ==", + "dependencies": { + "@smithy/types": "^1.1.1", + "@smithy/util-config-provider": "^1.0.2", + "@smithy/util-middleware": "^1.0.2", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@smithy/credential-provider-imds": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@smithy/credential-provider-imds/-/credential-provider-imds-1.0.2.tgz", + "integrity": "sha512-fLjCya+JOu2gPJpCiwSUyoLvT8JdNJmOaTOkKYBZoGf7CzqR6lluSyI+eboZnl/V0xqcfcqBG4tgqCISmWS3/w==", + "dependencies": { + "@smithy/node-config-provider": "^1.0.2", + "@smithy/property-provider": "^1.0.2", + "@smithy/types": "^1.1.1", + "@smithy/url-parser": "^1.0.2", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@smithy/eventstream-codec": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@smithy/eventstream-codec/-/eventstream-codec-1.0.2.tgz", + "integrity": "sha512-eW/XPiLauR1VAgHKxhVvgvHzLROUgTtqat2lgljztbH8uIYWugv7Nz+SgCavB+hWRazv2iYgqrSy74GvxXq/rg==", + "dependencies": { + "@aws-crypto/crc32": "3.0.0", + "@smithy/types": "^1.1.1", + "@smithy/util-hex-encoding": "^1.0.2", + "tslib": "^2.5.0" + } + }, + "node_modules/@smithy/fetch-http-handler": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-1.0.2.tgz", + "integrity": "sha512-kynyofLf62LvR8yYphPPdyHb8fWG3LepFinM/vWUTG2Q1pVpmPCM530ppagp3+q2p+7Ox0UvSqldbKqV/d1BpA==", + "dependencies": { + "@smithy/protocol-http": "^1.1.1", + "@smithy/querystring-builder": "^1.0.2", + "@smithy/types": "^1.1.1", + "@smithy/util-base64": "^1.0.2", + "tslib": "^2.5.0" + } + }, + "node_modules/@smithy/hash-node": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@smithy/hash-node/-/hash-node-1.0.2.tgz", + "integrity": "sha512-K6PKhcUNrJXtcesyzhIvNlU7drfIU7u+EMQuGmPw6RQDAg/ufUcfKHz4EcUhFAodUmN+rrejhRG9U6wxjeBOQA==", + "dependencies": { + "@smithy/types": "^1.1.1", + "@smithy/util-buffer-from": "^1.0.2", + "@smithy/util-utf8": "^1.0.2", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@smithy/invalid-dependency": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@smithy/invalid-dependency/-/invalid-dependency-1.0.2.tgz", + "integrity": "sha512-B1Y3Tsa6dfC+Vvb+BJMhTHOfFieeYzY9jWQSTR1vMwKkxsymD0OIAnEw8rD/RiDj/4E4RPGFdx9Mdgnyd6Bv5Q==", + "dependencies": { + "@smithy/types": "^1.1.1", + "tslib": "^2.5.0" + } + }, + "node_modules/@smithy/is-array-buffer": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-1.0.2.tgz", + "integrity": "sha512-pkyBnsBRpe+c/6ASavqIMRBdRtZNJEVJOEzhpxZ9JoAXiZYbkfaSMRA/O1dUxGdJ653GHONunnZ4xMo/LJ7utQ==", + "dependencies": { + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@smithy/middleware-content-length": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@smithy/middleware-content-length/-/middleware-content-length-1.0.2.tgz", + "integrity": "sha512-pa1/SgGIrSmnEr2c9Apw7CdU4l/HW0fK3+LKFCPDYJrzM0JdYpqjQzgxi31P00eAkL0EFBccpus/p1n2GF9urw==", + "dependencies": { + "@smithy/protocol-http": "^1.1.1", + "@smithy/types": "^1.1.1", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@smithy/middleware-endpoint": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@smithy/middleware-endpoint/-/middleware-endpoint-1.0.3.tgz", + "integrity": "sha512-GsWvTXMFjSgl617PCE2km//kIjjtvMRrR2GAuRDIS9sHiLwmkS46VWaVYy+XE7ubEsEtzZ5yK2e8TKDR6Qr5Lw==", + "dependencies": { + "@smithy/middleware-serde": "^1.0.2", + "@smithy/types": "^1.1.1", + "@smithy/url-parser": "^1.0.2", + "@smithy/util-middleware": "^1.0.2", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@smithy/middleware-retry": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@smithy/middleware-retry/-/middleware-retry-1.0.4.tgz", + "integrity": "sha512-G7uRXGFL8c3F7APnoIMTtNAHH8vT4F2qVnAWGAZaervjupaUQuRRHYBLYubK0dWzOZz86BtAXKieJ5p+Ni2Xpg==", + "dependencies": { + "@smithy/protocol-http": "^1.1.1", + "@smithy/service-error-classification": "^1.0.3", + "@smithy/types": "^1.1.1", + "@smithy/util-middleware": "^1.0.2", + "@smithy/util-retry": "^1.0.4", + "tslib": "^2.5.0", + "uuid": "^8.3.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@smithy/middleware-serde": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@smithy/middleware-serde/-/middleware-serde-1.0.2.tgz", + "integrity": "sha512-T4PcdMZF4xme6koUNfjmSZ1MLi7eoFeYCtodQNQpBNsS77TuJt1A6kt5kP/qxrTvfZHyFlj0AubACoaUqgzPeg==", + "dependencies": { + "@smithy/types": "^1.1.1", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@smithy/middleware-stack": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@smithy/middleware-stack/-/middleware-stack-1.0.2.tgz", + "integrity": "sha512-H7/uAQEcmO+eDqweEFMJ5YrIpsBwmrXSP6HIIbtxKJSQpAcMGY7KrR2FZgZBi1FMnSUOh+rQrbOyj5HQmSeUBA==", + "dependencies": { + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@smithy/node-config-provider": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@smithy/node-config-provider/-/node-config-provider-1.0.2.tgz", + "integrity": "sha512-HU7afWpTToU0wL6KseGDR2zojeyjECQfr8LpjAIeHCYIW7r360ABFf4EaplaJRMVoC3hD9FeltgI3/NtShOqCg==", + "dependencies": { + "@smithy/property-provider": "^1.0.2", + "@smithy/shared-ini-file-loader": "^1.0.2", + "@smithy/types": "^1.1.1", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@smithy/node-http-handler": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-1.0.3.tgz", + "integrity": "sha512-PcPUSzTbIb60VCJCiH0PU0E6bwIekttsIEf5Aoo/M0oTfiqsxHTn0Rcij6QoH6qJy6piGKXzLSegspXg5+Kq6g==", + "dependencies": { + "@smithy/abort-controller": "^1.0.2", + "@smithy/protocol-http": "^1.1.1", + "@smithy/querystring-builder": "^1.0.2", + "@smithy/types": "^1.1.1", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@smithy/property-provider": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@smithy/property-provider/-/property-provider-1.0.2.tgz", + "integrity": "sha512-pXDPyzKX8opzt38B205kDgaxda6LHcTfPvTYQZnwP6BAPp1o9puiCPjeUtkKck7Z6IbpXCPUmUQnzkUzWTA42Q==", + "dependencies": { + "@smithy/types": "^1.1.1", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, "node_modules/@smithy/protocol-http": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-1.0.1.tgz", - "integrity": "sha512-9OrEn0WfOVtBNYJUjUAn9AOiJ4lzERCJJ/JeZs8E6yajTGxBaFRxUnNBHiNqoDJVg076hY36UmEnPx7xXrvUSg==", + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-1.1.1.tgz", + "integrity": "sha512-mFLFa2sSvlUxm55U7B4YCIsJJIMkA6lHxwwqOaBkral1qxFz97rGffP/mmd4JDuin1EnygiO5eNJGgudiUgmDQ==", "dependencies": { - "@smithy/types": "^1.0.0", + "@smithy/types": "^1.1.1", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@smithy/querystring-builder": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@smithy/querystring-builder/-/querystring-builder-1.0.2.tgz", + "integrity": "sha512-6P/xANWrtJhMzTPUR87AbXwSBuz1SDHIfL44TFd/GT3hj6rA+IEv7rftEpPjayUiWRocaNnrCPLvmP31mobOyA==", + "dependencies": { + "@smithy/types": "^1.1.1", + "@smithy/util-uri-escape": "^1.0.2", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@smithy/querystring-parser": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@smithy/querystring-parser/-/querystring-parser-1.0.2.tgz", + "integrity": "sha512-IWxwxjn+KHWRRRB+K2Ngl+plTwo2WSgc2w+DvLy0DQZJh9UGOpw40d6q97/63GBlXIt4TEt5NbcFrO30CKlrsA==", + "dependencies": { + "@smithy/types": "^1.1.1", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@smithy/service-error-classification": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@smithy/service-error-classification/-/service-error-classification-1.0.3.tgz", + "integrity": "sha512-2eglIYqrtcUnuI71yweu7rSfCgt6kVvRVf0C72VUqrd0LrV1M0BM0eYN+nitp2CHPSdmMI96pi+dU9U/UqAMSA==", + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@smithy/shared-ini-file-loader": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@smithy/shared-ini-file-loader/-/shared-ini-file-loader-1.0.2.tgz", + "integrity": "sha512-bdQj95VN+lCXki+P3EsDyrkpeLn8xDYiOISBGnUG/AGPYJXN8dmp4EhRRR7XOoLoSs8anZHR4UcGEOzFv2jwGw==", + "dependencies": { + "@smithy/types": "^1.1.1", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@smithy/signature-v4": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-1.0.2.tgz", + "integrity": "sha512-rpKUhmCuPmpV5dloUkOb9w1oBnJatvKQEjIHGmkjRGZnC3437MTdzWej9TxkagcZ8NRRJavYnEUixzxM1amFig==", + "dependencies": { + "@smithy/eventstream-codec": "^1.0.2", + "@smithy/is-array-buffer": "^1.0.2", + "@smithy/types": "^1.1.1", + "@smithy/util-hex-encoding": "^1.0.2", + "@smithy/util-middleware": "^1.0.2", + "@smithy/util-uri-escape": "^1.0.2", + "@smithy/util-utf8": "^1.0.2", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@smithy/smithy-client": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@smithy/smithy-client/-/smithy-client-1.0.4.tgz", + "integrity": "sha512-gpo0Xl5Nyp9sgymEfpt7oa9P2q/GlM3VmQIdm+FeH0QEdYOQx3OtvwVmBYAMv2FIPWxkMZlsPYRTnEiBTK5TYg==", + "dependencies": { + "@smithy/middleware-stack": "^1.0.2", + "@smithy/types": "^1.1.1", + "@smithy/util-stream": "^1.0.2", "tslib": "^2.5.0" }, "engines": { @@ -3100,10 +3846,179 @@ } }, "node_modules/@smithy/types": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-1.0.0.tgz", - "integrity": "sha512-kc1m5wPBHQCTixwuaOh9vnak/iJm21DrSf9UK6yDE5S3mQQ4u11pqAUiKWnlrZnYkeLfAI9UEHj9OaMT1v5Umg==", + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@smithy/types/-/types-1.1.1.tgz", + "integrity": "sha512-tMpkreknl2gRrniHeBtdgQwaOlo39df8RxSrwsHVNIGXULy5XP6KqgScUw2m12D15wnJCKWxVhCX+wbrBW/y7g==", + "dependencies": { + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@smithy/url-parser": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@smithy/url-parser/-/url-parser-1.0.2.tgz", + "integrity": "sha512-0JRsDMQe53F6EHRWksdcavKDRjyqp8vrjakg8EcCUOa7PaFRRB1SO/xGZdzSlW1RSTWQDEksFMTCEcVEKmAoqA==", + "dependencies": { + "@smithy/querystring-parser": "^1.0.2", + "@smithy/types": "^1.1.1", + "tslib": "^2.5.0" + } + }, + "node_modules/@smithy/util-base64": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@smithy/util-base64/-/util-base64-1.0.2.tgz", + "integrity": "sha512-BCm15WILJ3SL93nusoxvJGMVfAMWHZhdeDZPtpAaskozuexd0eF6szdz4kbXaKp38bFCSenA6bkUHqaE3KK0dA==", + "dependencies": { + "@smithy/util-buffer-from": "^1.0.2", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@smithy/util-body-length-browser": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@smithy/util-body-length-browser/-/util-body-length-browser-1.0.2.tgz", + "integrity": "sha512-Xh8L06H2anF5BHjSYTg8hx+Itcbf4SQZnVMl4PIkCOsKtneMJoGjPRLy17lEzfoh/GOaa0QxgCP6lRMQWzNl4w==", + "dependencies": { + "tslib": "^2.5.0" + } + }, + "node_modules/@smithy/util-body-length-node": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@smithy/util-body-length-node/-/util-body-length-node-1.0.2.tgz", + "integrity": "sha512-nXHbZsUtvZeyfL4Ceds9nmy2Uh2AhWXohG4vWHyjSdmT8cXZlJdmJgnH6SJKDjyUecbu+BpKeVvSrA4cWPSOPA==", + "dependencies": { + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@smithy/util-buffer-from": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-1.0.2.tgz", + "integrity": "sha512-lHAYIyrBO9RANrPvccnPjU03MJnWZ66wWuC5GjWWQVfsmPwU6m00aakZkzHdUT6tGCkGacXSgArP5wgTgA+oCw==", + "dependencies": { + "@smithy/is-array-buffer": "^1.0.2", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@smithy/util-config-provider": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@smithy/util-config-provider/-/util-config-provider-1.0.2.tgz", + "integrity": "sha512-HOdmDm+3HUbuYPBABLLHtn8ittuRyy+BSjKOA169H+EMc+IozipvXDydf+gKBRAxUa4dtKQkLraypwppzi+PRw==", + "dependencies": { + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@smithy/util-defaults-mode-browser": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-browser/-/util-defaults-mode-browser-1.0.2.tgz", + "integrity": "sha512-J1u2PO235zxY7dg0+ZqaG96tFg4ehJZ7isGK1pCBEA072qxNPwIpDzUVGnLJkHZvjWEGA8rxIauDtXfB0qxeAg==", + "dependencies": { + "@smithy/property-provider": "^1.0.2", + "@smithy/types": "^1.1.1", + "bowser": "^2.11.0", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">= 10.0.0" + } + }, + "node_modules/@smithy/util-defaults-mode-node": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-node/-/util-defaults-mode-node-1.0.2.tgz", + "integrity": "sha512-9/BN63rlIsFStvI+AvljMh873Xw6bbI6b19b+PVYXyycQ2DDQImWcjnzRlHW7eP65CCUNGQ6otDLNdBQCgMXqg==", + "dependencies": { + "@smithy/config-resolver": "^1.0.2", + "@smithy/credential-provider-imds": "^1.0.2", + "@smithy/node-config-provider": "^1.0.2", + "@smithy/property-provider": "^1.0.2", + "@smithy/types": "^1.1.1", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">= 10.0.0" + } + }, + "node_modules/@smithy/util-hex-encoding": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@smithy/util-hex-encoding/-/util-hex-encoding-1.0.2.tgz", + "integrity": "sha512-Bxydb5rMJorMV6AuDDMOxro3BMDdIwtbQKHpwvQFASkmr52BnpDsWlxgpJi8Iq7nk1Bt4E40oE1Isy/7ubHGzg==", + "dependencies": { + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@smithy/util-middleware": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@smithy/util-middleware/-/util-middleware-1.0.2.tgz", + "integrity": "sha512-vtXK7GOR2BoseCX8NCGe9SaiZrm9M2lm/RVexFGyPuafTtry9Vyv7hq/vw8ifd/G/pSJ+msByfJVb1642oQHKw==", + "dependencies": { + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@smithy/util-retry": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@smithy/util-retry/-/util-retry-1.0.4.tgz", + "integrity": "sha512-RnZPVFvRoqdj2EbroDo3OsnnQU8eQ4AlnZTOGusbYKybH3269CFdrZfZJloe60AQjX7di3J6t/79PjwCLO5Khw==", + "dependencies": { + "@smithy/service-error-classification": "^1.0.3", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">= 14.0.0" + } + }, + "node_modules/@smithy/util-stream": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@smithy/util-stream/-/util-stream-1.0.2.tgz", + "integrity": "sha512-qyN2M9QFMTz4UCHi6GnBfLOGYKxQZD01Ga6nzaXFFC51HP/QmArU72e4kY50Z/EtW8binPxspP2TAsGbwy9l3A==", + "dependencies": { + "@smithy/fetch-http-handler": "^1.0.2", + "@smithy/node-http-handler": "^1.0.3", + "@smithy/types": "^1.1.1", + "@smithy/util-base64": "^1.0.2", + "@smithy/util-buffer-from": "^1.0.2", + "@smithy/util-hex-encoding": "^1.0.2", + "@smithy/util-utf8": "^1.0.2", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@smithy/util-uri-escape": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@smithy/util-uri-escape/-/util-uri-escape-1.0.2.tgz", + "integrity": "sha512-k8C0BFNS9HpBMHSgUDnWb1JlCQcFG+PPlVBq9keP4Nfwv6a9Q0yAfASWqUCtzjuMj1hXeLhn/5ADP6JxnID1Pg==", + "dependencies": { + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@smithy/util-utf8": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-1.0.2.tgz", + "integrity": "sha512-V4cyjKfJlARui0dMBfWJMQAmJzoW77i4N3EjkH/bwnE2Ngbl4tqD2Y0C/xzpzY/J1BdxeCKxAebVFk8aFCaSCw==", "dependencies": { + "@smithy/util-buffer-from": "^1.0.2", "tslib": "^2.5.0" }, "engines": { diff --git a/backend/package.json b/backend/package.json index d2630855..d352705c 100644 --- a/backend/package.json +++ b/backend/package.json @@ -22,6 +22,7 @@ }, "dependencies": { "@aws-sdk/client-s3": "^3.350.0", + "@aws-sdk/client-textract": "^3.370.0", "@nestjs/common": "^9.0.0", "@nestjs/core": "^9.0.0", "@nestjs/jwt": "^10.0.3", diff --git a/backend/src/app.module.ts b/backend/src/app.module.ts index 249e3fa9..241f4081 100644 --- a/backend/src/app.module.ts +++ b/backend/src/app.module.ts @@ -27,6 +27,8 @@ import { Folder } from './folders/entities/folder.entity'; import { S3Service } from './s3/s3.service'; import { ConversionService } from './conversion/conversion.service'; import { User } from './users/entities/user.entity'; +import { TextractController } from './textract/textract.controller'; +import { TextractModule } from './textract/textract.module'; @Module({ imports: [ @@ -43,6 +45,7 @@ import { User } from './users/entities/user.entity'; FoldersModule, S3Module, FileManagerModule, + TextractModule, ], controllers: [ AppController, @@ -50,6 +53,7 @@ import { User } from './users/entities/user.entity'; EditController, S3Controller, FileManagerController, + TextractController, ], providers: [ AppService, diff --git a/backend/src/textract/textract.controller.spec.ts b/backend/src/textract/textract.controller.spec.ts new file mode 100644 index 00000000..338bd580 --- /dev/null +++ b/backend/src/textract/textract.controller.spec.ts @@ -0,0 +1,18 @@ +import { Test, TestingModule } from '@nestjs/testing'; +import { TextractController } from './textract.controller'; + +describe('TextractController', () => { + let controller: TextractController; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + controllers: [TextractController], + }).compile(); + + controller = module.get(TextractController); + }); + + it('should be defined', () => { + expect(controller).toBeDefined(); + }); +}); diff --git a/backend/src/textract/textract.controller.ts b/backend/src/textract/textract.controller.ts new file mode 100644 index 00000000..8491112b --- /dev/null +++ b/backend/src/textract/textract.controller.ts @@ -0,0 +1,4 @@ +import { Controller } from '@nestjs/common'; + +@Controller('textract') +export class TextractController {} diff --git a/backend/src/textract/textract.module.ts b/backend/src/textract/textract.module.ts new file mode 100644 index 00000000..872a0482 --- /dev/null +++ b/backend/src/textract/textract.module.ts @@ -0,0 +1,7 @@ +import { Module } from '@nestjs/common'; +import { TextractService } from './textract.service'; + +@Module({ + providers: [TextractService] +}) +export class TextractModule {} diff --git a/backend/src/textract/textract.service.spec.ts b/backend/src/textract/textract.service.spec.ts new file mode 100644 index 00000000..353d69b8 --- /dev/null +++ b/backend/src/textract/textract.service.spec.ts @@ -0,0 +1,18 @@ +import { Test, TestingModule } from '@nestjs/testing'; +import { TextractService } from './textract.service'; + +describe('TextractService', () => { + let service: TextractService; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + providers: [TextractService], + }).compile(); + + service = module.get(TextractService); + }); + + it('should be defined', () => { + expect(service).toBeDefined(); + }); +}); diff --git a/backend/src/textract/textract.service.ts b/backend/src/textract/textract.service.ts new file mode 100644 index 00000000..be808923 --- /dev/null +++ b/backend/src/textract/textract.service.ts @@ -0,0 +1,33 @@ +import { Injectable } from '@nestjs/common'; +import { + TextractClient, + DetectDocumentTextCommand, + AnalyzeDocumentCommand, + FeatureType, + StartDocumentTextDetectionCommand, + StartDocumentAnalysisCommand, + GetDocumentTextDetectionCommand, + JobStatus, + GetDocumentAnalysisCommand, +} from '@aws-sdk/client-textract'; +import 'dotenv/config'; + +@Injectable() +export class TextractService { + awsS3BucketName = + process.env.AWS_S3_BUCKET_NAME; + awsS3BucketRegion = + process.env.AWS_S3_BUCKET_REGION; + awsS3AccessKeyId = + process.env.AWS_S3_ACCESS_KEY_ID; + awsS3SecretAccessKey = + process.env.AWS_S3_SECRET_ACCESS_KEY; + + private readonly s3Client = new TextractClient({ + credentials: { + accessKeyId: this.awsS3AccessKeyId, + secretAccessKey: this.awsS3SecretAccessKey, + }, + region: this.awsS3BucketRegion, + }); +} From 880fd6c6fb072ec38db821efea90a1487bdb5e6e Mon Sep 17 00:00:00 2001 From: Dylan Kapnias Date: Sun, 23 Jul 2023 14:04:25 +0200 Subject: [PATCH 06/14] Added Textract Service Principal to S3 bucket --- infrastructure/lib/s3-stack.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/infrastructure/lib/s3-stack.ts b/infrastructure/lib/s3-stack.ts index 153abbd9..fd69b34c 100644 --- a/infrastructure/lib/s3-stack.ts +++ b/infrastructure/lib/s3-stack.ts @@ -1,5 +1,5 @@ import { App, CfnOutput, RemovalPolicy, Stack, StackProps } from "aws-cdk-lib"; -import { PolicyStatement, User } from "aws-cdk-lib/aws-iam"; +import { PolicyStatement, ServicePrincipal, User } from "aws-cdk-lib/aws-iam"; import { Bucket, BucketAccessControl } from "aws-cdk-lib/aws-s3"; export class S3Stack extends Stack { @@ -14,6 +14,9 @@ export class S3Stack extends Stack { accessControl: BucketAccessControl.BUCKET_OWNER_FULL_CONTROL, }); + let textract = new ServicePrincipal("textract.amazonaws.com"); + bucket.grantReadWrite(textract); + const user = new User(this, "MyIAMUser", { userName: "writetopdf-webapp-test", }); From d0965fea001d4da32424aa0804e54c3744cf46c5 Mon Sep 17 00:00:00 2001 From: Dylan Kapnias Date: Sun, 23 Jul 2023 14:45:48 +0200 Subject: [PATCH 07/14] Added sns and sqs to CDK --- infrastructure/lib/s3-stack.ts | 33 ++++++++++++++++++++++++++++++--- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/infrastructure/lib/s3-stack.ts b/infrastructure/lib/s3-stack.ts index fd69b34c..9c74061f 100644 --- a/infrastructure/lib/s3-stack.ts +++ b/infrastructure/lib/s3-stack.ts @@ -1,11 +1,22 @@ import { App, CfnOutput, RemovalPolicy, Stack, StackProps } from "aws-cdk-lib"; -import { PolicyStatement, ServicePrincipal, User } from "aws-cdk-lib/aws-iam"; +import { + PolicyStatement, + Role, + ServicePrincipal, + User, +} from "aws-cdk-lib/aws-iam"; import { Bucket, BucketAccessControl } from "aws-cdk-lib/aws-s3"; +import { Topic } from "aws-cdk-lib/aws-sns"; +import { Alias } from "aws-cdk-lib/aws-kms"; +import { SqsSubscription } from "aws-cdk-lib/aws-sns-subscriptions"; +import { Queue } from "aws-cdk-lib/aws-sqs"; export class S3Stack extends Stack { constructor(scope: App, id: string, props?: StackProps) { super(scope, id, props); + let textract = new ServicePrincipal("textract.amazonaws.com"); + // Create S3 bucket to hold user assets const bucket = new Bucket(this, "WriteToPDFAppBucket", { publicReadAccess: false, @@ -13,10 +24,20 @@ export class S3Stack extends Stack { // blockPublicAccess: s3.BlockPublicAccess.BLOCK_ACLS, accessControl: BucketAccessControl.BUCKET_OWNER_FULL_CONTROL, }); - - let textract = new ServicePrincipal("textract.amazonaws.com"); bucket.grantReadWrite(textract); + let queue = new Queue(this, "writetopdf-textract-queue", {}); + + let topic = new Topic(this, "writetopdf-textract-topic", { + masterKey: Alias.fromAliasName(this, "defaultKey", "alias/aws/sns"), + }); + topic.addSubscription(new SqsSubscription(queue)); + + let role = new Role(this, "writetopdf-textract-role", { + assumedBy: textract, + }); + topic.grantPublish(role); + const user = new User(this, "MyIAMUser", { userName: "writetopdf-webapp-test", }); @@ -35,5 +56,11 @@ export class S3Stack extends Stack { new CfnOutput(this, "S3BucketRegion", { value: bucket.bucketRegionalDomainName, }); + + new CfnOutput(this, "TopicArn", { value: topic.topicArn }); + + new CfnOutput(this, "RoleArn", { value: role.roleArn }); + + new CfnOutput(this, "QueueUrl", { value: queue.queueUrl }); } } From 9887aab3ba637c6b03af0bd67cabc09e54223484 Mon Sep 17 00:00:00 2001 From: Dylan Kapnias Date: Sun, 23 Jul 2023 15:39:49 +0200 Subject: [PATCH 08/14] Super hopeful abomination of code --- backend/package-lock.json | 502 +++++++++++++++++++++++ backend/package.json | 1 + backend/src/textract/textract.service.ts | 124 +++++- 3 files changed, 624 insertions(+), 3 deletions(-) diff --git a/backend/package-lock.json b/backend/package-lock.json index ddac7b74..681a7338 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -10,6 +10,7 @@ "license": "UNLICENSED", "dependencies": { "@aws-sdk/client-s3": "^3.350.0", + "@aws-sdk/client-sqs": "^3.370.0", "@aws-sdk/client-textract": "^3.370.0", "@nestjs/common": "^9.0.0", "@nestjs/core": "^9.0.0", @@ -404,6 +405,470 @@ "node": ">=14.0.0" } }, + "node_modules/@aws-sdk/client-sqs": { + "version": "3.370.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-sqs/-/client-sqs-3.370.0.tgz", + "integrity": "sha512-HaF+Jb5dKtvv2uA5WhtPMPM4d6vczHGKV1x/CRFJv2NDwHJX6E84RZ+74008SCjdA3C1RqvErrjw0iSwYyN7CA==", + "dependencies": { + "@aws-crypto/sha256-browser": "3.0.0", + "@aws-crypto/sha256-js": "3.0.0", + "@aws-sdk/client-sts": "3.370.0", + "@aws-sdk/credential-provider-node": "3.370.0", + "@aws-sdk/middleware-host-header": "3.370.0", + "@aws-sdk/middleware-logger": "3.370.0", + "@aws-sdk/middleware-recursion-detection": "3.370.0", + "@aws-sdk/middleware-sdk-sqs": "3.370.0", + "@aws-sdk/middleware-signing": "3.370.0", + "@aws-sdk/middleware-user-agent": "3.370.0", + "@aws-sdk/types": "3.370.0", + "@aws-sdk/util-endpoints": "3.370.0", + "@aws-sdk/util-user-agent-browser": "3.370.0", + "@aws-sdk/util-user-agent-node": "3.370.0", + "@smithy/config-resolver": "^1.0.1", + "@smithy/fetch-http-handler": "^1.0.1", + "@smithy/hash-node": "^1.0.1", + "@smithy/invalid-dependency": "^1.0.1", + "@smithy/md5-js": "^1.0.1", + "@smithy/middleware-content-length": "^1.0.1", + "@smithy/middleware-endpoint": "^1.0.2", + "@smithy/middleware-retry": "^1.0.3", + "@smithy/middleware-serde": "^1.0.1", + "@smithy/middleware-stack": "^1.0.1", + "@smithy/node-config-provider": "^1.0.1", + "@smithy/node-http-handler": "^1.0.2", + "@smithy/protocol-http": "^1.1.0", + "@smithy/smithy-client": "^1.0.3", + "@smithy/types": "^1.1.0", + "@smithy/url-parser": "^1.0.1", + "@smithy/util-base64": "^1.0.1", + "@smithy/util-body-length-browser": "^1.0.1", + "@smithy/util-body-length-node": "^1.0.1", + "@smithy/util-defaults-mode-browser": "^1.0.1", + "@smithy/util-defaults-mode-node": "^1.0.1", + "@smithy/util-retry": "^1.0.3", + "@smithy/util-utf8": "^1.0.1", + "fast-xml-parser": "4.2.5", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/client-sqs/node_modules/@aws-sdk/client-sso": { + "version": "3.370.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso/-/client-sso-3.370.0.tgz", + "integrity": "sha512-0Ty1iHuzNxMQtN7nahgkZr4Wcu1XvqGfrQniiGdKKif9jG/4elxsQPiydRuQpFqN6b+bg7wPP7crFP1uTxx2KQ==", + "dependencies": { + "@aws-crypto/sha256-browser": "3.0.0", + "@aws-crypto/sha256-js": "3.0.0", + "@aws-sdk/middleware-host-header": "3.370.0", + "@aws-sdk/middleware-logger": "3.370.0", + "@aws-sdk/middleware-recursion-detection": "3.370.0", + "@aws-sdk/middleware-user-agent": "3.370.0", + "@aws-sdk/types": "3.370.0", + "@aws-sdk/util-endpoints": "3.370.0", + "@aws-sdk/util-user-agent-browser": "3.370.0", + "@aws-sdk/util-user-agent-node": "3.370.0", + "@smithy/config-resolver": "^1.0.1", + "@smithy/fetch-http-handler": "^1.0.1", + "@smithy/hash-node": "^1.0.1", + "@smithy/invalid-dependency": "^1.0.1", + "@smithy/middleware-content-length": "^1.0.1", + "@smithy/middleware-endpoint": "^1.0.2", + "@smithy/middleware-retry": "^1.0.3", + "@smithy/middleware-serde": "^1.0.1", + "@smithy/middleware-stack": "^1.0.1", + "@smithy/node-config-provider": "^1.0.1", + "@smithy/node-http-handler": "^1.0.2", + "@smithy/protocol-http": "^1.1.0", + "@smithy/smithy-client": "^1.0.3", + "@smithy/types": "^1.1.0", + "@smithy/url-parser": "^1.0.1", + "@smithy/util-base64": "^1.0.1", + "@smithy/util-body-length-browser": "^1.0.1", + "@smithy/util-body-length-node": "^1.0.1", + "@smithy/util-defaults-mode-browser": "^1.0.1", + "@smithy/util-defaults-mode-node": "^1.0.1", + "@smithy/util-retry": "^1.0.3", + "@smithy/util-utf8": "^1.0.1", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/client-sqs/node_modules/@aws-sdk/client-sso-oidc": { + "version": "3.370.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso-oidc/-/client-sso-oidc-3.370.0.tgz", + "integrity": "sha512-jAYOO74lmVXylQylqkPrjLzxvUnMKw476JCUTvCO6Q8nv3LzCWd76Ihgv/m9Q4M2Tbqi1iP2roVK5bstsXzEjA==", + "dependencies": { + "@aws-crypto/sha256-browser": "3.0.0", + "@aws-crypto/sha256-js": "3.0.0", + "@aws-sdk/middleware-host-header": "3.370.0", + "@aws-sdk/middleware-logger": "3.370.0", + "@aws-sdk/middleware-recursion-detection": "3.370.0", + "@aws-sdk/middleware-user-agent": "3.370.0", + "@aws-sdk/types": "3.370.0", + "@aws-sdk/util-endpoints": "3.370.0", + "@aws-sdk/util-user-agent-browser": "3.370.0", + "@aws-sdk/util-user-agent-node": "3.370.0", + "@smithy/config-resolver": "^1.0.1", + "@smithy/fetch-http-handler": "^1.0.1", + "@smithy/hash-node": "^1.0.1", + "@smithy/invalid-dependency": "^1.0.1", + "@smithy/middleware-content-length": "^1.0.1", + "@smithy/middleware-endpoint": "^1.0.2", + "@smithy/middleware-retry": "^1.0.3", + "@smithy/middleware-serde": "^1.0.1", + "@smithy/middleware-stack": "^1.0.1", + "@smithy/node-config-provider": "^1.0.1", + "@smithy/node-http-handler": "^1.0.2", + "@smithy/protocol-http": "^1.1.0", + "@smithy/smithy-client": "^1.0.3", + "@smithy/types": "^1.1.0", + "@smithy/url-parser": "^1.0.1", + "@smithy/util-base64": "^1.0.1", + "@smithy/util-body-length-browser": "^1.0.1", + "@smithy/util-body-length-node": "^1.0.1", + "@smithy/util-defaults-mode-browser": "^1.0.1", + "@smithy/util-defaults-mode-node": "^1.0.1", + "@smithy/util-retry": "^1.0.3", + "@smithy/util-utf8": "^1.0.1", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/client-sqs/node_modules/@aws-sdk/client-sts": { + "version": "3.370.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-sts/-/client-sts-3.370.0.tgz", + "integrity": "sha512-utFxOPWIzbN+3kc415Je2o4J72hOLNhgR2Gt5EnRSggC3yOnkC4GzauxG8n7n5gZGBX45eyubHyPOXLOIyoqQA==", + "dependencies": { + "@aws-crypto/sha256-browser": "3.0.0", + "@aws-crypto/sha256-js": "3.0.0", + "@aws-sdk/credential-provider-node": "3.370.0", + "@aws-sdk/middleware-host-header": "3.370.0", + "@aws-sdk/middleware-logger": "3.370.0", + "@aws-sdk/middleware-recursion-detection": "3.370.0", + "@aws-sdk/middleware-sdk-sts": "3.370.0", + "@aws-sdk/middleware-signing": "3.370.0", + "@aws-sdk/middleware-user-agent": "3.370.0", + "@aws-sdk/types": "3.370.0", + "@aws-sdk/util-endpoints": "3.370.0", + "@aws-sdk/util-user-agent-browser": "3.370.0", + "@aws-sdk/util-user-agent-node": "3.370.0", + "@smithy/config-resolver": "^1.0.1", + "@smithy/fetch-http-handler": "^1.0.1", + "@smithy/hash-node": "^1.0.1", + "@smithy/invalid-dependency": "^1.0.1", + "@smithy/middleware-content-length": "^1.0.1", + "@smithy/middleware-endpoint": "^1.0.2", + "@smithy/middleware-retry": "^1.0.3", + "@smithy/middleware-serde": "^1.0.1", + "@smithy/middleware-stack": "^1.0.1", + "@smithy/node-config-provider": "^1.0.1", + "@smithy/node-http-handler": "^1.0.2", + "@smithy/protocol-http": "^1.1.0", + "@smithy/smithy-client": "^1.0.3", + "@smithy/types": "^1.1.0", + "@smithy/url-parser": "^1.0.1", + "@smithy/util-base64": "^1.0.1", + "@smithy/util-body-length-browser": "^1.0.1", + "@smithy/util-body-length-node": "^1.0.1", + "@smithy/util-defaults-mode-browser": "^1.0.1", + "@smithy/util-defaults-mode-node": "^1.0.1", + "@smithy/util-retry": "^1.0.3", + "@smithy/util-utf8": "^1.0.1", + "fast-xml-parser": "4.2.5", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/client-sqs/node_modules/@aws-sdk/credential-provider-env": { + "version": "3.370.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.370.0.tgz", + "integrity": "sha512-raR3yP/4GGbKFRPP5hUBNkEmTnzxI9mEc2vJAJrcv4G4J4i/UP6ELiLInQ5eO2/VcV/CeKGZA3t7d1tsJ+jhCg==", + "dependencies": { + "@aws-sdk/types": "3.370.0", + "@smithy/property-provider": "^1.0.1", + "@smithy/types": "^1.1.0", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/client-sqs/node_modules/@aws-sdk/credential-provider-ini": { + "version": "3.370.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.370.0.tgz", + "integrity": "sha512-eJyapFKa4NrC9RfTgxlXnXfS9InG/QMEUPPVL+VhG7YS6nKqetC1digOYgivnEeu+XSKE0DJ7uZuXujN2Y7VAQ==", + "dependencies": { + "@aws-sdk/credential-provider-env": "3.370.0", + "@aws-sdk/credential-provider-process": "3.370.0", + "@aws-sdk/credential-provider-sso": "3.370.0", + "@aws-sdk/credential-provider-web-identity": "3.370.0", + "@aws-sdk/types": "3.370.0", + "@smithy/credential-provider-imds": "^1.0.1", + "@smithy/property-provider": "^1.0.1", + "@smithy/shared-ini-file-loader": "^1.0.1", + "@smithy/types": "^1.1.0", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/client-sqs/node_modules/@aws-sdk/credential-provider-node": { + "version": "3.370.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.370.0.tgz", + "integrity": "sha512-gkFiotBFKE4Fcn8CzQnMeab9TAR06FEAD02T4ZRYW1xGrBJOowmje9dKqdwQFHSPgnWAP+8HoTA8iwbhTLvjNA==", + "dependencies": { + "@aws-sdk/credential-provider-env": "3.370.0", + "@aws-sdk/credential-provider-ini": "3.370.0", + "@aws-sdk/credential-provider-process": "3.370.0", + "@aws-sdk/credential-provider-sso": "3.370.0", + "@aws-sdk/credential-provider-web-identity": "3.370.0", + "@aws-sdk/types": "3.370.0", + "@smithy/credential-provider-imds": "^1.0.1", + "@smithy/property-provider": "^1.0.1", + "@smithy/shared-ini-file-loader": "^1.0.1", + "@smithy/types": "^1.1.0", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/client-sqs/node_modules/@aws-sdk/credential-provider-process": { + "version": "3.370.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.370.0.tgz", + "integrity": "sha512-0BKFFZmUO779Xdw3u7wWnoWhYA4zygxJbgGVSyjkOGBvdkbPSTTcdwT1KFkaQy2kOXYeZPl+usVVRXs+ph4ejg==", + "dependencies": { + "@aws-sdk/types": "3.370.0", + "@smithy/property-provider": "^1.0.1", + "@smithy/shared-ini-file-loader": "^1.0.1", + "@smithy/types": "^1.1.0", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/client-sqs/node_modules/@aws-sdk/credential-provider-sso": { + "version": "3.370.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.370.0.tgz", + "integrity": "sha512-PFroYm5hcPSfC/jkZnCI34QFL3I7WVKveVk6/F3fud/cnP8hp6YjA9NiTNbqdFSzsyoiN/+e5fZgNKih8vVPTA==", + "dependencies": { + "@aws-sdk/client-sso": "3.370.0", + "@aws-sdk/token-providers": "3.370.0", + "@aws-sdk/types": "3.370.0", + "@smithy/property-provider": "^1.0.1", + "@smithy/shared-ini-file-loader": "^1.0.1", + "@smithy/types": "^1.1.0", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/client-sqs/node_modules/@aws-sdk/credential-provider-web-identity": { + "version": "3.370.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.370.0.tgz", + "integrity": "sha512-CFaBMLRudwhjv1sDzybNV93IaT85IwS+L8Wq6VRMa0mro1q9rrWsIZO811eF+k0NEPfgU1dLH+8Vc2qhw4SARQ==", + "dependencies": { + "@aws-sdk/types": "3.370.0", + "@smithy/property-provider": "^1.0.1", + "@smithy/types": "^1.1.0", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/client-sqs/node_modules/@aws-sdk/middleware-host-header": { + "version": "3.370.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-host-header/-/middleware-host-header-3.370.0.tgz", + "integrity": "sha512-CPXOm/TnOFC7KyXcJglICC7OiA7Kj6mT3ChvEijr56TFOueNHvJdV4aNIFEQy0vGHOWtY12qOWLNto/wYR1BAQ==", + "dependencies": { + "@aws-sdk/types": "3.370.0", + "@smithy/protocol-http": "^1.1.0", + "@smithy/types": "^1.1.0", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/client-sqs/node_modules/@aws-sdk/middleware-logger": { + "version": "3.370.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-logger/-/middleware-logger-3.370.0.tgz", + "integrity": "sha512-cQMq9SaZ/ORmTJPCT6VzMML7OxFdQzNkhMAgKpTDl+tdPWynlHF29E5xGoSzROnThHlQPCjogU0NZ8AxI0SWPA==", + "dependencies": { + "@aws-sdk/types": "3.370.0", + "@smithy/types": "^1.1.0", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/client-sqs/node_modules/@aws-sdk/middleware-recursion-detection": { + "version": "3.370.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-recursion-detection/-/middleware-recursion-detection-3.370.0.tgz", + "integrity": "sha512-L7ZF/w0lAAY/GK1khT8VdoU0XB7nWHk51rl/ecAg64J70dHnMOAg8n+5FZ9fBu/xH1FwUlHOkwlodJOgzLJjtg==", + "dependencies": { + "@aws-sdk/types": "3.370.0", + "@smithy/protocol-http": "^1.1.0", + "@smithy/types": "^1.1.0", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/client-sqs/node_modules/@aws-sdk/middleware-sdk-sts": { + "version": "3.370.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-sdk-sts/-/middleware-sdk-sts-3.370.0.tgz", + "integrity": "sha512-ykbsoVy0AJtVbuhAlTAMcaz/tCE3pT8nAp0L7CQQxSoanRCvOux7au0KwMIQVhxgnYid4dWVF6d00SkqU5MXRA==", + "dependencies": { + "@aws-sdk/middleware-signing": "3.370.0", + "@aws-sdk/types": "3.370.0", + "@smithy/types": "^1.1.0", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/client-sqs/node_modules/@aws-sdk/middleware-signing": { + "version": "3.370.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-signing/-/middleware-signing-3.370.0.tgz", + "integrity": "sha512-Dwr/RTCWOXdm394wCwICGT2VNOTMRe4IGPsBRJAsM24pm+EEqQzSS3Xu/U/zF4exuxqpMta4wec4QpSarPNTxA==", + "dependencies": { + "@aws-sdk/types": "3.370.0", + "@smithy/property-provider": "^1.0.1", + "@smithy/protocol-http": "^1.1.0", + "@smithy/signature-v4": "^1.0.1", + "@smithy/types": "^1.1.0", + "@smithy/util-middleware": "^1.0.1", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/client-sqs/node_modules/@aws-sdk/middleware-user-agent": { + "version": "3.370.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-user-agent/-/middleware-user-agent-3.370.0.tgz", + "integrity": "sha512-2+3SB6MtMAq1+gVXhw0Y3ONXuljorh6ijnxgTpv+uQnBW5jHCUiAS8WDYiDEm7i9euJPbvJfM8WUrSMDMU6Cog==", + "dependencies": { + "@aws-sdk/types": "3.370.0", + "@aws-sdk/util-endpoints": "3.370.0", + "@smithy/protocol-http": "^1.1.0", + "@smithy/types": "^1.1.0", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/client-sqs/node_modules/@aws-sdk/token-providers": { + "version": "3.370.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.370.0.tgz", + "integrity": "sha512-EyR2ZYr+lJeRiZU2/eLR+mlYU9RXLQvNyGFSAekJKgN13Rpq/h0syzXVFLP/RSod/oZenh/fhVZ2HwlZxuGBtQ==", + "dependencies": { + "@aws-sdk/client-sso-oidc": "3.370.0", + "@aws-sdk/types": "3.370.0", + "@smithy/property-provider": "^1.0.1", + "@smithy/shared-ini-file-loader": "^1.0.1", + "@smithy/types": "^1.1.0", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/client-sqs/node_modules/@aws-sdk/types": { + "version": "3.370.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.370.0.tgz", + "integrity": "sha512-8PGMKklSkRKjunFhzM2y5Jm0H2TBu7YRNISdYzXLUHKSP9zlMEYagseKVdmox0zKHf1LXVNuSlUV2b6SRrieCQ==", + "dependencies": { + "@smithy/types": "^1.1.0", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/client-sqs/node_modules/@aws-sdk/util-endpoints": { + "version": "3.370.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-endpoints/-/util-endpoints-3.370.0.tgz", + "integrity": "sha512-5ltVAnM79nRlywwzZN5i8Jp4tk245OCGkKwwXbnDU+gq7zT3CIOsct1wNZvmpfZEPGt/bv7/NyRcjP+7XNsX/g==", + "dependencies": { + "@aws-sdk/types": "3.370.0", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/client-sqs/node_modules/@aws-sdk/util-user-agent-browser": { + "version": "3.370.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-browser/-/util-user-agent-browser-3.370.0.tgz", + "integrity": "sha512-028LxYZMQ0DANKhW+AKFQslkScZUeYlPmSphrCIXgdIItRZh6ZJHGzE7J/jDsEntZOrZJsjI4z0zZ5W2idj04w==", + "dependencies": { + "@aws-sdk/types": "3.370.0", + "@smithy/types": "^1.1.0", + "bowser": "^2.11.0", + "tslib": "^2.5.0" + } + }, + "node_modules/@aws-sdk/client-sqs/node_modules/@aws-sdk/util-user-agent-node": { + "version": "3.370.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-node/-/util-user-agent-node-3.370.0.tgz", + "integrity": "sha512-33vxZUp8vxTT/DGYIR3PivQm07sSRGWI+4fCv63Rt7Q++fO24E0kQtmVAlikRY810I10poD6rwILVtITtFSzkg==", + "dependencies": { + "@aws-sdk/types": "3.370.0", + "@smithy/node-config-provider": "^1.0.1", + "@smithy/types": "^1.1.0", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + }, + "peerDependencies": { + "aws-crt": ">=1.0.0" + }, + "peerDependenciesMeta": { + "aws-crt": { + "optional": true + } + } + }, + "node_modules/@aws-sdk/client-sqs/node_modules/fast-xml-parser": { + "version": "4.2.5", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.2.5.tgz", + "integrity": "sha512-B9/wizE4WngqQftFPmdaMYlXoJlJOYxGQOanC77fq9k8+Z0v5dDSVh+3glErdIROP//s/jgb7ZuxKfB8nVyo0g==", + "funding": [ + { + "type": "paypal", + "url": "https://paypal.me/naturalintelligence" + }, + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "dependencies": { + "strnum": "^1.0.5" + }, + "bin": { + "fxparser": "src/cli/cli.js" + } + }, "node_modules/@aws-sdk/client-sso": { "version": "3.350.0", "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso/-/client-sso-3.350.0.tgz", @@ -1417,6 +1882,33 @@ "node": ">=14.0.0" } }, + "node_modules/@aws-sdk/middleware-sdk-sqs": { + "version": "3.370.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-sdk-sqs/-/middleware-sdk-sqs-3.370.0.tgz", + "integrity": "sha512-jk61LjuxYe3gfcxuPsAPuKCycTGC7bhd8VHGBjlndLmYIG6Y6t668fvVXTKsz+CWQLTpAjhkS8ON2nJKiMqaQw==", + "dependencies": { + "@aws-sdk/types": "3.370.0", + "@smithy/types": "^1.1.0", + "@smithy/util-hex-encoding": "^1.0.1", + "@smithy/util-utf8": "^1.0.1", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/middleware-sdk-sqs/node_modules/@aws-sdk/types": { + "version": "3.370.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.370.0.tgz", + "integrity": "sha512-8PGMKklSkRKjunFhzM2y5Jm0H2TBu7YRNISdYzXLUHKSP9zlMEYagseKVdmox0zKHf1LXVNuSlUV2b6SRrieCQ==", + "dependencies": { + "@smithy/types": "^1.1.0", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, "node_modules/@aws-sdk/middleware-sdk-sts": { "version": "3.347.0", "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-sdk-sts/-/middleware-sdk-sts-3.347.0.tgz", @@ -3647,6 +4139,16 @@ "node": ">=14.0.0" } }, + "node_modules/@smithy/md5-js": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@smithy/md5-js/-/md5-js-1.0.2.tgz", + "integrity": "sha512-0yUgIvIUt63Rb5+ErZTraQguc4Vu3Fw7NKJL0ozLnj1hcYDrt45pfQjUMztKBE7ve32vCnuSOA4LCAe3fudHZA==", + "dependencies": { + "@smithy/types": "^1.1.1", + "@smithy/util-utf8": "^1.0.2", + "tslib": "^2.5.0" + } + }, "node_modules/@smithy/middleware-content-length": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/@smithy/middleware-content-length/-/middleware-content-length-1.0.2.tgz", diff --git a/backend/package.json b/backend/package.json index 8efc85e9..66b87901 100644 --- a/backend/package.json +++ b/backend/package.json @@ -27,6 +27,7 @@ }, "dependencies": { "@aws-sdk/client-s3": "^3.350.0", + "@aws-sdk/client-sqs": "^3.370.0", "@aws-sdk/client-textract": "^3.370.0", "@nestjs/common": "^9.0.0", "@nestjs/core": "^9.0.0", diff --git a/backend/src/textract/textract.service.ts b/backend/src/textract/textract.service.ts index be808923..adcd5534 100644 --- a/backend/src/textract/textract.service.ts +++ b/backend/src/textract/textract.service.ts @@ -1,8 +1,6 @@ import { Injectable } from '@nestjs/common'; import { TextractClient, - DetectDocumentTextCommand, - AnalyzeDocumentCommand, FeatureType, StartDocumentTextDetectionCommand, StartDocumentAnalysisCommand, @@ -11,6 +9,12 @@ import { GetDocumentAnalysisCommand, } from '@aws-sdk/client-textract'; import 'dotenv/config'; +import { MarkdownFileDTO } from '../markdown_files/dto/markdown_file.dto'; +import { + DeleteMessageCommand, + ReceiveMessageCommand, + SQSClient, +} from '@aws-sdk/client-sqs'; @Injectable() export class TextractService { @@ -22,12 +26,126 @@ export class TextractService { process.env.AWS_S3_ACCESS_KEY_ID; awsS3SecretAccessKey = process.env.AWS_S3_SECRET_ACCESS_KEY; + snsTopicArn = + process.env.AWS_TEXTRACT_SNS_TOPIC_ARN; + roleArn = process.env.AWS_S3_TEXTRACT_ROLE_ARN; + queueUrl = + process.env.AWS_S3_TEXTRACT_QUEUE_URL; + + private readonly textractClient = + new TextractClient({ + credentials: { + accessKeyId: this.awsS3AccessKeyId, + secretAccessKey: + this.awsS3SecretAccessKey, + }, + region: this.awsS3BucketRegion, + }); - private readonly s3Client = new TextractClient({ + private readonly sqsClient = new SQSClient({ credentials: { accessKeyId: this.awsS3AccessKeyId, secretAccessKey: this.awsS3SecretAccessKey, }, region: this.awsS3BucketRegion, }); + + async _extractDocumentAsynchronous( + markdownFileDTO: MarkdownFileDTO, + extractType: string, + ) { + const input = { + DocumentLocation: { + S3Object: { + Bucket: this.awsS3BucketName, + Name: markdownFileDTO.Name, + }, + }, + NotificationChannel: { + SNSTopicArn: this.snsTopicArn, + RoleArn: this.roleArn, + }, + }; + + let command; + if (extractType === 'text') { + command = + new StartDocumentTextDetectionCommand( + input, + ); + } else { + input['FeatureTypes'] = [ + FeatureType.TABLES, + ]; + command = new StartDocumentAnalysisCommand( + input, + ); + } + + const { JobId: jobId } = + await this.textractClient.send(command); + console.log(`JobId: ${jobId}`); + + let waitTime = 0; + const getJob = async () => { + const { Messages } = + await this.sqsClient.send( + new ReceiveMessageCommand({ + QueueUrl: this.queueUrl, + MaxNumberOfMessages: 1, + }), + ); + if (Messages) { + console.log( + `Message[0]: ${Messages[0].Body}`, + ); + await this.sqsClient.send( + new DeleteMessageCommand({ + QueueUrl: this.queueUrl, + ReceiptHandle: + Messages[0].ReceiptHandle, + }), + ); + if ( + JSON.parse( + JSON.parse(Messages[0].Body).Message, + ).Status === JobStatus.SUCCEEDED + ) { + let getCommand; + if (extractType === 'text') { + getCommand = + new GetDocumentTextDetectionCommand( + { JobId: jobId }, + ); + } else { + getCommand = + new GetDocumentAnalysisCommand({ + JobId: jobId, + }); + } + const { Blocks } = + await this.textractClient.send( + getCommand, + ); + this.extraction = { + Name: this.imageData.objectKey, + ExtractType: extractType, + Children: + this._make_page_hierarchy(Blocks), + }; + this.inform(); + } + } else { + const tick = 5000; + waitTime += tick; + console.log( + `Waited ${ + waitTime / 1000 + } seconds. No messages yet.`, + ); + setTimeout(getJob, tick); + } + }; + await getJob(jobId); + } } From 549f6728c63da379179ca4841cbf32d5f5e1ced3 Mon Sep 17 00:00:00 2001 From: Dylan Kapnias Date: Sun, 23 Jul 2023 15:59:29 +0200 Subject: [PATCH 09/14] Does indeed compile --- backend/src/textract/textract.service.ts | 81 ++++++++++++++---------- 1 file changed, 47 insertions(+), 34 deletions(-) diff --git a/backend/src/textract/textract.service.ts b/backend/src/textract/textract.service.ts index adcd5534..ac5d52de 100644 --- a/backend/src/textract/textract.service.ts +++ b/backend/src/textract/textract.service.ts @@ -50,44 +50,55 @@ export class TextractService { region: this.awsS3BucketRegion, }); + extraction = null; + async _extractDocumentAsynchronous( markdownFileDTO: MarkdownFileDTO, extractType: string, ) { - const input = { - DocumentLocation: { - S3Object: { - Bucket: this.awsS3BucketName, - Name: markdownFileDTO.Name, - }, - }, - NotificationChannel: { - SNSTopicArn: this.snsTopicArn, - RoleArn: this.roleArn, - }, - }; - - let command; + let textCommand: StartDocumentTextDetectionCommand; + let tableCommand: StartDocumentAnalysisCommand; if (extractType === 'text') { - command = - new StartDocumentTextDetectionCommand( - input, - ); + textCommand = + new StartDocumentTextDetectionCommand({ + DocumentLocation: { + S3Object: { + Bucket: this.awsS3BucketName, + Name: markdownFileDTO.Name, + }, + }, + NotificationChannel: { + SNSTopicArn: this.snsTopicArn, + RoleArn: this.roleArn, + }, + }); } else { - input['FeatureTypes'] = [ - FeatureType.TABLES, - ]; - command = new StartDocumentAnalysisCommand( - input, - ); + tableCommand = + new StartDocumentAnalysisCommand({ + DocumentLocation: { + S3Object: { + Bucket: this.awsS3BucketName, + Name: markdownFileDTO.Name, + }, + }, + NotificationChannel: { + SNSTopicArn: this.snsTopicArn, + RoleArn: this.roleArn, + }, + FeatureTypes: [FeatureType.TABLES], + }); } const { JobId: jobId } = - await this.textractClient.send(command); + await this.textractClient.send( + extractType === 'text' + ? textCommand + : tableCommand, + ); console.log(`JobId: ${jobId}`); let waitTime = 0; - const getJob = async () => { + const getJob = async (jobId) => { const { Messages } = await this.sqsClient.send( new ReceiveMessageCommand({ @@ -111,29 +122,31 @@ export class TextractService { JSON.parse(Messages[0].Body).Message, ).Status === JobStatus.SUCCEEDED ) { - let getCommand; + let getTextCommand: GetDocumentTextDetectionCommand; + let getTableCommand: GetDocumentAnalysisCommand; if (extractType === 'text') { - getCommand = + getTextCommand = new GetDocumentTextDetectionCommand( { JobId: jobId }, ); } else { - getCommand = + getTableCommand = new GetDocumentAnalysisCommand({ JobId: jobId, }); } const { Blocks } = await this.textractClient.send( - getCommand, + extractType === 'text' + ? getTextCommand + : getTableCommand, ); this.extraction = { - Name: this.imageData.objectKey, + Name: markdownFileDTO.Name, ExtractType: extractType, - Children: - this._make_page_hierarchy(Blocks), + Extracted: Blocks, }; - this.inform(); + // this.inform(); } } else { const tick = 5000; From 9300ed59ef7888b8ea78e29c352e91b332a39fca Mon Sep 17 00:00:00 2001 From: Dylan Kapnias Date: Sun, 23 Jul 2023 19:39:02 +0200 Subject: [PATCH 10/14] Debugging changes --- backend/src/app.module.ts | 2 + backend/src/textract/textract.controller.ts | 32 +++- backend/src/textract/textract.service.ts | 190 +++++++++++++------- infrastructure/lib/s3-stack.ts | 12 +- 4 files changed, 160 insertions(+), 76 deletions(-) diff --git a/backend/src/app.module.ts b/backend/src/app.module.ts index 241f4081..1f09c824 100644 --- a/backend/src/app.module.ts +++ b/backend/src/app.module.ts @@ -29,6 +29,7 @@ import { ConversionService } from './conversion/conversion.service'; import { User } from './users/entities/user.entity'; import { TextractController } from './textract/textract.controller'; import { TextractModule } from './textract/textract.module'; +import { TextractService } from './textract/textract.service'; @Module({ imports: [ @@ -62,6 +63,7 @@ import { TextractModule } from './textract/textract.module'; FoldersService, S3Service, ConversionService, + TextractService, ], }) export class AppModule {} diff --git a/backend/src/textract/textract.controller.ts b/backend/src/textract/textract.controller.ts index 8491112b..e523a22f 100644 --- a/backend/src/textract/textract.controller.ts +++ b/backend/src/textract/textract.controller.ts @@ -1,4 +1,32 @@ -import { Controller } from '@nestjs/common'; +import { + Controller, + Post, + Body, + UseInterceptors, +} from '@nestjs/common'; +import { FileInterceptor } from '@nestjs/platform-express'; +import { MarkdownFileDTO } from '../markdown_files/dto/markdown_file.dto'; +import { TextractService } from './textract.service'; @Controller('textract') -export class TextractController {} +export class TextractController { + constructor( + private readonly textractService: TextractService, + ) {} + @Post('extract_image') + @UseInterceptors(FileInterceptor('file')) + async extractImage( + @Body() fileDTO: MarkdownFileDTO, + ExtractType: string, + ) { + const retVal = + await this.textractService._extractDocumentSynchronous( + fileDTO, + ExtractType, + ); + + console.log(retVal); + + return retVal; + } +} diff --git a/backend/src/textract/textract.service.ts b/backend/src/textract/textract.service.ts index ac5d52de..6a86106e 100644 --- a/backend/src/textract/textract.service.ts +++ b/backend/src/textract/textract.service.ts @@ -52,7 +52,118 @@ export class TextractService { extraction = null; - async _extractDocumentAsynchronous( + // async _extractDocumentAsynchronous( + // markdownFileDTO: MarkdownFileDTO, + // extractType: string, + // ) { + // console.log(markdownFileDTO); + // let textCommand: StartDocumentTextDetectionCommand; + // let tableCommand: StartDocumentAnalysisCommand; + // if (extractType === 'text') { + // textCommand = + // new StartDocumentTextDetectionCommand({ + // DocumentLocation: { + // S3Object: { + // Bucket: this.awsS3BucketName, + // Name: markdownFileDTO.MarkdownID, + // }, + // }, + // NotificationChannel: { + // SNSTopicArn: this.snsTopicArn, + // RoleArn: this.roleArn, + // }, + // }); + // } else { + // tableCommand = + // new StartDocumentAnalysisCommand({ + // DocumentLocation: { + // S3Object: { + // Bucket: this.awsS3BucketName, + // Name: markdownFileDTO.MarkdownID, + // }, + // }, + // NotificationChannel: { + // SNSTopicArn: this.snsTopicArn, + // RoleArn: this.roleArn, + // }, + // FeatureTypes: [FeatureType.TABLES], + // }); + // } + + // const { JobId: jobId } = + // await this.textractClient.send( + // extractType === 'text' + // ? textCommand + // : tableCommand, + // ); + // console.log(`JobId: ${jobId}`); + + // let waitTime = 0; + // const getJob = async (jobId) => { + // const { Messages } = + // await this.sqsClient.send( + // new ReceiveMessageCommand({ + // QueueUrl: this.queueUrl, + // MaxNumberOfMessages: 1, + // }), + // ); + // if (Messages) { + // console.log( + // `Message[0]: ${Messages[0].Body}`, + // ); + // await this.sqsClient.send( + // new DeleteMessageCommand({ + // QueueUrl: this.queueUrl, + // ReceiptHandle: + // Messages[0].ReceiptHandle, + // }), + // ); + // if ( + // JSON.parse( + // JSON.parse(Messages[0].Body).Message, + // ).Status === JobStatus.SUCCEEDED + // ) { + // let getTextCommand: GetDocumentTextDetectionCommand; + // let getTableCommand: GetDocumentAnalysisCommand; + // if (extractType === 'text') { + // getTextCommand = + // new GetDocumentTextDetectionCommand( + // { JobId: jobId }, + // ); + // } else { + // getTableCommand = + // new GetDocumentAnalysisCommand({ + // JobId: jobId, + // }); + // } + // const { Blocks } = + // await this.textractClient.send( + // extractType === 'text' + // ? getTextCommand + // : getTableCommand, + // ); + // this.extraction = { + // Name: markdownFileDTO.MarkdownID, + // ExtractType: extractType, + // Extracted: Blocks, + // }; + // return this.extraction; + // } + // } else { + // const tick = 5000; + // waitTime += tick; + // console.log( + // `Waited ${ + // waitTime / 1000 + // } seconds. No messages yet.`, + // ); + // setTimeout(getJob, tick); + // } + // }; + // await getJob(jobId); + // } + + async _extractDocumentSynchronous( markdownFileDTO: MarkdownFileDTO, extractType: string, ) { @@ -64,7 +175,7 @@ export class TextractService { DocumentLocation: { S3Object: { Bucket: this.awsS3BucketName, - Name: markdownFileDTO.Name, + Name: markdownFileDTO.MarkdownID, }, }, NotificationChannel: { @@ -78,7 +189,7 @@ export class TextractService { DocumentLocation: { S3Object: { Bucket: this.awsS3BucketName, - Name: markdownFileDTO.Name, + Name: markdownFileDTO.MarkdownID, }, }, NotificationChannel: { @@ -88,77 +199,20 @@ export class TextractService { FeatureTypes: [FeatureType.TABLES], }); } - - const { JobId: jobId } = + const textractResponse = await this.textractClient.send( extractType === 'text' ? textCommand : tableCommand, ); - console.log(`JobId: ${jobId}`); - let waitTime = 0; - const getJob = async (jobId) => { - const { Messages } = - await this.sqsClient.send( - new ReceiveMessageCommand({ - QueueUrl: this.queueUrl, - MaxNumberOfMessages: 1, - }), - ); - if (Messages) { - console.log( - `Message[0]: ${Messages[0].Body}`, - ); - await this.sqsClient.send( - new DeleteMessageCommand({ - QueueUrl: this.queueUrl, - ReceiptHandle: - Messages[0].ReceiptHandle, - }), - ); - if ( - JSON.parse( - JSON.parse(Messages[0].Body).Message, - ).Status === JobStatus.SUCCEEDED - ) { - let getTextCommand: GetDocumentTextDetectionCommand; - let getTableCommand: GetDocumentAnalysisCommand; - if (extractType === 'text') { - getTextCommand = - new GetDocumentTextDetectionCommand( - { JobId: jobId }, - ); - } else { - getTableCommand = - new GetDocumentAnalysisCommand({ - JobId: jobId, - }); - } - const { Blocks } = - await this.textractClient.send( - extractType === 'text' - ? getTextCommand - : getTableCommand, - ); - this.extraction = { - Name: markdownFileDTO.Name, - ExtractType: extractType, - Extracted: Blocks, - }; - // this.inform(); - } - } else { - const tick = 5000; - waitTime += tick; - console.log( - `Waited ${ - waitTime / 1000 - } seconds. No messages yet.`, - ); - setTimeout(getJob, tick); - } + this.extraction = { + Name: markdownFileDTO.MarkdownID, + ExtractType: extractType, + Children: textractResponse, }; - await getJob(jobId); + console.log(textractResponse); + + return textractResponse; } } diff --git a/infrastructure/lib/s3-stack.ts b/infrastructure/lib/s3-stack.ts index 9c74061f..15a7f035 100644 --- a/infrastructure/lib/s3-stack.ts +++ b/infrastructure/lib/s3-stack.ts @@ -49,6 +49,12 @@ export class S3Stack extends Stack { user.addToPolicy(policy); + new CfnOutput(this, "TopicArn", { value: topic.topicArn }); + + new CfnOutput(this, "RoleArn", { value: role.roleArn }); + + new CfnOutput(this, "QueueUrl", { value: queue.queueUrl }); + new CfnOutput(this, "S3BucketName", { value: bucket.bucketName, }); @@ -56,11 +62,5 @@ export class S3Stack extends Stack { new CfnOutput(this, "S3BucketRegion", { value: bucket.bucketRegionalDomainName, }); - - new CfnOutput(this, "TopicArn", { value: topic.topicArn }); - - new CfnOutput(this, "RoleArn", { value: role.roleArn }); - - new CfnOutput(this, "QueueUrl", { value: queue.queueUrl }); } } From b2ecd1d5d7f72667742368cbae0e849a808ff34e Mon Sep 17 00:00:00 2001 From: Dylan Kapnias Date: Sun, 23 Jul 2023 19:59:59 +0200 Subject: [PATCH 11/14] Synchronous call working --- backend/src/textract/textract.service.ts | 52 ++++++++++++------------ 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/backend/src/textract/textract.service.ts b/backend/src/textract/textract.service.ts index 6a86106e..574c3350 100644 --- a/backend/src/textract/textract.service.ts +++ b/backend/src/textract/textract.service.ts @@ -7,6 +7,8 @@ import { GetDocumentTextDetectionCommand, JobStatus, GetDocumentAnalysisCommand, + AnalyzeDocumentCommand, + DetectDocumentTextCommand, } from '@aws-sdk/client-textract'; import 'dotenv/config'; import { MarkdownFileDTO } from '../markdown_files/dto/markdown_file.dto'; @@ -167,37 +169,35 @@ export class TextractService { markdownFileDTO: MarkdownFileDTO, extractType: string, ) { - let textCommand: StartDocumentTextDetectionCommand; - let tableCommand: StartDocumentAnalysisCommand; + let textCommand: DetectDocumentTextCommand; + let tableCommand: AnalyzeDocumentCommand; if (extractType === 'text') { - textCommand = - new StartDocumentTextDetectionCommand({ - DocumentLocation: { + textCommand = new DetectDocumentTextCommand( + { + Document: { S3Object: { - Bucket: this.awsS3BucketName, - Name: markdownFileDTO.MarkdownID, + Bucket: + 'writetopdfs3stack-writetopdfappbucketfc6d0172-1rmwa22kdzix8', + // Bucket: this.awsS3BucketName, + Name: 'IMG_3601.jpeg', + // Name: markdownFileDTO.MarkdownID, }, }, - NotificationChannel: { - SNSTopicArn: this.snsTopicArn, - RoleArn: this.roleArn, - }, - }); + }, + ); } else { - tableCommand = - new StartDocumentAnalysisCommand({ - DocumentLocation: { - S3Object: { - Bucket: this.awsS3BucketName, - Name: markdownFileDTO.MarkdownID, - }, - }, - NotificationChannel: { - SNSTopicArn: this.snsTopicArn, - RoleArn: this.roleArn, + tableCommand = new AnalyzeDocumentCommand({ + Document: { + S3Object: { + Bucket: + 'writetopdfs3stack-writetopdfappbucketfc6d0172-1rmwa22kdzix8', + // Bucket: this.awsS3BucketName, + Name: 'IMG_3601.jpeg', + // Name: markdownFileDTO.MarkdownID, }, - FeatureTypes: [FeatureType.TABLES], - }); + }, + FeatureTypes: [FeatureType.TABLES], + }); } const textractResponse = await this.textractClient.send( @@ -211,7 +211,7 @@ export class TextractService { ExtractType: extractType, Children: textractResponse, }; - console.log(textractResponse); + console.log(textractResponse['Blocks']); return textractResponse; } From fa183d44fbd8caa5bf688927e680cdfbe49df6ea Mon Sep 17 00:00:00 2001 From: Dylan Kapnias Date: Sun, 23 Jul 2023 20:14:50 +0200 Subject: [PATCH 12/14] added hierarchy creation --- backend/src/textract/textract.service.ts | 52 ++++++++++++++++++------ 1 file changed, 39 insertions(+), 13 deletions(-) diff --git a/backend/src/textract/textract.service.ts b/backend/src/textract/textract.service.ts index 574c3350..2fe73620 100644 --- a/backend/src/textract/textract.service.ts +++ b/backend/src/textract/textract.service.ts @@ -176,11 +176,8 @@ export class TextractService { { Document: { S3Object: { - Bucket: - 'writetopdfs3stack-writetopdfappbucketfc6d0172-1rmwa22kdzix8', - // Bucket: this.awsS3BucketName, - Name: 'IMG_3601.jpeg', - // Name: markdownFileDTO.MarkdownID, + Bucket: this.awsS3BucketName, + Name: markdownFileDTO.MarkdownID, }, }, }, @@ -189,11 +186,8 @@ export class TextractService { tableCommand = new AnalyzeDocumentCommand({ Document: { S3Object: { - Bucket: - 'writetopdfs3stack-writetopdfappbucketfc6d0172-1rmwa22kdzix8', - // Bucket: this.awsS3BucketName, - Name: 'IMG_3601.jpeg', - // Name: markdownFileDTO.MarkdownID, + Bucket: this.awsS3BucketName, + Name: markdownFileDTO.MarkdownID, }, }, FeatureTypes: [FeatureType.TABLES], @@ -209,10 +203,42 @@ export class TextractService { this.extraction = { Name: markdownFileDTO.MarkdownID, ExtractType: extractType, - Children: textractResponse, + Children: this._make_page_hierarchy( + textractResponse['Blocks'], + ), }; - console.log(textractResponse['Blocks']); + console.log(textractResponse); - return textractResponse; + return this.extraction; + } + + _add_children(block, block_dict) { + const rels_list = block.Relationships || []; + rels_list.forEach((rels) => { + if (rels.Type === 'CHILD') { + block['Children'] = []; + rels.Ids.forEach((relId) => { + const kid = block_dict[relId]; + block['Children'].push(kid); + this._add_children(kid, block_dict); + }); + } + }); + } + + _make_page_hierarchy(blocks) { + const block_dict = {}; + blocks.forEach( + (block) => (block_dict[block.Id] = block), + ); + + const pages = []; + blocks.forEach((block) => { + if (block.BlockType === 'PAGE') { + pages.push(block); + this._add_children(block, block_dict); + } + }); + return pages; } } From 3ff192f270e626a27d8cbe21465d7101128d7f8f Mon Sep 17 00:00:00 2001 From: Dylan Kapnias Date: Sun, 23 Jul 2023 21:25:45 +0200 Subject: [PATCH 13/14] added debugging functions Need to fix jobId type error. Redo asynchronous function checks --- backend/src/textract/textract.controller.ts | 46 +++- backend/src/textract/textract.service.ts | 274 ++++++++++++-------- 2 files changed, 205 insertions(+), 115 deletions(-) diff --git a/backend/src/textract/textract.controller.ts b/backend/src/textract/textract.controller.ts index e523a22f..6a42ac34 100644 --- a/backend/src/textract/textract.controller.ts +++ b/backend/src/textract/textract.controller.ts @@ -20,7 +20,7 @@ export class TextractController { ExtractType: string, ) { const retVal = - await this.textractService._extractDocumentSynchronous( + await this.textractService._extractDocumentAsynchronous( fileDTO, ExtractType, ); @@ -29,4 +29,48 @@ export class TextractController { return retVal; } + + @Post('extract_test') + @UseInterceptors(FileInterceptor('file')) + async extract_test( + @Body() fileDTO: MarkdownFileDTO, + ExtractType: string, + ) { + const retVal = + await this.textractService.test_get( + ExtractType, + ); + + console.log(retVal); + + return retVal; + } + + @Post('extract_msg') + @UseInterceptors(FileInterceptor('file')) + async extract_msg( + @Body() fileDTO: MarkdownFileDTO, + ExtractType: string, + ) { + const retVal = + await this.textractService.test_msg(); + + console.log(retVal); + + return retVal; + } + + @Post('extract_del') + @UseInterceptors(FileInterceptor('file')) + async extract_del( + @Body() fileDTO: MarkdownFileDTO, + ExtractType: string, + ) { + const retVal = + await this.textractService.test_del(); + + console.log(retVal); + + return retVal; + } } diff --git a/backend/src/textract/textract.service.ts b/backend/src/textract/textract.service.ts index 2fe73620..ff4ae69f 100644 --- a/backend/src/textract/textract.service.ts +++ b/backend/src/textract/textract.service.ts @@ -54,116 +54,122 @@ export class TextractService { extraction = null; - // async _extractDocumentAsynchronous( - // markdownFileDTO: MarkdownFileDTO, - // extractType: string, - // ) { - // console.log(markdownFileDTO); - // let textCommand: StartDocumentTextDetectionCommand; - // let tableCommand: StartDocumentAnalysisCommand; - // if (extractType === 'text') { - // textCommand = - // new StartDocumentTextDetectionCommand({ - // DocumentLocation: { - // S3Object: { - // Bucket: this.awsS3BucketName, - // Name: markdownFileDTO.MarkdownID, - // }, - // }, - // NotificationChannel: { - // SNSTopicArn: this.snsTopicArn, - // RoleArn: this.roleArn, - // }, - // }); - // } else { - // tableCommand = - // new StartDocumentAnalysisCommand({ - // DocumentLocation: { - // S3Object: { - // Bucket: this.awsS3BucketName, - // Name: markdownFileDTO.MarkdownID, - // }, - // }, - // NotificationChannel: { - // SNSTopicArn: this.snsTopicArn, - // RoleArn: this.roleArn, - // }, - // FeatureTypes: [FeatureType.TABLES], - // }); - // } - - // const { JobId: jobId } = - // await this.textractClient.send( - // extractType === 'text' - // ? textCommand - // : tableCommand, - // ); - // console.log(`JobId: ${jobId}`); - - // let waitTime = 0; - // const getJob = async (jobId) => { - // const { Messages } = - // await this.sqsClient.send( - // new ReceiveMessageCommand({ - // QueueUrl: this.queueUrl, - // MaxNumberOfMessages: 1, - // }), - // ); - // if (Messages) { - // console.log( - // `Message[0]: ${Messages[0].Body}`, - // ); - // await this.sqsClient.send( - // new DeleteMessageCommand({ - // QueueUrl: this.queueUrl, - // ReceiptHandle: - // Messages[0].ReceiptHandle, - // }), - // ); - // if ( - // JSON.parse( - // JSON.parse(Messages[0].Body).Message, - // ).Status === JobStatus.SUCCEEDED - // ) { - // let getTextCommand: GetDocumentTextDetectionCommand; - // let getTableCommand: GetDocumentAnalysisCommand; - // if (extractType === 'text') { - // getTextCommand = - // new GetDocumentTextDetectionCommand( - // { JobId: jobId }, - // ); - // } else { - // getTableCommand = - // new GetDocumentAnalysisCommand({ - // JobId: jobId, - // }); - // } - // const { Blocks } = - // await this.textractClient.send( - // extractType === 'text' - // ? getTextCommand - // : getTableCommand, - // ); - // this.extraction = { - // Name: markdownFileDTO.MarkdownID, - // ExtractType: extractType, - // Extracted: Blocks, - // }; - // return this.extraction; - // } - // } else { - // const tick = 5000; - // waitTime += tick; - // console.log( - // `Waited ${ - // waitTime / 1000 - // } seconds. No messages yet.`, - // ); - // setTimeout(getJob, tick); - // } - // }; - // await getJob(jobId); - // } + async _extractDocumentAsynchronous( + markdownFileDTO: MarkdownFileDTO, + extractType: string, + ) { + console.log(markdownFileDTO); + let textCommand: StartDocumentTextDetectionCommand; + let tableCommand: StartDocumentAnalysisCommand; + if (extractType === 'text') { + textCommand = + new StartDocumentTextDetectionCommand({ + DocumentLocation: { + S3Object: { + Bucket: this.awsS3BucketName, + Name: markdownFileDTO.MarkdownID, + }, + }, + NotificationChannel: { + SNSTopicArn: this.snsTopicArn, + RoleArn: this.roleArn, + }, + }); + } else { + tableCommand = + new StartDocumentAnalysisCommand({ + DocumentLocation: { + S3Object: { + Bucket: this.awsS3BucketName, + Name: markdownFileDTO.MarkdownID, + }, + }, + NotificationChannel: { + SNSTopicArn: this.snsTopicArn, + RoleArn: this.roleArn, + }, + FeatureTypes: [FeatureType.TABLES], + }); + } + + const { JobId: jobId } = + await this.textractClient.send( + extractType === 'text' + ? textCommand + : tableCommand, + ); + console.log(`JobId: ${jobId}`); + + let waitTime = 0; + const getJob = async (jobId: string) => { + const { Messages } = + await this.sqsClient.send( + new ReceiveMessageCommand({ + QueueUrl: this.queueUrl, + MaxNumberOfMessages: 1, + }), + ); + if (Messages) { + console.log( + `Message[0]: ${Messages[0].Body}`, + ); + await this.sqsClient.send( + new DeleteMessageCommand({ + QueueUrl: this.queueUrl, + ReceiptHandle: + Messages[0].ReceiptHandle, + }), + ); + if ( + JSON.parse( + JSON.parse(Messages[0].Body).Message, + ).Status === JobStatus.SUCCEEDED + ) { + let getTextCommand: GetDocumentTextDetectionCommand; + let getTableCommand: GetDocumentAnalysisCommand; + if (extractType === 'text') { + getTextCommand = + new GetDocumentTextDetectionCommand( + { JobId: jobId }, + ); + } else { + getTableCommand = + new GetDocumentAnalysisCommand({ + JobId: jobId, + }); + } + const { Blocks } = + await this.textractClient.send( + extractType === 'text' + ? getTextCommand + : getTableCommand, + ); + this.extraction = { + Name: markdownFileDTO.MarkdownID, + ExtractType: extractType, + Extracted: Blocks, + }; + + // if (waitTime > 20000) { + // return; + // } + + console.log(this.extraction); + } + } else { + const tick = 5000; + waitTime += tick; + console.log( + `Waited ${ + waitTime / 1000 + } seconds. No messages yet.`, + ); + setTimeout(getJob, tick); + } + }; + await getJob(jobId); + return this.extraction; + } async _extractDocumentSynchronous( markdownFileDTO: MarkdownFileDTO, @@ -203,13 +209,14 @@ export class TextractService { this.extraction = { Name: markdownFileDTO.MarkdownID, ExtractType: extractType, - Children: this._make_page_hierarchy( - textractResponse['Blocks'], - ), + // Children: this._make_page_hierarchy( + // textractResponse['Blocks'], + // ), + Children: textractResponse, }; console.log(textractResponse); - return this.extraction; + return textractResponse; } _add_children(block, block_dict) { @@ -241,4 +248,43 @@ export class TextractService { }); return pages; } + + async test_get(jobId: string) { + const retVal = await this.textractClient.send( + new GetDocumentAnalysisCommand({ + JobId: + '3e8eb41e481cced553bd49108cbf3ec02b6397acfec80b5281118c022ddd91a4', + }), + ); + + return retVal; + } + + async test_msg() { + const { Messages } = + await this.sqsClient.send( + new ReceiveMessageCommand({ + QueueUrl: this.queueUrl, + MaxNumberOfMessages: 10, + }), + ); + + return Messages; + } + + async test_del() { + const { Messages } = + await this.sqsClient.send( + new ReceiveMessageCommand({ + QueueUrl: this.queueUrl, + MaxNumberOfMessages: 10, + }), + ); + await this.sqsClient.send( + new DeleteMessageCommand({ + QueueUrl: this.queueUrl, + ReceiptHandle: Messages[0].ReceiptHandle, + }), + ); + } } From b4988714ee02891b0675e7678ab744f18ecf8ed2 Mon Sep 17 00:00:00 2001 From: Dylan Kapnias Date: Mon, 24 Jul 2023 18:49:15 +0200 Subject: [PATCH 14/14] Added one endpoint to do both sync and async. --- backend/src/textract/textract.controller.ts | 3 ++- backend/src/textract/textract.service.ts | 28 +++++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/backend/src/textract/textract.controller.ts b/backend/src/textract/textract.controller.ts index 6a42ac34..aad90fcd 100644 --- a/backend/src/textract/textract.controller.ts +++ b/backend/src/textract/textract.controller.ts @@ -20,7 +20,8 @@ export class TextractController { ExtractType: string, ) { const retVal = - await this.textractService._extractDocumentAsynchronous( + await this.textractService.extractDocument( + 'sync', fileDTO, ExtractType, ); diff --git a/backend/src/textract/textract.service.ts b/backend/src/textract/textract.service.ts index ff4ae69f..b6153f2c 100644 --- a/backend/src/textract/textract.service.ts +++ b/backend/src/textract/textract.service.ts @@ -287,4 +287,32 @@ export class TextractService { }), ); } + + async extractDocument( + syncType: string, + markdownFileDTO: MarkdownFileDTO, + extractType: string, + ) { + let retVal; + try { + if (syncType === 'sync') { + retVal = + await this._extractDocumentSynchronous( + markdownFileDTO, + extractType, + ); + } else { + retVal = + await this._extractDocumentAsynchronous( + markdownFileDTO, + extractType, + ); + } + } catch (error) { + console.log(error.message); + } finally { + console.log('Finally'); + return retVal; + } + } }