Skip to content

Commit

Permalink
Merge pull request #2410 from StateVoicesNational/re/fix-exports
Browse files Browse the repository at this point in the history 
AWS-SDK: Fixing broken exports
  • Loading branch information
@engelhartrueben
engelhartrueben authored Aug 19, 2024
2 parents 50caf96 + be0bd30 commit 01d3bf2
Show file tree
Hide file tree
Showing 3 changed files with 117 additions and 49 deletions.
35 changes: 18 additions & 17 deletions src/containers/AdminCampaignStats.jsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -346,8 +346,8 @@ class AdminCampaignStats extends React.Component {
{campaign.exportResults.error && (
<div>Export failed: {campaign.exportResults.error}</div>
)}
{campaign.exportResults.campaignExportUrl &&
campaign.exportResults.campaignExportUrl.startsWith("http") ? (
{campaign.exportResults.campaignExportUrl && (
(campaign.exportResults.campaignExportUrl.startsWith("http")) ? (
<div>
Most recent export:
<a href={campaign.exportResults.campaignExportUrl} download>
Expand All @@ -360,15 +360,16 @@ class AdminCampaignStats extends React.Component {
Messages Export CSV
</a>
</div>
) : (
<div>
Local export was successful, saved on the server at:
<br />
{campaign.exportResults.campaignExportUrl}
<br />
{campaign.exportResults.campaignMessagesExportUrl}
</div>
)}
) : (campaign.exportResults.campaignExportUrl.startsWith("file://") && (
<div>
Local export was successful, saved on the server at:
<br />
{campaign.exportResults.campaignExportUrl}
<br />
{campaign.exportResults.campaignMessagesExportUrl}
</div>
)
))}
</div>
)}
{campaign.joinToken && campaign.useDynamicAssignment && (
Expand Down Expand Up @@ -424,21 +425,21 @@ class AdminCampaignStats extends React.Component {
message={
<span>
Export started -
{this.props.organizationData &&
this.props.organizationData.emailEnabled &&
" we'll e-mail you when it's done. "}
{campaign.cacheable && (
{(this.props.organizationData &&
this.props.organizationData.organization.emailEnabled) ?
" we'll e-mail you when it's done. " :
(campaign.cacheable && (
<span>
<Link
onClick={() => {
this.props.data.refetch();
}}
>
Reload the page
{" Reload the page"} {/*Hacky way to add a space at the beginning */}
</Link>{" "}
to see a download link when its ready.
</span>
)}
))}
</span>
}
autoHideDuration={campaign.cacheable ? null : 5000}
Expand Down
2 changes: 1 addition & 1 deletion src/server/models/cacheable_queries/campaign.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -254,7 +254,7 @@ const campaignCache = {
await r.redis
.MULTI()
.SET(exportCacheKey, JSON.stringify(data))
.EXPIRE(exportCacheKey, 43200)
.EXPIRE(exportCacheKey, 86400)
.exec();
}
},
Expand Down
129 changes: 98 additions & 31 deletions src/workers/jobs.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,14 @@ import { rawIngestMethod } from "../extensions/contact-loaders";

import { Lambda } from "@aws-sdk/client-lambda";
import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
import { GetObjectCommand, S3 } from "@aws-sdk/client-s3";
import {
CreateBucketCommand,
HeadBucketCommand,
GetObjectCommand,
waitUntilBucketExists,
S3Client,
PutObjectCommand
} from "@aws-sdk/client-s3";
import { SQS } from "@aws-sdk/client-sqs";
import Papa from "papaparse";
import moment from "moment";
Expand Down Expand Up @@ -861,46 +868,106 @@ export async function exportCampaign(job) {
(process.env.AWS_ACCESS_KEY_ID && process.env.AWS_SECRET_ACCESS_KEY)
) {
try {
const s3bucket = new S3({
// The transformation for params is not implemented.
// Refer to UPGRADING.md on aws-sdk-js-v3 for changes needed.
// Please create/upvote feature request on aws-sdk-js-codemod for params.
params: { Bucket: process.env.AWS_S3_BUCKET_NAME }
const client = new S3Client({
region: process.env.AWS_REGION
});
const bucketName = process.env.AWS_S3_BUCKET_NAME;

try {
// Check if the S3 bucket already exists
const verifyBucketCommand = new HeadBucketCommand({
Bucket: bucketName
});
await client.send(verifyBucketCommand);

console.log(`S3 bucket "${bucketName}" already exists.`);
} catch (error) {
if (error.name === "NotFound") {
console.log(
`S3 bucket "${bucketName}" not found. Creating a new bucket.`
);

try {
// Create the S3 bucket
const createBucketCommand = new CreateBucketCommand({
Bucket: bucketName
});
await client.send(createBucketCommand);

console.log(`S3 bucket "${bucketName}" created successfully.`);
} catch (createError) {
console.error(
`Error creating bucket "${bucketName}":`,
createError
);
}
} else {
console.error("Error checking bucket existence:", error);
}
}

// verifies that the bucket exists before moving forward
// if for some reason this fails, Spoke defensively deletes the job
await waitUntilBucketExists(
{ client, maxWaitTime: 15 },
{ Bucket: bucketName }
);

const campaignTitle = campaign.title
.replace(/ /g, "_")
.replace(/\//g, "_");
const key = `${campaignTitle}-${moment().format(
"YYYY-MM-DD-HH-mm-ss"
)}.csv`;
const messageKey = `${key}-messages.csv`;
let params = { Key: key, Body: campaignCsv };
await s3bucket.putObject(params);
params = { Key: key, Expires: 86400 };
const campaignExportUrl = await await getSignedUrl(s3bucket, new GetObjectCommand(params), {
expiresIn: "/* add value from 'Expires' from v2 call if present, else remove */"
});
params = { Key: messageKey, Body: messageCsv };
await s3bucket.putObject(params);
params = { Key: messageKey, Expires: 86400 };
const campaignMessagesExportUrl = await await getSignedUrl(s3bucket, new GetObjectCommand(params), {
expiresIn: "/* add value from 'Expires' from v2 call if present, else remove */"
});
let params = { Key: key,
Body: campaignCsv,
Bucket: bucketName };
await client.send(new PutObjectCommand(params));
params = { Key: key,
Expires: 86400,
Bucket: bucketName };
const campaignExportUrl = await getSignedUrl(client, new GetObjectCommand(params));
params = { Key: messageKey,
Body: messageCsv,
Bucket: bucketName };
await client.send(new PutObjectCommand(params));
params = { Key: messageKey,
Expires: 86400,
Bucket: bucketName };
const campaignMessagesExportUrl = await getSignedUrl(client, new GetObjectCommand(params));
exportResults.campaignExportUrl = campaignExportUrl;
exportResults.campaignMessagesExportUrl = campaignMessagesExportUrl;

await sendEmail({
to: user.email,
subject: `Export ready for ${campaign.title}`,
text: `Your Spoke exports are ready! These URLs will be valid for 24 hours.
Campaign export: ${campaignExportUrl}
Message export: ${campaignMessagesExportUrl}`
}).catch(err => {
log.error(err);
log.info(`Campaign Export URL - ${campaignExportUrl}`);
log.info(`Campaign Messages Export URL - ${campaignMessagesExportUrl}`);
});
log.info(`Successfully exported ${id}`);
// extreme check on email set-up
if ((
process.env.EMAIL_FROM &&
process.env.EMAIL_HOST &&
process.env.EMAIL_HOST_PASSWORD &&
process.env.EMAIL_HOST_PORT &&
process.env.EMAIL_HOST_USER) ||
(
process.env.MAILGUN_DOMAIN &&
process.env.MAILGUN_SMTP_LOGIN &&
process.env.MAILGUN_SMTP_PASSWORD &&
process.env.MAILGUN_SMTP_PORT &&
process.env.MAILGUN_SMTP_SERVER &&
process.env.MAILGUN_PUBLIC_KEY
)
) {
await sendEmail({
to: user.email,
subject: `Export ready for ${campaign.title}`,
text: `Your Spoke exports are ready! These URLs will be valid for 24 hours.
Campaign export: ${campaignExportUrl}
Message export: ${campaignMessagesExportUrl}`
}).catch(err => {
log.error(err);
log.info(`Campaign Export URL - ${campaignExportUrl}`);
log.info(`Campaign Messages Export URL - ${campaignMessagesExportUrl}`);
});
log.info(`Successfully exported ${id}`);
}
} catch (err) {
log.error(err);
exportResults.error = err.message;
Expand All @@ -927,7 +994,7 @@ export async function exportCampaign(job) {
log.debug(campaignCsv);
log.debug(messageCsv);
}
if (exportResults.campaignExportUrl) {
if (exportResults.campaignExportUrl || exportResults.error) {
exportResults.createdAt = String(new Date());
await cacheableData.campaign.saveExportData(campaign.id, exportResults);
}
Expand Down

0 comments on commit 01d3bf2

Please sign in to comment.