Refactor and migrate bulkupload to bulkuploadtask

This commit refactors all references of BulkUpload (and derivations) to BulkUploadTasks, along with refactoring the bulkUpload routes into a new task router, which is intended to contain all tasks moving forward.
PhilanthropyDataCommons · Nov 7, 2024 · c9ac193 · c9ac193
1 parent 76bd602
commit c9ac193
Show file tree

Hide file tree

Showing 35 changed files with 382 additions and 355 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -11,6 +11,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - Upgraded to use OpenAPI Specification 3.1.
 
+## 0.17.0 2024-11-07
+
+### Changed
+
+- `BulkUpload` is now `BulkUploadTask`.
+- `GET /bulkUpload` and `POST /bulkUpload` are now `GET /tasks/bulkUpload` and `POST /tasks/bulkUpload`.
+  Future jobs for the graphile-worker will be routed under `tasks` as well.
+
 ## 0.16.0 2024-11-7
 
 ### Added

diff --git a/src/__tests__/bulkUploads.int.test.ts → src/__tests__/bulkUploadTasks.int.test.ts b/src/__tests__/bulkUploads.int.test.ts → src/__tests__/bulkUploadTasks.int.test.ts
@@ -1,7 +1,7 @@
 import request from 'supertest';
 import { app } from '../app';
 import {
-	createBulkUpload,
+	createBulkUploadTask,
 	createUser,
 	loadSystemSource,
 	loadSystemUser,
@@ -13,66 +13,66 @@ import {
 	mockJwtWithoutSub as authHeaderWithNoSub,
 	mockJwtWithAdminRole as authHeaderWithAdminRole,
 } from '../test/mockJwt';
-import { BulkUploadStatus, keycloakUserIdToString } from '../types';
+import { TaskStatus, keycloakUserIdToString } from '../types';
 
-describe('/bulkUploads', () => {
+describe('/tasks/bulkUploads', () => {
 	describe('GET /', () => {
 		it('requires authentication', async () => {
-			await request(app).get('/bulkUploads').expect(401);
+			await request(app).get('/tasks/bulkUploads').expect(401);
 		});
 
 		it('requires a user', async () => {
 			await request(app)
-				.get('/bulkUploads')
+				.get('/tasks/bulkUploads')
 				.set(authHeaderWithNoSub)
 				.expect(401);
 		});
 
 		it('returns an empty Bundle when no data is present', async () => {
-			await request(app).get('/bulkUploads').set(authHeader).expect(200, {
+			await request(app).get('/tasks/bulkUploads').set(authHeader).expect(200, {
 				total: 0,
 				entries: [],
 			});
 		});
 
-		it('returns bulk uploads associated with the requesting user', async () => {
+		it('returns bulk upload tasks associated with the requesting user', async () => {
 			const systemUser = await loadSystemUser();
 			const systemSource = await loadSystemSource();
 			const testUser = await loadTestUser();
 			const thirdUser = await createUser({
 				keycloakUserId: '123e4567-e89b-12d3-a456-426614174000',
 			});
-			await createBulkUpload({
+			await createBulkUploadTask({
 				sourceId: systemSource.id,
 				fileName: 'foo.csv',
 				sourceKey: '96ddab90-1931-478d-8c02-a1dc80ae01e5-foo',
-				status: BulkUploadStatus.PENDING,
+				status: TaskStatus.PENDING,
 				createdBy: testUser.keycloakUserId,
 			});
-			await createBulkUpload({
+			await createBulkUploadTask({
 				sourceId: systemSource.id,
 				fileName: 'bar.csv',
 				sourceKey: '96ddab90-1931-478d-8c02-a1dc80ae01e5-bar',
-				status: BulkUploadStatus.COMPLETED,
+				status: TaskStatus.COMPLETED,
 				createdBy: testUser.keycloakUserId,
 			});
-			await createBulkUpload({
+			await createBulkUploadTask({
 				sourceId: systemSource.id,
 				fileName: 'baz.csv',
 				sourceKey: '96ddab90-1931-478d-8c02-a1dc80ae01e5-baz',
-				status: BulkUploadStatus.COMPLETED,
+				status: TaskStatus.COMPLETED,
 				createdBy: systemUser.keycloakUserId,
 			});
-			await createBulkUpload({
+			await createBulkUploadTask({
 				sourceId: systemSource.id,
 				fileName: 'boop.csv',
 				sourceKey: '96ddab90-1931-478d-8c02-a1dc80ae01e5-boop',
-				status: BulkUploadStatus.COMPLETED,
+				status: TaskStatus.COMPLETED,
 				createdBy: thirdUser.keycloakUserId,
 			});
 
 			await request(app)
-				.get('/bulkUploads')
+				.get('/tasks/bulkUploads')
 				.set(authHeader)
 				.expect(200)
 				.expect((res) =>
@@ -86,7 +86,7 @@ describe('/bulkUploads', () => {
 								fileName: 'bar.csv',
 								fileSize: null,
 								sourceKey: '96ddab90-1931-478d-8c02-a1dc80ae01e5-bar',
-								status: BulkUploadStatus.COMPLETED,
+								status: TaskStatus.COMPLETED,
 								createdAt: expectTimestamp,
 								createdBy: testUser.keycloakUserId,
 							},
@@ -97,7 +97,7 @@ describe('/bulkUploads', () => {
 								fileName: 'foo.csv',
 								fileSize: null,
 								sourceKey: '96ddab90-1931-478d-8c02-a1dc80ae01e5-foo',
-								status: BulkUploadStatus.PENDING,
+								status: TaskStatus.PENDING,
 								createdAt: expectTimestamp,
 								createdBy: testUser.keycloakUserId,
 							},
@@ -112,23 +112,23 @@ describe('/bulkUploads', () => {
 			const anotherUser = await createUser({
 				keycloakUserId: '123e4567-e89b-12d3-a456-426614174000',
 			});
-			await createBulkUpload({
+			await createBulkUploadTask({
 				sourceId: systemSource.id,
 				fileName: 'foo.csv',
 				sourceKey: '96ddab90-1931-478d-8c02-a1dc80ae01e5-foo',
-				status: BulkUploadStatus.PENDING,
+				status: TaskStatus.PENDING,
 				createdBy: testUser.keycloakUserId,
 			});
-			await createBulkUpload({
+			await createBulkUploadTask({
 				sourceId: systemSource.id,
 				fileName: 'bar.csv',
 				sourceKey: '96ddab90-1931-478d-8c02-a1dc80ae01e5-bar',
-				status: BulkUploadStatus.COMPLETED,
+				status: TaskStatus.COMPLETED,
 				createdBy: anotherUser.keycloakUserId,
 			});
 
 			await request(app)
-				.get('/bulkUploads')
+				.get('/tasks/bulkUploads')
 				.set(authHeaderWithAdminRole)
 				.expect(200)
 				.expect((res) =>
@@ -142,7 +142,7 @@ describe('/bulkUploads', () => {
 								fileName: 'bar.csv',
 								fileSize: null,
 								sourceKey: '96ddab90-1931-478d-8c02-a1dc80ae01e5-bar',
-								status: BulkUploadStatus.COMPLETED,
+								status: TaskStatus.COMPLETED,
 								createdAt: expectTimestamp,
 								createdBy: anotherUser.keycloakUserId,
 							},
@@ -153,7 +153,7 @@ describe('/bulkUploads', () => {
 								fileName: 'foo.csv',
 								fileSize: null,
 								sourceKey: '96ddab90-1931-478d-8c02-a1dc80ae01e5-foo',
-								status: BulkUploadStatus.PENDING,
+								status: TaskStatus.PENDING,
 								createdAt: expectTimestamp,
 								createdBy: testUser.keycloakUserId,
 							},
@@ -162,30 +162,30 @@ describe('/bulkUploads', () => {
 				);
 		});
 
-		it('returns uploads for specified createdBy user', async () => {
+		it('returns upload tasks for specified createdBy user', async () => {
 			const systemSource = await loadSystemSource();
 			const testUser = await loadTestUser();
 			const anotherUser = await createUser({
 				keycloakUserId: '123e4567-e89b-12d3-a456-426614174000',
 			});
-			await createBulkUpload({
+			await createBulkUploadTask({
 				sourceId: systemSource.id,
 				fileName: 'foo.csv',
 				sourceKey: '96ddab90-1931-478d-8c02-a1dc80ae01e5-foo',
-				status: BulkUploadStatus.PENDING,
+				status: TaskStatus.PENDING,
 				createdBy: testUser.keycloakUserId,
 			});
-			await createBulkUpload({
+			await createBulkUploadTask({
 				sourceId: systemSource.id,
 				fileName: 'bar.csv',
 				sourceKey: '96ddab90-1931-478d-8c02-a1dc80ae01e5-bar',
-				status: BulkUploadStatus.COMPLETED,
+				status: TaskStatus.COMPLETED,
 				createdBy: anotherUser.keycloakUserId,
 			});
 
 			await request(app)
 				.get(
-					`/bulkUploads?createdBy=${keycloakUserIdToString(anotherUser.keycloakUserId)}`,
+					`/tasks/bulkUploads?createdBy=${keycloakUserIdToString(anotherUser.keycloakUserId)}`,
 				)
 				.set(authHeaderWithAdminRole)
 				.expect(200)
@@ -200,7 +200,7 @@ describe('/bulkUploads', () => {
 								fileName: 'bar.csv',
 								fileSize: null,
 								sourceKey: '96ddab90-1931-478d-8c02-a1dc80ae01e5-bar',
-								status: BulkUploadStatus.COMPLETED,
+								status: TaskStatus.COMPLETED,
 								createdAt: expectTimestamp,
 								createdBy: anotherUser.keycloakUserId,
 							},
@@ -209,29 +209,29 @@ describe('/bulkUploads', () => {
 				);
 		});
 
-		it('returns uploads for the admin user when createdBy is set to me as an admin', async () => {
+		it('returns upload tasks for the admin user when createdBy is set to me as an admin', async () => {
 			const systemSource = await loadSystemSource();
 			const testUser = await loadTestUser();
 			const anotherUser = await createUser({
 				keycloakUserId: '123e4567-e89b-12d3-a456-426614174000',
 			});
-			await createBulkUpload({
+			await createBulkUploadTask({
 				sourceId: systemSource.id,
 				fileName: 'foo.csv',
 				sourceKey: '96ddab90-1931-478d-8c02-a1dc80ae01e5-foo',
-				status: BulkUploadStatus.PENDING,
+				status: TaskStatus.PENDING,
 				createdBy: testUser.keycloakUserId,
 			});
-			await createBulkUpload({
+			await createBulkUploadTask({
 				sourceId: systemSource.id,
 				fileName: 'bar.csv',
 				sourceKey: '96ddab90-1931-478d-8c02-a1dc80ae01e5-bar',
-				status: BulkUploadStatus.COMPLETED,
+				status: TaskStatus.COMPLETED,
 				createdBy: anotherUser.keycloakUserId,
 			});
 
 			await request(app)
-				.get(`/bulkUploads?createdBy=me`)
+				.get(`/tasks/bulkUploads?createdBy=me`)
 				.set(authHeaderWithAdminRole)
 				.expect(200)
 				.expect((res) =>
@@ -245,7 +245,7 @@ describe('/bulkUploads', () => {
 								fileName: 'foo.csv',
 								fileSize: null,
 								sourceKey: '96ddab90-1931-478d-8c02-a1dc80ae01e5-foo',
-								status: BulkUploadStatus.PENDING,
+								status: TaskStatus.PENDING,
 								createdAt: expectTimestamp,
 								createdBy: testUser.keycloakUserId,
 							},
@@ -259,17 +259,17 @@ describe('/bulkUploads', () => {
 			const testUser = await loadTestUser();
 			await Array.from(Array(20)).reduce(async (p, _, i) => {
 				await p;
-				await createBulkUpload({
+				await createBulkUploadTask({
 					sourceId: systemSource.id,
 					fileName: `bar-${i + 1}.csv`,
 					sourceKey: 'unprocessed/96ddab90-1931-478d-8c02-a1dc80ae01e5-bar',
-					status: BulkUploadStatus.COMPLETED,
+					status: TaskStatus.COMPLETED,
 					createdBy: testUser.keycloakUserId,
 				});
 			}, Promise.resolve());
 
 			await request(app)
-				.get('/bulkUploads')
+				.get('/tasks/bulkUploads')
 				.query({
 					_page: 2,
 					_count: 5,
@@ -288,7 +288,7 @@ describe('/bulkUploads', () => {
 								fileSize: null,
 								sourceKey:
 									'unprocessed/96ddab90-1931-478d-8c02-a1dc80ae01e5-bar',
-								status: BulkUploadStatus.COMPLETED,
+								status: TaskStatus.COMPLETED,
 								createdAt: expectTimestamp,
 								createdBy: testUser.keycloakUserId,
 							},
@@ -300,7 +300,7 @@ describe('/bulkUploads', () => {
 								fileSize: null,
 								sourceKey:
 									'unprocessed/96ddab90-1931-478d-8c02-a1dc80ae01e5-bar',
-								status: BulkUploadStatus.COMPLETED,
+								status: TaskStatus.COMPLETED,
 								createdAt: expectTimestamp,
 								createdBy: testUser.keycloakUserId,
 							},
@@ -312,7 +312,7 @@ describe('/bulkUploads', () => {
 								fileSize: null,
 								sourceKey:
 									'unprocessed/96ddab90-1931-478d-8c02-a1dc80ae01e5-bar',
-								status: BulkUploadStatus.COMPLETED,
+								status: TaskStatus.COMPLETED,
 								createdAt: expectTimestamp,
 								createdBy: testUser.keycloakUserId,
 							},
@@ -324,7 +324,7 @@ describe('/bulkUploads', () => {
 								fileSize: null,
 								sourceKey:
 									'unprocessed/96ddab90-1931-478d-8c02-a1dc80ae01e5-bar',
-								status: BulkUploadStatus.COMPLETED,
+								status: TaskStatus.COMPLETED,
 								createdAt: expectTimestamp,
 								createdBy: testUser.keycloakUserId,
 							},
@@ -336,7 +336,7 @@ describe('/bulkUploads', () => {
 								fileSize: null,
 								sourceKey:
 									'unprocessed/96ddab90-1931-478d-8c02-a1dc80ae01e5-bar',
-								status: BulkUploadStatus.COMPLETED,
+								status: TaskStatus.COMPLETED,
 								createdAt: expectTimestamp,
 								createdBy: testUser.keycloakUserId,
 							},
@@ -348,21 +348,21 @@ describe('/bulkUploads', () => {
 
 	describe('POST /', () => {
 		it('requires authentication', async () => {
-			await request(app).post('/bulkUploads').expect(401);
+			await request(app).post('/tasks/bulkUploads/').expect(401);
 		});
 
 		it('requires a user', async () => {
 			await request(app)
-				.post('/bulkUploads')
+				.post('/tasks/bulkUploads/')
 				.set(authHeaderWithNoSub)
 				.expect(401);
 		});
 
-		it('creates exactly one bulk upload', async () => {
+		it('creates exactly one bulk upload task', async () => {
 			const systemSource = await loadSystemSource();
-			const before = await loadTableMetrics('bulk_uploads');
+			const before = await loadTableMetrics('bulk_upload_tasks');
 			const result = await request(app)
-				.post('/bulkUploads')
+				.post('/tasks/bulkUploads/')
 				.type('application/json')
 				.set(authHeader)
 				.send({
@@ -371,7 +371,7 @@ describe('/bulkUploads', () => {
 					sourceKey: 'unprocessed/96ddab90-1931-478d-8c02-a1dc80ae01e5-bar',
 				})
 				.expect(201);
-			const after = await loadTableMetrics('bulk_uploads');
+			const after = await loadTableMetrics('bulk_upload_tasks');
 			const testUser = await loadTestUser();
 
 			expect(before.count).toEqual(0);
@@ -392,7 +392,7 @@ describe('/bulkUploads', () => {
 		it('returns 400 bad request when no file name is provided', async () => {
 			const systemSource = await loadSystemSource();
 			const result = await request(app)
-				.post('/bulkUploads')
+				.post('/tasks/bulkUploads')
 				.type('application/json')
 				.set(authHeader)
 				.send({
@@ -409,7 +409,7 @@ describe('/bulkUploads', () => {
 		it('returns 400 bad request when an invalid file name is provided', async () => {
 			const systemSource = await loadSystemSource();
 			const result = await request(app)
-				.post('/bulkUploads')
+				.post('/tasks/bulkUploads')
 				.type('application/json')
 				.set(authHeader)
 				.send({
@@ -426,7 +426,7 @@ describe('/bulkUploads', () => {
 
 		it('returns 400 bad request when an invalid source key is provided', async () => {
 			const result = await request(app)
-				.post('/bulkUploads')
+				.post('/tasks/bulkUploads')
 				.type('application/json')
 				.set(authHeader)
 				.send({
@@ -442,7 +442,7 @@ describe('/bulkUploads', () => {
 
 		it('returns 400 bad request when no source key is provided', async () => {
 			const result = await request(app)
-				.post('/bulkUploads')
+				.post('/tasks/bulkUploads')
 				.type('application/json')
 				.set(authHeader)
 				.send({