Skip to content

Commit

Permalink
Merge branch 'dev' into double-standards
Browse files Browse the repository at this point in the history
  • Loading branch information
mauberti-bc authored Aug 15, 2024
2 parents 2d8f3a4 + 27811fb commit c07bced
Show file tree
Hide file tree
Showing 36 changed files with 1,096 additions and 428 deletions.
44 changes: 22 additions & 22 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@

-include .env

# Apply the contents of the .env to the terminal, so that the docker-compose file can use them in its builds
# Apply the contents of the .env to the terminal, so that the compose file can use them in its builds
export $(shell sed 's/=.*//' .env)

## ------------------------------------------------------------------------------
Expand Down Expand Up @@ -46,13 +46,13 @@ close: ## Closes all project containers
@echo "==============================================="
@echo "Make: close - closing Docker containers"
@echo "==============================================="
@docker-compose -f docker-compose.yml down
@docker compose down

clean: ## Closes and cleans (removes) all project containers
@echo "==============================================="
@echo "Make: clean - closing and cleaning Docker containers"
@echo "==============================================="
@docker-compose -f docker-compose.yml down -v --rmi all --remove-orphans
@docker compose down -v --rmi all --remove-orphans

prune: ## Deletes ALL docker artifacts (even those not associated to this project)
@echo -n "Delete ALL docker artifacts? [y/n] " && read ans && [ $${ans:-n} = y ]
Expand All @@ -77,13 +77,13 @@ build-postgres: ## Builds the postgres db containers
@echo "==============================================="
@echo "Make: build-postgres - building postgres db images"
@echo "==============================================="
@docker-compose -f docker-compose.yml build db db_setup
@docker compose build db db_setup

run-postgres: ## Runs the postgres db containers
@echo "==============================================="
@echo "Make: run-postgres - running postgres db images"
@echo "==============================================="
@docker-compose -f docker-compose.yml up -d db db_setup
@docker compose up -d db db_setup

## ------------------------------------------------------------------------------
## Build/Run Backend Commands
Expand All @@ -94,13 +94,13 @@ build-backend: ## Builds all backend containers
@echo "==============================================="
@echo "Make: build-backend - building backend images"
@echo "==============================================="
@docker-compose -f docker-compose.yml build db db_setup api
@docker compose build db db_setup api

run-backend: ## Runs all backend containers
@echo "==============================================="
@echo "Make: run-backend - running backend images"
@echo "==============================================="
@docker-compose -f docker-compose.yml up -d db db_setup api
@docker compose up -d db db_setup api

## ------------------------------------------------------------------------------
## Build/Run Backend+Web Commands (backend + web frontend)
Expand All @@ -111,13 +111,13 @@ build-web: ## Builds all backend+web containers
@echo "==============================================="
@echo "Make: build-web - building web images"
@echo "==============================================="
@docker-compose -f docker-compose.yml build db db_setup api app
@docker compose build db db_setup api app

run-web: ## Runs all backend+web containers
@echo "==============================================="
@echo "Make: run-web - running web images"
@echo "==============================================="
@docker-compose -f docker-compose.yml up -d db db_setup api app
@docker compose up -d db db_setup api app

## ------------------------------------------------------------------------------
## Commands to shell into the target container
Expand All @@ -128,19 +128,19 @@ db-container: ## Executes into database container.
@echo "Make: Shelling into database container"
@echo "==============================================="
@export PGPASSWORD=$(DB_ADMIN_PASS)
@docker-compose exec db psql -U $(DB_ADMIN) -d $(DB_DATABASE)
@docker compose exec db psql -U $(DB_ADMIN) -d $(DB_DATABASE)

app-container: ## Executes into the app container.
@echo "==============================================="
@echo "Shelling into app container"
@echo "==============================================="
@docker-compose exec app bash
@docker compose exec app bash

api-container: ## Executes into the api container.
@echo "==============================================="
@echo "Shelling into api container"
@echo "==============================================="
@docker-compose exec api bash
@docker compose exec api bash

## ------------------------------------------------------------------------------
## Database migration commands
Expand All @@ -150,37 +150,37 @@ build-db-setup: ## Build the db knex setup (migrations + seeding) image
@echo "==============================================="
@echo "Make: build-db-setup - building db knex setup image"
@echo "==============================================="
@docker-compose -f docker-compose.yml build db_setup
@docker compose build db_setup

run-db-setup: ## Run the database migrations and seeding
@echo "==============================================="
@echo "Make: run-db-setup - running database migrations and seeding"
@echo "==============================================="
@docker-compose -f docker-compose.yml up db_setup
@docker compose up db_setup

build-db-migrate: ## Build the db knex migrations image
@echo "==============================================="
@echo "Make: build-db-migrate - building db knex migrate image"
@echo "==============================================="
@docker-compose -f docker-compose.yml build db_migrate
@docker compose build db_migrate

run-db-migrate: ## Run the database migrations
@echo "==============================================="
@echo "Make: run-db-migrate - running database migrations"
@echo "==============================================="
@docker-compose -f docker-compose.yml up db_migrate
@docker compose up db_migrate

build-db-rollback: ## Build the db knex rollback image
@echo "==============================================="
@echo "Make: build-db-rollback - building db knex rollback image"
@echo "==============================================="
@docker-compose -f docker-compose.yml build db_rollback
@docker compose build db_rollback

run-db-rollback: ## Rollback the latest database migrations
@echo "==============================================="
@echo "Make: run-db-rollback - rolling back the latest database migrations"
@echo "==============================================="
@docker-compose -f docker-compose.yml up db_rollback
@docker compose up db_rollback

## ------------------------------------------------------------------------------
## clamav commands
Expand All @@ -190,13 +190,13 @@ build-clamav: ## Build the clamav image
@echo "==============================================="
@echo "Make: build-clamav - building clamav image"
@echo "==============================================="
@docker-compose -f docker-compose.yml build clamav
@docker compose build clamav

run-clamav: ## Run clamav
@echo "==============================================="
@echo "Make: run-clamav - running clamav"
@echo "==============================================="
@docker-compose -f docker-compose.yml up -d clamav
@docker compose up -d clamav

## ------------------------------------------------------------------------------
## Run `npm` commands for all projects
Expand Down Expand Up @@ -314,11 +314,11 @@ pipeline-install: ## Runs `npm install` for all projects

args ?= --tail 2000 ## Default args if none are provided

log: ## Runs `docker-compose logs -f` for all containers
log: ## Runs `docker compose logs -f` for all containers
@echo "==============================================="
@echo "Running docker logs for the app container"
@echo "==============================================="
@docker-compose logs -f $(args)
@docker compose logs -f $(args)

log-app: ## Runs `docker logs <container> -f` for the app container
@echo "==============================================="
Expand Down
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,7 @@ Below are all of the relevant files that need to be updated when modifying envir
#### Local Development

- `env.docker`
- `docker-compose.yml`
- `compose.yml`
- `app/src/contexts/configContext.tsx`

#### Deployed to OpenShift
Expand Down
2 changes: 1 addition & 1 deletion api/.docker/api/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# ########################################################################################################
# This DockerFile is used for local development (via docker-compose) only.
# This DockerFile is used for local development (via compose.yml) only.
# ########################################################################################################

FROM node:20
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@ import { getDBConnection } from '../../../../../../../database/db';
import { HTTP400 } from '../../../../../../../errors/http-error';
import { csvFileSchema } from '../../../../../../../openapi/schemas/file';
import { authorizeRequestHandler } from '../../../../../../../request-handlers/security/authorization';
import { ImportCapturesService } from '../../../../../../../services/import-services/capture/import-captures-service';
import { importCSV } from '../../../../../../../services/import-services/csv-import-strategy';
import { ImportCapturesStrategy } from '../../../../../../../services/import-services/capture/import-captures-strategy';
import { importCSV } from '../../../../../../../services/import-services/import-csv';
import { scanFileForVirus } from '../../../../../../../utils/file-utils';
import { getLogger } from '../../../../../../../utils/logger';
import { parseMulterFile } from '../../../../../../../utils/media/media-utils';
Expand Down Expand Up @@ -142,7 +142,7 @@ export function importCsv(): RequestHandler {
throw new HTTP400('Malicious content detected, import cancelled.');
}

const importCsvCaptures = new ImportCapturesService(connection, surveyId);
const importCsvCaptures = new ImportCapturesStrategy(connection, surveyId);

// Pass CSV file and importer as dependencies
const capturesCreated = await importCSV(parseMulterFile(rawFile), importCsvCaptures);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import { expect } from 'chai';
import sinon from 'sinon';
import * as db from '../../../../../../database/db';
import { HTTP400 } from '../../../../../../errors/http-error';
import * as strategy from '../../../../../../services/import-services/csv-import-strategy';
import * as strategy from '../../../../../../services/import-services/import-csv';
import * as fileUtils from '../../../../../../utils/file-utils';
import { getMockDBConnection, getRequestHandlerMocks } from '../../../../../../__mocks__/db';
import { importCsv } from './import';
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@ import { getDBConnection } from '../../../../../../database/db';
import { HTTP400 } from '../../../../../../errors/http-error';
import { csvFileSchema } from '../../../../../../openapi/schemas/file';
import { authorizeRequestHandler } from '../../../../../../request-handlers/security/authorization';
import { ImportCrittersService } from '../../../../../../services/import-services/critter/import-critters-service';
import { importCSV } from '../../../../../../services/import-services/csv-import-strategy';
import { ImportCrittersStrategy } from '../../../../../../services/import-services/critter/import-critters-strategy';
import { importCSV } from '../../../../../../services/import-services/import-csv';
import { scanFileForVirus } from '../../../../../../utils/file-utils';
import { getLogger } from '../../../../../../utils/logger';
import { parseMulterFile } from '../../../../../../utils/media/media-utils';
Expand Down Expand Up @@ -144,8 +144,8 @@ export function importCsv(): RequestHandler {
throw new HTTP400('Malicious content detected, import cancelled.');
}

// Critter CSV import service - child of CSVImportStrategy
const importCsvCritters = new ImportCrittersService(connection, surveyId);
// Critter CSV import strategy - child of CSVImportStrategy
const importCsvCritters = new ImportCrittersStrategy(connection, surveyId);

const surveyCritterIds = await importCSV(parseMulterFile(rawFile), importCsvCritters);

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,161 @@
import { RequestHandler } from 'express';
import { Operation } from 'express-openapi';
import { PROJECT_PERMISSION, SYSTEM_ROLE } from '../../../../../../../constants/roles';
import { getDBConnection } from '../../../../../../../database/db';
import { HTTP400 } from '../../../../../../../errors/http-error';
import { csvFileSchema } from '../../../../../../../openapi/schemas/file';
import { authorizeRequestHandler } from '../../../../../../../request-handlers/security/authorization';
import { importCSV } from '../../../../../../../services/import-services/import-csv';
import { ImportMarkingsStrategy } from '../../../../../../../services/import-services/marking/import-markings-strategy';
import { scanFileForVirus } from '../../../../../../../utils/file-utils';
import { getLogger } from '../../../../../../../utils/logger';
import { parseMulterFile } from '../../../../../../../utils/media/media-utils';
import { getFileFromRequest } from '../../../../../../../utils/request';

const defaultLog = getLogger('/api/project/{projectId}/survey/{surveyId}/markings/import');

export const POST: Operation = [
authorizeRequestHandler((req) => {
return {
or: [
{
validProjectPermissions: [PROJECT_PERMISSION.COORDINATOR, PROJECT_PERMISSION.COLLABORATOR],
surveyId: Number(req.params.surveyId),
discriminator: 'ProjectPermission'
},
{
validSystemRoles: [SYSTEM_ROLE.DATA_ADMINISTRATOR],
discriminator: 'SystemRole'
}
]
};
}),
importCsv()
];

POST.apiDoc = {
description: 'Upload Critterbase CSV Markings file',
tags: ['observations'],
security: [
{
Bearer: []
}
],
parameters: [
{
in: 'path',
description: 'SIMS survey id',
name: 'projectId',
required: true,
schema: {
type: 'integer',
minimum: 1
}
},
{
in: 'path',
description: 'SIMS survey id',
name: 'surveyId',
required: true,
schema: {
type: 'integer',
minimum: 1
}
}
],
requestBody: {
description: 'Critterbase Markings CSV import file.',
content: {
'multipart/form-data': {
schema: {
type: 'object',
additionalProperties: false,
required: ['media'],
properties: {
media: {
description: 'Critterbase Markings CSV import file.',
type: 'array',
minItems: 1,
maxItems: 1,
items: csvFileSchema
}
}
}
}
}
},
responses: {
201: {
description: 'Marking import success.',
content: {
'application/json': {
schema: {
type: 'object',
additionalProperties: false,
properties: {
markingsCreated: {
description: 'Number of Critterbase markings created.',
type: 'integer'
}
}
}
}
}
},
400: {
$ref: '#/components/responses/400'
},
401: {
$ref: '#/components/responses/401'
},
403: {
$ref: '#/components/responses/403'
},
500: {
$ref: '#/components/responses/500'
},
default: {
$ref: '#/components/responses/default'
}
}
};

/**
* Imports a `Critterbase Marking CSV` which bulk adds markings to Critterbase.
*
* @return {*} {RequestHandler}
*/
export function importCsv(): RequestHandler {
return async (req, res) => {
const surveyId = Number(req.params.surveyId);
const rawFile = getFileFromRequest(req);

const connection = getDBConnection(req.keycloak_token);

try {
await connection.open();

// Check for viruses / malware
const virusScanResult = await scanFileForVirus(rawFile);

if (!virusScanResult) {
throw new HTTP400('Malicious content detected, import cancelled.');
}

const importCsvMarkingsStrategy = new ImportMarkingsStrategy(connection, surveyId);

// Pass CSV file and importer as dependencies
const markingsCreated = await importCSV(parseMulterFile(rawFile), importCsvMarkingsStrategy);

await connection.commit();

return res.status(201).json({ markingsCreated });
} catch (error) {
defaultLog.error({ label: 'importMarkingsCSV', message: 'error', error });
await connection.rollback();
throw error;
} finally {
connection.release();
}
};
}
Loading

0 comments on commit c07bced

Please sign in to comment.