Skip to content

Commit

Permalink
Add software entity
Browse files Browse the repository at this point in the history
  • Loading branch information
siiptuo committed Aug 22, 2023
1 parent 8c915fd commit 43a988f
Show file tree
Hide file tree
Showing 11 changed files with 298 additions and 25 deletions.
1 change: 1 addition & 0 deletions backend/fixtures/0-software.json
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
[{ "id": 1, "name": "cloudnetpy", "version": "1.0.4", "url": null }]
48 changes: 32 additions & 16 deletions backend/fixtures/2-regular_file.json
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,8 @@
"format": "HDF5 (NetCDF4)",
"site": "mace-head",
"version": "123",
"errorLevel": null
"errorLevel": null,
"software": [{ "id": 1 }]
},
{
"uuid": "bde7a35f-03aa-4bff-acfb-b4974ea9f217",
Expand All @@ -31,7 +32,8 @@
"format": "HDF5 (NetCDF4)",
"site": "mace-head",
"version": "123",
"errorLevel": null
"errorLevel": null,
"software": [{ "id": 1 }]
},
{
"uuid": "d21d6a9b-6804-4465-a026-74ec429fe17d",
Expand All @@ -48,7 +50,8 @@
"format": "HDF5 (NetCDF4)",
"site": "hyytiala",
"version": "123",
"errorLevel": null
"errorLevel": null,
"software": [{ "id": 1 }]
},
{
"uuid": "22b32746-faf0-4057-9076-ed2e698dcc34",
Expand All @@ -65,7 +68,8 @@
"format": "HDF5 (NetCDF4)",
"site": "bucharest",
"version": "123",
"errorLevel": null
"errorLevel": null,
"software": [{ "id": 1 }]
},
{
"uuid": "6cb32746-faf0-4057-9076-ed2e698dcf36",
Expand All @@ -82,7 +86,8 @@
"format": "HDF5 (NetCDF4)",
"site": "bucharest",
"version": "123",
"errorLevel": null
"errorLevel": null,
"software": [{ "id": 1 }]
},
{
"uuid": "8bb32746-faf0-4057-9076-ed2e698dcf36",
Expand All @@ -99,7 +104,8 @@
"format": "HDF5 (NetCDF4)",
"site": "bucharest",
"version": "123",
"errorLevel": null
"errorLevel": null,
"software": [{ "id": 1 }]
},
{
"uuid": "1bb32746-faf0-4057-9076-ed2e698dcf36",
Expand All @@ -121,7 +127,8 @@
"a45a2e9a-e39d-4af2-9798-5ea0fadf041e"
],
"version": "123",
"errorLevel": null
"errorLevel": null,
"software": [{ "id": 1 }]
},
{
"uuid": "2bb32746-faf0-4057-9076-ed2e698dcf36",
Expand All @@ -139,7 +146,8 @@
"site": "bucharest",
"sourceFileIds": ["1bb32746-faf0-4057-9076-ed2e698dcf36"],
"version": "123",
"errorLevel": "pass"
"errorLevel": "pass",
"software": [{ "id": 1 }]
},
{
"uuid": "3bb32746-faf0-4057-9076-ed2e698dcf36",
Expand All @@ -157,7 +165,8 @@
"format": "HDF5 (NetCDF4)",
"site": "bucharest",
"version": "123",
"errorLevel": null
"errorLevel": null,
"software": [{ "id": 1 }]
},
{
"uuid": "52b32746-faf0-4057-9076-ed2e698dcc34",
Expand All @@ -175,7 +184,8 @@
"format": "HDF5 (NetCDF4)",
"site": "bucharest",
"version": "123",
"errorLevel": null
"errorLevel": null,
"software": [{ "id": 1 }]
},
{
"uuid": "62b32746-faf0-4057-9076-ed2e698dcc34",
Expand All @@ -193,7 +203,8 @@
"format": "HDF5 (NetCDF4)",
"site": "bucharest",
"version": "123",
"errorLevel": "pass"
"errorLevel": "pass",
"software": [{ "id": 1 }]
},
{
"uuid": "72b32746-faf0-4057-9076-ed2e698dcc34",
Expand All @@ -211,7 +222,8 @@
"format": "HDF5 (NetCDF4)",
"site": "bucharest",
"version": "123",
"errorLevel": null
"errorLevel": null,
"software": [{ "id": 1 }]
},
{
"uuid": "82b32746-faf0-4057-9076-ed2e698dcc34",
Expand All @@ -229,7 +241,8 @@
"format": "HDF5 (NetCDF4)",
"site": "bucharest",
"version": "123",
"errorLevel": null
"errorLevel": null,
"software": [{ "id": 1 }]
},
{
"uuid": "acf78456-11b1-41a6-b2de-aa7590a75675",
Expand All @@ -248,7 +261,8 @@
"site": "bucharest",
"version": "123",
"quality": "qc",
"errorLevel": "error"
"errorLevel": "error",
"software": [{ "id": 1 }]
},
{
"uuid": "b6de8cf4-8825-47b0-aaa9-4fd413bbb0d7",
Expand All @@ -266,7 +280,8 @@
"site": "bucharest",
"version": "123",
"errorLevel": null,
"instrumentPid": "https://hdl.handle.net/123/bucharest_lidar"
"instrumentPid": "https://hdl.handle.net/123/bucharest_lidar",
"software": [{ "id": 1 }]
},
{
"uuid": "f036da43-c19c-4832-99f9-6cc88f3255c5",
Expand All @@ -284,6 +299,7 @@
"site": "bucharest",
"version": "123",
"errorLevel": null,
"instrumentPid": "https://hdl.handle.net/123/bucharest_radar"
"instrumentPid": "https://hdl.handle.net/123/bucharest_radar",
"software": [{ "id": 1 }]
}
]
7 changes: 7 additions & 0 deletions backend/src/entity/File.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ import {
Column,
Entity,
Index,
JoinTable,
ManyToMany,
ManyToOne,
OneToMany,
PrimaryColumn,
Expand All @@ -17,6 +19,7 @@ import { basename } from "path";
import { Model } from "./Model";
import { ModelVisualization } from "./ModelVisualization";
import { ErrorLevel } from "./QualityReport";
import { Software } from "./Software";

export enum Quality {
NRT = "nrt",
Expand Down Expand Up @@ -82,6 +85,10 @@ export abstract class File {
@Column({ default: "" })
processingVersion!: string;

@ManyToMany(() => Software)
@JoinTable()
software!: Software[];

get filename() {
return basename(this.s3key);
}
Expand Down
17 changes: 17 additions & 0 deletions backend/src/entity/Software.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
import { Column, Entity, Unique, PrimaryGeneratedColumn } from "typeorm";

@Entity()
@Unique(["name", "version"])
export class Software {
@PrimaryGeneratedColumn()
id?: number;

@Column()
name!: string;

@Column()
version!: string;

@Column({ type: "text", nullable: true })
url!: string | null;
}
3 changes: 3 additions & 0 deletions backend/src/lib/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,9 @@ export const augmentFile = (includeS3path: boolean) => (file: RegularFile | Mode
s3key: undefined,
s3path: includeS3path ? getS3pathForFile(file) : undefined,
model: "model" in file ? file.model : undefined,
software: file.software
? file.software.map((software) => ({ name: software.name, version: software.version, url: software.url }))
: undefined,
});

export const ssAuthString = () =>
Expand Down
81 changes: 81 additions & 0 deletions backend/src/migration/1692622438290-AddSoftware.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
import { MigrationInterface, QueryRunner } from "typeorm";

export class AddSoftware1692622438290 implements MigrationInterface {
name = "AddSoftware1692622438290";

public async up(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(
`CREATE TABLE "software" ("id" SERIAL NOT NULL, "name" character varying NOT NULL, "version" character varying NOT NULL, "url" text, CONSTRAINT "PK_3ceec82cc90b32643b07e8d9841" PRIMARY KEY ("id"))`
);
await queryRunner.query(
`CREATE TABLE "file_software_software" ("fileUuid" uuid NOT NULL, "softwareId" integer NOT NULL, CONSTRAINT "PK_46858ee5fd23ddd01a7370e0d4c" PRIMARY KEY ("fileUuid", "softwareId"))`
);
await queryRunner.query(`CREATE INDEX "IDX_d3e4502978abd25bdc51e42c60" ON "file_software_software" ("fileUuid") `);
await queryRunner.query(
`CREATE INDEX "IDX_94fcf91af85f113565d9d82ef0" ON "file_software_software" ("softwareId") `
);
await queryRunner.query(
`CREATE TABLE "regular_file_software_software" ("regularFileUuid" uuid NOT NULL, "softwareId" integer NOT NULL, CONSTRAINT "PK_beb1f28e62cd9e70183524086df" PRIMARY KEY ("regularFileUuid", "softwareId"))`
);
await queryRunner.query(
`CREATE INDEX "IDX_80069d8fc68b7b591221745c1e" ON "regular_file_software_software" ("regularFileUuid") `
);
await queryRunner.query(
`CREATE INDEX "IDX_bcbd69960eebffb7658aec8b94" ON "regular_file_software_software" ("softwareId") `
);
await queryRunner.query(
`CREATE TABLE "model_file_software_software" ("modelFileUuid" uuid NOT NULL, "softwareId" integer NOT NULL, CONSTRAINT "PK_aac272d389dd8c007c948ce991a" PRIMARY KEY ("modelFileUuid", "softwareId"))`
);
await queryRunner.query(
`CREATE INDEX "IDX_53d4543423792e10d0c9859b07" ON "model_file_software_software" ("modelFileUuid") `
);
await queryRunner.query(
`CREATE INDEX "IDX_64ecef712f34a271013c99768f" ON "model_file_software_software" ("softwareId") `
);
await queryRunner.query(
`ALTER TABLE "file_software_software" ADD CONSTRAINT "FK_d3e4502978abd25bdc51e42c604" FOREIGN KEY ("fileUuid") REFERENCES "file"("uuid") ON DELETE CASCADE ON UPDATE CASCADE`
);
await queryRunner.query(
`ALTER TABLE "file_software_software" ADD CONSTRAINT "FK_94fcf91af85f113565d9d82ef02" FOREIGN KEY ("softwareId") REFERENCES "software"("id") ON DELETE CASCADE ON UPDATE CASCADE`
);
await queryRunner.query(
`ALTER TABLE "regular_file_software_software" ADD CONSTRAINT "FK_80069d8fc68b7b591221745c1e4" FOREIGN KEY ("regularFileUuid") REFERENCES "regular_file"("uuid") ON DELETE CASCADE ON UPDATE CASCADE`
);
await queryRunner.query(
`ALTER TABLE "regular_file_software_software" ADD CONSTRAINT "FK_bcbd69960eebffb7658aec8b94a" FOREIGN KEY ("softwareId") REFERENCES "software"("id") ON DELETE CASCADE ON UPDATE CASCADE`
);
await queryRunner.query(
`ALTER TABLE "model_file_software_software" ADD CONSTRAINT "FK_53d4543423792e10d0c9859b077" FOREIGN KEY ("modelFileUuid") REFERENCES "model_file"("uuid") ON DELETE CASCADE ON UPDATE CASCADE`
);
await queryRunner.query(
`ALTER TABLE "model_file_software_software" ADD CONSTRAINT "FK_64ecef712f34a271013c99768f3" FOREIGN KEY ("softwareId") REFERENCES "software"("id") ON DELETE CASCADE ON UPDATE CASCADE`
);
}

public async down(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(
`ALTER TABLE "model_file_software_software" DROP CONSTRAINT "FK_64ecef712f34a271013c99768f3"`
);
await queryRunner.query(
`ALTER TABLE "model_file_software_software" DROP CONSTRAINT "FK_53d4543423792e10d0c9859b077"`
);
await queryRunner.query(
`ALTER TABLE "regular_file_software_software" DROP CONSTRAINT "FK_bcbd69960eebffb7658aec8b94a"`
);
await queryRunner.query(
`ALTER TABLE "regular_file_software_software" DROP CONSTRAINT "FK_80069d8fc68b7b591221745c1e4"`
);
await queryRunner.query(`ALTER TABLE "file_software_software" DROP CONSTRAINT "FK_94fcf91af85f113565d9d82ef02"`);
await queryRunner.query(`ALTER TABLE "file_software_software" DROP CONSTRAINT "FK_d3e4502978abd25bdc51e42c604"`);
await queryRunner.query(`DROP INDEX "public"."IDX_64ecef712f34a271013c99768f"`);
await queryRunner.query(`DROP INDEX "public"."IDX_53d4543423792e10d0c9859b07"`);
await queryRunner.query(`DROP TABLE "model_file_software_software"`);
await queryRunner.query(`DROP INDEX "public"."IDX_bcbd69960eebffb7658aec8b94"`);
await queryRunner.query(`DROP INDEX "public"."IDX_80069d8fc68b7b591221745c1e"`);
await queryRunner.query(`DROP TABLE "regular_file_software_software"`);
await queryRunner.query(`DROP INDEX "public"."IDX_94fcf91af85f113565d9d82ef0"`);
await queryRunner.query(`DROP INDEX "public"."IDX_d3e4502978abd25bdc51e42c60"`);
await queryRunner.query(`DROP TABLE "file_software_software"`);
await queryRunner.query(`DROP TABLE "software"`);
}
}
30 changes: 26 additions & 4 deletions backend/src/routes/file.ts
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import { SearchFileResponse } from "../entity/SearchFileResponse";
import { Visualization } from "../entity/Visualization";
import { ModelVisualization } from "../entity/ModelVisualization";
import { Product } from "../entity/Product";
import { Software } from "../entity/Software";

export class FileRoutes {
constructor(conn: Connection) {
Expand All @@ -35,6 +36,7 @@ export class FileRoutes {
this.modelVisualizationRepo = conn.getRepository<ModelVisualization>("model_visualization");
this.productRepo = conn.getRepository<Product>("product");
this.fileQualityRepo = conn.getRepository<FileQuality>("file_quality");
this.softwareRepo = conn.getRepository<Software>("software");
}

readonly conn: Connection;
Expand All @@ -46,13 +48,15 @@ export class FileRoutes {
readonly modelVisualizationRepo: Repository<ModelVisualization>;
readonly productRepo: Repository<Product>;
readonly fileQualityRepo: Repository<FileQuality>;
readonly softwareRepo: Repository<Software>;

file: RequestHandler = async (req: Request, res: Response, next) => {
const getFileByUuid = (repo: Repository<RegularFile | ModelFile>, isModel: boolean | undefined) => {
const qb = repo
.createQueryBuilder("file")
.leftJoinAndSelect("file.site", "site")
.leftJoinAndSelect("file.product", "product");
.leftJoinAndSelect("file.product", "product")
.leftJoinAndSelect("file.software", "software");
if (isModel) qb.leftJoinAndSelect("file.model", "model");
qb.where("file.uuid = :uuid", req.params);
return hideTestDataFromNormalUsers<RegularFile | ModelFile>(qb, req).getOne();
Expand Down Expand Up @@ -127,6 +131,10 @@ export class FileRoutes {
return next({ status: 400, errors: ["The specified file was not found in storage service"] });
}

if (file.software) {
file.software = await this.getSoftware(file.software);
}

try {
const findFileByName = (model: boolean) => {
const repo = model ? this.modelFileRepo : this.fileRepo;
Expand Down Expand Up @@ -155,14 +163,14 @@ export class FileRoutes {
} else {
await transactionalEntityManager.insert(SearchFile, searchFile);
}
await transactionalEntityManager.insert(FileClass, file);
await transactionalEntityManager.save(FileClass, file);
});
res.sendStatus(201);
} else if (existingFile.site.isTestSite || existingFile.volatile) {
// Replace existing
file.createdAt = existingFile.createdAt;
await this.conn.transaction(async (transactionalEntityManager) => {
await transactionalEntityManager.update(FileClass, { uuid: file.uuid }, file);
await transactionalEntityManager.save(FileClass, file);
await transactionalEntityManager.update(SearchFile, { uuid: file.uuid }, searchFile);
});
res.sendStatus(200);
Expand All @@ -171,7 +179,7 @@ export class FileRoutes {
if (isModel) return next({ status: 501, errors: ["Versioning is not supported for model files."] });
await this.conn.transaction(async (transactionalEntityManager) => {
file.createdAt = file.updatedAt;
await transactionalEntityManager.insert(FileClass, file);
await transactionalEntityManager.save(FileClass, file);
if (!file.legacy) {
// Don't display legacy files in search if cloudnet version is available
await transactionalEntityManager.delete(SearchFile, { uuid: existingFile.uuid });
Expand All @@ -186,6 +194,8 @@ export class FileRoutes {
});
}
} catch (e) {
console.error("============== ERROR =================");
console.error(e);
next({ status: 500, errors: e });
}
};
Expand Down Expand Up @@ -417,6 +427,18 @@ export class FileRoutes {
.getRawMany();
return products.map((prod) => prod.id);
}

getSoftware(software: Record<string, string>): Promise<Software[]> {
const promises = Object.entries(software).map(async ([name, version]) => {
const query = { name, version: version as string };
let software = await this.softwareRepo.findOne(query);
if (!software) {
software = await this.softwareRepo.save(query);
}
return software;
});
return Promise.all(promises);
}
}

function addCommonFilters<T>(qb: SelectQueryBuilder<T>, query: any) {
Expand Down
Loading

0 comments on commit 43a988f

Please sign in to comment.