Merge pull request #7 from icefoganalytics/main

Updates from IceFog

.vscode/extensions.json

@@ -7,7 +7,8 @@
     "vue.vscode-typescript-vue-plugin",
     "esbenp.prettier-vscode",
     "dbaeumer.vscode-eslint",
-    "jebbs.plantuml"
+    "jebbs.plantuml",
+    "thebearingedge.vscode-sql-lit"
   ],
   // List of extensions recommended by VS Code that should not be recommended for users of this workspace.
   "unwantedRecommendations": []

_Design/Entity Relationship Diagrams.wsd

@@ -3,9 +3,9 @@ entity "access_grants" {
   * id : int <<PK>>
   --
   * dataset_id : int <<FK>>
-  * owner_id : int <<FK>>
   * grant_level : nvarchar(255)
   * access_type : nvarchar(255)
+  creator_id : int <<FK>>
   support_id : int <<FK>>
   is_project_description_required : bit
   created_at : datetime2(0)
@@ -43,6 +43,27 @@ entity "dataset_fields" {
   * data_type : nvarchar(100)
   description : nvarchar(1000)
   note : nvarchar(MAX)
+  is_excluded_from_search : bit
+  created_at : datetime2(0)
+  updated_at : datetime2(0)
+  deleted_at : datetime2(0)
+}
+
+entity "dataset_integrations" {
+  * id : int <<PK>>
+  --
+  * dataset_id : int <<FK>>
+  * url : nvarchar(1024)
+  header_key : nvarchar(1024)
+  header_value : nvarchar(4000)
+  jmes_path_transform : nvarchar(1024)
+  raw_json_data : nvarchar(MAX)
+  parsed_json_data : nvarchar(MAX)
+  status : nvarchar(100)
+  error_code : nvarchar(100)
+  error_details : nvarchar(MAX)
+  last_success_at : datetime2(0)
+  last_failure_at : datetime2(0)
   created_at : datetime2(0)
   updated_at : datetime2(0)
   deleted_at : datetime2(0)
@@ -71,16 +92,11 @@ entity "datasets" {
   * slug : nvarchar(255)
   * name : nvarchar(255)
   * description : nvarchar(MAX)
-  subscription_url : nvarchar(1000)
-  subscription_access_code : nvarchar(255)
   is_spatial_data : bit
   is_live_data : bit
   terms_of_use : nvarchar(MAX)
   credits : nvarchar(MAX)
   owner_notes : nvarchar(MAX)
-  status : nvarchar(100)
-  error_code : nvarchar(100)
-  error_details : nvarchar(MAX)
   published_at : datetime2(0)
   deactivated_at : datetime2(0)
   created_at : datetime2(0)
@@ -122,7 +138,7 @@ entity "tags" {
 entity "user_group_memberships" {
   * id : int <<PK>>
   --
-  * userId : int <<FK>>
+  * user_id : int <<FK>>
   * department_id : int <<FK>>
   division_id : int <<FK>>
   branch_id : int <<FK>>
@@ -163,13 +179,29 @@ entity "users" {
   deleted_at : datetime2(0)
 }
 
+entity "visualization_controls" {
+  * id : int <<PK>>
+  --
+  * dataset_id : int <<FK>>
+  is_downloadable_as_csv : bit
+  has_search_row_limits : bit
+  search_row_limit_maximum : int
+  has_search_customizations : bit
+  has_fields_excluded_from_search : bit
+  created_at : datetime2(0)
+  updated_at : datetime2(0)
+  deleted_at : datetime2(0)
+}
+
 ' Define relationships
 access_grants }o--|| access_requests : access_grant_id
 datasets }o--|| access_grants : dataset_id
 datasets }o--|| dataset_fields : dataset_id
-datasets }o--|| taggings : taggable_id, tagging_type
+datasets }o--|| dataset_integrations : dataset_id
+datasets }o--|| taggings : taggable_id, tagging_type = 'Dataset'
 datasets |o--|| dataset_stewardships : dataset_id
 datasets }o--|| access_requests : dataset_id
+datasets }o--|| visualization_controls : dataset_id
 taggings ||--o{ tags : tag_id
 user_groups }o--|| dataset_stewardships : branch_id
 user_groups }o--|| dataset_stewardships : department_id
@@ -179,8 +211,8 @@ user_groups }o--|| user_group_memberships : branch_id
 user_groups }o--|| user_group_memberships : department_id
 user_groups }o--|| user_group_memberships : division_id
 user_groups }o--|| user_group_memberships : unit_id
-users }o--|| access_grants : owner_id
 users }o--|| access_grants : support_id
+users }o--|| access_grants : creator_id
 users }o--|| access_requests : requestor_id
 users }o--|| access_requests : denier_id
 users }o--|| access_requests : approver_id

api/package.json

@@ -20,14 +20,17 @@
     "@faker-js/faker": "^8.4.0",
     "axios": "^1.6.5",
     "cls-hooked": "^4.2.2",
+    "cookie-parser": "^1.4.6",
     "cors": "^2.8.5",
     "dotenv": "^16.3.1",
     "express": "^4.18.2",
     "express-jwt": "^8.4.1",
     "helmet": "^7.1.0",
+    "jmespath": "^0.16.0",
     "jwks-rsa": "^3.1.0",
     "lodash": "^4.17.21",
     "luxon": "^3.4.4",
+    "papaparse": "^5.4.1",
     "qs": "^6.11.2",
     "sequelize": "^6.35.2",
     "slugify": "^1.6.6",
@@ -36,11 +39,14 @@
   },
   "devDependencies": {
     "@types/cls-hooked": "^4.3.8",
+    "@types/cookie-parser": "^1.4.7",
     "@types/cors": "^2.8.17",
     "@types/express": "^4.17.21",
     "@types/jest": "^29.5.11",
+    "@types/jmespath": "^0.15.2",
     "@types/lodash": "^4.14.202",
     "@types/luxon": "^3.4.2",
+    "@types/papaparse": "^5.3.14",
     "@types/qs": "^6.9.11",
     "@types/supertest": "^6.0.2",
     "@typescript-eslint/eslint-plugin": "^6.21.0",

api/src/app.ts

@@ -1,4 +1,5 @@
 import express, { type Request, type Response } from "express"
+import cookieParser from "cookie-parser"
 import cors from "cors"
 import path from "path"
 import helmet from "helmet"
@@ -16,6 +17,7 @@ app.set("query parser", (params: string) => {
     strictNullHandling: true,
   })
 })
+app.use(cookieParser())
 app.use(express.json()) // for parsing application/json
 app.use(express.urlencoded({ extended: true })) // for parsing application/x-www-form-urlencoded
 app.use(

api/src/controllers/base-controller.ts

@@ -6,8 +6,14 @@ export type Actions = "index" | "show" | "new" | "edit" | "create" | "update" |
 
 type ControllerRequest = Request & {
   currentUser: User
+  format?: string
 }
 
+// Keep in sync with web/src/api/base-api.ts
+const MAX_PER_PAGE = 1000
+const MAX_PER_PAGE_EQUIVALENT = -1
+const DEFAULT_PER_PAGE = 10
+
 // See https://guides.rubyonrails.org/routing.html#crud-verbs-and-actions
 export class BaseController {
   protected request: ControllerRequest
@@ -61,23 +67,23 @@ export class BaseController {
     }
   }
 
-  index(): Promise<any> {
+  index(): Promise<unknown> {
     throw new Error("Not Implemented")
   }
 
-  create(): Promise<any> {
+  create(): Promise<unknown> {
     throw new Error("Not Implemented")
   }
 
-  show(): Promise<any> {
+  show(): Promise<unknown> {
     throw new Error("Not Implemented")
   }
 
-  update(): Promise<any> {
+  update(): Promise<unknown> {
     throw new Error("Not Implemented")
   }
 
-  destroy(): Promise<any> {
+  destroy(): Promise<unknown> {
     throw new Error("Not Implemented")
   }
 
@@ -91,6 +97,10 @@ export class BaseController {
     return this.request.currentUser
   }
 
+  get format() {
+    return this.request.format
+  }
+
   get params() {
     return this.request.params
   }
@@ -101,8 +111,8 @@ export class BaseController {
 
   get pagination() {
     const page = parseInt(this.query.page?.toString() || "") || 1
-    const perPage = parseInt(this.query.perPage?.toString() || "") || 10
-    const limit = Math.max(10, Math.min(perPage, 1000)) // restrict max limit to 1000 for safety
+    const perPage = parseInt(this.query.perPage?.toString() || "") || DEFAULT_PER_PAGE
+    const limit = this.determineLimit(perPage)
     const offset = (page - 1) * limit
     return {
       page,
@@ -111,6 +121,15 @@ export class BaseController {
       offset,
     }
   }
+
+  private determineLimit(perPage: number) {
+    if (perPage === MAX_PER_PAGE_EQUIVALENT) {
+      return MAX_PER_PAGE
+    }
+
+
+    return Math.min(perPage, MAX_PER_PAGE)
+  }
 }
 
 export default BaseController