Merge pull request #36 from datopian/feature/multipart-support

Multipart LFS support + refactor of LFS related code

.gitignore

@@ -2,6 +2,7 @@
 /node_modules
 
 # misc
+/.idea
 .DS_Store
 .env.local
 .env.development.local

lib/index.js

@@ -2,9 +2,7 @@ const frictionlessCkanMapper = require('frictionless-ckan-mapper-js')
 const axios = require('axios')
 
 const CkanAuthApi = require('./util/ckan-auth-api')
-const CkanUploadAPI = require('./util/ckan-upload-api')
-const ActionApi = require('./util/action-api')
-const Open = require('./file')
+const LfsClient = require('./lfs-client')
 
 const { camelToSnakeCase } = require('./util/general')
 
@@ -16,15 +14,15 @@ const { camelToSnakeCase } = require('./util/general')
  * @param {string} api
  */
 class Client {
-  constructor(apiKey, organizationId, datasetId, api, lfs) {
+  constructor(apiKey, organizationId, datasetId, api, lfsServerUrl) {
     this.apiKey = apiKey
     this.organizationId = organizationId
     this.datasetId = datasetId
     this.api = api
-    this.lfs = lfs
+    this.lfsServerUrl = lfsServerUrl
   }
 
-  async doBlobAuthz() {
+  async getUploadAuthToken() {
     // Create the scope to send to CkanAuthz
     let scope = [`obj:${this.organizationId}/${this.datasetId}/*:write`]
 
@@ -144,63 +142,31 @@ class Client {
   }
 
   /**
-   * The result of push blob method
-   * @typedef {Object} PushBlobResult
-   * @property {string} oid - oid
-   * @property {number} size - size of the file
-   * @property {string} name - resource name
-   * @property {boolean} success - Indicates whether the request was successful or not
-   * @property {boolean} fileExists - Indicates whether the resource exists or not
-   */
-  /**
-   * @param {Object} resource - This datajs resource. Please check https://github.com/datopian/data.js
-   * @param {function} onProgress a callback function to track the progress
-   * @return {Promise<PushBlobResult>} request result
+   * Upload a blob to storage
+   *
+   * This will return a promise that can be resolve to 'true' if the file has
+   * been uploaded successfully, or `false` if the file was not uploaded
+   * because it already exists in storage. Either case can be considered a
+   * success.
+   *
+   * Any kind of failure to upload will trigger an exception.
+   *
+   * @param {File} resource
+   * @param {CallableFunction} onProgress
+   * @returns {Promise<boolean>}
    */
   async pushBlob(resource, onProgress) {
     // Get the JWT token
-    const token = await this.doBlobAuthz()
-
-    // Given the JWT token and file size, will return signed URL, verify URL and JWT token
-    const lfs = await CkanAuthApi.requestFileUploadActions(
-      this.lfs,
-      token,
-      resource.descriptor.hash,
-      resource.size,
-      this.organizationId,
-      this.datasetId
-    )
-    const object = lfs.objects[0]
-    const result = {
-      oid: object.oid,
-      size: object.size,
-      name: resource.descriptor.name,
-      success: true,
-      fileExists: false,
-    }
+    const token = await this.getUploadAuthToken()
 
-    // Upload the file to cloud storage
-    if (object.actions) {
-      await CkanUploadAPI.pushDataToBlobStorage(
-        object.actions.upload,
-        resource,
-        onProgress
-      )
-      await CkanUploadAPI.verifyUpload(
-        lfs.objects[0].actions.verify,
-        resource.descriptor.hash,
-        resource.size
-      )
-      return result
-    } else {
-      // File is already in storage
-      result.fileExists = true
-      return result
-    }
+    const lfsClient = new LfsClient.GitLfsClient(this.lfsServerUrl, {
+      "Authorization": `Bearer ${token}`
+    })
+
+    return await lfsClient.upload(resource, this.organizationId, this.datasetId, onProgress);
   }
 }
 
 module.exports = {
   Client,
-  Open,
 }