Enable remote media fetching

.vscode/settings.json

@@ -8,7 +8,7 @@
     "vue"
   ],
   "editor.codeActionsOnSave": {
-    "source.fixAll.eslint": true
+    "source.fixAll.eslint": "explicit"
   },
   "i18n-ally.localesPaths": [
     "server/locales"

client/components/editor/editor-modal-media.vue

@@ -159,20 +159,26 @@
               v-toolbar.radius-7(:color='$vuetify.theme.dark ? `teal` : `teal lighten-5`', dense, flat)
                 v-icon.mr-3(:color='$vuetify.theme.dark ? `white` : `teal`') mdi-cloud-download
                 .body-2(:class='$vuetify.theme.dark ? `white--text` : `teal--text`') {{$t('editor:assets.fetchImage')}}
-                v-spacer
-                v-chip(label, color='white', small).teal--text coming soon
               v-text-field.mt-3(
                 v-model='remoteImageUrl'
                 outlined
                 color='teal'
                 single-line
                 placeholder='https://example.com/image.jpg'
+                :disabled='remoteImageLoading'
+                @keyup.enter='fetchRemoteAsset'
               )
             v-divider
             v-card-actions.pa-3
               .caption.grey--text.text-darken-2 Max 5 MB
               v-spacer
-              v-btn.px-4(color='teal', disabled) {{$t('common:actions.fetch')}}
+              v-btn.px-4(
+                color='teal'
+                dark
+                @click='fetchRemoteAsset'
+                :disabled='!remoteUrlIsValid || remoteImageLoading'
+                :loading='remoteImageLoading'
+              ) {{$t('common:actions.fetch')}}
 
           v-card.mt-3.radius-7.animated.fadeInRight.wait-p4s(:light='!$vuetify.theme.dark', :dark='$vuetify.theme.dark')
             v-card-text.pb-0
@@ -239,6 +245,7 @@ import listFolderAssetQuery from 'gql/editor/editor-media-query-folder-list.gql'
 import createAssetFolderMutation from 'gql/editor/editor-media-mutation-folder-create.gql'
 import renameAssetMutation from 'gql/editor/editor-media-mutation-asset-rename.gql'
 import deleteAssetMutation from 'gql/editor/editor-media-mutation-asset-delete.gql'
+import fetchRemoteAssetMutation from 'gql/editor/editor-media-mutation-asset-fetch.gql'
 
 const FilePond = vueFilePond()
 const localeSegmentRegex = /^[A-Z]{2}(-[A-Z]{2})?$/i
@@ -261,6 +268,7 @@ export default {
       assets: [],
       pagination: 1,
       remoteImageUrl: '',
+      remoteImageLoading: false,
       imageAlignments: [
         { text: 'None', value: '' },
         { text: 'Left', value: 'left' },
@@ -314,6 +322,21 @@ export default {
     currentAsset () {
       return _.find(this.assets, ['id', this.currentFileId]) || {}
     },
+    remoteUrlIsValid () {
+      if (!this.remoteImageUrl) {
+        return false
+      }
+      try {
+        const input = this.remoteImageUrl.trim()
+        if (!input) {
+          return false
+        }
+        const remoteUrl = new URL(input)
+        return ['http:', 'https:'].includes(remoteUrl.protocol)
+      } catch (err) {
+        return false
+      }
+    },
     filePondServerOpts () {
       const jwtToken = Cookies.get('jwt')
       return {
@@ -382,6 +405,43 @@ export default {
     browse () {
       this.$refs.pond.browse()
     },
+    async fetchRemoteAsset () {
+      if (!this.remoteUrlIsValid || this.remoteImageLoading) {
+        return
+      }
+      const folderId = this.currentFolderId || 0
+      const remoteUrl = this.remoteImageUrl.trim()
+      this.remoteImageLoading = true
+      this.$store.commit('loadingStart', 'editor-media-fetchremote')
+      try {
+        const resp = await this.$apollo.mutate({
+          mutation: fetchRemoteAssetMutation,
+          variables: {
+            folderId,
+            url: remoteUrl
+          }
+        })
+        const result = _.get(resp, 'data.assets.fetchRemoteAsset.responseResult', {})
+        if (result.succeeded) {
+          this.$store.commit('showNotification', {
+            message: result.message || 'Remote asset fetched successfully.',
+            style: 'success',
+            icon: 'check'
+          })
+          this.remoteImageUrl = ''
+          await this.$apollo.queries.assets.refetch()
+        } else if (result.message) {
+          this.$store.commit('pushGraphError', new Error(result.message))
+        } else {
+          this.$store.commit('pushGraphError', new Error(this.$t('editor:assets.uploadFailed')))
+        }
+      } catch (err) {
+        this.$store.commit('pushGraphError', err)
+      } finally {
+        this.remoteImageLoading = false
+        this.$store.commit('loadingStop', 'editor-media-fetchremote')
+      }
+    },
     async upload () {
       const files = this.$refs.pond.getFiles()
       if (files.length < 1) {

client/graph/editor/editor-media-mutation-asset-fetch.gql

@@ -0,0 +1,12 @@
+mutation ($folderId: Int!, $url: String!) {
+  assets {
+    fetchRemoteAsset(folderId: $folderId, url: $url) {
+      responseResult {
+        succeeded
+        errorCode
+        slug
+        message
+      }
+    }
+  }
+}

dev/containers/Dockerfile

@@ -1,7 +1,7 @@
 # -- DEV DOCKERFILE --
 # -- DO NOT USE IN PRODUCTION! --
 
-FROM node:18
+FROM node:20
 LABEL maintainer "requarks.io"
 
 RUN apt-get update && \

dev/containers/docker-compose.yml

@@ -52,6 +52,13 @@ services:
       - ../..:/wiki
       - /wiki/node_modules
       - /wiki/.git
+    tty: true
+    stdin_open: true
+    command: >
+      sh -c "
+        yarn install &&
+        yarn dev
+      "
 
 
 volumes:

server/graph/resolvers/asset.js

@@ -1,10 +1,19 @@
 const _ = require('lodash')
 const sanitize = require('sanitize-filename')
+const fs = require('fs-extra')
+const path = require('path')
+const mime = require('mime-types')
+const fetch = require('node-fetch')
+const { URL } = require('url')
+const FileType = require('file-type')
 const graphHelper = require('../../helpers/graph')
 const assetHelper = require('../../helpers/asset')
 
 /* global WIKI */
 
+const REMOTE_FETCH_TIMEOUT = 15000
+const REMOTE_ALLOWED_MIME_PREFIXES = ['image/', 'video/']
+
 module.exports = {
   Query: {
     async assets() { return {} }
@@ -189,6 +198,161 @@ module.exports = {
         return graphHelper.generateError(err)
       }
     },
+    /**
+     * Fetch remote media and store as asset
+     */
+    async fetchRemoteAsset(obj, args, context) {
+      let tempFilePath = null
+      try {
+        const user = context.req.user
+        const remoteUrlRaw = _.trim(args.url)
+        if (!remoteUrlRaw) {
+          throw new WIKI.Error.InputInvalid()
+        }
+
+        const maxFileSize = _.get(WIKI, 'config.uploads.maxFileSize', 0)
+        let targetFolderId = args.folderId === 0 ? null : args.folderId
+        let hierarchy = []
+        if (targetFolderId) {
+          hierarchy = await WIKI.models.assetFolders.getHierarchy(targetFolderId)
+          if (hierarchy.length < 1) {
+            throw new WIKI.Error.InputInvalid()
+          }
+        }
+        const folderPath = hierarchy.map(h => h.slug).join('/')
+
+        let parsedUrl
+        try {
+          parsedUrl = new URL(remoteUrlRaw)
+        } catch (err) {
+          throw new WIKI.Error.InputInvalid()
+        }
+        if (!['http:', 'https:'].includes(parsedUrl.protocol)) {
+          throw new WIKI.Error.InputInvalid()
+        }
+
+        const fetchOpts = {
+          timeout: REMOTE_FETCH_TIMEOUT
+        }
+        if (maxFileSize > 0) {
+          fetchOpts.size = maxFileSize + 1024
+        }
+
+        let response
+        try {
+          response = await fetch(remoteUrlRaw, fetchOpts)
+        } catch (err) {
+          throw new WIKI.Error.AssetFetchFailed()
+        }
+
+        if (!response.ok) {
+          throw new WIKI.Error.AssetFetchFailed()
+        }
+
+        const contentLengthHeader = response.headers.get('content-length')
+        if (contentLengthHeader && maxFileSize > 0 && _.toInteger(contentLengthHeader) > maxFileSize) {
+          throw new WIKI.Error.AssetFetchTooLarge()
+        }
+
+        let buffer
+        try {
+          buffer = await response.buffer()
+        } catch (err) {
+          if (err && err.type === 'max-size') {
+            throw new WIKI.Error.AssetFetchTooLarge()
+          }
+          throw new WIKI.Error.AssetFetchFailed()
+        }
+
+        if (!buffer || buffer.length < 1) {
+          throw new WIKI.Error.AssetFetchFailed()
+        }
+        if (maxFileSize > 0 && buffer.length > maxFileSize) {
+          throw new WIKI.Error.AssetFetchTooLarge()
+        }
+
+        let mimeType = (response.headers.get('content-type') || '').split(';')[0].trim().toLowerCase()
+        const detectedType = await FileType.fromBuffer(buffer)
+        if (detectedType && detectedType.mime) {
+          mimeType = detectedType.mime
+        }
+        if (!mimeType || !_.some(REMOTE_ALLOWED_MIME_PREFIXES, prefix => mimeType.startsWith(prefix))) {
+          throw new WIKI.Error.AssetFetchInvalidType()
+        }
+
+        const rawUrlSegment = decodeURIComponent(parsedUrl.pathname || '').split('/').filter(Boolean).pop() || ''
+        const rawExt = path.extname(rawUrlSegment)
+        const rawBase = rawExt ? rawUrlSegment.slice(0, -rawExt.length) : rawUrlSegment
+        let sanitizedBase = sanitize(rawBase.toLowerCase().replace(/[\s,;#]+/g, '_'))
+        if (!sanitizedBase) {
+          sanitizedBase = `remote_asset_${Date.now()}`
+        }
+
+        const normalizedRawExt = rawExt.toLowerCase().replace(/^\./, '')
+        const mimeExt = mime.extension(mimeType)
+        const allowedExts = (mime.extensions && mime.extensions[mimeType]) || []
+        let finalExt = ''
+        if (
+          normalizedRawExt &&
+          normalizedRawExt.length <= 8 &&
+          (allowedExts.length === 0 || allowedExts.includes(normalizedRawExt))
+        ) {
+          finalExt = `.${normalizedRawExt}`
+        } else if (mimeExt) {
+          finalExt = `.${mimeExt}`
+        }
+        if (!finalExt) {
+          throw new WIKI.Error.AssetFetchInvalidType()
+        }
+
+        if (sanitizedBase.length + finalExt.length > 255) {
+          sanitizedBase = sanitizedBase.substring(0, 255 - finalExt.length)
+        }
+
+        let finalName = sanitize(`${sanitizedBase}${finalExt}`).toLowerCase()
+        if (!finalName || finalName === finalExt) {
+          finalName = `remote_asset_${Date.now()}${finalExt}`
+        }
+        if (!finalName.endsWith(finalExt)) {
+          finalName = `${finalName}${finalExt}`
+        }
+
+        const assetPath = folderPath ? `${folderPath}/${finalName}` : finalName
+        if (!WIKI.auth.checkAccess(user, ['write:assets'], { path: assetPath })) {
+          throw new WIKI.Error.AssetUploadForbidden()
+        }
+
+        const uploadsDir = path.resolve(WIKI.ROOTPATH, WIKI.config.dataPath, 'uploads')
+        await fs.ensureDir(uploadsDir)
+        const tempFileName = `${Date.now()}-${Math.random().toString(36).substring(2, 8)}${finalExt}`
+        tempFilePath = path.join(uploadsDir, tempFileName)
+        await fs.writeFile(tempFilePath, buffer)
+
+        await WIKI.models.assets.upload({
+          mode: 'remote',
+          originalname: finalName,
+          mimetype: mimeType,
+          size: buffer.length,
+          folderId: targetFolderId,
+          path: tempFilePath,
+          assetPath,
+          user
+        })
+
+        await fs.remove(tempFilePath)
+        tempFilePath = null
+
+        return {
+          responseResult: graphHelper.generateSuccess('Remote asset fetched successfully.')
+        }
+      } catch (err) {
+        return graphHelper.generateError(err)
+      } finally {
+        if (tempFilePath) {
+          await fs.remove(tempFilePath).catch(() => {})
+        }
+      }
+    },
     /**
      * Flush Temporary Uploads
      */

server/graph/schemas/asset.graphql

@@ -45,6 +45,11 @@ type AssetMutation {
     id: Int!
   ): DefaultResponse @auth(requires: ["manage:system", "manage:assets"])
 
+  fetchRemoteAsset(
+    folderId: Int!
+    url: String!
+  ): DefaultResponse @auth(requires: ["manage:system", "write:assets"])
+
   flushTempUploads: DefaultResponse @auth(requires: ["manage:system"])
 }
 

server/helpers/error.js

@@ -37,6 +37,22 @@ module.exports = {
     message: 'You are not authorized to rename this asset to the requested name.',
     code: 2009
   }),
+  AssetUploadForbidden: CustomError('AssetUploadForbidden', {
+    message: 'You are not authorized to upload files to this folder.',
+    code: 2010
+  }),
+  AssetFetchFailed: CustomError('AssetFetchFailed', {
+    message: 'Failed to download the remote file. Make sure the URL is correct and accessible.',
+    code: 2011
+  }),
+  AssetFetchInvalidType: CustomError('AssetFetchInvalidType', {
+    message: 'Remote file type is not supported for import.',
+    code: 2012
+  }),
+  AssetFetchTooLarge: CustomError('AssetFetchTooLarge', {
+    message: 'Remote file exceeds the maximum allowed size.',
+    code: 2013
+  }),
   AuthAccountBanned: CustomError('AuthAccountBanned', {
     message: 'Your account has been disabled.',
     code: 1013