-
Notifications
You must be signed in to change notification settings - Fork 0
/
02_DownloadImages.js
64 lines (48 loc) · 1.72 KB
/
02_DownloadImages.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
import { createWriteStream, readFileSync } from 'fs'
import fetch from 'node-fetch'
import { createPathIfNotExists } from './00_Utils.js'
const downloadImage = async (url, path, fileName) => {
// fetch file
const response = await fetch(url)
// stream response body
const buffer = await response.arrayBuffer()
createPathIfNotExists(path, false)
// write file to disk
fileName = fileName || url.split("/").pop()
createWriteStream(`${path}/${fileName}`).write(Buffer.from(buffer))
}
const downloadImages = async (urls, path, fileNamesAsIndex) => {
urls.map(async (url, i) => {
console.log("Downloading " + url)
await downloadImage(url, path, fileNamesAsIndex ? i : undefined)
})
}
const getUrlsFromFile = (path) => {
const data = readFileSync(path, { encoding: 'utf-8' })
const urls = data.split(";")
console.log(`Found ${urls.length} Urls in ${path}`)
return urls
}
(async () => {
// TRAIN DATA
let type = "training"
// FULLYS
const fullyFiles = ["fully1.txt", "fully2.txt"]
fullyFiles.map(async file => {
const fullyUrls = getUrlsFromFile(`./urls/${type}/${file}`)
await downloadImages(fullyUrls, `./images/${type}/fully`)
})
// HARDTAILS
const hardtailFiles = ["hardtail1.txt", "hardtail2.txt"]
hardtailFiles.map(async file => {
const hardtailUrls = getUrlsFromFile(`./urls/${type}/${file}`)
await downloadImages(hardtailUrls, `./images/${type}/hardtail`)
})
// TEST DATA
type = "test"
const testFiles = ["test.txt"]
testFiles.map(async file => {
const testUrls = getUrlsFromFile(`./urls/${type}/${file}`)
await downloadImages(testUrls, `./images/${type}`)
})
})()