mirror of
https://github.com/Myxelium/Bridge-Multi.git
synced 2026-04-11 14:19:38 +00:00
Refactor download queue
This commit is contained in:
87
package-lock.json
generated
87
package-lock.json
generated
@@ -2124,15 +2124,6 @@
|
||||
"defer-to-connect": "^1.0.1"
|
||||
}
|
||||
},
|
||||
"@types/better-queue": {
|
||||
"version": "3.8.2",
|
||||
"resolved": "https://registry.npmjs.org/@types/better-queue/-/better-queue-3.8.2.tgz",
|
||||
"integrity": "sha512-KMFFgojmy10+npJiw/XkY2i+UD96NZmo4L0xuw8DRfEXB4a70rS6YHzqpb7Xgh1+TwWFsIHbsK8fFkvMTxEASA==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"@types/node": "*"
|
||||
}
|
||||
},
|
||||
"@types/cli-color": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/@types/cli-color/-/cli-color-2.0.0.tgz",
|
||||
@@ -2195,6 +2186,12 @@
|
||||
"integrity": "sha512-tHq6qdbT9U1IRSGf14CL0pUlULksvY9OZ+5eEgl1N7t+OA3tGvNpxJCzuKQlsNgCVwbAs670L1vcVQi8j9HjnA==",
|
||||
"dev": true
|
||||
},
|
||||
"@types/mv": {
|
||||
"version": "2.1.0",
|
||||
"resolved": "https://registry.npmjs.org/@types/mv/-/mv-2.1.0.tgz",
|
||||
"integrity": "sha512-9uDB9lojfIQipxfI308//b4c5isHs6uMo3kIMzv73FPdXmMnAk6iELHGI849cuuDPHy6aXBwN/q9gMzjRyhJ+w==",
|
||||
"dev": true
|
||||
},
|
||||
"@types/needle": {
|
||||
"version": "2.0.4",
|
||||
"resolved": "https://registry.npmjs.org/@types/needle/-/needle-2.0.4.tgz",
|
||||
@@ -2215,6 +2212,16 @@
|
||||
"integrity": "sha512-ce5d3q03Ex0sy4R14722Rmt6MT07Ua+k4FwDfdcToYJcMKNtRVQvJ6JCAPdAmAnbRb6CsX6aYb9m96NGod9uTw==",
|
||||
"dev": true
|
||||
},
|
||||
"@types/rimraf": {
|
||||
"version": "3.0.0",
|
||||
"resolved": "https://registry.npmjs.org/@types/rimraf/-/rimraf-3.0.0.tgz",
|
||||
"integrity": "sha512-7WhJ0MdpFgYQPXlF4Dx+DhgvlPCfz/x5mHaeDQAKhcenvQP1KCpLQ18JklAqeGMYSAT2PxLpzd0g2/HE7fj7hQ==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"@types/glob": "*",
|
||||
"@types/node": "*"
|
||||
}
|
||||
},
|
||||
"@types/source-list-map": {
|
||||
"version": "0.1.2",
|
||||
"resolved": "https://registry.npmjs.org/@types/source-list-map/-/source-list-map-0.1.2.tgz",
|
||||
@@ -3353,21 +3360,6 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"better-queue": {
|
||||
"version": "3.8.10",
|
||||
"resolved": "https://registry.npmjs.org/better-queue/-/better-queue-3.8.10.tgz",
|
||||
"integrity": "sha512-e3gwNZgDCnNWl0An0Tz6sUjKDV9m6aB+K9Xg//vYeo8+KiH8pWhLFxkawcXhm6FpM//GfD9IQv/kmvWCAVVpKA==",
|
||||
"requires": {
|
||||
"better-queue-memory": "^1.0.1",
|
||||
"node-eta": "^0.9.0",
|
||||
"uuid": "^3.0.0"
|
||||
}
|
||||
},
|
||||
"better-queue-memory": {
|
||||
"version": "1.0.4",
|
||||
"resolved": "https://registry.npmjs.org/better-queue-memory/-/better-queue-memory-1.0.4.tgz",
|
||||
"integrity": "sha512-SWg5wFIShYffEmJpI6LgbL8/3Dqhku7xI1oEiy6FroP9DbcZlG0ZDjxvPdP9t7hTGW40IpIcC6zVoGT1oxjOuA=="
|
||||
},
|
||||
"big.js": {
|
||||
"version": "5.2.2",
|
||||
"resolved": "https://registry.npmjs.org/big.js/-/big.js-5.2.2.tgz",
|
||||
@@ -4388,6 +4380,11 @@
|
||||
"integrity": "sha1-3dgA2gxmEnOTzKWVDqloo6rxJTs=",
|
||||
"dev": true
|
||||
},
|
||||
"comparators": {
|
||||
"version": "3.0.2",
|
||||
"resolved": "https://registry.npmjs.org/comparators/-/comparators-3.0.2.tgz",
|
||||
"integrity": "sha512-Tfum570X4APe7vEQdDjoMZssxeyW4xuH/oYbI/fXGwJZ/rIhzBrfykfHf6LUVhk0GzeFhI6oYuOr1y7PRV7tng=="
|
||||
},
|
||||
"component-emitter": {
|
||||
"version": "1.3.0",
|
||||
"resolved": "https://registry.npmjs.org/component-emitter/-/component-emitter-1.3.0.tgz",
|
||||
@@ -11679,6 +11676,38 @@
|
||||
"resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.8.tgz",
|
||||
"integrity": "sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA=="
|
||||
},
|
||||
"mv": {
|
||||
"version": "2.1.1",
|
||||
"resolved": "https://registry.npmjs.org/mv/-/mv-2.1.1.tgz",
|
||||
"integrity": "sha1-rmzg1vbV4KT32JN5jQPB6pVZtqI=",
|
||||
"requires": {
|
||||
"mkdirp": "~0.5.1",
|
||||
"ncp": "~2.0.0",
|
||||
"rimraf": "~2.4.0"
|
||||
},
|
||||
"dependencies": {
|
||||
"glob": {
|
||||
"version": "6.0.4",
|
||||
"resolved": "https://registry.npmjs.org/glob/-/glob-6.0.4.tgz",
|
||||
"integrity": "sha1-DwiGD2oVUSey+t1PnOJLGqtuTSI=",
|
||||
"requires": {
|
||||
"inflight": "^1.0.4",
|
||||
"inherits": "2",
|
||||
"minimatch": "2 || 3",
|
||||
"once": "^1.3.0",
|
||||
"path-is-absolute": "^1.0.0"
|
||||
}
|
||||
},
|
||||
"rimraf": {
|
||||
"version": "2.4.5",
|
||||
"resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.4.5.tgz",
|
||||
"integrity": "sha1-7nEM5dk6j9uFb7Xqj/Di11k0sto=",
|
||||
"requires": {
|
||||
"glob": "^6.0.1"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"nan": {
|
||||
"version": "2.14.1",
|
||||
"resolved": "https://registry.npmjs.org/nan/-/nan-2.14.1.tgz",
|
||||
@@ -11709,6 +11738,11 @@
|
||||
"integrity": "sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc=",
|
||||
"dev": true
|
||||
},
|
||||
"ncp": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/ncp/-/ncp-2.0.0.tgz",
|
||||
"integrity": "sha1-GVoh1sRuNh0vsSgbo4uR6d9727M="
|
||||
},
|
||||
"needle": {
|
||||
"version": "2.3.2",
|
||||
"resolved": "https://registry.npmjs.org/needle/-/needle-2.3.2.tgz",
|
||||
@@ -11805,11 +11839,6 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"node-eta": {
|
||||
"version": "0.9.0",
|
||||
"resolved": "https://registry.npmjs.org/node-eta/-/node-eta-0.9.0.tgz",
|
||||
"integrity": "sha1-n7CwmbzSoCGUDmA8ZCVNwAPZp6g="
|
||||
},
|
||||
"node-fetch": {
|
||||
"version": "2.6.0",
|
||||
"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.0.tgz",
|
||||
|
||||
@@ -34,11 +34,12 @@
|
||||
"@angular/platform-browser": "~9.1.4",
|
||||
"@angular/platform-browser-dynamic": "~9.1.4",
|
||||
"@angular/router": "~9.1.4",
|
||||
"better-queue": "^3.8.10",
|
||||
"cli-color": "^2.0.0",
|
||||
"comparators": "^3.0.2",
|
||||
"electron-window-state": "^5.0.3",
|
||||
"fomantic-ui": "^2.8.3",
|
||||
"jquery": "^3.4.1",
|
||||
"mv": "^2.1.1",
|
||||
"needle": "^2.3.2",
|
||||
"node-7z": "^2.0.5",
|
||||
"node-unrar-js": "^0.8.1",
|
||||
@@ -54,11 +55,12 @@
|
||||
"@angular/cli": "^9.1.4",
|
||||
"@angular/compiler-cli": "~9.1.4",
|
||||
"@angular/language-service": "~9.1.4",
|
||||
"@types/better-queue": "^3.8.2",
|
||||
"@types/cli-color": "^2.0.0",
|
||||
"@types/electron-window-state": "^2.0.33",
|
||||
"@types/mv": "^2.1.0",
|
||||
"@types/needle": "^2.0.4",
|
||||
"@types/node": "^12.11.1",
|
||||
"@types/rimraf": "^3.0.0",
|
||||
"@types/underscore": "^1.9.4",
|
||||
"@typescript-eslint/eslint-plugin": "^2.19.2",
|
||||
"@typescript-eslint/parser": "^2.19.2",
|
||||
|
||||
@@ -12,13 +12,6 @@
|
||||
<i class="redo icon"></i>
|
||||
Retry
|
||||
</button>
|
||||
<button
|
||||
*ngIf="download.type == 'warning'"
|
||||
class="ui right floated labeled icon button"
|
||||
(click)="continueDownload(download.versionID)">
|
||||
<i class="exclamation triangle icon"></i>
|
||||
Download Anyway
|
||||
</button>
|
||||
<div class="header">{{download.header}}</div>
|
||||
<div *ngIf="download.type != 'done'" class="description">{{download.description}}</div>
|
||||
<div *ngIf="download.type == 'done'" class="description">
|
||||
|
||||
@@ -38,11 +38,6 @@ export class DownloadsModalComponent {
|
||||
this.downloadService.retryDownload(versionID)
|
||||
}
|
||||
|
||||
continueDownload(versionID: number) {
|
||||
// TODO: test this
|
||||
this.downloadService.continueDownload(versionID)
|
||||
}
|
||||
|
||||
getBackgroundColor(download: DownloadProgress) {
|
||||
switch (download.type) {
|
||||
case 'good': return 'unset'
|
||||
|
||||
@@ -67,8 +67,4 @@ export class DownloadService {
|
||||
retryDownload(versionID: number) {
|
||||
this.electronService.sendIPC('download', { action: 'retry', versionID })
|
||||
}
|
||||
|
||||
continueDownload(versionID: number) {
|
||||
this.electronService.sendIPC('download', { action: 'continue', versionID })
|
||||
}
|
||||
}
|
||||
@@ -8,104 +8,150 @@ import { promisify } from 'util'
|
||||
import { randomBytes as _randomBytes } from 'crypto'
|
||||
import { mkdir as _mkdir } from 'fs'
|
||||
import { ProgressType, NewDownload } from 'src/electron/shared/interfaces/download.interface'
|
||||
import { downloadHandler } from './DownloadHandler'
|
||||
import { googleTimer } from './GoogleTimer'
|
||||
import { DriveFile } from 'src/electron/shared/interfaces/songDetails.interface'
|
||||
import { FileTransfer } from './FileTransfer'
|
||||
|
||||
const randomBytes = promisify(_randomBytes)
|
||||
const mkdir = promisify(_mkdir)
|
||||
|
||||
type EventCallback = {
|
||||
/** Note: this will not be the last event if `retry()` is called. */
|
||||
'error': () => void
|
||||
'complete': () => void
|
||||
}
|
||||
type Callbacks = { [E in keyof EventCallback]: EventCallback[E] }
|
||||
|
||||
export type DownloadError = { header: string; body: string }
|
||||
|
||||
export class ChartDownload {
|
||||
|
||||
// This changes if the user needs to click 'retry' or 'continue'
|
||||
run: () => void | Promise<void> = this.beginDownload.bind(this)
|
||||
cancel: () => void
|
||||
|
||||
isArchive: boolean
|
||||
title: string
|
||||
header: string
|
||||
description: string
|
||||
percent = 0
|
||||
type: ProgressType
|
||||
private retryFn: () => void | Promise<void>
|
||||
private cancelFn: () => void
|
||||
|
||||
private callbacks = {} as Callbacks
|
||||
private files: DriveFile[]
|
||||
allFilesProgress = 0
|
||||
individualFileProgressPortion: number
|
||||
private percent = 0 // Needs to be stored here because errors won't know the exact percent
|
||||
|
||||
private readonly individualFileProgressPortion: number
|
||||
private readonly destinationFolderName: string
|
||||
|
||||
private _allFilesProgress = 0
|
||||
get allFilesProgress() { return this._allFilesProgress }
|
||||
private _hasFailed = false
|
||||
/** If this chart download needs to be retried */
|
||||
get hasFailed() { return this._hasFailed }
|
||||
get isArchive() { return this.data.driveData.isArchive }
|
||||
|
||||
constructor(public versionID: number, private data: NewDownload) {
|
||||
this.updateGUI('', 'Waiting for other downloads to finish...', 'good')
|
||||
this.files = data.driveData.files
|
||||
this.isArchive = data.driveData.isArchive
|
||||
this.individualFileProgressPortion = 80 / this.files.length
|
||||
this.destinationFolderName = sanitizeFilename(`${this.data.artist} - ${this.data.avTagName} (${this.data.charter})`)
|
||||
}
|
||||
|
||||
/**
|
||||
* Changes this download to reflect that it is waiting in the download queue.
|
||||
* Calls `callback` when `event` fires. (no events will be fired after `this.cancel()` is called)
|
||||
*/
|
||||
setInQueue() {
|
||||
this.title = `${this.data.avTagName} - ${this.data.artist}`
|
||||
this.header = ''
|
||||
this.description = 'Waiting for other downloads to finish...'
|
||||
this.type = 'good'
|
||||
this.cancel = () => { /* do nothing */ }
|
||||
emitIPCEvent('download-updated', this)
|
||||
on<E extends keyof EventCallback>(event: E, callback: EventCallback[E]) {
|
||||
this.callbacks[event] = callback
|
||||
}
|
||||
|
||||
/**
|
||||
* Retries the last failed step if it is running.
|
||||
*/
|
||||
retry() { // Only allow it to be called once
|
||||
if (this.retryFn != undefined) {
|
||||
this._hasFailed = false
|
||||
const retryFn = this.retryFn
|
||||
this.retryFn = undefined
|
||||
retryFn()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Cancels the download if it is running.
|
||||
*/
|
||||
cancel() { // Only allow it to be called once
|
||||
if (this.cancelFn != undefined) {
|
||||
const cancelFn = this.cancelFn
|
||||
this.cancelFn = undefined
|
||||
cancelFn()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Updates the GUI with new information about this chart download.
|
||||
*/
|
||||
private updateGUI(header: string, description: string, type: ProgressType) {
|
||||
emitIPCEvent('download-updated', {
|
||||
versionID: this.versionID,
|
||||
title: `${this.data.avTagName} - ${this.data.artist}`,
|
||||
header: header,
|
||||
description: description,
|
||||
percent: this.percent,
|
||||
type: type
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Save the retry function, update the GUI, and call the `error` callback.
|
||||
*/
|
||||
private handleError(err: DownloadError, retry: () => void) {
|
||||
this._hasFailed = true
|
||||
this.retryFn = retry
|
||||
this.updateGUI(err.header, err.body, 'error')
|
||||
this.callbacks.error()
|
||||
}
|
||||
|
||||
/**
|
||||
* Starts the download process.
|
||||
*/
|
||||
async beginDownload() {
|
||||
// Create a temporary folder to store the downloaded files
|
||||
// CREATE DOWNLOAD DIRECTORY
|
||||
let chartPath: string
|
||||
try {
|
||||
chartPath = await this.createDownloadFolder()
|
||||
} catch (e) {
|
||||
this.run = this.beginDownload.bind(this) // Retry action
|
||||
this.error('Access Error', e.message)
|
||||
} catch (err) {
|
||||
this.retryFn = () => this.beginDownload()
|
||||
this.updateGUI('Access Error', err.message, 'error')
|
||||
return
|
||||
}
|
||||
|
||||
// For each download link in <this.files>, download the file to <chartPath>
|
||||
// DOWNLOAD FILES
|
||||
for (let i = 0; i < this.files.length; i++) {
|
||||
// INITIALIZE DOWNLOADER
|
||||
const downloader = new FileDownloader(this.files[i].webContentLink, chartPath)
|
||||
this.cancelFn = () => downloader.cancelDownload()
|
||||
|
||||
const downloadComplete = this.addDownloadEventListeners(downloader, i)
|
||||
|
||||
// DOWNLOAD THE NEXT FILE
|
||||
// Wait for google rate limit
|
||||
this.header = `[${this.files[i].name}] (file ${i + 1}/${this.files.length})`
|
||||
googleTimer.onTimerUpdate((remainingTime, totalTime) => {
|
||||
this.description = `Waiting for Google rate limit... (${remainingTime}s)`
|
||||
this.percent = this.allFilesProgress + interpolate(remainingTime, totalTime, 0, 0, this.individualFileProgressPortion / 2)
|
||||
this.type = 'good'
|
||||
emitIPCEvent('download-updated', this)
|
||||
})
|
||||
this.cancel = () => {
|
||||
googleTimer.removeCallbacks()
|
||||
this.onDownloadStop()
|
||||
downloader.cancelDownload()
|
||||
}
|
||||
await new Promise<void>(resolve => googleTimer.onTimerReady(resolve))
|
||||
|
||||
this.cancel = () => {
|
||||
this.onDownloadStop()
|
||||
downloader.cancelDownload()
|
||||
}
|
||||
downloader.beginDownload()
|
||||
await downloadComplete // Wait for this download to finish
|
||||
await downloadComplete
|
||||
}
|
||||
|
||||
// INITIALIZE FILE EXTRACTOR
|
||||
const destinationFolderName = sanitizeFilename(`${this.data.artist} - ${this.data.avTagName} (${this.data.charter})`)
|
||||
const extractor = new FileExtractor(chartPath, this.isArchive, destinationFolderName)
|
||||
this.cancel = () => extractor.cancelExtract() // Make cancel button cancel the file extraction
|
||||
this.addExtractorEventListeners(extractor)
|
||||
// EXTRACT FILES
|
||||
if (this.isArchive) {
|
||||
const extractor = new FileExtractor(chartPath)
|
||||
this.cancelFn = () => extractor.cancelExtract()
|
||||
|
||||
// EXTRACT THE DOWNLOADED ARCHIVE
|
||||
extractor.beginExtract()
|
||||
const extractComplete = this.addExtractorEventListeners(extractor)
|
||||
extractor.beginExtract()
|
||||
await extractComplete
|
||||
}
|
||||
|
||||
// TRANSFER FILES
|
||||
const transfer = new FileTransfer(chartPath, this.destinationFolderName)
|
||||
this.cancelFn = () => transfer.cancelTransfer()
|
||||
|
||||
const transferComplete = this.addTransferEventListeners(transfer)
|
||||
transfer.beginTransfer()
|
||||
await transferComplete
|
||||
|
||||
this.callbacks.complete()
|
||||
}
|
||||
|
||||
/**
|
||||
* Attempts to create a unique folder in Bridge's data paths. Throws an error if this fails.
|
||||
* Attempts to create a unique folder in Bridge's data paths.
|
||||
* @returns the new folder's path.
|
||||
* @throws an error if this fails.
|
||||
*/
|
||||
private async createDownloadFolder() {
|
||||
let retryCount = 0
|
||||
@@ -125,71 +171,38 @@ export class ChartDownload {
|
||||
throw new Error(`Bridge was unable to create a directory at [${chartPath}]`)
|
||||
}
|
||||
|
||||
/**
|
||||
* Stops the download and displays an error message.
|
||||
*/
|
||||
private error(header: string, description: string) {
|
||||
this.header = header
|
||||
this.description = description
|
||||
this.type = 'error'
|
||||
this.onDownloadStop()
|
||||
emitIPCEvent('download-updated', this)
|
||||
}
|
||||
|
||||
/**
|
||||
* If this was a google download, allows a new google download to start.
|
||||
*/
|
||||
private onDownloadStop() {
|
||||
downloadHandler.isGoogleDownloading = false
|
||||
downloadHandler.updateQueue()
|
||||
}
|
||||
|
||||
/**
|
||||
* Defines what happens in response to `FileDownloader` events.
|
||||
* @returns a `Promise` that resolves when the download finishes.
|
||||
*/
|
||||
private addDownloadEventListeners(downloader: FileDownloader, fileIndex: number) {
|
||||
let downloadHeader = `[${this.files[fileIndex].name}] (file ${fileIndex + 1}/${this.files.length})`
|
||||
let fileProgress = 0
|
||||
|
||||
downloader.on('request', () => {
|
||||
this.description = 'Sending request...'
|
||||
fileProgress = this.individualFileProgressPortion / 2
|
||||
this.percent = this.allFilesProgress + fileProgress
|
||||
this.type = 'good'
|
||||
emitIPCEvent('download-updated', this)
|
||||
downloader.on('waitProgress', (remainingSeconds: number, totalSeconds: number) => {
|
||||
this.percent = this._allFilesProgress + interpolate(remainingSeconds, totalSeconds, 0, 0, this.individualFileProgressPortion / 2)
|
||||
this.updateGUI(downloadHeader, `Waiting for Google rate limit... (${remainingSeconds}s)`, 'good')
|
||||
})
|
||||
|
||||
downloader.on('warning', (continueAnyway) => {
|
||||
this.description = 'WARNING'
|
||||
this.run = continueAnyway
|
||||
this.type = 'warning'
|
||||
this.onDownloadStop()
|
||||
emitIPCEvent('download-updated', this)
|
||||
downloader.on('requestSent', () => {
|
||||
fileProgress = this.individualFileProgressPortion / 2
|
||||
this.percent = this._allFilesProgress + fileProgress
|
||||
this.updateGUI(downloadHeader, 'Sending request...', 'good')
|
||||
})
|
||||
|
||||
downloader.on('downloadProgress', (bytesDownloaded) => {
|
||||
downloadHeader = `[${this.files[fileIndex].name}] (file ${fileIndex + 1}/${this.files.length})`
|
||||
const size = Number(this.files[fileIndex].size)
|
||||
this.header = `[${this.files[fileIndex].name}] (file ${fileIndex + 1}/${this.files.length})`
|
||||
this.description = `Downloading... (${Math.round(1000 * bytesDownloaded / size) / 10}%)`
|
||||
fileProgress = interpolate(bytesDownloaded, 0, size, this.individualFileProgressPortion / 2, this.individualFileProgressPortion)
|
||||
this.percent = this.allFilesProgress + fileProgress
|
||||
this.type = 'fastUpdate'
|
||||
emitIPCEvent('download-updated', this)
|
||||
this.percent = this._allFilesProgress + fileProgress
|
||||
this.updateGUI(downloadHeader, `Downloading... (${Math.round(1000 * bytesDownloaded / size) / 10}%)`, 'fastUpdate')
|
||||
})
|
||||
|
||||
downloader.on('error', (error, retry) => {
|
||||
this.header = error.header
|
||||
this.description = error.body
|
||||
this.type = 'error'
|
||||
this.run = () => { retry() }
|
||||
this.onDownloadStop()
|
||||
emitIPCEvent('download-updated', this)
|
||||
})
|
||||
downloader.on('error', this.handleError.bind(this))
|
||||
|
||||
// Returns a promise that resolves when the download is finished
|
||||
return new Promise<void>(resolve => {
|
||||
downloader.on('complete', () => {
|
||||
this.allFilesProgress += this.individualFileProgressPortion
|
||||
emitIPCEvent('download-updated', this)
|
||||
this._allFilesProgress += this.individualFileProgressPortion
|
||||
resolve()
|
||||
})
|
||||
})
|
||||
@@ -197,49 +210,51 @@ export class ChartDownload {
|
||||
|
||||
/**
|
||||
* Defines what happens in response to `FileExtractor` events.
|
||||
* @returns a `Promise` that resolves when the extraction finishes.
|
||||
*/
|
||||
private addExtractorEventListeners(extractor: FileExtractor) {
|
||||
let archive = ''
|
||||
|
||||
extractor.on('extract', (filename) => {
|
||||
extractor.on('start', (filename) => {
|
||||
archive = filename
|
||||
this.header = `[${archive}]`
|
||||
this.description = 'Extracting...'
|
||||
this.type = 'good'
|
||||
emitIPCEvent('download-updated', this)
|
||||
this.updateGUI(`[${archive}]`, 'Extracting...', 'good')
|
||||
})
|
||||
|
||||
extractor.on('extractProgress', (percent, filecount) => {
|
||||
this.header = `[${archive}] (${filecount} file${filecount == 1 ? '' : 's'} extracted)`
|
||||
this.description = `Extracting... (${percent}%)`
|
||||
this.percent = interpolate(percent, 0, 100, 80, 95)
|
||||
this.type = 'fastUpdate'
|
||||
emitIPCEvent('download-updated', this)
|
||||
this.updateGUI(`[${archive}] (${filecount} file${filecount == 1 ? '' : 's'} extracted)`, `Extracting... (${percent}%)`, 'fastUpdate')
|
||||
})
|
||||
|
||||
extractor.on('transfer', (filepath) => {
|
||||
this.header = 'Moving files to library folder...'
|
||||
this.description = filepath
|
||||
this.percent = 95
|
||||
this.type = 'good'
|
||||
emitIPCEvent('download-updated', this)
|
||||
extractor.on('error', this.handleError.bind(this))
|
||||
|
||||
return new Promise<void>(resolve => {
|
||||
extractor.on('complete', () => {
|
||||
this.percent = 95
|
||||
resolve()
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Defines what happens in response to `FileTransfer` events.
|
||||
* @returns a `Promise` that resolves when the transfer finishes.
|
||||
*/
|
||||
private addTransferEventListeners(transfer: FileTransfer) {
|
||||
let destinationFolder: string
|
||||
|
||||
transfer.on('start', (_destinationFolder) => {
|
||||
destinationFolder = _destinationFolder
|
||||
this.updateGUI('Moving files to library folder...', destinationFolder, 'good')
|
||||
})
|
||||
|
||||
extractor.on('error', (error, retry) => {
|
||||
this.header = error.header
|
||||
this.description = error.body
|
||||
this.type = 'error'
|
||||
this.run = retry
|
||||
emitIPCEvent('download-updated', this)
|
||||
})
|
||||
transfer.on('error', this.handleError.bind(this))
|
||||
|
||||
extractor.on('complete', (filepath) => {
|
||||
this.header = 'Download complete.'
|
||||
this.description = filepath
|
||||
this.percent = 100
|
||||
this.type = 'done'
|
||||
this.onDownloadStop()
|
||||
emitIPCEvent('download-updated', this)
|
||||
return new Promise<void>(resolve => {
|
||||
transfer.on('complete', () => {
|
||||
this.percent = 100
|
||||
this.updateGUI('Download complete.', destinationFolder, 'done')
|
||||
resolve()
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,46 +1,78 @@
|
||||
import { IPCEmitHandler } from '../../shared/IPCHandler'
|
||||
import { Download } from '../../shared/interfaces/download.interface'
|
||||
import { ChartDownload } from './ChartDownload'
|
||||
import { DownloadQueue } from './DownloadQueue'
|
||||
|
||||
class DownloadHandler implements IPCEmitHandler<'download'> {
|
||||
event: 'download' = 'download'
|
||||
|
||||
downloads: { [versionID: number]: ChartDownload } = {}
|
||||
downloadQueue: ChartDownload[] = []
|
||||
isGoogleDownloading = false // This is a lock controlled by only one ChartDownload at a time
|
||||
downloadQueue: DownloadQueue = new DownloadQueue()
|
||||
currentDownload: ChartDownload = undefined
|
||||
retryWaiting: ChartDownload[] = []
|
||||
|
||||
handler(data: Download) {
|
||||
if (data.action == 'add') {
|
||||
this.downloads[data.versionID] = new ChartDownload(data.versionID, data.data)
|
||||
}
|
||||
|
||||
const download = this.downloads[data.versionID]
|
||||
|
||||
if (data.action == 'cancel') {
|
||||
download.cancel() // Might change isGoogleDownloading and call updateQueue()
|
||||
this.downloadQueue = this.downloadQueue.filter(download => download.versionID != data.versionID)
|
||||
this.downloads[data.versionID] = undefined
|
||||
} else {
|
||||
download.setInQueue()
|
||||
this.downloadQueue.push(download) // Add, retry, or continue will re-add the download to the queue
|
||||
this.updateQueue()
|
||||
handler(data: Download) { // TODO: make sure UI can't add the same versionID more than once
|
||||
switch (data.action) {
|
||||
case 'add': this.addDownload(data); break
|
||||
case 'retry': this.retryDownload(data); break
|
||||
case 'cancel': this.cancelDownload(data); break
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Called when at least one download in the queue can potentially be started.
|
||||
*/
|
||||
updateQueue() {
|
||||
this.downloadQueue.sort((cd1: ChartDownload, cd2: ChartDownload) => {
|
||||
const value1 = 100 + (99 - cd1.allFilesProgress)
|
||||
const value2 = 100 + (99 - cd2.allFilesProgress)
|
||||
return value1 - value2 // Sorts in the order to get the most downloads completed early
|
||||
private addDownload(data: Download) {
|
||||
const newDownload = new ChartDownload(data.versionID, data.data)
|
||||
this.addDownloadEventListeners(newDownload)
|
||||
if (this.currentDownload == undefined) {
|
||||
this.currentDownload = newDownload
|
||||
newDownload.beginDownload()
|
||||
} else {
|
||||
this.downloadQueue.push(newDownload)
|
||||
}
|
||||
}
|
||||
|
||||
private retryDownload(data: Download) {
|
||||
const index = this.retryWaiting.findIndex(download => download.versionID == data.versionID)
|
||||
if (index != -1) {
|
||||
const retryDownload = this.retryWaiting.splice(index, 1)[0]
|
||||
if (this.currentDownload == undefined) {
|
||||
this.currentDownload = retryDownload
|
||||
retryDownload.retry()
|
||||
} else {
|
||||
this.downloadQueue.push(retryDownload)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private cancelDownload(data: Download) {
|
||||
if (this.currentDownload.versionID == data.versionID) {
|
||||
this.currentDownload.cancel()
|
||||
this.currentDownload = undefined
|
||||
this.startNextDownload()
|
||||
} else {
|
||||
this.downloadQueue.remove(data.versionID)
|
||||
}
|
||||
}
|
||||
|
||||
private addDownloadEventListeners(download: ChartDownload) {
|
||||
download.on('complete', () => {
|
||||
this.currentDownload = undefined
|
||||
this.startNextDownload()
|
||||
})
|
||||
|
||||
while (this.downloadQueue[0] != undefined && !(this.isGoogleDownloading)) {
|
||||
const nextDownload = this.downloadQueue.shift()
|
||||
nextDownload.run()
|
||||
this.isGoogleDownloading = true
|
||||
download.on('error', () => {
|
||||
this.retryWaiting.push(this.currentDownload)
|
||||
this.currentDownload = undefined
|
||||
this.startNextDownload()
|
||||
})
|
||||
}
|
||||
|
||||
private startNextDownload() {
|
||||
if (!this.downloadQueue.isEmpty()) {
|
||||
this.currentDownload = this.downloadQueue.pop()
|
||||
if (this.currentDownload.hasFailed) {
|
||||
this.currentDownload.retry()
|
||||
} else {
|
||||
this.currentDownload.beginDownload()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
43
src/electron/ipc/download/DownloadQueue.ts
Normal file
43
src/electron/ipc/download/DownloadQueue.ts
Normal file
@@ -0,0 +1,43 @@
|
||||
import Comparators from 'comparators'
|
||||
import { ChartDownload } from './ChartDownload'
|
||||
|
||||
export class DownloadQueue {
|
||||
|
||||
downloadQueue: ChartDownload[] = []
|
||||
|
||||
isEmpty() {
|
||||
return this.downloadQueue.length == 0
|
||||
}
|
||||
|
||||
push(chartDownload: ChartDownload) {
|
||||
this.downloadQueue.push(chartDownload)
|
||||
this.sort()
|
||||
}
|
||||
|
||||
pop() {
|
||||
return this.downloadQueue.pop()
|
||||
}
|
||||
|
||||
get(versionID: number) {
|
||||
return this.downloadQueue.find(download => download.versionID == versionID)
|
||||
}
|
||||
|
||||
remove(versionID: number) {
|
||||
const index = this.downloadQueue.findIndex(download => download.versionID == versionID)
|
||||
if (index != -1) {
|
||||
this.downloadQueue[index].cancel()
|
||||
this.downloadQueue.splice(index, 1)
|
||||
}
|
||||
}
|
||||
|
||||
private sort() { // TODO: make this order be reflected in the GUI (along with currentDownload)
|
||||
let comparator = Comparators.comparing('allFilesProgress', { reversed: true })
|
||||
|
||||
const prioritizeArchives = true
|
||||
if (prioritizeArchives) {
|
||||
comparator = comparator.thenComparing('isArchive', { reversed: true })
|
||||
}
|
||||
|
||||
this.downloadQueue.sort(comparator)
|
||||
}
|
||||
}
|
||||
@@ -1,25 +1,35 @@
|
||||
import { generateUUID, sanitizeFilename } from '../../shared/UtilFunctions'
|
||||
import * as fs from 'fs'
|
||||
import * as path from 'path'
|
||||
import { AnyFunction } from '../../shared/UtilFunctions'
|
||||
import { createWriteStream } from 'fs'
|
||||
import * as needle from 'needle'
|
||||
// TODO: replace needle with got (for cancel() method) (if before-headers event is possible?)
|
||||
import { getSettings } from '../SettingsHandler.ipc'
|
||||
import { googleTimer } from './GoogleTimer'
|
||||
import { DownloadError } from './ChartDownload'
|
||||
|
||||
type EventCallback = {
|
||||
'request': () => void
|
||||
'warning': (continueAnyway: () => void) => void
|
||||
'waitProgress': (remainingSeconds: number, totalSeconds: number) => void
|
||||
/** Note: this event can be called multiple times if the connection times out or a large file is downloaded */
|
||||
'requestSent': () => void
|
||||
'downloadProgress': (bytesDownloaded: number) => void
|
||||
'error': (error: DownloadError, retry: () => void) => void
|
||||
/** Note: after calling retry, the event lifecycle restarts */
|
||||
'error': (err: DownloadError, retry: () => void) => void
|
||||
'complete': () => void
|
||||
}
|
||||
type Callbacks = { [E in keyof EventCallback]: EventCallback[E] }
|
||||
|
||||
export type DownloadError = { header: string; body: string }
|
||||
const downloadErrors = {
|
||||
libraryFolder: () => { return { header: 'Library folder not specified', body: 'Please go to the settings to set your library folder.' } },
|
||||
timeout: (type: string) => { return { header: 'Timeout', body: `The download server could not be reached. (type=${type})` } },
|
||||
connectionError: (err: Error) => { return { header: 'Connection Error', body: `${err.name}: ${err.message}` } },
|
||||
responseError: (statusCode: number) => { return { header: 'Connection failed', body: `Server returned status code: ${statusCode}` } },
|
||||
htmlError: () => { return { header: 'Invalid response', body: 'Download server returned HTML instead of a file.' } }
|
||||
}
|
||||
|
||||
/**
|
||||
* Downloads a file from `url` to `destinationFolder` and verifies that its hash matches `expectedHash`.
|
||||
* Downloads a file from `url` to `fullPath`.
|
||||
* Will handle google drive virus scan warnings. Provides event listeners for download progress.
|
||||
* On error, provides the ability to retry.
|
||||
* Will only send download requests once every `getSettings().rateLimitDelay` seconds.
|
||||
*/
|
||||
export class FileDownloader {
|
||||
private readonly RETRY_MAX = 2
|
||||
@@ -30,29 +40,33 @@ export class FileDownloader {
|
||||
|
||||
/**
|
||||
* @param url The download link.
|
||||
* @param destinationFolder The path to where this file should be stored.
|
||||
* @param expectedHash The hash header value that is expected for this file.
|
||||
* @param fullPath The full path to where this file should be stored (including the filename).
|
||||
*/
|
||||
constructor(private url: string, private destinationFolder: string) { }
|
||||
constructor(private url: string, private fullPath: string) { }
|
||||
|
||||
/**
|
||||
* Calls `callback` when `event` fires.
|
||||
* Calls `callback` when `event` fires. (no events will be fired after `this.cancelDownload()` is called)
|
||||
*/
|
||||
on<E extends keyof EventCallback>(event: E, callback: EventCallback[E]) {
|
||||
this.callbacks[event] = callback
|
||||
}
|
||||
|
||||
/**
|
||||
* Download the file.
|
||||
* Download the file after waiting for the google rate limit.
|
||||
*/
|
||||
beginDownload() {
|
||||
// Check that the library folder has been specified
|
||||
console.log('Begin download...')
|
||||
if (getSettings().libraryPath == undefined) {
|
||||
this.callbacks.error({ header: 'Library folder not specified', body: 'Please go to the settings to set your library folder.' }, () => this.beginDownload())
|
||||
return
|
||||
}
|
||||
this.failDownload(downloadErrors.libraryFolder())
|
||||
} else {
|
||||
googleTimer.on('waitProgress', this.cancelable((remainingSeconds, totalSeconds) => {
|
||||
this.callbacks.waitProgress(remainingSeconds, totalSeconds)
|
||||
}))
|
||||
|
||||
this.requestDownload()
|
||||
googleTimer.on('complete', this.cancelable(() => {
|
||||
this.requestDownload()
|
||||
}))
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -60,64 +74,60 @@ export class FileDownloader {
|
||||
* @param cookieHeader the "cookie=" header to include this request.
|
||||
*/
|
||||
private requestDownload(cookieHeader?: string) {
|
||||
if (this.wasCanceled) { return } // CANCEL POINT
|
||||
this.callbacks.request()
|
||||
const uuid = generateUUID()
|
||||
this.callbacks.requestSent()
|
||||
const req = needle.get(this.url, {
|
||||
'follow_max': 10,
|
||||
'open_timeout': 5000,
|
||||
'headers': Object.assign({
|
||||
'User-Agent': 'PostmanRuntime/7.22.0',
|
||||
'Referer': this.url,
|
||||
'Accept': '*/*',
|
||||
'Postman-Token': uuid
|
||||
'Accept': '*/*'
|
||||
},
|
||||
(cookieHeader ? { 'Cookie': cookieHeader } : undefined)
|
||||
)
|
||||
})
|
||||
|
||||
req.on('timeout', (type: string) => {
|
||||
req.on('timeout', this.cancelable((type: string) => {
|
||||
this.retryCount++
|
||||
if (this.retryCount <= this.RETRY_MAX) {
|
||||
console.log(`TIMEOUT: Retry attempt ${this.retryCount}...`)
|
||||
this.requestDownload(cookieHeader)
|
||||
} else {
|
||||
this.callbacks.error({ header: 'Timeout', body: `The download server could not be reached. (type=${type})` }, () => this.beginDownload())
|
||||
this.failDownload(downloadErrors.timeout(type))
|
||||
}
|
||||
})
|
||||
}))
|
||||
|
||||
req.on('err', (err: Error) => {
|
||||
this.callbacks.error({ header: 'Connection Error', body: `${err.name}: ${err.message}` }, () => this.beginDownload())
|
||||
})
|
||||
req.on('err', this.cancelable((err: Error) => {
|
||||
this.failDownload(downloadErrors.connectionError(err))
|
||||
}))
|
||||
|
||||
req.on('header', (statusCode, headers: Headers) => {
|
||||
if (this.wasCanceled) { return } // CANCEL POINT
|
||||
req.on('header', this.cancelable((statusCode, headers: Headers) => {
|
||||
if (statusCode != 200) {
|
||||
this.callbacks.error({ header: 'Connection failed', body: `Server returned status code: ${statusCode}` }, () => this.beginDownload())
|
||||
this.failDownload(downloadErrors.responseError(statusCode))
|
||||
return
|
||||
}
|
||||
|
||||
const fileType = headers['content-type']
|
||||
if (fileType.startsWith('text/html')) {
|
||||
if (headers['content-type'].startsWith('text/html')) {
|
||||
this.handleHTMLResponse(req, headers['set-cookie'])
|
||||
} else {
|
||||
const fileName = this.getDownloadFileName(headers)
|
||||
this.handleDownloadResponse(req, fileName)
|
||||
this.handleDownloadResponse(req)
|
||||
}
|
||||
})
|
||||
}))
|
||||
}
|
||||
|
||||
/**
|
||||
* A Google Drive HTML response to a download request usually means this is the "file too large to scan for viruses" warning.
|
||||
* This function sends the request that results from clicking "download anyway", or throws an error if it can't be found.
|
||||
* This function sends the request that results from clicking "download anyway", or generates an error if it can't be found.
|
||||
* @param req The download request.
|
||||
* @param cookieHeader The "cookie=" header of this request.
|
||||
*/
|
||||
private handleHTMLResponse(req: NodeJS.ReadableStream, cookieHeader: string) {
|
||||
console.log('HTML Response...')
|
||||
let virusScanHTML = ''
|
||||
req.on('data', data => virusScanHTML += data)
|
||||
req.on('done', (err: Error) => {
|
||||
if (!err) {
|
||||
req.on('data', this.cancelable(data => virusScanHTML += data))
|
||||
req.on('done', this.cancelable((err: Error) => {
|
||||
if (err) {
|
||||
this.failDownload(downloadErrors.connectionError(err))
|
||||
} else {
|
||||
try {
|
||||
const confirmTokenRegex = /confirm=([0-9A-Za-z\-_]+)&/g
|
||||
const confirmTokenResults = confirmTokenRegex.exec(virusScanHTML)
|
||||
@@ -129,61 +139,57 @@ export class FileDownloader {
|
||||
const newHeader = `download_warning_${warningCode}=${confirmToken}; NID=${NID}`
|
||||
this.requestDownload(newHeader)
|
||||
} catch(e) {
|
||||
this.callbacks.error({ header: 'Invalid response', body: 'Download server returned HTML instead of a file.' }, () => this.beginDownload())
|
||||
this.failDownload(downloadErrors.htmlError())
|
||||
}
|
||||
} else {
|
||||
this.callbacks.error({ header: 'Connection Failed', body: `Connection failed while downloading HTML: ${err.name}` }, () => this.beginDownload())
|
||||
}
|
||||
})
|
||||
}))
|
||||
}
|
||||
|
||||
/**
|
||||
* Pipes the data from a download response to `fileName`.
|
||||
* Pipes the data from a download response to `this.fullPath`.
|
||||
* @param req The download request.
|
||||
* @param fileName The name of the output file.
|
||||
*/
|
||||
private handleDownloadResponse(req: NodeJS.ReadableStream, fileName: string) {
|
||||
private handleDownloadResponse(req: NodeJS.ReadableStream) {
|
||||
console.log('Download response...')
|
||||
this.callbacks.downloadProgress(0)
|
||||
let downloadedSize = 0
|
||||
const filePath = path.join(this.destinationFolder, fileName)
|
||||
req.pipe(fs.createWriteStream(filePath))
|
||||
req.on('data', (data) => {
|
||||
req.pipe(createWriteStream(this.fullPath))
|
||||
req.on('data', this.cancelable((data) => {
|
||||
downloadedSize += data.length
|
||||
this.callbacks.downloadProgress(downloadedSize)
|
||||
})
|
||||
}))
|
||||
|
||||
req.on('err', (err: Error) => {
|
||||
this.callbacks.error({ header: 'Connection Failed', body: `Connection failed while downloading file: ${err.name}` }, () => this.beginDownload())
|
||||
})
|
||||
req.on('err', this.cancelable((err: Error) => {
|
||||
this.failDownload(downloadErrors.connectionError(err))
|
||||
}))
|
||||
|
||||
req.on('end', () => {
|
||||
if (this.wasCanceled) { return } // CANCEL POINT
|
||||
req.on('end', this.cancelable(() => {
|
||||
this.callbacks.complete()
|
||||
})
|
||||
}))
|
||||
}
|
||||
|
||||
/**
|
||||
* Extracts the downloaded file's filename from `headers` or `this.url`, depending on the file's host server.
|
||||
* @param headers The response headers for this request.
|
||||
* Display an error message and provide a function to retry the download.
|
||||
*/
|
||||
private getDownloadFileName(headers: Headers) {
|
||||
if (headers['server'] && headers['server'] == 'cloudflare' || this.url.startsWith('https://public.fightthe.pw/')) {
|
||||
// Cloudflare and Chorus specific jazz
|
||||
return sanitizeFilename(decodeURIComponent(path.basename(this.url)))
|
||||
} else {
|
||||
// GDrive specific jazz
|
||||
const filenameRegex = /filename="(.*?)"/g
|
||||
const results = filenameRegex.exec(headers['content-disposition'])
|
||||
if (results == null) {
|
||||
console.log(`Warning: couldn't find filename in content-disposition header: [${headers['content-disposition']}]`)
|
||||
return 'unknownFilename'
|
||||
} else {
|
||||
return sanitizeFilename(results[1])
|
||||
}
|
||||
}
|
||||
private failDownload(error: DownloadError) {
|
||||
this.callbacks.error(error, this.cancelable(() => this.beginDownload()))
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop the process of downloading the file. (no more events will be fired after this is called)
|
||||
*/
|
||||
cancelDownload() {
|
||||
this.wasCanceled = true
|
||||
googleTimer.cancelTimer() // Prevents timer from trying to activate a download and resetting
|
||||
}
|
||||
|
||||
/**
|
||||
* Wraps a function that is able to be prevented if `this.cancelDownload()` was called.
|
||||
*/
|
||||
private cancelable<F extends AnyFunction>(fn: F) {
|
||||
return (...args: Parameters<F>): ReturnType<F> => {
|
||||
if (this.wasCanceled) { return }
|
||||
return fn(...Array.from(args))
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,163 +1,161 @@
|
||||
import { DownloadError } from './FileDownloader'
|
||||
import * as fs from 'fs'
|
||||
import { readdir, unlink, mkdir as _mkdir } from 'fs'
|
||||
import { promisify } from 'util'
|
||||
import { join, extname } from 'path'
|
||||
import { AnyFunction } from 'src/electron/shared/UtilFunctions'
|
||||
import * as node7z from 'node-7z'
|
||||
import * as zipBin from '7zip-bin'
|
||||
import { getSettings } from '../SettingsHandler.ipc'
|
||||
import { extractRar } from './RarExtractor'
|
||||
import * as unrarjs from 'node-unrar-js' // TODO find better rar library that has async extraction
|
||||
import { FailReason } from 'node-unrar-js/dist/js/extractor'
|
||||
import { DownloadError } from './ChartDownload'
|
||||
|
||||
const readdir = promisify(fs.readdir)
|
||||
const unlink = promisify(fs.unlink)
|
||||
const lstat = promisify(fs.lstat)
|
||||
const copyFile = promisify(fs.copyFile)
|
||||
const rmdir = promisify(fs.rmdir)
|
||||
const access = promisify(fs.access)
|
||||
const mkdir = promisify(fs.mkdir)
|
||||
const mkdir = promisify(_mkdir)
|
||||
|
||||
type EventCallback = {
|
||||
'extract': (filename: string) => void
|
||||
'start': (filename: string) => void
|
||||
'extractProgress': (percent: number, fileCount: number) => void
|
||||
'transfer': (filepath: string) => void
|
||||
'complete': (filepath: string) => void
|
||||
'error': (error: DownloadError, retry: () => void | Promise<void>) => void
|
||||
'error': (err: DownloadError, retry: () => void | Promise<void>) => void
|
||||
'complete': () => void
|
||||
}
|
||||
type Callbacks = { [E in keyof EventCallback]: EventCallback[E] }
|
||||
|
||||
const extractErrors = {
|
||||
readError: (err: NodeJS.ErrnoException) => { return { header: `Failed to read file (${err.code})`, body: `${err.name}: ${err.message}` } },
|
||||
emptyError: () => { return { header: 'Failed to extract archive', body: 'File archive was downloaded but could not be found' } },
|
||||
rarmkdirError: (err: NodeJS.ErrnoException, sourceFile: string) => {
|
||||
return { header: `Extracting archive failed. (${err.code})`, body: `${err.name}: ${err.message} (${sourceFile})`}
|
||||
},
|
||||
rarextractError: (result: { reason: FailReason; msg: string }, sourceFile: string) => {
|
||||
return { header: `Extracting archive failed: ${result.reason}`, body: `${result.msg} (${sourceFile})`}
|
||||
}
|
||||
}
|
||||
|
||||
export class FileExtractor {
|
||||
|
||||
private callbacks = {} as Callbacks
|
||||
private libraryFolder: string
|
||||
private wasCanceled = false
|
||||
constructor(private sourceFolder: string, private isArchive: boolean, private destinationFolderName: string) { }
|
||||
constructor(private sourceFolder: string) { }
|
||||
|
||||
/**
|
||||
* Calls `callback` when `event` fires.
|
||||
* Calls `callback` when `event` fires. (no events will be fired after `this.cancelDownload()` is called)
|
||||
*/
|
||||
on<E extends keyof EventCallback>(event: E, callback: EventCallback[E]) {
|
||||
this.callbacks[event] = callback
|
||||
}
|
||||
|
||||
/**
|
||||
* Starts the chart extraction process.
|
||||
* Extract the chart from `this.sourceFolder`. (assumes there is exactly one archive file in that folder)
|
||||
*/
|
||||
async beginExtract() {
|
||||
this.libraryFolder = getSettings().libraryPath
|
||||
const files = await readdir(this.sourceFolder)
|
||||
if (this.isArchive) {
|
||||
this.extract(files[0], extname(files[0]) == '.rar')
|
||||
} else {
|
||||
this.transfer()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Extracts the file at `filename` to `this.sourceFolder`.
|
||||
*/
|
||||
private async extract(filename: string, useRarExtractor: boolean) {
|
||||
await new Promise<void>(resolve => setTimeout(() => resolve(), 100)) // Wait for filesystem to process downloaded file...
|
||||
if (this.wasCanceled) { return } // CANCEL POINT
|
||||
this.callbacks.extract(filename)
|
||||
const source = join(this.sourceFolder, filename)
|
||||
|
||||
if (useRarExtractor) {
|
||||
|
||||
// Use node-unrar-js to extract the archive
|
||||
try {
|
||||
await extractRar(source, this.sourceFolder)
|
||||
} catch (err) {
|
||||
this.callbacks.error({
|
||||
header: 'Extract Failed.',
|
||||
body: `Unable to extract [${filename}]: ${err}`
|
||||
}, () => this.extract(filename, extname(filename) == '.rar'))
|
||||
return
|
||||
}
|
||||
this.transfer(source)
|
||||
|
||||
} else {
|
||||
|
||||
// Use node-7z to extract the archive
|
||||
const stream = node7z.extractFull(source, this.sourceFolder, { $progress: true, $bin: zipBin.path7za })
|
||||
|
||||
stream.on('progress', (progress: { percent: number; fileCount: number }) => {
|
||||
this.callbacks.extractProgress(progress.percent, progress.fileCount)
|
||||
})
|
||||
|
||||
let extractErrorOccured = false
|
||||
stream.on('error', () => {
|
||||
extractErrorOccured = true
|
||||
console.log(`Failed to extract [${filename}], retrying with .rar extractor...`)
|
||||
this.extract(filename, true)
|
||||
})
|
||||
|
||||
stream.on('end', () => {
|
||||
if (!extractErrorOccured) {
|
||||
this.transfer(source)
|
||||
beginExtract() {
|
||||
setTimeout(this.cancelable(() => {
|
||||
readdir(this.sourceFolder, (err, files) => {
|
||||
if (err) {
|
||||
this.callbacks.error(extractErrors.readError(err), () => this.beginExtract())
|
||||
} else if (files.length == 0) {
|
||||
this.callbacks.error(extractErrors.emptyError(), () => this.beginExtract())
|
||||
} else {
|
||||
this.callbacks.start(files[0])
|
||||
this.extract(join(this.sourceFolder, files[0]), extname(files[0]) == '.rar')
|
||||
}
|
||||
})
|
||||
}), 100) // Wait for filesystem to process downloaded file
|
||||
}
|
||||
|
||||
/**
|
||||
* Extracts the file at `fullPath` to `this.sourceFolder`.
|
||||
*/
|
||||
private async extract(fullPath: string, useRarExtractor: boolean) {
|
||||
if (useRarExtractor) {
|
||||
await this.extractRar(fullPath) // Use node-unrar-js to extract the archive
|
||||
} else {
|
||||
this.extract7z(fullPath) // Use node-7z to extract the archive
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Deletes the archive at `archiveFilepath`, then transfers the extracted chart to `this.libraryFolder`.
|
||||
* Extracts a .rar archive found at `fullPath` and puts the extracted results in `this.sourceFolder`.
|
||||
* @throws an `ExtractError` if this fails.
|
||||
*/
|
||||
private async transfer(archiveFilepath?: string) {
|
||||
// TODO: this fails if the extracted chart has nested folders
|
||||
// TODO: skip over "__MACOSX" folder
|
||||
// TODO: handle other common problems, like chart/audio files not named correctly
|
||||
if (this.wasCanceled) { return } // CANCEL POINT
|
||||
try {
|
||||
private async extractRar(fullPath: string) {
|
||||
const extractor = unrarjs.createExtractorFromFile(fullPath, this.sourceFolder)
|
||||
|
||||
// Create destiniation folder if it doesn't exist
|
||||
const destinationFolder = join(this.libraryFolder, this.destinationFolderName)
|
||||
this.callbacks.transfer(destinationFolder)
|
||||
try {
|
||||
await access(destinationFolder, fs.constants.F_OK)
|
||||
} catch (e) {
|
||||
await mkdir(destinationFolder)
|
||||
}
|
||||
const fileList = extractor.getFileList()
|
||||
|
||||
// Delete archive
|
||||
if (archiveFilepath != undefined) {
|
||||
try {
|
||||
await unlink(archiveFilepath)
|
||||
} catch (e) {
|
||||
if (e.code != 'ENOENT') {
|
||||
throw new Error(`Could not delete the archive file at [${archiveFilepath}]`)
|
||||
if (fileList[0].state != 'FAIL') {
|
||||
|
||||
// Create directories for nested archives (because unrarjs didn't feel like handling that automatically)
|
||||
const headers = fileList[1].fileHeaders
|
||||
for (const header of headers) {
|
||||
if (header.flags.directory) {
|
||||
try {
|
||||
await mkdir(join(this.sourceFolder, header.name), { recursive: true })
|
||||
} catch (err) {
|
||||
this.callbacks.error(extractErrors.rarmkdirError(err, fullPath), () => this.extract(fullPath, extname(fullPath) == '.rar'))
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Check if it extracted to a folder instead of a list of files
|
||||
let sourceFolder = this.sourceFolder
|
||||
let files = await readdir(sourceFolder)
|
||||
const isFolderArchive = (files.length < 2 && !(await lstat(join(sourceFolder, files[0]))).isFile())
|
||||
if (isFolderArchive) {
|
||||
sourceFolder = join(sourceFolder, files[0])
|
||||
files = await readdir(sourceFolder)
|
||||
}
|
||||
const extractResult = extractor.extractAll()
|
||||
|
||||
if (this.wasCanceled) { return } // CANCEL POINT
|
||||
|
||||
// Copy the files from the temporary directory to the destination
|
||||
for (const file of files) {
|
||||
await copyFile(join(sourceFolder, file), join(destinationFolder, file))
|
||||
await unlink(join(sourceFolder, file))
|
||||
}
|
||||
|
||||
// Delete the temporary folders
|
||||
await rmdir(sourceFolder)
|
||||
if (isFolderArchive) {
|
||||
await rmdir(join(sourceFolder, '..'))
|
||||
}
|
||||
this.callbacks.complete(destinationFolder)
|
||||
} catch (e) {
|
||||
this.callbacks.error({ header: 'Transfer Failed', body: `Unable to transfer downloaded files to the library folder: ${e.name}` }, () => this.transfer(archiveFilepath))
|
||||
if (extractResult[0].state == 'FAIL') {
|
||||
this.callbacks.error(extractErrors.rarextractError(extractResult[0], fullPath), () => this.extract(fullPath, extname(fullPath) == '.rar'))
|
||||
} else {
|
||||
this.deleteArchive(fullPath)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Extracts a .zip or .7z archive found at `fullPath` and puts the extracted results in `this.sourceFolder`.
|
||||
*/
|
||||
private extract7z(fullPath: string) {
|
||||
const stream = node7z.extractFull(fullPath, this.sourceFolder, { $progress: true, $bin: zipBin.path7za })
|
||||
|
||||
stream.on('progress', this.cancelable((progress: { percent: number; fileCount: number }) => {
|
||||
this.callbacks.extractProgress(progress.percent, progress.fileCount)
|
||||
}))
|
||||
|
||||
let extractErrorOccured = false
|
||||
stream.on('error', this.cancelable(() => {
|
||||
extractErrorOccured = true
|
||||
console.log(`Failed to extract [${fullPath}]; retrying with .rar extractor...`)
|
||||
this.extract(fullPath, true)
|
||||
}))
|
||||
|
||||
stream.on('end', this.cancelable(() => {
|
||||
if (!extractErrorOccured) {
|
||||
this.deleteArchive(fullPath)
|
||||
}
|
||||
}))
|
||||
}
|
||||
|
||||
/**
|
||||
* Tries to delete the archive at `fullPath`.
|
||||
*/
|
||||
private deleteArchive(fullPath: string) {
|
||||
unlink(fullPath, this.cancelable((err) => {
|
||||
if (err && err.code != 'ENOENT') {
|
||||
console.log(`Warning: failed to delete archive at [${fullPath}]`)
|
||||
}
|
||||
|
||||
this.callbacks.complete()
|
||||
}))
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop the process of extracting the file. (no more events will be fired after this is called)
|
||||
*/
|
||||
cancelExtract() {
|
||||
this.wasCanceled = true
|
||||
}
|
||||
|
||||
/**
|
||||
* Wraps a function that is able to be prevented if `this.cancelExtract()` was called.
|
||||
*/
|
||||
private cancelable<F extends AnyFunction>(fn: F) {
|
||||
return (...args: Parameters<F>): ReturnType<F> => {
|
||||
if (this.wasCanceled) { return }
|
||||
return fn(...Array.from(args))
|
||||
}
|
||||
}
|
||||
}
|
||||
111
src/electron/ipc/download/FileTransfer.ts
Normal file
111
src/electron/ipc/download/FileTransfer.ts
Normal file
@@ -0,0 +1,111 @@
|
||||
import { Dirent, readdir as _readdir } from 'fs'
|
||||
import { promisify } from 'util'
|
||||
import { getSettings } from '../SettingsHandler.ipc'
|
||||
import * as mv from 'mv'
|
||||
import { join } from 'path'
|
||||
import * as _rimraf from 'rimraf'
|
||||
import { DownloadError } from './ChartDownload'
|
||||
|
||||
const readdir = promisify(_readdir)
|
||||
const rimraf = promisify(_rimraf)
|
||||
|
||||
type EventCallback = {
|
||||
'start': (destinationFolder: string) => void
|
||||
'error': (err: DownloadError, retry: () => void | Promise<void>) => void
|
||||
'complete': () => void
|
||||
}
|
||||
type Callbacks = { [E in keyof EventCallback]: EventCallback[E] }
|
||||
|
||||
const transferErrors = {
|
||||
readError: (err: NodeJS.ErrnoException) => fsError(err, 'Failed to read file'),
|
||||
deleteError: (err: NodeJS.ErrnoException) => fsError(err, 'Failed to delete file'),
|
||||
rimrafError: (err: NodeJS.ErrnoException) => fsError(err, 'Failed to delete folder'),
|
||||
mvError: (err: NodeJS.ErrnoException) => fsError(err, 'Failed to move folder to library')
|
||||
}
|
||||
|
||||
function fsError(err: NodeJS.ErrnoException, description: string) {
|
||||
return { header: `${description} (${err.code})`, body: `${err.name}: ${err.message}` }
|
||||
}
|
||||
|
||||
export class FileTransfer {
|
||||
|
||||
private callbacks = {} as Callbacks
|
||||
private wasCanceled = false
|
||||
private destinationFolder: string
|
||||
private nestedSourceFolder: string // The top-level folder that is copied to the library folder
|
||||
constructor(private sourceFolder: string, destinationFolderName: string) {
|
||||
this.destinationFolder = join(getSettings().libraryPath, destinationFolderName)
|
||||
this.nestedSourceFolder = sourceFolder
|
||||
}
|
||||
|
||||
/**
|
||||
* Calls `callback` when `event` fires. (no events will be fired after `this.cancelDownload()` is called)
|
||||
*/
|
||||
on<E extends keyof EventCallback>(event: E, callback: EventCallback[E]) {
|
||||
this.callbacks[event] = callback
|
||||
}
|
||||
|
||||
async beginTransfer() {
|
||||
this.callbacks.start(this.destinationFolder)
|
||||
await this.cleanFolder()
|
||||
if (this.wasCanceled) { return }
|
||||
this.moveFolder()
|
||||
}
|
||||
|
||||
/**
|
||||
* Fixes common problems with the download chart folder.
|
||||
*/
|
||||
private async cleanFolder() {
|
||||
let files: Dirent[]
|
||||
try {
|
||||
files = await readdir(this.nestedSourceFolder, { withFileTypes: true })
|
||||
} catch (err) {
|
||||
this.callbacks.error(transferErrors.readError(err), () => this.cleanFolder())
|
||||
return
|
||||
}
|
||||
|
||||
// Remove nested folders
|
||||
if (files.length == 1 && !files[0].isFile()) {
|
||||
this.nestedSourceFolder = join(this.nestedSourceFolder, files[0].name)
|
||||
await this.cleanFolder()
|
||||
return
|
||||
}
|
||||
|
||||
// Delete '__MACOSX' folder
|
||||
for (const file of files) {
|
||||
if (!file.isFile() && file.name == '__MACOSX') {
|
||||
try {
|
||||
await rimraf(join(this.nestedSourceFolder, file.name))
|
||||
} catch (err) {
|
||||
this.callbacks.error(transferErrors.rimrafError(err), () => this.cleanFolder())
|
||||
return
|
||||
}
|
||||
} else {
|
||||
// TODO: handle other common problems, like chart/audio files not named correctly
|
||||
// TODO: this could have been a chart pack; have option to only keep selected chart?
|
||||
// (think about a user trying to download a set of charts in a pack)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Moves the downloaded chart to the library path.
|
||||
*/
|
||||
private moveFolder() {
|
||||
mv(this.nestedSourceFolder, this.destinationFolder, { mkdirp: true }, (err) => {
|
||||
if (err) {
|
||||
this.callbacks.error(transferErrors.mvError(err), () => this.moveFolder())
|
||||
} else {
|
||||
rimraf(this.sourceFolder) // Delete temp folder
|
||||
this.callbacks.complete()
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop the process of transfering the file. (no more events will be fired after this is called)
|
||||
*/
|
||||
cancelTransfer() {
|
||||
this.wasCanceled = true
|
||||
}
|
||||
}
|
||||
@@ -1,49 +1,72 @@
|
||||
import { getSettings } from '../SettingsHandler.ipc'
|
||||
|
||||
type EventCallback = {
|
||||
'waitProgress': (remainingSeconds: number, totalSeconds: number) => void
|
||||
'complete': () => void
|
||||
}
|
||||
type Callbacks = { [E in keyof EventCallback]?: EventCallback[E] }
|
||||
|
||||
class GoogleTimer {
|
||||
|
||||
private rateLimitCounter = Infinity
|
||||
private onReadyCallback: () => void
|
||||
private updateCallback: (remainingTime: number, totalTime: number) => void
|
||||
private callbacks: Callbacks = {}
|
||||
|
||||
/**
|
||||
* Initializes the timer to call the callbacks if they are defined.
|
||||
*/
|
||||
constructor() {
|
||||
setInterval(() => {
|
||||
this.rateLimitCounter++
|
||||
if (this.isReady() && this.onReadyCallback != undefined) {
|
||||
this.activateTimerReady()
|
||||
} else if (this.updateCallback != undefined) {
|
||||
const delay = getSettings().rateLimitDelay
|
||||
this.updateCallback(delay - this.rateLimitCounter, delay)
|
||||
}
|
||||
this.updateCallbacks()
|
||||
}, 1000)
|
||||
}
|
||||
|
||||
onTimerReady(callback: () => void) {
|
||||
this.onReadyCallback = callback
|
||||
if (this.isReady()) {
|
||||
this.activateTimerReady()
|
||||
/**
|
||||
* Calls `callback` when `event` fires. (no events will be fired after `this.cancelTimer()` is called)
|
||||
*/
|
||||
on<E extends keyof EventCallback>(event: E, callback: EventCallback[E]) {
|
||||
this.callbacks[event] = callback
|
||||
this.updateCallbacks() // Fire events immediately after the listeners have been added
|
||||
}
|
||||
|
||||
/**
|
||||
* Check the state of the callbacks and call them if necessary.
|
||||
*/
|
||||
private updateCallbacks() {
|
||||
if (this.hasTimerEnded() && this.callbacks.complete != undefined) {
|
||||
this.endTimer()
|
||||
} else if (this.callbacks.waitProgress != undefined) {
|
||||
const delay = getSettings().rateLimitDelay
|
||||
this.callbacks.waitProgress(delay - this.rateLimitCounter, delay)
|
||||
}
|
||||
}
|
||||
|
||||
onTimerUpdate(callback: (remainingTime: number, totalTime: number) => void) {
|
||||
this.updateCallback = callback
|
||||
/**
|
||||
* Prevents the callbacks from activating when the timer ends.
|
||||
*/
|
||||
cancelTimer() {
|
||||
this.callbacks = {}
|
||||
}
|
||||
|
||||
removeCallbacks() {
|
||||
this.onReadyCallback = undefined
|
||||
this.updateCallback = undefined
|
||||
}
|
||||
|
||||
private isReady() {
|
||||
/**
|
||||
* Checks if enough time has elapsed since the last timer activation.
|
||||
*/
|
||||
private hasTimerEnded() {
|
||||
return this.rateLimitCounter > getSettings().rateLimitDelay
|
||||
}
|
||||
|
||||
private activateTimerReady() {
|
||||
/**
|
||||
* Activates the completion callback and resets the timer.
|
||||
*/
|
||||
private endTimer() {
|
||||
this.rateLimitCounter = 0
|
||||
const onReadyCallback = this.onReadyCallback
|
||||
this.removeCallbacks()
|
||||
onReadyCallback()
|
||||
const completeCallback = this.callbacks.complete
|
||||
this.callbacks = {}
|
||||
completeCallback()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Important: this instance cannot be used by more than one file download at a time.
|
||||
*/
|
||||
export const googleTimer = new GoogleTimer()
|
||||
@@ -1,39 +0,0 @@
|
||||
import * as fs from 'fs'
|
||||
import { join } from 'path'
|
||||
import * as unrarjs from 'node-unrar-js'
|
||||
import { promisify } from 'util'
|
||||
|
||||
const mkdir = promisify(fs.mkdir)
|
||||
|
||||
/**
|
||||
* Extracts the archive at `sourceFile` to a new folder in `destinationFolder`. Throws an error when this fails.
|
||||
*/
|
||||
export async function extractRar(sourceFile: string, destinationFolder: string) {
|
||||
const extractor = unrarjs.createExtractorFromFile(sourceFile, destinationFolder)
|
||||
|
||||
const fileList = extractor.getFileList()
|
||||
|
||||
if (fileList[0].state != 'FAIL') {
|
||||
|
||||
// Create directories for nested archives (because unrarjs didn't feel like handling that automatically)
|
||||
const headers = fileList[1].fileHeaders
|
||||
for (const header of headers) {
|
||||
if (header.flags.directory) {
|
||||
try {
|
||||
await mkdir(join(destinationFolder, header.name), { recursive: true })
|
||||
} catch (e) {
|
||||
throw new Error(`Failed to extract directory: ${e}`)
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
console.log('Warning: failed to read .rar files: ', fileList[0].reason, fileList[0].msg)
|
||||
}
|
||||
|
||||
// Extract archive
|
||||
const extractResult = extractor.extractAll()
|
||||
|
||||
if (extractResult[0].state == 'FAIL') {
|
||||
throw new Error(`${extractResult[0].reason}: ${extractResult[0].msg}`)
|
||||
}
|
||||
}
|
||||
@@ -1,23 +1,7 @@
|
||||
const sanitize = require('sanitize-filename')
|
||||
|
||||
/**
|
||||
* @returns a random UUID
|
||||
*/
|
||||
export function generateUUID() { // Public Domain/MIT
|
||||
let d = new Date().getTime() // Timestamp
|
||||
let d2 = Date.now() // Time in microseconds since page-load or 0 if unsupported
|
||||
return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, (c) => {
|
||||
let r = Math.random() * 16 // Random number between 0 and 16
|
||||
if (d > 0) { // Use timestamp until depleted
|
||||
r = (d + r) % 16 | 0
|
||||
d = Math.floor(d / 16)
|
||||
} else { // Use microseconds since page-load if supported
|
||||
r = (d2 + r) % 16 | 0
|
||||
d2 = Math.floor(d2 / 16)
|
||||
}
|
||||
return (c === 'x' ? r : (r & 0x3 | 0x8)).toString(16)
|
||||
})
|
||||
}
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
export type AnyFunction = (...args: any) => any
|
||||
|
||||
/**
|
||||
* @returns `filename`, but with any invalid filename characters replaced with similar valid characters.
|
||||
@@ -37,10 +21,10 @@ export function sanitizeFilename(filename: string): string {
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts `val` from the range (`fromA`, `fromB`) to the range (`toA`, `toB`).
|
||||
* Converts `val` from the range (`fromStart`, `fromEnd`) to the range (`toStart`, `toEnd`).
|
||||
*/
|
||||
export function interpolate(val: number, fromA: number, fromB: number, toA: number, toB: number) {
|
||||
return ((val - fromA) / (fromB - fromA)) * (toB - toA) + toA
|
||||
export function interpolate(val: number, fromStart: number, fromEnd: number, toStart: number, toEnd: number) {
|
||||
return ((val - fromStart) / (fromEnd - fromStart)) * (toEnd - toStart) + toStart
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -4,7 +4,7 @@ import { DriveChart } from './songDetails.interface'
|
||||
* Represents a user's request to interact with the download system.
|
||||
*/
|
||||
export interface Download {
|
||||
action: 'add' | 'retry' | 'continue' | 'cancel'
|
||||
action: 'add' | 'retry' | 'cancel'
|
||||
versionID: number
|
||||
data?: NewDownload // Should be defined if action == 'add'
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user