diff --git a/src/app/core/services/download.service.ts b/src/app/core/services/download.service.ts index d3e5950..99ff74c 100644 --- a/src/app/core/services/download.service.ts +++ b/src/app/core/services/download.service.ts @@ -47,6 +47,8 @@ export class DownloadService { if (this.downloads.findIndex(oldDownload => oldDownload.versionID == download.versionID) == -1) { // If this is a new download item, don't call debouncedCallback; it may miss adding new versions to the list callback(download) + } else if (download.type == 'wait') { + callback(download) // Many wait events can be recieved at once } else { debouncedCallback(download) } diff --git a/src/electron/ipc/download/DownloadHandler.ts b/src/electron/ipc/download/DownloadHandler.ts index 4e42f1c..c5c2b48 100644 --- a/src/electron/ipc/download/DownloadHandler.ts +++ b/src/electron/ipc/download/DownloadHandler.ts @@ -47,23 +47,23 @@ export class DownloadHandler implements IPCEmitHandler<'download'> { const individualFileProgressPortion = 80 / fileKeys.length for (let i = 0; i < fileKeys.length; i++) { const typeHash = createHash('md5').update(data.data.links[fileKeys[i]]).digest('hex') - const downloader = new FileDownloader(data.data.links[fileKeys[i]], chartPath, data.data.links[typeHash]) + const downloader = await FileDownloader.asyncConstructor(data.data.links[fileKeys[i]], chartPath, fileKeys.length, data.data.links[typeHash]) this.downloadCallbacks[data.versionID].cancel = () => downloader.cancelDownload() // Make cancel button cancel this download let fileProgress = 0 - let waitTime: number - downloader.on('wait', (_waitTime) => { + let initialWaitTime: number + downloader.on('wait', (waitTime) => { download.header = `[${fileKeys[i]}] (file ${i + 1}/${fileKeys.length})` - download.description = `Waiting for Google rate limit... (${_waitTime}s)` - download.type = 'good' - waitTime = _waitTime + download.description = `Waiting for Google rate limit... (${waitTime}s)` + download.type = 'wait' + initialWaitTime = waitTime }) downloader.on('waitProgress', (secondsRemaining) => { download.description = `Waiting for Google rate limit... (${secondsRemaining}s)` - fileProgress = interpolate(secondsRemaining, waitTime, 0, 0, individualFileProgressPortion / 2) + fileProgress = interpolate(secondsRemaining, initialWaitTime, 0, 0, individualFileProgressPortion / 2) download.percent = allFilesProgress + fileProgress - download.type = 'good' + download.type = 'wait' emitIPCEvent('download-updated', download) }) diff --git a/src/electron/ipc/download/FileDownloader.ts b/src/electron/ipc/download/FileDownloader.ts index bfd2719..a0db389 100644 --- a/src/electron/ipc/download/FileDownloader.ts +++ b/src/electron/ipc/download/FileDownloader.ts @@ -3,6 +3,8 @@ import * as fs from 'fs' import * as path from 'path' import * as needle from 'needle' import InitSettingsHandler from '../InitSettingsHandler.ipc' +import { Settings } from 'src/electron/shared/Settings' + type EventCallback = { 'wait': (waitTime: number) => void 'waitProgress': (secondsRemaining: number) => void @@ -20,17 +22,39 @@ export type DownloadError = { header: string, body: string } export class FileDownloader { private RATE_LIMIT_DELAY: number private readonly RETRY_MAX = 2 - private static waitTime = 0 - private static clock: NodeJS.Timer + private static fileQueue: { // Stores the overall order that files should be downloaded + destinationFolder: string + fileCount: number + clock?: () => void + }[] = [] + private static waitTime: number + private static settings: Settings private callbacks = {} as Callbacks private retryCount: number private wasCanceled = false - constructor(private url: string, private destinationFolder: string, private expectedHash?: string) { - if (FileDownloader.clock == undefined) { - FileDownloader.clock = setInterval(() => FileDownloader.waitTime = Math.max(0, FileDownloader.waitTime - 1), 1000) + private constructor(private url: string, private destinationFolder: string, private numFiles: number, private expectedHash?: string) { } + static async asyncConstructor(url: string, destinationFolder: string, numFiles: number, expectedHash?: string) { + const downloader = new FileDownloader(url, destinationFolder, numFiles, expectedHash) + if (FileDownloader.settings == undefined) { + await downloader.firstInit() } + return downloader + } + + async firstInit() { + FileDownloader.settings = await InitSettingsHandler.getSettings() + FileDownloader.waitTime = 0 + setInterval(() => { + if (FileDownloader.waitTime > 0) { + FileDownloader.waitTime-- + } + FileDownloader.fileQueue.forEach(download => { if (download.clock != undefined) download.clock() }) + if (FileDownloader.waitTime == 0 && FileDownloader.fileQueue.length != 0) { + FileDownloader.waitTime = this.RATE_LIMIT_DELAY + } + }, 1000) } /** @@ -46,30 +70,69 @@ export class FileDownloader { * Wait RATE_LIMIT_DELAY seconds between each download, * then download the file. */ - async beginDownload() { - const settings = await InitSettingsHandler.getSettings() - if (settings.libraryPath == undefined) { - this.callbacks.error({header: 'Library folder not specified', body: 'Please go to the settings to set your library folder.'}, () => this.beginDownload()) + beginDownload() { + // Check that the library folder has been specified + if (FileDownloader.settings.libraryPath == undefined) { + this.callbacks.error({ header: 'Library folder not specified', body: 'Please go to the settings to set your library folder.' }, () => this.beginDownload()) return } - this.RATE_LIMIT_DELAY = (await InitSettingsHandler.getSettings()).rateLimitDelay - let waitTime = FileDownloader.waitTime - if (this.url.toLocaleLowerCase().includes('google')) { - FileDownloader.waitTime += this.RATE_LIMIT_DELAY - } else { - waitTime = 0 // Don't rate limit if not downloading from Google + + // Skip the fileQueue if the file is not from Google + if (!this.url.toLocaleLowerCase().includes('google')) { + this.requestDownload() + return } - this.callbacks.wait(waitTime) - const clock = setInterval(() => { - if (this.wasCanceled) { clearInterval(clock); return } // CANCEL POINT - waitTime-- - this.callbacks.waitProgress(waitTime) - if (waitTime <= 0) { - this.retryCount = 0 + + this.initWaitTime() + let queueWaitTime = this.getQueueWaitTime() + this.callbacks.wait(queueWaitTime + FileDownloader.waitTime) + if (queueWaitTime + FileDownloader.waitTime == 0) { + FileDownloader.waitTime = this.RATE_LIMIT_DELAY + this.requestDownload() + return + } + + const fileQueue = FileDownloader.fileQueue.find(queue => queue.destinationFolder == this.destinationFolder) + fileQueue.clock = () => { + if (this.wasCanceled) { this.removeFromQueue(); return } // CANCEL POINT + queueWaitTime = this.getQueueWaitTime() + if (queueWaitTime + FileDownloader.waitTime == 0) { this.requestDownload() - clearInterval(clock) + fileQueue.clock = undefined } - }, 1000) + this.callbacks.waitProgress(queueWaitTime + FileDownloader.waitTime) + } + } + + private initWaitTime() { + this.RATE_LIMIT_DELAY = FileDownloader.settings.rateLimitDelay + this.retryCount = 0 + const entry = FileDownloader.fileQueue.find(entry => entry.destinationFolder == this.destinationFolder) + if (entry == undefined) { + // Note: assumes that either all the chart files are from Google, or none of the chart files are from Google + FileDownloader.fileQueue.push({ destinationFolder: this.destinationFolder, fileCount: this.numFiles }) + } + } + + /** + * Returns the number of files in front of this file in the fileQueue + */ + private getQueueWaitTime() { + let fileCount = 0 + for (let entry of FileDownloader.fileQueue) { + if (entry.destinationFolder != this.destinationFolder) { + fileCount += entry.fileCount + } else { + break + } + } + + return fileCount * this.RATE_LIMIT_DELAY + } + + private removeFromQueue() { + const index = FileDownloader.fileQueue.findIndex(entry => entry.destinationFolder == this.destinationFolder) + FileDownloader.fileQueue.splice(index, 1) } /** @@ -77,7 +140,7 @@ export class FileDownloader { * @param cookieHeader the "cookie=" header to include this request. */ private requestDownload(cookieHeader?: string) { - if (this.wasCanceled) { return } // CANCEL POINT + if (this.wasCanceled) { this.removeFromQueue(); return } // CANCEL POINT this.callbacks.request() let uuid = generateUUID() const req = needle.get(this.url, { @@ -108,7 +171,7 @@ export class FileDownloader { }) req.on('header', (statusCode, headers: Headers) => { - if (this.wasCanceled) { return } // CANCEL POINT + if (this.wasCanceled) { this.removeFromQueue(); return } // CANCEL POINT if (statusCode != 200) { this.callbacks.error({ header: 'Connection failed', body: `Server returned status code: ${statusCode}` }, () => this.beginDownload()) return @@ -185,6 +248,11 @@ export class FileDownloader { req.on('end', () => { this.callbacks.complete() + const index = FileDownloader.fileQueue.findIndex(entry => entry.destinationFolder == this.destinationFolder) + FileDownloader.fileQueue[index].fileCount-- + if (FileDownloader.fileQueue[index].fileCount == 0) { + FileDownloader.fileQueue.splice(index, 1) + } }) } diff --git a/src/electron/shared/interfaces/download.interface.ts b/src/electron/shared/interfaces/download.interface.ts index 9334233..ca0d0dd 100644 --- a/src/electron/shared/interfaces/download.interface.ts +++ b/src/electron/shared/interfaces/download.interface.ts @@ -23,5 +23,5 @@ export interface DownloadProgress { header: string description: string percent: number - type: 'good' | 'warning' | 'error' | 'cancel' | 'done' + type: 'good' | 'warning' | 'error' | 'cancel' | 'done' | 'wait' } \ No newline at end of file