Rate limit wait period now adjusts correctly

src/app/core/services/download.service.ts

@@ -47,6 +47,8 @@ export class DownloadService {
       if (this.downloads.findIndex(oldDownload => oldDownload.versionID == download.versionID) == -1) {
         // If this is a new download item, don't call debouncedCallback; it may miss adding new versions to the list
         callback(download)
+      } else if (download.type == 'wait') {
+        callback(download) // Many wait events can be recieved at once
       } else {
         debouncedCallback(download)
       }

src/electron/ipc/download/DownloadHandler.ts

@@ -47,23 +47,23 @@ export class DownloadHandler implements IPCEmitHandler<'download'> {
     const individualFileProgressPortion = 80 / fileKeys.length
     for (let i = 0; i < fileKeys.length; i++) {
       const typeHash = createHash('md5').update(data.data.links[fileKeys[i]]).digest('hex')
-      const downloader = new FileDownloader(data.data.links[fileKeys[i]], chartPath, data.data.links[typeHash])
+      const downloader = await FileDownloader.asyncConstructor(data.data.links[fileKeys[i]], chartPath, fileKeys.length, data.data.links[typeHash])
       this.downloadCallbacks[data.versionID].cancel = () => downloader.cancelDownload() // Make cancel button cancel this download
       let fileProgress = 0
 
-      let waitTime: number
+      let initialWaitTime: number
-      downloader.on('wait', (_waitTime) => {
+      downloader.on('wait', (waitTime) => {
         download.header = `[${fileKeys[i]}] (file ${i + 1}/${fileKeys.length})`
-        download.description = `Waiting for Google rate limit... (${_waitTime}s)`
+        download.description = `Waiting for Google rate limit... (${waitTime}s)`
-        download.type = 'good'
+        download.type = 'wait'
-        waitTime = _waitTime
+        initialWaitTime = waitTime
       })
 
       downloader.on('waitProgress', (secondsRemaining) => {
         download.description = `Waiting for Google rate limit... (${secondsRemaining}s)`
-        fileProgress = interpolate(secondsRemaining, waitTime, 0, 0, individualFileProgressPortion / 2)
+        fileProgress = interpolate(secondsRemaining, initialWaitTime, 0, 0, individualFileProgressPortion / 2)
         download.percent = allFilesProgress + fileProgress
-        download.type = 'good'
+        download.type = 'wait'
         emitIPCEvent('download-updated', download)
       })
 

src/electron/ipc/download/FileDownloader.ts

@@ -3,6 +3,8 @@ import * as fs from 'fs'
 import * as path from 'path'
 import * as needle from 'needle'
 import InitSettingsHandler from '../InitSettingsHandler.ipc'
+import { Settings } from 'src/electron/shared/Settings'
+
 type EventCallback = {
   'wait': (waitTime: number) => void
   'waitProgress': (secondsRemaining: number) => void
@@ -20,17 +22,39 @@ export type DownloadError = { header: string, body: string }
 export class FileDownloader {
   private RATE_LIMIT_DELAY: number
   private readonly RETRY_MAX = 2
-  private static waitTime = 0
+  private static fileQueue: { // Stores the overall order that files should be downloaded
-  private static clock: NodeJS.Timer
+    destinationFolder: string
+    fileCount: number
+    clock?: () => void
+  }[] = []
+  private static waitTime: number
+  private static settings: Settings
 
   private callbacks = {} as Callbacks
   private retryCount: number
   private wasCanceled = false
 
-  constructor(private url: string, private destinationFolder: string, private expectedHash?: string) {
+  private constructor(private url: string, private destinationFolder: string, private numFiles: number, private expectedHash?: string) { }
-    if (FileDownloader.clock == undefined) {
+  static async asyncConstructor(url: string, destinationFolder: string, numFiles: number, expectedHash?: string) {
-      FileDownloader.clock = setInterval(() => FileDownloader.waitTime = Math.max(0, FileDownloader.waitTime - 1), 1000)
+    const downloader = new FileDownloader(url, destinationFolder, numFiles, expectedHash)
+    if (FileDownloader.settings == undefined) {
+      await downloader.firstInit()
     }
+    return downloader
+  }
+
+  async firstInit() {
+    FileDownloader.settings = await InitSettingsHandler.getSettings()
+    FileDownloader.waitTime = 0
+    setInterval(() => {
+      if (FileDownloader.waitTime > 0) {
+        FileDownloader.waitTime--
+      }
+      FileDownloader.fileQueue.forEach(download => { if (download.clock != undefined) download.clock() })
+      if (FileDownloader.waitTime == 0 && FileDownloader.fileQueue.length != 0) {
+        FileDownloader.waitTime = this.RATE_LIMIT_DELAY
+      }
+    }, 1000)
   }
 
   /**
@@ -46,30 +70,69 @@ export class FileDownloader {
    * Wait RATE_LIMIT_DELAY seconds between each download,
    * then download the file.
    */
-  async beginDownload() {
+  beginDownload() {
-    const settings = await InitSettingsHandler.getSettings()
+    // Check that the library folder has been specified
-    if (settings.libraryPath == undefined) {
+    if (FileDownloader.settings.libraryPath == undefined) {
-      this.callbacks.error({header: 'Library folder not specified', body: 'Please go to the settings to set your library folder.'}, () => this.beginDownload())
+      this.callbacks.error({ header: 'Library folder not specified', body: 'Please go to the settings to set your library folder.' }, () => this.beginDownload())
       return
     }
-    this.RATE_LIMIT_DELAY = (await InitSettingsHandler.getSettings()).rateLimitDelay
+
-    let waitTime = FileDownloader.waitTime
+    // Skip the fileQueue if the file is not from Google
-    if (this.url.toLocaleLowerCase().includes('google')) {
+    if (!this.url.toLocaleLowerCase().includes('google')) {
-      FileDownloader.waitTime += this.RATE_LIMIT_DELAY
+      this.requestDownload()
-    } else {
+      return
-      waitTime = 0 // Don't rate limit if not downloading from Google
     }
-    this.callbacks.wait(waitTime)
+
-    const clock = setInterval(() => {
+    this.initWaitTime()
-      if (this.wasCanceled) { clearInterval(clock); return } // CANCEL POINT
+    let queueWaitTime = this.getQueueWaitTime()
-      waitTime--
+    this.callbacks.wait(queueWaitTime + FileDownloader.waitTime)
-      this.callbacks.waitProgress(waitTime)
+    if (queueWaitTime + FileDownloader.waitTime == 0) {
-      if (waitTime <= 0) {
+      FileDownloader.waitTime = this.RATE_LIMIT_DELAY
-        this.retryCount = 0
+      this.requestDownload()
+      return
+    }
+
+    const fileQueue = FileDownloader.fileQueue.find(queue => queue.destinationFolder == this.destinationFolder)
+    fileQueue.clock = () => {
+      if (this.wasCanceled) { this.removeFromQueue(); return } // CANCEL POINT
+      queueWaitTime = this.getQueueWaitTime()
+      if (queueWaitTime + FileDownloader.waitTime == 0) {
         this.requestDownload()
-        clearInterval(clock)
+        fileQueue.clock = undefined
       }
-    }, 1000)
+      this.callbacks.waitProgress(queueWaitTime + FileDownloader.waitTime)
+    }
+  }
+
+  private initWaitTime() {
+    this.RATE_LIMIT_DELAY = FileDownloader.settings.rateLimitDelay
+    this.retryCount = 0
+    const entry = FileDownloader.fileQueue.find(entry => entry.destinationFolder == this.destinationFolder)
+    if (entry == undefined) {
+      // Note: assumes that either all the chart files are from Google, or none of the chart files are from Google
+      FileDownloader.fileQueue.push({ destinationFolder: this.destinationFolder, fileCount: this.numFiles })
+    }
+  }
+
+  /**
+   * Returns the number of files in front of this file in the fileQueue
+   */
+  private getQueueWaitTime() {
+    let fileCount = 0
+    for (let entry of FileDownloader.fileQueue) {
+      if (entry.destinationFolder != this.destinationFolder) {
+        fileCount += entry.fileCount
+      } else {
+        break
+      }
+    }
+
+    return fileCount * this.RATE_LIMIT_DELAY
+  }
+
+  private removeFromQueue() {
+    const index = FileDownloader.fileQueue.findIndex(entry => entry.destinationFolder == this.destinationFolder)
+    FileDownloader.fileQueue.splice(index, 1)
   }
 
   /**
@@ -77,7 +140,7 @@ export class FileDownloader {
    * @param cookieHeader the "cookie=" header to include this request.
    */
   private requestDownload(cookieHeader?: string) {
-    if (this.wasCanceled) { return } // CANCEL POINT
+    if (this.wasCanceled) { this.removeFromQueue(); return } // CANCEL POINT
     this.callbacks.request()
     let uuid = generateUUID()
     const req = needle.get(this.url, {
@@ -108,7 +171,7 @@ export class FileDownloader {
     })
 
     req.on('header', (statusCode, headers: Headers) => {
-      if (this.wasCanceled) { return } // CANCEL POINT
+      if (this.wasCanceled) { this.removeFromQueue(); return } // CANCEL POINT
       if (statusCode != 200) {
         this.callbacks.error({ header: 'Connection failed', body: `Server returned status code: ${statusCode}` }, () => this.beginDownload())
         return
@@ -185,6 +248,11 @@ export class FileDownloader {
 
     req.on('end', () => {
       this.callbacks.complete()
+      const index = FileDownloader.fileQueue.findIndex(entry => entry.destinationFolder == this.destinationFolder)
+      FileDownloader.fileQueue[index].fileCount--
+      if (FileDownloader.fileQueue[index].fileCount == 0) {
+        FileDownloader.fileQueue.splice(index, 1)
+      }
     })
   }
 

src/electron/shared/interfaces/download.interface.ts

@@ -23,5 +23,5 @@ export interface DownloadProgress {
   header: string
   description: string
   percent: number
-  type: 'good' | 'warning' | 'error' | 'cancel' | 'done'
+  type: 'good' | 'warning' | 'error' | 'cancel' | 'done' | 'wait'
 }