Refactor download queue

package-lock.json

@@ -2124,15 +2124,6 @@
         "defer-to-connect": "^1.0.1"
       }
     },
-    "@types/better-queue": {
-      "version": "3.8.2",
-      "resolved": "https://registry.npmjs.org/@types/better-queue/-/better-queue-3.8.2.tgz",
-      "integrity": "sha512-KMFFgojmy10+npJiw/XkY2i+UD96NZmo4L0xuw8DRfEXB4a70rS6YHzqpb7Xgh1+TwWFsIHbsK8fFkvMTxEASA==",
-      "dev": true,
-      "requires": {
-        "@types/node": "*"
-      }
-    },
     "@types/cli-color": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/@types/cli-color/-/cli-color-2.0.0.tgz",
@@ -2195,6 +2186,12 @@
       "integrity": "sha512-tHq6qdbT9U1IRSGf14CL0pUlULksvY9OZ+5eEgl1N7t+OA3tGvNpxJCzuKQlsNgCVwbAs670L1vcVQi8j9HjnA==",
       "dev": true
     },
+    "@types/mv": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/@types/mv/-/mv-2.1.0.tgz",
+      "integrity": "sha512-9uDB9lojfIQipxfI308//b4c5isHs6uMo3kIMzv73FPdXmMnAk6iELHGI849cuuDPHy6aXBwN/q9gMzjRyhJ+w==",
+      "dev": true
+    },
     "@types/needle": {
       "version": "2.0.4",
       "resolved": "https://registry.npmjs.org/@types/needle/-/needle-2.0.4.tgz",
@@ -2215,6 +2212,16 @@
       "integrity": "sha512-ce5d3q03Ex0sy4R14722Rmt6MT07Ua+k4FwDfdcToYJcMKNtRVQvJ6JCAPdAmAnbRb6CsX6aYb9m96NGod9uTw==",
       "dev": true
     },
+    "@types/rimraf": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/@types/rimraf/-/rimraf-3.0.0.tgz",
+      "integrity": "sha512-7WhJ0MdpFgYQPXlF4Dx+DhgvlPCfz/x5mHaeDQAKhcenvQP1KCpLQ18JklAqeGMYSAT2PxLpzd0g2/HE7fj7hQ==",
+      "dev": true,
+      "requires": {
+        "@types/glob": "*",
+        "@types/node": "*"
+      }
+    },
     "@types/source-list-map": {
       "version": "0.1.2",
       "resolved": "https://registry.npmjs.org/@types/source-list-map/-/source-list-map-0.1.2.tgz",
@@ -3353,21 +3360,6 @@
         }
       }
     },
-    "better-queue": {
-      "version": "3.8.10",
-      "resolved": "https://registry.npmjs.org/better-queue/-/better-queue-3.8.10.tgz",
-      "integrity": "sha512-e3gwNZgDCnNWl0An0Tz6sUjKDV9m6aB+K9Xg//vYeo8+KiH8pWhLFxkawcXhm6FpM//GfD9IQv/kmvWCAVVpKA==",
-      "requires": {
-        "better-queue-memory": "^1.0.1",
-        "node-eta": "^0.9.0",
-        "uuid": "^3.0.0"
-      }
-    },
-    "better-queue-memory": {
-      "version": "1.0.4",
-      "resolved": "https://registry.npmjs.org/better-queue-memory/-/better-queue-memory-1.0.4.tgz",
-      "integrity": "sha512-SWg5wFIShYffEmJpI6LgbL8/3Dqhku7xI1oEiy6FroP9DbcZlG0ZDjxvPdP9t7hTGW40IpIcC6zVoGT1oxjOuA=="
-    },
     "big.js": {
       "version": "5.2.2",
       "resolved": "https://registry.npmjs.org/big.js/-/big.js-5.2.2.tgz",
@@ -4388,6 +4380,11 @@
       "integrity": "sha1-3dgA2gxmEnOTzKWVDqloo6rxJTs=",
       "dev": true
     },
+    "comparators": {
+      "version": "3.0.2",
+      "resolved": "https://registry.npmjs.org/comparators/-/comparators-3.0.2.tgz",
+      "integrity": "sha512-Tfum570X4APe7vEQdDjoMZssxeyW4xuH/oYbI/fXGwJZ/rIhzBrfykfHf6LUVhk0GzeFhI6oYuOr1y7PRV7tng=="
+    },
     "component-emitter": {
       "version": "1.3.0",
       "resolved": "https://registry.npmjs.org/component-emitter/-/component-emitter-1.3.0.tgz",
@@ -11679,6 +11676,38 @@
       "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.8.tgz",
       "integrity": "sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA=="
     },
+    "mv": {
+      "version": "2.1.1",
+      "resolved": "https://registry.npmjs.org/mv/-/mv-2.1.1.tgz",
+      "integrity": "sha1-rmzg1vbV4KT32JN5jQPB6pVZtqI=",
+      "requires": {
+        "mkdirp": "~0.5.1",
+        "ncp": "~2.0.0",
+        "rimraf": "~2.4.0"
+      },
+      "dependencies": {
+        "glob": {
+          "version": "6.0.4",
+          "resolved": "https://registry.npmjs.org/glob/-/glob-6.0.4.tgz",
+          "integrity": "sha1-DwiGD2oVUSey+t1PnOJLGqtuTSI=",
+          "requires": {
+            "inflight": "^1.0.4",
+            "inherits": "2",
+            "minimatch": "2 || 3",
+            "once": "^1.3.0",
+            "path-is-absolute": "^1.0.0"
+          }
+        },
+        "rimraf": {
+          "version": "2.4.5",
+          "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.4.5.tgz",
+          "integrity": "sha1-7nEM5dk6j9uFb7Xqj/Di11k0sto=",
+          "requires": {
+            "glob": "^6.0.1"
+          }
+        }
+      }
+    },
     "nan": {
       "version": "2.14.1",
       "resolved": "https://registry.npmjs.org/nan/-/nan-2.14.1.tgz",
@@ -11709,6 +11738,11 @@
       "integrity": "sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc=",
       "dev": true
     },
+    "ncp": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/ncp/-/ncp-2.0.0.tgz",
+      "integrity": "sha1-GVoh1sRuNh0vsSgbo4uR6d9727M="
+    },
     "needle": {
       "version": "2.3.2",
       "resolved": "https://registry.npmjs.org/needle/-/needle-2.3.2.tgz",
@@ -11805,11 +11839,6 @@
         }
       }
     },
-    "node-eta": {
-      "version": "0.9.0",
-      "resolved": "https://registry.npmjs.org/node-eta/-/node-eta-0.9.0.tgz",
-      "integrity": "sha1-n7CwmbzSoCGUDmA8ZCVNwAPZp6g="
-    },
     "node-fetch": {
       "version": "2.6.0",
       "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.0.tgz",

package.json

@@ -34,11 +34,12 @@
     "@angular/platform-browser": "~9.1.4",
     "@angular/platform-browser-dynamic": "~9.1.4",
     "@angular/router": "~9.1.4",
-    "better-queue": "^3.8.10",
     "cli-color": "^2.0.0",
+    "comparators": "^3.0.2",
     "electron-window-state": "^5.0.3",
     "fomantic-ui": "^2.8.3",
     "jquery": "^3.4.1",
+    "mv": "^2.1.1",
     "needle": "^2.3.2",
     "node-7z": "^2.0.5",
     "node-unrar-js": "^0.8.1",
@@ -54,11 +55,12 @@
     "@angular/cli": "^9.1.4",
     "@angular/compiler-cli": "~9.1.4",
     "@angular/language-service": "~9.1.4",
-    "@types/better-queue": "^3.8.2",
     "@types/cli-color": "^2.0.0",
     "@types/electron-window-state": "^2.0.33",
+    "@types/mv": "^2.1.0",
     "@types/needle": "^2.0.4",
     "@types/node": "^12.11.1",
+    "@types/rimraf": "^3.0.0",
     "@types/underscore": "^1.9.4",
     "@typescript-eslint/eslint-plugin": "^2.19.2",
     "@typescript-eslint/parser": "^2.19.2",

src/app/components/browse/status-bar/downloads-modal/downloads-modal.component.html

@@ -12,13 +12,6 @@
         <i class="redo icon"></i>
         Retry
       </button>
-      <button
-        *ngIf="download.type == 'warning'"
-        class="ui right floated labeled icon button"
-        (click)="continueDownload(download.versionID)">
-        <i class="exclamation triangle icon"></i>
-        Download Anyway
-      </button>
       <div class="header">{{download.header}}</div>
       <div *ngIf="download.type != 'done'" class="description">{{download.description}}</div>
       <div *ngIf="download.type == 'done'" class="description">

src/app/components/browse/status-bar/downloads-modal/downloads-modal.component.ts

@@ -38,11 +38,6 @@ export class DownloadsModalComponent {
     this.downloadService.retryDownload(versionID)
   }
 
-  continueDownload(versionID: number) {
-    // TODO: test this
-    this.downloadService.continueDownload(versionID)
-  }
-
   getBackgroundColor(download: DownloadProgress) {
     switch (download.type) {
       case 'good': return 'unset'

src/app/core/services/download.service.ts

@@ -67,8 +67,4 @@ export class DownloadService {
   retryDownload(versionID: number) {
     this.electronService.sendIPC('download', { action: 'retry', versionID })
   }
-
-  continueDownload(versionID: number) {
-    this.electronService.sendIPC('download', { action: 'continue', versionID })
-  }
 }

src/electron/ipc/download/ChartDownload.ts

@@ -8,104 +8,150 @@ import { promisify } from 'util'
 import { randomBytes as _randomBytes } from 'crypto'
 import { mkdir as _mkdir } from 'fs'
 import { ProgressType, NewDownload } from 'src/electron/shared/interfaces/download.interface'
-import { downloadHandler } from './DownloadHandler'
-import { googleTimer } from './GoogleTimer'
 import { DriveFile } from 'src/electron/shared/interfaces/songDetails.interface'
+import { FileTransfer } from './FileTransfer'
 
 const randomBytes = promisify(_randomBytes)
 const mkdir = promisify(_mkdir)
 
+type EventCallback = {
+  /** Note: this will not be the last event if `retry()` is called. */
+  'error': () => void
+  'complete': () => void
+}
+type Callbacks = { [E in keyof EventCallback]: EventCallback[E] }
+
+export type DownloadError = { header: string; body: string }
+
 export class ChartDownload {
 
-  // This changes if the user needs to click 'retry' or 'continue'
+  private retryFn: () => void | Promise<void>
-  run: () => void | Promise<void> = this.beginDownload.bind(this)
+  private cancelFn: () => void
-  cancel: () => void
-
-  isArchive: boolean
-  title: string
-  header: string
-  description: string
-  percent = 0
-  type: ProgressType
 
+  private callbacks = {} as Callbacks
   private files: DriveFile[]
-  allFilesProgress = 0
+  private percent = 0 // Needs to be stored here because errors won't know the exact percent
-  individualFileProgressPortion: number
+
+  private readonly individualFileProgressPortion: number
+  private readonly destinationFolderName: string
+
+  private _allFilesProgress = 0
+  get allFilesProgress() { return this._allFilesProgress }
+  private _hasFailed = false
+  /** If this chart download needs to be retried */
+  get hasFailed() { return this._hasFailed }
+  get isArchive() { return this.data.driveData.isArchive }
 
   constructor(public versionID: number, private data: NewDownload) {
+    this.updateGUI('', 'Waiting for other downloads to finish...', 'good')
     this.files = data.driveData.files
-    this.isArchive = data.driveData.isArchive
     this.individualFileProgressPortion = 80 / this.files.length
+    this.destinationFolderName = sanitizeFilename(`${this.data.artist} - ${this.data.avTagName} (${this.data.charter})`)
   }
 
   /**
-   * Changes this download to reflect that it is waiting in the download queue.
+   * Calls `callback` when `event` fires. (no events will be fired after `this.cancel()` is called)
    */
-  setInQueue() {
+  on<E extends keyof EventCallback>(event: E, callback: EventCallback[E]) {
-    this.title = `${this.data.avTagName} - ${this.data.artist}`
+    this.callbacks[event] = callback
-    this.header = ''
+  }
-    this.description = 'Waiting for other downloads to finish...'
+
-    this.type = 'good'
+  /**
-    this.cancel = () => { /* do nothing */ }
+   * Retries the last failed step if it is running.
-    emitIPCEvent('download-updated', this)
+   */
+  retry() { // Only allow it to be called once
+    if (this.retryFn != undefined) {
+      this._hasFailed = false
+      const retryFn = this.retryFn
+      this.retryFn = undefined
+      retryFn()
+    }
+  }
+
+  /**
+   * Cancels the download if it is running.
+   */
+  cancel() { // Only allow it to be called once
+    if (this.cancelFn != undefined) {
+      const cancelFn = this.cancelFn
+      this.cancelFn = undefined
+      cancelFn()
+    }
+  }
+
+  /**
+   * Updates the GUI with new information about this chart download.
+   */
+  private updateGUI(header: string, description: string, type: ProgressType) {
+    emitIPCEvent('download-updated', {
+      versionID: this.versionID,
+      title: `${this.data.avTagName} - ${this.data.artist}`,
+      header: header,
+      description: description,
+      percent: this.percent,
+      type: type
+    })
+  }
+
+  /**
+   * Save the retry function, update the GUI, and call the `error` callback.
+   */
+  private handleError(err: DownloadError, retry: () => void) {
+    this._hasFailed = true
+    this.retryFn = retry
+    this.updateGUI(err.header, err.body, 'error')
+    this.callbacks.error()
   }
 
   /**
    * Starts the download process.
    */
   async beginDownload() {
-    // Create a temporary folder to store the downloaded files
+    // CREATE DOWNLOAD DIRECTORY
     let chartPath: string
     try {
       chartPath = await this.createDownloadFolder()
-    } catch (e) {
+    } catch (err) {
-      this.run = this.beginDownload.bind(this) // Retry action
+      this.retryFn = () => this.beginDownload()
-      this.error('Access Error', e.message)
+      this.updateGUI('Access Error', err.message, 'error')
       return
     }
 
-    // For each download link in <this.files>, download the file to <chartPath>
+    // DOWNLOAD FILES
     for (let i = 0; i < this.files.length; i++) {
-      // INITIALIZE DOWNLOADER
       const downloader = new FileDownloader(this.files[i].webContentLink, chartPath)
+      this.cancelFn = () => downloader.cancelDownload()
+
       const downloadComplete = this.addDownloadEventListeners(downloader, i)
-
-      // DOWNLOAD THE NEXT FILE
-      // Wait for google rate limit
-      this.header = `[${this.files[i].name}] (file ${i + 1}/${this.files.length})`
-      googleTimer.onTimerUpdate((remainingTime, totalTime) => {
-        this.description = `Waiting for Google rate limit... (${remainingTime}s)`
-        this.percent = this.allFilesProgress + interpolate(remainingTime, totalTime, 0, 0, this.individualFileProgressPortion / 2)
-        this.type = 'good'
-        emitIPCEvent('download-updated', this)
-      })
-      this.cancel = () => {
-        googleTimer.removeCallbacks()
-        this.onDownloadStop()
-        downloader.cancelDownload()
-      }
-      await new Promise<void>(resolve => googleTimer.onTimerReady(resolve))
-
-      this.cancel = () => {
-        this.onDownloadStop()
-        downloader.cancelDownload()
-      }
       downloader.beginDownload()
-      await downloadComplete // Wait for this download to finish
+      await downloadComplete
     }
 
-    // INITIALIZE FILE EXTRACTOR
+    // EXTRACT FILES
-    const destinationFolderName = sanitizeFilename(`${this.data.artist} - ${this.data.avTagName} (${this.data.charter})`)
+    if (this.isArchive) {
-    const extractor = new FileExtractor(chartPath, this.isArchive, destinationFolderName)
+      const extractor = new FileExtractor(chartPath)
-    this.cancel = () => extractor.cancelExtract() // Make cancel button cancel the file extraction
+      this.cancelFn = () => extractor.cancelExtract()
-    this.addExtractorEventListeners(extractor)
 
-    // EXTRACT THE DOWNLOADED ARCHIVE
+      const extractComplete = this.addExtractorEventListeners(extractor)
-    extractor.beginExtract()
+      extractor.beginExtract()
+      await extractComplete
+    }
+
+    // TRANSFER FILES
+    const transfer = new FileTransfer(chartPath, this.destinationFolderName)
+    this.cancelFn = () => transfer.cancelTransfer()
+
+    const transferComplete = this.addTransferEventListeners(transfer)
+    transfer.beginTransfer()
+    await transferComplete
+
+    this.callbacks.complete()
   }
 
   /**
-   * Attempts to create a unique folder in Bridge's data paths. Throws an error if this fails.
+   * Attempts to create a unique folder in Bridge's data paths.
+   * @returns the new folder's path.
+   * @throws an error if this fails.
    */
   private async createDownloadFolder() {
     let retryCount = 0
@@ -125,71 +171,38 @@ export class ChartDownload {
     throw new Error(`Bridge was unable to create a directory at [${chartPath}]`)
   }
 
-  /**
-   * Stops the download and displays an error message.
-   */
-  private error(header: string, description: string) {
-    this.header = header
-    this.description = description
-    this.type = 'error'
-    this.onDownloadStop()
-    emitIPCEvent('download-updated', this)
-  }
-
-  /**
-   * If this was a google download, allows a new google download to start.
-   */
-  private onDownloadStop() {
-    downloadHandler.isGoogleDownloading = false
-    downloadHandler.updateQueue()
-  }
-
   /**
    * Defines what happens in response to `FileDownloader` events.
+   * @returns a `Promise` that resolves when the download finishes.
    */
   private addDownloadEventListeners(downloader: FileDownloader, fileIndex: number) {
+    let downloadHeader = `[${this.files[fileIndex].name}] (file ${fileIndex + 1}/${this.files.length})`
     let fileProgress = 0
 
-    downloader.on('request', () => {
+    downloader.on('waitProgress', (remainingSeconds: number, totalSeconds: number) => {
-      this.description = 'Sending request...'
+      this.percent = this._allFilesProgress + interpolate(remainingSeconds, totalSeconds, 0, 0, this.individualFileProgressPortion / 2)
-      fileProgress = this.individualFileProgressPortion / 2
+      this.updateGUI(downloadHeader, `Waiting for Google rate limit... (${remainingSeconds}s)`, 'good')
-      this.percent = this.allFilesProgress + fileProgress
-      this.type = 'good'
-      emitIPCEvent('download-updated', this)
     })
 
-    downloader.on('warning', (continueAnyway) => {
+    downloader.on('requestSent', () => {
-      this.description = 'WARNING'
+      fileProgress = this.individualFileProgressPortion / 2
-      this.run = continueAnyway
+      this.percent = this._allFilesProgress + fileProgress
-      this.type = 'warning'
+      this.updateGUI(downloadHeader, 'Sending request...', 'good')
-      this.onDownloadStop()
-      emitIPCEvent('download-updated', this)
     })
 
     downloader.on('downloadProgress', (bytesDownloaded) => {
+      downloadHeader = `[${this.files[fileIndex].name}] (file ${fileIndex + 1}/${this.files.length})`
       const size = Number(this.files[fileIndex].size)
-      this.header = `[${this.files[fileIndex].name}] (file ${fileIndex + 1}/${this.files.length})`
-      this.description = `Downloading... (${Math.round(1000 * bytesDownloaded / size) / 10}%)`
       fileProgress = interpolate(bytesDownloaded, 0, size, this.individualFileProgressPortion / 2, this.individualFileProgressPortion)
-      this.percent = this.allFilesProgress + fileProgress
+      this.percent = this._allFilesProgress + fileProgress
-      this.type = 'fastUpdate'
+      this.updateGUI(downloadHeader, `Downloading... (${Math.round(1000 * bytesDownloaded / size) / 10}%)`, 'fastUpdate')
-      emitIPCEvent('download-updated', this)
     })
 
-    downloader.on('error', (error, retry) => {
+    downloader.on('error', this.handleError.bind(this))
-      this.header = error.header
-      this.description = error.body
-      this.type = 'error'
-      this.run = () => { retry() }
-      this.onDownloadStop()
-      emitIPCEvent('download-updated', this)
-    })
 
-    // Returns a promise that resolves when the download is finished
     return new Promise<void>(resolve => {
       downloader.on('complete', () => {
-        this.allFilesProgress += this.individualFileProgressPortion
+        this._allFilesProgress += this.individualFileProgressPortion
-        emitIPCEvent('download-updated', this)
         resolve()
       })
     })
@@ -197,49 +210,51 @@ export class ChartDownload {
 
   /**
    * Defines what happens in response to `FileExtractor` events.
+   * @returns a `Promise` that resolves when the extraction finishes.
    */
   private addExtractorEventListeners(extractor: FileExtractor) {
     let archive = ''
 
-    extractor.on('extract', (filename) => {
+    extractor.on('start', (filename) => {
       archive = filename
-      this.header = `[${archive}]`
+      this.updateGUI(`[${archive}]`, 'Extracting...', 'good')
-      this.description = 'Extracting...'
-      this.type = 'good'
-      emitIPCEvent('download-updated', this)
     })
 
     extractor.on('extractProgress', (percent, filecount) => {
-      this.header = `[${archive}] (${filecount} file${filecount == 1 ? '' : 's'} extracted)`
-      this.description = `Extracting... (${percent}%)`
       this.percent = interpolate(percent, 0, 100, 80, 95)
-      this.type = 'fastUpdate'
+      this.updateGUI(`[${archive}] (${filecount} file${filecount == 1 ? '' : 's'} extracted)`, `Extracting... (${percent}%)`, 'fastUpdate')
-      emitIPCEvent('download-updated', this)
     })
 
-    extractor.on('transfer', (filepath) => {
+    extractor.on('error', this.handleError.bind(this))
-      this.header = 'Moving files to library folder...'
+
-      this.description = filepath
+    return new Promise<void>(resolve => {
-      this.percent = 95
+      extractor.on('complete', () => {
-      this.type = 'good'
+        this.percent = 95
-      emitIPCEvent('download-updated', this)
+        resolve()
+      })
+    })
+  }
+
+  /**
+   * Defines what happens in response to `FileTransfer` events.
+   * @returns a `Promise` that resolves when the transfer finishes.
+   */
+  private addTransferEventListeners(transfer: FileTransfer) {
+    let destinationFolder: string
+
+    transfer.on('start', (_destinationFolder) => {
+      destinationFolder = _destinationFolder
+      this.updateGUI('Moving files to library folder...', destinationFolder, 'good')
     })
 
-    extractor.on('error', (error, retry) => {
+    transfer.on('error', this.handleError.bind(this))
-      this.header = error.header
-      this.description = error.body
-      this.type = 'error'
-      this.run = retry
-      emitIPCEvent('download-updated', this)
-    })
 
-    extractor.on('complete', (filepath) => {
+    return new Promise<void>(resolve => {
-      this.header = 'Download complete.'
+      transfer.on('complete', () => {
-      this.description = filepath
+        this.percent = 100
-      this.percent = 100
+        this.updateGUI('Download complete.', destinationFolder, 'done')
-      this.type = 'done'
+        resolve()
-      this.onDownloadStop()
+      })
-      emitIPCEvent('download-updated', this)
     })
   }
 }

src/electron/ipc/download/DownloadHandler.ts

@@ -1,46 +1,78 @@
 import { IPCEmitHandler } from '../../shared/IPCHandler'
 import { Download } from '../../shared/interfaces/download.interface'
 import { ChartDownload } from './ChartDownload'
+import { DownloadQueue } from './DownloadQueue'
 
 class DownloadHandler implements IPCEmitHandler<'download'> {
   event: 'download' = 'download'
 
-  downloads: { [versionID: number]: ChartDownload } = {}
+  downloadQueue: DownloadQueue = new DownloadQueue()
-  downloadQueue: ChartDownload[] = []
+  currentDownload: ChartDownload = undefined
-  isGoogleDownloading = false // This is a lock controlled by only one ChartDownload at a time
+  retryWaiting: ChartDownload[] = []
 
-  handler(data: Download) {
+  handler(data: Download) { // TODO: make sure UI can't add the same versionID more than once
-    if (data.action == 'add') {
+    switch (data.action) {
-      this.downloads[data.versionID] = new ChartDownload(data.versionID, data.data)
+      case 'add': this.addDownload(data); break
-    }
+      case 'retry': this.retryDownload(data); break
-
+      case 'cancel': this.cancelDownload(data); break
-    const download = this.downloads[data.versionID]
-
-    if (data.action == 'cancel') {
-      download.cancel() // Might change isGoogleDownloading and call updateQueue()
-      this.downloadQueue = this.downloadQueue.filter(download => download.versionID != data.versionID)
-      this.downloads[data.versionID] = undefined
-    } else {
-      download.setInQueue()
-      this.downloadQueue.push(download) // Add, retry, or continue will re-add the download to the queue
-      this.updateQueue()
     }
   }
 
-  /**
+  private addDownload(data: Download) {
-   * Called when at least one download in the queue can potentially be started.
+    const newDownload = new ChartDownload(data.versionID, data.data)
-   */
+    this.addDownloadEventListeners(newDownload)
-  updateQueue() {
+    if (this.currentDownload == undefined) {
-    this.downloadQueue.sort((cd1: ChartDownload, cd2: ChartDownload) => {
+      this.currentDownload = newDownload
-      const value1 = 100 + (99 - cd1.allFilesProgress)
+      newDownload.beginDownload()
-      const value2 = 100 + (99 - cd2.allFilesProgress)
+    } else {
-      return value1 - value2 // Sorts in the order to get the most downloads completed early
+      this.downloadQueue.push(newDownload)
+    }
+  }
+
+  private retryDownload(data: Download) {
+    const index = this.retryWaiting.findIndex(download => download.versionID == data.versionID)
+    if (index != -1) {
+      const retryDownload = this.retryWaiting.splice(index, 1)[0]
+      if (this.currentDownload == undefined) {
+        this.currentDownload = retryDownload
+        retryDownload.retry()
+      } else {
+        this.downloadQueue.push(retryDownload)
+      }
+    }
+  }
+
+  private cancelDownload(data: Download) {
+    if (this.currentDownload.versionID == data.versionID) {
+      this.currentDownload.cancel()
+      this.currentDownload = undefined
+      this.startNextDownload()
+    } else {
+      this.downloadQueue.remove(data.versionID)
+    }
+  }
+
+  private addDownloadEventListeners(download: ChartDownload) {
+    download.on('complete', () => {
+      this.currentDownload = undefined
+      this.startNextDownload()
     })
 
-    while (this.downloadQueue[0] != undefined && !(this.isGoogleDownloading)) {
+    download.on('error', () => {
-      const nextDownload = this.downloadQueue.shift()
+      this.retryWaiting.push(this.currentDownload)
-      nextDownload.run()
+      this.currentDownload = undefined
-      this.isGoogleDownloading = true
+      this.startNextDownload()
+    })
+  }
+
+  private startNextDownload() {
+    if (!this.downloadQueue.isEmpty()) {
+      this.currentDownload = this.downloadQueue.pop()
+      if (this.currentDownload.hasFailed) {
+        this.currentDownload.retry()
+      } else {
+        this.currentDownload.beginDownload()
+      }
     }
   }
 }

src/electron/ipc/download/DownloadQueue.ts

@@ -0,0 +1,43 @@
+import Comparators from 'comparators'
+import { ChartDownload } from './ChartDownload'
+
+export class DownloadQueue {
+
+  downloadQueue: ChartDownload[] = []
+
+  isEmpty() {
+    return this.downloadQueue.length == 0
+  }
+
+  push(chartDownload: ChartDownload) {
+    this.downloadQueue.push(chartDownload)
+    this.sort()
+  }
+
+  pop() {
+    return this.downloadQueue.pop()
+  }
+
+  get(versionID: number) {
+    return this.downloadQueue.find(download => download.versionID == versionID)
+  }
+
+  remove(versionID: number) {
+    const index = this.downloadQueue.findIndex(download => download.versionID == versionID)
+    if (index != -1) {
+      this.downloadQueue[index].cancel()
+      this.downloadQueue.splice(index, 1)
+    }
+  }
+
+  private sort() { // TODO: make this order be reflected in the GUI (along with currentDownload)
+    let comparator = Comparators.comparing('allFilesProgress', { reversed: true })
+
+    const prioritizeArchives = true
+    if (prioritizeArchives) {
+      comparator = comparator.thenComparing('isArchive', { reversed: true })
+    }
+
+    this.downloadQueue.sort(comparator)
+  }
+}

src/electron/ipc/download/FileDownloader.ts

@@ -1,25 +1,35 @@
-import { generateUUID, sanitizeFilename } from '../../shared/UtilFunctions'
+import { AnyFunction } from '../../shared/UtilFunctions'
-import * as fs from 'fs'
+import { createWriteStream } from 'fs'
-import * as path from 'path'
 import * as needle from 'needle'
 // TODO: replace needle with got (for cancel() method) (if before-headers event is possible?)
 import { getSettings } from '../SettingsHandler.ipc'
+import { googleTimer } from './GoogleTimer'
+import { DownloadError } from './ChartDownload'
 
 type EventCallback = {
-  'request': () => void
+  'waitProgress': (remainingSeconds: number, totalSeconds: number) => void
-  'warning': (continueAnyway: () => void) => void
+  /** Note: this event can be called multiple times if the connection times out or a large file is downloaded */
+  'requestSent': () => void
   'downloadProgress': (bytesDownloaded: number) => void
-  'error': (error: DownloadError, retry: () => void) => void
+  /** Note: after calling retry, the event lifecycle restarts */
+  'error': (err: DownloadError, retry: () => void) => void
   'complete': () => void
 }
 type Callbacks = { [E in keyof EventCallback]: EventCallback[E] }
 
-export type DownloadError = { header: string; body: string }
+const downloadErrors = {
+  libraryFolder: () => { return { header: 'Library folder not specified', body: 'Please go to the settings to set your library folder.' } },
+  timeout: (type: string) => { return { header: 'Timeout', body: `The download server could not be reached. (type=${type})` } },
+  connectionError: (err: Error) => { return { header: 'Connection Error', body: `${err.name}: ${err.message}` } },
+  responseError: (statusCode: number) => { return { header: 'Connection failed', body: `Server returned status code: ${statusCode}` } },
+  htmlError: () => { return { header: 'Invalid response', body: 'Download server returned HTML instead of a file.' } }
+}
 
 /**
- * Downloads a file from `url` to `destinationFolder` and verifies that its hash matches `expectedHash`.
+ * Downloads a file from `url` to `fullPath`.
  * Will handle google drive virus scan warnings. Provides event listeners for download progress.
  * On error, provides the ability to retry.
+ * Will only send download requests once every `getSettings().rateLimitDelay` seconds.
  */
 export class FileDownloader {
   private readonly RETRY_MAX = 2
@@ -30,29 +40,33 @@ export class FileDownloader {
 
   /**
    * @param url The download link.
-   * @param destinationFolder The path to where this file should be stored.
+   * @param fullPath The full path to where this file should be stored (including the filename).
-   * @param expectedHash The hash header value that is expected for this file.
    */
-  constructor(private url: string, private destinationFolder: string) { }
+  constructor(private url: string, private fullPath: string) { }
 
   /**
-   * Calls `callback` when `event` fires.
+   * Calls `callback` when `event` fires. (no events will be fired after `this.cancelDownload()` is called)
    */
   on<E extends keyof EventCallback>(event: E, callback: EventCallback[E]) {
     this.callbacks[event] = callback
   }
 
   /**
-   * Download the file.
+   * Download the file after waiting for the google rate limit.
    */
   beginDownload() {
-    // Check that the library folder has been specified
+    console.log('Begin download...')
     if (getSettings().libraryPath == undefined) {
-      this.callbacks.error({ header: 'Library folder not specified', body: 'Please go to the settings to set your library folder.' }, () => this.beginDownload())
+      this.failDownload(downloadErrors.libraryFolder())
-      return
+    } else {
-    }
+      googleTimer.on('waitProgress', this.cancelable((remainingSeconds, totalSeconds) => {
+        this.callbacks.waitProgress(remainingSeconds, totalSeconds)
+      }))
 
-    this.requestDownload()
+      googleTimer.on('complete', this.cancelable(() => {
+        this.requestDownload()
+      }))
+    }
   }
 
   /**
@@ -60,64 +74,60 @@ export class FileDownloader {
    * @param cookieHeader the "cookie=" header to include this request.
    */
   private requestDownload(cookieHeader?: string) {
-    if (this.wasCanceled) { return } // CANCEL POINT
+    this.callbacks.requestSent()
-    this.callbacks.request()
-    const uuid = generateUUID()
     const req = needle.get(this.url, {
       'follow_max': 10,
       'open_timeout': 5000,
       'headers': Object.assign({
-        'User-Agent': 'PostmanRuntime/7.22.0',
         'Referer': this.url,
-        'Accept': '*/*',
+        'Accept': '*/*'
-        'Postman-Token': uuid
       },
       (cookieHeader ? { 'Cookie': cookieHeader } : undefined)
       )
     })
 
-    req.on('timeout', (type: string) => {
+    req.on('timeout', this.cancelable((type: string) => {
       this.retryCount++
       if (this.retryCount <= this.RETRY_MAX) {
         console.log(`TIMEOUT: Retry attempt ${this.retryCount}...`)
         this.requestDownload(cookieHeader)
       } else {
-        this.callbacks.error({ header: 'Timeout', body: `The download server could not be reached. (type=${type})` }, () => this.beginDownload())
+        this.failDownload(downloadErrors.timeout(type))
       }
-    })
+    }))
 
-    req.on('err', (err: Error) => {
+    req.on('err', this.cancelable((err: Error) => {
-      this.callbacks.error({ header: 'Connection Error', body: `${err.name}: ${err.message}` }, () => this.beginDownload())
+      this.failDownload(downloadErrors.connectionError(err))
-    })
+    }))
 
-    req.on('header', (statusCode, headers: Headers) => {
+    req.on('header', this.cancelable((statusCode, headers: Headers) => {
-      if (this.wasCanceled) { return } // CANCEL POINT
       if (statusCode != 200) {
-        this.callbacks.error({ header: 'Connection failed', body: `Server returned status code: ${statusCode}` }, () => this.beginDownload())
+        this.failDownload(downloadErrors.responseError(statusCode))
         return
       }
 
-      const fileType = headers['content-type']
+      if (headers['content-type'].startsWith('text/html')) {
-      if (fileType.startsWith('text/html')) {
         this.handleHTMLResponse(req, headers['set-cookie'])
       } else {
-        const fileName = this.getDownloadFileName(headers)
+        this.handleDownloadResponse(req)
-        this.handleDownloadResponse(req, fileName)
       }
-    })
+    }))
   }
 
   /**
    * A Google Drive HTML response to a download request usually means this is the "file too large to scan for viruses" warning.
-   * This function sends the request that results from clicking "download anyway", or throws an error if it can't be found.
+   * This function sends the request that results from clicking "download anyway", or generates an error if it can't be found.
    * @param req The download request.
    * @param cookieHeader The "cookie=" header of this request.
    */
   private handleHTMLResponse(req: NodeJS.ReadableStream, cookieHeader: string) {
+    console.log('HTML Response...')
     let virusScanHTML = ''
-    req.on('data', data => virusScanHTML += data)
+    req.on('data', this.cancelable(data => virusScanHTML += data))
-    req.on('done', (err: Error) => {
+    req.on('done', this.cancelable((err: Error) => {
-      if (!err) {
+      if (err) {
+        this.failDownload(downloadErrors.connectionError(err))
+      } else {
         try {
           const confirmTokenRegex = /confirm=([0-9A-Za-z\-_]+)&/g
           const confirmTokenResults = confirmTokenRegex.exec(virusScanHTML)
@@ -129,61 +139,57 @@ export class FileDownloader {
           const newHeader = `download_warning_${warningCode}=${confirmToken}; NID=${NID}`
           this.requestDownload(newHeader)
         } catch(e) {
-          this.callbacks.error({ header: 'Invalid response', body: 'Download server returned HTML instead of a file.' }, () => this.beginDownload())
+          this.failDownload(downloadErrors.htmlError())
         }
-      } else {
-        this.callbacks.error({ header: 'Connection Failed', body: `Connection failed while downloading HTML: ${err.name}` }, () => this.beginDownload())
       }
-    })
+    }))
   }
 
   /**
-   * Pipes the data from a download response to `fileName`.
+   * Pipes the data from a download response to `this.fullPath`.
    * @param req The download request.
-   * @param fileName The name of the output file.
    */
-  private handleDownloadResponse(req: NodeJS.ReadableStream, fileName: string) {
+  private handleDownloadResponse(req: NodeJS.ReadableStream) {
+    console.log('Download response...')
     this.callbacks.downloadProgress(0)
     let downloadedSize = 0
-    const filePath = path.join(this.destinationFolder, fileName)
+    req.pipe(createWriteStream(this.fullPath))
-    req.pipe(fs.createWriteStream(filePath))
+    req.on('data', this.cancelable((data) => {
-    req.on('data', (data) => {
       downloadedSize += data.length
       this.callbacks.downloadProgress(downloadedSize)
-    })
+    }))
 
-    req.on('err', (err: Error) => {
+    req.on('err', this.cancelable((err: Error) => {
-      this.callbacks.error({ header: 'Connection Failed', body: `Connection failed while downloading file: ${err.name}` }, () => this.beginDownload())
+      this.failDownload(downloadErrors.connectionError(err))
-    })
+    }))
 
-    req.on('end', () => {
+    req.on('end', this.cancelable(() => {
-      if (this.wasCanceled) { return } // CANCEL POINT
       this.callbacks.complete()
-    })
+    }))
   }
 
   /**
-   * Extracts the downloaded file's filename from `headers` or `this.url`, depending on the file's host server.
+   * Display an error message and provide a function to retry the download.
-   * @param headers The response headers for this request.
    */
-  private getDownloadFileName(headers: Headers) {
+  private failDownload(error: DownloadError) {
-    if (headers['server'] && headers['server'] == 'cloudflare' || this.url.startsWith('https://public.fightthe.pw/')) {
+    this.callbacks.error(error, this.cancelable(() => this.beginDownload()))
-      // Cloudflare and Chorus specific jazz
-      return sanitizeFilename(decodeURIComponent(path.basename(this.url)))
-    } else {
-      // GDrive specific jazz
-      const filenameRegex = /filename="(.*?)"/g
-      const results = filenameRegex.exec(headers['content-disposition'])
-      if (results == null) {
-        console.log(`Warning: couldn't find filename in content-disposition header: [${headers['content-disposition']}]`)
-        return 'unknownFilename'
-      } else {
-        return sanitizeFilename(results[1])
-      }
-    }
   }
 
+  /**
+   * Stop the process of downloading the file. (no more events will be fired after this is called)
+   */
   cancelDownload() {
     this.wasCanceled = true
+    googleTimer.cancelTimer() // Prevents timer from trying to activate a download and resetting
+  }
+
+  /**
+   * Wraps a function that is able to be prevented if `this.cancelDownload()` was called.
+   */
+  private cancelable<F extends AnyFunction>(fn: F) {
+    return (...args: Parameters<F>): ReturnType<F> => {
+      if (this.wasCanceled) { return }
+      return fn(...Array.from(args))
+    }
   }
 }

src/electron/ipc/download/FileExtractor.ts

@@ -1,163 +1,161 @@
-import { DownloadError } from './FileDownloader'
+import { readdir, unlink, mkdir as _mkdir } from 'fs'
-import * as fs from 'fs'
 import { promisify } from 'util'
 import { join, extname } from 'path'
+import { AnyFunction } from 'src/electron/shared/UtilFunctions'
 import * as node7z from 'node-7z'
 import * as zipBin from '7zip-bin'
-import { getSettings } from '../SettingsHandler.ipc'
+import * as unrarjs from 'node-unrar-js' // TODO find better rar library that has async extraction
-import { extractRar } from './RarExtractor'
+import { FailReason } from 'node-unrar-js/dist/js/extractor'
+import { DownloadError } from './ChartDownload'
 
-const readdir = promisify(fs.readdir)
+const mkdir = promisify(_mkdir)
-const unlink = promisify(fs.unlink)
-const lstat = promisify(fs.lstat)
-const copyFile = promisify(fs.copyFile)
-const rmdir = promisify(fs.rmdir)
-const access = promisify(fs.access)
-const mkdir = promisify(fs.mkdir)
 
 type EventCallback = {
-  'extract': (filename: string) => void
+  'start': (filename: string) => void
   'extractProgress': (percent: number, fileCount: number) => void
-  'transfer': (filepath: string) => void
+  'error': (err: DownloadError, retry: () => void | Promise<void>) => void
-  'complete': (filepath: string) => void
+  'complete': () => void
-  'error': (error: DownloadError, retry: () => void | Promise<void>) => void
 }
 type Callbacks = { [E in keyof EventCallback]: EventCallback[E] }
 
+const extractErrors = {
+  readError: (err: NodeJS.ErrnoException) => { return { header: `Failed to read file (${err.code})`, body: `${err.name}: ${err.message}` } },
+  emptyError: () => { return { header: 'Failed to extract archive', body: 'File archive was downloaded but could not be found' } },
+  rarmkdirError: (err: NodeJS.ErrnoException, sourceFile: string) => {
+    return { header: `Extracting archive failed. (${err.code})`, body: `${err.name}: ${err.message} (${sourceFile})`}
+  },
+  rarextractError: (result: { reason: FailReason; msg: string }, sourceFile: string) => {
+    return { header: `Extracting archive failed: ${result.reason}`, body: `${result.msg} (${sourceFile})`}
+  }
+}
+
 export class FileExtractor {
 
   private callbacks = {} as Callbacks
-  private libraryFolder: string
   private wasCanceled = false
-  constructor(private sourceFolder: string, private isArchive: boolean, private destinationFolderName: string) { }
+  constructor(private sourceFolder: string) { }
 
   /**
-   * Calls `callback` when `event` fires.
+   * Calls `callback` when `event` fires. (no events will be fired after `this.cancelDownload()` is called)
    */
   on<E extends keyof EventCallback>(event: E, callback: EventCallback[E]) {
     this.callbacks[event] = callback
   }
 
   /**
-   * Starts the chart extraction process.
+   * Extract the chart from `this.sourceFolder`. (assumes there is exactly one archive file in that folder)
    */
-  async beginExtract() {
+  beginExtract() {
-    this.libraryFolder = getSettings().libraryPath
+    setTimeout(this.cancelable(() => {
-    const files = await readdir(this.sourceFolder)
+      readdir(this.sourceFolder, (err, files) => {
-    if (this.isArchive) {
+        if (err) {
-      this.extract(files[0], extname(files[0]) == '.rar')
+          this.callbacks.error(extractErrors.readError(err), () => this.beginExtract())
-    } else {
+        } else if (files.length == 0) {
-      this.transfer()
+          this.callbacks.error(extractErrors.emptyError(), () => this.beginExtract())
-    }
+        } else {
-  }
+          this.callbacks.start(files[0])
-
+          this.extract(join(this.sourceFolder, files[0]), extname(files[0]) == '.rar')
-  /**
-   * Extracts the file at `filename` to `this.sourceFolder`.
-   */
-  private async extract(filename: string, useRarExtractor: boolean) {
-    await new Promise<void>(resolve => setTimeout(() => resolve(), 100)) // Wait for filesystem to process downloaded file...
-    if (this.wasCanceled) { return } // CANCEL POINT
-    this.callbacks.extract(filename)
-    const source = join(this.sourceFolder, filename)
-
-    if (useRarExtractor) {
-
-      // Use node-unrar-js to extract the archive
-      try {
-        await extractRar(source, this.sourceFolder)
-      } catch (err) {
-        this.callbacks.error({
-          header: 'Extract Failed.',
-          body: `Unable to extract [${filename}]: ${err}`
-        }, () => this.extract(filename, extname(filename) == '.rar'))
-        return
-      }
-      this.transfer(source)
-
-    } else {
-
-      // Use node-7z to extract the archive
-      const stream = node7z.extractFull(source, this.sourceFolder, { $progress: true, $bin: zipBin.path7za })
-
-      stream.on('progress', (progress: { percent: number; fileCount: number }) => {
-        this.callbacks.extractProgress(progress.percent, progress.fileCount)
-      })
-
-      let extractErrorOccured = false
-      stream.on('error', () => {
-        extractErrorOccured = true
-        console.log(`Failed to extract [${filename}], retrying with .rar extractor...`)
-        this.extract(filename, true)
-      })
-
-      stream.on('end', () => {
-        if (!extractErrorOccured) {
-          this.transfer(source)
         }
       })
+    }), 100) // Wait for filesystem to process downloaded file
+  }
 
+  /**
+   * Extracts the file at `fullPath` to `this.sourceFolder`.
+   */
+  private async extract(fullPath: string, useRarExtractor: boolean) {
+    if (useRarExtractor) {
+      await this.extractRar(fullPath) // Use node-unrar-js to extract the archive
+    } else {
+      this.extract7z(fullPath) // Use node-7z to extract the archive
     }
   }
 
   /**
-   * Deletes the archive at `archiveFilepath`, then transfers the extracted chart to `this.libraryFolder`.
+   * Extracts a .rar archive found at `fullPath` and puts the extracted results in `this.sourceFolder`.
+   * @throws an `ExtractError` if this fails.
    */
-  private async transfer(archiveFilepath?: string) {
+  private async extractRar(fullPath: string) {
-    // TODO: this fails if the extracted chart has nested folders
+    const extractor = unrarjs.createExtractorFromFile(fullPath, this.sourceFolder)
-    // TODO: skip over "__MACOSX" folder
-    // TODO: handle other common problems, like chart/audio files not named correctly
-    if (this.wasCanceled) { return } // CANCEL POINT
-    try {
 
-      // Create destiniation folder if it doesn't exist
+    const fileList = extractor.getFileList()
-      const destinationFolder = join(this.libraryFolder, this.destinationFolderName)
-      this.callbacks.transfer(destinationFolder)
-      try {
-        await access(destinationFolder, fs.constants.F_OK)
-      } catch (e) {
-        await mkdir(destinationFolder)
-      }
 
-      // Delete archive
+    if (fileList[0].state != 'FAIL') {
-      if (archiveFilepath != undefined) {
+
-        try {
+      // Create directories for nested archives (because unrarjs didn't feel like handling that automatically)
-          await unlink(archiveFilepath)
+      const headers = fileList[1].fileHeaders
-        } catch (e) {
+      for (const header of headers) {
-          if (e.code != 'ENOENT') {
+        if (header.flags.directory) {
-            throw new Error(`Could not delete the archive file at [${archiveFilepath}]`)
+          try {
+            await mkdir(join(this.sourceFolder, header.name), { recursive: true })
+          } catch (err) {
+            this.callbacks.error(extractErrors.rarmkdirError(err, fullPath), () => this.extract(fullPath, extname(fullPath) == '.rar'))
+            return
           }
         }
       }
+    }
 
-      // Check if it extracted to a folder instead of a list of files
+    const extractResult = extractor.extractAll()
-      let sourceFolder = this.sourceFolder
-      let files = await readdir(sourceFolder)
-      const isFolderArchive = (files.length < 2 && !(await lstat(join(sourceFolder, files[0]))).isFile())
-      if (isFolderArchive) {
-        sourceFolder = join(sourceFolder, files[0])
-        files = await readdir(sourceFolder)
-      }
 
-      if (this.wasCanceled) { return } // CANCEL POINT
+    if (extractResult[0].state == 'FAIL') {
-
+      this.callbacks.error(extractErrors.rarextractError(extractResult[0], fullPath), () => this.extract(fullPath, extname(fullPath) == '.rar'))
-      // Copy the files from the temporary directory to the destination
+    } else {
-      for (const file of files) {
+      this.deleteArchive(fullPath)
-        await copyFile(join(sourceFolder, file), join(destinationFolder, file))
-        await unlink(join(sourceFolder, file))
-      }
-
-      // Delete the temporary folders
-      await rmdir(sourceFolder)
-      if (isFolderArchive) {
-        await rmdir(join(sourceFolder, '..'))
-      }
-      this.callbacks.complete(destinationFolder)
-    } catch (e) {
-      this.callbacks.error({ header: 'Transfer Failed', body: `Unable to transfer downloaded files to the library folder: ${e.name}` }, () => this.transfer(archiveFilepath))
     }
   }
 
+  /**
+   * Extracts a .zip or .7z archive found at `fullPath` and puts the extracted results in `this.sourceFolder`.
+   */
+  private extract7z(fullPath: string) {
+    const stream = node7z.extractFull(fullPath, this.sourceFolder, { $progress: true, $bin: zipBin.path7za })
+
+    stream.on('progress', this.cancelable((progress: { percent: number; fileCount: number }) => {
+      this.callbacks.extractProgress(progress.percent, progress.fileCount)
+    }))
+
+    let extractErrorOccured = false
+    stream.on('error', this.cancelable(() => {
+      extractErrorOccured = true
+      console.log(`Failed to extract [${fullPath}]; retrying with .rar extractor...`)
+      this.extract(fullPath, true)
+    }))
+
+    stream.on('end', this.cancelable(() => {
+      if (!extractErrorOccured) {
+        this.deleteArchive(fullPath)
+      }
+    }))
+  }
+
+  /**
+   * Tries to delete the archive at `fullPath`.
+   */
+  private deleteArchive(fullPath: string) {
+    unlink(fullPath, this.cancelable((err) => {
+      if (err && err.code != 'ENOENT') {
+        console.log(`Warning: failed to delete archive at [${fullPath}]`)
+      }
+
+      this.callbacks.complete()
+    }))
+  }
+
+  /**
+   * Stop the process of extracting the file. (no more events will be fired after this is called)
+   */
   cancelExtract() {
     this.wasCanceled = true
   }
+
+  /**
+   * Wraps a function that is able to be prevented if `this.cancelExtract()` was called.
+   */
+  private cancelable<F extends AnyFunction>(fn: F) {
+    return (...args: Parameters<F>): ReturnType<F> => {
+      if (this.wasCanceled) { return }
+      return fn(...Array.from(args))
+    }
+  }
 }

src/electron/ipc/download/FileTransfer.ts

@@ -0,0 +1,111 @@
+import { Dirent, readdir as _readdir } from 'fs'
+import { promisify } from 'util'
+import { getSettings } from '../SettingsHandler.ipc'
+import * as mv from 'mv'
+import { join } from 'path'
+import * as _rimraf from 'rimraf'
+import { DownloadError } from './ChartDownload'
+
+const readdir = promisify(_readdir)
+const rimraf = promisify(_rimraf)
+
+type EventCallback = {
+  'start': (destinationFolder: string) => void
+  'error': (err: DownloadError, retry: () => void | Promise<void>) => void
+  'complete': () => void
+}
+type Callbacks = { [E in keyof EventCallback]: EventCallback[E] }
+
+const transferErrors = {
+  readError: (err: NodeJS.ErrnoException) => fsError(err, 'Failed to read file'),
+  deleteError: (err: NodeJS.ErrnoException) => fsError(err, 'Failed to delete file'),
+  rimrafError: (err: NodeJS.ErrnoException) => fsError(err, 'Failed to delete folder'),
+  mvError: (err: NodeJS.ErrnoException) => fsError(err, 'Failed to move folder to library')
+}
+
+function fsError(err: NodeJS.ErrnoException, description: string) {
+  return { header: `${description} (${err.code})`, body: `${err.name}: ${err.message}` }
+}
+
+export class FileTransfer {
+
+  private callbacks = {} as Callbacks
+  private wasCanceled = false
+  private destinationFolder: string
+  private nestedSourceFolder: string // The top-level folder that is copied to the library folder
+  constructor(private sourceFolder: string, destinationFolderName: string) {
+    this.destinationFolder = join(getSettings().libraryPath, destinationFolderName)
+    this.nestedSourceFolder = sourceFolder
+  }
+
+  /**
+   * Calls `callback` when `event` fires. (no events will be fired after `this.cancelDownload()` is called)
+   */
+  on<E extends keyof EventCallback>(event: E, callback: EventCallback[E]) {
+    this.callbacks[event] = callback
+  }
+
+  async beginTransfer() {
+    this.callbacks.start(this.destinationFolder)
+    await this.cleanFolder()
+    if (this.wasCanceled) { return }
+    this.moveFolder()
+  }
+
+  /**
+   * Fixes common problems with the download chart folder.
+   */
+  private async cleanFolder() {
+    let files: Dirent[]
+    try {
+      files = await readdir(this.nestedSourceFolder, { withFileTypes: true })
+    } catch (err) {
+      this.callbacks.error(transferErrors.readError(err), () => this.cleanFolder())
+      return
+    }
+
+    // Remove nested folders
+    if (files.length == 1 && !files[0].isFile()) {
+      this.nestedSourceFolder = join(this.nestedSourceFolder, files[0].name)
+      await this.cleanFolder()
+      return
+    }
+
+    // Delete '__MACOSX' folder
+    for (const file of files) {
+      if (!file.isFile() && file.name == '__MACOSX') {
+        try {
+          await rimraf(join(this.nestedSourceFolder, file.name))
+        } catch (err) {
+          this.callbacks.error(transferErrors.rimrafError(err), () => this.cleanFolder())
+          return
+        }
+      } else {
+        // TODO: handle other common problems, like chart/audio files not named correctly
+        // TODO: this could have been a chart pack; have option to only keep selected chart?
+        // (think about a user trying to download a set of charts in a pack)
+      }
+    }
+  }
+
+  /**
+   * Moves the downloaded chart to the library path.
+   */
+  private moveFolder() {
+    mv(this.nestedSourceFolder, this.destinationFolder, { mkdirp: true }, (err) => {
+      if (err) {
+        this.callbacks.error(transferErrors.mvError(err), () => this.moveFolder())
+      } else {
+        rimraf(this.sourceFolder) // Delete temp folder
+        this.callbacks.complete()
+      }
+    })
+  }
+
+  /**
+   * Stop the process of transfering the file. (no more events will be fired after this is called)
+   */
+  cancelTransfer() {
+    this.wasCanceled = true
+  }
+}

src/electron/ipc/download/GoogleTimer.ts

@@ -1,49 +1,72 @@
 import { getSettings } from '../SettingsHandler.ipc'
 
+type EventCallback = {
+  'waitProgress': (remainingSeconds: number, totalSeconds: number) => void
+  'complete': () => void
+}
+type Callbacks = { [E in keyof EventCallback]?: EventCallback[E] }
+
 class GoogleTimer {
 
   private rateLimitCounter = Infinity
-  private onReadyCallback: () => void
+  private callbacks: Callbacks = {}
-  private updateCallback: (remainingTime: number, totalTime: number) => void
 
+  /**
+   * Initializes the timer to call the callbacks if they are defined.
+   */
   constructor() {
     setInterval(() => {
       this.rateLimitCounter++
-      if (this.isReady() && this.onReadyCallback != undefined) {
+      this.updateCallbacks()
-        this.activateTimerReady()
-      } else if (this.updateCallback != undefined) {
-        const delay = getSettings().rateLimitDelay
-        this.updateCallback(delay - this.rateLimitCounter, delay)
-      }
     }, 1000)
   }
 
-  onTimerReady(callback: () => void) {
+  /**
-    this.onReadyCallback = callback
+   * Calls `callback` when `event` fires. (no events will be fired after `this.cancelTimer()` is called)
-    if (this.isReady()) {
+   */
-      this.activateTimerReady()
+  on<E extends keyof EventCallback>(event: E, callback: EventCallback[E]) {
+    this.callbacks[event] = callback
+    this.updateCallbacks() // Fire events immediately after the listeners have been added
+  }
+
+  /**
+   * Check the state of the callbacks and call them if necessary.
+   */
+  private updateCallbacks() {
+    if (this.hasTimerEnded() && this.callbacks.complete != undefined) {
+      this.endTimer()
+    } else if (this.callbacks.waitProgress != undefined) {
+      const delay = getSettings().rateLimitDelay
+      this.callbacks.waitProgress(delay - this.rateLimitCounter, delay)
     }
   }
 
-  onTimerUpdate(callback: (remainingTime: number, totalTime: number) => void) {
+  /**
-    this.updateCallback = callback
+   * Prevents the callbacks from activating when the timer ends.
+   */
+  cancelTimer() {
+    this.callbacks = {}
   }
 
-  removeCallbacks() {
+  /**
-    this.onReadyCallback = undefined
+   * Checks if enough time has elapsed since the last timer activation.
-    this.updateCallback = undefined
+   */
-  }
+  private hasTimerEnded() {
-
-  private isReady() {
     return this.rateLimitCounter > getSettings().rateLimitDelay
   }
 
-  private activateTimerReady() {
+  /**
+   * Activates the completion callback and resets the timer.
+   */
+  private endTimer() {
     this.rateLimitCounter = 0
-    const onReadyCallback = this.onReadyCallback
+    const completeCallback = this.callbacks.complete
-    this.removeCallbacks()
+    this.callbacks = {}
-    onReadyCallback()
+    completeCallback()
   }
 }
 
+/**
+ * Important: this instance cannot be used by more than one file download at a time.
+ */
 export const googleTimer = new GoogleTimer()

src/electron/ipc/download/RarExtractor.ts

@@ -1,39 +0,0 @@
-import * as fs from 'fs'
-import { join } from 'path'
-import * as unrarjs from 'node-unrar-js'
-import { promisify } from 'util'
-
-const mkdir = promisify(fs.mkdir)
-
-/**
- * Extracts the archive at `sourceFile` to a new folder in `destinationFolder`. Throws an error when this fails.
- */
-export async function extractRar(sourceFile: string, destinationFolder: string) {
-  const extractor = unrarjs.createExtractorFromFile(sourceFile, destinationFolder)
-
-  const fileList = extractor.getFileList()
-
-  if (fileList[0].state != 'FAIL') {
-
-    // Create directories for nested archives (because unrarjs didn't feel like handling that automatically)
-    const headers = fileList[1].fileHeaders
-    for (const header of headers) {
-      if (header.flags.directory) {
-        try {
-          await mkdir(join(destinationFolder, header.name), { recursive: true })
-        } catch (e) {
-          throw new Error(`Failed to extract directory: ${e}`)
-        }
-      }
-    }
-  } else {
-    console.log('Warning: failed to read .rar files: ', fileList[0].reason, fileList[0].msg)
-  }
-
-  // Extract archive
-  const extractResult = extractor.extractAll()
-
-  if (extractResult[0].state == 'FAIL') {
-    throw new Error(`${extractResult[0].reason}: ${extractResult[0].msg}`)
-  }
-}

src/electron/shared/UtilFunctions.ts

@@ -1,23 +1,7 @@
 const sanitize = require('sanitize-filename')
 
-/**
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
- * @returns a random UUID
+export type AnyFunction = (...args: any) => any
- */
-export function generateUUID() { // Public Domain/MIT
-  let d = new Date().getTime() // Timestamp
-  let d2 = Date.now() // Time in microseconds since page-load or 0 if unsupported
-  return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, (c) => {
-    let r = Math.random() * 16 // Random number between 0 and 16
-    if (d > 0) { // Use timestamp until depleted
-      r = (d + r) % 16 | 0
-      d = Math.floor(d / 16)
-    } else { // Use microseconds since page-load if supported
-      r = (d2 + r) % 16 | 0
-      d2 = Math.floor(d2 / 16)
-    }
-    return (c === 'x' ? r : (r & 0x3 | 0x8)).toString(16)
-  })
-}
 
 /**
  * @returns `filename`, but with any invalid filename characters replaced with similar valid characters.
@@ -37,10 +21,10 @@ export function sanitizeFilename(filename: string): string {
 }
 
 /**
- * Converts `val` from the range (`fromA`, `fromB`) to the range (`toA`, `toB`).
+ * Converts `val` from the range (`fromStart`, `fromEnd`) to the range (`toStart`, `toEnd`).
  */
-export function interpolate(val: number, fromA: number, fromB: number, toA: number, toB: number) {
+export function interpolate(val: number, fromStart: number, fromEnd: number, toStart: number, toEnd: number) {
-  return ((val - fromA) / (fromB - fromA)) * (toB - toA) + toA
+  return ((val - fromStart) / (fromEnd - fromStart)) * (toEnd - toStart) + toStart
 }
 
 /**

src/electron/shared/interfaces/download.interface.ts

@@ -4,7 +4,7 @@ import { DriveChart } from './songDetails.interface'
  * Represents a user's request to interact with the download system.
  */
 export interface Download {
-  action: 'add' | 'retry' | 'continue' | 'cancel'
+  action: 'add' | 'retry' | 'cancel'
   versionID: number
   data?: NewDownload // Should be defined if action == 'add'
 }