More download bugfixes

package-lock.json

@@ -2142,6 +2142,15 @@
       "integrity": "sha512-Q1y515GcOdTHgagaVFhHnIFQ38ygs/kmxdNpvpou+raI9UO3YZcHDngBSYKQklcKlvA7iuQlmIKbzvmxcOE9CQ==",
       "dev": true
     },
+    "@types/destroy": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/@types/destroy/-/destroy-1.0.0.tgz",
+      "integrity": "sha512-nE3ePJLWPRu/qFHN8mj3fWnkr9K9ezwoiG4yOis2DuLeAawlnOOT/pM29JQkityrwfEvkblU5O9iS1bsiMqtDw==",
+      "dev": true,
+      "requires": {
+        "@types/node": "*"
+      }
+    },
     "@types/electron-window-state": {
       "version": "2.0.33",
       "resolved": "https://registry.npmjs.org/@types/electron-window-state/-/electron-window-state-2.0.33.tgz",
@@ -5423,8 +5432,7 @@
     "destroy": {
       "version": "1.0.4",
       "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz",
-      "integrity": "sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA=",
+      "integrity": "sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA="
-      "dev": true
     },
     "detect-file": {
       "version": "1.0.0",

package.json

@@ -36,6 +36,7 @@
     "@angular/router": "~9.1.4",
     "cli-color": "^2.0.0",
     "comparators": "^3.0.2",
+    "destroy": "^1.0.4",
     "electron-window-state": "^5.0.3",
     "fomantic-ui": "^2.8.3",
     "jquery": "^3.4.1",
@@ -56,6 +57,7 @@
     "@angular/compiler-cli": "~9.1.4",
     "@angular/language-service": "~9.1.4",
     "@types/cli-color": "^2.0.0",
+    "@types/destroy": "^1.0.0",
     "@types/electron-window-state": "^2.0.33",
     "@types/mv": "^2.1.0",
     "@types/needle": "^2.0.4",

src/app/components/browse/status-bar/downloads-modal/downloads-modal.component.ts

@@ -1,7 +1,7 @@
 import { Component, ChangeDetectorRef } from '@angular/core'
 import { DownloadProgress } from '../../../../../electron/shared/interfaces/download.interface'
 import { DownloadService } from '../../../../core/services/download.service'
-import { ElectronService } from 'src/app/core/services/electron.service'
+import { ElectronService } from '../../../../core/services/electron.service'
 
 @Component({
   selector: 'app-downloads-modal',
@@ -13,6 +13,10 @@ export class DownloadsModalComponent {
   downloads: DownloadProgress[] = []
 
   constructor(private electronService: ElectronService, private downloadService: DownloadService, ref: ChangeDetectorRef) {
+    electronService.receiveIPC('queue-updated', (order) => {
+      this.downloads.sort((a, b) => order.indexOf(a.versionID) - order.indexOf(b.versionID))
+    })
+
     downloadService.onDownloadUpdated(download => {
       const index = this.downloads.findIndex(thisDownload => thisDownload.versionID == download.versionID)
       if (index == -1) {

src/app/core/services/download.service.ts

@@ -1,7 +1,6 @@
 import { Injectable, EventEmitter } from '@angular/core'
 import { ElectronService } from './electron.service'
 import { NewDownload, DownloadProgress } from '../../../electron/shared/interfaces/download.interface'
-import * as _ from 'underscore'
 
 @Injectable({
   providedIn: 'root'
@@ -46,14 +45,7 @@ export class DownloadService {
   }
 
   onDownloadUpdated(callback: (download: DownloadProgress) => void) {
-    const debouncedCallback = _.throttle(callback, 30, { trailing: false })
+    this.downloadUpdatedEmitter.subscribe(callback)
-    this.downloadUpdatedEmitter.subscribe((download: DownloadProgress) => {
-      if (download.type == 'fastUpdate') { // 'good' updates can happen so frequently that the UI doesn't update correctly
-        debouncedCallback(download)
-      } else {
-        callback(download)
-      }
-    })
   }
 
   cancelDownload(versionID: number) {

src/electron/ipc/download/ChartDownload.ts

@@ -10,9 +10,11 @@ import { mkdir as _mkdir } from 'fs'
 import { ProgressType, NewDownload } from 'src/electron/shared/interfaces/download.interface'
 import { DriveFile } from 'src/electron/shared/interfaces/songDetails.interface'
 import { FileTransfer } from './FileTransfer'
+import * as _rimraf from 'rimraf'
 
 const randomBytes = promisify(_randomBytes)
 const mkdir = promisify(_mkdir)
+const rimraf = promisify(_rimraf)
 
 type EventCallback = {
   /** Note: this will not be the last event if `retry()` is called. */
@@ -31,6 +33,8 @@ export class ChartDownload {
   private callbacks = {} as Callbacks
   private files: DriveFile[]
   private percent = 0 // Needs to be stored here because errors won't know the exact percent
+  private chartPath: string
+  private dropFastUpdate = false
 
   private readonly individualFileProgressPortion: number
   private readonly destinationFolderName: string
@@ -68,6 +72,13 @@ export class ChartDownload {
     }
   }
 
+  /**
+   * Updates the GUI to indicate that a retry will be attempted.
+   */
+  displayRetrying() {
+    this.updateGUI('', 'Waiting for other downloads to finish to retry...', 'good')
+  }
+
   /**
    * Cancels the download if it is running.
    */
@@ -76,6 +87,7 @@ export class ChartDownload {
       const cancelFn = this.cancelFn
       this.cancelFn = undefined
       cancelFn()
+      rimraf(this.chartPath) // Delete temp folder
     }
   }
 
@@ -83,6 +95,15 @@ export class ChartDownload {
    * Updates the GUI with new information about this chart download.
    */
   private updateGUI(header: string, description: string, type: ProgressType) {
+    if (type == 'fastUpdate') {
+      if (this.dropFastUpdate) {
+        return
+      } else {
+        this.dropFastUpdate = true
+        setTimeout(() => this.dropFastUpdate = false, 30)
+      }
+    }
+
     emitIPCEvent('download-updated', {
       versionID: this.versionID,
       title: `${this.data.avTagName} - ${this.data.artist}`,
@@ -108,9 +129,8 @@ export class ChartDownload {
    */
   async beginDownload() {
     // CREATE DOWNLOAD DIRECTORY
-    let chartPath: string
     try {
-      chartPath = await this.createDownloadFolder()
+      this.chartPath = await this.createDownloadFolder()
     } catch (err) {
       this.retryFn = () => this.beginDownload()
       this.updateGUI('Access Error', err.message, 'error')
@@ -119,7 +139,7 @@ export class ChartDownload {
 
     // DOWNLOAD FILES
     for (let i = 0; i < this.files.length; i++) {
-      const downloader = new FileDownloader(this.files[i].webContentLink, join(chartPath, this.files[i].name))
+      const downloader = new FileDownloader(this.files[i].webContentLink, join(this.chartPath, this.files[i].name))
       this.cancelFn = () => downloader.cancelDownload()
 
       const downloadComplete = this.addDownloadEventListeners(downloader, i)
@@ -129,7 +149,7 @@ export class ChartDownload {
 
     // EXTRACT FILES
     if (this.isArchive) {
-      const extractor = new FileExtractor(chartPath)
+      const extractor = new FileExtractor(this.chartPath)
       this.cancelFn = () => extractor.cancelExtract()
 
       const extractComplete = this.addExtractorEventListeners(extractor)
@@ -138,7 +158,7 @@ export class ChartDownload {
     }
 
     // TRANSFER FILES
-    const transfer = new FileTransfer(chartPath, this.destinationFolderName)
+    const transfer = new FileTransfer(this.chartPath, this.destinationFolderName)
     this.cancelFn = () => transfer.cancelTransfer()
 
     const transferComplete = this.addTransferEventListeners(transfer)

src/electron/ipc/download/DownloadHandler.ts

@@ -29,10 +29,11 @@ class DownloadHandler implements IPCEmitHandler<'download'> {
     }
   }
 
-  private retryDownload(data: Download) { // TODO: cause this to send a GUI update that says waiting for download to finish...
+  private retryDownload(data: Download) {
     const index = this.retryWaiting.findIndex(download => download.versionID == data.versionID)
     if (index != -1) {
       const retryDownload = this.retryWaiting.splice(index, 1)[0]
+      retryDownload.displayRetrying()
       if (this.currentDownload == undefined) {
         this.currentDownload = retryDownload
         retryDownload.retry()

src/electron/ipc/download/DownloadQueue.ts

@@ -1,5 +1,6 @@
 import Comparators from 'comparators'
 import { ChartDownload } from './ChartDownload'
+import { emitIPCEvent } from '../../main'
 
 export class DownloadQueue {
 
@@ -15,7 +16,7 @@ export class DownloadQueue {
   }
 
   pop() {
-    return this.downloadQueue.pop()
+    return this.downloadQueue.shift()
   }
 
   get(versionID: number) {
@@ -27,6 +28,7 @@ export class DownloadQueue {
     if (index != -1) {
       this.downloadQueue[index].cancel()
       this.downloadQueue.splice(index, 1)
+      emitIPCEvent('queue-updated', this.downloadQueue.map(download => download.versionID))
     }
   }
 
@@ -39,5 +41,6 @@ export class DownloadQueue {
     }
 
     this.downloadQueue.sort(comparator)
+    emitIPCEvent('queue-updated', this.downloadQueue.map(download => download.versionID))
   }
 }

src/electron/ipc/download/FileDownloader.ts

@@ -2,6 +2,7 @@ import { AnyFunction } from '../../shared/UtilFunctions'
 import { createWriteStream } from 'fs'
 import * as needle from 'needle'
 // TODO: replace needle with got (for cancel() method) (if before-headers event is possible?)
+// TODO: add download throttle library and setting
 import { getSettings } from '../SettingsHandler.ipc'
 import { googleTimer } from './GoogleTimer'
 import { DownloadError } from './ChartDownload'
@@ -37,6 +38,7 @@ export class FileDownloader {
   private callbacks = {} as Callbacks
   private retryCount: number
   private wasCanceled = false
+  private req: NodeJS.ReadableStream
 
   /**
    * @param url The download link.
@@ -55,7 +57,6 @@ export class FileDownloader {
    * Download the file after waiting for the google rate limit.
    */
   beginDownload() {
-    console.log('Begin download...')
     if (getSettings().libraryPath == undefined) {
       this.failDownload(downloadErrors.libraryFolder())
     } else {
@@ -75,7 +76,7 @@ export class FileDownloader {
    */
   private requestDownload(cookieHeader?: string) {
     this.callbacks.requestSent()
-    const req = needle.get(this.url, {
+    this.req = needle.get(this.url, {
       'follow_max': 10,
       'open_timeout': 5000,
       'headers': Object.assign({
@@ -86,7 +87,7 @@ export class FileDownloader {
       )
     })
 
-    req.on('timeout', this.cancelable((type: string) => {
+    this.req.on('timeout', this.cancelable((type: string) => {
       this.retryCount++
       if (this.retryCount <= this.RETRY_MAX) {
         console.log(`TIMEOUT: Retry attempt ${this.retryCount}...`)
@@ -96,20 +97,20 @@ export class FileDownloader {
       }
     }))
 
-    req.on('err', this.cancelable((err: Error) => {
+    this.req.on('err', this.cancelable((err: Error) => {
       this.failDownload(downloadErrors.connectionError(err))
     }))
 
-    req.on('header', this.cancelable((statusCode, headers: Headers) => {
+    this.req.on('header', this.cancelable((statusCode, headers: Headers) => {
       if (statusCode != 200) {
         this.failDownload(downloadErrors.responseError(statusCode))
         return
       }
 
       if (headers['content-type'].startsWith('text/html')) {
-        this.handleHTMLResponse(req, headers['set-cookie'])
+        this.handleHTMLResponse(headers['set-cookie'])
       } else {
-        this.handleDownloadResponse(req)
+        this.handleDownloadResponse()
       }
     }))
   }
@@ -117,14 +118,12 @@ export class FileDownloader {
   /**
    * A Google Drive HTML response to a download request usually means this is the "file too large to scan for viruses" warning.
    * This function sends the request that results from clicking "download anyway", or generates an error if it can't be found.
-   * @param req The download request.
    * @param cookieHeader The "cookie=" header of this request.
    */
-  private handleHTMLResponse(req: NodeJS.ReadableStream, cookieHeader: string) {
+  private handleHTMLResponse(cookieHeader: string) {
-    console.log('HTML Response...')
     let virusScanHTML = ''
-    req.on('data', this.cancelable(data => virusScanHTML += data))
+    this.req.on('data', this.cancelable(data => virusScanHTML += data))
-    req.on('done', this.cancelable((err: Error) => {
+    this.req.on('done', this.cancelable((err: Error) => {
       if (err) {
         this.failDownload(downloadErrors.connectionError(err))
       } else {
@@ -149,21 +148,20 @@ export class FileDownloader {
    * Pipes the data from a download response to `this.fullPath`.
    * @param req The download request.
    */
-  private handleDownloadResponse(req: NodeJS.ReadableStream) {
+  private handleDownloadResponse() {
-    console.log('Download response...')
     this.callbacks.downloadProgress(0)
     let downloadedSize = 0
-    req.pipe(createWriteStream(this.fullPath))
+    this.req.pipe(createWriteStream(this.fullPath))
-    req.on('data', this.cancelable((data) => {
+    this.req.on('data', this.cancelable((data) => {
       downloadedSize += data.length
       this.callbacks.downloadProgress(downloadedSize)
     }))
 
-    req.on('err', this.cancelable((err: Error) => {
+    this.req.on('err', this.cancelable((err: Error) => {
       this.failDownload(downloadErrors.connectionError(err))
     }))
 
-    req.on('end', this.cancelable(() => {
+    this.req.on('end', this.cancelable(() => {
       this.callbacks.complete()
     }))
   }
@@ -181,6 +179,9 @@ export class FileDownloader {
   cancelDownload() {
     this.wasCanceled = true
     googleTimer.cancelTimer() // Prevents timer from trying to activate a download and resetting
+    if (this.req) {
+      // TODO: destroy request
+    }
   }
 
   /**

src/electron/ipc/download/FileTransfer.ts

@@ -17,14 +17,14 @@ type EventCallback = {
 type Callbacks = { [E in keyof EventCallback]: EventCallback[E] }
 
 const transferErrors = {
-  readError: (err: NodeJS.ErrnoException) => fsError(err, 'Failed to read file'),
+  readError: (err: NodeJS.ErrnoException) => fsError(err, 'Failed to read file.'),
-  deleteError: (err: NodeJS.ErrnoException) => fsError(err, 'Failed to delete file'),
+  deleteError: (err: NodeJS.ErrnoException) => fsError(err, 'Failed to delete file.'),
-  rimrafError: (err: NodeJS.ErrnoException) => fsError(err, 'Failed to delete folder'),
+  rimrafError: (err: NodeJS.ErrnoException) => fsError(err, 'Failed to delete folder.'),
-  mvError: (err: NodeJS.ErrnoException) => fsError(err, 'Failed to move folder to library')
+  mvError: (err: NodeJS.ErrnoException) => fsError(err, `Failed to move folder to library.${err.code == 'EPERM' ? ' (does the chart already exist?)' : ''}`)
 }
 
 function fsError(err: NodeJS.ErrnoException, description: string) {
-  return { header: `${description} (${err.code})`, body: `${err.name}: ${err.message}` }
+  return { header: description, body: `${err.name}: ${err.message}` }
 }
 
 export class FileTransfer {

src/electron/shared/IPCHandler.ts

@@ -76,6 +76,7 @@ export type IPCEmitEvents = {
   'download': Download
   'download-updated': DownloadProgress
   'set-settings': Settings
+  'queue-updated': number[]
 }
 
 /**