Replace Database connection with web server API

This commit is contained in:
Geomitron
2020-05-03 16:36:08 -04:00
parent 0b4e50174d
commit a3271bf164
20 changed files with 190 additions and 293 deletions

View File

@@ -6,6 +6,7 @@ import { AlbumArtService } from '../../../core/services/album-art.service'
import { DownloadService } from '../../../core/services/download.service'
import { groupBy } from 'src/electron/shared/UtilFunctions'
import { SearchService } from 'src/app/core/services/search.service'
import { DomSanitizer, SafeUrl } from '@angular/platform-browser'
@Component({
selector: 'app-chart-sidebar',
@@ -22,7 +23,8 @@ export class ChartSidebarComponent implements OnInit {
private electronService: ElectronService,
private albumArtService: AlbumArtService,
private downloadService: DownloadService,
private searchService: SearchService
private searchService: SearchService,
private sanitizer: DomSanitizer
) { }
ngOnInit() {
@@ -40,7 +42,8 @@ export class ChartSidebarComponent implements OnInit {
const albumArt = this.albumArtService.getImage(result.id)
const results = await this.electronService.invoke('song-details', result.id)
this.charts = groupBy(results, 'chartID').sort((v1, v2) => v1[0].avTagName.length - v2[0].avTagName.length)
this.charts.forEach(chart => chart.sort((v1, v2) => v2.lastModified - v1.lastModified))
// This sorting is very inefficient, but there's rarely more than two or three in this list, so it's fine
this.charts.forEach(chart => chart.sort((v1, v2) => new Date(v2.lastModified).getTime() - new Date(v1.lastModified).getTime()))
await this.selectChart(0)
this.initChartDropdown()
@@ -48,13 +51,13 @@ export class ChartSidebarComponent implements OnInit {
}
}
albumArtSrc = ''
albumArtSrc: SafeUrl = ''
/**
* Updates the sidebar to display the album art.
*/
updateAlbumArtSrc(albumArtBuffer?: Buffer) {
if (albumArtBuffer) {
this.albumArtSrc = 'data:image/jpg;base64,' + albumArtBuffer.toString('base64')
updateAlbumArtSrc(albumArtBase64String?: string) {
if (albumArtBase64String) {
this.albumArtSrc = this.sanitizer.bypassSecurityTrustUrl('data:image/jpg;base64,' + albumArtBase64String)
} else {
this.albumArtSrc = ''
}
@@ -168,7 +171,7 @@ export class ChartSidebarComponent implements OnInit {
* Converts the <lastModified> value to a user-readable format.
* @param lastModified The UNIX timestamp for the lastModified date.
*/
private getLastModifiedText(lastModified: number) {
private getLastModifiedText(lastModified: string) {
return new Date(lastModified).toLocaleDateString()
}
@@ -181,7 +184,7 @@ export class ChartSidebarComponent implements OnInit {
avTagName: this.selectedVersion.avTagName,
artist: this.songResult.artist,
charter: this.selectedVersion.charters, // TODO: get the charter name associated with this particular version
links: JSON.parse(this.selectedVersion.downloadLink)
driveData: this.selectedVersion.driveData
})
}
}

View File

@@ -81,7 +81,7 @@ export class StatusBarComponent {
avTagName: downloadVersion.avTagName,
artist: downloadSong.artist,
charter: downloadVersion.charters,
links: JSON.parse(downloadVersion.downloadLink)
driveData: downloadVersion.driveData
})
}
} else {
@@ -99,7 +99,7 @@ export class StatusBarComponent {
avTagName: version.avTagName,
artist: downloadSong.artist,
charter: version.charters,
links: JSON.parse(version.downloadLink)
driveData: version.driveData
})
}
}

View File

@@ -8,13 +8,13 @@ export class AlbumArtService {
constructor(private electronService: ElectronService) { }
private imageCache: { [songID: number]: Buffer } = {}
private imageCache: { [songID: number]: string } = {}
async getImage(songID: number): Promise<Buffer | null> {
async getImage(songID: number): Promise<string | null> {
if (this.imageCache[songID] == undefined) {
const albumArtResult = await this.electronService.invoke('album-art', songID)
if (albumArtResult) {
this.imageCache[songID] = albumArtResult.art
this.imageCache[songID] = albumArtResult.base64Art
} else {
this.imageCache[songID] = null
}

View File

@@ -1,6 +1,7 @@
import { IPCInvokeHandler } from '../shared/IPCHandler'
import Database from '../shared/Database'
import { AlbumArtResult } from '../shared/interfaces/songDetails.interface'
import * as needle from 'needle'
import { serverURL } from '../shared/Paths'
/**
* Handles the 'album-art' event.
@@ -12,20 +13,19 @@ class AlbumArtHandler implements IPCInvokeHandler<'album-art'> {
* @returns an `AlbumArtResult` object containing the album art for the song with `songID`.
*/
async handler(songID: number) {
const db = await Database.getInstance()
return db.sendQuery(this.getAlbumArtQuery(songID), 1) as Promise<AlbumArtResult>
}
/**
* @returns a database query that returns the album art for the song with `songID`.
*/
private getAlbumArtQuery(songID: number) {
return `
SELECT art
FROM AlbumArt
WHERE songID = ${songID};
`
return new Promise<AlbumArtResult>((resolve, reject) => {
needle.request(
'get',
serverURL + `/api/data/albumArt`, {
songID: songID
}, (err, response) => {
if (err) {
reject(err)
} else {
resolve(response.body)
}
})
})
}
}

View File

@@ -1,6 +1,7 @@
import { IPCInvokeHandler } from '../shared/IPCHandler'
import Database from '../shared/Database'
import { VersionResult } from '../shared/interfaces/songDetails.interface'
import { serverURL } from '../shared/Paths'
import * as needle from 'needle'
/**
* Handles the 'batch-song-details' event.
@@ -12,20 +13,19 @@ class BatchSongDetailsHandler implements IPCInvokeHandler<'batch-song-details'>
* @returns an array of all the chart versions with a songID found in `songIDs`.
*/
async handler(songIDs: number[]) {
const db = await Database.getInstance()
return db.sendQuery(this.getVersionQuery(songIDs)) as Promise<VersionResult[]>
}
/**
* @returns a database query that returns all the chart versions with a songID found in `songIDs`.
*/
private getVersionQuery(songIDs: number[]) {
return `
SELECT *
FROM VersionMetaFull
WHERE songID IN (${songIDs.join(',')});
`
return new Promise<VersionResult[]>((resolve, reject) => {
needle.request(
'get',
serverURL + `/api/data/songsVersions`, {
songIDs: songIDs
}, (err, response) => {
if (err) {
reject(err)
} else {
resolve(response.body)
}
})
})
}
}

View File

@@ -1,8 +1,7 @@
import { IPCInvokeHandler } from '../shared/IPCHandler'
import Database from '../shared/Database'
import { SongSearch, SearchType, SongResult } from '../shared/interfaces/search.interface'
import { escape } from 'mysql'
import * as needle from 'needle'
import { serverURL } from '../shared/Paths'
/**
* Handles the 'song-search' event.
*/
@@ -13,32 +12,32 @@ class SearchHandler implements IPCInvokeHandler<'song-search'> {
* @returns the top 20 songs that match `search`.
*/
async handler(search: SongSearch) {
const db = await Database.getInstance()
return db.sendQuery(this.getSearchQuery(search)) as Promise<SongResult[]>
return new Promise<SongResult[]>((resolve, reject) => {
needle.request(
'get',
serverURL + `/api/search/${this.getSearchType(search)}`, {
query: search.query,
limit: search.length,
offset: search.offset
}, (err, response) => {
if (err) {
reject(err)
} else {
resolve(response.body)
}
})
})
}
/**
* @returns a database query that returns the type of results expected by `search.type`.
* @returns the search api type that corresponds with `search.type`.
*/
private getSearchQuery(search: SongSearch) {
private getSearchType(search: SongSearch) {
switch (search.type) {
case SearchType.Any: return this.getGeneralSearchQuery(search)
case SearchType.Any: return 'general'
default: return '<<<ERROR>>>' // TODO: add more search types
}
}
/**
* @returns a database query that returns the top 20 songs that match `search`.
*/
private getGeneralSearchQuery(search: SongSearch) {
return `
SELECT id, name, artist, album, genre, year
FROM Song
WHERE MATCH (name,artist,album,genre) AGAINST (${escape(search.query)}) > 0
LIMIT ${search.length} OFFSET ${search.offset};
` // TODO: add parameters for the limit and offset
}
}
export const searchHandler = new SearchHandler()

View File

@@ -1,6 +1,7 @@
import { IPCInvokeHandler } from '../shared/IPCHandler'
import Database from '../shared/Database'
import { VersionResult } from '../shared/interfaces/songDetails.interface'
import * as needle from 'needle'
import { serverURL } from '../shared/Paths'
/**
* Handles the 'song-details' event.
@@ -12,20 +13,19 @@ class SongDetailsHandler implements IPCInvokeHandler<'song-details'> {
* @returns the chart versions with `songID`.
*/
async handler(songID: number) {
const db = await Database.getInstance()
return db.sendQuery(this.getVersionQuery(songID)) as Promise<VersionResult[]>
}
/**
* @returns a database query that returns the chart versions with `songID`.
*/
private getVersionQuery(songID: number) {
return `
SELECT *
FROM VersionMetaFull
WHERE songID = ${songID};
`
return new Promise<VersionResult[]>((resolve, reject) => {
needle.request(
'get',
serverURL + `/api/data/songVersions`, {
songID: songID
}, (err, response) => {
if (err) {
reject(err)
} else {
resolve(response.body)
}
})
})
}
}

View File

@@ -5,11 +5,12 @@ import { FileExtractor } from './FileExtractor'
import { sanitizeFilename, interpolate } from '../../shared/UtilFunctions'
import { emitIPCEvent } from '../../main'
import { promisify } from 'util'
import { randomBytes as _randomBytes, createHash } from 'crypto'
import { randomBytes as _randomBytes } from 'crypto'
import { mkdir as _mkdir } from 'fs'
import { ProgressType, NewDownload } from 'src/electron/shared/interfaces/download.interface'
import { downloadHandler } from './DownloadHandler'
import { googleTimer } from './GoogleTimer'
import { DriveFile } from 'src/electron/shared/interfaces/songDetails.interface'
const randomBytes = promisify(_randomBytes)
const mkdir = promisify(_mkdir)
@@ -21,26 +22,20 @@ export class ChartDownload {
cancel: () => void
isArchive: boolean
isGoogle: boolean
title: string
header: string
description: string
percent = 0
type: ProgressType
private fileKeys: string[]
private fileValues: string[]
private files: DriveFile[]
allFilesProgress = 0
individualFileProgressPortion: number
constructor(public versionID: number, private data: NewDownload) {
// Only iterate over the keys in data.links that have link values (not hashes)
this.fileKeys = Object.keys(data.links).filter(link => data.links[link].includes('.'))
this.fileValues = Object.values(data.links).filter(value => value.includes('.'))
this.isArchive = this.fileKeys.includes('archive')
this.isGoogle = !!this.fileValues.find(value => value.toLocaleLowerCase().includes('google'))
this.individualFileProgressPortion = 80 / this.fileKeys.length
this.files = data.driveData.files
this.isArchive = data.driveData.isArchive
this.individualFileProgressPortion = 80 / this.files.length
}
/**
@@ -69,31 +64,27 @@ export class ChartDownload {
return
}
// For each actual download link in <this.data.links>, download the file to <chartPath>
for (let i = 0; i < this.fileKeys.length; i++) {
// For each download link in <this.files>, download the file to <chartPath>
for (let i = 0; i < this.files.length; i++) {
// INITIALIZE DOWNLOADER
const typeHash = createHash('md5').update(this.fileValues[i]).digest('hex')
// <this.data.links[typeHash]> stores the expected hash value found in the download header
const downloader = new FileDownloader(this.fileValues[i], chartPath, this.data.links[typeHash])
const downloader = new FileDownloader(this.files[i].webContentLink, chartPath)
const downloadComplete = this.addDownloadEventListeners(downloader, i)
// DOWNLOAD THE NEXT FILE
if (this.isGoogle) { // If this is a google download...
// Wait for google rate limit
this.header = `[${this.fileKeys[i]}] (file ${i + 1}/${this.fileKeys.length})`
googleTimer.onTimerUpdate((remainingTime, totalTime) => {
this.description = `Waiting for Google rate limit... (${remainingTime}s)`
this.percent = this.allFilesProgress + interpolate(remainingTime, totalTime, 0, 0, this.individualFileProgressPortion / 2)
this.type = 'good'
emitIPCEvent('download-updated', this)
})
this.cancel = () => {
googleTimer.removeCallbacks()
this.onDownloadStop()
downloader.cancelDownload()
}
await new Promise<void>(resolve => googleTimer.onTimerReady(resolve))
// Wait for google rate limit
this.header = `[${this.files[i].originalFilename}] (file ${i + 1}/${this.files.length})`
googleTimer.onTimerUpdate((remainingTime, totalTime) => {
this.description = `Waiting for Google rate limit... (${remainingTime}s)`
this.percent = this.allFilesProgress + interpolate(remainingTime, totalTime, 0, 0, this.individualFileProgressPortion / 2)
this.type = 'good'
emitIPCEvent('download-updated', this)
})
this.cancel = () => {
googleTimer.removeCallbacks()
this.onDownloadStop()
downloader.cancelDownload()
}
await new Promise<void>(resolve => googleTimer.onTimerReady(resolve))
this.cancel = () => {
this.onDownloadStop()
@@ -149,10 +140,8 @@ export class ChartDownload {
* If this was a google download, allows a new google download to start.
*/
private onDownloadStop() {
if (this.isGoogle) {
downloadHandler.isGoogleDownloading = false
downloadHandler.updateQueue()
}
downloadHandler.isGoogleDownloading = false
downloadHandler.updateQueue()
}
/**
@@ -177,28 +166,12 @@ export class ChartDownload {
emitIPCEvent('download-updated', this)
})
let filesize = -1
downloader.on('download', (filename, _filesize) => {
this.header = `[${filename}] (file ${fileIndex + 1}/${this.fileKeys.length})`
if (_filesize != undefined) {
filesize = _filesize
this.description = 'Downloading... (0%)'
} else {
this.description = 'Downloading... (0 MB)'
}
this.type = 'good'
emitIPCEvent('download-updated', this)
})
downloader.on('downloadProgress', (bytesDownloaded) => {
if (filesize != -1) {
this.description = `Downloading... (${Math.round(1000 * bytesDownloaded / filesize) / 10}%)`
fileProgress = interpolate(bytesDownloaded, 0, filesize, this.individualFileProgressPortion / 2, this.individualFileProgressPortion)
this.percent = this.allFilesProgress + fileProgress
} else {
this.description = `Downloading... (${Math.round(bytesDownloaded / 1e+5) / 10} MB)`
this.percent = this.allFilesProgress + fileProgress
}
const size = Number(this.files[fileIndex].size)
this.header = `[${this.files[fileIndex].originalFilename}] (file ${fileIndex + 1}/${this.files.length})`
this.description = `Downloading... (${Math.round(1000 * bytesDownloaded / size) / 10}%)`
fileProgress = interpolate(bytesDownloaded, 0, size, this.individualFileProgressPortion / 2, this.individualFileProgressPortion)
this.percent = this.allFilesProgress + fileProgress
this.type = 'fastUpdate'
emitIPCEvent('download-updated', this)
})

View File

@@ -32,17 +32,15 @@ class DownloadHandler implements IPCEmitHandler<'download'> {
*/
updateQueue() {
this.downloadQueue.sort((cd1: ChartDownload, cd2: ChartDownload) => {
const value1 = (cd1.isGoogle ? 100 : 0) + (99 - cd1.allFilesProgress)
const value2 = (cd2.isGoogle ? 100 : 0) + (99 - cd2.allFilesProgress)
const value1 = 100 + (99 - cd1.allFilesProgress)
const value2 = 100 + (99 - cd2.allFilesProgress)
return value1 - value2 // Sorts in the order to get the most downloads completed early
})
while (this.downloadQueue[0] != undefined && !(this.downloadQueue[0].isGoogle && this.isGoogleDownloading)) {
while (this.downloadQueue[0] != undefined && !(this.isGoogleDownloading)) {
const nextDownload = this.downloadQueue.shift()
nextDownload.run()
if (nextDownload.isGoogle) {
this.isGoogleDownloading = true
}
this.isGoogleDownloading = true
}
}
}

View File

@@ -8,7 +8,6 @@ import { getSettings } from '../SettingsHandler.ipc'
type EventCallback = {
'request': () => void
'warning': (continueAnyway: () => void) => void
'download': (filename: string, filesize?: number) => void
'downloadProgress': (bytesDownloaded: number) => void
'error': (error: DownloadError, retry: () => void) => void
'complete': () => void
@@ -34,7 +33,7 @@ export class FileDownloader {
* @param destinationFolder The path to where this file should be stored.
* @param expectedHash The hash header value that is expected for this file.
*/
constructor(private url: string, private destinationFolder: string, private expectedHash?: string) { }
constructor(private url: string, private destinationFolder: string) { }
/**
* Calls `callback` when `event` fires.
@@ -103,14 +102,7 @@ export class FileDownloader {
this.handleHTMLResponse(req, headers['set-cookie'])
} else {
const fileName = this.getDownloadFileName(headers)
this.handleDownloadResponse(req, fileName, headers['content-length'])
if (this.expectedHash !== undefined && this.getDownloadHash(headers) !== this.expectedHash) {
req.pause()
this.callbacks.warning(() => {
req.resume()
})
}
this.handleDownloadResponse(req, fileName)
}
})
}
@@ -149,10 +141,9 @@ export class FileDownloader {
* Pipes the data from a download response to `fileName`.
* @param req The download request.
* @param fileName The name of the output file.
* @param contentLength The number of bytes to be downloaded. If undefined, download progress is indicated by MB, not %.
*/
private handleDownloadResponse(req: NodeJS.ReadableStream, fileName: string, contentLength?: number) {
this.callbacks.download(fileName, contentLength)
private handleDownloadResponse(req: NodeJS.ReadableStream, fileName: string) {
this.callbacks.downloadProgress(0)
let downloadedSize = 0
const filePath = path.join(this.destinationFolder, fileName)
req.pipe(fs.createWriteStream(filePath))
@@ -192,20 +183,6 @@ export class FileDownloader {
}
}
/**
* Extracts the downloaded file's hash from `headers`, depending on the file's host server.
* @param headers The response headers for this request.
*/
private getDownloadHash(headers: Headers): string {
if (headers['server'] && headers['server'] == 'cloudflare' || this.url.startsWith('https://public.fightthe.pw/')) {
// Cloudflare and Chorus specific jazz
return String(headers['content-length']) // No actual hash is provided in the header, so this is the next best thing
} else {
// GDrive specific jazz
return headers['x-goog-hash']
}
}
cancelDownload() {
this.wasCanceled = true
}

View File

@@ -105,6 +105,7 @@ export class FileExtractor {
private async transfer(archiveFilepath?: string) {
// TODO: this fails if the extracted chart has nested folders
// TODO: skip over "__MACOSX" folder
// TODO: handle other common problems, like chart/audio files not named correctly
if (this.wasCanceled) { return } // CANCEL POINT
try {

View File

@@ -5,7 +5,6 @@ import * as url from 'url'
// IPC Handlers
import { getIPCInvokeHandlers, getIPCEmitHandlers, IPCEmitEvents } from './shared/IPCHandler'
import Database from './shared/Database'
import { getSettingsHandler } from './ipc/SettingsHandler.ipc'
import { dataPath } from './shared/Paths'
@@ -86,7 +85,6 @@ function createBridgeWindow() {
}
mainWindow.on('closed', () => {
Database.closeConnection()
mainWindow = null // Dereference mainWindow when the window is closed
})
}

View File

@@ -1,83 +0,0 @@
import { Connection, createConnection } from 'mysql'
import { failQuery } from './ErrorMessages'
export default class Database {
// Singleton
private static database: Database
private constructor() { }
static async getInstance() {
if (this.database == undefined) {
this.database = new Database()
await this.database.initDatabaseConnection()
}
return this.database
}
private conn: Connection
/**
* Constructs a database connection to the chartmanager database.
*/
private async initDatabaseConnection() {
this.conn = createConnection({
host: 'chartmanager.cdtrqnlcxz86.us-east-1.rds.amazonaws.com',
port: 3306,
user: 'standarduser',
password: 'E4OZXWDPiX9exUpMhcQq', // Note: this login is read-only
database: 'chartmanagerdatabase',
multipleStatements: true,
charset: 'utf8mb4',
typeCast: (field, next) => { // Convert 1/0 to true/false
if (field.type == 'TINY' && field.length == 1) {
return (field.string() == '1') // 1 = true, 0 = false
} else {
return next()
}
}
})
// TODO: make this error message more user-friendly (retry option?)
return new Promise<void>((resolve, reject) => {
this.conn.connect(err => {
if (err) {
reject(`Failed to connect to database: ${err}`)
return
} else {
resolve()
}
})
})
}
/**
* Destroys the database connection.
*/
static closeConnection() {
if (this.database != undefined) {
this.database.conn.destroy()
}
}
/**
* Sends `query` to the database.
* @param queryStatement The nth response statement to be returned. If undefined, the entire response is returned.
* @returns the selected response statement, or an empty array if the query fails.
*/
async sendQuery<ResponseType>(query: string, queryStatement?: number) {
return new Promise<ResponseType[] | ResponseType>(resolve => {
this.conn.query(query, (err, results) => {
if (err) {
failQuery(query, err)
resolve([])
return
}
if (queryStatement !== undefined) {
resolve(results[queryStatement - 1])
} else {
resolve(results)
}
})
})
}
}

View File

@@ -6,4 +6,7 @@ export const dataPath = path.join(app.getPath('userData'), 'bridge_data')
export const libraryPath = path.join(dataPath, 'library.db')
export const settingsPath = path.join(dataPath, 'settings.json')
export const tempPath = path.join(dataPath, 'temp')
export const themesPath = path.join(dataPath, 'themes')
export const themesPath = path.join(dataPath, 'themes')
// URL
export const serverURL = '64.53.210.87'

View File

@@ -1,3 +1,5 @@
import { DriveChart } from './songDetails.interface'
/**
* Represents a user's request to interact with the download system.
*/
@@ -14,7 +16,7 @@ export interface NewDownload {
avTagName: string
artist: string
charter: string
links: { [type: string]: string }
driveData: DriveChart
}
/**

View File

@@ -2,7 +2,7 @@
* The image data for a song's album art.
*/
export interface AlbumArtResult {
art: Buffer
base64Art: string
}
/**
@@ -15,13 +15,12 @@ export interface VersionResult {
latestVersionID: number
latestSetlistVersionID: number
icon: string
name: string
driveData: DriveChart & { inChartPack: boolean }
avTagName: string
charters: string
charterIDs: string
tags: string | null
downloadLink: string
lastModified: number
lastModified: string
song_length: number
diff_band: number
diff_guitar: number
@@ -32,5 +31,53 @@ export interface VersionResult {
diff_guitarghl: number
diff_bassghl: number
songDataIncorrect: boolean
isUnusualAvTagName: boolean
year: string
chartMetadata: ChartMetadata
}
export interface DriveChart {
source: DriveSource
isArchive: boolean
downloadPath: string
filesHash: string
files: DriveFile[]
}
export interface DriveSource {
isSetlistSource: boolean
setlistIcon?: string
sourceUserIDs: number[]
sourceName: string
sourceDriveID: string
}
export interface DriveFile {
id: string
originalFilename: string
mimeType: string
webContentLink: string
modifiedTime: string
md5Checksum: string
size: string
}
export interface ChartMetadata {
hasSections: boolean
hasStarPower: boolean
hasForced: boolean
hasTap: boolean
hasOpen: {
[instrument: string]: boolean
}
hasSoloSections: boolean
hasLyrics: boolean
is120: boolean
hasBrokenNotes: boolean
noteCounts: {
[instrument: string]: {
[difficulty: string]: number
}
}
length: number
effectiveLength: number
}

2
src/typings.d.ts vendored
View File

@@ -6,7 +6,7 @@ interface NodeModule {
}
// @ts-ignore
declare let window: Window
// declare let window: Window
declare let $: any
interface Window {
process: any