Add "Tools" tab with chart issue scanner

src-electron/ipc/issue-scan/ExcelBuilder.ts

@@ -0,0 +1,240 @@
+import exceljs, { Borders } from 'exceljs'
+import _ from 'lodash'
+import { FolderIssueType, ScannedChart } from 'scan-chart'
+
+export function getChartIssues(charts: { chart: ScannedChart; path: string }[]) {
+	const chartIssues: {
+		path: string
+		artist: string
+		name: string
+		charter: string
+		errorName: string
+		errorDescription: string
+		fixMandatory: boolean
+	}[] = []
+
+	for (const chart of charts) {
+		const addIssue = (
+			errorName: string,
+			errorDescription: string,
+			fixMandatory: boolean,
+		) => {
+
+			chartIssues.push({
+				path: chart.path,
+				artist: removeStyleTags(chart.chart.artist ?? ''),
+				name: removeStyleTags(chart.chart.name ?? ''),
+				charter: removeStyleTags(chart.chart.charter ?? ''),
+				errorName,
+				errorDescription,
+				fixMandatory,
+			})
+		}
+
+		if (chart.chart.folderIssues.length > 0) {
+			for (const folderIssue of chart.chart.folderIssues) {
+				if (folderIssue.folderIssue === 'albumArtSize') {
+					continue
+				} // Ignored; .sng conversion fixes this
+				addIssue(
+					folderIssue.folderIssue,
+					folderIssue.description,
+					(
+						[
+							'noMetadata',
+							'invalidMetadata',
+							'noAudio',
+							'badAudio',
+							'noChart',
+							'invalidChart',
+							'badChart',
+						] satisfies FolderIssueType[] as FolderIssueType[]
+					).includes(folderIssue.folderIssue),
+				)
+			}
+		}
+
+		for (const metadataIssue of chart.chart.metadataIssues) {
+			addIssue(
+				metadataIssue.metadataIssue,
+				metadataIssue.description,
+				['"name"', '"artist"', '"charter"'].some(property => metadataIssue.description.includes(property)),
+			)
+		}
+
+		if (chart.chart.notesData) {
+			for (const issue of chart.chart.notesData.chartIssues) {
+				addIssue(
+					issue.noteIssue,
+					`${issue.instrument ? `[${issue.instrument}]` : ''}${issue.difficulty ? `[${issue.difficulty}]` : ''} ${issue.description}`,
+					issue.noteIssue === 'noNotes',
+				)
+			}
+		}
+	}
+
+	return chartIssues
+}
+
+export async function getIssuesXLSX(
+	chartIssues: Awaited<ReturnType<typeof getChartIssues>>,
+) {
+	const chartIssueHeaders = [
+		{ text: 'Artist', width: 160 / 7 },
+		{ text: 'Name', width: 400 / 7 },
+		{ text: 'Charter', width: 160 / 7 },
+		{ text: 'Issue Name', width: 160 / 7 },
+		{
+			text: 'Issue Description (a more detailed description of issue types can be '
+				+ 'found at https://drive.google.com/open?id=1UK7GsP4ZHJkOg8uREFRMY72svySaDlf0QRTGlk-ruYQ)',
+			width: 650 / 7,
+		},
+		{ text: 'Fix Mandatory?', width: 120 / 7 },
+		{ text: 'Path', width: 600 / 7 },
+	]
+	const chartIssueRows: (string | { text: string; hyperlink: string })[][] = []
+	for (const issue of chartIssues) {
+		chartIssueRows.push([
+			issue.artist,
+			issue.name,
+			issue.charter,
+			issue.errorName,
+			issue.errorDescription,
+			issue.fixMandatory ? 'yes' : 'no',
+			issue.path,
+		])
+	}
+
+	const gridlineBorderStyle = {
+		top: { style: 'thin', color: { argb: 'FFD0D0D0' } },
+		left: { style: 'thin', color: { argb: 'FFD0D0D0' } },
+		bottom: { style: 'thin', color: { argb: 'FFD0D0D0' } },
+		right: { style: 'thin', color: { argb: 'FFD0D0D0' } },
+	} satisfies Partial<Borders>
+	const workbook = new exceljs.Workbook()
+	workbook.creator = 'Chorus'
+	workbook.created = new Date()
+	workbook.modified = new Date()
+
+	const chartIssuesWorksheet = workbook.addWorksheet('Chart Issues', {
+		views: [{ state: 'frozen', ySplit: 1 }], // Sticky header row
+	})
+	chartIssuesWorksheet.autoFilter = {
+		from: { row: 1, column: 1 },
+		to: { row: chartIssueRows.length + 1, column: chartIssueHeaders.length },
+	}
+	chartIssueHeaders.forEach((header, index) => {
+		const cell = chartIssuesWorksheet.getCell(1, index + 1)
+		cell.value = header.text
+		cell.fill = {
+			type: 'pattern',
+			pattern: 'solid',
+			fgColor: { argb: 'FFD3D3D3' },
+		}
+		cell.font = { bold: true }
+		const column = chartIssuesWorksheet.getColumn(index + 1)
+		column.width = header.width
+		column.border = gridlineBorderStyle
+	})
+	chartIssuesWorksheet.addRows(chartIssueRows)
+	chartIssuesWorksheet.addConditionalFormatting({
+		ref: `A2:${columnNumberToLetter(chartIssueHeaders.length)}${chartIssueRows.length + 1
+			}`,
+		rules: [
+			{
+				type: 'expression',
+				priority: 99999,
+				formulae: ['MOD(ROW(),2)=0'],
+				style: {
+					fill: {
+						type: 'pattern',
+						pattern: 'solid',
+						fgColor: { argb: 'FFF7F7F7' },
+					},
+				},
+			},
+		],
+	})
+
+	return await workbook.xlsx.writeBuffer({ useStyles: true })
+}
+
+export function columnNumberToLetter(column: number) {
+	let temp,
+		letter = ''
+	while (column > 0) {
+		temp = (column - 1) % 26
+		letter = String.fromCharCode(temp + 65) + letter
+		column = (column - temp - 1) / 26
+	}
+	return letter
+}
+
+/**
+ * @returns an string representation of `ms` that looks like HH:MM:SS.mm
+ */
+export function msToExactTime(ms: number) {
+	const seconds = _.round((ms / 1000) % 60, 2)
+	const minutes = Math.floor((ms / 1000 / 60) % 60)
+	const hours = Math.floor((ms / 1000 / 60 / 60) % 24)
+	return `${hours ? `${hours}:` : ''}${_.padStart(
+		minutes + '',
+		2,
+		'0',
+	)}:${_.padStart(seconds.toFixed(2), 5, '0')}`
+}
+
+const allowedTags = [
+	'align',
+	'allcaps',
+	'alpha',
+	'b',
+	'br',
+	'color',
+	'cspace',
+	'font',
+	'font-weight',
+	'gradient',
+	'i',
+	'indent',
+	'line-height',
+	'line-indent',
+	'link',
+	'lowercase',
+	'margin',
+	'mark',
+	'mspace',
+	'nobr',
+	'noparse',
+	'page',
+	'pos',
+	'rotate',
+	's',
+	'size',
+	'smallcaps',
+	'space',
+	'sprite',
+	'strikethrough',
+	'style',
+	'sub',
+	'sup',
+	'u',
+	'uppercase',
+	'voffset',
+	'width',
+]
+const tagPattern = allowedTags.map(tag => `\\b${tag}\\b`).join('|')
+/**
+ * @returns `text` with all style tags removed. (e.g. "<color=#AEFFFF>Aren Eternal</color> & Geo" -> "Aren Eternal & Geo")
+ */
+export function removeStyleTags(text: string) {
+	let oldText = text
+	let newText = text
+	do {
+		oldText = newText
+		newText = newText
+			.replace(new RegExp(`<\\s*\\/?\\s*(?:#|${tagPattern})[^>]*>`, 'gi'), '')
+			.trim()
+	} while (newText !== oldText)
+	return newText
+}

src-electron/ipc/issue-scan/IssueScanHandler.ipc.ts

@@ -0,0 +1,183 @@
+import Bottleneck from 'bottleneck'
+import dayjs from 'dayjs'
+import { shell } from 'electron'
+import { createReadStream } from 'fs'
+import pkg from 'fs-extra'
+import _ from 'lodash'
+import { SngHeader, SngStream } from 'parse-sng'
+import { scanChartFolder, ScannedChart } from 'scan-chart'
+import { Readable } from 'stream'
+import { inspect } from 'util'
+
+import { appearsToBeChartFolder, getExtension, hasAlbumName, hasChartExtension, hasIniExtension, hasSngExtension } from '../../../src-shared/UtilFunctions.js'
+import { hasVideoExtension } from '../../ElectronUtilFunctions.js'
+import { emitIpcEvent } from '../../main.js'
+import { getSettings } from '../SettingsHandler.ipc.js'
+import { getChartIssues, getIssuesXLSX } from './ExcelBuilder.js'
+
+const { readdir, readFile, writeFile } = pkg
+export async function scanIssues() {
+	const settings = await getSettings()
+	if (!settings.issueScanPath || !settings.spreadsheetOutputPath) {
+		emitIpcEvent('updateIssueScan', {
+			status: 'error',
+			message: 'Scan path or output path were not properly defined.',
+		})
+		return
+	}
+
+	try {
+		const chartFolders = await getChartFolders(settings.issueScanPath)
+
+		const limiter = new Bottleneck({ maxConcurrent: 20 }) // Ensures memory use stays bounded
+
+		const charts: { chart: ScannedChart; path: string }[] = []
+		for (const chartFolder of chartFolders) {
+			limiter.schedule(async () => {
+				const isSng = chartFolder.files.length === 1 && hasSngExtension(chartFolder.files[0])
+				const files = isSng ? await getFilesFromSng([chartFolder.path, chartFolder.files[0]].join('/')) : await getFilesFromFolder(chartFolder)
+
+				const result: { chart: ScannedChart; path: string } = {
+					chart: scanChartFolder(files),
+					path: chartFolder.path,
+				}
+				charts.push(result)
+				emitIpcEvent('updateIssueScan', { status: 'progress', message: `${charts.length}/${chartFolders.length} scanned...` })
+			})
+		}
+
+		await new Promise<void>((resolve, reject) => {
+			limiter.on('error', err => {
+				reject(err)
+				limiter.stop()
+			})
+
+			limiter.on('idle', async () => {
+				const issues = getChartIssues(charts)
+				const xlsx = await getIssuesXLSX(issues)
+				const outputPath = [settings.spreadsheetOutputPath, `chart_issues_${dayjs().format('YYYY.MM.DD_HH.mm.ss')}.xlsx`].join('/')
+				await writeFile(outputPath, new Uint8Array(xlsx))
+				await new Promise<void>(resolve => setTimeout(resolve, 500)) // Delay for OS file processing
+				await shell.openPath(outputPath)
+				emitIpcEvent('updateIssueScan', {
+					status: 'done',
+					message: `${issues.length} issues found in ${charts.length} charts. Spreadsheet saved to ${outputPath}`,
+				})
+				resolve()
+			})
+		})
+	} catch (err) {
+		emitIpcEvent('updateIssueScan', { status: 'error', message: inspect(err) })
+	}
+}
+
+/**
+ * @returns valid chart folders in `path` and all its subdirectories.
+ */
+async function getChartFolders(path: string) {
+	const chartFolders: { path: string; files: string[] }[] = []
+
+	const entries = await readdir(path, { withFileTypes: true })
+
+	const subfolders = _.chain(entries)
+		.filter(entry => entry.isDirectory() && entry.name !== '__MACOSX') // Apple should follow the principle of least astonishment (smh)
+		.map(folder => getChartFolders([path, folder.name].join('/')))
+		.value()
+
+	chartFolders.push(..._.flatMap(await Promise.all(subfolders)))
+
+	const sngFiles = entries.filter(entry => !entry.isDirectory() && hasSngExtension(entry.name))
+	chartFolders.push(...sngFiles.map(sf => ({ path, files: [sf.name] })))
+
+	if (
+		subfolders.length === 0 && // Charts won't contain other charts
+		appearsToBeChartFolder(entries.map(entry => getExtension(entry.name)))
+	) {
+		chartFolders.push({
+			path,
+			files: entries.filter(entry => !entry.isDirectory()).map(entry => entry.name),
+		})
+		emitIpcEvent('updateIssueScan', { status: 'progress', message: `${chartFolders} charts found...` })
+	}
+
+	return chartFolders
+}
+
+async function getFilesFromSng(sngPath: string) {
+	const sngStream = new SngStream(Readable.toWeb(createReadStream(sngPath)) as ReadableStream<Uint8Array>, { generateSongIni: true })
+
+	let header: SngHeader
+	sngStream.on('header', h => header = h)
+	const isFileTruncated = (fileName: string) => {
+		const MAX_FILE_MIB = 2048
+		const MAX_FILES_MIB = 5000
+		const sortedFiles = _.sortBy(header.fileMeta, f => f.contentsLen)
+		let usedSizeMib = 0
+		for (const sortedFile of sortedFiles) {
+			usedSizeMib += Number(sortedFile.contentsLen / BigInt(1024) / BigInt(1024))
+			if (sortedFile.filename === fileName) {
+				return usedSizeMib > MAX_FILES_MIB || sortedFile.contentsLen / BigInt(1024) / BigInt(1024) >= MAX_FILE_MIB
+			}
+		}
+	}
+
+
+	const files: { fileName: string; data: Uint8Array }[] = []
+
+	await new Promise<void>((resolve, reject) => {
+		sngStream.on('file', async (fileName, fileStream, nextFile) => {
+			const matchingFileMeta = header.fileMeta.find(f => f.filename === fileName)
+			if (hasVideoExtension(fileName) || isFileTruncated(fileName) || !matchingFileMeta) {
+				const reader = fileStream.getReader()
+				// eslint-disable-next-line no-constant-condition
+				while (true) {
+					const result = await reader.read()
+					if (result.done) {
+						break
+					}
+				}
+			} else {
+				const data = new Uint8Array(Number(matchingFileMeta.contentsLen))
+				let offset = 0
+				const reader = fileStream.getReader()
+				// eslint-disable-next-line no-constant-condition
+				while (true) {
+					const result = await reader.read()
+					if (result.done) {
+						break
+					}
+					data.set(result.value, offset)
+					offset += result.value.length
+				}
+
+				files.push({ fileName, data })
+			}
+
+			if (nextFile) {
+				nextFile()
+			} else {
+				resolve()
+			}
+		})
+
+		sngStream.on('error', error => reject(error))
+
+		sngStream.start()
+	})
+
+	return files
+}
+
+async function getFilesFromFolder(chartFolder: { path: string; files: string[] }): Promise<{ fileName: string; data: Uint8Array }[]> {
+	const files: { fileName: string; data: Uint8Array }[] = []
+
+	for (const fileName of chartFolder.files) {
+		if (hasChartExtension(fileName) || hasIniExtension(fileName) || hasAlbumName(fileName)) {
+			files.push({ fileName, data: await readFile(chartFolder.path + '/' + fileName) })
+		} else {
+			files.push({ fileName, data: new Uint8Array() })
+		}
+	}
+
+	return files
+}