diff --git a/package.json b/package.json index c65cc5e0..1ef303f8 100644 --- a/package.json +++ b/package.json @@ -50,7 +50,6 @@ "didyoumean2": "^7.0.4", "discord.js": "^14.25.1", "emoji-regex": "^10.6.0", - "everything-json": "^1.2.1", "fen-validator": "^2.0.1", "font-finder": "^1.1.0", "gm": "^1.25.1", @@ -78,6 +77,7 @@ "semver": "^7.7.3", "sherlockjs": "^1.4.2", "stackblur-canvas": "^2.7.0", + "stream-json": "^1.9.1", "text-diff": "^1.0.1", "tictactoe-minimax-ai": "github:marianoheller/tic-tac-toe-minimax", "twitter-openapi-typescript": "^0.0.55", diff --git a/structures/JeopardyScrape.js b/structures/JeopardyScrape.js index b6232ce4..6c9ecdfb 100644 --- a/structures/JeopardyScrape.js +++ b/structures/JeopardyScrape.js @@ -1,7 +1,9 @@ const request = require('node-superfetch'); const cheerio = require('cheerio'); const fs = require('fs'); -const { JSON: eJSON } = require('everything-json'); +const { parser } = require('stream-json'); +const { pick } = require('stream-json/filters/Pick'); +const { streamArray } = require('stream-json/streamers/StreamArray'); const path = require('path'); const { checkFileExists } = require('../util/Util'); const rounds = ['jeopardy_round', 'double_jeopardy_round', 'final_jeopardy_round']; @@ -11,8 +13,8 @@ module.exports = class JeopardyScrape { Object.defineProperty(this, 'client', { value: client }); this.clues = []; - this.gameIDs = []; - this.seasons = []; + this.gameIDs = null; + this.seasons = null; this.imported = false; } @@ -69,28 +71,25 @@ module.exports = class JeopardyScrape { async importData() { const read = await fs.promises.readFile(path.join(__dirname, '..', 'jeopardy.json'), { encoding: 'utf8' }); - const file = await eJSON.parseAsync(read); - if (typeof file !== 'object' || Array.isArray(file)) return null; - if (!file.clues || !file.gameIDs || !file.seasons) return null; + const { seasons, gameIDs } = JSON.parse(read); + this.gameIDs = gameIDs; + this.seasons = seasons; + this.clues = await this.importClues(); this.imported = true; - for (const season of file.seasons) { - if (typeof season !== 'string') continue; - if (this.seasons.includes(season)) continue; - this.seasons.push(season); - } - for (const gameID of file.gameIDs) { - if (typeof gameID !== 'string') continue; - if (this.gameIDs.includes(gameID)) continue; - this.gameIDs.push(gameID); - } - for (const clue of file.clues) { - if (typeof clue !== 'object' || Array.isArray(clue)) continue; - if (this.clues.some(c => c.question === clue.question && c.answer === clue.answer)) continue; - this.clues.push(clue); - } return file; } + importClues() { + const pipeline = fs.createReadStream(path.join(__dirname, '..', 'jeopardy.json'), { encoding: 'utf8'}) + .pipe(parser()) + .pipe(pick({ filter: 'clues' })) + .pipe(streamArray()); + pipeline.on('data', ({ key, value }) => this.clues.push(value)); + return new Promise(res => { + pipeline.on('end', () => res(this.clues)); + }); + } + exportData() { const buf = Buffer.from(JSON.stringify({ clues: this.clues,