From 38dc7f2a38ea8e6814ac87863c0954b5e442de08 Mon Sep 17 00:00:00 2001 From: Yashaswini K P Date: Tue, 14 Jul 2026 16:36:54 +0530 Subject: [PATCH 1/3] fix: resolve timeframe data leak and unawaited writes --- scripts/sync-leaderboard.js | 236 +++++++++++++++++++++--------------- 1 file changed, 138 insertions(+), 98 deletions(-) diff --git a/scripts/sync-leaderboard.js b/scripts/sync-leaderboard.js index 03328f63..751bc0e5 100644 --- a/scripts/sync-leaderboard.js +++ b/scripts/sync-leaderboard.js @@ -2,12 +2,13 @@ const axios = require("axios"); const fs = require("fs"); +const fsPromises = require("fs").promises; const path = require("path"); const { setupRetryInterceptor } = require("./lib/retry-interceptor"); setupRetryInterceptor(); -function atomicWrite(filePath, data) { +async function atomicWrite(filePath, data) { const dir = path.dirname(filePath); const ext = path.extname(filePath); const base = path.basename(filePath, ext); @@ -16,8 +17,8 @@ function atomicWrite(filePath, data) { `${base}.tmp.${process.pid}.${Date.now()}${ext}`, ); - fs.writeFileSync(tmpPath, JSON.stringify(data, null, 2), "utf8"); - fs.renameSync(tmpPath, filePath); + await fsPromises.writeFile(tmpPath, JSON.stringify(data, null, 2), "utf8"); + await fsPromises.rename(tmpPath, filePath); } async function fetchData(url) { @@ -47,27 +48,35 @@ function getFileName(daysAgo) { return `${year}-${month}-${date}-${day}.json`; } -function updateUserData(user, DATA_DIR, badgesMap = null, ranksObj = null) { +async function updateUserDataAsync( + user, + DATA_DIR, + badgesMap = null, + ranksObj = null, +) { const userDataDir = path.join(DATA_DIR, "user-data"); - if (!fs.existsSync(userDataDir)) { - fs.mkdirSync(userDataDir, { recursive: true }); + try { + await fsPromises.mkdir(userDataDir, { recursive: true }); + } catch (err) { + if (err.code !== "EEXIST") throw err; } const userDataPath = path.join(userDataDir, `${user.id}.json`); let userData = { leaderboardRanks: {}, history: [], badges: [] }; let history = []; - if (fs.existsSync(userDataPath)) { - try { - const rawData = JSON.parse(fs.readFileSync(userDataPath, "utf8")); - if (Array.isArray(rawData)) { - history = rawData; - } else { - userData = rawData; - history = userData.history || []; - if (!userData.leaderboardRanks) userData.leaderboardRanks = {}; - } - } catch (err) { + try { + const fileHandle = await fsPromises.readFile(userDataPath, "utf8"); + const rawData = JSON.parse(fileHandle); + if (Array.isArray(rawData)) { + history = rawData; + } else { + userData = rawData; + history = userData.history || []; + if (!userData.leaderboardRanks) userData.leaderboardRanks = {}; + } + } catch (err) { + if (err.code !== "ENOENT") { console.error( `Failed to parse data for ${user.id}, resetting:`, err.message, @@ -100,17 +109,18 @@ function updateUserData(user, DATA_DIR, badgesMap = null, ranksObj = null) { if (ranksObj) { userData.leaderboardRanks = ranksObj; } - atomicWrite(userDataPath, userData); + await atomicWrite(userDataPath, userData); } -function checkHotStreak(userId, DATA_DIR, badgesMap) { +async function checkHotStreakAsync(userId, DATA_DIR, badgesMap) { const userDataDir = path.join(DATA_DIR, "user-data"); const userDataPath = path.join(userDataDir, `${userId}.json`); if (!fs.existsSync(userDataPath)) return; try { - const rawData = JSON.parse(fs.readFileSync(userDataPath, "utf8")); + const fileHandle = await fsPromises.readFile(userDataPath, "utf8"); + const rawData = JSON.parse(fileHandle); const history = Array.isArray(rawData) ? rawData : rawData.history || []; if (history.length >= 8) { @@ -266,7 +276,7 @@ async function processTimeframe( const previousFilepath = path.join(DATA_DIR, "daily", getFileName(daysAgo)); let previousData = []; try { - const rawData = fs.readFileSync(previousFilepath, "utf8"); + const rawData = await fsPromises.readFile(previousFilepath, "utf8"); previousData = JSON.parse(rawData); console.log(`Previous ${periodName}'s data loaded successfully`); } catch (err) { @@ -279,52 +289,64 @@ async function processTimeframe( console.log(" "); console.log(`Calculating ${periodName} progress...`); - for (let i = 0; i < data.length; i++) { - const previousIndex = previousData.findIndex( - (obj) => obj.id === data[i].id, - ); - if (previousIndex == -1) { - data.splice(i--, 1); - continue; - } - data[i].data.easySolved = Math.max( - 0, - data[i].data.easySolved - previousData[previousIndex].data.easySolved, - ); - data[i].data.mediumSolved = Math.max( - 0, - data[i].data.mediumSolved - previousData[previousIndex].data.mediumSolved, - ); - data[i].data.hardSolved = Math.max( - 0, - data[i].data.hardSolved - previousData[previousIndex].data.hardSolved, + + const PROFILE_CONCURRENCY = 20; + for (let i = 0; i < data.length; i += PROFILE_CONCURRENCY) { + const chunk = data.slice(i, i + PROFILE_CONCURRENCY); + await Promise.all( + chunk.map(async (item) => { + const previousIndex = previousData.findIndex( + (obj) => obj.id === item.id, + ); + if (previousIndex === -1) { + item._remove = true; + return; + } + item.data.easySolved = Math.max( + 0, + item.data.easySolved - previousData[previousIndex].data.easySolved, + ); + item.data.mediumSolved = Math.max( + 0, + item.data.mediumSolved - + previousData[previousIndex].data.mediumSolved, + ); + item.data.hardSolved = Math.max( + 0, + item.data.hardSolved - previousData[previousIndex].data.hardSolved, + ); + + item.score = + item.data.easySolved + + item.data.mediumSolved * 3 + + item.data.hardSolved * 5; + item.data.totalSolved = + item.data.easySolved + item.data.mediumSolved + item.data.hardSolved; + + if ( + periodName === "weekly" && + item.data.totalSolved >= 7 && + badgesMap + ) { + await checkHotStreakAsync(item.id, DATA_DIR, badgesMap); + } + }), ); - data[i].score = - data[i].data.easySolved + - data[i].data.mediumSolved * 3 + - data[i].data.hardSolved * 5; - data[i].data.totalSolved = - data[i].data.easySolved + - data[i].data.mediumSolved + - data[i].data.hardSolved; - - if (periodName === "weekly" && data[i].data.totalSolved >= 7 && badgesMap) { - checkHotStreak(data[i].id, DATA_DIR, badgesMap); - } } + const filteredData = data.filter((item) => !item._remove); console.log("Calculation done"); console.log(""); console.log("Sorting calculated data..."); - stableSortByScore(data); - assignCompetitionRanks(data); + stableSortByScore(filteredData); + assignCompetitionRanks(filteredData); console.log(`Writing sorted ${periodName} data to ${periodName}.json...`); const filepath = path.join(DATA_DIR, `${periodName}.json`); - await computeRankChanges(data, `${periodName}.json`, badgesMap); + await computeRankChanges(filteredData, `${periodName}.json`, badgesMap); try { - atomicWrite(filepath, data); + await atomicWrite(filepath, filteredData); console.log(`${periodName} data saved successfully`); - return data; + return filteredData; } catch (err) { console.error(`Failed to write ${periodName}.json: `, err.message); throw err; @@ -462,7 +484,7 @@ async function processTimeframe( console.log("Writing daily data to file..."); const filepath = path.join(DATA_DIR, "daily", getFileName(0)); try { - atomicWrite(filepath, overallData); + await atomicWrite(filepath, overallData); console.log("Daily data saved successfully"); } catch (err) { console.error(`Failed to write json file: `, err.message); @@ -480,7 +502,7 @@ async function processTimeframe( await computeRankChanges(overallData, "overall.json", badgesMap); try { - atomicWrite(overallFilepath, overallData); + await atomicWrite(overallFilepath, overallData); console.log("Daily data saved successfully"); } catch (err) { console.error(`Failed to write json file: `, err.message); @@ -592,7 +614,7 @@ async function processTimeframe( const noChanges = rankChanges.length === 0 && newUsers.length === 0 && totalNewSolves === 0; - atomicWrite(changesFilepath, { + await atomicWrite(changesFilepath, { sync_time: new Date().toISOString(), rank_changes: rankChanges, new_users: newUsers, @@ -617,45 +639,63 @@ async function processTimeframe( "Updating user data files with history, badges, and pre-calculated ranks...", ); let userDataFailures = 0; - overallData.forEach((user) => { - try { - const overallInfo = overallMap.get(user.id) || { - rank: "--", - change: "=", - }; - const dailyInfo = dailyMap.get(user.id) || { rank: "--", change: "=" }; - const weeklyInfo = weeklyMap.get(user.id) || { rank: "--", change: "=" }; - const monthlyInfo = monthlyMap.get(user.id) || { - rank: "--", - change: "=", - }; - const calculatedRanks = { - overall: { - rank: overallInfo.rank, - change: formatChange(overallInfo.change), - }, - daily: { rank: dailyInfo.rank, change: formatChange(dailyInfo.change) }, - weekly: { - rank: weeklyInfo.rank, - change: formatChange(weeklyInfo.change), - }, - monthly: { - rank: monthlyInfo.rank, - change: formatChange(monthlyInfo.change), - }, - }; + const DISK_CONCURRENCY_LIMIT = 20; - // Single write pass maps everything cleanly - updateUserData(user, DATA_DIR, badgesMap, calculatedRanks); - } catch (err) { - userDataFailures++; - console.error( - `Failed to completely map metadata for ${user.id}:`, - err.message, - ); - } - }); + for (let i = 0; i < overallData.length; i += DISK_CONCURRENCY_LIMIT) { + const batch = overallData.slice(i, i + DISK_CONCURRENCY_LIMIT); + + await Promise.all( + batch.map(async (user) => { + try { + const overallInfo = overallMap.get(user.id) || { + rank: "--", + change: "=", + }; + const dailyInfo = dailyMap.get(user.id) || { + rank: "--", + change: "=", + }; + const weeklyInfo = weeklyMap.get(user.id) || { + rank: "--", + change: "=", + }; + const monthlyInfo = monthlyMap.get(user.id) || { + rank: "--", + change: "=", + }; + + const calculatedRanks = { + overall: { + rank: overallInfo.rank, + change: formatChange(overallInfo.change), + }, + daily: { + rank: dailyInfo.rank, + change: formatChange(dailyInfo.change), + }, + weekly: { + rank: weeklyInfo.rank, + change: formatChange(weeklyInfo.change), + }, + monthly: { + rank: monthlyInfo.rank, + change: formatChange(monthlyInfo.change), + }, + }; + + // Executed asynchronously using fs.promises inside controlled chunk bounds + await updateUserDataAsync(user, DATA_DIR, badgesMap, calculatedRanks); + } catch (err) { + userDataFailures++; + console.error( + `Failed to completely map metadata for ${user.id}:`, + err.message, + ); + } + }), + ); + } if (userDataFailures > 0) { console.warn(`${userDataFailures} user data map update(s) failed.`); @@ -668,7 +708,7 @@ async function processTimeframe( try { const now = new Date(); const nextSync = new Date(now.getTime() + 15 * 60 * 1000); - atomicWrite(syncFilepath, { + await atomicWrite(syncFilepath, { lastSync: now.toISOString(), nextSync: nextSync.toISOString(), }); From bb636e35ea9b3be918cb693f84b4a0f0cf44ff79 Mon Sep 17 00:00:00 2001 From: Jagdish Prajapati Date: Thu, 16 Jul 2026 01:01:16 +0530 Subject: [PATCH 2/3] add missing tmp directory cleanup and lift redundant mkdir loop --- scripts/sync-leaderboard.js | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/scripts/sync-leaderboard.js b/scripts/sync-leaderboard.js index 751bc0e5..c608d6eb 100644 --- a/scripts/sync-leaderboard.js +++ b/scripts/sync-leaderboard.js @@ -55,12 +55,6 @@ async function updateUserDataAsync( ranksObj = null, ) { const userDataDir = path.join(DATA_DIR, "user-data"); - try { - await fsPromises.mkdir(userDataDir, { recursive: true }); - } catch (err) { - if (err.code !== "EEXIST") throw err; - } - const userDataPath = path.join(userDataDir, `${user.id}.json`); let userData = { leaderboardRanks: {}, history: [], badges: [] }; let history = []; @@ -116,8 +110,6 @@ async function checkHotStreakAsync(userId, DATA_DIR, badgesMap) { const userDataDir = path.join(DATA_DIR, "user-data"); const userDataPath = path.join(userDataDir, `${userId}.json`); - if (!fs.existsSync(userDataPath)) return; - try { const fileHandle = await fsPromises.readFile(userDataPath, "utf8"); const rawData = JSON.parse(fileHandle); @@ -149,10 +141,12 @@ async function checkHotStreakAsync(userId, DATA_DIR, badgesMap) { } } } catch (err) { - console.error( - `Failed parsing user data for badge verification on ${userId}:`, - err.message, - ); + if (err.code !== "ENOENT") { + console.error( + `Failed parsing user data for badge verification on ${userId}:`, + err.message, + ); + } } } @@ -358,7 +352,11 @@ async function processTimeframe( console.log(`Using data directory: ${DATA_DIR}`); // Clean up leftover tmp files from previous crashes - const tmpCleanupDirs = [DATA_DIR, path.join(DATA_DIR, "daily")]; + const tmpCleanupDirs = [ + DATA_DIR, + path.join(DATA_DIR, "daily"), + path.join(DATA_DIR, "user-data"), + ]; tmpCleanupDirs.forEach((dirPath) => { try { if (fs.existsSync(dirPath)) { @@ -638,6 +636,11 @@ async function processTimeframe( console.log( "Updating user data files with history, badges, and pre-calculated ranks...", ); + try { + await fsPromises.mkdir(path.join(DATA_DIR, "user-data"), { recursive: true }); + } catch (err) { + if (err.code !== "EEXIST") throw err; + } let userDataFailures = 0; const DISK_CONCURRENCY_LIMIT = 20; @@ -717,3 +720,4 @@ async function processTimeframe( console.error(`Failed to write sync file: `, err.message); } })(); + From 4acf4d97a9b0f271a0e2c9b41207293d2d7cf75d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 15 Jul 2026 19:32:07 +0000 Subject: [PATCH 3/3] style: auto-format code with Prettier (/format) --- scripts/sync-leaderboard.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/sync-leaderboard.js b/scripts/sync-leaderboard.js index c608d6eb..6f043b14 100644 --- a/scripts/sync-leaderboard.js +++ b/scripts/sync-leaderboard.js @@ -637,7 +637,9 @@ async function processTimeframe( "Updating user data files with history, badges, and pre-calculated ranks...", ); try { - await fsPromises.mkdir(path.join(DATA_DIR, "user-data"), { recursive: true }); + await fsPromises.mkdir(path.join(DATA_DIR, "user-data"), { + recursive: true, + }); } catch (err) { if (err.code !== "EEXIST") throw err; } @@ -720,4 +722,3 @@ async function processTimeframe( console.error(`Failed to write sync file: `, err.message); } })(); -