From 114985bbe905d3501dbabfedd727a7996c08e4c8 Mon Sep 17 00:00:00 2001 From: Aaron Date: Thu, 30 Apr 2026 00:14:29 -0500 Subject: [PATCH 1/5] feat: Improved memory managment and cleanup --- scripts/generate-gql-types.ts | 12 ++- src/@types/gql-ctx.d.ts | 4 +- src/apis/extension-store-apis.ts | 9 +-- src/dependencies.ts | 18 ++++- src/graphql/index.ts | 6 +- src/graphql/resolvers.ts | 14 ++-- src/plugins/context-plugin.ts | 2 +- src/services/cache.ts | 4 + src/services/chrome-crawler.ts | 2 +- src/services/chrome-web-store.ts | 15 +++- src/services/edge-addon-store.ts | 21 ++++-- src/services/edge-api.ts | 6 +- src/services/extension-store.ts | 110 ++++++++++++++-------------- src/services/firefox-addon-store.ts | 21 ++++-- src/services/firefox-api.ts | 6 +- src/services/in-memory-cache.ts | 14 ++++ src/services/redis-cache.ts | 20 +++++ src/utils/cache.ts | 41 ----------- src/utils/errors.ts | 22 ++++++ 19 files changed, 204 insertions(+), 143 deletions(-) create mode 100644 src/services/cache.ts create mode 100644 src/services/in-memory-cache.ts create mode 100644 src/services/redis-cache.ts delete mode 100644 src/utils/cache.ts create mode 100644 src/utils/errors.ts diff --git a/scripts/generate-gql-types.ts b/scripts/generate-gql-types.ts index 606e767..42e4320 100644 --- a/scripts/generate-gql-types.ts +++ b/scripts/generate-gql-types.ts @@ -76,13 +76,16 @@ function capitalizeFirstLetter(str: string): string { return str[0]!.toUpperCase() + str.substring(1); } -function getTsTypeString(gqlType: any): string { +function getTsTypeString(gqlType: any, isReturn?: boolean): string { if (gqlType.kind === "NON_NULL") - return `NonNullable<${getTsTypeString(gqlType.ofType)}>`; + return getTsTypeString(gqlType.ofType, isReturn).replace( + " | undefined", + "", + ); if (gqlType.kind === "LIST") - return `Array<${getTsTypeString(gqlType.ofType)}> | undefined`; + return `Array<${getTsTypeString(gqlType.ofType, isReturn)}> | undefined`; if (gqlType.kind === "SCALAR" || gqlType.kind === "OBJECT") - return `${gqlType.name} | undefined`; + return `${gqlType.name}${isReturn ? " | Error" : ""} | undefined`; logger.warn("Unknown GQL -> TS type", { gqlType }); return "unknown"; @@ -114,6 +117,7 @@ function writeObjectType(code: CodeBlockWriter, argTypes: any[], type: any) { }; args = `(args: ${argsType.name}, ctx: WxtQueueCtx)`; argTypes.push(argsType); + returnTypeStr = getTsTypeString(field.type, true); returnTypeStr = `Promise<${returnTypeStr}> | ${returnTypeStr}`; } code.writeLine(`"${field.name}"${args}: ${returnTypeStr}`); diff --git a/src/@types/gql-ctx.d.ts b/src/@types/gql-ctx.d.ts index 34d0038..4775d85 100644 --- a/src/@types/gql-ctx.d.ts +++ b/src/@types/gql-ctx.d.ts @@ -1,3 +1,5 @@ declare namespace Gql { - export type WxtQueueCtx = import("../dependencies").Dependencies; + export type WxtQueueCtx = { + deps: import("../dependencies").Dependencies; + }; } diff --git a/src/apis/extension-store-apis.ts b/src/apis/extension-store-apis.ts index 3326c15..3b5424b 100644 --- a/src/apis/extension-store-apis.ts +++ b/src/apis/extension-store-apis.ts @@ -22,11 +22,10 @@ export const extensionStoreApis = createApp({ index: z.coerce.number().int().min(0), }), }, - async ({ params, stores, set }) => { - const screenshotUrl = await stores[params.storeName].getScreenshotUrl( - params.id, - params.index, - ); + async ({ params, deps, set }) => { + const screenshotUrl = await deps.stores[ + params.storeName + ].getScreenshotUrl(params.id, params.index); if (!screenshotUrl) throw new NotFoundHttpError("Extension or screenshot not found"); diff --git a/src/dependencies.ts b/src/dependencies.ts index 3d74ca5..376473c 100644 --- a/src/dependencies.ts +++ b/src/dependencies.ts @@ -1,14 +1,24 @@ -import { createIocContainer } from "@aklinker1/zero-ioc"; +import { createIocContainer, transient } from "@aklinker1/zero-ioc"; import { createChromeWebStore } from "./services/chrome-web-store"; import { createFirefoxAddonStore } from "./services/firefox-addon-store"; import { createEdgeAddonStore } from "./services/edge-addon-store"; import type { ExtensionStores } from "./services/extension-stores"; import { ExtensionStoreName } from "./enums"; +import { createRedisCache } from "./services/redis-cache"; +import { createInMemoryCache } from "./services/in-memory-cache"; +import { createEdgeApi } from "./services/edge-api"; +import { createFirefoxApi } from "./services/firefox-api"; export const container = createIocContainer() - .register("chromeWebStore", createChromeWebStore) - .register("firefoxAddonStore", createFirefoxAddonStore) - .register("edgeAddonStore", createEdgeAddonStore) + .register( + "cache", + Bun.redis.connected ? createRedisCache : createInMemoryCache, + ) + .register("edgeApi", createEdgeApi) + .register("firefoxApi", createFirefoxApi) + .register("chromeWebStore", transient(createChromeWebStore)) + .register("firefoxAddonStore", transient(createFirefoxAddonStore)) + .register("edgeAddonStore", transient(createEdgeAddonStore)) .register( "stores", (deps) => diff --git a/src/graphql/index.ts b/src/graphql/index.ts index bf3eeef..21ea129 100644 --- a/src/graphql/index.ts +++ b/src/graphql/index.ts @@ -19,10 +19,14 @@ export function createGraphql() { logger.debug("Running query", { id, method, operationName }); + const ctx: Gql.WxtQueueCtx = { + deps: container.registrations, + }; + const response = await graphql({ schema, source: query, - contextValue: container.registrations, + contextValue: ctx, variableValues: variables, rootValue: rootResolver, }); diff --git a/src/graphql/resolvers.ts b/src/graphql/resolvers.ts index 101f38c..0325d02 100644 --- a/src/graphql/resolvers.ts +++ b/src/graphql/resolvers.ts @@ -1,8 +1,10 @@ export const rootResolver: Gql.RootResolver = { - chromeExtension: ({ id }, ctx) => ctx.chromeWebStore.getExtension(id), - chromeExtensions: ({ ids }, ctx) => ctx.chromeWebStore.getExtensions(ids), - firefoxAddon: ({ id }, ctx) => ctx.firefoxAddonStore.getExtension(id), - firefoxAddons: ({ ids }, ctx) => ctx.firefoxAddonStore.getExtensions(ids), - edgeAddon: ({ id }, ctx) => ctx.edgeAddonStore.getExtension(id), - edgeAddons: ({ ids }, ctx) => ctx.edgeAddonStore.getExtensions(ids), + chromeExtension: ({ id }, ctx) => ctx.deps.chromeWebStore.getExtension(id), + chromeExtensions: ({ ids }, ctx) => + ctx.deps.chromeWebStore.getExtensions(ids), + firefoxAddon: ({ id }, ctx) => ctx.deps.firefoxAddonStore.getExtension(id), + firefoxAddons: ({ ids }, ctx) => + ctx.deps.firefoxAddonStore.getExtensions(ids), + edgeAddon: ({ id }, ctx) => ctx.deps.edgeAddonStore.getExtension(id), + edgeAddons: ({ ids }, ctx) => ctx.deps.edgeAddonStore.getExtensions(ids), }; diff --git a/src/plugins/context-plugin.ts b/src/plugins/context-plugin.ts index 7b82881..e6f5336 100644 --- a/src/plugins/context-plugin.ts +++ b/src/plugins/context-plugin.ts @@ -2,5 +2,5 @@ import { createApp } from "@aklinker1/zeta"; import { container } from "../dependencies"; export const contextPlugin = createApp() - .decorate(container.resolveAll()) + .decorate({ deps: container.registrations }) .export(); diff --git a/src/services/cache.ts b/src/services/cache.ts new file mode 100644 index 0000000..1ead5aa --- /dev/null +++ b/src/services/cache.ts @@ -0,0 +1,4 @@ +export interface Cache { + get(key: string): Promise; + set(key: string, value: T): Promise; +} diff --git a/src/services/chrome-crawler.ts b/src/services/chrome-crawler.ts index 7d2bd66..d2aaf25 100644 --- a/src/services/chrome-crawler.ts +++ b/src/services/chrome-crawler.ts @@ -204,7 +204,7 @@ function tryExtract( errors.push(error as Error); } } - errors.forEach((err) => console.error(err)); + if (errors.length > 0) logger.error("Crawl errors", { errors }); throw new Error(`Could not extract "${field}" from HTML`, { cause: errors }); } diff --git a/src/services/chrome-web-store.ts b/src/services/chrome-web-store.ts index 2a2878d..415e0c5 100644 --- a/src/services/chrome-web-store.ts +++ b/src/services/chrome-web-store.ts @@ -1,8 +1,17 @@ +import type { Cache } from "./cache"; import { crawlExtension } from "./chrome-crawler"; -import { defineExtensionStore, type ExtensionStore } from "./extension-store"; +import { ExtensionStore } from "./extension-store"; export type ChromeWebStore = ExtensionStore; -export function createChromeWebStore() { - return defineExtensionStore((id) => crawlExtension(String(id), "en")); +export function createChromeWebStore({ + cache, +}: { + cache: Cache; +}): ChromeWebStore { + return new ExtensionStore({ + fetch: (id) => crawlExtension(String(id), "en"), + cacheKeyPrefix: "chrome-extension-", + cache, + }); } diff --git a/src/services/edge-addon-store.ts b/src/services/edge-addon-store.ts index b47a72f..4cf985e 100644 --- a/src/services/edge-addon-store.ts +++ b/src/services/edge-addon-store.ts @@ -1,10 +1,19 @@ -import { createEdgeApi } from "./edge-api"; -import { defineExtensionStore, type ExtensionStore } from "./extension-store"; +import type { Cache } from "./cache"; +import type { EdgeApi } from "./edge-api"; +import { ExtensionStore } from "./extension-store"; export type EdgeAddonStore = ExtensionStore; -export function createEdgeAddonStore() { - const api = createEdgeApi(); - - return defineExtensionStore((id) => api.getAddon(String(id))); +export function createEdgeAddonStore({ + cache, + edgeApi, +}: { + cache: Cache; + edgeApi: EdgeApi; +}): EdgeAddonStore { + return new ExtensionStore({ + fetch: (id) => edgeApi.getAddon(String(id)), + cacheKeyPrefix: "edge-addon-", + cache, + }); } diff --git a/src/services/edge-api.ts b/src/services/edge-api.ts index 807f6f0..30cb982 100644 --- a/src/services/edge-api.ts +++ b/src/services/edge-api.ts @@ -1,6 +1,7 @@ import { createLogger } from "@aklinker1/logger"; import { ExtensionStoreName } from "../enums"; import { buildScreenshotUrl } from "../utils/urls"; +import { FetchError } from "../utils/errors"; const logger = createLogger("edge-api"); @@ -41,13 +42,10 @@ export function createEdgeApi(): EdgeApi { const res = await fetch( `https://microsoftedge.microsoft.com/addons/getproductdetailsbycrxid/${crxid}`, ); - if (res.status !== 200) { - throw Error("Edge API request failed", { cause: res }); - } + if (res.status !== 200) throw new FetchError(res, await res.text()); const json = (await res.json()) as GetProductDetailsByCrxId200Response; logger.debug("Addon result", { crxid, json }); - return toGqlEdgeAddon(json); }; diff --git a/src/services/extension-store.ts b/src/services/extension-store.ts index c126e6a..d7ddcb7 100644 --- a/src/services/extension-store.ts +++ b/src/services/extension-store.ts @@ -1,74 +1,72 @@ -import { createCachedDataLoader } from "../utils/cache"; -import { HOUR_MS } from "../utils/time"; +import DataLoader from "dataloader"; +import type { Cache } from "./cache"; +import { createLogger } from "@aklinker1/logger"; + +const logger = createLogger("extension-store"); export type ExtensionId = string | number; -export interface ExtensionStore { +export class ExtensionStore { + private dataloader: DataLoader; + + constructor( + readonly options: { + cacheKeyPrefix: string; + fetch: (id: ExtensionId) => Promise; + cache: Cache; + }, + ) { + logger.info("CREATED EXTENSION STORE", { + prefix: options.cacheKeyPrefix, + }); + this.dataloader = new DataLoader( + async (ids): Promise> => { + const results = await Promise.allSettled( + ids.map(async (id) => { + const cacheKey = options.cacheKeyPrefix + id; + const cached = await options.cache.get(cacheKey); + if (cached) return cached; + + const result = await options.fetch(id); + if (result) await options.cache.set(cacheKey, result); + + return result; + }), + ); + return results.map((res) => + res.status === "fulfilled" ? res.value : res.reason, + ); + }, + ); + } + /** * Get an extension by it's ID. */ - getExtension: ( - extensionId: ExtensionId, - ) => Promise; + getExtension(extensionId: ExtensionId): Promise { + return this.dataloader.load(extensionId); + } /** * Get multiple extensions by their IDs. */ - getExtensions: ( + async getExtensions( extensionIds: ExtensionId[], - ) => Promise<(TGqlExtension | undefined)[]>; + ): Promise<(TGqlExtension | Error)[]> { + return this.dataloader.loadMany(extensionIds); + } /** * Get a screenshot given an index. */ - getScreenshotUrl( + async getScreenshotUrl( extensionId: ExtensionId, screenshotIndex: number, - ): Promise; -} - -export function defineExtensionStore( - fetch: (id: ExtensionId) => Promise, -): ExtensionStore { - const loader = createCachedDataLoader( - HOUR_MS, - async (ids) => { - const results = await Promise.allSettled(ids.map((id) => fetch(id))); - return results.map((res) => - res.status === "fulfilled" ? res.value : undefined, - ); - }, - ); - - const getExtension: ExtensionStore["getExtension"] = ( - extensionId, - ) => loader.load(extensionId); - - const getExtensions: ExtensionStore["getExtensions"] = async ( - extensionIds, - ) => { - const result = await loader.loadMany(extensionIds); - return result.map((item, index) => { - if (item instanceof Error) { - console.warn("Error loading extension:", extensionIds[index], item); - return undefined; - } - return item; - }); - }; - - const getScreenshotUrl: ExtensionStore["getScreenshotUrl"] = - async (extensionId, screenshotIndex) => { - const extension = await getExtension(extensionId); - const screenshot = extension?.screenshots.find( - (screenshot) => screenshot.index == screenshotIndex, - ); - return screenshot?.rawUrl; - }; - - return { - getExtension, - getExtensions, - getScreenshotUrl, - }; + ): Promise { + const extension = await this.getExtension(extensionId); + const screenshot = extension.screenshots.find( + (screenshot) => screenshot.index == screenshotIndex, + ); + return screenshot?.rawUrl; + } } diff --git a/src/services/firefox-addon-store.ts b/src/services/firefox-addon-store.ts index 3faa966..4046c95 100644 --- a/src/services/firefox-addon-store.ts +++ b/src/services/firefox-addon-store.ts @@ -1,10 +1,19 @@ -import { createFirefoxApi } from "./firefox-api"; -import { defineExtensionStore, type ExtensionStore } from "./extension-store"; +import type { Cache } from "./cache"; +import { ExtensionStore } from "./extension-store"; +import type { FirefoxApi } from "./firefox-api"; export type FirefoxAddonStore = ExtensionStore; -export function createFirefoxAddonStore() { - const api = createFirefoxApi(); - - return defineExtensionStore((id) => api.getAddon(id)); +export function createFirefoxAddonStore({ + cache, + firefoxApi, +}: { + cache: Cache; + firefoxApi: FirefoxApi; +}): FirefoxAddonStore { + return new ExtensionStore({ + fetch: (id) => firefoxApi.getAddon(String(id)), + cacheKeyPrefix: "firefox-addon-", + cache, + }); } diff --git a/src/services/firefox-api.ts b/src/services/firefox-api.ts index 311a4a1..8eeea2a 100644 --- a/src/services/firefox-api.ts +++ b/src/services/firefox-api.ts @@ -1,6 +1,7 @@ import { buildScreenshotUrl } from "../utils/urls"; import { ExtensionStoreName } from "../enums"; import { createLogger } from "@aklinker1/logger"; +import { FetchError } from "../utils/errors"; const logger = createLogger("firefox-api"); @@ -43,10 +44,7 @@ export function createFirefoxApi(): FirefoxApi { `https://addons.mozilla.org/api/v5/addons/addon/${idOrSlugOrGuid}`, ); const res = await fetch(url); - if (res.status !== 200) - throw Error( - `${url.href} failed with status: ${res.status} ${res.statusText}`, - ); + if (res.status !== 200) throw new FetchError(res, await res.text()); const json = (await res.json()) as GetAddon200Response; logger.debug("Addon result", { idOrSlugOrGuid, json }); diff --git a/src/services/in-memory-cache.ts b/src/services/in-memory-cache.ts new file mode 100644 index 0000000..aa057db --- /dev/null +++ b/src/services/in-memory-cache.ts @@ -0,0 +1,14 @@ +import type { Cache } from "./cache"; + +export function createInMemoryCache(): Cache { + let cache: Record = Object.create(null); + + return { + get: async (key: string) => { + return cache[key]; + }, + set: async (key: string, value: any) => { + cache[key] = value; + }, + }; +} diff --git a/src/services/redis-cache.ts b/src/services/redis-cache.ts new file mode 100644 index 0000000..9d2ca65 --- /dev/null +++ b/src/services/redis-cache.ts @@ -0,0 +1,20 @@ +import { DAY_MS } from "../utils/time"; +import type { Cache } from "./cache"; + +const EXPIRATION = DAY_MS / 1000; // 24 hours in seconds + +export function createRedisCache(): Cache { + return { + async get(key: string): Promise { + const value = await Bun.redis.get(key); + if (value === null) { + return undefined; + } + return JSON.parse(value) as T; + }, + async set(key: string, value: T): Promise { + await Bun.redis.set(key, JSON.stringify(value)); + await Bun.redis.expire(key, EXPIRATION); + }, + }; +} diff --git a/src/utils/cache.ts b/src/utils/cache.ts deleted file mode 100644 index c1a3827..0000000 --- a/src/utils/cache.ts +++ /dev/null @@ -1,41 +0,0 @@ -import DataLoader, { type CacheMap } from "dataloader"; - -export function createInMemoryCache(config: { - ttl: number; -}): CacheMap { - const cache = new Map>(); - return { - set(key, value) { - cache.set(key, { - setAt: Date.now(), - value, - }); - }, - get(key) { - const entry = cache.get(key); - if (entry === undefined) return undefined; - if (entry.setAt + config.ttl < Date.now()) return undefined; - return entry.value ?? undefined; - }, - clear() { - cache.clear(); - }, - delete(key) { - cache.delete(key); - }, - }; -} - -interface CacheEntry { - setAt: number; - value: T | null; -} - -export function createCachedDataLoader( - ttl: number, - batchLoadFn: DataLoader.BatchLoadFn, -) { - return new DataLoader(batchLoadFn, { - cacheMap: createInMemoryCache({ ttl }), - }); -} diff --git a/src/utils/errors.ts b/src/utils/errors.ts new file mode 100644 index 0000000..b44f9a3 --- /dev/null +++ b/src/utils/errors.ts @@ -0,0 +1,22 @@ +export class FetchError extends Error { + status: number; + body: any; + url: string; + + constructor(response: Response, body: string, options?: ErrorOptions) { + super( + `Fetch request failed with "${response.status} ${response.statusText || "undefined"}"`, + options, + ); + this.name = "FetchError"; + this.url = response.url; + this.status = response.status; + this.body = response.headers + .get("content-type") + ?.includes("application/json") + ? JSON.parse(body) + : body.length > 100 + ? body.slice(0, 100) + "..." + : body; + } +} From f81b5377ecf38083d125116d894aa6fd45930fb0 Mon Sep 17 00:00:00 2001 From: Aaron Date: Thu, 30 Apr 2026 00:26:08 -0500 Subject: [PATCH 2/5] Remove log --- src/services/extension-store.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/services/extension-store.ts b/src/services/extension-store.ts index d7ddcb7..4895f2f 100644 --- a/src/services/extension-store.ts +++ b/src/services/extension-store.ts @@ -16,9 +16,6 @@ export class ExtensionStore { cache: Cache; }, ) { - logger.info("CREATED EXTENSION STORE", { - prefix: options.cacheKeyPrefix, - }); this.dataloader = new DataLoader( async (ids): Promise> => { const results = await Promise.allSettled( From b1af8408ddc687ca273e1d0ecef3e2a6e3844e15 Mon Sep 17 00:00:00 2001 From: Aaron Date: Thu, 30 Apr 2026 00:30:20 -0500 Subject: [PATCH 3/5] Cleanup TTL --- src/services/in-memory-cache.ts | 9 +++++++++ src/services/redis-cache.ts | 5 +++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/services/in-memory-cache.ts b/src/services/in-memory-cache.ts index aa057db..53864ff 100644 --- a/src/services/in-memory-cache.ts +++ b/src/services/in-memory-cache.ts @@ -1,14 +1,23 @@ +import { HOUR_MS } from "../utils/time"; import type { Cache } from "./cache"; +const TTL = HOUR_MS; + export function createInMemoryCache(): Cache { let cache: Record = Object.create(null); + let ttl: Record = Object.create(null); return { get: async (key: string) => { + if (ttl[key] && Date.now() > ttl[key]) { + delete cache[key]; + delete ttl[key]; + } return cache[key]; }, set: async (key: string, value: any) => { cache[key] = value; + ttl[key] = Date.now() + TTL; }, }; } diff --git a/src/services/redis-cache.ts b/src/services/redis-cache.ts index 9d2ca65..adbb4ea 100644 --- a/src/services/redis-cache.ts +++ b/src/services/redis-cache.ts @@ -1,7 +1,8 @@ import { DAY_MS } from "../utils/time"; import type { Cache } from "./cache"; -const EXPIRATION = DAY_MS / 1000; // 24 hours in seconds +const TTL = DAY_MS; +const TTL_S = TTL / 1000; export function createRedisCache(): Cache { return { @@ -14,7 +15,7 @@ export function createRedisCache(): Cache { }, async set(key: string, value: T): Promise { await Bun.redis.set(key, JSON.stringify(value)); - await Bun.redis.expire(key, EXPIRATION); + await Bun.redis.expire(key, TTL_S); }, }; } From 6d19ecca9969c0b21587a091c123950975347b3a Mon Sep 17 00:00:00 2001 From: Aaron Date: Thu, 30 Apr 2026 00:30:56 -0500 Subject: [PATCH 4/5] Cleanup --- src/services/redis-cache.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/services/redis-cache.ts b/src/services/redis-cache.ts index adbb4ea..13caba2 100644 --- a/src/services/redis-cache.ts +++ b/src/services/redis-cache.ts @@ -8,9 +8,8 @@ export function createRedisCache(): Cache { return { async get(key: string): Promise { const value = await Bun.redis.get(key); - if (value === null) { - return undefined; - } + if (value == null) return undefined; + return JSON.parse(value) as T; }, async set(key: string, value: T): Promise { From 804276169cc4d97680ad643e5f0ac0452309f927 Mon Sep 17 00:00:00 2001 From: Aaron Date: Thu, 30 Apr 2026 00:34:40 -0500 Subject: [PATCH 5/5] Fix checks --- src/services/extension-store.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/services/extension-store.ts b/src/services/extension-store.ts index 4895f2f..123f14c 100644 --- a/src/services/extension-store.ts +++ b/src/services/extension-store.ts @@ -1,8 +1,5 @@ import DataLoader from "dataloader"; import type { Cache } from "./cache"; -import { createLogger } from "@aklinker1/logger"; - -const logger = createLogger("extension-store"); export type ExtensionId = string | number;