basecode · habansal · Aug 19, 2026 · Aug 19, 2026 · Aug 19, 2026
diff --git a/README.md b/README.md
@@ -4,10 +4,13 @@ Misc SEO research
 ## Setup
 Create .env file with the following content:
 ```
-SPACECAT_API_KEY=
-AHREFS_API_KEY=
+SPACECAT_SESSION_TOKEN=
+SEO_API_KEY=
+SEO_API_BASE_URL=https://api.semrush.com
 ```
-Retrieve the values from [Vault](https://vault-amer.adobe.net/ui/vault/secrets/aem_exp_success_eng/show/spacecat/seo).
+- `SPACECAT_SESSION_TOKEN`: IMS JWT session token. Sign in at [SpaceCat](https://spacecat.experiencecloud.live), then copy the token from your browser's `Authorization` request header (DevTools → Network tab).
+- `SEO_API_KEY`: Semrush API key. Retrieve from [Vault](https://vault-amer.adobe.net/ui/vault/secrets/aem_exp_success_eng/show/spacecat/seo).
+- `SEO_API_BASE_URL`: Semrush API base URL (default shown above, no trailing slash).
 
 Run `npm install` to install dependencies.
 
@@ -43,10 +46,7 @@ The following options are available for the assessments.
 They can be used either for running a single assessment or to run all assessments.
 If they are used to run all assessments, some options will only be used by their relevant assessments.
 
-- `topBacklinks=<number>` - Number of top backlinks (default 200), to run the audit for
-- `topPages=<number>` - Number of top pages (default 200), based on estimated organic traffic, to filter the backlinks
-- `onlyBacklinksInTopPages=<boolean>` - Only check backlinks that are in the top pages (default true)
-- `devBaseURL=<devBaseURL>` - Base URL of the development environment on which the backlinks should be checked. 
-- `sitemap=<sitemapUrl>` - Not used for now. Specify a specific sitemap location (default fetched from robots.txt or 
-  /sitemap.xml), especially useful for page in development as they are not listed yet in the robots.txt or sitemap_index.xml,
- to use instead of the top pages from Ahrefs
+- `topBacklinks=<number>` - Max number of broken backlinks to return from Semrush (default 200, max 100 per Semrush API)
+- `devBaseURL=<devBaseURL>` - Base URL of the development environment on which the backlinks should be checked.
+- `sitemap=<sitemapUrl>` - Not used for now. Specify a specific sitemap location (default fetched from robots.txt or
+  /sitemap.xml), especially useful for pages in development not yet listed in the robots.txt or sitemap_index.xml.
diff --git a/all-assessments.js b/all-assessments.js
@@ -10,7 +10,8 @@
  * governing permissions and limitations under the License.
  */
 import fs from 'fs';
-import SpaceCatSdk from 'spacecat-sdk/src/sdk.js';
+import { Buffer } from 'buffer';
+import fetch from 'node-fetch';
 import { json2csv } from 'json-2-csv';
 import path from 'path';
 import { composeAuditURL } from '@adobe/spacecat-shared-utils';
@@ -97,16 +98,41 @@ const setup = async () => {
     fs.mkdirSync(OUTPUT_DIR);
   }
 
-  console.log('Check if URL is qualified to be assessed. Needs to be part of spacecat catalogue');
-  const spaceCatSdk = new SpaceCatSdk(
-    { apiBaseUrl: SPACECAT_API_BASE_URL, apiKey: process.env.SPACECAT_API_KEY },
-  );
-  options.site = await spaceCatSdk.getSite(options.baseURL);
-  if (options.site.gitHubURL) {
-    options.hlxSiteURL = await gitHubURLToHlxSite(options.site.gitHubURL);
+  if (process.env.SPACECAT_SESSION_TOKEN) {
+    const lookupURLs = [options.baseURL];
+    const wwwStripped = options.baseURL.replace(/^(https?:\/\/)www\./, '$1');
+    if (wwwStripped !== options.baseURL) lookupURLs.push(wwwStripped);
+
+    let found = false;
+    for (const lookupURL of lookupURLs) {
+      const encodedURL = Buffer.from(lookupURL).toString('base64');
+      // eslint-disable-next-line no-await-in-loop
+      const siteResponse = await fetch(`${SPACECAT_API_BASE_URL}/sites/by-base-url/${encodedURL}`, {
+        headers: {
+          Authorization: `Bearer ${process.env.SPACECAT_SESSION_TOKEN}`,
+          'Content-Type': 'application/json',
+        },
+      });
+      if (siteResponse.ok) {
+        // eslint-disable-next-line no-await-in-loop
+        options.site = await siteResponse.json();
+        if (options.site.gitHubURL) {
+          // eslint-disable-next-line no-await-in-loop
+          options.hlxSiteURL = await gitHubURLToHlxSite(options.site.gitHubURL);
+        }
+        console.log(`Site found in SpaceCat catalogue: ${options.baseURL}`);
+        found = true;
+        break;
+      }
+    }
+    if (!found) {
+      console.warn(`Site not found in SpaceCat catalogue, continuing without SpaceCat metadata.`);
+    }
+  } else {
+    console.warn('SPACECAT_SESSION_TOKEN not set, skipping SpaceCat lookup.');
   }
 
-  const siteAuditUrl = await composeAuditURL(options.site.baseURL);
+  const siteAuditUrl = await composeAuditURL(options.site?.baseURL || options.baseURL);
   options.siteAuditURL = siteAuditUrl.replace(/\.html$/, '');
   if (options.devBaseURL) {
     options.devAuditURL = await composeAuditURL(options.devBaseURL);

diff --git a/all-broken-backlinks-enabled.js b/all-broken-backlinks-enabled.js
@@ -21,7 +21,7 @@ async function makeApiCall(method, url, data = null) {
     const response = await fetch(`${BASE_URL}${url}`, {
       method,
       headers: {
-        'x-api-key': process.env.SPACECAT_API_KEY,
+        Authorization: `Bearer ${process.env.SPACECAT_SESSION_TOKEN}`,
         'Content-Type': 'application/json',
       },
       ...(data ? { body: JSON.stringify(data) } : {}),

diff --git a/all-gsc-enabled.js b/all-gsc-enabled.js
@@ -21,7 +21,7 @@ async function makeApiCall(method, url, data = null) {
     const response = await fetch(`${BASE_URL}${url}`, {
       method,
       headers: {
-        'x-api-key': process.env.SPACECAT_API_KEY,
+        Authorization: `Bearer ${process.env.SPACECAT_SESSION_TOKEN}`,
         'Content-Type': 'application/json',
       },
       ...(data ? { body: JSON.stringify(data) } : {}),

diff --git a/all-jsonld-enabled.js b/all-jsonld-enabled.js
@@ -21,7 +21,7 @@ async function makeApiCall(method, url, data = null) {
     const response = await fetch(`${BASE_URL}${url}`, {
       method,
       headers: {
-        'x-api-key': process.env.SPACECAT_API_KEY,
+        Authorization: `Bearer ${process.env.SPACECAT_SESSION_TOKEN}`,
         'Content-Type': 'application/json',
       },
       ...(data ? { body: JSON.stringify(data) } : {}),

diff --git a/assessment/brokenBacklinks.js b/assessment/brokenBacklinks.js
@@ -11,91 +11,59 @@
  */
 
 import dotenv from 'dotenv';
-import AhrefsAPIClient from 'spacecat-audit-worker/src/support/ahrefs-client.js';
-import { filterOutValidBacklinks } from 'spacecat-audit-worker/src/backlinks/handler.js';
+import SeoClient from '@adobe/mysticat-shared-seo-client';
 import Assessment from './libs/assessment-lib.js';
-import HttpClient from './libs/fetch-client.js';
 import { prodToDevUrl } from './libs/page-provider.js';
 
 dotenv.config();
 
-const httpClient = HttpClient.getInstance();
-
-export const brokenBacklinksAudit = async (options, { ahrefsClient }, log = console) => {
+export const brokenBacklinksAudit = async (options, { seoClient }, log = console) => {
   const {
     hlxSiteURL, siteAuditURL, devAuditURL,
   } = options;
-  const topBacklinksResponse = await ahrefsClient
-    .getBacklinks(siteAuditURL, options.topBacklinks);
-  let topBacklinks = topBacklinksResponse?.result?.backlinks;
 
-  if (!topBacklinks || topBacklinks.length === 0) {
-    log.warn(`No backlinks found for the site URL: ${siteAuditURL}`);
-    return [];
-  }
-  log.info(`${topBacklinks.length} backlinks found for the site URL: ${siteAuditURL}: ${JSON.stringify(topBacklinks[0])}`);
+  const { result } = await seoClient.getBrokenBacklinks(siteAuditURL, options.topBacklinks);
+  const backlinks = result?.backlinks;
 
-  let topPagesUrls;
-  if (options.onlyBacklinksInTopPages) {
-    const topPagesResponse = await ahrefsClient
-      .getTopPages(siteAuditURL, options.topPages);
-    if (!topPagesResponse?.result?.pages
-      || topPagesResponse?.result?.pages.length === 0) {
-      log.warn(`No top pages found for the site URL: ${siteAuditURL}`);
-      return [];
-    }
-    topPagesUrls = topPagesResponse.result.pages.map((page) => page.url);
-    log.info(`${topPagesUrls.length} top pages found for the site URL: ${siteAuditURL}: ${JSON.stringify(topPagesUrls[0])}`);
-  }
-
-  // filter out backlinks that are not top pages
-  if (options.onlyBacklinksInTopPages) {
-    topBacklinks = topBacklinks
-      .filter((backlink) => topPagesUrls.includes(backlink.url_to));
-    log.info(`${topBacklinks.length} backlinks after filtering by top pages for the site URL: ${siteAuditURL}`);
+  if (!backlinks || backlinks.length === 0) {
+    log.warn(`No broken backlinks found for the site URL: ${siteAuditURL}`);
+    return [];
   }
+  log.info(`${backlinks.length} broken backlinks found for the site URL: ${siteAuditURL}: ${JSON.stringify(backlinks[0])}`);
 
-  topBacklinks = topBacklinks
-    .map((backlink) => ({
-      ...backlink,
-      original_url_to: backlink.url_to,
-      url_to: prodToDevUrl(backlink.url_to, {
-        hlxSiteURL,
-        devAuditURL,
-      }),
-    }));
-
-  const realBrokenBacklinks = await filterOutValidBacklinks(topBacklinks, log);
-  log.info(`${realBrokenBacklinks.length} backlinks after filtering out valid ones for the site URL: ${siteAuditURL}`);
-  return realBrokenBacklinks;
+  return backlinks.map((backlink) => ({
+    ...backlink,
+    original_url_to: backlink.url_to,
+    url_to: prodToDevUrl(backlink.url_to, {
+      hlxSiteURL,
+      devAuditURL,
+    }),
+  }));
 };
 
 export const brokenBacklinks = async (options) => {
   const { baseURL } = options;
   const title = 'Broken Backlinks';
   console.log(`Running broken backlinks audit for ${baseURL} with options: ${JSON.stringify(options)}`);
 
-  const ahrefsClient = new AhrefsAPIClient({
-    apiKey: process.env.AHREFS_API_KEY,
-    apiBaseUrl: 'https://api.ahrefs.com/v3',
-  }, httpClient.getFetch());
+  const seoClient = SeoClient.createFrom({ env: process.env, log: console });
 
   const assessment = new Assessment(options, title);
   assessment.setRowHeadersAndDefaults({
     original_url: '',
     url: '',
-    source: 'ahrefs',
+    source: 'semrush',
     title: '',
     url_from: '',
   });
 
-  const brokenBacklinksResult = await brokenBacklinksAudit(options, { ahrefsClient });
+  const brokenBacklinksResult = await brokenBacklinksAudit(options, { seoClient });
 
   brokenBacklinksResult.forEach((backlink) => {
     assessment.addColumn({
       original_url: backlink.original_url_to,
       url: backlink.url_to,
-      source: 'ahrefs',
+      source: 'semrush',
       title: backlink.title,
       url_from: backlink.url_from,
     });

diff --git a/assessment/brokenInternalLinks.js b/assessment/brokenInternalLinks.js
@@ -11,7 +11,7 @@
  */
 
 import * as cheerio from 'cheerio';
-import AhrefsAPIClient from 'spacecat-audit-worker/src/support/ahrefs-client.js';
+import SeoClient from '@adobe/mysticat-shared-seo-client';
 import Assessment from './libs/assessment-lib.js';
 import HttpClient from './libs/fetch-client.js';
 import PageProvider from './libs/page-provider.js';
@@ -87,13 +87,10 @@ async function checkForBrokenInternalLinks(url, assessment) {
   });
 }
 async function brokenInternalLinksAudit(assessment, options) {
-  const ahrefsClient = new AhrefsAPIClient({
-    apiKey: process.env.AHREFS_API_KEY,
-    apiBaseUrl: 'https://api.ahrefs.com/v3',
-  }, httpClient.getFetch());
+  const seoClient = SeoClient.createFrom({ env: process.env, log: console });
 
   const pageProvider = new PageProvider({
-    ahrefsClient,
+    seoClient,
   });
 
   const pages = await pageProvider

diff --git a/assessment/canonical.js b/assessment/canonical.js
@@ -11,7 +11,7 @@
  */
 
 import * as cheerio from 'cheerio';
-import AhrefsAPIClient from 'spacecat-audit-worker/src/support/ahrefs-client.js';
+import SeoClient from '@adobe/mysticat-shared-seo-client';
 import Assessment from './libs/assessment-lib.js';
 import { fetchAllPages } from './sitemap.js';
 import HttpClient from './libs/fetch-client.js';
@@ -131,13 +131,10 @@ const canonicalAudit = async (options, assessment) => {
   console.log(`Fetching pages on audit url ${siteAuditURL}, from sitemap ${sitemap ? `provided at ${sitemap}` : ''}`);
   const sitemapUrls = await fetchAllPages(baseURL, sitemap);
 
-  console.log(`Fetching top ${topPages} pages from Ahrefs`);
-  const ahrefsClient = new AhrefsAPIClient({
-    apiKey: process.env.AHREFS_API_KEY,
-    apiBaseUrl: 'https://api.ahrefs.com/v3',
-  }, httpClient.getFetch());
+  console.log(`Fetching top ${topPages} pages from Semrush`);
+  const seoClient = SeoClient.createFrom({ env: process.env, log: console });
 
-  const fetchTopPages = async (url) => ahrefsClient.getTopPages(url, topPages);
+  const fetchTopPages = async (url) => seoClient.getTopPages(url, { limit: topPages });
 
   const responseNoWWW = await fetchTopPages(siteAuditURL.replace(/^www\./, ''));
   const responseWithWWW = siteAuditURL.startsWith('www.') ? await fetchTopPages(siteAuditURL) : await fetchTopPages(`www.${siteAuditURL}`);

diff --git a/assessment/libs/page-provider.js b/assessment/libs/page-provider.js
@@ -38,8 +38,8 @@ export function prodToDevUrl(pageUrl, { hlxSiteURL, devAuditURL }) {
 }
 
 export default class PageProvider {
-  constructor({ ahrefsClient, sitemap }, log = console) {
-    this.ahrefsClient = ahrefsClient;
+  constructor({ seoClient, sitemap }, log = console) {
+    this.seoClient = seoClient;
     this.sitemap = sitemap;
     this.log = log;
   }
@@ -49,17 +49,17 @@ export default class PageProvider {
       limit = 100, siteAuditURL, devAuditURL, hlxSiteURL,
     } = options || {};
 
-    if (this.ahrefsClient) {
+    if (this.seoClient) {
       try {
-        const response = await this.ahrefsClient.getTopPages(siteAuditURL, limit);
+        const response = await this.seoClient.getTopPages(siteAuditURL, { limit });
         if (response?.result?.pages) {
           return response?.result?.pages.map((page) => ({
             prodUrl: page.url,
             devUrl: prodToDevUrl(page.url, { devAuditURL, hlxSiteURL }),
           }));
         }
       } catch (error) {
-        this.log.error(`Failed to get pages from Ahrefs: ${error.message}`);
+        this.log.error(`Failed to get top pages from Semrush: ${error.message}`);
       }
     }
 

diff --git a/assessment/sitemap.js b/assessment/sitemap.js
@@ -256,7 +256,8 @@ export async function fetchSitemapsFromSource(sources, origin) {
   const sitemapFetchPromises = sources.map(async (sitemapUrl) => {
     try {
       try {
-        const fetchedXml = await fetchSitemapXml(sitemapUrl);
+        const fetchUrl = typeof sitemapUrl === 'string' ? sitemapUrl : sitemapUrl.url;
+        const fetchedXml = await fetchSitemapXml(fetchUrl);
         return await parseSitemap(fetchedXml, sitemapUrl);
       } catch (fetchError) {
         return { url: sitemapUrl.url, source: sitemapUrl.source, error: fetchError.message };

diff --git a/enable-404.js b/enable-404.js
@@ -26,7 +26,7 @@ async function makeApiCall(method, url, data = null) {
     const response = await fetch(`${BASE_URL}${url}`, {
       method,
       headers: {
-        'x-api-key': process.env.SPACECAT_API_KEY,
+        Authorization: `Bearer ${process.env.SPACECAT_SESSION_TOKEN}`,
         'Content-Type': 'application/json',
       },
       ...(data ? { body: JSON.stringify(data) } : {}),