26-8-0 docs publication

.env.sample

@@ -1,4 +1,5 @@
 GATSBY_ALGOLIA_APP_ID=
 GATSBY_ALGOLIA_SEARCH_KEY=
 ALGOLIA_ADMIN_KEY=
-BUILD_ENV=LOCAL
+GATSBY_CLOUDFLARE_URL=
+BUILD_ENV=LOCAL

.github/workflows/deploy.yml

@@ -79,13 +79,15 @@ jobs:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
       - uses: actions/checkout@v2
 
-      # Use NodeJS v12.18.3
+      # Use NodeJS v20
       - uses: actions/setup-node@v2
         with:
-          node-version: '14.15.4'
+          node-version: '20'
 
       # Run npm install
       - name: Run npm install
+        env:
+          PACKAGECLOUD_TOKEN: ${{ secrets.PACKAGECLOUD_TOKEN }}
         run: npm install
 
       - name: create env file

.github/workflows/main.yml

@@ -22,13 +22,15 @@ jobs:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
       - uses: actions/checkout@v2
 
-      # Use NodeJS v14.19.0
+      # Use NodeJS v20
       - uses: actions/setup-node@v2
         with:
-          node-version: '14.15.4'
+          node-version: '20'
 
       # Run npm install
       - name: Run npm install
+        env:
+          PACKAGECLOUD_TOKEN: ${{ secrets.PACKAGECLOUD_TOKEN }}
         run: npm install
 
       # Runs tests

.npmrc

@@ -1 +1,3 @@
 registry = 'https://registry.npmjs.org/'
+@thoughtspot:registry=https://packagecloud.io/modeanalytics/tse/npm/
+//packagecloud.io/modeanalytics/tse/npm/:_authToken=${PACKAGECLOUD_TOKEN}

gatsby-browser.js

@@ -0,0 +1,31 @@
+exports.onClientEntry = () => {
+    // Dynamic imports so @thoughtspot/radiant-react (which reads `window` at
+    // module-load time) is never evaluated during Gatsby's SSR build.
+    Promise.all([
+        import('./src/contexts/FloatingAssistantContext'),
+        import('./src/components/FloatingAssistant'),
+        import('react'),
+        import('react-dom/client'),
+    ]).then(([
+        { FloatingAssistantProvider },
+        { default: FloatingAssistant },
+        React,
+        ReactDOM,
+    ]) => {
+        const container = document.createElement('div');
+        container.id = 'floating-assistant-root';
+        document.body.appendChild(container);
+
+        ReactDOM.createRoot(container).render(
+            React.createElement(
+                FloatingAssistantProvider,
+                null,
+                React.createElement(FloatingAssistant)
+            )
+        );
+    });
+};
+
+exports.onRouteUpdate = ({ location }) => {
+    window.dispatchEvent(new CustomEvent('gatsby-route-update', { detail: { location } }));
+};

gatsby-node.js

@@ -78,24 +78,69 @@ exports.onPostBuild = async ({ graphql, reporter }) => {
                         node {
                             document { title }
                             pageAttributes { pageid }
+                            fields { markdownBody }
+                            parent {
+                                ... on File {
+                                    sourceInstanceName
+                                    relativePath
+                                }
+                            }
                         }
                     }
                 }
             }
         `);
 
         if (result.errors) {
-            reporter.warn(`llms.txt generation: GraphQL errors — ${JSON.stringify(result.errors)}`);
+            reporter.warn(`Build-time generation: GraphQL errors — ${JSON.stringify(result.errors)}`);
             return;
         }
 
-        const pageMap = {};
+        // pageData keyed by pageid: { title, docPath }
+        // docPath is the URL-path segment (e.g. '/getting-started', '/tutorials/intro')
+        // derived from getDocLinkFromEdge so tutorials with subdirectories resolve correctly.
+        const pageData = {};
+        let mdCount = 0;
+        // Pages whose markdown exceeds this limit are truncated with a continuation link.
+        // Keeps all pages within the agent-score page-size threshold while preserving crawlability.
+        const MAX_MD_CHARS = 95_000;
+
         result.data.allAsciidoc.edges.forEach(({ node }) => {
             const pageid = node.pageAttributes?.pageid;
             const title = node.document?.title;
-            if (pageid && title) pageMap[pageid] = title;
+            const markdownBody = node.fields?.markdownBody;
+            const relativePath = node.parent?.relativePath || '';
+            // Auto-generated per-symbol SDK reference pages (scripts/Converter/index.ts) —
+            // represented in llms.txt by the single curated VisualEmbedSdk entry, not individually.
+            const isTypedocGenerated = relativePath.startsWith('generated/typedoc/');
+
+            if (!pageid || pageid.startsWith('nav-')) return;
+
+            const docPath = getDocLinkFromEdge({ node }); // e.g. '/getting-started' or '/tutorials/category/page'
+            if (title) pageData[pageid] = { title, docPath, isTypedocGenerated };
+
+            // Write static .md file — serves at /docs<docPath>.md for agent crawlers
+            if (markdownBody) {
+                let body = markdownBody;
+                if (body.length > MAX_MD_CHARS) {
+                    // Trim at the last complete line within the limit
+                    const cut = body.lastIndexOf('\n', MAX_MD_CHARS);
+                    body = body.slice(0, cut > MAX_MD_CHARS * 0.8 ? cut : MAX_MD_CHARS);
+                    body += `\n\n---\n\n> **Content truncated.** This page exceeds the inline size limit. View the complete documentation at [${SITE_URL}${docPath}](${SITE_URL}${docPath})\n`;
+                }
+                const header = `# ${title ?? pageid}\n\n> For the complete documentation index, see [llms.txt](${SITE_URL}/llms.txt)\n\nSource: ${SITE_URL}${docPath}\n\n`;
+                fsExtra.outputFileSync(
+                    `${__dirname}/public${docPath}.md`,
+                    header + body,
+                );
+                mdCount++;
+            }
         });
 
+        reporter.info(`[md-gen] Wrote ${mdCount} .md files`);
+
+        // Generate llms.txt — curated sections first, then any remaining pages
+        const coveredIds = new Set();
         const lines = [
             '# ThoughtSpot Developer Documentation',
             '',
@@ -104,21 +149,39 @@ exports.onPostBuild = async ({ graphql, reporter }) => {
         ];
 
         for (const section of LLMS_SECTIONS) {
-            lines.push(`## ${section.label}`);
+            const sectionLines = [];
             for (const pageId of section.pageIds) {
-                const title = pageMap[pageId];
-                if (title) lines.push(`- [${title}](${SITE_URL}/${pageId})`);
+                const data = pageData[pageId];
+                if (data) {
+                    sectionLines.push(`- [${data.title}](${SITE_URL}${data.docPath}.md)`);
+                    coveredIds.add(pageId);
+                }
+            }
+            if (sectionLines.length) {
+                lines.push(`## ${section.label}`);
+                lines.push(...sectionLines);
+                lines.push('');
             }
+        }
+
+        // Add pages that exist as Asciidoc nodes but aren't in any LLMS_SECTIONS entry.
+        // Excludes typedoc-generated pages — those are covered by the curated VisualEmbedSdk entry.
+        const uncovered = Object.entries(pageData).filter(
+            ([id, data]) => !coveredIds.has(id) && !data.isTypedocGenerated,
+        );
+        if (uncovered.length) {
+            lines.push('## Additional documentation');
+            uncovered.forEach(([, { title, docPath }]) => lines.push(`- [${title}](${SITE_URL}${docPath}.md)`));
             lines.push('');
         }
 
         fsExtra.writeFileSync(
             `${__dirname}/public/llms.txt`,
             lines.join('\n'),
         );
-        reporter.info(`llms.txt generated with ${Object.keys(pageMap).length} pages`);
+        reporter.info(`llms.txt: ${coveredIds.size} curated + ${uncovered.length} additional = ${coveredIds.size + uncovered.length} total pages`);
     } catch (err) {
-        reporter.warn(`llms.txt generation failed: ${err.message}`);
+        reporter.warn(`Build-time generation failed: ${err.message}`);
     }
 };
 exports.createPages = async function ({ actions, graphql }) {

gatsby-ssr.js

@@ -0,0 +1,41 @@
+const React = require('react');
+const RADIANT_SPRITE = require('./src/components/FloatingAssistant/radiantSprite');
+const { SITE_URL } = require('./src/configs/doc-configs');
+
+exports.onRenderBody = ({ setHeadComponents, setPreBodyComponents }) => {
+    setHeadComponents([
+        React.createElement('link', {
+            key: 'llms-txt',
+            rel: 'llms-txt',
+            href: `${SITE_URL}/llms.txt`,
+        }),
+    ]);
+
+    // Visually-hidden body element — picked up by agent crawlers that parse the DOM
+    // but ignore <head> link tags.
+    setPreBodyComponents([
+        React.createElement(
+            'div',
+            {
+                key: 'llms-txt-directive',
+                style: {
+                    position: 'absolute',
+                    width: '1px',
+                    height: '1px',
+                    overflow: 'hidden',
+                    clip: 'rect(0,0,0,0)',
+                    whiteSpace: 'nowrap',
+                },
+            },
+            React.createElement(
+                'a',
+                { href: `${SITE_URL}/llms.txt` },
+                'LLMs.txt: Complete documentation index for AI agents',
+            ),
+        ),
+        React.createElement('div', {
+            key: 'radiant-sprite',
+            dangerouslySetInnerHTML: { __html: RADIANT_SPRITE },
+        }),
+    ]);
+};

middleware.js

@@ -0,0 +1,26 @@
+import { rewrite, next } from '@vercel/functions'
+
+// Content-negotiation for AEO: agents (Claude Code, Cursor, OpenCode, etc.) send
+// `Accept: text/markdown` and expect the markdown sibling generated at build time.
+// Vercel's vercel.json `rewrites` can't do this because filesystem match takes
+// precedence over rewrites for a path that already resolves to an existing static
+// file, so this has to run in Middleware instead, which executes before that
+// filesystem check.
+export const config = {
+  matcher: '/:path*',
+}
+
+export default function middleware (request) {
+  const accept = request.headers.get('accept') || ''
+  if (!accept.includes('text/markdown')) return next()
+
+  const url = new URL(request.url)
+  const { pathname } = url
+  if (pathname.endsWith('.md')) return next()
+
+  const lastSegment = pathname.slice(pathname.lastIndexOf('/') + 1)
+  if (lastSegment.includes('.')) return next() // static asset (css/js/svg/png/...), not a doc page
+
+  url.pathname = pathname.endsWith('/') ? `${pathname}index.md` : `${pathname}.md`
+  return rewrite(url)
+}