Exclude language variants from llms.txt and llms-full.txt (#4165)

Author: nolannbiron

packages/gitbook/src/lib/sites.ts

@@ -39,6 +39,16 @@ export function getSiteStructureSections(
         : [];
 }
 
+/**
+ * Filter site spaces to only include those matching the given locale
+ */
+export function filterSiteSpacesByLocale(
+    siteSpaces: SiteSpace[],
+    locale: TranslationLanguage | undefined
+): SiteSpace[] {
+    return siteSpaces.filter((siteSpace) => siteSpace.space.language === locale);
+}
+
 /*
  * Gets all site spaces, in a site structure and overrides the title
  */

packages/gitbook/src/routes/llms-full.ts

@@ -3,7 +3,7 @@ import { throwIfDataError } from '@/lib/data';
 import { fromPageMarkdown, toPageMarkdown } from '@/lib/markdownPage';
 import { joinPath } from '@/lib/paths';
 import { getIndexablePages } from '@/lib/sitemap';
-import { getSiteStructureSections } from '@/lib/sites';
+import { filterSiteSpacesByLocale, getSiteStructureSections } from '@/lib/sites';
 import type { RevisionPageDocument, SiteSection, SiteSpace } from '@gitbook/api';
 import assertNever from 'assert-never';
 import type { Paragraph } from 'mdast';
@@ -118,7 +118,9 @@ export async function streamMarkdownFromSiteSpaces(
     const allPages: Array<{ page: RevisionPageDocument; siteSpace: SiteSpace; basePath: string }> =
         [];
 
-    for (const siteSpace of siteSpaces) {
+    const filteredSiteSpaces = filterSiteSpacesByLocale(siteSpaces, context.locale);
+
+    for (const siteSpace of filteredSiteSpaces) {
         const siteSpaceUrl = siteSpace.urls.published;
         if (!siteSpaceUrl) {
             continue;

packages/gitbook/src/routes/llms.ts

@@ -3,7 +3,7 @@ import { throwIfDataError } from '@/lib/data';
 import type { GitBookLinker } from '@/lib/links';
 import { joinPath } from '@/lib/paths';
 import { type FlatPageEntry, getIndexablePages } from '@/lib/sitemap';
-import { getLocalizedTitle, getSiteStructureSections } from '@/lib/sites';
+import { filterSiteSpacesByLocale, getLocalizedTitle, getSiteStructureSections } from '@/lib/sites';
 import type { SiteSection, SiteSpace } from '@gitbook/api';
 import assertNever from 'assert-never';
 import type { ListItem, Paragraph, Root, RootContent } from 'mdast';
@@ -131,8 +131,10 @@ async function getNodesFromSiteSpaces(
 ): Promise<RootContent[]> {
     const { dataFetcher, linker } = context;
 
+    const filteredSiteSpaces = filterSiteSpacesByLocale(siteSpaces, context.locale);
+
     const all = await Promise.all(
-        siteSpaces.map(async (siteSpace): Promise<RootContent[]> => {
+        filteredSiteSpaces.map(async (siteSpace): Promise<RootContent[]> => {
             const siteSpaceUrl = siteSpace.urls.published;
             if (!siteSpaceUrl) {
                 return [];