From 1f36e9effcfb09f846703fe83ff886b1d056e844 Mon Sep 17 00:00:00 2001 From: Aman Varshney Date: Tue, 12 Aug 2025 17:37:04 +0530 Subject: [PATCH] feat(web): add llms.txt --- apps/cli/src/utils/analytics.ts | 5 +- apps/web/next.config.mjs | 8 + apps/web/package.json | 3 + apps/web/src/app/docs/[[...slug]]/page.tsx | 8 + apps/web/src/app/llms-full.txt/route.ts | 12 ++ .../web/src/app/llms.mdx/[[...slug]]/route.ts | 21 ++ apps/web/src/components/ai/page-actions.tsx | 192 ++++++++++++++++++ apps/web/src/lib/get-llm-text.ts | 26 +++ bun.lock | 5 +- 9 files changed, 278 insertions(+), 2 deletions(-) create mode 100644 apps/web/src/app/llms-full.txt/route.ts create mode 100644 apps/web/src/app/llms.mdx/[[...slug]]/route.ts create mode 100644 apps/web/src/components/ai/page-actions.tsx create mode 100644 apps/web/src/lib/get-llm-text.ts diff --git a/apps/cli/src/utils/analytics.ts b/apps/cli/src/utils/analytics.ts index cafcb5f..4ea89fb 100644 --- a/apps/cli/src/utils/analytics.ts +++ b/apps/cli/src/utils/analytics.ts @@ -5,7 +5,10 @@ import { isTelemetryEnabled } from "./telemetry"; const POSTHOG_API_KEY = process.env.POSTHOG_API_KEY || ""; const POSTHOG_HOST = process.env.POSTHOG_HOST; -export async function trackProjectCreation(config: ProjectConfig, disableAnalytics = false) { +export async function trackProjectCreation( + config: ProjectConfig, + disableAnalytics = false, +) { if (!isTelemetryEnabled() || disableAnalytics) return; const sessionId = `cli_${crypto.randomUUID().replace(/-/g, "")}`; diff --git a/apps/web/next.config.mjs b/apps/web/next.config.mjs index 93355f1..f949a15 100644 --- a/apps/web/next.config.mjs +++ b/apps/web/next.config.mjs @@ -15,6 +15,14 @@ const config = { outputFileTracingExcludes: { "*": ["./**/*.js.map", "./**/*.mjs.map", "./**/*.cjs.map"], }, + async rewrites() { + return [ + { + source: "/docs/:path*.mdx", + destination: "/llms.mdx/:path*", + }, + ]; + }, }; export default withMDX(config); diff --git a/apps/web/package.json b/apps/web/package.json index 87ded92..479e5d8 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -42,6 +42,9 @@ "react-dom": "^19.1.1", "react-tweet": "^3.2.2", "recharts": "2.15.4", + "remark": "^15.0.1", + "remark-gfm": "^4.0.1", + "remark-mdx": "^3.1.0", "shiki": "^3.9.1", "sonner": "^2.0.6", "tailwind-merge": "^3.3.1" diff --git a/apps/web/src/app/docs/[[...slug]]/page.tsx b/apps/web/src/app/docs/[[...slug]]/page.tsx index b16dd5d..9949ad1 100644 --- a/apps/web/src/app/docs/[[...slug]]/page.tsx +++ b/apps/web/src/app/docs/[[...slug]]/page.tsx @@ -7,6 +7,7 @@ import { DocsTitle, } from "fumadocs-ui/page"; import { notFound } from "next/navigation"; +import { LLMCopyButton, ViewOptions } from "@/components/ai/page-actions"; import { source } from "@/lib/source"; export default async function Page(props: { @@ -22,6 +23,13 @@ export default async function Page(props: { {page.data.title} {page.data.description} +
+ + +
diff --git a/apps/web/src/app/llms-full.txt/route.ts b/apps/web/src/app/llms-full.txt/route.ts new file mode 100644 index 0000000..ed54c01 --- /dev/null +++ b/apps/web/src/app/llms-full.txt/route.ts @@ -0,0 +1,12 @@ +import { getLLMText } from "@/lib/get-llm-text"; +import { source } from "@/lib/source"; + +// cached forever +export const revalidate = false; + +export async function GET() { + const scan = source.getPages().map(getLLMText); + const scanned = await Promise.all(scan); + + return new Response(scanned.join("\n\n")); +} diff --git a/apps/web/src/app/llms.mdx/[[...slug]]/route.ts b/apps/web/src/app/llms.mdx/[[...slug]]/route.ts new file mode 100644 index 0000000..80ebdfb --- /dev/null +++ b/apps/web/src/app/llms.mdx/[[...slug]]/route.ts @@ -0,0 +1,21 @@ +import { notFound } from "next/navigation"; +import { type NextRequest, NextResponse } from "next/server"; +import { getLLMText } from "@/lib/get-llm-text"; +import { source } from "@/lib/source"; + +export const revalidate = false; + +export async function GET( + _req: NextRequest, + { params }: { params: Promise<{ slug?: string[] }> }, +) { + const { slug } = await params; + const page = source.getPage(slug); + if (!page) notFound(); + + return new NextResponse(await getLLMText(page)); +} + +export function generateStaticParams() { + return source.generateParams(); +} diff --git a/apps/web/src/components/ai/page-actions.tsx b/apps/web/src/components/ai/page-actions.tsx new file mode 100644 index 0000000..c08d1af --- /dev/null +++ b/apps/web/src/components/ai/page-actions.tsx @@ -0,0 +1,192 @@ +"use client"; +import { cva } from "class-variance-authority"; +import { buttonVariants } from "fumadocs-ui/components/ui/button"; +import { + Popover, + PopoverContent, + PopoverTrigger, +} from "fumadocs-ui/components/ui/popover"; +import { useCopyButton } from "fumadocs-ui/utils/use-copy-button"; +import { + Check, + ChevronDown, + Copy, + ExternalLinkIcon, + MessageCircleIcon, + SearchIcon, +} from "lucide-react"; +import { useMemo, useState } from "react"; +import { cn } from "../../../lib/cn"; + +const cache = new Map(); + +export function LLMCopyButton({ + /** + * A URL to fetch the raw Markdown/MDX content of page + */ + markdownUrl, +}: { + markdownUrl: string; +}) { + const [isLoading, setLoading] = useState(false); + const [checked, onClick] = useCopyButton(async () => { + const cached = cache.get(markdownUrl); + if (cached) return navigator.clipboard.writeText(cached); + + setLoading(true); + + try { + await navigator.clipboard.write([ + new ClipboardItem({ + "text/plain": fetch(markdownUrl).then(async (res) => { + const content = await res.text(); + cache.set(markdownUrl, content); + + return content; + }), + }), + ]); + } finally { + setLoading(false); + } + }); + + return ( + + ); +} + +const optionVariants = cva( + "inline-flex items-center gap-2 rounded-lg p-2 text-sm hover:bg-fd-accent hover:text-fd-accent-foreground [&_svg]:size-4", +); + +export function ViewOptions({ + markdownUrl, + githubUrl, +}: { + /** + * A URL to the raw Markdown/MDX content of page + */ + markdownUrl: string; + + /** + * Source file URL on GitHub + */ + githubUrl: string; +}) { + const items = useMemo(() => { + const fullMarkdownUrl = + typeof window !== "undefined" + ? new URL(markdownUrl, window.location.origin) + : "loading"; + const q = `Read ${fullMarkdownUrl}, I want to ask questions about it.`; + + return [ + { + title: "Open in GitHub", + href: githubUrl, + icon: ( + + GitHub + + + ), + }, + { + title: "Open in ChatGPT", + href: `https://chatgpt.com/?${new URLSearchParams({ + hints: "search", + q, + })}`, + icon: ( + + OpenAI + + + ), + }, + { + title: "Open in Claude", + href: `https://claude.ai/new?${new URLSearchParams({ + q, + })}`, + icon: ( + + Anthropic + + + ), + }, + { + title: "Open in T3 Chat", + href: `https://t3.chat/new?${new URLSearchParams({ + q, + })}`, + icon: , + }, + { + title: "Open in Scira AI", + href: `https://scira.ai/?${new URLSearchParams({ + q, + })}`, + icon: , + }, + ]; + }, [githubUrl, markdownUrl]); + + return ( + + + Open + + + + {items.map((item) => ( + + {item.icon} + {item.title} + + + ))} + + + ); +} diff --git a/apps/web/src/lib/get-llm-text.ts b/apps/web/src/lib/get-llm-text.ts new file mode 100644 index 0000000..1a5e231 --- /dev/null +++ b/apps/web/src/lib/get-llm-text.ts @@ -0,0 +1,26 @@ +import type { InferPageType } from "fumadocs-core/source"; +import { remarkInclude } from "fumadocs-mdx/config"; +import { remark } from "remark"; +import remarkGfm from "remark-gfm"; +import remarkMdx from "remark-mdx"; +import type { source } from "@/lib/source"; + +const processor = remark() + .use(remarkMdx) + // needed for Fumadocs MDX + .use(remarkInclude) + .use(remarkGfm); + +export async function getLLMText(page: InferPageType) { + const processed = await processor.process({ + path: page.data._file.absolutePath, + value: page.data.content, + }); + + return `# ${page.data.title} +URL: ${page.url} + +${page.data.description} + +${processed.value}`; +} diff --git a/bun.lock b/bun.lock index 62a3bd7..c059557 100644 --- a/bun.lock +++ b/bun.lock @@ -14,7 +14,7 @@ }, "apps/cli": { "name": "create-better-t-stack", - "version": "2.31.0", + "version": "2.32.1", "bin": { "create-better-t-stack": "dist/cli.js", }, @@ -71,6 +71,9 @@ "react-dom": "^19.1.1", "react-tweet": "^3.2.2", "recharts": "2.15.4", + "remark": "^15.0.1", + "remark-gfm": "^4.0.1", + "remark-mdx": "^3.1.0", "shiki": "^3.9.1", "sonner": "^2.0.6", "tailwind-merge": "^3.3.1",