feat(web): add llms.txt

2025-10-12 23:52:15 +00:00 · 2025-08-12 17:37:04 +05:30
parent 9273629d92
commit 1f36e9effc
9 changed files with 278 additions and 2 deletions
--- a/apps/cli/src/utils/analytics.ts
+++ b/apps/cli/src/utils/analytics.ts
@@ -5,7 +5,10 @@ import { isTelemetryEnabled } from "./telemetry";
 const POSTHOG_API_KEY = process.env.POSTHOG_API_KEY || "";
 const POSTHOG_HOST = process.env.POSTHOG_HOST;

-export async function trackProjectCreation(config: ProjectConfig, disableAnalytics = false) {
+export async function trackProjectCreation(
+	config: ProjectConfig,
+	disableAnalytics = false,
+) {
 	if (!isTelemetryEnabled() || disableAnalytics) return;

 	const sessionId = `cli_${crypto.randomUUID().replace(/-/g, "")}`;
--- a/apps/web/next.config.mjs
+++ b/apps/web/next.config.mjs
@@ -15,6 +15,14 @@ const config = {
 	outputFileTracingExcludes: {
 		"*": ["./**/*.js.map", "./**/*.mjs.map", "./**/*.cjs.map"],
 	},
+	async rewrites() {
+		return [
+			{
+				source: "/docs/:path*.mdx",
+				destination: "/llms.mdx/:path*",
+			},
+		];
+	},
 };

 export default withMDX(config);
--- a/apps/web/package.json
+++ b/apps/web/package.json
@@ -42,6 +42,9 @@
    "react-dom": "^19.1.1",
    "react-tweet": "^3.2.2",
    "recharts": "2.15.4",
+    "remark": "^15.0.1",
+    "remark-gfm": "^4.0.1",
+    "remark-mdx": "^3.1.0",
    "shiki": "^3.9.1",
    "sonner": "^2.0.6",
    "tailwind-merge": "^3.3.1"
--- a/apps/web/src/app/docs/[[...slug]]/page.tsx
+++ b/apps/web/src/app/docs/[[...slug]]/page.tsx
@@ -7,6 +7,7 @@ import {
 	DocsTitle,
 } from "fumadocs-ui/page";
 import { notFound } from "next/navigation";
+import { LLMCopyButton, ViewOptions } from "@/components/ai/page-actions";
 import { source } from "@/lib/source";

 export default async function Page(props: {
@@ -22,6 +23,13 @@ export default async function Page(props: {
 		<DocsPage toc={page.data.toc} full={page.data.full}>
 			<DocsTitle>{page.data.title}</DocsTitle>
 			<DocsDescription>{page.data.description}</DocsDescription>
+			<div className="flex flex-row items-center gap-2 border-b pt-2 pb-6">
+				<LLMCopyButton markdownUrl={`${page.url}.mdx`} />
+				<ViewOptions
+					markdownUrl={`${page.url}.mdx`}
+					githubUrl={`https://github.com/AmanVarshney01/create-better-t-stack/blob/main/apps/web/content/docs/${page.path}`}
+				/>
+			</div>
 			<DocsBody>
 				<MDX components={{ ...defaultMdxComponents, ...TabsComponents }} />
 			</DocsBody>
--- a/apps/web/src/app/llms-full.txt/route.ts
+++ b/apps/web/src/app/llms-full.txt/route.ts
@@ -0,0 +1,12 @@
+import { getLLMText } from "@/lib/get-llm-text";
+import { source } from "@/lib/source";
+
+// cached forever
+export const revalidate = false;
+
+export async function GET() {
+	const scan = source.getPages().map(getLLMText);
+	const scanned = await Promise.all(scan);
+
+	return new Response(scanned.join("\n\n"));
+}
--- a/apps/web/src/app/llms.mdx/[[...slug]]/route.ts
+++ b/apps/web/src/app/llms.mdx/[[...slug]]/route.ts
@@ -0,0 +1,21 @@
+import { notFound } from "next/navigation";
+import { type NextRequest, NextResponse } from "next/server";
+import { getLLMText } from "@/lib/get-llm-text";
+import { source } from "@/lib/source";
+
+export const revalidate = false;
+
+export async function GET(
+	_req: NextRequest,
+	{ params }: { params: Promise<{ slug?: string[] }> },
+) {
+	const { slug } = await params;
+	const page = source.getPage(slug);
+	if (!page) notFound();
+
+	return new NextResponse(await getLLMText(page));
+}
+
+export function generateStaticParams() {
+	return source.generateParams();
+}
--- a/apps/web/src/components/ai/page-actions.tsx
+++ b/apps/web/src/components/ai/page-actions.tsx
@@ -0,0 +1,192 @@
+"use client";
+import { cva } from "class-variance-authority";
+import { buttonVariants } from "fumadocs-ui/components/ui/button";
+import {
+	Popover,
+	PopoverContent,
+	PopoverTrigger,
+} from "fumadocs-ui/components/ui/popover";
+import { useCopyButton } from "fumadocs-ui/utils/use-copy-button";
+import {
+	Check,
+	ChevronDown,
+	Copy,
+	ExternalLinkIcon,
+	MessageCircleIcon,
+	SearchIcon,
+} from "lucide-react";
+import { useMemo, useState } from "react";
+import { cn } from "../../../lib/cn";
+
+const cache = new Map<string, string>();
+
+export function LLMCopyButton({
+	/**
+	 * A URL to fetch the raw Markdown/MDX content of page
+	 */
+	markdownUrl,
+}: {
+	markdownUrl: string;
+}) {
+	const [isLoading, setLoading] = useState(false);
+	const [checked, onClick] = useCopyButton(async () => {
+		const cached = cache.get(markdownUrl);
+		if (cached) return navigator.clipboard.writeText(cached);
+
+		setLoading(true);
+
+		try {
+			await navigator.clipboard.write([
+				new ClipboardItem({
+					"text/plain": fetch(markdownUrl).then(async (res) => {
+						const content = await res.text();
+						cache.set(markdownUrl, content);
+
+						return content;
+					}),
+				}),
+			]);
+		} finally {
+			setLoading(false);
+		}
+	});
+
+	return (
+		<button
+			type="button"
+			disabled={isLoading}
+			className={cn(
+				buttonVariants({
+					color: "secondary",
+					size: "sm",
+					className: "gap-2 [&_svg]:size-3.5 [&_svg]:text-fd-muted-foreground",
+				}),
+			)}
+			onClick={onClick}
+		>
+			{checked ? <Check /> : <Copy />}
+			Copy Markdown
+		</button>
+	);
+}
+
+const optionVariants = cva(
+	"inline-flex items-center gap-2 rounded-lg p-2 text-sm hover:bg-fd-accent hover:text-fd-accent-foreground [&_svg]:size-4",
+);
+
+export function ViewOptions({
+	markdownUrl,
+	githubUrl,
+}: {
+	/**
+	 * A URL to the raw Markdown/MDX content of page
+	 */
+	markdownUrl: string;
+
+	/**
+	 * Source file URL on GitHub
+	 */
+	githubUrl: string;
+}) {
+	const items = useMemo(() => {
+		const fullMarkdownUrl =
+			typeof window !== "undefined"
+				? new URL(markdownUrl, window.location.origin)
+				: "loading";
+		const q = `Read ${fullMarkdownUrl}, I want to ask questions about it.`;
+
+		return [
+			{
+				title: "Open in GitHub",
+				href: githubUrl,
+				icon: (
+					<svg fill="currentColor" role="img" viewBox="0 0 24 24">
+						<title>GitHub</title>
+						<path d="M12 .297c-6.63 0-12 5.373-12 12 0 5.303 3.438 9.8 8.205 11.385.6.113.82-.258.82-.577 0-.285-.01-1.04-.015-2.04-3.338.724-4.042-1.61-4.042-1.61C4.422 18.07 3.633 17.7 3.633 17.7c-1.087-.744.084-.729.084-.729 1.205.084 1.838 1.236 1.838 1.236 1.07 1.835 2.809 1.305 3.495.998.108-.776.417-1.305.76-1.605-2.665-.3-5.466-1.332-5.466-5.93 0-1.31.465-2.38 1.235-3.22-.135-.303-.54-1.523.105-3.176 0 0 1.005-.322 3.3 1.23.96-.267 1.98-.399 3-.405 1.02.006 2.04.138 3 .405 2.28-1.552 3.285-1.23 3.285-1.23.645 1.653.24 2.873.12 3.176.765.84 1.23 1.91 1.23 3.22 0 4.61-2.805 5.625-5.475 5.92.42.36.81 1.096.81 2.22 0 1.606-.015 2.896-.015 3.286 0 .315.21.69.825.57C20.565 22.092 24 17.592 24 12.297c0-6.627-5.373-12-12-12" />
+					</svg>
+				),
+			},
+			{
+				title: "Open in ChatGPT",
+				href: `https://chatgpt.com/?${new URLSearchParams({
+					hints: "search",
+					q,
+				})}`,
+				icon: (
+					<svg
+						role="img"
+						viewBox="0 0 24 24"
+						fill="currentColor"
+						xmlns="http://www.w3.org/2000/svg"
+					>
+						<title>OpenAI</title>
+						<path d="M22.2819 9.8211a5.9847 5.9847 0 0 0-.5157-4.9108 6.0462 6.0462 0 0 0-6.5098-2.9A6.0651 6.0651 0 0 0 4.9807 4.1818a5.9847 5.9847 0 0 0-3.9977 2.9 6.0462 6.0462 0 0 0 .7427 7.0966 5.98 5.98 0 0 0 .511 4.9107 6.051 6.051 0 0 0 6.5146 2.9001A5.9847 5.9847 0 0 0 13.2599 24a6.0557 6.0557 0 0 0 5.7718-4.2058 5.9894 5.9894 0 0 0 3.9977-2.9001 6.0557 6.0557 0 0 0-.7475-7.0729zm-9.022 12.6081a4.4755 4.4755 0 0 1-2.8764-1.0408l.1419-.0804 4.7783-2.7582a.7948.7948 0 0 0 .3927-.6813v-6.7369l2.02 1.1686a.071.071 0 0 1 .038.052v5.5826a4.504 4.504 0 0 1-4.4945 4.4944zm-9.6607-4.1254a4.4708 4.4708 0 0 1-.5346-3.0137l.142.0852 4.783 2.7582a.7712.7712 0 0 0 .7806 0l5.8428-3.3685v2.3324a.0804.0804 0 0 1-.0332.0615L9.74 19.9502a4.4992 4.4992 0 0 1-6.1408-1.6464zM2.3408 7.8956a4.485 4.485 0 0 1 2.3655-1.9728V11.6a.7664.7664 0 0 0 .3879.6765l5.8144 3.3543-2.0201 1.1685a.0757.0757 0 0 1-.071 0l-4.8303-2.7865A4.504 4.504 0 0 1 2.3408 7.872zm16.5963 3.8558L13.1038 8.364 15.1192 7.2a.0757.0757 0 0 1 .071 0l4.8303 2.7913a4.4944 4.4944 0 0 1-.6765 8.1042v-5.6772a.79.79 0 0 0-.407-.667zm2.0107-3.0231l-.142-.0852-4.7735-2.7818a.7759.7759 0 0 0-.7854 0L9.409 9.2297V6.8974a.0662.0662 0 0 1 .0284-.0615l4.8303-2.7866a4.4992 4.4992 0 0 1 6.6802 4.66zM8.3065 12.863l-2.02-1.1638a.0804.0804 0 0 1-.038-.0567V6.0742a4.4992 4.4992 0 0 1 7.3757-3.4537l-.142.0805L8.704 5.459a.7948.7948 0 0 0-.3927.6813zm1.0976-2.3654l2.602-1.4998 2.6069 1.4998v2.9994l-2.5974 1.4997-2.6067-1.4997Z" />
+					</svg>
+				),
+			},
+			{
+				title: "Open in Claude",
+				href: `https://claude.ai/new?${new URLSearchParams({
+					q,
+				})}`,
+				icon: (
+					<svg
+						fill="currentColor"
+						role="img"
+						viewBox="0 0 24 24"
+						xmlns="http://www.w3.org/2000/svg"
+					>
+						<title>Anthropic</title>
+						<path d="M17.3041 3.541h-3.6718l6.696 16.918H24Zm-10.6082 0L0 20.459h3.7442l1.3693-3.5527h7.0052l1.3693 3.5528h3.7442L10.5363 3.5409Zm-.3712 10.2232 2.2914-5.9456 2.2914 5.9456Z" />
+					</svg>
+				),
+			},
+			{
+				title: "Open in T3 Chat",
+				href: `https://t3.chat/new?${new URLSearchParams({
+					q,
+				})}`,
+				icon: <MessageCircleIcon />,
+			},
+			{
+				title: "Open in Scira AI",
+				href: `https://scira.ai/?${new URLSearchParams({
+					q,
+				})}`,
+				icon: <SearchIcon />,
+			},
+		];
+	}, [githubUrl, markdownUrl]);
+
+	return (
+		<Popover>
+			<PopoverTrigger
+				className={cn(
+					buttonVariants({
+						color: "secondary",
+						size: "sm",
+						className: "gap-2",
+					}),
+				)}
+			>
+				Open
+				<ChevronDown className="size-3.5 text-fd-muted-foreground" />
+			</PopoverTrigger>
+			<PopoverContent className="flex flex-col overflow-auto">
+				{items.map((item) => (
+					<a
+						key={item.href}
+						href={item.href}
+						rel="noreferrer noopener"
+						target="_blank"
+						className={cn(optionVariants())}
+					>
+						{item.icon}
+						{item.title}
+						<ExternalLinkIcon className="ms-auto size-3.5 text-fd-muted-foreground" />
+					</a>
+				))}
+			</PopoverContent>
+		</Popover>
+	);
+}
--- a/apps/web/src/lib/get-llm-text.ts
+++ b/apps/web/src/lib/get-llm-text.ts
@@ -0,0 +1,26 @@
+import type { InferPageType } from "fumadocs-core/source";
+import { remarkInclude } from "fumadocs-mdx/config";
+import { remark } from "remark";
+import remarkGfm from "remark-gfm";
+import remarkMdx from "remark-mdx";
+import type { source } from "@/lib/source";
+
+const processor = remark()
+	.use(remarkMdx)
+	// needed for Fumadocs MDX
+	.use(remarkInclude)
+	.use(remarkGfm);
+
+export async function getLLMText(page: InferPageType<typeof source>) {
+	const processed = await processor.process({
+		path: page.data._file.absolutePath,
+		value: page.data.content,
+	});
+
+	return `# ${page.data.title}
+URL: ${page.url}
+
+${page.data.description}
+
+${processed.value}`;
+}
--- a/bun.lock
+++ b/bun.lock
@@ -14,7 +14,7 @@
    },
    "apps/cli": {
      "name": "create-better-t-stack",
-      "version": "2.31.0",
+      "version": "2.32.1",
      "bin": {
        "create-better-t-stack": "dist/cli.js",
      },
@@ -71,6 +71,9 @@
        "react-dom": "^19.1.1",
        "react-tweet": "^3.2.2",
        "recharts": "2.15.4",
+        "remark": "^15.0.1",
+        "remark-gfm": "^4.0.1",
+        "remark-mdx": "^3.1.0",
        "shiki": "^3.9.1",
        "sonner": "^2.0.6",
        "tailwind-merge": "^3.3.1",