astro-discovery/src/generators/robots.ts

import type { RobotsConfig } from '../types.js';

/**
 * Default LLM bot user agents that should have access to llms.txt
 */
const DEFAULT_LLM_BOTS = [
  'Anthropic-AI',
  'Claude-Web',
  'GPTBot',
  'ChatGPT-User',
  'cohere-ai',
  'Google-Extended',
  'PerplexityBot',
  'Applebot-Extended',
];

/**
 * Generate robots.txt content
 *
 * @param config - Robots.txt configuration
 * @param siteURL - Site base URL
 * @returns Generated robots.txt content
 */
export function generateRobotsTxt(
  config: RobotsConfig,
  siteURL: URL
): string {
  const lines: string[] = [];

  // Header comment
  lines.push('# robots.txt');
  lines.push(`# Generated by @astrojs/discovery for ${siteURL.hostname}`);
  lines.push('');

  // Allow all bots by default
  if (config.allowAllBots !== false) {
    lines.push('User-agent: *');
    lines.push('Allow: /');
    lines.push('');
  }

  // Add sitemap reference
  lines.push('# Sitemaps');
  lines.push(`Sitemap: ${new URL('sitemap-index.xml', siteURL).href}`);
  lines.push('');

  // LLM-specific rules
  if (config.llmBots?.enabled !== false) {
    lines.push('# LLM-specific resources');
    lines.push('# AI assistants can find additional context at /llms.txt');
    lines.push('# See: https://github.com/anthropics/llm-txt');
    lines.push('');

    const agents = config.llmBots?.agents || DEFAULT_LLM_BOTS;
    agents.forEach(agent => {
      lines.push(`User-agent: ${agent}`);
    });
    lines.push('Allow: /llms.txt');
    lines.push('Allow: /llms-full.txt');
    lines.push('');
  }

  // Additional agent rules
  if (config.additionalAgents && config.additionalAgents.length > 0) {
    lines.push('# Custom agent rules');
    lines.push('');

    config.additionalAgents.forEach(agent => {
      lines.push(`User-agent: ${agent.userAgent}`);

      if (agent.allow && agent.allow.length > 0) {
        agent.allow.forEach(path => {
          lines.push(`Allow: ${path}`);
        });
      }

      if (agent.disallow && agent.disallow.length > 0) {
        agent.disallow.forEach(path => {
          lines.push(`Disallow: ${path}`);
        });
      }

      lines.push('');
    });
  }

  // Crawl delay
  if (config.crawlDelay) {
    lines.push('# Crawl delay (be nice to our server)');
    lines.push(`Crawl-delay: ${config.crawlDelay}`);
    lines.push('');
  }

  // Custom rules
  if (config.customRules) {
    lines.push('# Custom rules');
    lines.push(config.customRules.trim());
    lines.push('');
  }

  return lines.join('\n').trim() + '\n';
}