feat: initial implementation of @astrojs/discovery integration

This commit introduces a comprehensive Astro integration that automatically generates discovery files for websites: Features: - robots.txt with LLM bot support (Anthropic-AI, GPTBot, etc.) - llms.txt for AI assistant context and instructions - humans.txt for team credits and site information - Automatic sitemap integration via @astrojs/sitemap Technical Details: - TypeScript implementation with full type safety - Configurable HTTP caching headers - Custom template support for all generated files - Sensible defaults with extensive customization options - Date-based versioning (2025.11.03) Testing: - 34 unit tests covering all generators - Test coverage for robots.txt, llms.txt, and humans.txt - Integration with Vitest Documentation: - Comprehensive README with examples - API reference documentation - Contributing guidelines - Example configurations (minimal and full)
2025-11-03 07:36:39 -07:00 · 2025-11-03 07:36:39 -07:00 · d25dde4627
commit d25dde4627
25 changed files with 11001 additions and 0 deletions
--- a/src/generators/humans.ts
+++ b/src/generators/humans.ts
@ -0,0 +1,145 @@
+import type { HumansConfig } from '../types.js';
+
+/**
+ * Generate humans.txt content
+ *
+ * This file provides human-readable credits and information about
+ * the site, team, and technology stack.
+ *
+ * @param config - Humans.txt configuration
+ * @returns Generated humans.txt content
+ */
+export function generateHumansTxt(config: HumansConfig): string {
+  const lines: string[] = [];
+
+  // Team section
+  if (config.team && config.team.length > 0) {
+    lines.push('/* TEAM */');
+    lines.push('');
+
+    config.team.forEach((member, index) => {
+      if (index > 0) {
+        lines.push('');
+      }
+
+      lines.push(`  Name: ${member.name}`);
+      if (member.role) {
+        lines.push(`  Role: ${member.role}`);
+      }
+      if (member.contact) {
+        lines.push(`  Contact: ${member.contact}`);
+      }
+      if (member.location) {
+        lines.push(`  From: ${member.location}`);
+      }
+      if (member.twitter) {
+        lines.push(`  Twitter: ${member.twitter}`);
+      }
+      if (member.github) {
+        lines.push(`  GitHub: ${member.github}`);
+      }
+    });
+
+    lines.push('');
+  }
+
+  // Thanks section
+  if (config.thanks && config.thanks.length > 0) {
+    lines.push('/* THANKS */');
+    lines.push('');
+    config.thanks.forEach(thanks => {
+      lines.push(`  ${thanks}`);
+    });
+    lines.push('');
+  }
+
+  // Site section
+  if (config.site) {
+    lines.push('/* SITE */');
+    lines.push('');
+
+    const lastUpdate = config.site.lastUpdate === 'auto'
+      ? new Date().toISOString().split('T')[0]
+      : config.site.lastUpdate;
+
+    if (lastUpdate) {
+      lines.push(`  Last update: ${lastUpdate}`);
+    }
+    if (config.site.language) {
+      lines.push(`  Language: ${config.site.language}`);
+    }
+    if (config.site.doctype) {
+      lines.push(`  Doctype: ${config.site.doctype}`);
+    }
+    if (config.site.ide) {
+      lines.push(`  IDE: ${config.site.ide}`);
+    }
+
+    if (config.site.techStack && config.site.techStack.length > 0) {
+      lines.push(`  Tech Stack: ${config.site.techStack.join(', ')}`);
+    }
+    if (config.site.standards && config.site.standards.length > 0) {
+      lines.push(`  Standards: ${config.site.standards.join(', ')}`);
+    }
+    if (config.site.components && config.site.components.length > 0) {
+      lines.push(`  Components: ${config.site.components.join(', ')}`);
+    }
+    if (config.site.software && config.site.software.length > 0) {
+      lines.push(`  Software: ${config.site.software.join(', ')}`);
+    }
+
+    lines.push('');
+  }
+
+  // Story section
+  if (config.story) {
+    lines.push('/* THE STORY */');
+    lines.push('');
+
+    // Indent multi-line stories
+    const storyLines = config.story.trim().split('\n');
+    storyLines.forEach(line => {
+      lines.push(`  ${line.trim()}`);
+    });
+
+    lines.push('');
+  }
+
+  // Fun Facts section
+  if (config.funFacts && config.funFacts.length > 0) {
+    lines.push('/* FUN FACTS */');
+    lines.push('');
+    config.funFacts.forEach(fact => {
+      lines.push(`  ${fact}`);
+    });
+    lines.push('');
+  }
+
+  // Philosophy section
+  if (config.philosophy && config.philosophy.length > 0) {
+    lines.push('/* PHILOSOPHY */');
+    lines.push('');
+    config.philosophy.forEach(item => {
+      lines.push(`  "${item}"`);
+    });
+    lines.push('');
+  }
+
+  // Custom sections
+  if (config.customSections) {
+    Object.entries(config.customSections).forEach(([title, content]) => {
+      lines.push(`/* ${title.toUpperCase()} */`);
+      lines.push('');
+
+      // Indent custom content
+      const contentLines = content.trim().split('\n');
+      contentLines.forEach(line => {
+        lines.push(`  ${line.trim()}`);
+      });
+
+      lines.push('');
+    });
+  }
+
+  return lines.join('\n').trim() + '\n';
+}
--- a/src/generators/llms.ts
+++ b/src/generators/llms.ts
@ -0,0 +1,146 @@
+import type { LLMsConfig } from '../types.js';
+
+/**
+ * Generate llms.txt content
+ *
+ * This file provides context and instructions for AI assistants
+ * following the llms.txt specification.
+ *
+ * @param config - LLMs.txt configuration
+ * @param siteURL - Site base URL
+ * @returns Generated llms.txt content
+ */
+export async function generateLLMsTxt(
+  config: LLMsConfig,
+  siteURL: URL
+): Promise<string> {
+  const lines: string[] = [];
+
+  // Header with site name
+  const description = typeof config.description === 'function'
+    ? config.description()
+    : config.description;
+
+  lines.push(`# ${siteURL.hostname}`);
+  if (description) {
+    lines.push('');
+    lines.push(`> ${description}`);
+  }
+  lines.push('');
+  lines.push('---');
+  lines.push('');
+
+  // Site Information
+  lines.push('## Site Information');
+  lines.push('');
+  lines.push(`- **URL**: ${siteURL.href}`);
+  if (description) {
+    lines.push(`- **Description**: ${description}`);
+  }
+  lines.push('');
+
+  // Key Features
+  if (config.keyFeatures && config.keyFeatures.length > 0) {
+    lines.push('## Key Features');
+    lines.push('');
+    config.keyFeatures.forEach(feature => {
+      lines.push(`- ${feature}`);
+    });
+    lines.push('');
+  }
+
+  // Important Pages
+  if (config.importantPages) {
+    const pages = typeof config.importantPages === 'function'
+      ? await config.importantPages()
+      : config.importantPages;
+
+    if (pages.length > 0) {
+      lines.push('## Important Pages');
+      lines.push('');
+      pages.forEach(page => {
+        const url = new URL(page.path, siteURL).href;
+        lines.push(`- **[${page.name}](${url})**`);
+        if (page.description) {
+          lines.push(`  ${page.description}`);
+        }
+      });
+      lines.push('');
+    }
+  }
+
+  // Instructions for AI Assistants
+  if (config.instructions) {
+    lines.push('## Instructions for AI Assistants');
+    lines.push('');
+    lines.push(config.instructions.trim());
+    lines.push('');
+  }
+
+  // API Endpoints
+  if (config.apiEndpoints && config.apiEndpoints.length > 0) {
+    lines.push('## API Endpoints');
+    lines.push('');
+    config.apiEndpoints.forEach(endpoint => {
+      const method = endpoint.method || 'GET';
+      const fullUrl = new URL(endpoint.path, siteURL).href;
+      lines.push(`- \`${method} ${endpoint.path}\``);
+      lines.push(`  ${endpoint.description}`);
+      lines.push(`  Full URL: ${fullUrl}`);
+    });
+    lines.push('');
+  }
+
+  // Tech Stack
+  if (config.techStack) {
+    const hasAnyTech = Object.values(config.techStack).some(arr => arr && arr.length > 0);
+
+    if (hasAnyTech) {
+      lines.push('## Technical Stack');
+      lines.push('');
+
+      if (config.techStack.frontend && config.techStack.frontend.length > 0) {
+        lines.push(`- **Frontend**: ${config.techStack.frontend.join(', ')}`);
+      }
+      if (config.techStack.backend && config.techStack.backend.length > 0) {
+        lines.push(`- **Backend**: ${config.techStack.backend.join(', ')}`);
+      }
+      if (config.techStack.ai && config.techStack.ai.length > 0) {
+        lines.push(`- **AI/ML**: ${config.techStack.ai.join(', ')}`);
+      }
+      if (config.techStack.other && config.techStack.other.length > 0) {
+        lines.push(`- **Other**: ${config.techStack.other.join(', ')}`);
+      }
+      lines.push('');
+    }
+  }
+
+  // Brand Voice
+  if (config.brandVoice && config.brandVoice.length > 0) {
+    lines.push('## Brand Voice & Guidelines');
+    lines.push('');
+    config.brandVoice.forEach(item => {
+      lines.push(`- ${item}`);
+    });
+    lines.push('');
+  }
+
+  // Custom Sections
+  if (config.customSections) {
+    Object.entries(config.customSections).forEach(([title, content]) => {
+      lines.push(`## ${title}`);
+      lines.push('');
+      lines.push(content.trim());
+      lines.push('');
+    });
+  }
+
+  // Footer
+  lines.push('---');
+  lines.push('');
+  lines.push(`**Last Updated**: ${new Date().toISOString().split('T')[0]}`);
+  lines.push('');
+  lines.push('*This file was generated by [@astrojs/discovery](https://github.com/withastro/astro-discovery)*');
+
+  return lines.join('\n').trim() + '\n';
+}
--- a/src/generators/robots.ts
+++ b/src/generators/robots.ts
@ -0,0 +1,102 @@
+import type { RobotsConfig } from '../types.js';
+
+/**
+ * Default LLM bot user agents that should have access to llms.txt
+ */
+const DEFAULT_LLM_BOTS = [
+  'Anthropic-AI',
+  'Claude-Web',
+  'GPTBot',
+  'ChatGPT-User',
+  'cohere-ai',
+  'Google-Extended',
+  'PerplexityBot',
+  'Applebot-Extended',
+];
+
+/**
+ * Generate robots.txt content
+ *
+ * @param config - Robots.txt configuration
+ * @param siteURL - Site base URL
+ * @returns Generated robots.txt content
+ */
+export function generateRobotsTxt(
+  config: RobotsConfig,
+  siteURL: URL
+): string {
+  const lines: string[] = [];
+
+  // Header comment
+  lines.push('# robots.txt');
+  lines.push(`# Generated by @astrojs/discovery for ${siteURL.hostname}`);
+  lines.push('');
+
+  // Allow all bots by default
+  if (config.allowAllBots !== false) {
+    lines.push('User-agent: *');
+    lines.push('Allow: /');
+    lines.push('');
+  }
+
+  // Add sitemap reference
+  lines.push('# Sitemaps');
+  lines.push(`Sitemap: ${new URL('sitemap-index.xml', siteURL).href}`);
+  lines.push('');
+
+  // LLM-specific rules
+  if (config.llmBots?.enabled !== false) {
+    lines.push('# LLM-specific resources');
+    lines.push('# AI assistants can find additional context at /llms.txt');
+    lines.push('# See: https://github.com/anthropics/llm-txt');
+    lines.push('');
+
+    const agents = config.llmBots?.agents || DEFAULT_LLM_BOTS;
+    agents.forEach(agent => {
+      lines.push(`User-agent: ${agent}`);
+    });
+    lines.push('Allow: /llms.txt');
+    lines.push('Allow: /llms-full.txt');
+    lines.push('');
+  }
+
+  // Additional agent rules
+  if (config.additionalAgents && config.additionalAgents.length > 0) {
+    lines.push('# Custom agent rules');
+    lines.push('');
+
+    config.additionalAgents.forEach(agent => {
+      lines.push(`User-agent: ${agent.userAgent}`);
+
+      if (agent.allow && agent.allow.length > 0) {
+        agent.allow.forEach(path => {
+          lines.push(`Allow: ${path}`);
+        });
+      }
+
+      if (agent.disallow && agent.disallow.length > 0) {
+        agent.disallow.forEach(path => {
+          lines.push(`Disallow: ${path}`);
+        });
+      }
+
+      lines.push('');
+    });
+  }
+
+  // Crawl delay
+  if (config.crawlDelay) {
+    lines.push('# Crawl delay (be nice to our server)');
+    lines.push(`Crawl-delay: ${config.crawlDelay}`);
+    lines.push('');
+  }
+
+  // Custom rules
+  if (config.customRules) {
+    lines.push('# Custom rules');
+    lines.push(config.customRules.trim());
+    lines.push('');
+  }
+
+  return lines.join('\n').trim() + '\n';
+}