feat: initial implementation of @astrojs/discovery integration

This commit introduces a comprehensive Astro integration that automatically
generates discovery files for websites:

Features:
- robots.txt with LLM bot support (Anthropic-AI, GPTBot, etc.)
- llms.txt for AI assistant context and instructions
- humans.txt for team credits and site information
- Automatic sitemap integration via @astrojs/sitemap

Technical Details:
- TypeScript implementation with full type safety
- Configurable HTTP caching headers
- Custom template support for all generated files
- Sensible defaults with extensive customization options
- Date-based versioning (2025.11.03)

Testing:
- 34 unit tests covering all generators
- Test coverage for robots.txt, llms.txt, and humans.txt
- Integration with Vitest

Documentation:
- Comprehensive README with examples
- API reference documentation
- Contributing guidelines
- Example configurations (minimal and full)

src/config-store.ts

@@ -0,0 +1,15 @@
+import type { DiscoveryConfig } from './types.js';
+
+/**
+ * Shared configuration store
+ * This allows the integration to pass config to route handlers
+ */
+let globalConfig: DiscoveryConfig = {};
+
+export function setConfig(config: DiscoveryConfig): void {
+  globalConfig = config;
+}
+
+export function getConfig(): DiscoveryConfig {
+  return globalConfig;
+}

src/generators/humans.ts

@@ -0,0 +1,145 @@
+import type { HumansConfig } from '../types.js';
+
+/**
+ * Generate humans.txt content
+ *
+ * This file provides human-readable credits and information about
+ * the site, team, and technology stack.
+ *
+ * @param config - Humans.txt configuration
+ * @returns Generated humans.txt content
+ */
+export function generateHumansTxt(config: HumansConfig): string {
+  const lines: string[] = [];
+
+  // Team section
+  if (config.team && config.team.length > 0) {
+    lines.push('/* TEAM */');
+    lines.push('');
+
+    config.team.forEach((member, index) => {
+      if (index > 0) {
+        lines.push('');
+      }
+
+      lines.push(`  Name: ${member.name}`);
+      if (member.role) {
+        lines.push(`  Role: ${member.role}`);
+      }
+      if (member.contact) {
+        lines.push(`  Contact: ${member.contact}`);
+      }
+      if (member.location) {
+        lines.push(`  From: ${member.location}`);
+      }
+      if (member.twitter) {
+        lines.push(`  Twitter: ${member.twitter}`);
+      }
+      if (member.github) {
+        lines.push(`  GitHub: ${member.github}`);
+      }
+    });
+
+    lines.push('');
+  }
+
+  // Thanks section
+  if (config.thanks && config.thanks.length > 0) {
+    lines.push('/* THANKS */');
+    lines.push('');
+    config.thanks.forEach(thanks => {
+      lines.push(`  ${thanks}`);
+    });
+    lines.push('');
+  }
+
+  // Site section
+  if (config.site) {
+    lines.push('/* SITE */');
+    lines.push('');
+
+    const lastUpdate = config.site.lastUpdate === 'auto'
+      ? new Date().toISOString().split('T')[0]
+      : config.site.lastUpdate;
+
+    if (lastUpdate) {
+      lines.push(`  Last update: ${lastUpdate}`);
+    }
+    if (config.site.language) {
+      lines.push(`  Language: ${config.site.language}`);
+    }
+    if (config.site.doctype) {
+      lines.push(`  Doctype: ${config.site.doctype}`);
+    }
+    if (config.site.ide) {
+      lines.push(`  IDE: ${config.site.ide}`);
+    }
+
+    if (config.site.techStack && config.site.techStack.length > 0) {
+      lines.push(`  Tech Stack: ${config.site.techStack.join(', ')}`);
+    }
+    if (config.site.standards && config.site.standards.length > 0) {
+      lines.push(`  Standards: ${config.site.standards.join(', ')}`);
+    }
+    if (config.site.components && config.site.components.length > 0) {
+      lines.push(`  Components: ${config.site.components.join(', ')}`);
+    }
+    if (config.site.software && config.site.software.length > 0) {
+      lines.push(`  Software: ${config.site.software.join(', ')}`);
+    }
+
+    lines.push('');
+  }
+
+  // Story section
+  if (config.story) {
+    lines.push('/* THE STORY */');
+    lines.push('');
+
+    // Indent multi-line stories
+    const storyLines = config.story.trim().split('\n');
+    storyLines.forEach(line => {
+      lines.push(`  ${line.trim()}`);
+    });
+
+    lines.push('');
+  }
+
+  // Fun Facts section
+  if (config.funFacts && config.funFacts.length > 0) {
+    lines.push('/* FUN FACTS */');
+    lines.push('');
+    config.funFacts.forEach(fact => {
+      lines.push(`  ${fact}`);
+    });
+    lines.push('');
+  }
+
+  // Philosophy section
+  if (config.philosophy && config.philosophy.length > 0) {
+    lines.push('/* PHILOSOPHY */');
+    lines.push('');
+    config.philosophy.forEach(item => {
+      lines.push(`  "${item}"`);
+    });
+    lines.push('');
+  }
+
+  // Custom sections
+  if (config.customSections) {
+    Object.entries(config.customSections).forEach(([title, content]) => {
+      lines.push(`/* ${title.toUpperCase()} */`);
+      lines.push('');
+
+      // Indent custom content
+      const contentLines = content.trim().split('\n');
+      contentLines.forEach(line => {
+        lines.push(`  ${line.trim()}`);
+      });
+
+      lines.push('');
+    });
+  }
+
+  return lines.join('\n').trim() + '\n';
+}

src/generators/llms.ts

@@ -0,0 +1,146 @@
+import type { LLMsConfig } from '../types.js';
+
+/**
+ * Generate llms.txt content
+ *
+ * This file provides context and instructions for AI assistants
+ * following the llms.txt specification.
+ *
+ * @param config - LLMs.txt configuration
+ * @param siteURL - Site base URL
+ * @returns Generated llms.txt content
+ */
+export async function generateLLMsTxt(
+  config: LLMsConfig,
+  siteURL: URL
+): Promise<string> {
+  const lines: string[] = [];
+
+  // Header with site name
+  const description = typeof config.description === 'function'
+    ? config.description()
+    : config.description;
+
+  lines.push(`# ${siteURL.hostname}`);
+  if (description) {
+    lines.push('');
+    lines.push(`> ${description}`);
+  }
+  lines.push('');
+  lines.push('---');
+  lines.push('');
+
+  // Site Information
+  lines.push('## Site Information');
+  lines.push('');
+  lines.push(`- **URL**: ${siteURL.href}`);
+  if (description) {
+    lines.push(`- **Description**: ${description}`);
+  }
+  lines.push('');
+
+  // Key Features
+  if (config.keyFeatures && config.keyFeatures.length > 0) {
+    lines.push('## Key Features');
+    lines.push('');
+    config.keyFeatures.forEach(feature => {
+      lines.push(`- ${feature}`);
+    });
+    lines.push('');
+  }
+
+  // Important Pages
+  if (config.importantPages) {
+    const pages = typeof config.importantPages === 'function'
+      ? await config.importantPages()
+      : config.importantPages;
+
+    if (pages.length > 0) {
+      lines.push('## Important Pages');
+      lines.push('');
+      pages.forEach(page => {
+        const url = new URL(page.path, siteURL).href;
+        lines.push(`- **[${page.name}](${url})**`);
+        if (page.description) {
+          lines.push(`  ${page.description}`);
+        }
+      });
+      lines.push('');
+    }
+  }
+
+  // Instructions for AI Assistants
+  if (config.instructions) {
+    lines.push('## Instructions for AI Assistants');
+    lines.push('');
+    lines.push(config.instructions.trim());
+    lines.push('');
+  }
+
+  // API Endpoints
+  if (config.apiEndpoints && config.apiEndpoints.length > 0) {
+    lines.push('## API Endpoints');
+    lines.push('');
+    config.apiEndpoints.forEach(endpoint => {
+      const method = endpoint.method || 'GET';
+      const fullUrl = new URL(endpoint.path, siteURL).href;
+      lines.push(`- \`${method} ${endpoint.path}\``);
+      lines.push(`  ${endpoint.description}`);
+      lines.push(`  Full URL: ${fullUrl}`);
+    });
+    lines.push('');
+  }
+
+  // Tech Stack
+  if (config.techStack) {
+    const hasAnyTech = Object.values(config.techStack).some(arr => arr && arr.length > 0);
+
+    if (hasAnyTech) {
+      lines.push('## Technical Stack');
+      lines.push('');
+
+      if (config.techStack.frontend && config.techStack.frontend.length > 0) {
+        lines.push(`- **Frontend**: ${config.techStack.frontend.join(', ')}`);
+      }
+      if (config.techStack.backend && config.techStack.backend.length > 0) {
+        lines.push(`- **Backend**: ${config.techStack.backend.join(', ')}`);
+      }
+      if (config.techStack.ai && config.techStack.ai.length > 0) {
+        lines.push(`- **AI/ML**: ${config.techStack.ai.join(', ')}`);
+      }
+      if (config.techStack.other && config.techStack.other.length > 0) {
+        lines.push(`- **Other**: ${config.techStack.other.join(', ')}`);
+      }
+      lines.push('');
+    }
+  }
+
+  // Brand Voice
+  if (config.brandVoice && config.brandVoice.length > 0) {
+    lines.push('## Brand Voice & Guidelines');
+    lines.push('');
+    config.brandVoice.forEach(item => {
+      lines.push(`- ${item}`);
+    });
+    lines.push('');
+  }
+
+  // Custom Sections
+  if (config.customSections) {
+    Object.entries(config.customSections).forEach(([title, content]) => {
+      lines.push(`## ${title}`);
+      lines.push('');
+      lines.push(content.trim());
+      lines.push('');
+    });
+  }
+
+  // Footer
+  lines.push('---');
+  lines.push('');
+  lines.push(`**Last Updated**: ${new Date().toISOString().split('T')[0]}`);
+  lines.push('');
+  lines.push('*This file was generated by [@astrojs/discovery](https://github.com/withastro/astro-discovery)*');
+
+  return lines.join('\n').trim() + '\n';
+}

src/generators/robots.ts

@@ -0,0 +1,102 @@
+import type { RobotsConfig } from '../types.js';
+
+/**
+ * Default LLM bot user agents that should have access to llms.txt
+ */
+const DEFAULT_LLM_BOTS = [
+  'Anthropic-AI',
+  'Claude-Web',
+  'GPTBot',
+  'ChatGPT-User',
+  'cohere-ai',
+  'Google-Extended',
+  'PerplexityBot',
+  'Applebot-Extended',
+];
+
+/**
+ * Generate robots.txt content
+ *
+ * @param config - Robots.txt configuration
+ * @param siteURL - Site base URL
+ * @returns Generated robots.txt content
+ */
+export function generateRobotsTxt(
+  config: RobotsConfig,
+  siteURL: URL
+): string {
+  const lines: string[] = [];
+
+  // Header comment
+  lines.push('# robots.txt');
+  lines.push(`# Generated by @astrojs/discovery for ${siteURL.hostname}`);
+  lines.push('');
+
+  // Allow all bots by default
+  if (config.allowAllBots !== false) {
+    lines.push('User-agent: *');
+    lines.push('Allow: /');
+    lines.push('');
+  }
+
+  // Add sitemap reference
+  lines.push('# Sitemaps');
+  lines.push(`Sitemap: ${new URL('sitemap-index.xml', siteURL).href}`);
+  lines.push('');
+
+  // LLM-specific rules
+  if (config.llmBots?.enabled !== false) {
+    lines.push('# LLM-specific resources');
+    lines.push('# AI assistants can find additional context at /llms.txt');
+    lines.push('# See: https://github.com/anthropics/llm-txt');
+    lines.push('');
+
+    const agents = config.llmBots?.agents || DEFAULT_LLM_BOTS;
+    agents.forEach(agent => {
+      lines.push(`User-agent: ${agent}`);
+    });
+    lines.push('Allow: /llms.txt');
+    lines.push('Allow: /llms-full.txt');
+    lines.push('');
+  }
+
+  // Additional agent rules
+  if (config.additionalAgents && config.additionalAgents.length > 0) {
+    lines.push('# Custom agent rules');
+    lines.push('');
+
+    config.additionalAgents.forEach(agent => {
+      lines.push(`User-agent: ${agent.userAgent}`);
+
+      if (agent.allow && agent.allow.length > 0) {
+        agent.allow.forEach(path => {
+          lines.push(`Allow: ${path}`);
+        });
+      }
+
+      if (agent.disallow && agent.disallow.length > 0) {
+        agent.disallow.forEach(path => {
+          lines.push(`Disallow: ${path}`);
+        });
+      }
+
+      lines.push('');
+    });
+  }
+
+  // Crawl delay
+  if (config.crawlDelay) {
+    lines.push('# Crawl delay (be nice to our server)');
+    lines.push(`Crawl-delay: ${config.crawlDelay}`);
+    lines.push('');
+  }
+
+  // Custom rules
+  if (config.customRules) {
+    lines.push('# Custom rules');
+    lines.push(config.customRules.trim());
+    lines.push('');
+  }
+
+  return lines.join('\n').trim() + '\n';
+}

src/index.ts

@@ -0,0 +1,127 @@
+import type { AstroIntegration } from 'astro';
+import type { DiscoveryConfig } from './types.js';
+import sitemap from '@astrojs/sitemap';
+import { validateConfig } from './validators/config.js';
+import { setConfig } from './config-store.js';
+
+/**
+ * Astro Discovery Integration
+ *
+ * Automatically generates discovery files for your Astro site:
+ * - /robots.txt - Search engine and bot instructions
+ * - /llms.txt - AI assistant context and guidelines
+ * - /humans.txt - Human-readable credits and information
+ * - /sitemap-index.xml - Site structure for search engines
+ *
+ * @param userConfig - Optional configuration
+ * @returns Astro integration
+ *
+ * @example
+ * ```ts
+ * // astro.config.mjs
+ * import discovery from '@astrojs/discovery';
+ *
+ * export default defineConfig({
+ *   site: 'https://example.com',
+ *   integrations: [
+ *     discovery({
+ *       llms: {
+ *         description: 'My awesome site',
+ *         instructions: 'Be helpful and accurate'
+ *       }
+ *     })
+ *   ]
+ * });
+ * ```
+ */
+export default function discovery(
+  userConfig: DiscoveryConfig = {}
+): AstroIntegration {
+  // Merge with defaults and validate
+  const config = validateConfig(userConfig);
+
+  // Store config globally for route handlers to access
+  setConfig(config);
+
+  return {
+    name: '@astrojs/discovery',
+    hooks: {
+      'astro:config:setup': ({ config: astroConfig, injectRoute, updateConfig }) => {
+        // Ensure site is configured
+        if (!astroConfig.site) {
+          throw new Error(
+            '[@astrojs/discovery] The `site` option must be set in your Astro config.\n' +
+            'Example: site: "https://example.com"'
+          );
+        }
+
+        // Add sitemap integration
+        updateConfig({
+          integrations: [
+            sitemap(config.sitemap || {})
+          ]
+        });
+
+        // Inject dynamic routes for discovery files
+        if (config.robots?.enabled !== false) {
+          injectRoute({
+            pattern: '/robots.txt',
+            entrypoint: '@astrojs/discovery/routes/robots',
+            prerender: true
+          });
+        }
+
+        if (config.llms?.enabled !== false) {
+          injectRoute({
+            pattern: '/llms.txt',
+            entrypoint: '@astrojs/discovery/routes/llms',
+            prerender: true
+          });
+        }
+
+        if (config.humans?.enabled !== false) {
+          injectRoute({
+            pattern: '/humans.txt',
+            entrypoint: '@astrojs/discovery/routes/humans',
+            prerender: true
+          });
+        }
+      },
+
+      'astro:build:done': () => {
+        // Post-build notification
+        console.log('\n✨ @astrojs/discovery - Generated files:');
+
+        if (config.robots?.enabled !== false) {
+          console.log('  ✅ /robots.txt');
+        }
+        if (config.llms?.enabled !== false) {
+          console.log('  ✅ /llms.txt');
+        }
+        if (config.humans?.enabled !== false) {
+          console.log('  ✅ /humans.txt');
+        }
+
+        console.log('  ✅ /sitemap-index.xml');
+        console.log('');
+      }
+    }
+  };
+}
+
+// Named exports
+export type {
+  DiscoveryConfig,
+  RobotsConfig,
+  LLMsConfig,
+  HumansConfig,
+  SitemapConfig,
+  CachingConfig,
+  TemplateConfig,
+  ImportantPage,
+  APIEndpoint,
+  TechStack,
+  TeamMember,
+  SiteInfo,
+  SitemapItem,
+} from './types.js';

src/routes/humans.ts

@@ -0,0 +1,30 @@
+import type { APIRoute } from 'astro';
+import { generateHumansTxt } from '../generators/humans.js';
+import { getConfig } from '../config-store.js';
+
+/**
+ * API route for /humans.txt
+ */
+export const GET: APIRoute = ({ site }) => {
+  const config = getConfig();
+  const humansConfig = config.humans || {};
+  const siteURL = site || new URL('http://localhost:4321');
+
+  // Use custom template if provided
+  const content = config.templates?.humans
+    ? config.templates.humans(humansConfig, siteURL)
+    : generateHumansTxt(humansConfig);
+
+  // Get cache duration (default: 24 hours)
+  const cacheSeconds = config.caching?.humans ?? 86400;
+
+  return new Response(content, {
+    status: 200,
+    headers: {
+      'Content-Type': 'text/plain; charset=utf-8',
+      'Cache-Control': `public, max-age=${cacheSeconds}`,
+    },
+  });
+};
+
+export const prerender = true;

src/routes/llms.ts

@@ -0,0 +1,30 @@
+import type { APIRoute } from 'astro';
+import { generateLLMsTxt } from '../generators/llms.js';
+import { getConfig } from '../config-store.js';
+
+/**
+ * API route for /llms.txt
+ */
+export const GET: APIRoute = async ({ site }) => {
+  const config = getConfig();
+  const llmsConfig = config.llms || {};
+  const siteURL = site || new URL('http://localhost:4321');
+
+  // Use custom template if provided
+  const content = config.templates?.llms
+    ? await config.templates.llms(llmsConfig, siteURL)
+    : await generateLLMsTxt(llmsConfig, siteURL);
+
+  // Get cache duration (default: 1 hour)
+  const cacheSeconds = config.caching?.llms ?? 3600;
+
+  return new Response(content, {
+    status: 200,
+    headers: {
+      'Content-Type': 'text/plain; charset=utf-8',
+      'Cache-Control': `public, max-age=${cacheSeconds}`,
+    },
+  });
+};
+
+export const prerender = true;

src/routes/robots.ts

@@ -0,0 +1,30 @@
+import type { APIRoute } from 'astro';
+import { generateRobotsTxt } from '../generators/robots.js';
+import { getConfig } from '../config-store.js';
+
+/**
+ * API route for /robots.txt
+ */
+export const GET: APIRoute = ({ site }) => {
+  const config = getConfig();
+  const robotsConfig = config.robots || {};
+  const siteURL = site || new URL('http://localhost:4321');
+
+  // Use custom template if provided
+  const content = config.templates?.robots
+    ? config.templates.robots(robotsConfig, siteURL)
+    : generateRobotsTxt(robotsConfig, siteURL);
+
+  // Get cache duration (default: 1 hour)
+  const cacheSeconds = config.caching?.robots ?? 3600;
+
+  return new Response(content, {
+    status: 200,
+    headers: {
+      'Content-Type': 'text/plain; charset=utf-8',
+      'Cache-Control': `public, max-age=${cacheSeconds}`,
+    },
+  });
+};
+
+export const prerender = true;

src/types.ts

@@ -0,0 +1,235 @@
+/**
+ * Main configuration interface for the Astro Discovery integration
+ */
+export interface DiscoveryConfig {
+  /** Configuration for robots.txt generation */
+  robots?: RobotsConfig;
+  /** Configuration for llms.txt generation */
+  llms?: LLMsConfig;
+  /** Configuration for humans.txt generation */
+  humans?: HumansConfig;
+  /** Configuration passed to @astrojs/sitemap */
+  sitemap?: SitemapConfig;
+  /** HTTP cache control configuration */
+  caching?: CachingConfig;
+  /** Custom template functions */
+  templates?: TemplateConfig;
+}
+
+/**
+ * Configuration for robots.txt generation
+ */
+export interface RobotsConfig {
+  /** Enable/disable robots.txt generation (default: true) */
+  enabled?: boolean;
+  /** Crawl delay in seconds for polite crawlers */
+  crawlDelay?: number;
+  /** Allow all bots by default (default: true) */
+  allowAllBots?: boolean;
+  /** LLM-specific bot configuration */
+  llmBots?: {
+    /** Enable LLM bot rules (default: true) */
+    enabled?: boolean;
+    /** Custom LLM bot user agents */
+    agents?: string[];
+  };
+  /** Additional custom agent rules */
+  additionalAgents?: Array<{
+    /** User agent string */
+    userAgent: string;
+    /** Paths to allow */
+    allow?: string[];
+    /** Paths to disallow */
+    disallow?: string[];
+  }>;
+  /** Custom raw robots.txt content to append */
+  customRules?: string;
+}
+
+/**
+ * Configuration for llms.txt generation
+ */
+export interface LLMsConfig {
+  /** Enable/disable llms.txt generation (default: true) */
+  enabled?: boolean;
+  /** Site description for AI assistants (can be dynamic) */
+  description?: string | (() => string);
+  /** Key features of the site */
+  keyFeatures?: string[];
+  /** Important pages for AI to know about */
+  importantPages?: ImportantPage[] | (() => Promise<ImportantPage[]>);
+  /** Instructions for AI assistants */
+  instructions?: string;
+  /** API endpoints available */
+  apiEndpoints?: APIEndpoint[];
+  /** Technology stack information */
+  techStack?: TechStack;
+  /** Brand voice guidelines */
+  brandVoice?: string[];
+  /** Custom sections to add */
+  customSections?: Record<string, string>;
+}
+
+/**
+ * Configuration for humans.txt generation
+ */
+export interface HumansConfig {
+  /** Enable/disable humans.txt generation (default: true) */
+  enabled?: boolean;
+  /** Team members */
+  team?: TeamMember[];
+  /** Thank you notes */
+  thanks?: string[];
+  /** Site information */
+  site?: SiteInfo;
+  /** Project story/history */
+  story?: string;
+  /** Fun facts about the project */
+  funFacts?: string[];
+  /** Development philosophy */
+  philosophy?: string[];
+  /** Custom sections to add */
+  customSections?: Record<string, string>;
+}
+
+/**
+ * Configuration for sitemap generation (passed to @astrojs/sitemap)
+ * This is a simplified type - actual options are passed through to @astrojs/sitemap
+ */
+export interface SitemapConfig {
+  /** Filter function to exclude pages */
+  filter?: (page: string) => boolean;
+  /** Custom pages to include */
+  customPages?: string[];
+  /** Change frequency hint */
+  changefreq?: 'always' | 'hourly' | 'daily' | 'weekly' | 'monthly' | 'yearly' | 'never';
+  /** Priority hint (0.0 - 1.0) */
+  priority?: number;
+  /** Internationalization configuration */
+  i18n?: {
+    defaultLocale: string;
+    locales: Record<string, string>;
+  };
+  /** Last modification date */
+  lastmod?: Date;
+  /** Allow any other sitemap options from @astrojs/sitemap */
+  [key: string]: any;
+}
+
+/**
+ * HTTP caching configuration (in seconds)
+ */
+export interface CachingConfig {
+  /** Cache duration for robots.txt (default: 3600) */
+  robots?: number;
+  /** Cache duration for llms.txt (default: 3600) */
+  llms?: number;
+  /** Cache duration for humans.txt (default: 86400) */
+  humans?: number;
+  /** Cache duration for sitemap (default: 3600) */
+  sitemap?: number;
+}
+
+/**
+ * Custom template functions
+ */
+export interface TemplateConfig {
+  /** Custom robots.txt template */
+  robots?: (config: RobotsConfig, siteURL: URL) => string;
+  /** Custom llms.txt template */
+  llms?: (config: LLMsConfig, siteURL: URL) => string | Promise<string>;
+  /** Custom humans.txt template */
+  humans?: (config: HumansConfig, siteURL: URL) => string;
+}
+
+/**
+ * Important page definition for llms.txt
+ */
+export interface ImportantPage {
+  /** Page name/title */
+  name: string;
+  /** Path relative to site root */
+  path: string;
+  /** Optional description */
+  description?: string;
+}
+
+/**
+ * API endpoint definition for llms.txt
+ */
+export interface APIEndpoint {
+  /** Endpoint path */
+  path: string;
+  /** HTTP method (default: GET) */
+  method?: string;
+  /** Endpoint description */
+  description: string;
+}
+
+/**
+ * Technology stack information for llms.txt
+ */
+export interface TechStack {
+  /** Frontend technologies */
+  frontend?: string[];
+  /** Backend technologies */
+  backend?: string[];
+  /** AI/ML technologies */
+  ai?: string[];
+  /** Other technologies */
+  other?: string[];
+}
+
+/**
+ * Team member definition for humans.txt
+ */
+export interface TeamMember {
+  /** Full name */
+  name: string;
+  /** Role/title */
+  role?: string;
+  /** Contact email */
+  contact?: string;
+  /** Location */
+  location?: string;
+  /** Twitter handle */
+  twitter?: string;
+  /** GitHub username */
+  github?: string;
+}
+
+/**
+ * Site information for humans.txt
+ */
+export interface SiteInfo {
+  /** Last update date ('auto' for current date) */
+  lastUpdate?: string | 'auto';
+  /** Primary language */
+  language?: string;
+  /** Document type */
+  doctype?: string;
+  /** IDE/editor used */
+  ide?: string;
+  /** Technology stack */
+  techStack?: string[];
+  /** Web standards followed */
+  standards?: string[];
+  /** Components/libraries used */
+  components?: string[];
+  /** Software/tools used */
+  software?: string[];
+}
+
+/**
+ * Sitemap item interface (from @astrojs/sitemap)
+ */
+export interface SitemapItem {
+  url: string;
+  lastmod?: Date;
+  changefreq?: 'always' | 'hourly' | 'daily' | 'weekly' | 'monthly' | 'yearly' | 'never';
+  priority?: number;
+  links?: Array<{
+    url: string;
+    lang: string;
+  }>;
+}

src/validators/config.ts

@@ -0,0 +1,77 @@
+import type { DiscoveryConfig } from '../types.js';
+
+/**
+ * Default configuration values
+ */
+const DEFAULT_CONFIG: Required<Omit<DiscoveryConfig, 'templates'>> & { templates?: DiscoveryConfig['templates'] } = {
+  robots: {
+    enabled: true,
+    crawlDelay: 1,
+    allowAllBots: true,
+    llmBots: {
+      enabled: true,
+    },
+  },
+  llms: {
+    enabled: true,
+  },
+  humans: {
+    enabled: true,
+  },
+  sitemap: {},
+  caching: {
+    robots: 3600,    // 1 hour
+    llms: 3600,      // 1 hour
+    humans: 86400,   // 24 hours
+    sitemap: 3600,   // 1 hour
+  },
+};
+
+/**
+ * Validate and merge user configuration with defaults
+ *
+ * @param userConfig - User-provided configuration
+ * @returns Validated and merged configuration
+ */
+export function validateConfig(userConfig: DiscoveryConfig = {}): DiscoveryConfig {
+  const config: DiscoveryConfig = {
+    robots: {
+      ...DEFAULT_CONFIG.robots,
+      ...userConfig.robots,
+      llmBots: {
+        ...DEFAULT_CONFIG.robots.llmBots,
+        ...userConfig.robots?.llmBots,
+      },
+    },
+    llms: {
+      ...DEFAULT_CONFIG.llms,
+      ...userConfig.llms,
+    },
+    humans: {
+      ...DEFAULT_CONFIG.humans,
+      ...userConfig.humans,
+    },
+    sitemap: {
+      ...DEFAULT_CONFIG.sitemap,
+      ...userConfig.sitemap,
+    },
+    caching: {
+      ...DEFAULT_CONFIG.caching,
+      ...userConfig.caching,
+    },
+    templates: userConfig.templates,
+  };
+
+  // Validation warnings (non-breaking)
+  if (config.caching) {
+    Object.entries(config.caching).forEach(([key, value]) => {
+      if (value !== undefined && (value < 0 || value > 31536000)) {
+        console.warn(
+          `@astrojs/discovery: Cache duration for "${key}" should be between 0 and 31536000 seconds (1 year). Got: ${value}`
+        );
+      }
+    });
+  }
+
+  return config;
+}