feat: initial implementation of @astrojs/discovery integration
This commit introduces a comprehensive Astro integration that automatically generates discovery files for websites: Features: - robots.txt with LLM bot support (Anthropic-AI, GPTBot, etc.) - llms.txt for AI assistant context and instructions - humans.txt for team credits and site information - Automatic sitemap integration via @astrojs/sitemap Technical Details: - TypeScript implementation with full type safety - Configurable HTTP caching headers - Custom template support for all generated files - Sensible defaults with extensive customization options - Date-based versioning (2025.11.03) Testing: - 34 unit tests covering all generators - Test coverage for robots.txt, llms.txt, and humans.txt - Integration with Vitest Documentation: - Comprehensive README with examples - API reference documentation - Contributing guidelines - Example configurations (minimal and full)
This commit is contained in:
commit
d25dde4627
25 changed files with 11001 additions and 0 deletions
699
astro-discovery-implementation.md
Normal file
699
astro-discovery-implementation.md
Normal file
|
|
@ -0,0 +1,699 @@
|
|||
# @astrojs/discovery - Implementation Guide
|
||||
|
||||
> Technical implementation details for building the Astro discovery integration
|
||||
|
||||
## Package Structure
|
||||
|
||||
```
|
||||
@astrojs/discovery/
|
||||
├── package.json
|
||||
├── README.md
|
||||
├── LICENSE
|
||||
├── tsconfig.json
|
||||
├── src/
|
||||
│ ├── index.ts # Main entry point
|
||||
│ ├── types.ts # TypeScript definitions
|
||||
│ ├── generators/
|
||||
│ │ ├── robots.ts # robots.txt generation
|
||||
│ │ ├── llms.ts # llms.txt generation
|
||||
│ │ ├── humans.ts # humans.txt generation
|
||||
│ │ └── utils.ts # Shared utilities
|
||||
│ ├── templates/
|
||||
│ │ ├── robots.template.ts
|
||||
│ │ ├── llms.template.ts
|
||||
│ │ └── humans.template.ts
|
||||
│ └── validators/
|
||||
│ └── config.ts # Config validation
|
||||
├── dist/ # Built output
|
||||
└── tests/
|
||||
├── robots.test.ts
|
||||
├── llms.test.ts
|
||||
├── humans.test.ts
|
||||
└── integration.test.ts
|
||||
```
|
||||
|
||||
## Core Implementation
|
||||
|
||||
### 1. Main Integration File (`src/index.ts`)
|
||||
|
||||
```typescript
|
||||
import type { AstroIntegration } from 'astro';
|
||||
import type { DiscoveryConfig } from './types';
|
||||
import sitemap from '@astrojs/sitemap';
|
||||
import { generateRobotsTxt } from './generators/robots';
|
||||
import { generateLLMsTxt } from './generators/llms';
|
||||
import { generateHumansTxt } from './generators/humans';
|
||||
import { validateConfig } from './validators/config';
|
||||
|
||||
export default function discovery(
|
||||
userConfig: DiscoveryConfig = {}
|
||||
): AstroIntegration {
|
||||
// Merge with defaults
|
||||
const config = validateConfig(userConfig);
|
||||
|
||||
return {
|
||||
name: '@astrojs/discovery',
|
||||
hooks: {
|
||||
'astro:config:setup': ({ config: astroConfig, injectRoute, updateConfig }) => {
|
||||
// Ensure site is configured
|
||||
if (!astroConfig.site) {
|
||||
throw new Error(
|
||||
'@astrojs/discovery requires `site` to be set in astro.config.mjs'
|
||||
);
|
||||
}
|
||||
|
||||
// Add sitemap integration
|
||||
updateConfig({
|
||||
integrations: [
|
||||
sitemap(config.sitemap || {})
|
||||
]
|
||||
});
|
||||
|
||||
// Inject dynamic routes for discovery files
|
||||
if (config.robots?.enabled !== false) {
|
||||
injectRoute({
|
||||
pattern: '/robots.txt',
|
||||
entrypoint: '@astrojs/discovery/routes/robots.ts',
|
||||
prerender: true
|
||||
});
|
||||
}
|
||||
|
||||
if (config.llms?.enabled !== false) {
|
||||
injectRoute({
|
||||
pattern: '/llms.txt',
|
||||
entrypoint: '@astrojs/discovery/routes/llms.ts',
|
||||
prerender: true
|
||||
});
|
||||
}
|
||||
|
||||
if (config.humans?.enabled !== false) {
|
||||
injectRoute({
|
||||
pattern: '/humans.txt',
|
||||
entrypoint: '@astrojs/discovery/routes/humans.ts',
|
||||
prerender: true
|
||||
});
|
||||
}
|
||||
},
|
||||
|
||||
'astro:build:done': ({ dir, routes }) => {
|
||||
// Post-build validation
|
||||
console.log('✅ Discovery files generated:');
|
||||
if (config.robots?.enabled !== false) console.log(' - /robots.txt');
|
||||
if (config.llms?.enabled !== false) console.log(' - /llms.txt');
|
||||
if (config.humans?.enabled !== false) console.log(' - /humans.txt');
|
||||
console.log(' - /sitemap-index.xml');
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
// Named exports
|
||||
export type { DiscoveryConfig } from './types';
|
||||
```
|
||||
|
||||
### 2. Type Definitions (`src/types.ts`)
|
||||
|
||||
```typescript
|
||||
export interface DiscoveryConfig {
|
||||
robots?: RobotsConfig;
|
||||
llms?: LLMsConfig;
|
||||
humans?: HumansConfig;
|
||||
sitemap?: SitemapConfig;
|
||||
caching?: CachingConfig;
|
||||
templates?: TemplateConfig;
|
||||
}
|
||||
|
||||
export interface RobotsConfig {
|
||||
enabled?: boolean;
|
||||
crawlDelay?: number;
|
||||
allowAllBots?: boolean;
|
||||
llmBots?: {
|
||||
enabled?: boolean;
|
||||
agents?: string[];
|
||||
};
|
||||
additionalAgents?: Array<{
|
||||
userAgent: string;
|
||||
allow?: string[];
|
||||
disallow?: string[];
|
||||
}>;
|
||||
customRules?: string;
|
||||
}
|
||||
|
||||
export interface LLMsConfig {
|
||||
enabled?: boolean;
|
||||
description?: string | (() => string);
|
||||
keyFeatures?: string[];
|
||||
importantPages?: ImportantPage[] | (() => Promise<ImportantPage[]>);
|
||||
instructions?: string;
|
||||
apiEndpoints?: APIEndpoint[];
|
||||
techStack?: TechStack;
|
||||
brandVoice?: string[];
|
||||
customSections?: Record<string, string>;
|
||||
}
|
||||
|
||||
export interface HumansConfig {
|
||||
enabled?: boolean;
|
||||
team?: TeamMember[];
|
||||
thanks?: string[];
|
||||
site?: SiteInfo;
|
||||
story?: string;
|
||||
funFacts?: string[];
|
||||
philosophy?: string[];
|
||||
customSections?: Record<string, string>;
|
||||
}
|
||||
|
||||
export interface SitemapConfig {
|
||||
filter?: (page: string) => boolean;
|
||||
customPages?: string[];
|
||||
changefreq?: 'always' | 'hourly' | 'daily' | 'weekly' | 'monthly' | 'yearly' | 'never';
|
||||
priority?: number;
|
||||
}
|
||||
|
||||
export interface CachingConfig {
|
||||
robots?: number;
|
||||
llms?: number;
|
||||
humans?: number;
|
||||
sitemap?: number;
|
||||
}
|
||||
|
||||
export interface TemplateConfig {
|
||||
robots?: (config: RobotsConfig, siteURL: URL) => string;
|
||||
llms?: (config: LLMsConfig, siteURL: URL) => string;
|
||||
humans?: (config: HumansConfig, siteURL: URL) => string;
|
||||
}
|
||||
|
||||
export interface ImportantPage {
|
||||
name: string;
|
||||
path: string;
|
||||
description?: string;
|
||||
}
|
||||
|
||||
export interface APIEndpoint {
|
||||
path: string;
|
||||
method?: string;
|
||||
description: string;
|
||||
}
|
||||
|
||||
export interface TechStack {
|
||||
frontend?: string[];
|
||||
backend?: string[];
|
||||
ai?: string[];
|
||||
other?: string[];
|
||||
}
|
||||
|
||||
export interface TeamMember {
|
||||
name: string;
|
||||
role?: string;
|
||||
contact?: string;
|
||||
location?: string;
|
||||
twitter?: string;
|
||||
github?: string;
|
||||
}
|
||||
|
||||
export interface SiteInfo {
|
||||
lastUpdate?: string | 'auto';
|
||||
language?: string;
|
||||
doctype?: string;
|
||||
ide?: string;
|
||||
techStack?: string[];
|
||||
standards?: string[];
|
||||
components?: string[];
|
||||
software?: string[];
|
||||
}
|
||||
```
|
||||
|
||||
### 3. Robots.txt Generator (`src/generators/robots.ts`)
|
||||
|
||||
```typescript
|
||||
import type { RobotsConfig } from '../types';
|
||||
|
||||
const DEFAULT_LLM_BOTS = [
|
||||
'Anthropic-AI',
|
||||
'Claude-Web',
|
||||
'GPTBot',
|
||||
'ChatGPT-User',
|
||||
'cohere-ai',
|
||||
'Google-Extended'
|
||||
];
|
||||
|
||||
export function generateRobotsTxt(
|
||||
config: RobotsConfig,
|
||||
siteURL: URL
|
||||
): string {
|
||||
const lines: string[] = [];
|
||||
|
||||
// Allow all bots by default
|
||||
if (config.allowAllBots !== false) {
|
||||
lines.push('User-agent: *');
|
||||
lines.push('Allow: /');
|
||||
lines.push('');
|
||||
}
|
||||
|
||||
// Add sitemap
|
||||
lines.push('# Sitemaps');
|
||||
lines.push(`Sitemap: ${new URL('sitemap-index.xml', siteURL).href}`);
|
||||
lines.push('');
|
||||
|
||||
// LLM-specific rules
|
||||
if (config.llmBots?.enabled !== false) {
|
||||
lines.push('# LLM-specific resources');
|
||||
lines.push('# See: https://github.com/anthropics/llm-txt');
|
||||
|
||||
const agents = config.llmBots?.agents || DEFAULT_LLM_BOTS;
|
||||
agents.forEach(agent => {
|
||||
lines.push(`User-agent: ${agent}`);
|
||||
});
|
||||
lines.push('Allow: /llms.txt');
|
||||
lines.push('');
|
||||
}
|
||||
|
||||
// Additional agent rules
|
||||
if (config.additionalAgents) {
|
||||
config.additionalAgents.forEach(agent => {
|
||||
lines.push(`User-agent: ${agent.userAgent}`);
|
||||
|
||||
if (agent.allow) {
|
||||
agent.allow.forEach(path => {
|
||||
lines.push(`Allow: ${path}`);
|
||||
});
|
||||
}
|
||||
|
||||
if (agent.disallow) {
|
||||
agent.disallow.forEach(path => {
|
||||
lines.push(`Disallow: ${path}`);
|
||||
});
|
||||
}
|
||||
|
||||
lines.push('');
|
||||
});
|
||||
}
|
||||
|
||||
// Crawl delay
|
||||
if (config.crawlDelay) {
|
||||
lines.push('# Crawl delay (be nice to our server)');
|
||||
lines.push(`Crawl-delay: ${config.crawlDelay}`);
|
||||
lines.push('');
|
||||
}
|
||||
|
||||
// Custom rules
|
||||
if (config.customRules) {
|
||||
lines.push('# Custom rules');
|
||||
lines.push(config.customRules);
|
||||
lines.push('');
|
||||
}
|
||||
|
||||
return lines.join('\n');
|
||||
}
|
||||
```
|
||||
|
||||
### 4. LLMs.txt Generator (`src/generators/llms.ts`)
|
||||
|
||||
```typescript
|
||||
import type { LLMsConfig, ImportantPage } from '../types';
|
||||
|
||||
export async function generateLLMsTxt(
|
||||
config: LLMsConfig,
|
||||
siteURL: URL
|
||||
): Promise<string> {
|
||||
const lines: string[] = [];
|
||||
|
||||
// Header
|
||||
const description = typeof config.description === 'function'
|
||||
? config.description()
|
||||
: config.description;
|
||||
|
||||
lines.push(`# ${siteURL.hostname}`);
|
||||
if (description) {
|
||||
lines.push('');
|
||||
lines.push(`> ${description}`);
|
||||
}
|
||||
lines.push('');
|
||||
lines.push('---');
|
||||
lines.push('');
|
||||
|
||||
// Site Information
|
||||
lines.push('## Site Information');
|
||||
lines.push('');
|
||||
lines.push(`- **URL**: ${siteURL.href}`);
|
||||
if (description) {
|
||||
lines.push(`- **Description**: ${description}`);
|
||||
}
|
||||
lines.push('');
|
||||
|
||||
// Key Features
|
||||
if (config.keyFeatures && config.keyFeatures.length > 0) {
|
||||
lines.push('## Key Features');
|
||||
lines.push('');
|
||||
config.keyFeatures.forEach(feature => {
|
||||
lines.push(`- ${feature}`);
|
||||
});
|
||||
lines.push('');
|
||||
}
|
||||
|
||||
// Important Pages
|
||||
if (config.importantPages) {
|
||||
const pages = typeof config.importantPages === 'function'
|
||||
? await config.importantPages()
|
||||
: config.importantPages;
|
||||
|
||||
if (pages.length > 0) {
|
||||
lines.push('## Important Pages');
|
||||
lines.push('');
|
||||
pages.forEach(page => {
|
||||
const url = new URL(page.path, siteURL).href;
|
||||
lines.push(`- **${page.name}**: ${url}`);
|
||||
if (page.description) {
|
||||
lines.push(` ${page.description}`);
|
||||
}
|
||||
});
|
||||
lines.push('');
|
||||
}
|
||||
}
|
||||
|
||||
// Instructions for AI Assistants
|
||||
if (config.instructions) {
|
||||
lines.push('## For AI Assistants');
|
||||
lines.push('');
|
||||
lines.push(config.instructions);
|
||||
lines.push('');
|
||||
}
|
||||
|
||||
// API Endpoints
|
||||
if (config.apiEndpoints && config.apiEndpoints.length > 0) {
|
||||
lines.push('## API Endpoints');
|
||||
lines.push('');
|
||||
config.apiEndpoints.forEach(endpoint => {
|
||||
const method = endpoint.method || 'GET';
|
||||
lines.push(`- \`${method} ${endpoint.path}\` - ${endpoint.description}`);
|
||||
});
|
||||
lines.push('');
|
||||
}
|
||||
|
||||
// Tech Stack
|
||||
if (config.techStack) {
|
||||
lines.push('## Technical Stack');
|
||||
lines.push('');
|
||||
if (config.techStack.frontend) {
|
||||
lines.push(`- **Frontend**: ${config.techStack.frontend.join(', ')}`);
|
||||
}
|
||||
if (config.techStack.backend) {
|
||||
lines.push(`- **Backend**: ${config.techStack.backend.join(', ')}`);
|
||||
}
|
||||
if (config.techStack.ai) {
|
||||
lines.push(`- **AI**: ${config.techStack.ai.join(', ')}`);
|
||||
}
|
||||
if (config.techStack.other) {
|
||||
lines.push(`- **Other**: ${config.techStack.other.join(', ')}`);
|
||||
}
|
||||
lines.push('');
|
||||
}
|
||||
|
||||
// Brand Voice
|
||||
if (config.brandVoice && config.brandVoice.length > 0) {
|
||||
lines.push('## Brand Voice');
|
||||
lines.push('');
|
||||
config.brandVoice.forEach(item => {
|
||||
lines.push(`- ${item}`);
|
||||
});
|
||||
lines.push('');
|
||||
}
|
||||
|
||||
// Custom Sections
|
||||
if (config.customSections) {
|
||||
Object.entries(config.customSections).forEach(([title, content]) => {
|
||||
lines.push(`## ${title}`);
|
||||
lines.push('');
|
||||
lines.push(content);
|
||||
lines.push('');
|
||||
});
|
||||
}
|
||||
|
||||
// Footer
|
||||
lines.push('---');
|
||||
lines.push('');
|
||||
lines.push(`Last Updated: ${new Date().toISOString().split('T')[0]}`);
|
||||
|
||||
return lines.join('\n');
|
||||
}
|
||||
```
|
||||
|
||||
### 5. Humans.txt Generator (`src/generators/humans.ts`)
|
||||
|
||||
```typescript
|
||||
import type { HumansConfig } from '../types';
|
||||
|
||||
export function generateHumansTxt(config: HumansConfig): string {
|
||||
const lines: string[] = [];
|
||||
|
||||
// Team section
|
||||
if (config.team && config.team.length > 0) {
|
||||
lines.push('/* TEAM */');
|
||||
lines.push('');
|
||||
|
||||
config.team.forEach((member, index) => {
|
||||
if (index > 0) lines.push('');
|
||||
lines.push(`Name: ${member.name}`);
|
||||
if (member.role) lines.push(`Role: ${member.role}`);
|
||||
if (member.contact) lines.push(`Contact: ${member.contact}`);
|
||||
if (member.location) lines.push(`From: ${member.location}`);
|
||||
if (member.twitter) lines.push(`Twitter: ${member.twitter}`);
|
||||
if (member.github) lines.push(`GitHub: ${member.github}`);
|
||||
});
|
||||
|
||||
lines.push('');
|
||||
}
|
||||
|
||||
// Thanks section
|
||||
if (config.thanks && config.thanks.length > 0) {
|
||||
lines.push('/* THANKS */');
|
||||
lines.push('');
|
||||
config.thanks.forEach(thanks => {
|
||||
lines.push(`- ${thanks}`);
|
||||
});
|
||||
lines.push('');
|
||||
}
|
||||
|
||||
// Site section
|
||||
if (config.site) {
|
||||
lines.push('/* SITE */');
|
||||
lines.push('');
|
||||
|
||||
const lastUpdate = config.site.lastUpdate === 'auto'
|
||||
? new Date().toISOString().split('T')[0]
|
||||
: config.site.lastUpdate;
|
||||
|
||||
if (lastUpdate) lines.push(`Last update: ${lastUpdate}`);
|
||||
if (config.site.language) lines.push(`Language: ${config.site.language}`);
|
||||
if (config.site.doctype) lines.push(`Doctype: ${config.site.doctype}`);
|
||||
if (config.site.ide) lines.push(`IDE: ${config.site.ide}`);
|
||||
|
||||
if (config.site.techStack) {
|
||||
lines.push(`Tech Stack: ${config.site.techStack.join(', ')}`);
|
||||
}
|
||||
if (config.site.standards) {
|
||||
lines.push(`Standards: ${config.site.standards.join(', ')}`);
|
||||
}
|
||||
if (config.site.components) {
|
||||
lines.push(`Components: ${config.site.components.join(', ')}`);
|
||||
}
|
||||
if (config.site.software) {
|
||||
lines.push(`Software: ${config.site.software.join(', ')}`);
|
||||
}
|
||||
|
||||
lines.push('');
|
||||
}
|
||||
|
||||
// Story section
|
||||
if (config.story) {
|
||||
lines.push('/* THE STORY */');
|
||||
lines.push('');
|
||||
lines.push(config.story);
|
||||
lines.push('');
|
||||
}
|
||||
|
||||
// Fun Facts section
|
||||
if (config.funFacts && config.funFacts.length > 0) {
|
||||
lines.push('/* FUN FACTS */');
|
||||
lines.push('');
|
||||
config.funFacts.forEach(fact => {
|
||||
lines.push(`- ${fact}`);
|
||||
});
|
||||
lines.push('');
|
||||
}
|
||||
|
||||
// Philosophy section
|
||||
if (config.philosophy && config.philosophy.length > 0) {
|
||||
lines.push('/* PHILOSOPHY */');
|
||||
lines.push('');
|
||||
config.philosophy.forEach(item => {
|
||||
lines.push(`"${item}"`);
|
||||
});
|
||||
lines.push('');
|
||||
}
|
||||
|
||||
// Custom sections
|
||||
if (config.customSections) {
|
||||
Object.entries(config.customSections).forEach(([title, content]) => {
|
||||
lines.push(`/* ${title.toUpperCase()} */`);
|
||||
lines.push('');
|
||||
lines.push(content);
|
||||
lines.push('');
|
||||
});
|
||||
}
|
||||
|
||||
return lines.join('\n');
|
||||
}
|
||||
```
|
||||
|
||||
### 6. API Route Template (`routes/robots.ts`)
|
||||
|
||||
```typescript
|
||||
import type { APIRoute } from 'astro';
|
||||
import { generateRobotsTxt } from '../generators/robots';
|
||||
import { getConfig } from '../config';
|
||||
|
||||
export const GET: APIRoute = ({ site }) => {
|
||||
const config = getConfig();
|
||||
const siteURL = site || new URL('http://localhost:4321');
|
||||
|
||||
const content = config.templates?.robots
|
||||
? config.templates.robots(config.robots, siteURL)
|
||||
: generateRobotsTxt(config.robots, siteURL);
|
||||
|
||||
return new Response(content, {
|
||||
status: 200,
|
||||
headers: {
|
||||
'Content-Type': 'text/plain; charset=utf-8',
|
||||
'Cache-Control': `public, max-age=${config.caching?.robots || 3600}`,
|
||||
},
|
||||
});
|
||||
};
|
||||
```
|
||||
|
||||
## Testing Strategy
|
||||
|
||||
### Unit Tests
|
||||
|
||||
```typescript
|
||||
// tests/robots.test.ts
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { generateRobotsTxt } from '../src/generators/robots';
|
||||
|
||||
describe('generateRobotsTxt', () => {
|
||||
it('generates basic robots.txt', () => {
|
||||
const result = generateRobotsTxt({}, new URL('https://example.com'));
|
||||
expect(result).toContain('User-agent: *');
|
||||
expect(result).toContain('Sitemap: https://example.com/sitemap-index.xml');
|
||||
});
|
||||
|
||||
it('includes LLM bots when enabled', () => {
|
||||
const result = generateRobotsTxt(
|
||||
{ llmBots: { enabled: true } },
|
||||
new URL('https://example.com')
|
||||
);
|
||||
expect(result).toContain('Anthropic-AI');
|
||||
expect(result).toContain('GPTBot');
|
||||
});
|
||||
|
||||
it('respects custom crawl delay', () => {
|
||||
const result = generateRobotsTxt(
|
||||
{ crawlDelay: 5 },
|
||||
new URL('https://example.com')
|
||||
);
|
||||
expect(result).toContain('Crawl-delay: 5');
|
||||
});
|
||||
});
|
||||
```
|
||||
|
||||
### Integration Tests
|
||||
|
||||
```typescript
|
||||
// tests/integration.test.ts
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { testIntegration } from '@astrojs/test-utils';
|
||||
import discovery from '../src/index';
|
||||
|
||||
describe('discovery integration', () => {
|
||||
it('generates all discovery files', async () => {
|
||||
const fixture = await testIntegration({
|
||||
integrations: [discovery()],
|
||||
site: 'https://example.com'
|
||||
});
|
||||
|
||||
const files = await fixture.readdir('dist');
|
||||
expect(files).toContain('robots.txt');
|
||||
expect(files).toContain('llms.txt');
|
||||
expect(files).toContain('humans.txt');
|
||||
expect(files).toContain('sitemap-index.xml');
|
||||
});
|
||||
});
|
||||
```
|
||||
|
||||
## Build & Publish
|
||||
|
||||
### package.json
|
||||
|
||||
```json
|
||||
{
|
||||
"name": "@astrojs/discovery",
|
||||
"version": "1.0.0",
|
||||
"description": "Complete discovery integration for Astro",
|
||||
"type": "module",
|
||||
"exports": {
|
||||
".": "./dist/index.js",
|
||||
"./routes/*": "./dist/routes/*"
|
||||
},
|
||||
"files": [
|
||||
"dist",
|
||||
"README.md"
|
||||
],
|
||||
"scripts": {
|
||||
"build": "tsc",
|
||||
"test": "vitest",
|
||||
"prepublishOnly": "npm run build && npm test"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"astro": "^5.0.0"
|
||||
},
|
||||
"dependencies": {
|
||||
"@astrojs/sitemap": "^3.6.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@astrojs/test-utils": "^1.0.0",
|
||||
"typescript": "^5.3.0",
|
||||
"vitest": "^1.0.0"
|
||||
},
|
||||
"keywords": [
|
||||
"astro",
|
||||
"astro-integration",
|
||||
"robots",
|
||||
"sitemap",
|
||||
"llms",
|
||||
"humans",
|
||||
"discovery",
|
||||
"seo"
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
## Future Enhancements
|
||||
|
||||
1. **security.txt Support** - Add RFC 9116 security.txt generation
|
||||
2. **ads.txt Support** - For sites with advertising
|
||||
3. **manifest.json Support** - PWA manifest generation
|
||||
4. **RSS Feed Integration** - Optional RSS feed generation
|
||||
5. **OpenGraph Tags** - Meta tag injection
|
||||
6. **Structured Data** - JSON-LD schema.org markup
|
||||
7. **Analytics Integration** - Built-in analytics discovery
|
||||
8. **i18n Support** - Multi-language discovery files
|
||||
|
||||
## Resources
|
||||
|
||||
- [Astro Integration API](https://docs.astro.build/en/reference/integrations-reference/)
|
||||
- [humanstxt.org](https://humanstxt.org/)
|
||||
- [robots.txt spec](https://developers.google.com/search/docs/crawling-indexing/robots/intro)
|
||||
- [llms.txt proposal](https://github.com/anthropics/llm-txt)
|
||||
|
||||
---
|
||||
|
||||
**This integration is a proposal. Implementation details may vary based on Astro's API evolution.**
|
||||
Loading…
Add table
Add a link
Reference in a new issue