feat: initial implementation of @astrojs/discovery integration
This commit introduces a comprehensive Astro integration that automatically generates discovery files for websites: Features: - robots.txt with LLM bot support (Anthropic-AI, GPTBot, etc.) - llms.txt for AI assistant context and instructions - humans.txt for team credits and site information - Automatic sitemap integration via @astrojs/sitemap Technical Details: - TypeScript implementation with full type safety - Configurable HTTP caching headers - Custom template support for all generated files - Sensible defaults with extensive customization options - Date-based versioning (2025.11.03) Testing: - 34 unit tests covering all generators - Test coverage for robots.txt, llms.txt, and humans.txt - Integration with Vitest Documentation: - Comprehensive README with examples - API reference documentation - Contributing guidelines - Example configurations (minimal and full)
This commit is contained in:
commit
d25dde4627
25 changed files with 11001 additions and 0 deletions
155
tests/humans.test.ts
Normal file
155
tests/humans.test.ts
Normal file
|
|
@ -0,0 +1,155 @@
|
|||
import { describe, it, expect } from 'vitest';
|
||||
import { generateHumansTxt } from '../src/generators/humans.js';
|
||||
|
||||
describe('generateHumansTxt', () => {
|
||||
it('generates basic humans.txt structure', () => {
|
||||
const result = generateHumansTxt({});
|
||||
expect(result).toBeTruthy();
|
||||
});
|
||||
|
||||
it('includes team section', () => {
|
||||
const result = generateHumansTxt({
|
||||
team: [
|
||||
{
|
||||
name: 'Jane Doe',
|
||||
role: 'Developer',
|
||||
contact: 'jane@example.com',
|
||||
location: 'SF',
|
||||
twitter: '@jane',
|
||||
github: 'jane',
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
expect(result).toContain('/* TEAM */');
|
||||
expect(result).toContain('Name: Jane Doe');
|
||||
expect(result).toContain('Role: Developer');
|
||||
expect(result).toContain('Contact: jane@example.com');
|
||||
expect(result).toContain('From: SF');
|
||||
expect(result).toContain('Twitter: @jane');
|
||||
expect(result).toContain('GitHub: jane');
|
||||
});
|
||||
|
||||
it('includes multiple team members', () => {
|
||||
const result = generateHumansTxt({
|
||||
team: [
|
||||
{ name: 'Jane Doe' },
|
||||
{ name: 'John Smith' },
|
||||
],
|
||||
});
|
||||
|
||||
expect(result).toContain('Jane Doe');
|
||||
expect(result).toContain('John Smith');
|
||||
});
|
||||
|
||||
it('includes thanks section', () => {
|
||||
const result = generateHumansTxt({
|
||||
thanks: ['Coffee', 'Stack Overflow'],
|
||||
});
|
||||
|
||||
expect(result).toContain('/* THANKS */');
|
||||
expect(result).toContain('Coffee');
|
||||
expect(result).toContain('Stack Overflow');
|
||||
});
|
||||
|
||||
it('includes site section with auto date', () => {
|
||||
const result = generateHumansTxt({
|
||||
site: {
|
||||
lastUpdate: 'auto',
|
||||
language: 'English',
|
||||
doctype: 'HTML5',
|
||||
ide: 'VS Code',
|
||||
},
|
||||
});
|
||||
|
||||
const today = new Date().toISOString().split('T')[0];
|
||||
|
||||
expect(result).toContain('/* SITE */');
|
||||
expect(result).toContain(`Last update: ${today}`);
|
||||
expect(result).toContain('Language: English');
|
||||
expect(result).toContain('Doctype: HTML5');
|
||||
expect(result).toContain('IDE: VS Code');
|
||||
});
|
||||
|
||||
it('includes site section with custom date', () => {
|
||||
const result = generateHumansTxt({
|
||||
site: {
|
||||
lastUpdate: '2025-11-03',
|
||||
},
|
||||
});
|
||||
|
||||
expect(result).toContain('Last update: 2025-11-03');
|
||||
});
|
||||
|
||||
it('includes tech stack', () => {
|
||||
const result = generateHumansTxt({
|
||||
site: {
|
||||
techStack: ['Astro', 'TypeScript', 'React'],
|
||||
standards: ['HTML5', 'CSS3'],
|
||||
components: ['Astro Components'],
|
||||
software: ['VS Code', 'Git'],
|
||||
},
|
||||
});
|
||||
|
||||
expect(result).toContain('Tech Stack: Astro, TypeScript, React');
|
||||
expect(result).toContain('Standards: HTML5, CSS3');
|
||||
expect(result).toContain('Components: Astro Components');
|
||||
expect(result).toContain('Software: VS Code, Git');
|
||||
});
|
||||
|
||||
it('includes story section', () => {
|
||||
const result = generateHumansTxt({
|
||||
story: 'This is our story.\nIt spans multiple lines.',
|
||||
});
|
||||
|
||||
expect(result).toContain('/* THE STORY */');
|
||||
expect(result).toContain('This is our story.');
|
||||
});
|
||||
|
||||
it('includes fun facts', () => {
|
||||
const result = generateHumansTxt({
|
||||
funFacts: ['Built with love', 'Coffee powered'],
|
||||
});
|
||||
|
||||
expect(result).toContain('/* FUN FACTS */');
|
||||
expect(result).toContain('Built with love');
|
||||
expect(result).toContain('Coffee powered');
|
||||
});
|
||||
|
||||
it('includes philosophy section', () => {
|
||||
const result = generateHumansTxt({
|
||||
philosophy: ['Make it simple', 'Make it work'],
|
||||
});
|
||||
|
||||
expect(result).toContain('/* PHILOSOPHY */');
|
||||
expect(result).toContain('"Make it simple"');
|
||||
expect(result).toContain('"Make it work"');
|
||||
});
|
||||
|
||||
it('includes custom sections', () => {
|
||||
const result = generateHumansTxt({
|
||||
customSections: {
|
||||
'CONTACT': 'Email: info@example.com',
|
||||
'LICENSE': 'MIT License',
|
||||
},
|
||||
});
|
||||
|
||||
expect(result).toContain('/* CONTACT */');
|
||||
expect(result).toContain('Email: info@example.com');
|
||||
expect(result).toContain('/* LICENSE */');
|
||||
expect(result).toContain('MIT License');
|
||||
});
|
||||
|
||||
it('properly indents content', () => {
|
||||
const result = generateHumansTxt({
|
||||
team: [{ name: 'Jane' }],
|
||||
});
|
||||
|
||||
expect(result).toContain(' Name: Jane');
|
||||
});
|
||||
|
||||
it('ends with newline', () => {
|
||||
const result = generateHumansTxt({});
|
||||
expect(result.endsWith('\n')).toBe(true);
|
||||
});
|
||||
});
|
||||
166
tests/llms.test.ts
Normal file
166
tests/llms.test.ts
Normal file
|
|
@ -0,0 +1,166 @@
|
|||
import { describe, it, expect } from 'vitest';
|
||||
import { generateLLMsTxt } from '../src/generators/llms.js';
|
||||
|
||||
describe('generateLLMsTxt', () => {
|
||||
const testURL = new URL('https://example.com');
|
||||
|
||||
it('generates basic llms.txt with site URL', async () => {
|
||||
const result = await generateLLMsTxt({}, testURL);
|
||||
|
||||
expect(result).toContain('# example.com');
|
||||
expect(result).toContain('**URL**: https://example.com/');
|
||||
});
|
||||
|
||||
it('includes description when provided', async () => {
|
||||
const result = await generateLLMsTxt(
|
||||
{ description: 'Test site description' },
|
||||
testURL
|
||||
);
|
||||
|
||||
expect(result).toContain('> Test site description');
|
||||
expect(result).toContain('**Description**: Test site description');
|
||||
});
|
||||
|
||||
it('supports dynamic description function', async () => {
|
||||
const result = await generateLLMsTxt(
|
||||
{ description: () => 'Dynamic description' },
|
||||
testURL
|
||||
);
|
||||
|
||||
expect(result).toContain('> Dynamic description');
|
||||
});
|
||||
|
||||
it('includes key features', async () => {
|
||||
const result = await generateLLMsTxt(
|
||||
{
|
||||
keyFeatures: ['Feature 1', 'Feature 2', 'Feature 3'],
|
||||
},
|
||||
testURL
|
||||
);
|
||||
|
||||
expect(result).toContain('## Key Features');
|
||||
expect(result).toContain('- Feature 1');
|
||||
expect(result).toContain('- Feature 2');
|
||||
});
|
||||
|
||||
it('includes important pages', async () => {
|
||||
const result = await generateLLMsTxt(
|
||||
{
|
||||
importantPages: [
|
||||
{
|
||||
name: 'Docs',
|
||||
path: '/docs',
|
||||
description: 'Documentation',
|
||||
},
|
||||
],
|
||||
},
|
||||
testURL
|
||||
);
|
||||
|
||||
expect(result).toContain('## Important Pages');
|
||||
expect(result).toContain('[Docs]');
|
||||
expect(result).toContain('https://example.com/docs');
|
||||
expect(result).toContain('Documentation');
|
||||
});
|
||||
|
||||
it('supports async important pages function', async () => {
|
||||
const result = await generateLLMsTxt(
|
||||
{
|
||||
importantPages: async () => [
|
||||
{ name: 'Blog', path: '/blog' },
|
||||
],
|
||||
},
|
||||
testURL
|
||||
);
|
||||
|
||||
expect(result).toContain('[Blog]');
|
||||
});
|
||||
|
||||
it('includes AI instructions', async () => {
|
||||
const result = await generateLLMsTxt(
|
||||
{
|
||||
instructions: 'Be helpful and accurate',
|
||||
},
|
||||
testURL
|
||||
);
|
||||
|
||||
expect(result).toContain('## Instructions for AI Assistants');
|
||||
expect(result).toContain('Be helpful and accurate');
|
||||
});
|
||||
|
||||
it('includes API endpoints', async () => {
|
||||
const result = await generateLLMsTxt(
|
||||
{
|
||||
apiEndpoints: [
|
||||
{
|
||||
path: '/api/test',
|
||||
method: 'POST',
|
||||
description: 'Test endpoint',
|
||||
},
|
||||
],
|
||||
},
|
||||
testURL
|
||||
);
|
||||
|
||||
expect(result).toContain('## API Endpoints');
|
||||
expect(result).toContain('POST /api/test');
|
||||
expect(result).toContain('Test endpoint');
|
||||
});
|
||||
|
||||
it('includes tech stack', async () => {
|
||||
const result = await generateLLMsTxt(
|
||||
{
|
||||
techStack: {
|
||||
frontend: ['Astro', 'React'],
|
||||
backend: ['Node.js'],
|
||||
ai: ['Claude'],
|
||||
},
|
||||
},
|
||||
testURL
|
||||
);
|
||||
|
||||
expect(result).toContain('## Technical Stack');
|
||||
expect(result).toContain('**Frontend**: Astro, React');
|
||||
expect(result).toContain('**Backend**: Node.js');
|
||||
expect(result).toContain('**AI/ML**: Claude');
|
||||
});
|
||||
|
||||
it('includes brand voice', async () => {
|
||||
const result = await generateLLMsTxt(
|
||||
{
|
||||
brandVoice: ['Professional', 'Friendly'],
|
||||
},
|
||||
testURL
|
||||
);
|
||||
|
||||
expect(result).toContain('## Brand Voice & Guidelines');
|
||||
expect(result).toContain('- Professional');
|
||||
expect(result).toContain('- Friendly');
|
||||
});
|
||||
|
||||
it('includes custom sections', async () => {
|
||||
const result = await generateLLMsTxt(
|
||||
{
|
||||
customSections: {
|
||||
'Contact': 'Email: test@example.com',
|
||||
},
|
||||
},
|
||||
testURL
|
||||
);
|
||||
|
||||
expect(result).toContain('## Contact');
|
||||
expect(result).toContain('Email: test@example.com');
|
||||
});
|
||||
|
||||
it('includes last updated date', async () => {
|
||||
const result = await generateLLMsTxt({}, testURL);
|
||||
const today = new Date().toISOString().split('T')[0];
|
||||
|
||||
expect(result).toContain(`**Last Updated**: ${today}`);
|
||||
});
|
||||
|
||||
it('ends with newline', async () => {
|
||||
const result = await generateLLMsTxt({}, testURL);
|
||||
expect(result.endsWith('\n')).toBe(true);
|
||||
});
|
||||
});
|
||||
94
tests/robots.test.ts
Normal file
94
tests/robots.test.ts
Normal file
|
|
@ -0,0 +1,94 @@
|
|||
import { describe, it, expect } from 'vitest';
|
||||
import { generateRobotsTxt } from '../src/generators/robots.js';
|
||||
|
||||
describe('generateRobotsTxt', () => {
|
||||
const testURL = new URL('https://example.com');
|
||||
|
||||
it('generates basic robots.txt with defaults', () => {
|
||||
const result = generateRobotsTxt({}, testURL);
|
||||
|
||||
expect(result).toContain('User-agent: *');
|
||||
expect(result).toContain('Allow: /');
|
||||
expect(result).toContain('Sitemap: https://example.com/sitemap-index.xml');
|
||||
});
|
||||
|
||||
it('includes LLM bots when enabled', () => {
|
||||
const result = generateRobotsTxt(
|
||||
{ llmBots: { enabled: true } },
|
||||
testURL
|
||||
);
|
||||
|
||||
expect(result).toContain('Anthropic-AI');
|
||||
expect(result).toContain('GPTBot');
|
||||
expect(result).toContain('Claude-Web');
|
||||
expect(result).toContain('Allow: /llms.txt');
|
||||
});
|
||||
|
||||
it('excludes LLM bots when disabled', () => {
|
||||
const result = generateRobotsTxt(
|
||||
{ llmBots: { enabled: false } },
|
||||
testURL
|
||||
);
|
||||
|
||||
expect(result).not.toContain('Anthropic-AI');
|
||||
expect(result).not.toContain('GPTBot');
|
||||
});
|
||||
|
||||
it('respects custom crawl delay', () => {
|
||||
const result = generateRobotsTxt(
|
||||
{ crawlDelay: 5 },
|
||||
testURL
|
||||
);
|
||||
|
||||
expect(result).toContain('Crawl-delay: 5');
|
||||
});
|
||||
|
||||
it('includes custom agents', () => {
|
||||
const result = generateRobotsTxt(
|
||||
{
|
||||
additionalAgents: [
|
||||
{
|
||||
userAgent: 'CustomBot',
|
||||
allow: ['/api'],
|
||||
disallow: ['/admin'],
|
||||
},
|
||||
],
|
||||
},
|
||||
testURL
|
||||
);
|
||||
|
||||
expect(result).toContain('User-agent: CustomBot');
|
||||
expect(result).toContain('Allow: /api');
|
||||
expect(result).toContain('Disallow: /admin');
|
||||
});
|
||||
|
||||
it('includes custom rules', () => {
|
||||
const customRules = 'User-agent: SpecialBot\nCrawl-delay: 10';
|
||||
const result = generateRobotsTxt(
|
||||
{ customRules },
|
||||
testURL
|
||||
);
|
||||
|
||||
expect(result).toContain(customRules);
|
||||
});
|
||||
|
||||
it('allows custom LLM bot agents', () => {
|
||||
const result = generateRobotsTxt(
|
||||
{
|
||||
llmBots: {
|
||||
enabled: true,
|
||||
agents: ['CustomAI', 'AnotherBot'],
|
||||
},
|
||||
},
|
||||
testURL
|
||||
);
|
||||
|
||||
expect(result).toContain('CustomAI');
|
||||
expect(result).toContain('AnotherBot');
|
||||
});
|
||||
|
||||
it('ends with newline', () => {
|
||||
const result = generateRobotsTxt({}, testURL);
|
||||
expect(result.endsWith('\n')).toBe(true);
|
||||
});
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue