astro-discovery/tests/robots.test.ts

import { describe, it, expect } from 'vitest';
import { generateRobotsTxt } from '../src/generators/robots.js';

describe('generateRobotsTxt', () => {
  const testURL = new URL('https://example.com');

  it('generates basic robots.txt with defaults', () => {
    const result = generateRobotsTxt({}, testURL);

    expect(result).toContain('User-agent: *');
    expect(result).toContain('Allow: /');
    expect(result).toContain('Sitemap: https://example.com/sitemap-index.xml');
  });

  it('includes LLM bots when enabled', () => {
    const result = generateRobotsTxt(
      { llmBots: { enabled: true } },
      testURL
    );

    expect(result).toContain('Anthropic-AI');
    expect(result).toContain('GPTBot');
    expect(result).toContain('Claude-Web');
    expect(result).toContain('Allow: /llms.txt');
  });

  it('excludes LLM bots when disabled', () => {
    const result = generateRobotsTxt(
      { llmBots: { enabled: false } },
      testURL
    );

    expect(result).not.toContain('Anthropic-AI');
    expect(result).not.toContain('GPTBot');
  });

  it('respects custom crawl delay', () => {
    const result = generateRobotsTxt(
      { crawlDelay: 5 },
      testURL
    );

    expect(result).toContain('Crawl-delay: 5');
  });

  it('includes custom agents', () => {
    const result = generateRobotsTxt(
      {
        additionalAgents: [
          {
            userAgent: 'CustomBot',
            allow: ['/api'],
            disallow: ['/admin'],
          },
        ],
      },
      testURL
    );

    expect(result).toContain('User-agent: CustomBot');
    expect(result).toContain('Allow: /api');
    expect(result).toContain('Disallow: /admin');
  });

  it('includes custom rules', () => {
    const customRules = 'User-agent: SpecialBot\nCrawl-delay: 10';
    const result = generateRobotsTxt(
      { customRules },
      testURL
    );

    expect(result).toContain(customRules);
  });

  it('allows custom LLM bot agents', () => {
    const result = generateRobotsTxt(
      {
        llmBots: {
          enabled: true,
          agents: ['CustomAI', 'AnotherBot'],
        },
      },
      testURL
    );

    expect(result).toContain('CustomAI');
    expect(result).toContain('AnotherBot');
  });

  it('ends with newline', () => {
    const result = generateRobotsTxt({}, testURL);
    expect(result.endsWith('\n')).toBe(true);
  });
});