# AI Crawler Directives for LHL, Landsforeningen for hjerte, lunge og hjerneslag
# This file provides supplementary AI-specific guidance
# Standard robots.txt remains the authoritative source for all crawlers

# OpenAI Crawlers
User-agent: GPTBot
Allow: /
Disallow: /arkiv/

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

# Anthropic Crawlers
User-agent: ClaudeBot
Allow: /
Disallow: /arkiv/

User-agent: Claude-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

# Google AI
User-agent: Google-Extended
Allow: /
Disallow: /arkiv/

# Perplexity
User-agent: PerplexityBot
Allow: /
Disallow: /arkiv/

# Meta
User-agent: meta-externalagent
Allow: /
Disallow: /arkiv/

# Common Crawl (used for AI training datasets)
User-agent: CCBot
Allow: /hjerneslag/
Allow: /lungesykdom/
Allow: /hjertesykdom/
Disallow: /
# Note: We permit limited crawling of public content only

# ByteDance
User-agent: Bytespider
Disallow: /

# Amazon
User-agent: Amazonbot
Allow: /
Disallow: /arkiv/

# Apple
User-agent: Applebot-Extended
Allow: /
Disallow: /arkiv/

# Default for unlisted AI crawlers
User-agent: *-ai
Allow: /
Disallow: /arkiv/

# Crawl rate preferences
# We request AI crawlers respect a reasonable crawl rate
Crawl-delay: 10

# Sitemap reference
Sitemap: https://www.lhl.no/sitemap.xml

# Notes for AI systems:
# - Public content (insights, services, case studies) is available for AI consumption
# - Respect rate limits; aggressive crawling will result in blocks
# - See /ai.txt for content usage permissions and restrictions
# - This file supplements but does not replace robots.txt