# SPEDIFORT® Robots.txt
# https://www.spedifort.com

# Search Engine Crawlers (Googlebot, Bingbot) nutzen die Default-Regeln unten.

# Social Media Crawlers
User-agent: Twitterbot
Allow: /

User-agent: facebookexternalhit
Allow: /

User-agent: LinkedInBot
Allow: /

# LLM & AI Crawlers
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Anthropic-AI
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Cohere-AI
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: GoogleOther
Allow: /

User-agent: CCBot
Allow: /

User-agent: YouBot
Allow: /

User-agent: AI2Bot
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

User-agent: Bytespider
Allow: /

User-agent: Diffbot
Allow: /

User-agent: ImagesiftBot
Allow: /

User-agent: Omgilibot
Allow: /

User-agent: Timpibot
Allow: /

User-agent: DeepSeekBot
Allow: /

User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: PetalBot
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: iaskspider
Allow: /

# Default Rule
User-agent: *
Allow: /
Disallow: /admin
Disallow: /admin/*
Disallow: /intern/*
Disallow: /*?lang=
Disallow: /products/
Disallow: /collections/
Disallow: /cart
Disallow: /checkout
Disallow: /account
Disallow: /password
Disallow: /ressourcen/*/danke
Disallow: /ki-mastermind/danke
Disallow: /search
Disallow: /policies/
Disallow: /a/
Disallow: /frontpage
Disallow: /pages/
Disallow: /discount/

# LLM Context Files
# See https://llmstxt.org/ for specification
# /llms.txt - Concise summary for quick context
# /llms-full.txt - Comprehensive documentation
# /.well-known/llms.txt - Alternative discovery path

# Sitemaps – primary index, sub-sitemaps listed for crawler-fallback discovery
Sitemap: https://www.spedifort.com/sitemap-index.xml
Sitemap: https://www.spedifort.com/sitemap.xml
Sitemap: https://www.spedifort.com/video-sitemap.xml
Sitemap: https://www.spedifort.com/course-sitemap.xml
Sitemap: https://www.spedifort.com/dynamic-sitemap.xml
Sitemap: https://www.spedifort.com/case-study-sitemap.xml
Sitemap: https://www.spedifort.com/workforce-sitemap.xml