# Guide standard search engines away from MD files to prevent duplicate content

User-agent: Googlebot
Disallow: *.md
Disallow: /css/
Disallow: /jstopics/
Disallow: /js/
Disallow: /jspopups/

User-agent: bingbot
Disallow: *.md
Disallow: /css/
Disallow: /jstopics/
Disallow: /js/
Disallow: /jspopups/

User-agent: Slurp
Disallow: *.md
Disallow: /css/
Disallow: /jstopics/
Disallow: /js/
Disallow: /jspopups/

User-agent: Baiduspider
Disallow: *.md
Disallow: /css/
Disallow: /jstopics/
Disallow: /js/
Disallow: /jspopups/

User-agent: YandexBot
Disallow: *.md
Disallow: /css/
Disallow: /jstopics/
Disallow: /js/
Disallow: /jspopups/
# Explicitly allow everything for AI crawlers, including MD files

User-agent: GPTBot
Disallow: /css/
Disallow: /jstopics/
Disallow: /js/
Disallow: /jspopups/

User-agent: OAI-SearchBot
Disallow: /css/
Disallow: /jstopics/
Disallow: /js/
Disallow: /jspopups/

User-agent: anthropic-ai
Disallow: /css/
Disallow: /jstopics/
Disallow: /js/
Disallow: /jspopups/

User-agent: ClaudeBot
Disallow: /css/
Disallow: /jstopics/
Disallow: /js/
Disallow: /jspopups/

User-agent: PerplexityBot
Disallow: /css/
Disallow: /jstopics/
Disallow: /js/
Disallow: /jspopups/
# Exclude all irrelevant sub-folders for all

User-agent: *
Disallow: /css/
Disallow: /jstopics/
Disallow: /js/
Disallow: /jspopups/