# ===============================
# robots.txt for https://www.digiprima.com
# SEO + AI-Friendly | Security-Hardened Configuration
# Updated: May 2026
# ===============================
# BLOCK 1: Malicious & high-risk bots
# These often ignore robots.txt; combine with .htaccess / firewall
# -------------------------------------------------------
User-agent: Bytespider
Disallow: /
 
User-agent: Baiduspider
Disallow: /
 
User-agent: YisouSpider
Disallow: /
 
User-agent: Yandex
Disallow: /

User-agent: YandexBot
Disallow: /
 
User-agent: MJ12bot
Disallow: /
 
User-agent: DotBot
Disallow: /
 
User-agent: MauiBot
Disallow: /
 
User-agent: Nuclei
Disallow: /
 
# BLOCK 2: Chinese AI crawlers (low business value for our markets)
# -------------------------------------------------------

User-agent: DeepseekBot
Disallow: /

User-agent: Deepseek
Disallow: /
 
# BLOCK 3: AI training crawlers — protect content
# -------------------------------------------------------

User-agent: CCBot
Disallow: /

User-agent: PetalBot
Disallow: /
 
User-agent: serpstatbot
Disallow: /
 
# ALLOW: Legitimate AI search / visibility crawlers
# -------------------------------------------------------

User-agent: GPTBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Grok
Allow: /

User-agent: xai-grok
Allow: /

User-agent: SearchGPTBot
Allow: /
 
# ALLOW: Major search engines
# -------------------------------------------------------

User-agent: Googlebot
Allow: /

User-agent: Googlebot-Image
Allow: /

User-agent: Googlebot-Video
Allow: /

User-agent: AdsBot-Google
Allow: /

User-agent: Bingbot
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Slurp
Allow: /

User-agent: Applebot
Allow: /
 
# ALLOW: Social & advertising crawlers
# -------------------------------------------------------

User-agent: Facebot
Allow: /

User-agent: facebookexternalhit
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: Pinterestbot
Allow: /
 
# Image & Asset Directories


User-agent: *
Allow: /case-studies/images/
Allow: /home/images/
Allow: /about/images/
Allow: /blogs/images/
Allow: /contact/images/
Allow: /career/images/
Allow: /guide/images/
Allow: /privacy/images/
Allow: /terms/images/
 
# DEFAULT RULE
# -------------------------------------------------------
User-agent: *
Allow: /
 
# Sitemap
# -------------------------------------------------------
Sitemap: https://www.digiprima.com/sitemap-index.xml