# ========================
# Robots.txt for SEO & Security
# Website: https://gubbicivilengineers.com
# Created on: 15/03/2025
# ========================

# Allow All Search Engines to Crawl the Site
User-agent: *
Disallow:

# Block Unwanted Pages from Indexing (Prevents Duplicate & Low-Value Pages)
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /wp-includes/
Disallow: /cgi-bin/
Disallow: /tmp/
Disallow: /private/
Disallow: /cart/
Disallow: /checkout/
Disallow: /login/
Disallow: /register/
Disallow: /search/
Disallow: /tag/
Disallow: /*?s=
Disallow: /*?orderby=
Disallow: /*?add-to-cart=
Disallow: /*?replytocom
Disallow: /*?utm_source=
Disallow: /trackback/
Disallow: /*.php$

# Block Specific Pages from Indexing
Disallow: /blogs/author/krunal
Disallow: /privacy-policy
Disallow: /disclaimer
Disallow: /blogs/elementor-hf/footer_gubbi/
Disallow: /blogs/elementor-hf/gubbi_header/

# Allow Important Sections for Crawling
Allow: /wp-admin/admin-ajax.php
Allow: /wp-content/uploads/

# Sitemaps for Fast Indexing & Better Crawling
Sitemap: https://gubbicivilengineers.com/sitemap.xml
Sitemap: https://gubbicivilengineers.com/page-sitemap.xml
Sitemap: https://gubbicivilengineers.com/post-sitemap.xml



# Prevent Crawling of Plugins, Themes & Cache
Disallow: /wp-content/plugins/
Disallow: /wp-content/themes/
Disallow: /wp-content/cache/
Disallow: /wp-json/

# Blocking Sensitive & Unnecessary Files
Disallow: /readme.html
Disallow: /license.txt
Disallow: /?author=*
Disallow: /*/feed/

#Block AI Bots, Scrapers & Aggressive Crawlers
User-agent: GPTBot
Disallow: /
User-agent: ChatGPT-User
Disallow: /
User-agent: Claude
Disallow: /
User-agent: DataForSeoBot
Disallow: /
User-agent: AhrefsBot
Disallow: /
User-agent: SemrushBot
Disallow: /
User-agent: MJ12bot
Disallow: /
User-agent: DotBot
Disallow: /

# Google Ads & AdSense Bot Allowance (Ensures Ads are Properly Crawled)
User-agent: AdsBot-Google
Allow: /
User-agent: Mediapartners-Google
Allow: /

# Allow Image Crawlers for Image SEO
User-agent: Googlebot-Image
Allow: /wp-content/uploads/

# Allow Google Mobile Crawlers (Mobile SEO Optimization)
User-agent: Googlebot-Mobile
Allow: /

# Crawl Delay for Non-Google Bots (Prevents Server Overload)
Crawl-delay: 10

# Custom User-Agent Rules (For Bing & Other Crawlers)
User-agent: Bingbot
Crawl-delay: 5
User-agent: Yandex
Disallow: /

# ========================
# End of robots.txt file
# ========================