# =====================================================================
# robots.txt for pallyroofing.com
# Pally Roofing | Garrettsville, OH | The Friendliest Roofer
# Last updated: 2026-09-29
#
# Policy: search-first and AI-visibility-first.
#   1. Traditional search engines: full access to public content.
#   2. AI search, retrieval, and user-fetch agents: full access, so
#      Pally Roofing can be found and cited in ChatGPT, Claude,
#      Perplexity, Gemini, Copilot, Apple Intelligence, and similar.
#   3. AI training crawlers: allowed, so the brand, services, and
#      service area are part of what models know about local roofers.
#   4. WordPress system paths, internal search, and duplicate-URL
#      patterns are kept out of every crawler's path.
#   5. Low-value scrapers that add no search or AI visibility: blocked.
#
# Note: a crawler that matches a named group below ignores the
# "User-agent: *" group, so the housekeeping rules are repeated
# in each group on purpose.
#
# LLM content guide: https://pallyroofing.com/llms.txt
# =====================================================================
 
 
# ---------------------------------------------------------------------
# 1. DEFAULT RULES (Googlebot, Bingbot, and all other compliant bots)
# ---------------------------------------------------------------------
User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Allow: /wp-admin/admin-ajax.php
Allow: /wp-content/uploads/
Allow: /llms.txt
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /xmlrpc.php
Disallow: /readme.html
Disallow: /license.txt
Disallow: /cgi-bin/
Disallow: /?s=
Disallow: /*?s=
Disallow: /search/
Disallow: /*?replytocom=
Disallow: /*/trackback/
Disallow: /comments/feed/
Disallow: /*/comments/feed/
Disallow: /*?preview=true
Disallow: /*&preview=true
Disallow: /*?elementor_library=
Disallow: /elementor-hf/
 
 
# ---------------------------------------------------------------------
# 2. AI SEARCH, RETRIEVAL, AND USER-FETCH AGENTS
#    These build AI answer indexes and fetch pages live when a person
#    asks an assistant a question. Blocking them removes citations.
# ---------------------------------------------------------------------
 
# OpenAI: ChatGPT Search index + live browsing for ChatGPT users
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
# Anthropic: Claude search index + live fetches for Claude users
User-agent: Claude-SearchBot
User-agent: Claude-User
# Perplexity: answer index + live fetches for Perplexity users
User-agent: PerplexityBot
User-agent: Perplexity-User
# DuckDuckGo AI answers (Duck.ai / DuckAssist)
User-agent: DuckAssistBot
# Mistral Le Chat live fetches
User-agent: MistralAI-User
# Meta AI live link fetches
User-agent: meta-externalfetcher
# You.com
User-agent: YouBot
Allow: /
Allow: /wp-admin/admin-ajax.php
Allow: /llms.txt
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /xmlrpc.php
Disallow: /?s=
Disallow: /*?s=
Disallow: /search/
Disallow: /*?replytocom=
Disallow: /*?preview=true
Disallow: /*&preview=true
Disallow: /*?elementor_library=
 
 
# ---------------------------------------------------------------------
# 3. AI TRAINING AND MODEL-KNOWLEDGE CRAWLERS
#    Allowed so models "know" Pally Roofing as a Northeast Ohio metal
#    roofing entity. To opt out of training later while keeping AI
#    search citations, change "Allow: /" to "Disallow: /" in this
#    group only. Sections 1 and 2 are unaffected.
# ---------------------------------------------------------------------
 
# OpenAI model training
User-agent: GPTBot
# Anthropic model training
User-agent: ClaudeBot
# Google Gemini / Vertex AI grounding and training (does NOT affect
# Google Search rankings or AI Overviews eligibility)
User-agent: Google-Extended
# Apple Intelligence training (Applebot itself stays under section 1)
User-agent: Applebot-Extended
# Amazon (Alexa / Rufus / Nova)
User-agent: Amazonbot
# Meta AI training
User-agent: meta-externalagent
# Common Crawl (open dataset used by many LLMs)
User-agent: CCBot
# Cohere
User-agent: cohere-ai
User-agent: cohere-training-data-crawler
Allow: /
Allow: /wp-admin/admin-ajax.php
Allow: /llms.txt
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /xmlrpc.php
Disallow: /?s=
Disallow: /*?s=
Disallow: /search/
Disallow: /*?replytocom=
Disallow: /*?preview=true
Disallow: /*&preview=true
Disallow: /*?elementor_library=
 
 
# ---------------------------------------------------------------------
# 4. SEO TOOL CRAWLERS (kept, but slowed down)
#    Semrush and Ahrefs are used for audits and competitive data.
# ---------------------------------------------------------------------
User-agent: SemrushBot
User-agent: AhrefsBot
Crawl-delay: 5
Allow: /
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /xmlrpc.php
Disallow: /?s=
Disallow: /*?s=
Disallow: /search/
 
 
# ---------------------------------------------------------------------
# 5. BLOCKED: scrapers and crawlers with no search or AI-visibility
#    value to a local contractor. Some ignore robots.txt, so enforce
#    at the host, CDN, or WAF level as well.
# ---------------------------------------------------------------------
User-agent: Bytespider
User-agent: MJ12bot
User-agent: DotBot
User-agent: BLEXBot
User-agent: PetalBot
User-agent: Timpibot
User-agent: ImagesiftBot
User-agent: omgili
User-agent: omgilibot
User-agent: magpie-crawler
Disallow: /
 
 
# ---------------------------------------------------------------------
# SITEMAPS
# ---------------------------------------------------------------------
Sitemap: https://pallyroofing.com/sitemaps.xml