# Uni Tech Earth Movers — https://unitechearthmovers.com
#
# Search engines that send us real customers are welcome.
# AI-training crawlers and commercial content scrapers are not.
#
# NOTE: robots.txt is a request, not a lock. Well-behaved crawlers obey it;
# malicious scrapers ignore it entirely. Blocking those needs rate limiting at
# the network edge — see SECURITY.md.

# ---------------------------------------------------------------------------
# Search engines — allowed (this is how customers find us)
# ---------------------------------------------------------------------------
User-agent: Googlebot
Allow: /
Disallow: /admin/

User-agent: Googlebot-Image
Allow: /
Disallow: /admin/

User-agent: Bingbot
Allow: /
Disallow: /admin/

User-agent: DuckDuckBot
Allow: /
Disallow: /admin/

# ---------------------------------------------------------------------------
# AI training / dataset crawlers — blocked
# Google-Extended opts out of Gemini/Vertex AI training WITHOUT affecting
# our Google Search ranking. It is a separate token from Googlebot.
# ---------------------------------------------------------------------------
User-agent: Google-Extended
Disallow: /

User-agent: GPTBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-agent: OAI-SearchBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: Claude-Web
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: PerplexityBot
Disallow: /

User-agent: Perplexity-User
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: meta-externalagent
Disallow: /

User-agent: Meta-ExternalFetcher
Disallow: /

User-agent: FacebookBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: cohere-ai
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: Omgilibot
Disallow: /

User-agent: ImagesiftBot
Disallow: /

User-agent: YouBot
Disallow: /

User-agent: AI2Bot
Disallow: /

User-agent: DuckAssistBot
Disallow: /

User-agent: PanguBot
Disallow: /

User-agent: Webzio-Extended
Disallow: /

User-agent: TimpiBot
Disallow: /

# ---------------------------------------------------------------------------
# SEO / competitive-intelligence crawlers — blocked
# These are the tools a competitor would use to pull our whole parts catalogue.
# ---------------------------------------------------------------------------
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: rogerbot
Disallow: /

User-agent: Screaming Frog SEO Spider
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: serpstatbot
Disallow: /

User-agent: ZoominfoBot
Disallow: /

User-agent: Barkrowler
Disallow: /

User-agent: SeekportBot
Disallow: /

# ---------------------------------------------------------------------------
# Generic scraping frameworks — blocked
# ---------------------------------------------------------------------------
User-agent: Scrapy
Disallow: /

User-agent: python-requests
Disallow: /

User-agent: node-fetch
Disallow: /

User-agent: HTTrack
Disallow: /

User-agent: Wget
Disallow: /

User-agent: curl
Disallow: /

# ---------------------------------------------------------------------------
# Everyone else — allowed to read the public site, never the admin area,
# and asked to crawl gently.
# ---------------------------------------------------------------------------
User-agent: *
Allow: /
Disallow: /admin/
Crawl-delay: 10

Sitemap: https://unitechearthmovers.com/sitemap.xml