# =====================================================
# robots.txt for https://sendameal.com
# Purpose: SEO-friendly crawling + AI policy + noisy bot control
# Date: 2026-04-12
# =====================================================

# -----------------------------
# Default rules for all robots
# -----------------------------
User-agent: *
# Block sensitive / non-SEO pages
Disallow: /account.php
Disallow: /login.php
Disallow: /createaccount.php
Disallow: /forgotpassword.php
Disallow: /cart.php
Disallow: /checkout.php
Disallow: /checkout
Disallow: /finishorder.php
Disallow: /wishlist.php
Disallow: /compare
Disallow: /orders
Disallow: /admin/
# Block BigCommerce faceted search noise
Disallow: /*?_bc_fsnf=1*
Disallow: /*&_bc_fsnf=1*
# Everything else is allowed
Allow: /

# --------------------------------
# Major search engines (allowed)
# --------------------------------
User-agent: Googlebot
Allow: /
User-agent: Googlebot-Image
Allow: /
User-agent: Googlebot-Mobile
Allow: /
User-agent: Googlebot-News
Allow: /
User-agent: Googlebot-Video
Allow: /
User-agent: GoogleOther
Allow: /
User-agent: Google-InspectionTool
Allow: /
User-agent: Bingbot
Allow: /
User-agent: Slurp
Allow: /
User-agent: DuckDuckBot
Allow: /
User-agent: Baiduspider
Allow: /
User-agent: Applebot
Allow: /
User-agent: YandexBot
Disallow: /   # Low value for your market

# --------------------------------
# AI and assistant crawlers (opt-in)
# --------------------------------
User-agent: ChatGPT-User
Allow: /
User-agent: OAI-SearchBot
Allow: /
User-agent: GPTBot
Allow: /
User-agent: PerplexityBot
Allow: /
User-agent: Perplexity-User
Allow: /
User-agent: Claude-SearchBot
Allow: /
User-agent: Claude-User
Allow: /
User-agent: ClaudeBot
Allow: /
User-agent: anthropic-ai
Allow: /
User-agent: Google-Extended
Allow: /
User-agent: Google-CloudVertexBot
Allow: /
User-agent: GoogleOther-Image
Allow: /
User-agent: GoogleOther-Video
Allow: /
User-agent: Amazonbot
Allow: /
User-agent: FacebookBot
Allow: /
User-agent: facebookexternalhit
Allow: /
User-agent: DuckAssistBot
Allow: /
User-agent: MistralAI-User
Allow: /
User-agent: YouBot
Allow: /
User-agent: Meta-ExternalFetcher
Allow: /
User-agent: Meta-ExternalAgent
Allow: /
User-agent: cohere-ai
Allow: /
User-agent: Diffbot
Allow: /
User-agent: AI2Bot
Allow: /
User-agent: Ai2Bot-Dolma
Allow: /
User-agent: Timpibot
Allow: /
User-agent: Scrapy
Allow: /
User-agent: img2dataset
Allow: /
User-agent: omgili
Allow: /
User-agent: omgilibot
Allow: /
User-agent: ICC-Crawler
Allow: /
User-agent: ISSCyberRiskCrawler
Allow: /
User-agent: ImagesiftBot
Allow: /
User-agent: Kangaroo Bot
Allow: /
User-agent: FriendlyCrawler
Allow: /
User-agent: VelenPublicWebCrawler
Allow: /
User-agent: Webzio-Extended
Allow: /
User-agent: Sidetrade indexer bot
Allow: /
User-agent: iaskspider/2.0
Allow: /
User-agent: Manus Bot
Allow: /
User-agent: ProRataInc
Allow: /
User-agent: Novellum AI Crawl
Allow: /
User-agent: Anchor Browser
Allow: /

# --------------------------------
# Known non-beneficial / noisy bots
# (safe to block for ecommerce)
# --------------------------------

# SEO scrapers / generic crawlers with little value
User-agent: AhrefsBot
Disallow: /
User-agent: AhrefsSiteAudit
Disallow: /
User-agent: MJ12bot
Disallow: /
User-agent: SemrushBot
Disallow: /
User-agent: DotBot
Disallow: /
User-agent: dotbot
Disallow: /
User-agent: BLEXBot
Disallow: /
User-agent: Barkrowler
Disallow: /
User-agent: SEOkicks-Robot
Disallow: /
User-agent: SEOkicks
Disallow: /
User-agent: SISTRIX
Disallow: /
User-agent: SISTRIX Crawler
Disallow: /
User-agent: MegaIndex.ru
Disallow: /
User-agent: Exabot
Disallow: /
User-agent: Rogerbot
Disallow: /

# Site rippers / downloaders / email harvesters
User-agent: EmailCollector
Disallow: /
User-agent: EmailSiphon
Disallow: /
User-agent: EmailWolf
Disallow: /
User-agent: ExtractorPro
Disallow: /
User-agent: WebBandit
Disallow: /
User-agent: WebCopier
Disallow: /
User-agent: Teleport Pro
Disallow: /
User-agent: HTTrack
Disallow: /
User-agent: WebZIP
Disallow: /
User-agent: WebReaper
Disallow: /
User-agent: SiteSnagger
Disallow: /
User-agent: WebStripper
Disallow: /
User-agent: WebSpider
Disallow: /
User-agent: WebWhacker
Disallow: /
User-agent: WebDownloader
Disallow: /

# Very generic agents that are rarely needed directly
User-agent: Wget
Disallow: /
User-agent: curl
Disallow: /
User-agent: cURL
Disallow: /

# Catch some generic "crawler" agents on sensitive endpoints only
User-agent: *crawler*
Disallow: /cart.php
Disallow: /checkout.php
Disallow: /checkout
Disallow: /account.php
Disallow: /login.php
Disallow: /orders
Disallow: /wishlist.php
Disallow: /compare

# -----------------------------
# Sitemaps
# -----------------------------
Sitemap: https://sendameal.com/xmlsitemap.php
# AI-focused llms.txt descriptor (optional but useful for LLMs)
Sitemap: https://sendameal.com/llms.txt
User-agent: *
Disallow: /*?_bc_fsnf=1*
Disallow: /*&_bc_fsnf=1*

User-agent: AI2Bot
User-agent: Ai2Bot-Dolma
User-agent: Amazonbot
User-agent: Applebot
User-agent: Applebot-Extended
User-agent: Bytespider
User-agent: CCBot
User-agent: ChatGPT-User
User-agent: Claude-Web
User-agent: ClaudeBot
User-agent: Diffbot
User-agent: FacebookBot
User-agent: FriendlyCrawler
User-agent: GPTBot
User-agent: Google-Extended
User-agent: GoogleOther
User-agent: GoogleOther-Image
User-agent: GoogleOther-Video
User-agent: ICC-Crawler
User-agent: ISSCyberRiskCrawler
User-agent: ImagesiftBot
User-agent: Kangaroo Bot
User-agent: Meta-ExternalAgent
User-agent: Meta-ExternalAds
User-agent: Meta-ExternalFetcher
User-agent: Meta-WebIndexer
User-agent: OAI-SearchBot
User-agent: PerplexityBot
User-agent: PetalBot
User-agent: Scrapy
User-agent: Sidetrade indexer bot
User-agent: Timpibot
User-agent: VelenPublicWebCrawler
User-agent: Webzio-Extended
User-agent: YouBot
User-agent: anthropic-ai
User-agent: cohere-ai
User-agent: facebookexternalhit
User-agent: iaskspider/2.0
User-agent: img2dataset
User-agent: omgili
User-agent: omgilibot
Crawl-delay: 10
Disallow: /account.php
Disallow: /cart.php
Disallow: /checkout.php
Disallow: /checkout
Disallow: /admin/
Disallow: /*?_bc_fsnf=1*
Disallow: /*&_bc_fsnf=1*