# Grocefully Robots.txt
# Last updated: July 2026

# Allow all crawlers to access public pages
User-agent: *
Allow: /
Allow: /blog/
Allow: /blog/*

# Sitemaps are served by API routes and exposed via rewrites in next.config.js
# (e.g. /sitemap-uk-stores-1.xml -> /api/sitemap-uk-stores.xml). These Allow
# rules must come BEFORE the /api/ disallow below, otherwise any crawler that
# resolves the underlying path is blocked from the sitemaps it was pointed at.
Allow: /sitemap.xml
Allow: /sitemap-*.xml
Allow: /api/sitemap*

# Disallow admin/internal pages
Disallow: /studio/
Disallow: /studio/*
Disallow: /api/
Disallow: /api/*

# Paginated listing views serve byte-identical page-1 HTML: `?page=N` is a
# query parameter, not a distinct SSG route, and getStaticProps hardcodes
# page 1. Sort params likewise produce near-infinite permutations of the same
# grid. Keep crawl budget on the canonical listing URLs.
Disallow: /*?page=
Disallow: /*&page=
Disallow: /*?sort=
Disallow: /*&sort=

# Sitemap location
Sitemap: https://www.grocefully.com/sitemap.xml

# NOTE: `Crawl-delay` is deliberately NOT set (removed 2026-07-29).
# Google ignores it; Bing honours it, and a value of 1 caps Bing at roughly
# 86k pages/day — far below this site's URL count, leaving much of the
# catalogue permanently uncrawled. Only reinstate if a crawler is genuinely
# causing load problems.