# robots.txt for Magento 1.9.x & 2.x / v1.7 2023-01-30 # Sitemap (uncomment, change and add language/shop specific sitemaps, if running on multiple domains # keep in mind sitemap can only point to own domain so something like sitemapindex.php is needed) # Sitemap: http://example.com/sitemap.xml # # Crawlers Setup User-agent: * # # Allow paging (unless paging inside a listing with more params, as disallowed below) Allow: /*?p= # # Directories (technical images only) Disallow: /media/captcha/ Disallow: /media/customer/ Disallow: /media/dhl/ Disallow: /media/downloadable/ Disallow: /media/import/ Disallow: /media/pdf/ Disallow: /media/sales/ Disallow: /media/tmp/ Disallow: /media/xmlconnect/ # # Paths (leading * to make work for single- and multilocale versions) Disallow: */index.php/ Disallow: */catalog/product_compare/ Disallow: */catalog/category/view/ Disallow: */catalog/product/view/ Disallow: */catalog/product/gallery/ Disallow: */catalogsearch/ Disallow: */control/ Disallow: */customer/ Disallow: */customize/ Disallow: */newsletter/ Disallow: */poll/ Disallow: */review/ Disallow: */sendfriend/ Disallow: */tag/ Disallow: */wishlist/ Disallow: */checkout/ Disallow: */onestepcheckout/ # # Do not crawl sub category pages that are sorted or filtered. # NB: Avoid wider rules like /*?* as these would block also assets with timestamp/version as parameter! # # These are more specific, pick what you need - and do not forget to add your custom filters! Disallow: /*?dir* Disallow: /*?limit* Disallow: /*?mode* Disallow: /*?___from_store=* Disallow: /*?___store=* Disallow: /*?cat=* Disallow: /*?q=* Disallow: /*?price=* Disallow: /*?availability=* Disallow: /*?brand=* # # Paths that can be safely ignored (no clean URLs) Disallow: /*?p=*& Disallow: /*.php$ Disallow: /*?SID=