# For all robots User-agent: * # Block access to specific groups of pages in /en/ Disallow: /en/articles* Disallow: /en/*register/ Disallow: /en/*quickOrder/ Disallow: /en/*quickView* Disallow: /en/checkout Disallow: /en/flat-line/ Disallow: /en/rohs* # Block access to specific groups of pages in /fr/ Disallow: /fr/articles* Disallow: /fr/*register/ Disallow: /fr/*quickOrder/ Disallow: /fr/*quickView* Disallow: /fr/checkout Disallow: /fr/flat-line/ Disallow: /fr/rohs* Disallow: /invalidbase/ Disallow: */stock Disallow: */validCombis Allow: /en/cartridge* Allow: /fr/cartridge* # Allow search crawlers to discover the sitemap Sitemap: https://www.omega.ca/sitemap.xml # Block CazoodleBot as it does not present correct accept content headers User-agent: CazoodleBot Disallow: / # Block MJ12bot as it is just noise User-agent: MJ12bot Disallow: / # Block dotbot as it cannot parse base urls properly User-agent: dotbot/1.0 Disallow: / # Block Gigabot User-agent: Gigabot Disallow: /