# Wildcard user-agent means this applies to all robots who bother to read it
User-agent: *
# Allow all pages by default
Allow: /
# Block certain pages that we don't care for robots to index
Disallow: /*search-results
Disallow: /*shared-content
Disallow: /*examples
Disallow: /*testing-section
Disallow: /*server-health
Disallow: /*error-404
Disallow: /*product-finder
# Allow the CSS folder, so Google's mobile-testing tools work properly
Allow: /*shared-content/css/*
# Slow requests from robots to once per 20 seconds. Note that Googlebot doesn't obey this, use their Webmaster Tools instead: https://www.google.com/webmasters/tools
Crawl-delay: 20
# Tell the robots about the XML Sitemap that is being generated
Sitemap: https://www.hbfuller.com/sitemap-website.xml
# Tell the robots that the preferred host is www.hbfuller.com over https
Host: https://www.hbfuller.com
# Tell robots that they can ignore these parameters in URLs with querystrings. Should help cut down on the amount of requests they have to make to the same page.
Clean-param: tags&page