# Specific for moz.com / rogerbot
User-agent: rogerbot
Crawl-delay: 3

# All other crawlers
User-agent: *
Allow: /
Disallow: /new_php_pages.php
Disallow: /admin/_download-entire-client-list.php
Disallow: /admin/export
Disallow: /admin/
Disallow: /admin
Disallow: /function_test
Disallow: /function_test/
Disallow: /quote
Disallow: /quote/
Disallow: /quote-download
Disallow: /quote-download/
Disallow: /include/
Disallow: /offers/
Disallow: /offers
Disallow: /cms/forum.php?fpost_id=*

# As rec. by DataDial
Disallow: /search
Disallow: /search/
Disallow: /order/public/step2.php
Disallow: /account/
Disallow: /cms/forum.php?reply=*
Disallow: *printview=true

# As rec. by Chillibyte (link spam)
Disallow: */?link=*

# As rec. by Cloudflare
Disallow: /cdn-cgi/*

Sitemap: https://www.stl-training.co.uk/sitemaps/gen/main
Sitemap: https://www.stl-training.co.uk/sitemaps/gen/syl
Sitemap: https://www.stl-training.co.uk/sitemaps/gen/cms
Sitemap: https://www.stl-training.co.uk/sitemaps/gen/blog
Sitemap: https://www.stl-training.co.uk/sitemaps/gen/fpost


#################################
# robots.txt - for search engines
# see www.robotstxt.org for more information
# Can also use the meta tag "robots" on each page:
# eg. <meta name="robots" content="INDEX,FOLLOW">
# eg. <meta name="robots" content="NOINDEX,NOFOLLOW">

# Examples:

# the following allows all agents to index our entire site
# User-agent: *
# Allow: /

# the following disallows Googlebot (only) to our entire images directory, including subdirectories
# ie. other robots will have access to /images/
# User-agent: Googlebot
# Disallow: /images/