User-agent: *
Crawl-delay: 10
Sitemap: https://www.casinolistings.com/sitemap.xml
# Directories
Disallow: /includes/
Disallow: /modules/
Disallow: /profiles/
Disallow: /scripts/
Disallow: /themes/
# Files
Disallow: /CHANGELOG.txt
Disallow: /cron.php
Disallow: /INSTALL.mysql.txt
Disallow: /INSTALL.pgsql.txt
Disallow: /install.php
Disallow: /INSTALL.txt
Disallow: /LICENSE.txt
Disallow: /MAINTAINERS.txt
Disallow: /update.php
Disallow: /UPGRADE.txt
Disallow: /xmlrpc.php
# Paths (clean URLs)
Disallow: /admin/
Disallow: /aggregator/
Disallow: /comment/
Disallow: /contact/
Disallow: /logout/
Disallow: /node/add/
Disallow: /search/
Disallow: /user/
# Paths (no clean URLs)
Disallow: /?q=admin/
Disallow: /?q=aggregator/
Disallow: /?q=comment/
Disallow: /?q=contact/
Disallow: /?q=logout/
Disallow: /?q=node/add/
Disallow: /?q=search/
Disallow: /?q=user/
# Prevent duplicate and unwanted content from being crawled
Disallow: /node/
Disallow: /print/
Disallow: /forward/
Disallow: /comment/
Disallow: /tracker/
Disallow: /book/print/
Disallow: /share/
Disallow: /*sort=
Disallow: /filter/
Disallow: /ads/
Disallow: /*size=
Disallow: /goto/
Disallow: /download/
Disallow: /component/
Disallow: /*?utm_
Disallow: /flag/
Disallow: /aa/
Disallow: /fivestar/
Disallow: /*?page=0,
Disallow: /hybridauth/
Disallow: /forums/
Disallow: /feedback/
Disallow: /game/

Allow: /lib/flags/*
Disallow: /lib/

# Bad bots and crawlers
User-agent: AhrefsBot
User-agent: dotbot
User-agent: MJ12bot
User-agent: rightlander
User-agent: rogerbot
User-agent: Scrapy
User-agent: Screaming Frog SEO Spider
User-agent: SearchmetricsBot
User-agent: SemrushBot
User-agent: sistrix
User-agent: SEOENGWorldBot
User-agent: spbot
User-agent: WorldBrewBot
Disallow: /