#
# robots.txt
#
# This file is to prevent the crawling and indexing of certain parts
# of your site by web crawlers and spiders run by sites like Yahoo!
# and Google. By telling these "robots" where not to go on your site,
# you save bandwidth and server resources.
#
# This file will be ignored unless it is at the root of your host:
# Used:    http://example.com/robots.txt
# Ignored: http://example.com/site/robots.txt
#
# For more information about the robots.txt standard, see:
# http://www.robotstxt.org/robotstxt.html

User-agent: *
crawl-delay: 20

# Disallow unnecessary to scrape php files
Disallow: /app.php
Disallow: /common.php
Disallow: /common.php
Disallow: /common.php
Disallow: /common.php
Disallow: /config.php
Disallow: /cron.php
Disallow: /download.php
Disallow: /faq.php
Disallow: /mcp.php
Disallow: /posting.php
Disallow: /report.php
Disallow: /style.php
Disallow: /ucp.php
Disallow: /viewonline.php
Disallow: /memberlist.php
Disallow: /search.php

# Disallow unnecessary to scrape directories
Disallow: /styles
Disallow: /cache
Disallow: /includes
Disallow: /language
Disallow: /store
Disallow: /user


# Say no AI bots (Sadly, the worst bots will fake their user agent, or ignore this rule)
# Open AI ChatGPT
User-agent: GPTBot
Disallow: /

# Common Crawler
User-agent: CCbot
Disallow: /

# Anthropic Crawler for training
User-agent: anthropic-ai
Disallow: /

# Antrhopic Live Crawler for web app
User-agent: Claude-Web
Disallow: /

# Google Gemini AI
User-agent: Google-Extended
Disallow: /

# Babber.tech crawler
User-agent: barkrowler
crawl-delay: 20

#iAsk crawler
User-agent: iAskBot
Disallow: /

#iAsk crawler
User-agent: Amazonbot
Disallow: /