# robots.txt for http://www.sldictionary.org
##################################################
# Sign Language Dictionary
#
# For the advancement of deaf and hard-of-hearing communities around the world
# by providing a dictionary accessible to everyone.
#
# written by Dr. Jacob Tosado, c.2024
#
# Overview:
# The structure of this document is largely based on that of
# www.google.com and wikipedia.org [1, 4]. We generally encourage
# friendly, low-speed bots to view static pages, but not
# dynamically-generated ones. Recursion speed is a main deciding
# factor for whether a bot is disallowed. For example, Inktomi's
# Slurp/2.0 can react to the minimum delay between hits [1, 3].
# Please adjust the 'Crawl-delay' or other recursion limit of
# your bot to a low-speed for it to be allowed on this website.
#
# Other bots that have been identified as not being condusive to the
# mission of this website have also been disallowed. Examples are bots
# known to be used for advertising, mass downloading, or ignorance of
# SEO standards.
#
# References:
# [1] http://www.wikipedia.org/robots.txt
# [2] https://www.gnu.org/software/wget/
# [3] http://www.inktomi.com/
# [4] https://www.google.com/robots.txt
##################################################
User-agent: *
Disallow: /eng/dictionary/
Disallow: /eng/advanced_search

# Known to perform large amounts of spamming at https://en.wikipedia.org/?curid=NNNNNN
# and ignore their 429 response ratelimit [1];
# http://mj12bot.com/
User-agent: MJ12bot
Disallow: /

# Known to make too many requests per second [1].
User-agent: fast
Disallow: /

# Known to make too many hits per second [1];
# http://www.nameprotect.com/botinfo.html
User-agent: NPBot
Disallow: /

# Wget is known to be too fast in its recursive mode.
# Please use the --wait option to set the delay between hits [1,2].
User-agent: wget
Disallow: /

# This capture bot is known to download multitudes of pages with no public benefit [1];
# http://www.webreaper.net/
User-agent: WebReaper
Disallow: /

# Known advertising-related bots [1].
User-agent: Mediapartners-Google*
Disallow: /

# The following rules are applied to Crawlers that do obey rules, but are
# disallowed because they are known to not feed search engines [1].
User-agent: UbiCrawler
Disallow: /

User-agent: DOC
Disallow: /

User-agent: Zao
Disallow: /

# The following rules are applied to bots designed to copy entire sites [1].
User-agent: sitecheck.internetseer.com
Disallow: /

User-agent: Zealbot
Disallow: /

User-agent: MSIECrawler
Disallow: /

User-agent: SiteSnagger
Disallow: /

User-agent: WebStripper
Disallow: /

User-agent: WebCopier
Disallow: /

User-agent: Fetch
Disallow: /

User-agent: Offline Explorer
Disallow: /

User-agent: Teleport
Disallow: /

User-agent: TeleportPro
Disallow: /

User-agent: WebZIP
Disallow: /

User-agent: linko
Disallow: /

User-agent: HTTrack
Disallow: /

User-agent: Microsoft.URL.Control
Disallow: /

User-agent: Xenu
Disallow: /

User-agent: larbin
Disallow: /

User-agent: libwww
Disallow: /

User-agent: ZyBORG
Disallow: /

User-agent: Download Ninja
Disallow: /

# The 'grub' distributed client is known to behave *very* poorly [1].
User-agent: grub-client
Disallow: /

# Known to not follow robots.txt, but a rule has been added anyway [1].
User-agent: k2spider
Disallow: /

Sitemap: https://www.sldictionary.org/sitemap.xml