#
# robots.txt for stormlord.ca
#

# No ads here. Shoo.
User-agent: Mediapartners-Google*
Disallow: /

# Misc non-search crawlers.
User-agent: UbiCrawler
Disallow: /

User-agent: DOC
Disallow: /

User-agent: Zao
Disallow: /

# Some bots known to be trouble, particularly those designed to copy entire
# sites.
User-agent: sitecheck.internetseer.com
Disallow: /

User-agent: Zealbot
Disallow: /

User-agent: MSIECrawler
Disallow: /

User-agent: SiteSnagger
Disallow: /

User-agent: WebStripper
Disallow: /

User-agent: WebCopier
Disallow: /

User-agent: Fetch
Disallow: /

User-agent: Offline Explorer
Disallow: /

User-agent: Teleport
Disallow: /

User-agent: TeleportPro
Disallow: /

User-agent: WebZIP
Disallow: /

User-agent: linko
Disallow: /

User-agent: HTTrack
Disallow: /

User-agent: Microsoft.URL.Control
Disallow: /

User-agent: Xenu
Disallow: /

User-agent: larbin
Disallow: /

User-agent: libwww
Disallow: /

User-agent: ZyBORG
Disallow: /

User-agent: Download Ninja
Disallow: /

User-agent: NPBot
Disallow: /

# Misc. downloaders.
User-agent: WebReaper
Disallow: /

User-agent: wget
Disallow: /

User-agent: grub-client
Disallow: /

# All the rest (mainly Googlebot, Slurp (Yahoo), and MSNBot (Bing)). Using wildcards and end-of-string characters.
User-agent: *
Crawl-delay: 2
Disallow: /cgi-bin/
Disallow: /code/
Disallow: /publish/
Disallow: /apple-touch-icon.png$
Disallow: /favicon.ico$
Disallow: /logo.png$
## Duplicate content
Disallow: /tags/
Disallow: /project/type/
Disallow: /*/feed$
Disallow: /*/revisions$
Disallow: /*/revisions/
## Useless, boring stuff.
Disallow: /admin/
Disallow: /comment/reply/
Disallow: /contact/
Disallow: /logout$
Disallow: /filter/tips$
Disallow: /node$
Disallow: /node/
Disallow: /*/edit$
Disallow: /project/user$
Disallow: /project/issues/user$
Disallow: /search/
Disallow: /user$
Disallow: /user/
## Drupal files/dirs
Disallow: /includes/
Disallow: /misc/
Disallow: /modules/
Disallow: /profiles/
Disallow: /scripts/
Disallow: /sites/
Disallow: /themes/
Disallow: /CHANGELOG.txt$
Disallow: /cron.php$
Disallow: /INSTALL.mysql.txt$
Disallow: /INSTALL.pgsql.txt$
Disallow: /install.php$
Disallow: /INSTALL.txt$
Disallow: /LICENSE.txt$
Disallow: /MAINTAINERS.txt$
Disallow: /update.php$
Disallow: /UPGRADE.txt$
Disallow: /xmlrpc.php$
## Legacy wiki paths (will remove once content is copied).
Disallow: /w/
Disallow: /wiki/Special:Search
Disallow: /wiki/Special%3ASearch
Disallow: /wiki/Special:Random
Disallow: /wiki/Special%3ARandom