# robots.txt for http://www.wikipedia.org/ and friends
#
# Please note: There are a lot of pages on this site, and there are
# some misbehaved spiders out there that go _way_ too fast. If you're
# irresponsible, your access to the site may be blocked.
#

# Observed spamming large amounts of https://en.wikipedia.org/?curid=NNNNNN
# and ignoring 429 ratelimit responses, claims to respect robots:
# http://mj12bot.com/
User-agent: MJ12bot
Disallow: /

# advertising-related bots:
User-agent: Mediapartners-Google*
Disallow: /

# Wikipedia work bots:
User-agent: IsraBot
Disallow:

User-agent: Orthogaffe
Disallow:

# Crawlers that are kind enough to obey, but which we'd rather not have
# unless they're feeding search engines.
User-agent: UbiCrawler
Disallow: /

User-agent: DOC
Disallow: /

User-agent: Zao
Disallow: /

# Some bots are known to be trouble, particularly those designed to copy
# entire sites. Please obey robots.txt.
User-agent: sitecheck.internetseer.com
Disallow: /

User-agent: Zealbot
Disallow: /

User-agent: MSIECrawler
Disallow: /

User-agent: SiteSnagger
Disallow: /

User-agent: WebStripper
Disallow: /

User-agent: WebCopier
Disallow: /

User-agent: Fetch
Disallow: /

User-agent: Offline Explorer
Disallow: /

User-agent: Teleport
Disallow: /

User-agent: TeleportPro
Disallow: /

User-agent: WebZIP
Disallow: /

User-agent: linko
Disallow: /

User-agent: HTTrack
Disallow: /

User-agent: Microsoft.URL.Control
Disallow: /

User-agent: Xenu
Disallow: /

User-agent: larbin
Disallow: /

User-agent: libwww
Disallow: /

User-agent: ZyBORG
Disallow: /

User-agent: Download Ninja
Disallow: /

# Misbehaving: requests much too fast:
User-agent: fast
Disallow: /

#
# Sorry, wget in its recursive mode is a frequent problem.
# Please read the man page and use it properly; there is a
# --wait option you can use to set the delay between hits,
# for instance.
#
User-agent: wget
Disallow: /

#
# The 'grub' distributed client has been *very* poorly behaved.
#
User-agent: grub-client
Disallow: /

#
# Doesn't follow robots.txt anyway, but...
#
User-agent: k2spider
Disallow: /

#
# Hits many times per second, not acceptable
# http://www.nameprotect.com/botinfo.html
User-agent: NPBot
Disallow: /

# A capture bot, downloads gazillions of pages with no public benefit
# http://www.webreaper.net/
User-agent: WebReaper
Disallow: /

# Per their statement, semrushbot respects crawl-delay directives
# We want them to overall stay within reasonable request rates to
# the backend (20 rps); keeping in mind that the crawl-delay will
# be applied by site and not globally by the bot, 5 seconds seem
# like a reasonable approximation
User-agent: SemrushBot
Crawl-delay: 5

#
# Friendly, low-speed bots are welcome viewing article pages, but not
# dynamically-generated pages please.
#
# Inktomi's "Slurp" can read a minimum delay between hits; if your
# bot supports such a thing using the 'Crawl-delay' or another
# instruction, please let us know.
#
# There is a special exception for API mobileview to allow dynamic
# mobile web & app views to load section content.
# These views aren't HTTP-cached but use parser cache aggressively
# and don't expose special: pages etc.
#
# Another exception is for REST API documentation, located at
# /api/rest_v1/?doc.
#
User-agent: *
Allow: /w/api.php?action=mobileview&
Allow: /w/load.php?
Allow: /api/rest_v1/?doc
Allow: /w/rest.php/site/v1/sitemap
Disallow: /w/
Disallow: /api/
Disallow: /trap/
Disallow: /wiki/Special:
Disallow: /wiki/Spezial:
Disallow: /wiki/Spesial:
Disallow: /wiki/Special%3A
Disallow: /wiki/Spezial%3A
Disallow: /wiki/Spesial%3A
#
# ar:
Disallow: /wiki/%D8%AE%D8%A7%D8%B5:Search
Disallow: /wiki/%D8%AE%D8%A7%D8%B5%3ASearch
#
# dewiki:
# T6937
# sensible deletion and meta user discussion pages:
Disallow: /wiki/Wikipedia:L%C3%B6schkandidaten/
Disallow: /wiki/Wikipedia:Löschkandidaten/
Disallow: /wiki/Wikipedia:Vandalensperrung/
Disallow: /wiki/Wikipedia:Benutzersperrung/
Disallow: /wiki/Wikipedia:Vermittlungsausschuss/
Disallow: /wiki/Wikipedia:Administratoren/Probleme/
Disallow: /wiki/Wikipedia:Adminkandidaturen/
Disallow: /wiki/Wikipedia:Qualitätssicherung/
Disallow: /wiki/Wikipedia:Qualit%C3%A4tssicherung/
# 4937#5
Disallow: /wiki/Wikipedia:Vandalismusmeldung/
Disallow: /wiki/Wikipedia:Gesperrte_Lemmata/
Disallow: /wiki/Wikipedia:Löschprüfung/
Disallow: /wiki/Wikipedia:L%C3%B6schprüfung/
Disallow: /wiki/Wikipedia:Administratoren/Notizen/
Disallow: /wiki/Wikipedia:Schiedsgericht/Anfragen/
Disallow: /wiki/Wikipedia:L%C3%B6schpr%C3%BCfung/
# T14111
Disallow: /wiki/Wikipedia:Checkuser/
Disallow: /wiki/Wikipedia_Diskussion:Checkuser/
Disallow: /wiki/Wikipedia_Diskussion:Adminkandidaturen/
# T15961
Disallow: /wiki/Wikipedia:Spam-Blacklist-Log
Disallow: /wiki/Wikipedia%3ASpam-Blacklist-Log
Disallow: /wiki/Wikipedia_Diskussion:Spam-Blacklist-Log
Disallow: /wiki/Wikipedia_Diskussion%3ASpam-Blacklist-Log
#
# enwiki:
# Folks get annoyed when VfD discussions end up the number 1 google hit for
# their name. See T6776
Disallow: /wiki/Wikipedia:Articles_for_deletion/
Disallow: /wiki/Wikipedia%3AArticles_for_deletion/
Disallow: /wiki/Wikipedia:Votes_for_deletion/
Disallow: /wiki/Wikipedia%3AVotes_for_deletion/
Disallow: /wiki/Wikipedia:Pages_for_deletion/
Disallow: /wiki/Wikipedia%3APages_for_deletion/
Disallow: /wiki/Wikipedia:Miscellany_for_deletion/
Disallow: /wiki/Wikipedia%3AMiscellany_for_deletion/
Disallow: /wiki/Wikipedia:Miscellaneous_deletion/
Disallow: /wiki/Wikipedia%3AMiscellaneous_deletion/
Disallow: /wiki/Wikipedia:Copyright_problems
Disallow: /wiki/Wikipedia%3ACopyright_problems
Disallow: /wiki/Wikipedia:Protected_titles/
Disallow: /wiki/Wikipedia%3AProtected_titles/
# T15398
Disallow: /wiki/Wikipedia:WikiProject_Spam/
Disallow: /wiki/Wikipedia%3AWikiProject_Spam/
# T16075
Disallow: /wiki/MediaWiki:Spam-blacklist
Disallow: /wiki/MediaWiki%3ASpam-blacklist
Disallow: /wiki/MediaWiki_talk:Spam-blacklist
Disallow: /wiki/MediaWiki_talk%3ASpam-blacklist
# T13261
Disallow: /wiki/Wikipedia:Requests_for_arbitration/
Disallow: /wiki/Wikipedia%3ARequests_for_arbitration/
Disallow: /wiki/Wikipedia:Requests_for_comment/
Disallow: /wiki/Wikipedia%3ARequests_for_comment/
Disallow: /wiki/Wikipedia:Requests_for_adminship/
Disallow: /wiki/Wikipedia%3ARequests_for_adminship/
# T12288
Disallow: /wiki/Wikipedia_talk:Articles_for_deletion/
Disallow: /wiki/Wikipedia_talk%3AArticles_for_deletion/
Disallow: /wiki/Wikipedia_talk:Votes_for_deletion/
Disallow: /wiki/Wikipedia_talk%3AVotes_for_deletion/
Disallow: /wiki/Wikipedia_talk:Pages_for_deletion/
Disallow: /wiki/Wikipedia_talk%3APages_for_deletion/
Disallow: /wiki/Wikipedia_talk:Miscellany_for_deletion/
Disallow: /wiki/Wikipedia_talk%3AMiscellany_for_deletion/
Disallow: /wiki/Wikipedia_talk:Miscellaneous_deletion/
Disallow: /wiki/Wikipedia_talk%3AMiscellaneous_deletion/
# T16793
Disallow: /wiki/Wikipedia:Changing_username
Disallow: /wiki/Wikipedia%3AChanging_username
Disallow: /wiki/Wikipedia:Changing_username/
Disallow: /wiki/Wikipedia%3AChanging_username/
Disallow: /wiki/Wikipedia_talk:Changing_username
Disallow: /wiki/Wikipedia_talk%3AChanging_username
Disallow: /wiki/Wikipedia_talk:Changing_username/
Disallow: /wiki/Wikipedia_talk%3AChanging_username/
#
# eswiki:
# T8746
Disallow: /wiki/Wikipedia:Consultas_de_borrado/
Disallow: /wiki/Wikipedia%3AConsultas_de_borrado/
#
# fiwiki:
# T10695
Disallow: /wiki/Wikipedia:Poistettavat_sivut
Disallow: /wiki/K%C3%A4ytt%C3%A4j%C3%A4:
Disallow: /wiki/Käyttäjä:
Disallow: /wiki/Keskustelu_k%C3%A4ytt%C3%A4j%C3%A4st%C3%A4:
Disallow: /wiki/Keskustelu_käyttäjästä:
Disallow: /wiki/Wikipedia:Yll%C3%A4pit%C3%A4j%C3%A4t/
Disallow: /wiki/Wikipedia:Ylläpitäjät/
#
# hewiki:
Disallow: /wiki/%D7%9E%D7%99%D7%95%D7%97%D7%93:Search
Disallow: /wiki/%D7%9E%D7%99%D7%95%D7%97%D7%93%3ASearch
#T11517
Disallow: /wiki/ויקיפדיה:רשימת_מועמדים_למחיקה/
Disallow: /wiki/ויקיפדיה%3Aרשימת_מועמדים_למחיקה/
Disallow: /wiki/%D7%95%D7%99%D7%A7%D7%99%D7%A4%D7%93%D7%99%D7%94:%D7%A8%D7%A9%D7%99%D7%9E%D7%AA_%D7%9E%D7%95%D7%A2%D7%9E%D7%93%D7%99%D7%9D_%D7%9C%D7%9E%D7%97%D7%99%D7%A7%D7%94/
Disallow: /wiki/%D7%95%D7%99%D7%A7%D7%99%D7%A4%D7%93%D7%99%D7%94%3A%D7%A8%D7%A9%D7%99%D7%9E%D7%AA_%D7%9E%D7%95%D7%A2%D7%9E%D7%93%D7%99%D7%9D_%D7%9C%D7%9E%D7%97%D7%99%D7%A7%D7%94/
Disallow: /wiki/ויקיפדיה:ערכים_לא_קיימים_ומוגנים
Disallow: /wiki/ויקיפדיה%3Aערכים_לא_קיימים_ומוגנים
Disallow: /wiki/%D7%95%D7%99%D7%A7%D7%99%D7%A4%D7%93%D7%99%D7%94:%D7%A2%D7%A8%D7%9B%D7%99%D7%9D_%D7%9C%D7%90_%D7%A7%D7%99%D7%99%D7%9E%D7%99%D7%9D_%D7%95%D7%9E%D7%95%D7%92%D7%A0%D7%99%D7%9D
Disallow: /wiki/%D7%95%D7%99%D7%A7%D7%99%D7%A4%D7%93%D7%99%D7%94%3A%D7%A2%D7%A8%D7%9B%D7%99%D7%9D_%D7%9C%D7%90_%D7%A7%D7%99%D7%99%D7%9E%D7%99%D7%9D_%D7%95%D7%9E%D7%95%D7%92%D7%A0%D7%99%D7%9D
Disallow: /wiki/ויקיפדיה:דפים_לא_קיימים_ומוגנים
Disallow: /wiki/ויקיפדיה%3Aדפים_לא_קיימים_ומוגנים
Disallow: /wiki/%D7%95%D7%99%D7%A7%D7%99%D7%A4%D7%93%D7%99%D7%94:%D7%93%D7%A4%D7%99%D7%9D_%D7%9C%D7%90_%D7%A7%D7%99%D7%99%D7%9E%D7%99%D7%9D_%D7%95%D7%9E%D7%95%D7%92%D7%A0%D7%99%D7%9D
Disallow: /wiki/%D7%95%D7%99%D7%A7%D7%99%D7%A4%D7%93%D7%99%D7%94%3A%D7%93%D7%A4%D7%99%D7%9D_%D7%9C%D7%90_%D7%A7%D7%99%D7%99%D7%9E%D7%99%D7%9D_%D7%95%D7%9E%D7%95%D7%92%D7%A0%D7%99%D7%9D
#
# huwiki:
Disallow: /wiki/Speci%C3%A1lis:Search
Disallow: /wiki/Speci%C3%A1lis%3ASearch
#
# itwiki:
# T7545
Disallow: /wiki/Wikipedia:Pagine_da_cancellare
Disallow: /wiki/Wikipedia%3APagine_da_cancellare
Disallow: /wiki/Wikipedia:Utenti_problematici
Disallow: /wiki/Wikipedia%3AUtenti_problematici
Disallow: /wiki/Wikipedia:Vandalismi_in_corso
Disallow: /wiki/Wikipedia%3AVandalismi_in_corso
Disallow: /wiki/Wikipedia:Amministratori
Disallow: /wiki/Wikipedia%3AAmministratori
Disallow: /wiki/Wikipedia:Proposte_di_cancellazione_semplificata
Disallow: /wiki/Wikipedia%3AProposte_di_cancellazione_semplificata
Disallow: /wiki/Categoria:Da_cancellare_subito
Disallow: /wiki/Categoria%3ADa_cancellare_subito
Disallow: /wiki/Wikipedia:Sospette_violazioni_di_copyright
Disallow: /wiki/Wikipedia%3ASospette_violazioni_di_copyright
Disallow: /wiki/Categoria:Da_controllare_per_copyright
Disallow: /wiki/Categoria%3ADa_controllare_per_copyright
Disallow: /wiki/Progetto:Rimozione_contributi_sospetti
Disallow: /wiki/Progetto%3ARimozione_contributi_sospetti
Disallow: /wiki/Categoria:Da_cancellare_subito_per_violazione_integrale_copyright
Disallow: /wiki/Categoria%3ADa_cancellare_subito_per_violazione_integrale_copyright
Disallow: /wiki/Progetto:Cococo
Disallow: /wiki/Progetto%3ACococo
Disallow: /wiki/Discussioni_progetto:Cococo
Disallow: /wiki/Discussioni_progetto%3ACococo
#
# jawiki
Disallow: /wiki/%E7%89%B9%E5%88%A5:Search
Disallow: /wiki/%E7%89%B9%E5%88%A5%3ASearch
# T7239
Disallow: /wiki/Wikipedia:%E5%89%8A%E9%99%A4%E4%BE%9D%E9%A0%BC/
Disallow: /wiki/Wikipedia%3A%E5%89%8A%E9%99%A4%E4%BE%9D%E9%A0%BC/
Disallow: /wiki/Wikipedia:%E5%88%A9%E7%94%A8%E8%80%85%E3%83%9A%E3%83%BC%E3%82%B8%E3%81%AE%E5%89%8A%E9%99%A4%E4%BE%9D%E9%A0%BC
Disallow: /wiki/Wikipedia%3A%E5%88%A9%E7%94%A8%E8%80%85%E3%83%9A%E3%83%BC%E3%82%B8%E3%81%AE%E5%89%8A%E9%99%A4%E4%BE%9D%E9%A0%BC
# nowiki
# T13432
Disallow: /wiki/Bruker:
Disallow: /wiki/Bruker%3A
Disallow: /wiki/Brukerdiskusjon
Disallow: /wiki/Wikipedia:Administratorer
Disallow: /wiki/Wikipedia%3AAdministratorer
Disallow: /wiki/Wikipedia-diskusjon:Administratorer
Disallow: /wiki/Wikipedia-diskusjon%3AAdministratorer
Disallow: /wiki/Wikipedia:Sletting
Disallow: /wiki/Wikipedia%3ASletting
Disallow: /wiki/Wikipedia-diskusjon:Sletting
Disallow: /wiki/Wikipedia-diskusjon%3ASletting
#
# plwiki
# T10067
Disallow: /wiki/Wikipedia:Strony_do_usuni%C4%99cia
Disallow: /wiki/Wikipedia%3AStrony_do_usuni%C4%99cia
Disallow: /wiki/Wikipedia:Do_usuni%C4%99cia
Disallow: /wiki/Wikipedia%3ADo_usuni%C4%99cia
Disallow: /wiki/Wikipedia:SDU/
Disallow: /wiki/Wikipedia%3ASDU/
Disallow: /wiki/Wikipedia:Strony_podejrzane_o_naruszenie_praw_autorskich
Disallow: /wiki/Wikipedia%3AStrony_podejrzane_o_naruszenie_praw_autorskich
#
# ptwiki:
# T7394
Disallow: /wiki/Wikipedia:Páginas_para_eliminar/
Disallow: /wiki/Wikipedia:P%C3%A1ginas_para_eliminar/
Disallow: /wiki/Wikipedia%3AP%C3%A1ginas_para_eliminar/
Disallow: /wiki/Wikipedia_Discussão:Páginas_para_eliminar/
Disallow: /wiki/Wikipedia_Discuss%C3%A3o:P%C3%A1ginas_para_eliminar/
Disallow: /wiki/Wikipedia_Discuss%C3%A3o%3AP%C3%A1ginas_para_eliminar/
#
# rowiki:
# T14546
Disallow: /wiki/Wikipedia:Pagini_de_%C5%9Fters
Disallow: /wiki/Wikipedia%3APagini_de_%C5%9Fters
Disallow: /wiki/Discu%C5%A3ie_Wikipedia:Pagini_de_%C5%9Fters
Disallow: /wiki/Discu%C5%A3ie_Wikipedia%3APagini_de_%C5%9Fters
#
# ruwiki:
Disallow: /wiki/%D0%A1%D0%BF%D0%B5%D1%86%D0%B8%D0%B0%D0%BB%D1%8C%D0%BD%D1%8B%D0%B5:Search
Disallow: /wiki/%D0%A1%D0%BF%D0%B5%D1%86%D0%B8%D0%B0%D0%BB%D1%8C%D0%BD%D1%8B%D0%B5%3ASearch
#
# svwiki:
# T12229
Disallow: /wiki/Wikipedia%3ASidor_f%C3%B6reslagna_f%C3%B6r_radering
Disallow: /wiki/Wikipedia:Sidor_f%C3%B6reslagna_f%C3%B6r_radering
Disallow: /wiki/Wikipedia:Sidor_föreslagna_för_radering
Disallow: /wiki/Användare
Disallow: /wiki/Anv%C3%A4ndare
Disallow: /wiki/Användardiskussion
Disallow: /wiki/Anv%C3%A4ndardiskussion
Disallow: /wiki/Wikipedia:Skyddade_sidnamn
Disallow: /wiki/Wikipedia%3ASkyddade_sidnamn
# T13291
Disallow: /wiki/Wikipedia:Sidor_som_bör_raderas
Disallow: /wiki/Wikipedia:Sidor_som_b%C3%B6r_raderas
Disallow: /wiki/Wikipedia%3ASidor_som_b%C3%B6r_raderas
#
# zhwiki:
# T7104
Disallow: /wiki/Wikipedia:删除投票/侵权
Disallow: /wiki/Wikipedia:%E5%88%A0%E9%99%A4%E6%8A%95%E7%A5%A8/%E4%BE%B5%E6%9D%83
Disallow: /wiki/Wikipedia:删除投票和请求
Disallow: /wiki/Wikipedia:%E5%88%A0%E9%99%A4%E6%8A%95%E7%A5%A8%E5%92%8C%E8%AF%B7%E6%B1%82
Disallow: /wiki/Category:快速删除候选
Disallow: /wiki/Category:%E5%BF%AB%E9%80%9F%E5%88%A0%E9%99%A4%E5%80%99%E9%80%89
Disallow: /wiki/Category:维基百科需要翻译的文章
Disallow: /wiki/Category:%E7%BB%B4%E5%9F%BA%E7%99%BE%E7%A7%91%E9%9C%80%E8%A6%81%E7%BF%BB%E8%AF%91%E7%9A%84%E6%96%87%E7%AB%A0
#
# sister projects
#
# enwikinews:
# T7340
Disallow: /wiki/Portal:Prepared_stories/
Disallow: /wiki/Portal%3APrepared_stories/
#
# itwikinews
# T11138
Disallow: /wiki/Wikinotizie:Richieste_di_cancellazione
Disallow: /wiki/Wikinotizie:Sospette_violazioni_di_copyright
Disallow: /wiki/Categoria:Da_cancellare_subito
Disallow: /wiki/Categoria:Da_cancellare_subito_per_violazione_integrale_copyright
Disallow: /wiki/Wikinotizie:Storie_in_preparazione
#
# enwikiquote:
# T17095
Disallow: /wiki/Wikiquote:Votes_for_deletion/
Disallow: /wiki/Wikiquote%3AVotes_for_deletion/
Disallow: /wiki/Wikiquote_talk:Votes_for_deletion/
Disallow: /wiki/Wikiquote_talk%3AVotes_for_deletion/
Disallow: /wiki/Wikiquote:Votes_for_deletion_archive/
Disallow: /wiki/Wikiquote%3AVotes_for_deletion_archive/
Disallow: /wiki/Wikiquote_talk:Votes_for_deletion_archive/
Disallow: /wiki/Wikiquote_talk%3AVotes_for_deletion_archive/
#
# enwikibooks
Disallow: /wiki/Wikibooks:Votes_for_deletion
#
# working...
Disallow: /wiki/Fundraising_2007/comments
#
#
#
#----------------------------------------------------------#
#
#
#
 # <!--leave this line alone--><source lang="robots">
 # robots.txt for https://commons.wikimedia.org/
 #
 # Please note: There are a lot of pages on this site, and there are
 # some misbehaved spiders out there that go _way_ too fast. If you're
 # irresponsible, your access to the site may be blocked.
 #
Disallow: /wiki/Special:Random
Disallow: /wiki/Special%3ARandom
Disallow: /wiki/Special:Search
Disallow: /wiki/Special%3ASearch
#
#
# Folks get annoyed when deletion discussions end up
# the number 1 google hit for their name.
Disallow: /wiki/Commons:Deletion_requests/
Disallow: /wiki/Commons%3ADeletion_requests/
Disallow: /wiki/Commons:Categories_for_discussion/
Disallow: /wiki/Commons%3ACategories_for_discussion/
#
# Changing username might be annoying too
Disallow: /wiki/Commons:Changing_username
Disallow: /wiki/Commons%3AChanging_username
Disallow: /wiki/Commons_talk:Changing_username
Disallow: /wiki/Commons_talk%3AChanging_username
#
# Potentially sensitive discussions
Disallow: /wiki/Commons:Administrators'_noticeboard
Disallow: /wiki/Commons:Administrators%27_noticeboard
Disallow: /wiki/Commons%3AAdministrators%27_noticeboard
Disallow: /wiki/Commons_talk:Administrators%27_noticeboard
Disallow: /wiki/Commons_talk%3AAdministrators%27_noticeboard
#
# Category:Noindexed page
Disallow: /wiki/Category:Noindexed_pages
Disallow: /wiki/Category%3ANoindexed_pages
Disallow: /wiki/Category:Noindexed_user_pages
Disallow: /wiki/Category%3ANoindexed_user_pages
#
# Commons:Requests for checkuser
Disallow: /wiki/Commons:Requests_for_checkuser/
Disallow: /wiki/Commons%3ARequests_for_checkuser/
#
# Permission-related
Disallow: /wiki/Commons:Requests_and_votes/
Disallow: /wiki/Commons%3ARequests_and_votes/
Disallow: /wiki/Commons:Administrators/Requests/
Disallow: /wiki/Commons%3AAdministrators/Requests/
Disallow: /wiki/Commons:Administrators/Inactivity_section
Disallow: /wiki/Commons%3AAdministrators/Inactivity_section
Disallow: /wiki/Commons:Bureaucrats/Requests/
Disallow: /wiki/Commons%3ABureaucrats/Requests/
Disallow: /wiki/Commons:Checkusers/Requests/
Disallow: /wiki/Commons%3ACheckusers/Requests/
Disallow: /wiki/Commons:Oversighters/Requests/
Disallow: /wiki/Commons%3AOversighters/Requests/
Disallow: /wiki/Commons:License_review/Requests/
Disallow: /wiki/Commons%3ALicense_review/Requests/
Disallow: /wiki/Commons:Requests_for_rights/
Disallow: /wiki/Commons%3ARequests_for_rights/
# </source><!--leave this line alone-->


# robots.txt for http://www.wikipedia.org/ and friends
#
# Please note: There are a lot of pages on this site, and there are
# some misbehaved spiders out there that go _way_ too fast. If you're
# irresponsible, your access to the site may be blocked.
#

# Observed spamming large amounts of https://en.wikipedia.org/?curid=NNNNNN
# and ignoring 429 ratelimit responses, claims to respect robots:
# http://mj12bot.com/
User-agent: MJ12bot
Disallow: /

# advertising-related bots:
User-agent: Mediapartners-Google*
Disallow: /

# Wikipedia work bots:
User-agent: IsraBot
Disallow:

User-agent: Orthogaffe
Disallow:

# Crawlers that are kind enough to obey, but which we'd rather not have
# unless they're feeding search engines.
User-agent: UbiCrawler
Disallow: /

User-agent: DOC
Disallow: /

User-agent: Zao
Disallow: /

# Some bots are known to be trouble, particularly those designed to copy
# entire sites. Please obey robots.txt.
User-agent: sitecheck.internetseer.com
Disallow: /

User-agent: Zealbot
Disallow: /

User-agent: MSIECrawler
Disallow: /

User-agent: SiteSnagger
Disallow: /

User-agent: WebStripper
Disallow: /

User-agent: WebCopier
Disallow: /

User-agent: Fetch
Disallow: /

User-agent: Offline Explorer
Disallow: /

User-agent: Teleport
Disallow: /

User-agent: TeleportPro
Disallow: /

User-agent: WebZIP
Disallow: /

User-agent: linko
Disallow: /

User-agent: HTTrack
Disallow: /

User-agent: Microsoft.URL.Control
Disallow: /

User-agent: Xenu
Disallow: /

User-agent: larbin
Disallow: /

User-agent: libwww
Disallow: /

User-agent: ZyBORG
Disallow: /

User-agent: Download Ninja
Disallow: /

# Misbehaving, requests much too fast
User-agent: fast
Disallow: /

# Sorry, wget in its recursive mode is a frequent problem.
# Please read the man page and use it properly; there is a
# --wait option you can use to set the delay between hits,
# for instance.
User-agent: wget
Disallow: /

# The 'grub' distributed client has been *very* poorly behaved.
User-agent: grub-client
Disallow: /

# Doesn't follow robots.txt anyway, but...
User-agent: k2spider
Disallow: /

# Hits many times per second, not acceptable
# http://www.nameprotect.com/botinfo.html
User-agent: NPBot
Disallow: /

# A capture bot, downloads gazillions of pages with no public benefit
# http://www.webreaper.net/
User-agent: WebReaper
Disallow: /

# Per their statement, semrushbot respects crawl-delay directives
# We want them to overall stay within reasonable request rates to
# the backend (20 rps); keeping in mind that the crawl-delay will
# be applied by site and not globally by the bot, 5 seconds seem
# like a reasonable approximation
User-agent: SemrushBot
Crawl-delay: 5

#
# Friendly, low-speed bots are welcome viewing article pages, but not
# dynamically-generated pages please.
#
# Inktomi's "Slurp" can read a minimum delay between hits; if your
# bot supports such a thing using the 'Crawl-delay' or another
# instruction, please let us know.
#
# There is a special exception for API mobileview to allow dynamic
# mobile web & app views to load section content.
# These views aren't HTTP-cached but use parser cache aggressively
# and don't expose Special-pages and other uncached content.
#
# Another exception is for REST API documentation, located at
# /api/rest_v1/?doc.
#
User-agent: *
Allow: /w/api.php?action=mobileview&
Allow: /w/load.php?
Allow: /api/rest_v1/?doc
Allow: /w/rest.php/site/v1/sitemap
Disallow: /w/
Disallow: /api/
Disallow: /trap/
Disallow: /wiki/Special:
Disallow: /wiki/Spezial:
Disallow: /wiki/Spesial:
Disallow: /wiki/Special%3A
Disallow: /wiki/Spezial%3A
Disallow: /wiki/Spesial%3A
#
# ar:
Disallow: /wiki/%D8%AE%D8%A7%D8%B5:Search
Disallow: /wiki/%D8%AE%D8%A7%D8%B5%3ASearch
#
# dewiki:
# T6937
# sensible deletion and meta user discussion pages:
Disallow: /wiki/Wikipedia:L%C3%B6schkandidaten/
Disallow: /wiki/Wikipedia:Löschkandidaten/
Disallow: /wiki/Wikipedia:Vandalensperrung/
Disallow: /wiki/Wikipedia:Benutzersperrung/
Disallow: /wiki/Wikipedia:Vermittlungsausschuss/
Disallow: /wiki/Wikipedia:Administratoren/Probleme/
Disallow: /wiki/Wikipedia:Adminkandidaturen/
Disallow: /wiki/Wikipedia:Qualitätssicherung/
Disallow: /wiki/Wikipedia:Qualit%C3%A4tssicherung/
# 4937#5
Disallow: /wiki/Wikipedia:Vandalismusmeldung/
Disallow: /wiki/Wikipedia:Gesperrte_Lemmata/
Disallow: /wiki/Wikipedia:Löschprüfung/
Disallow: /wiki/Wikipedia:L%C3%B6schprüfung/
Disallow: /wiki/Wikipedia:Administratoren/Notizen/
Disallow: /wiki/Wikipedia:Schiedsgericht/Anfragen/
Disallow: /wiki/Wikipedia:L%C3%B6schpr%C3%BCfung/
# T14111
Disallow: /wiki/Wikipedia:Checkuser/
Disallow: /wiki/Wikipedia_Diskussion:Checkuser/
Disallow: /wiki/Wikipedia_Diskussion:Adminkandidaturen/
# T15961
Disallow: /wiki/Wikipedia:Spam-Blacklist-Log
Disallow: /wiki/Wikipedia%3ASpam-Blacklist-Log
Disallow: /wiki/Wikipedia_Diskussion:Spam-Blacklist-Log
Disallow: /wiki/Wikipedia_Diskussion%3ASpam-Blacklist-Log
#
# enwiki:
# Folks get annoyed when VfD discussions end up the number 1 google hit for
# their name. See T6776
Disallow: /wiki/Wikipedia:Articles_for_deletion/
Disallow: /wiki/Wikipedia%3AArticles_for_deletion/
Disallow: /wiki/Wikipedia:Votes_for_deletion/
Disallow: /wiki/Wikipedia%3AVotes_for_deletion/
Disallow: /wiki/Wikipedia:Pages_for_deletion/
Disallow: /wiki/Wikipedia%3APages_for_deletion/
Disallow: /wiki/Wikipedia:Miscellany_for_deletion/
Disallow: /wiki/Wikipedia%3AMiscellany_for_deletion/
Disallow: /wiki/Wikipedia:Miscellaneous_deletion/
Disallow: /wiki/Wikipedia%3AMiscellaneous_deletion/
Disallow: /wiki/Wikipedia:Copyright_problems
Disallow: /wiki/Wikipedia%3ACopyright_problems
Disallow: /wiki/Wikipedia:Protected_titles/
Disallow: /wiki/Wikipedia%3AProtected_titles/
# T15398
Disallow: /wiki/Wikipedia:WikiProject_Spam/
Disallow: /wiki/Wikipedia%3AWikiProject_Spam/
# T16075
Disallow: /wiki/MediaWiki:Spam-blacklist
Disallow: /wiki/MediaWiki%3ASpam-blacklist
Disallow: /wiki/MediaWiki_talk:Spam-blacklist
Disallow: /wiki/MediaWiki_talk%3ASpam-blacklist
# T13261
Disallow: /wiki/Wikipedia:Requests_for_arbitration/
Disallow: /wiki/Wikipedia%3ARequests_for_arbitration/
Disallow: /wiki/Wikipedia:Requests_for_comment/
Disallow: /wiki/Wikipedia%3ARequests_for_comment/
Disallow: /wiki/Wikipedia:Requests_for_adminship/
Disallow: /wiki/Wikipedia%3ARequests_for_adminship/
# T12288
Disallow: /wiki/Wikipedia_talk:Articles_for_deletion/
Disallow: /wiki/Wikipedia_talk%3AArticles_for_deletion/
Disallow: /wiki/Wikipedia_talk:Votes_for_deletion/
Disallow: /wiki/Wikipedia_talk%3AVotes_for_deletion/
Disallow: /wiki/Wikipedia_talk:Pages_for_deletion/
Disallow: /wiki/Wikipedia_talk%3APages_for_deletion/
Disallow: /wiki/Wikipedia_talk:Miscellany_for_deletion/
Disallow: /wiki/Wikipedia_talk%3AMiscellany_for_deletion/
Disallow: /wiki/Wikipedia_talk:Miscellaneous_deletion/
Disallow: /wiki/Wikipedia_talk%3AMiscellaneous_deletion/
# T16793
Disallow: /wiki/Wikipedia:Changing_username
Disallow: /wiki/Wikipedia%3AChanging_username
Disallow: /wiki/Wikipedia:Changing_username/
Disallow: /wiki/Wikipedia%3AChanging_username/
Disallow: /wiki/Wikipedia_talk:Changing_username
Disallow: /wiki/Wikipedia_talk%3AChanging_username
Disallow: /wiki/Wikipedia_talk:Changing_username/
Disallow: /wiki/Wikipedia_talk%3AChanging_username/
#
# eswiki:
# T8746
Disallow: /wiki/Wikipedia:Consultas_de_borrado/
Disallow: /wiki/Wikipedia%3AConsultas_de_borrado/
#
# fiwiki:
# T10695
Disallow: /wiki/Wikipedia:Poistettavat_sivut
Disallow: /wiki/K%C3%A4ytt%C3%A4j%C3%A4:
Disallow: /wiki/Käyttäjä:
Disallow: /wiki/Keskustelu_k%C3%A4ytt%C3%A4j%C3%A4st%C3%A4:
Disallow: /wiki/Keskustelu_käyttäjästä:
Disallow: /wiki/Wikipedia:Yll%C3%A4pit%C3%A4j%C3%A4t/
Disallow: /wiki/Wikipedia:Ylläpitäjät/
#
# hewiki:
Disallow: /wiki/%D7%9E%D7%99%D7%95%D7%97%D7%93:Search
Disallow: /wiki/%D7%9E%D7%99%D7%95%D7%97%D7%93%3ASearch
#T11517
Disallow: /wiki/ויקיפדיה:רשימת_מועמדים_למחיקה/
Disallow: /wiki/ויקיפדיה%3Aרשימת_מועמדים_למחיקה/
Disallow: /wiki/%D7%95%D7%99%D7%A7%D7%99%D7%A4%D7%93%D7%99%D7%94:%D7%A8%D7%A9%D7%99%D7%9E%D7%AA_%D7%9E%D7%95%D7%A2%D7%9E%D7%93%D7%99%D7%9D_%D7%9C%D7%9E%D7%97%D7%99%D7%A7%D7%94/
Disallow: /wiki/%D7%95%D7%99%D7%A7%D7%99%D7%A4%D7%93%D7%99%D7%94%3A%D7%A8%D7%A9%D7%99%D7%9E%D7%AA_%D7%9E%D7%95%D7%A2%D7%9E%D7%93%D7%99%D7%9D_%D7%9C%D7%9E%D7%97%D7%99%D7%A7%D7%94/
Disallow: /wiki/ויקיפדיה:ערכים_לא_קיימים_ומוגנים
Disallow: /wiki/ויקיפדיה%3Aערכים_לא_קיימים_ומוגנים
Disallow: /wiki/%D7%95%D7%99%D7%A7%D7%99%D7%A4%D7%93%D7%99%D7%94:%D7%A2%D7%A8%D7%9B%D7%99%D7%9D_%D7%9C%D7%90_%D7%A7%D7%99%D7%99%D7%9E%D7%99%D7%9D_%D7%95%D7%9E%D7%95%D7%92%D7%A0%D7%99%D7%9D
Disallow: /wiki/%D7%95%D7%99%D7%A7%D7%99%D7%A4%D7%93%D7%99%D7%94%3A%D7%A2%D7%A8%D7%9B%D7%99%D7%9D_%D7%9C%D7%90_%D7%A7%D7%99%D7%99%D7%9E%D7%99%D7%9D_%D7%95%D7%9E%D7%95%D7%92%D7%A0%D7%99%D7%9D
Disallow: /wiki/ויקיפדיה:דפים_לא_קיימים_ומוגנים
Disallow: /wiki/ויקיפדיה%3Aדפים_לא_קיימים_ומוגנים
Disallow: /wiki/%D7%95%D7%99%D7%A7%D7%99%D7%A4%D7%93%D7%99%D7%94:%D7%93%D7%A4%D7%99%D7%9D_%D7%9C%D7%90_%D7%A7%D7%99%D7%99%D7%9E%D7%99%D7%9D_%D7%95%D7%9E%D7%95%D7%92%D7%A0%D7%99%D7%9D
Disallow: /wiki/%D7%95%D7%99%D7%A7%D7%99%D7%A4%D7%93%D7%99%D7%94%3A%D7%93%D7%A4%D7%99%D7%9D_%D7%9C%D7%90_%D7%A7%D7%99%D7%99%D7%9E%D7%99%D7%9D_%D7%95%D7%9E%D7%95%D7%92%D7%A0%D7%99%D7%9D
#
# huwiki:
Disallow: /wiki/Speci%C3%A1lis:Search
Disallow: /wiki/Speci%C3%A1lis%3ASearch
#
# itwiki:
# T7545
Disallow: /wiki/Wikipedia:Pagine_da_cancellare
Disallow: /wiki/Wikipedia%3APagine_da_cancellare
Disallow: /wiki/Wikipedia:Utenti_problematici
Disallow: /wiki/Wikipedia%3AUtenti_problematici
Disallow: /wiki/Wikipedia:Vandalismi_in_corso
Disallow: /wiki/Wikipedia%3AVandalismi_in_corso
Disallow: /wiki/Wikipedia:Amministratori
Disallow: /wiki/Wikipedia%3AAmministratori
Disallow: /wiki/Wikipedia:Proposte_di_cancellazione_semplificata
Disallow: /wiki/Wikipedia%3AProposte_di_cancellazione_semplificata
Disallow: /wiki/Categoria:Da_cancellare_subito
Disallow: /wiki/Categoria%3ADa_cancellare_subito
Disallow: /wiki/Wikipedia:Sospette_violazioni_di_copyright
Disallow: /wiki/Wikipedia%3ASospette_violazioni_di_copyright
Disallow: /wiki/Categoria:Da_controllare_per_copyright
Disallow: /wiki/Categoria%3ADa_controllare_per_copyright
Disallow: /wiki/Progetto:Rimozione_contributi_sospetti
Disallow: /wiki/Progetto%3ARimozione_contributi_sospetti
Disallow: /wiki/Categoria:Da_cancellare_subito_per_violazione_integrale_copyright
Disallow: /wiki/Categoria%3ADa_cancellare_subito_per_violazione_integrale_copyright
Disallow: /wiki/Progetto:Cococo
Disallow: /wiki/Progetto%3ACococo
Disallow: /wiki/Discussioni_progetto:Cococo
Disallow: /wiki/Discussioni_progetto%3ACococo
#
# jawiki
Disallow: /wiki/%E7%89%B9%E5%88%A5:Search
Disallow: /wiki/%E7%89%B9%E5%88%A5%3ASearch
# T7239
Disallow: /wiki/Wikipedia:%E5%89%8A%E9%99%A4%E4%BE%9D%E9%A0%BC/
Disallow: /wiki/Wikipedia%3A%E5%89%8A%E9%99%A4%E4%BE%9D%E9%A0%BC/
Disallow: /wiki/Wikipedia:%E5%88%A9%E7%94%A8%E8%80%85%E3%83%9A%E3%83%BC%E3%82%B8%E3%81%AE%E5%89%8A%E9%99%A4%E4%BE%9D%E9%A0%BC
Disallow: /wiki/Wikipedia%3A%E5%88%A9%E7%94%A8%E8%80%85%E3%83%9A%E3%83%BC%E3%82%B8%E3%81%AE%E5%89%8A%E9%99%A4%E4%BE%9D%E9%A0%BC
# nowiki
# T13432
Disallow: /wiki/Bruker:
Disallow: /wiki/Bruker%3A
Disallow: /wiki/Brukerdiskusjon
Disallow: /wiki/Wikipedia:Administratorer
Disallow: /wiki/Wikipedia%3AAdministratorer
Disallow: /wiki/Wikipedia-diskusjon:Administratorer
Disallow: /wiki/Wikipedia-diskusjon%3AAdministratorer
Disallow: /wiki/Wikipedia:Sletting
Disallow: /wiki/Wikipedia%3ASletting
Disallow: /wiki/Wikipedia-diskusjon:Sletting
Disallow: /wiki/Wikipedia-diskusjon%3ASletting
#
# plwiki
# T10067
Disallow: /wiki/Wikipedia:Strony_do_usuni%C4%99cia
Disallow: /wiki/Wikipedia%3AStrony_do_usuni%C4%99cia
Disallow: /wiki/Wikipedia:Do_usuni%C4%99cia
Disallow: /wiki/Wikipedia%3ADo_usuni%C4%99cia
Disallow: /wiki/Wikipedia:SDU/
Disallow: /wiki/Wikipedia%3ASDU/
Disallow: /wiki/Wikipedia:Strony_podejrzane_o_naruszenie_praw_autorskich
Disallow: /wiki/Wikipedia%3AStrony_podejrzane_o_naruszenie_praw_autorskich
#
# ptwiki:
# T7394
Disallow: /wiki/Wikipedia:Páginas_para_eliminar/
Disallow: /wiki/Wikipedia:P%C3%A1ginas_para_eliminar/
Disallow: /wiki/Wikipedia%3AP%C3%A1ginas_para_eliminar/
Disallow: /wiki/Wikipedia_Discussão:Páginas_para_eliminar/
Disallow: /wiki/Wikipedia_Discuss%C3%A3o:P%C3%A1ginas_para_eliminar/
Disallow: /wiki/Wikipedia_Discuss%C3%A3o%3AP%C3%A1ginas_para_eliminar/
#
# rowiki:
# T14546
Disallow: /wiki/Wikipedia:Pagini_de_%C5%9Fters
Disallow: /wiki/Wikipedia%3APagini_de_%C5%9Fters
Disallow: /wiki/Discu%C5%A3ie_Wikipedia:Pagini_de_%C5%9Fters
Disallow: /wiki/Discu%C5%A3ie_Wikipedia%3APagini_de_%C5%9Fters
#
# ruwiki:
Disallow: /wiki/%D0%A1%D0%BF%D0%B5%D1%86%D0%B8%D0%B0%D0%BB%D1%8C%D0%BD%D1%8B%D0%B5:Search
Disallow: /wiki/%D0%A1%D0%BF%D0%B5%D1%86%D0%B8%D0%B0%D0%BB%D1%8C%D0%BD%D1%8B%D0%B5%3ASearch
#
# svwiki:
# T12229
Disallow: /wiki/Wikipedia%3ASidor_f%C3%B6reslagna_f%C3%B6r_radering
Disallow: /wiki/Wikipedia:Sidor_f%C3%B6reslagna_f%C3%B6r_radering
Disallow: /wiki/Wikipedia:Sidor_föreslagna_för_radering
Disallow: /wiki/Användare
Disallow: /wiki/Anv%C3%A4ndare
Disallow: /wiki/Användardiskussion
Disallow: /wiki/Anv%C3%A4ndardiskussion
Disallow: /wiki/Wikipedia:Skyddade_sidnamn
Disallow: /wiki/Wikipedia%3ASkyddade_sidnamn
# T13291
Disallow: /wiki/Wikipedia:Sidor_som_bör_raderas
Disallow: /wiki/Wikipedia:Sidor_som_b%C3%B6r_raderas
Disallow: /wiki/Wikipedia%3ASidor_som_b%C3%B6r_raderas
#
# zhwiki:
# T7104
Disallow: /wiki/Wikipedia:删除投票/侵权
Disallow: /wiki/Wikipedia:%E5%88%A0%E9%99%A4%E6%8A%95%E7%A5%A8/%E4%BE%B5%E6%9D%83
Disallow: /wiki/Wikipedia:删除投票和请求
Disallow: /wiki/Wikipedia:%E5%88%A0%E9%99%A4%E6%8A%95%E7%A5%A8%E5%92%8C%E8%AF%B7%E6%B1%82
Disallow: /wiki/Category:快速删除候选
Disallow: /wiki/Category:%E5%BF%AB%E9%80%9F%E5%88%A0%E9%99%A4%E5%80%99%E9%80%89
Disallow: /wiki/Category:维基百科需要翻译的文章
Disallow: /wiki/Category:%E7%BB%B4%E5%9F%BA%E7%99%BE%E7%A7%91%E9%9C%80%E8%A6%81%E7%BF%BB%E8%AF%91%E7%9A%84%E6%96%87%E7%AB%A0
#
# sister projects
#
# enwikinews:
# T7340
Disallow: /wiki/Portal:Prepared_stories/
Disallow: /wiki/Portal%3APrepared_stories/
#
# itwikinews
# T11138
Disallow: /wiki/Wikinotizie:Richieste_di_cancellazione
Disallow: /wiki/Wikinotizie:Sospette_violazioni_di_copyright
Disallow: /wiki/Categoria:Da_cancellare_subito
Disallow: /wiki/Categoria:Da_cancellare_subito_per_violazione_integrale_copyright
Disallow: /wiki/Wikinotizie:Storie_in_preparazione
#
# enwikiquote:
# T17095
Disallow: /wiki/Wikiquote:Votes_for_deletion/
Disallow: /wiki/Wikiquote%3AVotes_for_deletion/
Disallow: /wiki/Wikiquote_talk:Votes_for_deletion/
Disallow: /wiki/Wikiquote_talk%3AVotes_for_deletion/
Disallow: /wiki/Wikiquote:Votes_for_deletion_archive/
Disallow: /wiki/Wikiquote%3AVotes_for_deletion_archive/
Disallow: /wiki/Wikiquote_talk:Votes_for_deletion_archive/
Disallow: /wiki/Wikiquote_talk%3AVotes_for_deletion_archive/
#
# enwikibooks
Disallow: /wiki/Wikibooks:Votes_for_deletion
#

Sitemap: https://commons.wikimedia.org/w/rest.php/site/v1/sitemap/0

#
#
#----------------------------------------------------------#
#
#
#
 # <!--leave this line alone--><source lang="robots">
 # robots.txt for https://commons.wikimedia.org/
 #
 # Please note: There are a lot of pages on this site, and there are
 # some misbehaved spiders out there that go _way_ too fast. If you're
 # irresponsible, your access to the site may be blocked.
 #
Disallow: /wiki/Special:Random
Disallow: /wiki/Special%3ARandom
Disallow: /wiki/Special:Search
Disallow: /wiki/Special%3ASearch
#
#
# Folks get annoyed when deletion discussions end up
# the number 1 google hit for their name.
Disallow: /wiki/Commons:Deletion_requests/
Disallow: /wiki/Commons%3ADeletion_requests/
Disallow: /wiki/Commons:Categories_for_discussion/
Disallow: /wiki/Commons%3ACategories_for_discussion/
#
# Changing username might be annoying too
Disallow: /wiki/Commons:Changing_username
Disallow: /wiki/Commons%3AChanging_username
Disallow: /wiki/Commons_talk:Changing_username
Disallow: /wiki/Commons_talk%3AChanging_username
#
# Potentially sensitive discussions
Disallow: /wiki/Commons:Administrators'_noticeboard
Disallow: /wiki/Commons:Administrators%27_noticeboard
Disallow: /wiki/Commons%3AAdministrators%27_noticeboard
Disallow: /wiki/Commons_talk:Administrators%27_noticeboard
Disallow: /wiki/Commons_talk%3AAdministrators%27_noticeboard
#
# Category:Noindexed page
Disallow: /wiki/Category:Noindexed_pages
Disallow: /wiki/Category%3ANoindexed_pages
Disallow: /wiki/Category:Noindexed_user_pages
Disallow: /wiki/Category%3ANoindexed_user_pages
#
# Commons:Requests for checkuser
Disallow: /wiki/Commons:Requests_for_checkuser/
Disallow: /wiki/Commons%3ARequests_for_checkuser/
#
# Permission-related
Disallow: /wiki/Commons:Requests_and_votes/
Disallow: /wiki/Commons%3ARequests_and_votes/
Disallow: /wiki/Commons:Administrators/Requests/
Disallow: /wiki/Commons%3AAdministrators/Requests/
Disallow: /wiki/Commons:Administrators/Inactivity_section
Disallow: /wiki/Commons%3AAdministrators/Inactivity_section
Disallow: /wiki/Commons:Bureaucrats/Requests/
Disallow: /wiki/Commons%3ABureaucrats/Requests/
Disallow: /wiki/Commons:Checkusers/Requests/
Disallow: /wiki/Commons%3ACheckusers/Requests/
Disallow: /wiki/Commons:Oversighters/Requests/
Disallow: /wiki/Commons%3AOversighters/Requests/
Disallow: /wiki/Commons:License_review/Requests/
Disallow: /wiki/Commons%3ALicense_review/Requests/
Disallow: /wiki/Commons:Requests_for_rights/
Disallow: /wiki/Commons%3ARequests_for_rights/
# </source><!--leave this line alone-->


Sitemap: https://concepticon.clld.org/sitemap.xml


user-agent: *
Disallow: /area/
Allow: /area/_search$
Disallow: /bob/
Allow: /bob/_search$
Disallow: /cttest/
Allow: /cttest/_search$
Disallow: /ebob/
Allow: /ebob/_search$
Disallow: /estc/
Allow: /estc/_search$
Disallow: /estc_holdings/
Allow: /estc_holdings/_search$
Disallow: /estc_libraries/
Allow: /estc_libraries/_search$
Disallow: /holdinst/
Allow: /holdinst/_search$
Disallow: /hpbprov/
Allow: /hpbprov/_search$
Disallow: /ipi/
Allow: /ipi/_search$
Disallow: /istc/
Allow: /istc/_search$
Disallow: /kind/
Allow: /kind/_search$
Disallow: /kjd/
Allow: /kjd/_search$
Disallow: /kjd_class/
Allow: /kjd_class/_search$
Disallow: /kjd_top/
Allow: /kjd_top/_search$
Disallow: /mabfn/
Allow: /mabfn/_search$
Disallow: /matmed/
Allow: /matmed/_search$
Disallow: /mei/
Allow: /mei/_search$
Disallow: /owners/
Allow: /owners/_search$
Disallow: /patrimonit/
Allow: /patrimonit/_search$
Disallow: /polish-provenance/
Allow: /polish-provenance/_search$
Disallow: /sbti/
Allow: /sbti/_search$
Disallow: /siebmacher/
Allow: /siebmacher/_search$
Disallow: /stcn/
Allow: /stcn/_search$
Disallow: /stcn_copy/
Allow: /stcn_copy/_search$
Disallow: /stcn_lib/
Allow: /stcn_lib/_search$
Disallow: /stcn_persons/
Allow: /stcn_persons/_search$
Disallow: /stcn_printers/
Allow: /stcn_printers/_search$
Disallow: /stcn_subjects/
Allow: /stcn_subjects/_search$
Disallow: /thesaurus/
Allow: /thesaurus/_search$
Disallow: /zurich_prints/
Allow: /zurich_prints/_search$


User-agent: *
Disallow: /cgi-bin/
Disallow: /gistemp/graphs/
Disallow: /gistemp/station_data_v2/
Disallow: /gfx/
Disallow: /modelE/transient/
Disallow: /pub/
Disallow: /tmp/


User-agent: *
Disallow: /cgi-bin/
Disallow: /gistemp/graphs/
Disallow: /gistemp/graphs_v3/
Disallow: /gistemp/graphs_v4/
Disallow: /gistemp/station_data_v2/
Disallow: /gistemp/station_data_v3/
Disallow: /gistemp/station_data_v4/
Disallow: /gistemp/station_data_v4_globe/
Disallow: /gistemp/stdata/
Disallow: /gfx/
Disallow: /modelE/transient/
Disallow: /pub/
Disallow: /tmp/


User-agent: *
Disallow: /cgi-bin/
Disallow: /gistemp/graphs/
Disallow: /gistemp/graphs_v3/
Disallow: /gistemp/graphs_v4/
Disallow: /gistemp/station_data_v2/
Disallow: /gistemp/station_data_v3/
Disallow: /gistemp/station_data_v4/
Disallow: /gistemp/station_data_v4_globe/
Disallow: /gistemp/stdata/
Disallow: /gistemp/tabledata_v4/
Disallow: /gfx/
Disallow: /modelE/transient/
Disallow: /pub/
Disallow: /tmp/
Disallow: /work/


User-agent: *
Disallow: /cgi-bin/
Disallow: /cleared/
Disallow: /gistemp/graphs/
Disallow: /gistemp/graphs_v3/
Disallow: /gistemp/graphs_v4/
Disallow: /gistemp/station_data_v2/
Disallow: /gistemp/station_data_v3/
Disallow: /gistemp/station_data_v4/
Disallow: /gistemp/station_data_v4_globe/
Disallow: /gistemp/stdata/
Disallow: /gistemp/tabledata_v4/
Disallow: /gfx/
Disallow: /modelE/transient/
Disallow: /pub/
Disallow: /tmp/


User-agent: *
Disallow: /cgi-bin/
Disallow: /cleared/
Disallow: /gistemp/graphs/
Disallow: /gistemp/graphs_v3/
Disallow: /gistemp/graphs_v4/
Disallow: /gistemp/station_data_v2/
Disallow: /gistemp/station_data_v3/
Disallow: /gistemp/station_data_v4/
Disallow: /gistemp/station_data_v4_globe/
Disallow: /gistemp/stdata/
Disallow: /gistemp/tabledata_v4/
Disallow: /gfx/
Disallow: /modelE/transient/
Disallow: /pub/
Disallow: /tmp/

User-agent: Bingbot
Disallow: /cgi-bin/
Disallow: /cleared/
Disallow: /gistemp/graphs/
Disallow: /gistemp/graphs_v3/
Disallow: /gistemp/graphs_v4/
Disallow: /gistemp/station_data_v2/
Disallow: /gistemp/station_data_v3/
Disallow: /gistemp/station_data_v4/
Disallow: /gistemp/station_data_v4_globe/
Disallow: /gistemp/stdata/
Disallow: /gistemp/tabledata_v4/
Disallow: /gfx/
Disallow: /modelE/transient/
Disallow: /pub/
Disallow: /tmp/


User-agent: *
Disallow: /cgi-bin/
Disallow: /cleared/
Disallow: /gistemp/graphs/
Disallow: /gistemp/graphs_v3/
Disallow: /gistemp/graphs_v4/
Disallow: /gistemp/station_data_v2/
Disallow: /gistemp/station_data_v3/
Disallow: /gistemp/station_data_v4/
Disallow: /gistemp/station_data_v4_globe/
Disallow: /gistemp/stdata/
Disallow: /gistemp/tabledata_v4/
Disallow: /gfx/
Disallow: /modelE/transient/
Disallow: /pub/
Disallow: /tmp/
Disallow: /work/

User-agent: Bingbot
Disallow: /cgi-bin/
Disallow: /cleared/
Disallow: /gistemp/graphs/
Disallow: /gistemp/graphs_v3/
Disallow: /gistemp/graphs_v4/
Disallow: /gistemp/station_data_v2/
Disallow: /gistemp/station_data_v3/
Disallow: /gistemp/station_data_v4/
Disallow: /gistemp/station_data_v4_globe/
Disallow: /gistemp/stdata/
Disallow: /gistemp/tabledata_v4/
Disallow: /gfx/
Disallow: /modelE/transient/
Disallow: /pub/
Disallow: /tmp/
Disallow: /work/


# As a condition of accessing this website, you agree to abide by the following
# content signals:

# (a)  If a content-signal = yes, you may collect content for the corresponding
#      use.
# (b)  If a content-signal = no, you may not collect content for the
#      corresponding use.
# (c)  If the website operator does not include a content signal for a
#      corresponding use, the website operator neither grants nor restricts
#      permission via content signal with respect to the corresponding use.

# The content signals and their meanings are:

# search:   building a search index and providing search results (e.g., returning
#           hyperlinks and short excerpts from your website's contents). Search does not
#           include providing AI-generated search summaries.
# ai-input: inputting content into one or more AI models (e.g., retrieval
#           augmented generation, grounding, or other real-time taking of content for
#           generative AI search answers).
# ai-train: training or fine-tuning AI models.

# ANY RESTRICTIONS EXPRESSED VIA CONTENT SIGNALS ARE EXPRESS RESERVATIONS OF
# RIGHTS UNDER ARTICLE 4 OF THE EUROPEAN UNION DIRECTIVE 2019/790 ON COPYRIGHT
# AND RELATED RIGHTS IN THE DIGITAL SINGLE MARKET.


User-agent: *
Disallow: /Foyer/activateMembershipOneYear
Disallow: /Foyer/activateMembershipLifetime
Disallow: /exist/


# This file must be placed in the web root
# Meaning it must be accessible at http://dlib.nyu.edu/robots.txt
# If you don't see it there, the bots won't see it either!

User-agent: MegaIndex.ru
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: *
Disallow: /undercover/search/
Disallow: /files/books/
Disallow: /enm/
Disallow: /aco/book/*/display


User-agent: *
# Disable access to Discovery search and filters
Disallow: /files/nyupress/pdfs/*.pdf


User-agent: *
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php

Sitemap: https://earthsky.org/sitemapindex.xml

Sitemap: https://earthsky.org/sitemap.xml


User-agent: *
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-admin/

Sitemap: https://earthsky.org/sitemap.xml
Sitemap: https://earthsky.org/sitemap.rss
Sitemap: https://earthsky.org/sitemapindex.xml


User-agent: *
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php

User-agent: ChatGPT-User
Disallow: /

User-agent: GPTBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Claude-Web
Disallow: /

User-agent: PiplBot
Disallow: /

User-agent: CCbot
Disallow: /

User-agent: FacebookBot
Disallow: /

Sitemap: https://earthsky.org/sitemap.xml
Sitemap: https://earthsky.org/sitemap.rss


User-agent: *
Allow: /
Sitemap: https://eastasiastudent.net/sitemap.xml


User-agent: *
Disallow: /branch-snapshots/
Disallow: /commit-snapshots/
Disallow: /review-drafts/


# robots.txt for https://eprint.iacr.org/
# we also use other mechanisms to control crawling.
# for questions about this, send email to Kevin McCurley
User-agent: *
Disallow: /cgi-bin/
Disallow: /eprint-bin/
Disallow: /eprint-test-bin/
Crawl-delay: 20
User-agent: SemrushBot
Disallow: /
User-agent: SemrushBot-SA
Disallow: /
User-agent: AhrefsBot
Disallow: /
User-agent: SputnikBot
Disallow: /
User-agent: BLEXBot
Disallow: /


# robots.txt for https://eprint.iacr.org/
# we also use other mechanisms to control crawling.
# for questions about this, send email to Kevin McCurley
User-agent: *
Crawl-Delay: 9
User-agent: *
Disallow: /cgi-bin/
Disallow: /eprint-bin/
Disallow: /eprint-test-bin/
Crawl-delay: 20
User-agent: SemrushBot
Disallow: /
User-agent: SemrushBot-SA
Disallow: /
User-agent: AhrefsBot
Disallow: /
User-agent: SputnikBot
Disallow: /
User-agent: BLEXBot
Disallow: /


# robots.txt for https://eprint.iacr.org/
# we also use other mechanisms to control crawling.
# for questions about this, send email to Kevin McCurley
User-agent: *
Crawl-Delay: 9
User-agent: *
Disallow: /cgi-bin/
Disallow: /eprint-bin/
Disallow: /eprint-test-bin/
Crawl-delay: 20
User-agent: SemrushBot
Disallow: /
User-agent: SemrushBot-SA
Disallow: /
User-agent: AhrefsBot
Disallow: /
User-agent: SputnikBot
Disallow: /
User-agent: BLEXBot
Disallow: /
User-agent: MJ12bot
Disallow: /


# robots.txt for https://eprint.iacr.org/
# Metadata is available from this site using OAI-PMH,
# RSS, and ATOM. Full text PDFs are only
# available under a license specific to each paper.
# For questions about this, send
# email to Kevin McCurley
User-agent: *
Disallow: /archive
Disallow: /versions
Disallow: /search
Disallow: /complete/compact


# robots.txt for https://eprint.iacr.org/
# Metadata is available from this site using OAI-PMH,
# RSS, and ATOM. Full text PDFs are only
# available under a license specific to each paper.
# For questions about this, send
# email to Kevin McCurley
User-agent: *
Disallow: /archive
Disallow: /versions
Disallow: /search
Disallow: /complete/compact
Crawl-delay: 5


# robots.txt for https://eprint.iacr.org/
# Metadata is available from this site using OAI-PMH,
# RSS, and ATOM. Full text PDFs are only
# available under a license specific to each paper.
# For questions about this, send
# email to Kevin McCurley
User-agent: *
Disallow: /archive
Disallow: /versions
Disallow: /search
Disallow: /complete/compact
User-agent: AdsBot-Google
User-agent: AdsBot-Google-Mobile
User-agent: AhrefsBot
User-agent: AI2Bot
User-agent: Ai2Bot-Dolma
User-agent: Amazonbot
User-agent: amazon-kendra-
User-agent: anthropic-ai
User-agent: Applebot
User-agent: Applebot-Extended
User-agent: Awario
User-agent: bigsur.ai
User-agent: Brightbot 1.0
User-agent: Bytespider
User-agent: CCBot
User-agent: ChatGPT-User
User-agent: Claude-Web
User-agent: ClaudeBot
User-agent: ClearscopeBot
User-Agent: CloudVertexBot
User-agent: cohere-ai
User-agent: cohere-training-data-crawler
User-agent: Cotoyogi/
User-agent: Crawlspace
User-agent: Datadog
User-agent: Devin
User-agent: Diffbot
User-agent: DuckAssistBot
User-agent: FacebookBot
User-agent: FriendlyCrawler
User-agent: Google-Extended
User-agent: GoogleOther
User-agent: GoogleOther-Image
User-agent: GoogleOther-Video
User-agent: GPTBot
User-agent: iaskspider/2.0
User-agent: ICC-Crawler
User-agent: ImagesiftBot
User-agent: img2dataset
User-agent: imgproxy
User-agent: ISSCyberRiskCrawler
User-agent: Kangaroo Bot
User-agent: LINER Bot
User-agent: Meta-ExternalAgent
User-agent: Meta-ExternalFetcher
User-agent: netEstate NE Crawler
User-Agent: w4mwnpbXf3MFAbxOkJRw
User-agent: NovaAct
User-agent: OAI-SearchBot
User-agent: omgili
User-agent: omgilibot
User-agent: Operator
User-agent: outbrain
User-agent: PanguBot
User-agent: Perplexity-User
User-agent: PerplexityBot
User-agent: PetalBot
User-agent: Scrapy
User-agent: SemrushBot-OCOB
User-agent: SemrushBot-SWA
User-agent: Sidetrade indexer bot
User-agent: Timpibot
User-agent: VelenPublicWebCrawler
User-agent: Webzio-Extended
User-agent: YouBot
Disallow: /
Crawl-delay: 5


# robots.txt for https://eprint.iacr.org/
# Metadata is available from this site using OAI-PMH,
# RSS, and ATOM. Full text PDFs are only
# available under a license specific to each paper.
# For questions about this, send
# email to Kevin McCurley
User-agent: *
Disallow: /archive
Disallow: /versions
Disallow: /search
Disallow: /complete/compact
# crawlers should identify themselves and their intent.
User-agent: python-requests
Disallow: /
User-agent: Some-Agent
User-agent: some agent
User-agent: AdsBot-Google
User-agent: AdsBot-Google-Mobile
User-agent: AhrefsBot
User-agent: AI2Bot
User-agent: Ai2Bot-Dolma
User-agent: Amazonbot
User-agent: amazon-kendra
User-agent: amazon-QBusiness
User-agent: anthropic-ai
User-agent: Applebot
User-agent: Applebot-Extended
User-agent: Awario
User-agent: bigsur.ai
User-agent: Brightbot 1.0
User-agent: Bytespider
User-agent: CCBot
User-agent: ChatGPT-User
User-agent: Claude-Web
User-agent: ClaudeBot
User-agent: ClearscopeBot
User-Agent: CloudVertexBot
User-agent: cohere-ai
User-agent: cohere-training-data-crawler
User-agent: Cotoyogi/
User-agent: Crawlspace
User-agent: Datadog
User-agent: Devin
User-agent: Diffbot
User-agent: DuckAssistBot
User-agent: FacebookBot
User-agent: FriendlyCrawler
User-agent: Gemini-Deep-Research
User-agent: Google-Extended
User-agent: GoogleOther
User-agent: GoogleOther-Image
User-agent: GoogleOther-Video
User-agent: GPTBot
User-agent: iaskspider/2.0
User-agent: ICC-Crawler
User-agent: ImagesiftBot
User-agent: img2dataset
User-agent: imgproxy
User-agent: ISSCyberRiskCrawler
User-agent: Kangaroo Bot
User-agent: LINER Bot
User-agent: Meta-ExternalAgent
User-agent: Meta-ExternalFetcher
User-agent: netEstate NE Crawler
User-Agent: w4mwnpbXf3MFAbxOkJRw
User-agent: NovaAct
User-agent: OAI-SearchBot
User-agent: omgili
User-agent: omgilibot
User-agent: Operator
User-agent: outbrain
User-agent: PanguBot
User-agent: Perplexity-User
User-agent: PerplexityBot
User-agent: PetalBot
User-agent: quant
User-agent: Qwantbot
User-agent: Scrapy
User-agent: SemrushBot-OCOB
User-agent: SemrushBot-SWA
User-agent: Sidetrade indexer bot
User-agent: Timpibot
User-agent: VelenPublicWebCrawler
User-agent: Webzio-Extended
User-agent: YouBot
Disallow: /
Crawl-delay: 5


# robots.txt for https://eprint.iacr.org/
# Metadata is available from this site using OAI-PMH,
# RSS, and ATOM. Full text PDFs are only
# available under a license specific to each paper.
# For questions about this, send
# email to Kevin McCurley
User-agent: *
Disallow: /archive
Disallow: /versions
Disallow: /search
Disallow: /complete/compact
# crawlers should identify themselves and their intent.
User-agent: python-requests
Disallow: /
User-agent: Some-Agent
User-agent: some agent
user-agent: AcademicBotRTU
User-agent: AdsBot-Google
User-agent: AdsBot-Google-Mobile
User-agent: AhrefsBot
User-agent: AI2Bot
User-agent: Ai2Bot-Dolma
User-agent: Amazonbot
User-agent: amazon-kendra
User-agent: amazon-QBusiness
User-agent: anthropic-ai
User-agent: Applebot
User-agent: Applebot-Extended
User-agent: Awario
User-agent: bigsur.ai
User-agent: Brightbot 1.0
User-agent: Bytespider
User-agent: CCBot
User-agent: ChatGPT-User
User-agent: Claude-Web
User-agent: ClaudeBot
User-agent: ClearscopeBot
User-Agent: CloudVertexBot
User-agent: cohere-ai
User-agent: cohere-training-data-crawler
User-agent: Cotoyogi/
User-agent: Crawlspace
User-agent: Datadog
User-agent: Devin
User-agent: Diffbot
User-agent: DingTalkBot
User-agent: DuckAssistBot
User-agent: FacebookBot
User-agent: FriendlyCrawler
User-agent: Gemini-Deep-Research
User-agent: Google-Extended
User-agent: GoogleOther
User-agent: GoogleOther-Image
User-agent: GoogleOther-Video
User-agent: GPTBot
User-agent: iaskspider/2.0
User-agent: ICC-Crawler
User-agent: ImagesiftBot
User-agent: img2dataset
User-agent: imgproxy
User-agent: ISSCyberRiskCrawler
User-agent: Kangaroo Bot
User-agent: LINER Bot
User-agent: Meta-ExternalAgent
User-agent: Meta-ExternalFetcher
User-agent: netEstate NE Crawler
User-Agent: w4mwnpbXf3MFAbxOkJRw
User-agent: NovaAct
User-agent: OAI-SearchBot
User-agent: omgili
User-agent: omgilibot
User-agent: Operator
User-agent: outbrain
User-agent: PanguBot
User-agent: Perplexity-User
User-agent: PerplexityBot
User-agent: PetalBot
User-agent: quant
User-agent: Qwantbot
User-agent: Scrapy
User-agent: SemrushBot-OCOB
User-agent: SemrushBot-SWA
User-agent: Sidetrade indexer bot
User-agent: Timpibot
User-agent: VelenPublicWebCrawler
User-agent: Webzio-Extended
User-agent: YouBot
Disallow: /
Crawl-delay: 5


# robots.txt for https://eprint.iacr.org/
# Metadata is available from this site using OAI-PMH,
# RSS, and ATOM. Full text PDFs are only
# available under a license specific to each paper.
# For questions about this, send
# email to Kevin McCurley
User-agent: *
Crawl-delay: 5
Disallow: /archive
Disallow: /versions
Disallow: /search
Disallow: /complete/compact
# crawlers should identify themselves and their intent.
User-agent: python-file-reader
User-agent: python-requests
Disallow: /
User-agent: Some-Agent
User-agent: some agent
user-agent: AcademicBotRTU
User-agent: AdsBot-Google
User-agent: AdsBot-Google-Mobile
User-agent: AhrefsBot
User-agent: AI2Bot
User-agent: Ai2Bot-Dolma
User-agent: Amazonbot
User-agent: amazon-kendra
User-agent: amazon-QBusiness
User-agent: anthropic-ai
User-agent: Applebot
User-agent: Applebot-Extended
User-agent: Awario
User-agent: bigsur.ai
User-agent: Brightbot 1.0
User-agent: Bytespider
User-agent: CCBot
User-agent: ChatGPT-User
User-agent: Claude-Web
User-agent: ClaudeBot
User-agent: ClearscopeBot
User-Agent: CloudVertexBot
User-agent: cohere-ai
User-agent: cohere-training-data-crawler
User-agent: Cotoyogi/
User-agent: Crawlspace
User-agent: Datadog
User-agent: Devin
User-agent: Diffbot
User-agent: DingTalkBot
User-agent: DuckAssistBot
User-agent: FacebookBot
User-agent: FriendlyCrawler
User-agent: Gemini-Deep-Research
User-agent: Google-Extended
User-agent: GoogleOther
User-agent: GoogleOther-Image
User-agent: GoogleOther-Video
User-agent: GPTBot
User-agent: iaskspider/2.0
User-agent: ICC-Crawler
User-agent: ImagesiftBot
User-agent: img2dataset
User-agent: imgproxy
User-agent: ISSCyberRiskCrawler
User-agent: Kangaroo Bot
User-agent: LINER Bot
User-agent: Meta-ExternalAgent
User-agent: Meta-ExternalFetcher
User-agent: netEstate NE Crawler
User-Agent: w4mwnpbXf3MFAbxOkJRw
User-agent: NovaAct
User-agent: OAI-SearchBot
User-agent: omgili
User-agent: omgilibot
User-agent: Operator
User-agent: outbrain
User-agent: PanguBot
User-agent: Perplexity-User
User-agent: PerplexityBot
User-agent: PetalBot
User-agent: quant
User-agent: Qwantbot
User-agent: Scrapy
User-agent: SemrushBot-OCOB
User-agent: SemrushBot-SWA
User-agent: Sidetrade indexer bot
User-agent: Timpibot
User-agent: VelenPublicWebCrawler
User-agent: Webzio-Extended
User-agent: YouBot
Disallow: /


# robots.txt for https://eprint.iacr.org/
# Metadata is available from this site using OAI-PMH,
# RSS, and ATOM. Full text PDFs are only
# available under a license specific to each paper.
# For questions about this, send
# email to Kevin McCurley
User-agent: *
Crawl-delay: 5
Disallow: /archive
Disallow: /versions
Disallow: /search
Disallow: /complete/compact
# crawlers should identify themselves and their intent.
User-agent: python-file-reader
User-agent: python-requests
Disallow: /
User-agent: Some-Agent
User-agent: some agent
user-agent: AcademicBotRTU
User-agent: AdsBot-Google
User-agent: AdsBot-Google-Mobile
User-agent: AhrefsBot
User-agent: AI2Bot
User-agent: Ai2Bot-Dolma
User-agent: Amazonbot
User-agent: amazon-kendra
User-agent: amazon-QBusiness
User-agent: anthropic-ai
User-agent: Applebot
User-agent: Applebot-Extended
User-agent: Awario
User-agent: bigsur.ai
User-agent: Brightbot 1.0
User-agent: Bytespider
User-agent: CCBot
User-agent: ChatGPT-User
User-agent: Claude-Web
User-agent: ClaudeBot
User-agent: ClearscopeBot
User-Agent: CloudVertexBot
User-agent: cohere-ai
User-agent: cohere-training-data-crawler
User-agent: Cotoyogi/
User-agent: Crawlspace
User-agent: Datadog
User-agent: Devin
User-agent: Diffbot
User-agent: DingTalkBot
User-agent: DuckAssistBot
User-agent: FacebookBot
User-agent: FriendlyCrawler
User-agent: Gemini-Deep-Research
User-agent: Google-Extended
User-agent: GoogleOther
User-agent: GoogleOther-Image
User-agent: GoogleOther-Video
User-agent: GPTBot
User-agent: iaskspider/2.0
User-agent: ICC-Crawler
User-agent: ImagesiftBot
User-agent: img2dataset
User-agent: imgproxy
User-agent: ISSCyberRiskCrawler
User-agent: Kangaroo Bot
User-agent: LINER Bot
User-agent: Meta-ExternalAgent
User-agent: Meta-ExternalFetcher
User-agent: netEstate NE Crawler
User-Agent: w4mwnpbXf3MFAbxOkJRw
User-agent: NovaAct
User-agent: OAI-SearchBot
User-agent: omgili
User-agent: omgilibot
User-agent: Operator
User-agent: outbrain
User-agent: PanguBot
User-agent: Perplexity-User
User-agent: PerplexityBot
User-agent: PetalBot
User-agent: quant
User-agent: Qwantbot
User-agent: Scrapy
User-agent: SemrushBot
User-agent: SemrushBot-OCOB
User-agent: SemrushBot-SWA
User-agent: Sidetrade indexer bot
User-agent: Timpibot
User-agent: VelenPublicWebCrawler
User-agent: Webzio-Extended
User-agent: YouBot
Disallow: /


# robots.txt for https://eprint.iacr.org/
# Metadata is available from this site using OAI-PMH,
# RSS, and ATOM. Full text PDFs are only
# available under a license specific to each paper.
# For questions about this, send
# email to Kevin McCurley
User-agent: *
Crawl-delay: 5
Disallow: /archive
Disallow: /versions
Disallow: /search
Disallow: /complete/compact
# crawlers should identify themselves and their intent.
User-agent: python-file-reader
User-agent: python-requests
Disallow: /
User-agent: SiteimproveBot
User-agent: Claude-SearchBot
User-agent: ClaudeBot
User-agent: YisouSpider
User-agent: Some-Agent
User-agent: some agent
user-agent: AcademicBotRTU
User-agent: AdsBot-Google
User-agent: AdsBot-Google-Mobile
User-agent: AhrefsBot
User-agent: AI2Bot
User-agent: Ai2Bot-Dolma
User-agent: Amazonbot
User-agent: amazon-kendra
User-agent: amazon-QBusiness
User-agent: anthropic-ai
User-agent: Applebot
User-agent: Applebot-Extended
User-agent: Awario
User-agent: bigsur.ai
User-agent: Brightbot 1.0
User-agent: Bytespider
User-agent: CCBot
User-agent: ChatGPT-User
User-agent: Claude-Web
User-agent: ClaudeBot
User-agent: ClearscopeBot
User-Agent: CloudVertexBot
User-agent: cohere-ai
User-agent: cohere-training-data-crawler
User-agent: Cotoyogi/
User-agent: Crawlspace
User-agent: Datadog
User-agent: Devin
User-agent: Diffbot
User-agent: DingTalkBot
User-agent: dotbot
User-agent: DuckAssistBot
User-agent: FacebookBot
User-agent: FriendlyCrawler
User-agent: Gemini-Deep-Research
User-agent: Google-Extended
User-agent: GoogleOther
User-agent: GoogleOther-Image
User-agent: GoogleOther-Video
User-agent: GPTBot
User-agent: iaskspider/2.0
User-agent: ICC-Crawler
User-agent: ImagesiftBot
User-agent: img2dataset
User-agent: imgproxy
User-agent: ISSCyberRiskCrawler
User-agent: Kangaroo Bot
User-agent: LINER Bot
User-agent: Meta-ExternalAgent
User-agent: Meta-ExternalFetcher
User-agent: netEstate NE Crawler
User-Agent: w4mwnpbXf3MFAbxOkJRw
User-agent: NovaAct
User-agent: OAI-SearchBot
User-agent: omgili
User-agent: omgilibot
User-agent: Operator
User-agent: outbrain
User-agent: PanguBot
User-agent: Perplexity-User
User-agent: PerplexityBot
User-agent: PetalBot
User-agent: quant
User-agent: Qwantbot
User-agent: Scrapy
User-agent: SemrushBot
User-agent: SemrushBot-OCOB
User-agent: SemrushBot-SWA
User-agent: Sidetrade indexer bot
User-agent: Timpibot
User-agent: VelenPublicWebCrawler
User-agent: Webzio-Extended
User-agent: YouBot
Disallow: /


# As a condition of accessing this website, you agree to abide by the following
# content signals:

# (a)  If a Content-Signal = yes, you may collect content for the corresponding
#      use.
# (b)  If a Content-Signal = no, you may not collect content for the
#      corresponding use.
# (c)  If the website operator does not include a Content-Signal for a
#      corresponding use, the website operator neither grants nor restricts
#      permission via Content-Signal with respect to the corresponding use.

# The content signals and their meanings are:

# search:   building a search index and providing search results (e.g., returning
#           hyperlinks and short excerpts from your website's contents). Search does not
#           include providing AI-generated search summaries.
# ai-input: inputting content into one or more AI models (e.g., retrieval
#           augmented generation, grounding, or other real-time taking of content for
#           generative AI search answers).
# ai-train: training or fine-tuning AI models.

# ANY RESTRICTIONS EXPRESSED VIA CONTENT SIGNALS ARE EXPRESS RESERVATIONS OF
# RIGHTS UNDER ARTICLE 4 OF THE EUROPEAN UNION DIRECTIVE 2019/790 ON COPYRIGHT
# AND RELATED RIGHTS IN THE DIGITAL SINGLE MARKET.

# BEGIN Cloudflare Managed content

User-agent: *
Content-Signal: search=yes,ai-train=no
Allow: /

User-agent: Amazonbot
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: GPTBot
Disallow: /

User-agent: meta-externalagent
Disallow: /

# END Cloudflare Managed Content

# robots.txt for https://eprint.iacr.org/
# Metadata is available from this site using OAI-PMH,
# RSS, and ATOM. Full text PDFs are only
# available under a license specific to each paper.
# If you crawl this site then you must use a User-Agent that uniquely identifies
# your purpose (i.e., something like curl is inadequate). Crawling has recently
# starting causing so much load that it interferes with human usage.  If you
# disobey the crawl directives below or if you overcrawl, then your IP address(es)
# will be banned outright. For questions about this, send email to Kevin McCurley.
User-agent: *
Crawl-delay: 5
Crawl-Delay: 5
Disallow: /archive
Disallow: /versions
Disallow: /search
Disallow: /complete/compact
# crawlers should identify themselves and their intent.
User-agent: python-file-reader
Disallow: /
User-agent: python-requests
Disallow: /
User-agent: Thinkbot
Disallow: /
User-agent: coccocbot-web
Disallow: /
User-agent: GenAI
Disallow: /
User-agent: Twitterbot/1.0
Disallow: /
User-agent: SiteimproveBot
Disallow: /
User-agent: Claude-SearchBot
Disallow: /
User-agent: ClaudeBot
Disallow: /
User-agent: YisouSpider
Disallow: /
User-agent: Some-Agent
Disallow: /
User-agent: some agent
Disallow: /
user-agent: AcademicBotRTU
Disallow: /
User-agent: AdsBot-Google
Disallow: /
User-agent: AdsBot-Google-Mobile
Disallow: /
User-agent: AhrefsBot
Disallow: /
User-agent: AI2Bot
Disallow: /
User-agent: Ai2Bot-Dolma
Disallow: /
User-agent: Amazonbot
Disallow: /
User-agent: amazon-kendra
Disallow: /
User-agent: amazon-QBusiness
Disallow: /
User-agent: anthropic-ai
Disallow: /
User-agent: Applebot
Disallow: /
User-agent: Applebot-Extended
Disallow: /
User-agent: Awario
Disallow: /
User-agent: bigsur.ai
Disallow: /
User-agent: Brightbot 1.0
Disallow: /
User-agent: Bytespider
Disallow: /
User-agent: CCBot
Disallow: /
User-agent: ChatGPT-User
Disallow: /
User-agent: Claude-Web
Disallow: /
User-agent: ClaudeBot
Disallow: /
User-agent: ClearscopeBot
Disallow: /
User-Agent: CloudVertexBot
Disallow: /
User-agent: cohere-ai
Disallow: /
User-agent: cohere-training-data-crawler
Disallow: /
User-agent: Cotoyogi/
Disallow: /
User-agent: Crawlspace
Disallow: /
User-agent: Datadog
Disallow: /
User-agent: Devin
Disallow: /
User-agent: Diffbot
Disallow: /
User-agent: DingTalkBot
Disallow: /
User-agent: dotbot
Disallow: /
User-agent: DuckAssistBot
Disallow: /
User-agent: FacebookBot
Disallow: /
User-agent: FriendlyCrawler
Disallow: /
User-agent: Gemini-Deep-Research
Disallow: /
User-agent: Google-Extended
Disallow: /
User-agent: GoogleOther
Disallow: /
User-agent: GoogleOther-Image
Disallow: /
User-agent: GoogleOther-Video
Disallow: /
User-agent: GPTBot
Disallow: /
User-agent: iaskspider/2.0
Disallow: /
User-agent: ICC-Crawler
Disallow: /
User-agent: ImagesiftBot
Disallow: /
User-agent: img2dataset
Disallow: /
User-agent: imgproxy
Disallow: /
User-agent: ISSCyberRiskCrawler
Disallow: /
User-agent: Kangaroo Bot
Disallow: /
User-agent: LINER Bot
Disallow: /
User-agent: Meta-ExternalAgent
Disallow: /
User-agent: Meta-ExternalFetcher
Disallow: /
User-agent: netEstate NE Crawler
Disallow: /
User-Agent: w4mwnpbXf3MFAbxOkJRw
Disallow: /
User-agent: NovaAct
Disallow: /
User-agent: OAI-SearchBot
Disallow: /
User-agent: omgili
Disallow: /
User-agent: omgilibot
Disallow: /
User-agent: Operator
Disallow: /
User-agent: outbrain
Disallow: /
User-agent: PanguBot
Disallow: /
User-agent: Perplexity-User
Disallow: /
User-agent: PerplexityBot
Disallow: /
User-agent: PetalBot
Disallow: /
User-agent: quant
Disallow: /
User-agent: Qwantbot
Disallow: /
User-agent: Scrapy
Disallow: /
User-agent: SemrushBot
Disallow: /
User-agent: SemrushBot-OCOB
Disallow: /
User-agent: SemrushBot-SWA
Disallow: /
User-agent: Sidetrade indexer bot
Disallow: /
User-agent: Timpibot
Disallow: /
User-agent: VelenPublicWebCrawler
Disallow: /
User-agent: Webzio-Extended
Disallow: /
User-agent: YouBot
Disallow: /


# All Metadata is available from this site using OAI-PMH,
# RSS, and ATOM under a CC0 license.
#
# If you are trying to train an AI model, please look elsewhere.
# For questions about this, send email to Kevin McCurley.
Crawl-delay: 10
User-agent: Google-Extended
Disallow: /
User-agent: Googlebot
Disallow: *.pdf
Disallow: /archive
Disallow: /versions
Disallow: /search
Disallow: /complete/compact
Disallow: /submit
Allow: *
User-agent: bingbot
Disallow: /archive
Disallow: /versions
Disallow: /search
Disallow: /complete/compact
Disallow: /submit
Allow: *
User-agent: *
Allow: /oai
Allow: /rss
Disallow: /


# All Metadata is available from this site using OAI-PMH,
# RSS, and ATOM under a CC0 license.
#
# If you are trying to train an AI model, please look elsewhere.
# For questions about this, send email to Kevin McCurley.
User-agent: MJ12bot
Disallow: /
Crawl-delay: 10
User-agent: Google-Extended
Disallow: /
User-agent: Googlebot
Disallow: *.pdf
Disallow: /archive
Disallow: /versions
Disallow: /search
Disallow: /complete/compact
Disallow: /submit
Allow: *
User-agent: bingbot
Disallow: /archive
Disallow: /versions
Disallow: /search
Disallow: /complete/compact
Disallow: /submit
Allow: *
User-agent: *
Allow: /oai
Allow: /rss
Disallow: /


# robots.txt for https://eprint.iacr.org/
# Metadata is available from this site using OAI-PMH,
# RSS, and ATOM. Full text PDFs are only
# available under a license specific to each paper.
# Most AI crawlers willfully ignore the intent of the CC-BY license.
#
# For questions about this, send
# email to Kevin McCurley
#
# Note that we check the list of approved IP addresses for Googlebot
# and claude uses 160.79.104.0/21 and 160.79.104.0/23 and 2607:6bc0::/48
User-agent: Googlebot
User-agent: Claude-User
User-agent: Claude-SearchBot
Disallow: /archive
Disallow: /versions
Disallow: /submit
Disallow: /search
# they get it from other places with abstracts for better indexing
Disallow: /complete/compact
Allow: *
# ################################ bing
User-agent: bingbot
Crawl-delay: 5
Disallow: *pdf
Disallow: /archive
Disallow: /versions
Disallow: /submit
Disallow: /search
# they get it from other places with abstracts for better indexing
Disallow: /complete/compact
Allow: *
# duckduckgo
# ################################ duckduckgo
User-agent: DuckDuckBot/1.1
Crawl-delay: 5
Disallow: *pdf
Disallow: /archive
Disallow: /versions
Disallow: /submit
Disallow: /search
# they get it from other places with abstracts for better indexing
Disallow: /complete/compact
Allow: *
##########################################
# AI bots get limited access. No PDFs.
User-agent: Googleextended
User-agent: ChatGPT-User
User-agent: OAI-SearchBot
User-agent: GPTBot
# claude uses 160.79.104.0/21 and 160.79.104.0/23 and 2607:6bc0::/48
User-agent: ClaudeBot
Disallow: *.pdf
Disallow: /199*/*.pdf
Disallow: /200*/*.pdf
Disallow: /201*/*.pdf
Disallow: /202*/*.pdf
Allow: /199*/*
Allow: /200*/*
Allow: /201*/*
Allow: /202*/*
Disallow: /*/*pdf
Disallow: /archive
Disallow: /versions
Disallow: /submit
Disallow: /search
Disallow: *
# ################################ everyone else. Crawlers must be whitelisted and
# confirmed to be following rules, which many do not.
User-agent: *
Disallow: /


user-agent: *
Disallow: /


# Block all but AdsBot crawlers
User-agent: *
Disallow: /


# See http://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file
#
# To ban all spiders from the entire site uncomment the next two lines:
# User-agent: *
# Disallow: /


# See https://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file


#
# robots.txt
#
# This file is to prevent the crawling and indexing of certain parts
# of your site by web crawlers and spiders run by sites like Yahoo!
# and Google. By telling these "robots" where not to go on your site,
# you save bandwidth and server resources.
#
# This file will be ignored unless it is at the root of your host:
# Used:    http://example.com/robots.txt
# Ignored: http://example.com/site/robots.txt
#
# For more information about the robots.txt standard, see:
# http://www.robotstxt.org/robotstxt.html

User-agent: *
# CSS, JS, Images
Allow: /core/*.css$
Allow: /core/*.css?
Allow: /core/*.js$
Allow: /core/*.js?
Allow: /core/*.gif
Allow: /core/*.jpg
Allow: /core/*.jpeg
Allow: /core/*.png
Allow: /core/*.svg
Allow: /profiles/*.css$
Allow: /profiles/*.css?
Allow: /profiles/*.js$
Allow: /profiles/*.js?
Allow: /profiles/*.gif
Allow: /profiles/*.jpg
Allow: /profiles/*.jpeg
Allow: /profiles/*.png
Allow: /profiles/*.svg
# Directories
Disallow: /core/
Disallow: /profiles/
# Files
Disallow: /README.txt
Disallow: /web.config
# Paths (clean URLs)
Disallow: /admin/
Disallow: /comment/reply/
Disallow: /filter/tips
Disallow: /node/add/
Disallow: /search/
Disallow: /user/register/
Disallow: /user/password/
Disallow: /user/login/
Disallow: /user/logout/
Disallow: /press/
# Paths (no clean URLs)
Disallow: /index.php/admin/
Disallow: /index.php/comment/reply/
Disallow: /index.php/filter/tips
Disallow: /index.php/node/add/
Disallow: /index.php/search/
Disallow: /index.php/user/password/
Disallow: /index.php/user/register/
Disallow: /index.php/user/login/
Disallow: /index.php/user/logout/


#
# robots.txt
#
# This file is to prevent the crawling and indexing of certain parts
# of your site by web crawlers and spiders run by sites like Yahoo!
# and Google. By telling these "robots" where not to go on your site,
# you save bandwidth and server resources.
#
# This file will be ignored unless it is at the root of your host:
# Used:    http://example.com/robots.txt
# Ignored: http://example.com/site/robots.txt
#
# For more information about the robots.txt standard, see:
# http://www.robotstxt.org/robotstxt.html

User-agent: *
Crawl-delay: 10
# CSS, JS, Images
Allow: /core/*.css$
Allow: /core/*.css?
Allow: /core/*.js$
Allow: /core/*.js?
Allow: /core/*.gif
Allow: /core/*.jpg
Allow: /core/*.jpeg
Allow: /core/*.png
Allow: /core/*.svg
Allow: /profiles/*.css$
Allow: /profiles/*.css?
Allow: /profiles/*.js$
Allow: /profiles/*.js?
Allow: /profiles/*.gif
Allow: /profiles/*.jpg
Allow: /profiles/*.jpeg
Allow: /profiles/*.png
Allow: /profiles/*.svg
Allow: /cases/
# Directories
Disallow: /core/
Disallow: /profiles/
# Files
Disallow: /README.txt
Disallow: /web.config
# Paths (clean URLs)
Disallow: /admin/
Disallow: /comment/reply/
Disallow: /filter/tips
Disallow: /node/add/
Disallow: /search/
Disallow: /user/register/
Disallow: /user/password/
Disallow: /user/login/
Disallow: /user/logout/
Disallow: /press/
Disallow: /cases
Disallow: /cases/11290
Disallow: /cases/13208
# Paths (no clean URLs)
Disallow: /index.php/admin/
Disallow: /index.php/comment/reply/
Disallow: /index.php/filter/tips
Disallow: /index.php/node/add/
Disallow: /index.php/search/
Disallow: /index.php/user/password/
Disallow: /index.php/user/register/
Disallow: /index.php/user/login/
Disallow: /index.php/user/logout/


# As a condition of accessing this website, you agree to abide by the following
# content signals:

# (a)  If a content-signal = yes, you may collect content for the corresponding
#      use.
# (b)  If a content-signal = no, you may not collect content for the
#      corresponding use.
# (c)  If the website operator does not include a content signal for a
#      corresponding use, the website operator neither grants nor restricts
#      permission via content signal with respect to the corresponding use.

# The content signals and their meanings are:

# search:   building a search index and providing search results (e.g., returning
#           hyperlinks and short excerpts from your website's contents). Search does not
#           include providing AI-generated search summaries.
# ai-input: inputting content into one or more AI models (e.g., retrieval
#           augmented generation, grounding, or other real-time taking of content for
#           generative AI search answers).
# ai-train: training or fine-tuning AI models.

# ANY RESTRICTIONS EXPRESSED VIA CONTENT SIGNALS ARE EXPRESS RESERVATIONS OF
# RIGHTS UNDER ARTICLE 4 OF THE EUROPEAN UNION DIRECTIVE 2019/790 ON COPYRIGHT
# AND RELATED RIGHTS IN THE DIGITAL SINGLE MARKET.

# BEGIN Cloudflare Managed content

User-Agent: *
Content-signal: search=yes,ai-train=no
Allow: /

User-agent: Amazonbot
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: GPTBot
Disallow: /

User-agent: meta-externalagent
Disallow: /

# END Cloudflare Managed Content

#
# robots.txt
#
# This file is to prevent the crawling and indexing of certain parts
# of your site by web crawlers and spiders run by sites like Yahoo!
# and Google. By telling these "robots" where not to go on your site,
# you save bandwidth and server resources.
#
# This file will be ignored unless it is at the root of your host:
# Used:    http://example.com/robots.txt
# Ignored: http://example.com/site/robots.txt
#
# For more information about the robots.txt standard, see:
# http://www.robotstxt.org/robotstxt.html

User-agent: *
Crawl-delay: 10
# CSS, JS, Images
Allow: /core/*.css$
Allow: /core/*.css?
Allow: /core/*.js$
Allow: /core/*.js?
Allow: /core/*.gif
Allow: /core/*.jpg
Allow: /core/*.jpeg
Allow: /core/*.png
Allow: /core/*.svg
Allow: /profiles/*.css$
Allow: /profiles/*.css?
Allow: /profiles/*.js$
Allow: /profiles/*.js?
Allow: /profiles/*.gif
Allow: /profiles/*.jpg
Allow: /profiles/*.jpeg
Allow: /profiles/*.png
Allow: /profiles/*.svg
Allow: /cases/
# Directories
Disallow: /core/
Disallow: /profiles/
# Files
Disallow: /README.txt
Disallow: /web.config
# Paths (clean URLs)
Disallow: /admin/
Disallow: /comment/reply/
Disallow: /filter/tips
Disallow: /node/add/
Disallow: /search/
Disallow: /user/register/
Disallow: /user/password/
Disallow: /user/login/
Disallow: /user/logout/
Disallow: /press/
Disallow: /cases
Disallow: /cases/11290
Disallow: /cases/13208
# Paths (no clean URLs)
Disallow: /index.php/admin/
Disallow: /index.php/comment/reply/
Disallow: /index.php/filter/tips
Disallow: /index.php/node/add/
Disallow: /index.php/search/
Disallow: /index.php/user/password/
Disallow: /index.php/user/register/
Disallow: /index.php/user/login/
Disallow: /index.php/user/logout/


# As a condition of accessing this website, you agree to abide by the following
# content signals:

# (a)  If a Content-Signal = yes, you may collect content for the corresponding
#      use.
# (b)  If a Content-Signal = no, you may not collect content for the
#      corresponding use.
# (c)  If the website operator does not include a Content-Signal for a
#      corresponding use, the website operator neither grants nor restricts
#      permission via Content-Signal with respect to the corresponding use.

# The content signals and their meanings are:

# search:   building a search index and providing search results (e.g., returning
#           hyperlinks and short excerpts from your website's contents). Search does not
#           include providing AI-generated search summaries.
# ai-input: inputting content into one or more AI models (e.g., retrieval
#           augmented generation, grounding, or other real-time taking of content for
#           generative AI search answers).
# ai-train: training or fine-tuning AI models.
# use:      how AI systems may consume the content (immediate, reference, or full).

# ANY RESTRICTIONS EXPRESSED VIA CONTENT SIGNALS ARE EXPRESS RESERVATIONS OF
# RIGHTS UNDER ARTICLE 4 OF THE EUROPEAN UNION DIRECTIVE 2019/790 ON COPYRIGHT
# AND RELATED RIGHTS IN THE DIGITAL SINGLE MARKET.

# BEGIN Cloudflare Managed content

User-agent: *
Content-Signal: search=yes,ai-train=no,use=reference
Allow: /

User-agent: Amazonbot
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: CloudflareBrowserRenderingCrawler
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: GPTBot
Disallow: /

User-agent: meta-externalagent
Disallow: /

# END Cloudflare Managed Content

#
# robots.txt
#
# This file is to prevent the crawling and indexing of certain parts
# of your site by web crawlers and spiders run by sites like Yahoo!
# and Google. By telling these "robots" where not to go on your site,
# you save bandwidth and server resources.
#
# This file will be ignored unless it is at the root of your host:
# Used:    http://example.com/robots.txt
# Ignored: http://example.com/site/robots.txt
#
# For more information about the robots.txt standard, see:
# http://www.robotstxt.org/robotstxt.html

User-agent: *
# CSS, JS, Images
Allow: /core/*.css$
Allow: /core/*.css?
Allow: /core/*.js$
Allow: /core/*.js?
Allow: /core/*.gif
Allow: /core/*.jpg
Allow: /core/*.jpeg
Allow: /core/*.png
Allow: /core/*.svg
Allow: /profiles/*.css$
Allow: /profiles/*.css?
Allow: /profiles/*.js$
Allow: /profiles/*.js?
Allow: /profiles/*.gif
Allow: /profiles/*.jpg
Allow: /profiles/*.jpeg
Allow: /profiles/*.png
Allow: /profiles/*.svg
# Directories
Disallow: /core/
Disallow: /profiles/
# Files
Disallow: /README.md
Disallow: /composer/Metapackage/README.txt
Disallow: /composer/Plugin/ProjectMessage/README.md
Disallow: /composer/Plugin/Scaffold/README.md
Disallow: /composer/Plugin/VendorHardening/README.txt
Disallow: /composer/Template/README.txt
Disallow: /modules/README.txt
Disallow: /sites/README.txt
Disallow: /themes/README.txt
Disallow: /web.config
# Paths (clean URLs)
Disallow: /admin/
Disallow: /comment/reply/
Disallow: /filter/tips
Disallow: /node/add/
Disallow: /search/
Disallow: /user/register
Disallow: /user/password
Disallow: /user/login
Disallow: /user/logout
Disallow: /media/oembed
Disallow: /*/media/oembed
# Paths (no clean URLs)
Disallow: /index.php/admin/
Disallow: /index.php/comment/reply/
Disallow: /index.php/filter/tips
Disallow: /index.php/node/add/
Disallow: /index.php/search/
Disallow: /index.php/user/password
Disallow: /index.php/user/register
Disallow: /index.php/user/login
Disallow: /index.php/user/logout
Disallow: /index.php/media/oembed
Disallow: /index.php/*/media/oembed


User-agent: *
Disallow: /browse/
Disallow: /search/
Disallow: /admin/
Disallow: /api/
Disallow: /apiv2/
Disallow: /usersViewSingle
Disallow: /samplesViewSingle
Disallow: /packsViewSingle
Disallow: /tagsViewSingle
Disallow: /ratings/
Disallow: /geotags/
Disallow: /tickets/
Disallow: /embed/
Disallow: /*similar/
Disallow: /contact/


User-agent: *
Disallow: /browse/
Disallow: /search/
Disallow: /admin/
Disallow: /api/
Disallow: /apiv2/
Disallow: /usersViewSingle
Disallow: /samplesViewSingle
Disallow: /packsViewSingle
Disallow: /tagsViewSingle
Disallow: /ratings/
Disallow: /geotags/
Disallow: /tickets/
Disallow: /embed/
Disallow: /*similar/
Disallow: /contact/

Sitemap: https://freesound.org/sitemap.xml


# Discourage any crawling or indexing
Disallow: /


User-agent: *
Disallow: /


# If you would like to crawl GitHub contact us via https://support.github.com/
# We also provide an extensive API: https://developer.github.com/
User-agent: baidu
crawl-delay: 1

User-agent: *

Disallow: /*/pulse
Disallow: /*/tree/
Disallow: /*/wiki*
Disallow: /gist/
Disallow: /*/forks
Disallow: /*/stars
Disallow: /*/download
Disallow: /*/revisions
Disallow: /*/issues/new
Disallow: /*/issues/search
Disallow: /*/commits/
Disallow: /*/commits/*?author
Disallow: /*/commits/*?path
Disallow: /*/branches
Disallow: /*/tags
Disallow: /*/contributors
Disallow: /*/comments
Disallow: /*/stargazers
Disallow: /*/archive/
Disallow: /*/blame/
Disallow: /*/watchers
Disallow: /*/network
Disallow: /*/graphs
Disallow: /*/raw/
Disallow: /*/compare/
Disallow: /*/cache/
Disallow: /.git/
Disallow: */.git/
Disallow: /*.git$
Disallow: /search/advanced
Disallow: /search
Disallow: */search
Disallow: /*q=
Disallow: /*.atom

Disallow: /ekansa/Open-Context-Data
Disallow: /ekansa/opencontext-*
Disallow: */tarball/
Disallow: */zipball/

Disallow: /*source=*
Disallow: /*ref_cta=*
Disallow: /*plan=*
Disallow: /*return_to=*
Disallow: /*ref_loc=*
Disallow: /*setup_organization=*
Disallow: /*source_repo=*
Disallow: /*ref_page=*
Disallow: /*source=*
Disallow: /*referrer=*
Disallow: /*report=*
Disallow: /*author=*
Disallow: /*since=*
Disallow: /*until=*
Disallow: /*commits?author=*
Disallow: /*report-abuse?report=*
Disallow: /*tab=*

Disallow: /account-login
Disallow: /Explodingstuff/


# If you would like to crawl GitHub contact us via https://support.github.com/contact/
# We also provide an extensive API: https://developer.github.com/
User-agent: baidu
crawl-delay: 1

User-agent: *

Disallow: /*/pulse
Disallow: /*/tree/
Disallow: /*/wiki*
Disallow: /gist/
Disallow: /*/forks
Disallow: /*/stars
Disallow: /*/download
Disallow: /*/revisions
Disallow: /*/issues/new
Disallow: /*/issues/search
Disallow: /*/commits/
Disallow: /*/commits/*?author
Disallow: /*/commits/*?path
Disallow: /*/branches
Disallow: /*/tags
Disallow: /*/contributors
Disallow: /*/comments
Disallow: /*/stargazers
Disallow: /*/archive/
Disallow: /*/blame/
Disallow: /*/watchers
Disallow: /*/network
Disallow: /*/graphs
Disallow: /*/raw/
Disallow: /*/compare/
Disallow: /*/cache/
Disallow: /.git/
Disallow: */.git/
Disallow: /*.git$
Disallow: /search/advanced
Disallow: /search
Disallow: */search
Disallow: /*q=
Disallow: /*.atom

Disallow: /ekansa/Open-Context-Data
Disallow: /ekansa/opencontext-*
Disallow: */tarball/
Disallow: */zipball/

Disallow: /*source=*
Disallow: /*ref_cta=*
Disallow: /*plan=*
Disallow: /*return_to=*
Disallow: /*ref_loc=*
Disallow: /*setup_organization=*
Disallow: /*source_repo=*
Disallow: /*ref_page=*
Disallow: /*source=*
Disallow: /*referrer=*
Disallow: /*report=*
Disallow: /*author=*
Disallow: /*since=*
Disallow: /*until=*
Disallow: /*commits?author=*
Disallow: /*report-abuse?report=*
Disallow: /*tab=*

Disallow: /account-login
Disallow: /Explodingstuff/


# If you would like to crawl GitHub contact us via https://support.github.com/contact/
# We also provide an extensive API: https://docs.github.com
User-agent: baidu
crawl-delay: 1

User-agent: *

Disallow: /*/pulse
Disallow: /*/tree/
Disallow: /*/wiki*
Disallow: /gist/
Disallow: /*/forks
Disallow: /*/stars
Disallow: /*/download
Disallow: /*/revisions
Disallow: /*/issues/new
Disallow: /*/issues/search
Disallow: /*/commits/
Disallow: /*/commits/*?author
Disallow: /*/commits/*?path
Disallow: /*/branches
Disallow: /*/tags
Disallow: /*/contributors
Disallow: /*/comments
Disallow: /*/stargazers
Disallow: /*/archive/
Disallow: /*/blame/
Disallow: /*/watchers
Disallow: /*/network
Disallow: /*/graphs
Disallow: /*/raw/
Disallow: /*/compare/
Disallow: /*/cache/
Disallow: /.git/
Disallow: */.git/
Disallow: /*.git$
Disallow: /search/advanced
Disallow: /search
Disallow: */search
Disallow: /*q=
Disallow: /*.atom

Disallow: /ekansa/Open-Context-Data
Disallow: /ekansa/opencontext-*
Disallow: */tarball/
Disallow: */zipball/

Disallow: /*source=*
Disallow: /*ref_cta=*
Disallow: /*plan=*
Disallow: /*return_to=*
Disallow: /*ref_loc=*
Disallow: /*setup_organization=*
Disallow: /*source_repo=*
Disallow: /*ref_page=*
Disallow: /*source=*
Disallow: /*referrer=*
Disallow: /*report=*
Disallow: /*author=*
Disallow: /*since=*
Disallow: /*until=*
Disallow: /*commits?author=*
Disallow: /*report-abuse?report=*
Disallow: /*tab=*

Disallow: /account-login
Disallow: /Explodingstuff/


# If you would like to crawl GitHub contact us via https://support.github.com?tags=dotcom-robots
# We also provide an extensive API: https://docs.github.com
User-agent: baidu
crawl-delay: 1

User-agent: *

Disallow: /*/pulse
Disallow: /*/tree/
Disallow: /gist/
Disallow: /*/forks
Disallow: /*/stars
Disallow: /*/download
Disallow: /*/revisions
Disallow: /*/issues/new
Disallow: /*/issues/search
Disallow: /*/commits/
Disallow: /*/commits/*?author
Disallow: /*/commits/*?path
Disallow: /*/branches
Disallow: /*/tags
Disallow: /*/contributors
Disallow: /*/comments
Disallow: /*/stargazers
Disallow: /*/archive/
Disallow: /*/blame/
Disallow: /*/watchers
Disallow: /*/network
Disallow: /*/graphs
Disallow: /*/raw/
Disallow: /*/compare/
Disallow: /*/cache/
Disallow: /.git/
Disallow: */.git/
Disallow: /*.git$
Disallow: /search/advanced
Disallow: /search
Disallow: */search
Disallow: /*q=
Disallow: /*.atom

Disallow: /ekansa/Open-Context-Data
Disallow: /ekansa/opencontext-*
Disallow: */tarball/
Disallow: */zipball/

Disallow: /*source=*
Disallow: /*ref_cta=*
Disallow: /*plan=*
Disallow: /*return_to=*
Disallow: /*ref_loc=*
Disallow: /*setup_organization=*
Disallow: /*source_repo=*
Disallow: /*ref_page=*
Disallow: /*source=*
Disallow: /*referrer=*
Disallow: /*report=*
Disallow: /*author=*
Disallow: /*since=*
Disallow: /*until=*
Disallow: /*commits?author=*
Disallow: /*report-abuse?report=*
Disallow: /*tab=*

Disallow: /account-login
Disallow: /Explodingstuff/


# If you would like to crawl GitHub contact us via https://support.github.com?tags=dotcom-robots
# We also provide an extensive API: https://docs.github.com
User-agent: baidu
crawl-delay: 1

User-agent: *

Disallow: /*/pulse
Disallow: /*/tree/
Disallow: /gist/
Disallow: /*/forks
Disallow: /*/stars
Disallow: /*/download
Disallow: /*/revisions
Disallow: /*/issues/new
Disallow: /*/issues/search
Disallow: /*/commits/
Disallow: /*/commits/*?author
Disallow: /*/commits/*?path
Disallow: /*/branches
Disallow: /*/tags
Disallow: /*/contributors
Disallow: /*/comments
Disallow: /*/stargazers
Disallow: /*/archive/
Disallow: /*/blame/
Disallow: /*/watchers
Disallow: /*/network
Disallow: /*/graphs
Disallow: /*/raw/
Disallow: /*/compare/
Disallow: /*/cache/
Disallow: /.git/
Disallow: */.git/
Disallow: /*.git$
Disallow: /search/advanced
Disallow: /search
Disallow: */search
Disallow: /*q=
Disallow: /*.atom

Disallow: /ekansa/Open-Context-Data
Disallow: /ekansa/opencontext-*
Disallow: */tarball/
Disallow: */zipball/

Disallow: /*source=*
Disallow: /*ref_cta=*
Disallow: /*plan=*
Disallow: /*return_to=*
Disallow: /*ref_loc=*
Disallow: /*setup_organization=*
Disallow: /*source_repo=*
Disallow: /*ref_page=*
Disallow: /*source=*
Disallow: /*referrer=*
Disallow: /*report=*
Disallow: /*author=*
Disallow: /*since=*
Disallow: /*until=*
Disallow: /*commits?author=*
Disallow: /*report-abuse?report=*
Disallow: /*tab=*
Allow: /*?tab=achievements&achievement=*

Disallow: /account-login
Disallow: /Explodingstuff/


# If you would like to crawl GitHub contact us via https://support.github.com?tags=dotcom-robots
# We also provide an extensive API: https://docs.github.com
User-agent: baidu
crawl-delay: 1

User-agent: *

Disallow: /*/pulse
Disallow: /*/tree/
Disallow: /gist/
Disallow: /*/forks
Disallow: /*/stars
Disallow: /*/download
Disallow: /*/revisions
Disallow: /*/issues/new
Disallow: /*/issues/search
Disallow: /*/commits/
Disallow: /*/commits/*?author
Disallow: /*/commits/*?path
Disallow: /*/branches
Disallow: /*/tags
Disallow: /*/contributors
Disallow: /*/comments
Disallow: /*/stargazers
Disallow: /*/archive/
Disallow: /*/blame/
Disallow: /*/watchers
Disallow: /*/network
Disallow: /*/graphs
Disallow: /*/raw/
Disallow: /*/compare/
Disallow: /*/cache/
Disallow: /.git/
Disallow: */.git/
Disallow: /*.git$
Disallow: /search/advanced
Disallow: /search
Disallow: */search
Disallow: /*q=
Disallow: /*.atom$

Disallow: /ekansa/Open-Context-Data
Disallow: /ekansa/opencontext-*
Disallow: */tarball/
Disallow: */zipball/

Disallow: /*source=*
Disallow: /*ref_cta=*
Disallow: /*plan=*
Disallow: /*return_to=*
Disallow: /*ref_loc=*
Disallow: /*setup_organization=*
Disallow: /*source_repo=*
Disallow: /*ref_page=*
Disallow: /*source=*
Disallow: /*referrer=*
Disallow: /*report=*
Disallow: /*author=*
Disallow: /*since=*
Disallow: /*until=*
Disallow: /*commits?author=*
Disallow: /*report-abuse?report=*
Disallow: /*tab=*
Allow: /*?tab=achievements&achievement=*

Disallow: /account-login
Disallow: /Explodingstuff/


# If you would like to crawl GitHub contact us via https://support.github.com?tags=dotcom-robots
# We also provide an extensive API: https://docs.github.com
User-agent: baidu
crawl-delay: 1

User-agent: *

Disallow: /*/*/pulse
Disallow: /*/*/projects
Disallow: /*/*/forks
Disallow: /*/*/issues/new
Disallow: /*/*/issues/search
Disallow: /*/*/commits/
Disallow: /*/*/branches
Disallow: /*/*/contributors
Disallow: /*/*/tags
Disallow: /*/*/stargazers
Disallow: /*/*/watchers
Disallow: /*/*/network
Disallow: /*/*/graphs
Disallow: /*/*/compare

Disallow: /*/tree/
Disallow: /gist/
Disallow: /*/download
Disallow: /*/revisions
Disallow: /*/commits/*?author
Disallow: /*/commits/*?path
Disallow: /*/comments
Disallow: /*/archive/
Disallow: /*/blame/
Disallow: /*/raw/
Disallow: /*/cache/
Disallow: /.git/
Disallow: */.git/
Disallow: /*.git$
Disallow: /search/advanced
Disallow: /search
Disallow: */search
Disallow: /*q=
Disallow: /*.atom$

Disallow: /ekansa/Open-Context-Data
Disallow: /ekansa/opencontext-*
Disallow: */tarball/
Disallow: */zipball/

Disallow: /*source=*
Disallow: /*ref_cta=*
Disallow: /*plan=*
Disallow: /*return_to=*
Disallow: /*ref_loc=*
Disallow: /*setup_organization=*
Disallow: /*source_repo=*
Disallow: /*ref_page=*
Disallow: /*source=*
Disallow: /*referrer=*
Disallow: /*report=*
Disallow: /*author=*
Disallow: /*since=*
Disallow: /*until=*
Disallow: /*commits?author=*
Disallow: /*report-abuse?report=*
Disallow: /*tab=*
Allow: /*?tab=achievements&achievement=*

Disallow: /account-login
Disallow: /Explodingstuff/


# If you would like to crawl GitHub contact us via https://support.github.com?tags=dotcom-robots
# We also provide an extensive API: https://docs.github.com
User-agent: baidu
crawl-delay: 1

User-agent: *

Disallow: /*/*/pulse
Disallow: /*/*/projects
Disallow: /*/*/forks
Disallow: /*/*/issues/new
Disallow: /*/*/issues/search
Disallow: /*/*/commits/
Disallow: /*/*/branches
Disallow: /*/*/contributors
Disallow: /*/*/tags
Disallow: /*/*/stargazers
Disallow: /*/*/watchers
Disallow: /*/*/network
Disallow: /*/*/graphs
Disallow: /*/*/compare

Disallow: /*/tree/
Disallow: /gist/
Disallow: /*/download
Disallow: /*/revisions
Disallow: /*/commits/*?author
Disallow: /*/commits/*?path
Disallow: /*/comments
Disallow: /*/archive/
Disallow: /*/blame/
Disallow: /*/raw/
Disallow: /*/cache/
Disallow: /.git/
Disallow: */.git/
Disallow: /*.git$
Disallow: /search/advanced
Disallow: /search$
Disallow: /*q=
Disallow: /*.atom$

Disallow: /ekansa/Open-Context-Data
Disallow: /ekansa/opencontext-*
Disallow: */tarball/
Disallow: */zipball/

Disallow: /*source=*
Disallow: /*ref_cta=*
Disallow: /*plan=*
Disallow: /*return_to=*
Disallow: /*ref_loc=*
Disallow: /*setup_organization=*
Disallow: /*source_repo=*
Disallow: /*ref_page=*
Disallow: /*source=*
Disallow: /*referrer=*
Disallow: /*report=*
Disallow: /*author=*
Disallow: /*since=*
Disallow: /*until=*
Disallow: /*commits?author=*
Disallow: /*report-abuse?report=*
Disallow: /*tab=*
Allow: /*?tab=achievements&achievement=*

Disallow: /account-login
Disallow: /Explodingstuff/


# If you would like to crawl GitHub contact us via https://support.github.com?tags=dotcom-robots
# We also provide an extensive API: https://docs.github.com
User-agent: baidu
crawl-delay: 1

User-agent: *

Disallow: /*/*/pulse
Disallow: /*/*/projects
Disallow: /*/*/forks
Disallow: /*/*/issues/new
Disallow: /*/*/milestones/new
Disallow: /*/*/issues/search
Disallow: /*/*/commits/
Disallow: /*/*/branches
Disallow: /*/*/contributors
Disallow: /*/*/tags
Disallow: /*/*/stargazers
Disallow: /*/*/watchers
Disallow: /*/*/network
Disallow: /*/*/graphs
Disallow: /*/*/compare

Disallow: /*/tree/
Disallow: /gist/
Disallow: /*/download
Disallow: /*/revisions
Disallow: /*/commits/*?author
Disallow: /*/commits/*?path
Disallow: /*/comments
Disallow: /*/archive/
Disallow: /*/blame/
Disallow: /*/raw/
Disallow: /*/cache/
Disallow: /.git/
Disallow: */.git/
Disallow: /*.git$
Disallow: /search/advanced
Disallow: /search$
Disallow: /*q=
Disallow: /*.atom$

Disallow: /ekansa/Open-Context-Data
Disallow: /ekansa/opencontext-*
Disallow: */tarball/
Disallow: */zipball/

Disallow: /*source=*
Disallow: /*ref_cta=*
Disallow: /*plan=*
Disallow: /*return_to=*
Disallow: /*ref_loc=*
Disallow: /*setup_organization=*
Disallow: /*source_repo=*
Disallow: /*ref_page=*
Disallow: /*source=*
Disallow: /*referrer=*
Disallow: /*report=*
Disallow: /*author=*
Disallow: /*since=*
Disallow: /*until=*
Disallow: /*commits?author=*
Disallow: /*report-abuse?report=*
Disallow: /*tab=*
Allow: /*?tab=achievements&achievement=*

Disallow: /account-login
Disallow: /Explodingstuff/


# If you would like to crawl GitHub contact us via https://support.github.com?tags=dotcom-robots
# We also provide an extensive API: https://docs.github.com
User-agent: baidu
crawl-delay: 1

User-agent: *

Disallow: /*/*/pulse
Disallow: /*/*/projects
Disallow: /*/*/forks
Disallow: /*/*/issues/new
Disallow: /*/*/milestones/new
Disallow: /*/*/issues/search
Disallow: /*/*/commits/
Disallow: /*/*/branches
Disallow: /*/*/contributors
Disallow: /*/*/tags
Disallow: /*/*/stargazers
Disallow: /*/*/watchers
Disallow: /*/*/network
Disallow: /*/*/graphs
Disallow: /*/*/compare

Disallow: /*/tree/
Disallow: /gist/
Disallow: /*/download
Disallow: /*/revisions
Disallow: /*/commits/*?author
Disallow: /*/commits/*?path
Disallow: /*/comments
Disallow: /*/archive/
Disallow: /*/blame/
Disallow: /*/raw/
Disallow: /*/cache/
Disallow: /.git/
Disallow: */.git/
Disallow: /*.git$
Disallow: /search/advanced
Disallow: /search$
Disallow: /*q=
Disallow: /*.atom$

Disallow: /ekansa/Open-Context-Data
Disallow: /ekansa/opencontext-*
Disallow: */tarball/
Disallow: */zipball/

Disallow: /*source=*
Disallow: /*ref_cta=*
Disallow: /*plan=*
Disallow: /*return_to=*
Disallow: /*ref_loc=*
Disallow: /*setup_organization=*
Disallow: /*source_repo=*
Disallow: /*ref_page=*
Disallow: /*source=*
Disallow: /*referrer=*
Disallow: /*report=*
Disallow: /*author=*
Disallow: /*since=*
Disallow: /*until=*
Disallow: /*commits?author=*
Disallow: /*report-abuse?report=*
Disallow: /*tab=*
Allow: /*?tab=achievements&achievement=*

Disallow: /account-login
Disallow: /Explodingstuff/

Disallow: /copilot/
Disallow: /copilot/c/


# If you would like to crawl GitHub contact us via https://support.github.com?tags=dotcom-robots
# We also provide an extensive API: https://docs.github.com
User-agent: bingbot
Disallow: /ekansa/Open-Context-Data
Disallow: /ekansa/opencontext-*
Disallow: /account-login
Disallow: */tarball/
Disallow: */zipball/
Disallow: /Explodingstuff/
Disallow: /copilot/
Disallow: /copilot/c/

User-agent: adidxbot
Disallow: /ekansa/Open-Context-Data
Disallow: /ekansa/opencontext-*
Disallow: /account-login
Disallow: */tarball/
Disallow: */zipball/
Disallow: /Explodingstuff/
Disallow: /copilot/
Disallow: /copilot/c/

User-agent: BingPreview
Disallow: /ekansa/Open-Context-Data
Disallow: /ekansa/opencontext-*
Disallow: /account-login
Disallow: */tarball/
Disallow: */zipball/
Disallow: /Explodingstuff/
Disallow: /copilot/
Disallow: /copilot/c/

User-agent: baidu
crawl-delay: 1

User-agent: *

Disallow: /*/*/pulse
Disallow: /*/*/projects
Disallow: /*/*/forks
Disallow: /*/*/issues/new
Disallow: /*/*/milestones/new
Disallow: /*/*/issues/search
Disallow: /*/*/commits/
Disallow: /*/*/branches
Disallow: /*/*/contributors
Disallow: /*/*/tags
Disallow: /*/*/stargazers
Disallow: /*/*/watchers
Disallow: /*/*/network
Disallow: /*/*/graphs
Disallow: /*/*/compare

Disallow: /*/tree/
Disallow: /gist/
Disallow: /*/download
Disallow: /*/revisions
Disallow: /*/commits/*?author
Disallow: /*/commits/*?path
Disallow: /*/comments
Disallow: /*/archive/
Disallow: /*/blame/
Disallow: /*/raw/
Disallow: /*/cache/
Disallow: /.git/
Disallow: */.git/
Disallow: /*.git$
Disallow: /search/advanced
Disallow: /search$
Disallow: /*q=
Disallow: /*.atom$

Disallow: /ekansa/Open-Context-Data
Disallow: /ekansa/opencontext-*
Disallow: */tarball/
Disallow: */zipball/

Disallow: /*source=*
Disallow: /*ref_cta=*
Disallow: /*plan=*
Disallow: /*return_to=*
Disallow: /*ref_loc=*
Disallow: /*setup_organization=*
Disallow: /*source_repo=*
Disallow: /*ref_page=*
Disallow: /*source=*
Disallow: /*referrer=*
Disallow: /*report=*
Disallow: /*author=*
Disallow: /*since=*
Disallow: /*until=*
Disallow: /*commits?author=*
Disallow: /*report-abuse?report=*
Disallow: /*tab=*
Allow: /*?tab=achievements&achievement=*

Disallow: /account-login
Disallow: /Explodingstuff/

Disallow: /copilot/
Disallow: /copilot/c/


# See http://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file
#
# To ban all spiders from the entire site uncomment the next two lines:
# User-Agent: *
# Disallow: /

# Add a 1 second delay between successive requests to the same server, limits resources used by crawler
# Only some crawlers respect this setting, e.g. Googlebot does not
# Crawl-delay: 1

# Based on details in https://gitlab.com/gitlab-org/gitlab/blob/master/config/routes.rb,
# https://gitlab.com/gitlab-org/gitlab/blob/master/spec/routing, and using application

# Global routes
User-Agent: *
Disallow: /autocomplete/users
Disallow: /autocomplete/projects
Disallow: /search
Disallow: /admin
Disallow: /profile
Disallow: /dashboard
Disallow: /users
Disallow: /help
Disallow: /s/
Disallow: /-/profile
Disallow: /-/ide/
# Restrict allowed routes to avoid very ugly search results
Allow: /users/sign_in
Allow: /users/*/snippets

# Generic resource routes like new, edit, raw
# This will block routes like:
# - /projects/new
# - /gitlab-org/gitlab-foss/issues/123/-/edit
User-Agent: *
Disallow: /*/new
Disallow: /*/edit
Disallow: /*/raw
Disallow: /*/realtime_changes

# Group details
User-Agent: *
Disallow: /groups/*/analytics
Disallow: /groups/*/contribution_analytics
Disallow: /groups/*/group_members

# Project details
User-Agent: *
Disallow: /*/*.git$
Disallow: /*/archive/
Disallow: /*/repository/archive*
Disallow: /*/activity
Disallow: /*/blame
Disallow: /*/commits
Disallow: /*/commit
Disallow: /*/commit/*.patch
Disallow: /*/commit/*.diff
Disallow: /*/compare
Disallow: /*/network
Disallow: /*/graphs
Disallow: /*/merge_requests/*.patch
Disallow: /*/merge_requests/*.diff
Disallow: /*/merge_requests/*/diffs
Disallow: /*/deploy_keys
Disallow: /*/hooks
Disallow: /*/services
Disallow: /*/protected_branches
Disallow: /*/uploads/
Disallow: /*/project_members
Disallow: /*/settings
Disallow: /*/-/import
Disallow: /*/-/environments
Disallow: /*/-/jobs
Disallow: /*/-/requirements_management
Disallow: /*/-/pipelines
Disallow: /*/-/pipeline_schedules
Disallow: /*/-/dependencies
Disallow: /*/-/licenses
Disallow: /*/-/metrics
Disallow: /*/-/incidents
Disallow: /*/-/value_stream_analytics
Disallow: /*/-/analytics
Disallow: /*/insights