#
# robots.txt
#
# This file is to prevent the crawling and indexing of certain parts
# of your site by web crawlers and spiders run by sites like Yahoo!
# and Google. By telling these "robots" where not to go on your site,
# you save bandwidth and server resources.
#
# This file will be ignored unless it is at the root of your host:
# Used:    http://example.com/robots.txt
# Ignored: http://example.com/site/robots.txt
#
# For more information about the robots.txt standard, see:
# http://www.robotstxt.org/robotstxt.html

User-agent: *
Crawl-delay: 10
# CSS, JS, Images
Allow: /misc/*.css$
Allow: /misc/*.css?
Allow: /misc/*.js$
Allow: /misc/*.js?
Allow: /misc/*.gif
Allow: /misc/*.jpg
Allow: /misc/*.jpeg
Allow: /misc/*.png
Allow: /modules/*.css$
Allow: /modules/*.css?
Allow: /modules/*.js$
Allow: /modules/*.js?
Allow: /modules/*.gif
Allow: /modules/*.jpg
Allow: /modules/*.jpeg
Allow: /modules/*.png
Allow: /profiles/*.css$
Allow: /profiles/*.css?
Allow: /profiles/*.js$
Allow: /profiles/*.js?
Allow: /profiles/*.gif
Allow: /profiles/*.jpg
Allow: /profiles/*.jpeg
Allow: /profiles/*.png
Allow: /themes/*.css$
Allow: /themes/*.css?
Allow: /themes/*.js$
Allow: /themes/*.js?
Allow: /themes/*.gif
Allow: /themes/*.jpg
Allow: /themes/*.jpeg
Allow: /themes/*.png
# Directories
Disallow: /includes/
Disallow: /misc/
Disallow: /modules/
Disallow: /profiles/
Disallow: /scripts/
Disallow: /themes/
# Files
Disallow: /CHANGELOG.txt
Disallow: /cron.php
Disallow: /INSTALL.mysql.txt
Disallow: /INSTALL.pgsql.txt
Disallow: /INSTALL.sqlite.txt
Disallow: /install.php
Disallow: /INSTALL.txt
Disallow: /LICENSE.txt
Disallow: /MAINTAINERS.txt
Disallow: /update.php
Disallow: /UPGRADE.txt
Disallow: /xmlrpc.php
# Paths (clean URLs)
Disallow: /admin/
Disallow: /comment/reply/
Disallow: /filter/tips/
Disallow: /node/add/
Disallow: /search/
Disallow: /user/register/
Disallow: /user/password/
Disallow: /user/login/
Disallow: /user/logout/
# Paths (no clean URLs)
Disallow: /?q=admin/
Disallow: /?q=comment/reply/
Disallow: /?q=filter/tips/
Disallow: /?q=node/add/
Disallow: /?q=search/
Disallow: /?q=user/password/
Disallow: /?q=user/register/
Disallow: /?q=user/login/
Disallow: /?q=user/logout/


User-agent: *
Disallow: /login
Disallow: /register
Disallow: /submit
Disallow: /jms
Disallow: /articles/search
Disallow: /*/download/*/
Disallow: /journal_files/

Sitemap: https://openpsychologydata.metajnl.com/sitemap.xml


# Googlebot
User-agent: Googlebot
Allow: /

# Bing
User-agent: bingbot
Allow: /

# YandexBot
User-agent: YandexBot
Allow: /

# Yahoo
User-agent: Slurp
Allow: /

# AhrefsBot
User-agent: AhrefsBot
Disallow: /

# Barkrowler
User-agent: Barkrowler
Disallow: /

# Clickagy Intelligence Bot
User-agent: Clickagy Intelligence Bot v2
Disallow: /

# Cloudfind
User-agent: Cloudfind/1.0
Disallow: /

# DotBot
User-agent: dotbot
Disallow: /

# Dorkbot
User-agent: UT-Dorkbot/1.0
Disallow: /

# Grapeshot
User-agent: GrapeshotCrawler
Disallow: /

# LTX71
User-agent: ltx71
Disallow: /

# MegaIndex
User-agent: MegaIndex.ru/2.0
Disallow: /

User-agent: MegaIndex.ru
Disallow: /

User-agent: megaIndex.ru
Disallow: /

# mj12bot
User-Agent: MJ12bot
Disallow: /

# SemrushBot
User-Agent: SemrushBot
Disallow: /

# All other bots
User-agent: *
Disallow: /


# Googlebot
User-agent: Googlebot
Disallow: /print/*
Allow: /

# Bingbot
User-agent: Bingbot
Disallow: /print/*
Allow: /

# DuckDuckBot
User-agent: DuckDuckBot
Disallow: /print/*
Allow: /

# Applebot
User-agent: Applebot
Disallow: /print/*
Allow: /

# All other bots
User-agent: *
Disallow: /

Sitemap: https://openpsychologydata.metajnl.com/sitemap.xml


# Googlebot
User-agent: Googlebot
Disallow: /print/*
Allow: /

# Bingbot
User-agent: Bingbot
Disallow: /print/*
Allow: /

# DuckDuckBot
User-agent: DuckDuckBot
Disallow: /print/*
Allow: /

# Applebot
User-agent: Applebot
Disallow: /print/*
Allow: /

# All other bots
User-agent: *
Disallow: /

Sitemap: undefined/sitemap.xml


User-agent: *
Disallow: /ViewMedia.cfm
Disallow: /viewmedia.cfm
Disallow: /user/

User-agent: Googlebot
Disallow: /user/referencelogin.cfm
Disallow: /user/

User-agent: bingbot
Disallow: /user/referencelogin.cfm
Disallow: /user/

User-agent: AhrefsBot
Disallow: /

User-agent: Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0) LinkCheck by Siteimprove.com
Disallow: /


User-agent: *
Disallow: /ViewMedia.cfm
Disallow: /viewmedia.cfm
Disallow: /user/

User-agent: Googlebot
Disallow: /user/referencelogin.cfm
Disallow: /user/

User-agent: bingbot
Disallow: /user/referencelogin.cfm
Disallow: /user/

User-agent: AhrefsBot
Disallow: /

User-agent: Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0) LinkCheck by Siteimprove.com
Disallow: /

User-agent: PetalBot
Disallow: /


User-agent: *
Disallow: /ViewMedia.cfm
Disallow: /viewmedia.cfm
Disallow: /user/
Disallow: /openathens.cfm
Disallow: /getImage.cfm
Disalow: /getImagev2.cfm
Disallow: /getimage.cfm
Disalow: /getimagev2.cfm

User-agent: Googlebot
Disallow: /user/referencelogin.cfm
Disallow: /user/

User-agent: bingbot
Disallow: /user/referencelogin.cfm
Disallow: /user/

User-agent: AhrefsBot
Disallow: /

User-agent: Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0) LinkCheck by Siteimprove.com
Disallow: /

User-agent: PetalBot
Disallow: /


User-agent: *
Disallow: /ViewMedia.cfm
Disallow: /viewmedia.cfm
Disallow: /user/
Disallow: /openathens.cfm
Disallow: /getImage.cfm
Disalow: /getImagev2.cfm
Disallow: /getimage.cfm
Disalow: /getimagev2.cfm

User-agent: Googlebot
Disallow: /user/referencelogin.cfm
Disallow: /user/

User-agent: bingbot
Disallow: /user/referencelogin.cfm
Disallow: /user/

User-agent: AhrefsBot
Disallow: /

User-agent: Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0) LinkCheck by Siteimprove.com
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: GPTBot
Disallow: /


User-agent: *
Disallow: /ViewMedia.cfm
Disallow: /viewmedia.cfm
Disallow: /user/
Disallow: /openathens.cfm
Disallow: /getImage.cfm
Disalow: /getImagev2.cfm
Disallow: /getimage.cfm
Disalow: /getimagev2.cfm

User-agent: Googlebot
Disallow: /user/referencelogin.cfm
Disallow: /user/

User-agent: bingbot
Disallow: /user/referencelogin.cfm
Disallow: /user/

User-agent: AhrefsBot
Disallow: /

User-agent: Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0) LinkCheck by Siteimprove.com
Disallow: /

User-agent: PetalBot
User-agent: AdsBot-Google
User-agent: Amazonbot
User-agent: anthropic-ai
User-agent: Applebot
User-agent: AwarioRssBot
User-agent: AwarioSmartBot
User-agent: Bytespider
User-agent: CCBot
User-agent: ChatGPT-User
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: cohere-ai
User-agent: DataForSeoBot
User-agent: Diffbot
User-agent: FacebookBot
User-agent: FriendlyCrawler
User-agent: GPTBot
User-agent: img2dataset
User-agent: ImagesiftBot
User-agent: magpie-crawler
User-agent: Meltwater
User-agent: omgili
User-agent: omgilibot
User-agent: peer39_crawler
User-agent: peer39_crawler/1.0
User-agent: PerplexityBot
User-agent: PiplBot
User-agent: scoop.it
User-agent: Seekr
User-agent: YouBot
Disallow: /


User-agent: *
Disallow: /ViewMedia.cfm
Disallow: /viewmedia.cfm
Disallow: /user/
Disallow: /openathens.cfm
Disallow: /getImage.cfm
Disalow: /getImagev2.cfm
Disallow: /getimage.cfm
Disalow: /getimagev2.cfm

# Algolia-Crawler-Verif: FDE4C78B0290C718

User-agent: Googlebot
Disallow: /user/referencelogin.cfm
Disallow: /user/

User-agent: bingbot
Disallow: /user/referencelogin.cfm
Disallow: /user/

User-agent: AhrefsBot
Disallow: /

User-agent: Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0) LinkCheck by Siteimprove.com
Disallow: /

User-agent: PetalBot
User-agent: AdsBot-Google
User-agent: Amazonbot
User-agent: anthropic-ai
User-agent: Applebot
User-agent: AwarioRssBot
User-agent: AwarioSmartBot
User-agent: Bytespider
User-agent: CCBot
User-agent: ChatGPT-User
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: cohere-ai
User-agent: DataForSeoBot
User-agent: Diffbot
User-agent: FacebookBot
User-agent: FriendlyCrawler
User-agent: GPTBot
User-agent: img2dataset
User-agent: ImagesiftBot
User-agent: magpie-crawler
User-agent: Meltwater
User-agent: omgili
User-agent: omgilibot
User-agent: peer39_crawler
User-agent: peer39_crawler/1.0
User-agent: PerplexityBot
User-agent: PiplBot
User-agent: scoop.it
User-agent: Seekr
User-agent: YouBot
Disallow: /


User-agent: *
Disallow: /ViewMedia.cfm
Disallow: /viewmedia.cfm
Disallow: /user/
Disallow: /openathens.cfm
Disallow: /getImage.cfm
Disalow: /getImagev2.cfm
Disallow: /getimage.cfm
Disalow: /getimagev2.cfm

# Algolia-Crawler-Verif: FDE4C78B0290C718

User-agent: Googlebot
Disallow: /user/referencelogin.cfm
Disallow: /user/

User-agent: bingbot
Disallow: /user/referencelogin.cfm
Disallow: /user/

User-agent: AhrefsBot
Disallow: /

User-agent: Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0) LinkCheck by Siteimprove.com
Disallow: /

User-agent: PetalBot
User-agent: Amazonbot
User-agent: anthropic-ai
User-agent: Applebot
User-agent: AwarioRssBot
User-agent: AwarioSmartBot
User-agent: Bytespider
User-agent: CCBot
User-agent: ChatGPT-User
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: cohere-ai
User-agent: DataForSeoBot
User-agent: Diffbot
User-agent: FacebookBot
User-agent: FriendlyCrawler
User-agent: GPTBot
User-agent: img2dataset
User-agent: ImagesiftBot
User-agent: magpie-crawler
User-agent: Meltwater
User-agent: omgili
User-agent: omgilibot
User-agent: peer39_crawler
User-agent: peer39_crawler/1.0
User-agent: PerplexityBot
User-agent: PiplBot
User-agent: scoop.it
User-agent: Seekr
User-agent: YouBot
Disallow: /


User-agent: Betty-Reader
Allow: /
Disallow: /user/referencelogin.cfm
Disallow: /user/
Disallow: /viewmedia.cfm
Disallow: /getimagev2.cfm

User-agent: *
Disallow: /ViewMedia.cfm
Disallow: /viewmedia.cfm
Disallow: /user/
Disallow: /openathens.cfm
Disallow: /getImage.cfm
Disallow: /getImagev2.cfm
Disallow: /getimage.cfm
Disallow: /getimagev2.cfm
# Algolia-Crawler-Verif: FDE4C78B0290C718

User-agent: Googlebot
Disallow: /user/referencelogin.cfm
Disallow: /user/

User-agent: bingbot
Disallow: /user/referencelogin.cfm
Disallow: /user/

User-agent: AhrefsBot
Disallow: /

User-agent: Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0) LinkCheck by Siteimprove.com
Disallow: /

User-agent: PetalBot
User-agent: Amazonbot
User-agent: anthropic-ai
User-agent: Applebot
User-agent: AwarioRssBot
User-agent: AwarioSmartBot
User-agent: Bytespider
User-agent: CCBot
User-agent: ChatGPT-User
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: cohere-ai
User-agent: DataForSeoBot
User-agent: Diffbot
User-agent: FacebookBot
User-agent: FriendlyCrawler
User-agent: GPTBot
User-agent: img2dataset
User-agent: ImagesiftBot
User-agent: magpie-crawler
User-agent: Meltwater
User-agent: omgili
User-agent: omgilibot
User-agent: peer39_crawler
User-agent: peer39_crawler/1.0
User-agent: PerplexityBot
User-agent: PiplBot
User-agent: scoop.it
User-agent: Seekr
User-agent: YouBot
User-agent: facebookexternalhit
Disallow: /


User-agent: *
Disallow: /login
Disallow: /logout
Disallow: /random
Disallow: /big-map
Disallow: /signup.html
Disallow: /lost-password.html
Disallow: /actions/
Disallow: /invitations/


User-agent: Sogou web spider
  Disallow: /

User-agent: *
  Disallow: /login
  Disallow: /logout
  Disallow: /random
  Disallow: /big-map
  Disallow: /signup.html
  Disallow: /lost-password.html
  Disallow: /actions/
  Disallow: /invitations/


User-agent: *
Disallow: /wp-admin
Disallow: /wp-login.php

Sitemap: https://petapixel.com/sitemap_index.xml


User-agent: *
Disallow: /wp-admin/
Disallow: /wp-login.php

Sitemap: https://petapixel.com/sitemap_index.xml
Sitemap: https://petapixel.com/news-sitemap.xml


User-agent: *
Disallow: /wp-admin/
Disallow: /wp-login.php

Sitemap: https://petapixel.com/sitemap_index.xml


User-agent: *
Disallow: /wp-admin/
Disallow: /wp-login.php

User-agent: GPTBot
Disallow: /

Sitemap: https://petapixel.com/sitemap_index.xml


# Block some general WP endpoints
# -------------------------------
User-agent: *
User-agent: AdsBot-Google
Disallow: /wp-json/
Disallow: /?rest_route=
Disallow: /wp-admin/
Disallow: /wp-content/cache/
Disallow: /wp-content/plugins/
Disallow: /wp-login.php
Disallow: /xmlrpc.php

# Special handling for /wp-includes/
# ----------------------------------
User-agent: *
User-agent: AdsBot-Google
Disallow: /wp-includes/
Allow: /wp-includes/css/
Allow: /wp-includes/js/

# Block internal search
# ---------------------
User-agent: *
User-agent: AdsBot-Google
Disallow: /?s=
Disallow: /page/*/?s=
Disallow: /search/

# Block leaky Cloudflare endpoints
# --------------------------------
User-agent: *
User-agent: AdsBot-Google
Disallow: /cdn-cgi/

# Sitemap
# -------
Sitemap: https://petapixel.com/sitemap_index.xml


# See http://www.robotstxt.org/wc/norobots.html for documentation on how to use the robots.txt file
User-agent: *
Disallow: /cgi-bin/
Disallow: /gallery/
Disallow: /photos/
Disallow: /webmail/
Disallow: /Webmail/
Disallow: /mail/
Disallow: /Mail/
Disallow: /roundcubemail/
Disallow: /pacman/
Disallow: /PACMAN/
Disallow: /account/
Disallow: /student/
Disallow: /keys/
Disallow: /webnotes/
Disallow: /calendar/
Disallow: /webcal/
Disallow: /interview/
Disallow: /video/
Disallow: /download/
Disallow: /fedora/


# See http://www.robotstxt.org/wc/norobots.html for documentation on how to use the robots.txt file
User-agent: *
Disallow: /


crawl-delay: 5

User-agent: *
Disallow: /MathJax/
Disallow: /archives/
Disallow: /cgi-bin/
Disallow: /ck-editor/
Disallow: /css/
Disallow: /diffs/
Disallow: /font/
Disallow: /inc/
Disallow: /js/
Disallow: /referer/
Disallow: /rss/
Disallow: /subject-editors/
Disallow: /tmp/
Disallow: /usage/
Disallow: /wwwstat/
Disallow: /search/
Disallow: /perl/

User-Agent: ia_archiver
Disallow: /

User-Agent: W3C-checklink
Disallow:

user-agent: Pinterestbot
allow: /entries/


crawl-delay: 5

User-agent: *
Disallow: /MathJax/
Disallow: /archives/
Disallow: /cgi-bin/
Disallow: /ck-editor/
Disallow: /css/
Disallow: /diffs/
Disallow: /font/
Disallow: /inc/
Disallow: /js/
Disallow: /referer/
Disallow: /rss/
Disallow: /subject-editors/
Disallow: /tmp/
Disallow: /symbols/
Disallow: /usage/
Disallow: /wwwstat/
Disallow: /search/
Disallow: /perl/

User-Agent: ia_archiver
Disallow: /

User-Agent: W3C-checklink
Disallow:

user-agent: Pinterestbot
allow: /entries/


crawl-delay: 5

User-agent: *
Disallow: /MathJax/
Disallow: /mathjax/
Disallow: /archives/
Disallow: /Archives/
Disallow: /cgi-bin/
Disallow: /ck-editor/
Disallow: /css/
Disallow: /CSS/
Disallow: /diffs/
Disallow: /DIFFS/
Disallow: /font/
Disallow: /FONT/
Disallow: /inc/
Disallow: /INC/
Disallow: /js/
Disallow: /JS/
Disallow: /referer/
Disallow: /rss/
Disallow: /subject-editors/
Disallow: /tmp/
Disallow: /symbols/
Disallow: /Symbols/
Disallow: /usage/
Disallow: /Usage/
Disallow: /wwwstat/
Disallow: /Wwwstat/
Disallow: /search/
Disallow: /perl/

User-Agent: ia_archiver
Disallow: /

User-Agent: W3C-checklink
Disallow:

user-agent: Pinterestbot
allow: /entries/


crawl-delay: 5

User-agent: *
Disallow: /MathJax/
Disallow: /mathjax/
Disallow: /archiveS/
Disallow: /archivEs/
Disallow: /archivES/
Disallow: /archiVes/
Disallow: /archiVeS/
Disallow: /archiVEs/
Disallow: /archiVES/
Disallow: /archIves/
Disallow: /archIveS/
Disallow: /archIvEs/
Disallow: /archIvES/
Disallow: /archIVes/
Disallow: /archIVeS/
Disallow: /archIVEs/
Disallow: /archIVES/
Disallow: /arcHives/
Disallow: /arcHiveS/
Disallow: /arcHivEs/
Disallow: /arcHivES/
Disallow: /arcHiVes/
Disallow: /arcHiVeS/
Disallow: /arcHiVEs/
Disallow: /arcHiVES/
Disallow: /arcHIves/
Disallow: /arcHIveS/
Disallow: /arcHIvEs/
Disallow: /arcHIvES/
Disallow: /arcHIVes/
Disallow: /arcHIVeS/
Disallow: /arcHIVEs/
Disallow: /arcHIVES/
Disallow: /arChives/
Disallow: /arChiveS/
Disallow: /arChivEs/
Disallow: /arChivES/
Disallow: /arChiVes/
Disallow: /arChiVeS/
Disallow: /arChiVEs/
Disallow: /arChiVES/
Disallow: /arChIves/
Disallow: /arChIveS/
Disallow: /arChIvEs/
Disallow: /arChIvES/
Disallow: /arChIVes/
Disallow: /arChIVeS/
Disallow: /arChIVEs/
Disallow: /arChIVES/
Disallow: /arCHives/
Disallow: /arCHiveS/
Disallow: /arCHivEs/
Disallow: /arCHivES/
Disallow: /arCHiVes/
Disallow: /arCHiVeS/
Disallow: /arCHiVEs/
Disallow: /arCHiVES/
Disallow: /arCHIves/
Disallow: /arCHIveS/
Disallow: /arCHIvEs/
Disallow: /arCHIvES/
Disallow: /arCHIVes/
Disallow: /arCHIVeS/
Disallow: /arCHIVEs/
Disallow: /arCHIVES/
Disallow: /aRchives/
Disallow: /aRchiveS/
Disallow: /aRchivEs/
Disallow: /aRchivES/
Disallow: /aRchiVes/
Disallow: /aRchiVeS/
Disallow: /aRchiVEs/
Disallow: /aRchiVES/
Disallow: /aRchIves/
Disallow: /aRchIveS/
Disallow: /aRchIvEs/
Disallow: /aRchIvES/
Disallow: /aRchIVes/
Disallow: /aRchIVeS/
Disallow: /aRchIVEs/
Disallow: /aRchIVES/
Disallow: /aRcHives/
Disallow: /aRcHiveS/
Disallow: /aRcHivEs/
Disallow: /aRcHivES/
Disallow: /aRcHiVes/
Disallow: /aRcHiVeS/
Disallow: /aRcHiVEs/
Disallow: /aRcHiVES/
Disallow: /aRcHIves/
Disallow: /aRcHIveS/
Disallow: /aRcHIvEs/
Disallow: /aRcHIvES/
Disallow: /aRcHIVes/
Disallow: /aRcHIVeS/
Disallow: /aRcHIVEs/
Disallow: /aRcHIVES/
Disallow: /aRChives/
Disallow: /aRChiveS/
Disallow: /aRChivEs/
Disallow: /aRChivES/
Disallow: /aRChiVes/
Disallow: /aRChiVeS/
Disallow: /aRChiVEs/
Disallow: /aRChiVES/
Disallow: /aRChIves/
Disallow: /aRChIveS/
Disallow: /aRChIvEs/
Disallow: /aRChIvES/
Disallow: /aRChIVes/
Disallow: /aRChIVeS/
Disallow: /aRChIVEs/
Disallow: /aRChIVES/
Disallow: /aRCHives/
Disallow: /aRCHiveS/
Disallow: /aRCHivEs/
Disallow: /aRCHivES/
Disallow: /aRCHiVes/
Disallow: /aRCHiVeS/
Disallow: /aRCHiVEs/
Disallow: /aRCHiVES/
Disallow: /aRCHIves/
Disallow: /aRCHIveS/
Disallow: /aRCHIvEs/
Disallow: /aRCHIvES/
Disallow: /aRCHIVes/
Disallow: /aRCHIVeS/
Disallow: /aRCHIVEs/
Disallow: /aRCHIVES/
Disallow: /Archives/
Disallow: /ArchiveS/
Disallow: /ArchivEs/
Disallow: /ArchivES/
Disallow: /ArchiVes/
Disallow: /ArchiVeS/
Disallow: /ArchiVEs/
Disallow: /ArchiVES/
Disallow: /ArchIves/
Disallow: /ArchIveS/
Disallow: /ArchIvEs/
Disallow: /ArchIvES/
Disallow: /ArchIVes/
Disallow: /ArchIVeS/
Disallow: /ArchIVEs/
Disallow: /ArchIVES/
Disallow: /ArcHives/
Disallow: /ArcHiveS/
Disallow: /ArcHivEs/
Disallow: /ArcHivES/
Disallow: /ArcHiVes/
Disallow: /ArcHiVeS/
Disallow: /ArcHiVEs/
Disallow: /ArcHiVES/
Disallow: /ArcHIves/
Disallow: /ArcHIveS/
Disallow: /ArcHIvEs/
Disallow: /ArcHIvES/
Disallow: /ArcHIVes/
Disallow: /ArcHIVeS/
Disallow: /ArcHIVEs/
Disallow: /ArcHIVES/
Disallow: /ArChives/
Disallow: /ArChiveS/
Disallow: /ArChivEs/
Disallow: /ArChivES/
Disallow: /ArChiVes/
Disallow: /ArChiVeS/
Disallow: /ArChiVEs/
Disallow: /ArChiVES/
Disallow: /ArChIves/
Disallow: /ArChIveS/
Disallow: /ArChIvEs/
Disallow: /ArChIvES/
Disallow: /ArChIVes/
Disallow: /ArChIVeS/
Disallow: /ArChIVEs/
Disallow: /ArChIVES/
Disallow: /ArCHives/
Disallow: /ArCHiveS/
Disallow: /ArCHivEs/
Disallow: /ArCHivES/
Disallow: /ArCHiVes/
Disallow: /ArCHiVeS/
Disallow: /ArCHiVEs/
Disallow: /ArCHiVES/
Disallow: /ArCHIves/
Disallow: /ArCHIveS/
Disallow: /ArCHIvEs/
Disallow: /ArCHIvES/
Disallow: /ArCHIVes/
Disallow: /ArCHIVeS/
Disallow: /ArCHIVEs/
Disallow: /ArCHIVES/
Disallow: /ARchives/
Disallow: /ARchiveS/
Disallow: /ARchivEs/
Disallow: /ARchivES/
Disallow: /ARchiVes/
Disallow: /ARchiVeS/
Disallow: /ARchiVEs/
Disallow: /ARchiVES/
Disallow: /ARchIves/
Disallow: /ARchIveS/
Disallow: /ARchIvEs/
Disallow: /ARchIvES/
Disallow: /ARchIVes/
Disallow: /ARchIVeS/
Disallow: /ARchIVEs/
Disallow: /ARchIVES/
Disallow: /ARcHives/
Disallow: /ARcHiveS/
Disallow: /ARcHivEs/
Disallow: /ARcHivES/
Disallow: /ARcHiVes/
Disallow: /ARcHiVeS/
Disallow: /ARcHiVEs/
Disallow: /ARcHiVES/
Disallow: /ARcHIves/
Disallow: /ARcHIveS/
Disallow: /ARcHIvEs/
Disallow: /ARcHIvES/
Disallow: /ARcHIVes/
Disallow: /ARcHIVeS/
Disallow: /ARcHIVEs/
Disallow: /ARcHIVES/
Disallow: /ARChives/
Disallow: /ARChiveS/
Disallow: /ARChivEs/
Disallow: /ARChivES/
Disallow: /ARChiVes/
Disallow: /ARChiVeS/
Disallow: /ARChiVEs/
Disallow: /ARChiVES/
Disallow: /ARChIves/
Disallow: /ARChIveS/
Disallow: /ARChIvEs/
Disallow: /ARChIvES/
Disallow: /ARChIVes/
Disallow: /ARChIVeS/
Disallow: /ARChIVEs/
Disallow: /ARChIVES/
Disallow: /ARCHives/
Disallow: /ARCHiveS/
Disallow: /ARCHivEs/
Disallow: /ARCHivES/
Disallow: /ARCHiVes/
Disallow: /ARCHiVeS/
Disallow: /ARCHiVEs/
Disallow: /ARCHiVES/
Disallow: /ARCHIves/
Disallow: /ARCHIveS/
Disallow: /ARCHIvEs/
Disallow: /ARCHIvES/
Disallow: /ARCHIVes/
Disallow: /ARCHIVeS/
Disallow: /ARCHIVEs/
Disallow: /ARCHIVES/
Disallow: /cgi-bin/
Disallow: /ck-editor/
Disallow: /css/
Disallow: /CSS/
Disallow: /diffs/
Disallow: /DIFFS/
Disallow: /font/
Disallow: /FONT/
Disallow: /inc/
Disallow: /INC/
Disallow: /js/
Disallow: /JS/
Disallow: /referer/
Disallow: /rss/
Disallow: /subject-editors/
Disallow: /tmp/
Disallow: /symbols/
Disallow: /Symbols/
Disallow: /usage/
Disallow: /Usage/
Disallow: /wwwstat/
Disallow: /Wwwstat/
Disallow: /search/
Disallow: /perl/

User-Agent: ia_archiver
Disallow: /

User-Agent: W3C-checklink
Disallow:

user-agent: Pinterestbot
allow: /entries/


User-agent: *
Disallow: /


User-agent: *
Crawl-delay: 1
Allow: /$
Allow: /about/
Allow: /articles/
Allow: /sitemap/
Allow: /tools/
Allow: /pub/
Allow: /api/
Disallow: /

Sitemap: https://pmc.ncbi.nlm.nih.gov/sitemap/


User-agent: *
Crawl-delay: 1
Allow: /$
Allow: /about/
Allow: /articles/
Allow: /sitemap/
Allow: /tools/
Allow: /pub/
Allow: /api/
Allow: /tagging-guidelines/
Disallow: /

Sitemap: https://pmc.ncbi.nlm.nih.gov/sitemap/


User-agent: *
Allow: /
Disallow: /antilles/
Disallow: /App_Code/
Disallow: /Bin/

User-agent: Mediapartners-Google
Allow: /

User-agent: Adsbot-Google
Allow: /

User-agent: Googlebot-Image
Allow: /

User-agent: Googlebot-Mobile
Allow: /

User-agent: Googlebot
Allow: /
Disallow: /antilles/
Disallow: /App_Code/
Disallow: /Bin/

Sitemap: https://presidentlincoln.illinois.gov/sitemap.xml


# If you are regularly crawling WordPress.com sites, please use our firehose to receive real-time push updates instead.
# Please see https://developer.wordpress.com/docs/firehose/ for more details.

Sitemap: https://prodigious.blog/sitemap.xml
Sitemap: https://prodigious.blog/news-sitemap.xml

User-agent: *
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /wp-signup.php
Disallow: /press-this.php
Disallow: /remote-login.php
Disallow: /activate/
Disallow: /cgi-bin/
Disallow: /mshots/v1/
Disallow: /next/
Disallow: /public.api/

# This file was generated on Sat, 11 Dec 2021 08:58:38 +0000


# If you are regularly crawling WordPress.com sites, please use our firehose to receive real-time push updates instead.
# Please see https://developer.wordpress.com/docs/firehose/ for more details.

Sitemap: https://prodigious.blog/sitemap.xml
Sitemap: https://prodigious.blog/news-sitemap.xml

User-agent: *
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /wp-signup.php
Disallow: /press-this.php
Disallow: /remote-login.php
Disallow: /activate/
Disallow: /cgi-bin/
Disallow: /mshots/v1/
Disallow: /next/
Disallow: /public.api/

# This file was generated on Tue, 28 Jun 2022 15:16:32 +0000


# If you are regularly crawling WordPress.com sites, please use our firehose to receive real-time push updates instead.
# Please see https://developer.wordpress.com/docs/firehose/ for more details.

Sitemap: https://prodigious.blog/sitemap.xml
Sitemap: https://prodigious.blog/news-sitemap.xml

User-agent: *
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /wp-signup.php
Disallow: /press-this.php
Disallow: /remote-login.php
Disallow: /activate/
Disallow: /cgi-bin/
Disallow: /mshots/v1/
Disallow: /next/
Disallow: /public.api/

# This file was generated on Fri, 16 Sep 2022 08:01:59 +0000


# If you are regularly crawling WordPress.com sites, please use our firehose to receive real-time push updates instead.
# Please see https://developer.wordpress.com/docs/firehose/ for more details.

Sitemap: https://prodigious.blog/sitemap.xml
Sitemap: https://prodigious.blog/news-sitemap.xml

User-agent: *
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /wp-signup.php
Disallow: /press-this.php
Disallow: /remote-login.php
Disallow: /activate/
Disallow: /cgi-bin/
Disallow: /mshots/v1/
Disallow: /next/
Disallow: /public.api/

# This file was generated on Sat, 03 Dec 2022 02:24:53 +0000


# If you are regularly crawling WordPress.com sites, please use our firehose to receive real-time push updates instead.
# Please see https://developer.wordpress.com/docs/firehose/ for more details.

Sitemap: https://prodigious.blog/sitemap.xml
Sitemap: https://prodigious.blog/news-sitemap.xml

User-agent: *
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /wp-signup.php
Disallow: /press-this.php
Disallow: /remote-login.php
Disallow: /activate/
Disallow: /cgi-bin/
Disallow: /mshots/v1/
Disallow: /next/
Disallow: /public.api/

User-agent: GPTBot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: SentiBot
Disallow: /

User-agent: sentibot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: FacebookBot
Disallow: /

User-agent: omgili
Disallow: /

User-agent: omgilibot
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: Bingbot
Disallow: /

# This file was generated on Tue, 10 Oct 2023 14:09:26 +0000


# If you are regularly crawling WordPress.com sites, please use our firehose to receive real-time push updates instead.
# Please see https://developer.wordpress.com/docs/firehose/ for more details.

Sitemap: https://prodigious.blog/sitemap.xml
Sitemap: https://prodigious.blog/news-sitemap.xml

User-agent: *
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /wp-signup.php
Disallow: /press-this.php
Disallow: /remote-login.php
Disallow: /activate/
Disallow: /cgi-bin/
Disallow: /mshots/v1/
Disallow: /next/
Disallow: /public.api/

# This file was generated on Tue, 04 Jun 2024 09:05:58 +0000


# If you are regularly crawling WordPress.com sites, please use our firehose to receive real-time push updates instead.
# Please see https://developer.wordpress.com/docs/firehose/ for more details.

Sitemap: https://prodigious.blog/sitemap.xml
Sitemap: https://prodigious.blog/news-sitemap.xml

User-agent: *
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /wp-signup.php
Disallow: /press-this.php
Disallow: /remote-login.php
Disallow: /activate/
Disallow: /cgi-bin/
Disallow: /mshots/v1/
Disallow: /next/
Disallow: /public.api/

# This file was generated on Mon, 21 Apr 2025 05:04:01 +0000


# robots.txt
# ========================================================================
#
# Version: 2004-10-01, PubChem Team
#
# ========================================================================

User-agent: Googlebot
Disallow: /assay/asis.cgi
Disallow: /assay/bioactivity.html
Disallow: /assay/getGraphHeatmap.cgi
Disallow: /assay/assayHeatmap.cgi
Disallow: /assay/assaytool.cgi
Disallow: /assay/assaycluster.cgi
Disallow: /assay/redirect.cgi
Disallow: /assay/assaydownload.cgi
Disallow: /assaysrv/assaysrv.cgi
Disallow: /idexchange/idexchange.cgi
Disallow: /pc_fetch/pc_fetch.cgi
Disallow: /score_matrix/score_matrix.cgi
Disallow: /deposit/deposit.cgi
Disallow: /upload/upload.cgi
Disallow: /pug/pug.cgi
Disallow: /rest/rdf/
Disallow: /rest/upload/
Disallow: /rest/pug/
Disallow: /vw3d/
Disallow: /he/
Disallow: /widget
Disallow: /toc/summary_toc.cgi
Disallow: /assay/assay.cgi
Disallow: /pug_soap/pug_soap.cgi
Allow: /upload/docs/
Allow: /compound/
Allow: /substance/
Allow: /bioassay/
Allow: /rest/
Allow: /rest/pug/sourcetable/
Allow: /rest/pug/periodictable/
Allow: /sitemap/
Allow: /ghs/
Allow: /erg/

User-Agent: bingbot
Disallow: /assay/asis.cgi
Disallow: /assay/bioactivity.html
Disallow: /assay/getGraphHeatmap.cgi
Disallow: /assay/assayHeatmap.cgi
Disallow: /assay/assaytool.cgi
Disallow: /assay/assaycluster.cgi
Disallow: /assay/redirect.cgi
Disallow: /assay/assaydownload.cgi
Disallow: /assaysrv/assaysrv.cgi
Disallow: /idexchange/idexchange.cgi
Disallow: /pc_fetch/pc_fetch.cgi
Disallow: /score_matrix/score_matrix.cgi
Disallow: /deposit/deposit.cgi
Disallow: /upload/upload.cgi
Disallow: /pug/pug.cgi
Disallow: /rest/rdf/
Disallow: /rest/upload/
Disallow: /rest/pug/
Disallow: /vw3d/
Disallow: /he/
Disallow: /widget
Disallow: /assay/assay.cgi
Disallow: /pug_soap/pug_soap.cgi
Disallow: /toc/summary_toc.cgi
Allow: /upload/docs/
Allow: /compound/
Allow: /substance/
Allow: /bioassay/
Allow: /rest/
Allow: /rest/pug/sourcetable/
Allow: /rest/pug/periodictable/
Allow: /sitemap/
Allow: /ghs/
Allow: /erg/

User-agent: htdig/3.1.6
Disallow: /assay/asis.cgi
Disallow: /assay/bioactivity.html
Disallow: /assay/getGraphHeatmap.cgi
Disallow: /assay/assayHeatmap.cgi
Disallow: /assay/assaytool.cgi
Disallow: /assay/assaycluster.cgi
Disallow: /assay/redirect.cgi
Disallow: /assay/assaydownload.cgi
Disallow: /assaysrv/assaysrv.cgi
Disallow: /idexchange/idexchange.cgi
Disallow: /pc_fetch/pc_fetch.cgi
Disallow: /score_matrix/score_matrix.cgi
Disallow: /deposit/deposit.cgi
Disallow: /upload/upload.cgi
Disallow: /pug/pug.cgi
Disallow: /rest/rdf/
Disallow: /rest/upload/
Disallow: /rest/pug/
Disallow: /vw3d/
Disallow: /he/
Disallow: /widget
Disallow: /assay/assay.cgi
Disallow: /pug_soap/pug_soap.cgi
Disallow: /toc/summary_toc.cgi
Allow: /upload/docs/
Allow: /compound/
Allow: /substance/
Allow: /bioassay/
Allow: /rest/
Allow: /rest/pug/sourcetable/
Allow: /rest/pug/periodictable/
Allow: /sitemap/
Allow: /ghs/
Allow: /erg/

User-agent: Slurp
Disallow: /assay/asis.cgi
Disallow: /assay/bioactivity.html
Disallow: /assay/getGraphHeatmap.cgi
Disallow: /assay/assayHeatmap.cgi
Disallow: /assay/assaytool.cgi
Disallow: /assay/assaycluster.cgi
Disallow: /assay/redirect.cgi
Disallow: /assay/assaydownload.cgi
Disallow: /assaysrv/assaysrv.cgi
Disallow: /idexchange/idexchange.cgi
Disallow: /pc_fetch/pc_fetch.cgi
Disallow: /score_matrix/score_matrix.cgi
Disallow: /deposit/deposit.cgi
Disallow: /upload/upload.cgi
Disallow: /pug/pug.cgi
Disallow: /rest/rdf/
Disallow: /rest/upload/
Disallow: /rest/pug/
Disallow: /vw3d/
Disallow: /he/
Disallow: /widget
Disallow: /assay/assay.cgi
Disallow: /pug_soap/pug_soap.cgi
Disallow: /toc/summary_toc.cgi
Allow: /upload/docs/
Allow: /compound/
Allow: /substance/
Allow: /bioassay/
Allow: /rest/
Allow: /rest/pug/sourcetable/
Allow: /rest/pug/periodictable/
Allow: /sitemap/
Allow: /ghs/
Allow: /erg/

User-agent: MSNBot
Disallow: /assay/asis.cgi
Disallow: /assay/bioactivity.html
Disallow: /assay/getGraphHeatmap.cgi
Disallow: /assay/assayHeatmap.cgi
Disallow: /assay/assaytool.cgi
Disallow: /assay/assaycluster.cgi
Disallow: /assay/redirect.cgi
Disallow: /assay/assaydownload.cgi
Disallow: /assaysrv/assaysrv.cgi
Disallow: /idexchange/idexchange.cgi
Disallow: /pc_fetch/pc_fetch.cgi
Disallow: /score_matrix/score_matrix.cgi
Disallow: /deposit/deposit.cgi
Disallow: /upload/upload.cgi
Disallow: /pug/pug.cgi
Disallow: /rest/rdf/
Disallow: /rest/upload/
Disallow: /rest/pug/
Disallow: /vw3d/
Disallow: /he/
Disallow: /widget
Disallow: /assay/assay.cgi
Disallow: /pug_soap/pug_soap.cgi
Disallow: /toc/summary_toc.cgi
Allow: /upload/docs/
Allow: /compound/
Allow: /substance/
Allow: /rest/
Allow: /rest/pug/sourcetable/
Allow: /rest/pug/periodictable/
Allow: /sitemap/
Allow: /ghs/
Allow: /erg/

User-agent: Googlebot-Mobile
Disallow: /assay/asis.cgi
Disallow: /assay/bioactivity.html
Disallow: /assay/getGraphHeatmap.cgi
Disallow: /assay/assayHeatmap.cgi
Disallow: /assay/assaytool.cgi
Disallow: /assay/assaycluster.cgi
Disallow: /assay/redirect.cgi
Disallow: /assay/assaydownload.cgi
Disallow: /assaysrv/assaysrv.cgi
Disallow: /idexchange/idexchange.cgi
Disallow: /pc_fetch/pc_fetch.cgi
Disallow: /score_matrix/score_matrix.cgi
Disallow: /deposit/deposit.cgi
Disallow: /upload/upload.cgi
Disallow: /pug/pug.cgi
Disallow: /rest/rdf/
Disallow: /rest/upload/
Disallow: /rest/pug/
Disallow: /vw3d/
Disallow: /he/
Disallow: /widget
Disallow: /assay/assay.cgi
Disallow: /pug_soap/pug_soap.cgi
Disallow: /toc/summary_toc.cgi
Allow: /upload/docs/
Allow: /compound/
Allow: /substance/
Allow: /bioassay/
Allow: /rest/
Allow: /rest/pug/sourcetable/
Allow: /rest/pug/periodictable/
Allow: /sitemap/
Allow: /ghs/
Allow: /erg/

User-agent: Googlebot-Image
Allow: /image/imgsrv.fcgi
Allow: /image/img3d.cgi
Disallow: /he/

User-agent: Baiduspider
Disallow: /assay/asis.cgi
Disallow: /assay/bioactivity.html
Disallow: /assay/getGraphHeatmap.cgi
Disallow: /assay/assayHeatmap.cgi
Disallow: /assay/assaytool.cgi
Disallow: /assay/assaycluster.cgi
Disallow: /assay/redirect.cgi
Disallow: /assay/assaydownload.cgi
Disallow: /assaysrv/assaysrv.cgi
Disallow: /idexchange/idexchange.cgi
Disallow: /pc_fetch/pc_fetch.cgi
Disallow: /score_matrix/score_matrix.cgi
Disallow: /image/img3d.cgi
Disallow: /image/imagefly.cgi
Disallow: /image/structurefly.cgi
Disallow: /deposit/deposit.cgi
Disallow: /upload/upload.cgi
Disallow: /pug/pug.cgi
Disallow: /rest/rdf/
Disallow: /rest/upload/
Disallow: /rest/pug/
Disallow: /vw3d/
Disallow: /he/
Disallow: /widget
Disallow: /assay/assay.cgi
Disallow: /pug_soap/pug_soap.cgi
Disallow: /toc/summary_toc.cgi
Allow: /upload/docs/
Allow: /compound/
Allow: /substance/
Allow: /bioassay/
Allow: /rest/
Allow: /rest/pug/sourcetable/
Allow: /rest/pug/periodictable/
Allow: /sitemap/
Allow: /ghs/
Allow: /erg/

User-agent: Teleport Pro
Disallow: /assay/asis.cgi
Disallow: /assay/bioactivity.html
Disallow: /assay/getGraphHeatmap.cgi
Disallow: /assay/assayHeatmap.cgi
Disallow: /assay/assaytool.cgi
Disallow: /assay/assaycluster.cgi
Disallow: /assay/redirect.cgi
Disallow: /assay/assaydownload.cgi
Disallow: /assaysrv/assaysrv.cgi
Disallow: /idexchange/idexchange.cgi
Disallow: /pc_fetch/pc_fetch.cgi
Disallow: /score_matrix/score_matrix.cgi
Disallow: /image/img3d.cgi
Disallow: /image/imagefly.cgi
Disallow: /image/structurefly.cgi
Disallow: /deposit/deposit.cgi
Disallow: /upload/upload.cgi
Disallow: /pug/pug.cgi
Disallow: /rest/rdf/
Disallow: /rest/upload/
Disallow: /rest/pug/
Disallow: /vw3d/
Disallow: /he/
Disallow: /widget
Disallow: /assay/assay.cgi
Disallow: /pug_soap/pug_soap.cgi
Disallow: /toc/summary_toc.cgi
Allow: /upload/docs/
Allow: /compound/
Allow: /substance/
Allow: /bioassay/
Allow: /rest/
Allow: /rest/pug/sourcetable/
Allow: /rest/pug/periodictable/
Allow: /sitemap/
Allow: /ghs/
Allow: /erg/

User-agent: yacybot
Disallow: *

User-agent: Browsershots
Disallow:

User-agent: *
Disallow: /assay/asis.cgi
Disallow: /assay/bioactivity.html
Disallow: /assay/getGraphHeatmap.cgi
Disallow: /assay/assayHeatmap.cgi
Disallow: /assay/assaytool.cgi
Disallow: /assay/assaycluster.cgi
Disallow: /assay/redirect.cgi
Disallow: /assay/assaydownload.cgi
Disallow: /assaysrv/assaysrv.cgi
Disallow: /idexchange/idexchange.cgi
Disallow: /pc_fetch/pc_fetch.cgi
Disallow: /score_matrix/score_matrix.cgi
Disallow: /image/img3d.cgi
Disallow: /image/imagefly.cgi
Disallow: /image/structurefly.cgi
Disallow: /deposit/deposit.cgi
Disallow: /upload/upload.cgi
Disallow: /pug/pug.cgi
Disallow: /rest/rdf/
Disallow: /rest/upload/
Disallow: /rest/pug/
Disallow: /vw3d/
Disallow: /he/
Disallow: /widget
Disallow: /pug_soap/pug_soap.cgi

Sitemap: https://pubchem.ncbi.nlm.nih.gov/sitemap/pcsitemap.xml
Sitemap: https://pubchem.ncbi.nlm.nih.gov/sitemap/sitemap.cgi?p=annotation/sitemap_index.xml
Sitemap: https://pubchem.ncbi.nlm.nih.gov/sitemap/sitemap.cgi?p=assay/sitemap_index.xml
Sitemap: https://pubchem.ncbi.nlm.nih.gov/sitemap/sitemap.cgi?p=source/sitemap.xml
Sitemap: https://pubchem.ncbi.nlm.nih.gov/sitemap/sitemap.cgi?p=gene/sitemap_index.xml
Sitemap: https://pubchem.ncbi.nlm.nih.gov/sitemap/sitemap.cgi?p=protein/sitemap_index.xml
Sitemap: https://pubchem.ncbi.nlm.nih.gov/sitemap/sitemap.cgi?p=patent/sitemap_index.xml

# ==========================================================
# Version: 2004-10-01, PubChem Team
# Revision: 2015
# Revision: 2016
# Revision: 2017
# Revision: 2018
# Revision: 2019-06-04


# robots.txt
# ========================================================================
#
# Version: 2004-10-01, PubChem Team
#
# ========================================================================

User-agent: Googlebot
Disallow: /assay/asis.cgi
Disallow: /assay/bioactivity.html
Disallow: /assay/getGraphHeatmap.cgi
Disallow: /assay/assayHeatmap.cgi
Disallow: /assay/assaytool.cgi
Disallow: /assay/assaycluster.cgi
Disallow: /assay/redirect.cgi
Disallow: /assay/assaydownload.cgi
Disallow: /assaysrv/assaysrv.cgi
Disallow: /idexchange/idexchange.cgi
Disallow: /pc_fetch/pc_fetch.cgi
Disallow: /score_matrix/score_matrix.cgi
Disallow: /deposit/deposit.cgi
Disallow: /upload/upload.cgi
Disallow: /pug/pug.cgi
Disallow: /rest/rdf/
Disallow: /rest/upload/
Disallow: /rest/pug/
Disallow: /vw3d/
Disallow: /he/
Disallow: /widget
Disallow: /toc/summary_toc.cgi
Disallow: /assay/assay.cgi
Disallow: /pug_soap/pug_soap.cgi
Allow: /upload/docs/
Allow: /compound/
Allow: /substance/
Allow: /bioassay/
Allow: /rest/
Allow: /rest/pug/sourcetable/
Allow: /rest/pug/periodictable/
Allow: /sitemap/
Allow: /ghs/
Allow: /erg/

User-Agent: bingbot
Disallow: /assay/asis.cgi
Disallow: /assay/bioactivity.html
Disallow: /assay/getGraphHeatmap.cgi
Disallow: /assay/assayHeatmap.cgi
Disallow: /assay/assaytool.cgi
Disallow: /assay/assaycluster.cgi
Disallow: /assay/redirect.cgi
Disallow: /assay/assaydownload.cgi
Disallow: /assaysrv/assaysrv.cgi
Disallow: /idexchange/idexchange.cgi
Disallow: /pc_fetch/pc_fetch.cgi
Disallow: /score_matrix/score_matrix.cgi
Disallow: /deposit/deposit.cgi
Disallow: /upload/upload.cgi
Disallow: /pug/pug.cgi
Disallow: /rest/rdf/
Disallow: /rest/upload/
Disallow: /rest/pug/
Disallow: /vw3d/
Disallow: /he/
Disallow: /widget
Disallow: /assay/assay.cgi
Disallow: /pug_soap/pug_soap.cgi
Disallow: /toc/summary_toc.cgi
Allow: /upload/docs/
Allow: /compound/
Allow: /substance/
Allow: /bioassay/
Allow: /rest/
Allow: /rest/pug/sourcetable/
Allow: /rest/pug/periodictable/
Allow: /sitemap/
Allow: /ghs/
Allow: /erg/

User-agent: htdig/3.1.6
Disallow: /assay/asis.cgi
Disallow: /assay/bioactivity.html
Disallow: /assay/getGraphHeatmap.cgi
Disallow: /assay/assayHeatmap.cgi
Disallow: /assay/assaytool.cgi
Disallow: /assay/assaycluster.cgi
Disallow: /assay/redirect.cgi
Disallow: /assay/assaydownload.cgi
Disallow: /assaysrv/assaysrv.cgi
Disallow: /idexchange/idexchange.cgi
Disallow: /pc_fetch/pc_fetch.cgi
Disallow: /score_matrix/score_matrix.cgi
Disallow: /deposit/deposit.cgi
Disallow: /upload/upload.cgi
Disallow: /pug/pug.cgi
Disallow: /rest/rdf/
Disallow: /rest/upload/
Disallow: /rest/pug/
Disallow: /vw3d/
Disallow: /he/
Disallow: /widget
Disallow: /assay/assay.cgi
Disallow: /pug_soap/pug_soap.cgi
Disallow: /toc/summary_toc.cgi
Allow: /upload/docs/
Allow: /compound/
Allow: /substance/
Allow: /bioassay/
Allow: /rest/
Allow: /rest/pug/sourcetable/
Allow: /rest/pug/periodictable/
Allow: /sitemap/
Allow: /ghs/
Allow: /erg/

User-agent: Slurp
Disallow: /assay/asis.cgi
Disallow: /assay/bioactivity.html
Disallow: /assay/getGraphHeatmap.cgi
Disallow: /assay/assayHeatmap.cgi
Disallow: /assay/assaytool.cgi
Disallow: /assay/assaycluster.cgi
Disallow: /assay/redirect.cgi
Disallow: /assay/assaydownload.cgi
Disallow: /assaysrv/assaysrv.cgi
Disallow: /idexchange/idexchange.cgi
Disallow: /pc_fetch/pc_fetch.cgi
Disallow: /score_matrix/score_matrix.cgi
Disallow: /deposit/deposit.cgi
Disallow: /upload/upload.cgi
Disallow: /pug/pug.cgi
Disallow: /rest/rdf/
Disallow: /rest/upload/
Disallow: /rest/pug/
Disallow: /vw3d/
Disallow: /he/
Disallow: /widget
Disallow: /assay/assay.cgi
Disallow: /pug_soap/pug_soap.cgi
Disallow: /toc/summary_toc.cgi
Allow: /upload/docs/
Allow: /compound/
Allow: /substance/
Allow: /bioassay/
Allow: /rest/
Allow: /rest/pug/sourcetable/
Allow: /rest/pug/periodictable/
Allow: /sitemap/
Allow: /ghs/
Allow: /erg/

User-agent: MSNBot
Disallow: /assay/asis.cgi
Disallow: /assay/bioactivity.html
Disallow: /assay/getGraphHeatmap.cgi
Disallow: /assay/assayHeatmap.cgi
Disallow: /assay/assaytool.cgi
Disallow: /assay/assaycluster.cgi
Disallow: /assay/redirect.cgi
Disallow: /assay/assaydownload.cgi
Disallow: /assaysrv/assaysrv.cgi
Disallow: /idexchange/idexchange.cgi
Disallow: /pc_fetch/pc_fetch.cgi
Disallow: /score_matrix/score_matrix.cgi
Disallow: /deposit/deposit.cgi
Disallow: /upload/upload.cgi
Disallow: /pug/pug.cgi
Disallow: /rest/rdf/
Disallow: /rest/upload/
Disallow: /rest/pug/
Disallow: /vw3d/
Disallow: /he/
Disallow: /widget
Disallow: /assay/assay.cgi
Disallow: /pug_soap/pug_soap.cgi
Disallow: /toc/summary_toc.cgi
Allow: /upload/docs/
Allow: /compound/
Allow: /substance/
Allow: /rest/
Allow: /rest/pug/sourcetable/
Allow: /rest/pug/periodictable/
Allow: /sitemap/
Allow: /ghs/
Allow: /erg/

User-agent: Googlebot-Mobile
Disallow: /assay/asis.cgi
Disallow: /assay/bioactivity.html
Disallow: /assay/getGraphHeatmap.cgi
Disallow: /assay/assayHeatmap.cgi
Disallow: /assay/assaytool.cgi
Disallow: /assay/assaycluster.cgi
Disallow: /assay/redirect.cgi
Disallow: /assay/assaydownload.cgi
Disallow: /assaysrv/assaysrv.cgi
Disallow: /idexchange/idexchange.cgi
Disallow: /pc_fetch/pc_fetch.cgi
Disallow: /score_matrix/score_matrix.cgi
Disallow: /deposit/deposit.cgi
Disallow: /upload/upload.cgi
Disallow: /pug/pug.cgi
Disallow: /rest/rdf/
Disallow: /rest/upload/
Disallow: /rest/pug/
Disallow: /vw3d/
Disallow: /he/
Disallow: /widget
Disallow: /assay/assay.cgi
Disallow: /pug_soap/pug_soap.cgi
Disallow: /toc/summary_toc.cgi
Allow: /upload/docs/
Allow: /compound/
Allow: /substance/
Allow: /bioassay/
Allow: /rest/
Allow: /rest/pug/sourcetable/
Allow: /rest/pug/periodictable/
Allow: /sitemap/
Allow: /ghs/
Allow: /erg/

User-agent: Googlebot-Image
Allow: /image/imgsrv.fcgi
Allow: /image/img3d.cgi
Disallow: /he/

User-agent: Baiduspider
Disallow: /assay/asis.cgi
Disallow: /assay/bioactivity.html
Disallow: /assay/getGraphHeatmap.cgi
Disallow: /assay/assayHeatmap.cgi
Disallow: /assay/assaytool.cgi
Disallow: /assay/assaycluster.cgi
Disallow: /assay/redirect.cgi
Disallow: /assay/assaydownload.cgi
Disallow: /assaysrv/assaysrv.cgi
Disallow: /idexchange/idexchange.cgi
Disallow: /pc_fetch/pc_fetch.cgi
Disallow: /score_matrix/score_matrix.cgi
Disallow: /image/img3d.cgi
Disallow: /image/imagefly.cgi
Disallow: /image/structurefly.cgi
Disallow: /deposit/deposit.cgi
Disallow: /upload/upload.cgi
Disallow: /pug/pug.cgi
Disallow: /rest/rdf/
Disallow: /rest/upload/
Disallow: /rest/pug/
Disallow: /vw3d/
Disallow: /he/
Disallow: /widget
Disallow: /assay/assay.cgi
Disallow: /pug_soap/pug_soap.cgi
Disallow: /toc/summary_toc.cgi
Allow: /upload/docs/
Allow: /compound/
Allow: /substance/
Allow: /bioassay/
Allow: /rest/
Allow: /rest/pug/sourcetable/
Allow: /rest/pug/periodictable/
Allow: /sitemap/
Allow: /ghs/
Allow: /erg/

User-agent: Teleport Pro
Disallow: /assay/asis.cgi
Disallow: /assay/bioactivity.html
Disallow: /assay/getGraphHeatmap.cgi
Disallow: /assay/assayHeatmap.cgi
Disallow: /assay/assaytool.cgi
Disallow: /assay/assaycluster.cgi
Disallow: /assay/redirect.cgi
Disallow: /assay/assaydownload.cgi
Disallow: /assaysrv/assaysrv.cgi
Disallow: /idexchange/idexchange.cgi
Disallow: /pc_fetch/pc_fetch.cgi
Disallow: /score_matrix/score_matrix.cgi
Disallow: /image/img3d.cgi
Disallow: /image/imagefly.cgi
Disallow: /image/structurefly.cgi
Disallow: /deposit/deposit.cgi
Disallow: /upload/upload.cgi
Disallow: /pug/pug.cgi
Disallow: /rest/rdf/
Disallow: /rest/upload/
Disallow: /rest/pug/
Disallow: /vw3d/
Disallow: /he/
Disallow: /widget
Disallow: /assay/assay.cgi
Disallow: /pug_soap/pug_soap.cgi
Disallow: /toc/summary_toc.cgi
Allow: /upload/docs/
Allow: /compound/
Allow: /substance/
Allow: /bioassay/
Allow: /rest/
Allow: /rest/pug/sourcetable/
Allow: /rest/pug/periodictable/
Allow: /sitemap/
Allow: /ghs/
Allow: /erg/

User-agent: yacybot
Disallow: *

User-agent: Browsershots
Disallow:

User-agent: *
Disallow: /assay/asis.cgi
Disallow: /assay/bioactivity.html
Disallow: /assay/getGraphHeatmap.cgi
Disallow: /assay/assayHeatmap.cgi
Disallow: /assay/assaytool.cgi
Disallow: /assay/assaycluster.cgi
Disallow: /assay/redirect.cgi
Disallow: /assay/assaydownload.cgi
Disallow: /assaysrv/assaysrv.cgi
Disallow: /idexchange/idexchange.cgi
Disallow: /pc_fetch/pc_fetch.cgi
Disallow: /score_matrix/score_matrix.cgi
Disallow: /image/img3d.cgi
Disallow: /image/imagefly.cgi
Disallow: /image/structurefly.cgi
Disallow: /deposit/deposit.cgi
Disallow: /upload/upload.cgi
Disallow: /pug/pug.cgi
Disallow: /rest/rdf/
Disallow: /rest/upload/
Disallow: /rest/pug/
Disallow: /vw3d/
Disallow: /he/
Disallow: /widget
Disallow: /pug_soap/pug_soap.cgi

Sitemap: https://pubchem.ncbi.nlm.nih.gov/sitemap/pcsitemap.xml
Sitemap: https://pubchem.ncbi.nlm.nih.gov/sitemap/sitemap.cgi?p=annotation/sitemap_index.xml
Sitemap: https://pubchem.ncbi.nlm.nih.gov/sitemap/sitemap.cgi?p=assay/sitemap_index.xml
Sitemap: https://pubchem.ncbi.nlm.nih.gov/sitemap/sitemap.cgi?p=source/sitemap.xml
Sitemap: https://pubchem.ncbi.nlm.nih.gov/sitemap/sitemap.cgi?p=gene/sitemap_index.xml
Sitemap: https://pubchem.ncbi.nlm.nih.gov/sitemap/sitemap.cgi?p=protein/sitemap_index.xml
Sitemap: https://pubchem.ncbi.nlm.nih.gov/sitemap/sitemap.cgi?p=patent/sitemap_index.xml
Sitemap: https://pubchem.ncbi.nlm.nih.gov/sitemap/sitemap.cgi?p=concept/sitemap_1.xml

# ==========================================================
# Version: 2004-10-01, PubChem Team
# Revision: 2015
# Revision: 2016
# Revision: 2017
# Revision: 2018
# Revision: 2019-06-04
# Revision: 2022


# robots.txt
# ========================================================================
#
# Version: 2004-10-01, PubChem Team
#
# ========================================================================

User-agent: Googlebot
Disallow: /assay/asis.cgi
Disallow: /assay/bioactivity.html
Disallow: /assay/getGraphHeatmap.cgi
Disallow: /assay/assayHeatmap.cgi
Disallow: /assay/assaytool.cgi
Disallow: /assay/assaycluster.cgi
Disallow: /assay/redirect.cgi
Disallow: /assay/assaydownload.cgi
Disallow: /assaysrv/assaysrv.cgi
Disallow: /idexchange/idexchange.cgi
Disallow: /pc_fetch/pc_fetch.cgi
Disallow: /score_matrix/score_matrix.cgi
Disallow: /deposit/deposit.cgi
Disallow: /upload/upload.cgi
Disallow: /pug/pug.cgi
Disallow: /rest/rdf/
Disallow: /rest/upload/
Disallow: /rest/pug/
Disallow: /vw3d/
Disallow: /he/
Disallow: /widget
Disallow: /toc/summary_toc.cgi
Disallow: /assay/assay.cgi
Disallow: /pug_soap/pug_soap.cgi
Disallow: /unified_search/structure_search.cgi
Disallow: /search_3d/search_3d_front_end.cgi
Allow: /upload/docs/
Allow: /compound/
Allow: /substance/
Allow: /bioassay/
Allow: /rest/
Allow: /rest/pug/sourcetable/
Allow: /rest/pug/periodictable/
Allow: /sitemap/
Allow: /ghs/
Allow: /erg/

User-Agent: bingbot
Disallow: /assay/asis.cgi
Disallow: /assay/bioactivity.html
Disallow: /assay/getGraphHeatmap.cgi
Disallow: /assay/assayHeatmap.cgi
Disallow: /assay/assaytool.cgi
Disallow: /assay/assaycluster.cgi
Disallow: /assay/redirect.cgi
Disallow: /assay/assaydownload.cgi
Disallow: /assaysrv/assaysrv.cgi
Disallow: /idexchange/idexchange.cgi
Disallow: /pc_fetch/pc_fetch.cgi
Disallow: /score_matrix/score_matrix.cgi
Disallow: /deposit/deposit.cgi
Disallow: /upload/upload.cgi
Disallow: /pug/pug.cgi
Disallow: /rest/rdf/
Disallow: /rest/upload/
Disallow: /rest/pug/
Disallow: /vw3d/
Disallow: /he/
Disallow: /widget
Disallow: /assay/assay.cgi
Disallow: /pug_soap/pug_soap.cgi
Disallow: /unified_search/structure_search.cgi
Disallow: /search_3d/search_3d_front_end.cgi
Disallow: /toc/summary_toc.cgi
Allow: /upload/docs/
Allow: /compound/
Allow: /substance/
Allow: /bioassay/
Allow: /rest/
Allow: /rest/pug/sourcetable/
Allow: /rest/pug/periodictable/
Allow: /sitemap/
Allow: /ghs/
Allow: /erg/

User-agent: htdig/3.1.6
Disallow: /assay/asis.cgi
Disallow: /assay/bioactivity.html
Disallow: /assay/getGraphHeatmap.cgi
Disallow: /assay/assayHeatmap.cgi
Disallow: /assay/assaytool.cgi
Disallow: /assay/assaycluster.cgi
Disallow: /assay/redirect.cgi
Disallow: /assay/assaydownload.cgi
Disallow: /assaysrv/assaysrv.cgi
Disallow: /idexchange/idexchange.cgi
Disallow: /pc_fetch/pc_fetch.cgi
Disallow: /score_matrix/score_matrix.cgi
Disallow: /deposit/deposit.cgi
Disallow: /upload/upload.cgi
Disallow: /pug/pug.cgi
Disallow: /rest/rdf/
Disallow: /rest/upload/
Disallow: /rest/pug/
Disallow: /vw3d/
Disallow: /he/
Disallow: /widget
Disallow: /assay/assay.cgi
Disallow: /pug_soap/pug_soap.cgi
Disallow: /unified_search/structure_search.cgi
Disallow: /search_3d/search_3d_front_end.cgi
Disallow: /toc/summary_toc.cgi
Allow: /upload/docs/
Allow: /compound/
Allow: /substance/
Allow: /bioassay/
Allow: /rest/
Allow: /rest/pug/sourcetable/
Allow: /rest/pug/periodictable/
Allow: /sitemap/
Allow: /ghs/
Allow: /erg/

User-agent: Slurp
Disallow: /assay/asis.cgi
Disallow: /assay/bioactivity.html
Disallow: /assay/getGraphHeatmap.cgi
Disallow: /assay/assayHeatmap.cgi
Disallow: /assay/assaytool.cgi
Disallow: /assay/assaycluster.cgi
Disallow: /assay/redirect.cgi
Disallow: /assay/assaydownload.cgi
Disallow: /assaysrv/assaysrv.cgi
Disallow: /idexchange/idexchange.cgi
Disallow: /pc_fetch/pc_fetch.cgi
Disallow: /score_matrix/score_matrix.cgi
Disallow: /deposit/deposit.cgi
Disallow: /upload/upload.cgi
Disallow: /pug/pug.cgi
Disallow: /rest/rdf/
Disallow: /rest/upload/
Disallow: /rest/pug/
Disallow: /vw3d/
Disallow: /he/
Disallow: /widget
Disallow: /assay/assay.cgi
Disallow: /pug_soap/pug_soap.cgi
Disallow: /unified_search/structure_search.cgi
Disallow: /search_3d/search_3d_front_end.cgi
Disallow: /toc/summary_toc.cgi
Allow: /upload/docs/
Allow: /compound/
Allow: /substance/
Allow: /bioassay/
Allow: /rest/
Allow: /rest/pug/sourcetable/
Allow: /rest/pug/periodictable/
Allow: /sitemap/
Allow: /ghs/
Allow: /erg/

User-agent: MSNBot
Disallow: /assay/asis.cgi
Disallow: /assay/bioactivity.html
Disallow: /assay/getGraphHeatmap.cgi
Disallow: /assay/assayHeatmap.cgi
Disallow: /assay/assaytool.cgi
Disallow: /assay/assaycluster.cgi
Disallow: /assay/redirect.cgi
Disallow: /assay/assaydownload.cgi
Disallow: /assaysrv/assaysrv.cgi
Disallow: /idexchange/idexchange.cgi
Disallow: /pc_fetch/pc_fetch.cgi
Disallow: /score_matrix/score_matrix.cgi
Disallow: /deposit/deposit.cgi
Disallow: /upload/upload.cgi
Disallow: /pug/pug.cgi
Disallow: /rest/rdf/
Disallow: /rest/upload/
Disallow: /rest/pug/
Disallow: /vw3d/
Disallow: /he/
Disallow: /widget
Disallow: /assay/assay.cgi
Disallow: /pug_soap/pug_soap.cgi
Disallow: /unified_search/structure_search.cgi
Disallow: /search_3d/search_3d_front_end.cgi
Disallow: /toc/summary_toc.cgi
Allow: /upload/docs/
Allow: /compound/
Allow: /substance/
Allow: /rest/
Allow: /rest/pug/sourcetable/
Allow: /rest/pug/periodictable/
Allow: /sitemap/
Allow: /ghs/
Allow: /erg/

User-agent: Googlebot-Mobile
Disallow: /assay/asis.cgi
Disallow: /assay/bioactivity.html
Disallow: /assay/getGraphHeatmap.cgi
Disallow: /assay/assayHeatmap.cgi
Disallow: /assay/assaytool.cgi
Disallow: /assay/assaycluster.cgi
Disallow: /assay/redirect.cgi
Disallow: /assay/assaydownload.cgi
Disallow: /assaysrv/assaysrv.cgi
Disallow: /idexchange/idexchange.cgi
Disallow: /pc_fetch/pc_fetch.cgi
Disallow: /score_matrix/score_matrix.cgi
Disallow: /deposit/deposit.cgi
Disallow: /upload/upload.cgi
Disallow: /pug/pug.cgi
Disallow: /rest/rdf/
Disallow: /rest/upload/
Disallow: /rest/pug/
Disallow: /vw3d/
Disallow: /he/
Disallow: /widget
Disallow: /assay/assay.cgi
Disallow: /pug_soap/pug_soap.cgi
Disallow: /unified_search/structure_search.cgi
Disallow: /search_3d/search_3d_front_end.cgi
Disallow: /toc/summary_toc.cgi
Allow: /upload/docs/
Allow: /compound/
Allow: /substance/
Allow: /bioassay/
Allow: /rest/
Allow: /rest/pug/sourcetable/
Allow: /rest/pug/periodictable/
Allow: /sitemap/
Allow: /ghs/
Allow: /erg/

User-agent: Googlebot-Image
Allow: /image/imgsrv.fcgi
Allow: /image/img3d.cgi
Disallow: /he/

User-agent: Baiduspider
Disallow: /assay/asis.cgi
Disallow: /assay/bioactivity.html
Disallow: /assay/getGraphHeatmap.cgi
Disallow: /assay/assayHeatmap.cgi
Disallow: /assay/assaytool.cgi
Disallow: /assay/assaycluster.cgi
Disallow: /assay/redirect.cgi
Disallow: /assay/assaydownload.cgi
Disallow: /assaysrv/assaysrv.cgi
Disallow: /idexchange/idexchange.cgi
Disallow: /pc_fetch/pc_fetch.cgi
Disallow: /score_matrix/score_matrix.cgi
Disallow: /image/img3d.cgi
Disallow: /image/imagefly.cgi
Disallow: /image/structurefly.cgi
Disallow: /deposit/deposit.cgi
Disallow: /upload/upload.cgi
Disallow: /pug/pug.cgi
Disallow: /rest/rdf/
Disallow: /rest/upload/
Disallow: /rest/pug/
Disallow: /vw3d/
Disallow: /he/
Disallow: /widget
Disallow: /assay/assay.cgi
Disallow: /pug_soap/pug_soap.cgi
Disallow: /unified_search/structure_search.cgi
Disallow: /search_3d/search_3d_front_end.cgi
Disallow: /toc/summary_toc.cgi
Allow: /upload/docs/
Allow: /compound/
Allow: /substance/
Allow: /bioassay/
Allow: /rest/
Allow: /rest/pug/sourcetable/
Allow: /rest/pug/periodictable/
Allow: /sitemap/
Allow: /ghs/
Allow: /erg/

User-agent: Teleport Pro
Disallow: /assay/asis.cgi
Disallow: /assay/bioactivity.html
Disallow: /assay/getGraphHeatmap.cgi
Disallow: /assay/assayHeatmap.cgi
Disallow: /assay/assaytool.cgi
Disallow: /assay/assaycluster.cgi
Disallow: /assay/redirect.cgi
Disallow: /assay/assaydownload.cgi
Disallow: /assaysrv/assaysrv.cgi
Disallow: /idexchange/idexchange.cgi
Disallow: /pc_fetch/pc_fetch.cgi
Disallow: /score_matrix/score_matrix.cgi
Disallow: /image/img3d.cgi
Disallow: /image/imagefly.cgi
Disallow: /image/structurefly.cgi
Disallow: /deposit/deposit.cgi
Disallow: /upload/upload.cgi
Disallow: /pug/pug.cgi
Disallow: /rest/rdf/
Disallow: /rest/upload/
Disallow: /rest/pug/
Disallow: /vw3d/
Disallow: /he/
Disallow: /widget
Disallow: /assay/assay.cgi
Disallow: /pug_soap/pug_soap.cgi
Disallow: /unified_search/structure_search.cgi
Disallow: /search_3d/search_3d_front_end.cgi
Disallow: /toc/summary_toc.cgi
Allow: /upload/docs/
Allow: /compound/
Allow: /substance/
Allow: /bioassay/
Allow: /rest/
Allow: /rest/pug/sourcetable/
Allow: /rest/pug/periodictable/
Allow: /sitemap/
Allow: /ghs/
Allow: /erg/

User-agent: yacybot
Disallow: *

User-agent: Browsershots
Disallow:

User-agent: *
Disallow: /assay/asis.cgi
Disallow: /assay/bioactivity.html
Disallow: /assay/getGraphHeatmap.cgi
Disallow: /assay/assayHeatmap.cgi
Disallow: /assay/assaytool.cgi
Disallow: /assay/assaycluster.cgi
Disallow: /assay/redirect.cgi
Disallow: /assay/assaydownload.cgi
Disallow: /assaysrv/assaysrv.cgi
Disallow: /idexchange/idexchange.cgi
Disallow: /pc_fetch/pc_fetch.cgi
Disallow: /score_matrix/score_matrix.cgi
Disallow: /image/img3d.cgi
Disallow: /image/imagefly.cgi
Disallow: /image/structurefly.cgi
Disallow: /deposit/deposit.cgi
Disallow: /upload/upload.cgi
Disallow: /pug/pug.cgi
Disallow: /rest/rdf/
Disallow: /rest/upload/
Disallow: /rest/pug/
Disallow: /vw3d/
Disallow: /he/
Disallow: /widget
Disallow: /pug_soap/pug_soap.cgi
Disallow: /unified_search/structure_search.cgi
Disallow: /search_3d/search_3d_front_end.cgi

Sitemap: https://pubchem.ncbi.nlm.nih.gov/sitemap/pcsitemap.xml
Sitemap: https://pubchem.ncbi.nlm.nih.gov/sitemap/sitemap.cgi?p=annotation/sitemap_index.xml
Sitemap: https://pubchem.ncbi.nlm.nih.gov/sitemap/sitemap.cgi?p=assay/sitemap_index.xml
Sitemap: https://pubchem.ncbi.nlm.nih.gov/sitemap/sitemap.cgi?p=source/sitemap.xml
Sitemap: https://pubchem.ncbi.nlm.nih.gov/sitemap/sitemap.cgi?p=gene/sitemap_index.xml
Sitemap: https://pubchem.ncbi.nlm.nih.gov/sitemap/sitemap.cgi?p=protein/sitemap_index.xml
Sitemap: https://pubchem.ncbi.nlm.nih.gov/sitemap/sitemap.cgi?p=patent/sitemap_index.xml
Sitemap: https://pubchem.ncbi.nlm.nih.gov/sitemap/sitemap.cgi?p=concept/sitemap_1.xml

# ==========================================================
# Version: 2004-10-01, PubChem Team
# Revision: 2015
# Revision: 2016
# Revision: 2017
# Revision: 2018
# Revision: 2019
# Revision: 2022


# robots.txt
# ========================================================================
#
# Version: 2004-10-01, PubChem Team
#
# ========================================================================

User-agent: Googlebot
Disallow: /assay/asis.cgi
Disallow: /assay/bioactivity.html
Disallow: /assay/getGraphHeatmap.cgi
Disallow: /assay/assayHeatmap.cgi
Disallow: /assay/assaytool.cgi
Disallow: /assay/assaycluster.cgi
Disallow: /assay/redirect.cgi
Disallow: /assay/assaydownload.cgi
Disallow: /assaysrv/assaysrv.cgi
Disallow: /idexchange/idexchange.cgi
Disallow: /pc_fetch/pc_fetch.cgi
Disallow: /score_matrix/score_matrix.cgi
Disallow: /deposit/deposit.cgi
Disallow: /upload/upload.cgi
Disallow: /pug/pug.cgi
Disallow: /rest/rdf/
Disallow: /rest/upload/
Disallow: /rest/pug/
Disallow: /vw3d/
Disallow: /he/
Disallow: /widget
Disallow: /toc/summary_toc.cgi
Disallow: /assay/assay.cgi
Disallow: /pug_soap/pug_soap.cgi
Disallow: /unified_search/structure_search.cgi
Disallow: /search_3d/search_3d_front_end.cgi
Allow: /upload/docs/
Allow: /compound/
Allow: /substance/
Allow: /bioassay/
Allow: /rest/
Allow: /rest/pug/sourcetable/
Allow: /rest/pug/periodictable/
Allow: /sitemap/
Allow: /ghs/
Allow: /docs/

User-Agent: bingbot
Disallow: /assay/asis.cgi
Disallow: /assay/bioactivity.html
Disallow: /assay/getGraphHeatmap.cgi
Disallow: /assay/assayHeatmap.cgi
Disallow: /assay/assaytool.cgi
Disallow: /assay/assaycluster.cgi
Disallow: /assay/redirect.cgi
Disallow: /assay/assaydownload.cgi
Disallow: /assaysrv/assaysrv.cgi
Disallow: /idexchange/idexchange.cgi
Disallow: /pc_fetch/pc_fetch.cgi
Disallow: /score_matrix/score_matrix.cgi
Disallow: /deposit/deposit.cgi
Disallow: /upload/upload.cgi
Disallow: /pug/pug.cgi
Disallow: /rest/rdf/
Disallow: /rest/upload/
Disallow: /rest/pug/
Disallow: /vw3d/
Disallow: /he/
Disallow: /widget
Disallow: /assay/assay.cgi
Disallow: /pug_soap/pug_soap.cgi
Disallow: /unified_search/structure_search.cgi
Disallow: /search_3d/search_3d_front_end.cgi
Disallow: /toc/summary_toc.cgi
Allow: /upload/docs/
Allow: /compound/
Allow: /substance/
Allow: /bioassay/
Allow: /rest/
Allow: /rest/pug/sourcetable/
Allow: /rest/pug/periodictable/
Allow: /sitemap/
Allow: /ghs/
Allow: /docs/

User-agent: htdig/3.1.6
Disallow: /assay/asis.cgi
Disallow: /assay/bioactivity.html
Disallow: /assay/getGraphHeatmap.cgi
Disallow: /assay/assayHeatmap.cgi
Disallow: /assay/assaytool.cgi
Disallow: /assay/assaycluster.cgi
Disallow: /assay/redirect.cgi
Disallow: /assay/assaydownload.cgi
Disallow: /assaysrv/assaysrv.cgi
Disallow: /idexchange/idexchange.cgi
Disallow: /pc_fetch/pc_fetch.cgi
Disallow: /score_matrix/score_matrix.cgi
Disallow: /deposit/deposit.cgi
Disallow: /upload/upload.cgi
Disallow: /pug/pug.cgi
Disallow: /rest/rdf/
Disallow: /rest/upload/
Disallow: /rest/pug/
Disallow: /vw3d/
Disallow: /he/
Disallow: /widget
Disallow: /assay/assay.cgi
Disallow: /pug_soap/pug_soap.cgi
Disallow: /unified_search/structure_search.cgi
Disallow: /search_3d/search_3d_front_end.cgi
Disallow: /toc/summary_toc.cgi
Allow: /upload/docs/
Allow: /compound/
Allow: /substance/
Allow: /bioassay/
Allow: /rest/
Allow: /rest/pug/sourcetable/
Allow: /rest/pug/periodictable/
Allow: /sitemap/
Allow: /ghs/
Allow: /docs/

User-agent: Slurp
Disallow: /assay/asis.cgi
Disallow: /assay/bioactivity.html
Disallow: /assay/getGraphHeatmap.cgi
Disallow: /assay/assayHeatmap.cgi
Disallow: /assay/assaytool.cgi
Disallow: /assay/assaycluster.cgi
Disallow: /assay/redirect.cgi
Disallow: /assay/assaydownload.cgi
Disallow: /assaysrv/assaysrv.cgi
Disallow: /idexchange/idexchange.cgi
Disallow: /pc_fetch/pc_fetch.cgi
Disallow: /score_matrix/score_matrix.cgi
Disallow: /deposit/deposit.cgi
Disallow: /upload/upload.cgi
Disallow: /pug/pug.cgi
Disallow: /rest/rdf/
Disallow: /rest/upload/
Disallow: /rest/pug/
Disallow: /vw3d/
Disallow: /he/
Disallow: /widget
Disallow: /assay/assay.cgi
Disallow: /pug_soap/pug_soap.cgi
Disallow: /unified_search/structure_search.cgi
Disallow: /search_3d/search_3d_front_end.cgi
Disallow: /toc/summary_toc.cgi
Allow: /upload/docs/
Allow: /compound/
Allow: /substance/
Allow: /bioassay/
Allow: /rest/
Allow: /rest/pug/sourcetable/
Allow: /rest/pug/periodictable/
Allow: /sitemap/
Allow: /ghs/
Allow: /docs/

User-agent: MSNBot
Disallow: /assay/asis.cgi
Disallow: /assay/bioactivity.html
Disallow: /assay/getGraphHeatmap.cgi
Disallow: /assay/assayHeatmap.cgi
Disallow: /assay/assaytool.cgi
Disallow: /assay/assaycluster.cgi
Disallow: /assay/redirect.cgi
Disallow: /assay/assaydownload.cgi
Disallow: /assaysrv/assaysrv.cgi
Disallow: /idexchange/idexchange.cgi
Disallow: /pc_fetch/pc_fetch.cgi
Disallow: /score_matrix/score_matrix.cgi
Disallow: /deposit/deposit.cgi
Disallow: /upload/upload.cgi
Disallow: /pug/pug.cgi
Disallow: /rest/rdf/
Disallow: /rest/upload/
Disallow: /rest/pug/
Disallow: /vw3d/
Disallow: /he/
Disallow: /widget
Disallow: /assay/assay.cgi
Disallow: /pug_soap/pug_soap.cgi
Disallow: /unified_search/structure_search.cgi
Disallow: /search_3d/search_3d_front_end.cgi
Disallow: /toc/summary_toc.cgi
Allow: /upload/docs/
Allow: /compound/
Allow: /substance/
Allow: /rest/
Allow: /rest/pug/sourcetable/
Allow: /rest/pug/periodictable/
Allow: /sitemap/
Allow: /ghs/
Allow: /docs/

User-agent: Googlebot-Mobile
Disallow: /assay/asis.cgi
Disallow: /assay/bioactivity.html
Disallow: /assay/getGraphHeatmap.cgi
Disallow: /assay/assayHeatmap.cgi
Disallow: /assay/assaytool.cgi
Disallow: /assay/assaycluster.cgi
Disallow: /assay/redirect.cgi
Disallow: /assay/assaydownload.cgi
Disallow: /assaysrv/assaysrv.cgi
Disallow: /idexchange/idexchange.cgi
Disallow: /pc_fetch/pc_fetch.cgi
Disallow: /score_matrix/score_matrix.cgi
Disallow: /deposit/deposit.cgi
Disallow: /upload/upload.cgi
Disallow: /pug/pug.cgi
Disallow: /rest/rdf/
Disallow: /rest/upload/
Disallow: /rest/pug/
Disallow: /vw3d/
Disallow: /he/
Disallow: /widget
Disallow: /assay/assay.cgi
Disallow: /pug_soap/pug_soap.cgi
Disallow: /unified_search/structure_search.cgi
Disallow: /search_3d/search_3d_front_end.cgi
Disallow: /toc/summary_toc.cgi
Allow: /upload/docs/
Allow: /compound/
Allow: /substance/
Allow: /bioassay/
Allow: /rest/
Allow: /rest/pug/sourcetable/
Allow: /rest/pug/periodictable/
Allow: /sitemap/
Allow: /ghs/
Allow: /docs/

User-agent: Googlebot-Image
Allow: /image/imgsrv.fcgi
Allow: /image/img3d.cgi
Disallow: /he/

User-agent: Baiduspider
Disallow: /assay/asis.cgi
Disallow: /assay/bioactivity.html
Disallow: /assay/getGraphHeatmap.cgi
Disallow: /assay/assayHeatmap.cgi
Disallow: /assay/assaytool.cgi
Disallow: /assay/assaycluster.cgi
Disallow: /assay/redirect.cgi
Disallow: /assay/assaydownload.cgi
Disallow: /assaysrv/assaysrv.cgi
Disallow: /idexchange/idexchange.cgi
Disallow: /pc_fetch/pc_fetch.cgi
Disallow: /score_matrix/score_matrix.cgi
Disallow: /image/img3d.cgi
Disallow: /image/imagefly.cgi
Disallow: /image/structurefly.cgi
Disallow: /deposit/deposit.cgi
Disallow: /upload/upload.cgi
Disallow: /pug/pug.cgi
Disallow: /rest/rdf/
Disallow: /rest/upload/
Disallow: /rest/pug/
Disallow: /vw3d/
Disallow: /he/
Disallow: /widget
Disallow: /assay/assay.cgi
Disallow: /pug_soap/pug_soap.cgi
Disallow: /unified_search/structure_search.cgi
Disallow: /search_3d/search_3d_front_end.cgi
Disallow: /toc/summary_toc.cgi
Allow: /upload/docs/
Allow: /compound/
Allow: /substance/
Allow: /bioassay/
Allow: /rest/
Allow: /rest/pug/sourcetable/
Allow: /rest/pug/periodictable/
Allow: /sitemap/
Allow: /ghs/
Allow: /docs/

User-agent: Teleport Pro
Disallow: /assay/asis.cgi
Disallow: /assay/bioactivity.html
Disallow: /assay/getGraphHeatmap.cgi
Disallow: /assay/assayHeatmap.cgi
Disallow: /assay/assaytool.cgi
Disallow: /assay/assaycluster.cgi
Disallow: /assay/redirect.cgi
Disallow: /assay/assaydownload.cgi
Disallow: /assaysrv/assaysrv.cgi
Disallow: /idexchange/idexchange.cgi
Disallow: /pc_fetch/pc_fetch.cgi
Disallow: /score_matrix/score_matrix.cgi
Disallow: /image/img3d.cgi
Disallow: /image/imagefly.cgi
Disallow: /image/structurefly.cgi
Disallow: /deposit/deposit.cgi
Disallow: /upload/upload.cgi
Disallow: /pug/pug.cgi
Disallow: /rest/rdf/
Disallow: /rest/upload/
Disallow: /rest/pug/
Disallow: /vw3d/
Disallow: /he/
Disallow: /widget
Disallow: /assay/assay.cgi
Disallow: /pug_soap/pug_soap.cgi
Disallow: /unified_search/structure_search.cgi
Disallow: /search_3d/search_3d_front_end.cgi
Disallow: /toc/summary_toc.cgi
Allow: /upload/docs/
Allow: /compound/
Allow: /substance/
Allow: /bioassay/
Allow: /rest/
Allow: /rest/pug/sourcetable/
Allow: /rest/pug/periodictable/
Allow: /sitemap/
Allow: /ghs/
Allow: /docs/

User-agent: yacybot
Disallow: *

User-agent: Browsershots
Disallow:

User-agent: *
Disallow: /assay/asis.cgi
Disallow: /assay/bioactivity.html
Disallow: /assay/getGraphHeatmap.cgi
Disallow: /assay/assayHeatmap.cgi
Disallow: /assay/assaytool.cgi
Disallow: /assay/assaycluster.cgi
Disallow: /assay/redirect.cgi
Disallow: /assay/assaydownload.cgi
Disallow: /assaysrv/assaysrv.cgi
Disallow: /idexchange/idexchange.cgi
Disallow: /pc_fetch/pc_fetch.cgi
Disallow: /score_matrix/score_matrix.cgi
Disallow: /image/img3d.cgi
Disallow: /image/imagefly.cgi
Disallow: /image/structurefly.cgi
Disallow: /deposit/deposit.cgi
Disallow: /upload/upload.cgi
Disallow: /pug/pug.cgi
Disallow: /rest/rdf/
Disallow: /rest/upload/
Disallow: /rest/pug/
Disallow: /vw3d/
Disallow: /he/
Disallow: /widget
Disallow: /pug_soap/pug_soap.cgi
Disallow: /unified_search/structure_search.cgi
Disallow: /search_3d/search_3d_front_end.cgi

Sitemap: https://pubchem.ncbi.nlm.nih.gov/sitemap/pcsitemap.xml
Sitemap: https://pubchem.ncbi.nlm.nih.gov/sitemap/sitemap.cgi?p=annotation/sitemap_index.xml
Sitemap: https://pubchem.ncbi.nlm.nih.gov/sitemap/sitemap.cgi?p=assay/sitemap_index.xml
Sitemap: https://pubchem.ncbi.nlm.nih.gov/sitemap/sitemap.cgi?p=source/sitemap.xml
Sitemap: https://pubchem.ncbi.nlm.nih.gov/sitemap/sitemap.cgi?p=gene/sitemap_index.xml
Sitemap: https://pubchem.ncbi.nlm.nih.gov/sitemap/sitemap.cgi?p=protein/sitemap_index.xml
Sitemap: https://pubchem.ncbi.nlm.nih.gov/sitemap/sitemap.cgi?p=patent/sitemap_index.xml
Sitemap: https://pubchem.ncbi.nlm.nih.gov/sitemap/sitemap.cgi?p=concept/sitemap_1.xml
Sitemap: https://pubchem.ncbi.nlm.nih.gov/sitemap/sitemap.cgi?p=docs

# ==========================================================
# Version: 2004-10-01, PubChem Team
# Revision: 2005-2022


# robots.txt
# ========================================================================
#
# Version: 2004-10-01, PubChem Team
#
# ========================================================================

User-agent: Googlebot
Disallow: /assay/asis.cgi
Disallow: /assay/bioactivity.html
Disallow: /assay/getGraphHeatmap.cgi
Disallow: /assay/assayHeatmap.cgi
Disallow: /assay/assaytool.cgi
Disallow: /assay/assaycluster.cgi
Disallow: /assay/redirect.cgi
Disallow: /assay/assaydownload.cgi
Disallow: /assaysrv/assaysrv.cgi
Disallow: /idexchange/idexchange.cgi
Disallow: /pc_fetch/pc_fetch.cgi
Disallow: /score_matrix/score_matrix.cgi
Disallow: /deposit/deposit.cgi
Disallow: /upload/upload.cgi
Disallow: /pug/pug.cgi
Disallow: /rest/rdf/
Disallow: /rest/upload/
Disallow: /rest/pug/
Disallow: /vw3d/
Disallow: /he/
Disallow: /widget
Disallow: /toc/summary_toc.cgi
Disallow: /assay/assay.cgi
Disallow: /pug_soap/pug_soap.cgi
Disallow: /unified_search/structure_search.cgi
Disallow: /search_3d/search_3d_front_end.cgi
Allow: /upload/docs/
Allow: /compound/
Allow: /substance/
Allow: /bioassay/
Allow: /rest/
Allow: /rest/pug/sourcetable/
Allow: /rest/pug/periodictable/
Allow: /sitemap/
Allow: /ghs/
Allow: /docs/

User-Agent: bingbot
Disallow: /assay/asis.cgi
Disallow: /assay/bioactivity.html
Disallow: /assay/getGraphHeatmap.cgi
Disallow: /assay/assayHeatmap.cgi
Disallow: /assay/assaytool.cgi
Disallow: /assay/assaycluster.cgi
Disallow: /assay/redirect.cgi
Disallow: /assay/assaydownload.cgi
Disallow: /assaysrv/assaysrv.cgi
Disallow: /idexchange/idexchange.cgi
Disallow: /pc_fetch/pc_fetch.cgi
Disallow: /score_matrix/score_matrix.cgi
Disallow: /deposit/deposit.cgi
Disallow: /upload/upload.cgi
Disallow: /pug/pug.cgi
Disallow: /rest/rdf/
Disallow: /rest/upload/
Disallow: /rest/pug/
Disallow: /vw3d/
Disallow: /he/
Disallow: /widget
Disallow: /assay/assay.cgi
Disallow: /pug_soap/pug_soap.cgi
Disallow: /unified_search/structure_search.cgi
Disallow: /search_3d/search_3d_front_end.cgi
Disallow: /toc/summary_toc.cgi
Allow: /upload/docs/
Allow: /compound/
Allow: /substance/
Allow: /bioassay/
Allow: /rest/
Allow: /rest/pug/sourcetable/
Allow: /rest/pug/periodictable/
Allow: /sitemap/
Allow: /ghs/
Allow: /docs/

User-agent: htdig/3.1.6
Disallow: /assay/asis.cgi
Disallow: /assay/bioactivity.html
Disallow: /assay/getGraphHeatmap.cgi
Disallow: /assay/assayHeatmap.cgi
Disallow: /assay/assaytool.cgi
Disallow: /assay/assaycluster.cgi
Disallow: /assay/redirect.cgi
Disallow: /assay/assaydownload.cgi
Disallow: /assaysrv/assaysrv.cgi
Disallow: /idexchange/idexchange.cgi
Disallow: /pc_fetch/pc_fetch.cgi
Disallow: /score_matrix/score_matrix.cgi
Disallow: /deposit/deposit.cgi
Disallow: /upload/upload.cgi
Disallow: /pug/pug.cgi
Disallow: /rest/rdf/
Disallow: /rest/upload/
Disallow: /rest/pug/
Disallow: /vw3d/
Disallow: /he/
Disallow: /widget
Disallow: /assay/assay.cgi
Disallow: /pug_soap/pug_soap.cgi
Disallow: /unified_search/structure_search.cgi
Disallow: /search_3d/search_3d_front_end.cgi
Disallow: /toc/summary_toc.cgi
Allow: /upload/docs/
Allow: /compound/
Allow: /substance/
Allow: /bioassay/
Allow: /rest/
Allow: /rest/pug/sourcetable/
Allow: /rest/pug/periodictable/
Allow: /sitemap/
Allow: /ghs/
Allow: /docs/

User-agent: Slurp
Disallow: /assay/asis.cgi
Disallow: /assay/bioactivity.html
Disallow: /assay/getGraphHeatmap.cgi
Disallow: /assay/assayHeatmap.cgi
Disallow: /assay/assaytool.cgi
Disallow: /assay/assaycluster.cgi
Disallow: /assay/redirect.cgi
Disallow: /assay/assaydownload.cgi
Disallow: /assaysrv/assaysrv.cgi
Disallow: /idexchange/idexchange.cgi
Disallow: /pc_fetch/pc_fetch.cgi
Disallow: /score_matrix/score_matrix.cgi
Disallow: /deposit/deposit.cgi
Disallow: /upload/upload.cgi
Disallow: /pug/pug.cgi
Disallow: /rest/rdf/
Disallow: /rest/upload/
Disallow: /rest/pug/
Disallow: /vw3d/
Disallow: /he/
Disallow: /widget
Disallow: /assay/assay.cgi
Disallow: /pug_soap/pug_soap.cgi
Disallow: /unified_search/structure_search.cgi
Disallow: /search_3d/search_3d_front_end.cgi
Disallow: /toc/summary_toc.cgi
Allow: /upload/docs/
Allow: /compound/
Allow: /substance/
Allow: /bioassay/
Allow: /rest/
Allow: /rest/pug/sourcetable/
Allow: /rest/pug/periodictable/
Allow: /sitemap/
Allow: /ghs/
Allow: /docs/

User-agent: MSNBot
Disallow: /assay/asis.cgi
Disallow: /assay/bioactivity.html
Disallow: /assay/getGraphHeatmap.cgi
Disallow: /assay/assayHeatmap.cgi
Disallow: /assay/assaytool.cgi
Disallow: /assay/assaycluster.cgi
Disallow: /assay/redirect.cgi
Disallow: /assay/assaydownload.cgi
Disallow: /assaysrv/assaysrv.cgi
Disallow: /idexchange/idexchange.cgi
Disallow: /pc_fetch/pc_fetch.cgi
Disallow: /score_matrix/score_matrix.cgi
Disallow: /deposit/deposit.cgi
Disallow: /upload/upload.cgi
Disallow: /pug/pug.cgi
Disallow: /rest/rdf/
Disallow: /rest/upload/
Disallow: /rest/pug/
Disallow: /vw3d/
Disallow: /he/
Disallow: /widget
Disallow: /assay/assay.cgi
Disallow: /pug_soap/pug_soap.cgi
Disallow: /unified_search/structure_search.cgi
Disallow: /search_3d/search_3d_front_end.cgi
Disallow: /toc/summary_toc.cgi
Allow: /upload/docs/
Allow: /compound/
Allow: /substance/
Allow: /rest/
Allow: /rest/pug/sourcetable/
Allow: /rest/pug/periodictable/
Allow: /sitemap/
Allow: /ghs/
Allow: /docs/

User-agent: Googlebot-Mobile
Disallow: /assay/asis.cgi
Disallow: /assay/bioactivity.html
Disallow: /assay/getGraphHeatmap.cgi
Disallow: /assay/assayHeatmap.cgi
Disallow: /assay/assaytool.cgi
Disallow: /assay/assaycluster.cgi
Disallow: /assay/redirect.cgi
Disallow: /assay/assaydownload.cgi
Disallow: /assaysrv/assaysrv.cgi
Disallow: /idexchange/idexchange.cgi
Disallow: /pc_fetch/pc_fetch.cgi
Disallow: /score_matrix/score_matrix.cgi
Disallow: /deposit/deposit.cgi
Disallow: /upload/upload.cgi
Disallow: /pug/pug.cgi
Disallow: /rest/rdf/
Disallow: /rest/upload/
Disallow: /rest/pug/
Disallow: /vw3d/
Disallow: /he/
Disallow: /widget
Disallow: /assay/assay.cgi
Disallow: /pug_soap/pug_soap.cgi
Disallow: /unified_search/structure_search.cgi
Disallow: /search_3d/search_3d_front_end.cgi
Disallow: /toc/summary_toc.cgi
Allow: /upload/docs/
Allow: /compound/
Allow: /substance/
Allow: /bioassay/
Allow: /rest/
Allow: /rest/pug/sourcetable/
Allow: /rest/pug/periodictable/
Allow: /sitemap/
Allow: /ghs/
Allow: /docs/

User-agent: Googlebot-Image
Allow: /image/imgsrv.fcgi
Allow: /image/img3d.cgi
Disallow: /he/

User-agent: Baiduspider
Disallow: /assay/asis.cgi
Disallow: /assay/bioactivity.html
Disallow: /assay/getGraphHeatmap.cgi
Disallow: /assay/assayHeatmap.cgi
Disallow: /assay/assaytool.cgi
Disallow: /assay/assaycluster.cgi
Disallow: /assay/redirect.cgi
Disallow: /assay/assaydownload.cgi
Disallow: /assaysrv/assaysrv.cgi
Disallow: /idexchange/idexchange.cgi
Disallow: /pc_fetch/pc_fetch.cgi
Disallow: /score_matrix/score_matrix.cgi
Disallow: /image/img3d.cgi
Disallow: /image/imagefly.cgi
Disallow: /image/structurefly.cgi
Disallow: /deposit/deposit.cgi
Disallow: /upload/upload.cgi
Disallow: /pug/pug.cgi
Disallow: /rest/rdf/
Disallow: /rest/upload/
Disallow: /rest/pug/
Disallow: /vw3d/
Disallow: /he/
Disallow: /widget
Disallow: /assay/assay.cgi
Disallow: /pug_soap/pug_soap.cgi
Disallow: /unified_search/structure_search.cgi
Disallow: /search_3d/search_3d_front_end.cgi
Disallow: /toc/summary_toc.cgi
Allow: /upload/docs/
Allow: /compound/
Allow: /substance/
Allow: /bioassay/
Allow: /rest/
Allow: /rest/pug/sourcetable/
Allow: /rest/pug/periodictable/
Allow: /sitemap/
Allow: /ghs/
Allow: /docs/

User-agent: Teleport Pro
Disallow: /assay/asis.cgi
Disallow: /assay/bioactivity.html
Disallow: /assay/getGraphHeatmap.cgi
Disallow: /assay/assayHeatmap.cgi
Disallow: /assay/assaytool.cgi
Disallow: /assay/assaycluster.cgi
Disallow: /assay/redirect.cgi
Disallow: /assay/assaydownload.cgi
Disallow: /assaysrv/assaysrv.cgi
Disallow: /idexchange/idexchange.cgi
Disallow: /pc_fetch/pc_fetch.cgi
Disallow: /score_matrix/score_matrix.cgi
Disallow: /image/img3d.cgi
Disallow: /image/imagefly.cgi
Disallow: /image/structurefly.cgi
Disallow: /deposit/deposit.cgi
Disallow: /upload/upload.cgi
Disallow: /pug/pug.cgi
Disallow: /rest/rdf/
Disallow: /rest/upload/
Disallow: /rest/pug/
Disallow: /vw3d/
Disallow: /he/
Disallow: /widget
Disallow: /assay/assay.cgi
Disallow: /pug_soap/pug_soap.cgi
Disallow: /unified_search/structure_search.cgi
Disallow: /search_3d/search_3d_front_end.cgi
Disallow: /toc/summary_toc.cgi
Allow: /upload/docs/
Allow: /compound/
Allow: /substance/
Allow: /bioassay/
Allow: /rest/
Allow: /rest/pug/sourcetable/
Allow: /rest/pug/periodictable/
Allow: /sitemap/
Allow: /ghs/
Allow: /docs/

User-agent: yacybot
Disallow: *

User-agent: Browsershots
Disallow:

User-agent: *
Disallow: /assay/asis.cgi
Disallow: /assay/bioactivity.html
Disallow: /assay/getGraphHeatmap.cgi
Disallow: /assay/assayHeatmap.cgi
Disallow: /assay/assaytool.cgi
Disallow: /assay/assaycluster.cgi
Disallow: /assay/redirect.cgi
Disallow: /assay/assaydownload.cgi
Disallow: /assaysrv/assaysrv.cgi
Disallow: /idexchange/idexchange.cgi
Disallow: /pc_fetch/pc_fetch.cgi
Disallow: /score_matrix/score_matrix.cgi
Disallow: /image/img3d.cgi
Disallow: /image/imagefly.cgi
Disallow: /image/structurefly.cgi
Disallow: /deposit/deposit.cgi
Disallow: /upload/upload.cgi
Disallow: /pug/pug.cgi
Disallow: /rest/rdf/
Disallow: /rest/upload/
Disallow: /rest/pug/
Disallow: /vw3d/
Disallow: /he/
Disallow: /widget
Disallow: /pug_soap/pug_soap.cgi
Disallow: /unified_search/structure_search.cgi
Disallow: /search_3d/search_3d_front_end.cgi

Sitemap: https://pubchem.ncbi.nlm.nih.gov/sitemap/pcsitemap.xml
Sitemap: https://pubchem.ncbi.nlm.nih.gov/sitemap/sitemap.cgi?p=annotation/sitemap_index.xml
Sitemap: https://pubchem.ncbi.nlm.nih.gov/sitemap/sitemap.cgi?p=assay/sitemap_index.xml
Sitemap: https://pubchem.ncbi.nlm.nih.gov/sitemap/sitemap.cgi?p=source/sitemap.xml
Sitemap: https://pubchem.ncbi.nlm.nih.gov/sitemap/sitemap.cgi?p=gene/sitemap_index.xml
Sitemap: https://pubchem.ncbi.nlm.nih.gov/sitemap/sitemap.cgi?p=protein/sitemap_index.xml
Sitemap: https://pubchem.ncbi.nlm.nih.gov/sitemap/sitemap.cgi?p=patent/sitemap_index.xml
Sitemap: https://pubchem.ncbi.nlm.nih.gov/sitemap/sitemap.cgi?p=concept/sitemap_1.xml
Sitemap: https://pubchem.ncbi.nlm.nih.gov/sitemap/sitemap.cgi?p=docs
Sitemap: https://pubchem.ncbi.nlm.nih.gov/sitemap/sitemap.cgi?p=cell/sitemap_index.xml
Sitemap: https://pubchem.ncbi.nlm.nih.gov/sitemap/sitemap.cgi?p=taxonomy/sitemap_index.xml

# ==========================================================
# Version: 2004-10-01, PubChem Team
# Revision: 2005-2023


# robots.txt
# ========================================================================
#
# Version: 2004-10-01, PubChem Team
#
# ========================================================================

User-agent: Googlebot
Disallow: /assay/asis.cgi
Disallow: /assay/bioactivity.html
Disallow: /assay/getGraphHeatmap.cgi
Disallow: /assay/assayHeatmap.cgi
Disallow: /assay/assaytool.cgi
Disallow: /assay/assaycluster.cgi
Disallow: /assay/redirect.cgi
Disallow: /assay/assaydownload.cgi
Disallow: /assaysrv/assaysrv.cgi
Disallow: /idexchange/idexchange.cgi
Disallow: /pc_fetch/pc_fetch.cgi
Disallow: /score_matrix/score_matrix.cgi
Disallow: /deposit/deposit.cgi
Disallow: /upload/upload.cgi
Disallow: /pug/pug.cgi
Disallow: /rest/rdf/
Disallow: /rest/upload/
Disallow: /rest/pug/
Disallow: /vw3d/
Disallow: /he/
Disallow: /sdq/
Disallow: /widget
Disallow: /toc/summary_toc.cgi
Disallow: /assay/assay.cgi
Disallow: /pug_soap/pug_soap.cgi
Disallow: /unified_search/structure_search.cgi
Disallow: /search_3d/search_3d_front_end.cgi
Allow: /upload/docs/
Allow: /compound/
Allow: /substance/
Allow: /bioassay/
Allow: /rest/
Allow: /rest/pug/sourcetable/
Allow: /rest/pug/periodictable/
Allow: /sitemap/
Allow: /ghs/
Allow: /docs/

User-Agent: bingbot
Disallow: /assay/asis.cgi
Disallow: /assay/bioactivity.html
Disallow: /assay/getGraphHeatmap.cgi
Disallow: /assay/assayHeatmap.cgi
Disallow: /assay/assaytool.cgi
Disallow: /assay/assaycluster.cgi
Disallow: /assay/redirect.cgi
Disallow: /assay/assaydownload.cgi
Disallow: /assaysrv/assaysrv.cgi
Disallow: /idexchange/idexchange.cgi
Disallow: /pc_fetch/pc_fetch.cgi
Disallow: /score_matrix/score_matrix.cgi
Disallow: /deposit/deposit.cgi
Disallow: /upload/upload.cgi
Disallow: /pug/pug.cgi
Disallow: /rest/rdf/
Disallow: /rest/upload/
Disallow: /rest/pug/
Disallow: /vw3d/
Disallow: /he/
Disallow: /sdq/
Disallow: /widget
Disallow: /assay/assay.cgi
Disallow: /pug_soap/pug_soap.cgi
Disallow: /unified_search/structure_search.cgi
Disallow: /search_3d/search_3d_front_end.cgi
Disallow: /toc/summary_toc.cgi
Allow: /upload/docs/
Allow: /compound/
Allow: /substance/
Allow: /bioassay/
Allow: /rest/
Allow: /rest/pug/sourcetable/
Allow: /rest/pug/periodictable/
Allow: /sitemap/
Allow: /ghs/
Allow: /docs/

User-agent: htdig/3.1.6
Disallow: /assay/asis.cgi
Disallow: /assay/bioactivity.html
Disallow: /assay/getGraphHeatmap.cgi
Disallow: /assay/assayHeatmap.cgi
Disallow: /assay/assaytool.cgi
Disallow: /assay/assaycluster.cgi
Disallow: /assay/redirect.cgi
Disallow: /assay/assaydownload.cgi
Disallow: /assaysrv/assaysrv.cgi
Disallow: /idexchange/idexchange.cgi
Disallow: /pc_fetch/pc_fetch.cgi
Disallow: /score_matrix/score_matrix.cgi
Disallow: /deposit/deposit.cgi
Disallow: /upload/upload.cgi
Disallow: /pug/pug.cgi
Disallow: /rest/rdf/
Disallow: /rest/upload/
Disallow: /rest/pug/
Disallow: /vw3d/
Disallow: /he/
Disallow: /sdq/
Disallow: /widget
Disallow: /assay/assay.cgi
Disallow: /pug_soap/pug_soap.cgi
Disallow: /unified_search/structure_search.cgi
Disallow: /search_3d/search_3d_front_end.cgi
Disallow: /toc/summary_toc.cgi
Allow: /upload/docs/
Allow: /compound/
Allow: /substance/
Allow: /bioassay/
Allow: /rest/
Allow: /rest/pug/sourcetable/
Allow: /rest/pug/periodictable/
Allow: /sitemap/
Allow: /ghs/
Allow: /docs/

User-agent: Slurp
Disallow: /assay/asis.cgi
Disallow: /assay/bioactivity.html
Disallow: /assay/getGraphHeatmap.cgi
Disallow: /assay/assayHeatmap.cgi
Disallow: /assay/assaytool.cgi
Disallow: /assay/assaycluster.cgi
Disallow: /assay/redirect.cgi
Disallow: /assay/assaydownload.cgi
Disallow: /assaysrv/assaysrv.cgi
Disallow: /idexchange/idexchange.cgi
Disallow: /pc_fetch/pc_fetch.cgi
Disallow: /score_matrix/score_matrix.cgi
Disallow: /deposit/deposit.cgi
Disallow: /upload/upload.cgi
Disallow: /pug/pug.cgi
Disallow: /rest/rdf/
Disallow: /rest/upload/
Disallow: /rest/pug/
Disallow: /vw3d/
Disallow: /he/
Disallow: /sdq/
Disallow: /widget
Disallow: /assay/assay.cgi
Disallow: /pug_soap/pug_soap.cgi
Disallow: /unified_search/structure_search.cgi
Disallow: /search_3d/search_3d_front_end.cgi
Disallow: /toc/summary_toc.cgi
Allow: /upload/docs/
Allow: /compound/
Allow: /substance/
Allow: /bioassay/
Allow: /rest/
Allow: /rest/pug/sourcetable/
Allow: /rest/pug/periodictable/
Allow: /sitemap/
Allow: /ghs/
Allow: /docs/

User-agent: MSNBot
Disallow: /assay/asis.cgi
Disallow: /assay/bioactivity.html
Disallow: /assay/getGraphHeatmap.cgi
Disallow: /assay/assayHeatmap.cgi
Disallow: /assay/assaytool.cgi
Disallow: /assay/assaycluster.cgi
Disallow: /assay/redirect.cgi
Disallow: /assay/assaydownload.cgi
Disallow: /assaysrv/assaysrv.cgi
Disallow: /idexchange/idexchange.cgi
Disallow: /pc_fetch/pc_fetch.cgi
Disallow: /score_matrix/score_matrix.cgi
Disallow: /deposit/deposit.cgi
Disallow: /upload/upload.cgi
Disallow: /pug/pug.cgi
Disallow: /rest/rdf/
Disallow: /rest/upload/
Disallow: /rest/pug/
Disallow: /vw3d/
Disallow: /he/
Disallow: /sdq/
Disallow: /widget
Disallow: /assay/assay.cgi
Disallow: /pug_soap/pug_soap.cgi
Disallow: /unified_search/structure_search.cgi
Disallow: /search_3d/search_3d_front_end.cgi
Disallow: /toc/summary_toc.cgi
Allow: /upload/docs/
Allow: /compound/
Allow: /substance/
Allow: /rest/
Allow: /rest/pug/sourcetable/
Allow: /rest/pug/periodictable/
Allow: /sitemap/
Allow: /ghs/
Allow: /docs/

User-agent: Googlebot-Mobile
Disallow: /assay/asis.cgi
Disallow: /assay/bioactivity.html
Disallow: /assay/getGraphHeatmap.cgi
Disallow: /assay/assayHeatmap.cgi
Disallow: /assay/assaytool.cgi
Disallow: /assay/assaycluster.cgi
Disallow: /assay/redirect.cgi
Disallow: /assay/assaydownload.cgi
Disallow: /assaysrv/assaysrv.cgi
Disallow: /idexchange/idexchange.cgi
Disallow: /pc_fetch/pc_fetch.cgi
Disallow: /score_matrix/score_matrix.cgi
Disallow: /deposit/deposit.cgi
Disallow: /upload/upload.cgi
Disallow: /pug/pug.cgi
Disallow: /rest/rdf/
Disallow: /rest/upload/
Disallow: /rest/pug/
Disallow: /vw3d/
Disallow: /he/
Disallow: /sdq/
Disallow: /widget
Disallow: /assay/assay.cgi
Disallow: /pug_soap/pug_soap.cgi
Disallow: /unified_search/structure_search.cgi
Disallow: /search_3d/search_3d_front_end.cgi
Disallow: /toc/summary_toc.cgi
Allow: /upload/docs/
Allow: /compound/
Allow: /substance/
Allow: /bioassay/
Allow: /rest/
Allow: /rest/pug/sourcetable/
Allow: /rest/pug/periodictable/
Allow: /sitemap/
Allow: /ghs/
Allow: /docs/

User-agent: Googlebot-Image
Allow: /image/imgsrv.fcgi
Allow: /image/img3d.cgi
Disallow: /he/
Disallow: /sdq/

User-agent: Baiduspider
Disallow: /assay/asis.cgi
Disallow: /assay/bioactivity.html
Disallow: /assay/getGraphHeatmap.cgi
Disallow: /assay/assayHeatmap.cgi
Disallow: /assay/assaytool.cgi
Disallow: /assay/assaycluster.cgi
Disallow: /assay/redirect.cgi
Disallow: /assay/assaydownload.cgi
Disallow: /assaysrv/assaysrv.cgi
Disallow: /idexchange/idexchange.cgi
Disallow: /pc_fetch/pc_fetch.cgi
Disallow: /score_matrix/score_matrix.cgi
Disallow: /image/img3d.cgi
Disallow: /image/imagefly.cgi
Disallow: /image/structurefly.cgi
Disallow: /deposit/deposit.cgi
Disallow: /upload/upload.cgi
Disallow: /pug/pug.cgi
Disallow: /rest/rdf/
Disallow: /rest/upload/
Disallow: /rest/pug/
Disallow: /vw3d/
Disallow: /he/
Disallow: /sdq/
Disallow: /widget
Disallow: /assay/assay.cgi
Disallow: /pug_soap/pug_soap.cgi
Disallow: /unified_search/structure_search.cgi
Disallow: /search_3d/search_3d_front_end.cgi
Disallow: /toc/summary_toc.cgi
Allow: /upload/docs/
Allow: /compound/
Allow: /substance/
Allow: /bioassay/
Allow: /rest/
Allow: /rest/pug/sourcetable/
Allow: /rest/pug/periodictable/
Allow: /sitemap/
Allow: /ghs/
Allow: /docs/

User-agent: Teleport Pro
Disallow: /assay/asis.cgi
Disallow: /assay/bioactivity.html
Disallow: /assay/getGraphHeatmap.cgi
Disallow: /assay/assayHeatmap.cgi
Disallow: /assay/assaytool.cgi
Disallow: /assay/assaycluster.cgi
Disallow: /assay/redirect.cgi
Disallow: /assay/assaydownload.cgi
Disallow: /assaysrv/assaysrv.cgi
Disallow: /idexchange/idexchange.cgi
Disallow: /pc_fetch/pc_fetch.cgi
Disallow: /score_matrix/score_matrix.cgi
Disallow: /image/img3d.cgi
Disallow: /image/imagefly.cgi
Disallow: /image/structurefly.cgi
Disallow: /deposit/deposit.cgi
Disallow: /upload/upload.cgi
Disallow: /pug/pug.cgi
Disallow: /rest/rdf/
Disallow: /rest/upload/
Disallow: /rest/pug/
Disallow: /vw3d/
Disallow: /he/
Disallow: /sdq/
Disallow: /widget
Disallow: /assay/assay.cgi
Disallow: /pug_soap/pug_soap.cgi
Disallow: /unified_search/structure_search.cgi
Disallow: /search_3d/search_3d_front_end.cgi
Disallow: /toc/summary_toc.cgi
Allow: /upload/docs/
Allow: /compound/
Allow: /substance/
Allow: /bioassay/
Allow: /rest/
Allow: /rest/pug/sourcetable/
Allow: /rest/pug/periodictable/
Allow: /sitemap/
Allow: /ghs/
Allow: /docs/

User-agent: yacybot
Disallow: *

User-agent: Browsershots
Disallow:

User-agent: *
Disallow: /assay/asis.cgi
Disallow: /assay/bioactivity.html
Disallow: /assay/getGraphHeatmap.cgi
Disallow: /assay/assayHeatmap.cgi
Disallow: /assay/assaytool.cgi
Disallow: /assay/assaycluster.cgi
Disallow: /assay/redirect.cgi
Disallow: /assay/assaydownload.cgi
Disallow: /assaysrv/assaysrv.cgi
Disallow: /idexchange/idexchange.cgi
Disallow: /pc_fetch/pc_fetch.cgi
Disallow: /score_matrix/score_matrix.cgi
Disallow: /image/img3d.cgi
Disallow: /image/imagefly.cgi
Disallow: /image/structurefly.cgi
Disallow: /deposit/deposit.cgi
Disallow: /upload/upload.cgi
Disallow: /pug/pug.cgi
Disallow: /rest/rdf/
Disallow: /rest/upload/
Disallow: /rest/pug/
Disallow: /vw3d/
Disallow: /he/
Disallow: /sdq/
Disallow: /widget
Disallow: /pug_soap/pug_soap.cgi
Disallow: /unified_search/structure_search.cgi
Disallow: /search_3d/search_3d_front_end.cgi

Sitemap: https://pubchem.ncbi.nlm.nih.gov/sitemap/pcsitemap.xml
Sitemap: https://pubchem.ncbi.nlm.nih.gov/sitemap/sitemap.cgi?p=concept/sitemap_index.xml
Sitemap: https://pubchem.ncbi.nlm.nih.gov/sitemap/sitemap.cgi?p=annotation/sitemap_index.xml
Sitemap: https://pubchem.ncbi.nlm.nih.gov/sitemap/sitemap.cgi?p=assay/sitemap_index.xml
Sitemap: https://pubchem.ncbi.nlm.nih.gov/sitemap/sitemap.cgi?p=source/sitemap.xml
Sitemap: https://pubchem.ncbi.nlm.nih.gov/sitemap/sitemap.cgi?p=gene/sitemap_index.xml
Sitemap: https://pubchem.ncbi.nlm.nih.gov/sitemap/sitemap.cgi?p=protein/sitemap_index.xml
Sitemap: https://pubchem.ncbi.nlm.nih.gov/sitemap/sitemap.cgi?p=patent/sitemap_index.xml
Sitemap: https://pubchem.ncbi.nlm.nih.gov/sitemap/sitemap.cgi?p=docs
Sitemap: https://pubchem.ncbi.nlm.nih.gov/sitemap/sitemap.cgi?p=cell/sitemap_index.xml
Sitemap: https://pubchem.ncbi.nlm.nih.gov/sitemap/sitemap.cgi?p=taxonomy/sitemap_index.xml

# ==========================================================
# Version: 2004-10-01, PubChem Team
# Revision: 2005-2023


User-agent: *
Disallow:


User-agent: AhrefsBot
Disallow: /

User-agent: *
Disallow:


User-agent: AhrefsBot
Disallow: /

User-agent: Googlebot
Disallow: /search

User-agent: *
Disallow:


User-agent: *
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php


User-agent: *
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php

Disallow: https://www.pwlcapital.com/wp-content/uploads/wpo-plugins-tables-list.json


User-agent: *
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php

Disallow: /wp-content/uploads/wpo-plugins-tables-list.json


User-agent: *
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php

User-agent: *
Disallow: /wp-content/uploads/wpo-plugins-tables-list.json


User-agent: *
Disallow: /wp-content/uploads/wpo-plugins-tables-list.json

# START YOAST BLOCK
# ---------------------------
User-agent: *
Disallow:

Sitemap: https://www.pwlcapital.com/sitemap_index.xml
# ---------------------------
# END YOAST BLOCK


User-agent: *
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php

Sitemap: https://pwlcapital.com/sitemap_index.xml


User-agent: *
Disallow: /
User-agent: GPTBot
Disallow: /
User-agent: Meta-ExernalAgent
Disallow: /
User-agent: mirall
Disallow: /


User-agent: *
Disallow: /contact
Disallow: /validate
Disallow: /_next/

User-agent: Google-Extended
Disallow: /

User-agent: GPTBot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: FacebookBot
Disallow: /

User-agent: Omglilibot
Disallow: /

Sitemap: https://reference-global.com/sitemap.xml


User-agent: *
Disallow: /contact
Disallow: /validate
Disallow: /_next/
Disallow: /search
Disallow: /pl
Disallow: /fr
Disallow: /es
Disallow: /it
Disallow: /de

User-agent: Google-Extended
Disallow: /

User-agent: GPTBot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: FacebookBot
Disallow: /

User-agent: Omglilibot
Disallow: /

Sitemap: https://reference-global.com/sitemap.xml


# The FULL URL to the DSpace sitemaps
# The http://cclibaerade-2.dmz.cranfield.ac.uk:8080/xmlui will be auto-filled with the value in dspace.cfg
# XML sitemap is listed first as it is preferred by most search engines
Sitemap: http://cclibaerade-2.dmz.cranfield.ac.uk:8080/xmlui/sitemap
Sitemap: http://cclibaerade-2.dmz.cranfield.ac.uk:8080/xmlui/htmlmap

##########################
# Default Access Group
# (NOTE: blank lines are not allowable in a group record)
##########################
User-agent: *
# Disable access to Discovery search and filters
Disallow: /discover
Disallow: /search-filter
#
# Optionally uncomment the following line ONLY if sitemaps are working
# and you have verified that your site is being indexed correctly.
# Disallow: /browse
#
# If you have configured DSpace (Solr-based) Statistics to be publicly
# accessible, then you may not want this content to be indexed
# Disallow: /statistics
#
# You also may wish to disallow access to the following paths, in order
# to stop web spiders from accessing user-based content
# Disallow: /contact
# Disallow: /feedback
# Disallow: /forgot
# Disallow: /login
# Disallow: /register

##############################
# Section for misbehaving bots
# The following directives to block specific robots were borrowed from Wikipedia's robots.txt
##############################

# advertising-related bots:
User-agent: Mediapartners-Google*
Disallow: /

# Crawlers that are kind enough to obey, but which we'd rather not have
# unless they're feeding search engines.
User-agent: UbiCrawler
Disallow: /

User-agent: DOC
Disallow: /

User-agent: Zao
Disallow: /

# Some bots are known to be trouble, particularly those designed to copy
# entire sites. Please obey robots.txt.
User-agent: sitecheck.internetseer.com
Disallow: /

User-agent: Zealbot
Disallow: /

User-agent: MSIECrawler
Disallow: /

User-agent: SiteSnagger
Disallow: /

User-agent: WebStripper
Disallow: /

User-agent: WebCopier
Disallow: /

User-agent: Fetch
Disallow: /

User-agent: Offline Explorer
Disallow: /

User-agent: Teleport
Disallow: /

User-agent: TeleportPro
Disallow: /

User-agent: WebZIP
Disallow: /

User-agent: linko
Disallow: /

User-agent: HTTrack
Disallow: /

User-agent: Microsoft.URL.Control
Disallow: /

User-agent: Xenu
Disallow: /

User-agent: larbin
Disallow: /

User-agent: libwww
Disallow: /

User-agent: ZyBORG
Disallow: /

User-agent: Download Ninja
Disallow: /

# Misbehaving: requests much too fast:
User-agent: fast
Disallow: /

#
# If your DSpace is going down because of someone using recursive wget,
# you can activate the following rule.
#
# If your own faculty is bringing down your dspace with recursive wget,
# you can advise them to use the --wait option to set the delay between hits.
#
#User-agent: wget
#Disallow: /

#
# The 'grub' distributed client has been *very* poorly behaved.
#
User-agent: grub-client
Disallow: /

#
# Doesn't follow robots.txt anyway, but...
#
User-agent: k2spider
Disallow: /

#
# Hits many times per second, not acceptable
# http://www.nameprotect.com/botinfo.html
User-agent: NPBot
Disallow: /

# A capture bot, downloads gazillions of pages with no public benefit
# http://www.webreaper.net/
User-agent: WebReaper
Disallow: /


User-agent: *
Disallow: /action
Disallow: /help
Disallow: /search
Disallow: /feedback
Disallow: /rss
Disallow: /page/account-confirmation-thanks
Disallow: /media
Disallow: /medical-research
Disallow: /servlet/linkout
Disallow: /na101/
Disallow: /na101v1/
Disallow: /na102/
Disallow: /doi/mlt/
Disallow: /topic
Disallow: /author/
Allow: /action/showJournal
Allow: /action/showPublications
Allow: /action/showXml
Allow: /action/showTopic
Allow: /action/showBook
Allow: /action/showCoverImage
Crawl-delay: 1
Disallow: /results


User-agent: *
Disallow: /action
Disallow: /help
Disallow: /search
Disallow: /feedback
Disallow: /rss
Disallow: /page/account-confirmation-thanks
Disallow: /media
Disallow: /medical-research
Disallow: /servlet/linkout
Disallow: /na101/
Disallow: /na101v1/
Disallow: /na102/
Disallow: /doi/mlt/
Disallow: /topic
Disallow: /author/
Allow: /action/showJournal
Allow: /action/showPublications
Allow: /action/showXml
Allow: /action/showTopic
Allow: /action/showBook
Allow: /action/showCoverImage
Crawl-delay: 1
Disallow: /results
Disallow: /action/ajaxShowRecommended


User-agent: *
Disallow: /action
Disallow: /help
Disallow: /search
Disallow: /feedback
Disallow: /rss
Disallow: /page/account-confirmation-thanks
Disallow: /media
Disallow: /medical-research
Disallow: /servlet/linkout
Disallow: /na101/
Disallow: /na101v1/
Disallow: /na102/
Disallow: /doi/mlt/
Disallow: /topic
Disallow: /author/
Disallow: /results
Disallow: /action/ajaxShowRecommended
Allow: /action/showJournal
Allow: /action/showPublications
Allow: /action/showXml
Allow: /action/showTopic
Allow: /action/showBook
Allow: /action/showCoverImage

Crawl-delay: 1


User-agent: *
Disallow: /action
Disallow: /help
Disallow: /search
Disallow: /feedback
Disallow: /rss
Disallow: /page/account-confirmation-thanks
Disallow: /media
Disallow: /medical-research
Disallow: /servlet/linkout
Disallow: /na101/
Disallow: /na101v1/
Disallow: /na102/
Disallow: /doi/mlt/
Disallow: /topic
Disallow: /author/
Disallow: /results
Disallow: /action/ajaxShowRecommended
Allow: /action/showJournal
Allow: /action/showPublications
Allow: /action/showXml
Allow: /action/showTopic
Allow: /action/showBook
Allow: /action/showCoverImage

User-agent: facebookexternalhit
User-agent: LinkedInBot
User-agent: Twitterbot
Allow: /

Crawl-delay: 1


User-agent: *
Disallow: /action
Disallow: /help
Disallow: /search
Disallow: /feedback
Disallow: /rss
Disallow: /page/account-confirmation-thanks
Disallow: /media
Disallow: /medical-research
Disallow: /servlet/linkout
Disallow: /na101/
Disallow: /na101v1/
Disallow: /na102/
Disallow: /doi/mlt/
Disallow: /topic
Disallow: /author/
Disallow: /doi/metrics/
Disallow: /results
Disallow: /action/ajaxShowRecommended
Allow: /action/showJournal
Allow: /action/showPublications
Allow: /action/showXml
Allow: /action/showTopic
Allow: /action/showBook
Allow: /action/showCoverImage

User-agent: facebookexternalhit
User-agent: LinkedInBot
User-agent: Twitterbot
Allow: /

Crawl-delay: 1


User-agent: *
Disallow: /action
Disallow: /help
Disallow: /search
Disallow: /feedback
Disallow: /rss
Disallow: /page/account-confirmation-thanks
Disallow: /media
Disallow: /medical-research
Disallow: /servlet/linkout
Disallow: /na101/
Disallow: /na101v1/
Disallow: /na102/
Disallow: /doi/mlt/
Disallow: /topic
Disallow: /author/
Disallow: /doi/metrics/
Disallow: /results
Disallow: /action/ajaxShowRecommended
Disallow: /history/
Allow: /action/showJournal
Allow: /action/showPublications
Allow: /action/showXml
Allow: /action/showTopic
Allow: /action/showBook
Allow: /action/showCoverImage

User-agent: facebookexternalhit
User-agent: LinkedInBot
User-agent: Twitterbot
Allow: /

Crawl-delay: 1


User-agent: *
Disallow: /action
Disallow: /help
Disallow: /search
Disallow: /feedback
Disallow: /rss
Disallow: /page/account-confirmation-thanks
Disallow: /media
Disallow: /medical-research
Disallow: /servlet/linkout
Disallow: /na101/
Disallow: /na101v1/
Disallow: /na102/
Disallow: /doi/mlt/
Disallow: /topic
Disallow: /author/
Disallow: /doi/metrics/
Disallow: /results
Disallow: /action/ajaxShowRecommended
Disallow: /authored-by/
Disallow: /history/
Allow: /action/showJournal
Allow: /action/showPublications
Allow: /action/showXml
Allow: /action/showTopic
Allow: /action/showBook
Allow: /action/showCoverImage

User-agent: facebookexternalhit
User-agent: LinkedInBot
User-agent: Twitterbot
Allow: /

User-agent: GPTBot
Disallow: /

Crawl-delay: 1


User-agent: *
Disallow: /action
Disallow: /help
Disallow: /search
Disallow: /feedback
Disallow: /rss
Disallow: /page/account-confirmation-thanks
Disallow: /media
Disallow: /medical-research
Disallow: /servlet/linkout
Disallow: /na101/
Disallow: /na101v1/
Disallow: /na102/
Disallow: /doi/mlt/
Disallow: /topic
Disallow: /author/
Disallow: /doi/metrics/
Disallow: /results
Disallow: /action/ajaxShowRecommended
Disallow: /authored-by/
Disallow: /history/
Allow: /action/showJournal
Allow: /action/showPublications
Allow: /action/showXml
Allow: /action/showTopic
Allow: /action/showBook
Allow: /action/showCoverImage

User-agent: facebookexternalhit
User-agent: LinkedInBot
User-agent: Twitterbot
Allow: /

User-agent: GPTBot
Disallow: /

User-agent: Google-Extended
Disallow: /

Crawl-delay: 1


User-agent: *
Disallow: /action
Disallow: /help
Disallow: /search
Disallow: /feedback
Disallow: /rss
Disallow: /page/account-confirmation-thanks
Disallow: /media
Disallow: /medical-research
Disallow: /servlet/linkout
Disallow: /na101/
Disallow: /na101v1/
Disallow: /na102/
Disallow: /doi/mlt/
Disallow: /topic
Disallow: /author/
Disallow: /doi/metrics/
Disallow: /results
Disallow: /action/ajaxShowRecommended
Disallow: /authored-by/
Disallow: /history/
Allow: /action/showJournal
Allow: /action/showPublications
Allow: /action/showXml
Allow: /action/showTopic
Allow: /action/showBook
Allow: /action/showCoverImage
Allow: /ads.txt

User-agent: facebookexternalhit
User-agent: LinkedInBot
User-agent: Twitterbot
Allow: /

User-agent: GPTBot
Disallow: /

User-agent: Google-Extended
Disallow: /

Crawl-delay: 1


Sitemap: https://rss.onlinelibrary.wiley.com/sitemap-index-1.txt
User-agent: *
Disallow: /action
Disallow: /help
Disallow: /search
Disallow: /feedback
Disallow: /rss
Disallow: /page/account-confirmation-thanks
Disallow: /media
Disallow: /medical-research
Disallow: /servlet/linkout
Disallow: /na101/
Disallow: /na101v1/
Disallow: /na102/
Disallow: /doi/mlt/
Disallow: /topic
Disallow: /author/
Disallow: /doi/metrics/
Disallow: /results
Disallow: /action/ajaxShowRecommended
Disallow: /authored-by/
Disallow: /history/
Allow: /action/showJournal
Allow: /action/showPublications
Allow: /action/showXml
Allow: /action/showTopic
Allow: /action/showBook
Allow: /action/showCoverImage
Allow: /ads.txt

User-agent: facebookexternalhit
User-agent: LinkedInBot
User-agent: Twitterbot
Allow: /

User-agent: GPTBot
Disallow: /

User-agent: Google-Extended
Disallow: /

Crawl-delay: 1