Sitemap: https://www.jidonline.org/custom_pages.gz

User-agent: *
Disallow: /action
Disallow: /help
Disallow: /search
Disallow: /feedback
Disallow: /rss
Disallow: /action/clickThrough
Disallow: /action/showLogin
Disallow: /page/account-confirmation-thanks
Disallow: /media
Disallow: /medical-research
Disallow: /servlet/linkout
Disallow: /na101/
Disallow: /na101v1/
Disallow: /na102/
Disallow: /doi/mlt/
Allow: /action/showJournal
Allow: /action/showXml
Allow: /series
Allow: /isbn
Allow: /doi/book

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
User-agent: Twitterbot
Allow: /

User-agent: GPTBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Google-Extended
Disallow: /

Crawl-delay: 1


Sitemap: https://www.jidonline.org/sitemap-index-1.txt
Sitemap: https://www.jidonline.org/custom_pages.gz

User-agent: *
Disallow: /action
Disallow: /help
Disallow: /search
Disallow: /feedback
Disallow: /rss
Disallow: /action/clickThrough
Disallow: /action/showLogin
Disallow: /page/account-confirmation-thanks
Disallow: /media
Disallow: /medical-research
Disallow: /servlet/linkout
Disallow: /na101/
Disallow: /na101v1/
Disallow: /na102/
Disallow: /doi/mlt/
Allow: /action/showJournal
Allow: /action/showXml
Allow: /series
Allow: /isbn
Allow: /doi/book

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: GPTBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Google-Extended
Disallow: /

Crawl-delay: 1


Sitemap: https://www.jidonline.org/sitemap-index-1.txt
Sitemap: https://www.jidonline.org/custom_pages.gz

User-agent: *
Disallow: /action
Disallow: /help
Disallow: /search
Disallow: /feedback
Disallow: /rss
Disallow: /action/clickThrough
Disallow: /action/showLogin
Disallow: /page/account-confirmation-thanks
Disallow: /media
Disallow: /medical-research
Disallow: /servlet/linkout
Disallow: /na101/
Disallow: /na101v1/
Disallow: /na102/
Disallow: /doi/mlt/
Allow: /action/showJournal
Allow: /action/showXml
Allow: /series
Allow: /isbn
Allow: /doi/book
Allow: /.well-known/tdmrep.json

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: GPTBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Google-Extended
Disallow: /

Crawl-delay: 1


Sitemap: https://www.jidonline.org/sitemap-index-1.txt
Sitemap: https://www.jidonline.org/custom_pages.gz

User-agent: *
Disallow: /action
Disallow: /help
Disallow: /search
Disallow: /feedback
Disallow: /rss
Disallow: /action/clickThrough
Disallow: /action/showLogin
Disallow: /page/account-confirmation-thanks
Disallow: /media
Disallow: /medical-research
Disallow: /servlet/linkout
Disallow: /na101/
Disallow: /na101v1/
Disallow: /na102/
Disallow: /doi/mlt/
Disallow: /cms/attachment/92c526c4-ef1f-4065-b12b-ca4ae0f3e84f/gr1*
Allow: /action/showJournal
Allow: /action/showXml
Allow: /series
Allow: /isbn
Allow: /doi/book
Allow: /.well-known/tdmrep.json

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: GPTBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: PerplexityBot
Disallow: /

Crawl-delay: 1


Sitemap: https://www.jidonline.org/sitemap-index-1.txt
Sitemap: https://www.jidonline.org/custom_pages.gz

User-agent: *
Disallow: /action
Disallow: /help
Disallow: /search
Disallow: /feedback
Disallow: /rss
Disallow: /action/clickThrough
Disallow: /action/showLogin
Disallow: /page/account-confirmation-thanks
Disallow: /media
Disallow: /medical-research
Disallow: /servlet/linkout
Disallow: /na101/
Disallow: /na101v1/
Disallow: /na102/
Disallow: /doi/mlt/
Disallow: /cms/attachment/92c526c4-ef1f-4065-b12b-ca4ae0f3e84f/gr1*
Disallow: /personalize/addFavoritePublication
Disallow: /pb/widgets/*
Allow: /action/showJournal
Allow: /action/showXml
Allow: /series
Allow: /isbn
Allow: /doi/book
Allow: /.well-known/tdmrep.json

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: GPTBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: PerplexityBot
Disallow: /

Crawl-delay: 1


Sitemap: https://www.jidonline.org/sitemap-index-1.txt
Sitemap: https://www.jidonline.org/custom_pages.gz

User-agent: *
Disallow: /action
Disallow: /help
Disallow: /search
Disallow: /feedback
Disallow: /rss
Disallow: /action/clickThrough
Disallow: /action/showLogin
Disallow: /page/account-confirmation-thanks
Disallow: /media
Disallow: /medical-research
Disallow: /servlet/linkout
Disallow: /na101/
Disallow: /na101v1/
Disallow: /na102/
Disallow: /doi/mlt/
Disallow: /cms/attachment/92c526c4-ef1f-4065-b12b-ca4ae0f3e84f/gr1*
Disallow: /personalize/addFavoritePublication
Disallow: /pb/widgets/*
Allow: /action/showJournal
Allow: /action/showXml
Allow: /series
Allow: /isbn
Allow: /doi/book
Allow: /.well-known/tdmrep.json

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: GPTBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: PerplexityBot
Disallow: /

User-agent: AI2Bot
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: cohere-training-data-crawler
Disallow: /

User-agent: cohere-ai
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: iAskBot/1.0
Disallow: /

User-agent: meta-externalagent
Disallow: /

User-agent: MistralAI-User
Disallow: /

User-agent: omgili
Disallow: /

User-agent: TimpiBot
Disallow: /

User-agent: YouBot
Disallow: /

Crawl-delay: 1


Sitemap: https://www.jidonline.org/sitemap-index-1.txt
Sitemap: https://www.jidonline.org/custom_pages.gz

User-agent: *
Disallow: /action
Disallow: /help
Disallow: /search
Disallow: /feedback
Disallow: /rss
Disallow: /action/clickThrough
Disallow: /action/showLogin
Disallow: /page/account-confirmation-thanks
Disallow: /media
Disallow: /medical-research
Disallow: /servlet/linkout
Disallow: /na101/
Disallow: /na101v1/
Disallow: /na102/
Disallow: /doi/mlt/
Disallow: /cms/attachment/92c526c4-ef1f-4065-b12b-ca4ae0f3e84f/gr1*
Disallow: /personalize/addFavoritePublication
Disallow: /pb/widgets/*
Allow: /action/showJournal
Allow: /action/showXml
Allow: /series
Allow: /isbn
Allow: /doi/book
Allow: /.well-known/*

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: GPTBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: PerplexityBot
Disallow: /

User-agent: AI2Bot
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: cohere-training-data-crawler
Disallow: /

User-agent: cohere-ai
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: iAskBot/1.0
Disallow: /

User-agent: meta-externalagent
Disallow: /

User-agent: MistralAI-User
Disallow: /

User-agent: omgili
Disallow: /

User-agent: TimpiBot
Disallow: /

User-agent: YouBot
Disallow: /

Crawl-delay: 1


Sitemap: https://www.journalofdairyscience.org/custom_pages.gz
User-agent: Twitterbot
Disallow:
User-agent: *
Disallow: /action
Disallow: /help
Disallow: /search
Disallow: /feedback
Disallow: /rss
Disallow: /action/clickThrough
Disallow: /action/showLogin
Disallow: /page/account-confirmation-thanks
Disallow: /media
Disallow: /medical-research
Disallow: /servlet/linkout
Disallow: /na101/
Disallow: /na101v1/
Disallow: /na102/
Disallow: /doi/mlt/
Allow: /action/showJournal
Allow: /action/showXml
Allow: /series
Allow: /isbn
Allow: /doi/book
Allow: /action/generateSitemap
Crawl-delay: 1


Sitemap: https://www.journalofdairyscience.org:443/custom_pages.gz
User-agent: Twitterbot
Disallow:
User-agent: *
Disallow: /action
Disallow: /help
Disallow: /search
Disallow: /feedback
Disallow: /rss
Disallow: /action/clickThrough
Disallow: /action/showLogin
Disallow: /page/account-confirmation-thanks
Disallow: /media
Disallow: /medical-research
Disallow: /servlet/linkout
Disallow: /na101/
Disallow: /na101v1/
Disallow: /na102/
Disallow: /doi/mlt/
Allow: /action/showJournal
Allow: /action/showXml
Allow: /series
Allow: /isbn
Allow: /doi/book
Allow: /action/generateSitemap
Crawl-delay: 1


Sitemap: https://www.journalofdairyscience.org:443/custom_pages.gz

User-agent: *
Disallow: /action
Disallow: /help
Disallow: /search
Disallow: /feedback
Disallow: /rss
Disallow: /action/clickThrough
Disallow: /action/showLogin
Disallow: /page/account-confirmation-thanks
Disallow: /media
Disallow: /medical-research
Disallow: /servlet/linkout
Disallow: /na101/
Disallow: /na101v1/
Disallow: /na102/
Disallow: /doi/mlt/
Allow: /action/showJournal
Allow: /action/showXml
Allow: /series
Allow: /isbn
Allow: /doi/book
Allow: /action/generateSitemap

User-agent: Twitterbot
Allow: /

Crawl-delay: 1


Sitemap: https://www.journalofdairyscience.org/custom_pages.gz

User-agent: *
Disallow: /action
Disallow: /help
Disallow: /search
Disallow: /feedback
Disallow: /rss
Disallow: /action/clickThrough
Disallow: /action/showLogin
Disallow: /page/account-confirmation-thanks
Disallow: /media
Disallow: /medical-research
Disallow: /servlet/linkout
Disallow: /na101/
Disallow: /na101v1/
Disallow: /na102/
Disallow: /doi/mlt/
Allow: /action/showJournal
Allow: /action/showXml
Allow: /series
Allow: /isbn
Allow: /doi/book
Allow: /action/generateSitemap

User-agent: Twitterbot
Allow: /

Crawl-delay: 1


Sitemap: https://www.journalofdairyscience.org/custom_pages.gz

User-agent: *
Disallow: /action
Disallow: /help
Disallow: /search
Disallow: /feedback
Disallow: /rss
Disallow: /action/clickThrough
Disallow: /action/showLogin
Disallow: /page/account-confirmation-thanks
Disallow: /media
Disallow: /medical-research
Disallow: /servlet/linkout
Disallow: /na101/
Disallow: /na101v1/
Disallow: /na102/
Disallow: /doi/mlt/
Allow: /action/showJournal
Allow: /action/showXml
Allow: /series
Allow: /isbn
Allow: /doi/book
Allow: /action/generateSitemap

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
User-agent: Twitterbot
Allow: /

Crawl-delay: 1


Sitemap: https://www.journalofdairyscience.org/custom_pages.gz

User-agent: *
Disallow: /action
Disallow: /help
Disallow: /search
Disallow: /feedback
Disallow: /rss
Disallow: /action/clickThrough
Disallow: /action/showLogin
Disallow: /page/account-confirmation-thanks
Disallow: /media
Disallow: /medical-research
Disallow: /servlet/linkout
Disallow: /na101/
Disallow: /na101v1/
Disallow: /na102/
Disallow: /doi/mlt/
Allow: /action/showJournal
Allow: /action/showXml
Allow: /series
Allow: /isbn
Allow: /doi/book

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
User-agent: Twitterbot
Allow: /

Crawl-delay: 1


Sitemap: https://www.journalofdairyscience.org/custom_pages.gz

User-agent: *
Disallow: /action
Disallow: /help
Disallow: /search
Disallow: /feedback
Disallow: /rss
Disallow: /action/clickThrough
Disallow: /action/showLogin
Disallow: /page/account-confirmation-thanks
Disallow: /media
Disallow: /medical-research
Disallow: /servlet/linkout
Disallow: /na101/
Disallow: /na101v1/
Disallow: /na102/
Disallow: /doi/mlt/
Allow: /action/showJournal
Allow: /action/showXml
Allow: /series
Allow: /isbn
Allow: /doi/book

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
User-agent: Twitterbot
Allow: /

User-agent: GPTBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-agent: CCBot
Disallow: /

Crawl-delay: 1


Sitemap: https://www.journalofdairyscience.org/custom_pages.gz

User-agent: *
Disallow: /action
Disallow: /help
Disallow: /search
Disallow: /feedback
Disallow: /rss
Disallow: /action/clickThrough
Disallow: /action/showLogin
Disallow: /page/account-confirmation-thanks
Disallow: /media
Disallow: /medical-research
Disallow: /servlet/linkout
Disallow: /na101/
Disallow: /na101v1/
Disallow: /na102/
Disallow: /doi/mlt/
Allow: /action/showJournal
Allow: /action/showXml
Allow: /series
Allow: /isbn
Allow: /doi/book

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
User-agent: Twitterbot
Allow: /

User-agent: GPTBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Google-Extended
Disallow: /

Crawl-delay: 1


Sitemap: https://www.journalofdairyscience.org/sitemap-index-1.txt
Sitemap: https://www.journalofdairyscience.org/custom_pages.gz

User-agent: *
Disallow: /action
Disallow: /help
Disallow: /search
Disallow: /feedback
Disallow: /rss
Disallow: /action/clickThrough
Disallow: /action/showLogin
Disallow: /page/account-confirmation-thanks
Disallow: /media
Disallow: /medical-research
Disallow: /servlet/linkout
Disallow: /na101/
Disallow: /na101v1/
Disallow: /na102/
Disallow: /doi/mlt/
Allow: /action/showJournal
Allow: /action/showXml
Allow: /series
Allow: /isbn
Allow: /doi/book

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: GPTBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Google-Extended
Disallow: /

Crawl-delay: 1


Sitemap: https://www.journalofdairyscience.org/sitemap-index-1.txt
Sitemap: https://www.journalofdairyscience.org/custom_pages.gz

User-agent: *
Disallow: /action
Disallow: /help
Disallow: /search
Disallow: /feedback
Disallow: /rss
Disallow: /action/clickThrough
Disallow: /action/showLogin
Disallow: /page/account-confirmation-thanks
Disallow: /media
Disallow: /medical-research
Disallow: /servlet/linkout
Disallow: /na101/
Disallow: /na101v1/
Disallow: /na102/
Disallow: /doi/mlt/
Allow: /action/showJournal
Allow: /action/showXml
Allow: /series
Allow: /isbn
Allow: /doi/book
Allow: /.well-known/tdmrep.json

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: GPTBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Google-Extended
Disallow: /

Crawl-delay: 1


Sitemap: https://www.journalofdairyscience.org/sitemap-index-1.txt
Sitemap: https://www.journalofdairyscience.org/custom_pages.gz

User-agent: *
Disallow: /action
Disallow: /help
Disallow: /search
Disallow: /feedback
Disallow: /rss
Disallow: /action/clickThrough
Disallow: /action/showLogin
Disallow: /page/account-confirmation-thanks
Disallow: /media
Disallow: /medical-research
Disallow: /servlet/linkout
Disallow: /na101/
Disallow: /na101v1/
Disallow: /na102/
Disallow: /doi/mlt/
Disallow: /cms/attachment/92c526c4-ef1f-4065-b12b-ca4ae0f3e84f/gr1*
Allow: /action/showJournal
Allow: /action/showXml
Allow: /series
Allow: /isbn
Allow: /doi/book
Allow: /.well-known/tdmrep.json

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: GPTBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: PerplexityBot
Disallow: /

Crawl-delay: 1


Sitemap: https://www.journalofdairyscience.org/sitemap-index-1.txt
Sitemap: https://www.journalofdairyscience.org/custom_pages.gz

User-agent: *
Disallow: /action
Disallow: /help
Disallow: /search
Disallow: /feedback
Disallow: /rss
Disallow: /action/clickThrough
Disallow: /action/showLogin
Disallow: /page/account-confirmation-thanks
Disallow: /media
Disallow: /medical-research
Disallow: /servlet/linkout
Disallow: /na101/
Disallow: /na101v1/
Disallow: /na102/
Disallow: /doi/mlt/
Disallow: /cms/attachment/92c526c4-ef1f-4065-b12b-ca4ae0f3e84f/gr1*
Disallow: /personalize/addFavoritePublication
Disallow: /pb/widgets/*
Allow: /action/showJournal
Allow: /action/showXml
Allow: /series
Allow: /isbn
Allow: /doi/book
Allow: /.well-known/tdmrep.json

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: GPTBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: PerplexityBot
Disallow: /

Crawl-delay: 1


Sitemap: https://www.journalofdairyscience.org/sitemap-index-1.txt
Sitemap: https://www.journalofdairyscience.org/custom_pages.gz

User-agent: *
Disallow: /action
Disallow: /help
Disallow: /search
Disallow: /feedback
Disallow: /rss
Disallow: /action/clickThrough
Disallow: /action/showLogin
Disallow: /page/account-confirmation-thanks
Disallow: /media
Disallow: /medical-research
Disallow: /servlet/linkout
Disallow: /na101/
Disallow: /na101v1/
Disallow: /na102/
Disallow: /doi/mlt/
Disallow: /cms/attachment/92c526c4-ef1f-4065-b12b-ca4ae0f3e84f/gr1*
Disallow: /personalize/addFavoritePublication
Disallow: /pb/widgets/*
Allow: /action/showJournal
Allow: /action/showXml
Allow: /series
Allow: /isbn
Allow: /doi/book
Allow: /.well-known/tdmrep.json

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: GPTBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: PerplexityBot
Disallow: /

User-agent: AI2Bot
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: cohere-training-data-crawler
Disallow: /

User-agent: cohere-ai
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: iAskBot/1.0
Disallow: /

User-agent: meta-externalagent
Disallow: /

User-agent: MistralAI-User
Disallow: /

User-agent: omgili
Disallow: /

User-agent: TimpiBot
Disallow: /

User-agent: YouBot
Disallow: /

Crawl-delay: 1


Sitemap: https://www.journalofdairyscience.org/sitemap-index-1.txt
Sitemap: https://www.journalofdairyscience.org/custom_pages.gz

User-agent: *
Disallow: /action
Disallow: /help
Disallow: /search
Disallow: /feedback
Disallow: /rss
Disallow: /action/clickThrough
Disallow: /action/showLogin
Disallow: /page/account-confirmation-thanks
Disallow: /media
Disallow: /medical-research
Disallow: /servlet/linkout
Disallow: /na101/
Disallow: /na101v1/
Disallow: /na102/
Disallow: /doi/mlt/
Disallow: /cms/attachment/92c526c4-ef1f-4065-b12b-ca4ae0f3e84f/gr1*
Disallow: /personalize/addFavoritePublication
Disallow: /pb/widgets/*
Allow: /action/showJournal
Allow: /action/showXml
Allow: /series
Allow: /isbn
Allow: /doi/book
Allow: /.well-known/*

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: GPTBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: PerplexityBot
Disallow: /

User-agent: AI2Bot
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: cohere-training-data-crawler
Disallow: /

User-agent: cohere-ai
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: iAskBot/1.0
Disallow: /

User-agent: meta-externalagent
Disallow: /

User-agent: MistralAI-User
Disallow: /

User-agent: omgili
Disallow: /

User-agent: TimpiBot
Disallow: /

User-agent: YouBot
Disallow: /

Crawl-delay: 1


User-agent: *
Disallow: /*_pdf

User-agent: Googlebot
Allow: /

User-agent: TurnitinBot
Allow: /

User-agent: NBDCbot(biosciencedbc.jp/dbsearch)
Allow: /

User-agent: ciniibot/1.0
Allow: /

User-agent: ASCPro16
Allow: /

User-agent: Summon
Allow: /

User-agent: bingbot
Crawl-delay: 5


Sitemap: http://www.jstor.org/sitemap.xml

User-agent: Googlebot
Disallow: /action
Disallow: /help
Disallow: /search
Disallow: /sustainability/search
Disallow: /feedback
Disallow: /rss
Disallow: /action/clickThrough
Disallow: /token
Disallow: /topic
Disallow: /servlet/linkout
Disallow: /stoken
Disallow: /stable/pdfreader
Disallow: /stable/view
Disallow: /doi/abs
Disallow: /userimages
Disallow: /stable/suppl
Disallow: /stable/full
Disallow: /start-session
Disallow: /citation
Disallow: /purchase
Disallow: /clockss-manifest
Disallow: /institutionSearch
Disallow: /register
Disallow: /api
Disallow: /dfr/results
Disallow: /action/doImageSearch
Allow: /doi/xml
Allow: /action/showPdfForWebcrawler
Allow: /action/showJournal
Allow: /action/showPublication
Allow: /action/showXml
Allow: /action/showSubscriptionJournalsAsXml

User-agent: *
Disallow: /action
Disallow: /help
Disallow: /search
Disallow: /sustainability/search
Disallow: /feedback
Disallow: /rss
Disallow: /action/clickThrough
Disallow: /token
Disallow: /topic
Disallow: /servlet/linkout
Disallow: /stoken
Disallow: /stable/pdfreader
Disallow: /stable/view
Disallow: /doi/abs
Disallow: /userimages
Disallow: /stable/suppl
Disallow: /stable/full
Disallow: /start-session
Disallow: /citation
Disallow: /purchase
Disallow: /clockss-manifest
Disallow: /institutionSearch
Disallow: /register
Disallow: /api
Disallow: /dfr/results
Disallow: /action/doImageSearch
Disallow: /doi/xml
Allow: /action/showPdfForWebcrawler
Allow: /action/showJournal
Allow: /action/showPublication
Allow: /action/showXml
Allow: /action/showSubscriptionJournalsAsXml


Sitemap: http://www.jstor.org/sitemap.xml

User-agent: Googlebot
Disallow: /action
Disallow: /help
Disallow: /search
Disallow: /sustainability/search
Disallow: /feedback
Disallow: /rss
Disallow: /action/clickThrough
Disallow: /token
Disallow: /topic
Disallow: /servlet/linkout
Disallow: /stoken
Disallow: /stable/pdfreader
Disallow: /stable/view
Disallow: /doi/abs
Disallow: /userimages
Disallow: /stable/suppl
Disallow: /stable/full
Disallow: /start-session
Disallow: /citation
Disallow: /purchase
Disallow: /clockss-manifest
Disallow: /institutionSearch
Disallow: /register
Disallow: /api
Disallow: /dfr/results
Disallow: /action/doImageSearch
Allow: /doi/xml
Allow: /action/showPdfForWebcrawler
Allow: /action/showJournal
Allow: /action/showPublication
Allow: /action/showXml
Allow: /action/showSubscriptionJournalsAsXml
Disallow: /ui_log

User-agent: *
Disallow: /action
Disallow: /help
Disallow: /search
Disallow: /sustainability/search
Disallow: /feedback
Disallow: /rss
Disallow: /action/clickThrough
Disallow: /token
Disallow: /topic
Disallow: /servlet/linkout
Disallow: /stoken
Disallow: /stable/pdfreader
Disallow: /stable/view
Disallow: /doi/abs
Disallow: /userimages
Disallow: /stable/suppl
Disallow: /stable/full
Disallow: /start-session
Disallow: /citation
Disallow: /purchase
Disallow: /clockss-manifest
Disallow: /institutionSearch
Disallow: /register
Disallow: /api
Disallow: /dfr/results
Disallow: /action/doImageSearch
Disallow: /doi/xml
Allow: /action/showPdfForWebcrawler
Allow: /action/showJournal
Allow: /action/showPublication
Allow: /action/showXml
Allow: /action/showSubscriptionJournalsAsXml
Disallow: /ui_log


Sitemap: https://www.jstor.org/sitemap.xml

User-agent: Googlebot
Disallow: /action
Disallow: /api
Disallow: /citation
Disallow: /clockss-manifest
Disallow: /doi/abs
Disallow: /feedback
Disallow: /institutionSearch
Disallow: /purchase
Disallow: /register
Disallow: /stable/full
Disallow: /stable/suppl
Disallow: /stable/view
Disallow: /start-session
Disallow: /stoken
Disallow: /token
Disallow: /topic
Disallow: /ui_log
Disallow: /userimages
Allow: /action/showJournal
Allow: /action/showPublication
Allow: /action/showSubscriptionJournalsAsXml
Allow: /action/showXml
Allow: /doi/xml

User-agent: *
Disallow: /action
Disallow: /api
Disallow: /citation
Disallow: /clockss-manifest
Disallow: /doi/abs
Disallow: /doi/xml
Disallow: /feedback
Disallow: /institutionSearch
Disallow: /purchase
Disallow: /register
Disallow: /stable/full
Disallow: /stable/suppl
Disallow: /stable/view
Disallow: /start-session
Disallow: /stoken
Disallow: /token
Disallow: /topic
Disallow: /ui_log
Disallow: /userimages
Allow: /action/showJournal
Allow: /action/showPublication
Allow: /action/showSubscriptionJournalsAsXml
Allow: /action/showXml


Sitemap: https://www.jstor.org/sitemap.xml

User-agent: Googlebot
Disallow: /action
Disallow: /api
Disallow: /citation
Disallow: /clockss-manifest
Disallow: /doi/abs
Disallow: /feedback
Disallow: /purchase
Disallow: /register/
Disallow: /stable/full
Disallow: /stable/suppl
Disallow: /stable/view
Disallow: /start-session
Disallow: /stoken
Disallow: /token
Disallow: /topic
Disallow: /ui_log
Disallow: /userimages
Allow: /action/showLogin
Allow: /action/showJournal
Allow: /action/showPublication
Allow: /action/showSubscriptionJournalsAsXml
Allow: /action/showXml
Allow: /doi/xml

User-agent: *
Disallow: /action
Disallow: /api
Disallow: /citation
Disallow: /clockss-manifest
Disallow: /doi/abs
Disallow: /doi/xml
Disallow: /feedback
Disallow: /purchase
Disallow: /register/
Disallow: /stable/full
Disallow: /stable/suppl
Disallow: /stable/view
Disallow: /start-session
Disallow: /stoken
Disallow: /token
Disallow: /topic
Disallow: /ui_log
Disallow: /userimages
Allow: /action/showLogin
Allow: /action/showJournal
Allow: /action/showPublication
Allow: /action/showSubscriptionJournalsAsXml
Allow: /action/showXml


Sitemap: https://www.jstor.org/sitemap.xml

User-agent: Googlebot
Disallow: /action
Disallow: /api
Disallow: /citation
Disallow: /clockss-manifest
Disallow: /doi/abs
Disallow: /feedback
Disallow: /purchase
Disallow: /register/
Disallow: /stable/full
Disallow: /stable/suppl
Disallow: /stable/view
Disallow: /start-session
Disallow: /stoken
Disallow: /tc/accept
Disallow: /token
Disallow: /topic
Disallow: /ui_log
Disallow: /userimages
Allow: /action/showLogin
Allow: /action/showJournal
Allow: /action/showPublication
Allow: /action/showSubscriptionJournalsAsXml
Allow: /action/showXml
Allow: /doi/xml

User-agent: *
Disallow: /action
Disallow: /api
Disallow: /citation
Disallow: /clockss-manifest
Disallow: /doi/abs
Disallow: /doi/xml
Disallow: /feedback
Disallow: /purchase
Disallow: /register/
Disallow: /stable/full
Disallow: /stable/suppl
Disallow: /stable/view
Disallow: /start-session
Disallow: /stoken
Disallow: /tc/accept
Disallow: /token
Disallow: /topic
Disallow: /ui_log
Disallow: /userimages
Allow: /action/showLogin
Allow: /action/showJournal
Allow: /action/showPublication
Allow: /action/showSubscriptionJournalsAsXml
Allow: /action/showXml

# Disallow crawling for the purposes of training models

# https://support.anthropic.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler
User-agent: ClaudeBot
# https://commoncrawl.org/faq
User-agent: CCBot
# https://platform.openai.com/docs/gptbot
User-agent: GPTBot
# https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers
User-agent: Google-Extended
Disallow: /


User-Agent: *
Disallow: /search
Disallow: /*searchTerm
Disallow: /deals/compare
Disallow: /*/deals/compare
Disallow: *jwsource=*

Sitemap: https://www.livescience.com/sitemap.xml


User-Agent: *
Disallow: /search/*
Disallow: /*searchTerm
Disallow: /deals/compare
Disallow: /*/deals/compare
Disallow: *jwsource=*

Sitemap: https://www.livescience.com/sitemap.xml


User-Agent: *
Disallow: /search*
Disallow: /*searchTerm
Disallow: /deals/compare
Disallow: /*/deals/compare
Disallow: *jwsource=*

Disallow: *%7B*%7D*

Sitemap: https://www.livescience.com/sitemap.xml


##################################################
# Vanilla-wide rules                             #
##################################################
User-agent: *

# Common path patterns (* prefix to handle localisation)
Disallow: */deals/compare
Disallow: */html/

# Common query string patterns
Disallow: *searchTerm=*
Disallow: *sortBy=*
Disallow: *productBrand=*
Disallow: *jwsource=*
Disallow: *region-switch=*
Disallow: *%7B*%7D*

##################################################
# Sitemaps                                       #
##################################################
User-agent: *
https://www.livescience.com/sitemap.xml


##################################################
# Vanilla-wide rules                             #
##################################################
User-agent: *

# Common path patterns (* prefix to handle localisation)
Disallow: */deals/compare
Disallow: */html/

# Common query string patterns
Disallow: *searchTerm=*
Disallow: *sortBy=*
Disallow: *productBrand=*
Disallow: *jwsource=*
Disallow: *region-switch=*
Disallow: *%7B*%7D*

##################################################
# Sitemaps                                       #
##################################################
User-agent: *
Sitemap: https://www.livescience.com/sitemap.xml


##################################################
# Vanilla-wide rules                             #
##################################################
User-agent: *

# Common path patterns (* prefix to handle localisation)
Disallow: */deals/compare
Disallow: */html/

# Common query string patterns
Disallow: *searchTerm=*
Disallow: *sortBy=*
Disallow: *productBrand=*
Disallow: *jwsource=*
Disallow: *%7B*%7D*

##################################################
# Sitemaps                                       #
##################################################
User-agent: *
Sitemap: https://www.livescience.com/sitemap.xml


##################################################
# Vanilla-wide rules                             #
##################################################
User-agent: *

# Common path patterns (* prefix to handle localisation)
Disallow: */deals/compare
Disallow: */html/

# Common query string patterns
Disallow: *searchTerm=*
Disallow: *sortBy=*
Disallow: *productBrand=*
Disallow: *%7B*%7D*

##################################################
# Sitemaps                                       #
##################################################
User-agent: *
Sitemap: https://www.livescience.com/sitemap.xml


##################################################
# Vanilla-wide rules                             #
##################################################
User-agent: *

# Common path patterns (* prefix to handle localisation)
Disallow: */deals/compare
Disallow: */html/

# Common query string patterns
Disallow: *searchTerm=*
Disallow: *sortBy=*
Disallow: *productBrand=*
Disallow: *%7B*%7D*

# Infinite scroll paths
Disallow: /infinite-scroll-article/*
Disallow: /infinite-scroll-review/*
Disallow: /infinite-scroll-recipe/*

##################################################
# Sitemaps                                       #
##################################################
User-agent: *
Sitemap: https://www.livescience.com/sitemap.xml


##################################################
# Vanilla-wide rules                             #
##################################################
User-agent: *

# Common path patterns (* prefix to handle localisation)
Disallow: */deals/compare
Disallow: */html/
Disallow: */p/*/embed/captioned

# Common query string patterns
Disallow: *searchTerm=*
Disallow: *sortBy=*
Disallow: *productBrand=*
Disallow: *region-switch=*
Disallow: *%7B*%7D*

# Infinite scroll paths
Disallow: /infinite-scroll-article/*
Disallow: /infinite-scroll-review/*
Disallow: /infinite-scroll-recipe/*

##################################################
# Sitemaps                                       #
##################################################
User-agent: *
Sitemap: https://www.livescience.com/sitemap.xml


##################################################
# Vanilla-wide rules                             #
##################################################
User-agent: GPTBot
Disallow: /

User-agent: *

# Common path patterns (* prefix to handle localisation)
Disallow: */deals/compare
Disallow: */html/
Disallow: */p/*/embed/captioned

# Common query string patterns
Disallow: *searchTerm=*
Disallow: *sortBy=*
Disallow: *productBrand=*
Disallow: *region-switch=*
Disallow: *%7B*%7D*

# Infinite scroll paths
Disallow: /infinite-scroll-article/*
Disallow: /infinite-scroll-review/*
Disallow: /infinite-scroll-recipe/*

##################################################
# Sitemaps                                       #
##################################################
User-agent: *
Sitemap: https://www.livescience.com/sitemap.xml


##################################################
# Vanilla-wide rules                             #
##################################################
User-agent: GPTBot
Disallow: /

User-agent: *

# Common path patterns (* prefix to handle localisation)
Disallow: */deals/compare
Disallow: */html/
Disallow: */p/*/embed/captioned

# Common query string patterns
Disallow: *searchTerm=*
Disallow: *sortBy=*
Disallow: *productBrand=*
Disallow: *%7B*%7D*

# Infinite scroll paths
Disallow: /infinite-scroll-article/*
Disallow: /infinite-scroll-review/*
Disallow: /infinite-scroll-recipe/*

##################################################
# Sitemaps                                       #
##################################################
User-agent: *
Sitemap: https://www.livescience.com/sitemap.xml


##################################################
# Vanilla-wide rules                             #
##################################################
User-agent: *

# Common path patterns (* prefix to handle localisation)
Disallow: */deals/compare
Disallow: */html/
Disallow: */p/*/embed/captioned

# Common query string patterns
Disallow: *searchTerm=*
Disallow: *sortBy=*
Disallow: *productBrand=*
Disallow: *%7B*%7D*

# Infinite scroll paths
Disallow: /infinite-scroll-article/*
Disallow: /infinite-scroll-review/*
Disallow: /infinite-scroll-recipe/*

##################################################
# Sitemaps                                       #
##################################################
User-agent: *
Sitemap: https://www.livescience.com/sitemap.xml


##################################################
# Vanilla-wide rules                             #
##################################################
User-agent: *

# Common path patterns (* prefix to handle localisation)
Disallow: */deals/compare
Disallow: */html/
Disallow: */p/*/embed/captioned

# Common query string patterns
Disallow: *searchTerm=*
Disallow: *sortBy=*
Disallow: *productBrand=*
Disallow: *%7B*%7D*

# Infinite scroll paths
Disallow: /infinite-scroll-article/*
Disallow: /infinite-scroll-review/*
Disallow: /infinite-scroll-recipe/*

##################################################
# Sitemaps                                       #
##################################################
User-agent: *
Sitemap: https://www.livescience.com/sitemap.xml
Sitemap: https://www.livescience.com/sitemap-news.xml


##################################################
# Vanilla-wide rules                             #
##################################################
User-agent: *

# Common path patterns (* prefix to handle localisation)
Disallow: */deals/compare
Disallow: */html/
Disallow: */p/*/embed/captioned
Disallow: */outlink*

# Common query string patterns
Disallow: *searchTerm=*
Disallow: *sortBy=*
Disallow: *productBrand=*
Disallow: *%7B*%7D*
Disallow: *seenMatchId=*

# Infinite scroll paths
Disallow: /infinite-scroll-article/*
Disallow: /infinite-scroll-review/*
Disallow: /infinite-scroll-recipe/*

##################################################
# Sitemaps                                       #
##################################################
User-agent: *
Sitemap: https://www.livescience.com/sitemap.xml
Sitemap: https://www.livescience.com/sitemap-news.xml


##################################################
# Vanilla-wide rules                             #
##################################################
User-agent: *

# Common path patterns (* prefix to handle localisation)
Disallow: */deals/compare
Disallow: */html/
Disallow: */p/*/embed/captioned
Disallow: */outlink*

# Common query string patterns
Disallow: *searchTerm=*
Disallow: *sortBy=*
Disallow: *productBrand=*
Disallow: *%7B*%7D*
Disallow: *seenMatchId=*

# Infinite scroll paths
Disallow: /infinite-scroll-article/*
Disallow: /infinite-scroll-review/*
Disallow: /infinite-scroll-recipe/*

# AI crawlers
User-agent: bytespider
User-agent: mistralai
User-agent: cohere
User-agent: ai2bot
User-agent: youbot
User-agent: omgili
User-agent: diffbot
User-agent: kangaroo
User-agent: img2dataset
User-agent: amazonbot
User-agent: amazon-qbusiness
Disallow: /

##################################################
# Sitemaps                                       #
##################################################
User-agent: *
Sitemap: https://www.livescience.com/sitemap.xml
Sitemap: https://www.livescience.com/sitemap-news.xml


##################################################
# Vanilla-wide rules                             #
##################################################
User-agent: *

# Common path patterns (* prefix to handle localisation)
Disallow: */deals/compare
Disallow: */html/
Disallow: */p/*/embed/captioned
Disallow: */outlink*

# Common query string patterns
Disallow: *searchTerm=*
Disallow: *sortBy=*
Disallow: *productBrand=*
Disallow: *%7B*%7D*
Disallow: *seenMatchId=*

# Infinite scroll paths
Disallow: /infinite-scroll-article/*
Disallow: /infinite-scroll-review/*
Disallow: /infinite-scroll-recipe/*

# AI crawlers
User-agent: bytespider
User-agent: mistralai
User-agent: cohere
User-agent: ai2bot
User-agent: youbot
User-agent: omgili
User-agent: diffbot
User-agent: kangaroo
User-agent: img2dataset
User-agent: amazonbot
User-agent: amazon-qbusiness
User-Agent: meta-externalagent
User-Agent: meta-webindexer
Disallow: /

##################################################
# Sitemaps                                       #
##################################################
User-agent: *
Sitemap: https://www.livescience.com/sitemap.xml
Sitemap: https://www.livescience.com/sitemap-news.xml


#
# robots.txt
#
# This file is to prevent the crawling and indexing of certain parts
# of your site by web crawlers and spiders run by sites like Yahoo!
# and Google. By telling these "robots" where not to go on your site,
# you save bandwidth and server resources.
#
# This file will be ignored unless it is at the root of your host:
# Used:    http://example.com/robots.txt
# Ignored: http://example.com/site/robots.txt
#
# For more information about the robots.txt standard, see:
# http://www.robotstxt.org/robotstxt.html

User-agent: Lucidworks-Anda/2.0
Crawl-delay: 0

User-agent: *
Crawl-delay: 5

# CSS, JS, Images
Allow: */misc/*.css$
Allow: */misc/*.css?
Allow: */misc/*.js$
Allow: */misc/*.js?
Allow: */misc/*.gif
Allow: */misc/*.jpg
Allow: */misc/*.jpeg
Allow: */misc/*.png
Allow: */modules/*.css$
Allow: */modules/*.css?
Allow: */modules/*.js$
Allow: */modules/*.js?
Allow: */modules/*.gif
Allow: */modules/*.jpg
Allow: */modules/*.jpeg
Allow: */modules/*.png
Allow: */profiles/*.css$
Allow: */profiles/*.css?
Allow: */profiles/*.js$
Allow: */profiles/*.js?
Allow: */profiles/*.gif
Allow: */profiles/*.jpg
Allow: */profiles/*.jpeg
Allow: */profiles/*.png
Allow: */themes/*.css$
Allow: */themes/*.css?
Allow: */themes/*.js$
Allow: */themes/*.js?
Allow: */themes/*.gif
Allow: */themes/*.jpg
Allow: */themes/*.jpeg
Allow: */themes/*.png
Allow: */study/2020-2021
# Directories
Disallow: */includes/
Disallow: */modules/
Disallow: */profiles/
Disallow: */scripts/
Disallow: */themes/
Disallow: */study/2009-2010
Disallow: */study/2010-2011
Disallow: */study/2011-2012
Disallow: */study/2012-2013
Disallow: */study/2013-2014
Disallow: */study/2014-2015
Disallow: */study/2015-2016
Disallow: */study/2016-2017
Disallow: */study/2017-2018
Disallow: */study/2018-2019
Disallow: */study/2019-2020
# Files
Disallow: */CHANGELOG.txt
Disallow: */cron.php
Disallow: */INSTALL.mysql.txt
Disallow: */INSTALL.pgsql.txt
Disallow: */INSTALL.sqlite.txt
Disallow: */install.php
Disallow: */INSTALL.txt
Disallow: */LICENSE.txt
Disallow: */MAINTAINERS.txt
Disallow: */update.php
Disallow: */UPGRADE.txt
Disallow: */xmlrpc.php
Disallow: */misc/favicon.ico
# Paths (clean URLs)
Disallow: */admin/
Disallow: */comment/reply/
Disallow: */filter/tips/
Disallow: */node/add/
Disallow: */search/
Disallow: */user/register/
Disallow: */user/password/
Disallow: */user/login/
Disallow: */user/logout/
Disallow: */user
# Paths (no clean URLs)
Disallow: */?q=admin/
Disallow: */?q=comment/reply/
Disallow: */?q=filter/tips/
Disallow: */?q=node/add/
Disallow: */?q=search/
Disallow: */?q=user/password/
Disallow: */?q=user/register/
Disallow: */?q=user/login/
Disallow: */?q=user/logout/


#
# robots.txt
#
# This file is to prevent the crawling and indexing of certain parts
# of your site by web crawlers and spiders run by sites like Yahoo!
# and Google. By telling these "robots" where not to go on your site,
# you save bandwidth and server resources.
#
# This file will be ignored unless it is at the root of your host:
# Used:    http://example.com/robots.txt
# Ignored: http://example.com/site/robots.txt
#
# For more information about the robots.txt standard, see:
# http://www.robotstxt.org/robotstxt.html

User-agent: Lucidworks-Anda/2.0
Crawl-delay: 0

User-agent: *
Crawl-delay: 5

# CSS, JS, Images
Allow: */misc/*.css$
Allow: */misc/*.css?
Allow: */misc/*.js$
Allow: */misc/*.js?
Allow: */misc/*.gif
Allow: */misc/*.jpg
Allow: */misc/*.jpeg
Allow: */misc/*.png
Allow: */modules/*.css$
Allow: */modules/*.css?
Allow: */modules/*.js$
Allow: */modules/*.js?
Allow: */modules/*.gif
Allow: */modules/*.jpg
Allow: */modules/*.jpeg
Allow: */modules/*.png
Allow: */profiles/*.css$
Allow: */profiles/*.css?
Allow: */profiles/*.js$
Allow: */profiles/*.js?
Allow: */profiles/*.gif
Allow: */profiles/*.jpg
Allow: */profiles/*.jpeg
Allow: */profiles/*.png
Allow: */themes/*.css$
Allow: */themes/*.css?
Allow: */themes/*.js$
Allow: */themes/*.js?
Allow: */themes/*.gif
Allow: */themes/*.jpg
Allow: */themes/*.jpeg
Allow: */themes/*.png
Allow: */study/2021-2022
# Directories
Disallow: */includes/
Disallow: */modules/
Disallow: */profiles/
Disallow: */scripts/
Disallow: */themes/
Disallow: */study/2009-2010
Disallow: */study/2010-2011
Disallow: */study/2011-2012
Disallow: */study/2012-2013
Disallow: */study/2013-2014
Disallow: */study/2014-2015
Disallow: */study/2015-2016
Disallow: */study/2016-2017
Disallow: */study/2017-2018
Disallow: */study/2018-2019
Disallow: */study/2019-2020
Disallow: */study/2020-2021
# Files
Disallow: */CHANGELOG.txt
Disallow: */cron.php
Disallow: */INSTALL.mysql.txt
Disallow: */INSTALL.pgsql.txt
Disallow: */INSTALL.sqlite.txt
Disallow: */install.php
Disallow: */INSTALL.txt
Disallow: */LICENSE.txt
Disallow: */MAINTAINERS.txt
Disallow: */update.php
Disallow: */UPGRADE.txt
Disallow: */xmlrpc.php
Disallow: */misc/favicon.ico
# Paths (clean URLs)
Disallow: */admin/
Disallow: */comment/reply/
Disallow: */filter/tips/
Disallow: */node/add/
Disallow: */search/
Disallow: */user/register/
Disallow: */user/password/
Disallow: */user/login/
Disallow: */user/logout/
Disallow: */user
# Paths (no clean URLs)
Disallow: */?q=admin/
Disallow: */?q=comment/reply/
Disallow: */?q=filter/tips/
Disallow: */?q=node/add/
Disallow: */?q=search/
Disallow: */?q=user/password/
Disallow: */?q=user/register/
Disallow: */?q=user/login/
Disallow: */?q=user/logout/


#
# robots.txt
#
# This file is to prevent the crawling and indexing of certain parts
# of your site by web crawlers and spiders run by sites like Yahoo!
# and Google. By telling these "robots" where not to go on your site,
# you save bandwidth and server resources.
#
# This file will be ignored unless it is at the root of your host:
# Used:    http://example.com/robots.txt
# Ignored: http://example.com/site/robots.txt
#
# For more information about the robots.txt standard, see:
# http://www.robotstxt.org/robotstxt.html

User-agent: Lucidworks-Anda/2.0
Crawl-delay: 0

User-agent: *
Crawl-delay: 5

# CSS, JS, Images
Allow: */misc/*.css$
Allow: */misc/*.css?
Allow: */misc/*.js$
Allow: */misc/*.js?
Allow: */misc/*.gif
Allow: */misc/*.jpg
Allow: */misc/*.jpeg
Allow: */misc/*.png
Allow: */modules/*.css$
Allow: */modules/*.css?
Allow: */modules/*.js$
Allow: */modules/*.js?
Allow: */modules/*.gif
Allow: */modules/*.jpg
Allow: */modules/*.jpeg
Allow: */modules/*.png
Allow: */profiles/*.css$
Allow: */profiles/*.css?
Allow: */profiles/*.js$
Allow: */profiles/*.js?
Allow: */profiles/*.gif
Allow: */profiles/*.jpg
Allow: */profiles/*.jpeg
Allow: */profiles/*.png
Allow: */themes/*.css$
Allow: */themes/*.css?
Allow: */themes/*.js$
Allow: */themes/*.js?
Allow: */themes/*.gif
Allow: */themes/*.jpg
Allow: */themes/*.jpeg
Allow: */themes/*.png
Allow: */study/2022-2023
# Directories
Disallow: */includes/
Disallow: */modules/
Disallow: */profiles/
Disallow: */scripts/
Disallow: */themes/
Disallow: */study/2009-2010
Disallow: */study/2010-2011
Disallow: */study/2011-2012
Disallow: */study/2012-2013
Disallow: */study/2013-2014
Disallow: */study/2014-2015
Disallow: */study/2015-2016
Disallow: */study/2016-2017
Disallow: */study/2017-2018
Disallow: */study/2018-2019
Disallow: */study/2019-2020
Disallow: */study/2020-2021
Disallow: */study/2021-2022
# Files
Disallow: */CHANGELOG.txt
Disallow: */cron.php
Disallow: */INSTALL.mysql.txt
Disallow: */INSTALL.pgsql.txt
Disallow: */INSTALL.sqlite.txt
Disallow: */install.php
Disallow: */INSTALL.txt
Disallow: */LICENSE.txt
Disallow: */MAINTAINERS.txt
Disallow: */update.php
Disallow: */UPGRADE.txt
Disallow: */xmlrpc.php
Disallow: */misc/favicon.ico
# Paths (clean URLs)
Disallow: */admin/
Disallow: */comment/reply/
Disallow: */filter/tips/
Disallow: */node/add/
Disallow: */search/
Disallow: */user/register/
Disallow: */user/password/
Disallow: */user/login/
Disallow: */user/logout/
Disallow: */user
# Paths (no clean URLs)
Disallow: */?q=admin/
Disallow: */?q=comment/reply/
Disallow: */?q=filter/tips/
Disallow: */?q=node/add/
Disallow: */?q=search/
Disallow: */?q=user/password/
Disallow: */?q=user/register/
Disallow: */?q=user/login/
Disallow: */?q=user/logout/


#
# robots.txt
#
# This file is to prevent the crawling and indexing of certain parts
# of your site by web crawlers and spiders run by sites like Yahoo!
# and Google. By telling these "robots" where not to go on your site,
# you save bandwidth and server resources.
#
# This file will be ignored unless it is at the root of your host:
# Used:    http://example.com/robots.txt
# Ignored: http://example.com/site/robots.txt
#
# For more information about the robots.txt standard, see:
# http://www.robotstxt.org/robotstxt.html

User-agent: Lucidworks-Anda/2.0
Crawl-delay: 0

User-agent: *
Crawl-delay: 5

# CSS, JS, Images
Allow: */misc/*.css$
Allow: */misc/*.css?
Allow: */misc/*.js$
Allow: */misc/*.js?
Allow: */misc/*.gif
Allow: */misc/*.jpg
Allow: */misc/*.jpeg
Allow: */misc/*.png
Allow: */modules/*.css$
Allow: */modules/*.css?
Allow: */modules/*.js$
Allow: */modules/*.js?
Allow: */modules/*.gif
Allow: */modules/*.jpg
Allow: */modules/*.jpeg
Allow: */modules/*.png
Allow: */profiles/*.css$
Allow: */profiles/*.css?
Allow: */profiles/*.js$
Allow: */profiles/*.js?
Allow: */profiles/*.gif
Allow: */profiles/*.jpg
Allow: */profiles/*.jpeg
Allow: */profiles/*.png
Allow: */themes/*.css$
Allow: */themes/*.css?
Allow: */themes/*.js$
Allow: */themes/*.js?
Allow: */themes/*.gif
Allow: */themes/*.jpg
Allow: */themes/*.jpeg
Allow: */themes/*.png
Allow: */study/2023-2024
# Directories
Disallow: */includes/
Disallow: */modules/
Disallow: */profiles/
Disallow: */scripts/
Disallow: */themes/
Disallow: */study/2009-2010
Disallow: */study/2010-2011
Disallow: */study/2011-2012
Disallow: */study/2012-2013
Disallow: */study/2013-2014
Disallow: */study/2014-2015
Disallow: */study/2015-2016
Disallow: */study/2016-2017
Disallow: */study/2017-2018
Disallow: */study/2018-2019
Disallow: */study/2019-2020
Disallow: */study/2020-2021
Disallow: */study/2021-2022
Disallow: */study/2022-2023
# Files
Disallow: */CHANGELOG.txt
Disallow: */cron.php
Disallow: */INSTALL.mysql.txt
Disallow: */INSTALL.pgsql.txt
Disallow: */INSTALL.sqlite.txt
Disallow: */install.php
Disallow: */INSTALL.txt
Disallow: */LICENSE.txt
Disallow: */MAINTAINERS.txt
Disallow: */update.php
Disallow: */UPGRADE.txt
Disallow: */xmlrpc.php
Disallow: */misc/favicon.ico
# Paths (clean URLs)
Disallow: */admin/
Disallow: */comment/reply/
Disallow: */filter/tips/
Disallow: */node/add/
Disallow: */search/
Disallow: */user/register/
Disallow: */user/password/
Disallow: */user/login/
Disallow: */user/logout/
Disallow: */user
# Paths (no clean URLs)
Disallow: */?q=admin/
Disallow: */?q=comment/reply/
Disallow: */?q=filter/tips/
Disallow: */?q=node/add/
Disallow: */?q=search/
Disallow: */?q=user/password/
Disallow: */?q=user/register/
Disallow: */?q=user/login/
Disallow: */?q=user/logout/


#
# robots.txt
#
# This file is to prevent the crawling and indexing of certain parts
# of your site by web crawlers and spiders run by sites like Yahoo!
# and Google. By telling these "robots" where not to go on your site,
# you save bandwidth and server resources.
#
# This file will be ignored unless it is at the root of your host:
# Used:    http://example.com/robots.txt
# Ignored: http://example.com/site/robots.txt
#
# For more information about the robots.txt standard, see:
# http://www.robotstxt.org/robotstxt.html

User-agent: Lucidworks-Anda/2.0
Crawl-delay: 0

User-agent: *
Crawl-delay: 5

# CSS, JS, Images
Allow: */misc/*.css$
Allow: */misc/*.css?
Allow: */misc/*.js$
Allow: */misc/*.js?
Allow: */misc/*.gif
Allow: */misc/*.jpg
Allow: */misc/*.jpeg
Allow: */misc/*.png
Allow: */modules/*.css$
Allow: */modules/*.css?
Allow: */modules/*.js$
Allow: */modules/*.js?
Allow: */modules/*.gif
Allow: */modules/*.jpg
Allow: */modules/*.jpeg
Allow: */modules/*.png
Allow: */profiles/*.css$
Allow: */profiles/*.css?
Allow: */profiles/*.js$
Allow: */profiles/*.js?
Allow: */profiles/*.gif
Allow: */profiles/*.jpg
Allow: */profiles/*.jpeg
Allow: */profiles/*.png
Allow: */themes/*.css$
Allow: */themes/*.css?
Allow: */themes/*.js$
Allow: */themes/*.js?
Allow: */themes/*.gif
Allow: */themes/*.jpg
Allow: */themes/*.jpeg
Allow: */themes/*.png
Allow: */study/2023-2024
# Directories
Disallow: */includes/
Disallow: */modules/
Disallow: */profiles/
Disallow: */scripts/
Disallow: */themes/
Disallow: */study/2009-2010
Disallow: */study/2010-2011
Disallow: */study/2011-2012
Disallow: */study/2012-2013
Disallow: */study/2013-2014
Disallow: */study/2014-2015
Disallow: */study/2015-2016
Disallow: */study/2016-2017
Disallow: */study/2017-2018
Disallow: */study/2018-2019
Disallow: */study/2019-2020
Disallow: */study/2020-2021
Disallow: */study/2021-2022
Disallow: */study/2022-2023
# Files
Disallow: */CHANGELOG.txt
Disallow: */cron.php
Disallow: */INSTALL.mysql.txt
Disallow: */INSTALL.pgsql.txt
Disallow: */INSTALL.sqlite.txt
Disallow: */install.php
Disallow: */INSTALL.txt
Disallow: */LICENSE.txt
Disallow: */MAINTAINERS.txt
Disallow: */update.php
Disallow: */UPGRADE.txt
Disallow: */xmlrpc.php
Disallow: */misc/favicon.ico
# Paths (clean URLs)
Disallow: */admin/
Disallow: */comment/reply/
Disallow: */filter/tips/
Disallow: */node/add/
Disallow: */search/
Disallow: /study/*/courses/search
Disallow: /study/*/programs/search
Disallow: /study/*/search/all
Disallow: */user/register/
Disallow: */user/password/
Disallow: */user/login/
Disallow: */user/logout/
Disallow: */user
# Paths (no clean URLs)
Disallow: */?q=admin/
Disallow: */?q=comment/reply/
Disallow: */?q=filter/tips/
Disallow: */?q=node/add/
Disallow: */?q=search/
Disallow: /?q=study/*/courses/search
Disallow: /?q=study/*/programs/search
Disallow: /?q=study/*/search/all
Disallow: */?q=user/password/
Disallow: */?q=user/register/
Disallow: */?q=user/login/
Disallow: */?q=user/logout/


#
# robots.txt
#
# This file is to prevent the crawling and indexing of certain parts
# of your site by web crawlers and spiders run by sites like Yahoo!
# and Google. By telling these "robots" where not to go on your site,
# you save bandwidth and server resources.
#
# This file will be ignored unless it is at the root of your host:
# Used:    http://example.com/robots.txt
# Ignored: http://example.com/site/robots.txt
#
# For more information about the robots.txt standard, see:
# http://www.robotstxt.org/robotstxt.html

User-agent: Lucidworks-Anda/2.0
Crawl-delay: 0

User-agent: *
Crawl-delay: 5

# CSS, JS, Images
Allow: */misc/*.css$
Allow: */misc/*.css?
Allow: */misc/*.js$
Allow: */misc/*.js?
Allow: */misc/*.gif
Allow: */misc/*.jpg
Allow: */misc/*.jpeg
Allow: */misc/*.png
Allow: */modules/*.css$
Allow: */modules/*.css?
Allow: */modules/*.js$
Allow: */modules/*.js?
Allow: */modules/*.gif
Allow: */modules/*.jpg
Allow: */modules/*.jpeg
Allow: */modules/*.png
Allow: */profiles/*.css$
Allow: */profiles/*.css?
Allow: */profiles/*.js$
Allow: */profiles/*.js?
Allow: */profiles/*.gif
Allow: */profiles/*.jpg
Allow: */profiles/*.jpeg
Allow: */profiles/*.png
Allow: */themes/*.css$
Allow: */themes/*.css?
Allow: */themes/*.js$
Allow: */themes/*.js?
Allow: */themes/*.gif
Allow: */themes/*.jpg
Allow: */themes/*.jpeg
Allow: */themes/*.png
Allow: */study/2024-2025
# Directories
Disallow: */includes/
Disallow: */modules/
Disallow: */profiles/
Disallow: */scripts/
Disallow: */themes/
Disallow: */study/2009-2010
Disallow: */study/2010-2011
Disallow: */study/2011-2012
Disallow: */study/2012-2013
Disallow: */study/2013-2014
Disallow: */study/2014-2015
Disallow: */study/2015-2016
Disallow: */study/2016-2017
Disallow: */study/2017-2018
Disallow: */study/2018-2019
Disallow: */study/2019-2020
Disallow: */study/2020-2021
Disallow: */study/2021-2022
Disallow: */study/2022-2023
Disallow: */study/2023-2024
# Files
Disallow: */CHANGELOG.txt
Disallow: */cron.php
Disallow: */INSTALL.mysql.txt
Disallow: */INSTALL.pgsql.txt
Disallow: */INSTALL.sqlite.txt
Disallow: */install.php
Disallow: */INSTALL.txt
Disallow: */LICENSE.txt
Disallow: */MAINTAINERS.txt
Disallow: */update.php
Disallow: */UPGRADE.txt
Disallow: */xmlrpc.php
Disallow: */misc/favicon.ico
# Paths (clean URLs)
Disallow: */admin/
Disallow: */comment/reply/
Disallow: */filter/tips/
Disallow: */node/add/
Disallow: */search/
Disallow: /study/*/courses/search
Disallow: /study/*/programs/search
Disallow: /study/*/search/all
Disallow: */user/register/
Disallow: */user/password/
Disallow: */user/login/
Disallow: */user/logout/
Disallow: */user
# Paths (no clean URLs)
Disallow: */?q=admin/
Disallow: */?q=comment/reply/
Disallow: */?q=filter/tips/
Disallow: */?q=node/add/
Disallow: */?q=search/
Disallow: /?q=study/*/courses/search
Disallow: /?q=study/*/programs/search
Disallow: /?q=study/*/search/all
Disallow: */?q=user/password/
Disallow: */?q=user/register/
Disallow: */?q=user/login/
Disallow: */?q=user/logout/


#
# robots.txt
#
# This file is to prevent the crawling and indexing of certain parts
# of your site by web crawlers and spiders run by sites like Yahoo!
# and Google. By telling these "robots" where not to go on your site,
# you save bandwidth and server resources.
#
# This file will be ignored unless it is at the root of your host:
# Used:    http://example.com/robots.txt
# Ignored: http://example.com/site/robots.txt
#
# For more information about the robots.txt standard, see:
# http://www.robotstxt.org/robotstxt.html

User-agent: Lucidworks-Anda/2.0
Crawl-delay: 0

User-agent: *
Crawl-delay: 5

# CSS, JS, Images
Allow: */misc/*.css$
Allow: */misc/*.css?
Allow: */misc/*.js$
Allow: */misc/*.js?
Allow: */misc/*.gif
Allow: */misc/*.jpg
Allow: */misc/*.jpeg
Allow: */misc/*.png
Allow: */modules/*.css$
Allow: */modules/*.css?
Allow: */modules/*.js$
Allow: */modules/*.js?
Allow: */modules/*.gif
Allow: */modules/*.jpg
Allow: */modules/*.jpeg
Allow: */modules/*.png
Allow: */profiles/*.css$
Allow: */profiles/*.css?
Allow: */profiles/*.js$
Allow: */profiles/*.js?
Allow: */profiles/*.gif
Allow: */profiles/*.jpg
Allow: */profiles/*.jpeg
Allow: */profiles/*.png
Allow: */themes/*.css$
Allow: */themes/*.css?
Allow: */themes/*.js$
Allow: */themes/*.js?
Allow: */themes/*.gif
Allow: */themes/*.jpg
Allow: */themes/*.jpeg
Allow: */themes/*.png
# Directories
Disallow: */includes/
Disallow: */modules/
Disallow: */profiles/
Disallow: */scripts/
Disallow: */themes/
Disallow: */study/2009-2010
Disallow: */study/2010-2011
Disallow: */study/2011-2012
Disallow: */study/2012-2013
Disallow: */study/2013-2014
Disallow: */study/2014-2015
Disallow: */study/2015-2016
Disallow: */study/2016-2017
Disallow: */study/2017-2018
Disallow: */study/2018-2019
Disallow: */study/2019-2020
Disallow: */study/2020-2021
Disallow: */study/2021-2022
Disallow: */study/2022-2023
Disallow: */study/2023-2024
Disallow: */study/2024-2025
# Files
Disallow: */CHANGELOG.txt
Disallow: */cron.php
Disallow: */INSTALL.mysql.txt
Disallow: */INSTALL.pgsql.txt
Disallow: */INSTALL.sqlite.txt
Disallow: */install.php
Disallow: */INSTALL.txt
Disallow: */LICENSE.txt
Disallow: */MAINTAINERS.txt
Disallow: */update.php
Disallow: */UPGRADE.txt
Disallow: */xmlrpc.php
Disallow: */misc/favicon.ico
# Paths (clean URLs)
Disallow: */admin/
Disallow: */comment/reply/
Disallow: */filter/tips/
Disallow: */node/add/
Disallow: */search/
Disallow: /study/*/courses/search
Disallow: /study/*/programs/search
Disallow: /study/*/search/all
Disallow: */user/register/
Disallow: */user/password/
Disallow: */user/login/
Disallow: */user/logout/
Disallow: */user
# Paths (no clean URLs)
Disallow: */?q=admin/
Disallow: */?q=comment/reply/
Disallow: */?q=filter/tips/
Disallow: */?q=node/add/
Disallow: */?q=search/
Disallow: /?q=study/*/courses/search
Disallow: /?q=study/*/programs/search
Disallow: /?q=study/*/search/all
Disallow: */?q=user/password/
Disallow: */?q=user/register/
Disallow: */?q=user/login/
Disallow: */?q=user/logout/


#
# robots.txt
#
# This file is to prevent the crawling and indexing of certain parts
# of your site by web crawlers and spiders run by sites like Yahoo!
# and Google. By telling these "robots" where not to go on your site,
# you save bandwidth and server resources.
#
# This file will be ignored unless it is at the root of your host:
# Used:    http://example.com/robots.txt
# Ignored: http://example.com/site/robots.txt
#
# For more information about the robots.txt standard, see:
# http://www.robotstxt.org/robotstxt.html

User-agent: Lucidworks-Anda/2.0
Crawl-delay: 0

User-agent: *
Crawl-delay: 5

# CSS, JS, Images
Allow: */misc/*.css$
Allow: */misc/*.css?
Allow: */misc/*.js$
Allow: */misc/*.js?
Allow: */misc/*.gif
Allow: */misc/*.jpg
Allow: */misc/*.jpeg
Allow: */misc/*.png
Allow: */modules/*.css$
Allow: */modules/*.css?
Allow: */modules/*.js$
Allow: */modules/*.js?
Allow: */modules/*.gif
Allow: */modules/*.jpg
Allow: */modules/*.jpeg
Allow: */modules/*.png
Allow: */profiles/*.css$
Allow: */profiles/*.css?
Allow: */profiles/*.js$
Allow: */profiles/*.js?
Allow: */profiles/*.gif
Allow: */profiles/*.jpg
Allow: */profiles/*.jpeg
Allow: */profiles/*.png
Allow: */themes/*.css$
Allow: */themes/*.css?
Allow: */themes/*.js$
Allow: */themes/*.js?
Allow: */themes/*.gif
Allow: */themes/*.jpg
Allow: */themes/*.jpeg
Allow: */themes/*.png
# Directories
Disallow: */includes/
Disallow: */modules/
Disallow: */profiles/
Disallow: */scripts/
Disallow: */themes/
Disallow: */study/2009-2010
Disallow: */study/2010-2011
Disallow: */study/2011-2012
Disallow: */study/2012-2013
Disallow: */study/2013-2014
Disallow: */study/2014-2015
Disallow: */study/2015-2016
Disallow: */study/2016-2017
Disallow: */study/2017-2018
Disallow: */study/2018-2019
Disallow: */study/2019-2020
Disallow: */study/2020-2021
Disallow: */study/2021-2022
Disallow: */study/2022-2023
Disallow: */study/2023-2024
Disallow: */study/2024-2025
# Files
Disallow: */CHANGELOG.txt
Disallow: */cron.php
Disallow: */INSTALL.mysql.txt
Disallow: */INSTALL.pgsql.txt
Disallow: */INSTALL.sqlite.txt
Disallow: */install.php
Disallow: */INSTALL.txt
Disallow: */LICENSE.txt
Disallow: */MAINTAINERS.txt
Disallow: */update.php
Disallow: */UPGRADE.txt
Disallow: */xmlrpc.php
Disallow: */misc/favicon.ico
# Paths (clean URLs)
Disallow: */admin/
Disallow: */comment/reply/
Disallow: */filter/tips/
Disallow: */node/add/
Disallow: */search/
Disallow: /*/people/*
Disallow: /*/events/*
Disallow: /undergraduate-admissions/programs?*
Disallow: /gradapplicants/programs?*
Disallow: /study/*/courses/search
Disallow: /study/*/programs/search
Disallow: /study/*/search/all
Disallow: */user/register/
Disallow: */user/password/
Disallow: */user/login/
Disallow: */user/logout/
Disallow: */user
# Paths (no clean URLs)
Disallow: */?q=admin/
Disallow: */?q=comment/reply/
Disallow: */?q=filter/tips/
Disallow: */?q=node/add/
Disallow: */?q=search/
Disallow: /?q=study/*/courses/search
Disallow: /?q=study/*/programs/search
Disallow: /?q=study/*/search/all
Disallow: */?q=user/password/
Disallow: */?q=user/register/
Disallow: */?q=user/login/
Disallow: */?q=user/logout/


#
# robots.txt
#
# This file is to prevent the crawling and indexing of certain parts
# of your site by web crawlers and spiders run by sites like Yahoo!
# and Google. By telling these "robots" where not to go on your site,
# you save bandwidth and server resources.
#
# This file will be ignored unless it is at the root of your host:
# Used:    http://example.com/robots.txt
# Ignored: http://example.com/site/robots.txt
#
# For more information about the robots.txt standard, see:
# http://www.robotstxt.org/robotstxt.html

Sitemap: https://www.mcgill.ca/root/sitemap-index.xml
Sitemap: https://www.mcgill.ca/sitemap.xml

User-agent: Lucidworks-Anda/2.0
Crawl-delay: 0

User-agent: Elastic-Crawler
Crawl-delay: 0

User-agent: *
Crawl-delay: 5

# CSS, JS, Images
Allow: */misc/*.css$
Allow: */misc/*.css?
Allow: */misc/*.js$
Allow: */misc/*.js?
Allow: */misc/*.gif
Allow: */misc/*.jpg
Allow: */misc/*.jpeg
Allow: */misc/*.png
Allow: */modules/*.css$
Allow: */modules/*.css?
Allow: */modules/*.js$
Allow: */modules/*.js?
Allow: */modules/*.gif
Allow: */modules/*.jpg
Allow: */modules/*.jpeg
Allow: */modules/*.png
Allow: */profiles/*.css$
Allow: */profiles/*.css?
Allow: */profiles/*.js$
Allow: */profiles/*.js?
Allow: */profiles/*.gif
Allow: */profiles/*.jpg
Allow: */profiles/*.jpeg
Allow: */profiles/*.png
Allow: */themes/*.css$
Allow: */themes/*.css?
Allow: */themes/*.js$
Allow: */themes/*.js?
Allow: */themes/*.gif
Allow: */themes/*.jpg
Allow: */themes/*.jpeg
Allow: */themes/*.png
# Directories
Disallow: */includes/
Disallow: */modules/
Disallow: */profiles/
Disallow: */scripts/
Disallow: */themes/
Disallow: */study/2009-2010
Disallow: */study/2010-2011
Disallow: */study/2011-2012
Disallow: */study/2012-2013
Disallow: */study/2013-2014
Disallow: */study/2014-2015
Disallow: */study/2015-2016
Disallow: */study/2016-2017
Disallow: */study/2017-2018
Disallow: */study/2018-2019
Disallow: */study/2019-2020
Disallow: */study/2020-2021
Disallow: */study/2021-2022
Disallow: */study/2022-2023
Disallow: */study/2023-2024
Disallow: */study/2024-2025
# Files
Disallow: */CHANGELOG.txt
Disallow: */cron.php
Disallow: */INSTALL.mysql.txt
Disallow: */INSTALL.pgsql.txt
Disallow: */INSTALL.sqlite.txt
Disallow: */install.php
Disallow: */INSTALL.txt
Disallow: */LICENSE.txt
Disallow: */MAINTAINERS.txt
Disallow: */update.php
Disallow: */UPGRADE.txt
Disallow: */xmlrpc.php
Disallow: */misc/favicon.ico
# Paths (clean URLs)
Disallow: */admin/
Disallow: */comment/reply/
Disallow: */filter/tips/
Disallow: */node/add/
Disallow: */search/
Disallow: /*/people/*
Disallow: /*/events/*
Disallow: /undergraduate-admissions/programs?*
Disallow: /gradapplicants/programs?*
Disallow: /study/*/courses/search
Disallow: /study/*/programs/search
Disallow: /study/*/search/all
Disallow: */user/register/
Disallow: */user/password/
Disallow: */user/login/
Disallow: */user/logout/
Disallow: */user
# Paths (no clean URLs)
Disallow: */?q=admin/
Disallow: */?q=comment/reply/
Disallow: */?q=filter/tips/
Disallow: */?q=node/add/
Disallow: */?q=search/
Disallow: /?q=study/*/courses/search
Disallow: /?q=study/*/programs/search
Disallow: /?q=study/*/search/all
Disallow: */?q=user/password/
Disallow: */?q=user/register/
Disallow: */?q=user/login/
Disallow: */?q=user/logout/
Disallow: /*.zip$
Disallow: /*.gif$
Disallow: /*.jpg$
Disallow: /*.jpeg$
Disallow: /*.png$
Disallow: /*.tif$
Disallow: /*.tiff$
Disallow: /*.dll$
Disallow: /*.exe$
Disallow: /*.class$
Disallow: /*.wmv$
Disallow: /*.m4v$
Disallow: /*.jar$
Disallow: /*.gz$
Disallow: /*.tar$
Disallow: /*.css$
Disallow: /*.inc$
Disallow: /*.js$
Disallow: /*.js.php$
Disallow: /*.swf$
Disallow: /*.fla$
Disallow: /*.psd$
Disallow: /*.m4a$
Disallow: /*.m4p$
Disallow: /*.aac$
Disallow: /*.m2a$
Disallow: /*.m2v$
Disallow: /*.sit$
Disallow: /*.dmg$
Disallow: /*.wma$
Disallow: /*.mdb$
Disallow: /*.tar.gz2$
Disallow: /*.rar$


#
# robots.txt
#
# This file is to prevent the crawling and indexing of certain parts
# of your site by web crawlers and spiders run by sites like Yahoo!
# and Google. By telling these "robots" where not to go on your site,
# you save bandwidth and server resources.
#
# This file will be ignored unless it is at the root of your host:
# Used:    http://example.com/robots.txt
# Ignored: http://example.com/site/robots.txt
#
# For more information about the robots.txt standard, see:
# http://www.robotstxt.org/robotstxt.html

Sitemap: https://www.mcgill.ca/root/sitemap-index.xml
Sitemap: https://www.mcgill.ca/sitemap.xml

User-agent: Lucidworks-Anda/2.0
Crawl-delay: 0

User-agent: Elastic-Crawler
Crawl-delay: 0

User-agent: *
Crawl-delay: 5

User-agent: archive.org_bot
Allow: /study/*

# CSS, JS, Images
Allow: */misc/*.css$
Allow: */misc/*.css?
Allow: */misc/*.js$
Allow: */misc/*.js?
Allow: */misc/*.gif
Allow: */misc/*.jpg
Allow: */misc/*.jpeg
Allow: */misc/*.png
Allow: */modules/*.css$
Allow: */modules/*.css?
Allow: */modules/*.js$
Allow: */modules/*.js?
Allow: */modules/*.gif
Allow: */modules/*.jpg
Allow: */modules/*.jpeg
Allow: */modules/*.png
Allow: */profiles/*.css$
Allow: */profiles/*.css?
Allow: */profiles/*.js$
Allow: */profiles/*.js?
Allow: */profiles/*.gif
Allow: */profiles/*.jpg
Allow: */profiles/*.jpeg
Allow: */profiles/*.png
Allow: */themes/*.css$
Allow: */themes/*.css?
Allow: */themes/*.js$
Allow: */themes/*.js?
Allow: */themes/*.gif
Allow: */themes/*.jpg
Allow: */themes/*.jpeg
Allow: */themes/*.png
# Directories
Disallow: */includes/
Disallow: */modules/
Disallow: */profiles/
Disallow: */scripts/
Disallow: */themes/
# eCals
Disallow: /study/*
# Files
Disallow: */CHANGELOG.txt
Disallow: */cron.php
Disallow: */INSTALL.mysql.txt
Disallow: */INSTALL.pgsql.txt
Disallow: */INSTALL.sqlite.txt
Disallow: */install.php
Disallow: */INSTALL.txt
Disallow: */LICENSE.txt
Disallow: */MAINTAINERS.txt
Disallow: */update.php
Disallow: */UPGRADE.txt
Disallow: */xmlrpc.php
Disallow: */misc/favicon.ico
# Paths (clean URLs)
Disallow: */admin/
Disallow: */comment/reply/
Disallow: */filter/tips/
Disallow: */node/add/
Disallow: */search/
Disallow: /*/people/*
Disallow: /*/events/*
Disallow: /undergraduate-admissions/programs?*
Disallow: /gradapplicants/programs?*
Disallow: */user/register/
Disallow: */user/password/
Disallow: */user/login/
Disallow: */user/logout/
Disallow: */user
# Paths (no clean URLs)
Disallow: */?q=admin/
Disallow: */?q=comment/reply/
Disallow: */?q=filter/tips/
Disallow: */?q=node/add/
Disallow: */?q=search/
Disallow: /?q=study/*/courses/search
Disallow: /?q=study/*/programs/search
Disallow: /?q=study/*/search/all
Disallow: */?q=user/password/
Disallow: */?q=user/register/
Disallow: */?q=user/login/
Disallow: */?q=user/logout/
Disallow: /*.zip$
Disallow: /*.gif$
Disallow: /*.jpg$
Disallow: /*.jpeg$
Disallow: /*.png$
Disallow: /*.tif$
Disallow: /*.tiff$
Disallow: /*.dll$
Disallow: /*.exe$
Disallow: /*.class$
Disallow: /*.wmv$
Disallow: /*.m4v$
Disallow: /*.jar$
Disallow: /*.gz$
Disallow: /*.tar$
Disallow: /*.css$
Disallow: /*.inc$
Disallow: /*.js$
Disallow: /*.js.php$
Disallow: /*.swf$
Disallow: /*.fla$
Disallow: /*.psd$
Disallow: /*.m4a$
Disallow: /*.m4p$
Disallow: /*.aac$
Disallow: /*.m2a$
Disallow: /*.m2v$
Disallow: /*.sit$
Disallow: /*.dmg$
Disallow: /*.wma$
Disallow: /*.mdb$
Disallow: /*.tar.gz2$
Disallow: /*.rar$


User-agent: Mediapartners-Google*
Disallow:

User-agent: Nutch
Crawl-delay: 5
Disallow:

User-agent: Slurp
Disallow: /*.gif$
Disallow: /*.jpg$

User-agent: *
Disallow: /linkfwd.php
Disallow: /counters.php

# Sitemaps
Sitemap: https://www.medicalnewstoday.com/sitemap.xml


User-agent: Mediapartners-Google*
Disallow:

User-agent: Nutch
Crawl-delay: 5
Disallow:

User-agent: Slurp
Disallow: /*.gif$
Disallow: /*.jpg$

User-agent: *
Crawl-delay: 5
Disallow: /linkfwd.php
Disallow: /counters.php

# Sitemaps
Sitemap: https://www.medicalnewstoday.com/sitemap.xml


User-agent: Mediapartners-Google*
Disallow:

User-agent: Nutch
Crawl-delay: 5
Disallow:

User-agent: Slurp
Disallow: /*.gif$
Disallow: /*.jpg$

User-agent: *
Crawl-delay: 5
Disallow: /linkfwd.php
Disallow: /counters.php

# Wordpress Previews
Disallow: /articles/mnt-*

# Sitemaps
Sitemap: https://www.medicalnewstoday.com/sitemap.xml


User-agent: Mediapartners-Google*
Disallow:

User-agent: Nutch
Crawl-delay: 5
Disallow:

User-agent: Slurp
Disallow: /*.gif$
Disallow: /*.jpg$

User-agent: *
Crawl-delay: 5
Disallow: /linkfwd.php
Disallow: /counters.php

# Wordpress Previews
Disallow: /articles/mnt-*

# Infinite Scroll API
Disallow: /api/content/

# Sitemaps
Sitemap: https://www.medicalnewstoday.com/sitemap.xml


User-agent: Mediapartners-Google*
Disallow:

User-agent: Nutch
Crawl-delay: 5
Disallow:

User-agent: Slurp
Disallow: /*.gif$
Disallow: /*.jpg$

User-agent: *
Crawl-delay: 5
Disallow: /linkfwd.php
Disallow: /counters.php

# Wordpress Previews
Disallow: /articles/mnt-*

# Infinite Scroll API
Disallow: /api/content/

User-agent: GPTBot
Disallow: /

# Sitemaps
Sitemap: https://www.medicalnewstoday.com/sitemap.xml


User-agent: Mediapartners-Google*
Disallow:

User-agent: Nutch
Crawl-delay: 5
Disallow:

User-agent: Slurp
Disallow: /*.gif$
Disallow: /*.jpg$

User-agent: *
Crawl-delay: 5
Disallow: /linkfwd.php
Disallow: /counters.php

# Wordpress Previews
Disallow: /articles/mnt-*
Disallow: /program/mnt-*

# Infinite Scroll API
Disallow: /api/content/

User-agent: GPTBot
Disallow: /

# Sitemaps
Sitemap: https://www.medicalnewstoday.com/sitemap.xml


User-agent: Mediapartners-Google*
Disallow:

User-agent: Nutch
Crawl-delay: 5
Disallow:

User-agent: Slurp
Disallow: /*.gif$
Disallow: /*.jpg$

User-agent: *
Crawl-delay: 5
Disallow: /linkfwd.php
Disallow: /counters.php

# Wordpress Previews
Disallow: /articles/mnt-*
Disallow: /program/mnt-*

# Infinite Scroll API
Disallow: /api/content/

User-agent: GPTBot
Disallow: /

# Sitemaps
Sitemap: https://www.medicalnewstoday.com/sitemap.xml

# Widget Sampler
Disallow: /articles/widget-sampler

# Static Test Articles
Disallow: /test/


User-agent: Mediapartners-Google*
Disallow:

User-agent: Nutch
Crawl-delay: 5
Disallow:

User-agent: Slurp
Disallow: /*.gif$
Disallow: /*.jpg$

User-agent: *
Crawl-delay: 5
Disallow: /linkfwd.php
Disallow: /counters.php

# Wordpress Previews
Disallow: /articles/mnt-*
Disallow: /program/mnt-*

# API Routes
Disallow: /api/*

# Invalid URLs
Disallow: */null$
Disallow: */inline$

User-agent: GPTBot
Disallow: /

# Sitemaps
Sitemap: https://www.medicalnewstoday.com/sitemap.xml

# Widget Sampler
Disallow: /articles/widget-sampler

# Static Test Articles
Disallow: /test/


User-agent: Mediapartners-Google*
Disallow:

User-agent: Nutch
Crawl-delay: 5
Disallow:

User-agent: Slurp
Disallow: /*.gif$
Disallow: /*.jpg$

User-agent: *
Crawl-delay: 5
Disallow: /linkfwd.php
Disallow: /counters.php

# Wordpress Previews
Disallow: /articles/mnt-*
Disallow: /program/mnt-*

# API Routes
Disallow: /api/*

# Invalid URLs
Disallow: */null$
Disallow: */inline$

User-agent: GPTBot
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: FacebookBot
Disallow: /

User-agent: Meta-ExternalAgent
Disallow: /

User-agent: omgili
Disallow: /

User-agent: Timpibot
Disallow: /

User-agent: Webzio-Extended
Disallow: /

# Sitemaps
Sitemap: https://www.medicalnewstoday.com/sitemap.xml

# Widget Sampler
Disallow: /articles/widget-sampler

# Static Test Articles
Disallow: /test/


User-agent: Mediapartners-Google*
Disallow:

User-agent: Nutch
Crawl-delay: 5
Disallow:

User-agent: Slurp
Disallow: /*.gif$
Disallow: /*.jpg$

User-agent: *
Crawl-delay: 5
Disallow: /linkfwd.php
Disallow: /counters.php

# Wordpress Previews
Disallow: /articles/mnt-*
Disallow: /program/mnt-*

# API Routes
Disallow: /api/*

# Invalid URLs
Disallow: */null$
Disallow: */inline$

User-agent: GPTBot
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: FacebookBot
Disallow: /

User-agent: Meta-ExternalAgent
Disallow: /

User-agent: omgili
Disallow: /

User-agent: Timpibot
Disallow: /

User-agent: Webzio-Extended
Disallow: /

# Sitemaps
Sitemap: https://www.medicalnewstoday.com/sitemap.xml

# Widget Sampler
Disallow: /articles/widget-sampler

# Static Test Articles
Disallow: /test/

# Block AMP URLs
Disallow: /amp/


User-agent: Mediapartners-Google*
Disallow:

User-agent: Nutch
Crawl-delay: 5
Disallow:

User-agent: Slurp
Disallow: /*.gif$
Disallow: /*.jpg$

User-agent: *
Crawl-delay: 5
Disallow: /linkfwd.php
Disallow: /counters.php

# Wordpress Previews
Disallow: /articles/mnt-*
Disallow: /program/mnt-*

# API Routes
Disallow: /api/*

# Invalid URLs
Disallow: */null$
Disallow: */inline$

User-agent: GPTBot
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: FacebookBot
Disallow: /

User-agent: Meta-ExternalAgent
Disallow: /

User-agent: omgili
Disallow: /

User-agent: Timpibot
Disallow: /

User-agent: Webzio-Extended
Disallow: /

User-agent: DDM-DCipher/1.0.7
Disallow: /

# Sitemaps
Sitemap: https://www.medicalnewstoday.com/sitemap.xml

# Widget Sampler
Disallow: /articles/widget-sampler

# Static Test Articles
Disallow: /test/

# Block AMP URLs
Disallow: /amp/


User-agent: Mediapartners-Google*
Disallow:

User-agent: Nutch
Crawl-delay: 5
Disallow:

User-agent: Slurp
Disallow: /*.gif$
Disallow: /*.jpg$

User-agent: Amazonbot
Disallow:

User-agent: *
Crawl-delay: 5
Disallow: /linkfwd.php
Disallow: /counters.php

# Wordpress Previews
Disallow: /articles/mnt-*
Disallow: /program/mnt-*

# API Routes
Disallow: /api/*

# Invalid URLs
Disallow: */null$
Disallow: */inline$

User-agent: GPTBot
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: FacebookBot
Disallow: /

User-agent: Meta-ExternalAgent
Disallow: /

User-agent: omgili
Disallow: /

User-agent: Timpibot
Disallow: /

User-agent: Webzio-Extended
Disallow: /

User-agent: DDM-DCipher/1.0.7
Disallow: /

# Sitemaps
Sitemap: https://www.medicalnewstoday.com/sitemap.xml

# Widget Sampler
Disallow: /articles/widget-sampler

# Static Test Articles
Disallow: /test/

# Block AMP URLs
Disallow: /amp/


User-agent: Mediapartners-Google*
Disallow:

User-agent: Nutch
Crawl-delay: 5
Disallow:

User-agent: Slurp
Disallow: /*.gif$
Disallow: /*.jpg$

# --- Amazon / AWS crawlers (begin) ---

# AmazonAdbot is intentionally allowed (ad targeting — see HLMR-6504)
User-agent: AmazonAdbot
Disallow:

User-agent: Amazonbot
Disallow: /

User-agent: Amzn-SearchBot
Disallow: /

User-agent: Amzn-User
Disallow: /

User-agent: amazon-kendra
Disallow: /

User-agent: amazon-QBusiness
Disallow: /

User-agent: bedrockbot
Disallow: /

User-agent: aws-quick-on-behalf-of-
Disallow: /

User-agent: AmazonProductDiscoverybot
Disallow: /

User-agent: AmazonBuyForMe
Disallow: /

User-agent: AmazonSellerInitiatedListing
Disallow: /

User-agent: NovaAct
Disallow: /

# --- Amazon / AWS crawlers (end) ---

User-agent: *
Crawl-delay: 5
Disallow: /linkfwd.php
Disallow: /counters.php

# Wordpress Previews
Disallow: /articles/mnt-*
Disallow: /program/mnt-*

# API Routes
Disallow: /api/*

# Invalid URLs
Disallow: */null$
Disallow: */inline$

User-agent: GPTBot
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: FacebookBot
Disallow: /

User-agent: Meta-ExternalAgent
Disallow: /

User-agent: omgili
Disallow: /

User-agent: Timpibot
Disallow: /

User-agent: Webzio-Extended
Disallow: /

User-agent: DDM-DCipher/1.0.7
Disallow: /

# Sitemaps
Sitemap: https://www.medicalnewstoday.com/sitemap.xml

# Widget Sampler
Disallow: /articles/widget-sampler

# Static Test Articles
Disallow: /test/

# Block AMP URLs
Disallow: /amp/


User-agent: Swiftbot
Disallow:
Disallow: /blog/facet/Authors/
Disallow: /blog/facet/Categories/
Disallow: /blog/facet/Tags/
Disallow: /blog/facet/Date/
Disallow: /about-us/press-office/news/corporate/facet/Topic/
Disallow: /about-us/press-office/news/corporate/facet/Year/
Disallow: /about-us/press-office/news/weather-and-climate/facet/Topic/
Disallow: /about-us/press-office/news/weather-and-climate/facet/Year/
User-agent: *
Disallow: /alive/
Disallow: /brand/
Disallow: /conference/
Disallow: /images/
Disallow: /lib/
Disallow: /public/data/
Disallow: /public/weather/lib/
Disallow: /eurocontrol/
Disallow: /magister/
Disallow: /premium/
Disallow: /weather/africa/lam/
Disallow: /alive/
Disallow: /images/
Disallow: /lib/
Disallow: /staff
Disallow: /weather/test/
Disallow: /blog/facet/Authors/
Disallow: /blog/facet/Categories/
Disallow: /blog/facet/Tags/
Disallow: /blog/facet/Date/
Disallow: /about-us/press-office/news/corporate/facet/Topic/
Disallow: /about-us/press-office/news/corporate/facet/Year/
Disallow: /about-us/press-office/news/weather-and-climate/facet/Topic/
Disallow: /about-us/press-office/news/weather-and-climate/facet/Year/
Sitemap: https://www.metoffice.gov.uk/sitemap.xml
Sitemap: https://www.metoffice.gov.uk/sitemap-domestic-forecast.xml
Sitemap: https://www.metoffice.gov.uk/sitemap-international-forecast.xml
Sitemap: https://www.metoffice.gov.uk/sitemap-international-hub.xml
Sitemap: https://www.metoffice.gov.uk/sitemap-regional-hub.xml


User-agent: Swiftbot
Disallow:
Disallow: /blog/facet/Authors/
Disallow: /blog/facet/Categories/
Disallow: /blog/facet/Tags/
Disallow: /blog/facet/Date/
Disallow: /about-us/press-office/news/corporate/facet/Topic/
Disallow: /about-us/press-office/news/corporate/facet/Year/
Disallow: /about-us/press-office/news/weather-and-climate/facet/Topic/
Disallow: /about-us/press-office/news/weather-and-climate/facet/Year/
User-agent: *
Disallow: /alive/
Disallow: /brand/
Disallow: /conference/
Disallow: /images/
Disallow: /lib/
Disallow: /public/data/
Disallow: /public/weather/lib/
Disallow: /eurocontrol/
Disallow: /magister/
Disallow: /premium/
Disallow: /weather/africa/lam/
Disallow: /alive/
Disallow: /images/
Disallow: /lib/
Disallow: /staff
Disallow: /weather/test/
Disallow: /blog/facet/Authors/
Disallow: /blog/facet/Categories/
Disallow: /blog/facet/Tags/
Disallow: /blog/facet/Date/
Disallow: /about-us/press-office/news/corporate/facet/Topic/
Disallow: /about-us/press-office/news/corporate/facet/Year/
Disallow: /about-us/press-office/news/weather-and-climate/facet/Topic/
Disallow: /about-us/press-office/news/weather-and-climate/facet/Year/
Sitemap: https://www.metoffice.gov.uk/sitemap.xml
Sitemap: https://www.metoffice.gov.uk/sitemap-domestic-forecast.xml
Sitemap: https://www.metoffice.gov.uk/sitemap-international-forecast.xml
Sitemap: https://www.metoffice.gov.uk/sitemap-international-hub.xml
Sitemap: https://www.metoffice.gov.uk/sitemap-regional-hub.xml
Sitemap: https://www.metoffice.gov.uk/sitemap-international-observation.xml
Sitemap: https://www.metoffice.gov.uk/sitemap-domestic-observation.xml


User-agent: Swiftbot
Disallow:
Disallow: /blog/facet/Authors/
Disallow: /blog/facet/Categories/
Disallow: /blog/facet/Tags/
Disallow: /blog/facet/Date/
Disallow: /about-us/press-office/news/corporate/facet/Topic/
Disallow: /about-us/press-office/news/corporate/facet/Year/
Disallow: /about-us/press-office/news/weather-and-climate/facet/Topic/
Disallow: /about-us/press-office/news/weather-and-climate/facet/Year/
User-agent: *
Disallow: /alive/
Disallow: /brand/
Disallow: /conference/
Disallow: /images/
Disallow: /lib/
Disallow: /public/data/
Disallow: /public/weather/lib/
Disallow: /eurocontrol/
Disallow: /magister/
Disallow: /premium/
Disallow: /weather/africa/lam/
Disallow: /alive/
Disallow: /images/
Disallow: /lib/
Disallow: /staff
Disallow: /weather/test/
Disallow: /blog/facet/Authors/
Disallow: /blog/facet/Categories/
Disallow: /blog/facet/Tags/
Disallow: /blog/facet/Date/
Disallow: /about-us/press-office/news/corporate/facet/Topic/
Disallow: /about-us/press-office/news/corporate/facet/Year/
Disallow: /about-us/press-office/news/weather-and-climate/facet/Topic/
Disallow: /about-us/press-office/news/weather-and-climate/facet/Year/
Sitemap: https://www.metoffice.gov.uk/sitemap.xml
Sitemap: https://www.metoffice.gov.uk/sitemap-domestic-forecast.xml
Sitemap: https://www.metoffice.gov.uk/sitemap-international-forecast.xml
Sitemap: https://www.metoffice.gov.uk/sitemap-international-hub.xml
Sitemap: https://www.metoffice.gov.uk/sitemap-regional-hub.xml
Sitemap: https://www.metoffice.gov.uk/sitemap-international-observation.xml
Sitemap: https://www.metoffice.gov.uk/sitemap-domestic-observation.xml
Sitemap: https://www.metoffice.gov.uk/sitemap-beach.xml


User-agent: Swiftbot
Disallow:
Disallow: /blog/facet/Authors/
Disallow: /blog/facet/Categories/
Disallow: /blog/facet/Tags/
Disallow: /blog/facet/Date/
Disallow: /about-us/press-office/news/corporate/facet/Topic/
Disallow: /about-us/press-office/news/corporate/facet/Year/
Disallow: /about-us/press-office/news/weather-and-climate/facet/Topic/
Disallow: /about-us/press-office/news/weather-and-climate/facet/Year/
User-agent: *
Disallow: /alive/
Disallow: /brand/
Disallow: /conference/
Disallow: /images/
Disallow: /lib/
Disallow: /public/data/
Disallow: /public/weather/lib/
Disallow: /eurocontrol/
Disallow: /magister/
Disallow: /premium/
Disallow: /weather/africa/lam/
Disallow: /alive/
Disallow: /images/
Disallow: /lib/
Disallow: /staff
Disallow: /weather/test/
Disallow: /blog/facet/Authors/
Disallow: /blog/facet/Categories/
Disallow: /blog/facet/Tags/
Disallow: /blog/facet/Date/
Disallow: /about-us/press-office/news/corporate/facet/Topic/
Disallow: /about-us/press-office/news/corporate/facet/Year/
Disallow: /about-us/press-office/news/weather-and-climate/facet/Topic/
Disallow: /about-us/press-office/news/weather-and-climate/facet/Year/
User-agent: proximic
Disallow: /
Disallow: /blog/facet/Authors/
Disallow: /blog/facet/Categories/
Disallow: /blog/facet/Tags/
Disallow: /blog/facet/Date/
Disallow: /about-us/press-office/news/corporate/facet/Topic/
Disallow: /about-us/press-office/news/corporate/facet/Year/
Disallow: /about-us/press-office/news/weather-and-climate/facet/Topic/
Disallow: /about-us/press-office/news/weather-and-climate/facet/Year/
Sitemap: https://www.metoffice.gov.uk/sitemap.xml
Sitemap: https://www.metoffice.gov.uk/sitemap-domestic-forecast.xml
Sitemap: https://www.metoffice.gov.uk/sitemap-international-forecast.xml
Sitemap: https://www.metoffice.gov.uk/sitemap-international-hub.xml
Sitemap: https://www.metoffice.gov.uk/sitemap-regional-hub.xml
Sitemap: https://www.metoffice.gov.uk/sitemap-international-observation.xml
Sitemap: https://www.metoffice.gov.uk/sitemap-domestic-observation.xml
Sitemap: https://www.metoffice.gov.uk/sitemap-beach.xml


User-agent: Swiftbot
Disallow:
Disallow: /about-us/news-and-media/media-centre/corporate-news/facet/Topic/
Disallow: /about-us/news-and-media/media-centre/corporate-news/facet/Year/
Disallow: /about-us/news-and-media/media-centre/weather-and-climate-news/facet/Topic/
Disallow: /about-us/news-and-media/media-centre/weather-and-climate-news/facet/Year/
Disallow: /blog/facet/Authors/
Disallow: /blog/facet/Categories/
Disallow: /blog/facet/Tags/
Disallow: /blog/facet/Date/
Disallow: /about-us/press-office/news/corporate/facet/Topic/
Disallow: /about-us/press-office/news/corporate/facet/Year/
Disallow: /about-us/press-office/news/weather-and-climate/facet/Topic/
Disallow: /about-us/press-office/news/weather-and-climate/facet/Year/
User-agent: *
Disallow: /alive/
Disallow: /brand/
Disallow: /conference/
Disallow: /images/
Disallow: /lib/
Disallow: /public/data/
Disallow: /public/weather/lib/
Disallow: /eurocontrol/
Disallow: /magister/
Disallow: /premium/
Disallow: /weather/africa/lam/
Disallow: /alive/
Disallow: /images/
Disallow: /lib/
Disallow: /staff
Disallow: /weather/test/
Disallow: /about-us/news-and-media/media-centre/corporate-news/facet/Topic/
Disallow: /about-us/news-and-media/media-centre/corporate-news/facet/Year/
Disallow: /about-us/news-and-media/media-centre/weather-and-climate-news/facet/Topic/
Disallow: /about-us/news-and-media/media-centre/weather-and-climate-news/facet/Year/
Disallow: /blog/facet/Authors/
Disallow: /blog/facet/Categories/
Disallow: /blog/facet/Tags/
Disallow: /blog/facet/Date/
Disallow: /about-us/press-office/news/corporate/facet/Topic/
Disallow: /about-us/press-office/news/corporate/facet/Year/
Disallow: /about-us/press-office/news/weather-and-climate/facet/Topic/
Disallow: /about-us/press-office/news/weather-and-climate/facet/Year/
User-agent: proximic
Disallow: /
Disallow: /about-us/news-and-media/media-centre/corporate-news/facet/Topic/
Disallow: /about-us/news-and-media/media-centre/corporate-news/facet/Year/
Disallow: /about-us/news-and-media/media-centre/weather-and-climate-news/facet/Topic/
Disallow: /about-us/news-and-media/media-centre/weather-and-climate-news/facet/Year/
Disallow: /blog/facet/Authors/
Disallow: /blog/facet/Categories/
Disallow: /blog/facet/Tags/
Disallow: /blog/facet/Date/
Disallow: /about-us/press-office/news/corporate/facet/Topic/
Disallow: /about-us/press-office/news/corporate/facet/Year/
Disallow: /about-us/press-office/news/weather-and-climate/facet/Topic/
Disallow: /about-us/press-office/news/weather-and-climate/facet/Year/
Sitemap: https://www.metoffice.gov.uk/sitemap.xml
Sitemap: https://www.metoffice.gov.uk/sitemap-domestic-forecast.xml
Sitemap: https://www.metoffice.gov.uk/sitemap-international-forecast.xml
Sitemap: https://www.metoffice.gov.uk/sitemap-international-hub.xml
Sitemap: https://www.metoffice.gov.uk/sitemap-regional-hub.xml
Sitemap: https://www.metoffice.gov.uk/sitemap-international-observation.xml
Sitemap: https://www.metoffice.gov.uk/sitemap-domestic-observation.xml
Sitemap: https://www.metoffice.gov.uk/sitemap-beach.xml


User-agent: Swiftbot
Disallow:
Disallow: /about-us/news-and-media/media-centre/corporate-news/facet/Topic/
Disallow: /about-us/news-and-media/media-centre/corporate-news/facet/Year/
Disallow: /about-us/news-and-media/media-centre/weather-and-climate-news/facet/Topic/
Disallow: /about-us/news-and-media/media-centre/weather-and-climate-news/facet/Year/
Disallow: /blog/facet/Authors/
Disallow: /blog/facet/Categories/
Disallow: /blog/facet/Tags/
Disallow: /blog/facet/Date/
Disallow: /about-us/press-office/news/corporate/facet/Topic/
Disallow: /about-us/press-office/news/corporate/facet/Year/
Disallow: /about-us/press-office/news/weather-and-climate/facet/Topic/
Disallow: /about-us/press-office/news/weather-and-climate/facet/Year/
User-agent: *
Disallow: /alive/
Disallow: /brand/
Disallow: /conference/
Disallow: /images/
Disallow: /lib/
Disallow: /public/data/
Disallow: /public/weather/lib/
Disallow: /eurocontrol/
Disallow: /magister/
Disallow: /premium/
Disallow: /weather/africa/lam/
Disallow: /alive/
Disallow: /images/
Disallow: /lib/
Disallow: /staff
Disallow: /weather/test/
Disallow: /about-us/news-and-media/media-centre/corporate-news/facet/Topic/
Disallow: /about-us/news-and-media/media-centre/corporate-news/facet/Year/
Disallow: /about-us/news-and-media/media-centre/weather-and-climate-news/facet/Topic/
Disallow: /about-us/news-and-media/media-centre/weather-and-climate-news/facet/Year/
Disallow: /blog/facet/Authors/
Disallow: /blog/facet/Categories/
Disallow: /blog/facet/Tags/
Disallow: /blog/facet/Date/
Disallow: /about-us/press-office/news/corporate/facet/Topic/
Disallow: /about-us/press-office/news/corporate/facet/Year/
Disallow: /about-us/press-office/news/weather-and-climate/facet/Topic/
Disallow: /about-us/press-office/news/weather-and-climate/facet/Year/
User-agent: proximic
Disallow: /
Disallow: /about-us/news-and-media/media-centre/corporate-news/facet/Topic/
Disallow: /about-us/news-and-media/media-centre/corporate-news/facet/Year/
Disallow: /about-us/news-and-media/media-centre/weather-and-climate-news/facet/Topic/
Disallow: /about-us/news-and-media/media-centre/weather-and-climate-news/facet/Year/
Disallow: /blog/facet/Authors/
Disallow: /blog/facet/Categories/
Disallow: /blog/facet/Tags/
Disallow: /blog/facet/Date/
Disallow: /about-us/press-office/news/corporate/facet/Topic/
Disallow: /about-us/press-office/news/corporate/facet/Year/
Disallow: /about-us/press-office/news/weather-and-climate/facet/Topic/
Disallow: /about-us/press-office/news/weather-and-climate/facet/Year/
Sitemap: https://www.metoffice.gov.uk/sitemap.xml
Sitemap: https://www.metoffice.gov.uk/sitemap-international-hub.xml
Sitemap: https://www.metoffice.gov.uk/sitemap-regional-hub.xml
Sitemap: https://www.metoffice.gov.uk/sitemap-international-observation.xml
Sitemap: https://www.metoffice.gov.uk/sitemap-domestic-observation.xml


User-agent: Swiftbot
Disallow:
Disallow: /blog/facet/Authors/
Disallow: /blog/facet/Categories/
Disallow: /blog/facet/Tags/
Disallow: /blog/facet/Date/
Disallow: /about-us/press-office/news/corporate/facet/Topic/
Disallow: /about-us/press-office/news/corporate/facet/Year/
Disallow: /about-us/press-office/news/weather-and-climate/facet/Topic/
Disallow: /about-us/press-office/news/weather-and-climate/facet/Year/
Disallow: /about-us/news-and-media/media-centre/corporate-news/facet/Topic/
Disallow: /about-us/news-and-media/media-centre/corporate-news/facet/Year/
Disallow: /about-us/news-and-media/media-centre/weather-and-climate-news/facet/Topic/
Disallow: /about-us/news-and-media/media-centre/weather-and-climate-news/facet/Year/
User-agent: *
Disallow: /alive/
Disallow: /brand/
Disallow: /conference/
Disallow: /images/
Disallow: /lib/
Disallow: /public/data/
Disallow: /public/weather/lib/
Disallow: /eurocontrol/
Disallow: /magister/
Disallow: /premium/
Disallow: /weather/africa/lam/
Disallow: /alive/
Disallow: /images/
Disallow: /lib/
Disallow: /staff
Disallow: /weather/test/
Disallow: /blog/facet/Authors/
Disallow: /blog/facet/Categories/
Disallow: /blog/facet/Tags/
Disallow: /blog/facet/Date/
Disallow: /about-us/press-office/news/corporate/facet/Topic/
Disallow: /about-us/press-office/news/corporate/facet/Year/
Disallow: /about-us/press-office/news/weather-and-climate/facet/Topic/
Disallow: /about-us/press-office/news/weather-and-climate/facet/Year/
Disallow: /about-us/news-and-media/media-centre/corporate-news/facet/Topic/
Disallow: /about-us/news-and-media/media-centre/corporate-news/facet/Year/
Disallow: /about-us/news-and-media/media-centre/weather-and-climate-news/facet/Topic/
Disallow: /about-us/news-and-media/media-centre/weather-and-climate-news/facet/Year/
User-agent: proximic
Disallow: /
Disallow: /blog/facet/Authors/
Disallow: /blog/facet/Categories/
Disallow: /blog/facet/Tags/
Disallow: /blog/facet/Date/
Disallow: /about-us/press-office/news/corporate/facet/Topic/
Disallow: /about-us/press-office/news/corporate/facet/Year/
Disallow: /about-us/press-office/news/weather-and-climate/facet/Topic/
Disallow: /about-us/press-office/news/weather-and-climate/facet/Year/
Disallow: /about-us/news-and-media/media-centre/corporate-news/facet/Topic/
Disallow: /about-us/news-and-media/media-centre/corporate-news/facet/Year/
Disallow: /about-us/news-and-media/media-centre/weather-and-climate-news/facet/Topic/
Disallow: /about-us/news-and-media/media-centre/weather-and-climate-news/facet/Year/
Sitemap: https://www.metoffice.gov.uk/sitemap.xml
Sitemap: https://www.metoffice.gov.uk/sitemap-international-hub.xml
Sitemap: https://www.metoffice.gov.uk/sitemap-regional-hub.xml
Sitemap: https://www.metoffice.gov.uk/sitemap-international-observation.xml
Sitemap: https://www.metoffice.gov.uk/sitemap-domestic-observation.xml


User-Agent: Swiftbot

User-Agent: *
Disallow: /alive/
Disallow: /brand/
Disallow: /conference/
Disallow: /images/
Disallow: /lib/
Disallow: /public/data/
Disallow: /public/weather/lib/
Disallow: /eurocontrol/
Disallow: /magister/
Disallow: /premium/
Disallow: /weather/africa/lam/
Disallow: /staff
Disallow: /weather/test/

User-Agent: proximic
Disallow: /

Sitemap: https://www.metoffice.gov.uk/sitemap.xml


User-agent: *
Sitemap: https://www.nature.com/sitemap.xml
Sitemap: https://www.nature.com/subjects/sitemap.xml
Sitemap: https://www.nature.com/naturecareers/sitemap.xml
Disallow: /*.pdf$
Disallow: /*.PDF$
Disallow: /*/*/*/pf/
Disallow: /*.otmi$
Disallow: /*/*/*/*/otmi/
Disallow: /*/*/*/*/fp/
Disallow: /protocolexchange/labgroups/
Disallow: /naturecareers/jobs/search
Disallow: /*draft=*
Disallow: /*proof=true*


User-agent: *
Sitemap: https://www.nature.com/sitemap.xml
Sitemap: https://www.nature.com/subjects/sitemap.xml
Sitemap: https://www.nature.com/naturecareers/sitemap.xml
Disallow: /search
Disallow: /*.pdf$
Disallow: /*.PDF$
Disallow: /*/*/*/pf/
Disallow: /*.otmi$
Disallow: /*/*/*/*/otmi/
Disallow: /*/*/*/*/fp/
Disallow: /protocolexchange/labgroups/
Disallow: /naturecareers/jobs/search
Disallow: /*draft=*
Disallow: /*proof=true*


User-agent: *
Sitemap: https://www.nature.com/sitemap.xml
Sitemap: https://www.nature.com/subjects/sitemap.xml
Sitemap: https://www.nature.com/naturecareers/sitemap.xml
Disallow: /search
Disallow: /*.pdf$
Disallow: /*.PDF$
Disallow: /*/*/*/pf/
Disallow: /*.otmi$
Disallow: /*/*/*/*/otmi/
Disallow: /*/*/*/*/fp/
Disallow: /protocolexchange/labgroups/
Disallow: /naturecareers/jobs/search
Disallow: /*draft=*
Disallow: /my-account
Disallow: /*proof=true*


User-agent: *
Sitemap: https://www.nature.com/sitemap.xml
Sitemap: https://www.nature.com/subjects/sitemap.xml
Sitemap: https://www.nature.com/naturecareers/sitemap.xml
Disallow: /search
Disallow: /*.pdf$
Disallow: /*.PDF$
Disallow: /*/*/*/pf/
Disallow: /*.otmi$
Disallow: /*/*/*/*/otmi/
Disallow: /*/*/*/*/fp/
Disallow: /protocolexchange/labgroups/
Disallow: /naturecareers/jobs/search
Disallow: /*draft=*
Disallow: /my-account
Disallow: /*proof=true*
Disallow: /*/figures
Disallow: /*/tables
Disallow: /*/schemes
Disallow: /*/structures
Disallow: /*/metrics


User-agent: *
Sitemap: https://www.nature.com/sitemap.xml
Sitemap: https://www.nature.com/subjects/sitemap.xml
Sitemap: https://www.nature.com/naturecareers/sitemap.xml
Disallow: /search
Disallow: /*.pdf$
Disallow: /*.PDF$
Disallow: /*/*/*/pf/
Disallow: /*.otmi$
Disallow: /*/*/*/*/otmi/
Disallow: /*/*/*/*/fp/
Disallow: /protocolexchange/labgroups/
Disallow: /naturecareers/jobs/search
Disallow: /*draft=*
Disallow: /my-account
Disallow: /*proof=*
Disallow: /*/figures
Disallow: /*/tables
Disallow: /*/schemes
Disallow: /*/structures
Disallow: /*/metrics
Disallow: /platform/contextual


User-agent: *
Sitemap: https://www.nature.com/sitemap.xml
Sitemap: https://www.nature.com/subjects/sitemap.xml
Sitemap: https://www.nature.com/naturecareers/sitemap.xml
Disallow: /search
Disallow: /*.pdf$
Disallow: /*.PDF$
Disallow: /*/*/*/pf/
Disallow: /*.otmi$
Disallow: /*/*/*/*/otmi/
Disallow: /*/*/*/*/fp/
Disallow: /protocolexchange/labgroups/
Disallow: /naturecareers/jobs/search
Disallow: /*draft=*
Disallow: /*foxtrotcallback=*
Disallow: /my-account
Disallow: /*proof=*
Disallow: /*/figures
Disallow: /*/tables
Disallow: /*/schemes
Disallow: /*/structures
Disallow: /*/metrics
Disallow: /platform/contextual


User-agent: *
Sitemap: https://www.nature.com/sitemap.xml
Sitemap: https://www.nature.com/subjects/sitemap.xml
Sitemap: https://www.nature.com/naturecareers/sitemap.xml
Disallow: /search
Disallow: /*.pdf$
Disallow: /*.PDF$
Disallow: /*/*/*/pf/
Disallow: /*.otmi$
Disallow: /*/*/*/*/otmi/
Disallow: /*/*/*/*/fp/
Disallow: /protocolexchange/labgroups/
Disallow: /naturecareers/jobs/search
Disallow: /*draft=*
Disallow: /*foxtrotcallback=*
Disallow: /*utm_source=*
Disallow: /my-account
Disallow: /*proof=*
Disallow: /*/figures
Disallow: /*/tables
Disallow: /*/schemes
Disallow: /*/structures
Disallow: /*/metrics
Disallow: /platform/contextual


User-agent: *
Sitemap: https://www.nature.com/sitemap.xml
Sitemap: https://www.nature.com/subjects/sitemap.xml
Sitemap: https://www.nature.com/naturecareers/sitemap.xml
Disallow: /search
Disallow: /*.pdf$
Disallow: /*.PDF$
Disallow: /*/*/*/pf/
Disallow: /*.otmi$
Disallow: /*/*/*/*/otmi/
Disallow: /*/*/*/*/fp/
Disallow: /protocolexchange/labgroups/
Disallow: /naturecareers/jobs/search
Disallow: /*draft=*
Disallow: /*foxtrotcallback=*
Disallow: /*utm_source=*
Disallow: /*origin=*
Disallow: /my-account
Disallow: /*proof=*
Disallow: /*/figures
Disallow: /*/tables
Disallow: /*/schemes
Disallow: /*/structures
Disallow: /*/metrics
Disallow: /platform/contextual


User-agent: *
Sitemap: https://www.nature.com/sitemap.xml
Sitemap: https://www.nature.com/subjects/sitemap.xml
Sitemap: https://www.nature.com/naturecareers/sitemap.xml
Disallow: /search
Disallow: /*.pdf$
Disallow: /*.PDF$
Disallow: /*/*/*/pf/
Disallow: /*.otmi$
Disallow: /*/*/*/*/otmi/
Disallow: /*/*/*/*/fp/
Disallow: /protocolexchange/labgroups/
Disallow: /naturecareers/jobs/search
Disallow: /webcasts/*
Disallow: /*draft=*
Disallow: /*foxtrotcallback=*
Disallow: /*utm_source=*
Disallow: /*origin=*
Disallow: /my-account
Disallow: /*proof=*
Disallow: /*/figures
Disallow: /*/tables
Disallow: /*/schemes
Disallow: /*/structures
Disallow: /*/metrics
Disallow: /platform/contextual


User-agent: *
Sitemap: https://www.nature.com/sitemap.xml
Sitemap: https://www.nature.com/subjects/sitemap.xml
Sitemap: https://www.nature.com/naturecareers/sitemap.xml
Disallow: /search
Disallow: /*.pdf$
Disallow: /*.PDF$
Disallow: /*/*/*/pf/
Disallow: /*.otmi$
Disallow: /*/*/*/*/otmi/
Disallow: /*/*/*/*/fp/
Disallow: /protocolexchange/labgroups/
Disallow: /naturecareers/jobs/search
Disallow: /webcasts/*
Disallow: /*draft=*
Disallow: /*foxtrotcallback=*
Disallow: /*origin=*
Disallow: /my-account
Disallow: /*proof=*
Disallow: /*/figures
Disallow: /*/tables
Disallow: /*/schemes
Disallow: /*/structures
Disallow: /*/metrics
Disallow: /platform/contextual


User-agent: *
Sitemap: https://www.nature.com/sitemap.xml
Sitemap: https://www.nature.com/subjects/sitemap.xml
Sitemap: https://www.nature.com/naturecareers/sitemap.xml
Disallow: /search
Disallow: /*.pdf$
Disallow: /*.PDF$
Disallow: /*/*/*/pf/
Disallow: /*.otmi$
Disallow: /*/*/*/*/otmi/
Disallow: /*/*/*/*/fp/
Disallow: /protocolexchange/labgroups/
Disallow: /naturecareers/jobs/search
Disallow: /webcasts/*
Disallow: /*draft=*
Disallow: /*foxtrotcallback=*
Disallow: /*origin=*
Disallow: /my-account
Disallow: /*proof=*
Disallow: /*/figures
Disallow: /*/tables
Disallow: /*/schemes
Disallow: /*/structures
Disallow: /*/metrics
Disallow: /platform/contextual
Disallow: /articles/*.ris


User-agent: *
Sitemap: https://www.nature.com/sitemap.xml
Sitemap: https://www.nature.com/subjects/sitemap.xml
Sitemap: https://www.nature.com/naturecareers/sitemap.xml
Disallow: /search
Disallow: /*.pdf$
Disallow: /*.PDF$
Disallow: /*/*/*/pf/
Disallow: /*.otmi$
Disallow: /*/*/*/*/otmi/
Disallow: /*/*/*/*/fp/
Disallow: /protocolexchange/labgroups/
Disallow: /naturecareers/jobs/search
Disallow: /webcasts/*
Disallow: /*draft=*
Disallow: /*foxtrotcallback=*
Disallow: /*origin=*
Disallow: /my-account
Disallow: /*proof=t%2Btarget%3D\*
Disallow: /*proof=*
Disallow: /*/figures
Disallow: /*/tables
Disallow: /*/schemes
Disallow: /*/structures
Disallow: /*/metrics
Disallow: /platform/contextual
Disallow: /articles/*.ris