User-agent: *
Crawl-delay: 10

Disallow: /labs/mercury-lab
Disallow: /labs/hif
Disallow: /labs/rml
Disallow: /labs/rsil
Disallow: /rsil-themes/
Disallow: /rsil-topics/
Disallow: /centers/earthquakesc
Disallow: /earthquakesc-themes/
Disallow: /earthquakesc-topics/
Disallow: /earthquakesc-non-theme-topics/
Disallow: /centers/ema
Disallow: /ema-themes/
Disallow: /ema-topics/
Disallow: /ema-non-theme-topics/
Disallow: /centers/mercury
Disallow: /mercury-themes/
Disallow: /mercury-topics/
Disallow: /mercury-non-theme-topics/
Disallow: /centers/hif
Disallow: /hif-themes/
Disallow: /hif-topics/
Disallow: /hif-non-theme-topics/
Disallow: /dorian-themes/
Disallow: /dorian-topics/
Disallow: /dorian-non-theme-topics/

Disallow: /centers/testten
Disallow: /testten-themes/
Disallow: /testten-topics/
Disallow: /testten-non-theme-topics/
Disallow: /centers/tcc
Disallow: /tcc-themes/
Disallow: /tcc-topics/
Disallow: /tcc-non-theme-topics/
Disallow: /centers/hydroacoustics
Disallow: /hydroacoustics-themes/
Disallow: /hydroacoustics-topics/
Disallow: /hydroacoustics-non-theme-topics/
Disallow: /centers/states
Disallow: /states-themes/
Disallow: /states-topics/
Disallow: /states-non-theme-topics/
Disallow: /centers/newhurricane
Disallow: /newhurricane-themes/
Disallow: /newhurricane-topics/
Disallow: /newhurricane-non-theme-topics/
Disallow: /centers/newhurricane
Disallow: /newhurricane-themes/
Disallow: /newhurricane-topics/
Disallow: /newhurricane-non-theme-topics/
Disallow: /centers/hurricanetemp
Disallow: /hurricanetemp-themes/
Disallow: /hurricanetemp-topics/
Disallow: /hurricanetemp-non-theme-topics/
Disallow: /centers/stormtemplate
Disallow: /stormtemplate-themes/
Disallow: /stormtemplate-topics/
Disallow: /stormtemplate-non-theme-topics/
Disallow: /centers/stormtemp
Disallow: /stormtemp-themes/
Disallow: /stormtemp-topics/
Disallow: /stormtemp-non-theme-topics/
Disallow: /centers/subductionzone
Disallow: /subductionzone-themes/
Disallow: /subductionzone-topics/
Disallow: /subductionzone-non-theme-topics/
Disallow: /centers/marco
Disallow: /marco-themes/
Disallow: /marco-topics/
Disallow: /marco-non-theme-topics/
Disallow: /centers/laura
Disallow: /laura-themes/
Disallow: /laura-topics/
Disallow: /laura-non-theme-topics/
Disallow: /centers/sally
Disallow: /sally-themes/
Disallow: /sally-topics/
Disallow: /sally-non-theme-topics/
Disallow: /centers/mmfs-lab
Disallow: /mmfs-lab-themes/
Disallow: /mmfs-lab-topics/
Disallow: /mmfs-lab-non-theme-topics/
Disallow: /centers/pec-lab
Disallow: /pec-lab-themes/
Disallow: /pec-lab-topics/
Disallow: /pec-lab-non-theme-topics/
Disallow: /centers/ocr-lab
Disallow: /ocr-lab-themes/
Disallow: /ocr-lab-topics/
Disallow: /ocr-lab-non-theme-topics/
Disallow: /centers/deepsea
Disallow: /deepsea-themes/
Disallow: /deepsea-topics/
Disallow: /deepsea-non-theme-topics/
Disallow: /centers/envhealthprog
Disallow: /envhealthprog-themes/
Disallow: /envhealthprog-topics/
Disallow: /envhealthprog-non-theme-topics/
Disallow: /centers/gemsc
Disallow: /gemsc-themes/
Disallow: /gemsc-topics/
Disallow: /gemsc-non-theme-topics/
#-----GLOBALS-------------
#Global non-indexing of any item matching paths below. Do not add individual elsewhere.
Disallow: /media/images/*logo*
Disallow: /science-centers-and-observatories/
Disallow: /locations/
Disallow: /program/
Disallow: /*-themes/
Disallow: /*-topics/
Disallow: /*-non-theme-topics/
Disallow: /cost-centers/
Disallow: /connect/
Disallow: /cost-center/
Disallow: /science-centers/
Disallow: /special-topics/
Disallow: /includes/
Disallow: /misc/
Disallow: /modules/
Disallow: /profiles/
Disallow: /scripts/
Disallow: /themes/
Disallow: /CHANGELOG.txt
Disallow: /cron.php
Disallow: /INSTALL.mysql.txt
Disallow: /INSTALL.pgsql.txt
Disallow: /INSTALL.sqlite.txt
Disallow: /install.php
Disallow: /INSTALL.txt
Disallow: /LICENSE.txt
Disallow: /MAINTAINERS.txt
Disallow: /update.php
Disallow: /UPGRADE.txt
Disallow: /xmlrpc.php
# Paths (clean URLs)
Disallow: /admin/
Disallow: /comment/reply/
Disallow: /filter/tips/
Disallow: /node/
Disallow: /node/add/
Disallow: /search/
Disallow: /user/register/
Disallow: /user/password/
Disallow: /user/login/
Disallow: /user/logout/
Disallow: /employees
Disallow: /science-support/communications-and-publishing/about/employees
# Paths (no clean URLs)
Disallow: /?q=admin/
Disallow: /?q=comment/reply/
Disallow: /?q=filter/tips/
Disallow: /?q=node/
Disallow: /?q=node/add/
Disallow: /?q=search/
Disallow: /?q=user/password/
Disallow: /?q=user/register/
Disallow: /?q=user/login/
Disallow: /?q=user/logout/
#-----END GLOBALS-------

#SCIENCE EXPLORER
Disallow: /science/science-explorer/
Disallow: /science-explorer-results

#PUBS
Disallow:	/products/publications/
Disallow:	/centers/pubs
Disallow:	/pubs-themes/
Disallow:	/pubs-topics/

#ECOSYSTEMS
#ENVIRONMENTS PROGRAM
Disallow: /centers/enviroprog
Disallow: /program/environments-program
Disallow: /enviroprog-themes/
Disallow: /enviroprog-topics/
#ENERGY AND WILDLIFE PROGRAM
Disallow: /centers/ewp
Disallow: /program/energy-wildlife
Disallow: /ewp-themes/
Disallow: /ewp-topics/
#FISHERIES
Disallow: /centers/fisheries
Disallow: /program/fisheries-program
Disallow: /fisheries-themes/
Disallow: /fisheries-topics/
#FISH AND WILDLIFE DISEASE
Disallow: /centers/fwd
Disallow: /program/fish-wildlife-disease
Disallow: /fwd-themes/
Disallow: /fwd-topics/
#INVASIVE SPECIES PROGRAM
Disallow: /centers/invasives
Disallow: /program/invasive-species-program
Disallow: /invasives_themes/
Disallow: /invasives_topics/
#SSSP AKA SAGE GROUSE AND SAGEBRUSH ECOSYSTEM
Disallow: /centers/sssp
Disallow: /sage-grouse-sagebrush-steppe
Disallow: /sssp-themes/
Disallow: /sssp-topics/
#STP STATUS AND TRENDS PROGRAM
Disallow: /centers/wp
Disallow: /program/status-and-trends-program
Disallow: /stp-themes/
Disallow: /stp-topics/
#WILDLIFE PROGRAM
Disallow: /centers/wp
Disallow: /program/wildlife-program
Disallow: /wp-themes/
Disallow: /wp-topics/

#------
#WRET site for classroom
Disallow: /centers/wret
Disallow: /wret-themes/
Disallow: /wret-topics/

#------
#SPECIAL TOPICS
#FLOODS
Disallow: /floods-themes/
Disallow: /floods-topics/
#BIG SUR
Disallow: /big-sur-themes/
#MICROBIOME
Disallow: /microbiome-themes/
Disallow: /microbiome-topics/
#MISSISSIPPI
Disallow: /mississippi_themes/
Disallow: /mississippi_topics/
#------

#MERGED NOT DELETED YET
#ILIA-WATER
Disallow: /centers/ilia-water
Disallow: /ilia-water-themes/
Disallow: /ilia-water-topics/
#INKY
Disallow: /centers/inkywsc
Disallow: /inkywsc-themes/
Disallow: /inkywsc-topics/
#MI-OH
Disallow: /centers/mi-oh-water
Disallow: /mi-water-themes/
Disallow: /mi-water-topics/

#---------------------
#Sites allow ie now live
#---------------------
#ASC
Disallow: /asc-themes/
Disallow: /asc-topics/
#ASTRO
Disallow: /astrogeo-sc-themes/
Disallow: /astrogeo-sc-topics/
#AZ-WATER
Disallow: /az-water-themes/
Disallow: /az-water-topics/
#CA-WATER
Disallow: /ca-water-themes/
Disallow: /ca-water-topics/
#CA WATER LS
Disallow: /ca-water-ls-themes/
Disallow: /ca-water-ls-topics/
#CAR-FL-WATER
Disallow: /car-fl-water-themes/
Disallow: /car-fl-water-topics/
#CASC PROGRAM
Disallow: /casc-themes/
Disallow: /casc-topics/
#CASC CENTER
Disallow: /casc-sc-themes/
Disallow: /casc-sc-topics/
Disallow: /casc-sc-non-theme-topics/
#CBA
Disallow: /cba-themes/
Disallow: /cba-topics/
#CDI
Disallow: /cdi-themes/
Disallow: /cdi-topics/
#CBP CONTAMINANT BIOLOGY PROGRAM
Disallow: /ncbp-themes/
Disallow: /ncbp-topics/
Disallow: /ncbp-non-theme-topics/
#CERC
Disallow: /cerc-themes/
Disallow: /cerc-topics/
#CERSC
Disallow: /cersc-themes/
Disallow: /cersc-topics/
#CGGSC
Disallow: /cggsc-themes/
Disallow: /cggsc-topics/
#CM-WATER
Disallow: /cm-water-themes/
Disallow: /cm-water-topics/
#CMERSC
Disallow: /cmersc-themes/
Disallow: /cmersc-topics/
#CMGP AKA COASTAL MARINE HAZARDS AND RESOURCES PROGRAM
Disallow: /cmgp-themes/
Disallow: /cmgp-topics/
#CO-WATER
Disallow: /co-water-themes/
Disallow: /co-water-topics/
#DAKOTA
Disallow: /dakota-water-themes/
Disallow: /dakota-water-topics/
#DML
Disallow: /dml-themes/
Disallow: /dml-topics/
Disallow:	/dml-capability-themes/
Disallow:	/dml-capabilities-topics/
#EERSC
Disallow: /eersc-themes/
Disallow: /eersc-topics/
#EGGL
Disallow: /eggl-themes/
Disallow: /eggl-topics/
#EARTHMRI
Disallow: /earthmri-themes/
Disallow: /earthmri-topics/
Disallow: /earthmri-non-theme-topics/
#EHMA
Disallow: /ehma-themes/
Disallow: /ehma-topics/
#EHP EARTHQUAKE HAZARDS PROGRAM
Disallow: /ehp-themes/
Disallow: /ehp-topics/
#EMERSC
Disallow: /emersc-themes/
Disallow: /emersc-topics/
#EMP ENERGY AND MINERALS PROGRAM
Disallow: /emp-themes/
Disallow: /emp-topics/
#ENERGY & MINERALS MA
Disallow: /emma-themes/
Disallow: /emma-topics/
#EROS
Disallow: /eros-themes/
Disallow: /eros-topics/
#ERP ENERGY RESOURCES PROGRAM
Disallow: /erp-themes/
Disallow: /erp-topics/
#FIRE
Disallow: /fire-themes/
Disallow: /fire-topics/
#FBGC
Disallow: /fbgc-themes/
Disallow: /fbgc-topics/
#FORT
Disallow: /fort-topics/
Disallow: /fort-themes/
#FRESC
Disallow: /fresc-themes/
Disallow: /fresc-topics/
#GAP GAP ANALYSIS PROJECT
Disallow: /gap-themes/
Disallow: /gap-topics/
#GAPP
Disallow: /centers/gapp
Disallow: /gapp-themes/
Disallow: /gapp-topics/
#GECSC
Disallow: /gecsc-themes/
Disallow: /gecsc-topics/
#GEOHAZARDS
Disallow: /geohazards-themes/
Disallow: /geohazards-topics/
#GEOMAGNETISM
Disallow: /geomagnetism-themes/
Disallow: /geomagnetism-topics/
#GLR
Disallow: /glr-themes/
Disallow: /glr-topics/
#GLSC
Disallow: /glsc-themes/
Disallow: /glsc-topics/
#GMEG
Disallow: /gmeg-themes/
Disallow: /gmeg-topics/
#GOM
Disallow: /gom-themes/
Disallow: /gom-topics/
#GWSIP GROUNDWATER AND STREAMFLOW INFORMATION PROGRAM
Disallow: /gwsip-themes/
Disallow: /gwsip-topics/
#HURRICANE FLORENCE
Disallow: /hurrflorence-themes/
Disallow: /hurrflorence-topics/
#HURRICANE MICHAEL
Disallow: /hurrmichael-themes/
Disallow: /hurrmichael-topics/
#ID-WATER
Disallow: /id-water-themes/
Disallow: /id-water-topics/
#INNOVATION CENTER
Disallow: /innovation-themes/
Disallow: /innovation-topics/
#LEETOWN
Disallow: /lsc-themes/
Disallow: /lsc-topics/
#LHP LANDSLIDE HAZARDS PROGRAM
Disallow: /program/landslide-hazards-program
Disallow: /lhp-themes/
Disallow: /lhp-topics/
#LRMA LAND RESOURCES MA
Disallow: /lrma-themes/
Disallow: /lrma-topics/
#LSP LAND CHANGE SCIENCE PROGRAM
Disallow: /lcsp-themes/
Disallow: /lcsp-topics/
#MENDENHALL
Disallow: /mendenhall-themes/
Disallow: /mendenhall-topics/
#MRL
Disallow: /mrl-themes/
Disallow: /mrl-topics/
#MRP MINERAL RESOURCES PROGRAM
Disallow: /mrp-themes/
Disallow: /mrp-topics/
#NCGMP NATIONAL COOPERATIVE GEOLOGIC MAPPING PROGRAM
Disallow: /ncgmp-themes/
Disallow: /ncgmp-topics/
#NE-WATER
Disallow: /ne-water-themes/
Disallow: /ne-water-topics/
#NEHP
Disallow: /nehp-themes/
Disallow: /nehp-topics/
#NGGDPP NATIONAL GEOGRAPHIC AND GEOSPATIAL DATA PRESERVATION PROGRAM
Disallow: /nggdpp-themes/
Disallow: /nggdpp-topics/
#NHMA
Disallow:	/nhma-themes/
Disallow:	/nhma-topics/
#NJ-WATER
Disallow:	/nj-water-themes/
Disallow:	/nj-water-topics/
#NPWRC
Disallow: /npwrc-themes/
Disallow: /npwrc-topics/
#NWHC
Disallow:	/nwhc-themes/
Disallow:	/nwhc-topics/
#NV-WATER
Disallow:	/nv-water-themes/
Disallow:	/nv-water-topics/
#NWQL
Disallow:	/nwql-themes/
Disallow:	/nwql-topics/
Disallow:	/nwql-capability-themes/
Disallow:	/nwql-capabilities-topics/
#NWQP NATIONAL WATER QUALITY PROGRAM
Disallow: /nwqp-themes/
Disallow: /nwqp-topics/
#PA-WATER
Disallow:	/pa-water-themes/
Disallow:	/pa-water-topics/
#PCMSC
Disallow:	/pcmsc-themes/
Disallow:	/pcmsc-topics/
#PIWSC
Disallow:	/piwsc-themes/
Disallow:	/piwsc-topics/
#POWELL CENTER
Disallow: /powell-ctr-themes/
Disallow: /powell-ctr-topics/
#MD-DE-DC
Disallow: /md-de-dc-water-themes/
Disallow: /md-de-dc-water-topics/
#OR-WATER
Disallow:	/or-water-themes/
Disallow:	/or-water-topics/
#ORGANIC MATTER RESEARCH LAB OMRL
Disallow: /omrl-themes/
Disallow: /omrl-topics/
Disallow:	/omrl-capability-themes/
Disallow:	/omrl-capabilities-topics/
#PWRC
Disallow:	/pwrc-themes/
Disallow:	/pwrc-topics/
#RBPGL
Disallow: /rbpgl-themes/
Disallow: /rbpgl-topics/
#RSP REMOTE SENSING PHENOLOGY
Disallow: /phenology-themes/
Disallow: /phenology-topics/
#SAFRR
Disallow: /safrr-themes/
Disallow: /safrr-topics/
#SDC
Disallow: /sdc-themes/
Disallow: /sdc-topics/
#SPCMSC
Disallow: /spcmsc-themes/
Disallow: /spcmsc-topics/
#SPECLAB
Disallow: /spec-lab-themes/
Disallow: /spec-lab-topics/
Disallow: /spec-lab-capability-themes/
Disallow:	/spec-lab-capabilities-topics/
#TX-WATER
Disallow: /tx-water-themes/
Disallow: /tx-water-topics/
#TOXIC SUBSTANCES HYDROLOGY PROGRAM
Disallow: /tshp-themes/
Disallow: /tshp-topics/
Disallow: /tshp-non-theme-topics/
#UMID-WATER
Disallow: /umid-water-themes/
Disallow: /umid-water-topics/
#UT-WATER
Disallow: /ut-water-themes/
Disallow: /ut-water-topics/
#VA-WV
Disallow: /va-wv-water-themes/
Disallow: /va-wv-water-topics/
#WA-WATER
Disallow: /wa-water-themes/
Disallow: /wa-water-topics/
#WAUSP WATER AVAILABILITY AND USE SCIENCE PROGRAM
Disallow: /wausp-themes/
Disallow: /wausp-topics/
#WATER SCIENCE SCHOOL
Disallow: /water-school-themes/
Disallow: /water-school-topics/
#WERC
Disallow: /werc-themes/
Disallow: /werc-topics/
#WFRC
Disallow: /wfrc-themes/
Disallow: /wfrc-topics/
#WGSC
Disallow: /wgsc-themes/
Disallow: /wgsc-topics/
#WHCMSC
Disallow: /whcmsc-themes/
Disallow: /whcmsc-topics/
#WMA
Disallow: /wma-themes/
Disallow: /wma-topics/
#WRRP AKA WRRI
Disallow: /wrri-themes/
Disallow: /wrri-topics/
#WY-MT-WATER
Disallow: /wy-mt-water-themes/
Disallow: /wy-mt-water-topics/`
#FOIA-FAQ
Disallow: /faqs/how-do-i-check-status-my-us-geological-survey-freedom-information-act-foia-request


#robots.txt
#
User-agent:*
Disallow: /blogs/
Disallow: /water/southatlantic/usgs/
Disallow: /water/southatlantic/realtime/go2/


User-agent: *
Crawl-delay: 10

Disallow: /rsil-themes/
Disallow: /rsil-topics/
Disallow: /centers/earthquakesc
Disallow: /earthquakesc-themes/
Disallow: /earthquakesc-topics/
Disallow: /earthquakesc-non-theme-topics/
Disallow: /dorian-themes/
Disallow: /dorian-topics/
Disallow: /dorian-non-theme-topics/

Disallow: /centers/testten
Disallow: /testten-themes/
Disallow: /testten-topics/
Disallow: /testten-non-theme-topics/

Disallow: /centers/newhurricane
Disallow: /newhurricane-themes/
Disallow: /newhurricane-topics/
Disallow: /newhurricane-non-theme-topics/
Disallow: /centers/newhurricane
Disallow: /newhurricane-themes/
Disallow: /newhurricane-topics/
Disallow: /newhurricane-non-theme-topics/
Disallow: /centers/hurricanetemp
Disallow: /hurricanetemp-themes/
Disallow: /hurricanetemp-topics/
Disallow: /hurricanetemp-non-theme-topics/
Disallow: /centers/stormtemplate
Disallow: /stormtemplate-themes/
Disallow: /stormtemplate-topics/
Disallow: /stormtemplate-non-theme-topics/
Disallow: /centers/stormtemp
Disallow: /stormtemp-themes/
Disallow: /stormtemp-topics/
Disallow: /stormtemp-non-theme-topics/
Disallow: /gemsc-themes/
Disallow: /gemsc-topics/
Disallow: /gemsc-non-theme-topics/
Disallow: /volscicenter-themes/
Disallow: /volscicenter-topics/
Disallow: /volscicenter-non-theme-topics/
Disallow: /eesc-themes/
Disallow: /eesc-topics/
Disallow: /eesc-non-theme-topics/
#-----GLOBALS-------------
#Global non-indexing of any item matching paths below. Do not add individuals elsewhere.
Disallow: /media/images/*logo*
Disallow: /science-centers-and-observatories/
Disallow: /locations/
Disallow: /program/
Disallow: /*-themes/
#Disallow: /*-topics/
Disallow: /*-non-theme-topics/
Disallow: /cost-centers/
Disallow: /connect/
Disallow: /cost-center/
Disallow: /science-centers/
#Disallow: /special-topics/
Disallow: /includes/
Disallow: /misc/
Disallow: /modules/
Disallow: /profiles/
Disallow: /scripts/
Disallow: /themes/
Disallow: /CHANGELOG.txt
Disallow: /cron.php
Disallow: /INSTALL.mysql.txt
Disallow: /INSTALL.pgsql.txt
Disallow: /INSTALL.sqlite.txt
Disallow: /install.php
Disallow: /INSTALL.txt
Disallow: /LICENSE.txt
Disallow: /MAINTAINERS.txt
Disallow: /update.php
Disallow: /UPGRADE.txt
Disallow: /xmlrpc.php
# Paths (clean URLs)
Disallow: /admin/
Disallow: /comment/reply/
Disallow: /filter/tips/
Disallow: /node/
Disallow: /node/add/
Disallow: /search/
Disallow: /user/register/
Disallow: /user/password/
Disallow: /user/login/
Disallow: /user/logout/
Disallow: /employees
Disallow: /science-support/communications-and-publishing/about/employees
# Paths (no clean URLs)
Disallow: /?q=admin/
Disallow: /?q=comment/reply/
Disallow: /?q=filter/tips/
Disallow: /?q=node/
Disallow: /?q=node/add/
Disallow: /?q=search/
Disallow: /?q=user/password/
Disallow: /?q=user/register/
Disallow: /?q=user/login/
Disallow: /?q=user/logout/
#-----END GLOBALS-------

#SCIENCE EXPLORER
Disallow: /science/science-explorer/
Disallow: /science-explorer-results

#PUBS
Disallow:	/products/publications/
Disallow:	/centers/pubs
Disallow:	/pubs-themes/
Disallow:	/pubs-topics/

#ECOSYSTEMS
#ENVIRONMENTS PROGRAM
Disallow: /centers/enviroprog
Disallow: /program/environments-program
Disallow: /enviroprog-themes/
Disallow: /enviroprog-topics/
#ENERGY AND WILDLIFE PROGRAM
Disallow: /centers/ewp
Disallow: /program/energy-wildlife
Disallow: /ewp-themes/
Disallow: /ewp-topics/
#FISHERIES
Disallow: /centers/fisheries
Disallow: /program/fisheries-program
Disallow: /fisheries-themes/
Disallow: /fisheries-topics/
#FISH AND WILDLIFE DISEASE
Disallow: /centers/fwd
Disallow: /program/fish-wildlife-disease
Disallow: /fwd-themes/
Disallow: /fwd-topics/
#INVASIVE SPECIES PROGRAM
Disallow: /centers/invasives
Disallow: /program/invasive-species-program
Disallow: /invasives_themes/
Disallow: /invasives_topics/
#SSSP AKA SAGE GROUSE AND SAGEBRUSH ECOSYSTEM
Disallow: /centers/sssp
Disallow: /sage-grouse-sagebrush-steppe
Disallow: /sssp-themes/
Disallow: /sssp-topics/
#STP STATUS AND TRENDS PROGRAM
Disallow: /centers/wp
Disallow: /program/status-and-trends-program
Disallow: /stp-themes/
Disallow: /stp-topics/
#WILDLIFE PROGRAM
Disallow: /centers/wp
Disallow: /program/wildlife-program
Disallow: /wp-themes/
Disallow: /wp-topics/

#------
#WRET site for classroom
Disallow: /centers/wret
Disallow: /wret-themes/
Disallow: /wret-topics/

#------
#SPECIAL TOPICS
#FLOODS
Disallow: /floods-themes/
Disallow: /floods-topics/
#BIG SUR
Disallow: /big-sur-themes/
#MICROBIOME
Disallow: /microbiome-themes/
Disallow: /microbiome-topics/
#MISSISSIPPI
Disallow: /mississippi_themes/
Disallow: /mississippi_topics/
#------

#MERGED NOT DELETED YET
#ILIA-WATER
Disallow: /centers/ilia-water
Disallow: /ilia-water-themes/
Disallow: /ilia-water-topics/
#INKY
Disallow: /centers/inkywsc
Disallow: /inkywsc-themes/
Disallow: /inkywsc-topics/
#MI-OH
Disallow: /centers/mi-oh-water
Disallow: /mi-water-themes/
Disallow: /mi-water-topics/

#---------------------
#Sites allow ie now live
#---------------------
#ASC
Disallow: /asc-themes/
Disallow: /asc-topics/
#ASTRO
Disallow: /astrogeo-sc-themes/
Disallow: /astrogeo-sc-topics/
#AZ-WATER
Disallow: /az-water-themes/
Disallow: /az-water-topics/
#CA-WATER
Disallow: /ca-water-themes/
Disallow: /ca-water-topics/
#CA WATER LS
Disallow: /ca-water-ls-themes/
Disallow: /ca-water-ls-topics/
#CAR-FL-WATER
Disallow: /car-fl-water-themes/
Disallow: /car-fl-water-topics/
#CASC PROGRAM
Disallow: /casc-themes/
Disallow: /casc-topics/
#CASC CENTER
Disallow: /casc-sc-themes/
Disallow: /casc-sc-topics/
Disallow: /casc-sc-non-theme-topics/
#CBA
Disallow: /cba-themes/
Disallow: /cba-topics/
#CDI
Disallow: /cdi-themes/
Disallow: /cdi-topics/
#CBP CONTAMINANT BIOLOGY PROGRAM
Disallow: /ncbp-themes/
Disallow: /ncbp-topics/
Disallow: /ncbp-non-theme-topics/
#CERC
Disallow: /cerc-themes/
Disallow: /cerc-topics/
#CERSC
Disallow: /cersc-themes/
Disallow: /cersc-topics/
#CGGSC
Disallow: /cggsc-themes/
Disallow: /cggsc-topics/
#CM-WATER
Disallow: /cm-water-themes/
Disallow: /cm-water-topics/
#CMERSC
Disallow: /cmersc-themes/
Disallow: /cmersc-topics/
#CMGP AKA COASTAL MARINE HAZARDS AND RESOURCES PROGRAM
Disallow: /cmgp-themes/
Disallow: /cmgp-topics/
#CO-WATER
Disallow: /co-water-themes/
Disallow: /co-water-topics/
#DAKOTA
Disallow: /dakota-water-themes/
Disallow: /dakota-water-topics/
#DML
Disallow: /dml-themes/
Disallow: /dml-topics/
Disallow:	/dml-capability-themes/
Disallow:	/dml-capabilities-topics/
#EERSC
Disallow: /eersc-themes/
Disallow: /eersc-topics/
#EGGL
Disallow: /eggl-themes/
Disallow: /eggl-topics/
#EARTHMRI
Disallow: /earthmri-themes/
Disallow: /earthmri-topics/
Disallow: /earthmri-non-theme-topics/
#EHMA
Disallow: /ehma-themes/
Disallow: /ehma-topics/
#EHP EARTHQUAKE HAZARDS PROGRAM
Disallow: /ehp-themes/
Disallow: /ehp-topics/
#EMERSC
Disallow: /emersc-themes/
Disallow: /emersc-topics/
#EMP ENERGY AND MINERALS PROGRAM
Disallow: /emp-themes/
Disallow: /emp-topics/
#ENERGY & MINERALS MA
Disallow: /emma-themes/
Disallow: /emma-topics/
#EROS
Disallow: /eros-themes/
Disallow: /eros-topics/
#ERP ENERGY RESOURCES PROGRAM
Disallow: /erp-themes/
Disallow: /erp-topics/
#FIRE
Disallow: /fire-themes/
Disallow: /fire-topics/
#FBGC
Disallow: /fbgc-themes/
Disallow: /fbgc-topics/
#FORT
Disallow: /fort-topics/
Disallow: /fort-themes/
#FRESC
Disallow: /fresc-themes/
Disallow: /fresc-topics/
#GAP GAP ANALYSIS PROJECT
Disallow: /gap-themes/
Disallow: /gap-topics/
#GAPP
Disallow: /centers/gapp
Disallow: /gapp-themes/
Disallow: /gapp-topics/
#GECSC
Disallow: /gecsc-themes/
Disallow: /gecsc-topics/
#GEOHAZARDS
Disallow: /geohazards-themes/
Disallow: /geohazards-topics/
#GEOMAGNETISM
Disallow: /geomagnetism-themes/
Disallow: /geomagnetism-topics/
#GLR
Disallow: /glr-themes/
Disallow: /glr-topics/
#GLSC
Disallow: /glsc-themes/
Disallow: /glsc-topics/
#GMEG
Disallow: /gmeg-themes/
Disallow: /gmeg-topics/
#GOM
Disallow: /gom-themes/
Disallow: /gom-topics/
#GWSIP GROUNDWATER AND STREAMFLOW INFORMATION PROGRAM
Disallow: /gwsip-themes/
Disallow: /gwsip-topics/
#HURRICANE FLORENCE
Disallow: /hurrflorence-themes/
Disallow: /hurrflorence-topics/
#HURRICANE MICHAEL
Disallow: /hurrmichael-themes/
Disallow: /hurrmichael-topics/
#ID-WATER
Disallow: /id-water-themes/
Disallow: /id-water-topics/
#INNOVATION CENTER
Disallow: /innovation-themes/
Disallow: /innovation-topics/
#LEETOWN
Disallow: /lsc-themes/
Disallow: /lsc-topics/
#LHP LANDSLIDE HAZARDS PROGRAM
Disallow: /program/landslide-hazards-program
Disallow: /lhp-themes/
Disallow: /lhp-topics/
#LRMA LAND RESOURCES MA
Disallow: /lrma-themes/
Disallow: /lrma-topics/
#LSP LAND CHANGE SCIENCE PROGRAM
Disallow: /lcsp-themes/
Disallow: /lcsp-topics/
#MENDENHALL
Disallow: /mendenhall-themes/
Disallow: /mendenhall-topics/
#MRL
Disallow: /mrl-themes/
Disallow: /mrl-topics/
#MRP MINERAL RESOURCES PROGRAM
Disallow: /mrp-themes/
Disallow: /mrp-topics/
#NCGMP NATIONAL COOPERATIVE GEOLOGIC MAPPING PROGRAM
Disallow: /ncgmp-themes/
Disallow: /ncgmp-topics/
#NE-WATER
Disallow: /ne-water-themes/
Disallow: /ne-water-topics/
#NEHP
Disallow: /nehp-themes/
Disallow: /nehp-topics/
#NGGDPP NATIONAL GEOGRAPHIC AND GEOSPATIAL DATA PRESERVATION PROGRAM
Disallow: /nggdpp-themes/
Disallow: /nggdpp-topics/
#NHMA
Disallow:	/nhma-themes/
Disallow:	/nhma-topics/
#NJ-WATER
Disallow:	/nj-water-themes/
Disallow:	/nj-water-topics/
#NPWRC
Disallow: /npwrc-themes/
Disallow: /npwrc-topics/
#NWHC
Disallow:	/nwhc-themes/
Disallow:	/nwhc-topics/
#NV-WATER
Disallow:	/nv-water-themes/
Disallow:	/nv-water-topics/
#NWQL
Disallow:	/nwql-themes/
Disallow:	/nwql-topics/
Disallow:	/nwql-capability-themes/
Disallow:	/nwql-capabilities-topics/
#NWQP NATIONAL WATER QUALITY PROGRAM
Disallow: /nwqp-themes/
Disallow: /nwqp-topics/
#PA-WATER
Disallow:	/pa-water-themes/
Disallow:	/pa-water-topics/
#PCMSC
Disallow:	/pcmsc-themes/
Disallow:	/pcmsc-topics/
#PIWSC
Disallow:	/piwsc-themes/
Disallow:	/piwsc-topics/
#POWELL CENTER
Disallow: /powell-ctr-themes/
Disallow: /powell-ctr-topics/
#MD-DE-DC
Disallow: /md-de-dc-water-themes/
Disallow: /md-de-dc-water-topics/
#OR-WATER
Disallow:	/or-water-themes/
Disallow:	/or-water-topics/
#ORGANIC MATTER RESEARCH LAB OMRL
Disallow: /omrl-themes/
Disallow: /omrl-topics/
Disallow:	/omrl-capability-themes/
Disallow:	/omrl-capabilities-topics/
#PWRC
Disallow:	/pwrc-themes/
Disallow:	/pwrc-topics/
#RBPGL
Disallow: /rbpgl-themes/
Disallow: /rbpgl-topics/
#RSP REMOTE SENSING PHENOLOGY
Disallow: /phenology-themes/
Disallow: /phenology-topics/
#SAFRR
Disallow: /safrr-themes/
Disallow: /safrr-topics/
#SDC
Disallow: /sdc-themes/
Disallow: /sdc-topics/
#SPCMSC
Disallow: /spcmsc-themes/
Disallow: /spcmsc-topics/
#SPECLAB
Disallow: /spec-lab-themes/
Disallow: /spec-lab-topics/
Disallow: /spec-lab-capability-themes/
Disallow:	/spec-lab-capabilities-topics/
#TX-WATER
Disallow: /tx-water-themes/
Disallow: /tx-water-topics/
#TOXIC SUBSTANCES HYDROLOGY PROGRAM
Disallow: /tshp-themes/
Disallow: /tshp-topics/
Disallow: /tshp-non-theme-topics/
#UMID-WATER
Disallow: /umid-water-themes/
Disallow: /umid-water-topics/
#UT-WATER
Disallow: /ut-water-themes/
Disallow: /ut-water-topics/
#VA-WV
Disallow: /va-wv-water-themes/
Disallow: /va-wv-water-topics/
#WA-WATER
Disallow: /wa-water-themes/
Disallow: /wa-water-topics/
#WAUSP WATER AVAILABILITY AND USE SCIENCE PROGRAM
Disallow: /wausp-themes/
Disallow: /wausp-topics/
#WATER SCIENCE SCHOOL
Disallow: /water-school-themes/
Disallow: /water-school-topics/
#WERC
Disallow: /werc-themes/
Disallow: /werc-topics/
#WFRC
Disallow: /wfrc-themes/
Disallow: /wfrc-topics/
#WGSC
Disallow: /wgsc-themes/
Disallow: /wgsc-topics/
#WHCMSC
Disallow: /whcmsc-themes/
Disallow: /whcmsc-topics/
#WMA
Disallow: /wma-themes/
Disallow: /wma-topics/
#WRRP AKA WRRI
Disallow: /wrri-themes/
Disallow: /wrri-topics/
#WY-MT-WATER
Disallow: /wy-mt-water-themes/
Disallow: /wy-mt-water-topics/`
#FOIA-FAQ
Disallow: /faqs/how-do-i-check-status-my-us-geological-survey-freedom-information-act-foia-request


User-agent: *
Crawl-delay: 10

Disallow: /rsil-themes/
Disallow: /rsil-topics/
Disallow: /centers/earthquakesc
Disallow: /earthquakesc-themes/
Disallow: /earthquakesc-topics/
Disallow: /earthquakesc-non-theme-topics/
Disallow: /dorian-themes/
Disallow: /dorian-topics/
Disallow: /dorian-non-theme-topics/

Disallow: /centers/testten
Disallow: /testten-themes/
Disallow: /testten-topics/
Disallow: /testten-non-theme-topics/

Disallow: /centers/newhurricane
Disallow: /newhurricane-themes/
Disallow: /newhurricane-topics/
Disallow: /newhurricane-non-theme-topics/
Disallow: /centers/newhurricane
Disallow: /newhurricane-themes/
Disallow: /newhurricane-topics/
Disallow: /newhurricane-non-theme-topics/
Disallow: /centers/hurricanetemp
Disallow: /hurricanetemp-themes/
Disallow: /hurricanetemp-topics/
Disallow: /hurricanetemp-non-theme-topics/
Disallow: /centers/stormtemplate
Disallow: /stormtemplate-themes/
Disallow: /stormtemplate-topics/
Disallow: /stormtemplate-non-theme-topics/
Disallow: /centers/stormtemp
Disallow: /stormtemp-themes/
Disallow: /stormtemp-topics/
Disallow: /stormtemp-non-theme-topics/
Disallow: /gemsc-themes/
Disallow: /gemsc-topics/
Disallow: /gemsc-non-theme-topics/
Disallow: /volscicenter-themes/
Disallow: /volscicenter-topics/
Disallow: /volscicenter-non-theme-topics/
Disallow: /eesc-themes/
Disallow: /eesc-topics/
Disallow: /eesc-non-theme-topics/
Disallow: /centers/henri
Disallow: /henri-themes/
Disallow: /henri-topics/
Disallow: /henri-non-theme-topics/
#-----GLOBALS-------------
#Global non-indexing of any item matching paths below. Do not add individuals elsewhere.
Disallow: /media/images/*logo*
Disallow: /science-centers-and-observatories/
Disallow: /locations/
Disallow: /program/
Disallow: /*-themes/
#Disallow: /*-topics/
Disallow: /*-non-theme-topics/
Disallow: /cost-centers/
Disallow: /connect/
Disallow: /cost-center/
Disallow: /science-centers/
#Disallow: /special-topics/
Disallow: /includes/
Disallow: /misc/
Disallow: /modules/
Disallow: /profiles/
Disallow: /scripts/
Disallow: /themes/
Disallow: /CHANGELOG.txt
Disallow: /cron.php
Disallow: /INSTALL.mysql.txt
Disallow: /INSTALL.pgsql.txt
Disallow: /INSTALL.sqlite.txt
Disallow: /install.php
Disallow: /INSTALL.txt
Disallow: /LICENSE.txt
Disallow: /MAINTAINERS.txt
Disallow: /update.php
Disallow: /UPGRADE.txt
Disallow: /xmlrpc.php
# Paths (clean URLs)
Disallow: /admin/
Disallow: /comment/reply/
Disallow: /filter/tips/
Disallow: /node/
Disallow: /node/add/
Disallow: /search/
Disallow: /user/register/
Disallow: /user/password/
Disallow: /user/login/
Disallow: /user/logout/
Disallow: /employees
Disallow: /science-support/communications-and-publishing/about/employees
# Paths (no clean URLs)
Disallow: /?q=admin/
Disallow: /?q=comment/reply/
Disallow: /?q=filter/tips/
Disallow: /?q=node/
Disallow: /?q=node/add/
Disallow: /?q=search/
Disallow: /?q=user/password/
Disallow: /?q=user/register/
Disallow: /?q=user/login/
Disallow: /?q=user/logout/
#-----END GLOBALS-------

#SCIENCE EXPLORER
Disallow: /science/science-explorer/
Disallow: /science-explorer-results

#PUBS
Disallow:	/products/publications/
Disallow:	/centers/pubs
Disallow:	/pubs-themes/
Disallow:	/pubs-topics/

#ECOSYSTEMS
#ENVIRONMENTS PROGRAM
Disallow: /centers/enviroprog
Disallow: /program/environments-program
Disallow: /enviroprog-themes/
Disallow: /enviroprog-topics/
#ENERGY AND WILDLIFE PROGRAM
Disallow: /centers/ewp
Disallow: /program/energy-wildlife
Disallow: /ewp-themes/
Disallow: /ewp-topics/
#FISHERIES
Disallow: /centers/fisheries
Disallow: /program/fisheries-program
Disallow: /fisheries-themes/
Disallow: /fisheries-topics/
#FISH AND WILDLIFE DISEASE
Disallow: /centers/fwd
Disallow: /program/fish-wildlife-disease
Disallow: /fwd-themes/
Disallow: /fwd-topics/
#INVASIVE SPECIES PROGRAM
Disallow: /centers/invasives
Disallow: /program/invasive-species-program
Disallow: /invasives_themes/
Disallow: /invasives_topics/
#SSSP AKA SAGE GROUSE AND SAGEBRUSH ECOSYSTEM
Disallow: /centers/sssp
Disallow: /sage-grouse-sagebrush-steppe
Disallow: /sssp-themes/
Disallow: /sssp-topics/
#STP STATUS AND TRENDS PROGRAM
Disallow: /centers/wp
Disallow: /program/status-and-trends-program
Disallow: /stp-themes/
Disallow: /stp-topics/
#WILDLIFE PROGRAM
Disallow: /centers/wp
Disallow: /program/wildlife-program
Disallow: /wp-themes/
Disallow: /wp-topics/

#------
#WRET site for classroom
Disallow: /centers/wret
Disallow: /wret-themes/
Disallow: /wret-topics/

#------
#SPECIAL TOPICS
#FLOODS
Disallow: /floods-themes/
Disallow: /floods-topics/
#BIG SUR
Disallow: /big-sur-themes/
#MICROBIOME
Disallow: /microbiome-themes/
Disallow: /microbiome-topics/
#MISSISSIPPI
Disallow: /mississippi_themes/
Disallow: /mississippi_topics/
#------

#MERGED NOT DELETED YET
#ILIA-WATER
Disallow: /centers/ilia-water
Disallow: /ilia-water-themes/
Disallow: /ilia-water-topics/
#INKY
Disallow: /centers/inkywsc
Disallow: /inkywsc-themes/
Disallow: /inkywsc-topics/
#MI-OH
Disallow: /centers/mi-oh-water
Disallow: /mi-water-themes/
Disallow: /mi-water-topics/

#---------------------
#Sites allow ie now live
#---------------------
#ASC
Disallow: /asc-themes/
Disallow: /asc-topics/
#ASTRO
Disallow: /astrogeo-sc-themes/
Disallow: /astrogeo-sc-topics/
#AZ-WATER
Disallow: /az-water-themes/
Disallow: /az-water-topics/
#CA-WATER
Disallow: /ca-water-themes/
Disallow: /ca-water-topics/
#CA WATER LS
Disallow: /ca-water-ls-themes/
Disallow: /ca-water-ls-topics/
#CAR-FL-WATER
Disallow: /car-fl-water-themes/
Disallow: /car-fl-water-topics/
#CASC PROGRAM
Disallow: /casc-themes/
Disallow: /casc-topics/
#CASC CENTER
Disallow: /casc-sc-themes/
Disallow: /casc-sc-topics/
Disallow: /casc-sc-non-theme-topics/
#CBA
Disallow: /cba-themes/
Disallow: /cba-topics/
#CDI
Disallow: /cdi-themes/
Disallow: /cdi-topics/
#CBP CONTAMINANT BIOLOGY PROGRAM
Disallow: /ncbp-themes/
Disallow: /ncbp-topics/
Disallow: /ncbp-non-theme-topics/
#CERC
Disallow: /cerc-themes/
Disallow: /cerc-topics/
#CERSC
Disallow: /cersc-themes/
Disallow: /cersc-topics/
#CGGSC
Disallow: /cggsc-themes/
Disallow: /cggsc-topics/
#CM-WATER
Disallow: /cm-water-themes/
Disallow: /cm-water-topics/
#CMERSC
Disallow: /cmersc-themes/
Disallow: /cmersc-topics/
#CMGP AKA COASTAL MARINE HAZARDS AND RESOURCES PROGRAM
Disallow: /cmgp-themes/
Disallow: /cmgp-topics/
#CO-WATER
Disallow: /co-water-themes/
Disallow: /co-water-topics/
#DAKOTA
Disallow: /dakota-water-themes/
Disallow: /dakota-water-topics/
#DML
Disallow: /dml-themes/
Disallow: /dml-topics/
Disallow:	/dml-capability-themes/
Disallow:	/dml-capabilities-topics/
#EERSC
Disallow: /eersc-themes/
Disallow: /eersc-topics/
#EGGL
Disallow: /eggl-themes/
Disallow: /eggl-topics/
#EARTHMRI
Disallow: /earthmri-themes/
Disallow: /earthmri-topics/
Disallow: /earthmri-non-theme-topics/
#EHMA
Disallow: /ehma-themes/
Disallow: /ehma-topics/
#EHP EARTHQUAKE HAZARDS PROGRAM
Disallow: /ehp-themes/
Disallow: /ehp-topics/
#EMERSC
Disallow: /emersc-themes/
Disallow: /emersc-topics/
#EMP ENERGY AND MINERALS PROGRAM
Disallow: /emp-themes/
Disallow: /emp-topics/
#ENERGY & MINERALS MA
Disallow: /emma-themes/
Disallow: /emma-topics/
#EROS
Disallow: /eros-themes/
Disallow: /eros-topics/
#ERP ENERGY RESOURCES PROGRAM
Disallow: /erp-themes/
Disallow: /erp-topics/
#FIRE
Disallow: /fire-themes/
Disallow: /fire-topics/
#FBGC
Disallow: /fbgc-themes/
Disallow: /fbgc-topics/
#FORT
Disallow: /fort-topics/
Disallow: /fort-themes/
#FRESC
Disallow: /fresc-themes/
Disallow: /fresc-topics/
#GAP GAP ANALYSIS PROJECT
Disallow: /gap-themes/
Disallow: /gap-topics/
#GAPP
Disallow: /centers/gapp
Disallow: /gapp-themes/
Disallow: /gapp-topics/
#GECSC
Disallow: /gecsc-themes/
Disallow: /gecsc-topics/
#GEOHAZARDS
Disallow: /geohazards-themes/
Disallow: /geohazards-topics/
#GEOMAGNETISM
Disallow: /geomagnetism-themes/
Disallow: /geomagnetism-topics/
#GLR
Disallow: /glr-themes/
Disallow: /glr-topics/
#GLSC
Disallow: /glsc-themes/
Disallow: /glsc-topics/
#GMEG
Disallow: /gmeg-themes/
Disallow: /gmeg-topics/
#GOM
Disallow: /gom-themes/
Disallow: /gom-topics/
#GWSIP GROUNDWATER AND STREAMFLOW INFORMATION PROGRAM
Disallow: /gwsip-themes/
Disallow: /gwsip-topics/
#HURRICANE FLORENCE
Disallow: /hurrflorence-themes/
Disallow: /hurrflorence-topics/
#HURRICANE MICHAEL
Disallow: /hurrmichael-themes/
Disallow: /hurrmichael-topics/
#ID-WATER
Disallow: /id-water-themes/
Disallow: /id-water-topics/
#INNOVATION CENTER
Disallow: /innovation-themes/
Disallow: /innovation-topics/
#LEETOWN
Disallow: /lsc-themes/
Disallow: /lsc-topics/
#LHP LANDSLIDE HAZARDS PROGRAM
Disallow: /program/landslide-hazards-program
Disallow: /lhp-themes/
Disallow: /lhp-topics/
#LRMA LAND RESOURCES MA
Disallow: /lrma-themes/
Disallow: /lrma-topics/
#LSP LAND CHANGE SCIENCE PROGRAM
Disallow: /lcsp-themes/
Disallow: /lcsp-topics/
#MENDENHALL
Disallow: /mendenhall-themes/
Disallow: /mendenhall-topics/
#MRL
Disallow: /mrl-themes/
Disallow: /mrl-topics/
#MRP MINERAL RESOURCES PROGRAM
Disallow: /mrp-themes/
Disallow: /mrp-topics/
#NCGMP NATIONAL COOPERATIVE GEOLOGIC MAPPING PROGRAM
Disallow: /ncgmp-themes/
Disallow: /ncgmp-topics/
#NE-WATER
Disallow: /ne-water-themes/
Disallow: /ne-water-topics/
#NEHP
Disallow: /nehp-themes/
Disallow: /nehp-topics/
#NGGDPP NATIONAL GEOGRAPHIC AND GEOSPATIAL DATA PRESERVATION PROGRAM
Disallow: /nggdpp-themes/
Disallow: /nggdpp-topics/
#NHMA
Disallow:	/nhma-themes/
Disallow:	/nhma-topics/
#NJ-WATER
Disallow:	/nj-water-themes/
Disallow:	/nj-water-topics/
#NPWRC
Disallow: /npwrc-themes/
Disallow: /npwrc-topics/
#NWHC
Disallow:	/nwhc-themes/
Disallow:	/nwhc-topics/
#NV-WATER
Disallow:	/nv-water-themes/
Disallow:	/nv-water-topics/
#NWQL
Disallow:	/nwql-themes/
Disallow:	/nwql-topics/
Disallow:	/nwql-capability-themes/
Disallow:	/nwql-capabilities-topics/
#NWQP NATIONAL WATER QUALITY PROGRAM
Disallow: /nwqp-themes/
Disallow: /nwqp-topics/
#PA-WATER
Disallow:	/pa-water-themes/
Disallow:	/pa-water-topics/
#PCMSC
Disallow:	/pcmsc-themes/
Disallow:	/pcmsc-topics/
#PIWSC
Disallow:	/piwsc-themes/
Disallow:	/piwsc-topics/
#POWELL CENTER
Disallow: /powell-ctr-themes/
Disallow: /powell-ctr-topics/
#MD-DE-DC
Disallow: /md-de-dc-water-themes/
Disallow: /md-de-dc-water-topics/
#OR-WATER
Disallow:	/or-water-themes/
Disallow:	/or-water-topics/
#ORGANIC MATTER RESEARCH LAB OMRL
Disallow: /omrl-themes/
Disallow: /omrl-topics/
Disallow:	/omrl-capability-themes/
Disallow:	/omrl-capabilities-topics/
#PWRC
Disallow:	/pwrc-themes/
Disallow:	/pwrc-topics/
#RBPGL
Disallow: /rbpgl-themes/
Disallow: /rbpgl-topics/
#RSP REMOTE SENSING PHENOLOGY
Disallow: /phenology-themes/
Disallow: /phenology-topics/
#SAFRR
Disallow: /safrr-themes/
Disallow: /safrr-topics/
#SDC
Disallow: /sdc-themes/
Disallow: /sdc-topics/
#SPCMSC
Disallow: /spcmsc-themes/
Disallow: /spcmsc-topics/
#SPECLAB
Disallow: /spec-lab-themes/
Disallow: /spec-lab-topics/
Disallow: /spec-lab-capability-themes/
Disallow:	/spec-lab-capabilities-topics/
#TX-WATER
Disallow: /tx-water-themes/
Disallow: /tx-water-topics/
#TOXIC SUBSTANCES HYDROLOGY PROGRAM
Disallow: /tshp-themes/
Disallow: /tshp-topics/
Disallow: /tshp-non-theme-topics/
#UMID-WATER
Disallow: /umid-water-themes/
Disallow: /umid-water-topics/
#UT-WATER
Disallow: /ut-water-themes/
Disallow: /ut-water-topics/
#VA-WV
Disallow: /va-wv-water-themes/
Disallow: /va-wv-water-topics/
#WA-WATER
Disallow: /wa-water-themes/
Disallow: /wa-water-topics/
#WAUSP WATER AVAILABILITY AND USE SCIENCE PROGRAM
Disallow: /wausp-themes/
Disallow: /wausp-topics/
#WATER SCIENCE SCHOOL
Disallow: /water-school-themes/
Disallow: /water-school-topics/
#WERC
Disallow: /werc-themes/
Disallow: /werc-topics/
#WFRC
Disallow: /wfrc-themes/
Disallow: /wfrc-topics/
#WGSC
Disallow: /wgsc-themes/
Disallow: /wgsc-topics/
#WHCMSC
Disallow: /whcmsc-themes/
Disallow: /whcmsc-topics/
#WMA
Disallow: /wma-themes/
Disallow: /wma-topics/
#WRRP AKA WRRI
Disallow: /wrri-themes/
Disallow: /wrri-topics/
#WY-MT-WATER
Disallow: /wy-mt-water-themes/
Disallow: /wy-mt-water-topics/`
#FOIA-FAQ
Disallow: /faqs/how-do-i-check-status-my-us-geological-survey-freedom-information-act-foia-request


#
# robots.txt
#
# This file is to prevent the crawling and indexing of certain parts
# of your site by web crawlers and spiders run by sites like Yahoo!
# and Google. By telling these "robots" where not to go on your site,
# you save bandwidth and server resources.
#
# This file will be ignored unless it is at the root of your host:
# Used:    http://example.com/robots.txt
# Ignored: http://example.com/site/robots.txt
#
# For more information about the robots.txt standard, see:
# http://www.robotstxt.org/robotstxt.html

User-agent: *
# CSS, JS, Images
Allow: /core/*.css$
Allow: /core/*.css?
Allow: /core/*.js$
Allow: /core/*.js?
Allow: /core/*.gif
Allow: /core/*.jpg
Allow: /core/*.jpeg
Allow: /core/*.png
Allow: /core/*.svg
Allow: /profiles/*.css$
Allow: /profiles/*.css?
Allow: /profiles/*.js$
Allow: /profiles/*.js?
Allow: /profiles/*.gif
Allow: /profiles/*.jpg
Allow: /profiles/*.jpeg
Allow: /profiles/*.png
Allow: /profiles/*.svg
# Directories
Disallow: /core/
Disallow: /profiles/
# Files
Disallow: /README.txt
Disallow: /web.config
# Paths (clean URLs)
Disallow: /admin/
Disallow: /comment/reply/
Disallow: /filter/tips
Disallow: /node/add/
Disallow: /search/
Disallow: /user/register
Disallow: /user/password
Disallow: /user/login
Disallow: /user/logout
# Paths (no clean URLs)
Disallow: /index.php/admin/
Disallow: /index.php/comment/reply/
Disallow: /index.php/filter/tips
Disallow: /index.php/node/add/
Disallow: /index.php/search/
Disallow: /index.php/user/password
Disallow: /index.php/user/register
Disallow: /index.php/user/login
Disallow: /index.php/user/logout


#
# robots.txt
#
# This file is to prevent the crawling and indexing of certain parts
# of your site by web crawlers and spiders run by sites like Yahoo!
# and Google. By telling these "robots" where not to go on your site,
# you save bandwidth and server resources.
#
# This file will be ignored unless it is at the root of your host:
# Used:    http://example.com/robots.txt
# Ignored: http://example.com/site/robots.txt
#
# For more information about the robots.txt standard, see:
# http://www.robotstxt.org/robotstxt.html

User-agent: *
# CSS, JS, Images
Allow: /core/*.css$
Allow: /core/*.css?
Allow: /core/*.js$
Allow: /core/*.js?
Allow: /core/*.gif
Allow: /core/*.jpg
Allow: /core/*.jpeg
Allow: /core/*.png
Allow: /core/*.svg
Allow: /profiles/*.css$
Allow: /profiles/*.css?
Allow: /profiles/*.js$
Allow: /profiles/*.js?
Allow: /profiles/*.gif
Allow: /profiles/*.jpg
Allow: /profiles/*.jpeg
Allow: /profiles/*.png
Allow: /profiles/*.svg
# Directories
Disallow: /core/
Disallow: /profiles/
# Files
Disallow: /README.txt
Disallow: /web.config
# Paths (clean URLs)
Disallow: /admin/
Disallow: /comment/reply/
Disallow: /filter/tips
Disallow: /node/add/
Disallow: /search/
Disallow: /user/register/
Disallow: /user/password/
Disallow: /user/login/
Disallow: /user/logout/
# Paths (no clean URLs)
Disallow: /index.php/admin/
Disallow: /index.php/comment/reply/
Disallow: /index.php/filter/tips
Disallow: /index.php/node/add/
Disallow: /index.php/search/
Disallow: /index.php/user/password/
Disallow: /index.php/user/register/
Disallow: /index.php/user/login/
Disallow: /index.php/user/logout/


User-agent: *
Disallow: /


#
# robots.txt
#
# This file is to prevent the crawling and indexing of certain parts
# of your site by web crawlers and spiders run by sites like Yahoo!
# and Google. By telling these "robots" where not to go on your site,
# you save bandwidth and server resources.
#
# This file will be ignored unless it is at the root of your host:
# Used:    http://example.com/robots.txt
# Ignored: http://example.com/site/robots.txt
#
# For more information about the robots.txt standard, see:
# http://www.robotstxt.org/robotstxt.html

User-agent: *
# CSS, JS, Images
Allow: /core/*.css$
Allow: /core/*.css?
Allow: /core/*.js$
Allow: /core/*.js?
Allow: /core/*.gif
Allow: /core/*.jpg
Allow: /core/*.jpeg
Allow: /core/*.png
Allow: /core/*.svg
Allow: /profiles/*.css$
Allow: /profiles/*.css?
Allow: /profiles/*.js$
Allow: /profiles/*.js?
Allow: /profiles/*.gif
Allow: /profiles/*.jpg
Allow: /profiles/*.jpeg
Allow: /profiles/*.png
Allow: /profiles/*.svg
# Directories
Disallow: /core/
Disallow: /profiles/
# Files
Disallow: /README.txt
Disallow: /web.config
# Paths (clean URLs)
Disallow: /admin/
Disallow: /comment/reply/
Disallow: /filter/tips
Disallow: /node/add/
Disallow: /search/
Disallow: /user/register/
Disallow: /user/password/
Disallow: /user/login/
Disallow: /user/logout/
# Paths (no clean URLs)
Disallow: /index.php/admin/
Disallow: /index.php/comment/reply/
Disallow: /index.php/filter/tips
Disallow: /index.php/node/add/
Disallow: /index.php/search/
Disallow: /index.php/user/password/
Disallow: /index.php/user/register/
Disallow: /index.php/user/login/
Disallow: /index.php/user/logout/

Sitemap: https://www.usgs.gov/sitemap.xml


# This space intentionally left blank
User-Agent: *
Disallow: /wp-admin/*

Sitemap: https://www.vvsor.nl/sitemap_index.xml


### Version Information #
###################################################
### Version: V3.2018.05.828
### Updated: Tue May  8 11:37:04 SAST 2018
### Bad Bot Count: 527
###################################################
### Version Information ##

User-agent: 360Spider
Disallow:/
User-agent: 404checker
Disallow:/
User-agent: 404enemy
Disallow:/
User-agent: 80legs
Disallow:/
User-agent: Abonti
Disallow:/
User-agent: Aboundex
Disallow:/
User-agent: Acunetix
Disallow:/
User-agent: ADmantX
Disallow:/
User-agent: AfD-Verbotsverfahren
Disallow:/
User-agent: AhrefsBot
Disallow:/
User-agent: AIBOT
Disallow:/
User-agent: AiHitBot
Disallow:/
User-agent: Aipbot
Disallow:/
User-agent: Alexibot
Disallow:/
User-agent: Alligator
Disallow:/
User-agent: AllSubmitter
Disallow:/
User-agent: AlphaBot
Disallow:/
User-agent: Anarchie
Disallow:/
User-agent: Apexoo
Disallow:/
User-agent: ASPSeek
Disallow:/
User-agent: Asterias
Disallow:/
User-agent: Attach
Disallow:/
User-agent: autoemailspider
Disallow:/
User-agent: BackDoorBot
Disallow:/
User-agent: Backlink-Ceck
Disallow:/
User-agent: backlink-check
Disallow:/
User-agent: BacklinkCrawler
Disallow:/
User-agent: BackStreet
Disallow:/
User-agent: BackWeb
Disallow:/
User-agent: Badass
Disallow:/
User-agent: Bandit
Disallow:/
User-agent: Barkrowler
Disallow:/
User-agent: BatchFTP
Disallow:/
User-agent: Battleztar Bazinga
Disallow:/
User-agent: BBBike
Disallow:/
User-agent: BDCbot
Disallow:/
User-agent: BDFetch
Disallow:/
User-agent: BetaBot
Disallow:/
User-agent: Bigfoot
Disallow:/
User-agent: Bitacle
Disallow:/
User-agent: Blackboard
Disallow:/
User-agent: Black Hole
Disallow:/
User-agent: BlackWidow
Disallow:/
User-agent: BLEXBot
Disallow:/
User-agent: Blow
Disallow:/
User-agent: BlowFish
Disallow:/
User-agent: Boardreader
Disallow:/
User-agent: Bolt
Disallow:/
User-agent: BotALot
Disallow:/
User-agent: Brandprotect
Disallow:/
User-agent: Brandwatch
Disallow:/
User-agent: Bubing
Disallow:/
User-agent: Buddy
Disallow:/
User-agent: BuiltBotTough
Disallow:/
User-agent: BuiltWith
Disallow:/
User-agent: Bullseye
Disallow:/
User-agent: BunnySlippers
Disallow:/
User-agent: BuzzSumo
Disallow:/
User-agent: Calculon
Disallow:/
User-agent: CATExplorador
Disallow:/
User-agent: CazoodleBot
Disallow:/
User-agent: CCBot
Disallow:/
User-agent: Cegbfeieh
Disallow:/
User-agent: CheeseBot
Disallow:/
User-agent: CherryPicker
Disallow:/
User-agent: ChinaClaw
Disallow:/
User-agent: Chlooe
Disallow:/
User-agent: Claritybot
Disallow:/
User-agent: Cliqzbot
Disallow:/
User-agent: Cloud mapping
Disallow:/
User-agent: coccocbot-web
Disallow:/
User-agent: Cogentbot
Disallow:/
User-agent: cognitiveseo
Disallow:/
User-agent: Collector
Disallow:/
User-agent: com.plumanalytics
Disallow:/
User-agent: Copier
Disallow:/
User-agent: CopyRightCheck
Disallow:/
User-agent: Copyscape
Disallow:/
User-agent: Cosmos
Disallow:/
User-agent: Craftbot
Disallow:/
User-agent: crawler4j
Disallow:/
User-agent: crawler.feedback
Disallow:/
User-agent: CrazyWebCrawler
Disallow:/
User-agent: Crescent
Disallow:/
User-agent: CSHttp
Disallow:/
User-agent: Curious
Disallow:/
User-agent: Custo
Disallow:/
User-agent: DatabaseDriverMysqli
Disallow:/
User-agent: DataCha0s
Disallow:/
User-agent: DBLBot
Disallow:/
User-agent: demandbase-bot
Disallow:/
User-agent: Demon
Disallow:/
User-agent: Deusu
Disallow:/
User-agent: Devil
Disallow:/
User-agent: Digincore
Disallow:/
User-agent: DigitalPebble
Disallow:/
User-agent: DIIbot
Disallow:/
User-agent: Dirbuster
Disallow:/
User-agent: Disco
Disallow:/
User-agent: Discobot
Disallow:/
User-agent: Discoverybot
Disallow:/
User-agent: DittoSpyder
Disallow:/
User-agent: DnyzBot
Disallow:/
User-agent: DomainAppender
Disallow:/
User-agent: DomainCrawler
Disallow:/
User-agent: DomainSigmaCrawler
Disallow:/
User-agent: DomainStatsBot
Disallow:/
User-agent: Dotbot
Disallow:/
User-agent: Download Wonder
Disallow:/
User-agent: Dragonfly
Disallow:/
User-agent: Drip
Disallow:/
User-agent: DTS Agent
Disallow:/
User-agent: EasyDL
Disallow:/
User-agent: Ebingbong
Disallow:/
User-agent: eCatch
Disallow:/
User-agent: ECCP/1.0
Disallow:/
User-agent: Ecxi
Disallow:/
User-agent: EirGrabber
Disallow:/
User-agent: EMail Siphon
Disallow:/
User-agent: EMail Wolf
Disallow:/
User-agent: EroCrawler
Disallow:/
User-agent: evc-batch
Disallow:/
User-agent: Evil
Disallow:/
User-agent: Exabot
Disallow:/
User-agent: Express WebPictures
Disallow:/
User-agent: ExtLinksBot
Disallow:/
User-agent: Extractor
Disallow:/
User-agent: ExtractorPro
Disallow:/
User-agent: Extreme Picture Finder
Disallow:/
User-agent: EyeNetIE
Disallow:/
User-agent: Ezooms
Disallow:/
User-agent: FDM
Disallow:/
User-agent: FemtosearchBot
Disallow:/
User-agent: FHscan
Disallow:/
User-agent: Fimap
Disallow:/
User-agent: Firefox/7.0
Disallow:/
User-agent: FlashGet
Disallow:/
User-agent: Flunky
Disallow:/
User-agent: Foobot
Disallow:/
User-agent: fq
Disallow:/
User-agent: Freeuploader
Disallow:/
User-agent: FrontPage
Disallow:/
User-agent: Fyrebot
Disallow:/
User-agent: GalaxyBot
Disallow:/
User-agent: Genieo
Disallow:/
User-agent: GermCrawler
Disallow:/
User-agent: Getintent
Disallow:/
User-agent: GetRight
Disallow:/
User-agent: GetWeb
Disallow:/
User-agent: Gigablast
Disallow:/
User-agent: Gigabot
Disallow:/
User-agent: G-i-g-a-b-o-t
Disallow:/
User-agent: Go-Ahead-Got-It
Disallow:/
User-agent: Gotit
Disallow:/
User-agent: GoZilla
Disallow:/
User-agent: Go!Zilla
Disallow:/
User-agent: Grabber
Disallow:/
User-agent: GrabNet
Disallow:/
User-agent: Grafula
Disallow:/
User-agent: GrapeFX
Disallow:/
User-agent: GrapeshotCrawler
Disallow:/
User-agent: GridBot
Disallow:/
User-agent: GT::WWW
Disallow:/
User-agent: Haansoft
Disallow:/
User-agent: HaosouSpider
Disallow:/
User-agent: Harvest
Disallow:/
User-agent: Havij
Disallow:/
User-agent: HEADMasterSEO
Disallow:/
User-agent: Heritrix
Disallow:/
User-agent: Hloader
Disallow:/
User-agent: HMView
Disallow:/
User-agent: HTMLparser
Disallow:/
User-agent: HTTP::Lite
Disallow:/
User-agent: HTTrack
Disallow:/
User-agent: Humanlinks
Disallow:/
User-agent: HybridBot
Disallow:/
User-agent: Iblog
Disallow:/
User-agent: IDBot
Disallow:/
User-agent: Id-search
Disallow:/
User-agent: IlseBot
Disallow:/
User-agent: Image Fetch
Disallow:/
User-agent: Image Sucker
Disallow:/
User-agent: IndeedBot
Disallow:/
User-agent: Indy Library
Disallow:/
User-agent: InfoNaviRobot
Disallow:/
User-agent: InfoTekies
Disallow:/
User-agent: instabid
Disallow:/
User-agent: Intelliseek
Disallow:/
User-agent: InterGET
Disallow:/
User-agent: Internet Ninja
Disallow:/
User-agent: InternetSeer
Disallow:/
User-agent: internetVista monitor
Disallow:/
User-agent: ips-agent
Disallow:/
User-agent: Iria
Disallow:/
User-agent: IRLbot
Disallow:/
User-agent: Iskanie
Disallow:/
User-agent: IstellaBot
Disallow:/
User-agent: JamesBOT
Disallow:/
User-agent: Jbrofuzz
Disallow:/
User-agent: JennyBot
Disallow:/
User-agent: JetCar
Disallow:/
User-agent: JikeSpider
Disallow:/
User-agent: JOC Web Spider
Disallow:/
User-agent: Joomla
Disallow:/
User-agent: Jorgee
Disallow:/
User-agent: JustView
Disallow:/
User-agent: Jyxobot
Disallow:/
User-agent: Kenjin Spider
Disallow:/
User-agent: Keyword Density
Disallow:/
User-agent: Kozmosbot
Disallow:/
User-agent: Lanshanbot
Disallow:/
User-agent: Larbin
Disallow:/
User-agent: LeechFTP
Disallow:/
User-agent: LeechGet
Disallow:/
User-agent: LexiBot
Disallow:/
User-agent: Lftp
Disallow:/
User-agent: LibWeb
Disallow:/
User-agent: Libwhisker
Disallow:/
User-agent: Lightspeedsystems
Disallow:/
User-agent: Likse
Disallow:/
User-agent: Linkdexbot
Disallow:/
User-agent: LinkextractorPro
Disallow:/
User-agent: LinkpadBot
Disallow:/
User-agent: LinkScan
Disallow:/
User-agent: LinksManager
Disallow:/
User-agent: LinkWalker
Disallow:/
User-agent: LinqiaMetadataDownloaderBot
Disallow:/
User-agent: LinqiaRSSBot
Disallow:/
User-agent: LinqiaScrapeBot
Disallow:/
User-agent: Lipperhey
Disallow:/
User-agent: Litemage_walker
Disallow:/
User-agent: Lmspider
Disallow:/
User-agent: LNSpiderguy
Disallow:/
User-agent: Ltx71
Disallow:/
User-agent: lwp-request
Disallow:/
User-agent: LWP::Simple
Disallow:/
User-agent: lwp-trivial
Disallow:/
User-agent: Magnet
Disallow:/
User-agent: Mag-Net
Disallow:/
User-agent: magpie-crawler
Disallow:/
User-agent: Mail.RU_Bot
Disallow:/
User-agent: Majestic12
Disallow:/
User-agent: MarkMonitor
Disallow:/
User-agent: MarkWatch
Disallow:/
User-agent: Masscan
Disallow:/
User-agent: Mass Downloader
Disallow:/
User-agent: Mata Hari
Disallow:/
User-agent: MauiBot
Disallow:/
User-agent: Meanpathbot
Disallow:/
User-agent: mediawords
Disallow:/
User-agent: MegaIndex.ru
Disallow:/
User-agent: Metauri
Disallow:/
User-agent: MFC_Tear_Sample
Disallow:/
User-agent: Microsoft Data Access
Disallow:/
User-agent: Microsoft URL Control
Disallow:/
User-agent: MIDown tool
Disallow:/
User-agent: MIIxpc
Disallow:/
User-agent: Mister PiX
Disallow:/
User-agent: MJ12bot
Disallow:/
User-agent: Mojeek
Disallow:/
User-agent: Morfeus Fucking Scanner
Disallow:/
User-agent: MSFrontPage
Disallow:/
User-agent: MSIECrawler
Disallow:/
User-agent: Msrabot
Disallow:/
User-agent: MS Web Services Client Protocol
Disallow:/
User-agent: muhstik-scan
Disallow:/
User-agent: Musobot
Disallow:/
User-agent: Name Intelligence
Disallow:/
User-agent: Nameprotect
Disallow:/
User-agent: Navroad
Disallow:/
User-agent: NearSite
Disallow:/
User-agent: Needle
Disallow:/
User-agent: Nessus
Disallow:/
User-agent: NetAnts
Disallow:/
User-agent: Netcraft
Disallow:/
User-agent: netEstate NE Crawler
Disallow:/
User-agent: NetLyzer
Disallow:/
User-agent: NetMechanic
Disallow:/
User-agent: NetSpider
Disallow:/
User-agent: Nettrack
Disallow:/
User-agent: Net Vampire
Disallow:/
User-agent: Netvibes
Disallow:/
User-agent: NetZIP
Disallow:/
User-agent: NextGenSearchBot
Disallow:/
User-agent: Nibbler
Disallow:/
User-agent: NICErsPRO
Disallow:/
User-agent: Niki-bot
Disallow:/
User-agent: Nikto
Disallow:/
User-agent: NimbleCrawler
Disallow:/
User-agent: Ninja
Disallow:/
User-agent: Nmap
Disallow:/
User-agent: NPbot
Disallow:/
User-agent: Nutch
Disallow:/
User-agent: NYU
Disallow:/
User-agent: oBot
Disallow:/
User-agent: Octopus
Disallow:/
User-agent: Offline Explorer
Disallow:/
User-agent: Offline Navigator
Disallow:/
User-agent: Openfind
Disallow:/
User-agent: OpenLinkProfiler
Disallow:/
User-agent: Openvas
Disallow:/
User-agent: OrangeBot
Disallow:/
User-agent: OrangeSpider
Disallow:/
User-agent: OutclicksBot
Disallow:/
User-agent: OutfoxBot
Disallow:/
User-agent: PageAnalyzer
Disallow:/
User-agent: Page Analyzer
Disallow:/
User-agent: PageGrabber
Disallow:/
User-agent: page scorer
Disallow:/
User-agent: PageScorer
Disallow:/
User-agent: Panscient
Disallow:/
User-agent: Papa Foto
Disallow:/
User-agent: Pavuk
Disallow:/
User-agent: pcBrowser
Disallow:/
User-agent: PECL::HTTP
Disallow:/
User-agent: PeoplePal
Disallow:/
User-agent: PHPCrawl
Disallow:/
User-agent: Picscout
Disallow:/
User-agent: Picsearch
Disallow:/
User-agent: PictureFinder
Disallow:/
User-agent: Pimonster
Disallow:/
User-agent: Pi-Monster
Disallow:/
User-agent: Pixray
Disallow:/
User-agent: PleaseCrawl
Disallow:/
User-agent: plumanalytics
Disallow:/
User-agent: Pockey
Disallow:/
User-agent: POE-Component-Client-HTTP
Disallow:/
User-agent: Probethenet
Disallow:/
User-agent: ProPowerBot
Disallow:/
User-agent: ProWebWalker
Disallow:/
User-agent: Psbot
Disallow:/
User-agent: Pump
Disallow:/
User-agent: PyCurl
Disallow:/
User-agent: QueryN Metasearch
Disallow:/
User-agent: RankActive
Disallow:/
User-agent: RankActiveLinkBot
Disallow:/
User-agent: RankFlex
Disallow:/
User-agent: RankingBot
Disallow:/
User-agent: RankingBot2
Disallow:/
User-agent: Rankivabot
Disallow:/
User-agent: RankurBot
Disallow:/
User-agent: RealDownload
Disallow:/
User-agent: Reaper
Disallow:/
User-agent: RebelMouse
Disallow:/
User-agent: Recorder
Disallow:/
User-agent: RedesScrapy
Disallow:/
User-agent: ReGet
Disallow:/
User-agent: RepoMonkey
Disallow:/
User-agent: Ripper
Disallow:/
User-agent: RocketCrawler
Disallow:/
User-agent: Rogerbot
Disallow:/
User-agent: SalesIntelligent
Disallow:/
User-agent: SBIder
Disallow:/
User-agent: ScanAlert
Disallow:/
User-agent: Scanbot
Disallow:/
User-agent: scan.lol
Disallow:/
User-agent: Scrapy
Disallow:/
User-agent: Screaming
Disallow:/
User-agent: ScreenerBot
Disallow:/
User-agent: Searchestate
Disallow:/
User-agent: SearchmetricsBot
Disallow:/
User-agent: Semrush
Disallow:/
User-agent: SemrushBot
Disallow:/
User-agent: SEOkicks
Disallow:/
User-agent: SEOlyticsCrawler
Disallow:/
User-agent: Seomoz
Disallow:/
User-agent: SEOprofiler
Disallow:/
User-agent: seoscanners
Disallow:/
User-agent: SEOstats
Disallow:/
User-agent: sexsearcher
Disallow:/
User-agent: Seznam
Disallow:/
User-agent: SeznamBot
Disallow:/
User-agent: Shodan
Disallow:/
User-agent: Siphon
Disallow:/
User-agent: SISTRIX
Disallow:/
User-agent: Sitebeam
Disallow:/
User-agent: SiteExplorer
Disallow:/
User-agent: Siteimprove
Disallow:/
User-agent: SiteLockSpider
Disallow:/
User-agent: SiteSnagger
Disallow:/
User-agent: SiteSucker
Disallow:/
User-agent: Site Sucker
Disallow:/
User-agent: Sitevigil
Disallow:/
User-agent: Slackbot-LinkExpanding
Disallow:/
User-agent: SlySearch
Disallow:/
User-agent: SmartDownload
Disallow:/
User-agent: SMTBot
Disallow:/
User-agent: Snake
Disallow:/
User-agent: Snapbot
Disallow:/
User-agent: Snoopy
Disallow:/
User-agent: SocialRankIOBot
Disallow:/
User-agent: Sogou web spider
Disallow:/
User-agent: Sosospider
Disallow:/
User-agent: Sottopop
Disallow:/
User-agent: SpaceBison
Disallow:/
User-agent: Spammen
Disallow:/
User-agent: SpankBot
Disallow:/
User-agent: Spanner
Disallow:/
User-agent: Spbot
Disallow:/
User-agent: Spinn3r
Disallow:/
User-agent: SputnikBot
Disallow:/
User-agent: Sqlmap
Disallow:/
User-agent: Sqlworm
Disallow:/
User-agent: Sqworm
Disallow:/
User-agent: Steeler
Disallow:/
User-agent: Stripper
Disallow:/
User-agent: Sucker
Disallow:/
User-agent: Sucuri
Disallow:/
User-agent: SuperBot
Disallow:/
User-agent: SuperHTTP
Disallow:/
User-agent: Surfbot
Disallow:/
User-agent: SurveyBot
Disallow:/
User-agent: Suzuran
Disallow:/
User-agent: Swiftbot
Disallow:/
User-agent: sysscan
Disallow:/
User-agent: Szukacz
Disallow:/
User-agent: T0PHackTeam
Disallow:/
User-agent: T8Abot
Disallow:/
User-agent: tAkeOut
Disallow:/
User-agent: Teleport
Disallow:/
User-agent: TeleportPro
Disallow:/
User-agent: Telesoft
Disallow:/
User-agent: Telesphoreo
Disallow:/
User-agent: Telesphorep
Disallow:/
User-agent: The Intraformant
Disallow:/
User-agent: TheNomad
Disallow:/
User-agent: TightTwatBot
Disallow:/
User-agent: Titan
Disallow:/
User-agent: Toata
Disallow:/
User-agent: Toweyabot
Disallow:/
User-agent: Trendiction
Disallow:/
User-agent: Trendictionbot
Disallow:/
User-agent: trendiction.com
Disallow:/
User-agent: trendiction.de
Disallow:/
User-agent: True_Robot
Disallow:/
User-agent: Turingos
Disallow:/
User-agent: Turnitin
Disallow:/
User-agent: TurnitinBot
Disallow:/
User-agent: TwengaBot
Disallow:/
User-agent: Twice
Disallow:/
User-agent: Typhoeus
Disallow:/
User-agent: UnisterBot
Disallow:/
User-agent: URLy.Warning
Disallow:/
User-agent: URLy Warning
Disallow:/
User-agent: Vacuum
Disallow:/
User-agent: Vagabondo
Disallow:/
User-agent: VB Project
Disallow:/
User-agent: VCI
Disallow:/
User-agent: VeriCiteCrawler
Disallow:/
User-agent: VidibleScraper
Disallow:/
User-agent: Virusdie
Disallow:/
User-agent: VoidEYE
Disallow:/
User-agent: Voil
Disallow:/
User-agent: Voltron
Disallow:/
User-agent: Wallpapers/3.0
Disallow:/
User-agent: WallpapersHD
Disallow:/
User-agent: WASALive-Bot
Disallow:/
User-agent: WBSearchBot
Disallow:/
User-agent: Webalta
Disallow:/
User-agent: WebAuto
Disallow:/
User-agent: Web Auto
Disallow:/
User-agent: WebBandit
Disallow:/
User-agent: WebCollage
Disallow:/
User-agent: Web Collage
Disallow:/
User-agent: WebCopier
Disallow:/
User-agent: WEBDAV
Disallow:/
User-agent: WebEnhancer
Disallow:/
User-agent: Web Enhancer
Disallow:/
User-agent: WebFetch
Disallow:/
User-agent: Web Fetch
Disallow:/
User-agent: WebFuck
Disallow:/
User-agent: Web Fuck
Disallow:/
User-agent: WebGo IS
Disallow:/
User-agent: WebImageCollector
Disallow:/
User-agent: WebLeacher
Disallow:/
User-agent: WebmasterWorldForumBot
Disallow:/
User-agent: webmeup-crawler
Disallow:/
User-agent: WebPix
Disallow:/
User-agent: Web Pix
Disallow:/
User-agent: WebReaper
Disallow:/
User-agent: WebSauger
Disallow:/
User-agent: Web Sauger
Disallow:/
User-agent: Webshag
Disallow:/
User-agent: WebsiteExtractor
Disallow:/
User-agent: WebsiteQuester
Disallow:/
User-agent: Website Quester
Disallow:/
User-agent: Webster
Disallow:/
User-agent: WebStripper
Disallow:/
User-agent: WebSucker
Disallow:/
User-agent: Web Sucker
Disallow:/
User-agent: WebWhacker
Disallow:/
User-agent: WebZIP
Disallow:/
User-agent: WeSEE
Disallow:/
User-agent: Whack
Disallow:/
User-agent: Whacker
Disallow:/
User-agent: Whatweb
Disallow:/
User-agent: Who.is Bot
Disallow:/
User-agent: Widow
Disallow:/
User-agent: WinHTTrack
Disallow:/
User-agent: WiseGuys Robot
Disallow:/
User-agent: WISENutbot
Disallow:/
User-agent: Wonderbot
Disallow:/
User-agent: Woobot
Disallow:/
User-agent: Wotbox
Disallow:/
User-agent: Wprecon
Disallow:/
User-agent: WPScan
Disallow:/
User-agent: WWW-Collector-E
Disallow:/
User-agent: WWW-Mechanize
Disallow:/
User-agent: WWW::Mechanize
Disallow:/
User-agent: WWWOFFLE
Disallow:/
User-agent: x09Mozilla
Disallow:/
User-agent: x22Mozilla
Disallow:/
User-agent: Xaldon_WebSpider
Disallow:/
User-agent: Xaldon WebSpider
Disallow:/
User-agent: Xenu
Disallow:/
User-agent: xpymep1.exe
Disallow:/
User-agent: YoudaoBot
Disallow:/
User-agent: Zade
Disallow:/
User-agent: Zauba
Disallow:/
User-agent: zauba.io
Disallow:/
User-agent: Zermelo
Disallow:/
User-agent: Zeus
Disallow:/
User-agent: zgrab
Disallow:/
User-agent: Zitebot
Disallow:/
User-agent: ZmEu
Disallow:/
User-agent: ZumBot
Disallow:/
User-agent: ZyBorg
Disallow:/

Sitemap: http://www.who.int/sitemaps/sitemapindex.xml


User-agent: *

User-agent: W3C-checklink
Disallow: /cgi-bin/
Disallow: /libs/
Disallow: /tmp/
Disallow: /etc/

Disallow: /about/adrm/data-linkage/
allow: /etc.clientlibs/census/clientlibs
allow: /etc/clientlibs/granite
allow: /etc/clientlibs/foundation

User-agent: Googlebot
Crawl-delay: 3
Disallow: /cgi-bin/
Disallow: /libs/
Disallow: /tmp/
Disallow: /etc/

Disallow: /about/adrm/data-linkage/
allow: /etc.clientlibs/census/clientlibs
allow: /etc/clientlibs/granite
allow: /etc/clientlibs/foundation

User-agent: Yahoo! Slurp
Crawl-delay: 3
Disallow: /cgi-bin/
Disallow: /libs/
Disallow: /tmp/
Disallow: /etc/

Disallow: /about/adrm/data-linkage/
allow: /etc.clientlibs/census/clientlibs
allow: /etc/clientlibs/granite
allow: /etc/clientlibs/foundation

User-agent: bingbot
Crawl-delay: 3
Disallow: /cgi-bin/
Disallow: /libs/
Disallow: /tmp/
Disallow: /etc/

Disallow: /about/adrm/data-linkage/
allow: /etc.clientlibs/census/clientlibs
allow: /etc/clientlibs/granite
allow: /etc/clientlibs/foundation

User-agent: usasearch
SITEMAP: https://www.census.gov/sitemapindex/sitemap.xml
SITEMAP: https://www.census.gov/quickfacts/fact/sitemap/US/PST045217
SITEMAP: https://2020census.gov/en/sitemapindex/sitemap.xml


User-agent: *

User-agent: W3C-checklink
Disallow: /cgi-bin/
Disallow: /libs/
Disallow: /tmp/
Disallow: /etc/
Disallow: /about/adrm/data-linkage/
allow: /etc.clientlibs/census/clientlibs
allow: /etc/clientlibs/granite
allow: /etc/clientlibs/foundation

User-agent: Googlebot
Crawl-delay: 3
Disallow: /cgi-bin/
Disallow: /libs/
Disallow: /tmp/
Disallow: /etc/
Disallow: /about/adrm/data-linkage/
allow: /etc.clientlibs/census/clientlibs
allow: /etc/clientlibs/granite
allow: /etc/clientlibs/foundation

User-agent: Yahoo! Slurp
Crawl-delay: 3
Disallow: /cgi-bin/
Disallow: /libs/
Disallow: /tmp/
Disallow: /etc/
Disallow: /about/adrm/data-linkage/
allow: /etc.clientlibs/census/clientlibs
allow: /etc/clientlibs/granite
allow: /etc/clientlibs/foundation

User-agent: bingbot
Crawl-delay: 3
Disallow: /cgi-bin/
Disallow: /libs/
Disallow: /tmp/
Disallow: /etc/
Disallow: /about/adrm/data-linkage/
allow: /etc.clientlibs/census/clientlibs
allow: /etc/clientlibs/granite
allow: /etc/clientlibs/foundation

User-agent: usasearch
SITEMAP: https://www.census.gov/sitemapindex/sitemap.xml
SITEMAP: https://www.census.gov/quickfacts/fact/sitemap/US/PST045217
SITEMAP: https://2020census.gov/en/sitemapindex/sitemap.xml


User-agent: *

User-agent: RavenCrawler
Disallow: /

User-agent: MegaIndex
Disallow: /

User-agent: MegaIndex.ru/2.0
Disallow: /

User-agent: BLP_bbot/0.1
Disallow: /

User-agent: BLP_bbot
Disallow: /

User-agent: TweetmemeBot
Disallow: /

User-agent: Applebot
Crawl-delay: 30
Disallow: /cgi-bin/

User-agent: Googlebot
Crawl-delay: 30
Disallow: /cgi-bin/

User-agent: Yahoo! Slurp
Crawl-delay: 30
Disallow: /cgi-bin/

User-agent: bingbot
Crawl-delay: 30
Disallow: /cgi-bin/


User-agent: *

User-agent: W3C-checklink
Disallow: /cgi-bin/
Disallow: /libs/
Disallow: /tmp/
Disallow: /etc/

Disallow: /about/adrm/data-linkage/
allow: /etc.clientlibs/census/clientlibs
allow: /etc/clientlibs/granite
allow: /etc/clientlibs/foundation

User-agent: Googlebot
Crawl-delay: 3
Disallow: /cgi-bin/
Disallow: /libs/
Disallow: /tmp/
Disallow: /etc/

Disallow: /about/adrm/data-linkage/
allow: /etc.clientlibs/census/clientlibs
allow: /etc/clientlibs/granite
allow: /etc/clientlibs/foundation

User-agent: Yahoo! Slurp
Crawl-delay: 3
Disallow: /cgi-bin/
Disallow: /libs/
Disallow: /tmp/
Disallow: /etc/

Disallow: /about/adrm/data-linkage/
allow: /etc.clientlibs/census/clientlibs
allow: /etc/clientlibs/granite
allow: /etc/clientlibs/foundation

User-agent: bingbot
Crawl-delay: 3
Disallow: /cgi-bin/
Disallow: /libs/
Disallow: /tmp/
Disallow: /etc/

Disallow: /about/adrm/data-linkage/
allow: /etc.clientlibs/census/clientlibs
allow: /etc/clientlibs/granite
allow: /etc/clientlibs/foundation

User-agent: usasearch
SITEMAP: https://www.census.gov/sitemapindex/sitemap.xml
SITEMAP: https://www.census.gov/quickfacts/fact/sitemap/US/PST045217


User-agent: *

User-agent: RavenCrawler
Disallow: /

User-agent: MegaIndex
Disallow: /

User-agent: MegaIndex.ru/2.0
Disallow: /

User-agent: BLP_bbot/0.1
Disallow: /

User-agent: BLP_bbot
Disallow: /

User-agent: TweetmemeBot
Disallow: /

User-agent: Applebot
Crawl-delay: 30

User-agent: Googlebot
Crawl-delay: 30

User-agent: Yahoo! Slurp
Crawl-delay: 30

User-agent: bingbot
Crawl-delay: 30


User-agent: *

User-agent: W3C-checklink
Disallow: /cgi-bin/
Disallow: /libs/
Disallow: /tmp/
Disallow: /etc/

Disallow: /about/adrm/data-linkage/
allow: /etc.clientlibs/census/clientlibs
allow: /etc/clientlibs/granite
allow: /etc/clientlibs/foundation

User-agent: Googlebot
Crawl-delay: 15
Disallow: /cgi-bin/
Disallow: /libs/
Disallow: /tmp/
Disallow: /etc/

Disallow: /about/adrm/data-linkage/
allow: /etc.clientlibs/census/clientlibs
allow: /etc/clientlibs/granite
allow: /etc/clientlibs/foundation

User-agent: Yahoo! Slurp
Crawl-delay: 3
Disallow: /cgi-bin/
Disallow: /libs/
Disallow: /tmp/
Disallow: /etc/

Disallow: /about/adrm/data-linkage/
allow: /etc.clientlibs/census/clientlibs
allow: /etc/clientlibs/granite
allow: /etc/clientlibs/foundation

User-agent: bingbot
Crawl-delay: 3
Disallow: /cgi-bin/
Disallow: /libs/
Disallow: /tmp/
Disallow: /etc/

Disallow: /about/adrm/data-linkage/
allow: /etc.clientlibs/census/clientlibs
allow: /etc/clientlibs/granite
allow: /etc/clientlibs/foundation

User-agent: usasearch
SITEMAP: https://www.census.gov/sitemapindex/sitemap.xml
SITEMAP: https://www.census.gov/quickfacts/fact/sitemap/US/PST045217


User-agent: *

User-agent: W3C-checklink
Disallow: /cgi-bin/
Disallow: /libs/
Disallow: /tmp/
Disallow: /etc/

Disallow: /about/adrm/data-linkage/
allow: /etc.clientlibs/census/clientlibs
allow: /etc/clientlibs/granite
allow: /etc/clientlibs/foundation

User-agent: Googlebot
Crawl-delay: 15
Disallow: /cgi-bin/
Disallow: /libs/
Disallow: /tmp/
Disallow: /etc/

Disallow: /about/adrm/data-linkage/
allow: /etc.clientlibs/census/clientlibs
allow: /etc/clientlibs/granite
allow: /etc/clientlibs/foundation

User-agent: Yahoo! Slurp
Crawl-delay: 3
Disallow: /cgi-bin/
Disallow: /libs/
Disallow: /tmp/
Disallow: /etc/

Disallow: /about/adrm/data-linkage/
allow: /etc.clientlibs/census/clientlibs
allow: /etc/clientlibs/granite
allow: /etc/clientlibs/foundation

User-agent: bingbot
Crawl-delay: 3
Disallow: /cgi-bin/
Disallow: /libs/
Disallow: /tmp/
Disallow: /etc/

Disallow: /about/adrm/data-linkage/
allow: /etc.clientlibs/census/clientlibs
allow: /etc/clientlibs/granite
allow: /etc/clientlibs/foundation

User-agent: usasearch
SITEMAP: https://www.census.gov/sitemapindex/sitemap.xml


User-agent: *

User-agent: W3C-checklink
Disallow: /cgi-bin/
Disallow: /libs/
Disallow: /tmp/
Disallow: /etc/

Disallow: /about/adrm/data-linkage/
allow: /etc.clientlibs/census/clientlibs
allow: /etc/clientlibs/granite
allow: /etc/clientlibs/foundation

User-agent: Googlebot
Crawl-delay: 15
Disallow: /cgi-bin/
Disallow: /libs/
Disallow: /tmp/
Disallow: /etc/

Disallow: /about/adrm/data-linkage/
allow: /etc.clientlibs/census/clientlibs
allow: /etc/clientlibs/granite
allow: /etc/clientlibs/foundation

User-agent: Yahoo! Slurp
Crawl-delay: 3
Disallow: /cgi-bin/
Disallow: /libs/
Disallow: /tmp/
Disallow: /etc/

Disallow: /about/adrm/data-linkage/
allow: /etc.clientlibs/census/clientlibs
allow: /etc/clientlibs/granite
allow: /etc/clientlibs/foundation

User-agent: bingbot
Crawl-delay: 3
Disallow: /cgi-bin/
Disallow: /libs/
Disallow: /tmp/
Disallow: /etc/

Disallow: /about/adrm/data-linkage/
allow: /etc.clientlibs/census/clientlibs
allow: /etc/clientlibs/granite
allow: /etc/clientlibs/foundation

User-agent: usasearch
Disallow: /quickfacts/
Noindex: /quickfacts/

SITEMAP: https://www.census.gov/sitemapindex/sitemap.xml


User-agent: *
Disallow: /search-results.html

User-agent: W3C-checklink
Disallow: /cgi-bin/
Disallow: /libs/
Disallow: /tmp/
Disallow: /etc/

Disallow: /about/adrm/data-linkage/
allow: /etc.clientlibs/census/clientlibs
allow: /etc/clientlibs/granite
allow: /etc/clientlibs/foundation

User-agent: Googlebot
Crawl-delay: 15
Disallow: /cgi-bin/
Disallow: /libs/
Disallow: /tmp/
Disallow: /etc/

Disallow: /about/adrm/data-linkage/
allow: /etc.clientlibs/census/clientlibs
allow: /etc/clientlibs/granite
allow: /etc/clientlibs/foundation

User-agent: Yahoo! Slurp
Crawl-delay: 3
Disallow: /cgi-bin/
Disallow: /libs/
Disallow: /tmp/
Disallow: /etc/

Disallow: /about/adrm/data-linkage/
allow: /etc.clientlibs/census/clientlibs
allow: /etc/clientlibs/granite
allow: /etc/clientlibs/foundation

User-agent: bingbot
Crawl-delay: 3
Disallow: /cgi-bin/
Disallow: /libs/
Disallow: /tmp/
Disallow: /etc/

Disallow: /about/adrm/data-linkage/
allow: /etc.clientlibs/census/clientlibs
allow: /etc/clientlibs/granite
allow: /etc/clientlibs/foundation

User-agent: usasearch
Disallow: /quickfacts/
Noindex: /quickfacts/

SITEMAP: https://www.census.gov/sitemapindex/sitemap.xml


# robots.txt for http://www.wikipedia.org/ and friends
#
# Please note: There are a lot of pages on this site, and there are
# some misbehaved spiders out there that go _way_ too fast. If you're
# irresponsible, your access to the site may be blocked.
#

# Observed spamming large amounts of https://en.wikipedia.org/?curid=NNNNNN
# and ignoring 429 ratelimit responses, claims to respect robots:
# http://mj12bot.com/
User-agent: MJ12bot
Disallow: /

# advertising-related bots:
User-agent: Mediapartners-Google*
Disallow: /

# Wikipedia work bots:
User-agent: IsraBot
Disallow:

User-agent: Orthogaffe
Disallow:

# Crawlers that are kind enough to obey, but which we'd rather not have
# unless they're feeding search engines.
User-agent: UbiCrawler
Disallow: /

User-agent: DOC
Disallow: /

User-agent: Zao
Disallow: /

# Some bots are known to be trouble, particularly those designed to copy
# entire sites. Please obey robots.txt.
User-agent: sitecheck.internetseer.com
Disallow: /

User-agent: Zealbot
Disallow: /

User-agent: MSIECrawler
Disallow: /

User-agent: SiteSnagger
Disallow: /

User-agent: WebStripper
Disallow: /

User-agent: WebCopier
Disallow: /

User-agent: Fetch
Disallow: /

User-agent: Offline Explorer
Disallow: /

User-agent: Teleport
Disallow: /

User-agent: TeleportPro
Disallow: /

User-agent: WebZIP
Disallow: /

User-agent: linko
Disallow: /

User-agent: HTTrack
Disallow: /

User-agent: Microsoft.URL.Control
Disallow: /

User-agent: Xenu
Disallow: /

User-agent: larbin
Disallow: /

User-agent: libwww
Disallow: /

User-agent: ZyBORG
Disallow: /

User-agent: Download Ninja
Disallow: /

# Misbehaving: requests much too fast:
User-agent: fast
Disallow: /

#
# Sorry, wget in its recursive mode is a frequent problem.
# Please read the man page and use it properly; there is a
# --wait option you can use to set the delay between hits,
# for instance.
#
User-agent: wget
Disallow: /

#
# The 'grub' distributed client has been *very* poorly behaved.
#
User-agent: grub-client
Disallow: /

#
# Doesn't follow robots.txt anyway, but...
#
User-agent: k2spider
Disallow: /

#
# Hits many times per second, not acceptable
# http://www.nameprotect.com/botinfo.html
User-agent: NPBot
Disallow: /

# A capture bot, downloads gazillions of pages with no public benefit
# http://www.webreaper.net/
User-agent: WebReaper
Disallow: /

#
# Friendly, low-speed bots are welcome viewing article pages, but not
# dynamically-generated pages please.
#
# Inktomi's "Slurp" can read a minimum delay between hits; if your
# bot supports such a thing using the 'Crawl-delay' or another
# instruction, please let us know.
#
# There is a special exception for API mobileview to allow dynamic
# mobile web & app views to load section content.
# These views aren't HTTP-cached but use parser cache aggressively
# and don't expose special: pages etc.
#
# Another exception is for REST API documentation, located at
# /api/rest_v1/?doc.
#
User-agent: *
Allow: /w/api.php?action=mobileview&
Allow: /w/load.php?
Allow: /api/rest_v1/?doc
Disallow: /w/
Disallow: /api/
Disallow: /trap/
Disallow: /wiki/Special:
Disallow: /wiki/Spezial:
Disallow: /wiki/Spesial:
Disallow: /wiki/Special%3A
Disallow: /wiki/Spezial%3A
Disallow: /wiki/Spesial%3A

#
# ar:
Disallow: /wiki/%D8%AE%D8%A7%D8%B5:Search
Disallow: /wiki/%D8%AE%D8%A7%D8%B5%3ASearch
#
# dewiki:
# T6937
# sensible deletion and meta user discussion pages:
Disallow: /wiki/Wikipedia:L%C3%B6schkandidaten/
Disallow: /wiki/Wikipedia:Löschkandidaten/
Disallow: /wiki/Wikipedia:Vandalensperrung/
Disallow: /wiki/Wikipedia:Benutzersperrung/
Disallow: /wiki/Wikipedia:Vermittlungsausschuss/
Disallow: /wiki/Wikipedia:Administratoren/Probleme/
Disallow: /wiki/Wikipedia:Adminkandidaturen/
Disallow: /wiki/Wikipedia:Qualitätssicherung/
Disallow: /wiki/Wikipedia:Qualit%C3%A4tssicherung/
# 4937#5
Disallow: /wiki/Wikipedia:Vandalismusmeldung/
Disallow: /wiki/Wikipedia:Gesperrte_Lemmata/
Disallow: /wiki/Wikipedia:Löschprüfung/
Disallow: /wiki/Wikipedia:L%C3%B6schprüfung/
Disallow: /wiki/Wikipedia:Administratoren/Notizen/
Disallow: /wiki/Wikipedia:Schiedsgericht/Anfragen/
Disallow: /wiki/Wikipedia:L%C3%B6schpr%C3%BCfung/
# T14111
Disallow: /wiki/Wikipedia:Checkuser/
Disallow: /wiki/Wikipedia_Diskussion:Checkuser/
Disallow: /wiki/Wikipedia_Diskussion:Adminkandidaturen/
# T15961
Disallow: /wiki/Wikipedia:Spam-Blacklist-Log
Disallow: /wiki/Wikipedia%3ASpam-Blacklist-Log
Disallow: /wiki/Wikipedia_Diskussion:Spam-Blacklist-Log
Disallow: /wiki/Wikipedia_Diskussion%3ASpam-Blacklist-Log
#
# enwiki:
# Folks get annoyed when VfD discussions end up the number 1 google hit for
# their name. See T6776
Disallow: /wiki/Wikipedia:Articles_for_deletion/
Disallow: /wiki/Wikipedia%3AArticles_for_deletion/
Disallow: /wiki/Wikipedia:Votes_for_deletion/
Disallow: /wiki/Wikipedia%3AVotes_for_deletion/
Disallow: /wiki/Wikipedia:Pages_for_deletion/
Disallow: /wiki/Wikipedia%3APages_for_deletion/
Disallow: /wiki/Wikipedia:Miscellany_for_deletion/
Disallow: /wiki/Wikipedia%3AMiscellany_for_deletion/
Disallow: /wiki/Wikipedia:Miscellaneous_deletion/
Disallow: /wiki/Wikipedia%3AMiscellaneous_deletion/
Disallow: /wiki/Wikipedia:Copyright_problems
Disallow: /wiki/Wikipedia%3ACopyright_problems
Disallow: /wiki/Wikipedia:Protected_titles/
Disallow: /wiki/Wikipedia%3AProtected_titles/
# T15398
Disallow: /wiki/Wikipedia:WikiProject_Spam/
Disallow: /wiki/Wikipedia%3AWikiProject_Spam/
# T16075
Disallow: /wiki/MediaWiki:Spam-blacklist
Disallow: /wiki/MediaWiki%3ASpam-blacklist
Disallow: /wiki/MediaWiki_talk:Spam-blacklist
Disallow: /wiki/MediaWiki_talk%3ASpam-blacklist
# T13261
Disallow: /wiki/Wikipedia:Requests_for_arbitration/
Disallow: /wiki/Wikipedia%3ARequests_for_arbitration/
Disallow: /wiki/Wikipedia:Requests_for_comment/
Disallow: /wiki/Wikipedia%3ARequests_for_comment/
Disallow: /wiki/Wikipedia:Requests_for_adminship/
Disallow: /wiki/Wikipedia%3ARequests_for_adminship/
# T12288
Disallow: /wiki/Wikipedia_talk:Articles_for_deletion/
Disallow: /wiki/Wikipedia_talk%3AArticles_for_deletion/
Disallow: /wiki/Wikipedia_talk:Votes_for_deletion/
Disallow: /wiki/Wikipedia_talk%3AVotes_for_deletion/
Disallow: /wiki/Wikipedia_talk:Pages_for_deletion/
Disallow: /wiki/Wikipedia_talk%3APages_for_deletion/
Disallow: /wiki/Wikipedia_talk:Miscellany_for_deletion/
Disallow: /wiki/Wikipedia_talk%3AMiscellany_for_deletion/
Disallow: /wiki/Wikipedia_talk:Miscellaneous_deletion/
Disallow: /wiki/Wikipedia_talk%3AMiscellaneous_deletion/
# T16793
Disallow: /wiki/Wikipedia:Changing_username
Disallow: /wiki/Wikipedia%3AChanging_username
Disallow: /wiki/Wikipedia:Changing_username/
Disallow: /wiki/Wikipedia%3AChanging_username/
Disallow: /wiki/Wikipedia_talk:Changing_username
Disallow: /wiki/Wikipedia_talk%3AChanging_username
Disallow: /wiki/Wikipedia_talk:Changing_username/
Disallow: /wiki/Wikipedia_talk%3AChanging_username/
#
# eswiki:
# T8746
Disallow: /wiki/Wikipedia:Consultas_de_borrado/
Disallow: /wiki/Wikipedia%3AConsultas_de_borrado/
#
# fiwiki:
# T10695
Disallow: /wiki/Wikipedia:Poistettavat_sivut
Disallow: /wiki/K%C3%A4ytt%C3%A4j%C3%A4:
Disallow: /wiki/Käyttäjä:
Disallow: /wiki/Keskustelu_k%C3%A4ytt%C3%A4j%C3%A4st%C3%A4:
Disallow: /wiki/Keskustelu_käyttäjästä:
Disallow: /wiki/Wikipedia:Yll%C3%A4pit%C3%A4j%C3%A4t/
Disallow: /wiki/Wikipedia:Ylläpitäjät/
#
# hewiki:
Disallow: /wiki/%D7%9E%D7%99%D7%95%D7%97%D7%93:Search
Disallow: /wiki/%D7%9E%D7%99%D7%95%D7%97%D7%93%3ASearch
#T11517
Disallow: /wiki/ויקיפדיה:רשימת_מועמדים_למחיקה/
Disallow: /wiki/ויקיפדיה%3Aרשימת_מועמדים_למחיקה/
Disallow: /wiki/%D7%95%D7%99%D7%A7%D7%99%D7%A4%D7%93%D7%99%D7%94:%D7%A8%D7%A9%D7%99%D7%9E%D7%AA_%D7%9E%D7%95%D7%A2%D7%9E%D7%93%D7%99%D7%9D_%D7%9C%D7%9E%D7%97%D7%99%D7%A7%D7%94/
Disallow: /wiki/%D7%95%D7%99%D7%A7%D7%99%D7%A4%D7%93%D7%99%D7%94%3A%D7%A8%D7%A9%D7%99%D7%9E%D7%AA_%D7%9E%D7%95%D7%A2%D7%9E%D7%93%D7%99%D7%9D_%D7%9C%D7%9E%D7%97%D7%99%D7%A7%D7%94/
Disallow: /wiki/ויקיפדיה:ערכים_לא_קיימים_ומוגנים
Disallow: /wiki/ויקיפדיה%3Aערכים_לא_קיימים_ומוגנים
Disallow: /wiki/%D7%95%D7%99%D7%A7%D7%99%D7%A4%D7%93%D7%99%D7%94:%D7%A2%D7%A8%D7%9B%D7%99%D7%9D_%D7%9C%D7%90_%D7%A7%D7%99%D7%99%D7%9E%D7%99%D7%9D_%D7%95%D7%9E%D7%95%D7%92%D7%A0%D7%99%D7%9D
Disallow: /wiki/%D7%95%D7%99%D7%A7%D7%99%D7%A4%D7%93%D7%99%D7%94%3A%D7%A2%D7%A8%D7%9B%D7%99%D7%9D_%D7%9C%D7%90_%D7%A7%D7%99%D7%99%D7%9E%D7%99%D7%9D_%D7%95%D7%9E%D7%95%D7%92%D7%A0%D7%99%D7%9D
Disallow: /wiki/ויקיפדיה:דפים_לא_קיימים_ומוגנים
Disallow: /wiki/ויקיפדיה%3Aדפים_לא_קיימים_ומוגנים
Disallow: /wiki/%D7%95%D7%99%D7%A7%D7%99%D7%A4%D7%93%D7%99%D7%94:%D7%93%D7%A4%D7%99%D7%9D_%D7%9C%D7%90_%D7%A7%D7%99%D7%99%D7%9E%D7%99%D7%9D_%D7%95%D7%9E%D7%95%D7%92%D7%A0%D7%99%D7%9D
Disallow: /wiki/%D7%95%D7%99%D7%A7%D7%99%D7%A4%D7%93%D7%99%D7%94%3A%D7%93%D7%A4%D7%99%D7%9D_%D7%9C%D7%90_%D7%A7%D7%99%D7%99%D7%9E%D7%99%D7%9D_%D7%95%D7%9E%D7%95%D7%92%D7%A0%D7%99%D7%9D
#
# huwiki:
Disallow: /wiki/Speci%C3%A1lis:Search
Disallow: /wiki/Speci%C3%A1lis%3ASearch
#
# itwiki:
# T7545
Disallow: /wiki/Wikipedia:Pagine_da_cancellare
Disallow: /wiki/Wikipedia%3APagine_da_cancellare
Disallow: /wiki/Wikipedia:Utenti_problematici
Disallow: /wiki/Wikipedia%3AUtenti_problematici
Disallow: /wiki/Wikipedia:Vandalismi_in_corso
Disallow: /wiki/Wikipedia%3AVandalismi_in_corso
Disallow: /wiki/Wikipedia:Amministratori
Disallow: /wiki/Wikipedia%3AAmministratori
Disallow: /wiki/Wikipedia:Proposte_di_cancellazione_semplificata
Disallow: /wiki/Wikipedia%3AProposte_di_cancellazione_semplificata
Disallow: /wiki/Categoria:Da_cancellare_subito
Disallow: /wiki/Categoria%3ADa_cancellare_subito
Disallow: /wiki/Wikipedia:Sospette_violazioni_di_copyright
Disallow: /wiki/Wikipedia%3ASospette_violazioni_di_copyright
Disallow: /wiki/Categoria:Da_controllare_per_copyright
Disallow: /wiki/Categoria%3ADa_controllare_per_copyright
Disallow: /wiki/Progetto:Rimozione_contributi_sospetti
Disallow: /wiki/Progetto%3ARimozione_contributi_sospetti
Disallow: /wiki/Categoria:Da_cancellare_subito_per_violazione_integrale_copyright
Disallow: /wiki/Categoria%3ADa_cancellare_subito_per_violazione_integrale_copyright
Disallow: /wiki/Progetto:Cococo
Disallow: /wiki/Progetto%3ACococo
Disallow: /wiki/Discussioni_progetto:Cococo
Disallow: /wiki/Discussioni_progetto%3ACococo
#
# jawiki
Disallow: /wiki/%E7%89%B9%E5%88%A5:Search
Disallow: /wiki/%E7%89%B9%E5%88%A5%3ASearch
# T7239
Disallow: /wiki/Wikipedia:%E5%89%8A%E9%99%A4%E4%BE%9D%E9%A0%BC/
Disallow: /wiki/Wikipedia%3A%E5%89%8A%E9%99%A4%E4%BE%9D%E9%A0%BC/
Disallow: /wiki/Wikipedia:%E5%88%A9%E7%94%A8%E8%80%85%E3%83%9A%E3%83%BC%E3%82%B8%E3%81%AE%E5%89%8A%E9%99%A4%E4%BE%9D%E9%A0%BC
Disallow: /wiki/Wikipedia%3A%E5%88%A9%E7%94%A8%E8%80%85%E3%83%9A%E3%83%BC%E3%82%B8%E3%81%AE%E5%89%8A%E9%99%A4%E4%BE%9D%E9%A0%BC
# nowiki
# T13432
Disallow: /wiki/Bruker:
Disallow: /wiki/Bruker%3A
Disallow: /wiki/Brukerdiskusjon
Disallow: /wiki/Wikipedia:Administratorer
Disallow: /wiki/Wikipedia%3AAdministratorer
Disallow: /wiki/Wikipedia-diskusjon:Administratorer
Disallow: /wiki/Wikipedia-diskusjon%3AAdministratorer
Disallow: /wiki/Wikipedia:Sletting
Disallow: /wiki/Wikipedia%3ASletting
Disallow: /wiki/Wikipedia-diskusjon:Sletting
Disallow: /wiki/Wikipedia-diskusjon%3ASletting
#
# plwiki
# T10067
Disallow: /wiki/Wikipedia:Strony_do_usuni%C4%99cia
Disallow: /wiki/Wikipedia%3AStrony_do_usuni%C4%99cia
Disallow: /wiki/Wikipedia:Do_usuni%C4%99cia
Disallow: /wiki/Wikipedia%3ADo_usuni%C4%99cia
Disallow: /wiki/Wikipedia:SDU/
Disallow: /wiki/Wikipedia%3ASDU/
Disallow: /wiki/Wikipedia:Strony_podejrzane_o_naruszenie_praw_autorskich
Disallow: /wiki/Wikipedia%3AStrony_podejrzane_o_naruszenie_praw_autorskich
#
# ptwiki:
# T7394
Disallow: /wiki/Wikipedia:Páginas_para_eliminar/
Disallow: /wiki/Wikipedia:P%C3%A1ginas_para_eliminar/
Disallow: /wiki/Wikipedia%3AP%C3%A1ginas_para_eliminar/
Disallow: /wiki/Wikipedia_Discussão:Páginas_para_eliminar/
Disallow: /wiki/Wikipedia_Discuss%C3%A3o:P%C3%A1ginas_para_eliminar/
Disallow: /wiki/Wikipedia_Discuss%C3%A3o%3AP%C3%A1ginas_para_eliminar/
#
# rowiki:
# T14546
Disallow: /wiki/Wikipedia:Pagini_de_%C5%9Fters
Disallow: /wiki/Wikipedia%3APagini_de_%C5%9Fters
Disallow: /wiki/Discu%C5%A3ie_Wikipedia:Pagini_de_%C5%9Fters
Disallow: /wiki/Discu%C5%A3ie_Wikipedia%3APagini_de_%C5%9Fters
#
# ruwiki:
Disallow: /wiki/%D0%A1%D0%BF%D0%B5%D1%86%D0%B8%D0%B0%D0%BB%D1%8C%D0%BD%D1%8B%D0%B5:Search
Disallow: /wiki/%D0%A1%D0%BF%D0%B5%D1%86%D0%B8%D0%B0%D0%BB%D1%8C%D0%BD%D1%8B%D0%B5%3ASearch
#
# svwiki:
# T12229
Disallow: /wiki/Wikipedia%3ASidor_f%C3%B6reslagna_f%C3%B6r_radering
Disallow: /wiki/Wikipedia:Sidor_f%C3%B6reslagna_f%C3%B6r_radering
Disallow: /wiki/Wikipedia:Sidor_föreslagna_för_radering
Disallow: /wiki/Användare
Disallow: /wiki/Anv%C3%A4ndare
Disallow: /wiki/Användardiskussion
Disallow: /wiki/Anv%C3%A4ndardiskussion
Disallow: /wiki/Wikipedia:Skyddade_sidnamn
Disallow: /wiki/Wikipedia%3ASkyddade_sidnamn
# T13291
Disallow: /wiki/Wikipedia:Sidor_som_bör_raderas
Disallow: /wiki/Wikipedia:Sidor_som_b%C3%B6r_raderas
Disallow: /wiki/Wikipedia%3ASidor_som_b%C3%B6r_raderas
#
# zhwiki:
# T7104
Disallow: /wiki/Wikipedia:删除投票/侵权
Disallow: /wiki/Wikipedia:%E5%88%A0%E9%99%A4%E6%8A%95%E7%A5%A8/%E4%BE%B5%E6%9D%83
Disallow: /wiki/Wikipedia:删除投票和请求
Disallow: /wiki/Wikipedia:%E5%88%A0%E9%99%A4%E6%8A%95%E7%A5%A8%E5%92%8C%E8%AF%B7%E6%B1%82
Disallow: /wiki/Category:快速删除候选
Disallow: /wiki/Category:%E5%BF%AB%E9%80%9F%E5%88%A0%E9%99%A4%E5%80%99%E9%80%89
Disallow: /wiki/Category:维基百科需要翻译的文章
Disallow: /wiki/Category:%E7%BB%B4%E5%9F%BA%E7%99%BE%E7%A7%91%E9%9C%80%E8%A6%81%E7%BF%BB%E8%AF%91%E7%9A%84%E6%96%87%E7%AB%A0
#
# sister projects
#
# enwikinews:
# T7340
Disallow: /wiki/Portal:Prepared_stories/
Disallow: /wiki/Portal%3APrepared_stories/
#
# itwikinews
# T11138
Disallow: /wiki/Wikinotizie:Richieste_di_cancellazione
Disallow: /wiki/Wikinotizie:Sospette_violazioni_di_copyright
Disallow: /wiki/Categoria:Da_cancellare_subito
Disallow: /wiki/Categoria:Da_cancellare_subito_per_violazione_integrale_copyright
Disallow: /wiki/Wikinotizie:Storie_in_preparazione
#
# enwikiquote:
# T17095
Disallow: /wiki/Wikiquote:Votes_for_deletion/
Disallow: /wiki/Wikiquote%3AVotes_for_deletion/
Disallow: /wiki/Wikiquote_talk:Votes_for_deletion/
Disallow: /wiki/Wikiquote_talk%3AVotes_for_deletion/
Disallow: /wiki/Wikiquote:Votes_for_deletion_archive/
Disallow: /wiki/Wikiquote%3AVotes_for_deletion_archive/
Disallow: /wiki/Wikiquote_talk:Votes_for_deletion_archive/
Disallow: /wiki/Wikiquote_talk%3AVotes_for_deletion_archive/
#
# enwikibooks
Disallow: /wiki/Wikibooks:Votes_for_deletion
#
# working...
Disallow: /wiki/Fundraising_2007/comments
#
#
#
#----------------------------------------------------------#
#
#
#
 # <!-- Please do not remove the space at the start of this line, it breaks the rendering.  http://www.robotstxt.org/orig.html says spaces before comments are OK. --><pre>
#
# robots.txt for http://www.wikisource.org/ and friends
#
# Please note: There are a lot of pages on this site, and there are
# some misbehaved spiders out there that go _way_ too fast. If you're
# irresponsible, your access to the site may be blocked.
#
# Edit at http://zh.wikisource.org/w/index.php?title=MediaWiki:Robots.txt&action=edit
#
# Please check any changes using a syntax validator such as http://tool.motoricerca.info/robots-checker.phtml
# Enter http://zh.wikisource.org/robots.txt as the URL to check.
#

# advertising-related bots:
User-agent: Mediapartners-Google*
Disallow: /

# Wikipedia work bots:
User-agent: IsraBot
Disallow:

User-agent: Orthogaffe
Disallow:

# Crawlers that are kind enough to obey, but which we'd rather not have
# unless they're feeding search engines.
User-agent: UbiCrawler
Disallow: /

User-agent: DOC
Disallow: /

User-agent: Zao
Disallow: /

# Some bots are known to be trouble, particularly those designed to copy
# entire sites. Please obey robots.txt.
User-agent: sitecheck.internetseer.com
Disallow: /

User-agent: Zealbot
Disallow: /

User-agent: MSIECrawler
Disallow: /

User-agent: SiteSnagger
Disallow: /

User-agent: WebStripper
Disallow: /

User-agent: WebCopier
Disallow: /

User-agent: Fetch
Disallow: /

User-agent: Offline Explorer
Disallow: /

User-agent: Teleport
Disallow: /

User-agent: TeleportPro
Disallow: /

User-agent: WebZIP
Disallow: /

User-agent: linko
Disallow: /

User-agent: HTTrack
Disallow: /

User-agent: Microsoft.URL.Control
Disallow: /

User-agent: Xenu
Disallow: /

User-agent: larbin
Disallow: /

User-agent: libwww
Disallow: /

User-agent: ZyBORG
Disallow: /

User-agent: Download Ninja
Disallow: /

#
# Sorry, wget in its recursive mode is a frequent problem.
# Please read the man page and use it properly; there is a
# --wait option you can use to set the delay between hits,
# for instance.
#
User-agent: wget
Disallow: /

#
# The 'grub' distributed client has been *very* poorly behaved.
#
User-agent: grub-client
Disallow: /

#
# Doesn't follow robots.txt anyway, but...
#
User-agent: k2spider
Disallow: /

#
# Hits many times per second, not acceptable
# http://www.nameprotect.com/botinfo.html
User-agent: NPBot
Disallow: /

# A capture bot, downloads gazillions of pages with no public benefit
# http://www.webreaper.net/
User-agent: WebReaper
Disallow: /

# Don't allow the wayback-maschine to index user-pages
#User-agent: ia_archiver
#Disallow: /wiki/User
#Disallow: /wiki/Benutzer

#
# Friendly, low-speed bots are welcome viewing article pages, but not
# dynamically-generated pages please.
#
# Inktomi's "Slurp" can read a minimum delay between hits; if your
# bot supports such a thing using the 'Crawl-delay' or another
# instruction, please let us know.
#
User-agent: *
Disallow: /w/
Disallow: /trap/
Disallow: /wiki/Special:Random
Disallow: /wiki/Special%3ARandom
Disallow: /wiki/Special:Search
Disallow: /wiki/Special%3ASearch
Disallow:http://zh.wikisource.org/wiki/Category:%E6%80%80%E7%96%91%E4%BE%B5%E7%8A%AF%E7%89%88%E6%9D%83%E6%96%87%E7%AB%A0

Sitemap: http://sitemap.wikimedia.org/zhwikisource/latest/zhwikisource-latest-abstract.xml
Sitemap: http://sitemap.wikimedia.org/zhwikisource/latest/zhwikisource-latest-abstract.xml-rss.xml
Sitemap: http://zh.wikisource.org/w/index.php?title=Special:NewPages&feed=rss&namespace=all

# </pre>