#
# robots.txt
#
# This file is to prevent the crawling and indexing of certain parts
# of your site by web crawlers and spiders run by sites like Yahoo!
# and Google. By telling these "robots" where not to go on your site,
# you save bandwidth and server resources.
#
# This file will be ignored unless it is at the root of your host:
# Used:    http://example.com/robots.txt
# Ignored: http://example.com/site/robots.txt
#
# For more information about the robots.txt standard, see:
# http://www.robotstxt.org/wc/robots.html
#
# For syntax checking, see:
# http://www.sxw.org.uk/computing/robots/check.html

User-agent: *
Crawl-delay: 10
# Directories
Disallow: /includes/
Disallow: /misc/
Disallow: /modules/
Allow: /profiles/openscholar/themes/
Disallow: /profiles/
Disallow: /scripts/
Disallow: /themes/
# Files
Disallow: /CHANGELOG.txt
Disallow: /cron.php
Disallow: /INSTALL.mysql.txt
Disallow: /INSTALL.pgsql.txt
Disallow: /INSTALL.sqlite.txt
Disallow: /install.php
Disallow: /INSTALL.txt
Disallow: /LICENSE.txt
Disallow: /MAINTAINERS.txt
Disallow: /update.php
Disallow: /UPGRADE.txt
Disallow: /xmlrpc.php
# Paths (clean URLs)
Disallow: /admin/
Disallow: /comment/reply/
Disallow: /filter/tips/
Disallow: /node/add/
Disallow: /search/
Disallow: /user/register/
Disallow: /user/password/
Disallow: /user/login/
Disallow: /user/logout/
# Paths (no clean URLs)
Disallow: /?q=admin/
Disallow: /?q=comment/reply/
Disallow: /?q=filter/tips/
Disallow: /?q=node/add/
Disallow: /?q=search/
Disallow: /?q=user/password/
Disallow: /?q=user/register/
Disallow: /?q=user/login/
Disallow: /?q=user/logout/
Disallow: /crustal-dynamics/
Disallow: /astronomy/
Disallow: /atweb/
Disallow: /testfowkes/
Disallow: /gabriel/
Disallow: /nora-hampl/
Disallow: /joanna-nizynska/
Disallow: /jacobsen/
Disallow: /kahne/
Disallow: /shair/
Disallow: /annualmeeting/
Disallow: /hwpi/
Disallow: /socquantworkshop/
Disallow: /1636society-membership/
Disallow: /advisingcorner/
Disallow: /stuart-project/
Disallow: /huit-intranet/
Disallow: /nilagiatest/
Disallow: /itatti/
Disallow: /uds-sensusaccess/
Disallow: /ivy/
Disallow: /nilagiatest3/
Disallow: /nilagia-lab-test-site/
Disallow: /soch/
Disallow: /hks/
Disallow: /hms/
Disallow: /hiltresearch/
Disallow: /slt/
Disallow: /digital-content-connect/
Disallow: /resources/
Disallow: /itcrb/
Disallow: /itattitest/
Disallow: /academictech/
Disallow: /admin-enews/
Disallow: /nilagia-test-event/
Disallow: /projectdelivery/
Disallow: /learning/
Disallow: /service_desk_test/
Disallow: /unifiedcommunications/
Disallow: /botlib_projects/
Disallow: /mixedmethodsworkshop/
Disallow: /harvard/
Disallow: /jillhooley/
Disallow: /lciportal/
Disallow: /cavanaughlab/
Disallow: /nilagia-test-subsite/
Disallow: /nm-test/
Disallow: /employers/
Disallow: /advising_materials/
Disallow: /myfad/
Disallow: /bokcenter-videos/
Disallow: /a_harvard_education/
Disallow: /hacc/
Disallow: /fasscit/
Disallow: /amdp-globalmap/
Disallow: /debivortlab/
Disallow: /able/
Disallow: /eric-heller/
Disallow: /facultysearch/
Disallow: /umt-test/
Disallow: /professors_and_pastries/
Disallow: /graduate_student_site/
Disallow: /umt/
Disallow: /inas2015/
Disallow: /fep/
Disallow: /eric-temp/
Disallow: /wolkovichlab/
Disallow: /media_accessibility/
Disallow: /osp-nfs/
Disallow: /admin-login/
Disallow: /govwrites/
Disallow: /share/
Disallow: /hms-mso/
Disallow: /travelservices_reimbursement/
Disallow: /card_services/
Disallow: /posts/
Disallow: /bomblieslab/
Disallow: /hwp_test/
Disallow: /harvardkey/
Disallow: /combeslab/
Disallow: /srivastavalab/
Disallow: /spierce/
Disallow: /online_accessibility/
Disallow: /sjm-test/
Disallow: /hbi-protected/
Disallow: /govteaches/
Disallow: /mediasite-recordings/
Disallow: /hsdmintranet/
Disallow: /hopkinslab/
Disallow: /richardson/
Disallow: /hura/
Disallow: /brain-initiative/
Disallow: /susansuleiman/
Disallow: /andes_amazon_initiative/
Disallow: /fas_glacier/
Disallow: /ajones/
Disallow: /onecampus-sandbox/
Disallow: /accessibility_tpg/
Disallow: /procurement/
Disallow: /knolllab/
Disallow: /procurement-splash/
Disallow: /harvard_peer_advising_fellows/
Disallow: /olveczky-internal/
Disallow: /internal/
Disallow: /hoekstra-lab-internal/
Disallow: /comp-lit-alumni-connect/
Disallow: /webinars/
Disallow: /digging/
Disallow: /minerals/
Disallow: /ali2ndsite/
Disallow: /mrct-landing/
Disallow: /diversity_new/
Disallow: /cwd_getorganized/
Disallow: /cwd_bic/
Disallow: /hmsc_resources/
Disallow: /campus_of_the_future/
Disallow: /jsinger/
Disallow: /oebinternal/
Disallow: /hwp-sample-academic-department/
Disallow: /girguis.lab.members/
Disallow: /academic_resources_student_advisory_board/
Disallow: /taskforcevideo/
Disallow: /est_repository/
Disallow: /libsearching/
Disallow: /fad-ccs/
Disallow: /hwp-sample-department-minimal/
Disallow: /test2_sjm/
Disallow: /jri/
Disallow: /tax/
Disallow: /spectra/
Disallow: /hwpscholars/
Disallow: /geobiology-symposium-2016/
Disallow: /secretsofthesiege/
Disallow: /web/
Disallow: /mrct/
Disallow: /saultest/
Disallow: /client-services/
Disallow: /mp_test/
Disallow: /kentontesting/
Disallow: /radcliffe/
Disallow: /internalcontrols/
Disallow: /seas-courses/
Disallow: /administration/
Disallow: /levi/
Disallow: /vmo/
Disallow: /financialmanagersforum/
Disallow: /myfac/
Disallow: /hds-intranet/
Disallow: /msotest/
Disallow: /cogneuro/
Disallow: /seas-courselist/
Disallow: /hscrb-guide/
Disallow: /ldcnetwork/
Disallow: /redirects/
Disallow: /testing_kim/
Disallow: /fingerprinting/
Disallow: /grs/
Disallow: /jonathan-donais/
Disallow: /complit-intranet/
Disallow: /slo/
Disallow: /titleixreviewcommittee/
Disallow: /evp2/
Disallow: /grid/
Disallow: /torman_lop/
Disallow: /africa/
Disallow: /marytestsubsite/
Disallow: /marytestsubsite2/
Disallow: /anothony_sandbox/
Disallow: /hopitest/
Disallow: /rlpi/
Disallow: /evp/
Disallow: /cai/
Disallow: /entomology-department-specimen-ledgers/
Disallow: /mcz-entomology-department-specimen-ledgers/
Disallow: /specimen-ledgers/
Disallow: /hunt/
Disallow: /tlfwebinars/
Disallow: /huit-pmo-1/
Disallow: /psnetwork/
Disallow: /finding/
Disallow: /joe/
Disallow: /pha/
Disallow: /academic/
Disallow: /flyresearch/
Disallow: /jkloppenberg/
Disallow: /naomitest/
Disallow: /healthcare/
Disallow: /tll-im/
Disallow: /hks-publications/
Disallow: /hks-pubs-ksr/
Disallow: /thecitizen/
Disallow: /designencounters/
Disallow: /publicservice/
Disallow: /training/
Disallow: /marshalkm/
Disallow: /huit-pmo/
Disallow: /aeo/
Disallow: /huitintranet2/
Disallow: /libraryresources/
Disallow: /cloudrevamp/
Disallow: /execed/
Disallow: /cloud2/
Disallow: /kates-site/
Disallow: /newhuitsite/
Disallow: /lab-site/
Disallow: /wellbeing/
Disallow: /friedmanlab/
Disallow: /mary-architecture/
Disallow: /chemcmm/
Disallow: /content-strategy/
Disallow: /cmmchemistry/
Disallow: /drclasmcaoredirect/
Disallow: /signdemo/
Disallow: /mofo-officehours/
Disallow: /giribettest/
Disallow: /beijing/
Disallow: /fsan/
Disallow: /currier/
Disallow: /opensdp/
Disallow: /fashrtest/
Disallow: /fsan-intranet/
Disallow: /testcal/
Disallow: /bok/
Disallow: /harvardshorts/
Disallow: /community/
Disallow: /hwp-officehours/
Disallow: /ccva/
Disallow: /phi-beta-kappa/
Disallow: /advising_programs/
Disallow: /ogcfaq/
Disallow: /hrpractitioners/
Disallow: /tormanhc2/
Disallow: /expos/
Disallow: /test3_sjm/
Disallow: /student-test/
Disallow: /nasaabisscruise_julyaug2017/
Disallow: /rbarro/
Disallow: /alberto-alesina/
Disallow: /james-stock/
Disallow: /dudleyundergrad/
Disallow: /concentrations/
Disallow: /rll-tutoring/
Disallow: /rebeccafischer/
Disallow: /ypo/
Disallow: /rll-intranet/
Disallow: /rogerfu/
Disallow: /leap/
Disallow: /weiss_fund/
Disallow: /grouper_demo/
Disallow: /grouper-test/
Disallow: /kenton_grouper_test/
Disallow: /grouper_test/
Disallow: /it_security/
Disallow: /officemanual/
Disallow: /classicsintranet/
Disallow: /plaza-webcam/
Disallow: /metro-redirect/
Disallow: /bureau-study-counsel/
Disallow: /bgltq/
Disallow: /bureau-of-study-counsel/
Disallow: /torman_group_hwp_client/
Disallow: /torman_group_os_managed/
Disallow: /accessibilitywiki/
Disallow: /gsf/
Disallow: /guidelines/
Disallow: /bsc-2/
Disallow: /hmsc_private/
Disallow: /texere/
Disallow: /hmsc_intranet/
Disallow: /visibly-better/
Disallow: /kasper/
Disallow: /kenton/
Disallow: /hcf-fy15/
Disallow: /lish_temp/
Disallow: /matt/
Disallow: /odrip/
Disallow: /aisp-practice/
Disallow: /tix/
Disallow: /practicetest/
Disallow: /bokcenter/
Disallow: /elevate/
Disallow: /tdmtheowlanswers/
Disallow: /hwp3/
Disallow: /zhangtest/
Disallow: /tormanhc_001/
Disallow: /yusuf/
Disallow: /2018pdsymposium/
Disallow: /tlt/
Disallow: /cepr-sdp/
Disallow: /profdev/
Disallow: /jconforti/
Disallow: /mlsp-teachers/
Disallow: /advisingfortnight/
Disallow: /torman_grouper_client/
Disallow: /torman_grouper_dev/
Disallow: /hutchins/
Disallow: /mammalogy/
Disallow: /herpetology/
Disallow: /reu/
Disallow: /titleix/
Disallow: /b2p/
Disallow: /gdpr/
Disallow: /resilience-consortium/
Disallow: /hds-temporary/
Disallow: /studentemployment/
Disallow: /3ash/
Disallow: /kramertest/
Disallow: /chge-redirect/
Disallow: /chge/
Disallow: /faculty-diversity/
Disallow: /shinesummit/
Disallow: /may18/
Disallow: /ctrrtaskforce-temp/
Disallow: /gdprprivacy/
Disallow: /hmstheme/
Disallow: /transparency/
Disallow: /english/
Disallow: /odr/
Disallow: /statistics/