#
# robots.txt
#
# This file is to prevent the crawling and indexing of certain parts
# of your site by web crawlers and spiders run by sites like Yahoo!
# and Google. By telling these "robots" where not to go on your site,
# you save bandwidth and server resources.
#
# This file will be ignored unless it is at the root of your host:
# Used:    http://example.com/robots.txt
# Ignored: http://example.com/site/robots.txt
#
# For more information about the robots.txt standard, see:
# http://www.robotstxt.org/wc/robots.html
#
# For syntax checking, see:
# http://www.sxw.org.uk/computing/robots/check.html

User-agent: *
Crawl-delay: 10
# Directories
Disallow: /includes/
Disallow: /misc/
Disallow: /modules/
Allow: /profiles/openscholar/themes/
Disallow: /profiles/
Disallow: /scripts/
Disallow: /themes/
# Files
Disallow: /CHANGELOG.txt
Disallow: /cron.php
Disallow: /INSTALL.mysql.txt
Disallow: /INSTALL.pgsql.txt
Disallow: /INSTALL.sqlite.txt
Disallow: /install.php
Disallow: /INSTALL.txt
Disallow: /LICENSE.txt
Disallow: /MAINTAINERS.txt
Disallow: /update.php
Disallow: /UPGRADE.txt
Disallow: /xmlrpc.php
# Paths (clean URLs)
Disallow: /admin/
Disallow: /comment/reply/
Disallow: /filter/tips/
Disallow: /node/add/
Disallow: /search/
Disallow: /user/register/
Disallow: /user/password/
Disallow: /user/login/
Disallow: /user/logout/
# Paths (no clean URLs)
Disallow: /?q=admin/
Disallow: /?q=comment/reply/
Disallow: /?q=filter/tips/
Disallow: /?q=node/add/
Disallow: /?q=search/
Disallow: /?q=user/password/
Disallow: /?q=user/register/
Disallow: /?q=user/login/
Disallow: /?q=user/logout/
Disallow: /atweb/
Disallow: /kahne/
Disallow: /shair/
Disallow: /annualmeeting/
Disallow: /advisingcorner/
Disallow: /huit-intranet/
Disallow: /uds-sensusaccess/
Disallow: /ivy/
Disallow: /soch/
Disallow: /resources/
Disallow: /itcrb/
Disallow: /academictech/
Disallow: /admin-enews/
Disallow: /projectdelivery/
Disallow: /learning/
Disallow: /unifiedcommunications/
Disallow: /mixedmethodsworkshop/
Disallow: /cavanaughlab/
Disallow: /employers/
Disallow: /advising_materials/
Disallow: /bokcenter-videos/
Disallow: /a_harvard_education/
Disallow: /hacc/
Disallow: /fasscit/
Disallow: /amdp-globalmap/
Disallow: /debivortlab/
Disallow: /able/
Disallow: /eric-heller/
Disallow: /facultysearch/
Disallow: /graduate_student_site/
Disallow: /fep/
Disallow: /osp-nfs/
Disallow: /admin-login/
Disallow: /govwrites/
Disallow: /posts/
Disallow: /spierce/
Disallow: /hbi-protected/
Disallow: /govteaches/
Disallow: /mediasite-recordings/
Disallow: /hsdmintranet/
Disallow: /hura/
Disallow: /susansuleiman/
Disallow: /andes_amazon_initiative/
Disallow: /procurement/
Disallow: /procurement-splash/
Disallow: /harvard_peer_advising_fellows/
Disallow: /olveczky-internal/
Disallow: /internal/
Disallow: /hoekstra-lab-internal/
Disallow: /webinars/
Disallow: /digging/
Disallow: /minerals/
Disallow: /ali2ndsite/
Disallow: /mrct-landing/
Disallow: /cwd_bic/
Disallow: /hmsc_resources/
Disallow: /campus_of_the_future/
Disallow: /jsinger/
Disallow: /oebinternal/
Disallow: /hwp-sample-academic-department/
Disallow: /academic_resources_student_advisory_board/
Disallow: /libsearching/
Disallow: /fad-ccs/
Disallow: /hwp-sample-department-minimal/
Disallow: /jri/
Disallow: /spectra/
Disallow: /secretsofthesiege/
Disallow: /radcliffe/
Disallow: /internalcontrols/
Disallow: /financialmanagersforum/
Disallow: /myfac/
Disallow: /msotest/
Disallow: /ldcnetwork/
Disallow: /redirects/
Disallow: /fingerprinting/
Disallow: /grs/
Disallow: /complit-intranet/
Disallow: /titleixreviewcommittee/
Disallow: /grid/
Disallow: /evp/
Disallow: /tlfwebinars/
Disallow: /huit-pmo-1/
Disallow: /academic/
Disallow: /flyresearch/
Disallow: /jkloppenberg/
Disallow: /publicservice/
Disallow: /huit-pmo/
Disallow: /huitintranet2/
Disallow: /cloudrevamp/
Disallow: /execed/
Disallow: /cloud2/
Disallow: /friedmanlab/
Disallow: /chemcmm/
Disallow: /drclasmcaoredirect/
Disallow: /beijing/
Disallow: /fsan/
Disallow: /currier/
Disallow: /bok/
Disallow: /harvardshorts/
Disallow: /community/
Disallow: /ccva/
Disallow: /ogcfaq/
Disallow: /hrpractitioners/
Disallow: /tormanhc2/
Disallow: /expos/
Disallow: /student-test/
Disallow: /rbarro/
Disallow: /alberto-alesina/
Disallow: /james-stock/
Disallow: /concentrations/
Disallow: /rll-tutoring/
Disallow: /ypo/
Disallow: /rogerfu/
Disallow: /leap/
Disallow: /grouper_demo/
Disallow: /officemanual/
Disallow: /classicsintranet/
Disallow: /metro-redirect/
Disallow: /texere/
Disallow: /hmsc_intranet/
Disallow: /kasper/
Disallow: /lish_temp/
Disallow: /matt/
Disallow: /tix/
Disallow: /bokcenter/
Disallow: /tdmtheowlanswers/
Disallow: /zhangtest/
Disallow: /2018pdsymposium/
Disallow: /cepr-sdp/
Disallow: /advisingfortnight/
Disallow: /reu/
Disallow: /b2p/
Disallow: /gdpr/
Disallow: /chge-redirect/
Disallow: /chge/
Disallow: /shinesummit/
Disallow: /ctrrtaskforce-temp/
Disallow: /gdprprivacy/
Disallow: /hmstheme/
Disallow: /statistics/
Disallow: /resilience/
Disallow: /hbi-2/
Disallow: /kramertest1/
Disallow: /chinaphilanthropy/
Disallow: /title-ix/
Disallow: /hcs/
Disallow: /insights/
Disallow: /hds-temporary/
Disallow: /share_shadow/
Disallow: /firstyear/
Disallow: /fdd/
Disallow: /emedesigntoolkit/
Disallow: /dso-redirect/
Disallow: /harvardhillelboard/
Disallow: /webpublishing/
Disallow: /case/
Disallow: /resilience-redirect/
Disallow: /resilience-consortium/
Disallow: /upolicies/
Disallow: /diversity/
Disallow: /sis/
Disallow: /labadminmanual/
Disallow: /halc-redirect/
Disallow: /halccom-redirect/
Disallow: /iq-redirect/
Disallow: /brochure/
Disallow: /hscrb/
Disallow: /henrytestsite/
Disallow: /titleixdisclosure/
Disallow: /maryslanguagecenter/
Disallow: /zaentz/
Disallow: /flag/
Disallow: /signs/
Disallow: /enterprisearchitecture/
Disallow: /edlabs/
Disallow: /hcf-fy15/
Disallow: /gsf/
Disallow: /bostonharborislands/
Disallow: /beatyaleweek/
Disallow: /lcassets/
Disallow: /haatestsite/
Disallow: /osl-redirect/
Disallow: /fdo-redirect/
Disallow: /osl/
Disallow: /soch-redirect/
Disallow: /thegame-redirect/
Disallow: /wintersession-redirect/
Disallow: /arcsign/
Disallow: /sochsign/
Disallow: /institutionalresearch/
Disallow: /fasadmin/
Disallow: /procurement-travel/
Disallow: /styleguide/
Disallow: /liaisons/
Disallow: /mahindra/
Disallow: /it-security-new/
Disallow: /edredesign-portal/
Disallow: /covid-19/
Disallow: /semiticmuseum-redirect/
Disallow: /semiticmuseum-redirect/
Disallow: /hupd-redirect/
Disallow: /brazil-drclas-redirect/
Disallow: /mx-drclas-redirect/
Disallow: /ro-drclas-redirect/
Disallow: /drclasregionaloffice/
Disallow: /drclasbraziloffice/
Disallow: /drclasmcaoffice/
Disallow: /parents-fas-redirect/
Disallow: /parents/
Disallow: /drclas/
Disallow: /dmca/
Disallow: /hilt/
Disallow: /collegeorientation/
Disallow: /hds-rpp/
Disallow: /hcf-fy21/
Disallow: /hds-intranet/
Disallow: /osp/
Disallow: /gsf-fy21/
Disallow: /lichens/
Disallow: /huri-tcup-events/
Disallow: /hds-temporary2/
Disallow: /hds-temporary2/
Disallow: /hds-rlp/
Disallow: /accessibilitykey/
Disallow: /engagement/
Disallow: /2021gordonlabvirtualreunion/
Disallow: /ofa/
Disallow: /huit/
Disallow: /w4w/
Disallow: /economics/
Disallow: /seo/
Disallow: /drewfaust/
Disallow: /central-payroll/
Disallow: /thesis/
Disallow: /erg/
Disallow: /ovpr-test/
Disallow: /titleix/
Disallow: /mahindra-ihum-usi/
Disallow: /huitintranet/
Disallow: /oie-test/
Disallow: /econ-campus-reboot/
Disallow: /nra/
Disallow: /ash-innovations-redirect/
Disallow: /biocaribe/
Disallow: /hbs_test/
Disallow: /hbstraining/
Disallow: /kahlilchaarperez/
Disallow: /harvardworks/
Disallow: /coldwarlinkages/
Disallow: /risk-services/
Disallow: /revista/
Disallow: /it-academy/
Disallow: /hppm/
Disallow: /haa-intranet/
Disallow: /environmentalhealthandsafety/
Disallow: /fas-registrar/
Disallow: /hms-grad-ed/
Disallow: /resourceguide/
Disallow: /harvard-allston-redirect/
Disallow: /afvs-alumni/
Disallow: /guidelines/