#
# robots.txt
#
# This file is to prevent the crawling and indexing of certain parts
# of your site by web crawlers and spiders run by sites like Yahoo!
# and Google. By telling these "robots" where not to go on your site,
# you save bandwidth and server resources.
#
# This file will be ignored unless it is at the root of your host:
# Used:    http://example.com/robots.txt
# Ignored: http://example.com/site/robots.txt
#
# For more information about the robots.txt standard, see:
# http://www.robotstxt.org/robotstxt.html
#
# For syntax checking, see:
# http://www.frobee.com/robots-txt-check

User-agent: *
Crawl-delay: 10
# Directories
Disallow: /includes/
Disallow: /misc/
Disallow: /promote-slide/
Disallow: /modules/
Disallow: /profiles/
Disallow: /scripts/
Disallow: /themes/
Disallow: /files/white-papers/internet-of-things/
# Files
Disallow: /CHANGELOG.txt
Disallow: /cron.php
Disallow: /INSTALL.mysql.txt
Disallow: /INSTALL.pgsql.txt
Disallow: /INSTALL.sqlite.txt
Disallow: /install.php
Disallow: /INSTALL.txt
Disallow: /LICENSE.txt
Disallow: /MAINTAINERS.txt
Disallow: /update.php
Disallow: /UPGRADE.txt
Disallow: /xmlrpc.php
# Paths (clean URLs)
Disallow: /admin/
Disallow: /comment/reply/
Disallow: /filter/tips/
Disallow: /node/add/
Disallow: /search/
Disallow: /*/search/
Disallow: /user/register/
Disallow: /user/password/
Disallow: /user/login/
Disallow: /user/logout/
# Paths to pages we do not want to index
Disallow: /virtual-loadmaster-eula
Disallow: /node/
Disallow: /*/node/
# Paths (no clean URLs)
Disallow: /?q=admin/
Disallow: /?q=comment/reply/
Disallow: /?q=filter/tips/
Disallow: /?q=node/add/
Disallow: /?q=search/
Disallow: /?q=user/password/
Disallow: /?q=user/register/
Disallow: /?q=user/login/
Disallow: /?q=user/logout/
# Paths to pages we do not want to index
Disallow: /virtual-loadmaster-eula
Disallow: /node/
Disallow: /*/node/
Disallow: /kempdev/
Disallow: /*/kempdev/
Disallow: /*/newpartner/
Disallow: /newpartner
Disallow: /taxonomy/term/
Disallow: /*/taxonomy/term/
Disallow: /tags/
Disallow: /*/tags/
Disallow: /*/sitemap-old
Disallow: /*/saitomatupu
Disallow: /files/assets/white-papers/
Disallow: /protected-node/
Disallow: /*/protected-node/
Disallow: /kempdev-old/
Disallow: /*/kempdev-old/
Disallow: /*/news-link/
Disallow: /news-link/
Disallow: /kemp-link/
Disallow: /*/kemp-link/
Disallow: /buy-company/
Disallow: /*/buy-company/
Disallow: /kemp-pdf/
Disallow: /*/kemp-pdf/
Disallow: /document-doesnt-exist/
Disallow: /smix/
Disallow: /smix/*
Disallow: /files/assets/web/esg/ESG-Lab-Review-KEMP-LoadMaster-Solution-Feb-2016.pdf
Disallow: /contact-sdn/
Disallow: /promote-slide/
Disallow: /*/promote-slide/
Disallow: /kemp360/vision/login/
Disallow: /*/kemp360/vision/login/
Disallow: /kemp360/vision/launch/
Disallow: /*/kemp360/vision/launch/
Disallow: /partner-resources/
Disallow: /*/partner-resources/
Disallow: *tanx*
# no index for /en/ 
Disallow: /en/
# Disallow specific extensions
Disallow: /*.inc$
Disallow: /*/document-doesnt-exist/
Disallow: /kemp-technologies-KCP-NFR-Program/
Disallow: *tech-tuesday*
Disallow: /activity-request-form/
Disallow: /*/activity-request-form/
Disallow: /support/feedback/
Disallow: /*/support/feedback/
Disallow: /kemp-documentation/
Disallow: /*/kemp-documentation/
Disallow: /partner-training/
Disallow: /*/partner-training/
Disallow: /partner-training-certification/
Disallow: /*/partner-training-certification/
Disallow: /partner-training-register/
Disallow: /*/partner-training-register/
Disallow: *kemp-sku*
Disallow: *sku-type*
Disallow: /service-provider-metered-licensing-overview/
Disallow: /*/service-provider-metered-licensing-overview/
Disallow: partner-type/*
Disallow: /*/partner-type/*
Disallow: /mela-trial-download/
Disallow: /*/mela-trial-download/
Disallow: /mela-login-register/
Disallow: /*/mela-login-register/
Disallow: /mela-trial-register-password/
Disallow: /*/mela-trial-register-password/
Disallow: /*/kemp-taxonomy/*
Disallow: /files/assets/documentation-input/*
Disallow: /newrelic/*
Disallow: /*/newrelic/*
Disallow: /*/free-vlm-register/
Disallow: /files/assets/web/research/*
Disallow: /*/kemp-funnel/*
Disallow: /files/assets/web/partner-portal/*
Disallow: /*/server-load-balancing-appliances/virtual-loadbalancer/vlm-overview/
Disallow: /files/assets/documentation/7.1/*
Disallow: /*/register-project-webform/
Disallow: /files/pdf/*
Disallow: /node/*/submission/*
Disallow: /*/node/*/submission/*
Disallow: /files/packages/
Disallow: /user/logout/
Allow: /binge/always-on/
Allow: /binge/education/
Allow: /binge/f5/
Allow: /binge/citrix/
Allow: /binge/remote-working/
Allow: /*/binge/always-on/
Allow: /*/binge/education/
Allow: /*/binge/f5/
Allow: /*/binge/citrix/
Allow: /*/binge/remote-working/
Disallow: /*/binge/*
Disallow: /binge/*
Disallow: /partners/directory/*
Disallow: /*/partners/directory/*
Disallow: /krel/*/files/packages/*
Disallow: /*/files/packages/*
Disallow: /flowmon-probe-download-hypervisor/
Disallow: /*/flowmon-probe-download-hypervisor/
Disallow: /flowmon-collector-download-hypervisor/
Disallow: /*/flowmon-collector-download-hypervisor/
Disallow:  /config-tool/download/*
Disallow:  /*/config-tool/download/*
Disallow:  /*/?preview_id*
Disallow:  /index.php?*
Disallow:  /pdf-file-view/?file=*

User-agent: SurdotlyBot
Disallow: /