## all bots
User-agent: *
Disallow: /_notes/
Disallow: /inc/
Disallow: /includes/
Disallow: /masters/
Disallow: /test/
Disallow: /Templates/
Disallow: /Connections/
Disallow: /scripts/
Disallow: /go/
Disallow: /projects/
Disallow: /stuff/

## overide specific file
# Allow: /folder1/myfile.html
# Disallow: /folder1/

## specific bots ... 
# User-agent: fooBot
# Disallow: /_notes/

## throttle requests / prevent hammering
Crawl-delay: 10

## google sitemaps
Sitemap: http://pjkix.com/sitemap/sitemap-index.xml

## extended standard
Request-rate: 1/5         # maximum rate is one page every 5 seconds
Visit-time: 0600-0845     # only visit between 06:00 and 08:45 UTC (GMT)