24 lines
423 B
Plaintext
24 lines
423 B
Plaintext
User-agent: *
|
|
Allow: /
|
|
|
|
# Disallow authenticated pages
|
|
|
|
Disallow: /intent
|
|
Disallow: /settings
|
|
Disallow: /blocks
|
|
Disallow: /bookmarks
|
|
Disallow: /compose
|
|
Disallow: /conversations
|
|
Disallow: /domain_blocks
|
|
Disallow: /favourites
|
|
Disallow: /home
|
|
Disallow: /mutes
|
|
Disallow: /notifications
|
|
Disallow: /pinned
|
|
Disallow: /search
|
|
Disallow: /settings
|
|
Disallow: /share-target
|
|
|
|
# Wait 1 second between successive requests.
|
|
Crawl-delay: 1
|