# Generally, block infinite traps;
User-agent: *
Disallow: /wayback/
Disallow: /ukwa/search
Disallow: /en/ukwa/search
Disallow: /cy/ukwa/search
Disallow: /gd/ukwa/search
Disallow: /datasets/
Disallow: /shine/

# Allow Twitterbot so social cards work:
User-agent: Twitterbot
Allow: /wayback/

User-agent: facebookexternalhit
Allow: /wayback/

# Allow search engines to index specific sites:
User-agent: *
# As requested in https://github.com/ukwa/ukwa-services/issues/96
Allow: /wayback/archive/*/http://www.europeandialogue.org/