# Authenticated portals and tokenized share links return the SPA shell with
# HTTP 200 to a crawler, so nothing stops them being indexed as thin duplicate
# pages except this file.
#
# The trailing slashes matter: `/employer/` does not match `/employers`, and
# `/candidate/` does not match `/candidates/privacy-policy`.
User-agent: *
Disallow: /employer/
Disallow: /candidate/
Disallow: /admin/
Disallow: /job
Disallow: /pitch
Disallow: /main-page

# Social unfurlers need to fetch tokenized share links to build link previews,
# and they do not feed the search index.
User-agent: Twitterbot
Allow: /

User-agent: facebookexternalhit
Allow: /

Sitemap: https://gettink.ai/sitemap.xml