# Linkster Robots.txt
User-agent: *
Allow: /

# Sitemaps
Sitemap: https://www.linkster.co/sitemap-index.xml

# Crawl-delay for respectful crawling
Crawl-delay: 1

# Disallow access to internal areas
Disallow: /admin/
Disallow: /api/
Disallow: /_astro/

# Allow search engines to access everything else
Allow: /de/
Allow: /*.js$
Allow: /*.css$
Allow: /*.png$
Allow: /*.jpg$
Allow: /*.jpeg$
Allow: /*.gif$
Allow: /*.svg$
Allow: /*.webp$
Allow: /*.woff2$

# Specific user agents
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Slurp
Allow: /

User-agent: DuckDuckBot
Allow: /

# Block bad bots
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: MJ12bot
Disallow: /