# Robots.txt for Clearfork Intelligence # https://clearforkintelligence.com # Allow all crawlers User-agent: * Allow: / # Specific directives for search engines User-agent: Googlebot Allow: / Crawl-delay: 0 User-agent: Bingbot Allow: / Crawl-delay: 0 User-agent: Slurp Allow: / Crawl-delay: 0 User-agent: DuckDuckBot Allow: / # Block bad bots User-agent: AhrefsBot Disallow: / User-agent: SemrushBot Disallow: / User-agent: DotBot Disallow: / User-agent: MJ12bot Disallow: / # Sitemap location Sitemap: https://clearforkintelligence.com/sitemap.xml # Allow access to CSS and JS files Allow: /css/ Allow: /js/ Allow: /images/ # Prevent crawling of temporary or private files Disallow: /*.tmp$ Disallow: /*.bak$ Disallow: /*.old$ Disallow: /private/ Disallow: /admin/