User-agent: * # Allow AJAX endpoint Allow: /wp-admin/admin-ajax.php # Block WordPress admin areas Disallow: /wp-admin/ # Block search results Disallow: /?s= Disallow: /search/ # Block login pages Disallow: /wp-login.php Disallow: /wp-register.php # Optional: Block author archives if not used Disallow: /author/ # Sitemap Sitemap: https://www.darwill.com/sitemap_index.xml # Google Search, Google AI Overviews, and Google AI Mode User-agent: Googlebot Allow: / # Google Gemini and other Google AI systems User-agent: Google-Extended Allow: / # ChatGPT search User-agent: OAI-SearchBot Allow: / # OpenAI model crawler User-agent: GPTBot Allow: / # OpenAI user requested retrieval User-agent: ChatGPT-User Allow: / # Perplexity search User-agent: PerplexityBot Allow: / # Perplexity user requested retrieval User-agent: Perplexity-User Allow: / # Anthropic Claude crawler User-agent: ClaudeBot Allow: / # Anthropic Claude search crawler User-agent: Claude-SearchBot Allow: / # Anthropic Claude user requested retrieval User-agent: Claude-User Allow: / # Common Crawl dataset used by AI systems User-agent: CCBot Allow: / # Apple AI related crawler User-agent: Applebot Allow: / # Amazon / Alexa AI crawler User-agent: Amazonbot Allow: / # DuckDuckGo AI assistant crawler User-agent: DuckAssistBot Allow: / # Internet Archive crawler User-agent: ia_archiver Allow: / # Microsoft Bing Search and Copilot User-agent: Bingbot Allow: / # Microsoft AI crawler User-agent: BingPreview Allow: /