# ========================================================= # villa.edu robots.txt # Updated 2026-04-18 — Fixed group-override bugs for throttled bots # ========================================================= # --- Default rules for all crawlers (incl. Googlebot) --- User-agent: * Disallow: /wp-admin/ Allow: /wp-admin/admin-ajax.php Disallow: /wp-login.php Disallow: /xmlrpc.php Disallow: /wp-cron.php Disallow: /?wp-cmd=cron Disallow: /?s= Disallow: /*?replytocom Disallow: /feed/ Disallow: /comments/feed/ # Legacy AI1EC calendar URLs (plugin removed 2026-03-24) Disallow: /home-showcase/action~posterboard/ Disallow: /home-showcase/action~agenda/ Disallow: /home-showcase/action~oneday/ Disallow: /home-showcase/action~month/ Disallow: /home-showcase/action~week/ Disallow: /home-showcase/action~stream/ Disallow: /calendar/action~posterboard/ Disallow: /calendar/action~agenda/ Disallow: /calendar/action~oneday/ Disallow: /calendar/action~month/ Disallow: /calendar/action~week/ Disallow: /calendar/action~stream/ Disallow: /*?ai1ec_render_js # --- Bingbot: throttle + default disallows --- User-agent: bingbot Crawl-delay: 10 Disallow: /wp-admin/ Allow: /wp-admin/admin-ajax.php Disallow: /wp-login.php Disallow: /xmlrpc.php Disallow: /wp-cron.php Disallow: /?wp-cmd=cron Disallow: /?s= Disallow: /*?replytocom Disallow: /feed/ Disallow: /comments/feed/ Disallow: /home-showcase/action~posterboard/ Disallow: /home-showcase/action~agenda/ Disallow: /home-showcase/action~oneday/ Disallow: /home-showcase/action~month/ Disallow: /home-showcase/action~week/ Disallow: /home-showcase/action~stream/ Disallow: /calendar/action~posterboard/ Disallow: /calendar/action~agenda/ Disallow: /calendar/action~oneday/ Disallow: /calendar/action~month/ Disallow: /calendar/action~week/ Disallow: /calendar/action~stream/ Disallow: /*?ai1ec_render_js # --- AhrefsBot: throttle + default disallows --- User-agent: AhrefsBot Crawl-delay: 15 Disallow: /wp-admin/ Allow: /wp-admin/admin-ajax.php Disallow: /wp-login.php Disallow: /xmlrpc.php Disallow: /wp-cron.php Disallow: /?wp-cmd=cron Disallow: /?s= Disallow: /*?replytocom Disallow: /feed/ Disallow: /comments/feed/ Disallow: /home-showcase/action~posterboard/ Disallow: /home-showcase/action~agenda/ Disallow: /home-showcase/action~oneday/ Disallow: /home-showcase/action~month/ Disallow: /home-showcase/action~week/ Disallow: /home-showcase/action~stream/ Disallow: /calendar/action~posterboard/ Disallow: /calendar/action~agenda/ Disallow: /calendar/action~oneday/ Disallow: /calendar/action~month/ Disallow: /calendar/action~week/ Disallow: /calendar/action~stream/ Disallow: /*?ai1ec_render_js # --- SemrushBot: throttle + default disallows --- User-agent: SemrushBot Crawl-delay: 20 Disallow: /wp-admin/ Allow: /wp-admin/admin-ajax.php Disallow: /wp-login.php Disallow: /xmlrpc.php Disallow: /wp-cron.php Disallow: /?wp-cmd=cron Disallow: /?s= Disallow: /*?replytocom Disallow: /feed/ Disallow: /comments/feed/ Disallow: /home-showcase/action~posterboard/ Disallow: /home-showcase/action~agenda/ Disallow: /home-showcase/action~oneday/ Disallow: /home-showcase/action~month/ Disallow: /home-showcase/action~week/ Disallow: /home-showcase/action~stream/ Disallow: /calendar/action~posterboard/ Disallow: /calendar/action~agenda/ Disallow: /calendar/action~oneday/ Disallow: /calendar/action~month/ Disallow: /calendar/action~week/ Disallow: /calendar/action~stream/ Disallow: /*?ai1ec_render_js # --- facebookexternalhit: throttle + default disallows --- # Note: Facebook ignores Crawl-delay in practice; kept for documentation. User-agent: facebookexternalhit Crawl-delay: 30 Disallow: /wp-admin/ Allow: /wp-admin/admin-ajax.php Disallow: /wp-login.php Disallow: /xmlrpc.php Disallow: /wp-cron.php Disallow: /?wp-cmd=cron Disallow: /?s= Disallow: /*?replytocom Disallow: /feed/ Disallow: /comments/feed/ Disallow: /home-showcase/action~posterboard/ Disallow: /home-showcase/action~agenda/ Disallow: /home-showcase/action~oneday/ Disallow: /home-showcase/action~month/ Disallow: /home-showcase/action~week/ Disallow: /home-showcase/action~stream/ Disallow: /calendar/action~posterboard/ Disallow: /calendar/action~agenda/ Disallow: /calendar/action~oneday/ Disallow: /calendar/action~month/ Disallow: /calendar/action~week/ Disallow: /calendar/action~stream/ Disallow: /*?ai1ec_render_js # --- Applebot: throttle + default disallows --- User-agent: Applebot Crawl-delay: 15 Disallow: /wp-admin/ Allow: /wp-admin/admin-ajax.php Disallow: /wp-login.php Disallow: /xmlrpc.php Disallow: /wp-cron.php Disallow: /?wp-cmd=cron Disallow: /?s= Disallow: /*?replytocom Disallow: /feed/ Disallow: /comments/feed/ Disallow: /home-showcase/action~posterboard/ Disallow: /home-showcase/action~agenda/ Disallow: /home-showcase/action~oneday/ Disallow: /home-showcase/action~month/ Disallow: /home-showcase/action~week/ Disallow: /home-showcase/action~stream/ Disallow: /calendar/action~posterboard/ Disallow: /calendar/action~agenda/ Disallow: /calendar/action~oneday/ Disallow: /calendar/action~month/ Disallow: /calendar/action~week/ Disallow: /calendar/action~stream/ Disallow: /*?ai1ec_render_js # --- Amazonbot: throttle + default disallows --- User-agent: Amazonbot Crawl-delay: 90 Disallow: /wp-admin/ Allow: /wp-admin/admin-ajax.php Disallow: /wp-login.php Disallow: /xmlrpc.php Disallow: /wp-cron.php Disallow: /?wp-cmd=cron Disallow: /?s= Disallow: /*?replytocom Disallow: /feed/ Disallow: /comments/feed/ Disallow: /home-showcase/action~posterboard/ Disallow: /home-showcase/action~agenda/ Disallow: /home-showcase/action~oneday/ Disallow: /home-showcase/action~month/ Disallow: /home-showcase/action~week/ Disallow: /home-showcase/action~stream/ Disallow: /calendar/action~posterboard/ Disallow: /calendar/action~agenda/ Disallow: /calendar/action~oneday/ Disallow: /calendar/action~month/ Disallow: /calendar/action~week/ Disallow: /calendar/action~stream/ Disallow: /*?ai1ec_render_js # --- AI / LLM crawlers --- # Allow ChatGPT Search indexing User-agent: OAI-SearchBot Allow: / # Allow user-requested ChatGPT browsing User-agent: ChatGPT-User Allow: / # Block OpenAI training crawler User-agent: GPTBot Disallow: / # Block other AI / scraper bots User-agent: CCBot Disallow: / User-agent: ClaudeBot Disallow: / User-agent: anthropic-ai Disallow: / User-agent: Google-Extended Disallow: / User-agent: Bytespider Disallow: / User-agent: PetalBot Disallow: / # --- Sitemap reference --- Sitemap: https://www.villa.edu/sitemap_index.xml