# https://www.robotstxt.org/robotstxt.html # Default rules for all crawlers User-agent: * Disallow: /wp-admin/ Allow: /wp-admin/admin-ajax.php # Stop crawlers from endless parameter crawl space Disallow: /*?q= Disallow: /*?m= # Block known bad actors User-agent: Bytespider Crawl-delay: 10 Disallow: / User-agent: PetalBot Crawl-delay: 10 Disallow: /