User-agent: * # Delay included for SiteSucker - https://ricks-apps.com/osx/sitesucker/archive/5.x/5.1.x/5.1/manuals/en/pgs/Request.html Crawl-delay: 30 # Discourage robots from downloading large files Disallow: /pluginfile.php/