1 files changed, 6 insertions, 0 deletions

@@ -65,6 +65,7 @@ main() {
                 weed > /tmp/weeds.html &&
                         mv /tmp/weeds.html "$SFEED_OUTPUT/weeds/index.html";
         }'
+        robotstxt >"$SFEED_OUTPUT/robots.txt"
         if [ -f /tmp/runsfeed.ok ]; then
                 echo >&2 'Done.'
         else
@@ -171,6 +172,11 @@ archive() ( # sfeed_archive AGE FEED ...
         done
 )
 
+robotstxt() {
+        echo "User-Agent: *"
+        echo "Disallow: /"
+}
+
 if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
         main "$@"
 fi