diff --git a/robots.txt b/robots.txt index d8777d0..81f294c 100644 --- a/robots.txt +++ b/robots.txt @@ -3,6 +3,10 @@ # http://stackoverflow.com/a/44711 # Boaz answered The trick is using the $ to mark the end of URL. -User-Agent: * -Disallow: /* -Allow: /$ +User-agent: Twitterbot +Disallow: + +User-agent: * +Disallow: / + +#Allow: /$