diff --git a/public/robots.txt b/public/robots.txt index e9e57dc..3dad881 100644 --- a/public/robots.txt +++ b/public/robots.txt @@ -1,3 +1,56 @@ -# https://www.robotstxt.org/robotstxt.html -User-agent: * -Disallow: +# As a condition of accessing this website, you agree to abide by the following +# content signals: + +# (a) If a content-signal = yes, you may collect content for the corresponding +# use. +# (b) If a content-signal = no, you may not collect content for the +# corresponding use. +# (c) If the website operator does not include a content signal for a +# corresponding use, the website operator neither grants nor restricts +# permission via content signal with respect to the corresponding use. + +# The content signals and their meanings are: + +# search: building a search index and providing search results (e.g., returning +# hyperlinks and short excerpts from your website's contents). Search does not +# include providing AI-generated search summaries. +# ai-input: inputting content into one or more AI models (e.g., retrieval +# augmented generation, grounding, or other real-time taking of content for +# generative AI search answers). +# ai-train: training or fine-tuning AI models. + +# ANY RESTRICTIONS EXPRESSED VIA CONTENT SIGNALS ARE EXPRESS RESERVATIONS OF +# RIGHTS UNDER ARTICLE 4 OF THE EUROPEAN UNION DIRECTIVE 2019/790 ON COPYRIGHT +# AND RELATED RIGHTS IN THE DIGITAL SINGLE MARKET. + +# BEGIN Cloudflare Managed content + +User-Agent: * +Content-signal: search=yes,ai-train=no +Allow: / + +User-agent: Amazonbot +Disallow: / + +User-agent: Applebot-Extended +Disallow: / + +User-agent: Bytespider +Disallow: / + +User-agent: CCBot +Disallow: / + +User-agent: ClaudeBot +Disallow: / + +User-agent: Google-Extended +Disallow: / + +User-agent: GPTBot +Disallow: / + +User-agent: meta-externalagent +Disallow: / + +# END Cloudflare Managed Content