diff --git a/public/robots.txt b/public/robots.txt index 129aafa..d52673d 100644 --- a/public/robots.txt +++ b/public/robots.txt @@ -16,6 +16,12 @@ Disallow: / User-agent: Applebot Disallow: / +# AI Data Scraper +# https://darkvisitors.com/agents/applebot-extended + +User-agent: Applebot-Extended +Disallow: / + # AI Data Scraper # https://darkvisitors.com/agents/bytespider