aaaaaa
This commit is contained in:
parent
5f63527955
commit
f51dc7e078
31 changed files with 331 additions and 88 deletions
60
src/data/caddy.json
Normal file
60
src/data/caddy.json
Normal file
|
@ -0,0 +1,60 @@
|
|||
{
|
||||
"redirects": { "/now": "/me/now" },
|
||||
"blockedCrawlers": [
|
||||
"AI2Bot",
|
||||
"Ai2Bot-Dolma",
|
||||
"aiHitBot",
|
||||
"Amazonbot",
|
||||
"anthropic-ai",
|
||||
"Applebot",
|
||||
"Applebot-Extended",
|
||||
"Brightbot 1.0",
|
||||
"Bytespider",
|
||||
"CCBot",
|
||||
"ChatGPT-User",
|
||||
"Claude-Web",
|
||||
"ClaudeBot",
|
||||
"cohere-ai",
|
||||
"cohere-training-data-crawler",
|
||||
"Cotoyogi",
|
||||
"Crawlspace",
|
||||
"Diffbot",
|
||||
"DuckAssistBot",
|
||||
"FacebookBot",
|
||||
"Factset_spyderbot",
|
||||
"FirecrawlAgent",
|
||||
"FriendlyCrawler",
|
||||
"Google-Extended",
|
||||
"GoogleOther",
|
||||
"GoogleOther-Image",
|
||||
"GoogleOther-Video",
|
||||
"GPTBot",
|
||||
"iaskspider/2.0",
|
||||
"ICC-Crawler",
|
||||
"ImagesiftBot",
|
||||
"img2dataset",
|
||||
"imgproxy",
|
||||
"ISSCyberRiskCrawler",
|
||||
"Kangaroo Bot",
|
||||
"Meta-ExternalAgent",
|
||||
"Meta-ExternalFetcher",
|
||||
"NovaAct",
|
||||
"OAI-SearchBot",
|
||||
"omgili",
|
||||
"omgilibot",
|
||||
"Operator",
|
||||
"PanguBot",
|
||||
"Perplexity-User",
|
||||
"PerplexityBot",
|
||||
"PetalBot",
|
||||
"Scrapy",
|
||||
"SemrushBot-OCOB",
|
||||
"SemrushBot-SWA",
|
||||
"Sidetrade indexer bot",
|
||||
"TikTokSpider",
|
||||
"Timpibot",
|
||||
"VelenPublicWebCrawler",
|
||||
"Webzio-Extended",
|
||||
"YouBot"
|
||||
]
|
||||
}
|
Loading…
Add table
Add a link
Reference in a new issue