-
Notifications
You must be signed in to change notification settings - Fork 1
/
next-sitemap.config.js
98 lines (77 loc) · 1.76 KB
/
next-sitemap.config.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
/** @type {import('next-sitemap').IConfig} */
// Source: https://www.foundationwebdev.com/2023/11/which-web-crawlers-are-associated-with-ai-crawlers
const llmUserAgents = [
// Anthrophic
"anthropic-ai",
"Claude-Web",
// Apple
"Applebot-Extended",
// ByteDance
"Bytespider",
// Common Crawl
"CCBot",
// OpenAI
"ChatGPT-User",
"GPTBot",
// Cohere AI
"cohere-ai",
// Diffbot
"Diffbot",
// Meta
"FacebookBot",
// Google
"GoogleOther",
"Google-Extended",
// The Hive
"ImagesiftBot",
// Webz.io
"OmigiliBot",
"Omigili",
];
// Source: https://darkvisitors.com/agents
const otherUserAgents = [
// Adbeat
"adbeat_bot",
// Google
"Adsbot-Google",
"Adsbot-Google-Mobile",
// aiHit
"aiHitBot",
// Anders Pink
"AndersPinkBot",
// Awario
"AwarioBot",
"AwarioSmartBot",
// Bitsight
"BitSightBot",
];
const fullyBlockedUserAgents = [...llmUserAgents, ...otherUserAgents];
module.exports = {
siteUrl: "https://hkamran.com",
changefreq: "daily",
priority: 0.7,
sitemapSize: 5000,
generateRobotsTxt: true,
exclude: ["/_next/", "/api/", "/*.png"],
robotsTxtOptions: {
policies: [
{
userAgent: "*",
allow: "/",
disallow: ["/_next/", "/api/", "/*.png"],
},
...fullyBlockedUserAgents.map((userAgent) => ({
userAgent,
disallow: "/",
})),
//{
// userAgent: "GPTBot",
// disallow: "/",
//},
//{
// userAgent: "Applebot-Extended",
// disallow: "/",
//},
],
},
};