1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
User-agent: *
Crawl-delay: 10
Disallow: /wp-admin/
Disallow: /go/
Disallow: /wp-content/plugins/
Disallow: /?s=*
Disallow: /author/
Disallow: /astra-advanced-hook/
Disallow: /theme
Disallow: /demo/*
Disallow: /*/page/*
Disallow: /*blackhole
Disallow: /?blackhole
Allow: /wp-admin/admin-ajax.php

User-agent: YandexBot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: Googlebot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: YaK
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: MJ12bot
Disallow: /

Sitemap: https://www.xxx.com/sitemap.xml