Modify robots.txt for maintenance and sitemap update
Updated robots.txt to disallow maintenance and change sitemap reference.
This commit is contained in:
1 parent
5b7742b132
commit
d085783872
1 file changed
+4
-2
+4
-2
@@ -1,11 +1,11 @@
|
||||
# Robots.txt for ssbg04/Portfolioo Portfolio
|
||||
# This file helps search engines navigate and index your portfolio site
|
||||
|
||||
User-agent: *
|
||||
Allow: /
|
||||
Disallow: /api/
|
||||
Disallow: /.vercel/
|
||||
Disallow: /.astro/
|
||||
Disallow: /maintenance
|
||||
|
||||
# Allow specific crawlers
|
||||
User-agent: Googlebot
|
||||
@@ -14,8 +14,10 @@ Allow: /
|
||||
User-agent: Bingbot
|
||||
Allow: /
|
||||
|
||||
Host: crischarles.com
|
||||
|
||||
# Crawl delay (optional - uncomment if needed)
|
||||
# Crawl-delay: 1
|
||||
|
||||
# Reference to sitemap
|
||||
Sitemap: https://crischarles.com/sitemap.xml
|
||||
Sitemap: https://crischarles.com/sitemap-index.xml
|
||||
Reference in new issue
Block a user