fix(seo): add HSTS headers, clean crawler directives, and exclude /links from sitemap

.gitignore

@@ -26,7 +26,7 @@ Sanity-studio/
 
 # production
 /build
-
+.agents/
 # misc
 .DS_Store
 *.pem
@@ -56,3 +56,4 @@ next-env.d.ts
 # astro
 **/.astro/
 dist/
+

astro.config.mjs

@@ -24,6 +24,7 @@ export default defineConfig({
         !page.includes('/maintenance') &&
         !page.includes('/chat') &&
         !page.includes('/gallery') &&
+        !page.includes('/links') &&
         !page.includes('/_image'),
       serialize(item) {
         if (item.url === 'https://crischarles.com' || item.url === 'https://crischarles.com/') {

public/_headers

@@ -0,0 +1,5 @@
+/*
+  Strict-Transport-Security: max-age=31536000; includeSubDomains; preload
+  X-Content-Type-Options: nosniff
+  X-Frame-Options: SAMEORIGIN
+  Referrer-Policy: strict-origin-when-cross-origin

public/robots.txt

@@ -14,20 +14,5 @@ User-agent: Googlebot-Image
 Disallow: /gallery
 Disallow: /gallery/
 
-# Allow specific crawlers
-User-agent: Googlebot
-Allow: /
-Disallow: /api/
-Disallow: /.vercel/
-Disallow: /.astro/
-Disallow: /maintenance
-Disallow: /gallery
-Disallow: /gallery/
-
-User-agent: Bingbot
-Allow: /
-Disallow: /gallery
-Disallow: /gallery/
-
 # Reference to sitemap
 Sitemap: https://crischarles.com/sitemap-index.xml