Add sitemap and robots.txt (#163)

* yarn add next-sitemap and config

* add sitemap to .gitignore

* commit generated robots.txt

.gitignore

@@ -34,3 +34,6 @@ yarn-error.log*
 # typescript
 *.tsbuildinfo
 next-env.d.ts
+
+# Sitemaps
+sitemap*.xml

next-sitemap.config.js

@@ -0,0 +1,6 @@
+/** @type {import('next-sitemap').IConfig} */
+
+module.exports = {
+  siteUrl: 'https://geth.ethereum.org',
+  generateRobotsTxt: true
+};

package.json

@@ -7,6 +7,7 @@
   "scripts": {
     "dev": "next dev",
     "build": "npm run lint && next build && npm run format:fix",
+    "postbuild": "next-sitemap",
     "start": "next start",
     "lint": "next lint",
     "format:fix": "prettier . --write --config .prettierrc --ignore-path .prettierignore --loglevel warn"
@@ -23,6 +24,7 @@
     "gray-matter": "^4.0.3",
     "js-yaml": "^4.1.0",
     "next": "^13.0.2",
+    "next-sitemap": "^3.1.32",
     "react": "18.2.0",
     "react-dom": "18.2.0",
     "react-markdown": "^8.0.3",

public/robots.txt

@@ -0,0 +1,9 @@
+# *
+User-agent: *
+Allow: /
+
+# Host
+Host: https://geth.ethereum.org
+
+# Sitemaps
+Sitemap: https://geth.ethereum.org/sitemap.xml

yarn.lock

@@ -806,6 +806,11 @@
   resolved "https://registry.yarnpkg.com/@chakra-ui/visually-hidden/-/visually-hidden-2.0.9.tgz#b43a3dd0bc1108954ad0eeb50d0261887ab5e31c"
   integrity sha512-PkNxrRGp9H3bdqEaoo8XGt/AL9UuGRTom0/9XJa+G/Dj8Cy1sDuamOWk3pN/ZQs46RokfK9Uh5LqPY5dwSDweg==
 
+"@corex/deepmerge@^4.0.29":
+  version "4.0.29"
+  resolved "https://registry.yarnpkg.com/@corex/deepmerge/-/deepmerge-4.0.29.tgz#af9debf07d7f6b0d2a9d04a266abf2c1418ed2f6"
+  integrity sha512-q/yVUnqckA8Do+EvAfpy7RLdumnBy9ZsducMUtZTvpdbJC7azEf1hGtnYYxm0QfphYxjwggv6XtH64prvS1W+A==
+
 "@ctrl/tinycolor@^3.4.0":
   version "3.4.1"
   resolved "https://registry.npmjs.org/@ctrl/tinycolor/-/tinycolor-3.4.1.tgz"
@@ -3215,6 +3220,14 @@ natural-compare@^1.4.0:
   resolved "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz"
   integrity sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==
 
+next-sitemap@^3.1.32:
+  version "3.1.32"
+  resolved "https://registry.yarnpkg.com/next-sitemap/-/next-sitemap-3.1.32.tgz#e4a7227cab23b5e5c68bc54d335b86d3ff1e05f8"
+  integrity sha512-jkIKpwLXpWWTPfmDO46+6nu4+qpar4CjvUwCR9rYZHWtzE/wFfaCVFKpGtFMl6MFjpu8GjiE6kWFEa7uF3bzzg==
+  dependencies:
+    "@corex/deepmerge" "^4.0.29"
+    minimist "^1.2.6"
+
 next@^13.0.2:
   version "13.0.2"
   resolved "https://registry.npmjs.org/next/-/next-13.0.2.tgz"