Merge pull request #11317 from cxxboy/robots_update_script

script to automate updates to robots.txt (v2)

ReleaseChecklist.md

@@ -61,7 +61,6 @@
 ### Documentation
  - [ ] Build the new version on https://readthedocs.org/projects/solidity/ (select `latest` at the bottom of the page and click `BUILD`).
  - [ ] In the admin panel, select `Versions` in the menu and set the default version to the released one.
- - [ ] If it is a non-breaking release, block indexing of previous release version in the ``robots.txt`` file.
 
 ### Release solc-js
  - [ ] Wait until solc-bin was properly deployed. You can test this via remix - a test run through remix is advisable anyway.

docs/_static/robots.txt

@@ -1,95 +0,0 @@
-Sitemap: http://docs.soliditylang.org/sitemap.xml
-
-# Prevent documentation for the development branches from showing up in search results.
-Disallow: /en/develop/
-Disallow: /en/breaking/
-
-# Prevent documentation for the older Solidity versions from showing up in search results.
-
-Disallow: /en/v0.1.2/
-Disallow: /en/v0.1.3/
-Disallow: /en/v0.1.4/
-Disallow: /en/v0.1.5/
-Disallow: /en/v0.1.6/
-Disallow: /en/v0.1.7/
-Disallow: /en/v0.2.0/
-Disallow: /en/v0.2.1/
-Disallow: /en/v0.2.2/
-Disallow: /en/v0.3.0/
-Disallow: /en/v0.3.1/
-Disallow: /en/v0.3.2/
-Disallow: /en/v0.3.3/
-Disallow: /en/v0.3.4/
-Disallow: /en/v0.3.5/
-Disallow: /en/v0.3.6/
-Disallow: /en/v0.4.0/
-Disallow: /en/v0.4.1/
-Disallow: /en/v0.4.2/
-Disallow: /en/v0.4.3/
-Disallow: /en/v0.4.4/
-Disallow: /en/v0.4.5/
-Disallow: /en/v0.4.6/
-Disallow: /en/v0.4.7/
-Disallow: /en/v0.4.8/
-Disallow: /en/v0.4.9/
-Disallow: /en/v0.4.10/
-Disallow: /en/v0.4.11/
-Disallow: /en/v0.4.12/
-Disallow: /en/v0.4.13/
-Disallow: /en/v0.4.14/
-Disallow: /en/v0.4.15/
-Disallow: /en/v0.4.16/
-Disallow: /en/v0.4.17/
-Disallow: /en/v0.4.18/
-Disallow: /en/v0.4.19/
-Disallow: /en/v0.4.20/
-Disallow: /en/v0.4.21/
-Disallow: /en/v0.4.22/
-Disallow: /en/v0.4.23/
-Disallow: /en/v0.4.24/
-Disallow: /en/v0.4.25/
-Disallow: /en/v0.4.26/
-Disallow: /en/v0.5.0/
-Disallow: /en/v0.5.1/
-Disallow: /en/v0.5.2/
-Disallow: /en/v0.5.3/
-Disallow: /en/v0.5.4/
-Disallow: /en/v0.5.5/
-Disallow: /en/v0.5.6/
-Disallow: /en/v0.5.7/
-Disallow: /en/v0.5.8/
-Disallow: /en/v0.5.9/
-Disallow: /en/v0.5.10/
-Disallow: /en/v0.5.11/
-Disallow: /en/v0.5.12/
-Disallow: /en/v0.5.13/
-Disallow: /en/v0.5.14/
-Disallow: /en/v0.5.15/
-Disallow: /en/v0.5.16/
-Disallow: /en/v0.5.17/
-Disallow: /en/v0.6.0/
-Disallow: /en/v0.6.1/
-Disallow: /en/v0.6.2/
-Disallow: /en/v0.6.3/
-Disallow: /en/v0.6.4/
-Disallow: /en/v0.6.5/
-Disallow: /en/v0.6.6/
-Disallow: /en/v0.6.7/
-Disallow: /en/v0.6.8/
-Disallow: /en/v0.6.9/
-Disallow: /en/v0.6.10/
-Disallow: /en/v0.6.11/
-Disallow: /en/v0.6.12/
-Disallow: /en/v0.7.0/
-Disallow: /en/v0.7.1/
-Disallow: /en/v0.7.2/
-Disallow: /en/v0.7.3/
-Disallow: /en/v0.7.4/
-Disallow: /en/v0.7.5/
-# Allow the last patch release of the 0.7.x series
-#Disallow: /en/v0.7.6/
-Disallow: /en/v0.8.0/
-Disallow: /en/v0.8.1/
-Disallow: /en/v0.8.2/
-# Allow the latest release
-#Disallow: /en/v0.8.3/

docs/robots.txt.template

@@ -0,0 +1,14 @@
+User-Agent: *
+Sitemap: http://docs.soliditylang.org/sitemap.xml
+
+Allow: /en/latest/
+Allow: /en/v0.7.6/
+Allow: /en/v{{ LATEST_VERSION }}/
+Allow: /_/downloads/en/latest/
+Allow: /_/downloads/en/0.7.6/
+Allow: /_/downloads/en/{{ LATEST_VERSION }}/
+
+# Prevent documentation for the development branches and older Solidity
+# versions from showing up in search results.
+Disallow: /en/*
+Disallow: /_/downloads/en/*

scripts/docs.sh

@@ -27,6 +27,7 @@
 #------------------------------------------------------------------------------
 
 set -e
+scripts/update_robotstxt.sh
 cd docs
 pip3 install -r requirements.txt
 sphinx-build -nW -b html -d _build/doctrees . _build/html

scripts/update_robotstxt.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+
+#------------------------------------------------------------------------------
+# solidity is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# solidity is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with solidity.  If not, see <http://www.gnu.org/licenses/>
+#
+# (c) 2021 solidity contributors.
+#------------------------------------------------------------------------------
+
+set -eu
+
+repo_root="$(dirname "$0")/.."
+robots_template_path="${repo_root}/docs/robots.txt.template"
+robots_txt_path="${repo_root}/docs/_static/robots.txt"
+latest_version=$("${repo_root}/scripts/get_version.sh")
+
+sed -E -e "s/\{\{[[:space:]]LATEST_VERSION[[:space:]]\}\}/${latest_version}/g" "$robots_template_path" > "$robots_txt_path"