Merge pull request #10972 from ethereum/docs_seo

[DOCS] Create robots.txt / Avoid search results showing docs of old Solidity versions

ReleaseChecklist.md

@@ -55,8 +55,9 @@
  - [ ] Wait for the ``~ethereum/ubuntu/ethereum-static`` PPA build to be finished and published for *all platforms*. SERIOUSLY: DO NOT PROCEED EARLIER!!! *After* the static builds are *published*, copy the static package to the ``~ethereum/ubuntu/ethereum`` PPA for the destination series ``Trusty``, ``Xenial`` and ``Bionic`` while selecting ``Copy existing binaries``.
 
 ### Documentation
- - [ ] Build the new version on https://readthedocs.org/projects/solidity/ (select `latest` at the bottom of the page and click `BUILD`)
+ - [ ] Build the new version on https://readthedocs.org/projects/solidity/ (select `latest` at the bottom of the page and click `BUILD`).
  - [ ] In the admin panel, select `Versions` in the menu and set the default version to the released one.
+ - [ ] If it is a non-breaking release, block indexing of previous release version in the ``robots.txt`` file.
 
 ### Release solc-js
  - [ ] Wait until solc-bin was properly deployed. You can test this via remix - a test run through remix is advisable anyway.

docs/_static/robots.txt

@@ -0,0 +1,95 @@
+Sitemap: http://docs.soliditylang.org/sitemap.xml
+
+# Prevent documentation for the development branches from showing up in search results.
+Disallow: /en/develop/
+Disallow: /en/breaking/
+
+# Prevent documentation for the older Solidity versions from showing up in search results.
+
+Disallow: /en/v0.1.2/
+Disallow: /en/v0.1.3/
+Disallow: /en/v0.1.4/
+Disallow: /en/v0.1.5/
+Disallow: /en/v0.1.6/
+Disallow: /en/v0.1.7/
+Disallow: /en/v0.2.0/
+Disallow: /en/v0.2.1/
+Disallow: /en/v0.2.2/
+Disallow: /en/v0.3.0/
+Disallow: /en/v0.3.1/
+Disallow: /en/v0.3.2/
+Disallow: /en/v0.3.3/
+Disallow: /en/v0.3.4/
+Disallow: /en/v0.3.5/
+Disallow: /en/v0.3.6/
+Disallow: /en/v0.4.0/
+Disallow: /en/v0.4.1/
+Disallow: /en/v0.4.2/
+Disallow: /en/v0.4.3/
+Disallow: /en/v0.4.4/
+Disallow: /en/v0.4.5/
+Disallow: /en/v0.4.6/
+Disallow: /en/v0.4.7/
+Disallow: /en/v0.4.8/
+Disallow: /en/v0.4.9/
+Disallow: /en/v0.4.10/
+Disallow: /en/v0.4.11/
+Disallow: /en/v0.4.12/
+Disallow: /en/v0.4.13/
+Disallow: /en/v0.4.14/
+Disallow: /en/v0.4.15/
+Disallow: /en/v0.4.16/
+Disallow: /en/v0.4.17/
+Disallow: /en/v0.4.18/
+Disallow: /en/v0.4.19/
+Disallow: /en/v0.4.20/
+Disallow: /en/v0.4.21/
+Disallow: /en/v0.4.22/
+Disallow: /en/v0.4.23/
+Disallow: /en/v0.4.24/
+Disallow: /en/v0.4.25/
+Disallow: /en/v0.4.26/
+Disallow: /en/v0.5.0/
+Disallow: /en/v0.5.1/
+Disallow: /en/v0.5.2/
+Disallow: /en/v0.5.3/
+Disallow: /en/v0.5.4/
+Disallow: /en/v0.5.5/
+Disallow: /en/v0.5.6/
+Disallow: /en/v0.5.7/
+Disallow: /en/v0.5.8/
+Disallow: /en/v0.5.9/
+Disallow: /en/v0.5.10/
+Disallow: /en/v0.5.11/
+Disallow: /en/v0.5.12/
+Disallow: /en/v0.5.13/
+Disallow: /en/v0.5.14/
+Disallow: /en/v0.5.15/
+Disallow: /en/v0.5.16/
+Disallow: /en/v0.5.17/
+Disallow: /en/v0.6.0/
+Disallow: /en/v0.6.1/
+Disallow: /en/v0.6.2/
+Disallow: /en/v0.6.3/
+Disallow: /en/v0.6.4/
+Disallow: /en/v0.6.5/
+Disallow: /en/v0.6.6/
+Disallow: /en/v0.6.7/
+Disallow: /en/v0.6.8/
+Disallow: /en/v0.6.9/
+Disallow: /en/v0.6.10/
+Disallow: /en/v0.6.11/
+Disallow: /en/v0.6.12/
+Disallow: /en/v0.7.0/
+Disallow: /en/v0.7.1/
+Disallow: /en/v0.7.2/
+Disallow: /en/v0.7.3/
+Disallow: /en/v0.7.4/
+Disallow: /en/v0.7.5/
+# Allow the last patch release of the 0.7.x series
+#Disallow: /en/v0.7.6/
+Disallow: /en/v0.8.0/
+Disallow: /en/v0.8.1/
+Disallow: /en/v0.8.2/
+# Allow the latest release
+#Disallow: /en/v0.8.3/

docs/conf.py

@@ -156,7 +156,7 @@ html_js_files = ["js/toggle.js"]
 # Add any extra paths that contain custom files (such as robots.txt or
 # .htaccess) here, relative to this directory. These files are copied
 # directly to the root of the documentation.
-html_extra_path = ["_static/css"]
+html_extra_path = ["_static/css", "_static/robots.txt"]
 
 # If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
 # using the given strftime format.