Add robots.txt file to prevent indexing outdated docs (#3002)

Closes #2912

conf.py

@@ -91,6 +91,8 @@ html_logo = 'img/docs_logo.png'
 # These folders are copied to the documentation's HTML output
 html_static_path = ['_static']
 
+html_extra_path = ['robots.txt']
+
 # These paths are either relative to html_static_path
 # or fully qualified paths (eg. https://...)
 html_css_files = [

robots.txt

@@ -0,0 +1,6 @@
+user-agent: *
+disallow: /*/3.1
+disallow: /*/3.0
+disallow: /*/2.0
+
+sitemap: https://docs.godotengine.org/sitemap.xml