Reduce test requirements duplication, sync flake8 and related (#28538)

* Generate pre-commit test dependencies instead of duplicating

* Upgrade/sync to flake8 3.7.9, flake8-docstrings 1.5.0, and pydocstyle 4.0.1

https://flake8.readthedocs.io/en/latest/release-notes/3.7.9.html
https://gitlab.com/pycqa/flake8-docstrings/blob/1.4.0/HISTORY.rst
https://gitlab.com/pycqa/flake8-docstrings/blob/1.5.0/HISTORY.rst
http://www.pydocstyle.org/en/4.0.1/release_notes.html

* Include requirements_test.txt from *_all.txt instead of copying

.pre-commit-config-all.yaml

@@ -19,12 +19,12 @@ repos:
           - --quiet
         files: ^((homeassistant|script|tests)/.+)?[^/]+\.py$
 -   repo: https://gitlab.com/pycqa/flake8
-    rev: 3.7.8
+    rev: 3.7.9
     hooks:
     -   id: flake8
         additional_dependencies:
-          - flake8-docstrings==1.3.1
-          - pydocstyle==4.0.0
+          - flake8-docstrings==1.5.0
+          - pydocstyle==4.0.1
         files: ^(homeassistant|script|tests)/.+\.py$
 # Using a local "system" mypy instead of the mypy hook, because its
 # results depend on what is installed. And the mypy hook runs in a

.pre-commit-config.yaml

@@ -15,10 +15,10 @@ repos:
           - --quiet
         files: ^((homeassistant|script|tests)/.+)?[^/]+\.py$
 -   repo: https://gitlab.com/pycqa/flake8
-    rev: 3.7.8
+    rev: 3.7.9
     hooks:
     -   id: flake8
         additional_dependencies:
-          - flake8-docstrings==1.3.1
-          - pydocstyle==4.0.0
+          - flake8-docstrings==1.5.0
+          - pydocstyle==4.0.1
         files: ^(homeassistant|script|tests)/.+\.py$

requirements_test.txt

@@ -2,16 +2,12 @@
 # make new things fail. Manually update these pins when pulling in a
 # new version
 
-# When updating this file, update .pre-commit-config*.yaml too
+-r requirements_test_pre_commit.txt
 asynctest==0.13.0
-black==19.10b0
 codecov==2.0.15
-flake8-docstrings==1.5.0
-flake8==3.7.8
 mock-open==1.3.1
 mypy==0.740
 pre-commit==1.20.0
-pydocstyle==4.0.1
 pylint==2.4.3
 astroid==2.3.2
 pytest-aiohttp==0.3.0

requirements_test_all.txt

@@ -1,28 +1,7 @@
-# Home Assistant test
-# linters such as flake8 and pylint should be pinned, as new releases
-# make new things fail. Manually update these pins when pulling in a
-# new version
-
-# When updating this file, update .pre-commit-config*.yaml too
-asynctest==0.13.0
-black==19.10b0
-codecov==2.0.15
-flake8-docstrings==1.5.0
-flake8==3.7.8
-mock-open==1.3.1
-mypy==0.740
-pre-commit==1.20.0
-pydocstyle==4.0.1
-pylint==2.4.3
-astroid==2.3.2
-pytest-aiohttp==0.3.0
-pytest-cov==2.8.1
-pytest-sugar==0.9.2
-pytest-timeout==1.3.3
-pytest==5.2.2
-requests_mock==1.7.0
-responses==0.10.6
+# Home Assistant tests, full dependency set
+# Automatically generated by gen_requirements_all.py, do not edit
 
+-r requirements_test.txt
 
 # homeassistant.components.homekit
 HAP-python==2.6.0

requirements_test_pre_commit.txt

@@ -0,0 +1,6 @@
+# Automatically generated from .pre-commit-config-all.yaml by gen_requirements_all.py, do not edit
+
+black==19.10b0
+flake8-docstrings==1.5.0
+flake8==3.7.9
+pydocstyle==4.0.1

script/gen_requirements_all.py

@@ -8,6 +8,8 @@ import pkgutil
 import re
 import sys
 
+from homeassistant.util.yaml.loader import load_yaml
+
 from script.hassfest.model import Integration
 
 COMMENT_REQUIREMENTS = (
@@ -225,10 +227,11 @@ def requirements_all_output(reqs):
 def requirements_test_output(reqs):
     """Generate output for test_requirements."""
     output = []
-    output.append("# Home Assistant test")
-    output.append("\n")
-    output.append(Path("requirements_test.txt").read_text())
-    output.append("\n")
+    output.append("# Home Assistant tests, full dependency set\n")
+    output.append(
+        f"# Automatically generated by {Path(__file__).name}, do not edit\n\n"
+    )
+    output.append("-r requirements_test.txt\n")
 
     filtered = {
         requirement: modules
@@ -246,6 +249,24 @@ def requirements_test_output(reqs):
     return "".join(output)
 
 
+def requirements_pre_commit_output():
+    """Generate output for pre-commit dependencies."""
+    source = ".pre-commit-config-all.yaml"
+    pre_commit_conf = load_yaml(source)
+    reqs = []
+    for repo in (x for x in pre_commit_conf["repos"] if x.get("rev")):
+        for hook in repo["hooks"]:
+            reqs.append(f"{hook['id']}=={repo['rev']}")
+            reqs.extend(x for x in hook.get("additional_dependencies", ()))
+    output = [
+        f"# Automatically generated "
+        f"from {source} by {Path(__file__).name}, do not edit",
+        "",
+    ]
+    output.extend(sorted(reqs))
+    return "\n".join(output) + "\n"
+
+
 def gather_constraints():
     """Construct output for constraint file."""
     return (
@@ -285,10 +306,12 @@ def main(validate):
 
     reqs_file = requirements_all_output(data)
     reqs_test_file = requirements_test_output(data)
+    reqs_pre_commit_file = requirements_pre_commit_output()
     constraints = gather_constraints()
 
     files = (
         ("requirements_all.txt", reqs_file),
+        ("requirements_test_pre_commit.txt", reqs_pre_commit_file),
         ("requirements_test_all.txt", reqs_test_file),
         ("homeassistant/package_constraints.txt", constraints),
     )