Generate requirements per supported architecture (#115708)

* Generate requirements per supported architecture

* Don't store wheels requirements in the repo

* Dry run

* Set Python version

* Install base packages

* Fix

* Fix

* Fix

* Fix typo

Co-authored-by: Martin Hjelmare <marhje52@gmail.com>

* Genarate requirements_all_pytest.txt

* Fix hassfest

* Reenable building wheels

* Remove unneeded code

* Address review comment

* Fix lying comment

* Add tests, address review comments

* Deduplicate

* Fix file name

* Add comment

---------

Co-authored-by: Martin Hjelmare <marhje52@gmail.com>

.github/workflows/ci.yaml

@@ -97,7 +97,8 @@ jobs:
             hashFiles('requirements_test.txt', 'requirements_test_pre_commit.txt') }}-${{
             hashFiles('requirements.txt') }}-${{
             hashFiles('requirements_all.txt') }}-${{
-            hashFiles('homeassistant/package_constraints.txt') }}" >> $GITHUB_OUTPUT
+            hashFiles('homeassistant/package_constraints.txt') }}-${{
+            hashFiles('script/gen_requirements_all.py') }}" >> $GITHUB_OUTPUT
       - name: Generate partial pre-commit restore key
         id: generate_pre-commit_cache_key
         run: >-
@@ -497,8 +498,9 @@ jobs:
           python --version
           pip install "$(grep '^uv' < requirements_test.txt)"
           uv pip install -U "pip>=21.3.1" setuptools wheel
-          uv pip install -r requirements_all.txt
+          uv pip install -r requirements.txt
-          uv pip install "$(grep 'python-gammu' < requirements_all.txt | sed -e 's|# python-gammu|python-gammu|g')"
+          python -m script.gen_requirements_all ci
+          uv pip install -r requirements_all_pytest.txt
           uv pip install -r requirements_test.txt
           uv pip install -e . --config-settings editable_mode=compat
 

.github/workflows/wheels.yml

@@ -14,6 +14,10 @@ on:
       - "homeassistant/package_constraints.txt"
       - "requirements_all.txt"
       - "requirements.txt"
+      - "script/gen_requirements_all.py"
+
+env:
+  DEFAULT_PYTHON: "3.12"
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref_name}}
@@ -30,6 +34,21 @@ jobs:
       - name: Checkout the repository
         uses: actions/checkout@v4.1.3
 
+      - name: Set up Python ${{ env.DEFAULT_PYTHON }}
+        id: python
+        uses: actions/setup-python@v5.1.0
+        with:
+          python-version: ${{ env.DEFAULT_PYTHON }}
+          check-latest: true
+
+      - name: Create Python virtual environment
+        run: |
+          python -m venv venv
+          . venv/bin/activate
+          python --version
+          pip install "$(grep '^uv' < requirements_test.txt)"
+          uv pip install -r requirements.txt
+
       - name: Get information
         id: info
         uses: home-assistant/actions/helpers/info@master
@@ -76,6 +95,17 @@ jobs:
           path: ./requirements_diff.txt
           overwrite: true
 
+      - name: Generate requirements
+        run: |
+          . venv/bin/activate
+          python -m script.gen_requirements_all ci
+
+      - name: Upload requirements_all_wheels
+        uses: actions/upload-artifact@v4.3.1
+        with:
+          name: requirements_all_wheels
+          path: ./requirements_all_wheels_*.txt
+
   core:
     name: Build Core wheels ${{ matrix.abi }} for ${{ matrix.arch }} (musllinux_1_2)
     if: github.repository_owner == 'home-assistant'
@@ -138,30 +168,10 @@ jobs:
         with:
           name: requirements_diff
 
-      - name: (Un)comment packages
+      - name: Download requirements_all_wheels
-        run: |
+        uses: actions/download-artifact@v4.1.4
-          requirement_files="requirements_all.txt requirements_diff.txt"
+        with:
-          for requirement_file in ${requirement_files}; do
+          name: requirements_all_wheels
-            sed -i "s|# pyuserinput|pyuserinput|g" ${requirement_file}
-            sed -i "s|# evdev|evdev|g" ${requirement_file}
-            sed -i "s|# pycups|pycups|g" ${requirement_file}
-            sed -i "s|# decora-wifi|decora-wifi|g" ${requirement_file}
-            sed -i "s|# python-gammu|python-gammu|g" ${requirement_file}
-
-            # Some packages are not buildable on armhf anymore
-            if [ "${{ matrix.arch }}" = "armhf" ]; then
-
-              # Pandas has issues building on armhf, it is expected they
-              # will drop the platform in the near future (they consider it
-              # "flimsy" on 386). The following packages depend on pandas,
-              # so we comment them out.
-              sed -i "s|env-canada|# env-canada|g" ${requirement_file}
-              sed -i "s|noaa-coops|# noaa-coops|g" ${requirement_file}
-              sed -i "s|pyezviz|# pyezviz|g" ${requirement_file}
-              sed -i "s|pykrakenapi|# pykrakenapi|g" ${requirement_file}
-            fi
-
-          done
 
       - name: Split requirements all
         run: |
@@ -169,7 +179,7 @@ jobs:
           # This is to prevent the build from running out of memory when
           # resolving packages on 32-bits systems (like armhf, armv7).
 
-          split -l $(expr $(expr $(cat requirements_all.txt | wc -l) + 1) / 3) requirements_all.txt requirements_all.txt
+          split -l $(expr $(expr $(cat requirements_all.txt | wc -l) + 1) / 3) requirements_all_wheels_${{ matrix.arch }}.txt requirements_all.txt
 
       - name: Create requirements for cython<3
         run: |

script/gen_requirements_all.py

@@ -17,7 +17,10 @@ from typing import Any
 from homeassistant.util.yaml.loader import load_yaml
 from script.hassfest.model import Integration
 
-COMMENT_REQUIREMENTS = (
+# Requirements which can't be installed on all systems because they rely on additional
+# system packages. Requirements listed in EXCLUDED_REQUIREMENTS_ALL will be commented-out
+# in requirements_all.txt and requirements_test_all.txt.
+EXCLUDED_REQUIREMENTS_ALL = {
     "atenpdu",  # depends on pysnmp which is not maintained at this time
     "avea",  # depends on bluepy
     "avion",
@@ -36,10 +39,39 @@ COMMENT_REQUIREMENTS = (
     "pyuserinput",
     "tensorflow",
     "tf-models-official",
-)
+}
 
-COMMENT_REQUIREMENTS_NORMALIZED = {
+# Requirements excluded by EXCLUDED_REQUIREMENTS_ALL which should be included when
-    commented.lower().replace("_", "-") for commented in COMMENT_REQUIREMENTS
+# building integration wheels for all architectures.
+INCLUDED_REQUIREMENTS_WHEELS = {
+    "decora-wifi",
+    "evdev",
+    "pycups",
+    "python-gammu",
+    "pyuserinput",
+}
+
+
+# Requirements to exclude or include when running github actions.
+# Requirements listed in "exclude" will be commented-out in
+# requirements_all_{action}.txt
+# Requirements listed in "include" must be listed in EXCLUDED_REQUIREMENTS_CI, and
+# will be included in requirements_all_{action}.txt
+
+OVERRIDDEN_REQUIREMENTS_ACTIONS = {
+    "pytest": {"exclude": set(), "include": {"python-gammu"}},
+    "wheels_aarch64": {"exclude": set(), "include": INCLUDED_REQUIREMENTS_WHEELS},
+    # Pandas has issues building on armhf, it is expected they
+    # will drop the platform in the near future (they consider it
+    # "flimsy" on 386). The following packages depend on pandas,
+    # so we comment them out.
+    "wheels_armhf": {
+        "exclude": {"env-canada", "noaa-coops", "pyezviz", "pykrakenapi"},
+        "include": INCLUDED_REQUIREMENTS_WHEELS,
+    },
+    "wheels_armv7": {"exclude": set(), "include": INCLUDED_REQUIREMENTS_WHEELS},
+    "wheels_amd64": {"exclude": set(), "include": INCLUDED_REQUIREMENTS_WHEELS},
+    "wheels_i386": {"exclude": set(), "include": INCLUDED_REQUIREMENTS_WHEELS},
 }
 
 IGNORE_PIN = ("colorlog>2.1,<3", "urllib3")
@@ -254,6 +286,12 @@ def gather_recursive_requirements(
     return reqs
 
 
+def _normalize_package_name(package_name: str) -> str:
+    """Normalize a package name."""
+    # pipdeptree needs lowercase and dash instead of underscore or period as separator
+    return package_name.lower().replace("_", "-").replace(".", "-")
+
+
 def normalize_package_name(requirement: str) -> str:
     """Return a normalized package name from a requirement string."""
     # This function is also used in hassfest.
@@ -262,12 +300,24 @@ def normalize_package_name(requirement: str) -> str:
         return ""
 
     # pipdeptree needs lowercase and dash instead of underscore or period as separator
-    return match.group(1).lower().replace("_", "-").replace(".", "-")
+    return _normalize_package_name(match.group(1))
 
 
 def comment_requirement(req: str) -> bool:
     """Comment out requirement. Some don't install on all systems."""
-    return normalize_package_name(req) in COMMENT_REQUIREMENTS_NORMALIZED
+    return normalize_package_name(req) in EXCLUDED_REQUIREMENTS_ALL
+
+
+def process_action_requirement(req: str, action: str) -> str:
+    """Process requirement for a specific github action."""
+    normalized_package_name = normalize_package_name(req)
+    if normalized_package_name in OVERRIDDEN_REQUIREMENTS_ACTIONS[action]["exclude"]:
+        return f"# {req}"
+    if normalized_package_name in OVERRIDDEN_REQUIREMENTS_ACTIONS[action]["include"]:
+        return req
+    if normalized_package_name in EXCLUDED_REQUIREMENTS_ALL:
+        return f"# {req}"
+    return req
 
 
 def gather_modules() -> dict[str, list[str]] | None:
@@ -353,6 +403,16 @@ def generate_requirements_list(reqs: dict[str, list[str]]) -> str:
     return "".join(output)
 
 
+def generate_action_requirements_list(reqs: dict[str, list[str]], action: str) -> str:
+    """Generate a pip file based on requirements."""
+    output = []
+    for pkg, requirements in sorted(reqs.items(), key=itemgetter(0)):
+        output.extend(f"\n# {req}" for req in sorted(requirements))
+        processed_pkg = process_action_requirement(pkg, action)
+        output.append(f"\n{processed_pkg}\n")
+    return "".join(output)
+
+
 def requirements_output() -> str:
     """Generate output for requirements."""
     output = [
@@ -379,6 +439,18 @@ def requirements_all_output(reqs: dict[str, list[str]]) -> str:
     return "".join(output)
 
 
+def requirements_all_action_output(reqs: dict[str, list[str]], action: str) -> str:
+    """Generate output for requirements_all_{action}."""
+    output = [
+        f"# Home Assistant Core, full dependency set for {action}\n",
+        GENERATED_MESSAGE,
+        "-r requirements.txt\n",
+    ]
+    output.append(generate_action_requirements_list(reqs, action))
+
+    return "".join(output)
+
+
 def requirements_test_all_output(reqs: dict[str, list[str]]) -> str:
     """Generate output for test_requirements."""
     output = [
@@ -459,7 +531,7 @@ def diff_file(filename: str, content: str) -> list[str]:
     )
 
 
-def main(validate: bool) -> int:
+def main(validate: bool, ci: bool) -> int:
     """Run the script."""
     if not os.path.isfile("requirements_all.txt"):
         print("Run this from HA root dir")
@@ -472,17 +544,28 @@ def main(validate: bool) -> int:
 
     reqs_file = requirements_output()
     reqs_all_file = requirements_all_output(data)
+    reqs_all_action_files = {
+        action: requirements_all_action_output(data, action)
+        for action in OVERRIDDEN_REQUIREMENTS_ACTIONS
+    }
     reqs_test_all_file = requirements_test_all_output(data)
+    # Always calling requirements_pre_commit_output is intentional to ensure
+    # the code is called by the pre-commit hooks.
     reqs_pre_commit_file = requirements_pre_commit_output()
     constraints = gather_constraints()
 
-    files = (
+    files = [
         ("requirements.txt", reqs_file),
         ("requirements_all.txt", reqs_all_file),
         ("requirements_test_pre_commit.txt", reqs_pre_commit_file),
         ("requirements_test_all.txt", reqs_test_all_file),
         ("homeassistant/package_constraints.txt", constraints),
-    )
+    ]
+    if ci:
+        files.extend(
+            (f"requirements_all_{action}.txt", reqs_all_file)
+            for action, reqs_all_file in reqs_all_action_files.items()
+        )
 
     if validate:
         errors = []
@@ -511,4 +594,5 @@ def main(validate: bool) -> int:
 
 if __name__ == "__main__":
     _VAL = sys.argv[-1] == "validate"
-    sys.exit(main(_VAL))
+    _CI = sys.argv[-1] == "ci"
+    sys.exit(main(_VAL, _CI))

script/hassfest/requirements.py

@@ -15,13 +15,13 @@ from awesomeversion import AwesomeVersion, AwesomeVersionStrategy
 from tqdm import tqdm
 
 import homeassistant.util.package as pkg_util
-from script.gen_requirements_all import COMMENT_REQUIREMENTS, normalize_package_name
+from script.gen_requirements_all import (
+    EXCLUDED_REQUIREMENTS_ALL,
+    normalize_package_name,
+)
 
 from .model import Config, Integration
 
-IGNORE_PACKAGES = {
-    commented.lower().replace("_", "-") for commented in COMMENT_REQUIREMENTS
-}
 PACKAGE_REGEX = re.compile(
     r"^(?:--.+\s)?([-_,\.\w\d\[\]]+)(==|>=|<=|~=|!=|<|>|===)*(.*)$"
 )
@@ -116,7 +116,7 @@ def validate_requirements(integration: Integration) -> None:
                 f"Failed to normalize package name from requirement {req}",
             )
             return
-        if package in IGNORE_PACKAGES:
+        if package in EXCLUDED_REQUIREMENTS_ALL:
             continue
         integration_requirements.add(req)
         integration_packages.add(package)

tests/script/__init__.py

@@ -0,0 +1 @@
+"""Tests for scripts."""

tests/script/test_gen_requirements_all.py

@@ -0,0 +1,25 @@
+"""Tests for the gen_requirements_all script."""
+
+from script import gen_requirements_all
+
+
+def test_overrides_normalized() -> None:
+    """Test override lists are using normalized package names."""
+    for req in gen_requirements_all.EXCLUDED_REQUIREMENTS_ALL:
+        assert req == gen_requirements_all._normalize_package_name(req)
+    for req in gen_requirements_all.INCLUDED_REQUIREMENTS_WHEELS:
+        assert req == gen_requirements_all._normalize_package_name(req)
+    for overrides in gen_requirements_all.OVERRIDDEN_REQUIREMENTS_ACTIONS.values():
+        for req in overrides["exclude"]:
+            assert req == gen_requirements_all._normalize_package_name(req)
+        for req in overrides["include"]:
+            assert req == gen_requirements_all._normalize_package_name(req)
+
+
+def test_include_overrides_subsets() -> None:
+    """Test packages in include override lists are present in the exclude list."""
+    for req in gen_requirements_all.INCLUDED_REQUIREMENTS_WHEELS:
+        assert req in gen_requirements_all.EXCLUDED_REQUIREMENTS_ALL
+    for overrides in gen_requirements_all.OVERRIDDEN_REQUIREMENTS_ACTIONS.values():
+        for req in overrides["include"]:
+            assert req in gen_requirements_all.EXCLUDED_REQUIREMENTS_ALL