Migrate AutoGPT agent to poetry (#5219)

Inspired by #1102

* Migrate AutoGPT agent to poetry

  Co-authored-by: rickythefox <richard@ginzburg.se>

* Rewrite automatic dependency check (check_requirements.py) for poetry

* Sort dependencies

* Add instructions for poetry to README

.github/workflows/autogpt-ci.yml

@@ -57,13 +57,13 @@ jobs:
       - name: Set up Python dependency cache
         uses: actions/cache@v3
         with:
-          path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}-${{ steps.get_date.outputs.date }}
+          path: ~/.cache/pypoetry
+          key: ${{ runner.os }}-poetry-${{ hashFiles('autogpts/autogpt/pyproject.toml') }}-${{ steps.get_date.outputs.date }}
 
-      - name: Install dependencies
+      - name: Install Python dependencies
         run: |
-          python -m pip install --upgrade pip
-          pip install -r requirements.txt
+          curl -sSL https://install.python-poetry.org | python3 -
+          poetry install
 
       - name: Lint with flake8
         run: flake8
@@ -156,20 +156,21 @@ jobs:
       - name: Set up Python dependency cache
         uses: actions/cache@v3
         with:
-          path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}-${{ steps.get_date.outputs.date }}
+          path: ~/.cache/pypoetry
+          key: ${{ runner.os }}-poetry-${{ hashFiles('autogpts/autogpt/pyproject.toml') }}-${{ steps.get_date.outputs.date }}
 
       - name: Install Python dependencies
         run: |
-          python -m pip install --upgrade pip
-          pip install -r requirements.txt
+          curl -sSL https://install.python-poetry.org | python3 -
+          poetry install
 
       - name: Run pytest with coverage
         run: |
-          pytest -vv --cov=autogpt --cov-branch --cov-report term-missing --cov-report xml \
+          poetry run pytest -vv \
+            --cov=autogpt --cov-branch --cov-report term-missing --cov-report xml \
             --numprocesses=logical --durations=10 \
             tests/unit tests/integration tests/challenges
-          python tests/challenges/utils/build_current_score.py
+          poetry run python tests/challenges/utils/build_current_score.py
         env:
           CI: true
           PROXY: ${{ github.event_name == 'pull_request_target' && secrets.PROXY || '' }}

.github/workflows/autogpt-docker-ci.yml

@@ -117,7 +117,7 @@ jobs:
         run: |
           set +e
           test_output=$(
-            docker run --env CI --env OPENAI_API_KEY --entrypoint python ${{ env.IMAGE_NAME }} -m \
+            docker run --env CI --env OPENAI_API_KEY ${{ env.IMAGE_NAME }} run \
             pytest -v --cov=autogpt --cov-branch --cov-report term-missing \
               --numprocesses=4 --durations=10 \
               tests/unit tests/integration 2>&1

.github/workflows/benchmark-ci.yml

@@ -162,8 +162,8 @@ jobs:
           elif [ "$AGENT_NAME" == "Auto-GPT" ]; then
             python -m venv venv
             source venv/bin/activate
-            pip install -r requirements.txt
-            pip uninstall agbenchmark -y
+            curl -sSL https://install.python-poetry.org | python3 -
+            poetry install --without benchmark
           elif [ "$AGENT_NAME" == "mini-agi" ]; then
             python -m venv venv
             source venv/bin/activate

.github/workflows/benchmarks.yml

@@ -39,13 +39,13 @@ jobs:
       - name: Set up Python dependency cache
         uses: actions/cache@v3
         with:
-          path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}-${{ steps.get_date.outputs.date }}
+          path: ~/.cache/pypoetry
+          key: ${{ runner.os }}-poetry-${{ hashFiles('autogpts/autogpt/pyproject.toml') }}-${{ steps.get_date.outputs.date }}
 
       - name: Install Python dependencies
         run: |
-          python -m pip install --upgrade pip
-          pip install -r requirements.txt
+          curl -sSL https://install.python-poetry.org | python3 -
+          poetry install
 
       - name: Run pytest with coverage
         run: |

autogpts/autogpt/.devcontainer/devcontainer.json

@@ -46,11 +46,11 @@
   // "forwardPorts": [],
 
   // Use 'postCreateCommand' to run commands after the container is created.
-  // "postCreateCommand": "pip3 install --user -r requirements.txt",
+  // "postCreateCommand": "poetry install",
 
   // Set `remoteUser` to `root` to connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root.
   "remoteUser": "vscode",
 
   // Add the freshly containerized repo to the list of safe repositories
-  "postCreateCommand": "git config --global --add safe.directory /workspace/Auto-GPT && pip3 install --user -r requirements.txt"
-}
+  "postCreateCommand": "git config --global --add safe.directory /workspace/Auto-GPT && poetry install"
+}

autogpts/autogpt/.dockerignore

@@ -9,6 +9,4 @@ logs/*
 agbenchmark_config/logs/*
 agbenchmark_config/reports/*
 
-*.md
 *.png
-!BULLETIN.md

autogpts/autogpt/Dockerfile

@@ -17,30 +17,37 @@ RUN apt-get update && apt-get install -y \
 # Set environment variables
 ENV PIP_NO_CACHE_DIR=yes \
     PYTHONUNBUFFERED=1 \
-    PYTHONDONTWRITEBYTECODE=1
+    PYTHONDONTWRITEBYTECODE=1 \
+    POETRY_HOME="/opt/poetry" \
+    POETRY_VIRTUALENVS_IN_PROJECT=0 \
+    POETRY_NO_INTERACTION=1
 
-# Install the required python packages globally
-ENV PATH="$PATH:/root/.local/bin"
-COPY requirements.txt .
+# Install and configure Poetry
+RUN curl -sSL https://install.python-poetry.org | python3 -
+ENV PATH="$POETRY_HOME/bin:$PATH"
+RUN poetry config installer.max-workers 10
+
+WORKDIR /app
+COPY pyproject.toml poetry.lock ./
 
 # Set the entrypoint
-ENTRYPOINT ["python", "-m", "autogpt", "--install-plugin-deps"]
+ENTRYPOINT ["poetry"]
+CMD ["run", "autogpt", "--install-plugin-deps"]
 
 # dev build -> include everything
 FROM autogpt-base as autogpt-dev
-RUN pip install --no-cache-dir -r requirements.txt
-WORKDIR /app
+RUN poetry install --no-root
 ONBUILD COPY . ./
 
 # release build -> include bare minimum
 FROM autogpt-base as autogpt-release
-RUN sed -i '/Items below this point will not be included in the Docker Image/,$d' requirements.txt && \
-	pip install --no-cache-dir -r requirements.txt
-WORKDIR /app
+RUN poetry install --no-root --without dev,benchmark
 ONBUILD COPY autogpt/ ./autogpt
 ONBUILD COPY scripts/ ./scripts
 ONBUILD COPY plugins/ ./plugins
 ONBUILD COPY prompt_settings.yaml ./prompt_settings.yaml
+ONBUILD COPY README.md ./README.md
 ONBUILD RUN mkdir ./data
 
-FROM autogpt-${BUILD_TYPE} AS auto-gpt
+FROM autogpt-${BUILD_TYPE} AS autogpt
+RUN poetry install --only-root

autogpts/autogpt/README.md

@@ -60,6 +60,11 @@ Please see the [documentation][docs] for full setup instructions and configurati
 [docs/usage]: https://docs.agpt.co/usage/
 [docs/plugins]: https://docs.agpt.co/plugins/
 
+## 🏗️ Setting up for development
+1. Make sure `poetry` is installed: `python3 -m pip install poetry`
+2. Install all dependencies: `poetry install`
+
+
 <h2 align="center"> 💖 Help Fund Auto-GPT's Development 💖</h2>
 <p align="center">
 If you can spare a coffee, you can help to cover the costs of developing Auto-GPT and help to push the boundaries of fully autonomous AI!

autogpts/autogpt/pyproject.toml

@@ -1,25 +1,103 @@
-[build-system]
-requires = ["hatchling"]
-build-backend = "hatchling.build"
-
-[project]
+[tool.poetry]
 name = "agpt"
 version = "0.4.7"
 authors = [
-  { name="Torantulino", email="support@agpt.co" },
+  "Significant Gravitas <support@agpt.co>",
 ]
 readme = "README.md"
-requires-python = ">=3.10"
+description = "An open-source attempt to make GPT-4 autonomous"
+homepage = "https://github.com/Significant-Gravitas/Auto-GPT/tree/master/autogpts/autogpt"
 classifiers = [
     "Programming Language :: Python :: 3",
     "License :: OSI Approved :: MIT License",
     "Operating System :: OS Independent",
 ]
-description = "An open-source attempt to make GPT-4 autonomous"
+packages = [{ include = "autogpt" }]
+
+
+[tool.poetry.scripts]
+autogpt = "autogpt.app.cli:main"
+
+
+[tool.poetry.dependencies]
+python = "^3.10"
+agent-protocol = "^0.2.3"
+beautifulsoup4 = "^4.12.2"
+charset-normalizer = "^3.1.0"
+click = "*"
+colorama = "^0.4.6"
+distro = "^1.8.0"
+docker = "*"
+duckduckgo-search = "^3.0.2"
+en-core-web-sm = {url = "https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.5.0/en_core_web_sm-3.5.0-py3-none-any.whl"}
+ftfy = "^6.1.1"
+google-api-python-client = "*"
+gTTS = "^2.3.1"
+inflection = "*"
+jsonschema = "*"
+markdown = "*"
+openai = "^0.27.10"
+orjson = "^3.8.10"
+Pillow = "*"
+pinecone-client = "^2.2.1"
+playsound = "~1.2.2"
+prompt_toolkit = "^3.0.38"
+pydantic = "*"
+pylatexenc = "*"
+PyPDF2 = "*"
+python-docx = "*"
+python-dotenv = "^1.0.0"
+pyyaml = "^6.0"
+readability-lxml = "^0.8.1"
+redis = "*"
+requests = "*"
+selenium = "^4.11.2"
+spacy = "^3.0.0"
+tiktoken = "^0.3.3"
+webdriver-manager = "*"
+
+# web server
+fastapi = "*"
+uvicorn = "*"
+
+# OpenAI and Generic plugins import
+openapi-python-client = "^0.13.4"
+
+[tool.poetry.group.dev.dependencies]
+auto-gpt-plugin-template = {git = "https://github.com/Significant-Gravitas/Auto-GPT-Plugin-Template", rev = "0.1.0"}
+black = "*"
+coverage = "*"
+flake8 = "*"
+gitpython = "^3.1.32"
+isort = "*"
+mypy = "*"
+numpy = "*"
+pre-commit = "*"
+types-beautifulsoup4 = "*"
+types-colorama = "*"
+types-Markdown = "*"
+types-Pillow = "*"
+
+# Testing dependencies
+asynctest = "*"
+pytest = "*"
+pytest-asyncio = "*"
+pytest-benchmark = "*"
+pytest-cov = "*"
+pytest-integration = "*"
+pytest-mock = "*"
+pytest-recording = "*"
+pytest-xdist = "*"
+vcrpy = {git = "https://github.com/Significant-Gravitas/vcrpy.git", rev = "master"}
+
+[tool.poetry.group.benchmark.dependencies]
+agbenchmark = "0.0.9"
+
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"
 
-[project.urls]
-"Homepage" = "https://github.com/Significant-Gravitas/Auto-GPT"
-"Bug Tracker" = "https://github.com/Significant-Gravitas/Auto-GPT"
 
 [tool.black]
 line-length = 88
@@ -56,6 +134,7 @@ skip = '''
 
 '''
 
+
 [tool.pytest.ini_options]
 markers = [
     "requires_openai_api_key",

autogpts/autogpt/requirements.txt

@@ -1,71 +0,0 @@
-beautifulsoup4>=4.12.2
-colorama==0.4.6
-distro==1.8.0
-openai==0.27.8
-playsound==1.2.2
-python-dotenv==1.0.0
-pyyaml==6.0
-PyPDF2
-python-docx
-markdown
-pylatexenc
-readability-lxml==0.8.1
-requests
-tiktoken==0.3.3
-gTTS==2.3.1
-docker
-duckduckgo-search==3.0.2
-google-api-python-client #(https://developers.google.com/custom-search/v1/overview)
-pinecone-client==2.2.1
-redis
-orjson==3.8.10
-ftfy>=6.1.1
-Pillow
-selenium>=4.11.2
-webdriver-manager
-jsonschema
-click
-charset-normalizer>=3.1.0
-spacy>=3.0.0,<4.0.0
-en-core-web-sm @ https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.5.0/en_core_web_sm-3.5.0-py3-none-any.whl
-prompt_toolkit>=3.0.38
-pydantic
-inflection
-agbenchmark>=0.0.9
-agent-protocol>=0.1.1
-
-# web server
-fastapi
-uvicorn
-
-##Dev
-coverage
-flake8
-numpy
-pre-commit
-black
-isort
-gitpython>=3.1.32
-auto-gpt-plugin-template @ git+https://github.com/Significant-Gravitas/Auto-GPT-Plugin-Template@0.1.0
-mypy
-types-Markdown
-types-beautifulsoup4
-types-colorama
-types-Pillow
-
-# OpenAI and Generic plugins import
-openapi-python-client==0.13.4
-
-# Items below this point will not be included in the Docker Image
-
-# Testing dependencies
-pytest
-asynctest
-pytest-asyncio
-pytest-benchmark
-pytest-cov
-pytest-integration
-pytest-mock
-vcrpy @ git+https://github.com/Significant-Gravitas/vcrpy.git@master
-pytest-recording
-pytest-xdist

autogpts/autogpt/run.bat

@@ -15,10 +15,13 @@ pause
 exit /B 1
 
 :Found
-%PYTHON_CMD% scripts/check_requirements.py requirements.txt
+%PYTHON_CMD% scripts/check_requirements.py
 if errorlevel 1 (
-    echo Installing missing packages...
-    %PYTHON_CMD% -m pip install -r requirements.txt
+    echo
+    %PYTHON_CMD% -m poetry install --without dev
+    echo
+    echo Finished installing packages! Starting AutoGPT...
+    echo
 )
 %PYTHON_CMD% -m autogpt %*
-pause
+pause

autogpts/autogpt/run.sh

@@ -16,14 +16,16 @@ function find_python_command() {
 PYTHON_CMD=$(find_python_command)
 
 if $PYTHON_CMD -c "import sys; sys.exit(sys.version_info < (3, 10))"; then
-    $PYTHON_CMD scripts/check_requirements.py requirements.txt
+    $PYTHON_CMD scripts/check_requirements.py
     if [ $? -eq 1 ]
     then
-        echo Installing missing packages...
-        $PYTHON_CMD -m pip install -r requirements.txt
+        echo
+        $PYTHON_CMD -m poetry install --without dev
+        echo
+        echo "Finished installing packages! Starting AutoGPT..."
+        echo
     fi
     $PYTHON_CMD -m autogpt "$@"
-    read -p "Press any key to continue..."
 else
     echo "Python 3.10 or higher is required to run Auto GPT."
 fi

autogpts/autogpt/scripts/check_requirements.py

@@ -1,36 +1,37 @@
-import re
+import contextlib
+import os
 import sys
+from importlib.metadata import version
 
-import pkg_resources
+try:
+    import poetry.factory
+except ModuleNotFoundError:
+    os.system(f"{sys.executable} -m pip install 'poetry>=1.6.1,<2.0.0'")
+
+from poetry.factory import Factory
+from poetry.core.constraints.version.version import Version
 
 
 def main():
-    requirements_file = sys.argv[1]
-    with open(requirements_file, "r") as f:
-        required_packages = [
-            line.strip().split("#")[0].strip() for line in f.readlines()
-        ]
-
-    installed_packages = {pkg.key: pkg.version for pkg in pkg_resources.working_set}
+    poetry_project = Factory().create_poetry()
+    # repository = poetry_project.locker.locked_repository()
+    # dependencies = repository.packages
+    dependency_group = poetry_project.package.dependency_group("main")
 
     missing_packages = []
-    for required_package in required_packages:
-        if not required_package:  # Skip empty lines
-            continue
-        pkg = pkg_resources.Requirement.parse(required_package)
-        if (
-            pkg.key not in installed_packages
-            or pkg_resources.parse_version(installed_packages[pkg.key])
-            not in pkg.specifier
-        ):
-            missing_packages.append(str(pkg))
+    for dep in dependency_group.dependencies:
+        # Try to verify that the installed version is suitable
+        with contextlib.suppress(ModuleNotFoundError):
+            installed_version = version(dep.name)   # if this fails -> not installed
+            if dep.constraint.allows(Version.parse(installed_version)):
+                continue
+        # If the above verification fails, mark the package as missing
+        missing_packages.append(str(dep))
 
     if missing_packages:
         print("Missing packages:")
         print(", ".join(missing_packages))
         sys.exit(1)
-    else:
-        print("All packages are installed.")
 
 
 if __name__ == "__main__":

docs/content/configuration/memory.md

@@ -153,14 +153,14 @@ data objects. To set up a Weaviate database, check out their [Quickstart Tutoria
 
 Although still experimental, [Embedded Weaviate](https://weaviate.io/developers/weaviate/installation/embedded)
 is supported which allows the Auto-GPT process itself to start a Weaviate instance.
-To enable it, set `USE_WEAVIATE_EMBEDDED` to `True` and make sure you `pip install "weaviate-client>=3.15.4"`.
+To enable it, set `USE_WEAVIATE_EMBEDDED` to `True` and make sure you `poetry add weaviate-client@^3.15.4`.
 
 #### Install the Weaviate client
 
 Install the Weaviate client before usage.
 
 ```shell
-$ pip install weaviate-client
+$ poetry add weaviate-client
 ```
 
 #### Setting up environment variables

docs/content/setup.md

@@ -230,8 +230,8 @@ docker run -it --env-file=.env -v $PWD:/app --rm auto-gpt --gpt3only --continuou
 Create a virtual environment to run in.
 
 ```shell
-python -m venv venvAutoGPT
-source venvAutoGPT/bin/activate
+python -m venv .venv
+source .venv/bin/activate
 pip3 install --upgrade pip
 ```