134 lines
4.3 KiB
YAML
134 lines
4.3 KiB
YAML
name: Hackathon
|
|
|
|
on:
|
|
workflow_dispatch:
|
|
inputs:
|
|
agents:
|
|
description: "Agents to run (comma-separated)"
|
|
required: false
|
|
default: "autogpt" # Default agents if none are specified
|
|
|
|
jobs:
|
|
matrix-setup:
|
|
runs-on: ubuntu-latest
|
|
# Service containers to run with `matrix-setup`
|
|
services:
|
|
# Label used to access the service container
|
|
postgres:
|
|
# Docker Hub image
|
|
image: postgres
|
|
# Provide the password for postgres
|
|
env:
|
|
POSTGRES_PASSWORD: postgres
|
|
# Set health checks to wait until postgres has started
|
|
options: >-
|
|
--health-cmd pg_isready
|
|
--health-interval 10s
|
|
--health-timeout 5s
|
|
--health-retries 5
|
|
ports:
|
|
# Maps tcp port 5432 on service container to the host
|
|
- 5432:5432
|
|
outputs:
|
|
matrix: ${{ steps.set-matrix.outputs.matrix }}
|
|
env-name: ${{ steps.set-matrix.outputs.env-name }}
|
|
steps:
|
|
- id: set-matrix
|
|
run: |
|
|
if [ "${{ github.event_name }}" == "schedule" ]; then
|
|
echo "::set-output name=env-name::production"
|
|
echo "::set-output name=matrix::[ 'irrelevant']"
|
|
elif [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
|
|
IFS=',' read -ra matrix_array <<< "${{ github.event.inputs.agents }}"
|
|
matrix_string="[ \"$(echo "${matrix_array[@]}" | sed 's/ /", "/g')\" ]"
|
|
echo "::set-output name=env-name::production"
|
|
echo "::set-output name=matrix::$matrix_string"
|
|
else
|
|
echo "::set-output name=env-name::testing"
|
|
echo "::set-output name=matrix::[ 'irrelevant' ]"
|
|
fi
|
|
|
|
tests:
|
|
environment:
|
|
name: "${{ needs.matrix-setup.outputs.env-name }}"
|
|
needs: matrix-setup
|
|
env:
|
|
min-python-version: "3.10"
|
|
name: "${{ matrix.agent-name }}"
|
|
runs-on: ubuntu-latest
|
|
services:
|
|
# Label used to access the service container
|
|
postgres:
|
|
# Docker Hub image
|
|
image: postgres
|
|
# Provide the password for postgres
|
|
env:
|
|
POSTGRES_PASSWORD: postgres
|
|
# Set health checks to wait until postgres has started
|
|
options: >-
|
|
--health-cmd pg_isready
|
|
--health-interval 10s
|
|
--health-timeout 5s
|
|
--health-retries 5
|
|
ports:
|
|
# Maps tcp port 5432 on service container to the host
|
|
- 5432:5432
|
|
timeout-minutes: 50
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
agent-name: ${{fromJson(needs.matrix-setup.outputs.matrix)}}
|
|
steps:
|
|
- name: Print Environment Name
|
|
run: |
|
|
echo "Matrix Setup Environment Name: ${{ needs.matrix-setup.outputs.env-name }}"
|
|
|
|
- name: Check Docker Container
|
|
id: check
|
|
run: docker ps
|
|
|
|
- name: Checkout repository
|
|
uses: actions/checkout@v4
|
|
with:
|
|
fetch-depth: 0
|
|
submodules: true
|
|
|
|
- name: Set up Python ${{ env.min-python-version }}
|
|
uses: actions/setup-python@v5
|
|
with:
|
|
python-version: ${{ env.min-python-version }}
|
|
|
|
- id: get_date
|
|
name: Get date
|
|
run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT
|
|
|
|
- name: Install Poetry
|
|
run: |
|
|
curl -sSL https://install.python-poetry.org | python -
|
|
|
|
- name: Install Node.js
|
|
uses: actions/setup-node@v4
|
|
with:
|
|
node-version: v18.15
|
|
|
|
- name: Run benchmark
|
|
run: |
|
|
link=$(jq -r '.["github_repo_url"]' arena/$AGENT_NAME.json)
|
|
branch=$(jq -r '.["branch_to_benchmark"]' arena/$AGENT_NAME.json)
|
|
git clone "$link" -b "$branch" "$AGENT_NAME"
|
|
cd $AGENT_NAME
|
|
cp ./$AGENT_NAME/.env.example ./$AGENT_NAME/.env || echo "file not found"
|
|
./run agent start $AGENT_NAME
|
|
cd ../benchmark
|
|
poetry install
|
|
poetry run agbenchmark --no-dep
|
|
env:
|
|
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
|
SERP_API_KEY: ${{ secrets.SERP_API_KEY }}
|
|
SERPAPI_API_KEY: ${{ secrets.SERP_API_KEY }}
|
|
WEAVIATE_API_KEY: ${{ secrets.WEAVIATE_API_KEY }}
|
|
WEAVIATE_URL: ${{ secrets.WEAVIATE_URL }}
|
|
GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
|
|
GOOGLE_CUSTOM_SEARCH_ENGINE_ID: ${{ secrets.GOOGLE_CUSTOM_SEARCH_ENGINE_ID }}
|
|
AGENT_NAME: ${{ matrix.agent-name }}
|