From 56e2cfbf45fe7eb774ea0faf7792505b8484d348 Mon Sep 17 00:00:00 2001 From: Travis Dent Date: Thu, 30 Jan 2025 18:56:23 -0800 Subject: [PATCH 01/11] Create separate test environments for each framework. --- pyproject.toml | 4 +--- tests/test_cli_templates.py | 3 ++- tests/test_cli_tools.py | 3 ++- tests/test_frameworks.py | 4 ++-- tests/test_frameworks_langgraph.py | 3 ++- tests/test_generation_agent.py | 5 ++--- tests/test_generation_files.py | 3 ++- tests/test_generation_tasks.py | 5 ++--- tests/test_generation_tool.py | 4 ++-- tests/test_inputs_config.py | 3 ++- tests/test_log.py | 6 +++--- tests/test_project_run.py | 5 ++--- tests/test_tasks_config.py | 3 ++- tests/test_templates_config.py | 3 ++- tox.ini | 25 ++++++++++++++++++++----- 15 files changed, 48 insertions(+), 31 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index eb708eaf..acade32f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,14 +46,12 @@ test = [ crewai = [ "crewai==0.100.0", "crewai-tools==0.33.0", + "shapely==2.0.6" # crewai-tools needs this, but 2.0.7 is broken ] langgraph = [ "langgraph>=0.2.61", "langchain>=0.3.14", ] -all = [ - "agentstack[dev,test,crewai,langgraph]", -] [tool.setuptools.package-data] agentstack = ["templates/**/*"] diff --git a/tests/test_cli_templates.py b/tests/test_cli_templates.py index f9bba704..86a0220b 100644 --- a/tests/test_cli_templates.py +++ b/tests/test_cli_templates.py @@ -11,7 +11,8 @@ class CLITemplatesTest(unittest.TestCase): def setUp(self): - self.project_dir = Path(BASE_PATH / 'tmp/cli_templates') + self.framework = os.getenv('TEST_FRAMEWORK') + self.project_dir = BASE_PATH / 'tmp' / self.framework / 'cli_templates' os.makedirs(self.project_dir, exist_ok=True) os.chdir(self.project_dir) diff --git a/tests/test_cli_tools.py b/tests/test_cli_tools.py index 05c4ad67..5237aeff 100644 --- a/tests/test_cli_tools.py +++ b/tests/test_cli_tools.py @@ -17,7 +17,8 @@ # TODO parameterized framework class CLIToolsTest(unittest.TestCase): def setUp(self): - self.project_dir = Path(BASE_PATH / 'tmp/cli_tools') + self.framework = os.getenv('TEST_FRAMEWORK') + self.project_dir = BASE_PATH / 'tmp' / self.framework / 'cli_tools' os.makedirs(self.project_dir, exist_ok=True) os.chdir(self.project_dir) diff --git a/tests/test_frameworks.py b/tests/test_frameworks.py index 2952d916..66e8f188 100644 --- a/tests/test_frameworks.py +++ b/tests/test_frameworks.py @@ -3,7 +3,7 @@ from pathlib import Path import shutil import unittest -from parameterized import parameterized, parameterized_class +from parameterized import parameterized from agentstack.conf import ConfigFile, set_path from agentstack.exceptions import ValidationError @@ -16,9 +16,9 @@ BASE_PATH = Path(__file__).parent -@parameterized_class([{"framework": framework} for framework in frameworks.SUPPORTED_FRAMEWORKS]) class TestFrameworks(unittest.TestCase): def setUp(self): + self.framework = os.getenv('TEST_FRAMEWORK') self.project_dir = BASE_PATH / 'tmp' / self.framework / 'test_frameworks' os.makedirs(self.project_dir) diff --git a/tests/test_frameworks_langgraph.py b/tests/test_frameworks_langgraph.py index 68d6c83d..44265260 100644 --- a/tests/test_frameworks_langgraph.py +++ b/tests/test_frameworks_langgraph.py @@ -18,7 +18,8 @@ class FrameworksLanggraphTest(unittest.TestCase): def setUp(self): - self.project_dir = BASE_PATH / 'tmp/frameworks/langgraph' + self.framework = os.getenv('TEST_FRAMEWORK') + self.project_dir = BASE_PATH / 'tmp' / self.framework / 'langgraph' conf.set_path(self.project_dir) os.makedirs(self.project_dir / 'src/config') diff --git a/tests/test_generation_agent.py b/tests/test_generation_agent.py index 16dc08aa..40a75d94 100644 --- a/tests/test_generation_agent.py +++ b/tests/test_generation_agent.py @@ -2,7 +2,6 @@ from pathlib import Path import shutil import unittest -from parameterized import parameterized_class import ast from agentstack.conf import ConfigFile, set_path @@ -15,10 +14,10 @@ BASE_PATH = Path(__file__).parent -@parameterized_class([{"framework": framework} for framework in frameworks.SUPPORTED_FRAMEWORKS]) class TestGenerationAgent(unittest.TestCase): def setUp(self): - self.project_dir = BASE_PATH / 'tmp' / 'agent_generation' + self.framework = os.getenv('TEST_FRAMEWORK') + self.project_dir = BASE_PATH / 'tmp' / self.framework / 'agent_generation' os.makedirs(self.project_dir) os.makedirs(self.project_dir / 'src') diff --git a/tests/test_generation_files.py b/tests/test_generation_files.py index 8ed151f7..2f1ba4f3 100644 --- a/tests/test_generation_files.py +++ b/tests/test_generation_files.py @@ -18,7 +18,8 @@ # TODO copy files to working directory class GenerationFilesTest(unittest.TestCase): def setUp(self): - self.project_dir = BASE_PATH / "tmp" / "generation_files" + self.framework = os.getenv('TEST_FRAMEWORK') + self.project_dir = BASE_PATH / 'tmp' / self.framework / 'generation_files' os.makedirs(self.project_dir) shutil.copy(BASE_PATH / "fixtures/agentstack.json", self.project_dir / "agentstack.json") diff --git a/tests/test_generation_tasks.py b/tests/test_generation_tasks.py index e2356194..51b010d2 100644 --- a/tests/test_generation_tasks.py +++ b/tests/test_generation_tasks.py @@ -2,7 +2,6 @@ from pathlib import Path import shutil import unittest -from parameterized import parameterized_class import ast from agentstack.conf import ConfigFile, set_path @@ -16,10 +15,10 @@ BASE_PATH = Path(__file__).parent -@parameterized_class([{"framework": framework} for framework in frameworks.SUPPORTED_FRAMEWORKS]) class TestGenerationAgent(unittest.TestCase): def setUp(self): - self.project_dir = BASE_PATH / 'tmp' / 'agent_generation' + self.framework = os.getenv('TEST_FRAMEWORK') + self.project_dir = BASE_PATH / 'tmp' / self.framework / 'agent_generation' os.makedirs(self.project_dir) os.makedirs(self.project_dir / 'src') diff --git a/tests/test_generation_tool.py b/tests/test_generation_tool.py index 23247d09..9b3b9a8c 100644 --- a/tests/test_generation_tool.py +++ b/tests/test_generation_tool.py @@ -15,10 +15,10 @@ # TODO parameterize all tools -@parameterized_class([{"framework": framework} for framework in frameworks.SUPPORTED_FRAMEWORKS]) class TestGenerationTool(unittest.TestCase): def setUp(self): - self.project_dir = BASE_PATH / 'tmp' / 'tool_generation' + self.framework = os.getenv('TEST_FRAMEWORK') + self.project_dir = BASE_PATH / 'tmp' / self.framework / 'tool_generation' os.makedirs(self.project_dir) os.makedirs(self.project_dir / 'src') diff --git a/tests/test_inputs_config.py b/tests/test_inputs_config.py index 23ea37d5..d259da20 100644 --- a/tests/test_inputs_config.py +++ b/tests/test_inputs_config.py @@ -11,7 +11,8 @@ class InputsConfigTest(unittest.TestCase): def setUp(self): - self.project_dir = BASE_PATH / "tmp/inputs_config" + self.framework = os.getenv('TEST_FRAMEWORK') + self.project_dir = BASE_PATH / 'tmp' / self.framework / 'inputs_config' os.makedirs(self.project_dir) os.makedirs(self.project_dir / "src/config") diff --git a/tests/test_log.py b/tests/test_log.py index d058d56d..8db42bbb 100644 --- a/tests/test_log.py +++ b/tests/test_log.py @@ -1,5 +1,5 @@ import unittest -import sys +import os, sys import io import logging import shutil @@ -12,8 +12,8 @@ class TestLog(unittest.TestCase): def setUp(self): - # Create test directory if it doesn't exist - self.test_dir = BASE_PATH / 'tmp/test_log' + self.framework = os.getenv('TEST_FRAMEWORK') + self.test_dir = BASE_PATH / 'tmp' / self.framework / 'test_log' self.test_dir.mkdir(parents=True, exist_ok=True) # Set log file to test directory diff --git a/tests/test_project_run.py b/tests/test_project_run.py index cb30f133..758f57af 100644 --- a/tests/test_project_run.py +++ b/tests/test_project_run.py @@ -2,7 +2,6 @@ from pathlib import Path import shutil import unittest -from parameterized import parameterized_class from agentstack import conf from agentstack.conf import ConfigFile @@ -12,10 +11,10 @@ BASE_PATH = Path(__file__).parent -@parameterized_class([{"framework": framework} for framework in frameworks.SUPPORTED_FRAMEWORKS]) class ProjectRunTest(unittest.TestCase): def setUp(self): - self.project_dir = BASE_PATH / 'tmp/project_run' / self.framework + self.framework = os.getenv('TEST_FRAMEWORK') + self.project_dir = BASE_PATH / 'tmp' / self.framework / 'project_run' os.makedirs(self.project_dir) os.makedirs(self.project_dir / 'src') diff --git a/tests/test_tasks_config.py b/tests/test_tasks_config.py index 06959b9b..22631683 100644 --- a/tests/test_tasks_config.py +++ b/tests/test_tasks_config.py @@ -13,7 +13,8 @@ class AgentConfigTest(unittest.TestCase): def setUp(self): - self.project_dir = BASE_PATH / 'tmp/task_config' + self.framework = os.getenv('TEST_FRAMEWORK') + self.project_dir = BASE_PATH / 'tmp' / self.framework / 'task_config' os.makedirs(self.project_dir / 'src/config') conf.set_path(self.project_dir) diff --git a/tests/test_templates_config.py b/tests/test_templates_config.py index 045baded..bb0d715a 100644 --- a/tests/test_templates_config.py +++ b/tests/test_templates_config.py @@ -21,7 +21,8 @@ class TemplateConfigTest(unittest.TestCase): def setUp(self): - self.project_dir = BASE_PATH / 'tmp/template_config' + self.framework = os.getenv('TEST_FRAMEWORK') + self.project_dir = BASE_PATH / 'tmp' / self.framework / 'template_config' os.makedirs(self.project_dir, exist_ok=True) def tearDown(self): diff --git a/tox.ini b/tox.ini index aa870ebd..d4f997d6 100644 --- a/tox.ini +++ b/tox.ini @@ -3,11 +3,16 @@ # test suite on all supported python versions. To use it, "pip install tox" # and then run "tox" from this directory. +# run just 3.12 tests with `tox -epy312-crewai,py312-langgraph,report` +# TODO I have not found a way to alias this, yet. + [tox] -envlist = py310,py311,py312 +envlist = py{310,311,312}-{crewai,langgraph} [testenv] -extras = all # install dependencies for all frameworks, too +extras = + crewai: crewai # installs agentstack[crewai] + langgraph: langgraph deps = pytest parameterized @@ -15,16 +20,25 @@ deps = mypy: mypy commands = coverage run --source=. --omit="**/tmp/**,./tmp/**/*" -m pytest -v {posargs} - coverage report -m - coverage xml mypy: mypy agentstack setenv = AGENTSTACK_IS_TEST_ENV = 1 AGENTSTACK_TELEMETRY_OPT_OUT = 1 AGENTSTACK_UPDATE_DISABLE = 1 + crewai: TEST_FRAMEWORK = crewai + langgraph: TEST_FRAMEWORK = langgraph + +[testenv:report] +deps = coverage +skip_install = true +commands = + coverage combine + coverage report -m + coverage xml [coverage:run] branch = True +parallel = True source = . omit = **/tmp/** @@ -40,4 +54,5 @@ exclude_lines = pragma: no cover ^\s*\.\.\.\s*$ pass - if TYPE_CHECKING: \ No newline at end of file + if TYPE_CHECKING: + From 689559d1dd444f47b4bd9f68b9955b14c3511159 Mon Sep 17 00:00:00 2001 From: Travis Dent Date: Thu, 30 Jan 2025 20:16:14 -0800 Subject: [PATCH 02/11] Update github action. --- .github/workflows/python-testing.yml | 1 + tox.ini | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/.github/workflows/python-testing.yml b/.github/workflows/python-testing.yml index ab40e60a..5ce842fd 100644 --- a/.github/workflows/python-testing.yml +++ b/.github/workflows/python-testing.yml @@ -26,6 +26,7 @@ jobs: strategy: matrix: python-version: [3.10,3.11,3.12] + framework: [crewai, langgraph] steps: - uses: actions/checkout@v2 diff --git a/tox.ini b/tox.ini index d4f997d6..55dc26a7 100644 --- a/tox.ini +++ b/tox.ini @@ -3,9 +3,15 @@ # test suite on all supported python versions. To use it, "pip install tox" # and then run "tox" from this directory. +# Adding a New Framework +# modify `envlist`, add extras to install optional-dependencies, add a new +# `TEST_FRAMEWORK` section to `setenv` and update the github action. + # run just 3.12 tests with `tox -epy312-crewai,py312-langgraph,report` # TODO I have not found a way to alias this, yet. +# codecov is configured to run on all frameworks and then be combined at the end. + [tox] envlist = py{310,311,312}-{crewai,langgraph} From eef4fafb447d5cb58f0ecca370b5c7c3f49a9f84 Mon Sep 17 00:00:00 2001 From: Travis Dent Date: Thu, 30 Jan 2025 20:22:53 -0800 Subject: [PATCH 03/11] Ensure tests run for one specific python version. --- .github/workflows/python-testing.yml | 6 ++++-- tox.ini | 6 ++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-testing.yml b/.github/workflows/python-testing.yml index 5ce842fd..ea1a6bc3 100644 --- a/.github/workflows/python-testing.yml +++ b/.github/workflows/python-testing.yml @@ -33,8 +33,10 @@ jobs: - name: Set up Python uses: actions/setup-python@v2 with: - python-version: '3.11' # Use a default Python version for running tox + python-version: ${{ matrix.python-version }} - name: Install tox run: pip install tox + - name: Install tox-gh-actions + run: pip install tox-gh-actions - name: Run tests with tox - run: tox \ No newline at end of file + run: tox -e py${{ matrix.python-version }}-${{ matrix.framework }} \ No newline at end of file diff --git a/tox.ini b/tox.ini index 55dc26a7..ef98da19 100644 --- a/tox.ini +++ b/tox.ini @@ -15,6 +15,12 @@ [tox] envlist = py{310,311,312}-{crewai,langgraph} +[gh-actions] +python = + 3.10: py310 + 3.11: py311 + 3.12: py312 + [testenv] extras = crewai: crewai # installs agentstack[crewai] From 266ce0b7b877c830017793134518aea478823386 Mon Sep 17 00:00:00 2001 From: Travis Dent Date: Thu, 30 Jan 2025 20:24:42 -0800 Subject: [PATCH 04/11] Try running tox with 3.11 --- .github/workflows/python-testing.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-testing.yml b/.github/workflows/python-testing.yml index ea1a6bc3..f1fc797c 100644 --- a/.github/workflows/python-testing.yml +++ b/.github/workflows/python-testing.yml @@ -33,7 +33,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v2 with: - python-version: ${{ matrix.python-version }} + python-version: 3.11 # setup up with 3.11 - name: Install tox run: pip install tox - name: Install tox-gh-actions From 8fc3baeb42d6bd8aabf6766bf5811f157f1cd9c7 Mon Sep 17 00:00:00 2001 From: Travis Dent Date: Thu, 30 Jan 2025 20:29:07 -0800 Subject: [PATCH 05/11] Run both frameworks in one test. --- .github/workflows/python-testing.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/python-testing.yml b/.github/workflows/python-testing.yml index f1fc797c..4e6ce215 100644 --- a/.github/workflows/python-testing.yml +++ b/.github/workflows/python-testing.yml @@ -26,7 +26,6 @@ jobs: strategy: matrix: python-version: [3.10,3.11,3.12] - framework: [crewai, langgraph] steps: - uses: actions/checkout@v2 @@ -37,6 +36,6 @@ jobs: - name: Install tox run: pip install tox - name: Install tox-gh-actions - run: pip install tox-gh-actions + run: pip install tox-gh-actions # allows tox to get the current python version - name: Run tests with tox - run: tox -e py${{ matrix.python-version }}-${{ matrix.framework }} \ No newline at end of file + run: tox \ No newline at end of file From be74578fd5e046a792f223f8e0b3b0d188948b8d Mon Sep 17 00:00:00 2001 From: Travis Dent Date: Thu, 30 Jan 2025 20:58:23 -0800 Subject: [PATCH 06/11] testing workflows on main :/ --- .github/workflows/python-testing.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-testing.yml b/.github/workflows/python-testing.yml index 4e6ce215..cd3e9d75 100644 --- a/.github/workflows/python-testing.yml +++ b/.github/workflows/python-testing.yml @@ -31,8 +31,8 @@ jobs: - uses: actions/checkout@v2 - name: Set up Python uses: actions/setup-python@v2 - with: - python-version: 3.11 # setup up with 3.11 + # with: + # python-version: 3.11 # setup up with 3.11 - name: Install tox run: pip install tox - name: Install tox-gh-actions From 67800c40100c77286e3f406a18e71d5df36b9803 Mon Sep 17 00:00:00 2001 From: Travis Dent Date: Thu, 30 Jan 2025 21:01:04 -0800 Subject: [PATCH 07/11] try matrix version again --- .github/workflows/python-testing.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-testing.yml b/.github/workflows/python-testing.yml index cd3e9d75..41cee93e 100644 --- a/.github/workflows/python-testing.yml +++ b/.github/workflows/python-testing.yml @@ -31,8 +31,8 @@ jobs: - uses: actions/checkout@v2 - name: Set up Python uses: actions/setup-python@v2 - # with: - # python-version: 3.11 # setup up with 3.11 + with: + python-version: ${{ matrix.python-version }} - name: Install tox run: pip install tox - name: Install tox-gh-actions From 6f084003e1ab69f7aee3d03e6c5da2d47d3ff043 Mon Sep 17 00:00:00 2001 From: Travis Dent Date: Thu, 30 Jan 2025 21:04:02 -0800 Subject: [PATCH 08/11] I think v5 is supposed to have 3.10 --- .github/workflows/python-testing.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-testing.yml b/.github/workflows/python-testing.yml index 41cee93e..9d9e90f4 100644 --- a/.github/workflows/python-testing.yml +++ b/.github/workflows/python-testing.yml @@ -30,7 +30,7 @@ jobs: steps: - uses: actions/checkout@v2 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Install tox From 8865622f7e68ac700218ee4b28977cd41fe57369 Mon Sep 17 00:00:00 2001 From: Travis Dent Date: Thu, 30 Jan 2025 21:09:27 -0800 Subject: [PATCH 09/11] try versions as strings --- .github/workflows/python-testing.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-testing.yml b/.github/workflows/python-testing.yml index 9d9e90f4..10e2c5ba 100644 --- a/.github/workflows/python-testing.yml +++ b/.github/workflows/python-testing.yml @@ -25,7 +25,7 @@ jobs: strategy: matrix: - python-version: [3.10,3.11,3.12] + python-version: ["3.10","3.11","3.12"] steps: - uses: actions/checkout@v2 From 237a43c84f7639cb30b7f380a1e117469299d5b7 Mon Sep 17 00:00:00 2001 From: Travis Dent Date: Thu, 30 Jan 2025 21:14:07 -0800 Subject: [PATCH 10/11] got ittttttttt --- .github/workflows/python-testing.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-testing.yml b/.github/workflows/python-testing.yml index 10e2c5ba..0d94dfd1 100644 --- a/.github/workflows/python-testing.yml +++ b/.github/workflows/python-testing.yml @@ -25,7 +25,7 @@ jobs: strategy: matrix: - python-version: ["3.10","3.11","3.12"] + python-version: ["3.10","3.11","3.12"] # versions need to be strings steps: - uses: actions/checkout@v2 @@ -38,4 +38,4 @@ jobs: - name: Install tox-gh-actions run: pip install tox-gh-actions # allows tox to get the current python version - name: Run tests with tox - run: tox \ No newline at end of file + run: tox # we can't pass arguments to tox when using tox-gh-actions, so all frameworks happen in one run \ No newline at end of file From d006f1a5d180aef527cd2dc1db3c9959b28f707f Mon Sep 17 00:00:00 2001 From: Travis Dent Date: Thu, 30 Jan 2025 21:18:34 -0800 Subject: [PATCH 11/11] Update comments for future us --- tox.ini | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tox.ini b/tox.ini index ef98da19..2e39b7dd 100644 --- a/tox.ini +++ b/tox.ini @@ -1,21 +1,19 @@ -# tox (https://tox.readthedocs.io/) is a tool for running tests -# in multiple virtualenvs. This configuration file will run the -# test suite on all supported python versions. To use it, "pip install tox" -# and then run "tox" from this directory. - # Adding a New Framework -# modify `envlist`, add extras to install optional-dependencies, add a new -# `TEST_FRAMEWORK` section to `setenv` and update the github action. +# modify `envlist`, add extras to install optional-dependencies, and add a new +# `TEST_FRAMEWORK` section to `setenv`. +# Developers # run just 3.12 tests with `tox -epy312-crewai,py312-langgraph,report` # TODO I have not found a way to alias this, yet. +# Coverage # codecov is configured to run on all frameworks and then be combined at the end. [tox] envlist = py{310,311,312}-{crewai,langgraph} [gh-actions] +# converts python versions to tox envlist values for github actions python = 3.10: py310 3.11: py311 @@ -23,6 +21,7 @@ python = [testenv] extras = + # framework specific dependencies crewai: crewai # installs agentstack[crewai] langgraph: langgraph deps = @@ -37,6 +36,7 @@ setenv = AGENTSTACK_IS_TEST_ENV = 1 AGENTSTACK_TELEMETRY_OPT_OUT = 1 AGENTSTACK_UPDATE_DISABLE = 1 + # environment variables to determine active framework inside tests crewai: TEST_FRAMEWORK = crewai langgraph: TEST_FRAMEWORK = langgraph