From 88744f09b94853abfa1811a602a05c2f812f2966 Mon Sep 17 00:00:00 2001 From: Travis Dent Date: Tue, 7 Jan 2025 10:24:28 -0800 Subject: [PATCH 1/5] Full coverage for tasks.py --- tests/test_tasks_config.py | 56 +++++++++++++++++++++++++++++++++++++- 1 file changed, 55 insertions(+), 1 deletion(-) diff --git a/tests/test_tasks_config.py b/tests/test_tasks_config.py index 394b8ced..06959b9b 100644 --- a/tests/test_tasks_config.py +++ b/tests/test_tasks_config.py @@ -5,7 +5,8 @@ import importlib.resources from pathlib import Path from agentstack import conf -from agentstack.tasks import TaskConfig, TASKS_FILENAME +from agentstack.tasks import TaskConfig, TASKS_FILENAME, get_all_task_names, get_all_tasks +from agentstack.exceptions import ValidationError BASE_PATH = Path(__file__).parent @@ -76,3 +77,56 @@ def test_write_none_values(self): agent: > """ ) + + def test_yaml_error(self): + # Create an invalid YAML file + with open(self.project_dir / TASKS_FILENAME, 'w') as f: + f.write(""" +task_name: + description: "This is a valid line" + invalid_yaml: "This line is missing a colon" + nested_key: "This will cause a YAML error" +""") + + # Attempt to load the config, which should raise a ValidationError + with self.assertRaises(ValidationError) as context: + TaskConfig("task_name") + + def test_pydantic_validation_error(self): + # Create a YAML file with an invalid field type + with open(self.project_dir / TASKS_FILENAME, 'w') as f: + f.write(""" +task_name: + description: "This is a valid description" + expected_output: "This is a valid expected output" + agent: 123 # This should be a string, not an integer +""") + + # Attempt to load the config, which should raise a ValidationError + with self.assertRaises(ValidationError) as context: + TaskConfig("task_name") + + def test_get_all_task_names(self): + shutil.copy(BASE_PATH / "fixtures/tasks_max.yaml", self.project_dir / TASKS_FILENAME) + + task_names = get_all_task_names() + self.assertEqual(set(task_names), {"task_name", "task_name_two"}) + self.assertEqual(task_names, ["task_name", "task_name_two"]) + + def test_get_all_task_names_missing_file(self): + if os.path.exists(self.project_dir / TASKS_FILENAME): + os.remove(self.project_dir / TASKS_FILENAME) + non_existent_file_task_names = get_all_task_names() + self.assertEqual(non_existent_file_task_names, []) + + def test_get_all_task_names_empty_file(self): + with open(self.project_dir / TASKS_FILENAME, 'w') as f: + f.write("") + + empty_task_names = get_all_task_names() + self.assertEqual(empty_task_names, []) + + def test_get_all_tasks(self): + shutil.copy(BASE_PATH / "fixtures/tasks_max.yaml", self.project_dir / TASKS_FILENAME) + for task in get_all_tasks(): + self.assertIsInstance(task, TaskConfig) From 0fcb6a1399c106e2c94bc2aeb02aabe150858c00 Mon Sep 17 00:00:00 2001 From: Travis Dent Date: Tue, 7 Jan 2025 10:40:30 -0800 Subject: [PATCH 2/5] Full coverage for agents.py --- tests/test_agents_config.py | 58 ++++++++++++++++++++++++++++++++++++- 1 file changed, 57 insertions(+), 1 deletion(-) diff --git a/tests/test_agents_config.py b/tests/test_agents_config.py index 58811a95..fb9acea8 100644 --- a/tests/test_agents_config.py +++ b/tests/test_agents_config.py @@ -5,7 +5,8 @@ import importlib.resources from pathlib import Path from agentstack import conf -from agentstack.agents import AgentConfig, AGENTS_FILENAME +from agentstack.agents import AgentConfig, AGENTS_FILENAME, get_all_agent_names, get_all_agents +from agentstack.exceptions import ValidationError BASE_PATH = Path(__file__).parent @@ -83,3 +84,58 @@ def test_write_none_values(self): llm: """ ) + + def test_yaml_error(self): + # Create an invalid YAML file + with open(self.project_dir / AGENTS_FILENAME, 'w') as f: + f.write(""" +agent_name: + role: "This is a valid line" + invalid_yaml: "This line is missing a colon" + nested_key: "This will cause a YAML error" +""") + + # Attempt to load the config, which should raise a ValidationError + with self.assertRaises(ValidationError) as context: + AgentConfig("agent_name") + + def test_pydantic_validation_error(self): + # Create a YAML file with an invalid field type + with open(self.project_dir / AGENTS_FILENAME, 'w') as f: + f.write(""" +agent_name: + role: "This is a valid role" + goal: "This is a valid goal" + backstory: "This is a valid backstory" + llm: 123 # This should be a string, not an integer +""") + + # Attempt to load the config, which should raise a ValidationError + with self.assertRaises(ValidationError) as context: + AgentConfig("agent_name") + + def test_get_all_agent_names(self): + shutil.copy(BASE_PATH / "fixtures/agents_max.yaml", self.project_dir / AGENTS_FILENAME) + + agent_names = get_all_agent_names() + self.assertEqual(set(agent_names), {"agent_name", "second_agent_name"}) + self.assertEqual(agent_names, ["agent_name", "second_agent_name"]) + + def test_get_all_agent_names_missing_file(self): + if os.path.exists(self.project_dir / AGENTS_FILENAME): + os.remove(self.project_dir / AGENTS_FILENAME) + non_existent_file_agent_names = get_all_agent_names() + self.assertEqual(non_existent_file_agent_names, []) + + def test_get_all_agent_names_empty_file(self): + with open(self.project_dir / AGENTS_FILENAME, 'w') as f: + f.write("") + + empty_agent_names = get_all_agent_names() + self.assertEqual(empty_agent_names, []) + + def test_get_all_agents(self): + shutil.copy(BASE_PATH / "fixtures/agents_max.yaml", self.project_dir / AGENTS_FILENAME) + + for agent in get_all_agents(): + self.assertIsInstance(agent, AgentConfig) From 1cc2b08e2154aca7b646ba0e72513aa4034915b2 Mon Sep 17 00:00:00 2001 From: Travis Dent Date: Tue, 7 Jan 2025 11:10:45 -0800 Subject: [PATCH 3/5] Full coverage for inputs.py --- tests/test_inputs_config.py | 62 ++++++++++++++++++++++++++++++++++++- 1 file changed, 61 insertions(+), 1 deletion(-) diff --git a/tests/test_inputs_config.py b/tests/test_inputs_config.py index 3ca883b0..23ea37d5 100644 --- a/tests/test_inputs_config.py +++ b/tests/test_inputs_config.py @@ -3,7 +3,8 @@ import unittest from pathlib import Path from agentstack import conf -from agentstack.inputs import InputsConfig +from agentstack.inputs import InputsConfig, get_inputs, add_input_for_run +from agentstack.exceptions import ValidationError BASE_PATH = Path(__file__).parent @@ -30,3 +31,62 @@ def test_maximal_input_config(self): assert config['input_name'] == "This in an input" assert config['input_name_2'] == "This is another input" assert config.to_dict() == {'input_name': "This in an input", 'input_name_2': "This is another input"} + + def test_yaml_error(self): + # Create an invalid YAML file + with open(self.project_dir / "src/config/inputs.yaml", 'w') as f: + f.write(""" +input_name: "This is a valid line" +invalid_yaml: "This line is missing a colon" + nested_key: "This will cause a YAML error" +""") + + # Attempt to load the config, which should raise a ValidationError + with self.assertRaises(ValidationError) as context: + InputsConfig() + + def test_create_inputs_file_if_not_exists(self): + # Ensure the inputs file doesn't exist + inputs_file = self.project_dir / "src/config/inputs.yaml" + if inputs_file.exists(): + inputs_file.unlink() + + # Create an InputsConfig instance and set a value + with InputsConfig() as config: + config['test_key'] = 'test_value' + + # Check that the file was created + self.assertTrue(inputs_file.exists()) + + def test_inputs_config_contains(self): + # Create an InputsConfig instance and set some values + with InputsConfig() as config: + config['existing_key'] = 'some_value' + config['another_key'] = 'another_value' + + # Test the __contains__ method + self.assertTrue('existing_key' in config) + self.assertTrue('another_key' in config) + self.assertFalse('non_existing_key' in config) + + def test_get_inputs(self): + # Set up some initial inputs + with InputsConfig() as config: + config['saved_key'] = 'saved_value' + + # Test get_inputs without run inputs + inputs = get_inputs() + self.assertEqual(inputs['saved_key'], 'saved_value') + + # Add a run input + add_input_for_run('run_key', 'run_value') + + # Test get_inputs with run inputs + inputs = get_inputs() + self.assertEqual(inputs['saved_key'], 'saved_value') + self.assertEqual(inputs['run_key'], 'run_value') + + # Test that run inputs override saved inputs + add_input_for_run('saved_key', 'overridden_value') + inputs = get_inputs() + self.assertEqual(inputs['saved_key'], 'overridden_value') From c3c35ec0540a6e389987e1955c798a20da1948ee Mon Sep 17 00:00:00 2001 From: Travis Dent Date: Tue, 7 Jan 2025 13:47:12 -0800 Subject: [PATCH 4/5] Full coverage for proj_templates.py --- agentstack/cli/cli.py | 13 +-- agentstack/proj_templates.py | 44 +++++--- tests/test_templates_config.py | 192 ++++++++++++++++++++++++++++++++- 3 files changed, 219 insertions(+), 30 deletions(-) diff --git a/agentstack/cli/cli.py b/agentstack/cli/cli.py index 1644f109..ca1b9493 100644 --- a/agentstack/cli/cli.py +++ b/agentstack/cli/cli.py @@ -56,18 +56,7 @@ def init_project_builder( template_data = None if template is not None: - if template.startswith("https://"): - try: - template_data = TemplateConfig.from_url(template) - except Exception as e: - print(term_color(f"Failed to fetch template data from {template}.\n{e}", 'red')) - sys.exit(1) - else: - try: - template_data = TemplateConfig.from_template_name(template) - except Exception as e: - print(term_color(f"Failed to load template {template}.\n{e}", 'red')) - sys.exit(1) + template_data = TemplateConfig.from_user_input(template) if template_data: project_details = { diff --git a/agentstack/proj_templates.py b/agentstack/proj_templates.py index 2e4ce194..d8bc0796 100644 --- a/agentstack/proj_templates.py +++ b/agentstack/proj_templates.py @@ -69,9 +69,9 @@ def to_v3(self) -> 'TemplateConfig': framework=self.framework, method=self.method, manager_agent=None, - agents=[TemplateConfig.Agent(**agent.dict()) for agent in self.agents], - tasks=[TemplateConfig.Task(**task.dict()) for task in self.tasks], - tools=[TemplateConfig.Tool(**tool.dict()) for tool in self.tools], + agents=[TemplateConfig.Agent(**agent.model_dump()) for agent in self.agents], + tasks=[TemplateConfig.Task(**task.model_dump()) for task in self.tasks], + tools=[TemplateConfig.Tool(**tool.model_dump()) for tool in self.tools], inputs=self.inputs, ) @@ -144,17 +144,22 @@ def write_to_file(self, filename: Path): f.write(json.dumps(model_dump, indent=4)) @classmethod - def from_template_name(cls, name: str) -> 'TemplateConfig': - # if url - if name.startswith('https://'): - return cls.from_url(name) - - # if .json file - if name.endswith('.json'): - path = os.getcwd() / Path(name) + def from_user_input(cls, identifier: str): + """ + Load a template from a user-provided identifier. + Three cases will be tried: A URL, a file path, or a template name. + """ + if identifier.startswith('https://'): + return cls.from_url(identifier) + + if identifier.endswith('.json'): + path = Path() / identifier return cls.from_file(path) - # if named template + return cls.from_template_name(identifier) + + @classmethod + def from_template_name(cls, name: str) -> 'TemplateConfig': path = get_package_path() / f'templates/proj_templates/{name}.json' if not name in get_all_template_names(): raise ValidationError(f"Template {name} not bundled with agentstack.") @@ -162,10 +167,14 @@ def from_template_name(cls, name: str) -> 'TemplateConfig': @classmethod def from_file(cls, path: Path) -> 'TemplateConfig': + print(path) if not os.path.exists(path): raise ValidationError(f"Template {path} not found.") - with open(path, 'r') as f: - return cls.from_json(json.load(f)) + try: + with open(path, 'r') as f: + return cls.from_json(json.load(f)) + except json.JSONDecodeError as e: + raise ValidationError(f"Error decoding template JSON.\n{e}") @classmethod def from_url(cls, url: str) -> 'TemplateConfig': @@ -174,7 +183,10 @@ def from_url(cls, url: str) -> 'TemplateConfig': response = requests.get(url) if response.status_code != 200: raise ValidationError(f"Failed to fetch template from {url}") - return cls.from_json(response.json()) + try: + return cls.from_json(response.json()) + except json.JSONDecodeError as e: + raise ValidationError(f"Error decoding template JSON.\n{e}") @classmethod def from_json(cls, data: dict) -> 'TemplateConfig': @@ -193,8 +205,6 @@ def from_json(cls, data: dict) -> 'TemplateConfig': for error in e.errors(): err_msg += f"{' '.join([str(loc) for loc in error['loc']])}: {error['msg']}\n" raise ValidationError(err_msg) - except json.JSONDecodeError as e: - raise ValidationError(f"Error decoding template JSON.\n{e}") def get_all_template_paths() -> list[Path]: diff --git a/tests/test_templates_config.py b/tests/test_templates_config.py index 47926c0f..63779d5d 100644 --- a/tests/test_templates_config.py +++ b/tests/test_templates_config.py @@ -1,9 +1,17 @@ from pathlib import Path import json import unittest +import os +import shutil +from unittest.mock import patch from parameterized import parameterized from agentstack.exceptions import ValidationError -from agentstack.proj_templates import TemplateConfig, get_all_template_names, get_all_template_paths +from agentstack.proj_templates import ( + TemplateConfig, + get_all_template_names, + get_all_template_paths, + get_all_templates, +) BASE_PATH = Path(__file__).parent VALID_TEMPLATE_URL = "https://raw.githubusercontent.com/AgentOps-AI/AgentStack/13a6e335fb163b932ed037562fcedbc269f0d5a5/agentstack/templates/proj_templates/content_creator.json" @@ -11,6 +19,13 @@ class TemplateConfigTest(unittest.TestCase): + def setUp(self): + self.project_dir = BASE_PATH / 'tmp/template_config' + os.makedirs(self.project_dir, exist_ok=True) + + def tearDown(self): + shutil.rmtree(self.project_dir) + @parameterized.expand([(x,) for x in get_all_template_names()]) def test_all_configs_from_template_name(self, template_name: str): config = TemplateConfig.from_template_name(template_name) @@ -34,3 +49,178 @@ def test_load_template_from_valid_url(self): def load_template_from_invalid_url(self): with self.assertRaises(ValidationError): TemplateConfig.from_url(INVALID_TEMPLATE_URL) + + def test_write_to_file_with_json_suffix(self): + config = TemplateConfig.from_template_name("content_creator") + file_path = self.project_dir / "test_template.json" + config.write_to_file(file_path) + + # Check if file exists + self.assertTrue(file_path.exists()) + + # Read the file and check if it's valid JSON + with open(file_path, 'r') as f: + written_data = json.load(f) + + # Compare written data with original config + self.assertEqual(written_data["name"], config.name) + self.assertEqual(written_data["description"], config.description) + self.assertEqual(written_data["template_version"], config.template_version) + + def test_write_to_file_without_suffix(self): + config = TemplateConfig.from_template_name("content_creator") + file_path = self.project_dir / "test_template" + config.write_to_file(file_path) + + # Check if file exists with .json suffix + expected_path = file_path.with_suffix('.json') + self.assertTrue(expected_path.exists()) + + # Read the file and check if it's valid JSON + with open(expected_path, 'r') as f: + written_data = json.load(f) + + # Compare written data with original config + self.assertEqual(written_data["name"], config.name) + self.assertEqual(written_data["description"], config.description) + self.assertEqual(written_data["template_version"], config.template_version) + + def test_from_user_input_url(self): + config = TemplateConfig.from_user_input(VALID_TEMPLATE_URL) + self.assertEqual(config.name, "content_creator") + self.assertEqual(config.template_version, 3) + + def test_from_user_input_name(self): + config = TemplateConfig.from_user_input('content_creator') + self.assertEqual(config.name, "content_creator") + self.assertEqual(config.template_version, 3) + + def test_from_user_input_local_file(self): + test_file = self.project_dir / 'test_local_template.json' + + test_data = { + "name": "test_local", + "description": "Test local file", + "template_version": 3, + "framework": "test", + "method": "test", + "manager_agent": None, + "agents": [], + "tasks": [], + "tools": [], + "inputs": {}, + } + + with open(test_file, 'w') as f: + json.dump(test_data, f, indent=2) + + config = TemplateConfig.from_user_input(str(test_file)) + self.assertEqual(config.name, "test_local") + self.assertEqual(config.template_version, 3) + + def test_from_file_missing_file(self): + non_existent_path = Path("/path/to/non_existent_file.json") + with self.assertRaises(ValidationError) as context: + TemplateConfig.from_file(non_existent_path) + + def test_from_url_invalid_url(self): + invalid_url = "not_a_valid_url" + with self.assertRaises(ValidationError) as context: + TemplateConfig.from_url(invalid_url) + + @patch('agentstack.proj_templates.requests.get') + def test_from_url_non_200_response(self, mock_get): + mock_response = mock_get.return_value + mock_response.status_code = 404 + + invalid_url = "https://example.com/non_existent_template.json" + with self.assertRaises(ValidationError) as context: + TemplateConfig.from_url(invalid_url) + mock_get.assert_called_once_with(invalid_url) + + def test_from_json_invalid_version(self): + invalid_template = { + "name": "invalid_version_template", + "description": "A template with an invalid version", + "template_version": 999, # Invalid version + "framework": "test", + "method": "test", + "manager_agent": None, + "agents": [], + "tasks": [], + "tools": [], + "inputs": {}, + } + with self.assertRaises(ValidationError) as context: + TemplateConfig.from_json(invalid_template) + + def test_from_json_pydantic_validation_error(self): + invalid_template = { + "name": "invalid_template", + "description": "A template with invalid data", + "template_version": 3, + "framework": "test", + "method": "test", + "manager_agent": None, + "agents": [ + { + "name": "Invalid Agent", + "role": "Tester", + "goal": "Test invalid data", + "backstory": "This agent has an invalid model", + "allow_delegation": False, + "model": 123, # This should be a string, not an integer + } + ], + "tasks": [], + "tools": [], + "inputs": {}, + } + with self.assertRaises(ValidationError) as context: + TemplateConfig.from_json(invalid_template) + + def test_from_file_invalid_json(self): + temp_file = self.project_dir / 'invalid_template.json' + with open(temp_file, 'w') as f: + f.write("This is not valid JSON") + + try: + with self.assertRaises(ValidationError) as context: + TemplateConfig.from_file(temp_file) + finally: + os.unlink(temp_file) + + @patch('agentstack.proj_templates.requests.get') + def test_from_url_invalid_json(self, mock_get): + mock_response = mock_get.return_value + mock_response.status_code = 200 + mock_response.json.side_effect = json.JSONDecodeError("Invalid JSON", "", 0) + + invalid_url = "https://example.com/invalid_json_template.json" + with self.assertRaises(ValidationError) as context: + TemplateConfig.from_url(invalid_url) + mock_get.assert_called_once_with(invalid_url) + + def test_get_all_templates(self): + for template in get_all_templates(): + self.assertIsInstance(template, TemplateConfig) + + def test_get_all_template_names(self): + for name in get_all_template_names(): + self.assertIsInstance(name, str) + + def test_get_all_template_paths(self): + for path in get_all_template_paths(): + self.assertIsInstance(path, Path) + + @patch('agentstack.proj_templates.get_package_path') + @patch('pathlib.Path.iterdir') + def test_get_all_template_paths_no_json_files(self, mock_iterdir, mock_get_package_path): + mock_get_package_path.return_value = Path('/mock/path') + mock_iterdir.return_value = [Path('file1.txt'), Path('file2.csv')] # No JSON files + + paths = get_all_template_paths() + + self.assertEqual(paths, []) + mock_get_package_path.assert_called_once() + mock_iterdir.assert_called_once() From f5d93b4a891c28a0d82a336a486630ae11c509a2 Mon Sep 17 00:00:00 2001 From: Travis Dent Date: Tue, 14 Jan 2025 14:31:59 -0800 Subject: [PATCH 5/5] Rogue print statement --- agentstack/proj_templates.py | 1 - 1 file changed, 1 deletion(-) diff --git a/agentstack/proj_templates.py b/agentstack/proj_templates.py index d8bc0796..b92acd09 100644 --- a/agentstack/proj_templates.py +++ b/agentstack/proj_templates.py @@ -167,7 +167,6 @@ def from_template_name(cls, name: str) -> 'TemplateConfig': @classmethod def from_file(cls, path: Path) -> 'TemplateConfig': - print(path) if not os.path.exists(path): raise ValidationError(f"Template {path} not found.") try: