From 20509f134d540a9aa8bc0b423b0106bfcc915492 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 29 Dec 2025 21:19:02 +0000 Subject: [PATCH 1/6] Initial plan From e3160e55a9d2e8e4d369e4d4e3940bda5cc2c8ed Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 29 Dec 2025 21:26:40 +0000 Subject: [PATCH 2/6] Fix CSV quote stripping and add test Co-authored-by: luigiw <1483379+luigiw@users.noreply.github.com> --- .../azure/ai/evaluation/_evaluate/_utils.py | 3 +- .../tests/unittests/data/test_csv_quotes.csv | 4 ++ .../tests/unittests/test_evaluate.py | 47 +++++++++++++++++++ 3 files changed, 53 insertions(+), 1 deletion(-) create mode 100644 sdk/evaluation/azure-ai-evaluation/tests/unittests/data/test_csv_quotes.csv diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_utils.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_utils.py index 7050ecef15ce..10b56dd5a32b 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_utils.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_utils.py @@ -1,6 +1,7 @@ # --------------------------------------------------------- # Copyright (c) Microsoft Corporation. All rights reserved. # --------------------------------------------------------- +import csv import json import logging import os @@ -479,7 +480,7 @@ def __init__(self, filename: Union[os.PathLike, str]): self.filename = filename def load(self) -> pd.DataFrame: - return pd.read_csv(self.filename, dtype=str) + return pd.read_csv(self.filename, dtype=str, quoting=csv.QUOTE_NONE, escapechar="\\") class DataLoaderFactory: diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/data/test_csv_quotes.csv b/sdk/evaluation/azure-ai-evaluation/tests/unittests/data/test_csv_quotes.csv new file mode 100644 index 000000000000..d9e04ce14a10 --- /dev/null +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/data/test_csv_quotes.csv @@ -0,0 +1,4 @@ +response,ground_truth +test,"test" +"quoted",quoted +"start,middle"end diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py index e110eb369369..f4f8e43f3634 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py @@ -1652,3 +1652,50 @@ def test_log_metrics_and_instance_results_onedp_no_redundant_tags(self, mock_cli call_args = mock_client.start_evaluation_run.call_args eval_upload = call_args[1]["evaluation"] assert eval_upload.tags == tags + + def test_csv_preserves_quotes_in_values(self): + """Test that CSV loading preserves quotes in cell values. + + This test validates the fix for the issue where custom code evaluators + were dropping leading and trailing quotation marks from parameter values. + The issue occurs when a CSV cell value starts AND ends with quotes. + """ + # Get the test CSV file + csv_file = _get_file("test_csv_quotes.csv") + + # Define a custom evaluator that checks if quotes are preserved + def quote_checker(response: str, ground_truth: str): + """Custom evaluator that checks if values match exactly.""" + return { + "match": 1 if response == ground_truth else 0, + "response_value": response, + "ground_truth_value": ground_truth, + } + + # Run evaluation with the custom evaluator + result = evaluate( + data=csv_file, + evaluators={"quote_checker": quote_checker}, + ) + + # Verify the results + assert result is not None + row_result_df = pd.DataFrame(result["rows"]) + + # Check that we have the expected rows + assert len(row_result_df) == 3 + + # Row 0: response='test', ground_truth='"test"' - should NOT match + assert row_result_df["outputs.quote_checker.response_value"][0] == "test" + assert row_result_df["outputs.quote_checker.ground_truth_value"][0] == '"test"' + assert row_result_df["outputs.quote_checker.match"][0] == 0 + + # Row 1: response='"quoted"', ground_truth='quoted' - should NOT match + assert row_result_df["outputs.quote_checker.response_value"][1] == '"quoted"' + assert row_result_df["outputs.quote_checker.ground_truth_value"][1] == "quoted" + assert row_result_df["outputs.quote_checker.match"][1] == 0 + + # Row 2: response='"start', ground_truth='middle"end' - should NOT match + assert row_result_df["outputs.quote_checker.response_value"][2] == '"start' + assert row_result_df["outputs.quote_checker.ground_truth_value"][2] == 'middle"end' + assert row_result_df["outputs.quote_checker.match"][2] == 0 From eb4e32ea2f9b70be27124d3f39c0326e443ef528 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 29 Dec 2025 21:31:18 +0000 Subject: [PATCH 3/6] Fix malformed CSV test data based on code review Co-authored-by: luigiw <1483379+luigiw@users.noreply.github.com> --- .../tests/unittests/data/test_csv_quotes.csv | 2 +- .../azure-ai-evaluation/tests/unittests/test_evaluate.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/data/test_csv_quotes.csv b/sdk/evaluation/azure-ai-evaluation/tests/unittests/data/test_csv_quotes.csv index d9e04ce14a10..decf129a5777 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/data/test_csv_quotes.csv +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/data/test_csv_quotes.csv @@ -1,4 +1,4 @@ response,ground_truth test,"test" "quoted",quoted -"start,middle"end +start,"end diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py index f4f8e43f3634..143249649029 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py @@ -1695,7 +1695,7 @@ def quote_checker(response: str, ground_truth: str): assert row_result_df["outputs.quote_checker.ground_truth_value"][1] == "quoted" assert row_result_df["outputs.quote_checker.match"][1] == 0 - # Row 2: response='"start', ground_truth='middle"end' - should NOT match - assert row_result_df["outputs.quote_checker.response_value"][2] == '"start' - assert row_result_df["outputs.quote_checker.ground_truth_value"][2] == 'middle"end' + # Row 2: response='start', ground_truth='"end' - should NOT match + assert row_result_df["outputs.quote_checker.response_value"][2] == 'start' + assert row_result_df["outputs.quote_checker.ground_truth_value"][2] == '"end' assert row_result_df["outputs.quote_checker.match"][2] == 0 From dc0921cbef54a34b92c387868e9063c42ec14ba7 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 29 Dec 2025 21:33:14 +0000 Subject: [PATCH 4/6] Fix CSV test data and update test expectations Co-authored-by: luigiw <1483379+luigiw@users.noreply.github.com> --- .../tests/unittests/data/test_csv_quotes.csv | 2 +- .../azure-ai-evaluation/tests/unittests/test_evaluate.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/data/test_csv_quotes.csv b/sdk/evaluation/azure-ai-evaluation/tests/unittests/data/test_csv_quotes.csv index decf129a5777..280d8481baec 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/data/test_csv_quotes.csv +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/data/test_csv_quotes.csv @@ -1,4 +1,4 @@ response,ground_truth test,"test" "quoted",quoted -start,"end +start,"""end" \ No newline at end of file diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py index 143249649029..bbc32f58c589 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py @@ -1695,7 +1695,7 @@ def quote_checker(response: str, ground_truth: str): assert row_result_df["outputs.quote_checker.ground_truth_value"][1] == "quoted" assert row_result_df["outputs.quote_checker.match"][1] == 0 - # Row 2: response='start', ground_truth='"end' - should NOT match + # Row 2: response='start', ground_truth='"""end"' - should NOT match assert row_result_df["outputs.quote_checker.response_value"][2] == 'start' - assert row_result_df["outputs.quote_checker.ground_truth_value"][2] == '"end' + assert row_result_df["outputs.quote_checker.ground_truth_value"][2] == '"""end"' assert row_result_df["outputs.quote_checker.match"][2] == 0 From fc3782b2aed0721a1fac3a9ae56d2551c51bb152 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 29 Dec 2025 21:34:29 +0000 Subject: [PATCH 5/6] Add explanatory comment for CSV quote handling Co-authored-by: luigiw <1483379+luigiw@users.noreply.github.com> --- .../azure/ai/evaluation/_evaluate/_utils.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_utils.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_utils.py index 10b56dd5a32b..a36e4ae73bef 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_utils.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_utils.py @@ -480,6 +480,10 @@ def __init__(self, filename: Union[os.PathLike, str]): self.filename = filename def load(self) -> pd.DataFrame: + # Use QUOTE_NONE to preserve quotation marks as literal characters in cell values. + # By default, pandas treats quotes as CSV field delimiters and strips them. + # This ensures that values like "test" are read as "test" (with quotes), not test. + # The escapechar allows escaping special characters like commas within values. return pd.read_csv(self.filename, dtype=str, quoting=csv.QUOTE_NONE, escapechar="\\") From c2e38dfe5d959c671a999741c3956960954443eb Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 29 Dec 2025 21:36:32 +0000 Subject: [PATCH 6/6] Add detailed documentation to test explaining quote handling behavior Co-authored-by: luigiw <1483379+luigiw@users.noreply.github.com> --- .../azure-ai-evaluation/tests/unittests/test_evaluate.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py index bbc32f58c589..6c47a67cbb49 100644 --- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py +++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py @@ -1659,6 +1659,13 @@ def test_csv_preserves_quotes_in_values(self): This test validates the fix for the issue where custom code evaluators were dropping leading and trailing quotation marks from parameter values. The issue occurs when a CSV cell value starts AND ends with quotes. + + Test CSV contains: + - Row 0: test,"test" - unquoted vs quoted value + - Row 1: "quoted",quoted - quoted vs unquoted value + - Row 2: start,"""end" - tests that even multiple quotes are preserved as literals + + With QUOTE_NONE, all quotes are treated as literal characters, not delimiters. """ # Get the test CSV file csv_file = _get_file("test_csv_quotes.csv") @@ -1696,6 +1703,7 @@ def quote_checker(response: str, ground_truth: str): assert row_result_df["outputs.quote_checker.match"][1] == 0 # Row 2: response='start', ground_truth='"""end"' - should NOT match + # Note: With QUOTE_NONE, """end" is read as the literal string """end" assert row_result_df["outputs.quote_checker.response_value"][2] == 'start' assert row_result_df["outputs.quote_checker.ground_truth_value"][2] == '"""end"' assert row_result_df["outputs.quote_checker.match"][2] == 0