From b6ff99fb1785219ab37fe65dc2981278f8805a08 Mon Sep 17 00:00:00 2001 From: Subrata Paitandi Date: Tue, 27 Jan 2026 10:47:00 +0000 Subject: [PATCH 01/10] segfault fix --- mssql_python/pybind/connection/connection.cpp | 28 ++++++++++++++++++- mssql_python/pybind/connection/connection.h | 4 +++ mssql_python/pybind/ddbc_bindings.cpp | 27 ++++++++++++------ mssql_python/pybind/ddbc_bindings.h | 6 ++++ 4 files changed, 55 insertions(+), 10 deletions(-) diff --git a/mssql_python/pybind/connection/connection.cpp b/mssql_python/pybind/connection/connection.cpp index 1fe4d2132..d61971b32 100644 --- a/mssql_python/pybind/connection/connection.cpp +++ b/mssql_python/pybind/connection/connection.cpp @@ -94,6 +94,19 @@ void Connection::connect(const py::dict& attrs_before) { void Connection::disconnect() { if (_dbcHandle) { LOG("Disconnecting from database"); + + // CRITICAL FIX: Mark all child statement handles as implicitly freed + // When we free the DBC handle below, the ODBC driver will automatically free + // all child STMT handles. We need to tell the SqlHandle objects about this + // so they don't try to free the handles again during their destruction. + LOG("Marking %zu child statement handles as implicitly freed", _childStatementHandles.size()); + for (auto& weakHandle : _childStatementHandles) { + if (auto handle = weakHandle.lock()) { + handle->markImplicitlyFreed(); + } + } + _childStatementHandles.clear(); + SQLRETURN ret = SQLDisconnect_ptr(_dbcHandle->get()); checkError(ret); // triggers SQLFreeHandle via destructor, if last owner @@ -173,7 +186,20 @@ SqlHandlePtr Connection::allocStatementHandle() { SQLHANDLE stmt = nullptr; SQLRETURN ret = SQLAllocHandle_ptr(SQL_HANDLE_STMT, _dbcHandle->get(), &stmt); checkError(ret); - return std::make_shared(static_cast(SQL_HANDLE_STMT), stmt); + auto stmtHandle = std::make_shared(static_cast(SQL_HANDLE_STMT), stmt); + + // Track this child handle so we can mark it as implicitly freed when connection closes + // Use weak_ptr to avoid circular references and allow normal cleanup + _childStatementHandles.push_back(stmtHandle); + + // Clean up expired weak_ptrs periodically to avoid unbounded growth + // Remove entries where the weak_ptr is expired (object was already destroyed) + _childStatementHandles.erase( + std::remove_if(_childStatementHandles.begin(), _childStatementHandles.end(), + [](const std::weak_ptr& wp) { return wp.expired(); }), + _childStatementHandles.end()); + + return stmtHandle; } SQLRETURN Connection::setAttribute(SQLINTEGER attribute, py::object value) { diff --git a/mssql_python/pybind/connection/connection.h b/mssql_python/pybind/connection/connection.h index d007106af..6bdb596bd 100644 --- a/mssql_python/pybind/connection/connection.h +++ b/mssql_python/pybind/connection/connection.h @@ -61,6 +61,10 @@ class Connection { std::chrono::steady_clock::time_point _lastUsed; std::wstring wstrStringBuffer; // wstr buffer for string attribute setting std::string strBytesBuffer; // string buffer for byte attributes setting + + // Track child statement handles to mark them as implicitly freed when connection closes + // Uses weak_ptr to avoid circular references and allow normal cleanup + std::vector> _childStatementHandles; }; class ConnectionHandle { diff --git a/mssql_python/pybind/ddbc_bindings.cpp b/mssql_python/pybind/ddbc_bindings.cpp index f49d860a8..32baaf0f9 100644 --- a/mssql_python/pybind/ddbc_bindings.cpp +++ b/mssql_python/pybind/ddbc_bindings.cpp @@ -1144,6 +1144,10 @@ SQLSMALLINT SqlHandle::type() const { return _type; } +void SqlHandle::markImplicitlyFreed() { + _implicitly_freed = true; +} + /* * IMPORTANT: Never log in destructors - it causes segfaults. * During program exit, C++ destructors may run AFTER Python shuts down. @@ -1169,16 +1173,19 @@ void SqlHandle::free() { return; } - // Always clean up ODBC resources, regardless of Python state + // CRITICAL FIX: Check if handle was already implicitly freed by parent handle + // When Connection::disconnect() frees the DBC handle, the ODBC driver automatically + // frees all child STMT handles. We track this state to avoid double-free attempts. + // This approach avoids calling ODBC functions on potentially-freed handles, which + // would cause use-after-free errors. + if (_implicitly_freed) { + _handle = nullptr; // Just clear the pointer, don't call ODBC functions + return; + } + + // Handle is valid and not implicitly freed, proceed with normal freeing SQLFreeHandle_ptr(_type, _handle); _handle = nullptr; - - // Only log if Python is not shutting down (to avoid segfault) - if (!pythonShuttingDown) { - // Don't log during destruction - even in normal cases it can be - // problematic If logging is needed, use explicit close() methods - // instead - } } } @@ -4360,7 +4367,9 @@ PYBIND11_MODULE(ddbc_bindings, m) { .def_readwrite("ddbcErrorMsg", &ErrorInfo::ddbcErrorMsg); py::class_(m, "SqlHandle") - .def("free", &SqlHandle::free, "Free the handle"); + .def("free", &SqlHandle::free, "Free the handle") + .def("markImplicitlyFreed", &SqlHandle::markImplicitlyFreed, + "Mark handle as implicitly freed by parent handle"); py::class_(m, "Connection") .def(py::init(), py::arg("conn_str"), diff --git a/mssql_python/pybind/ddbc_bindings.h b/mssql_python/pybind/ddbc_bindings.h index 391903ef2..87391b552 100644 --- a/mssql_python/pybind/ddbc_bindings.h +++ b/mssql_python/pybind/ddbc_bindings.h @@ -378,10 +378,16 @@ class SqlHandle { SQLHANDLE get() const; SQLSMALLINT type() const; void free(); + + // Mark this handle as implicitly freed (freed by parent handle) + // This prevents double-free attempts when the ODBC driver automatically + // frees child handles (e.g., STMT handles when DBC handle is freed) + void markImplicitlyFreed(); private: SQLSMALLINT _type; SQLHANDLE _handle; + bool _implicitly_freed = false; // Tracks if handle was freed by parent }; using SqlHandlePtr = std::shared_ptr; From b7f105f1297a7b5a18818b5b67f36b9d20f99074 Mon Sep 17 00:00:00 2001 From: Subrata Paitandi Date: Tue, 27 Jan 2026 11:06:46 +0000 Subject: [PATCH 02/10] test addition and linting fix --- mssql_python/pybind/connection/connection.cpp | 17 +- mssql_python/pybind/connection/connection.h | 2 +- mssql_python/pybind/ddbc_bindings.cpp | 11 +- mssql_python/pybind/ddbc_bindings.h | 2 +- ...st_016_connection_invalidation_segfault.py | 351 ++++++++++++++++++ 5 files changed, 366 insertions(+), 17 deletions(-) create mode 100644 tests/test_016_connection_invalidation_segfault.py diff --git a/mssql_python/pybind/connection/connection.cpp b/mssql_python/pybind/connection/connection.cpp index d61971b32..7c5e756a4 100644 --- a/mssql_python/pybind/connection/connection.cpp +++ b/mssql_python/pybind/connection/connection.cpp @@ -94,19 +94,20 @@ void Connection::connect(const py::dict& attrs_before) { void Connection::disconnect() { if (_dbcHandle) { LOG("Disconnecting from database"); - + // CRITICAL FIX: Mark all child statement handles as implicitly freed // When we free the DBC handle below, the ODBC driver will automatically free // all child STMT handles. We need to tell the SqlHandle objects about this // so they don't try to free the handles again during their destruction. - LOG("Marking %zu child statement handles as implicitly freed", _childStatementHandles.size()); + LOG("Marking %zu child statement handles as implicitly freed", + _childStatementHandles.size()); for (auto& weakHandle : _childStatementHandles) { if (auto handle = weakHandle.lock()) { handle->markImplicitlyFreed(); } } _childStatementHandles.clear(); - + SQLRETURN ret = SQLDisconnect_ptr(_dbcHandle->get()); checkError(ret); // triggers SQLFreeHandle via destructor, if last owner @@ -187,18 +188,18 @@ SqlHandlePtr Connection::allocStatementHandle() { SQLRETURN ret = SQLAllocHandle_ptr(SQL_HANDLE_STMT, _dbcHandle->get(), &stmt); checkError(ret); auto stmtHandle = std::make_shared(static_cast(SQL_HANDLE_STMT), stmt); - + // Track this child handle so we can mark it as implicitly freed when connection closes // Use weak_ptr to avoid circular references and allow normal cleanup _childStatementHandles.push_back(stmtHandle); - + // Clean up expired weak_ptrs periodically to avoid unbounded growth // Remove entries where the weak_ptr is expired (object was already destroyed) _childStatementHandles.erase( std::remove_if(_childStatementHandles.begin(), _childStatementHandles.end(), [](const std::weak_ptr& wp) { return wp.expired(); }), _childStatementHandles.end()); - + return stmtHandle; } @@ -334,7 +335,7 @@ bool Connection::reset() { disconnect(); return false; } - + // SQL_ATTR_RESET_CONNECTION does NOT reset the transaction isolation level. // Explicitly reset it to the default (SQL_TXN_READ_COMMITTED) to prevent // isolation level settings from leaking between pooled connection usages. @@ -346,7 +347,7 @@ bool Connection::reset() { disconnect(); return false; } - + updateLastUsed(); return true; } diff --git a/mssql_python/pybind/connection/connection.h b/mssql_python/pybind/connection/connection.h index 6bdb596bd..4bda21a08 100644 --- a/mssql_python/pybind/connection/connection.h +++ b/mssql_python/pybind/connection/connection.h @@ -61,7 +61,7 @@ class Connection { std::chrono::steady_clock::time_point _lastUsed; std::wstring wstrStringBuffer; // wstr buffer for string attribute setting std::string strBytesBuffer; // string buffer for byte attributes setting - + // Track child statement handles to mark them as implicitly freed when connection closes // Uses weak_ptr to avoid circular references and allow normal cleanup std::vector> _childStatementHandles; diff --git a/mssql_python/pybind/ddbc_bindings.cpp b/mssql_python/pybind/ddbc_bindings.cpp index 32baaf0f9..52b9d21ee 100644 --- a/mssql_python/pybind/ddbc_bindings.cpp +++ b/mssql_python/pybind/ddbc_bindings.cpp @@ -2900,7 +2900,6 @@ SQLRETURN SQLGetData_wrap(SqlHandlePtr StatementHandle, SQLUSMALLINT colCount, p // Cache decimal separator to avoid repeated system calls - for (SQLSMALLINT i = 1; i <= colCount; ++i) { SQLWCHAR columnName[256]; SQLSMALLINT columnNameLen; @@ -3622,8 +3621,6 @@ SQLRETURN FetchBatchData(SQLHSTMT hStmt, ColumnBuffers& buffers, py::list& colum columnInfos[col].processedColumnSize + 1; // +1 for null terminator } - - // Performance: Build function pointer dispatch table (once per batch) // This eliminates the switch statement from the hot loop - 10,000 rows × 10 // cols reduces from 100,000 switch evaluations to just 10 switch @@ -4040,8 +4037,8 @@ SQLRETURN FetchMany_wrap(SqlHandlePtr StatementHandle, py::list& rows, int fetch lobColumns.push_back(i + 1); // 1-based } } - - // Initialized to 0 for LOB path counter; overwritten by ODBC in non-LOB path; + + // Initialized to 0 for LOB path counter; overwritten by ODBC in non-LOB path; SQLULEN numRowsFetched = 0; // If we have LOBs → fall back to row-by-row fetch + SQLGetData_wrap if (!lobColumns.empty()) { @@ -4073,7 +4070,7 @@ SQLRETURN FetchMany_wrap(SqlHandlePtr StatementHandle, py::list& rows, int fetch LOG("FetchMany_wrap: Error when binding columns - SQLRETURN=%d", ret); return ret; } - + SQLSetStmtAttr_ptr(hStmt, SQL_ATTR_ROW_ARRAY_SIZE, (SQLPOINTER)(intptr_t)fetchSize, 0); SQLSetStmtAttr_ptr(hStmt, SQL_ATTR_ROWS_FETCHED_PTR, &numRowsFetched, 0); @@ -4368,7 +4365,7 @@ PYBIND11_MODULE(ddbc_bindings, m) { py::class_(m, "SqlHandle") .def("free", &SqlHandle::free, "Free the handle") - .def("markImplicitlyFreed", &SqlHandle::markImplicitlyFreed, + .def("markImplicitlyFreed", &SqlHandle::markImplicitlyFreed, "Mark handle as implicitly freed by parent handle"); py::class_(m, "Connection") diff --git a/mssql_python/pybind/ddbc_bindings.h b/mssql_python/pybind/ddbc_bindings.h index 87391b552..190c9bd1a 100644 --- a/mssql_python/pybind/ddbc_bindings.h +++ b/mssql_python/pybind/ddbc_bindings.h @@ -378,7 +378,7 @@ class SqlHandle { SQLHANDLE get() const; SQLSMALLINT type() const; void free(); - + // Mark this handle as implicitly freed (freed by parent handle) // This prevents double-free attempts when the ODBC driver automatically // frees child handles (e.g., STMT handles when DBC handle is freed) diff --git a/tests/test_016_connection_invalidation_segfault.py b/tests/test_016_connection_invalidation_segfault.py new file mode 100644 index 000000000..b92826aa6 --- /dev/null +++ b/tests/test_016_connection_invalidation_segfault.py @@ -0,0 +1,351 @@ +""" +Test for connection invalidation segfault scenario (Issue: Use-after-free on statement handles) + +This test reproduces the segfault that occurred in SQLAlchemy's RealReconnectTest +when connection invalidation triggered automatic freeing of child statement handles +by the ODBC driver, followed by Python GC attempting to free the same handles again. + +The fix uses state tracking where Connection marks child handles as "implicitly freed" +before disconnecting, preventing SqlHandle::free() from calling ODBC functions on +already-freed handles. + +Background: +- When Connection::disconnect() frees a DBC handle, ODBC automatically frees all child STMT handles +- Python SqlHandle objects weren't aware of this implicit freeing +- GC later tried to free those handles again via SqlHandle::free(), causing use-after-free +- Fix: Connection tracks children in _childStatementHandles vector and marks them as + implicitly freed before DBC is freed +""" + +import gc +import pytest +from mssql_python import connect, DatabaseError, OperationalError + + +def test_connection_invalidation_with_multiple_cursors(conn_str): + """ + Test connection invalidation scenario that previously caused segfaults. + + This test: + 1. Creates a connection with multiple active cursors + 2. Executes queries on those cursors to create statement handles + 3. Simulates connection invalidation by closing the connection + 4. Forces garbage collection to trigger handle cleanup + 5. Verifies no segfault occurs during the cleanup process + + Previously, this would crash because: + - Closing connection freed the DBC handle + - ODBC driver automatically freed all child STMT handles + - Python GC later tried to free those same STMT handles + - Result: use-after-free crash (segfault) + + With the fix: + - Connection marks all child handles as "implicitly freed" before closing + - SqlHandle::free() checks the flag and skips ODBC calls if true + - Result: No crash, clean shutdown + """ + # Create connection + conn = connect(conn_str) + + # Create multiple cursors with statement handles + cursors = [] + for i in range(5): + cursor = conn.cursor() + cursor.execute("SELECT 1 AS id, 'test' AS name") + cursor.fetchall() # Fetch results to complete the query + cursors.append(cursor) + + # Close connection without explicitly closing cursors first + # This simulates the invalidation scenario where connection is lost + conn.close() + + # Force garbage collection to trigger cursor cleanup + # This is where the segfault would occur without the fix + cursors = None + gc.collect() + + # If we reach here without crashing, the fix is working + assert True + + +def test_connection_invalidation_without_cursor_close(conn_str): + """ + Test that cursors are properly cleaned up when connection is closed + without explicitly closing the cursors. + + This mimics the SQLAlchemy scenario where connection pools may + invalidate connections without first closing all cursors. + """ + conn = connect(conn_str) + + # Create cursors and execute queries + cursor1 = conn.cursor() + cursor1.execute("SELECT 1") + cursor1.fetchone() + + cursor2 = conn.cursor() + cursor2.execute("SELECT 2") + cursor2.fetchone() + + cursor3 = conn.cursor() + cursor3.execute("SELECT 3") + cursor3.fetchone() + + # Close connection with active cursors + conn.close() + + # Trigger GC - should not crash + del cursor1, cursor2, cursor3 + gc.collect() + + assert True + + +def test_repeated_connection_invalidation_cycles(conn_str): + """ + Test repeated connection invalidation cycles to ensure no memory + corruption or handle leaks occur across multiple iterations. + + This stress test simulates the scenario from SQLAlchemy's + RealReconnectTest which ran multiple invalidation tests in sequence. + """ + for iteration in range(10): + # Create connection + conn = connect(conn_str) + + # Create and use cursors + for cursor_num in range(3): + cursor = conn.cursor() + cursor.execute(f"SELECT {iteration} AS iteration, {cursor_num} AS cursor_num") + result = cursor.fetchone() + assert result[0] == iteration + assert result[1] == cursor_num + + # Close connection (invalidate) + conn.close() + + # Force GC after each iteration + gc.collect() + + # Final GC to clean up any remaining references + gc.collect() + assert True + + +def test_connection_close_with_uncommitted_transaction(conn_str): + """ + Test that closing a connection with an uncommitted transaction + properly cleans up statement handles without crashing. + """ + conn = connect(conn_str) + cursor = conn.cursor() + + try: + # Start a transaction + cursor.execute("CREATE TABLE #temp_test (id INT, name VARCHAR(50))") + cursor.execute("INSERT INTO #temp_test VALUES (1, 'test')") + # Don't commit - leave transaction open + + # Close connection without commit or rollback + conn.close() + + # Trigger GC + del cursor + gc.collect() + + assert True + except Exception as e: + pytest.fail(f"Unexpected exception during connection close: {e}") + + +def test_cursor_after_connection_invalidation_raises_error(conn_str): + """ + Test that attempting to use a cursor after connection is closed + raises an appropriate error rather than crashing. + """ + conn = connect(conn_str) + cursor = conn.cursor() + cursor.execute("SELECT 1") + cursor.fetchone() + + # Close connection + conn.close() + + # Attempting to execute on cursor should raise an error, not crash + with pytest.raises((DatabaseError, OperationalError)): + cursor.execute("SELECT 2") + + # GC should not crash + del cursor + gc.collect() + + +def test_multiple_connections_concurrent_invalidation(conn_str): + """ + Test that multiple connections can be invalidated concurrently + without interfering with each other's handle cleanup. + """ + connections = [] + all_cursors = [] + + # Create multiple connections with cursors + for conn_num in range(5): + conn = connect(conn_str) + connections.append(conn) + + for cursor_num in range(3): + cursor = conn.cursor() + cursor.execute(f"SELECT {conn_num} AS conn, {cursor_num} AS cursor") + cursor.fetchone() + all_cursors.append(cursor) + + # Close all connections + for conn in connections: + conn.close() + + # Clear references and force GC + connections = None + all_cursors = None + gc.collect() + + assert True + + +def test_connection_invalidation_with_prepared_statements(conn_str): + """ + Test connection invalidation when cursors have prepared statements. + This ensures statement handles are properly marked as implicitly freed. + """ + conn = connect(conn_str) + + # Create cursor with parameterized query (prepared statement) + cursor = conn.cursor() + cursor.execute("SELECT ? AS value", (42,)) + result = cursor.fetchone() + assert result[0] == 42 + + # Execute another parameterized query + cursor.execute("SELECT ? AS name, ? AS age", ("John", 30)) + result = cursor.fetchone() + assert result[0] == "John" + assert result[1] == 30 + + # Close connection with prepared statements + conn.close() + + # GC should handle cleanup without crash + del cursor + gc.collect() + + assert True + + +def test_verify_markImplicitlyFreed_method_exists(): + """ + Verify that the markImplicitlyFreed method exists on SqlHandle. + This is the core of the segfault fix. + """ + from mssql_python import ddbc_bindings + + # Verify the method exists + assert hasattr( + ddbc_bindings.SqlHandle, "markImplicitlyFreed" + ), "SqlHandle should have markImplicitlyFreed method" + + # Verify free method also exists + assert hasattr(ddbc_bindings.SqlHandle, "free"), "SqlHandle should have free method" + + +def test_connection_invalidation_with_fetchall(conn_str): + """ + Test connection invalidation when cursors have fetched all results. + This ensures all statement handle states are properly cleaned up. + """ + conn = connect(conn_str) + + cursor = conn.cursor() + cursor.execute("SELECT number FROM (VALUES (1), (2), (3), (4), (5)) AS numbers(number)") + results = cursor.fetchall() + assert len(results) == 5 + + # Close connection after fetchall + conn.close() + + # GC cleanup should work without issues + del cursor + gc.collect() + + assert True + + +def test_nested_connection_cursor_cleanup(conn_str): + """ + Test nested connection/cursor creation and cleanup pattern. + This mimics complex application patterns where connections + and cursors are created in nested scopes. + """ + + def inner_function(connection): + cursor = connection.cursor() + cursor.execute("SELECT 'inner' AS scope") + return cursor.fetchone() + + def outer_function(conn_str): + conn = connect(conn_str) + result = inner_function(conn) + conn.close() + return result + + # Run multiple times to ensure no accumulated state issues + for _ in range(5): + result = outer_function(conn_str) + assert result[0] == "inner" + gc.collect() + + # Final cleanup + gc.collect() + assert True + + +if __name__ == "__main__": + # Allow running this test file directly for quick verification + import sys + + if len(sys.argv) > 1: + conn_str = sys.argv[1] + print("Running connection invalidation segfault tests...") + + test_connection_invalidation_with_multiple_cursors(conn_str) + print("✓ test_connection_invalidation_with_multiple_cursors passed") + + test_connection_invalidation_without_cursor_close(conn_str) + print("✓ test_connection_invalidation_without_cursor_close passed") + + test_repeated_connection_invalidation_cycles(conn_str) + print("✓ test_repeated_connection_invalidation_cycles passed") + + test_connection_close_with_uncommitted_transaction(conn_str) + print("✓ test_connection_close_with_uncommitted_transaction passed") + + test_cursor_after_connection_invalidation_raises_error(conn_str) + print("✓ test_cursor_after_connection_invalidation_raises_error passed") + + test_multiple_connections_concurrent_invalidation(conn_str) + print("✓ test_multiple_connections_concurrent_invalidation passed") + + test_connection_invalidation_with_prepared_statements(conn_str) + print("✓ test_connection_invalidation_with_prepared_statements passed") + + test_verify_markImplicitlyFreed_method_exists() + print("✓ test_verify_markImplicitlyFreed_method_exists passed") + + test_connection_invalidation_with_fetchall(conn_str) + print("✓ test_connection_invalidation_with_fetchall passed") + + test_nested_connection_cursor_cleanup(conn_str) + print("✓ test_nested_connection_cursor_cleanup passed") + + print("\n✓✓✓ All connection invalidation segfault tests passed! ✓✓✓") + else: + print("Usage: python test_016_connection_invalidation_segfault.py ") + print("Or run with pytest: pytest test_016_connection_invalidation_segfault.py") From b532a63a653e255111a1e4c37f5306dbe83613e5 Mon Sep 17 00:00:00 2001 From: Subrata Paitandi Date: Fri, 30 Jan 2026 07:01:56 +0000 Subject: [PATCH 03/10] mssql_python/pybind/connection/connection.h --- mssql_python/pybind/ddbc_bindings.cpp | 2 +- ...st_016_connection_invalidation_segfault.py | 57 +++---------------- 2 files changed, 9 insertions(+), 50 deletions(-) diff --git a/mssql_python/pybind/ddbc_bindings.cpp b/mssql_python/pybind/ddbc_bindings.cpp index 52b9d21ee..0f262fd37 100644 --- a/mssql_python/pybind/ddbc_bindings.cpp +++ b/mssql_python/pybind/ddbc_bindings.cpp @@ -4365,7 +4365,7 @@ PYBIND11_MODULE(ddbc_bindings, m) { py::class_(m, "SqlHandle") .def("free", &SqlHandle::free, "Free the handle") - .def("markImplicitlyFreed", &SqlHandle::markImplicitlyFreed, + .def("mark_implicitly_freed", &SqlHandle::markImplicitlyFreed, "Mark handle as implicitly freed by parent handle"); py::class_(m, "Connection") diff --git a/tests/test_016_connection_invalidation_segfault.py b/tests/test_016_connection_invalidation_segfault.py index b92826aa6..3d509ecde 100644 --- a/tests/test_016_connection_invalidation_segfault.py +++ b/tests/test_016_connection_invalidation_segfault.py @@ -195,7 +195,7 @@ def test_multiple_connections_concurrent_invalidation(conn_str): for cursor_num in range(3): cursor = conn.cursor() - cursor.execute(f"SELECT {conn_num} AS conn, {cursor_num} AS cursor") + cursor.execute(f"SELECT {conn_num} AS conn, {cursor_num} AS cursor_num") cursor.fetchone() all_cursors.append(cursor) @@ -203,6 +203,9 @@ def test_multiple_connections_concurrent_invalidation(conn_str): for conn in connections: conn.close() + # Verify we have cursors alive (keep them referenced until after connection close) + assert len(all_cursors) == 15 # 5 connections * 3 cursors each + # Clear references and force GC connections = None all_cursors = None @@ -240,17 +243,17 @@ def test_connection_invalidation_with_prepared_statements(conn_str): assert True -def test_verify_markImplicitlyFreed_method_exists(): +def test_verify_mark_implicitly_freed_method_exists(): """ - Verify that the markImplicitlyFreed method exists on SqlHandle. + Verify that the mark_implicitly_freed method exists on SqlHandle. This is the core of the segfault fix. """ from mssql_python import ddbc_bindings # Verify the method exists assert hasattr( - ddbc_bindings.SqlHandle, "markImplicitlyFreed" - ), "SqlHandle should have markImplicitlyFreed method" + ddbc_bindings.SqlHandle, "mark_implicitly_freed" + ), "SqlHandle should have mark_implicitly_freed method" # Verify free method also exists assert hasattr(ddbc_bindings.SqlHandle, "free"), "SqlHandle should have free method" @@ -305,47 +308,3 @@ def outer_function(conn_str): # Final cleanup gc.collect() assert True - - -if __name__ == "__main__": - # Allow running this test file directly for quick verification - import sys - - if len(sys.argv) > 1: - conn_str = sys.argv[1] - print("Running connection invalidation segfault tests...") - - test_connection_invalidation_with_multiple_cursors(conn_str) - print("✓ test_connection_invalidation_with_multiple_cursors passed") - - test_connection_invalidation_without_cursor_close(conn_str) - print("✓ test_connection_invalidation_without_cursor_close passed") - - test_repeated_connection_invalidation_cycles(conn_str) - print("✓ test_repeated_connection_invalidation_cycles passed") - - test_connection_close_with_uncommitted_transaction(conn_str) - print("✓ test_connection_close_with_uncommitted_transaction passed") - - test_cursor_after_connection_invalidation_raises_error(conn_str) - print("✓ test_cursor_after_connection_invalidation_raises_error passed") - - test_multiple_connections_concurrent_invalidation(conn_str) - print("✓ test_multiple_connections_concurrent_invalidation passed") - - test_connection_invalidation_with_prepared_statements(conn_str) - print("✓ test_connection_invalidation_with_prepared_statements passed") - - test_verify_markImplicitlyFreed_method_exists() - print("✓ test_verify_markImplicitlyFreed_method_exists passed") - - test_connection_invalidation_with_fetchall(conn_str) - print("✓ test_connection_invalidation_with_fetchall passed") - - test_nested_connection_cursor_cleanup(conn_str) - print("✓ test_nested_connection_cursor_cleanup passed") - - print("\n✓✓✓ All connection invalidation segfault tests passed! ✓✓✓") - else: - print("Usage: python test_016_connection_invalidation_segfault.py ") - print("Or run with pytest: pytest test_016_connection_invalidation_segfault.py") From 245a75d606e1e1fddd7076aec131f1c2761570bb Mon Sep 17 00:00:00 2001 From: Subrata Paitandi Date: Fri, 30 Jan 2026 07:02:38 +0000 Subject: [PATCH 04/10] mssql_python/pybind/connection/connection.h --- SEGFAULT_REPRODUCTION_REPORT.md | 81 ++++++ WINDOWS_TEST_INSTRUCTIONS.md | 259 ++++++++++++++++++++ docker-scripts/entrypoint.sh | 79 ++++++ docker-scripts/entrypoint_sqlalchemy.sh | 130 ++++++++++ docker-scripts/mssql_setup.sh | 11 + docker-scripts/mssql_setup.sql | 21 ++ mssql_python/pybind/connection/connection.h | 1 + run_sqlalchemy_segfault_test.sh | 86 +++++++ run_windows_test.sh | 160 ++++++++++++ setup_and_run_windows_test.sh | 225 +++++++++++++++++ test_windows_segfault.ps1 | 171 +++++++++++++ test_windows_segfault_simple.py | 192 +++++++++++++++ 12 files changed, 1416 insertions(+) create mode 100644 SEGFAULT_REPRODUCTION_REPORT.md create mode 100644 WINDOWS_TEST_INSTRUCTIONS.md create mode 100644 docker-scripts/entrypoint.sh create mode 100644 docker-scripts/entrypoint_sqlalchemy.sh create mode 100644 docker-scripts/mssql_setup.sh create mode 100644 docker-scripts/mssql_setup.sql create mode 100644 run_sqlalchemy_segfault_test.sh create mode 100755 run_windows_test.sh create mode 100755 setup_and_run_windows_test.sh create mode 100644 test_windows_segfault.ps1 create mode 100644 test_windows_segfault_simple.py diff --git a/SEGFAULT_REPRODUCTION_REPORT.md b/SEGFAULT_REPRODUCTION_REPORT.md new file mode 100644 index 000000000..b1a6ca822 --- /dev/null +++ b/SEGFAULT_REPRODUCTION_REPORT.md @@ -0,0 +1,81 @@ +# SQLAlchemy Segfault Reproduction - Success + +## Summary +Successfully reproduced the segmentation fault issue that occurs when using the PyPI version of `mssql-python` with SQLAlchemy's reconnect tests. + +## Test Setup +- **Docker Image**: `mssql-sqlalchemy-segfault:clean` +- **Base OS**: Fedora 39 +- **SQL Server**: 2022 (RTM-CU22-GDR) 16.0.4230.2 +- **Python**: 3.12.7 +- **mssql-python**: Installed from PyPI (version without fix) +- **SQLAlchemy**: 2.1.0b1.dev0 (from Gerrit review 6149) + +## Test Execution +The Docker container: +1. Started SQL Server 2022 +2. Created test database and user +3. Cloned SQLAlchemy from gerrit.sqlalchemy.org +4. Fetched specific review (refs/changes/49/6149/14) containing reconnect tests +5. Installed SQLAlchemy and dependencies +6. Ran `test/engine/test_reconnect.py::RealReconnectTest` with mssql-python + +## Result +**SEGFAULT REPRODUCED ON FIRST TEST RUN** + +``` +test/engine/test_reconnect.py::RealReconnectTest_mssql+mssqlpython_16_0_4230_2::test_close PASSED +[tests continued...] + +Segmentation fault (core dumped) +*** CRASH DETECTED ON RUN 1 *** +``` + +## Root Cause +The segfault is caused by a **use-after-free bug** in mssql-python: + +### The Problem: +1. SQLAlchemy creates connections with multiple active cursors/statement handles +2. When connection invalidation occurs (e.g., during reconnect tests), the connection is closed +3. Closing the connection frees the DBC (database connection) handle +4. ODBC driver automatically frees all child STMT (statement) handles when DBC is freed +5. Python garbage collector later tries to free those same STMT handles +6. **Result**: Use-after-free → Segmentation Fault + +### Technical Details: +- When `Connection::disconnect()` frees a DBC handle, ODBC spec dictates that all child statement handles are implicitly freed +- The Python `SqlHandle` objects weren't aware of this implicit freeing +- When Python's GC ran and called `SqlHandle::free()` on those handles, it attempted to call ODBC functions on already-freed handles +- This caused memory corruption and crashes + +## The Fix +The fix (already implemented in the local workspace) uses state tracking: +- `Connection` maintains a `_childStatementHandles` vector +- Before disconnecting, `Connection` marks all child handles as "implicitly freed" +- `SqlHandle::free()` checks the flag and skips ODBC calls if the handle was already freed +- Result: Clean shutdown, no crashes + +## Files Created +1. **Dockerfile.sqlalchemy-segfault** - Clean Dockerfile for reproduction +2. **docker-scripts/entrypoint.sh** - Main test execution script +3. **docker-scripts/mssql_setup.sh** - SQL Server initialization +4. **docker-scripts/mssql_setup.sql** - Database setup SQL +5. **sqlalchemy-segfault-test.log** - Complete test execution log + +## How to Reproduce +```bash +# Build the Docker image +docker build -f Dockerfile.sqlalchemy-segfault -t mssql-sqlalchemy-segfault:clean . + +# Run the test (will crash with segfault) +docker run --rm mssql-sqlalchemy-segfault:clean +``` + +## Next Steps +To verify the fix works: +1. Use `Dockerfile.test-fix` which installs the local version with the fix +2. Run the same test suite +3. Confirm it completes 10 runs without crashing + +## Log Location +Full execution log: `/home/subrata/SegFaultRepro/mssql-python/sqlalchemy-segfault-test.log` diff --git a/WINDOWS_TEST_INSTRUCTIONS.md b/WINDOWS_TEST_INSTRUCTIONS.md new file mode 100644 index 000000000..0256685bb --- /dev/null +++ b/WINDOWS_TEST_INSTRUCTIONS.md @@ -0,0 +1,259 @@ +# Windows Segfault Fix - Testing Instructions + +## Prerequisites + +1. **Windows OS** (tested on Windows 10/11) +2. **Python 3.12 or 3.13** installed +3. **Visual Studio 2019 or later** with C++ build tools +4. **SQL Server** (LocalDB, Express, or full) running locally or accessible +5. **ODBC Driver 18 for SQL Server** installed + +## Setup Steps + +### 1. Install Build Dependencies + +Open PowerShell as Administrator and install required tools: + +```powershell +# Install Python (if not already installed) +# Download from: https://www.python.org/downloads/ + +# Install Visual Studio Build Tools +# Download from: https://visualstudio.microsoft.com/downloads/ +# Select "Desktop development with C++" workload + +# Install ODBC Driver 18 +# Download from: https://learn.microsoft.com/en-us/sql/connect/odbc/download-odbc-driver-for-sql-server +``` + +### 2. Clone and Navigate to Repository + +```powershell +cd C:\Users\YourUsername +git clone https://github.com/your-repo/mssql-python.git +cd mssql-python +git checkout subrata-ms/GCSegFault # The branch with the fix +``` + +### 3. Build the C++ Extension with Fix + +```powershell +# Navigate to pybind directory +cd mssql_python\pybind + +# Build for your architecture (x64, x86, or arm64) +.\build.bat x64 + +# Return to root +cd ..\.. +``` + +### 4. Install the Package + +```powershell +# Install in development mode +pip install -e . + +# Install testing dependencies +pip install pytest sqlalchemy greenlet typing-extensions +``` + +### 5. Verify Installation + +```powershell +python -c "import mssql_python; print(f'Version: {mssql_python.__version__}'); from mssql_python.ddbc_bindings import Connection; print('C++ extension loaded!')" +``` + +Expected output: +``` +Version: 1.2.0 +C++ extension loaded! +``` + +## Running Tests + +### Option 1: Simple Python Test (Recommended for Quick Validation) + +1. **Update connection string** in `test_windows_segfault_simple.py`: + ```python + CONN_STR = "Driver={ODBC Driver 18 for SQL Server};Server=localhost;Database=test;UID=sa;PWD=YourPassword;Encrypt=No" + ``` + +2. **Run the test**: + ```powershell + python test_windows_segfault_simple.py + ``` + +3. **Expected output**: + ``` + ============================================================ + SUCCESS: No segfault detected! + The fix is working correctly. + ============================================================ + ``` + +### Option 2: PowerShell Script (Full Test Suite) + +```powershell +# Run with your SQL Server credentials +.\test_windows_segfault.ps1 -SqlServer "localhost" -Database "test" -Username "sa" -Password "YourPassword" -TestRuns 10 +``` + +### Option 3: Run Local Unit Tests + +```powershell +# Run the connection invalidation test +pytest tests\test_016_connection_invalidation_segfault.py -v +``` + +### Option 4: Full SQLAlchemy Reconnect Tests + +If you want to reproduce the exact scenario from the stack trace: + +```powershell +# Clone SQLAlchemy (if not already done) +cd .. +git clone https://github.com/sqlalchemy/sqlalchemy.git +cd sqlalchemy + +# Fetch the specific gerrit review with reconnect tests +git fetch https://gerrit.sqlalchemy.org/sqlalchemy/sqlalchemy refs/changes/49/6149/14 +git checkout FETCH_HEAD + +# Install SQLAlchemy +pip install -e . + +# Run the reconnect tests +pytest test\engine\test_reconnect.py::RealReconnectTest --dburi "mssql+mssqlpython://sa:YourPassword@localhost/test?Encrypt=No" --disable-asyncio -v +``` + +## What the Tests Validate + +### 1. **Cursor Destruction During Initialization** +- Tests the exact scenario from the stack trace +- Creates cursors and triggers GC during object initialization +- Ensures `__del__` doesn't cause segfaults when called prematurely + +### 2. **Connection Invalidation** +- Closes connections without explicitly closing cursors +- Mimics SQLAlchemy's connection pool behavior +- Verifies child handles are properly marked as freed + +### 3. **Threading Scenario** +- Tests cursor cleanup during lock creation (RLock scenario) +- Ensures thread-safety of the fix +- Validates no deadlocks occur + +### 4. **Multiple Iterations** +- Runs 10+ iterations to ensure stability +- Catches intermittent issues +- Validates fix works consistently + +## Expected Results + +### ✅ Success Indicators +- All tests complete without crashes +- No segmentation faults +- No "Access Violation" errors +- Clean exit codes (0) + +### ❌ Failure Indicators +- Python crashes with exit code +- "Access Violation" exceptions +- Tests hanging indefinitely +- Segmentation fault errors + +## Troubleshooting + +### Build Errors + +**Error: "CMake not found"** +```powershell +# Install CMake +pip install cmake +``` + +**Error: "Visual Studio not found"** +- Install Visual Studio Build Tools with C++ support +- Or set environment variable: `$env:VS_PATH = "C:\Path\To\VS"` + +**Error: "Python.h not found"** +- Ensure Python development headers are installed +- Reinstall Python with "Include development tools" option + +### Runtime Errors + +**Error: "ODBC Driver not found"** +```powershell +# Check installed ODBC drivers +Get-OdbcDriver +``` + +**Error: "Cannot connect to SQL Server"** +- Verify SQL Server is running: `Get-Service MSSQL*` +- Check firewall settings +- Test connection: `sqlcmd -S localhost -U sa -P YourPassword` + +**Error: "Import failed: DLL load failed"** +- Ensure Visual C++ Redistributable is installed +- Check PATH includes Python and ODBC driver directories + +## Comparing Before/After + +### Test WITHOUT Fix (PyPI Version) + +```powershell +# Install PyPI version +pip uninstall mssql-python -y +pip install mssql-python==1.2.0 + +# Run test - should crash +python test_windows_segfault_simple.py +``` + +Expected: **Crash/Segfault** + +### Test WITH Fix (Your Build) + +```powershell +# Install your fixed version +pip uninstall mssql-python -y +cd C:\path\to\mssql-python +pip install -e . + +# Run test - should succeed +python test_windows_segfault_simple.py +``` + +Expected: **Success** + +## Reporting Results + +When reporting test results, include: + +1. **Environment**: + - Windows version + - Python version (`python --version`) + - ODBC Driver version + - SQL Server version + +2. **Test Output**: + - Full console output + - Any error messages + - Stack traces (if crash occurs) + +3. **Build Info**: + - Architecture (x64/x86/arm64) + - Compiler version + - Build warnings/errors + +4. **Comparison**: + - Results with PyPI version (should fail) + - Results with fixed version (should pass) + +## Additional Resources + +- ODBC Driver Download: https://learn.microsoft.com/en-us/sql/connect/odbc/download-odbc-driver-for-sql-server +- Python Windows Installation: https://www.python.org/downloads/windows/ +- Visual Studio Build Tools: https://visualstudio.microsoft.com/downloads/ +- SQL Server Express: https://www.microsoft.com/en-us/sql-server/sql-server-downloads diff --git a/docker-scripts/entrypoint.sh b/docker-scripts/entrypoint.sh new file mode 100644 index 000000000..d416db733 --- /dev/null +++ b/docker-scripts/entrypoint.sh @@ -0,0 +1,79 @@ +#!/bin/bash +set -e +set -x +export ACCEPT_EULA='Y' +export MSSQL_SA_PASSWORD='wh0_CAR;ES!!' +export MSSQL_PID=Developer + +# Start SQL Server in the background +/opt/mssql/bin/sqlservr & PID=$! + +# Wait for SQL Server to be ready +echo "Waiting for SQL Server to start..." +sleep 30s + +# Clone SQLAlchemy from gerrit +echo "" +echo "=========================================" +echo "Cloning SQLAlchemy from gerrit..." +echo "=========================================" +cd / +git clone https://gerrit.sqlalchemy.org/sqlalchemy/sqlalchemy + +cd /sqlalchemy + +# Fetch the specific review +echo "" +echo "Fetching gerrit review 6149..." +git fetch https://gerrit.sqlalchemy.org/sqlalchemy/sqlalchemy refs/changes/49/6149/14 +git checkout FETCH_HEAD + +# Install SQLAlchemy and test dependencies +echo "" +echo "Installing SQLAlchemy and dependencies..." +pip3 install -e . +pip3 install pytest greenlet typing-extensions + +# Run the test suite multiple times to trigger the segfault +echo "" +echo "=========================================" +echo "Testing PyPI version (expected to SEGFAULT)" +echo "=========================================" +echo "This will run the test suite in a loop until it crashes or completes 10 runs." +echo "" + +CRASH_DETECTED=0 +for i in {1..10}; do + echo "" + echo "=== Test Run $i ===" + + if ! pytest test/engine/test_reconnect.py::RealReconnectTest \ + --dburi "mssql+mssqlpython://scott:tiger^5HHH@localhost:1433/test?Encrypt=No" \ + --disable-asyncio -s -v 2>&1; then + + echo "" + echo "*** CRASH DETECTED ON RUN $i ***" + CRASH_DETECTED=1 + break + fi + + echo "Run $i completed successfully" + sleep 1 +done + +if [ $CRASH_DETECTED -eq 0 ]; then + echo "" + echo "*** No crash in 10 runs. The issue is intermittent and may require more runs. ***" +else + echo "" + echo "=========================================" + echo "*** SEGFAULT REPRODUCED! ***" + echo "This demonstrates the use-after-free bug" + echo "=========================================" +fi + +# Shut down SQL Server cleanly +echo "" +echo "Shutting down SQL Server..." +kill ${PID} 2>/dev/null || true +sleep 5s diff --git a/docker-scripts/entrypoint_sqlalchemy.sh b/docker-scripts/entrypoint_sqlalchemy.sh new file mode 100644 index 000000000..7dfa2ba07 --- /dev/null +++ b/docker-scripts/entrypoint_sqlalchemy.sh @@ -0,0 +1,130 @@ +#!/bin/bash +set -e + +echo "==========================================" +echo "Starting SQL Server..." +echo "==========================================" + +# Start SQL Server +/opt/mssql/bin/sqlservr & +SQLSERVER_PID=$! + +# Wait for SQL Server to be ready +echo "Waiting for SQL Server to start..." +for i in {1..60}; do + if /opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P 'wh0_CAR;ES!!' -Q "SELECT 1" &>/dev/null; then + echo "SQL Server is ready!" + break + fi + echo "Waiting... ($i/60)" + sleep 2 +done + +# Create test database +echo "" +echo "==========================================" +echo "Setting up test database..." +echo "==========================================" +/opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P 'wh0_CAR;ES!!' -Q " +IF NOT EXISTS(SELECT * FROM sys.databases WHERE name = 'test') +BEGIN + CREATE DATABASE test +END +" || true + +echo "" +echo "==========================================" +echo "Installing SQLAlchemy from Gerrit..." +echo "==========================================" + +cd /tmp + +# Clone SQLAlchemy if not already present +if [ ! -d "sqlalchemy" ]; then + git clone https://github.com/sqlalchemy/sqlalchemy.git +fi + +cd sqlalchemy + +# Fetch and checkout the specific gerrit change with mssql-python support +echo "Fetching gerrit change 6149/14..." +git fetch https://gerrit.sqlalchemy.org/sqlalchemy/sqlalchemy refs/changes/49/6149/14 +git checkout FETCH_HEAD + +# Install SQLAlchemy +pip3 install -e . + +SA_VERSION=$(python3 -c "import sqlalchemy; print(sqlalchemy.__version__)") +echo "SQLAlchemy $SA_VERSION installed" + +echo "" +echo "==========================================" +echo "Running SQLAlchemy RealReconnectTest..." +echo "==========================================" +echo "" + +# Connection URI for SQLAlchemy +CONN_URI="mssql+mssqlpython://sa:wh0_CAR;ES!!@localhost/test?Encrypt=No" + +# Run the reconnect tests multiple times to check for segfaults +ITERATIONS=${TEST_ITERATIONS:-10} +echo "Running $ITERATIONS test iterations..." +echo "" + +PASS_COUNT=0 +FAIL_COUNT=0 +CRASH_DETECTED=0 + +for ((i=1; i<=ITERATIONS; i++)); do + echo " Iteration $i/$ITERATIONS..." + + if pytest test/engine/test_reconnect.py::RealReconnectTest \ + --dburi "$CONN_URI" \ + --disable-asyncio \ + -v \ + --tb=short 2>&1; then + echo " PASS" + ((PASS_COUNT++)) + else + EXIT_CODE=$? + echo " FAIL (exit code: $EXIT_CODE)" + ((FAIL_COUNT++)) + + # Check if it was a segfault (exit code 139 or 'Segmentation fault' message) + if [ $EXIT_CODE -eq 139 ] || [ $EXIT_CODE -eq 11 ]; then + echo " *** SEGFAULT DETECTED ***" + CRASH_DETECTED=1 + break + fi + fi + + sleep 1 +done + +echo "" +echo "==========================================" +echo "Test Results:" +echo " Passed: $PASS_COUNT / $ITERATIONS" +echo " Failed: $FAIL_COUNT / $ITERATIONS" +echo "==========================================" +echo "" + +if [ $CRASH_DETECTED -eq 1 ]; then + echo "*** SEGFAULT DETECTED - FIX DID NOT WORK ***" + echo "==========================================" + kill $SQLSERVER_PID 2>/dev/null || true + exit 1 +elif [ $FAIL_COUNT -eq 0 ]; then + echo "*** SUCCESS! No segfaults in $ITERATIONS runs! ***" + echo "The fix is working correctly!" + echo "==========================================" + kill $SQLSERVER_PID 2>/dev/null || true + exit 0 +else + echo "*** PARTIAL SUCCESS ***" + echo "No segfaults detected, but some test failures occurred" + echo "The fix prevents crashes!" + echo "==========================================" + kill $SQLSERVER_PID 2>/dev/null || true + exit 0 +fi diff --git a/docker-scripts/mssql_setup.sh b/docker-scripts/mssql_setup.sh new file mode 100644 index 000000000..0a82f4249 --- /dev/null +++ b/docker-scripts/mssql_setup.sh @@ -0,0 +1,11 @@ +#!/bin/bash +set -x +export ACCEPT_EULA='Y' +export MSSQL_SA_PASSWORD='wh0_CAR;ES!!' +export MSSQL_PID=Developer +/opt/mssql/bin/mssql-conf set memory.memorylimitmb 4096 +/opt/mssql/bin/sqlservr & PID=$! +sleep 30s +/opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P "${MSSQL_SA_PASSWORD}" -d master -i /mssql_setup.sql +kill ${PID} +sleep 10s diff --git a/docker-scripts/mssql_setup.sql b/docker-scripts/mssql_setup.sql new file mode 100644 index 000000000..6e897481b --- /dev/null +++ b/docker-scripts/mssql_setup.sql @@ -0,0 +1,21 @@ +sp_configure 'show advanced options', 1; +GO +RECONFIGURE; +go +sp_configure 'max server memory', 4096; +go +sp_configure 'max worker threads', 512; +go +RECONFIGURE; +go +CREATE DATABASE test; +go +CREATE LOGIN scott with password='tiger^5HHH'; +GRANT CONTROL SERVER TO scott; +USE test; +go +CREATE SCHEMA test_schema; +go +ALTER DATABASE test SET ALLOW_SNAPSHOT_ISOLATION ON +ALTER DATABASE test SET READ_COMMITTED_SNAPSHOT ON +go diff --git a/mssql_python/pybind/connection/connection.h b/mssql_python/pybind/connection/connection.h index 4bda21a08..4b6f068bd 100644 --- a/mssql_python/pybind/connection/connection.h +++ b/mssql_python/pybind/connection/connection.h @@ -5,6 +5,7 @@ #include "../ddbc_bindings.h" #include #include +#include // Represents a single ODBC database connection. // Manages connection handles. diff --git a/run_sqlalchemy_segfault_test.sh b/run_sqlalchemy_segfault_test.sh new file mode 100644 index 000000000..5d02b0b72 --- /dev/null +++ b/run_sqlalchemy_segfault_test.sh @@ -0,0 +1,86 @@ +#!/bin/bash + +# Script to reproduce the segfault issue with SQLAlchemy +# This script will: +# 1. Clone SQLAlchemy from gerrit +# 2. Fetch the specific review that includes the reconnect test +# 3. Install SQLAlchemy and dependencies +# 4. Run the reconnect test multiple times to trigger the segfault + +set -e +set -x + +WORK_DIR="/tmp/sqlalchemy-segfault-test" +DB_URI="mssql+mssqlpython://scott:tiger^5HHH@localhost:1433/test?Encrypt=No" + +echo "========================================" +echo "SQLAlchemy Segfault Reproduction Test" +echo "========================================" +echo "" +echo "This test will attempt to reproduce a segfault that occurs" +echo "when SQLAlchemy's connection pool invalidates connections" +echo "with active cursors/statement handles." +echo "" + +# Clean up any previous test directory +rm -rf "$WORK_DIR" +mkdir -p "$WORK_DIR" +cd "$WORK_DIR" + +# Clone SQLAlchemy from gerrit +echo "Cloning SQLAlchemy from gerrit..." +git clone https://gerrit.sqlalchemy.org/sqlalchemy/sqlalchemy + +cd sqlalchemy + +# Fetch the specific review that includes the reconnect test +echo "Fetching gerrit review 6149..." +git fetch https://gerrit.sqlalchemy.org/sqlalchemy/sqlalchemy refs/changes/49/6149/14 +git checkout FETCH_HEAD + +# Install SQLAlchemy and test dependencies +echo "Installing SQLAlchemy and dependencies..." +pip3 install --user -e . +pip3 install --user pytest greenlet typing-extensions + +# Run the test suite multiple times to trigger the segfault +echo "" +echo "========================================" +echo "Running reconnect tests to reproduce segfault..." +echo "========================================" +echo "This will run the test suite in a loop until it crashes or completes 10 runs." +echo "" + +CRASH_DETECTED=0 +for i in {1..10}; do + echo "" + echo "=== Test Run $i ===" + + if ! pytest test/engine/test_reconnect.py::RealReconnectTest \ + --dburi "$DB_URI" \ + --disable-asyncio -s -v 2>&1; then + + echo "" + echo "*** CRASH DETECTED ON RUN $i ***" + CRASH_DETECTED=1 + break + fi + + echo "Run $i completed successfully" + sleep 1 +done + +echo "" +if [ $CRASH_DETECTED -eq 0 ]; then + echo "========================================" + echo "No crash detected in 10 runs." + echo "The issue may be intermittent or already fixed." + echo "========================================" +else + echo "========================================" + echo "*** SEGFAULT REPRODUCED! ***" + echo "========================================" +fi + +echo "" +echo "Test directory: $WORK_DIR" diff --git a/run_windows_test.sh b/run_windows_test.sh new file mode 100755 index 000000000..141abe950 --- /dev/null +++ b/run_windows_test.sh @@ -0,0 +1,160 @@ +#!/bin/bash +# Helper script to run Windows tests from WSL + +set -e + +echo "==========================================" +echo "Launching Windows Test from WSL" +echo "==========================================" +echo "" + +# Convert current path to Windows path +WINDOWS_PATH=$(wslpath -w "$PWD") +echo "Windows Path: $WINDOWS_PATH" +echo "" + +# Check if Python is installed on Windows +echo "Checking Windows Python installation..." +if ! powershell.exe -Command "python --version" 2>/dev/null; then + echo "" + echo "ERROR: Python not found in Windows PATH" + echo "Please install Python on Windows: https://www.python.org/downloads/" + exit 1 +fi + +PYTHON_VERSION=$(powershell.exe -Command "python --version 2>&1" | tr -d '\r') +echo "Found: $PYTHON_VERSION" +echo "" + +# Ask user for connection details +echo "==========================================" +echo "SQL Server Connection Details" +echo "==========================================" +read -p "SQL Server (default: localhost): " SQL_SERVER +SQL_SERVER=${SQL_SERVER:-localhost} + +read -p "Database (default: test): " DATABASE +DATABASE=${DATABASE:-test} + +read -p "Username (default: sa): " USERNAME +USERNAME=${USERNAME:-sa} + +read -sp "Password: " PASSWORD +echo "" + +read -p "Number of test runs (default: 10): " TEST_RUNS +TEST_RUNS=${TEST_RUNS:-10} + +echo "" +echo "==========================================" +echo "Starting Windows Build and Test" +echo "==========================================" +echo "" + +# Build the C++ extension on Windows +echo "Step 1: Building C++ extension on Windows..." +echo "This requires Visual Studio with C++ tools installed" +echo "" + +powershell.exe -NoProfile -ExecutionPolicy Bypass -Command " + cd '$WINDOWS_PATH' + Write-Host 'Current directory:' (Get-Location) -ForegroundColor Cyan + Write-Host '' + + # Check for Visual Studio + Write-Host 'Checking for Visual Studio...' -ForegroundColor Yellow + if (-not (Test-Path 'C:\Program Files\Microsoft Visual Studio')) { + Write-Host 'WARNING: Visual Studio not found in default location' -ForegroundColor Red + Write-Host 'Build may fail without Visual Studio C++ tools' -ForegroundColor Red + Write-Host '' + } + + # Build the extension + Write-Host 'Building C++ extension...' -ForegroundColor Yellow + cd mssql_python\pybind + + if (-not (Test-Path 'build.bat')) { + Write-Host 'ERROR: build.bat not found!' -ForegroundColor Red + exit 1 + } + + & cmd /c 'build.bat x64 2>&1' + if (\$LASTEXITCODE -ne 0) { + Write-Host 'ERROR: Build failed!' -ForegroundColor Red + Write-Host '' + Write-Host 'Troubleshooting:' -ForegroundColor Yellow + Write-Host '1. Install Visual Studio 2019+ with C++ build tools' -ForegroundColor Gray + Write-Host '2. Open Visual Studio Developer Command Prompt' -ForegroundColor Gray + Write-Host '3. Run: cd \"$WINDOWS_PATH\mssql_python\pybind\" && build.bat x64' -ForegroundColor Gray + exit 1 + } + + cd ..\.. + Write-Host '✓ Build completed successfully' -ForegroundColor Green + Write-Host '' + + # Install the package + Write-Host 'Step 2: Installing package in development mode...' -ForegroundColor Yellow + python -m pip install -e . 2>&1 | Out-Null + if (\$LASTEXITCODE -ne 0) { + Write-Host 'ERROR: Installation failed!' -ForegroundColor Red + exit 1 + } + Write-Host '✓ Package installed' -ForegroundColor Green + Write-Host '' + + # Verify extension loads + Write-Host 'Step 3: Verifying C++ extension...' -ForegroundColor Yellow + python -c 'import mssql_python; print(f\"Version: {mssql_python.__version__}\"); from mssql_python.ddbc_bindings import Connection; print(\"✓ C++ extension loaded\")' + if (\$LASTEXITCODE -ne 0) { + Write-Host 'ERROR: Extension verification failed!' -ForegroundColor Red + exit 1 + } + Write-Host '' + + # Install test dependencies + Write-Host 'Step 4: Installing test dependencies...' -ForegroundColor Yellow + python -m pip install pytest sqlalchemy greenlet typing-extensions 2>&1 | Out-Null + Write-Host '✓ Dependencies installed' -ForegroundColor Green + Write-Host '' + + # Update connection string in test file + Write-Host 'Step 5: Updating connection string...' -ForegroundColor Yellow + \$connStr = \"Driver={ODBC Driver 18 for SQL Server};Server=$SQL_SERVER;Database=$DATABASE;UID=$USERNAME;PWD=$PASSWORD;Encrypt=No\" + \$testFile = Get-Content 'test_windows_segfault_simple.py' -Raw + \$testFile = \$testFile -replace 'CONN_STR = \".*?\"', \"CONN_STR = \`\"\$connStr\`\"\" + \$testFile | Set-Content 'test_windows_segfault_simple.py' -NoNewline + Write-Host '✓ Connection string updated' -ForegroundColor Green + Write-Host '' + + # Run the simple test + Write-Host 'Step 6: Running Windows segfault test...' -ForegroundColor Yellow + Write-Host '========================================' -ForegroundColor Cyan + python test_windows_segfault_simple.py + \$testResult = \$LASTEXITCODE + Write-Host '========================================' -ForegroundColor Cyan + Write-Host '' + + if (\$testResult -eq 0) { + Write-Host '✓✓✓ ALL TESTS PASSED! ✓✓✓' -ForegroundColor Green + Write-Host 'The fix is working correctly on Windows.' -ForegroundColor Green + } else { + Write-Host '✗✗✗ TESTS FAILED! ✗✗✗' -ForegroundColor Red + Write-Host 'The segfault issue still exists.' -ForegroundColor Red + } + + exit \$testResult +" + +EXIT_CODE=$? + +echo "" +echo "==========================================" +if [ $EXIT_CODE -eq 0 ]; then + echo "✓ Windows test completed successfully!" +else + echo "✗ Windows test failed with exit code: $EXIT_CODE" +fi +echo "==========================================" + +exit $EXIT_CODE diff --git a/setup_and_run_windows_test.sh b/setup_and_run_windows_test.sh new file mode 100755 index 000000000..f6d3c6ac4 --- /dev/null +++ b/setup_and_run_windows_test.sh @@ -0,0 +1,225 @@ +#!/bin/bash +# Copy repository to Windows and run tests there + +set -e + +echo "==========================================" +echo "Windows Test Setup from WSL" +echo "==========================================" +echo "" + +# Get Windows directory +echo "We need to copy the repository to a Windows-accessible directory" +echo "because Windows CMD doesn't support UNC paths (\\\\wsl.localhost\\...)" +echo "" +read -p "Windows directory path (e.g., C:\\Temp\\mssql-test): " WIN_DIR + +if [ -z "$WIN_DIR" ]; then + WIN_DIR="C:\\Temp\\mssql-python-test" + echo "Using default: $WIN_DIR" +fi + +# Convert to Windows format if needed +WIN_DIR=$(echo "$WIN_DIR" | sed 's/\//\\/g') + +# Convert to Linux path for copying +LINUX_PATH=$(wslpath "$WIN_DIR" 2>/dev/null || echo "") +if [ -z "$LINUX_PATH" ]; then + echo "ERROR: Could not convert Windows path to Linux path" + exit 1 +fi + +echo "" +echo "Linux path: $LINUX_PATH" +echo "Windows path: $WIN_DIR" +echo "" + +# Create directory and copy files +echo "Copying repository to Windows..." +mkdir -p "$LINUX_PATH" + +# Copy necessary files +rsync -av --progress \ + --exclude='.git' \ + --exclude='__pycache__' \ + --exclude='*.pyc' \ + --exclude='.pytest_cache' \ + --exclude='build' \ + --exclude='*.egg-info' \ + --exclude='.venv' \ + --exclude='venv' \ + ./ "$LINUX_PATH/" + +echo "" +echo "✓ Files copied successfully" +echo "" + +# Get connection details +echo "==========================================" +echo "SQL Server Connection Details" +echo "==========================================" +read -p "SQL Server (default: localhost): " SQL_SERVER +SQL_SERVER=${SQL_SERVER:-localhost} + +read -p "Database (default: test): " DATABASE +DATABASE=${DATABASE:-test} + +read -p "Username (default: sa): " USERNAME +USERNAME=${USERNAME:-sa} + +read -sp "Password: " PASSWORD +echo "" +echo "" + +# Create a PowerShell script to run in Windows +cat > "$LINUX_PATH/run_test_windows.ps1" << 'PSEOF' +param( + [string]$SqlServer = "localhost", + [string]$Database = "test", + [string]$Username = "sa", + [string]$Password = "" +) + +$ErrorActionPreference = "Stop" + +Write-Host "==========================================" -ForegroundColor Cyan +Write-Host "Windows C++ Build and Test" -ForegroundColor Cyan +Write-Host "==========================================" -ForegroundColor Cyan +Write-Host "" +Write-Host "Python: $(python --version)" -ForegroundColor Green +Write-Host "Location: $PWD" -ForegroundColor Gray +Write-Host "" + +# Check for Visual Studio +Write-Host "Step 1: Checking build environment..." -ForegroundColor Yellow +$vsPath = & "${env:ProgramFiles(x86)}\Microsoft Visual Studio\Installer\vswhere.exe" -latest -property installationPath 2>$null +if ($vsPath) { + Write-Host " ✓ Visual Studio found: $vsPath" -ForegroundColor Green +} else { + Write-Host " ⚠ Visual Studio not detected" -ForegroundColor Yellow + Write-Host " Build may fail without Visual Studio C++ tools" -ForegroundColor Yellow +} +Write-Host "" + +# Build C++ extension +Write-Host "Step 2: Building C++ extension..." -ForegroundColor Yellow +Push-Location "mssql_python\pybind" + +if (-not (Test-Path "build.bat")) { + Write-Host " ERROR: build.bat not found!" -ForegroundColor Red + Pop-Location + exit 1 +} + +Write-Host " Running: build.bat x64" -ForegroundColor Gray +$buildOutput = & cmd /c "build.bat x64 2>&1" +$buildExit = $LASTEXITCODE + +if ($buildExit -ne 0) { + Write-Host "" + Write-Host " ✗ Build failed!" -ForegroundColor Red + Write-Host "" + Write-Host "Build output:" -ForegroundColor Yellow + Write-Host $buildOutput + Write-Host "" + Write-Host "Troubleshooting:" -ForegroundColor Yellow + Write-Host " 1. Open 'Developer Command Prompt for VS'" -ForegroundColor Gray + Write-Host " 2. Navigate to: $PWD" -ForegroundColor Gray + Write-Host " 3. Run: cd mssql_python\pybind && build.bat x64" -ForegroundColor Gray + Pop-Location + exit 1 +} + +Pop-Location +Write-Host " ✓ C++ extension built successfully" -ForegroundColor Green +Write-Host "" + +# Install package +Write-Host "Step 3: Installing package..." -ForegroundColor Yellow +$installOutput = python -m pip install -e . 2>&1 +if ($LASTEXITCODE -ne 0) { + Write-Host " ✗ Installation failed!" -ForegroundColor Red + Write-Host $installOutput + exit 1 +} +Write-Host " ✓ Package installed" -ForegroundColor Green +Write-Host "" + +# Verify extension +Write-Host "Step 4: Verifying C++ extension..." -ForegroundColor Yellow +$verifyOutput = python -c "import mssql_python; print(f' Version: {mssql_python.__version__}'); from mssql_python.ddbc_bindings import Connection; print(' ✓ Extension loaded')" 2>&1 +if ($LASTEXITCODE -ne 0) { + Write-Host " ✗ Verification failed!" -ForegroundColor Red + Write-Host $verifyOutput + exit 1 +} +Write-Host $verifyOutput +Write-Host "" + +# Install dependencies +Write-Host "Step 5: Installing test dependencies..." -ForegroundColor Yellow +python -m pip install -q pytest sqlalchemy greenlet typing-extensions +Write-Host " ✓ Dependencies installed" -ForegroundColor Green +Write-Host "" + +# Update connection string +Write-Host "Step 6: Configuring connection..." -ForegroundColor Yellow +$connStr = "Driver={ODBC Driver 18 for SQL Server};Server=$SqlServer;Database=$Database;UID=$Username;PWD=$Password;Encrypt=No" +Write-Host " Server: $SqlServer/$Database" -ForegroundColor Gray + +$testFile = Get-Content 'test_windows_segfault_simple.py' -Raw +$testFile = $testFile -replace 'CONN_STR = ".*?"', "CONN_STR = `"$connStr`"" +$testFile | Set-Content 'test_windows_segfault_simple.py' -NoNewline +Write-Host " ✓ Connection configured" -ForegroundColor Green +Write-Host "" + +# Run test +Write-Host "Step 7: Running segfault test..." -ForegroundColor Yellow +Write-Host "==========================================" -ForegroundColor Cyan +Write-Host "" + +python test_windows_segfault_simple.py +$testExit = $LASTEXITCODE + +Write-Host "" +Write-Host "==========================================" -ForegroundColor Cyan +if ($testExit -eq 0) { + Write-Host "✓✓✓ SUCCESS! All tests passed! ✓✓✓" -ForegroundColor Green + Write-Host "The segfault fix is working on Windows." -ForegroundColor Green +} else { + Write-Host "✗✗✗ FAILURE! Tests failed! ✗✗✗" -ForegroundColor Red + Write-Host "The issue may still exist." -ForegroundColor Red +} +Write-Host "==========================================" -ForegroundColor Cyan + +exit $testExit +PSEOF + +echo "==========================================" +echo "Starting Windows test..." +echo "==========================================" +echo "" + +# Run PowerShell script +powershell.exe -NoProfile -ExecutionPolicy Bypass -File "$(wslpath -w "$LINUX_PATH/run_test_windows.ps1")" -SqlServer "$SQL_SERVER" -Database "$DATABASE" -Username "$USERNAME" -Password "$PASSWORD" + +EXIT_CODE=$? + +echo "" +echo "==========================================" +if [ $EXIT_CODE -eq 0 ]; then + echo "✓ Windows test completed successfully!" + echo "" + echo "The C++ fix is working correctly on Windows!" +else + echo "✗ Windows test failed with exit code: $EXIT_CODE" +fi +echo "==========================================" +echo "" +echo "Files are located at: $WIN_DIR" +echo "You can also run tests manually from Windows:" +echo " 1. Open PowerShell" +echo " 2. cd $WIN_DIR" +echo " 3. .\\run_test_windows.ps1 -SqlServer '$SQL_SERVER' -Database '$DATABASE' -Username '$USERNAME' -Password 'yourpass'" + +exit $EXIT_CODE diff --git a/test_windows_segfault.ps1 b/test_windows_segfault.ps1 new file mode 100644 index 000000000..30fd5650a --- /dev/null +++ b/test_windows_segfault.ps1 @@ -0,0 +1,171 @@ +# PowerShell script to reproduce and test the segfault fix on Windows +# Run this from the mssql-python repository root + +param( + [string]$SqlServer = "localhost", + [string]$Database = "test", + [string]$Username = "sa", + [string]$Password = "", + [int]$TestRuns = 10 +) + +Write-Host "========================================" -ForegroundColor Cyan +Write-Host "Windows Segfault Reproduction Test" -ForegroundColor Cyan +Write-Host "========================================" -ForegroundColor Cyan +Write-Host "" + +# Check Python version +$pythonVersion = python --version 2>&1 +Write-Host "Python Version: $pythonVersion" -ForegroundColor Green +Write-Host "" + +# Step 1: Build the C++ extension with the fix +Write-Host "Step 1: Building C++ extension with fix..." -ForegroundColor Yellow +Write-Host "-------------------------------------" -ForegroundColor Yellow + +if (Test-Path "mssql_python\pybind\build") { + Write-Host "Cleaning existing build directory..." -ForegroundColor Gray + Remove-Item -Path "mssql_python\pybind\build" -Recurse -Force -ErrorAction SilentlyContinue +} + +Push-Location "mssql_python\pybind" +Write-Host "Running build.bat..." -ForegroundColor Gray +$buildOutput = & cmd /c "build.bat" 2>&1 +if ($LASTEXITCODE -ne 0) { + Write-Host "ERROR: Build failed!" -ForegroundColor Red + Write-Host $buildOutput + Pop-Location + exit 1 +} +Write-Host "✓ C++ extension built successfully" -ForegroundColor Green +Pop-Location +Write-Host "" + +# Step 2: Install the package in development mode +Write-Host "Step 2: Installing mssql-python with fix..." -ForegroundColor Yellow +Write-Host "-------------------------------------" -ForegroundColor Yellow +$installOutput = python -m pip install -e . 2>&1 +if ($LASTEXITCODE -ne 0) { + Write-Host "ERROR: Installation failed!" -ForegroundColor Red + Write-Host $installOutput + exit 1 +} +Write-Host "✓ Package installed successfully" -ForegroundColor Green +Write-Host "" + +# Step 3: Verify the C++ extension loads +Write-Host "Step 3: Verifying C++ extension..." -ForegroundColor Yellow +Write-Host "-------------------------------------" -ForegroundColor Yellow +$verifyScript = @" +import mssql_python +print('✓ mssql_python imported') +from mssql_python.ddbc_bindings import Connection +print('✓ C++ extension loaded successfully') +import sys +print(f'Python: {sys.version}') +print(f'mssql_python: {mssql_python.__version__}') +"@ + +$verifyOutput = python -c $verifyScript 2>&1 +if ($LASTEXITCODE -ne 0) { + Write-Host "ERROR: Extension verification failed!" -ForegroundColor Red + Write-Host $verifyOutput + exit 1 +} +Write-Host $verifyOutput +Write-Host "" + +# Step 4: Check if SQLAlchemy is available +Write-Host "Step 4: Checking SQLAlchemy installation..." -ForegroundColor Yellow +Write-Host "-------------------------------------" -ForegroundColor Yellow +$sqlalchemyCheck = python -c "import sqlalchemy; print(f'SQLAlchemy version: {sqlalchemy.__version__}')" 2>&1 +if ($LASTEXITCODE -ne 0) { + Write-Host "SQLAlchemy not found. Installing..." -ForegroundColor Yellow + python -m pip install sqlalchemy pytest greenlet typing-extensions 2>&1 | Out-Null + Write-Host "✓ SQLAlchemy installed" -ForegroundColor Green +} else { + Write-Host $sqlalchemyCheck -ForegroundColor Green +} +Write-Host "" + +# Step 5: Run the connection invalidation test +Write-Host "Step 5: Running connection invalidation test..." -ForegroundColor Yellow +Write-Host "-------------------------------------" -ForegroundColor Yellow + +if ($Password -eq "") { + Write-Host "ERROR: SQL Server password not provided!" -ForegroundColor Red + Write-Host "Usage: .\test_windows_segfault.ps1 -SqlServer 'localhost' -Database 'test' -Username 'sa' -Password 'YourPassword'" -ForegroundColor Yellow + Write-Host "" + Write-Host "Or set connection string and run basic test:" -ForegroundColor Yellow + Write-Host " `$env:TEST_CONN_STR = 'mssql+mssqlpython://user:pass@localhost/test'" -ForegroundColor Yellow + exit 1 +} + +$connStr = "mssql+mssqlpython://${Username}:${Password}@${SqlServer}/${Database}?Encrypt=No" +Write-Host "Connection: mssql+mssqlpython://${Username}:***@${SqlServer}/${Database}" -ForegroundColor Gray +Write-Host "" + +# Run the local test if it exists +if (Test-Path "tests\test_016_connection_invalidation_segfault.py") { + Write-Host "Running local connection invalidation test..." -ForegroundColor Cyan + $testResult = pytest tests\test_016_connection_invalidation_segfault.py -v --tb=short 2>&1 + if ($LASTEXITCODE -ne 0) { + Write-Host "Test execution encountered issues:" -ForegroundColor Yellow + Write-Host $testResult + } else { + Write-Host "✓ Local tests passed!" -ForegroundColor Green + } + Write-Host "" +} + +# Step 6: Run multiple iterations of SQLAlchemy reconnect tests +Write-Host "Step 6: Running SQLAlchemy reconnect tests (${TestRuns} iterations)..." -ForegroundColor Yellow +Write-Host "-------------------------------------" -ForegroundColor Yellow + +# Check if SQLAlchemy test directory exists +if (-not (Test-Path "..\sqlalchemy\test\engine\test_reconnect.py")) { + Write-Host "SQLAlchemy test suite not found." -ForegroundColor Yellow + Write-Host "To run full SQLAlchemy tests:" -ForegroundColor Yellow + Write-Host " 1. Clone SQLAlchemy: git clone https://github.com/sqlalchemy/sqlalchemy.git" -ForegroundColor Gray + Write-Host " 2. Install it: cd sqlalchemy && pip install -e ." -ForegroundColor Gray + Write-Host " 3. Run: pytest test\engine\test_reconnect.py::RealReconnectTest --dburi '$connStr' -v" -ForegroundColor Gray + Write-Host "" +} else { + Write-Host "Running RealReconnectTest from SQLAlchemy..." -ForegroundColor Cyan + + $crashDetected = $false + for ($i = 1; $i -le $TestRuns; $i++) { + Write-Host " Test Run $i/$TestRuns..." -ForegroundColor Gray + + Push-Location "..\sqlalchemy" + $testOutput = pytest test\engine\test_reconnect.py::RealReconnectTest --dburi $connStr --disable-asyncio -v 2>&1 + $exitCode = $LASTEXITCODE + Pop-Location + + if ($exitCode -ne 0) { + Write-Host "" + Write-Host " ✗ CRASH DETECTED ON RUN $i" -ForegroundColor Red + Write-Host $testOutput | Select-String -Pattern "Error|Exception|Segmentation" + $crashDetected = $true + break + } + + Write-Host " ✓ Run $i completed successfully" -ForegroundColor Green + Start-Sleep -Seconds 1 + } + + Write-Host "" + if (-not $crashDetected) { + Write-Host "========================================" -ForegroundColor Green + Write-Host "SUCCESS! No crash in $TestRuns runs." -ForegroundColor Green + Write-Host "The fix appears to be working!" -ForegroundColor Green + Write-Host "========================================" -ForegroundColor Green + } else { + Write-Host "========================================" -ForegroundColor Red + Write-Host "FIX DID NOT WORK - Segfault occurred!" -ForegroundColor Red + Write-Host "========================================" -ForegroundColor Red + } +} + +Write-Host "" +Write-Host "Test completed." -ForegroundColor Cyan diff --git a/test_windows_segfault_simple.py b/test_windows_segfault_simple.py new file mode 100644 index 000000000..0bcac23c7 --- /dev/null +++ b/test_windows_segfault_simple.py @@ -0,0 +1,192 @@ +""" +Simple Python script to reproduce the Windows segfault issue with the fix. +This mimics the stack trace scenario: cursor destruction during object creation. + +Run: python test_windows_segfault_simple.py +""" + +import gc +import sys +import threading +from mssql_python import connect, DatabaseError, OperationalError + +# Connection string - update with your SQL Server details +CONN_STR = "Driver={ODBC Driver 18 for SQL Server};Server=localhost;Database=test;UID=sa;PWD=YourPassword;Encrypt=No" + + +def test_cursor_destruction_during_init(): + """ + Test that mimics the stack trace where __del__ is called during object initialization. + This tests the fix for premature cursor destruction. + """ + print("=" * 60) + print("Test: Cursor Destruction During Initialization") + print("=" * 60) + + try: + conn = connect(CONN_STR) + print("✓ Connection established") + + # Create multiple cursors and trigger GC during initialization + cursors = [] + for i in range(5): + cursor = conn.cursor() + cursor.execute("SELECT 1 AS test_col") + cursor.fetchall() + cursors.append(cursor) + + # Force garbage collection to trigger __del__ on unreferenced objects + gc.collect() + print(f" Cursor {i+1} created and GC triggered") + + # Close connection WITHOUT explicitly closing cursors first + # This simulates the invalidation scenario + print("\n✓ Closing connection without closing cursors...") + conn.close() + + # Force garbage collection to trigger cursor cleanup + print("✓ Triggering garbage collection...") + cursors = None + gc.collect() + gc.collect() # Call twice to ensure cleanup + + print("\n" + "=" * 60) + print("SUCCESS: No segfault detected!") + print("The fix is working correctly.") + print("=" * 60) + return True + + except Exception as e: + print("\n" + "=" * 60) + print(f"ERROR: {type(e).__name__}: {e}") + print("=" * 60) + import traceback + traceback.print_exc() + return False + + +def test_connection_invalidation_multiple_times(iterations=10): + """ + Run the connection invalidation test multiple times to ensure stability. + """ + print("\n" + "=" * 60) + print(f"Running Connection Invalidation Test ({iterations} iterations)") + print("=" * 60) + + failures = 0 + for i in range(iterations): + print(f"\nIteration {i+1}/{iterations}...", end=" ") + try: + conn = connect(CONN_STR) + cursor = conn.cursor() + cursor.execute("SELECT 1") + cursor.fetchone() + conn.close() + gc.collect() + print("✓ PASS") + except Exception as e: + print(f"✗ FAIL: {e}") + failures += 1 + + print("\n" + "=" * 60) + if failures == 0: + print(f"✓ All {iterations} iterations passed!") + print("The fix is stable across multiple runs.") + else: + print(f"✗ {failures}/{iterations} iterations failed") + print("=" * 60) + + return failures == 0 + + +def test_threading_scenario(): + """ + Test that mimics the threading.RLock scenario from the stack trace. + This ensures the fix works when cursors are destroyed during lock creation. + """ + print("\n" + "=" * 60) + print("Test: Threading + Cursor Destruction") + print("=" * 60) + + def worker(): + try: + conn = connect(CONN_STR) + cursor = conn.cursor() + cursor.execute("SELECT 1") + cursor.fetchone() + conn.close() + gc.collect() + return True + except Exception as e: + print(f" Thread error: {e}") + return False + + threads = [] + for i in range(5): + t = threading.Thread(target=worker) + threads.append(t) + t.start() + print(f" Thread {i+1} started") + + print("\n Waiting for threads to complete...") + for t in threads: + t.join() + + print("\n" + "=" * 60) + print("✓ Threading test completed without crashes") + print("=" * 60) + + return True + + +def main(): + """Main test runner""" + print("\n" + "=" * 60) + print("Windows Segfault Fix - Validation Tests") + print("=" * 60) + print(f"Python version: {sys.version}") + print(f"Platform: {sys.platform}") + print("=" * 60) + print("\nIMPORTANT: Update CONN_STR in the script with your SQL Server details") + print("=" * 60) + + try: + # Import and print version info + import mssql_python + print(f"\nmssql_python version: {mssql_python.__version__}") + from mssql_python.ddbc_bindings import Connection + print("✓ C++ extension loaded successfully") + except Exception as e: + print(f"\n✗ Failed to load mssql_python: {e}") + print("\nMake sure to build and install the package first:") + print(" 1. cd mssql_python\\pybind") + print(" 2. build.bat") + print(" 3. cd ..\\..") + print(" 4. pip install -e .") + return False + + # Run tests + print("\n") + test1_passed = test_cursor_destruction_during_init() + + if test1_passed: + test2_passed = test_connection_invalidation_multiple_times(10) + test3_passed = test_threading_scenario() + + if test1_passed and test2_passed and test3_passed: + print("\n" + "=" * 60) + print("ALL TESTS PASSED!") + print("The segfault fix is working correctly on Windows.") + print("=" * 60) + return True + + print("\n" + "=" * 60) + print("SOME TESTS FAILED") + print("Please review the errors above.") + print("=" * 60) + return False + + +if __name__ == "__main__": + success = main() + sys.exit(0 if success else 1) From 97c009d1d89155266fab1436b2aa0282146ee526 Mon Sep 17 00:00:00 2001 From: Subrata Paitandi Date: Fri, 30 Jan 2026 07:09:43 +0000 Subject: [PATCH 05/10] fixed issues --- mssql_python/pybind/connection/connection.h | 1 - 1 file changed, 1 deletion(-) diff --git a/mssql_python/pybind/connection/connection.h b/mssql_python/pybind/connection/connection.h index 4b6f068bd..4bda21a08 100644 --- a/mssql_python/pybind/connection/connection.h +++ b/mssql_python/pybind/connection/connection.h @@ -5,7 +5,6 @@ #include "../ddbc_bindings.h" #include #include -#include // Represents a single ODBC database connection. // Manages connection handles. From 498dfef29a7d084f6ed89a60a9908715fb17e0bb Mon Sep 17 00:00:00 2001 From: Subrata Paitandi Date: Fri, 30 Jan 2026 07:20:05 +0000 Subject: [PATCH 06/10] cleanning the unnecessary files --- SEGFAULT_REPRODUCTION_REPORT.md | 81 -------- WINDOWS_TEST_INSTRUCTIONS.md | 259 ------------------------ docker-scripts/entrypoint.sh | 79 -------- docker-scripts/entrypoint_sqlalchemy.sh | 130 ------------ docker-scripts/mssql_setup.sh | 11 - docker-scripts/mssql_setup.sql | 21 -- run_sqlalchemy_segfault_test.sh | 86 -------- run_windows_test.sh | 160 --------------- setup_and_run_windows_test.sh | 225 -------------------- test_windows_segfault.ps1 | 171 ---------------- test_windows_segfault_simple.py | 192 ------------------ 11 files changed, 1415 deletions(-) delete mode 100644 SEGFAULT_REPRODUCTION_REPORT.md delete mode 100644 WINDOWS_TEST_INSTRUCTIONS.md delete mode 100644 docker-scripts/entrypoint.sh delete mode 100644 docker-scripts/entrypoint_sqlalchemy.sh delete mode 100644 docker-scripts/mssql_setup.sh delete mode 100644 docker-scripts/mssql_setup.sql delete mode 100644 run_sqlalchemy_segfault_test.sh delete mode 100755 run_windows_test.sh delete mode 100755 setup_and_run_windows_test.sh delete mode 100644 test_windows_segfault.ps1 delete mode 100644 test_windows_segfault_simple.py diff --git a/SEGFAULT_REPRODUCTION_REPORT.md b/SEGFAULT_REPRODUCTION_REPORT.md deleted file mode 100644 index b1a6ca822..000000000 --- a/SEGFAULT_REPRODUCTION_REPORT.md +++ /dev/null @@ -1,81 +0,0 @@ -# SQLAlchemy Segfault Reproduction - Success - -## Summary -Successfully reproduced the segmentation fault issue that occurs when using the PyPI version of `mssql-python` with SQLAlchemy's reconnect tests. - -## Test Setup -- **Docker Image**: `mssql-sqlalchemy-segfault:clean` -- **Base OS**: Fedora 39 -- **SQL Server**: 2022 (RTM-CU22-GDR) 16.0.4230.2 -- **Python**: 3.12.7 -- **mssql-python**: Installed from PyPI (version without fix) -- **SQLAlchemy**: 2.1.0b1.dev0 (from Gerrit review 6149) - -## Test Execution -The Docker container: -1. Started SQL Server 2022 -2. Created test database and user -3. Cloned SQLAlchemy from gerrit.sqlalchemy.org -4. Fetched specific review (refs/changes/49/6149/14) containing reconnect tests -5. Installed SQLAlchemy and dependencies -6. Ran `test/engine/test_reconnect.py::RealReconnectTest` with mssql-python - -## Result -**SEGFAULT REPRODUCED ON FIRST TEST RUN** - -``` -test/engine/test_reconnect.py::RealReconnectTest_mssql+mssqlpython_16_0_4230_2::test_close PASSED -[tests continued...] - -Segmentation fault (core dumped) -*** CRASH DETECTED ON RUN 1 *** -``` - -## Root Cause -The segfault is caused by a **use-after-free bug** in mssql-python: - -### The Problem: -1. SQLAlchemy creates connections with multiple active cursors/statement handles -2. When connection invalidation occurs (e.g., during reconnect tests), the connection is closed -3. Closing the connection frees the DBC (database connection) handle -4. ODBC driver automatically frees all child STMT (statement) handles when DBC is freed -5. Python garbage collector later tries to free those same STMT handles -6. **Result**: Use-after-free → Segmentation Fault - -### Technical Details: -- When `Connection::disconnect()` frees a DBC handle, ODBC spec dictates that all child statement handles are implicitly freed -- The Python `SqlHandle` objects weren't aware of this implicit freeing -- When Python's GC ran and called `SqlHandle::free()` on those handles, it attempted to call ODBC functions on already-freed handles -- This caused memory corruption and crashes - -## The Fix -The fix (already implemented in the local workspace) uses state tracking: -- `Connection` maintains a `_childStatementHandles` vector -- Before disconnecting, `Connection` marks all child handles as "implicitly freed" -- `SqlHandle::free()` checks the flag and skips ODBC calls if the handle was already freed -- Result: Clean shutdown, no crashes - -## Files Created -1. **Dockerfile.sqlalchemy-segfault** - Clean Dockerfile for reproduction -2. **docker-scripts/entrypoint.sh** - Main test execution script -3. **docker-scripts/mssql_setup.sh** - SQL Server initialization -4. **docker-scripts/mssql_setup.sql** - Database setup SQL -5. **sqlalchemy-segfault-test.log** - Complete test execution log - -## How to Reproduce -```bash -# Build the Docker image -docker build -f Dockerfile.sqlalchemy-segfault -t mssql-sqlalchemy-segfault:clean . - -# Run the test (will crash with segfault) -docker run --rm mssql-sqlalchemy-segfault:clean -``` - -## Next Steps -To verify the fix works: -1. Use `Dockerfile.test-fix` which installs the local version with the fix -2. Run the same test suite -3. Confirm it completes 10 runs without crashing - -## Log Location -Full execution log: `/home/subrata/SegFaultRepro/mssql-python/sqlalchemy-segfault-test.log` diff --git a/WINDOWS_TEST_INSTRUCTIONS.md b/WINDOWS_TEST_INSTRUCTIONS.md deleted file mode 100644 index 0256685bb..000000000 --- a/WINDOWS_TEST_INSTRUCTIONS.md +++ /dev/null @@ -1,259 +0,0 @@ -# Windows Segfault Fix - Testing Instructions - -## Prerequisites - -1. **Windows OS** (tested on Windows 10/11) -2. **Python 3.12 or 3.13** installed -3. **Visual Studio 2019 or later** with C++ build tools -4. **SQL Server** (LocalDB, Express, or full) running locally or accessible -5. **ODBC Driver 18 for SQL Server** installed - -## Setup Steps - -### 1. Install Build Dependencies - -Open PowerShell as Administrator and install required tools: - -```powershell -# Install Python (if not already installed) -# Download from: https://www.python.org/downloads/ - -# Install Visual Studio Build Tools -# Download from: https://visualstudio.microsoft.com/downloads/ -# Select "Desktop development with C++" workload - -# Install ODBC Driver 18 -# Download from: https://learn.microsoft.com/en-us/sql/connect/odbc/download-odbc-driver-for-sql-server -``` - -### 2. Clone and Navigate to Repository - -```powershell -cd C:\Users\YourUsername -git clone https://github.com/your-repo/mssql-python.git -cd mssql-python -git checkout subrata-ms/GCSegFault # The branch with the fix -``` - -### 3. Build the C++ Extension with Fix - -```powershell -# Navigate to pybind directory -cd mssql_python\pybind - -# Build for your architecture (x64, x86, or arm64) -.\build.bat x64 - -# Return to root -cd ..\.. -``` - -### 4. Install the Package - -```powershell -# Install in development mode -pip install -e . - -# Install testing dependencies -pip install pytest sqlalchemy greenlet typing-extensions -``` - -### 5. Verify Installation - -```powershell -python -c "import mssql_python; print(f'Version: {mssql_python.__version__}'); from mssql_python.ddbc_bindings import Connection; print('C++ extension loaded!')" -``` - -Expected output: -``` -Version: 1.2.0 -C++ extension loaded! -``` - -## Running Tests - -### Option 1: Simple Python Test (Recommended for Quick Validation) - -1. **Update connection string** in `test_windows_segfault_simple.py`: - ```python - CONN_STR = "Driver={ODBC Driver 18 for SQL Server};Server=localhost;Database=test;UID=sa;PWD=YourPassword;Encrypt=No" - ``` - -2. **Run the test**: - ```powershell - python test_windows_segfault_simple.py - ``` - -3. **Expected output**: - ``` - ============================================================ - SUCCESS: No segfault detected! - The fix is working correctly. - ============================================================ - ``` - -### Option 2: PowerShell Script (Full Test Suite) - -```powershell -# Run with your SQL Server credentials -.\test_windows_segfault.ps1 -SqlServer "localhost" -Database "test" -Username "sa" -Password "YourPassword" -TestRuns 10 -``` - -### Option 3: Run Local Unit Tests - -```powershell -# Run the connection invalidation test -pytest tests\test_016_connection_invalidation_segfault.py -v -``` - -### Option 4: Full SQLAlchemy Reconnect Tests - -If you want to reproduce the exact scenario from the stack trace: - -```powershell -# Clone SQLAlchemy (if not already done) -cd .. -git clone https://github.com/sqlalchemy/sqlalchemy.git -cd sqlalchemy - -# Fetch the specific gerrit review with reconnect tests -git fetch https://gerrit.sqlalchemy.org/sqlalchemy/sqlalchemy refs/changes/49/6149/14 -git checkout FETCH_HEAD - -# Install SQLAlchemy -pip install -e . - -# Run the reconnect tests -pytest test\engine\test_reconnect.py::RealReconnectTest --dburi "mssql+mssqlpython://sa:YourPassword@localhost/test?Encrypt=No" --disable-asyncio -v -``` - -## What the Tests Validate - -### 1. **Cursor Destruction During Initialization** -- Tests the exact scenario from the stack trace -- Creates cursors and triggers GC during object initialization -- Ensures `__del__` doesn't cause segfaults when called prematurely - -### 2. **Connection Invalidation** -- Closes connections without explicitly closing cursors -- Mimics SQLAlchemy's connection pool behavior -- Verifies child handles are properly marked as freed - -### 3. **Threading Scenario** -- Tests cursor cleanup during lock creation (RLock scenario) -- Ensures thread-safety of the fix -- Validates no deadlocks occur - -### 4. **Multiple Iterations** -- Runs 10+ iterations to ensure stability -- Catches intermittent issues -- Validates fix works consistently - -## Expected Results - -### ✅ Success Indicators -- All tests complete without crashes -- No segmentation faults -- No "Access Violation" errors -- Clean exit codes (0) - -### ❌ Failure Indicators -- Python crashes with exit code -- "Access Violation" exceptions -- Tests hanging indefinitely -- Segmentation fault errors - -## Troubleshooting - -### Build Errors - -**Error: "CMake not found"** -```powershell -# Install CMake -pip install cmake -``` - -**Error: "Visual Studio not found"** -- Install Visual Studio Build Tools with C++ support -- Or set environment variable: `$env:VS_PATH = "C:\Path\To\VS"` - -**Error: "Python.h not found"** -- Ensure Python development headers are installed -- Reinstall Python with "Include development tools" option - -### Runtime Errors - -**Error: "ODBC Driver not found"** -```powershell -# Check installed ODBC drivers -Get-OdbcDriver -``` - -**Error: "Cannot connect to SQL Server"** -- Verify SQL Server is running: `Get-Service MSSQL*` -- Check firewall settings -- Test connection: `sqlcmd -S localhost -U sa -P YourPassword` - -**Error: "Import failed: DLL load failed"** -- Ensure Visual C++ Redistributable is installed -- Check PATH includes Python and ODBC driver directories - -## Comparing Before/After - -### Test WITHOUT Fix (PyPI Version) - -```powershell -# Install PyPI version -pip uninstall mssql-python -y -pip install mssql-python==1.2.0 - -# Run test - should crash -python test_windows_segfault_simple.py -``` - -Expected: **Crash/Segfault** - -### Test WITH Fix (Your Build) - -```powershell -# Install your fixed version -pip uninstall mssql-python -y -cd C:\path\to\mssql-python -pip install -e . - -# Run test - should succeed -python test_windows_segfault_simple.py -``` - -Expected: **Success** - -## Reporting Results - -When reporting test results, include: - -1. **Environment**: - - Windows version - - Python version (`python --version`) - - ODBC Driver version - - SQL Server version - -2. **Test Output**: - - Full console output - - Any error messages - - Stack traces (if crash occurs) - -3. **Build Info**: - - Architecture (x64/x86/arm64) - - Compiler version - - Build warnings/errors - -4. **Comparison**: - - Results with PyPI version (should fail) - - Results with fixed version (should pass) - -## Additional Resources - -- ODBC Driver Download: https://learn.microsoft.com/en-us/sql/connect/odbc/download-odbc-driver-for-sql-server -- Python Windows Installation: https://www.python.org/downloads/windows/ -- Visual Studio Build Tools: https://visualstudio.microsoft.com/downloads/ -- SQL Server Express: https://www.microsoft.com/en-us/sql-server/sql-server-downloads diff --git a/docker-scripts/entrypoint.sh b/docker-scripts/entrypoint.sh deleted file mode 100644 index d416db733..000000000 --- a/docker-scripts/entrypoint.sh +++ /dev/null @@ -1,79 +0,0 @@ -#!/bin/bash -set -e -set -x -export ACCEPT_EULA='Y' -export MSSQL_SA_PASSWORD='wh0_CAR;ES!!' -export MSSQL_PID=Developer - -# Start SQL Server in the background -/opt/mssql/bin/sqlservr & PID=$! - -# Wait for SQL Server to be ready -echo "Waiting for SQL Server to start..." -sleep 30s - -# Clone SQLAlchemy from gerrit -echo "" -echo "=========================================" -echo "Cloning SQLAlchemy from gerrit..." -echo "=========================================" -cd / -git clone https://gerrit.sqlalchemy.org/sqlalchemy/sqlalchemy - -cd /sqlalchemy - -# Fetch the specific review -echo "" -echo "Fetching gerrit review 6149..." -git fetch https://gerrit.sqlalchemy.org/sqlalchemy/sqlalchemy refs/changes/49/6149/14 -git checkout FETCH_HEAD - -# Install SQLAlchemy and test dependencies -echo "" -echo "Installing SQLAlchemy and dependencies..." -pip3 install -e . -pip3 install pytest greenlet typing-extensions - -# Run the test suite multiple times to trigger the segfault -echo "" -echo "=========================================" -echo "Testing PyPI version (expected to SEGFAULT)" -echo "=========================================" -echo "This will run the test suite in a loop until it crashes or completes 10 runs." -echo "" - -CRASH_DETECTED=0 -for i in {1..10}; do - echo "" - echo "=== Test Run $i ===" - - if ! pytest test/engine/test_reconnect.py::RealReconnectTest \ - --dburi "mssql+mssqlpython://scott:tiger^5HHH@localhost:1433/test?Encrypt=No" \ - --disable-asyncio -s -v 2>&1; then - - echo "" - echo "*** CRASH DETECTED ON RUN $i ***" - CRASH_DETECTED=1 - break - fi - - echo "Run $i completed successfully" - sleep 1 -done - -if [ $CRASH_DETECTED -eq 0 ]; then - echo "" - echo "*** No crash in 10 runs. The issue is intermittent and may require more runs. ***" -else - echo "" - echo "=========================================" - echo "*** SEGFAULT REPRODUCED! ***" - echo "This demonstrates the use-after-free bug" - echo "=========================================" -fi - -# Shut down SQL Server cleanly -echo "" -echo "Shutting down SQL Server..." -kill ${PID} 2>/dev/null || true -sleep 5s diff --git a/docker-scripts/entrypoint_sqlalchemy.sh b/docker-scripts/entrypoint_sqlalchemy.sh deleted file mode 100644 index 7dfa2ba07..000000000 --- a/docker-scripts/entrypoint_sqlalchemy.sh +++ /dev/null @@ -1,130 +0,0 @@ -#!/bin/bash -set -e - -echo "==========================================" -echo "Starting SQL Server..." -echo "==========================================" - -# Start SQL Server -/opt/mssql/bin/sqlservr & -SQLSERVER_PID=$! - -# Wait for SQL Server to be ready -echo "Waiting for SQL Server to start..." -for i in {1..60}; do - if /opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P 'wh0_CAR;ES!!' -Q "SELECT 1" &>/dev/null; then - echo "SQL Server is ready!" - break - fi - echo "Waiting... ($i/60)" - sleep 2 -done - -# Create test database -echo "" -echo "==========================================" -echo "Setting up test database..." -echo "==========================================" -/opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P 'wh0_CAR;ES!!' -Q " -IF NOT EXISTS(SELECT * FROM sys.databases WHERE name = 'test') -BEGIN - CREATE DATABASE test -END -" || true - -echo "" -echo "==========================================" -echo "Installing SQLAlchemy from Gerrit..." -echo "==========================================" - -cd /tmp - -# Clone SQLAlchemy if not already present -if [ ! -d "sqlalchemy" ]; then - git clone https://github.com/sqlalchemy/sqlalchemy.git -fi - -cd sqlalchemy - -# Fetch and checkout the specific gerrit change with mssql-python support -echo "Fetching gerrit change 6149/14..." -git fetch https://gerrit.sqlalchemy.org/sqlalchemy/sqlalchemy refs/changes/49/6149/14 -git checkout FETCH_HEAD - -# Install SQLAlchemy -pip3 install -e . - -SA_VERSION=$(python3 -c "import sqlalchemy; print(sqlalchemy.__version__)") -echo "SQLAlchemy $SA_VERSION installed" - -echo "" -echo "==========================================" -echo "Running SQLAlchemy RealReconnectTest..." -echo "==========================================" -echo "" - -# Connection URI for SQLAlchemy -CONN_URI="mssql+mssqlpython://sa:wh0_CAR;ES!!@localhost/test?Encrypt=No" - -# Run the reconnect tests multiple times to check for segfaults -ITERATIONS=${TEST_ITERATIONS:-10} -echo "Running $ITERATIONS test iterations..." -echo "" - -PASS_COUNT=0 -FAIL_COUNT=0 -CRASH_DETECTED=0 - -for ((i=1; i<=ITERATIONS; i++)); do - echo " Iteration $i/$ITERATIONS..." - - if pytest test/engine/test_reconnect.py::RealReconnectTest \ - --dburi "$CONN_URI" \ - --disable-asyncio \ - -v \ - --tb=short 2>&1; then - echo " PASS" - ((PASS_COUNT++)) - else - EXIT_CODE=$? - echo " FAIL (exit code: $EXIT_CODE)" - ((FAIL_COUNT++)) - - # Check if it was a segfault (exit code 139 or 'Segmentation fault' message) - if [ $EXIT_CODE -eq 139 ] || [ $EXIT_CODE -eq 11 ]; then - echo " *** SEGFAULT DETECTED ***" - CRASH_DETECTED=1 - break - fi - fi - - sleep 1 -done - -echo "" -echo "==========================================" -echo "Test Results:" -echo " Passed: $PASS_COUNT / $ITERATIONS" -echo " Failed: $FAIL_COUNT / $ITERATIONS" -echo "==========================================" -echo "" - -if [ $CRASH_DETECTED -eq 1 ]; then - echo "*** SEGFAULT DETECTED - FIX DID NOT WORK ***" - echo "==========================================" - kill $SQLSERVER_PID 2>/dev/null || true - exit 1 -elif [ $FAIL_COUNT -eq 0 ]; then - echo "*** SUCCESS! No segfaults in $ITERATIONS runs! ***" - echo "The fix is working correctly!" - echo "==========================================" - kill $SQLSERVER_PID 2>/dev/null || true - exit 0 -else - echo "*** PARTIAL SUCCESS ***" - echo "No segfaults detected, but some test failures occurred" - echo "The fix prevents crashes!" - echo "==========================================" - kill $SQLSERVER_PID 2>/dev/null || true - exit 0 -fi diff --git a/docker-scripts/mssql_setup.sh b/docker-scripts/mssql_setup.sh deleted file mode 100644 index 0a82f4249..000000000 --- a/docker-scripts/mssql_setup.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash -set -x -export ACCEPT_EULA='Y' -export MSSQL_SA_PASSWORD='wh0_CAR;ES!!' -export MSSQL_PID=Developer -/opt/mssql/bin/mssql-conf set memory.memorylimitmb 4096 -/opt/mssql/bin/sqlservr & PID=$! -sleep 30s -/opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P "${MSSQL_SA_PASSWORD}" -d master -i /mssql_setup.sql -kill ${PID} -sleep 10s diff --git a/docker-scripts/mssql_setup.sql b/docker-scripts/mssql_setup.sql deleted file mode 100644 index 6e897481b..000000000 --- a/docker-scripts/mssql_setup.sql +++ /dev/null @@ -1,21 +0,0 @@ -sp_configure 'show advanced options', 1; -GO -RECONFIGURE; -go -sp_configure 'max server memory', 4096; -go -sp_configure 'max worker threads', 512; -go -RECONFIGURE; -go -CREATE DATABASE test; -go -CREATE LOGIN scott with password='tiger^5HHH'; -GRANT CONTROL SERVER TO scott; -USE test; -go -CREATE SCHEMA test_schema; -go -ALTER DATABASE test SET ALLOW_SNAPSHOT_ISOLATION ON -ALTER DATABASE test SET READ_COMMITTED_SNAPSHOT ON -go diff --git a/run_sqlalchemy_segfault_test.sh b/run_sqlalchemy_segfault_test.sh deleted file mode 100644 index 5d02b0b72..000000000 --- a/run_sqlalchemy_segfault_test.sh +++ /dev/null @@ -1,86 +0,0 @@ -#!/bin/bash - -# Script to reproduce the segfault issue with SQLAlchemy -# This script will: -# 1. Clone SQLAlchemy from gerrit -# 2. Fetch the specific review that includes the reconnect test -# 3. Install SQLAlchemy and dependencies -# 4. Run the reconnect test multiple times to trigger the segfault - -set -e -set -x - -WORK_DIR="/tmp/sqlalchemy-segfault-test" -DB_URI="mssql+mssqlpython://scott:tiger^5HHH@localhost:1433/test?Encrypt=No" - -echo "========================================" -echo "SQLAlchemy Segfault Reproduction Test" -echo "========================================" -echo "" -echo "This test will attempt to reproduce a segfault that occurs" -echo "when SQLAlchemy's connection pool invalidates connections" -echo "with active cursors/statement handles." -echo "" - -# Clean up any previous test directory -rm -rf "$WORK_DIR" -mkdir -p "$WORK_DIR" -cd "$WORK_DIR" - -# Clone SQLAlchemy from gerrit -echo "Cloning SQLAlchemy from gerrit..." -git clone https://gerrit.sqlalchemy.org/sqlalchemy/sqlalchemy - -cd sqlalchemy - -# Fetch the specific review that includes the reconnect test -echo "Fetching gerrit review 6149..." -git fetch https://gerrit.sqlalchemy.org/sqlalchemy/sqlalchemy refs/changes/49/6149/14 -git checkout FETCH_HEAD - -# Install SQLAlchemy and test dependencies -echo "Installing SQLAlchemy and dependencies..." -pip3 install --user -e . -pip3 install --user pytest greenlet typing-extensions - -# Run the test suite multiple times to trigger the segfault -echo "" -echo "========================================" -echo "Running reconnect tests to reproduce segfault..." -echo "========================================" -echo "This will run the test suite in a loop until it crashes or completes 10 runs." -echo "" - -CRASH_DETECTED=0 -for i in {1..10}; do - echo "" - echo "=== Test Run $i ===" - - if ! pytest test/engine/test_reconnect.py::RealReconnectTest \ - --dburi "$DB_URI" \ - --disable-asyncio -s -v 2>&1; then - - echo "" - echo "*** CRASH DETECTED ON RUN $i ***" - CRASH_DETECTED=1 - break - fi - - echo "Run $i completed successfully" - sleep 1 -done - -echo "" -if [ $CRASH_DETECTED -eq 0 ]; then - echo "========================================" - echo "No crash detected in 10 runs." - echo "The issue may be intermittent or already fixed." - echo "========================================" -else - echo "========================================" - echo "*** SEGFAULT REPRODUCED! ***" - echo "========================================" -fi - -echo "" -echo "Test directory: $WORK_DIR" diff --git a/run_windows_test.sh b/run_windows_test.sh deleted file mode 100755 index 141abe950..000000000 --- a/run_windows_test.sh +++ /dev/null @@ -1,160 +0,0 @@ -#!/bin/bash -# Helper script to run Windows tests from WSL - -set -e - -echo "==========================================" -echo "Launching Windows Test from WSL" -echo "==========================================" -echo "" - -# Convert current path to Windows path -WINDOWS_PATH=$(wslpath -w "$PWD") -echo "Windows Path: $WINDOWS_PATH" -echo "" - -# Check if Python is installed on Windows -echo "Checking Windows Python installation..." -if ! powershell.exe -Command "python --version" 2>/dev/null; then - echo "" - echo "ERROR: Python not found in Windows PATH" - echo "Please install Python on Windows: https://www.python.org/downloads/" - exit 1 -fi - -PYTHON_VERSION=$(powershell.exe -Command "python --version 2>&1" | tr -d '\r') -echo "Found: $PYTHON_VERSION" -echo "" - -# Ask user for connection details -echo "==========================================" -echo "SQL Server Connection Details" -echo "==========================================" -read -p "SQL Server (default: localhost): " SQL_SERVER -SQL_SERVER=${SQL_SERVER:-localhost} - -read -p "Database (default: test): " DATABASE -DATABASE=${DATABASE:-test} - -read -p "Username (default: sa): " USERNAME -USERNAME=${USERNAME:-sa} - -read -sp "Password: " PASSWORD -echo "" - -read -p "Number of test runs (default: 10): " TEST_RUNS -TEST_RUNS=${TEST_RUNS:-10} - -echo "" -echo "==========================================" -echo "Starting Windows Build and Test" -echo "==========================================" -echo "" - -# Build the C++ extension on Windows -echo "Step 1: Building C++ extension on Windows..." -echo "This requires Visual Studio with C++ tools installed" -echo "" - -powershell.exe -NoProfile -ExecutionPolicy Bypass -Command " - cd '$WINDOWS_PATH' - Write-Host 'Current directory:' (Get-Location) -ForegroundColor Cyan - Write-Host '' - - # Check for Visual Studio - Write-Host 'Checking for Visual Studio...' -ForegroundColor Yellow - if (-not (Test-Path 'C:\Program Files\Microsoft Visual Studio')) { - Write-Host 'WARNING: Visual Studio not found in default location' -ForegroundColor Red - Write-Host 'Build may fail without Visual Studio C++ tools' -ForegroundColor Red - Write-Host '' - } - - # Build the extension - Write-Host 'Building C++ extension...' -ForegroundColor Yellow - cd mssql_python\pybind - - if (-not (Test-Path 'build.bat')) { - Write-Host 'ERROR: build.bat not found!' -ForegroundColor Red - exit 1 - } - - & cmd /c 'build.bat x64 2>&1' - if (\$LASTEXITCODE -ne 0) { - Write-Host 'ERROR: Build failed!' -ForegroundColor Red - Write-Host '' - Write-Host 'Troubleshooting:' -ForegroundColor Yellow - Write-Host '1. Install Visual Studio 2019+ with C++ build tools' -ForegroundColor Gray - Write-Host '2. Open Visual Studio Developer Command Prompt' -ForegroundColor Gray - Write-Host '3. Run: cd \"$WINDOWS_PATH\mssql_python\pybind\" && build.bat x64' -ForegroundColor Gray - exit 1 - } - - cd ..\.. - Write-Host '✓ Build completed successfully' -ForegroundColor Green - Write-Host '' - - # Install the package - Write-Host 'Step 2: Installing package in development mode...' -ForegroundColor Yellow - python -m pip install -e . 2>&1 | Out-Null - if (\$LASTEXITCODE -ne 0) { - Write-Host 'ERROR: Installation failed!' -ForegroundColor Red - exit 1 - } - Write-Host '✓ Package installed' -ForegroundColor Green - Write-Host '' - - # Verify extension loads - Write-Host 'Step 3: Verifying C++ extension...' -ForegroundColor Yellow - python -c 'import mssql_python; print(f\"Version: {mssql_python.__version__}\"); from mssql_python.ddbc_bindings import Connection; print(\"✓ C++ extension loaded\")' - if (\$LASTEXITCODE -ne 0) { - Write-Host 'ERROR: Extension verification failed!' -ForegroundColor Red - exit 1 - } - Write-Host '' - - # Install test dependencies - Write-Host 'Step 4: Installing test dependencies...' -ForegroundColor Yellow - python -m pip install pytest sqlalchemy greenlet typing-extensions 2>&1 | Out-Null - Write-Host '✓ Dependencies installed' -ForegroundColor Green - Write-Host '' - - # Update connection string in test file - Write-Host 'Step 5: Updating connection string...' -ForegroundColor Yellow - \$connStr = \"Driver={ODBC Driver 18 for SQL Server};Server=$SQL_SERVER;Database=$DATABASE;UID=$USERNAME;PWD=$PASSWORD;Encrypt=No\" - \$testFile = Get-Content 'test_windows_segfault_simple.py' -Raw - \$testFile = \$testFile -replace 'CONN_STR = \".*?\"', \"CONN_STR = \`\"\$connStr\`\"\" - \$testFile | Set-Content 'test_windows_segfault_simple.py' -NoNewline - Write-Host '✓ Connection string updated' -ForegroundColor Green - Write-Host '' - - # Run the simple test - Write-Host 'Step 6: Running Windows segfault test...' -ForegroundColor Yellow - Write-Host '========================================' -ForegroundColor Cyan - python test_windows_segfault_simple.py - \$testResult = \$LASTEXITCODE - Write-Host '========================================' -ForegroundColor Cyan - Write-Host '' - - if (\$testResult -eq 0) { - Write-Host '✓✓✓ ALL TESTS PASSED! ✓✓✓' -ForegroundColor Green - Write-Host 'The fix is working correctly on Windows.' -ForegroundColor Green - } else { - Write-Host '✗✗✗ TESTS FAILED! ✗✗✗' -ForegroundColor Red - Write-Host 'The segfault issue still exists.' -ForegroundColor Red - } - - exit \$testResult -" - -EXIT_CODE=$? - -echo "" -echo "==========================================" -if [ $EXIT_CODE -eq 0 ]; then - echo "✓ Windows test completed successfully!" -else - echo "✗ Windows test failed with exit code: $EXIT_CODE" -fi -echo "==========================================" - -exit $EXIT_CODE diff --git a/setup_and_run_windows_test.sh b/setup_and_run_windows_test.sh deleted file mode 100755 index f6d3c6ac4..000000000 --- a/setup_and_run_windows_test.sh +++ /dev/null @@ -1,225 +0,0 @@ -#!/bin/bash -# Copy repository to Windows and run tests there - -set -e - -echo "==========================================" -echo "Windows Test Setup from WSL" -echo "==========================================" -echo "" - -# Get Windows directory -echo "We need to copy the repository to a Windows-accessible directory" -echo "because Windows CMD doesn't support UNC paths (\\\\wsl.localhost\\...)" -echo "" -read -p "Windows directory path (e.g., C:\\Temp\\mssql-test): " WIN_DIR - -if [ -z "$WIN_DIR" ]; then - WIN_DIR="C:\\Temp\\mssql-python-test" - echo "Using default: $WIN_DIR" -fi - -# Convert to Windows format if needed -WIN_DIR=$(echo "$WIN_DIR" | sed 's/\//\\/g') - -# Convert to Linux path for copying -LINUX_PATH=$(wslpath "$WIN_DIR" 2>/dev/null || echo "") -if [ -z "$LINUX_PATH" ]; then - echo "ERROR: Could not convert Windows path to Linux path" - exit 1 -fi - -echo "" -echo "Linux path: $LINUX_PATH" -echo "Windows path: $WIN_DIR" -echo "" - -# Create directory and copy files -echo "Copying repository to Windows..." -mkdir -p "$LINUX_PATH" - -# Copy necessary files -rsync -av --progress \ - --exclude='.git' \ - --exclude='__pycache__' \ - --exclude='*.pyc' \ - --exclude='.pytest_cache' \ - --exclude='build' \ - --exclude='*.egg-info' \ - --exclude='.venv' \ - --exclude='venv' \ - ./ "$LINUX_PATH/" - -echo "" -echo "✓ Files copied successfully" -echo "" - -# Get connection details -echo "==========================================" -echo "SQL Server Connection Details" -echo "==========================================" -read -p "SQL Server (default: localhost): " SQL_SERVER -SQL_SERVER=${SQL_SERVER:-localhost} - -read -p "Database (default: test): " DATABASE -DATABASE=${DATABASE:-test} - -read -p "Username (default: sa): " USERNAME -USERNAME=${USERNAME:-sa} - -read -sp "Password: " PASSWORD -echo "" -echo "" - -# Create a PowerShell script to run in Windows -cat > "$LINUX_PATH/run_test_windows.ps1" << 'PSEOF' -param( - [string]$SqlServer = "localhost", - [string]$Database = "test", - [string]$Username = "sa", - [string]$Password = "" -) - -$ErrorActionPreference = "Stop" - -Write-Host "==========================================" -ForegroundColor Cyan -Write-Host "Windows C++ Build and Test" -ForegroundColor Cyan -Write-Host "==========================================" -ForegroundColor Cyan -Write-Host "" -Write-Host "Python: $(python --version)" -ForegroundColor Green -Write-Host "Location: $PWD" -ForegroundColor Gray -Write-Host "" - -# Check for Visual Studio -Write-Host "Step 1: Checking build environment..." -ForegroundColor Yellow -$vsPath = & "${env:ProgramFiles(x86)}\Microsoft Visual Studio\Installer\vswhere.exe" -latest -property installationPath 2>$null -if ($vsPath) { - Write-Host " ✓ Visual Studio found: $vsPath" -ForegroundColor Green -} else { - Write-Host " ⚠ Visual Studio not detected" -ForegroundColor Yellow - Write-Host " Build may fail without Visual Studio C++ tools" -ForegroundColor Yellow -} -Write-Host "" - -# Build C++ extension -Write-Host "Step 2: Building C++ extension..." -ForegroundColor Yellow -Push-Location "mssql_python\pybind" - -if (-not (Test-Path "build.bat")) { - Write-Host " ERROR: build.bat not found!" -ForegroundColor Red - Pop-Location - exit 1 -} - -Write-Host " Running: build.bat x64" -ForegroundColor Gray -$buildOutput = & cmd /c "build.bat x64 2>&1" -$buildExit = $LASTEXITCODE - -if ($buildExit -ne 0) { - Write-Host "" - Write-Host " ✗ Build failed!" -ForegroundColor Red - Write-Host "" - Write-Host "Build output:" -ForegroundColor Yellow - Write-Host $buildOutput - Write-Host "" - Write-Host "Troubleshooting:" -ForegroundColor Yellow - Write-Host " 1. Open 'Developer Command Prompt for VS'" -ForegroundColor Gray - Write-Host " 2. Navigate to: $PWD" -ForegroundColor Gray - Write-Host " 3. Run: cd mssql_python\pybind && build.bat x64" -ForegroundColor Gray - Pop-Location - exit 1 -} - -Pop-Location -Write-Host " ✓ C++ extension built successfully" -ForegroundColor Green -Write-Host "" - -# Install package -Write-Host "Step 3: Installing package..." -ForegroundColor Yellow -$installOutput = python -m pip install -e . 2>&1 -if ($LASTEXITCODE -ne 0) { - Write-Host " ✗ Installation failed!" -ForegroundColor Red - Write-Host $installOutput - exit 1 -} -Write-Host " ✓ Package installed" -ForegroundColor Green -Write-Host "" - -# Verify extension -Write-Host "Step 4: Verifying C++ extension..." -ForegroundColor Yellow -$verifyOutput = python -c "import mssql_python; print(f' Version: {mssql_python.__version__}'); from mssql_python.ddbc_bindings import Connection; print(' ✓ Extension loaded')" 2>&1 -if ($LASTEXITCODE -ne 0) { - Write-Host " ✗ Verification failed!" -ForegroundColor Red - Write-Host $verifyOutput - exit 1 -} -Write-Host $verifyOutput -Write-Host "" - -# Install dependencies -Write-Host "Step 5: Installing test dependencies..." -ForegroundColor Yellow -python -m pip install -q pytest sqlalchemy greenlet typing-extensions -Write-Host " ✓ Dependencies installed" -ForegroundColor Green -Write-Host "" - -# Update connection string -Write-Host "Step 6: Configuring connection..." -ForegroundColor Yellow -$connStr = "Driver={ODBC Driver 18 for SQL Server};Server=$SqlServer;Database=$Database;UID=$Username;PWD=$Password;Encrypt=No" -Write-Host " Server: $SqlServer/$Database" -ForegroundColor Gray - -$testFile = Get-Content 'test_windows_segfault_simple.py' -Raw -$testFile = $testFile -replace 'CONN_STR = ".*?"', "CONN_STR = `"$connStr`"" -$testFile | Set-Content 'test_windows_segfault_simple.py' -NoNewline -Write-Host " ✓ Connection configured" -ForegroundColor Green -Write-Host "" - -# Run test -Write-Host "Step 7: Running segfault test..." -ForegroundColor Yellow -Write-Host "==========================================" -ForegroundColor Cyan -Write-Host "" - -python test_windows_segfault_simple.py -$testExit = $LASTEXITCODE - -Write-Host "" -Write-Host "==========================================" -ForegroundColor Cyan -if ($testExit -eq 0) { - Write-Host "✓✓✓ SUCCESS! All tests passed! ✓✓✓" -ForegroundColor Green - Write-Host "The segfault fix is working on Windows." -ForegroundColor Green -} else { - Write-Host "✗✗✗ FAILURE! Tests failed! ✗✗✗" -ForegroundColor Red - Write-Host "The issue may still exist." -ForegroundColor Red -} -Write-Host "==========================================" -ForegroundColor Cyan - -exit $testExit -PSEOF - -echo "==========================================" -echo "Starting Windows test..." -echo "==========================================" -echo "" - -# Run PowerShell script -powershell.exe -NoProfile -ExecutionPolicy Bypass -File "$(wslpath -w "$LINUX_PATH/run_test_windows.ps1")" -SqlServer "$SQL_SERVER" -Database "$DATABASE" -Username "$USERNAME" -Password "$PASSWORD" - -EXIT_CODE=$? - -echo "" -echo "==========================================" -if [ $EXIT_CODE -eq 0 ]; then - echo "✓ Windows test completed successfully!" - echo "" - echo "The C++ fix is working correctly on Windows!" -else - echo "✗ Windows test failed with exit code: $EXIT_CODE" -fi -echo "==========================================" -echo "" -echo "Files are located at: $WIN_DIR" -echo "You can also run tests manually from Windows:" -echo " 1. Open PowerShell" -echo " 2. cd $WIN_DIR" -echo " 3. .\\run_test_windows.ps1 -SqlServer '$SQL_SERVER' -Database '$DATABASE' -Username '$USERNAME' -Password 'yourpass'" - -exit $EXIT_CODE diff --git a/test_windows_segfault.ps1 b/test_windows_segfault.ps1 deleted file mode 100644 index 30fd5650a..000000000 --- a/test_windows_segfault.ps1 +++ /dev/null @@ -1,171 +0,0 @@ -# PowerShell script to reproduce and test the segfault fix on Windows -# Run this from the mssql-python repository root - -param( - [string]$SqlServer = "localhost", - [string]$Database = "test", - [string]$Username = "sa", - [string]$Password = "", - [int]$TestRuns = 10 -) - -Write-Host "========================================" -ForegroundColor Cyan -Write-Host "Windows Segfault Reproduction Test" -ForegroundColor Cyan -Write-Host "========================================" -ForegroundColor Cyan -Write-Host "" - -# Check Python version -$pythonVersion = python --version 2>&1 -Write-Host "Python Version: $pythonVersion" -ForegroundColor Green -Write-Host "" - -# Step 1: Build the C++ extension with the fix -Write-Host "Step 1: Building C++ extension with fix..." -ForegroundColor Yellow -Write-Host "-------------------------------------" -ForegroundColor Yellow - -if (Test-Path "mssql_python\pybind\build") { - Write-Host "Cleaning existing build directory..." -ForegroundColor Gray - Remove-Item -Path "mssql_python\pybind\build" -Recurse -Force -ErrorAction SilentlyContinue -} - -Push-Location "mssql_python\pybind" -Write-Host "Running build.bat..." -ForegroundColor Gray -$buildOutput = & cmd /c "build.bat" 2>&1 -if ($LASTEXITCODE -ne 0) { - Write-Host "ERROR: Build failed!" -ForegroundColor Red - Write-Host $buildOutput - Pop-Location - exit 1 -} -Write-Host "✓ C++ extension built successfully" -ForegroundColor Green -Pop-Location -Write-Host "" - -# Step 2: Install the package in development mode -Write-Host "Step 2: Installing mssql-python with fix..." -ForegroundColor Yellow -Write-Host "-------------------------------------" -ForegroundColor Yellow -$installOutput = python -m pip install -e . 2>&1 -if ($LASTEXITCODE -ne 0) { - Write-Host "ERROR: Installation failed!" -ForegroundColor Red - Write-Host $installOutput - exit 1 -} -Write-Host "✓ Package installed successfully" -ForegroundColor Green -Write-Host "" - -# Step 3: Verify the C++ extension loads -Write-Host "Step 3: Verifying C++ extension..." -ForegroundColor Yellow -Write-Host "-------------------------------------" -ForegroundColor Yellow -$verifyScript = @" -import mssql_python -print('✓ mssql_python imported') -from mssql_python.ddbc_bindings import Connection -print('✓ C++ extension loaded successfully') -import sys -print(f'Python: {sys.version}') -print(f'mssql_python: {mssql_python.__version__}') -"@ - -$verifyOutput = python -c $verifyScript 2>&1 -if ($LASTEXITCODE -ne 0) { - Write-Host "ERROR: Extension verification failed!" -ForegroundColor Red - Write-Host $verifyOutput - exit 1 -} -Write-Host $verifyOutput -Write-Host "" - -# Step 4: Check if SQLAlchemy is available -Write-Host "Step 4: Checking SQLAlchemy installation..." -ForegroundColor Yellow -Write-Host "-------------------------------------" -ForegroundColor Yellow -$sqlalchemyCheck = python -c "import sqlalchemy; print(f'SQLAlchemy version: {sqlalchemy.__version__}')" 2>&1 -if ($LASTEXITCODE -ne 0) { - Write-Host "SQLAlchemy not found. Installing..." -ForegroundColor Yellow - python -m pip install sqlalchemy pytest greenlet typing-extensions 2>&1 | Out-Null - Write-Host "✓ SQLAlchemy installed" -ForegroundColor Green -} else { - Write-Host $sqlalchemyCheck -ForegroundColor Green -} -Write-Host "" - -# Step 5: Run the connection invalidation test -Write-Host "Step 5: Running connection invalidation test..." -ForegroundColor Yellow -Write-Host "-------------------------------------" -ForegroundColor Yellow - -if ($Password -eq "") { - Write-Host "ERROR: SQL Server password not provided!" -ForegroundColor Red - Write-Host "Usage: .\test_windows_segfault.ps1 -SqlServer 'localhost' -Database 'test' -Username 'sa' -Password 'YourPassword'" -ForegroundColor Yellow - Write-Host "" - Write-Host "Or set connection string and run basic test:" -ForegroundColor Yellow - Write-Host " `$env:TEST_CONN_STR = 'mssql+mssqlpython://user:pass@localhost/test'" -ForegroundColor Yellow - exit 1 -} - -$connStr = "mssql+mssqlpython://${Username}:${Password}@${SqlServer}/${Database}?Encrypt=No" -Write-Host "Connection: mssql+mssqlpython://${Username}:***@${SqlServer}/${Database}" -ForegroundColor Gray -Write-Host "" - -# Run the local test if it exists -if (Test-Path "tests\test_016_connection_invalidation_segfault.py") { - Write-Host "Running local connection invalidation test..." -ForegroundColor Cyan - $testResult = pytest tests\test_016_connection_invalidation_segfault.py -v --tb=short 2>&1 - if ($LASTEXITCODE -ne 0) { - Write-Host "Test execution encountered issues:" -ForegroundColor Yellow - Write-Host $testResult - } else { - Write-Host "✓ Local tests passed!" -ForegroundColor Green - } - Write-Host "" -} - -# Step 6: Run multiple iterations of SQLAlchemy reconnect tests -Write-Host "Step 6: Running SQLAlchemy reconnect tests (${TestRuns} iterations)..." -ForegroundColor Yellow -Write-Host "-------------------------------------" -ForegroundColor Yellow - -# Check if SQLAlchemy test directory exists -if (-not (Test-Path "..\sqlalchemy\test\engine\test_reconnect.py")) { - Write-Host "SQLAlchemy test suite not found." -ForegroundColor Yellow - Write-Host "To run full SQLAlchemy tests:" -ForegroundColor Yellow - Write-Host " 1. Clone SQLAlchemy: git clone https://github.com/sqlalchemy/sqlalchemy.git" -ForegroundColor Gray - Write-Host " 2. Install it: cd sqlalchemy && pip install -e ." -ForegroundColor Gray - Write-Host " 3. Run: pytest test\engine\test_reconnect.py::RealReconnectTest --dburi '$connStr' -v" -ForegroundColor Gray - Write-Host "" -} else { - Write-Host "Running RealReconnectTest from SQLAlchemy..." -ForegroundColor Cyan - - $crashDetected = $false - for ($i = 1; $i -le $TestRuns; $i++) { - Write-Host " Test Run $i/$TestRuns..." -ForegroundColor Gray - - Push-Location "..\sqlalchemy" - $testOutput = pytest test\engine\test_reconnect.py::RealReconnectTest --dburi $connStr --disable-asyncio -v 2>&1 - $exitCode = $LASTEXITCODE - Pop-Location - - if ($exitCode -ne 0) { - Write-Host "" - Write-Host " ✗ CRASH DETECTED ON RUN $i" -ForegroundColor Red - Write-Host $testOutput | Select-String -Pattern "Error|Exception|Segmentation" - $crashDetected = $true - break - } - - Write-Host " ✓ Run $i completed successfully" -ForegroundColor Green - Start-Sleep -Seconds 1 - } - - Write-Host "" - if (-not $crashDetected) { - Write-Host "========================================" -ForegroundColor Green - Write-Host "SUCCESS! No crash in $TestRuns runs." -ForegroundColor Green - Write-Host "The fix appears to be working!" -ForegroundColor Green - Write-Host "========================================" -ForegroundColor Green - } else { - Write-Host "========================================" -ForegroundColor Red - Write-Host "FIX DID NOT WORK - Segfault occurred!" -ForegroundColor Red - Write-Host "========================================" -ForegroundColor Red - } -} - -Write-Host "" -Write-Host "Test completed." -ForegroundColor Cyan diff --git a/test_windows_segfault_simple.py b/test_windows_segfault_simple.py deleted file mode 100644 index 0bcac23c7..000000000 --- a/test_windows_segfault_simple.py +++ /dev/null @@ -1,192 +0,0 @@ -""" -Simple Python script to reproduce the Windows segfault issue with the fix. -This mimics the stack trace scenario: cursor destruction during object creation. - -Run: python test_windows_segfault_simple.py -""" - -import gc -import sys -import threading -from mssql_python import connect, DatabaseError, OperationalError - -# Connection string - update with your SQL Server details -CONN_STR = "Driver={ODBC Driver 18 for SQL Server};Server=localhost;Database=test;UID=sa;PWD=YourPassword;Encrypt=No" - - -def test_cursor_destruction_during_init(): - """ - Test that mimics the stack trace where __del__ is called during object initialization. - This tests the fix for premature cursor destruction. - """ - print("=" * 60) - print("Test: Cursor Destruction During Initialization") - print("=" * 60) - - try: - conn = connect(CONN_STR) - print("✓ Connection established") - - # Create multiple cursors and trigger GC during initialization - cursors = [] - for i in range(5): - cursor = conn.cursor() - cursor.execute("SELECT 1 AS test_col") - cursor.fetchall() - cursors.append(cursor) - - # Force garbage collection to trigger __del__ on unreferenced objects - gc.collect() - print(f" Cursor {i+1} created and GC triggered") - - # Close connection WITHOUT explicitly closing cursors first - # This simulates the invalidation scenario - print("\n✓ Closing connection without closing cursors...") - conn.close() - - # Force garbage collection to trigger cursor cleanup - print("✓ Triggering garbage collection...") - cursors = None - gc.collect() - gc.collect() # Call twice to ensure cleanup - - print("\n" + "=" * 60) - print("SUCCESS: No segfault detected!") - print("The fix is working correctly.") - print("=" * 60) - return True - - except Exception as e: - print("\n" + "=" * 60) - print(f"ERROR: {type(e).__name__}: {e}") - print("=" * 60) - import traceback - traceback.print_exc() - return False - - -def test_connection_invalidation_multiple_times(iterations=10): - """ - Run the connection invalidation test multiple times to ensure stability. - """ - print("\n" + "=" * 60) - print(f"Running Connection Invalidation Test ({iterations} iterations)") - print("=" * 60) - - failures = 0 - for i in range(iterations): - print(f"\nIteration {i+1}/{iterations}...", end=" ") - try: - conn = connect(CONN_STR) - cursor = conn.cursor() - cursor.execute("SELECT 1") - cursor.fetchone() - conn.close() - gc.collect() - print("✓ PASS") - except Exception as e: - print(f"✗ FAIL: {e}") - failures += 1 - - print("\n" + "=" * 60) - if failures == 0: - print(f"✓ All {iterations} iterations passed!") - print("The fix is stable across multiple runs.") - else: - print(f"✗ {failures}/{iterations} iterations failed") - print("=" * 60) - - return failures == 0 - - -def test_threading_scenario(): - """ - Test that mimics the threading.RLock scenario from the stack trace. - This ensures the fix works when cursors are destroyed during lock creation. - """ - print("\n" + "=" * 60) - print("Test: Threading + Cursor Destruction") - print("=" * 60) - - def worker(): - try: - conn = connect(CONN_STR) - cursor = conn.cursor() - cursor.execute("SELECT 1") - cursor.fetchone() - conn.close() - gc.collect() - return True - except Exception as e: - print(f" Thread error: {e}") - return False - - threads = [] - for i in range(5): - t = threading.Thread(target=worker) - threads.append(t) - t.start() - print(f" Thread {i+1} started") - - print("\n Waiting for threads to complete...") - for t in threads: - t.join() - - print("\n" + "=" * 60) - print("✓ Threading test completed without crashes") - print("=" * 60) - - return True - - -def main(): - """Main test runner""" - print("\n" + "=" * 60) - print("Windows Segfault Fix - Validation Tests") - print("=" * 60) - print(f"Python version: {sys.version}") - print(f"Platform: {sys.platform}") - print("=" * 60) - print("\nIMPORTANT: Update CONN_STR in the script with your SQL Server details") - print("=" * 60) - - try: - # Import and print version info - import mssql_python - print(f"\nmssql_python version: {mssql_python.__version__}") - from mssql_python.ddbc_bindings import Connection - print("✓ C++ extension loaded successfully") - except Exception as e: - print(f"\n✗ Failed to load mssql_python: {e}") - print("\nMake sure to build and install the package first:") - print(" 1. cd mssql_python\\pybind") - print(" 2. build.bat") - print(" 3. cd ..\\..") - print(" 4. pip install -e .") - return False - - # Run tests - print("\n") - test1_passed = test_cursor_destruction_during_init() - - if test1_passed: - test2_passed = test_connection_invalidation_multiple_times(10) - test3_passed = test_threading_scenario() - - if test1_passed and test2_passed and test3_passed: - print("\n" + "=" * 60) - print("ALL TESTS PASSED!") - print("The segfault fix is working correctly on Windows.") - print("=" * 60) - return True - - print("\n" + "=" * 60) - print("SOME TESTS FAILED") - print("Please review the errors above.") - print("=" * 60) - return False - - -if __name__ == "__main__": - success = main() - sys.exit(0 if success else 1) From 88a6167b8eb464275e3308eb88bb62a4606e9078 Mon Sep 17 00:00:00 2001 From: Subrata Paitandi Date: Fri, 30 Jan 2026 08:40:23 +0000 Subject: [PATCH 07/10] review comment --- mssql_python/pybind/ddbc_bindings.cpp | 4 +--- tests/test_016_connection_invalidation_segfault.py | 13 ++++--------- 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/mssql_python/pybind/ddbc_bindings.cpp b/mssql_python/pybind/ddbc_bindings.cpp index 0f262fd37..b4a88a741 100644 --- a/mssql_python/pybind/ddbc_bindings.cpp +++ b/mssql_python/pybind/ddbc_bindings.cpp @@ -4364,9 +4364,7 @@ PYBIND11_MODULE(ddbc_bindings, m) { .def_readwrite("ddbcErrorMsg", &ErrorInfo::ddbcErrorMsg); py::class_(m, "SqlHandle") - .def("free", &SqlHandle::free, "Free the handle") - .def("mark_implicitly_freed", &SqlHandle::markImplicitlyFreed, - "Mark handle as implicitly freed by parent handle"); + .def("free", &SqlHandle::free, "Free the handle"); py::class_(m, "Connection") .def(py::init(), py::arg("conn_str"), diff --git a/tests/test_016_connection_invalidation_segfault.py b/tests/test_016_connection_invalidation_segfault.py index 3d509ecde..4ae07306a 100644 --- a/tests/test_016_connection_invalidation_segfault.py +++ b/tests/test_016_connection_invalidation_segfault.py @@ -243,19 +243,14 @@ def test_connection_invalidation_with_prepared_statements(conn_str): assert True -def test_verify_mark_implicitly_freed_method_exists(): +def test_verify_sqlhandle_free_method_exists(): """ - Verify that the mark_implicitly_freed method exists on SqlHandle. - This is the core of the segfault fix. + Verify that the free method exists on SqlHandle. + The segfault fix uses markImplicitlyFreed internally in C++ (not exposed to Python). """ from mssql_python import ddbc_bindings - # Verify the method exists - assert hasattr( - ddbc_bindings.SqlHandle, "mark_implicitly_freed" - ), "SqlHandle should have mark_implicitly_freed method" - - # Verify free method also exists + # Verify free method exists assert hasattr(ddbc_bindings.SqlHandle, "free"), "SqlHandle should have free method" From bdf1a974edafa58ca7671012a1e90503a377f7cb Mon Sep 17 00:00:00 2001 From: Subrata Paitandi Date: Fri, 30 Jan 2026 08:47:15 +0000 Subject: [PATCH 08/10] review comment --- mssql_python/pybind/connection/connection.cpp | 36 +++++++++++++++---- mssql_python/pybind/connection/connection.h | 5 +++ 2 files changed, 34 insertions(+), 7 deletions(-) diff --git a/mssql_python/pybind/connection/connection.cpp b/mssql_python/pybind/connection/connection.cpp index 7c5e756a4..d916b2d6f 100644 --- a/mssql_python/pybind/connection/connection.cpp +++ b/mssql_python/pybind/connection/connection.cpp @@ -99,6 +99,18 @@ void Connection::disconnect() { // When we free the DBC handle below, the ODBC driver will automatically free // all child STMT handles. We need to tell the SqlHandle objects about this // so they don't try to free the handles again during their destruction. + + // First compact: remove expired weak_ptrs (they're already destroyed) + size_t originalSize = _childStatementHandles.size(); + _childStatementHandles.erase( + std::remove_if(_childStatementHandles.begin(), _childStatementHandles.end(), + [](const std::weak_ptr& wp) { return wp.expired(); }), + _childStatementHandles.end()); + + LOG("Compacted child handles: %zu -> %zu (removed %zu expired)", + originalSize, _childStatementHandles.size(), + originalSize - _childStatementHandles.size()); + LOG("Marking %zu child statement handles as implicitly freed", _childStatementHandles.size()); for (auto& weakHandle : _childStatementHandles) { @@ -107,6 +119,7 @@ void Connection::disconnect() { } } _childStatementHandles.clear(); + _allocationsSinceCompaction = 0; SQLRETURN ret = SQLDisconnect_ptr(_dbcHandle->get()); checkError(ret); @@ -192,13 +205,22 @@ SqlHandlePtr Connection::allocStatementHandle() { // Track this child handle so we can mark it as implicitly freed when connection closes // Use weak_ptr to avoid circular references and allow normal cleanup _childStatementHandles.push_back(stmtHandle); - - // Clean up expired weak_ptrs periodically to avoid unbounded growth - // Remove entries where the weak_ptr is expired (object was already destroyed) - _childStatementHandles.erase( - std::remove_if(_childStatementHandles.begin(), _childStatementHandles.end(), - [](const std::weak_ptr& wp) { return wp.expired(); }), - _childStatementHandles.end()); + _allocationsSinceCompaction++; + + // Compact expired weak_ptrs only periodically to avoid O(n²) overhead + // This keeps allocation fast (O(1) amortized) while preventing unbounded growth + // disconnect() also compacts, so this is just for long-lived connections with many cursors + if (_allocationsSinceCompaction >= COMPACTION_INTERVAL) { + size_t originalSize = _childStatementHandles.size(); + _childStatementHandles.erase( + std::remove_if(_childStatementHandles.begin(), _childStatementHandles.end(), + [](const std::weak_ptr& wp) { return wp.expired(); }), + _childStatementHandles.end()); + _allocationsSinceCompaction = 0; + LOG("Periodic compaction: %zu -> %zu handles (removed %zu expired)", + originalSize, _childStatementHandles.size(), + originalSize - _childStatementHandles.size()); + } return stmtHandle; } diff --git a/mssql_python/pybind/connection/connection.h b/mssql_python/pybind/connection/connection.h index 4bda21a08..3f5178538 100644 --- a/mssql_python/pybind/connection/connection.h +++ b/mssql_python/pybind/connection/connection.h @@ -65,6 +65,11 @@ class Connection { // Track child statement handles to mark them as implicitly freed when connection closes // Uses weak_ptr to avoid circular references and allow normal cleanup std::vector> _childStatementHandles; + + // Counter for periodic compaction of expired weak_ptrs + // Compact every N allocations to avoid O(n²) overhead in hot path + size_t _allocationsSinceCompaction = 0; + static constexpr size_t COMPACTION_INTERVAL = 100; }; class ConnectionHandle { From 80e37f21dfaecd95026e30cc7379d708936a7150 Mon Sep 17 00:00:00 2001 From: Subrata Paitandi Date: Fri, 30 Jan 2026 09:08:24 +0000 Subject: [PATCH 09/10] latest review comment --- mssql_python/pybind/connection/connection.cpp | 8 ++++++++ mssql_python/pybind/ddbc_bindings.cpp | 11 +++++++++++ mssql_python/pybind/ddbc_bindings.h | 9 +++++++++ 3 files changed, 28 insertions(+) diff --git a/mssql_python/pybind/connection/connection.cpp b/mssql_python/pybind/connection/connection.cpp index d916b2d6f..8d2caae3b 100644 --- a/mssql_python/pybind/connection/connection.cpp +++ b/mssql_python/pybind/connection/connection.cpp @@ -115,6 +115,14 @@ void Connection::disconnect() { _childStatementHandles.size()); for (auto& weakHandle : _childStatementHandles) { if (auto handle = weakHandle.lock()) { + // SAFETY ASSERTION: Only STMT handles should be in this vector + // This is guaranteed by allocStatementHandle() which only creates STMT handles + // If this assertion fails, it indicates a serious bug in handle tracking + if (handle->type() != SQL_HANDLE_STMT) { + LOG_ERROR("CRITICAL: Non-STMT handle (type=%d) found in _childStatementHandles. " + "This will cause a handle leak!", handle->type()); + continue; // Skip marking to prevent leak + } handle->markImplicitlyFreed(); } } diff --git a/mssql_python/pybind/ddbc_bindings.cpp b/mssql_python/pybind/ddbc_bindings.cpp index b4a88a741..2cf04fe0d 100644 --- a/mssql_python/pybind/ddbc_bindings.cpp +++ b/mssql_python/pybind/ddbc_bindings.cpp @@ -1145,6 +1145,17 @@ SQLSMALLINT SqlHandle::type() const { } void SqlHandle::markImplicitlyFreed() { + // SAFETY: Only STMT handles should be marked as implicitly freed. + // When a DBC handle is freed, the ODBC driver automatically frees all child STMT handles. + // Other handle types (ENV, DBC, DESC) are NOT automatically freed by parents. + // Calling this on wrong handle types will cause silent handle leaks. + if (_type != SQL_HANDLE_STMT) { + // Log error but don't throw - we're likely in cleanup/destructor path + LOG_ERROR("SAFETY VIOLATION: Attempted to mark non-STMT handle as implicitly freed. " + "Handle type=%d. This will cause handle leak. Only STMT handles are " + "automatically freed by parent DBC handles.", _type); + return; // Refuse to mark - let normal free() handle it + } _implicitly_freed = true; } diff --git a/mssql_python/pybind/ddbc_bindings.h b/mssql_python/pybind/ddbc_bindings.h index 190c9bd1a..fd9e7db71 100644 --- a/mssql_python/pybind/ddbc_bindings.h +++ b/mssql_python/pybind/ddbc_bindings.h @@ -382,6 +382,15 @@ class SqlHandle { // Mark this handle as implicitly freed (freed by parent handle) // This prevents double-free attempts when the ODBC driver automatically // frees child handles (e.g., STMT handles when DBC handle is freed) + // + // SAFETY CONSTRAINTS: + // - ONLY call this on SQL_HANDLE_STMT handles + // - ONLY call this when the parent DBC handle is about to be freed + // - Calling on other handle types (ENV, DBC, DESC) will cause HANDLE LEAKS + // - The ODBC spec only guarantees automatic freeing of STMT handles by DBC parents + // + // Current usage: Connection::disconnect() marks all tracked STMT handles + // before freeing the DBC handle. void markImplicitlyFreed(); private: From 563b8ad69e62fa2eb42aeca44ea672fdc19fe0cb Mon Sep 17 00:00:00 2001 From: Subrata Paitandi Date: Fri, 30 Jan 2026 09:28:11 +0000 Subject: [PATCH 10/10] review comments --- mssql_python/pybind/connection/connection.cpp | 103 ++++++++++-------- mssql_python/pybind/connection/connection.h | 16 +++ 2 files changed, 74 insertions(+), 45 deletions(-) diff --git a/mssql_python/pybind/connection/connection.cpp b/mssql_python/pybind/connection/connection.cpp index 8d2caae3b..32ed55075 100644 --- a/mssql_python/pybind/connection/connection.cpp +++ b/mssql_python/pybind/connection/connection.cpp @@ -100,34 +100,40 @@ void Connection::disconnect() { // all child STMT handles. We need to tell the SqlHandle objects about this // so they don't try to free the handles again during their destruction. - // First compact: remove expired weak_ptrs (they're already destroyed) - size_t originalSize = _childStatementHandles.size(); - _childStatementHandles.erase( - std::remove_if(_childStatementHandles.begin(), _childStatementHandles.end(), - [](const std::weak_ptr& wp) { return wp.expired(); }), - _childStatementHandles.end()); - - LOG("Compacted child handles: %zu -> %zu (removed %zu expired)", - originalSize, _childStatementHandles.size(), - originalSize - _childStatementHandles.size()); - - LOG("Marking %zu child statement handles as implicitly freed", - _childStatementHandles.size()); - for (auto& weakHandle : _childStatementHandles) { - if (auto handle = weakHandle.lock()) { - // SAFETY ASSERTION: Only STMT handles should be in this vector - // This is guaranteed by allocStatementHandle() which only creates STMT handles - // If this assertion fails, it indicates a serious bug in handle tracking - if (handle->type() != SQL_HANDLE_STMT) { - LOG_ERROR("CRITICAL: Non-STMT handle (type=%d) found in _childStatementHandles. " - "This will cause a handle leak!", handle->type()); - continue; // Skip marking to prevent leak + // THREAD-SAFETY: Lock mutex to safely access _childStatementHandles + // This protects against concurrent allocStatementHandle() calls or GC finalizers + { + std::lock_guard lock(_childHandlesMutex); + + // First compact: remove expired weak_ptrs (they're already destroyed) + size_t originalSize = _childStatementHandles.size(); + _childStatementHandles.erase( + std::remove_if(_childStatementHandles.begin(), _childStatementHandles.end(), + [](const std::weak_ptr& wp) { return wp.expired(); }), + _childStatementHandles.end()); + + LOG("Compacted child handles: %zu -> %zu (removed %zu expired)", + originalSize, _childStatementHandles.size(), + originalSize - _childStatementHandles.size()); + + LOG("Marking %zu child statement handles as implicitly freed", + _childStatementHandles.size()); + for (auto& weakHandle : _childStatementHandles) { + if (auto handle = weakHandle.lock()) { + // SAFETY ASSERTION: Only STMT handles should be in this vector + // This is guaranteed by allocStatementHandle() which only creates STMT handles + // If this assertion fails, it indicates a serious bug in handle tracking + if (handle->type() != SQL_HANDLE_STMT) { + LOG_ERROR("CRITICAL: Non-STMT handle (type=%d) found in _childStatementHandles. " + "This will cause a handle leak!", handle->type()); + continue; // Skip marking to prevent leak + } + handle->markImplicitlyFreed(); } - handle->markImplicitlyFreed(); } - } - _childStatementHandles.clear(); - _allocationsSinceCompaction = 0; + _childStatementHandles.clear(); + _allocationsSinceCompaction = 0; + } // Release lock before potentially slow SQLDisconnect call SQLRETURN ret = SQLDisconnect_ptr(_dbcHandle->get()); checkError(ret); @@ -210,25 +216,32 @@ SqlHandlePtr Connection::allocStatementHandle() { checkError(ret); auto stmtHandle = std::make_shared(static_cast(SQL_HANDLE_STMT), stmt); - // Track this child handle so we can mark it as implicitly freed when connection closes - // Use weak_ptr to avoid circular references and allow normal cleanup - _childStatementHandles.push_back(stmtHandle); - _allocationsSinceCompaction++; - - // Compact expired weak_ptrs only periodically to avoid O(n²) overhead - // This keeps allocation fast (O(1) amortized) while preventing unbounded growth - // disconnect() also compacts, so this is just for long-lived connections with many cursors - if (_allocationsSinceCompaction >= COMPACTION_INTERVAL) { - size_t originalSize = _childStatementHandles.size(); - _childStatementHandles.erase( - std::remove_if(_childStatementHandles.begin(), _childStatementHandles.end(), - [](const std::weak_ptr& wp) { return wp.expired(); }), - _childStatementHandles.end()); - _allocationsSinceCompaction = 0; - LOG("Periodic compaction: %zu -> %zu handles (removed %zu expired)", - originalSize, _childStatementHandles.size(), - originalSize - _childStatementHandles.size()); - } + // THREAD-SAFETY: Lock mutex before modifying _childStatementHandles + // This protects against concurrent disconnect() or allocStatementHandle() calls, + // or GC finalizers running from different threads + { + std::lock_guard lock(_childHandlesMutex); + + // Track this child handle so we can mark it as implicitly freed when connection closes + // Use weak_ptr to avoid circular references and allow normal cleanup + _childStatementHandles.push_back(stmtHandle); + _allocationsSinceCompaction++; + + // Compact expired weak_ptrs only periodically to avoid O(n²) overhead + // This keeps allocation fast (O(1) amortized) while preventing unbounded growth + // disconnect() also compacts, so this is just for long-lived connections with many cursors + if (_allocationsSinceCompaction >= COMPACTION_INTERVAL) { + size_t originalSize = _childStatementHandles.size(); + _childStatementHandles.erase( + std::remove_if(_childStatementHandles.begin(), _childStatementHandles.end(), + [](const std::weak_ptr& wp) { return wp.expired(); }), + _childStatementHandles.end()); + _allocationsSinceCompaction = 0; + LOG("Periodic compaction: %zu -> %zu handles (removed %zu expired)", + originalSize, _childStatementHandles.size(), + originalSize - _childStatementHandles.size()); + } + } // Release lock return stmtHandle; } diff --git a/mssql_python/pybind/connection/connection.h b/mssql_python/pybind/connection/connection.h index 3f5178538..6c6f1e63c 100644 --- a/mssql_python/pybind/connection/connection.h +++ b/mssql_python/pybind/connection/connection.h @@ -5,10 +5,19 @@ #include "../ddbc_bindings.h" #include #include +#include // Represents a single ODBC database connection. // Manages connection handles. // Note: This class does NOT implement pooling logic directly. +// +// THREADING MODEL (per DB-API 2.0 threadsafety=1): +// - Connections should NOT be shared between threads in normal usage +// - However, _childStatementHandles is mutex-protected because: +// 1. Python GC/finalizers can run from any thread +// 2. Native code may release GIL during blocking ODBC calls +// 3. Provides safety if user accidentally shares connection +// - All accesses to _childStatementHandles are guarded by _childHandlesMutex class Connection { public: @@ -64,12 +73,19 @@ class Connection { // Track child statement handles to mark them as implicitly freed when connection closes // Uses weak_ptr to avoid circular references and allow normal cleanup + // THREAD-SAFETY: All accesses must be guarded by _childHandlesMutex std::vector> _childStatementHandles; // Counter for periodic compaction of expired weak_ptrs // Compact every N allocations to avoid O(n²) overhead in hot path + // THREAD-SAFETY: Protected by _childHandlesMutex size_t _allocationsSinceCompaction = 0; static constexpr size_t COMPACTION_INTERVAL = 100; + + // Mutex protecting _childStatementHandles and _allocationsSinceCompaction + // Prevents data races between allocStatementHandle() and disconnect(), + // or concurrent GC finalizers running from different threads + mutable std::mutex _childHandlesMutex; }; class ConnectionHandle {