diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 9dda9365..3b50c9a6 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -32,7 +32,7 @@ jobs: strategy: fail-fast: false matrix: - language: [ 'cpp', 'python' ] + language: [ 'cpp' ] # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] # Learn more about CodeQL language support at https://git.io/codeql-language-support diff --git a/.github/workflows/ubuntu_build.yml b/.github/workflows/ubuntu_build.yml index bb6fe8a3..e85da3cf 100644 --- a/.github/workflows/ubuntu_build.yml +++ b/.github/workflows/ubuntu_build.yml @@ -12,7 +12,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] + python-version: ["3.12"] services: @@ -40,23 +40,9 @@ jobs: ACCEPT_EULA: Y SA_PASSWORD: StrongPassword2022 - postgres: - image: postgres:13 - env: - POSTGRES_DB: postgres_db - POSTGRES_USER: postgres_user - POSTGRES_PASSWORD: postgres_pwd - ports: - - 5432:5432 - # needed because the postgres container does not provide a healthcheck - options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 steps: - - name: Start MySQL service - run: | - sudo systemctl start mysql.service - - name: Check initial setup run: | echo '*** echo $PATH' @@ -77,36 +63,6 @@ jobs: echo '*** ls -l /usr/lib/x86_64-linux-gnu/odbc' ls -l /opt/microsoft/msodbcsql18/lib64 || true - - name: Install ODBC driver for PostgreSQL - run: | - echo "*** apt-get install the driver" - sudo apt-get install --yes odbc-postgresql - echo '*** ls -l /usr/lib/x86_64-linux-gnu/odbc' - ls -l /usr/lib/x86_64-linux-gnu/odbc || true - echo '*** add full paths to Postgres .so files in /etc/odbcinst.ini' - sudo sed -i 's|Driver=psqlodbca.so|Driver=/usr/lib/x86_64-linux-gnu/odbc/psqlodbca.so|g' /etc/odbcinst.ini - sudo sed -i 's|Driver=psqlodbcw.so|Driver=/usr/lib/x86_64-linux-gnu/odbc/psqlodbcw.so|g' /etc/odbcinst.ini - sudo sed -i 's|Setup=libodbcpsqlS.so|Setup=/usr/lib/x86_64-linux-gnu/odbc/libodbcpsqlS.so|g' /etc/odbcinst.ini - - - name: Install ODBC driver for MySQL - run: | - cd "$RUNNER_TEMP" - echo "*** download driver zip file" - curl --silent --show-error --write-out "$CURL_OUTPUT_FORMAT" -O "https://www.mirrorservice.org/sites/ftp.mysql.com/Downloads/Connector-ODBC/8.0/${MYSQL_DRIVER}.tar.gz" - ls -l "${MYSQL_DRIVER}.tar.gz" - tar -xz -f "${MYSQL_DRIVER}.tar.gz" - echo "*** copy driver file to /usr/lib" - sudo cp -v "${MYSQL_DRIVER}/lib/libmyodbc8a.so" /usr/lib/x86_64-linux-gnu/odbc/ - sudo chmod a+r /usr/lib/x86_64-linux-gnu/odbc/libmyodbc8a.so - echo "*** create odbcinst.ini entry" - echo '[MySQL ODBC 8.0 ANSI Driver]' > mysql_odbcinst.ini - echo 'Driver = /usr/lib/x86_64-linux-gnu/odbc/libmyodbc8a.so' >> mysql_odbcinst.ini - echo 'UsageCount = 1' >> mysql_odbcinst.ini - echo 'Threading = 2' >> mysql_odbcinst.ini - sudo odbcinst -i -d -f mysql_odbcinst.ini - env: - CURL_OUTPUT_FORMAT: '%{http_code} %{filename_effective} %{size_download} %{time_total}\n' - MYSQL_DRIVER: mysql-connector-odbc-8.0.22-linux-glibc2.12-x86-64bit - name: Check ODBC setup run: | @@ -135,27 +91,7 @@ jobs: docker exec -i "${{ job.services.mssql2022.id }}" /opt/mssql-tools18/bin/sqlcmd -S localhost -U SA -P 'StrongPassword2022' -C -Q "SELECT @@VERSION" || sleep 5 docker exec -i "${{ job.services.mssql2022.id }}" /opt/mssql-tools18/bin/sqlcmd -S localhost -U SA -P 'StrongPassword2022' -C -Q "CREATE DATABASE test" - - name: Create test database in PostgreSQL - run: | - echo "*** get version" - psql -c "SELECT version()" - echo "*** create database" - psql -c "CREATE DATABASE test WITH encoding='UTF8' LC_COLLATE='en_US.utf8' LC_CTYPE='en_US.utf8'" - echo "*** list databases" - psql -l - env: - PGHOST: localhost - PGPORT: 5432 - PGDATABASE: postgres_db - PGUSER: postgres_user - PGPASSWORD: postgres_pwd - - name: Create test database in MySQL - run: | - echo "*** get status" - mysql --user=root --password=root --execute "STATUS" - echo "*** create database" - mysql --user=root --password=root --execute "CREATE DATABASE test" - uses: actions/checkout@v4.1.1 @@ -186,19 +122,7 @@ jobs: echo "*** pyodbc drivers" python -c "import pyodbc; print('\n'.join(sorted(pyodbc.drivers())))" - - name: Run PostgreSQL tests - env: - PYODBC_POSTGRESQL: "DRIVER={PostgreSQL Unicode};SERVER=localhost;PORT=5432;UID=postgres_user;PWD=postgres_pwd;DATABASE=test" - run: | - cd "$GITHUB_WORKSPACE" - python -m pytest "./tests/postgresql_test.py" - - name: Run MySQL tests - env: - PYODBC_MYSQL: "DRIVER={MySQL ODBC 8.0 ANSI Driver};SERVER=localhost;UID=root;PWD=root;DATABASE=test;CHARSET=utf8mb4" - run: | - cd "$GITHUB_WORKSPACE" - python -m pytest "./tests/mysql_test.py" - name: Run SQL Server 2017 tests env: diff --git a/src/cursor.cpp b/src/cursor.cpp index 08002453..91fa58c5 100644 --- a/src/cursor.cpp +++ b/src/cursor.cpp @@ -24,6 +24,7 @@ #include "getdata.h" #include "dbspecific.h" #include +#include enum { @@ -2382,6 +2383,298 @@ static PyObject* Cursor_exit(PyObject* self, PyObject* args) Py_RETURN_NONE; } +static char column_bind_insert_doc[] = +"column_bind_insert(sql, rows)\n" +"\n" +"Insert multiple rows efficiently using column-wise parameter binding.\n" +"\n" +"Arguments:\n" +" sql -- The SQL INSERT statement with parameter placeholders (e.g. ?, ?, ...).\n" +" rows -- A list of tuples, each tuple representing a row of data.\n" +"\n" +"This method transposes row-wise data into column-wise buffers and binds parameters\n" +"by column, reducing overhead and improving bulk insert performance.\n" +"\n" +"Supported data types:\n" +" int, float, str (UTF-8), datetime.date, datetime.datetime, None (as NULL).\n" +"\n" +"Returns None on success or raises an exception on error.\n"; +static PyObject* Cursor_column_bind_insert(PyObject* self, PyObject* args) { + Cursor* cursor = (Cursor*)self; + PyObject* sql; + PyObject* pyRows; + + if (!PyArg_ParseTuple(args, "OO!", &sql, &PyList_Type, &pyRows)) + return nullptr; + if (!PyUnicode_Check(sql)) + return PyErr_Format(PyExc_TypeError, "SQL must be a string"); + + size_t rowCount = PyList_Size(pyRows); + if (rowCount == 0) + Py_RETURN_NONE; + + PyObject* firstRow = PyList_GetItem(pyRows, 0); + if (!PyTuple_Check(firstRow)) + return PyErr_Format(PyExc_TypeError, "Expected list of tuples"); + + size_t colCount = PyTuple_Size(firstRow); + std::vector> buffers(colCount); + std::vector> indicators(colCount); + std::vector c_types(colCount); + + for (size_t col = 0; col < colCount; ++col) { + PyObject* sample = PyTuple_GetItem(firstRow, col); + if (PyLong_Check(sample)) c_types[col] = SQL_C_SLONG; + else if (PyFloat_Check(sample)) c_types[col] = SQL_C_DOUBLE; + else if (PyUnicode_Check(sample)) c_types[col] = SQL_C_CHAR; + else if (PyDateTime_Check(sample)) c_types[col] = SQL_C_TYPE_TIMESTAMP; + else if (PyDate_Check(sample)) c_types[col] = SQL_C_TYPE_DATE; + else if (sample == Py_None) c_types[col] = SQL_C_CHAR; // Assume char placeholder + else return PyErr_Format(PyExc_TypeError, "Unsupported type in column %zu", col); + + buffers[col].resize(rowCount); + indicators[col].resize(rowCount); + } + + // Allocate C++ structures + std::vector> ts_storage(colCount); + std::vector> date_storage(colCount); + std::vector> int_storage(colCount); + std::vector> float_storage(colCount); + std::vector str_storage; + + for (size_t row = 0; row < rowCount; ++row) { + PyObject* rowObj = PyList_GetItem(pyRows, row); + if (!PyTuple_Check(rowObj)) return PyErr_Format(PyExc_TypeError, "Each row must be a tuple"); + + for (size_t col = 0; col < colCount; ++col) { + PyObject* val = PyTuple_GetItem(rowObj, col); + SQLLEN& ind = indicators[col][row]; + + if (val == Py_None) { + buffers[col][row] = nullptr; + ind = SQL_NULL_DATA; + } else { + switch (c_types[col]) { + case SQL_C_SLONG: + if (!PyLong_Check(val)) return PyErr_Format(PyExc_TypeError, "Expected int"); + int_storage[col].resize(rowCount); + int_storage[col][row] = (int32_t)PyLong_AsLong(val); + buffers[col][row] = &int_storage[col][row]; + ind = sizeof(int32_t); + break; + case SQL_C_DOUBLE: + if (!PyFloat_Check(val)) return PyErr_Format(PyExc_TypeError, "Expected float"); + float_storage[col].resize(rowCount); + float_storage[col][row] = PyFloat_AsDouble(val); + buffers[col][row] = &float_storage[col][row]; + ind = sizeof(double); + break; + case SQL_C_CHAR: { + PyObject* bytes = PyUnicode_AsEncodedString(val, "utf-8", "strict"); + if (!bytes) return nullptr; + const char* b = PyBytes_AsString(bytes); + str_storage.emplace_back(b); + buffers[col][row] = (void*)str_storage.back().c_str(); + ind = (SQLLEN)str_storage.back().size(); + Py_DECREF(bytes); + break; + } + case SQL_C_TYPE_TIMESTAMP: + ts_storage[col].resize(rowCount); + ts_storage[col][row].year = PyDateTime_GET_YEAR(val); + ts_storage[col][row].month = PyDateTime_GET_MONTH(val); + ts_storage[col][row].day = PyDateTime_GET_DAY(val); + ts_storage[col][row].hour = PyDateTime_DATE_GET_HOUR(val); + ts_storage[col][row].minute = PyDateTime_DATE_GET_MINUTE(val); + ts_storage[col][row].second = PyDateTime_DATE_GET_SECOND(val); + ts_storage[col][row].fraction = PyDateTime_DATE_GET_MICROSECOND(val) * 1000; + buffers[col][row] = &ts_storage[col][row]; + ind = sizeof(SQL_TIMESTAMP_STRUCT); + break; + case SQL_C_TYPE_DATE: + date_storage[col].resize(rowCount); + date_storage[col][row].year = PyDateTime_GET_YEAR(val); + date_storage[col][row].month = PyDateTime_GET_MONTH(val); + date_storage[col][row].day = PyDateTime_GET_DAY(val); + buffers[col][row] = &date_storage[col][row]; + ind = sizeof(SQL_DATE_STRUCT); + break; + default: + return PyErr_Format(PyExc_TypeError, "Unhandled C type"); + } + } + } + } + + SQLSetStmtAttr(cursor->hstmt, SQL_ATTR_PARAM_BIND_TYPE, SQL_PARAM_BIND_BY_COLUMN, 0); + SQLSetStmtAttr(cursor->hstmt, SQL_ATTR_PARAMSET_SIZE, (SQLPOINTER)rowCount, 0); + + for (size_t col = 0; col < colCount; ++col) { + SQLBindParameter(cursor->hstmt, (SQLUSMALLINT)(col + 1), SQL_PARAM_INPUT, + c_types[col], SQL_UNKNOWN_TYPE, 0, 0, + (SQLPOINTER)buffers[col].data(), 0, indicators[col].data()); + } + + PyObject* encoded_sql = PyUnicode_AsEncodedString(sql, "utf-8", "strict"); + if (!encoded_sql) return nullptr; + const char* sql_text = PyBytes_AsString(encoded_sql); + RETCODE ret = SQLPrepare(cursor->hstmt, (SQLCHAR*)sql_text, SQL_NTS); + Py_DECREF(encoded_sql); + if (!SQL_SUCCEEDED(ret)) + return RaiseErrorFromHandle(cur->cnxn, "SQLPrepare", cursor->hstmt, SQL_HANDLE_STMT); + + ret = SQLExecute(cursor->hstmt); + if (!SQL_SUCCEEDED(ret)) + return RaiseErrorFromHandle(cur->cnxn, "SQLExecute", cursor->hstmt, SQL_HANDLE_STMT); + + Py_RETURN_NONE; +} + +static char bcp_init_doc[] = "bcp_init(table, database=None)\n\nInitializes the bulk copy operation for the specified table."; +static PyObject* Cursor_bcp_init(PyObject* self, PyObject* args, PyObject* kwargs) { + Cursor* cursor = (Cursor*)self; + Connection* conn = cursor->cnxn; + const char* table = nullptr; + const char* database = nullptr; + static const char* kwlist[] = {"table", "database", nullptr}; + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s|s", (char**)kwlist, &table, &database)) + return nullptr; + + RETCODE ret = bcp_init(conn->hdbc, (LPCSTR)table, nullptr, nullptr, DB_IN); + if (!SQL_SUCCEEDED(ret)) + return RaiseErrorFromHandle(cursor->cnxn, "bcp_init", conn->hdbc, SQL_HANDLE_DBC); + + Py_RETURN_NONE; +} + +static char bcp_bind_doc[] = "bcp_bind(value, column, c_type=SQL_C_CHAR, data_len=0)\n\nBinds a value to a BCP column. Supports strings, integers, floats, dates, datetimes, and unicode."; +static PyObject* Cursor_bcp_bind(PyObject* self, PyObject* args, PyObject* kwargs) { + Cursor* cursor = (Cursor*)self; + Connection* conn = cursor->cnxn; + PyObject* value; + int col_index; + int c_type = SQL_C_CHAR; + int data_len = 0; + + static const char* kwlist[] = {"value", "column", "c_type", "data_len", nullptr}; + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "Oi|ii", (char**)kwlist, &value, &col_index, &c_type, &data_len)) + return nullptr; + + void* buffer = nullptr; + DBINT cbData = 0; + + if (value == Py_None) { + buffer = nullptr; + cbData = SQL_NULL_DATA; // Indicates NULL to BCP API + } else if (PyUnicode_Check(value)) { + if (c_type == SQL_C_WCHAR) { + Py_ssize_t size = PyUnicode_GET_LENGTH(value); + wchar_t* wdata = (wchar_t*)malloc((size + 1) * sizeof(wchar_t)); + if (!wdata) return PyErr_NoMemory(); + Py_ssize_t copied = PyUnicode_AsWideChar(value, wdata, size); + if (copied < 0) { + free(wdata); + return nullptr; + } + wdata[copied] = 0; + buffer = wdata; + cbData = (DBINT)(copied * sizeof(wchar_t)); + } else { + PyObject* bytes = PyUnicode_AsEncodedString(value, "utf-8", "strict"); + if (!bytes) return nullptr; + buffer = malloc(PyBytes_GET_SIZE(bytes)); + memcpy(buffer, PyBytes_AS_STRING(bytes), PyBytes_GET_SIZE(bytes)); + cbData = (DBINT)PyBytes_GET_SIZE(bytes); + Py_DECREF(bytes); + } + } else if (PyBytes_Check(value)) { + buffer = malloc(PyBytes_GET_SIZE(value)); + memcpy(buffer, PyBytes_AS_STRING(value), PyBytes_GET_SIZE(value)); + cbData = (DBINT)PyBytes_GET_SIZE(value); + } else if (PyLong_Check(value)) { + buffer = malloc(sizeof(DBINT)); + *(DBINT*)buffer = (DBINT)PyLong_AsLong(value); + cbData = sizeof(DBINT); + } else if (PyFloat_Check(value)) { + buffer = malloc(sizeof(double)); + *(double*)buffer = PyFloat_AsDouble(value); + cbData = sizeof(double); + } else if (PyDateTime_Check(value)) { + SQL_TIMESTAMP_STRUCT* ts = (SQL_TIMESTAMP_STRUCT*)malloc(sizeof(SQL_TIMESTAMP_STRUCT)); + ts->year = PyDateTime_GET_YEAR(value); + ts->month = PyDateTime_GET_MONTH(value); + ts->day = PyDateTime_GET_DAY(value); + ts->hour = PyDateTime_DATE_GET_HOUR(value); + ts->minute = PyDateTime_DATE_GET_MINUTE(value); + ts->second = PyDateTime_DATE_GET_SECOND(value); + ts->fraction = PyDateTime_DATE_GET_MICROSECOND(value) * 1000; + buffer = ts; + cbData = sizeof(SQL_TIMESTAMP_STRUCT); + } else if (PyDate_Check(value)) { + SQL_DATE_STRUCT* ds = (SQL_DATE_STRUCT*)malloc(sizeof(SQL_DATE_STRUCT)); + ds->year = PyDateTime_GET_YEAR(value); + ds->month = PyDateTime_GET_MONTH(value); + ds->day = PyDateTime_GET_DAY(value); + buffer = ds; + cbData = sizeof(SQL_DATE_STRUCT); + } else if (PyTime_Check(value)) { + SQL_TIME_STRUCT* t = (SQL_TIME_STRUCT*)malloc(sizeof(SQL_TIME_STRUCT)); + t->hour = PyDateTime_TIME_GET_HOUR(value); + t->minute = PyDateTime_TIME_GET_MINUTE(value); + t->second = PyDateTime_TIME_GET_SECOND(value); + buffer = t; + cbData = sizeof(SQL_TIME_STRUCT); + } else { + PyErr_SetString(PyExc_TypeError, "Unsupported type for bcp_bind"); + return nullptr; + } + + // Track buffer to free later if not NULL + if (buffer != nullptr) + cursor->bcp_bound_buffers.push_back(buffer); + + RETCODE ret = bcp_bind(conn->hdbc, (BYTE*)buffer, 0, cbData, nullptr, 0, c_type, col_index); + if (!SQL_SUCCEEDED(ret)) + return RaiseErrorFromHandle(conn, "bcp_bind", conn->hdbc, SQL_HANDLE_DBC); + + Py_RETURN_NONE; +} + +static char bcp_sendrow_doc[] = "bcp_sendrow()\n\nSends a single row of bound data to the server."; +static PyObject* Cursor_bcp_sendrow(PyObject* self, PyObject* args) { + Cursor* cursor = (Cursor*)self; + Connection* conn = cursor->connection; + RETCODE ret = bcp_sendrow(conn->hdbc); + if (!SQL_SUCCEEDED(ret)) + return RaiseErrorFromHandle(conn, "bcp_sendrow", conn->hdbc, SQL_HANDLE_DBC); + Py_RETURN_NONE; +} + +static char bcp_batch_doc[] = "bcp_batch()\n\nSends all pending rows in the current batch to the server."; +static PyObject* Cursor_bcp_batch(PyObject* self, PyObject* args) { + Cursor* cursor = (Cursor*)self; + Connection* conn = cursor->connection; + DBINT result = bcp_batch(conn->hdbc); + if (result == -1) + return RaiseErrorFromHandle(conn, "bcp_batch", conn->hdbc, SQL_HANDLE_DBC); + return PyLong_FromLong(result); +} + +static char bcp_done_doc[] = "bcp_done()\n\nFinalizes the bulk copy operation and frees internal buffers."; +static PyObject* Cursor_bcp_done(PyObject* self, PyObject* args) { + Cursor* cursor = (Cursor*)self; + Connection* conn = cursor->connection; + DBINT result = bcp_done(conn->hdbc); + for (auto ptr : cursor->bcp_bound_buffers) + free(ptr); + cursor->bcp_bound_buffers.clear(); + if (result == -1) + return RaiseErrorFromHandle(conn, "bcp_done", conn->hdbc, SQL_HANDLE_DBC); + return PyLong_FromLong(result); +} + + static PyMethodDef Cursor_methods[] = { @@ -2411,6 +2704,12 @@ static PyMethodDef Cursor_methods[] = {"cancel", (PyCFunction)Cursor_cancel, METH_NOARGS, cancel_doc}, {"__enter__", Cursor_enter, METH_NOARGS, enter_doc }, {"__exit__", Cursor_exit, METH_VARARGS, exit_doc }, + {"column_bind_insert", (PyCFunction)Cursor_column_bind_insert, METH_VARARGS, column_bind_insert_doc}, + {"bcp_init", (PyCFunction)Cursor_bcp_init, METH_VARARGS | METH_KEYWORDS, bcp_init_doc }, + {"bcp_bind", (PyCFunction)Cursor_bcp_bind, METH_VARARGS | METH_KEYWORDS, bcp_bind_doc }, + {"bcp_sendrow", (PyCFunction)Cursor_bcp_sendrow, METH_NOARGS, bcp_sendrow_doc }, + {"bcp_batch", (PyCFunction)Cursor_bcp_batch, METH_NOARGS, bcp_batch_doc }, + {"bcp_done", (PyCFunction)Cursor_bcp_done, METH_NOARGS, bcp_done_doc }, {0, 0, 0, 0} }; diff --git a/src/cursor.h b/src/cursor.h index 657eb483..65e0a3a4 100644 --- a/src/cursor.h +++ b/src/cursor.h @@ -12,6 +12,8 @@ #ifndef CURSOR_H #define CURSOR_H +#undef max +#include struct Connection; @@ -154,6 +156,9 @@ struct Cursor // The messages attribute described in the DB API 2.0 specification. // Contains a list of all non-data messages provided by the driver, retrieved using SQLGetDiagRec. PyObject* messages; + + // A buffer for use during bcp data loads + std::vector bcp_bound_buffers; }; int GetDiagRecs(Cursor* cur); diff --git a/src/dbspecific.h b/src/dbspecific.h index e6af0a93..543d2b1d 100644 --- a/src/dbspecific.h +++ b/src/dbspecific.h @@ -40,4 +40,67 @@ struct PYSQLGUID byte Data4[8]; }; +// --------------------------------------------------------------------------- +// BCP ODBC Extension Definitions +// Minimal self-contained header replacements for Microsoft's bcp.h +// Allows pyODBC to call BCP APIs without additional SDK headers. +// --------------------------------------------------------------------------- + +#ifndef DBINT +typedef long DBINT; // 32-bit signed integer for BCP sizes/counts +#endif + +// Return codes +#ifndef BCP_SUCCESS +#define BCP_SUCCESS 0 +#endif +#ifndef BCP_ERROR +#define BCP_ERROR (-1) +#endif +#ifndef BCP_EOF +#define BCP_EOF (-2) +#endif + +// Directions for bcp_init +#ifndef DB_IN +#define DB_IN 1 // From program to SQL Server +#endif +#ifndef DB_OUT +#define DB_OUT 2 // From SQL Server to program +#endif + +// Options for bcp_control +#ifndef BCPMAXERRS +#define BCPMAXERRS 1 +#endif +#ifndef BCPFIRST +#define BCPFIRST 2 +#endif +#ifndef BCPLAST +#define BCPLAST 3 +#endif +#ifndef BCPBATCH +#define BCPBATCH 4 +#endif +#ifndef BCPKEEPNULLS +#define BCPKEEPNULLS 5 +#endif +#ifndef BCPABORT +#define BCPABORT 6 +#endif +#ifndef BCPHINTS +#define BCPHINTS 7 +#endif +#ifndef BCPKEEPIDENTITY +#define BCPKEEPIDENTITY 8 +#endif + +// Special data length markers for bcp_collen +#ifndef SQL_NULL_DATA +#define SQL_NULL_DATA (-1) +#endif +#ifndef SQL_VARLEN_DATA +#define SQL_VARLEN_DATA (-10) +#endif + #endif // DBSPECIFIC_H diff --git a/tests/sqlserver_test.py b/tests/sqlserver_test.py index ab51ce0d..01402d2b 100755 --- a/tests/sqlserver_test.py +++ b/tests/sqlserver_test.py @@ -1465,6 +1465,157 @@ def test_emoticons_as_literal(cursor: pyodbc.Cursor): assert result == v +def test_bcp_load_single_row(cursor: pyodbc.Cursor): + test_table = "bcp_test_single" + cursor.execute(f""" + CREATE TABLE {test_table} ( + id INT, + val VARCHAR(100), + dt DATETIME + ) + """) + cursor.bcp_init(test_table) + cursor.bcp_bind(1, 1) + cursor.bcp_bind('test value', 2) + cursor.bcp_bind(datetime.datetime(2025, 7, 28, 12, 34, 56), 3) + cursor.bcp_sendrow() + cursor.bcp_done() + + cursor.execute(f"SELECT id, val, dt FROM {test_table}") + row = cursor.fetchone() + assert row[0] == 1 + assert row[1] == 'test value' + assert row[2] == datetime.datetime(2025, 7, 28, 12, 34, 56) + cursor.execute(f"DROP TABLE {test_table}") + +def test_bcp_load_multiple_rows_and_batch(cursor: pyodbc.Cursor): + test_table = "bcp_test_multiple" + cursor.execute(f""" + CREATE TABLE {test_table} ( + id INT, + val VARCHAR(100), + dt DATETIME + ) + """) + cursor.bcp_init(test_table) + + for i in range(5): + cursor.bcp_bind(i, 1) + cursor.bcp_bind(f"row{i}", 2) + cursor.bcp_bind(datetime.datetime(2025, 1, i + 1, 0, 0, 0), 3) + cursor.bcp_sendrow() + + cursor.bcp_batch() + cursor.bcp_done() + + cursor.execute(f"SELECT COUNT(*) FROM {test_table}") + count = cursor.fetchone()[0] + assert count == 5 + cursor.execute(f"DROP TABLE {test_table}") + +def test_bcp_load_empty_rows(cursor: pyodbc.Cursor): + test_table = "bcp_test_empty" + cursor.execute(f"CREATE TABLE {test_table} (id INT)") + cursor.bcp_init(test_table) + # No binds or sendrow called + cursor.bcp_done() + cursor.execute(f"SELECT COUNT(*) FROM {test_table}") + count = cursor.fetchone()[0] + assert count == 0 + cursor.execute(f"DROP TABLE {test_table}") + +def test_bcp_bind_null_values(cursor: pyodbc.Cursor): + test_table = "bcp_test_nulls" + cursor.execute(f""" + CREATE TABLE {test_table} ( + id INT, + val VARCHAR(100), + dt DATETIME + ) + """) + cursor.bcp_init(test_table) + cursor.bcp_bind(None, 1) + cursor.bcp_bind(None, 2) + cursor.bcp_bind(None, 3) + cursor.bcp_sendrow() + cursor.bcp_done() + + cursor.execute(f"SELECT id, val, dt FROM {test_table}") + row = cursor.fetchone() + assert row[0] is None + assert row[1] is None + assert row[2] is None + cursor.execute(f"DROP TABLE {test_table}") + +def test_bcp_bind_invalid_type(cursor: pyodbc.Cursor): + test_table = "bcp_test_invalid" + cursor.execute(f"CREATE TABLE {test_table} (id INT)") + cursor.bcp_init(test_table) + + try: + cursor.bcp_bind(object(), 1) # Unsupported type + except TypeError: + pass + else: + assert False, "Expected TypeError for unsupported type" + + cursor.bcp_done() + cursor.execute(f"DROP TABLE {test_table}") + +def test_bcp_invalid_column_index(cursor: pyodbc.Cursor): + test_table = "bcp_test_invalid_col" + cursor.execute(f"CREATE TABLE {test_table} (id INT)") + cursor.bcp_init(test_table) + try: + cursor.bcp_bind(1, 99) # Invalid column index + except Exception as e: + assert "column" in str(e).lower() or "index" in str(e).lower() + else: + assert False, "Expected error for invalid column index" + cursor.bcp_done() + cursor.execute(f"DROP TABLE {test_table}") + + +def performance_test(cursor: pyodbc.Cursor): + # benchmark_column_bind_vs_fastexecmany.py + # Setup: drop/create test table + cursor.execute("DROP TABLE IF EXISTS test_perf") + cursor.execute(""" + CREATE TABLE test_perf + ( + id INT, + value FLOAT, + description VARCHAR(100), + created_at DATETIME + ) + """) + + # Generate data + rowcount = 10000 + data = [ + (i, float(i) * 1.1, f"row {i}", datetime.datetime(2020, 1, (i % 28) + 1, 12, 0, 0)) + for i in range(rowcount) + ] + + # Method 1: fast_executemany + cursor.fast_executemany = True + start = time.time() + cursor.executemany("INSERT INTO test_perf VALUES (?, ?, ?, ?)", data) + end = time.time() + print(f"fast_executemany: {end - start:.3f} seconds") + + # Method 2: column_bind_insert + # Assuming your pyodbc build exposes column_bind_insert on Cursor + start = time.time() + cursor.column_bind_insert("INSERT INTO test_perf VALUES (?, ?, ?, ?)", data) + end = time.time() + print(f"column_bind_insert: {end - start:.3f} seconds") + + # Verify inserted rows count + cursor.execute("SELECT COUNT(*) FROM test_perf") + print("Total rows inserted:", cursor.fetchone()[0]) + + def _test_tvp(cursor: pyodbc.Cursor, diff_schema): # Test table value parameters (TVP). I like the explanation here: #