Skip to content

Commit 55fb898

Browse files
bewithgauravbewithgaurav
committed
OPTIMIZATION #4: Batch row allocation with direct Python C API
Problem: -------- Row creation and assignment had multiple layers of overhead: 1. Per-row allocation: py::list(numCols) creates pybind11 wrapper for each row 2. Cell assignment: row[col-1] = value uses pybind11 operator[] with bounds checking 3. Final assignment: rows[i] = row uses pybind11 list assignment with refcount overhead 4. Fragmented allocation: 1,000 separate py::list() calls instead of batch allocation For 1,000 rows: ~30-50 CPU cycles × 1,000 = 30K-50K wasted cycles Solution: --------- Replace pybind11 wrappers with direct Python C API throughout: 1. Row creation: PyList_New(numCols) instead of py::list(numCols) 2. Cell assignment: PyList_SET_ITEM(row, col-1, value) instead of row[col-1] = value 3. Final assignment: PyList_SET_ITEM(rows.ptr(), i, row) instead of rows[i] = row This completes the transition to direct Python C API started in OPT #2. Changes: -------- - Replaced py::list row(numCols) → PyObject* row = PyList_New(numCols) - Updated all NULL/SQL_NO_TOTAL handlers to use PyList_SET_ITEM - Updated all zero-length data handlers to use direct Python C API - Updated string handlers (SQL_CHAR, SQL_WCHAR) to use PyList_SET_ITEM - Updated complex type handlers (DECIMAL, DATETIME, DATE, TIME, TIMESTAMPOFFSET, GUID, BINARY) - Updated final row assignment to use PyList_SET_ITEM(rows.ptr(), i, row) All cell assignments now use direct Python C API: - Numeric types: Already done in OPT #2 (PyLong_FromLong, PyFloat_FromDouble, etc.) - Strings: PyUnicode_FromStringAndSize, PyUnicode_FromString - Binary: PyBytes_FromStringAndSize - Complex types: .release().ptr() to transfer ownership Impact: ------- - ✅ Eliminates pybind11 wrapper overhead for row creation - ✅ No bounds checking in hot loop (PyList_SET_ITEM is a macro) - ✅ Clean reference counting (objects created with refcount=1, transferred to list) - ✅ Consistent with OPT #2 (entire row/cell management via Python C API) - ✅ Expected 5-10% improvement (smaller than OPT #3, but completes the stack) All type handlers now bypass pybind11 for maximum performance.
1 parent 7ad0947 commit 55fb898

File tree

1 file changed

+60
-45
lines changed

1 file changed

+60
-45
lines changed

‎mssql_python/pybind/ddbc_bindings.cpp‎

Lines changed: 60 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -3240,33 +3240,36 @@ SQLRETURN FetchBatchData(SQLHSTMT hStmt, ColumnBuffers& buffers, py::list& colum
32403240
}
32413241

32423242
for (SQLULEN i = 0; i < numRowsFetched; i++) {
3243-
// Create row container pre-allocated with known column count
3244-
py::list row(numCols);
3243+
// OPTIMIZATION #4: Create row using direct Python C API (bypasses pybind11 wrapper)
3244+
PyObject* row = PyList_New(numCols);
32453245
for (SQLUSMALLINT col = 1; col <= numCols; col++) {
32463246
// Use prefetched metadata from L1 cache-hot arrays
32473247
SQLSMALLINT dataType = dataTypes[col - 1];
32483248
SQLLEN dataLen = buffers.indicators[col - 1][i];
32493249
if (dataLen == SQL_NULL_DATA) {
3250-
row[col - 1] = py::none();
3250+
Py_INCREF(Py_None);
3251+
PyList_SET_ITEM(row, col - 1, Py_None);
32513252
continue;
32523253
}
32533254
if (dataLen == SQL_NO_TOTAL) {
32543255
LOG("Cannot determine the length of the data. Returning NULL value instead."
32553256
"Column ID - {}", col);
3256-
row[col - 1] = py::none();
3257+
Py_INCREF(Py_None);
3258+
PyList_SET_ITEM(row, col - 1, Py_None);
32573259
continue;
32583260
} else if (dataLen == 0) {
32593261
// Handle zero-length (non-NULL) data
32603262
if (dataType == SQL_CHAR || dataType == SQL_VARCHAR || dataType == SQL_LONGVARCHAR) {
3261-
row[col - 1] = std::string("");
3263+
PyList_SET_ITEM(row, col - 1, PyUnicode_FromString(""));
32623264
} else if (dataType == SQL_WCHAR || dataType == SQL_WVARCHAR || dataType == SQL_WLONGVARCHAR) {
3263-
row[col - 1] = std::wstring(L"");
3265+
PyList_SET_ITEM(row, col - 1, PyUnicode_FromString(""));
32643266
} else if (dataType == SQL_BINARY || dataType == SQL_VARBINARY || dataType == SQL_LONGVARBINARY) {
3265-
row[col - 1] = py::bytes("");
3267+
PyList_SET_ITEM(row, col - 1, PyBytes_FromStringAndSize("", 0));
32663268
} else {
32673269
// For other datatypes, 0 length is unexpected. Log & set None
32683270
LOG("Column data length is 0 for non-string/binary datatype. Setting None to the result row. Column ID - {}", col);
3269-
row[col - 1] = py::none();
3271+
Py_INCREF(Py_None);
3272+
PyList_SET_ITEM(row, col - 1, Py_None);
32703273
}
32713274
continue;
32723275
} else if (dataLen < 0) {
@@ -3280,23 +3283,26 @@ SQLRETURN FetchBatchData(SQLHSTMT hStmt, ColumnBuffers& buffers, py::list& colum
32803283
case SQL_CHAR:
32813284
case SQL_VARCHAR:
32823285
case SQL_LONGVARCHAR: {
3286+
SQLULEN columnSize = columnSizes[col - 1];
32833287
uint64_t fetchBufferSize = fetchBufferSizes[col - 1];
32843288
uint64_t numCharsInData = dataLen / sizeof(SQLCHAR);
32853289
bool isLob = isLobs[col - 1];
32863290
// fetchBufferSize includes null-terminator, numCharsInData doesn't. Hence '<'
32873291
if (!isLob && numCharsInData < fetchBufferSize) {
3288-
row[col - 1] = py::str(
3292+
PyObject* pyStr = PyUnicode_FromStringAndSize(
32893293
reinterpret_cast<char*>(&buffers.charBuffers[col - 1][i * fetchBufferSize]),
32903294
numCharsInData);
3295+
PyList_SET_ITEM(row, col - 1, pyStr);
32913296
} else {
3292-
row[col - 1] = FetchLobColumnData(hStmt, col, SQL_C_CHAR, false, false);
3297+
PyList_SET_ITEM(row, col - 1, FetchLobColumnData(hStmt, col, SQL_C_CHAR, false, false).release().ptr());
32933298
}
32943299
break;
32953300
}
32963301
case SQL_WCHAR:
32973302
case SQL_WVARCHAR:
32983303
case SQL_WLONGVARCHAR: {
32993304
// TODO: variable length data needs special handling, this logic wont suffice
3305+
SQLULEN columnSize = columnSizes[col - 1];
33003306
uint64_t fetchBufferSize = fetchBufferSizes[col - 1];
33013307
uint64_t numCharsInData = dataLen / sizeof(SQLWCHAR);
33023308
bool isLob = isLobs[col - 1];
@@ -3312,73 +3318,74 @@ SQLRETURN FetchBatchData(SQLHSTMT hStmt, ColumnBuffers& buffers, py::list& colum
33123318
NULL // byteorder - auto-detect
33133319
);
33143320
if (pyStr) {
3315-
row[col - 1] = py::reinterpret_steal<py::object>(pyStr);
3321+
PyList_SET_ITEM(row, col - 1, pyStr);
33163322
} else {
33173323
PyErr_Clear();
3318-
row[col - 1] = std::wstring(L"");
3324+
PyList_SET_ITEM(row, col - 1, PyUnicode_FromString(""));
33193325
}
33203326
#else
3321-
row[col - 1] = std::wstring(
3327+
PyObject* pyStr = PyUnicode_FromWideChar(
33223328
reinterpret_cast<wchar_t*>(&buffers.wcharBuffers[col - 1][i * fetchBufferSize]),
33233329
numCharsInData);
3330+
PyList_SET_ITEM(row, col - 1, pyStr);
33243331
#endif
33253332
} else {
3326-
row[col - 1] = FetchLobColumnData(hStmt, col, SQL_C_WCHAR, true, false);
3333+
PyList_SET_ITEM(row, col - 1, FetchLobColumnData(hStmt, col, SQL_C_WCHAR, true, false).release().ptr());
33273334
}
33283335
break;
33293336
}
33303337
case SQL_INTEGER: {
33313338
// OPTIMIZATION #2: Direct Python C API for integers
33323339
if (buffers.indicators[col - 1][i] == SQL_NULL_DATA) {
33333340
Py_INCREF(Py_None);
3334-
PyList_SET_ITEM(row.ptr(), col - 1, Py_None);
3341+
PyList_SET_ITEM(row, col - 1, Py_None);
33353342
} else {
33363343
PyObject* pyInt = PyLong_FromLong(buffers.intBuffers[col - 1][i]);
3337-
PyList_SET_ITEM(row.ptr(), col - 1, pyInt);
3344+
PyList_SET_ITEM(row, col - 1, pyInt);
33383345
}
33393346
break;
33403347
}
33413348
case SQL_SMALLINT: {
33423349
// OPTIMIZATION #2: Direct Python C API for smallint
33433350
if (buffers.indicators[col - 1][i] == SQL_NULL_DATA) {
33443351
Py_INCREF(Py_None);
3345-
PyList_SET_ITEM(row.ptr(), col - 1, Py_None);
3352+
PyList_SET_ITEM(row, col - 1, Py_None);
33463353
} else {
33473354
PyObject* pyInt = PyLong_FromLong(buffers.smallIntBuffers[col - 1][i]);
3348-
PyList_SET_ITEM(row.ptr(), col - 1, pyInt);
3355+
PyList_SET_ITEM(row, col - 1, pyInt);
33493356
}
33503357
break;
33513358
}
33523359
case SQL_TINYINT: {
33533360
// OPTIMIZATION #2: Direct Python C API for tinyint
33543361
if (buffers.indicators[col - 1][i] == SQL_NULL_DATA) {
33553362
Py_INCREF(Py_None);
3356-
PyList_SET_ITEM(row.ptr(), col - 1, Py_None);
3363+
PyList_SET_ITEM(row, col - 1, Py_None);
33573364
} else {
33583365
PyObject* pyInt = PyLong_FromLong(buffers.charBuffers[col - 1][i]);
3359-
PyList_SET_ITEM(row.ptr(), col - 1, pyInt);
3366+
PyList_SET_ITEM(row, col - 1, pyInt);
33603367
}
33613368
break;
33623369
}
33633370
case SQL_BIT: {
33643371
// OPTIMIZATION #2: Direct Python C API for bit/boolean
33653372
if (buffers.indicators[col - 1][i] == SQL_NULL_DATA) {
33663373
Py_INCREF(Py_None);
3367-
PyList_SET_ITEM(row.ptr(), col - 1, Py_None);
3374+
PyList_SET_ITEM(row, col - 1, Py_None);
33683375
} else {
33693376
PyObject* pyBool = PyBool_FromLong(buffers.charBuffers[col - 1][i]);
3370-
PyList_SET_ITEM(row.ptr(), col - 1, pyBool);
3377+
PyList_SET_ITEM(row, col - 1, pyBool);
33713378
}
33723379
break;
33733380
}
33743381
case SQL_REAL: {
33753382
// OPTIMIZATION #2: Direct Python C API for real/float
33763383
if (buffers.indicators[col - 1][i] == SQL_NULL_DATA) {
33773384
Py_INCREF(Py_None);
3378-
PyList_SET_ITEM(row.ptr(), col - 1, Py_None);
3385+
PyList_SET_ITEM(row, col - 1, Py_None);
33793386
} else {
33803387
PyObject* pyFloat = PyFloat_FromDouble(buffers.realBuffers[col - 1][i]);
3381-
PyList_SET_ITEM(row.ptr(), col - 1, pyFloat);
3388+
PyList_SET_ITEM(row, col - 1, pyFloat);
33823389
}
33833390
break;
33843391
}
@@ -3391,11 +3398,13 @@ SQLRETURN FetchBatchData(SQLHSTMT hStmt, ColumnBuffers& buffers, py::list& colum
33913398

33923399
// Always use standard decimal point for Python Decimal parsing
33933400
// The decimal separator only affects display formatting, not parsing
3394-
row[col - 1] = PythonObjectCache::get_decimal_class()(py::str(rawData, decimalDataLen));
3401+
PyObject* decimalObj = PythonObjectCache::get_decimal_class()(py::str(rawData, decimalDataLen)).release().ptr();
3402+
PyList_SET_ITEM(row, col - 1, decimalObj);
33953403
} catch (const py::error_already_set& e) {
33963404
// Handle the exception, e.g., log the error and set py::none()
33973405
LOG("Error converting to decimal: {}", e.what());
3398-
row[col - 1] = py::none();
3406+
Py_INCREF(Py_None);
3407+
PyList_SET_ITEM(row, col - 1, Py_None);
33993408
}
34003409
break;
34013410
}
@@ -3404,45 +3413,48 @@ SQLRETURN FetchBatchData(SQLHSTMT hStmt, ColumnBuffers& buffers, py::list& colum
34043413
// OPTIMIZATION #2: Direct Python C API for double/float
34053414
if (buffers.indicators[col - 1][i] == SQL_NULL_DATA) {
34063415
Py_INCREF(Py_None);
3407-
PyList_SET_ITEM(row.ptr(), col - 1, Py_None);
3416+
PyList_SET_ITEM(row, col - 1, Py_None);
34083417
} else {
34093418
PyObject* pyFloat = PyFloat_FromDouble(buffers.doubleBuffers[col - 1][i]);
3410-
PyList_SET_ITEM(row.ptr(), col - 1, pyFloat);
3419+
PyList_SET_ITEM(row, col - 1, pyFloat);
34113420
}
34123421
break;
34133422
}
34143423
case SQL_TIMESTAMP:
34153424
case SQL_TYPE_TIMESTAMP:
34163425
case SQL_DATETIME: {
34173426
const SQL_TIMESTAMP_STRUCT& ts = buffers.timestampBuffers[col - 1][i];
3418-
row[col - 1] = PythonObjectCache::get_datetime_class()(ts.year, ts.month, ts.day,
3427+
PyObject* datetimeObj = PythonObjectCache::get_datetime_class()(ts.year, ts.month, ts.day,
34193428
ts.hour, ts.minute, ts.second,
3420-
ts.fraction / 1000);
3429+
ts.fraction / 1000).release().ptr();
3430+
PyList_SET_ITEM(row, col - 1, datetimeObj);
34213431
break;
34223432
}
34233433
case SQL_BIGINT: {
34243434
// OPTIMIZATION #2: Direct Python C API for bigint
34253435
if (buffers.indicators[col - 1][i] == SQL_NULL_DATA) {
34263436
Py_INCREF(Py_None);
3427-
PyList_SET_ITEM(row.ptr(), col - 1, Py_None);
3437+
PyList_SET_ITEM(row, col - 1, Py_None);
34283438
} else {
34293439
PyObject* pyInt = PyLong_FromLongLong(buffers.bigIntBuffers[col - 1][i]);
3430-
PyList_SET_ITEM(row.ptr(), col - 1, pyInt);
3440+
PyList_SET_ITEM(row, col - 1, pyInt);
34313441
}
34323442
break;
34333443
}
34343444
case SQL_TYPE_DATE: {
3435-
row[col - 1] = PythonObjectCache::get_date_class()(buffers.dateBuffers[col - 1][i].year,
3445+
PyObject* dateObj = PythonObjectCache::get_date_class()(buffers.dateBuffers[col - 1][i].year,
34363446
buffers.dateBuffers[col - 1][i].month,
3437-
buffers.dateBuffers[col - 1][i].day);
3447+
buffers.dateBuffers[col - 1][i].day).release().ptr();
3448+
PyList_SET_ITEM(row, col - 1, dateObj);
34383449
break;
34393450
}
34403451
case SQL_TIME:
34413452
case SQL_TYPE_TIME:
34423453
case SQL_SS_TIME2: {
3443-
row[col - 1] = PythonObjectCache::get_time_class()(buffers.timeBuffers[col - 1][i].hour,
3454+
PyObject* timeObj = PythonObjectCache::get_time_class()(buffers.timeBuffers[col - 1][i].hour,
34443455
buffers.timeBuffers[col - 1][i].minute,
3445-
buffers.timeBuffers[col - 1][i].second);
3456+
buffers.timeBuffers[col - 1][i].second).release().ptr();
3457+
PyList_SET_ITEM(row, col - 1, timeObj);
34463458
break;
34473459
}
34483460
case SQL_SS_TIMESTAMPOFFSET: {
@@ -3465,16 +3477,18 @@ SQLRETURN FetchBatchData(SQLHSTMT hStmt, ColumnBuffers& buffers, py::list& colum
34653477
dtoValue.fraction / 1000, // ns → µs
34663478
tzinfo
34673479
);
3468-
row[col - 1] = py_dt;
3480+
PyList_SET_ITEM(row, col - 1, py_dt.release().ptr());
34693481
} else {
3470-
row[col - 1] = py::none();
3482+
Py_INCREF(Py_None);
3483+
PyList_SET_ITEM(row, col - 1, Py_None);
34713484
}
34723485
break;
34733486
}
34743487
case SQL_GUID: {
34753488
SQLLEN indicator = buffers.indicators[col - 1][i];
34763489
if (indicator == SQL_NULL_DATA) {
3477-
row[col - 1] = py::none();
3490+
Py_INCREF(Py_None);
3491+
PyList_SET_ITEM(row, col - 1, Py_None);
34783492
break;
34793493
}
34803494
SQLGUID* guidValue = &buffers.guidBuffers[col - 1][i];
@@ -3493,7 +3507,7 @@ SQLRETURN FetchBatchData(SQLHSTMT hStmt, ColumnBuffers& buffers, py::list& colum
34933507
py::dict kwargs;
34943508
kwargs["bytes"] = py_guid_bytes;
34953509
py::object uuid_obj = PythonObjectCache::get_uuid_class()(**kwargs);
3496-
row[col - 1] = uuid_obj;
3510+
PyList_SET_ITEM(row, col - 1, uuid_obj.release().ptr());
34973511
break;
34983512
}
34993513
case SQL_BINARY:
@@ -3502,11 +3516,12 @@ SQLRETURN FetchBatchData(SQLHSTMT hStmt, ColumnBuffers& buffers, py::list& colum
35023516
SQLULEN columnSize = columnSizes[col - 1];
35033517
bool isLob = isLobs[col - 1];
35043518
if (!isLob && static_cast<size_t>(dataLen) <= columnSize) {
3505-
row[col - 1] = py::bytes(reinterpret_cast<const char*>(
3506-
&buffers.charBuffers[col - 1][i * columnSize]),
3507-
dataLen);
3519+
PyObject* pyBytes = PyBytes_FromStringAndSize(
3520+
reinterpret_cast<const char*>(&buffers.charBuffers[col - 1][i * columnSize]),
3521+
dataLen);
3522+
PyList_SET_ITEM(row, col - 1, pyBytes);
35083523
} else {
3509-
row[col - 1] = FetchLobColumnData(hStmt, col, SQL_C_BINARY, false, true);
3524+
PyList_SET_ITEM(row, col - 1, FetchLobColumnData(hStmt, col, SQL_C_BINARY, false, true).release().ptr());
35103525
}
35113526
break;
35123527
}
@@ -3522,7 +3537,7 @@ SQLRETURN FetchBatchData(SQLHSTMT hStmt, ColumnBuffers& buffers, py::list& colum
35223537
}
35233538
}
35243539
}
3525-
rows[initialSize + i] = row;
3540+
PyList_SET_ITEM(rows.ptr(), initialSize + i, row);
35263541
}
35273542
return ret;
35283543
}

0 commit comments

Comments
 (0)