From 95eac916d7ca4119d6e566e88c0b26b0c699d2a0 Mon Sep 17 00:00:00 2001 From: Joshua Howard Date: Mon, 4 Nov 2024 15:17:51 -0600 Subject: [PATCH 1/2] Format patches/sqlite README as markdown --- patches/sqlite/{README => README.md} | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) rename patches/sqlite/{README => README.md} (69%) diff --git a/patches/sqlite/README b/patches/sqlite/README.md similarity index 69% rename from patches/sqlite/README rename to patches/sqlite/README.md index 43e8d9b5b86..1d129b66501 100644 --- a/patches/sqlite/README +++ b/patches/sqlite/README.md @@ -1,5 +1,6 @@ -The patches for SQLite are applied against the plain source, not the -amalgamated source. +# Patching SQLite + +The patches for SQLite are applied against the plain source, not the amalgamated source. SQLite comes in two forms. First, there's the "plain" form. It looks a lot like a typical open-source C project: there's a bunch of .c and .h @@ -17,21 +18,28 @@ any patches that needed modification with their fixed versions. Example, assuming the new SQLite has been downloaded into the current directory: -$ unzip sqlite-src-$VERSION.zip -$ mv sqlite-src-$VERSION sqlite-src-pristine -$ unzip sqlite-src-$VERSION.zip # yes, again -$ mv sqlite-src-$VERSION sqlite-src-modified +```bash +export VERSION=3470000 +unzip sqlite-src-$VERSION.zip +mv sqlite-src-$VERSION sqlite-src-pristine +unzip sqlite-src-$VERSION.zip # yes, again +mv sqlite-src-$VERSION sqlite-src-modified +``` Now patch: -$ cd sqlite-src-modified -$ patch -p1 < /path/to/workerd/patches/sqlite/0001-row-counts-plain.patch -$ ./configure && make test +```bash +cd sqlite-src-modified +patch -p1 < /path/to/workerd/patches/sqlite/0001-row-counts-plain.patch +./configure && make test +``` Make sure the tests pass. If the patch needed any modification, regenerate it: -$ diff -u5 -r sqlite-src-pristine sqlite-src-modified \ +```bash +diff -u5 -r sqlite-src-pristine sqlite-src-modified \ | grep -v "Only in sqlite-src-modified" \ > /path/to/workerd/patches/sqlite/0001-row-counts-plain.patch +``` Repeat for each patch. From 140323672dca97047a8a36ae11a1bf65fb92aa60 Mon Sep 17 00:00:00 2001 From: Joshua Howard Date: Mon, 4 Nov 2024 17:33:48 -0600 Subject: [PATCH 2/2] Update SQLite to version 3.47 --- WORKSPACE | 6 +- patches/sqlite/0001-row-counts-plain.patch | 131 ++++++++++----------- 2 files changed, 68 insertions(+), 69 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index 0b5723a1da1..f87babf1ac3 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -44,9 +44,9 @@ http_archive( "//:patches/sqlite/0002-macOS-missing-PATH-fix.patch", "//:patches/sqlite/0003-sqlite-complete-early-exit.patch", ], - sha256 = "ab9aae38a11b931f35d8d1c6d62826d215579892e6ffbf89f20bdce106a9c8c5", - strip_prefix = "sqlite-src-3440000", - url = "https://sqlite.org/2023/sqlite-src-3440000.zip", + sha256 = "f59c349bedb470203586a6b6d10adb35f2afefa49f91e55a672a36a09a8fedf7", + strip_prefix = "sqlite-src-3470000", + url = "https://sqlite.org/2024/sqlite-src-3470000.zip", ) load("@rules_python//python:repositories.bzl", "py_repositories", "python_register_toolchains") diff --git a/patches/sqlite/0001-row-counts-plain.patch b/patches/sqlite/0001-row-counts-plain.patch index 266d66ffb8d..58a0b53a562 100644 --- a/patches/sqlite/0001-row-counts-plain.patch +++ b/patches/sqlite/0001-row-counts-plain.patch @@ -1,26 +1,26 @@ -diff -u5 -r sqlite-src-3440000.pristine/src/shell.c.in sqlite-src-3440000/src/shell.c.in ---- sqlite-src-3440000.pristine/src/shell.c.in 2023-11-01 04:31:37.000000000 -0700 -+++ sqlite-src-3440000/src/shell.c.in 2023-11-03 15:06:27.994222697 -0700 -@@ -3381,10 +3381,15 @@ - raw_printf(pArg->out, "Reprepare operations: %d\n", iCur); - iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_RUN, bReset); - raw_printf(pArg->out, "Number of times run: %d\n", iCur); +diff -u5 -r sqlite-src-pristine/src/shell.c.in sqlite-src-modified/src/shell.c.in +--- sqlite-src-pristine/src/shell.c.in 2024-10-21 11:47:53 ++++ sqlite-src-modified/src/shell.c.in 2024-11-05 08:16:15 +@@ -3411,10 +3411,15 @@ + sqlite3_fprintf(out, + "Number of times run: %d\n", iCur); iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_MEMUSED, bReset); - raw_printf(pArg->out, "Memory used by prepared stmt: %d\n", iCur); + sqlite3_fprintf(out, + "Memory used by prepared stmt: %d\n", iCur); + + iCur = sqlite3_stmt_status(pArg->pStmt, LIBSQL_STMTSTATUS_ROWS_READ, bReset); -+ raw_printf(pArg->out, "Rows read: %d\n", iCur); ++ sqlite3_fprintf(pArg->out, "Rows read: %d\n", iCur); + iCur = sqlite3_stmt_status(pArg->pStmt, LIBSQL_STMTSTATUS_ROWS_WRITTEN, bReset); -+ raw_printf(pArg->out, "Rows written: %d\n", iCur); ++ sqlite3_fprintf(pArg->out, "Rows written: %d\n", iCur); } #ifdef __linux__ displayLinuxIoStats(pArg->out); #endif -diff -u5 -r sqlite-src-3440000.pristine/src/sqlite.h.in sqlite-src-3440000/src/sqlite.h.in ---- sqlite-src-3440000.pristine/src/sqlite.h.in 2023-11-01 04:31:37.000000000 -0700 -+++ sqlite-src-3440000/src/sqlite.h.in 2023-11-03 15:06:27.994222697 -0700 -@@ -8815,10 +8815,18 @@ +diff -u5 -r sqlite-src-pristine/src/sqlite.h.in sqlite-src-modified/src/sqlite.h.in +--- sqlite-src-pristine/src/sqlite.h.in 2024-10-21 11:47:53 ++++ sqlite-src-modified/src/sqlite.h.in 2024-11-05 08:13:50 +@@ -8907,20 +8907,32 @@ ** used to store the prepared statement. ^This value is not actually ** a counter, and so the resetFlg parameter to sqlite3_stmt_status() ** is ignored when the opcode is SQLITE_STMTSTATUS_MEMUSED. @@ -39,55 +39,24 @@ diff -u5 -r sqlite-src-3440000.pristine/src/sqlite.h.in sqlite-src-3440000/src/s #define SQLITE_STMTSTATUS_SORT 2 #define SQLITE_STMTSTATUS_AUTOINDEX 3 #define SQLITE_STMTSTATUS_VM_STEP 4 -@@ -8826,10 +8834,14 @@ + #define SQLITE_STMTSTATUS_REPREPARE 5 #define SQLITE_STMTSTATUS_RUN 6 #define SQLITE_STMTSTATUS_FILTER_MISS 7 #define SQLITE_STMTSTATUS_FILTER_HIT 8 #define SQLITE_STMTSTATUS_MEMUSED 99 - ++ +#define LIBSQL_STMTSTATUS_BASE 1024 +#define LIBSQL_STMTSTATUS_ROWS_READ LIBSQL_STMTSTATUS_BASE + 1 +#define LIBSQL_STMTSTATUS_ROWS_WRITTEN LIBSQL_STMTSTATUS_BASE + 2 -+ + /* ** CAPI3REF: Custom Page Cache Object ** ** The sqlite3_pcache type is opaque. It is implemented by - ** the pluggable module. The SQLite core has no knowledge of -diff -u5 -r sqlite-src-3440000.pristine/src/vdbeapi.c sqlite-src-3440000/src/vdbeapi.c ---- sqlite-src-3440000.pristine/src/vdbeapi.c 2023-11-01 04:31:37.000000000 -0700 -+++ sqlite-src-3440000/src/vdbeapi.c 2023-11-03 15:23:46.669473054 -0700 -@@ -2036,11 +2036,11 @@ - int sqlite3_stmt_status(sqlite3_stmt *pStmt, int op, int resetFlag){ - Vdbe *pVdbe = (Vdbe*)pStmt; - u32 v; - #ifdef SQLITE_ENABLE_API_ARMOR - if( !pStmt -- || (op!=SQLITE_STMTSTATUS_MEMUSED && (op<0||op>=ArraySize(pVdbe->aCounter))) -+ || (op!=SQLITE_STMTSTATUS_MEMUSED && (op<0||(op>=ArraySize(pVdbe->aCounter)&&oplookaside.pEnd = db->lookaside.pStart; - sqlite3VdbeDelete(pVdbe); - db->pnBytesFreed = 0; - db->lookaside.pEnd = db->lookaside.pTrueEnd; - sqlite3_mutex_leave(db->mutex); -+ }else if( op>=LIBSQL_STMTSTATUS_BASE ){ -+ v = pVdbe->aLibsqlCounter[op - LIBSQL_STMTSTATUS_BASE]; -+ if( resetFlag ) pVdbe->aLibsqlCounter[op - LIBSQL_STMTSTATUS_BASE] = 0; - }else{ - v = pVdbe->aCounter[op]; - if( resetFlag ) pVdbe->aCounter[op] = 0; - } - return (int)v; -diff -u5 -r sqlite-src-3440000.pristine/src/vdbe.c sqlite-src-3440000/src/vdbe.c ---- sqlite-src-3440000.pristine/src/vdbe.c 2023-11-01 04:31:37.000000000 -0700 -+++ sqlite-src-3440000/src/vdbe.c 2024-01-30 10:49:55.735518937 -0800 -@@ -3708,10 +3708,11 @@ +diff -u5 -r sqlite-src-pristine/src/vdbe.c sqlite-src-modified/src/vdbe.c +--- sqlite-src-pristine/src/vdbe.c 2024-10-21 11:47:53 ++++ sqlite-src-modified/src/vdbe.c 2024-11-05 08:13:50 +@@ -3737,10 +3737,11 @@ if( pOp->p3 ){ nEntry = sqlite3BtreeRowCountEst(pCrsr); }else{ @@ -99,7 +68,7 @@ diff -u5 -r sqlite-src-3440000.pristine/src/vdbe.c sqlite-src-3440000/src/vdbe.c pOut = out2Prerelease(p, pOp); pOut->u.i = nEntry; goto check_for_interrupt; -@@ -4867,10 +4868,11 @@ +@@ -4900,10 +4901,11 @@ if( eqOnly && r.eqSeen==0 ){ assert( res!=0 ); goto seek_not_found; @@ -111,7 +80,7 @@ diff -u5 -r sqlite-src-3440000.pristine/src/vdbe.c sqlite-src-3440000/src/vdbe.c #endif if( oc>=OP_SeekGE ){ assert( oc==OP_SeekGE || oc==OP_SeekGT ); if( res<0 || (res==0 && oc==OP_SeekGT) ){ -@@ -5436,10 +5438,11 @@ +@@ -5470,10 +5472,11 @@ pC->nullRow = 0; pC->cacheStatus = CACHE_STALE; pC->deferredMoveto = 0; @@ -123,7 +92,7 @@ diff -u5 -r sqlite-src-3440000.pristine/src/vdbe.c sqlite-src-3440000/src/vdbe.c if( pOp->p2==0 ){ rc = SQLITE_CORRUPT_BKPT; }else{ -@@ -5693,10 +5696,11 @@ +@@ -5727,10 +5730,11 @@ } if( pOp->p5 & OPFLAG_ISNOOP ) break; #endif @@ -135,7 +104,7 @@ diff -u5 -r sqlite-src-3440000.pristine/src/vdbe.c sqlite-src-3440000/src/vdbe.c if( pOp->p5 & OPFLAG_LASTROWID ) db->lastRowid = x.nKey; } assert( (pData->flags & (MEM_Blob|MEM_Str))!=0 || pData->n==0 ); -@@ -5886,10 +5890,11 @@ +@@ -5920,10 +5924,11 @@ pC->seekResult = 0; if( rc ) goto abort_due_to_error; @@ -147,7 +116,7 @@ diff -u5 -r sqlite-src-3440000.pristine/src/vdbe.c sqlite-src-3440000/src/vdbe.c db->xUpdateCallback(db->pUpdateArg, SQLITE_DELETE, zDb, pTab->zName, pC->movetoTarget); assert( pC->iDb>=0 ); -@@ -6173,10 +6178,11 @@ +@@ -6207,10 +6212,11 @@ rc = sqlite3BtreeLast(pCrsr, &res); pC->nullRow = (u8)res; pC->deferredMoveto = 0; @@ -159,7 +128,7 @@ diff -u5 -r sqlite-src-3440000.pristine/src/vdbe.c sqlite-src-3440000/src/vdbe.c if( res ) goto jump_to_p2; } break; -@@ -6282,10 +6288,11 @@ +@@ -6326,10 +6332,11 @@ pC->deferredMoveto = 0; pC->cacheStatus = CACHE_STALE; } @@ -171,7 +140,7 @@ diff -u5 -r sqlite-src-3440000.pristine/src/vdbe.c sqlite-src-3440000/src/vdbe.c if( res ) goto jump_to_p2; } break; -@@ -6387,10 +6394,11 @@ +@@ -6431,10 +6438,11 @@ pC->cacheStatus = CACHE_STALE; VdbeBranchTaken(rc==SQLITE_OK,2); if( rc==SQLITE_OK ){ @@ -183,7 +152,7 @@ diff -u5 -r sqlite-src-3440000.pristine/src/vdbe.c sqlite-src-3440000/src/vdbe.c #endif goto jump_to_p2_and_check_for_interrupt; } -@@ -6438,10 +6446,11 @@ +@@ -6482,10 +6490,11 @@ assert( pC!=0 ); assert( !isSorter(pC) ); pIn2 = &aMem[pOp->p2]; @@ -195,7 +164,7 @@ diff -u5 -r sqlite-src-3440000.pristine/src/vdbe.c sqlite-src-3440000/src/vdbe.c rc = ExpandBlob(pIn2); if( rc ) goto abort_due_to_error; x.nKey = pIn2->n; -@@ -6838,10 +6847,11 @@ +@@ -6882,10 +6891,11 @@ assert( p->readOnly==0 ); assert( DbMaskTest(p->btreeMask, pOp->p2) ); rc = sqlite3BtreeClearTable(db->aDb[pOp->p2].pBt, (u32)pOp->p1, &nChange); @@ -207,7 +176,7 @@ diff -u5 -r sqlite-src-3440000.pristine/src/vdbe.c sqlite-src-3440000/src/vdbe.c memAboutToChange(p, &aMem[pOp->p3]); aMem[pOp->p3].u.i += nChange; } -@@ -8391,10 +8401,11 @@ +@@ -8466,10 +8476,11 @@ ** some other method is next invoked on the save virtual table cursor. */ rc = pModule->xNext(pCur->uc.pVCur); @@ -219,7 +188,7 @@ diff -u5 -r sqlite-src-3440000.pristine/src/vdbe.c sqlite-src-3440000/src/vdbe.c if( !res ){ /* If there is data, jump to P2 */ goto jump_to_p2_and_check_for_interrupt; -@@ -8512,10 +8523,11 @@ +@@ -8587,10 +8598,11 @@ rc = SQLITE_OK; }else{ p->errorAction = ((pOp->p5==OE_Replace) ? OE_Abort : pOp->p5); @@ -231,9 +200,9 @@ diff -u5 -r sqlite-src-3440000.pristine/src/vdbe.c sqlite-src-3440000/src/vdbe.c if( rc ) goto abort_due_to_error; } break; -diff -u5 -r sqlite-src-3440000.pristine/src/vdbeInt.h sqlite-src-3440000/src/vdbeInt.h ---- sqlite-src-3440000.pristine/src/vdbeInt.h 2023-11-01 04:31:37.000000000 -0700 -+++ sqlite-src-3440000/src/vdbeInt.h 2023-11-03 15:06:27.998222610 -0700 +diff -u5 -r sqlite-src-pristine/src/vdbeInt.h sqlite-src-modified/src/vdbeInt.h +--- sqlite-src-pristine/src/vdbeInt.h 2024-10-21 11:47:53 ++++ sqlite-src-modified/src/vdbeInt.h 2024-11-05 08:13:50 @@ -496,10 +496,11 @@ bft bIsReader:1; /* True for statements that read */ bft haveEqpOps:1; /* Bytecode supports EXPLAIN QUERY PLAN */ @@ -246,3 +215,33 @@ diff -u5 -r sqlite-src-3440000.pristine/src/vdbeInt.h sqlite-src-3440000/src/vdb char *zNormSql; /* Normalization of the associated SQL statement */ DblquoteStr *pDblStr; /* List of double-quoted string literals */ #endif +diff -u5 -r sqlite-src-pristine/src/vdbeapi.c sqlite-src-modified/src/vdbeapi.c +--- sqlite-src-pristine/src/vdbeapi.c 2024-10-21 11:47:53 ++++ sqlite-src-modified/src/vdbeapi.c 2024-11-05 08:13:50 +@@ -2072,11 +2072,11 @@ + int sqlite3_stmt_status(sqlite3_stmt *pStmt, int op, int resetFlag){ + Vdbe *pVdbe = (Vdbe*)pStmt; + u32 v; + #ifdef SQLITE_ENABLE_API_ARMOR + if( !pStmt +- || (op!=SQLITE_STMTSTATUS_MEMUSED && (op<0||op>=ArraySize(pVdbe->aCounter))) ++ || (op!=SQLITE_STMTSTATUS_MEMUSED && (op<0||(op>=ArraySize(pVdbe->aCounter)&&oplookaside.pEnd = db->lookaside.pStart; + sqlite3VdbeDelete(pVdbe); + db->pnBytesFreed = 0; + db->lookaside.pEnd = db->lookaside.pTrueEnd; + sqlite3_mutex_leave(db->mutex); ++ }else if( op>=LIBSQL_STMTSTATUS_BASE ){ ++ v = pVdbe->aLibsqlCounter[op - LIBSQL_STMTSTATUS_BASE]; ++ if( resetFlag ) pVdbe->aLibsqlCounter[op - LIBSQL_STMTSTATUS_BASE] = 0; + }else{ + v = pVdbe->aCounter[op]; + if( resetFlag ) pVdbe->aCounter[op] = 0; + } + return (int)v;