Skip to content

Commit

Permalink
Implement comdb2_filenames
Browse files Browse the repository at this point in the history
Signed-off-by: Morgan Douglas <mdouglas47@bloomberg.net>
  • Loading branch information
morgando committed Sep 26, 2024
1 parent b597b19 commit 7dbbaba
Show file tree
Hide file tree
Showing 7 changed files with 243 additions and 76 deletions.
1 change: 1 addition & 0 deletions sqlite/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ add_library(sqlite
ext/comdb2/crons.c
ext/comdb2/ezsystables.c
ext/comdb2/files.c
ext/comdb2/filenames.c
ext/comdb2/fingerprints.c
ext/comdb2/functions.c
ext/comdb2/indexuse.c
Expand Down
1 change: 1 addition & 0 deletions sqlite/ext/comdb2/comdb2systblInt.h
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ extern const sqlite3_module systblLogicalOpsModule;
extern const sqlite3_module systblSystabsModule;
extern const sqlite3_module systblFdbInfoModule;
extern const sqlite3_module systblFilesModule;
extern const sqlite3_module systblFilenamesModule;
extern sqlite3_module systblSchemaVersionsModule;
extern sqlite3_module systblTablePermissionsModule;
extern sqlite3_module systblSystabPermissionsModule;
Expand Down
124 changes: 124 additions & 0 deletions sqlite/ext/comdb2/filenames.c
Original file line number Diff line number Diff line change
@@ -0,0 +1,124 @@
#include <assert.h>
#include <comdb2systblInt.h>
#include <ezsystables.h>
#include "files.h"

static int filenamesBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo)
{
const int rc = filesBestIndex(tab, pIdxInfo);
assert(rc != SQLITE_OK
|| pIdxInfo->idxNum == 0
|| pIdxInfo->idxNum == FILES_FILE_PATTERN_FLAG);

return rc;
}

static int filenamesConnect(sqlite3 *db, void *pAux, int argc,
const char *const *argv, sqlite3_vtab **ppVtab,
char **pzErr)
{
sqlite3_vtab *pNew;
int rc;

rc = sqlite3_declare_vtab( db, "CREATE TABLE x(filename, dir, type)");
if (rc == SQLITE_OK) {
pNew = *ppVtab = sqlite3_malloc(sizeof(*pNew));
if (pNew == 0) return SQLITE_NOMEM;
memset(pNew, 0, sizeof(*pNew));
}
return rc;
}

static int filenamesDisconnect(sqlite3_vtab *pVtab)
{
sqlite3_free(pVtab);
return SQLITE_OK;
}

static int filenamesOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor)
{
return filesOpen(p, ppCursor);
}

static int filenamesClose(sqlite3_vtab_cursor *cur)
{
return filesClose(cur);
}

static int filenamesFilter(sqlite3_vtab_cursor *pVtabCursor, int idxNum,
const char *idxStr, int argc, sqlite3_value **argv)
{
assert(idxNum == 0
|| idxNum == FILES_FILE_PATTERN_FLAG);

const int just_get_filenames = 1;
return _filesFilter(pVtabCursor, idxNum, idxStr, argc, argv, just_get_filenames);
}

static int filenamesEof(sqlite3_vtab_cursor *cur)
{
systbl_files_cursor *pCur = (systbl_files_cursor *)cur;
return (pCur->rowid >= pCur->nfiles) ? 1 : 0;
}

static int filenamesNext(sqlite3_vtab_cursor *cur)
{
systbl_files_cursor * const pCur = (systbl_files_cursor *) cur;
logmsg(LOGMSG_DEBUG, "%s:%d done processing %s\n", __func__, __LINE__,
pCur->files[pCur->rowid].name);
++pCur->rowid;
if (!filenamesEof(cur)) {
logmsg(LOGMSG_DEBUG, "%s:%d processing %s\n", __func__, __LINE__,
pCur->files[pCur->rowid].name);
}
return SQLITE_OK;
}

static int
filenamesColumn(sqlite3_vtab_cursor *cur, /* The cursor */
sqlite3_context *ctx, /* First argument to sqlite3_result_...() */
int i /* Which column to return */
) {
assert(i == FILES_COLUMN_FILENAME
|| i == FILES_COLUMN_DIR
|| i == FILES_COLUMN_TYPE);
return filesColumn(cur, ctx, i);
}

static int filenamesRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid)
{
systbl_files_cursor *pCur = (systbl_files_cursor *)cur;
*pRowid = pCur->rowid;
return SQLITE_OK;
}

/*
** This following structure defines all the methods for the
** generate_series virtual table.
*/
const sqlite3_module systblFilenamesModule = {
0, /* iVersion */
0, /* xCreate */
filenamesConnect, /* xConnect */
filenamesBestIndex, /* xBestIndex */
filenamesDisconnect, /* xDisconnect */
0, /* xDestroy */
filenamesOpen, /* xOpen - open a cursor */
filenamesClose, /* xClose - close a cursor */
filenamesFilter, /* xFilter - configure scan constraints */
filenamesNext, /* xNext - advance a cursor */
filenamesEof, /* xEof - check for end of scan */
filenamesColumn, /* xColumn - read data */
filenamesRowid, /* xRowid - read data */
0, /* xUpdate */
0, /* xBegin */
0, /* xSync */
0, /* xCommit */
0, /* xRollback */
0, /* xFindMethod */
0, /* xRename */
0, /* xSavepoint */
0, /* xRelease */
0, /* xRollbackTo */
0, /* xShadowName */
.access_flag = CDB2_ALLOW_USER};
84 changes: 21 additions & 63 deletions sqlite/ext/comdb2/files.c
Original file line number Diff line number Diff line change
Expand Up @@ -26,13 +26,12 @@
#include <comdb2systblInt.h>
#include <ezsystables.h>
#include <bb_oscompat.h>
#include <comdb2.h>
#include <errno.h>
#include "ar_wrap.h"
#include "cdb2_constants.h"
#include "sqliteInt.h"
#include "sql.h"
#include "list.h"

#include "files.h"

typedef unsigned char u_int8_t;

Expand All @@ -48,58 +47,6 @@ extern int patternCompare(const u8 *, const u8 *, const struct compareInfo *,
extern struct dbenv *thedb;
extern char gbl_dbname[MAX_DBNAME_LENGTH];

/* Column numbers */
enum {
FILES_COLUMN_FILENAME,
FILES_COLUMN_DIR,
FILES_COLUMN_TYPE,
FILES_COLUMN_CONTENT,
FILES_COLUMN_CONTENT_OFFSET,
FILES_COLUMN_CONTENT_SIZE,
FILES_COLUMN_CHUNK_SIZE,
};

enum {
FILES_FILE_PATTERN_FLAG = 1,
FILES_CHUNK_SIZE_FLAG = 2,
};

enum {
FILES_TYPE_UNKNOWN,
FILES_TYPE_CHECKPOINT,
FILES_TYPE_BERKDB,
FILES_TYPE_LOGFILE, /* Always keep LOGFILE in the end for them to sort
higher */
};

typedef struct db_chunk {
uint8_t *buffer;
size_t size;
size_t offset;
LINKC_T(struct db_chunk) lnk;
} db_chunk_t;

typedef struct db_file {
char *name; /* Name of the file */
char *dir; /* Name of the directory */
int type;
int chunk_seq;
struct db_chunk current_chunk;
dbfile_info *info;
} db_file_t;

typedef struct {
sqlite3_vtab_cursor base;
sqlite3_int64 rowid;
db_file_t *files;
size_t nfiles;
off_t content_offset;
size_t content_size;
size_t chunk_size;
char *file_pattern;
struct log_delete_state log_delete_state;
} systbl_files_cursor;

static const char *print_file_type(int type)
{
switch (type) {
Expand Down Expand Up @@ -492,7 +439,7 @@ static int read_dir(const char *dirname, db_file_t **files, int *count, char *fi
return rc;
}

static int get_files(void **data, size_t *npoints, char *file_pattern, size_t chunk_size)
int get_files(void **data, size_t *npoints, char *file_pattern, size_t chunk_size)
{
db_file_t *files = NULL;
int count = 0;
Expand Down Expand Up @@ -531,7 +478,7 @@ static int filesDisconnect(sqlite3_vtab *pVtab)
return SQLITE_OK;
}

static int filesOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor)
int filesOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor)
{
systbl_files_cursor *pCur;

Expand All @@ -554,7 +501,7 @@ static int filesOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor)
return SQLITE_OK;
}

static int filesClose(sqlite3_vtab_cursor *cur)
int filesClose(sqlite3_vtab_cursor *cur)
{
systbl_files_cursor *pCur = (systbl_files_cursor *)cur;

Expand All @@ -575,7 +522,7 @@ static int filesNext(sqlite3_vtab_cursor *cur)
return (read_next_chunk(pCur)) ? SQLITE_ERROR : SQLITE_OK;
}

static int
int
filesColumn(sqlite3_vtab_cursor *cur, /* The cursor */
sqlite3_context *ctx, /* First argument to sqlite3_result_...() */
int i /* Which column to return */
Expand Down Expand Up @@ -639,8 +586,8 @@ static int file_cmp(const void *file1, const void *file2)
return strcmp(f1->name, f2->name);
}

static int filesFilter(sqlite3_vtab_cursor *pVtabCursor, int idxNum,
const char *idxStr, int argc, sqlite3_value **argv)
int _filesFilter(sqlite3_vtab_cursor *pVtabCursor, int idxNum,
const char *idxStr, int argc, sqlite3_value **argv, const int just_get_filenames)
{
systbl_files_cursor *pCur = (systbl_files_cursor *)pVtabCursor;
int i = 0;
Expand All @@ -666,11 +613,22 @@ static int filesFilter(sqlite3_vtab_cursor *pVtabCursor, int idxNum,
pCur->rowid = 0;

qsort(pCur->files, pCur->nfiles, sizeof(db_file_t), file_cmp);

if (just_get_filenames) {
return SQLITE_OK;
} else {
return (read_next_chunk(pCur)) ? SQLITE_ERROR : SQLITE_OK;
}
}

return (read_next_chunk(pCur)) ? SQLITE_ERROR : SQLITE_OK;
static int filesFilter(sqlite3_vtab_cursor *pVtabCursor, int idxNum,
const char *idxStr, int argc, sqlite3_value **argv)
{
const int just_get_filenames = 0;
return _filesFilter(pVtabCursor, idxNum, idxStr, argc, argv, just_get_filenames);
}

static int filesBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo)
int filesBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo)
{
int i; /* Loop over constraints */
int idxNum = 0; /* The query plan bitmask */
Expand Down
68 changes: 68 additions & 0 deletions sqlite/ext/comdb2/files.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
#ifndef __COMDB2_FILES_H_
#define __COMDB2_FILES_H_

#include <comdb2.h>
#include "list.h"
#include "ar_wrap.h"

/* Column numbers */
enum {
FILES_COLUMN_FILENAME,
FILES_COLUMN_DIR,
FILES_COLUMN_TYPE,
FILES_COLUMN_CONTENT,
FILES_COLUMN_CONTENT_OFFSET,
FILES_COLUMN_CONTENT_SIZE,
FILES_COLUMN_CHUNK_SIZE,
};

enum {
FILES_FILE_PATTERN_FLAG = 1,
FILES_CHUNK_SIZE_FLAG = 2,
};

enum {
FILES_TYPE_UNKNOWN,
FILES_TYPE_CHECKPOINT,
FILES_TYPE_BERKDB,
FILES_TYPE_LOGFILE, /* Always keep LOGFILE in the end for them to sort
higher */
};

typedef struct db_chunk {
uint8_t *buffer;
size_t size;
size_t offset;
LINKC_T(struct db_chunk) lnk;
} db_chunk_t;

typedef struct db_file {
char *name; /* Name of the file */
char *dir; /* Name of the directory */
int type;
int chunk_seq;
struct db_chunk current_chunk;
dbfile_info *info;
} db_file_t;

typedef struct {
sqlite3_vtab_cursor base;
sqlite3_int64 rowid;
db_file_t *files;
size_t nfiles;
off_t content_offset;
size_t content_size;
size_t chunk_size;
char *file_pattern;
struct log_delete_state log_delete_state;
} systbl_files_cursor;

int get_files(void **data, size_t *npoints, char *file_pattern, size_t chunk_size);
int filesOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor);
int filesClose(sqlite3_vtab_cursor *cur);
int filesColumn(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int i);
int filesBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo);
int _filesFilter(sqlite3_vtab_cursor *pVtabCursor, int idxNum,
const char *idxStr, int argc, sqlite3_value **argv, const int just_get_filenames);

#endif
2 changes: 2 additions & 0 deletions sqlite/ext/comdb2/tables.c
Original file line number Diff line number Diff line change
Expand Up @@ -168,6 +168,8 @@ int comdb2SystblInit(
rc = systblTablePropertiesInit(db);
if (rc == SQLITE_OK)
rc = sqlite3_create_module(db, "comdb2_files", &systblFilesModule, 0);
if (rc == SQLITE_OK)
rc = sqlite3_create_module(db, "comdb2_filenames", &systblFilenamesModule, 0);
if (rc == SQLITE_OK)
rc = systblTimepartInit(db);
if (rc == SQLITE_OK)
Expand Down
Loading

0 comments on commit 7dbbaba

Please sign in to comment.