From fca6bb0813f707f48c287260d4094e5a90a84d76 Mon Sep 17 00:00:00 2001 From: Ibrar Ahmed Date: Mon, 2 Jun 2014 18:14:20 +0500 Subject: [PATCH 01/43] Connection Pooler for mongodb similar to PostgreSQL FDW. This will avoid multiple connection and disconnection to MongoDB Server. FDW's Options functions separated from "mongo_fdw.c" and placed in "option.c" file. The newly created connection's pooling functions placed into "connection.c" file. --- Makefile | 2 +- mongo_fdw.c | 245 ++++++---------------------------------------------- mongo_fdw.h | 28 +++++- 3 files changed, 56 insertions(+), 219 deletions(-) diff --git a/Makefile b/Makefile index b4543d8..82bf10c 100644 --- a/Makefile +++ b/Makefile @@ -17,7 +17,7 @@ MONGO_OBJS = $(MONGO_PATH)/bson.os $(MONGO_PATH)/encoding.os $(MONGO_PATH)/md5.o $(MONGO_PATH)/mongo.os $(MONGO_PATH)/numbers.os $(MONGO_PATH)/env_posix.os PG_CPPFLAGS = --std=c99 -I$(MONGO_PATH) -OBJS = mongo_fdw.o mongo_query.o $(MONGO_OBJS) +OBJS = connection.o option.o mongo_fdw.o mongo_query.o $(MONGO_OBJS) EXTENSION = mongo_fdw DATA = mongo_fdw--1.0.sql diff --git a/mongo_fdw.c b/mongo_fdw.c index daa5438..634e4d1 100644 --- a/mongo_fdw.c +++ b/mongo_fdw.c @@ -26,6 +26,7 @@ #include "optimizer/plancat.h" #include "optimizer/planmain.h" #include "optimizer/restrictinfo.h" +#include "storage/ipc.h" #include "utils/array.h" #include "utils/builtins.h" #include "utils/date.h" @@ -33,6 +34,7 @@ #include "utils/lsyscache.h" #include "utils/rel.h" #include "utils/memutils.h" +#include "storage/ipc.h" #if PG_VERSION_NUM >= 90300 #include "access/htup_details.h" @@ -40,7 +42,6 @@ /* Local functions forward declarations */ -static StringInfo OptionNamesString(Oid currentContextId); static void MongoGetForeignRelSize(PlannerInfo *root, RelOptInfo *baserel, Oid foreignTableId); static void MongoGetForeignPaths(PlannerInfo *root, RelOptInfo *baserel, @@ -57,8 +58,6 @@ static void MongoReScanForeignScan(ForeignScanState *scanState); static Const * SerializeDocument(bson *document); static bson * DeserializeDocument(Const *constant); static double ForeignTableDocumentCount(Oid foreignTableId); -static MongoFdwOptions * MongoGetOptions(Oid foreignTableId); -static char * MongoGetOptionValue(Oid foreignTableId, const char *optionName); static HTAB * ColumnMappingHash(Oid foreignTableId, List *columnList); static void FillTupleSlot(const bson *bsonDocument, const char *bsonDocumentKey, HTAB *columnMappingHash, Datum *columnValues, @@ -74,14 +73,25 @@ static bool MongoAnalyzeForeignTable(Relation relation, static int MongoAcquireSampleRows(Relation relation, int errorLevel, HeapTuple *sampleRows, int targetRowCount, double *totalRowCount, double *totalDeadRowCount); +static void mongo_fdw_exit(int code, Datum arg); + +extern PGDLLEXPORT void _PG_init(void); /* declarations for dynamic loading */ PG_MODULE_MAGIC; PG_FUNCTION_INFO_V1(mongo_fdw_handler); -PG_FUNCTION_INFO_V1(mongo_fdw_validator); +/* + * Library load-time initalization, sets on_proc_exit() callback for + * backend shutdown. + */ +void +_PG_init(void) +{ + on_proc_exit(&mongo_fdw_exit, PointerGetDatum(NULL)); +} /* * mongo_fdw_handler creates and returns a struct with pointers to foreign table @@ -91,7 +101,6 @@ Datum mongo_fdw_handler(PG_FUNCTION_ARGS) { FdwRoutine *fdwRoutine = makeNode(FdwRoutine); - fdwRoutine->GetForeignRelSize = MongoGetForeignRelSize; fdwRoutine->GetForeignPaths = MongoGetForeignPaths; fdwRoutine->GetForeignPlan = MongoGetForeignPlan; @@ -105,92 +114,13 @@ mongo_fdw_handler(PG_FUNCTION_ARGS) PG_RETURN_POINTER(fdwRoutine); } - /* - * mongo_fdw_validator validates options given to one of the following commands: - * foreign data wrapper, server, user mapping, or foreign table. This function - * errors out if the given option name or its value is considered invalid. + * Exit callback function. */ -Datum -mongo_fdw_validator(PG_FUNCTION_ARGS) -{ - Datum optionArray = PG_GETARG_DATUM(0); - Oid optionContextId = PG_GETARG_OID(1); - List *optionList = untransformRelOptions(optionArray); - ListCell *optionCell = NULL; - - foreach(optionCell, optionList) - { - DefElem *optionDef = (DefElem *) lfirst(optionCell); - char *optionName = optionDef->defname; - bool optionValid = false; - - int32 optionIndex = 0; - for (optionIndex = 0; optionIndex < ValidOptionCount; optionIndex++) - { - const MongoValidOption *validOption = &(ValidOptionArray[optionIndex]); - - if ((optionContextId == validOption->optionContextId) && - (strncmp(optionName, validOption->optionName, NAMEDATALEN) == 0)) - { - optionValid = true; - break; - } - } - - /* if invalid option, display an informative error message */ - if (!optionValid) - { - StringInfo optionNamesString = OptionNamesString(optionContextId); - - ereport(ERROR, (errcode(ERRCODE_FDW_INVALID_OPTION_NAME), - errmsg("invalid option \"%s\"", optionName), - errhint("Valid options in this context are: %s", - optionNamesString->data))); - } - - /* if port option is given, error out if its value isn't an integer */ - if (strncmp(optionName, OPTION_NAME_PORT, NAMEDATALEN) == 0) - { - char *optionValue = defGetString(optionDef); - int32 portNumber = pg_atoi(optionValue, sizeof(int32), 0); - (void) portNumber; - } - } - - PG_RETURN_VOID(); -} - - -/* - * OptionNamesString finds all options that are valid for the current context, - * and concatenates these option names in a comma separated string. - */ -static StringInfo -OptionNamesString(Oid currentContextId) +static void +mongo_fdw_exit(int code, Datum arg) { - StringInfo optionNamesString = makeStringInfo(); - bool firstOptionPrinted = false; - - int32 optionIndex = 0; - for (optionIndex = 0; optionIndex < ValidOptionCount; optionIndex++) - { - const MongoValidOption *validOption = &(ValidOptionArray[optionIndex]); - - /* if option belongs to current context, append option name */ - if (currentContextId == validOption->optionContextId) - { - if (firstOptionPrinted) - { - appendStringInfoString(optionNamesString, ", "); - } - - appendStringInfoString(optionNamesString, validOption->optionName); - firstOptionPrinted = true; - } - } - - return optionNamesString; + cleanup_connection(); } @@ -390,13 +320,11 @@ MongoBeginForeignScan(ForeignScanState *scanState, int executorFlags) { mongo *mongoConnection = NULL; mongo_cursor *mongoCursor = NULL; - int32 connectStatus = MONGO_ERROR; Oid foreignTableId = InvalidOid; List *columnList = NIL; HTAB *columnMappingHash = NULL; char *addressName = NULL; int32 portNumber = 0; - int32 errorCode = 0; StringInfo namespaceName = NULL; ForeignScan *foreignScan = NULL; List *foreignPrivateList = NIL; @@ -418,22 +346,11 @@ MongoBeginForeignScan(ForeignScanState *scanState, int executorFlags) addressName = mongoFdwOptions->addressName; portNumber = mongoFdwOptions->portNumber; - mongoConnection = mongo_create(); - mongo_init(mongoConnection); - - connectStatus = mongo_connect(mongoConnection, addressName, portNumber); - if (connectStatus != MONGO_OK) - { - errorCode = (int32) mongoConnection->err; - - mongo_destroy(mongoConnection); - mongo_dispose(mongoConnection); - - ereport(ERROR, (errmsg("could not connect to %s:%d", addressName, portNumber), - errhint("Mongo driver connection error: %d", errorCode))); - } - - /* deserialize query document; and create column info hash */ + /* + * Get connection to the foreign server. Connection manager will + * establish new connection if necessary. + */ + mongoConnection = GetConnection(addressName, portNumber); foreignScan = (ForeignScan *) scanState->ss.ps.plan; foreignPrivateList = foreignScan->fdw_private; Assert(list_length(foreignPrivateList) == 2); @@ -641,122 +558,18 @@ ForeignTableDocumentCount(Oid foreignTableId) MongoFdwOptions *options = NULL; mongo *mongoConnection = NULL; const bson *emptyQuery = NULL; - int32 status = MONGO_ERROR; double documentCount = 0.0; /* resolve foreign table options; and connect to mongo server */ options = MongoGetOptions(foreignTableId); - mongoConnection = mongo_create(); - mongo_init(mongoConnection); - - status = mongo_connect(mongoConnection, options->addressName, options->portNumber); - if (status == MONGO_OK) - { - documentCount = mongo_count(mongoConnection, options->databaseName, - options->collectionName, emptyQuery); - } - else - { - documentCount = -1.0; - } - - mongo_destroy(mongoConnection); - mongo_dispose(mongoConnection); - + mongoConnection = GetConnection(options->addressName, options->portNumber); + documentCount = mongo_count(mongoConnection, options->databaseName, + options->collectionName, emptyQuery); return documentCount; } -/* - * MongoGetOptions returns the option values to be used when connecting to and - * querying MongoDB. To resolve these values, the function checks the foreign - * table's options, and if not present, falls back to default values. - */ -static MongoFdwOptions * -MongoGetOptions(Oid foreignTableId) -{ - MongoFdwOptions *mongoFdwOptions = NULL; - char *addressName = NULL; - char *portName = NULL; - int32 portNumber = 0; - char *databaseName = NULL; - char *collectionName = NULL; - - addressName = MongoGetOptionValue(foreignTableId, OPTION_NAME_ADDRESS); - if (addressName == NULL) - { - addressName = pstrdup(DEFAULT_IP_ADDRESS); - } - - portName = MongoGetOptionValue(foreignTableId, OPTION_NAME_PORT); - if (portName == NULL) - { - portNumber = DEFAULT_PORT_NUMBER; - } - else - { - portNumber = pg_atoi(portName, sizeof(int32), 0); - } - - databaseName = MongoGetOptionValue(foreignTableId, OPTION_NAME_DATABASE); - if (databaseName == NULL) - { - databaseName = pstrdup(DEFAULT_DATABASE_NAME); - } - - collectionName = MongoGetOptionValue(foreignTableId, OPTION_NAME_COLLECTION); - if (collectionName == NULL) - { - collectionName = get_rel_name(foreignTableId); - } - - mongoFdwOptions = (MongoFdwOptions *) palloc0(sizeof(MongoFdwOptions)); - mongoFdwOptions->addressName = addressName; - mongoFdwOptions->portNumber = portNumber; - mongoFdwOptions->databaseName = databaseName; - mongoFdwOptions->collectionName = collectionName; - - return mongoFdwOptions; -} - - -/* - * MongoGetOptionValue walks over foreign table and foreign server options, and - * looks for the option with the given name. If found, the function returns the - * option's value. - */ -static char * -MongoGetOptionValue(Oid foreignTableId, const char *optionName) -{ - ForeignTable *foreignTable = NULL; - ForeignServer *foreignServer = NULL; - List *optionList = NIL; - ListCell *optionCell = NULL; - char *optionValue = NULL; - - foreignTable = GetForeignTable(foreignTableId); - foreignServer = GetForeignServer(foreignTable->serverid); - - optionList = list_concat(optionList, foreignTable->options); - optionList = list_concat(optionList, foreignServer->options); - - foreach(optionCell, optionList) - { - DefElem *optionDef = (DefElem *) lfirst(optionCell); - char *optionDefName = optionDef->defname; - - if (strncmp(optionDefName, optionName, NAMEDATALEN) == 0) - { - optionValue = defGetString(optionDef); - break; - } - } - - return optionValue; -} - - /* * ColumnMappingHash creates a hash table that maps column names to column index * and types. This table helps us quickly translate BSON document key/values to @@ -1197,10 +1010,8 @@ MongoFreeScanState(MongoFdwExecState *executionState) mongo_cursor_destroy(executionState->mongoCursor); mongo_cursor_dispose(executionState->mongoCursor); - - /* also close the connection to mongo server */ - mongo_destroy(executionState->mongoConnection); - mongo_dispose(executionState->mongoConnection); + /* Release remote connection */ + ReleaseConnection(executionState->mongoConnection); } diff --git a/mongo_fdw.h b/mongo_fdw.h index 60947a2..52c5ef7 100644 --- a/mongo_fdw.h +++ b/mongo_fdw.h @@ -15,6 +15,8 @@ #include "bson.h" #include "mongo.h" +#include "postgres.h" +#include "utils/hsearch.h" #include "fmgr.h" #include "catalog/pg_foreign_server.h" #include "catalog/pg_foreign_table.h" @@ -22,6 +24,27 @@ #include "nodes/pg_list.h" #include "nodes/relation.h" #include "utils/timestamp.h" +#include "access/reloptions.h" +#include "catalog/pg_type.h" +#include "commands/defrem.h" +#include "commands/explain.h" +#include "commands/vacuum.h" +#include "foreign/fdwapi.h" +#include "foreign/foreign.h" +#include "nodes/makefuncs.h" +#include "optimizer/cost.h" +#include "optimizer/pathnode.h" +#include "optimizer/plancat.h" +#include "optimizer/planmain.h" +#include "optimizer/restrictinfo.h" +#include "utils/array.h" +#include "utils/builtins.h" +#include "utils/date.h" +#include "utils/hsearch.h" +#include "utils/lsyscache.h" +#include "utils/rel.h" +#include "utils/memutils.h" + /* Defines for valid option names */ @@ -113,7 +136,6 @@ typedef struct ColumnMapping Oid columnTypeId; int32 columnTypeMod; Oid columnArrayTypeId; - } ColumnMapping; @@ -125,6 +147,10 @@ extern List * ColumnList(RelOptInfo *baserel); /* Function declarations for foreign data wrapper */ extern Datum mongo_fdw_handler(PG_FUNCTION_ARGS); extern Datum mongo_fdw_validator(PG_FUNCTION_ARGS); +extern MongoFdwOptions * MongoGetOptions(Oid foreignTableId); +mongo* GetConnection(char *host, int32 port); +void cleanup_connection(void); +void ReleaseConnection(mongo *conn); #endif /* MONGO_FDW_H */ From 955350c200433888aa02b81b38c363685b21b830 Mon Sep 17 00:00:00 2001 From: Ibrar Ahmed Date: Mon, 2 Jun 2014 18:15:58 +0500 Subject: [PATCH 02/43] Plemove DeserializeDocument and SerializeDocument functions which are causing the server to crash while query is used in any container, i.e in PlpgSQl function. --- mongo_fdw.c | 79 +++++++---------------------------------------------- 1 file changed, 10 insertions(+), 69 deletions(-) diff --git a/mongo_fdw.c b/mongo_fdw.c index 634e4d1..4b81a89 100644 --- a/mongo_fdw.c +++ b/mongo_fdw.c @@ -55,8 +55,6 @@ static void MongoBeginForeignScan(ForeignScanState *scanState, int executorFlags static TupleTableSlot * MongoIterateForeignScan(ForeignScanState *scanState); static void MongoEndForeignScan(ForeignScanState *scanState); static void MongoReScanForeignScan(ForeignScanState *scanState); -static Const * SerializeDocument(bson *document); -static bson * DeserializeDocument(Const *constant); static double ForeignTableDocumentCount(Oid foreignTableId); static HTAB * ColumnMappingHash(Oid foreignTableId, List *columnList); static void FillTupleSlot(const bson *bsonDocument, const char *bsonDocumentKey, @@ -247,7 +245,6 @@ MongoGetForeignPlan(PlannerInfo *root, RelOptInfo *baserel, Oid foreignTableId, List *foreignPrivateList = NIL; List *opExpressionList = NIL; bson *queryDocument = NULL; - Const *queryBuffer = NULL; List *columnList = NIL; /* @@ -267,22 +264,22 @@ MongoGetForeignPlan(PlannerInfo *root, RelOptInfo *baserel, Oid foreignTableId, */ opExpressionList = ApplicableOpExpressionList(baserel); queryDocument = QueryDocument(foreignTableId, opExpressionList); - queryBuffer = SerializeDocument(queryDocument); - - /* only clean up the query struct, but not its data */ - bson_dispose(queryDocument); /* we don't need to serialize column list as lists are copiable */ columnList = ColumnList(baserel); /* construct foreign plan with query document and column list */ - foreignPrivateList = list_make2(queryBuffer, columnList); + foreignPrivateList = list_make2(columnList, restrictionClauses); + + /* only clean up the query struct, but not its data */ + bson_dispose(queryDocument); /* create the foreign scan node */ foreignScan = make_foreignscan(targetList, restrictionClauses, scanRangeTableIndex, NIL, /* no expressions to evaluate */ foreignPrivateList); + return foreignScan; } @@ -328,7 +325,6 @@ MongoBeginForeignScan(ForeignScanState *scanState, int executorFlags) StringInfo namespaceName = NULL; ForeignScan *foreignScan = NULL; List *foreignPrivateList = NIL; - Const *queryBuffer = NULL; bson *queryDocument = NULL; MongoFdwOptions *mongoFdwOptions = NULL; MongoFdwExecState *executionState = NULL; @@ -353,12 +349,12 @@ MongoBeginForeignScan(ForeignScanState *scanState, int executorFlags) mongoConnection = GetConnection(addressName, portNumber); foreignScan = (ForeignScan *) scanState->ss.ps.plan; foreignPrivateList = foreignScan->fdw_private; - Assert(list_length(foreignPrivateList) == 2); + Assert(list_length(foreignPrivateList) == 1); + + columnList = list_nth(foreignPrivateList, 0); - queryBuffer = (Const *) linitial(foreignPrivateList); - queryDocument = DeserializeDocument(queryBuffer); + queryDocument = QueryDocument(foreignTableId, NIL); - columnList = (List *) lsecond(foreignPrivateList); columnMappingHash = ColumnMappingHash(foreignTableId, columnList); namespaceName = makeStringInfo(); @@ -496,57 +492,6 @@ MongoReScanForeignScan(ForeignScanState *scanState) executionState->mongoCursor = mongoCursor; } - -/* - * SerializeDocument serializes the document's data to a constant, as advised in - * foreign/fdwapi.h. Note that this function shallow-copies the document's data; - * and the caller should therefore not free it. - */ -static Const * -SerializeDocument(bson *document) -{ - Const *serializedDocument = NULL; - Datum documentDatum = 0; - - /* - * We access document data and wrap a datum around it. Note that even when - * we have an empty document, the document size can't be zero according to - * bson apis. - */ - const char *documentData = bson_data(document); - int32 documentSize = bson_buffer_size(document); - Assert(documentSize != 0); - - documentDatum = CStringGetDatum(documentData); - serializedDocument = makeConst(CSTRINGOID, -1, InvalidOid, documentSize, - documentDatum, false, false); - - return serializedDocument; -} - - -/* - * DeserializeDocument deserializes the constant to a bson document. For this, - * the function creates a document, and explicitly sets the document's data. - */ -static bson * -DeserializeDocument(Const *constant) -{ - bson *document = NULL; - Datum documentDatum = constant->constvalue; - char *documentData = DatumGetCString(documentDatum); - - Assert(constant->constlen > 0); - Assert(constant->constisnull == false); - - document = bson_create(); - bson_init_size(document, 0); - bson_init_finished_data(document, documentData); - - return document; -} - - /* * ForeignTableDocumentCount connects to the MongoDB server, and queries it for * the number of documents in the foreign collection. On success, the function @@ -1096,7 +1041,6 @@ MongoAcquireSampleRows(Relation relation, int errorLevel, HTAB *columnMappingHash = NULL; mongo_cursor *mongoCursor = NULL; bson *queryDocument = NULL; - Const *queryBuffer = NULL; List *columnList = NIL; ForeignScanState *scanState = NULL; List *foreignPrivateList = NIL; @@ -1130,14 +1074,11 @@ MongoAcquireSampleRows(Relation relation, int errorLevel, foreignTableId = RelationGetRelid(relation); queryDocument = QueryDocument(foreignTableId, NIL); - queryBuffer = SerializeDocument(queryDocument); + foreignPrivateList = list_make1(columnList); /* only clean up the query struct, but not its data */ bson_dispose(queryDocument); - /* construct foreign plan with query document and column list */ - foreignPrivateList = list_make2(queryBuffer, columnList); - foreignScan = makeNode(ForeignScan); foreignScan->fdw_private = foreignPrivateList; From d8d8f48cb8050c36b4b755335d0880e4dc64343f Mon Sep 17 00:00:00 2001 From: Ibrar Ahmed Date: Mon, 2 Jun 2014 18:18:40 +0500 Subject: [PATCH 03/43] Write support (INSERT/UPDATE/DELETE). --- mongo_fdw.c | 654 ++++++++++++++++++++++++++++++++++++++++++++++---- mongo_fdw.h | 35 ++- mongo_query.c | 85 ++++--- mongo_query.h | 2 +- 4 files changed, 679 insertions(+), 97 deletions(-) diff --git a/mongo_fdw.c b/mongo_fdw.c index 4b81a89..75593eb 100644 --- a/mongo_fdw.c +++ b/mongo_fdw.c @@ -12,6 +12,12 @@ #include "postgres.h" #include "mongo_fdw.h" +#include "mongo_query.h" + +#include "postgres.h" +#include "bson.h" +#include "mongo_fdw.h" +#include "mongo_query.h" #include "access/reloptions.h" #include "catalog/pg_type.h" @@ -34,8 +40,27 @@ #include "utils/lsyscache.h" #include "utils/rel.h" #include "utils/memutils.h" -#include "storage/ipc.h" - +#include "access/sysattr.h" +#include "commands/defrem.h" +#include "commands/explain.h" +#include "commands/vacuum.h" +#include "foreign/fdwapi.h" +#include "funcapi.h" +#include "miscadmin.h" +#include "nodes/makefuncs.h" +#include "nodes/nodeFuncs.h" +#include "optimizer/cost.h" +#include "optimizer/pathnode.h" +#include "optimizer/paths.h" +#include "optimizer/planmain.h" +#include "optimizer/prep.h" +#include "optimizer/restrictinfo.h" +#include "optimizer/var.h" +#include "parser/parsetree.h" +#include "utils/builtins.h" +#include "utils/guc.h" +#include "utils/lsyscache.h" +#include "utils/memutils.h" #if PG_VERSION_NUM >= 90300 #include "access/htup_details.h" #endif @@ -55,16 +80,56 @@ static void MongoBeginForeignScan(ForeignScanState *scanState, int executorFlags static TupleTableSlot * MongoIterateForeignScan(ForeignScanState *scanState); static void MongoEndForeignScan(ForeignScanState *scanState); static void MongoReScanForeignScan(ForeignScanState *scanState); + +static TupleTableSlot *MongoExecForeignUpdate(EState *estate, + ResultRelInfo *resultRelInfo, + TupleTableSlot *slot, + TupleTableSlot *planSlot); +static TupleTableSlot *MongoExecForeignDelete(EState *estate, + ResultRelInfo *resultRelInfo, + TupleTableSlot *slot, + TupleTableSlot *planSlot); +static void MongoEndForeignModify(EState *estate, + ResultRelInfo *resultRelInfo); + +static void MongoAddForeignUpdateTargets(Query *parsetree, + RangeTblEntry *target_rte, + Relation target_relation); + +static void MongoBeginForeignModify(ModifyTableState *mtstate, + ResultRelInfo *resultRelInfo, + List *fdw_private, + int subplan_index, + int eflags); + +static TupleTableSlot *MongoExecForeignInsert(EState *estate, + ResultRelInfo *resultRelInfo, + TupleTableSlot *slot, + TupleTableSlot *planSlot); + +static List *MongoPlanForeignModify(PlannerInfo *root, + ModifyTable *plan, + Index resultRelation, + int subplan_index); + +static void +MongoExplainForeignModify(ModifyTableState *mtstate, + ResultRelInfo *rinfo, + List *fdw_private, + int subplan_index, + ExplainState *es); + +/* local functions */ static double ForeignTableDocumentCount(Oid foreignTableId); static HTAB * ColumnMappingHash(Oid foreignTableId, List *columnList); static void FillTupleSlot(const bson *bsonDocument, const char *bsonDocumentKey, - HTAB *columnMappingHash, Datum *columnValues, - bool *columnNulls); + HTAB *columnMappingHash, Datum *columnValues, + bool *columnNulls); static bool ColumnTypesCompatible(bson_type bsonType, Oid columnTypeId); static Datum ColumnValueArray(bson_iterator *bsonIterator, Oid valueTypeId); static Datum ColumnValue(bson_iterator *bsonIterator, Oid columnTypeId, int32 columnTypeMod); -static void MongoFreeScanState(MongoFdwExecState *executionState); +static void MongoFreeScanState(MongoFdwModifyState *fmstate); static bool MongoAnalyzeForeignTable(Relation relation, AcquireSampleRowsFunc *acquireSampleRowsFunc, BlockNumber *totalPageCount); @@ -76,6 +141,20 @@ static void mongo_fdw_exit(int code, Datum arg); extern PGDLLEXPORT void _PG_init(void); +/* + * Generate new 24 character's rowid (MongoDB's "_id") + * from a string using pad character 'A'. + */ +#define UNIQUE_OID(x, z) do \ +{ \ + char y[25]; \ + memset(y, 'A', sizeof(y));\ + y[24] = 0; \ + strcpy(y, x); \ + bson_oid_from_string(&z, y);\ +} while(0); + + /* declarations for dynamic loading */ PG_MODULE_MAGIC; @@ -102,13 +181,29 @@ mongo_fdw_handler(PG_FUNCTION_ARGS) fdwRoutine->GetForeignRelSize = MongoGetForeignRelSize; fdwRoutine->GetForeignPaths = MongoGetForeignPaths; fdwRoutine->GetForeignPlan = MongoGetForeignPlan; - fdwRoutine->ExplainForeignScan = MongoExplainForeignScan; fdwRoutine->BeginForeignScan = MongoBeginForeignScan; fdwRoutine->IterateForeignScan = MongoIterateForeignScan; fdwRoutine->ReScanForeignScan = MongoReScanForeignScan; fdwRoutine->EndForeignScan = MongoEndForeignScan; fdwRoutine->AnalyzeForeignTable = MongoAnalyzeForeignTable; + /* support for insert / update / delete */ + fdwRoutine->ExecForeignInsert = MongoExecForeignInsert; + fdwRoutine->BeginForeignModify = MongoBeginForeignModify; + fdwRoutine->PlanForeignModify = MongoPlanForeignModify; + fdwRoutine->AddForeignUpdateTargets = MongoAddForeignUpdateTargets; + fdwRoutine->ExecForeignUpdate = MongoExecForeignUpdate; + fdwRoutine->ExecForeignDelete = MongoExecForeignDelete; + fdwRoutine->EndForeignModify = MongoEndForeignModify; + + /* support for EXPLAIN */ + fdwRoutine->ExplainForeignScan = MongoExplainForeignScan; + fdwRoutine->ExplainForeignModify = NULL; + fdwRoutine->ExplainForeignModify = MongoExplainForeignModify; + + /* support for ANALYSE */ + fdwRoutine->AnalyzeForeignTable = MongoAnalyzeForeignTable; + PG_RETURN_POINTER(fdwRoutine); } @@ -275,7 +370,7 @@ MongoGetForeignPlan(PlannerInfo *root, RelOptInfo *baserel, Oid foreignTableId, bson_dispose(queryDocument); /* create the foreign scan node */ - foreignScan = make_foreignscan(targetList, restrictionClauses, + foreignScan = make_foreignscan(targetList, restrictionClauses, scanRangeTableIndex, NIL, /* no expressions to evaluate */ foreignPrivateList); @@ -302,9 +397,34 @@ MongoExplainForeignScan(ForeignScanState *scanState, ExplainState *explainState) appendStringInfo(namespaceName, "%s.%s", mongoFdwOptions->databaseName, mongoFdwOptions->collectionName); + MongoFreeOptions(mongoFdwOptions); + ExplainPropertyText("Foreign Namespace", namespaceName->data, explainState); } +static void +MongoExplainForeignModify(ModifyTableState *mtstate, + ResultRelInfo *rinfo, + List *fdw_private, + int subplan_index, + ExplainState *es) +{ + MongoFdwOptions *mongoFdwOptions = NULL; + StringInfo namespaceName = NULL; + Oid foreignTableId = InvalidOid; + + foreignTableId = RelationGetRelid(rinfo->ri_RelationDesc); + mongoFdwOptions = MongoGetOptions(foreignTableId); + + /* construct fully qualified collection name */ + namespaceName = makeStringInfo(); + appendStringInfo(namespaceName, "%s.%s", mongoFdwOptions->databaseName, + mongoFdwOptions->collectionName); + + MongoFreeOptions(mongoFdwOptions); + ExplainPropertyText("Foreign Namespace", namespaceName->data, es); +} + /* * MongoBeginForeignScan connects to the MongoDB server, and opens a cursor that @@ -327,7 +447,7 @@ MongoBeginForeignScan(ForeignScanState *scanState, int executorFlags) List *foreignPrivateList = NIL; bson *queryDocument = NULL; MongoFdwOptions *mongoFdwOptions = NULL; - MongoFdwExecState *executionState = NULL; + MongoFdwModifyState *fmstate = NULL; /* if Explain with no Analyze, do nothing */ if (executorFlags & EXEC_FLAG_EXPLAIN_ONLY) @@ -367,13 +487,14 @@ MongoBeginForeignScan(ForeignScanState *scanState, int executorFlags) mongo_cursor_set_query(mongoCursor, queryDocument); /* create and set foreign execution state */ - executionState = (MongoFdwExecState *) palloc0(sizeof(MongoFdwExecState)); - executionState->columnMappingHash = columnMappingHash; - executionState->mongoConnection = mongoConnection; - executionState->mongoCursor = mongoCursor; - executionState->queryDocument = queryDocument; - - scanState->fdw_state = (void *) executionState; + fmstate = (MongoFdwModifyState *) palloc0(sizeof(MongoFdwModifyState)); + fmstate->columnMappingHash = columnMappingHash; + fmstate->mongoConnection = mongoConnection; + fmstate->mongoCursor = mongoCursor; + fmstate->queryDocument = queryDocument; + fmstate->mongoFdwOptions = mongoFdwOptions; + + scanState->fdw_state = (void *) fmstate; } @@ -385,10 +506,10 @@ MongoBeginForeignScan(ForeignScanState *scanState, int executorFlags) static TupleTableSlot * MongoIterateForeignScan(ForeignScanState *scanState) { - MongoFdwExecState *executionState = (MongoFdwExecState *) scanState->fdw_state; + MongoFdwModifyState *fmstate = (MongoFdwModifyState *) scanState->fdw_state; TupleTableSlot *tupleSlot = scanState->ss.ss_ScanTupleSlot; - mongo_cursor *mongoCursor = executionState->mongoCursor; - HTAB *columnMappingHash = executionState->columnMappingHash; + mongo_cursor *mongoCursor = fmstate->mongoCursor; + HTAB *columnMappingHash = fmstate->columnMappingHash; int32 cursorStatus = MONGO_ERROR; TupleDesc tupleDescriptor = tupleSlot->tts_tupleDescriptor; @@ -415,7 +536,7 @@ MongoIterateForeignScan(ForeignScanState *scanState) const char *bsonDocumentKey = NULL; /* top level document */ FillTupleSlot(bsonDocument, bsonDocumentKey, - columnMappingHash, columnValues, columnNulls); + columnMappingHash, columnValues, columnNulls); ExecStoreVirtualTuple(tupleSlot); } @@ -429,7 +550,7 @@ MongoIterateForeignScan(ForeignScanState *scanState) mongo_cursor_error_t errorCode = mongoCursor->err; if (errorCode != MONGO_CURSOR_EXHAUSTED) { - MongoFreeScanState(executionState); + MongoFreeScanState(fmstate); ereport(ERROR, (errmsg("could not iterate over mongo collection"), errhint("Mongo driver cursor error code: %d", errorCode))); @@ -447,12 +568,12 @@ MongoIterateForeignScan(ForeignScanState *scanState) static void MongoEndForeignScan(ForeignScanState *scanState) { - MongoFdwExecState *executionState = (MongoFdwExecState *) scanState->fdw_state; + MongoFdwModifyState *fmstate = (MongoFdwModifyState *) scanState->fdw_state; /* if we executed a query, reclaim mongo related resources */ - if (executionState != NULL) + if (fmstate != NULL) { - MongoFreeScanState(executionState); + MongoFreeScanState(fmstate); } } @@ -465,16 +586,16 @@ MongoEndForeignScan(ForeignScanState *scanState) static void MongoReScanForeignScan(ForeignScanState *scanState) { - MongoFdwExecState *executionState = (MongoFdwExecState *) scanState->fdw_state; - mongo *mongoConnection = executionState->mongoConnection; + MongoFdwModifyState *fmstate = (MongoFdwModifyState *) scanState->fdw_state; + mongo *mongoConnection = fmstate->mongoConnection; MongoFdwOptions *mongoFdwOptions = NULL; mongo_cursor *mongoCursor = NULL; StringInfo namespaceName = NULL; Oid foreignTableId = InvalidOid; /* close down the old cursor */ - mongo_cursor_destroy(executionState->mongoCursor); - mongo_cursor_dispose(executionState->mongoCursor); + mongo_cursor_destroy(fmstate->mongoCursor); + mongo_cursor_dispose(fmstate->mongoCursor); /* reconstruct full collection name */ foreignTableId = RelationGetRelid(scanState->ss.ss_currentRelation); @@ -484,14 +605,448 @@ MongoReScanForeignScan(ForeignScanState *scanState) appendStringInfo(namespaceName, "%s.%s", mongoFdwOptions->databaseName, mongoFdwOptions->collectionName); + MongoFreeOptions(mongoFdwOptions); + /* reconstruct cursor for collection name and set query */ mongoCursor = mongo_cursor_create(); mongo_cursor_init(mongoCursor, mongoConnection, namespaceName->data); - mongo_cursor_set_query(mongoCursor, executionState->queryDocument); + mongo_cursor_set_query(mongoCursor, fmstate->queryDocument); + fmstate->mongoCursor = mongoCursor; +} + +static List * +MongoPlanForeignModify(PlannerInfo *root, + ModifyTable *plan, + Index resultRelation, + int subplan_index) +{ + CmdType operation = plan->operation; + RangeTblEntry *rte = planner_rt_fetch(resultRelation, root); + Relation rel; + List *targetAttrs = NIL; + + /* + * Core code already has some lock on each rel being planned, so we can + * use NoLock here. + */ + rel = heap_open(rte->relid, NoLock); + + if (operation == CMD_INSERT) + { + TupleDesc tupdesc = RelationGetDescr(rel); + int attnum; - executionState->mongoCursor = mongoCursor; + for (attnum = 1; attnum <= tupdesc->natts; attnum++) + { + Form_pg_attribute attr = tupdesc->attrs[attnum - 1]; + + if (!attr->attisdropped) + targetAttrs = lappend_int(targetAttrs, attnum); + } + } + else if (operation == CMD_UPDATE) + { + Bitmapset *tmpset = bms_copy(rte->modifiedCols); + AttrNumber col; + + while ((col = bms_first_member(tmpset)) >= 0) + { + col += FirstLowInvalidHeapAttributeNumber; + if (col <= InvalidAttrNumber) /* shouldn't happen */ + elog(ERROR, "system-column update is not supported"); + + /* + * We also disallow updates to the first column which + * happens to be the row identifier in MongoDb (_id) + */ + if (col == 1) /* shouldn't happen */ + elog(ERROR, "row identifier column update is not supported"); + + targetAttrs = lappend_int(targetAttrs, col); + } + /* We also want the rowid column to be available for the update */ + targetAttrs = lcons_int(1, targetAttrs); + } + else + { + targetAttrs = lcons_int(1, targetAttrs); + } + /* + * RETURNING list not supported + */ + if (plan->returningLists) + elog(ERROR, "RETURNING is not supported by this FDW"); + + heap_close(rel, NoLock); + + return list_make1(targetAttrs); +} + + +/* + * MongoBeginForeignModify + * Begin an insert/update/delete operation on a foreign table + */ +static void +MongoBeginForeignModify(ModifyTableState *mtstate, + ResultRelInfo *resultRelInfo, + List *fdw_private, + int subplan_index, + int eflags) +{ + MongoFdwModifyState *fmstate = NULL; + Relation rel = resultRelInfo->ri_RelationDesc; + AttrNumber n_params = 0; + Oid typefnoid = InvalidOid; + bool isvarlena = false; + ListCell *lc = NULL; + Oid foreignTableId = InvalidOid; + + /* + * Do nothing in EXPLAIN (no ANALYZE) case. resultRelInfo->ri_FdwState + * stays NULL. + */ + if (eflags & EXEC_FLAG_EXPLAIN_ONLY) + return; + + foreignTableId = RelationGetRelid(rel); + + /* Begin constructing MongoFdwModifyState. */ + fmstate = (MongoFdwModifyState *) palloc0(sizeof(MongoFdwModifyState)); + + fmstate->rel = rel; + fmstate->mongoFdwOptions = MongoGetOptions(foreignTableId); + + fmstate->target_attrs = (List *) list_nth(fdw_private, 0); + + n_params = list_length(fmstate->target_attrs) + 1; + fmstate->p_flinfo = (FmgrInfo *) palloc0(sizeof(FmgrInfo) * n_params); + fmstate->p_nums = 0; + + /* Set up for remaining transmittable parameters */ + foreach(lc, fmstate->target_attrs) + { + int attnum = lfirst_int(lc); + Form_pg_attribute attr = RelationGetDescr(rel)->attrs[attnum - 1]; + + Assert(!attr->attisdropped); + + getTypeOutputInfo(attr->atttypid, &typefnoid, &isvarlena); + fmgr_info(typefnoid, &fmstate->p_flinfo[fmstate->p_nums]); + fmstate->p_nums++; + } + Assert(fmstate->p_nums <= n_params); + + resultRelInfo->ri_FdwState = fmstate; } + +/* + * MongoExecForeignInsert + * Insert one row into a foreign table + */ +static TupleTableSlot * +MongoExecForeignInsert(EState *estate, + ResultRelInfo *resultRelInfo, + TupleTableSlot *slot, + TupleTableSlot *planSlot) +{ + MongoFdwOptions *options = NULL; + mongo *mongoConnection = NULL; + Oid foreignTableId = InvalidOid; + bson *b = NULL; + bson_oid_t bsonObjectId; + char *outputString; + Oid outputFunctionId = InvalidOid; + bool typeVarLength = false; + Oid typoid = InvalidOid; + Datum value; + bool isnull = false; + char qualname[255]; + + MongoFdwModifyState *fmstate = (MongoFdwModifyState *) resultRelInfo->ri_FdwState; + + foreignTableId = RelationGetRelid(resultRelInfo->ri_RelationDesc); + + /* resolve foreign table options; and connect to mongo server */ + options = fmstate->mongoFdwOptions; + + mongoConnection = GetConnection(options->addressName, options->portNumber); + + b = bson_create(); + bson_init(b); + + typoid = get_atttype(foreignTableId, 1); + + /* get following parameters from slot */ + if (slot != NULL && fmstate->target_attrs != NIL) + { + ListCell *lc; + + foreach(lc, fmstate->target_attrs) + { + int attnum = lfirst_int(lc); + value = slot_getattr(slot, attnum, &isnull); + /* + * We also disallow null values to the first column which + * happens to be the row identifier in MongoDb (_id). + */ + if (attnum == 1) + { + if (isnull) + elog(ERROR, "null value for first column (row identifier column) is not supported"); + + getTypeOutputInfo(typoid, &outputFunctionId, &typeVarLength); + outputString = OidOutputFunctionCall(outputFunctionId, value); + + /* MongoDB support 24 character's _id (row identifier) */ + if (strlen(outputString) > 24) + elog(ERROR, "first column (row identifier column) should be max 24 characters"); + + UNIQUE_OID(outputString, bsonObjectId); + + if(bson_append_oid(b, "_id", &bsonObjectId) == MONGO_ERROR) + ereport(ERROR, + (errcode(ERRCODE_FDW_ERROR), + errmsg("insert failed, invalid value"))); + } + if(AppenMongoValue(b, slot->tts_tupleDescriptor->attrs[attnum - 1]->attname.data, value, + isnull, slot->tts_tupleDescriptor->attrs[attnum -1]->atttypid) == MONGO_ERROR) + ereport(ERROR, (errmsg("failed to update row %d", slot->tts_tupleDescriptor->attrs[attnum -1]->atttypid), + errhint("Mongo driver insert error: %d", mongoConnection->err))); + } + } + bson_finish(b); + + sprintf(qualname,"%s.%s", options->databaseName, options->collectionName); + + if (mongo_insert(mongoConnection, qualname, b , NULL) != MONGO_OK) + ereport(ERROR, + (errcode(ERRCODE_FDW_ERROR), + errmsg("insert failed"), + errhint("Mongo driver insert error: %d", mongoConnection->err))); + + bson_destroy(b); + bson_dispose(b); + + return slot; +} + + +/* + * MongoAddForeignUpdateTargets + * Add column(s) needed for update/delete on a foreign table + */ +static void +MongoAddForeignUpdateTargets(Query *parsetree, + RangeTblEntry *target_rte, + Relation target_relation) +{ + Var *var; + const char *attrname; + TargetEntry *tle; + + /* + * What we need is the rowid which is the first column + */ + Form_pg_attribute attr = + RelationGetDescr(target_relation)->attrs[0]; + + /* Make a Var representing the desired value */ + var = makeVar(parsetree->resultRelation, + 1, + attr->atttypid, + attr->atttypmod, + InvalidOid, + 0); + + /* Wrap it in a TLE with the right name ... */ + attrname = NameStr(attr->attname); + + tle = makeTargetEntry((Expr *) var, + list_length(parsetree->targetList) + 1, + pstrdup(attrname), + true); + + /* ... and add it to the query's targetlist */ + parsetree->targetList = lappend(parsetree->targetList, tle); +} + + +/* + * postgresExecForeignUpdate + * Update one row in a foreign table + */ +static TupleTableSlot * +MongoExecForeignUpdate(EState *estate, + ResultRelInfo *resultRelInfo, + TupleTableSlot *slot, + TupleTableSlot *planSlot) +{ + MongoFdwOptions *options = NULL; + mongo *mongoConnection = NULL; + Datum datum = 0; + bool isNull = false; + Oid foreignTableId = InvalidOid; + char *columnName = NULL; + Oid typoid = InvalidOid; + bson *b = NULL; + bson *op = NULL; + char qualname[255]; + + MongoFdwModifyState *fmstate = (MongoFdwModifyState *) resultRelInfo->ri_FdwState; + + foreignTableId = RelationGetRelid(resultRelInfo->ri_RelationDesc); + + /* resolve foreign table options; and connect to mongo server */ + options = fmstate->mongoFdwOptions; + + mongoConnection = GetConnection(options->addressName, options->portNumber); + + /* Get the id that was passed up as a resjunk column */ + datum = ExecGetJunkAttribute(planSlot, 1, &isNull); + + columnName = get_relid_attribute_name(foreignTableId, 1); + + typoid = get_atttype(foreignTableId, 1); + + b = bson_create(); + bson_init(b); + + bson_append_start_object(b, "$set"); + + /* Get following parameters from slot, and append to the bson object */ + if (slot != NULL && fmstate->target_attrs != NIL) + { + ListCell *lc; + + foreach(lc, fmstate->target_attrs) + { + int attnum = lfirst_int(lc); + Datum value; + bool isnull; + + value = slot_getattr(slot, attnum, &isnull); + AppenMongoValue(b, slot->tts_tupleDescriptor->attrs[attnum - 1]->attname.data, value, + isnull ? true : false, slot->tts_tupleDescriptor->attrs[attnum - 1]->atttypid); + } + } + bson_append_finish_object(b); + bson_finish(b); + + op = bson_create(); + bson_init(op); + + /* Append where clause in bson object for particular rowid */ + if (AppenMongoValue(op, columnName, datum, false, typoid) == MONGO_ERROR) + { + bson_destroy(b); + bson_dispose(b); + + bson_destroy(op); + bson_dispose(op); + + ereport(ERROR, (errmsg("failed to update row"), + errhint("Mongo driver update error: %d", mongoConnection->err))); + + return NULL; + } + bson_finish(op); + + sprintf(qualname,"%s.%s", options->databaseName, options->collectionName); + + /* We are ready to update the row into MongoDB */ + if (mongo_update(mongoConnection, qualname, op, b, MONGO_UPDATE_BASIC, 0) == MONGO_ERROR) + ereport(ERROR, (errmsg("failed to update row"), + errhint("Mongo driver update error: %d", mongoConnection->err))); + + bson_destroy(b); + bson_dispose(b); + + bson_destroy(op); + bson_dispose(op); + + /* Return NULL if nothing was updated on the remote end */ + return slot; +} + + +/* + * MongoExecForeignDelete + * Delete one row from a foreign table + */ +static TupleTableSlot * +MongoExecForeignDelete(EState *estate, + ResultRelInfo *resultRelInfo, + TupleTableSlot *slot, + TupleTableSlot *planSlot) +{ + MongoFdwOptions *options = NULL; + mongo *mongoConnection = NULL; + Datum datum = 0; + bool isNull = false; + Oid foreignTableId = InvalidOid; + char *columnName = NULL; + Oid typoid = InvalidOid; + bson *b = NULL; + char qualname[255]; + + MongoFdwModifyState *fmstate = (MongoFdwModifyState *) resultRelInfo->ri_FdwState; + + foreignTableId = RelationGetRelid(resultRelInfo->ri_RelationDesc); + + /* resolve foreign table options; and connect to mongo server */ + options = fmstate->mongoFdwOptions; + + mongoConnection = GetConnection(options->addressName, options->portNumber); + + /* Get the id that was passed up as a resjunk column */ + datum = ExecGetJunkAttribute(planSlot, 1, &isNull); + + columnName = get_relid_attribute_name(foreignTableId, 1); + + typoid = get_atttype(foreignTableId, 1); + + b = bson_create(); + bson_init(b); + + if (AppenMongoValue(b, columnName, datum, false, typoid) == MONGO_ERROR) + { + bson_destroy(b); + bson_dispose(b); + ereport(ERROR, (errmsg("failed to delete row"), + errhint("Mongo driver delete error: %d", mongoConnection->err))); + + return NULL; + } + bson_finish(b); + + sprintf(qualname,"%s.%s", options->databaseName, options->collectionName); + + /* Now we are ready to delete a single document from MongoDB */ + if (mongo_remove(mongoConnection, qualname, b , NULL) != MONGO_OK) + ereport(ERROR, (errmsg("failed to delete row"), + errhint("Mongo driver delete error: %d", mongoConnection->err))); + + bson_destroy(b); + bson_dispose(b); + + /* Return NULL if nothing was updated on the remote end */ + return slot; +} + + +/* + * MongoEndForeignModify + * Finish an insert/update/delete operation on a foreign table + */ +static void +MongoEndForeignModify(EState *estate, ResultRelInfo *resultRelInfo) +{ + +} + + /* * ForeignTableDocumentCount connects to the MongoDB server, and queries it for * the number of documents in the foreign collection. On success, the function @@ -511,6 +1066,9 @@ ForeignTableDocumentCount(Oid foreignTableId) mongoConnection = GetConnection(options->addressName, options->portNumber); documentCount = mongo_count(mongoConnection, options->databaseName, options->collectionName, emptyQuery); + + MongoFreeOptions(options); + return documentCount; } @@ -615,7 +1173,7 @@ FillTupleSlot(const bson *bsonDocument, const char *bsonDocumentKey, bson subObject; bson_iterator_subobject(&bsonIterator, &subObject); FillTupleSlot(&subObject, bsonFullKey, - columnMappingHash, columnValues, columnNulls); + columnMappingHash, columnValues, columnNulls); continue; } @@ -714,7 +1272,7 @@ ColumnTypesCompatible(bson_type bsonType, Oid columnTypeId) } break; } - case NAMEOID: + case NAMEOID: { /* * We currently overload the NAMEOID type to represent the BSON @@ -943,20 +1501,20 @@ ColumnValue(bson_iterator *bsonIterator, Oid columnTypeId, int32 columnTypeMod) * all Mongo related resources allocated for the foreign scan. */ static void -MongoFreeScanState(MongoFdwExecState *executionState) +MongoFreeScanState(MongoFdwModifyState *fmstate) { - if (executionState == NULL) + if (fmstate == NULL) { return; } - bson_destroy(executionState->queryDocument); - bson_dispose(executionState->queryDocument); + bson_destroy(fmstate->queryDocument); + bson_dispose(fmstate->queryDocument); - mongo_cursor_destroy(executionState->mongoCursor); - mongo_cursor_dispose(executionState->mongoCursor); + mongo_cursor_destroy(fmstate->mongoCursor); + mongo_cursor_dispose(fmstate->mongoCursor); /* Release remote connection */ - ReleaseConnection(executionState->mongoConnection); + ReleaseConnection(fmstate->mongoConnection); } @@ -1045,7 +1603,7 @@ MongoAcquireSampleRows(Relation relation, int errorLevel, ForeignScanState *scanState = NULL; List *foreignPrivateList = NIL; ForeignScan *foreignScan = NULL; - MongoFdwExecState *executionState = NULL; + MongoFdwModifyState *fmstate = NULL; char *relationName = NULL; int executorFlags = 0; MemoryContext oldContext = CurrentMemoryContext; @@ -1086,9 +1644,9 @@ MongoAcquireSampleRows(Relation relation, int errorLevel, MongoBeginForeignScan(scanState, executorFlags); - executionState = (MongoFdwExecState *) scanState->fdw_state; - mongoCursor = executionState->mongoCursor; - columnMappingHash = executionState->columnMappingHash; + fmstate = (MongoFdwModifyState *) scanState->fdw_state; + mongoCursor = fmstate->mongoCursor; + columnMappingHash = fmstate->columnMappingHash; /* * Use per-tuple memory context to prevent leak of memory used to read @@ -1128,7 +1686,7 @@ MongoAcquireSampleRows(Relation relation, int errorLevel, MemoryContextSwitchTo(tupleContext); FillTupleSlot(bsonDocument, bsonDocumentKey, - columnMappingHash, columnValues, columnNulls); + columnMappingHash, columnValues, columnNulls); MemoryContextSwitchTo(oldContext); } @@ -1141,14 +1699,14 @@ MongoAcquireSampleRows(Relation relation, int errorLevel, mongo_cursor_error_t errorCode = mongoCursor->err; if (errorCode != MONGO_CURSOR_EXHAUSTED) { - MongoFreeScanState(executionState); + MongoFreeScanState(fmstate); ereport(ERROR, (errmsg("could not iterate over mongo collection"), errhint("Mongo driver cursor error code: %d", errorCode))); } break; - } + } /* * The first targetRowCount sample rows are simply copied into the @@ -1187,8 +1745,8 @@ MongoAcquireSampleRows(Relation relation, int errorLevel, heap_freetuple(sampleRows[rowIndex]); sampleRows[rowIndex] = heap_form_tuple(tupleDescriptor, - columnValues, - columnNulls); + columnValues, + columnNulls); } rowCountToSkip -= 1; @@ -1199,7 +1757,7 @@ MongoAcquireSampleRows(Relation relation, int errorLevel, /* clean up */ MemoryContextDelete(tupleContext); - MongoFreeScanState(executionState); + MongoFreeScanState(fmstate); pfree(columnValues); pfree(columnNulls); diff --git a/mongo_fdw.h b/mongo_fdw.h index 52c5ef7..8be4d10 100644 --- a/mongo_fdw.h +++ b/mongo_fdw.h @@ -110,17 +110,33 @@ typedef struct MongoFdwOptions /* - * MongoFdwExecState keeps foreign data wrapper specific execution state that we - * create and hold onto when executing the query. - */ -typedef struct MongoFdwExecState +* MongoFdwExecState keeps foreign data wrapper specific execution state that we +* create and hold onto when executing the query. +*/ +/* +* Execution state of a foreign insert/update/delete operation. +*/ +typedef struct MongoFdwModifyState { - struct HTAB *columnMappingHash; - mongo *mongoConnection; - mongo_cursor *mongoCursor; - bson *queryDocument; + Relation rel; /* relcache entry for the foreign table */ + + List *target_attrs; /* list of target attribute numbers */ + + /* info about parameters for prepared statement */ + int p_nums; /* number of parameters to transmit */ + FmgrInfo *p_flinfo; /* output conversion functions for them */ + + struct HTAB *columnMappingHash; + + mongo *mongoConnection; /* MongoDB connection */ + mongo_cursor *mongoCursor; /* MongoDB cursor */ + bson *queryDocument; /* Bson Document */ + + MongoFdwOptions *mongoFdwOptions; -} MongoFdwExecState; + /* working memory context */ + MemoryContext temp_cxt; /* context for per-tuple temporary data */ +} MongoFdwModifyState; /* @@ -148,6 +164,7 @@ extern List * ColumnList(RelOptInfo *baserel); extern Datum mongo_fdw_handler(PG_FUNCTION_ARGS); extern Datum mongo_fdw_validator(PG_FUNCTION_ARGS); extern MongoFdwOptions * MongoGetOptions(Oid foreignTableId); +extern void MongoFreeOptions(MongoFdwOptions *mongoFdwOptions); mongo* GetConnection(char *host, int32 port); void cleanup_connection(void); diff --git a/mongo_query.c b/mongo_query.c index fb2ac33..426e801 100644 --- a/mongo_query.c +++ b/mongo_query.c @@ -13,6 +13,7 @@ #include "postgres.h" #include "mongo_fdw.h" +#include "mongo_query.h" #include "catalog/pg_type.h" #include "nodes/makefuncs.h" @@ -363,59 +364,68 @@ ColumnOperatorList(Var *column, List *operatorList) static void AppendConstantValue(bson *queryDocument, const char *keyName, Const *constant) { - Datum constantValue = constant->constvalue; - Oid constantTypeId = constant->consttype; - - bool constantNull = constant->constisnull; - if (constantNull) + if (constant->constisnull) { bson_append_null(queryDocument, keyName); return; } + AppenMongoValue(queryDocument, keyName, constant->constvalue, constant->consttype, false); +} + +int32 +AppenMongoValue(bson *queryDocument, const char *keyName, Datum value, bool isnull, Oid id) +{ + int32 status = MONGO_ERROR; - switch(constantTypeId) + if (isnull) + { + status = bson_append_null(queryDocument, keyName); + return status; + } + + switch(id) { case INT2OID: { - int16 value = DatumGetInt16(constantValue); - bson_append_int(queryDocument, keyName, (int) value); + int16 valueInt = DatumGetInt16(value); + status = bson_append_int(queryDocument, keyName, (int) valueInt); break; } case INT4OID: { - int32 value = DatumGetInt32(constantValue); - bson_append_int(queryDocument, keyName, value); + int32 valueInt = DatumGetInt32(value); + status = bson_append_int(queryDocument, keyName, valueInt); break; } case INT8OID: { - int64 value = DatumGetInt64(constantValue); - bson_append_long(queryDocument, keyName, value); + int64 valueLong = DatumGetInt64(value); + status = bson_append_long(queryDocument, keyName, valueLong); break; } case FLOAT4OID: { - float4 value = DatumGetFloat4(constantValue); - bson_append_double(queryDocument, keyName, (double) value); + float4 valueFloat = DatumGetFloat4(value); + status = bson_append_double(queryDocument, keyName, (double) valueFloat); break; } case FLOAT8OID: { - float8 value = DatumGetFloat8(constantValue); - bson_append_double(queryDocument, keyName, value); + float8 valueFloat = DatumGetFloat8(value); + status = bson_append_double(queryDocument, keyName, valueFloat); break; } case NUMERICOID: { - Datum valueDatum = DirectFunctionCall1(numeric_float8, constantValue); - float8 value = DatumGetFloat8(valueDatum); - bson_append_double(queryDocument, keyName, value); + Datum valueDatum = DirectFunctionCall1(numeric_float8, value); + float8 valueFloat = DatumGetFloat8(valueDatum); + status = bson_append_double(queryDocument, keyName, valueFloat); break; } case BOOLOID: { - bool value = DatumGetBool(constantValue); - bson_append_int(queryDocument, keyName, (int) value); + bool valueBool = DatumGetBool(value); + status = bson_append_int(queryDocument, keyName, (int) valueBool); break; } case BPCHAROID: @@ -425,46 +435,42 @@ AppendConstantValue(bson *queryDocument, const char *keyName, Const *constant) char *outputString = NULL; Oid outputFunctionId = InvalidOid; bool typeVarLength = false; - - getTypeOutputInfo(constantTypeId, &outputFunctionId, &typeVarLength); - outputString = OidOutputFunctionCall(outputFunctionId, constantValue); - - bson_append_string(queryDocument, keyName, outputString); + getTypeOutputInfo(id, &outputFunctionId, &typeVarLength); + outputString = OidOutputFunctionCall(outputFunctionId, value); + status = bson_append_string(queryDocument, keyName, outputString); break; } - case NAMEOID: + case NAMEOID: { char *outputString = NULL; Oid outputFunctionId = InvalidOid; bool typeVarLength = false; bson_oid_t bsonObjectId; memset(bsonObjectId.bytes, 0, sizeof(bsonObjectId.bytes)); - - getTypeOutputInfo(constantTypeId, &outputFunctionId, &typeVarLength); - outputString = OidOutputFunctionCall(outputFunctionId, constantValue); + getTypeOutputInfo(id, &outputFunctionId, &typeVarLength); + outputString = OidOutputFunctionCall(outputFunctionId, value); bson_oid_from_string(&bsonObjectId, outputString); - - bson_append_oid(queryDocument, keyName, &bsonObjectId); + status = bson_append_oid(queryDocument, keyName, &bsonObjectId); break; } case DATEOID: { - Datum valueDatum = DirectFunctionCall1(date_timestamp, constantValue); + Datum valueDatum = DirectFunctionCall1(date_timestamp, value); Timestamp valueTimestamp = DatumGetTimestamp(valueDatum); int64 valueMicroSecs = valueTimestamp + POSTGRES_TO_UNIX_EPOCH_USECS; int64 valueMilliSecs = valueMicroSecs / 1000; - bson_append_date(queryDocument, keyName, valueMilliSecs); + status = bson_append_date(queryDocument, keyName, valueMilliSecs); break; } case TIMESTAMPOID: case TIMESTAMPTZOID: { - Timestamp valueTimestamp = DatumGetTimestamp(constantValue); + Timestamp valueTimestamp = DatumGetTimestamp(value); int64 valueMicroSecs = valueTimestamp + POSTGRES_TO_UNIX_EPOCH_USECS; int64 valueMilliSecs = valueMicroSecs / 1000; - bson_append_date(queryDocument, keyName, valueMilliSecs); + status = bson_append_date(queryDocument, keyName, valueMilliSecs); break; } default: @@ -474,12 +480,13 @@ AppendConstantValue(bson *queryDocument, const char *keyName, Const *constant) * byte arrays are easy to add, but they need testing. Other types * such as money or inet, do not have equivalents in MongoDB. */ - ereport(ERROR, (errcode(ERRCODE_FDW_INVALID_DATA_TYPE), - errmsg("cannot convert constant value to BSON value"), - errhint("Constant value data type: %u", constantTypeId))); + ereport(ERROR, (errcode(ERRCODE_FDW_INVALID_DATA_TYPE), + errmsg("cannot convert constant value to BSON value"), + errhint("Constant value data type: %u", id))); break; } } + return status; } diff --git a/mongo_query.h b/mongo_query.h index 0a75e98..87b95e5 100644 --- a/mongo_query.h +++ b/mongo_query.h @@ -12,7 +12,7 @@ #ifndef MONGO_QUERY_H #define MONGO_QUERY_H - +int32 AppenMongoValue(bson *queryDocument, const char *keyName, Datum value, bool isnull, Oid id); #endif /* MONGO_QUERY_H */ From 5f881c9e5fb3ebf84274829eab14aadab48552e8 Mon Sep 17 00:00:00 2001 From: Ibrar Ahmed Date: Mon, 2 Jun 2014 18:21:25 +0500 Subject: [PATCH 04/43] MongoDB's Meta Driver support added. --- Makefile | 35 ---- mongo_fdw.c | 441 +++++++++++++++++++++++--------------------------- mongo_fdw.h | 83 ++++++---- mongo_query.c | 69 ++++---- mongo_query.h | 5 +- 5 files changed, 291 insertions(+), 342 deletions(-) delete mode 100644 Makefile diff --git a/Makefile b/Makefile deleted file mode 100644 index 82bf10c..0000000 --- a/Makefile +++ /dev/null @@ -1,35 +0,0 @@ -# mongo_fdw/Makefile -# -# Copyright (c) 2012-2014 Citus Data, Inc. -# - -MODULE_big = mongo_fdw - -# -# We assume we are running on a POSIX compliant system (Linux, OSX). If you are -# on another platform, change env_posix.os in MONGO_OBJS with the appropriate -# environment object file. -# - -MONGO_DRIVER = mongo-c-driver-v0.6 -MONGO_PATH = $(MONGO_DRIVER)/src -MONGO_OBJS = $(MONGO_PATH)/bson.os $(MONGO_PATH)/encoding.os $(MONGO_PATH)/md5.os \ - $(MONGO_PATH)/mongo.os $(MONGO_PATH)/numbers.os $(MONGO_PATH)/env_posix.os - -PG_CPPFLAGS = --std=c99 -I$(MONGO_PATH) -OBJS = connection.o option.o mongo_fdw.o mongo_query.o $(MONGO_OBJS) - -EXTENSION = mongo_fdw -DATA = mongo_fdw--1.0.sql - -$(MONGO_DRIVER)/%.os: - $(MAKE) -C $(MONGO_DRIVER) $*.os - -# -# Users need to specify their Postgres installation path through pg_config. For -# example: /usr/local/pgsql/bin/pg_config or /usr/lib/postgresql/9.1/bin/pg_config -# - -PG_CONFIG = pg_config -PGXS := $(shell $(PG_CONFIG) --pgxs) -include $(PGXS) diff --git a/mongo_fdw.c b/mongo_fdw.c index 75593eb..2bc67b1 100644 --- a/mongo_fdw.c +++ b/mongo_fdw.c @@ -10,12 +10,9 @@ *------------------------------------------------------------------------- */ -#include "postgres.h" -#include "mongo_fdw.h" -#include "mongo_query.h" - #include "postgres.h" #include "bson.h" +#include "mongo_wrapper.h" #include "mongo_fdw.h" #include "mongo_query.h" @@ -68,14 +65,14 @@ /* Local functions forward declarations */ static void MongoGetForeignRelSize(PlannerInfo *root, RelOptInfo *baserel, - Oid foreignTableId); + Oid foreignTableId); static void MongoGetForeignPaths(PlannerInfo *root, RelOptInfo *baserel, - Oid foreignTableId); + Oid foreignTableId); static ForeignScan * MongoGetForeignPlan(PlannerInfo *root, RelOptInfo *baserel, - Oid foreignTableId, ForeignPath *bestPath, - List *targetList, List *restrictionClauses); + Oid foreignTableId, ForeignPath *bestPath, + List *targetList, List *restrictionClauses); static void MongoExplainForeignScan(ForeignScanState *scanState, - ExplainState *explainState); + ExplainState *explainState); static void MongoBeginForeignScan(ForeignScanState *scanState, int executorFlags); static TupleTableSlot * MongoIterateForeignScan(ForeignScanState *scanState); static void MongoEndForeignScan(ForeignScanState *scanState); @@ -90,7 +87,7 @@ static TupleTableSlot *MongoExecForeignDelete(EState *estate, TupleTableSlot *slot, TupleTableSlot *planSlot); static void MongoEndForeignModify(EState *estate, - ResultRelInfo *resultRelInfo); + ResultRelInfo *resultRelInfo); static void MongoAddForeignUpdateTargets(Query *parsetree, RangeTblEntry *target_rte, @@ -103,35 +100,33 @@ static void MongoBeginForeignModify(ModifyTableState *mtstate, int eflags); static TupleTableSlot *MongoExecForeignInsert(EState *estate, - ResultRelInfo *resultRelInfo, - TupleTableSlot *slot, - TupleTableSlot *planSlot); + ResultRelInfo *resultRelInfo, + TupleTableSlot *slot, + TupleTableSlot *planSlot); static List *MongoPlanForeignModify(PlannerInfo *root, - ModifyTable *plan, - Index resultRelation, - int subplan_index); + ModifyTable *plan, + Index resultRelation, + int subplan_index); static void MongoExplainForeignModify(ModifyTableState *mtstate, - ResultRelInfo *rinfo, - List *fdw_private, - int subplan_index, - ExplainState *es); + ResultRelInfo *rinfo, List *fdw_private, + int subplan_index, ExplainState *es); /* local functions */ static double ForeignTableDocumentCount(Oid foreignTableId); static HTAB * ColumnMappingHash(Oid foreignTableId, List *columnList); -static void FillTupleSlot(const bson *bsonDocument, const char *bsonDocumentKey, - HTAB *columnMappingHash, Datum *columnValues, - bool *columnNulls); -static bool ColumnTypesCompatible(bson_type bsonType, Oid columnTypeId); -static Datum ColumnValueArray(bson_iterator *bsonIterator, Oid valueTypeId); -static Datum ColumnValue(bson_iterator *bsonIterator, Oid columnTypeId, +static void FillTupleSlot(const BSON *bsonDocument, const char *bsonDocumentKey, + HTAB *columnMappingHash, Datum *columnValues, + bool *columnNulls); +static bool ColumnTypesCompatible(BSON_TYPE bsonType, Oid columnTypeId); +static Datum ColumnValueArray(BSON_ITERATOR *bsonIterator, Oid valueTypeId); +static Datum ColumnValue(BSON_ITERATOR *bsonIterator, Oid columnTypeId, int32 columnTypeMod); static void MongoFreeScanState(MongoFdwModifyState *fmstate); static bool MongoAnalyzeForeignTable(Relation relation, - AcquireSampleRowsFunc *acquireSampleRowsFunc, + AcquireSampleRowsFunc *acquireSampleRowsFunc, BlockNumber *totalPageCount); static int MongoAcquireSampleRows(Relation relation, int errorLevel, HeapTuple *sampleRows, int targetRowCount, @@ -151,10 +146,9 @@ extern PGDLLEXPORT void _PG_init(void); memset(y, 'A', sizeof(y));\ y[24] = 0; \ strcpy(y, x); \ - bson_oid_from_string(&z, y);\ + BsonOidFromString(&z, y);\ } while(0); - /* declarations for dynamic loading */ PG_MODULE_MAGIC; @@ -198,7 +192,6 @@ mongo_fdw_handler(PG_FUNCTION_ARGS) /* support for EXPLAIN */ fdwRoutine->ExplainForeignScan = MongoExplainForeignScan; - fdwRoutine->ExplainForeignModify = NULL; fdwRoutine->ExplainForeignModify = MongoExplainForeignModify; /* support for ANALYSE */ @@ -339,7 +332,7 @@ MongoGetForeignPlan(PlannerInfo *root, RelOptInfo *baserel, Oid foreignTableId, ForeignScan *foreignScan = NULL; List *foreignPrivateList = NIL; List *opExpressionList = NIL; - bson *queryDocument = NULL; + BSON *queryDocument = NULL; List *columnList = NIL; /* @@ -364,13 +357,13 @@ MongoGetForeignPlan(PlannerInfo *root, RelOptInfo *baserel, Oid foreignTableId, columnList = ColumnList(baserel); /* construct foreign plan with query document and column list */ - foreignPrivateList = list_make2(columnList, restrictionClauses); + foreignPrivateList = list_make2(columnList, opExpressionList); - /* only clean up the query struct, but not its data */ - bson_dispose(queryDocument); + /* only clean up the query struct */ + BsonDestroy(queryDocument); /* create the foreign scan node */ - foreignScan = make_foreignscan(targetList, restrictionClauses, + foreignScan = make_foreignscan(targetList, restrictionClauses, scanRangeTableIndex, NIL, /* no expressions to evaluate */ foreignPrivateList); @@ -435,25 +428,23 @@ MongoExplainForeignModify(ModifyTableState *mtstate, static void MongoBeginForeignScan(ForeignScanState *scanState, int executorFlags) { - mongo *mongoConnection = NULL; - mongo_cursor *mongoCursor = NULL; + MONGO_CONN *mongoConnection = NULL; + MONGO_CURSOR *mongoCursor = NULL; Oid foreignTableId = InvalidOid; List *columnList = NIL; HTAB *columnMappingHash = NULL; char *addressName = NULL; int32 portNumber = 0; - StringInfo namespaceName = NULL; ForeignScan *foreignScan = NULL; List *foreignPrivateList = NIL; - bson *queryDocument = NULL; + BSON *queryDocument = NULL; MongoFdwOptions *mongoFdwOptions = NULL; MongoFdwModifyState *fmstate = NULL; + List *opExpressionList = NIL; /* if Explain with no Analyze, do nothing */ if (executorFlags & EXEC_FLAG_EXPLAIN_ONLY) - { return; - } foreignTableId = RelationGetRelid(scanState->ss.ss_currentRelation); mongoFdwOptions = MongoGetOptions(foreignTableId); @@ -467,24 +458,20 @@ MongoBeginForeignScan(ForeignScanState *scanState, int executorFlags) * establish new connection if necessary. */ mongoConnection = GetConnection(addressName, portNumber); + foreignScan = (ForeignScan *) scanState->ss.ps.plan; foreignPrivateList = foreignScan->fdw_private; - Assert(list_length(foreignPrivateList) == 1); + Assert(list_length(foreignPrivateList) == 2); columnList = list_nth(foreignPrivateList, 0); + opExpressionList = list_nth(foreignPrivateList, 1); - queryDocument = QueryDocument(foreignTableId, NIL); + queryDocument = QueryDocument(foreignTableId, opExpressionList); columnMappingHash = ColumnMappingHash(foreignTableId, columnList); - namespaceName = makeStringInfo(); - appendStringInfo(namespaceName, "%s.%s", mongoFdwOptions->databaseName, - mongoFdwOptions->collectionName); - /* create cursor for collection name and set query */ - mongoCursor = mongo_cursor_create(); - mongo_cursor_init(mongoCursor, mongoConnection, namespaceName->data); - mongo_cursor_set_query(mongoCursor, queryDocument); + mongoCursor = MongoCursorCreate(mongoConnection, mongoFdwOptions->databaseName, mongoFdwOptions->collectionName, queryDocument); /* create and set foreign execution state */ fmstate = (MongoFdwModifyState *) palloc0(sizeof(MongoFdwModifyState)); @@ -508,9 +495,8 @@ MongoIterateForeignScan(ForeignScanState *scanState) { MongoFdwModifyState *fmstate = (MongoFdwModifyState *) scanState->fdw_state; TupleTableSlot *tupleSlot = scanState->ss.ss_ScanTupleSlot; - mongo_cursor *mongoCursor = fmstate->mongoCursor; + MONGO_CURSOR *mongoCursor = fmstate->mongoCursor; HTAB *columnMappingHash = fmstate->columnMappingHash; - int32 cursorStatus = MONGO_ERROR; TupleDesc tupleDescriptor = tupleSlot->tts_tupleDescriptor; Datum *columnValues = tupleSlot->tts_values; @@ -529,32 +515,33 @@ MongoIterateForeignScan(ForeignScanState *scanState) memset(columnValues, 0, columnCount * sizeof(Datum)); memset(columnNulls, true, columnCount * sizeof(bool)); - cursorStatus = mongo_cursor_next(mongoCursor); - if (cursorStatus == MONGO_OK) + if (MongoCursorNext(mongoCursor, NULL)) { - const bson *bsonDocument = mongo_cursor_bson(mongoCursor); + const BSON *bsonDocument = MongoCursorBson(mongoCursor); const char *bsonDocumentKey = NULL; /* top level document */ FillTupleSlot(bsonDocument, bsonDocumentKey, - columnMappingHash, columnValues, columnNulls); + columnMappingHash, columnValues, columnNulls); ExecStoreVirtualTuple(tupleSlot); } else { + #ifndef META_DRIVER /* * The following is a courtesy check. In practice when Mongo shuts down, * mongo_cursor_next() could possibly crash. This function first frees - * cursor->reply, and then references reply in mongo_cursor_destroy(). + * cursor->reply, and then references reply in mongo_cursor__destroy(). */ + mongo_cursor_error_t errorCode = mongoCursor->err; if (errorCode != MONGO_CURSOR_EXHAUSTED) { MongoFreeScanState(fmstate); - ereport(ERROR, (errmsg("could not iterate over mongo collection"), - errhint("Mongo driver cursor error code: %d", errorCode))); + errhint("Mongo driver cursor error code: %d", errorCode))); } + #endif } return tupleSlot; @@ -573,6 +560,11 @@ MongoEndForeignScan(ForeignScanState *scanState) /* if we executed a query, reclaim mongo related resources */ if (fmstate != NULL) { + if (fmstate->mongoFdwOptions) + { + MongoFreeOptions(fmstate->mongoFdwOptions); + fmstate->mongoFdwOptions = NULL; + } MongoFreeScanState(fmstate); } } @@ -587,31 +579,23 @@ static void MongoReScanForeignScan(ForeignScanState *scanState) { MongoFdwModifyState *fmstate = (MongoFdwModifyState *) scanState->fdw_state; - mongo *mongoConnection = fmstate->mongoConnection; + MONGO_CONN *mongoConnection = fmstate->mongoConnection; MongoFdwOptions *mongoFdwOptions = NULL; - mongo_cursor *mongoCursor = NULL; - StringInfo namespaceName = NULL; Oid foreignTableId = InvalidOid; /* close down the old cursor */ - mongo_cursor_destroy(fmstate->mongoCursor); - mongo_cursor_dispose(fmstate->mongoCursor); + MongoCursorDestroy(fmstate->mongoCursor); /* reconstruct full collection name */ foreignTableId = RelationGetRelid(scanState->ss.ss_currentRelation); mongoFdwOptions = MongoGetOptions(foreignTableId); - namespaceName = makeStringInfo(); - appendStringInfo(namespaceName, "%s.%s", mongoFdwOptions->databaseName, - mongoFdwOptions->collectionName); - - MongoFreeOptions(mongoFdwOptions); - /* reconstruct cursor for collection name and set query */ - mongoCursor = mongo_cursor_create(); - mongo_cursor_init(mongoCursor, mongoConnection, namespaceName->data); - mongo_cursor_set_query(mongoCursor, fmstate->queryDocument); - fmstate->mongoCursor = mongoCursor; + fmstate->mongoCursor = MongoCursorCreate(mongoConnection, + fmstate->mongoFdwOptions->databaseName, + fmstate->mongoFdwOptions->collectionName, + fmstate->queryDocument); + MongoFreeOptions(mongoFdwOptions); } static List * @@ -623,7 +607,7 @@ MongoPlanForeignModify(PlannerInfo *root, CmdType operation = plan->operation; RangeTblEntry *rte = planner_rt_fetch(resultRelation, root); Relation rel; - List *targetAttrs = NIL; + List* targetAttrs = NIL; /* * Core code already has some lock on each rel being planned, so we can @@ -646,7 +630,7 @@ MongoPlanForeignModify(PlannerInfo *root, } else if (operation == CMD_UPDATE) { - Bitmapset *tmpset = bms_copy(rte->modifiedCols); + Bitmapset *tmpset = bms_copy(rte->modifiedCols); AttrNumber col; while ((col = bms_first_member(tmpset)) >= 0) @@ -654,7 +638,6 @@ MongoPlanForeignModify(PlannerInfo *root, col += FirstLowInvalidHeapAttributeNumber; if (col <= InvalidAttrNumber) /* shouldn't happen */ elog(ERROR, "system-column update is not supported"); - /* * We also disallow updates to the first column which * happens to be the row identifier in MongoDb (_id) @@ -684,8 +667,7 @@ MongoPlanForeignModify(PlannerInfo *root, /* - * MongoBeginForeignModify - * Begin an insert/update/delete operation on a foreign table + * Begin an insert/update/delete operation on a foreign table */ static void MongoBeginForeignModify(ModifyTableState *mtstate, @@ -742,8 +724,7 @@ MongoBeginForeignModify(ModifyTableState *mtstate, /* - * MongoExecForeignInsert - * Insert one row into a foreign table + * Insert one row into a foreign table. */ static TupleTableSlot * MongoExecForeignInsert(EState *estate, @@ -752,9 +733,9 @@ MongoExecForeignInsert(EState *estate, TupleTableSlot *planSlot) { MongoFdwOptions *options = NULL; - mongo *mongoConnection = NULL; + MONGO_CONN *mongoConnection = NULL; Oid foreignTableId = InvalidOid; - bson *b = NULL; + BSON *b = NULL; bson_oid_t bsonObjectId; char *outputString; Oid outputFunctionId = InvalidOid; @@ -762,7 +743,7 @@ MongoExecForeignInsert(EState *estate, Oid typoid = InvalidOid; Datum value; bool isnull = false; - char qualname[255]; + MongoFdwModifyState *fmstate = (MongoFdwModifyState *) resultRelInfo->ri_FdwState; @@ -773,8 +754,7 @@ MongoExecForeignInsert(EState *estate, mongoConnection = GetConnection(options->addressName, options->portNumber); - b = bson_create(); - bson_init(b); + b = BsonCreate(); typoid = get_atttype(foreignTableId, 1); @@ -787,12 +767,13 @@ MongoExecForeignInsert(EState *estate, { int attnum = lfirst_int(lc); value = slot_getattr(slot, attnum, &isnull); - /* - * We also disallow null values to the first column which - * happens to be the row identifier in MongoDb (_id). - */ if (attnum == 1) { + /* + * We also disallow null values to the first column which + * happens to be the row identifier in MongoDb (_id). + */ + if (isnull) elog(ERROR, "null value for first column (row identifier column) is not supported"); @@ -805,46 +786,40 @@ MongoExecForeignInsert(EState *estate, UNIQUE_OID(outputString, bsonObjectId); - if(bson_append_oid(b, "_id", &bsonObjectId) == MONGO_ERROR) + /* Append rowid field which is "_id" in MongoDB */ + if(!BsonAppendOid(b, "_id", &bsonObjectId)) ereport(ERROR, (errcode(ERRCODE_FDW_ERROR), errmsg("insert failed, invalid value"))); } - if(AppenMongoValue(b, slot->tts_tupleDescriptor->attrs[attnum - 1]->attname.data, value, - isnull, slot->tts_tupleDescriptor->attrs[attnum -1]->atttypid) == MONGO_ERROR) - ereport(ERROR, (errmsg("failed to update row %d", slot->tts_tupleDescriptor->attrs[attnum -1]->atttypid), - errhint("Mongo driver insert error: %d", mongoConnection->err))); + AppenMongoValue(b, slot->tts_tupleDescriptor->attrs[attnum - 1]->attname.data, value, + isnull, slot->tts_tupleDescriptor->attrs[attnum -1]->atttypid); } } - bson_finish(b); - - sprintf(qualname,"%s.%s", options->databaseName, options->collectionName); + BsonFinish(b); - if (mongo_insert(mongoConnection, qualname, b , NULL) != MONGO_OK) - ereport(ERROR, - (errcode(ERRCODE_FDW_ERROR), - errmsg("insert failed"), - errhint("Mongo driver insert error: %d", mongoConnection->err))); + /* Now we are ready to insert tuple / document into MongoDB */ + MongoInsert(mongoConnection, options->databaseName, options->collectionName, b); - bson_destroy(b); - bson_dispose(b); + BsonDestroy(b); return slot; } /* - * MongoAddForeignUpdateTargets - * Add column(s) needed for update/delete on a foreign table + * Add column(s) needed for update/delete on a foreign table, we are using + * first column as row identification column, so we are adding that into target + * list. */ static void MongoAddForeignUpdateTargets(Query *parsetree, RangeTblEntry *target_rte, Relation target_relation) { - Var *var; - const char *attrname; - TargetEntry *tle; + Var *var = NULL; + const char *attrname = NULL; + TargetEntry *tle = NULL; /* * What we need is the rowid which is the first column @@ -873,10 +848,6 @@ MongoAddForeignUpdateTargets(Query *parsetree, } -/* - * postgresExecForeignUpdate - * Update one row in a foreign table - */ static TupleTableSlot * MongoExecForeignUpdate(EState *estate, ResultRelInfo *resultRelInfo, @@ -884,15 +855,15 @@ MongoExecForeignUpdate(EState *estate, TupleTableSlot *planSlot) { MongoFdwOptions *options = NULL; - mongo *mongoConnection = NULL; + MONGO_CONN *mongoConnection = NULL; Datum datum = 0; bool isNull = false; Oid foreignTableId = InvalidOid; char *columnName = NULL; Oid typoid = InvalidOid; - bson *b = NULL; - bson *op = NULL; - char qualname[255]; + BSON *b = NULL; + BSON *op = NULL; + BSON set; MongoFdwModifyState *fmstate = (MongoFdwModifyState *) resultRelInfo->ri_FdwState; @@ -910,12 +881,10 @@ MongoExecForeignUpdate(EState *estate, typoid = get_atttype(foreignTableId, 1); - b = bson_create(); - bson_init(b); - - bson_append_start_object(b, "$set"); + b = BsonCreate(); + BsonAppendStartObject(b, "$set", &set); - /* Get following parameters from slot, and append to the bson object */ + /* get following parameters from slot */ if (slot != NULL && fmstate->target_attrs != NIL) { ListCell *lc; @@ -927,50 +896,36 @@ MongoExecForeignUpdate(EState *estate, bool isnull; value = slot_getattr(slot, attnum, &isnull); +#ifdef META_DRIVER + AppenMongoValue(&set, slot->tts_tupleDescriptor->attrs[attnum - 1]->attname.data, value, + isnull ? true : false, slot->tts_tupleDescriptor->attrs[attnum - 1]->atttypid); +#else AppenMongoValue(b, slot->tts_tupleDescriptor->attrs[attnum - 1]->attname.data, value, isnull ? true : false, slot->tts_tupleDescriptor->attrs[attnum - 1]->atttypid); +#endif } } - bson_append_finish_object(b); - bson_finish(b); + BsonAppendFinishObject(b, &set); + BsonFinish(b); - op = bson_create(); - bson_init(op); - - /* Append where clause in bson object for particular rowid */ - if (AppenMongoValue(op, columnName, datum, false, typoid) == MONGO_ERROR) + op = BsonCreate(); + if (!AppenMongoValue(op, columnName, datum, false, typoid)) { - bson_destroy(b); - bson_dispose(b); - - bson_destroy(op); - bson_dispose(op); - - ereport(ERROR, (errmsg("failed to update row"), - errhint("Mongo driver update error: %d", mongoConnection->err))); - + BsonDestroy(b); return NULL; } - bson_finish(op); - - sprintf(qualname,"%s.%s", options->databaseName, options->collectionName); + BsonFinish(op); /* We are ready to update the row into MongoDB */ - if (mongo_update(mongoConnection, qualname, op, b, MONGO_UPDATE_BASIC, 0) == MONGO_ERROR) - ereport(ERROR, (errmsg("failed to update row"), - errhint("Mongo driver update error: %d", mongoConnection->err))); + MongoUpdate(mongoConnection, options->databaseName, options->collectionName, op, b); - bson_destroy(b); - bson_dispose(b); - - bson_destroy(op); - bson_dispose(op); + BsonDestroy(op); + BsonDestroy(b); /* Return NULL if nothing was updated on the remote end */ return slot; } - /* * MongoExecForeignDelete * Delete one row from a foreign table @@ -982,14 +937,13 @@ MongoExecForeignDelete(EState *estate, TupleTableSlot *planSlot) { MongoFdwOptions *options = NULL; - mongo *mongoConnection = NULL; + MONGO_CONN *mongoConnection = NULL; Datum datum = 0; bool isNull = false; Oid foreignTableId = InvalidOid; char *columnName = NULL; Oid typoid = InvalidOid; - bson *b = NULL; - char qualname[255]; + BSON *b = NULL; MongoFdwModifyState *fmstate = (MongoFdwModifyState *) resultRelInfo->ri_FdwState; @@ -1007,35 +961,23 @@ MongoExecForeignDelete(EState *estate, typoid = get_atttype(foreignTableId, 1); - b = bson_create(); - bson_init(b); - - if (AppenMongoValue(b, columnName, datum, false, typoid) == MONGO_ERROR) + b = BsonCreate(); + if (!AppenMongoValue(b,columnName, datum, false, typoid)) { - bson_destroy(b); - bson_dispose(b); - ereport(ERROR, (errmsg("failed to delete row"), - errhint("Mongo driver delete error: %d", mongoConnection->err))); - + BsonDestroy(b); return NULL; } - bson_finish(b); - - sprintf(qualname,"%s.%s", options->databaseName, options->collectionName); + BsonFinish(b); /* Now we are ready to delete a single document from MongoDB */ - if (mongo_remove(mongoConnection, qualname, b , NULL) != MONGO_OK) - ereport(ERROR, (errmsg("failed to delete row"), - errhint("Mongo driver delete error: %d", mongoConnection->err))); + MongoDelete(mongoConnection, options->databaseName, options->collectionName, b); - bson_destroy(b); - bson_dispose(b); + BsonDestroy(b); /* Return NULL if nothing was updated on the remote end */ return slot; } - /* * MongoEndForeignModify * Finish an insert/update/delete operation on a foreign table @@ -1043,10 +985,19 @@ MongoExecForeignDelete(EState *estate, static void MongoEndForeignModify(EState *estate, ResultRelInfo *resultRelInfo) { - + MongoFdwModifyState *fmstate = (MongoFdwModifyState *) resultRelInfo->ri_FdwState; + if (fmstate) + { + if (fmstate->mongoFdwOptions) + { + MongoFreeOptions(fmstate->mongoFdwOptions); + fmstate->mongoFdwOptions = NULL; + } + MongoFreeScanState(fmstate); + pfree(fmstate); + } } - /* * ForeignTableDocumentCount connects to the MongoDB server, and queries it for * the number of documents in the foreign collection. On success, the function @@ -1056,16 +1007,16 @@ static double ForeignTableDocumentCount(Oid foreignTableId) { MongoFdwOptions *options = NULL; - mongo *mongoConnection = NULL; - const bson *emptyQuery = NULL; + MONGO_CONN *mongoConnection = NULL; + const BSON *emptyQuery = NULL; double documentCount = 0.0; /* resolve foreign table options; and connect to mongo server */ options = MongoGetOptions(foreignTableId); mongoConnection = GetConnection(options->addressName, options->portNumber); - documentCount = mongo_count(mongoConnection, options->databaseName, - options->collectionName, emptyQuery); + + MongoAggregateCount(mongoConnection, options->databaseName, options->collectionName, emptyQuery); MongoFreeOptions(options); @@ -1133,16 +1084,16 @@ ColumnMappingHash(Oid foreignTableId, List *columnList) * passed as NULL. */ static void -FillTupleSlot(const bson *bsonDocument, const char *bsonDocumentKey, +FillTupleSlot(const BSON *bsonDocument, const char *bsonDocumentKey, HTAB *columnMappingHash, Datum *columnValues, bool *columnNulls) { - bson_iterator bsonIterator = { NULL, 0 }; - bson_iterator_init(&bsonIterator, bsonDocument); + BSON_ITERATOR bsonIterator = { NULL, 0 }; + BsonIterInit(&bsonIterator, (BSON*)bsonDocument); - while (bson_iterator_next(&bsonIterator)) + while (BsonIterNext(&bsonIterator)) { - const char *bsonKey = bson_iterator_key(&bsonIterator); - bson_type bsonType = bson_iterator_type(&bsonIterator); + const char *bsonKey = BsonIterKey(&bsonIterator); + BSON_TYPE bsonType = BsonIterType(&bsonIterator); ColumnMapping *columnMapping = NULL; Oid columnTypeId = InvalidOid; @@ -1168,22 +1119,22 @@ FillTupleSlot(const bson *bsonDocument, const char *bsonDocumentKey, } /* recurse into nested objects */ - if (bsonType == BSON_OBJECT) + if (bsonType == BSON_TYPE_DOCUMENT) { - bson subObject; - bson_iterator_subobject(&bsonIterator, &subObject); + BSON subObject; + BsonIterSubObject(&bsonIterator, &subObject); FillTupleSlot(&subObject, bsonFullKey, - columnMappingHash, columnValues, columnNulls); + columnMappingHash, columnValues, columnNulls); continue; } - /* look up the corresponding column for this bson key */ + /* look up the corresponding column for this BSON key */ hashKey = (void *) bsonFullKey; columnMapping = (ColumnMapping *) hash_search(columnMappingHash, hashKey, HASH_FIND, &handleFound); - /* if no corresponding column or null bson value, continue */ - if (columnMapping == NULL || bsonType == BSON_NULL) + /* if no corresponding column or null BSON value, continue */ + if (columnMapping == NULL || bsonType == BSON_TYPE_NULL) { continue; } @@ -1192,7 +1143,7 @@ FillTupleSlot(const bson *bsonDocument, const char *bsonDocumentKey, columnTypeId = columnMapping->columnTypeId; columnArrayTypeId = columnMapping->columnArrayTypeId; - if (OidIsValid(columnArrayTypeId) && bsonType == BSON_ARRAY) + if (OidIsValid(columnArrayTypeId) && bsonType == BSON_TYPE_ARRAY) { compatibleTypes = true; } @@ -1235,7 +1186,7 @@ FillTupleSlot(const bson *bsonDocument, const char *bsonDocumentKey, * internal conversions applied by BSON APIs. */ static bool -ColumnTypesCompatible(bson_type bsonType, Oid columnTypeId) +ColumnTypesCompatible(BSON_TYPE bsonType, Oid columnTypeId) { bool compatibleTypes = false; @@ -1246,8 +1197,8 @@ ColumnTypesCompatible(bson_type bsonType, Oid columnTypeId) case INT8OID: case FLOAT4OID: case FLOAT8OID: case NUMERICOID: { - if (bsonType == BSON_INT || bsonType == BSON_LONG || - bsonType == BSON_DOUBLE) + if (bsonType == BSON_TYPE_INT32 || bsonType == BSON_TYPE_INT64 || + bsonType == BSON_TYPE_DOUBLE) { compatibleTypes = true; } @@ -1255,8 +1206,8 @@ ColumnTypesCompatible(bson_type bsonType, Oid columnTypeId) } case BOOLOID: { - if (bsonType == BSON_INT || bsonType == BSON_LONG || - bsonType == BSON_DOUBLE || bsonType == BSON_BOOL) + if (bsonType == BSON_TYPE_INT32 || bsonType == BSON_TYPE_INT64 || + bsonType == BSON_TYPE_DOUBLE || bsonType == BSON_TYPE_BOOL) { compatibleTypes = true; } @@ -1266,7 +1217,7 @@ ColumnTypesCompatible(bson_type bsonType, Oid columnTypeId) case VARCHAROID: case TEXTOID: { - if (bsonType == BSON_STRING) + if (bsonType == BSON_TYPE_UTF8) { compatibleTypes = true; } @@ -1279,7 +1230,7 @@ ColumnTypesCompatible(bson_type bsonType, Oid columnTypeId) * object identifier. We can safely overload this 64-byte data type * since it's reserved for internal use in PostgreSQL. */ - if (bsonType == BSON_OID) + if (bsonType == BSON_TYPE_OID) { compatibleTypes = true; } @@ -1289,7 +1240,7 @@ ColumnTypesCompatible(bson_type bsonType, Oid columnTypeId) case TIMESTAMPOID: case TIMESTAMPTZOID: { - if (bsonType == BSON_DATE) + if (bsonType == BSON_TYPE_DATE_TIME) { compatibleTypes = true; } @@ -1303,7 +1254,7 @@ ColumnTypesCompatible(bson_type bsonType, Oid columnTypeId) * such as money or inet, do not have equivalents in MongoDB. */ ereport(ERROR, (errcode(ERRCODE_FDW_INVALID_DATA_TYPE), - errmsg("cannot convert bson type to column type"), + errmsg("cannot convert BSON type to column type"), errhint("Column type: %u", (uint32) columnTypeId))); break; } @@ -1320,7 +1271,7 @@ ColumnTypesCompatible(bson_type bsonType, Oid columnTypeId) * datum from element datums, and returns the array datum. */ static Datum -ColumnValueArray(bson_iterator *bsonIterator, Oid valueTypeId) +ColumnValueArray(BSON_ITERATOR *bsonIterator, Oid valueTypeId) { Datum *columnValueArray = palloc0(INITIAL_ARRAY_CAPACITY * sizeof(Datum)); uint32 arrayCapacity = INITIAL_ARRAY_CAPACITY; @@ -1333,16 +1284,15 @@ ColumnValueArray(bson_iterator *bsonIterator, Oid valueTypeId) char typeAlignment = 0; int16 typeLength = 0; - bson_iterator bsonSubIterator = { NULL, 0 }; - bson_iterator_subiterator(bsonIterator, &bsonSubIterator); - - while (bson_iterator_next(&bsonSubIterator)) + BSON_ITERATOR bsonSubIterator = { NULL, 0 }; + BsonIterSubIter(bsonIterator, &bsonSubIterator); + while (BsonIterNext(&bsonSubIterator)) { - bson_type bsonType = bson_iterator_type(&bsonSubIterator); + BSON_TYPE bsonType = BsonIterType(&bsonSubIterator); bool compatibleTypes = false; compatibleTypes = ColumnTypesCompatible(bsonType, valueTypeId); - if (bsonType == BSON_NULL || !compatibleTypes) + if (bsonType == BSON_TYPE_NULL || !compatibleTypes) { continue; } @@ -1373,7 +1323,7 @@ ColumnValueArray(bson_iterator *bsonIterator, Oid valueTypeId) * datum. The function then returns this datum. */ static Datum -ColumnValue(bson_iterator *bsonIterator, Oid columnTypeId, int32 columnTypeMod) +ColumnValue(BSON_ITERATOR *bsonIterator, Oid columnTypeId, int32 columnTypeMod) { Datum columnValue = 0; @@ -1381,37 +1331,37 @@ ColumnValue(bson_iterator *bsonIterator, Oid columnTypeId, int32 columnTypeMod) { case INT2OID: { - int16 value = (int16) bson_iterator_int(bsonIterator); + int16 value = (int16) BsonIterInt32(bsonIterator); columnValue = Int16GetDatum(value); break; } case INT4OID: { - int32 value = bson_iterator_int(bsonIterator); + int32 value = BsonIterInt32(bsonIterator); columnValue = Int32GetDatum(value); break; } case INT8OID: { - int64 value = bson_iterator_long(bsonIterator); + int64 value = BsonIterInt64(bsonIterator); columnValue = Int64GetDatum(value); break; } case FLOAT4OID: { - float4 value = (float4) bson_iterator_double(bsonIterator); + float4 value = (float4) BsonIterDouble(bsonIterator); columnValue = Float4GetDatum(value); break; } case FLOAT8OID: { - float8 value = bson_iterator_double(bsonIterator); + float8 value = BsonIterDouble(bsonIterator); columnValue = Float8GetDatum(value); break; } case NUMERICOID: { - float8 value = bson_iterator_double(bsonIterator); + float8 value = BsonIterDouble(bsonIterator); Datum valueDatum = Float8GetDatum(value); /* overlook type modifiers for numeric */ @@ -1420,13 +1370,13 @@ ColumnValue(bson_iterator *bsonIterator, Oid columnTypeId, int32 columnTypeMod) } case BOOLOID: { - bool value = bson_iterator_bool(bsonIterator); + bool value = BsonIterBool(bsonIterator); columnValue = BoolGetDatum(value); break; } case BPCHAROID: { - const char *value = bson_iterator_string(bsonIterator); + const char *value = BsonIterString(bsonIterator); Datum valueDatum = CStringGetDatum(value); columnValue = DirectFunctionCall3(bpcharin, valueDatum, @@ -1436,7 +1386,7 @@ ColumnValue(bson_iterator *bsonIterator, Oid columnTypeId, int32 columnTypeMod) } case VARCHAROID: { - const char *value = bson_iterator_string(bsonIterator); + const char *value = BsonIterString(bsonIterator); Datum valueDatum = CStringGetDatum(value); columnValue = DirectFunctionCall3(varcharin, valueDatum, @@ -1446,16 +1396,16 @@ ColumnValue(bson_iterator *bsonIterator, Oid columnTypeId, int32 columnTypeMod) } case TEXTOID: { - const char *value = bson_iterator_string(bsonIterator); + const char *value = BsonIterString(bsonIterator); columnValue = CStringGetTextDatum(value); break; } - case NAMEOID: + case NAMEOID: { char value[NAMEDATALEN]; Datum valueDatum = 0; - bson_oid_t *bsonObjectId = bson_iterator_oid(bsonIterator); + bson_oid_t *bsonObjectId = (bson_oid_t*) BsonIterOid(bsonIterator); bson_oid_to_string(bsonObjectId, value); valueDatum = CStringGetDatum(value); @@ -1466,7 +1416,7 @@ ColumnValue(bson_iterator *bsonIterator, Oid columnTypeId, int32 columnTypeMod) } case DATEOID: { - int64 valueMillis = bson_iterator_date(bsonIterator); + int64 valueMillis = BsonIterDate(bsonIterator); int64 timestamp = (valueMillis * 1000L) - POSTGRES_TO_UNIX_EPOCH_USECS; Datum timestampDatum = TimestampGetDatum(timestamp); @@ -1476,7 +1426,7 @@ ColumnValue(bson_iterator *bsonIterator, Oid columnTypeId, int32 columnTypeMod) case TIMESTAMPOID: case TIMESTAMPTZOID: { - int64 valueMillis = bson_iterator_date(bsonIterator); + int64 valueMillis = BsonIterDate(bsonIterator); int64 timestamp = (valueMillis * 1000L) - POSTGRES_TO_UNIX_EPOCH_USECS; /* overlook type modifiers for timestamp */ @@ -1486,7 +1436,7 @@ ColumnValue(bson_iterator *bsonIterator, Oid columnTypeId, int32 columnTypeMod) default: { ereport(ERROR, (errcode(ERRCODE_FDW_INVALID_DATA_TYPE), - errmsg("cannot convert bson type to column type"), + errmsg("cannot convert BSON type to column type"), errhint("Column type: %u", (uint32) columnTypeId))); break; } @@ -1504,15 +1454,20 @@ static void MongoFreeScanState(MongoFdwModifyState *fmstate) { if (fmstate == NULL) - { return; + + if (fmstate->queryDocument) + { + BsonDestroy(fmstate->queryDocument); + fmstate->queryDocument = NULL; } - bson_destroy(fmstate->queryDocument); - bson_dispose(fmstate->queryDocument); + if (fmstate->mongoCursor) + { + MongoCursorDestroy(fmstate->mongoCursor); + fmstate->mongoCursor = NULL; + } - mongo_cursor_destroy(fmstate->mongoCursor); - mongo_cursor_dispose(fmstate->mongoCursor); /* Release remote connection */ ReleaseConnection(fmstate->mongoConnection); } @@ -1535,6 +1490,7 @@ MongoAnalyzeForeignTable(Relation relation, double foreignTableSize = 0; foreignTableId = RelationGetRelid(relation); + documentCount = ForeignTableDocumentCount(foreignTableId); if (documentCount > 0.0) @@ -1597,8 +1553,8 @@ MongoAcquireSampleRows(Relation relation, int errorLevel, AttrNumber columnCount = 0; AttrNumber columnId = 0; HTAB *columnMappingHash = NULL; - mongo_cursor *mongoCursor = NULL; - bson *queryDocument = NULL; + MONGO_CURSOR *mongoCursor = NULL; + BSON *queryDocument = NULL; List *columnList = NIL; ForeignScanState *scanState = NULL; List *foreignPrivateList = NIL; @@ -1635,7 +1591,7 @@ MongoAcquireSampleRows(Relation relation, int errorLevel, foreignPrivateList = list_make1(columnList); /* only clean up the query struct, but not its data */ - bson_dispose(queryDocument); + BsonDestroy(queryDocument); foreignScan = makeNode(ForeignScan); foreignScan->fdw_private = foreignPrivateList; @@ -1666,8 +1622,6 @@ MongoAcquireSampleRows(Relation relation, int errorLevel, for (;;) { - int32 cursorStatus = MONGO_ERROR; - /* check for user-requested abort or sleep */ vacuum_delay_point(); @@ -1675,10 +1629,9 @@ MongoAcquireSampleRows(Relation relation, int errorLevel, memset(columnValues, 0, columnCount * sizeof(Datum)); memset(columnNulls, true, columnCount * sizeof(bool)); - cursorStatus = mongo_cursor_next(mongoCursor); - if (cursorStatus == MONGO_OK) + if(MongoCursorNext(mongoCursor, NULL)) { - const bson *bsonDocument = mongo_cursor_bson(mongoCursor); + const BSON *bsonDocument = MongoCursorBson(mongoCursor); const char *bsonDocumentKey = NULL; /* top level document */ /* fetch next tuple */ @@ -1686,25 +1639,27 @@ MongoAcquireSampleRows(Relation relation, int errorLevel, MemoryContextSwitchTo(tupleContext); FillTupleSlot(bsonDocument, bsonDocumentKey, - columnMappingHash, columnValues, columnNulls); + columnMappingHash, columnValues, columnNulls); MemoryContextSwitchTo(oldContext); } else { + #ifndef META_DRIVER /* * The following is a courtesy check. In practice when Mongo shuts down, - * mongo_cursor_next() could possibly crash. + * mongo_cursor__next() could possibly crash. */ mongo_cursor_error_t errorCode = mongoCursor->err; + if (errorCode != MONGO_CURSOR_EXHAUSTED) { MongoFreeScanState(fmstate); - ereport(ERROR, (errmsg("could not iterate over mongo collection"), + ereport(ERROR, (errmsg("could not iterate over mongo 11collection"), errhint("Mongo driver cursor error code: %d", errorCode))); } - + #endif break; } @@ -1745,8 +1700,8 @@ MongoAcquireSampleRows(Relation relation, int errorLevel, heap_freetuple(sampleRows[rowIndex]); sampleRows[rowIndex] = heap_form_tuple(tupleDescriptor, - columnValues, - columnNulls); + columnValues, + columnNulls); } rowCountToSkip -= 1; diff --git a/mongo_fdw.h b/mongo_fdw.h index 8be4d10..f72af94 100644 --- a/mongo_fdw.h +++ b/mongo_fdw.h @@ -12,11 +12,16 @@ #ifndef MONGO_FDW_H #define MONGO_FDW_H +#include "config.h" +#include "mongo_wrapper.h" #include "bson.h" -#include "mongo.h" -#include "postgres.h" -#include "utils/hsearch.h" +#ifdef META_DRIVER + #include "mongoc.h" +#else + #include "mongo.h" +#endif + #include "fmgr.h" #include "catalog/pg_foreign_server.h" #include "catalog/pg_foreign_table.h" @@ -45,7 +50,29 @@ #include "utils/rel.h" #include "utils/memutils.h" - +#ifdef META_DRIVER + #define BSON bson_t + #define BSON_TYPE bson_type_t + #define BSON_ITERATOR bson_iter_t + #define MONGO_CONN mongoc_client_t + #define MONGO_CURSOR mongoc_cursor_t +#else + #define BSON bson + #define BSON_TYPE bson_type + #define BSON_ITERATOR bson_iterator + #define MONGO_CONN mongo + #define MONGO_CURSOR mongo_cursor + #define BSON_TYPE_DOCUMENT BSON_OBJECT + #define BSON_TYPE_NULL BSON_NULL + #define BSON_TYPE_ARRAY BSON_ARRAY + #define BSON_TYPE_INT32 BSON_INT + #define BSON_TYPE_INT64 BSON_LONG + #define BSON_TYPE_DOUBLE BSON_DOUBLE + #define BSON_TYPE_BOOL BSON_BOOL + #define BSON_TYPE_UTF8 BSON_STRING + #define BSON_TYPE_OID BSON_OID + #define BSON_TYPE_DATE_TIME BSON_DATE +#endif /* Defines for valid option names */ #define OPTION_NAME_ADDRESS "address" @@ -105,37 +132,35 @@ typedef struct MongoFdwOptions int32 portNumber; char *databaseName; char *collectionName; - } MongoFdwOptions; /* -* MongoFdwExecState keeps foreign data wrapper specific execution state that we -* create and hold onto when executing the query. -*/ + * MongoFdwExecState keeps foreign data wrapper specific execution state that we + * create and hold onto when executing the query. + */ /* -* Execution state of a foreign insert/update/delete operation. -*/ + * Execution state of a foreign insert/update/delete operation. + */ typedef struct MongoFdwModifyState { - Relation rel; /* relcache entry for the foreign table */ - - List *target_attrs; /* list of target attribute numbers */ + Relation rel; /* relcache entry for the foreign table */ + List *target_attrs; /* list of target attribute numbers */ /* info about parameters for prepared statement */ - int p_nums; /* number of parameters to transmit */ - FmgrInfo *p_flinfo; /* output conversion functions for them */ + int p_nums; /* number of parameters to transmit */ + FmgrInfo *p_flinfo; /* output conversion functions for them */ - struct HTAB *columnMappingHash; + struct HTAB *columnMappingHash; - mongo *mongoConnection; /* MongoDB connection */ - mongo_cursor *mongoCursor; /* MongoDB cursor */ - bson *queryDocument; /* Bson Document */ + MONGO_CONN *mongoConnection; /* MongoDB connection */ + MONGO_CURSOR *mongoCursor; /* MongoDB cursor */ + BSON *queryDocument; /* Bson Document */ - MongoFdwOptions *mongoFdwOptions; + MongoFdwOptions *mongoFdwOptions; /* working memory context */ - MemoryContext temp_cxt; /* context for per-tuple temporary data */ + MemoryContext temp_cxt; /* context for per-tuple temporary data */ } MongoFdwModifyState; @@ -155,19 +180,23 @@ typedef struct ColumnMapping } ColumnMapping; +extern MONGO_CONN *GetConnection(char *host, int32 port); +extern void cleanup_connection(void); +extern void ReleaseConnection(MONGO_CONN* conn); + +extern StringInfo OptionNamesString(Oid currentContextId); + /* Function declarations related to creating the mongo query */ extern List * ApplicableOpExpressionList(RelOptInfo *baserel); -extern bson * QueryDocument(Oid relationId, List *opExpressionList); +extern BSON * QueryDocument(Oid relationId, List *opExpressionList); extern List * ColumnList(RelOptInfo *baserel); +extern MongoFdwOptions * MongoGetOptions(Oid foreignTableId); +extern void MongoFreeOptions(MongoFdwOptions *mongoFdwOptions); + /* Function declarations for foreign data wrapper */ extern Datum mongo_fdw_handler(PG_FUNCTION_ARGS); extern Datum mongo_fdw_validator(PG_FUNCTION_ARGS); -extern MongoFdwOptions * MongoGetOptions(Oid foreignTableId); -extern void MongoFreeOptions(MongoFdwOptions *mongoFdwOptions); -mongo* GetConnection(char *host, int32 port); -void cleanup_connection(void); -void ReleaseConnection(mongo *conn); #endif /* MONGO_FDW_H */ diff --git a/mongo_query.c b/mongo_query.c index 426e801..a559a0b 100644 --- a/mongo_query.c +++ b/mongo_query.c @@ -12,6 +12,13 @@ */ #include "postgres.h" +#include "mongo_wrapper.h" + +#ifdef META_DRIVER + #include "mongoc.h" +#else + #include "mongo.h" +#endif #include "mongo_fdw.h" #include "mongo_query.h" @@ -33,7 +40,7 @@ static char * MongoOperatorName(const char *operatorName); static List * EqualityOperatorList(List *operatorList); static List * UniqueColumnList(List *operatorList); static List * ColumnOperatorList(Var *column, List *operatorList); -static void AppendConstantValue(bson *queryDocument, const char *keyName, +static void AppendConstantValue(BSON *queryDocument, const char *keyName, Const *constant); @@ -154,7 +161,7 @@ FindArgumentOfType(List *argumentList, NodeTag argumentType) * "l_shipdate >= date '1994-01-01' AND l_shipdate < date '1995-01-01'" become * "l_shipdate: { $gte: new Date(757382400000), $lt: new Date(788918400000) }". */ -bson * +BSON * QueryDocument(Oid relationId, List *opExpressionList) { List *equalityOperatorList = NIL; @@ -162,12 +169,9 @@ QueryDocument(Oid relationId, List *opExpressionList) List *columnList = NIL; ListCell *equalityOperatorCell = NULL; ListCell *columnCell = NULL; - bson *queryDocument = NULL; - int documentStatus = BSON_OK; - - queryDocument = bson_create(); - bson_init(queryDocument); + BSON *queryDocument = NULL; + queryDocument = BsonCreate(); /* * We distinguish between equality expressions and others since we need to * insert the latter (<, >, <=, >=, <>) as separate sub-documents into the @@ -210,6 +214,7 @@ QueryDocument(Oid relationId, List *opExpressionList) char *columnName = NULL; List *columnOperatorList = NIL; ListCell *columnOperatorCell = NULL; + BSON r; columnId = column->varattno; columnName = get_relid_attribute_name(relationId, columnId); @@ -218,7 +223,7 @@ QueryDocument(Oid relationId, List *opExpressionList) columnOperatorList = ColumnOperatorList(column, comparisonOperatorList); /* for comparison expressions, start a sub-document */ - bson_append_start_object(queryDocument, columnName); + BsonAppendStartObject(queryDocument, columnName, &r); foreach(columnOperatorCell, columnOperatorList) { @@ -234,15 +239,12 @@ QueryDocument(Oid relationId, List *opExpressionList) AppendConstantValue(queryDocument, mongoOperatorName, constant); } - - bson_append_finish_object(queryDocument); + BsonAppendFinishObject(queryDocument, &r); } - - documentStatus = bson_finish(queryDocument); - if (documentStatus != BSON_OK) + if (!BsonFinish(queryDocument)) { ereport(ERROR, (errmsg("could not create document for query"), - errhint("BSON error: %s", queryDocument->errstr))); + errhint("BSON error"))); } return queryDocument; @@ -362,24 +364,23 @@ ColumnOperatorList(Var *column, List *operatorList) * its MongoDB equivalent. */ static void -AppendConstantValue(bson *queryDocument, const char *keyName, Const *constant) +AppendConstantValue(BSON *queryDocument, const char *keyName, Const *constant) { if (constant->constisnull) { - bson_append_null(queryDocument, keyName); + BsonAppendNull(queryDocument, keyName); return; } - AppenMongoValue(queryDocument, keyName, constant->constvalue, constant->consttype, false); + AppenMongoValue(queryDocument, keyName, constant->constvalue, false, constant->consttype); } -int32 -AppenMongoValue(bson *queryDocument, const char *keyName, Datum value, bool isnull, Oid id) +bool +AppenMongoValue(BSON *queryDocument, const char *keyName, Datum value, bool isnull, Oid id) { - int32 status = MONGO_ERROR; - + bool status; if (isnull) { - status = bson_append_null(queryDocument, keyName); + status = BsonAppendNull(queryDocument, keyName); return status; } @@ -388,44 +389,44 @@ AppenMongoValue(bson *queryDocument, const char *keyName, Datum value, bool isnu case INT2OID: { int16 valueInt = DatumGetInt16(value); - status = bson_append_int(queryDocument, keyName, (int) valueInt); + status = BsonAppendInt32(queryDocument, keyName, (int) valueInt); break; } case INT4OID: { int32 valueInt = DatumGetInt32(value); - status = bson_append_int(queryDocument, keyName, valueInt); + status = BsonAppendInt32(queryDocument, keyName, valueInt); break; } case INT8OID: { int64 valueLong = DatumGetInt64(value); - status = bson_append_long(queryDocument, keyName, valueLong); + status = BsonAppendInt64(queryDocument, keyName, valueLong); break; } case FLOAT4OID: { float4 valueFloat = DatumGetFloat4(value); - status = bson_append_double(queryDocument, keyName, (double) valueFloat); + status = BsonAppendDouble(queryDocument, keyName, (double) valueFloat); break; } case FLOAT8OID: { float8 valueFloat = DatumGetFloat8(value); - status = bson_append_double(queryDocument, keyName, valueFloat); + status = BsonAppendDouble(queryDocument, keyName, valueFloat); break; } case NUMERICOID: { Datum valueDatum = DirectFunctionCall1(numeric_float8, value); float8 valueFloat = DatumGetFloat8(valueDatum); - status = bson_append_double(queryDocument, keyName, valueFloat); + status = BsonAppendDouble(queryDocument, keyName, valueFloat); break; } case BOOLOID: { bool valueBool = DatumGetBool(value); - status = bson_append_int(queryDocument, keyName, (int) valueBool); + status = BsonAppendBool(queryDocument, keyName, (int) valueBool); break; } case BPCHAROID: @@ -437,7 +438,7 @@ AppenMongoValue(bson *queryDocument, const char *keyName, Datum value, bool isnu bool typeVarLength = false; getTypeOutputInfo(id, &outputFunctionId, &typeVarLength); outputString = OidOutputFunctionCall(outputFunctionId, value); - status = bson_append_string(queryDocument, keyName, outputString); + status = BsonAppendUTF8(queryDocument, keyName, outputString); break; } case NAMEOID: @@ -449,8 +450,8 @@ AppenMongoValue(bson *queryDocument, const char *keyName, Datum value, bool isnu memset(bsonObjectId.bytes, 0, sizeof(bsonObjectId.bytes)); getTypeOutputInfo(id, &outputFunctionId, &typeVarLength); outputString = OidOutputFunctionCall(outputFunctionId, value); - bson_oid_from_string(&bsonObjectId, outputString); - status = bson_append_oid(queryDocument, keyName, &bsonObjectId); + BsonOidFromString(&bsonObjectId, outputString); + status = BsonAppendOid(queryDocument, keyName, &bsonObjectId); break; } case DATEOID: @@ -460,7 +461,7 @@ AppenMongoValue(bson *queryDocument, const char *keyName, Datum value, bool isnu int64 valueMicroSecs = valueTimestamp + POSTGRES_TO_UNIX_EPOCH_USECS; int64 valueMilliSecs = valueMicroSecs / 1000; - status = bson_append_date(queryDocument, keyName, valueMilliSecs); + status = BsonAppendDate(queryDocument, keyName, valueMilliSecs); break; } case TIMESTAMPOID: @@ -470,7 +471,7 @@ AppenMongoValue(bson *queryDocument, const char *keyName, Datum value, bool isnu int64 valueMicroSecs = valueTimestamp + POSTGRES_TO_UNIX_EPOCH_USECS; int64 valueMilliSecs = valueMicroSecs / 1000; - status = bson_append_date(queryDocument, keyName, valueMilliSecs); + status = BsonAppendDate(queryDocument, keyName, valueMilliSecs); break; } default: diff --git a/mongo_query.h b/mongo_query.h index 87b95e5..607bb2d 100644 --- a/mongo_query.h +++ b/mongo_query.h @@ -12,7 +12,6 @@ #ifndef MONGO_QUERY_H #define MONGO_QUERY_H -int32 AppenMongoValue(bson *queryDocument, const char *keyName, Datum value, bool isnull, Oid id); +bool AppenMongoValue(BSON *queryDocument, const char *keyName, Datum value, bool isnull, Oid id); - -#endif /* MONGO_QUERY_H */ +#endif /* MONGO_QUERY_H */ From 3dd8ef6685d7d3a5709a367367d4ea726d29003a Mon Sep 17 00:00:00 2001 From: Ibrar Ahmed Date: Mon, 2 Jun 2014 18:25:02 +0500 Subject: [PATCH 05/43] Added files missed during last commits. --- Makefile.legacy | 42 +++++ Makefile.meta | 44 +++++ config.h | 5 + connection.c | 141 +++++++++++++++ mongo_wrapper.c | 318 +++++++++++++++++++++++++++++++++ mongo_wrapper.h | 62 +++++++ mongo_wrapper_meta.c | 407 +++++++++++++++++++++++++++++++++++++++++++ option.c | 232 ++++++++++++++++++++++++ 8 files changed, 1251 insertions(+) create mode 100644 Makefile.legacy create mode 100644 Makefile.meta create mode 100644 config.h create mode 100644 connection.c create mode 100644 mongo_wrapper.c create mode 100644 mongo_wrapper.h create mode 100644 mongo_wrapper_meta.c create mode 100644 option.c diff --git a/Makefile.legacy b/Makefile.legacy new file mode 100644 index 0000000..88e6ce0 --- /dev/null +++ b/Makefile.legacy @@ -0,0 +1,42 @@ +# mongo_fdw/Makefile +# +# Copyright (c) 2012-2014 Citus Data, Inc. +# + +MODULE_big = mongo_fdw + +# +# We assume we are running on a POSIX compliant system (Linux, OSX). If you are +# on another platform, change env_posix.os in MONGO_OBJS with the appropriate +# environment object file. +# + +MONGO_DRIVER = mongo-c-driver-v0.6 +MONGO_PATH = $(MONGO_DRIVER)/src +MONGO_OBJS = $(MONGO_PATH)/bson.os $(MONGO_PATH)/encoding.os $(MONGO_PATH)/md5.os $(MONGO_PATH)/mongo.os $(MONGO_PATH)/numbers.os $(MONGO_PATH)/env_posix.os + +PG_CPPFLAGS = --std=c99 -I$(MONGO_PATH) +OBJS = connection.o option.o mongo_wrapper.o mongo_fdw.o mongo_query.o $(MONGO_OBJS) + +EXTENSION = mongo_fdw +DATA = mongo_fdw--1.0.sql + +$(MONGO_DRIVER)/%.os: + $(MAKE) -C $(MONGO_DRIVER) $*.os + +# +# Users need to specify their Postgres installation path through pg_config. For +# example: /usr/local/pgsql/bin/pg_config or /usr/lib/postgresql/9.1/bin/pg_config +# + +PG_CONFIG = pg_config +PGXS := $(shell $(PG_CONFIG) --pgxs) +include $(PGXS) + +ifndef MAJORVERSION + MAJORVERSION := $(basename $(VERSION)) +endif + +ifeq (,$(findstring $(MAJORVERSION), 9.3 9.4)) + $(error PostgreSQL 9.3 or 9.4 is required to compile this extension) +endif diff --git a/Makefile.meta b/Makefile.meta new file mode 100644 index 0000000..10a12e4 --- /dev/null +++ b/Makefile.meta @@ -0,0 +1,44 @@ +# mongo_fdw/Makefile +# +# Copyright (c) 2012-2014 Citus Data, Inc. +# + +MODULE_big = mongo_fdw + +# +# We assume we are running on a POSIX compliant system (Linux, OSX). If you are +# on another platform, change env_posix.os in MONGO_OBJS with the appropriate +# environment object file. +# + +MONGO_DRIVER = mongo-c-meta-driver +MONGO_PATH = $(MONGO_DRIVER)/src/mongoc +MONGO_INCLUDE = -I$(MONGO_DRIVER)/src/libbson/src/bson/ -I$(MONGO_PATH) +PG_CPPFLAGS = --std=c99 $(MONGO_INCLUDE) +SHLIB_LINK = -L$(MONGO_DRIVER)/.libs -lmongoc-1.0 +SHLIB_LINK += -L$(MONGO_DRIVER)/src/libbson/.libs -lbson-1.0 + +OBJS = connection.o option.o mongo_wrapper_meta.o mongo_fdw.o mongo_query.o + +EXTENSION = mongo_fdw +DATA = mongo_fdw--1.0.sql + +$(MONGO_DRIVER)/%.os: + $(MAKE) -C $(MONGO_DRIVER) $*.os + +# +# Users need to specify their Postgres installation path through pg_config. For +# example: /usr/local/pgsql/bin/pg_config or /usr/lib/postgresql/9.1/bin/pg_config +# + +PG_CONFIG = pg_config +PGXS := $(shell $(PG_CONFIG) --pgxs) +include $(PGXS) + +ifndef MAJORVERSION + MAJORVERSION := $(basename $(VERSION)) +endif + +ifeq (,$(findstring $(MAJORVERSION), 9.3 9.4)) + $(error PostgreSQL 9.3 or 9.4 is required to compile this extension) +endif diff --git a/config.h b/config.h new file mode 100644 index 0000000..6139f3e --- /dev/null +++ b/config.h @@ -0,0 +1,5 @@ +/* + * Define if you want to compile the MongoFDW with Meta C Driver, otherwise + * it will compile using MongoDB legacy C Driver +*/ +/* #define META_DRIVER */ diff --git a/connection.c b/connection.c new file mode 100644 index 0000000..9b8660e --- /dev/null +++ b/connection.c @@ -0,0 +1,141 @@ +/*------------------------------------------------------------------------- + * + * connection.c + * Connection management functions for mongo_fdw + * + * Portions Copyright (c) 2004-2014, EnterpriseDB Corporation. + * + *------------------------------------------------------------------------- + */ +#include + + +#include "postgres.h" +#include "mongo_wrapper.h" +#include "mongo_fdw.h" + +#include "access/xact.h" +#include "mb/pg_wchar.h" +#include "miscadmin.h" +#include "utils/hsearch.h" +#include "utils/memutils.h" +#include "utils/resowner.h" + +/* Length of host */ +#define HOST_LEN 256 + +/* + * Connection cache hash table entry + * + * The lookup key in this hash table is the foreign Mongo server name / IP + * and the server port number. (We use just one connection per user per foreign server, + * so that we can ensure all scans use the same snapshot during a query.) + * + * The "conn" pointer can be NULL if we don't currently have a live connection. + */ +typedef struct ConnCacheKey +{ + char host[HOST_LEN]; /* MongoDB's host name / IP address */ + int32 port; /* MongoDB's port number */ +} ConnCacheKey; + +typedef struct ConnCacheEntry +{ + ConnCacheKey key; /* hash key (must be first) */ + MONGO_CONN *conn; /* connection to foreign server, or NULL */ +} ConnCacheEntry; + +/* + * Connection cache (initialized on first use) + */ +static HTAB *ConnectionHash = NULL; + +/* + * GetConnection: + * Get a mong connection which can be used to execute queries on + * the remote Mongo server with the user's authorization. A new connection + * is established if we don't already have a suitable one. + */ +MONGO_CONN* +GetConnection(char *host, int32 port) +{ + bool found; + ConnCacheEntry *entry; + ConnCacheKey key; + + /* First time through, initialize connection cache hashtable */ + if (ConnectionHash == NULL) + { + HASHCTL ctl; + MemSet(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(ConnCacheKey); + ctl.entrysize = sizeof(ConnCacheEntry); + ctl.hash = tag_hash; + /* allocate ConnectionHash in the cache context */ + ctl.hcxt = CacheMemoryContext; + ConnectionHash = hash_create("mongo_fdw connections", 8, + &ctl, + HASH_ELEM | HASH_FUNCTION | HASH_CONTEXT); + } + + /* Create hash key for the entry */ + memset(key.host, 0, HOST_LEN); + strncpy(key.host, host, HOST_LEN); + key.port = port; + + /* + * Find or create cached entry for requested connection. + */ + entry = hash_search(ConnectionHash, &key, HASH_ENTER, &found); + if (!found) + { + /* initialize new hashtable entry (key is already filled in) */ + entry->conn = NULL; + } + if (entry->conn == NULL) + { + entry->conn = MongoConnect(host, port); + elog(DEBUG3, "new mongo_fdw connection %p for server \"%s:%d\"", + entry->conn, host, port); + } + + return entry->conn; +} + +/* + * cleanup_connection: + * Delete all the cache entries on backend exists. + */ +void +cleanup_connection() +{ + HASH_SEQ_STATUS scan; + ConnCacheEntry *entry; + + if (ConnectionHash == NULL) + return; + + hash_seq_init(&scan, ConnectionHash); + while ((entry = (ConnCacheEntry *) hash_seq_search(&scan))) + { + if (entry->conn == NULL) + continue; + + elog(DEBUG3, "disconnecting mongo_fdw connection %p", entry->conn); + MongoDisconnect(entry->conn); + entry->conn = NULL; + } +} + +/* + * Release connection created by calling GetConnection. + */ +void +ReleaseConnection(MONGO_CONN *conn) +{ + /* + * We don't close the connection indvisually here, will do all connection + * cleanup on the backend exit. + */ +} + diff --git a/mongo_wrapper.c b/mongo_wrapper.c new file mode 100644 index 0000000..9c77c41 --- /dev/null +++ b/mongo_wrapper.c @@ -0,0 +1,318 @@ +/*------------------------------------------------------------------------- + * + * mongo_wrapper.c + * + * Wrapper functions for MongoDB's old legacy Driver. + * + * Portions Copyright (c) 2004-2014, EnterpriseDB Corporation. + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" +#include "mongo_wrapper.h" + +#ifdef META_DRIVER + #include "mongoc.h" +#else + #include "mongo.h" +#endif + +#include "mongo_fdw.h" + +#define QUAL_STRING_LEN 512 + +MONGO_CONN* +MongoConnect(const char* host, const unsigned short port) +{ + MONGO_CONN *conn; + conn = mongo_create(); + mongo_init(conn); + + if (mongo_connect(conn, host, port) != MONGO_OK) + { + int err = conn->err; + mongo_destroy(conn); + mongo_dispose(conn); + ereport(ERROR, (errmsg("could not connect to %s:%d", host, port), + errhint("Mongo driver connection error: %d", err))); + } + return conn; +} + +void +MongoDisconnect(MONGO_CONN* conn) +{ + mongo_destroy(conn); + mongo_dispose(conn); +} + +bool +MongoInsert(MONGO_CONN* conn, char* database, char *collection, bson* b) +{ + char qual[QUAL_STRING_LEN]; + + snprintf (qual, QUAL_STRING_LEN, "%s.%s", database, collection); + if (mongo_insert(conn, qual, b, NULL) != MONGO_OK) + ereport(ERROR, (errmsg("failed to insert row"), + errhint("Mongo driver insert error: %d", conn->err))); + return true; +} + + +bool +MongoUpdate(MONGO_CONN* conn, char* database, char *collection, BSON* b, BSON* op) +{ + char qual[QUAL_STRING_LEN]; + + snprintf (qual, QUAL_STRING_LEN, "%s.%s", database, collection); + if (mongo_update(conn, qual, b, op, MONGO_UPDATE_BASIC, 0) != MONGO_OK) + ereport(ERROR, (errmsg("failed to update row"), + errhint("Mongo driver update error: %d", conn->err))); + return true; +} + + +bool +MongoDelete(MONGO_CONN* conn, char* database, char *collection, BSON* b) +{ + char qual[QUAL_STRING_LEN]; + + snprintf (qual, QUAL_STRING_LEN, "%s.%s", database, collection); + if (mongo_remove(conn, qual, b , NULL) != MONGO_OK) + ereport(ERROR, (errmsg("failed to delete row"), + errhint("Mongo driver delete error: %d", conn->err))); + + return true; +} + + +MONGO_CURSOR* +MongoCursorCreate(MONGO_CONN* conn, char* database, char *collection, BSON* q) +{ + MONGO_CURSOR* c; + char qual[QUAL_STRING_LEN]; + + snprintf (qual, QUAL_STRING_LEN, "%s.%s", database, collection); + c = mongo_cursor_create(); + mongo_cursor_init(c, conn , qual); + mongo_cursor_set_query(c, q); + return c; +} + + +const bson* +MongoCursorBson(MONGO_CURSOR* c) +{ + return mongo_cursor_bson(c); +} + + +bool +MongoCursorNext(MONGO_CURSOR* c, BSON* b) +{ + return (mongo_cursor_next(c) == MONGO_OK); +} + +void +MongoCursorDestroy(MONGO_CURSOR* c) +{ + mongo_cursor_destroy(c); + mongo_cursor_dispose(c); +} + +BSON* +BsonCreate() +{ + BSON *b = NULL; + b = bson_create(); + bson_init(b); + return b; +} + +void +BsonDestroy(BSON *b) +{ + bson_destroy(b); + bson_dispose(b); +} + +bool +BsonIterInit(BSON_ITERATOR *it, BSON *b) +{ + bson_iterator_init(it, b); + return true; +} + +bool +BsonIterSubObject(BSON_ITERATOR *it, BSON *b) +{ + bson_iterator_subobject(it, b); + return true; +} + +int32_t +BsonIterInt32(BSON_ITERATOR *it) +{ + return bson_iterator_int(it); +} + + +int64_t +BsonIterInt64(BSON_ITERATOR *it) +{ + return bson_iterator_long(it); +} + + +double +BsonIterDouble(BSON_ITERATOR *it) +{ + return bson_iterator_double(it); +} + + +bool +BsonIterBool(BSON_ITERATOR *it) +{ + return bson_iterator_bool(it); +} + + +const char* +BsonIterString(BSON_ITERATOR *it) +{ + return bson_iterator_string(it); +} + +const bson_oid_t * +BsonIterOid(BSON_ITERATOR *it) +{ + return bson_iterator_oid(it); +} + + +time_t +BsonIterDate(BSON_ITERATOR *it) +{ + return bson_iterator_date(it); +} + + +const char* +BsonIterKey(BSON_ITERATOR *it) +{ + return bson_iterator_key(it); +} + +int +BsonIterType(BSON_ITERATOR *it) +{ + return bson_iterator_type(it); +} + +int +BsonIterNext(BSON_ITERATOR *it) +{ + return bson_iterator_next(it); +} + + +bool +BsonIterSubIter(BSON_ITERATOR *it, BSON_ITERATOR* sub) +{ + bson_iterator_subiterator(it, sub); + return true; +} + + +void +BsonOidFromString(bson_oid_t *o, char* str) +{ + bson_oid_from_string(o, str); +} + + +bool +BsonAppendOid(BSON *b, const char* key, bson_oid_t *v) +{ + return (bson_append_oid(b, key, v) == MONGO_OK); +} + +bool +BsonAppendBool(BSON *b, const char* key, bool v) +{ + return (bson_append_int(b, key, v) == MONGO_OK); +} + +bool +BsonAppendNull(BSON *b, const char* key) +{ + return (bson_append_null(b, key) == MONGO_OK); +} + + +bool +BsonAppendInt32(BSON *b, const char* key, int v) +{ + return (bson_append_int(b, key, v) == MONGO_OK); +} + + +bool +BsonAppendInt64(BSON *b, const char* key, int64_t v) +{ + return (bson_append_long(b, key, v) == MONGO_OK); +} + +bool +BsonAppendDouble(BSON *b, const char* key, double v) +{ + return (bson_append_double(b, key, v) == MONGO_OK); +} + +bool +BsonAppendUTF8(BSON *b, const char* key, char *v) +{ + return (bson_append_string(b, key, v) == MONGO_OK); +} + +bool +BsonAppendDate(BSON *b, const char* key, time_t v) +{ + return (bson_append_date(b, key, v) == MONGO_OK); +} + + +bool +BsonAppendStartObject(BSON* b, char *key, BSON* r) +{ + return (bson_append_start_object(b, key) == MONGO_OK); +} + +bool +BsonAppendFinishObject(BSON* b, BSON* r) +{ + return (bson_append_finish_object(b) == MONGO_OK); +} + + +bool +BsonAppendBson(BSON* b, char *key, BSON* c) +{ + return (bson_append_bson(b, key, c) == MONGO_OK); +} + + +bool +BsonFinish(BSON* b) +{ + return (bson_finish(b) == MONGO_OK); +} + +int +MongoAggregateCount(MONGO_CONN* conn, const char* database, const char* collection, const BSON* b) +{ + return mongo_count(conn, database, collection, b); +} + diff --git a/mongo_wrapper.h b/mongo_wrapper.h new file mode 100644 index 0000000..6cc8c21 --- /dev/null +++ b/mongo_wrapper.h @@ -0,0 +1,62 @@ +/*------------------------------------------------------------------------- + * + * mongo_wrapper.h + * + *------------------------------------------------------------------------- + */ + +#ifndef MONGO_WRAPPER_H +#define MONGO_WRAPPER_H + +#include "mongo_fdw.h" +#include "bson.h" + +#ifdef META_DRIVER + #include "mongoc.h" +#else + #include "mongo.h" +#endif + +MONGO_CONN* MongoConnect(const char* host, const unsigned short port); +void MongoDisconnect(MONGO_CONN* conn); +bool MongoInsert(MONGO_CONN* conn, char* database, char *collection, BSON* b); +bool MongoUpdate(MONGO_CONN* conn, char* database, char *collection, BSON* b, BSON* op); +bool MongoDelete(MONGO_CONN* conn, char* database, char *collection, BSON* b); +MONGO_CURSOR* MongoCursorCreate(MONGO_CONN* conn, char* database, char *collection, BSON* q); +const BSON* MongoCursorBson(MONGO_CURSOR* c); +bool MongoCursorNext(MONGO_CURSOR* c, BSON* b); +void MongoCursorDestroy(MONGO_CURSOR* c); +int MongoAggregateCount(MONGO_CONN* conn, const char* database, const char* collection, const BSON* b); + +BSON* BsonCreate(void); +void BsonDestroy(BSON *b); + +bool BsonIterInit(BSON_ITERATOR *it, BSON *b); +bool BsonIterSubObject(BSON_ITERATOR *it, BSON *b); +int32_t BsonIterInt32(BSON_ITERATOR *it); +int64_t BsonIterInt64(BSON_ITERATOR *it); +double BsonIterDouble(BSON_ITERATOR *it); +bool BsonIterBool(BSON_ITERATOR *it); +const char* BsonIterString(BSON_ITERATOR *it); +const bson_oid_t * BsonIterOid(BSON_ITERATOR *it); +time_t BsonIterDate(BSON_ITERATOR *it); +const char* BsonIterKey(BSON_ITERATOR *it); +int BsonIterType(BSON_ITERATOR *it); +int BsonIterNext(BSON_ITERATOR *it); +bool BsonIterSubIter(BSON_ITERATOR *it, BSON_ITERATOR* sub); +void BsonOidFromString(bson_oid_t *o, char* str); + +BSON *BsonCreate(); +bool BsonAppendOid(BSON *b, const char* key, bson_oid_t *v); +bool BsonAppendBool(BSON *b, const char* key, bool v); +bool BsonAppendNull(BSON *b, const char* key); +bool BsonAppendInt32(BSON *b, const char* key, int v); +bool BsonAppendInt64(BSON *b, const char* key, int64_t v); +bool BsonAppendDouble(BSON *b, const char* key, double v); +bool BsonAppendUTF8(BSON *b, const char* key, char *v); +bool BsonAppendDate(BSON *b, const char* key, time_t v); +bool BsonAppendStartObject(BSON* b, char *key, BSON *r); +bool BsonAppendFinishObject(BSON* b, BSON* r); +bool BsonAppendBson(BSON* b, char *key, BSON* c); +bool BsonFinish(BSON* b); +#endif diff --git a/mongo_wrapper_meta.c b/mongo_wrapper_meta.c new file mode 100644 index 0000000..33ec2c5 --- /dev/null +++ b/mongo_wrapper_meta.c @@ -0,0 +1,407 @@ +/*------------------------------------------------------------------------- + * + * mongo_wrapper.c + * + * Wrapper functions for MongoDB's new Meta Driver. + * + * Portions Copyright (c) 2004-2014, EnterpriseDB Corporation. + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" +#include + +#include "mongo_wrapper.h" + +/* + * Connect to MongoDB server using Host/ip and Port number. + */ +MONGO_CONN* +MongoConnect(const char* host, const unsigned short port) +{ + MONGO_CONN *client = NULL; + char* uri = NULL; + + uri = bson_strdup_printf ("mongodb://%s:%hu/", host, port); + + client = mongoc_client_new(uri); + + bson_free(uri); + + if (client == NULL) + ereport(ERROR, (errmsg("could not connect to %s:%d", host, port), + errhint("Mongo driver connection error"))); + return client; +} + +/* + * Disconnect from MongoDB server. + */ +void +MongoDisconnect(MONGO_CONN* conn) +{ + if (conn) + mongoc_client_destroy(conn); +} + + +/* + * Insert a document 'b' into MongoDB. + */ +bool +MongoInsert(MONGO_CONN* conn, char *database, char* collection, BSON* b) +{ + mongoc_collection_t *c = NULL; + bson_error_t error; + bool r = false; + + c = mongoc_client_get_collection(conn, database, collection); + + r = mongoc_collection_insert(c, MONGOC_INSERT_NONE, b, NULL, &error); + mongoc_collection_destroy(c); + if (!r) + ereport(ERROR, (errmsg("failed to insert row"), + errhint("Mongo error: \"%s\"", error.message))); + return true; +} + + +/* + * Update a document 'b' into MongoDB. + */ +bool +MongoUpdate(MONGO_CONN* conn, char* database, char *collection, BSON* b, BSON* op) +{ + mongoc_collection_t *c = NULL; + bson_error_t error; + bool r = false; + + c = mongoc_client_get_collection (conn, database, collection); + + r = mongoc_collection_update(c, MONGOC_UPDATE_NONE, b, op, NULL, &error); + mongoc_collection_destroy(c); + if (!r) + ereport(ERROR, (errmsg("failed to update row"), + errhint("Mongo error: \"%s\"", error.message))); + return true; +} + + +/* + * Delete MongoDB's document. + */ +bool +MongoDelete(MONGO_CONN* conn, char* database, char *collection, BSON* b) +{ + mongoc_collection_t *c = NULL; + bson_error_t error; + bool r = false; + + c = mongoc_client_get_collection (conn, database, collection); + + r = mongoc_collection_delete(c, MONGOC_DELETE_SINGLE_REMOVE, b, NULL, &error); + mongoc_collection_destroy(c); + if (!r) + ereport(ERROR, (errmsg("failed to delete row"), + errhint("Mongo error: \"%s\"", error.message))); + return true; +} + +/* + * Performs a query against the configured MongoDB server and return + * cursor which can be destroyed by calling mongoc_cursor_current. + */ +MONGO_CURSOR* +MongoCursorCreate(MONGO_CONN* conn, char* database, char *collection, BSON* q) +{ + mongoc_collection_t *c = NULL; + MONGO_CURSOR *cur = NULL; + bson_error_t error; + + c = mongoc_client_get_collection (conn, database, collection); + cur = mongoc_collection_find(c, MONGOC_QUERY_NONE, 0, 0, 0, q, NULL, NULL); + mongoc_cursor_error(cur, &error); + if (!cur) + ereport(ERROR, (errmsg("failed to create cursor"), + errhint("Mongo error: \"%s\"", error.message))); + + mongoc_collection_destroy(c); + return cur; +} + + +/* + * Destroy cursor created by calling MongoCursorCreate function. + */ +void +MongoCursorDestroy(MONGO_CURSOR* c) +{ + mongoc_cursor_destroy(c); +} + + +/* + * Get the current document from cursor. + */ +const BSON* +MongoCursorBson(MONGO_CURSOR* c) +{ + return mongoc_cursor_current(c); +} + +/* + * Get the next document from the cursor. + */ +bool +MongoCursorNext(MONGO_CURSOR* c, BSON *b) +{ + return mongoc_cursor_next(c, (const BSON**) &b); +} + + +/* + * Allocates a new bson_t structure, and also initialize the bson + * object. After that point objects can be appended to that bson + * object and can be iterated. A newly allocated bson_t that should + * be freed with bson_destroy(). + */ +BSON* +BsonCreate(void) +{ + BSON *b = NULL; + b = bson_new(); + bson_init(b); + return b; +} + +/* + * Destroy Bson objected created by BsonCreate function. + */ +void +BsonDestroy(BSON *b) +{ + bson_destroy(b); +} + + +/* + * Initialize the bson Iterator. + */ +bool +BsonIterInit(BSON_ITERATOR *it, BSON *b) +{ + return bson_iter_init(it, b); +} + + +bool +BsonIterSubObject(BSON_ITERATOR *it, BSON *b) +{ + /* TODO: Need to see the Meta Driver equalient for "bson_iterator_subobject" */ + return true; +} + +int32_t +BsonIterInt32(BSON_ITERATOR *it) +{ + return bson_iter_int32(it); +} + + +int64_t +BsonIterInt64(BSON_ITERATOR *it) +{ + return bson_iter_int64(it); +} + + +double +BsonIterDouble(BSON_ITERATOR *it) +{ + return bson_iter_double(it); +} + + +bool +BsonIterBool(BSON_ITERATOR *it) +{ + return bson_iter_bool(it); +} + + +const char* +BsonIterString(BSON_ITERATOR *it) +{ + uint32_t len = 0; + return bson_iter_utf8(it, &len); +} + + +const bson_oid_t * +BsonIterOid(BSON_ITERATOR *it) +{ + return bson_iter_oid(it); +} + + +time_t +BsonIterDate(BSON_ITERATOR *it) +{ + return bson_iter_date_time(it); +} + + +const char* +BsonIterKey(BSON_ITERATOR *it) +{ + return bson_iter_key(it); +} + +int +BsonIterType(BSON_ITERATOR *it) +{ + return bson_iter_type(it); +} + +int +BsonIterNext(BSON_ITERATOR *it) +{ + return bson_iter_next(it); +} + + +bool +BsonIterSubIter(BSON_ITERATOR *it, BSON_ITERATOR* sub) +{ + return bson_iter_recurse(it, sub); +} + + +void +BsonOidFromString(bson_oid_t *o, char* str) +{ + bson_oid_init_from_string(o, str); +} + + +bool +BsonAppendOid(BSON *b, const char* key, bson_oid_t *v) +{ + return bson_append_oid(b, key, strlen(key), v); +} + +bool +BsonAppendBool(BSON *b, const char* key, bool v) +{ + return bson_append_bool(b, key, -1, v); +} + +bool +BsonAppendStartObject(BSON* b, char *key, BSON* r) +{ + return bson_append_document_begin(b, key, strlen(key), r); +} + + +bool +BsonAppendFinishObject(BSON* b, BSON* r) +{ + return bson_append_document_end(b, r); +} + + +bool +BsonAppendNull(BSON *b, const char* key) +{ + return bson_append_null(b, key, strlen(key)); +} + + +bool +BsonAppendInt32(BSON *b, const char* key, int v) +{ + return bson_append_int32(b, key, strlen(key), v); +} + + +bool +BsonAppendInt64(BSON *b, const char* key, int64_t v) +{ + return bson_append_int64(b, key, strlen(key), v); +} + +bool +BsonAppendDouble(BSON *b, const char* key, double v) +{ + return bson_append_double(b, key, strlen(key), v); +} + +bool +BsonAppendUTF8(BSON *b, const char* key, char *v) +{ + + return bson_append_utf8(b, key, strlen(key), v, strlen(v)); +} + + +bool +BsonAppendDate(BSON *b, const char* key, time_t v) +{ + return bson_append_date_time(b, key, strlen(key), v); +} + + +bool +BsonAppendBson(BSON* b, char *key, BSON* c) +{ + return bson_append_document(b, key, strlen(key), c); +} + + +bool +BsonFinish(BSON* b) +{ + /* + * There is no need for bson_finish in Meta Driver. + * We are doing nothing, just because of compatiblity with legacy + * driver. + */ + return true; +} + +/* + * Count the number of documents. + */ +int +MongoAggregateCount(MONGO_CONN* conn, const char* database, const char* collection, const BSON* b) +{ + BSON *cmd = NULL; + BSON *out; + double count = -1; + bool r = false; + bson_error_t error; + mongoc_collection_t *c = NULL; + + c = mongoc_client_get_collection (conn, database, collection); + + cmd = BsonCreate(); + out = BsonCreate(); + BsonAppendUTF8(cmd, "count", (char*)collection); + if (b) /* not empty */ + BsonAppendBson(cmd, "query", (BSON*)b); + + BsonFinish(cmd); + r = mongoc_collection_command_simple(c, cmd, NULL, out, &error); + if (r) + { + bson_iter_t it; + if (bson_iter_init_find(&it, out, "n")) + count = BsonIterDouble(&it); + } + mongoc_collection_destroy(c); + + BsonDestroy(out); + BsonDestroy(cmd); + return (int)count; +} diff --git a/option.c b/option.c new file mode 100644 index 0000000..b7730ab --- /dev/null +++ b/option.c @@ -0,0 +1,232 @@ +/*------------------------------------------------------------------------- + * + * option.c + * + * Function definitions for MongoDB foreign data wrapper. These functions access + * data stored in MongoDB through the official C driver. + * + * Copyright (c) 2012-2014 Citus Data, Inc. + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" +#include "mongo_wrapper.h" +#include "mongo_fdw.h" + +#include "access/reloptions.h" +#include "catalog/pg_type.h" +#include "commands/defrem.h" +#include "commands/explain.h" +#include "commands/vacuum.h" +#include "foreign/fdwapi.h" +#include "foreign/foreign.h" +#include "nodes/makefuncs.h" +#include "optimizer/cost.h" +#include "optimizer/pathnode.h" +#include "optimizer/plancat.h" +#include "optimizer/planmain.h" +#include "optimizer/restrictinfo.h" +#include "utils/array.h" +#include "utils/builtins.h" +#include "utils/date.h" +#include "utils/hsearch.h" +#include "utils/lsyscache.h" +#include "utils/rel.h" +#include "utils/memutils.h" + + +static char * MongoGetOptionValue(Oid foreignTableId, const char *optionName); + +/* + * Validate the generic options given to a FOREIGN DATA WRAPPER, SERVER, + * USER MAPPING or FOREIGN TABLE that uses postgres_fdw. + * + * Raise an ERROR if the option or its value is considered invalid. + */ +extern Datum mongo_fdw_validator(PG_FUNCTION_ARGS); + +PG_FUNCTION_INFO_V1(mongo_fdw_validator); + +/* + * mongo_fdw_validator validates options given to one of the following commands: + * foreign data wrapper, server, user mapping, or foreign table. This function + * errors out if the given option name or its value is considered invalid. + */ +Datum +mongo_fdw_validator(PG_FUNCTION_ARGS) +{ + Datum optionArray = PG_GETARG_DATUM(0); + Oid optionContextId = PG_GETARG_OID(1); + List *optionList = untransformRelOptions(optionArray); + ListCell *optionCell = NULL; + + foreach(optionCell, optionList) + { + DefElem *optionDef = (DefElem *) lfirst(optionCell); + char *optionName = optionDef->defname; + bool optionValid = false; + + int32 optionIndex = 0; + for (optionIndex = 0; optionIndex < ValidOptionCount; optionIndex++) + { + const MongoValidOption *validOption = &(ValidOptionArray[optionIndex]); + + if ((optionContextId == validOption->optionContextId) && + (strncmp(optionName, validOption->optionName, NAMEDATALEN) == 0)) + { + optionValid = true; + break; + } + } + + /* if invalid option, display an informative error message */ + if (!optionValid) + { + StringInfo optionNamesString = OptionNamesString(optionContextId); + + ereport(ERROR, (errcode(ERRCODE_FDW_INVALID_OPTION_NAME), + errmsg("invalid option \"%s\"", optionName), + errhint("Valid options in this context are: %s", + optionNamesString->data))); + } + + /* if port option is given, error out if its value isn't an integer */ + if (strncmp(optionName, OPTION_NAME_PORT, NAMEDATALEN) == 0) + { + char *optionValue = defGetString(optionDef); + int32 portNumber = pg_atoi(optionValue, sizeof(int32), 0); + (void) portNumber; + } + } + PG_RETURN_VOID(); +} + +/* + * OptionNamesString finds all options that are valid for the current context, + * and concatenates these option names in a comma separated string. + */ +StringInfo +OptionNamesString(Oid currentContextId) +{ + StringInfo optionNamesString = makeStringInfo(); + bool firstOptionPrinted = false; + + int32 optionIndex = 0; + for (optionIndex = 0; optionIndex < ValidOptionCount; optionIndex++) + { + const MongoValidOption *validOption = &(ValidOptionArray[optionIndex]); + + /* if option belongs to current context, append option name */ + if (currentContextId == validOption->optionContextId) + { + if (firstOptionPrinted) + appendStringInfoString(optionNamesString, ", "); + + appendStringInfoString(optionNamesString, validOption->optionName); + firstOptionPrinted = true; + } + } + return optionNamesString; +} + + +/* + * MongoGetOptions returns the option values to be used when connecting to and + * querying MongoDB. To resolve these values, the function checks the foreign + * table's options, and if not present, falls back to default values. + */ +MongoFdwOptions * +MongoGetOptions(Oid foreignTableId) +{ + MongoFdwOptions *mongoFdwOptions = NULL; + char *addressName = NULL; + char *portName = NULL; + int32 portNumber = 0; + char *databaseName = NULL; + char *collectionName = NULL; + + addressName = MongoGetOptionValue(foreignTableId, OPTION_NAME_ADDRESS); + if (addressName == NULL) + { + addressName = pstrdup(DEFAULT_IP_ADDRESS); + } + + portName = MongoGetOptionValue(foreignTableId, OPTION_NAME_PORT); + if (portName == NULL) + { + portNumber = DEFAULT_PORT_NUMBER; + } + else + { + portNumber = pg_atoi(portName, sizeof(int32), 0); + } + + databaseName = MongoGetOptionValue(foreignTableId, OPTION_NAME_DATABASE); + if (databaseName == NULL) + { + databaseName = pstrdup(DEFAULT_DATABASE_NAME); + } + + collectionName = MongoGetOptionValue(foreignTableId, OPTION_NAME_COLLECTION); + if (collectionName == NULL) + { + collectionName = get_rel_name(foreignTableId); + } + + mongoFdwOptions = (MongoFdwOptions *) palloc0(sizeof(MongoFdwOptions)); + mongoFdwOptions->addressName = addressName; + mongoFdwOptions->portNumber = portNumber; + mongoFdwOptions->databaseName = databaseName; + mongoFdwOptions->collectionName = collectionName; + + return mongoFdwOptions; +} + + +void +MongoFreeOptions(MongoFdwOptions *mongoFdwOptions) +{ + if (mongoFdwOptions) + { + pfree(mongoFdwOptions->addressName); + pfree(mongoFdwOptions->databaseName); + pfree(mongoFdwOptions); + } +} + +/* + * MongoGetOptionValue walks over foreign table and foreign server options, and + * looks for the option with the given name. If found, the function returns the + * option's value. + */ +static char * +MongoGetOptionValue(Oid foreignTableId, const char *optionName) +{ + ForeignTable *foreignTable = NULL; + ForeignServer *foreignServer = NULL; + List *optionList = NIL; + ListCell *optionCell = NULL; + char *optionValue = NULL; + + foreignTable = GetForeignTable(foreignTableId); + foreignServer = GetForeignServer(foreignTable->serverid); + + optionList = list_concat(optionList, foreignTable->options); + optionList = list_concat(optionList, foreignServer->options); + + foreach(optionCell, optionList) + { + DefElem *optionDef = (DefElem *) lfirst(optionCell); + char *optionDefName = optionDef->defname; + + if (strncmp(optionDefName, optionName, NAMEDATALEN) == 0) + { + optionValue = defGetString(optionDef); + break; + } + } + + return optionValue; +} + From aacb0c80dd89c32357d9d89e7307d04a7b4002b3 Mon Sep 17 00:00:00 2001 From: Ibrar Ahmed Date: Mon, 2 Jun 2014 18:49:15 +0500 Subject: [PATCH 06/43] Restrict first column name of MongoDB's foreign table to "_id". --- mongo_fdw.c | 45 +++++++++++---------------------------------- 1 file changed, 11 insertions(+), 34 deletions(-) diff --git a/mongo_fdw.c b/mongo_fdw.c index 2bc67b1..899130a 100644 --- a/mongo_fdw.c +++ b/mongo_fdw.c @@ -136,19 +136,6 @@ static void mongo_fdw_exit(int code, Datum arg); extern PGDLLEXPORT void _PG_init(void); -/* - * Generate new 24 character's rowid (MongoDB's "_id") - * from a string using pad character 'A'. - */ -#define UNIQUE_OID(x, z) do \ -{ \ - char y[25]; \ - memset(y, 'A', sizeof(y));\ - y[24] = 0; \ - strcpy(y, x); \ - BsonOidFromString(&z, y);\ -} while(0); - /* declarations for dynamic loading */ PG_MODULE_MAGIC; @@ -767,33 +754,23 @@ MongoExecForeignInsert(EState *estate, { int attnum = lfirst_int(lc); value = slot_getattr(slot, attnum, &isnull); + + /* first column of MongoDB's foreign table must be _id */ + if (strcmp(slot->tts_tupleDescriptor->attrs[0]->attname.data, "_id") != 0) + elog(ERROR, "first colum of MongoDB's foreign table must be \"_id\""); + if (attnum == 1) { /* - * We also disallow null values to the first column which - * happens to be the row identifier in MongoDb (_id). + * Ignore the value of first column which is row identifier in MongoDb (_id) + * and let MongoDB to insert the unique value for that column. */ - - if (isnull) - elog(ERROR, "null value for first column (row identifier column) is not supported"); - - getTypeOutputInfo(typoid, &outputFunctionId, &typeVarLength); - outputString = OidOutputFunctionCall(outputFunctionId, value); - - /* MongoDB support 24 character's _id (row identifier) */ - if (strlen(outputString) > 24) - elog(ERROR, "first column (row identifier column) should be max 24 characters"); - - UNIQUE_OID(outputString, bsonObjectId); - - /* Append rowid field which is "_id" in MongoDB */ - if(!BsonAppendOid(b, "_id", &bsonObjectId)) - ereport(ERROR, - (errcode(ERRCODE_FDW_ERROR), - errmsg("insert failed, invalid value"))); } - AppenMongoValue(b, slot->tts_tupleDescriptor->attrs[attnum - 1]->attname.data, value, + else + { + AppenMongoValue(b, slot->tts_tupleDescriptor->attrs[attnum - 1]->attname.data, value, isnull, slot->tts_tupleDescriptor->attrs[attnum -1]->atttypid); + } } } BsonFinish(b); From b67102ce516e8cb8dfda36a6bda744885b0afa9a Mon Sep 17 00:00:00 2001 From: Ibrar Ahmed Date: Mon, 2 Jun 2014 18:53:44 +0500 Subject: [PATCH 07/43] README Changes --- CONTRIBUTING.md | 4 +- README.md | 137 +++++++++++++++++++++++++++++++++++------------- 2 files changed, 104 insertions(+), 37 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index df9d264..bd5c88b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -11,7 +11,7 @@ Using Issues `mongo_fdw`'s maintainers prefer that bug reports, feature requests, and pull requests are submitted as [GitHub Issues][1]. If you think you require personal -assistance, please **do not** open an issue: email `engage` `@` `citusdata.com` +assistance, please **do not** open an issue: email `ibrar.ahmed` `@` `enterprisedb.com` instead. @@ -81,5 +81,5 @@ permitted by law. Finally, you confirm that you own said copyright, have the legal authority to grant said license, and in doing so are not violating any grant of rights you have made to third parties, including your employer. -[1]: https://github.com/citusdata/mongo_fdw/issues +[1]: https://github.com/EnterpriseDB/mongo_fdw/issues [2]: LICENSE diff --git a/README.md b/README.md index 4b024d5..9065d78 100644 --- a/README.md +++ b/README.md @@ -1,29 +1,41 @@ -MongoDB FDW for PostgreSQL -========================== +MongoDB Foreign Date Wrapper for PostgreSQL +=========================================== -This PostgreSQL extension implements a Foreign Data Wrapper (FDW) for -[MongoDB][1]. For an example demonstrating this wrapper's use, see [our blog -post][2]. Please also note that this version of `mongo_fdw` only works with -PostgreSQL 9.2 or 9.3. +This [MongoDB][1] extension implements the PostgreSQL's Foreign Data Wrapper for. + +Please also note that this version of `mongo_fdw` only works with +PostgreSQL Version 9.3 and greater. Installation ------------ -The MongoDB FDW includes the official MongoDB C Driver version 0.6. When you -type `make`, the C driver's source code also gets automatically compiled and -linked. +This [MongoDB][1] FDW is compatible with two [MongoDB][1]'s ""C"" drivers, [legacy driver][6] and [MongoDB][1]'s [Meta Driver][7]. This is compile time decision which driver you want to use with this FDW. + +The [MongoDB][1] FDW includes the [legacy MongoDB C Driver][6] version 0.6. When you +type `make -f Makefile.legacy`, the C driver's source code also gets automatically compiled and linked. The other option is to compile using [MongoDB's Meta Driver][7]. You need to download the Meta C driver from MongoDB site. To build on POSIX-compliant systems (like Linux and OS X), you need to ensure the `pg_config` executable is in your path when you run `make`. This executable is typically in your PostgreSQL installation's `bin` directory. For example: +Compile using MonoDB's legacy C driver. + +```sh +PATH=/usr/local/pgsql/bin/:$PATH make -f Makefile.legacy +sudo PATH=/usr/local/pgsql/bin/:$PATH make install +``` + +Compile using MonoDB's Meta C driver. + +Add `#deine META_DRIVER` in `config.h` file, then + ```sh -PATH=/usr/local/pgsql/bin/:$PATH make +PATH=/usr/local/pgsql/bin/:$PATH make -f Makefile.meta sudo PATH=/usr/local/pgsql/bin/:$PATH make install ``` -Note that we have tested the `mongo_fdw` extension only on Fedora and Ubuntu +Note that we have tested the `mongo_fdw` extension only on MacOS X, Fedora and Ubuntu systems. If you run into issues on other systems, please [let us know][3]. @@ -52,38 +64,90 @@ default value mentioned above. estimating costs for the query execution plan. To see selected execution plans for a query, just run `EXPLAIN`. -We also currently use the internal PostgreSQL `NAME` type to represent the BSON -object identifier type (the `_id` field). - ```sql +Examples with MongoDB equelent statments. + -- load extension first time after install -CREATE EXTENSION mongo_fdw; +`CREATE EXTENSION mongo_fdw;` -- create server object -CREATE SERVER mongo_server FOREIGN DATA WRAPPER mongo_fdw -OPTIONS (address '127.0.0.1', port '27017'); +`CREATE SERVER mongo_server + FOREIGN DATA WRAPPER mongo_fdw + OPTIONS (address '127.0.0.1', port '27017');` -- create foreign table -CREATE FOREIGN TABLE customer_reviews +`CREATE FOREIGN TABLE warehouse( + _id NAME, + warehouse_id int, + warehouse_name text, + warehouse_created timestamptz) +SERVER mongo_server + OPTIONS (database 'db', collection 'warehouse');` + + +-- select from table +`SELECT * FROM warehouse WHERE warehouse_id = 1;` + +_id | warehouse_id | warehouse_name | warehouse_created +------------------------+----------------+--------------------------- +53720b1904864dc1f5a571a0| 1 | UPS | 12-DEC-14 12:12:10 +05:00 + + +`db.warehouse.find({"warehouse_id" : 1}).pretty()` +{ + "_id" : ObjectId("53720b1904864dc1f5a571a0"), + "warehouse_id" : 1, + "warehouse_name" : "UPS", + "warehouse_created" : ISODate("2014-12-12T07:12:10Z") +} + + +-- insert row in table +`INSERT INTO warehouse values (0, 1, 'UPS', to_date('2014-12-12T07:12:10Z'));` + +`db.warehouse.insert` +( + { + "warehouse_id" : NumberInt(1), + "warehouse_name" : "UPS", + "warehouse_created" : ISODate("2014-12-12T07:12:10Z") + } +); + +-- delete row from table +`DELETE FROM warehouse where warehouse_id = 3;` + +> db.warehouse.remove({"warehouse_id" : 2}) + + +-- update a row of table +`UPDATE warehouse set warehouse_name = 'UPS_NEW' where warehouse_id = 1;` + +db.warehouse.update ( - _id NAME, - customer_id TEXT, - review_date TIMESTAMP, - review_rating INTEGER, - product_id CHAR(10), - product_title TEXT, - product_group TEXT, - product_category TEXT, - similar_product_ids CHAR(10)[] + { + "warehouse_id" : 1 + }, + { + "warehouse_id" : 1, + "warehouse_name" : "UPS_NEW" + } ) -SERVER mongo_server -OPTIONS (database 'test', collection 'customer_reviews'); --- collect data distribution statistics -ANALYZE customer_reviews; +-- explain a table +`EXPLAIN SELECT * FROM warehouse WHERE warehouse_id = 1; + QUERY PLAN + ----------------------------------------------------------------- + Foreign Scan on warehouse (cost=0.00..0.00 rows=1000 width=44) + Filter: (warehouse_id = 1) + Foreign Namespace: db.warehouse + Planning time: 0.671 ms +(4 rows)` + +-- collect data distribution statistics` +ANALYZE warehouse; ``` - Limitations ----------- @@ -103,7 +167,7 @@ Contributing Have a fix for a bug or an idea for a great new feature? Great! Check out the contribution guidelines [here][4]. For all other types of questions or comments -about the wrapper please contact us at `engage` `@` `citusdata.com`. +about the wrapper please contact us at `ibrar.ahmed` `@` `enterprisedb.com`. Support @@ -118,7 +182,8 @@ Reported bugs will be addressed by apparent severity. License ------- -Copyright © 2012–2014 Citus Data, Inc. +Portions Copyright © 2004-2014, EnterpriseDB Corporation. +Portions Copyright © 2012–2014 Citus Data, Inc. This program is free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License as published by the Free @@ -129,6 +194,8 @@ See the [`LICENSE`][5] file for full details. [1]: http://www.mongodb.com [2]: http://www.citusdata.com/blog/51-run-sql-on-mongodb -[3]: https://github.com/citusdata/mongo_fdw/issues/new +[3]: https://github.com/ibrarahmad/mongo_fdw/issues/new [4]: CONTRIBUTING.md [5]: LICENSE +[6]: https://github.com/mongodb/mongo-c-driver-legacy +[7]: https://github.com/mongodb/mongo-meta-driver From a501e6d8492c23b07385bd362b10669b435080a7 Mon Sep 17 00:00:00 2001 From: Ibrar Ahmed Date: Mon, 2 Jun 2014 18:59:56 +0500 Subject: [PATCH 08/43] README file typos and cosmetic changes. --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 9065d78..45dbd19 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -MongoDB Foreign Date Wrapper for PostgreSQL +MongoDB Foreign Data Wrapper for PostgreSQL =========================================== This [MongoDB][1] extension implements the PostgreSQL's Foreign Data Wrapper for. @@ -10,7 +10,7 @@ PostgreSQL Version 9.3 and greater. Installation ------------ -This [MongoDB][1] FDW is compatible with two [MongoDB][1]'s ""C"" drivers, [legacy driver][6] and [MongoDB][1]'s [Meta Driver][7]. This is compile time decision which driver you want to use with this FDW. +This [MongoDB][1] FDW is compatible with two [MongoDB][1]'s 'C' drivers, [MongoDB's legacy driver][6] and [MongoDB's Meta Driver][7]. This is compile time decision which driver you want to use with this FDW. The [MongoDB][1] FDW includes the [legacy MongoDB C Driver][6] version 0.6. When you type `make -f Makefile.legacy`, the C driver's source code also gets automatically compiled and linked. The other option is to compile using [MongoDB's Meta Driver][7]. You need to download the Meta C driver from MongoDB site. @@ -183,6 +183,7 @@ License ------- Portions Copyright © 2004-2014, EnterpriseDB Corporation. + Portions Copyright © 2012–2014 Citus Data, Inc. This program is free software: you can redistribute it and/or modify it under From 4ead1e4e1941f328602ffd4e32c338034a02a497 Mon Sep 17 00:00:00 2001 From: Ibrar Ahmed Date: Mon, 2 Jun 2014 19:03:13 +0500 Subject: [PATCH 09/43] Compiler warning of unused variable fixed. --- mongo_fdw.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/mongo_fdw.c b/mongo_fdw.c index 899130a..29ef87a 100644 --- a/mongo_fdw.c +++ b/mongo_fdw.c @@ -723,10 +723,6 @@ MongoExecForeignInsert(EState *estate, MONGO_CONN *mongoConnection = NULL; Oid foreignTableId = InvalidOid; BSON *b = NULL; - bson_oid_t bsonObjectId; - char *outputString; - Oid outputFunctionId = InvalidOid; - bool typeVarLength = false; Oid typoid = InvalidOid; Datum value; bool isnull = false; From 2e5a4914c06a47426dd9498b80253b49d4e540e4 Mon Sep 17 00:00:00 2001 From: Ibrar Ahmed Date: Mon, 2 Jun 2014 19:05:40 +0500 Subject: [PATCH 10/43] Default Makefile to compile FDW with MongoDB's legacy Driver. --- Makefile | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 Makefile diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..88e6ce0 --- /dev/null +++ b/Makefile @@ -0,0 +1,42 @@ +# mongo_fdw/Makefile +# +# Copyright (c) 2012-2014 Citus Data, Inc. +# + +MODULE_big = mongo_fdw + +# +# We assume we are running on a POSIX compliant system (Linux, OSX). If you are +# on another platform, change env_posix.os in MONGO_OBJS with the appropriate +# environment object file. +# + +MONGO_DRIVER = mongo-c-driver-v0.6 +MONGO_PATH = $(MONGO_DRIVER)/src +MONGO_OBJS = $(MONGO_PATH)/bson.os $(MONGO_PATH)/encoding.os $(MONGO_PATH)/md5.os $(MONGO_PATH)/mongo.os $(MONGO_PATH)/numbers.os $(MONGO_PATH)/env_posix.os + +PG_CPPFLAGS = --std=c99 -I$(MONGO_PATH) +OBJS = connection.o option.o mongo_wrapper.o mongo_fdw.o mongo_query.o $(MONGO_OBJS) + +EXTENSION = mongo_fdw +DATA = mongo_fdw--1.0.sql + +$(MONGO_DRIVER)/%.os: + $(MAKE) -C $(MONGO_DRIVER) $*.os + +# +# Users need to specify their Postgres installation path through pg_config. For +# example: /usr/local/pgsql/bin/pg_config or /usr/lib/postgresql/9.1/bin/pg_config +# + +PG_CONFIG = pg_config +PGXS := $(shell $(PG_CONFIG) --pgxs) +include $(PGXS) + +ifndef MAJORVERSION + MAJORVERSION := $(basename $(VERSION)) +endif + +ifeq (,$(findstring $(MAJORVERSION), 9.3 9.4)) + $(error PostgreSQL 9.3 or 9.4 is required to compile this extension) +endif From e31a1936fd76feccb5030792e19c48df08ce76e3 Mon Sep 17 00:00:00 2001 From: Ibrar Ahmed Date: Mon, 2 Jun 2014 19:12:43 +0500 Subject: [PATCH 11/43] Licence information update --- Makefile | 4 +++- Makefile.legacy | 6 ++++-- Makefile.meta | 6 ++++-- connection.c | 5 ++++- mongo_fdw--1.0.sql | 3 ++- mongo_fdw.c | 4 +++- mongo_fdw.control | 3 ++- mongo_fdw.h | 5 ++++- mongo_query.c | 5 ++++- mongo_query.h | 5 ++++- mongo_wrapper.c | 5 ++++- mongo_wrapper_meta.c | 7 +++++-- option.c | 5 ++++- 13 files changed, 47 insertions(+), 16 deletions(-) diff --git a/Makefile b/Makefile index 88e6ce0..11cdf69 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,8 @@ # mongo_fdw/Makefile # -# Copyright (c) 2012-2014 Citus Data, Inc. +# Portions Copyright © 2004-2014, EnterpriseDB Corporation. +# +# Portions Copyright © 2012–2014 Citus Data, Inc. # MODULE_big = mongo_fdw diff --git a/Makefile.legacy b/Makefile.legacy index 88e6ce0..a9f0ed3 100644 --- a/Makefile.legacy +++ b/Makefile.legacy @@ -1,6 +1,8 @@ -# mongo_fdw/Makefile +# mongo_fdw/Makefile.legacy # -# Copyright (c) 2012-2014 Citus Data, Inc. +# Portions Copyright © 2004-2014, EnterpriseDB Corporation. +# +# Portions Copyright © 2012–2014 Citus Data, Inc. # MODULE_big = mongo_fdw diff --git a/Makefile.meta b/Makefile.meta index 10a12e4..569d2f0 100644 --- a/Makefile.meta +++ b/Makefile.meta @@ -1,6 +1,8 @@ -# mongo_fdw/Makefile +# mongo_fdw/Makefile.meta # -# Copyright (c) 2012-2014 Citus Data, Inc. +# Portions Copyright © 2004-2014, EnterpriseDB Corporation. +# +# Portions Copyright © 2012–2014 Citus Data, Inc. # MODULE_big = mongo_fdw diff --git a/connection.c b/connection.c index 9b8660e..aefe924 100644 --- a/connection.c +++ b/connection.c @@ -3,7 +3,10 @@ * connection.c * Connection management functions for mongo_fdw * - * Portions Copyright (c) 2004-2014, EnterpriseDB Corporation. + * Portions Copyright © 2004-2014, EnterpriseDB Corporation. + * + * Portions Copyright © 2012–2014 Citus Data, Inc. + * * *------------------------------------------------------------------------- */ diff --git a/mongo_fdw--1.0.sql b/mongo_fdw--1.0.sql index af83adf..b495ced 100644 --- a/mongo_fdw--1.0.sql +++ b/mongo_fdw--1.0.sql @@ -1,6 +1,7 @@ /* mongo_fdw/mongo_fdw--1.0.sql */ --- Copyright (c) 2012-2014 Citus Data, Inc. +-- Portions Copyright © 2004-2014, EnterpriseDB Corporation. +-- Portions Copyright © 2012–2014 Citus Data, Inc. -- complain if script is sourced in psql, rather than via CREATE EXTENSION \echo Use "CREATE EXTENSION mongo_fdw" to load this file. \quit diff --git a/mongo_fdw.c b/mongo_fdw.c index 29ef87a..65d5d6c 100644 --- a/mongo_fdw.c +++ b/mongo_fdw.c @@ -5,7 +5,9 @@ * Function definitions for MongoDB foreign data wrapper. These functions access * data stored in MongoDB through the official C driver. * - * Copyright (c) 2012-2014 Citus Data, Inc. + * Portions Copyright © 2004-2014, EnterpriseDB Corporation. + * + * Portions Copyright © 2012–2014 Citus Data, Inc. * *------------------------------------------------------------------------- */ diff --git a/mongo_fdw.control b/mongo_fdw.control index c0fbe86..a457792 100644 --- a/mongo_fdw.control +++ b/mongo_fdw.control @@ -1,6 +1,7 @@ # mongo_fdw extension # -# Copyright (c) 2012-2014 Citus Data, Inc. +# Portions Copyright © 2004-2014, EnterpriseDB Corporation. +# Portions Copyright © 2012–2014 Citus Data, Inc. # comment = 'foreign data wrapper for MongoDB access' default_version = '1.0' diff --git a/mongo_fdw.h b/mongo_fdw.h index f72af94..ba40cde 100644 --- a/mongo_fdw.h +++ b/mongo_fdw.h @@ -4,7 +4,10 @@ * * Type and function declarations for MongoDB foreign data wrapper. * - * Copyright (c) 2012-2014 Citus Data, Inc. + * Portions Copyright © 2004-2014, EnterpriseDB Corporation. + * + * Portions Copyright © 2012–2014 Citus Data, Inc. + * * *------------------------------------------------------------------------- */ diff --git a/mongo_query.c b/mongo_query.c index a559a0b..fae7543 100644 --- a/mongo_query.c +++ b/mongo_query.c @@ -6,7 +6,10 @@ * that queries are sent through the official MongoDB C driver, and apply query * optimizations to reduce the amount of data fetched from the driver. * - * Copyright (c) 2012-2014 Citus Data, Inc. + * Portions Copyright © 2004-2014, EnterpriseDB Corporation. + * + * Portions Copyright © 2012–2014 Citus Data, Inc. + * * *------------------------------------------------------------------------- */ diff --git a/mongo_query.h b/mongo_query.h index 607bb2d..66f0c9f 100644 --- a/mongo_query.h +++ b/mongo_query.h @@ -4,7 +4,10 @@ * * Type and function declarations for constructing queries to send to MongoDB. * - * Copyright (c) 2012-2014 Citus Data, Inc. + * Portions Copyright © 2004-2014, EnterpriseDB Corporation. + * + * Portions Copyright © 2012–2014 Citus Data, Inc. + * * *------------------------------------------------------------------------- */ diff --git a/mongo_wrapper.c b/mongo_wrapper.c index 9c77c41..e4c3604 100644 --- a/mongo_wrapper.c +++ b/mongo_wrapper.c @@ -4,7 +4,10 @@ * * Wrapper functions for MongoDB's old legacy Driver. * - * Portions Copyright (c) 2004-2014, EnterpriseDB Corporation. + * Portions Copyright © 2004-2014, EnterpriseDB Corporation. + * + * Portions Copyright © 2012–2014 Citus Data, Inc. + * * *------------------------------------------------------------------------- */ diff --git a/mongo_wrapper_meta.c b/mongo_wrapper_meta.c index 33ec2c5..70074c9 100644 --- a/mongo_wrapper_meta.c +++ b/mongo_wrapper_meta.c @@ -1,10 +1,13 @@ /*------------------------------------------------------------------------- * - * mongo_wrapper.c + * mongo_wrapper_meta.c * * Wrapper functions for MongoDB's new Meta Driver. * - * Portions Copyright (c) 2004-2014, EnterpriseDB Corporation. + * Portions Copyright © 2004-2014, EnterpriseDB Corporation. + * + * Portions Copyright © 2012–2014 Citus Data, Inc. + * * *------------------------------------------------------------------------- */ diff --git a/option.c b/option.c index b7730ab..5504373 100644 --- a/option.c +++ b/option.c @@ -5,7 +5,10 @@ * Function definitions for MongoDB foreign data wrapper. These functions access * data stored in MongoDB through the official C driver. * - * Copyright (c) 2012-2014 Citus Data, Inc. + * Portions Copyright © 2004-2014, EnterpriseDB Corporation. + * + * Portions Copyright © 2012–2014 Citus Data, Inc. + * * *------------------------------------------------------------------------- */ From 0fe3122d48cd890cd6b0eb3c276d5bda62aafb8f Mon Sep 17 00:00:00 2001 From: Ibrar Ahmed Date: Mon, 2 Jun 2014 19:27:32 +0500 Subject: [PATCH 12/43] README file changes. --- README.md | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 45dbd19..a614783 100644 --- a/README.md +++ b/README.md @@ -10,14 +10,14 @@ PostgreSQL Version 9.3 and greater. Installation ------------ -This [MongoDB][1] FDW is compatible with two [MongoDB][1]'s 'C' drivers, [MongoDB's legacy driver][6] and [MongoDB's Meta Driver][7]. This is compile time decision which driver you want to use with this FDW. +This [MongoDB][1] FDW is compatible with two [MongoDB][1]'s 'C' drivers, [MongoDB's legacy driver][6] and [MongoDB's Meta Driver][7]. This is a compile time decision which MongoDB's driver this FDW will use. The [MongoDB][1] FDW includes the [legacy MongoDB C Driver][6] version 0.6. When you -type `make -f Makefile.legacy`, the C driver's source code also gets automatically compiled and linked. The other option is to compile using [MongoDB's Meta Driver][7]. You need to download the Meta C driver from MongoDB site. +Type `make -f Makefile. legacy`, the C driver's source code which gets automatically compiled and linked. The other option is to compile utilizing [MongoDB's Meta Driver][7]. You require to download the [Meta C driver][7] from MongoDB site. -To build on POSIX-compliant systems (like Linux and OS X), you need to ensure -the `pg_config` executable is in your path when you run `make`. This executable -is typically in your PostgreSQL installation's `bin` directory. For example: +To build on POSIX-compliant systems (like Linux and OS X), you require to ascertain +The `pg_config` executable is in your path when you run make`. This executable +Is typically in your PostgreSQL installation's bin directory. For example: Compile using MonoDB's legacy C driver. @@ -65,7 +65,7 @@ estimating costs for the query execution plan. To see selected execution plans for a query, just run `EXPLAIN`. ```sql -Examples with MongoDB equelent statments. +Examples with MongoDB equivalent statments. -- load extension first time after install `CREATE EXTENSION mongo_fdw;` @@ -153,9 +153,7 @@ Limitations * If the BSON document key contains uppercase letters or occurs within a nested document, `mongo_fdw` requires the corresponding column names to be - declared in double quotes. For example, a nested field such as `"review": { - "Votes": 19 }` should be declared as `"review.Votes" INTEGER` in the `CREATE - TABLE` statement. + declared in double quotes. * Note that PostgreSQL limits column names to 63 characters by default. If you need column names that are longer, you can increase the `NAMEDATALEN` From 9f971d9a3f5d24916cfa0df1b665997d4ad1c903 Mon Sep 17 00:00:00 2001 From: Ibrar Ahmed Date: Wed, 4 Jun 2014 19:55:51 +0500 Subject: [PATCH 13/43] Numeric array support for MongoDB's legacy and meta driver added --- README.md | 2 +- .../docs/buildscripts/__init__.py | 0 mongo_fdw.c | 11 +++++- mongo_fdw.h | 1 + mongo_query.c | 38 +++++++++++++++++++ mongo_query.h | 3 ++ mongo_wrapper.c | 11 ++++++ mongo_wrapper.h | 2 + mongo_wrapper_meta.c | 11 ++++++ 9 files changed, 77 insertions(+), 2 deletions(-) delete mode 100644 mongo-c-driver-v0.6/docs/buildscripts/__init__.py diff --git a/README.md b/README.md index a614783..4c775f8 100644 --- a/README.md +++ b/README.md @@ -197,4 +197,4 @@ See the [`LICENSE`][5] file for full details. [4]: CONTRIBUTING.md [5]: LICENSE [6]: https://github.com/mongodb/mongo-c-driver-legacy -[7]: https://github.com/mongodb/mongo-meta-driver +[7]: https://github.com/mongodb/mongo-c-driver diff --git a/mongo-c-driver-v0.6/docs/buildscripts/__init__.py b/mongo-c-driver-v0.6/docs/buildscripts/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/mongo_fdw.c b/mongo_fdw.c index 65d5d6c..d0fa33e 100644 --- a/mongo_fdw.c +++ b/mongo_fdw.c @@ -303,7 +303,7 @@ MongoGetForeignPaths(PlannerInfo *root, RelOptInfo *baserel, Oid foreignTableId) NULL, /* no outer rel either */ NIL); /* no fdw_private data */ - /* add foreign path as the only possible path */ + /* add foreign path as the only possible path */ add_path(baserel, foreignPath); } @@ -869,6 +869,9 @@ MongoExecForeignUpdate(EState *estate, int attnum = lfirst_int(lc); Datum value; bool isnull; + + if (strcmp("_id", slot->tts_tupleDescriptor->attrs[attnum - 1]->attname.data) == 0) + continue; value = slot_getattr(slot, attnum, &isnull); #ifdef META_DRIVER @@ -1221,6 +1224,12 @@ ColumnTypesCompatible(BSON_TYPE bsonType, Oid columnTypeId) } break; } + case NUMERICARRAY_OID: + { + if (bsonType == BSON_TYPE_ARRAY) + compatibleTypes = true; + break; + } default: { /* diff --git a/mongo_fdw.h b/mongo_fdw.h index ba40cde..11f2a60 100644 --- a/mongo_fdw.h +++ b/mongo_fdw.h @@ -53,6 +53,7 @@ #include "utils/rel.h" #include "utils/memutils.h" + #ifdef META_DRIVER #define BSON bson_t #define BSON_TYPE bson_type_t diff --git a/mongo_query.c b/mongo_query.c index fae7543..1a58a65 100644 --- a/mongo_query.c +++ b/mongo_query.c @@ -477,6 +477,44 @@ AppenMongoValue(BSON *queryDocument, const char *keyName, Datum value, bool isnu status = BsonAppendDate(queryDocument, keyName, valueMilliSecs); break; } + case NUMERICARRAY_OID: + { + ArrayType *array; + Oid elmtype; + int16 elmlen; + bool elmbyval; + char elmalign; + int num_elems; + Datum *elem_values; + bool *elem_nulls; + int i; + BSON t; + + array = DatumGetArrayTypeP(value); + elmtype = ARR_ELEMTYPE(array); + get_typlenbyvalalign(elmtype, &elmlen, &elmbyval, &elmalign); + + deconstruct_array(array, elmtype, elmlen, elmbyval, elmalign, &elem_values, &elem_nulls, &num_elems); + + BsonAppendStartArray(queryDocument, keyName, &t); + for (i = 0; i < num_elems; i++) + { + if (elem_nulls[i]) + continue; + + Datum valueDatum = DirectFunctionCall1(numeric_float8, elem_values[i]); + float8 valueFloat = DatumGetFloat8(valueDatum); +#ifdef META_DRIVER + status = BsonAppendDouble(&t, keyName, valueFloat); +#else + status = BsonAppendDouble(queryDocument, keyName, valueFloat); +#endif + } + BsonAppendFinishArray(queryDocument, &t); + pfree(elem_values); + pfree(elem_nulls); + break; + } default: { /* diff --git a/mongo_query.h b/mongo_query.h index 66f0c9f..d159dd3 100644 --- a/mongo_query.h +++ b/mongo_query.h @@ -15,6 +15,9 @@ #ifndef MONGO_QUERY_H #define MONGO_QUERY_H + +#define NUMERICARRAY_OID 1231 + bool AppenMongoValue(BSON *queryDocument, const char *keyName, Datum value, bool isnull, Oid id); #endif /* MONGO_QUERY_H */ diff --git a/mongo_wrapper.c b/mongo_wrapper.c index e4c3604..3c93ab4 100644 --- a/mongo_wrapper.c +++ b/mongo_wrapper.c @@ -287,6 +287,17 @@ BsonAppendDate(BSON *b, const char* key, time_t v) } +bool BsonAppendStartArray(BSON *b, const char* key, BSON* c) +{ + return (bson_append_start_array(b, key) == MONGO_OK); +} + + +bool BsonAppendFinishArray(BSON *b, BSON *c) +{ + return (bson_append_finish_array(b) == MONGO_OK); +} + bool BsonAppendStartObject(BSON* b, char *key, BSON* r) { diff --git a/mongo_wrapper.h b/mongo_wrapper.h index 6cc8c21..594ab1f 100644 --- a/mongo_wrapper.h +++ b/mongo_wrapper.h @@ -55,6 +55,8 @@ bool BsonAppendInt64(BSON *b, const char* key, int64_t v); bool BsonAppendDouble(BSON *b, const char* key, double v); bool BsonAppendUTF8(BSON *b, const char* key, char *v); bool BsonAppendDate(BSON *b, const char* key, time_t v); +bool BsonAppendStartArray(BSON *b, const char* key, BSON* c); +bool BsonAppendFinishArray(BSON *b, BSON *c); bool BsonAppendStartObject(BSON* b, char *key, BSON *r); bool BsonAppendFinishObject(BSON* b, BSON* r); bool BsonAppendBson(BSON* b, char *key, BSON* c); diff --git a/mongo_wrapper_meta.c b/mongo_wrapper_meta.c index 70074c9..5e77804 100644 --- a/mongo_wrapper_meta.c +++ b/mongo_wrapper_meta.c @@ -361,6 +361,17 @@ BsonAppendBson(BSON* b, char *key, BSON* c) return bson_append_document(b, key, strlen(key), c); } +bool BsonAppendStartArray(BSON *b, const char* key, BSON* c) +{ + return bson_append_array_begin(b, key, -1, c); +} + + +bool BsonAppendFinishArray(BSON *b, BSON* c) +{ + return bson_append_array_end(b, c); +} + bool BsonFinish(BSON* b) From bce5fc3c5bdf841ac637af6e088f9f72d2cd4bec Mon Sep 17 00:00:00 2001 From: Ibrar Ahmed Date: Thu, 5 Jun 2014 13:19:05 +0500 Subject: [PATCH 14/43] README file changes --- README.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 4c775f8..9bce266 100644 --- a/README.md +++ b/README.md @@ -1,32 +1,32 @@ MongoDB Foreign Data Wrapper for PostgreSQL =========================================== -This [MongoDB][1] extension implements the PostgreSQL's Foreign Data Wrapper for. +This [MongoDB][1] extension implements the PostgreSQL's Foreign Data Wrapper. -Please also note that this version of `mongo_fdw` only works with +Please note that this version of `mongo_fdw` only works with PostgreSQL Version 9.3 and greater. Installation ------------ -This [MongoDB][1] FDW is compatible with two [MongoDB][1]'s 'C' drivers, [MongoDB's legacy driver][6] and [MongoDB's Meta Driver][7]. This is a compile time decision which MongoDB's driver this FDW will use. +This [MongoDB][1] FDW is compatible with two [MongoDB][1]'s 'C' drivers, [MongoDB's Legacy driver][6] and [MongoDB's C Driver][7]. This is a compile time decision which MongoDB's driver this FDW will use. -The [MongoDB][1] FDW includes the [legacy MongoDB C Driver][6] version 0.6. When you -Type `make -f Makefile. legacy`, the C driver's source code which gets automatically compiled and linked. The other option is to compile utilizing [MongoDB's Meta Driver][7]. You require to download the [Meta C driver][7] from MongoDB site. +The [MongoDB][1] FDW includes the [MongoDB's Legacy C Driver][6] version 0.6. When you +Type `make -f Makefile. legacy` or `make`, the [MongoDB's Legacy C Driver][6]'s source code which gets automatically compiled and linked. The other option is to compile utilizing [MongoDB's C Driver][7]. You require to download the [MongoDB's C driver][7] from [MongoDB][1] site. To build on POSIX-compliant systems (like Linux and OS X), you require to ascertain The `pg_config` executable is in your path when you run make`. This executable Is typically in your PostgreSQL installation's bin directory. For example: -Compile using MonoDB's legacy C driver. +Compile using [MongoDB][1] legacy C driver. ```sh PATH=/usr/local/pgsql/bin/:$PATH make -f Makefile.legacy sudo PATH=/usr/local/pgsql/bin/:$PATH make install ``` -Compile using MonoDB's Meta C driver. +Compile using [MongoDB][1]'s C driver. Add `#deine META_DRIVER` in `config.h` file, then @@ -193,7 +193,7 @@ See the [`LICENSE`][5] file for full details. [1]: http://www.mongodb.com [2]: http://www.citusdata.com/blog/51-run-sql-on-mongodb -[3]: https://github.com/ibrarahmad/mongo_fdw/issues/new +[3]: https://github.com/enterprisedb/mongo_fdw/issues/new [4]: CONTRIBUTING.md [5]: LICENSE [6]: https://github.com/mongodb/mongo-c-driver-legacy From 1fd94bc95a808eecc553918bb2df67515e4359d1 Mon Sep 17 00:00:00 2001 From: Ibrar Ahmed Date: Fri, 6 Jun 2014 15:26:54 +0500 Subject: [PATCH 15/43] README file changes. --- README.md | 175 ++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 125 insertions(+), 50 deletions(-) diff --git a/README.md b/README.md index 9bce266..76abb33 100644 --- a/README.md +++ b/README.md @@ -1,58 +1,134 @@ -MongoDB Foreign Data Wrapper for PostgreSQL -=========================================== +# MongoDB Foreign Data Wrapper for PostgreSQL This [MongoDB][1] extension implements the PostgreSQL's Foreign Data Wrapper. Please note that this version of `mongo_fdw` only works with -PostgreSQL Version 9.3 and greater. +PostgreSQL Version **9.3** and greater. +## 1 - Installation -Installation ------------- +This [MongoDB][1] Foreign Data Wrapper is compatible with two [MongoDB][1]'s 'C' drivers, [MongoDB's Legacy C driver][6] and [MongoDB's C Driver][7]. [MongoDB][1]'s driver to use is decided at compile time using a different Makefile. -This [MongoDB][1] FDW is compatible with two [MongoDB][1]'s 'C' drivers, [MongoDB's Legacy driver][6] and [MongoDB's C Driver][7]. This is a compile time decision which MongoDB's driver this FDW will use. +### 1.1 - Compiling Using 'Legacy C' Driver### -The [MongoDB][1] FDW includes the [MongoDB's Legacy C Driver][6] version 0.6. When you -Type `make -f Makefile. legacy` or `make`, the [MongoDB's Legacy C Driver][6]'s source code which gets automatically compiled and linked. The other option is to compile utilizing [MongoDB's C Driver][7]. You require to download the [MongoDB's C driver][7] from [MongoDB][1] site. +The [MongoDB][1]'s FDW includes the source code of [MongoDB's Legacy C Driver][6] version **0.6**. + +1 - Export `pg_config` Path To build on POSIX-compliant systems (like Linux and OS X), you require to ascertain -The `pg_config` executable is in your path when you run make`. This executable -Is typically in your PostgreSQL installation's bin directory. For example: +that `pg_config` executable is in your path when you run make. This executable is typically in your PostgreSQL installation's bin directory. + +```sh +# export PATH=/usr/local/pgsql/bin:$PATH +``` + +2 - Compile code + +```sh +# make +``` + +OR + +```sh +# make -f Makefile.legacy +``` + +*Note: [MongoDB's Legacy C driver][6] will be compiled automatically.* + +3 - Install -Compile using [MongoDB][1] legacy C driver. +```sh +# make install +``` +OR ```sh -PATH=/usr/local/pgsql/bin/:$PATH make -f Makefile.legacy -sudo PATH=/usr/local/pgsql/bin/:$PATH make install +# make -f Makefile.legacy install ``` -Compile using [MongoDB][1]'s C driver. +### 1.2 - Compile Using [MongoDB][7]'s C Driver + +The source code of [MongoDB's C Driver][7] is not part of this repository. It can be downloaded from + +https://github.com/mongodb/mongo-c-driver + + +#### 1.2.1 - Install MongoDB's C Driver + +1 - Download [MongoDB's C Driver][7] -Add `#deine META_DRIVER` in `config.h` file, then +```sh +git clone https://github.com/mongodb/mongo-c-driver.git mongo-c-meta-driver +``` + +2 - Change directory ```sh -PATH=/usr/local/pgsql/bin/:$PATH make -f Makefile.meta -sudo PATH=/usr/local/pgsql/bin/:$PATH make install +cd mongo-c-meta-driver ``` -Note that we have tested the `mongo_fdw` extension only on MacOS X, Fedora and Ubuntu -systems. If you run into issues on other systems, please [let us know][3]. +3 - Configure Driver +```sh +./autogen.sh +``` -Usage ------ +4 - Compile Driver + +```sh +make +``` + +5 - Install Driver + +```sh +make install +``` + +*Note: Make sure you have permission to "/usr/local" (default installation location) folder.* + +#### 1.2.2 - Install Foreign Data Wrapper + +1 - Change configuration + +Add `#deine META_DRIVER` in `config.h` file + +1 - Export `pg_config` Path + +To build on POSIX-compliant systems (like Linux and OS X), you require to ascertain +that `pg_config` executable is in your path when you run make. This executable is typically in your PostgreSQL installation's bin directory. + +```sh +# export PATH=/usr/local/pgsql/bin:$PATH +``` + +2 - Compile code + +```sh +# make -f Makefile.meta +``` +2 - Install + +```sh +# make -f Makefile.meta install +``` + +*Note that we have verified the `mongo_fdw` extension only on MacOS X, Fedora and Ubuntu +systems. If you run into issues on other systems, please [let us know][3]* + + +## 2 - Usage The following parameters can be set on a MongoDB foreign server object: - * `address`: the address or hostname of the MongoDB server. - Defaults to `127.0.0.1` - * `port`: the port number of the MongoDB server. Defaults to `27017` + * **`address`**: the address or hostname of the MongoDB server Defaults to `127.0.0.1` + * **`port`**: the port number of the MongoDB server. Defaults to `27017` The following parameters can be set on a MongoDB foreign table object: - * `database`: the name of the MongoDB database to query. Defaults to `test` - * `collection`: the name of the MongoDB collection to query. Defaults to - the foreign table name used in the relevant `CREATE` command + * **`database`**: the name of the MongoDB database to query. Defaults to `test` + * **`collection`**: the name of the MongoDB collection to query. Defaults to the foreign table name used in the relevant `CREATE` command As an example, the following commands demonstrate loading the `mongo_fdw` wrapper, creating a server, and then creating a foreign table associated with @@ -64,36 +140,38 @@ default value mentioned above. estimating costs for the query execution plan. To see selected execution plans for a query, just run `EXPLAIN`. +Examples with [MongoDB][1]'s equivalent statments. + ```sql -Examples with MongoDB equivalent statments. -- load extension first time after install -`CREATE EXTENSION mongo_fdw;` +CREATE EXTENSION mongo_fdw; -- create server object -`CREATE SERVER mongo_server +CREATE SERVER mongo_server FOREIGN DATA WRAPPER mongo_fdw - OPTIONS (address '127.0.0.1', port '27017');` + OPTIONS (address '127.0.0.1', port '27017'); -- create foreign table -`CREATE FOREIGN TABLE warehouse( +CREATE FOREIGN TABLE warehouse( _id NAME, warehouse_id int, warehouse_name text, warehouse_created timestamptz) SERVER mongo_server - OPTIONS (database 'db', collection 'warehouse');` + OPTIONS (database 'db', collection 'warehouse'); +-- Note: first column of the table must be "_id" of type "NAME". -- select from table -`SELECT * FROM warehouse WHERE warehouse_id = 1;` +SELECT * FROM warehouse WHERE warehouse_id = 1; -_id | warehouse_id | warehouse_name | warehouse_created + _id | warehouse_id | warehouse_name | warehouse_created ------------------------+----------------+--------------------------- 53720b1904864dc1f5a571a0| 1 | UPS | 12-DEC-14 12:12:10 +05:00 -`db.warehouse.find({"warehouse_id" : 1}).pretty()` +db.warehouse.find({"warehouse_id" : 1}).pretty() { "_id" : ObjectId("53720b1904864dc1f5a571a0"), "warehouse_id" : 1, @@ -103,9 +181,9 @@ _id | warehouse_id | warehouse_name | warehouse_created -- insert row in table -`INSERT INTO warehouse values (0, 1, 'UPS', to_date('2014-12-12T07:12:10Z'));` +INSERT INTO warehouse values (0, 1, 'UPS', to_date('2014-12-12T07:12:10Z')); -`db.warehouse.insert` +db.warehouse.insert ( { "warehouse_id" : NumberInt(1), @@ -115,13 +193,13 @@ _id | warehouse_id | warehouse_name | warehouse_created ); -- delete row from table -`DELETE FROM warehouse where warehouse_id = 3;` +DELETE FROM warehouse where warehouse_id = 3; > db.warehouse.remove({"warehouse_id" : 2}) -- update a row of table -`UPDATE warehouse set warehouse_name = 'UPS_NEW' where warehouse_id = 1;` +UPDATE warehouse set warehouse_name = 'UPS_NEW' where warehouse_id = 1; db.warehouse.update ( @@ -135,21 +213,21 @@ db.warehouse.update ) -- explain a table -`EXPLAIN SELECT * FROM warehouse WHERE warehouse_id = 1; +EXPLAIN SELECT * FROM warehouse WHERE warehouse_id = 1; QUERY PLAN ----------------------------------------------------------------- Foreign Scan on warehouse (cost=0.00..0.00 rows=1000 width=44) Filter: (warehouse_id = 1) Foreign Namespace: db.warehouse Planning time: 0.671 ms -(4 rows)` +(4 rows) -- collect data distribution statistics` ANALYZE warehouse; + ``` -Limitations ------------ +## 3 - Limitations * If the BSON document key contains uppercase letters or occurs within a nested document, `mongo_fdw` requires the corresponding column names to be @@ -160,16 +238,14 @@ Limitations constant in `src/include/pg_config_manual.h`, compile, and reinstall. -Contributing ------------- +## 4 - Contributing Have a fix for a bug or an idea for a great new feature? Great! Check out the contribution guidelines [here][4]. For all other types of questions or comments about the wrapper please contact us at `ibrar.ahmed` `@` `enterprisedb.com`. -Support -------- +## 5 - Support This project will be modified to maintain compatibility with new PostgreSQL releases. The project owners set aside a day every month to look over open @@ -177,8 +253,7 @@ issues and support emails, but are not engaged in active feature development. Reported bugs will be addressed by apparent severity. -License -------- +## 6 - License Portions Copyright © 2004-2014, EnterpriseDB Corporation. From 2007c722609421403299d540e57fd3653c71cff5 Mon Sep 17 00:00:00 2001 From: Ibrar Ahmed Date: Fri, 6 Jun 2014 17:13:52 +0500 Subject: [PATCH 16/43] Throw error in case type of first column of MongoDB's foreign Table not "name". --- mongo_fdw.c | 5 ++++- mongo_query.c | 6 ++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/mongo_fdw.c b/mongo_fdw.c index d0fa33e..7115fd0 100644 --- a/mongo_fdw.c +++ b/mongo_fdw.c @@ -725,7 +725,7 @@ MongoExecForeignInsert(EState *estate, MONGO_CONN *mongoConnection = NULL; Oid foreignTableId = InvalidOid; BSON *b = NULL; - Oid typoid = InvalidOid; + Oid typoid; Datum value; bool isnull = false; @@ -757,6 +757,9 @@ MongoExecForeignInsert(EState *estate, if (strcmp(slot->tts_tupleDescriptor->attrs[0]->attname.data, "_id") != 0) elog(ERROR, "first colum of MongoDB's foreign table must be \"_id\""); + if (typoid != NAMEOID) + elog(ERROR, "type of first colum of MongoDB's foreign table must be \"name\""); + if (attnum == 1) { /* diff --git a/mongo_query.c b/mongo_query.c index 1a58a65..7708db6 100644 --- a/mongo_query.c +++ b/mongo_query.c @@ -499,11 +499,13 @@ AppenMongoValue(BSON *queryDocument, const char *keyName, Datum value, bool isnu BsonAppendStartArray(queryDocument, keyName, &t); for (i = 0; i < num_elems; i++) { + Datum valueDatum; + float8 valueFloat; if (elem_nulls[i]) continue; - Datum valueDatum = DirectFunctionCall1(numeric_float8, elem_values[i]); - float8 valueFloat = DatumGetFloat8(valueDatum); + valueDatum = DirectFunctionCall1(numeric_float8, elem_values[i]); + valueFloat = DatumGetFloat8(valueDatum); #ifdef META_DRIVER status = BsonAppendDouble(&t, keyName, valueFloat); #else From 990bdf1413ce8ef0631249ce7abfb32bee9b71b0 Mon Sep 17 00:00:00 2001 From: Ibrar Ahmed Date: Fri, 6 Jun 2014 20:09:09 +0500 Subject: [PATCH 17/43] Wrong BSON object was passed in case of Meta Drive --- mongo_query.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mongo_query.c b/mongo_query.c index 7708db6..246b715 100644 --- a/mongo_query.c +++ b/mongo_query.c @@ -239,8 +239,11 @@ QueryDocument(Oid relationId, List *opExpressionList) operatorName = get_opname(columnOperator->opno); mongoOperatorName = MongoOperatorName(operatorName); - +#ifdef META_DRIVER + AppendConstantValue(&r, mongoOperatorName, constant); +#else AppendConstantValue(queryDocument, mongoOperatorName, constant); +#endif } BsonAppendFinishObject(queryDocument, &r); } From 87b026cbedd3046da4f5daabade7c0fea7687a0b Mon Sep 17 00:00:00 2001 From: Ibrar Ahmed Date: Mon, 9 Jun 2014 17:55:01 +0500 Subject: [PATCH 18/43] The "analyze" statement throwing error while retrieving rows from MongoDB. --- mongo_fdw.c | 46 ++++++++++++++++++++++++-------------------- mongo_query.c | 1 + mongo_wrapper.c | 2 +- mongo_wrapper.h | 2 +- mongo_wrapper_meta.c | 4 ++-- 5 files changed, 30 insertions(+), 25 deletions(-) diff --git a/mongo_fdw.c b/mongo_fdw.c index 7115fd0..8c4b256 100644 --- a/mongo_fdw.c +++ b/mongo_fdw.c @@ -516,13 +516,15 @@ MongoIterateForeignScan(ForeignScanState *scanState) } else { - #ifndef META_DRIVER - /* - * The following is a courtesy check. In practice when Mongo shuts down, - * mongo_cursor_next() could possibly crash. This function first frees - * cursor->reply, and then references reply in mongo_cursor__destroy(). - */ - + #ifdef META_DRIVER + bson_error_t error; + if (mongoc_cursor_error (mongoCursor, &error)) + { + MongoFreeScanState(fmstate); + ereport(ERROR, (errmsg("could not iterate over mongo collection"), + errhint("Mongo driver error: %s", error.message))); + } + #else mongo_cursor_error_t errorCode = mongoCursor->err; if (errorCode != MONGO_CURSOR_EXHAUSTED) { @@ -997,7 +999,7 @@ ForeignTableDocumentCount(Oid foreignTableId) mongoConnection = GetConnection(options->addressName, options->portNumber); - MongoAggregateCount(mongoConnection, options->databaseName, options->collectionName, emptyQuery); + documentCount = MongoAggregateCount(mongoConnection, options->databaseName, options->collectionName, emptyQuery); MongoFreeOptions(options); @@ -1575,7 +1577,7 @@ MongoAcquireSampleRows(Relation relation, int errorLevel, foreignTableId = RelationGetRelid(relation); queryDocument = QueryDocument(foreignTableId, NIL); - foreignPrivateList = list_make1(columnList); + foreignPrivateList = list_make2(columnList, NULL); /* only clean up the query struct, but not its data */ BsonDestroy(queryDocument); @@ -1632,20 +1634,22 @@ MongoAcquireSampleRows(Relation relation, int errorLevel, } else { - #ifndef META_DRIVER - /* - * The following is a courtesy check. In practice when Mongo shuts down, - * mongo_cursor__next() could possibly crash. - */ - mongo_cursor_error_t errorCode = mongoCursor->err; - - if (errorCode != MONGO_CURSOR_EXHAUSTED) + #ifdef META_DRIVER + bson_error_t error; + if (mongoc_cursor_error (mongoCursor, &error)) { MongoFreeScanState(fmstate); - ereport(ERROR, (errmsg("could not iterate over mongo 11collection"), - errhint("Mongo driver cursor error code: %d", - errorCode))); + ereport(ERROR, (errmsg("could not iterate over mongo collection"), + errhint("Mongo driver error: %s", error.message))); } + #else + mongo_cursor_error_t errorCode = mongoCursor->err; + if (errorCode != MONGO_CURSOR_EXHAUSTED) + { + MongoFreeScanState(fmstate); + ereport(ERROR, (errmsg("could not iterate over mongo collection"), + errhint("Mongo driver cursor error code: %d", errorCode))); + } #endif break; } @@ -1706,7 +1710,7 @@ MongoAcquireSampleRows(Relation relation, int errorLevel, /* emit some interesting relation info */ relationName = RelationGetRelationName(relation); - ereport(errorLevel, (errmsg("\"%s\": collection contains %.0f rows; %d rows in sample", + ereport(errorLevel, (errmsg("\"%s\": collection contains %.0f rows; %d rows in sample", relationName, rowCount, sampleRowCount))); (*totalRowCount) = rowCount; diff --git a/mongo_query.c b/mongo_query.c index 246b715..8b95f1c 100644 --- a/mongo_query.c +++ b/mongo_query.c @@ -247,6 +247,7 @@ QueryDocument(Oid relationId, List *opExpressionList) } BsonAppendFinishObject(queryDocument, &r); } + if (!BsonFinish(queryDocument)) { ereport(ERROR, (errmsg("could not create document for query"), diff --git a/mongo_wrapper.c b/mongo_wrapper.c index 3c93ab4..5dd53fe 100644 --- a/mongo_wrapper.c +++ b/mongo_wrapper.c @@ -324,7 +324,7 @@ BsonFinish(BSON* b) return (bson_finish(b) == MONGO_OK); } -int +double MongoAggregateCount(MONGO_CONN* conn, const char* database, const char* collection, const BSON* b) { return mongo_count(conn, database, collection, b); diff --git a/mongo_wrapper.h b/mongo_wrapper.h index 594ab1f..5d023f1 100644 --- a/mongo_wrapper.h +++ b/mongo_wrapper.h @@ -26,7 +26,7 @@ MONGO_CURSOR* MongoCursorCreate(MONGO_CONN* conn, char* database, char *collecti const BSON* MongoCursorBson(MONGO_CURSOR* c); bool MongoCursorNext(MONGO_CURSOR* c, BSON* b); void MongoCursorDestroy(MONGO_CURSOR* c); -int MongoAggregateCount(MONGO_CONN* conn, const char* database, const char* collection, const BSON* b); +double MongoAggregateCount(MONGO_CONN* conn, const char* database, const char* collection, const BSON* b); BSON* BsonCreate(void); void BsonDestroy(BSON *b); diff --git a/mongo_wrapper_meta.c b/mongo_wrapper_meta.c index 5e77804..bea95dd 100644 --- a/mongo_wrapper_meta.c +++ b/mongo_wrapper_meta.c @@ -387,7 +387,7 @@ BsonFinish(BSON* b) /* * Count the number of documents. */ -int +double MongoAggregateCount(MONGO_CONN* conn, const char* database, const char* collection, const BSON* b) { BSON *cmd = NULL; @@ -417,5 +417,5 @@ MongoAggregateCount(MONGO_CONN* conn, const char* database, const char* collecti BsonDestroy(out); BsonDestroy(cmd); - return (int)count; + return count; } From 9f7e1255747166b374fa99622c7bd59ac94f3bf1 Mon Sep 17 00:00:00 2001 From: Ahsan Hadi Date: Tue, 24 Jun 2014 16:03:33 +0500 Subject: [PATCH 19/43] Misc changes to readme --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 76abb33..1bef2dc 100644 --- a/README.md +++ b/README.md @@ -242,7 +242,7 @@ ANALYZE warehouse; Have a fix for a bug or an idea for a great new feature? Great! Check out the contribution guidelines [here][4]. For all other types of questions or comments -about the wrapper please contact us at `ibrar.ahmed` `@` `enterprisedb.com`. +about the wrapper please contact us at `mongo_fdw` `@` `enterprisedb.com`. ## 5 - Support @@ -252,6 +252,7 @@ releases. The project owners set aside a day every month to look over open issues and support emails, but are not engaged in active feature development. Reported bugs will be addressed by apparent severity. +As with many open source projects, you may be able to obtain support via the public mailing list (`mongo_fdw` `@` `enterprisedb.com`). If you need commercial support, please contact the EnterpriseDB sales team, or check whether your existing PostgreSQL support provider can also support mongo_fdw. ## 6 - License From 19f16f495f848514add0cb9f37599750b62cbd92 Mon Sep 17 00:00:00 2001 From: Ahsan Hadi Date: Tue, 24 Jun 2014 16:05:50 +0500 Subject: [PATCH 20/43] Update CONTRIBUTING.md --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index bd5c88b..e490f03 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -11,7 +11,7 @@ Using Issues `mongo_fdw`'s maintainers prefer that bug reports, feature requests, and pull requests are submitted as [GitHub Issues][1]. If you think you require personal -assistance, please **do not** open an issue: email `ibrar.ahmed` `@` `enterprisedb.com` +assistance, please **do not** open an issue: email `mongo_fdw` `@` `enterprisedb.com` instead. From 08bbee63259378a3fced1f6891cf8a5e9dbb2069 Mon Sep 17 00:00:00 2001 From: Ibrar Ahmed Date: Thu, 10 Jul 2014 12:38:13 +0500 Subject: [PATCH 21/43] Authentication using USER mapping --- connection.c | 4 ++-- mongo_fdw.c | 16 +++++++++++----- mongo_fdw.h | 19 ++++++++++++++----- mongo_wrapper.c | 13 ++++++++++++- mongo_wrapper.h | 2 +- mongo_wrapper_meta.c | 7 +++++-- option.c | 11 ++++++++++- 7 files changed, 55 insertions(+), 17 deletions(-) diff --git a/connection.c b/connection.c index aefe924..94c8e18 100644 --- a/connection.c +++ b/connection.c @@ -60,7 +60,7 @@ static HTAB *ConnectionHash = NULL; * is established if we don't already have a suitable one. */ MONGO_CONN* -GetConnection(char *host, int32 port) +GetConnection(char *host, int32 port, char *databaseName, char *user, char *password) { bool found; ConnCacheEntry *entry; @@ -97,7 +97,7 @@ GetConnection(char *host, int32 port) } if (entry->conn == NULL) { - entry->conn = MongoConnect(host, port); + entry->conn = MongoConnect(host, port, databaseName, user, password); elog(DEBUG3, "new mongo_fdw connection %p for server \"%s:%d\"", entry->conn, host, port); } diff --git a/mongo_fdw.c b/mongo_fdw.c index 8c4b256..6695868 100644 --- a/mongo_fdw.c +++ b/mongo_fdw.c @@ -424,6 +424,9 @@ MongoBeginForeignScan(ForeignScanState *scanState, int executorFlags) HTAB *columnMappingHash = NULL; char *addressName = NULL; int32 portNumber = 0; + char *databaseName= NULL; + char *username= NULL; + char *password= NULL; ForeignScan *foreignScan = NULL; List *foreignPrivateList = NIL; BSON *queryDocument = NULL; @@ -441,12 +444,15 @@ MongoBeginForeignScan(ForeignScanState *scanState, int executorFlags) /* resolve hostname and port number; and connect to mongo server */ addressName = mongoFdwOptions->addressName; portNumber = mongoFdwOptions->portNumber; + databaseName = mongoFdwOptions->databaseName; + username = mongoFdwOptions->username; + password = mongoFdwOptions->password; /* * Get connection to the foreign server. Connection manager will * establish new connection if necessary. */ - mongoConnection = GetConnection(addressName, portNumber); + mongoConnection = GetConnection(addressName, portNumber, databaseName, username, password); foreignScan = (ForeignScan *) scanState->ss.ps.plan; foreignPrivateList = foreignScan->fdw_private; @@ -739,7 +745,7 @@ MongoExecForeignInsert(EState *estate, /* resolve foreign table options; and connect to mongo server */ options = fmstate->mongoFdwOptions; - mongoConnection = GetConnection(options->addressName, options->portNumber); + mongoConnection = GetConnection(options->addressName, options->portNumber, options->databaseName, options->username, options->password); b = BsonCreate(); @@ -852,7 +858,7 @@ MongoExecForeignUpdate(EState *estate, /* resolve foreign table options; and connect to mongo server */ options = fmstate->mongoFdwOptions; - mongoConnection = GetConnection(options->addressName, options->portNumber); + mongoConnection = GetConnection(options->addressName, options->portNumber, options->databaseName, options->username, options->password); /* Get the id that was passed up as a resjunk column */ datum = ExecGetJunkAttribute(planSlot, 1, &isNull); @@ -935,7 +941,7 @@ MongoExecForeignDelete(EState *estate, /* resolve foreign table options; and connect to mongo server */ options = fmstate->mongoFdwOptions; - mongoConnection = GetConnection(options->addressName, options->portNumber); + mongoConnection = GetConnection(options->addressName, options->portNumber, options->databaseName, options->username, options->password); /* Get the id that was passed up as a resjunk column */ datum = ExecGetJunkAttribute(planSlot, 1, &isNull); @@ -997,7 +1003,7 @@ ForeignTableDocumentCount(Oid foreignTableId) /* resolve foreign table options; and connect to mongo server */ options = MongoGetOptions(foreignTableId); - mongoConnection = GetConnection(options->addressName, options->portNumber); + mongoConnection = GetConnection(options->addressName, options->portNumber, options->databaseName, options->username, options->password); documentCount = MongoAggregateCount(mongoConnection, options->databaseName, options->collectionName, emptyQuery); diff --git a/mongo_fdw.h b/mongo_fdw.h index 11f2a60..a4e9efa 100644 --- a/mongo_fdw.h +++ b/mongo_fdw.h @@ -52,7 +52,7 @@ #include "utils/lsyscache.h" #include "utils/rel.h" #include "utils/memutils.h" - +#include "catalog/pg_user_mapping.h" #ifdef META_DRIVER #define BSON bson_t @@ -83,6 +83,8 @@ #define OPTION_NAME_PORT "port" #define OPTION_NAME_DATABASE "database" #define OPTION_NAME_COLLECTION "collection" +#define OPTION_NAME_USERNAME "username" +#define OPTION_NAME_PASSWORD "password" /* Default values for option parameters */ #define DEFAULT_IP_ADDRESS "127.0.0.1" @@ -112,16 +114,21 @@ typedef struct MongoValidOption /* Array of options that are valid for mongo_fdw */ -static const uint32 ValidOptionCount = 4; +static const uint32 ValidOptionCount = 6; static const MongoValidOption ValidOptionArray[] = { /* foreign server options */ { OPTION_NAME_ADDRESS, ForeignServerRelationId }, - { OPTION_NAME_PORT, ForeignServerRelationId }, + { OPTION_NAME_PORT, ForeignServerRelationId }, /* foreign table options */ { OPTION_NAME_DATABASE, ForeignTableRelationId }, - { OPTION_NAME_COLLECTION, ForeignTableRelationId } + { OPTION_NAME_COLLECTION, ForeignTableRelationId }, + + /* User mapping options */ + { OPTION_NAME_USERNAME, UserMappingRelationId }, + { OPTION_NAME_PASSWORD, UserMappingRelationId } + }; @@ -136,6 +143,8 @@ typedef struct MongoFdwOptions int32 portNumber; char *databaseName; char *collectionName; + char *username; + char *password; } MongoFdwOptions; @@ -184,7 +193,7 @@ typedef struct ColumnMapping } ColumnMapping; -extern MONGO_CONN *GetConnection(char *host, int32 port); +extern MONGO_CONN *GetConnection(char *host, int32 port, char *databaneName, char *user, char *password); extern void cleanup_connection(void); extern void ReleaseConnection(MONGO_CONN* conn); diff --git a/mongo_wrapper.c b/mongo_wrapper.c index 5dd53fe..6026878 100644 --- a/mongo_wrapper.c +++ b/mongo_wrapper.c @@ -26,7 +26,7 @@ #define QUAL_STRING_LEN 512 MONGO_CONN* -MongoConnect(const char* host, const unsigned short port) +MongoConnect(const char* host, const unsigned short port, char* databaseName, char *user, char *password) { MONGO_CONN *conn; conn = mongo_create(); @@ -40,6 +40,17 @@ MongoConnect(const char* host, const unsigned short port) ereport(ERROR, (errmsg("could not connect to %s:%d", host, port), errhint("Mongo driver connection error: %d", err))); } + if (user && password) + { + if (mongo_cmd_authenticate(conn, databaseName, user, password) != MONGO_OK) + { + int err = conn->err; + mongo_destroy(conn); + mongo_dispose(conn); + ereport(ERROR, (errmsg("could not connect to %s:%d", host, port), + errhint("Mongo driver connection error: %d", err))); + } + } return conn; } diff --git a/mongo_wrapper.h b/mongo_wrapper.h index 5d023f1..6a957bf 100644 --- a/mongo_wrapper.h +++ b/mongo_wrapper.h @@ -17,7 +17,7 @@ #include "mongo.h" #endif -MONGO_CONN* MongoConnect(const char* host, const unsigned short port); +MONGO_CONN* MongoConnect(const char* host, const unsigned short port, char *databaseName, char *user, char *password); void MongoDisconnect(MONGO_CONN* conn); bool MongoInsert(MONGO_CONN* conn, char* database, char *collection, BSON* b); bool MongoUpdate(MONGO_CONN* conn, char* database, char *collection, BSON* b, BSON* op); diff --git a/mongo_wrapper_meta.c b/mongo_wrapper_meta.c index bea95dd..f92513f 100644 --- a/mongo_wrapper_meta.c +++ b/mongo_wrapper_meta.c @@ -21,12 +21,15 @@ * Connect to MongoDB server using Host/ip and Port number. */ MONGO_CONN* -MongoConnect(const char* host, const unsigned short port) +MongoConnect(const char* host, const unsigned short port, char* databaseName, char *user, char *password) { MONGO_CONN *client = NULL; char* uri = NULL; - uri = bson_strdup_printf ("mongodb://%s:%hu/", host, port); + if (user && password) + uri = bson_strdup_printf ("mongodb://%s:%s@%s:%hu/", user, password, host, port); + else + uri = bson_strdup_printf ("mongodb://%s:%hu/", host, port); client = mongoc_client_new(uri); diff --git a/option.c b/option.c index 5504373..1f1fa75 100644 --- a/option.c +++ b/option.c @@ -37,7 +37,7 @@ #include "utils/lsyscache.h" #include "utils/rel.h" #include "utils/memutils.h" - +#include "miscadmin.h" static char * MongoGetOptionValue(Oid foreignTableId, const char *optionName); @@ -148,6 +148,8 @@ MongoGetOptions(Oid foreignTableId) int32 portNumber = 0; char *databaseName = NULL; char *collectionName = NULL; + char *username= NULL; + char *password= NULL; addressName = MongoGetOptionValue(foreignTableId, OPTION_NAME_ADDRESS); if (addressName == NULL) @@ -176,12 +178,16 @@ MongoGetOptions(Oid foreignTableId) { collectionName = get_rel_name(foreignTableId); } + username = MongoGetOptionValue(foreignTableId, OPTION_NAME_USERNAME); + password = MongoGetOptionValue(foreignTableId, OPTION_NAME_PASSWORD); mongoFdwOptions = (MongoFdwOptions *) palloc0(sizeof(MongoFdwOptions)); mongoFdwOptions->addressName = addressName; mongoFdwOptions->portNumber = portNumber; mongoFdwOptions->databaseName = databaseName; mongoFdwOptions->collectionName = collectionName; + mongoFdwOptions->username = username; + mongoFdwOptions->password = password; return mongoFdwOptions; } @@ -210,13 +216,16 @@ MongoGetOptionValue(Oid foreignTableId, const char *optionName) ForeignServer *foreignServer = NULL; List *optionList = NIL; ListCell *optionCell = NULL; + UserMapping *mapping= NULL; char *optionValue = NULL; foreignTable = GetForeignTable(foreignTableId); foreignServer = GetForeignServer(foreignTable->serverid); + mapping = GetUserMapping(GetUserId(), foreignTable->serverid); optionList = list_concat(optionList, foreignTable->options); optionList = list_concat(optionList, foreignServer->options); + optionList = list_concat(optionList, mapping->options); foreach(optionCell, optionList) { From f4f6c423633155c02d8df39301b815bff6b000a2 Mon Sep 17 00:00:00 2001 From: Ibrar Ahmed Date: Thu, 10 Jul 2014 12:59:57 +0500 Subject: [PATCH 22/43] README Changes for user mapping --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 1bef2dc..354a08d 100644 --- a/README.md +++ b/README.md @@ -152,6 +152,11 @@ CREATE SERVER mongo_server FOREIGN DATA WRAPPER mongo_fdw OPTIONS (address '127.0.0.1', port '27017'); +-- create user mapping +CREATE USER MAPPING FOR postgres + SERVER mongo_server + OPTIONS (username 'mongo_user', password 'mongo_pass'); + -- create foreign table CREATE FOREIGN TABLE warehouse( _id NAME, From 9da15a0ffa9954f2fc6aadf90067b7e95179443a Mon Sep 17 00:00:00 2001 From: Ibrar Ahmed Date: Tue, 30 Sep 2014 23:36:50 +0500 Subject: [PATCH 23/43] License header change --- config.h | 17 +++++++++++++++++ mongo_fdw.c | 11 +++++++---- mongo_fdw.h | 9 ++++++--- mongo_query.c | 11 ++++++----- mongo_query.h | 9 ++++++--- mongo_wrapper.c | 9 ++++++--- mongo_wrapper.h | 11 ++++++++++- mongo_wrapper_meta.c | 10 +++++++--- option.c | 10 ++++++---- 9 files changed, 71 insertions(+), 26 deletions(-) diff --git a/config.h b/config.h index 6139f3e..ee6ce48 100644 --- a/config.h +++ b/config.h @@ -1,3 +1,20 @@ +/*------------------------------------------------------------------------- + * + * config.h + * Foreign-data wrapper for remote MongoDB servers + * + * Portions Copyright (c) 2012-2014, PostgreSQL Global Development Group + * + * Portions Copyright (c) 2004-2014, EnterpriseDB Corporation. + * + * Portions Copyright (c) 2012–2014 Citus Data, Inc. + * + * IDENTIFICATION + * config.h + * + *------------------------------------------------------------------------- + */ + /* * Define if you want to compile the MongoFDW with Meta C Driver, otherwise * it will compile using MongoDB legacy C Driver diff --git a/mongo_fdw.c b/mongo_fdw.c index 510968d..6485559 100644 --- a/mongo_fdw.c +++ b/mongo_fdw.c @@ -1,13 +1,16 @@ /*------------------------------------------------------------------------- * * mongo_fdw.c + * Foreign-data wrapper for remote MongoDB servers * - * Function definitions for MongoDB foreign data wrapper. These functions access - * data stored in MongoDB through the official C driver. + * Portions Copyright (c) 2012-2014, PostgreSQL Global Development Group * - * Portions Copyright © 2004-2014, EnterpriseDB Corporation. + * Portions Copyright (c) 2004-2014, EnterpriseDB Corporation. * - * Portions Copyright © 2012–2014 Citus Data, Inc. + * Portions Copyright (c) 2012–2014 Citus Data, Inc. + * + * IDENTIFICATION + * mongo_fdw.c * *------------------------------------------------------------------------- */ diff --git a/mongo_fdw.h b/mongo_fdw.h index a4e9efa..c040fd0 100644 --- a/mongo_fdw.h +++ b/mongo_fdw.h @@ -1,13 +1,16 @@ /*------------------------------------------------------------------------- * * mongo_fdw.h + * Foreign-data wrapper for remote MongoDB servers * - * Type and function declarations for MongoDB foreign data wrapper. + * Portions Copyright (c) 2012-2014, PostgreSQL Global Development Group * - * Portions Copyright © 2004-2014, EnterpriseDB Corporation. + * Portions Copyright (c) 2004-2014, EnterpriseDB Corporation. * - * Portions Copyright © 2012–2014 Citus Data, Inc. + * Portions Copyright (c) 2012–2014 Citus Data, Inc. * + * IDENTIFICATION + * mongo_fdw.h * *------------------------------------------------------------------------- */ diff --git a/mongo_query.c b/mongo_query.c index 1d4b7a3..d8356e5 100644 --- a/mongo_query.c +++ b/mongo_query.c @@ -1,15 +1,16 @@ /*------------------------------------------------------------------------- * * mongo_query.c + * Foreign-data wrapper for remote MongoDB servers * - * Function definitions for sending queries to MongoDB. These functions assume - * that queries are sent through the official MongoDB C driver, and apply query - * optimizations to reduce the amount of data fetched from the driver. + * Portions Copyright (c) 2012-2014, PostgreSQL Global Development Group * - * Portions Copyright © 2004-2014, EnterpriseDB Corporation. + * Portions Copyright (c) 2004-2014, EnterpriseDB Corporation. * - * Portions Copyright © 2012–2014 Citus Data, Inc. + * Portions Copyright (c) 2012–2014 Citus Data, Inc. * + * IDENTIFICATION + * mongo_query.c * *------------------------------------------------------------------------- */ diff --git a/mongo_query.h b/mongo_query.h index d159dd3..9ba8508 100644 --- a/mongo_query.h +++ b/mongo_query.h @@ -1,13 +1,16 @@ /*------------------------------------------------------------------------- * * mongo_query.h + * Foreign-data wrapper for remote MongoDB servers * - * Type and function declarations for constructing queries to send to MongoDB. + * Portions Copyright (c) 2012-2014, PostgreSQL Global Development Group * - * Portions Copyright © 2004-2014, EnterpriseDB Corporation. + * Portions Copyright (c) 2004-2014, EnterpriseDB Corporation. * - * Portions Copyright © 2012–2014 Citus Data, Inc. + * Portions Copyright (c) 2012–2014 Citus Data, Inc. * + * IDENTIFICATION + * mongo_query.h * *------------------------------------------------------------------------- */ diff --git a/mongo_wrapper.c b/mongo_wrapper.c index d20b678..99dadd5 100644 --- a/mongo_wrapper.c +++ b/mongo_wrapper.c @@ -1,13 +1,16 @@ /*------------------------------------------------------------------------- * * mongo_wrapper.c + * Foreign-data wrapper for remote MongoDB servers * - * Wrapper functions for MongoDB's old legacy Driver. + * Portions Copyright (c) 2012-2014, PostgreSQL Global Development Group * - * Portions Copyright © 2004-2014, EnterpriseDB Corporation. + * Portions Copyright (c) 2004-2014, EnterpriseDB Corporation. * - * Portions Copyright © 2012–2014 Citus Data, Inc. + * Portions Copyright (c) 2012–2014 Citus Data, Inc. * + * IDENTIFICATION + * mongo_wrapper.c * *------------------------------------------------------------------------- */ diff --git a/mongo_wrapper.h b/mongo_wrapper.h index 6a957bf..f3a26d8 100644 --- a/mongo_wrapper.h +++ b/mongo_wrapper.h @@ -1,10 +1,19 @@ /*------------------------------------------------------------------------- * * mongo_wrapper.h + * Foreign-data wrapper for remote MongoDB servers + * + * Portions Copyright (c) 2012-2014, PostgreSQL Global Development Group + * + * Portions Copyright (c) 2004-2014, EnterpriseDB Corporation. + * + * Portions Copyright (c) 2012–2014 Citus Data, Inc. + * + * IDENTIFICATION + * mongo_wrapper.h * *------------------------------------------------------------------------- */ - #ifndef MONGO_WRAPPER_H #define MONGO_WRAPPER_H diff --git a/mongo_wrapper_meta.c b/mongo_wrapper_meta.c index f92513f..400bd12 100644 --- a/mongo_wrapper_meta.c +++ b/mongo_wrapper_meta.c @@ -1,17 +1,21 @@ /*------------------------------------------------------------------------- * * mongo_wrapper_meta.c + * Foreign-data wrapper for remote MongoDB servers * - * Wrapper functions for MongoDB's new Meta Driver. + * Portions Copyright (c) 2012-2014, PostgreSQL Global Development Group * - * Portions Copyright © 2004-2014, EnterpriseDB Corporation. + * Portions Copyright (c) 2004-2014, EnterpriseDB Corporation. * - * Portions Copyright © 2012–2014 Citus Data, Inc. + * Portions Copyright (c) 2012–2014 Citus Data, Inc. * + * IDENTIFICATION + * mongo_wrapper_meta.c * *------------------------------------------------------------------------- */ + #include "postgres.h" #include diff --git a/option.c b/option.c index 1f1fa75..bbd3ed0 100644 --- a/option.c +++ b/option.c @@ -1,14 +1,16 @@ /*------------------------------------------------------------------------- * * option.c + * Foreign-data wrapper for remote MongoDB servers * - * Function definitions for MongoDB foreign data wrapper. These functions access - * data stored in MongoDB through the official C driver. + * Portions Copyright (c) 2012-2014, PostgreSQL Global Development Group * - * Portions Copyright © 2004-2014, EnterpriseDB Corporation. + * Portions Copyright (c) 2004-2014, EnterpriseDB Corporation. * - * Portions Copyright © 2012–2014 Citus Data, Inc. + * Portions Copyright (c) 2012–2014 Citus Data, Inc. * + * IDENTIFICATION + * option.c * *------------------------------------------------------------------------- */ From b6aa5e145102265ba6ccb9c9f7c0e6396f806aea Mon Sep 17 00:00:00 2001 From: Ibrar Ahmed Date: Tue, 30 Sep 2014 23:44:00 +0500 Subject: [PATCH 24/43] PostgreSQL Version 9.5 support --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 4e24993..afcb3e3 100644 --- a/Makefile +++ b/Makefile @@ -44,6 +44,6 @@ ifndef MAJORVERSION MAJORVERSION := $(basename $(VERSION)) endif -ifeq (,$(findstring $(MAJORVERSION), 9.3 9.4)) +ifeq (,$(findstring $(MAJORVERSION), 9.3 9.4 9.5)) $(error PostgreSQL 9.3 or 9.4 is required to compile this extension) endif From f81428339d7fdc328b17668a8f67dbb8aebf092b Mon Sep 17 00:00:00 2001 From: Ibrar Ahmed Date: Tue, 30 Sep 2014 23:45:40 +0500 Subject: [PATCH 25/43] PostgreSQL Version 9.5 support --- Makefile | 2 +- Makefile.meta | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index afcb3e3..a65c28e 100644 --- a/Makefile +++ b/Makefile @@ -45,5 +45,5 @@ ifndef MAJORVERSION endif ifeq (,$(findstring $(MAJORVERSION), 9.3 9.4 9.5)) - $(error PostgreSQL 9.3 or 9.4 is required to compile this extension) + $(error PostgreSQL 9.3, 9.4 or 9.5 is required to compile this extension) endif diff --git a/Makefile.meta b/Makefile.meta index 569d2f0..52a527a 100644 --- a/Makefile.meta +++ b/Makefile.meta @@ -41,6 +41,6 @@ ifndef MAJORVERSION MAJORVERSION := $(basename $(VERSION)) endif -ifeq (,$(findstring $(MAJORVERSION), 9.3 9.4)) - $(error PostgreSQL 9.3 or 9.4 is required to compile this extension) +ifeq (,$(findstring $(MAJORVERSION), 9.3 9.4 9.5)) + $(error PostgreSQL 9.3, 9.4 or 9.5 is required to compile this extension) endif From c9b34617ff2d7c588d96489db1bde0a938065841 Mon Sep 17 00:00:00 2001 From: Ibrar Ahmed Date: Tue, 30 Sep 2014 23:56:26 +0500 Subject: [PATCH 26/43] README File changes --- README.md | 58 +++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 41 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 2725555..2042ad9 100644 --- a/README.md +++ b/README.md @@ -5,8 +5,8 @@ This [MongoDB][1] extension implements the PostgreSQL's Foreign Data Wrapper. Please note that this version of `mongo_fdw` only works with PostgreSQL Version **9.3** and greater. -## 1 - Installation - +Installation +------------ The MongoDB FDW depends on the official MongoDB C Driver version 0.8 and includes it as a git submodule. If you are cloning this repository for the first time, be sure to pass the --recursive option to git clone in order to @@ -19,14 +19,37 @@ not up-to-date, run git submodule update --init. When you type `make`, the C driver's source code also gets automatically compiled and linked. -*Note: Make sure you have permission to "/usr/local" (default installation location) folder.* - -*Note that we have verified the `mongo_fdw` extension only on MacOS X, Fedora and Ubuntu -systems. If you run into issues on other systems, please [let us know][3]* - - -## 2 - Usage - +`*Note: Make sure you have permission to "/usr/local" (default installation location) folder.*` + +`*Note that we have verified the `mongo_fdw` extension only on MacOS X, Fedora and Ubuntu systems. If you run into issues on other systems, please [let us know][3]*` + +Enhancments +----------- +The following enhancements are added to the latest version of mongo_fdw + +Write-able FDW +-------------- +The previous version was only read-only, the latest version provides the write capability. +The user can now issue insert/update and delete statements for the foreign tables using the mongo_fdw. + +Connection Pooling +------------------ +The latest version comes with a connection pooler that utilises the same mongo +database connection for all the queries in the same session. The previous version +would open a new mongodb connection for every query. +This is a performance enhancement. + +New MongoDB C Driver Support +---------------------------- +The third enhancement is to add a new [MongoDB][1]' C driver. The current implementation is +based on the legacy driver of MongoDB. But [MongoDB][1] is provided completely new library +for driver called MongoDB's Meta Driver. So I have added support of that driver. +Now compile time option is available to use legacy and Meta driver. I am sure there +are many other benefits of the new Mongo-C-driver that we are not leveraging but we +will adopt those as we learn more about the new C driver. + +Usage +----- The following parameters can be set on a MongoDB foreign server object: * **`address`**: the address or hostname of the MongoDB server Defaults to `127.0.0.1` @@ -139,7 +162,8 @@ ANALYZE warehouse; ``` -## 3 - Limitations +Limitations +----------- * If the BSON document key contains uppercase letters or occurs within a nested document, `mongo_fdw` requires the corresponding column names to be @@ -150,15 +174,15 @@ ANALYZE warehouse; constant in `src/include/pg_config_manual.h`, compile, and reinstall. -## 4 - Contributing - +Contributing +------------ Have a fix for a bug or an idea for a great new feature? Great! Check out the contribution guidelines [here][4]. For all other types of questions or comments about the wrapper please contact us at `mongo_fdw` `@` `enterprisedb.com`. -## 5 - Support - +Support +------- This project will be modified to maintain compatibility with new PostgreSQL releases. The project owners set aside a day every month to look over open issues and support emails, but are not engaged in active feature development. @@ -166,8 +190,8 @@ Reported bugs will be addressed by apparent severity. As with many open source projects, you may be able to obtain support via the public mailing list (`mongo_fdw` `@` `enterprisedb.com`). If you need commercial support, please contact the EnterpriseDB sales team, or check whether your existing PostgreSQL support provider can also support mongo_fdw. -## 6 - License - +License +------- Portions Copyright © 2004-2014, EnterpriseDB Corporation. Portions Copyright © 2012–2014 Citus Data, Inc. From e5d1ec08e3bc73bc0971a02d70df73f9a1289dd9 Mon Sep 17 00:00:00 2001 From: Ibrar Ahmed Date: Wed, 1 Oct 2014 15:02:14 +0500 Subject: [PATCH 27/43] Error message typos fix r Please enter the commit message for your changes. Lines starting --- mongo_fdw.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mongo_fdw.c b/mongo_fdw.c index 6485559..997b27e 100644 --- a/mongo_fdw.c +++ b/mongo_fdw.c @@ -776,10 +776,10 @@ MongoExecForeignInsert(EState *estate, /* first column of MongoDB's foreign table must be _id */ if (strcmp(slot->tts_tupleDescriptor->attrs[0]->attname.data, "_id") != 0) - elog(ERROR, "first colum of MongoDB's foreign table must be \"_id\""); + elog(ERROR, "first column of MongoDB's foreign table must be \"_id\""); if (typoid != NAMEOID) - elog(ERROR, "type of first colum of MongoDB's foreign table must be \"name\""); + elog(ERROR, "type of first column of MongoDB's foreign table must be \"NAME\""); if (attnum == 1) { From 81673981612e66df97f3e3d5811fea131789abf0 Mon Sep 17 00:00:00 2001 From: Ibrar Ahmed Date: Wed, 1 Oct 2014 15:04:30 +0500 Subject: [PATCH 28/43] README typo fix --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2042ad9..b5e413b 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ compiled and linked. `*Note that we have verified the `mongo_fdw` extension only on MacOS X, Fedora and Ubuntu systems. If you run into issues on other systems, please [let us know][3]*` -Enhancments +Enhancements ----------- The following enhancements are added to the latest version of mongo_fdw From a4b043c0fedcdc294becfbe1a644b27c7fd54733 Mon Sep 17 00:00:00 2001 From: Ibrar Ahmed Date: Wed, 1 Oct 2014 15:18:21 +0500 Subject: [PATCH 29/43] Function name change in connection.c and option. c file to avoid conflict with other libraries. --- connection.c | 22 +++++++++++++--------- mongo_fdw.c | 38 +++++++++++++++++++------------------- mongo_fdw.h | 16 ++++++++-------- option.c | 42 ++++++++++++++++-------------------------- 4 files changed, 56 insertions(+), 62 deletions(-) diff --git a/connection.c b/connection.c index 94c8e18..6a9464e 100644 --- a/connection.c +++ b/connection.c @@ -1,12 +1,16 @@ /*------------------------------------------------------------------------- * * connection.c - * Connection management functions for mongo_fdw + * Foreign-data wrapper for remote MongoDB servers * - * Portions Copyright © 2004-2014, EnterpriseDB Corporation. + * Portions Copyright (c) 2012-2014, PostgreSQL Global Development Group * - * Portions Copyright © 2012–2014 Citus Data, Inc. + * Portions Copyright (c) 2004-2014, EnterpriseDB Corporation. * + * Portions Copyright (c) 2012–2014 Citus Data, Inc. + * + * IDENTIFICATION + * connection.c * *------------------------------------------------------------------------- */ @@ -54,13 +58,13 @@ typedef struct ConnCacheEntry static HTAB *ConnectionHash = NULL; /* - * GetConnection: + * mongo_get_connection: * Get a mong connection which can be used to execute queries on * the remote Mongo server with the user's authorization. A new connection * is established if we don't already have a suitable one. */ MONGO_CONN* -GetConnection(char *host, int32 port, char *databaseName, char *user, char *password) +mongo_get_connection(char *host, int32 port, char *databaseName, char *user, char *password) { bool found; ConnCacheEntry *entry; @@ -106,11 +110,11 @@ GetConnection(char *host, int32 port, char *databaseName, char *user, char *pass } /* - * cleanup_connection: + * mongo_cleanup_connection: * Delete all the cache entries on backend exists. */ void -cleanup_connection() +mongo_cleanup_connection() { HASH_SEQ_STATUS scan; ConnCacheEntry *entry; @@ -131,10 +135,10 @@ cleanup_connection() } /* - * Release connection created by calling GetConnection. + * Release connection created by calling mongo_get_connection. */ void -ReleaseConnection(MONGO_CONN *conn) +mongo_release_connection(MONGO_CONN *conn) { /* * We don't close the connection indvisually here, will do all connection diff --git a/mongo_fdw.c b/mongo_fdw.c index 997b27e..80858b8 100644 --- a/mongo_fdw.c +++ b/mongo_fdw.c @@ -208,7 +208,7 @@ mongo_fdw_handler(PG_FUNCTION_ARGS) static void mongo_fdw_exit(int code, Datum arg) { - cleanup_connection(); + mongo_cleanup_connection(); } @@ -385,14 +385,14 @@ MongoExplainForeignScan(ForeignScanState *scanState, ExplainState *explainState) Oid foreignTableId = InvalidOid; foreignTableId = RelationGetRelid(scanState->ss.ss_currentRelation); - mongoFdwOptions = MongoGetOptions(foreignTableId); + mongoFdwOptions = mongo_get_options(foreignTableId); /* construct fully qualified collection name */ namespaceName = makeStringInfo(); appendStringInfo(namespaceName, "%s.%s", mongoFdwOptions->databaseName, mongoFdwOptions->collectionName); - MongoFreeOptions(mongoFdwOptions); + mongo_free_options(mongoFdwOptions); ExplainPropertyText("Foreign Namespace", namespaceName->data, explainState); } @@ -409,14 +409,14 @@ MongoExplainForeignModify(ModifyTableState *mtstate, Oid foreignTableId = InvalidOid; foreignTableId = RelationGetRelid(rinfo->ri_RelationDesc); - mongoFdwOptions = MongoGetOptions(foreignTableId); + mongoFdwOptions = mongo_get_options(foreignTableId); /* construct fully qualified collection name */ namespaceName = makeStringInfo(); appendStringInfo(namespaceName, "%s.%s", mongoFdwOptions->databaseName, mongoFdwOptions->collectionName); - MongoFreeOptions(mongoFdwOptions); + mongo_free_options(mongoFdwOptions); ExplainPropertyText("Foreign Namespace", namespaceName->data, es); } @@ -452,7 +452,7 @@ MongoBeginForeignScan(ForeignScanState *scanState, int executorFlags) return; foreignTableId = RelationGetRelid(scanState->ss.ss_currentRelation); - mongoFdwOptions = MongoGetOptions(foreignTableId); + mongoFdwOptions = mongo_get_options(foreignTableId); /* resolve hostname and port number; and connect to mongo server */ addressName = mongoFdwOptions->addressName; @@ -465,7 +465,7 @@ MongoBeginForeignScan(ForeignScanState *scanState, int executorFlags) * Get connection to the foreign server. Connection manager will * establish new connection if necessary. */ - mongoConnection = GetConnection(addressName, portNumber, databaseName, username, password); + mongoConnection = mongo_get_connection(addressName, portNumber, databaseName, username, password); foreignScan = (ForeignScan *) scanState->ss.ps.plan; foreignPrivateList = foreignScan->fdw_private; @@ -572,7 +572,7 @@ MongoEndForeignScan(ForeignScanState *scanState) { if (fmstate->mongoFdwOptions) { - MongoFreeOptions(fmstate->mongoFdwOptions); + mongo_free_options(fmstate->mongoFdwOptions); fmstate->mongoFdwOptions = NULL; } MongoFreeScanState(fmstate); @@ -598,14 +598,14 @@ MongoReScanForeignScan(ForeignScanState *scanState) /* reconstruct full collection name */ foreignTableId = RelationGetRelid(scanState->ss.ss_currentRelation); - mongoFdwOptions = MongoGetOptions(foreignTableId); + mongoFdwOptions = mongo_get_options(foreignTableId); /* reconstruct cursor for collection name and set query */ fmstate->mongoCursor = MongoCursorCreate(mongoConnection, fmstate->mongoFdwOptions->databaseName, fmstate->mongoFdwOptions->collectionName, fmstate->queryDocument); - MongoFreeOptions(mongoFdwOptions); + mongo_free_options(mongoFdwOptions); } static List * @@ -707,7 +707,7 @@ MongoBeginForeignModify(ModifyTableState *mtstate, fmstate = (MongoFdwModifyState *) palloc0(sizeof(MongoFdwModifyState)); fmstate->rel = rel; - fmstate->mongoFdwOptions = MongoGetOptions(foreignTableId); + fmstate->mongoFdwOptions = mongo_get_options(foreignTableId); fmstate->target_attrs = (List *) list_nth(fdw_private, 0); @@ -758,7 +758,7 @@ MongoExecForeignInsert(EState *estate, /* resolve foreign table options; and connect to mongo server */ options = fmstate->mongoFdwOptions; - mongoConnection = GetConnection(options->addressName, options->portNumber, options->databaseName, options->username, options->password); + mongoConnection = mongo_get_connection(options->addressName, options->portNumber, options->databaseName, options->username, options->password); b = BsonCreate(); @@ -871,7 +871,7 @@ MongoExecForeignUpdate(EState *estate, /* resolve foreign table options; and connect to mongo server */ options = fmstate->mongoFdwOptions; - mongoConnection = GetConnection(options->addressName, options->portNumber, options->databaseName, options->username, options->password); + mongoConnection = mongo_get_connection(options->addressName, options->portNumber, options->databaseName, options->username, options->password); /* Get the id that was passed up as a resjunk column */ datum = ExecGetJunkAttribute(planSlot, 1, &isNull); @@ -954,7 +954,7 @@ MongoExecForeignDelete(EState *estate, /* resolve foreign table options; and connect to mongo server */ options = fmstate->mongoFdwOptions; - mongoConnection = GetConnection(options->addressName, options->portNumber, options->databaseName, options->username, options->password); + mongoConnection = mongo_get_connection(options->addressName, options->portNumber, options->databaseName, options->username, options->password); /* Get the id that was passed up as a resjunk column */ datum = ExecGetJunkAttribute(planSlot, 1, &isNull); @@ -992,7 +992,7 @@ MongoEndForeignModify(EState *estate, ResultRelInfo *resultRelInfo) { if (fmstate->mongoFdwOptions) { - MongoFreeOptions(fmstate->mongoFdwOptions); + mongo_free_options(fmstate->mongoFdwOptions); fmstate->mongoFdwOptions = NULL; } MongoFreeScanState(fmstate); @@ -1014,13 +1014,13 @@ ForeignTableDocumentCount(Oid foreignTableId) double documentCount = 0.0; /* resolve foreign table options; and connect to mongo server */ - options = MongoGetOptions(foreignTableId); + options = mongo_get_options(foreignTableId); - mongoConnection = GetConnection(options->addressName, options->portNumber, options->databaseName, options->username, options->password); + mongoConnection = mongo_get_connection(options->addressName, options->portNumber, options->databaseName, options->username, options->password); documentCount = MongoAggregateCount(mongoConnection, options->databaseName, options->collectionName, emptyQuery); - MongoFreeOptions(options); + mongo_free_options(options); return documentCount; } @@ -1693,7 +1693,7 @@ MongoFreeScanState(MongoFdwModifyState *fmstate) } /* Release remote connection */ - ReleaseConnection(fmstate->mongoConnection); + mongo_release_connection(fmstate->mongoConnection); } diff --git a/mongo_fdw.h b/mongo_fdw.h index c040fd0..8d38079 100644 --- a/mongo_fdw.h +++ b/mongo_fdw.h @@ -195,21 +195,21 @@ typedef struct ColumnMapping Oid columnArrayTypeId; } ColumnMapping; +/* options.c */ +extern MongoFdwOptions * mongo_get_options(Oid foreignTableId); +extern void mongo_free_options(MongoFdwOptions *mongoFdwOptions); +extern StringInfo mongo_option_names_string(Oid currentContextId); -extern MONGO_CONN *GetConnection(char *host, int32 port, char *databaneName, char *user, char *password); -extern void cleanup_connection(void); -extern void ReleaseConnection(MONGO_CONN* conn); - -extern StringInfo OptionNamesString(Oid currentContextId); +/* connection.c */ +extern MONGO_CONN *mongo_get_connection(char *host, int32 port, char *databaneName, char *user, char *password); +extern void mongo_cleanup_connection(void); +extern void mongo_release_connection(MONGO_CONN* conn); /* Function declarations related to creating the mongo query */ extern List * ApplicableOpExpressionList(RelOptInfo *baserel); extern BSON * QueryDocument(Oid relationId, List *opExpressionList); extern List * ColumnList(RelOptInfo *baserel); -extern MongoFdwOptions * MongoGetOptions(Oid foreignTableId); -extern void MongoFreeOptions(MongoFdwOptions *mongoFdwOptions); - /* Function declarations for foreign data wrapper */ extern Datum mongo_fdw_handler(PG_FUNCTION_ARGS); extern Datum mongo_fdw_validator(PG_FUNCTION_ARGS); diff --git a/option.c b/option.c index bbd3ed0..eb8991a 100644 --- a/option.c +++ b/option.c @@ -41,7 +41,7 @@ #include "utils/memutils.h" #include "miscadmin.h" -static char * MongoGetOptionValue(Oid foreignTableId, const char *optionName); +static char * mongo_get_option_value(Oid foreignTableId, const char *optionName); /* * Validate the generic options given to a FOREIGN DATA WRAPPER, SERVER, @@ -88,7 +88,7 @@ mongo_fdw_validator(PG_FUNCTION_ARGS) /* if invalid option, display an informative error message */ if (!optionValid) { - StringInfo optionNamesString = OptionNamesString(optionContextId); + StringInfo optionNamesString = mongo_option_names_string(optionContextId); ereport(ERROR, (errcode(ERRCODE_FDW_INVALID_OPTION_NAME), errmsg("invalid option \"%s\"", optionName), @@ -108,11 +108,11 @@ mongo_fdw_validator(PG_FUNCTION_ARGS) } /* - * OptionNamesString finds all options that are valid for the current context, + * mongo_option_names_string finds all options that are valid for the current context, * and concatenates these option names in a comma separated string. */ StringInfo -OptionNamesString(Oid currentContextId) +mongo_option_names_string(Oid currentContextId) { StringInfo optionNamesString = makeStringInfo(); bool firstOptionPrinted = false; @@ -137,12 +137,12 @@ OptionNamesString(Oid currentContextId) /* - * MongoGetOptions returns the option values to be used when connecting to and + * mongo_get_options returns the option values to be used when connecting to and * querying MongoDB. To resolve these values, the function checks the foreign * table's options, and if not present, falls back to default values. */ MongoFdwOptions * -MongoGetOptions(Oid foreignTableId) +mongo_get_options(Oid foreignTableId) { MongoFdwOptions *mongoFdwOptions = NULL; char *addressName = NULL; @@ -153,35 +153,26 @@ MongoGetOptions(Oid foreignTableId) char *username= NULL; char *password= NULL; - addressName = MongoGetOptionValue(foreignTableId, OPTION_NAME_ADDRESS); + addressName = mongo_get_option_value(foreignTableId, OPTION_NAME_ADDRESS); if (addressName == NULL) - { addressName = pstrdup(DEFAULT_IP_ADDRESS); - } - portName = MongoGetOptionValue(foreignTableId, OPTION_NAME_PORT); + portName = mongo_get_option_value(foreignTableId, OPTION_NAME_PORT); if (portName == NULL) - { portNumber = DEFAULT_PORT_NUMBER; - } else - { portNumber = pg_atoi(portName, sizeof(int32), 0); - } - databaseName = MongoGetOptionValue(foreignTableId, OPTION_NAME_DATABASE); + databaseName = mongo_get_option_value(foreignTableId, OPTION_NAME_DATABASE); if (databaseName == NULL) - { databaseName = pstrdup(DEFAULT_DATABASE_NAME); - } - collectionName = MongoGetOptionValue(foreignTableId, OPTION_NAME_COLLECTION); + collectionName = mongo_get_option_value(foreignTableId, OPTION_NAME_COLLECTION); if (collectionName == NULL) - { collectionName = get_rel_name(foreignTableId); - } - username = MongoGetOptionValue(foreignTableId, OPTION_NAME_USERNAME); - password = MongoGetOptionValue(foreignTableId, OPTION_NAME_PASSWORD); + + username = mongo_get_option_value(foreignTableId, OPTION_NAME_USERNAME); + password = mongo_get_option_value(foreignTableId, OPTION_NAME_PASSWORD); mongoFdwOptions = (MongoFdwOptions *) palloc0(sizeof(MongoFdwOptions)); mongoFdwOptions->addressName = addressName; @@ -196,7 +187,7 @@ MongoGetOptions(Oid foreignTableId) void -MongoFreeOptions(MongoFdwOptions *mongoFdwOptions) +mongo_free_options(MongoFdwOptions *mongoFdwOptions) { if (mongoFdwOptions) { @@ -207,12 +198,12 @@ MongoFreeOptions(MongoFdwOptions *mongoFdwOptions) } /* - * MongoGetOptionValue walks over foreign table and foreign server options, and + * mongo_get_option_value walks over foreign table and foreign server options, and * looks for the option with the given name. If found, the function returns the * option's value. */ static char * -MongoGetOptionValue(Oid foreignTableId, const char *optionName) +mongo_get_option_value(Oid foreignTableId, const char *optionName) { ForeignTable *foreignTable = NULL; ForeignServer *foreignServer = NULL; @@ -240,7 +231,6 @@ MongoGetOptionValue(Oid foreignTableId, const char *optionName) break; } } - return optionValue; } From da603d525b22fd4598975127ab2790b70d8321a3 Mon Sep 17 00:00:00 2001 From: Ibrar Ahmed Date: Wed, 1 Oct 2014 15:28:05 +0500 Subject: [PATCH 30/43] README Changes --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b5e413b..cec383d 100644 --- a/README.md +++ b/README.md @@ -19,9 +19,10 @@ not up-to-date, run git submodule update --init. When you type `make`, the C driver's source code also gets automatically compiled and linked. -`*Note: Make sure you have permission to "/usr/local" (default installation location) folder.*` +Note: Make sure you have permission to "/usr/local" (default installation location) folder. -`*Note that we have verified the `mongo_fdw` extension only on MacOS X, Fedora and Ubuntu systems. If you run into issues on other systems, please [let us know][3]*` +Note that we have verified the `mongo_fdw` extension only on MacOS X, +Fedora and Ubuntu systems. If you run into issues on other systems, please [let us know][3] Enhancements ----------- From 8e6ca32eb91e9169c6013d51c805f6e6d9985be3 Mon Sep 17 00:00:00 2001 From: TruongSinh Tran-Nguyen Date: Thu, 2 Oct 2014 14:48:41 +0300 Subject: [PATCH 31/43] read/write binary data (bytea) --- mongo_fdw.c | 18 ++++++++++++++++++ mongo_fdw.h | 1 + mongo_query.c | 15 +++++++++++++++ mongo_wrapper.c | 13 +++++++++++++ mongo_wrapper.h | 3 +++ 5 files changed, 50 insertions(+) diff --git a/mongo_fdw.c b/mongo_fdw.c index 80858b8..a4cc8c3 100644 --- a/mongo_fdw.c +++ b/mongo_fdw.c @@ -1227,6 +1227,14 @@ ColumnTypesCompatible(BSON_TYPE bsonType, Oid columnTypeId) } break; } + case BYTEAOID: + { + if (bsonType == BSON_TYPE_BINDATA) + { + compatibleTypes = true; + } + break; + } case NAMEOID: { /* @@ -1432,6 +1440,16 @@ ColumnValue(BSON_ITERATOR *bsonIterator, Oid columnTypeId, int32 columnTypeMod) Int32GetDatum(columnTypeMod)); break; } + case BYTEAOID: + { + int value_len = BsonIterBinLen(bsonIterator); + const char *value = BsonIterBinData(bsonIterator); + bytea *result = (bytea *)palloc(value_len + VARHDRSZ); + memcpy(VARDATA(result), value, value_len); + SET_VARSIZE(result, value_len + VARHDRSZ); + columnValue = PointerGetDatum(result); + break; + } case DATEOID: { int64 valueMillis = BsonIterDate(bsonIterator); diff --git a/mongo_fdw.h b/mongo_fdw.h index 8d38079..9a6eb99 100644 --- a/mongo_fdw.h +++ b/mongo_fdw.h @@ -75,6 +75,7 @@ #define BSON_TYPE_INT32 BSON_INT #define BSON_TYPE_INT64 BSON_LONG #define BSON_TYPE_DOUBLE BSON_DOUBLE + #define BSON_TYPE_BINDATA BSON_BINDATA #define BSON_TYPE_BOOL BSON_BOOL #define BSON_TYPE_UTF8 BSON_STRING #define BSON_TYPE_OID BSON_OID diff --git a/mongo_query.c b/mongo_query.c index d8356e5..77945b8 100644 --- a/mongo_query.c +++ b/mongo_query.c @@ -449,6 +449,21 @@ AppenMongoValue(BSON *queryDocument, const char *keyName, Datum value, bool isnu status = BsonAppendUTF8(queryDocument, keyName, outputString); break; } + case BYTEAOID: + { + int len; + char *data; + char *result = DatumGetPointer(value); + if (VARATT_IS_1B(result)) { + len = VARSIZE_1B(result) - VARHDRSZ_SHORT; + data = VARDATA_1B(result); + } else { + len = VARSIZE_4B(result) - VARHDRSZ; + data = VARDATA_4B(result); + } + status = BsonAppendBinary(queryDocument, keyName, data, len); + break; + } case NAMEOID: { char *outputString = NULL; diff --git a/mongo_wrapper.c b/mongo_wrapper.c index 99dadd5..cbe0b2a 100644 --- a/mongo_wrapper.c +++ b/mongo_wrapper.c @@ -202,6 +202,15 @@ BsonIterString(BSON_ITERATOR *it) return bson_iterator_string(it); } +const char* BsonIterBinData(BSON_ITERATOR *it) +{ + return bson_iterator_bin_data(it); +} + +int BsonIterBinLen(BSON_ITERATOR *it) +{ + return bson_iterator_bin_len(it); +} const bson_oid_t * BsonIterOid(BSON_ITERATOR *it) { @@ -294,6 +303,10 @@ BsonAppendUTF8(BSON *b, const char* key, char *v) return (bson_append_string(b, key, v) == MONGO_OK); } +bool BsonAppendBinary(BSON *b, const char* key, char *v, size_t len) +{ + return (bson_append_binary(b, key, BSON_BIN_BINARY, v, len) == MONGO_OK); +} bool BsonAppendDate(BSON *b, const char* key, time_t v) { diff --git a/mongo_wrapper.h b/mongo_wrapper.h index f3a26d8..b08640e 100644 --- a/mongo_wrapper.h +++ b/mongo_wrapper.h @@ -47,6 +47,8 @@ int64_t BsonIterInt64(BSON_ITERATOR *it); double BsonIterDouble(BSON_ITERATOR *it); bool BsonIterBool(BSON_ITERATOR *it); const char* BsonIterString(BSON_ITERATOR *it); +const char* BsonIterBinData(BSON_ITERATOR *it); +int BsonIterBinLen(BSON_ITERATOR *it); const bson_oid_t * BsonIterOid(BSON_ITERATOR *it); time_t BsonIterDate(BSON_ITERATOR *it); const char* BsonIterKey(BSON_ITERATOR *it); @@ -63,6 +65,7 @@ bool BsonAppendInt32(BSON *b, const char* key, int v); bool BsonAppendInt64(BSON *b, const char* key, int64_t v); bool BsonAppendDouble(BSON *b, const char* key, double v); bool BsonAppendUTF8(BSON *b, const char* key, char *v); +bool BsonAppendBinary(BSON *b, const char* key, char *v, size_t len); bool BsonAppendDate(BSON *b, const char* key, time_t v); bool BsonAppendStartArray(BSON *b, const char* key, BSON* c); bool BsonAppendFinishArray(BSON *b, BSON *c); From 7a3d32e8f6374ec712ba24939290208e5874bf72 Mon Sep 17 00:00:00 2001 From: TruongSinh Tran-Nguyen Date: Tue, 28 Oct 2014 11:43:18 +0200 Subject: [PATCH 32/43] support write text[] (TEXTARRAYOID) --- mongo_query.c | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/mongo_query.c b/mongo_query.c index 77945b8..c9b43b0 100644 --- a/mongo_query.c +++ b/mongo_query.c @@ -537,6 +537,42 @@ AppenMongoValue(BSON *queryDocument, const char *keyName, Datum value, bool isnu pfree(elem_nulls); break; } + case TEXTARRAYOID: + { + ArrayType *array; + Oid elmtype; + int16 elmlen; + bool elmbyval; + char elmalign; + int num_elems; + Datum *elem_values; + bool *elem_nulls; + int i; + BSON t; + + array = DatumGetArrayTypeP(value); + elmtype = ARR_ELEMTYPE(array); + get_typlenbyvalalign(elmtype, &elmlen, &elmbyval, &elmalign); + + deconstruct_array(array, elmtype, elmlen, elmbyval, elmalign, &elem_values, &elem_nulls, &num_elems); + + BsonAppendStartArray(queryDocument, keyName, &t); + for (i = 0; i < num_elems; i++) + { + if (elem_nulls[i]) + continue; + char *valueString = NULL; + Oid outputFunctionId = InvalidOid; + bool typeVarLength = false; + getTypeOutputInfo(TEXTOID, &outputFunctionId, &typeVarLength); + valueString = OidOutputFunctionCall(outputFunctionId, elem_values[i]); + status = BsonAppendUTF8(queryDocument, keyName, valueString); + } + BsonAppendFinishArray(queryDocument, &t); + pfree(elem_values); + pfree(elem_nulls); + break; + } default: { /* From c25671b625dab8a25008b426ca3407a2af31b293 Mon Sep 17 00:00:00 2001 From: Ibrar Ahmed Date: Thu, 30 Oct 2014 22:25:51 +0500 Subject: [PATCH 33/43] Error #7: Fixed compiler warning --- mongo_fdw.c | 2 +- mongo_query.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mongo_fdw.c b/mongo_fdw.c index a4cc8c3..6844abf 100644 --- a/mongo_fdw.c +++ b/mongo_fdw.c @@ -1618,7 +1618,7 @@ DumpJson(StringInfo output, const char *bsonData, bool isArray) { appendStringInfo(output, "%d", bson_iterator_int(&i)); break; case BSON_LONG: - appendStringInfo(output, "%ld", (uint64_t)bson_iterator_long(&i)); + appendStringInfo(output, "%lld", (uint64_t)bson_iterator_long(&i)); break; case BSON_TIMESTAMP: ereport(ERROR, (errcode(ERRCODE_FDW_INVALID_DATA_TYPE), diff --git a/mongo_query.c b/mongo_query.c index c9b43b0..c041b30 100644 --- a/mongo_query.c +++ b/mongo_query.c @@ -559,11 +559,11 @@ AppenMongoValue(BSON *queryDocument, const char *keyName, Datum value, bool isnu BsonAppendStartArray(queryDocument, keyName, &t); for (i = 0; i < num_elems; i++) { - if (elem_nulls[i]) - continue; char *valueString = NULL; Oid outputFunctionId = InvalidOid; bool typeVarLength = false; + if (elem_nulls[i]) + continue; getTypeOutputInfo(TEXTOID, &outputFunctionId, &typeVarLength); valueString = OidOutputFunctionCall(outputFunctionId, elem_values[i]); status = BsonAppendUTF8(queryDocument, keyName, valueString); From 579452745bed9da9945fe969c3e9b60dfbc80e53 Mon Sep 17 00:00:00 2001 From: Ibrar Ahmed Date: Fri, 31 Oct 2014 00:20:51 +0500 Subject: [PATCH 34/43] Error #8: Using serverid and userid for connection pooling instead of hot and port. --- connection.c | 23 ++++----- mongo_fdw.c | 121 +++++++++++++++++++++++++++++++++++++----------- mongo_fdw.h | 12 ++--- mongo_wrapper.c | 4 +- option.c | 31 +++++++------ 5 files changed, 128 insertions(+), 63 deletions(-) diff --git a/connection.c b/connection.c index 6a9464e..83560c2 100644 --- a/connection.c +++ b/connection.c @@ -34,16 +34,14 @@ /* * Connection cache hash table entry * - * The lookup key in this hash table is the foreign Mongo server name / IP - * and the server port number. (We use just one connection per user per foreign server, + * The lookup key in this hash table is the foreign server OID plus the user + * mapping OID. (We use just one connection per user per foreign server, * so that we can ensure all scans use the same snapshot during a query.) - * - * The "conn" pointer can be NULL if we don't currently have a live connection. */ typedef struct ConnCacheKey { - char host[HOST_LEN]; /* MongoDB's host name / IP address */ - int32 port; /* MongoDB's port number */ + Oid serverid; /* OID of foreign server */ + Oid userid; /* OID of local user whose mapping we use */ } ConnCacheKey; typedef struct ConnCacheEntry @@ -64,7 +62,7 @@ static HTAB *ConnectionHash = NULL; * is established if we don't already have a suitable one. */ MONGO_CONN* -mongo_get_connection(char *host, int32 port, char *databaseName, char *user, char *password) +mongo_get_connection(ForeignServer *server, UserMapping *user, MongoFdwOptions *opt) { bool found; ConnCacheEntry *entry; @@ -85,10 +83,9 @@ mongo_get_connection(char *host, int32 port, char *databaseName, char *user, cha HASH_ELEM | HASH_FUNCTION | HASH_CONTEXT); } - /* Create hash key for the entry */ - memset(key.host, 0, HOST_LEN); - strncpy(key.host, host, HOST_LEN); - key.port = port; + /* Create hash key for the entry. Assume no pad bytes in key struct */ + key.serverid = server->serverid; + key.userid = user->userid; /* * Find or create cached entry for requested connection. @@ -101,9 +98,9 @@ mongo_get_connection(char *host, int32 port, char *databaseName, char *user, cha } if (entry->conn == NULL) { - entry->conn = MongoConnect(host, port, databaseName, user, password); + entry->conn = MongoConnect(opt->svr_address, opt->svr_port, opt->svr_database, opt->svr_username, opt->svr_password); elog(DEBUG3, "new mongo_fdw connection %p for server \"%s:%d\"", - entry->conn, host, port); + entry->conn, opt->svr_address, opt->svr_port); } return entry->conn; diff --git a/mongo_fdw.c b/mongo_fdw.c index 6844abf..d50c71e 100644 --- a/mongo_fdw.c +++ b/mongo_fdw.c @@ -389,7 +389,7 @@ MongoExplainForeignScan(ForeignScanState *scanState, ExplainState *explainState) /* construct fully qualified collection name */ namespaceName = makeStringInfo(); - appendStringInfo(namespaceName, "%s.%s", mongoFdwOptions->databaseName, + appendStringInfo(namespaceName, "%s.%s", mongoFdwOptions->svr_database, mongoFdwOptions->collectionName); mongo_free_options(mongoFdwOptions); @@ -413,7 +413,7 @@ MongoExplainForeignModify(ModifyTableState *mtstate, /* construct fully qualified collection name */ namespaceName = makeStringInfo(); - appendStringInfo(namespaceName, "%s.%s", mongoFdwOptions->databaseName, + appendStringInfo(namespaceName, "%s.%s", mongoFdwOptions->svr_database, mongoFdwOptions->collectionName); mongo_free_options(mongoFdwOptions); @@ -435,17 +435,21 @@ MongoBeginForeignScan(ForeignScanState *scanState, int executorFlags) Oid foreignTableId = InvalidOid; List *columnList = NIL; HTAB *columnMappingHash = NULL; - char *addressName = NULL; - int32 portNumber = 0; - char *databaseName= NULL; - char *username= NULL; - char *password= NULL; ForeignScan *foreignScan = NULL; List *foreignPrivateList = NIL; BSON *queryDocument = NULL; MongoFdwOptions *mongoFdwOptions = NULL; MongoFdwModifyState *fmstate = NULL; List *opExpressionList = NIL; + RangeTblEntry *rte; + EState *estate = scanState->ss.ps.state; + ForeignScan *fsplan = (ForeignScan *) scanState->ss.ps.plan; + + Oid userid; + ForeignServer *server; + UserMapping *user; + ForeignTable *table; + /* if Explain with no Analyze, do nothing */ if (executorFlags & EXEC_FLAG_EXPLAIN_ONLY) @@ -454,18 +458,25 @@ MongoBeginForeignScan(ForeignScanState *scanState, int executorFlags) foreignTableId = RelationGetRelid(scanState->ss.ss_currentRelation); mongoFdwOptions = mongo_get_options(foreignTableId); - /* resolve hostname and port number; and connect to mongo server */ - addressName = mongoFdwOptions->addressName; - portNumber = mongoFdwOptions->portNumber; - databaseName = mongoFdwOptions->databaseName; - username = mongoFdwOptions->username; - password = mongoFdwOptions->password; + fmstate = (MongoFdwModifyState *) palloc0(sizeof(MongoFdwModifyState)); + /* + * Identify which user to do the remote access as. This should match what + * ExecCheckRTEPerms() does. + */ + rte = rt_fetch(fsplan->scan.scanrelid, estate->es_range_table); + userid = rte->checkAsUser ? rte->checkAsUser : GetUserId(); + + /* Get info about foreign table. */ + fmstate->rel = scanState->ss.ss_currentRelation; + table = GetForeignTable(RelationGetRelid(fmstate->rel)); + server = GetForeignServer(table->serverid); + user = GetUserMapping(userid, server->serverid); /* * Get connection to the foreign server. Connection manager will * establish new connection if necessary. */ - mongoConnection = mongo_get_connection(addressName, portNumber, databaseName, username, password); + mongoConnection = mongo_get_connection(server, user, mongoFdwOptions); foreignScan = (ForeignScan *) scanState->ss.ps.plan; foreignPrivateList = foreignScan->fdw_private; @@ -479,10 +490,9 @@ MongoBeginForeignScan(ForeignScanState *scanState, int executorFlags) columnMappingHash = ColumnMappingHash(foreignTableId, columnList); /* create cursor for collection name and set query */ - mongoCursor = MongoCursorCreate(mongoConnection, mongoFdwOptions->databaseName, mongoFdwOptions->collectionName, queryDocument); + mongoCursor = MongoCursorCreate(mongoConnection, mongoFdwOptions->svr_database, mongoFdwOptions->collectionName, queryDocument); /* create and set foreign execution state */ - fmstate = (MongoFdwModifyState *) palloc0(sizeof(MongoFdwModifyState)); fmstate->columnMappingHash = columnMappingHash; fmstate->mongoConnection = mongoConnection; fmstate->mongoCursor = mongoCursor; @@ -602,7 +612,7 @@ MongoReScanForeignScan(ForeignScanState *scanState) /* reconstruct cursor for collection name and set query */ fmstate->mongoCursor = MongoCursorCreate(mongoConnection, - fmstate->mongoFdwOptions->databaseName, + fmstate->mongoFdwOptions->svr_database, fmstate->mongoFdwOptions->collectionName, fmstate->queryDocument); mongo_free_options(mongoFdwOptions); @@ -750,15 +760,28 @@ MongoExecForeignInsert(EState *estate, Datum value; bool isnull = false; + Oid userid; + ForeignServer *server; + UserMapping *user; + ForeignTable *table; MongoFdwModifyState *fmstate = (MongoFdwModifyState *) resultRelInfo->ri_FdwState; foreignTableId = RelationGetRelid(resultRelInfo->ri_RelationDesc); - /* resolve foreign table options; and connect to mongo server */ - options = fmstate->mongoFdwOptions; + userid = GetUserId(); - mongoConnection = mongo_get_connection(options->addressName, options->portNumber, options->databaseName, options->username, options->password); + /* Get info about foreign table. */ + table = GetForeignTable(RelationGetRelid(fmstate->rel)); + server = GetForeignServer(table->serverid); + user = GetUserMapping(userid, server->serverid); + + /* + * Get connection to the foreign server. Connection manager will + * establish new connection if necessary. + */ + options = fmstate->mongoFdwOptions; + mongoConnection = mongo_get_connection(server, user, options); b = BsonCreate(); @@ -798,7 +821,7 @@ MongoExecForeignInsert(EState *estate, BsonFinish(b); /* Now we are ready to insert tuple / document into MongoDB */ - MongoInsert(mongoConnection, options->databaseName, options->collectionName, b); + MongoInsert(mongoConnection, options->svr_database, options->collectionName, b); BsonDestroy(b); @@ -863,6 +886,11 @@ MongoExecForeignUpdate(EState *estate, BSON *b = NULL; BSON *op = NULL; BSON set; + Oid userid = GetUserId(); + ForeignServer *server; + UserMapping *user; + ForeignTable *table; + MongoFdwModifyState *fmstate = (MongoFdwModifyState *) resultRelInfo->ri_FdwState; @@ -871,7 +899,16 @@ MongoExecForeignUpdate(EState *estate, /* resolve foreign table options; and connect to mongo server */ options = fmstate->mongoFdwOptions; - mongoConnection = mongo_get_connection(options->addressName, options->portNumber, options->databaseName, options->username, options->password); + /* Get info about foreign table. */ + table = GetForeignTable(foreignTableId); + server = GetForeignServer(table->serverid); + user = GetUserMapping(userid, server->serverid); + + /* + * Get connection to the foreign server. Connection manager will + * establish new connection if necessary. + */ + mongoConnection = mongo_get_connection(server, user, options); /* Get the id that was passed up as a resjunk column */ datum = ExecGetJunkAttribute(planSlot, 1, &isNull); @@ -919,7 +956,7 @@ MongoExecForeignUpdate(EState *estate, BsonFinish(op); /* We are ready to update the row into MongoDB */ - MongoUpdate(mongoConnection, options->databaseName, options->collectionName, op, b); + MongoUpdate(mongoConnection, options->svr_database, options->collectionName, op, b); BsonDestroy(op); BsonDestroy(b); @@ -947,6 +984,12 @@ MongoExecForeignDelete(EState *estate, Oid typoid = InvalidOid; BSON *b = NULL; + Oid userid = GetUserId(); + ForeignServer *server; + UserMapping *user; + ForeignTable *table; + + MongoFdwModifyState *fmstate = (MongoFdwModifyState *) resultRelInfo->ri_FdwState; foreignTableId = RelationGetRelid(resultRelInfo->ri_RelationDesc); @@ -954,7 +997,16 @@ MongoExecForeignDelete(EState *estate, /* resolve foreign table options; and connect to mongo server */ options = fmstate->mongoFdwOptions; - mongoConnection = mongo_get_connection(options->addressName, options->portNumber, options->databaseName, options->username, options->password); + /* Get info about foreign table. */ + table = GetForeignTable(foreignTableId); + server = GetForeignServer(table->serverid); + user = GetUserMapping(userid, server->serverid); + + /* + * Get connection to the foreign server. Connection manager will + * establish new connection if necessary. + */ + mongoConnection = mongo_get_connection(server, user, options); /* Get the id that was passed up as a resjunk column */ datum = ExecGetJunkAttribute(planSlot, 1, &isNull); @@ -972,7 +1024,7 @@ MongoExecForeignDelete(EState *estate, BsonFinish(b); /* Now we are ready to delete a single document from MongoDB */ - MongoDelete(mongoConnection, options->databaseName, options->collectionName, b); + MongoDelete(mongoConnection, options->svr_database, options->collectionName, b); BsonDestroy(b); @@ -1012,13 +1064,28 @@ ForeignTableDocumentCount(Oid foreignTableId) MONGO_CONN *mongoConnection = NULL; const BSON *emptyQuery = NULL; double documentCount = 0.0; + Oid userid = GetUserId(); + ForeignServer *server; + UserMapping *user; + ForeignTable *table; + + + /* Get info about foreign table. */ + table = GetForeignTable(foreignTableId); + server = GetForeignServer(table->serverid); + user = GetUserMapping(userid, server->serverid); /* resolve foreign table options; and connect to mongo server */ options = mongo_get_options(foreignTableId); - mongoConnection = mongo_get_connection(options->addressName, options->portNumber, options->databaseName, options->username, options->password); + /* + * Get connection to the foreign server. Connection manager will + * establish new connection if necessary. + */ + mongoConnection = mongo_get_connection(server, user, options); + - documentCount = MongoAggregateCount(mongoConnection, options->databaseName, options->collectionName, emptyQuery); + documentCount = MongoAggregateCount(mongoConnection, options->svr_database, options->collectionName, emptyQuery); mongo_free_options(options); diff --git a/mongo_fdw.h b/mongo_fdw.h index 9a6eb99..9960ec3 100644 --- a/mongo_fdw.h +++ b/mongo_fdw.h @@ -143,12 +143,12 @@ static const MongoValidOption ValidOptionArray[] = */ typedef struct MongoFdwOptions { - char *addressName; - int32 portNumber; - char *databaseName; + char *svr_address; + int32 svr_port; + char *svr_database; char *collectionName; - char *username; - char *password; + char *svr_username; + char *svr_password; } MongoFdwOptions; @@ -202,7 +202,7 @@ extern void mongo_free_options(MongoFdwOptions *mongoFdwOptions); extern StringInfo mongo_option_names_string(Oid currentContextId); /* connection.c */ -extern MONGO_CONN *mongo_get_connection(char *host, int32 port, char *databaneName, char *user, char *password); +MONGO_CONN* mongo_get_connection(ForeignServer *server, UserMapping *user, MongoFdwOptions *opt); extern void mongo_cleanup_connection(void); extern void mongo_release_connection(MONGO_CONN* conn); diff --git a/mongo_wrapper.c b/mongo_wrapper.c index cbe0b2a..fe05ef6 100644 --- a/mongo_wrapper.c +++ b/mongo_wrapper.c @@ -47,11 +47,11 @@ MongoConnect(const char* host, const unsigned short port, char* databaseName, ch { if (mongo_cmd_authenticate(conn, databaseName, user, password) != MONGO_OK) { - int err = conn->err; + char *str = pstrdup(conn->errstr); mongo_destroy(conn); mongo_dealloc(conn); ereport(ERROR, (errmsg("could not connect to %s:%d", host, port), - errhint("Mongo driver connection error: %d", err))); + errhint("Mongo driver connection error: %s", str))); } } return conn; diff --git a/option.c b/option.c index eb8991a..b6519ba 100644 --- a/option.c +++ b/option.c @@ -148,10 +148,10 @@ mongo_get_options(Oid foreignTableId) char *addressName = NULL; char *portName = NULL; int32 portNumber = 0; - char *databaseName = NULL; + char *svr_database = NULL; char *collectionName = NULL; - char *username= NULL; - char *password= NULL; + char *svr_username= NULL; + char *svr_password= NULL; addressName = mongo_get_option_value(foreignTableId, OPTION_NAME_ADDRESS); if (addressName == NULL) @@ -163,24 +163,25 @@ mongo_get_options(Oid foreignTableId) else portNumber = pg_atoi(portName, sizeof(int32), 0); - databaseName = mongo_get_option_value(foreignTableId, OPTION_NAME_DATABASE); - if (databaseName == NULL) - databaseName = pstrdup(DEFAULT_DATABASE_NAME); + svr_database = mongo_get_option_value(foreignTableId, OPTION_NAME_DATABASE); + if (svr_database == NULL) + svr_database = pstrdup(DEFAULT_DATABASE_NAME); collectionName = mongo_get_option_value(foreignTableId, OPTION_NAME_COLLECTION); if (collectionName == NULL) collectionName = get_rel_name(foreignTableId); - username = mongo_get_option_value(foreignTableId, OPTION_NAME_USERNAME); - password = mongo_get_option_value(foreignTableId, OPTION_NAME_PASSWORD); + svr_username = mongo_get_option_value(foreignTableId, OPTION_NAME_USERNAME); + svr_password = mongo_get_option_value(foreignTableId, OPTION_NAME_PASSWORD); mongoFdwOptions = (MongoFdwOptions *) palloc0(sizeof(MongoFdwOptions)); - mongoFdwOptions->addressName = addressName; - mongoFdwOptions->portNumber = portNumber; - mongoFdwOptions->databaseName = databaseName; + + mongoFdwOptions->svr_address = addressName; + mongoFdwOptions->svr_port = portNumber; + mongoFdwOptions->svr_database = svr_database; mongoFdwOptions->collectionName = collectionName; - mongoFdwOptions->username = username; - mongoFdwOptions->password = password; + mongoFdwOptions->svr_username = svr_username; + mongoFdwOptions->svr_password = svr_password; return mongoFdwOptions; } @@ -191,8 +192,8 @@ mongo_free_options(MongoFdwOptions *mongoFdwOptions) { if (mongoFdwOptions) { - pfree(mongoFdwOptions->addressName); - pfree(mongoFdwOptions->databaseName); + pfree(mongoFdwOptions->svr_address); + pfree(mongoFdwOptions->svr_database); pfree(mongoFdwOptions); } } From 005b342173feb489c36e971d9f1abe2f6f823e6e Mon Sep 17 00:00:00 2001 From: Ibrar Ahmed Date: Fri, 31 Oct 2014 00:44:00 +0500 Subject: [PATCH 35/43] Fixed some formating issues. --- connection.c | 6 +- mongo_fdw.c | 409 ++++++++++++++++++++++++++------------------------- option.c | 38 ++--- 3 files changed, 227 insertions(+), 226 deletions(-) diff --git a/connection.c b/connection.c index 83560c2..9276420 100644 --- a/connection.c +++ b/connection.c @@ -64,9 +64,9 @@ static HTAB *ConnectionHash = NULL; MONGO_CONN* mongo_get_connection(ForeignServer *server, UserMapping *user, MongoFdwOptions *opt) { - bool found; - ConnCacheEntry *entry; - ConnCacheKey key; + bool found; + ConnCacheEntry *entry; + ConnCacheKey key; /* First time through, initialize connection cache hashtable */ if (ConnectionHash == NULL) diff --git a/mongo_fdw.c b/mongo_fdw.c index d50c71e..f8c3535 100644 --- a/mongo_fdw.c +++ b/mongo_fdw.c @@ -71,72 +71,72 @@ /* Local functions forward declarations */ static void MongoGetForeignRelSize(PlannerInfo *root, RelOptInfo *baserel, - Oid foreignTableId); + Oid foreignTableId); static void MongoGetForeignPaths(PlannerInfo *root, RelOptInfo *baserel, - Oid foreignTableId); + Oid foreignTableId); static ForeignScan * MongoGetForeignPlan(PlannerInfo *root, RelOptInfo *baserel, - Oid foreignTableId, ForeignPath *bestPath, - List *targetList, List *restrictionClauses); + Oid foreignTableId, ForeignPath *bestPath, + List *targetList, List *restrictionClauses); static void MongoExplainForeignScan(ForeignScanState *scanState, - ExplainState *explainState); + ExplainState *explainState); static void MongoBeginForeignScan(ForeignScanState *scanState, int executorFlags); static TupleTableSlot * MongoIterateForeignScan(ForeignScanState *scanState); static void MongoEndForeignScan(ForeignScanState *scanState); static void MongoReScanForeignScan(ForeignScanState *scanState); static TupleTableSlot *MongoExecForeignUpdate(EState *estate, - ResultRelInfo *resultRelInfo, - TupleTableSlot *slot, - TupleTableSlot *planSlot); + ResultRelInfo *resultRelInfo, + TupleTableSlot *slot, + TupleTableSlot *planSlot); static TupleTableSlot *MongoExecForeignDelete(EState *estate, - ResultRelInfo *resultRelInfo, - TupleTableSlot *slot, - TupleTableSlot *planSlot); + ResultRelInfo *resultRelInfo, + TupleTableSlot *slot, + TupleTableSlot *planSlot); static void MongoEndForeignModify(EState *estate, - ResultRelInfo *resultRelInfo); + ResultRelInfo *resultRelInfo); static void MongoAddForeignUpdateTargets(Query *parsetree, - RangeTblEntry *target_rte, - Relation target_relation); + RangeTblEntry *target_rte, + Relation target_relation); static void MongoBeginForeignModify(ModifyTableState *mtstate, - ResultRelInfo *resultRelInfo, - List *fdw_private, - int subplan_index, - int eflags); + ResultRelInfo *resultRelInfo, + List *fdw_private, + int subplan_index, + int eflags); static TupleTableSlot *MongoExecForeignInsert(EState *estate, - ResultRelInfo *resultRelInfo, - TupleTableSlot *slot, - TupleTableSlot *planSlot); + ResultRelInfo *resultRelInfo, + TupleTableSlot *slot, + TupleTableSlot *planSlot); static List *MongoPlanForeignModify(PlannerInfo *root, - ModifyTable *plan, - Index resultRelation, - int subplan_index); + ModifyTable *plan, + Index resultRelation, + int subplan_index); static void MongoExplainForeignModify(ModifyTableState *mtstate, - ResultRelInfo *rinfo, List *fdw_private, - int subplan_index, ExplainState *es); + ResultRelInfo *rinfo, List *fdw_private, + int subplan_index, ExplainState *es); /* local functions */ static double ForeignTableDocumentCount(Oid foreignTableId); static HTAB * ColumnMappingHash(Oid foreignTableId, List *columnList); static void FillTupleSlot(const BSON *bsonDocument, const char *bsonDocumentKey, - HTAB *columnMappingHash, Datum *columnValues, - bool *columnNulls); + HTAB *columnMappingHash, Datum *columnValues, + bool *columnNulls); static bool ColumnTypesCompatible(BSON_TYPE bsonType, Oid columnTypeId); static Datum ColumnValueArray(BSON_ITERATOR *bsonIterator, Oid valueTypeId); static Datum ColumnValue(BSON_ITERATOR *bsonIterator, Oid columnTypeId, int32 columnTypeMod); static void MongoFreeScanState(MongoFdwModifyState *fmstate); static bool MongoAnalyzeForeignTable(Relation relation, - AcquireSampleRowsFunc *acquireSampleRowsFunc, - BlockNumber *totalPageCount); + AcquireSampleRowsFunc *acquireSampleRowsFunc, + BlockNumber *totalPageCount); static int MongoAcquireSampleRows(Relation relation, int errorLevel, - HeapTuple *sampleRows, int targetRowCount, - double *totalRowCount, double *totalDeadRowCount); + HeapTuple *sampleRows, int targetRowCount, + double *totalRowCount, double *totalDeadRowCount); static void mongo_fdw_exit(int code, Datum arg); extern PGDLLEXPORT void _PG_init(void); @@ -250,22 +250,22 @@ MongoGetForeignRelSize(PlannerInfo *root, RelOptInfo *baserel, Oid foreignTableI static void MongoGetForeignPaths(PlannerInfo *root, RelOptInfo *baserel, Oid foreignTableId) { - double tupleFilterCost = baserel->baserestrictcost.per_tuple; - double inputRowCount = 0.0; - double documentSelectivity = 0.0; - double foreignTableSize = 0; - int32 documentWidth = 0; - BlockNumber pageCount = 0; - double totalDiskAccessCost = 0.0; - double cpuCostPerDoc = 0.0; - double cpuCostPerRow = 0.0; - double totalCpuCost = 0.0; - double connectionCost = 0.0; - double documentCount = 0.0; - List *opExpressionList = NIL; - Cost startupCost = 0.0; - Cost totalCost = 0.0; - Path *foreignPath = NULL; + double tupleFilterCost = baserel->baserestrictcost.per_tuple; + double inputRowCount = 0.0; + double documentSelectivity = 0.0; + double foreignTableSize = 0; + int32 documentWidth = 0; + BlockNumber pageCount = 0; + double totalDiskAccessCost = 0.0; + double cpuCostPerDoc = 0.0; + double cpuCostPerRow = 0.0; + double totalCpuCost = 0.0; + double connectionCost = 0.0; + double documentCount = 0.0; + List *opExpressionList = NIL; + Cost startupCost = 0.0; + Cost totalCost = 0.0; + Path *foreignPath = NULL; documentCount = ForeignTableDocumentCount(foreignTableId); if (documentCount > 0.0) @@ -330,12 +330,12 @@ static ForeignScan * MongoGetForeignPlan(PlannerInfo *root, RelOptInfo *baserel, Oid foreignTableId, ForeignPath *bestPath, List *targetList, List *restrictionClauses) { - Index scanRangeTableIndex = baserel->relid; - ForeignScan *foreignScan = NULL; - List *foreignPrivateList = NIL; - List *opExpressionList = NIL; - BSON *queryDocument = NULL; - List *columnList = NIL; + Index scanRangeTableIndex = baserel->relid; + ForeignScan *foreignScan = NULL; + List *foreignPrivateList = NIL; + List *opExpressionList = NIL; + BSON *queryDocument = NULL; + List *columnList = NIL; /* * We push down applicable restriction clauses to MongoDB, but for simplicity @@ -380,9 +380,9 @@ MongoGetForeignPlan(PlannerInfo *root, RelOptInfo *baserel, Oid foreignTableId, static void MongoExplainForeignScan(ForeignScanState *scanState, ExplainState *explainState) { - MongoFdwOptions *mongoFdwOptions = NULL; - StringInfo namespaceName = NULL; - Oid foreignTableId = InvalidOid; + MongoFdwOptions *mongoFdwOptions = NULL; + StringInfo namespaceName = NULL; + Oid foreignTableId = InvalidOid; foreignTableId = RelationGetRelid(scanState->ss.ss_currentRelation); mongoFdwOptions = mongo_get_options(foreignTableId); @@ -404,9 +404,9 @@ MongoExplainForeignModify(ModifyTableState *mtstate, int subplan_index, ExplainState *es) { - MongoFdwOptions *mongoFdwOptions = NULL; - StringInfo namespaceName = NULL; - Oid foreignTableId = InvalidOid; + MongoFdwOptions *mongoFdwOptions = NULL; + StringInfo namespaceName = NULL; + Oid foreignTableId = InvalidOid; foreignTableId = RelationGetRelid(rinfo->ri_RelationDesc); mongoFdwOptions = mongo_get_options(foreignTableId); @@ -430,25 +430,24 @@ MongoExplainForeignModify(ModifyTableState *mtstate, static void MongoBeginForeignScan(ForeignScanState *scanState, int executorFlags) { - MONGO_CONN *mongoConnection = NULL; - MONGO_CURSOR *mongoCursor = NULL; - Oid foreignTableId = InvalidOid; - List *columnList = NIL; - HTAB *columnMappingHash = NULL; - ForeignScan *foreignScan = NULL; - List *foreignPrivateList = NIL; - BSON *queryDocument = NULL; - MongoFdwOptions *mongoFdwOptions = NULL; - MongoFdwModifyState *fmstate = NULL; - List *opExpressionList = NIL; - RangeTblEntry *rte; - EState *estate = scanState->ss.ps.state; - ForeignScan *fsplan = (ForeignScan *) scanState->ss.ps.plan; - - Oid userid; - ForeignServer *server; - UserMapping *user; - ForeignTable *table; + MONGO_CONN *mongoConnection = NULL; + MONGO_CURSOR *mongoCursor = NULL; + Oid foreignTableId = InvalidOid; + List *columnList = NIL; + HTAB *columnMappingHash = NULL; + ForeignScan *foreignScan = NULL; + List *foreignPrivateList = NIL; + BSON *queryDocument = NULL; + MongoFdwOptions *mongoFdwOptions = NULL; + MongoFdwModifyState *fmstate = NULL; + List *opExpressionList = NIL; + RangeTblEntry *rte; + EState *estate = scanState->ss.ps.state; + ForeignScan *fsplan = (ForeignScan *) scanState->ss.ps.plan; + Oid userid; + ForeignServer *server; + UserMapping *user; + ForeignTable *table; /* if Explain with no Analyze, do nothing */ @@ -512,14 +511,13 @@ static TupleTableSlot * MongoIterateForeignScan(ForeignScanState *scanState) { MongoFdwModifyState *fmstate = (MongoFdwModifyState *) scanState->fdw_state; - TupleTableSlot *tupleSlot = scanState->ss.ss_ScanTupleSlot; - MONGO_CURSOR *mongoCursor = fmstate->mongoCursor; - HTAB *columnMappingHash = fmstate->columnMappingHash; - - TupleDesc tupleDescriptor = tupleSlot->tts_tupleDescriptor; - Datum *columnValues = tupleSlot->tts_values; - bool *columnNulls = tupleSlot->tts_isnull; - int32 columnCount = tupleDescriptor->natts; + TupleTableSlot *tupleSlot = scanState->ss.ss_ScanTupleSlot; + MONGO_CURSOR *mongoCursor = fmstate->mongoCursor; + HTAB *columnMappingHash = fmstate->columnMappingHash; + TupleDesc tupleDescriptor = tupleSlot->tts_tupleDescriptor; + Datum *columnValues = tupleSlot->tts_values; + bool *columnNulls = tupleSlot->tts_isnull; + int32 columnCount = tupleDescriptor->natts; /* * We execute the protocol to load a virtual tuple into a slot. We first @@ -598,10 +596,10 @@ MongoEndForeignScan(ForeignScanState *scanState) static void MongoReScanForeignScan(ForeignScanState *scanState) { - MongoFdwModifyState *fmstate = (MongoFdwModifyState *) scanState->fdw_state; - MONGO_CONN *mongoConnection = fmstate->mongoConnection; - MongoFdwOptions *mongoFdwOptions = NULL; - Oid foreignTableId = InvalidOid; + MongoFdwModifyState *fmstate = (MongoFdwModifyState *) scanState->fdw_state; + MONGO_CONN *mongoConnection = fmstate->mongoConnection; + MongoFdwOptions *mongoFdwOptions = NULL; + Oid foreignTableId = InvalidOid; /* close down the old cursor */ MongoCursorDestroy(fmstate->mongoCursor); @@ -624,10 +622,10 @@ MongoPlanForeignModify(PlannerInfo *root, Index resultRelation, int subplan_index) { - CmdType operation = plan->operation; - RangeTblEntry *rte = planner_rt_fetch(resultRelation, root); - Relation rel; - List* targetAttrs = NIL; + CmdType operation = plan->operation; + RangeTblEntry *rte = planner_rt_fetch(resultRelation, root); + Relation rel; + List *targetAttrs = NIL; /* * Core code already has some lock on each rel being planned, so we can @@ -696,13 +694,13 @@ MongoBeginForeignModify(ModifyTableState *mtstate, int subplan_index, int eflags) { - MongoFdwModifyState *fmstate = NULL; - Relation rel = resultRelInfo->ri_RelationDesc; - AttrNumber n_params = 0; - Oid typefnoid = InvalidOid; - bool isvarlena = false; - ListCell *lc = NULL; - Oid foreignTableId = InvalidOid; + MongoFdwModifyState *fmstate = NULL; + Relation rel = resultRelInfo->ri_RelationDesc; + AttrNumber n_params = 0; + Oid typefnoid = InvalidOid; + bool isvarlena = false; + ListCell *lc = NULL; + Oid foreignTableId = InvalidOid; /* * Do nothing in EXPLAIN (no ANALYZE) case. resultRelInfo->ri_FdwState @@ -752,18 +750,17 @@ MongoExecForeignInsert(EState *estate, TupleTableSlot *slot, TupleTableSlot *planSlot) { - MongoFdwOptions *options = NULL; - MONGO_CONN *mongoConnection = NULL; - Oid foreignTableId = InvalidOid; - BSON *b = NULL; - Oid typoid; - Datum value; - bool isnull = false; - - Oid userid; - ForeignServer *server; - UserMapping *user; - ForeignTable *table; + MongoFdwOptions *options = NULL; + MONGO_CONN *mongoConnection = NULL; + Oid foreignTableId = InvalidOid; + BSON *b = NULL; + Oid typoid; + Datum value; + bool isnull = false; + Oid userid; + ForeignServer *server; + UserMapping *user; + ForeignTable *table; MongoFdwModifyState *fmstate = (MongoFdwModifyState *) resultRelInfo->ri_FdwState; @@ -839,9 +836,9 @@ MongoAddForeignUpdateTargets(Query *parsetree, RangeTblEntry *target_rte, Relation target_relation) { - Var *var = NULL; - const char *attrname = NULL; - TargetEntry *tle = NULL; + Var *var = NULL; + const char *attrname = NULL; + TargetEntry *tle = NULL; /* * What we need is the rowid which is the first column @@ -876,20 +873,20 @@ MongoExecForeignUpdate(EState *estate, TupleTableSlot *slot, TupleTableSlot *planSlot) { - MongoFdwOptions *options = NULL; - MONGO_CONN *mongoConnection = NULL; - Datum datum = 0; - bool isNull = false; - Oid foreignTableId = InvalidOid; - char *columnName = NULL; - Oid typoid = InvalidOid; - BSON *b = NULL; - BSON *op = NULL; - BSON set; - Oid userid = GetUserId(); - ForeignServer *server; - UserMapping *user; - ForeignTable *table; + MongoFdwOptions *options = NULL; + MONGO_CONN *mongoConnection = NULL; + Datum datum = 0; + bool isNull = false; + Oid foreignTableId = InvalidOid; + char *columnName = NULL; + Oid typoid = InvalidOid; + BSON *b = NULL; + BSON *op = NULL; + BSON set; + Oid userid = GetUserId(); + ForeignServer *server; + UserMapping *user; + ForeignTable *table; MongoFdwModifyState *fmstate = (MongoFdwModifyState *) resultRelInfo->ri_FdwState; @@ -975,19 +972,18 @@ MongoExecForeignDelete(EState *estate, TupleTableSlot *slot, TupleTableSlot *planSlot) { - MongoFdwOptions *options = NULL; - MONGO_CONN *mongoConnection = NULL; - Datum datum = 0; - bool isNull = false; - Oid foreignTableId = InvalidOid; - char *columnName = NULL; - Oid typoid = InvalidOid; - BSON *b = NULL; - - Oid userid = GetUserId(); - ForeignServer *server; - UserMapping *user; - ForeignTable *table; + MongoFdwOptions *options = NULL; + MONGO_CONN *mongoConnection = NULL; + Datum datum = 0; + bool isNull = false; + Oid foreignTableId = InvalidOid; + char *columnName = NULL; + Oid typoid = InvalidOid; + BSON *b = NULL; + Oid userid = GetUserId(); + ForeignServer *server; + UserMapping *user; + ForeignTable *table; MongoFdwModifyState *fmstate = (MongoFdwModifyState *) resultRelInfo->ri_FdwState; @@ -1060,14 +1056,14 @@ MongoEndForeignModify(EState *estate, ResultRelInfo *resultRelInfo) static double ForeignTableDocumentCount(Oid foreignTableId) { - MongoFdwOptions *options = NULL; - MONGO_CONN *mongoConnection = NULL; - const BSON *emptyQuery = NULL; - double documentCount = 0.0; - Oid userid = GetUserId(); - ForeignServer *server; - UserMapping *user; - ForeignTable *table; + MongoFdwOptions *options = NULL; + MONGO_CONN *mongoConnection = NULL; + const BSON *emptyQuery = NULL; + double documentCount = 0.0; + Oid userid = GetUserId(); + ForeignServer *server; + UserMapping *user; + ForeignTable *table; /* Get info about foreign table. */ @@ -1101,9 +1097,9 @@ ForeignTableDocumentCount(Oid foreignTableId) static HTAB * ColumnMappingHash(Oid foreignTableId, List *columnList) { - ListCell *columnCell = NULL; - const long hashTableSize = 2048; - HTAB *columnMappingHash = NULL; + ListCell *columnCell = NULL; + const long hashTableSize = 2048; + HTAB *columnMappingHash = NULL; /* create hash table */ HASHCTL hashInfo; @@ -1366,16 +1362,16 @@ ColumnTypesCompatible(BSON_TYPE bsonType, Oid columnTypeId) static Datum ColumnValueArray(BSON_ITERATOR *bsonIterator, Oid valueTypeId) { - Datum *columnValueArray = palloc0(INITIAL_ARRAY_CAPACITY * sizeof(Datum)); - uint32 arrayCapacity = INITIAL_ARRAY_CAPACITY; - uint32 arrayGrowthFactor = 2; - uint32 arrayIndex = 0; + Datum *columnValueArray = palloc0(INITIAL_ARRAY_CAPACITY * sizeof(Datum)); + uint32 arrayCapacity = INITIAL_ARRAY_CAPACITY; + uint32 arrayGrowthFactor = 2; + uint32 arrayIndex = 0; - ArrayType *columnValueObject = NULL; - Datum columnValueDatum = 0; - bool typeByValue = false; - char typeAlignment = 0; - int16 typeLength = 0; + ArrayType *columnValueObject = NULL; + Datum columnValueDatum = 0; + bool typeByValue = false; + char typeAlignment = 0; + int16 typeLength = 0; BSON_ITERATOR bsonSubIterator = { NULL, 0 }; BsonIterSubIter(bsonIterator, &bsonSubIterator); @@ -1582,12 +1578,15 @@ ColumnValue(BSON_ITERATOR *bsonIterator, Oid columnTypeId, int32 columnTypeMod) * [1] http://docs.mongodb.org/manual/reference/mongodb-extended-json/ */ void -DumpJson(StringInfo output, const char *bsonData, bool isArray) { - bson_iterator i; - const char *key; - bool isFirstElement; - char beginSymbol, endSymbol; - bson_type t; +DumpJson(StringInfo output, const char *bsonData, bool isArray) +{ + bson_iterator i; + const char *key; + bool isFirstElement; + char beginSymbol, endSymbol; + bson_type t; + + bson_iterator_from_buffer(&i, bsonData); if (isArray) @@ -1712,10 +1711,12 @@ DumpJson(StringInfo output, const char *bsonData, bool isArray) { * EscapeJsonString escapes a string for safe inclusion in JSON. */ const char * -EscapeJsonString(const char *string) { - StringInfo buffer; - const char *ptr; - int i, segmentStartIdx, len; +EscapeJsonString(const char *string) +{ + StringInfo buffer; + const char *ptr; + int i, segmentStartIdx, len; + bool needsEscaping = false; for (ptr = string; *ptr; ++ptr) { if (*ptr == '"' || *ptr == '\r' || *ptr == '\n' || *ptr == '\t'\ @@ -1790,13 +1791,13 @@ MongoAnalyzeForeignTable(Relation relation, AcquireSampleRowsFunc *acquireSampleRowsFunc, BlockNumber *totalPageCount) { - BlockNumber pageCount = 0; - int attributeCount = 0; - int32 *attributeWidths = NULL; - Oid foreignTableId = InvalidOid; - int32 documentWidth = 0; - double documentCount = 0.0; - double foreignTableSize = 0; + BlockNumber pageCount = 0; + int attributeCount = 0; + int32 *attributeWidths = NULL; + Oid foreignTableId = InvalidOid; + int32 documentWidth = 0; + double documentCount = 0.0; + double foreignTableSize = 0; foreignTableId = RelationGetRelid(relation); @@ -1850,29 +1851,29 @@ MongoAcquireSampleRows(Relation relation, int errorLevel, HeapTuple *sampleRows, int targetRowCount, double *totalRowCount, double *totalDeadRowCount) { - int sampleRowCount = 0; - double rowCount = 0; - double rowCountToSkip = -1; /* -1 means not set yet */ - double randomState = 0; - Datum *columnValues = NULL; - bool *columnNulls = NULL; - Oid foreignTableId = InvalidOid; - TupleDesc tupleDescriptor = NULL; - Form_pg_attribute *attributesPtr = NULL; - AttrNumber columnCount = 0; - AttrNumber columnId = 0; - HTAB *columnMappingHash = NULL; - MONGO_CURSOR *mongoCursor = NULL; - BSON *queryDocument = NULL; - List *columnList = NIL; - ForeignScanState *scanState = NULL; - List *foreignPrivateList = NIL; - ForeignScan *foreignScan = NULL; - MongoFdwModifyState *fmstate = NULL; - char *relationName = NULL; - int executorFlags = 0; - MemoryContext oldContext = CurrentMemoryContext; - MemoryContext tupleContext = NULL; + int sampleRowCount = 0; + double rowCount = 0; + double rowCountToSkip = -1; /* -1 means not set yet */ + double randomState = 0; + Datum *columnValues = NULL; + bool *columnNulls = NULL; + Oid foreignTableId = InvalidOid; + TupleDesc tupleDescriptor = NULL; + Form_pg_attribute *attributesPtr = NULL; + AttrNumber columnCount = 0; + AttrNumber columnId = 0; + HTAB *columnMappingHash = NULL; + MONGO_CURSOR *mongoCursor = NULL; + BSON *queryDocument = NULL; + List *columnList = NIL; + ForeignScanState *scanState = NULL; + List *foreignPrivateList = NIL; + ForeignScan *foreignScan = NULL; + MongoFdwModifyState *fmstate = NULL; + char *relationName = NULL; + int executorFlags = 0; + MemoryContext oldContext = CurrentMemoryContext; + MemoryContext tupleContext = NULL; /* create list of columns in the relation */ tupleDescriptor = RelationGetDescr(relation); diff --git a/option.c b/option.c index b6519ba..c78e1e2 100644 --- a/option.c +++ b/option.c @@ -61,9 +61,9 @@ PG_FUNCTION_INFO_V1(mongo_fdw_validator); Datum mongo_fdw_validator(PG_FUNCTION_ARGS) { - Datum optionArray = PG_GETARG_DATUM(0); - Oid optionContextId = PG_GETARG_OID(1); - List *optionList = untransformRelOptions(optionArray); + Datum optionArray = PG_GETARG_DATUM(0); + Oid optionContextId = PG_GETARG_OID(1); + List *optionList = untransformRelOptions(optionArray); ListCell *optionCell = NULL; foreach(optionCell, optionList) @@ -114,8 +114,8 @@ mongo_fdw_validator(PG_FUNCTION_ARGS) StringInfo mongo_option_names_string(Oid currentContextId) { - StringInfo optionNamesString = makeStringInfo(); - bool firstOptionPrinted = false; + StringInfo optionNamesString = makeStringInfo(); + bool firstOptionPrinted = false; int32 optionIndex = 0; for (optionIndex = 0; optionIndex < ValidOptionCount; optionIndex++) @@ -144,14 +144,14 @@ mongo_option_names_string(Oid currentContextId) MongoFdwOptions * mongo_get_options(Oid foreignTableId) { - MongoFdwOptions *mongoFdwOptions = NULL; - char *addressName = NULL; - char *portName = NULL; - int32 portNumber = 0; - char *svr_database = NULL; - char *collectionName = NULL; - char *svr_username= NULL; - char *svr_password= NULL; + MongoFdwOptions *mongoFdwOptions = NULL; + char *addressName = NULL; + char *portName = NULL; + int32 portNumber = 0; + char *svr_database = NULL; + char *collectionName = NULL; + char *svr_username= NULL; + char *svr_password= NULL; addressName = mongo_get_option_value(foreignTableId, OPTION_NAME_ADDRESS); if (addressName == NULL) @@ -206,12 +206,12 @@ mongo_free_options(MongoFdwOptions *mongoFdwOptions) static char * mongo_get_option_value(Oid foreignTableId, const char *optionName) { - ForeignTable *foreignTable = NULL; - ForeignServer *foreignServer = NULL; - List *optionList = NIL; - ListCell *optionCell = NULL; - UserMapping *mapping= NULL; - char *optionValue = NULL; + ForeignTable *foreignTable = NULL; + ForeignServer *foreignServer = NULL; + List *optionList = NIL; + ListCell *optionCell = NULL; + UserMapping *mapping= NULL; + char *optionValue = NULL; foreignTable = GetForeignTable(foreignTableId); foreignServer = GetForeignServer(foreignTable->serverid); From 2ad3238288563364a2be362dc9445abc0eeaa8d5 Mon Sep 17 00:00:00 2001 From: Ibrar Ahmed Date: Fri, 31 Oct 2014 00:56:58 +0500 Subject: [PATCH 36/43] Change the Option's variable name from mongoFdwOptions to options --- mongo_fdw.c | 60 ++++++++++++++++++++++++++--------------------------- mongo_fdw.h | 16 +++++++------- option.c | 28 ++++++++++++------------- 3 files changed, 52 insertions(+), 52 deletions(-) diff --git a/mongo_fdw.c b/mongo_fdw.c index f8c3535..6c880c5 100644 --- a/mongo_fdw.c +++ b/mongo_fdw.c @@ -380,19 +380,19 @@ MongoGetForeignPlan(PlannerInfo *root, RelOptInfo *baserel, Oid foreignTableId, static void MongoExplainForeignScan(ForeignScanState *scanState, ExplainState *explainState) { - MongoFdwOptions *mongoFdwOptions = NULL; + MongoFdwOptions *options = NULL; StringInfo namespaceName = NULL; Oid foreignTableId = InvalidOid; foreignTableId = RelationGetRelid(scanState->ss.ss_currentRelation); - mongoFdwOptions = mongo_get_options(foreignTableId); + options = mongo_get_options(foreignTableId); /* construct fully qualified collection name */ namespaceName = makeStringInfo(); - appendStringInfo(namespaceName, "%s.%s", mongoFdwOptions->svr_database, - mongoFdwOptions->collectionName); + appendStringInfo(namespaceName, "%s.%s", options->svr_database, + options->collectionName); - mongo_free_options(mongoFdwOptions); + mongo_free_options(options); ExplainPropertyText("Foreign Namespace", namespaceName->data, explainState); } @@ -404,19 +404,19 @@ MongoExplainForeignModify(ModifyTableState *mtstate, int subplan_index, ExplainState *es) { - MongoFdwOptions *mongoFdwOptions = NULL; + MongoFdwOptions *options = NULL; StringInfo namespaceName = NULL; Oid foreignTableId = InvalidOid; foreignTableId = RelationGetRelid(rinfo->ri_RelationDesc); - mongoFdwOptions = mongo_get_options(foreignTableId); + options = mongo_get_options(foreignTableId); /* construct fully qualified collection name */ namespaceName = makeStringInfo(); - appendStringInfo(namespaceName, "%s.%s", mongoFdwOptions->svr_database, - mongoFdwOptions->collectionName); + appendStringInfo(namespaceName, "%s.%s", options->svr_database, + options->collectionName); - mongo_free_options(mongoFdwOptions); + mongo_free_options(options); ExplainPropertyText("Foreign Namespace", namespaceName->data, es); } @@ -438,7 +438,7 @@ MongoBeginForeignScan(ForeignScanState *scanState, int executorFlags) ForeignScan *foreignScan = NULL; List *foreignPrivateList = NIL; BSON *queryDocument = NULL; - MongoFdwOptions *mongoFdwOptions = NULL; + MongoFdwOptions *options = NULL; MongoFdwModifyState *fmstate = NULL; List *opExpressionList = NIL; RangeTblEntry *rte; @@ -455,7 +455,7 @@ MongoBeginForeignScan(ForeignScanState *scanState, int executorFlags) return; foreignTableId = RelationGetRelid(scanState->ss.ss_currentRelation); - mongoFdwOptions = mongo_get_options(foreignTableId); + options = mongo_get_options(foreignTableId); fmstate = (MongoFdwModifyState *) palloc0(sizeof(MongoFdwModifyState)); /* @@ -475,7 +475,7 @@ MongoBeginForeignScan(ForeignScanState *scanState, int executorFlags) * Get connection to the foreign server. Connection manager will * establish new connection if necessary. */ - mongoConnection = mongo_get_connection(server, user, mongoFdwOptions); + mongoConnection = mongo_get_connection(server, user, options); foreignScan = (ForeignScan *) scanState->ss.ps.plan; foreignPrivateList = foreignScan->fdw_private; @@ -489,14 +489,14 @@ MongoBeginForeignScan(ForeignScanState *scanState, int executorFlags) columnMappingHash = ColumnMappingHash(foreignTableId, columnList); /* create cursor for collection name and set query */ - mongoCursor = MongoCursorCreate(mongoConnection, mongoFdwOptions->svr_database, mongoFdwOptions->collectionName, queryDocument); + mongoCursor = MongoCursorCreate(mongoConnection, options->svr_database, options->collectionName, queryDocument); /* create and set foreign execution state */ fmstate->columnMappingHash = columnMappingHash; fmstate->mongoConnection = mongoConnection; fmstate->mongoCursor = mongoCursor; fmstate->queryDocument = queryDocument; - fmstate->mongoFdwOptions = mongoFdwOptions; + fmstate->options = options; scanState->fdw_state = (void *) fmstate; } @@ -578,10 +578,10 @@ MongoEndForeignScan(ForeignScanState *scanState) /* if we executed a query, reclaim mongo related resources */ if (fmstate != NULL) { - if (fmstate->mongoFdwOptions) + if (fmstate->options) { - mongo_free_options(fmstate->mongoFdwOptions); - fmstate->mongoFdwOptions = NULL; + mongo_free_options(fmstate->options); + fmstate->options = NULL; } MongoFreeScanState(fmstate); } @@ -598,7 +598,7 @@ MongoReScanForeignScan(ForeignScanState *scanState) { MongoFdwModifyState *fmstate = (MongoFdwModifyState *) scanState->fdw_state; MONGO_CONN *mongoConnection = fmstate->mongoConnection; - MongoFdwOptions *mongoFdwOptions = NULL; + MongoFdwOptions *options = NULL; Oid foreignTableId = InvalidOid; /* close down the old cursor */ @@ -606,14 +606,14 @@ MongoReScanForeignScan(ForeignScanState *scanState) /* reconstruct full collection name */ foreignTableId = RelationGetRelid(scanState->ss.ss_currentRelation); - mongoFdwOptions = mongo_get_options(foreignTableId); + options = mongo_get_options(foreignTableId); /* reconstruct cursor for collection name and set query */ fmstate->mongoCursor = MongoCursorCreate(mongoConnection, - fmstate->mongoFdwOptions->svr_database, - fmstate->mongoFdwOptions->collectionName, + fmstate->options->svr_database, + fmstate->options->collectionName, fmstate->queryDocument); - mongo_free_options(mongoFdwOptions); + mongo_free_options(options); } static List * @@ -715,7 +715,7 @@ MongoBeginForeignModify(ModifyTableState *mtstate, fmstate = (MongoFdwModifyState *) palloc0(sizeof(MongoFdwModifyState)); fmstate->rel = rel; - fmstate->mongoFdwOptions = mongo_get_options(foreignTableId); + fmstate->options = mongo_get_options(foreignTableId); fmstate->target_attrs = (List *) list_nth(fdw_private, 0); @@ -777,7 +777,7 @@ MongoExecForeignInsert(EState *estate, * Get connection to the foreign server. Connection manager will * establish new connection if necessary. */ - options = fmstate->mongoFdwOptions; + options = fmstate->options; mongoConnection = mongo_get_connection(server, user, options); b = BsonCreate(); @@ -894,7 +894,7 @@ MongoExecForeignUpdate(EState *estate, foreignTableId = RelationGetRelid(resultRelInfo->ri_RelationDesc); /* resolve foreign table options; and connect to mongo server */ - options = fmstate->mongoFdwOptions; + options = fmstate->options; /* Get info about foreign table. */ table = GetForeignTable(foreignTableId); @@ -991,7 +991,7 @@ MongoExecForeignDelete(EState *estate, foreignTableId = RelationGetRelid(resultRelInfo->ri_RelationDesc); /* resolve foreign table options; and connect to mongo server */ - options = fmstate->mongoFdwOptions; + options = fmstate->options; /* Get info about foreign table. */ table = GetForeignTable(foreignTableId); @@ -1038,10 +1038,10 @@ MongoEndForeignModify(EState *estate, ResultRelInfo *resultRelInfo) MongoFdwModifyState *fmstate = (MongoFdwModifyState *) resultRelInfo->ri_FdwState; if (fmstate) { - if (fmstate->mongoFdwOptions) + if (fmstate->options) { - mongo_free_options(fmstate->mongoFdwOptions); - fmstate->mongoFdwOptions = NULL; + mongo_free_options(fmstate->options); + fmstate->options = NULL; } MongoFreeScanState(fmstate); pfree(fmstate); diff --git a/mongo_fdw.h b/mongo_fdw.h index 9960ec3..b2874cd 100644 --- a/mongo_fdw.h +++ b/mongo_fdw.h @@ -161,20 +161,20 @@ typedef struct MongoFdwOptions */ typedef struct MongoFdwModifyState { - Relation rel; /* relcache entry for the foreign table */ - List *target_attrs; /* list of target attribute numbers */ + Relation rel; /* relcache entry for the foreign table */ + List *target_attrs; /* list of target attribute numbers */ /* info about parameters for prepared statement */ - int p_nums; /* number of parameters to transmit */ - FmgrInfo *p_flinfo; /* output conversion functions for them */ + int p_nums; /* number of parameters to transmit */ + FmgrInfo *p_flinfo; /* output conversion functions for them */ struct HTAB *columnMappingHash; - MONGO_CONN *mongoConnection; /* MongoDB connection */ + MONGO_CONN *mongoConnection; /* MongoDB connection */ MONGO_CURSOR *mongoCursor; /* MongoDB cursor */ - BSON *queryDocument; /* Bson Document */ + BSON *queryDocument; /* Bson Document */ - MongoFdwOptions *mongoFdwOptions; + MongoFdwOptions *options; /* working memory context */ MemoryContext temp_cxt; /* context for per-tuple temporary data */ @@ -198,7 +198,7 @@ typedef struct ColumnMapping /* options.c */ extern MongoFdwOptions * mongo_get_options(Oid foreignTableId); -extern void mongo_free_options(MongoFdwOptions *mongoFdwOptions); +extern void mongo_free_options(MongoFdwOptions *options); extern StringInfo mongo_option_names_string(Oid currentContextId); /* connection.c */ diff --git a/option.c b/option.c index c78e1e2..51230bb 100644 --- a/option.c +++ b/option.c @@ -144,7 +144,7 @@ mongo_option_names_string(Oid currentContextId) MongoFdwOptions * mongo_get_options(Oid foreignTableId) { - MongoFdwOptions *mongoFdwOptions = NULL; + MongoFdwOptions *options = NULL; char *addressName = NULL; char *portName = NULL; int32 portNumber = 0; @@ -174,27 +174,27 @@ mongo_get_options(Oid foreignTableId) svr_username = mongo_get_option_value(foreignTableId, OPTION_NAME_USERNAME); svr_password = mongo_get_option_value(foreignTableId, OPTION_NAME_PASSWORD); - mongoFdwOptions = (MongoFdwOptions *) palloc0(sizeof(MongoFdwOptions)); + options = (MongoFdwOptions *) palloc0(sizeof(MongoFdwOptions)); - mongoFdwOptions->svr_address = addressName; - mongoFdwOptions->svr_port = portNumber; - mongoFdwOptions->svr_database = svr_database; - mongoFdwOptions->collectionName = collectionName; - mongoFdwOptions->svr_username = svr_username; - mongoFdwOptions->svr_password = svr_password; + options->svr_address = addressName; + options->svr_port = portNumber; + options->svr_database = svr_database; + options->collectionName = collectionName; + options->svr_username = svr_username; + options->svr_password = svr_password; - return mongoFdwOptions; + return options; } void -mongo_free_options(MongoFdwOptions *mongoFdwOptions) +mongo_free_options(MongoFdwOptions *options) { - if (mongoFdwOptions) + if (options) { - pfree(mongoFdwOptions->svr_address); - pfree(mongoFdwOptions->svr_database); - pfree(mongoFdwOptions); + pfree(options->svr_address); + pfree(options->svr_database); + pfree(options); } } From cc070a8d29d17e7307509a5f587450ac47f03fcc Mon Sep 17 00:00:00 2001 From: TruongSinh Tran-Nguyen Date: Fri, 7 Nov 2014 10:20:15 +0200 Subject: [PATCH 37/43] write JSON (object and array) --- .gitmodules | 3 +++ Makefile | 12 ++++++--- mongo_query.c | 69 ++++++++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 79 insertions(+), 5 deletions(-) diff --git a/.gitmodules b/.gitmodules index 8da9629..53af7f8 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,6 @@ [submodule "mongo-c-driver"] path = mongo-c-driver url = ../../mongodb/mongo-c-driver.git +[submodule "json-c"] + path = json-c + url = https://github.com/json-c/json-c.git diff --git a/Makefile b/Makefile index a65c28e..bb9d3d0 100644 --- a/Makefile +++ b/Makefile @@ -12,14 +12,16 @@ MODULE_big = mongo_fdw # on another platform, change env_posix.os in MONGO_OBJS with the appropriate # environment object file. # - MONGO_DRIVER = mongo-c-driver MONGO_PATH = $(MONGO_DRIVER)/src MONGO_OBJS = $(MONGO_PATH)/bson.os $(MONGO_PATH)/encoding.os $(MONGO_PATH)/md5.os \ $(MONGO_PATH)/mongo.os $(MONGO_PATH)/numbers.os $(MONGO_PATH)/env.os - -PG_CPPFLAGS = --std=c99 -I$(MONGO_PATH) -OBJS = connection.o option.o mongo_wrapper.o mongo_fdw.o mongo_query.o $(MONGO_OBJS) +LIBJSON = json-c +LIBJSON_OBJS = $(LIBJSON)/json_util.o $(LIBJSON)/json_object.o $(LIBJSON)/json_tokener.o \ + $(LIBJSON)/json_object_iterator.o $(LIBJSON)/printbuf.o $(LIBJSON)/linkhash.o \ + $(LIBJSON)/arraylist.o $(LIBJSON)/random_seed.o $(LIBJSON)/debug.o +PG_CPPFLAGS = --std=c99 -I$(MONGO_PATH) -I$(LIBJSON) +OBJS = connection.o option.o mongo_wrapper.o mongo_fdw.o mongo_query.o $(MONGO_OBJS) $(LIBJSON_OBJS) EXTENSION = mongo_fdw DATA = mongo_fdw--1.0.sql @@ -30,6 +32,8 @@ REGRESS_OPTS = --inputdir=test --outputdir=test \ $(MONGO_DRIVER)/%.os: $(MAKE) -C $(MONGO_DRIVER) $*.os +#$(LIBJSON)/json.o: +# $(MAKE) -C $(LIBJSON) # # Users need to specify their Postgres installation path through pg_config. For diff --git a/mongo_query.c b/mongo_query.c index c041b30..0121188 100644 --- a/mongo_query.c +++ b/mongo_query.c @@ -36,7 +36,9 @@ #include "utils/lsyscache.h" #include "utils/numeric.h" #include "utils/timestamp.h" - +#include "bson.h" +#include "json.h" +#include "bits.h" /* Local functions forward declarations */ static Expr * FindArgumentOfType(List *argumentList, NodeTag argumentType); @@ -48,6 +50,53 @@ static void AppendConstantValue(BSON *queryDocument, const char *keyName, Const *constant); +bool json_to_bson_append_element( bson *bb , const char *k , struct json_object *v ); + +bool json_to_bson_append_element( bson *bb , const char *k , struct json_object *v ) { + bool status; + status = true; + if ( ! v ) { + bson_append_null( bb , k ); + return status; + } + + switch ( json_object_get_type( v ) ) { + case json_type_int: + bson_append_int( bb , k , json_object_get_int( v ) ); + break; + case json_type_boolean: + bson_append_bool( bb , k , json_object_get_boolean( v ) ); + break; + case json_type_double: + bson_append_double( bb , k , json_object_get_double( v ) ); + break; + case json_type_string: + bson_append_string( bb , k , json_object_get_string( v ) ); + break; + case json_type_object: + bson_append_start_object( bb , k ); + json_object_object_foreach( v, kk, vv ) { + json_to_bson_append_element( bb , kk , vv ); + } + bson_append_finish_object( bb ); + break; + case json_type_array: + bson_append_start_array( bb , k ); + int i; + char buf[10]; + for ( i=0; i Date: Fri, 7 Nov 2014 11:36:57 +0200 Subject: [PATCH 38/43] $date and $oid --- mongo_fdw.c | 2 +- mongo_query.c | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/mongo_fdw.c b/mongo_fdw.c index 6c880c5..5ce6764 100644 --- a/mongo_fdw.c +++ b/mongo_fdw.c @@ -1638,7 +1638,7 @@ DumpJson(StringInfo output, const char *bsonData, bool isArray) { char oidhex[25]; bson_oid_to_string(bson_iterator_oid(&i), oidhex); - appendStringInfo(output, "\"%s\"", oidhex); + appendStringInfo(output, "{\"$oid\":\"%s\"}", oidhex); break; } case BSON_BOOL: diff --git a/mongo_query.c b/mongo_query.c index 0121188..cb7abf2 100644 --- a/mongo_query.c +++ b/mongo_query.c @@ -74,12 +74,29 @@ bool json_to_bson_append_element( bson *bb , const char *k , struct json_object bson_append_string( bb , k , json_object_get_string( v ) ); break; case json_type_object: + { + struct json_object *joj = NULL; + joj = json_object_object_get( v, "$oid" ); + if (joj != NULL) { + bson_oid_t bsonObjectId; + memset(bsonObjectId.bytes, 0, sizeof(bsonObjectId.bytes)); + BsonOidFromString(&bsonObjectId, json_object_get_string(joj) ); + status = BsonAppendOid( bb, k , &bsonObjectId); + break; + } + joj = json_object_object_get( v, "$date" ); + if (joj != NULL) { + status = BsonAppendDate( bb, k , json_object_get_int64(joj)); + break; + } + bson_append_start_object( bb , k ); json_object_object_foreach( v, kk, vv ) { json_to_bson_append_element( bb , kk , vv ); } bson_append_finish_object( bb ); break; + } case json_type_array: bson_append_start_array( bb , k ); int i; From 9b2679d2bf532f0d02e2b0fd212a61e104e7e929 Mon Sep 17 00:00:00 2001 From: TruongSinh Tran-Nguyen Date: Tue, 11 Nov 2014 15:26:15 +0200 Subject: [PATCH 39/43] NAMEARRAYOID --- mongo_query.c | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/mongo_query.c b/mongo_query.c index cb7abf2..b19cf39 100644 --- a/mongo_query.c +++ b/mongo_query.c @@ -639,6 +639,45 @@ AppenMongoValue(BSON *queryDocument, const char *keyName, Datum value, bool isnu pfree(elem_nulls); break; } + case 1003: // NAMEARRAYOID + { + ArrayType *array; + Oid elmtype; + int16 elmlen; + bool elmbyval; + char elmalign; + int num_elems; + Datum *elem_values; + bool *elem_nulls; + int i; + BSON t; + + array = DatumGetArrayTypeP(value); + elmtype = ARR_ELEMTYPE(array); + get_typlenbyvalalign(elmtype, &elmlen, &elmbyval, &elmalign); + + deconstruct_array(array, elmtype, elmlen, elmbyval, elmalign, &elem_values, &elem_nulls, &num_elems); + + BsonAppendStartArray(queryDocument, keyName, &t); + for (i = 0; i < num_elems; i++) + { + if (elem_nulls[i]) + continue; + char *valueString = NULL; + Oid outputFunctionId = InvalidOid; + bool typeVarLength = false; + bson_oid_t bsonObjectId; + memset(bsonObjectId.bytes, 0, sizeof(bsonObjectId.bytes)); + getTypeOutputInfo(NAMEOID, &outputFunctionId, &typeVarLength); + valueString = OidOutputFunctionCall(outputFunctionId, elem_values[i]); + BsonOidFromString(&bsonObjectId, valueString); + status = BsonAppendOid(queryDocument, keyName, &bsonObjectId); + } + BsonAppendFinishArray(queryDocument, &t); + pfree(elem_values); + pfree(elem_nulls); + break; + } case JSONOID: { char *outputString = NULL; From 4421151c0768409a728c52fff1b07224b691c8c3 Mon Sep 17 00:00:00 2001 From: TruongSinh Tran-Nguyen Date: Fri, 14 Nov 2014 14:40:26 +0200 Subject: [PATCH 40/43] Ignore the value of first column which is row identifier in MongoDb (_id) and let MongoDB to insert the unique value for that column *only if it is null* --- mongo_fdw.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/mongo_fdw.c b/mongo_fdw.c index 5ce6764..2f8c2c0 100644 --- a/mongo_fdw.c +++ b/mongo_fdw.c @@ -798,14 +798,12 @@ MongoExecForeignInsert(EState *estate, if (strcmp(slot->tts_tupleDescriptor->attrs[0]->attname.data, "_id") != 0) elog(ERROR, "first column of MongoDB's foreign table must be \"_id\""); - if (typoid != NAMEOID) - elog(ERROR, "type of first column of MongoDB's foreign table must be \"NAME\""); - - if (attnum == 1) + if (attnum == 1 && isnull) { /* * Ignore the value of first column which is row identifier in MongoDb (_id) - * and let MongoDB to insert the unique value for that column. + * and let MongoDB to insert the unique value for that column + * only if it is null */ } else From ff7236da66210faa30bc2aa2b8de3c8c5ceded53 Mon Sep 17 00:00:00 2001 From: TruongSinh Tran-Nguyen Date: Wed, 26 Nov 2014 10:37:38 +0200 Subject: [PATCH 41/43] allow RETURNING --- mongo_fdw.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mongo_fdw.c b/mongo_fdw.c index 2f8c2c0..07eb334 100644 --- a/mongo_fdw.c +++ b/mongo_fdw.c @@ -674,9 +674,10 @@ MongoPlanForeignModify(PlannerInfo *root, } /* * RETURNING list not supported - */ + * #truongsinh: allow RETURNING for now if (plan->returningLists) elog(ERROR, "RETURNING is not supported by this FDW"); + */ heap_close(rel, NoLock); From 46d45eab4445402b59f00977c93c2ee5ea94c2e7 Mon Sep 17 00:00:00 2001 From: TruongSinh Tran-Nguyen Date: Sat, 29 Nov 2014 00:13:54 +0200 Subject: [PATCH 42/43] fix bug when writing array of numeric, text or name --- mongo_query.c | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/mongo_query.c b/mongo_query.c index b19cf39..851d2b9 100644 --- a/mongo_query.c +++ b/mongo_query.c @@ -592,10 +592,12 @@ AppenMongoValue(BSON *queryDocument, const char *keyName, Datum value, bool isnu valueDatum = DirectFunctionCall1(numeric_float8, elem_values[i]); valueFloat = DatumGetFloat8(valueDatum); + char *index = malloc(snprintf(0, 0, "%d", i)+1); + sprintf(index, "%d", i); #ifdef META_DRIVER - status = BsonAppendDouble(&t, keyName, valueFloat); + status = BsonAppendDouble(&t, index, valueFloat); #else - status = BsonAppendDouble(queryDocument, keyName, valueFloat); + status = BsonAppendDouble(queryDocument, index, valueFloat); #endif } BsonAppendFinishArray(queryDocument, &t); @@ -632,7 +634,9 @@ AppenMongoValue(BSON *queryDocument, const char *keyName, Datum value, bool isnu continue; getTypeOutputInfo(TEXTOID, &outputFunctionId, &typeVarLength); valueString = OidOutputFunctionCall(outputFunctionId, elem_values[i]); - status = BsonAppendUTF8(queryDocument, keyName, valueString); + char *index = malloc(snprintf(0, 0, "%d", i)+1); + sprintf(index, "%d", i); + status = BsonAppendUTF8(queryDocument, index, valueString); } BsonAppendFinishArray(queryDocument, &t); pfree(elem_values); @@ -671,7 +675,9 @@ AppenMongoValue(BSON *queryDocument, const char *keyName, Datum value, bool isnu getTypeOutputInfo(NAMEOID, &outputFunctionId, &typeVarLength); valueString = OidOutputFunctionCall(outputFunctionId, elem_values[i]); BsonOidFromString(&bsonObjectId, valueString); - status = BsonAppendOid(queryDocument, keyName, &bsonObjectId); + char *index = malloc(snprintf(0, 0, "%d", i)+1); + sprintf(index, "%d", i); + status = BsonAppendOid(queryDocument, index, &bsonObjectId); } BsonAppendFinishArray(queryDocument, &t); pfree(elem_values); From 80bb2ab50ea50cef24acb790256f65d88a49327f Mon Sep 17 00:00:00 2001 From: TruongSinh Tran-Nguyen Date: Wed, 3 Dec 2014 12:43:50 +0200 Subject: [PATCH 43/43] fix BsonAppendBool --- mongo_query.c | 2 +- mongo_wrapper.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mongo_query.c b/mongo_query.c index 851d2b9..87200c0 100644 --- a/mongo_query.c +++ b/mongo_query.c @@ -500,7 +500,7 @@ AppenMongoValue(BSON *queryDocument, const char *keyName, Datum value, bool isnu case BOOLOID: { bool valueBool = DatumGetBool(value); - status = BsonAppendBool(queryDocument, keyName, (int) valueBool); + status = BsonAppendBool(queryDocument, keyName, valueBool); break; } case BPCHAROID: diff --git a/mongo_wrapper.c b/mongo_wrapper.c index fe05ef6..0b9e1d1 100644 --- a/mongo_wrapper.c +++ b/mongo_wrapper.c @@ -268,7 +268,7 @@ BsonAppendOid(BSON *b, const char* key, bson_oid_t *v) bool BsonAppendBool(BSON *b, const char* key, bool v) { - return (bson_append_int(b, key, v) == MONGO_OK); + return (bson_append_bool(b, key, v) == MONGO_OK); } bool