From 0a4cbe4e9f1052ba6e5f3b1e9ae523c390f70f74 Mon Sep 17 00:00:00 2001 From: Pascal Desmarets Date: Wed, 8 May 2019 22:00:42 +0200 Subject: [PATCH] v0.1.21 RE of specialized datatypes --- package.json | 4 +- .../field_level/fieldLevelConfig.json | 1 + reverse_engineering/SchemaCreator.js | 114 ++++++- reverse_engineering/api.js | 318 +++++++++++------- 4 files changed, 311 insertions(+), 126 deletions(-) diff --git a/package.json b/package.json index 23de486..bb0b163 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "Elasticsearch", - "version": "0.1.20", - "versionDate": "2019-05-07", + "version": "0.1.21", + "versionDate": "2019-05-08", "author": "hackolade", "engines": { "hackolade": "1.12.7", diff --git a/properties_pane/field_level/fieldLevelConfig.json b/properties_pane/field_level/fieldLevelConfig.json index 1216e95..3f2753e 100644 --- a/properties_pane/field_level/fieldLevelConfig.json +++ b/properties_pane/field_level/fieldLevelConfig.json @@ -1085,6 +1085,7 @@ making sure that you maintain a proper JSON format. "propertyName": "relations", "propertyKeyword": "relations", "propertyType": "group", + "isTargetProperty": true, "structure": [ { "propertyName": "parent", diff --git a/reverse_engineering/SchemaCreator.js b/reverse_engineering/SchemaCreator.js index 01c8553..833f2c2 100644 --- a/reverse_engineering/SchemaCreator.js +++ b/reverse_engineering/SchemaCreator.js @@ -14,7 +14,9 @@ const snippets = { "geometrycollection": require(snippetsPath + "geoshape-geometrycollection.json"), "multilinestring": require(snippetsPath + "geoshape-multilinestring.json"), "multipolygon": require(snippetsPath + "geoshape-multipolygon.json"), - "polygon": require(snippetsPath + "geoshape-polygon.json") + "polygon": require(snippetsPath + "geoshape-polygon.json"), + "completionArray": require(snippetsPath + "completionArray.json"), + "completionObject": require(snippetsPath + "completionObject.json") }; const helper = require('../helper/helper'); @@ -78,7 +80,7 @@ module.exports = { sample = sample || {}; schema.properties = this.getServiceFields(sample); - schema.properties._source.properties = this.getFields(elasticMapping.properties, sample._source); + schema.properties._source.properties = this.getFields(elasticMapping.properties, sample._source, elasticMapping.properties); if (elasticMapping.dynamic) { schema.dynamic = elasticMapping.dynamic; @@ -87,19 +89,19 @@ module.exports = { return schema; }, - getFields(properties, sample) { + getFields(properties, sample, mapping) { let schema = {}; for (let fieldName in properties) { const currentSample = sample && sample[fieldName]; - schema[fieldName] = this.getField(properties[fieldName], currentSample); + schema[fieldName] = this.getField(properties[fieldName], currentSample, mapping); } return schema; }, - getField(fieldData, sample) { + getField(fieldData, sample, mapping) { let schema = {}; if (!fieldData) { @@ -116,7 +118,7 @@ module.exports = { ].indexOf(schema.type) !== -1; if (hasProperties) { - let properties = this.getFields(fieldData.properties, sample); + let properties = this.getFields(fieldData.properties, sample, mapping); if (isArrayType) { schema.items = [{ @@ -128,6 +130,10 @@ module.exports = { } } + if (fieldData.copy_to) { + fieldData.copy_to = this.getCopyToPath(fieldData.copy_to, mapping); + } + if (Array.isArray(sample) && !isArrayType) { schema = { type: 'array', @@ -135,10 +141,16 @@ module.exports = { }; } - if (schema.type === 'geo-shape' || schema.type === 'geo-point') { + if ([ + 'geo-shape', 'geo-point' + ].includes(schema.type)) { schema = this.handleSnippet(schema); } + if (schema.type === 'completion') { + schema = this.handleCompletionSnippet(schema); + } + schema = this.setProperties(schema, fieldData); return schema; @@ -178,7 +190,23 @@ module.exports = { case "binary": case "nested": case "date": + case "token_count": + case "murmur3": + case "annotated_text": + case "percolator": + case "ip": + case "dense_vector": + case "sparse_vector": + case "alias": + case "rank_feature": + case "rank_features": + case "join": return { type }; + case "completion": + return { + type, + subType: this.getCompletionSubtype(value) + }; case "geo_point": return { type: "geo-point", @@ -310,6 +338,22 @@ module.exports = { } }, + getCompletionSubtype(value) { + if (Array.isArray(value)) { + return "array"; + } else { + return "object"; + } + }, + + handleCompletionSnippet(schema) { + return Object.assign({}, this.handleSnippet(Object.assign({}, schema, { + subType: schema.subType === 'array' ? 'completionArray' : 'completionObject' + })), { + subType: schema.subType + }); + }, + handleSnippet(schema) { const snippet = snippets[schema.subType]; if (snippet) { @@ -363,11 +407,67 @@ module.exports = { for (let propName in properties) { if (propName === 'fields') { schema["stringfields"] = JSON.stringify(properties[propName], null, 4); + } else if (propName === 'relations') { + schema[propName] = getRelations(properties[propName]); } else { schema[propName] = properties[propName]; } } return schema; + }, + + getCopyToPath(copyToValue, mapping) { + const copyTo = Array.isArray(copyToValue) ? copyToValue : [ copyToValue ]; + + const result = copyTo.reduce((result, propertyName) => { + return [ + ...result, + ...findPropertiesInMapping(propertyName, mapping) + ]; + }, []); + + return result; + } +}; + +const getRelations = (relations) => { + if (typeof relations !== 'object') { + return []; } + + return Object.keys(relations).map((parentName) => { + const children = Array.isArray(relations[parentName]) ? relations[parentName] : [ relations[parentName] ]; + + return { + parent: parentName, + children: children.map(item => ({ name: item })) + }; + }, {}); +}; + +const findPropertiesInMapping = (propertyName, mapping) => { + const getPaths = (propertyName, properties, path) => { + return Object.keys(properties).reduce((result, name) => { + const property = properties[name]; + + if (property.properties) { + result = [ + ...result, + ...getPaths(propertyName, property.properties, [...path, name]) + ]; + } + + if (propertyName === name) { + result = [ + ...result, + [...path, name].join('.') + ]; + } + + return result; + }, []); + }; + + return getPaths(propertyName, mapping, []); }; diff --git a/reverse_engineering/api.js b/reverse_engineering/api.js index 32e2072..7ac4305 100644 --- a/reverse_engineering/api.js +++ b/reverse_engineering/api.js @@ -112,33 +112,33 @@ module.exports = { const { includeSystemCollection } = connectionInfo; - client.indices.getMapping() - .then(data => { - let result = []; - - for (let index in data) { - if (!includeSystemCollection && index[0] === '.') { - continue; - } - - let dbItem = { - dbName: index, - dbCollections: [] - }; - - if (data[index].mappings) { - dbItem.dbCollections = Object.keys(data[index].mappings); - } + client.info().then(info => { + const majorVersion = +info.version.number.split('.').shift(); + + return getIndexes(client, includeSystemCollection) + .then(indexes => { + return Object.keys(indexes).map(indexName => { + let dbItem = { + dbName: indexName, + dbCollections: [] + }; - result.push(dbItem); - } + if (majorVersion < 7 && indexes[indexName].mappings) { + dbItem.dbCollections = Object.keys(indexes[indexName].mappings); + } - cb(null, result); - }) - .catch(err => { + return dbItem; + }); + }); + }) + .then( + (data) => { + cb(null, data); + }, err => { logger.log('error', err); cb(err); - }); + } + ); }); }, @@ -190,7 +190,9 @@ module.exports = { }, (client, modelInfo, getData) => { - getSchemaMapping(types, client).then((jsonSchemas) => { + const indexTypes = getTypesByVersion(modelInfo.version, types, indices); + + getSchemaMapping(indexTypes, client).then((jsonSchemas) => { getData(null, client, modelInfo, jsonSchemas); }, (err) => { logger.log('error', err, 'Error of getting schema'); @@ -203,9 +205,12 @@ module.exports = { }, (client, modelInfo, jsonSchemas, next) => { + const indexTypes = getTypesByVersion(modelInfo.version, types, indices); + async.map(indices, (indexName, nextIndex) => { let bucketInfo = Object.assign(getBucketData(jsonSchemas[indexName] || {}), defaultBucketInfo); - if (!types[indexName]) { + + if (!indexTypes[indexName]) { if (includeEmptyCollection) { nextIndex(null, [{ dbName: indexName, @@ -217,100 +222,39 @@ module.exports = { nextIndex(null, [{}]); } } else { - async.map(types[indexName], (typeName, nextType) => { - async.waterfall([ - (getSampleDocSize) => { - client.count({ - index: indexName, - type: typeName - }, (err, response) => { - getSampleDocSize(err, response); - }); - }, - - (response, searchData) => { - const per = recordSamplingSettings.relative.value; - const size = (recordSamplingSettings.active === 'absolute') - ? recordSamplingSettings.absolute.value - : Math.round(response.count / 100 * per); - const count = size > MAX_DOCUMENTS ? MAX_DOCUMENTS : size; - - searchData(null, count); - }, - - (size, getTypeData) => { - client.search({ - index: indexName, - type: typeName, - size - }, (err, data) => { - getTypeData(err, data); - }); - }, - - (data, nextCallback) => { - let documents = data.hits.hits; - const documentTemplate = documents.reduce((tpl, doc) => _.merge(tpl, doc), {}); - - let documentsPackage = { - dbName: indexName, - collectionName: typeName, - documents, - indexes: [], - bucketIndexes: [], - views: [], - validation: false, - emptyBucket: false, - containerLevelKeys, - bucketInfo - }; - - const hasJsonSchema = jsonSchemas && jsonSchemas[indexName] && jsonSchemas[indexName].mappings && jsonSchemas[indexName].mappings[typeName]; - - if (hasJsonSchema) { - documentsPackage.validation = { - jsonSchema: SchemaCreator.getSchema( - jsonSchemas[indexName].mappings[typeName], - documentTemplate - ) - }; - } - - if (fieldInference.active === 'field') { - documentsPackage.documentTemplate = documentTemplate; - } - - nextCallback(null, documentsPackage); + const majorVersion = +modelInfo.version.split('.').shift(); + const schemaData = { + indexName, recordSamplingSettings, containerLevelKeys, bucketInfo, jsonSchemas, fieldInference, client + }; + + if (majorVersion >= 7) { + getIndexTypeData('', schemaData).then((docPackage) => { + if (shouldPackageBeAdded(docPackage, includeEmptyCollection)) { + nextIndex(null, [ docPackage ]); + } else { + nextIndex(null, []); } - ], nextType); - }, (err, typeData) => { - if (err) { - nextIndex(err, typeData); - } else { - const filterData = typeData.filter(docPackage => { - if (!includeEmptyCollection) { - if ( - docPackage.documents.length === 0 - && - docPackage.validation - && - docPackage.validation.jsonSchema - && - docPackage.validation.jsonSchema.properties - && - docPackage.validation.jsonSchema.properties._source - && - _.isEmpty(docPackage.validation.jsonSchema.properties._source.properties) - ) { - return false; - } - } - - return true; + }, err => { + nextIndex(err); + }); + } else { + async.map(indexTypes[indexName], (typeName, nextType) => { + getIndexTypeData(typeName, schemaData).then((docPackage) => { + nextType(null, docPackage); + }, err => { + nextType(err); }); - nextIndex(null, filterData); - } - }); + }, (err, typeData) => { + if (err) { + nextIndex(err, typeData); + } else { + const filterData = typeData.filter(docPackage => { + return shouldPackageBeAdded(docPackage, includeEmptyCollection); + }); + nextIndex(null, filterData); + } + }); + } } }, (err, items) => { next(err, items, modelInfo); @@ -327,6 +271,146 @@ module.exports = { } }; +const shouldPackageBeAdded = (docPackage, includeEmptyCollection) => { + if (includeEmptyCollection) { + return true; + } + + if ( + docPackage.documents.length === 0 + && + docPackage.validation + && + docPackage.validation.jsonSchema + && + docPackage.validation.jsonSchema.properties + && + docPackage.validation.jsonSchema.properties._source + && + _.isEmpty(docPackage.validation.jsonSchema.properties._source.properties) + ) { + return false; + } + + return true; +}; + +const getIndexTypeData = (typeName, { + indexName, + recordSamplingSettings, + containerLevelKeys, + bucketInfo, + jsonSchemas, + fieldInference, + client +}) => new Promise((resolve, reject) => { + async.waterfall([ + (getSampleDocSize) => { + client.count({ + index: indexName, + type: typeName + }, (err, response) => { + getSampleDocSize(err, response); + }); + }, + + (response, searchData) => { + const per = recordSamplingSettings.relative.value; + const size = (recordSamplingSettings.active === 'absolute') + ? recordSamplingSettings.absolute.value + : Math.round(response.count / 100 * per); + const count = size > MAX_DOCUMENTS ? MAX_DOCUMENTS : size; + + searchData(null, count); + }, + + (size, getTypeData) => { + client.search({ + index: indexName, + type: typeName, + size + }, (err, data) => { + getTypeData(err, data); + }); + }, + + (data, nextCallback) => { + let documents = data.hits.hits; + const documentTemplate = documents.reduce((tpl, doc) => _.merge(tpl, doc), {}); + + let documentsPackage = { + dbName: indexName, + collectionName: typeName || "_doc", + documents, + indexes: [], + bucketIndexes: [], + views: [], + validation: false, + emptyBucket: false, + containerLevelKeys, + bucketInfo + }; + + const mappingJsonSchema = typeName + ? jsonSchemas && jsonSchemas[indexName] && jsonSchemas[indexName].mappings && jsonSchemas[indexName].mappings[typeName] + : jsonSchemas && jsonSchemas[indexName] && jsonSchemas[indexName].mappings; + const hasJsonSchema = Boolean(mappingJsonSchema); + + if (hasJsonSchema) { + documentsPackage.validation = { + jsonSchema: SchemaCreator.getSchema( + mappingJsonSchema, + documentTemplate + ) + }; + } + + if (fieldInference.active === 'field') { + documentsPackage.documentTemplate = documentTemplate; + } + + nextCallback(null, documentsPackage); + } + ], (err, data) => { + if (err) { + reject(err); + } else { + resolve(data); + } + }); +}); + +const getTypesByVersion = (version, types, indexes) => { + const majorVersion = +version.split('.').shift(); + + if (majorVersion < 7) { + return types; + } + + indexes = Array.isArray(indexes) ? indexes : []; + + return indexes.reduce((result, indexName) => { + return Object.assign({}, result, { [indexName]: [] }); + }, {}); +}; + +const getIndexes = (client, includeSystemCollection) => { + return client.indices.getMapping() + .then(data => { + return Object.keys(data).filter(indexName => { + if (!includeSystemCollection && indexName[0] === '.') { + return false; + } else { + return true; + } + }).reduce((result, indexName) => { + return Object.assign({}, result, { + [indexName]: data[indexName] + }); + }, {}); + }); +}; + function getSamplingInfo(recordSamplingSettings, fieldInference){ let samplingInfo = {}; let value = recordSamplingSettings[recordSamplingSettings.active].value;