diff --git a/.gitignore b/.gitignore index 59a616eb..10ebefcd 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ xunit.xml db.log data/ sslcert +.idea/ diff --git a/app/models/build.js b/app/models/build.js index 57f1654f..6976c445 100644 --- a/app/models/build.js +++ b/app/models/build.js @@ -1,9 +1,7 @@ - /*! * Module dependencies */ // native modules -var crypto = require('crypto'); var fs = require('fs'); var path = require('path'); const zlib = require('zlib'); @@ -15,152 +13,138 @@ var uuid = require('node-uuid'); var mongoose = require('mongoose'); var QueryPlugin = require('mongoose-query'); var mime = require('mime'); -var nconf = require('nconf'); - -function checksum (str, algorithm, encoding) { - return crypto - .createHash(algorithm || 'sha1') - .update(str, 'binary') - .digest(encoding || 'hex') -} +var tools = require('../tools'); +var checksum = tools.checksum; +var filedb = tools.filedb; +var file_provider = filedb.provider; +var FileSchema = require('./file'); var Schema = mongoose.Schema; -var filedb = nconf.get('filedb'); - /** * Location schema */ var Location = new Schema({ - url: {type: String }, // fs://... or http://... or ftp://... or sft://... - auth: { - usr: { type: StringĀ }, - pwd: { type: String } - } + url: {type: String}, // fs://... or http://... or ftp://... or sft://... + auth: { + usr: {type: String}, + pwd: {type: String} + } }); /** * Issue schema */ var Issue = new Schema({ - time: {type: Date, default: Date.now}, - type: { type: String, enum: ['github', 'jira'] }, - url: { type: String } + time: {type: Date, default: Date.now}, + type: {type: String, enum: ['github', 'jira']}, + url: {type: String} }); /** * Build schema */ var BuildSchema = new Schema({ - name: {type: String, required: true }, - type: { type: String, enum: ['app', 'lib'], default: 'app'}, - cre: { - user: {type: String}, - time: {type: Date, default: Date.now}, - host: {type: String} - }, - mod: { - user: {type: String}, - time: {type: Date, default: Date.now} - }, - uuid: { type: String, default: uuid.v4, index: true }, - vcs: [ - new Schema({ - name: { type: String }, //e.g. "github" - system: { type: String, enum: ['git','SVN', 'CSV'], default: 'git' }, - type: {type: String, enum: ['PR']}, - commitId: { type: String }, - branch: { type: String }, - base_branch: {type: String}, - base_commit: {type: String}, - pr_number: {type: String}, - url: { type: String }, - clean_wa: { type: Boolean } - }) - ], - ci: { - system: {type: String, enum: ['Jenkins', 'travisCI', 'circleCI']}, - location: Location, - job: { - name: {type: String}, - number: {type: String} - } - }, - compiledBy: { type: String, enum: ['CI', 'manual'] }, - changeId: {type: String}, // e.g. when gerrit build - configuration: { - name: { type: String }, - compiler: { - name: {type: String}, - version: {type: String}, - macros: [{ - key: {type: String}, - value: {type: String} - }], - flags: [{ - key: {type: String}, - value: {type: String} - }] + name: {type: String, required: true}, + type: {type: String, enum: ['app', 'lib'], default: 'app'}, + cre: { + user: {type: String}, + time: {type: Date, default: Date.now}, + host: {type: String} }, - linker: { - name: {type: String}, - version: {type: String}, - flags: [{ - key: {type: String}, - value: {type: String} - }] + mod: { + user: {type: String}, + time: {type: Date, default: Date.now} }, - toolchain: { - name: {type: String}, - version: {type: String} - } - }, - memory: { - summary: { - heap: {type: Number}, - static_ram: {type: Number}, - total_flash: {type: Number}, - stack: {type: Number}, - total_ram: {type: Number} - } - }, - files: [{ - //buffer limit 16MB when attached to document! - name: { type: String }, - mime_type: { type: String }, - base64: { type: String }, - data: { type: Buffer }, - size: { type: Number }, - sha1: { type: String }, - sha256: { type: String } - }], - issues: [ Issue ], - // build target device - target: { - type: { type: String, enum: ['simulate','hardware'], default: 'hardware', required: true}, - os: { type: String, enum: ['win32', 'win64', 'unix32', 'unix64', 'mbedOS', 'unknown'] }, - simulator: { - bt: { type: String }, - network: { type: String } + uuid: {type: String, default: uuid.v4, index: true}, + vcs: [ + new Schema({ + name: {type: String}, //e.g. "github" + system: {type: String, enum: ['git', 'SVN', 'CSV'], default: 'git'}, + type: {type: String, enum: ['PR']}, + commitId: {type: String}, + branch: {type: String}, + base_branch: {type: String}, + base_commit: {type: String}, + pr_number: {type: String}, + url: {type: String}, + clean_wa: {type: Boolean} + }) + ], + ci: { + system: {type: String, enum: ['Jenkins', 'travisCI', 'circleCI']}, + location: Location, + job: { + name: {type: String}, + number: {type: String} + } + }, + compiledBy: {type: String, enum: ['CI', 'manual']}, + changeId: {type: String}, // e.g. when gerrit build + configuration: { + name: {type: String}, + compiler: { + name: {type: String}, + version: {type: String}, + macros: [{ + key: {type: String}, + value: {type: String} + }], + flags: [{ + key: {type: String}, + value: {type: String} + }] + }, + linker: { + name: {type: String}, + version: {type: String}, + flags: [{ + key: {type: String}, + value: {type: String} + }] + }, + toolchain: { + name: {type: String}, + version: {type: String} + } + }, + memory: { + summary: { + heap: {type: Number}, + static_ram: {type: Number}, + total_flash: {type: Number}, + stack: {type: Number}, + total_ram: {type: Number} + } }, - hw: { - vendor: { type: String }, - model: {type: String}, - rev: { type: String }, - meta: { type: String } + files: [ FileSchema ], + issues: [Issue], + // build target device + target: { + type: {type: String, enum: ['simulate', 'hardware'], default: 'hardware', required: true}, + os: {type: String, enum: ['win32', 'win64', 'unix32', 'unix64', 'mbedOS', 'unknown']}, + simulator: { + bt: {type: String}, + network: {type: String} + }, + hw: { + vendor: {type: String}, + model: {type: String}, + rev: {type: String}, + meta: {type: String} + } } - } }); -BuildSchema.set('toObject', { virtuals: true }); +BuildSchema.set('toObject', {virtuals: true}); //BuildSchema.set('toJSON', { virtuals: true }); /** * Build plugin */ -BuildSchema.plugin( QueryPlugin ); //install QueryPlugin +BuildSchema.plugin(QueryPlugin); //install QueryPlugin /** * Add your @@ -170,131 +154,104 @@ BuildSchema.plugin( QueryPlugin ); //install QueryPlugin */ /* -BuildSchema.path('location').validate(function (value, respond) { - if( value.length === 0 ) respond(false); - else respond(true); + BuildSchema.path('location').validate(function (value, respond) { + if( value.length === 0 ) respond(false); + else respond(true); }, '{PATH} missing'); */ -function handleFiledb(file) { - var target = path.join(filedb, file.sha1+'.gz'); - var fileData = file.data; - fs.exists(target, function(exists) { - if (exists) { - winston.warn('File %s exists already (filename: %s)', file.name, target); - return; - } - winston.warn('Store file %s (filename: %s)', file.name, target); - zlib.gzip(fileData, function (error, result) { - if (error) { - return winston.warn(error); - } - fs.writeFile(target, result, function (err) { - if (err) { - winston.warn(err); - } - }); - }); - }); -} + BuildSchema.pre('validate', function (next) { - var err; - if( _.isArray(this.files) ) { - for(i=0;i { @@ -106,6 +149,7 @@ ResultSchema.pre('validate', function (next) { }); return; } + next(err); }); ResultSchema.virtual('exec.sut.sha1'); diff --git a/app/tools/checksum.js b/app/tools/checksum.js new file mode 100644 index 00000000..d584614c --- /dev/null +++ b/app/tools/checksum.js @@ -0,0 +1,13 @@ + +/*! + * Module dependencies + */ +// native modules +var crypto = require('crypto'); + +module.exports = function checksum (str, algorithm, encoding) { + return crypto + .createHash(algorithm || 'sha1') + .update(str, 'binary') + .digest(encoding || 'hex') +} diff --git a/app/tools/filedb.js b/app/tools/filedb.js new file mode 100644 index 00000000..e7c19d3d --- /dev/null +++ b/app/tools/filedb.js @@ -0,0 +1,64 @@ +// native modules +var fs = require('fs'); +var path = require('path'); +const zlib = require('zlib'); + +// 3rd party modules +var _ = require('lodash'); +var winston = require('winston'); +var nconf = require('nconf'); + +var filedb = nconf.get('filedb'); +var checksum = require('./checksum'); + +module.exports.provider = filedb; +module.exports.readFile = function readFile(file, callback) { + var source = path.join(filedb, file.sha1+'.gz'); + winston.debug('loading source: ', source); + fs.readFile(source, function(err, buffer) { + if(err) { + return callback(err); + } + winston.debug("file readed"); + zlib.gunzip(buffer, function (error, data) { + winston.debug("data gunzipped"); + callback(error, data?_.merge({}, file, {data: data}):null); + }); + }); +}; + +module.exports.storeFile = function storeFile(file, callback) { + + if(!_.has(file, 'size')) { + file.size = file.data.length; + } + if(!_.has(file, 'sha1')) { + file.sha1 = checksum(file.data, 'sha1'); + } + if(!_.isFunction(callback)) { + callback = function(){}; + } + + var target = path.join(filedb, file.sha1+'.gz'); + var fileData = file.data; + fs.exists(target, function(exists) { + if (exists) { + winston.warn('File %s exists already (filename: %s)', file.name, target); + return callback(); + } + winston.warn('Store file %s (filename: %s)', file.name, target); + zlib.gzip(fileData, function (error, result) { + if (error) { + winston.warn(error); + return callback(error); + } + fs.writeFile(target, result, function (err) { + if (err) { + winston.warn(err); + return callback(err); + } + callback(); + }); + }); + }); +} \ No newline at end of file diff --git a/app/tools/index.js b/app/tools/index.js new file mode 100644 index 00000000..e2ef10d1 --- /dev/null +++ b/app/tools/index.js @@ -0,0 +1,4 @@ +module.exports = { + filedb: require('./filedb'), + checksum: require('./checksum') +};