Skip to content

Commit

Permalink
Merge pull request #62 from linkedconnections/development
Browse files Browse the repository at this point in the history 
v2.0.0
  • Loading branch information
@julianrojas87
julianrojas87 authored Jun 25, 2022
2 parents 5cf7625 + d2a225b commit 1646a58
Show file tree
Hide file tree
Showing 6 changed files with 1,719 additions and 1,709 deletions.
2 changes: 1 addition & 1 deletion bin/gtfsrt2json.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ console.error("GTFS-RT to JSON converter use --help to discover how to use it");

program
.option('-r --real-time <realTime>', 'URL/path to gtfs-rt feed')
.option('-H --headers <headers>', 'Extra HTTP headers for requesting the gtfs files. E.g., {\\"api-Key\\":\\"someApiKey\\"}')
.option('-H --headers <headers>', 'Extra HTTP headers for requesting the gtfs files. E.g., {\\"some-header\\":\\"some-API-Key\\"}')
.parse(process.argv);

if (!program.realTime) {
Expand Down
30 changes: 20 additions & 10 deletions bin/gtfsrt2lc.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,17 +3,19 @@
const fs = require('fs');
const GtfsIndex = require('../lib/GtfsIndex');
const Gtfsrt2LC = require('../lib/Gtfsrt2LC');
var program = require('commander');
const { Level } = require('level');
const program = require('commander');

program
.option('-r --real-time <realTime>', 'URL/path to gtfs-rt feed')
.option('-s --static <static>', 'URL/path to static gtfs feed')
.option('-u --uris-template <template>', 'Templates for Linked Connection URIs following the RFC 6570 specification')
.option('-u --uris-template <urisTemplate>', 'Templates for Linked Connection URIs following the RFC 6570 specification')
.option('-H --headers <headers>', 'Extra HTTP headers for requesting the gtfs files. E.g., {\\"api-Key\\":\\"someApiKey\\"}')
.option('-f --format <format>', 'Output serialization format. Choose from json, jsonld, turtle, ntriples and csv. (Default: json)')
.option('-S --store <store>', 'Store type: LevelStore (uses your harddisk to avoid that you run out of RAM) or MemStore (default)')
.option('-S --store <store>', 'Store type: LevelStore (uses your hard disk to avoid that you run out of RAM) or MemStore (default)')
.option('-g --grep', 'Use grep to index only the trips present in the GTFS-RT. Useful for dealing with big GTFS feeds in memory.')
.option('-d --deduce', 'Create additional indexes to identify Trips on GTFS-RT feeds that do not provide a trip_id')
.option('-h --history <history>', 'Path to historic Connection LevelStore for differential updates')
.parse(process.argv);

if (!program.realTime) {
Expand All @@ -39,7 +41,7 @@ if (!program.store) {
}

// Load URIs template
var template = null;
let template = null;
try {
template = JSON.parse(fs.readFileSync(program.urisTemplate, 'utf8'));
} catch (err) {
Expand All @@ -48,22 +50,29 @@ try {
process.exit();
}
// Get resulting data format
var format = program.format || 'json';
const format = program.format || 'json';
// Set HTTP custom headers, e.g., API keys
var headers = {};
let headers = {};
if (program.headers) {
try {
headers = JSON.parse(program.headers);
} catch (err) {
console.error(err);
console.error('Please provide a valid JSON string for the extra HTTP headers');
console.error("GTFS-RT to linked connections converter use --help to discover how to use it");
process.exit();
}
}

var t0 = new Date();
var gtfsrt2lc = new Gtfsrt2LC({ path: program.realTime, uris: template, headers: headers });
var gtfsIndexer = new GtfsIndex({ path: program.static, headers: headers });
// Load historic connections store (if any)
let historyDB = null;
if(program.history) {
historyDB = new Level(program.history, { valueEncoding: 'json' });
}

let t0 = new Date();
const gtfsrt2lc = new Gtfsrt2LC({ path: program.realTime, uris: template, headers: headers });
const gtfsIndexer = new GtfsIndex({ path: program.static, headers: headers });

async function processUpdate(store, grep, deduce) {
console.error("Converting the GTFS-RT feed to Linked Connections");
Expand All @@ -76,7 +85,8 @@ async function processUpdate(store, grep, deduce) {
}
// Get GTFS indexes (stops.txt, routes.txt, trips.txt, stop_times.txt)
console.error("Creating the GTFS indexes needed for the conversion");
let indexes = await gtfsIndexer.getIndexes({ store: store, trips: trips, deduce: deduce });
const indexes = await gtfsIndexer.getIndexes({ store: store, trips: trips, deduce: deduce });
indexes.historyDB = historyDB;
console.error(`GTFS indexing process took ${new Date().getTime() - t0.getTime()} ms`);
t0 = new Date();
gtfsrt2lc.setIndexes(indexes);
Expand Down
55 changes: 21 additions & 34 deletions lib/GtfsIndex.js
View file
Original file line number Diff line number Diff line change
@@ -1,13 +1,12 @@
const url = require('url');
const { URL } = require('url');
const { request } = require('undici');
const util = require('util');
const fs = require('fs');
const unzip = require('unzipper');
const csv = require('fast-csv');
const del = require('del');
const http = require('follow-redirects').http;
const https = require('follow-redirects').https;
const childProcess = require('child_process');
const level = require('level');
const { Level } = require('level');

const exec = util.promisify(childProcess.exec);

Expand All @@ -22,15 +21,14 @@ class GtfsIndex {
return new Promise(async (resolve, reject) => {
try {
await this.cleanUp();
// Download/access static GTFS feed
// Download or load from disk static GTFS feed
if (this.path.startsWith('http') || this.path.startsWith('https')) {
let download_url = url.parse(this.path);
if (!download_url.protocol) {
reject('Please provide a valid url or a path to a GTFS feed');
const downloadURL = new URL(this.path);
if (!downloadURL.protocol) {
reject('Please provide a valid URL or a path to a GTFS feed');
return;
} else {
download_url.headers = this.headers;
await this.doRequest(download_url);
await this.download(this.path, this.headers);
}
} else {
if (!fs.existsSync(this.path)) {
Expand All @@ -49,25 +47,14 @@ class GtfsIndex {
});
}

doRequest(url) {
return new Promise((resolve, reject) => {
let req = null;
if (url.protocol === 'https:') {
req = https.request(url, async res => {
await this.unzip(res)
resolve();
});
} else {
req = http.request(url, async res => {
await this.unzip(res)
resolve();
});
}
req.on('error', err => {
reject(err);
});
req.end();
});
async download(url, headers) {
const res = await request(url, { method: 'GET', headers });

if(res.statusCode === 200) {
await this.unzip(await res.body);
} else {
throw new Error(`Error on HTTP request: ${url}, Message: ${await res.body.text()}`);
}
}

unzip(res) {
Expand Down Expand Up @@ -142,10 +129,10 @@ class GtfsIndex {
await del(['.rt_indexes'], { force: true });
fs.mkdirSync('.rt_indexes');

stops_index = level('.rt_indexes/.stops');
routes_index = level('.rt_indexes/.routes');
trips_index = level('.rt_indexes/.trips');
stop_times_index = level('.rt_indexes/.stop_times');
stops_index = new Level('.rt_indexes/.stops', { valueEncoding: 'json' });
routes_index = new Level('.rt_indexes/.routes', { valueEncoding: 'json' });
trips_index = new Level('.rt_indexes/.trips', { valueEncoding: 'json' });
stop_times_index = new Level('.rt_indexes/.stop_times', { valueEncoding: 'json' });

tp = this.createIndex(this.auxPath + '/trips.txt', trips_index, 'trip_id', tripsByRoute);
// Make sure stop_times.txt is ordered by stop_sequence
Expand Down Expand Up @@ -239,7 +226,7 @@ class GtfsIndex {
if (map instanceof Map) {
map.set(currentTripId, currentTrip);
} else {
await map.put(currentTripId, currentTrip, { valueEncoding: 'json' });
await map.put(currentTripId, currentTrip);
}
// Prepare for next trip
currentTrip = [];
Expand Down
Loading

0 comments on commit 1646a58

Please sign in to comment.