Skip to content

Commit

Permalink
feat(stop_time): add ability to interpolate stop times
Browse files Browse the repository at this point in the history
  • Loading branch information
@evansiroky
evansiroky committed Jan 15, 2018
1 parent 9613072 commit 562b085
Show file tree
Hide file tree
Showing 69 changed files with 542 additions and 137 deletions.
105 changes: 45 additions & 60 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ Currently works only with PostgreSQL (including PostGIS), MySQL (with spatial ca
## Table of Contents

* [Installation](#installation)
* [Usage](#usage)
* [API](#api)

## Installation

Expand All @@ -19,6 +19,7 @@ In order to use this library, you must also install the additional libraries in

npm install pg --save
npm install pg-copy-streams --save
npm install pg-query-stream --save
npm install pg-hstore --save

#### With pg and node v0.10.x
Expand All @@ -37,66 +38,13 @@ You must also install the package `promise-polyfill` and write additional code.

Usage with SQLite requires that sqlite is installed and is available via a unix command line.

## Usage:
## API:

### Downloading the GTFS File:
### GTFS(options)

```js
var GTFS = require('gtfs-sequelize');

var downloadConfig = {
gtfsUrl: 'http://feed.rvtd.org/googleFeeds/static/google_transit.zip',
downloadsDir: 'downloads'
};

var gtfs = GTFS(downloadConfig);
gtfs.downloadGtfs(function() {
//download has finished callback
});
```

### Loading GTFS into Database:

```js
var GTFS = require('gtfs-sequelize');
var pgConfig = {
database: 'postgres://gtfs_sequelize:gtfs_sequelize@localhost:5432/gtfs-sequelize-test',
downloadsDir: 'downloads',
gtfsFileOrFolder: 'google_transit.zip',
sequelizeOptions: {
logging: false
}
}
Create a new GTFS API.

var gtfs = GTFS(pgConfig);
gtfs.loadGtfs(function() {
//database loading has finished callback
});
```

### Loading into a DB with PostGIS installed:

```js
var GTFS = require('gtfs-sequelize');
var pgConfig = {
database: 'postgres://gtfs_sequelize:gtfs_sequelize@localhost:5432/gtfs-sequelize-test',
downloadsDir: 'downloads',
gtfsFileOrFolder: 'google_transit.zip',
spatial: true,
sequelizeOptions: {
logging: false
}
}
var gtfs = GTFS(pgConfig);
gtfs.loadGtfs(function() {
//database loading has finished callback
});
```

### Querying a specific schema within a DB:
Example:

```js
var GTFS = require('gtfs-sequelize');
Expand All @@ -107,8 +55,7 @@ var pgConfig = {
gtfsFileOrFolder: 'google_transit.zip',
spatial: true,
sequelizeOptions: {
logging: false,
schema: 'test_schema'
logging: false
}
}

Expand All @@ -117,3 +64,41 @@ gtfs.loadGtfs(function() {
//database loading has finished callback
});
```

#### options

| Key | Value |
| -- | -- |
| database | A database connection string. You must specify a user and a database in your connection string. The database must already exist, but the tables within the db do not need to exist. |
| downloadsDir | The directory where you want the feed zip fils downloaded to or where you're going to read the feed read from. |
| gtfsFileOrFolder | The (zip) file or folder to load the gtfs from |
| interpolateStopTimes | Default is undefined. If true, after loading the stop_times table, all stop_times with undefined arrival and departure times will be updated to include interpolated arrival and departure times. |
| sequelizeOptions | Options to pass to sequelize. Note: to use a specific schema you'll want to pass something like this: `{ schema: 'your_schema' }` |
| spatial | Default is undefined. If true, spatial tables for the shapes and stops will be created. |

### gtfs.connectToDatabase()

Return a sequelize api of the database.

Example:

```js
var db = gtfs.connectToDatabase()

db.stop.findAll()
.then(stops => {
console.log(stops)
})
```

### gtfs.downloadGtfs(callback)

If a url is provided, the feed will be attempted to be downloaded. Works with `http`, `https` and `ftp`.

### gtfs.interpolateStopTimes(callback)

Interpolate stop_times with undefined arrival and departure times. If you load a gtfs with the `interpolateStopTimes` flag set to true, you don't need to call this.

### gtfs.loadGtfs(callback)

Load the gtfs into the database.
45 changes: 26 additions & 19 deletions index.js
View file
Original file line number Diff line number Diff line change
@@ -1,40 +1,47 @@
var path = require('path'),
downloadGtfs = require('./lib/download.js'),
Database = require('./models'),
loadgtfs = require('./lib/gtfsLoader.js');

module.exports = function(config) {
const downloadGtfs = require('./lib/download.js')
const loadgtfs = require('./lib/gtfsLoader.js')
const operations = require('./lib/operations')
const Database = require('./models')

var connectToDatabase = function(rawModels) {
var db = Database(config.database, config.sequelizeOptions ? config.sequelizeOptions : {});
if(!rawModels && config.spatial) {
db.stop = db.sequelize.import('models/spatial/stop.js');
db.shape_gis = db.sequelize.import('models/spatial/shape_gis.js');
db.trip = db.sequelize.import('models/spatial/trip.js');
module.exports = function (config) {
const connectToDatabase = function (rawModels) {
const db = Database(config.database, config.sequelizeOptions ? config.sequelizeOptions : {})
if (!rawModels && config.spatial) {
db.stop = db.sequelize.import('models/spatial/stop.js')
db.shape_gis = db.sequelize.import('models/spatial/shape_gis.js')
db.trip = db.sequelize.import('models/spatial/trip.js')
// reassociate spatially-enable models
db.stop.associate(db);
db.shape_gis.associate(db);
db.trip.associate(db);
db.stop.associate(db)
db.shape_gis.associate(db)
db.trip.associate(db)
}
return db;
return db
}

var download = function(callback) {
downloadGtfs(config.gtfsUrl, config.downloadsDir, callback);
const download = function (callback) {
downloadGtfs(config.gtfsUrl, config.downloadsDir, callback)
}

var loadGtfs = function(callback) {
const interpolateStopTimes = function (callback) {
const db = connectToDatabase()
operations.interpolateStopTimes(db, callback)
}

const loadGtfs = function (callback) {
loadgtfs(config.downloadsDir,
config.gtfsFileOrFolder,
connectToDatabase(true),
config.spatial,
callback);
config.interpolateStopTimes,
callback)
}

return {
config: config,
connectToDatabase: connectToDatabase,
downloadGtfs: download,
interpolateStopTimes: interpolateStopTimes,
loadGtfs: loadGtfs
}
}
91 changes: 67 additions & 24 deletions lib/gtfsLoader.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,12 @@ var rimraf = require('rimraf')
var unzip = require('unzip2')
var uuid = require('uuid')

const operations = require('./operations')
var util = require('./util.js')

var DATE_FORMAT = 'YYYYMMDD'
var lastAgencyId, numAgencies
let hasShapeTable = false

// convert dateString to moment
var toMoment = function (dateString) {
Expand All @@ -30,7 +32,7 @@ var toSecondsAfterMidnight = function (timeString) {
parseInt(timeArr[2])
}

var loadGtfs = function (extractedFolder, db, isSpatial, callback) {
var loadGtfs = function (extractedFolder, db, isSpatial, interpolateStopTimes, callback) {
numAgencies = 0
lastAgencyId = null

Expand Down Expand Up @@ -74,24 +76,31 @@ var loadGtfs = function (extractedFolder, db, isSpatial, callback) {
}

var postProcess = function (postProcessCallback) {
const postprocesses = []
if (isSpatial) {
var dialect = db.sequelize.options.dialect
if (['postgres', 'mysql'].indexOf(dialect) === -1) {
var err = Error('Spatial columns not supported for dialect ' + dialect + '.')
postProcessCallback(err)
} else {
async.series([
makeStopGeom,
makeShapeTable
],
function (err, results) {
postProcessCallback(err)
}
)
return postProcessCallback(err)
}
} else {
postProcessCallback()
postprocesses.push(makeStopGeom)
postprocesses.push(makeShapeTable)
}

if (interpolateStopTimes) {
postprocesses.push(doInterpolation)
}

async.series(
postprocesses,
function (err, results) {
postProcessCallback(err)
}
)
}

function doInterpolation (interpolationCallback) {
operations.interpolateStopTimes(db, interpolationCallback)
}

var makeStopGeom = function (seriesCallback) {
Expand Down Expand Up @@ -141,6 +150,10 @@ var loadGtfs = function (extractedFolder, db, isSpatial, callback) {
}

var makeShapeTable = function (seriesCallback) {
if (!hasShapeTable) {
console.log('shape table does not exist, skipping creation of shape_gis table')
return seriesCallback()
}
console.log('creating shape_gis table')
var processShape = function (shapePoint, shapeCallback) {
db.shape.findAll({
Expand Down Expand Up @@ -315,7 +328,7 @@ var insertCSVInTable = function (insertCfg, callback) {
// prepare processing function, but don't run it until file existance is confirmed
var processTable = function () {
insertCfg.model.sync({force: true}).then(function () {
var streamInserterCfg = util.makeInserterConfig(insertCfg.model)
var streamInserterCfg = util.makeStreamerConfig(insertCfg.model)
var inserter = dbStreamer.getInserter(streamInserterCfg)

inserter.connect(function (err) {
Expand Down Expand Up @@ -487,13 +500,13 @@ var loadCalendarDates = function (extractedFolder, db, callback) {
var processCalendarDates = function () {
db.calendar_date.sync({force: true}).then(function () {
var serviceIdsNotInCalendar = []
var calendarInserterConfig = util.makeInserterConfig(db.calendar)
var calendarInserterConfig = util.makeStreamerConfig(db.calendar)

// create inserter for calendar dates
var calendarInserter = dbStreamer.getInserter(calendarInserterConfig)
calendarInserter.connect(function (err, client) {
if (err) return callback(err)
var calendarDateInserterConfig = util.makeInserterConfig(db.calendar_date)
var calendarDateInserterConfig = util.makeStreamerConfig(db.calendar_date)
calendarDateInserterConfig.client = client
calendarDateInserterConfig.deferUntilEnd = true

Expand Down Expand Up @@ -602,6 +615,12 @@ var loadStopTimes = function (extractedFolder, db, callback) {
// change arrival and departure times into integer of seconds after midnight
line.arrival_time = toSecondsAfterMidnight(line.arrival_time)
line.departure_time = toSecondsAfterMidnight(line.departure_time)

if (line.arrival_time !== null && line.departure_time === null) {
line.departure_time = line.arrival_time
} else if (line.departure_time !== null && line.arrival_time === null) {
line.arrival_time = line.departure_time
}
return line
}
},
Expand All @@ -625,11 +644,28 @@ var loadFareRules = function (extractedFolder, db, callback) {
}

var loadShapes = function (extractedFolder, db, callback) {
insertCSVInTable({
filename: path.join(extractedFolder, 'shapes.txt'),
model: db.shape
},
callback)
const filename = path.join(extractedFolder, 'shapes.txt')
fs.stat(
filename,
function (err, stats) {
if (!err) {
// shapes.txt exists
hasShapeTable = true
insertCSVInTable(
{
filename: path.join(extractedFolder, 'shapes.txt'),
model: db.shape
},
callback
)
} else if (err && err.code === 'ENOENT') {
console.log(`${filename} <--- FILE NOT FOUND. SKIPPING.`)
callback()
} else if (err) {
callback(err)
}
}
)
}

var loadFrequencies = function (extractedFolder, db, callback) {
Expand Down Expand Up @@ -661,7 +697,14 @@ var loadFeedInfo = function (extractedFolder, db, callback) {
callback)
}

module.exports = function (downloadsDir, gtfsFileOrFolder, db, isSpatial, callback) {
module.exports = function (
downloadsDir,
gtfsFileOrFolder,
db,
isSpatial,
interpolateStopTimes,
callback
) {
// determine if gtfs is a file or folder
var gtfsPath = path.join(downloadsDir, gtfsFileOrFolder)
fs.lstat(gtfsPath, function (err, stats) {
Expand All @@ -671,7 +714,7 @@ module.exports = function (downloadsDir, gtfsFileOrFolder, db, isSpatial, callba
}

if (stats.isDirectory()) {
loadGtfs(gtfsPath, db, isSpatial, callback)
loadGtfs(gtfsPath, db, isSpatial, interpolateStopTimes, callback)
} else {
// create unzipper (assuming gtfs is in zip file)
var extractFolder = path.join(downloadsDir, 'google_transit')
Expand All @@ -680,7 +723,7 @@ module.exports = function (downloadsDir, gtfsFileOrFolder, db, isSpatial, callba
// create handler to process gtfs upon completion of unzip
extractor.on('close',
function () {
loadGtfs(extractFolder, db, isSpatial, callback)
loadGtfs(extractFolder, db, isSpatial, interpolateStopTimes, callback)
}
)

Expand Down
Loading

0 comments on commit 562b085

Please sign in to comment.