diff --git a/README.md b/README.md index edd708f1..ea42ea6e 100644 --- a/README.md +++ b/README.md @@ -279,6 +279,23 @@ API along with your API token. } ``` +- When importing multiple agencies their IDs may overlap - specify prefix added to every ID in feed to maintain uniqueness. Provided string will be prepended to every ID in feed, number will be added (number IDs required, otherwise error will be thrown) + +```json +{ + "agencies": [ + { + "path": "/path/to/the/gtfs.zip", + "prefix": "A" + }, + { + "path": "/path/to/the/othergtfs.zip", + "prefix": 10000 + } + ] +} +``` + ### csvOptions {Object} Add options to be passed to [`csv-parse`](https://csv.js.org/parse/) with the key `csvOptions`. This is an optional parameter. diff --git a/lib/import.js b/lib/import.js index 40fad1bc..41818b9c 100644 --- a/lib/import.js +++ b/lib/import.js @@ -452,11 +452,35 @@ const importLines = (task, lines, model, totalLineCount) => { const placeholders = []; const values = []; - while (lines.length > 0) { - const line = lines.pop(); - placeholders.push(`(${fieldNames.map(() => '?').join(', ')})`); - values.push(...fieldNames.map((fieldName) => line[fieldName])); + //Let's check, if we will use prefixes at all - if not, don't repeat ifs in loop and do it once + if(task.prefix) + { + const prefixes = (() => { + if(typeof task.prefix === "number") { + return model.schema + .filter((column) => column.name !== 'id') + .map((column) => (column.prefix ? `${task.prefix}+` : '')); + } else { + return model.schema + .filter((column) => column.name !== 'id') + .map((column) => (column.prefix ? `'${task.prefix}'||` : '')); + }})(); + + while (lines.length > 0) { + const line = lines.pop(); + placeholders.push(`(${prefixes.map((pref) => pref + '?').join(', ')})`); + values.push(...fieldNames.map((fieldName) => line[fieldName])); + } } + else + { + while (lines.length > 0) { + const line = lines.pop(); + placeholders.push(`(${fieldNames.map(() => '?').join(', ')})`); + values.push(...fieldNames.map((fieldName) => line[fieldName])); + } + } + try { db.prepare( @@ -588,6 +612,7 @@ export async function importGtfs(initialConfig) { csvOptions: config.csvOptions || {}, ignoreDuplicates: config.ignoreDuplicates, sqlitePath: config.sqlitePath, + prefix: (agency.prefix ? agency.prefix : false), log, warn: logWarning, error: logError, diff --git a/models/gtfs-plus/calendar-attributes.js b/models/gtfs-plus/calendar-attributes.js index 7619df7a..4f40a027 100644 --- a/models/gtfs-plus/calendar-attributes.js +++ b/models/gtfs-plus/calendar-attributes.js @@ -7,6 +7,7 @@ const model = { name: 'service_id', type: 'varchar(255)', primary: true, + prefix: true, }, { name: 'service_description', diff --git a/models/gtfs-plus/directions.js b/models/gtfs-plus/directions.js index 2160c1e4..17bf625d 100644 --- a/models/gtfs-plus/directions.js +++ b/models/gtfs-plus/directions.js @@ -8,6 +8,7 @@ const model = { type: 'varchar(255)', required: true, primary: true, + prefix: true, }, { name: 'direction_id', diff --git a/models/gtfs-plus/route-attributes.js b/models/gtfs-plus/route-attributes.js index d9ba1871..8d51ba7a 100644 --- a/models/gtfs-plus/route-attributes.js +++ b/models/gtfs-plus/route-attributes.js @@ -7,6 +7,7 @@ const model = { name: 'route_id', type: 'varchar(255)', primary: true, + prefix: true, }, { name: 'category', diff --git a/models/gtfs-plus/stop-attributes.js b/models/gtfs-plus/stop-attributes.js index 51a810ba..9e2705c8 100644 --- a/models/gtfs-plus/stop-attributes.js +++ b/models/gtfs-plus/stop-attributes.js @@ -8,6 +8,7 @@ const model = { type: 'varchar(255)', required: true, primary: true, + prefix: true, }, { name: 'accessibility_id', diff --git a/models/gtfs-ride/board-alight.js b/models/gtfs-ride/board-alight.js index 2f09a41d..b5904b87 100644 --- a/models/gtfs-ride/board-alight.js +++ b/models/gtfs-ride/board-alight.js @@ -8,12 +8,14 @@ const model = { type: 'varchar(255)', required: true, index: true, + prefix: true, }, { name: 'stop_id', type: 'varchar(255)', required: true, index: true, + prefix: true, }, { name: 'stop_sequence', diff --git a/models/gtfs-ride/rider-trip.js b/models/gtfs-ride/rider-trip.js index 2f2ec55d..4864a549 100644 --- a/models/gtfs-ride/rider-trip.js +++ b/models/gtfs-ride/rider-trip.js @@ -7,21 +7,25 @@ const model = { name: 'rider_id', type: 'varchar(255)', primary: true, + prefix: true, }, { name: 'agency_id', type: 'varchar(255)', index: true, + prefix: true, }, { name: 'trip_id', type: 'varchar(255)', index: true, + prefix: true, }, { name: 'boarding_stop_id', type: 'varchar(255)', index: true, + prefix: true, }, { name: 'boarding_stop_sequence', @@ -33,6 +37,7 @@ const model = { name: 'alighting_stop_id', type: 'varchar(255)', index: true, + prefix: true, }, { name: 'alighting_stop_sequence', diff --git a/models/gtfs-ride/ridership.js b/models/gtfs-ride/ridership.js index c1720ad6..3d32e908 100644 --- a/models/gtfs-ride/ridership.js +++ b/models/gtfs-ride/ridership.js @@ -47,6 +47,7 @@ const model = { name: 'service_id', type: 'varchar(255)', index: true, + prefix: true, }, { name: 'monday', @@ -94,11 +95,13 @@ const model = { name: 'agency_id', type: 'varchar(255)', index: true, + prefix: true, }, { name: 'route_id', type: 'varchar(255)', index: true, + prefix: true, }, { name: 'direction_id', @@ -110,10 +113,12 @@ const model = { { name: 'trip_id', type: 'varchar(255)', + prefix: true, }, { name: 'stop_id', type: 'varchar(255)', + prefix: true, }, ], }; diff --git a/models/gtfs-ride/trip-capacity.js b/models/gtfs-ride/trip-capacity.js index ce073ff8..fc0f1c08 100644 --- a/models/gtfs-ride/trip-capacity.js +++ b/models/gtfs-ride/trip-capacity.js @@ -7,11 +7,13 @@ const model = { name: 'agency_id', type: 'varchar(255)', index: true, + prefix: true, }, { name: 'trip_id', type: 'varchar(255)', index: true, + prefix: true, }, { name: 'service_date', diff --git a/models/gtfs/agency.js b/models/gtfs/agency.js index d1731758..8549efdd 100644 --- a/models/gtfs/agency.js +++ b/models/gtfs/agency.js @@ -5,6 +5,7 @@ const model = { name: 'agency_id', type: 'varchar(255)', primary: true, + prefix: true, }, { name: 'agency_name', diff --git a/models/gtfs/areas.js b/models/gtfs/areas.js index 80a674cc..1759c2cc 100644 --- a/models/gtfs/areas.js +++ b/models/gtfs/areas.js @@ -6,6 +6,7 @@ const model = { type: 'varchar(255)', required: true, primary: true, + prefix: true, }, { name: 'area_name', diff --git a/models/gtfs/attributions.js b/models/gtfs/attributions.js index a8d37113..3d09fc88 100644 --- a/models/gtfs/attributions.js +++ b/models/gtfs/attributions.js @@ -6,18 +6,22 @@ const model = { type: 'varchar(255)', primary: true, required: true, + prefix: true, }, { name: 'agency_id', - type: 'varchar(255)', + type: 'varchar(255)', + prefix: true, }, { name: 'route_id', type: 'varchar(255)', + prefix: true, }, { name: 'trip_id', type: 'varchar(255)', + prefix: true, }, { name: 'organization_name', diff --git a/models/gtfs/calendar-dates.js b/models/gtfs/calendar-dates.js index fe39fe93..b053581e 100644 --- a/models/gtfs/calendar-dates.js +++ b/models/gtfs/calendar-dates.js @@ -6,6 +6,7 @@ const model = { type: 'varchar(255)', required: true, primary: true, + prefix: true, }, { name: 'date', diff --git a/models/gtfs/calendar.js b/models/gtfs/calendar.js index 778f6508..75ef9664 100644 --- a/models/gtfs/calendar.js +++ b/models/gtfs/calendar.js @@ -6,6 +6,7 @@ const model = { type: 'varchar(255)', required: true, primary: true, + prefix: true, }, { name: 'monday', diff --git a/models/gtfs/fare-attributes.js b/models/gtfs/fare-attributes.js index fc50756c..a7aa5bf1 100644 --- a/models/gtfs/fare-attributes.js +++ b/models/gtfs/fare-attributes.js @@ -6,6 +6,7 @@ const model = { type: 'varchar(255)', required: true, primary: true, + prefix: true, }, { name: 'price', @@ -33,6 +34,7 @@ const model = { { name: 'agency_id', type: 'varchar(255)', + prefix: true, }, { name: 'transfer_duration', diff --git a/models/gtfs/fare-leg-rules.js b/models/gtfs/fare-leg-rules.js index feb4eb8b..991680c6 100644 --- a/models/gtfs/fare-leg-rules.js +++ b/models/gtfs/fare-leg-rules.js @@ -4,27 +4,32 @@ const model = { { name: 'leg_group_id', type: 'varchar(255)', + prefix: true, }, { name: 'network_id', type: 'varchar(255)', primary: true, + prefix: true, }, { name: 'from_area_id', type: 'varchar(255)', primary: true, + prefix: true, }, { name: 'to_area_id', type: 'varchar(255)', primary: true, + prefix: true, }, { name: 'fare_product_id', type: 'varchar(255)', required: true, primary: true, + prefix: true, }, ], }; diff --git a/models/gtfs/fare-products.js b/models/gtfs/fare-products.js index 160913b6..dd6166c5 100644 --- a/models/gtfs/fare-products.js +++ b/models/gtfs/fare-products.js @@ -6,6 +6,7 @@ const model = { type: 'varchar(255)', required: true, primary: true, + prefix: true, }, { name: 'fare_product_name', @@ -15,6 +16,7 @@ const model = { name: 'fare_media_id', type: 'varchar(255)', primary: true, + prefix: true, }, { name: 'amount', diff --git a/models/gtfs/fare-rules.js b/models/gtfs/fare-rules.js index 5185361d..00fc694e 100644 --- a/models/gtfs/fare-rules.js +++ b/models/gtfs/fare-rules.js @@ -5,22 +5,27 @@ const model = { name: 'fare_id', type: 'varchar(255)', required: true, + prefix: true, }, { name: 'route_id', type: 'varchar(255)', + prefix: true, }, { name: 'origin_id', type: 'varchar(255)', + prefix: true, }, { name: 'destination_id', type: 'varchar(255)', + prefix: true, }, { name: 'contains_id', type: 'varchar(255)', + prefix: true, }, ], }; diff --git a/models/gtfs/fare-transfer-rules.js b/models/gtfs/fare-transfer-rules.js index 5b710d51..3a776c7e 100644 --- a/models/gtfs/fare-transfer-rules.js +++ b/models/gtfs/fare-transfer-rules.js @@ -5,11 +5,13 @@ const model = { name: 'from_leg_group_id', type: 'varchar(255)', primary: true, + prefix: true, }, { name: 'to_leg_group_id', type: 'varchar(255)', primary: true, + prefix: true, }, { name: 'transfer_count', @@ -20,6 +22,7 @@ const model = { { name: 'transfer_id', type: 'varchar(255)', + prefix: true, }, { name: 'duration_limit', @@ -44,6 +47,7 @@ const model = { name: 'fare_product_id', type: 'varchar(255)', primary: true, + prefix: true, }, ], }; diff --git a/models/gtfs/frequencies.js b/models/gtfs/frequencies.js index 95deebd8..afc1ad1f 100644 --- a/models/gtfs/frequencies.js +++ b/models/gtfs/frequencies.js @@ -6,6 +6,7 @@ const model = { type: 'varchar(255)', required: true, primary: true, + prefix: true, }, { name: 'start_time', diff --git a/models/gtfs/levels.js b/models/gtfs/levels.js index 3a7e56f7..cb633316 100644 --- a/models/gtfs/levels.js +++ b/models/gtfs/levels.js @@ -6,6 +6,7 @@ const model = { type: 'varchar(255)', primary: true, required: true, + prefix: true, }, { name: 'level_index', diff --git a/models/gtfs/pathways.js b/models/gtfs/pathways.js index ff03869e..3dc65604 100644 --- a/models/gtfs/pathways.js +++ b/models/gtfs/pathways.js @@ -6,16 +6,19 @@ const model = { type: 'varchar(255)', primary: true, required: true, + prefix: true, }, { name: 'from_stop_id', type: 'varchar(255)', required: true, + prefix: true, }, { name: 'to_stop_id', type: 'varchar(255)', required: true, + prefix: true, }, { name: 'pathway_mode', diff --git a/models/gtfs/routes.js b/models/gtfs/routes.js index 9a94681a..a08a7f8e 100644 --- a/models/gtfs/routes.js +++ b/models/gtfs/routes.js @@ -6,10 +6,12 @@ const model = { type: 'varchar(255)', primary: true, required: true, + prefix: true, }, { name: 'agency_id', type: 'varchar(255)', + prefix: true, }, { name: 'route_short_name', @@ -68,6 +70,7 @@ const model = { { name: 'network_id', type: 'varchar(255)', + prefix: true, }, ], }; diff --git a/models/gtfs/shapes.js b/models/gtfs/shapes.js index e85af3fc..30afe99d 100644 --- a/models/gtfs/shapes.js +++ b/models/gtfs/shapes.js @@ -6,6 +6,7 @@ const model = { type: 'varchar(255)', required: true, primary: true, + prefix: true, }, { name: 'shape_pt_lat', diff --git a/models/gtfs/stop-areas.js b/models/gtfs/stop-areas.js index 7426edc8..cfebc7cc 100644 --- a/models/gtfs/stop-areas.js +++ b/models/gtfs/stop-areas.js @@ -5,11 +5,13 @@ const model = { name: 'area_id', type: 'varchar(255)', required: true, + prefix: true, }, { name: 'stop_id', type: 'varchar(255)', required: true, + prefix: true, }, ], }; diff --git a/models/gtfs/stop-times.js b/models/gtfs/stop-times.js index 270bb17d..41edde0d 100644 --- a/models/gtfs/stop-times.js +++ b/models/gtfs/stop-times.js @@ -6,6 +6,7 @@ const model = { type: 'varchar(255)', required: true, primary: true, + prefix: true, }, { name: 'arrival_time', @@ -29,6 +30,7 @@ const model = { name: 'stop_id', type: 'varchar(255)', required: true, + prefix: true, }, { name: 'stop_sequence', diff --git a/models/gtfs/stops.js b/models/gtfs/stops.js index 5abbac87..75ba1f93 100644 --- a/models/gtfs/stops.js +++ b/models/gtfs/stops.js @@ -6,6 +6,7 @@ const model = { type: 'varchar(255)', primary: true, required: true, + prefix: true, }, { name: 'stop_code', @@ -41,6 +42,7 @@ const model = { { name: 'zone_id', type: 'varchar(255)', + prefix: true, }, { name: 'stop_url', @@ -70,6 +72,7 @@ const model = { { name: 'level_id', type: 'varchar(255)', + prefix: true, }, { name: 'platform_code', diff --git a/models/gtfs/transfers.js b/models/gtfs/transfers.js index 4dd37695..a516e91b 100644 --- a/models/gtfs/transfers.js +++ b/models/gtfs/transfers.js @@ -5,31 +5,37 @@ const model = { name: 'from_stop_id', type: 'varchar(255)', primary: true, + prefix: true, }, { name: 'to_stop_id', type: 'varchar(255)', primary: true, + prefix: true, }, { name: 'from_route_id', type: 'varchar(255)', primary: true, + prefix: true, }, { name: 'to_route_id', type: 'varchar(255)', primary: true, + prefix: true, }, { name: 'from_trip_id', type: 'varchar(255)', primary: true, + prefix: true, }, { name: 'to_trip_id', type: 'varchar(255)', primary: true, + prefix: true, }, { name: 'transfer_type', diff --git a/models/gtfs/translations.js b/models/gtfs/translations.js index d82524d6..04f7e6ba 100644 --- a/models/gtfs/translations.js +++ b/models/gtfs/translations.js @@ -28,11 +28,13 @@ const model = { name: 'record_id', type: 'varchar(255)', primary: true, + prefix: true, }, { name: 'record_sub_id', type: 'varchar(255)', primary: true, + prefix: true, }, { name: 'field_value', diff --git a/models/gtfs/trips.js b/models/gtfs/trips.js index f9a5a9f0..65d597d4 100644 --- a/models/gtfs/trips.js +++ b/models/gtfs/trips.js @@ -6,18 +6,21 @@ const model = { type: 'varchar(255)', required: true, index: true, + prefix: true, }, { name: 'service_id', type: 'varchar(255)', required: true, index: true, + prefix: true, }, { name: 'trip_id', type: 'varchar(255)', primary: true, required: true, + prefix: true, }, { name: 'trip_headsign', @@ -40,11 +43,13 @@ const model = { name: 'block_id', type: 'varchar(255)', index: true, + prefix: true, }, { name: 'shape_id', type: 'varchar(255)', index: true, + prefix: true, }, { name: 'wheelchair_accessible', diff --git a/models/non-standard/timetable-notes-references.js b/models/non-standard/timetable-notes-references.js index 28f5527b..10d0ccb8 100644 --- a/models/non-standard/timetable-notes-references.js +++ b/models/non-standard/timetable-notes-references.js @@ -5,26 +5,31 @@ const model = { { name: 'note_id', type: 'varchar(255)', + prefix: true, }, { name: 'timetable_id', type: 'varchar(255)', index: true, + prefix: true, }, { name: 'route_id', type: 'varchar(255)', index: true, + prefix: true, }, { name: 'trip_id', type: 'varchar(255)', index: true, + prefix: true, }, { name: 'stop_id', type: 'varchar(255)', index: true, + prefix: true, }, { name: 'stop_sequence', diff --git a/models/non-standard/timetable-notes.js b/models/non-standard/timetable-notes.js index c331d4db..23747321 100644 --- a/models/non-standard/timetable-notes.js +++ b/models/non-standard/timetable-notes.js @@ -6,6 +6,7 @@ const model = { name: 'note_id', type: 'varchar(255)', primary: true, + prefix: true, }, { name: 'symbol', diff --git a/models/non-standard/timetable-pages.js b/models/non-standard/timetable-pages.js index 8ea2638f..45ce8151 100644 --- a/models/non-standard/timetable-pages.js +++ b/models/non-standard/timetable-pages.js @@ -6,6 +6,7 @@ const model = { name: 'timetable_page_id', type: 'varchar(255)', primary: true, + prefix: true, }, { name: 'timetable_page_label', diff --git a/models/non-standard/timetable-stop-order.js b/models/non-standard/timetable-stop-order.js index a5cefd4a..08d4dbc2 100644 --- a/models/non-standard/timetable-stop-order.js +++ b/models/non-standard/timetable-stop-order.js @@ -6,15 +6,18 @@ const model = { name: 'id', type: 'integer', primary: true, + prefix: true, }, { name: 'timetable_id', type: 'varchar(255)', index: true, + prefix: true, }, { name: 'stop_id', type: 'varchar(255)', + prefix: true, }, { name: 'stop_sequence', diff --git a/models/non-standard/timetables.js b/models/non-standard/timetables.js index 23a3c156..18611bc1 100644 --- a/models/non-standard/timetables.js +++ b/models/non-standard/timetables.js @@ -6,14 +6,17 @@ const model = { name: 'id', type: 'integer', primary: true, + prefix: true, }, { name: 'timetable_id', type: 'varchar(255)', + prefix: true, }, { name: 'route_id', type: 'varchar(255)', + prefix: true, }, { name: 'direction_id', diff --git a/models/non-standard/trips-dated-vehicle-journey.js b/models/non-standard/trips-dated-vehicle-journey.js index ed18521c..5223c7f3 100644 --- a/models/non-standard/trips-dated-vehicle-journey.js +++ b/models/non-standard/trips-dated-vehicle-journey.js @@ -7,6 +7,7 @@ const model = { type: 'varchar(255)', required: true, index: true, + prefix: true, }, { name: 'operating_day_date', diff --git a/models/ods/deadhead-times.js b/models/ods/deadhead-times.js index b9594c1c..3506494c 100644 --- a/models/ods/deadhead-times.js +++ b/models/ods/deadhead-times.js @@ -7,12 +7,14 @@ const model = { name: 'id', type: 'integer', primary: true, + prefix: true, }, { name: 'deadhead_id', type: 'varchar(255)', required: true, index: true, + prefix: true, }, { name: 'arrival_time', @@ -37,10 +39,12 @@ const model = { { name: 'ops_location_id', type: 'varchar(255)', + prefix: true, }, { name: 'stop_id', type: 'varchar(255)', + prefix: true, }, { name: 'location_sequence', diff --git a/models/ods/deadheads.js b/models/ods/deadheads.js index 4a57aeef..3f64cd4e 100644 --- a/models/ods/deadheads.js +++ b/models/ods/deadheads.js @@ -8,42 +8,50 @@ const model = { type: 'varchar(255)', primary: true, required: true, + prefix: true, }, { name: 'service_id', type: 'varchar(255)', required: true, + prefix: true, }, { name: 'block_id', type: 'varchar(255)', required: true, index: true, + prefix: true, }, { name: 'shape_id', type: 'varchar(255)', index: true, + prefix: true, }, { name: 'to_trip_id', type: 'varchar(255)', index: true, + prefix: true, }, { name: 'from_trip_id', type: 'varchar(255)', index: true, + prefix: true, }, { name: 'to_deadhead_id', type: 'varchar(255)', index: true, + prefix: true, }, { name: 'from_deadhead_id', type: 'varchar(255)', index: true, + prefix: true, }, ], }; diff --git a/models/ods/ops-locations.js b/models/ods/ops-locations.js index b67eb773..bb6a1b85 100644 --- a/models/ods/ops-locations.js +++ b/models/ods/ops-locations.js @@ -8,6 +8,7 @@ const model = { type: 'varchar(255)', primary: true, required: true, + prefix: true, }, { name: 'ops_location_code', diff --git a/models/ods/run-events.js b/models/ods/run-events.js index 3de23809..ce5ce965 100644 --- a/models/ods/run-events.js +++ b/models/ods/run-events.js @@ -8,11 +8,13 @@ const model = { type: 'varchar(255)', primary: true, required: true, + prefix: true, }, { name: 'piece_id', type: 'varchar(255)', required: true, + prefix: true, }, { name: 'event_type', @@ -47,6 +49,7 @@ const model = { { name: 'event_from_location_id', type: 'varchar(255)', + prefix: true, }, { name: 'event_to_location_type', @@ -58,6 +61,7 @@ const model = { { name: 'event_to_location_id', type: 'varchar(255)', + prefix: true, }, ], }; diff --git a/test/mocha/import-gtfs.js b/test/mocha/import-gtfs.js index 443a1d14..59c2db08 100644 --- a/test/mocha/import-gtfs.js +++ b/test/mocha/import-gtfs.js @@ -63,16 +63,14 @@ describe('importGtfs():', function () { routes.length.should.equal(4); }); - it("should throw an error when importing from local filesystem which doesn't exist", async () => { - return importGtfs({ - ...config, - agencies: [ - { - path: '/does/not/exist', - }, - ], - }).should.be.rejected(); - }); + it("should throw an error when importing from local filesystem which doesn't exist", async () => importGtfs({ + ...config, + agencies: [ + { + path: '/does/not/exist', + }, + ], + }).should.be.rejected()); }); describe('Verify data imported into database', () => {