Prefixes for IDs

README.md

@@ -279,6 +279,23 @@ API along with your API token.
 }
 ```
 
+- When importing multiple agencies their IDs may overlap - specify prefix added to every ID in feed to maintain uniqueness. Provided string will be prepended to every ID in feed, number will be added (number IDs required, otherwise error will be thrown)
+
+```json
+{
+  "agencies": [
+    {
+      "path": "/path/to/the/gtfs.zip",
+      "prefix": "A"
+    },
+    {
+      "path": "/path/to/the/othergtfs.zip",
+      "prefix": 10000
+    }
+  ]
+}
+```
+
 ### csvOptions
 
 {Object} Add options to be passed to [`csv-parse`](https://csv.js.org/parse/) with the key `csvOptions`. This is an optional parameter.

lib/import.js

@@ -452,11 +452,35 @@ const importLines = (task, lines, model, totalLineCount) => {
   const placeholders = [];
   const values = [];
 
-  while (lines.length > 0) {
-    const line = lines.pop();
-    placeholders.push(`(${fieldNames.map(() => '?').join(', ')})`);
-    values.push(...fieldNames.map((fieldName) => line[fieldName]));
+  //Let's check, if we will use prefixes at all - if not, don't repeat ifs in loop and do it once
+  if(task.prefix)
+  {
+    const prefixes = (() => {
+    if(typeof task.prefix === "number") {
+      return model.schema
+      .filter((column) => column.name !== 'id')
+      .map((column) => (column.prefix ? `${task.prefix}+` : ''));
+    } else {
+      return model.schema
+      .filter((column) => column.name !== 'id')
+      .map((column) => (column.prefix ? `'${task.prefix}'||` : ''));
+    }})();
+
+    while (lines.length > 0) {
+      const line = lines.pop();
+      placeholders.push(`(${prefixes.map((pref) => pref + '?').join(', ')})`);
+      values.push(...fieldNames.map((fieldName) => line[fieldName]));
+    }
   }
+  else
+  {
+    while (lines.length > 0) {
+      const line = lines.pop();
+      placeholders.push(`(${fieldNames.map(() => '?').join(', ')})`);
+      values.push(...fieldNames.map((fieldName) => line[fieldName]));
+    }
+  }
+
 
   try {
     db.prepare(
@@ -588,6 +612,7 @@ export async function importGtfs(initialConfig) {
         csvOptions: config.csvOptions || {},
         ignoreDuplicates: config.ignoreDuplicates,
         sqlitePath: config.sqlitePath,
+        prefix: (agency.prefix ? agency.prefix : false),
         log,
         warn: logWarning,
         error: logError,

models/gtfs-plus/calendar-attributes.js

@@ -7,6 +7,7 @@ const model = {
       name: 'service_id',
       type: 'varchar(255)',
       primary: true,
+      prefix: true,
     },
     {
       name: 'service_description',

models/gtfs-plus/directions.js

@@ -8,6 +8,7 @@ const model = {
       type: 'varchar(255)',
       required: true,
       primary: true,
+      prefix: true,
     },
     {
       name: 'direction_id',

models/gtfs-plus/route-attributes.js

@@ -7,6 +7,7 @@ const model = {
       name: 'route_id',
       type: 'varchar(255)',
       primary: true,
+      prefix: true,
     },
     {
       name: 'category',

models/gtfs-plus/stop-attributes.js

@@ -8,6 +8,7 @@ const model = {
       type: 'varchar(255)',
       required: true,
       primary: true,
+      prefix: true,
     },
     {
       name: 'accessibility_id',

models/gtfs-ride/board-alight.js

@@ -8,12 +8,14 @@ const model = {
       type: 'varchar(255)',
       required: true,
       index: true,
+      prefix: true,
     },
     {
       name: 'stop_id',
       type: 'varchar(255)',
       required: true,
       index: true,
+      prefix: true,
     },
     {
       name: 'stop_sequence',

models/gtfs-ride/rider-trip.js

@@ -7,21 +7,25 @@ const model = {
       name: 'rider_id',
       type: 'varchar(255)',
       primary: true,
+      prefix: true,
     },
     {
       name: 'agency_id',
       type: 'varchar(255)',
       index: true,
+      prefix: true,
     },
     {
       name: 'trip_id',
       type: 'varchar(255)',
       index: true,
+      prefix: true,
     },
     {
       name: 'boarding_stop_id',
       type: 'varchar(255)',
       index: true,
+      prefix: true,
     },
     {
       name: 'boarding_stop_sequence',
@@ -33,6 +37,7 @@ const model = {
       name: 'alighting_stop_id',
       type: 'varchar(255)',
       index: true,
+      prefix: true,
     },
     {
       name: 'alighting_stop_sequence',

models/gtfs-ride/ridership.js

@@ -47,6 +47,7 @@ const model = {
       name: 'service_id',
       type: 'varchar(255)',
       index: true,
+      prefix: true,
     },
     {
       name: 'monday',
@@ -94,11 +95,13 @@ const model = {
       name: 'agency_id',
       type: 'varchar(255)',
       index: true,
+      prefix: true,
     },
     {
       name: 'route_id',
       type: 'varchar(255)',
       index: true,
+      prefix: true,
     },
     {
       name: 'direction_id',
@@ -110,10 +113,12 @@ const model = {
     {
       name: 'trip_id',
       type: 'varchar(255)',
+      prefix: true,
     },
     {
       name: 'stop_id',
       type: 'varchar(255)',
+      prefix: true,
     },
   ],
 };

models/gtfs-ride/trip-capacity.js

@@ -7,11 +7,13 @@ const model = {
       name: 'agency_id',
       type: 'varchar(255)',
       index: true,
+      prefix: true,
     },
     {
       name: 'trip_id',
       type: 'varchar(255)',
       index: true,
+      prefix: true,
     },
     {
       name: 'service_date',

models/gtfs/agency.js

@@ -5,6 +5,7 @@ const model = {
       name: 'agency_id',
       type: 'varchar(255)',
       primary: true,
+      prefix: true,
     },
     {
       name: 'agency_name',

models/gtfs/areas.js

@@ -6,6 +6,7 @@ const model = {
       type: 'varchar(255)',
       required: true,
       primary: true,
+      prefix: true,
     },
     {
       name: 'area_name',

models/gtfs/attributions.js

@@ -6,18 +6,22 @@ const model = {
       type: 'varchar(255)',
       primary: true,
       required: true,
+      prefix: true,
     },
     {
       name: 'agency_id',
-      type: 'varchar(255)',
+      type: 'varchar(255)',      
+      prefix: true,
     },
     {
       name: 'route_id',
       type: 'varchar(255)',
+      prefix: true,      
     },
     {
       name: 'trip_id',
       type: 'varchar(255)',
+      prefix: true,
     },
     {
       name: 'organization_name',

models/gtfs/calendar-dates.js

@@ -6,6 +6,7 @@ const model = {
       type: 'varchar(255)',
       required: true,
       primary: true,
+      prefix: true,
     },
     {
       name: 'date',

models/gtfs/calendar.js

@@ -6,6 +6,7 @@ const model = {
       type: 'varchar(255)',
       required: true,
       primary: true,
+      prefix: true,
     },
     {
       name: 'monday',

models/gtfs/fare-attributes.js

@@ -6,6 +6,7 @@ const model = {
       type: 'varchar(255)',
       required: true,
       primary: true,
+      prefix: true,
     },
     {
       name: 'price',
@@ -33,6 +34,7 @@ const model = {
     {
       name: 'agency_id',
       type: 'varchar(255)',
+      prefix: true,
     },
     {
       name: 'transfer_duration',

models/gtfs/fare-leg-rules.js

@@ -4,27 +4,32 @@ const model = {
     {
       name: 'leg_group_id',
       type: 'varchar(255)',
+      prefix: true,
     },
     {
       name: 'network_id',
       type: 'varchar(255)',
       primary: true,
+      prefix: true,
     },
     {
       name: 'from_area_id',
       type: 'varchar(255)',
       primary: true,
+      prefix: true,
     },
     {
       name: 'to_area_id',
       type: 'varchar(255)',
       primary: true,
+      prefix: true,
     },
     {
       name: 'fare_product_id',
       type: 'varchar(255)',
       required: true,
       primary: true,
+      prefix: true,
     },
   ],
 };

models/gtfs/fare-products.js

@@ -6,6 +6,7 @@ const model = {
       type: 'varchar(255)',
       required: true,
       primary: true,
+      prefix: true,
     },
     {
       name: 'fare_product_name',
@@ -15,6 +16,7 @@ const model = {
       name: 'fare_media_id',
       type: 'varchar(255)',
       primary: true,
+      prefix: true,
     },
     {
       name: 'amount',

models/gtfs/fare-rules.js

@@ -5,22 +5,27 @@ const model = {
       name: 'fare_id',
       type: 'varchar(255)',
       required: true,
+      prefix: true,
     },
     {
       name: 'route_id',
       type: 'varchar(255)',
+      prefix: true,
     },
     {
       name: 'origin_id',
       type: 'varchar(255)',
+      prefix: true,
     },
     {
       name: 'destination_id',
       type: 'varchar(255)',
+      prefix: true,
     },
     {
       name: 'contains_id',
       type: 'varchar(255)',
+      prefix: true,
     },
   ],
 };

models/gtfs/fare-transfer-rules.js

@@ -5,11 +5,13 @@ const model = {
       name: 'from_leg_group_id',
       type: 'varchar(255)',
       primary: true,
+      prefix: true,
     },
     {
       name: 'to_leg_group_id',
       type: 'varchar(255)',
       primary: true,
+      prefix: true,
     },
     {
       name: 'transfer_count',
@@ -20,6 +22,7 @@ const model = {
     {
       name: 'transfer_id',
       type: 'varchar(255)',
+      prefix: true,
     },
     {
       name: 'duration_limit',
@@ -44,6 +47,7 @@ const model = {
       name: 'fare_product_id',
       type: 'varchar(255)',
       primary: true,
+      prefix: true,
     },
   ],
 };

models/gtfs/frequencies.js

@@ -6,6 +6,7 @@ const model = {
       type: 'varchar(255)',
       required: true,
       primary: true,
+      prefix: true,
     },
     {
       name: 'start_time',

models/gtfs/levels.js

@@ -6,6 +6,7 @@ const model = {
       type: 'varchar(255)',
       primary: true,
       required: true,
+      prefix: true,
     },
     {
       name: 'level_index',