DuckDB [todo]

Author: derhuerst

.eslintrc.json

@@ -12,7 +12,7 @@
 		"node_modules"
 	],
 	"rules": {
-		"no-unused-vars": "off",
+		"no-unused-vars": "warn",
 		"no-irregular-whitespace": "off"
 	}
 }

.gitignore

@@ -10,8 +10,11 @@ pnpm-debug.log
 /shrinkwrap.yaml
 
 /test/amtrak-gtfs-2021-10-06
+/test/*.duckdb
 
 /*.gtfs
 /*.gtfs.zip
 /*.gtfs.tar.gz
 /*.gtfs.tar.zst
+
+/*.duckdb

Dockerfile

@@ -9,11 +9,6 @@ LABEL org.opencontainers.image.licenses="(Apache-2.0 AND Prosperity-3.0.0)"
 
 WORKDIR /app
 
-# Both moreutils (providing sponge) and postgresql-client (providing psql) are not required but come in handy for users.
-RUN apk add --no-cache \
-	postgresql-client \
-	moreutils
-
 ADD package.json /app
 RUN npm install --production && npm cache clean --force
 

cli.js

@@ -44,9 +44,6 @@ const {
 		'lower-case-lang-codes': {
 			type: 'boolean',
 		},
-		'stops-location-index': {
-			type: 'boolean',
-		},
 		'stats-by-route-date': {
 			type: 'string',
 		},
@@ -59,21 +56,6 @@ const {
 		'schema': {
 			type: 'string',
 		},
-		'postgraphile': {
-			type: 'boolean',
-		},
-		'postgraphile-password': {
-			type: 'string',
-		},
-		'postgrest': {
-			type: 'boolean',
-		},
-		'postgrest-password': {
-			type: 'string',
-		},
-		'postgrest-query-cost-limit': {
-			type: 'string',
-		},
 		'import-metadata': {
 			type: 'boolean',
 		}
@@ -84,7 +66,7 @@ const {
 if (flags.help) {
 	process.stdout.write(`
 Usage:
-    gtfs-to-sql [options] [--] <gtfs-file> ...
+    import-gtfs-into-duckdb [options] [--] <path-to-duckdb> <gtfs-file> ...
 Options:
     --silent                  -s  Don't show files being converted.
     --require-dependencies    -d  Require files that the specified GTFS files depend
@@ -102,8 +84,6 @@ Options:
     --routes-without-agency-id    Don't require routes.txt items to have an agency_id.
     --stops-without-level-id      Don't require stops.txt items to have a level_id.
                                     Default if levels.txt has not been provided.
-    --stops-location-index        Create a spatial index on stops.stop_loc for efficient
-                                    queries by geolocation.
     --lower-case-lang-codes       Accept Language Codes (e.g. in feed_info.feed_lang)
                                     with a different casing than the official BCP-47
                                     language tags (as specified by the GTFS spec),
@@ -124,34 +104,18 @@ Options:
                                     currently running trips over time, by hour.
                                     Like --stats-by-route-date, this flag accepts
                                     none, view & materialized-view.
-    --schema                      The schema to use for the database. Default: public
-                                    Even when importing into a schema other than \`public\`,
-                                    a function \`public.gtfs_via_postgres_import_version()\`
-                                    gets created, to ensure that multiple imports into the
-                                    same database are all made using the same version. See
-                                    also multiple-datasets.md in the docs.
-    --postgraphile                Tweak generated SQL for PostGraphile usage.
-                                    https://www.graphile.org/postgraphile/
-    --postgraphile-password       Password for the PostGraphile PostgreSQL user.
-                                    Default: $POSTGRAPHILE_PGPASSWORD, fallback random.
-    --postgrest                   Tweak generated SQL for PostgREST usage.
-                                    Please combine it with --schema.
-                                    https://postgrest.org/
-    --postgrest-password          Password for the PostgREST PostgreSQL user \`web_anon\`.
-                                    Default: $POSTGREST_PGPASSWORD, fallback random.
-    --postgrest-query-cost-limit  Define a cost limit [1] for queries executed by PostgREST
-                                    on behalf of a user. It is only enforced if
-                                    pg_plan_filter [2] is installed in the database!
-                                    Must be a positive float. Default: none
-                                    [1] https://www.postgresql.org/docs/14/using-explain.html
-                                    [2] https://github.com/pgexperts/pg_plan_filter
+    --schema                      The schema to use for the database. Default: main
+                                    May not contain \`.\`.
     --import-metadata             Create functions returning import metadata:
                                     - gtfs_data_imported_at (timestamp with time zone)
                                     - gtfs_via_postgres_version (text)
                                     - gtfs_via_postgres_options (jsonb)
+Notes:
+    If you just want to check if the GTFS data can be imported but don't care about the
+    resulting DuckDB database file, you can import into an in-memory database by specifying
+    \`:memory:\` as the <path-to-duckdb>.
 Examples:
-    gtfs-to-sql some-gtfs/*.txt | sponge | psql -b # import into PostgreSQL
-    gtfs-to-sql -u -- some-gtfs/*.txt | gzip >gtfs.sql.gz # generate a gzipped SQL dump
+    import-gtfs-into-duckdb some-gtfs.duckdb some-gtfs/*.txt
 
 [1] https://developers.google.com/transit/gtfs/reference/extended-route-types
 [2] https://groups.google.com/g/gtfs-changes/c/keT5rTPS7Y0/m/71uMz2l6ke0J
@@ -165,11 +129,11 @@ if (flags.version) {
 }
 
 const {basename, extname} = require('path')
-const {pipeline} = require('stream')
 const convertGtfsToSql = require('./index')
-const DataError = require('./lib/data-error')
 
-const files = args.map((file) => {
+const [pathToDb] = args
+
+const files = args.slice(1).map((file) => {
 	const name = basename(file, extname(file))
 	return {name, file}
 })
@@ -185,9 +149,7 @@ const opt = {
 	statsByRouteIdAndDate: flags['stats-by-route-date'] || 'none',
 	statsByAgencyIdAndRouteIdAndStopAndHour: flags['stats-by-agency-route-stop-hour'] || 'none',
 	statsActiveTripsByHour: flags['stats-active-trips-by-hour'] || 'none',
-	schema: flags['schema'] || 'public',
-	postgraphile: !!flags.postgraphile,
-	postgrest: !!flags.postgrest,
+	schema: flags['schema'] || 'main',
 	importMetadata: !!flags['import-metadata'],
 }
 if ('stops-without-level-id' in flags) {
@@ -196,31 +158,11 @@ if ('stops-without-level-id' in flags) {
 if ('lower-case-lang-codes' in flags) {
 	opt.lowerCaseLanguageCodes = flags['lower-case-lang-codes']
 }
-if ('postgraphile-password' in flags) {
-	opt.postgraphilePassword = flags['postgraphile-password']
-}
-if ('postgrest-password' in flags) {
-	opt.postgrestPassword = flags['postgrest-password']
-}
-if ('postgrest-query-cost-limit' in flags) {
-	const limit = parseFloat(flags['postgrest-query-cost-limit'])
-	if (!Number.isFinite(limit) || limit < 0) {
-		console.error('Invalid --postgrest-query-cost-limit value.')
-		process.exit(1)
-	}
-	opt.lowerCaseLanguageCodes = limit
-}
 
-pipeline(
-	convertGtfsToSql(files, opt),
-	process.stdout,
-	(err) => {
-		if (!err) return;
-		if (err instanceof DataError) {
-			console.error(String(err))
-		} else if (err.code !== 'EPIPE') {
-			console.error(err)
-		}
-		process.exit(1)
+convertGtfsToSql(pathToDb, files, opt)
+.catch((err) => {
+	if (err.code !== 'EPIPE') { // todo: check still necessary? we don't pipe anymore
+		console.error(err)
 	}
-)
+	process.exit(1)
+})

docs/import-metadata.md

@@ -12,7 +12,7 @@ SELECT gtfs_via_postgres_version()
 -- 4.5.3
 
 SELECT gtfs_via_postgres_options()
--- {"schema": "public", "silent": false, "importStart": 1681417454781, "postgraphile": false, "importMetadata": true, … }
+-- {"schema": "public", "silent": false, "importStart": 1681417454781, "importMetadata": true, … }
 SELECT (gtfs_via_postgres_options())['tripsWithoutShapeId']
 -- true
 ```

docs/multiple-datasets.md

@@ -8,14 +8,14 @@ As an example, let's import two datasets ([Paris](https://en.wikipedia.org/wiki/
 wget -U 'gtfs-via-postgres demo' -O paris.gtfs.zip 'https://eu.ftp.opendatasoft.com/stif/GTFS/IDFM-gtfs.zip'
 unzip -d paris.gtfs paris.gtfs.zip
 gtfs-to-sql --require-dependencies \
-	--schema paris -- paris.gtfs/*.txt \
-	| sponge | psql -b
+	--schema paris multiple-datasets.duckdb -- \
+	paris.gtfs/*.txt
 
 wget -U 'gtfs-via-postgres demo' -O berlin.gtfs.zip 'https://www.vbb.de/vbbgtfs'
 unzip -d berlin.gtfs berlin.gtfs.zip
 gtfs-to-sql --require-dependencies \
-	--schema berlin -- berlin.gtfs/*.txt \
-	| sponge | psql -b
+	--schema berlin multiple-datasets.duckdb -- \
+	berlin.gtfs/*.txt
 ```
 
 We can now do queries across both datasets, for example finding the geographically furthest 2 stops:
@@ -28,6 +28,7 @@ SELECT
 FROM
 	paris.stops paris,
 	berlin.stops berlin
+-- todo: does this operator work in DuckDB?
 ORDER BY paris.stop_loc <-> berlin.stop_loc DESC
 LIMIT 100
 ```

example.sh

@@ -1,36 +1,39 @@
 #!/bin/sh
 
 set -e
+set -u
 set -o pipefail
 
-2>&1 echo "importing into PostgreSQL:"
+rm -f example.duckdb
+
+2>&1 echo "importing into example.duckdb:"
 ./cli.js --ignore-unsupported --require-dependencies --trips-without-shape-id --silent \
-	node_modules/sample-gtfs-feed/gtfs/*.txt \
-	| sponge | psql -b
+	example.duckdb \
+	node_modules/sample-gtfs-feed/gtfs/*.txt
 
 2>&1 echo "\nfetching a connection during DST switch:"
-psql -c "$(cat <<- EOM
+duckdb -csv -c "$(cat <<- EOM
 	SELECT
 		trip_id, route_id,
 		from_stop_id, t_departure,
-		stop_sequence,
+		from_stop_sequence,
 		to_stop_id, t_arrival
-	FROM connections
+	FROM "main.connections"
 	WHERE trip_id = 'during-dst-1'
-	AND t_departure > '2019-03-31T01:55+01' AND t_departure < '2019-03-31T03:00+02'
+	AND t_departure > '2019-03-31T01:55:00+01:00' AND t_departure < '2019-03-31T03:00:00+02:00'
 	-- AND route_id = 'D'
 	-- AND from_stop_id = 'airport'
-EOM)"
+EOM)" example.duckdb
 
 2>&1 echo "\nfetching the departure at the same time:"
-psql -c "$(cat <<- EOM
+duckdb -csv -c "$(cat <<- EOM
 	SELECT
 		trip_id, route_id,
 		stop_id, t_departure,
 		stop_sequence
-	FROM arrivals_departures
+	FROM "main.arrivals_departures"
 	WHERE trip_id = 'during-dst-1'
-	AND t_departure > '2019-03-31T01:55+01' AND t_departure < '2019-03-31T03:00+02'
+	AND t_departure > '2019-03-31T01:55:00+01:00' AND t_departure < '2019-03-31T03:00:00+02:00'
 	-- AND route_id = 'D'
 	-- AND stop_id = 'airport'
-EOM)"
+EOM)" example.duckdb