WIP

Author: derhuerst

index.js

@@ -61,6 +61,8 @@ const convertGtfsToSql = async (pathToDb, files, opt = {}) => {
 			dep: [],
 		},
 
+		// todo: currently doesn't fail if *neither* calendar nor calendar_dates is present!
+
 		// special handling of calendar/calendar_dates:
 		// service_days relies on *both* calendar's & calendar_dates' tables to
 		// be present, so we add mock tasks here. Each of these mock tasks get

lib/deps.js

@@ -79,7 +79,8 @@ const getDependencies = (opt, files) => {
 			),
 			...(files.includes('calendar') ? ['calendar'] : []),
 			...(files.includes('calendar_dates') ? ['calendar_dates'] : []),
-			// not supported yet: attributions
+			// todo: support attributions
+			// related: https://github.com/hove-io/transit_model/pull/994/files
 			// not supported yet: fare_attributes/fare_rules
 			// not supported yet: frequencies
 			// not supported yet: transfers

lib/feed_info.js

@@ -19,6 +19,7 @@ CREATE TABLE feed_info (
 	feed_start_date DATE,
 	feed_end_date DATE,
 	feed_version TEXT,
+	-- todo: optional, how to handle this?
 	feed_contact_email TEXT,
 	feed_contact_url TEXT
 );

lib/shapes.js

@@ -30,6 +30,8 @@ CREATE TABLE shapes (
 
 INSERT INTO shapes
 -- WITH
+--	-- todo: explicitly specify if we want materialization!
+--	-- see also https://github.com/duckdb/duckdb/pull/17459
 -- 	csv_columns AS (
 -- 		SELECT list(column_name) AS cols
 -- 		FROM (
@@ -42,6 +44,8 @@ INSERT INTO shapes
 -- 			)
 -- 		) columns
 -- 	),
+--	-- todo: explicitly specify if we want materialization!
+--	-- see also https://github.com/duckdb/duckdb/pull/17459
 -- 	table_columns AS (
 -- 		SELECT list(column_name)
 -- 		FROM (

lib/stats_active_trips_by_hour.js

@@ -22,39 +22,53 @@ const createStatsActiveTripsByHourView = async (db, _, opt) => {
 -- see also https://github.com/duckdb/duckdb/discussions/3638
 CREATE TABLE feed_time_frame AS
 WITH
+	-- todo: explicitly specify if we want materialization!
+	-- see also https://github.com/duckdb/duckdb/pull/17459
 	dates AS (
 		SELECT
 			min("date") AS min,
 			max("date") AS max
 		FROM service_days
 	),
+	-- todo: explicitly specify if we want materialization!
+	-- see also https://github.com/duckdb/duckdb/pull/17459
 	date_offset AS (
 		SELECT
 			largest AS o
 		FROM largest_arr_dep_time
 	),
+	-- todo: explicitly specify if we want materialization!
+	-- see also https://github.com/duckdb/duckdb/pull/17459
 	date_min_max AS (
 		SELECT
 			-- todo
 			date_trunc('day', min + o) AS min,
 			date_trunc('day', max - o) AS max
 		FROM dates, date_offset 
 	),
+	-- todo: explicitly specify if we want materialization!
+	-- see also https://github.com/duckdb/duckdb/pull/17459
 	min_dep AS (
 		SELECT min("t_departure") AS t
 		FROM arrivals_departures, date_min_max
 		WHERE date <= (SELECT min FROM date_min_max)
 	),
+	-- todo: explicitly specify if we want materialization!
+	-- see also https://github.com/duckdb/duckdb/pull/17459
 	min_arr AS (
 		SELECT min("t_arrival") AS t
 		FROM arrivals_departures, date_min_max
 		WHERE date <= (SELECT min FROM date_min_max)
 	),
+	-- todo: explicitly specify if we want materialization!
+	-- see also https://github.com/duckdb/duckdb/pull/17459
 	max_dep AS (
 		SELECT min("t_departure") AS t
 		FROM arrivals_departures, date_min_max
 		WHERE date >= (SELECT max FROM date_min_max)
 	),
+	-- todo: explicitly specify if we want materialization!
+	-- see also https://github.com/duckdb/duckdb/pull/17459
 	max_arr AS (
 		SELECT min("t_arrival") AS t
 		FROM arrivals_departures, date_min_max

lib/stop_times.js

@@ -40,7 +40,7 @@ CREATE TABLE stop_times (
 	arrival_time INTERVAL,
 	departure_time INTERVAL,
 	stop_id TEXT NOT NULL,
-	FOREIGN KEY (stop_id) REFERENCES stops,
+	FOREIGN KEY (stop_id) REFERENCES stops(stop_id),
 	stop_sequence INT NOT NULL,
 	stop_sequence_consec INT,
 	stop_headsign TEXT,
@@ -87,7 +87,11 @@ FROM read_csv(
 		drop_off_type: 'INTEGER',
 		${has_shape_dist_traveled ? `shape_dist_traveled: 'REAL',` : ``}
 		${has_timepoint ? `timepoint: 'INTEGER',` : ``}
-	}
+	},
+	-- skip DuckDB's automatic format detection
+	delim = ',',
+	quote = '"',
+	escape = '"'
 );
 
 -- For a primary key, DuckDB doesn't create an index automatically.
@@ -107,6 +111,8 @@ CREATE INDEX stop_times_trip_id_stop_sequence_consec ON stop_times (trip_id, sto
 -- see also https://github.com/duckdb/duckdb/discussions/3638
 CREATE TABLE largest_arr_dep_time AS
 WITH
+	-- todo: explicitly specify if we want materialization!
+	-- see also https://github.com/duckdb/duckdb/pull/17459
 	largest_departure_time AS (
 		SELECT departure_time
 		FROM stop_times stop_times
@@ -119,6 +125,8 @@ WITH
 		ORDER BY departure_time DESC
 		LIMIT 1
 	),
+	-- todo: explicitly specify if we want materialization!
+	-- see also https://github.com/duckdb/duckdb/pull/17459
 	largest_arrival_time AS (
 		SELECT arrival_time
 		FROM stop_times stop_times

lib/stops.js

@@ -82,6 +82,7 @@ CREATE TABLE stops (
 	-- In stops.txt, *any* row's parent_station might reference *any* other row. Essentially, stops.txt describes a tree.
 	-- As of DuckDB v1.0.0, it *seems* like adding a foreign key constraint here doesn't work, even if we order the stops to put parents before their children (see below).
 	-- todo: Report this with DuckDB? Alternatively, add the constraint after the import (see below).
+	-- maybe related? https://github.com/duckdb/duckdb/issues/10574
 	-- FOREIGN KEY (parent_station) REFERENCES stops,
 	stop_timezone TEXT,
 	FOREIGN KEY (stop_timezone) REFERENCES valid_timezones,
@@ -96,6 +97,8 @@ INSERT INTO stops
 -- todo: handle the CSV file having *additional* columns
 BY NAME
 WITH RECURSIVE
+	-- todo: explicitly specify if we want materialization!
+	-- see also https://github.com/duckdb/duckdb/pull/17459
 	stops AS (
 		SELECT
 			${has_stop_code ? `` : `NULL AS stop_code,`}
@@ -146,6 +149,8 @@ WITH RECURSIVE
 		)
 	),
 	-- order the stops to put parents before their children
+	-- todo: explicitly specify if we want materialization!
+	-- see also https://github.com/duckdb/duckdb/pull/17459
 	stops_sorted_by_parents AS (
 		(
 			SELECT