DuckDB rewrite: use ISO 8601 timestamps including seconds 📝

derhuerst · derhuerst · commit 163fcd69b86a · 2025-08-13T17:40:46.000+02:00
diff --git a/benchmark/arrs_deps_by_route_name_and_time.sql b/benchmark/arrs_deps_by_route_name_and_time.sql
@@ -1,6 +1,6 @@
 SELECT *
 FROM arrivals_departures
 WHERE route_short_name = 'S1'
-AND t_departure >= '2022-08-09T07:10+02' AND t_departure <= '2022-08-09T07:30+02'
-AND date >= dates_filter_min('2022-08-09T07:10+02')
-AND date <= dates_filter_max('2022-08-09T07:30+02')
+AND t_departure >= '2022-08-09T07:10:00+02' AND t_departure <= '2022-08-09T07:30:00+02'
+AND date >= dates_filter_min('2022-08-09T07:10:00+02')
+AND date <= dates_filter_max('2022-08-09T07:30:00+02')
diff --git a/benchmark/arrs_deps_by_station_and_time.sql b/benchmark/arrs_deps_by_station_and_time.sql
@@ -1,6 +1,6 @@
 SELECT *
 FROM arrivals_departures
 WHERE station_id = 'de:11000:900100001' -- S+U Friedrichstr. (Berlin)
-AND t_departure >= '2022-08-09T07:10+02' AND t_departure <= '2022-08-09T07:30+02'
-AND date >= dates_filter_min('2022-08-09T07:10+02')
-AND date <= dates_filter_max('2022-08-09T07:30+02')
+AND t_departure >= '2022-08-09T07:10:00+02' AND t_departure <= '2022-08-09T07:30:00+02'
+AND date >= dates_filter_min('2022-08-09T07:10:00+02')
+AND date <= dates_filter_max('2022-08-09T07:30:00+02')
diff --git a/benchmark/arrs_deps_by_station_and_time_seq_0.sql b/benchmark/arrs_deps_by_station_and_time_seq_0.sql
@@ -1,7 +1,7 @@
 SELECT *
 FROM arrivals_departures
 WHERE station_id = 'de:11000:900100001' -- S+U Friedrichstr. (Berlin)
-AND t_departure >= '2022-08-09T07:10+02' AND t_departure <= '2022-08-09T07:30+02'
-AND date >= dates_filter_min('2022-08-09T07:10+02')
-AND date <= dates_filter_max('2022-08-09T07:30+02')
+AND t_departure >= '2022-08-09T07:10:00+02' AND t_departure <= '2022-08-09T07:30:00+02'
+AND date >= dates_filter_min('2022-08-09T07:10:00+02')
+AND date <= dates_filter_max('2022-08-09T07:30:00+02')
 AND stop_sequence = 0
diff --git a/benchmark/arrs_deps_by_stop_and_time.sql b/benchmark/arrs_deps_by_stop_and_time.sql
@@ -1,6 +1,6 @@
 SELECT *
 FROM arrivals_departures
 WHERE stop_id = 'de:11000:900100001::4' -- S+U Friedrichstr. (Berlin)
-AND t_departure >= '2022-08-09T07:10+02' AND t_departure <= '2022-08-09T07:30+02'
-AND date >= dates_filter_min('2022-08-09T07:10+02')
-AND date <= dates_filter_max('2022-08-09T07:30+02')
+AND t_departure >= '2022-08-09T07:10:00+02' AND t_departure <= '2022-08-09T07:30:00+02'
+AND date >= dates_filter_min('2022-08-09T07:10:00+02')
+AND date <= dates_filter_max('2022-08-09T07:30:00+02')
diff --git a/benchmark/arrs_deps_by_time.sql b/benchmark/arrs_deps_by_time.sql
@@ -1,5 +1,5 @@
 SELECT *
 FROM arrivals_departures
-WHERE t_departure >= '2022-08-09T07:10+02' AND t_departure <= '2022-08-09T07:30+02'
-AND date >= dates_filter_min('2022-08-09T07:10+02'::timestamp with time zone)
-AND date <= dates_filter_max('2022-08-09T07:30+02'::timestamp with time zone)
+WHERE t_departure >= '2022-08-09T07:10:00+02' AND t_departure <= '2022-08-09T07:30:00+02'
+AND date >= dates_filter_min('2022-08-09T07:10:00+02'::timestamp with time zone)
+AND date <= dates_filter_max('2022-08-09T07:30:00+02'::timestamp with time zone)
diff --git a/benchmark/arrs_deps_by_time_manual.sql b/benchmark/arrs_deps_by_time_manual.sql
@@ -1,5 +1,5 @@
 SELECT *
 FROM arrivals_departures
-WHERE t_departure >= '2022-08-09T07:10+02' AND t_departure <= '2022-08-09T07:30+02'
+WHERE t_departure >= '2022-08-09T07:10:00+02' AND t_departure <= '2022-08-09T07:30:00+02'
 AND date >= '2022-08-08'
 AND date <= '2022-08-09'
diff --git a/benchmark/connections_by_route_name_and_time.sql b/benchmark/connections_by_route_name_and_time.sql
@@ -1,6 +1,6 @@
 SELECT *
 FROM connections
 WHERE route_short_name = 'S1'
-AND t_departure >= '2022-08-09T07:10+02' AND t_departure <= '2022-08-09T07:30+02'
-AND date >= dates_filter_min('2022-08-09T07:10+02')
-AND date <= dates_filter_max('2022-08-09T07:30+02')
+AND t_departure >= '2022-08-09T07:10:00+02' AND t_departure <= '2022-08-09T07:30:00+02'
+AND date >= dates_filter_min('2022-08-09T07:10:00+02')
+AND date <= dates_filter_max('2022-08-09T07:30:00+02')
diff --git a/benchmark/connections_by_station_and_time.sql b/benchmark/connections_by_station_and_time.sql
@@ -1,6 +1,6 @@
 SELECT *
 FROM connections
 WHERE from_station_id = 'de:11000:900100001' -- S+U Friedrichstr. (Berlin)
-AND t_departure >= '2022-08-09T07:10+02' AND t_departure <= '2022-08-09T07:30+02'
-AND date >= dates_filter_min('2022-08-09T07:10+02')
-AND date <= dates_filter_max('2022-08-09T07:30+02')
+AND t_departure >= '2022-08-09T07:10:00+02' AND t_departure <= '2022-08-09T07:30:00+02'
+AND date >= dates_filter_min('2022-08-09T07:10:00+02')
+AND date <= dates_filter_max('2022-08-09T07:30:00+02')
diff --git a/benchmark/connections_by_station_and_time_seq_0.sql b/benchmark/connections_by_station_and_time_seq_0.sql
@@ -1,7 +1,7 @@
 SELECT *
 FROM connections
 WHERE from_station_id = 'de:11000:900100001' -- S+U Friedrichstr. (Berlin)
-AND t_departure >= '2022-08-09T07:10+02' AND t_departure <= '2022-08-09T07:30+02'
-AND date >= dates_filter_min('2022-08-09T07:10+02')
-AND date <= dates_filter_max('2022-08-09T07:30+02')
+AND t_departure >= '2022-08-09T07:10:00+02' AND t_departure <= '2022-08-09T07:30:00+02'
+AND date >= dates_filter_min('2022-08-09T07:10:00+02')
+AND date <= dates_filter_max('2022-08-09T07:30:00+02')
 AND from_stop_sequence = 0
diff --git a/benchmark/connections_by_stop_and_time.sql b/benchmark/connections_by_stop_and_time.sql
@@ -1,6 +1,6 @@
 SELECT *
 FROM connections
 WHERE from_stop_id = 'de:11000:900100001::4' -- S+U Friedrichstr. (Berlin)
-AND t_departure >= '2022-08-09T07:10+02' AND t_departure <= '2022-08-09T07:30+02'
-AND date >= dates_filter_min('2022-08-09T07:10+02')
-AND date <= dates_filter_max('2022-08-09T07:30+02')
+AND t_departure >= '2022-08-09T07:10:00+02' AND t_departure <= '2022-08-09T07:30:00+02'
+AND date >= dates_filter_min('2022-08-09T07:10:00+02')
+AND date <= dates_filter_max('2022-08-09T07:30:00+02')
diff --git a/benchmark/connections_by_time.sql b/benchmark/connections_by_time.sql
@@ -1,7 +1,7 @@
 SELECT *
 FROM connections
-WHERE t_departure >= '2022-08-09T07:10+02' AND t_departure <= '2022-08-09T07:30+02'
-AND date >= dates_filter_min('2022-08-09T07:10+02'::timestamp with time zone)
-AND date <= dates_filter_max('2022-08-09T07:30+02'::timestamp with time zone)
+WHERE t_departure >= '2022-08-09T07:10:00+02' AND t_departure <= '2022-08-09T07:30:00+02'
+AND date >= dates_filter_min('2022-08-09T07:10:00+02'::timestamp with time zone)
+AND date <= dates_filter_max('2022-08-09T07:30:00+02'::timestamp with time zone)
 ORDER BY t_departure
 LIMIT 100
diff --git a/benchmark/connections_by_time_manual.sql b/benchmark/connections_by_time_manual.sql
@@ -1,6 +1,6 @@
 SELECT *
 FROM connections
-WHERE t_departure >= '2022-08-09T07:10+02' AND t_departure <= '2022-08-09T07:30+02'
+WHERE t_departure >= '2022-08-09T07:10:00+02' AND t_departure <= '2022-08-09T07:30:00+02'
 AND date >= '2022-08-08'
 AND date <= '2022-08-09'
 ORDER BY t_departure
diff --git a/example.sh b/example.sh
@@ -18,7 +18,7 @@ psql -c "$(cat <<- EOM
 		to_stop_id, t_arrival
 	FROM connections
 	WHERE trip_id = 'during-dst-1'
-	AND t_departure > '2019-03-31T01:55+01' AND t_departure < '2019-03-31T03:00+02'
+	AND t_departure > '2019-03-31T01:55:00+01:00' AND t_departure < '2019-03-31T03:00:00+02:00'
 	-- AND route_id = 'D'
 	-- AND from_stop_id = 'airport'
 EOM)"
@@ -31,7 +31,7 @@ psql -c "$(cat <<- EOM
 		stop_sequence
 	FROM arrivals_departures
 	WHERE trip_id = 'during-dst-1'
-	AND t_departure > '2019-03-31T01:55+01' AND t_departure < '2019-03-31T03:00+02'
+	AND t_departure > '2019-03-31T01:55:00+01:00' AND t_departure < '2019-03-31T03:00:00+02:00'
 	-- AND route_id = 'D'
 	-- AND stop_id = 'airport'
 EOM)"
diff --git a/readme.md b/readme.md
@@ -101,13 +101,13 @@ In addition to a table for each GTFS file, `gtfs-via-duckdb` adds these views to
 - `stats_by_agency_route_stop_hour` provides the number of arrivals/departures by agency ID, route ID, stop ID & hour. – [read more](docs/analysis/feed-by-agency-route-stop-and-hour.md)
 - In contrast to `stats_by_route_date` & `stats_by_agency_route_stop_hour`, `stats_active_trips_by_hour` provides the number of *currently running* trips for each hour in the feeds period of time.
 
-As an example, we're going to use the `arrivals_departures` view to query all *absolute* departures at `de:11000:900120003` (*S Ostkreuz Bhf (Berlin)*) between `2022-03-23T12:30+01` and  `2022-03-23T12:35+01`:
+As an example, we're going to use the `arrivals_departures` view to query all *absolute* departures at `de:11000:900120003` (*S Ostkreuz Bhf (Berlin)*) between `2022-03-23T12:30:00+01` and  `2022-03-23T12:35:00+01`:
 
 ```sql
 SELECT *
 FROM arrivals_departures
 WHERE station_id = 'de:11000:900120003'
-AND t_departure >= '2022-03-23T12:30+01' AND t_departure <= '2022-03-23T12:35+01'
+AND t_departure >= '2022-03-23T12:30:00+01' AND t_departure <= '2022-03-23T12:35:00+01'
 ```
 
 `route_id` | `route_short_name` | `route_type` | `trip_id` | `date` | `stop_sequence` | `t_arrival` | `t_departure` | `stop_id` | `stop_name` | `station_id` | `station_name`
@@ -275,14 +275,14 @@ This means that, in order to determine all *absolute* points in time where a par
 
 Let's consider two examples:
 
-- A `departure_time` of `26:59:00` with a trip running on `2021-03-01`: The time, applied to this specific date, "extends" into the following day, so it actually departs at `2021-03-02T02:59+01`.
-- A departure time of `03:01:00` with a trip running on `2021-03-28`: This is when the standard -> DST switch happens in the `Europe/Berlin` timezone. Because the dep. time refers to noon - 12h (*not* to midnight), it actually happens at `2021-03-28T03:01+02` which is *not* `3h1m` after `2021-03-28T00:00+01`.
+- A `departure_time` of `26:59:00` with a trip running on `2021-03-01`: The time, applied to this specific date, "extends" into the following day, so it actually departs at `2021-03-02T02:59:00+01`.
+- A departure time of `03:01:00` with a trip running on `2021-03-28`: This is when the standard -> DST switch happens in the `Europe/Berlin` timezone. Because the dep. time refers to noon - 12h (*not* to midnight), it actually happens at `2021-03-28T03:01:00+02` which is *not* `3h1m` after `2021-03-28T00:00:00+01`.
 
 `gtfs-via-duckdb` always prioritizes correctness over speed. Because it follows the GTFS semantics, when filtering `arrivals_departures` by *absolute* departure date+time, it cannot automatically filter `service_days` (which is `calendar` and `calendar_dates` combined), because **even a date *before* the date of the desired departure time frame might still end up *within*, when combined with a `departure_time` of e.g. `27:30:00`**; Instead, it has to consider all `service_days` and apply the `departure_time` to all of them to check if they're within the range.
 
 However, if you determine your feed's largest `arrival_time`/`departure_time`, you can filter on `date` when querying `arrivals_departures`; This allows PostgreSQL to reduce the number of joins and calendar calculations by orders of magnitude, speeding up your queries significantly. `gtfs-via-postgres` provides two low-level helper functions `largest_arrival_time()` & `largest_departure_time()` for this, as well as two high-level helper functions `dates_filter_min(t_min)` & `dates_filter_max(t_max)` (see below).
 
-For example, when querying all *absolute* departures at `de:11000:900120003` (*S Ostkreuz Bhf (Berlin)*) between `2022-03-23T12:30+01` and  `2022-03-23T12:35+01` within the [2022-02-25 *VBB* feed](https://vbb-gtfs.jannisr.de/2022-02-25/), filtering by `date` speeds it up nicely (Apple M1, PostgreSQL 14.2):
+For example, when querying all *absolute* departures at `de:11000:900120003` (*S Ostkreuz Bhf (Berlin)*) between `2022-03-23T12:30:00+01` and  `2022-03-23T12:35:00+01` within the [2022-02-25 *VBB* feed](https://vbb-gtfs.jannisr.de/2022-02-25/), filtering by `date` speeds it up nicely (Apple M1, PostgreSQL 14.2):
 
 `station_id` filter | `date` filter | query time | nr of results
 -|-|-|-
@@ -300,10 +300,10 @@ Using `dates_filter_min(t_min)` & `dates_filter_max(t_max)`, we can easily filte
 SELECT *
 FROM arrivals_departures
 -- filter by absolute departure date+time
-WHERE t_departure >= '2022-03-23T12:30+01' AND t_departure <= '2022-03-23T12:35+01'
+WHERE t_departure >= '2022-03-23T12:30:00+01' AND t_departure <= '2022-03-23T12:35:00+01'
 -- allow "cutoffs" by filtering by date
-AND "date" >= dates_filter_min('2022-03-23T12:30+01') -- evaluates to 2023-03-22
-AND "date" <= dates_filter_max('2022-03-23T12:35+01') -- evaluates to 2023-03-23
+AND "date" >= dates_filter_min('2022-03-23T12:30:00+01') -- evaluates to 2023-03-22
+AND "date" <= dates_filter_max('2022-03-23T12:35:00+01') -- evaluates to 2023-03-23
 ```
 
 
diff --git a/test/amtrak-gtfs-2021-10-06.sh b/test/amtrak-gtfs-2021-10-06.sh
@@ -76,7 +76,7 @@ SELECT nr_of_arrs
 FROM amtrak.stats_by_agency_route_stop_hour
 WHERE route_id = '40751' -- Acela
 AND stop_id = 'PHL' -- Philadelphia
-AND effective_hour = '2022-07-24T09:00-05'
+AND effective_hour = '2022-07-24 09:00:00-05:00'
 EOF
 )
 acelaPhillyStat=$(psql --csv -t -c "$acelaPhillyStatQuery" | tail -n 1)
@@ -87,16 +87,16 @@ fi
 
 nrOfActiveTripsQuery=$(cat << EOF
 SELECT nr_of_active_trips
-FROM amtrak.stats_active_trips_by_hour
-WHERE "hour" = '2021-11-26T04:00-05'
+FROM stats_active_trips_by_hour
+WHERE "hour" = '2021-11-26 04:00:00-05:00'
 EOF
 )
 # Note: I'm not sure if 127 is correct, but it is in the right ballpark. 🙈
 # The following query yields 150 connections, and it doesn't contain those who depart earlier and arrive later.
 # SELECT DISTINCT ON (trip_id) *
 # FROM amtrak.connections
-# WHERE t_departure >= '2021-11-26T02:00-05'
-# AND t_arrival <= '2021-11-26T06:00-05'
+# WHERE t_departure >= '2021-11-26 02:00:00-05:00'
+# AND t_arrival <= '2021-11-26 06:00:00-05:00'
 nrOfActiveTrips=$(psql --csv -t -c "$nrOfActiveTripsQuery" | tail -n 1)
 if [[ "$nrOfActiveTrips" != "127" ]]; then
 	echo "unexpected no. of active trips at 2021-11-26T04:00-05: $nrOfActiveTrips" 1>&2
diff --git a/test/sample-gtfs-feed.sh b/test/sample-gtfs-feed.sh
@@ -93,7 +93,7 @@ connection_during_dst=$(cat << EOF
 		extract(epoch from t_departure)::integer as dep
 	FROM connections
 	WHERE trip_id = 'during-dst-1'
-	AND t_departure = '2019-03-31T01:58+01'
+	AND t_departure = '2019-03-31T01:58:00+01:00'
 EOF
 )
 dst1=$(psql --csv -t -c "$connection_during_dst" | head -n 1)
