DuckDB rewrite: remove PostgreSQL-specific features 💥✅💚📝

Author: derhuerst

.github/workflows/test.yml

@@ -54,26 +54,6 @@ jobs:
         PGPASSWORD: password
         PGDATABASE: postgres
 
-    - name: install PostgREST
-      run: |
-        set -euo pipefail
-        set -x
-        dl_url="$(
-          curl -fsSL \
-            -H "User-Agent: $user_agent" \
-            -H 'Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}' \
-            'https://api.github.com/repos/PostgREST/postgrest/releases/latest' \
-            | jq -rc '.assets[] | select(.name | test("linux-static-x86-64")) | .browser_download_url'
-        )"
-        wget -nv -U "$user_agent" \
-          --header='Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}' \
-          -O /tmp/postgrest.tar.xz \
-          "$dl_url"
-        tar -C /usr/local/bin -J -x postgrest </tmp/postgrest.tar.xz
-        /usr/local/bin/postgrest --version
-      env:
-        user_agent: 'public-transport/gtfs-via-postgres CI'
-
     - run: npm install
 
     - run: npm run lint

cli.js

@@ -59,21 +59,6 @@ const {
 		'schema': {
 			type: 'string',
 		},
-		'postgraphile': {
-			type: 'boolean',
-		},
-		'postgraphile-password': {
-			type: 'string',
-		},
-		'postgrest': {
-			type: 'boolean',
-		},
-		'postgrest-password': {
-			type: 'string',
-		},
-		'postgrest-query-cost-limit': {
-			type: 'string',
-		},
 		'import-metadata': {
 			type: 'boolean',
 		}
@@ -130,21 +115,6 @@ Options:
                                     gets created, to ensure that multiple imports into the
                                     same database are all made using the same version. See
                                     also multiple-datasets.md in the docs.
-    --postgraphile                Tweak generated SQL for PostGraphile usage.
-                                    https://www.graphile.org/postgraphile/
-    --postgraphile-password       Password for the PostGraphile PostgreSQL user.
-                                    Default: $POSTGRAPHILE_PGPASSWORD, fallback random.
-    --postgrest                   Tweak generated SQL for PostgREST usage.
-                                    Please combine it with --schema.
-                                    https://postgrest.org/
-    --postgrest-password          Password for the PostgREST PostgreSQL user \`web_anon\`.
-                                    Default: $POSTGREST_PGPASSWORD, fallback random.
-    --postgrest-query-cost-limit  Define a cost limit [1] for queries executed by PostgREST
-                                    on behalf of a user. It is only enforced if
-                                    pg_plan_filter [2] is installed in the database!
-                                    Must be a positive float. Default: none
-                                    [1] https://www.postgresql.org/docs/14/using-explain.html
-                                    [2] https://github.com/pgexperts/pg_plan_filter
     --import-metadata             Create functions returning import metadata:
                                     - gtfs_data_imported_at (timestamp with time zone)
                                     - gtfs_via_postgres_version (text)
@@ -186,8 +156,6 @@ const opt = {
 	statsByAgencyIdAndRouteIdAndStopAndHour: flags['stats-by-agency-route-stop-hour'] || 'none',
 	statsActiveTripsByHour: flags['stats-active-trips-by-hour'] || 'none',
 	schema: flags['schema'] || 'public',
-	postgraphile: !!flags.postgraphile,
-	postgrest: !!flags.postgrest,
 	importMetadata: !!flags['import-metadata'],
 }
 if ('stops-without-level-id' in flags) {
@@ -196,20 +164,6 @@ if ('stops-without-level-id' in flags) {
 if ('lower-case-lang-codes' in flags) {
 	opt.lowerCaseLanguageCodes = flags['lower-case-lang-codes']
 }
-if ('postgraphile-password' in flags) {
-	opt.postgraphilePassword = flags['postgraphile-password']
-}
-if ('postgrest-password' in flags) {
-	opt.postgrestPassword = flags['postgrest-password']
-}
-if ('postgrest-query-cost-limit' in flags) {
-	const limit = parseFloat(flags['postgrest-query-cost-limit'])
-	if (!Number.isFinite(limit) || limit < 0) {
-		console.error('Invalid --postgrest-query-cost-limit value.')
-		process.exit(1)
-	}
-	opt.lowerCaseLanguageCodes = limit
-}
 
 pipeline(
 	convertGtfsToSql(files, opt),

docs/import-metadata.md

@@ -12,7 +12,7 @@ SELECT gtfs_via_postgres_version()
 -- 4.5.3
 
 SELECT gtfs_via_postgres_options()
--- {"schema": "public", "silent": false, "importStart": 1681417454781, "postgraphile": false, "importMetadata": true, … }
+-- {"schema": "public", "silent": false, "importStart": 1681417454781, "importMetadata": true, … }
 SELECT (gtfs_via_postgres_options())['tripsWithoutShapeId']
 -- true
 ```

docs/postgrest.md

@@ -26,13 +26,6 @@ const convertGtfsToSql = async function* (files, opt = {}) {
 		statsByAgencyIdAndRouteIdAndStopAndHour: 'none',
 		statsActiveTripsByHour: 'none',
 		schema: 'public',
-		postgraphile: false,
-		postgraphilePassword: process.env.POSTGRAPHILE_PGPASSWORD || null,
-		postgrest: false,
-		postgrestPassword: process.env.POSTGREST_PASSWORD || null,
-		// see https://github.com/pgexperts/pg_plan_filter
-		// see also https://www.postgresql.org/docs/14/using-explain.html
-		postgrestQueryCostLimit: null, // or float
 		importMetadata: false,
 		...opt,
 	}
@@ -47,16 +40,6 @@ const convertGtfsToSql = async function* (files, opt = {}) {
 		statsByAgencyIdAndRouteIdAndStopAndHour,
 		statsActiveTripsByHour,
 	} = opt
-	let postgraphilePassword = opt.postgraphilePassword
-	if (opt.postgraphile && postgraphilePassword === null) {
-		postgraphilePassword = randomBytes(10).toString('hex')
-		console.error(`PostGraphile PostgreSQL user's password:`, postgraphilePassword)
-	}
-	let postgrestPassword = opt.postgrestPassword
-	if (opt.postgrest && postgrestPassword === null) {
-		postgrestPassword = randomBytes(10).toString('hex')
-		console.error(`PostrREST PostgreSQL user's password:`, postgrestPassword)
-	}
 
 	if (ignoreUnsupportedFiles) {
 		files = files.filter(f => !!formatters[f.name])
@@ -255,104 +238,6 @@ LANGUAGE sql;
 	}
 
 	yield `\
-
-${opt.postgraphile ? `\
--- seal imported data
--- todo:
--- > Be careful with public schema.It already has a lot of default privileges that you maybe don't want... See documentation[1].
--- > [1]: postgresql.org/docs/11/ddl-schemas.html#DDL-SCHEMAS-PRIV
-DO $$
-BEGIN
-	-- https://stackoverflow.com/questions/8092086/create-postgresql-role-user-if-it-doesnt-exist#8099557
-	IF EXISTS (
-		SELECT FROM pg_catalog.pg_roles
-		WHERE rolname = 'postgraphile'
-	) THEN
-		RAISE NOTICE 'Role "postgraphile" already exists, skipping creation.';
-	ELSE
-		CREATE ROLE postgraphile LOGIN PASSWORD '${opt.postgraphilePassword}'; -- todo: escape properly
-	END IF;
-END
-$$;
-DO $$
-    DECLARE
-        db TEXT := current_database();
-    BEGIN
-    	-- todo: grant just on $opt.schema instead?
-        EXECUTE format('GRANT ALL PRIVILEGES ON DATABASE %I TO %I', db, 'postgraphile');
-    END
-$$;
-GRANT USAGE ON SCHEMA "${opt.schema}" TO postgraphile;
--- https://stackoverflow.com/questions/760210/how-do-you-create-a-read-only-user-in-postgresql#comment50679407_762649
-REVOKE CREATE ON SCHEMA "${opt.schema}" FROM PUBLIC;
-GRANT SELECT ON ALL TABLES IN SCHEMA "${opt.schema}" TO postgraphile;
--- ALTER DEFAULT PRIVILEGES IN SCHEMA "${opt.schema}" GRANT SELECT ON TABLES TO postgraphile;
--- todo: set search_path? https://stackoverflow.com/questions/760210/how-do-you-create-a-read-only-user-in-postgresql#comment33535263_762649
-` : ''}
-
-${opt.postgrest ? `\
-${opt.schema !== 'public' ? `\
--- pattern from https://stackoverflow.com/a/8099557
-DO
-$$
-BEGIN
-	-- Roles are shared across databases, so we have remove previously configured privileges.
-	-- This might of course interfere with other programs running on the DBMS!
-	-- todo: find a cleaner solution
-	IF EXISTS (
-		SELECT FROM pg_catalog.pg_roles
-		WHERE  rolname = 'web_anon'
-	) THEN
-		RAISE WARNING 'Role web_anon already exists. Reassigning owned DB objects to current_user().';
-		REASSIGN OWNED BY web_anon TO SESSION_USER;
-	ELSE
-		BEGIN
-			CREATE ROLE web_anon NOLOGIN NOINHERIT;
-		EXCEPTION
-			WHEN duplicate_object THEN
-				RAISE NOTICE 'Role web_anon was just created by a concurrent transaction.';
-		END;
-	END IF;
-	IF EXISTS (
-		SELECT FROM pg_catalog.pg_roles
-		WHERE  rolname = 'postgrest'
-	) THEN
-		RAISE WARNING 'Role postgrest already exists. Reassigning owned DB objects to current_user().';
-		REASSIGN OWNED BY postgrest TO SESSION_USER;
-	ELSE
-		BEGIN
-			CREATE ROLE postgrest LOGIN NOINHERIT NOCREATEDB NOCREATEROLE NOSUPERUSER PASSWORD '${postgrestPassword}';
-		EXCEPTION
-			WHEN duplicate_object THEN
-				RAISE NOTICE 'Role postgrest was just created by a concurrent transaction.';
-		END;
-	END IF;
-END
-$$;
-
-
--- https://postgrest.org/en/stable/tutorials/tut0.html#step-4-create-database-for-api
--- https://postgrest.org/en/stable/explanations/db_authz.html
--- todo: is this secure?
-GRANT USAGE ON SCHEMA "${opt.schema}" TO web_anon;
-GRANT SELECT ON ALL TABLES IN SCHEMA "${opt.schema}" TO web_anon;
-GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA "${opt.schema}" TO web_anon;
-GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA "${opt.schema}" TO web_anon;
-
-GRANT web_anon TO postgrest;
-
-${opt.postgrestQueryCostLimit !== null ? `
--- If pg_plan_filter is installed, limit the cost of queries made by PostgREST users.
-ALTER USER web_anon SET plan_filter.statement_cost_limit = ${opt.postgrestQueryCostLimit};
-` : ''}
-
-COMMENT ON SCHEMA "${opt.schema}" IS
-$$GTFS REST API
-This REST API is created by running [PostgREST](https://postgrest.org/) on top of a [PostgreSQL](https://www.postgresql.org) DB generated using [${pkg.name} v${pkg.version}](${pkg.homepage || pkg.repository}).
-$$;
-` : ''}
-` : ''}
-
 COMMIT;`
 }
 

index.js

@@ -41,10 +41,6 @@ const afterAll = (opt) => `\
 
 CREATE INDEX ON "${opt.schema}".calendar_dates (service_id);
 CREATE INDEX ON "${opt.schema}".calendar_dates (exception_type);
-
-${opt.postgraphile ? `\
-COMMENT ON TABLE "${opt.schema}".calendar_dates IS E'@foreignKey (service_id) references calendar|@fieldName calendar';
-` : ''}
 `
 
 module.exports = {

lib/calendar_dates.js

@@ -84,11 +84,6 @@ const formatPathwaysRow = (p) => {
 
 const afterAll = (opt) => `\
 \\.
-
-${opt.postgraphile ? `\
-CREATE INDEX ON "${opt.schema}".pathways (from_stop_id);
-CREATE INDEX ON "${opt.schema}".pathways (to_stop_id);
-` : ''}
 `
 
 module.exports = {

lib/pathways.js

@@ -22,10 +22,6 @@ AS $$
 	);
 $$ language sql STABLE;
 
-${opt.postgraphile ? `\
-COMMENT ON FUNCTION "${opt.schema}".is_bcp_47_tag IS E'@omit';
-` : ''}
-
 -- todo [breaking]: remove
 CREATE OR REPLACE FUNCTION "${opt.schema}".is_valid_lang_code(
 	input TEXT
@@ -36,9 +32,6 @@ AS $$
 	SELECT "${opt.schema}".is_bcp_47_tag(input);
 $$ language sql STABLE;
 
-${opt.postgraphile ? `\
-COMMENT ON FUNCTION "${opt.schema}".is_valid_lang_code IS E'@omit';
-` : ''}
 `,
 }
 const is_timezone = {
@@ -59,9 +52,6 @@ AS $$
 	END;
 $$ language plpgsql STABLE;
 
-${opt.postgraphile ? `\
-COMMENT ON FUNCTION "${opt.schema}".is_timezone IS E'@omit';
-` : ''}
 `,
 }
 const shape_exists = {
@@ -79,9 +69,6 @@ AS $$
 	);
 $$ language sql STABLE;
 
-${opt.postgraphile ? `\
-COMMENT ON FUNCTION "${opt.schema}".shape_exists IS E'@omit';
-` : ''}
 `,
 }
 

lib/prerequisites.js

@@ -257,10 +257,6 @@ CREATE TYPE "${opt.schema}".route_type_val AS ENUM (
 ${extRouteTypes.map(([route_type, desc]) => `, '${route_type}' -- ${desc}`).join('\n')}
 );
 CREATE CAST ("${opt.schema}".route_type_val AS text) WITH INOUT AS IMPLICIT;
--- todo [breaking]: use small table as enum? https://www.graphile.org/postgraphile/enums/#with-enum-tables
-${opt.postgraphile ? `\
-COMMENT ON TYPE "${opt.schema}".route_type_val IS E'@enum\\n@enumName RouteType\\n';
-` : ''}
 
 CREATE TABLE "${opt.schema}".routes (
 	route_id TEXT PRIMARY KEY,
@@ -328,9 +324,6 @@ const afterAll = (opt) => `\
 \\.
 
 CREATE INDEX ON "${opt.schema}".routes (route_short_name);
-${opt.postgraphile ? `\
-CREATE INDEX ON "${opt.schema}".routes (agency_id);
-` : ''}
 `
 
 module.exports = {

lib/routes.js

@@ -68,9 +68,6 @@ CREATE INDEX ON "${opt.schema}".service_days (date);
 -- apparently the unique index (service_id, date) doesn't speed up queries
 CREATE INDEX ON "${opt.schema}".service_days (service_id, date);
 
-${opt.postgraphile ? `\
-COMMENT ON MATERIALIZED VIEW "${opt.schema}".service_days IS E'@name serviceDates\\n@primaryKey service_id,date';
-` : ''}
 `
 
 module.exports = {