Merge pull request #11 from NikolayS/tablespaces_and_load_profiles

New report: Load Profile + tablespaces info for Table Sizes report

Author: NikolayS

sql/1_basic.sql

@@ -1,4 +1,4 @@
---Basic Node Information (master/replica, lag, DB size, tmp files)
+--Node Information: master/replica, lag, DB size, tmp files, etc
 with data as (
   select s.*
   from pg_stat_database s

sql/2_table_sizes.sql

@@ -1,8 +1,9 @@
---General Table Size Information
+--Table Size
 
 with data as (
   select
     c.oid,
+    (select spcname from pg_tablespace where oid = reltablespace) as tblspace,
     nspname as schema_name,
     relname as table_name,
     c.reltuples as row_estimate,
@@ -12,10 +13,11 @@ with data as (
     pg_total_relation_size(c.oid) - pg_indexes_size(c.oid) - coalesce(pg_total_relation_size(reltoastrelid), 0) as table_bytes
   from pg_class c
   left join pg_namespace n on n.oid = c.relnamespace
-  where relkind = 'r'
+  where relkind = 'r' and nspname <> 'pg_catalog'
 ), data2 as (
   select
     null::oid as oid,
+    null as tblspace,
     null as schema_name,
     '*** TOTAL ***' as table_name,
     sum(row_estimate) as row_estimate,
@@ -25,23 +27,37 @@ with data as (
     sum(table_bytes) as table_bytes
   from data
   union all
-  select null::oid, null, null, null, null, null, null, null
+  select
+    null::oid as oid,
+    null,
+    null as schema_name,
+    '    tablespace: [' || coalesce(tblspace, 'pg_default') || ']' as table_name,
+    sum(row_estimate) as row_estimate,
+    sum(total_bytes) as total_bytes,
+    sum(index_bytes) as index_bytes,
+    sum(toast_bytes) as toast_bytes,
+    sum(table_bytes) as table_bytes
+  from data
+  where (select count(distinct coalesce(tblspace, 'pg_default')) from data) > 1 -- don't show this part if there are no custom tablespaces
+  group by tblspace
+  union all
+  select null::oid, null, null, null, null, null, null, null, null
   union all
   select * from data
 )
 select
-  coalesce(nullif(schema_name, 'public') || '.', '') || table_name as "Table",
+  coalesce(nullif(schema_name, 'public') || '.', '') || table_name || coalesce(' [' || tblspace || ']', '') as "Table",
   '~' || case
     when row_estimate > 10^12 then round(row_estimate::numeric / 10^12::numeric, 0)::text || 'T'
     when row_estimate > 10^9 then round(row_estimate::numeric / 10^9::numeric, 0)::text || 'B'
     when row_estimate > 10^6 then round(row_estimate::numeric / 10^6::numeric, 0)::text || 'M'
     when row_estimate > 10^3 then round(row_estimate::numeric / 10^3::numeric, 0)::text || 'k'
     else row_estimate::text
   end as "Rows",
-  pg_size_pretty(total_bytes) || ' (' || round(100 * total_bytes::numeric / sum(total_bytes) over (partition by (schema_name is null)), 2)::text || '%)' as "Total Size",
-  pg_size_pretty(table_bytes) || ' (' || round(100 * table_bytes::numeric / sum(table_bytes) over (partition by (schema_name is null)), 2)::text || '%)' as "Table Size",
-  pg_size_pretty(index_bytes) || ' (' || round(100 * index_bytes::numeric / sum(index_bytes) over (partition by (schema_name is null)), 2)::text || '%)' as "Index(es) Size",
-  pg_size_pretty(toast_bytes) || ' (' || round(100 * toast_bytes::numeric / sum(toast_bytes) over (partition by (schema_name is null)), 2)::text || '%)' as "TOAST Size"
+  pg_size_pretty(total_bytes) || ' (' || round(100 * total_bytes::numeric / sum(total_bytes) over (partition by (schema_name is null), left(table_name, 3) = '***'), 2)::text || '%)' as "Total Size",
+  pg_size_pretty(table_bytes) || ' (' || round(100 * table_bytes::numeric / sum(table_bytes) over (partition by (schema_name is null), left(table_name, 3) = '***'), 2)::text || '%)' as "Table Size",
+  pg_size_pretty(index_bytes) || ' (' || round(100 * index_bytes::numeric / sum(index_bytes) over (partition by (schema_name is null), left(table_name, 3) = '***'), 2)::text || '%)' as "Index(es) Size",
+  pg_size_pretty(toast_bytes) || ' (' || round(100 * toast_bytes::numeric / sum(toast_bytes) over (partition by (schema_name is null), left(table_name, 3) = '***'), 2)::text || '%)' as "TOAST Size"
 \if :postgres_dba_wide
   ,
   row_estimate,
@@ -51,6 +67,7 @@ select
   toast_bytes,
   schema_name,
   table_name,
+  tblspace,
   oid
 \endif
 from data2

sql/3_load_profiles.sql

@@ -0,0 +1,100 @@
+--Load Profile
+
+with data as (
+  select
+    s.relname as table_name,
+    s.schemaname as schema_name,
+    (select spcname from pg_tablespace where oid = reltablespace) as tblspace,
+    *,
+    case when n_tup_upd = 0 then null else n_tup_hot_upd::numeric / n_tup_upd end as upd_hot_ratio,
+    seq_tup_read + coalesce(idx_tup_fetch, 0) - n_tup_del - n_tup_upd as tuples_selected,
+    seq_tup_read + coalesce(idx_tup_fetch, 0) + n_tup_ins as processed_tup_total -- we don't add _del and _upd here (already counted via seq_ & idx_)
+  from pg_stat_user_tables s
+  join pg_class c on c.oid = relid
+), data2 as (
+  select
+    0 as ord,
+    '*** TOTAL ***' as table_name,
+    null as schema_name,
+    null as tblspace,
+    sum(seq_tup_read) as seq_tup_read,
+    sum(idx_tup_fetch) as idx_tup_fetch,
+    sum(tuples_selected) as tuples_selected,
+    sum(n_tup_ins) as n_tup_ins,
+    sum(n_tup_del) as n_tup_del,
+    sum(n_tup_upd) as n_tup_upd,
+    sum(n_tup_hot_upd) as n_tup_hot_upd,
+    avg(upd_hot_ratio) as upd_hot_ratio,
+    sum(processed_tup_total) as processed_tup_total
+  from data
+  union all
+  select
+    1 as ord,
+    '    tablespace: [' || coalesce(tblspace, 'pg_default') || ']' as table_name,
+    null as schema_name,
+    null, -- we don't need to pass real tblspace value for this aggregated record further
+    sum(seq_tup_read) as seq_tup_read,
+    sum(idx_tup_fetch) as idx_tup_fetch,
+    sum(tuples_selected) as tuples_selected,
+    sum(n_tup_ins) as n_tup_ins,
+    sum(n_tup_del) as n_tup_del,
+    sum(n_tup_upd) as n_tup_upd,
+    sum(n_tup_hot_upd) as n_tup_hot_upd,
+    avg(upd_hot_ratio) as upd_hot_ratio,
+    sum(processed_tup_total) as processed_tup_total
+  from data
+  where (select count(distinct coalesce(tblspace, 'pg_default')) from data) > 1 -- don't show this part if there are no custom tablespaces
+  group by tblspace
+  union all
+  select 3, null, null, null, null, null, null, null, null, null, null, null, null
+  union all
+  select 4, table_name, schema_name, tblspace, seq_tup_read, idx_tup_fetch, tuples_selected, n_tup_ins, n_tup_del, n_tup_upd, n_tup_hot_upd, upd_hot_ratio, processed_tup_total
+  from data
+)
+select
+  coalesce(nullif(schema_name, 'public') || '.', '') || table_name || coalesce(' [' || tblspace || ']', '') as "Table",
+  (
+    with ops as (
+      select * from data2 d2 where d2.schema_name is not distinct from data2.schema_name and d2.table_name = data2.table_name
+    ), ops_ratios(opname, ratio) as (
+      select
+        'select',
+        case when processed_tup_total > 0 then tuples_selected::numeric / processed_tup_total else 0 end
+      from ops
+      union all
+      select
+        'insert',
+        case when processed_tup_total > 0 then n_tup_ins::numeric / processed_tup_total else 0 end
+      from ops
+      union all
+      select
+        'delete',
+        case when processed_tup_total > 0 then n_tup_del::numeric / processed_tup_total else 0 end
+      from ops
+      union all
+      select
+        'update',
+        case when processed_tup_total > 0 then n_tup_upd::numeric / processed_tup_total else 0 end
+      from ops
+    )
+    select
+      case
+        when ratio > .7 then upper(opname) || ' ~' || round(100 * ratio, 2)::text || '%'
+        else 'Mixed: ' || (
+          select string_agg(upper(left(opname, 1)) || ' ~' || round(100 * ratio, 2)::text || '%', ', ' order by ratio desc)
+          from (select * from ops_ratios where ratio > .2) _
+        )
+      end
+    from ops_ratios
+    order by ratio desc
+    limit 1
+  ) as "Load Type",
+  processed_tup_total as "Total (S+I+D+U)",
+  tuples_selected as "SELECTed",
+  n_tup_ins as "INSERTed",
+  n_tup_del as "DELETEd",
+  n_tup_upd as "UPDATEd",
+  round(100 * upd_hot_ratio, 2) as "HOT-updated, %",
+  case when seq_tup_read + coalesce(idx_tup_fetch, 0) > 0 then round(100 * seq_tup_read::numeric / (seq_tup_read + coalesce(idx_tup_fetch, 0)), 2) else 0 end as "SeqScan, %"
+from data2
+order by ord, processed_tup_total desc;

sql/a1_alignment_padding.sql

@@ -1,4 +1,4 @@
---Alignmet Padding Analysis: how many bytes can be saved if columns are ordered better?
+--[EXPERIMENTAL] Alignment Padding. How many bytes can be saved if columns are ordered better?
 
 -- TODO: not-yet-analyzed tables – show a warning (cannot get n_live_tup -> cannot get total bytes)
 -- TODO: NULLs

sql/s1_pg_stat_statements_top_total.sql

@@ -20,7 +20,7 @@ select
   min(min_time) as min_time,
   -- stddev_time, -- https://stats.stackexchange.com/questions/55999/is-it-possible-to-find-the-combined-standard-deviation
   sum(rows) as rows,
-  userid,
+  (select usename from pg_user where usesysid = userid) as usr,
   (select datname from pg_database where oid = dbid) as db,
   query,
   sum(shared_blks_hit) as shared_blks_hit,

start.psql

@@ -14,9 +14,10 @@ select regexp_replace(version(), '^PostgreSQL (\d+\.\d+).*$', e'\\1')::numeric >
   reset client_min_messages;
 \endif
 \echo '\033[1;35mMenu:\033[0m'
-\echo '   1 – Basic Node Information (master/replica, lag, DB size, tmp files)'
-\echo '   2 – General Table Size Information'
-\echo '  a1 – Alignment Padding Analysis: how many bytes can be saved if columns are ordered better?'
+\echo '   1 – Node Information: master/replica, lag, DB size, tmp files, etc'
+\echo '   2 – Table Size'
+\echo '   3 – Load Profile'
+\echo '  a1 – [EXPERIMENTAL] Alignment Padding. How many bytes can be saved if columns are ordered better?'
 \echo '  b1 – Tables Bloat, rough estimation'
 \echo '  b2 – B-tree Indexes Bloat, rough estimation'
 \echo '  b3 – Tables Bloat, more precise (requires pgstattuple extension; expensive)'
@@ -40,6 +41,7 @@ select regexp_replace(version(), '^PostgreSQL (\d+\.\d+).*$', e'\\1')::numeric >
 select
 :d_stp::text = '1' as d_step_is_1,
 :d_stp::text = '2' as d_step_is_2,
+:d_stp::text = '3' as d_step_is_3,
 :d_stp::text = 'a1' as d_step_is_a1,
 :d_stp::text = 'b1' as d_step_is_b1,
 :d_stp::text = 'b2' as d_step_is_b2,
@@ -75,6 +77,10 @@ set postgres_dba.wide = 'on';
   \ir ./sql/2_table_sizes.sql
   \prompt 'Press <Enter> to continue…' d_dummy
   \ir ./start.psql
+\elif :d_step_is_3
+  \ir ./sql/3_load_profiles.sql
+  \prompt 'Press <Enter> to continue…' d_dummy
+  \ir ./start.psql
 \elif :d_step_is_a1
   \ir ./sql/a1_alignment_padding.sql
   \prompt 'Press <Enter> to continue…' d_dummy