Merge branch 'master' of github.com:NikolayS/postgres_dba into postgres10

Conflicts:
	start.psql

Author: NikolayS

README.md

@@ -4,9 +4,11 @@ The missing set of useful tools for Postgres DBA and mere mortals.
 
 :warning: The project is in its very early stage. If you have great ideas, feel free to create a pull request or open an issue.
 
+<img width="709" alt="screen shot 2018-03-17 at 14 34 04" src="https://user-images.githubusercontent.com/1345402/37560150-78b4b24e-29f0-11e8-9b3f-7d68b13631e1.png">
+
 ## Questions?
 
-Questions? Ideas? Write me: samokhvalov@gmail.com, Nikolay Samokhvalov.
+Questions? Ideas? Contact me: samokhvalov@gmail.com, Nikolay Samokhvalov.
 
 ## Credits
 
@@ -45,8 +47,6 @@ And type `:dba <Enter>` in psql. (Or `\i /path/to/postgres_dba/start.psql` if yo
 
 – it will open interactive menu.
 
-<img width="779" alt="screen shot 2018-01-05 at 13 14 30" src="https://user-images.githubusercontent.com/1345402/34628761-6b98b988-f21a-11e7-8e5c-ab2580389a5c.png">
-
 ### Connect to Remote Postgres Server
 What to do if you need to connect to a remote Postgres server? Usually, Postgres is behind a firewall and/or doesn't listen to a public network interface. So you need to be able to connect to the server using SSH. If you can do it, then just create SSH tunnel (assuming that Postgres listens to default port 5432 on that server:
 

sql/1_basic.sql

@@ -1,4 +1,4 @@
---Basic Node Information (master/replica, lag, DB size, tmp files)
+--Node Information: master/replica, lag, DB size, tmp files, etc
 with data as (
   select s.*
   from pg_stat_database s

sql/2_table_sizes.sql

@@ -1,4 +1,4 @@
---General Table Size Information
+--Table Size
 
 with data as (
   select
@@ -38,6 +38,7 @@ with data as (
     sum(toast_bytes) as toast_bytes,
     sum(table_bytes) as table_bytes
   from data
+  where (select count(distinct coalesce(tblspace, 'pg_default')) from data) > 1 -- don't show this part if there are no custom tablespaces
   group by tblspace
   union all
   select null::oid, null, null, null, null, null, null, null, null

sql/3_load_profiles.sql

@@ -0,0 +1,100 @@
+--Load Profile
+
+with data as (
+  select
+    s.relname as table_name,
+    s.schemaname as schema_name,
+    (select spcname from pg_tablespace where oid = reltablespace) as tblspace,
+    *,
+    case when n_tup_upd = 0 then null else n_tup_hot_upd::numeric / n_tup_upd end as upd_hot_ratio,
+    seq_tup_read + coalesce(idx_tup_fetch, 0) - n_tup_del - n_tup_upd as tuples_selected,
+    seq_tup_read + coalesce(idx_tup_fetch, 0) + n_tup_ins as processed_tup_total -- we don't add _del and _upd here (already counted via seq_ & idx_)
+  from pg_stat_user_tables s
+  join pg_class c on c.oid = relid
+), data2 as (
+  select
+    0 as ord,
+    '*** TOTAL ***' as table_name,
+    null as schema_name,
+    null as tblspace,
+    sum(seq_tup_read) as seq_tup_read,
+    sum(idx_tup_fetch) as idx_tup_fetch,
+    sum(tuples_selected) as tuples_selected,
+    sum(n_tup_ins) as n_tup_ins,
+    sum(n_tup_del) as n_tup_del,
+    sum(n_tup_upd) as n_tup_upd,
+    sum(n_tup_hot_upd) as n_tup_hot_upd,
+    avg(upd_hot_ratio) as upd_hot_ratio,
+    sum(processed_tup_total) as processed_tup_total
+  from data
+  union all
+  select
+    1 as ord,
+    '    tablespace: [' || coalesce(tblspace, 'pg_default') || ']' as table_name,
+    null as schema_name,
+    null, -- we don't need to pass real tblspace value for this aggregated record further
+    sum(seq_tup_read) as seq_tup_read,
+    sum(idx_tup_fetch) as idx_tup_fetch,
+    sum(tuples_selected) as tuples_selected,
+    sum(n_tup_ins) as n_tup_ins,
+    sum(n_tup_del) as n_tup_del,
+    sum(n_tup_upd) as n_tup_upd,
+    sum(n_tup_hot_upd) as n_tup_hot_upd,
+    avg(upd_hot_ratio) as upd_hot_ratio,
+    sum(processed_tup_total) as processed_tup_total
+  from data
+  where (select count(distinct coalesce(tblspace, 'pg_default')) from data) > 1 -- don't show this part if there are no custom tablespaces
+  group by tblspace
+  union all
+  select 3, null, null, null, null, null, null, null, null, null, null, null, null
+  union all
+  select 4, table_name, schema_name, tblspace, seq_tup_read, idx_tup_fetch, tuples_selected, n_tup_ins, n_tup_del, n_tup_upd, n_tup_hot_upd, upd_hot_ratio, processed_tup_total
+  from data
+)
+select
+  coalesce(nullif(schema_name, 'public') || '.', '') || table_name || coalesce(' [' || tblspace || ']', '') as "Table",
+  (
+    with ops as (
+      select * from data2 d2 where d2.schema_name is not distinct from data2.schema_name and d2.table_name = data2.table_name
+    ), ops_ratios(opname, ratio) as (
+      select
+        'select',
+        case when processed_tup_total > 0 then tuples_selected::numeric / processed_tup_total else 0 end
+      from ops
+      union all
+      select
+        'insert',
+        case when processed_tup_total > 0 then n_tup_ins::numeric / processed_tup_total else 0 end
+      from ops
+      union all
+      select
+        'delete',
+        case when processed_tup_total > 0 then n_tup_del::numeric / processed_tup_total else 0 end
+      from ops
+      union all
+      select
+        'update',
+        case when processed_tup_total > 0 then n_tup_upd::numeric / processed_tup_total else 0 end
+      from ops
+    )
+    select
+      case
+        when ratio > .7 then upper(opname) || ' ~' || round(100 * ratio, 2)::text || '%'
+        else 'Mixed: ' || (
+          select string_agg(upper(left(opname, 1)) || ' ~' || round(100 * ratio, 2)::text || '%', ', ' order by ratio desc)
+          from (select * from ops_ratios where ratio > .2) _
+        )
+      end
+    from ops_ratios
+    order by ratio desc
+    limit 1
+  ) as "Load Type",
+  processed_tup_total as "Total (S+I+D+U)",
+  tuples_selected as "SELECTed",
+  n_tup_ins as "INSERTed",
+  n_tup_del as "DELETEd",
+  n_tup_upd as "UPDATEd",
+  round(100 * upd_hot_ratio, 2) as "HOT-updated, %",
+  case when seq_tup_read + coalesce(idx_tup_fetch, 0) > 0 then round(100 * seq_tup_read::numeric / (seq_tup_read + coalesce(idx_tup_fetch, 0)), 2) else 0 end as "SeqScan, %"
+from data2
+order by ord, processed_tup_total desc;

sql/a1_alignment_padding.sql

@@ -1,4 +1,4 @@
---Alignmet Padding Analysis: how many bytes can be saved if columns are ordered better?
+--[EXPERIMENTAL] Alignment Padding. How many bytes can be saved if columns are ordered better?
 
 -- TODO: not-yet-analyzed tables – show a warning (cannot get n_live_tup -> cannot get total bytes)
 -- TODO: NULLs

start.psql

@@ -27,9 +27,10 @@ select regexp_replace(version(), '^PostgreSQL (\d+\.\d+).*$', e'\\1')::numeric >
   reset client_min_messages;
 \endif
 \echo '\033[1;35mMenu:\033[0m'
-\echo '   1 – Basic Node Information (master/replica, lag, DB size, tmp files)'
-\echo '   2 – General Table Size Information'
-\echo '  a1 – Alignmet Padding Analysis: how many bytes can be saved if columns are ordered better?'
+\echo '   1 – Node Information: master/replica, lag, DB size, tmp files, etc'
+\echo '   2 – Table Size'
+\echo '   3 – Load Profile'
+\echo '  a1 – [EXPERIMENTAL] Alignment Padding. How many bytes can be saved if columns are ordered better?'
 \echo '  b1 – Tables Bloat, rough estimation'
 \echo '  b2 – B-tree Indexes Bloat, rough estimation'
 \echo '  b3 – Tables Bloat, more precise (requires pgstattuple extension; expensive)'
@@ -53,6 +54,7 @@ select regexp_replace(version(), '^PostgreSQL (\d+\.\d+).*$', e'\\1')::numeric >
 select
 :d_stp::text = '1' as d_step_is_1,
 :d_stp::text = '2' as d_step_is_2,
+:d_stp::text = '3' as d_step_is_3,
 :d_stp::text = 'a1' as d_step_is_a1,
 :d_stp::text = 'b1' as d_step_is_b1,
 :d_stp::text = 'b2' as d_step_is_b2,
@@ -88,6 +90,10 @@ set postgres_dba.wide = 'on';
   \ir ./sql/2_table_sizes.sql
   \prompt 'Press <Enter> to continue…' d_dummy
   \ir ./start.psql
+\elif :d_step_is_3
+  \ir ./sql/3_load_profiles.sql
+  \prompt 'Press <Enter> to continue…' d_dummy
+  \ir ./start.psql
 \elif :d_step_is_a1
   \ir ./sql/a1_alignment_padding.sql
   \prompt 'Press <Enter> to continue…' d_dummy