Tablewise Mnesia Metrics ... Optimized collect_metric for summaries and better metric registrations (#113)

* persisting metrics using prometheus_sup:register_metrics/1

* adding per table size and memory usage in prometheus_mnesia_collector (#2)

* adding per table size and memory usage in prometheus_mnesia_collector

* fixing doc

* making prometheus_summary:collect_metric more efficient (#3)

* fixing formatting issues

Author: SoniCoder

src/collectors/mnesia/prometheus_mnesia_collector.erl

@@ -122,6 +122,8 @@ add_metric_family({Name, Type, Help, Metrics}, Callback) ->
 metrics(EnabledMetrics) ->
   {Participants, Coordinators} = get_tm_info(EnabledMetrics),
   MemoryUsage = get_memory_usage(),
+  TablewiseMemoryUsage = get_tablewise_memory_usage(),
+  TablewiseSize = get_tablewise_size(),
 
   [{held_locks, gauge,
     "Number of held locks.",
@@ -149,7 +151,14 @@ metrics(EnabledMetrics) ->
     fun() -> mnesia:system_info(transaction_restarts) end},
    {memory_usage_bytes, gauge,
     "Total number of bytes allocated by all mnesia tables",
-    fun() -> MemoryUsage end}].
+    fun() -> MemoryUsage end},
+   {tablewise_memory_usage_bytes, gauge,
+    "Number of bytes allocated per mnesia table",
+    fun() -> TablewiseMemoryUsage end},
+   {tablewise_size, gauge,
+    "Number of rows present per table",
+    fun() -> TablewiseSize end}
+    ].
 
 %%====================================================================
 %% Private Parts
@@ -189,4 +198,19 @@ get_memory_usage() ->
   Calculator = fun(Tab, Sum) ->
                  mnesia:table_info(Tab, memory) + Sum
                end,
-  lists:foldl(Calculator, 0, mnesia:system_info(tables)) * WordSize.
+  lists:foldl(Calculator, 0, mnesia:system_info(tables)) * WordSize.
+
+get_tablewise_memory_usage() ->
+  WordSize = erlang:system_info(wordsize),
+  Calculator =
+    fun(Tab, Acc) ->
+      [{[{table, Tab}], mnesia:table_info(Tab, memory) * WordSize} | Acc]
+    end,
+  lists:foldl(Calculator, [], mnesia:system_info(tables)).
+
+get_tablewise_size() ->
+  Calculator =
+    fun(Tab, Acc) ->
+      [{[{table, Tab}], mnesia:table_info(Tab, size)} | Acc]
+    end,
+  lists:foldl(Calculator, [], mnesia:system_info(tables)).

src/metrics/prometheus_summary.erl

@@ -294,12 +294,21 @@ values(Registry, Name) ->
       DU = prometheus_metric:mf_duration_unit(MF),
       Labels = prometheus_metric:mf_labels(MF),
       MFValues = load_all_values(Registry, Name),
-      [begin
-         {Count, Sum} = reduce_label_values(LabelValues, MFValues),
-         {lists:zip(Labels, LabelValues), Count,
-          prometheus_time:maybe_convert_to_du(DU, Sum)}
-       end ||
-        LabelValues <- collect_unique_labels(MFValues)]
+      ReducedMap = lists:foldl(
+        fun([L, C, IS, FS], ResAcc) ->
+          {PrevCount, PrevSum} = maps:get(L, ResAcc, {0, 0}),
+          ResAcc#{L => {PrevCount + C, PrevSum + IS + FS}}
+        end,
+      #{},
+      MFValues),
+      ReducedMapList = lists:sort(maps:to_list(ReducedMap)),
+      lists:foldr(
+        fun({LabelValues, {Count, Sum}}, Acc) ->
+          [{lists:zip(Labels, LabelValues), Count,
+          prometheus_time:maybe_convert_to_du(DU, Sum)} | Acc]
+        end,
+        [],
+      ReducedMapList)
   end.
 
 %%====================================================================
@@ -322,13 +331,22 @@ collect_mf(Registry, Callback) ->
 %% @private
 collect_metrics(Name, {CLabels, Labels, Registry, DU}) ->
   MFValues = load_all_values(Registry, Name),
-  [begin
-     {Count, Sum} = reduce_label_values(LabelValues, MFValues),
-     prometheus_model_helpers:summary_metric(
-       CLabels ++ lists:zip(Labels, LabelValues), Count,
-       prometheus_time:maybe_convert_to_du(DU, Sum))
-   end ||
-    LabelValues <- collect_unique_labels(MFValues)].
+  ReducedMap = lists:foldl(
+        fun([L, C, IS, FS], ResAcc) ->
+          {PrevCount, PrevSum} = maps:get(L, ResAcc, {0, 0}),
+          ResAcc#{L => {PrevCount + C, PrevSum + IS + FS}}
+        end,
+      #{},
+      MFValues),
+  ReducedMapList = lists:sort(maps:to_list(ReducedMap)),
+  lists:foldr(
+    fun({LabelValues, {Count, Sum}}, Acc) ->
+      [prometheus_model_helpers:summary_metric(
+      CLabels ++ lists:zip(Labels, LabelValues), Count,
+      prometheus_time:maybe_convert_to_du(DU, Sum)) | Acc]
+    end,
+    [],
+  ReducedMapList).
 
 %%====================================================================
 %% Private Parts
@@ -371,13 +389,6 @@ key(Registry, Name, LabelValues) ->
   Rnd = X band (?WIDTH-1),
   {Registry, Name, LabelValues, Rnd}.
 
-collect_unique_labels(MFValues) ->
-  lists:usort([L || [L, _, _, _] <- MFValues]).
-
-reduce_label_values(Labels, MFValues) ->
-  {lists:sum([C || [L, C, _, _] <- MFValues, L == Labels]),
-   lists:sum([IS + FS || [L, _, IS, FS] <- MFValues, L == Labels])}.
-
 reduce_values(Values) ->
   {lists:sum([C || [C, _, _] <- Values]),
    lists:sum([IS + FS || [_, IS, FS] <- Values])}.

src/prometheus_sup.erl

@@ -9,6 +9,7 @@
 -export([start_link/0]).
 %% Supervisor callbacks
 -export([init/1]).
+-export([register_metrics/1]).
 
 -behaviour(supervisor).
 
@@ -62,6 +63,12 @@ register_collectors() ->
 register_metrics() ->
   [declare_metric(Decl) || Decl <- default_metrics()].
 
+register_metrics(Metrics) ->
+  DefaultMetrics0 = default_metrics(),
+  DefaultMetrics1 = lists:usort(DefaultMetrics0 ++ Metrics),
+  application:set_env(prometheus, default_metrics, DefaultMetrics1),
+  [declare_metric(Decl) || Decl <- Metrics].
+
 setup_instrumenters() ->
   [prometheus_instrumenter:setup(Instrumenter) ||
     Instrumenter <- prometheus_instrumenter:enabled_instrumenters()].