Fetch lag per consumer group

lib/kafkaex_lag_exporter/utils.ex

@@ -0,0 +1,70 @@
+# source code taken from https://github.com/reachfh/brod_group_subscriber_example
+
+defmodule KafkaexLagExporter.KafkaUtils do
+  @moduledoc "Utility functions for dealing with Kafka"
+
+  @default_client :client1
+
+  @type endpoint() :: {host :: atom(), port :: non_neg_integer()}
+
+  def connection, do: connection(@default_client)
+  @spec connection(atom) :: {list({charlist, non_neg_integer}), Keyword.t()}
+  def connection(client) do
+    clients = Application.get_env(:brod, :clients)
+    config = clients[client]
+
+    endpoints = config[:endpoints] || [{~c"localhost", 9092}]
+
+    sock_opts =
+      case Keyword.fetch(config, :ssl) do
+        {:ok, ssl_opts} ->
+          [ssl: ssl_opts]
+
+        :error ->
+          []
+      end
+
+    {endpoints, sock_opts}
+  end
+
+  @spec resolve_offsets(binary(), :earliest | :latest, atom()) ::
+          list({non_neg_integer(), integer()})
+  def resolve_offsets(topic, type, client) do
+    {endpoints, sock_opts} = connection(client)
+
+    {:ok, partitions_count} = :brod.get_partitions_count(client, topic)
+
+    for i <- Range.new(0, partitions_count - 1),
+        {:ok, offset} = :brod.resolve_offset(endpoints, topic, i, type, sock_opts) do
+      {i, offset}
+    end
+  end
+
+  @spec fetch_committed_offsets(binary(), binary(), atom()) ::
+          {non_neg_integer(), non_neg_integer()}
+  def fetch_committed_offsets(_topic, consumer_group, client) do
+    {endpoints, sock_opts} = connection(client)
+    {:ok, response} = :brod.fetch_committed_offsets(endpoints, sock_opts, consumer_group)
+
+    for r <- response,
+        pr <- r[:partitions],
+        do: {pr[:partition_index], pr[:committed_offset]}
+  end
+
+  @spec lag(binary(), binary(), atom()) :: list({non_neg_integer(), integer()})
+  def lag(topic, consumer_group, client) do
+    offsets = resolve_offsets(topic, :latest, client)
+    committed_offsets = fetch_committed_offsets(topic, consumer_group, client)
+
+    for {{part, current}, {_part2, committed}} <- Enum.zip(offsets, committed_offsets) do
+      {part, current - committed}
+    end
+  end
+
+  @spec lag_total(binary(), binary(), atom()) :: non_neg_integer()
+  def lag_total(topic, consumer_group, client) do
+    for {_part, recs} <- lag(topic, consumer_group, client), reduce: 0 do
+      acc -> acc + recs
+    end
+  end
+end