feat(relay): Add more prometheus metrics

Author: kkirkov

beacon-light-client/solidity/tasks/start-publishing.ts

@@ -11,7 +11,10 @@ import {
   isSupportedFollowNetwork,
 } from '@dendreth/relay/utils/get_current_network_config';
 import { getGenericLogger } from '@dendreth/utils/ts-utils/logger';
-import { initPrometheusSetup } from '@dendreth/utils/ts-utils/prometheus-utils';
+import {
+  initPrometheusSetup,
+  registerGaugesForStartPublishing,
+} from '@dendreth/utils/ts-utils/prometheus-utils';
 
 const logger = getGenericLogger();
 
@@ -73,6 +76,7 @@ task('start-publishing', 'Run relayer')
       }
 
       initPrometheusSetup(args.prometheusPort, networkName);
+      registerGaugesForStartPublishing();
     }
 
     const currentConfig = await getNetworkConfig(args.followNetwork);

libs/typescript/ts-utils/prometheus-utils.ts

@@ -11,7 +11,7 @@ let followNetwork: string;
 
 export function initPrometheusSetup(port?: number, curFollowNetwork?: string) {
   const app = express();
-
+  prometheusInitProving();
   if (!port) {
     // Only for pollUpdates
     port = 2999;
@@ -20,12 +20,14 @@ export function initPrometheusSetup(port?: number, curFollowNetwork?: string) {
     followNetwork = curFollowNetwork;
   }
 
+  // Only expose the metrics endpoint if not already initialized
   app.get('/metrics', async (req, res) => {
+    res.set('Content-Type', register.contentType);
     res.end(await register.metrics());
   });
 
   app.listen(port, () => {
-    console.log(`Express listening on port ${port}`);
+    console.log(`Prometheus metrics exposed on port ${port}`);
   });
 
   return client;
@@ -69,3 +71,89 @@ export async function prometheusTiming<T>(func: () => T, funcName: string) {
     }
   }
 }
+
+export function registerGaugesForProver() {
+  register.registerMetric(timesGettingInputsForProofGeneration);
+  register.registerMetric(numberOfProofGenerated);
+}
+
+export function registerGaugesForStartPublishing() {
+  register.registerMetric(accountBalanceGauge);
+  register.registerMetric(previousSlot);
+  register.registerMetric(transactionForSlot);
+  register.registerMetric(currentNetworkSlot);
+  register.registerMetric(minutesDelayPrevSlot);
+  register.registerMetric(minutesDelayTransaction);
+  register.registerMetric(numberOfProofPublished);
+}
+
+export const accountBalanceGauge = new client.Gauge({
+  name: 'account_balance',
+  help: 'Current balance of the account',
+  labelNames: ['network'],
+});
+
+export const previousSlot = new client.Gauge({
+  name: 'previous_slot',
+  help: 'Previous slot on the chain',
+  labelNames: ['network'],
+});
+
+export const transactionForSlot = new client.Gauge({
+  name: 'transaction_slot',
+  help: 'Transaction publishing for slot',
+  labelNames: ['network'],
+});
+
+export const currentNetworkSlot = new client.Gauge({
+  name: 'current_network_slot',
+  help: 'Current slot on chian',
+  labelNames: ['network'],
+});
+
+export const minutesDelayPrevSlot = new client.Gauge({
+  name: 'minutes_delay_prev_slot',
+  help: 'How behind is the last slot',
+  labelNames: ['network'],
+});
+
+export const minutesDelayTransaction = new client.Gauge({
+  name: 'minutes_delay_transaction',
+  help: 'How behind is the transaction',
+  labelNames: ['network'],
+});
+
+export async function prometheusInitProving() {
+  timesGettingInputsForProofGeneration.reset();
+  numberOfProofGenerated.reset();
+  numberOfProofPublished.reset();
+}
+
+export function incrementInputsForProofGeneration() {
+  timesGettingInputsForProofGeneration.inc();
+}
+
+export function incrementProofGenerated() {
+  numberOfProofGenerated.inc();
+}
+
+export function incrementProofPublished(network) {
+  numberOfProofPublished.labels(network).inc();
+}
+export const timesGettingInputsForProofGeneration = new client.Counter({
+  name: 'times_getting_inputs_for_proof_generation',
+  help: 'The number of times inputs for proof generation were requested(since last restart)',
+  labelNames: ['network'],
+});
+
+export const numberOfProofGenerated = new client.Counter({
+  name: 'number_of_proof_generated',
+  help: 'The number of proofs generated(since last restart)',
+  labelNames: ['network'],
+});
+
+export const numberOfProofPublished = new client.Counter({
+  name: 'number_of_proof_published',
+  help: 'The number of proofs published(since last restart)',
+  labelNames: ['network'],
+});

relay/implementations/prover.ts

@@ -1,6 +1,10 @@
 import { IProver } from '@/abstraction/prover-interface';
 import { ProofInputType, Proof, WitnessGeneratorInput } from '@/types/types';
 import { getGenericLogger } from '@dendreth/utils/ts-utils/logger';
+import {
+  incrementInputsForProofGeneration,
+  incrementProofGenerated,
+} from '@dendreth/utils/ts-utils/prometheus-utils';
 
 const logger = getGenericLogger();
 export class Prover implements IProver {
@@ -24,6 +28,7 @@ export class Prover implements IProver {
     await this.callInput(proofInput.proofInput);
 
     logger.info('Input send waiting for proof generation');
+    incrementInputsForProofGeneration();
 
     st = await this.getStatus();
 
@@ -35,6 +40,7 @@ export class Prover implements IProver {
     }
 
     logger.info('Proof successfully generated');
+    incrementProofGenerated();
 
     const proof = JSON.parse(st.proof);
 

relay/implementations/publish_evm_transaction.ts

@@ -3,6 +3,7 @@ import Web3 from 'web3';
 import { FeeHistoryResult } from 'web3-eth';
 import { groth16 } from 'snarkjs';
 import { getGenericLogger } from '@dendreth/utils/ts-utils/logger';
+import { incrementProofPublished } from '@dendreth/utils/ts-utils/prometheus-utils';
 import { explorerUrls } from '@dendreth/utils/ts-utils/common-utils';
 
 const logger = getGenericLogger();
@@ -73,6 +74,7 @@ export async function publishTransaction(
         });
       }
       if (chainName) {
+        incrementProofPublished(chainName);
         logger.info(
           'Transaction uploaded at: ',
           explorerUrls[chainName].tx(transaction.hash),

relay/workers/prover/prover-worker.ts

@@ -7,12 +7,18 @@ import { Prover } from '@/implementations/prover';
 import { PROOF_GENERATOR_QUEUE } from '@/constants/constants';
 import { checkConfig } from '@dendreth/utils/ts-utils/common-utils';
 import yargs from 'yargs';
+import {
+  initPrometheusSetup,
+  registerGaugesForProver,
+} from '@dendreth/utils/ts-utils/prometheus-utils';
 
 (async () => {
   const proverConfig = {
     REDIS_HOST: process.env.REDIS_HOST || 'localhost',
     REDIS_PORT: Number(process.env.REDIS_PORT) || 6379,
   };
+  initPrometheusSetup(3000);
+  registerGaugesForProver();
 
   checkConfig(proverConfig);