Generate metrics from model's layers

aiu_fms_testing_utils/utils/__init__.py

@@ -9,6 +9,7 @@
 # Third Party
 from aiu_fms_testing_utils.utils.aiu_setup import dprint
 from fms.utils.tokenizers import BaseTokenizer
+from fms.utils.generation import pad_input_ids
 import torch
 import torch.nn as nn
 
@@ -166,3 +167,19 @@ def sample_squad_v2_qa_requests(
         prompt_length_max,
         seed,
     )
+
+def prepare_inputs(batch_size, seq_length, tokenizer, sharegpt_path, seed=0):
+    prompts_and_sizes = sample_sharegpt_requests(
+        sharegpt_path,
+        batch_size,
+        tokenizer,
+        int(seq_length / 2),
+        seq_length,
+        seed,
+    )
+    prompt_list = []
+    for prompt, _ in prompts_and_sizes:
+        prompt_list.append(ids_for_prompt(prompt, tokenizer))
+
+    input_ids, padding_kwargs = pad_input_ids(prompt_list, min_pad_length=seq_length)
+    return input_ids, padding_kwargs

aiu_fms_testing_utils/utils/metrics_utils.py

@@ -0,0 +1,72 @@
+import numpy as np
+import torch
+import torch.nn as nn
+
+
+def abs_diff_linalg_norm(res_vector):
+    """
+    Calculates the Euclidean norm (also known as the L2 norm) of a given array res_vector. This is equivalent to finding the square
+    root of the sum of the squares of all the elements in the array. It's a fundamental operation in linear algebra and is often used 
+    to measure the "length" or "magnitude" of a vector. More at https://numpy.org/devdocs/reference/generated/numpy.linalg.norm.html
+    Args:
+        res_vector (list): The list of abs diff
+
+    Returns:
+        float: "magnitude" of the diff vector.
+    """
+    return np.linalg.norm(res_vector)
+
+def list_mean(val_list):
+    """
+    Calculates the mean for all the values in a given list.
+    Args:
+        val_list (list): The list of values
+
+    Returns:
+        float: mean value calculated.
+    """
+    return np.mean(val_list)
+
+def tensor_abs_diff(tensor1, tensor2):
+    """
+    Calculate the absolute difference between two tensors.
+
+    Args:
+        tensor1 (torch.Tensor): The first input tensor.
+        tensor2 (torch.Tensor): The second input tensor.
+
+    Returns:
+        torch.Tensor: The absolute difference tensor.
+
+    Example:
+        >>> tensor1 = torch.tensor([1, 2, 3])
+        >>> tensor2 = torch.tensor([4, 5, 6])
+        >>> abs_diff(tensor1, tensor2)
+        torch.tensor([3, 3, 3])
+    """
+    abs_diff = torch.abs(tensor1 - tensor2)
+    abs_diff[abs_diff == 0.0] = 1e-6
+    return abs_diff
+
+def tensor_cos_sim(tensor1, tensor2):
+    """
+    Computes the cosine similarity between two tensors.
+
+    Args:
+        tensor1 (torch.Tensor): The first input tensor.
+        tensor2 (torch.Tensor): The second input tensor.
+
+    Returns:
+        torch.Tensor: The cosine similarity between the two input tensors.
+
+    Example:
+        >>> import torch
+        >>> tensor1 = torch.randn(3, 5)
+        >>> tensor2 = torch.randn(3, 5)
+        >>> sim = cos_sim(tensor1, tensor2)
+        >>> print(sim)
+    """
+    cos = nn.CosineSimilarity(dim=-1)
+    cos_sim = cos(tensor1, tensor2)
+    cos_sim[cos_sim == 0.0] = 1e-6
+    return cos_sim