update data_formats to new format/pylint/etc

Author: AlexanderViand

p-isa_tools/data_formats/proto/heracles/data.proto

@@ -43,9 +43,9 @@ message FHEContext {
     repeated uint32        psi         = 6;  // 2n-th root of Z_{q_i}, implies elsewhere used n-th root omega = psi^2
     uint32                 q_size      = 7;  // n(Q), for SEAL it is always ( key_rns_num - 1 ), OpenFHE (cc->GetElementParams()->GetParams().size())
     uint32                 alpha       = 8;  // ceil(q_size/dnum)
-    oneof scheme_specific {
-        CKKSSpecific        ckks_info   = 9;
-        BGVSpecific        bgv_info    = 10;
+    oneof                  scheme_specific {
+        CKKSSpecific ckks_info = 9;
+        BGVSpecific  bgv_info  = 10;
     }
 }
 
@@ -76,10 +76,9 @@ message TestVector {
 
 // TODO: merge this with above
 message Data {
-    DCRTPoly   dcrtpoly   = 1; // this is only used for v2
+    DCRTPoly dcrtpoly = 1;  // this is only used for v2
 }
 
-
 // MAIN INTERFACE FOR CONSUMERs &
 // ROOT TYPES FOR SERIALIZATIONS
 //=================================
@@ -129,17 +128,16 @@ message BGVPlaintextSpecific {
 }
 
 message CKKSSpecific {
-    Keys                keys                    = 1;
-    uint32              composite_degree        = 2; // BASE_NUM_LEVELS_TO_DROP
+    Keys   keys             = 1;
+    uint32 composite_degree = 2;  // BASE_NUM_LEVELS_TO_DROP
     // Scaling factors
-    repeated double     scaling_factor_real     = 3; // size: q_size (CryptoParametersRNS->GetScalingFactorReal())
-    repeated double     scaling_factor_real_big = 4; // size: q_size - 1(CryptoParametersRNS->GetScalingFactorRealBig())
+    repeated double     scaling_factor_real     = 3;  // size: q_size (CryptoParametersRNS->GetScalingFactorReal())
+    repeated double     scaling_factor_real_big = 4;  // size: q_size - 1(CryptoParametersRNS->GetScalingFactorRealBig())
     map<string, uint32> metadata_extra          = 5;
 }
 
-
 message Keys {
-    KeySwitch          relin_key     = 1;
+    KeySwitch              relin_key     = 1;
     map<uint32, KeySwitch> rotation_keys = 2;
 }
 
@@ -169,7 +167,7 @@ message Plaintext {
 
 // DCRTPoly form
 message DCRTPoly {
-    repeated Polynomial polys       = 1; // size = order
+    repeated Polynomial polys       = 1;  // size = order
     bool                in_ntt_form = 2;
 }
 
@@ -182,8 +180,6 @@ message Polynomial {
 message RNSPolynomial {
     repeated uint32 coeffs  = 1;  // repeated a power-of-two times with power-of-two
     uint32          modulus = 2;  // need modulus in case curr_rns < max_rns, as it will use part of Q and all of P
-
-
 }
 
 message HECRNSPolynomial {

p-isa_tools/data_formats/proto/heracles/fhe_trace.proto

@@ -18,56 +18,56 @@ message Trace {
     // Sequence of HE instructions
     repeated Instruction   instructions = 1;
     heracles.common.Scheme scheme       = 2;
-    uint32                 N            = 3; // poly modulus degree
+    uint32                 N            = 3;  // poly modulus degree
     uint32                 key_rns_num  = 4;
-    uint32                 q_size       = 5; // n(Q)
-    uint32                 dnum         = 6; // digit size
-    uint32                 alpha        = 7; // ceil(n(Q)/dnum) Note: key_rns_num=n(Q) + n(P)
+    uint32                 q_size       = 5;  // n(Q)
+    uint32                 dnum         = 6;  // digit size
+    uint32                 alpha        = 7;  // ceil(n(Q)/dnum) Note: key_rns_num=n(Q) + n(P)
 }
 
 message Instruction {
-    string                 op              = 1;
-    uint32                 plaintext_index = 2;  // which plaintext algebra used, can be ignored for CKKS. Used as index into `plaintext_specific` field of `heracles.data.BGVSpecific` object inside `heracles.data.FHEContext`.
-    Operands               args            = 3;  // inputs/outputs and additional params
-    string                 evalop_name     = 4;  // (OpenFHE specific) Evaluator level call tracking, helps identifying what eval op invoked atomic ops
+    string   op              = 1;
+    uint32   plaintext_index = 2;  // which plaintext algebra used, can be ignored for CKKS. Used as index into `plaintext_specific` field of `heracles.data.BGVSpecific` object inside `heracles.data.FHEContext`.
+    Operands args            = 3;  // inputs/outputs and additional params
+    string   evalop_name     = 4;  // (OpenFHE specific) Evaluator level call tracking, helps identifying what eval op invoked atomic ops
 }
 
 message Operands {
-    repeated OperandObject      dests   = 1;
-    repeated OperandObject      srcs   = 2;
-    map<string, Parameter>      params = 3;
+    repeated OperandObject dests  = 1;
+    repeated OperandObject srcs   = 2;
+    map<string, Parameter> params = 3;
 }
 
 message Parameter {
-    string          value   = 1;
-    ValueType       type    = 2;
+    string    value = 1;
+    ValueType type  = 2;
 }
 enum ValueType {
-    UINT32              = 0 [
+    UINT32 = 0 [
         (valuetype_name) = "UINT32"
     ];
-    UINT64              = 1 [
+    UINT64 = 1 [
         (valuetype_name) = "UINT64"
     ];
-    INT32               = 2 [
+    INT32 = 2 [
         (valuetype_name) = "INT32"
     ];
-    INT64               = 3 [
+    INT64 = 3 [
         (valuetype_name) = "INT64"
     ];
-    FLOAT               = 4 [
+    FLOAT = 4 [
         (valuetype_name) = "FLOAT"
     ];
-    DOUBLE              = 5 [
+    DOUBLE = 5 [
         (valuetype_name) = "DOUBLE"
     ];
-    STRING              = 6 [
+    STRING = 6 [
         (valuetype_name) = "STRING"
     ];
 }
 
 message OperandObject {
     string symbol_name = 1;
-    uint32 num_rns     = 2; // size = curr_rns of dcrtpoly
-    uint32 order       = 3; // typically 2 for ct/pt (can be 3), single DCRTPoly will always be 1
+    uint32 num_rns     = 2;  // size = curr_rns of dcrtpoly
+    uint32 order       = 3;  // typically 2 for ct/pt (can be 3), single DCRTPoly will always be 1
 }

p-isa_tools/data_formats/python/heracles/data/io.py

@@ -4,32 +4,27 @@
 
 # TODO: create also C++ variants of below; given how simple and stable these functions should be just in replicated form, not shared code
 
-import heracles.proto.data_pb2 as hpd
-import heracles.proto.common_pb2 as hpc
-import heracles.proto.fhe_trace_pb2 as hpf
-import google.protobuf.json_format as gpj
 import json
-import heracles.data.transform as hdt
-from google.protobuf.json_format import MessageToJson, MessageToDict
-from glob import glob
-import re
 import sys
 
+import heracles.proto.data_pb2 as hpd
+from google.protobuf.json_format import MessageToDict
+
 # load & store functions
 # ===============================
 
 
 def parse_manifest(filename: str) -> dict:
-    manifest = dict()
-    with open(filename, "r") as fp:
+    manifest: dict = {}
+    with open(filename) as fp:
         cur_field = None
         found_first_field = False
         for linenum, cur_line in enumerate(fp):
             cur_line = cur_line.rstrip()
             if cur_line.startswith("[") and cur_line.endswith("]"):
                 cur_field = cur_line[1:-1]
                 found_first_field = True
-                manifest[cur_field] = dict()
+                manifest[cur_field] = {}
                 continue
 
             if not found_first_field:
@@ -60,7 +55,7 @@ def generate_manifest(filename: str, manifest: dict):
 # re-check
 def store_hec_context_json(filename: str, context: hpd.FHEContext):
     print(
-        f"Warning: Dumping FHE Context data trace to json can take a long time",
+        "Warning: Dumping FHE Context data trace to json can take a long time",
         file=sys.stderr,
     )
     with open(filename, "w") as fp:
@@ -70,7 +65,7 @@ def store_hec_context_json(filename: str, context: hpd.FHEContext):
 # re-check
 def store_testvector_json(filename: str, testvector: hpd.TestVector):
     print(
-        f"Warning: Dumping TestVector data trace to json can take a long time",
+        "Warning: Dumping TestVector data trace to json can take a long time",
         file=sys.stderr,
     )
     with open(filename, "w") as fp:
@@ -95,15 +90,13 @@ def load_hec_context_from_manifest(manifest: dict) -> hpd.FHEContext:
 
 
 def store_hec_context(filename: str, context_pb: hpd.FHEContext) -> dict:
-    hec_context_manifest = {"context": dict()}
+    hec_context_manifest: dict = {"context": {}}
     tmp_context = hpd.FHEContext()
     tmp_context.CopyFrom(context_pb)
 
     if tmp_context.ByteSize() > 1 << 30:
-        hec_context_manifest["rotation_keys"] = dict()
-        for gkct, (ge, gk_pb) in enumerate(
-            tmp_context.ckks_info.keys.rotation_keys.items()
-        ):
+        hec_context_manifest["rotation_keys"] = {}
+        for gkct, (ge, gk_pb) in enumerate(tmp_context.ckks_info.keys.rotation_keys.items()):
             parts_fn = f"{filename}_hec_context_part_{gkct + 1}"
             hec_context_manifest["rotation_keys"][ge] = parts_fn
             with open(parts_fn, "wb") as fp:
@@ -137,7 +130,7 @@ def load_testvector_from_manifest(manifest: dict) -> hpd.TestVector:
 
 
 def store_testvector(filename: str, testvector_pb: hpd.TestVector) -> dict:
-    testvector_manifest = {"testvector": dict()}
+    testvector_manifest: dict = {"testvector": {}}
     if testvector_pb.ByteSize() > 1 << 30:
         for tvct, (sym, data_pb) in enumerate(testvector_pb.sym_data_map.items()):
             parts_fn = f"{filename}_testvector_part_{tvct}"
@@ -171,9 +164,7 @@ def load_data_trace(filename: str) -> tuple[hpd.FHEContext, hpd.TestVector]:
     )
 
 
-def store_data_trace(
-    filename: str, context_pb: hpd.FHEContext, testvector_pb: hpd.TestVector
-):
+def store_data_trace(filename: str, context_pb: hpd.FHEContext, testvector_pb: hpd.TestVector):
     generate_manifest(
         filename,
         {