13
13
14
14
logger = getLogger (__name__ )
15
15
16
+ SENS_PG_TYPES = ["text" , "integer" , "bigint" , "character" , "json" ]
17
+
16
18
17
19
class TaggedFields :
18
20
def __init__ (
@@ -264,18 +266,12 @@ def check_sensitive_data_in_fld(
264
266
return dict_matches
265
267
266
268
267
- def exclude_tagged_fields (
268
- field_info : FieldInfo , tagged_fields : List [Optional [TaggedFields ]]
269
- ):
270
- for field in tagged_fields :
271
- if (
272
- field .nspname == field_info .nspname
273
- and field .relname == field_info .relname
274
- and field .column_name == field_info .column_name
275
- ):
276
- # res[task["obj_id"]] = task
277
- # scanning_flag = False
278
- return {field_info .obj_id : field_info }
269
+ def check_sens_pg_types (field_type : str ):
270
+ """Check if actual field type is sens."""
271
+ for pg_type in SENS_PG_TYPES :
272
+ if pg_type in field_type :
273
+ return True
274
+ return False
279
275
280
276
281
277
async def scan_obj_func (
@@ -292,10 +288,7 @@ async def scan_obj_func(
292
288
logger .debug ("====>>> Process[%s]: Started task %s" % (name , str (field_info )))
293
289
294
290
start_t = time .time ()
295
- if not (
296
- field_info .type in ("text" , "integer" , "bigint" )
297
- or field_info .type .find ("character varying" ) > - 1
298
- ):
291
+ if not check_sens_pg_types (field_info .type ):
299
292
logger .debug (
300
293
"========> Process[%s]: scan_obj_func: task %s skipped by field type %s"
301
294
% (name , str (field_info ), "[integer, text, bigint, character varying(x)]" )
@@ -312,11 +305,13 @@ async def scan_obj_func(
312
305
and field .relname == field_info .relname
313
306
and field .column_name == field_info .column_name
314
307
):
315
- res [field_info .obj_id ] = field_info
308
+ if ":sens" in field .column_comment :
309
+ res [field_info .obj_id ] = field_info
316
310
scanning_flag = False
317
311
break
318
312
319
313
if scan_mode == ScanMode .PARTIAL and scanning_flag :
314
+ # TODO: Create check for bigger than 10MB fields
320
315
fld_data = await db_conn .fetch (
321
316
"""SELECT distinct(\" %s\" )::text FROM \" %s\" .\" %s\" WHERE \" %s\" is not null LIMIT %s"""
322
317
% (
0 commit comments