Skip to content

Commit 5352968

Browse files
committed
chore: Add semantic versioning
1 parent 592730e commit 5352968

File tree

4 files changed

+180
-170
lines changed

4 files changed

+180
-170
lines changed

create_dict.py

Lines changed: 12 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,8 @@
1313

1414
logger = getLogger(__name__)
1515

16+
SENS_PG_TYPES = ["text", "integer", "bigint", "character", "json"]
17+
1618

1719
class TaggedFields:
1820
def __init__(
@@ -264,18 +266,12 @@ def check_sensitive_data_in_fld(
264266
return dict_matches
265267

266268

267-
def exclude_tagged_fields(
268-
field_info: FieldInfo, tagged_fields: List[Optional[TaggedFields]]
269-
):
270-
for field in tagged_fields:
271-
if (
272-
field.nspname == field_info.nspname
273-
and field.relname == field_info.relname
274-
and field.column_name == field_info.column_name
275-
):
276-
# res[task["obj_id"]] = task
277-
# scanning_flag = False
278-
return {field_info.obj_id: field_info}
269+
def check_sens_pg_types(field_type: str):
270+
"""Check if actual field type is sens."""
271+
for pg_type in SENS_PG_TYPES:
272+
if pg_type in field_type:
273+
return True
274+
return False
279275

280276

281277
async def scan_obj_func(
@@ -292,10 +288,7 @@ async def scan_obj_func(
292288
logger.debug("====>>> Process[%s]: Started task %s" % (name, str(field_info)))
293289

294290
start_t = time.time()
295-
if not (
296-
field_info.type in ("text", "integer", "bigint")
297-
or field_info.type.find("character varying") > -1
298-
):
291+
if not check_sens_pg_types(field_info.type):
299292
logger.debug(
300293
"========> Process[%s]: scan_obj_func: task %s skipped by field type %s"
301294
% (name, str(field_info), "[integer, text, bigint, character varying(x)]")
@@ -312,11 +305,13 @@ async def scan_obj_func(
312305
and field.relname == field_info.relname
313306
and field.column_name == field_info.column_name
314307
):
315-
res[field_info.obj_id] = field_info
308+
if ":sens" in field.column_comment:
309+
res[field_info.obj_id] = field_info
316310
scanning_flag = False
317311
break
318312

319313
if scan_mode == ScanMode.PARTIAL and scanning_flag:
314+
# TODO: Create check for bigger than 10MB fields
320315
fld_data = await db_conn.fetch(
321316
"""SELECT distinct(\"%s\")::text FROM \"%s\".\"%s\" WHERE \"%s\" is not null LIMIT %s"""
322317
% (

0 commit comments

Comments
 (0)