Importer rework - various fortimanager enhancements

roles/database/files/sql/creation/fworch-create-constraints.sql

@@ -17,8 +17,8 @@ Alter Table "import_control" add Constraint "control_id_stop_time_unique" UNIQUE
 Alter Table "object" add Constraint "obj_altkey" UNIQUE ("mgm_id","zone_id","obj_uid","obj_create");
 ALTER TABLE object ADD CONSTRAINT object_obj_ip_is_host CHECK (is_single_ip(obj_ip));
 ALTER TABLE object ADD CONSTRAINT object_obj_ip_end_is_host CHECK (is_single_ip(obj_ip_end));
-ALTER TABLE object ADD CONSTRAINT object_obj_ip_not_null CHECK (obj_ip IS NOT NULL OR obj_typ_id=2);
-ALTER TABLE object ADD CONSTRAINT object_obj_ip_end_not_null CHECK (obj_ip_end IS NOT NULL OR obj_typ_id=2);
+ALTER TABLE "object" ADD CONSTRAINT object_obj_ip_not_null CHECK NOT (obj_ip IS NULL AND obj_typ_id IN (1, 3, 4));
+ALTER TABLE "object" ADD CONSTRAINT object_obj_ip_end_not_null CHECK NOT (obj_ip_end IS NULL AND obj_typ_id IN (1, 3, 4));
 ALTER TABLE owner ADD CONSTRAINT owner_name_unique_in_tenant UNIQUE ("name","tenant_id");
 ALTER TABLE owner_network ADD CONSTRAINT port_in_valid_range CHECK (port > 0 and port <= 65535);
 ALTER TABLE owner_network ADD CONSTRAINT owner_network_ip_is_host CHECK (is_single_ip(ip));

roles/database/files/upgrade/9.0.sql

@@ -1061,6 +1061,12 @@ ALTER TABLE "rulebase_link" ADD COLUMN IF NOT EXISTS "removed" BIGINT;
 -- add obj type access-role for cp import
 INSERT INTO stm_obj_typ (obj_typ_id,obj_typ_name) VALUES (21,'access-role') ON CONFLICT DO NOTHING;
 
+-- change ip not Null constraint, only applicable if obj_typ_name is host, network or machines_range
+ALTER TABLE "object" DROP CONSTRAINT IF EXISTS "object_obj_ip_not_null" CASCADE;
+ALTER TABLE "object" DROP CONSTRAINT IF EXISTS "object_obj_ip_end_not_null" CASCADE;
+ALTER TABLE "object" ADD CONSTRAINT object_obj_ip_not_null CHECK NOT (obj_ip IS NULL AND obj_typ_id IN (1, 3, 4));
+ALTER TABLE "object" ADD CONSTRAINT object_obj_ip_end_not_null CHECK NOT (obj_ip_end IS NULL AND obj_typ_id IN (1, 3, 4));
+
 -- remove dev_id fk and set nullable if column exists
 ALTER TABLE changelog_rule DROP CONSTRAINT IF EXISTS changelog_rule_dev_id_fkey;
 

roles/importer/files/importer/checkpointR8x/cp_rule.py

@@ -24,7 +24,7 @@
     - migrate section headers from rule to ordering element 
     ...
 """
-def normalize_rulebases (nativeConfig, native_config_global, importState, normalized_config_dict, 
+def normalize_rulebases(nativeConfig, native_config_global, importState, normalized_config_dict, 
                          normalized_config_global, is_global_loop_iteration):
 
     normalized_config_dict['policies'] = []

roles/importer/files/importer/common.py

@@ -92,8 +92,9 @@ def import_management(mgmId=None, ssl_verification=None, debug_level_in=0,
     except (FwoApiWriteError, FwoImporterError) as e:
         importState.addError(f"FwoApiWriteError or FwoImporterError: {str(e.args)} - aborting import")
         rollBackExceptionHandler(importState, configImporter=config_importer, exc=e, errorText="")
-    except FwoImporterErrorInconsistencies:
+    except FwoImporterErrorInconsistencies as e:
         importState.delete_import() # delete whole import
+        importState.addError(str(e.args))
     except ValueError:
         importState.addError("ValueError - aborting import")
         raise

roles/importer/files/importer/fortiadom5ff/fmgr_consts.py

@@ -2,10 +2,12 @@
 nw_obj_types = ['firewall/address', 'firewall/address6', 'firewall/addrgrp',
                 'firewall/addrgrp6', 'firewall/ippool', 'firewall/vip', 'system/external-resource',
                 'firewall/wildcard-fqdn/custom', 'firewall/wildcard-fqdn/group']
-# TODO add 
+
 svc_obj_types = ['application/list', 'application/group', 'application/categories',
                  'application/custom', 'firewall/service/custom', 'firewall/service/group']
 
+nat_types = ['central/dnat', 'central/dnat6', 'firewall/central-snat-map']
+
 # delte_v: beide typen können weg
 # v4_object_types = ['nw_obj_global_firewall/address', 'nw_obj_global_firewall/addrgrp']
 # v6_object_types = ['nw_obj_adom_firewall/address', 'nw_obj_adom_firewall/addrgrp','nw_obj_global_firewall/address', \

roles/importer/files/importer/fortiadom5ff/fmgr_getter.py

@@ -98,7 +98,6 @@ def update_config_with_fortinet_api_call(config_json, sid, api_base_url, api_pat
         # adding options
         if len(options)>0:
             payload['params'][0].update({'option': options})
-            # payload['params'][0].update({'filter': options})
 
         result = fortinet_api_call(sid, api_base_url, api_path, payload=payload, method=method)
         full_result.extend(result)
@@ -159,12 +158,3 @@ def parse_device_and_vdom(fmgr_device, mgm_details_device, device_vdom_dict, fou
                 else:
                     device_vdom_dict.update({fmgr_device['name']: {fmgr_vdom['name']: ''}})
     return found_fmgr_device
-
-def get_policy_packages_from_manager(sid, fm_api_url, adom=''):
-    if adom == '':
-        url = '/pm/pkg/global'
-    else:
-        url = '/pm/pkg/adom/' + adom
-    policy_packages_result = fortinet_api_call(sid, fm_api_url, url)
-
-    return policy_packages_result

roles/importer/files/importer/fortiadom5ff/fmgr_network.py

@@ -2,14 +2,14 @@
 import ipaddress
 from fwo_log import getFwoLogger
 from fwo_const import list_delimiter, nat_postfix
-from fmgr_zone import add_zone_if_missing
+from fmgr_zone import find_zones_in_normalized_config
 from fwo_config import readConfig
 from model_controllers.import_state_controller import ImportStateController
 from copy import deepcopy
 from fwo_exceptions import FwoImporterErrorInconsistencies
 
 
-def normalize_network_objects(import_state: ImportStateController, native_config, native_config_global, normalized_config, normalized_config_global, nw_obj_types):
+def normalize_network_objects(native_config, normalized_config_adom, normalized_config_global, nw_obj_types):
     nw_objects = []
 
     if 'objects' not in native_config:
@@ -19,7 +19,7 @@ def normalize_network_objects(import_state: ImportStateController, native_config
         if not(current_obj_type in nw_obj_types and 'data' in native_config['objects'][current_obj_type]):
             continue
         for obj_orig in native_config['objects'][current_obj_type]['data']:
-            normalize_network_object(obj_orig, nw_objects, normalized_config, native_config['objects'], current_obj_type)
+            normalize_network_object(obj_orig, nw_objects, normalized_config_adom, normalized_config_global, native_config['objects'], current_obj_type)
 
     if native_config.get('is-super-manager',False):
         # finally add "Original" network object for natting (only in global domain)
@@ -28,7 +28,7 @@ def normalize_network_objects(import_state: ImportStateController, native_config
         nw_objects.append(create_network_object(name=original_obj_name, type='network', ip='0.0.0.0', ip_end='255.255.255.255',\
             uid=original_obj_uid, zone='global', color='black', comment='"original" network object created by FWO importer for NAT purposes'))
 
-    normalized_config.update({'network_objects': nw_objects})
+    normalized_config_adom.update({'network_objects': nw_objects})
 
 def get_obj_member_refs_list(obj_orig, native_config_objects, current_obj_type):
     obj_member_refs_list = []
@@ -52,8 +52,7 @@ def exclude_object_types_in_member_ref_search(obj_type, current_obj_type):
             skip_member_ref_loop = True
     return skip_member_ref_loop
 
-def normalize_network_object(obj_orig, nw_objects, normalized_config, native_config_objects, current_obj_type):
-    obj_zone = 'global'
+def normalize_network_object(obj_orig, nw_objects, normalized_config_adom, normalized_config_global, native_config_objects, current_obj_type):
     obj = {}
     obj.update({'obj_name': obj_orig['name']})
     if 'subnet' in obj_orig: # ipv4 object
@@ -98,17 +97,12 @@ def normalize_network_object(obj_orig, nw_objects, normalized_config, native_con
 
     obj.update({'obj_uid': obj_orig.get('uuid', obj_orig['name'])})  # using name as fallback, but this should not happen
 
-    # here only picking first associated interface as zone:
-    if 'associated-interface' in obj_orig and len(obj_orig['associated-interface'])>0: # and obj_orig['associated-interface'][0] != 'any':
-        obj_zone = deepcopy(obj_orig['associated-interface'][0])
-        # adding zone if it not yet exists
-        obj_zone = add_zone_if_missing (normalized_config, obj_zone)
-    obj.update({'obj_zone': obj_zone })
+    associated_interfaces = find_zones_in_normalized_config(
+        obj_orig.get('associated-interface', []), normalized_config_adom, normalized_config_global)
+    obj.update({'obj_zone': list_delimiter.join(associated_interfaces)})
 
-    #obj.update({'control_id': import_state.ImportId})
     nw_objects.append(obj)
 
-
 def _parse_subnet (obj, obj_orig):
     ipa = ipaddress.ip_network(str(obj_orig['subnet'][0]) + '/' + str(obj_orig['subnet'][1]))
     if ipa.num_addresses > 1: