Merge pull request #15 from Pwnzer0tt1/dev-nfproxy

1.5.1 Release

Author: domysh

backend/binsrc/nfregex.cpp

@@ -50,6 +50,21 @@ void config_updater (){
 }
 
 int main(int argc, char *argv[]){
+
+	char * test_regex = getenv("FIREGEX_TEST_REGEX");
+	if (test_regex != nullptr){
+		cerr << "[info] [main] Testing regex: " << test_regex << endl;
+		try{
+			RegexRules::compile_regex(test_regex);
+			cerr << "[info] [main] Test passed" << endl;
+			return 0;
+		}catch(const std::exception& e){
+			cerr << "[error] [updater] Test failed" << endl;
+			cout << e.what() << flush;
+			return 1;
+		}
+	}
+
 	int n_of_threads = 1;
    	char * n_threads_str = getenv("NTHREADS");
    	if (n_threads_str != nullptr) n_of_threads = ::atoi(n_threads_str);

backend/binsrc/regex/regex_rules.cpp

@@ -59,6 +59,26 @@ class RegexRules{
 	public:
 		regex_ruleset output_ruleset, input_ruleset;
 
+	static void compile_regex(char* regex){
+		hs_database_t* db = nullptr;
+		hs_compile_error_t *compile_err = nullptr;
+		if (
+			hs_compile(
+				regex,
+				HS_FLAG_SINGLEMATCH | HS_FLAG_ALLOWEMPTY,
+				HS_MODE_BLOCK,
+				nullptr, &db, &compile_err
+			) != HS_SUCCESS
+		) {
+			string err = string(compile_err->message);
+			hs_free_compile_error(compile_err);
+			throw runtime_error(err);
+		}else{
+			hs_free_database(db);
+		}
+		
+	}
+
 	private:
 		static inline u_int16_t glob_seq = 0;
 		u_int16_t version;
@@ -77,6 +97,8 @@ class RegexRules{
 			}
 		}
 
+
+
 		void fill_ruleset(vector<pair<string, decoded_regex>> & decoded, regex_ruleset & ruleset){
 			size_t n_of_regex = decoded.size();
 			if (n_of_regex == 0){

backend/modules/nfregex/firegex.py

@@ -1,7 +1,6 @@
 from modules.nfregex.nftables import FiregexTables
 from utils import run_func
 from modules.nfregex.models import Service, Regex
-import re
 import os
 import asyncio
 import traceback
@@ -10,6 +9,20 @@
 
 nft = FiregexTables()
 
+async def test_regex_validity(regex: str) -> bool:
+    proxy_binary_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),"../cppregex")
+    process = await asyncio.create_subprocess_exec(
+        proxy_binary_path,
+        stdout=asyncio.subprocess.PIPE,
+        stdin=asyncio.subprocess.DEVNULL,
+        env={"FIREGEX_TEST_REGEX": regex},
+    )
+    await process.wait()
+    if process.returncode != 0:
+        message = (await process.stdout.read()).decode()
+        return False, message
+    return True, "ok"
+
 class RegexFilter:
     def __init__(
         self, regex,
@@ -44,7 +57,6 @@ def compile(self):
             self.regex = self.regex.encode()
         if not isinstance(self.regex, bytes):
             raise Exception("Invalid Regex Paramether")
-        re.compile(self.regex) # raise re.error if it's invalid!
         case_sensitive = "1" if self.is_case_sensitive else "0"
         if self.input_mode:
             yield case_sensitive + "C" + self.regex.hex()

backend/routers/nfregex.py

@@ -1,5 +1,4 @@
 from base64 import b64decode
-import re
 import secrets
 import sqlite3
 from fastapi import APIRouter, Response, HTTPException
@@ -9,6 +8,7 @@
 from utils.sqlite import SQLite
 from utils import ip_parse, refactor_name, socketio_emit, PortType
 from utils.models import ResetRequest, StatusMessageModel
+from modules.nfregex.firegex import test_regex_validity
 
 class ServiceModel(BaseModel):
     status: str
@@ -299,10 +299,9 @@ async def regex_disable(regex_id: int):
 @app.post('/regexes', response_model=StatusMessageModel)
 async def add_new_regex(form: RegexAddForm):
     """Add a new regex"""
-    try:
-        re.compile(b64decode(form.regex))
-    except Exception:
-        raise HTTPException(status_code=400, detail="Invalid regex")
+    regex_correct, message = await test_regex_validity(b64decode(form.regex))
+    if not regex_correct:
+        raise HTTPException(status_code=400, detail=f"Invalid regex: {message}")
     try:
         db.query("INSERT INTO regexes (service_id, regex, mode, is_case_sensitive, active ) VALUES (?, ?, ?, ?, ?);", 
                 form.service_id, form.regex, form.mode, form.is_case_sensitive, True if form.active is None else form.active )

start.py

@@ -100,11 +100,11 @@ def gen_args(args_to_parse: list[str]|None = None):
     #Start Command
     parser_start = subcommands.add_parser('start', help='Start the firewall')
     parser_start.add_argument('--threads', "-t", type=int, required=False, help='Number of threads started for each service/utility', default=-1)
-    parser_start.add_argument('--psw-no-interactive',type=str, required=False, help='Password for no-interactive mode', default=None)
-    parser_start.add_argument('--startup-psw','-P', required=False, action="store_true", help='Insert password in the startup screen of firegex', default=False)
+    parser_start.add_argument('--startup-psw','-P', required=False, help='Insert password in the startup screen of firegex', type=str, default=None)
+    parser_start.add_argument('--psw-on-web', required=False, help='Setup firegex password on the web interface', action="store_true", default=False)
     parser_start.add_argument('--port', "-p", type=int, required=False, help='Port where open the web service of the firewall', default=4444)
     parser_start.add_argument('--logs', required=False, action="store_true", help='Show firegex logs', default=False)
-    parser_start.add_argument('--version', '-v', required=False, type=str , help='Version of the firegex image to use', default="latest")
+    parser_start.add_argument('--version', '-v', required=False, type=str , help='Version of the firegex image to use', default=None)
 
     #Stop Command
     parser_stop = subcommands.add_parser('stop', help='Stop the firewall')
@@ -221,10 +221,10 @@ def write_compose(skip_password = True):
             }))
 
 def get_password():
-    if volume_exists() or args.startup_psw:
+    if volume_exists() or args.psw_on_web:
         return None
-    if args.psw_no_interactive:
-        return args.psw_no_interactive
+    if args.startup_psw:
+        return args.startup_psw
     psw_set = None
     while True:
         while True:

tests/README.md

@@ -79,14 +79,37 @@ You will find a new benchmark.csv file containg the results.
 
 The test was performed on:
 - Macbook Air M2 16GB RAM
-- On a VM powered by OrbStack with Ubuntu 24.04.1 LTS aarch64
-- 6.12.10-orbstack-00297-gf8f6e015b993
+- On a VM powered by OrbStack with Fedora Linux 41 (Container Image) aarch64
+- Linux 6.12.13-orbstack-00304-gede1cf3337c4
 
-Command: `./benchmark.py -p testpassword -r 50 -d 1 -s 60`
+Command: `./benchmark.py -p testpassword -r 50 -d 1 -s 50`
 
-### NOTE: 8 threads performance do not change due to the fact that the source and destination ip is always the same, so the packets are sent to the same thread by the kernel.
+NOTE: 8 threads performance before 2.5.0 do not change due to the fact that the source and destination ip is always the same, so the packets are sent to the same thread by the kernel.
 [https://netfilter.vger.kernel.narkive.com/sTP7613Y/meaning-of-nfqueue-s-queue-balance-option](https://netfilter.vger.kernel.narkive.com/sTP7613Y/meaning-of-nfqueue-s-queue-balance-option)
 
 Internally the kernel hashes the source and dest ip and choose the target thread based on the hash. If the source and dest ip are the same, the hash will be the same and the packets will be sent to the same thread.
+This is a problem in a CTF, where we usually have a NAT to hide real IPs.
+
+Firegex 2.5.0 changes the way the threads are assigned to the packets, this is done userland, so we can have a better distribution of the packets between the threads.
+
+The charts are labeled as follows: `[version]-[n_thread]T` eg. `2.5.0-8T` means Firegex version 2.5.0 with 8 threads.
 
 ![Firegex Benchmark](results/Benchmark-chart.png)
+
+
+From the benchmark above we can't see the real advantage of multithreading in 2.5.1, we can better see the advantage of multithreading in the chart below where a fake load in filtering is done.
+
+The load is simulated by this code:
+```cpp
+volatile int x = 0;
+for (int i=0; i<50000; i++){
+    x+=1;
+}
+```
+
+![Firegex Benchmark](results/Benchmark-chart-with-load.png)
+
+In the chart above we can see that the 2.5.1 version with 8 threads has a better performance than the 2.5.1 version with 1 threads, and we can see it as much as the load increases.
+
+This particular advantage will be more noticeable with nfproxy module that is not implemented yet.
+

tests/benchmark.py

@@ -45,13 +45,22 @@ def exit_test(code):
 
 #Create new Service
 
+srvs = firegex.nf_get_services()
+for ele in srvs:
+    if ele['name'] == args.service_name:
+        firegex.nf_delete_service(ele['service_id'])
+
 service_id = firegex.nf_add_service(args.service_name, args.port, "tcp", "127.0.0.1/24")
 if service_id:
     puts(f"Sucessfully created service {service_id} ✔", color=colors.green)
 else:
     puts("Test Failed: Failed to create service ✗", color=colors.red)
     exit(1)
 
+args.port = int(args.port)
+args.duration = int(args.duration)
+args.num_of_streams = int(args.num_of_streams)
+
 #Start iperf3
 def startServer():
     server = iperf3.Server()
@@ -66,6 +75,8 @@ def getReading(port):
     client.duration = args.duration
     client.server_hostname = '127.0.0.1'
     client.port = port
+    client.zerocopy = True
+    client.verbose = False
     client.protocol = 'tcp'
     client.num_streams = args.num_of_streams
     return round(client.run().json['end']['sum_received']['bits_per_second']/8e+6 , 3)
@@ -74,6 +85,19 @@ def getReading(port):
 server.start()
 sleep(1)
 
+custom_regex = [
+        '(?:[a-z0-9!#$%&\'*+/=?^_`{|}~-]+(?:\\.[a-z0-9!#$%&\'*+/=?^_`{|}~-]+)*|"(?:[\\x01-\\x08\\x0b\\x0c\\x0e-\\x1f\\x21\\x23-\\x5b\\x5d-\\x7f]|\\\\[\\x01-\\x09\\x0b\\x0c\\x0e-\\x7f])*")@(?:(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?|\\[(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?|[a-z0-9-]*[a-z0-9]:(?:[\\x01-\\x08\\x0b\\x0c\\x0e-\\x1f\\x21-\\x5a\\x53-\\x7f]|\\\\[\\x01-\\x09\\x0b\\x0c\\x0e-\\x7f])+)\\])'
+]
+
+def gen_regex():
+    """
+    if len(custom_regex) == 0:
+        regex = secrets.token_hex(8)
+    else:
+        regex = custom_regex.pop()
+    """
+    regex = secrets.token_hex(20)
+    return base64.b64encode(bytes(regex.encode())).decode()
 
 #Get baseline reading 
 puts("Baseline without proxy: ", color=colors.blue, end='')
@@ -95,7 +119,7 @@ def getReading(port):
 
 #Add all the regexs
 for i in range(1,args.num_of_regexes+1):
-    regex = base64.b64encode(bytes(secrets.token_hex(16).encode())).decode()
+    regex = gen_regex()
     if not firegex.nf_add_regex(service_id,regex,"B",active=True,is_case_sensitive=False): 
         puts("Benchmark Failed: Couldn't add the regex ✗", color=colors.red)
         exit_test(1)

tests/results/2.3.3-1T.csv

@@ -0,0 +1,51 @@
+0,4090.616
+1,2211.62
+2,1165.45
+3,849.39
+4,828.635
+5,741.537
+6,632.721
+7,624.772
+8,529.234
+9,469.688
+10,336.33
+11,427.783
+12,400.662
+13,335.086
+14,342.042
+15,307.283
+16,239.694
+17,295.163
+18,285.787
+19,254.402
+20,250.553
+21,227.146
+22,238.747
+23,234.718
+24,210.484
+25,210.697
+26,205.943
+27,202.568
+28,194.341
+29,189.916
+30,154.228
+31,168.922
+32,173.623
+33,125.431
+34,162.154
+35,149.865
+36,150.088
+37,146.085
+38,137.182
+39,138.686
+40,136.302
+41,132.707
+42,100.928
+43,126.414
+44,125.271
+45,117.839
+46,89.494
+47,116.939
+48,112.517
+49,111.369
+50,108.568

tests/results/2.3.3-8T.csv

@@ -0,0 +1,51 @@
+0,3789.988
+1,2069.487
+2,1484.554
+3,956.972
+4,1052.873
+5,739.658
+6,534.722
+7,638.524
+8,573.833
+9,531.658
+10,476.167
+11,443.746
+12,406.027
+13,385.739
+14,341.563
+15,318.699
+16,303.722
+17,284.924
+18,284.336
+19,267.32
+20,202.74
+21,243.849
+22,226.082
+23,214.348
+24,216.8
+25,188.98
+26,158.68
+27,166.556
+28,148.287
+29,149.681
+30,177.043
+31,175.321
+32,165.312
+33,166.943
+34,159.026
+35,156.759
+36,150.216
+37,144.932
+38,146.088
+39,135.897
+40,136.99
+41,128.557
+42,100.307
+43,103.249
+44,123.49
+45,120.39
+46,118.055
+47,115.0
+48,112.593
+49,109.55
+50,109.512