From 3275af51418629cf9b2d9f299250e43ea8bf237e Mon Sep 17 00:00:00 2001 From: Siyuan Date: Tue, 11 Feb 2025 18:48:58 -0500 Subject: [PATCH 01/12] modify run script on genie3 --- scripts/run-dctcp-tput-experiment.sh | 10 ++-- scripts/sosp24-experiments/flows_exp.sh | 50 +++++++++++++++----- scripts/sosp24-experiments/ringbuffer_exp.sh | 43 ++++++++++++----- 3 files changed, 75 insertions(+), 28 deletions(-) diff --git a/scripts/run-dctcp-tput-experiment.sh b/scripts/run-dctcp-tput-experiment.sh index a8bc94945..936df8c05 100755 --- a/scripts/run-dctcp-tput-experiment.sh +++ b/scripts/run-dctcp-tput-experiment.sh @@ -38,8 +38,8 @@ eval set -- "$OPTS" exp="benny-test" server="192.168.11.116" client="192.168.11.117" -server_intf="ens2f1np1" -client_intf="ens2f1" +server_intf="ens2f0np0" +client_intf="ens2f0" num_servers=5 num_clients=5 init_port=3000 @@ -182,7 +182,7 @@ function cleanup() { sleep 2 sudo ip link set $server_intf up sleep 2 - sudo bash /home/benny/restart.sh +# sudo bash /home/benny/restart.sh } @@ -207,7 +207,7 @@ fi #### setup and start servers echo "setting up server config..." -sudo bash /home/benny/restart.sh +# sudo bash /home/benny/restart.sh cd $setup_dir sudo bash setup-envir.sh -i $server_intf -a $server -m $mtu -d $ddio --ring_buffer $ring_buffer --buf $buf -f 1 -r 0 -p 0 -e 1 -o 1 cd - @@ -243,7 +243,7 @@ echo "done logging..." cd - #transfer sender-side info back to receiver -sshpass -p benny ssh benny@192.168.11.117 -- "sudo rm /dev/null; sudo mknod /dev/null c 1 3; sudo chmod 666 /dev/null" +# sshpass -p benny ssh benny@192.168.11.117 -- "sudo rm /dev/null; sudo mknod /dev/null c 1 3; sudo chmod 666 /dev/null" sshpass -p $password scp $uname@$ssh_hostname:$setup_dir/reports/$exp-RUN-$j/retx.rpt $setup_dir/reports/$exp-RUN-$j/retx.rpt sleep $(($dur * 2)) diff --git a/scripts/sosp24-experiments/flows_exp.sh b/scripts/sosp24-experiments/flows_exp.sh index b03160530..d7f779116 100755 --- a/scripts/sosp24-experiments/flows_exp.sh +++ b/scripts/sosp24-experiments/flows_exp.sh @@ -2,17 +2,45 @@ cd .. echo "Running flow5 experiment... this may take a few minutes" -sudo bash run-dctcp-tput-experiment.sh -E "flow5" -M 4000 --num_servers 5 --num_clients 5 -c "4,8,12,16,20" --ring_buffer 256 --buf 1 --mlc_cores 'none' --bandwidth "100g" --server_intf ens2f1np1 > /dev/null 2>&1 -python3 report-tput-metrics.py flow5 tput,drops,acks,iommu,cpu -echo "Running flow10 experiment... this may take a few minutes" -sudo bash run-dctcp-tput-experiment.sh -E "flow10" -M 4000 --num_servers 10 --num_clients 10 -c "4,8,12,16,20" --ring_buffer 256 --buf 1 --mlc_cores 'none' --bandwidth "100g" --server_intf ens2f1np1 > /dev/null 2>&1 -python3 report-tput-metrics.py flow10 tput,drops,acks,iommu,cpu -echo "Running flow20 experiment... this may take a few minutes" -sudo bash run-dctcp-tput-experiment.sh -E "flow20" -M 4000 --num_servers 20 --num_clients 20 -c "4,8,12,16,20" --ring_buffer 256 --buf 1 --mlc_cores 'none' --bandwidth "100g" --server_intf ens2f1np1 > /dev/null 2>&1 -python3 report-tput-metrics.py flow20 tput,drops,acks,iommu,cpu +iommu_on=$(grep -o intel_iommu=on /proc/cmdline) +iommu_config="" +if [ -z $iommu_on ]; then + iommu_config="iommu-off" +else + iommu_config="iommu-on" +fi -echo "Running flow40 experiment... this may take a few minutes" -sudo bash run-dctcp-tput-experiment.sh -E "flow40" -M 4000 --num_servers 40 --num_clients 40 -c "4,8,12,16,20" --ring_buffer 256 --buf 1 --mlc_cores 'none' --bandwidth "100g" --server_intf ens2f1np1 > /dev/null 2>&1 -python3 report-tput-metrics.py flow40 tput,drops,acks,iommu,cpu +# pause the frame +sudo ethtool --pause ens2f0np0 tx off rx off +ssh benny@genie04.cs.cornell.edu "sudo ethtool --pause ens2f0 tx off rx off" + +sleep 1 + +for i in 5 10 20 40; do + format_i=$(printf "%02d\n" $i) + exp_name="$(uname -r)-flow${format_i}-${iommu_config}-ofed24.10" + echo $exp_name + sudo bash run-dctcp-tput-experiment.sh -E $exp_name -M 4000 --num_servers $i --num_clients $i -c "4,8,12,16,20" --ring_buffer 256 --buf 1 --mlc_cores 'none' --bandwidth "100g" \ + --server_intf ens2f0np0 --client_intf ens2f0 +# > /dev/null 2>&1 + python3 report-tput-metrics.py $exp_name tput,drops,acks,iommu,cpu +done + +# sudo bash run-dctcp-tput-experiment.sh -E "flow5-iommu-on" -M 4000 --num_servers 5 --num_clients 5 -c "4,8,12,16,20" --ring_buffer 256 --buf 1 --mlc_cores 'none' --bandwidth "100g" \ +# --server_intf ens2f0np0 --client_intf ens2f0 +# # > /dev/null 2>&1 +# python3 report-tput-metrics.py flow5 tput,drops,acks,iommu,cpu + +# echo "Running flow10 experiment... this may take a few minutes" +# sudo bash run-dctcp-tput-experiment.sh -E "flow10" -M 4000 --num_servers 10 --num_clients 10 -c "4,8,12,16,20" --ring_buffer 256 --buf 1 --mlc_cores 'none' --bandwidth "100g" --server_intf ens2f1np1 > /dev/null 2>&1 +# python3 report-tput-metrics.py flow10 tput,drops,acks,iommu,cpu + +# echo "Running flow20 experiment... this may take a few minutes" +# sudo bash run-dctcp-tput-experiment.sh -E "flow20" -M 4000 --num_servers 20 --num_clients 20 -c "4,8,12,16,20" --ring_buffer 256 --buf 1 --mlc_cores 'none' --bandwidth "100g" --server_intf ens2f1np1 > /dev/null 2>&1 +# python3 report-tput-metrics.py flow20 tput,drops,acks,iommu,cpu + +# echo "Running flow40 experiment... this may take a few minutes" +# sudo bash run-dctcp-tput-experiment.sh -E "flow40" -M 4000 --num_servers 40 --num_clients 40 -c "4,8,12,16,20" --ring_buffer 256 --buf 1 --mlc_cores 'none' --bandwidth "100g" --server_intf ens2f1np1 > /dev/null 2>&1 +# python3 report-tput-metrics.py flow40 tput,drops,acks,iommu,cpu diff --git a/scripts/sosp24-experiments/ringbuffer_exp.sh b/scripts/sosp24-experiments/ringbuffer_exp.sh index b8229a2b4..fdc9e416a 100755 --- a/scripts/sosp24-experiments/ringbuffer_exp.sh +++ b/scripts/sosp24-experiments/ringbuffer_exp.sh @@ -1,19 +1,38 @@ -./clean_logs.sh +# ./clean_logs.sh cd .. echo "Running ring buffer 2048 experiment... this may take a few minutes" -sudo bash run-dctcp-tput-experiment.sh -E "ring_buffer-2048" -M 4000 --num_servers 5 --num_clients 5 -c "4,8,12,16,20" --ring_buffer 2048 --buf 1 --mlc_cores 'none' --bandwidth "100g" --server_intf ens2f1np1 > /dev/null 2>&1 -python3 report-tput-metrics.py ring_buffer-2048 tput,drops,acks,iommu,cpu +iommu_on=$(grep -o intel_iommu=on /proc/cmdline) +iommu_config="" +if [ -z $iommu_on ]; then + iommu_config="iommu-off" +else + iommu_config="iommu-on" +fi -echo "Running ring buffer 1024 experiment... this may take a few minutes" -sudo bash run-dctcp-tput-experiment.sh -E "ring_buffer-1024" -M 4000 --num_servers 5 --num_clients 5 -c "4,8,12,16,20" --ring_buffer 1024 --buf 1 --mlc_cores 'none' --bandwidth "100g" --server_intf ens2f1np1 > /dev/null 2>&1 -python3 report-tput-metrics.py ring_buffer-1024 tput,drops,acks,iommu,cpu +sudo ethtool --pause ens2f0np0 tx off rx off +ssh benny@genie04.cs.cornell.edu "sudo ethtool --pause ens2f0 tx off rx off" -echo "Running ring buffer 512 experiment... this may take a few minutes" -sudo bash run-dctcp-tput-experiment.sh -E "ring_buffer-512" -M 4000 --num_servers 5 --num_clients 5 -c "4,8,12,16,20" --ring_buffer 512 --buf 1 --mlc_cores 'none' --bandwidth "100g" --server_intf ens2f1np1 > /dev/null 2>&1 -python3 report-tput-metrics.py ring_buffer-512 tput,drops,acks,iommu,cpu +for i in 256 512 1024 2048; do + format_i=$(printf "%04d\n" $i) + exp_name="$(uname -r)-${iommu_config}-ring_buffer-${format_i}" + echo $exp_name + sudo bash run-dctcp-tput-experiment.sh -E $exp_name -M 4000 --num_servers 5 --num_clients 5 -c "4,8,12,16,20" --ring_buffer $i --buf 1 --mlc_cores 'none' --bandwidth "100g"\ + --server_intf ens2f0np0 --client_intf ens2f0 + python3 report-tput-metrics.py $exp_name tput,drops,acks,iommu,cpu +done +# --server_intf ens2f1np1 > /dev/null 2>&1 +# python3 report-tput-metrics.py ring_buffer-2048 tput,drops,acks,iommu,cpu -echo "Running ring buffer 256 experiment... this may take a few minutes" -sudo bash run-dctcp-tput-experiment.sh -E "ring_buffer-256" -M 4000 --num_servers 5 --num_clients 5 -c "4,8,12,16,20" --ring_buffer 256 --buf 1 --mlc_cores 'none' --bandwidth "100g" --server_intf ens2f1np1 > /dev/null 2>&1 -python3 report-tput-metrics.py ring_buffer-256 tput,drops,acks,iommu,cpu +# echo "Running ring buffer 1024 experiment... this may take a few minutes" +# sudo bash run-dctcp-tput-experiment.sh -E "ring_buffer-1024" -M 4000 --num_servers 5 --num_clients 5 -c "4,8,12,16,20" --ring_buffer 1024 --buf 1 --mlc_cores 'none' --bandwidth "100g" --server_intf ens2f1np1 > /dev/null 2>&1 +# python3 report-tput-metrics.py ring_buffer-1024 tput,drops,acks,iommu,cpu + +# echo "Running ring buffer 512 experiment... this may take a few minutes" +# sudo bash run-dctcp-tput-experiment.sh -E "ring_buffer-512" -M 4000 --num_servers 5 --num_clients 5 -c "4,8,12,16,20" --ring_buffer 512 --buf 1 --mlc_cores 'none' --bandwidth "100g" --server_intf ens2f1np1 > /dev/null 2>&1 +# python3 report-tput-metrics.py ring_buffer-512 tput,drops,acks,iommu,cpu + +# echo "Running ring buffer 256 experiment... this may take a few minutes" +# sudo bash run-dctcp-tput-experiment.sh -E "ring_buffer-256" -M 4000 --num_servers 5 --num_clients 5 -c "4,8,12,16,20" --ring_buffer 256 --buf 1 --mlc_cores 'none' --bandwidth "100g" --server_intf ens2f1np1 > /dev/null 2>&1 +# python3 report-tput-metrics.py ring_buffer-256 tput,drops,acks,iommu,cpu From e92c363744a601a2d14cb9b44e32d65d1aadfec8 Mon Sep 17 00:00:00 2001 From: Siyuan Date: Wed, 12 Feb 2025 20:56:21 -0500 Subject: [PATCH 02/12] reboot utils --- grub_linux/grub_linux_6.0.3_off | 80 ++++++++++++++++++++++++++++++++ grub_linux/grub_linux_6.0.3_on | 80 ++++++++++++++++++++++++++++++++ grub_linux/grub_linux_6.12.9_off | 80 ++++++++++++++++++++++++++++++++ grub_linux/grub_linux_6.12.9_on | 80 ++++++++++++++++++++++++++++++++ reboot-6.0.3-iommu-off.sh | 20 ++++++++ reboot-6.0.3-iommu-on.sh | 22 +++++++++ reboot-6.12.9-iommu-off.sh | 20 ++++++++ reboot-6.12.9-iommu-on.sh | 20 ++++++++ 8 files changed, 402 insertions(+) create mode 100644 grub_linux/grub_linux_6.0.3_off create mode 100644 grub_linux/grub_linux_6.0.3_on create mode 100644 grub_linux/grub_linux_6.12.9_off create mode 100644 grub_linux/grub_linux_6.12.9_on create mode 100755 reboot-6.0.3-iommu-off.sh create mode 100755 reboot-6.0.3-iommu-on.sh create mode 100755 reboot-6.12.9-iommu-off.sh create mode 100755 reboot-6.12.9-iommu-on.sh diff --git a/grub_linux/grub_linux_6.0.3_off b/grub_linux/grub_linux_6.0.3_off new file mode 100644 index 000000000..3697eaefe --- /dev/null +++ b/grub_linux/grub_linux_6.0.3_off @@ -0,0 +1,80 @@ +# If you change this file, run 'update-grub' afterwards to update +# /boot/grub/grub.cfg. +# For full documentation of the options in this file, see: +# info -f grub -n 'Simple configuration' + +#GRUB_DEFAULT="1>0" # 5.9.1dizzi +#GRUB_DEFAULT="1>Ubuntu, with Linux 5.6.0-cpu" +#GRUB_DEFAULT="1>Ubuntu, with Linux 5.6.0-netchannel" +#GRUB_DEFAULT="1>Ubuntu, with Linux 5.13.0-51-generic" +#GRUB_DEFAULT="1>Ubuntu, with Linux 5.4.0-124-generic" +#GRUB_DEFAULT="1>Ubuntu, with Linux 5.4.0-137-generic" +#GRUB_DEFAULT="1>Ubuntu, with Linux 6.0.3-qizhe+" +#GRUB_DEFAULT="1>Ubuntu, with Linux 6.0.3sol+" +#GRUB_DEFAULT="1>Ubuntu, with Linux 6.0.3fands+" +#GRUB_DEFAULT="1>Ubuntu, with Linux 6.0.3preserve+" +GRUB_DEFAULT="1>Ubuntu, with Linux 6.0.3+" +#GRUB_DEFAULT="1>Ubuntu, with Linux 6.0.3inv-ack+" +#GRUB_DEFAULT="1>Ubuntu, with Linux 6.0.3no-batch+" +#GRUB_DEFAULT="1>Ubuntu, with Linux 6.0.3iova-log+" +#GRUB_DEFAULT="1>Ubuntu, with Linux 6.0.3-damn+" +#GRUB_DEFAULT="1>Ubuntu, with Linux 4.7.0-damn-ne-p+" +#GRUB_DEFAULT="1>Ubuntu, with Linux 5.1.12-damn-ne+" +#GRUB_DEFAULT="1>Ubuntu, with Linux 5.1.12iovasolution+" +#GRUB_DEFAULT="1>Ubuntu, with Linux 5.1.12+" +#GRUB_DEFAULT="1>Ubuntu, with Linux 6.0.3" +#GRUB_DEFAULT="1>Ubuntu, with Linux 5.13.8" +#GRUB_DEFAULT="1>Ubuntu, with Linux 5.4.0-050400-generic" # main line kernel +#GRUB_DEFAULT="1>Ubuntu, with Linux 5.4.0-48-generic" +#GRUB_DEFAULT="1>Ubuntu, with Linux 5.6.0-rc5-qizhe" +#GRUB_DEFAULT="1>Ubuntu, with Linux 5.4.78-sched" +#GRUB_DEFAULT="1>Ubuntu, with Linux 5.4.43-jaehyun" +#GRUB_DEFAULT="1>Ubuntu, with Linux 5.4.43-abhishek" +#GRUB_DEFAULT="1>6" # 5.4.60-i10 +#GRUB_DEFAULT="1>10" # 5.4.43-jaehyun +#GRUB_DEFAULT="1>Ubuntu, with Linux 5.8.0-48-generic" # generic +#GRUB_DEFAULT="1>Ubuntu, with Linux 4.11.0hyperd" # 4.11 hyperd +#GRUB_DEFAULT="1>Ubuntu, with Linux 4.19.155-mptcp+" # 4.19 mptcp +#GRUB_TIMEOUT_STYLE=hidden +GRUB_TIMEOUT=3 +GRUB_DISTRIBUTOR=`lsb_release -i -s 2> /dev/null || echo Debian` +#GRUB_CMDLINE_LINUX_DEFAULT="cgroup_enable=memory swapaccount=1 intel_iommu=on" +#GRUB_CMDLINE_LINUX_DEFAULT="quiet splash default_hugepagesz=1G hugepagesz=1G hugepages=32" +#GRUB_CMDLINE_LINUX_DEFAULT="quiet splash intel_iommu=on intel_iommu=strict" +#GRUB_CMDLINE_LINUX_DEFAULT="quiet splash intel_iommu=on" +#GRUB_CMDLINE_LINUX_DEFAULT="init_on_alloc=0" +#GRUB_CMDLINE_LINUX="net.ifnames=0 biosdevname=0" + +# Uncomment to enable BadRAM filtering, modify to suit your needs +# This works with Linux (no patch required) and with any kernel that obtains +# the memory map information from GRUB (GNU Mach, kernel of FreeBSD ...) +#GRUB_BADRAM="0x01234567,0xfefefefe,0x89abcdef,0xefefefef" + +# Uncomment to disable graphical terminal (grub-pc only) +#GRUB_TERMINAL=console + +# The resolution used on graphical terminal +# note that you can use only modes which your graphic card supports via VBE +# you can see them in real GRUB with the command `vbeinfo' +#GRUB_GFXMODE=640x480 + +# Uncomment if you don't want GRUB to pass "root=UUID=xxx" parameter to Linux +#GRUB_DISABLE_LINUX_UUID=true + +# Uncomment to disable generation of recovery mode menu entries +#GRUB_DISABLE_RECOVERY="true" + +# Uncomment to get a beep at grub start +#GRUB_INIT_TUNE="480 440 1" + +#GRUB_CMDLINE_LINUX="" + +#GRUB_CMDLINE_LINUX_DEFAULT="quiet splash" + +GRUB_CMDLINE_LINUX="" +#iommu.strict=1 +#GRUB_CMDLINE_LINUX_DEFAULT="intel_iommu=on iommu.strict=1 init_on_free=0 init_on_alloc=0 noresume" +GRUB_CMDLINE_LINUX_DEFAULT="intel_iommu=off init_on_free=0 init_on_alloc=0 noresume" + + +#GRUB_CMDLINE_LINUX_DEFAULT="" diff --git a/grub_linux/grub_linux_6.0.3_on b/grub_linux/grub_linux_6.0.3_on new file mode 100644 index 000000000..34cbd4142 --- /dev/null +++ b/grub_linux/grub_linux_6.0.3_on @@ -0,0 +1,80 @@ +# If you change this file, run 'update-grub' afterwards to update +# /boot/grub/grub.cfg. +# For full documentation of the options in this file, see: +# info -f grub -n 'Simple configuration' + +#GRUB_DEFAULT="1>0" # 5.9.1dizzi +#GRUB_DEFAULT="1>Ubuntu, with Linux 5.6.0-cpu" +#GRUB_DEFAULT="1>Ubuntu, with Linux 5.6.0-netchannel" +#GRUB_DEFAULT="1>Ubuntu, with Linux 5.13.0-51-generic" +#GRUB_DEFAULT="1>Ubuntu, with Linux 5.4.0-124-generic" +#GRUB_DEFAULT="1>Ubuntu, with Linux 5.4.0-137-generic" +#GRUB_DEFAULT="1>Ubuntu, with Linux 6.0.3-qizhe+" +#GRUB_DEFAULT="1>Ubuntu, with Linux 6.0.3sol+" +#GRUB_DEFAULT="1>Ubuntu, with Linux 6.0.3fands+" +#GRUB_DEFAULT="1>Ubuntu, with Linux 6.0.3preserve+" +GRUB_DEFAULT="1>Ubuntu, with Linux 6.0.3+" +#GRUB_DEFAULT="1>Ubuntu, with Linux 6.0.3inv-ack+" +#GRUB_DEFAULT="1>Ubuntu, with Linux 6.0.3no-batch+" +#GRUB_DEFAULT="1>Ubuntu, with Linux 6.0.3iova-log+" +#GRUB_DEFAULT="1>Ubuntu, with Linux 6.0.3-damn+" +#GRUB_DEFAULT="1>Ubuntu, with Linux 4.7.0-damn-ne-p+" +#GRUB_DEFAULT="1>Ubuntu, with Linux 5.1.12-damn-ne+" +#GRUB_DEFAULT="1>Ubuntu, with Linux 5.1.12iovasolution+" +#GRUB_DEFAULT="1>Ubuntu, with Linux 5.1.12+" +#GRUB_DEFAULT="1>Ubuntu, with Linux 6.0.3" +#GRUB_DEFAULT="1>Ubuntu, with Linux 5.13.8" +#GRUB_DEFAULT="1>Ubuntu, with Linux 5.4.0-050400-generic" # main line kernel +#GRUB_DEFAULT="1>Ubuntu, with Linux 5.4.0-48-generic" +#GRUB_DEFAULT="1>Ubuntu, with Linux 5.6.0-rc5-qizhe" +#GRUB_DEFAULT="1>Ubuntu, with Linux 5.4.78-sched" +#GRUB_DEFAULT="1>Ubuntu, with Linux 5.4.43-jaehyun" +#GRUB_DEFAULT="1>Ubuntu, with Linux 5.4.43-abhishek" +#GRUB_DEFAULT="1>6" # 5.4.60-i10 +#GRUB_DEFAULT="1>10" # 5.4.43-jaehyun +#GRUB_DEFAULT="1>Ubuntu, with Linux 5.8.0-48-generic" # generic +#GRUB_DEFAULT="1>Ubuntu, with Linux 4.11.0hyperd" # 4.11 hyperd +#GRUB_DEFAULT="1>Ubuntu, with Linux 4.19.155-mptcp+" # 4.19 mptcp +#GRUB_TIMEOUT_STYLE=hidden +GRUB_TIMEOUT=3 +GRUB_DISTRIBUTOR=`lsb_release -i -s 2> /dev/null || echo Debian` +#GRUB_CMDLINE_LINUX_DEFAULT="cgroup_enable=memory swapaccount=1 intel_iommu=on" +#GRUB_CMDLINE_LINUX_DEFAULT="quiet splash default_hugepagesz=1G hugepagesz=1G hugepages=32" +#GRUB_CMDLINE_LINUX_DEFAULT="quiet splash intel_iommu=on intel_iommu=strict" +#GRUB_CMDLINE_LINUX_DEFAULT="quiet splash intel_iommu=on" +#GRUB_CMDLINE_LINUX_DEFAULT="init_on_alloc=0" +#GRUB_CMDLINE_LINUX="net.ifnames=0 biosdevname=0" + +# Uncomment to enable BadRAM filtering, modify to suit your needs +# This works with Linux (no patch required) and with any kernel that obtains +# the memory map information from GRUB (GNU Mach, kernel of FreeBSD ...) +#GRUB_BADRAM="0x01234567,0xfefefefe,0x89abcdef,0xefefefef" + +# Uncomment to disable graphical terminal (grub-pc only) +#GRUB_TERMINAL=console + +# The resolution used on graphical terminal +# note that you can use only modes which your graphic card supports via VBE +# you can see them in real GRUB with the command `vbeinfo' +#GRUB_GFXMODE=640x480 + +# Uncomment if you don't want GRUB to pass "root=UUID=xxx" parameter to Linux +#GRUB_DISABLE_LINUX_UUID=true + +# Uncomment to disable generation of recovery mode menu entries +#GRUB_DISABLE_RECOVERY="true" + +# Uncomment to get a beep at grub start +#GRUB_INIT_TUNE="480 440 1" + +#GRUB_CMDLINE_LINUX="" + +#GRUB_CMDLINE_LINUX_DEFAULT="quiet splash" + +GRUB_CMDLINE_LINUX="" +#iommu.strict=1 +GRUB_CMDLINE_LINUX_DEFAULT="intel_iommu=on iommu.strict=1 init_on_free=0 init_on_alloc=0 noresume" +#GRUB_CMDLINE_LINUX_DEFAULT="intel_iommu=off init_on_free=0 init_on_alloc=0 noresume" + + +#GRUB_CMDLINE_LINUX_DEFAULT="" diff --git a/grub_linux/grub_linux_6.12.9_off b/grub_linux/grub_linux_6.12.9_off new file mode 100644 index 000000000..6c44870ff --- /dev/null +++ b/grub_linux/grub_linux_6.12.9_off @@ -0,0 +1,80 @@ +# If you change this file, run 'update-grub' afterwards to update +# /boot/grub/grub.cfg. +# For full documentation of the options in this file, see: +# info -f grub -n 'Simple configuration' + +#GRUB_DEFAULT="1>0" # 5.9.1dizzi +#GRUB_DEFAULT="1>Ubuntu, with Linux 5.6.0-cpu" +#GRUB_DEFAULT="1>Ubuntu, with Linux 5.6.0-netchannel" +#GRUB_DEFAULT="1>Ubuntu, with Linux 5.13.0-51-generic" +#GRUB_DEFAULT="1>Ubuntu, with Linux 5.4.0-124-generic" +#GRUB_DEFAULT="1>Ubuntu, with Linux 5.4.0-137-generic" +#GRUB_DEFAULT="1>Ubuntu, with Linux 6.0.3-qizhe+" +#GRUB_DEFAULT="1>Ubuntu, with Linux 6.0.3sol+" +#GRUB_DEFAULT="1>Ubuntu, with Linux 6.0.3fands+" +#GRUB_DEFAULT="1>Ubuntu, with Linux 6.0.3preserve+" +GRUB_DEFAULT="1>Ubuntu, with Linux 6.12.9-vanilla" +#GRUB_DEFAULT="1>Ubuntu, with Linux 6.0.3inv-ack+" +#GRUB_DEFAULT="1>Ubuntu, with Linux 6.0.3no-batch+" +#GRUB_DEFAULT="1>Ubuntu, with Linux 6.0.3iova-log+" +#GRUB_DEFAULT="1>Ubuntu, with Linux 6.0.3-damn+" +#GRUB_DEFAULT="1>Ubuntu, with Linux 4.7.0-damn-ne-p+" +#GRUB_DEFAULT="1>Ubuntu, with Linux 5.1.12-damn-ne+" +#GRUB_DEFAULT="1>Ubuntu, with Linux 5.1.12iovasolution+" +#GRUB_DEFAULT="1>Ubuntu, with Linux 5.1.12+" +#GRUB_DEFAULT="1>Ubuntu, with Linux 6.0.3" +#GRUB_DEFAULT="1>Ubuntu, with Linux 5.13.8" +#GRUB_DEFAULT="1>Ubuntu, with Linux 5.4.0-050400-generic" # main line kernel +#GRUB_DEFAULT="1>Ubuntu, with Linux 5.4.0-48-generic" +#GRUB_DEFAULT="1>Ubuntu, with Linux 5.6.0-rc5-qizhe" +#GRUB_DEFAULT="1>Ubuntu, with Linux 5.4.78-sched" +#GRUB_DEFAULT="1>Ubuntu, with Linux 5.4.43-jaehyun" +#GRUB_DEFAULT="1>Ubuntu, with Linux 5.4.43-abhishek" +#GRUB_DEFAULT="1>6" # 5.4.60-i10 +#GRUB_DEFAULT="1>10" # 5.4.43-jaehyun +#GRUB_DEFAULT="1>Ubuntu, with Linux 5.8.0-48-generic" # generic +#GRUB_DEFAULT="1>Ubuntu, with Linux 4.11.0hyperd" # 4.11 hyperd +#GRUB_DEFAULT="1>Ubuntu, with Linux 4.19.155-mptcp+" # 4.19 mptcp +#GRUB_TIMEOUT_STYLE=hidden +GRUB_TIMEOUT=3 +GRUB_DISTRIBUTOR=`lsb_release -i -s 2> /dev/null || echo Debian` +#GRUB_CMDLINE_LINUX_DEFAULT="cgroup_enable=memory swapaccount=1 intel_iommu=on" +#GRUB_CMDLINE_LINUX_DEFAULT="quiet splash default_hugepagesz=1G hugepagesz=1G hugepages=32" +#GRUB_CMDLINE_LINUX_DEFAULT="quiet splash intel_iommu=on intel_iommu=strict" +#GRUB_CMDLINE_LINUX_DEFAULT="quiet splash intel_iommu=on" +#GRUB_CMDLINE_LINUX_DEFAULT="init_on_alloc=0" +#GRUB_CMDLINE_LINUX="net.ifnames=0 biosdevname=0" + +# Uncomment to enable BadRAM filtering, modify to suit your needs +# This works with Linux (no patch required) and with any kernel that obtains +# the memory map information from GRUB (GNU Mach, kernel of FreeBSD ...) +#GRUB_BADRAM="0x01234567,0xfefefefe,0x89abcdef,0xefefefef" + +# Uncomment to disable graphical terminal (grub-pc only) +#GRUB_TERMINAL=console + +# The resolution used on graphical terminal +# note that you can use only modes which your graphic card supports via VBE +# you can see them in real GRUB with the command `vbeinfo' +#GRUB_GFXMODE=640x480 + +# Uncomment if you don't want GRUB to pass "root=UUID=xxx" parameter to Linux +#GRUB_DISABLE_LINUX_UUID=true + +# Uncomment to disable generation of recovery mode menu entries +#GRUB_DISABLE_RECOVERY="true" + +# Uncomment to get a beep at grub start +#GRUB_INIT_TUNE="480 440 1" + +#GRUB_CMDLINE_LINUX="" + +#GRUB_CMDLINE_LINUX_DEFAULT="quiet splash" + +GRUB_CMDLINE_LINUX="" +#iommu.strict=1 +#GRUB_CMDLINE_LINUX_DEFAULT="intel_iommu=on iommu.strict=1 init_on_free=0 init_on_alloc=0 noresume" +GRUB_CMDLINE_LINUX_DEFAULT="intel_iommu=off init_on_free=0 init_on_alloc=0 noresume" + + +#GRUB_CMDLINE_LINUX_DEFAULT="" diff --git a/grub_linux/grub_linux_6.12.9_on b/grub_linux/grub_linux_6.12.9_on new file mode 100644 index 000000000..dc91f166d --- /dev/null +++ b/grub_linux/grub_linux_6.12.9_on @@ -0,0 +1,80 @@ +# If you change this file, run 'update-grub' afterwards to update +# /boot/grub/grub.cfg. +# For full documentation of the options in this file, see: +# info -f grub -n 'Simple configuration' + +#GRUB_DEFAULT="1>0" # 5.9.1dizzi +#GRUB_DEFAULT="1>Ubuntu, with Linux 5.6.0-cpu" +#GRUB_DEFAULT="1>Ubuntu, with Linux 5.6.0-netchannel" +#GRUB_DEFAULT="1>Ubuntu, with Linux 5.13.0-51-generic" +#GRUB_DEFAULT="1>Ubuntu, with Linux 5.4.0-124-generic" +#GRUB_DEFAULT="1>Ubuntu, with Linux 5.4.0-137-generic" +#GRUB_DEFAULT="1>Ubuntu, with Linux 6.0.3-qizhe+" +#GRUB_DEFAULT="1>Ubuntu, with Linux 6.0.3sol+" +#GRUB_DEFAULT="1>Ubuntu, with Linux 6.0.3fands+" +#GRUB_DEFAULT="1>Ubuntu, with Linux 6.0.3preserve+" +GRUB_DEFAULT="1>Ubuntu, with Linux 6.12.9-vanilla" +#GRUB_DEFAULT="1>Ubuntu, with Linux 6.0.3inv-ack+" +#GRUB_DEFAULT="1>Ubuntu, with Linux 6.0.3no-batch+" +#GRUB_DEFAULT="1>Ubuntu, with Linux 6.0.3iova-log+" +#GRUB_DEFAULT="1>Ubuntu, with Linux 6.0.3-damn+" +#GRUB_DEFAULT="1>Ubuntu, with Linux 4.7.0-damn-ne-p+" +#GRUB_DEFAULT="1>Ubuntu, with Linux 5.1.12-damn-ne+" +#GRUB_DEFAULT="1>Ubuntu, with Linux 5.1.12iovasolution+" +#GRUB_DEFAULT="1>Ubuntu, with Linux 5.1.12+" +#GRUB_DEFAULT="1>Ubuntu, with Linux 6.0.3" +#GRUB_DEFAULT="1>Ubuntu, with Linux 5.13.8" +#GRUB_DEFAULT="1>Ubuntu, with Linux 5.4.0-050400-generic" # main line kernel +#GRUB_DEFAULT="1>Ubuntu, with Linux 5.4.0-48-generic" +#GRUB_DEFAULT="1>Ubuntu, with Linux 5.6.0-rc5-qizhe" +#GRUB_DEFAULT="1>Ubuntu, with Linux 5.4.78-sched" +#GRUB_DEFAULT="1>Ubuntu, with Linux 5.4.43-jaehyun" +#GRUB_DEFAULT="1>Ubuntu, with Linux 5.4.43-abhishek" +#GRUB_DEFAULT="1>6" # 5.4.60-i10 +#GRUB_DEFAULT="1>10" # 5.4.43-jaehyun +#GRUB_DEFAULT="1>Ubuntu, with Linux 5.8.0-48-generic" # generic +#GRUB_DEFAULT="1>Ubuntu, with Linux 4.11.0hyperd" # 4.11 hyperd +#GRUB_DEFAULT="1>Ubuntu, with Linux 4.19.155-mptcp+" # 4.19 mptcp +#GRUB_TIMEOUT_STYLE=hidden +GRUB_TIMEOUT=3 +GRUB_DISTRIBUTOR=`lsb_release -i -s 2> /dev/null || echo Debian` +#GRUB_CMDLINE_LINUX_DEFAULT="cgroup_enable=memory swapaccount=1 intel_iommu=on" +#GRUB_CMDLINE_LINUX_DEFAULT="quiet splash default_hugepagesz=1G hugepagesz=1G hugepages=32" +#GRUB_CMDLINE_LINUX_DEFAULT="quiet splash intel_iommu=on intel_iommu=strict" +#GRUB_CMDLINE_LINUX_DEFAULT="quiet splash intel_iommu=on" +#GRUB_CMDLINE_LINUX_DEFAULT="init_on_alloc=0" +#GRUB_CMDLINE_LINUX="net.ifnames=0 biosdevname=0" + +# Uncomment to enable BadRAM filtering, modify to suit your needs +# This works with Linux (no patch required) and with any kernel that obtains +# the memory map information from GRUB (GNU Mach, kernel of FreeBSD ...) +#GRUB_BADRAM="0x01234567,0xfefefefe,0x89abcdef,0xefefefef" + +# Uncomment to disable graphical terminal (grub-pc only) +#GRUB_TERMINAL=console + +# The resolution used on graphical terminal +# note that you can use only modes which your graphic card supports via VBE +# you can see them in real GRUB with the command `vbeinfo' +#GRUB_GFXMODE=640x480 + +# Uncomment if you don't want GRUB to pass "root=UUID=xxx" parameter to Linux +#GRUB_DISABLE_LINUX_UUID=true + +# Uncomment to disable generation of recovery mode menu entries +#GRUB_DISABLE_RECOVERY="true" + +# Uncomment to get a beep at grub start +#GRUB_INIT_TUNE="480 440 1" + +#GRUB_CMDLINE_LINUX="" + +#GRUB_CMDLINE_LINUX_DEFAULT="quiet splash" + +GRUB_CMDLINE_LINUX="" +#iommu.strict=1 +GRUB_CMDLINE_LINUX_DEFAULT="intel_iommu=on iommu.strict=1 init_on_free=0 init_on_alloc=0 noresume" +#GRUB_CMDLINE_LINUX_DEFAULT="intel_iommu=off init_on_free=0 init_on_alloc=0 noresume" + + +#GRUB_CMDLINE_LINUX_DEFAULT="" diff --git a/reboot-6.0.3-iommu-off.sh b/reboot-6.0.3-iommu-off.sh new file mode 100755 index 000000000..4f7e05992 --- /dev/null +++ b/reboot-6.0.3-iommu-off.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +kernel="6.0.3" +iommu="off" + +grub_src="grub_linux/grub_linux_${kernel}_${iommu}" +if [ -f $grub_src ]; then +# echo "File does not exist." + sudo cp $grub_src /etc/default/grub +else + echo "$grub_src doesn't exist" + exit 1 +fi + + +sudo update-grub2 + +echo "Reboot to ${kernel} IOMMU=${iommu} in 10 seconds." + +sudo reboot \ No newline at end of file diff --git a/reboot-6.0.3-iommu-on.sh b/reboot-6.0.3-iommu-on.sh new file mode 100755 index 000000000..391d09074 --- /dev/null +++ b/reboot-6.0.3-iommu-on.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +kernel="6.0.3" +iommu="on" + +grub_src="grub_linux/grub_linux_${kernel}_${iommu}" + +if [ -f $grub_src ]; then +# echo "File does not exist." + sudo cp $grub_src /etc/default/grub +else + echo "$grub_src doesn't exist" + exit 1 +fi + + +sudo update-grub2 + + +echo "Reboot to ${kernel} IOMMU=${iommu} in 10 seconds." + +sudo reboot \ No newline at end of file diff --git a/reboot-6.12.9-iommu-off.sh b/reboot-6.12.9-iommu-off.sh new file mode 100755 index 000000000..63407add7 --- /dev/null +++ b/reboot-6.12.9-iommu-off.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +kernel="6.12.9" +iommu="off" + +grub_src="grub_linux/grub_linux_${kernel}_${iommu}" +if [ -f $grub_src ]; then +# echo "File does not exist." + sudo cp $grub_src /etc/default/grub +else + echo "$grub_src doesn't exist" + exit 1 +fi + + +sudo update-grub2 + +echo "Reboot to ${kernel} IOMMU=${iommu} in 10 seconds." + +sudo reboot \ No newline at end of file diff --git a/reboot-6.12.9-iommu-on.sh b/reboot-6.12.9-iommu-on.sh new file mode 100755 index 000000000..c15c3d557 --- /dev/null +++ b/reboot-6.12.9-iommu-on.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +kernel="6.12.9" +iommu="on" + +grub_src="grub_linux/grub_linux_${kernel}_${iommu}" +if [ -f $grub_src ]; then +# echo "File does not exist." + sudo cp $grub_src /etc/default/grub +else + echo "$grub_src doesn't exist" + exit 1 +fi + + +sudo update-grub2 + +echo "Reboot to ${kernel} IOMMU=${iommu} in 10 seconds." + +sudo reboot \ No newline at end of file From afc5568da29a58dc7fcb2f39359c263b9930c4de Mon Sep 17 00:00:00 2001 From: Siyuan Date: Wed, 12 Feb 2025 20:57:10 -0500 Subject: [PATCH 03/12] plot script --- scripts/sosp24-experiments/plot.py | 252 +++++++++++++++++++++++++++++ 1 file changed, 252 insertions(+) create mode 100644 scripts/sosp24-experiments/plot.py diff --git a/scripts/sosp24-experiments/plot.py b/scripts/sosp24-experiments/plot.py new file mode 100644 index 000000000..0b64f4e72 --- /dev/null +++ b/scripts/sosp24-experiments/plot.py @@ -0,0 +1,252 @@ + +import matplotlib.pyplot as plt +import numpy as np + +def get_tput(filename): + with open(filename, "r") as file: + lines = file.readlines() + + # Extract the third number from the second line (index 2) + if len(lines) > 1: # Ensure there are at least two lines + tput = lines[1].strip().split(",")[2] + # print(tput) + return float(tput) + else: + print("File does not contain enough lines.") + +# get_tput("/home/benny/Fast-and-Safe-IO-Memory-Protection-Siyuan/utils/reports/6.12.9-vanilla-flow05-iommu-off/tput_metrics.dat") + +def parse_results(path): + results = np.genfromtxt(path, dtype=float, delimiter=',', names=True) + return results + +def get_data(prefix, iommu_str, suffix=""): + + flows = ["05", "10", "20", "40"] + folders = [ + prefix + f"flow{f}-" + iommu_str + suffix for f in flows + ] + + files = [ + "../../utils/reports/" + f + "/tput_metrics.dat" for f in folders + ] + + data = [ + parse_results(f) for f in files + ] + + return data + +def get_data_ring(prefix, iommu_str, suffix=""): + + x_labels = ["0256", "0512", "1024", "2048"] + folders = [ + prefix + iommu_str + "-ring_buffer-"+ x for x in x_labels + ] + + files = [ + "../../utils/reports/" + f + "/tput_metrics.dat" for f in folders + ] + + data = [ + parse_results(f) for f in files + ] + + return data + + +def misses_per_page(misses, tput_mean): + # a bit of a round-a-bout way from when I used per desc, but it works so not touching it! + mbs_per_second = tput_mean * 125 + descriptors_per_second = mbs_per_second * 4 + misses_per_page = misses / descriptors_per_second + # GETTING MISSES PER PAGE + misses_per_page = misses_per_page / 64 + return misses_per_page + +def get_misses_per_page(data): + # tput = data['net_tput_mean'] + acks_page = [] + iotlb_miss_page = [] + l1_miss_page = [] + l2_miss_page = [] + l3_miss_page = [] + + # acks_page = misses_per_page(sent_packets, tput) + for idx in range(len(data)): + tp = data[idx]['net_tput_mean'] + + iotlb_miss_page.append(misses_per_page(data[idx]['iotlb_misses_mean'], tp)) + l1_miss_page.append(misses_per_page(data[idx]['l1_misses_mean'], tp)) + l2_miss_page.append(misses_per_page(data[idx]['l2_misses_mean'], tp)) + l3_miss_page.append(misses_per_page(data[idx]['l3_misses_mean'], tp)) + acks_page.append(misses_per_page(data[idx]['sent_packets_mean']/20, tp)) + + return iotlb_miss_page, l1_miss_page, l2_miss_page, l3_miss_page, acks_page + +def plot_tput(iommu_off_data, iommu_on_data, x_labels, title): + bar_width = 0.35 + # plt.plot(iommu_off_data, iommu_on_data) + x = np.arange(len(x_labels)) + plt.bar(x - bar_width/2, iommu_off_data, bar_width, label='IOMMU off') + plt.bar(x + bar_width/2, iommu_on_data, bar_width, label='IOMMU on') + + plt.xlabel("# of flows") + + plt.ylabel("Throughput (Gbps)") + plt.title(title) + plt.xticks(x, x_labels) + plt.legend() + + plt.savefig(title + '.png') + print('Saved plot to ' + title + '.png') + plt.close() + +def plot_drop_rate(iommu_off_data, iommu_on_data, x_labels, title): + bar_width = 0.35 + # plt.plot(iommu_off_data, iommu_on_data) + x = np.arange(len(x_labels)) + plt.bar(x - bar_width/2, iommu_off_data, bar_width, label='IOMMU off') + plt.bar(x + bar_width/2, iommu_on_data, bar_width, label='IOMMU on') + + plt.xlabel("# of flows") + + plt.ylabel("Drop rate") + plt.title(title) + plt.xticks(x, x_labels) + plt.legend() + + plt.savefig(title + '.png') + print('Saved plot to ' + title + '.png') + plt.close() + +def plot_iommu_misses_stats(iommu_on_data, x_labels, title): + iotlb_miss_page, l1_miss_page, l2_miss_page, l3_miss_page, acks_page = get_misses_per_page(iommu_on_data) + + bar_width = 0.35 + # plt.plot(iommu_off_data, iommu_on_data) + x = np.arange(len(x_labels)) + plt.bar(x, iotlb_miss_page, bar_width, label='IOMMU TLB misses') + # plt.bar(x + bar_width/2, iommu_on_data, bar_width, label='IOMMU on') + + plt.xlabel("# of flows") + + plt.ylabel("IOTLB misses per page") + plt.title(title + 'IOTLB-miss') + plt.xticks(x, x_labels) + plt.legend() + plt.ylim(0, 3) + + + file_name = title + 'IOTLB-miss.png' + plt.savefig(file_name) + print('Saved plot to ' + file_name) + plt.close() + + + # plot L1, L2, L3 misses + + x = np.arange(len(x_labels)) + plt.bar(x, acks_page, bar_width, label='ACKs per page') + # plt.bar(x + bar_width/2, iommu_on_data, bar_width, label='IOMMU on') + + plt.xlabel("# of flows") + + plt.ylabel("Acks per page") + plt.title(title + 'IOTLB-miss') + plt.xticks(x, x_labels) + plt.legend() + plt.ylim(0, 0.15) + + + file_name = title + 'Acks.png' + plt.savefig(file_name) + print('Saved plot to ' + file_name) + plt.close() + + + # plot L1, L2, L3 misses + bar_width = 0.25 + + plt.bar(x - bar_width, l1_miss_page, bar_width, label='L1') + plt.bar(x, l2_miss_page, bar_width, label='L2') + plt.bar(x + bar_width, l3_miss_page, bar_width, label='L3') + + plt.xlabel("# of flows") + + plt.ylabel("Misses per page") + plt.title(title + 'L1-L2-L3-miss') + plt.xticks(x, x_labels) + plt.legend() + plt.ylim(0, 1.0) + + file_name = title + 'L1-L2-L3-miss.png' + plt.savefig(file_name) + print('Saved plot to ' + file_name) + plt.close() + +def plot_all_subplots(iommu_off_all_data, iommu_on_all_data, x_labels, title_key): + plot_tput( + iommu_off_data = [ r['net_tput_mean'] for r in iommu_off_all_data ], + iommu_on_data = [ r['net_tput_mean'] for r in iommu_on_all_data ], + x_labels = x_labels, + title = title_key + '-tput' + ) + + plot_drop_rate( + iommu_off_data = [ r['retx_rate_mean'] for r in iommu_off_all_data ], + iommu_on_data = [ r['retx_rate_mean'] for r in iommu_on_all_data ], + x_labels = x_labels, + title = title_key + 'drop-rate' + ) + + plot_iommu_misses_stats( + iommu_on_data = iommu_on_all_data, + x_labels = x_labels, + title = title_key + '-misses' + ) + + +def plot_tput_f_and_s(): + x_labels = ["05", "10", "20", "40"] + iommu_off_all_data = get_data(prefix="", iommu_str="iommu-off") + iommu_on_all_data = get_data(prefix="", iommu_str="iommu-on") + + plot_all_subplots(iommu_off_all_data, iommu_on_all_data, x_labels, 'Fast-safe-setup') + + +def plot_tput_new_kernel(): + x_labels = ["05", "10", "20", "40"] + iommu_off_all_data = get_data(prefix="6.12.9-vanilla-", iommu_str="iommu-off") + iommu_on_all_data = get_data(prefix="6.12.9-vanilla-", iommu_str="iommu-on") + + plot_all_subplots(iommu_off_all_data, iommu_on_all_data, x_labels, '6.12.9') + +def plot_tput_new_ofed(): + x_labels = ["05", "10", "20", "40"] + iommu_off_all_data = get_data(prefix="6.12.9-vanilla-", iommu_str="iommu-off", suffix="-ofed24.10") + iommu_on_all_data = get_data(prefix="6.12.9-vanilla-", iommu_str="iommu-on", suffix="-ofed24.10-paused") + + plot_all_subplots(iommu_off_all_data, iommu_on_all_data, x_labels, '6.12.9-ofed24.10') + +def plot_tput_new_ofed2(): + x_labels = ["05", "10", "20", "40"] + iommu_off_all_data = get_data(prefix="6.12.9-vanilla-", iommu_str="iommu-off", suffix="-ofed24.10") + iommu_on_all_data = get_data(prefix="6.12.9-vanilla-", iommu_str="iommu-on", suffix="-ofed24.10-paused3") + + plot_all_subplots(iommu_off_all_data, iommu_on_all_data, x_labels, '6.12.9-ofed24.10-paused3') + +def plot_ring_buf_exp(): + x_labels = ["256", "512", "1024", "2048"] + iommu_off_all_data = get_data_ring(prefix="6.12.9-vanilla-", iommu_str="iommu-off") + iommu_on_all_data = get_data_ring(prefix="6.12.9-vanilla-", iommu_str="iommu-on") + + plot_all_subplots(iommu_off_all_data, iommu_on_all_data, x_labels, '6.12.9-ring-buffer') + +# plot_tput_f_and_s() +# plot_tput_new_kernel() +# plot_tput_new_ofed() + +# plot_tput_new_ofed2() + +plot_ring_buf_exp() \ No newline at end of file From 352b5a8acbc4d08d4745db0f66647beb47e02562 Mon Sep 17 00:00:00 2001 From: Siyuan Date: Sat, 15 Feb 2025 14:45:40 -0500 Subject: [PATCH 04/12] prep --- prep.sh | 81 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100755 prep.sh diff --git a/prep.sh b/prep.sh new file mode 100755 index 000000000..feadac5af --- /dev/null +++ b/prep.sh @@ -0,0 +1,81 @@ +#!/bin/bash + + + +home="$(realpath ~)" +install_pcm () { + cd $home + git clone --recursive https://github.com/intel/pcm + cd pcm + mkdir build + cd build + cmake .. + cmake --build . + + cmake --build . --parallel + + ls bin + # sudo apt install libasan11 +} + +install_mlc () { + cd $home + wget https://downloadmirror.intel.com/834254/mlc_v3.11b.tgz + mkdir -p mlc + mv mlc_v3.11b.tgz mlc + cd mlc + tar -xvf mlc_v3.11b.tgz + + ls Linux +} + +install_rdt () { + cd $home + git clone https://github.com/intel/intel-cmt-cat.git + cd intel-cmt-cat/ + make + sudo make install + + sudo modprobe msr +} + +install_iperf3 () { + cd $home + git clone https://github.com/esnet/iperf.git + cd iperf + git checkout 3.18 + ./bootstrap.sh + ./configure; make; sudo make install + sudo ldconfig +} + +install_netperf () { + cd $home + git clone https://github.com/HewlettPackard/netperf.git + cd netperf + cp $home/Fast-and-Safe-IO-Memory-Protection/utils/tcp/netperf-logging.diff . + git apply netperf-logging.diff + + ./autogen.sh + ./configure + sudo apt install texinfo + + make CFLAGS=-fcommon + sudo make install +} + + + +# cd $home +# git clone https://github.com/Terabit-Ethernet/Understanding-network-stack-overheads-SIGCOMM-2021 +# git clone https://github.com/aliireza/ddio-bench.git + +sudo utils/setup-envir.sh --home $home \ + --intf enp202s0f0np0 \ + --addr 10.10.1.2 + +# install_pcm +# install_mlc +# install_rdt +# install_iperf3 +# install_netperf \ No newline at end of file From 0e5a2f4665fb478881636d30e38ca73770b512fa Mon Sep 17 00:00:00 2001 From: Siyuan Date: Mon, 12 May 2025 15:20:49 -0400 Subject: [PATCH 05/12] count script --- .../sosp24-experiments/count_invalidation.py | 127 ++++++++++++++++++ scripts/sosp24-experiments/flows_exp.sh | 55 ++++++-- 2 files changed, 172 insertions(+), 10 deletions(-) create mode 100644 scripts/sosp24-experiments/count_invalidation.py diff --git a/scripts/sosp24-experiments/count_invalidation.py b/scripts/sosp24-experiments/count_invalidation.py new file mode 100644 index 000000000..703b9d90d --- /dev/null +++ b/scripts/sosp24-experiments/count_invalidation.py @@ -0,0 +1,127 @@ + + + +def count_invalidation(file: str) -> dict: + """ + iperf3-14236 [004] b..2. 484.455404: intel_iommu_iotlb_sync_map: intel_iommu_iotlb_sync_map: core: 4, iova: cc990000 size: 0x1000 + iperf3-14236 [004] b..2. 484.455407: mlx5e_xmit: TX map: mlx5e_dma_push_with_log: core: 4, size: 66, addr: cc9904fe + iperf3-14236 [004] ..s1. 484.455411: mlx5e_tx_wi_dma_unmap.isra.0: TX unmap: mlx5e_tx_dma_unmap: core: 4, size: 66, addr: cc9904fe + iperf3-14236 [004] ..s1. 484.455413: mlx5e_page_dma_unmap: RX unmap: mlx5e_page_dma_unmap: core: 4, iova=f5dff000 pfn=400138 + iperf3-14236 [004] ..s1. 484.455415: mlx5e_page_dma_unmap: RX unmap: mlx5e_page_dma_unmap: core: 4, iova=f2607000 pfn=3e765d + iperf3-14236 [004] ..s1. 484.455416: mlx5e_page_dma_unmap: RX unmap: mlx5e_page_dma_unmap: core: 4, iova=ece5a000 pfn=3f4915 + iperf3-14236 [004] ..s1. 484.455417: mlx5e_page_dma_unmap: RX unmap: mlx5e_page_dma_unmap: core: 4, iova=ecb0f000 pfn=3fa2a3 + """ + + count_dict = { + "total": 0, + + "RX map": 0, + "RX unmap": 0, + "TX map": 0, + "TX unmap": 0, + + "page_pool_put_unrefed_page": 0, + "recycle_in_cache": 0, + + "start_time": -1.0, + "end_time": -1.0, + + "total_per_sec" : 0.0, + "RX map_per_sec": 0.0, + "RX unmap_per_sec": 0.0, + "TX map_per_sec": 0.0, + "TX unmap_per_sec": 0.0, + } + + with open(file, "r") as f: + for line in f: + line = line.strip() + operation_happens = False + if line: + # Check if the line contains "RX map" or "TX map" + if "RX map" in line: + count_dict["RX map"] += 1 + operation_happens = True + elif "TX map" in line: + count_dict["TX map"] += 1 + operation_happens = True + elif "RX unmap" in line: + count_dict["RX unmap"] += 1 + operation_happens = True + elif "TX unmap" in line: + count_dict["TX unmap"] += 1 + operation_happens = True + elif "page_pool_put_unrefed_page" in line: + count_dict["page_pool_put_unrefed_page"] += 1 + operation_happens = True + elif "page_pool_recycle_in_cache" in line: + count_dict["recycle_in_cache"] += 1 + operation_happens = True + + + if operation_happens: + # Extract the time from the line + time_str = line.split()[3].strip()[:-1] + time_val = float(time_str) + + # Update start and end times + if count_dict["start_time"] < 0.0 or time_val < count_dict["start_time"]: + count_dict["start_time"] = time_val + if time_val > count_dict["end_time"]: + count_dict["end_time"] = time_val + + + + count_dict["total"] = count_dict["RX map"] + count_dict["TX map"] + count_dict["RX unmap"] + count_dict["TX unmap"] + + # Calculate per second rate + duration_sec = count_dict["end_time"] - count_dict["start_time"] + if duration_sec > 0: + count_dict["total_per_sec"] = count_dict["total"] / duration_sec + count_dict["RX map_per_sec"] = count_dict["RX map"] / duration_sec + count_dict["RX unmap_per_sec"] = count_dict["RX unmap"] / duration_sec + count_dict["TX map_per_sec"] = count_dict["TX map"] / duration_sec + count_dict["TX unmap_per_sec"] = count_dict["TX unmap"] / duration_sec + + return count_dict + + +def save_to_csv(stat_dict: dict, file: str): + """ + Save the statistics to a CSV file. + """ + import csv + + with open(file, "w", newline="") as csvfile: + fieldnames = stat_dict.keys() + writer = csv.DictWriter(csvfile, fieldnames=fieldnames) + + writer.writeheader() + writer.writerow(stat_dict) + +if __name__ == "__main__": + import argparse + import os + import sys + import time + + parser = argparse.ArgumentParser( + description="Count the number of invalidations in a given directory." + ) + parser.add_argument( + "--dir", + type=str, + default=".", + help="Directory to count invalidations in (default: current directory)", + ) + + args = parser.parse_args() + + stat_dict = count_invalidation(os.path.join(args.dir, "iova.log")) + + print(stat_dict) + + count_path = os.path.join(args.dir, "invalidation_count.csv") + + save_to_csv(stat_dict, count_path) + print(f"Saved to {count_path}") \ No newline at end of file diff --git a/scripts/sosp24-experiments/flows_exp.sh b/scripts/sosp24-experiments/flows_exp.sh index d7f779116..9620ba683 100755 --- a/scripts/sosp24-experiments/flows_exp.sh +++ b/scripts/sosp24-experiments/flows_exp.sh @@ -1,6 +1,12 @@ -./clean_logs.sh +#!/bin/bash + +# ./clean_logs.sh +# start_dir=$(pwd) + +# ./clean_logs.sh cd .. +working_dir=$(pwd) echo "Running flow5 experiment... this may take a few minutes" @@ -12,20 +18,49 @@ else iommu_config="iommu-on" fi +ofed_version=$(ofed_info -n) + # pause the frame sudo ethtool --pause ens2f0np0 tx off rx off ssh benny@genie04.cs.cornell.edu "sudo ethtool --pause ens2f0 tx off rx off" -sleep 1 +sleep 10 + +warmup_time=10 + +# 5 10 20 40 +for i in 5 40; do + for j in $(seq 1 1 1) # start from 1, increment by 2 until 10 + do + cd $working_dir + cur_time=$(date +"%Y-%m-%d-%H-%M-%S") + format_i=$(printf "%02d\n" $i) + # exp_name="$(uname -r)-flow${format_i}-${iommu_config}-ofed$(ofed_version)-test2-siyuan" + # exp_name="$(uname -r)-${iommu_config}-flow-${format_i}-core4-warmup${warmup_time}-leshna" + exp_name="$(uname -r)-${iommu_config}-flow-${format_i}-core4-warmup${warmup_time}-siyuan-test" + echo $exp_name + exp_name="${exp_name}-${cur_time}" + sudo bash run-dctcp-tput-experiment.sh -E $exp_name -M 4000 --num_servers $i --num_clients $i -c "4,8,12,16,20" --ring_buffer 256 --buf 1 --mlc_cores 'none' --bandwidth "100g" \ + --server_intf ens2f0np0 --client_intf ens2f0 --warmup $warmup_time + # > /dev/null 2>&1 + python3 report-tput-metrics.py $exp_name tput,drops,acks,iommu,cpu + cd ../utils/reports/$exp_name + + sudo bash -c "cat /sys/kernel/debug/tracing/trace > iova.log" + sudo bash -c "rg iperf3 iova.log > iperf_iova.log" + sudo bash -c "rg 'core: 16' iova.log > iperf_iova_core16.log" + # sudo bash -c "rg core iova.log > core_iova.log" + + cd $working_dir + sudo chmod +666 -R ../utils/reports/$exp_name + + python sosp24-experiments/plot_iova_logging.py \ + --exp_folder "../utils/reports/$exp_name" \ + --log_file "iova.log" -for i in 5 10 20 40; do - format_i=$(printf "%02d\n" $i) - exp_name="$(uname -r)-flow${format_i}-${iommu_config}-ofed24.10" - echo $exp_name - sudo bash run-dctcp-tput-experiment.sh -E $exp_name -M 4000 --num_servers $i --num_clients $i -c "4,8,12,16,20" --ring_buffer 256 --buf 1 --mlc_cores 'none' --bandwidth "100g" \ - --server_intf ens2f0np0 --client_intf ens2f0 -# > /dev/null 2>&1 - python3 report-tput-metrics.py $exp_name tput,drops,acks,iommu,cpu + python3 sosp24-experiments/count_invalidation.py --dir "../utils/reports/$exp_name" + done + done # sudo bash run-dctcp-tput-experiment.sh -E "flow5-iommu-on" -M 4000 --num_servers 5 --num_clients 5 -c "4,8,12,16,20" --ring_buffer 256 --buf 1 --mlc_cores 'none' --bandwidth "100g" \ From 0eee2fd0ee134d50adedcf84ed96237a653c7a8c Mon Sep 17 00:00:00 2001 From: Leshna Date: Mon, 23 Jun 2025 21:11:50 -0600 Subject: [PATCH 06/12] simplified setup script for complete passthrough --- utils/setup-host.sh | 98 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 98 insertions(+) create mode 100644 utils/setup-host.sh diff --git a/utils/setup-host.sh b/utils/setup-host.sh new file mode 100644 index 000000000..858d63033 --- /dev/null +++ b/utils/setup-host.sh @@ -0,0 +1,98 @@ +SCRIPT_NAME="setup-envir" + +#default values +MTU=4000 +INTF="virbr0" +IP="192.168.122.1" +TCP_SOCKET_BUF_MB=1 +ECN_ENABLED=1 +HWPREF_ENABLED=1 +RDMA=0 +RING_BUFFER_SIZE=1024 + +help() +{ + echo "Usage: $SCRIPT_NAME + [ --intf (interface name, eg. ens2f0) ] + [ --ip (ip address for the interface) ] + [ -m | --mtu (MTU size in bytes; default=4000 for TCP, 4096 for RDMA) ] + [ -r | --ring-buffer (size of Rx ring buffer. Note: opt must be set to change this)] + [ --socket-buf (TCP socket buffer size (in MB)) ] + [ --ecn (Enable ECN support in Linux stack) ] + [ --hwpref (Enable hardware prefetching) ] + [ --rdma (=0/1, whether the setup is for running RDMA experiments (MTU offset will be different)) ] + [ -h | --help ]" + exit 2 +} + +SHORT=m:,r:,h +LONG=mtu:,ring-buffer:,intf:,ip:,socket-buf:,ecn:,hwpref:,rdma:,help +PARSED_OPTS=$(getopt -a -n $SCRIPT_NAME --options $SHORT --longoptions $LONG -- "$@") + +VALID_ARGUMENTS=$# +if [ "$VALID_ARGUMENTS" -eq 0 ]; then + help +fi +eval set -- "$PARSED_OPTS" + +while :;do + case "$1" in + -m | --mtu) MTU="$2"; shift 2 ;; + -r | --ring-buffer) RING_BUFFER_SIZE="$2"; shift 2 ;; + --intf) INTF="$2"; shift 2 ;; + --ip) IP="$2"; shift 2 ;; + --socket-buf) TCP_SOCKET_BUF_MB="$2"; shift 2 ;; + --ecn) ECN_ENABLED="$2"; shift 2 ;; + --hwpref) HWPREF_ENABLED="$2"; shift 2 ;; + --rdma) RDMA="$2"; shift 2 ;; + -h | --help) help ;; + --) shift; break ;; + *) echo "Unexpected option: $1"; help ;; + esac +done + +log_info() { + echo "[INFO] - $1" +} + +if [ "$RDMA" -eq 1 ]; then + log_info "Configuring MTU according to RDMA supported values..." + MTU=$(($MTU + 96)) + if [ "$MTU" -lt 1280 ]; then + log_info "Requested physical MTU size is $MTU, updating to 1280" + MTU=1280 + fi + log_info "MTU configured to $MTU" +fi + +# setup the interface +log_info "Setting up the interface $INTF..." +ifconfig $INTF up +ifconfig $INTF $IP +ifconfig $INTF mtu $MTU + +#disable TCP buffer auto-tuning, and set the buffer size to the specified size +log_info "Setting up the socket buffer size to be ${TCP_SOCKET_BUF_MB}MB" +echo 0 > /proc/sys/net/ipv4/tcp_moderate_rcvbuf +#Set TCP receive buffer size to be 1MB (other 1MB is for the application buffer) +echo "$(($TCP_SOCKET_BUF_MB * 2000000)) $(($TCP_SOCKET_BUF_MB * 2000000)) $(($TCP_SOCKET_BUF_MB * 2000000))" > /proc/sys/net/ipv4/tcp_rmem +#Set TCP send buffer size to be 1MB +echo "$(($TCP_SOCKET_BUF_MB * 1000000)) $(($TCP_SOCKET_BUF_MB * 1000000)) $(($TCP_SOCKET_BUF_MB * 1000000))" > /proc/sys/net/ipv4/tcp_wmem + +#Enable TCP ECN support at the senders/receivers +if [ "$ECN_ENABLED" = 1 ]; then + log_info "Enabling ECN support..." + echo 1 > /proc/sys/net/ipv4/tcp_ecn +fi + +#Enable prefetching +if [ "$HWPREF_ENABLED" -eq 1 ]; then + log_info "Enabling hardware prefetching..." + modprobe msr + wrmsr -a 0x1a4 0 +else + log_info "Disabling hardware prefetching..." + modprobe msr + wrmsr -a 0x1a4 1 +fi + From 270fddfa82bee486ae66cad0a65fd562be3eae6c Mon Sep 17 00:00:00 2001 From: Leshna Date: Mon, 23 Jun 2025 21:35:23 -0600 Subject: [PATCH 07/12] setup script for bare metal case --- utils/setup-bare-metal.sh | 162 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 162 insertions(+) create mode 100644 utils/setup-bare-metal.sh diff --git a/utils/setup-bare-metal.sh b/utils/setup-bare-metal.sh new file mode 100644 index 000000000..9f9bf9860 --- /dev/null +++ b/utils/setup-bare-metal.sh @@ -0,0 +1,162 @@ +SCRIPT_NAME="setup-bare-metal" + +#default values +MTU=4000 +DDIO_ENABLED=1 +INTF="eno12409np1" +IP="10.10.1.2" +NIC_BUS="0x6f" +TCP_OPTIMIZATION_ENABLED=1 +TCP_SOCKET_BUF_MB=1 +ECN_ENABLED=1 +HWPREF_ENABLED=1 +RDMA=0 +PFC_ENABLED=0 +RING_BUFFER_SIZE=1024 +DEPS_DIR="/users/Leshna" + +help() +{ + echo "Usage: $SCRIPT_NAME + [ --dep (path to dependencies directories)] + [ --intf (interface name, eg. ens2f0) ] + [ --ip (ip address for the interface) ] + [ --nic-bus (NIC's PCI bus number) ] + [ -m | --mtu (MTU size in bytes; default=4000 for TCP, 4096 for RDMA) ] + [ -d | --ddio (=0/1, whether DDIO should be disabled/enabled; default=0) ] + [ -r | --ring-buffer (size of Rx ring buffer. Note: opt must be set to change this)] + [ --opt (enable TCP optimization TSO,GRO,aRFS) ] + [ --socket-buf (TCP socket buffer size (in MB)) ] + [ --ecn (Enable ECN support in Linux stack) ] + [ --hwpref (Enable hardware prefetching) ] + [ --rdma (=0/1, whether the setup is for running RDMA experiments (MTU offset will be different)) ] + [ --pfc (=0/1, disable or enable PFC) ] + [ -h | --help ]" + exit 2 +} + +SHORT=m:,d:,r:,h +LONG=dep:,nic-bus:,mtu:,ddio:,ring-buffer:,intf:,ip:,opt:,socket-buf:,ecn:,hwpref:,rdma:,pfc:,help +PARSED_OPTS=$(getopt -a -n $SCRIPT_NAME --options $SHORT --longoptions $LONG -- "$@") + +VALID_ARGUMENTS=$# +if [ "$VALID_ARGUMENTS" -eq 0 ]; then + help +fi +eval set -- "$PARSED_OPTS" + +while :;do + case "$1" in + --dep) DEPS_DIR="$2"; shift 2 ;; + --nic-bus) NIC_BUS="$2"; shift 2 ;; + -m | --mtu) MTU="$2"; shift 2 ;; + -d | --ddio) DDIO_ENABLED="$2"; shift 2 ;; + -r | --ring-buffer) RING_BUFFER_SIZE="$2"; shift 2 ;; + --intf) INTF="$2"; shift 2 ;; + --ip) IP="$2"; shift 2 ;; + --opt) TCP_OPTIMIZATION_ENABLED="$2"; shift 2 ;; + --socket-buf) TCP_SOCKET_BUF_MB="$2"; shift 2 ;; + --ecn) ECN_ENABLED="$2"; shift 2 ;; + --hwpref) HWPREF_ENABLED="$2"; shift 2 ;; + --rdma) RDMA="$2"; shift 2 ;; + --pfc) PFC_ENABLED="$2"; shift 2 ;; + -h | --help) help ;; + --) shift; break ;; + *) echo "Unexpected option: $1"; help ;; + esac +done + +log_info() { + echo "[INFO] - $1" +} + +if [ "$RDMA" -eq 1 ]; then + log_info "Configuring MTU according to RDMA supported values..." + MTU=$(($MTU + 96)) + if [ "$MTU" -lt 1280 ]; then + log_info "Requested physical MTU size is $MTU, updating to 1280" + MTU=1280 + fi + log_info "MTU configured to $MTU" +fi + +# setup the interface +log_info "Setting up the interface $INTF..." +ifconfig $INTF up +ifconfig $INTF $IP +ifconfig $INTF mtu $MTU + +#disable TCP buffer auto-tuning, and set the buffer size to the specified size +log_info "Setting up the socket buffer size to be ${TCP_SOCKET_BUF_MB}MB" +echo 0 > /proc/sys/net/ipv4/tcp_moderate_rcvbuf +#Set TCP receive buffer size to be 1MB (other 1MB is for the application buffer) +echo "$(($TCP_SOCKET_BUF_MB * 2000000)) $(($TCP_SOCKET_BUF_MB * 2000000)) $(($TCP_SOCKET_BUF_MB * 2000000))" > /proc/sys/net/ipv4/tcp_rmem +#Set TCP send buffer size to be 1MB +echo "$(($TCP_SOCKET_BUF_MB * 1000000)) $(($TCP_SOCKET_BUF_MB * 1000000)) $(($TCP_SOCKET_BUF_MB * 1000000))" > /proc/sys/net/ipv4/tcp_wmem + +#Enable TCP ECN support at the senders/receivers +if [ "$ECN_ENABLED" = 1 ]; then + log_info "Enabling ECN support..." + echo 1 > /proc/sys/net/ipv4/tcp_ecn +fi + +#Enable aRFS, TSO, GRO for the interface +if [ "$TCP_OPTIMIZATION_ENABLED" -eq 1 ]; then + cd ${DEPS_DIR}/Understanding-network-stack-overheads-SIGCOMM-2021 + log_info "Enabling TCP optimizations (TSO, GRO, aRFS)..." + sudo python3 network_setup.py $INTF --arfs --mtu $MTU --sock-size --tso --gro --ring-buffer $RING_BUFFER_SIZE + cd - +fi + + +#Enable/disable DDIO +cd ${DEPS_DIR}/ddio-bench/ +if [ "$DDIO_ENABLED" -eq 1 ]; then + log_info "Enabling DDIO..." + sudo ./ddio-tool -b $NIC_BUS enable +else + log_info "Disabling DDIO..." + sudo ./ddio-tool -b $NIC_BUS disable +fi +cd - + + +#Enable prefetching +if [ "$HWPREF_ENABLED" -eq 1 ]; then + log_info "Enabling hardware prefetching..." + modprobe msr + wrmsr -a 0x1a4 0 +else + log_info "Disabling hardware prefetching..." + modprobe msr + wrmsr -a 0x1a4 1 +fi + +#Enable PFC (on QoS 0) +if [ "$PFC_ENABLED" -eq 1 ] +then + log_info "Enabling PFC..." + # mlnx_qos -i $INTF --pfc 1,0,0,0,0,0,0,0 + sudo lldptool -T -i $INTF -V PFC willing=no enabled=0 + tc_wrap.py -i $INTF + tc_wrap.py -i $INTF -u 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 + + # To enable on other QoS, modify the above code accordingly + # For eg., to enable PFC on QoS 1 or 2 us the code below + + # Qos 1 + # mlnx_qos -i $INTF --pfc 0,1,0,0,0,0,0,0 + # tc_wrap.py -i $INTF + # tc_wrap.py -i $INTF -u 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1 + + # Qos 2 + # mlnx_qos -i $INTF --pfc 0,0,1,0,0,0,0,0 + # tc_wrap.py -i $INTF + # tc_wrap.py -i $INTF -u 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2 +else + log_info "Disabling PFC..." + sudo lldptool -T -i $INTF -V PFC willing=no enabled= + # sudo mlnx_qos -i $INTF --pfc 0,0,0,0,0,0,0,0 +fi + + From cd73a498970bf5f284423950a4a2607bfff49eca Mon Sep 17 00:00:00 2001 From: Leshna Date: Mon, 23 Jun 2025 23:53:49 -0600 Subject: [PATCH 08/12] refactor record-metrics --- utils/record-host-metrics.sh | 390 ++++++++++++++--------------------- utils/tcp/run-netapp-tput.sh | 112 ++++------ 2 files changed, 193 insertions(+), 309 deletions(-) diff --git a/utils/record-host-metrics.sh b/utils/record-host-metrics.sh index 3ce941f8f..71f751feb 100644 --- a/utils/record-host-metrics.sh +++ b/utils/record-host-metrics.sh @@ -1,140 +1,83 @@ +#default values +SCRIPT_NAME="record-host-metrics" + +DEP_DIR="/home/schai" +OUT_DIR="test" +DURATION_S=30 +TYPE=1 +CPU_UTIL_REPORTING=1 +CPU_MASK=0 +RETX_REPORTING=1 +TCP_LOG_REPORTING=0 +FLAMEGRAPH_REPORTING=0 +BANDWIDTH_REPORTING=1 +PCIE_REPORTING=1 +MEMBW_REPORTING=1 +IIO_REPORTING=0 +PFC_REPORTING=0 +INTF=enp8s0 + +cur_dir=$PWD + help() { - echo "Usage: record-host-metrics [ -H | --home (home directory)] + echo "Usage: $SCRIPT_NAME [ --dep (dependency directory)] [ -o | --outdir (name of the output directory which will store the records; default=test) ] - [ -d | --dur (duration in seconds to record each metric; default=30s) ] - [ -c | --cpu_util (=0/1, disable/enable recording cpu utilization) ) ] - [ -C | --cores (comma separated values of cpu cores to log utilization, eg., '0,4,8,12') ) ] - [ -r | --retx (=0/1, disable/enable recording retransmission rate (should be done at TCP senders) ) ] - [ -T | --tcplog (=0/1, disable/enable recording TCP log (should be done at TCP senders) ) ] - [ -b | --bw (=0/1, disable/enable recording app-level bandwidth ) ] + [ --dur (duration in seconds to record each metric; default=30s) ] + [ --cpu-util (=0/1, disable/enable recording cpu utilization) ) ] + [ -c | --cores (comma separated values of cpu cores to log utilization, eg., '0,4,8,12') ) ] + [ --retx (=0/1, disable/enable recording retransmission rate (should be done at TCP senders) ) ] + [ --tcplog (=0/1, disable/enable recording TCP log (should be done at TCP senders) ) ] + [ --bw (=0/1, disable/enable recording app-level bandwidth ) ] [ -f | --flame (=0/1, disable/enable recording flamegraph (for cores specified via -C/--cores option) ) ] - [ -P | --pcie (=0/1, disable/enable recording PCIe bandwidth) ] - [ -M | --membw (=0/1, disable/enable recording memory bandwidth) ] - [ -I | --iio (=0/1, disable/enable recording IIO occupancy) ] - [ -p | --pfc (=0/1, disable/enable recording PFC pause triggers) ] - [ -i | --intf (interface name, over which to record PFC triggers) ] + [ --pcie (=0/1, disable/enable recording PCIe bandwidth) ] + [ --membw (=0/1, disable/enable recording memory bandwidth) ] + [ --iio (=0/1, disable/enable recording IIO occupancy) ] + [ --pfc (=0/1, disable/enable recording PFC pause triggers) ] + [ --intf (interface name, over which to record PFC triggers) ] [ -t | --type (=0/1, experiment type -- 0 for TCP, 1 for RDMA) ] [ -h | --help ]" exit 2 } -SHORT=H:,o:,d:,c:,C:,r:,T:,b:,f:,P:,M:,I:,p:,i:,t:,h -LONG=home:,outdir:,dur:,cpu_util:,cores:,retx:,tcplog:,bw:,flame:,pcie:,membw:,iio:,pfc:,intf:,type:,help -OPTS=$(getopt -a -n record-host-metrics --options $SHORT --longoptions $LONG -- "$@") +SHORT=o:,c:,f:,t:,h +LONG=dep:,outdir:,dur:,cpu-util:,cores:,retx:,tcplog:,bw:,flame:,pcie:,membw:,iio:,pfc:,intf:,type:,help +OPTS=$(getopt -a -n $SCRIPT_NAME --options $SHORT --longoptions $LONG -- "$@") VALID_ARGUMENTS=$# # Returns the count of arguments that are in short or long options - if [ "$VALID_ARGUMENTS" -eq 0 ]; then help fi - eval set -- "$OPTS" -#default values -home='/home/benny' -outdir='test' -dur=30 -type=1 -cpu_util=1 -cores=0 -retx=1 -tcplog=0 -flame=0 -bw=1 -pcie=1 -membw=1 -iio=0 -pfc=0 -intf=ens2f0 - -cur_dir=$PWD - #TODO: add input config file to specify NUMA node and PCIe slot for PCIe, MemBW and IIO occupancy logging - - -while : -do +while :; do case "$1" in - -H | --home ) - home="$2" - shift 2 - ;; - -o | --outdir ) - outdir="$2" - shift 2 - ;; - -d | --dur ) - dur="$2" - shift 2 - ;; - -c | --cpu_util ) - cpu_util="$2" - shift 2 - ;; - -C | --cores ) - cores="$2" - shift 2 - ;; - -r | --retx ) - retx="$2" - shift 2 - ;; - -T | --tcplog ) - tcplog="$2" - shift 2 - ;; - -b | --bw ) - bw="$2" - shift 2 - ;; - -f | --flame ) - flame="$2" - shift 2 - ;; - -P | --pcie ) - pcie="$2" - shift 2 - ;; - -M | --membw ) - membw="$2" - shift 2 - ;; - -I | --iio ) - iio="$2" - shift 2 - ;; - -p | --pfc ) - pfc="$2" - shift 2 - ;; - -i | --intf ) - intf="$2" - shift 2 - ;; - -t | --type ) - type="$2" - shift 2 - ;; - -h | --help) - help - ;; - --) - shift; - break - ;; - *) - echo "Unexpected option: $1" - help - ;; + --dep) DEP_DIR="$2"; shift 2 ;; + -o | --outdir) OUT_DIR="$2"; shift 2 ;; + --dur) DURATION_S="$2"; shift 2 ;; + --cpu-util) CPU_UTIL_REPORTING="$2"; shift 2 ;; + -c | --cores) CPU_MASK="$2"; shift 2 ;; + --retx) RETX_REPORTING="$2"; shift 2 ;; + --tcplog) TCP_LOG_REPORTING="$2"; shift 2 ;; + --bw) BANDWIDTH_REPORTING="$2"; shift 2 ;; + -f | --flame) FLAMEGRAPH_REPORTING="$2"; shift 2 ;; + --pcie) PCIE_REPORTING="$2"; shift 2 ;; + --membw) MEMBW_REPORTING="$2"; shift 2 ;; + --iio) IIO_REPORTING="$2"; shift 2 ;; + --pfc) PFC_REPORTING="$2"; shift 2 ;; + --intf) INTF="$2"; shift 2 ;; + -t | --type) TYPE="$2"; shift 2 ;; + -h | --help) help ;; + --) shift; break ;; + *) echo "Unexpected option: $1"; help ;; esac done - mkdir -p logs #Directory to store collected logs -mkdir -p logs/$outdir #Directory to store collected logs +mkdir -p logs/$OUT_DIR #Directory to store collected logs mkdir -p reports #Directory to store parsed metrics -mkdir -p reports/$outdir #Directory to store parsed metrics +mkdir -p reports/$OUT_DIR #Directory to store parsed metrics function dump_netstat() { local SLEEP_TIME=$1 @@ -145,72 +88,70 @@ function dump_netstat() { sleep $SLEEP_TIME echo "After measurement" netstat -s - } function dump_pciebw() { modprobe msr - sudo taskset -c 31 $home/pcm/build/bin/pcm-iio 1 -csv=logs/$outdir/pcie.csv & + sudo taskset -c 15 $DEP_DIR/pcm/build/bin/pcm-iio 1 -csv=logs/$OUT_DIR/pcie.csv & } function parse_pciebw() { #TODO: make more general, parse PCIe bandwidth for any given socket and IIO stack - echo "PCIe_wr_tput: " $(cat logs/$outdir/pcie.csv | grep "Socket0,IIO Stack 2 - PCIe1,Part0" | awk -F ',' '{ sum += $4/1000000000.0; n++ } END { if (n > 0) printf "%.3f", sum / n * 8 ; }') > reports/$outdir/pcie.rpt - echo "PCIe_rd_tput: " $(cat logs/$outdir/pcie.csv | grep "Socket0,IIO Stack 2 - PCIe1,Part0" | awk -F ',' '{ sum += $5/1000000000.0; n++ } END { if (n > 0) printf "%0.3f", sum / n * 8 ; }') >> reports/$outdir/pcie.rpt - echo "IOTLB_hits: " $(cat logs/$outdir/pcie.csv | grep "Socket0,IIO Stack 2 - PCIe1,Part0" | awk -F ',' '{ sum += $8; n++ } END { if (n > 0) printf "%0.3f", sum / n; }') >> reports/$outdir/pcie.rpt - echo "IOTLB_misses: " $(cat logs/$outdir/pcie.csv | grep "Socket0,IIO Stack 2 - PCIe1,Part0" | awk -F ',' '{ sum += $9; n++ } END { if (n > 0) printf "%0.3f", sum / n; }') >> reports/$outdir/pcie.rpt - echo "CTXT_Miss: " $(cat logs/$outdir/pcie.csv | grep "Socket0,IIO Stack 2 - PCIe1,Part0" | awk -F ',' '{ sum += $10; n++ } END { if (n > 0) printf "%0.3f", sum / n; }') >> reports/$outdir/pcie.rpt - echo "L1_Miss: " $(cat logs/$outdir/pcie.csv | grep "Socket0,IIO Stack 2 - PCIe1,Part0" | awk -F ',' '{ sum += $11; n++ } END { if (n > 0) printf "%0.3f", sum / n; }') >> reports/$outdir/pcie.rpt - echo "L2_Miss: " $(cat logs/$outdir/pcie.csv | grep "Socket0,IIO Stack 2 - PCIe1,Part0" | awk -F ',' '{ sum += $12; n++ } END { if (n > 0) printf "%0.3f", sum / n; }') >> reports/$outdir/pcie.rpt - echo "L3_Miss: " $(cat logs/$outdir/pcie.csv | grep "Socket0,IIO Stack 2 - PCIe1,Part0" | awk -F ',' '{ sum += $13; n++ } END { if (n > 0) printf "%0.3f", sum / n; }') >> reports/$outdir/pcie.rpt - echo "Mem_Read: " $(cat logs/$outdir/pcie.csv | grep "Socket0,IIO Stack 2 - PCIe1,Part0" | awk -F ',' '{ sum += $14; n++ } END { if (n > 0) printf "%0.3f", sum / n; }') >> reports/$outdir/pcie.rpt + echo "PCIe_wr_tput: " $(cat logs/$OUT_DIR/pcie.csv | grep "Socket0,IIO Stack 2 - PCIe1,Part0" | awk -F ',' '{ sum += $4/1000000000.0; n++ } END { if (n > 0) printf "%.3f", sum / n * 8 ; }') > reports/$OUT_DIR/pcie.rpt + echo "PCIe_rd_tput: " $(cat logs/$OUT_DIR/pcie.csv | grep "Socket0,IIO Stack 2 - PCIe1,Part0" | awk -F ',' '{ sum += $5/1000000000.0; n++ } END { if (n > 0) printf "%0.3f", sum / n * 8 ; }') >> reports/$OUT_DIR/pcie.rpt + echo "IOTLB_hits: " $(cat logs/$OUT_DIR/pcie.csv | grep "Socket0,IIO Stack 2 - PCIe1,Part0" | awk -F ',' '{ sum += $8; n++ } END { if (n > 0) printf "%0.3f", sum / n; }') >> reports/$OUT_DIR/pcie.rpt + echo "IOTLB_misses: " $(cat logs/$OUT_DIR/pcie.csv | grep "Socket0,IIO Stack 2 - PCIe1,Part0" | awk -F ',' '{ sum += $9; n++ } END { if (n > 0) printf "%0.3f", sum / n; }') >> reports/$OUT_DIR/pcie.rpt + echo "CTXT_Miss: " $(cat logs/$OUT_DIR/pcie.csv | grep "Socket0,IIO Stack 2 - PCIe1,Part0" | awk -F ',' '{ sum += $10; n++ } END { if (n > 0) printf "%0.3f", sum / n; }') >> reports/$OUT_DIR/pcie.rpt + echo "L1_Miss: " $(cat logs/$OUT_DIR/pcie.csv | grep "Socket0,IIO Stack 2 - PCIe1,Part0" | awk -F ',' '{ sum += $11; n++ } END { if (n > 0) printf "%0.3f", sum / n; }') >> reports/$OUT_DIR/pcie.rpt + echo "L2_Miss: " $(cat logs/$OUT_DIR/pcie.csv | grep "Socket0,IIO Stack 2 - PCIe1,Part0" | awk -F ',' '{ sum += $12; n++ } END { if (n > 0) printf "%0.3f", sum / n; }') >> reports/$OUT_DIR/pcie.rpt + echo "L3_Miss: " $(cat logs/$OUT_DIR/pcie.csv | grep "Socket0,IIO Stack 2 - PCIe1,Part0" | awk -F ',' '{ sum += $13; n++ } END { if (n > 0) printf "%0.3f", sum / n; }') >> reports/$OUT_DIR/pcie.rpt + echo "Mem_Read: " $(cat logs/$OUT_DIR/pcie.csv | grep "Socket0,IIO Stack 2 - PCIe1,Part0" | awk -F ',' '{ sum += $14; n++ } END { if (n > 0) printf "%0.3f", sum / n; }') >> reports/$OUT_DIR/pcie.rpt } function dump_membw() { modprobe msr - sudo taskset -c 31 $home/pcm/build/bin/pcm-memory 1 -columns=5 + sudo taskset -c 15 $home/pcm/build/bin/pcm-memory 1 -columns=5 } function parse_membw() { #TODO: make more general, parse memory bandwidth for any given number of sockets - echo "Node0_rd_bw: " $(cat logs/$outdir/membw.log | grep "NODE 0 Mem Read" | awk '{ sum += $8; n++ } END { if (n > 0) printf "%f\n", sum / n; }') > reports/$outdir/membw.rpt - echo "Node0_wr_bw: " $(cat logs/$outdir/membw.log | grep "NODE 0 Mem Write" | awk '{ sum += $7; n++ } END { if (n > 0) printf "%f\n", sum / n; }') >> reports/$outdir/membw.rpt - echo "Node0_total_bw: " $(cat logs/$outdir/membw.log | grep "NODE 0 Memory" | awk '{ sum += $6; n++ } END { if (n > 0) printf "%f\n", sum / n; }') >> reports/$outdir/membw.rpt - echo "Node1_rd_bw: " $(cat logs/$outdir/membw.log | grep "NODE 1 Mem Read" | awk '{ sum += $16; n++ } END { if (n > 0) printf "%f\n", sum / n; }') >> reports/$outdir/membw.rpt - echo "Node1_wr_bw: " $(cat logs/$outdir/membw.log | grep "NODE 1 Mem Write" | awk '{ sum += $14; n++ } END { if (n > 0) printf "%f\n", sum / n; }') >> reports/$outdir/membw.rpt - echo "Node1_total_bw: " $(cat logs/$outdir/membw.log | grep "NODE 1 Memory" | awk '{ sum += $12; n++ } END { if (n > 0) printf "%f\n", sum / n; }') >> reports/$outdir/membw.rpt - echo "Node2_rd_bw: " $(cat logs/$outdir/membw.log | grep "NODE 2 Mem Read" | awk '{ sum += $24; n++ } END { if (n > 0) printf "%f\n", sum / n; }') >> reports/$outdir/membw.rpt - echo "Node2_wr_bw: " $(cat logs/$outdir/membw.log | grep "NODE 2 Mem Write" | awk '{ sum += $21; n++ } END { if (n > 0) printf "%f\n", sum / n; }') >> reports/$outdir/membw.rpt - echo "Node2_total_bw: " $(cat logs/$outdir/membw.log | grep "NODE 2 Memory" | awk '{ sum += $18; n++ } END { if (n > 0) printf "%f\n", sum / n; }') >> reports/$outdir/membw.rpt - echo "Node3_rd_bw: " $(cat logs/$outdir/membw.log | grep "NODE 3 Mem Read" | awk '{ sum += $32; n++ } END { if (n > 0) printf "%f\n", sum / n; }') >> reports/$outdir/membw.rpt - echo "Node3_wr_bw: " $(cat logs/$outdir/membw.log | grep "NODE 3 Mem Write" | awk '{ sum += $28; n++ } END { if (n > 0) printf "%f\n", sum / n; }') >> reports/$outdir/membw.rpt - echo "Node3_total_bw: " $(cat logs/$outdir/membw.log | grep "NODE 3 Memory" | awk '{ sum += $24; n++ } END { if (n > 0) printf "%f\n", sum / n; }') >> reports/$outdir/membw.rpt + echo "Node0_rd_bw: " $(cat logs/$OUT_DIR/membw.log | grep "NODE 0 Mem Read" | awk '{ sum += $8; n++ } END { if (n > 0) printf "%f\n", sum / n; }') > reports/$OUT_DIR/membw.rpt + echo "Node0_wr_bw: " $(cat logs/$OUT_DIR/membw.log | grep "NODE 0 Mem Write" | awk '{ sum += $7; n++ } END { if (n > 0) printf "%f\n", sum / n; }') >> reports/$OUT_DIR/membw.rpt + echo "Node0_total_bw: " $(cat logs/$OUT_DIR/membw.log | grep "NODE 0 Memory" | awk '{ sum += $6; n++ } END { if (n > 0) printf "%f\n", sum / n; }') >> reports/$OUT_DIR/membw.rpt + echo "Node1_rd_bw: " $(cat logs/$OUT_DIR/membw.log | grep "NODE 1 Mem Read" | awk '{ sum += $16; n++ } END { if (n > 0) printf "%f\n", sum / n; }') >> reports/$OUT_DIR/membw.rpt + echo "Node1_wr_bw: " $(cat logs/$OUT_DIR/membw.log | grep "NODE 1 Mem Write" | awk '{ sum += $14; n++ } END { if (n > 0) printf "%f\n", sum / n; }') >> reports/$OUT_DIR/membw.rpt + echo "Node1_total_bw: " $(cat logs/$OUT_DIR/membw.log | grep "NODE 1 Memory" | awk '{ sum += $12; n++ } END { if (n > 0) printf "%f\n", sum / n; }') >> reports/$OUT_DIR/membw.rpt + echo "Node2_rd_bw: " $(cat logs/$OUT_DIR/membw.log | grep "NODE 2 Mem Read" | awk '{ sum += $24; n++ } END { if (n > 0) printf "%f\n", sum / n; }') >> reports/$OUT_DIR/membw.rpt + echo "Node2_wr_bw: " $(cat logs/$OUT_DIR/membw.log | grep "NODE 2 Mem Write" | awk '{ sum += $21; n++ } END { if (n > 0) printf "%f\n", sum / n; }') >> reports/$OUT_DIR/membw.rpt + echo "Node2_total_bw: " $(cat logs/$OUT_DIR/membw.log | grep "NODE 2 Memory" | awk '{ sum += $18; n++ } END { if (n > 0) printf "%f\n", sum / n; }') >> reports/$OUT_DIR/membw.rpt + echo "Node3_rd_bw: " $(cat logs/$OUT_DIR/membw.log | grep "NODE 3 Mem Read" | awk '{ sum += $32; n++ } END { if (n > 0) printf "%f\n", sum / n; }') >> reports/$OUT_DIR/membw.rpt + echo "Node3_wr_bw: " $(cat logs/$OUT_DIR/membw.log | grep "NODE 3 Mem Write" | awk '{ sum += $28; n++ } END { if (n > 0) printf "%f\n", sum / n; }') >> reports/$OUT_DIR/membw.rpt + echo "Node3_total_bw: " $(cat logs/$OUT_DIR/membw.log | grep "NODE 3 Memory" | awk '{ sum += $24; n++ } END { if (n > 0) printf "%f\n", sum / n; }') >> reports/$OUT_DIR/membw.rpt } function collect_pfc() { #assuming PFC is enabled for QoS 0 - sudo ethtool -S $intf | grep pause > logs/$outdir/pause.before.log - sleep $dur - sudo ethtool -S $intf | grep pause > logs/$outdir/pause.after.log + sudo ethtool -S $INTF | grep pause > logs/$OUT_DIR/pause.before.log + sleep $DURATION_S + sudo ethtool -S $INTF | grep pause > logs/$OUT_DIR/pause.after.log - pause_before=$(cat logs/$outdir/pause.before.log | grep "tx_prio0_pause" | head -n1 | awk '{ printf $2 }') - pause_duration_before=$(cat logs/$outdir/pause.before.log | grep "tx_prio0_pause_duration" | awk '{ printf $2 }') - pause_after=$(cat logs/$outdir/pause.after.log | grep "tx_prio0_pause" | head -n1 | awk '{ printf $2 }') - pause_duration_after=$(cat logs/$outdir/pause.after.log | grep "tx_prio0_pause_duration" | awk '{ printf $2 }') + pause_before=$(cat logs/$OUT_DIR/pause.before.log | grep "tx_prio0_pause" | head -n1 | awk '{ printf $2 }') + pause_duration_before=$(cat logs/$OUT_DIR/pause.before.log | grep "tx_prio0_pause_duration" | awk '{ printf $2 }') + pause_after=$(cat logs/$OUT_DIR/pause.after.log | grep "tx_prio0_pause" | head -n1 | awk '{ printf $2 }') + pause_duration_after=$(cat logs/$OUT_DIR/pause.after.log | grep "tx_prio0_pause_duration" | awk '{ printf $2 }') - echo "pauses_before: "$pause_before > logs/$outdir/pause.log - echo "pause_duration_before: "$pause_duration_before >> logs/$outdir/pause.log - echo "pauses_after: "$pause_after >> logs/$outdir/pause.log - echo "pause_duration_after: "$pause_duration_after >> logs/$outdir/pause.log + echo "pauses_before: "$pause_before > logs/$OUT_DIR/pause.log + echo "pause_duration_before: "$pause_duration_before >> logs/$OUT_DIR/pause.log + echo "pauses_after: "$pause_after >> logs/$OUT_DIR/pause.log + echo "pause_duration_after: "$pause_duration_after >> logs/$OUT_DIR/pause.log # echo $pause_before, $pause_after - echo "print(($pause_after - $pause_before)/$dur)" | lua > reports/$outdir/pause.rpt + echo "print(($pause_after - $pause_before)/$DURATION_S)" | lua > reports/$OUT_DIR/pause.rpt # echo $pause_duration_before, $pause_duration_after - echo "print(($pause_duration_after - $pause_duration_before)/$dur)" | lua >> reports/$outdir/pause.rpt + echo "print(($pause_duration_after - $pause_duration_before)/$DURATION_S)" | lua >> reports/$OUT_DIR/pause.rpt } - function compile_if_needed() { local source_file=$1 local executable=$2 @@ -229,111 +170,90 @@ function compile_if_needed() { fi } - - -if [ "$type" = 0 ] -then +if [ "$TYPE" -eq 0 ]; then echo "Collecting TCP experiment metrics..." - if [ "$cpu_util" = 1 ] - then - echo "Collecting CPU utilization for cores $cores..." - sar -P $cores 1 1000 > logs/$outdir/cpu_util.log & - sleep $dur - sudo pkill -9 -f "sar" - python3 cpu_util.py logs/$outdir/cpu_util.log > reports/$outdir/cpu_util.rpt + if [ "$CPU_UTIL_REPORTING" -eq 1 ]; then + echo "Collecting CPU utilization for cores $CPU_MASK..." + sar -P $CPU_MASK 1 1000 > logs/$OUT_DIR/cpu_util.log & + sleep $DURATION_S + sudo pkill -9 -f "sar" + python3 cpu_util.py logs/$OUT_DIR/cpu_util.log > reports/$OUT_DIR/cpu_util.rpt fi - # if ["$bw" = 1 ] - # then - # echo "Collecting app bandwidth..." - # echo "Avg_iperf_tput: " $(cat logs/$outdir/iperf.bw.log | grep "60.*-90.*" | awk '{ sum += $7; n++ } END { if (n > 0) printf "%.3f", sum/1000; }') > reports/$outdir/iperf.bw.rpt - # fi - - if [ "$retx" = 1 ] - then - echo "Collecting retransmission rate..." - dump_netstat $dur > logs/$outdir/retx.log - cat logs/$outdir/retx.log | grep -E "segment|TCPLostRetransmit" > retx.out - python3 print_retx_rate.py retx.out $dur > reports/$outdir/retx.rpt + if [ "$BANDWIDTH_REPORTING" -eq 1 ]; then + echo "Collecting app bandwidth..." + echo "Avg_iperf_tput: " $(cat logs/$OUT_DIR/iperf.bw.log | grep "60.*-90.*" | awk '{ sum += $7; n++ } END { if (n > 0) printf "%.3f", sum/1000; }') > reports/$OUT_DIR/iperf.bw.rpt fi - if [ "$tcplog" = 1 ] - then - echo "Collecting tcplog..." - cd /sys/kernel/debug/tracing - echo > trace - echo 1 > events/tcp/tcp_probe/enable - sleep 2 - echo 0 > events/tcp/tcp_probe/enable - sleep 2 - cp trace $cur_dir/logs/$outdir/tcp.trace.log - echo > trace - cd - - python3 parse_tcplog.py $outdir + if [ "$RETX_REPORTING" -eq 1 ]; then + echo "Collecting retransmission rate..." + dump_netstat $DURATION_S > logs/$OUT_DIR/retx.log + cat logs/$OUT_DIR/retx.log | grep -E "segment|TCPLostRetransmit" > retx.out + python3 print_retx_rate.py retx.out $DURATION_S > reports/$OUT_DIR/retx.rpt fi -elif [ "$type" = 1 ] -then - echo "Collecting RDMA experiment metrics..." - if [ "$pfc" = 1 ] - then + if [ "$TCP_LOG_REPORTING" -eq 1 ]; then + echo "Collecting tcplog..." + cd /sys/kernel/debug/tracing + echo > trace + echo 1 > events/tcp/tcp_probe/enable + sleep 2 + echo 0 > events/tcp/tcp_probe/enable + sleep 2 + cp trace $cur_dir/logs/$OUT_DIR/tcp.trace.log + echo > trace + cd - + python3 parse_tcplog.py $OUT_DIR + fi +elif [ "$TYPE" -eq 1 ]; then + echo "Collecting RDMA experiment metrics..." + + if [ "$PFC_REPORTING" -eq 1 ]; then echo "Collecting PFC triggers at RDMA server..." collect_pfc - fi - + fi else - echo "Incorrect type..." - help + echo "Incorrect type..." + help fi - -if [ "$pcie" = 1 ] -then - echo "Collecting PCIe bandwidth..." - dump_pciebw - sleep $dur - sudo pkill -9 -f "pcm" - parse_pciebw +if [ "$PCIE_REPORTING" -eq 1 ]; then + echo "Collecting PCIe bandwidth..." + dump_pciebw + sleep $DURATION_S + sudo pkill -9 -f "pcm" + parse_pciebw fi - -if [ "$membw" = 1 ] -then - echo "Collecting Memory bandwidth..." - dump_membw > logs/$outdir/membw.log & - sleep 30 - sleep $dur - sudo pkill -9 -f "pcm" - parse_membw +if [ "$MEMBW_REPORTING" -eq 1 ]; then + echo "Collecting Memory bandwidth..." + dump_membw > logs/$OUT_DIR/membw.log & + sleep 30 + sleep $DURATION_S + sudo pkill -9 -f "pcm" + parse_membw fi - -if [ "$iio" = 1 ] -then - echo "Collecting IIO occupancy..." - # gcc collect_iio_occ.c -o collect_iio_occ - compile_if_needed collect_iio_occ.c collect_iio_occ - taskset -c 28 ./collect_iio_occ & - sleep 5 - sudo pkill -2 -f collect_iio_occ - sleep 5 - mv iio.log logs/$outdir/iio.log - #TODO: make more generic and add a parser to create report for iio occupancy logging from userspace +if [ "$IIO_REPORTING" -eq 1 ]; then + echo "Collecting IIO occupancy..." + compile_if_needed collect_iio_occ.c collect_iio_occ + taskset -c 14 ./collect_iio_occ & + sleep 5 + sudo pkill -2 -f collect_iio_occ + sleep 5 + mv iio.log logs/$OUT_DIR/iio.log + #TODO: make more generic and add a parser to create report for iio occupancy logging from userspace fi - -if [ "$flame" = 1 ] -then +if [ "$FLAMEGRAPH_REPORTING" -eq 1 ]; then sudo rm -f out.perf-folded echo "Creating Flame Graph..." - sudo perf record -C $cores -g -F 99 -- sleep $dur - sudo perf script | $home/FlameGraph/stackcollapse-perf.pl > out.perf-folded - sudo $home/FlameGraph/flamegraph.pl out.perf-folded > logs/$outdir/perf-kernel-flame.svg + sudo perf record -C $CPU_MASK -g -F 99 -- sleep $DURATION_S + sudo perf script | $DEP_DIR/FlameGraph/stackcollapse-perf.pl > out.perf-folded + sudo $DEP_DIR/FlameGraph/flamegraph.pl out.perf-folded > logs/$OUT_DIR/perf-kernel-flame.svg # also collect cache miss rates - sudo perf stat -C $cores -e LLC-load,LLC-load-misses,l2_rqsts.all_demand_miss,l2_rqsts.all_demand_references -o logs/$outdir/llc.miss.log sleep 2 + sudo perf stat -C $CPU_MASK -e LLC-load,LLC-load-misses,l2_rqsts.all_demand_miss,l2_rqsts.all_demand_references -o logs/$OUT_DIR/llc.miss.log sleep 2 #loadmisses=$(cat logs/$4/$3/llc.miss.log | grep "LLC-load-misses" | awk '{ printf $1 }') #loads=$(cat logs/$4/$3/llc.miss.log | grep "LLC-load " | awk '{ printf $1 }') - - fi diff --git a/utils/tcp/run-netapp-tput.sh b/utils/tcp/run-netapp-tput.sh index a4fcddc1b..90b2c4661 100644 --- a/utils/tcp/run-netapp-tput.sh +++ b/utils/tcp/run-netapp-tput.sh @@ -1,113 +1,77 @@ #!/bin/bash +SCRIPT_NAME="run-netapp-tput" + +#default values +MODE="server" +OUT_DIR="tcptest" +SERVER_IP="192.168.11.127" +CPU_MASK="0,1,2,3,4" +NUM_SERVERS=5 +NUM_CLIENTS=5 +PORT=3000 +BANDWIDTH="100g" help() { - echo "Usage: run-netapp-tput [ -m | --mode (=client/server) ] + echo "Usage: $SCRIPT_NAME [ --mode (=client/server) ] [ -o | --outdir (output directory to store the application stats log; default='test')] - [ -S | --num_servers (number of server instances)] - [ -C | --num_clients (number of client instances, only use this option at client)] + [ -n | --num_servers (number of server instances)] + [ -N | --num_clients (number of client instances, only use this option at client)] [ -p | --port (port number for the first connection) ] - [ -a | --addr (ip address of the server, only use this option at client) ] + [ --server-ip (ip address of the server, only use this option at client) ] [ -c | --cores (comma separated cpu core values to run the clients/servers at, for eg., cpu=4,8,12,16; if the number of clients/servers > the number of input cpu cores, the clients/servers will round-robin over the provided input cores; recommended to run on NUMA node local to the NIC for maximum performance) ] [ -b | --bandwidth (bandwidth to send at in bits/sec)] [ -h | --help ]" exit 2 } -SHORT=m:,o:,S:,C:,p:,a:,c:,b:h -LONG=mode:,outdir:,num_servers:,num_clients:,port:,addr:,cores:,bandwidth:,help +SHORT=o:,n:,N:,p:,c:,b:h +LONG=mode:,outdir:,num_servers:,num_clients:,port:,server-ip:,cores:,bandwidth:,help OPTS=$(getopt -a -n run-netapp-tput --options $SHORT --longoptions $LONG -- "$@") - VALID_ARGUMENTS=$# # Returns the count of arguments that are in short or long options - if [ "$VALID_ARGUMENTS" -eq 0 ]; then help fi - eval set -- "$OPTS" -#default values -mode="server" -outdir="tcptest" -addr="192.168.10.122" -cores="4,8,12,16" -num_servers=4 -num_clients=4 -port=3000 -bandwidth="100g" - - -while : -do +while :;do case "$1" in - -m | --mode ) - mode="$2" - shift 2 - ;; - -o | --outdir ) - outdir="$2" - shift 2 - ;; - -S | --num_servers ) - num_servers="$2" - shift 2 - ;; - -C | --num_clients ) - num_clients="$2" - shift 2 - ;; - -p | --port ) - port="$2" - shift 2 - ;; - -a | --addr ) - addr="$2" - shift 2 - ;; - -c | --cores ) - cores="$2" - shift 2 - ;; - -b | --bandwidth ) - bandwidth="$2" - shift 2 - ;; - -h | --help) - help - ;; - --) - shift; - break - ;; - *) - echo "Unexpected option: $1" - help - ;; + --mode) MODE="$2"; shift 2 ;; + -o | --outdir) OUT_DIR="$2"; shift 2 ;; + -n | --num_servers) NUM_SERVERS="$2"; shift 2 ;; + -N | --num_clients) NUM_CLIENTS="$2"; shift 2 ;; + -p | --port) PORT="$2"; shift 2 ;; + --server-ip) SERVER_IP="$2"; shift 2 ;; + -c | --cores) CPU_MASK="$2"; shift 2 ;; + -b | --bandwidth) BANDWIDTH="$2"; shift 2 ;; + -h | --help) help ;; + --) shift; break ;; + *) echo "Unexpected option: $1"; help ;; esac done -IFS=',' read -ra core_values <<< $cores +IFS=',' read -ra core_values <<< $CPU_MASK mkdir -p ../reports #Directory to store collected logs -mkdir -p ../reports/$outdir #Directory to store collected logs +mkdir -p ../reports/$OUT_DIR #Directory to store collected logs mkdir -p ../logs #Directory to store collected logs -mkdir -p ../logs/$outdir #Directory to store collected logs -rm ../logs/$outdir/iperf.bw.log +mkdir -p ../logs/$OUT_DIR #Directory to store collected logs +rm ../logs/$OUT_DIR/iperf.bw.log function collect_stats() { echo "Collecting app throughput for TCP server..." - echo "Avg_iperf_tput: " $(cat ../logs/$outdir/iperf.bw.log | grep "30.*-60.*" | awk '{ sum += $7; n++ } END { if (n > 0) printf "%.3f", sum/1000; }') > ../reports/$outdir/iperf.bw.rpt + echo "Avg_iperf_tput: " $(cat ../logs/$OUT_DIR/iperf.bw.log | grep "30.*-60.*" | awk '{ sum += $7; n++ } END { if (n > 0) printf "%.3f", sum/1000; }') > ../reports/$OUT_DIR/iperf.bw.rpt } counter=0 if [ "$mode" = "server" ] then sudo pkill -9 -f iperf #kill existing iperf servers/clients - while [ $counter -lt $num_servers ]; do + while [ $counter -lt $NUM_SERVERS ]; do index=$(( counter % ${#core_values[@]} )) core=${core_values[index]} echo "Starting server $counter on core $core" - taskset -c $core nice -n -20 iperf3 -s --port $(($port + $counter)) -i 30 -f m --logfile ../logs/$outdir/iperf.bw.log & + sudo taskset -c $core nice -n -20 iperf3 -s --port $(($PORT + $counter)) -i 30 -f m --logfile ../logs/$OUT_DIR/iperf.bw.log & ((counter++)) done echo "waiting for few minutes before collecting stats..." @@ -117,11 +81,11 @@ then elif [ "$mode" = "client" ] then sudo pkill -9 -f iperf #kill existing iperf servers/clients - while [ $counter -lt $num_clients ]; do + while [ $counter -lt $NUM_CLIENTS ]; do index=$(( counter % ${#core_values[@]} )) core=${core_values[index]} echo "Starting client $counter on core $core" - taskset -c $core nice -n -20 iperf3 -c $addr --port $(($port+$(($counter%$num_servers)))) -t 10000 -C dctcp -b $bandwidth & + taskset -c $core nice -n -20 iperf3 -c $SERVER_IP --port $(($PORT+$(($counter%$NUM_SERVERS)))) -t 10000 -C dctcp -b $BANDWIDTH & ((counter++)) done else From 37259352701b121e5f7c55ce9532e5e621c93a31 Mon Sep 17 00:00:00 2001 From: Leshna Date: Tue, 24 Jun 2025 00:33:58 -0600 Subject: [PATCH 09/12] fix --- utils/record-host-metrics.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/record-host-metrics.sh b/utils/record-host-metrics.sh index 71f751feb..d8bda58ea 100644 --- a/utils/record-host-metrics.sh +++ b/utils/record-host-metrics.sh @@ -110,7 +110,7 @@ function parse_pciebw() { function dump_membw() { modprobe msr - sudo taskset -c 15 $home/pcm/build/bin/pcm-memory 1 -columns=5 + sudo taskset -c 15 $DEP_DIR/pcm/build/bin/pcm-memory 1 -columns=5 } function parse_membw() { From 97755b2b529f2d1fef782ee561442890a9e22883 Mon Sep 17 00:00:00 2001 From: Leshna Date: Tue, 24 Jun 2025 00:38:14 -0600 Subject: [PATCH 10/12] sudo --- utils/record-host-metrics.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utils/record-host-metrics.sh b/utils/record-host-metrics.sh index d8bda58ea..fd2a9c8fa 100644 --- a/utils/record-host-metrics.sh +++ b/utils/record-host-metrics.sh @@ -91,7 +91,7 @@ function dump_netstat() { } function dump_pciebw() { - modprobe msr + sudo modprobe msr sudo taskset -c 15 $DEP_DIR/pcm/build/bin/pcm-iio 1 -csv=logs/$OUT_DIR/pcie.csv & } @@ -109,7 +109,7 @@ function parse_pciebw() { } function dump_membw() { - modprobe msr + sudo modprobe msr sudo taskset -c 15 $DEP_DIR/pcm/build/bin/pcm-memory 1 -columns=5 } From 3f8b2ff2b56e73cdf1a09ad8ca99a3e12d897393 Mon Sep 17 00:00:00 2001 From: Leshna Date: Tue, 24 Jun 2025 01:08:10 -0600 Subject: [PATCH 11/12] parsing --- utils/record-host-metrics.sh | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/utils/record-host-metrics.sh b/utils/record-host-metrics.sh index fd2a9c8fa..05197e0be 100644 --- a/utils/record-host-metrics.sh +++ b/utils/record-host-metrics.sh @@ -16,6 +16,7 @@ MEMBW_REPORTING=1 IIO_REPORTING=0 PFC_REPORTING=0 INTF=enp8s0 +PCIE_PATTERN="Socket0,IIO Stack 2 - PCIe0,Part0" cur_dir=$PWD @@ -35,13 +36,14 @@ help() [ --iio (=0/1, disable/enable recording IIO occupancy) ] [ --pfc (=0/1, disable/enable recording PFC pause triggers) ] [ --intf (interface name, over which to record PFC triggers) ] - [ -t | --type (=0/1, experiment type -- 0 for TCP, 1 for RDMA) ] + [ -t | --type (=0/1, experiment type -- 0 for TCP, 1 for RDMA) ] + [ --pattern ] [ -h | --help ]" exit 2 } SHORT=o:,c:,f:,t:,h -LONG=dep:,outdir:,dur:,cpu-util:,cores:,retx:,tcplog:,bw:,flame:,pcie:,membw:,iio:,pfc:,intf:,type:,help +LONG=dep:,outdir:,dur:,cpu-util:,cores:,retx:,tcplog:,bw:,flame:,pcie:,membw:,iio:,pfc:,intf:,type:,pattern:,help OPTS=$(getopt -a -n $SCRIPT_NAME --options $SHORT --longoptions $LONG -- "$@") VALID_ARGUMENTS=$# # Returns the count of arguments that are in short or long options @@ -67,6 +69,7 @@ while :; do --iio) IIO_REPORTING="$2"; shift 2 ;; --pfc) PFC_REPORTING="$2"; shift 2 ;; --intf) INTF="$2"; shift 2 ;; + --pattern) PCIE_PATTERN="$2"; shift 2 ;; -t | --type) TYPE="$2"; shift 2 ;; -h | --help) help ;; --) shift; break ;; @@ -97,15 +100,15 @@ function dump_pciebw() { function parse_pciebw() { #TODO: make more general, parse PCIe bandwidth for any given socket and IIO stack - echo "PCIe_wr_tput: " $(cat logs/$OUT_DIR/pcie.csv | grep "Socket0,IIO Stack 2 - PCIe1,Part0" | awk -F ',' '{ sum += $4/1000000000.0; n++ } END { if (n > 0) printf "%.3f", sum / n * 8 ; }') > reports/$OUT_DIR/pcie.rpt - echo "PCIe_rd_tput: " $(cat logs/$OUT_DIR/pcie.csv | grep "Socket0,IIO Stack 2 - PCIe1,Part0" | awk -F ',' '{ sum += $5/1000000000.0; n++ } END { if (n > 0) printf "%0.3f", sum / n * 8 ; }') >> reports/$OUT_DIR/pcie.rpt - echo "IOTLB_hits: " $(cat logs/$OUT_DIR/pcie.csv | grep "Socket0,IIO Stack 2 - PCIe1,Part0" | awk -F ',' '{ sum += $8; n++ } END { if (n > 0) printf "%0.3f", sum / n; }') >> reports/$OUT_DIR/pcie.rpt - echo "IOTLB_misses: " $(cat logs/$OUT_DIR/pcie.csv | grep "Socket0,IIO Stack 2 - PCIe1,Part0" | awk -F ',' '{ sum += $9; n++ } END { if (n > 0) printf "%0.3f", sum / n; }') >> reports/$OUT_DIR/pcie.rpt - echo "CTXT_Miss: " $(cat logs/$OUT_DIR/pcie.csv | grep "Socket0,IIO Stack 2 - PCIe1,Part0" | awk -F ',' '{ sum += $10; n++ } END { if (n > 0) printf "%0.3f", sum / n; }') >> reports/$OUT_DIR/pcie.rpt - echo "L1_Miss: " $(cat logs/$OUT_DIR/pcie.csv | grep "Socket0,IIO Stack 2 - PCIe1,Part0" | awk -F ',' '{ sum += $11; n++ } END { if (n > 0) printf "%0.3f", sum / n; }') >> reports/$OUT_DIR/pcie.rpt - echo "L2_Miss: " $(cat logs/$OUT_DIR/pcie.csv | grep "Socket0,IIO Stack 2 - PCIe1,Part0" | awk -F ',' '{ sum += $12; n++ } END { if (n > 0) printf "%0.3f", sum / n; }') >> reports/$OUT_DIR/pcie.rpt - echo "L3_Miss: " $(cat logs/$OUT_DIR/pcie.csv | grep "Socket0,IIO Stack 2 - PCIe1,Part0" | awk -F ',' '{ sum += $13; n++ } END { if (n > 0) printf "%0.3f", sum / n; }') >> reports/$OUT_DIR/pcie.rpt - echo "Mem_Read: " $(cat logs/$OUT_DIR/pcie.csv | grep "Socket0,IIO Stack 2 - PCIe1,Part0" | awk -F ',' '{ sum += $14; n++ } END { if (n > 0) printf "%0.3f", sum / n; }') >> reports/$OUT_DIR/pcie.rpt + echo "PCIe_wr_tput: " $(cat logs/$OUT_DIR/pcie.csv | grep "$PCIE_PATTERN" | awk -F ',' '{ sum += $4/1000000000.0; n++ } END { if (n > 0) printf "%.3f", sum / n * 8 ; }') > reports/$OUT_DIR/pcie.rpt + echo "PCIe_rd_tput: " $(cat logs/$OUT_DIR/pcie.csv | grep "$PCIE_PATTERN" | awk -F ',' '{ sum += $5/1000000000.0; n++ } END { if (n > 0) printf "%0.3f", sum / n * 8 ; }') >> reports/$OUT_DIR/pcie.rpt + echo "IOTLB_hits: " $(cat logs/$OUT_DIR/pcie.csv | grep "$PCIE_PATTERN" | awk -F ',' '{ sum += $8; n++ } END { if (n > 0) printf "%0.3f", sum / n; }') >> reports/$OUT_DIR/pcie.rpt + echo "IOTLB_misses: " $(cat logs/$OUT_DIR/pcie.csv | grep "$PCIE_PATTERN" | awk -F ',' '{ sum += $9; n++ } END { if (n > 0) printf "%0.3f", sum / n; }') >> reports/$OUT_DIR/pcie.rpt + echo "CTXT_Miss: " $(cat logs/$OUT_DIR/pcie.csv | grep "$PCIE_PATTERN" | awk -F ',' '{ sum += $10; n++ } END { if (n > 0) printf "%0.3f", sum / n; }') >> reports/$OUT_DIR/pcie.rpt + echo "L1_Miss: " $(cat logs/$OUT_DIR/pcie.csv | grep "$PCIE_PATTERN" | awk -F ',' '{ sum += $11; n++ } END { if (n > 0) printf "%0.3f", sum / n; }') >> reports/$OUT_DIR/pcie.rpt + echo "L2_Miss: " $(cat logs/$OUT_DIR/pcie.csv | grep "$PCIE_PATTERN" | awk -F ',' '{ sum += $12; n++ } END { if (n > 0) printf "%0.3f", sum / n; }') >> reports/$OUT_DIR/pcie.rpt + echo "L3_Miss: " $(cat logs/$OUT_DIR/pcie.csv | grep "$PCIE_PATTERN" | awk -F ',' '{ sum += $13; n++ } END { if (n > 0) printf "%0.3f", sum / n; }') >> reports/$OUT_DIR/pcie.rpt + echo "Mem_Read: " $(cat logs/$OUT_DIR/pcie.csv | grep "$PCIE_PATTERN" | awk -F ',' '{ sum += $14; n++ } END { if (n > 0) printf "%0.3f", sum / n; }') >> reports/$OUT_DIR/pcie.rpt } function dump_membw() { From 6de35aa16d11e0413a15a8a6a393238c51beb60a Mon Sep 17 00:00:00 2001 From: Leshna Date: Tue, 24 Jun 2025 03:04:28 -0600 Subject: [PATCH 12/12] fix --- utils/record-host-metrics.sh | 18 +++++++++--------- utils/tcp/run-netapp-tput.sh | 10 ++++------ 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/utils/record-host-metrics.sh b/utils/record-host-metrics.sh index 05197e0be..2ab36a7e4 100644 --- a/utils/record-host-metrics.sh +++ b/utils/record-host-metrics.sh @@ -100,15 +100,15 @@ function dump_pciebw() { function parse_pciebw() { #TODO: make more general, parse PCIe bandwidth for any given socket and IIO stack - echo "PCIe_wr_tput: " $(cat logs/$OUT_DIR/pcie.csv | grep "$PCIE_PATTERN" | awk -F ',' '{ sum += $4/1000000000.0; n++ } END { if (n > 0) printf "%.3f", sum / n * 8 ; }') > reports/$OUT_DIR/pcie.rpt - echo "PCIe_rd_tput: " $(cat logs/$OUT_DIR/pcie.csv | grep "$PCIE_PATTERN" | awk -F ',' '{ sum += $5/1000000000.0; n++ } END { if (n > 0) printf "%0.3f", sum / n * 8 ; }') >> reports/$OUT_DIR/pcie.rpt - echo "IOTLB_hits: " $(cat logs/$OUT_DIR/pcie.csv | grep "$PCIE_PATTERN" | awk -F ',' '{ sum += $8; n++ } END { if (n > 0) printf "%0.3f", sum / n; }') >> reports/$OUT_DIR/pcie.rpt - echo "IOTLB_misses: " $(cat logs/$OUT_DIR/pcie.csv | grep "$PCIE_PATTERN" | awk -F ',' '{ sum += $9; n++ } END { if (n > 0) printf "%0.3f", sum / n; }') >> reports/$OUT_DIR/pcie.rpt - echo "CTXT_Miss: " $(cat logs/$OUT_DIR/pcie.csv | grep "$PCIE_PATTERN" | awk -F ',' '{ sum += $10; n++ } END { if (n > 0) printf "%0.3f", sum / n; }') >> reports/$OUT_DIR/pcie.rpt - echo "L1_Miss: " $(cat logs/$OUT_DIR/pcie.csv | grep "$PCIE_PATTERN" | awk -F ',' '{ sum += $11; n++ } END { if (n > 0) printf "%0.3f", sum / n; }') >> reports/$OUT_DIR/pcie.rpt - echo "L2_Miss: " $(cat logs/$OUT_DIR/pcie.csv | grep "$PCIE_PATTERN" | awk -F ',' '{ sum += $12; n++ } END { if (n > 0) printf "%0.3f", sum / n; }') >> reports/$OUT_DIR/pcie.rpt - echo "L3_Miss: " $(cat logs/$OUT_DIR/pcie.csv | grep "$PCIE_PATTERN" | awk -F ',' '{ sum += $13; n++ } END { if (n > 0) printf "%0.3f", sum / n; }') >> reports/$OUT_DIR/pcie.rpt - echo "Mem_Read: " $(cat logs/$OUT_DIR/pcie.csv | grep "$PCIE_PATTERN" | awk -F ',' '{ sum += $14; n++ } END { if (n > 0) printf "%0.3f", sum / n; }') >> reports/$OUT_DIR/pcie.rpt + echo "PCIe_wr_tput: " $(cat logs/$OUT_DIR/pcie.csv | grep "$PCIE_PATTERN" | awk -F ',' '{ sum += $6/1000000000.0; n++ } END { if (n > 0) printf "%.3f", sum / n * 8 ; }') > reports/$OUT_DIR/pcie.rpt + echo "PCIe_rd_tput: " $(cat logs/$OUT_DIR/pcie.csv | grep "$PCIE_PATTERN" | awk -F ',' '{ sum += $7/1000000000.0; n++ } END { if (n > 0) printf "%0.3f", sum / n * 8 ; }') >> reports/$OUT_DIR/pcie.rpt + echo "IOTLB_hits: " $(cat logs/$OUT_DIR/pcie.csv | grep "$PCIE_PATTERN" | awk -F ',' '{ sum += $10; n++ } END { if (n > 0) printf "%0.3f", sum / n; }') >> reports/$OUT_DIR/pcie.rpt + echo "IOTLB_misses: " $(cat logs/$OUT_DIR/pcie.csv | grep "$PCIE_PATTERN" | awk -F ',' '{ sum += $11; n++ } END { if (n > 0) printf "%0.3f", sum / n; }') >> reports/$OUT_DIR/pcie.rpt + echo "CTXT_Miss: " $(cat logs/$OUT_DIR/pcie.csv | grep "$PCIE_PATTERN" | awk -F ',' '{ sum += $12; n++ } END { if (n > 0) printf "%0.3f", sum / n; }') >> reports/$OUT_DIR/pcie.rpt + echo "L1_Miss: " $(cat logs/$OUT_DIR/pcie.csv | grep "$PCIE_PATTERN" | awk -F ',' '{ sum += $13; n++ } END { if (n > 0) printf "%0.3f", sum / n; }') >> reports/$OUT_DIR/pcie.rpt + echo "L2_Miss: " $(cat logs/$OUT_DIR/pcie.csv | grep "$PCIE_PATTERN" | awk -F ',' '{ sum += $14; n++ } END { if (n > 0) printf "%0.3f", sum / n; }') >> reports/$OUT_DIR/pcie.rpt + echo "L3_Miss: " $(cat logs/$OUT_DIR/pcie.csv | grep "$PCIE_PATTERN" | awk -F ',' '{ sum += $15; n++ } END { if (n > 0) printf "%0.3f", sum / n; }') >> reports/$OUT_DIR/pcie.rpt + echo "Mem_Read: " $(cat logs/$OUT_DIR/pcie.csv | grep "$PCIE_PATTERN" | awk -F ',' '{ sum += $16; n++ } END { if (n > 0) printf "%0.3f", sum / n; }') >> reports/$OUT_DIR/pcie.rpt } function dump_membw() { diff --git a/utils/tcp/run-netapp-tput.sh b/utils/tcp/run-netapp-tput.sh index 90b2c4661..cd308a892 100644 --- a/utils/tcp/run-netapp-tput.sh +++ b/utils/tcp/run-netapp-tput.sh @@ -25,7 +25,7 @@ help() exit 2 } -SHORT=o:,n:,N:,p:,c:,b:h +SHORT=o:,n:,N:,p:,c:,b:,h LONG=mode:,outdir:,num_servers:,num_clients:,port:,server-ip:,cores:,bandwidth:,help OPTS=$(getopt -a -n run-netapp-tput --options $SHORT --longoptions $LONG -- "$@") VALID_ARGUMENTS=$# # Returns the count of arguments that are in short or long options @@ -56,7 +56,7 @@ mkdir -p ../reports #Directory to store collected logs mkdir -p ../reports/$OUT_DIR #Directory to store collected logs mkdir -p ../logs #Directory to store collected logs mkdir -p ../logs/$OUT_DIR #Directory to store collected logs -rm ../logs/$OUT_DIR/iperf.bw.log +rm -f ../logs/$OUT_DIR/iperf.bw.log function collect_stats() { echo "Collecting app throughput for TCP server..." @@ -64,8 +64,7 @@ function collect_stats() { } counter=0 -if [ "$mode" = "server" ] -then +if [ "$MODE" = "server" ]; then sudo pkill -9 -f iperf #kill existing iperf servers/clients while [ $counter -lt $NUM_SERVERS ]; do index=$(( counter % ${#core_values[@]} )) @@ -78,8 +77,7 @@ then sleep 120 echo "collecting stats..." collect_stats -elif [ "$mode" = "client" ] -then +elif [ "$MODE" = "client" ]; then sudo pkill -9 -f iperf #kill existing iperf servers/clients while [ $counter -lt $NUM_CLIENTS ]; do index=$(( counter % ${#core_values[@]} ))