Figure 5

Bursts with #32-batching, Snort-filter (VM)

low-latency/figures/burst-32batch/vm/pcap/latencies-pre-rate10000-snort-filter.pdf.svg

Bursts with #4-batching, Snort-filter (VM)

low-latency/figures/burst-4batch/vm/pcap/latencies-pre-rate10000-snort-filter.pdf.svg

Steps to reproduce measurements

Images

Images used for testing

Loadgen, timestamper

Linux machine 4.9.0-8-amd64 #1 SMP Debian 4.9.110-3+deb9u6 (2018-10-08) x86_64 GNU/Linux

Device under test (VM host and VM)

Linux machine 4.19.0-4-rt-amd64 #1 SMP PREEMPT RT Debian 4.19.28-2 (2019-03-15) x86_64 GNU/Linux

Evaluator

Linux machine 4.19.12 #1 SMP Fri Feb 1 21:28:58 CET 2019 x86_64 GNU/Linux

Setup

Setup loadgen

git clone https://github.com/gallenmu/MoonGen /root/moongen
cd /root/moongen
git checkout 0450d0f84ac900843031b61a9f28a8194add147d 
/root/moongen/build.sh
/root/moongen/bind-interfaces.sh
/root/moongen/setup-hugetlbfs.sh

Configuration VM: setup device under test (DuT) for Snort-filter:

Boot parameter VM host

boot=live nosoftlockup isolcpus=1,2,3 intel_idle.max_cstate=0 intel_pstate=disable BOOTIF=01-ac-1f-6b-7c-87-b0 skew_tick=1 irqaffinity=0 intel_iommu=on noprompt lvm console=ttyS0,115200 apparmor=0 noeject vga=791 dhcp audit=0 rcu_nocbs=1,2,3 mce=ignore_ce nmi_watchdog=0 random.trust_cpu=on hostname=machine tsc=reliable idle=poll

Setup VM host

echo 'deb http://deb.debian.org/debian experimental main contrib non-free
deb-src http://deb.debian.org/debian experimental main contrib non-free' >> /etc/apt/sources.list

apt update
apt install -t experimental -y virt-manager qemu-system pkg-config libvirt-dev python-libvirt

# bind rcu processes to core 0
# recommended by https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/7/html-single/performance_tuning_guide/#sect-Red_Hat_Enterprise_Linux-Performance_Tuning_Guide-Configuration_suggestions-Configuring_kernel_tick_time
for i in `pgrep rcu[^c]` ; do taskset -pc 0 $i ; done

# activate virtual functions
rmmod ixgbe
modprobe ixgbe max_vfs=1

ip link set up dev eno7
ip link set up dev eno8

# disable ipv6 to avoid pf chatting on vf connection
sysctl net.ipv6.conf.eno7.disable_ipv6=1
sysctl net.ipv6.conf.eno8.disable_ipv6=1

# download network config
git clone https://github.com/gallenmu/low-latency /root/low-latency

# mgmt interface of VM
virsh net-define /root/low-latency/configs/vm/net.xml
virsh net-start net

# measurement interfaces of VM
virsh net-define /root/low-latency/configs/vm/passthrough7.xml
virsh net-define /root/low-latency/configs/vm/passthrough8.xml
virsh net-start passthrough7
virsh net-start passthrough8

virt-install --cpu host-passthrough --memory 16384 --vcpus=3 --cpuset=1-3 --boot=network \
--name vm --nodisks --network="network=net,mac=11:22:33:44:55:66,model=virtio" \
--network="network=net7,mac=52:54:00:8d:9d:ad,model=virtio" \
--network="network=net8,mac=52:54:00:21:f8:29,model=virtio" \
--noautoconsole --graphics none --dry-run --controller="type=usb,model=none" --print-xml \
--console "pty,target_type=virtio" \
--cputune="vcpupin0.cpuset=1,vcpupin0.vcpu=0,vcpupin1.cpuset=2,vcpupin1.vcpu=1,vcpupin2.cpuset=3,vcpupin2.vcpu=2" > vm.xml

virsh define vm.xml
virsh start vm

Cache partitioning VM host

apt install ethtool intel-cmt-cat

# needed for cat tool
modprobe msr

pqos -R
pqos -a "llc:1=2;llc:2=0-1,3"
pqos -e "llc:1=0xff;llc:2=0xf00"
pqos -s

Boot parameter VM

boot=live irqaffinity=0 isolcpus=1,2 intel_pstate=disable BOOTIF=01-52-54-00-6d-00-02 skew_tick=1 nosoftlockup noprompt lvm intel_idle.max_cstate=0 console=ttyS0,115200 noeject vga=791 dhcp audit=0 rcu_nocbs=1,2 mce=ignore_ce nmi_watchdog=0 random.trust_cpu=on idle=poll hostname=vm tsc=reliable

Var A: Setup VM Snort #32 batching

apt-get install -y build-essential \
        autotools-dev libdumbnet-dev libluajit-5.1-dev libpcap-dev \
        libpcre3-dev zlib1g-dev pkg-config libhwloc-dev cmake liblzma-dev \
        openssl libssl-dev cpputest libsqlite3-dev libtool git autoconf \
        bison flex asciidoc source-highlight \
        libgoogle-perftools-dev

# bind rcu processes to core 0
# recommended by https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/7/html-single/performance_tuning_guide/#sect-Red_Hat_Enterprise_Linux-Performance_Tuning_Guide-Configuration_suggestions-Configuring_kernel_tick_time
for i in `pgrep rcu[^c]` ; do taskset -pc 0 $i ; done

# disable rt throttling
# recommended https://wiki.opnfv.org/display/kvm/Nfv-kvm-tuning
echo -1 > /proc/sys/kernel/sched_rt_runtime_us

# set virt memory stats collector 3600s interval (default 1s)
# recommended by https://access.redhat.com/sites/default/files/attachments/201501-perf-brief-low-latency-tuning-rhel7-v1.1.pdf
sysctl vm.stat_interval=3600

# download config data
git clone https://github.com/gallenmu/low-latency /root/low-latency

# setup dpdk for daq
git clone https://github.com/gallenmu/napadpdk /root/napadpdk
cd /root/napadpdk
git checkout f819cd087d48332060b15da5afee27cb5a9a508e
make config T=x86_64-native-linuxapp-gcc
cd build
make -j 3
mv /root/napadpdk/build /root/napadpdk/x86_64-native-linuxapp-gcc
export RTE_SDK=/root/napadpdk

# build l2fwd
export PQOS_INSTALL_PATH=/usr/lib
cd /root/napadpdk/examples/l2fwd-cat
make

# dpdk_config: load driver, bind X552 interfaces, allocate hugepages
modprobe uio
insmod /root/napadpdk/x86_64-native-linuxapp-gcc/kmod/igb_uio.ko
for id in $(/root/napadpdk/usertools/dpdk-devbind.py --status | grep -v Active | grep X552 | cut -f 1 -d " ")
do
        echo "Binding interface $id to DPDK"
        /root/napadpdk/usertools/dpdk-devbind.py  --bind=igb_uio $id
done
mkdir -p /mnt/huge
mount -t hugetlbfs nodev /mnt/huge
echo 512 > /sys/devices/system/node/node0/hugepages/hugepages-2048kB/nr_hugepages

# build daq
git clone https://github.com/gallenmu/daq_dpdk_multiqueue /root/daq
cd /root/daq
git checkout 5b25e4192b10858cbce434f46f5c8abec1360661 # batched processing #32
cd /root/daq/daq-2.2.2
autoreconf -f -i
./configure --disable-dump-module --disable-ipfw-module --disable-ipq-module --disable-nfq-module --disable-pcap-module --disable-netmap-module --disable-afpacket-module --enable-static --disable-shared --prefix=/opt/snort
make -j 3
make install

# build snort
export PATH=$PATH:/opt/snort/bin/
git clone https://github.com/gallenmu/snort-3.0.0-beta /root/snort-3.0.0
cd /root/snort-3.0.0
./configure_cmake.sh --prefix=/opt/snort --with-daq-libraries=/opt/snort/lib --with-daq-includes=/opt/snort/include --enable-static-daq --disable-docs
cd build
make -j 3
make install

Var B: Setup VM Snort #4 batching

apt-get install -y build-essential \
        autotools-dev libdumbnet-dev libluajit-5.1-dev libpcap-dev \
        libpcre3-dev zlib1g-dev pkg-config libhwloc-dev cmake liblzma-dev \
        openssl libssl-dev cpputest libsqlite3-dev libtool git autoconf \
        bison flex asciidoc source-highlight \
        libgoogle-perftools-dev

# bind rcu processes to core 0
# recommended by https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/7/html-single/performance_tuning_guide/#sect-Red_Hat_Enterprise_Linux-Performance_Tuning_Guide-Configuration_suggestions-Configuring_kernel_tick_time
for i in `pgrep rcu[^c]` ; do taskset -pc 0 $i ; done

# disable rt throttling
# recommended https://wiki.opnfv.org/display/kvm/Nfv-kvm-tuning
echo -1 > /proc/sys/kernel/sched_rt_runtime_us

# set virt memory stats collector 3600s interval (default 1s)
# recommended by https://access.redhat.com/sites/default/files/attachments/201501-perf-brief-low-latency-tuning-rhel7-v1.1.pdf
sysctl vm.stat_interval=3600

# download config data
git clone https://github.com/gallenmu/low-latency /root/low-latency

# setup dpdk for daq
git clone https://github.com/gallenmu/napadpdk /root/napadpdk
cd /root/napadpdk
git checkout f819cd087d48332060b15da5afee27cb5a9a508e
make config T=x86_64-native-linuxapp-gcc
cd build
make -j 3
mv /root/napadpdk/build /root/napadpdk/x86_64-native-linuxapp-gcc
export RTE_SDK=/root/napadpdk

# build l2fwd
export PQOS_INSTALL_PATH=/usr/lib
cd /root/napadpdk/examples/l2fwd-cat
make

# dpdk_config: load driver, bind X552 interfaces, allocate hugepages
modprobe uio
insmod /root/napadpdk/x86_64-native-linuxapp-gcc/kmod/igb_uio.ko
for id in $(/root/napadpdk/usertools/dpdk-devbind.py --status | grep -v Active | grep X552 | cut -f 1 -d " ")
do
        echo "Binding interface $id to DPDK"
        /root/napadpdk/usertools/dpdk-devbind.py  --bind=igb_uio $id
done
mkdir -p /mnt/huge
mount -t hugetlbfs nodev /mnt/huge
echo 512 > /sys/devices/system/node/node0/hugepages/hugepages-2048kB/nr_hugepages

# build daq
git clone https://github.com/gallenmu/daq_dpdk_multiqueue /root/daq
cd /root/daq
git checkout 24a9bf33e52e90c03c90a406cb31d62fedc54422 # batched processing #4
cd /root/daq/daq-2.2.2
autoreconf -f -i
./configure --disable-dump-module --disable-ipfw-module --disable-ipq-module --disable-nfq-module --disable-pcap-module --disable-netmap-module --disable-afpacket-module --enable-static --disable-shared --prefix=/opt/snort
make -j 3
make install

# build snort
export PATH=$PATH:/opt/snort/bin/
git clone https://github.com/gallenmu/snort-3.0.0-beta /root/snort-3.0.0
cd /root/snort-3.0.0
./configure_cmake.sh --prefix=/opt/snort --with-daq-libraries=/opt/snort/lib --with-daq-includes=/opt/snort/include --enable-static-daq --disable-docs
cd build
make -j 3
make install

Setup timestamper

git clone https://github.com/gallenmu/MoonGen /root/moongen
cd /root/moongen
git checkout 0450d0f84ac900843031b61a9f28a8194add147d 
/root/moongen/build.sh
/root/moongen/bind-interfaces.sh
/root/moongen/setup-hugetlbfs.sh

Experiment execution

1. Start Snort-filter on the DuT:

Snort-filter

taskset -c 1 /opt/snort/bin/snort -c "/root/low-latency/configs/snort/snort-communityrules.lua" --daq-dir /opt/snort/lib/daq --daq dpdk --daq-var dpdk_argc="-n1" -i "dpdk0:dpdk1" -Q -z 1

2. Wait until forwarder has been started, then start MoonGen on the LoadGen:

Start MoonGen

# $RATE is the number of generated packets per second
# $PACKETS is the number of packets to be generated
# $BURST amount of packets in a burst on the wire
/root/moongen/build/MoonGen /root/moongen/examples/moonsniff/traffic-gen.lua --src-mac 52:54:00:8d:9d:ad --dst-mac 52:54:00:21:f8:29 --fix-packetrate $RATE --packets $PACKETS --warm-up 30 --l4-dst 53 --flows 10 --burst $BURST 1 0

3. After Moongen on the LoadGen has been started 1000 packets are sent to warm up the DuT. After that packet generation is paused for 30 seconds. In this pause MoonGen's packet sniffer on the timestamper should be started to record the actual measurement:

Start MoonGen

/root/moongen/build/MoonGen /root/moongen/examples/moonsniff/sniffer.lua 1 0 --capture --time 60 --snaplen 256

4. The timestamper stops recording after 60 seconds and creates the two pcap files latencies-pre.pcap and latencies-post.pcap which can be used for evaluation.

Steps to reproduce figures

Raw data

Pcaps (Snort-filter #32-batch (VM), Snort-filter #4-batch (VM))

Evaluation execution

Setup evaluator

# install dependencies
apt install postgresql postgresql-client parallel python3-pip texlive-full zstd
python3 -m pip install pypacker

Generate figures

git clone https://github.com/gallenmu/low-latency /root/low-latency

#decompress pcaps
cd /root/low-latency/measurements
for i in $(find . -iname *.zst); do zstd --rm -d $i; done;

#create figures for batch size #32
cd /root/low-latency/figures/burst-32batch/vm/pcap
su postgres -c "createuser -s root"
dropdb --if-exists root
createdb root
export PGDATABASE=root
for i in /root/low-latency/measurements/burst-32batch/vm/pcap/*-pre-*.pcap; do /root/low-latency/scripts/pcap-import.sh $i; /root/low-latency/scripts/pcap-analysis.sh $i; done;
python3 /root/low-latency/scripts/burst-tex-create.py .
for i in *.tex; do lualatex $i; done

#create figures for batch size #4
cd /root/low-latency/figures/burst-4batch/vm/pcap
su postgres -c "createuser -s root"
dropdb --if-exists root
createdb root
export PGDATABASE=root
for i in /root/low-latency/measurements/burst-4batch/vm/pcap/*-pre-*.pcap; do /root/low-latency/scripts/pcap-import.sh $i; /root/low-latency/scripts/pcap-analysis.sh $i; done;
python3 /root/low-latency/scripts/burst-tex-create.py .
for i in *.tex; do lualatex $i; done

Output

The precompiled output files can be found in the respective subfolders the burst-32batch folder and burst-4batch folder.