1#!/usr/bin/env python3 2# Copyright (c) PLUMgrid, Inc. 3# Licensed under the Apache License, Version 2.0 (the "License") 4 5# This program implements a topology likes below: 6# pem: physical endpoint manager, implemented as a bpf program 7# 8# vm1 <--------+ +----> bridge1 <----+ 9# V V V 10# pem router 11# ^ ^ ^ 12# vm2 <--------+ +----> bridge2 <----+ 13# 14# The vm1, vm2 and router are implemented as namespaces. 15# The linux bridge device is used to provice bridge functionality. 16# pem bpf will be attached to related network devices for vm1, vm1, bridge1 and bridge2. 17# 18# vm1 and vm2 are in different subnet. For vm1 to communicate to vm2, 19# the packet will have to travel from vm1 to pem, bridge1, router, bridge2, pem, and 20# then come to vm2. 21# 22# When this test is run with verbose mode (ctest -R <test_name> -V), 23# the following printout is observed on my local box: 24# 25# ...... 26# 9: PING 200.1.1.1 (200.1.1.1) 56(84) bytes of data. 27# 9: 64 bytes from 200.1.1.1: icmp_req=1 ttl=63 time=0.090 ms 28# 9: 64 bytes from 200.1.1.1: icmp_req=2 ttl=63 time=0.032 ms 29# 9: 30# 9: --- 200.1.1.1 ping statistics --- 31# 9: 2 packets transmitted, 2 received, 0% packet loss, time 999ms 32# 9: rtt min/avg/max/mdev = 0.032/0.061/0.090/0.029 ms 33# 9: [ ID] Interval Transfer Bandwidth 34# 9: [ 5] 0.0- 1.0 sec 3.80 GBytes 32.6 Gbits/sec 35# 9: Starting netserver with host 'IN(6)ADDR_ANY' port '12865' and family AF_UNSPEC 36# 9: MIGRATED TCP STREAM TEST from 0.0.0.0 (0.0.0.0) port 0 AF_INET to 200.1.1.1 (200.1.1.1) port 0 AF_INET : demo 37# 9: Recv Send Send 38# 9: Socket Socket Message Elapsed 39# 9: Size Size Size Time Throughput 40# 9: bytes bytes bytes secs. 10^6bits/sec 41# 9: 42# 9: 87380 16384 65160 1.00 39940.46 43# 9: MIGRATED TCP REQUEST/RESPONSE TEST from 0.0.0.0 (0.0.0.0) port 0 AF_INET to 200.1.1.1 (200.1.1.1) port 0 AF_INET : demo : first burst 0 44# 9: Local /Remote 45# 9: Socket Size Request Resp. Elapsed Trans. 46# 9: Send Recv Size Size Time Rate 47# 9: bytes Bytes bytes bytes secs. per sec 48# 9: 49# 9: 16384 87380 1 1 1.00 46387.80 50# 9: 16384 87380 51# 9: . 52# 9: ---------------------------------------------------------------------- 53# 9: Ran 1 test in 7.495s 54# 9: 55# 9: OK 56 57from ctypes import c_uint 58from bcc import BPF 59from pyroute2 import IPRoute, NetNS, IPDB, NSPopen 60from utils import NSPopenWithCheck, mayFail 61import sys 62from time import sleep 63from unittest import main, TestCase 64import subprocess 65from simulation import Simulation 66 67arg1 = sys.argv.pop(1) 68ipr = IPRoute() 69ipdb = IPDB(nl=ipr) 70sim = Simulation(ipdb) 71 72allocated_interfaces = set(ipdb.interfaces.keys()) 73 74def get_next_iface(prefix): 75 i = 0 76 while True: 77 iface = "{0}{1}".format(prefix, i) 78 if iface not in allocated_interfaces: 79 allocated_interfaces.add(iface) 80 return iface 81 i += 1 82 83class TestBPFSocket(TestCase): 84 def setup_br(self, br, veth_rt_2_br, veth_pem_2_br, veth_br_2_pem): 85 # create veth which connecting pem and br 86 with ipdb.create(ifname=veth_pem_2_br, kind="veth", peer=veth_br_2_pem) as v: 87 v.up() 88 ipdb.interfaces[veth_br_2_pem].up().commit() 89 subprocess.call(["sysctl", "-q", "-w", "net.ipv6.conf." + veth_pem_2_br + ".disable_ipv6=1"]) 90 subprocess.call(["sysctl", "-q", "-w", "net.ipv6.conf." + veth_br_2_pem + ".disable_ipv6=1"]) 91 92 # set up the bridge and add router interface as one of its slaves 93 with ipdb.create(ifname=br, kind="bridge") as br1: 94 br1.add_port(ipdb.interfaces[veth_pem_2_br]) 95 br1.add_port(ipdb.interfaces[veth_rt_2_br]) 96 br1.up() 97 subprocess.call(["sysctl", "-q", "-w", "net.ipv6.conf." + br + ".disable_ipv6=1"]) 98 99 def set_default_const(self): 100 self.ns1 = "ns1" 101 self.ns2 = "ns2" 102 self.ns_router = "ns_router" 103 self.br1 = get_next_iface("br") 104 self.veth_pem_2_br1 = "v20" 105 self.veth_br1_2_pem = "v21" 106 self.br2 = get_next_iface("br") 107 self.veth_pem_2_br2 = "v22" 108 self.veth_br2_2_pem = "v23" 109 110 self.vm1_ip = "100.1.1.1" 111 self.vm2_ip = "200.1.1.1" 112 self.vm1_rtr_ip = "100.1.1.254" 113 self.vm2_rtr_ip = "200.1.1.254" 114 self.vm1_rtr_mask = "100.1.1.0/24" 115 self.vm2_rtr_mask = "200.1.1.0/24" 116 117 def attach_filter(self, ifname, fd, name): 118 ifindex = ipdb.interfaces[ifname].index 119 ipr.tc("add", "ingress", ifindex, "ffff:") 120 ipr.tc("add-filter", "bpf", ifindex, ":1", fd=fd, name=name, 121 parent="ffff:", action="drop", classid=1) 122 123 def config_maps(self): 124 # pem just relays packets between VM and its corresponding 125 # slave link in the bridge interface 126 ns1_ifindex = self.ns1_eth_out.index 127 ns2_ifindex = self.ns2_eth_out.index 128 br1_ifindex = ipdb.interfaces[self.veth_br1_2_pem].index 129 br2_ifindex = ipdb.interfaces[self.veth_br2_2_pem].index 130 self.pem_dest[c_uint(ns1_ifindex)] = c_uint(br1_ifindex) 131 self.pem_dest[c_uint(br1_ifindex)] = c_uint(ns1_ifindex) 132 self.pem_dest[c_uint(ns2_ifindex)] = c_uint(br2_ifindex) 133 self.pem_dest[c_uint(br2_ifindex)] = c_uint(ns2_ifindex) 134 135 # tc filter setup with bpf programs attached 136 self.attach_filter(self.veth_br1_2_pem, self.pem_fn.fd, self.pem_fn.name) 137 self.attach_filter(self.veth_br2_2_pem, self.pem_fn.fd, self.pem_fn.name) 138 139 @mayFail("This fails on github actions environment, and needs to be fixed") 140 def test_brb2(self): 141 try: 142 b = BPF(src_file=arg1.encode(), debug=0) 143 self.pem_fn = b.load_func(b"pem", BPF.SCHED_CLS) 144 self.pem_dest= b.get_table(b"pem_dest") 145 self.pem_stats = b.get_table(b"pem_stats") 146 147 # set up the topology 148 self.set_default_const() 149 (ns1_ipdb, self.ns1_eth_out, _) = sim._create_ns(self.ns1, ipaddr=self.vm1_ip+'/24', 150 fn=self.pem_fn, action='drop', 151 disable_ipv6=True) 152 (ns2_ipdb, self.ns2_eth_out, _) = sim._create_ns(self.ns2, ipaddr=self.vm2_ip+'/24', 153 fn=self.pem_fn, action='drop', 154 disable_ipv6=True) 155 ns1_ipdb.routes.add({'dst': self.vm2_rtr_mask, 'gateway': self.vm1_rtr_ip}).commit() 156 ns2_ipdb.routes.add({'dst': self.vm1_rtr_mask, 'gateway': self.vm2_rtr_ip}).commit() 157 158 (_, self.nsrtr_eth0_out, _) = sim._create_ns(self.ns_router, ipaddr=self.vm1_rtr_ip+'/24', 159 disable_ipv6=True) 160 (rt_ipdb, self.nsrtr_eth1_out, _) = sim._ns_add_ifc(self.ns_router, "eth1", "ns_router2", 161 ipaddr=self.vm2_rtr_ip+'/24', 162 disable_ipv6=True) 163 # enable ip forwarding in router ns 164 nsp = NSPopen(rt_ipdb.nl.netns, ["sysctl", "-w", "net.ipv4.ip_forward=1"]) 165 nsp.wait(); nsp.release() 166 167 # for each VM connecting to pem, there will be a corresponding veth connecting to the bridge 168 self.setup_br(self.br1, self.nsrtr_eth0_out.ifname, self.veth_pem_2_br1, self.veth_br1_2_pem) 169 self.setup_br(self.br2, self.nsrtr_eth1_out.ifname, self.veth_pem_2_br2, self.veth_br2_2_pem) 170 171 # load the program and configure maps 172 self.config_maps() 173 174 # ping 175 nsp = NSPopen(ns1_ipdb.nl.netns, ["ping", self.vm2_ip, "-c", "2"]); nsp.wait(); nsp.release() 176 # one arp request/reply, 2 icmp request/reply per VM, total 6 packets per VM, 12 packets total 177 self.assertEqual(self.pem_stats[c_uint(0)].value, 12) 178 179 nsp_server = NSPopenWithCheck(ns2_ipdb.nl.netns, ["iperf", "-s", "-xSC"]) 180 sleep(1) 181 nsp = NSPopen(ns1_ipdb.nl.netns, ["iperf", "-c", self.vm2_ip, "-t", "1", "-xSC"]) 182 nsp.wait(); nsp.release() 183 nsp_server.kill(); nsp_server.wait(); nsp_server.release() 184 185 nsp_server = NSPopenWithCheck(ns2_ipdb.nl.netns, ["netserver", "-D"]) 186 sleep(1) 187 nsp = NSPopenWithCheck(ns1_ipdb.nl.netns, ["netperf", "-l", "1", "-H", self.vm2_ip, "--", "-m", "65160"]) 188 nsp.wait(); nsp.release() 189 nsp = NSPopen(ns1_ipdb.nl.netns, ["netperf", "-l", "1", "-H", self.vm2_ip, "-t", "TCP_RR"]) 190 nsp.wait(); nsp.release() 191 nsp_server.kill(); nsp_server.wait(); nsp_server.release() 192 193 finally: 194 if self.br1 in ipdb.interfaces: ipdb.interfaces[self.br1].remove().commit() 195 if self.br2 in ipdb.interfaces: ipdb.interfaces[self.br2].remove().commit() 196 if self.veth_pem_2_br1 in ipdb.interfaces: ipdb.interfaces[self.veth_pem_2_br1].remove().commit() 197 if self.veth_pem_2_br2 in ipdb.interfaces: ipdb.interfaces[self.veth_pem_2_br2].remove().commit() 198 sim.release() 199 ipdb.release() 200 201 202if __name__ == "__main__": 203 main() 204