1#!/usr/bin/env python3 2# Copyright (c) PLUMgrid, Inc. 3# Licensed under the Apache License, Version 2.0 (the "License") 4 5# This program implements a topology likes below: 6# pem: physical endpoint manager, implemented as a bpf program 7# 8# vm1 <--------+ +----> bridge1 <----+ 9# V V V 10# pem router 11# ^ ^ ^ 12# vm2 <--------+ +----> bridge2 <----+ 13# 14# The vm1, vm2 and router are implemented as namespaces. 15# The bridge is implemented with limited functionality in bpf program. 16# 17# vm1 and vm2 are in different subnet. For vm1 to communicate to vm2, 18# the packet will have to travel from vm1 to pem, bridge1, router, bridge2, pem, and 19# then come to vm2. 20# 21# When this test is run with verbose mode (ctest -R <test_name> -V), 22# the following printout is observed on my local box: 23# 24# ...... 25# 8: ARPING 100.1.1.254 from 100.1.1.1 eth0 26# 8: Unicast reply from 100.1.1.254 [76:62:B5:5C:8C:6F] 0.533ms 27# 8: Sent 1 probes (1 broadcast(s)) 28# 8: Received 1 response(s) 29# 8: ARPING 200.1.1.254 from 200.1.1.1 eth0 30# 8: Unicast reply from 200.1.1.254 [F2:F0:B4:ED:7B:1B] 0.524ms 31# 8: Sent 1 probes (1 broadcast(s)) 32# 8: Received 1 response(s) 33# 8: PING 200.1.1.1 (200.1.1.1) 56(84) bytes of data. 34# 8: 64 bytes from 200.1.1.1: icmp_req=1 ttl=63 time=0.074 ms 35# 8: 64 bytes from 200.1.1.1: icmp_req=2 ttl=63 time=0.061 ms 36# 8: 37# 8: --- 200.1.1.1 ping statistics --- 38# 8: 2 packets transmitted, 2 received, 0% packet loss, time 999ms 39# 8: rtt min/avg/max/mdev = 0.061/0.067/0.074/0.010 ms 40# 8: [ ID] Interval Transfer Bandwidth 41# 8: [ 5] 0.0- 1.0 sec 4.00 GBytes 34.3 Gbits/sec 42# 8: Starting netserver with host 'IN(6)ADDR_ANY' port '12865' and family AF_UNSPEC 43# 8: MIGRATED TCP STREAM TEST from 0.0.0.0 (0.0.0.0) port 0 AF_INET to 200.1.1.1 (200.1.1.1) port 0 AF_INET : demo 44# 8: Recv Send Send 45# 8: Socket Socket Message Elapsed 46# 8: Size Size Size Time Throughput 47# 8: bytes bytes bytes secs. 10^6bits/sec 48# 8: 49# 8: 87380 16384 65160 1.00 41991.68 50# 8: MIGRATED TCP REQUEST/RESPONSE TEST from 0.0.0.0 (0.0.0.0) port 0 AF_INET to 200.1.1.1 (200.1.1.1) port 0 AF_INET : demo : first burst 0 51# 8: Local /Remote 52# 8: Socket Size Request Resp. Elapsed Trans. 53# 8: Send Recv Size Size Time Rate 54# 8: bytes Bytes bytes bytes secs. per sec 55# 8: 56# 8: 16384 87380 1 1 1.00 48645.53 57# 8: 16384 87380 58# 8: . 59# 8: ---------------------------------------------------------------------- 60# 8: Ran 1 test in 11.296s 61# 8: 62# 8: OK 63 64from ctypes import c_uint 65from netaddr import IPAddress, EUI 66from bcc import BPF 67from pyroute2 import IPRoute, NetNS, IPDB, NSPopen 68from utils import NSPopenWithCheck, skipUnlessHasBinaries 69import sys 70from time import sleep 71from unittest import main, TestCase 72from simulation import Simulation 73 74arg1 = sys.argv.pop(1) 75ipr = IPRoute() 76ipdb = IPDB(nl=ipr) 77sim = Simulation(ipdb) 78 79class TestBPFSocket(TestCase): 80 def set_default_const(self): 81 self.ns1 = "ns1" 82 self.ns2 = "ns2" 83 self.ns_router = "ns_router" 84 self.vm1_ip = "100.1.1.1" 85 self.vm2_ip = "200.1.1.1" 86 self.vm1_rtr_ip = "100.1.1.254" 87 self.vm2_rtr_ip = "200.1.1.254" 88 self.vm1_rtr_mask = "100.1.1.0/24" 89 self.vm2_rtr_mask = "200.1.1.0/24" 90 91 def get_table(self, b): 92 self.jump = b.get_table(b"jump") 93 94 self.pem_dest = b.get_table(b"pem_dest") 95 self.pem_port = b.get_table(b"pem_port") 96 self.pem_ifindex = b.get_table(b"pem_ifindex") 97 self.pem_stats = b.get_table(b"pem_stats") 98 99 self.br1_dest = b.get_table(b"br1_dest") 100 self.br1_mac = b.get_table(b"br1_mac") 101 self.br1_rtr = b.get_table(b"br1_rtr") 102 103 self.br2_dest = b.get_table(b"br2_dest") 104 self.br2_mac = b.get_table(b"br2_mac") 105 self.br2_rtr = b.get_table(b"br2_rtr") 106 107 def connect_ports(self, prog_id_pem, prog_id_br, curr_pem_pid, curr_br_pid, 108 br_dest_map, br_mac_map, ifindex, vm_mac, vm_ip): 109 self.pem_dest[c_uint(curr_pem_pid)] = self.pem_dest.Leaf(prog_id_br, curr_br_pid) 110 br_dest_map[c_uint(curr_br_pid)] = br_dest_map.Leaf(prog_id_pem, curr_pem_pid) 111 self.pem_port[c_uint(curr_pem_pid)] = c_uint(ifindex) 112 self.pem_ifindex[c_uint(ifindex)] = c_uint(curr_pem_pid) 113 mac_addr = br_mac_map.Key(int(EUI(vm_mac))) 114 br_mac_map[mac_addr] = c_uint(curr_br_pid) 115 116 def config_maps(self): 117 # program id 118 prog_id_pem = 1 119 prog_id_br1 = 2 120 prog_id_br2 = 3 121 122 # initial port id and table pointers 123 curr_pem_pid = 0 124 curr_br1_pid = 0 125 curr_br2_pid = 0 126 127 # configure jump table 128 self.jump[c_uint(prog_id_pem)] = c_uint(self.pem_fn.fd) 129 self.jump[c_uint(prog_id_br1)] = c_uint(self.br1_fn.fd) 130 self.jump[c_uint(prog_id_br2)] = c_uint(self.br2_fn.fd) 131 132 # connect pem and br1 133 curr_pem_pid = curr_pem_pid + 1 134 curr_br1_pid = curr_br1_pid + 1 135 self.connect_ports(prog_id_pem, prog_id_br1, curr_pem_pid, curr_br1_pid, 136 self.br1_dest, self.br1_mac, 137 self.ns1_eth_out.index, self.vm1_mac, self.vm1_ip) 138 139 # connect pem and br2 140 curr_pem_pid = curr_pem_pid + 1 141 curr_br2_pid = curr_br2_pid + 1 142 self.connect_ports(prog_id_pem, prog_id_br2, curr_pem_pid, curr_br2_pid, 143 self.br2_dest, self.br2_mac, 144 self.ns2_eth_out.index, self.vm2_mac, self.vm2_ip) 145 146 # connect <br1, rtr> and <br2, rtr> 147 self.br1_rtr[c_uint(0)] = c_uint(self.nsrtr_eth0_out.index) 148 self.br2_rtr[c_uint(0)] = c_uint(self.nsrtr_eth1_out.index) 149 150 @skipUnlessHasBinaries( 151 ["arping", "iperf", "netperf", "netserver", "ping"], 152 "iperf and netperf packages must be installed.") 153 def test_brb(self): 154 try: 155 b = BPF(src_file=arg1.encode(), debug=0) 156 self.pem_fn = b.load_func(b"pem", BPF.SCHED_CLS) 157 self.br1_fn = b.load_func(b"br1", BPF.SCHED_CLS) 158 self.br2_fn = b.load_func(b"br2", BPF.SCHED_CLS) 159 self.get_table(b) 160 161 # set up the topology 162 self.set_default_const() 163 (ns1_ipdb, self.ns1_eth_out, _) = sim._create_ns(self.ns1, ipaddr=self.vm1_ip+'/24', 164 fn=self.pem_fn, action='drop', 165 disable_ipv6=True) 166 (ns2_ipdb, self.ns2_eth_out, _) = sim._create_ns(self.ns2, ipaddr=self.vm2_ip+'/24', 167 fn=self.pem_fn, action='drop', 168 disable_ipv6=True) 169 ns1_ipdb.routes.add({'dst': self.vm2_rtr_mask, 'gateway': self.vm1_rtr_ip}).commit() 170 ns2_ipdb.routes.add({'dst': self.vm1_rtr_mask, 'gateway': self.vm2_rtr_ip}).commit() 171 self.vm1_mac = ns1_ipdb.interfaces['eth0'].address 172 self.vm2_mac = ns2_ipdb.interfaces['eth0'].address 173 174 (_, self.nsrtr_eth0_out, _) = sim._create_ns(self.ns_router, ipaddr=self.vm1_rtr_ip+'/24', 175 fn=self.br1_fn, action='drop', 176 disable_ipv6=True) 177 (rt_ipdb, self.nsrtr_eth1_out, _) = sim._ns_add_ifc(self.ns_router, "eth1", "ns_router2", 178 ipaddr=self.vm2_rtr_ip+'/24', 179 fn=self.br2_fn, action='drop', 180 disable_ipv6=True) 181 nsp = NSPopen(rt_ipdb.nl.netns, ["sysctl", "-w", "net.ipv4.ip_forward=1"]) 182 nsp.wait(); nsp.release() 183 184 # configure maps 185 self.config_maps() 186 187 # our bridge is not smart enough, so send arping for router learning to prevent router 188 # from sending out arp request 189 nsp = NSPopen(ns1_ipdb.nl.netns, 190 ["arping", "-w", "1", "-c", "1", "-I", "eth0", self.vm1_rtr_ip]) 191 nsp.wait(); nsp.release() 192 nsp = NSPopen(ns2_ipdb.nl.netns, 193 ["arping", "-w", "1", "-c", "1", "-I", "eth0", self.vm2_rtr_ip]) 194 nsp.wait(); nsp.release() 195 196 # ping 197 nsp = NSPopen(ns1_ipdb.nl.netns, ["ping", self.vm2_ip, "-c", "2"]) 198 nsp.wait(); nsp.release() 199 # pem_stats only counts pem->bridge traffic, each VM has 4: arping/arp request/2 icmp request 200 # total 8 packets should be counted 201 self.assertEqual(self.pem_stats[c_uint(0)].value, 8) 202 203 nsp_server = NSPopenWithCheck(ns2_ipdb.nl.netns, ["iperf", "-s", "-xSC"]) 204 sleep(1) 205 nsp = NSPopen(ns1_ipdb.nl.netns, ["iperf", "-c", self.vm2_ip, "-t", "1", "-xSC"]) 206 nsp.wait(); nsp.release() 207 nsp_server.kill(); nsp_server.wait(); nsp_server.release() 208 209 nsp_server = NSPopenWithCheck(ns2_ipdb.nl.netns, ["netserver", "-D"]) 210 sleep(1) 211 nsp = NSPopenWithCheck(ns1_ipdb.nl.netns, ["netperf", "-l", "1", "-H", self.vm2_ip, "--", "-m", "65160"]) 212 nsp.wait(); nsp.release() 213 nsp = NSPopen(ns1_ipdb.nl.netns, ["netperf", "-l", "1", "-H", self.vm2_ip, "-t", "TCP_RR"]) 214 nsp.wait(); nsp.release() 215 nsp_server.kill(); nsp_server.wait(); nsp_server.release() 216 217 finally: 218 sim.release() 219 ipdb.release() 220 221 222if __name__ == "__main__": 223 main() 224