1 /*
2  * Copyright (C) 2019 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 // The resulting .o needs to load on Android T+
18 #define BPFLOADER_MIN_VER BPFLOADER_MAINLINE_T_VERSION
19 
20 #include "bpf_net_helpers.h"
21 #include "clatd.h"
22 #include "clat_mark.h"
23 
24 DEFINE_BPF_MAP_GRW(clat_ingress6_map, HASH, ClatIngress6Key, ClatIngress6Value, 16, AID_SYSTEM)
25 
nat64(struct __sk_buff * skb,const struct rawip_bool rawip,const struct kver_uint kver)26 static inline __always_inline int nat64(struct __sk_buff* skb,
27                                         const struct rawip_bool rawip,
28                                         const struct kver_uint kver) {
29     const bool is_ethernet = !rawip.rawip;
30 
31     // Require ethernet dst mac address to be our unicast address.
32     if (is_ethernet && (skb->pkt_type != PACKET_HOST)) return TC_ACT_PIPE;
33 
34     // Must be meta-ethernet IPv6 frame
35     if (skb->protocol != htons(ETH_P_IPV6)) return TC_ACT_PIPE;
36 
37     const int l2_header_size = is_ethernet ? sizeof(struct ethhdr) : 0;
38 
39     // Not clear if this is actually necessary considering we use DPA (Direct Packet Access),
40     // but we need to make sure we can read the IPv6 header reliably so that we can set
41     // skb->mark = 0xDeadC1a7 for packets we fail to offload.
42     try_make_writable(skb, l2_header_size + sizeof(struct ipv6hdr));
43 
44     void* data = (void*)(long)skb->data;
45     const void* data_end = (void*)(long)skb->data_end;
46     const struct ethhdr* const eth = is_ethernet ? data : NULL;  // used iff is_ethernet
47     const struct ipv6hdr* const ip6 = is_ethernet ? (void*)(eth + 1) : data;
48 
49     // Must have (ethernet and) ipv6 header
50     if (data + l2_header_size + sizeof(*ip6) > data_end) return TC_ACT_PIPE;
51 
52     // Ethertype - if present - must be IPv6
53     if (is_ethernet && (eth->h_proto != htons(ETH_P_IPV6))) return TC_ACT_PIPE;
54 
55     // IP version must be 6
56     if (ip6->version != 6) return TC_ACT_PIPE;
57 
58     // Maximum IPv6 payload length that can be translated to IPv4
59     // Note: technically this check is too strict for an IPv6 fragment,
60     // which by virtue of stripping the extra 8 byte fragment extension header,
61     // could thus be 8 bytes larger and still fit in an ipv4 packet post
62     // translation.  However... who ever heard of receiving ~64KB frags...
63     // fragments are kind of by definition smaller than ingress device mtu,
64     // and thus, on the internet, very very unlikely to exceed 1500 bytes.
65     if (ntohs(ip6->payload_len) > 0xFFFF - sizeof(struct iphdr)) return TC_ACT_PIPE;
66 
67     ClatIngress6Key k = {
68             .iif = skb->ifindex,
69             .pfx96.in6_u.u6_addr32 =
70                     {
71                             ip6->saddr.in6_u.u6_addr32[0],
72                             ip6->saddr.in6_u.u6_addr32[1],
73                             ip6->saddr.in6_u.u6_addr32[2],
74                     },
75             .local6 = ip6->daddr,
76     };
77 
78     ClatIngress6Value* v = bpf_clat_ingress6_map_lookup_elem(&k);
79 
80     if (!v) return TC_ACT_PIPE;
81 
82     __u8 proto = ip6->nexthdr;
83     __be16 ip_id = 0;
84     __be16 frag_off = htons(IP_DF);
85     __u16 tot_len = ntohs(ip6->payload_len) + sizeof(struct iphdr);  // cannot overflow, see above
86 
87     if (proto == IPPROTO_FRAGMENT) {
88         // Fragment handling requires bpf_skb_adjust_room which is 4.14+
89         if (!KVER_IS_AT_LEAST(kver, 4, 14, 0)) return TC_ACT_PIPE;
90 
91         // Must have (ethernet and) ipv6 header and ipv6 fragment extension header
92         if (data + l2_header_size + sizeof(*ip6) + sizeof(struct frag_hdr) > data_end)
93             return TC_ACT_PIPE;
94         const struct frag_hdr *frag = (const struct frag_hdr *)(ip6 + 1);
95         proto = frag->nexthdr;
96         // RFC6145: use bottom 16-bits of network endian 32-bit IPv6 ID field for 16-bit IPv4 field.
97         // this is equivalent to: ip_id = htons(ntohl(frag->identification));
98         ip_id = frag->identification >> 16;
99         // Conversion of 16-bit IPv6 frag offset to 16-bit IPv4 frag offset field.
100         // IPv6 is '13 bits of offset in multiples of 8' + 2 zero bits + more fragment bit
101         // IPv4 is zero bit + don't frag bit + more frag bit + '13 bits of offset in multiples of 8'
102         frag_off = ntohs(frag->frag_off);
103         frag_off = ((frag_off & 1) << 13) | (frag_off >> 3);
104         frag_off = htons(frag_off);
105         // Note that by construction tot_len is guaranteed to not underflow here
106         tot_len -= sizeof(struct frag_hdr);
107         // This is a badly formed IPv6 packet with less payload than the size of an IPv6 Frag EH
108         if (tot_len < sizeof(struct iphdr)) return TC_ACT_PIPE;
109     }
110 
111     switch (proto) {
112         case IPPROTO_TCP:      // For TCP, UDP & UDPLITE the checksum neutrality of the chosen
113         case IPPROTO_UDP:      // IPv6 address means there is no need to update their checksums.
114         case IPPROTO_UDPLITE:  //
115         case IPPROTO_GRE:      // We do not need to bother looking at GRE/ESP headers,
116         case IPPROTO_ESP:      // since there is never a checksum to update.
117             break;
118 
119         default:  // do not know how to handle anything else
120             // Mark ingress non-offloaded clat packet for dropping in ip6tables bw_raw_PREROUTING.
121             // Non-offloaded clat packet is going to be handled by clat daemon and ip6tables. The
122             // duplicate one in ip6tables is not necessary.
123             skb->mark = CLAT_MARK;
124             return TC_ACT_PIPE;
125     }
126 
127     struct ethhdr eth2;  // used iff is_ethernet
128     if (is_ethernet) {
129         eth2 = *eth;                     // Copy over the ethernet header (src/dst mac)
130         eth2.h_proto = htons(ETH_P_IP);  // But replace the ethertype
131     }
132 
133     struct iphdr ip = {
134             .version = 4,                                                      // u4
135             .ihl = sizeof(struct iphdr) / sizeof(__u32),                       // u4
136             .tos = (ip6->priority << 4) + (ip6->flow_lbl[0] >> 4),             // u8
137             .tot_len = htons(tot_len),                                         // be16
138             .id = ip_id,                                                       // be16
139             .frag_off = frag_off,                                              // be16
140             .ttl = ip6->hop_limit,                                             // u8
141             .protocol = proto,                                                 // u8
142             .check = 0,                                                        // u16
143             .saddr = ip6->saddr.in6_u.u6_addr32[3],                            // be32
144             .daddr = v->local4.s_addr,                                         // be32
145     };
146 
147     // Calculate the IPv4 one's complement checksum of the IPv4 header.
148     __wsum sum4 = 0;
149     for (unsigned i = 0; i < sizeof(ip) / sizeof(__u16); ++i) {
150         sum4 += ((__u16*)&ip)[i];
151     }
152     // Note that sum4 is guaranteed to be non-zero by virtue of ip.version == 4
153     sum4 = (sum4 & 0xFFFF) + (sum4 >> 16);  // collapse u32 into range 1 .. 0x1FFFE
154     sum4 = (sum4 & 0xFFFF) + (sum4 >> 16);  // collapse any potential carry into u16
155     ip.check = (__u16)~sum4;                // sum4 cannot be zero, so this is never 0xFFFF
156 
157     // Calculate the *negative* IPv6 16-bit one's complement checksum of the IPv6 header.
158     __wsum sum6 = 0;
159     // We'll end up with a non-zero sum due to ip6->version == 6 (which has '0' bits)
160     for (unsigned i = 0; i < sizeof(*ip6) / sizeof(__u16); ++i) {
161         sum6 += ~((__u16*)ip6)[i];  // note the bitwise negation
162     }
163 
164     // Note that there is no L4 checksum update: we are relying on the checksum neutrality
165     // of the ipv6 address chosen by netd's ClatdController.
166 
167     // Packet mutations begin - point of no return, but if this first modification fails
168     // the packet is probably still pristine, so let clatd handle it.
169     if (bpf_skb_change_proto(skb, htons(ETH_P_IP), 0)) {
170         // Mark ingress non-offloaded clat packet for dropping in ip6tables bw_raw_PREROUTING.
171         // Non-offloaded clat packet is going to be handled by clat daemon and ip6tables. The
172         // duplicate one in ip6tables is not necessary.
173         skb->mark = CLAT_MARK;
174         return TC_ACT_PIPE;
175     }
176 
177     // This takes care of updating the skb->csum field for a CHECKSUM_COMPLETE packet.
178     //
179     // In such a case, skb->csum is a 16-bit one's complement sum of the entire payload,
180     // thus we need to subtract out the ipv6 header's sum, and add in the ipv4 header's sum.
181     // However, by construction of ip.check above the checksum of an ipv4 header is zero.
182     // Thus we only need to subtract the ipv6 header's sum, which is the same as adding
183     // in the sum of the bitwise negation of the ipv6 header.
184     //
185     // bpf_csum_update() always succeeds if the skb is CHECKSUM_COMPLETE and returns an error
186     // (-ENOTSUPP) if it isn't.  So we just ignore the return code.
187     //
188     // if (skb->ip_summed == CHECKSUM_COMPLETE)
189     //   return (skb->csum = csum_add(skb->csum, csum));
190     // else
191     //   return -ENOTSUPP;
192     bpf_csum_update(skb, sum6);
193 
194     // Technically 'kver < KVER_4_14' already implies 'frag_off == htons(IP_DF)' due to logic above,
195     // thus the initial 'kver >= KVER_4_14' check here is entirely superfluous.
196     //
197     // However, we *need* the compiler (when compiling the program for 4.9) to entirely
198     // optimize out the call to bpf_skb_adjust_room() bpf helper: it's not enough for it to emit
199     // an unreachable call to it, it must *not* emit it at all (otherwise the 4.9 kernel's
200     // bpf verifier will refuse to load a program with an unknown bpf helper call)
201     //
202     // This is easiest to achieve by being very explicit in the if clause,
203     // better safe than sorry...
204     //
205     // Note: we currently have no TreeHugger coverage for 4.9-T devices (there are no such
206     // Pixel or cuttlefish devices), so likely you won't notice for months if this breaks...
207     if (KVER_IS_AT_LEAST(kver, 4, 14, 0) && frag_off != htons(IP_DF)) {
208         // If we're converting an IPv6 Fragment, we need to trim off 8 more bytes
209         // We're beyond recovery on error here... but hard to imagine how this could fail.
210         if (bpf_skb_adjust_room(skb, -(__s32)sizeof(struct frag_hdr), BPF_ADJ_ROOM_NET, /*flags*/0))
211             return TC_ACT_SHOT;
212     }
213 
214     try_make_writable(skb, l2_header_size + sizeof(struct iphdr));
215 
216     // bpf_skb_change_proto() invalidates all pointers - reload them.
217     data = (void*)(long)skb->data;
218     data_end = (void*)(long)skb->data_end;
219 
220     // I cannot think of any valid way for this error condition to trigger, however I do
221     // believe the explicit check is required to keep the in kernel ebpf verifier happy.
222     if (data + l2_header_size + sizeof(struct iphdr) > data_end) return TC_ACT_SHOT;
223 
224     if (is_ethernet) {
225         struct ethhdr* new_eth = data;
226 
227         // Copy over the updated ethernet header
228         *new_eth = eth2;
229 
230         // Copy over the new ipv4 header.
231         *(struct iphdr*)(new_eth + 1) = ip;
232     } else {
233         // Copy over the new ipv4 header without an ethernet header.
234         *(struct iphdr*)data = ip;
235     }
236 
237     // Count successfully translated packet
238     __sync_fetch_and_add(&v->packets, 1);
239     __sync_fetch_and_add(&v->bytes, skb->len - l2_header_size);
240 
241     // Redirect, possibly back to same interface, so tcpdump sees packet twice.
242     if (v->oif) return bpf_redirect(v->oif, BPF_F_INGRESS);
243 
244     // Just let it through, tcpdump will not see IPv4 packet.
245     return TC_ACT_PIPE;
246 }
247 
248 DEFINE_BPF_PROG_KVER("schedcls/ingress6/clat_ether$4_14", AID_ROOT, AID_SYSTEM, sched_cls_ingress6_clat_ether_4_14, KVER_4_14)
249 (struct __sk_buff* skb) {
250     return nat64(skb, ETHER, KVER_4_14);
251 }
252 
253 DEFINE_BPF_PROG_KVER_RANGE("schedcls/ingress6/clat_ether$4_9", AID_ROOT, AID_SYSTEM, sched_cls_ingress6_clat_ether_4_9, KVER_NONE, KVER_4_14)
254 (struct __sk_buff* skb) {
255     return nat64(skb, ETHER, KVER_NONE);
256 }
257 
258 DEFINE_BPF_PROG_KVER("schedcls/ingress6/clat_rawip$4_14", AID_ROOT, AID_SYSTEM, sched_cls_ingress6_clat_rawip_4_14, KVER_4_14)
259 (struct __sk_buff* skb) {
260     return nat64(skb, RAWIP, KVER_4_14);
261 }
262 
263 DEFINE_BPF_PROG_KVER_RANGE("schedcls/ingress6/clat_rawip$4_9", AID_ROOT, AID_SYSTEM, sched_cls_ingress6_clat_rawip_4_9, KVER_NONE, KVER_4_14)
264 (struct __sk_buff* skb) {
265     return nat64(skb, RAWIP, KVER_NONE);
266 }
267 
268 DEFINE_BPF_MAP_GRW(clat_egress4_map, HASH, ClatEgress4Key, ClatEgress4Value, 16, AID_SYSTEM)
269 
270 DEFINE_BPF_PROG("schedcls/egress4/clat_rawip", AID_ROOT, AID_SYSTEM, sched_cls_egress4_clat_rawip)
271 (struct __sk_buff* skb) {
272     // Must be meta-ethernet IPv4 frame
273     if (skb->protocol != htons(ETH_P_IP)) return TC_ACT_PIPE;
274 
275     // Possibly not needed, but for consistency with nat64 up above
276     try_make_writable(skb, sizeof(struct iphdr));
277 
278     void* data = (void*)(long)skb->data;
279     const void* data_end = (void*)(long)skb->data_end;
280     const struct iphdr* const ip4 = data;
281 
282     // Must have ipv4 header
283     if (data + sizeof(*ip4) > data_end) return TC_ACT_PIPE;
284 
285     // IP version must be 4
286     if (ip4->version != 4) return TC_ACT_PIPE;
287 
288     // We cannot handle IP options, just standard 20 byte == 5 dword minimal IPv4 header
289     if (ip4->ihl != 5) return TC_ACT_PIPE;
290 
291     // Calculate the IPv4 one's complement checksum of the IPv4 header.
292     __wsum sum4 = 0;
293     for (unsigned i = 0; i < sizeof(*ip4) / sizeof(__u16); ++i) {
294         sum4 += ((__u16*)ip4)[i];
295     }
296     // Note that sum4 is guaranteed to be non-zero by virtue of ip4->version == 4
297     sum4 = (sum4 & 0xFFFF) + (sum4 >> 16);  // collapse u32 into range 1 .. 0x1FFFE
298     sum4 = (sum4 & 0xFFFF) + (sum4 >> 16);  // collapse any potential carry into u16
299     // for a correct checksum we should get *a* zero, but sum4 must be positive, ie 0xFFFF
300     if (sum4 != 0xFFFF) return TC_ACT_PIPE;
301 
302     // Minimum IPv4 total length is the size of the header
303     if (ntohs(ip4->tot_len) < sizeof(*ip4)) return TC_ACT_PIPE;
304 
305     // We are incapable of dealing with IPv4 fragments
306     if (ip4->frag_off & ~htons(IP_DF)) return TC_ACT_PIPE;
307 
308     switch (ip4->protocol) {
309         case IPPROTO_TCP:      // For TCP, UDP & UDPLITE the checksum neutrality of the chosen
310         case IPPROTO_UDPLITE:  // IPv6 address means there is no need to update their checksums.
311         case IPPROTO_GRE:      // We do not need to bother looking at GRE/ESP headers,
312         case IPPROTO_ESP:      // since there is never a checksum to update.
313             break;
314 
315         case IPPROTO_UDP:      // See above comment, but must also have UDP header...
316             if (data + sizeof(*ip4) + sizeof(struct udphdr) > data_end) return TC_ACT_PIPE;
317             const struct udphdr* uh = (const struct udphdr*)(ip4 + 1);
318             // If IPv4/UDP checksum is 0 then fallback to clatd so it can calculate the
319             // checksum.  Otherwise the network or more likely the NAT64 gateway might
320             // drop the packet because in most cases IPv6/UDP packets with a zero checksum
321             // are invalid. See RFC 6935.  TODO: calculate checksum via bpf_csum_diff()
322             if (!uh->check) return TC_ACT_PIPE;
323             break;
324 
325         default:  // do not know how to handle anything else
326             return TC_ACT_PIPE;
327     }
328 
329     ClatEgress4Key k = {
330             .iif = skb->ifindex,
331             .local4.s_addr = ip4->saddr,
332     };
333 
334     ClatEgress4Value* v = bpf_clat_egress4_map_lookup_elem(&k);
335 
336     if (!v) return TC_ACT_PIPE;
337 
338     // Translating without redirecting doesn't make sense.
339     if (!v->oif) return TC_ACT_PIPE;
340 
341     // This implementation is currently limited to rawip.
342     if (v->oifIsEthernet) return TC_ACT_PIPE;
343 
344     struct ipv6hdr ip6 = {
345             .version = 6,                                    // __u8:4
346             .priority = ip4->tos >> 4,                       // __u8:4
347             .flow_lbl = {(ip4->tos & 0xF) << 4, 0, 0},       // __u8[3]
348             .payload_len = htons(ntohs(ip4->tot_len) - 20),  // __be16
349             .nexthdr = ip4->protocol,                        // __u8
350             .hop_limit = ip4->ttl,                           // __u8
351             .saddr = v->local6,                              // struct in6_addr
352             .daddr = v->pfx96,                               // struct in6_addr
353     };
354     ip6.daddr.in6_u.u6_addr32[3] = ip4->daddr;
355 
356     // Calculate the IPv6 16-bit one's complement checksum of the IPv6 header.
357     __wsum sum6 = 0;
358     // We'll end up with a non-zero sum due to ip6.version == 6
359     for (unsigned i = 0; i < sizeof(ip6) / sizeof(__u16); ++i) {
360         sum6 += ((__u16*)&ip6)[i];
361     }
362 
363     // Note that there is no L4 checksum update: we are relying on the checksum neutrality
364     // of the ipv6 address chosen by netd's ClatdController.
365 
366     // Packet mutations begin - point of no return, but if this first modification fails
367     // the packet is probably still pristine, so let clatd handle it.
368     if (bpf_skb_change_proto(skb, htons(ETH_P_IPV6), 0)) return TC_ACT_PIPE;
369 
370     // This takes care of updating the skb->csum field for a CHECKSUM_COMPLETE packet.
371     //
372     // In such a case, skb->csum is a 16-bit one's complement sum of the entire payload,
373     // thus we need to subtract out the ipv4 header's sum, and add in the ipv6 header's sum.
374     // However, we've already verified the ipv4 checksum is correct and thus 0.
375     // Thus we only need to add the ipv6 header's sum.
376     //
377     // bpf_csum_update() always succeeds if the skb is CHECKSUM_COMPLETE and returns an error
378     // (-ENOTSUPP) if it isn't.  So we just ignore the return code (see above for more details).
379     bpf_csum_update(skb, sum6);
380 
381     // bpf_skb_change_proto() invalidates all pointers - reload them.
382     data = (void*)(long)skb->data;
383     data_end = (void*)(long)skb->data_end;
384 
385     // I cannot think of any valid way for this error condition to trigger, however I do
386     // believe the explicit check is required to keep the in kernel ebpf verifier happy.
387     if (data + sizeof(ip6) > data_end) return TC_ACT_SHOT;
388 
389     // Copy over the new ipv6 header without an ethernet header.
390     *(struct ipv6hdr*)data = ip6;
391 
392     // Count successfully translated packet
393     __sync_fetch_and_add(&v->packets, 1);
394     __sync_fetch_and_add(&v->bytes, skb->len);
395 
396     // Redirect to non v4-* interface.  Tcpdump only sees packet after this redirect.
397     return bpf_redirect(v->oif, 0 /* this is effectively BPF_F_EGRESS */);
398 }
399 
400 LICENSE("Apache 2.0");
401 CRITICAL("Connectivity");
402