Hi folks,
let me preface this by saying that I know I am combining a wild amount
of things in here, but sometimes that is where things take you.
So the scenario:
I'm using squid and tproxy to transparently proxy HTTP traffic
I am also using tun2socks to send this HTTP traffic to an upstream proxy
I'm using policy based routing to determine which traffic to send to tun2socks
The individual components are all working independently but in
combination it does not. Specifically, as can be seen in the table
below, it seems the routing decision somehow changes between the inet
route output hook and the uip nat postrouting hook.
I'm increasingly convinced that I am somehow triggering some kind of
super weird bug, but I'm more than happy to find out that I am doing
things wrong.
Any help would be greatly appreciated!
Kind regards
Nicolai Moore
nftables rules being used:
table inet filter {
set internet_adapters {
type ifname
elements = { "ens224",
"tunU-ens224" }
}
set internet_ipv4_addrs {
type ipv4_addr
elements = { 10.34.1.121, 203.0.113.2 }
}
set internet_ipv4_networks {
type ipv4_addr
flags interval
auto-merge
elements = { 10.34.0.0/16, 203.0.113.2 }
}
set local_adapters {
type ifname
elements = { "ens256" }
}
set local_ipv4_addrs {
type ipv4_addr
elements = { 10.111.1.1 }
}
set local_ipv4_networks {
type ipv4_addr
flags interval
auto-merge
elements = { 10.111.1.0/24 }
}
chain prerouting {
type filter hook prerouting priority mangle; policy accept;
ct state new ip saddr 10.111.1.2 meta
nftrace set 1
jump pakete-prerouting
}
chain input {
type filter hook input priority security; policy drop;
iif "lo" accept
ip protocol icmp accept
ip6 nexthdr ipv6-icmp accept
meta mark & 0x00010000 != 0x00010000 ct state
established,related accept
meta mark & 0x00010000 != 0x00010000 accept
meta mark & 0x00010000 == 0x00010000 jump pakete-input
}
chain forward {
type filter hook forward priority security; policy drop;
jump pakete-forward
}
chain output {
type filter hook output priority security; policy accept;
meta mark & 0x00020000 == 0x00020000 jump pakete-output
}
chain pakete-prerouting {
counter packets 3743 bytes 1577550
meta mark set ct mark & 0xfff4fff7
meta l4proto tcp socket transparent 1
meta mark set meta mark or 0x10000
meta mark set meta mark or 0x6
iifname @internet_adapters ip daddr @internet_ipv4_networks accept
iifname @local_adapters ip daddr @local_ipv4_networks accept
iifname @local_adapters tcp dport 80 meta
mark set meta mark | 0x00010000 tproxy to :31080 accept
}
chain pakete-output {
counter packets 0 bytes 0
meta mark & 0x00000002 == 0x00000002 counter packets 0
bytes 0 accept
meta mark & 0x00000001 == 0x00000001 counter packets 0
bytes 0 accept
counter packets 0 bytes 0 reject
}
chain pakete-input {
counter packets 0 bytes 0
ct mark set meta mark
meta mark & 0x00000002 == 0x00000002 counter packets 0
bytes 0 accept
meta mark & 0x00000001 == 0x00000001 counter packets 0
bytes 0 accept
counter packets 0 bytes 0 drop
}
chain pakete-forward {
counter packets 0 bytes 0
ct mark set meta mark
meta mark & 0x00000002 == 0x00000002 counter packets 0
bytes 0 accept
counter packets 0 bytes 0 drop
}
}
table inet route {
chain output {
type route hook output priority mangle; policy accept;
meta mark set ct mark & 0xfff4fff7
ct state new ip saddr 10.111.1.2 meta nftrace set 1
}
}
table ip nat {
set internet_adapters {
type ifname
elements = { "ens224",
"tunU-ens224" }
}
set local_ipv4_networks {
type ipv4_addr
flags interval
auto-merge
elements = { 10.111.1.0/24 }
}
chain postrouting {
type nat hook postrouting priority srcnat; policy accept;
jump connectionconf-postrouting
}
chain connectionconf-postrouting {
counter packets 60 bytes 3892
ip saddr @local_ipv4_networks oifname @internet_adapters
counter packets 0 bytes 0 masquerade
}
}
ip rule
0: from all lookup local
31900: from all fwmark 0x10000/0x10000 lookup squid_tproxy
32000: from all lookup main
32766: from all ipproto tcp dport 80 lookup upstream_proxy_default
32767: from all lookup default
nft monitor trace output from curling http://1.1.1.1
trace id 34202808 inet filter prerouting packet: iif "ens256" ether
saddr 00:50:56:a1:22:4e ether daddr 00:50:56:a1:21:94 ip saddr
10.111.1.2 ip daddr 1.1.1.1 ip dscp cs0 ip ecn not-ect ip ttl 64 ip id
3223 ip protocol tcp ip length 60 tcp sport 50952 tcp dport 80 tcp
flags == syn tcp window 29200
trace id 34202808 inet filter prerouting rule ct state new ip saddr
10.111.1.2 meta nftrace set 1 (verdict continue)
trace id 34202808 inet filter prerouting rule jump pakete-prerouting
(verdict jump pakete-prerouting)
trace id 34202808 inet filter pakete-prerouting rule counter packets
4054 bytes 1626001 (verdict continue)
trace id 34202808 inet filter pakete-prerouting rule meta mark set ct
mark & 0xfff4fff7 (verdict continue)
trace id 34202808 inet filter pakete-prerouting rule meta mark set
meta mark | 0x00000006 (verdict continue)
trace id 34202808 inet filter pakete-prerouting rule iifname
@local_adapters tcp dport 80 meta mark set meta mark | 0x00010000
tproxy to :31080 counter packets 0 bytes 0 accept (verdict accept)
trace id 34202808 inet filter input packet: iif "ens256" ether saddr
00:50:56:a1:22:4e ether daddr 00:50:56:a1:21:94 ip saddr 10.111.1.2 ip
daddr 1.1.1.1 ip dscp cs0 ip ecn not-ect ip ttl 64 ip id 3223 ip
protocol tcp ip length 60 tcp sport 50952 tcp dport 80 tcp flags ==
syn tcp window 29200
trace id 34202808 inet filter input rule meta mark & 0x00010000 ==
0x00010000 jump pakete-input (verdict jump pakete-input)
trace id 34202808 inet filter pakete-input rule counter packets 0
bytes 0 (verdict continue)
trace id 34202808 inet filter pakete-input rule ct mark set meta mark
(verdict continue)
trace id 34202808 inet filter pakete-input rule meta mark & 0x00000002
== 0x00000002 counter packets 0 bytes 0 accept (verdict accept)
trace id 3368ad3d inet route output packet: oif "tunU-ens224" ip saddr
10.111.1.2 ip daddr 1.1.1.1 ip dscp cs0 ip ecn not-ect ip ttl 64 ip id
41644 ip protocol tcp ip length 60 tcp sport 40955 tcp dport 80 tcp
flags == syn tcp window 64240
trace id 3368ad3d inet route output rule ct state new ip saddr
10.111.1.2 meta nftrace set 1 (verdict continue)
trace id 3368ad3d inet route output verdict continue
trace id 3368ad3d inet route output policy accept
trace id 3368ad3d inet filter output packet: oif "tunU-ens224" ip
saddr 10.111.1.2 ip daddr 1.1.1.1 ip dscp cs0 ip ecn not-ect ip ttl 64
ip id 41644 ip protocol tcp ip length 60 tcp sport 40955 tcp dport 80
tcp flags == syn tcp window 64240
trace id 3368ad3d inet filter output verdict continue
trace id 3368ad3d inet filter output policy accept
trace id 3368ad3d ip nat postrouting packet: oif "ens224" ip saddr
10.111.1.2 ip daddr 1.1.1.1 ip dscp cs0 ip ecn not-ect ip ttl 64 ip id
41644 ip length 60 tcp sport 40955 tcp dport 80 tcp flags == syn tcp
window 64240
trace id 3368ad3d ip nat postrouting rule jump
connectionconf-postrouting (verdict jump connectionconf-postrouting)
trace id 3368ad3d ip nat connectionconf-postrouting rule counter
packets 73 bytes 4891 (verdict continue)
trace id 3368ad3d ip nat connectionconf-postrouting rule ip saddr
@local_ipv4_networks oifname @internet_adapters counter packets 0
bytes 0 masquerade (verdict accept)
Hopefully relevant versions:
nftables 1.0.5
kernel 5.10.105
squid 4.11
iproute 5.11
