Hi Greg,
Thanks for your response. I changed my config to use br0. The behavior
I observed is different and not what I wanted. I've attached the
traces and the iptables-save output at the end of the email. With my
previous configuration, the order of the trace was as follows:
raw:PREROUTING:policy:2
mangle:PREROUTING:rule:1
nat:PREROUTING:policy:1
mangle:INPUT:policy:1
filter:INPUT:policy:1
nat:INPUT:policy:1
However, with the new configuration, the order is as follows:
raw:PREROUTING:policy:2
mangle:PREROUTING:policy:3
mangle:FORWARD:policy:1
filter:FORWARD:policy:1
mangle:POSTROUTING:policy:1
It is trying to take the forwarding path which is not what I want. I
want the traffic to be delivered to my application which is not
happening in this case either.
The iptables -v doesn't show the rule being hit with the new
configuration. However, I see it being hit in the older configuration.
I've attached the iptables -v output for both cases at the end of the
email.
Please let me know if there any other debugging options I can try. Thanks.
Dk.
--------------------------------------------------------------------------------------------------------------------------------------------------
Jul 9 17:45:47 localhost kernel: TRACE: raw:PREROUTING:policy:2
IN=enp2s0 OUT= MAC=00:30:18:08:06:e8:0c:c4:7a:b5:be:eb:08:00
SRC=192.168.10.200 DST=192.168.10.50 LEN=60 TOS=0x00 PREC=0x00 TTL=64
ID=40946 DF PROTO=TCP SPT=38912 DPT=80 SEQ=747233776 ACK=0
WINDOW=29200 RES=0x00 SYN URGP=0 OPT
(020405B40402080A3CFB7E020000000001030307)
Jul 9 17:45:47 localhost kernel: TRACE: mangle:PREROUTING:policy:3
IN=enp2s0 OUT= MAC=00:30:18:08:06:e8:0c:c4:7a:b5:be:eb:08:00
SRC=192.168.10.200 DST=192.168.10.50 LEN=60 TOS=0x00 PREC=0x00 TTL=64
ID=40946 DF PROTO=TCP SPT=38912 DPT=80 SEQ=747233776 ACK=0
WINDOW=29200 RES=0x00 SYN URGP=0 OPT
(020405B40402080A3CFB7E020000000001030307)
Jul 9 17:45:47 localhost kernel: TRACE: mangle:FORWARD:policy:1
IN=enp2s0 OUT=enp0s31f6 MAC=00:30:18:08:06:e8:0c:c4:7a:b5:be:eb:08:00
SRC=192.168.10.200 DST=192.168.10.50 LEN=60 TOS=0x00 PREC=0x00 TTL=63
ID=40946 DF PROTO=TCP SPT=38912 DPT=80 SEQ=747233776 ACK=0
WINDOW=29200 RES=0x00 SYN URGP=0 OPT
(020405B40402080A3CFB7E020000000001030307)
Jul 9 17:45:47 localhost kernel: TRACE: filter:FORWARD:policy:1
IN=enp2s0 OUT=enp0s31f6 MAC=00:30:18:08:06:e8:0c:c4:7a:b5:be:eb:08:00
SRC=192.168.10.200 DST=192.168.10.50 LEN=60 TOS=0x00 PREC=0x00 TTL=63
ID=40946 DF PROTO=TCP SPT=38912 DPT=80 SEQ=747233776 ACK=0
WINDOW=29200 RES=0x00 SYN URGP=0 OPT
(020405B40402080A3CFB7E020000000001030307)
Jul 9 17:45:47 localhost kernel: TRACE: mangle:POSTROUTING:policy:1
IN= OUT=enp0s31f6 SRC=192.168.10.200 DST=192.168.10.50 LEN=60 TOS=0x00
PREC=0x00 TTL=63 ID=40946 DF PROTO=TCP SPT=38912 DPT=80 SEQ=747233776
ACK=0 WINDOW=29200 RES=0x00 SYN URGP=0 OPT
(020405B40402080A3CFB7E020000000001030307)
--------------------------------------------------------------------------------------------------------------------------------------------------
[root@localhost ~]# iptables-save
# Generated by iptables-save v1.4.21 on Tue Jul 9 17:49:13 2019
*nat
:PREROUTING ACCEPT [7:1312]
:INPUT ACCEPT [7:1312]
:OUTPUT ACCEPT [0:0]
:POSTROUTING ACCEPT [0:0]
COMMIT
# Completed on Tue Jul 9 17:49:13 2019
# Generated by iptables-save v1.4.21 on Tue Jul 9 17:49:13 2019
*raw
:PREROUTING ACCEPT [593:55820]
:OUTPUT ACCEPT [285:75634]
-A PREROUTING -p tcp -m tcp --dport 80 -j TRACE
-A OUTPUT -p tcp -m tcp --dport 80 -j TRACE
COMMIT
# Completed on Tue Jul 9 17:49:13 2019
# Generated by iptables-save v1.4.21 on Tue Jul 9 17:49:13 2019
*filter
:INPUT ACCEPT [72:5760]
:FORWARD ACCEPT [1:60]
:OUTPUT ACCEPT [39:5884]
COMMIT
# Completed on Tue Jul 9 17:49:13 2019
# Generated by iptables-save v1.4.21 on Tue Jul 9 17:49:13 2019
*mangle
:PREROUTING ACCEPT [73:5820]
:INPUT ACCEPT [72:5760]
:FORWARD ACCEPT [1:60]
:OUTPUT ACCEPT [39:5884]
:POSTROUTING ACCEPT [40:5944]
-A PREROUTING -i br0 -p tcp -m tcp --dport 80 -j TPROXY --on-port 80
--on-ip 0.0.0.0 --tproxy-mark 0x1/0x1
-A PREROUTING -i br0 -p tcp -m tcp --sport 80 -j MARK --set-xmark 0x1/0x1
COMMIT
# Completed on Tue Jul 9 17:49:13 2019
--------------------------------------------------------------------------------------------------------------------------------------------------
[root@localhost log]# iptables -t mangle -v -L -n
Chain PREROUTING (policy ACCEPT 286 packets, 27386 bytes)
pkts bytes target prot opt in out source destination
0 0 TPROXY tcp -- br0 * 0.0.0.0/0
0.0.0.0/0 tcp dpt:80 TPROXY redirect 0.0.0.0:8080 mark
0x1/0x1
0 0 MARK tcp -- br0 * 0.0.0.0/0
0.0.0.0/0 tcp spt:80 MARK or 0x1
Chain INPUT (policy ACCEPT 281 packets, 27086 bytes)
pkts bytes target prot opt in out source destination
Chain FORWARD (policy ACCEPT 5 packets, 300 bytes)
pkts bytes target prot opt in out source destination
Chain OUTPUT (policy ACCEPT 127 packets, 21822 bytes)
pkts bytes target prot opt in out source destination
Chain POSTROUTING (policy ACCEPT 132 packets, 22122 bytes)
pkts bytes target prot opt in out source destination
--------------------------------------------------------------------------------------------------------------------------------------------------
[root@localhost ~]# iptables -t mangle -L -n -v
Chain PREROUTING (policy ACCEPT 97 packets, 8152 bytes)
pkts bytes target prot opt in out source destination
2 120 TPROXY tcp -- enp2s0 * 0.0.0.0/0
0.0.0.0/0 tcp dpt:80 TPROXY redirect 0.0.0.0:8080 mark
0x1/0x1
0 0 MARK tcp -- enp1s0f3 * 0.0.0.0/0
0.0.0.0/0 tcp spt:80 MARK or 0x1
Chain INPUT (policy ACCEPT 97 packets, 8152 bytes)
pkts bytes target prot opt in out source destination
Chain FORWARD (policy ACCEPT 0 packets, 0 bytes)
pkts bytes target prot opt in out source destination
Chain OUTPUT (policy ACCEPT 49 packets, 6228 bytes)
pkts bytes target prot opt in out source destination
Chain POSTROUTING (policy ACCEPT 49 packets, 6228 bytes)
pkts bytes target prot opt in out source destination
On Tue, Jul 9, 2019 at 10:10 AM Gregory Vander Schueren
<gregory.vanderschueren@xxxxxxxxxxxx> wrote:
>
> Hi Dk,
>
> If I recall correctly, when using brouting the IP stack sees the bridge as the incoming interface. Try removing -i enp2s0 in your TPROXY rule to see if it helps (or use -i br0).
>
> You can also get packet counters with iptables (-v) to validate that packets hit your TPROXY rule.
>
> Regards,
> Gregory
>
> On Tue, 9 Jul 2019, 04:22 Dk Jack, <dnj0496@xxxxxxxxx> wrote:
>>
>> Hi,
>> I am trying to get my application to work transparent proxy mode i.e
>> in inline Linux bridge
>> mode. I've configured my Linux system as shown later in the end of the
>> email. I've turned
>> off rp_filters and enabled ip forwarding as suggested. I am following
>> these instructions for Apache Traffic server.
>>
>> https://docs.trafficserver.apache.org/en/6.2.x/admin-guide/configuration/transparent-proxy/bridge.en.html
>>
>> The traffic seem to be coming into my bridged system and getting
>> forwarded up the stack as per the configuration. However, my
>> application is not receiving the traffic. Enabling tracing for
>> port 80 shows the logs at the end of the email.
>>
>> Is my configuration correct? If so, why is my redirect rule that
>> modifies the dest. port from
>> 80 to 8080 not changing the port. Since my application is listening on
>> 8080, could this be the
>> reason my application is not seeing the traffic. If not, what else can
>> I look at to debug this
>> issue? Are there any other counters I can look at (or traces that I
>> can enable) to determing
>> where the packets are getting lost?
>>
>> Thanks for the help in advance.
>>
>> regards,
>> Dk.
>>
>> OS: Centos 7.5
>> Kernel Version: 3.10
>>
>> PS: My config.
>>
>> ----------------------------- Config --------------------------------
>>
>> echo 1 > /proc/sys/net/ipv4/ip_forward
>> brctl addbr br0
>> ifconfig br0 up
>> brctl stp br0 off
>> brctl addif br0 enp2s0
>> brctl addif br0 enp1s0f3
>> ifconfig enp2s0 0 0.0.0.0
>> ifconfig enp1s0f3 0 0.0.0.0
>> ifconfig br0 192.168.150.150 netmask 255.255.255.0 up
>> echo 0 > /proc/sys/net/ipv4/conf/br0/rp_filter
>> echo 0 > /proc/sys/net/ipv4/conf/enp1s0f3/rp_filter
>> echo 0 > /proc/sys/net/ipv4/conf/enp2s0/rp_filter
>> ip rule delete fwmark 1/1 > /dev/null 2>&1
>> ip rule add fwmark 1/1 table 1
>> ip route add local 0/0 dev lo table 1
>> iptables -t mangle -A PREROUTING -i enp2s0 -p tcp -m tcp --dport 80 -j
>> TPROXY --on-ip 0.0.0.0 --on-port 8080 --tproxy-mark 1/1
>> iptables -t mangle -A PREROUTING -i enp1s0f3 -p tcp -m tcp --sport 80
>> -j MARK --set-mark 1/1
>> iptables -t filter --flush FORWARD
>> iptables -t filter --flush INPUT
>> ebtables -t broute -F
>> ebtables -t broute -A BROUTING -p IPv4 --ip-proto tcp --ip-dport 80 -j
>> redirect --redirect-target DROP
>> ebtables -t broute -A BROUTING -p IPv4 --ip-proto tcp --ip-sport 80
>> -j redirect --redirect-target DROP
>>
>> ------------------------- TRACES ----------------------------------
>>
>> Jul 8 15:57:26 localhost kernel: TRACE: raw:PREROUTING:policy:2
>> IN=enp2s0 OUT= MAC=00:30:18:08:06:e8:0c:c4:7a:b5:be:eb:08:00
>> SRC=192.168.10.200 DST=192.168.10.50 LEN=60 TOS=0x00 PREC=0x00 TTL=64
>> ID=43187 DF PROTO=TCP SPT=35294 DPT=80 SEQ=2080332442 ACK=0
>> WINDOW=29200 RES=0x00 SYN URGP=0 OPT
>> (020405B40402080A3771F2290000000001030307)
>> Jul 8 15:57:26 localhost kernel: TRACE: mangle:PREROUTING:rule:1
>> IN=enp2s0 OUT= MAC=00:30:18:08:06:e8:0c:c4:7a:b5:be:eb:08:00
>> SRC=192.168.10.200 DST=192.168.10.50 LEN=60 TOS=0x00 PREC=0x00 TTL=64
>> ID=43187 DF PROTO=TCP SPT=35294 DPT=80 SEQ=2080332442 ACK=0
>> WINDOW=29200 RES=0x00 SYN URGP=0 OPT
>> (020405B40402080A3771F2290000000001030307)
>> Jul 8 15:57:26 localhost kernel: TRACE: nat:PREROUTING:policy:1
>> IN=enp2s0 OUT= MAC=00:30:18:08:06:e8:0c:c4:7a:b5:be:eb:08:00
>> SRC=192.168.10.200 DST=192.168.10.50 LEN=60 TOS=0x00 PREC=0x00 TTL=64
>> ID=43187 DF PROTO=TCP SPT=35294 DPT=80 SEQ=2080332442 ACK=0
>> WINDOW=29200 RES=0x00 SYN URGP=0 OPT
>> (020405B40402080A3771F2290000000001030307) UID=1001 GID=1001 MARK=0x1
>> Jul 8 15:57:26 localhost kernel: TRACE: mangle:INPUT:policy:1
>> IN=enp2s0 OUT= MAC=00:30:18:08:06:e8:0c:c4:7a:b5:be:eb:08:00
>> SRC=192.168.10.200 DST=192.168.10.50 LEN=60 TOS=0x00 PREC=0x00 TTL=64
>> ID=43187 DF PROTO=TCP SPT=35294 DPT=80 SEQ=2080332442 ACK=0
>> WINDOW=29200 RES=0x00 SYN URGP=0 OPT
>> (020405B40402080A3771F2290000000001030307) UID=1001 GID=1001 MARK=0x1
>> Jul 8 15:57:26 localhost kernel: TRACE: filter:INPUT:policy:1
>> IN=enp2s0 OUT= MAC=00:30:18:08:06:e8:0c:c4:7a:b5:be:eb:08:00
>> SRC=192.168.10.200 DST=192.168.10.50 LEN=60 TOS=0x00 PREC=0x00 TTL=64
>> ID=43187 DF PROTO=TCP SPT=35294 DPT=80 SEQ=2080332442 ACK=0
>> WINDOW=29200 RES=0x00 SYN URGP=0 OPT
>> (020405B40402080A3771F2290000000001030307) UID=1001 GID=1001 MARK=0x1
>> Jul 8 15:57:26 localhost kernel: TRACE: nat:INPUT:policy:1 IN=enp2s0
>> OUT= MAC=00:30:18:08:06:e8:0c:c4:7a:b5:be:eb:08:00 SRC=192.168.10.200
>> DST=192.168.10.50 LEN=60 TOS=0x00 PREC=0x00 TTL=64 ID=43187 DF
>> PROTO=TCP SPT=35294 DPT=80 SEQ=2080332442 ACK=0 WINDOW=29200 RES=0x00
>> SYN URGP=0 OPT (020405B40402080A3771F2290000000001030307) UID=1001
>> GID=1001 MARK=0x1
>
>
> ________________________________
>
> Disclaimer: https://www.tessares.net/mail-disclaimer/
>
>
