Avatar of layer47
layer47
 asked on

IPtables performance drops when being access by two Source IP's

Hi we have setup IPtables as per below.

The problem is that after a certain level of load around 9k Per source IP (We have Load coming from two source IP's) It suddenly looks like it gets rate limited.

we have check many setting but must be missing something..,,

Table: nat
Chain PREROUTING (policy ACCEPT)
num  target     prot opt source               destination
1    VIP-0002   all  --  0.0.0.0/0            10.4.8.112          [goto] /* VIP-0002 */
2    VIP-0003   all  --  0.0.0.0/0            10.4.8.113          [goto] /* VIP-0003 */

Chain INPUT (policy ACCEPT)
num  target     prot opt source               destination

Chain OUTPUT (policy ACCEPT)
num  target     prot opt source               destination

Chain POSTROUTING (policy ACCEPT)
num  target     prot opt source               destination
1    PR-0002-0001-0001  tcp  --  0.0.0.0/0            10.4.8.201          [goto] tcp dpt:83 state NEW
2    PR-0003-0001-0001  tcp  --  0.0.0.0/0            10.4.8.203          [goto] tcp dpt:83 state NEW
3    PR-0003-0001-0002  tcp  --  0.0.0.0/0            10.4.8.203          [goto] tcp dpt:83 state NEW
4    PR-0003-0001-0003  tcp  --  0.0.0.0/0            10.4.8.203          [goto] tcp dpt:83 state NEW
5    PR-0003-0001-0004  tcp  --  0.0.0.0/0            10.4.8.203          [goto] tcp dpt:83 state NEW
6    PR-0002-0001-0002  tcp  --  0.0.0.0/0            10.4.8.201          [goto] tcp dpt:83 state NEW
7    PR-0003-0001-0005  tcp  --  0.0.0.0/0            10.4.8.203          [goto] tcp dpt:83 state NEW

Chain PR-0002-0001-0001 (1 references)
num  target     prot opt source               destination
1    SNAT       all  --  0.0.0.0/0            0.0.0.0/0           /* PR-RIP-0002-0001-0001 */ to:10.4.8.11

Chain PR-0002-0001-0002 (1 references)
num  target     prot opt source               destination
1    SNAT       all  --  0.0.0.0/0            0.0.0.0/0           /* PR-RIP-0002-0001-0002 */ to:10.4.8.11

Chain PR-0003-0001-0001 (1 references)
num  target     prot opt source               destination
1    SNAT       all  --  0.0.0.0/0            0.0.0.0/0           /* PR-RIP-0003-0001-0001 */ to:10.4.8.11

Chain PR-0003-0001-0002 (1 references)
num  target     prot opt source               destination
1    SNAT       all  --  0.0.0.0/0            0.0.0.0/0           /* PR-RIP-0003-0001-0002 */ to:10.4.8.11

Chain PR-0003-0001-0003 (1 references)
num  target     prot opt source               destination
1    SNAT       all  --  0.0.0.0/0            0.0.0.0/0           /* PR-RIP-0003-0001-0003 */ to:10.4.8.11

Chain PR-0003-0001-0004 (1 references)
num  target     prot opt source               destination
1    SNAT       all  --  0.0.0.0/0            0.0.0.0/0           /* PR-RIP-0003-0001-0004 */ to:10.4.8.11

Chain PR-0003-0001-0005 (1 references)
num  target     prot opt source               destination
1    SNAT       all  --  0.0.0.0/0            0.0.0.0/0           /* PR-RIP-0003-0001-0005 */ to:10.4.8.11

Chain RIP-RR-0002-0001-0001 (1 references)
num  target     prot opt source               destination
1    SET        all  --  0.0.0.0/0            0.0.0.0/0           /* Removes VSR-RR-0002-0001 from LB-RR-0002-0001-0001 */ del-set LB-RR-0002-0001-0001 dst
2    SET        all  --  0.0.0.0/0            0.0.0.0/0           /* Adds VSR-RR-0002-0001 to LB-RR-0002-0001-0002 */ add-set LB-RR-0002-0001-0002 dst
3    DNAT       tcp  --  0.0.0.0/0            0.0.0.0/0           /* PREROUTING: VSR-RR-0002-0001 LB-RR-0002-0001-0001 */ to:10.4.8.201:83

Chain RIP-RR-0002-0001-0002 (1 references)
num  target     prot opt source               destination
1    SET        all  --  0.0.0.0/0            0.0.0.0/0           /* Removes VSR-RR-0002-0001 from LB-RR-0002-0001-0002 */ del-set LB-RR-0002-0001-0002 dst
2    SET        all  --  0.0.0.0/0            0.0.0.0/0           /* Adds VSR-RR-0002-0001 to LB-RR-0002-0001-0001 */ add-set LB-RR-0002-0001-0001 dst
3    DNAT       tcp  --  0.0.0.0/0            0.0.0.0/0           /* PREROUTING: VSR-RR-0002-0001 LB-RR-0002-0001-0002 */ to:10.4.8.201:83

Chain RIP-RR-0003-0001-0001 (1 references)
num  target     prot opt source               destination
1    SET        all  --  0.0.0.0/0            0.0.0.0/0           /* Removes VSR-RR-0003-0001 from LB-RR-0003-0001-0001 */ del-set LB-RR-0003-0001-0001 dst
2    SET        all  --  0.0.0.0/0            0.0.0.0/0           /* Adds VSR-RR-0003-0001 to LB-RR-0003-0001-0005 */ add-set LB-RR-0003-0001-0005 dst
3    DNAT       tcp  --  0.0.0.0/0            0.0.0.0/0           /* PREROUTING: VSR-RR-0003-0001 LB-RR-0003-0001-0001 */ to:10.4.8.203:83

Chain RIP-RR-0003-0001-0002 (1 references)
num  target     prot opt source               destination
1    SET        all  --  0.0.0.0/0            0.0.0.0/0           /* Removes VSR-RR-0003-0001 from LB-RR-0003-0001-0002 */ del-set LB-RR-0003-0001-0002 dst
2    SET        all  --  0.0.0.0/0            0.0.0.0/0           /* Adds VSR-RR-0003-0001 to LB-RR-0003-0001-0001 */ add-set LB-RR-0003-0001-0001 dst
3    DNAT       tcp  --  0.0.0.0/0            0.0.0.0/0           /* PREROUTING: VSR-RR-0003-0001 LB-RR-0003-0001-0002 */ to:10.4.8.203:83

Chain RIP-RR-0003-0001-0003 (1 references)
num  target     prot opt source               destination
1    SET        all  --  0.0.0.0/0            0.0.0.0/0           /* Removes VSR-RR-0003-0001 from LB-RR-0003-0001-0003 */ del-set LB-RR-0003-0001-0003 dst
2    SET        all  --  0.0.0.0/0            0.0.0.0/0           /* Adds VSR-RR-0003-0001 to LB-RR-0003-0001-0002 */ add-set LB-RR-0003-0001-0002 dst
3    DNAT       tcp  --  0.0.0.0/0            0.0.0.0/0           /* PREROUTING: VSR-RR-0003-0001 LB-RR-0003-0001-0003 */ to:10.4.8.203:83

Chain RIP-RR-0003-0001-0004 (1 references)
num  target     prot opt source               destination
1    SET        all  --  0.0.0.0/0            0.0.0.0/0           /* Removes VSR-RR-0003-0001 from LB-RR-0003-0001-0004 */ del-set LB-RR-0003-0001-0004 dst
2    SET        all  --  0.0.0.0/0            0.0.0.0/0           /* Adds VSR-RR-0003-0001 to LB-RR-0003-0001-0003 */ add-set LB-RR-0003-0001-0003 dst
3    DNAT       tcp  --  0.0.0.0/0            0.0.0.0/0           /* PREROUTING: VSR-RR-0003-0001 LB-RR-0003-0001-0004 */ to:10.4.8.203:83

Chain RIP-RR-0003-0001-0005 (1 references)
num  target     prot opt source               destination
1    SET        all  --  0.0.0.0/0            0.0.0.0/0           /* Removes VSR-RR-0003-0001 from LB-RR-0003-0001-0005 */ del-set LB-RR-0003-0001-0005 dst
2    SET        all  --  0.0.0.0/0            0.0.0.0/0           /* Adds VSR-RR-0003-0001 to LB-RR-0003-0001-0004 */ add-set LB-RR-0003-0001-0004 dst
3    DNAT       tcp  --  0.0.0.0/0            0.0.0.0/0           /* PREROUTING: VSR-RR-0003-0001 LB-RR-0003-0001-0005 */ to:10.4.8.203:83

Chain VIP-0002 (1 references)
num  target     prot opt source               destination
1    VSR-RR-0002-0001  tcp  --  0.0.0.0/0            0.0.0.0/0           [goto] tcp dpt:80 state NEW /* RR-0002-0001 */

Chain VIP-0003 (1 references)
num  target     prot opt source               destination
1    VSR-RR-0003-0001  tcp  --  0.0.0.0/0            0.0.0.0/0           [goto] tcp dpt:80 state NEW /* RR-0003-0001 */

Chain VSR-RR-0002-0001 (1 references)
num  target     prot opt source               destination
1    RIP-RR-0002-0001-0002  all  --  0.0.0.0/0            0.0.0.0/0           [goto] match-set LB-RR-0002-0001-0002 dst /* RR-NEXT-0001 DNAT to RIPip:10.4.8.201 RIPport:83 */
2    RIP-RR-0002-0001-0001  all  --  0.0.0.0/0            0.0.0.0/0           [goto] /* RR-NEXT-0002 DNAT to RIPip:10.4.8.201 RIPport:83 */

Chain VSR-RR-0003-0001 (1 references)
num  target     prot opt source               destination
1    RIP-RR-0003-0001-0005  all  --  0.0.0.0/0            0.0.0.0/0           [goto] match-set LB-RR-0003-0001-0005 dst /* RR-NEXT-0004 DNAT to RIPip:10.4.8.203 RIPport:83 */
2    RIP-RR-0003-0001-0004  all  --  0.0.0.0/0            0.0.0.0/0           [goto] match-set LB-RR-0003-0001-0004 dst /* RR-NEXT-0003 DNAT to RIPip:10.4.8.203 RIPport:83 */
3    RIP-RR-0003-0001-0003  all  --...
Linux NetworkingLinux SecuritySoftware Firewalls

Avatar of undefined
Last Comment
gheist

8/22/2022 - Mon
gheist

Can you upload output of iptables-save | tee mu_tables_ee
layer47

ASKER
Hi,

From one source ip it's seems to get to 65k cps,  but if you have more source ip s then it's less.

Also web have in read conntrack and max hash size as well as ephemeral ports, file handles etc.

 
[root@jetnexus ~]# iptables-save | tee mu_tables_ee
# Generated by iptables-save v1.4.7 on Sun May 18 20:21:09 2014
*filter
:INPUT ACCEPT [349776:126260616]
:FORWARD ACCEPT [32897207:3995884837]
:OUTPUT ACCEPT [317044:203387357]
COMMIT
# Completed on Sun May 18 20:21:09 2014
# Generated by iptables-save v1.4.7 on Sun May 18 20:21:09 2014
*nat
:PREROUTING ACCEPT [24823:1310538]
:POSTROUTING ACCEPT [24673:1480554]
:OUTPUT ACCEPT [24673:1480554]
:PR-0001-0001-0001 - [0:0]
:PR-0001-0001-0002 - [0:0]
:PR-0001-0001-0003 - [0:0]
:PR-0002-0001-0001 - [0:0]
:RIP-RR-0001-0001-0001 - [0:0]
:RIP-RR-0001-0001-0002 - [0:0]
:RIP-RR-0001-0001-0003 - [0:0]
:RIP-RR-0002-0001-0001 - [0:0]
:VIP-0001 - [0:0]
:VIP-0002 - [0:0]
:VSR-RR-0001-0001 - [0:0]
:VSR-RR-0002-0001 - [0:0]
-A PREROUTING -d 10.4.8.131/32 -m comment --comment "VIP-0001" -g VIP-0001
-A PREROUTING -d 10.4.8.132/32 -m comment --comment "VIP-0002" -g VIP-0002
-A POSTROUTING -d 10.4.8.203/32 -p tcp -m tcp --dport 83 -m state --state NEW -g PR-0001-0001-0001
-A POSTROUTING -d 10.4.8.203/32 -p tcp -m tcp --dport 83 -m state --state NEW -g PR-0002-0001-0001
-A POSTROUTING -d 10.4.8.201/32 -p tcp -m tcp --dport 83 -m state --state NEW -g PR-0001-0001-0002
-A POSTROUTING -d 10.4.8.201/32 -p tcp -m tcp --dport 83 -m state --state NEW -g PR-0001-0001-0003
-A PR-0001-0001-0001 -m comment --comment "PR-RIP-0001-0001-0001" -j SNAT --to-source 10.4.8.31
-A PR-0001-0001-0002 -m comment --comment "PR-RIP-0001-0001-0002" -j SNAT --to-source 10.4.8.31
-A PR-0001-0001-0003 -m comment --comment "PR-RIP-0001-0001-0003" -j SNAT --to-source 10.4.8.31
-A PR-0002-0001-0001 -m comment --comment "PR-RIP-0002-0001-0001" -j SNAT --to-source 10.4.8.31
-A RIP-RR-0001-0001-0001 -m comment --comment "Removes VSR-RR-0001-0001 from LB-RR-0001-0001-0001" -j SET --del-set LB-RR-0001-0001-0001 dst
-A RIP-RR-0001-0001-0001 -m comment --comment "Adds VSR-RR-0001-0001 to LB-RR-0001-0001-0003" -j SET --add-set LB-RR-0001-0001-0003 dst
-A RIP-RR-0001-0001-0001 -p tcp -m comment --comment "PREROUTING: VSR-RR-0001-0001 LB-RR-0001-0001-0001" -j DNAT --to-destination 10.4.8.203:                                83
-A RIP-RR-0001-0001-0002 -m comment --comment "Removes VSR-RR-0001-0001 from LB-RR-0001-0001-0002" -j SET --del-set LB-RR-0001-0001-0002 dst
-A RIP-RR-0001-0001-0002 -m comment --comment "Adds VSR-RR-0001-0001 to LB-RR-0001-0001-0001" -j SET --add-set LB-RR-0001-0001-0001 dst
-A RIP-RR-0001-0001-0002 -p tcp -m comment --comment "PREROUTING: VSR-RR-0001-0001 LB-RR-0001-0001-0002" -j DNAT --to-destination 10.4.8.201:                                83
-A RIP-RR-0001-0001-0003 -m comment --comment "Removes VSR-RR-0001-0001 from LB-RR-0001-0001-0003" -j SET --del-set LB-RR-0001-0001-0003 dst
-A RIP-RR-0001-0001-0003 -m comment --comment "Adds VSR-RR-0001-0001 to LB-RR-0001-0001-0002" -j SET --add-set LB-RR-0001-0001-0002 dst
-A RIP-RR-0001-0001-0003 -p tcp -m comment --comment "PREROUTING: VSR-RR-0001-0001 LB-RR-0001-0001-0003" -j DNAT --to-destination 10.4.8.201:                                83
-A RIP-RR-0002-0001-0001 -m comment --comment "Removes VSR-RR-0002-0001 from LB-RR-0002-0001-0001" -j SET --del-set LB-RR-0002-0001-0001 dst
-A RIP-RR-0002-0001-0001 -m comment --comment "Adds VSR-RR-0002-0001 to LB-RR-0002-0001-0001" -j SET --add-set LB-RR-0002-0001-0001 dst
-A RIP-RR-0002-0001-0001 -p tcp -m comment --comment "PREROUTING: VSR-RR-0002-0001 LB-RR-0002-0001-0001" -j DNAT --to-destination 10.4.8.203:                                83
-A VIP-0001 -p tcp -m tcp --dport 80 -m state --state NEW -m comment --comment "RR-0001-0001" -g VSR-RR-0001-0001
-A VIP-0002 -p tcp -m tcp --dport 80 -m state --state NEW -m comment --comment "RR-0002-0001" -g VSR-RR-0002-0001
-A VSR-RR-0001-0001 -m set --match-set LB-RR-0001-0001-0003 dst -m comment --comment "RR-NEXT-0002 DNAT to RIPip:10.4.8.201 RIPport:83" -g RI                                P-RR-0001-0001-0003
-A VSR-RR-0001-0001 -m set --match-set LB-RR-0001-0001-0002 dst -m comment --comment "RR-NEXT-0001 DNAT to RIPip:10.4.8.201 RIPport:83" -g RI                                P-RR-0001-0001-0002
-A VSR-RR-0001-0001 -m comment --comment "RR-NEXT-0003 DNAT to RIPip:10.4.8.203 RIPport:83" -g RIP-RR-0001-0001-0001
-A VSR-RR-0002-0001 -m comment --comment "RR-NEXT-0001 DNAT to RIPip:10.4.8.203 RIPport:83" -g RIP-RR-0002-0001-0001
COMMIT
ASKER CERTIFIED SOLUTION
gheist

Log in or sign up to see answer
Become an EE member today7-DAY FREE TRIAL
Members can start a 7-Day Free trial then enjoy unlimited access to the platform
Sign up - Free for 7 days
or
Learn why we charge membership fees
We get it - no one likes a content blocker. Take one extra minute and find out why we block content.
Not exactly the question you had in mind?
Sign up for an EE membership and get your own personalized solution. With an EE membership, you can ask unlimited troubleshooting, research, or opinion questions.
ask a question
Experts Exchange has (a) saved my job multiple times, (b) saved me hours, days, and even weeks of work, and often (c) makes me look like a superhero! This place is MAGIC!
Walt Forbes