Link to home
Start Free TrialLog in
Avatar of layer47
layer47

asked on

IPtables performance drops when being access by two Source IP's

Hi we have setup IPtables as per below.

The problem is that after a certain level of load around 9k Per source IP (We have Load coming from two source IP's) It suddenly looks like it gets rate limited.

we have check many setting but must be missing something..,,

Table: nat
Chain PREROUTING (policy ACCEPT)
num  target     prot opt source               destination
1    VIP-0002   all  --  0.0.0.0/0            10.4.8.112          [goto] /* VIP-0002 */
2    VIP-0003   all  --  0.0.0.0/0            10.4.8.113          [goto] /* VIP-0003 */

Chain INPUT (policy ACCEPT)
num  target     prot opt source               destination

Chain OUTPUT (policy ACCEPT)
num  target     prot opt source               destination

Chain POSTROUTING (policy ACCEPT)
num  target     prot opt source               destination
1    PR-0002-0001-0001  tcp  --  0.0.0.0/0            10.4.8.201          [goto] tcp dpt:83 state NEW
2    PR-0003-0001-0001  tcp  --  0.0.0.0/0            10.4.8.203          [goto] tcp dpt:83 state NEW
3    PR-0003-0001-0002  tcp  --  0.0.0.0/0            10.4.8.203          [goto] tcp dpt:83 state NEW
4    PR-0003-0001-0003  tcp  --  0.0.0.0/0            10.4.8.203          [goto] tcp dpt:83 state NEW
5    PR-0003-0001-0004  tcp  --  0.0.0.0/0            10.4.8.203          [goto] tcp dpt:83 state NEW
6    PR-0002-0001-0002  tcp  --  0.0.0.0/0            10.4.8.201          [goto] tcp dpt:83 state NEW
7    PR-0003-0001-0005  tcp  --  0.0.0.0/0            10.4.8.203          [goto] tcp dpt:83 state NEW

Chain PR-0002-0001-0001 (1 references)
num  target     prot opt source               destination
1    SNAT       all  --  0.0.0.0/0            0.0.0.0/0           /* PR-RIP-0002-0001-0001 */ to:10.4.8.11

Chain PR-0002-0001-0002 (1 references)
num  target     prot opt source               destination
1    SNAT       all  --  0.0.0.0/0            0.0.0.0/0           /* PR-RIP-0002-0001-0002 */ to:10.4.8.11

Chain PR-0003-0001-0001 (1 references)
num  target     prot opt source               destination
1    SNAT       all  --  0.0.0.0/0            0.0.0.0/0           /* PR-RIP-0003-0001-0001 */ to:10.4.8.11

Chain PR-0003-0001-0002 (1 references)
num  target     prot opt source               destination
1    SNAT       all  --  0.0.0.0/0            0.0.0.0/0           /* PR-RIP-0003-0001-0002 */ to:10.4.8.11

Chain PR-0003-0001-0003 (1 references)
num  target     prot opt source               destination
1    SNAT       all  --  0.0.0.0/0            0.0.0.0/0           /* PR-RIP-0003-0001-0003 */ to:10.4.8.11

Chain PR-0003-0001-0004 (1 references)
num  target     prot opt source               destination
1    SNAT       all  --  0.0.0.0/0            0.0.0.0/0           /* PR-RIP-0003-0001-0004 */ to:10.4.8.11

Chain PR-0003-0001-0005 (1 references)
num  target     prot opt source               destination
1    SNAT       all  --  0.0.0.0/0            0.0.0.0/0           /* PR-RIP-0003-0001-0005 */ to:10.4.8.11

Chain RIP-RR-0002-0001-0001 (1 references)
num  target     prot opt source               destination
1    SET        all  --  0.0.0.0/0            0.0.0.0/0           /* Removes VSR-RR-0002-0001 from LB-RR-0002-0001-0001 */ del-set LB-RR-0002-0001-0001 dst
2    SET        all  --  0.0.0.0/0            0.0.0.0/0           /* Adds VSR-RR-0002-0001 to LB-RR-0002-0001-0002 */ add-set LB-RR-0002-0001-0002 dst
3    DNAT       tcp  --  0.0.0.0/0            0.0.0.0/0           /* PREROUTING: VSR-RR-0002-0001 LB-RR-0002-0001-0001 */ to:10.4.8.201:83

Chain RIP-RR-0002-0001-0002 (1 references)
num  target     prot opt source               destination
1    SET        all  --  0.0.0.0/0            0.0.0.0/0           /* Removes VSR-RR-0002-0001 from LB-RR-0002-0001-0002 */ del-set LB-RR-0002-0001-0002 dst
2    SET        all  --  0.0.0.0/0            0.0.0.0/0           /* Adds VSR-RR-0002-0001 to LB-RR-0002-0001-0001 */ add-set LB-RR-0002-0001-0001 dst
3    DNAT       tcp  --  0.0.0.0/0            0.0.0.0/0           /* PREROUTING: VSR-RR-0002-0001 LB-RR-0002-0001-0002 */ to:10.4.8.201:83

Chain RIP-RR-0003-0001-0001 (1 references)
num  target     prot opt source               destination
1    SET        all  --  0.0.0.0/0            0.0.0.0/0           /* Removes VSR-RR-0003-0001 from LB-RR-0003-0001-0001 */ del-set LB-RR-0003-0001-0001 dst
2    SET        all  --  0.0.0.0/0            0.0.0.0/0           /* Adds VSR-RR-0003-0001 to LB-RR-0003-0001-0005 */ add-set LB-RR-0003-0001-0005 dst
3    DNAT       tcp  --  0.0.0.0/0            0.0.0.0/0           /* PREROUTING: VSR-RR-0003-0001 LB-RR-0003-0001-0001 */ to:10.4.8.203:83

Chain RIP-RR-0003-0001-0002 (1 references)
num  target     prot opt source               destination
1    SET        all  --  0.0.0.0/0            0.0.0.0/0           /* Removes VSR-RR-0003-0001 from LB-RR-0003-0001-0002 */ del-set LB-RR-0003-0001-0002 dst
2    SET        all  --  0.0.0.0/0            0.0.0.0/0           /* Adds VSR-RR-0003-0001 to LB-RR-0003-0001-0001 */ add-set LB-RR-0003-0001-0001 dst
3    DNAT       tcp  --  0.0.0.0/0            0.0.0.0/0           /* PREROUTING: VSR-RR-0003-0001 LB-RR-0003-0001-0002 */ to:10.4.8.203:83

Chain RIP-RR-0003-0001-0003 (1 references)
num  target     prot opt source               destination
1    SET        all  --  0.0.0.0/0            0.0.0.0/0           /* Removes VSR-RR-0003-0001 from LB-RR-0003-0001-0003 */ del-set LB-RR-0003-0001-0003 dst
2    SET        all  --  0.0.0.0/0            0.0.0.0/0           /* Adds VSR-RR-0003-0001 to LB-RR-0003-0001-0002 */ add-set LB-RR-0003-0001-0002 dst
3    DNAT       tcp  --  0.0.0.0/0            0.0.0.0/0           /* PREROUTING: VSR-RR-0003-0001 LB-RR-0003-0001-0003 */ to:10.4.8.203:83

Chain RIP-RR-0003-0001-0004 (1 references)
num  target     prot opt source               destination
1    SET        all  --  0.0.0.0/0            0.0.0.0/0           /* Removes VSR-RR-0003-0001 from LB-RR-0003-0001-0004 */ del-set LB-RR-0003-0001-0004 dst
2    SET        all  --  0.0.0.0/0            0.0.0.0/0           /* Adds VSR-RR-0003-0001 to LB-RR-0003-0001-0003 */ add-set LB-RR-0003-0001-0003 dst
3    DNAT       tcp  --  0.0.0.0/0            0.0.0.0/0           /* PREROUTING: VSR-RR-0003-0001 LB-RR-0003-0001-0004 */ to:10.4.8.203:83

Chain RIP-RR-0003-0001-0005 (1 references)
num  target     prot opt source               destination
1    SET        all  --  0.0.0.0/0            0.0.0.0/0           /* Removes VSR-RR-0003-0001 from LB-RR-0003-0001-0005 */ del-set LB-RR-0003-0001-0005 dst
2    SET        all  --  0.0.0.0/0            0.0.0.0/0           /* Adds VSR-RR-0003-0001 to LB-RR-0003-0001-0004 */ add-set LB-RR-0003-0001-0004 dst
3    DNAT       tcp  --  0.0.0.0/0            0.0.0.0/0           /* PREROUTING: VSR-RR-0003-0001 LB-RR-0003-0001-0005 */ to:10.4.8.203:83

Chain VIP-0002 (1 references)
num  target     prot opt source               destination
1    VSR-RR-0002-0001  tcp  --  0.0.0.0/0            0.0.0.0/0           [goto] tcp dpt:80 state NEW /* RR-0002-0001 */

Chain VIP-0003 (1 references)
num  target     prot opt source               destination
1    VSR-RR-0003-0001  tcp  --  0.0.0.0/0            0.0.0.0/0           [goto] tcp dpt:80 state NEW /* RR-0003-0001 */

Chain VSR-RR-0002-0001 (1 references)
num  target     prot opt source               destination
1    RIP-RR-0002-0001-0002  all  --  0.0.0.0/0            0.0.0.0/0           [goto] match-set LB-RR-0002-0001-0002 dst /* RR-NEXT-0001 DNAT to RIPip:10.4.8.201 RIPport:83 */
2    RIP-RR-0002-0001-0001  all  --  0.0.0.0/0            0.0.0.0/0           [goto] /* RR-NEXT-0002 DNAT to RIPip:10.4.8.201 RIPport:83 */

Chain VSR-RR-0003-0001 (1 references)
num  target     prot opt source               destination
1    RIP-RR-0003-0001-0005  all  --  0.0.0.0/0            0.0.0.0/0           [goto] match-set LB-RR-0003-0001-0005 dst /* RR-NEXT-0004 DNAT to RIPip:10.4.8.203 RIPport:83 */
2    RIP-RR-0003-0001-0004  all  --  0.0.0.0/0            0.0.0.0/0           [goto] match-set LB-RR-0003-0001-0004 dst /* RR-NEXT-0003 DNAT to RIPip:10.4.8.203 RIPport:83 */
3    RIP-RR-0003-0001-0003  all  --...
Avatar of gheist
gheist
Flag of Belgium image

Can you upload output of iptables-save | tee mu_tables_ee
Avatar of layer47
layer47

ASKER

Hi,

From one source ip it's seems to get to 65k cps,  but if you have more source ip s then it's less.

Also web have in read conntrack and max hash size as well as ephemeral ports, file handles etc.

 
[root@jetnexus ~]# iptables-save | tee mu_tables_ee
# Generated by iptables-save v1.4.7 on Sun May 18 20:21:09 2014
*filter
:INPUT ACCEPT [349776:126260616]
:FORWARD ACCEPT [32897207:3995884837]
:OUTPUT ACCEPT [317044:203387357]
COMMIT
# Completed on Sun May 18 20:21:09 2014
# Generated by iptables-save v1.4.7 on Sun May 18 20:21:09 2014
*nat
:PREROUTING ACCEPT [24823:1310538]
:POSTROUTING ACCEPT [24673:1480554]
:OUTPUT ACCEPT [24673:1480554]
:PR-0001-0001-0001 - [0:0]
:PR-0001-0001-0002 - [0:0]
:PR-0001-0001-0003 - [0:0]
:PR-0002-0001-0001 - [0:0]
:RIP-RR-0001-0001-0001 - [0:0]
:RIP-RR-0001-0001-0002 - [0:0]
:RIP-RR-0001-0001-0003 - [0:0]
:RIP-RR-0002-0001-0001 - [0:0]
:VIP-0001 - [0:0]
:VIP-0002 - [0:0]
:VSR-RR-0001-0001 - [0:0]
:VSR-RR-0002-0001 - [0:0]
-A PREROUTING -d 10.4.8.131/32 -m comment --comment "VIP-0001" -g VIP-0001
-A PREROUTING -d 10.4.8.132/32 -m comment --comment "VIP-0002" -g VIP-0002
-A POSTROUTING -d 10.4.8.203/32 -p tcp -m tcp --dport 83 -m state --state NEW -g PR-0001-0001-0001
-A POSTROUTING -d 10.4.8.203/32 -p tcp -m tcp --dport 83 -m state --state NEW -g PR-0002-0001-0001
-A POSTROUTING -d 10.4.8.201/32 -p tcp -m tcp --dport 83 -m state --state NEW -g PR-0001-0001-0002
-A POSTROUTING -d 10.4.8.201/32 -p tcp -m tcp --dport 83 -m state --state NEW -g PR-0001-0001-0003
-A PR-0001-0001-0001 -m comment --comment "PR-RIP-0001-0001-0001" -j SNAT --to-source 10.4.8.31
-A PR-0001-0001-0002 -m comment --comment "PR-RIP-0001-0001-0002" -j SNAT --to-source 10.4.8.31
-A PR-0001-0001-0003 -m comment --comment "PR-RIP-0001-0001-0003" -j SNAT --to-source 10.4.8.31
-A PR-0002-0001-0001 -m comment --comment "PR-RIP-0002-0001-0001" -j SNAT --to-source 10.4.8.31
-A RIP-RR-0001-0001-0001 -m comment --comment "Removes VSR-RR-0001-0001 from LB-RR-0001-0001-0001" -j SET --del-set LB-RR-0001-0001-0001 dst
-A RIP-RR-0001-0001-0001 -m comment --comment "Adds VSR-RR-0001-0001 to LB-RR-0001-0001-0003" -j SET --add-set LB-RR-0001-0001-0003 dst
-A RIP-RR-0001-0001-0001 -p tcp -m comment --comment "PREROUTING: VSR-RR-0001-0001 LB-RR-0001-0001-0001" -j DNAT --to-destination 10.4.8.203:                                83
-A RIP-RR-0001-0001-0002 -m comment --comment "Removes VSR-RR-0001-0001 from LB-RR-0001-0001-0002" -j SET --del-set LB-RR-0001-0001-0002 dst
-A RIP-RR-0001-0001-0002 -m comment --comment "Adds VSR-RR-0001-0001 to LB-RR-0001-0001-0001" -j SET --add-set LB-RR-0001-0001-0001 dst
-A RIP-RR-0001-0001-0002 -p tcp -m comment --comment "PREROUTING: VSR-RR-0001-0001 LB-RR-0001-0001-0002" -j DNAT --to-destination 10.4.8.201:                                83
-A RIP-RR-0001-0001-0003 -m comment --comment "Removes VSR-RR-0001-0001 from LB-RR-0001-0001-0003" -j SET --del-set LB-RR-0001-0001-0003 dst
-A RIP-RR-0001-0001-0003 -m comment --comment "Adds VSR-RR-0001-0001 to LB-RR-0001-0001-0002" -j SET --add-set LB-RR-0001-0001-0002 dst
-A RIP-RR-0001-0001-0003 -p tcp -m comment --comment "PREROUTING: VSR-RR-0001-0001 LB-RR-0001-0001-0003" -j DNAT --to-destination 10.4.8.201:                                83
-A RIP-RR-0002-0001-0001 -m comment --comment "Removes VSR-RR-0002-0001 from LB-RR-0002-0001-0001" -j SET --del-set LB-RR-0002-0001-0001 dst
-A RIP-RR-0002-0001-0001 -m comment --comment "Adds VSR-RR-0002-0001 to LB-RR-0002-0001-0001" -j SET --add-set LB-RR-0002-0001-0001 dst
-A RIP-RR-0002-0001-0001 -p tcp -m comment --comment "PREROUTING: VSR-RR-0002-0001 LB-RR-0002-0001-0001" -j DNAT --to-destination 10.4.8.203:                                83
-A VIP-0001 -p tcp -m tcp --dport 80 -m state --state NEW -m comment --comment "RR-0001-0001" -g VSR-RR-0001-0001
-A VIP-0002 -p tcp -m tcp --dport 80 -m state --state NEW -m comment --comment "RR-0002-0001" -g VSR-RR-0002-0001
-A VSR-RR-0001-0001 -m set --match-set LB-RR-0001-0001-0003 dst -m comment --comment "RR-NEXT-0002 DNAT to RIPip:10.4.8.201 RIPport:83" -g RI                                P-RR-0001-0001-0003
-A VSR-RR-0001-0001 -m set --match-set LB-RR-0001-0001-0002 dst -m comment --comment "RR-NEXT-0001 DNAT to RIPip:10.4.8.201 RIPport:83" -g RI                                P-RR-0001-0001-0002
-A VSR-RR-0001-0001 -m comment --comment "RR-NEXT-0003 DNAT to RIPip:10.4.8.203 RIPport:83" -g RIP-RR-0001-0001-0001
-A VSR-RR-0002-0001 -m comment --comment "RR-NEXT-0001 DNAT to RIPip:10.4.8.203 RIPport:83" -g RIP-RR-0002-0001-0001
COMMIT
ASKER CERTIFIED SOLUTION
Avatar of gheist
gheist
Flag of Belgium image

Link to home
membership
This solution is only available to members.
To access this solution, you must be a member of Experts Exchange.
Start Free Trial