mwhc
asked on
How to make this sql processing faster?
Hi,
My code below does this,
-it reads rows from a db table row by row
-every row that is reads it gets the user info and send data to the user info via a method sendText()
Now, my current code takes very long to run especially when there's lots of rows. My requests are,
-Suggestion on how to load my rows of data into memory then send one by one from mem
-Suggestion on improving sql processing of this code.
----CODE--------
/*
* Classname: FortuneTellerReadingsDeliv ery.java Description: Generic class to send
* readings to fortune teller subscribers
*
* Author: melwong Date started: 11-Mar-2004
*
* Version: 1.0
*/
package com.jm.fortuneteller;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement ;
import java.sql.ResultSet;
import java.sql.SQLException;
import org.apache.commons.dbutils .DbUtils;
import com.jm.config.DbConfig;
import com.jm.config.FileConfig;
import com.jm.config.PropertyFile Reader;
import com.jm.gateway.OutboundRes ponse;
import com.jm.util.KeywordProcess or;
public class FortuneTellerReadingsDeliv ery {
private Connection conn = null;
//Melvin:16/03/06
public FortuneTellerReadingsDeliv ery(Connec tion conn) {
this.conn = conn;
}
public void processDelivery (String message) {
int[] fortuneReadings = null;
int groupId = 0;
int serviceId = 0;
int subkeyword1Id = 0;
String lineFeed = "";
String serviceKeyword = "";
String fortuneCookie = "";
String readings = "";
String phoneNumber = "";
String sqlStr = "";
//Melvin:16/03/06
//Connection conn = null;
PreparedStatement statement = null;
ResultSet rs = null;
KeywordProcessor keyPro = null;
OutboundResponse outRes = null;
FortuneTellerUtils util = null;
keyPro = new KeywordProcessor();
outRes = new OutboundResponse();
util = new FortuneTellerUtils();
lineFeed = System.getProperty("line.s eparator") ;
//To get the subscribers table
serviceId = keyPro.getServiceId(messag e);
subkeyword1Id = keyPro.getSubkeyword1Id(me ssage, serviceId);
groupId = keyPro.getGroupId(subkeywo rd1Id);
serviceKeyword = keyPro.getServiceKeyword(g roupId);
//Melvin:16/03/06
//Get one fortune cookie for all subscribers
fortuneCookie = util.getFortuneCookie();
try {
//Melvin:16/03/06
//DbUtils.loadDriver(DbCon fig.dbDriv er);
//conn = DriverManager.getConnectio n(DbConfig .dbUrl);
sqlStr = "SELECT DISTINCT phoneNumber FROM "
+ serviceKeyword + "_fortuneteller_users"
+ " WHERE inactive = 0";
statement = conn
.prepareStatement(sqlStr);
rs = statement.executeQuery();
while (rs.next()) {
phoneNumber = rs.getString(1);
fortuneReadings = util.getFortuneReadings();
//Melvin:16/03/06
//fortuneCookie = util.getFortuneCookie();
readings = "Love:" + fortuneReadings[0] + "%" + lineFeed
+ "Money:" + fortuneReadings[1] + "%" + lineFeed
+ "Attitude:" + fortuneReadings[2] + "%";
//Melvin: 02/08/05
outRes.processText(groupId , "Your today's fortune cookie: " + fortuneCookie + lineFeed + readings, phoneNumber);
//outRes.processText(group Id, fortuneCookie + lineFeed + readings, phoneNumber);
}
} catch (SQLException e) {
throw new FortuneTellerException(thi s.getClass ().toStrin g(), e);
} finally{
DbUtils.closeQuietly(rs);
DbUtils.closeQuietly(state ment);
//Melvin:16/03/06
//DbUtils.closeQuietly(con n);
}
}
public static void main(String args[]) {
String message = "";
String absPath = "";
PropertyFileReader readProp = null;
DbConfig initDbConfig = null;
FortuneTellerReadingsDeliv ery fortuneTellerReadingsDeliv ery = null;
//Melvin:16/03/06
Connection conn = null;
if (args.length != 0) {
absPath = args[0];
for (int i = 1; i < args.length; i++) {
message += args[i];
message += " ";
}
message = message.trim();
System.out.println("absPat h=" + absPath);
System.out.println("messag e=" + message);
} else {
System.err
.println("FortuneTellerRea dingsDeliv ery: ERROR: Invalid command line arguments.");
System.err.println("\tFort uneTellerR eadingsDel ivery: Usage:");
System.err
.println("\tjava FortuneTellerReadingsDeliv ery <full webapp path> <full SMS message>");
System.err
.println("\tE.g: java FortuneTellerReadingsDeliv ery C:\\tomcat\\webapps\\jm HI FC");
System.exit(1);
}
//Read properties must process first, then initialize db
readProp = new PropertyFileReader(absPath );
//To initialize the db stuffs
initDbConfig = new DbConfig();
//To get URL for WAP Push and app full path
FileConfig.setSysConfig();
//Melvin:16/03/06
try {
DbUtils.loadDriver(DbConfi g.dbDriver );
conn = DriverManager.getConnectio n(DbConfig .dbUrl);
fortuneTellerReadingsDeliv ery = new FortuneTellerReadingsDeliv ery(conn);
fortuneTellerReadingsDeliv ery.proces sDelivery( message);
} catch (SQLException e) {
throw new FortuneTellerException("Fo rtuneTelle rReadingsD elivery", e);
} finally {
DbUtils.closeQuietly(conn) ;
}
//fortuneTellerReadingsDel ivery = new FortuneTellerReadingsDeliv ery();
//fortuneTellerReadingsDel ivery.proc essDeliver y(message) ;
}
}
---EOF CODE-----
My code below does this,
-it reads rows from a db table row by row
-every row that is reads it gets the user info and send data to the user info via a method sendText()
Now, my current code takes very long to run especially when there's lots of rows. My requests are,
-Suggestion on how to load my rows of data into memory then send one by one from mem
-Suggestion on improving sql processing of this code.
----CODE--------
/*
* Classname: FortuneTellerReadingsDeliv
* readings to fortune teller subscribers
*
* Author: melwong Date started: 11-Mar-2004
*
* Version: 1.0
*/
package com.jm.fortuneteller;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement
import java.sql.ResultSet;
import java.sql.SQLException;
import org.apache.commons.dbutils
import com.jm.config.DbConfig;
import com.jm.config.FileConfig;
import com.jm.config.PropertyFile
import com.jm.gateway.OutboundRes
import com.jm.util.KeywordProcess
public class FortuneTellerReadingsDeliv
private Connection conn = null;
//Melvin:16/03/06
public FortuneTellerReadingsDeliv
this.conn = conn;
}
public void processDelivery (String message) {
int[] fortuneReadings = null;
int groupId = 0;
int serviceId = 0;
int subkeyword1Id = 0;
String lineFeed = "";
String serviceKeyword = "";
String fortuneCookie = "";
String readings = "";
String phoneNumber = "";
String sqlStr = "";
//Melvin:16/03/06
//Connection conn = null;
PreparedStatement statement = null;
ResultSet rs = null;
KeywordProcessor keyPro = null;
OutboundResponse outRes = null;
FortuneTellerUtils util = null;
keyPro = new KeywordProcessor();
outRes = new OutboundResponse();
util = new FortuneTellerUtils();
lineFeed = System.getProperty("line.s
//To get the subscribers table
serviceId = keyPro.getServiceId(messag
subkeyword1Id = keyPro.getSubkeyword1Id(me
groupId = keyPro.getGroupId(subkeywo
serviceKeyword = keyPro.getServiceKeyword(g
//Melvin:16/03/06
//Get one fortune cookie for all subscribers
fortuneCookie = util.getFortuneCookie();
try {
//Melvin:16/03/06
//DbUtils.loadDriver(DbCon
//conn = DriverManager.getConnectio
sqlStr = "SELECT DISTINCT phoneNumber FROM "
+ serviceKeyword + "_fortuneteller_users"
+ " WHERE inactive = 0";
statement = conn
.prepareStatement(sqlStr);
rs = statement.executeQuery();
while (rs.next()) {
phoneNumber = rs.getString(1);
fortuneReadings = util.getFortuneReadings();
//Melvin:16/03/06
//fortuneCookie = util.getFortuneCookie();
readings = "Love:" + fortuneReadings[0] + "%" + lineFeed
+ "Money:" + fortuneReadings[1] + "%" + lineFeed
+ "Attitude:" + fortuneReadings[2] + "%";
//Melvin: 02/08/05
outRes.processText(groupId
//outRes.processText(group
}
} catch (SQLException e) {
throw new FortuneTellerException(thi
} finally{
DbUtils.closeQuietly(rs);
DbUtils.closeQuietly(state
//Melvin:16/03/06
//DbUtils.closeQuietly(con
}
}
public static void main(String args[]) {
String message = "";
String absPath = "";
PropertyFileReader readProp = null;
DbConfig initDbConfig = null;
FortuneTellerReadingsDeliv
//Melvin:16/03/06
Connection conn = null;
if (args.length != 0) {
absPath = args[0];
for (int i = 1; i < args.length; i++) {
message += args[i];
message += " ";
}
message = message.trim();
System.out.println("absPat
System.out.println("messag
} else {
System.err
.println("FortuneTellerRea
System.err.println("\tFort
System.err
.println("\tjava FortuneTellerReadingsDeliv
System.err
.println("\tE.g: java FortuneTellerReadingsDeliv
System.exit(1);
}
//Read properties must process first, then initialize db
readProp = new PropertyFileReader(absPath
//To initialize the db stuffs
initDbConfig = new DbConfig();
//To get URL for WAP Push and app full path
FileConfig.setSysConfig();
//Melvin:16/03/06
try {
DbUtils.loadDriver(DbConfi
conn = DriverManager.getConnectio
fortuneTellerReadingsDeliv
fortuneTellerReadingsDeliv
} catch (SQLException e) {
throw new FortuneTellerException("Fo
} finally {
DbUtils.closeQuietly(conn)
}
//fortuneTellerReadingsDel
//fortuneTellerReadingsDel
}
}
---EOF CODE-----
Don't use a PreparedSTatement, just use a standard one using createStatement()
And if your result set size is large then try adjusting the fetch size of the resukt set
And if your result set size is large then try adjusting the fetch size of the resukt set
ASKER
Hi CEHJ,
a. What is PS
>>Does the below need to be called in the loop?
>>fortuneReadings = util.getFortuneReadings();
Yes is does.
To objects,
How do I adjust the fetch size of the result set?
a. What is PS
>>Does the below need to be called in the loop?
>>fortuneReadings = util.getFortuneReadings();
Yes is does.
To objects,
How do I adjust the fetch size of the result set?
ASKER
BTW,
What do you mean by "make sure there's an index on phone number column". This is my table structure,
CREATE TABLE `just_fortuneteller_users` (
`userId` int(11) NOT NULL auto_increment,
`phoneNumber` varchar(100) NOT NULL default '',
`registrationTime` datetime NOT NULL default '0000-00-00 00:00:00',
`totalMessage` int(11) NOT NULL default '0',
`inactive` int(11) NOT NULL default '0',
PRIMARY KEY (`userId`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1;
Is this ok?
Melv
What do you mean by "make sure there's an index on phone number column". This is my table structure,
CREATE TABLE `just_fortuneteller_users`
`userId` int(11) NOT NULL auto_increment,
`phoneNumber` varchar(100) NOT NULL default '',
`registrationTime` datetime NOT NULL default '0000-00-00 00:00:00',
`totalMessage` int(11) NOT NULL default '0',
`inactive` int(11) NOT NULL default '0',
PRIMARY KEY (`userId`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1;
Is this ok?
Melv
ASKER CERTIFIED SOLUTION
membership
This solution is only available to members.
To access this solution, you must be a member of Experts Exchange.
> How do I adjust the fetch size of the result set?
rs.setFetchSize(size);
http://java.sun.com/j2se/1.5.0/docs/api/java/sql/ResultSet.html#setFetchSize(int)
rs.setFetchSize(size);
http://java.sun.com/j2se/1.5.0/docs/api/java/sql/ResultSet.html#setFetchSize(int)
ASKER
Hi,
I'm learning on indexing now. But I found out that indexing is useful especially in the WHERE statement. Will it be useful in my phoneNumber col since it's not in the WHERE statement?
I'm learning on indexing now. But I found out that indexing is useful especially in the WHERE statement. Will it be useful in my phoneNumber col since it's not in the WHERE statement?
doubt it, your better off indexing on inactive.
>>But I found out that indexing is useful especially in the WHERE statement.
>>your better off indexing on inactive.
Sorry mwhc, inactive is the correct column
>>your better off indexing on inactive.
Sorry mwhc, inactive is the correct column
a. bind the parameter to the PS
statement.setString(1, serviceKeyword);
b. make sure there's an index on phone number column
Does the below need to be called in the loop?
>>fortuneReadings = util.getFortuneReadings();