Delphi HTTPSend - HTTPMethod GET never completes

Hey,

I have this code.. occasionally it just hangs forever.. on the HTTP.HTTPMethod get call.. been 10 minutes and it still hasn't timed out, or returned anything or even failed.  What would cause the method to do this?
This code:

      if HTTP.HTTPMethod('GET',url) then //<-----here
        begin
          addlog('GET ACTY');

Open in new window


function TBaseThread.doGet(url: string; useragent: string; UseProxy:Integer) : string;
var
  HTTP:THTTPSend;
  NewUrl:String;
  n: integer;
  SL,badproxy:TStringList;
  retries:integer;

begin
repeat
application.processmessages;
sleep(100);
until paused[myindex]=false;
  result:= ''; // or use 'error codes' in the result, similar to the debug messages
  retries := 1;
  addlog('get attempts max: '+inttostr(retryattempts));
  SL:=TStringList.Create;
  HTTP := THTTPSend.Create;

  repeat
  addlog('Do GET');
  sleep(100);
    http.Clear;
    http.sock.ResetLastError;
    CookieStream.Position := 0;
    HTTP.Cookies.LoadFromStream(CookieStream);
    HTTP.UserAgent := useragent;
    HTTP.KeepAlive := true;
//    HTTP.KeepAliveTimeout := connecttimeout;
    Http.Timeout := connecttimeout;
    http.Sock.SetRecvTimeout(readtimeout);
    http.Sock.SetSendTimeout(readtimeout);
    http.Sock.HTTPTunnelTimeout := connecttimeout;
    http.Sock.SocksTimeout := connecttimeout;
    http.Sock.SetTimeout(readtimeout);

    HTTP.TargetPort := '80';
    HTTP.TargetHost := url;
    HTTP.Protocol :='1.1';
    HTTP.MimeType :='application/x-www-form-urlencoded';

    if UseProxy = 1 then
      begin
        if(ptype = 'socks4')then
          begin
            if (pUser <> '') and (pPass <> '') then
              begin
            http.Sock.SocksType := ST_socks5;
            http.Sock.SocksUsername := pUser;
            http.Sock.SocksPassword := pPass;
            end;//if username and pass

            http.Sock.SocksIP := pHost;
            http.Sock.SocksPort := pPort;


          end
        else
          begin
            if (pUser <> '') and (pPass <> '') then
              begin
                HTTP.ProxyHost := pHost;
                HTTP.ProxyPort := pPort;
                HTTP.ProxyUser := pUser;
                HTTP.ProxyPass := pPass;
              end
            else
              begin
                HTTP.ProxyHost := pHost;
                HTTP.ProxyPort := pPort;
                HTTP.ProxyUser := '';
                HTTP.ProxyPass := '';
              end;
          end;
      end;

    repeat
    sleep(100);
    addlog(copy(url,1,40));
    addlog(copy(url,41,length(url)));
    http.clear;
      if HTTP.HTTPMethod('GET',url) then
        begin
          addlog('GET ACTY');
          n := FoundLocationStrNum(HTTP.Headers);
          if (n >= 0) and (n <= HTTP.Headers.count) then
            begin
              try
               addlog('BOOm GET');
                NewUrl := StringReplace(HTTP.Headers.Strings[n],'Location: ','',[]);
              except
                addlog('Error with headers stringreplace 3');
              end;
              newurl := StringReplace(newurl, ':443', '',[rfReplaceAll, rfIgnoreCase]);
              url:= newurl;
            end;//if > 0
        end else begin
      sl.Add(URL);
      sl.SaveToFile('debug/FAILEDGET.html');
        addlog('failed get');
        end;
        addlog('http result: ' +inttostr(http.resultcode));
    until not(HTTP.Resultcode = 301) and not(HTTP.Resultcode = 302) and not(HTTP.Resultcode = 307);  //and (HTTP.Resultcode = 200)
      addlog('middle get');
  //error checking and retrying
  if((http.Sock.LastError <> 0) or (http.Document.Size <= 0)) then begin

          //addlog(pHost+ ' ' + pPort + ' ' +puser+' '+ppass);
          addlog('GET WinSock Error: '+inttostr(http.Sock.LastError) + ' ' + http.Sock.GetErrorDescEx);
          http.Clear;
          inc(retries);
          addlog('Retrying connection...GET');
          sleep(1000);

    end else begin
      //Clear SL String List, and load http into it
      sl.Clear;
      http.Document.Position := 0;
      SL.LoadFromStream(HTTP.Document);
  if(pos('Redirecting...',sl.Text) > 0) then begin
    addlog('redirecting get');
      newurl := findvalue2('url=',sl.Text,'"');
      http.Clear;
      HTTP.HTTPMethod('GET',newurl);
      sl.Clear;
      http.Document.Position := 0;
      SL.LoadFromStream(HTTP.Document);
    end;//redirecting

    end;
  until ((http.Sock.LastError = 0) or (http.Document.Size > 0)) or (retries > retryattempts);


  //save new cookie to stream
  CookieStream.Position := 0;
  if CookieStream <> nil then
    HTTP.Cookies.SaveToStream(CookieStream);

  //write debug
  if(debugmode) then
    begin
      if(not(directoryexists('debug')))then
        begin
          createdir('debug');
        end;
     //   addlog('write debug');
   //   sl.Add(URL);
      sl.SaveToFile('debug/GET_'+email+'_Action_'+inttostr(debugpos)+'_.html');
      inc(debugpos);
    end;

  //return result text
  addlog('return get html');
  Result:= SL.Text;

  SL.Free;
  HTTP.Free;
end;

Open in new window

nickdelphi777Asked:
Who is Participating?
 
jimyXCommented:
Make sure to use compatible SSL Lib. with indy & delphi versions.
Which Delphi/Indy versions you are using please?
0
 
nickdelphi777Author Commented:
Happens randomly.. on different urls.. what is going on? Its like HTTPMethod never returns a result. Just hangs.
0
 
jimyXCommented:
Does it happen on the exact same URLs?

Try to log the HTTP Status by using the OnStatus event.

Different prospective, have you tried Tidhttp (indy package)?
0
Free Tool: Port Scanner

Check which ports are open to the outside world. Helps make sure that your firewall rules are working as intended.

One of a set of tools we are providing to everyone as a way of saying thank you for being a part of the community.

 
nickdelphi777Author Commented:
Happens on different urls.. yeah

I'm trying to code a Idhttp function that replicates that one.. but its going to take me awhile. :( so frustrating
0
 
nickdelphi777Author Commented:
idHTTP is driving me nuts.. I have the 2 dlls i need in the same folder and its saying i'm missing libraries for SSL.

var
http:TIDHttp;
html:String;
code:integer;
MySSL:TiDSSLIOHandlerSocket;
begin
http := tIdHttp.Create(nil);

//setup

http.HandleRedirects := true;
http.AllowCookies := true;
try
http.IOHandler := TiDSSLIOHandlerSocket.Create(http);

except
 on e: exception do begin
  showmessage(e.Message);
 end;
end;
try
html := http.Get('http://yahoo.com');
except
 on E: eIdHTTPProtocolException do begin
  code := http.ResponseCode;
  showmessage('Error: ' +inttostr(code));
 end;
 on E: Exception do begin
 showmessage('Real error: '+e.Message);
 end;
end;
http.free;
memo3.text := html;

Open in new window

0
 
nickdelphi777Author Commented:
I fixed it.. but have new issue.. I store my cookies in text files, i load the cookie before the request, and save the new cookies to the text file after the request. However. If you look at my new Idhttp code the cookie is being loaded but the session does not work. So the cookie is not being loaded correctly I've tried a lot of things.

function TBaseThread.doGet(url: string; useragent: string; UseProxy:Integer) : string;
var
http:TIDHttp;
html:String;
code:integer;
myssl: TIdSSLIOHandlerSocket;
TheCookie:TidCookieManager;
i:integer;
boolretry:boolean;
retries:integer;
sl:TStringList;
label
retry;
begin
Result := '';
http := tIdHttp.Create(nil);
TheCookie := TidCookieManager.Create(HTTP);

//setup
retries := 1;
HTTP.Request.ContentType:='application/x-www-form-urlencoded';
HTTP.Request.Connection:='Keep-Alive';
http.Request.Method := hmGet;
HTTP.Request.UserAgent := useragent;
HTTP.HandleRedirects := True;
HTTP.AllowCookies := True;
HTTP.CookieManager := TheCookie;
http.Port := 80;
http.Host := url;
http.ProtocolVersion := pv1_1;


myssl := Tidssliohandlersocket.Create(http);
myssl.SSLOptions.Method := sslvTLSv1;
http.IOHandler := myssl;

//load cookie
try
for i := 0 to Cookie.Count-1 do begin
thecookie.CookieCollection.AddSrcCookie(Cookie.Strings[i]);
HTTP.Request.CustomHeaders.Values['Cookie']:=Cookie.Strings[i];
//HTTP.Request.CustomHeaders.add('Cookie:'+Cookie.Strings[i]);//not working
addlog(Cookie.Strings[i]);
end;
except
addlog('Error Loading Cookie in GET');
end;
boolretry := false;

//Set Time Outs
http.ReadTimeout := ReadTimeout;
http.ConnectTimeout := ConnectTimeout;

//setup proxy
    if UseProxy = 1 then
      begin
        if(ptype = 'socks4')then
          begin
            if (pUser <> '') and (pPass <> '') then
              begin
            http.Socket.SocksInfo.Authentication := saUsernamePassword;
            http.Socket.SocksInfo.Host := pHost;
            http.Socket.SocksInfo.Port := Strtoint64(pPort);
            http.Socket.SocksInfo.Username := pUser;
            http.Socket.SocksInfo.Password := pPass;
            http.Socket.SocksInfo.Version := svSocks5;
            end;//if username and pass
            http.Socket.SocksInfo.Authentication := saNoAuthentication;
            http.Socket.SocksInfo.Host := pHost;
            http.Socket.SocksInfo.Port := Strtoint64(pPort);
            http.Socket.SocksInfo.Version := svSocks4;
          end else begin
            if (pUser <> '') and (pPass <> '') then
              begin
                http.ProxyParams.BasicAuthentication := true;
                http.ProxyParams.ProxyServer := pHost;
                http.ProxyParams.ProxyPort := Strtoint64(pPort);
                http.ProxyParams.ProxyUsername := pUser;
                http.ProxyParams.ProxyPassword := pPass;
              end else begin
                http.ProxyParams.ProxyServer := pHost;
                http.ProxyParams.ProxyPort := Strtoint64(pPort);
                http.ProxyParams.ProxyUsername := '';
                http.ProxyParams.ProxyPassword := '';
              end;
          end;
      end;


retry:
try
Result := http.Get(url);
except
 on E: Exception do begin
code := http.ResponseCode;
Addlog(Inttostr(code) + ' ' + e.Message);
boolretry := false;
if(retries < retryattempts) then begin
addlog('retrying');
 inc(retries);
 sleep(1000);
 boolretry := true;
end;
end;
end;

//retry boolean
if(boolretry) then goto retry;

//save cookie
try
for i := 0 to TheCookie.CookieCollection.Count-1 do begin
//HTTP.Request.RawHeaders.Add('Cookie'+HTTP.Request.RawHeaders.NameValueSeparator+TheCookie.CookieCollection.Items[i].CookieText);
//addlog('Cookie'+HTTP.Request.RawHeaders.NameValueSeparator+TheCookie.CookieCollection.Items[i].CookieText);
end;
except
Addlog('Error in saving cookie to global GET');
end;


//write debug
  if(debugmode) then
    begin
      if(not(directoryexists('debug')))then
        begin
          createdir('debug');
        end;
      sl := tstringlist.create;
      sl.Text := result;
      sl.SaveToFile('debug/GET_'+email+'_Action_'+inttostr(debugpos)+'_.html');
      sl.free;
      inc(debugpos);
    end;

TheCookie.Free;
http.free;
end;

Open in new window

0
 
nickdelphi777Author Commented:
Nevermind fixed it.. with non stop playing around with the code haha

try
for i := 0 to Cookie.Count-1 do begin
cookiestr := cookiestr+ copy(Cookie.Strings[i],1,pos(';',Cookie.Strings[i]))+ ' ';
end;
HTTP.Request.CustomHeaders.add('Cookie: '+cookiestr);//not working
except
addlog('Error Loading Cookie in GET');
end;

Open in new window

0
 
jimyXCommented:
Good determination.

Did it solve the earlier issue of frozen "GET URL"?
0
 
nickdelphi777Author Commented:
Yes it did thanks
0
Question has a verified solution.

Are you are experiencing a similar issue? Get a personalized answer when you ask a related question.

Have a better answer? Share it in a comment.

All Courses

From novice to tech pro — start learning today.