asked on

Delphi Out of Memory Leak with SL.LoadFromStream(HTTP.Document);

Hey everyone,

After awhile of running my multi threaded program.. at around 170 mbs used of RAM it will crash and say Out of Memory in Madexcept's bug report.

It is telling me this line is the problem:
SL.LoadFromStream(HTTP.Document);

Can you please review my code and tell me why this could be happening?

function TBaseThread.doGet(url: string; useragent: string; UseProxy:Integer) : string;
var
  HTTP:THTTPSend;
  NewUrl:String;
  n: integer;
  SL:TStringList;
  retries:integer;
label
retry,redirectagain;
begin
retries := 1;
SL:=TStringList.Create;
HTTP := THTTPSend.Create;

   retry:
    CookieStream.Position := 0;
    HTTP.Cookies.LoadFromStream(CookieStream);
    http.document.Clear;
    HTTP.UserAgent := useragent;
    HTTP.KeepAlive := true;
    HTTP.KeepAliveTimeout := connecttimeout;
    HTTP.Timeout := connecttimeout;
    HTTP.TargetPort := '80';
    HTTP.TargetHost := url;
    HTTP.Protocol :='1.1';
    HTTP.MimeType :='application/x-www-form-urlencoded';
    http.Sock.SetRecvTimeout(readtimeout);


if UseProxy = 1 then begin
 if(ptype = 'socks4')then begin
      if (pUser <> '') and (pPass <> '') then begin
          http.Sock.SocksUsername := pUser;
          http.Sock.SocksPassword := pPass;
      end;
      http.Sock.SocksIP := pHost;
      http.Sock.SocksPort := pPort;
      http.Sock.SocksTimeout := readtimeout;
      http.Sock.HTTPTunnelTimeout := connecttimeout;
     end else begin
     http.Sock.HTTPTunnelTimeout := connecttimeout;
 if (pUser <> '') and (pPass <> '') then begin
    HTTP.ProxyHost := pHost;
    HTTP.ProxyPort := pPort;
    HTTP.ProxyUser := pUser;
    HTTP.ProxyPass := pPass;
    end else begin
    HTTP.ProxyHost := pHost;
    HTTP.ProxyPort := pPort;
    HTTP.ProxyUser := '';
    HTTP.ProxyPass := '';
    end;
     end;

end;

if HTTP.HTTPMethod('GET',url) then begin
        case HTTP.Resultcode of
        301, 302, 307: begin
 n := FoundLocationStrNum(HTTP.Headers);
 if (n >= 0) and (n <= HTTP.Headers.count) then begin
          try
            NewUrl := StringReplace(HTTP.Headers.Strings[n],'Location: ','',[]);
          except
            addlog('Error with headers stringreplace 3');
          end;
          newurl := StringReplace(newurl, ':443', '',[rfReplaceAll, rfIgnoreCase]);
    redirectagain:
    http.Document.Clear;
    if HTTP.HTTPMethod('GET',newurl) then begin
        case HTTP.Resultcode of
        301, 302, 307: begin
        n := FoundLocationStrNum(HTTP.Headers);
        if (n >= 0) and (n <= HTTP.Headers.count) then begin
		  
          try
            NewUrl := StringReplace(HTTP.Headers.Strings[n],'Location: ','',[]);
          except
            addlog('Error with headers stringreplace 4');
          end;
		  
         newurl := StringReplace(newurl, ':443', '',[rfReplaceAll, rfIgnoreCase]);

         goto redirectagain;
        end;//if n>0
		
        end;//case2
		end;
		end;


         end;//if n>0
        end;//case
        end;
		end;


//Clear SL String List, and load http into it
sl.Clear;
http.Document.Position := 0;
SL.LoadFromStream(HTTP.Document); //<----memory leak "out of memory" occurring here according to mad except


if(pos('Redirecting...',sl.Text) > 0) then begin
newurl := findvalue2('url=',sl.Text,'"');
http.document.Clear;
HTTP.HTTPMethod('GET',newurl);
sl.Clear;
http.Document.Position := 0;
SL.LoadFromStream(HTTP.Document);
end;


//error checking and retrying
if((http.Sock.LastError <> 0) or (http.Document.Size <= 0)) then begin
addlog('GET WinSock Error: '+inttostr(http.Sock.LastError) + ' ' + http.Sock.GetErrorDescEx);
http.sock.ResetLastError;
if (retries <= retryattempts) then begin
inc(retries);
addlog('Retrying connection...GET');
sleep(1000);
goto retry;
end;
end;



//save new cookie to stream
CookieStream.Position := 0;
if CookieStream <> nil then HTTP.Cookies.SaveToStream(CookieStream);

//write debug
if(debugmode) then begin
if(not(directoryexists('debug')))then begin
 createdir('debug');
end;
sl.Add(URL);
sl.SaveToFile('debug/GET_'+email+'_Action_'+inttostr(debugpos)+'_.html');
inc(debugpos);
end;

//return result text
doGet := SL.Text;

SL.Free;
HTTP.Free;
end;

Open in new window

jimyX

First of all, avoid using Goto. Use Repeat-Until instead, unless you really, really can not.

function TBaseThread.doGet(url: string; useragent: string; UseProxy:Integer) : string;
...
begin
  ...
  Repeat
    
  Until some condition;
// in the case of "Goto retry;" condition might be: ((http.Sock.LastError = 0) or (http.Document.Size > 0)) or (retries > retryattempts)
  ...
end;

Open in new window

Now, the following might not be direct solution to solve the out of memory error, but might narrow the possibilities:
These lines:
//Clear SL String List, and load http into it
sl.Clear;
http.Document.Position := 0;
SL.LoadFromStream(HTTP.Document);

You put the above lines (as per the excerpt code, line 99, 100 & 101) where you do not know, for sure, if there is result to load on sl or not.
Better move them to the end (after line 124) where you know for sure you ended successfully and you can assign the http.doc content to the sl.

Final code might look something like:

function TBaseThread.doGet(url: string; useragent: string; UseProxy:Integer) : string;
var
  HTTP:THTTPSend;
  NewUrl:String;
  n: integer;
  SL:TStringList;
  retries:integer;
begin
  result:= ''; // or use 'error codes' in the result, similar to the debug messages
  retries := 1;
  SL:=TStringList.Create;
  HTTP := THTTPSend.Create;

  repeat
    CookieStream.Position := 0;
    HTTP.Cookies.LoadFromStream(CookieStream);
    http.document.Clear;
    HTTP.UserAgent := useragent;
    HTTP.KeepAlive := true;
    HTTP.KeepAliveTimeout := connecttimeout;
    HTTP.Timeout := connecttimeout;
    HTTP.TargetPort := '80';
    HTTP.TargetHost := url;
    HTTP.Protocol :='1.1';
    HTTP.MimeType :='application/x-www-form-urlencoded';
    http.Sock.SetRecvTimeout(readtimeout);

    if UseProxy = 1 then
      begin
        if(ptype = 'socks4')then
          begin
            if (pUser <> '') and (pPass <> '') then
              begin
                http.Sock.SocksUsername := pUser;
                http.Sock.SocksPassword := pPass;
              end;

            http.Sock.SocksIP := pHost;
            http.Sock.SocksPort := pPort;
            http.Sock.SocksTimeout := readtimeout;
            http.Sock.HTTPTunnelTimeout := connecttimeout;
          end
        else
          begin
            http.Sock.HTTPTunnelTimeout := connecttimeout;
            if (pUser <> '') and (pPass <> '') then
              begin
                HTTP.ProxyHost := pHost;
                HTTP.ProxyPort := pPort;
                HTTP.ProxyUser := pUser;
                HTTP.ProxyPass := pPass;
              end
            else
              begin
                HTTP.ProxyHost := pHost;
                HTTP.ProxyPort := pPort;
                HTTP.ProxyUser := '';
                HTTP.ProxyPass := '';
              end;
          end;
      end;

    repeat
      if HTTP.HTTPMethod('GET',url) then
        begin
          n := FoundLocationStrNum(HTTP.Headers);
          if (n >= 0) and (n <= HTTP.Headers.count) then
            begin
              try
                NewUrl := StringReplace(HTTP.Headers.Strings[n],'Location: ','',[]);
              except
                addlog('Error with headers stringreplace 3');
              end;
              newurl := StringReplace(newurl, ':443', '',[rfReplaceAll, rfIgnoreCase]);
              url:= newurl;
            end;
        end;
    until not (HTTP.Resultcode in [301, 302, 307]);  //and (HTTP.Resultcode = 200)

  if(pos('Redirecting...',sl.Text) > 0) then
    begin
      newurl := findvalue2('url=',sl.Text,'"');
      http.document.Clear;
      HTTP.HTTPMethod('GET',newurl);
      sl.Clear;
      http.Document.Position := 0;
      SL.LoadFromStream(HTTP.Document);
    end;

  //error checking and retrying
  if((http.Sock.LastError <> 0) or (http.Document.Size <= 0)) then
    begin
      addlog('GET WinSock Error: '+inttostr(http.Sock.LastError) + ' ' + http.Sock.GetErrorDescEx);
      http.sock.ResetLastError;
      if (retries <= retryattempts) then
        begin
          inc(retries);
          addlog('Retrying connection...GET');
          sleep(1000);
        end;
    end
  else   // or use if success-condition then
    begin
      //Clear SL String List, and load http into it
      sl.Clear;
      http.Document.Position := 0;
      SL.LoadFromStream(HTTP.Document);
    end;
  until ((http.Sock.LastError = 0) or (http.Document.Size > 0)) or (retries > retryattempts);


  //save new cookie to stream
  CookieStream.Position := 0;
  if CookieStream <> nil then
    HTTP.Cookies.SaveToStream(CookieStream);

  //write debug
  if(debugmode) then
    begin
      if(not(directoryexists('debug')))then
        begin
          createdir('debug');
        end;
      sl.Add(URL);
      sl.SaveToFile('debug/GET_'+email+'_Action_'+inttostr(debugpos)+'_.html');
      inc(debugpos);
    end;

  //return result text
  Result:= SL.Text;

  SL.Free;
  HTTP.Free;
end;

Open in new window

You may need to tune it a bit more.

After executing if you get that error again make sure to show the debug messages you implemented.

nickdelphi777

ASKER

Still the same exact issue, it says Out of Memory after abut 5 - 10 minutes of running.. ram seems stable but it still happens.

your code I used:

function TBaseThread.doGet(url: string; useragent: string; UseProxy:Integer) : string;
var
  HTTP:THTTPSend;
  NewUrl:String;
  n: integer;
  SL:TStringList;
  retries:integer;
begin
  result:= ''; // or use 'error codes' in the result, similar to the debug messages
  retries := 1;
  SL:=TStringList.Create;
  HTTP := THTTPSend.Create;

  repeat
    CookieStream.Position := 0;
    HTTP.Cookies.LoadFromStream(CookieStream);
    http.document.Clear;
    HTTP.UserAgent := useragent;
    HTTP.KeepAlive := true;
    HTTP.KeepAliveTimeout := connecttimeout;
    HTTP.Timeout := connecttimeout;
    HTTP.TargetPort := '80';
    HTTP.TargetHost := url;
    HTTP.Protocol :='1.1';
    HTTP.MimeType :='application/x-www-form-urlencoded';
    http.Sock.SetRecvTimeout(readtimeout);

    if UseProxy = 1 then
      begin
        if(ptype = 'socks4')then
          begin
            if (pUser <> '') and (pPass <> '') then
              begin
                http.Sock.SocksUsername := pUser;
                http.Sock.SocksPassword := pPass;
              end;

            http.Sock.SocksIP := pHost;
            http.Sock.SocksPort := pPort;
            http.Sock.SocksTimeout := readtimeout;
            http.Sock.HTTPTunnelTimeout := connecttimeout;
          end
        else
          begin
            http.Sock.HTTPTunnelTimeout := connecttimeout;
            if (pUser <> '') and (pPass <> '') then
              begin
                HTTP.ProxyHost := pHost;
                HTTP.ProxyPort := pPort;
                HTTP.ProxyUser := pUser;
                HTTP.ProxyPass := pPass;
              end
            else
              begin
                HTTP.ProxyHost := pHost;
                HTTP.ProxyPort := pPort;
                HTTP.ProxyUser := '';
                HTTP.ProxyPass := '';
              end;
          end;
      end;

    repeat
      if HTTP.HTTPMethod('GET',url) then
        begin
          n := FoundLocationStrNum(HTTP.Headers);
          if (n >= 0) and (n <= HTTP.Headers.count) then
            begin
              try
                NewUrl := StringReplace(HTTP.Headers.Strings[n],'Location: ','',[]);
              except
                addlog('Error with headers stringreplace 3');
              end;
              newurl := StringReplace(newurl, ':443', '',[rfReplaceAll, rfIgnoreCase]);
              url:= newurl;
            end;
        end;
    until not(HTTP.Resultcode = 301) and not(HTTP.Resultcode = 302) and not(HTTP.Resultcode = 307);  //and (HTTP.Resultcode = 200)


  //error checking and retrying
  if((http.Sock.LastError <> 0) or (http.Document.Size <= 0)) then
    begin
      addlog('GET WinSock Error: '+inttostr(http.Sock.LastError) + ' ' + http.Sock.GetErrorDescEx);
      http.sock.ResetLastError;
      if (retries <= retryattempts) then
        begin
          inc(retries);
          addlog('Retrying connection...GET');
          sleep(1000);
        end;
    end
  else   // or use if success-condition then
    begin
      //Clear SL String List, and load http into it
      sl.Clear;
      http.Document.Position := 0;
      SL.LoadFromStream(HTTP.Document);
  if(pos('Redirecting...',sl.Text) > 0) then
    begin
      newurl := findvalue2('url=',sl.Text,'"');
      http.document.Clear;
      HTTP.HTTPMethod('GET',newurl);
      sl.Clear;
      http.Document.Position := 0;
      SL.LoadFromStream(HTTP.Document);
    end;

    end;
  until ((http.Sock.LastError = 0) or (http.Document.Size > 0)) or (retries > retryattempts);


  //save new cookie to stream
  CookieStream.Position := 0;
  if CookieStream <> nil then
    HTTP.Cookies.SaveToStream(CookieStream);

  //write debug
  if(debugmode) then
    begin
      if(not(directoryexists('debug')))then
        begin
          createdir('debug');
        end;
      sl.Add(URL);
      sl.SaveToFile('debug/GET_'+email+'_Action_'+inttostr(debugpos)+'_.html');
      inc(debugpos);
    end;

  //return result text
  Result:= SL.Text;

  SL.Free;
  HTTP.Free;
end;

Open in new window

You can see the last few lines it calls in the stack, the line number it shows goes directly to LoadFromStream:

jimyX

> it says Out of Memory after abut 5 - 10 minutes of running
I suspect that you do not have a problem in your code, it's all about managing your threads.

See my reply in your other thread.

MerijnB

What Delphi version are you using?

ASKER CERTIFIED SOLUTION

jimyX

membership

This solution is only available to members.

To access this solution, you must be a member of Experts Exchange.

Start Free Trial