Split the files

Hi Chris (Wint),

If you remember you helped me few months ago with the following code:
Now I got a problem whenever the number of the files to be read and spilt is more than for example 150 or the file has so many rows (more than a million lines) it's going to almost freeze the system by using memory and system will hang. Would you please help me to fix this problem?

Thank you so much,
Mark

using System;
using System.Collections;
using System.Collections.Specialized;
using System.IO;
using System.Threading;
using System.Xml;

namespace MultiThreadFileReader
{
      /////////////////////////////////////////////////////
      // Program Class: Launches the program.
      /////////////////////////////////////////////////////
      internal class Program
      {
            private static void Main(string[] args)
            {
                  try
                  {
                        new MultiThreadFileReader(args);
                  }
                  catch (System.Exception e)
                  {
                        Console.WriteLine("An exception was caught whilst running the MultiThreadFileReader:"
                              + Environment.NewLine + e.ToString());
                  }
                  ////---Console.WriteLine("Press ENTER to exit.");
                  ///----Console.ReadLine();
            }
      }


      /////////////////////////////////////////////////////
      // MultiThreadFileReader Class: Runs the reading etc.
      /////////////////////////////////////////////////////
      internal class MultiThreadFileReader
      {
            /// <summary>Run the MultiThreadFileReader.</summary>
            /// <param name="args">The directory name to read files from.</param>
            internal MultiThreadFileReader(string[] args)
            {
                  DateTime startTime = DateTime.Now;
                  string dirToRead = null;
                  string settingsFile = null;
                  int numThreads = -1;

                  #region Initial Checks
                  //Usage:
                  //  MyProg.exe -d <dirToRead> -s <settingsFile> [-t ##]
                  //We also need to check that the number of args is correct, we *need* a settings file and *need* a dir to read.
                  //-t / -threads is the only arg we don't need (as we can default it).

                  //NEW!!

                  if (args.Length < 4)
                  {
                        DisplayUsageMessage();
                        return;
                  }

                  for (int i = 0; i < args.Length; i++)
                  {
                        switch (args[i])
                        {
                                    //The user can use either -t or -threads to set the number of threads to use.
                              case "-t":
                              case "-threads":
                                    i++;
                                    numThreads = Convert.ToInt32(args[i]);
                                    break;
                                    //The user can use -d or -dir to indicate the directory to read in from.
                              case "-d":
                              case "-dir":
                                    i++;
                                    dirToRead = args[i];
                                    break;
                                    //The user can use -s or -settings to indicate the location of the settings file to read in.
                              case "-s":
                              case "-settings":
                                    i++;
                                    settingsFile = args[i];
                                    break;
                        }
                  }

                  //Have the settings file and directory to read from been set?
                  if (settingsFile == null || settingsFile.Length == 0 || dirToRead == null || dirToRead.Length == 0)
                  {
                        DisplayUsageMessage();
                        return;
                  }

                  //Does the settings file exist?
                  if (!File.Exists(settingsFile))
                  {
                        Console.WriteLine("The settings file given: " + settingsFile + ", doesn't exist, I can't continue!");
                        return;
                  }

                  //Does the directory to read exist?
                  if (!Directory.Exists(dirToRead))
                  {
                        Console.WriteLine("This Path (" + dirToRead + ") does not exists.");
                        return;
                  }

                  //Get the files.
                  string[] fileEntries = Directory.GetFiles(dirToRead);
                  if (fileEntries == null || fileEntries.Length == 0)
                  {
                        Console.Out.WriteLine("The path '" + dirToRead + "' contains no files, there is nothing for me to read!!");
                        return;
                  }

                  //Create a new StringCollection (an ArrayList, but specialized to handle only strings),
                  //this means we can avoid casting and use it type-safely.
                  StringCollection filesToRead = new StringCollection();

                  //If the filename has '.log' anywhere in it, it will be read.
                  //i.e. my.LOG.txt, my.LOG, my.LOGarithm, etc. Not just the extension!!
                  //This can be changed by using 'FileInfo' classes instead of just strings.
                  foreach (string fileName in fileEntries)
                  {
                        if (fileName.IndexOf(".log") != -1)
                              filesToRead.Add(fileName);
                  }

                  if (filesToRead.Count == 0)
                  {
                        Console.Out.WriteLine("The path '" + dirToRead + "' contains no files I can read (log files), there is nothing I can do!!");
                        return;
                  }
                  #endregion Initial Checks


                  #region Read in the Settings file and create a collection of paths.
                  //Maybe make this typesafe by making extending collectionbase to make a PathCollection.
                  //Or in C# 2.0 return a List<Path> collection...
                  ArrayList myPaths = ExtractPathsFromSettingsFile(settingsFile);
                  if (myPaths == null || myPaths.Count == 0)
                  {
                        Console.WriteLine("I have no 'Paths' to run from, perhaps the settings file was malformed?");
                        return;
                  }
                  #endregion Read in the Settings file and create a collection of paths.

                  #region Create New Threads & Do Reading
                  //Create a new thread reader for each of the files. NB, we aren't going to be starting them until later!
                  FileOpenerThread[] openFiles = new FileOpenerThread[filesToRead.Count];
                  for (int i = 0; i < filesToRead.Count; i++)
                        if (filesToRead[i] != null && filesToRead[i].Length != 0)
                              openFiles[i] = new FileOpenerThread(filesToRead[i], myPaths);

                  //This is where we set any negative value for the number of threads to be the number of files to read.
                  //i.e. if the user passes in 0, or any negative value, they will get a new thread for each file.
                  if (numThreads <= 0 || numThreads > filesToRead.Count)
                        numThreads = filesToRead.Count;

                  int started = 0, runningThreads = 0, end = started + numThreads;

                  //Here we want to start numThreads, so we go through the threads we have, starting them until we reach the
                  //numThreads limit, OR we go past the number of files in the directory!
                  for (int i = started; i < end; i++)
                  {
                        if (i >= openFiles.Length) //Are we attempting to open more threads than files?
                              break;
                        openFiles[i].Start();
                        started++;
                        runningThreads++;
                  }

                  //A local variable to make the rest of this program wait until all the files are written.
                  bool finished = false;
                  while (!finished)
                  {
                        finished = true;
                        //Sleep to prevent 100% CPU usage.
                        Thread.Sleep(300);
                        int nonFinishedThreads = 0;
                        //Check if the thread has finished...
                        foreach (FileOpenerThread thread in openFiles)
                        {
                              if (!thread.Finished)
                              {
                                    nonFinishedThreads++;
                                    finished = false;
                                    if (nonFinishedThreads == numThreads)
                                          break;
                              }
                              else if (thread.Finished && !thread.Checked)
                              {
                                    //If the variable 'start' (which is defined up about 30 lines) is >= openFiles.Length
                                    //then we do nothing, else we need to start another thread, as we know the previous one
                                    //has finished.
                                    runningThreads--;
                                    thread.BeenChecked();
                                    thread.Stop(); //Just incase...
                                    if (!(started >= openFiles.Length))
                                    {
                                          runningThreads++;
                                          openFiles[started].Start();
                                          started++;
                                    }
                              }
                        }
                  }
                  #endregion Create New Threads & Do Reading

                  #region Finalise
                  /* All the threads have finished, so now we can commit all the 'Paths'.
                   * We do it this way so that the creation of a StreamWriter occurs only once for each
                   * output file.
                   * */
                  foreach (Path path in myPaths)
                        path.Commit();
                  TimeSpan timeTaken = DateTime.Now - startTime;
                  Console.Out.WriteLine("NumThreads: " + numThreads + ", Time taken (ms): " + timeTaken.TotalMilliseconds);
                  Console.Out.WriteLine("=====================================================================");
                  Console.Out.WriteLine("Finished reading files");
                        ////, press [ENTER] to exit.");
                  #endregion Finalise
            }

            //NEW!!
            /// <summary>Extract the 'Paths' from the settings file given.</summary>
            /// <param name="settingsFilename">The filename of the settings file.</param>
            /// <returns>A collection of paths, or null if the file doesn't exist.</returns>
            private ArrayList ExtractPathsFromSettingsFile(string settingsFilename)
            {
                  ArrayList myPaths = new ArrayList();

                  if (!File.Exists(settingsFilename))
                        return null;

                  string filename = "";
                  string search1 = "";
                  string search2 = "";
                  long fileSize = 0;

                  XmlTextReader sReader = new XmlTextReader(settingsFilename);
                  while (sReader.Read())
                  {
                        switch (sReader.NodeType)
                        {
                              case XmlNodeType.Element:
                                    if (sReader.Name.Equals("path"))
                                    {
                                          if (sReader.AttributeCount != 4)
                                                break;

                                          sReader.MoveToAttribute("output");
                                          filename = sReader.Value;

                                          sReader.MoveToAttribute("search1");
                                          search1 = sReader.Value;

                                          sReader.MoveToAttribute("search2");
                                          search2 = sReader.Value;

                                          sReader.MoveToAttribute("filesize");
                                          fileSize = Convert.ToInt64(sReader.Value);
                                          myPaths.Add(new Path(filename, search1, search2, fileSize));
                                    }
                                    //reset all vals:
                                    filename = search1 = search2 = "";
                                    fileSize = 0;
                                    break;
                        }
                  }
                  return myPaths;
            }

            //NEW!!
            /// <summary>Just display a the usage message.</summary>
            private void DisplayUsageMessage()
            {
                  Console.Out.WriteLine(
                        "Usage:\r\n------------------------------------\r\n" +
                        "My.Exe -d <dir> -s <settings> [-t ##]\r\n" +
                        "\t -d <dir> = the directory you want to read the '.log' files from." +
                        "\t -s <settings> = The settings the file to read in." +
                        "\t -t ## = optional, the number of threads to read the log files on.");
            }
      }

      /////////////////////////////////////////////////////
      // Path Class: Writes the file and contains search info.
      /////////////////////////////////////////////////////
      /// <summary>A class to store the search strings, filenames etc for use in 'Arrays' method of the 'FileOpenerThread' class.</summary>
      internal class Path
      {
            #region Member Variables & Properties
            /// <summary>The size that the file is allowed to get (in bytes) before a new file is opened.</summary>
            private long _FileSize;

            /// <summary>The output FileInfo instance that is written to.</summary>
            private FileInfo _OutputFileInfo;

            /// <summary>All the lines read in previously.</summary>
            private StringCollection _Lines = new StringCollection();

            /// <summary>The first string that must be found in the line to be checked.</summary>
            private string _SearchString1;

            /// <summary>The second string that must be found in the line to be checked.</summary>
            private string _SearchString2;
            #endregion Member Variables & Properties

            #region Constructor
            /// <summary>Create a new Path instance.</summary>
            /// <param name="filename">The filename to output results to.</param>
            /// <param name="search1">The first search string.</param>
            /// <param name="search2">The second search string.</param>
            /// <param name="fileSize">The size in bytes the file can be before it needs to be closed and a new file opened.</param>
            public Path(string filename, string search1, string search2, long fileSize)
            {
                  _OutputFileInfo = new FileInfo(filename);
                  _SearchString1 = search1;
                  _SearchString2 = search2;
                  if (fileSize > 0)
                        _FileSize = fileSize;
                  else
                        _FileSize = long.MaxValue;
            }
            #endregion Constructor

            #region Checking a Line
            /// <summary>Check if a given line contains both of the SearchStrings.</summary>
            /// <param name="line">The line to check.</param>
            public void CheckLine(string line)
            {
                  if (line == null || _SearchString1 == null || _SearchString2 == null)
                        return;
                  if (line.IndexOf(_SearchString1) != -1 && line.IndexOf(_SearchString2) != -1)
                        AppendText(line);
            }

            /// <summary>Append the line to the private Collection of lines.</summary>
            /// <param name="line">The line to append.</param>
            private void AppendText(string line)
            {
                  lock (_Lines)
                  {
                        if (_Lines == null)
                              _Lines = new StringCollection();
                        _Lines.Add(line);
                  }
            }

            /// <summary>Commit *all* the lines read in to the files.</summary>
            public void Commit()
            {
                  if (_Lines == null || _Lines.Count <= 0 || _OutputFileInfo == null)
                        return;

                  _OutputFileInfo = new FileInfo(GenerateFileName(_OutputFileInfo.FullName));
                  Console.Out.WriteLine("Writing " + _Lines.Count + " lines to: " + _OutputFileInfo.FullName);
                  foreach (string line in _Lines)
                        this.WriteLine(line);
            }

            //NEW!!
            /// <summary>Generate a new filename (if required).</summary>
            /// <param name="filenameToCheck">The filename to check.</param>
            /// <returns>The new filename if needed.</returns>
            private string GenerateFileName(string filenameToCheck)
            {
                  //File doesn't exist, so can't be bigger than limit.
                  if (!File.Exists(filenameToCheck))
                        return filenameToCheck;

                  //File does exist, check size:
                  FileInfo fi = new FileInfo(filenameToCheck);

                  //Size ok, so return the name.
                  if (!(fi.Length > _FileSize))
                        return filenameToCheck;

                  //Size isn't ok, so we need a new filename:
                  //Check if this is a file with an _# on it...
                  string name = "";
                  int fileNumber = 0;

                  int indexOfUnderscore = fi.FullName.LastIndexOf("_");
                  if (indexOfUnderscore == -1)
                        name = fi.FullName.Substring(0, fi.FullName.Length - fi.Extension.Length);
                  else
                  {
                        name = fi.FullName.Substring(0, indexOfUnderscore);
                        fileNumber = Convert.ToInt32(fi.FullName.Substring(indexOfUnderscore + 1, fi.FullName.Length - (indexOfUnderscore + 1) - fi.Extension.Length));
                  }

                  fileNumber++;
                  return GenerateFileName(name + "_" + fileNumber + fi.Extension);
            }

            //NEW!!
            /// <summary>Write a line to the file.
            /// This checks the file size and compares it to the file length (which is in bytes).</summary>
            /// <remarks>If the file is bigger than allowed the _FileWriter will be closed, and a new one will be created
            /// of the new 'advanced' name (givenName_#.extension).</remarks>
            /// <param name="line">The line to write.</param>
            private void WriteLine(string line)
            {
                  _OutputFileInfo.Refresh();
                  if (_OutputFileInfo.Exists)
                  {
                        if (_OutputFileInfo.Length > _FileSize)
                        {
                              Console.WriteLine("File changing to: " + _OutputFileInfo.FullName + " as size (" + _OutputFileInfo.Length + ") was > " + _FileSize + "bytes");
                              _OutputFileInfo = new FileInfo(GenerateFileName(_OutputFileInfo.FullName));                    
                        }
                  }
                  StreamWriter _FileWriter = _OutputFileInfo.AppendText();
                  _FileWriter.WriteLine(line);
                  _FileWriter.Close();
            }
            #endregion Checking a Line
      }

      /////////////////////////////////////////////////////
      // FileOpenerThread Class: Opens a file on a new thread.
      /////////////////////////////////////////////////////
      /// <summary>Will open a file on a new thread and retrieve the entire contents.</summary>
      internal class FileOpenerThread
      {
            #region Member Variables & Properties
            /// <summary>The filename to read, set in the constructor.</summary>
            private string mFilename;

            /// <summary>Has the thread finished reading the file?</summary>
            private bool mFinished = false;

            /// <summary>Has this thread been checked by the calling method?</summary>
            private bool mChecked = false;

            /// <summary>The current contents of the file.</summary>
            private string mContentsOfFile;

            /// <summary>The thread that will do the reading.</summary>
            private Thread mReaderThread;

            /// <summary>Used for disposing and stopping the threads.</summary>
            private bool mAlive = true;

            /// <summary>The collection of "Path's" that contain the search information etc.</summary>
            private ArrayList mPaths;

            /// <summary>Gets the contents of the file.</summary>
            /// <remarks>Will return an error message if an attempt is made to access the contents before the file is read.</remarks>
            public string ContentsOfFile
            {
                  get
                  {
                        if (Finished)
                              return mContentsOfFile;
                        return "File reading (" + Filename + ") either hasn't begun, or hasn't finished.";
                  }
            }

            /// <summary>Gets the filename the thread is running on.</summary>
            public string Filename
            {
                  get { return mFilename; }
            }

            /// <summary>Gets whether the file has been read or not.</summary>
            public bool Finished
            {
                  get { return mFinished; }
            }

            /// <summary>Gets whether this thread has been checked by the calling method.</summary>
            public bool Checked
            {
                  get { return mChecked; }
            }
            #endregion Member Variables & Properties

            #region Constructor
            /// <summary>Create a new instance of the FileOpenerThread passing in the file to open.</summary>
            /// <param name="filename">The file to open.</param>
            /// <param name="paths">The paths to write to.</param>
            public FileOpenerThread(string filename, ArrayList paths)
            {
                  mFilename = filename;
                  mReaderThread = new Thread(new ThreadStart(ReadFile));
                  mPaths = paths;
            }
            #endregion Constructor

            #region Start / Stop / Checked
            /// <summary>Start the thread to begin reading.</summary>
            public void Start()
            {
                  mFinished = false;
                  mReaderThread.Start();
            }

            /// <summary>Stop the thread, the thread will read one more line and then die.</summary>
            public void Stop()
            {
                  mAlive = false;
            }

            /// <summary>The file thread has been checked.</summary>
            public void BeenChecked()
            {
                  mChecked = true;
            }
            #endregion Start / Stop / Checked

            #region Parse The File
            /// <summary>Read the file.</summary>
            private void ReadFile()
            {
                  if (Filename.Length == 0 || !File.Exists(Filename))
                  {
                        mContentsOfFile = "Filename: " + Filename + " doesn't exist.";
                        mFinished = true;
                        return;
                  }
                  StreamReader myFile = new StreamReader(Filename);
                  string line = myFile.ReadLine();
                  while (line != null)
                  {
                        if (!mAlive)
                              break;
                        Arrays(line);
                        line = myFile.ReadLine();
                  }
                  myFile.Close();
                  mFinished = true;
            }

            /// <summary>Check each line read in against the paths given.</summary>
            /// <param name="lines">The line to check.</param>
            public void Arrays(string lines)
            {
                  if (lines == null || mPaths == null)
                  {
                        Console.Out.WriteLine("ERROR: Null line/Null Paths read in, can't deal with this, as such am exiting 'Arrays'.");
                        return;
                  }
                  foreach (Path path in mPaths)
                        path.CheckLine(lines);
            }
            #endregion Parse The File
      }
}