asked on

Writing Random Data C# - Performance.

Hi,

Ok I have some code below that I would like to improve - it works but it runs very slowly.

What I am trying to do is write a program that generates a "random" data file of a
size determined by the user in GB which is chosen as a multiple. so if they
select 5.5 the 5.5 GB of random data is written.

It runs really slowly in the generate data phase and I guess the file write phase.

Obviously I am not doing something correct and it performs poorly. My guess is
List<Byte> file_data = new List<Byte>();

So I am interested to see if the performance can be improved or are my expectations unrealistic ? :)

Thanks,

Ward.

using System;
using System.Collections.Generic;
using System.Collections;
using System.ComponentModel;
using System.Data;
using System.Drawing;
using System.Linq;
using System.Text;
using System.Windows.Forms;
using System.IO;

namespace FileGen
{
    public partial class Form1 : Form
    {
        private string file_name;
        decimal total_file_size;

        public Form1()
        {
            InitializeComponent();
        }

        private void fnUpdate_Progress(decimal position)
        {

            if (position % 1048576 == 0)
            {
                decimal percentage = (position / total_file_size) * 100;
                percentage = Math.Round(percentage, 2);
                label_Position.Text = position.ToString("#,##0") + " / " + total_file_size.ToString("#,##0") + " (" + percentage.ToString() + "%)";
                progressBar_File.Value = (int)percentage;
            }

            Application.DoEvents();
        }

        private void fnCalculate_File_Size(decimal gb)
        {
            decimal gigabyte = 1073741824;

            decimal temp_file_size = gb * gigabyte;
            total_file_size = temp_file_size;

            string temp_format = temp_file_size.ToString("#,##0");

            label_FileSize.Text = temp_format + " bytes.";
            Application.DoEvents();

        }

        private void button_Exit_Click(object sender, EventArgs e)
        {
            Application.Exit();
        }

        private void button_Path_Click(object sender, EventArgs e)
        {

            SaveFileDialog saveFileDialog1 = new SaveFileDialog();

            saveFileDialog1.Filter = "dat files (*.dat)|*.dat|All files (*.*)|*.*";
            saveFileDialog1.FilterIndex = 2;
            saveFileDialog1.RestoreDirectory = true;

            if (saveFileDialog1.ShowDialog() == DialogResult.OK)
            {               
                file_name = saveFileDialog1.FileName;
                textBox_Path.Text = file_name;
                button_Generate.Enabled = true;
            }

        }

        private void Form1_Load(object sender, EventArgs e)
        {
            comboFileSize.SelectedIndex = 11;
            file_name = "";
            progressBar_File.Minimum = 0;
            progressBar_File.Maximum = 100;
            progressBar_File.Step = 1;
            progressBar_File.Value = 0;
            label_FileSize.Text = "";
            label_Status.Text = "";
            label_Position.Text = "";
            fnCalculate_File_Size(4);
        }

        private Byte RandomNumber(int min, int max)
        {
            Random random = new Random();
            return (Byte)random.Next(min, max);
        }

        private void button_Generate_Click(object sender, EventArgs e)
        {

            List<Byte> file_data = new List<Byte>();
            Byte data_byte;
            decimal file_pos;

            if (file_name == "")
            {
                return;
            }

            // ---------------------------- Very SLOW Here ------------------------ 

            label_Status.Text = "Generating data...";
            Application.DoEvents();

            for (file_pos = 1; file_pos <= total_file_size; file_pos++)
            {
                data_byte = RandomNumber(0, 255);
                file_data.Add(data_byte);
                fnUpdate_Progress(file_pos);
            }                     

            FileStream fs1 = new FileStream(file_name, FileMode.OpenOrCreate,
            FileAccess.Write);
            BinaryWriter bw = new BinaryWriter(fs1);

            label_Status.Text = "Writing data to file...";
            Application.DoEvents();

            file_pos = 1;

            // ---------------------- Probably SLOW Here? -----------------------------

            foreach (Byte b in file_data)
            {
                bw.Write(b);
                fnUpdate_Progress(file_pos);
                file_pos++;
            }
            
            bw.Close();
            fs1.Close();

            Application.Exit();
        }


        private void comboFileSize_SelectedValueChanged(object sender, EventArgs e)
        {
            string gb_size;
            decimal gb;

            gb_size = (string)comboFileSize.SelectedItem;

            gb = Convert.ToDecimal(gb_size);

            fnCalculate_File_Size(gb);
        }
    }
}

Open in new window

ASKER CERTIFIED SOLUTION

Fernando Soto

membership

This solution is only available to members.

To access this solution, you must be a member of Experts Exchange.

Start Free Trial

Fernando Soto

You will need to check the limits on file size and chunk sizes used to make sure that all adds up.

r3nder

try switching loops - use LINQ TakeAny();

//17%  slower for average sized collections using any()
// find implemented via LINQ
    public static bool FindViaLinq(IEnumerable<int> list, int target)
    {
        return list.Any(item => item == target);
    }
 
 
    // find implemented via standard iteration
    public static bool FindViaIteration(IEnumerable<int> list, int target)
    {
        foreach (var i in list)
        {
            if (i == target)
            {
                return true;
            }
        }
 
        return false;
    }


//90% slower using takewhile()
// Linq form
    public static int GetTargetPosition1(IEnumerable<int> list, int target)
    {
        return list.TakeWhile(item => item != target).Count();
    }
 
    // traditionally iterative form
    public static int GetTargetPosition2(IEnumerable<int> list, int target)
    {
        int count = 0;
 
        foreach (var i in list)
        {
            if(i == target)
            {
                break;
            }
 
            ++count;
        }
 
        return count;
    }
//Much faster  
// a newer method that uses linq but evaluates the count in a closure.
    public static int TakeWhileViaLinq2(IEnumerable<int> list, int target)
    {
        int count = 0;
        list.Any(item =>
            {
                if(item == target)
                {
                    return true;
                }
 
                ++count;
                return false;
            });
        return count;
    }

Open in new window

Najam Uddin

if your requirement is just a file of specified size, without any concern of its content, than you can use
"fsutil file createnew C:\1mbfile.txt 1000000

ie fsutil file createnew <name of file> <size in bytes>