PHP fputcsv() utf-8 magento error question...

<?php // demo/temp_helpchrisplz.php
error_reporting(E_ALL);


// SEE http://www.experts-exchange.com/Web_Development/Internet_Marketing/Search_Engine_Optimization_SEO/Q_28396768.html
// REF http://php.net/manual/en/function.utf8-encode.php
// REF http://php.net/manual/en/function.ord.php
// REF http://php.net/manual/en/function.decbin.php
// REF http://www.asciitable.com/
// REF http://en.wikipedia.org/wiki/UTF-8


// TRY THE SCRIPT BOTH WITH AND WITHOUT THIS STATEMENT (FIRST PREVAILS)
echo '<meta charset="utf8" />';        // GARBLES NON-UTF-8
echo '<meta charset="iso-8859-1" />';  // GARBLES UTF-8

// PREFORMAT FOR EASY VISUALIZATION
echo '<pre>';


// THE TEST DATA MAY CONTAIN INVALID UTF-8 CHARACTERS
$url = 'http://christophersowerby.com/csv/fixedSERVER.CSV';
$txt = file($url, FILE_IGNORE_NEW_LINES);

// TRY TO FIX THE NON-UTF-8 CHARACTERS
foreach ($txt as $row => $str)
{
    // TRY TO FIX THE NON-UTF-8 CHARACTERS
    $str = utf8_encode($str);
    $txt[$row] = $str;
}

// TEST THE RESULTING DATA SET
$sig = array();
mb_internal_encoding("UTF-8");
foreach ($txt as $row => $str)
{
    // VERIFY UTF-8 CHARACTERS
    if (!mb_check_encoding($str)) $sig[$row] = $str;
}
if (!empty($sig)) var_dump($sig);
if ( empty($sig))
{
    echo "UTF-8 CONVERSION SUCCESSFUL" . PHP_EOL;
    // FILE PUT CONTENTS
}


// LOOK FOR UTF-8 CHARACTERS OR FAILURES
ob_start();
$sig = array();

// WITH EACH ROW OF THE CSV FILE
foreach ($txt as $row => $str)
{
    echo PHP_EOL . "ROW: " .str_pad($row, 4, ' ', STR_PAD_LEFT);
    echo ' ' . $str;

    // SKIP VALID UTF-8 STRINGS
    if (mb_check_encoding($str)) continue;

	// LOOK AT THE TEST STRING BYTE-BY-BYTE (_NOT_ THE SAME AS CHARACTER-BY-CHARACTER)
	$arr = str_split($str);
	foreach ($arr as $ptr => $chr)
	{
	    $ord = ord($chr);
	    $err = FALSE;

	    // PRINTABLE INFO FOR THE BYTE VALUES
	    $hex  = strtoupper(dechex($ord));
	    $bin  = decbin($ord);
	    $ptrp = str_pad($ptr, 3, ' ', STR_PAD_LEFT);
	    $ordp = str_pad($ord, 3, ' ', STR_PAD_LEFT);

	    // FLAG THIS CHARACTER IF THE BYTE-CODE IS GT 127
	    if ($ord > 127)
	    {
	        // echo PHP_EOL . " <b>BYTE: $ptrp</a>, CHR: $chr ORD: $ordp, HEX: $hex, BIN: $bin</b>";

	        // GET POINTERS TO THE NEXT CHARACTERS
	        $pp1 = $ptr + 1;
	        $pp2 = $ptr + 2;
	        $pp3 = $ptr + 3;

	        // IF A FOUR-BYTE UTF-8 CHARACTER, NEXT 3 BYTES MUST START WITH '10'
	        $sub = substr($bin, 0, 5);
	        if ($sub == '11110')
	        {
	            $chs = array();
	            $chs[$pp1] = $arr[$pp1];
	            $chs[$pp2] = $arr[$pp2];
	            $chs[$pp3] = $arr[$pp3];
	            foreach ($chs as $ppp => $nxt)
	            {
	                $cod = decbin(ord($nxt));
	                $cod = str_pad($cod, 8, '0', STR_PAD_LEFT);
	                $utf = substr($cod,0,2);
	                if ($utf !== '10')
	                {
	                    echo ", ERROR IN BYTE $ppp: $cod";
	                    $err = TRUE;
	                }
	            }
	        }

	        // IF A THREE-BYTE UTF-8 CHARACTER, NEXT 2 BYTES MUST START WITH '10'
	        $sub = substr($bin, 0, 4);
	        if ($sub == '1110')
	        {
	            $chs = array();
	            $chs[$pp1] = $arr[$pp1];
	            $chs[$pp2] = $arr[$pp2];
	            foreach ($chs as $ppp => $nxt)
	            {
	                $cod = decbin(ord($nxt));
	                $cod = str_pad($cod, 8, '0', STR_PAD_LEFT);
	                $utf = substr($cod,0,2);
	                if ($utf !== '10')
	                {
	                    echo ", ERROR IN BYTE $ppp: $cod";
	                    $err = TRUE;
	                }
	            }
	        }

	        // IF A TWO BYTE UTF-8 CHARACTER, NEXT 1 BYTE MUST START WITH '10'
	        $sub = substr($bin, 0, 3);
	        if ($sub == '110')
	        {
	            $chs = array();
	            $chs[$pp1] = $arr[$pp1];
	            foreach ($chs as $ppp => $nxt)
	            {
	                $cod = decbin(ord($nxt));
	                $cod = str_pad($cod, 8, '0', STR_PAD_LEFT);
	                $utf = substr($cod,0,2);
	                if ($utf !== '10')
	                {
	                    echo ", ERROR IN BYTE $ppp: $cod";
	                    $err = TRUE;
	                }
	            }
	        }
	    }
	    $sig[$row] = $row;
	}
}
$out = ob_get_clean();

// IF THERE WERE ANY CHARACTERS FLAGGED
if (!empty($sig))
{
    echo PHP_EOL . '<b>POSSIBLE UTF-8 ERRORS</b>';
    echo PHP_EOL;
    print_r($sig);
    echo PHP_EOL;
}

echo PHP_EOL . '<b>ENTIRE DATA SET FOLLOWS:</b>';
echo $out;

Select all Open in new window