Skip to content

Commit 6a2dc85

Browse files
author
Radovan Janjic
committed
ADDED: Auto-detect EOL for CSV importer
1 parent 2668c0c commit 6a2dc85

File tree

7 files changed

+55
-13
lines changed

7 files changed

+55
-13
lines changed

‎MySQL_wrapper.class.php‎

Lines changed: 28 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -371,16 +371,21 @@ function arrayToInsert($table, $data, $ignore = FALSE, $duplicateupdate = NULL)
371371
* @param integer $ignore - Number of ignored rows (Default: 1)
372372
* @param array $update - If row fields needed to be updated eg date format or increment (SQL format only @FIELD is variable with content of that field in CSV row) $update = array('SOME_DATE' => 'STR_TO_DATE(@SOME_DATE, "%d/%m/%Y")', 'SOME_INCREMENT' => '@SOME_INCREMENT + 1')
373373
* @param string $getColumnsFrom - Get Columns Names from (file or table) - this is important if there is update while inserting (Default: file)
374-
* @param string $newLine - New line delimiter (Default: \n)
374+
* @param string $newLine - New line delimiter (Default: auto detection use \n, \r\n ...)
375375
* @return number of inserted rows or false
376376
*/
377-
function importCSV2Table($file, $table, $delimiter = ',', $enclosure = '"', $escape = '\\', $ignore = 1, $update = array(), $getColumnsFrom = 'file', $newLine = '\n') {
377+
function importCSV2Table($file, $table, $delimiter = ',', $enclosure = '"', $escape = '\\', $ignore = 1, $update = array(), $getColumnsFrom = 'file', $newLine = FALSE) {
378378
$file = file_exists($file) ? realpath($file) : NULL;
379379
$file = realpath($file);
380380
if (!file_exists($file)) {
381381
$this->error('ERROR', "Import CSV to Table - File: {$file} doesn't exist.");
382382
return FALSE;
383383
}
384+
385+
if ($newLine === FALSE) {
386+
$newLine = $this->detectEOL($file);
387+
}
388+
384389
$sql = "LOAD DATA LOCAL INFILE '{$this->escape($file)}' " .
385390
"INTO TABLE `{$table}` " .
386391
"COLUMNS TERMINATED BY '{$delimiter}' " .
@@ -420,10 +425,10 @@ function importCSV2Table($file, $table, $delimiter = ',', $enclosure = '"', $esc
420425
* @param integer $ignore - Number of ignored rows (Default: 1)
421426
* @param array $update - If row fields needed to be updated eg date format or increment (SQL format only @FIELD is variable with content of that field in CSV row) $update = array('SOME_DATE' => 'STR_TO_DATE(@SOME_DATE, "%d/%m/%Y")', 'SOME_INCREMENT' => '@SOME_INCREMENT + 1')
422427
* @param string $getColumnsFrom - Get Columns Names from (file or table) - this is important if there is update while inserting (Default: file)
423-
* @param string $newLine - New line delimiter (Default: \n)
428+
* @param string $newLine - New line delimiter (Default: auto detection use \n, \r\n ...)
424429
* @return number of inserted rows or false
425430
*/
426-
function importUpdateCSV2Table($file, $table, $delimiter = ',', $enclosure = '"', $escape = '\\', $ignore = 1, $update = array(), $getColumnsFrom = 'file', $newLine = '\n') {
431+
function importUpdateCSV2Table($file, $table, $delimiter = ',', $enclosure = '"', $escape = '\\', $ignore = 1, $update = array(), $getColumnsFrom = 'file', $newLine = FALSE) {
427432
$tmp_name = "{$table}_tmp_" . rand();
428433

429434
// Create tmp table
@@ -432,7 +437,7 @@ function importUpdateCSV2Table($file, $table, $delimiter = ',', $enclosure = '"'
432437
// Remove auto_increment if exists
433438
$change = array();
434439
$this->query("SHOW COLUMNS FROM `{$tmp_name}` WHERE `Key` NOT LIKE '';");
435-
if ($this->affected > 0) {
440+
if($this->affected > 0){
436441
while ($row = $this->fetchArray()) {
437442
$change[$row['Field']] = "CHANGE `{$row['Field']}` `{$row['Field']}` {$row['Type']}";
438443
}
@@ -595,10 +600,10 @@ function query2CSV($sql, $file, $delimiter = ',', $enclosure = '"', $escape = '\
595600
* @param integer $ignore - Number of ignored rows (Default: 1)
596601
* @param array $update - If row fields needed to be updated eg date format or increment (SQL format only @FIELD is variable with content of that field in CSV row) $update = array('SOME_DATE' => 'STR_TO_DATE(@SOME_DATE, "%d/%m/%Y")', 'SOME_INCREMENT' => '@SOME_INCREMENT + 1')
597602
* @param string $getColumnsFrom - Get Columns Names from (file or generate) - this is important if there is update while inserting (Default: file)
598-
* @param string $newLine - New line delimiter (Default: \n)
603+
* @param string $newLine - New line delimiter (Default: auto detection use \n, \r\n ...)
599604
* @return number of inserted rows or false
600605
*/
601-
function createTableFromCSV($file, $table, $delimiter = ',', $enclosure = '"', $escape = '\\', $ignore = 1, $update = array(), $getColumnsFrom = 'file', $newLine = '\r\n') {
606+
function createTableFromCSV($file, $table, $delimiter = ',', $enclosure = '"', $escape = '\\', $ignore = 1, $update = array(), $getColumnsFrom = 'file', $newLine = FALSE) {
602607
$file = file_exists($file) ? realpath($file) : NULL;
603608
if ($file === NULL) {
604609
$this->error('ERROR', "Create Table form CSV - File: {$file} doesn't exist.");
@@ -878,4 +883,20 @@ function getMicrotime() {
878883
list($usec, $sec) = explode(" ", microtime());
879884
return ((float) $usec + (float) $sec);
880885
}
886+
887+
/** Detect EOL from file
888+
* @param string - File path
889+
* @retrun - EOL chr
890+
*/
891+
function detectEOL($file) {
892+
$f = fopen($file, 'r');
893+
$line = fgets($f);
894+
fclose($f);
895+
foreach (array("\r\n", "\r", "\n") as $eol) {
896+
if (substr_compare($line, $eol, -strlen($eol)) === 0) {
897+
return $eol;
898+
}
899+
}
900+
return FALSE;
901+
}
881902
}

‎example.php‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@
5555

5656
// Connect
5757
$db->connect();
58+
// Import and update all data
5859
// Close connection
5960
$db->close();
6061
///////////////////////////////////////////////////////////////////////////////////////////
@@ -563,7 +564,7 @@
563564
$db->connect();
564565

565566
$db->dropTable('csv_to_table_test');
566-
$db->createTableFromCSV('test_files/countrylist.csv', 'csv_to_table_test', ',', '"', '\\', 1, array(), 'file', '\r\n');
567+
$db->createTableFromCSV('test_files/countrylist.csv', 'csv_to_table_test');
567568

568569
$db->dropTable('csv_to_table_test_no_column_names');
569570
$db->createTableFromCSV('test_files/countrylist1.csv', 'csv_to_table_test_no_column_names', ',', '"', '\\', 0, array(), 'generate', '\r\n');
@@ -591,6 +592,10 @@
591592
$db = new MySQL_wrapper(MySQL_HOST, MySQL_USER, MySQL_PASS, MySQL_DB);
592593
// Connect
593594
$db->connect();
595+
596+
// Import and update all data
597+
$db->importUpdateCSV2Table('test_files/countrylist.csv', 'csv_to_table_test');
598+
594599
// Import and update all data
595600
$db->importUpdateCSV2Table('test_files/countrylist.csv', 'csv_to_table_test', ',', '"', '\\', 1, array(), 'file', '\r\n');
596601
// More options

‎log-mysql.txt‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,3 +17,7 @@
1717
2014-01-14 11:12:48 - ERROR -> NO -> 1064 - DESC -> You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near '!@#$' at line 1 - CALL -> Function query in C:\xampp\htdocs\Git\PHP_MySQL_wrapper\example.php on line 441
1818
2014-01-14 17:14:14 - QUERY -> EXEC -> 0.00538516 -> SHOW COLUMNS FROM `table`;
1919
2014-01-14 17:14:14 - ERROR -> NO -> 1064 - DESC -> You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near '!@#$' at line 1 - CALL -> Function query in C:\xampp\htdocs\Git\PHP_MySQL_wrapper\example.php on line 469
20+
2014-01-15 11:10:45 - QUERY -> EXEC -> 0.00558519 -> SHOW COLUMNS FROM `table`;
21+
2014-01-15 11:10:45 - ERROR -> NO -> 1064 - DESC -> You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near '!@#$' at line 1 - CALL -> Function query in C:\xampp\htdocs\Git\PHP_MySQL_wrapper\example.php on line 469
22+
2014-01-15 11:13:38 - QUERY -> EXEC -> 0.00299191 -> SHOW COLUMNS FROM `table`;
23+
2014-01-15 11:13:38 - ERROR -> NO -> 1064 - DESC -> You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near '!@#$' at line 1 - CALL -> Function query in C:\xampp\htdocs\Git\PHP_MySQL_wrapper\example.php on line 469

‎test_files/test-1.txt‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
"id","firstname","surname","email","date"
2-
"1","foo","bar","rade@it-radionica.com","2014-01-14"
3-
"2","Radovana","Janjic","rade@it-radionica.com","2014-01-14"
2+
"1","foo","bar","rade@it-radionica.com","2014-01-15"
3+
"2","Radovana","Janjic","rade@it-radionica.com","2014-01-15"
44
"3","Radovan","Janjic'","rade@it-radionica.com","2012-11-04"
55
"4","Radovan","Janjic","rade@it-radionica.com","2012-11-04"
66
"5","Radovan","Janjic","rade@it-radionica.com","2012-11-04"
@@ -11,3 +11,7 @@
1111
"10","Radovan","Janjic","rade@it-radionica.com","2012-11-04"
1212
"12","Radovan","Janjic","rade@it-radionica.com","2014-01-14"
1313
"13","Radovan","Janjic","rade@it-radionica.com","2014-01-14"
14+
"15","Radovan","Janjic","rade@it-radionica.com","2014-01-15"
15+
"16","Radovan","Janjic","rade@it-radionica.com","2014-01-15"
16+
"18","Radovan","Janjic","rade@it-radionica.com","2014-01-15"
17+
"19","Radovan","Janjic","rade@it-radionica.com","2014-01-15"

‎test_files/test-2.txt‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,3 +11,7 @@
1111
"Radovan","Janjic"
1212
"Radovan","Janjic"
1313
"Radovan","Janjic"
14+
"Radovan","Janjic"
15+
"Radovan","Janjic"
16+
"Radovan","Janjic"
17+
"Radovan","Janjic"

‎test_files/test-3.txt‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
"firstname","surname","date"
2-
"foo","bar","2014-01-14"
3-
"Radovana","Janjic","2014-01-14"
2+
"foo","bar","2014-01-15"
3+
"Radovana","Janjic","2014-01-15"
44
"Radovan","Janjic'","2012-11-04"
55
"Radovan","Janjic","2012-11-04"
66
"Radovan","Janjic","2012-11-04"
@@ -11,3 +11,7 @@
1111
"Radovan","Janjic","2012-11-04"
1212
"Radovan","Janjic","2014-01-14"
1313
"Radovan","Janjic","2014-01-14"
14+
"Radovan","Janjic","2014-01-15"
15+
"Radovan","Janjic","2014-01-15"
16+
"Radovan","Janjic","2014-01-15"
17+
"Radovan","Janjic","2014-01-15"

‎test_files/test-4.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
"id","firstname","surname","email","date"
2-
"2","Radovana","Janjic","rade@it-radionica.com","2014-01-14"
2+
"2","Radovana","Janjic","rade@it-radionica.com","2014-01-15"
33
"3","Radovan","Janjic'","rade@it-radionica.com","2012-11-04"
44
"4","Radovan","Janjic","rade@it-radionica.com","2012-11-04"
55
"5","Radovan","Janjic","rade@it-radionica.com","2012-11-04"

0 commit comments

Comments
 (0)