从1000行批量导入CSV行

时间:2015-12-21 10:56:26

标签: php fgetcsv

我写了一个脚本,将CSV文件导入应用程序数据库,但是要导入的最终文件超过70k行,而我的脚本速度很慢。在它甚至导入第10行之前,它会在服务器上达到最大执行时间限制。

有没有办法让这个脚本批量导入行,这些行将作为单独的执行运行而不完全重写?

$job_strings[] = 'ImportAccounts';

function ImportAccounts(){

global $sugar_config;
$cfg = new Configurator();
$emailAddress = $cfg->config['IA']['IA_email'];
$emailSubject = 'Accounts Importing: Error Report';
$import_file = $cfg->config['IA']['IA_path'];

try {
    $GLOBALS['log']->fatal('The Account import works');
    //Used to handle mac line endings
    ini_set('auto_detect_line_endings', TRUE);
    $errors = array();
    //check file exists if not throw an error
    if (!file_exists($import_file)) {
        throw new Exception('The import file is not found.');
        $GLOBALS['log']->fatal('The Account import csv file is not found.');
    }
    //Open the csv file
    if ($file = fopen($import_file, "r")) {
        $i = 1;
        while (($data = fgetcsv($file, 100000, ',')) !== FALSE) {//Loop through all rows in the cvs file
            if ($i == 1) {//Get column headers
                $header_count = count($data);
            }
            if ($i > 1) {
                $col_count = count($data);
                // Bail out of the loop if columns are incorrect
                if ($header_count != $col_count) {
                    $error_msg = 'Error: Incorrect number of columns in the .CVS file at line: ' . $i . '. Import halted no records were imported after this point.';
                    $errors[] = $error_msg;
                    $GLOBALS['log']->fatal('Incorrect number of columns in the .CVS file at line: \' . $i . \'. Import halted no records were imported after this point.');
                    break;
                } else {
                    if (!IsNullOrEmptyString($data[21])) { //If ID is specified update the existing record
                        $account = new Account();
                        $e_account = $account->retrieve($data[21]);
                        //Log error if the ID is not recognised.
                        if (IsNullOrEmptyString($e_account)) {
                            $error_msg = 'Error: Record at line: ' . $i . ' does not have a valid ID field or the record does not exist. Record was not Imported!' . "\r\n";
                            $errors[] = $error_msg;
                        } else {//update record
                            $e_account->wg_accountnumber_c = $data[0];
                            $e_account->name = $data[1];
                            $e_account->billing_address_street = $data[2];
                            $e_account->billing_address_street_2 = $data[3];
                            $e_account->billing_address_street_3 = $data[4];
                            $e_account->billing_address_street_4 = $data[5];
                            $e_account->billing_address_state = $data[6];
                            $e_account->billing_address_postalcode = $data[7];
                            $e_account->phone_office = $data[8];
                            //check email
                            if (!IsNullOrEmptyString($data[9])) {
                                if (filter_var($data[9], FILTER_VALIDATE_EMAIL)) {
                                    $e_account->email1 = $data[9];
                                } else {
                                    $error_msg = 'Error: Record at line: ' . $i . '. The Record was imported successfully but may contain an invalid email address.' . "\r\n";
                                    $errors[] = $error_msg;
                                }
                            }

                            $e_account->phone_fax = $data[10];

                            if (!IsNullOrEmptyString($data[11])) {
                                $e_account->website = $data[11];
                            }

                            $e_account->wg_gl_ordertype_c = $data[12];
                            $e_account->industry = $data[13];
                            $e_account->user_id_c = $data[14];
                            $e_account->wg_sellingcompany_c = $data[15];
                            $e_account->wg_creditterms_c = $data[16];
                            $e_account->wg_credithold_c = $data[17];
                            $e_account->wg_proforma_c = $data[18];
                            $e_account->wg_revenue_ytd_c = $data[19];
                            $e_account->annual_revenue = $data[20];
                            $e_account->save();
                        }

                    } else { //If no ID is specified create a new record
                        $e_account = new Account();
                        $e_account->wg_accountnumber_c = $data[0];
                        $e_account->name = $data[1];
                        $e_account->billing_address_street = $data[2];
                        $e_account->billing_address_street_2 = $data[3];
                        $e_account->billing_address_street_3 = $data[4];
                        $e_account->billing_address_street_4 = $data[5];
                        $e_account->billing_address_state = $data[6];
                        $e_account->billing_address_postalcode = $data[7];
                        $e_account->phone_office = $data[8];
                        //check email
                        if (!IsNullOrEmptyString($data[9])) {
                            if (filter_var($data[9], FILTER_VALIDATE_EMAIL)) {
                                $e_account->email1 = $data[9];
                            } else {
                                $error_msg = 'Error: Record at line: ' . $i . '. The Record was imported successfully but may contain an invalid email address.' . "\r\n";
                                $errors[] = $error_msg;
                            }
                        }

                        $e_account->phone_fax = $data[10];

                        if (!IsNullOrEmptyString($data[11])) {
                            $e_account->website = $data[11];
                        }

                        $e_account->wg_gl_ordertype_c = $data[12];
                        $e_account->industry = $data[13];
                        $e_account->user_id_c = $data[14];
                        $e_account->wg_sellingcompany_c = $data[15];
                        $e_account->wg_creditterms_c = $data[16];
                        $e_account->wg_credithold_c = $data[17];
                        $e_account->wg_proforma_c = $data[18];
                        $e_account->wg_revenue_ytd_c = $data[19];
                        $e_account->annual_revenue = $data[20];
                        $e_account->save();
                    }

                }

            }
            $i++;
        }
        ini_set('auto_detect_line_endings', FALSE);
        fclose($file);
        //send error report email
        if (!empty($errors)) {
            $emailBody = 'The following errors occurred during Account import at: ' . date("Y-m-d H:i:s") . "\r\n";
            foreach ($errors as $error) {
                $emailBody .= "\r\n" . $error;
            }
            //Send email
            sendEmail($emailAddress, $emailSubject, $emailBody);
        }
    } else {
        throw new Exception('The Import file failed to open.');
        $GLOBALS['log']->fatal('The import csv file failed to open.');
    }
}
catch ( Exception $e ) { //Catch errors where the import file fails to open or is not found

    $error_msg = 'Error: '.$e->getMessage(); ;
    $errors[] = $error_msg;

    $emailBody = 'The following errors occurred during Account import at: ' . date("Y-m-d H:i:s") . "\r\n";
    foreach ($errors as $error) {
        $emailBody .= "\r\n" . $error;
    }
    //Send email
    sendEmail($emailAddress, $emailSubject, $emailBody);
}

return true;
}

0 个答案:

没有答案
相关问题