diff --git a/.toxic.json b/.toxic.json index d24776f28356..f5f5c6fb2781 100644 --- a/.toxic.json +++ b/.toxic.json @@ -32,7 +32,7 @@ "CRM_Contribute_Form_Contribution_Main::formRule()": "toxicAlert", "CRM_Contribute_Form_Contribution_Main::submit()": "toxicAlert", "CRM_Contribute_Form_Task_Invoice::printPDF()": "toxicAlert", - "CRM_Contribute_Import_Parser::run()": "toxicAlert", + "CRM_Contribute_Import_Parser_Contribution::run()": "toxicAlert", "CRM_Core_BAO_ActionScheduleTest::setUp()": "toxicAlert", "CRM_Core_BAO_CustomField::formatCustomField()": "toxicAlert", "CRM_Core_BAO_CustomGroup::getTree()": "toxicAlert", diff --git a/CRM/Contribute/Import/Form/Preview.php b/CRM/Contribute/Import/Form/Preview.php index e45de43a25b8..9a6ba068dea8 100644 --- a/CRM/Contribute/Import/Form/Preview.php +++ b/CRM/Contribute/Import/Form/Preview.php @@ -54,17 +54,17 @@ public function preProcess() { } if ($invalidRowCount) { - $urlParams = 'type=' . CRM_Import_Parser::ERROR . '&parser=CRM_Contribute_Import_Parser'; + $urlParams = 'type=' . CRM_Import_Parser::ERROR . '&parser=CRM_Contribute_Import_Parser_Contribution'; $this->set('downloadErrorRecordsUrl', CRM_Utils_System::url('civicrm/export', $urlParams)); } if ($conflictRowCount) { - $urlParams = 'type=' . CRM_Import_Parser::CONFLICT . '&parser=CRM_Contribute_Import_Parser'; + $urlParams = 'type=' . CRM_Import_Parser::CONFLICT . '&parser=CRM_Contribute_Import_Parser_Contribution'; $this->set('downloadConflictRecordsUrl', CRM_Utils_System::url('civicrm/export', $urlParams)); } if ($mismatchCount) { - $urlParams = 'type=' . CRM_Import_Parser::NO_MATCH . '&parser=CRM_Contribute_Import_Parser'; + $urlParams = 'type=' . CRM_Import_Parser::NO_MATCH . '&parser=CRM_Contribute_Import_Parser_Contribution'; $this->set('downloadMismatchRecordsUrl', CRM_Utils_System::url('civicrm/export', $urlParams)); } @@ -160,11 +160,11 @@ public function postProcess() { fclose($fd); $this->set('errorFile', $errorFile); - $urlParams = 'type=' . CRM_Import_Parser::ERROR . '&parser=CRM_Contribute_Import_Parser'; + $urlParams = 'type=' . CRM_Import_Parser::ERROR . '&parser=CRM_Contribute_Import_Parser_Contribution'; $this->set('downloadErrorRecordsUrl', CRM_Utils_System::url('civicrm/export', $urlParams)); - $urlParams = 'type=' . CRM_Import_Parser::CONFLICT . '&parser=CRM_Contribute_Import_Parser'; + $urlParams = 'type=' . CRM_Import_Parser::CONFLICT . '&parser=CRM_Contribute_Import_Parser_Contribution'; $this->set('downloadConflictRecordsUrl', CRM_Utils_System::url('civicrm/export', $urlParams)); - $urlParams = 'type=' . CRM_Import_Parser::NO_MATCH . '&parser=CRM_Contribute_Import_Parser'; + $urlParams = 'type=' . CRM_Import_Parser::NO_MATCH . '&parser=CRM_Contribute_Import_Parser_Contribution'; $this->set('downloadMismatchRecordsUrl', CRM_Utils_System::url('civicrm/export', $urlParams)); } } diff --git a/CRM/Contribute/Import/Form/Summary.php b/CRM/Contribute/Import/Form/Summary.php index 4d86b06058b5..6714befabc7e 100644 --- a/CRM/Contribute/Import/Form/Summary.php +++ b/CRM/Contribute/Import/Form/Summary.php @@ -35,13 +35,13 @@ public function preProcess() { $invalidRowCount = $this->get('invalidRowCount'); $invalidSoftCreditRowCount = $this->get('invalidSoftCreditRowCount'); if ($invalidSoftCreditRowCount) { - $urlParams = 'type=' . CRM_Contribute_Import_Parser::SOFT_CREDIT_ERROR . '&parser=CRM_Contribute_Import_Parser'; + $urlParams = 'type=' . CRM_Contribute_Import_Parser_Contribution::SOFT_CREDIT_ERROR . '&parser=CRM_Contribute_Import_Parser_Contribution'; $this->set('downloadSoftCreditErrorRecordsUrl', CRM_Utils_System::url('civicrm/export', $urlParams)); } $validSoftCreditRowCount = $this->get('validSoftCreditRowCount'); $invalidPledgePaymentRowCount = $this->get('invalidPledgePaymentRowCount'); if ($invalidPledgePaymentRowCount) { - $urlParams = 'type=' . CRM_Contribute_Import_Parser::PLEDGE_PAYMENT_ERROR . '&parser=CRM_Contribute_Import_Parser'; + $urlParams = 'type=' . CRM_Contribute_Import_Parser_Contribution::PLEDGE_PAYMENT_ERROR . '&parser=CRM_Contribute_Import_Parser_Contribution'; $this->set('downloadPledgePaymentErrorRecordsUrl', CRM_Utils_System::url('civicrm/export', $urlParams)); } $validPledgePaymentRowCount = $this->get('validPledgePaymentRowCount'); @@ -50,11 +50,11 @@ public function preProcess() { $onDuplicate = $this->get('onDuplicate'); $mismatchCount = $this->get('unMatchCount'); if ($duplicateRowCount > 0) { - $urlParams = 'type=' . CRM_Import_Parser::DUPLICATE . '&parser=CRM_Contribute_Import_Parser'; + $urlParams = 'type=' . CRM_Import_Parser::DUPLICATE . '&parser=CRM_Contribute_Import_Parser_Contribution'; $this->set('downloadDuplicateRecordsUrl', CRM_Utils_System::url('civicrm/export', $urlParams)); } elseif ($mismatchCount) { - $urlParams = 'type=' . CRM_Import_Parser::NO_MATCH . '&parser=CRM_Contribute_Import_Parser'; + $urlParams = 'type=' . CRM_Import_Parser::NO_MATCH . '&parser=CRM_Contribute_Import_Parser_Contribution'; $this->set('downloadMismatchRecordsUrl', CRM_Utils_System::url('civicrm/export', $urlParams)); } else { diff --git a/CRM/Contribute/Import/Parser.php b/CRM/Contribute/Import/Parser.php deleted file mode 100644 index 8cd494f3869f..000000000000 --- a/CRM/Contribute/Import/Parser.php +++ /dev/null @@ -1,667 +0,0 @@ -_contactType = 'Individual'; - break; - - case self::CONTACT_HOUSEHOLD: - $this->_contactType = 'Household'; - break; - - case self::CONTACT_ORGANIZATION: - $this->_contactType = 'Organization'; - } - - $this->init(); - - $this->_haveColumnHeader = $skipColumnHeader; - - $this->_separator = $separator; - - $fd = fopen($fileName, "r"); - if (!$fd) { - return FALSE; - } - - $this->_lineCount = $this->_warningCount = $this->_validSoftCreditRowCount = $this->_validPledgePaymentRowCount = 0; - $this->_invalidRowCount = $this->_validCount = $this->_invalidSoftCreditRowCount = $this->_invalidPledgePaymentRowCount = 0; - $this->_totalCount = $this->_conflictCount = 0; - - $this->_errors = []; - $this->_warnings = []; - $this->_conflicts = []; - $this->_pledgePaymentErrors = []; - $this->_softCreditErrors = []; - if ($statusID) { - $this->progressImport($statusID); - $startTimestamp = $currTimestamp = $prevTimestamp = time(); - } - - $this->_fileSize = number_format(filesize($fileName) / 1024.0, 2); - - if ($mode == self::MODE_MAPFIELD) { - $this->_rows = []; - } - else { - $this->_activeFieldCount = count($this->_activeFields); - } - - while (!feof($fd)) { - $this->_lineCount++; - - $values = fgetcsv($fd, 8192, $separator); - if (!$values) { - continue; - } - - self::encloseScrub($values); - - // skip column header if we're not in mapfield mode - if ($mode != self::MODE_MAPFIELD && $skipColumnHeader) { - $skipColumnHeader = FALSE; - continue; - } - - /* trim whitespace around the values */ - - $empty = TRUE; - foreach ($values as $k => $v) { - $values[$k] = trim($v, " \t\r\n"); - } - - if (CRM_Utils_System::isNull($values)) { - continue; - } - - $this->_totalCount++; - - if ($mode == self::MODE_MAPFIELD) { - $returnCode = $this->mapField($values); - } - elseif ($mode == self::MODE_PREVIEW) { - $returnCode = $this->preview($values); - } - elseif ($mode == self::MODE_SUMMARY) { - $returnCode = $this->summary($values); - } - elseif ($mode == self::MODE_IMPORT) { - $returnCode = $this->import($onDuplicate, $values); - if ($statusID && (($this->_lineCount % 50) == 0)) { - $prevTimestamp = $this->progressImport($statusID, FALSE, $startTimestamp, $prevTimestamp, $totalRowCount); - } - } - else { - $returnCode = self::ERROR; - } - - // note that a line could be valid but still produce a warning - if ($returnCode == self::VALID) { - $this->_validCount++; - if ($mode == self::MODE_MAPFIELD) { - $this->_rows[] = $values; - $this->_activeFieldCount = max($this->_activeFieldCount, count($values)); - } - } - - if ($returnCode == self::SOFT_CREDIT) { - $this->_validSoftCreditRowCount++; - $this->_validCount++; - if ($mode == self::MODE_MAPFIELD) { - $this->_rows[] = $values; - $this->_activeFieldCount = max($this->_activeFieldCount, count($values)); - } - } - - if ($returnCode == self::PLEDGE_PAYMENT) { - $this->_validPledgePaymentRowCount++; - $this->_validCount++; - if ($mode == self::MODE_MAPFIELD) { - $this->_rows[] = $values; - $this->_activeFieldCount = max($this->_activeFieldCount, count($values)); - } - } - - if ($returnCode == self::WARNING) { - $this->_warningCount++; - if ($this->_warningCount < $this->_maxWarningCount) { - $this->_warningCount[] = $line; - } - } - - if ($returnCode == self::ERROR) { - $this->_invalidRowCount++; - $recordNumber = $this->_lineCount; - if ($this->_haveColumnHeader) { - $recordNumber--; - } - array_unshift($values, $recordNumber); - $this->_errors[] = $values; - } - - if ($returnCode == self::PLEDGE_PAYMENT_ERROR) { - $this->_invalidPledgePaymentRowCount++; - $recordNumber = $this->_lineCount; - if ($this->_haveColumnHeader) { - $recordNumber--; - } - array_unshift($values, $recordNumber); - $this->_pledgePaymentErrors[] = $values; - } - - if ($returnCode == self::SOFT_CREDIT_ERROR) { - $this->_invalidSoftCreditRowCount++; - $recordNumber = $this->_lineCount; - if ($this->_haveColumnHeader) { - $recordNumber--; - } - array_unshift($values, $recordNumber); - $this->_softCreditErrors[] = $values; - } - - if ($returnCode == self::CONFLICT) { - $this->_conflictCount++; - $recordNumber = $this->_lineCount; - if ($this->_haveColumnHeader) { - $recordNumber--; - } - array_unshift($values, $recordNumber); - $this->_conflicts[] = $values; - } - - if ($returnCode == self::DUPLICATE) { - if ($returnCode == self::MULTIPLE_DUPE) { - /* TODO: multi-dupes should be counted apart from singles - * on non-skip action */ - } - $this->_duplicateCount++; - $recordNumber = $this->_lineCount; - if ($this->_haveColumnHeader) { - $recordNumber--; - } - array_unshift($values, $recordNumber); - $this->_duplicates[] = $values; - if ($onDuplicate != self::DUPLICATE_SKIP) { - $this->_validCount++; - } - } - - // we give the derived class a way of aborting the process - // note that the return code could be multiple code or'ed together - if ($returnCode == self::STOP) { - break; - } - - // if we are done processing the maxNumber of lines, break - if ($this->_maxLinesToProcess > 0 && $this->_validCount >= $this->_maxLinesToProcess) { - break; - } - } - - fclose($fd); - - if ($mode == self::MODE_PREVIEW || $mode == self::MODE_IMPORT) { - $customHeaders = $mapper; - - $customfields = CRM_Core_BAO_CustomField::getFields('Contribution'); - foreach ($customHeaders as $key => $value) { - if ($id = CRM_Core_BAO_CustomField::getKeyID($value)) { - $customHeaders[$key] = $customfields[$id][0]; - } - } - if ($this->_invalidRowCount) { - // removed view url for invlaid contacts - $headers = array_merge([ - ts('Line Number'), - ts('Reason'), - ], $customHeaders); - $this->_errorFileName = self::errorFileName(self::ERROR); - self::exportCSV($this->_errorFileName, $headers, $this->_errors); - } - - if ($this->_invalidPledgePaymentRowCount) { - // removed view url for invlaid contacts - $headers = array_merge([ - ts('Line Number'), - ts('Reason'), - ], $customHeaders); - $this->_pledgePaymentErrorsFileName = self::errorFileName(self::PLEDGE_PAYMENT_ERROR); - self::exportCSV($this->_pledgePaymentErrorsFileName, $headers, $this->_pledgePaymentErrors); - } - - if ($this->_invalidSoftCreditRowCount) { - // removed view url for invlaid contacts - $headers = array_merge([ - ts('Line Number'), - ts('Reason'), - ], $customHeaders); - $this->_softCreditErrorsFileName = self::errorFileName(self::SOFT_CREDIT_ERROR); - self::exportCSV($this->_softCreditErrorsFileName, $headers, $this->_softCreditErrors); - } - - if ($this->_conflictCount) { - $headers = array_merge([ - ts('Line Number'), - ts('Reason'), - ], $customHeaders); - $this->_conflictFileName = self::errorFileName(self::CONFLICT); - self::exportCSV($this->_conflictFileName, $headers, $this->_conflicts); - } - if ($this->_duplicateCount) { - $headers = array_merge([ - ts('Line Number'), - ts('View Contribution URL'), - ], $customHeaders); - - $this->_duplicateFileName = self::errorFileName(self::DUPLICATE); - self::exportCSV($this->_duplicateFileName, $headers, $this->_duplicates); - } - } - return $this->fini(); - } - - /** - * Given a list of the importable field keys that the user has selected - * set the active fields array to this list - * - * @param array $fieldKeys mapped array of values - */ - public function setActiveFields($fieldKeys) { - $this->_activeFieldCount = count($fieldKeys); - foreach ($fieldKeys as $key) { - if (empty($this->_fields[$key])) { - $this->_activeFields[] = new CRM_Contribute_Import_Field('', ts('- do not import -')); - } - else { - $this->_activeFields[] = clone($this->_fields[$key]); - } - } - } - - /** - * Store the soft credit field information. - * - * This was perhaps done this way on the believe that a lot of code pain - * was worth it to avoid negligible-cost array iterations. Perhaps we could prioritise - * readability & maintainability next since we can just work with functions to retrieve - * data from the metadata. - * - * @param array $elements - */ - public function setActiveFieldSoftCredit($elements) { - foreach ((array) $elements as $i => $element) { - $this->_activeFields[$i]->_softCreditField = $element; - } - } - - /** - * Store the soft credit field type information. - * - * This was perhaps done this way on the believe that a lot of code pain - * was worth it to avoid negligible-cost array iterations. Perhaps we could prioritise - * readability & maintainability next since we can just work with functions to retrieve - * data from the metadata. - * - * @param array $elements - */ - public function setActiveFieldSoftCreditType($elements) { - foreach ((array) $elements as $i => $element) { - $this->_activeFields[$i]->_softCreditType = $element; - } - } - - /** - * Format the field values for input to the api. - * - * @return array - * (reference ) associative array of name/value pairs - */ - public function &getActiveFieldParams() { - $params = []; - for ($i = 0; $i < $this->_activeFieldCount; $i++) { - if (isset($this->_activeFields[$i]->_value)) { - if (isset($this->_activeFields[$i]->_softCreditField)) { - if (!isset($params[$this->_activeFields[$i]->_name])) { - $params[$this->_activeFields[$i]->_name] = []; - } - $params[$this->_activeFields[$i]->_name][$i][$this->_activeFields[$i]->_softCreditField] = $this->_activeFields[$i]->_value; - if (isset($this->_activeFields[$i]->_softCreditType)) { - $params[$this->_activeFields[$i]->_name][$i]['soft_credit_type_id'] = $this->_activeFields[$i]->_softCreditType; - } - } - - if (!isset($params[$this->_activeFields[$i]->_name])) { - if (!isset($this->_activeFields[$i]->_softCreditField)) { - $params[$this->_activeFields[$i]->_name] = $this->_activeFields[$i]->_value; - } - } - } - } - return $params; - } - - /** - * @param string $name - * @param $title - * @param int $type - * @param string $headerPattern - * @param string $dataPattern - */ - public function addField($name, $title, $type = CRM_Utils_Type::T_INT, $headerPattern = '//', $dataPattern = '//') { - if (empty($name)) { - $this->_fields['doNotImport'] = new CRM_Contribute_Import_Field($name, $title, $type, $headerPattern, $dataPattern); - } - else { - $tempField = CRM_Contact_BAO_Contact::importableFields('All', NULL); - if (!array_key_exists($name, $tempField)) { - $this->_fields[$name] = new CRM_Contribute_Import_Field($name, $title, $type, $headerPattern, $dataPattern); - } - else { - $this->_fields[$name] = new CRM_Contact_Import_Field($name, $title, $type, $headerPattern, $dataPattern, - CRM_Utils_Array::value('hasLocationType', $tempField[$name]) - ); - } - } - } - - /** - * Store parser values. - * - * @param CRM_Core_Session $store - * - * @param int $mode - */ - public function set($store, $mode = self::MODE_SUMMARY) { - $store->set('fileSize', $this->_fileSize); - $store->set('lineCount', $this->_lineCount); - $store->set('separator', $this->_separator); - $store->set('fields', $this->getSelectValues()); - $store->set('fieldTypes', $this->getSelectTypes()); - - $store->set('headerPatterns', $this->getHeaderPatterns()); - $store->set('dataPatterns', $this->getDataPatterns()); - $store->set('columnCount', $this->_activeFieldCount); - - $store->set('totalRowCount', $this->_totalCount); - $store->set('validRowCount', $this->_validCount); - $store->set('invalidRowCount', $this->_invalidRowCount); - $store->set('invalidSoftCreditRowCount', $this->_invalidSoftCreditRowCount); - $store->set('validSoftCreditRowCount', $this->_validSoftCreditRowCount); - $store->set('invalidPledgePaymentRowCount', $this->_invalidPledgePaymentRowCount); - $store->set('validPledgePaymentRowCount', $this->_validPledgePaymentRowCount); - $store->set('conflictRowCount', $this->_conflictCount); - - switch ($this->_contactType) { - case 'Individual': - $store->set('contactType', CRM_Import_Parser::CONTACT_INDIVIDUAL); - break; - - case 'Household': - $store->set('contactType', CRM_Import_Parser::CONTACT_HOUSEHOLD); - break; - - case 'Organization': - $store->set('contactType', CRM_Import_Parser::CONTACT_ORGANIZATION); - } - - if ($this->_invalidRowCount) { - $store->set('errorsFileName', $this->_errorFileName); - } - if ($this->_conflictCount) { - $store->set('conflictsFileName', $this->_conflictFileName); - } - if (isset($this->_rows) && !empty($this->_rows)) { - $store->set('dataValues', $this->_rows); - } - - if ($this->_invalidPledgePaymentRowCount) { - $store->set('pledgePaymentErrorsFileName', $this->_pledgePaymentErrorsFileName); - } - - if ($this->_invalidSoftCreditRowCount) { - $store->set('softCreditErrorsFileName', $this->_softCreditErrorsFileName); - } - - if ($mode == self::MODE_IMPORT) { - $store->set('duplicateRowCount', $this->_duplicateCount); - if ($this->_duplicateCount) { - $store->set('duplicatesFileName', $this->_duplicateFileName); - } - } - } - - /** - * Export data to a CSV file. - * - * @param string $fileName - * @param array $header - * @param array $data - */ - public static function exportCSV($fileName, $header, $data) { - $output = []; - $fd = fopen($fileName, 'w'); - - foreach ($header as $key => $value) { - $header[$key] = "\"$value\""; - } - $config = CRM_Core_Config::singleton(); - $output[] = implode($config->fieldSeparator, $header); - - foreach ($data as $datum) { - foreach ($datum as $key => $value) { - if (isset($value[0]) && is_array($value)) { - foreach ($value[0] as $k1 => $v1) { - if ($k1 == 'location_type_id') { - continue; - } - $datum[$k1] = $v1; - } - } - else { - $datum[$key] = "\"$value\""; - } - } - $output[] = implode($config->fieldSeparator, $datum); - } - fwrite($fd, implode("\n", $output)); - fclose($fd); - } - - /** - * Determines the file extension based on error code. - * - * @param int $type - * Error code constant. - * - * @return string - */ - public static function errorFileName($type) { - $fileName = NULL; - if (empty($type)) { - return $fileName; - } - - $config = CRM_Core_Config::singleton(); - $fileName = $config->uploadDir . "sqlImport"; - - switch ($type) { - case CRM_Contribute_Import_Parser::SOFT_CREDIT_ERROR: - $fileName .= '.softCreditErrors'; - break; - - case CRM_Contribute_Import_Parser::PLEDGE_PAYMENT_ERROR: - $fileName .= '.pledgePaymentErrors'; - break; - - default: - $fileName = parent::errorFileName($type); - break; - } - - return $fileName; - } - - /** - * Determines the file name based on error code. - * - * @param int $type - * Error code constant. - * - * @return string - */ - public static function saveFileName($type) { - $fileName = NULL; - if (empty($type)) { - return $fileName; - } - - switch ($type) { - case CRM_Contribute_Import_Parser::SOFT_CREDIT_ERROR: - $fileName = 'Import_Soft_Credit_Errors.csv'; - break; - - case CRM_Contribute_Import_Parser::PLEDGE_PAYMENT_ERROR: - $fileName = 'Import_Pledge_Payment_Errors.csv'; - break; - - default: - $fileName = parent::saveFileName($type); - break; - } - - return $fileName; - } - -} diff --git a/CRM/Contribute/Import/Parser/Contribution.php b/CRM/Contribute/Import/Parser/Contribution.php index a8a98ef50dfa..d5386b77e8f8 100644 --- a/CRM/Contribute/Import/Parser/Contribution.php +++ b/CRM/Contribute/Import/Parser/Contribution.php @@ -18,7 +18,7 @@ /** * Class to parse contribution csv files. */ -class CRM_Contribute_Import_Parser_Contribution extends CRM_Contribute_Import_Parser { +class CRM_Contribute_Import_Parser_Contribution extends CRM_Import_Parser { protected $_mapperKeys; @@ -51,6 +51,654 @@ public function __construct(&$mapperKeys, $mapperSoftCredit = [], $mapperPhoneTy $this->_mapperSoftCreditType = &$mapperSoftCreditType; } + /** + * Contribution-specific result codes + * @see CRM_Import_Parser result code constants + */ + const SOFT_CREDIT = 512, SOFT_CREDIT_ERROR = 1024, PLEDGE_PAYMENT = 2048, PLEDGE_PAYMENT_ERROR = 4096; + + /** + * @var string + */ + protected $_fileName; + + /** + * Imported file size + * @var int + */ + protected $_fileSize; + + /** + * Separator being used + * @var string + */ + protected $_separator; + + /** + * Total number of lines in file + * @var int + */ + protected $_lineCount; + + /** + * Running total number of valid soft credit rows + * @var int + */ + protected $_validSoftCreditRowCount; + + /** + * Running total number of invalid soft credit rows + * @var int + */ + protected $_invalidSoftCreditRowCount; + + /** + * Running total number of valid pledge payment rows + * @var int + */ + protected $_validPledgePaymentRowCount; + + /** + * Running total number of invalid pledge payment rows + * @var int + */ + protected $_invalidPledgePaymentRowCount; + + /** + * Array of pledge payment error lines, bounded by MAX_ERROR + * @var array + */ + protected $_pledgePaymentErrors; + + /** + * Array of pledge payment error lines, bounded by MAX_ERROR + * @var array + */ + protected $_softCreditErrors; + + /** + * Filename of pledge payment error data + * + * @var string + */ + protected $_pledgePaymentErrorsFileName; + + /** + * Filename of soft credit error data + * + * @var string + */ + protected $_softCreditErrorsFileName; + + /** + * Whether the file has a column header or not + * + * @var bool + */ + protected $_haveColumnHeader; + + /** + * @param string $fileName + * @param string $separator + * @param $mapper + * @param bool $skipColumnHeader + * @param int $mode + * @param int $contactType + * @param int $onDuplicate + * @param int $statusID + * @param int $totalRowCount + * + * @return mixed + * @throws Exception + */ + public function run( + $fileName, + $separator, + &$mapper, + $skipColumnHeader = FALSE, + $mode = self::MODE_PREVIEW, + $contactType = self::CONTACT_INDIVIDUAL, + $onDuplicate = self::DUPLICATE_SKIP, + $statusID = NULL, + $totalRowCount = NULL + ) { + if (!is_array($fileName)) { + throw new CRM_Core_Exception('Unable to determine import file'); + } + $fileName = $fileName['name']; + + switch ($contactType) { + case self::CONTACT_INDIVIDUAL: + $this->_contactType = 'Individual'; + break; + + case self::CONTACT_HOUSEHOLD: + $this->_contactType = 'Household'; + break; + + case self::CONTACT_ORGANIZATION: + $this->_contactType = 'Organization'; + } + + $this->init(); + + $this->_haveColumnHeader = $skipColumnHeader; + + $this->_separator = $separator; + + $fd = fopen($fileName, "r"); + if (!$fd) { + return FALSE; + } + + $this->_lineCount = $this->_warningCount = $this->_validSoftCreditRowCount = $this->_validPledgePaymentRowCount = 0; + $this->_invalidRowCount = $this->_validCount = $this->_invalidSoftCreditRowCount = $this->_invalidPledgePaymentRowCount = 0; + $this->_totalCount = $this->_conflictCount = 0; + + $this->_errors = []; + $this->_warnings = []; + $this->_conflicts = []; + $this->_pledgePaymentErrors = []; + $this->_softCreditErrors = []; + if ($statusID) { + $this->progressImport($statusID); + $startTimestamp = $currTimestamp = $prevTimestamp = time(); + } + + $this->_fileSize = number_format(filesize($fileName) / 1024.0, 2); + + if ($mode == self::MODE_MAPFIELD) { + $this->_rows = []; + } + else { + $this->_activeFieldCount = count($this->_activeFields); + } + + while (!feof($fd)) { + $this->_lineCount++; + + $values = fgetcsv($fd, 8192, $separator); + if (!$values) { + continue; + } + + self::encloseScrub($values); + + // skip column header if we're not in mapfield mode + if ($mode != self::MODE_MAPFIELD && $skipColumnHeader) { + $skipColumnHeader = FALSE; + continue; + } + + /* trim whitespace around the values */ + + $empty = TRUE; + foreach ($values as $k => $v) { + $values[$k] = trim($v, " \t\r\n"); + } + + if (CRM_Utils_System::isNull($values)) { + continue; + } + + $this->_totalCount++; + + if ($mode == self::MODE_MAPFIELD) { + $returnCode = $this->mapField($values); + } + elseif ($mode == self::MODE_PREVIEW) { + $returnCode = $this->preview($values); + } + elseif ($mode == self::MODE_SUMMARY) { + $returnCode = $this->summary($values); + } + elseif ($mode == self::MODE_IMPORT) { + $returnCode = $this->import($onDuplicate, $values); + if ($statusID && (($this->_lineCount % 50) == 0)) { + $prevTimestamp = $this->progressImport($statusID, FALSE, $startTimestamp, $prevTimestamp, $totalRowCount); + } + } + else { + $returnCode = self::ERROR; + } + + // note that a line could be valid but still produce a warning + if ($returnCode == self::VALID) { + $this->_validCount++; + if ($mode == self::MODE_MAPFIELD) { + $this->_rows[] = $values; + $this->_activeFieldCount = max($this->_activeFieldCount, count($values)); + } + } + + if ($returnCode == self::SOFT_CREDIT) { + $this->_validSoftCreditRowCount++; + $this->_validCount++; + if ($mode == self::MODE_MAPFIELD) { + $this->_rows[] = $values; + $this->_activeFieldCount = max($this->_activeFieldCount, count($values)); + } + } + + if ($returnCode == self::PLEDGE_PAYMENT) { + $this->_validPledgePaymentRowCount++; + $this->_validCount++; + if ($mode == self::MODE_MAPFIELD) { + $this->_rows[] = $values; + $this->_activeFieldCount = max($this->_activeFieldCount, count($values)); + } + } + + if ($returnCode == self::WARNING) { + $this->_warningCount++; + if ($this->_warningCount < $this->_maxWarningCount) { + $this->_warningCount[] = $line; + } + } + + if ($returnCode == self::ERROR) { + $this->_invalidRowCount++; + $recordNumber = $this->_lineCount; + if ($this->_haveColumnHeader) { + $recordNumber--; + } + array_unshift($values, $recordNumber); + $this->_errors[] = $values; + } + + if ($returnCode == self::PLEDGE_PAYMENT_ERROR) { + $this->_invalidPledgePaymentRowCount++; + $recordNumber = $this->_lineCount; + if ($this->_haveColumnHeader) { + $recordNumber--; + } + array_unshift($values, $recordNumber); + $this->_pledgePaymentErrors[] = $values; + } + + if ($returnCode == self::SOFT_CREDIT_ERROR) { + $this->_invalidSoftCreditRowCount++; + $recordNumber = $this->_lineCount; + if ($this->_haveColumnHeader) { + $recordNumber--; + } + array_unshift($values, $recordNumber); + $this->_softCreditErrors[] = $values; + } + + if ($returnCode == self::CONFLICT) { + $this->_conflictCount++; + $recordNumber = $this->_lineCount; + if ($this->_haveColumnHeader) { + $recordNumber--; + } + array_unshift($values, $recordNumber); + $this->_conflicts[] = $values; + } + + if ($returnCode == self::DUPLICATE) { + if ($returnCode == self::MULTIPLE_DUPE) { + /* TODO: multi-dupes should be counted apart from singles + * on non-skip action */ + } + $this->_duplicateCount++; + $recordNumber = $this->_lineCount; + if ($this->_haveColumnHeader) { + $recordNumber--; + } + array_unshift($values, $recordNumber); + $this->_duplicates[] = $values; + if ($onDuplicate != self::DUPLICATE_SKIP) { + $this->_validCount++; + } + } + + // we give the derived class a way of aborting the process + // note that the return code could be multiple code or'ed together + if ($returnCode == self::STOP) { + break; + } + + // if we are done processing the maxNumber of lines, break + if ($this->_maxLinesToProcess > 0 && $this->_validCount >= $this->_maxLinesToProcess) { + break; + } + } + + fclose($fd); + + if ($mode == self::MODE_PREVIEW || $mode == self::MODE_IMPORT) { + $customHeaders = $mapper; + + $customfields = CRM_Core_BAO_CustomField::getFields('Contribution'); + foreach ($customHeaders as $key => $value) { + if ($id = CRM_Core_BAO_CustomField::getKeyID($value)) { + $customHeaders[$key] = $customfields[$id][0]; + } + } + if ($this->_invalidRowCount) { + // removed view url for invlaid contacts + $headers = array_merge([ + ts('Line Number'), + ts('Reason'), + ], $customHeaders); + $this->_errorFileName = self::errorFileName(self::ERROR); + self::exportCSV($this->_errorFileName, $headers, $this->_errors); + } + + if ($this->_invalidPledgePaymentRowCount) { + // removed view url for invlaid contacts + $headers = array_merge([ + ts('Line Number'), + ts('Reason'), + ], $customHeaders); + $this->_pledgePaymentErrorsFileName = self::errorFileName(self::PLEDGE_PAYMENT_ERROR); + self::exportCSV($this->_pledgePaymentErrorsFileName, $headers, $this->_pledgePaymentErrors); + } + + if ($this->_invalidSoftCreditRowCount) { + // removed view url for invlaid contacts + $headers = array_merge([ + ts('Line Number'), + ts('Reason'), + ], $customHeaders); + $this->_softCreditErrorsFileName = self::errorFileName(self::SOFT_CREDIT_ERROR); + self::exportCSV($this->_softCreditErrorsFileName, $headers, $this->_softCreditErrors); + } + + if ($this->_conflictCount) { + $headers = array_merge([ + ts('Line Number'), + ts('Reason'), + ], $customHeaders); + $this->_conflictFileName = self::errorFileName(self::CONFLICT); + self::exportCSV($this->_conflictFileName, $headers, $this->_conflicts); + } + if ($this->_duplicateCount) { + $headers = array_merge([ + ts('Line Number'), + ts('View Contribution URL'), + ], $customHeaders); + + $this->_duplicateFileName = self::errorFileName(self::DUPLICATE); + self::exportCSV($this->_duplicateFileName, $headers, $this->_duplicates); + } + } + return $this->fini(); + } + + /** + * Given a list of the importable field keys that the user has selected + * set the active fields array to this list + * + * @param array $fieldKeys mapped array of values + */ + public function setActiveFields($fieldKeys) { + $this->_activeFieldCount = count($fieldKeys); + foreach ($fieldKeys as $key) { + if (empty($this->_fields[$key])) { + $this->_activeFields[] = new CRM_Contribute_Import_Field('', ts('- do not import -')); + } + else { + $this->_activeFields[] = clone($this->_fields[$key]); + } + } + } + + /** + * Store the soft credit field information. + * + * This was perhaps done this way on the believe that a lot of code pain + * was worth it to avoid negligible-cost array iterations. Perhaps we could prioritise + * readability & maintainability next since we can just work with functions to retrieve + * data from the metadata. + * + * @param array $elements + */ + public function setActiveFieldSoftCredit($elements) { + foreach ((array) $elements as $i => $element) { + $this->_activeFields[$i]->_softCreditField = $element; + } + } + + /** + * Store the soft credit field type information. + * + * This was perhaps done this way on the believe that a lot of code pain + * was worth it to avoid negligible-cost array iterations. Perhaps we could prioritise + * readability & maintainability next since we can just work with functions to retrieve + * data from the metadata. + * + * @param array $elements + */ + public function setActiveFieldSoftCreditType($elements) { + foreach ((array) $elements as $i => $element) { + $this->_activeFields[$i]->_softCreditType = $element; + } + } + + /** + * Format the field values for input to the api. + * + * @return array + * (reference ) associative array of name/value pairs + */ + public function &getActiveFieldParams() { + $params = []; + for ($i = 0; $i < $this->_activeFieldCount; $i++) { + if (isset($this->_activeFields[$i]->_value)) { + if (isset($this->_activeFields[$i]->_softCreditField)) { + if (!isset($params[$this->_activeFields[$i]->_name])) { + $params[$this->_activeFields[$i]->_name] = []; + } + $params[$this->_activeFields[$i]->_name][$i][$this->_activeFields[$i]->_softCreditField] = $this->_activeFields[$i]->_value; + if (isset($this->_activeFields[$i]->_softCreditType)) { + $params[$this->_activeFields[$i]->_name][$i]['soft_credit_type_id'] = $this->_activeFields[$i]->_softCreditType; + } + } + + if (!isset($params[$this->_activeFields[$i]->_name])) { + if (!isset($this->_activeFields[$i]->_softCreditField)) { + $params[$this->_activeFields[$i]->_name] = $this->_activeFields[$i]->_value; + } + } + } + } + return $params; + } + + /** + * @param string $name + * @param $title + * @param int $type + * @param string $headerPattern + * @param string $dataPattern + */ + public function addField($name, $title, $type = CRM_Utils_Type::T_INT, $headerPattern = '//', $dataPattern = '//') { + if (empty($name)) { + $this->_fields['doNotImport'] = new CRM_Contribute_Import_Field($name, $title, $type, $headerPattern, $dataPattern); + } + else { + $tempField = CRM_Contact_BAO_Contact::importableFields('All', NULL); + if (!array_key_exists($name, $tempField)) { + $this->_fields[$name] = new CRM_Contribute_Import_Field($name, $title, $type, $headerPattern, $dataPattern); + } + else { + $this->_fields[$name] = new CRM_Contact_Import_Field($name, $title, $type, $headerPattern, $dataPattern, + CRM_Utils_Array::value('hasLocationType', $tempField[$name]) + ); + } + } + } + + /** + * Store parser values. + * + * @param CRM_Core_Session $store + * + * @param int $mode + */ + public function set($store, $mode = self::MODE_SUMMARY) { + $store->set('fileSize', $this->_fileSize); + $store->set('lineCount', $this->_lineCount); + $store->set('separator', $this->_separator); + $store->set('fields', $this->getSelectValues()); + $store->set('fieldTypes', $this->getSelectTypes()); + + $store->set('headerPatterns', $this->getHeaderPatterns()); + $store->set('dataPatterns', $this->getDataPatterns()); + $store->set('columnCount', $this->_activeFieldCount); + + $store->set('totalRowCount', $this->_totalCount); + $store->set('validRowCount', $this->_validCount); + $store->set('invalidRowCount', $this->_invalidRowCount); + $store->set('invalidSoftCreditRowCount', $this->_invalidSoftCreditRowCount); + $store->set('validSoftCreditRowCount', $this->_validSoftCreditRowCount); + $store->set('invalidPledgePaymentRowCount', $this->_invalidPledgePaymentRowCount); + $store->set('validPledgePaymentRowCount', $this->_validPledgePaymentRowCount); + $store->set('conflictRowCount', $this->_conflictCount); + + switch ($this->_contactType) { + case 'Individual': + $store->set('contactType', CRM_Import_Parser::CONTACT_INDIVIDUAL); + break; + + case 'Household': + $store->set('contactType', CRM_Import_Parser::CONTACT_HOUSEHOLD); + break; + + case 'Organization': + $store->set('contactType', CRM_Import_Parser::CONTACT_ORGANIZATION); + } + + if ($this->_invalidRowCount) { + $store->set('errorsFileName', $this->_errorFileName); + } + if ($this->_conflictCount) { + $store->set('conflictsFileName', $this->_conflictFileName); + } + if (isset($this->_rows) && !empty($this->_rows)) { + $store->set('dataValues', $this->_rows); + } + + if ($this->_invalidPledgePaymentRowCount) { + $store->set('pledgePaymentErrorsFileName', $this->_pledgePaymentErrorsFileName); + } + + if ($this->_invalidSoftCreditRowCount) { + $store->set('softCreditErrorsFileName', $this->_softCreditErrorsFileName); + } + + if ($mode == self::MODE_IMPORT) { + $store->set('duplicateRowCount', $this->_duplicateCount); + if ($this->_duplicateCount) { + $store->set('duplicatesFileName', $this->_duplicateFileName); + } + } + } + + /** + * Export data to a CSV file. + * + * @param string $fileName + * @param array $header + * @param array $data + */ + public static function exportCSV($fileName, $header, $data) { + $output = []; + $fd = fopen($fileName, 'w'); + + foreach ($header as $key => $value) { + $header[$key] = "\"$value\""; + } + $config = CRM_Core_Config::singleton(); + $output[] = implode($config->fieldSeparator, $header); + + foreach ($data as $datum) { + foreach ($datum as $key => $value) { + if (isset($value[0]) && is_array($value)) { + foreach ($value[0] as $k1 => $v1) { + if ($k1 == 'location_type_id') { + continue; + } + $datum[$k1] = $v1; + } + } + else { + $datum[$key] = "\"$value\""; + } + } + $output[] = implode($config->fieldSeparator, $datum); + } + fwrite($fd, implode("\n", $output)); + fclose($fd); + } + + /** + * Determines the file extension based on error code. + * + * @param int $type + * Error code constant. + * + * @return string + */ + public static function errorFileName($type) { + $fileName = NULL; + if (empty($type)) { + return $fileName; + } + + $config = CRM_Core_Config::singleton(); + $fileName = $config->uploadDir . "sqlImport"; + + switch ($type) { + case self::SOFT_CREDIT_ERROR: + $fileName .= '.softCreditErrors'; + break; + + case self::PLEDGE_PAYMENT_ERROR: + $fileName .= '.pledgePaymentErrors'; + break; + + default: + $fileName = parent::errorFileName($type); + break; + } + + return $fileName; + } + + /** + * Determines the file name based on error code. + * + * @param int $type + * Error code constant. + * + * @return string + */ + public static function saveFileName($type) { + $fileName = NULL; + if (empty($type)) { + return $fileName; + } + + switch ($type) { + case self::SOFT_CREDIT_ERROR: + $fileName = 'Import_Soft_Credit_Errors.csv'; + break; + + case self::PLEDGE_PAYMENT_ERROR: + $fileName = 'Import_Pledge_Payment_Errors.csv'; + break; + + default: + $fileName = parent::saveFileName($type); + break; + } + + return $fileName; + } + /** * The initializer code, called before the processing */ @@ -245,10 +893,10 @@ public function import($onDuplicate, &$values) { if ($formatError) { array_unshift($values, $formatError['error_message']); if (CRM_Utils_Array::value('error_data', $formatError) == 'soft_credit') { - return CRM_Contribute_Import_Parser::SOFT_CREDIT_ERROR; + return self::SOFT_CREDIT_ERROR; } if (CRM_Utils_Array::value('error_data', $formatError) == 'pledge_payment') { - return CRM_Contribute_Import_Parser::PLEDGE_PAYMENT_ERROR; + return self::PLEDGE_PAYMENT_ERROR; } return CRM_Import_Parser::ERROR; } @@ -325,7 +973,7 @@ public function import($onDuplicate, &$values) { //return soft valid since we need to show how soft credits were added if (!empty($formatted['soft_credit'])) { - return CRM_Contribute_Import_Parser::SOFT_CREDIT; + return self::SOFT_CREDIT; } // process pledge payment assoc w/ the contribution @@ -379,7 +1027,7 @@ public function import($onDuplicate, &$values) { //return soft valid since we need to show how soft credits were added if (!empty($formatted['soft_credit'])) { - return CRM_Contribute_Import_Parser::SOFT_CREDIT; + return self::SOFT_CREDIT; } // process pledge payment assoc w/ the contribution @@ -446,7 +1094,7 @@ public function import($onDuplicate, &$values) { //return soft valid since we need to show how soft credits were added if (!empty($formatted['soft_credit'])) { - return CRM_Contribute_Import_Parser::SOFT_CREDIT; + return self::SOFT_CREDIT; } // process pledge payment assoc w/ the contribution @@ -477,7 +1125,7 @@ public function processPledgePayments(array $formatted) { $formatted['total_amount'] ); - return CRM_Contribute_Import_Parser::PLEDGE_PAYMENT; + return self::PLEDGE_PAYMENT; } } diff --git a/tests/phpunit/CRM/Contribute/Import/Parser/ContributionTest.php b/tests/phpunit/CRM/Contribute/Import/Parser/ContributionTest.php index 2982049681d7..faac17af3f81 100644 --- a/tests/phpunit/CRM/Contribute/Import/Parser/ContributionTest.php +++ b/tests/phpunit/CRM/Contribute/Import/Parser/ContributionTest.php @@ -70,7 +70,7 @@ public function testImportParserWithSoftCreditsByExternalIdentifier(string $thou ]; $mapperSoftCredit = [NULL, NULL, NULL, 'external_identifier']; $mapperSoftCreditType = [NULL, NULL, NULL, '1']; - $this->runImport($values, CRM_Import_Parser::DUPLICATE_UPDATE, CRM_Contribute_Import_Parser::SOFT_CREDIT, $mapperSoftCredit, NULL, $mapperSoftCreditType); + $this->runImport($values, CRM_Import_Parser::DUPLICATE_UPDATE, CRM_Contribute_Import_Parser_Contribution::SOFT_CREDIT, $mapperSoftCredit, NULL, $mapperSoftCreditType); $contributionsOfMainContact = Contribution::get()->addWhere('contact_id', '=', $contact1Id)->execute(); $this->assertCount(1, $contributionsOfMainContact, 'Contribution not added for primary contact');