From 8dc9763a2d9de8cdffa311fc0f2cb7ff8572ef94 Mon Sep 17 00:00:00 2001 From: Eileen McNaughton Date: Thu, 7 Apr 2022 10:39:09 +1200 Subject: [PATCH] Simplify class inheritance CRM_Contribute_Import_Parser_Contribution is the only class in our universe to extend CRM_Contribute_Import_Parser and this class only adds confusion to the mix as functions are 'distributed' between them. This removes the inheritance, leaving only the constant that is still used in the now-deprecated class --- .toxic.json | 2 +- CRM/Contribute/Import/Form/Preview.php | 12 +- CRM/Contribute/Import/Form/Summary.php | 8 +- CRM/Contribute/Import/Parser.php | 667 ------------------ CRM/Contribute/Import/Parser/Contribution.php | 662 ++++++++++++++++- .../Import/Parser/ContributionTest.php | 2 +- 6 files changed, 667 insertions(+), 686 deletions(-) delete mode 100644 CRM/Contribute/Import/Parser.php diff --git a/.toxic.json b/.toxic.json index d24776f28356..f5f5c6fb2781 100644 --- a/.toxic.json +++ b/.toxic.json @@ -32,7 +32,7 @@ "CRM_Contribute_Form_Contribution_Main::formRule()": "toxicAlert", "CRM_Contribute_Form_Contribution_Main::submit()": "toxicAlert", "CRM_Contribute_Form_Task_Invoice::printPDF()": "toxicAlert", - "CRM_Contribute_Import_Parser::run()": "toxicAlert", + "CRM_Contribute_Import_Parser_Contribution::run()": "toxicAlert", "CRM_Core_BAO_ActionScheduleTest::setUp()": "toxicAlert", "CRM_Core_BAO_CustomField::formatCustomField()": "toxicAlert", "CRM_Core_BAO_CustomGroup::getTree()": "toxicAlert", diff --git a/CRM/Contribute/Import/Form/Preview.php b/CRM/Contribute/Import/Form/Preview.php index e45de43a25b8..9a6ba068dea8 100644 --- a/CRM/Contribute/Import/Form/Preview.php +++ b/CRM/Contribute/Import/Form/Preview.php @@ -54,17 +54,17 @@ public function preProcess() { } if ($invalidRowCount) { - $urlParams = 'type=' . CRM_Import_Parser::ERROR . '&parser=CRM_Contribute_Import_Parser'; + $urlParams = 'type=' . CRM_Import_Parser::ERROR . '&parser=CRM_Contribute_Import_Parser_Contribution'; $this->set('downloadErrorRecordsUrl', CRM_Utils_System::url('civicrm/export', $urlParams)); } if ($conflictRowCount) { - $urlParams = 'type=' . CRM_Import_Parser::CONFLICT . '&parser=CRM_Contribute_Import_Parser'; + $urlParams = 'type=' . CRM_Import_Parser::CONFLICT . '&parser=CRM_Contribute_Import_Parser_Contribution'; $this->set('downloadConflictRecordsUrl', CRM_Utils_System::url('civicrm/export', $urlParams)); } if ($mismatchCount) { - $urlParams = 'type=' . CRM_Import_Parser::NO_MATCH . '&parser=CRM_Contribute_Import_Parser'; + $urlParams = 'type=' . CRM_Import_Parser::NO_MATCH . '&parser=CRM_Contribute_Import_Parser_Contribution'; $this->set('downloadMismatchRecordsUrl', CRM_Utils_System::url('civicrm/export', $urlParams)); } @@ -160,11 +160,11 @@ public function postProcess() { fclose($fd); $this->set('errorFile', $errorFile); - $urlParams = 'type=' . CRM_Import_Parser::ERROR . '&parser=CRM_Contribute_Import_Parser'; + $urlParams = 'type=' . CRM_Import_Parser::ERROR . '&parser=CRM_Contribute_Import_Parser_Contribution'; $this->set('downloadErrorRecordsUrl', CRM_Utils_System::url('civicrm/export', $urlParams)); - $urlParams = 'type=' . CRM_Import_Parser::CONFLICT . '&parser=CRM_Contribute_Import_Parser'; + $urlParams = 'type=' . CRM_Import_Parser::CONFLICT . '&parser=CRM_Contribute_Import_Parser_Contribution'; $this->set('downloadConflictRecordsUrl', CRM_Utils_System::url('civicrm/export', $urlParams)); - $urlParams = 'type=' . CRM_Import_Parser::NO_MATCH . '&parser=CRM_Contribute_Import_Parser'; + $urlParams = 'type=' . CRM_Import_Parser::NO_MATCH . '&parser=CRM_Contribute_Import_Parser_Contribution'; $this->set('downloadMismatchRecordsUrl', CRM_Utils_System::url('civicrm/export', $urlParams)); } } diff --git a/CRM/Contribute/Import/Form/Summary.php b/CRM/Contribute/Import/Form/Summary.php index 4d86b06058b5..6714befabc7e 100644 --- a/CRM/Contribute/Import/Form/Summary.php +++ b/CRM/Contribute/Import/Form/Summary.php @@ -35,13 +35,13 @@ public function preProcess() { $invalidRowCount = $this->get('invalidRowCount'); $invalidSoftCreditRowCount = $this->get('invalidSoftCreditRowCount'); if ($invalidSoftCreditRowCount) { - $urlParams = 'type=' . CRM_Contribute_Import_Parser::SOFT_CREDIT_ERROR . '&parser=CRM_Contribute_Import_Parser'; + $urlParams = 'type=' . CRM_Contribute_Import_Parser_Contribution::SOFT_CREDIT_ERROR . '&parser=CRM_Contribute_Import_Parser_Contribution'; $this->set('downloadSoftCreditErrorRecordsUrl', CRM_Utils_System::url('civicrm/export', $urlParams)); } $validSoftCreditRowCount = $this->get('validSoftCreditRowCount'); $invalidPledgePaymentRowCount = $this->get('invalidPledgePaymentRowCount'); if ($invalidPledgePaymentRowCount) { - $urlParams = 'type=' . CRM_Contribute_Import_Parser::PLEDGE_PAYMENT_ERROR . '&parser=CRM_Contribute_Import_Parser'; + $urlParams = 'type=' . CRM_Contribute_Import_Parser_Contribution::PLEDGE_PAYMENT_ERROR . '&parser=CRM_Contribute_Import_Parser_Contribution'; $this->set('downloadPledgePaymentErrorRecordsUrl', CRM_Utils_System::url('civicrm/export', $urlParams)); } $validPledgePaymentRowCount = $this->get('validPledgePaymentRowCount'); @@ -50,11 +50,11 @@ public function preProcess() { $onDuplicate = $this->get('onDuplicate'); $mismatchCount = $this->get('unMatchCount'); if ($duplicateRowCount > 0) { - $urlParams = 'type=' . CRM_Import_Parser::DUPLICATE . '&parser=CRM_Contribute_Import_Parser'; + $urlParams = 'type=' . CRM_Import_Parser::DUPLICATE . '&parser=CRM_Contribute_Import_Parser_Contribution'; $this->set('downloadDuplicateRecordsUrl', CRM_Utils_System::url('civicrm/export', $urlParams)); } elseif ($mismatchCount) { - $urlParams = 'type=' . CRM_Import_Parser::NO_MATCH . '&parser=CRM_Contribute_Import_Parser'; + $urlParams = 'type=' . CRM_Import_Parser::NO_MATCH . '&parser=CRM_Contribute_Import_Parser_Contribution'; $this->set('downloadMismatchRecordsUrl', CRM_Utils_System::url('civicrm/export', $urlParams)); } else { diff --git a/CRM/Contribute/Import/Parser.php b/CRM/Contribute/Import/Parser.php deleted file mode 100644 index 8cd494f3869f..000000000000 --- a/CRM/Contribute/Import/Parser.php +++ /dev/null @@ -1,667 +0,0 @@ -_contactType = 'Individual'; - break; - - case self::CONTACT_HOUSEHOLD: - $this->_contactType = 'Household'; - break; - - case self::CONTACT_ORGANIZATION: - $this->_contactType = 'Organization'; - } - - $this->init(); - - $this->_haveColumnHeader = $skipColumnHeader; - - $this->_separator = $separator; - - $fd = fopen($fileName, "r"); - if (!$fd) { - return FALSE; - } - - $this->_lineCount = $this->_warningCount = $this->_validSoftCreditRowCount = $this->_validPledgePaymentRowCount = 0; - $this->_invalidRowCount = $this->_validCount = $this->_invalidSoftCreditRowCount = $this->_invalidPledgePaymentRowCount = 0; - $this->_totalCount = $this->_conflictCount = 0; - - $this->_errors = []; - $this->_warnings = []; - $this->_conflicts = []; - $this->_pledgePaymentErrors = []; - $this->_softCreditErrors = []; - if ($statusID) { - $this->progressImport($statusID); - $startTimestamp = $currTimestamp = $prevTimestamp = time(); - } - - $this->_fileSize = number_format(filesize($fileName) / 1024.0, 2); - - if ($mode == self::MODE_MAPFIELD) { - $this->_rows = []; - } - else { - $this->_activeFieldCount = count($this->_activeFields); - } - - while (!feof($fd)) { - $this->_lineCount++; - - $values = fgetcsv($fd, 8192, $separator); - if (!$values) { - continue; - } - - self::encloseScrub($values); - - // skip column header if we're not in mapfield mode - if ($mode != self::MODE_MAPFIELD && $skipColumnHeader) { - $skipColumnHeader = FALSE; - continue; - } - - /* trim whitespace around the values */ - - $empty = TRUE; - foreach ($values as $k => $v) { - $values[$k] = trim($v, " \t\r\n"); - } - - if (CRM_Utils_System::isNull($values)) { - continue; - } - - $this->_totalCount++; - - if ($mode == self::MODE_MAPFIELD) { - $returnCode = $this->mapField($values); - } - elseif ($mode == self::MODE_PREVIEW) { - $returnCode = $this->preview($values); - } - elseif ($mode == self::MODE_SUMMARY) { - $returnCode = $this->summary($values); - } - elseif ($mode == self::MODE_IMPORT) { - $returnCode = $this->import($onDuplicate, $values); - if ($statusID && (($this->_lineCount % 50) == 0)) { - $prevTimestamp = $this->progressImport($statusID, FALSE, $startTimestamp, $prevTimestamp, $totalRowCount); - } - } - else { - $returnCode = self::ERROR; - } - - // note that a line could be valid but still produce a warning - if ($returnCode == self::VALID) { - $this->_validCount++; - if ($mode == self::MODE_MAPFIELD) { - $this->_rows[] = $values; - $this->_activeFieldCount = max($this->_activeFieldCount, count($values)); - } - } - - if ($returnCode == self::SOFT_CREDIT) { - $this->_validSoftCreditRowCount++; - $this->_validCount++; - if ($mode == self::MODE_MAPFIELD) { - $this->_rows[] = $values; - $this->_activeFieldCount = max($this->_activeFieldCount, count($values)); - } - } - - if ($returnCode == self::PLEDGE_PAYMENT) { - $this->_validPledgePaymentRowCount++; - $this->_validCount++; - if ($mode == self::MODE_MAPFIELD) { - $this->_rows[] = $values; - $this->_activeFieldCount = max($this->_activeFieldCount, count($values)); - } - } - - if ($returnCode == self::WARNING) { - $this->_warningCount++; - if ($this->_warningCount < $this->_maxWarningCount) { - $this->_warningCount[] = $line; - } - } - - if ($returnCode == self::ERROR) { - $this->_invalidRowCount++; - $recordNumber = $this->_lineCount; - if ($this->_haveColumnHeader) { - $recordNumber--; - } - array_unshift($values, $recordNumber); - $this->_errors[] = $values; - } - - if ($returnCode == self::PLEDGE_PAYMENT_ERROR) { - $this->_invalidPledgePaymentRowCount++; - $recordNumber = $this->_lineCount; - if ($this->_haveColumnHeader) { - $recordNumber--; - } - array_unshift($values, $recordNumber); - $this->_pledgePaymentErrors[] = $values; - } - - if ($returnCode == self::SOFT_CREDIT_ERROR) { - $this->_invalidSoftCreditRowCount++; - $recordNumber = $this->_lineCount; - if ($this->_haveColumnHeader) { - $recordNumber--; - } - array_unshift($values, $recordNumber); - $this->_softCreditErrors[] = $values; - } - - if ($returnCode == self::CONFLICT) { - $this->_conflictCount++; - $recordNumber = $this->_lineCount; - if ($this->_haveColumnHeader) { - $recordNumber--; - } - array_unshift($values, $recordNumber); - $this->_conflicts[] = $values; - } - - if ($returnCode == self::DUPLICATE) { - if ($returnCode == self::MULTIPLE_DUPE) { - /* TODO: multi-dupes should be counted apart from singles - * on non-skip action */ - } - $this->_duplicateCount++; - $recordNumber = $this->_lineCount; - if ($this->_haveColumnHeader) { - $recordNumber--; - } - array_unshift($values, $recordNumber); - $this->_duplicates[] = $values; - if ($onDuplicate != self::DUPLICATE_SKIP) { - $this->_validCount++; - } - } - - // we give the derived class a way of aborting the process - // note that the return code could be multiple code or'ed together - if ($returnCode == self::STOP) { - break; - } - - // if we are done processing the maxNumber of lines, break - if ($this->_maxLinesToProcess > 0 && $this->_validCount >= $this->_maxLinesToProcess) { - break; - } - } - - fclose($fd); - - if ($mode == self::MODE_PREVIEW || $mode == self::MODE_IMPORT) { - $customHeaders = $mapper; - - $customfields = CRM_Core_BAO_CustomField::getFields('Contribution'); - foreach ($customHeaders as $key => $value) { - if ($id = CRM_Core_BAO_CustomField::getKeyID($value)) { - $customHeaders[$key] = $customfields[$id][0]; - } - } - if ($this->_invalidRowCount) { - // removed view url for invlaid contacts - $headers = array_merge([ - ts('Line Number'), - ts('Reason'), - ], $customHeaders); - $this->_errorFileName = self::errorFileName(self::ERROR); - self::exportCSV($this->_errorFileName, $headers, $this->_errors); - } - - if ($this->_invalidPledgePaymentRowCount) { - // removed view url for invlaid contacts - $headers = array_merge([ - ts('Line Number'), - ts('Reason'), - ], $customHeaders); - $this->_pledgePaymentErrorsFileName = self::errorFileName(self::PLEDGE_PAYMENT_ERROR); - self::exportCSV($this->_pledgePaymentErrorsFileName, $headers, $this->_pledgePaymentErrors); - } - - if ($this->_invalidSoftCreditRowCount) { - // removed view url for invlaid contacts - $headers = array_merge([ - ts('Line Number'), - ts('Reason'), - ], $customHeaders); - $this->_softCreditErrorsFileName = self::errorFileName(self::SOFT_CREDIT_ERROR); - self::exportCSV($this->_softCreditErrorsFileName, $headers, $this->_softCreditErrors); - } - - if ($this->_conflictCount) { - $headers = array_merge([ - ts('Line Number'), - ts('Reason'), - ], $customHeaders); - $this->_conflictFileName = self::errorFileName(self::CONFLICT); - self::exportCSV($this->_conflictFileName, $headers, $this->_conflicts); - } - if ($this->_duplicateCount) { - $headers = array_merge([ - ts('Line Number'), - ts('View Contribution URL'), - ], $customHeaders); - - $this->_duplicateFileName = self::errorFileName(self::DUPLICATE); - self::exportCSV($this->_duplicateFileName, $headers, $this->_duplicates); - } - } - return $this->fini(); - } - - /** - * Given a list of the importable field keys that the user has selected - * set the active fields array to this list - * - * @param array $fieldKeys mapped array of values - */ - public function setActiveFields($fieldKeys) { - $this->_activeFieldCount = count($fieldKeys); - foreach ($fieldKeys as $key) { - if (empty($this->_fields[$key])) { - $this->_activeFields[] = new CRM_Contribute_Import_Field('', ts('- do not import -')); - } - else { - $this->_activeFields[] = clone($this->_fields[$key]); - } - } - } - - /** - * Store the soft credit field information. - * - * This was perhaps done this way on the believe that a lot of code pain - * was worth it to avoid negligible-cost array iterations. Perhaps we could prioritise - * readability & maintainability next since we can just work with functions to retrieve - * data from the metadata. - * - * @param array $elements - */ - public function setActiveFieldSoftCredit($elements) { - foreach ((array) $elements as $i => $element) { - $this->_activeFields[$i]->_softCreditField = $element; - } - } - - /** - * Store the soft credit field type information. - * - * This was perhaps done this way on the believe that a lot of code pain - * was worth it to avoid negligible-cost array iterations. Perhaps we could prioritise - * readability & maintainability next since we can just work with functions to retrieve - * data from the metadata. - * - * @param array $elements - */ - public function setActiveFieldSoftCreditType($elements) { - foreach ((array) $elements as $i => $element) { - $this->_activeFields[$i]->_softCreditType = $element; - } - } - - /** - * Format the field values for input to the api. - * - * @return array - * (reference ) associative array of name/value pairs - */ - public function &getActiveFieldParams() { - $params = []; - for ($i = 0; $i < $this->_activeFieldCount; $i++) { - if (isset($this->_activeFields[$i]->_value)) { - if (isset($this->_activeFields[$i]->_softCreditField)) { - if (!isset($params[$this->_activeFields[$i]->_name])) { - $params[$this->_activeFields[$i]->_name] = []; - } - $params[$this->_activeFields[$i]->_name][$i][$this->_activeFields[$i]->_softCreditField] = $this->_activeFields[$i]->_value; - if (isset($this->_activeFields[$i]->_softCreditType)) { - $params[$this->_activeFields[$i]->_name][$i]['soft_credit_type_id'] = $this->_activeFields[$i]->_softCreditType; - } - } - - if (!isset($params[$this->_activeFields[$i]->_name])) { - if (!isset($this->_activeFields[$i]->_softCreditField)) { - $params[$this->_activeFields[$i]->_name] = $this->_activeFields[$i]->_value; - } - } - } - } - return $params; - } - - /** - * @param string $name - * @param $title - * @param int $type - * @param string $headerPattern - * @param string $dataPattern - */ - public function addField($name, $title, $type = CRM_Utils_Type::T_INT, $headerPattern = '//', $dataPattern = '//') { - if (empty($name)) { - $this->_fields['doNotImport'] = new CRM_Contribute_Import_Field($name, $title, $type, $headerPattern, $dataPattern); - } - else { - $tempField = CRM_Contact_BAO_Contact::importableFields('All', NULL); - if (!array_key_exists($name, $tempField)) { - $this->_fields[$name] = new CRM_Contribute_Import_Field($name, $title, $type, $headerPattern, $dataPattern); - } - else { - $this->_fields[$name] = new CRM_Contact_Import_Field($name, $title, $type, $headerPattern, $dataPattern, - CRM_Utils_Array::value('hasLocationType', $tempField[$name]) - ); - } - } - } - - /** - * Store parser values. - * - * @param CRM_Core_Session $store - * - * @param int $mode - */ - public function set($store, $mode = self::MODE_SUMMARY) { - $store->set('fileSize', $this->_fileSize); - $store->set('lineCount', $this->_lineCount); - $store->set('separator', $this->_separator); - $store->set('fields', $this->getSelectValues()); - $store->set('fieldTypes', $this->getSelectTypes()); - - $store->set('headerPatterns', $this->getHeaderPatterns()); - $store->set('dataPatterns', $this->getDataPatterns()); - $store->set('columnCount', $this->_activeFieldCount); - - $store->set('totalRowCount', $this->_totalCount); - $store->set('validRowCount', $this->_validCount); - $store->set('invalidRowCount', $this->_invalidRowCount); - $store->set('invalidSoftCreditRowCount', $this->_invalidSoftCreditRowCount); - $store->set('validSoftCreditRowCount', $this->_validSoftCreditRowCount); - $store->set('invalidPledgePaymentRowCount', $this->_invalidPledgePaymentRowCount); - $store->set('validPledgePaymentRowCount', $this->_validPledgePaymentRowCount); - $store->set('conflictRowCount', $this->_conflictCount); - - switch ($this->_contactType) { - case 'Individual': - $store->set('contactType', CRM_Import_Parser::CONTACT_INDIVIDUAL); - break; - - case 'Household': - $store->set('contactType', CRM_Import_Parser::CONTACT_HOUSEHOLD); - break; - - case 'Organization': - $store->set('contactType', CRM_Import_Parser::CONTACT_ORGANIZATION); - } - - if ($this->_invalidRowCount) { - $store->set('errorsFileName', $this->_errorFileName); - } - if ($this->_conflictCount) { - $store->set('conflictsFileName', $this->_conflictFileName); - } - if (isset($this->_rows) && !empty($this->_rows)) { - $store->set('dataValues', $this->_rows); - } - - if ($this->_invalidPledgePaymentRowCount) { - $store->set('pledgePaymentErrorsFileName', $this->_pledgePaymentErrorsFileName); - } - - if ($this->_invalidSoftCreditRowCount) { - $store->set('softCreditErrorsFileName', $this->_softCreditErrorsFileName); - } - - if ($mode == self::MODE_IMPORT) { - $store->set('duplicateRowCount', $this->_duplicateCount); - if ($this->_duplicateCount) { - $store->set('duplicatesFileName', $this->_duplicateFileName); - } - } - } - - /** - * Export data to a CSV file. - * - * @param string $fileName - * @param array $header - * @param array $data - */ - public static function exportCSV($fileName, $header, $data) { - $output = []; - $fd = fopen($fileName, 'w'); - - foreach ($header as $key => $value) { - $header[$key] = "\"$value\""; - } - $config = CRM_Core_Config::singleton(); - $output[] = implode($config->fieldSeparator, $header); - - foreach ($data as $datum) { - foreach ($datum as $key => $value) { - if (isset($value[0]) && is_array($value)) { - foreach ($value[0] as $k1 => $v1) { - if ($k1 == 'location_type_id') { - continue; - } - $datum[$k1] = $v1; - } - } - else { - $datum[$key] = "\"$value\""; - } - } - $output[] = implode($config->fieldSeparator, $datum); - } - fwrite($fd, implode("\n", $output)); - fclose($fd); - } - - /** - * Determines the file extension based on error code. - * - * @param int $type - * Error code constant. - * - * @return string - */ - public static function errorFileName($type) { - $fileName = NULL; - if (empty($type)) { - return $fileName; - } - - $config = CRM_Core_Config::singleton(); - $fileName = $config->uploadDir . "sqlImport"; - - switch ($type) { - case CRM_Contribute_Import_Parser::SOFT_CREDIT_ERROR: - $fileName .= '.softCreditErrors'; - break; - - case CRM_Contribute_Import_Parser::PLEDGE_PAYMENT_ERROR: - $fileName .= '.pledgePaymentErrors'; - break; - - default: - $fileName = parent::errorFileName($type); - break; - } - - return $fileName; - } - - /** - * Determines the file name based on error code. - * - * @param int $type - * Error code constant. - * - * @return string - */ - public static function saveFileName($type) { - $fileName = NULL; - if (empty($type)) { - return $fileName; - } - - switch ($type) { - case CRM_Contribute_Import_Parser::SOFT_CREDIT_ERROR: - $fileName = 'Import_Soft_Credit_Errors.csv'; - break; - - case CRM_Contribute_Import_Parser::PLEDGE_PAYMENT_ERROR: - $fileName = 'Import_Pledge_Payment_Errors.csv'; - break; - - default: - $fileName = parent::saveFileName($type); - break; - } - - return $fileName; - } - -} diff --git a/CRM/Contribute/Import/Parser/Contribution.php b/CRM/Contribute/Import/Parser/Contribution.php index a8a98ef50dfa..d5386b77e8f8 100644 --- a/CRM/Contribute/Import/Parser/Contribution.php +++ b/CRM/Contribute/Import/Parser/Contribution.php @@ -18,7 +18,7 @@ /** * Class to parse contribution csv files. */ -class CRM_Contribute_Import_Parser_Contribution extends CRM_Contribute_Import_Parser { +class CRM_Contribute_Import_Parser_Contribution extends CRM_Import_Parser { protected $_mapperKeys; @@ -51,6 +51,654 @@ public function __construct(&$mapperKeys, $mapperSoftCredit = [], $mapperPhoneTy $this->_mapperSoftCreditType = &$mapperSoftCreditType; } + /** + * Contribution-specific result codes + * @see CRM_Import_Parser result code constants + */ + const SOFT_CREDIT = 512, SOFT_CREDIT_ERROR = 1024, PLEDGE_PAYMENT = 2048, PLEDGE_PAYMENT_ERROR = 4096; + + /** + * @var string + */ + protected $_fileName; + + /** + * Imported file size + * @var int + */ + protected $_fileSize; + + /** + * Separator being used + * @var string + */ + protected $_separator; + + /** + * Total number of lines in file + * @var int + */ + protected $_lineCount; + + /** + * Running total number of valid soft credit rows + * @var int + */ + protected $_validSoftCreditRowCount; + + /** + * Running total number of invalid soft credit rows + * @var int + */ + protected $_invalidSoftCreditRowCount; + + /** + * Running total number of valid pledge payment rows + * @var int + */ + protected $_validPledgePaymentRowCount; + + /** + * Running total number of invalid pledge payment rows + * @var int + */ + protected $_invalidPledgePaymentRowCount; + + /** + * Array of pledge payment error lines, bounded by MAX_ERROR + * @var array + */ + protected $_pledgePaymentErrors; + + /** + * Array of pledge payment error lines, bounded by MAX_ERROR + * @var array + */ + protected $_softCreditErrors; + + /** + * Filename of pledge payment error data + * + * @var string + */ + protected $_pledgePaymentErrorsFileName; + + /** + * Filename of soft credit error data + * + * @var string + */ + protected $_softCreditErrorsFileName; + + /** + * Whether the file has a column header or not + * + * @var bool + */ + protected $_haveColumnHeader; + + /** + * @param string $fileName + * @param string $separator + * @param $mapper + * @param bool $skipColumnHeader + * @param int $mode + * @param int $contactType + * @param int $onDuplicate + * @param int $statusID + * @param int $totalRowCount + * + * @return mixed + * @throws Exception + */ + public function run( + $fileName, + $separator, + &$mapper, + $skipColumnHeader = FALSE, + $mode = self::MODE_PREVIEW, + $contactType = self::CONTACT_INDIVIDUAL, + $onDuplicate = self::DUPLICATE_SKIP, + $statusID = NULL, + $totalRowCount = NULL + ) { + if (!is_array($fileName)) { + throw new CRM_Core_Exception('Unable to determine import file'); + } + $fileName = $fileName['name']; + + switch ($contactType) { + case self::CONTACT_INDIVIDUAL: + $this->_contactType = 'Individual'; + break; + + case self::CONTACT_HOUSEHOLD: + $this->_contactType = 'Household'; + break; + + case self::CONTACT_ORGANIZATION: + $this->_contactType = 'Organization'; + } + + $this->init(); + + $this->_haveColumnHeader = $skipColumnHeader; + + $this->_separator = $separator; + + $fd = fopen($fileName, "r"); + if (!$fd) { + return FALSE; + } + + $this->_lineCount = $this->_warningCount = $this->_validSoftCreditRowCount = $this->_validPledgePaymentRowCount = 0; + $this->_invalidRowCount = $this->_validCount = $this->_invalidSoftCreditRowCount = $this->_invalidPledgePaymentRowCount = 0; + $this->_totalCount = $this->_conflictCount = 0; + + $this->_errors = []; + $this->_warnings = []; + $this->_conflicts = []; + $this->_pledgePaymentErrors = []; + $this->_softCreditErrors = []; + if ($statusID) { + $this->progressImport($statusID); + $startTimestamp = $currTimestamp = $prevTimestamp = time(); + } + + $this->_fileSize = number_format(filesize($fileName) / 1024.0, 2); + + if ($mode == self::MODE_MAPFIELD) { + $this->_rows = []; + } + else { + $this->_activeFieldCount = count($this->_activeFields); + } + + while (!feof($fd)) { + $this->_lineCount++; + + $values = fgetcsv($fd, 8192, $separator); + if (!$values) { + continue; + } + + self::encloseScrub($values); + + // skip column header if we're not in mapfield mode + if ($mode != self::MODE_MAPFIELD && $skipColumnHeader) { + $skipColumnHeader = FALSE; + continue; + } + + /* trim whitespace around the values */ + + $empty = TRUE; + foreach ($values as $k => $v) { + $values[$k] = trim($v, " \t\r\n"); + } + + if (CRM_Utils_System::isNull($values)) { + continue; + } + + $this->_totalCount++; + + if ($mode == self::MODE_MAPFIELD) { + $returnCode = $this->mapField($values); + } + elseif ($mode == self::MODE_PREVIEW) { + $returnCode = $this->preview($values); + } + elseif ($mode == self::MODE_SUMMARY) { + $returnCode = $this->summary($values); + } + elseif ($mode == self::MODE_IMPORT) { + $returnCode = $this->import($onDuplicate, $values); + if ($statusID && (($this->_lineCount % 50) == 0)) { + $prevTimestamp = $this->progressImport($statusID, FALSE, $startTimestamp, $prevTimestamp, $totalRowCount); + } + } + else { + $returnCode = self::ERROR; + } + + // note that a line could be valid but still produce a warning + if ($returnCode == self::VALID) { + $this->_validCount++; + if ($mode == self::MODE_MAPFIELD) { + $this->_rows[] = $values; + $this->_activeFieldCount = max($this->_activeFieldCount, count($values)); + } + } + + if ($returnCode == self::SOFT_CREDIT) { + $this->_validSoftCreditRowCount++; + $this->_validCount++; + if ($mode == self::MODE_MAPFIELD) { + $this->_rows[] = $values; + $this->_activeFieldCount = max($this->_activeFieldCount, count($values)); + } + } + + if ($returnCode == self::PLEDGE_PAYMENT) { + $this->_validPledgePaymentRowCount++; + $this->_validCount++; + if ($mode == self::MODE_MAPFIELD) { + $this->_rows[] = $values; + $this->_activeFieldCount = max($this->_activeFieldCount, count($values)); + } + } + + if ($returnCode == self::WARNING) { + $this->_warningCount++; + if ($this->_warningCount < $this->_maxWarningCount) { + $this->_warningCount[] = $line; + } + } + + if ($returnCode == self::ERROR) { + $this->_invalidRowCount++; + $recordNumber = $this->_lineCount; + if ($this->_haveColumnHeader) { + $recordNumber--; + } + array_unshift($values, $recordNumber); + $this->_errors[] = $values; + } + + if ($returnCode == self::PLEDGE_PAYMENT_ERROR) { + $this->_invalidPledgePaymentRowCount++; + $recordNumber = $this->_lineCount; + if ($this->_haveColumnHeader) { + $recordNumber--; + } + array_unshift($values, $recordNumber); + $this->_pledgePaymentErrors[] = $values; + } + + if ($returnCode == self::SOFT_CREDIT_ERROR) { + $this->_invalidSoftCreditRowCount++; + $recordNumber = $this->_lineCount; + if ($this->_haveColumnHeader) { + $recordNumber--; + } + array_unshift($values, $recordNumber); + $this->_softCreditErrors[] = $values; + } + + if ($returnCode == self::CONFLICT) { + $this->_conflictCount++; + $recordNumber = $this->_lineCount; + if ($this->_haveColumnHeader) { + $recordNumber--; + } + array_unshift($values, $recordNumber); + $this->_conflicts[] = $values; + } + + if ($returnCode == self::DUPLICATE) { + if ($returnCode == self::MULTIPLE_DUPE) { + /* TODO: multi-dupes should be counted apart from singles + * on non-skip action */ + } + $this->_duplicateCount++; + $recordNumber = $this->_lineCount; + if ($this->_haveColumnHeader) { + $recordNumber--; + } + array_unshift($values, $recordNumber); + $this->_duplicates[] = $values; + if ($onDuplicate != self::DUPLICATE_SKIP) { + $this->_validCount++; + } + } + + // we give the derived class a way of aborting the process + // note that the return code could be multiple code or'ed together + if ($returnCode == self::STOP) { + break; + } + + // if we are done processing the maxNumber of lines, break + if ($this->_maxLinesToProcess > 0 && $this->_validCount >= $this->_maxLinesToProcess) { + break; + } + } + + fclose($fd); + + if ($mode == self::MODE_PREVIEW || $mode == self::MODE_IMPORT) { + $customHeaders = $mapper; + + $customfields = CRM_Core_BAO_CustomField::getFields('Contribution'); + foreach ($customHeaders as $key => $value) { + if ($id = CRM_Core_BAO_CustomField::getKeyID($value)) { + $customHeaders[$key] = $customfields[$id][0]; + } + } + if ($this->_invalidRowCount) { + // removed view url for invlaid contacts + $headers = array_merge([ + ts('Line Number'), + ts('Reason'), + ], $customHeaders); + $this->_errorFileName = self::errorFileName(self::ERROR); + self::exportCSV($this->_errorFileName, $headers, $this->_errors); + } + + if ($this->_invalidPledgePaymentRowCount) { + // removed view url for invlaid contacts + $headers = array_merge([ + ts('Line Number'), + ts('Reason'), + ], $customHeaders); + $this->_pledgePaymentErrorsFileName = self::errorFileName(self::PLEDGE_PAYMENT_ERROR); + self::exportCSV($this->_pledgePaymentErrorsFileName, $headers, $this->_pledgePaymentErrors); + } + + if ($this->_invalidSoftCreditRowCount) { + // removed view url for invlaid contacts + $headers = array_merge([ + ts('Line Number'), + ts('Reason'), + ], $customHeaders); + $this->_softCreditErrorsFileName = self::errorFileName(self::SOFT_CREDIT_ERROR); + self::exportCSV($this->_softCreditErrorsFileName, $headers, $this->_softCreditErrors); + } + + if ($this->_conflictCount) { + $headers = array_merge([ + ts('Line Number'), + ts('Reason'), + ], $customHeaders); + $this->_conflictFileName = self::errorFileName(self::CONFLICT); + self::exportCSV($this->_conflictFileName, $headers, $this->_conflicts); + } + if ($this->_duplicateCount) { + $headers = array_merge([ + ts('Line Number'), + ts('View Contribution URL'), + ], $customHeaders); + + $this->_duplicateFileName = self::errorFileName(self::DUPLICATE); + self::exportCSV($this->_duplicateFileName, $headers, $this->_duplicates); + } + } + return $this->fini(); + } + + /** + * Given a list of the importable field keys that the user has selected + * set the active fields array to this list + * + * @param array $fieldKeys mapped array of values + */ + public function setActiveFields($fieldKeys) { + $this->_activeFieldCount = count($fieldKeys); + foreach ($fieldKeys as $key) { + if (empty($this->_fields[$key])) { + $this->_activeFields[] = new CRM_Contribute_Import_Field('', ts('- do not import -')); + } + else { + $this->_activeFields[] = clone($this->_fields[$key]); + } + } + } + + /** + * Store the soft credit field information. + * + * This was perhaps done this way on the believe that a lot of code pain + * was worth it to avoid negligible-cost array iterations. Perhaps we could prioritise + * readability & maintainability next since we can just work with functions to retrieve + * data from the metadata. + * + * @param array $elements + */ + public function setActiveFieldSoftCredit($elements) { + foreach ((array) $elements as $i => $element) { + $this->_activeFields[$i]->_softCreditField = $element; + } + } + + /** + * Store the soft credit field type information. + * + * This was perhaps done this way on the believe that a lot of code pain + * was worth it to avoid negligible-cost array iterations. Perhaps we could prioritise + * readability & maintainability next since we can just work with functions to retrieve + * data from the metadata. + * + * @param array $elements + */ + public function setActiveFieldSoftCreditType($elements) { + foreach ((array) $elements as $i => $element) { + $this->_activeFields[$i]->_softCreditType = $element; + } + } + + /** + * Format the field values for input to the api. + * + * @return array + * (reference ) associative array of name/value pairs + */ + public function &getActiveFieldParams() { + $params = []; + for ($i = 0; $i < $this->_activeFieldCount; $i++) { + if (isset($this->_activeFields[$i]->_value)) { + if (isset($this->_activeFields[$i]->_softCreditField)) { + if (!isset($params[$this->_activeFields[$i]->_name])) { + $params[$this->_activeFields[$i]->_name] = []; + } + $params[$this->_activeFields[$i]->_name][$i][$this->_activeFields[$i]->_softCreditField] = $this->_activeFields[$i]->_value; + if (isset($this->_activeFields[$i]->_softCreditType)) { + $params[$this->_activeFields[$i]->_name][$i]['soft_credit_type_id'] = $this->_activeFields[$i]->_softCreditType; + } + } + + if (!isset($params[$this->_activeFields[$i]->_name])) { + if (!isset($this->_activeFields[$i]->_softCreditField)) { + $params[$this->_activeFields[$i]->_name] = $this->_activeFields[$i]->_value; + } + } + } + } + return $params; + } + + /** + * @param string $name + * @param $title + * @param int $type + * @param string $headerPattern + * @param string $dataPattern + */ + public function addField($name, $title, $type = CRM_Utils_Type::T_INT, $headerPattern = '//', $dataPattern = '//') { + if (empty($name)) { + $this->_fields['doNotImport'] = new CRM_Contribute_Import_Field($name, $title, $type, $headerPattern, $dataPattern); + } + else { + $tempField = CRM_Contact_BAO_Contact::importableFields('All', NULL); + if (!array_key_exists($name, $tempField)) { + $this->_fields[$name] = new CRM_Contribute_Import_Field($name, $title, $type, $headerPattern, $dataPattern); + } + else { + $this->_fields[$name] = new CRM_Contact_Import_Field($name, $title, $type, $headerPattern, $dataPattern, + CRM_Utils_Array::value('hasLocationType', $tempField[$name]) + ); + } + } + } + + /** + * Store parser values. + * + * @param CRM_Core_Session $store + * + * @param int $mode + */ + public function set($store, $mode = self::MODE_SUMMARY) { + $store->set('fileSize', $this->_fileSize); + $store->set('lineCount', $this->_lineCount); + $store->set('separator', $this->_separator); + $store->set('fields', $this->getSelectValues()); + $store->set('fieldTypes', $this->getSelectTypes()); + + $store->set('headerPatterns', $this->getHeaderPatterns()); + $store->set('dataPatterns', $this->getDataPatterns()); + $store->set('columnCount', $this->_activeFieldCount); + + $store->set('totalRowCount', $this->_totalCount); + $store->set('validRowCount', $this->_validCount); + $store->set('invalidRowCount', $this->_invalidRowCount); + $store->set('invalidSoftCreditRowCount', $this->_invalidSoftCreditRowCount); + $store->set('validSoftCreditRowCount', $this->_validSoftCreditRowCount); + $store->set('invalidPledgePaymentRowCount', $this->_invalidPledgePaymentRowCount); + $store->set('validPledgePaymentRowCount', $this->_validPledgePaymentRowCount); + $store->set('conflictRowCount', $this->_conflictCount); + + switch ($this->_contactType) { + case 'Individual': + $store->set('contactType', CRM_Import_Parser::CONTACT_INDIVIDUAL); + break; + + case 'Household': + $store->set('contactType', CRM_Import_Parser::CONTACT_HOUSEHOLD); + break; + + case 'Organization': + $store->set('contactType', CRM_Import_Parser::CONTACT_ORGANIZATION); + } + + if ($this->_invalidRowCount) { + $store->set('errorsFileName', $this->_errorFileName); + } + if ($this->_conflictCount) { + $store->set('conflictsFileName', $this->_conflictFileName); + } + if (isset($this->_rows) && !empty($this->_rows)) { + $store->set('dataValues', $this->_rows); + } + + if ($this->_invalidPledgePaymentRowCount) { + $store->set('pledgePaymentErrorsFileName', $this->_pledgePaymentErrorsFileName); + } + + if ($this->_invalidSoftCreditRowCount) { + $store->set('softCreditErrorsFileName', $this->_softCreditErrorsFileName); + } + + if ($mode == self::MODE_IMPORT) { + $store->set('duplicateRowCount', $this->_duplicateCount); + if ($this->_duplicateCount) { + $store->set('duplicatesFileName', $this->_duplicateFileName); + } + } + } + + /** + * Export data to a CSV file. + * + * @param string $fileName + * @param array $header + * @param array $data + */ + public static function exportCSV($fileName, $header, $data) { + $output = []; + $fd = fopen($fileName, 'w'); + + foreach ($header as $key => $value) { + $header[$key] = "\"$value\""; + } + $config = CRM_Core_Config::singleton(); + $output[] = implode($config->fieldSeparator, $header); + + foreach ($data as $datum) { + foreach ($datum as $key => $value) { + if (isset($value[0]) && is_array($value)) { + foreach ($value[0] as $k1 => $v1) { + if ($k1 == 'location_type_id') { + continue; + } + $datum[$k1] = $v1; + } + } + else { + $datum[$key] = "\"$value\""; + } + } + $output[] = implode($config->fieldSeparator, $datum); + } + fwrite($fd, implode("\n", $output)); + fclose($fd); + } + + /** + * Determines the file extension based on error code. + * + * @param int $type + * Error code constant. + * + * @return string + */ + public static function errorFileName($type) { + $fileName = NULL; + if (empty($type)) { + return $fileName; + } + + $config = CRM_Core_Config::singleton(); + $fileName = $config->uploadDir . "sqlImport"; + + switch ($type) { + case self::SOFT_CREDIT_ERROR: + $fileName .= '.softCreditErrors'; + break; + + case self::PLEDGE_PAYMENT_ERROR: + $fileName .= '.pledgePaymentErrors'; + break; + + default: + $fileName = parent::errorFileName($type); + break; + } + + return $fileName; + } + + /** + * Determines the file name based on error code. + * + * @param int $type + * Error code constant. + * + * @return string + */ + public static function saveFileName($type) { + $fileName = NULL; + if (empty($type)) { + return $fileName; + } + + switch ($type) { + case self::SOFT_CREDIT_ERROR: + $fileName = 'Import_Soft_Credit_Errors.csv'; + break; + + case self::PLEDGE_PAYMENT_ERROR: + $fileName = 'Import_Pledge_Payment_Errors.csv'; + break; + + default: + $fileName = parent::saveFileName($type); + break; + } + + return $fileName; + } + /** * The initializer code, called before the processing */ @@ -245,10 +893,10 @@ public function import($onDuplicate, &$values) { if ($formatError) { array_unshift($values, $formatError['error_message']); if (CRM_Utils_Array::value('error_data', $formatError) == 'soft_credit') { - return CRM_Contribute_Import_Parser::SOFT_CREDIT_ERROR; + return self::SOFT_CREDIT_ERROR; } if (CRM_Utils_Array::value('error_data', $formatError) == 'pledge_payment') { - return CRM_Contribute_Import_Parser::PLEDGE_PAYMENT_ERROR; + return self::PLEDGE_PAYMENT_ERROR; } return CRM_Import_Parser::ERROR; } @@ -325,7 +973,7 @@ public function import($onDuplicate, &$values) { //return soft valid since we need to show how soft credits were added if (!empty($formatted['soft_credit'])) { - return CRM_Contribute_Import_Parser::SOFT_CREDIT; + return self::SOFT_CREDIT; } // process pledge payment assoc w/ the contribution @@ -379,7 +1027,7 @@ public function import($onDuplicate, &$values) { //return soft valid since we need to show how soft credits were added if (!empty($formatted['soft_credit'])) { - return CRM_Contribute_Import_Parser::SOFT_CREDIT; + return self::SOFT_CREDIT; } // process pledge payment assoc w/ the contribution @@ -446,7 +1094,7 @@ public function import($onDuplicate, &$values) { //return soft valid since we need to show how soft credits were added if (!empty($formatted['soft_credit'])) { - return CRM_Contribute_Import_Parser::SOFT_CREDIT; + return self::SOFT_CREDIT; } // process pledge payment assoc w/ the contribution @@ -477,7 +1125,7 @@ public function processPledgePayments(array $formatted) { $formatted['total_amount'] ); - return CRM_Contribute_Import_Parser::PLEDGE_PAYMENT; + return self::PLEDGE_PAYMENT; } } diff --git a/tests/phpunit/CRM/Contribute/Import/Parser/ContributionTest.php b/tests/phpunit/CRM/Contribute/Import/Parser/ContributionTest.php index 2982049681d7..faac17af3f81 100644 --- a/tests/phpunit/CRM/Contribute/Import/Parser/ContributionTest.php +++ b/tests/phpunit/CRM/Contribute/Import/Parser/ContributionTest.php @@ -70,7 +70,7 @@ public function testImportParserWithSoftCreditsByExternalIdentifier(string $thou ]; $mapperSoftCredit = [NULL, NULL, NULL, 'external_identifier']; $mapperSoftCreditType = [NULL, NULL, NULL, '1']; - $this->runImport($values, CRM_Import_Parser::DUPLICATE_UPDATE, CRM_Contribute_Import_Parser::SOFT_CREDIT, $mapperSoftCredit, NULL, $mapperSoftCreditType); + $this->runImport($values, CRM_Import_Parser::DUPLICATE_UPDATE, CRM_Contribute_Import_Parser_Contribution::SOFT_CREDIT, $mapperSoftCredit, NULL, $mapperSoftCreditType); $contributionsOfMainContact = Contribution::get()->addWhere('contact_id', '=', $contact1Id)->execute(); $this->assertCount(1, $contributionsOfMainContact, 'Contribution not added for primary contact');