Server : Apache System : Linux server2.corals.io 4.18.0-348.2.1.el8_5.x86_64 #1 SMP Mon Nov 15 09:17:08 EST 2021 x86_64 User : corals ( 1002) PHP Version : 7.4.33 Disable Function : exec,passthru,shell_exec,system Directory : /home/corals/old/vendor/magento/module-import-export/Model/Import/Entity/ |
<?php
/**
* Copyright © Magento, Inc. All rights reserved.
* See COPYING.txt for license details.
*/
namespace Magento\ImportExport\Model\Import\Entity;
use Magento\Framework\App\ObjectManager;
use Magento\Framework\App\ResourceConnection;
use Magento\Framework\Exception\LocalizedException;
use Magento\Framework\Serialize\Serializer\Json;
use Magento\ImportExport\Model\Import as ImportExport;
use Magento\ImportExport\Model\Import\AbstractSource;
use Magento\ImportExport\Model\Import\EntityInterface;
use Magento\ImportExport\Model\Import\ErrorProcessing\ProcessingError;
use Magento\ImportExport\Model\Import\ErrorProcessing\ProcessingErrorAggregatorInterface;
use Magento\ImportExport\Model\ResourceModel\Import\Data as DataSourceModel;
/**
* Import entity abstract model
*
* phpcs:disable Magento2.Classes.AbstractApi
* @api
*
* @SuppressWarnings(PHPMD.TooManyFields)
* @SuppressWarnings(PHPMD.CouplingBetweenObjects)
* @since 100.0.2
*/
abstract class AbstractEntity implements EntityInterface
{
/**
* Database constants
*/
public const DB_MAX_PACKET_COEFFICIENT = 900000;
public const DB_MAX_PACKET_DATA = 1048576;
public const DB_MAX_VARCHAR_LENGTH = 256;
public const DB_MAX_TEXT_LENGTH = 65536;
public const ERROR_CODE_SYSTEM_EXCEPTION = 'systemException';
public const ERROR_CODE_COLUMN_NOT_FOUND = 'columnNotFound';
public const ERROR_CODE_COLUMN_EMPTY_HEADER = 'columnEmptyHeader';
public const ERROR_CODE_COLUMN_NAME_INVALID = 'columnNameInvalid';
public const ERROR_CODE_ATTRIBUTE_NOT_VALID = 'attributeNotInvalid';
public const ERROR_CODE_DUPLICATE_UNIQUE_ATTRIBUTE = 'duplicateUniqueAttribute';
public const ERROR_CODE_ILLEGAL_CHARACTERS = 'illegalCharacters';
public const ERROR_CODE_INVALID_ATTRIBUTE = 'invalidAttributeName';
public const ERROR_CODE_WRONG_QUOTES = 'wrongQuotes';
public const ERROR_CODE_COLUMNS_NUMBER = 'wrongColumnsNumber';
public const ERROR_CODE_CATEGORY_NOT_VALID = 'categoryNotValid';
/**
* @var array
*/
protected $errorMessageTemplates = [
self::ERROR_CODE_SYSTEM_EXCEPTION => 'General system exception happened',
self::ERROR_CODE_COLUMN_NOT_FOUND => 'We can\'t find required columns: %s.',
self::ERROR_CODE_COLUMN_EMPTY_HEADER => 'Columns number: "%s" have empty headers',
self::ERROR_CODE_COLUMN_NAME_INVALID => 'Column names: "%s" are invalid',
self::ERROR_CODE_ATTRIBUTE_NOT_VALID => "Please correct the value for '%s'.",
self::ERROR_CODE_DUPLICATE_UNIQUE_ATTRIBUTE => "Duplicate Unique Attribute for '%s'",
self::ERROR_CODE_ILLEGAL_CHARACTERS => "Illegal character used for attribute %s",
self::ERROR_CODE_INVALID_ATTRIBUTE => 'Header contains invalid attribute(s): "%s"',
self::ERROR_CODE_WRONG_QUOTES => "Curly quotes used instead of straight quotes",
self::ERROR_CODE_COLUMNS_NUMBER => "Number of columns does not correspond to the number of rows in the header",
];
/**
* Validation failure message template definitions
*
* @var array
*/
protected $_messageTemplates = [];
/**
* DB connection.
*
* @var \Magento\Framework\DB\Adapter\AdapterInterface
*/
protected $_connection;
/**
* Has data process validation done?8
*
* @var bool
*/
protected $_dataValidated = false;
/**
* @var array
*/
protected $validColumnNames = [];
/**
* If we should check column names
*
* @var bool
*/
protected $needColumnCheck = false;
/**
* DB data source model.
*
* @var DataSourceModel
*/
protected $_dataSourceModel;
/**
* @var int
*/
protected $_entityTypeId;
/**
* Error codes with arrays of corresponding row numbers.
*
* @var array
*/
protected $_errors = [];
/**
* Flag to disable import.
*
* @var bool
*/
protected $_importAllowed = true;
/**
* Attributes with index (not label) value.
*
* @var array
*/
protected $_indexValueAttributes = [];
/**
* Entity model parameters.
*
* @var array
*/
protected $_parameters = [];
/**
* Column names that holds values with particular meaning.
*
* @var string[]
*/
protected $_specialAttributes = [];
/**
* Permanent entity columns.
*
* @var string[]
*/
protected $_permanentAttributes = [];
/**
* Number of entities processed by validation.
*
* @var int
*/
protected $_processedEntitiesCount = 0;
/**
* Number of rows processed by validation.
*
* @var int
*/
protected $_processedRowsCount = 0;
/**
* Array of numbers of validated rows as keys and boolean TRUE as values.
*
* @var array
*/
protected $_validatedRows = [];
/**
* Source model.
*
* @var AbstractSource
*/
protected $_source;
/**
* Array of unique attributes
*
* @var array
*/
protected $_uniqueAttributes = [];
/**
* @var \Magento\ImportExport\Helper\Data
*/
protected $_importExportData;
/**
* @var \Magento\Framework\Json\Helper\Data
*/
protected $jsonHelper;
/**
* Magento string lib
*
* @var \Magento\Framework\Stdlib\StringUtils
*/
protected $string;
/**
* @var \Magento\ImportExport\Model\ResourceModel\Helper
*/
protected $_resourceHelper;
/**
* Count if created items
*
* @var int
*/
protected $countItemsCreated = 0;
/**
* Count if updated items
*
* @var int
*/
protected $countItemsUpdated = 0;
/**
* Count if deleted items
*
* @var int
*/
protected $countItemsDeleted = 0;
/**
* Need to log in import history
*
* @var bool
*/
protected $logInHistory = false;
/**
* @var ProcessingErrorAggregatorInterface
*/
protected $errorAggregator;
/**
* Product metadata pool
*
* @var \Magento\Framework\EntityManager\MetadataPool
* @since 100.1.0
*/
protected $metadataPool;
/**
* Json Serializer Instance
*
* @var Json
*/
private $serializer;
/**
* Ids of saved data in DB
*
* @var array
*/
private array $ids = [];
/**
* @param \Magento\Framework\Json\Helper\Data $jsonHelper
* @param \Magento\ImportExport\Helper\Data $importExportData
* @param \Magento\ImportExport\Model\ResourceModel\Import\Data $importData
* @param \Magento\Eav\Model\Config $config
* @param ResourceConnection $resource
* @param \Magento\ImportExport\Model\ResourceModel\Helper $resourceHelper
* @param \Magento\Framework\Stdlib\StringUtils $string
* @param ProcessingErrorAggregatorInterface $errorAggregator
* @throws \Magento\Framework\Exception\LocalizedException
*/
public function __construct(
\Magento\Framework\Json\Helper\Data $jsonHelper,
\Magento\ImportExport\Helper\Data $importExportData,
\Magento\ImportExport\Model\ResourceModel\Import\Data $importData,
\Magento\Eav\Model\Config $config,
ResourceConnection $resource,
\Magento\ImportExport\Model\ResourceModel\Helper $resourceHelper,
\Magento\Framework\Stdlib\StringUtils $string,
ProcessingErrorAggregatorInterface $errorAggregator
) {
$this->jsonHelper = $jsonHelper;
$this->_importExportData = $importExportData;
$this->_resourceHelper = $resourceHelper;
$this->string = $string;
$this->errorAggregator = $errorAggregator;
foreach ($this->errorMessageTemplates as $errorCode => $message) {
$this->getErrorAggregator()->addErrorMessageTemplate($errorCode, $message);
}
$entityType = $config->getEntityType($this->getEntityTypeCode());
$this->_entityTypeId = $entityType->getEntityTypeId();
$this->_dataSourceModel = $importData;
$this->_connection = $resource->getConnection();
}
/**
* Inner source object getter.
*
* @return AbstractSource
* @throws \Magento\Framework\Exception\LocalizedException
*/
protected function _getSource()
{
if (!$this->_source) {
throw new LocalizedException(__('Please specify a source.'));
}
return $this->_source;
}
/**
* Import data rows.
*
* @abstract
* @return boolean
*/
abstract protected function _importData();
/**
* Returns boolean TRUE if row scope is default (fundamental) scope.
*
* @param array $rowData
* @return bool
* @SuppressWarnings(PHPMD.UnusedFormalParameter)
*/
protected function _isRowScopeDefault(array $rowData)
{
return true;
}
/**
* Change row data before saving in DB table.
*
* @param array $rowData
* @return array
*/
protected function _prepareRowForDb(array $rowData)
{
/**
* Convert all empty strings to null values, as
* a) we don't use empty string in DB
* b) empty strings instead of numeric values will product errors in Sql Server
*/
foreach ($rowData as $key => $val) {
if ($val === '') {
$rowData[$key] = null;
}
}
return $rowData;
}
/**
* Add errors to error aggregator
*
* @param string $code
* @param array|mixed $errors
* @return void
*/
protected function addErrors($code, $errors)
{
if ($errors) {
$this->getErrorAggregator()->addError(
$code,
ProcessingError::ERROR_LEVEL_CRITICAL,
null,
implode('", "', $errors)
);
}
}
/**
* Validate data rows and save bunches to DB.
*
* @return $this
* @SuppressWarnings(PHPMD.CyclomaticComplexity)
*/
protected function _saveValidatedBunches()
{
$source = $this->_getSource();
$currentDataSize = 0;
$bunchRows = [];
$startNewBunch = false;
$nextRowBackup = [];
$maxDataSize = $this->_resourceHelper->getMaxDataSize();
$bunchSize = $this->_importExportData->getBunchSize();
$skuSet = [];
$source->rewind();
$this->_dataSourceModel->cleanProcessedBunches();
while ($source->valid() || $bunchRows) {
if ($startNewBunch || !$source->valid()) {
$this->ids[] =
$this->_dataSourceModel->saveBunch($this->getEntityTypeCode(), $this->getBehavior(), $bunchRows);
$bunchRows = $nextRowBackup;
$currentDataSize = strlen($this->getSerializer()->serialize($bunchRows));
$startNewBunch = false;
$nextRowBackup = [];
}
if ($source->valid()) {
try {
$rowData = $source->current();
if (array_key_exists('sku', $rowData)) {
$skuSet[$rowData['sku']] = true;
}
} catch (\InvalidArgumentException $e) {
$this->addRowError($e->getMessage(), $this->_processedRowsCount);
$this->_processedRowsCount++;
$source->next();
continue;
}
$this->_processedRowsCount++;
if ($this->validateRow($rowData, $source->key())) {
// add row to bunch for save
$rowData = $this->_prepareRowForDb($rowData);
$rowSize = strlen($this->jsonHelper->jsonEncode($rowData) ?? '');
$isBunchSizeExceeded = $bunchSize > 0 && count($bunchRows) >= $bunchSize;
if ($currentDataSize + $rowSize >= $maxDataSize || $isBunchSizeExceeded) {
$startNewBunch = true;
$nextRowBackup = [$source->key() => $rowData];
} else {
$bunchRows[$source->key()] = $rowData;
$currentDataSize += $rowSize;
}
}
$source->next();
}
}
$this->_processedEntitiesCount = (count($skuSet)) ?: $this->_processedRowsCount;
return $this;
}
/**
* Get Serializer instance
*
* Workaround. Only way to implement dependency and not to break inherited child classes
*
* @return Json
*/
private function getSerializer()
{
if (null === $this->serializer) {
$this->serializer = ObjectManager::getInstance()->get(Json::class);
}
return $this->serializer;
}
/**
* Add error with corresponding current data source row number.
*
* @param string $errorCode Error code or simply column name
* @param int $errorRowNum Row number.
* @param string $colName OPTIONAL Column name.
* @param string $errorMessage OPTIONAL Column name.
* @param string $errorLevel
* @param string $errorDescription
* @return $this
*/
public function addRowError(
$errorCode,
$errorRowNum,
$colName = null,
$errorMessage = null,
$errorLevel = ProcessingError::ERROR_LEVEL_CRITICAL,
$errorDescription = null
) {
$errorCode = (string)$errorCode;
$this->getErrorAggregator()->addError(
$errorCode,
$errorLevel,
$errorRowNum,
$colName,
$errorMessage,
$errorDescription
);
return $this;
}
/**
* Add message template for specific error code from outside.
*
* @param string $errorCode Error code
* @param string $message Message template
* @return $this
*/
public function addMessageTemplate($errorCode, $message)
{
$this->getErrorAggregator()->addErrorMessageTemplate($errorCode, $message);
return $this;
}
/**
* Returns attributes all values in label-value or value-value pairs form. Labels are lower-cased.
*
* @param \Magento\Eav\Model\Entity\Attribute\AbstractAttribute $attribute
* @param array $indexValAttrs OPTIONAL Additional attributes' codes with index values.
* @return array
*/
public function getAttributeOptions(
\Magento\Eav\Model\Entity\Attribute\AbstractAttribute $attribute,
$indexValAttrs = []
) {
$options = [];
if ($attribute->usesSource()) {
// merge global entity index value attributes
$indexValAttrs = array_merge($indexValAttrs, $this->_indexValueAttributes);
// should attribute have index (option value) instead of a label?
$index = in_array($attribute->getAttributeCode(), $indexValAttrs) ? 'value' : 'label';
// only default (admin) store values used
$attribute->setStoreId(\Magento\Store\Model\Store::DEFAULT_STORE_ID);
try {
foreach ($attribute->getSource()->getAllOptions(false) as $option) {
$value = is_array($option['value']) ? $option['value'] : [$option];
foreach ($value as $innerOption) {
if (strlen($innerOption['value'] ?? '')) {
// skip ' -- Please Select -- ' option
$options[strtolower($innerOption[$index] ?? '')] = $innerOption['value'];
}
}
}
// phpcs:disable Magento2.CodeAnalysis.EmptyBlock.DetectedCatch
} catch (\Exception $e) {
// ignore exceptions connected with source models
}
}
return $options;
}
/**
* Import behavior getter.
*
* @return string
*/
public function getBehavior()
{
if (!isset(
$this->_parameters['behavior']
) ||
$this->_parameters['behavior'] != ImportExport::BEHAVIOR_APPEND &&
$this->_parameters['behavior'] != ImportExport::BEHAVIOR_ADD_UPDATE &&
$this->_parameters['behavior'] != ImportExport::BEHAVIOR_REPLACE &&
$this->_parameters['behavior'] != ImportExport::BEHAVIOR_CUSTOM &&
$this->_parameters['behavior'] != ImportExport::BEHAVIOR_DELETE
) {
return ImportExport::getDefaultBehavior();
}
return $this->_parameters['behavior'];
}
/**
* EAV entity type code getter.
*
* @abstract
* @return string
*/
abstract public function getEntityTypeCode();
/**
* Entity type ID getter.
*
* @return int
*/
public function getEntityTypeId()
{
return $this->_entityTypeId;
}
/**
* Returns number of checked entities.
*
* @return int
*/
public function getProcessedEntitiesCount()
{
return $this->_processedEntitiesCount;
}
/**
* Returns number of checked rows.
*
* @return int
*/
public function getProcessedRowsCount()
{
return $this->_processedRowsCount;
}
/**
* Source object getter.
*
* @return AbstractSource
* @throws \Magento\Framework\Exception\LocalizedException
*/
public function getSource()
{
if (!$this->_source) {
throw new LocalizedException(__('The source is not set.'));
}
return $this->_source;
}
/**
* Import process start.
*
* @return bool Result of operation.
*/
public function importData()
{
return $this->_importData();
}
/**
* Is attribute contains particular data (not plain entity attribute).
*
* @param string $attrCode
* @return bool
*/
public function isAttributeParticular($attrCode)
{
return in_array($attrCode, $this->_specialAttributes);
}
/**
* Check one attribute. Can be overridden in child.
*
* @param string $attrCode Attribute code
* @param array $attrParams Attribute params
* @param array $rowData Row data
* @param int $rowNum
* @return boolean
* @SuppressWarnings(PHPMD.CyclomaticComplexity)
*/
public function isAttributeValid($attrCode, array $attrParams, array $rowData, $rowNum)
{
switch ($attrParams['type']) {
case 'varchar':
$val = $this->string->cleanString($rowData[$attrCode]);
$valid = $this->string->strlen($val) < self::DB_MAX_VARCHAR_LENGTH;
break;
case 'decimal':
$val = trim($rowData[$attrCode] ?? '');
$valid = (double)$val == $val;
break;
case 'select':
case 'multiselect':
$valid = isset($attrParams['options'][strtolower($rowData[$attrCode] ?? '')]);
break;
case 'int':
$val = trim($rowData[$attrCode] ?? '');
$valid = (int)$val == $val;
break;
case 'datetime':
$val = trim($rowData[$attrCode] ?? '');
$valid = strtotime($val) !== false;
break;
case 'text':
$val = $this->string->cleanString($rowData[$attrCode]);
$valid = $this->string->strlen($val) < self::DB_MAX_TEXT_LENGTH;
break;
default:
$valid = true;
break;
}
if (!$valid) {
$this->addRowError(self::ERROR_CODE_ATTRIBUTE_NOT_VALID, $rowNum, $attrCode);
} elseif (!empty($attrParams['is_unique'])) {
if (isset($this->_uniqueAttributes[$attrCode][$rowData[$attrCode]])) {
$this->addRowError(self::ERROR_CODE_DUPLICATE_UNIQUE_ATTRIBUTE, $rowNum, $attrCode);
return false;
}
$this->_uniqueAttributes[$attrCode][$rowData[$attrCode]] = true;
}
return (bool)$valid;
}
/**
* Import possibility getter.
*
* @return bool
*/
public function isImportAllowed()
{
return $this->_importAllowed;
}
/**
* Returns TRUE if row is valid and not in skipped rows array.
*
* @param array $rowData
* @param int $rowNum
* @return bool
*/
public function isRowAllowedToImport(array $rowData, $rowNum)
{
$this->validateRow($rowData, $rowNum);
return !$this->getErrorAggregator()->isRowInvalid($rowNum);
}
/**
* Retrieve message template
*
* @param string $errorCode
* @return null|string
*/
public function retrieveMessageTemplate($errorCode)
{
if (isset($this->_messageTemplates[$errorCode])) {
return $this->_messageTemplates[$errorCode];
}
return null;
}
/**
* Is import need to log in history.
*
* @return bool
*/
public function isNeedToLogInHistory()
{
return $this->logInHistory;
}
/**
* Validate data row.
*
* @param array $rowData
* @param int $rowNum
* @return boolean
*/
abstract public function validateRow(array $rowData, $rowNum);
/**
* Set data from outside to change behavior. I.e. for setting some default parameters etc.
*
* @param array $params
* @return $this
*/
public function setParameters(array $params)
{
$this->_parameters = $params;
return $this;
}
/**
* Get data from outside to change behavior. I.e. for setting some default parameters etc.
*
* @return array $params
*/
public function getParameters()
{
return $this->_parameters;
}
/**
* Source model setter.
*
* @param AbstractSource $source
* @return $this
*/
public function setSource(AbstractSource $source)
{
$this->_source = $source;
$this->_dataValidated = false;
return $this;
}
/**
* Validate data.
*
* @return ProcessingErrorAggregatorInterface
* @throws \Magento\Framework\Exception\LocalizedException
* @SuppressWarnings(PHPMD.CyclomaticComplexity)
*/
public function validateData()
{
if (!$this->_dataValidated) {
$this->getErrorAggregator()->clear();
// do all permanent columns exist?
$absentColumns = array_diff($this->_permanentAttributes, $this->getSource()->getColNames());
$this->addErrors(self::ERROR_CODE_COLUMN_NOT_FOUND, $absentColumns);
if (ImportExport::BEHAVIOR_DELETE != $this->getBehavior()) {
// check attribute columns names validity
$columnNumber = 0;
$emptyHeaderColumns = [];
$invalidColumns = [];
$invalidAttributes = [];
foreach ($this->getSource()->getColNames() as $columnName) {
$columnNumber++;
if (!$this->isAttributeParticular($columnName)) {
if (trim($columnName ?? '') == '') {
$emptyHeaderColumns[] = $columnNumber;
} elseif (!$columnName || !preg_match('/^[a-z][a-z0-9_]*$/', $columnName)) {
$invalidColumns[] = $columnName;
} elseif ($this->needColumnCheck && !in_array($columnName, $this->getValidColumnNames())) {
$invalidAttributes[] = $columnName;
}
}
}
$this->addErrors(self::ERROR_CODE_INVALID_ATTRIBUTE, $invalidAttributes);
$this->addErrors(self::ERROR_CODE_COLUMN_EMPTY_HEADER, $emptyHeaderColumns);
$this->addErrors(self::ERROR_CODE_COLUMN_NAME_INVALID, $invalidColumns);
}
if (!$this->getErrorAggregator()->getErrorsCount()) {
$this->_saveValidatedBunches();
$this->_dataValidated = true;
}
}
return $this->getErrorAggregator();
}
/**
* Get error aggregator object
*
* @return ProcessingErrorAggregatorInterface
*/
public function getErrorAggregator()
{
return $this->errorAggregator;
}
/**
* Get count of created items
*
* @return int
*/
public function getCreatedItemsCount()
{
return $this->countItemsCreated;
}
/**
* Get count of updated items
*
* @return int
*/
public function getUpdatedItemsCount()
{
return $this->countItemsUpdated;
}
/**
* Get count of deleted items
*
* @return int
*/
public function getDeletedItemsCount()
{
return $this->countItemsDeleted;
}
/**
* Retrieve valid column names
*
* @return array
*/
public function getValidColumnNames()
{
return $this->validColumnNames;
}
/**
* Get product metadata pool
*
* @return \Magento\Framework\EntityManager\MetadataPool
* @since 100.1.0
*/
protected function getMetadataPool()
{
if (!$this->metadataPool) {
$this->metadataPool = ObjectManager::getInstance()
->get(\Magento\Framework\EntityManager\MetadataPool::class);
}
return $this->metadataPool;
}
/**
* Retrieve Ids of Validated Rows
*
* @return array
*/
public function getIds() : array
{
return $this->ids;
}
/**
* Set Ids of Validated Rows
*
* @param array $ids
* @return void
*/
public function setIds(array $ids)
{
$this->ids = $ids;
}
/**
* Gets the currently used DataSourceModel
*
* @return DataSourceModel
*/
public function getDataSourceModel() : DataSourceModel
{
return $this->_dataSourceModel;
}
}