Skip to content

Fix encoding issue when source is encoded in utf8 (issue #284) #456

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
cholletk wants to merge 4 commits into from
Closed

Fix encoding issue when source is encoded in utf8 (issue #284) #456

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
fcffbe4
Fix encoding issue when source is encoded in utf8 (issue #284)
cholletk Dec 21, 2020
c70eeb6
Modifying changelog for the fix to issue #284
cholletk Dec 21, 2020
ce96cee
Remove line return (#284)
cholletk Dec 21, 2020
84bfab7
Compliance with cs fixer (#284)
cholletk Dec 21, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,8 @@
[#437](https://github.com/nextcloud/cookbook/pull/437) @christianlupus
- Corrected bugs in CI system
[#447](https://github.com/nextcloud/cookbook/pull/447) @christianlupus
- Correct encoding error when importing UTF8 Encoded recipe
[#284](https://github.com/nextcloud/cookbook/issues/284) @cholletk

### Removed
- Travis build system
Expand Down
63 changes: 33 additions & 30 deletions lib/Service/RecipeService.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ public function getRecipeById(int $id) {

return $this->parseRecipeFile($file);
}

/**
* Get a recipe's modification time by its folder id.
*
Expand Down Expand Up @@ -114,7 +114,7 @@ public function checkRecipe(array $json): array {
if (!$json) {
throw new Exception('Recipe array was null');
}

if (empty($json['name'])) {
throw new Exception('Field "name" is required');
}
Expand Down Expand Up @@ -146,7 +146,7 @@ public function checkRecipe(array $json): array {
if (empty($img)) {
continue;
}

$image_matches = [];

preg_match_all('!\d+!', $img, $image_matches);
Expand Down Expand Up @@ -195,7 +195,7 @@ public function checkRecipe(array $json): array {
$json['image'] .= '?' . $image_url['query'];
}
}


// Make sure that "recipeCategory" is a string
if (isset($json['recipeCategory'])) {
Expand All @@ -210,7 +210,7 @@ public function checkRecipe(array $json): array {

$json['recipeCategory'] = $this->cleanUpString($json['recipeCategory'], false, true);


// Make sure that "recipeYield" is an integer which is at least 1
if (isset($json['recipeYield']) && $json['recipeYield']) {
$regex_matches = [];
Expand Down Expand Up @@ -388,7 +388,7 @@ public function checkRecipe(array $json): array {
if (isset($duration_matches[1][0]) && !empty($duration_matches[1][0])) {
$duration_hours = intval($duration_matches[1][0]);
}

if (isset($duration_matches[2][0]) && !empty($duration_matches[2][0])) {
$duration_minutes = intval($duration_matches[2][0]);
}
Expand All @@ -409,7 +409,7 @@ public function checkRecipe(array $json): array {
} else {
$json['nutrition'] = [];
}

return $json;
}

Expand All @@ -426,6 +426,9 @@ private function parseRecipeHtml($url, $html) {
// Make sure we don't have any encoded entities in the HTML string
$html = html_entity_decode($html);

// Convert utf8 entity
$html = utf8_decode($html);

// Start document parser
$document = new \DOMDocument();

Expand All @@ -441,7 +444,7 @@ private function parseRecipeHtml($url, $html) {
} finally {
libxml_use_internal_errors($libxml_previous_state);
}

$xpath = new \DOMXPath($document);

$json_ld_elements = $xpath->query("//*[@type='application/ld+json']");
Expand Down Expand Up @@ -489,7 +492,7 @@ private function parseRecipeHtml($url, $html) {

// Parse HTML if JSON couldn't be found
$json = [];

$recipes = $xpath->query("//*[@itemtype='http://schema.org/Recipe']");

if (!isset($recipes[0])) {
Expand All @@ -514,7 +517,7 @@ private function parseRecipeHtml($url, $html) {
case 'images':
case 'thumbnail':
$prop = 'image';

if (!isset($json[$prop]) || !is_array($json[$prop])) {
$json[$prop] = [];
}
Expand All @@ -533,7 +536,7 @@ private function parseRecipeHtml($url, $html) {
case 'recipeIngredient':
case 'ingredients':
$prop = 'recipeIngredient';

if (!isset($json[$prop]) || !is_array($json[$prop])) {
$json[$prop] = [];
}
Expand All @@ -546,15 +549,15 @@ private function parseRecipeHtml($url, $html) {
} else {
array_push($json[$prop], $prop_element->nodeValue);
}

break;

case 'recipeInstructions':
case 'instructions':
case 'steps':
case 'guide':
$prop = 'recipeInstructions';

if (!isset($json[$prop]) || !is_array($json[$prop])) {
$json[$prop] = [];
}
Expand Down Expand Up @@ -590,7 +593,7 @@ private function parseRecipeHtml($url, $html) {
// Make one final desparate attempt at getting the instructions
if (!isset($json['recipeInstructions']) || !$json['recipeInstructions'] || sizeof($json['recipeInstructions']) < 1) {
$json['recipeInstructions'] = [];

$step_elements = $recipes[0]->getElementsByTagName('p');

foreach ($step_elements as $step_element) {
Expand All @@ -601,23 +604,23 @@ private function parseRecipeHtml($url, $html) {
array_push($json['recipeInstructions'], $step_element->nodeValue);
}
}

return $this->checkRecipe($json);
}

private function display_libxml_errors($url, $errors) {
$error_counter = [];
$by_error_code = [];

foreach ($errors as $error) {
$count = array_key_exists($error->code, $error_counter) ? $error_counter[$error->code] : 0;
$error_counter[$error->code] = $count + 1;
$by_error_code[$error->code] = $error;
}

foreach ($error_counter as $code => $count) {
$error = $by_error_code[$code];

switch ($error->level) {
case LIBXML_ERR_WARNING:
$error_message = "libxml: Warning $error->code ";
Expand All @@ -634,7 +637,7 @@ private function display_libxml_errors($url, $errors) {

$error_message .= "occurred " . $count . " times while parsing " . $url . ". Last time in line $error->line" .
" and column $error->column: " . $error->message;

$this->logger->warning($error_message);
}
}
Expand Down Expand Up @@ -688,7 +691,7 @@ public function addRecipe($json) {
if ($user_folder->nodeExists($json['name'])) {
throw new Exception('Another recipe with that name already exists');
}

$recipe_folder->move($new_path);
}

Expand Down Expand Up @@ -864,29 +867,29 @@ public function getRecipeFiles() {
public function updateSearchIndex() {
$this->migrateFolderStructure();
}

private function migrateFolderStructure() {
// Remove old cache folder if needed
$legacy_cache_path = '/cookbook/cache';

if ($this->root->nodeExists($legacy_cache_path)) {
$this->root->get($legacy_cache_path)->delete();
}

// Restructure files if needed
$user_folder = $this->getFolderForUser();

foreach ($user_folder->getDirectoryListing() as $node) {
// Move JSON files from the user directory into its own folder
if ($this->isRecipeFile($node)) {
$recipe_name = str_replace('.json', '', $node->getName());

$node->move($node->getPath() . '_tmp');

$recipe_folder = $user_folder->newFolder($recipe_name);

$node->move($recipe_folder->getPath() . '/recipe.json');

// Rename folders with .json extensions (this was likely caused by a migration bug)
} elseif ($node instanceof Folder && strpos($node->getName(), '.json')) {
$node->move(str_replace('.json', '', $node->getPath()));
Expand All @@ -902,7 +905,7 @@ private function migrateFolderStructure() {
public function getAllKeywordsInSearchIndex() {
return $this->db->findAllKeywords($this->user_id);
}

/**
* Gets all categories from the index
*
Expand Down Expand Up @@ -1198,7 +1201,7 @@ private function cleanUpString($str, $preserve_newlines = false, $remove_slashes
} else {
$str = str_replace(["\t", "\\"], '', $str);
}

$str = html_entity_decode($str);

return $str;
Expand Down