From dbed4b66e08f8294e2ad1ff22f0a15364b3d7e6d Mon Sep 17 00:00:00 2001 From: Lee Evans Date: Tue, 11 Jul 2023 02:44:53 +0100 Subject: [PATCH] Add some more config options Allow retry on incomplete responses Little refactor and tidy --- config/ai-translate.php | 4 ++++ src/Console/TranslateStrings.php | 33 ++++++++++++++++++++++++-------- src/Helpers/FileHelper.php | 4 ++-- src/Helpers/OpenAiHelper.php | 21 +++++++------------- 4 files changed, 38 insertions(+), 24 deletions(-) diff --git a/config/ai-translate.php b/config/ai-translate.php index ec83704..68ca58d 100644 --- a/config/ai-translate.php +++ b/config/ai-translate.php @@ -48,6 +48,10 @@ // If your translations are lots of short strings, this number could be much higher. 'max_lines_per_request' => 40, + //If Chat GPT throws an error, how many times should we gracefully retry with exponential backoff + //This is useful for timeout errors. + 'max_retries'=>5, + //Copy any required from here to target_locales above to enable translation of those longuages. 'known_locales' => [ 'af' => 'Afrikaans', diff --git a/src/Console/TranslateStrings.php b/src/Console/TranslateStrings.php index 6a45b94..a75de59 100644 --- a/src/Console/TranslateStrings.php +++ b/src/Console/TranslateStrings.php @@ -4,6 +4,7 @@ use Illuminate\Console\Command; use Illuminate\Support\Arr; +use Illuminate\Support\Facades\Log; use Symfony\Component\VarExporter\VarExporter; use Visualbuilder\AiTranslate\Helpers\FileHelper; use Visualbuilder\AiTranslate\Helpers\OpenAiHelper; @@ -68,6 +69,7 @@ private function init() { $this->sourceLocale = config('ai-translate.source-locale'); $this->overwrite = (bool) $this->option('force'); + $this->maxRetries = config('ai-translate.max_retries'); $this->findSourceFiles(); $this->estimateCostAndSelectModel(); } @@ -83,7 +85,11 @@ private function findSourceFiles() $this->files = array_merge($jsonFiles, $phpFiles); } - + + /** + * Chunk source and setup target + * @return void + */ private function translateFiles() { foreach ($this->files as $sourceFile) { @@ -223,7 +229,7 @@ public function readLanguageFile($file) if(! file_exists($file)) { return []; } - switch (FileHelper::getExtention($file)) { + switch (FileHelper::getExtension($file)) { case('php'): return include($file); case('json'): @@ -307,7 +313,7 @@ private function handleTargetLocale($file, $chunks, $targetLocale, $targetLangua public function createCheckOrEmptyTargetFile($file, $locale) { // Replace source locale with target locale in the path - switch (FileHelper::getExtention($file)) { + switch (FileHelper::getExtension($file)) { case('php'): //php files are in their own locale dir $newFile = str_replace('/'.$this->sourceLocale.'/', '/'.$locale.'/', $file); @@ -329,7 +335,7 @@ public function createCheckOrEmptyTargetFile($file, $locale) mkdir($directory, 0755, true); } - switch (FileHelper::getExtention($file)) { + switch (FileHelper::getExtension($file)) { case('php'): $comment = "/**\n * Auto Translated by visualbuilder/ai-translate on ".date("d/m/Y")."\n */"; file_put_contents($newFile, "handleRetryFailure($retryCount, $targetFile); } - + private function ensureArray($var, $name) { + if (!is_array($var)) { + $this->error("{$name} is not an array."); + Log::error("{$name} is not an array.", ['content' => $var]); + throw new \Exception("{$name} is not an array."); + } + } + public function appendResponse($filename, $translatedChunk) { if(! count($translatedChunk)) { @@ -400,16 +413,19 @@ public function appendResponse($filename, $translatedChunk) // Undot the translated chunk $undottedTranslatedChunk = Arr::undot($translatedChunk); - + + $this->ensureArray($existingContent,'existingContent: '.$filename); + $this->ensureArray($undottedTranslatedChunk,'undottedTranslatedChunk: '); + // Merge new translations with existing content $newContent = array_merge($existingContent, $undottedTranslatedChunk); $comment = "/**\n * Auto Translated by visualbuilder/ai-translate on ".date("d/m/Y")."\n */"; - switch (FileHelper::getExtention($filename)) { + switch (FileHelper::getExtension($filename)) { case('php'): $output = "newLine(); $this->error('An error occurred while processing the file: '.$targetFile); $this->error('Error message: '.$exception->getMessage()); $this->info("Retrying $retryCount / $this->maxRetries Times in ".pow(2, $retryCount).' seconds'); diff --git a/src/Helpers/FileHelper.php b/src/Helpers/FileHelper.php index d8d3b00..62e5291 100644 --- a/src/Helpers/FileHelper.php +++ b/src/Helpers/FileHelper.php @@ -90,7 +90,7 @@ private static function getFiles($dir, $basepath, $fileType = '*') return $files; } - public static function getExtention($filename) + public static function getExtension($filename) { return pathinfo($filename, PATHINFO_EXTENSION); } @@ -98,7 +98,7 @@ public static function getExtention($filename) public static function countItemsAndStringLengths($filename) { - switch (self::getExtention($filename)) { + switch (self::getExtension($filename)) { case('php'): $translations = include($filename); diff --git a/src/Helpers/OpenAiHelper.php b/src/Helpers/OpenAiHelper.php index 845f3ee..62d1bf6 100644 --- a/src/Helpers/OpenAiHelper.php +++ b/src/Helpers/OpenAiHelper.php @@ -50,7 +50,6 @@ public static function translateChunk($command, $chunk, $sourceLocale, $targetLa $lines = array_map(function ($line) use (&$placeholders) { return preg_replace_callback('/:(\w+)/', function ($matches) use (&$placeholders) { $placeholders[] = $matches[0]; // Store the placeholders - return '***'; // Replace them with *** }, $line); }, $lines); @@ -58,7 +57,7 @@ public static function translateChunk($command, $chunk, $sourceLocale, $targetLa $placeholdersUsed = count($placeholders); // Add line numbers to each string - $lines = array_map(function ($k, $v) { return ($k + 1) . ". {$v}"; }, array_keys($lines), $lines); + $lines = self::addLineNumbersToArrayofStrings($lines); $linesString = implode("\n", $lines); @@ -67,9 +66,9 @@ public static function translateChunk($command, $chunk, $sourceLocale, $targetLa // Get the total tokens from all the strings in the chunk $totalTokens = OpenAiHelper::estimateTokensFromString($prompt.$linesString); - $command->comment("Request tokens: $totalTokens"); + $command->comment("Tokens: $totalTokens"); $command->comment("Request: $prompt"); - $command->warn('Source Lines'); + $command->warn('Source Lines: '); self::displayArrayInNumberedLines($originalLines, $command); $response = OpenAI::chat()->create([ @@ -98,7 +97,7 @@ public static function translateChunk($command, $chunk, $sourceLocale, $targetLa $translatedStrings = explode("\n", trim($translatedContent)); // Remove the line numbers from the translated strings - $translatedStrings = array_map(function ($s) { return preg_replace('/^\d+\.\s/', '', $s); }, $translatedStrings); + $translatedStrings = array_map(function ($s) { return preg_replace('/^\d+\.\s/', '', $s); },$translatedStrings); @@ -109,20 +108,14 @@ public static function translateChunk($command, $chunk, $sourceLocale, $targetLa }, $line); }, $translatedStrings); - $translatedChunk = []; // Combine the original keys with the translated strings if (count(array_keys($chunk)) === count($translatedStrings)) { $translatedChunk = array_combine(array_keys($chunk), $translatedStrings); } else { // Handle situation when the translation result doesn't match with the original number of keys - $command->error("Mismatch in translation keys and translated strings for chunk. Keys: " . count(array_keys($chunk)) . " Translated Strings: " . count($translatedStrings)); - $command->newLine(); - - // assign untranslated lines to their original value. - // Actually this is not useful. Better not to exist - // foreach (array_keys($chunk) as $i => $key) { - // $translatedChunk[$key] = $translatedStrings[$i] ?? $chunk[$key]; - // } + // Sometimes some data is lost. + // Exceptions will trigger a retry upto max_retries. + throw new \Exception("Mismatch in source keys and translation. Keys: " . count(array_keys($chunk)) . " Translated Strings: " . count($translatedStrings)); } if($placeholdersUsed > 0) {