Skip to content

Commit

Permalink
Add some more config options
Browse files Browse the repository at this point in the history 
Allow retry on incomplete responses
Little refactor and tidy
  • Loading branch information
@cannycookie
cannycookie committed Jul 11, 2023
1 parent 2ab076c commit dbed4b6
Show file tree
Hide file tree
Showing 4 changed files with 38 additions and 24 deletions.
4 changes: 4 additions & 0 deletions config/ai-translate.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,10 @@
// If your translations are lots of short strings, this number could be much higher.
'max_lines_per_request' => 40,

//If Chat GPT throws an error, how many times should we gracefully retry with exponential backoff
//This is useful for timeout errors.
'max_retries'=>5,

//Copy any required from here to target_locales above to enable translation of those longuages.
'known_locales' => [
'af' => 'Afrikaans',
Expand Down
33 changes: 25 additions & 8 deletions src/Console/TranslateStrings.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@

use Illuminate\Console\Command;
use Illuminate\Support\Arr;
use Illuminate\Support\Facades\Log;
use Symfony\Component\VarExporter\VarExporter;
use Visualbuilder\AiTranslate\Helpers\FileHelper;
use Visualbuilder\AiTranslate\Helpers\OpenAiHelper;
Expand Down Expand Up @@ -68,6 +69,7 @@ private function init()
{
$this->sourceLocale = config('ai-translate.source-locale');
$this->overwrite = (bool) $this->option('force');
$this->maxRetries = config('ai-translate.max_retries');
$this->findSourceFiles();
$this->estimateCostAndSelectModel();
}
Expand All @@ -83,7 +85,11 @@ private function findSourceFiles()

$this->files = array_merge($jsonFiles, $phpFiles);
}


/**
* Chunk source and setup target
* @return void
*/
private function translateFiles()
{
foreach ($this->files as $sourceFile) {
Expand Down Expand Up @@ -223,7 +229,7 @@ public function readLanguageFile($file)
if(! file_exists($file)) {
return [];
}
switch (FileHelper::getExtention($file)) {
switch (FileHelper::getExtension($file)) {
case('php'):
return include($file);
case('json'):
Expand Down Expand Up @@ -307,7 +313,7 @@ private function handleTargetLocale($file, $chunks, $targetLocale, $targetLangua
public function createCheckOrEmptyTargetFile($file, $locale)
{
// Replace source locale with target locale in the path
switch (FileHelper::getExtention($file)) {
switch (FileHelper::getExtension($file)) {
case('php'):
//php files are in their own locale dir
$newFile = str_replace('/'.$this->sourceLocale.'/', '/'.$locale.'/', $file);
Expand All @@ -329,7 +335,7 @@ public function createCheckOrEmptyTargetFile($file, $locale)
mkdir($directory, 0755, true);
}

switch (FileHelper::getExtention($file)) {
switch (FileHelper::getExtension($file)) {
case('php'):
$comment = "/**\n * Auto Translated by visualbuilder/ai-translate on ".date("d/m/Y")."\n */";
file_put_contents($newFile, "<?php\n\n".$comment."\n\nreturn [\n\n];\n");
Expand Down Expand Up @@ -389,7 +395,14 @@ private function processChunk($targetFile, $chunk, $targetLanguage)
}
$this->handleRetryFailure($retryCount, $targetFile);
}

private function ensureArray($var, $name) {
if (!is_array($var)) {
$this->error("{$name} is not an array.");
Log::error("{$name} is not an array.", ['content' => $var]);
throw new \Exception("{$name} is not an array.");
}
}

public function appendResponse($filename, $translatedChunk)
{
if(! count($translatedChunk)) {
Expand All @@ -400,16 +413,19 @@ public function appendResponse($filename, $translatedChunk)

// Undot the translated chunk
$undottedTranslatedChunk = Arr::undot($translatedChunk);


$this->ensureArray($existingContent,'existingContent: '.$filename);
$this->ensureArray($undottedTranslatedChunk,'undottedTranslatedChunk: ');

// Merge new translations with existing content
$newContent = array_merge($existingContent, $undottedTranslatedChunk);

$comment = "/**\n * Auto Translated by visualbuilder/ai-translate on ".date("d/m/Y")."\n */";

switch (FileHelper::getExtention($filename)) {
switch (FileHelper::getExtension($filename)) {
case('php'):
$output = "<?php\n\n".$comment."\nreturn ".VarExporter::export($newContent).";\n";
// no break
break;
case('json'):
$output = json_encode($newContent, JSON_PRETTY_PRINT | JSON_UNESCAPED_UNICODE);
}
Expand All @@ -420,6 +436,7 @@ public function appendResponse($filename, $translatedChunk)
private function handleRetry(&$retryCount, $exception, $targetFile)
{
$retryCount++;
$this->newLine();
$this->error('An error occurred while processing the file: '.$targetFile);
$this->error('Error message: '.$exception->getMessage());
$this->info("Retrying $retryCount / $this->maxRetries Times in ".pow(2, $retryCount).' seconds');
Expand Down
4 changes: 2 additions & 2 deletions src/Helpers/FileHelper.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,15 +90,15 @@ private static function getFiles($dir, $basepath, $fileType = '*')
return $files;
}

public static function getExtention($filename)
public static function getExtension($filename)
{
return pathinfo($filename, PATHINFO_EXTENSION);
}

public static function countItemsAndStringLengths($filename)
{

switch (self::getExtention($filename)) {
switch (self::getExtension($filename)) {
case('php'):
$translations = include($filename);

Expand Down
21 changes: 7 additions & 14 deletions src/Helpers/OpenAiHelper.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,15 +50,14 @@ public static function translateChunk($command, $chunk, $sourceLocale, $targetLa
$lines = array_map(function ($line) use (&$placeholders) {
return preg_replace_callback('/:(\w+)/', function ($matches) use (&$placeholders) {
$placeholders[] = $matches[0]; // Store the placeholders

return '***'; // Replace them with ***
}, $line);
}, $lines);

$placeholdersUsed = count($placeholders);

// Add line numbers to each string
$lines = array_map(function ($k, $v) { return ($k + 1) . ". {$v}"; }, array_keys($lines), $lines);
$lines = self::addLineNumbersToArrayofStrings($lines);

$linesString = implode("\n", $lines);

Expand All @@ -67,9 +66,9 @@ public static function translateChunk($command, $chunk, $sourceLocale, $targetLa

// Get the total tokens from all the strings in the chunk
$totalTokens = OpenAiHelper::estimateTokensFromString($prompt.$linesString);
$command->comment("Request tokens: $totalTokens");
$command->comment("Tokens: $totalTokens");
$command->comment("Request: $prompt");
$command->warn('Source Lines');
$command->warn('Source Lines: ');
self::displayArrayInNumberedLines($originalLines, $command);

$response = OpenAI::chat()->create([
Expand Down Expand Up @@ -98,7 +97,7 @@ public static function translateChunk($command, $chunk, $sourceLocale, $targetLa
$translatedStrings = explode("\n", trim($translatedContent));

// Remove the line numbers from the translated strings
$translatedStrings = array_map(function ($s) { return preg_replace('/^\d+\.\s/', '', $s); }, $translatedStrings);
$translatedStrings = array_map(function ($s) { return preg_replace('/^\d+\.\s/', '', $s); },$translatedStrings);



Expand All @@ -109,20 +108,14 @@ public static function translateChunk($command, $chunk, $sourceLocale, $targetLa
}, $line);
}, $translatedStrings);

$translatedChunk = [];
// Combine the original keys with the translated strings
if (count(array_keys($chunk)) === count($translatedStrings)) {
$translatedChunk = array_combine(array_keys($chunk), $translatedStrings);
} else {
// Handle situation when the translation result doesn't match with the original number of keys
$command->error("Mismatch in translation keys and translated strings for chunk. Keys: " . count(array_keys($chunk)) . " Translated Strings: " . count($translatedStrings));
$command->newLine();

// assign untranslated lines to their original value.
// Actually this is not useful. Better not to exist
// foreach (array_keys($chunk) as $i => $key) {
// $translatedChunk[$key] = $translatedStrings[$i] ?? $chunk[$key];
// }
// Sometimes some data is lost.
// Exceptions will trigger a retry upto max_retries.
throw new \Exception("Mismatch in source keys and translation. Keys: " . count(array_keys($chunk)) . " Translated Strings: " . count($translatedStrings));
}

if($placeholdersUsed > 0) {
Expand Down

0 comments on commit dbed4b6

Please sign in to comment.