diff --git a/.php-cs-fixer.cache b/.php-cs-fixer.cache index 4b70f47..ead170b 100644 --- a/.php-cs-fixer.cache +++ b/.php-cs-fixer.cache @@ -1 +1 @@ -{"php":"8.2.7","version":"3.20.0","indent":" ","lineEnding":"\n","rules":{"blank_line_after_opening_tag":true,"blank_line_between_import_groups":true,"blank_lines_before_namespace":true,"class_definition":{"inline_constructor_arguments":false,"space_before_parenthesis":true},"compact_nullable_typehint":true,"curly_braces_position":{"allow_single_line_empty_anonymous_classes":true},"declare_equal_normalize":true,"lowercase_cast":true,"lowercase_static_reference":true,"new_with_braces":true,"no_blank_lines_after_class_opening":true,"no_leading_import_slash":true,"no_whitespace_in_blank_line":true,"ordered_class_elements":{"order":["use_trait"]},"ordered_imports":{"sort_algorithm":"alpha"},"return_type_declaration":true,"short_scalar_cast":true,"single_import_per_statement":{"group_to_single_imports":false},"single_trait_insert_per_statement":true,"ternary_operator_spaces":true,"visibility_required":true,"blank_line_after_namespace":true,"constant_case":true,"control_structure_braces":true,"control_structure_continuation_position":true,"elseif":true,"function_declaration":true,"indentation_type":true,"line_ending":true,"lowercase_keywords":true,"method_argument_space":{"on_multiline":"ensure_fully_multiline","keep_multiple_spaces_after_comma":true},"no_break_comment":true,"no_closing_tag":true,"no_multiple_statements_per_line":true,"no_space_around_double_colon":true,"no_spaces_after_function_name":true,"no_spaces_inside_parenthesis":true,"no_trailing_whitespace":true,"no_trailing_whitespace_in_comment":true,"single_blank_line_at_eof":true,"single_class_element_per_statement":{"elements":["property"]},"single_line_after_imports":true,"statement_indentation":true,"switch_case_semicolon_to_colon":true,"switch_case_space":true,"encoding":true,"full_opening_tag":true,"array_syntax":{"syntax":"short"},"no_unused_imports":true,"not_operator_with_successor_space":true,"trailing_comma_in_multiline":true,"phpdoc_scalar":true,"unary_operator_spaces":true,"binary_operator_spaces":true,"blank_line_before_statement":{"statements":["break","continue","declare","return","throw","try"]},"phpdoc_single_line_var_spacing":true,"phpdoc_var_without_name":true,"class_attributes_separation":{"elements":{"method":"one"}}},"hashes":{"src\/Console\/InstallCommand.php":"147be8dd6fb08df5cc28b96c9c9c095d","src\/Console\/TranslateStrings.php":"f110b77b3a58e856fd4f7945db142620","src\/Helpers\/FileHelper.php":"43f73a0a4462d3645b2a819b818a5d26","src\/Helpers\/OpenAiHelper.php":"845cde34a5ce4fe9594faddae6d51c8d","src\/AiTranslateServiceProvider.php":"b1bff69a366d9d989f324fecd430aca4","tests\/TestCase.php":"e6219ed5a379f12f69434c02c23e7022","tests\/Pest.php":"6e5dfd4d87636df2e7b436612c442111","tests\/Unit\/SetupTest.php":"a3ff27305662d6ea6674a89eef7eb33a"}} \ No newline at end of file +{"php":"8.2.7","version":"3.20.0","indent":" ","lineEnding":"\n","rules":{"blank_line_after_opening_tag":true,"blank_line_between_import_groups":true,"blank_lines_before_namespace":true,"class_definition":{"inline_constructor_arguments":false,"space_before_parenthesis":true},"compact_nullable_typehint":true,"curly_braces_position":{"allow_single_line_empty_anonymous_classes":true},"declare_equal_normalize":true,"lowercase_cast":true,"lowercase_static_reference":true,"new_with_braces":true,"no_blank_lines_after_class_opening":true,"no_leading_import_slash":true,"no_whitespace_in_blank_line":true,"ordered_class_elements":{"order":["use_trait"]},"ordered_imports":{"sort_algorithm":"alpha"},"return_type_declaration":true,"short_scalar_cast":true,"single_import_per_statement":{"group_to_single_imports":false},"single_trait_insert_per_statement":true,"ternary_operator_spaces":true,"visibility_required":true,"blank_line_after_namespace":true,"constant_case":true,"control_structure_braces":true,"control_structure_continuation_position":true,"elseif":true,"function_declaration":true,"indentation_type":true,"line_ending":true,"lowercase_keywords":true,"method_argument_space":{"on_multiline":"ensure_fully_multiline","keep_multiple_spaces_after_comma":true},"no_break_comment":true,"no_closing_tag":true,"no_multiple_statements_per_line":true,"no_space_around_double_colon":true,"no_spaces_after_function_name":true,"no_spaces_inside_parenthesis":true,"no_trailing_whitespace":true,"no_trailing_whitespace_in_comment":true,"single_blank_line_at_eof":true,"single_class_element_per_statement":{"elements":["property"]},"single_line_after_imports":true,"statement_indentation":true,"switch_case_semicolon_to_colon":true,"switch_case_space":true,"encoding":true,"full_opening_tag":true,"array_syntax":{"syntax":"short"},"no_unused_imports":true,"not_operator_with_successor_space":true,"trailing_comma_in_multiline":true,"phpdoc_scalar":true,"unary_operator_spaces":true,"binary_operator_spaces":true,"blank_line_before_statement":{"statements":["break","continue","declare","return","throw","try"]},"phpdoc_single_line_var_spacing":true,"phpdoc_var_without_name":true,"class_attributes_separation":{"elements":{"method":"one"}}},"hashes":{"src\/Console\/InstallCommand.php":"147be8dd6fb08df5cc28b96c9c9c095d","src\/Console\/TranslateStrings.php":"371b7be6aa3d9578f2da41fd49907377","src\/Helpers\/FileHelper.php":"43f73a0a4462d3645b2a819b818a5d26","src\/Helpers\/OpenAiHelper.php":"845cde34a5ce4fe9594faddae6d51c8d","src\/AiTranslateServiceProvider.php":"b1bff69a366d9d989f324fecd430aca4","tests\/TestCase.php":"e6219ed5a379f12f69434c02c23e7022","tests\/Pest.php":"6e5dfd4d87636df2e7b436612c442111","tests\/Unit\/SetupTest.php":"a3ff27305662d6ea6674a89eef7eb33a"}} \ No newline at end of file diff --git a/src/Console/TranslateStrings.php b/src/Console/TranslateStrings.php index 5625732..34e4a99 100644 --- a/src/Console/TranslateStrings.php +++ b/src/Console/TranslateStrings.php @@ -25,65 +25,72 @@ class TranslateStrings extends Command * @var string */ protected $description = "Translates all language files in these directories: "; - protected $model = ""; - protected $maxInputStringLength = 2000; - protected $maxRetries = 5; - protected $overwrite = false; - protected $sourceData = []; - protected $sourceLocale = ""; - protected $totalPromptTokens = 0; + protected $model = ""; + protected $maxInputStringLength = 2000; + protected $maxRetries = 5; + protected $overwrite = false; + protected $sourceData = []; + protected $sourceLocale = ""; + protected $totalPromptTokens = 0; protected $totalCompletionTokens = 0; - - public function __construct() { + + public function __construct() + { parent::__construct(); - + $dirs = [ 'source_directories' => [ 'lang', 'resources/lang', ], ]; - + $dirString = implode("\n", $dirs[ 'source_directories' ]); - + $this->signature = $dirString; $this->setDescription( (string) "Translates all PHP & JSON language files in these directories: \n\n\t- ".implode( - "\n\t- ", config('ai-translate.source_directories') + "\n\t- ", + config('ai-translate.source_directories') )."\n\n Source Locale: ".config('ai-translate.source-locale')."\n\n Target Languages: \n\t- ".implode( - "\n\t- ", config('ai-translate.target_locales') + "\n\t- ", + config('ai-translate.target_locales') ) ); } - + /** * Execute the console command. * * @return int */ - public function handle() { + public function handle() + { $this->init(); $files = $this->findSourceFiles(); $this->translateFiles($files); - + return Command::SUCCESS; } - - private function init() { + + private function init() + { $this->sourceLocale = config('ai-translate.source-locale'); $this->overwrite = (bool) $this->option('force'); } - - private function findSourceFiles() { + + private function findSourceFiles() + { $jsonFiles = FileHelper::getJsonSourceFileList($this->sourceLocale); $phpFiles = FileHelper::getLanguageFileList($this->sourceLocale); - + return array_merge($jsonFiles, $phpFiles); } - - private function translateFiles($files) { + + private function translateFiles($files) + { $this->estimateCostAndSelectModel($files); - + foreach ($files as $sourceFile) { $this->sourceData = $this->readLanguageFile($sourceFile); $chunks = $this->chunkArray($this->sourceData); @@ -91,13 +98,13 @@ private function translateFiles($files) { $this->handleTargetLocale($sourceFile, $chunks, $targetLocale, $targetLanguage); } } - + $this->newLine(); $this->info("Total prompt tokens used: ".$this->totalPromptTokens); $this->info("Total completion tokens used: ".$this->totalCompletionTokens); $this->warn("Total Cost: $".$this->getCost()); } - + /** * Parse the input files and estimate the cost of translation * Ask the user which model they would like to use if they have not specified @@ -109,19 +116,20 @@ private function translateFiles($files) { * * @return true */ - private function estimateCostAndSelectModel($files) { + private function estimateCostAndSelectModel($files) + { $targetLanguages = config('ai-translate.target_locales'); $targetCount = count($targetLanguages); - + $this->info("Found ".count($files)." files to translate into $targetCount languages"); - + $stats = []; $totalItems = 0; $totalLength = 0; $sourceTokensTotal = 0; $targetTokensTotal = 0; $totalTokens = 0; - + foreach ($files as $file) { $lengths = FileHelper::countItemsAndStringLengths($file); $sourceTokens = OpenAiHelper::estimateTokens($lengths[ 'totalLength' ]); @@ -132,9 +140,9 @@ private function estimateCostAndSelectModel($files) { $sourceTokens, $sourceTokens * $targetCount, $sourceTokens + ($sourceTokens * $targetCount), - + ]; - + // Accumulate totals $totalItems += $lengths[ 'itemCount' ]; $totalLength += $lengths[ 'totalLength' ]; @@ -142,21 +150,22 @@ private function estimateCostAndSelectModel($files) { $targetTokensTotal += $sourceTokens * $targetCount; $totalTokens += $sourceTokens + ($sourceTokens * $targetCount); } - + // Append totals to stats $stats[] = ['Total', $totalItems, $totalLength, $sourceTokensTotal, $targetTokensTotal, $totalTokens]; - + $this->table( - ['File', 'Lines', 'Total String Length', 'Source Tokens', 'Target Tokens', 'Total Tokens'], $stats + ['File', 'Lines', 'Total String Length', 'Source Tokens', 'Target Tokens', 'Total Tokens'], + $stats ); - + $this->newLine(); - $this->info('Cost Estimations per AI model for '.number_format($totalTokens,0,"",",").' tokens'); - + $this->info('Cost Estimations per AI model for '.number_format($totalTokens, 0, "", ",").' tokens'); + $totals = end($stats); - + $data = []; - + foreach (config('ai-translate.ai-models') as $key => $model) { $data[] = [ $key, @@ -164,45 +173,49 @@ private function estimateCostAndSelectModel($files) { round(($totals[ 4 ] / 1000) * $model[ 'output_price' ], 2), round( (($totals[ 3 ] / 1000) * $model[ 'input_price' ]) + (($totals[ 4 ] / 1000) - * $model[ 'output_price' ]), 2 + * $model[ 'output_price' ]), + 2 ), ]; } - + $this->table( - ['Model', 'Input Cost ($)', 'Output Cost ($)', 'Total Cost ($)'], $data + ['Model', 'Input Cost ($)', 'Output Cost ($)', 'Total Cost ($)'], + $data ); $this->warn('Please notes costs are estimations only, prices will vary subject to the destination languages translated'); - - if(!$this->option('cheapest') && !$this->option('best')) { + + if(! $this->option('cheapest') && ! $this->option('best')) { $model = $this->choice( - 'Which model would you like to use?', array_keys(config('ai-translate.ai-models')), array_keys(config('ai-translate.ai-models'))[ 0 ] + 'Which model would you like to use?', + array_keys(config('ai-translate.ai-models')), + array_keys(config('ai-translate.ai-models'))[ 0 ] ); - } - else { + } else { $models = config('ai-translate.ai-models'); $model = $this->option('cheapest') ? array_keys(config('ai-translate.ai-models'))[ 0 ] : end($models); } - + $this->info("Going to translate using $model"); $this->model = $model; $this->setMaxInputStringLength(); - + return true; } - + /** * Deliberately keeping the chunks smaller to avoid timeout issues with long inputs. * With GPT-3.5 this equates to 2700 characters input per request * * @return void */ - private function setMaxInputStringLength() { + private function setMaxInputStringLength() + { $maxTokens = config('ai-translate.ai-models')[ $this->model ][ 'max_tokens' ]; $maxCharactersTotal = $maxTokens * 4; $this->maxInputStringLength = round($maxCharactersTotal / 6, 0); } - + /** * They should always be an array so we can just include it. * @@ -210,8 +223,9 @@ private function setMaxInputStringLength() { * * @return mixed */ - public function readLanguageFile($file) { - if(!file_exists($file)) { + public function readLanguageFile($file) + { + if(! file_exists($file)) { return []; } switch (FileHelper::getExtention($file)) { @@ -221,7 +235,7 @@ public function readLanguageFile($file) { return json_decode(file_get_contents($file), true); } } - + /** * Splits a file into chunks of a given length * @@ -230,22 +244,23 @@ public function readLanguageFile($file) { * * @return array The chunks */ - public function chunkArray($array) { + public function chunkArray($array) + { $chunks = []; $chunk = []; $length = 0; - + // Flatten the array using Laravel's helper function $flattenedArray = Arr::dot($array); - + foreach ($flattenedArray as $key => $value) { // If the value is an empty array, skip to the next iteration if(is_array($value) && empty($value)) { continue; } - + $itemLength = strlen($value); - if($length + $itemLength > $this->maxInputStringLength && !empty($chunk)) { + if($length + $itemLength > $this->maxInputStringLength && ! empty($chunk)) { // If adding this item will exceed the max length, save the current chunk and start a new one $chunks[] = $chunk; $chunk = []; @@ -254,14 +269,14 @@ public function chunkArray($array) { $chunk[ $key ] = $value; $length += $itemLength; } - if(!empty($chunk)) { + if(! empty($chunk)) { // Add the last chunk if it's not empty $chunks[] = $chunk; } - + return $chunks; } - + /** * Create the target file and process the input chunked array * @@ -272,53 +287,55 @@ public function chunkArray($array) { * * @return void */ - private function handleTargetLocale($file, $chunks, $targetLocale, $targetLanguage) { + private function handleTargetLocale($file, $chunks, $targetLocale, $targetLanguage) + { if($targetFile = $this->createCheckOrEmptyTargetFile($file, $targetLocale)) { $this->handleExistingTargetFile($targetFile, $chunks); - + foreach ($chunks as $chunk) { $this->processChunk($targetFile, $chunk, $targetLanguage); } } } - - public function createCheckOrEmptyTargetFile($file, $locale) { + + public function createCheckOrEmptyTargetFile($file, $locale) + { // Replace source locale with target locale in the path switch (FileHelper::getExtention($file)) { case('php'): //php files are in their own locale dir $newFile = str_replace('/'.$this->sourceLocale.'/', '/'.$locale.'/', $file); - + break; case('json'): //Json files are in the same dir $newFile = str_replace($this->sourceLocale.'.json', $locale.'.json', $file); } - + // If overwrite is false and file already exists, make sure it contains an array - if(!$this->overwrite && file_exists($newFile)) { + if(! $this->overwrite && file_exists($newFile)) { return is_array($this->readLanguageFile($newFile)) ? $newFile : false; } - + // Create directory structure if it doesn't exist $directory = dirname($newFile); - if(!file_exists($directory)) { + if(! file_exists($directory)) { mkdir($directory, 0755, true); } - + switch (FileHelper::getExtention($file)) { case('php'): $comment = "/**\n * Auto Translated by visualbuilder/ai-translate on ".date("d/m/Y")."\n */"; file_put_contents($newFile, "readLanguageFile($targetFile); - if(count($targetArray) !== 0 && !$this->overwrite) { + if(count($targetArray) !== 0 && ! $this->overwrite) { $diffArray = array_diff_key($this->sourceData, $targetArray); $chunks = $this->chunkArray($diffArray); } } - + /** * Gracefully try translation, with some retries if fails and save the result to the target * @@ -344,55 +362,58 @@ private function handleExistingTargetFile($targetFile, &$chunks) { * * @return void */ - private function processChunk($targetFile, $chunk, $targetLanguage) { + private function processChunk($targetFile, $chunk, $targetLanguage) + { $retryCount = 0; $complete = false; - while ($retryCount < $this->maxRetries && !$complete) { + while ($retryCount < $this->maxRetries && ! $complete) { try { $translatedChunk = OpenAiHelper::translateChunk($this, $chunk, $this->sourceLocale, $targetLanguage, $this->model); $this->appendResponse($targetFile, $translatedChunk[ 'translatedChunk' ]); $complete = true; - } - catch (\Exception $e) { + } catch (\Exception $e) { $this->handleRetry($retryCount, $e, $targetFile); } } $this->handleRetryFailure($retryCount, $targetFile); } - - public function appendResponse($filename, $translatedChunk) { + + public function appendResponse($filename, $translatedChunk) + { // Fetch existing content $existingContent = $this->readLanguageFile($filename); - + // Undot the translated chunk $undottedTranslatedChunk = Arr::undot($translatedChunk); - + // Merge new translations with existing content $newContent = array_merge($existingContent, $undottedTranslatedChunk); - + $comment = "/**\n * Auto Translated by visualbuilder/ai-translate on ".date("d/m/Y")."\n */"; - + switch (FileHelper::getExtention($filename)) { case('php'): $output = "error('An error occurred while processing the file: '.$targetFile); $this->error('Error message: '.$exception->getMessage()); $this->info("Retrying $retryCount / $this->maxRetries Times in ".pow(2, $retryCount).' seconds'); usleep(pow(2, $retryCount) * 1000000); // Wait for 2^retryCount seconds } - - private function handleRetryFailure($retryCount, $targetFile) { + + private function handleRetryFailure($retryCount, $targetFile) + { if($retryCount === $this->maxRetries) { $this->error("Request failed after ".$this->maxRetries." retries"); if(file_exists($targetFile)) { @@ -400,15 +421,16 @@ private function handleRetryFailure($retryCount, $targetFile) { } } } - - private function getCost() { + + private function getCost() + { $cost = ($this->totalPromptTokens / 1000) * config('ai-translate.ai-models')[ $this->model ][ 'input_price' ]; $cost += ($this->totalCompletionTokens / 1000) * config('ai-translate.ai-models')[ $this->model ][ 'output_price' ]; - + return $cost; } - + /** * Recursively merges arrays * @@ -417,16 +439,16 @@ private function getCost() { * * @return array The merged array */ - private function mergeArray($array1, $array2) { + private function mergeArray($array1, $array2) + { foreach ($array2 as $key => $value) { if(array_key_exists($key, $array1) && is_array($array1[ $key ]) && is_array($value)) { $array1[ $key ] = $this->mergeArray($array1[ $key ], $value); - } - else { + } else { $array1[ $key ] = $value; } } - + return $array1; } }