From 9245951fb90b4996b1d9c2722600bf2da3f82d5e Mon Sep 17 00:00:00 2001 From: mk-mxp <55182845+mk-mxp@users.noreply.github.com> Date: Sun, 3 Mar 2024 11:44:37 +0100 Subject: [PATCH] Test generator, step 1 (#638) It's working, but not complete. --- README.md | 15 ++ contribution/generator/.env | 20 +++ contribution/generator/.env.test | 6 + contribution/generator/.gitignore | 17 ++ contribution/generator/README.md | 20 +++ contribution/generator/bin/console | 17 ++ contribution/generator/composer.json | 71 ++++++++ contribution/generator/config/bundles.php | 6 + .../generator/config/packages/cache.yaml | 19 +++ .../generator/config/packages/framework.yaml | 16 ++ .../generator/config/packages/routing.yaml | 10 ++ contribution/generator/config/preload.php | 5 + contribution/generator/config/routes.yaml | 5 + .../generator/config/routes/framework.yaml | 4 + contribution/generator/config/services.yaml | 26 +++ contribution/generator/phpunit.xml.dist | 27 +++ contribution/generator/public/index.php | 9 + .../src/Command/CreateTestsCommand.php | 71 ++++++++ .../generator/src/Controller/.gitignore | 0 contribution/generator/src/Kernel.php | 11 ++ contribution/generator/src/TestGenerator.php | 92 +++++++++++ .../generator/src/TrackData/CanonicalData.php | 21 +++ .../src/TrackData/CanonicalData/TestCase.php | 18 ++ .../generator/src/TrackData/Exercise.php | 23 +++ .../src/TrackData/PracticeExercise.php | 156 ++++++++++++++++++ contribution/generator/symfony.lock | 81 +++++++++ contribution/generator/tests/bootstrap.php | 11 ++ phpunit.xml | 8 - 28 files changed, 777 insertions(+), 8 deletions(-) create mode 100644 contribution/generator/.env create mode 100644 contribution/generator/.env.test create mode 100644 contribution/generator/.gitignore create mode 100644 contribution/generator/README.md create mode 100755 contribution/generator/bin/console create mode 100644 contribution/generator/composer.json create mode 100644 contribution/generator/config/bundles.php create mode 100644 contribution/generator/config/packages/cache.yaml create mode 100644 contribution/generator/config/packages/framework.yaml create mode 100644 contribution/generator/config/packages/routing.yaml create mode 100644 contribution/generator/config/preload.php create mode 100644 contribution/generator/config/routes.yaml create mode 100644 contribution/generator/config/routes/framework.yaml create mode 100644 contribution/generator/config/services.yaml create mode 100644 contribution/generator/phpunit.xml.dist create mode 100644 contribution/generator/public/index.php create mode 100644 contribution/generator/src/Command/CreateTestsCommand.php create mode 100644 contribution/generator/src/Controller/.gitignore create mode 100644 contribution/generator/src/Kernel.php create mode 100644 contribution/generator/src/TestGenerator.php create mode 100644 contribution/generator/src/TrackData/CanonicalData.php create mode 100644 contribution/generator/src/TrackData/CanonicalData/TestCase.php create mode 100644 contribution/generator/src/TrackData/Exercise.php create mode 100644 contribution/generator/src/TrackData/PracticeExercise.php create mode 100644 contribution/generator/symfony.lock create mode 100644 contribution/generator/tests/bootstrap.php delete mode 100644 phpunit.xml diff --git a/README.md b/README.md index 906b6caa..d9d17082 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,7 @@ The following system dependencies are required: - `composer`, as recommended in the [PHP track installation docs][exercism-track-installation-composer]. - [`bash` shell][gnu-bash] +- PHP V8.2+ CLI Run the following commands to get started with this project: @@ -51,6 +52,20 @@ composer lint:fix # Automatically fix codestyle issues - CI is run on all pull requests, it must pass the required checks for merge. - CI is running all tests on PHP 8.0 to PHP 8.2 +## Generating new practice exercises + +Use `bin/configlet create --practice-exercise ` to create the exercism resources required. +This provides you with the directories and files in `exercises/practice/`. +Look into `tests.toml` for which test cases **not** to implement / generate and mark them with `include = false`. + +Test generator MVP used like this: + +```shell +composer -d contribution/generator install +contribution/generator/bin/console app:create-tests '' +composer lint:fix +``` + [exercism-configlet]: https://exercism.org/docs/building/configlet [exercism-docs]: https://exercism.org/docs [exercism-track-home]: https://exercism.org/docs/tracks/php diff --git a/contribution/generator/.env b/contribution/generator/.env new file mode 100644 index 00000000..900185a0 --- /dev/null +++ b/contribution/generator/.env @@ -0,0 +1,20 @@ +# In all environments, the following files are loaded if they exist, +# the latter taking precedence over the former: +# +# * .env contains default values for the environment variables needed by the app +# * .env.local uncommitted file with local overrides +# * .env.$APP_ENV committed environment-specific defaults +# * .env.$APP_ENV.local uncommitted environment-specific overrides +# +# Real environment variables win over .env files. +# +# DO NOT DEFINE PRODUCTION SECRETS IN THIS FILE NOR IN ANY OTHER COMMITTED FILES. +# https://symfony.com/doc/current/configuration/secrets.html +# +# Run "composer dump-env prod" to compile .env files for production use (requires symfony/flex >=1.2). +# https://symfony.com/doc/current/best_practices.html#use-environment-variables-for-infrastructure-configuration + +###> symfony/framework-bundle ### +APP_ENV=dev +APP_SECRET=1be0b2dbd34333efb23cfefcff0ff718 +###< symfony/framework-bundle ### diff --git a/contribution/generator/.env.test b/contribution/generator/.env.test new file mode 100644 index 00000000..9e7162f0 --- /dev/null +++ b/contribution/generator/.env.test @@ -0,0 +1,6 @@ +# define your env variables for the test env here +KERNEL_CLASS='App\Kernel' +APP_SECRET='$ecretf0rt3st' +SYMFONY_DEPRECATIONS_HELPER=999999 +PANTHER_APP_ENV=panther +PANTHER_ERROR_SCREENSHOT_DIR=./var/error-screenshots diff --git a/contribution/generator/.gitignore b/contribution/generator/.gitignore new file mode 100644 index 00000000..d2d8d075 --- /dev/null +++ b/contribution/generator/.gitignore @@ -0,0 +1,17 @@ +# IDEs +.idea + +###> symfony/framework-bundle ### +/.env.local +/.env.local.php +/.env.*.local +/config/secrets/prod/prod.decrypt.private.php +/public/bundles/ +/var/ +/vendor/ +###< symfony/framework-bundle ### + +###> phpunit/phpunit ### +/phpunit.xml +.phpunit.result.cache +###< phpunit/phpunit ### diff --git a/contribution/generator/README.md b/contribution/generator/README.md new file mode 100644 index 00000000..09fa6828 --- /dev/null +++ b/contribution/generator/README.md @@ -0,0 +1,20 @@ +# Auto Creating of tests for Exercism PHP Track + +This is a small poc on how we could auto generate tests for the PHP track based on the https://github.com/exercism/problem-specifications/. + +How to test it: + +``` +git clone https://github.com/tomasnorre/exercism-tests-generation.git +cd exercism-tests-generation +composer install +bin/console app:create-tests +vendor/bin/phpunit src/Command/NucleotideCountTest.php +``` + +If you now make a `git status` you will see that the `src/Command/NucleotideCountTest.php` and you can now inspect the auto generated tests. + +It's all based on the `nikic/php-parser` and the https://github.com/exercism/problem-specifications/ repository, I have made a local copy of that on file +for now to spare the http-requests. + +Let me know what you think. diff --git a/contribution/generator/bin/console b/contribution/generator/bin/console new file mode 100755 index 00000000..c933dc53 --- /dev/null +++ b/contribution/generator/bin/console @@ -0,0 +1,17 @@ +#!/usr/bin/env php +=8.2", + "ext-ctype": "*", + "ext-iconv": "*", + "nikic/php-parser": "^5.0", + "symfony/console": "7.0.*", + "symfony/dotenv": "7.0.*", + "symfony/flex": "^2", + "symfony/framework-bundle": "7.0.*", + "symfony/runtime": "7.0.*", + "symfony/yaml": "7.0.*" + }, + "require-dev": { + "phpunit/phpunit": "^11.0", + "symfony/maker-bundle": "^1.54" + }, + "config": { + "allow-plugins": { + "php-http/discovery": true, + "symfony/flex": true, + "symfony/runtime": true + }, + "sort-packages": true + }, + "autoload": { + "psr-4": { + "App\\": "src/" + } + }, + "autoload-dev": { + "psr-4": { + "App\\Tests\\": "tests/" + } + }, + "replace": { + "symfony/polyfill-ctype": "*", + "symfony/polyfill-iconv": "*", + "symfony/polyfill-php72": "*", + "symfony/polyfill-php73": "*", + "symfony/polyfill-php74": "*", + "symfony/polyfill-php80": "*", + "symfony/polyfill-php81": "*", + "symfony/polyfill-php82": "*" + }, + "scripts": { + "auto-scripts": { + "cache:clear": "symfony-cmd", + "assets:install %PUBLIC_DIR%": "symfony-cmd" + }, + "post-install-cmd": [ + "@auto-scripts" + ], + "post-update-cmd": [ + "@auto-scripts" + ] + }, + "conflict": { + "symfony/symfony": "*" + }, + "extra": { + "symfony": { + "allow-contrib": false, + "require": "7.0.*" + } + } +} diff --git a/contribution/generator/config/bundles.php b/contribution/generator/config/bundles.php new file mode 100644 index 00000000..ffeb8610 --- /dev/null +++ b/contribution/generator/config/bundles.php @@ -0,0 +1,6 @@ + ['all' => true], + Symfony\Bundle\MakerBundle\MakerBundle::class => ['dev' => true], +]; diff --git a/contribution/generator/config/packages/cache.yaml b/contribution/generator/config/packages/cache.yaml new file mode 100644 index 00000000..6899b720 --- /dev/null +++ b/contribution/generator/config/packages/cache.yaml @@ -0,0 +1,19 @@ +framework: + cache: + # Unique name of your app: used to compute stable namespaces for cache keys. + #prefix_seed: your_vendor_name/app_name + + # The "app" cache stores to the filesystem by default. + # The data in this cache should persist between deploys. + # Other options include: + + # Redis + #app: cache.adapter.redis + #default_redis_provider: redis://localhost + + # APCu (not recommended with heavy random-write workloads as memory fragmentation can cause perf issues) + #app: cache.adapter.apcu + + # Namespaced pools use the above "app" backend by default + #pools: + #my.dedicated.cache: null diff --git a/contribution/generator/config/packages/framework.yaml b/contribution/generator/config/packages/framework.yaml new file mode 100644 index 00000000..877eb25d --- /dev/null +++ b/contribution/generator/config/packages/framework.yaml @@ -0,0 +1,16 @@ +# see https://symfony.com/doc/current/reference/configuration/framework.html +framework: + secret: '%env(APP_SECRET)%' + #csrf_protection: true + + # Note that the session will be started ONLY if you read or write from it. + session: true + + #esi: true + #fragments: true + +when@test: + framework: + test: true + session: + storage_factory_id: session.storage.factory.mock_file diff --git a/contribution/generator/config/packages/routing.yaml b/contribution/generator/config/packages/routing.yaml new file mode 100644 index 00000000..8166181c --- /dev/null +++ b/contribution/generator/config/packages/routing.yaml @@ -0,0 +1,10 @@ +framework: + router: + # Configure how to generate URLs in non-HTTP contexts, such as CLI commands. + # See https://symfony.com/doc/current/routing.html#generating-urls-in-commands + #default_uri: http://localhost + +when@prod: + framework: + router: + strict_requirements: null diff --git a/contribution/generator/config/preload.php b/contribution/generator/config/preload.php new file mode 100644 index 00000000..5ebcdb21 --- /dev/null +++ b/contribution/generator/config/preload.php @@ -0,0 +1,5 @@ + + + + + + + + + + + + + + + + tests + + + + + + diff --git a/contribution/generator/public/index.php b/contribution/generator/public/index.php new file mode 100644 index 00000000..9982c218 --- /dev/null +++ b/contribution/generator/public/index.php @@ -0,0 +1,9 @@ +trackRoot = realpath($projectDir . '/../..'); + + parent::__construct(); + } + + protected function configure(): void + { + $this->addArgument('exercise', InputArgument::REQUIRED, 'Exercise slug'); + } + + protected function execute(InputInterface $input, OutputInterface $output): int + { + $exerciseSlug = $input->getArgument('exercise'); + $exercise = new PracticeExercise( + $this->trackRoot, + $exerciseSlug, + ); + $testGenerator = new TestGenerator(); + + $io = new SymfonyStyle($input, $output); + $io->writeln('Generating tests for ' . $exerciseSlug . ' in ' . $exercise->pathToExercise()); + + \file_put_contents( + // TODO: Make '$exercise->pathToTestFile()' + $exercise->pathToExercise() + . '/' + . $this->inPascalCase($exerciseSlug) + . 'Test.php', + $testGenerator->createTestsFor( + $exercise->canonicalData(), + $this->inPascalCase($exerciseSlug) + ), + ); + // TODO: Make '$exercise->pathToStudentsFile()' + // TODO: Make '$testGenerator->studentsFileFor()' + + $io->success('Generating Tests - Finished'); + return Command::SUCCESS; + } + + private function inPascalCase(string $text): string + { + return \str_replace(" ", "", \ucwords(\str_replace("-", " ", $text))); + } +} diff --git a/contribution/generator/src/Controller/.gitignore b/contribution/generator/src/Controller/.gitignore new file mode 100644 index 00000000..e69de29b diff --git a/contribution/generator/src/Kernel.php b/contribution/generator/src/Kernel.php new file mode 100644 index 00000000..779cd1f2 --- /dev/null +++ b/contribution/generator/src/Kernel.php @@ -0,0 +1,11 @@ +builderFactory = new BuilderFactory(); + + $class = $this->builderFactory->class( + $exerciseClass . "Test" + )->makeFinal()->extend('TestCase'); + $class->setDocComment( + "/**\n * " . implode("\n * ", $canonicalData->comments) . "\n */" + ); + + // Include Setup Method + $methodSetup = 'setUpBeforeClass'; + $method = $this->builderFactory->method($methodSetup) + ->makePublic() + ->makeStatic() + ->setReturnType('void') + ->addStmt( + $this->builderFactory->funcCall( + "require_once", + [$exerciseClass . ".php"] + ), + ); + + $class->addStmt($method); + + foreach ($canonicalData->testCases as $case) { + // Generate a method for each test case + $description = \ucfirst($case->description); + $methodName = ucfirst(str_replace('-', ' ', $description)); + $methodName = 'test' . str_replace(' ', '', ucwords($methodName)); + + // $exceptionClassName = new Node\Name\FullyQualified('Exception'); + $method = $this->builderFactory->method($methodName) + ->makePublic() + ->setReturnType('void') + ->setDocComment("/**\n * uuid: {$case->uuid}\n * @testdox {$description}\n */") + ->addStmt( + $this->builderFactory->funcCall( + '$this->markTestIncomplete', + [ 'This test has not been implemented yet.' ], + ) + ) + ; + // if (isset($case->expected->error)) { + // $method->addStmt( + // $this->builderFactory->funcCall('$this->expectException', + // [new Node\Arg(new Node\Expr\ClassConstFetch($exceptionClassName, 'class'))] + // ) + // ) + // ->addStmt($this->builderFactory->funcCall($case->property, [$case->input->strand ?? 'unknown'])) + // ; + // } else { + // $method->addStmt( + // $this->builderFactory->funcCall('$this->assertEquals', [ + // $case->expected, + // $this->builderFactory->funcCall($case->property, [$case->input->strand ?? 'unknown']) + // ]) + // ); + // } + + $class->addStmt($method); + } + + $namespace = new Namespace_(new Node\Name('Tests')); + $namespace->stmts[] = $this->builderFactory->use(TestCase::class)->getNode(); + $namespace->stmts[] = $class->getNode(); + + $printer = new PrettyPrinter\Standard(); + + return $printer->prettyPrintFile([$namespace]) . PHP_EOL; + } +} diff --git a/contribution/generator/src/TrackData/CanonicalData.php b/contribution/generator/src/TrackData/CanonicalData.php new file mode 100644 index 00000000..cedcee7a --- /dev/null +++ b/contribution/generator/src/TrackData/CanonicalData.php @@ -0,0 +1,21 @@ +pathToConfiglet = $trackRoot . '/bin/configlet'; + $this->pathToPracticeExercises = $trackRoot . '/exercises/practice/'; + $this->pathToExercise = + $this->pathToPracticeExercises . $this->exerciseSlug; + } + + public function pathToExercise(): string + { + return $this->pathToExercise; + } + + public function canonicalData(): CanonicalData + { + $this->ensureConfigletCanBeUsed(); + $this->ensurePracticeExerciseCanBeUsed(); + $this->pathToCachedCanonicalDataFromConfiglet(); + $this->ensurePathToCanonicalDataCanBeUsed(); + + return $this->hydratedCanonicalData(); + } + + private function hydratedCanonicalData(): CanonicalData + { + $canonicalData = \json_decode( + json: \file_get_contents($this->pathToCanonicalData), + flags: JSON_THROW_ON_ERROR + ); + + // TODO: Validate + return new CanonicalData( + $canonicalData->exercise, + $this->hydrateTestCasesFrom($canonicalData->cases), + $canonicalData->comments, + ); + } + + private function hydrateTestCasesFrom(array $rawData): array + { + // TODO: Validate + return array_map(fn ($case) => new TestCase( + $case->uuid ?? null, + $case->description ?? null, + $case->property ?? null, + $case->input ?? null, + $case->expected ?? null, + $case->comments ?? [], + ), $rawData); + } + + private function ensureConfigletCanBeUsed(): void + { + if ( + !( + is_executable($this->pathToConfiglet) + && is_file($this->pathToConfiglet) + ) + ) { + throw new RuntimeException( + 'configlet not found. Run the generator from track root.' + . ' Fetch configlet and create exercise with configlet first!' + ); + } + } + + private function ensurePracticeExerciseCanBeUsed(): void + { + if ( + !( + is_writable($this->pathToExercise) + && is_dir($this->pathToExercise) + ) + ) { + throw new RuntimeException( + 'Cannot write to exercise directory. Create exercise with' + . ' configlet first or check access rights!' + ); + } + } + + private function pathToCachedCanonicalDataFromConfiglet(): void + { + /* + When running configlet with detailed output (-v d) and a command that + requires problem specification data (e.g. info), it prints the location + of the cache as the first line. To avoid an HTTP call, use the offline + mode (-o). + + Pipe the output through 'head' to get the first line only, then 'cut' + the 5th field to get the path only. + + configlet may fail when there is no cached data (offline mode), which + tells us, that the exercise hasn't been generated before (the cache is + required for that, too). So BASH must use `-eo pipefail` to get the + failure code back. + */ + $command = 'bash -c \'set -eo pipefail; ' + . $this->pathToConfiglet + . ' -v d -t ' + . $this->trackRoot + . ' info -o | head -1 | cut -d " " -f 5\'' + ; + $resultCode = 1; + + $configletCache = \exec(command: $command, result_code: $resultCode); + if ($configletCache === false || $resultCode !== 0) { + throw new RuntimeException( + '"configlet" could not provide cached canonical data.' + . ' Create exercise with configlet first!' + ); + } + + $this->pathToCanonicalData = \sprintf( + '%1$s/exercises/%2$s/canonical-data.json', + $configletCache, + $this->exerciseSlug + ); + } + + private function ensurePathToCanonicalDataCanBeUsed(): void + { + if ( + !( + is_readable($this->pathToCanonicalData) + && is_file($this->pathToCanonicalData) + ) + ) { + throw new RuntimeException( + 'Cannot read "configlet" provided cached canonical data from ' + . $this->pathToCanonicalData + . '. Check exercise slug or access rights!' + ); + } + } +} diff --git a/contribution/generator/symfony.lock b/contribution/generator/symfony.lock new file mode 100644 index 00000000..e6e2f486 --- /dev/null +++ b/contribution/generator/symfony.lock @@ -0,0 +1,81 @@ +{ + "phpunit/phpunit": { + "version": "11.0", + "recipe": { + "repo": "github.com/symfony/recipes", + "branch": "main", + "version": "9.6", + "ref": "7364a21d87e658eb363c5020c072ecfdc12e2326" + }, + "files": [ + ".env.test", + "phpunit.xml.dist", + "tests/bootstrap.php" + ] + }, + "symfony/console": { + "version": "7.0", + "recipe": { + "repo": "github.com/symfony/recipes", + "branch": "main", + "version": "5.3", + "ref": "da0c8be8157600ad34f10ff0c9cc91232522e047" + }, + "files": [ + "bin/console" + ] + }, + "symfony/flex": { + "version": "2.4", + "recipe": { + "repo": "github.com/symfony/recipes", + "branch": "main", + "version": "1.0", + "ref": "146251ae39e06a95be0fe3d13c807bcf3938b172" + }, + "files": [ + ".env" + ] + }, + "symfony/framework-bundle": { + "version": "7.0", + "recipe": { + "repo": "github.com/symfony/recipes", + "branch": "main", + "version": "7.0", + "ref": "6356c19b9ae08e7763e4ba2d9ae63043efc75db5" + }, + "files": [ + "config/packages/cache.yaml", + "config/packages/framework.yaml", + "config/preload.php", + "config/routes/framework.yaml", + "config/services.yaml", + "public/index.php", + "src/Controller/.gitignore", + "src/Kernel.php" + ] + }, + "symfony/maker-bundle": { + "version": "1.54", + "recipe": { + "repo": "github.com/symfony/recipes", + "branch": "main", + "version": "1.0", + "ref": "fadbfe33303a76e25cb63401050439aa9b1a9c7f" + } + }, + "symfony/routing": { + "version": "7.0", + "recipe": { + "repo": "github.com/symfony/recipes", + "branch": "main", + "version": "7.0", + "ref": "21b72649d5622d8f7da329ffb5afb232a023619d" + }, + "files": [ + "config/packages/routing.yaml", + "config/routes.yaml" + ] + } +} diff --git a/contribution/generator/tests/bootstrap.php b/contribution/generator/tests/bootstrap.php new file mode 100644 index 00000000..469dccee --- /dev/null +++ b/contribution/generator/tests/bootstrap.php @@ -0,0 +1,11 @@ +bootEnv(dirname(__DIR__).'/.env'); +} diff --git a/phpunit.xml b/phpunit.xml deleted file mode 100644 index 5cd996e7..00000000 --- a/phpunit.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - ./ - - -