Add infection test tool (#1306)

* run unit tests in random order running in random order highlights some interference between tests (mostly logging being enabled), and is a requirement for mutation testing (ie, all tests must pass when executed in a random order). * adding infection mutation testing this adds the ability to run infection. It generates a lot of output, which is an exercise for another PR... * infection max threads
open-telemetry · May 15, 2024 · 4f32817 · 4f32817
1 parent 500f6ee
commit 4f32817
Show file tree

Hide file tree

Showing 20 changed files with 91 additions and 17 deletions.
diff --git a/Makefile b/Makefile
@@ -43,6 +43,8 @@ psalm-info: ## Run psalm and show info
 	$(DC_RUN_PHP) env XDEBUG_MODE=off vendor-bin/psalm/vendor/bin/psalm --show-info=true --threads=1
 phpstan: ## Run phpstan
 	$(DC_RUN_PHP) env XDEBUG_MODE=off vendor/bin/phpstan analyse --memory-limit=256M
+infection: ## Run infection (mutation testing)
+	$(DC_RUN_PHP) env XDEBUG_MODE=coverage php -d memory_limit=1024M vendor-bin/infection/vendor/bin/infection --threads=max
 packages-composer: ## Validate composer packages
 	$(DC_RUN_PHP) env XDEBUG_MODE=off vendor/bin/otel packages:composer:validate
 benchmark: ## Run phpbench

diff --git a/infection.json5 b/infection.json5
@@ -0,0 +1,19 @@
+{
+    "$schema": "https://raw.githubusercontent.com/infection/infection/0.28.1/resources/schema.json",
+    "source": {
+        "directories": [
+            "src"
+        ],
+        "excludes": [
+            "Composer"
+        ]
+    },
+    "logs": {
+        "text": "var/infection/infection.log",
+        "html": "var/infection/infection.html",
+    },
+    "timeout": 1,
+    "mutators": {
+        "@default": true
+    }
+}
diff --git a/phpunit.xml.dist b/phpunit.xml.dist
@@ -6,6 +6,7 @@
     bootstrap="./tests/bootstrap.php"
     cacheResult="false"
     colors="false"
+    executionOrder="random"
     processIsolation="false"
     stopOnError="false"
     stopOnFailure="false"

diff --git a/src/API/Behavior/Internal/Logging.php b/src/API/Behavior/Internal/Logging.php
@@ -5,6 +5,7 @@
 namespace OpenTelemetry\API\Behavior\Internal;
 
 use OpenTelemetry\API\Behavior\Internal\LogWriter\LogWriterInterface;
+use OpenTelemetry\API\Behavior\Internal\LogWriter\NoopLogWriter;
 use Psr\Log\LogLevel;
 
 /**
@@ -87,4 +88,9 @@ public static function reset(): void
         self::$logLevel = null;
         self::$writer = null;
     }
+
+    public static function disable(): void
+    {
+        self::$writer = new NoopLogWriter();
+    }
 }
diff --git a/src/API/LoggerHolder.php b/src/API/LoggerHolder.php
@@ -38,13 +38,17 @@ public static function isSet(): bool
         return null !== self::$logger;
     }
 
+    /**
+     * @internal
+     */
     public static function unset(): void
     {
         self::$logger = null;
     }
 
     /**
      * Disable psr-3 logging
+     * @internal
      */
     public static function disable(): void
     {

diff --git a/tests/Unit/API/Behavior/Internal/LogWriterFactoryTest.php b/tests/Unit/API/Behavior/Internal/LogWriterFactoryTest.php
@@ -21,6 +21,11 @@ class LogWriterFactoryTest extends TestCase
 {
     use EnvironmentVariables;
 
+    public function setUp(): void
+    {
+        LoggerHolder::unset();
+    }
+
     public function tearDown(): void
     {
         self::restoreEnvironmentVariables();

diff --git a/tests/Unit/API/Behavior/LogsMessagesTraitTest.php b/tests/Unit/API/Behavior/LogsMessagesTraitTest.php
@@ -23,6 +23,7 @@ class LogsMessagesTraitTest extends TestCase
 
     public function setUp(): void
     {
+        Logging::reset();
         $this->writer = $this->createMock(LogWriterInterface::class);
         Logging::setLogWriter($this->writer);
     }

diff --git a/tests/Unit/API/Common/Time/ClockTest.php b/tests/Unit/API/Common/Time/ClockTest.php
@@ -14,6 +14,16 @@
  */
 class ClockTest extends TestCase
 {
+    public function setUp(): void
+    {
+        Clock::reset();
+    }
+
+    public function tearDown(): void
+    {
+        Clock::reset();
+    }
+
     public function test_default_is_system_clock(): void
     {
         $this->assertInstanceOf(SystemClock::class, Clock::getDefault());

diff --git a/tests/Unit/API/Instrumentation/InstrumentationTest.php b/tests/Unit/API/Instrumentation/InstrumentationTest.php
@@ -33,6 +33,11 @@
  */
 final class InstrumentationTest extends TestCase
 {
+    public function setUp(): void
+    {
+        Globals::reset();
+    }
+
     public function tearDown(): void
     {
         Globals::reset();

diff --git a/tests/Unit/API/LoggerHolderTest.php b/tests/Unit/API/LoggerHolderTest.php
@@ -19,6 +19,11 @@ public function tearDown(): void
         LoggerHolder::unset();
     }
 
+    public function setUp(): void
+    {
+        LoggerHolder::unset();
+    }
+
     public function test_constructor(): void
     {
         $logger = $this->createMock(LoggerInterface::class);

diff --git a/tests/Unit/API/Trace/Propagation/TraceContextPropagatorTest.php b/tests/Unit/API/Trace/Propagation/TraceContextPropagatorTest.php
@@ -4,7 +4,7 @@
 
 namespace OpenTelemetry\Tests\Unit\API\Trace\Propagation;
 
-use OpenTelemetry\API\LoggerHolder;
+use OpenTelemetry\API\Behavior\Internal\Logging;
 use OpenTelemetry\API\Trace\Propagation\TraceContextPropagator;
 use OpenTelemetry\API\Trace\SpanContext;
 use OpenTelemetry\API\Trace\SpanContextInterface;
@@ -16,7 +16,6 @@
 use OpenTelemetry\Context\ContextInterface;
 use OpenTelemetry\SDK\Trace\Span;
 use PHPUnit\Framework\TestCase;
-use Psr\Log\NullLogger;
 
 /**
  * @covers \OpenTelemetry\API\Trace\Propagation\TraceContextPropagator
@@ -35,7 +34,7 @@ class TraceContextPropagatorTest extends TestCase
 
     protected function setUp(): void
     {
-        LoggerHolder::set(new NullLogger());
+        Logging::disable();
         $this->traceContextPropagator = TraceContextPropagator::getInstance();
         $this->traceState = (new TraceState())->with('bar', 'baz')->with('foo', 'bar');
     }

diff --git a/tests/Unit/Contrib/Otlp/LogsExporterTest.php b/tests/Unit/Contrib/Otlp/LogsExporterTest.php
@@ -4,6 +4,7 @@
 
 namespace OpenTelemetry\Tests\Unit\Contrib\Otlp;
 
+use OpenTelemetry\API\Behavior\Internal\Logging;
 use OpenTelemetry\Contrib\Otlp\LogsExporter;
 use OpenTelemetry\SDK\Common\Export\TransportInterface;
 use OpenTelemetry\SDK\Common\Future\CompletedFuture;
@@ -24,6 +25,7 @@ public function setUp(): void
         $this->transport = $this->createMock(TransportInterface::class);
         $this->transport->method('contentType')->willReturn('application/x-protobuf');
         $this->exporter = new LogsExporter($this->transport);
+        Logging::disable();
     }
 
     public function test_export_with_transport_failure(): void

diff --git a/tests/Unit/Contrib/Otlp/SpanExporterTest.php b/tests/Unit/Contrib/Otlp/SpanExporterTest.php
@@ -5,6 +5,7 @@
 namespace OpenTelemetry\Tests\Unit\Contrib\Otlp;
 
 use function fseek;
+use OpenTelemetry\API\Behavior\Internal\Logging;
 use OpenTelemetry\API\Trace\SpanContext;
 use OpenTelemetry\Contrib\Otlp\SpanExporter;
 use OpenTelemetry\SDK\Common\Export\Stream\StreamTransport;
@@ -26,6 +27,7 @@ class SpanExporterTest extends TestCase
 
     public function setUp(): void
     {
+        Logging::disable();
         $this->transport = $this->createMock(TransportInterface::class);
         $this->transport->method('contentType')->willReturn('application/x-protobuf');
         $this->exporter = new SpanExporter($this->transport);

diff --git a/tests/Unit/SDK/Metrics/MeterProviderFactoryTest.php b/tests/Unit/SDK/Metrics/MeterProviderFactoryTest.php
@@ -5,13 +5,12 @@
 namespace OpenTelemetry\Tests\Unit\SDK\Metrics;
 
 use AssertWell\PHPUnitGlobalState\EnvironmentVariables;
-use OpenTelemetry\API\LoggerHolder;
+use OpenTelemetry\API\Behavior\Internal\Logging;
 use OpenTelemetry\API\Metrics\MeterInterface;
 use OpenTelemetry\SDK\Common\Configuration\KnownValues;
 use OpenTelemetry\SDK\Common\Configuration\Variables;
 use OpenTelemetry\SDK\Metrics\MeterProviderFactory;
 use PHPUnit\Framework\TestCase;
-use Psr\Log\NullLogger;
 
 /**
  * @covers \OpenTelemetry\SDK\Metrics\MeterProviderFactory
@@ -22,7 +21,7 @@ class MeterProviderFactoryTest extends TestCase
 
     public function setUp(): void
     {
-        LoggerHolder::set(new NullLogger());
+        Logging::disable();
     }
 
     public function tearDown(): void

diff --git a/tests/Unit/SDK/Metrics/Stream/MetricStreamTest.php b/tests/Unit/SDK/Metrics/Stream/MetricStreamTest.php
@@ -6,6 +6,7 @@
 
 use function current;
 use function extension_loaded;
+use OpenTelemetry\API\Behavior\Internal\Logging;
 use OpenTelemetry\Context\Context;
 use OpenTelemetry\SDK\Common\Attribute\Attributes;
 use OpenTelemetry\SDK\Metrics\Aggregation\SumAggregation;
@@ -44,6 +45,11 @@
  */
 final class MetricStreamTest extends TestCase
 {
+    public function setUp(): void
+    {
+        Logging::disable();
+    }
+
     public function test_asynchronous_single_data_point(): void
     {
         $s = new AsynchronousMetricStream(new SumAggregation(), 3);

diff --git a/tests/Unit/SDK/Propagation/PropagatorFactoryTest.php b/tests/Unit/SDK/Propagation/PropagatorFactoryTest.php
@@ -6,6 +6,7 @@
 
 use AssertWell\PHPUnitGlobalState\EnvironmentVariables;
 use OpenTelemetry\API\Baggage\Propagation\BaggagePropagator;
+use OpenTelemetry\API\Behavior\Internal\Logging;
 use OpenTelemetry\API\LoggerHolder;
 use OpenTelemetry\API\Trace\Propagation\TraceContextPropagator;
 use OpenTelemetry\Context\Propagation\MultiTextMapPropagator;
@@ -18,7 +19,6 @@
 use OpenTelemetry\SDK\Common\Configuration\Variables;
 use OpenTelemetry\SDK\Propagation\PropagatorFactory;
 use PHPUnit\Framework\TestCase;
-use Psr\Log\NullLogger;
 
 /**
  * @covers \OpenTelemetry\SDK\Propagation\PropagatorFactory
@@ -29,7 +29,8 @@ class PropagatorFactoryTest extends TestCase
 
     public function setUp(): void
     {
-        LoggerHolder::set(new NullLogger());
+        LoggerHolder::disable();
+        Logging::disable();
     }
 
     public function tearDown(): void

diff --git a/tests/Unit/SDK/Resource/ResourceInfoTest.php b/tests/Unit/SDK/Resource/ResourceInfoTest.php
@@ -7,14 +7,13 @@
 use AssertWell\PHPUnitGlobalState\EnvironmentVariables;
 use Composer\InstalledVersions;
 use Generator;
-use OpenTelemetry\API\LoggerHolder;
+use OpenTelemetry\API\Behavior\Internal\Logging;
 use OpenTelemetry\SDK\Common\Attribute\Attributes;
 use OpenTelemetry\SDK\Resource\Detectors;
 use OpenTelemetry\SDK\Resource\ResourceInfo;
 use OpenTelemetry\SDK\Resource\ResourceInfoFactory;
 use OpenTelemetry\SemConv\ResourceAttributes;
 use PHPUnit\Framework\TestCase;
-use Psr\Log\NullLogger;
 
 /**
  * @covers \OpenTelemetry\SDK\Resource\ResourceInfo
@@ -25,7 +24,7 @@ class ResourceInfoTest extends TestCase
 
     public function setUp(): void
     {
-        LoggerHolder::set(new NullLogger());
+        Logging::disable();
     }
 
     public function tearDown(): void

diff --git a/tests/Unit/SDK/SdkAutoloaderTest.php b/tests/Unit/SDK/SdkAutoloaderTest.php
@@ -5,8 +5,8 @@
 namespace OpenTelemetry\Tests\Unit\SDK;
 
 use AssertWell\PHPUnitGlobalState\EnvironmentVariables;
+use OpenTelemetry\API\Behavior\Internal\Logging;
 use OpenTelemetry\API\Globals;
-use OpenTelemetry\API\LoggerHolder;
 use OpenTelemetry\API\Logs\NoopEventLoggerProvider;
 use OpenTelemetry\API\Logs\NoopLoggerProvider;
 use OpenTelemetry\API\Metrics\Noop\NoopMeterProvider;
@@ -15,7 +15,6 @@
 use OpenTelemetry\SDK\Common\Configuration\Variables;
 use OpenTelemetry\SDK\SdkAutoloader;
 use PHPUnit\Framework\TestCase;
-use Psr\Log\NullLogger;
 
 /**
  * @covers \OpenTelemetry\SDK\SdkAutoloader
@@ -26,7 +25,7 @@ class SdkAutoloaderTest extends TestCase
 
     public function setUp(): void
     {
-        LoggerHolder::set(new NullLogger());
+        Logging::disable();
         Globals::reset();
     }
 

diff --git a/tests/Unit/SDK/Trace/SpanExporter/AbstractExporterTestCase.php b/tests/Unit/SDK/Trace/SpanExporter/AbstractExporterTestCase.php
@@ -6,14 +6,13 @@
 
 use Mockery;
 use Mockery\Adapter\Phpunit\MockeryTestCase;
-use OpenTelemetry\API\LoggerHolder;
+use OpenTelemetry\API\Behavior\Internal\Logging;
 use OpenTelemetry\SDK\Common\Export\TransportInterface;
 use OpenTelemetry\SDK\Common\Future\CompletedFuture;
 use OpenTelemetry\SDK\Common\Future\ErrorFuture;
 use OpenTelemetry\SDK\Common\Future\FutureInterface;
 use OpenTelemetry\SDK\Trace\SpanExporterInterface;
 use OpenTelemetry\Tests\Unit\SDK\Util\SpanData;
-use Psr\Log\NullLogger;
 
 /**
  * @psalm-suppress UndefinedInterfaceMethod
@@ -25,7 +24,7 @@ abstract class AbstractExporterTestCase extends MockeryTestCase
 
     public function setUp(): void
     {
-        LoggerHolder::set(new NullLogger());
+        Logging::disable();
         $this->future = Mockery::mock(FutureInterface::class);
         $this->future->allows([
             'map' => $this->future,

diff --git a/vendor-bin/infection/composer.json b/vendor-bin/infection/composer.json
@@ -0,0 +1,10 @@
+{
+    "require-dev": {
+        "infection/infection": "^0.28.1"
+    },
+    "config": {
+        "allow-plugins": {
+            "infection/extension-installer": true
+        }
+    }
+}