Merging final GDK for Unreal 0.13.1 release

Release 0.13.1

.buildkite/premerge.steps.yaml

@@ -1,5 +1,5 @@
 ---
-ci_version: &ci_version "1.1"
+ci_version: &ci_version "1.2"
 # This is designed to trap and retry failures because agent lost
 # connection. Agent exits with -1 in this case.
 agent_transients: &agent_transients
@@ -32,10 +32,10 @@ steps:
       message: "gdk-4.26 ${BUILDKITE_MESSAGE}"
       env: 
         BUILD_TYPE: "GDK"
-        GDK_BRANCH: "${BUILDKITE_BRANCH}"
-        ENGINE_BRANCH: "${ENGINE_BRANCH_426:-4.26-SpatialOSUnrealGDK-0.13.0}" # NOTE: temp fix for new ci release interop issues
+        GDK_BRANCH: "main_branch::${BUILDKITE_BRANCH}"
+        ENGINE_BRANCH: "${ENGINE_BRANCH_426:-match_branch_name_pref_engine::4.26-SpatialOSUnrealGDK-0.13.1}"
         ENGINE_MAJOR: "4.26"
-        PROJECT_BRANCH: "${PROJECT_BRANCH:-0.13.0}" # NOTE: temp fix for new ci release interop issues
+        PROJECT_BRANCH: "${PROJECT_BRANCH:-match_branch_name_pref::0.13.1}"
         USE_FASTBUILD: "True"
         IS_BUILDKITE_BUILD: "True"
         BUILD_ANDROID: "False"
@@ -51,10 +51,10 @@ steps:
       message: "gdk-4.25 ${BUILDKITE_MESSAGE}"
       env: 
         BUILD_TYPE: "GDK" # GDK or ENGINE
-        GDK_BRANCH: "${BUILDKITE_BRANCH}"
-        ENGINE_BRANCH: "${ENGINE_BRANCH_425:-4.25-SpatialOSUnrealGDK-0.13.0}" # NOTE: temp fix for new ci release interop issues
+        GDK_BRANCH: "main_branch::${BUILDKITE_BRANCH}"
+        ENGINE_BRANCH: "${ENGINE_BRANCH_425:-match_branch_name_pref_engine::4.25-SpatialOSUnrealGDK-0.13.1}"
         ENGINE_MAJOR: "4.25"
-        PROJECT_BRANCH: "${PROJECT_BRANCH:-0.13.0}" # NOTE: temp fix for new ci release interop issues
+        PROJECT_BRANCH: "${PROJECT_BRANCH:-match_branch_name_pref::0.13.1}"
         USE_FASTBUILD: "True"
         IS_BUILDKITE_BUILD: "True"
         BUILD_ANDROID: "False"

CHANGELOG.md

@@ -9,6 +9,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [`x.y.z`] - Unreleased
 
+## [`0.13.1`] - 2021-06-02
+### Breaking changes:
+- Event tracing has been optimised to reduce overhead when tracing events in general and in particular when events are not sampled. The tracing API has been modified to accommodate these improvements. You will have to modify your project if you use the API.
+
+### Features:
+- Added `SpatialExecServerCmd` console command with one command `StartInsights` to dynamically enable insights capturing.
+  - Format: SpatialExecServerCmd < server > < command > < args >
+  - Example usage: "SpatialExecServerCmd local StartInsights -trace=CustomChannel -tracefile=MyNewTrace"
+- Visual Logger now supports multi-worker environments.
+- The Unreal GDK has been updated to run against SpatialOS 15.1.0, older versions of SpatialOS will no longer work with the Unreal GDK.
+- Event tracing rotating log support (via `bEnableEventTracingRotatingLogs`, `EventTracingRotatingLogsMaxFileSizeBytes` and `EventTracingRotatingLogsMaxFileCount`). 
+- Event tracing filter support (configured via `UEventTracingSamplingSettings`).
+
 ## [`0.13.0`] - 2021-05-17
 ### Breaking changes:
 - Removed support for Unreal Engine 4.24.
@@ -64,7 +77,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Fixed a bug where on initial replication, actors with replicated TArrays would not have the array cleared if the local state was not empty.
 - Fixed an issue with replicating references to stably named dynamically added subobjects of dynamic actors.
 - Fixed an issue during client logout where a client's corresponding Actors were not cleaned up correctly.
-- Reverted a fix relating to the `dbghelp` file that previously caused the Editor to crash when loading the Session Front End. Our fix is no longer necessary, as Epic have fixed the issue and we've adopted their fix.
 - Fixed issue with `SpatialDebugger` crashing when client travelling.
 
 ## [`0.12.0`] - 2021-02-01

RequireSetup

@@ -1,4 +1,4 @@
 Increment the below number whenever it is required to run Setup.bat as part of a new commit.
 Our git hooks will detect this file has been updated and automatically run Setup.bat on pull.
 
-85
+86

SetupIncTraceLibs.bat

@@ -18,8 +18,8 @@ call :MarkStartOfBlock "Create folders"
 call :MarkEndOfBlock "Create folders"
 
 call :MarkStartOfBlock "Retrieve dependencies"	
-	spatial package retrieve internal        trace-dynamic-x86_64-vc141_md-win32        %PINNED_CORE_SDK_VERSION% 	"%CORE_SDK_DIR%\trace_lib\trace-win32.zip"
-	spatial package retrieve internal        trace-dynamic-x86_64-clang1000-linux       %PINNED_CORE_SDK_VERSION% 	"%CORE_SDK_DIR%\trace_lib\trace-linux.zip"
+	spatial package retrieve internal        trace-dynamic-x86_64-vc141_md-win32        15.0.1        "%CORE_SDK_DIR%\trace_lib\trace-win32.zip"
+	spatial package retrieve internal        trace-dynamic-x86_64-clang1000-linux       15.0.1        "%CORE_SDK_DIR%\trace_lib\trace-linux.zip"
 call :MarkEndOfBlock "Retrieve dependencies"
 
 REM There is a race condition between retrieve and unzip, add version call to stall briefly

SpatialGDK/Extras/core-sdk.version

@@ -1,2 +1,2 @@
-15.0.1
+15.1.0
 // If changing version, update SpatialGDK/Source/SpatialGDK/Public/Utils/WorkerVersionCheck.h too

SpatialGDK/Extras/schema/debug_metrics.schema

@@ -8,9 +8,16 @@ type ModifySettingPayload {
     float setting_value = 2;
 }
 
+type ExecServerCommandPayload {
+    string server_worker = 1;
+    string command = 2;
+    string args = 3;
+}
+
 component DebugMetrics {
     id = 9984;
     command Void start_rpc_metrics(Void);
     command Void stop_rpc_metrics(Void);
     command Void modify_spatial_settings(ModifySettingPayload);
+    command Void exec_server_command(ExecServerCommandPayload);
 }

SpatialGDK/Extras/schema/server_worker.schema

@@ -2,6 +2,7 @@
 package unreal;
 
 import "unreal/gdk/core_types.schema";
+import "unreal/gdk/debug_metrics.schema";
 import "unreal/gdk/spawner.schema";
 import "unreal/generated/rpc_endpoints.schema";
 import "improbable/standard_library.schema";
@@ -22,15 +23,16 @@ component ServerWorker {
     bool ready_to_begin_play = 2;
     EntityId server_system_entity_id = 3;
     command ForwardSpawnPlayerResponse forward_spawn_player(ForwardSpawnPlayerRequest);
+    command Void exec_server_command(ExecServerCommandPayload);
 }
 
 component_set ServerWorkerAuthComponentSet {
   id = 9908;
   components = [
-	improbable.Position,
-	improbable.Metadata,
-	improbable.Interest,
-	unreal.ServerWorker,
-	unreal.generated.UnrealCrossServerSenderRPCs
+    improbable.Position,
+    improbable.Metadata,
+    improbable.Interest,
+    unreal.ServerWorker,
+    unreal.generated.UnrealCrossServerSenderRPCs
   ];
 }

SpatialGDK/Source/SpatialGDK/Private/EngineClasses/SpatialActorChannel.cpp

@@ -20,7 +20,6 @@
 #include "EngineStats.h"
 #include "Interop/ActorSystem.h"
 #include "Interop/Connection/SpatialEventTracer.h"
-#include "Interop/Connection/SpatialTraceEventBuilder.h"
 #include "Interop/GlobalStateManager.h"
 #include "Interop/SpatialReceiver.h"
 #include "Interop/SpatialSender.h"

SpatialGDK/Source/SpatialGDK/Private/EngineClasses/SpatialGameInstance.cpp

@@ -244,9 +244,9 @@ void USpatialGameInstance::Init()
 void USpatialGameInstance::HandleOnConnected(USpatialNetDriver& NetDriver)
 {
 	UE_LOG(LogSpatialGameInstance, Log, TEXT("Successfully connected to SpatialOS"));
-	SpatialWorkerId = SpatialConnectionManager->GetWorkerConnection()->GetWorkerId();
+	SetSpatialWorkerId(SpatialConnectionManager->GetWorkerConnection()->GetWorkerId());
 #if TRACE_LIB_ACTIVE
-	SpatialLatencyTracer->SetWorkerId(SpatialWorkerId);
+	SpatialLatencyTracer->SetWorkerId(GetSpatialWorkerId());
 
 	USpatialWorkerConnection* WorkerConnection = SpatialConnectionManager->GetWorkerConnection();
 	WorkerConnection->OnEnqueueMessage.AddUObject(SpatialLatencyTracer, &USpatialLatencyTracer::OnEnqueueMessage);

SpatialGDK/Source/SpatialGDK/Private/EngineClasses/SpatialNetDriver.cpp

@@ -3113,22 +3113,14 @@ int64 USpatialNetDriver::GetClientID() const
 	return SpatialConstants::INVALID_ENTITY_ID;
 }
 
-int64 USpatialNetDriver::GetActorEntityId(AActor& Actor)
+int64 USpatialNetDriver::GetActorEntityId(const AActor& Actor) const
 {
 	if (PackageMap == nullptr)
 	{
 		return SpatialConstants::INVALID_ENTITY_ID;
 	}
 
-	int64 EntityId = PackageMap->GetEntityIdFromObject(&Actor);
-	if (EntityId == SpatialConstants::INVALID_ENTITY_ID)
-	{
-		if (IsServer() && Actor.GetIsReplicated() && (Actor.Role == ROLE_Authority))
-		{
-			EntityId = PackageMap->AllocateEntityIdAndResolveActor(&Actor);
-		}
-	}
-	return EntityId;
+	return PackageMap->GetEntityIdFromObject(&Actor);
 }
 
 bool USpatialNetDriver::HasTimedOut(const float Interval, uint64& TimeStamp)

SpatialGDK/Source/SpatialGDK/Private/EngineClasses/SpatialNetDriverRPC.cpp

@@ -4,7 +4,6 @@
 #include "EngineClasses/SpatialNetBitReader.h"
 #include "EngineClasses/SpatialNetDriver.h"
 #include "EngineClasses/SpatialPackageMapClient.h"
-#include "Interop/Connection/SpatialTraceEventBuilder.h"
 #include "Interop/Connection/SpatialWorkerConnection.h"
 #include "Utils/RPCRingBuffer.h"
 #include "Utils/RepLayoutUtils.h"
@@ -19,10 +18,12 @@ void FRPCMetaData::ComputeSpanId(SpatialGDK::SpatialEventTracer& Tracer, Spatial
 {
 	TArray<FSpatialGDKSpanId> ComponentUpdateSpans = Tracer.GetAndConsumeSpansForComponent(EntityComponent);
 
+	const Trace_SpanIdType* Causes = reinterpret_cast<const Trace_SpanIdType*>(ComponentUpdateSpans.GetData());
 	SpanId = Tracer.TraceEvent(
-		FSpatialTraceEventBuilder::CreateReceiveRPC(EventTraceUniqueId::GenerateForNamedRPC(EntityComponent.EntityId, RPCName, RPCId)),
-		/* Causes */ reinterpret_cast<const Trace_SpanIdType*>(ComponentUpdateSpans.GetData()),
-		/* NumCauses */ ComponentUpdateSpans.Num());
+		RECEIVE_RPC_EVENT_NAME, "", Causes, ComponentUpdateSpans.Num(),
+		[this, EntityComponent, RPCId](FSpatialTraceEventDataBuilder& EventBuilder) {
+			EventBuilder.AddLinearTraceId(EventTraceUniqueId::GenerateForNamedRPC(EntityComponent.EntityId, RPCName, RPCId));
+		});
 }
 
 void FRPCPayload::ReadFromSchema(const Schema_Object* RPCObject)
@@ -43,9 +44,11 @@ void FSpatialNetDriverRPC::OnRPCSent(SpatialGDK::SpatialEventTracer& EventTracer
 									 Worker_EntityId EntityId, Worker_ComponentId ComponentId, uint64 RPCId,
 									 const FSpatialGDKSpanId& SpanId)
 {
-	const EventTraceUniqueId LinearTraceId = EventTraceUniqueId::GenerateForNamedRPC(EntityId, Name, RPCId);
-	const FSpatialGDKSpanId NewSpanId = EventTracer.TraceEvent(FSpatialTraceEventBuilder::CreateSendRPC(LinearTraceId),
-															   /* Causes */ SpanId.GetConstId(), /* NumCauses */ 1);
+	FSpatialGDKSpanId NewSpanId =
+		EventTracer.TraceEvent(SEND_RPC_EVENT_NAME, "", SpanId.GetConstId(), /* NumCauses */ 1,
+							   [EntityId, Name, RPCId](FSpatialTraceEventDataBuilder& EventBuilder) {
+								   EventBuilder.AddLinearTraceId(EventTraceUniqueId::GenerateForNamedRPC(EntityId, Name, RPCId));
+							   });
 
 	if (OutUpdates.Num() == 0 || OutUpdates.Last().EntityId != EntityId || OutUpdates.Last().Update.component_id != ComponentId)
 	{
@@ -191,9 +194,13 @@ struct FSpatialNetDriverRPC::RAIIUpdateContext : FStackOnly
 			FSpatialGDKSpanId SpanId;
 			if (RPCSystem.EventTracer != nullptr)
 			{
-				SpanId = RPCSystem.EventTracer->TraceEvent(
-					FSpatialTraceEventBuilder::CreateMergeSendRPCs(Update.EntityId, Update.Update.component_id),
-					/* Causes */ Update.Spans.GetData()->GetConstId(), /* NumCauses */ Update.Spans.Num());
+				TArray<FSpatialGDKSpanId>& Causes = Update.Spans;
+
+				SpanId = RPCSystem.EventTracer->TraceEvent(MERGE_SEND_RPCS_EVENT_NAME, "", Causes.GetData()->GetConstId(), Causes.Num(),
+														   [Update](FSpatialTraceEventDataBuilder& EventBuilder) {
+															   EventBuilder.AddEntityId(Update.EntityId);
+															   EventBuilder.AddComponentId(Update.Update.component_id);
+														   });
 			}
 			RPCSystem.NetDriver.Connection->SendComponentUpdate(Update.EntityId, &Update.Update, SpanId);
 		}
@@ -376,8 +383,11 @@ bool FSpatialNetDriverRPC::ApplyRPC(Worker_EntityId EntityId, SpatialGDK::Receiv
 	const bool bUseEventTracer = EventTracer != nullptr;
 	if (bUseEventTracer)
 	{
-		FSpatialGDKSpanId SpanId = EventTracer->TraceEvent(FSpatialTraceEventBuilder::CreateApplyRPC(TargetObject, Function),
-														   /* Causes */ MetaData.SpanId.GetConstId(), /* NumCauses */ 1);
+		FSpatialGDKSpanId SpanId = EventTracer->TraceEvent(APPLY_RPC_EVENT_NAME, "", MetaData.SpanId.GetConstId(), /* NumCauses */ 1,
+														   [TargetObject, Function](FSpatialTraceEventDataBuilder& EventBuilder) {
+															   EventBuilder.AddObject(TargetObject);
+															   EventBuilder.AddFunction(Function);
+														   });
 		EventTracer->AddToStack(SpanId);
 	}
 

SpatialGDK/Source/SpatialGDK/Private/EngineClasses/SpatialPackageMapClient.cpp

@@ -617,7 +617,7 @@ void FSpatialNetGUIDCache::RemoveEntityNetGUID(Worker_EntityId EntityId)
 
 	if (UnrealMetadata->NativeClass.IsStale())
 	{
-		UE_LOG(LogSpatialPackageMap, Warning, TEXT("Attempting to remove stale object from package map - %s"), *UnrealMetadata->ClassPath);
+		UE_LOG(LogSpatialPackageMap, Log, TEXT("Attempting to remove stale object from package map - %s"), *UnrealMetadata->ClassPath);
 	}
 	else
 	{
@@ -695,7 +695,7 @@ void FSpatialNetGUIDCache::RemoveSubobjectNetGUID(const FUnrealObjectRef& Subobj
 
 	if (UnrealMetadata->NativeClass.IsStale())
 	{
-		UE_LOG(LogSpatialPackageMap, Warning, TEXT("Attempting to remove stale subobject from package map - %s"),
+		UE_LOG(LogSpatialPackageMap, Log, TEXT("Attempting to remove stale subobject from package map - %s"),
 			   *UnrealMetadata->ClassPath);
 	}
 	else