Proposed changes to model compilation telemetry

onnxruntime/core/platform/telemetry.cc

@@ -81,8 +81,34 @@ void Telemetry::LogSessionCreation(uint32_t session_id, int64_t ir_version, cons
   ORT_UNUSED_PARAMETER(captureState);
 }
 
-void Telemetry::LogCompileModel(uint32_t session_id) const {
+void Telemetry::LogCompileModelStart(uint32_t session_id,
+                                     const std::string& input_source,
+                                     const std::string& output_target,
+                                     uint32_t flags,
+                                     int graph_optimization_level,
+                                     bool embed_ep_context,
+                                     bool has_external_initializers_file,
+                                     const std::vector<std::string>& execution_provider_ids) const {
   ORT_UNUSED_PARAMETER(session_id);
+  ORT_UNUSED_PARAMETER(input_source);
+  ORT_UNUSED_PARAMETER(output_target);
+  ORT_UNUSED_PARAMETER(flags);
+  ORT_UNUSED_PARAMETER(graph_optimization_level);
+  ORT_UNUSED_PARAMETER(embed_ep_context);
+  ORT_UNUSED_PARAMETER(has_external_initializers_file);
+  ORT_UNUSED_PARAMETER(execution_provider_ids);
+}
+
+void Telemetry::LogCompileModelComplete(uint32_t session_id,
+                                        bool success,
+                                        uint32_t error_code,
+                                        uint32_t error_category,
+                                        const std::string& error_message) const {
+  ORT_UNUSED_PARAMETER(session_id);
+  ORT_UNUSED_PARAMETER(success);
+  ORT_UNUSED_PARAMETER(error_code);
+  ORT_UNUSED_PARAMETER(error_category);
+  ORT_UNUSED_PARAMETER(error_message);
 }
 
 void Telemetry::LogRuntimeError(uint32_t session_id, const common::Status& status, const char* file,

onnxruntime/core/platform/telemetry.h

@@ -66,7 +66,20 @@ class Telemetry {
                                   const std::string& loadedFrom, const std::vector<std::string>& execution_provider_ids,
                                   bool use_fp16, bool captureState) const;
 
-  virtual void LogCompileModel(uint32_t session_id) const;
+  virtual void LogCompileModelStart(uint32_t session_id,
+                                    const std::string& input_source,
+                                    const std::string& output_target,
+                                    uint32_t flags,
+                                    int graph_optimization_level,
+                                    bool embed_ep_context,
+                                    bool has_external_initializers_file,
+                                    const std::vector<std::string>& execution_provider_ids) const;
+
+  virtual void LogCompileModelComplete(uint32_t session_id,
+                                       bool success,
+                                       uint32_t error_code,
+                                       uint32_t error_category,
+                                       const std::string& error_message) const;
 
   virtual void LogRuntimeError(uint32_t session_id, const common::Status& status, const char* file,
                                const char* function, uint32_t line) const;

onnxruntime/core/platform/windows/telemetry.cc

@@ -334,18 +334,67 @@ void WindowsTelemetry::LogSessionCreation(uint32_t session_id, int64_t ir_versio
   }
 }
 
-void WindowsTelemetry::LogCompileModel(uint32_t session_id) const {
+void WindowsTelemetry::LogCompileModelStart(uint32_t session_id,
+                                            const std::string& input_source,
+                                            const std::string& output_target,
+                                            uint32_t flags,
+                                            int graph_optimization_level,
+                                            bool embed_ep_context,
+                                            bool has_external_initializers_file,
+                                            const std::vector<std::string>& execution_provider_ids) const {
   if (global_register_count_ == 0 || enabled_ == false)
     return;
 
+  std::string execution_provider_string;
+  bool first = true;
+  for (const auto& ep_id : execution_provider_ids) {
+    if (first) {
+      first = false;
+    } else {
+      execution_provider_string += ',';
+    }
+    execution_provider_string += ep_id;
+  }
+
   TraceLoggingWrite(telemetry_provider_handle,
-                    "CompileModel",
+                    "CompileModelStart",
                     TraceLoggingBool(true, "UTCReplace_AppSessionGuid"),
                     TelemetryPrivacyDataTag(PDT_ProductAndServiceUsage),
                     TraceLoggingKeyword(MICROSOFT_KEYWORD_MEASURES),
+                    TraceLoggingLevel(WINEVENT_LEVEL_INFO),
                     // Telemetry info
                     TraceLoggingUInt8(0, "schemaVersion"),
-                    TraceLoggingUInt32(session_id, "sessionId"));
+                    TraceLoggingUInt32(session_id, "sessionId"),
+                    TraceLoggingString(input_source.c_str(), "inputSource"),
+                    TraceLoggingString(output_target.c_str(), "outputTarget"),
+                    TraceLoggingUInt32(flags, "flags"),
+                    TraceLoggingInt32(graph_optimization_level, "graphOptimizationLevel"),
+                    TraceLoggingBool(embed_ep_context, "embedEpContext"),
+                    TraceLoggingBool(has_external_initializers_file, "hasExternalInitializersFile"),
+                    TraceLoggingString(execution_provider_string.c_str(), "executionProviderIds"));
+}
+
+void WindowsTelemetry::LogCompileModelComplete(uint32_t session_id,
+                                               bool success,
+                                               uint32_t error_code,
+                                               uint32_t error_category,
+                                               const std::string& error_message) const {
+  if (global_register_count_ == 0 || enabled_ == false)
+    return;
+
+  TraceLoggingWrite(telemetry_provider_handle,
+                    "CompileModelComplete",
+                    TraceLoggingBool(true, "UTCReplace_AppSessionGuid"),
+                    TelemetryPrivacyDataTag(PDT_ProductAndServiceUsage),
+                    TraceLoggingKeyword(MICROSOFT_KEYWORD_MEASURES),
+                    TraceLoggingLevel(WINEVENT_LEVEL_INFO),
+                    // Telemetry info
+                    TraceLoggingUInt8(0, "schemaVersion"),
+                    TraceLoggingUInt32(session_id, "sessionId"),
+                    TraceLoggingBool(success, "success"),
+                    TraceLoggingUInt32(error_code, "errorCode"),
+                    TraceLoggingUInt32(error_category, "errorCategory"),
+                    TraceLoggingString(error_message.c_str(), "errorMessage"));
 }
 
 void WindowsTelemetry::LogRuntimeError(uint32_t session_id, const common::Status& status, const char* file,

onnxruntime/core/platform/windows/telemetry.h

@@ -59,7 +59,20 @@ class WindowsTelemetry : public Telemetry {
                           const std::string& loadedFrom, const std::vector<std::string>& execution_provider_ids,
                           bool use_fp16, bool captureState) const override;
 
-  void LogCompileModel(uint32_t session_id) const override;
+  void LogCompileModelStart(uint32_t session_id,
+                            const std::string& input_source,
+                            const std::string& output_target,
+                            uint32_t flags,
+                            int graph_optimization_level,
+                            bool embed_ep_context,
+                            bool has_external_initializers_file,
+                            const std::vector<std::string>& execution_provider_ids) const override;
+
+  void LogCompileModelComplete(uint32_t session_id,
+                               bool success,
+                               uint32_t error_code,
+                               uint32_t error_category,
+                               const std::string& error_message) const override;
 
   void LogRuntimeError(uint32_t session_id, const common::Status& status, const char* file,
                        const char* function, uint32_t line) const override;

onnxruntime/core/session/model_compilation_options.cc

@@ -293,5 +293,53 @@ Status ModelCompilationOptions::Check() const {
   return Status::OK();
 }
 
+std::string ModelCompilationOptions::GetInputSourceForTelemetry() const {
+  return InputModelComesFromFile() ? "file" : "buffer";
+}
+
+std::string ModelCompilationOptions::GetOutputTargetForTelemetry() const {
+  const epctx::ModelGenOptions& options = session_options_.value.ep_context_gen_options;
+
+  if (options.TryGetOutputModelPath() != nullptr) {
+    return "file";
+  }
+  if (options.TryGetOutputModelBuffer() != nullptr) {
+    return "buffer";
+  }
+  if (options.TryGetOutputModelWriteFunc() != nullptr) {
+    return "callback";
+  }
+
+  // Default: output path will be derived from input path
+  return "file";
+}
+
+uint32_t ModelCompilationOptions::GetFlagsForTelemetry() const {
+  const epctx::ModelGenOptions& options = session_options_.value.ep_context_gen_options;
+  uint32_t flags = OrtCompileApiFlags_NONE;
+
+  if (options.error_if_output_file_exists) {
+    flags |= OrtCompileApiFlags_ERROR_IF_OUTPUT_FILE_EXISTS;
+  }
+  if (options.action_if_no_compiled_nodes == epctx::ModelGenOptions::ActionIfNoCompiledNodes::kReturnError) {
+    flags |= OrtCompileApiFlags_ERROR_IF_NO_NODES_COMPILED;
+  }
+
+  return flags;
+}
+
+int ModelCompilationOptions::GetGraphOptimizationLevelForTelemetry() const {
+  return static_cast<int>(session_options_.value.graph_optimization_level);
+}
+
+bool ModelCompilationOptions::GetEmbedEpContextForTelemetry() const {
+  return session_options_.value.ep_context_gen_options.embed_ep_context_in_model;
+}
+
+bool ModelCompilationOptions::HasExternalInitializersFileForTelemetry() const {
+  return session_options_.value.ep_context_gen_options.TryGetExternalInitializerFileInfo() != nullptr ||
+         session_options_.value.ep_context_gen_options.TryGetInitializerHandler() != nullptr;
+}
+
 }  // namespace onnxruntime
 #endif  // !defined(ORT_MINIMAL_BUILD)

onnxruntime/core/session/model_compilation_options.h

@@ -159,6 +159,44 @@ class ModelCompilationOptions {
   /// <returns>An error status if the compilation options are invalid</returns>
   Status Check() const;
 
+  // Telemetry helper methods
+
+  /// <summary>
+  /// Returns a string describing the input source type: "file" or "buffer".
+  /// </summary>
+  /// <returns>"file" or "buffer"</returns>
+  std::string GetInputSourceForTelemetry() const;
+
+  /// <summary>
+  /// Returns a string describing the output target type: "file", "buffer", or "callback".
+  /// </summary>
+  /// <returns>"file", "buffer", or "callback"</returns>
+  std::string GetOutputTargetForTelemetry() const;
+
+  /// <summary>
+  /// Returns the flags value for telemetry.
+  /// </summary>
+  /// <returns>The flags value</returns>
+  uint32_t GetFlagsForTelemetry() const;
+
+  /// <summary>
+  /// Returns the graph optimization level for telemetry.
+  /// </summary>
+  /// <returns>The graph optimization level as an integer</returns>
+  int GetGraphOptimizationLevelForTelemetry() const;
+
+  /// <summary>
+  /// Returns whether EP context embedding is enabled.
+  /// </summary>
+  /// <returns>True if EP context is embedded in model</returns>
+  bool GetEmbedEpContextForTelemetry() const;
+
+  /// <summary>
+  /// Returns whether external initializers file is configured.
+  /// </summary>
+  /// <returns>True if external initializers file is configured</returns>
+  bool HasExternalInitializersFileForTelemetry() const;
+
  private:
   void ResetInputModelSettings();
 

onnxruntime/core/session/utils.cc

@@ -9,6 +9,8 @@
 #include "core/framework/error_code_helper.h"
 #include "core/framework/execution_provider.h"
 #include "core/framework/provider_options.h"
+#include "core/platform/env.h"
+#include "core/platform/telemetry.h"
 #include "core/session/abi_session_options_impl.h"
 #include "core/session/environment.h"
 #include "core/session/inference_session.h"
@@ -388,25 +390,56 @@ namespace onnxruntime {
 Status CompileModel(const Environment& env, const ModelCompilationOptions& model_compile_options) {
   ORT_RETURN_IF_ERROR(model_compile_options.Check());
 
+  const Telemetry& telemetry_provider = Env::Default().GetTelemetryProvider();
+
   std::unique_ptr<onnxruntime::InferenceSession> session;
   const OrtSessionOptions* session_options = &model_compile_options.GetSessionOptions();
 
+  Status status;
+
   if (model_compile_options.InputModelComesFromFile()) {
     const std::filesystem::path& input_model_path = model_compile_options.GetInputModelPath();
-    ORT_RETURN_IF_ERROR(ToStatusAndRelease(CreateSessionAndLoadModelImpl(session_options, env,
-                                                                         input_model_path.c_str(),
-                                                                         nullptr, 0, session)));
+    status = ToStatusAndRelease(CreateSessionAndLoadModelImpl(session_options, env,
+                                                              input_model_path.c_str(),
+                                                              nullptr, 0, session));
   } else {
-    ORT_RETURN_IF_ERROR(
-        ToStatusAndRelease(CreateSessionAndLoadModelImpl(session_options, env, nullptr,
-                                                         model_compile_options.GetInputModelData(),
-                                                         model_compile_options.GetInputModelDataSize(),
-                                                         session)));
+    status = ToStatusAndRelease(CreateSessionAndLoadModelImpl(session_options, env, nullptr,
+                                                              model_compile_options.GetInputModelData(),
+                                                              model_compile_options.GetInputModelDataSize(),
+                                                              session));
   }
 
-  Env::Default().GetTelemetryProvider().LogCompileModel(session->GetCurrentSessionId());
-  ORT_RETURN_IF_ERROR(ToStatusAndRelease(InitializeSession(session_options, *session)));
-  return Status::OK();
+  if (!status.IsOK()) {
+    telemetry_provider.LogCompileModelComplete(
+        0,  // No session ID available
+        false,
+        static_cast<uint32_t>(status.Code()),
+        static_cast<uint32_t>(status.Category()),
+        status.ErrorMessage());
+    return status;
+  }
+
+  // Log start event now that we have the session ID and can get registered EP types
+  telemetry_provider.LogCompileModelStart(
+      session->GetCurrentSessionId(),
+      model_compile_options.GetInputSourceForTelemetry(),
+      model_compile_options.GetOutputTargetForTelemetry(),
+      model_compile_options.GetFlagsForTelemetry(),
+      model_compile_options.GetGraphOptimizationLevelForTelemetry(),
+      model_compile_options.GetEmbedEpContextForTelemetry(),
+      model_compile_options.HasExternalInitializersFileForTelemetry(),
+      session->GetRegisteredProviderTypes());
+
+  status = ToStatusAndRelease(InitializeSession(session_options, *session));
+
+  telemetry_provider.LogCompileModelComplete(
+      session->GetCurrentSessionId(),
+      status.IsOK(),
+      status.IsOK() ? 0 : static_cast<uint32_t>(status.Code()),
+      status.IsOK() ? 0 : static_cast<uint32_t>(status.Category()),
+      status.IsOK() ? "" : status.ErrorMessage());
+
+  return status;
 }
 
 Status LoadPluginOrProviderBridge(const std::string& registration_name,