Add Content Understanding 2025-05-01-preview

specification/ai/ContentUnderstanding/audioVisualContent.tsp

@@ -28,14 +28,27 @@ model AudioVisualContent extends MediaContent {
   @doc("Height of each video frame in pixels, if applicable.")
   height?: int32;
 
+  @added(Versions.v2025_05_01_preview)
+  @doc("List of camera shot changes in the video, represented by its timestamp in milliseconds.  Only if returnDetails is true.")
+  cameraShotTimesMs?: int64[];
+
   @doc("List of key frames in the video, represented by its timestamp in milliseconds.  Only if returnDetails is true.")
   keyFrameTimesMs?: int64[];
 
   @doc("List of transcript phrases.  Only if returnDetails is true.")
   transcriptPhrases?: TranscriptPhrase[];
 
+  @removed(Versions.v2025_05_01_preview)
   @doc("List of faces in the video.  Only if enableFace and returnDetails are true.")
   faces?: ImageFace[];
+
+  @added(Versions.v2025_05_01_preview)
+  @doc("List of detected persons in the video.  Only if enableFace and returnDetails are true.")
+  persons?: DetectedPerson[];
+
+  @added(Versions.v2025_05_01_preview)
+  @doc("List of audio visual segments.  Only if enableSegmentation and returnDetails are true.")
+  segments?: AudioVisualSegment[];
 }
 
 @doc("Transcript phrase.")
@@ -81,3 +94,35 @@ model TranscriptWord {
   @doc("Span of the word in the markdown content.")
   span?: ContentSpan;
 }
+
+@added(Versions.v2025_05_01_preview)
+@doc("Audio visual segment, such as a scene, chapter, etc.")
+model AudioVisualSegment {
+  @doc("Segment ID.")
+  segmentId: string;
+
+  @doc("Start time of the segment in milliseconds.")
+  startTimeMs: int64;
+
+  @doc("End time of the segment in milliseconds.")
+  endTimeMs: int64;
+
+  @doc("Short description of the segment.")
+  description: string;
+
+  @doc("Span of the segment in the markdown content.")
+  span?: ContentSpan;
+}
+
+@added(Versions.v2025_05_01_preview)
+@doc("Detected person.")
+model DetectedPerson {
+  @doc("Person identifier in the optional person directory if found.  Otherwise, each unknown person is assigned a unique `Person-{Number}`.")
+  personId?: string;
+
+  @doc("Confidence of the person identification, if a person directory is provided.")
+  confidence?: float32;
+
+  @doc("Encoded source that identifies the position of the person in the input content.")
+  source?: SourceExpression;
+}

specification/ai/ContentUnderstanding/classifierModels.tsp

@@ -0,0 +1,162 @@
+import "@azure-tools/typespec-azure-core";
+import "@typespec/rest";
+import "@typespec/http";
+
+using Azure.Core;
+using TypeSpec.Rest;
+using TypeSpec.Http;
+using TypeSpec.Versioning;
+
+namespace ContentUnderstanding;
+
+@added(Versions.v2025_05_01_preview)
+@doc("Classifier that classifies content into categories with optional splitting.")
+@resource("classifiers")
+model ContentClassifier {
+  @doc("The unique identifier of the classifier.")
+  @key
+  @visibility(Lifecycle.Read)
+  @pattern("^[a-zA-Z0-9._-]{1,64}$")
+  classifierId: string;
+
+  @doc("A description of the classifier.")
+  description?: string;
+
+  @doc("Tags associated with the classifier.")
+  tags?: Record<string>;
+
+  @doc("The status of the classifier.")
+  @visibility(Lifecycle.Read)
+  status: ResourceStatus;
+
+  @doc("The date and time when the classifier was created.")
+  @visibility(Lifecycle.Read)
+  createdAt: utcDateTime;
+
+  @doc("The date and time when the classifier was last modified.")
+  @visibility(Lifecycle.Read)
+  lastModifiedAt: utcDateTime;
+
+  @doc("Warnings encountered while creating the classifier.")
+  @visibility(Lifecycle.Read)
+  warnings?: Azure.Core.Foundations.Error[];
+
+  // TODO: Do we want to provide an overall instruction?
+
+  @doc("The categories to classify against.")
+  @visibility(Lifecycle.Create, Lifecycle.Read)
+  categories: Record<ClassifierCategory>;
+
+  @doc("Mode used to split input into content objects.")
+  @visibility(Lifecycle.Create, Lifecycle.Read)
+  splitMode?: ClassifierSplitMode = ClassifierSplitMode.noSplit;
+
+  @added(Versions.v2025_05_01_preview)
+  @doc("The location where the data may be processed.")
+  @visibility(Lifecycle.Create, Lifecycle.Read)
+  processingLocation?: ProcessingLocation = ProcessingLocation.geography;
+}
+
+@added(Versions.v2025_05_01_preview)
+@doc("A classifier category.")
+model ClassifierCategory {
+  @doc("The description of the category.")
+  @visibility(Lifecycle.Create, Lifecycle.Read)
+  description?: string;
+
+  @doc("Optional analyzer used to process the content.")
+  analyzerId?: string;
+}
+
+@added(Versions.v2025_05_01_preview)
+@doc("Mode used to split input into content objects.")
+union ClassifierSplitMode {
+  string,
+
+  @doc("Treat the entire input as a single content object.")
+  noSplit: "noSplit",
+
+  @doc("Split each page of the input into a separate content object.")
+  perPage: "perPage",
+
+  @doc("Split the input into content objects based on the input and category definitions.")
+  auto: "auto",
+}
+
+@added(Versions.v2025_05_01_preview)
+@doc("Classify operation parameters.")
+model ClassifyParameters {
+  // @doc("Range of the input to classify (ex. `1-3;5;9-`).  Document content uses 1-based page numbers, while audio visual content uses integer milliseconds.")
+  // @query
+  // range?: RangeExpression;
+
+  @added(Versions.v2025_05_01_preview)
+  @doc("The encoding format for content spans in the response.")
+  @query
+  stringEncoding?: StringEncoding = StringEncoding.codePoint;
+
+  @added(Versions.v2025_05_01_preview)
+  @doc("The location where the data may be processed.")
+  @query
+  processingLocation?: ProcessingLocation;
+}
+
+@added(Versions.v2025_05_01_preview)
+@doc("Classify operation request.")
+model ClassifyRequest {
+  ...ClassifyParameters;
+
+  @doc("The URL of the document to classify.")
+  url?: url;
+}
+
+@added(Versions.v2025_05_01_preview)
+@doc("Classify operation request directly from binary content.")
+model ClassifyBinaryRequest {
+  ...ClassifyParameters;
+
+  #suppress "@azure-tools/typespec-azure-core/no-closed-literal-union" "Conflicts with @typespec/http/content-type-string"
+  @doc("Request content type.")
+  @header
+  contentType: string = "application/octet-stream";
+
+  @doc("The binary content of the document to classify.")
+  @bodyRoot
+  input: bytes;
+}
+
+// @added(Versions.v2025_05_01_preview)
+// @doc("...")
+// model AdHocClassifyRequest {
+//   ...ClassifyRequest;
+
+//   @doc("The content categories to classify against.")
+//   categories?: Record<ClassifierCategory>;
+
+//   @doc("Mode used to split input into content objects.")
+//   splitMode?: ClassifierSplitMode = ClassifierSplitMode.noSplit;
+// }
+
+@added(Versions.v2025_05_01_preview)
+@doc("Classify operation result.")
+model ClassifyResult {
+  @doc("The unique identifier of the classifier.")
+  @pattern("^[a-zA-Z0-9._-]{1,64}$")
+  classifierId?: string;
+
+  @doc("The version of the API used to classify the document.")
+  apiVersion?: string;
+
+  @doc("The date and time when the result was created.")
+  createdAt?: utcDateTime;
+
+  @doc("Warnings encountered while classifying the document.")
+  warnings?: Azure.Core.Foundations.Error[];
+
+  @added(Versions.v2025_05_01_preview)
+  @doc("The string encoding used for content spans.")
+  stringEncoding?: StringEncoding = StringEncoding.codePoint;
+
+  @doc("The classified content.")
+  contents: MediaContent[];
+}

specification/ai/ContentUnderstanding/classifierRoutes.tsp

@@ -0,0 +1,91 @@
+import "@azure-tools/typespec-azure-core";
+import "@typespec/rest";
+import "@typespec/http";
+import "./classifierModels.tsp";
+
+using Azure.Core;
+using Azure.Core.Traits;
+using TypeSpec.Rest;
+using TypeSpec.Http;
+using TypeSpec.Versioning;
+
+namespace ContentUnderstanding;
+
+@added(Versions.v2025_05_01_preview)
+interface ContentClassifiers {
+  @doc("Get the status of a classifier creation operation.")
+  getOperationStatus is Operations.GetResourceOperationStatus<
+    ContentClassifier,
+    ContentClassifier
+  >;
+
+  @doc("Create a new classifier asynchronously.")
+  @pollingOperation(ContentClassifiers.getOperationStatus)
+  createOrReplace is Operations.LongRunningResourceCreateOrReplace<ContentClassifier>;
+
+  @doc("Update classifier properties.")
+  update is Operations.ResourceUpdate<ContentClassifier>;
+
+  @doc("Get classifier properties.")
+  get is Operations.ResourceRead<ContentClassifier>;
+
+  @doc("Delete classifier.")
+  delete is Operations.ResourceDelete<ContentClassifier>;
+
+  @doc("List classifiers.")
+  list is Operations.ResourceList<ContentClassifier>;
+
+  @doc("Classify content with optional splitting.")
+  @sharedRoute
+  @action("classify")
+  @pollingOperation(ContentClassifiers.getResult)
+  classify is Operations.LongRunningResourceAction<
+    ContentClassifier,
+    ClassifyRequest,
+    ClassifyResult
+  >;
+
+  @doc("Classify content with optional splitting.")
+  @sharedRoute
+  @action("classify")
+  @pollingOperation(ContentClassifiers.getResult)
+  classifyBinary is Operations.LongRunningResourceAction<
+    ContentClassifier,
+    ClassifyBinaryRequest,
+    ClassifyResult
+  >;
+
+  // @added(Versions.v2025_05_01_preview)
+  // @doc("Classify input and route to analyzers for processing using adhoc classifier.")
+  // @pollingOperation(ContentClassifiers.getResult)
+  // @action("classify")
+  // adHocClassify is Operations.LongRunningResourceCollectionAction<
+  //   ContentClassifier,
+  //   AdHocClassifyRequest,
+  //   ClassifyResult
+  // >;
+
+  #suppress "@azure-tools/typespec-azure-core/use-standard-operations" "This is a custom operation status endpoint."
+  @doc("Get the result of a classifier operation.")
+  @route("/classifierResults/{operationId}")
+  getResult is Foundations.GetOperationStatus<{}, ClassifyResult>;
+
+  // #suppress "@azure-tools/typespec-azure-core/use-standard-operations" "Doesn't fit standard ops"
+  // @added(Versions.v2025_05_01_preview)
+  // @doc("Mark the result of a classifier operation for deletion.")
+  // @route("/classifierResults/{operationId}")
+  // @delete
+  // deleteResult is Foundations.Operation<
+  //   {
+  //     @doc("Classifier identifier.")
+  //     @path
+  //     classifierId: string;
+
+  //     @doc("Operation identifier.")
+  //     @path
+  //     operationId: string;
+  //   },
+  //   TypeSpec.Http.NoContentResponse,
+  //   ServiceTraits
+  // >;
+}