Project import generated by Copybara.

GitOrigin-RevId: 796203faee20d7aae2876aac8ca5a1827dee4fe3

Author: MediaPipe Team

Diff for: WORKSPACE

@@ -64,6 +64,12 @@ http_archive(
     sha256 = "267103f8a1e9578978aa1dc256001e6529ef593e5aea38193d31c2872ee025e8",
     strip_prefix = "glog-0.3.5",
     build_file = "@//third_party:glog.BUILD",
+    patches = [
+        "@//third_party:com_github_glog_glog_9779e5ea6ef59562b030248947f787d1256132ae.diff"
+    ],
+    patch_args = [
+        "-p1",
+    ],
 )
 
 # libyuv

Diff for: mediapipe/calculators/audio/audio_decoder_calculator.cc

@@ -61,7 +61,9 @@ class AudioDecoderCalculator : public CalculatorBase {
 ::mediapipe::Status AudioDecoderCalculator::GetContract(
     CalculatorContract* cc) {
   cc->InputSidePackets().Tag("INPUT_FILE_PATH").Set<std::string>();
-
+  if (cc->InputSidePackets().HasTag("OPTIONS")) {
+    cc->InputSidePackets().Tag("OPTIONS").Set<mediapipe::AudioDecoderOptions>();
+  }
   cc->Outputs().Tag("AUDIO").Set<Matrix>();
   if (cc->Outputs().HasTag("AUDIO_HEADER")) {
     cc->Outputs().Tag("AUDIO_HEADER").SetNone();
@@ -72,7 +74,9 @@ ::mediapipe::Status AudioDecoderCalculator::GetContract(
 ::mediapipe::Status AudioDecoderCalculator::Open(CalculatorContext* cc) {
   const std::string& input_file_path =
       cc->InputSidePackets().Tag("INPUT_FILE_PATH").Get<std::string>();
-  const auto& decoder_options = cc->Options<mediapipe::AudioDecoderOptions>();
+  const auto& decoder_options =
+      tool::RetrieveOptions(cc->Options<mediapipe::AudioDecoderOptions>(),
+                            cc->InputSidePackets(), "OPTIONS");
   decoder_ = absl::make_unique<AudioDecoder>();
   MP_RETURN_IF_ERROR(decoder_->Initialize(input_file_path, decoder_options));
   std::unique_ptr<mediapipe::TimeSeriesHeader> header =

Diff for: mediapipe/calculators/audio/stabilized_log_calculator.cc

@@ -75,8 +75,13 @@ class StabilizedLogCalculator : public CalculatorBase {
 
   ::mediapipe::Status Process(CalculatorContext* cc) override {
     auto input_matrix = cc->Inputs().Index(0).Get<Matrix>();
+    if (input_matrix.array().isNaN().any()) {
+      return ::mediapipe::InvalidArgumentError("NaN input to log operation.");
+    }
     if (check_nonnegativity_) {
-      CHECK_GE(input_matrix.minCoeff(), 0);
+      if (input_matrix.minCoeff() < 0.0) {
+        return ::mediapipe::OutOfRangeError("Negative input to log operation.");
+      }
     }
     std::unique_ptr<Matrix> output_frame(new Matrix(
         output_scale_ * (input_matrix.array() + stabilizer_).log().matrix()));

Diff for: mediapipe/calculators/audio/stabilized_log_calculator_test.cc

@@ -11,6 +11,7 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
+#include <cmath>
 
 #include "Eigen/Core"
 #include "mediapipe/calculators/audio/stabilized_log_calculator.pb.h"
@@ -108,13 +109,22 @@ TEST_F(StabilizedLogCalculatorTest, ZerosAreStabilized) {
       runner_->Outputs().Index(0).packets[0].Get<Matrix>());
 }
 
-TEST_F(StabilizedLogCalculatorTest, NegativeValuesCheckFail) {
+TEST_F(StabilizedLogCalculatorTest, NanValuesReturnError) {
+  InitializeGraph();
+  FillInputHeader();
+  AppendInputPacket(
+      new Matrix(Matrix::Constant(kNumChannels, kNumSamples, std::nanf(""))),
+      0 /* timestamp */);
+  ASSERT_FALSE(RunGraph().ok());
+}
+
+TEST_F(StabilizedLogCalculatorTest, NegativeValuesReturnError) {
   InitializeGraph();
   FillInputHeader();
   AppendInputPacket(
       new Matrix(Matrix::Constant(kNumChannels, kNumSamples, -1.0)),
       0 /* timestamp */);
-  ASSERT_DEATH(RunGraphNoReturn(), "");
+  ASSERT_FALSE(RunGraph().ok());
 }
 
 TEST_F(StabilizedLogCalculatorTest, NegativeValuesDoNotCheckFailIfCheckIsOff) {

Diff for: mediapipe/calculators/audio/time_series_framer_calculator.cc

@@ -56,6 +56,14 @@ namespace mediapipe {
 // If pad_final_packet is true, all input samples will be emitted and the final
 // packet will be zero padded as necessary.  If pad_final_packet is false, some
 // samples may be dropped at the end of the stream.
+//
+// If use_local_timestamp is true, the output packet's timestamp is based on the
+// last sample of the packet. The timestamp of this sample is inferred by
+// input_packet_timesamp + local_sample_index / sampling_rate_. If false, the
+// output packet's timestamp is based on the cumulative timestamping, which is
+// done by adopting the timestamp of the first sample of the packet and this
+// sample's timestamp is inferred by initial_input_timestamp_ +
+// cumulative_completed_samples / sample_rate_.
 class TimeSeriesFramerCalculator : public CalculatorBase {
  public:
   static ::mediapipe::Status GetContract(CalculatorContract* cc) {
@@ -86,11 +94,26 @@ class TimeSeriesFramerCalculator : public CalculatorBase {
   void FrameOutput(CalculatorContext* cc);
 
   Timestamp CurrentOutputTimestamp() {
+    if (use_local_timestamp_) {
+      return current_timestamp_;
+    }
+    return CumulativeOutputTimestamp();
+  }
+
+  Timestamp CumulativeOutputTimestamp() {
     return initial_input_timestamp_ +
            round(cumulative_completed_samples_ / sample_rate_ *
                  Timestamp::kTimestampUnitsPerSecond);
   }
 
+  // Returns the timestamp of a sample on a base, which is usually the time
+  // stamp of a packet.
+  Timestamp CurrentSampleTimestamp(const Timestamp& timestamp_base,
+                                   int64 number_of_samples) {
+    return timestamp_base + round(number_of_samples / sample_rate_ *
+                                  Timestamp::kTimestampUnitsPerSecond);
+  }
+
   // The number of input samples to advance after the current output frame is
   // emitted.
   int next_frame_step_samples() const {
@@ -118,22 +141,27 @@ class TimeSeriesFramerCalculator : public CalculatorBase {
   // any overlap).
   int64 cumulative_completed_samples_;
   Timestamp initial_input_timestamp_;
+  // The current timestamp is updated along with the incoming packets.
+  Timestamp current_timestamp_;
   int num_channels_;
 
   // Each entry in this deque consists of a single sample, i.e. a
-  // single column vector.
-  std::deque<Matrix> sample_buffer_;
+  // single column vector, and its timestamp.
+  std::deque<std::pair<Matrix, Timestamp>> sample_buffer_;
 
   bool use_window_;
   Matrix window_;
+
+  bool use_local_timestamp_;
 };
 REGISTER_CALCULATOR(TimeSeriesFramerCalculator);
 
 void TimeSeriesFramerCalculator::EnqueueInput(CalculatorContext* cc) {
   const Matrix& input_frame = cc->Inputs().Index(0).Get<Matrix>();
 
   for (int i = 0; i < input_frame.cols(); ++i) {
-    sample_buffer_.emplace_back(input_frame.col(i));
+    sample_buffer_.emplace_back(std::make_pair(
+        input_frame.col(i), CurrentSampleTimestamp(cc->InputTimestamp(), i)));
   }
 
   cumulative_input_samples_ += input_frame.cols();
@@ -151,14 +179,16 @@ void TimeSeriesFramerCalculator::FrameOutput(CalculatorContext* cc) {
         new Matrix(num_channels_, frame_duration_samples_));
     for (int i = 0; i < std::min(frame_step_samples, frame_duration_samples_);
          ++i) {
-      output_frame->col(i) = sample_buffer_.front();
+      output_frame->col(i) = sample_buffer_.front().first;
+      current_timestamp_ = sample_buffer_.front().second;
       sample_buffer_.pop_front();
     }
     const int frame_overlap_samples =
         frame_duration_samples_ - frame_step_samples;
     if (frame_overlap_samples > 0) {
       for (int i = 0; i < frame_overlap_samples; ++i) {
-        output_frame->col(i + frame_step_samples) = sample_buffer_[i];
+        output_frame->col(i + frame_step_samples) = sample_buffer_[i].first;
+        current_timestamp_ = sample_buffer_[i].second;
       }
     } else {
       samples_still_to_drop_ = -frame_overlap_samples;
@@ -178,6 +208,7 @@ void TimeSeriesFramerCalculator::FrameOutput(CalculatorContext* cc) {
 ::mediapipe::Status TimeSeriesFramerCalculator::Process(CalculatorContext* cc) {
   if (initial_input_timestamp_ == Timestamp::Unstarted()) {
     initial_input_timestamp_ = cc->InputTimestamp();
+    current_timestamp_ = initial_input_timestamp_;
   }
 
   EnqueueInput(cc);
@@ -195,7 +226,8 @@ ::mediapipe::Status TimeSeriesFramerCalculator::Close(CalculatorContext* cc) {
     std::unique_ptr<Matrix> output_frame(new Matrix);
     output_frame->setZero(num_channels_, frame_duration_samples_);
     for (int i = 0; i < sample_buffer_.size(); ++i) {
-      output_frame->col(i) = sample_buffer_[i];
+      output_frame->col(i) = sample_buffer_[i].first;
+      current_timestamp_ = sample_buffer_[i].second;
     }
 
     cc->Outputs().Index(0).Add(output_frame.release(),
@@ -258,6 +290,7 @@ ::mediapipe::Status TimeSeriesFramerCalculator::Open(CalculatorContext* cc) {
   cumulative_output_frames_ = 0;
   samples_still_to_drop_ = 0;
   initial_input_timestamp_ = Timestamp::Unstarted();
+  current_timestamp_ = Timestamp::Unstarted();
 
   std::vector<double> window_vector;
   use_window_ = false;
@@ -282,6 +315,7 @@ ::mediapipe::Status TimeSeriesFramerCalculator::Open(CalculatorContext* cc) {
                                           frame_duration_samples_)
                   .cast<float>();
   }
+  use_local_timestamp_ = framer_options.use_local_timestamp();
 
   return ::mediapipe::OkStatus();
 }

Diff for: mediapipe/calculators/audio/time_series_framer_calculator.proto

@@ -62,4 +62,11 @@ message TimeSeriesFramerCalculatorOptions {
     HANN = 2;
   }
   optional WindowFunction window_function = 4 [default = NONE];
+
+  // If use_local_timestamp is true, the output packet's timestamp is based on
+  // the last sample of the packet and it's inferred from the latest input
+  // packet's timestamp.  If false, the output packet's timestamp is based on
+  // the cumulative timestamping, which is inferred from the intial input
+  // timestamp and the cumulative number of samples.
+  optional bool use_local_timestamp = 6 [default = false];
 }

Diff for: mediapipe/calculators/audio/time_series_framer_calculator_test.cc

@@ -35,6 +35,8 @@ namespace mediapipe {
 namespace {
 
 const int kInitialTimestampOffsetMicroseconds = 4;
+const int kGapBetweenPacketsInSeconds = 1;
+const int kUniversalInputPacketSize = 50;
 
 class TimeSeriesFramerCalculatorTest
     : public TimeSeriesCalculatorTest<TimeSeriesFramerCalculatorOptions> {
@@ -391,5 +393,93 @@ TEST_F(TimeSeriesFramerCalculatorWindowingSanityTest, HannWindowSanityCheck) {
   RunAndTestSinglePacketAverage(0.5f);
 }
 
-}  // anonymous namespace
+// A simple test class that checks the local packet time stamp. This class
+// generate a series of packets with and without gaps between packets and tests
+// the behavior with cumulative timestamping and local packet timestamping.
+class TimeSeriesFramerCalculatorTimestampingTest
+    : public TimeSeriesFramerCalculatorTest {
+ protected:
+  // Creates test input and saves a reference copy.
+  void InitializeInputForTimeStampingTest() {
+    concatenated_input_samples_.resize(0, num_input_channels_);
+    num_input_samples_ = 0;
+    for (int i = 0; i < 10; ++i) {
+      // This range of packet sizes was chosen such that some input
+      // packets will be smaller than the output packet size and other
+      // input packets will be larger.
+      int packet_size = kUniversalInputPacketSize;
+      double timestamp_seconds = kInitialTimestampOffsetMicroseconds * 1.0e-6 +
+                                 num_input_samples_ / input_sample_rate_;
+      if (options_.use_local_timestamp()) {
+        timestamp_seconds += kGapBetweenPacketsInSeconds * i;
+      }
+
+      Matrix* data_frame =
+          NewTestFrame(num_input_channels_, packet_size, timestamp_seconds);
+
+      AppendInputPacket(data_frame, round(timestamp_seconds *
+                                          Timestamp::kTimestampUnitsPerSecond));
+      num_input_samples_ += packet_size;
+    }
+  }
+
+  void CheckOutputTimestamps() {
+    int num_full_packets = output().packets.size();
+    if (options_.pad_final_packet()) {
+      num_full_packets -= 1;
+    }
+
+    int64 num_samples = 0;
+    for (int packet_num = 0; packet_num < num_full_packets; ++packet_num) {
+      const Packet& packet = output().packets[packet_num];
+      num_samples += FrameDurationSamples();
+      double expected_timestamp =
+          options_.use_local_timestamp()
+              ? GetExpectedLocalTimestampForSample(num_samples - 1)
+              : GetExpectedCumulativeTimestamp(num_samples - 1);
+      ASSERT_NEAR(packet.Timestamp().Seconds(), expected_timestamp, 1e-10);
+    }
+  }
+
+  ::mediapipe::Status RunTimestampTest() {
+    InitializeGraph();
+    InitializeInputForTimeStampingTest();
+    FillInputHeader();
+    return RunGraph();
+  }
+
+ private:
+  // Returns the timestamp in seconds based on local timestamping.
+  double GetExpectedLocalTimestampForSample(int sample_index) {
+    return kInitialTimestampOffsetMicroseconds * 1.0e-6 +
+           sample_index / input_sample_rate_ +
+           (sample_index / kUniversalInputPacketSize) *
+               kGapBetweenPacketsInSeconds;
+  }
+
+  // Returns the timestamp inseconds based on cumulative timestamping.
+  double GetExpectedCumulativeTimestamp(int sample_index) {
+    return kInitialTimestampOffsetMicroseconds * 1.0e-6 +
+           sample_index / FrameDurationSamples() * FrameDurationSamples() /
+               input_sample_rate_;
+  }
+};
+
+TEST_F(TimeSeriesFramerCalculatorTimestampingTest, UseLocalTimeStamp) {
+  options_.set_frame_duration_seconds(100.0 / input_sample_rate_);
+  options_.set_use_local_timestamp(true);
+
+  MP_ASSERT_OK(RunTimestampTest());
+  CheckOutputTimestamps();
+}
+
+TEST_F(TimeSeriesFramerCalculatorTimestampingTest, UseCumulativeTimeStamp) {
+  options_.set_frame_duration_seconds(100.0 / input_sample_rate_);
+  options_.set_use_local_timestamp(false);
+
+  MP_ASSERT_OK(RunTimestampTest());
+  CheckOutputTimestamps();
+}
+
+}  // namespace
 }  // namespace mediapipe

Diff for: mediapipe/calculators/core/BUILD

@@ -166,7 +166,13 @@ cc_library(
         "//mediapipe/framework/port:ret_check",
         "//mediapipe/framework/port:status",
         "@org_tensorflow//tensorflow/lite:framework",
-    ],
+    ] + select({
+        "//mediapipe/gpu:disable_gpu": [],
+        "//mediapipe:ios": [],
+        "//conditions:default": [
+            "@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_buffer",
+        ],
+    }),
     alwayslink = 1,
 )
 

Diff for: mediapipe/calculators/core/concatenate_vector_calculator.cc

@@ -19,6 +19,10 @@
 #include "mediapipe/framework/formats/landmark.pb.h"
 #include "tensorflow/lite/interpreter.h"
 
+#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__APPLE__)
+#include "tensorflow/lite/delegates/gpu/gl/gl_buffer.h"
+#endif  //  !MEDIAPIPE_DISABLE_GPU
+
 namespace mediapipe {
 
 // Example config:
@@ -45,4 +49,11 @@ REGISTER_CALCULATOR(ConcatenateTfLiteTensorVectorCalculator);
 typedef ConcatenateVectorCalculator<::mediapipe::NormalizedLandmark>
     ConcatenateLandmarkVectorCalculator;
 REGISTER_CALCULATOR(ConcatenateLandmarkVectorCalculator);
+
+#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__APPLE__)
+typedef ConcatenateVectorCalculator<::tflite::gpu::gl::GlBuffer>
+    ConcatenateGlBufferVectorCalculator;
+REGISTER_CALCULATOR(ConcatenateGlBufferVectorCalculator);
+#endif
+
 }  // namespace mediapipe