Skip to content

Commit 7bad8fc

Browse files
MediaPipe Teamchuoling
MediaPipe Team
authored and
chuoling
committed
Project import generated by Copybara.
GitOrigin-RevId: d0039a576e2db9c0fcefffd26a527df74cbe145b
1 parent 024f7bf commit 7bad8fc

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

45 files changed

+1566
-227
lines changed

Diff for: ‎README.md

+3
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
* [Object Detection and Tracking](mediapipe/docs/object_tracking_mobile_gpu.md)
2020
* [Objectron: 3D Object Detection and Tracking](mediapipe/docs/objectron_mobile_gpu.md)
2121
* [AutoFlip: Intelligent Video Reframing](mediapipe/docs/autoflip.md)
22+
* [KNIFT: Template Matching with Neural Image Features](mediapipe/docs/template_matching_mobile_cpu.md)
2223

2324
![face_detection](mediapipe/docs/images/mobile/face_detection_android_gpu_small.gif)
2425
![face_mesh](mediapipe/docs/images/mobile/face_mesh_android_gpu_small.gif)
@@ -29,6 +30,7 @@
2930
![object_tracking](mediapipe/docs/images/mobile/object_tracking_android_gpu_small.gif)
3031
![objectron_shoes](mediapipe/docs/images/mobile/objectron_shoe_android_gpu_small.gif)
3132
![objectron_chair](mediapipe/docs/images/mobile/objectron_chair_android_gpu_small.gif)
33+
![template_matching](mediapipe/docs/images/mobile/template_matching_android_cpu_small.gif)
3234

3335
## Installation
3436
Follow these [instructions](mediapipe/docs/install.md).
@@ -53,6 +55,7 @@ Search MediaPipe Github repository using [Google Open Source code search](https:
5355
* [YouTube Channel](https://www.youtube.com/channel/UCObqmpuSMx-usADtL_qdMAw)
5456

5557
## Publications
58+
* [MediaPipe KNIFT: Template-based Feature Matching](https://mediapipe.page.link/knift-blog)
5659
* [Alfred Camera: Smart camera features using MediaPipe](https://developers.googleblog.com/2020/03/alfred-camera-smart-camera-features-using-mediapipe.html)
5760
* [MediaPipe Objectron: Real-time 3D Object Detection on Mobile Devices](https://mediapipe.page.link/objectron-aiblog)
5861
* [AutoFlip: An Open Source Framework for Intelligent Video Reframing](https://mediapipe.page.link/autoflip)

Diff for: ‎mediapipe/calculators/image/image_properties_calculator.cc

+17-10
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,11 @@
1919
#include "mediapipe/gpu/gpu_buffer.h"
2020
#endif // !MEDIAPIPE_DISABLE_GPU
2121

22+
namespace {
23+
constexpr char kImageFrameTag[] = "IMAGE";
24+
constexpr char kGpuBufferTag[] = "IMAGE_GPU";
25+
} // namespace
26+
2227
namespace mediapipe {
2328

2429
// Extracts image properties from the input image and outputs the properties.
@@ -40,13 +45,14 @@ namespace mediapipe {
4045
class ImagePropertiesCalculator : public CalculatorBase {
4146
public:
4247
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
43-
RET_CHECK(cc->Inputs().HasTag("IMAGE") ^ cc->Inputs().HasTag("IMAGE_GPU"));
44-
if (cc->Inputs().HasTag("IMAGE")) {
45-
cc->Inputs().Tag("IMAGE").Set<ImageFrame>();
48+
RET_CHECK(cc->Inputs().HasTag(kImageFrameTag) ^
49+
cc->Inputs().HasTag(kGpuBufferTag));
50+
if (cc->Inputs().HasTag(kImageFrameTag)) {
51+
cc->Inputs().Tag(kImageFrameTag).Set<ImageFrame>();
4652
}
4753
#if !defined(MEDIAPIPE_DISABLE_GPU)
48-
if (cc->Inputs().HasTag("IMAGE_GPU")) {
49-
cc->Inputs().Tag("IMAGE_GPU").Set<::mediapipe::GpuBuffer>();
54+
if (cc->Inputs().HasTag(kGpuBufferTag)) {
55+
cc->Inputs().Tag(kGpuBufferTag).Set<::mediapipe::GpuBuffer>();
5056
}
5157
#endif // !MEDIAPIPE_DISABLE_GPU
5258

@@ -66,16 +72,17 @@ class ImagePropertiesCalculator : public CalculatorBase {
6672
int width;
6773
int height;
6874

69-
if (cc->Inputs().HasTag("IMAGE") && !cc->Inputs().Tag("IMAGE").IsEmpty()) {
70-
const auto& image = cc->Inputs().Tag("IMAGE").Get<ImageFrame>();
75+
if (cc->Inputs().HasTag(kImageFrameTag) &&
76+
!cc->Inputs().Tag(kImageFrameTag).IsEmpty()) {
77+
const auto& image = cc->Inputs().Tag(kImageFrameTag).Get<ImageFrame>();
7178
width = image.Width();
7279
height = image.Height();
7380
}
7481
#if !defined(MEDIAPIPE_DISABLE_GPU)
75-
if (cc->Inputs().HasTag("IMAGE_GPU") &&
76-
!cc->Inputs().Tag("IMAGE_GPU").IsEmpty()) {
82+
if (cc->Inputs().HasTag(kGpuBufferTag) &&
83+
!cc->Inputs().Tag(kGpuBufferTag).IsEmpty()) {
7784
const auto& image =
78-
cc->Inputs().Tag("IMAGE_GPU").Get<mediapipe::GpuBuffer>();
85+
cc->Inputs().Tag(kGpuBufferTag).Get<mediapipe::GpuBuffer>();
7986
width = image.width();
8087
height = image.height();
8188
}

Diff for: ‎mediapipe/calculators/image/image_transformation_calculator.cc

+46-39
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,9 @@ namespace mediapipe {
4747
#endif // !MEDIAPIPE_DISABLE_GPU
4848

4949
namespace {
50+
constexpr char kImageFrameTag[] = "IMAGE";
51+
constexpr char kGpuBufferTag[] = "IMAGE_GPU";
52+
5053
int RotationModeToDegrees(mediapipe::RotationMode_Mode rotation) {
5154
switch (rotation) {
5255
case mediapipe::RotationMode_Mode_UNKNOWN:
@@ -95,7 +98,7 @@ mediapipe::ScaleMode_Mode ParseScaleMode(
9598
// Scales, rotates, and flips images horizontally or vertically.
9699
//
97100
// Input:
98-
// One of the following two tags:
101+
// One of the following tags:
99102
// IMAGE: ImageFrame representing the input image.
100103
// IMAGE_GPU: GpuBuffer representing the input image.
101104
//
@@ -113,7 +116,7 @@ mediapipe::ScaleMode_Mode ParseScaleMode(
113116
// corresponding field in the calculator options.
114117
//
115118
// Output:
116-
// One of the following two tags:
119+
// One of the following tags:
117120
// IMAGE - ImageFrame representing the output image.
118121
// IMAGE_GPU - GpuBuffer representing the output image.
119122
//
@@ -152,7 +155,8 @@ mediapipe::ScaleMode_Mode ParseScaleMode(
152155
// Note: To enable horizontal or vertical flipping, specify them in the
153156
// calculator options. Flipping is applied after rotation.
154157
//
155-
// Note: Only scale mode STRETCH is currently supported on CPU.
158+
// Note: Input defines output, so only matchig types supported:
159+
// IMAGE -> IMAGE or IMAGE_GPU -> IMAGE_GPU
156160
//
157161
class ImageTransformationCalculator : public CalculatorBase {
158162
public:
@@ -186,7 +190,7 @@ class ImageTransformationCalculator : public CalculatorBase {
186190

187191
bool use_gpu_ = false;
188192
#if !defined(MEDIAPIPE_DISABLE_GPU)
189-
GlCalculatorHelper helper_;
193+
GlCalculatorHelper gpu_helper_;
190194
std::unique_ptr<QuadRenderer> rgb_renderer_;
191195
std::unique_ptr<QuadRenderer> yuv_renderer_;
192196
std::unique_ptr<QuadRenderer> ext_rgb_renderer_;
@@ -197,21 +201,22 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
197201
// static
198202
::mediapipe::Status ImageTransformationCalculator::GetContract(
199203
CalculatorContract* cc) {
200-
RET_CHECK(cc->Inputs().HasTag("IMAGE") ^ cc->Inputs().HasTag("IMAGE_GPU"));
201-
RET_CHECK(cc->Outputs().HasTag("IMAGE") ^ cc->Outputs().HasTag("IMAGE_GPU"));
204+
// Only one input can be set, and the output type must match.
205+
RET_CHECK(cc->Inputs().HasTag(kImageFrameTag) ^
206+
cc->Inputs().HasTag(kGpuBufferTag));
202207

203208
bool use_gpu = false;
204209

205-
if (cc->Inputs().HasTag("IMAGE")) {
206-
RET_CHECK(cc->Outputs().HasTag("IMAGE"));
207-
cc->Inputs().Tag("IMAGE").Set<ImageFrame>();
208-
cc->Outputs().Tag("IMAGE").Set<ImageFrame>();
210+
if (cc->Inputs().HasTag(kImageFrameTag)) {
211+
RET_CHECK(cc->Outputs().HasTag(kImageFrameTag));
212+
cc->Inputs().Tag(kImageFrameTag).Set<ImageFrame>();
213+
cc->Outputs().Tag(kImageFrameTag).Set<ImageFrame>();
209214
}
210215
#if !defined(MEDIAPIPE_DISABLE_GPU)
211-
if (cc->Inputs().HasTag("IMAGE_GPU")) {
212-
RET_CHECK(cc->Outputs().HasTag("IMAGE_GPU"));
213-
cc->Inputs().Tag("IMAGE_GPU").Set<GpuBuffer>();
214-
cc->Outputs().Tag("IMAGE_GPU").Set<GpuBuffer>();
216+
if (cc->Inputs().HasTag(kGpuBufferTag)) {
217+
RET_CHECK(cc->Outputs().HasTag(kGpuBufferTag));
218+
cc->Inputs().Tag(kGpuBufferTag).Set<GpuBuffer>();
219+
cc->Outputs().Tag(kGpuBufferTag).Set<GpuBuffer>();
215220
use_gpu |= true;
216221
}
217222
#endif // !MEDIAPIPE_DISABLE_GPU
@@ -259,7 +264,7 @@ ::mediapipe::Status ImageTransformationCalculator::Open(CalculatorContext* cc) {
259264

260265
options_ = cc->Options<ImageTransformationCalculatorOptions>();
261266

262-
if (cc->Inputs().HasTag("IMAGE_GPU")) {
267+
if (cc->Inputs().HasTag(kGpuBufferTag)) {
263268
use_gpu_ = true;
264269
}
265270

@@ -300,7 +305,7 @@ ::mediapipe::Status ImageTransformationCalculator::Open(CalculatorContext* cc) {
300305
if (use_gpu_) {
301306
#if !defined(MEDIAPIPE_DISABLE_GPU)
302307
// Let the helper access the GL context information.
303-
MP_RETURN_IF_ERROR(helper_.Open(cc));
308+
MP_RETURN_IF_ERROR(gpu_helper_.Open(cc));
304309
#else
305310
RET_CHECK_FAIL() << "GPU processing not enabled.";
306311
#endif // !MEDIAPIPE_DISABLE_GPU
@@ -328,18 +333,14 @@ ::mediapipe::Status ImageTransformationCalculator::Process(
328333

329334
if (use_gpu_) {
330335
#if !defined(MEDIAPIPE_DISABLE_GPU)
331-
if (cc->Inputs().Tag("IMAGE_GPU").IsEmpty()) {
332-
// Image is missing, hence no way to produce output image. (Timestamp
333-
// bound will be updated automatically.)
336+
if (cc->Inputs().Tag(kGpuBufferTag).IsEmpty()) {
334337
return ::mediapipe::OkStatus();
335338
}
336-
return helper_.RunInGlContext(
339+
return gpu_helper_.RunInGlContext(
337340
[this, cc]() -> ::mediapipe::Status { return RenderGpu(cc); });
338341
#endif // !MEDIAPIPE_DISABLE_GPU
339342
} else {
340-
if (cc->Inputs().Tag("IMAGE").IsEmpty()) {
341-
// Image is missing, hence no way to produce output image. (Timestamp
342-
// bound will be updated automatically.)
343+
if (cc->Inputs().Tag(kImageFrameTag).IsEmpty()) {
343344
return ::mediapipe::OkStatus();
344345
}
345346
return RenderCpu(cc);
@@ -354,7 +355,7 @@ ::mediapipe::Status ImageTransformationCalculator::Close(
354355
QuadRenderer* rgb_renderer = rgb_renderer_.release();
355356
QuadRenderer* yuv_renderer = yuv_renderer_.release();
356357
QuadRenderer* ext_rgb_renderer = ext_rgb_renderer_.release();
357-
helper_.RunInGlContext([rgb_renderer, yuv_renderer, ext_rgb_renderer] {
358+
gpu_helper_.RunInGlContext([rgb_renderer, yuv_renderer, ext_rgb_renderer] {
358359
if (rgb_renderer) {
359360
rgb_renderer->GlTeardown();
360361
delete rgb_renderer;
@@ -376,17 +377,21 @@ ::mediapipe::Status ImageTransformationCalculator::Close(
376377

377378
::mediapipe::Status ImageTransformationCalculator::RenderCpu(
378379
CalculatorContext* cc) {
379-
const auto& input_img = cc->Inputs().Tag("IMAGE").Get<ImageFrame>();
380-
cv::Mat input_mat = formats::MatView(&input_img);
381-
cv::Mat scaled_mat;
380+
cv::Mat input_mat;
381+
mediapipe::ImageFormat::Format format;
382382

383-
const int input_width = input_img.Width();
384-
const int input_height = input_img.Height();
383+
const auto& input = cc->Inputs().Tag(kImageFrameTag).Get<ImageFrame>();
384+
input_mat = formats::MatView(&input);
385+
format = input.Format();
386+
387+
const int input_width = input_mat.cols;
388+
const int input_height = input_mat.rows;
385389
if (!output_height_ || !output_width_) {
386390
output_height_ = input_height;
387391
output_width_ = input_width;
388392
}
389393

394+
cv::Mat scaled_mat;
390395
if (scale_mode_ == mediapipe::ScaleMode_Mode_STRETCH) {
391396
cv::resize(input_mat, scaled_mat, cv::Size(output_width_, output_height_));
392397
} else {
@@ -443,18 +448,20 @@ ::mediapipe::Status ImageTransformationCalculator::RenderCpu(
443448
}
444449

445450
std::unique_ptr<ImageFrame> output_frame(
446-
new ImageFrame(input_img.Format(), output_width, output_height));
451+
new ImageFrame(format, output_width, output_height));
447452
cv::Mat output_mat = formats::MatView(output_frame.get());
448453
flipped_mat.copyTo(output_mat);
449-
cc->Outputs().Tag("IMAGE").Add(output_frame.release(), cc->InputTimestamp());
454+
cc->Outputs()
455+
.Tag(kImageFrameTag)
456+
.Add(output_frame.release(), cc->InputTimestamp());
450457

451458
return ::mediapipe::OkStatus();
452459
}
453460

454461
::mediapipe::Status ImageTransformationCalculator::RenderGpu(
455462
CalculatorContext* cc) {
456463
#if !defined(MEDIAPIPE_DISABLE_GPU)
457-
const auto& input = cc->Inputs().Tag("IMAGE_GPU").Get<GpuBuffer>();
464+
const auto& input = cc->Inputs().Tag(kGpuBufferTag).Get<GpuBuffer>();
458465
const int input_width = input.width();
459466
const int input_height = input.height();
460467

@@ -485,11 +492,11 @@ ::mediapipe::Status ImageTransformationCalculator::RenderGpu(
485492
{"video_frame_y", "video_frame_uv"}));
486493
}
487494
renderer = yuv_renderer_.get();
488-
src1 = helper_.CreateSourceTexture(input, 0);
495+
src1 = gpu_helper_.CreateSourceTexture(input, 0);
489496
} else // NOLINT(readability/braces)
490497
#endif // iOS
491498
{
492-
src1 = helper_.CreateSourceTexture(input);
499+
src1 = gpu_helper_.CreateSourceTexture(input);
493500
#if defined(TEXTURE_EXTERNAL_OES)
494501
if (src1.target() == GL_TEXTURE_EXTERNAL_OES) {
495502
if (!ext_rgb_renderer_) {
@@ -515,10 +522,10 @@ ::mediapipe::Status ImageTransformationCalculator::RenderGpu(
515522
mediapipe::FrameRotation rotation =
516523
mediapipe::FrameRotationFromDegrees(RotationModeToDegrees(rotation_));
517524

518-
auto dst = helper_.CreateDestinationTexture(output_width, output_height,
519-
input.format());
525+
auto dst = gpu_helper_.CreateDestinationTexture(output_width, output_height,
526+
input.format());
520527

521-
helper_.BindFramebuffer(dst); // GL_TEXTURE0
528+
gpu_helper_.BindFramebuffer(dst); // GL_TEXTURE0
522529
glActiveTexture(GL_TEXTURE1);
523530
glBindTexture(src1.target(), src1.name());
524531

@@ -533,8 +540,8 @@ ::mediapipe::Status ImageTransformationCalculator::RenderGpu(
533540
// Execute GL commands, before getting result.
534541
glFlush();
535542

536-
auto output = dst.GetFrame<GpuBuffer>();
537-
cc->Outputs().Tag("IMAGE_GPU").Add(output.release(), cc->InputTimestamp());
543+
auto output = dst.template GetFrame<GpuBuffer>();
544+
cc->Outputs().Tag(kGpuBufferTag).Add(output.release(), cc->InputTimestamp());
538545

539546
#endif // !MEDIAPIPE_DISABLE_GPU
540547

0 commit comments

Comments
 (0)