Skip to content

Commit

Permalink
Merge pull request #1148 from bradh/shared_iref_2024-03-19
Browse files Browse the repository at this point in the history 
libheif: support for shared auxl alpha images
  • Loading branch information
@farindk
farindk authored Jun 11, 2024
2 parents 09ec676 + 9138f32 commit 58ba30d
Show file tree
Hide file tree
Showing 2 changed files with 104 additions and 126 deletions.
209 changes: 99 additions & 110 deletions libheif/context.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -573,8 +573,6 @@ Error HeifContext::interpret_heif_file()


// --- read through properties for each image and extract image resolutions
// Note: this has to be executed before assigning the auxiliary images below because we will only
// merge the alpha image with the main image when their resolutions are the same.

for (auto& pair : m_all_images) {
auto& image = pair.second;
Expand Down Expand Up @@ -656,34 +654,29 @@ Error HeifContext::interpret_heif_file()
// --- this is a thumbnail image, attach to the main image

std::vector<heif_item_id> refs = ref.to_item_ID;
if (refs.size() != 1) {
return Error(heif_error_Invalid_input,
heif_suberror_Unspecified,
"Too many thumbnail references");
}

image->set_is_thumbnail_of(refs[0]);
for (heif_item_id ref: refs) {
image->set_is_thumbnail();

auto master_iter = m_all_images.find(refs[0]);
if (master_iter == m_all_images.end()) {
return Error(heif_error_Invalid_input,
heif_suberror_Nonexisting_item_referenced,
"Thumbnail references a non-existing image");
}
auto master_iter = m_all_images.find(ref);
if (master_iter == m_all_images.end()) {
return Error(heif_error_Invalid_input,
heif_suberror_Nonexisting_item_referenced,
"Thumbnail references a non-existing image");
}

if (master_iter->second->is_thumbnail()) {
return Error(heif_error_Invalid_input,
heif_suberror_Nonexisting_item_referenced,
"Thumbnail references another thumbnail");
}
if (master_iter->second->is_thumbnail()) {
return Error(heif_error_Invalid_input,
heif_suberror_Nonexisting_item_referenced,
"Thumbnail references another thumbnail");
}

if (image.get() == master_iter->second.get()) {
return Error(heif_error_Invalid_input,
heif_suberror_Nonexisting_item_referenced,
"Recursive thumbnail image detected");
if (image.get() == master_iter->second.get()) {
return Error(heif_error_Invalid_input,
heif_suberror_Nonexisting_item_referenced,
"Recursive thumbnail image detected");
}
master_iter->second->add_thumbnail(image);
}
master_iter->second->add_thumbnail(image);

remove_top_level_image(image);
}
else if (type == fourcc("auxl")) {
Expand Down Expand Up @@ -714,39 +707,30 @@ Error HeifContext::interpret_heif_file()
}

std::vector<heif_item_id> refs = ref.to_item_ID;
if (refs.size() != 1) {
return Error(heif_error_Invalid_input,
heif_suberror_Unspecified,
"Too many auxiliary image references");
}


// alpha channel

if (auxC_property->get_aux_type() == "urn:mpeg:avc:2015:auxid:1" || // HEIF (avc)
auxC_property->get_aux_type() == "urn:mpeg:hevc:2015:auxid:1" || // HEIF (h265)
auxC_property->get_aux_type() == "urn:mpeg:mpegB:cicp:systems:auxiliary:alpha") { // MIAF

auto master_iter = m_all_images.find(refs[0]);
if (master_iter == m_all_images.end()) {
return Error(heif_error_Invalid_input,
heif_suberror_Nonexisting_item_referenced,
"Non-existing alpha image referenced");
}

auto master_img = master_iter->second;

if (image.get() == master_img.get()) {
return Error(heif_error_Invalid_input,
heif_suberror_Nonexisting_item_referenced,
"Recursive alpha image detected");
}
for (heif_item_id ref: refs) {
auto master_iter = m_all_images.find(ref);
if (master_iter == m_all_images.end()) {
return Error(heif_error_Invalid_input,
heif_suberror_Nonexisting_item_referenced,
"Non-existing alpha image referenced");
}

auto master_img = master_iter->second;

if (image->get_width() == master_img->get_width() &&
image->get_height() == master_img->get_height()) {
if (image.get() == master_img.get()) {
return Error(heif_error_Invalid_input,
heif_suberror_Nonexisting_item_referenced,
"Recursive alpha image detected");
}

image->set_is_alpha_channel_of(refs[0], true);
image->set_is_alpha_channel();
master_img->set_alpha_channel(image);
}
}
Expand All @@ -756,54 +740,58 @@ Error HeifContext::interpret_heif_file()

if (auxC_property->get_aux_type() == "urn:mpeg:hevc:2015:auxid:2" || // HEIF
auxC_property->get_aux_type() == "urn:mpeg:mpegB:cicp:systems:auxiliary:depth") { // AVIF
image->set_is_depth_channel_of(refs[0]);

auto master_iter = m_all_images.find(refs[0]);
if (master_iter == m_all_images.end()) {
return Error(heif_error_Invalid_input,
heif_suberror_Nonexisting_item_referenced,
"Non-existing depth image referenced");
}
if (image.get() == master_iter->second.get()) {
return Error(heif_error_Invalid_input,
heif_suberror_Nonexisting_item_referenced,
"Recursive depth image detected");
}
master_iter->second->set_depth_channel(image);
image->set_is_depth_channel();

for (heif_item_id ref: refs) {
auto master_iter = m_all_images.find(ref);
if (master_iter == m_all_images.end()) {
return Error(heif_error_Invalid_input,
heif_suberror_Nonexisting_item_referenced,
"Non-existing depth image referenced");
}
if (image.get() == master_iter->second.get()) {
return Error(heif_error_Invalid_input,
heif_suberror_Nonexisting_item_referenced,
"Recursive depth image detected");
}
master_iter->second->set_depth_channel(image);

auto subtypes = auxC_property->get_subtypes();
auto subtypes = auxC_property->get_subtypes();

std::vector<std::shared_ptr<SEIMessage>> sei_messages;
err = decode_hevc_aux_sei_messages(subtypes, sei_messages);
std::vector<std::shared_ptr<SEIMessage>> sei_messages;
err = decode_hevc_aux_sei_messages(subtypes, sei_messages);

for (auto& msg : sei_messages) {
auto depth_msg = std::dynamic_pointer_cast<SEIMessage_depth_representation_info>(msg);
if (depth_msg) {
image->set_depth_representation_info(*depth_msg);
for (auto& msg : sei_messages) {
auto depth_msg = std::dynamic_pointer_cast<SEIMessage_depth_representation_info>(msg);
if (depth_msg) {
image->set_depth_representation_info(*depth_msg);
}
}
}
}


// --- generic aux image

image->set_is_aux_image_of(refs[0], auxC_property->get_aux_type());
image->set_is_aux_image(auxC_property->get_aux_type());

auto master_iter = m_all_images.find(refs[0]);
if (master_iter == m_all_images.end()) {
return Error(heif_error_Invalid_input,
heif_suberror_Nonexisting_item_referenced,
"Non-existing aux image referenced");
}
if (image.get() == master_iter->second.get()) {
return Error(heif_error_Invalid_input,
heif_suberror_Nonexisting_item_referenced,
"Recursive aux image detected");
}
for (heif_item_id ref: refs) {
auto master_iter = m_all_images.find(ref);
if (master_iter == m_all_images.end()) {
return Error(heif_error_Invalid_input,
heif_suberror_Nonexisting_item_referenced,
"Non-existing aux image referenced");
}
if (image.get() == master_iter->second.get()) {
return Error(heif_error_Invalid_input,
heif_suberror_Nonexisting_item_referenced,
"Recursive aux image detected");
}

master_iter->second->add_aux_image(image);
master_iter->second->add_aux_image(image);

remove_top_level_image(image);
remove_top_level_image(image);
}
}
else {
// 'image' is a normal image, keep it as a top-level image
Expand Down Expand Up @@ -915,21 +903,17 @@ Error HeifContext::interpret_heif_file()
for (const auto& ref : references) {
if (ref.header.get_short_type() == fourcc("cdsc")) {
std::vector<uint32_t> refs = ref.to_item_ID;
if (refs.size() != 1) {
return Error(heif_error_Invalid_input,
heif_suberror_Unspecified,
"Metadata not correctly assigned to image");
}

uint32_t exif_image_id = refs[0];
auto img_iter = m_all_images.find(exif_image_id);
if (img_iter == m_all_images.end()) {
return Error(heif_error_Invalid_input,
heif_suberror_Nonexisting_item_referenced,
"Metadata assigned to non-existing image");
for(uint32_t ref: refs) {
uint32_t exif_image_id = ref;
auto img_iter = m_all_images.find(exif_image_id);
if (img_iter == m_all_images.end()) {
return Error(heif_error_Invalid_input,
heif_suberror_Nonexisting_item_referenced,
"Metadata assigned to non-existing image");
}
img_iter->second->add_metadata(metadata);
}

img_iter->second->add_metadata(metadata);
}
else if (ref.header.get_short_type() == fourcc("prem")) {
uint32_t color_image_id = ref.from_item_ID;
Expand Down Expand Up @@ -964,20 +948,17 @@ Error HeifContext::interpret_heif_file()
for (const auto& ref : references) {
if (ref.header.get_short_type() == fourcc("cdsc")) {
std::vector<uint32_t> refs = ref.to_item_ID;
if (refs.size() != 1) {
return Error(heif_error_Invalid_input,
heif_suberror_Unspecified,
"Region item not correctly assigned to image");
}
uint32_t image_id = refs[0];
auto img_iter = m_all_images.find(image_id);
if (img_iter == m_all_images.end()) {
return Error(heif_error_Invalid_input,
heif_suberror_Nonexisting_item_referenced,
"Region item assigned to non-existing image");
for (uint32_t ref: refs) {
uint32_t image_id = ref;
auto img_iter = m_all_images.find(image_id);
if (img_iter == m_all_images.end()) {
return Error(heif_error_Invalid_input,
heif_suberror_Nonexisting_item_referenced,
"Region item assigned to non-existing image");
}
img_iter->second->add_region_item_id(id);
m_region_items.push_back(region_item);
}
img_iter->second->add_region_item_id(id);
m_region_items.push_back(region_item);
}

/* When the geometry 'mask' of a region is represented by a mask stored in
Expand Down Expand Up @@ -1599,6 +1580,14 @@ Error HeifContext::decode_image_planar(heif_item_id ID,
heif_suberror_Unsupported_color_conversion);
}

if ((alpha_image->get_width() != img->get_width()) || (alpha_image->get_height() != img->get_height())) {
std::shared_ptr<HeifPixelImage> scaled_alpha;
err = alpha->scale_nearest_neighbor(scaled_alpha, img->get_width(), img->get_height());
if (err) {
return err;
}
alpha = std::move(scaled_alpha);
}
img->transfer_plane_from_image_as(alpha, channel, heif_channel_Alpha);

if (imginfo->is_premultiplied_alpha()) {
Expand Down
21 changes: 5 additions & 16 deletions libheif/context.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -134,10 +134,9 @@ class HeifContext : public ErrorBuffer

// -- thumbnails

void set_is_thumbnail_of(heif_item_id id)
void set_is_thumbnail()
{
m_is_thumbnail = true;
m_thumbnail_ref_id = id;
}

void add_thumbnail(const std::shared_ptr<Image>& img) { m_thumbnails.push_back(img); }
Expand All @@ -149,11 +148,9 @@ class HeifContext : public ErrorBuffer

// --- alpha channel

void set_is_alpha_channel_of(heif_item_id id, bool consumed)
void set_is_alpha_channel()
{
m_is_alpha_channel = true;
m_alpha_channel_ref_id = id;
m_implicitly_consumed_alpha = consumed;
}

void set_alpha_channel(std::shared_ptr<Image> img) { m_alpha_channel = std::move(img); }
Expand All @@ -169,10 +166,9 @@ class HeifContext : public ErrorBuffer

// --- depth channel

void set_is_depth_channel_of(heif_item_id id)
void set_is_depth_channel()
{
m_is_depth_channel = true;
m_depth_channel_ref_id = id;
}

void set_depth_channel(std::shared_ptr<Image> img) { m_depth_channel = std::move(img); }
Expand Down Expand Up @@ -201,10 +197,9 @@ class HeifContext : public ErrorBuffer

// --- generic aux image

void set_is_aux_image_of(heif_item_id id, const std::string& aux_type)
void set_is_aux_image(const std::string& aux_type)
{
m_is_aux_image = true;
m_aux_image_ref_id = id;
m_aux_image_type = aux_type;
}

Expand All @@ -222,8 +217,7 @@ class HeifContext : public ErrorBuffer
else {
std::vector<std::shared_ptr<Image>> auxImgs;
for (const auto& aux : m_aux_images) {
if ((aux_image_filter & LIBHEIF_AUX_IMAGE_FILTER_OMIT_ALPHA) &&
aux->is_alpha_channel() && aux->m_implicitly_consumed_alpha) {
if ((aux_image_filter & LIBHEIF_AUX_IMAGE_FILTER_OMIT_ALPHA) && aux->is_alpha_channel()) {
continue;
}

Expand Down Expand Up @@ -290,24 +284,19 @@ class HeifContext : public ErrorBuffer
bool m_is_primary = false;

bool m_is_thumbnail = false;
heif_item_id m_thumbnail_ref_id = 0;

std::vector<std::shared_ptr<Image>> m_thumbnails;

bool m_is_alpha_channel = false;
bool m_premultiplied_alpha = false;
bool m_implicitly_consumed_alpha = false; // alpha data was integrated into main color image
heif_item_id m_alpha_channel_ref_id = 0;
std::shared_ptr<Image> m_alpha_channel;

bool m_is_depth_channel = false;
heif_item_id m_depth_channel_ref_id = 0;
std::shared_ptr<Image> m_depth_channel;

bool m_has_depth_representation_info = false;
struct heif_depth_representation_info m_depth_representation_info;

heif_item_id m_aux_image_ref_id = 0;
bool m_is_aux_image = false;
std::string m_aux_image_type;
std::vector<std::shared_ptr<Image>> m_aux_images;
Expand Down

0 comments on commit 58ba30d

Please sign in to comment.