About DensePose

[leviome]

Thanks for sharing the nice work!

I'm doing experiments with customed data. But I'm not clear with "UV unwrap" which is handled by DensePose.

├── densepose // Store the pre-defined UV unwrap. The directory name must be the same as the densepose directory name in the config file.
├── Camera_B1
├── 00000_IUV.png

Could you please give me a sample of 00000_IUV.png?

[smontode24]

Hi @leviome , I am extracting the IUV map as well with detectron 2, the images look like this in my case:

With detectron 2 I created a script to extract the IUV maps from the .pkl files that detectron2 generates, here's the code, you can adapt it to your needs:

import numpy as np
import torch
from PIL import Image
import pickle
from os.path import join
import os
from tqdm import tqdm

"""
Process densepose extracted data of IUV maps (in .pkl files) into IUV images in the ZJUMoCap folder structure.
 - DensePose: https://github.com/facebookresearch/detectron2/blob/main/projects/DensePose/README.md
 - ZJUMoCap structure: https://github.com/fanegg/UV-Volumes/blob/main/INSTALL.md#get-the-pre-defined-uv-unwrap-in-densepose
"""

seq_name = "CoreView_313"
path_to_seq = f"/home/sergio/data/human/zjumocap/{seq_name}/densepose" # 
path_to_densepose_data = "/home/sergio/data/densepose_raw/" # Path to pkl files
cam_idxs = list(np.arange(1,20)) + [22, 23]

for idx in cam_idxs:
    seq_cam_name = "Camera (" + str(idx) + ")" # Structure from sequences 313, 315, ... (There are others with different ones)
    cam_densepose_path = join(path_to_seq, seq_cam_name)
    os.makedirs(cam_densepose_path, exist_ok=True)
    densepose_data = torch.load(join(path_to_densepose_data, f'cam_{idx}.pkl'), map_location="cpu")

    for img_idx in tqdm(range(len(densepose_data)), desc=f"{seq_name}: Extracting cam. {idx}"):
        # Extract IUV map from pkl
        dp_item = densepose_data[img_idx]
        i = dp_item['pred_densepose'][0].labels.cpu().numpy()
        uv = dp_item['pred_densepose'][0].uv.cpu().numpy()
        iuv = np.stack((uv[1,:,:], uv[0,:,:], i))
        iuv = np.transpose(iuv, (1,2,0))
        iuv_img = Image.fromarray(np.uint8(iuv*255), "RGB")
        
        # iuv_img.show()

        # Show person IUV with original image size, at its corresponding position and save to 
        # corresponding file in the folder structure
        box = dp_item["pred_boxes_XYXY"][0]
        box[2] = box[2] - box[0]
        box[3] = box[3] - box[1]
        x, y, w, h = [int(v) for v in box]

        img_filename = dp_item["file_name"]
        img = Image.open(img_filename)
        img_w, img_h = img.size
        
        bg=np.zeros((img_h, img_w, 3))
        bg[y:y+h, x:x+w, :]=iuv
        bg_img = Image.fromarray(np.uint8(bg*255), "RGB")
        name_iuv_file = img_filename.split("/")[-1]
        name_iuv_file = name_iuv_file[:-4]+"_IUV.jpg"
        path_to_save_IUV = join(cam_densepose_path, name_iuv_file)
        bg_img.save(path_to_save_IUV)

[fanegg]

Hi @leviome, you can download the sample from here.

[fanegg]

Hi @leviome , I am extracting the IUV map as well with detectron 2, the images look like this in my case:

With detectron 2 I created a script to extract the IUV maps from the .pkl files that detectron2 generates, here's the code, you can adapt it to your needs:

import numpy as np
import torch
from PIL import Image
import pickle
from os.path import join
import os
from tqdm import tqdm

"""
Process densepose extracted data of IUV maps (in .pkl files) into IUV images in the ZJUMoCap folder structure.
 - DensePose: https://github.com/facebookresearch/detectron2/blob/main/projects/DensePose/README.md
 - ZJUMoCap structure: https://github.com/fanegg/UV-Volumes/blob/main/INSTALL.md#get-the-pre-defined-uv-unwrap-in-densepose
"""

seq_name = "CoreView_313"
path_to_seq = f"/home/sergio/data/human/zjumocap/{seq_name}/densepose" # 
path_to_densepose_data = "/home/sergio/data/densepose_raw/" # Path to pkl files
cam_idxs = list(np.arange(1,20)) + [22, 23]

for idx in cam_idxs:
    seq_cam_name = "Camera (" + str(idx) + ")" # Structure from sequences 313, 315, ... (There are others with different ones)
    cam_densepose_path = join(path_to_seq, seq_cam_name)
    os.makedirs(cam_densepose_path, exist_ok=True)
    densepose_data = torch.load(join(path_to_densepose_data, f'cam_{idx}.pkl'), map_location="cpu")

    for img_idx in tqdm(range(len(densepose_data)), desc=f"{seq_name}: Extracting cam. {idx}"):
        # Extract IUV map from pkl
        dp_item = densepose_data[img_idx]
        i = dp_item['pred_densepose'][0].labels.cpu().numpy()
        uv = dp_item['pred_densepose'][0].uv.cpu().numpy()
        iuv = np.stack((uv[1,:,:], uv[0,:,:], i))
        iuv = np.transpose(iuv, (1,2,0))
        iuv_img = Image.fromarray(np.uint8(iuv*255), "RGB")
        
        # iuv_img.show()

        # Show person IUV with original image size, at its corresponding position and save to 
        # corresponding file in the folder structure
        box = dp_item["pred_boxes_XYXY"][0]
        box[2] = box[2] - box[0]
        box[3] = box[3] - box[1]
        x, y, w, h = [int(v) for v in box]

        img_filename = dp_item["file_name"]
        img = Image.open(img_filename)
        img_w, img_h = img.size
        
        bg=np.zeros((img_h, img_w, 3))
        bg[y:y+h, x:x+w, :]=iuv
        bg_img = Image.fromarray(np.uint8(bg*255), "RGB")
        name_iuv_file = img_filename.split("/")[-1]
        name_iuv_file = name_iuv_file[:-4]+"_IUV.jpg"
        path_to_save_IUV = join(cam_densepose_path, name_iuv_file)
        bg_img.save(path_to_save_IUV)

Excellent! Thank you for your sharing. I think it would save others a lot of time!

[fanzhongyi]

Hi @leviome , I am extracting the IUV map as well with detectron 2, the images look like this in my case:

With detectron 2 I created a script to extract the IUV maps from the .pkl files that detectron2 generates, here's the code, you can adapt it to your needs:

import numpy as np
import torch
from PIL import Image
import pickle
from os.path import join
import os
from tqdm import tqdm

"""
Process densepose extracted data of IUV maps (in .pkl files) into IUV images in the ZJUMoCap folder structure.
 - DensePose: https://github.com/facebookresearch/detectron2/blob/main/projects/DensePose/README.md
 - ZJUMoCap structure: https://github.com/fanegg/UV-Volumes/blob/main/INSTALL.md#get-the-pre-defined-uv-unwrap-in-densepose
"""

seq_name = "CoreView_313"
path_to_seq = f"/home/sergio/data/human/zjumocap/{seq_name}/densepose" # 
path_to_densepose_data = "/home/sergio/data/densepose_raw/" # Path to pkl files
cam_idxs = list(np.arange(1,20)) + [22, 23]

for idx in cam_idxs:
    seq_cam_name = "Camera (" + str(idx) + ")" # Structure from sequences 313, 315, ... (There are others with different ones)
    cam_densepose_path = join(path_to_seq, seq_cam_name)
    os.makedirs(cam_densepose_path, exist_ok=True)
    densepose_data = torch.load(join(path_to_densepose_data, f'cam_{idx}.pkl'), map_location="cpu")

    for img_idx in tqdm(range(len(densepose_data)), desc=f"{seq_name}: Extracting cam. {idx}"):
        # Extract IUV map from pkl
        dp_item = densepose_data[img_idx]
        i = dp_item['pred_densepose'][0].labels.cpu().numpy()
        uv = dp_item['pred_densepose'][0].uv.cpu().numpy()
        iuv = np.stack((uv[1,:,:], uv[0,:,:], i))
        iuv = np.transpose(iuv, (1,2,0))
        iuv_img = Image.fromarray(np.uint8(iuv*255), "RGB")
        
        # iuv_img.show()

        # Show person IUV with original image size, at its corresponding position and save to 
        # corresponding file in the folder structure
        box = dp_item["pred_boxes_XYXY"][0]
        box[2] = box[2] - box[0]
        box[3] = box[3] - box[1]
        x, y, w, h = [int(v) for v in box]

        img_filename = dp_item["file_name"]
        img = Image.open(img_filename)
        img_w, img_h = img.size
        
        bg=np.zeros((img_h, img_w, 3))
        bg[y:y+h, x:x+w, :]=iuv
        bg_img = Image.fromarray(np.uint8(bg*255), "RGB")
        name_iuv_file = img_filename.split("/")[-1]
        name_iuv_file = name_iuv_file[:-4]+"_IUV.jpg"
        path_to_save_IUV = join(cam_densepose_path, name_iuv_file)
        bg_img.save(path_to_save_IUV)

I am confused about the Code snippets:

iuv = np.stack((uv[1,:,:], uv[0,:,:], i))
	...
bg=np.zeros((img_h, img_w, 3))
bg[y:y+h, x:x+w, :]=iuv
bg_img = Image.fromarray(np.uint8(bg*255), "RGB")

The datatype of block index I is Integer, which will be erased when saving *_IUV.jpg file? I'm not sure if I misunderstood ? Or need another processing when loading to Dataset ?

[swanilee]

Thank you for your code @smontode24.

Then, If we use that code as is, we need additional pre-process with images.
So, I modified the code a bit.

If you use this code right away, it works according to the code without any problems.

import numpy as np
import torch
from PIL import Image
import pickle
from os.path import join
import os
from tqdm import tqdm
import cv2

"""
Process densepose extracted data of IUV maps (in .pkl files) into IUV images in the ZJUMoCap folder structure.
 - DensePose: https://github.com/facebookresearch/detectron2/blob/main/projects/DensePose/README.md
 - ZJUMoCap structure: https://github.com/fanegg/UV-Volumes/blob/main/INSTALL.md#get-the-pre-defined-uv-unwrap-in-densepose
"""

seq_name = "CoreView_313"
path_to_seq = f"/EasyMocap/{seq_name}/densepose" # 
path_to_densepose_data = "/EasyMocap/CoreView_313/densepose/" # Path to pkl files
cam_idxs = list(np.arange(1,20)) + [22, 23]

for idx in cam_idxs:
    seq_cam_name = "Camera (" + str(idx) + ")" # Structure from sequences 313, 315, ... (There are others with different ones)
    cam_densepose_path = join(path_to_seq, seq_cam_name)
    os.makedirs(cam_densepose_path, exist_ok=True)
    densepose_data = torch.load(join(path_to_densepose_data, f'cam_{idx}.pkl'), map_location="cpu")

    for img_idx in tqdm(range(len(densepose_data)), desc=f"{seq_name}: Extracting cam. {idx}"):
        # Extract IUV map from pkl
        dp_item = densepose_data[img_idx]
        dp_item['file_name'] = "/EasyMocap/"+'/'.join(dp_item['file_name'].split('/')[-4:])
        i = dp_item['pred_densepose'][0].labels.cpu().numpy()
        uv = dp_item['pred_densepose'][0].uv.cpu().numpy()
        iuv = np.stack((uv[1,:,:], uv[0,:,:], i))
        iuv = np.transpose(iuv, (1,2,0))
        iuv_img = Image.fromarray(np.uint8(iuv*255), "RGB")
        
        # iuv_img.show()

        # Show person IUV with original image size, at its corresponding position and save to 
        # corresponding file in the folder structure
        box = dp_item["pred_boxes_XYXY"][0]
        box[2] = box[2] - box[0]
        box[3] = box[3] - box[1]
        x, y, w, h = [int(v) for v in box]

        img_filename = dp_item["file_name"]
        img = Image.open(img_filename)
        img_w, img_h = img.size
        
        bg=np.zeros((img_h, img_w, 3))
        bg[y:y+h, x:x+w, :]=iuv
        bg[:,:,:2] = bg[:,:,:2] * 255
        bg = np.uint8(bg)[:,:,::-1]

        name_iuv_file = img_filename.split("/")[-1]
        name_iuv_file = name_iuv_file[:-4]+"_IUV.png"
        path_to_save_IUV = join(cam_densepose_path, name_iuv_file)

        cv2.imwrite(path_to_save_IUV, bg)

😀 Result IUV Image :