Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 2 additions & 10 deletions nemo_curator/stages/video/clipping/video_frame_extraction.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,17 +24,9 @@
from nemo_curator.stages.base import ProcessingStage
from nemo_curator.stages.resources import Resources
from nemo_curator.tasks.video import VideoTask
from nemo_curator.utils.nvcodec_utils import _PYNVC_AVAILABLE, PyNvcFrameExtractor
from nemo_curator.utils.operation_utils import make_pipeline_named_temporary_file

try:
from nemo_curator.utils.nvcodec_utils import PyNvcFrameExtractor

_PYNVC_AVAILABLE = True
except (ImportError, RuntimeError):
logger.warning("PyNvcFrameExtractor not available, PyNvCodec mode will fall back to FFmpeg")
PyNvcFrameExtractor = None
_PYNVC_AVAILABLE = False


def get_frames_from_ffmpeg(
video_file: Path,
Expand Down Expand Up @@ -123,7 +115,7 @@ def setup(self, worker_metadata: WorkerMetadata | None = None) -> None: # noqa:
worker_metadata (WorkerMetadata, optional): Information about the worker (provided by some backends)
"""
if self.decoder_mode == "pynvc":
if _PYNVC_AVAILABLE and PyNvcFrameExtractor is not None:
if _PYNVC_AVAILABLE:
self.pynvc_frame_extractor = PyNvcFrameExtractor(
width=self.output_hw[1],
height=self.output_hw[0],
Expand Down
13 changes: 7 additions & 6 deletions nemo_curator/utils/nvcodec_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,21 +22,21 @@

try:
import cvcuda # type: ignore[import-untyped]
import nvcv # type: ignore[import-untyped]
import pycuda.driver as cuda # type: ignore[import-untyped]
import PyNvVideoCodec as Nvc # type: ignore[import-untyped]

pixel_format_to_cvcuda_code = {
Nvc.Pixel_Format.YUV444: cvcuda.ColorConversion.YUV2RGB, # type: ignore[import-untyped]
Nvc.Pixel_Format.NV12: cvcuda.ColorConversion.YUV2RGB_NV12, # type: ignore[import-untyped]
}
_PYNVC_AVAILABLE = True
except (ImportError, RuntimeError):
logger.warning("PyNvVideoCodec is not installed, some features will be disabled.")
Nvc = None
cvcuda = None
nvcv = None
cuda = None
pixel_format_to_cvcuda_code = {}
_PYNVC_AVAILABLE = False


class FrameExtractionPolicy(enum.Enum):
Expand Down Expand Up @@ -268,17 +268,18 @@ def generate_decoded_frames(self) -> list[torch.Tensor]:
for packet in self.nvDemux:
list_frames = self.nvDec.Decode(packet)
for decoded_frame in list_frames:
nvcv_tensor = nvcv.as_tensor(nvcv.as_image(decoded_frame.nvcv_image(), nvcv.Format.U8))
if nvcv_tensor.layout == "NCHW":
nchw_shape = nvcv_tensor.shape
# TODO: Remove the use of nvcv_image. It's deprecated
cvcuda_tensor = cvcuda.as_tensor(cvcuda.as_image(decoded_frame.nvcv_image(), cvcuda.Format.U8))
Comment on lines +271 to +272

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Deprecated nvcv_image() method still in use

The TODO comment acknowledges nvcv_image() is deprecated, but the method is still called on the decoded frame object. If PyNvVideoCodec eventually removes nvcv_image() from its DecodedFrame API, this will fail at runtime with an AttributeError. The TODO should either track a GitHub issue or be resolved now by using the alternative non-deprecated frame access API exposed by PyNvVideoCodec.

if cvcuda_tensor.layout == "NCHW":
nchw_shape = cvcuda_tensor.shape
nhwc_shape = (nchw_shape[0], nchw_shape[2], nchw_shape[3], nchw_shape[1])
torch_nhwc = torch.empty(
nhwc_shape,
dtype=torch.uint8,
device=f"cuda:{self.device_id}",
)
cvcuda_nhwc = cvcuda.as_tensor(torch_nhwc.cuda(self.device_id), "NHWC")
cvcuda.reformat_into(cvcuda_nhwc, nvcv_tensor, stream=self.cvcuda_stream)
cvcuda.reformat_into(cvcuda_nhwc, cvcuda_tensor, stream=self.cvcuda_stream)
# Push the decoded frame with the reformatted frame to keep it alive.
self.input_frame_list.put(torch_nhwc)
else:
Expand Down
83 changes: 42 additions & 41 deletions tests/utils/test_nvcodec_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,6 @@ def test_video_batch_decoder_init_without_dependencies(self) -> None:

@patch("nemo_curator.utils.nvcodec_utils.cuda", None)
@patch("nemo_curator.utils.nvcodec_utils.cvcuda", None)
@patch("nemo_curator.utils.nvcodec_utils.nvcv", None)
@patch("nemo_curator.utils.nvcodec_utils.Nvc", None)
def test_py_nvc_frame_extractor_without_dependencies(self) -> None:
"""Test PyNvcFrameExtractor fails gracefully when dependencies are missing."""
Expand All @@ -92,7 +91,7 @@ def test_module_imports_gracefully_without_dependencies(self) -> None:
"""Test that the module can be imported even when GPU dependencies are missing."""
# If we got here, the import was successful
# This test verifies that import failures are handled gracefully
from nemo_curator.utils import nvcodec_utils
from nemo_curator.utils import nvcodec_utils # noqa: PLC0415, RUF100

# Verify the module has expected attributes
assert hasattr(nvcodec_utils, "FrameExtractionPolicy")
Expand Down Expand Up @@ -528,10 +527,7 @@ def test_init(self, _mock_logger: Any, mock_nvc: Any) -> None:
@patch("nemo_curator.utils.nvcodec_utils.Nvc")
@patch("nemo_curator.utils.nvcodec_utils.torch")
@patch("nemo_curator.utils.nvcodec_utils.cvcuda")
@patch("nemo_curator.utils.nvcodec_utils.nvcv")
def test_generate_decoded_frames(
self, _mock_nvcv: Any, _mock_cvcuda: Any, _mock_torch: Any, mock_nvc: Any
) -> None:
def test_generate_decoded_frames(self, _mock_cvcuda: Any, _mock_torch: Any, mock_nvc: Any) -> None:
"""Test generate_decoded_frames method."""
# Setup mocks
mock_demux = Mock()
Expand Down Expand Up @@ -655,10 +651,7 @@ def test_get_next_frames_multiple_frames(self, _mock_torch: Any, mock_nvc: Any)
@patch("nemo_curator.utils.nvcodec_utils.Nvc")
@patch("nemo_curator.utils.nvcodec_utils.torch")
@patch("nemo_curator.utils.nvcodec_utils.cvcuda")
@patch("nemo_curator.utils.nvcodec_utils.nvcv")
def test_generate_decoded_frames_with_frames(
self, mock_nvcv: Any, mock_cvcuda: Any, mock_torch: Any, mock_nvc: Any
) -> None:
def test_generate_decoded_frames_with_frames(self, mock_cvcuda: Any, mock_torch: Any, mock_nvc: Any) -> None:
"""Test generate_decoded_frames with actual frame processing."""
# Setup mocks
mock_demux = Mock()
Expand All @@ -677,20 +670,27 @@ def test_generate_decoded_frames_with_frames(
mock_decoded_frame.nvcv_image.return_value = Mock()

# Mock tensor operations
mock_nvcv_tensor = Mock()
mock_nvcv_tensor.layout = "NCHW"
mock_nvcv_tensor.shape = (1, 3, 480, 640) # NCHW format
mock_nvcv.as_tensor.return_value = mock_nvcv_tensor
mock_nvcv.as_image.return_value = Mock()
mock_nvcv.Format.U8 = Mock()
mock_cvcuda_tensor = Mock()
mock_cvcuda_tensor.layout = "NCHW"
mock_cvcuda_tensor.shape = (1, 3, 480, 640) # NCHW format
mock_cvcuda.as_tensor.return_value = mock_cvcuda_tensor
Comment thread
abhinavg4 marked this conversation as resolved.
Resolved
mock_cvcuda.as_image.return_value = Mock()
mock_cvcuda.Format.U8 = Mock()

# Mock torch tensor
mock_torch_nhwc = Mock()
mock_torch.empty.return_value = mock_torch_nhwc

# Mock cvcuda tensor
# Mock cvcuda tensor for NHWC conversion
mock_cvcuda_nhwc = Mock()
mock_cvcuda.as_tensor.return_value = mock_cvcuda_nhwc

# Setup side effect for as_tensor to return different mocks
def as_tensor_side_effect(*args: Any, **_kwargs: Any) -> Any:
if len(args) == 2 and isinstance(args[1], str):
return mock_cvcuda_nhwc
return mock_cvcuda_tensor
Comment thread
abhinavg4 marked this conversation as resolved.
Resolved

mock_cvcuda.as_tensor.side_effect = as_tensor_side_effect
Comment on lines +676 to +693

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

as_tensor initial return_value is shadowed by side_effect

At line 676, mock_cvcuda.as_tensor.return_value = mock_cvcuda_tensor is set, but is immediately overridden by mock_cvcuda.as_tensor.side_effect = as_tensor_side_effect at line 693. When a side_effect is set on a Mock, return_value is ignored entirely — Python's unittest.mock always invokes the side_effect callable instead. The line 676 assignment is dead code and should be removed to avoid confusing readers about which mock controls the return value.

Suggested change
mock_cvcuda.as_tensor.return_value = mock_cvcuda_tensor
mock_cvcuda.as_image.return_value = Mock()
mock_cvcuda.Format.U8 = Mock()
# Mock torch tensor
mock_torch_nhwc = Mock()
mock_torch.empty.return_value = mock_torch_nhwc
# Mock cvcuda tensor
# Mock cvcuda tensor for NHWC conversion
mock_cvcuda_nhwc = Mock()
mock_cvcuda.as_tensor.return_value = mock_cvcuda_nhwc
# Setup side effect for as_tensor to return different mocks
def as_tensor_side_effect(*args: Any, **_kwargs: Any) -> Any:
if len(args) == 2 and isinstance(args[1], str):
return mock_cvcuda_nhwc
return mock_cvcuda_tensor
mock_cvcuda.as_tensor.side_effect = as_tensor_side_effect
mock_cvcuda.as_image.return_value = Mock()
mock_cvcuda.Format.U8 = Mock()


# Mock demux iteration - return one packet with one frame
mock_demux.__iter__ = Mock(return_value=iter([mock_packet]))
Expand All @@ -707,9 +707,8 @@ def test_generate_decoded_frames_with_frames(
result = decoder.generate_decoded_frames()

# Verify frame processing was called
mock_nvcv.as_tensor.assert_called_once()
mock_cvcuda.as_tensor.assert_called()
mock_torch.empty.assert_called_once()
mock_cvcuda.as_tensor.assert_called_once()
mock_cvcuda.reformat_into.assert_called_once()

# Should return the processed frames
Expand All @@ -718,9 +717,8 @@ def test_generate_decoded_frames_with_frames(
@patch("nemo_curator.utils.nvcodec_utils.Nvc")
@patch("nemo_curator.utils.nvcodec_utils.torch")
@patch("nemo_curator.utils.nvcodec_utils.cvcuda")
@patch("nemo_curator.utils.nvcodec_utils.nvcv")
def test_generate_decoded_frames_unexpected_layout(
self, mock_nvcv: Any, _mock_cvcuda: Any, _mock_torch: Any, mock_nvc: Any
self, mock_cvcuda: Any, _mock_torch: Any, mock_nvc: Any
) -> None:
"""Test generate_decoded_frames with unexpected tensor layout."""
# Setup mocks
Expand All @@ -740,11 +738,11 @@ def test_generate_decoded_frames_unexpected_layout(
mock_decoded_frame.nvcv_image.return_value = Mock()

# Mock tensor with unexpected layout
mock_nvcv_tensor = Mock()
mock_nvcv_tensor.layout = "NHWC" # Unexpected layout - should be NCHW
mock_nvcv.as_tensor.return_value = mock_nvcv_tensor
mock_nvcv.as_image.return_value = Mock()
mock_nvcv.Format.U8 = Mock()
mock_cvcuda_tensor = Mock()
mock_cvcuda_tensor.layout = "NHWC" # Unexpected layout - should be NCHW
mock_cvcuda.as_tensor.return_value = mock_cvcuda_tensor
mock_cvcuda.as_image.return_value = Mock()
mock_cvcuda.Format.U8 = Mock()

# Mock demux iteration
mock_demux.__iter__ = Mock(return_value=iter([mock_packet]))
Expand All @@ -764,10 +762,7 @@ def test_generate_decoded_frames_unexpected_layout(
@patch("nemo_curator.utils.nvcodec_utils.Nvc")
@patch("nemo_curator.utils.nvcodec_utils.torch")
@patch("nemo_curator.utils.nvcodec_utils.cvcuda")
@patch("nemo_curator.utils.nvcodec_utils.nvcv")
def test_generate_decoded_frames_partial_batch(
self, mock_nvcv: Any, mock_cvcuda: Any, mock_torch: Any, mock_nvc: Any
) -> None:
def test_generate_decoded_frames_partial_batch(self, mock_cvcuda: Any, mock_torch: Any, mock_nvc: Any) -> None:
"""Test generate_decoded_frames with partial batch (less frames than batch_size)."""
# Setup mocks
mock_demux = Mock()
Expand All @@ -789,17 +784,24 @@ def test_generate_decoded_frames_partial_batch(
mock_frames.append(frame)

# Mock tensor operations
mock_nvcv_tensor = Mock()
mock_nvcv_tensor.layout = "NCHW"
mock_nvcv_tensor.shape = (1, 3, 480, 640)
mock_nvcv.as_tensor.return_value = mock_nvcv_tensor
mock_nvcv.as_image.return_value = Mock()
mock_nvcv.Format.U8 = Mock()
mock_cvcuda_tensor = Mock()
mock_cvcuda_tensor.layout = "NCHW"
mock_cvcuda_tensor.shape = (1, 3, 480, 640)
mock_cvcuda.as_tensor.return_value = mock_cvcuda_tensor
Comment thread
abhinavg4 marked this conversation as resolved.
Resolved
mock_cvcuda.as_image.return_value = Mock()
mock_cvcuda.Format.U8 = Mock()

mock_torch_nhwc = Mock()
mock_torch.empty.return_value = mock_torch_nhwc
mock_cvcuda_nhwc = Mock()
mock_cvcuda.as_tensor.return_value = mock_cvcuda_nhwc

# Setup side effect for as_tensor to return different mocks
def as_tensor_side_effect(*args: Any, **_kwargs: Any) -> Any:
if len(args) == 2 and isinstance(args[1], str):
return mock_cvcuda_nhwc
return mock_cvcuda_tensor
Comment thread
abhinavg4 marked this conversation as resolved.
Resolved

mock_cvcuda.as_tensor.side_effect = as_tensor_side_effect
Comment on lines +790 to +804

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same dead return_value assignment shadowed by side_effect

mock_cvcuda.as_tensor.return_value = mock_cvcuda_tensor at line 790 is set but immediately overridden when mock_cvcuda.as_tensor.side_effect = as_tensor_side_effect is assigned at line 804. As with test_generate_decoded_frames_with_frames, the return_value is never consulted once a side_effect is active, making the assignment dead code.

Suggested change
mock_cvcuda.as_tensor.return_value = mock_cvcuda_tensor
mock_cvcuda.as_image.return_value = Mock()
mock_cvcuda.Format.U8 = Mock()
mock_torch_nhwc = Mock()
mock_torch.empty.return_value = mock_torch_nhwc
mock_cvcuda_nhwc = Mock()
mock_cvcuda.as_tensor.return_value = mock_cvcuda_nhwc
# Setup side effect for as_tensor to return different mocks
def as_tensor_side_effect(*args: Any, **_kwargs: Any) -> Any:
if len(args) == 2 and isinstance(args[1], str):
return mock_cvcuda_nhwc
return mock_cvcuda_tensor
mock_cvcuda.as_tensor.side_effect = as_tensor_side_effect
mock_cvcuda.as_image.return_value = Mock()
mock_cvcuda.Format.U8 = Mock()


# Mock demux to return frames across multiple packets then end
decode_calls = []
Expand All @@ -825,9 +827,8 @@ def test_generate_decoded_frames_partial_batch(
assert len(result) == 2

# Verify frame processing was called
mock_nvcv.as_tensor.assert_called()
mock_torch.empty.assert_called()
mock_cvcuda.as_tensor.assert_called()
mock_torch.empty.assert_called()
mock_cvcuda.reformat_into.assert_called()


Expand Down Expand Up @@ -1178,7 +1179,7 @@ def test_error_messages_are_helpful(self) -> None:
def test_all_classes_can_be_imported(self) -> None:
"""Test that all public classes can be imported regardless of dependency availability."""
# All these should be importable even when dependencies are missing
from nemo_curator.utils.nvcodec_utils import (
from nemo_curator.utils.nvcodec_utils import ( # noqa: PLC0415, RUF100
FrameExtractionPolicy,
NvVideoDecoder,
PyNvcFrameExtractor,
Expand Down
Loading