From 7728468f5e5993ec6fbcf68dd4c213599a664788 Mon Sep 17 00:00:00 2001 From: Bruno Martin Date: Sun, 5 Apr 2026 00:47:08 +0200 Subject: [PATCH] Add MetalFX Video Super Resolution for iOS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit adds an optional Video Super Resolution path for iOS/tvOS. - Introduce a Video​Super​Resolution renderer backed by Metal and Core Image. - Convert decoded YUV frames to RGB and upscale them with MetalFX SpatialScaler. - Preserve the existing rendering path as the default fallback. - Add a user-facing toggle and hide it when MetalFX. - Show "Video ​Enhancement: ​Metal​FX " in the performance overlay when active. - Run the VSR path on a dedicated queue and cache session resources for better latency. - Preserve HDR/color attachments through the converted output frames. --- Limelight/Database/TemporarySettings.h | 1 + Limelight/Database/TemporarySettings.m | 2 + Limelight/Stream/StreamManager.m | 5 +- Limelight/Stream/VideoDecoderRenderer.m | 236 +++++- Limelight/Stream/VideoSuperResolution.h | 29 + Limelight/Stream/VideoSuperResolution.m | 766 ++++++++++++++++++ .../ViewControllers/SettingsViewController.h | 2 + .../ViewControllers/SettingsViewController.m | 33 + Moonlight.xcodeproj/project.pbxproj | 10 + iPad.storyboard | 79 +- iPhone.storyboard | 73 +- 11 files changed, 1170 insertions(+), 66 deletions(-) create mode 100644 Limelight/Stream/VideoSuperResolution.h create mode 100644 Limelight/Stream/VideoSuperResolution.m diff --git a/Limelight/Database/TemporarySettings.h b/Limelight/Database/TemporarySettings.h index da35b30c1..ba5e924c0 100644 --- a/Limelight/Database/TemporarySettings.h +++ b/Limelight/Database/TemporarySettings.h @@ -31,6 +31,7 @@ @property (nonatomic) BOOL playAudioOnPC; @property (nonatomic) BOOL optimizeGames; @property (nonatomic) BOOL enableHdr; +@property (nonatomic) BOOL videoSuperResolution; @property (nonatomic) BOOL btMouseSupport; @property (nonatomic) BOOL absoluteTouchMode; @property (nonatomic) BOOL statsOverlay; diff --git a/Limelight/Database/TemporarySettings.m b/Limelight/Database/TemporarySettings.m index e627e0c1f..5c8d43924 100644 --- a/Limelight/Database/TemporarySettings.m +++ b/Limelight/Database/TemporarySettings.m @@ -40,6 +40,7 @@ - (id) initFromSettings:(Settings*)settings { self.useFramePacing = [[NSUserDefaults standardUserDefaults] integerForKey:@"useFramePacing"] != 0; self.playAudioOnPC = [[NSUserDefaults standardUserDefaults] boolForKey:@"audioOnPC"]; self.enableHdr = [[NSUserDefaults standardUserDefaults] boolForKey:@"enableHdr"]; + self.videoSuperResolution = [[NSUserDefaults standardUserDefaults] boolForKey:@"videoSuperResolution"]; self.optimizeGames = [[NSUserDefaults standardUserDefaults] boolForKey:@"optimizeGames"]; self.multiController = [[NSUserDefaults standardUserDefaults] boolForKey:@"multipleControllers"]; self.swapABXYButtons = [[NSUserDefaults standardUserDefaults] boolForKey:@"swapABXYButtons"]; @@ -78,6 +79,7 @@ - (id) initFromSettings:(Settings*)settings { self.useFramePacing = settings.useFramePacing; self.playAudioOnPC = settings.playAudioOnPC; self.enableHdr = settings.enableHdr; + self.videoSuperResolution = [[NSUserDefaults standardUserDefaults] boolForKey:@"videoSuperResolution"]; self.optimizeGames = settings.optimizeGames; self.multiController = settings.multiController; self.swapABXYButtons = settings.swapABXYButtons; diff --git a/Limelight/Stream/StreamManager.m b/Limelight/Stream/StreamManager.m index e776feef0..3c8e0f71f 100644 --- a/Limelight/Stream/StreamManager.m +++ b/Limelight/Stream/StreamManager.m @@ -149,6 +149,7 @@ - (BOOL) resumeApp:(HttpManager*)hMan receiveSessionUrl:(NSString**)sessionUrl { - (NSString*) getStatsOverlayText { video_stats_t stats; + BOOL videoSuperResolutionEnabled = [[NSUserDefaults standardUserDefaults] boolForKey:@"videoSuperResolution"]; if (!_connection) { return nil; @@ -179,11 +180,13 @@ - (NSString*) getStatsOverlayText { } float interval = stats.endTime - stats.startTime; - return [NSString stringWithFormat:@"Video stream: %dx%d %.2f FPS (Codec: %@)\nFrames dropped by your network connection: %.2f%%\nAverage network latency: %@%@", + NSString* videoEnhancementString = videoSuperResolutionEnabled ? @"\nVideo Enhancement: MetalFX" : @""; + return [NSString stringWithFormat:@"Video stream: %dx%d %.2f FPS (Codec: %@)%@\nFrames dropped by your network connection: %.2f%%\nAverage network latency: %@%@", _config.width, _config.height, stats.totalFrames / interval, [_connection getActiveCodecName], + videoEnhancementString, stats.networkDroppedFrames / interval, latencyString, hostProcessingString]; diff --git a/Limelight/Stream/VideoDecoderRenderer.m b/Limelight/Stream/VideoDecoderRenderer.m index a2adf08db..ed4b17192 100644 --- a/Limelight/Stream/VideoDecoderRenderer.m +++ b/Limelight/Stream/VideoDecoderRenderer.m @@ -8,6 +8,9 @@ #import "VideoDecoderRenderer.h" #import "StreamView.h" +#import "VideoSuperResolution.h" + +@import VideoToolbox; #include #include @@ -19,6 +22,8 @@ extern int ff_isom_write_av1c(AVIOContext *pb, const uint8_t *buf, int size, int write_seq_header); +static const void* kVideoSuperResolutionQueueKey = &kVideoSuperResolutionQueueKey; + @implementation VideoDecoderRenderer { StreamView* _view; id _callbacks; @@ -27,11 +32,19 @@ @implementation VideoDecoderRenderer { AVSampleBufferDisplayLayer* displayLayer; int videoFormat; int frameRate; + int videoWidth; + int videoHeight; NSMutableArray *parameterSetBuffers; NSData *masteringDisplayColorVolume; NSData *contentLightLevelInfo; CMVideoFormatDescriptionRef formatDesc; + VideoSuperResolution* _videoSuperResolution; + BOOL _useVideoSuperResolution; + BOOL _videoSuperResolutionHdrEnabled; + dispatch_queue_t _videoSuperResolutionQueue; + VTDecompressionSessionRef _decompressionSession; + CGSize _videoSuperResolutionTargetSize; CADisplayLink* _displayLink; BOOL framePacing; @@ -85,18 +98,199 @@ - (id)initWithView:(StreamView*)view callbacks:(id)callback _callbacks = callbacks; _streamAspectRatio = aspectRatio; framePacing = useFramePacing; + _useVideoSuperResolution = [[NSUserDefaults standardUserDefaults] boolForKey:@"videoSuperResolution"]; parameterSetBuffers = [[NSMutableArray alloc] init]; + if (_useVideoSuperResolution) { + _videoSuperResolution = [[VideoSuperResolution alloc] init]; + // Build the long-lived GPU state once up front. Size-dependent resources are configured later. + [_videoSuperResolution initializeResources]; + [_videoSuperResolution setHdrEnabled:NO]; + + // Keep the optional VSR decode/conversion path off the main thread. + _videoSuperResolutionQueue = dispatch_queue_create("com.moonlight.VideoSuperResolution", DISPATCH_QUEUE_SERIAL); + + // Tag the queue so we can safely invalidate the VT session from either the queue itself + // or another thread without introducing a cross-thread lifetime bug. + dispatch_queue_set_specific(_videoSuperResolutionQueue, kVideoSuperResolutionQueueKey, (void*)kVideoSuperResolutionQueueKey, NULL); + } [self reinitializeDisplayLayer]; return self; } +- (CGSize)videoSuperResolutionTargetSize +{ + __block CGRect displayBounds = CGRectZero; + __block CGFloat screenScale = 1.0; + void (^readUIState)(void) = ^{ + UIScreen* screen = self->_view.window.screen ?: UIScreen.mainScreen; + + // The display layer bounds define the actual presentation size after aspect-ratio fitting. + displayBounds = self->displayLayer.bounds; + screenScale = screen.nativeScale > 0.0 ? screen.nativeScale : screen.scale; + }; + + if ([NSThread isMainThread]) { + readUIState(); + } + else { + dispatch_sync(dispatch_get_main_queue(), readUIState); + } + + return CGSizeMake(MAX(1.0, CGRectGetWidth(displayBounds) * screenScale), + MAX(1.0, CGRectGetHeight(displayBounds) * screenScale)); +} + +- (void)invalidateVideoSuperResolutionDecoder +{ + if (_videoSuperResolutionQueue == NULL) { + if (_decompressionSession != NULL) { + VTDecompressionSessionInvalidate(_decompressionSession); + CFRelease(_decompressionSession); + _decompressionSession = NULL; + } + return; + } + + // The decompression session is owned by the VSR queue, so all invalidation must run there. + void (^invalidateBlock)(void) = ^{ + if (self->_decompressionSession != NULL) { + VTDecompressionSessionInvalidate(self->_decompressionSession); + CFRelease(self->_decompressionSession); + self->_decompressionSession = NULL; + } + }; + + if (dispatch_get_specific(kVideoSuperResolutionQueueKey) == kVideoSuperResolutionQueueKey) { + invalidateBlock(); + } + else { + dispatch_sync(_videoSuperResolutionQueue, invalidateBlock); + } +} + +- (BOOL)setupVideoSuperResolutionDecoder +{ + if (!_useVideoSuperResolution || formatDesc == NULL) { + return NO; + } + + // Rebuild the secondary decode session when the stream format or HDR mode changes. + [self invalidateVideoSuperResolutionDecoder]; + + NSDictionary* imageBufferAttributes = @{ + // Request a Metal-compatible decoder output so VideoSuperResolution can wrap it directly + // as a CVMetalTexture without an extra CPU copy. + (id)kCVPixelBufferPixelFormatTypeKey : @(_videoSuperResolutionHdrEnabled ? kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange : kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange), + (id)kCVPixelBufferMetalCompatibilityKey : @YES, + (id)kCVPixelBufferIOSurfacePropertiesKey : @{}, + }; + + OSStatus status = VTDecompressionSessionCreate(kCFAllocatorDefault, + formatDesc, + NULL, + (__bridge CFDictionaryRef)imageBufferAttributes, + NULL, + &_decompressionSession); + if (status != noErr || _decompressionSession == NULL) { + Log(LOG_E, @"Failed to create VideoSuperResolution decompression session: %d", (int)status); + [self invalidateVideoSuperResolutionDecoder]; + return NO; + } + + return YES; +} + +- (BOOL)enqueueVideoSuperResolutionSampleBuffer:(CMSampleBufferRef)sampleBuffer decodeUnit:(PDECODE_UNIT)du +{ + BOOL isIDRFrame = du->frameType == FRAME_TYPE_IDR; + CFRetain(sampleBuffer); + + dispatch_async(_videoSuperResolutionQueue, ^{ + @autoreleasepool { + // Lazily create the secondary VT session on the queue that will own and use it. + if (self->_decompressionSession == NULL && ![self setupVideoSuperResolutionDecoder]) { + CFRelease(sampleBuffer); + return; + } + + __block CMSampleBufferRef rgbSampleBuffer = NULL; + __block BOOL decodeCompleted = NO; + VTDecodeInfoFlags infoFlags = 0; + + // Decode into a Metal-compatible pixel buffer, then hand that image buffer to the + // VSR path for YUV->RGB conversion and optional MetalFX scaling. + OSStatus status = VTDecompressionSessionDecodeFrameWithOutputHandler(self->_decompressionSession, + sampleBuffer, + 0, + &infoFlags, + ^(OSStatus decodeStatus, + VTDecodeInfoFlags decodeInfoFlags, + CVImageBufferRef _Nullable imageBuffer, + CMTime presentationTimeStamp, + CMTime presentationDuration) { + decodeCompleted = YES; + + if (decodeStatus != noErr || imageBuffer == nil) { + Log(LOG_E, @"VideoSuperResolution decode failed: %d flags: %u", (int)decodeStatus, (unsigned int)decodeInfoFlags); + return; + } + + // The source sample buffer is passed through so the RGB output can inherit its + // HDR and color attachments before it is enqueued for display. + rgbSampleBuffer = [self->_videoSuperResolution copyRGBSampleBufferFromImageBuffer:imageBuffer + sourceSampleBuffer:sampleBuffer + presentationTimeStamp:presentationTimeStamp + duration:presentationDuration]; + }); + + CFRelease(sampleBuffer); + + if (status != noErr) { + Log(LOG_E, @"VTDecompressionSessionDecodeFrame failed: %d", (int)status); + [self invalidateVideoSuperResolutionDecoder]; + return; + } + + if (!decodeCompleted || rgbSampleBuffer == NULL) { + return; + } + + dispatch_async(dispatch_get_main_queue(), ^{ + // AVSampleBufferDisplayLayer remains a main-thread-facing object even though + // decoding and conversion happen on the dedicated VSR queue. + [self->displayLayer enqueueSampleBuffer:rgbSampleBuffer]; + + if (isIDRFrame) { + self->displayLayer.hidden = NO; + [self->_callbacks videoContentShown]; + } + + CFRelease(rgbSampleBuffer); + }); + } + }); + + return YES; +} + - (void)setupWithVideoFormat:(int)videoFormat width:(int)videoWidth height:(int)videoHeight frameRate:(int)frameRate { self->videoFormat = videoFormat; + self->videoWidth = videoWidth; + self->videoHeight = videoHeight; self->frameRate = frameRate; + + if (_useVideoSuperResolution) { + _videoSuperResolutionTargetSize = [self videoSuperResolutionTargetSize]; + [_videoSuperResolution setHdrEnabled:_videoSuperResolutionHdrEnabled]; + if (![_videoSuperResolution configureWithInputSize:CGSizeMake(self->videoWidth, self->videoHeight) + outputSize:_videoSuperResolutionTargetSize]) { + Log(LOG_W, @"VideoSuperResolution session resource configuration failed"); + } + } } - (void)start @@ -142,6 +336,12 @@ - (void)displayLinkCallback:(CADisplayLink *)sender - (void)stop { + if (_videoSuperResolutionQueue != NULL) { + dispatch_sync(_videoSuperResolutionQueue, ^{ + }); + } + + [self invalidateVideoSuperResolutionDecoder]; [_displayLink invalidate]; } @@ -507,6 +707,10 @@ - (int)submitDecodeBuffer:(unsigned char *)data length:(int)length bufferType:(i // Unsupported codec! abort(); } + + if (_useVideoSuperResolution && ![self setupVideoSuperResolutionDecoder]) { + Log(LOG_W, @"VideoSuperResolution setup failed. Falling back to default display pipeline."); + } } if (formatDesc == NULL) { @@ -594,15 +798,18 @@ - (int)submitDecodeBuffer:(unsigned char *)data length:(int)length bufferType:(i return DR_NEED_IDR; } - // Enqueue the next frame - [self->displayLayer enqueueSampleBuffer:sampleBuffer]; - - if (du->frameType == FRAME_TYPE_IDR) { - // Ensure the layer is visible now - self->displayLayer.hidden = NO; + if (!_useVideoSuperResolution || ![self enqueueVideoSuperResolutionSampleBuffer:sampleBuffer decodeUnit:du]) { + // Enqueue the next frame on the existing path if VideoSuperResolution is disabled + // or if the optional YUV-to-RGB path couldn't process this frame. + [self->displayLayer enqueueSampleBuffer:sampleBuffer]; - // Tell our parent VC to hide the progress indicator - [self->_callbacks videoContentShown]; + if (du->frameType == FRAME_TYPE_IDR) { + // Ensure the layer is visible now + self->displayLayer.hidden = NO; + + // Tell our parent VC to hide the progress indicator + [self->_callbacks videoContentShown]; + } } // Dereference the buffers @@ -616,6 +823,19 @@ - (int)submitDecodeBuffer:(unsigned char *)data length:(int)length bufferType:(i - (void)setHdrMode:(BOOL)enabled { SS_HDR_METADATA hdrMetadata; + if (_useVideoSuperResolution && _videoSuperResolutionHdrEnabled != enabled) { + _videoSuperResolutionHdrEnabled = enabled; + [_videoSuperResolution setHdrEnabled:enabled]; + + if (displayLayer != nil && videoWidth > 0 && videoHeight > 0) { + _videoSuperResolutionTargetSize = [self videoSuperResolutionTargetSize]; + [_videoSuperResolution configureWithInputSize:CGSizeMake(videoWidth, videoHeight) + outputSize:_videoSuperResolutionTargetSize]; + } + + [self invalidateVideoSuperResolutionDecoder]; + } + BOOL hasMetadata = enabled && LiGetHdrMetadata(&hdrMetadata); BOOL metadataChanged = NO; diff --git a/Limelight/Stream/VideoSuperResolution.h b/Limelight/Stream/VideoSuperResolution.h new file mode 100644 index 000000000..8c62209ef --- /dev/null +++ b/Limelight/Stream/VideoSuperResolution.h @@ -0,0 +1,29 @@ +// +// VideoSuperResolution.h +// Moonlight +// + +#import +@import CoreMedia; +@import CoreVideo; +@import Metal; + +@interface VideoSuperResolution : NSObject + +// Creates the Metal/Core Image state shared by the whole streaming session. +- (void)initializeResources; + +// Rebuilds cached resources when the stream switches between SDR and HDR. +- (void)setHdrEnabled:(BOOL)enabled; + +// Prepares all resources that only depend on the fixed input and output sizes for the session. +- (BOOL)configureWithInputSize:(CGSize)inputSize outputSize:(CGSize)outputSize; + +// Converts the decoder output from YUV to RGB, optionally upscales it with MetalFX, +// and returns a new sample buffer ready for AVSampleBufferDisplayLayer. +- (CMSampleBufferRef _Nullable)copyRGBSampleBufferFromImageBuffer:(CVImageBufferRef _Nonnull)imageBuffer + sourceSampleBuffer:(CMSampleBufferRef _Nullable)sourceSampleBuffer + presentationTimeStamp:(CMTime)presentationTimeStamp + duration:(CMTime)duration CF_RETURNS_RETAINED; + +@end diff --git a/Limelight/Stream/VideoSuperResolution.m b/Limelight/Stream/VideoSuperResolution.m new file mode 100644 index 000000000..f4b6f140a --- /dev/null +++ b/Limelight/Stream/VideoSuperResolution.m @@ -0,0 +1,766 @@ +// +// VideoSuperResolution.m +// Moonlight +// + +#import "VideoSuperResolution.h" +@import CoreImage; + +#if __has_include() +@import MetalFX; +#define MOONLIGHT_HAS_METALFX 1 +#else +#define MOONLIGHT_HAS_METALFX 0 +#endif + +@implementation VideoSuperResolution { + // Metal objects reused for the whole session. + id _device; + id _commandQueue; + CIContext* _ciContext; + + // Explicit SDR and HDR destination spaces let us control YUV->RGB conversion. + CGColorSpaceRef _rgbColorSpace; + CGColorSpaceRef _hdrColorSpace; + BOOL _isAvailable; + BOOL _hdrEnabled; + CVMetalTextureCacheRef _textureCache; + +#if MOONLIGHT_HAS_METALFX + // MetalFX resources created once for a given input/output size pair. + id _spatialScaler; + id _upscaledTexture; +#endif + + // Cached format/pool objects avoid per-frame allocations during streaming. + CMVideoFormatDescriptionRef _outputFormatDescription; + CVPixelBufferPoolRef _rgbPixelBufferPool; + CVPixelBufferPoolRef _upscaledPixelBufferPool; + size_t _inputWidth; + size_t _inputHeight; + size_t _outputWidth; + size_t _outputHeight; + BOOL _isConfigured; +} + +// Copies one attachment while preserving the requested propagation mode. +- (void)copyAttachmentForKey:(CFStringRef)key + fromBuffer:(CVBufferRef)sourceBuffer + toBuffer:(CVBufferRef)destinationBuffer + mode:(CVAttachmentMode)mode +{ + if (key == NULL || sourceBuffer == NULL || destinationBuffer == NULL) { + return; + } + + CFTypeRef value = CVBufferGetAttachment(sourceBuffer, key, NULL); + if (value != NULL) { + CVBufferSetAttachment(destinationBuffer, key, value, mode); + } +} + +// Uses the source image buffer color space when available so the RGB conversion matches +// the decoder output as closely as possible. Fall back to explicit SDR/HDR spaces otherwise. +- (CGColorSpaceRef)destinationColorSpaceForImageBuffer:(CVImageBufferRef)imageBuffer +{ + CGColorSpaceRef sourceColorSpace = NULL; + if (imageBuffer != NULL) { + sourceColorSpace = (CGColorSpaceRef)CVBufferGetAttachment(imageBuffer, + kCVImageBufferCGColorSpaceKey, + NULL); + } + + if (sourceColorSpace != NULL) { + return sourceColorSpace; + } + + if (_hdrEnabled && _hdrColorSpace != NULL) { + return _hdrColorSpace; + } + + return _rgbColorSpace; +} + +// Propagates only the attachments that are still meaningful after converting the buffer to RGB. +- (void)copyAttachmentsFromBuffer:(CVBufferRef)sourceBuffer toBuffer:(CVBufferRef)destinationBuffer +{ + if (sourceBuffer == NULL || destinationBuffer == NULL) { + return; + } + + [self copyAttachmentForKey:kCVImageBufferCleanApertureKey + fromBuffer:sourceBuffer + toBuffer:destinationBuffer + mode:kCVAttachmentMode_ShouldPropagate]; + [self copyAttachmentForKey:kCVImageBufferPreferredCleanApertureKey + fromBuffer:sourceBuffer + toBuffer:destinationBuffer + mode:kCVAttachmentMode_ShouldPropagate]; + [self copyAttachmentForKey:kCVImageBufferPixelAspectRatioKey + fromBuffer:sourceBuffer + toBuffer:destinationBuffer + mode:kCVAttachmentMode_ShouldPropagate]; + [self copyAttachmentForKey:kCVImageBufferDisplayDimensionsKey + fromBuffer:sourceBuffer + toBuffer:destinationBuffer + mode:kCVAttachmentMode_ShouldPropagate]; + [self copyAttachmentForKey:kCVImageBufferMasteringDisplayColorVolumeKey + fromBuffer:sourceBuffer + toBuffer:destinationBuffer + mode:kCVAttachmentMode_ShouldPropagate]; + [self copyAttachmentForKey:kCVImageBufferContentLightLevelInfoKey + fromBuffer:sourceBuffer + toBuffer:destinationBuffer + mode:kCVAttachmentMode_ShouldPropagate]; + + [self copyAttachmentForKey:kCVImageBufferColorPrimariesKey + fromBuffer:sourceBuffer + toBuffer:destinationBuffer + mode:kCVAttachmentMode_ShouldPropagate]; + [self copyAttachmentForKey:kCVImageBufferTransferFunctionKey + fromBuffer:sourceBuffer + toBuffer:destinationBuffer + mode:kCVAttachmentMode_ShouldPropagate]; + + CGColorSpaceRef colorSpace = [self destinationColorSpaceForImageBuffer:(CVImageBufferRef)sourceBuffer]; + if (colorSpace != NULL) { + CVBufferSetAttachment(destinationBuffer, + kCVImageBufferCGColorSpaceKey, + colorSpace, + kCVAttachmentMode_ShouldPropagate); + } + + // The output buffer is RGB, so source YCbCr matrix/chroma metadata no longer applies. + CVBufferRemoveAttachment(destinationBuffer, kCVImageBufferYCbCrMatrixKey); + CVBufferRemoveAttachment(destinationBuffer, kCVImageBufferChromaLocationTopFieldKey); + CVBufferRemoveAttachment(destinationBuffer, kCVImageBufferChromaLocationBottomFieldKey); + CVBufferRemoveAttachment(destinationBuffer, kCVImageBufferChromaSubsamplingKey); +} + +// Builds a fresh format description from the actual output pixel buffer so AVFoundation sees +// the real colorimetry of the converted frame instead of a stale generic description. +- (CMVideoFormatDescriptionRef)copyFormatDescriptionForImageBuffer:(CVImageBufferRef)imageBuffer CF_RETURNS_RETAINED +{ + if (imageBuffer == NULL) { + return nil; + } + + CMVideoFormatDescriptionRef formatDescription = NULL; + OSStatus status = CMVideoFormatDescriptionCreateForImageBuffer(kCFAllocatorDefault, + imageBuffer, + &formatDescription); + if (status != noErr) { + NSLog(@"VideoSuperResolution: Failed to create per-frame format description: %d", (int)status); + return nil; + } + + return formatDescription; +} + +// Preserves sample-buffer level attachments such as timing-agnostic HDR metadata. +- (void)copyAttachmentsFromSampleBuffer:(CMSampleBufferRef)sourceSampleBuffer + toSampleBuffer:(CMSampleBufferRef)destinationSampleBuffer +{ + if (sourceSampleBuffer == NULL || destinationSampleBuffer == NULL) { + return; + } + + CFDictionaryRef propagatedAttachments = CMCopyDictionaryOfAttachments(kCFAllocatorDefault, + sourceSampleBuffer, + kCMAttachmentMode_ShouldPropagate); + if (propagatedAttachments != NULL) { + CMSetAttachments(destinationSampleBuffer, propagatedAttachments, kCMAttachmentMode_ShouldPropagate); + CFRelease(propagatedAttachments); + } + + CFDictionaryRef nonPropagatedAttachments = CMCopyDictionaryOfAttachments(kCFAllocatorDefault, + sourceSampleBuffer, + kCMAttachmentMode_ShouldNotPropagate); + if (nonPropagatedAttachments != NULL) { + CMSetAttachments(destinationSampleBuffer, nonPropagatedAttachments, kCMAttachmentMode_ShouldNotPropagate); + CFRelease(nonPropagatedAttachments); + } +} + +- (void)initializeResources +{ + if (_device != nil && _commandQueue != nil) { + return; + } + + // Metal is required for both the Core Image conversion path and the optional MetalFX scaler. + _device = MTLCreateSystemDefaultDevice(); + if (_device == nil) { + NSLog(@"VideoSuperResolution: Metal device unavailable"); + return; + } + + _commandQueue = [_device newCommandQueue]; + if (_commandQueue == nil) { + NSLog(@"VideoSuperResolution: Failed to create Metal command queue"); + _device = nil; + return; + } + + // Core Image performs the YUV->RGB conversion on the GPU to keep the CPU path light. + _ciContext = [CIContext contextWithMTLDevice:_device]; + _rgbColorSpace = CGColorSpaceCreateWithName(kCGColorSpaceITUR_709); + if (@available(iOS 12.3, tvOS 12.3, *)) { + _hdrColorSpace = CGColorSpaceCreateWithName(kCGColorSpaceExtendedLinearITUR_2020); + } + + // Older deployment targets don't expose the dedicated HDR linear 2020 space. + // Fall back to a sane RGB space so the conversion path still compiles and runs. + if (_hdrColorSpace == NULL) { + _hdrColorSpace = CGColorSpaceCreateWithName(kCGColorSpaceITUR_709); + } + + // The texture cache bridges CVPixelBuffer and Metal texture objects without manual copies. + CVReturn cacheStatus = CVMetalTextureCacheCreate(kCFAllocatorDefault, NULL, _device, NULL, &_textureCache); + if (cacheStatus != kCVReturnSuccess) { + NSLog(@"VideoSuperResolution: Failed to create Metal texture cache: %d", cacheStatus); + _ciContext = nil; + if (_rgbColorSpace != NULL) { + CGColorSpaceRelease(_rgbColorSpace); + _rgbColorSpace = NULL; + } + if (_hdrColorSpace != NULL) { + CGColorSpaceRelease(_hdrColorSpace); + _hdrColorSpace = NULL; + } + _commandQueue = nil; + _device = nil; + return; + } + + // MetalFX may be unavailable either because the OS is too old or because the SDK target + // being compiled (notably some simulator environments) does not expose the framework. +#if MOONLIGHT_HAS_METALFX + if (@available(iOS 16.0, tvOS 16.0, *)) { + _isAvailable = YES; + } + else { + _isAvailable = NO; + NSLog(@"VideoSuperResolution: MetalFX SpatialScaler unavailable on this OS version"); + } +#else + _isAvailable = NO; +#endif +} + +- (void)setHdrEnabled:(BOOL)enabled +{ + if (_hdrEnabled == enabled) { + return; + } + + _hdrEnabled = enabled; + _isConfigured = NO; + + // HDR changes the pixel format, working color space, and MetalFX processing mode, + // so every size-dependent cached object must be rebuilt. + if (_outputFormatDescription != NULL) { + CFRelease(_outputFormatDescription); + _outputFormatDescription = NULL; + } + + if (_rgbPixelBufferPool != NULL) { + CFRelease(_rgbPixelBufferPool); + _rgbPixelBufferPool = NULL; + } + + if (_upscaledPixelBufferPool != NULL) { + CFRelease(_upscaledPixelBufferPool); + _upscaledPixelBufferPool = NULL; + } + +#if MOONLIGHT_HAS_METALFX + _spatialScaler = nil; + _upscaledTexture = nil; +#endif +} + +// Creates a reusable pool for buffers that stay GPU-compatible all the way through the pipeline. +- (CVPixelBufferPoolRef)createPixelBufferPoolWithWidth:(size_t)width + height:(size_t)height + pixelFormat:(OSType)pixelFormat CF_RETURNS_RETAINED +{ + NSDictionary* pixelBufferAttributes = @{ + (id)kCVPixelBufferWidthKey : @(width), + (id)kCVPixelBufferHeightKey : @(height), + (id)kCVPixelBufferPixelFormatTypeKey : @(pixelFormat), + (id)kCVPixelBufferMetalCompatibilityKey : @YES, + (id)kCVPixelBufferIOSurfacePropertiesKey : @{}, + }; + NSDictionary* poolAttributes = @{ + (id)kCVPixelBufferPoolMinimumBufferCountKey : @3, + }; + + CVPixelBufferPoolRef pool = NULL; + CVReturn status = CVPixelBufferPoolCreate(kCFAllocatorDefault, + (__bridge CFDictionaryRef)poolAttributes, + (__bridge CFDictionaryRef)pixelBufferAttributes, + &pool); + if (status != kCVReturnSuccess) { + NSLog(@"VideoSuperResolution: Failed to create pixel buffer pool: %d", status); + return nil; + } + + return pool; +} + +// Pulls a buffer from a pool instead of allocating a brand-new pixel buffer per frame. +- (CVPixelBufferRef)copyPixelBufferFromPool:(CVPixelBufferPoolRef)pixelBufferPool CF_RETURNS_RETAINED +{ + if (pixelBufferPool == NULL) { + return nil; + } + + CVPixelBufferRef pixelBuffer = NULL; + CVReturn status = CVPixelBufferPoolCreatePixelBuffer(kCFAllocatorDefault, pixelBufferPool, &pixelBuffer); + if (status != kCVReturnSuccess) { + NSLog(@"VideoSuperResolution: Failed to create pixel buffer from pool: %d", status); + return nil; + } + + return pixelBuffer; +} + +// The intermediate RGB buffer always matches the decoder output size. +- (BOOL)prepareRGBPixelBufferPoolWithWidth:(size_t)width height:(size_t)height +{ + if (_rgbPixelBufferPool != NULL && + _inputWidth == width && + _inputHeight == height) { + return YES; + } + + if (_rgbPixelBufferPool != NULL) { + CFRelease(_rgbPixelBufferPool); + _rgbPixelBufferPool = NULL; + } + + _rgbPixelBufferPool = [self createPixelBufferPoolWithWidth:width + height:height + pixelFormat:_hdrEnabled ? kCVPixelFormatType_64RGBAHalf : kCVPixelFormatType_32BGRA]; + return _rgbPixelBufferPool != NULL; +} + +// Keeps a cached output format description for the common case where attachments do not force +// us to create a per-frame description later. +- (BOOL)prepareOutputFormatDescription +{ + if (_outputFormatDescription != NULL) { + return YES; + } + + CVPixelBufferRef outputPixelBuffer = nil; + if (_upscaledPixelBufferPool != NULL) { + outputPixelBuffer = [self copyPixelBufferFromPool:_upscaledPixelBufferPool]; + } + else if (_rgbPixelBufferPool != NULL) { + outputPixelBuffer = [self copyPixelBufferFromPool:_rgbPixelBufferPool]; + } + + if (outputPixelBuffer == NULL) { + return NO; + } + + OSStatus status = CMVideoFormatDescriptionCreateForImageBuffer(kCFAllocatorDefault, + outputPixelBuffer, + &_outputFormatDescription); + CFRelease(outputPixelBuffer); + if (status != noErr) { + NSLog(@"VideoSuperResolution: Failed to create output format description: %d", (int)status); + _outputFormatDescription = NULL; + return NO; + } + + return YES; +} + +// Creates the MetalFX scaler and the private texture that MetalFX requires as its render target. +- (BOOL)prepareUpscalerForInputWidth:(size_t)inputWidth + inputHeight:(size_t)inputHeight + outputWidth:(size_t)outputWidth + outputHeight:(size_t)outputHeight +{ +#if !MOONLIGHT_HAS_METALFX + return NO; +#else + if (!_isAvailable) { + return NO; + } + + if (_spatialScaler != nil && + _inputWidth == inputWidth && + _inputHeight == inputHeight && + _outputWidth == outputWidth && + _outputHeight == outputHeight) { + return YES; + } + + if (@available(iOS 16.0, tvOS 16.0, *)) { + MTLPixelFormat outputPixelFormat = _hdrEnabled ? MTLPixelFormatRGBA16Float : MTLPixelFormatBGRA8Unorm; + + // MetalFX writes into a private texture first, then we blit into a CVPixelBuffer-backed texture. + MTLTextureDescriptor* outputDescriptor = + [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:outputPixelFormat + width:outputWidth + height:outputHeight + mipmapped:NO]; + outputDescriptor.storageMode = MTLStorageModePrivate; + + MTLFXSpatialScalerDescriptor* descriptor = [[MTLFXSpatialScalerDescriptor alloc] init]; + descriptor.inputWidth = inputWidth; + descriptor.inputHeight = inputHeight; + descriptor.outputWidth = outputWidth; + descriptor.outputHeight = outputHeight; + descriptor.colorTextureFormat = outputPixelFormat; + descriptor.outputTextureFormat = outputPixelFormat; + descriptor.colorProcessingMode = _hdrEnabled ? MTLFXSpatialScalerColorProcessingModeHDR : MTLFXSpatialScalerColorProcessingModePerceptual; + + // The scaler describes the exact texture usage flags required by the driver. + id spatialScaler = [descriptor newSpatialScalerWithDevice:_device]; + if (spatialScaler == nil) { + NSLog(@"VideoSuperResolution: Failed to create MetalFX spatial scaler"); + return NO; + } + + outputDescriptor.usage = spatialScaler.outputTextureUsage; + + id outputTexture = [_device newTextureWithDescriptor:outputDescriptor]; + if (outputTexture == nil) { + NSLog(@"VideoSuperResolution: Failed to create private output texture"); + return NO; + } + + spatialScaler.inputContentWidth = inputWidth; + spatialScaler.inputContentHeight = inputHeight; + spatialScaler.outputTexture = outputTexture; + + _spatialScaler = spatialScaler; + _upscaledTexture = outputTexture; + _inputWidth = inputWidth; + _inputHeight = inputHeight; + _outputWidth = outputWidth; + _outputHeight = outputHeight; + + // The upscaled output is exposed through a CVPixelBuffer pool because AVSampleBufferDisplayLayer + // still expects image buffers, not raw Metal textures. + if (_upscaledPixelBufferPool != NULL) { + CFRelease(_upscaledPixelBufferPool); + _upscaledPixelBufferPool = NULL; + } + + _upscaledPixelBufferPool = [self createPixelBufferPoolWithWidth:outputWidth + height:outputHeight + pixelFormat:_hdrEnabled ? kCVPixelFormatType_64RGBAHalf : kCVPixelFormatType_32BGRA]; + if (_upscaledPixelBufferPool == NULL) { + _spatialScaler = nil; + return NO; + } + + if (_outputFormatDescription != NULL) { + CFRelease(_outputFormatDescription); + _outputFormatDescription = NULL; + } + + return YES; + } + + return NO; +#endif +} + +// This is the one-shot session configuration path called once the input stream size and the +// device display size are known. +- (BOOL)configureWithInputSize:(CGSize)inputSize outputSize:(CGSize)outputSize +{ + if (_device == nil || _ciContext == nil) { + return NO; + } + + size_t inputWidth = MAX(1, (size_t)llround(inputSize.width)); + size_t inputHeight = MAX(1, (size_t)llround(inputSize.height)); + size_t outputWidth = MAX(1, (size_t)llround(outputSize.width)); + size_t outputHeight = MAX(1, (size_t)llround(outputSize.height)); + + if (_isConfigured && + _inputWidth == inputWidth && + _inputHeight == inputHeight && + _outputWidth == outputWidth && + _outputHeight == outputHeight) { + return YES; + } + + if (_outputFormatDescription != NULL) { + CFRelease(_outputFormatDescription); + _outputFormatDescription = NULL; + } + + if (![self prepareRGBPixelBufferPoolWithWidth:inputWidth height:inputHeight]) { + _isConfigured = NO; + return NO; + } + + // Only enable MetalFX when the target is strictly larger than the decoded frame. + if (outputWidth > inputWidth && outputHeight > inputHeight) { + if (![self prepareUpscalerForInputWidth:inputWidth + inputHeight:inputHeight + outputWidth:outputWidth + outputHeight:outputHeight]) { + _isConfigured = NO; + return NO; + } + } + else { + if (_upscaledPixelBufferPool != NULL) { + CFRelease(_upscaledPixelBufferPool); + _upscaledPixelBufferPool = NULL; + } +#if MOONLIGHT_HAS_METALFX + _spatialScaler = nil; + _upscaledTexture = nil; +#endif + _inputWidth = inputWidth; + _inputHeight = inputHeight; + _outputWidth = inputWidth; + _outputHeight = inputHeight; + } + + // Keep a cached format description around for fast sample buffer creation. + if (![self prepareOutputFormatDescription]) { + _isConfigured = NO; + return NO; + } + + _isConfigured = YES; + return YES; +} + +// Runs the MetalFX stage and copies the private result into a pixel-buffer-backed texture. +- (CVPixelBufferRef)copyUpscaledPixelBufferFromRGBPixelBuffer:(CVPixelBufferRef)rgbPixelBuffer + CF_RETURNS_RETAINED +{ +#if !MOONLIGHT_HAS_METALFX + return nil; +#else + if (!_isConfigured || _spatialScaler == nil || _upscaledPixelBufferPool == NULL) { + return nil; + } + + size_t inputWidth = _inputWidth; + size_t inputHeight = _inputHeight; + size_t outputWidth = _outputWidth; + size_t outputHeight = _outputHeight; + + // Wrap the RGB intermediate buffer as a Metal texture without copying it. + CVMetalTextureRef inputTextureRef = NULL; + CVReturn cvStatus = CVMetalTextureCacheCreateTextureFromImage(kCFAllocatorDefault, + _textureCache, + rgbPixelBuffer, + NULL, + _hdrEnabled ? MTLPixelFormatRGBA16Float : MTLPixelFormatBGRA8Unorm, + inputWidth, + inputHeight, + 0, + &inputTextureRef); + if (cvStatus != kCVReturnSuccess || inputTextureRef == NULL) { + NSLog(@"VideoSuperResolution: Failed to create input texture from RGB pixel buffer: %d", cvStatus); + return nil; + } + + id inputTexture = CVMetalTextureGetTexture(inputTextureRef); + if (inputTexture == nil) { + CFRelease(inputTextureRef); + return nil; + } + + CVPixelBufferRef outputPixelBuffer = [self copyPixelBufferFromPool:_upscaledPixelBufferPool]; + if (outputPixelBuffer == NULL) { + CFRelease(inputTextureRef); + return nil; + } + + // Wrap the displayable output pixel buffer as a Metal texture for the final blit. + CVMetalTextureRef outputTextureRef = NULL; + cvStatus = CVMetalTextureCacheCreateTextureFromImage(kCFAllocatorDefault, + _textureCache, + outputPixelBuffer, + NULL, + _hdrEnabled ? MTLPixelFormatRGBA16Float : MTLPixelFormatBGRA8Unorm, + outputWidth, + outputHeight, + 0, + &outputTextureRef); + if (cvStatus != kCVReturnSuccess || outputTextureRef == NULL) { + NSLog(@"VideoSuperResolution: Failed to create output texture from pixel buffer: %d", cvStatus); + CFRelease(inputTextureRef); + CFRelease(outputPixelBuffer); + return nil; + } + + id outputTexture = CVMetalTextureGetTexture(outputTextureRef); + if (outputTexture == nil) { + CFRelease(inputTextureRef); + CFRelease(outputTextureRef); + CFRelease(outputPixelBuffer); + return nil; + } + + id commandBuffer = [_commandQueue commandBuffer]; + if (commandBuffer == nil) { + CFRelease(inputTextureRef); + CFRelease(outputTextureRef); + CFRelease(outputPixelBuffer); + return nil; + } + + // MetalFX writes into the private texture declared during configuration. + _spatialScaler.colorTexture = inputTexture; + [_spatialScaler encodeToCommandBuffer:commandBuffer]; + + // Copy the private result into the CVPixelBuffer-backed texture consumed by AVFoundation. + id blitEncoder = [commandBuffer blitCommandEncoder]; + [blitEncoder copyFromTexture:_upscaledTexture + sourceSlice:0 + sourceLevel:0 + sourceOrigin:MTLOriginMake(0, 0, 0) + sourceSize:MTLSizeMake(outputWidth, outputHeight, 1) + toTexture:outputTexture + destinationSlice:0 + destinationLevel:0 + destinationOrigin:MTLOriginMake(0, 0, 0)]; + [blitEncoder endEncoding]; + + [commandBuffer commit]; + + // Do not wait here: blocking the CPU on every frame causes severe input latency. + // [commandBuffer waitUntilCompleted]; + + CFRelease(inputTextureRef); + CFRelease(outputTextureRef); + + return outputPixelBuffer; +#endif +} + +// Converts the decoder output into an RGB sample buffer that preserves the source timing and +// the HDR/color metadata expected by AVSampleBufferDisplayLayer. +- (CMSampleBufferRef)copyRGBSampleBufferFromImageBuffer:(CVImageBufferRef)imageBuffer + sourceSampleBuffer:(CMSampleBufferRef)sourceSampleBuffer + presentationTimeStamp:(CMTime)presentationTimeStamp + duration:(CMTime)duration +{ + if (_device == nil || _ciContext == nil || imageBuffer == nil || !_isConfigured) { + return nil; + } + + CVPixelBufferRef pixelBuffer = (CVPixelBufferRef)imageBuffer; + if (CVPixelBufferGetWidth(pixelBuffer) != _inputWidth || + CVPixelBufferGetHeight(pixelBuffer) != _inputHeight) { + return nil; + } + + CVPixelBufferRef rgbPixelBuffer = [self copyPixelBufferFromPool:_rgbPixelBufferPool]; + if (rgbPixelBuffer == NULL) { + return nil; + } + + CIImage* image = [CIImage imageWithCVPixelBuffer:pixelBuffer]; + CGColorSpaceRef destinationColorSpace = [self destinationColorSpaceForImageBuffer:imageBuffer]; + + // Core Image performs the actual YUV->RGB conversion here. + [_ciContext render:image + toCVPixelBuffer:rgbPixelBuffer + bounds:CGRectMake(0, 0, _inputWidth, _inputHeight) + colorSpace:destinationColorSpace]; + [self copyAttachmentsFromBuffer:imageBuffer toBuffer:rgbPixelBuffer]; + + CVPixelBufferRef outputPixelBuffer = rgbPixelBuffer; +#if MOONLIGHT_HAS_METALFX + if (_spatialScaler != nil) { + // Upscaling is optional; the conversion-only path still uses the same metadata flow. + CVPixelBufferRef upscaledPixelBuffer = [self copyUpscaledPixelBufferFromRGBPixelBuffer:rgbPixelBuffer]; + if (upscaledPixelBuffer != NULL) { + [self copyAttachmentsFromBuffer:rgbPixelBuffer toBuffer:upscaledPixelBuffer]; + outputPixelBuffer = upscaledPixelBuffer; + CFRelease(rgbPixelBuffer); + } + } +#endif + + if (_outputFormatDescription == NULL) { + CFRelease(outputPixelBuffer); + return nil; + } + + CMSampleTimingInfo sampleTiming = { + .duration = duration, + .presentationTimeStamp = presentationTimeStamp, + .decodeTimeStamp = kCMTimeInvalid, + }; + + // Prefer a per-frame format description so the latest HDR attachments are visible to the display layer. + CMVideoFormatDescriptionRef outputFormatDescription = [self copyFormatDescriptionForImageBuffer:outputPixelBuffer]; + if (outputFormatDescription == NULL) { + outputFormatDescription = _outputFormatDescription; + if (outputFormatDescription != NULL) { + CFRetain(outputFormatDescription); + } + } + + if (outputFormatDescription == NULL) { + CFRelease(outputPixelBuffer); + return nil; + } + + CMSampleBufferRef rgbSampleBuffer = NULL; + OSStatus status = CMSampleBufferCreateReadyWithImageBuffer(kCFAllocatorDefault, + outputPixelBuffer, + outputFormatDescription, + &sampleTiming, + &rgbSampleBuffer); + CFRelease(outputFormatDescription); + CFRelease(outputPixelBuffer); + + if (status != noErr) { + NSLog(@"VideoSuperResolution: Failed to create RGB sample buffer: %d", (int)status); + return nil; + } + + // Preserve sample-level attachments after the new CMSampleBuffer has been created. + [self copyAttachmentsFromSampleBuffer:sourceSampleBuffer toSampleBuffer:rgbSampleBuffer]; + + return rgbSampleBuffer; +} + +- (void)dealloc +{ + if (_outputFormatDescription != NULL) { + CFRelease(_outputFormatDescription); + } + + if (_rgbPixelBufferPool != NULL) { + CFRelease(_rgbPixelBufferPool); + } + + if (_upscaledPixelBufferPool != NULL) { + CFRelease(_upscaledPixelBufferPool); + } + + if (_textureCache != NULL) { + CFRelease(_textureCache); + } + + if (_rgbColorSpace != NULL) { + CGColorSpaceRelease(_rgbColorSpace); + } + + if (_hdrColorSpace != NULL) { + CGColorSpaceRelease(_hdrColorSpace); + } +} + +@end diff --git a/Limelight/ViewControllers/SettingsViewController.h b/Limelight/ViewControllers/SettingsViewController.h index 1a79162da..e53fb02e7 100644 --- a/Limelight/ViewControllers/SettingsViewController.h +++ b/Limelight/ViewControllers/SettingsViewController.h @@ -11,6 +11,7 @@ @interface SettingsViewController : UIViewController @property (strong, nonatomic) IBOutlet UILabel *bitrateLabel; +@property (strong, nonatomic) IBOutlet UILabel *videoSuperResolutionLabel; @property (strong, nonatomic) IBOutlet UISlider *bitrateSlider; @property (strong, nonatomic) IBOutlet UISegmentedControl *framerateSelector; @property (strong, nonatomic) IBOutlet UISegmentedControl *resolutionSelector; @@ -23,6 +24,7 @@ @property (strong, nonatomic) IBOutlet UISegmentedControl *audioOnPCSelector; @property (strong, nonatomic) IBOutlet UISegmentedControl *codecSelector; @property (strong, nonatomic) IBOutlet UISegmentedControl *hdrSelector; +@property (strong, nonatomic) IBOutlet UISegmentedControl *videoSuperResolutionSelector; @property (strong, nonatomic) IBOutlet UISegmentedControl *framePacingSelector; @property (strong, nonatomic) IBOutlet UISegmentedControl *btMouseSelector; @property (strong, nonatomic) IBOutlet UISegmentedControl *statsOverlaySelector; diff --git a/Limelight/ViewControllers/SettingsViewController.m b/Limelight/ViewControllers/SettingsViewController.m index bbe8eb2de..41f80e4f0 100644 --- a/Limelight/ViewControllers/SettingsViewController.m +++ b/Limelight/ViewControllers/SettingsViewController.m @@ -13,9 +13,17 @@ #import #import +#if __has_include() +#import +#define MOONLIGHT_HAS_METALFX 1 +#else +#define MOONLIGHT_HAS_METALFX 0 +#endif + @implementation SettingsViewController { NSInteger _bitrate; NSInteger _lastSelectedResolutionIndex; + BOOL _videoSuperResolutionSupported; } @dynamic overrideUserInterfaceStyle; @@ -126,6 +134,19 @@ BOOL isCustomResolution(CGSize res) { return YES; } +- (BOOL)isVideoSuperResolutionSupported { +#if !MOONLIGHT_HAS_METALFX + return NO; +#else + if (@available(iOS 16.0, tvOS 16.0, *)) { + id device = MTLCreateSystemDefaultDevice(); + return device != nil && [MTLFXSpatialScalerDescriptor supportsDevice:device]; + } + + return NO; +#endif +} + - (void)viewDidLoad { [super viewDidLoad]; @@ -239,6 +260,16 @@ - (void)viewDidLoad { else { [self.hdrSelector setSelectedSegmentIndex:currentSettings.enableHdr ? 1 : 0]; } + _videoSuperResolutionSupported = [self isVideoSuperResolutionSupported]; + if (_videoSuperResolutionSupported) { + [self.videoSuperResolutionSelector setSelectedSegmentIndex:currentSettings.videoSuperResolution ? 1 : 0]; + } + else { + [self.videoSuperResolutionSelector setSelectedSegmentIndex:0]; + [self.videoSuperResolutionSelector setHidden:YES]; + [self.videoSuperResolutionLabel setHidden:YES]; + [[NSUserDefaults standardUserDefaults] setBool:NO forKey:@"videoSuperResolution"]; + } [self.touchModeSelector setSelectedSegmentIndex:currentSettings.absoluteTouchMode ? 1 : 0]; [self.touchModeSelector addTarget:self action:@selector(touchModeChanged) forControlEvents:UIControlEventValueChanged]; @@ -538,6 +569,8 @@ - (void) saveSettings { BOOL absoluteTouchMode = [self.touchModeSelector selectedSegmentIndex] == 1; BOOL statsOverlay = [self.statsOverlaySelector selectedSegmentIndex] == 1; BOOL enableHdr = [self.hdrSelector selectedSegmentIndex] == 1; + BOOL videoSuperResolution = _videoSuperResolutionSupported && [self.videoSuperResolutionSelector selectedSegmentIndex] == 1; + [[NSUserDefaults standardUserDefaults] setBool:videoSuperResolution forKey:@"videoSuperResolution"]; [dataMan saveSettingsWithBitrate:_bitrate framerate:framerate height:height diff --git a/Moonlight.xcodeproj/project.pbxproj b/Moonlight.xcodeproj/project.pbxproj index 6f055579b..a625efc39 100644 --- a/Moonlight.xcodeproj/project.pbxproj +++ b/Moonlight.xcodeproj/project.pbxproj @@ -41,6 +41,8 @@ 98CFB82F1CAD481B0048EF74 /* libmoonlight-common.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 98AB2E841CAD46840089BB98 /* libmoonlight-common.a */; }; 98D5856D1C0EA79600F6CC00 /* TemporaryHost.m in Sources */ = {isa = PBXBuildFile; fileRef = 98D5856C1C0EA79600F6CC00 /* TemporaryHost.m */; }; 98D585701C0ED0E800F6CC00 /* TemporarySettings.m in Sources */ = {isa = PBXBuildFile; fileRef = 98D5856F1C0ED0E800F6CC00 /* TemporarySettings.m */; }; + A1D986BF2F8417100030618A /* VideoSuperResolution.m in Sources */ = {isa = PBXBuildFile; fileRef = A1D986BE2F8417100030618A /* VideoSuperResolution.m */; }; + A1D986C02F8417100030618A /* VideoSuperResolution.m in Sources */ = {isa = PBXBuildFile; fileRef = A1D986BE2F8417100030618A /* VideoSuperResolution.m */; }; DC1F5A07206436B20037755F /* ConnectionHelper.m in Sources */ = {isa = PBXBuildFile; fileRef = DC1F5A06206436B20037755F /* ConnectionHelper.m */; }; F74BEF9C2C1A705600224667 /* OpenSSL in Frameworks */ = {isa = PBXBuildFile; productRef = F74BEF9B2C1A705600224667 /* OpenSSL */; }; F74BEFA42C1A80E400224667 /* OpenSSL in Frameworks */ = {isa = PBXBuildFile; productRef = F74BEFA32C1A80E400224667 /* OpenSSL */; }; @@ -212,6 +214,9 @@ 98D5856C1C0EA79600F6CC00 /* TemporaryHost.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; name = TemporaryHost.m; path = Database/TemporaryHost.m; sourceTree = ""; }; 98D5856E1C0ED0E800F6CC00 /* TemporarySettings.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = TemporarySettings.h; path = Database/TemporarySettings.h; sourceTree = ""; }; 98D5856F1C0ED0E800F6CC00 /* TemporarySettings.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; name = TemporarySettings.m; path = Database/TemporarySettings.m; sourceTree = ""; }; + A1C4B35C2D8E100100A1B2C3 /* MetalFX.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = MetalFX.framework; path = System/Library/Frameworks/MetalFX.framework; sourceTree = SDKROOT; }; + A1D986BD2F8417100030618A /* VideoSuperResolution.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = VideoSuperResolution.h; sourceTree = ""; }; + A1D986BE2F8417100030618A /* VideoSuperResolution.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = VideoSuperResolution.m; sourceTree = ""; }; D4746EEA1CBC740C006FB401 /* Moonlight-Bridging-Header.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = "Moonlight-Bridging-Header.h"; path = "Input/Moonlight-Bridging-Header.h"; sourceTree = ""; }; DC1F5A05206436B10037755F /* ConnectionHelper.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = ConnectionHelper.h; sourceTree = ""; }; DC1F5A06206436B20037755F /* ConnectionHelper.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = ConnectionHelper.m; sourceTree = ""; }; @@ -425,6 +430,7 @@ 98882A042AF60F5300C5A11C /* libavutil.a */, 98882A092AF60F9000C5A11C /* libavutil.a */, 98B9CE6C27B2144B00B473C4 /* AVKit.framework */, + A1C4B35C2D8E100100A1B2C3 /* MetalFX.framework */, 98181BEC2791281100E43572 /* CoreMotion.framework */, 98181BE82791275D00E43572 /* libSDL2.a */, 98181BEA2791278300E43572 /* libSDL2.a */, @@ -551,6 +557,8 @@ FB89461C19F646E200339C8A /* VideoDecoderRenderer.h */, FB89461D19F646E200339C8A /* VideoDecoderRenderer.m */, 9803CCAB254F9EAF00EE185E /* ConnectionCallbacks.h */, + A1D986BD2F8417100030618A /* VideoSuperResolution.h */, + A1D986BE2F8417100030618A /* VideoSuperResolution.m */, ); path = Stream; sourceTree = ""; @@ -852,6 +860,7 @@ FB1A67DC2132460400507771 /* KeyboardSupport.m in Sources */, FB1A67CD213245F800507771 /* DataManager.m in Sources */, FB1A67CF213245F800507771 /* TemporaryApp.m in Sources */, + A1D986C02F8417100030618A /* VideoSuperResolution.m in Sources */, FB1A67D1213245F800507771 /* TemporaryHost.m in Sources */, FB1A67D3213245F800507771 /* TemporarySettings.m in Sources */, FB1A67C3213245EA00507771 /* AppAssetManager.m in Sources */, @@ -893,6 +902,7 @@ FB9AFD401A7E127D00872C98 /* AppListResponse.m in Sources */, FB89463419F646E200339C8A /* Utils.m in Sources */, FBDE86E619F82297001C18A8 /* UIAppView.m in Sources */, + A1D986BF2F8417100030618A /* VideoSuperResolution.m in Sources */, FB89462F19F646E200339C8A /* Connection.m in Sources */, FB9AFD321A7D867C00872C98 /* AppAssetRetriever.m in Sources */, FB4678FF1A565DAC00377732 /* WakeOnLanManager.m in Sources */, diff --git a/iPad.storyboard b/iPad.storyboard index 9017b15b9..cbf064279 100644 --- a/iPad.storyboard +++ b/iPad.storyboard @@ -1,9 +1,9 @@ - + - + @@ -12,7 +12,7 @@ - + @@ -59,7 +59,7 @@ - + - + + + + + + + + + + + + - + @@ -161,14 +178,14 @@ - + @@ -180,14 +197,14 @@ - + @@ -197,14 +214,14 @@ - + @@ -214,14 +231,14 @@ - + @@ -231,14 +248,14 @@ - + @@ -248,14 +265,14 @@ - + @@ -267,14 +284,14 @@ - + @@ -284,14 +301,14 @@ - + @@ -301,14 +318,14 @@ - + @@ -318,14 +335,14 @@ - + @@ -346,6 +363,8 @@ + + @@ -366,7 +385,7 @@ - +