Resolution switching without reconnecting.

- Based on https://github.com/jsorg71/xrdp/tree/dynamic_monitor
- Tested with xorgxrdp
- Tested with vnc
- Only works with single monitor.
- Update documentation to clarify the difference between MSTSC and
Microsoft Remote Desktop.
- Compatible with NVENC and xorgxrdp_helper
- Compatible with Nvidia.
- Updates to include ms-rdpedisp.h header for the 2.2.2 specification of
the protocol.
- State machine for resizing.
- Mechanisms to make sure the key frame is always sent on resize.
- Adds new struct that shares the number of monitors with xrdp_client_info.h
- Modification to the original resize setup that works with /gfx.
- YAMI compat.
- Updating to librfxcodec branch that is also a merge from mainline.
- Made sure RFX progressive works.

Depends on neutrinolabs/xorgxrdp#183

Makefile.am

@@ -66,7 +66,8 @@ SUBDIRS = \
   $(XRDPVRDIR) \
   $(ULALACADIR) \
   tests \
-  tools
+  tools \
+  xorgxrdp_helper
 
 distclean-local:
 	-rm -f xrdp_configure_options.h

common/ms-rdpbcgr.h

@@ -95,10 +95,10 @@
 /* the maximum height of the virtual desktop resulting from the union of the monitors  */
 /* contained in the monitorDefArray field MUST NOT exceed 32,766 pixels. */
 /* The minimum permitted size of the virtual desktop is 200 x 200 pixels. */
-#define CLIENT_MONITOR_DATA_MINIMUM_VIRTUAL_DESKTOP_WIDTH  0xC8
-#define CLIENT_MONITOR_DATA_MINIMUM_VIRTUAL_DESKTOP_HEIGHT 0xC8
-#define CLIENT_MONITOR_DATA_MAXIMUM_VIRTUAL_DESKTOP_WIDTH  0x7FFE
-#define CLIENT_MONITOR_DATA_MAXIMUM_VIRTUAL_DESKTOP_HEIGHT 0x7FFE
+#define CLIENT_MONITOR_DATA_MINIMUM_VIRTUAL_DESKTOP_WIDTH  0xC8   // 200
+#define CLIENT_MONITOR_DATA_MINIMUM_VIRTUAL_DESKTOP_HEIGHT 0xC8   // 200
+#define CLIENT_MONITOR_DATA_MAXIMUM_VIRTUAL_DESKTOP_WIDTH  0x7FFE // 32766
+#define CLIENT_MONITOR_DATA_MAXIMUM_VIRTUAL_DESKTOP_HEIGHT 0x7FFE // 32766
 
 /* 2.2.1.3.6.1 Monitor Definition (TS_MONITOR_DEF) */
 #define TS_MONITOR_PRIMARY 0x00000001

common/ms-rdpedisp.h

@@ -38,4 +38,7 @@
 #define ORIENTATION_LANDSCAPE_FLIPPED                      180
 #define ORIENTATION_PORTRAIT_FLIPPED                       270
 
+/* Display Control Monitor Layout (2.2.2.2.1) */
+#define DISPLAYCONTROL_MONITOR_PRIMARY         0x00000001
+
 #endif /* MS_RDPEDISP_H */

common/pixman-region.h

@@ -68,9 +68,11 @@ pixman_bool_t           pixman_region_subtract           (pixman_region16_t *reg
         /**/                                              pixman_region16_t *reg_m,
         /**/                                              pixman_region16_t *reg_s);
 pixman_bool_t           pixman_region_intersect          (pixman_region16_t *new_reg,
-        /**/                                              pixman_region16_t *reg1,
-        /**/                                              pixman_region16_t *reg2);
+        pixman_region16_t *reg1,
+        pixman_region16_t *reg2);
 pixman_box16_t         *pixman_region_rectangles         (pixman_region16_t *region,
-        /**/                                              int               *n_rects);
+        int *n_rects);
+pixman_bool_t           pixman_region_not_empty          (pixman_region16_t *region);
+pixman_box16_t         *pixman_region_extents            (pixman_region16_t *region);
 
 #endif

common/trans.h

@@ -57,6 +57,8 @@ enum xrdp_source
     XRDP_SOURCE_SESMAN,
     XRDP_SOURCE_CHANSRV,
     XRDP_SOURCE_MOD,
+    XORGXRDP_SOURCE_XORG,
+    XORGXRDP_SOURCE_XRDP,
 
     XRDP_SOURCE_MAX_COUNT
 };

common/xrdp_client_info.h

@@ -213,6 +213,7 @@ struct xrdp_client_info
      * data */
     unsigned int session_physical_width; /* in mm */
     unsigned int session_physical_height; /* in mm */
+    int gfx;
 };
 
 /* yyyymmdd of last incompatible change to xrdp_client_info */

configure.ac

@@ -173,6 +173,21 @@ AC_ARG_ENABLE(pixman, AS_HELP_STRING([--enable-pixman],
               [], [enable_pixman=no])
 AM_CONDITIONAL(XRDP_PIXMAN, [test x$enable_pixman = xyes])
 
+#example XRDP_NVENC_CFLAGS="-I/usr/local/cuda-11.3/include -I$HOME/Video_Codec_SDK_11.0.10/Interface" XRDP_NVENC_LIBS="-lcuda -lnvidia-encode" ./configure --enable-nvenc
+AC_ARG_ENABLE(nvenc, AS_HELP_STRING([--enable-nvenc],
+              [Use nvenc library (default: no), env vars XRDP_NVENC_CFLAGS and
+               XRDP_NVENC_LIBS should be set if used]),
+              [], [enable_nvenc=no])
+AM_CONDITIONAL(XRDP_NVENC, [test x$enable_nvenc = xyes])
+AC_ARG_ENABLE(yami, AS_HELP_STRING([--enable-yami],
+              [Use yami library (default: no)]),
+              [], [enable_yami=no])
+AM_CONDITIONAL(XRDP_YAMI, [test x$enable_yami = xyes])
+AC_ARG_ENABLE(x264, AS_HELP_STRING([--enable-x264],
+              [Use x264 library (default: no)]),
+              [], [enable_x264=no])
+AM_CONDITIONAL(XRDP_X264, [test x$enable_x264 = xyes])
+
 AC_ARG_ENABLE(painter, AS_HELP_STRING([--disable-painter],
               [Do not use included painter library (default: no)]),
               [], [enable_painter=yes])
@@ -405,6 +420,32 @@ fi
 
 AS_IF( [test "x$enable_pixman" = "xyes"] , [PKG_CHECK_MODULES(PIXMAN, pixman-1 >= 0.1.0)] )
 
+AS_IF( [test "x$enable_x264" = "xyes"] , [PKG_CHECK_MODULES(XRDP_X264, x264 >= 0.3.0)] )
+
+if test "x$enable_nvenc" = "xyes"
+then
+if test ! -z "$XRDP_NVENC_CFLAGS"
+then
+  AC_SUBST(XRDP_NVENC_CFLAGS, ["$XRDP_NVENC_CFLAGS"])
+fi
+if test ! -z "$XRDP_NVENC_LIBS"
+then
+  AC_SUBST(XRDP_NVENC_LIBS, ["$XRDP_NVENC_LIBS"])
+fi
+fi
+
+if test "x$enable_yami" = "xyes"
+then
+if test ! -z "$XRDP_YAMI_CFLAGS"
+then
+  AC_SUBST(XRDP_YAMI_CFLAGS, ["$XRDP_YAMI_CFLAGS"])
+fi
+if test ! -z "$XRDP_YAMI_LIBS"
+then
+  AC_SUBST(XRDP_YAMI_LIBS, ["$XRDP_YAMI_LIBS"])
+fi
+fi
+
 # checking for TurboJPEG
 if test "x$enable_tjpeg" = "xyes"
 then
@@ -528,6 +569,7 @@ AC_CONFIG_FILES([
   xrdp/Makefile
   xrdpvr/Makefile
   xup/Makefile
+  xorgxrdp_helper/Makefile
 ])
 
 AC_REQUIRE_AUX_FILE([tap-driver.sh])
@@ -542,6 +584,9 @@ echo "  fdkaac                  $enable_fdkaac"
 echo "  jpeg                    $enable_jpeg"
 echo "  turbo jpeg              $enable_tjpeg"
 echo "  rfxcodec                $enable_rfxcodec"
+echo "  nvenc                   $enable_nvenc"
+echo "  yami                    $enable_yami"
+echo "  x264                    $enable_x264"
 echo "  painter                 $enable_painter"
 echo "  pixman                  $enable_pixman"
 echo "  fuse                    $enable_fuse"

libxrdp/libxrdp.c

@@ -1137,6 +1137,7 @@ int EXPORT_CC
 libxrdp_reset(struct xrdp_session *session,
               unsigned int width, unsigned int height, int bpp)
 {
+    LOG_DEVEL(LOG_LEVEL_TRACE, "libxrdp_reset:");
     if (session->client_info != 0)
     {
         struct xrdp_client_info *client_info = session->client_info;
@@ -1375,6 +1376,10 @@ libxrdp_send_to_channel(struct xrdp_session *session, int channel_id,
         free_stream(s);
         return 1;
     }
+    else
+    {
+        LOG(LOG_LEVEL_TRACE, "libxrdp_send_to_channel: xrdp_channel_init successful!");
+    }
 
     /* here we make a copy of the data */
     out_uint8a(s, data, data_len);
@@ -2258,3 +2263,15 @@ libxrdp_process_monitor_ex_stream(struct stream *s,
 
     return 0;
 }
+int EXPORT_CC
+libxrdp_planar_compress(char *in_data, int width, int height,
+                        struct stream *s, int bpp, int byte_limit,
+                        int start_line, struct stream *temp_s,
+                        int e, int flags)
+{
+    return xrdp_bitmap32_compress(in_data, width, height,
+                                  s, bpp, byte_limit,
+                                  start_line, temp_s,
+                                  e, flags);
+}
+

libxrdp/libxrdpinc.h

@@ -306,6 +306,11 @@ libxrdp_fastpath_send_frame_marker(struct xrdp_session *session,
 int EXPORT_CC
 libxrdp_send_session_info(struct xrdp_session *session, const char *data,
                           int data_bytes);
+int EXPORT_CC
+libxrdp_planar_compress(char *in_data, int width, int height,
+                        struct stream *s, int bpp, int byte_limit,
+                        int start_line, struct stream *temp_s,
+                        int e, int flags);
 
 /**
  * Processes a stream that is based on either

libxrdp/xrdp_sec.c

@@ -24,8 +24,11 @@
 
 #include "libxrdp.h"
 #include "ms-rdpbcgr.h"
+#include "ms-rdpedisp.h"
 #include "log.h"
 #include "string_calls.h"
+#include <limits.h>
+
 
 /* some compilers need unsigned char to avoid warnings */
 static tui8 g_pad_54[40] =
@@ -2081,6 +2084,15 @@ xrdp_sec_process_mcs_data_CS_CORE(struct xrdp_sec *self, struct stream *s)
     {
         client_info->bpp = 32;
     }
+    if (earlyCapabilityFlags & 0x100) /* RNS_UD_CS_SUPPORT_DYNVC_GFX_PROTOCOL */
+    {
+        LOG_DEVEL(LOG_LEVEL_INFO, "client supports gfx");
+        self->rdp_layer->client_info.gfx = 1;
+    }
+    else
+    {
+        LOG_DEVEL(LOG_LEVEL_INFO, "client DOES NOT support gfx");
+    }
 
     if (!s_check_rem(s, 64))
     {

scripts/install_xrdp_build_dependencies_with_apt.sh

@@ -69,7 +69,9 @@ in
             libssl-dev \
             libx11-dev \
             libxrandr-dev \
-            libxfixes-dev"
+            libxfixes-dev \
+            libepoxy-dev \
+            libepoxy0"
 
         case "$FEATURE_SET"
         in
@@ -111,7 +113,9 @@ in
             libxfixes-dev:i386 \
             libxrandr-dev:i386 \
             libxrender-dev:i386 \
-            libfuse-dev:i386"
+            libfuse-dev:i386 \
+            libepoxy-dev:i386 \
+            libepoxy0:i386"
 
         dpkg --add-architecture i386
         dpkg --print-architecture

sesman/sesman.ini.in

@@ -120,7 +120,7 @@ EnableSyslog=true
 param=Xorg
 ; Leave the rest paramaters as-is unless you understand what will happen.
 param=-config
-param=xrdp/xorg.conf
+param=xrdp/xorg_nvidia.conf
 param=-noreset
 param=-nolisten
 param=tcp
@@ -173,3 +173,4 @@ EnableSyslog=true
 
 [SessionVariables]
 PULSE_SCRIPT=@sesmansysconfdir@/pulse/default.pa
+XRDP_USE_HELPER=1

xorgxrdp_helper/Makefile.am

@@ -0,0 +1,35 @@
+AM_CPPFLAGS = \
+  -I$(top_srcdir)/common
+
+XRDP_EXTRA_LIBS =
+XRDP_EXTRA_SOURCES =
+
+if XRDP_NVENC
+AM_CPPFLAGS += -DXRDP_NVENC
+AM_CPPFLAGS += $(XRDP_NVENC_CFLAGS)
+XRDP_EXTRA_LIBS += $(XRDP_NVENC_LIBS)
+XRDP_EXTRA_SOURCES += xorgxrdp_helper_nvenc.c xorgxrdp_helper_nvenc.h
+endif
+
+if XRDP_YAMI
+AM_CPPFLAGS += -DXRDP_YAMI
+AM_CPPFLAGS += $(XRDP_YAMI_CFLAGS)
+XRDP_EXTRA_LIBS += $(XRDP_YAMI_LIBS)
+XRDP_EXTRA_SOURCES += xorgxrdp_helper_yami.c xorgxrdp_helper_yami.h
+endif
+
+bin_PROGRAMS = \
+  xorgxrdp_helper
+
+xorgxrdp_helper_SOURCES = \
+  xorgxrdp_helper.c \
+  xorgxrdp_helper.h \
+  xorgxrdp_helper_x11.c \
+  xorgxrdp_helper_x11.h \
+  $(XRDP_EXTRA_SOURCES)
+
+xorgxrdp_helper_LDADD = \
+  $(top_builddir)/common/libcommon.la \
+  $(XRDP_EXTRA_LIBS) \
+  -lX11 -lepoxy
+