Resolution switching without reconnecting.

- Based on https://github.com/jsorg71/xrdp/tree/dynamic_monitor
- Tested with xorgxrdp
- Tested with vnc
- Only works with single monitor.
- Update documentation to clarify the difference between MSTSC and
Microsoft Remote Desktop.
- Compatible with NVENC and xorgxrdp_helper
- Compatible with Nvidia.
- Updates to include ms-rdpedisp.h header for the 2.2.2 specification of
the protocol.
- State machine for resizing.
- Mechanisms to make sure the key frame is always sent on resize.
- Adds new struct that shares the number of monitors with xrdp_client_info.h
- Modification to the original resize setup that works with /gfx.
- YAMI compat.
- Updating to librfxcodec branch that is also a merge from mainline.
- Made sure RFX progressive works.

Depends on neutrinolabs/xorgxrdp#183

.gitmodules

@@ -1,7 +1,7 @@
 [submodule "librfxcodec"]
 	path = librfxcodec
-	url = https://github.com/neutrinolabs/librfxcodec.git
-	branch = .
+	url = https://github.com/Nexarian/librfxcodec.git
+	branch = egfx_mainline_merge
 	ignore = untracked
 
 [submodule "libpainter"]

.vscode/settings.json

@@ -0,0 +1,5 @@
+{
+    "files.associations": {
+        "trans.h": "c"
+    }
+}

Makefile.am

@@ -59,7 +59,8 @@ SUBDIRS = \
   pkgconfig \
   $(XRDPVRDIR) \
   tests \
-  tools
+  tools \
+  xorgxrdp_helper
 
 distclean-local:
 	-rm -f xrdp_configure_options.h

common/ms-rdpbcgr.h

@@ -94,10 +94,10 @@
 /* the maximum height of the virtual desktop resulting from the union of the monitors  */
 /* contained in the monitorDefArray field MUST NOT exceed 32,766 pixels. */
 /* The minimum permitted size of the virtual desktop is 200 x 200 pixels. */
-#define CLIENT_MONITOR_DATA_MINIMUM_VIRTUAL_DESKTOP_WIDTH  0xC8
-#define CLIENT_MONITOR_DATA_MINIMUM_VIRTUAL_DESKTOP_HEIGHT 0xC8
-#define CLIENT_MONITOR_DATA_MAXIMUM_VIRTUAL_DESKTOP_WIDTH  0x7FFE
-#define CLIENT_MONITOR_DATA_MAXIMUM_VIRTUAL_DESKTOP_HEIGHT 0x7FFE
+#define CLIENT_MONITOR_DATA_MINIMUM_VIRTUAL_DESKTOP_WIDTH  0xC8   // 200
+#define CLIENT_MONITOR_DATA_MINIMUM_VIRTUAL_DESKTOP_HEIGHT 0xC8   // 200
+#define CLIENT_MONITOR_DATA_MAXIMUM_VIRTUAL_DESKTOP_WIDTH  0x7FFE // 32766
+#define CLIENT_MONITOR_DATA_MAXIMUM_VIRTUAL_DESKTOP_HEIGHT 0x7FFE // 32766
 
 /* 2.2.1.3.6.1 Monitor Definition (TS_MONITOR_DEF) */
 #define TS_MONITOR_PRIMARY 0x00000001

common/ms-rdpedisp.h

@@ -38,4 +38,7 @@
 #define ORIENTATION_LANDSCAPE_FLIPPED                      180
 #define ORIENTATION_PORTRAIT_FLIPPED                       270
 
+/* Display Control Monitor Layout (2.2.2.2.1) */
+#define DISPLAYCONTROL_MONITOR_PRIMARY         0x00000001
+
 #endif /* MS_RDPEDISP_H */

common/pixman-region.h

@@ -68,9 +68,11 @@ pixman_bool_t           pixman_region_subtract           (pixman_region16_t *reg
         /**/                                              pixman_region16_t *reg_m,
         /**/                                              pixman_region16_t *reg_s);
 pixman_bool_t           pixman_region_intersect          (pixman_region16_t *new_reg,
-        /**/                                              pixman_region16_t *reg1,
-        /**/                                              pixman_region16_t *reg2);
+        pixman_region16_t *reg1,
+        pixman_region16_t *reg2);
 pixman_box16_t         *pixman_region_rectangles         (pixman_region16_t *region,
-        /**/                                              int               *n_rects);
+        int *n_rects);
+pixman_bool_t           pixman_region_not_empty          (pixman_region16_t *region);
+pixman_box16_t         *pixman_region_extents            (pixman_region16_t *region);
 
 #endif

common/trans.c

@@ -1065,3 +1065,23 @@ trans_shutdown_tls_mode(struct trans *self)
 
     return 0;
 }
+
+/******************************************************************************/
+int
+trans_use_helper()
+{
+    const char *xrdp_use_helper = getenv("XRDP_USE_HELPER");
+    if (xrdp_use_helper == NULL)
+    {
+        return 0;
+    }
+    if (g_strcmp(xrdp_use_helper, "0") == 0)
+    {
+        return 0;
+    }
+    if (g_strcmp(xrdp_use_helper, "1") == 0)
+    {
+        return 1;
+    }
+    return 0;
+}

common/trans.h

@@ -57,6 +57,8 @@ enum xrdp_source
     XRDP_SOURCE_SESMAN,
     XRDP_SOURCE_CHANSRV,
     XRDP_SOURCE_MOD,
+    XORGXRDP_SOURCE_XORG,
+    XORGXRDP_SOURCE_XRDP,
 
     XRDP_SOURCE_MAX_COUNT
 };
@@ -179,5 +181,7 @@ int
 trans_shutdown_tls_mode(struct trans *self);
 int
 trans_tcp_force_read_s(struct trans *self, struct stream *in_s, int size);
+int
+trans_use_helper();
 
 #endif

common/xrdp_client_info.h

@@ -66,6 +66,58 @@ struct display_size_description
     unsigned int session_height;
 };
 
+enum display_resize_state
+{
+    WMRZ_QUEUED = 0,
+    WMRZ_ENCODER_DELETE,
+    WMRZ_EGFX_DELETE_SURFACE,
+    WMRZ_EGFX_CONN_CLOSE,
+    WMRZ_EGFX_CONN_CLOSING,
+    WMRZ_EGFX_CONN_CLOSED,
+    WRMZ_EGFX_DELETE,
+    WMRZ_SERVER_MONITOR_RESIZE,
+    WMRZ_SERVER_VERSION_MESSAGE_START,
+    WMRZ_SERVER_MONITOR_MESSAGE_PROCESSING,
+    WMRZ_SERVER_MONITOR_MESSAGE_PROCESSED,
+    WMRZ_XRDP_CORE_RESIZE,
+    WMRZ_EGFX_INITIALIZE,
+    WMRZ_EGFX_INITALIZING,
+    WMRZ_EGFX_INITIALIZED,
+    WMRZ_SERVER_INVALIDATE,
+    WMRZ_COMPLETE,
+    WMRZ_ERROR
+};
+
+struct dynamic_monitor_description
+{
+    struct display_size_description description;
+    enum display_resize_state state;
+    int last_state_update_timestamp;
+    int start_time;
+};
+
+#define XRDP_DISPLAY_RESIZE_STATE_TO_STR(status) \
+    ((status) == WMRZ_QUEUED ? "WMRZ_QUEUED" : \
+     (status) == WMRZ_ENCODER_DELETE ? "WMRZ_ENCODER_DELETE" : \
+     (status) == WMRZ_EGFX_DELETE_SURFACE ? "EGFX_DELETE_SURFACE" : \
+     (status) == WMRZ_EGFX_CONN_CLOSE ? "EGFX_CONN_CLOSE" : \
+     (status) == WMRZ_EGFX_CONN_CLOSING ? "EGFX_CONN_CLOSING" : \
+     (status) == WMRZ_EGFX_CONN_CLOSED ? "EGFX_CONN_CLOSED" : \
+     (status) == WRMZ_EGFX_DELETE ? "EGFX_DELETE" : \
+     (status) == WMRZ_SERVER_MONITOR_RESIZE ? "SERVER_MONITOR_RESIZE" : \
+     (status) == WMRZ_SERVER_VERSION_MESSAGE_START ? "SERVER_VERSION_MESSAGE_START" : \
+     (status) == WMRZ_SERVER_MONITOR_MESSAGE_PROCESSING ? "SERVER_MONITOR_MESSAGE_PROCESSING" : \
+     (status) == WMRZ_SERVER_MONITOR_MESSAGE_PROCESSED ? "SERVER_MONITOR_MESSAGE_PROCESSED" : \
+     (status) == WMRZ_XRDP_CORE_RESIZE ? "XRDP_CORE_RESIZE" : \
+     (status) == WMRZ_EGFX_INITIALIZE ? "EGFX_INITIALIZE" : \
+     (status) == WMRZ_EGFX_INITALIZING ? "EGFX_INITALIZING" : \
+     (status) == WMRZ_EGFX_INITIALIZED ? "EGFX_INITIALIZED" : \
+     (status) == WMRZ_SERVER_INVALIDATE ? "SERVER_INVALIDATE" : \
+     (status) == WMRZ_COMPLETE ? "COMPLETE" : \
+     (status) == WMRZ_ERROR ? "ERROR" : \
+     "unknown" \
+    )
+
 /**
  * Information about the xrdp client
  *
@@ -204,6 +256,7 @@ struct xrdp_client_info
 
     /* xrdp.override_* values */
     struct xrdp_keyboard_overrides xrdp_keyboard_overrides;
+    int gfx;
 };
 
 /* yyyymmdd of last incompatible change to xrdp_client_info */

libxrdp/libxrdp.c

@@ -1374,6 +1374,10 @@ libxrdp_send_to_channel(struct xrdp_session *session, int channel_id,
         free_stream(s);
         return 1;
     }
+    else
+    {
+        LOG(LOG_LEVEL_TRACE, "libxrdp_send_to_channel: xrdp_channel_init successful!");
+    }
 
     /* here we make a copy of the data */
     out_uint8a(s, data, data_len);
@@ -2136,3 +2140,15 @@ libxrdp_process_monitor_stream(struct stream *s,
     }
     return 0;
 }
+int EXPORT_CC
+libxrdp_planar_compress(char *in_data, int width, int height,
+                        struct stream *s, int bpp, int byte_limit,
+                        int start_line, struct stream *temp_s,
+                        int e, int flags)
+{
+    return xrdp_bitmap32_compress(in_data, width, height,
+                                  s, bpp, byte_limit,
+                                  start_line, temp_s,
+                                  e, flags);
+}
+

libxrdp/libxrdpinc.h

@@ -306,6 +306,11 @@ libxrdp_fastpath_send_frame_marker(struct xrdp_session *session,
 int EXPORT_CC
 libxrdp_send_session_info(struct xrdp_session *session, const char *data,
                           int data_bytes);
+int EXPORT_CC
+libxrdp_planar_compress(char *in_data, int width, int height,
+                        struct stream *s, int bpp, int byte_limit,
+                        int start_line, struct stream *temp_s,
+                        int e, int flags);
 
 /**
  * Processes a stream that is based on either

libxrdp/xrdp_sec.c

@@ -24,8 +24,11 @@
 
 #include "libxrdp.h"
 #include "ms-rdpbcgr.h"
+#include "ms-rdpedisp.h"
 #include "log.h"
 #include "string_calls.h"
+#include <limits.h>
+
 
 /* some compilers need unsigned char to avoid warnings */
 static tui8 g_pad_54[40] =
@@ -2073,6 +2076,15 @@ xrdp_sec_process_mcs_data_CS_CORE(struct xrdp_sec *self, struct stream *s)
     {
         self->rdp_layer->client_info.bpp = 32;
     }
+    if (earlyCapabilityFlags & 0x100) /* RNS_UD_CS_SUPPORT_DYNVC_GFX_PROTOCOL */
+    {
+        LOG_DEVEL(LOG_LEVEL_INFO, "client supports gfx");
+        self->rdp_layer->client_info.gfx = 1;
+    }
+    else
+    {
+        LOG_DEVEL(LOG_LEVEL_INFO, "client DOES NOT support gfx");
+    }
 
     if (!s_check_rem(s, 64))
     {

scripts/install_xrdp_build_dependencies_with_apt.sh

@@ -58,7 +58,7 @@ PACKAGES=" \
     make \
     nasm \
     pkg-config \
-    check \
+    check
     "
 
 case "$ARCH"
@@ -69,7 +69,9 @@ in
             libssl-dev \
             libx11-dev \
             libxrandr-dev \
-            libxfixes-dev"
+            libxfixes-dev \
+            libepoxy-dev \
+            libepoxy0"
 
         case "$FEATURE_SET"
         in
@@ -111,7 +113,9 @@ in
             libxfixes-dev:i386 \
             libxrandr-dev:i386 \
             libxrender-dev:i386 \
-            libfuse-dev:i386"
+            libfuse-dev:i386 \
+            libepoxy-dev:i386 \
+            libepoxy0:i386"
 
         dpkg --add-architecture i386
         dpkg --print-architecture

sesman/sesman.ini.in

@@ -113,7 +113,7 @@ EnableSyslog=true
 param=Xorg
 ; Leave the rest paramaters as-is unless you understand what will happen.
 param=-config
-param=xrdp/xorg.conf
+param=xrdp/xorg_nvidia.conf
 param=-noreset
 param=-nolisten
 param=tcp
@@ -166,3 +166,4 @@ EnableSyslog=true
 
 [SessionVariables]
 PULSE_SCRIPT=@sesmansysconfdir@/pulse/default.pa
+XRDP_USE_HELPER=1

xorgxrdp_helper/Makefile.am

@@ -0,0 +1,35 @@
+AM_CPPFLAGS = \
+  -I$(top_srcdir)/common
+
+XRDP_EXTRA_LIBS =
+XRDP_EXTRA_SOURCES =
+
+if XRDP_NVENC
+AM_CPPFLAGS += -DXRDP_NVENC
+AM_CPPFLAGS += $(XRDP_NVENC_CFLAGS)
+XRDP_EXTRA_LIBS += $(XRDP_NVENC_LIBS)
+XRDP_EXTRA_SOURCES += xorgxrdp_helper_nvenc.c xorgxrdp_helper_nvenc.h
+endif
+
+if XRDP_YAMI
+AM_CPPFLAGS += -DXRDP_YAMI
+AM_CPPFLAGS += $(XRDP_YAMI_CFLAGS)
+XRDP_EXTRA_LIBS += $(XRDP_YAMI_LIBS)
+XRDP_EXTRA_SOURCES += xorgxrdp_helper_yami.c xorgxrdp_helper_yami.h
+endif
+
+bin_PROGRAMS = \
+  xorgxrdp_helper
+
+xorgxrdp_helper_SOURCES = \
+  xorgxrdp_helper.c \
+  xorgxrdp_helper.h \
+  xorgxrdp_helper_x11.c \
+  xorgxrdp_helper_x11.h \
+  $(XRDP_EXTRA_SOURCES)
+
+xorgxrdp_helper_LDADD = \
+  $(top_builddir)/common/libcommon.la \
+  $(XRDP_EXTRA_LIBS) \
+  -lX11 -lepoxy
+