diff --git a/kitty/mouse.c b/kitty/mouse.c
index d1eb769171e..576f56c1d81 100644
--- a/kitty/mouse.c
+++ b/kitty/mouse.c
@@ -130,11 +130,11 @@ distance_to_window(Window *w, OSWindow *os_window) {
 static bool clamp_to_window = false;
 
 static inline bool
-cell_for_pos(Window *w, unsigned int *x, unsigned int *y, OSWindow *os_window) {
+cell_for_pos(Window *w, unsigned int *x, unsigned int *y, unsigned int *edge_x, unsigned int *edge_y, OSWindow *os_window) {
     WindowGeometry *g = &w->geometry;
     Screen *screen = w->render_data.screen;
     if (!screen) return false;
-    unsigned int qx = 0, qy = 0;
+    double qx = 0, qy = 0;
     double mouse_x = global_state.callback_os_window->mouse_x;
     double mouse_y = global_state.callback_os_window->mouse_y;
     double left = window_left(w, os_window), top = window_top(w, os_window), right = window_right(w, os_window), bottom = window_bottom(w, os_window);
@@ -144,12 +144,15 @@ cell_for_pos(Window *w, unsigned int *x, unsigned int *y, OSWindow *os_window) {
     }
     w->mouse_pos.x = mouse_x - left; w->mouse_pos.y = mouse_y - top;
     if (mouse_x < left || mouse_y < top || mouse_x > right || mouse_y > bottom) return false;
-    if (mouse_x >= g->right) qx = screen->columns - 1;
-    else if (mouse_x >= g->left) qx = (unsigned int)((double)(mouse_x - g->left) / os_window->fonts_data->cell_width);
-    if (mouse_y >= g->bottom) qy = screen->lines - 1;
-    else if (mouse_y >= g->top) qy = (unsigned int)((double)(mouse_y - g->top) / os_window->fonts_data->cell_height);
+    if (mouse_x >= g->right) qx = screen->columns - 0.5;
+    else if (mouse_x >= g->left) qx = ((double)(mouse_x - g->left) / os_window->fonts_data->cell_width);
+    if (mouse_y >= g->bottom) qy = screen->lines - 0.5;
+    else if (mouse_y >= g->top) qy = ((double)(mouse_y - g->top) / os_window->fonts_data->cell_height);
     if (qx < screen->columns && qy < screen->lines) {
-        *x = qx; *y = qy;
+        *x = (unsigned int)qx;
+        *y = (unsigned int)qy;
+        *edge_x = (unsigned int)(qx + 0.5);
+        *edge_y = (unsigned int)(qy + 0.5);
         return true;
     }
     return false;
@@ -165,14 +168,15 @@ update_drag(bool from_button, Window *w, bool is_release, int modifiers) {
             global_state.active_drag_in_window = 0;
             w->last_drag_scroll_at = 0;
             if (screen->selection.in_progress)
-                screen_update_selection(screen, w->mouse_pos.cell_x, w->mouse_pos.cell_y, true);
+                screen_update_selection(screen, w->mouse_pos.edge_x, screen->selection.rectangle_select ? w->mouse_pos.edge_y : w->mouse_pos.cell_y, true);
         }
         else {
+            bool rectangle_select = modifiers == (int)OPT(rectangle_select_modifiers) || modifiers == ((int)OPT(rectangle_select_modifiers) | GLFW_MOD_SHIFT);
             global_state.active_drag_in_window = w->id;
-            screen_start_selection(screen, w->mouse_pos.cell_x, w->mouse_pos.cell_y, modifiers == (int)OPT(rectangle_select_modifiers) || modifiers == ((int)OPT(rectangle_select_modifiers) | GLFW_MOD_SHIFT), EXTEND_CELL);
+            screen_start_selection(screen, w->mouse_pos.edge_x, rectangle_select ? w->mouse_pos.edge_y : w->mouse_pos.cell_y, rectangle_select, EXTEND_CELL);
         }
     } else if (screen->selection.in_progress) {
-        screen_update_selection(screen, w->mouse_pos.cell_x, w->mouse_pos.cell_y, false);
+        screen_update_selection(screen, w->mouse_pos.edge_x, screen->selection.rectangle_select ? w->mouse_pos.edge_y : w->mouse_pos.cell_y, false);
     }
 }
 
@@ -267,18 +271,21 @@ detect_url(Screen *screen, unsigned int x, unsigned int y) {
 
 
 HANDLER(handle_move_event) {
-    unsigned int x = 0, y = 0;
+    unsigned int x = 0, y = 0, edge_x = 0, edge_y = 0;
     if (OPT(focus_follows_mouse)) {
         Tab *t = global_state.callback_os_window->tabs + global_state.callback_os_window->active_tab;
         if (window_idx != t->active_window) {
             call_boss(switch_focus_to, "I", window_idx);
         }
     }
-    if (!cell_for_pos(w, &x, &y, global_state.callback_os_window)) return;
+    if (!cell_for_pos(w, &x, &y, &edge_x, &edge_y, global_state.callback_os_window)) return;
+
     Screen *screen = w->render_data.screen;
     detect_url(screen, x, y);
     bool mouse_cell_changed = x != w->mouse_pos.cell_x || y != w->mouse_pos.cell_y;
+    bool mouse_edge_changed = edge_x != w->mouse_pos.edge_x || edge_y != w->mouse_pos.edge_y;
     w->mouse_pos.cell_x = x; w->mouse_pos.cell_y = y;
+    w->mouse_pos.edge_x = edge_x; w->mouse_pos.edge_y = edge_y;
     bool handle_in_kitty = (
             (screen->modes.mouse_tracking_mode == ANY_MODE ||
             (screen->modes.mouse_tracking_mode == MOTION_MODE && button >= 0)) &&
@@ -287,7 +294,7 @@ HANDLER(handle_move_event) {
     if (handle_in_kitty) {
         if (screen->selection.in_progress && button == GLFW_MOUSE_BUTTON_LEFT) {
             double now = monotonic();
-            if ((now - w->last_drag_scroll_at) >= 0.02 || mouse_cell_changed) {
+            if ((now - w->last_drag_scroll_at) >= 0.02 || mouse_edge_changed || mouse_cell_changed) {
                 update_drag(false, w, false, 0);
                 w->last_drag_scroll_at = monotonic();
             }
diff --git a/kitty/screen.c b/kitty/screen.c
index a91f832e94e..3acdad6e9c9 100644
--- a/kitty/screen.c
+++ b/kitty/screen.c
@@ -1500,7 +1500,7 @@ screen_update_cell_data(Screen *self, void *address, FONTS_DATA_HANDLE fonts_dat
 
 static inline bool
 is_selection_empty(Screen *self, unsigned int start_x, unsigned int start_y, unsigned int end_x, unsigned int end_y) {
-    return (start_x >= self->columns || start_y >= self->lines || end_x >= self->columns || end_y >= self->lines || (start_x == end_x && start_y == end_y)) ? true : false;
+    return start_x >= self->columns || start_y >= self->lines || end_x >= self->columns || end_y >= self->lines;
 }
 
 static inline void
@@ -2030,7 +2030,14 @@ screen_is_selection_dirty(Screen *self) {
 void
 screen_start_selection(Screen *self, index_type x, index_type y, bool rectangle_select, SelectionExtendMode extend_mode) {
 #define A(attr, val) self->selection.attr = val;
-    A(start_x, x); A(end_x, x); A(start_y, y); A(end_y, y); A(start_scrolled_by, self->scrolled_by); A(end_scrolled_by, self->scrolled_by);
+    A(anchor_x, x);
+    A(anchor_y, y);
+    A(start_x, UINT_MAX);
+    A(start_y, UINT_MAX);
+    A(end_x, UINT_MAX);
+    A(end_y, UINT_MAX);
+    A(start_scrolled_by, self->scrolled_by);
+    A(end_scrolled_by, self->scrolled_by);
     A(in_progress, true); A(rectangle_select, rectangle_select); A(extend_mode, extend_mode);
 #undef A
 }
@@ -2044,27 +2051,55 @@ screen_mark_url(Screen *self, index_type start_x, index_type start_y, index_type
 
 void
 screen_update_selection(Screen *self, index_type x, index_type y, bool ended) {
-    self->selection.end_x = x; self->selection.end_y = y; self->selection.end_scrolled_by = self->scrolled_by;
     if (ended) self->selection.in_progress = false;
-    index_type start, end;
-    bool found = false;
-    bool extending_leftwards = self->selection.end_y < self->selection.start_y || (self->selection.end_y == self->selection.start_y && self->selection.end_x < self->selection.start_x);
-    switch(self->selection.extend_mode) {
+    bool empty = false;
+    bool extending_leftwards = y < self->selection.anchor_y || (y == self->selection.anchor_y && x <= self->selection.anchor_x);
+    if (self->selection.rectangle_select) {
+        if (x == self->selection.anchor_x || y == self->selection.anchor_y) {
+            empty = true;
+        } else {
+            self->selection.start_x = MIN(self->selection.anchor_x, x);
+            self->selection.end_x = MAX(self->selection.anchor_x, x) - 1;
+            self->selection.start_y = MIN(self->selection.anchor_y, y);
+            self->selection.end_y = MAX(self->selection.anchor_y, y) - 1;
+        }
+    } else {
+        if (self->selection.extend_mode == EXTEND_CELL && x == self->selection.anchor_x && y == self->selection.anchor_y) {
+            empty = true;
+        } else if (extending_leftwards) {
+            self->selection.start_x = x;
+            self->selection.start_y = y;
+            self->selection.end_x = self->selection.anchor_x - 1;
+            self->selection.end_y = self->selection.anchor_y;
+        } else {
+            self->selection.start_x = self->selection.anchor_x;
+            self->selection.start_y = self->selection.anchor_y;
+            self->selection.end_x = x - 1;
+            self->selection.end_y = y;
+        }
+    }
+
+    if (empty) {
+        self->selection.start_x = self->selection.start_y = self->selection.end_x = self->selection.end_y = UINT_MAX;
+    } else {
+        index_type start, end;
+        bool found = false;
+        switch(self->selection.extend_mode) {
         case EXTEND_WORD: {
             index_type y1 = y, y2;
-            found = screen_selection_range_for_word(self, x, &y1, &y2, &start, &end);
+            found = screen_selection_range_for_word(self, x - (extending_leftwards ? 0 : 1), &y1, &y2, &start, &end);
             if (found) {
                 if (extending_leftwards) {
                     self->selection.end_x = start; self->selection.end_y = y1;
-                    y1 = self->selection.start_y;
-                    found = screen_selection_range_for_word(self, self->selection.start_x, &y1, &y2, &start, &end);
+                    y1 = self->selection.anchor_y;
+                    found = screen_selection_range_for_word(self, self->selection.anchor_x, &y1, &y2, &start, &end);
                     if (found) {
                         self->selection.start_x = end; self->selection.start_y = y2;
                     }
                 } else {
                     self->selection.end_x = end; self->selection.end_y = y2;
-                    y1 = self->selection.start_y;
-                    found = screen_selection_range_for_word(self, self->selection.start_x, &y1, &y2, &start, &end);
+                    y1 = self->selection.anchor_y;
+                    found = screen_selection_range_for_word(self, self->selection.anchor_x, &y1, &y2, &start, &end);
                     if (found) {
                         self->selection.start_x = start; self->selection.start_y = y1;
                     }
@@ -2074,8 +2109,8 @@ screen_update_selection(Screen *self, index_type x, index_type y, bool ended) {
             break;
         }
         case EXTEND_LINE: {
-            index_type top_line = extending_leftwards ? self->selection.end_y : self->selection.start_y;
-            index_type bottom_line = extending_leftwards ? self->selection.start_y : self->selection.end_y;
+            index_type top_line = self->selection.start_y;
+            index_type bottom_line = self->selection.end_y;
             while(top_line > 0 && visual_line_(self, top_line)->continued) top_line--;
             while(bottom_line < self->lines - 1 && visual_line_(self, bottom_line + 1)->continued) bottom_line++;
             found = screen_selection_range_for_line(self, top_line, &start, &end);
@@ -2098,6 +2133,7 @@ screen_update_selection(Screen *self, index_type x, index_type y, bool ended) {
         }
         case EXTEND_CELL:
             break;
+        }
     }
     call_boss(set_primary_selection, NULL);
 }
diff --git a/kitty/screen.h b/kitty/screen.h
index 06a463230fa..3eac3b8d5e3 100644
--- a/kitty/screen.h
+++ b/kitty/screen.h
@@ -25,7 +25,7 @@ typedef struct {
 typedef enum SelectionExtendModes { EXTEND_CELL, EXTEND_WORD, EXTEND_LINE } SelectionExtendMode;
 
 typedef struct {
-    unsigned int start_x, start_y, start_scrolled_by, end_x, end_y, end_scrolled_by;
+    unsigned int anchor_x, anchor_y, start_x, start_y, start_scrolled_by, end_x, end_y, end_scrolled_by;
     bool in_progress, rectangle_select;
     SelectionExtendMode extend_mode;
 } Selection;
diff --git a/kitty/state.h b/kitty/state.h
index e8d5ddf1201..5e343d41c5a 100644
--- a/kitty/state.h
+++ b/kitty/state.h
@@ -70,6 +70,7 @@ typedef struct {
     ScreenRenderData render_data;
     struct {
         unsigned int cell_x, cell_y;
+        unsigned int edge_x, edge_y;
         double x, y;
     } mouse_pos;
     WindowGeometry geometry;
diff --git a/kitty_tests/screen.py b/kitty_tests/screen.py
old mode 100644
new mode 100755
index 9b31da3d02e..4dfb13126fc
--- a/kitty_tests/screen.py
+++ b/kitty_tests/screen.py
@@ -423,7 +423,7 @@ def test_selection_as_text(self):
                 s.carriage_return(), s.linefeed()
             s.draw(str(i) * s.columns)
         s.start_selection(0, 0, False)
-        s.update_selection(4, 4, True)
+        s.update_selection(5, 4, True)
         expected = ('55555', '\n66666', '\n77777', '\n88888', '\n99999')
         self.ae(s.text_for_selection(), expected)
         s.scroll(2, True)