diff --git a/core/src/main/resources/hudson/widgets/HistoryWidget/index.jelly b/core/src/main/resources/hudson/widgets/HistoryWidget/index.jelly
index 8d4663bb00dde..6b158ddc4ea6c 100644
--- a/core/src/main/resources/hudson/widgets/HistoryWidget/index.jelly
+++ b/core/src/main/resources/hudson/widgets/HistoryWidget/index.jelly
@@ -56,7 +56,7 @@ THE SOFTWARE.
         <div id="jenkins-build-history" class="app-builds-container__items">
         </div>
 
-        <div class="app-builds-container__controls" id="controls">
+        <div class="app-builds-container__controls jenkins-hidden" id="controls">
           <button class="jenkins-button jenkins-button--tertiary jenkins-card__unveil" id="up">
             <l:icon src="symbol-arrow-left" />
             <span class="jenkins-visually-hidden">${%Newer builds}</span>
diff --git a/war/src/main/js/pages/project/builds-card.js b/war/src/main/js/pages/project/builds-card.js
index 5f2e5341e98c6..3e364a2ace347 100644
--- a/war/src/main/js/pages/project/builds-card.js
+++ b/war/src/main/js/pages/project/builds-card.js
@@ -26,16 +26,28 @@ const updateBuildsRefreshInterval = 5000;
  */
 function load(options = {}) {
   /** @type {QueryParameters} */
+  cancelRefreshTimeout();
   const params = Object.assign({}, options, { search: pageSearchInput.value });
+  const paginationOrFirst =
+    buildHistoryPage.dataset.pageHasUp === "false" ||
+    "older-than" in params ||
+    "newer-than" in params;
 
-  // Avoid fetching if the page isn't active
+  // Avoid fetching if the page isn't visible
   if (document.hidden) {
-    if (buildHistoryPage.dataset.pageHasUp === "false") {
-      createRefreshTimeout();
-    }
     return;
   }
 
+  createRefreshTimeout();
+
+  // When we're not on the first page and this is not a load due to pagination
+  // we need to set the correct value for older-than so we fetch the same set of runs
+  if (!paginationOrFirst) {
+    params["older-than"] = (
+      BigInt(buildHistoryPage.dataset.pageEntryNewest) + 1n
+    ).toString();
+  }
+
   fetch(ajaxUrl + toQueryString(params)).then((rsp) => {
     if (rsp.ok) {
       rsp.text().then((responseText) => {
@@ -69,6 +81,7 @@ function load(options = {}) {
           pageHasDown: innerChild.dataset.pageHasDown === "true",
           pageEntryNewest: innerChild.dataset.pageEntryNewest,
           pageEntryOldest: innerChild.dataset.pageEntryOldest,
+          activeRuns: innerChild.dataset.activeRuns,
         });
       });
     } else {
@@ -95,13 +108,6 @@ function updateCardControls(parameters) {
     !parameters.pageHasDown,
   );
 
-  // We only want the list to refresh if the user is on the first page of results
-  if (!parameters.pageHasUp) {
-    createRefreshTimeout();
-  } else {
-    cancelRefreshTimeout();
-  }
-
   buildHistoryPage.dataset.pageEntryNewest = parameters.pageEntryNewest;
   buildHistoryPage.dataset.pageEntryOldest = parameters.pageEntryOldest;
   buildHistoryPage.dataset.pageHasUp = parameters.pageHasUp;
@@ -112,7 +118,6 @@ paginationPrevious.addEventListener("click", () => {
 });
 
 paginationNext.addEventListener("click", () => {
-  cancelRefreshTimeout();
   load({ "older-than": buildHistoryPage.dataset.pageEntryOldest });
 });
 
@@ -143,4 +148,8 @@ document.addEventListener("DOMContentLoaded", function () {
   });
 
   load();
+
+  window.addEventListener("focus", function () {
+    load();
+  });
 });