TEST: Tightens file-based condition in peer-recovery

As a replica always keeps a safe commit and starts peer-recovery with
that commit; file-based recovery  only happens if new operations are
added to the primary and the required translog is not fully retained. In
the test, we tried to produce this condition by flushing a new commit in
order to trim all translog. However, if the new global checkpoint is not
persisted yet, we will keep two commits and not trim translog. This
commit tightens the file-based condition in the test by waiting for the
global checkpoint persisted properly on the new primary before flushing.

Close #28209
Relates #28181

Author: dnhatn

server/src/test/java/org/elasticsearch/index/replication/RecoveryDuringReplicationTests.java

@@ -215,7 +215,6 @@ public void testRecoveryToReplicaThatReceivedExtraDocument() throws Exception {
     }
 
     @TestLogging("org.elasticsearch.index.shard:TRACE,org.elasticsearch.indices.recovery:TRACE")
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/28209")
     public void testRecoveryAfterPrimaryPromotion() throws Exception {
         try (ReplicationGroup shards = createGroup(2)) {
             shards.startAll();
@@ -268,7 +267,12 @@ public void testRecoveryAfterPrimaryPromotion() throws Exception {
                 );
                 newPrimary.indexSettings().updateIndexMetaData(builder.build());
                 newPrimary.onSettingsChanged();
-                shards.syncGlobalCheckpoint();
+                // Make sure the global checkpoint on the new primary is persisted properly,
+                // otherwise the deletion policy won't trim translog
+                assertBusy(() -> {
+                    shards.syncGlobalCheckpoint();
+                    assertThat(newPrimary.getTranslog().getLastSyncedGlobalCheckpoint(), equalTo(newPrimary.seqNoStats().getMaxSeqNo()));
+                });
                 newPrimary.flush(new FlushRequest());
                 uncommittedOpsOnPrimary = shards.indexDocs(randomIntBetween(0, 10));
                 totalDocs += uncommittedOpsOnPrimary;