Bugfix: repair rescan when sync stuck and allow sync to continue

build.gradle

@@ -3,8 +3,8 @@ buildscript {
         kotlin_version = '1.9.23'
         coroutinesVersion = '1.6.4'
         ok_http_version = '4.9.1'
-        dashjVersion = '21.1.3'
-        dppVersion = "1.5.1-SNAPSHOT"
+        dashjVersion = '21.1.4-SNAPSHOT'
+        dppVersion = "1.5.2-SNAPSHOT"
         hiltVersion = '2.51'
         hiltCompilerVersion = '1.2.0'
         hiltWorkVersion = '1.0.0'

common/src/main/java/org/dash/wallet/common/Configuration.java

@@ -95,7 +95,7 @@ public class Configuration {
     public static final String PREFS_KEY_SHOW_NOTIFICATIONS_EXPLAINER = "show_notifications_explainer";
     public static final String PREFS_KEY_SHOW_TAX_CATEGORY_EXPLAINER = "show_tax_catagory_explainer";
     public static final String PREFS_KEY_SHOW_TAX_CATEGORY_INSTALLTIME = "show_tax_catagory_install_time";
-
+    public static final String PREFS_KEY_RESET_BLOCKCHAIN_PENDING = "reset_blockchain_pending";
     private static final long DISABLE_NOTIFICATIONS = -1;
 
     // CrowdNode
@@ -526,6 +526,17 @@ public void setTaxCategoryInstallTime(long time) {
         prefs.edit().putLong(PREFS_KEY_SHOW_TAX_CATEGORY_INSTALLTIME, time).apply();
     }
 
+    // reset blockchain pending
+    public boolean isResetBlockchainPending() {
+        return prefs.getBoolean(PREFS_KEY_RESET_BLOCKCHAIN_PENDING, false);
+    }
+    public void setResetBlockchainPending() {
+        prefs.edit().putBoolean(PREFS_KEY_RESET_BLOCKCHAIN_PENDING, true).apply();
+    }
+    public void clearResetBlockchainPending() {
+        prefs.edit().putBoolean(PREFS_KEY_RESET_BLOCKCHAIN_PENDING, false).apply();
+    }
+
     // CrowdNode
 
     @NonNull

integrations/crowdnode/src/main/java/org/dash/wallet/integrations/crowdnode/api/CrowdNodeApi.kt

@@ -865,11 +865,11 @@ class CrowdNodeApiAggregator @Inject constructor(
      * obtains the current CrowdNode fee on masternode rewards once per 24 hours
      */
     private suspend fun refreshFees() {
-        val lastFeeRequest = config.get(CrowdNodeConfig.LAST_FEE_REQUEST)
+        val lastFeeRequest = 0;//config.get(CrowdNodeConfig.LAST_FEE_REQUEST)
         if (lastFeeRequest == null || (lastFeeRequest + TimeUnit.DAYS.toMillis(1)) < System.currentTimeMillis()) {
             val feeInfo = webApi.getFees(accountAddress)
             log.info("crowdnode feeInfo: {}", feeInfo)
-            val fee = feeInfo.getNormal()?.fee
+            val fee = feeInfo.first().getNormal()?.fee
             fee?.let {
                 config.set(CrowdNodeConfig.FEE_PERCENTAGE, fee)
                 config.set(CrowdNodeConfig.LAST_FEE_REQUEST, System.currentTimeMillis())

integrations/crowdnode/src/main/java/org/dash/wallet/integrations/crowdnode/api/CrowdNodeWebApi.kt

@@ -69,7 +69,7 @@ interface CrowdNodeEndpoint {
     @GET("odata/apifundings/GetFeeJson(address='{address}')")
     suspend fun getFees(
         @Path("address") address: String
-    ): Response<FeeInfo>
+    ): Response<List<FeeInfo>>
 
     @GET("odata/apiaddresses/IsApiAddressInUse(address='{address}')")
     suspend fun isAddressInUse(
@@ -274,14 +274,14 @@ open class CrowdNodeWebApi @Inject constructor(
         }
     }
 
-    open suspend fun getFees(address: Address?): FeeInfo {
+    open suspend fun getFees(address: Address?): List<FeeInfo> {
         return try {
             val response = endpoint.getFees(address?.toBase58() ?: "")
 
             return if (response.isSuccessful) {
                 response.body()!!
             } else {
-                FeeInfo.default
+                listOf(FeeInfo.default)
             }
         } catch (ex: Exception) {
             log.error("Error in getFees: $ex")
@@ -290,7 +290,7 @@ open class CrowdNodeWebApi @Inject constructor(
                 analyticsService.logError(ex)
             }
 
-            FeeInfo.default
+            listOf(FeeInfo.default)
         }
     }
 

integrations/crowdnode/src/main/java/org/dash/wallet/integrations/crowdnode/model/CrowdNodeFeeInfo.kt

@@ -5,12 +5,13 @@ import com.google.gson.annotations.SerializedName
 import kotlinx.parcelize.Parcelize
 import kotlinx.parcelize.RawValue
 
+@Parcelize
 data class FeeLadder(
     val name: String,
     val type: String,
     val amount: Double,
     val fee: Double
-)
+) : Parcelable
 
 /*
   [
@@ -34,16 +35,27 @@ data class FeeLadder(
  */
 @Parcelize
 data class FeeInfo(
-    @SerializedName("FeeLadder") val feeLadder: @RawValue List<FeeLadder>
+    @SerializedName("Key") val key: String,
+    @SerializedName("Value") val value: @RawValue List<FeeLadder>
 ) : Parcelable {
     companion object {
         const val DEFAULT_FEE = 35.0
         const val DEFAULT_AMOUNT = 100.0
         const val KEY_FEELADDER = "FeeLadder"
         const val TYPE_NORMAL = "Normal"
         const val TYPE_TRUSTLESS = "Trustless"
-        val default = FeeInfo(listOf(FeeLadder("", TYPE_NORMAL, DEFAULT_AMOUNT, DEFAULT_FEE)))
+        val default = FeeInfo("FeeLadder", listOf(FeeLadder("", TYPE_NORMAL, DEFAULT_AMOUNT, DEFAULT_FEE)))
     }
 
-    fun getNormal() = feeLadder.find { it.type == FeeInfo.TYPE_NORMAL }
+    fun getNormal() = value.find { it.type == TYPE_NORMAL }
+}
+
+@Parcelize
+data class FeeInfoResponse(
+    @SerializedName("FeeInfo") val feeInfoList: List<FeeInfo>
+) : Parcelable {
+    companion object {
+        val default = FeeInfoResponse(listOf(FeeInfo.default))
+    }
+    fun getNormal() = feeInfoList.first().value.find { it.type == FeeInfo.TYPE_NORMAL }
 }

wallet/src/de/schildbach/wallet/WalletApplication.java

@@ -558,16 +558,34 @@ private void afterLoadWallet() {
             //txes with fees that were too low or dust that were stuck and could not be sent.  In a later version
             //the fees were fixed, then those stuck transactions became inconsistant and the exception is thrown.
             if (x.getMessage().contains("Inconsistent spent tx:")) {
-                File blockChainFile = new File(getDir("blockstore", Context.MODE_PRIVATE), Constants.Files.BLOCKCHAIN_FILENAME);
-                blockChainFile.delete();
+                deleteBlockchainFiles();
             } else throw x;
         }
 
+        // did blockchain rescan fail
+        if (config.isResetBlockchainPending()) {
+            log.info("failed to finish reset earlier, performing now...");
+            deleteBlockchainFiles();
+            Toast.makeText(this, "finishing blockchain rescan", Toast.LENGTH_LONG).show();
+            WalletApplicationExt.INSTANCE.clearDatabases(this, false);
+            config.clearResetBlockchainPending();
+        }
+
         // make sure there is at least one recent backup
         if (!getFileStreamPath(Constants.Files.WALLET_KEY_BACKUP_PROTOBUF).exists())
             backupWallet();
     }
 
+    private void deleteBlockchainFiles() {
+        File blockChainFile = new File(getDir("blockstore", Context.MODE_PRIVATE), Constants.Files.BLOCKCHAIN_FILENAME);
+        blockChainFile.delete();
+        File headerChainFile = new File(getDir("blockstore", Context.MODE_PRIVATE), Constants.Files.HEADERS_FILENAME);
+        headerChainFile.delete();
+        // TODO: should we have a backup blockchain file?
+        // File backupBlockChainFile = new File(getDir("blockstore", Context.MODE_PRIVATE), Constants.Files.BACKUP_BLOCKCHAIN_FILENAME);
+        // backupBlockChainFile.delete();
+    }
+
     @SuppressWarnings("ResultOfMethodCallIgnored")
     private void initLogging() {
         // create log dir

wallet/src/de/schildbach/wallet/service/BlockchainServiceImpl.java

@@ -120,6 +120,9 @@
 import java.util.Set;
 import java.util.concurrent.Executor;
 import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -990,11 +993,13 @@ public void onCreate() {
         }
 
         try {
+            log.info("loading blockchain file");
             blockStore = new SPVBlockStore(Constants.NETWORK_PARAMETERS, blockChainFile);
             blockStore.getChainHead(); // detect corruptions as early as possible
-
+            log.info("loading header file");
             headerStore = new SPVBlockStore(Constants.NETWORK_PARAMETERS, headerChainFile);
             headerStore.getChainHead(); // detect corruptions as early as possible
+            verifyBlockStores();
 
             final long earliestKeyCreationTime = wallet.getEarliestKeyCreationTime();
 
@@ -1139,6 +1144,7 @@ public int onStartCommand(final Intent intent, final int flags, final int startI
                 log.info("will remove blockchain on service shutdown");
 
                 resetBlockchainOnShutdown = true;
+                config.setResetBlockchainPending();
                 stopSelf();
             } else if (BlockchainService.ACTION_WIPE_WALLET.equals(action)) {
                 log.info("will remove blockchain and delete walletFile on service shutdown");
@@ -1290,6 +1296,9 @@ public void onDestroy() {
             }
             //Clear the blockchain identity
             WalletApplicationExt.INSTANCE.clearDatabases(application, false);
+            if (resetBlockchainOnShutdown) {
+                config.clearResetBlockchainPending();
+            }
         }
 
         closeStream(mnlistinfoBootStrapStream);
@@ -1481,4 +1490,103 @@ private void closeStream(InputStream mnlistinfoBootStrapStream) {
             }
         }
     }
+
+    // TODO: should we have a backup blockchain file?
+//    private NewBestBlockListener newBestBlockListener = block -> {
+//        try {
+//            backupBlockStore.put(block);
+//        } catch (BlockStoreException x) {
+//            throw new RuntimeException(x);
+//        }
+//    };
+
+    private boolean verifyBlockStore(BlockStore store) throws BlockStoreException {
+            StoredBlock cursor = store.getChainHead();
+            for (int i = 0; i < 10; ++i) {
+                cursor = cursor.getPrev(store);
+                if (cursor == null || cursor.getHeader().equals(Constants.NETWORK_PARAMETERS.getGenesisBlock())) {
+                    break;
+                }
+            }
+            return true;
+    }
+
+    private boolean verifyBlockStore(BlockStore store, ScheduledExecutorService scheduledExecutorService) {
+        try {
+            ScheduledFuture<Boolean> future = scheduledExecutorService.schedule(() -> verifyBlockStore(store), 100, TimeUnit.MILLISECONDS);
+            return future.get(1, TimeUnit.SECONDS);
+        } catch (Exception e) {
+            log.warn("verification of blockstore failed:", e);
+            return false;
+        }
+    }
+
+    // TODO: should we have a backup blockchain file?
+//    public static void copyFile(File source, File destination) throws IOException {
+//        try (FileChannel sourceChannel = new FileInputStream(source).getChannel();
+//             FileChannel destChannel = new FileOutputStream(destination).getChannel()) {
+//            sourceChannel.transferTo(0, sourceChannel.size(), destChannel);
+//        }
+//    }
+//
+//
+//    private void replaceBlockStore(BlockStore a, File aFile, BlockStore b, File bFile) throws BlockStoreException {
+//        try {
+//            a.close();
+//            b.close();
+//            copyFile(bFile, aFile);
+//        } catch (IOException e) {
+//            throw new RuntimeException(e);
+//        }
+//    }
+
+    private void verifyBlockStores() throws BlockStoreException {
+        ScheduledExecutorService scheduledExecutorService = new ScheduledThreadPoolExecutor(1);
+        log.info("verifying backupBlockStore");
+//        boolean verifiedBackupBlockStore = false;
+        boolean verifiedHeaderStore = false;
+        boolean verifiedBlockStore = false;
+//        if (!(verifiedBackupBlockStore = verifyBlockStore(backupBlockStore, scheduledExecutorService))) {
+//            log.info("backupBlockStore verification failed");
+//        }
+
+        log.info("verifying headerStore");
+        if (!(verifiedHeaderStore = verifyBlockStore(headerStore, scheduledExecutorService))) {
+            log.info("headerStore verification failed");
+        }
+
+        log.info("verifying blockStore");
+        if (!(verifiedBlockStore = verifyBlockStore(blockStore, scheduledExecutorService))) {
+            log.info("blockStore verification failed");
+        }
+        // TODO: should we have a backup blockchain file?
+//        if (!verifiedBlockStore) {
+//            if (verifiedBackupBlockStore &&
+//                    !backupBlockStore.getChainHead().getHeader().getHash().equals(Constants.NETWORK_PARAMETERS.getGenesisBlock().getHash())) {
+//                log.info("replacing blockStore with backup");
+//                replaceBlockStore(blockStore, blockChainFile, backupBlockStore, backupBlockChainFile);
+//                log.info("reloading blockStore");
+//                blockStore = new SPVBlockStore(Constants.NETWORK_PARAMETERS, blockChainFile);
+//                blockStore.getChainHead(); // detect corruptions as early as possible
+//                log.info("reloading backup blockchain file");
+//                backupBlockStore = new SPVBlockStore(Constants.NETWORK_PARAMETERS, blockChainFile);
+//                backupBlockStore.getChainHead(); // detect corruptions as early as possible
+//                verifyBlockStores();
+//            } /*else if (verifiedHeaderStore) {
+//                log.info("replacing blockStore with header");
+//                replaceBlockStore(blockStore, blockChainFile, headerStore, headerChainFile);
+//                log.info("reloading blockStore");
+//                blockStore = new SPVBlockStore(Constants.NETWORK_PARAMETERS, blockChainFile);
+//                blockStore.getChainHead(); // detect corruptions as early as possible
+//                log.info("reloading header file");
+//                headerStore = new SPVBlockStore(Constants.NETWORK_PARAMETERS, headerChainFile);
+//                headerStore.getChainHead(); // detect corruptions as early as possible
+//                verifyBlockStores();
+//            } else*/ {
+//                // get blocks from platform here...
+//                throw new BlockStoreException("can't verify and recover");
+//            }
+//        }
+        log.info("blockstore files verified: {}, {}", verifiedBlockStore, verifiedHeaderStore);
+    }
 }

wallet/src/de/schildbach/wallet/ui/more/SettingsViewModel.kt

@@ -28,7 +28,6 @@ import de.schildbach.wallet.service.MixingStatus
 import kotlinx.coroutines.flow.Flow
 import kotlinx.coroutines.flow.launchIn
 import kotlinx.coroutines.flow.onEach
-import kotlinx.coroutines.launch
 import org.bitcoinj.wallet.Wallet
 import org.bitcoinj.wallet.WalletEx
 import org.dash.wallet.common.WalletDataProvider