Unify JSON serialization format of transactions

API-CHANGELOG.md

@@ -141,6 +141,31 @@ In API version 2, the following methods are no longer available:
 - `tx_history` - Instead, use other methods such as `account_tx` or `ledger` with the `transactions` field set to `true`.
 - `ledger_header` - Instead, use the `ledger` method.
 
+#### Modifications to JSON transaction element in V2
+
+In API version 2, JSON elements for transaction output have been changed and made consistent for all methods which output transactions:
+
+- JSON transaction element is named `tx_json`
+- Binary transaction element is named `tx_blob`
+- JSON transaction metadata element is named `meta`
+- Binary transaction metadata element is named `meta_blob`
+
+Additionally, these elements are now consistently available next to `tx_json` (i.e. sibling elements), where possible:
+
+- `hash` - Transaction ID. This data was stored inside transaction output in API version 1, but in API version 2 is a sibling element.
+- `ledger_index` - Ledger index (only set on validated ledgers)
+- `ledger_hash` - Ledger hash (only set on closed or validated ledgers)
+- `close_time_iso` - Ledger close time expressed in ISO 8601 time format (only set on validated ledgers)
+- `validated` - Bool element set to `true` if the transaction is in a validated ledger, otherwise `false`
+
+This change affects the following methods:
+
+- `tx` - Transaction data moved into element `tx_json` (was inline inside `result`) or, if binary output was requested, moved from `tx` to `tx_blob`. Renamed binary transaction metadata element (if it was requsted) from `meta` to `meta_blob`. Changed location of `hash` and added new elements
+- `account_tx` - Renamed transaction element from `tx` to `tx_json`. Renamed binary transaction metadata element (if it was requsted) from `meta` to `meta_blob`. Changed location of `hash` and added new elements
+- `transaction_entry` - Renamed transaction metadata element from `metadata` to `meta`. Changed location of `hash` and added new elements
+- `subscribe` - Renamed transaction element from `transaction` to `tx_json`. Changed location of `hash` and added new elements
+- `sign`, `sign_for`, `submit` and `submit_multisigned` - Changed location of `hash` element.
+
 #### Modifications to account_info response in V2
 
 - `signer_lists` is returned in the root of the response. Previously, in API version 1, it was nested under `account_data`. (https://github.com/XRPLF/rippled/pull/3770)

src/ripple/app/ledger/LedgerMaster.h

@@ -215,6 +215,8 @@ class LedgerMaster : public AbstractFetchPackContainer
     void
     clearLedger(std::uint32_t seq);
     bool
+    isValidated(ReadView const& ledger);
+    bool
     getValidatedRange(std::uint32_t& minVal, std::uint32_t& maxVal);
     bool
     getFullValidatedRange(std::uint32_t& minVal, std::uint32_t& maxVal);

src/ripple/app/ledger/LedgerToJson.h

@@ -21,8 +21,10 @@
 #define RIPPLE_APP_LEDGER_LEDGERTOJSON_H_INCLUDED
 
 #include <ripple/app/ledger/Ledger.h>
+#include <ripple/app/ledger/LedgerMaster.h>
 #include <ripple/app/misc/TxQ.h>
 #include <ripple/basics/StringUtilities.h>
+#include <ripple/basics/chrono.h>
 #include <ripple/json/Object.h>
 #include <ripple/protocol/STTx.h>
 #include <ripple/protocol/jss.h>
@@ -41,6 +43,8 @@ struct LedgerFill
         LedgerEntryType t = ltANY)
         : ledger(l), options(o), txQueue(std::move(q)), type(t), context(ctx)
     {
+        if (context)
+            closeTime = context->ledgerMaster.getCloseTimeBySeq(ledger.seq());
     }
 
     enum Options {
@@ -58,6 +62,7 @@ struct LedgerFill
     std::vector<TxQ::TxDetails> txQueue;
     LedgerEntryType type;
     RPC::Context* context;
+    std::optional<NetClock::time_point> closeTime;
 };
 
 /** Given a Ledger and options, fill a Json::Object or Json::Value with a

src/ripple/app/ledger/impl/LedgerMaster.cpp

@@ -603,6 +603,55 @@ LedgerMaster::clearLedger(std::uint32_t seq)
     mCompleteLedgers.erase(seq);
 }
 
+bool
+LedgerMaster::isValidated(ReadView const& ledger)
+{
+    if (app_.config().reporting())
+        return true;  // Reporting mode only supports validated ledger
+
+    if (ledger.open())
+        return false;
+
+    if (ledger.info().validated)
+        return true;
+
+    auto const seq = ledger.info().seq;
+    try
+    {
+        // Use the skip list in the last validated ledger to see if ledger
+        // comes before the last validated ledger (and thus has been
+        // validated).
+        auto hash = walkHashBySeq(seq, InboundLedger::Reason::GENERIC);
+
+        if (!hash || ledger.info().hash != *hash)
+        {
+            // This ledger's hash is not the hash of the validated ledger
+            if (hash)
+            {
+                assert(hash->isNonZero());
+                uint256 valHash =
+                    app_.getRelationalDatabase().getHashByIndex(seq);
+                if (valHash == ledger.info().hash)
+                {
+                    // SQL database doesn't match ledger chain
+                    clearLedger(seq);
+                }
+            }
+            return false;
+        }
+    }
+    catch (SHAMapMissingNode const& mn)
+    {
+        auto stream = app_.journal("IsValidated").warn();  // TODO Better name ?
+        JLOG(stream) << "Ledger #" << seq << ": " << mn.what();
+        return false;
+    }
+
+    // Mark ledger as validated to save time if we see it again.
+    ledger.info().validated = true;
+    return true;
+}
+
 // returns Ledgers we have all the nodes for
 bool
 LedgerMaster::getFullValidatedRange(

src/ripple/app/ledger/impl/LedgerToJson.cpp

@@ -17,12 +17,14 @@
 */
 //==============================================================================
 
+#include <ripple/app/ledger/LedgerMaster.h>
 #include <ripple/app/ledger/LedgerToJson.h>
 #include <ripple/app/main/Application.h>
 #include <ripple/app/misc/DeliverMax.h>
 #include <ripple/app/misc/TxQ.h>
 #include <ripple/basics/base_uint.h>
 #include <ripple/core/Pg.h>
+#include <ripple/protocol/jss.h>
 #include <ripple/rpc/Context.h>
 #include <ripple/rpc/DeliveredAmount.h>
 
@@ -83,6 +85,7 @@ fillJson(Object& json, bool closed, LedgerInfo const& info, bool bFull)
         json[jss::close_time_human] = to_string(info.closeTime);
         if (!getCloseAgree(info))
             json[jss::close_time_estimated] = true;
+        json[jss::close_time_iso] = to_string_iso(info.closeTime);
     }
 }
 
@@ -118,24 +121,70 @@ fillJsonTx(
     if (bBinary)
     {
         txJson[jss::tx_blob] = serializeHex(*txn);
+        if (fill.context->apiVersion > 1)
+            txJson[jss::hash] = to_string(txn->getTransactionID());
+
+        auto const json_meta =
+            (fill.context->apiVersion > 1 ? jss::meta_blob : jss::meta);
         if (stMeta)
-            txJson[jss::meta] = serializeHex(*stMeta);
+            txJson[json_meta] = serializeHex(*stMeta);
     }
     else
     {
-        copyFrom(txJson, txn->getJson(JsonOptions::none));
-        RPC::insertDeliverMax(txJson, txnType, fill.context->apiVersion);
-        if (stMeta)
+        if (fill.context->apiVersion > 1)
+        {
+            std::string hash;
+            copyFrom(
+                txJson[jss::tx_json],
+                txn->getJson(
+                    JsonOptions::none, false, {std::optional(std::ref(hash))}));
+            txJson[jss::hash] = hash;
+            RPC::insertDeliverMax(
+                txJson[jss::tx_json], txnType, fill.context->apiVersion);
+
+            if (stMeta)
+            {
+                txJson[jss::meta] = stMeta->getJson(JsonOptions::none);
+
+                // If applicable, insert delivered amount
+                if (txnType == ttPAYMENT || txnType == ttCHECK_CASH)
+                    RPC::insertDeliveredAmount(
+                        txJson[jss::meta],
+                        fill.ledger,
+                        txn,
+                        {txn->getTransactionID(), fill.ledger.seq(), *stMeta});
+            }
+
+            if (!fill.ledger.open())
+                txJson[jss::ledger_hash] = to_string(fill.ledger.info().hash);
+
+            const bool validated =
+                fill.context->ledgerMaster.isValidated(fill.ledger);
+            txJson[jss::validated] = validated;
+            if (validated)
+            {
+                txJson[jss::ledger_index] = to_string(fill.ledger.seq());
+                if (fill.closeTime)
+                    txJson[jss::close_time_iso] =
+                        to_string_iso(*fill.closeTime);
+            }
+        }
+        else
         {
-            txJson[jss::metaData] = stMeta->getJson(JsonOptions::none);
-
-            // If applicable, insert delivered amount
-            if (txnType == ttPAYMENT || txnType == ttCHECK_CASH)
-                RPC::insertDeliveredAmount(
-                    txJson[jss::metaData],
-                    fill.ledger,
-                    txn,
-                    {txn->getTransactionID(), fill.ledger.seq(), *stMeta});
+            copyFrom(txJson, txn->getJson(JsonOptions::none));
+            RPC::insertDeliverMax(txJson, txnType, fill.context->apiVersion);
+            if (stMeta)
+            {
+                txJson[jss::metaData] = stMeta->getJson(JsonOptions::none);
+
+                // If applicable, insert delivered amount
+                if (txnType == ttPAYMENT || txnType == ttCHECK_CASH)
+                    RPC::insertDeliveredAmount(
+                        txJson[jss::metaData],
+                        fill.ledger,
+                        txn,
+                        {txn->getTransactionID(), fill.ledger.seq(), *stMeta});
+            }
         }
     }
 
@@ -254,7 +303,11 @@ fillJsonQueue(Object& json, LedgerFill const& fill)
         if (tx.lastResult)
             txJson["last_result"] = transToken(*tx.lastResult);
 
-        txJson[jss::tx] = fillJsonTx(fill, bBinary, bExpanded, tx.txn, nullptr);
+        auto&& temp = fillJsonTx(fill, bBinary, bExpanded, tx.txn, nullptr);
+        if (fill.context->apiVersion > 1)
+            copyFrom(txJson, temp);
+        else
+            copyFrom(txJson[jss::tx], temp);
     }
 }
 

src/ripple/app/misc/NetworkOPs.cpp

@@ -63,6 +63,7 @@
 #include <ripple/protocol/BuildInfo.h>
 #include <ripple/protocol/Feature.h>
 #include <ripple/protocol/STParsedJSON.h>
+#include <ripple/protocol/jss.h>
 #include <ripple/resource/Fees.h>
 #include <ripple/resource/ResourceManager.h>
 #include <ripple/rpc/BookChanges.h>
@@ -74,6 +75,7 @@
 
 #include <algorithm>
 #include <mutex>
+#include <optional>
 #include <string>
 #include <tuple>
 #include <unordered_map>
@@ -3101,7 +3103,12 @@ NetworkOPsImp::transJson(
     transResultInfo(result, sToken, sHuman);
 
     jvObj[jss::type] = "transaction";
-    jvObj[jss::transaction] = transaction->getJson(JsonOptions::none);
+    // NOTE jvObj which is not a finished object for either API version. After
+    // it's populated, we need to finish it for a specific API version. This is
+    // done in a loop, near the end of this function.
+    std::string hash = {};
+    jvObj[jss::transaction] =
+        transaction->getJson(JsonOptions::none, false, {std::ref(hash)});
 
     if (meta)
     {
@@ -3110,13 +3117,16 @@ NetworkOPsImp::transJson(
             jvObj[jss::meta], *ledger, transaction, meta->get());
     }
 
+    if (!ledger->open())
+        jvObj[jss::ledger_hash] = to_string(ledger->info().hash);
+
     if (validated)
     {
         jvObj[jss::ledger_index] = ledger->info().seq;
-        jvObj[jss::ledger_hash] = to_string(ledger->info().hash);
         jvObj[jss::transaction][jss::date] =
             ledger->info().closeTime.time_since_epoch().count();
         jvObj[jss::validated] = true;
+        jvObj[jss::close_time_iso] = to_string_iso(ledger->info().closeTime);
 
         // WRITEME: Put the account next seq here
     }
@@ -3165,11 +3175,21 @@ NetworkOPsImp::transJson(
         assert(index < MultiApiJson::size);
         if (index != lastIndex)
         {
-            RPC::insertDeliverMax(
-                multiObj.val[index][jss::transaction],
-                transaction->getTxnType(),
-                apiVersion);
             lastIndex = index;
+
+            Json::Value& jvTx = multiObj.val[index];
+            RPC::insertDeliverMax(
+                jvTx[jss::transaction], transaction->getTxnType(), apiVersion);
+
+            if (apiVersion > 1)
+            {
+                jvTx[jss::tx_json] = jvTx.removeMember(jss::transaction);
+                jvTx[jss::hash] = hash;
+            }
+            else
+            {
+                jvTx[jss::transaction][jss::hash] = hash;
+            }
         }
     }
 

src/ripple/app/misc/Transaction.h

@@ -27,7 +27,7 @@
 #include <ripple/protocol/STTx.h>
 #include <ripple/protocol/TER.h>
 #include <ripple/protocol/TxMeta.h>
-#include <boost/optional.hpp>
+
 #include <optional>
 #include <variant>
 
@@ -99,13 +99,13 @@ class Transaction : public std::enable_shared_from_this<Transaction>,
     LedgerIndex
     getLedger() const
     {
-        return mInLedger;
+        return mLedgerIndex;
     }
 
     bool
     isValidated() const
     {
-        return mInLedger != 0;
+        return mLedgerIndex != 0;
     }
 
     TransStatus
@@ -138,7 +138,7 @@ class Transaction : public std::enable_shared_from_this<Transaction>,
     void
     setLedger(LedgerIndex ledger)
     {
-        mInLedger = ledger;
+        mLedgerIndex = ledger;
     }
 
     /**
@@ -305,8 +305,13 @@ class Transaction : public std::enable_shared_from_this<Transaction>,
             validatedLedger, fee, accountSeq, availableSeq);
     }
 
+    /// Same as similar overload for STTx::getJson
     Json::Value
-    getJson(JsonOptions options, bool binary = false) const;
+    getJson(
+        JsonOptions options,
+        bool binary = false,
+        bool showInLedger = false,
+        std::optional<std::reference_wrapper<std::string>> hash = {}) const;
 
     // Information used to locate a transaction.
     // Contains a nodestore hash and ledger sequence pair if the transaction was
@@ -386,7 +391,7 @@ class Transaction : public std::enable_shared_from_this<Transaction>,
 
     uint256 mTransactionID;
 
-    LedgerIndex mInLedger = 0;
+    LedgerIndex mLedgerIndex = 0;
     TransStatus mStatus = INVALID;
     TER mResult = temUNCERTAIN;
     bool mApplying = false;