Unify JSON serialization format of transactions

src/ripple/app/ledger/LedgerMaster.h

@@ -215,6 +215,8 @@ class LedgerMaster : public AbstractFetchPackContainer
     void
     clearLedger(std::uint32_t seq);
     bool
+    isValidated(ReadView const& ledger);
+    bool
     getValidatedRange(std::uint32_t& minVal, std::uint32_t& maxVal);
     bool
     getFullValidatedRange(std::uint32_t& minVal, std::uint32_t& maxVal);

src/ripple/app/ledger/impl/LedgerMaster.cpp

@@ -603,6 +603,55 @@ LedgerMaster::clearLedger(std::uint32_t seq)
     mCompleteLedgers.erase(seq);
 }
 
+bool
+LedgerMaster::isValidated(ReadView const& ledger)
+{
+    if (app_.config().reporting())
+        return haveLedger(ledger.seq());  // TODO Is this correct ?
+
+    if (ledger.open())
+        return false;
+
+    if (ledger.info().validated)
+        return true;
+
+    auto const seq = ledger.info().seq;
+    try
+    {
+        // Use the skip list in the last validated ledger to see if ledger
+        // comes before the last validated ledger (and thus has been
+        // validated).
+        auto hash = walkHashBySeq(seq, InboundLedger::Reason::GENERIC);
+
+        if (!hash || ledger.info().hash != *hash)
+        {
+            // This ledger's hash is not the hash of the validated ledger
+            if (hash)
+            {
+                assert(hash->isNonZero());
+                uint256 valHash =
+                    app_.getRelationalDatabase().getHashByIndex(seq);
+                if (valHash == ledger.info().hash)
+                {
+                    // SQL database doesn't match ledger chain
+                    clearLedger(seq);
+                }
+            }
+            return false;
+        }
+    }
+    catch (SHAMapMissingNode const& mn)
+    {
+        auto stream = app_.journal("IsValidated").warn();  // TODO Better name ?
+        JLOG(stream) << "Ledger #" << seq << ": " << mn.what();
+        return false;
+    }
+
+    // Mark ledger as validated to save time if we see it again.
+    ledger.info().validated = true;
+    return true;
+}
+
 // returns Ledgers we have all the nodes for
 bool
 LedgerMaster::getFullValidatedRange(

src/ripple/app/ledger/impl/LedgerToJson.cpp

@@ -17,12 +17,14 @@
 */
 //==============================================================================
 
+#include <ripple/app/ledger/LedgerMaster.h>
 #include <ripple/app/ledger/LedgerToJson.h>
 #include <ripple/app/main/Application.h>
 #include <ripple/app/misc/DeliverMax.h>
 #include <ripple/app/misc/TxQ.h>
 #include <ripple/basics/base_uint.h>
 #include <ripple/core/Pg.h>
+#include <ripple/protocol/jss.h>
 #include <ripple/rpc/Context.h>
 #include <ripple/rpc/DeliveredAmount.h>
 
@@ -83,6 +85,7 @@ fillJson(Object& json, bool closed, LedgerInfo const& info, bool bFull)
         json[jss::close_time_human] = to_string(info.closeTime);
         if (!getCloseAgree(info))
             json[jss::close_time_estimated] = true;
+        json[jss::close_time_iso] = to_string_iso(info.closeTime);
     }
 }
 
@@ -118,24 +121,69 @@ fillJsonTx(
     if (bBinary)
     {
         txJson[jss::tx_blob] = serializeHex(*txn);
+        if (fill.context->apiVersion > 1)
+            txJson[jss::hash] = to_string(txn->getTransactionID());
+
+        auto const json_meta =
+            (fill.context->apiVersion > 1 ? jss::meta_blob : jss::meta);
         if (stMeta)
-            txJson[jss::meta] = serializeHex(*stMeta);
+            txJson[json_meta] = serializeHex(*stMeta);
     }
     else
     {
-        copyFrom(txJson, txn->getJson(JsonOptions::none));
-        RPC::insertDeliverMax(txJson, txnType, fill.context->apiVersion);
-        if (stMeta)
+        if (fill.context->apiVersion > 1)
         {
-            txJson[jss::metaData] = stMeta->getJson(JsonOptions::none);
-
-            // If applicable, insert delivered amount
-            if (txnType == ttPAYMENT || txnType == ttCHECK_CASH)
-                RPC::insertDeliveredAmount(
-                    txJson[jss::metaData],
-                    fill.ledger,
-                    txn,
-                    {txn->getTransactionID(), fill.ledger.seq(), *stMeta});
+            std::string hash;
+            copyFrom(
+                txJson[jss::tx_json],
+                txn->getJson(
+                    JsonOptions::none, false, {std::optional(std::ref(hash))}));
+            txJson[jss::hash] = hash;
+            RPC::insertDeliverMax(
+                txJson[jss::tx_json], txnType, fill.context->apiVersion);
+
+            if (stMeta)
+            {
+                txJson[jss::meta] = stMeta->getJson(JsonOptions::none);
+
+                // If applicable, insert delivered amount
+                if (txnType == ttPAYMENT || txnType == ttCHECK_CASH)
+                    RPC::insertDeliveredAmount(
+                        txJson[jss::meta],
+                        fill.ledger,
+                        txn,
+                        {txn->getTransactionID(), fill.ledger.seq(), *stMeta});
+            }
+
+            const bool validated =
+                fill.context->ledgerMaster.isValidated(fill.ledger);
+            txJson[jss::validated] = validated;
+            if (validated)
+            {
+                txJson[jss::ledger_index] = to_string(fill.ledger.seq());
+                txJson[jss::ledger_hash] = to_string(fill.ledger.info().hash);
+                if (auto close_time =
+                        fill.context->ledgerMaster.getCloseTimeBySeq(
+                            fill.ledger.seq()))
+                    txJson[jss::close_time_iso] = to_string_iso(*close_time);
+            }
+        }
+        else
+        {
+            copyFrom(txJson, txn->getJson(JsonOptions::none));
+            RPC::insertDeliverMax(txJson, txnType, fill.context->apiVersion);
+            if (stMeta)
+            {
+                txJson[jss::metaData] = stMeta->getJson(JsonOptions::none);
+
+                // If applicable, insert delivered amount
+                if (txnType == ttPAYMENT || txnType == ttCHECK_CASH)
+                    RPC::insertDeliveredAmount(
+                        txJson[jss::metaData],
+                        fill.ledger,
+                        txn,
+                        {txn->getTransactionID(), fill.ledger.seq(), *stMeta});
+            }
         }
     }
 
@@ -254,7 +302,11 @@ fillJsonQueue(Object& json, LedgerFill const& fill)
         if (tx.lastResult)
             txJson["last_result"] = transToken(*tx.lastResult);
 
-        txJson[jss::tx] = fillJsonTx(fill, bBinary, bExpanded, tx.txn, nullptr);
+        auto&& temp = fillJsonTx(fill, bBinary, bExpanded, tx.txn, nullptr);
+        if (fill.context->apiVersion > 1)
+            copyFrom(txJson, temp);
+        else
+            copyFrom(txJson[jss::tx], temp);
     }
 }
 

src/ripple/app/misc/NetworkOPs.cpp

@@ -63,6 +63,7 @@
 #include <ripple/protocol/BuildInfo.h>
 #include <ripple/protocol/Feature.h>
 #include <ripple/protocol/STParsedJSON.h>
+#include <ripple/protocol/jss.h>
 #include <ripple/resource/Fees.h>
 #include <ripple/resource/ResourceManager.h>
 #include <ripple/rpc/BookChanges.h>
@@ -74,6 +75,7 @@
 
 #include <algorithm>
 #include <mutex>
+#include <optional>
 #include <string>
 #include <tuple>
 #include <unordered_map>
@@ -3101,7 +3103,12 @@ NetworkOPsImp::transJson(
     transResultInfo(result, sToken, sHuman);
 
     jvObj[jss::type] = "transaction";
-    jvObj[jss::transaction] = transaction->getJson(JsonOptions::none);
+    // NOTE jvObj which is not a finished object for either API version. After
+    // it's populated, we need to finish it for a specific API version. This is
+    // done in a loop, near the end of this function.
+    std::string hash = {};
+    jvObj[jss::transaction] =
+        transaction->getJson(JsonOptions::none, false, {std::ref(hash)});
 
     if (meta)
     {
@@ -3117,6 +3124,8 @@ NetworkOPsImp::transJson(
         jvObj[jss::transaction][jss::date] =
             ledger->info().closeTime.time_since_epoch().count();
         jvObj[jss::validated] = true;
+        if (auto close_time = m_ledgerMaster.getCloseTimeBySeq(ledger->seq()))
+            jvObj[jss::close_time_iso] = to_string_iso(*close_time);
 
         // WRITEME: Put the account next seq here
     }
@@ -3165,11 +3174,20 @@ NetworkOPsImp::transJson(
         assert(index < MultiApiJson::size);
         if (index != lastIndex)
         {
+            Json::Value& jvTx = multiObj.val[index];
             RPC::insertDeliverMax(
-                multiObj.val[index][jss::transaction],
-                transaction->getTxnType(),
-                apiVersion);
+                jvTx[jss::transaction], transaction->getTxnType(), apiVersion);
             lastIndex = index;
+
+            if (apiVersion > 1)
+            {
+                jvTx[jss::tx_json] = jvTx.removeMember(jss::transaction);
+                jvTx[jss::hash] = hash;
+            }
+            else
+            {
+                jvTx[jss::transaction][jss::hash] = hash;
+            }
         }
     }
 

src/ripple/app/misc/Transaction.h

@@ -27,7 +27,7 @@
 #include <ripple/protocol/STTx.h>
 #include <ripple/protocol/TER.h>
 #include <ripple/protocol/TxMeta.h>
-#include <boost/optional.hpp>
+
 #include <optional>
 #include <variant>
 
@@ -305,8 +305,13 @@ class Transaction : public std::enable_shared_from_this<Transaction>,
             validatedLedger, fee, accountSeq, availableSeq);
     }
 
+    /// Same as similar overload for STTx::getJson
     Json::Value
-    getJson(JsonOptions options, bool binary = false) const;
+    getJson(
+        JsonOptions options,
+        bool binary = false,
+        bool showInLedger = false,
+        std::optional<std::reference_wrapper<std::string>> hash = {}) const;
 
     // Information used to locate a transaction.
     // Contains a nodestore hash and ledger sequence pair if the transaction was

src/ripple/app/misc/impl/Transaction.cpp

@@ -165,13 +165,19 @@ Transaction::load(
 
 // options 1 to include the date of the transaction
 Json::Value
-Transaction::getJson(JsonOptions options, bool binary) const
+Transaction::getJson(
+    JsonOptions options,
+    bool binary,
+    bool showInLedger,
+    std::optional<std::reference_wrapper<std::string>> hash) const
 {
-    Json::Value ret(mTransaction->getJson(JsonOptions::none, binary));
+    Json::Value ret(mTransaction->getJson(JsonOptions::none, binary, hash));
 
     if (mInLedger)
     {
-        ret[jss::inLedger] = mInLedger;  // Deprecated.
+        if (showInLedger)
+            ret[jss::inLedger] = mInLedger;  // Deprecated.
+
         ret[jss::ledger_index] = mInLedger;
 
         if (options == JsonOptions::include_date)

src/ripple/basics/chrono.h

@@ -25,6 +25,7 @@
 #include <ripple/beast/clock/abstract_clock.h>
 #include <ripple/beast/clock/basic_seconds_clock.h>
 #include <ripple/beast/clock/manual_clock.h>
+
 #include <chrono>
 #include <cstdint>
 #include <string>
@@ -43,8 +44,19 @@ using weeks = std::chrono::
 /** Clock for measuring the network time.
 
     The epoch is January 1, 2000
-    epoch_offset = days(10957);  // 2000-01-01
+
+    epoch_offset
+    = date(2000-01-01) - date(1970-0-01)
+    = days(10957)
+    = seconds(946684800)
 */
+
+constexpr static std::chrono::seconds epoch_offset =
+    date::sys_days{date::year{2000} / 1 / 1} -
+    date::sys_days{date::year{1970} / 1 / 1};
+
+static_assert(epoch_offset.count() == 946684800);
+
 class NetClock
 {
 public:
@@ -71,7 +83,24 @@ to_string(NetClock::time_point tp)
     // 2000-01-01 00:00:00 UTC is 946684800s from 1970-01-01 00:00:00 UTC
     using namespace std::chrono;
     return to_string(
-        system_clock::time_point{tp.time_since_epoch() + 946684800s});
+        system_clock::time_point{tp.time_since_epoch() + epoch_offset});
+}
+
+template <class Duration>
+std::string
+to_string_iso(date::sys_time<Duration> tp)
+{
+    using namespace std::chrono;
+    return date::format("%FT%H:%M:%OSZ", tp);
+}
+
+inline std::string
+to_string_iso(NetClock::time_point tp)
+{
+    // 2000-01-01 00:00:00 UTC is 946684800s from 1970-01-01 00:00:00 UTC
+    using namespace std::chrono;
+    return to_string_iso(
+        system_clock::time_point{tp.time_since_epoch() + epoch_offset});
 }
 
 /** A clock for measuring elapsed time.

src/ripple/core/TimeKeeper.h

@@ -22,6 +22,7 @@
 
 #include <ripple/basics/chrono.h>
 #include <ripple/beast/clock/abstract_clock.h>
+
 #include <atomic>
 
 namespace ripple {
@@ -37,7 +38,7 @@ class TimeKeeper : public beast::abstract_clock<NetClock>
     adjust(std::chrono::system_clock::time_point when)
     {
         return time_point(std::chrono::duration_cast<duration>(
-            when.time_since_epoch() - days(10957)));
+            when.time_since_epoch() - epoch_offset));
     }
 
 public:

src/ripple/protocol/STTx.h

@@ -108,8 +108,16 @@ class STTx final : public STObject, public CountedObject<STTx>
 
     Json::Value
     getJson(JsonOptions options) const override;
+
+    /// If `hash` is set, will store hash inside the provided string. Otherwise
+    /// hash will be stored as nested jss::hash element inside the returned JSON
+    /// Additionally, if `hash` is set and `binary` is true, will not create
+    /// nested jss::tx for binary hex; instead will return it as JSON string
     Json::Value
-    getJson(JsonOptions options, bool binary) const;
+    getJson(
+        JsonOptions options,
+        bool binary,
+        std::optional<std::reference_wrapper<std::string>> hash = {}) const;
 
     void
     sign(PublicKey const& publicKey, SecretKey const& secretKey);