Skip to content

fix: disable nanosecond timestamp when only EnableTimestampNanosecond is enabled #1528

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Jun 7, 2024
Merged

fix: disable nanosecond timestamp when only EnableTimestampNanosecond is enabled #1528

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
cfa2e59
fix: disable nanosecond timestamp when only EnableTimestampNanosecond…
henryzhx8 Jun 7, 2024
aeb2baf
polish
henryzhx8 Jun 7, 2024
f6c5b2e
polish
henryzhx8 Jun 7, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 4 additions & 3 deletions core/common/TimeUtil.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -302,10 +302,11 @@ void SetLogTime(sls_logs::Log* log, time_t second) {
log->set_time(second);
}

void SetLogTimeWithNano(sls_logs::Log* log, time_t second, long nanosecond) {
// Usage: set nanosecond first, and then discard at LogProcess@ProcessBufferLegacy
void SetLogTimeWithNano(sls_logs::Log* log, time_t second, std::optional<uint32_t> nanosecond) {
log->set_time(second);
log->set_time_ns(nanosecond);
if (nanosecond) {
log->set_time_ns(nanosecond.value());
}
}

LogtailTime GetCurrentLogtailTime() {
Expand Down
3 changes: 2 additions & 1 deletion core/common/TimeUtil.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@

#pragma once
#include <ctime>
#include <optional>
#include <string>
#include <thread>

Expand Down Expand Up @@ -83,7 +84,7 @@ uint64_t GetPreciseTimestampFromLogtailTime(LogtailTime logTime, const PreciseTi

void SetLogTime(sls_logs::Log* log, time_t second);

void SetLogTimeWithNano(sls_logs::Log* log, time_t second, long nanosecond);
void SetLogTimeWithNano(sls_logs::Log* log, time_t second, std::optional<uint32_t> nanosecond);

LogtailTime GetCurrentLogtailTime();

Expand Down
4 changes: 3 additions & 1 deletion core/models/LogEvent.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,9 @@ Json::Value LogEvent::ToJson(bool enableEventMeta) const {
Json::Value root;
root["type"] = static_cast<int>(GetType());
root["timestamp"] = GetTimestamp();
root["timestampNanosecond"] = GetTimestampNanosecond();
if (GetTimestampNanosecond()) {
root["timestampNanosecond"] = static_cast<int32_t>(GetTimestampNanosecond().value());
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

这里的类型是不是也可以统一成uint32_t?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

json不认识uint，只认识int，不这样搞的话，在单测那里，会出现uint != int的情况，只能强制转成int

}
if (enableEventMeta) {
root["fileOffset"] = GetPosition().first;
root["rawSize"] = GetPosition().second;
Expand Down
4 changes: 3 additions & 1 deletion core/models/MetricEvent.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,9 @@ Json::Value MetricEvent::ToJson(bool enableEventMeta) const {
Json::Value root;
root["type"] = static_cast<int>(GetType());
root["timestamp"] = GetTimestamp();
root["timestampNanosecond"] = GetTimestampNanosecond();
if (GetTimestampNanosecond()) {
root["timestampNanosecond"] = static_cast<int32_t>(GetTimestampNanosecond().value());
}
root["name"] = mName.to_string();
root["value"] = MetricValueToJson(mValue);
if (!mTags.mInner.empty()) {
Expand Down
11 changes: 8 additions & 3 deletions core/models/PipelineEvent.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
#include <ctime>
#include <map>
#include <memory>
#include <optional>
#include <string>

#include "common/memory/SourceBuffer.h"
Expand All @@ -41,9 +42,13 @@ class PipelineEvent {

Type GetType() const { return mType; }
time_t GetTimestamp() const { return mTimestamp; }
long GetTimestampNanosecond() const { return mTimestampNanosecond; }
std::optional<uint32_t> GetTimestampNanosecond() const { return mTimestampNanosecond; }
void SetTimestamp(time_t t) { mTimestamp = t; }
void SetTimestamp(time_t t, long ns) {
void SetTimestamp(time_t t, uint32_t ns) {
mTimestamp = t;
mTimestampNanosecond = ns; // Only nanosecond part
}
void SetTimestamp(time_t t, std::optional<uint32_t> ns) {
mTimestamp = t;
mTimestampNanosecond = ns; // Only nanosecond part
}
Expand All @@ -64,7 +69,7 @@ class PipelineEvent {

Type mType = Type::NONE;
time_t mTimestamp = 0;
long mTimestampNanosecond = 0;
std::optional<uint32_t> mTimestampNanosecond;
PipelineEventGroup* mPipelineEventGroupPtr = nullptr;
};

Expand Down
10 changes: 6 additions & 4 deletions core/models/SpanEvent.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -299,17 +299,19 @@ size_t SpanEvent::DataSize() const {
linksSize += item.DataSize();
}
// TODO: for enum, it seems more reasonable to use actual string size instead of size of enum
return PipelineEvent::DataSize() + mTraceId.size() + mSpanId.size() + mTraceState.size() + mParentSpanId.size() + mName.size()
+ sizeof(decltype(mKind)) + sizeof(decltype(mStartTimeNs)) + sizeof(decltype(mEndTimeNs)) + mTags.DataSize()
+ eventsSize + linksSize + sizeof(decltype(mStatus)) + mScopeTags.DataSize();
return PipelineEvent::DataSize() + mTraceId.size() + mSpanId.size() + mTraceState.size() + mParentSpanId.size()
+ mName.size() + sizeof(decltype(mKind)) + sizeof(decltype(mStartTimeNs)) + sizeof(decltype(mEndTimeNs))
+ mTags.DataSize() + eventsSize + linksSize + sizeof(decltype(mStatus)) + mScopeTags.DataSize();
}

#ifdef APSARA_UNIT_TEST_MAIN
Json::Value SpanEvent::ToJson(bool enableEventMeta) const {
Json::Value root;
root["type"] = static_cast<int>(GetType());
root["timestamp"] = GetTimestamp();
root["timestampNanosecond"] = GetTimestampNanosecond();
if (GetTimestampNanosecond()) {
root["timestampNanosecond"] = static_cast<int32_t>(GetTimestampNanosecond().value());
}
root["traceId"] = mTraceId.to_string();
root["spanId"] = mSpanId.to_string();
if (!mTraceState.empty()) {
Expand Down
2 changes: 1 addition & 1 deletion core/processor/daemon/LogProcess.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -231,7 +231,7 @@ void* LogProcess::ProcessLoop(int32_t threadNo) {

{
ReadLock lock(mAccessProcessThreadRWL);

std::unique_ptr<ProcessQueueItem> item;
std::string configName;
if (!ProcessQueueManager::GetInstance()->PopItem(threadNo, item, configName)) {
Expand Down
2 changes: 1 addition & 1 deletion core/spl/PipelineEventGroupInput.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ void PipelineEventGroupInput::getTimeColumns(std::vector<uint32_t>& times,
for (const auto &event : mLogGroup->GetEvents()) {
const LogEvent& sourceEvent = event.Cast<LogEvent>();
times.emplace_back(sourceEvent.GetTimestamp());
timeNanos.emplace_back(sourceEvent.GetTimestampNanosecond());
timeNanos.emplace_back(sourceEvent.GetTimestampNanosecond() ? sourceEvent.GetTimestampNanosecond().value() : 0);
}
}

Expand Down
4 changes: 2 additions & 2 deletions core/unittest/models/MetricEventUnittest.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,7 @@ void MetricEventUnittest::TestSize() {
}

void MetricEventUnittest::TestToJson() {
mMetricEvent->SetTimestamp(12345678901);
mMetricEvent->SetTimestamp(12345678901, 0);
mMetricEvent->SetName("test");
mMetricEvent->SetValue(UntypedSingleValue{10.0});
mMetricEvent->SetTag(string("key1"), string("value1"));
Expand Down Expand Up @@ -170,7 +170,7 @@ void MetricEventUnittest::TestFromJson() {
mMetricEvent->FromJson(eventJson);

APSARA_TEST_EQUAL(12345678901, mMetricEvent->GetTimestamp());
APSARA_TEST_EQUAL(0L, mMetricEvent->GetTimestampNanosecond());
APSARA_TEST_EQUAL(0L, mMetricEvent->GetTimestampNanosecond().value());
APSARA_TEST_EQUAL("test", mMetricEvent->GetName());
APSARA_TEST_TRUE(mMetricEvent->Is<UntypedSingleValue>());
APSARA_TEST_EQUAL(10.0, mMetricEvent->GetValue<UntypedSingleValue>()->mValue);
Expand Down
4 changes: 3 additions & 1 deletion core/unittest/models/SpanEventUnittest.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -247,7 +247,7 @@ void SpanEventUnittest::TestSize() {
}

void SpanEventUnittest::TestToJson() {
mSpanEvent->SetTimestamp(12345678901);
mSpanEvent->SetTimestamp(12345678901, 0);
mSpanEvent->SetTraceId("test_trace_id");
mSpanEvent->SetSpanId("test_span_id");
mSpanEvent->SetTraceState("normal");
Expand Down Expand Up @@ -343,6 +343,8 @@ void SpanEventUnittest::TestFromJson() {
ParseJsonTable(eventStr, eventJson, errorMsg);
mSpanEvent->FromJson(eventJson);

APSARA_TEST_EQUAL(12345678901, mSpanEvent->GetTimestamp());
APSARA_TEST_EQUAL(0L, mSpanEvent->GetTimestampNanosecond().value());
APSARA_TEST_EQUAL("test_trace_id", mSpanEvent->GetTraceId().to_string());
APSARA_TEST_EQUAL("test_span_id", mSpanEvent->GetSpanId().to_string());
APSARA_TEST_EQUAL("normal", mSpanEvent->GetTraceState().to_string());
Expand Down
7 changes: 0 additions & 7 deletions core/unittest/processor/ProcessorMergeMultilineLogNativeUnittest.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -365,7 +365,6 @@ void ProcessorMergeMultilineLogNativeUnittest::TestProcess() {
{
"name": "",
"timestamp": 0,
"timestampNanosecond": 0,
"type": 2,
"value": {
"type": "unknown"
Expand Down Expand Up @@ -469,7 +468,6 @@ void ProcessorMergeMultilineLogNativeUnittest::TestProcess() {
{
"name": "",
"timestamp": 0,
"timestampNanosecond": 0,
"type": 2,
"value": {
"type": "unknown"
Expand Down Expand Up @@ -526,7 +524,6 @@ void ProcessorMergeMultilineLogNativeUnittest::TestProcess() {
{
"name": "",
"timestamp": 0,
"timestampNanosecond": 0,
"type": 2,
"value": {
"type": "unknown"
Expand Down Expand Up @@ -1412,7 +1409,6 @@ void ProcessEventsWithPartLogUnittest::TestProcess() {
{
"name": "",
"timestamp": 0,
"timestampNanosecond": 0,
"type": 2,
"value": {
"type": "unknown"
Expand Down Expand Up @@ -1556,7 +1552,6 @@ void ProcessEventsWithPartLogUnittest::TestProcess() {
{
"name": "",
"timestamp": 0,
"timestampNanosecond": 0,
"type": 2,
"value": {
"type": "unknown"
Expand Down Expand Up @@ -1697,7 +1692,6 @@ void ProcessEventsWithPartLogUnittest::TestProcess() {
{
"name": "",
"timestamp": 0,
"timestampNanosecond": 0,
"type": 2,
"value": {
"type": "unknown"
Expand Down Expand Up @@ -1827,7 +1821,6 @@ void ProcessEventsWithPartLogUnittest::TestProcess() {
{
"name": "",
"timestamp": 0,
"timestampNanosecond": 0,
"type": 2,
"value": {
"type": "unknown"
Expand Down
10 changes: 0 additions & 10 deletions core/unittest/processor/ProcessorParseApsaraNativeUnittest.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -491,15 +491,13 @@ void ProcessorParseApsaraNativeUnittest::TestMultipleLines() {
"__raw__": "[2023-09-04 13:15"
},
"timestamp": 12345678901,
"timestampNanosecond": 0,
"type": 1
},
{
"contents": {
"__raw__": ":50.1]\t[ERROR]\t[1]\t/ilogtail/AppConfigBase.cpp:1\t\tAppConfigBase AppConfigBase:1"
},
"timestamp": 12345678901,
"timestampNanosecond": 0,
"type": 1
},
{
Expand Down Expand Up @@ -863,7 +861,6 @@ void ProcessorParseApsaraNativeUnittest::TestProcessKeyOverwritten() {
"rawLog": "value1"
},
"timestamp": 12345678901,
"timestampNanosecond": 0,
"type": 1
}
]
Expand Down Expand Up @@ -939,7 +936,6 @@ void ProcessorParseApsaraNativeUnittest::TestUploadRawLog() {
"rawLog": "value1"
},
"timestamp": 12345678901,
"timestampNanosecond": 0,
"type": 1
}
]
Expand Down Expand Up @@ -1050,7 +1046,6 @@ void ProcessorParseApsaraNativeUnittest::TestProcessEventKeepUnmatch() {
"rawLog" : "value1"
},
"timestamp" : 12345678901,
"timestampNanosecond": 0,
"type" : 1
},
{
Expand All @@ -1059,7 +1054,6 @@ void ProcessorParseApsaraNativeUnittest::TestProcessEventKeepUnmatch() {
"rawLog" : "value1"
},
"timestamp" : 12345678901,
"timestampNanosecond": 0,
"type" : 1
},
{
Expand All @@ -1068,7 +1062,6 @@ void ProcessorParseApsaraNativeUnittest::TestProcessEventKeepUnmatch() {
"rawLog" : "value1"
},
"timestamp" : 12345678901,
"timestampNanosecond": 0,
"type" : 1
},
{
Expand All @@ -1077,7 +1070,6 @@ void ProcessorParseApsaraNativeUnittest::TestProcessEventKeepUnmatch() {
"rawLog" : "value1"
},
"timestamp" : 12345678901,
"timestampNanosecond": 0,
"type" : 1
},
{
Expand All @@ -1086,7 +1078,6 @@ void ProcessorParseApsaraNativeUnittest::TestProcessEventKeepUnmatch() {
"rawLog" : "value1"
},
"timestamp" : 12345678901,
"timestampNanosecond": 0,
"type" : 1
}
]
Expand Down Expand Up @@ -1292,7 +1283,6 @@ void ProcessorParseApsaraNativeUnittest::TestProcessEventMicrosecondUnmatch() {
"rawLog": "[2023-09-04 13:18:04"
},
"timestamp": 12345678901,
"timestampNanosecond": 0,
"type": 1
}
]
Expand Down
Loading
Loading