diff --git a/fe/fe-core/src/main/java/com/starrocks/common/AuditLog.java b/fe/fe-core/src/main/java/com/starrocks/common/AuditLog.java
index 043aa16e2e253..4a14fbcc7968e 100644
--- a/fe/fe-core/src/main/java/com/starrocks/common/AuditLog.java
+++ b/fe/fe-core/src/main/java/com/starrocks/common/AuditLog.java
@@ -25,6 +25,7 @@ public class AuditLog {
public static final AuditLog SLOW_AUDIT = new AuditLog("audit.slow_query");
public static final AuditLog QUERY_AUDIT = new AuditLog("audit.query");
+ public static final AuditLog BIG_QUERY_AUDIT = new AuditLog("big_query.query");
private Logger logger;
@@ -36,6 +37,10 @@ public static AuditLog getSlowAudit() {
return SLOW_AUDIT;
}
+ public static AuditLog getBigQueryAudit() {
+ return BIG_QUERY_AUDIT;
+ }
+
public AuditLog(String auditName) {
logger = LogManager.getLogger(auditName);
}
diff --git a/fe/fe-core/src/main/java/com/starrocks/common/Config.java b/fe/fe-core/src/main/java/com/starrocks/common/Config.java
index 6944311b5591a..1a3ad867a7cda 100644
--- a/fe/fe-core/src/main/java/com/starrocks/common/Config.java
+++ b/fe/fe-core/src/main/java/com/starrocks/common/Config.java
@@ -157,6 +157,47 @@ public class Config extends ConfigBase {
@ConfField
public static String dump_log_delete_age = "7d";
+ /**
+ * big_query_log_dir:
+ * This specifies FE big query log dir.
+ * Dump log fe.big_query.log contains all information about big query.
+ * The structure of each log record is very similar to the audit log.
+ * If the cpu cost of a query exceeds big_query_log_cpu_second_threshold,
+ * or scan rows exceeds big_query_log_scan_rows_threshold,
+ * or scan bytes exceeds big_query_log_scan_bytes_threshold,
+ * we will consider it as a big query.
+ * These thresholds are defined by the user.
+ *
+ * big_query_log_roll_num:
+ * Maximal FE log files to be kept within an big_query_log_roll_interval.
+ *
+ * big_query_log_modules:
+ * Informations for all big queries.
+ *
+ * big_query_log_roll_interval:
+ * DAY: log suffix is yyyyMMdd
+ * HOUR: log suffix is yyyyMMddHH
+ *
+ * big_query_log_delete_age:
+ * default is 7 days, if log's last modify time is 7 days ago, it will be deleted.
+ * support format:
+ * 7d 7 days
+ * 10h 10 hours
+ * 60m 60 mins
+ * 120s 120 seconds
+ */
+ @ConfField
+ public static String big_query_log_dir = StarRocksFE.STARROCKS_HOME_DIR + "/log";
+ @ConfField
+ public static int big_query_log_roll_num = 10;
+ @ConfField
+ public static String[] big_query_log_modules = {"query"};
+ @ConfField
+ public static String big_query_log_roll_interval = "DAY";
+ @ConfField
+ public static String big_query_log_delete_age = "7d";
+
+
/**
* plugin_dir:
* plugin install directory
diff --git a/fe/fe-core/src/main/java/com/starrocks/common/Log4jConfig.java b/fe/fe-core/src/main/java/com/starrocks/common/Log4jConfig.java
index c9a0bacb5a78f..3f357d8fd6736 100644
--- a/fe/fe-core/src/main/java/com/starrocks/common/Log4jConfig.java
+++ b/fe/fe-core/src/main/java/com/starrocks/common/Log4jConfig.java
@@ -103,6 +103,21 @@ public class Log4jConfig extends XmlConfiguration {
" \n" +
" \n" +
" \n" +
+ " \n" +
+ " \n" +
+ " %d{yyyy-MM-dd HH:mm:ss,SSS} [%c{1}] %m%n\n" +
+ " \n" +
+ " \n" +
+ " \n" +
+ " \n" +
+ " \n" +
+ " \n" +
+ " \n" +
+ " \n" +
+ " \n" +
+ " \n" +
+ " \n" +
+ " \n" +
" \n" +
" \n" +
" \n" +
@@ -115,6 +130,9 @@ public class Log4jConfig extends XmlConfiguration {
" \n" +
" \n" +
" \n" +
+ " \n" +
+ " \n" +
+ " \n" +
" \n" +
" \n" +
" \n" +
@@ -136,6 +154,7 @@ public class Log4jConfig extends XmlConfiguration {
private static String[] verboseModules;
private static String[] auditModules;
private static String[] dumpModules;
+ private static String[] bigQueryModules;
private static void reconfig() throws IOException {
String newXmlConfTemplate = xmlConfTemplate;
@@ -191,6 +210,20 @@ private static void reconfig() throws IOException {
throw new IOException("dump_log_roll_interval config error: " + Config.dump_log_roll_interval);
}
+ // big query log config
+ String bigQueryLogDir = Config.big_query_log_dir;
+ String bigQueryLogRollPattern = "%d{yyyyMMdd}";
+ String bigQueryLogRollNum = String.valueOf(Config.big_query_log_roll_num);
+ String bigQueryLogRollMaxSize = String.valueOf(Config.log_roll_size_mb);
+ String bigQueryLogDeleteAge = String.valueOf(Config.big_query_log_delete_age);
+ if (Config.big_query_log_roll_interval.equals("HOUR")) {
+ bigQueryLogRollPattern = "%d{yyyyMMddHH}";
+ } else if (Config.big_query_log_roll_interval.equals("DAY")) {
+ bigQueryLogRollPattern = "%d{yyyyMMdd}";
+ } else {
+ throw new IOException("big_query_log_roll_interval config error: " + Config.big_query_log_roll_interval);
+ }
+
// verbose modules and audit log modules
StringBuilder sb = new StringBuilder();
for (String s : verboseModules) {
@@ -202,6 +235,9 @@ private static void reconfig() throws IOException {
for (String s : dumpModules) {
sb.append("");
}
+ for (String s : bigQueryModules) {
+ sb.append("");
+ }
newXmlConfTemplate = newXmlConfTemplate.replaceAll("",
sb.toString());
@@ -226,6 +262,12 @@ private static void reconfig() throws IOException {
properties.put("dump_roll_num", dumpRollNum);
properties.put("dump_log_delete_age", dumpDeleteAge);
+ properties.put("big_query_log_dir", bigQueryLogDir);
+ properties.put("big_query_file_pattern", bigQueryLogRollPattern);
+ properties.put("big_query_roll_maxsize", bigQueryLogRollMaxSize);
+ properties.put("big_query_roll_num", bigQueryLogRollNum);
+ properties.put("big_query_log_delete_age", bigQueryLogDeleteAge);
+
strSub = new StrSubstitutor(new Interpolator(properties));
newXmlConfTemplate = strSub.replace(newXmlConfTemplate);
@@ -269,6 +311,7 @@ public static synchronized void initLogging() throws IOException {
verboseModules = Config.sys_log_verbose_modules;
auditModules = Config.audit_log_modules;
dumpModules = Config.dump_log_modules;
+ bigQueryModules = Config.big_query_log_modules;
reconfig();
}
diff --git a/fe/fe-core/src/main/java/com/starrocks/plugin/AuditEvent.java b/fe/fe-core/src/main/java/com/starrocks/plugin/AuditEvent.java
index 29516f66b6c35..4858c3b8a81f5 100644
--- a/fe/fe-core/src/main/java/com/starrocks/plugin/AuditEvent.java
+++ b/fe/fe-core/src/main/java/com/starrocks/plugin/AuditEvent.java
@@ -102,6 +102,12 @@ public enum EventType {
public double planMemCosts = 0.0;
@AuditField(value = "PendingTimeMs")
public long pendingTimeMs = 0;
+ @AuditField(value = "BigQueryLogCPUSecondThreshold")
+ public long bigQueryLogCPUSecondThreshold = -1;
+ @AuditField(value = "BigQueryLogScanBytesThreshold")
+ public long bigQueryLogScanBytesThreshold = -1;
+ @AuditField(value = "BigQueryLogScanRowsThreshold")
+ public long bigQueryLogScanRowsThreshold = -1;
public static class AuditEventBuilder {
@@ -242,6 +248,21 @@ public AuditEventBuilder setPendingTimeMs(long pendingTimeMs) {
return this;
}
+ public AuditEventBuilder setBigQueryLogCPUSecondThreshold(long bigQueryLogCPUSecondThreshold) {
+ auditEvent.bigQueryLogCPUSecondThreshold = bigQueryLogCPUSecondThreshold;
+ return this;
+ }
+
+ public AuditEventBuilder setBigQueryLogScanBytesThreshold(long bigQueryLogScanBytesThreshold) {
+ auditEvent.bigQueryLogScanBytesThreshold = bigQueryLogScanBytesThreshold;
+ return this;
+ }
+
+ public AuditEventBuilder setBigQueryLogScanRowsThreshold(long bigQueryLogScanRowsThreshold) {
+ auditEvent.bigQueryLogScanRowsThreshold = bigQueryLogScanRowsThreshold;
+ return this;
+ }
+
public AuditEvent build() {
return this.auditEvent;
}
diff --git a/fe/fe-core/src/main/java/com/starrocks/qe/AuditLogBuilder.java b/fe/fe-core/src/main/java/com/starrocks/qe/AuditLogBuilder.java
index e668d499d5468..c1975ef9a74d2 100644
--- a/fe/fe-core/src/main/java/com/starrocks/qe/AuditLogBuilder.java
+++ b/fe/fe-core/src/main/java/com/starrocks/qe/AuditLogBuilder.java
@@ -77,6 +77,14 @@ public void exec(AuditEvent event) {
continue;
}
+ // fields related to big queries are not written into audit log by default,
+ // they will be written into big query log.
+ if (af.value().equals("BigQueryLogCPUSecondThreshold") ||
+ af.value().equals("BigQueryLogScanBytesThreshold") ||
+ af.value().equals("BigQueryLogScanRowsThreshold")) {
+ continue;
+ }
+
if (af.value().equals("Time")) {
queryTime = (long) f.get(event);
}
@@ -89,8 +97,30 @@ public void exec(AuditEvent event) {
if (queryTime > Config.qe_slow_log_ms) {
AuditLog.getSlowAudit().log(auditLog);
}
+
+ if (isBigQuery(event)) {
+ sb.append("|bigQueryLogCPUSecondThreshold=").append(event.bigQueryLogCPUSecondThreshold);
+ sb.append("|bigQueryLogScanBytesThreshold=").append(event.bigQueryLogScanBytesThreshold);
+ sb.append("|bigQueryLogScanRowsThreshold=").append(event.bigQueryLogScanRowsThreshold);
+ String bigQueryLog = sb.toString();
+ AuditLog.getBigQueryAudit().log(bigQueryLog);
+ }
} catch (Exception e) {
LOG.debug("failed to process audit event", e);
}
}
+
+ private boolean isBigQuery(AuditEvent event) {
+ if (event.bigQueryLogCPUSecondThreshold >= 0 &&
+ event.cpuCostNs > event.bigQueryLogCPUSecondThreshold * 1000000000L) {
+ return true;
+ }
+ if (event.bigQueryLogScanBytesThreshold >= 0 && event.scanBytes > event.bigQueryLogScanBytesThreshold) {
+ return true;
+ }
+ if (event.bigQueryLogScanRowsThreshold >= 0 && event.scanRows > event.bigQueryLogScanRowsThreshold) {
+ return true;
+ }
+ return false;
+ }
}
diff --git a/fe/fe-core/src/main/java/com/starrocks/qe/ConnectProcessor.java b/fe/fe-core/src/main/java/com/starrocks/qe/ConnectProcessor.java
index 66754f786d1d1..6f20009e57453 100644
--- a/fe/fe-core/src/main/java/com/starrocks/qe/ConnectProcessor.java
+++ b/fe/fe-core/src/main/java/com/starrocks/qe/ConnectProcessor.java
@@ -171,6 +171,14 @@ public void auditAfterExec(String origStmt, StatementBase parsedStmt, PQueryStat
}
}
ctx.getAuditEventBuilder().setIsQuery(true);
+ if (ctx.getSessionVariable().isEnableBigQueryLog()) {
+ ctx.getAuditEventBuilder().setBigQueryLogCPUSecondThreshold(
+ ctx.getSessionVariable().getBigQueryLogCPUSecondThreshold());
+ ctx.getAuditEventBuilder().setBigQueryLogScanBytesThreshold(
+ ctx.getSessionVariable().getBigQueryLogScanBytesThreshold());
+ ctx.getAuditEventBuilder().setBigQueryLogScanRowsThreshold(
+ ctx.getSessionVariable().getBigQueryLogScanRowsThreshold());
+ }
} else {
ctx.getAuditEventBuilder().setIsQuery(false);
}
diff --git a/fe/fe-core/src/main/java/com/starrocks/qe/SessionVariable.java b/fe/fe-core/src/main/java/com/starrocks/qe/SessionVariable.java
index 54ffa930079cf..be92304cd02b9 100644
--- a/fe/fe-core/src/main/java/com/starrocks/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/com/starrocks/qe/SessionVariable.java
@@ -305,6 +305,11 @@ public class SessionVariable implements Serializable, Writable, Cloneable {
public static final String ENABLE_MATERIALIZED_VIEW_UNION_REWRITE = "enable_materialized_view_union_rewrite";
public static final String ENABLE_RULE_BASED_MATERIALIZED_VIEW_REWRITE = "enable_rule_based_materialized_view_rewrite";
+ public static final String ENABLE_BIG_QUERY_LOG = "enable_big_query_log";
+ public static final String BIG_QUERY_LOG_CPU_SECOND_THRESHOLD = "big_query_log_cpu_second_threshold";
+ public static final String BIG_QUERY_LOG_SCAN_BYTES_THRESHOLD = "big_query_log_scan_bytes_threshold";
+ public static final String BIG_QUERY_LOG_SCAN_ROWS_THRESHOLD = "big_query_log_scan_rows_threshold";
+
public static final List DEPRECATED_VARIABLES = ImmutableList.builder()
.add(CODEGEN_LEVEL)
.add(ENABLE_SPILLING)
@@ -753,6 +758,26 @@ public boolean getUseScanBlockCache() {
@VarAttr(name = ENABLE_RULE_BASED_MATERIALIZED_VIEW_REWRITE)
private boolean enableRuleBasedMaterializedViewRewrite = true;
+ // if enable_big_query_log = true and cpu/io cost of a query exceeds the related threshold,
+ // the information will be written to the big query log
+ @VarAttr(name = ENABLE_BIG_QUERY_LOG)
+ private boolean enableBigQueryLog = true;
+ // the value is set for testing,
+ // if a query needs to perform 10s for computing tasks at full load on three 16-core machines,
+ // we treat it as a big query, so set this value to 480(10 * 16 * 3).
+ // Users need to set up according to their own scenario.
+ @VarAttr(name = BIG_QUERY_LOG_CPU_SECOND_THRESHOLD)
+ private long bigQueryLogCPUSecondThreshold = 480;
+ // the value is set for testing, if a query needs to scan more than 10GB of data, we treat it as a big query.
+ // Users need to set up according to their own scenario.
+ @VarAttr(name = BIG_QUERY_LOG_SCAN_BYTES_THRESHOLD)
+ private long bigQueryLogScanBytesThreshold = 1024L * 1024 * 1024 * 10;
+ // the value is set for testing, if a query need to scan more than 1 billion rows of data,
+ // we treat it as a big query.
+ // Users need to set up according to their own scenario.
+ @VarAttr(name = BIG_QUERY_LOG_SCAN_ROWS_THRESHOLD)
+ private long bigQueryLogScanRowsThreshold = 1000000000L;
+
public boolean getEnablePopulateBlockCache() {
return enablePopulateBlockCache;
}
@@ -1421,6 +1446,38 @@ public void setEnableRuleBasedMaterializedViewRewrite(boolean enableRuleBasedMat
this.enableRuleBasedMaterializedViewRewrite = enableRuleBasedMaterializedViewRewrite;
}
+ public boolean isEnableBigQueryLog() {
+ return enableBigQueryLog;
+ }
+
+ public void setEnableBigQueryLog(boolean enableBigQueryLog) {
+ this.enableBigQueryLog = enableBigQueryLog;
+ }
+
+ public long getBigQueryLogCPUSecondThreshold() {
+ return this.bigQueryLogCPUSecondThreshold;
+ }
+
+ public void setBigQueryLogCpuSecondThreshold(long bigQueryLogCPUSecondThreshold) {
+ this.bigQueryLogCPUSecondThreshold = bigQueryLogCPUSecondThreshold;
+ }
+
+ public long getBigQueryLogScanBytesThreshold() {
+ return bigQueryLogScanBytesThreshold;
+ }
+
+ public void setBigQueryLogScanBytesThreshold(long bigQueryLogScanBytesThreshold) {
+ this.bigQueryLogScanBytesThreshold = bigQueryLogScanBytesThreshold;
+ }
+
+ public long getBigQueryLogScanRowsThreshold() {
+ return bigQueryLogScanRowsThreshold;
+ }
+
+ public void setBigQueryLogScanRowsThreshold(long bigQueryLogScanRowsThreshold) {
+ this.bigQueryLogScanRowsThreshold = bigQueryLogScanRowsThreshold;
+ }
+
// Serialize to thrift object
// used for rest api
public TQueryOptions toThrift() {