Add new GetLogs API

client_interface.go

@@ -221,6 +221,7 @@ type ClientInterface interface {
 
 	GetLogsV2(project, logstore string, req *GetLogRequest) (*GetLogsResponse, error)
 	GetLogLinesV2(project, logstore string, req *GetLogRequest) (*GetLogLinesResponse, error)
+	GetLogsV3(project, logstore string, req *GetLogRequest) (*GetLogsV3Response, error)
 
 	// GetHistogramsToCompleted query logs with [from, to) time range to completed
 	GetHistogramsToCompleted(project, logstore string, topic string, from int64, to int64, queryExp string) (*GetHistogramsResponse, error)

client_store.go

@@ -230,6 +230,12 @@ func (c *Client) GetLogsV2(project, logstore string, req *GetLogRequest) (*GetLo
 	return ls.GetLogsV2(req)
 }
 
+// GetLogsV3 ...
+func (c *Client) GetLogsV3(project, logstore string, req *GetLogRequest) (*GetLogsV3Response, error) {
+	ls := convertLogstore(c, project, logstore)
+	return ls.GetLogsV3(req)
+}
+
 // GetLogsToCompletedV2 ...
 func (c *Client) GetLogsToCompletedV2(project, logstore string, req *GetLogRequest) (*GetLogsResponse, error) {
 	ls := convertLogstore(c, project, logstore)

config.go

@@ -21,6 +21,7 @@ const (
 	RequestIDHeader = "x-log-requestid"
 
 	GetLogsQueryInfo = "X-Log-Query-Info"
+	BodyRawSize      = "X-Log-Bodyrawsize"
 	HasSQLHeader     = "x-log-has-sql"
 	ETLVersion       = 2
 	ETLType          = "ETL"

log_store.go

@@ -719,6 +719,55 @@ func (s *LogStore) GetLogsV2(req *GetLogRequest) (*GetLogsResponse, error) {
 	return logRsp, err
 }
 
+// GetLogsV3 query logs with [from, to) time range
+func (s *LogStore) GetLogsV3(req *GetLogRequest) (*GetLogsV3Response, error) {
+	reqBody, err := json.Marshal(req)
+	if err != nil {
+		return nil, err
+	}
+	h := map[string]string{
+		"x-log-bodyrawsize": fmt.Sprintf("%v", len(reqBody)),
+		"Content-Type":      "application/json",
+		"Accept-Encoding":   "lz4",
+	}
+	uri := fmt.Sprintf("/logstores/%s/logs", s.Name)
+	r, err := request(s.project, "POST", uri, h, reqBody)
+	if err != nil {
+		return nil, NewClientError(err)
+	}
+	defer r.Body.Close()
+
+	respBody, _ := ioutil.ReadAll(r.Body)
+	if r.StatusCode != http.StatusOK {
+		err := new(Error)
+		if jErr := json.Unmarshal(respBody, err); jErr != nil {
+			return nil, NewBadResponseError(string(respBody), r.Header, r.StatusCode)
+		}
+		return nil, err
+	}
+	if _, ok := r.Header[BodyRawSize]; ok {
+		if len(r.Header[BodyRawSize]) > 0 {
+			bodyRawSize, err := strconv.ParseInt(r.Header[BodyRawSize][0], 10, 64)
+			if err != nil {
+				return nil, NewBadResponseError(string(respBody), r.Header, r.StatusCode)
+			}
+			out := make([]byte, bodyRawSize)
+			if bodyRawSize != 0 {
+				len, err := lz4.UncompressBlock(respBody, out)
+				if err != nil || int64(len) != bodyRawSize {
+					return nil, NewBadResponseError(string(respBody), r.Header, r.StatusCode)
+				}
+			}
+			respBody = out
+		}
+	}
+	var result GetLogsV3Response
+	if err = json.Unmarshal(respBody, &result); err != nil {
+		return nil, NewBadResponseError(string(respBody), r.Header, r.StatusCode)
+	}
+	return &result, nil
+}
+
 // GetContextLogs ...
 func (s *LogStore) GetContextLogs(backLines int32, forwardLines int32,
 	packID string, packMeta string) (*GetContextLogsResponse, error) {

model.go

@@ -10,14 +10,14 @@ import (
 
 // GetLogRequest for GetLogsV2
 type GetLogRequest struct {
-	From     int64  // unix time, eg time.Now().Unix() - 900
-	To       int64  // unix time, eg time.Now().Unix()
-	Topic    string // @note topic is not used anymore, use __topic__ : xxx in query instead
-	Lines    int64  // max 100; offset, lines and reverse is ignored when use SQL in query
-	Offset   int64
-	Reverse  bool
-	Query    string
-	PowerSQL bool
+	From     int64  `json:"from"`  // unix time, eg time.Now().Unix() - 900
+	To       int64  `json:"to"`    // unix time, eg time.Now().Unix()
+	Topic    string `json:"topic"` // @note topic is not used anymore, use __topic__ : xxx in query instead
+	Lines    int64  `json:"lines"` // max 100; offset, lines and reverse is ignored when use SQL in query
+	Offset   int64  `json:"offset"`
+	Reverse  bool   `json:"reverse"`
+	Query    string `json:"query"`
+	PowerSQL bool   `json:"powerSql"`
 }
 
 func (glr *GetLogRequest) ToURLParams() url.Values {
@@ -62,6 +62,25 @@ type GetLogsResponse struct {
 	Header   http.Header         `json:"header"`
 }
 
+type GetLogsV3ResponseMeta struct {
+	Progress           string  `json:"progress"`
+	AggQuery           string  `json:"aggQuery"`
+	WhereQuery         string  `json:"whereQuery"`
+	HasSQL             bool    `json:"hasSQL"`
+	ProcessedRows      int64   `json:"processedRows"`
+	ElapsedMillisecond int64   `json:"elapsedMillisecond"`
+	CpuSec             float64 `json:"cpuSec"`
+	CpuCores           float64 `json:"cpuCores"`
+	Limited            int64   `json:"limited"`
+	Count              int64   `json:"count"`
+}
+
+// GetLogsV3Response defines response from GetLogs call
+type GetLogsV3Response struct {
+	Meta GetLogsV3ResponseMeta `json:"meta"`
+	Logs []map[string]string   `json:"data"`
+}
+
 // GetLogLinesResponse defines response from GetLogLines call
 // note: GetLogLinesResponse.Logs is nil when use GetLogLinesResponse
 type GetLogLinesResponse struct {

token_auto_update_client.go

@@ -799,6 +799,16 @@ func (c *TokenAutoUpdateClient) GetLogsV2(project, logstore string, req *GetLogR
 	return
 }
 
+func (c *TokenAutoUpdateClient) GetLogsV3(project, logstore string, req *GetLogRequest) (r *GetLogsV3Response, err error) {
+	for i := 0; i < c.maxTryTimes; i++ {
+		r, err = c.logClient.GetLogsV3(project, logstore, req)
+		if !c.processError(err) {
+			return
+		}
+	}
+	return
+}
+
 func (c *TokenAutoUpdateClient) GetLogsToCompletedV2(project, logstore string, req *GetLogRequest) (r *GetLogsResponse, err error) {
 	for i := 0; i < c.maxTryTimes; i++ {
 		r, err = c.logClient.GetLogsToCompletedV2(project, logstore, req)
@@ -1765,4 +1775,4 @@ func (c *TokenAutoUpdateClient) PublishAlertEvent(project string, alertResult []
 		}
 	}
 	return err
-}
+}