Skip to content

Commit

Permalink
refactor writer to have split schema
Browse files Browse the repository at this point in the history
  • Loading branch information
@mh-park
mh-park committed Jul 11, 2017
1 parent 433b7e0 commit db202fc
Show file tree
Hide file tree
Showing 3 changed files with 207 additions and 167 deletions.
272 changes: 142 additions & 130 deletions plugin/storage/es/spanstore/schema.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,10 @@

package spanstore

// TODO: divide the span and service type into separate indices
import "fmt"

// TODO: resolve traceID concerns (may not require any changes here)
const spanMapping = `{
const mapping = `{
"settings":{
"index.mapping.nested_fields.limit":50,
"index.requests.cache.enable":true,
Expand All @@ -34,133 +35,144 @@ const spanMapping = `{
"enabled":false
}
},
"span":{
"properties":{
"traceID":{
"type":"keyword",
"ignore_above":256
},
"parentSpanID":{
"type":"keyword",
"ignore_above":256
},
"spanID":{
"type":"keyword",
"ignore_above":256
},
"operationName":{
"type":"keyword",
"ignore_above":256
},
"startTime":{
"type":"long"
},
"duration":{
"type":"long"
},
"flags":{
"type":"integer"
},
"logs":{
"properties":{
"timestamp":{
"type":"long"
},
"fields":{
"type":"nested",
"dynamic":false,
"properties":{
"key":{
"type":"keyword",
"ignore_above":256
},
"value":{
"type":"keyword",
"ignore_above":256
},
"tagType":{
"type":"keyword",
"ignore_above":256
}
}
}
}
},
"process":{
"properties":{
"serviceName":{
"type":"keyword",
"ignore_above":256
},
"tags":{
"type":"nested",
"dynamic":false,
"properties":{
"key":{
"type":"keyword",
"ignore_above":256
},
"value":{
"type":"keyword",
"ignore_above":256
},
"tagType":{
"type":"keyword",
"ignore_above":256
}
}
}
}
},
"references":{
"type":"nested",
"dynamic":false,
"properties":{
"refType":{
"type":"keyword",
"ignore_above":256
},
"traceID":{
"type":"keyword",
"ignore_above":256
},
"spanID":{
"type":"keyword",
"ignore_above":256
}
}
},
"tags":{
"type":"nested",
"dynamic":false,
"properties":{
"key":{
"type":"keyword",
"ignore_above":256
},
"value":{
"type":"keyword",
"ignore_above":256
},
"tagType":{
"type":"keyword",
"ignore_above":256
}
}
}
}
},
"service":{
"properties":{
"serviceName":{
"type":"keyword",
"ignore_above":256
},
"operationName":{
"type":"keyword",
"ignore_above":256
}
}
}
%s
}
}`

var (
spanMapping = fmt.Sprintf(
mapping,
`"span":{
"properties":{
"traceID":{
"type":"keyword",
"ignore_above":256
},
"parentSpanID":{
"type":"keyword",
"ignore_above":256
},
"spanID":{
"type":"keyword",
"ignore_above":256
},
"operationName":{
"type":"keyword",
"ignore_above":256
},
"startTime":{
"type":"long"
},
"duration":{
"type":"long"
},
"flags":{
"type":"integer"
},
"logs":{
"properties":{
"timestamp":{
"type":"long"
},
"fields":{
"type":"nested",
"dynamic":false,
"properties":{
"key":{
"type":"keyword",
"ignore_above":256
},
"value":{
"type":"keyword",
"ignore_above":256
},
"tagType":{
"type":"keyword",
"ignore_above":256
}
}
}
}
},
"process":{
"properties":{
"serviceName":{
"type":"keyword",
"ignore_above":256
},
"tags":{
"type":"nested",
"dynamic":false,
"properties":{
"key":{
"type":"keyword",
"ignore_above":256
},
"value":{
"type":"keyword",
"ignore_above":256
},
"tagType":{
"type":"keyword",
"ignore_above":256
}
}
}
}
},
"references":{
"type":"nested",
"dynamic":false,
"properties":{
"refType":{
"type":"keyword",
"ignore_above":256
},
"traceID":{
"type":"keyword",
"ignore_above":256
},
"spanID":{
"type":"keyword",
"ignore_above":256
}
}
},
"tags":{
"type":"nested",
"dynamic":false,
"properties":{
"key":{
"type":"keyword",
"ignore_above":256
},
"value":{
"type":"keyword",
"ignore_above":256
},
"tagType":{
"type":"keyword",
"ignore_above":256
}
}
}
}
}`,
)

serviceMapping = fmt.Sprintf(
mapping,
`"service":{
"properties":{
"serviceName":{
"type":"keyword",
"ignore_above":256
},
"operationName":{
"type":"keyword",
"ignore_above":256
}
}
}`,
)
)
24 changes: 14 additions & 10 deletions plugin/storage/es/spanstore/writer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,38 +63,42 @@ func NewSpanWriter(client es.Client, logger *zap.Logger) *SpanWriter {

// WriteSpan writes a span and its corresponding service:operation in ElasticSearch
func (s *SpanWriter) WriteSpan(span *model.Span) error {
jaegerIndexName := spanIndexName(span)
spanIndexName, serviceIndexName := indexNames(span)
// Convert model.Span into json.Span
jsonSpan := json.FromDomainEmbedProcess(span)

if err := s.checkAndCreateIndex(jaegerIndexName, jsonSpan); err != nil {
if err := s.checkAndCreateIndex(spanIndexName, serviceIndexName, jsonSpan); err != nil {
return err
}
if err := s.writeService(jaegerIndexName, jsonSpan); err != nil {
if err := s.writeService(serviceIndexName, jsonSpan); err != nil {
return err
}
if err := s.writeSpan(jaegerIndexName, jsonSpan); err != nil {
if err := s.writeSpan(spanIndexName, jsonSpan); err != nil {
return err
}
return nil
}

func spanIndexName(span *model.Span) string {
func indexNames(span *model.Span) (string, string) {
spanDate := span.StartTime.Format("2006-01-02")
return "jaeger-" + spanDate
return "jaeger-span-" + spanDate, "jaeger-service-" + spanDate
}

// Check if index exists, and create index if it does not.
func (s *SpanWriter) checkAndCreateIndex(indexName string, jsonSpan *jModel.Span) error {
func (s *SpanWriter) checkAndCreateIndex(spanIndexName string, serviceIndexName string, jsonSpan *jModel.Span) error {
// TODO: We don't need to check every write. Try to pull this out of WriteSpan.
exists, err := s.client.IndexExists(indexName).Do(s.ctx)
exists, err := s.client.IndexExists(spanIndexName).Do(s.ctx)
if err != nil {
return s.logError(jsonSpan, err, "Failed to find index", s.logger)
}
if !exists {
_, err = s.client.CreateIndex(indexName).Body(spanMapping).Do(s.ctx)
_, err = s.client.CreateIndex(spanIndexName).Body(spanMapping).Do(s.ctx)
if err != nil {
return s.logError(jsonSpan, err, "Failed to create index", s.logger)
return s.logError(jsonSpan, err, "Failed to create span index", s.logger)
}
_, err = s.client.CreateIndex(serviceIndexName).Body(serviceMapping).Do(s.ctx)
if err != nil {
return s.logError(jsonSpan, err, "Failed to create service:operation index", s.logger)
}
}
return nil
Expand Down
Loading

0 comments on commit db202fc

Please sign in to comment.