Skip to content

Commit 48c60f8

Browse files
michalpristasmichalpristas
authored
[Ingest Manager] Split index restrictions into type,dataset, namespace parts (#21406)
[Ingest Manager] Split index restrictions into type,dataset, namespace parts (#21406)
1 parent 13a5463 commit 48c60f8

File tree

2 files changed

+77
-93
lines changed

2 files changed

+77
-93
lines changed

‎x-pack/elastic-agent/pkg/agent/application/filters/stream_checker.go‎

Lines changed: 50 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@
55
package filters
66

77
import (
8-
"fmt"
98
"strings"
109

1110
"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors"
@@ -47,11 +46,7 @@ func StreamChecker(log *logger.Logger, ast *transpiler.AST) error {
4746
if nsNode, found := inputNode.Find("data_stream.namespace"); found {
4847
nsKey, ok := nsNode.(*transpiler.Key)
4948
if ok {
50-
newNamespace := nsKey.Value().(transpiler.Node).String()
51-
if !isValid(newNamespace) {
52-
return ErrInvalidNamespace
53-
}
54-
namespace = newNamespace
49+
namespace = nsKey.Value().(transpiler.Node).String()
5550
}
5651
} else {
5752
dsNode, found := inputNode.Find("data_stream")
@@ -63,17 +58,17 @@ func StreamChecker(log *logger.Logger, ast *transpiler.AST) error {
6358
if found {
6459
nsKey, ok := nsNode.(*transpiler.Key)
6560
if ok {
66-
newNamespace := nsKey.Value().(transpiler.Node).String()
67-
if !isValid(newNamespace) {
68-
return ErrInvalidNamespace
69-
}
70-
namespace = newNamespace
61+
namespace = nsKey.Value().(transpiler.Node).String()
7162
}
7263
}
7364
}
7465
}
7566
}
7667

68+
if !matchesNamespaceContraints(namespace) {
69+
return ErrInvalidNamespace
70+
}
71+
7772
// get the type, longest type for now is metrics
7873
datasetType := "metrics"
7974
if nsNode, found := inputNode.Find("data_stream.type"); found {
@@ -100,6 +95,10 @@ func StreamChecker(log *logger.Logger, ast *transpiler.AST) error {
10095
}
10196
}
10297

98+
if !matchesTypeConstraints(datasetType) {
99+
return ErrInvalidIndex
100+
}
101+
103102
streamsNode, ok := inputNode.Find("streams")
104103
if ok {
105104
streamsList, ok := streamsNode.Value().(*transpiler.List)
@@ -119,11 +118,8 @@ func StreamChecker(log *logger.Logger, ast *transpiler.AST) error {
119118
if dsNameNode, found := streamMap.Find("data_stream.dataset"); found {
120119
dsKey, ok := dsNameNode.(*transpiler.Key)
121120
if ok {
122-
newDataset := dsKey.Value().(transpiler.Node).String()
123-
if !isValid(newDataset) {
124-
return ErrInvalidDataset
125-
}
126-
datasetName = newDataset
121+
datasetName = dsKey.Value().(transpiler.Node).String()
122+
break
127123
}
128124
} else {
129125
datasetNode, found := streamMap.Find("data_stream")
@@ -137,61 +133,74 @@ func StreamChecker(log *logger.Logger, ast *transpiler.AST) error {
137133
if found {
138134
dsKey, ok := dsNameNode.(*transpiler.Key)
139135
if ok {
140-
newDataset := dsKey.Value().(transpiler.Node).String()
141-
if !isValid(newDataset) {
142-
return ErrInvalidDataset
143-
}
144-
datasetName = newDataset
136+
datasetName = dsKey.Value().(transpiler.Node).String()
137+
break
145138
}
146139
}
147140
}
148141
}
149142
}
150143
}
151144
}
152-
153-
if indexName := fmt.Sprintf("%s-%s-%s", datasetType, datasetName, namespace); !matchesIndexContraints(indexName) {
154-
return ErrInvalidIndex
145+
if !matchesDatasetConstraints(datasetName) {
146+
return ErrInvalidDataset
155147
}
156148
}
157149

158150
return nil
159151
}
160152

161-
// The only two requirement are that it has only characters allowed in an Elasticsearch index name
162-
// and does NOT contain a `-`.
163-
func isValid(namespace string) bool {
164-
return matchesIndexContraints(namespace) && !strings.Contains(namespace, "-")
165-
}
166-
167153
// The only two requirement are that it has only characters allowed in an Elasticsearch index name
168154
// Index names must meet the following criteria:
155+
// Not longer than 100 bytes
169156
// Lowercase only
170157
// Cannot include \, /, *, ?, ", <, >, |, ` ` (space character), ,, #
158+
func matchesNamespaceContraints(namespace string) bool {
159+
// length restriction is in bytes, not characters
160+
if len(namespace) <= 0 || len(namespace) > 100 {
161+
return false
162+
}
163+
164+
return isCharactersetValid(namespace)
165+
}
166+
167+
// matchesTypeConstraints fails for following rules. As type is first element of resulting index prefix restrictions need to be applied.
168+
// Not longer than 20 bytes
169+
// Lowercase only
171170
// Cannot start with -, _, +
172-
// Cannot be . or ..
173-
func matchesIndexContraints(namespace string) bool {
174-
// Cannot be . or ..
175-
if namespace == "." || namespace == ".." {
171+
// Cannot include \, /, *, ?, ", <, >, |, ` ` (space character), ,, #
172+
func matchesTypeConstraints(dsType string) bool {
173+
// length restriction is in bytes, not characters
174+
if len(dsType) <= 0 || len(dsType) > 20 {
176175
return false
177176
}
178177

179-
if len(namespace) <= 0 || len(namespace) > 255 {
178+
if strings.HasPrefix(dsType, "-") || strings.HasPrefix(dsType, "_") || strings.HasPrefix(dsType, "+") {
180179
return false
181180
}
182181

183-
// Lowercase only
184-
if strings.ToLower(namespace) != namespace {
182+
return isCharactersetValid(dsType)
183+
}
184+
185+
// matchesDatasetConstraints fails for following rules
186+
// Not longer than 100 bytes
187+
// Lowercase only
188+
// Cannot include \, /, *, ?, ", <, >, |, ` ` (space character), ,, #
189+
func matchesDatasetConstraints(dataset string) bool {
190+
// length restriction is in bytes, not characters
191+
if len(dataset) <= 0 || len(dataset) > 100 {
185192
return false
186193
}
187194

188-
// Cannot include \, /, *, ?, ", <, >, |, ` ` (space character), ,, #
189-
if strings.ContainsAny(namespace, "\\/*?\"<>| ,#") {
195+
return isCharactersetValid(dataset)
196+
}
197+
198+
func isCharactersetValid(input string) bool {
199+
if strings.ToLower(input) != input {
190200
return false
191201
}
192202

193-
// Cannot start with -, _, +
194-
if strings.HasPrefix(namespace, "-") || strings.HasPrefix(namespace, "_") || strings.HasPrefix(namespace, "+") {
203+
if strings.ContainsAny(input, "\\/*?\"<>| ,#:") {
195204
return false
196205
}
197206

‎x-pack/elastic-agent/pkg/agent/application/filters/stream_checker_test.go‎

Lines changed: 27 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -93,25 +93,6 @@ func TestStreamCheck(t *testing.T) {
9393
},
9494
result: ErrInvalidDataset,
9595
},
96-
97-
{
98-
name: "dataset invalid dot - compact",
99-
configMap: map[string]interface{}{
100-
"inputs": []map[string]interface{}{
101-
{"streams": []map[string]interface{}{{"data_stream.dataset": "."}}},
102-
},
103-
},
104-
result: ErrInvalidDataset,
105-
},
106-
{
107-
name: "dataset invalid dotdot- compact",
108-
configMap: map[string]interface{}{
109-
"inputs": []map[string]interface{}{
110-
{"streams": []map[string]interface{}{{"data_stream.dataset": ".."}}},
111-
},
112-
},
113-
result: ErrInvalidDataset,
114-
},
11596
{
11697
name: "dataset invalid uppercase - compact",
11798
configMap: map[string]interface{}{
@@ -139,39 +120,13 @@ func TestStreamCheck(t *testing.T) {
139120
},
140121
result: ErrInvalidDataset,
141122
},
142-
{
143-
name: "dataset invalid invalid prefix- compact",
144-
configMap: map[string]interface{}{
145-
"inputs": []map[string]interface{}{
146-
{"streams": []map[string]interface{}{{"data_stream.dataset": "_isthisvalid"}}},
147-
},
148-
},
149-
result: ErrInvalidDataset,
150-
},
151-
152123
{
153124
name: "namespace invalid - compact",
154125
configMap: map[string]interface{}{
155126
"inputs": []map[string]interface{}{{"data_stream.namespace": ""}},
156127
},
157128
result: ErrInvalidNamespace,
158129
},
159-
{
160-
name: "namespace invalid name 1 - compact",
161-
configMap: map[string]interface{}{
162-
"inputs": []map[string]interface{}{
163-
{"data_stream.namespace": "."},
164-
},
165-
},
166-
result: ErrInvalidNamespace,
167-
},
168-
{
169-
name: "namespace invalid name 2 - compact",
170-
configMap: map[string]interface{}{
171-
"inputs": []map[string]interface{}{{"data_stream.namespace": ".."}},
172-
},
173-
result: ErrInvalidNamespace,
174-
},
175130
{
176131
name: "namespace invalid name uppercase - compact",
177132
configMap: map[string]interface{}{
@@ -193,13 +148,6 @@ func TestStreamCheck(t *testing.T) {
193148
},
194149
result: ErrInvalidNamespace,
195150
},
196-
{
197-
name: "namespace invalid name invalid prefix - compact",
198-
configMap: map[string]interface{}{
199-
"inputs": []map[string]interface{}{{"data_stream.namespace": "+isitok"}},
200-
},
201-
result: ErrInvalidNamespace,
202-
},
203151
{
204152
name: "namespace invalid - long",
205153
configMap: map[string]interface{}{
@@ -274,6 +222,33 @@ func TestStreamCheck(t *testing.T) {
274222
},
275223
result: nil,
276224
},
225+
{
226+
name: "type invalid prefix _",
227+
configMap: map[string]interface{}{
228+
"inputs": []map[string]interface{}{
229+
{"data_stream.type": "_type"},
230+
},
231+
},
232+
result: ErrInvalidIndex,
233+
},
234+
{
235+
name: "type invalid prefix -",
236+
configMap: map[string]interface{}{
237+
"inputs": []map[string]interface{}{
238+
{"data_stream.type": "-type"},
239+
},
240+
},
241+
result: ErrInvalidIndex,
242+
},
243+
{
244+
name: "type invalid prefix +",
245+
configMap: map[string]interface{}{
246+
"inputs": []map[string]interface{}{
247+
{"data_stream.type": "+type"},
248+
},
249+
},
250+
result: ErrInvalidIndex,
251+
},
277252
}
278253

279254
log, err := logger.New("")

0 commit comments

Comments
 (0)