Skip to content

Commit c0d7f6a

Browse files
mmetcmmetc
authored
refact pkg/parser: extract+embed NodeConfig in Node struct (#4343)
1 parent e487da9 commit c0d7f6a

File tree

6 files changed

+194
-69
lines changed

6 files changed

+194
-69
lines changed

‎.golangci.yml‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -123,7 +123,8 @@ linters:
123123
- '!**/pkg/csplugin/broker.go'
124124
- '!**/pkg/leakybucket/bucketstore_test.go'
125125
- '!**/pkg/leakybucket/manager_load.go'
126-
- '!**/pkg/parser/node.go'
126+
- '!**/pkg/parser/node_config.go'
127+
- '!**/pkg/parser/node_subnodes_test.go'
127128
- '!**/pkg/parser/node_test.go'
128129
- '!**/pkg/parser/parsing_test.go'
129130
- '!**/pkg/parser/stage.go'
@@ -658,6 +659,12 @@ linters:
658659
- embeddedstructfieldcheck
659660
text: "embedded fields should be listed before regular fields"
660661

662+
# TODO: remove when NodeConfig is not embedded anymore
663+
- linters:
664+
- staticcheck
665+
path: pkg/parser/(.+).go
666+
text: 'QF1008: could remove embedded field "NodeConfig" from selector'
667+
661668
paths:
662669
- third_party$
663670
- builtin$

‎pkg/hubtest/hubtest_item.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -210,13 +210,13 @@ func (t *HubTestItem) InstallHub(ctx context.Context) error {
210210
}
211211

212212
if len(t.Config.OverrideStatics) > 0 {
213-
n := parser.Node{
213+
cfg := parser.NodeConfig{
214214
Name: "overrides",
215215
Filter: "1==1",
216216
Statics: t.Config.OverrideStatics,
217217
}
218218

219-
b, err := yaml.Marshal(n)
219+
b, err := yaml.Marshal(cfg)
220220
if err != nil {
221221
return fmt.Errorf("unable to serialize overrides: %w", err)
222222
}

‎pkg/parser/node.go‎

Lines changed: 28 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -12,62 +12,57 @@ import (
1212
"github.com/expr-lang/expr/vm"
1313
"github.com/prometheus/client_golang/prometheus"
1414
log "github.com/sirupsen/logrus"
15-
yaml "gopkg.in/yaml.v2"
1615

1716
"github.com/crowdsecurity/grokky"
1817

19-
"github.com/crowdsecurity/crowdsec/pkg/enrichment"
2018
"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
2119
"github.com/crowdsecurity/crowdsec/pkg/logging"
2220
"github.com/crowdsecurity/crowdsec/pkg/metrics"
2321
"github.com/crowdsecurity/crowdsec/pkg/pipeline"
2422
)
2523

2624
type Node struct {
27-
FormatVersion string `yaml:"format"`
28-
// Enable config + runtime debug of node via config o/
29-
Debug bool `yaml:"debug,omitempty"`
30-
// If enabled, the node (and its child) will report their own statistics
31-
Profiling bool `yaml:"profiling,omitempty"`
32-
// Name, author, description and reference(s) for parser pattern
33-
Name string `yaml:"name,omitempty"`
34-
Author string `yaml:"author,omitempty"`
35-
Description string `yaml:"description,omitempty"`
36-
References []string `yaml:"references,omitempty"`
25+
NodeConfig `yaml:",inline"`
3726
// if debug is present in the node, keep its specific Logger in runtime structure
3827
Logger *log.Entry `yaml:"-"`
39-
// This is mostly a hack to make writing less repetitive.
40-
// relying on stage, we know which field to parse, and we
41-
// can also promote log to next stage on success
42-
Stage string `yaml:"stage,omitempty"`
43-
// OnSuccess allows to tag a node to be able to move log to next stage on success
44-
OnSuccess string `yaml:"onsuccess,omitempty"`
4528
rn string // this is only for us in debug, a random generated name for each node
4629
// Filter is executed at runtime (with current log line as context)
4730
// and must succeed or node is exited
48-
Filter string `yaml:"filter,omitempty"`
4931
RunTimeFilter *vm.Program `yaml:"-"` // the actual compiled filter
5032
// If node has leafs, execute all of them until one asks for a 'break'
51-
LeavesNodes []Node `yaml:"nodes,omitempty"`
33+
LeavesNodes []Node `yaml:"-"`
5234
// Flag used to describe when to 'break' or return an 'error'
5335
EnrichFunctions EnricherCtx
5436

55-
/* If the node is actually a leaf, it can have : grok, enrich, statics */
56-
// pattern_syntax are named grok patterns that are re-utilized over several grok patterns
57-
SubGroks yaml.MapSlice `yaml:"pattern_syntax,omitempty"`
58-
59-
// Holds a grok pattern
60-
Grok GrokPattern `yaml:"grok,omitempty"`
6137
RuntimeGrok RuntimeGrokPattern `yaml:"-"`
62-
// Statics can be present in any type of node and is executed last
63-
Statics []Static `yaml:"statics,omitempty"`
6438
RuntimeStatics []RuntimeStatic `yaml:"-"`
65-
// Stash allows to capture data from the log line and store it in an accessible cache
66-
Stashes []Stash `yaml:"stash,omitempty"`
6739
RuntimeStashes []RuntimeStash `yaml:"-"`
68-
// Whitelists
69-
Whitelist Whitelist `yaml:"whitelist,omitempty"`
70-
Data []*enrichment.DataProvider `yaml:"data,omitempty"`
40+
}
41+
42+
func (n *Node) UnmarshalYAML(unmarshal func(any) error) error {
43+
var cfg NodeConfig
44+
if err := unmarshal(&cfg); err != nil {
45+
return err
46+
}
47+
48+
// Reset node and assign config.
49+
*n = Node{NodeConfig: cfg}
50+
n.initRuntimeChildrenFromConfig()
51+
return nil
52+
}
53+
54+
func (n *Node) initRuntimeChildrenFromConfig() {
55+
subNodes := n.NodeConfig.SubNodes
56+
if len(subNodes) == 0 {
57+
n.LeavesNodes = nil
58+
return
59+
}
60+
n.LeavesNodes = make([]Node, len(subNodes))
61+
for i := range subNodes {
62+
child := Node{NodeConfig: subNodes[i]}
63+
child.initRuntimeChildrenFromConfig()
64+
n.LeavesNodes[i] = child
65+
}
7166
}
7267

7368
func (n *Node) validate(ectx EnricherCtx) error {

‎pkg/parser/node_config.go‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
package parser
2+
3+
import (
4+
yaml "gopkg.in/yaml.v2"
5+
6+
"github.com/crowdsecurity/crowdsec/pkg/enrichment"
7+
)
8+
9+
// NodeConfig is the YAML shape of a parser node.
10+
type NodeConfig struct {
11+
FormatVersion string `yaml:"format"`
12+
// Enable config + runtime debug of node via config o/
13+
Debug bool `yaml:"debug,omitempty"`
14+
// If enabled, the node (and its child) will report their own statistics
15+
Profiling bool `yaml:"profiling,omitempty"`
16+
// Name, author, description and reference(s) for parser pattern
17+
Name string `yaml:"name,omitempty"`
18+
Author string `yaml:"author,omitempty"`
19+
Description string `yaml:"description,omitempty"`
20+
References []string `yaml:"references,omitempty"`
21+
// This is mostly a hack to make writing less repetitive.
22+
// relying on stage, we know which field to parse, and we
23+
// can also promote log to next stage on success
24+
Stage string `yaml:"stage,omitempty"`
25+
// OnSuccess allows to tag a node to be able to move log to next stage on success
26+
OnSuccess string `yaml:"onsuccess,omitempty"`
27+
Filter string `yaml:"filter,omitempty"`
28+
29+
SubNodes []NodeConfig `yaml:"nodes,omitempty"`
30+
31+
/* If the node is actually a leaf, it can have : grok, enrich, statics */
32+
// pattern_syntax are named grok patterns that are re-utilized over several grok patterns
33+
SubGroks yaml.MapSlice `yaml:"pattern_syntax,omitempty"`
34+
35+
// Holds a grok pattern
36+
Grok GrokPattern `yaml:"grok,omitempty"`
37+
// Statics can be present in any type of node and is executed last
38+
Statics []Static `yaml:"statics,omitempty"`
39+
// Stash allows to capture data from the log line and store it in an accessible cache
40+
Stashes []Stash `yaml:"stash,omitempty"`
41+
Whitelist Whitelist `yaml:"whitelist,omitempty"`
42+
Data []*enrichment.DataProvider `yaml:"data,omitempty"`
43+
}

‎pkg/parser/node_subnodes_test.go‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
package parser
2+
3+
import (
4+
"testing"
5+
6+
"github.com/stretchr/testify/assert"
7+
"github.com/stretchr/testify/require"
8+
yaml "gopkg.in/yaml.v2"
9+
)
10+
11+
func mustUnmarshalNode(t *testing.T, raw string) Node {
12+
t.Helper()
13+
14+
var n Node
15+
require.NoError(t, yaml.Unmarshal([]byte(raw), &n))
16+
return n
17+
}
18+
19+
func mustUnmarshalNodeConfig(t *testing.T, raw string) NodeConfig {
20+
t.Helper()
21+
22+
var cfg NodeConfig
23+
require.NoError(t, yaml.Unmarshal([]byte(raw), &cfg))
24+
return cfg
25+
}
26+
27+
func assertNames(t *testing.T, nodes []Node, want ...string) {
28+
t.Helper()
29+
30+
require.Len(t, nodes, len(want))
31+
for i, w := range want {
32+
assert.Equal(t, w, nodes[i].Name, "node[%d].Name", i)
33+
}
34+
}
35+
36+
func TestNodeUnmarshalYAML_PopulatesConfigNodesAndRuntimeLeaves(t *testing.T) {
37+
raw := `
38+
name: root
39+
stage: s00-raw
40+
nodes:
41+
- name: child1
42+
stage: s01-parse
43+
filter: "evt.Line.Labels.type == 'a'"
44+
- name: child2
45+
stage: s02-enrich
46+
nodes:
47+
- name: grandchild
48+
stage: s03-final
49+
`
50+
51+
n := mustUnmarshalNode(t, raw)
52+
53+
// Root config
54+
assert.Equal(t, "root", n.Name)
55+
require.Len(t, n.NodeConfig.SubNodes, 2)
56+
57+
// Runtime mirror
58+
assertNames(t, n.LeavesNodes, "child1", "child2")
59+
assert.Equal(t, "evt.Line.Labels.type == 'a'", n.LeavesNodes[0].Filter)
60+
61+
// Nested: child2 -> grandchild (config + runtime mirror)
62+
child2 := n.LeavesNodes[1]
63+
require.Len(t, child2.NodeConfig.SubNodes, 1)
64+
assertNames(t, child2.LeavesNodes, "grandchild")
65+
}
66+
67+
func TestNodeConfigUnmarshalYAML_PopulatesNodes(t *testing.T) {
68+
raw := `
69+
name: root
70+
nodes:
71+
- name: child
72+
`
73+
74+
cfg := mustUnmarshalNodeConfig(t, raw)
75+
76+
assert.Equal(t, "root", cfg.Name)
77+
require.Len(t, cfg.SubNodes, 1)
78+
assert.Equal(t, "child", cfg.SubNodes[0].Name)
79+
}

‎pkg/parser/node_test.go‎

Lines changed: 34 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -12,58 +12,59 @@ func TestParserConfigs(t *testing.T) {
1212
t.Fatalf("unable to load patterns : %s", err)
1313
}
1414

15-
/*the actual tests*/
15+
// the actual tests
1616
var CfgTests = []struct {
17-
NodeCfg *Node
18-
Compiles bool
19-
Valid bool
17+
cfg NodeConfig
18+
compiles bool
19+
valid bool
2020
}{
21-
//valid node with grok pattern
22-
{&Node{Debug: true, Stage: "s00", Grok: GrokPattern{RegexpValue: "^x%{DATA:extr}$", TargetField: "t"}}, true, true},
23-
//bad filter
24-
{&Node{Debug: true, Stage: "s00", Filter: "ratata"}, false, false},
25-
//empty node
26-
{&Node{Debug: true, Stage: "s00", Filter: "true"}, false, false},
27-
//bad subgrok
28-
{&Node{Debug: true, Stage: "s00", SubGroks: yaml.MapSlice{{Key: string("FOOBAR"), Value: string("[a-$")}}}, false, true},
29-
//valid node with grok pattern
30-
{&Node{Debug: true, Stage: "s00", SubGroks: yaml.MapSlice{{Key: string("FOOBAR"), Value: string("[a-z]")}}, Grok: GrokPattern{RegexpValue: "^x%{FOOBAR:extr}$", TargetField: "t"}}, true, true},
31-
//bad node success
32-
{&Node{Debug: true, Stage: "s00", OnSuccess: "ratat", Grok: GrokPattern{RegexpValue: "^x%{DATA:extr}$", TargetField: "t"}}, false, false},
33-
//ok node success
34-
{&Node{Debug: true, Stage: "s00", OnSuccess: "continue", Grok: GrokPattern{RegexpValue: "^x%{DATA:extr}$", TargetField: "t"}}, true, true},
35-
//valid node with grok sub-pattern used by name
36-
{&Node{Debug: true, Stage: "s00", SubGroks: yaml.MapSlice{{Key: string("FOOBARx"), Value: string("[a-z] %{DATA:lol}$")}}, Grok: GrokPattern{RegexpName: "FOOBARx", TargetField: "t"}}, true, true},
37-
//node with unexisting grok pattern
38-
{&Node{Debug: true, Stage: "s00", Grok: GrokPattern{RegexpName: "RATATA", TargetField: "t"}}, false, true},
39-
//node with grok pattern dependencies
40-
{&Node{Debug: true, Stage: "s00", SubGroks: yaml.MapSlice{
21+
// valid node with grok pattern
22+
{NodeConfig{Debug: true, Stage: "s00", Grok: GrokPattern{RegexpValue: "^x%{DATA:extr}$", TargetField: "t"}}, true, true},
23+
// bad filter
24+
{NodeConfig{Debug: true, Stage: "s00", Filter: "ratata"}, false, false},
25+
// empty node
26+
{NodeConfig{Debug: true, Stage: "s00", Filter: "true"}, false, false},
27+
// bad subgrok
28+
{NodeConfig{Debug: true, Stage: "s00", SubGroks: yaml.MapSlice{{Key: string("FOOBAR"), Value: string("[a-$")}}}, false, true},
29+
// valid node with grok pattern
30+
{NodeConfig{Debug: true, Stage: "s00", SubGroks: yaml.MapSlice{{Key: string("FOOBAR"), Value: string("[a-z]")}}, Grok: GrokPattern{RegexpValue: "^x%{FOOBAR:extr}$", TargetField: "t"}}, true, true},
31+
// bad node success
32+
{NodeConfig{Debug: true, Stage: "s00", OnSuccess: "ratat", Grok: GrokPattern{RegexpValue: "^x%{DATA:extr}$", TargetField: "t"}}, false, false},
33+
// ok node success
34+
{NodeConfig{Debug: true, Stage: "s00", OnSuccess: "continue", Grok: GrokPattern{RegexpValue: "^x%{DATA:extr}$", TargetField: "t"}}, true, true},
35+
// valid node with grok sub-pattern used by name
36+
{NodeConfig{Debug: true, Stage: "s00", SubGroks: yaml.MapSlice{{Key: string("FOOBARx"), Value: string("[a-z] %{DATA:lol}$")}}, Grok: GrokPattern{RegexpName: "FOOBARx", TargetField: "t"}}, true, true},
37+
// node with unexisting grok pattern
38+
{NodeConfig{Debug: true, Stage: "s00", Grok: GrokPattern{RegexpName: "RATATA", TargetField: "t"}}, false, true},
39+
// node with grok pattern dependencies
40+
{NodeConfig{Debug: true, Stage: "s00", SubGroks: yaml.MapSlice{
4141
{Key: string("SUBGROK"), Value: string("[a-z]")},
4242
{Key: string("MYGROK"), Value: string("[a-z]%{SUBGROK}")},
4343
}, Grok: GrokPattern{RegexpValue: "^x%{MYGROK:extr}$", TargetField: "t"}}, true, true},
44-
//node with broken grok pattern dependencies
45-
{&Node{Debug: true, Stage: "s00", SubGroks: yaml.MapSlice{
44+
// node with broken grok pattern dependencies
45+
{NodeConfig{Debug: true, Stage: "s00", SubGroks: yaml.MapSlice{
4646
{Key: string("SUBGROKBIS"), Value: string("[a-z]%{MYGROKBIS}")},
4747
{Key: string("MYGROKBIS"), Value: string("[a-z]")},
4848
}, Grok: GrokPattern{RegexpValue: "^x%{MYGROKBIS:extr}$", TargetField: "t"}}, false, true},
4949
}
5050

51-
for idx := range CfgTests {
52-
err := CfgTests[idx].NodeCfg.compile(pctx, EnricherCtx{})
53-
if CfgTests[idx].Compiles && err != nil {
51+
for idx, tc := range CfgTests {
52+
node := &Node{NodeConfig: tc.cfg}
53+
err := node.compile(pctx, EnricherCtx{})
54+
if tc.compiles && err != nil {
5455
t.Fatalf("Compile: (%d/%d) expected valid, got : %s", idx+1, len(CfgTests), err)
5556
}
5657

57-
if !CfgTests[idx].Compiles && err == nil {
58+
if !tc.compiles && err == nil {
5859
t.Fatalf("Compile: (%d/%d) expected error", idx+1, len(CfgTests))
5960
}
6061

61-
err = CfgTests[idx].NodeCfg.validate(EnricherCtx{})
62-
if CfgTests[idx].Valid && err != nil {
62+
err = node.validate(EnricherCtx{})
63+
if tc.valid && err != nil {
6364
t.Fatalf("Valid: (%d/%d) expected valid, got : %s", idx+1, len(CfgTests), err)
6465
}
6566

66-
if !CfgTests[idx].Valid && err == nil {
67+
if !tc.valid && err == nil {
6768
t.Fatalf("Valid: (%d/%d) expected error", idx+1, len(CfgTests))
6869
}
6970
}

0 commit comments

Comments
 (0)