From 7da54800ad44cb2bbd00ac34e7f2a2fe7978884b Mon Sep 17 00:00:00 2001 From: Steve McDaniel Date: Mon, 8 May 2023 12:16:02 -0600 Subject: [PATCH 1/4] Added caching for dotpathtoslice Signed-off-by: Steve McDaniel --- go.mod | 2 +- internal/bloblang/parser/mapping_parser.go | 21 +++++++- internal/bloblang/query/functions.go | 51 ++++++++++++++++--- internal/bloblang/query/methods.go | 13 ++++- internal/bloblang/query/methods_structured.go | 41 +++++++++++++-- 5 files changed, 116 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index 885f5b71ac..2ca0768ddd 100644 --- a/go.mod +++ b/go.mod @@ -55,6 +55,7 @@ require ( github.com/gorilla/mux v1.8.0 github.com/gorilla/websocket v1.5.0 github.com/gosimple/slug v1.13.1 + github.com/hashicorp/golang-lru v0.5.4 github.com/hashicorp/golang-lru/v2 v2.0.1 github.com/influxdata/go-syslog/v3 v3.0.0 github.com/influxdata/influxdb1-client v0.0.0-20220302092344-a9ab5670611c @@ -217,7 +218,6 @@ require ( github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect github.com/hashicorp/go-immutable-radix v1.3.0 // indirect github.com/hashicorp/go-uuid v1.0.3 // indirect - github.com/hashicorp/golang-lru v0.5.4 // indirect github.com/imdario/mergo v0.3.13 // indirect github.com/jcmturner/aescts/v2 v2.0.0 // indirect github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect diff --git a/internal/bloblang/parser/mapping_parser.go b/internal/bloblang/parser/mapping_parser.go index 178a67a87c..2e10ba3715 100644 --- a/internal/bloblang/parser/mapping_parser.go +++ b/internal/bloblang/parser/mapping_parser.go @@ -6,11 +6,21 @@ import ( "strings" "github.com/Jeffail/gabs/v2" + lru "github.com/hashicorp/golang-lru/v2" "github.com/benthosdev/benthos/v4/internal/bloblang/mapping" "github.com/benthosdev/benthos/v4/internal/bloblang/query" ) +var dotPathCache *lru.Cache[string, []string] + +func init() { + var err error + if dotPathCache, err = lru.New[string, []string](1024); err != nil { + panic(err) + } +} + // ParseMapping parses a bloblang mapping and returns an executor to run it, or // an error if the parsing fails. // @@ -461,7 +471,16 @@ func pathParser() Func { if sequence[1] != nil { pathParts := sequence[1].([]any)[1].(DelimitedResult).Primary for _, p := range pathParts { - path = append(path, gabs.DotPathToSlice(p.(string))...) + dotPathSlice, ok := dotPathCache.Get(p.(string)) + + if ok { + path = append(path, dotPathSlice...) + } else { + dotPathSlice := gabs.DotPathToSlice(p.(string)) + dotPathCache.Add(p.(string), dotPathSlice) + path = append(path, dotPathSlice...) + } + } } diff --git a/internal/bloblang/query/functions.go b/internal/bloblang/query/functions.go index 8cc441a108..be5a98ce4c 100644 --- a/internal/bloblang/query/functions.go +++ b/internal/bloblang/query/functions.go @@ -11,12 +11,22 @@ import ( "github.com/Jeffail/gabs/v2" "github.com/gofrs/uuid" + lru "github.com/hashicorp/golang-lru/v2" gonanoid "github.com/matoous/go-nanoid/v2" "github.com/segmentio/ksuid" "github.com/benthosdev/benthos/v4/internal/tracing" ) +var dotPathCache *lru.Cache[string, []string] + +func init() { + var err error + if dotPathCache, err = lru.New[string, []string](1024); err != nil { + panic(err) + } +} + type fieldFunction struct { namedContext string fromRoot bool @@ -104,8 +114,14 @@ func (f *fieldFunction) Close(ctx context.Context) error { // return a field from a named context. func NewNamedContextFieldFunction(namedContext, pathStr string) Function { var path []string + var ok bool if len(pathStr) > 0 { - path = gabs.DotPathToSlice(pathStr) + path, ok = dotPathCache.Get(pathStr) + + if !ok { + path = gabs.DotPathToSlice(pathStr) + dotPathCache.Add(pathStr, path) + } } return &fieldFunction{namedContext: namedContext, fromRoot: false, path: path} } @@ -114,8 +130,14 @@ func NewNamedContextFieldFunction(namedContext, pathStr string) Function { // current context. func NewFieldFunction(pathStr string) Function { var path []string + var ok bool if len(pathStr) > 0 { - path = gabs.DotPathToSlice(pathStr) + path, ok = dotPathCache.Get(pathStr) + + if !ok { + path = gabs.DotPathToSlice(pathStr) + dotPathCache.Add(pathStr, path) + } } return &fieldFunction{ path: path, @@ -125,9 +147,17 @@ func NewFieldFunction(pathStr string) Function { // NewRootFieldFunction creates a query function that returns a field from the // root context. func NewRootFieldFunction(pathStr string) Function { - var path []string + var ( + path []string + ok bool + ) if len(pathStr) > 0 { - path = gabs.DotPathToSlice(pathStr) + path, ok = dotPathCache.Get(pathStr) + + if !ok { + path = gabs.DotPathToSlice(pathStr) + dotPathCache.Add(pathStr, path) + } } return &fieldFunction{ fromRoot: true, @@ -440,9 +470,18 @@ func jsonFunction(args *ParsedParams) (Function, error) { if err != nil { return nil, err } - var argPath []string + var ( + argPath []string + ok bool + ) + if len(path) > 0 { - argPath = gabs.DotPathToSlice(path) + argPath, ok = dotPathCache.Get(path) + + if !ok { + argPath = gabs.DotPathToSlice(path) + dotPathCache.Add(path, argPath) + } } return ClosureFunction("json path `"+SliceToDotPath(argPath...)+"`", func(ctx FunctionContext) (any, error) { jPart, err := ctx.MsgBatch.Get(ctx.Index).AsStructured() diff --git a/internal/bloblang/query/methods.go b/internal/bloblang/query/methods.go index 1327df36cf..f5fcb77ed2 100644 --- a/internal/bloblang/query/methods.go +++ b/internal/bloblang/query/methods.go @@ -274,7 +274,18 @@ func (g *getMethod) QueryTargets(ctx TargetsContext) (TargetsContext, []TargetPa // NewGetMethod creates a new get method. func NewGetMethod(target Function, pathStr string) (Function, error) { - path := gabs.DotPathToSlice(pathStr) + var ( + path []string + ok bool + ) + + path, ok = dotPathCache.Get(pathStr) + + if !ok { + path = gabs.DotPathToSlice(pathStr) + dotPathCache.Add(pathStr, path) + } + switch t := target.(type) { case *getMethod: newPath := append([]string{}, t.path...) diff --git a/internal/bloblang/query/methods_structured.go b/internal/bloblang/query/methods_structured.go index 76ad8212a8..fb8e810b11 100644 --- a/internal/bloblang/query/methods_structured.go +++ b/internal/bloblang/query/methods_structured.go @@ -290,7 +290,19 @@ var _ = registerSimpleMethod( if err != nil { return nil, err } - path := gabs.DotPathToSlice(pathStr) + + var ( + path []string + ok bool + ) + + path, ok = dotPathCache.Get(pathStr) + + if !ok { + path = gabs.DotPathToSlice(pathStr) + dotPathCache.Add(pathStr, path) + } + return func(v any, ctx FunctionContext) (any, error) { return gabs.Wrap(v).Exists(path...), nil }, nil @@ -325,7 +337,18 @@ Exploding objects results in an object where the keys match the target object, a if err != nil { return nil, err } - path := gabs.DotPathToSlice(pathRaw) + var ( + path []string + ok bool + ) + + path, ok = dotPathCache.Get(pathRaw) + + if !ok { + path = gabs.DotPathToSlice(pathRaw) + dotPathCache.Add(pathRaw, path) + } + return func(v any, ctx FunctionContext) (any, error) { rootMap, ok := v.(map[string]any) if !ok { @@ -1690,7 +1713,19 @@ If a key within a nested path does not exist or is not an object then it is not if err != nil { return nil, fmt.Errorf("argument %v: %w", i, err) } - excludeList = append(excludeList, gabs.DotPathToSlice(argStr)) + var ( + path []string + ok bool + ) + + path, ok = dotPathCache.Get(argStr) + + if !ok { + path = gabs.DotPathToSlice(argStr) + dotPathCache.Add(argStr, path) + } + + excludeList = append(excludeList, path) } return func(v any, ctx FunctionContext) (any, error) { m, ok := v.(map[string]any) From 1c1718de9b0b65ee94b874202a2618faaa14a7f3 Mon Sep 17 00:00:00 2001 From: Steve McDaniel Date: Mon, 8 May 2023 12:26:56 -0600 Subject: [PATCH 2/4] Add caching to query/mapping_parser Signed-off-by: Steve McDaniel --- internal/bloblang/parser/mapping_parser.go | 25 ++++++++++++++-------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/internal/bloblang/parser/mapping_parser.go b/internal/bloblang/parser/mapping_parser.go index 2e10ba3715..c3f47e14b7 100644 --- a/internal/bloblang/parser/mapping_parser.go +++ b/internal/bloblang/parser/mapping_parser.go @@ -12,7 +12,9 @@ import ( "github.com/benthosdev/benthos/v4/internal/bloblang/query" ) -var dotPathCache *lru.Cache[string, []string] +var ( + dotPathCache *lru.Cache[string, []string] +) func init() { var err error @@ -471,16 +473,21 @@ func pathParser() Func { if sequence[1] != nil { pathParts := sequence[1].([]any)[1].(DelimitedResult).Primary for _, p := range pathParts { - dotPathSlice, ok := dotPathCache.Get(p.(string)) - - if ok { - path = append(path, dotPathSlice...) - } else { - dotPathSlice := gabs.DotPathToSlice(p.(string)) - dotPathCache.Add(p.(string), dotPathSlice) - path = append(path, dotPathSlice...) + var ( + dotPathSlice []string + ok bool + pathString string = p.(string) + ) + + dotPathSlice, ok = dotPathCache.Get(pathString) + + if !ok { + dotPathSlice = gabs.DotPathToSlice(pathString) + dotPathCache.Add(pathString, dotPathSlice) } + path = append(path, dotPathSlice...) + } } From aa6ee4dbed575e45ab8eee6bf831bcdd44b4ce41 Mon Sep 17 00:00:00 2001 From: Steve McDaniel Date: Mon, 8 May 2023 12:29:11 -0600 Subject: [PATCH 3/4] Run Make deps Signed-off-by: Steve McDaniel --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 2ca0768ddd..885f5b71ac 100644 --- a/go.mod +++ b/go.mod @@ -55,7 +55,6 @@ require ( github.com/gorilla/mux v1.8.0 github.com/gorilla/websocket v1.5.0 github.com/gosimple/slug v1.13.1 - github.com/hashicorp/golang-lru v0.5.4 github.com/hashicorp/golang-lru/v2 v2.0.1 github.com/influxdata/go-syslog/v3 v3.0.0 github.com/influxdata/influxdb1-client v0.0.0-20220302092344-a9ab5670611c @@ -218,6 +217,7 @@ require ( github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect github.com/hashicorp/go-immutable-radix v1.3.0 // indirect github.com/hashicorp/go-uuid v1.0.3 // indirect + github.com/hashicorp/golang-lru v0.5.4 // indirect github.com/imdario/mergo v0.3.13 // indirect github.com/jcmturner/aescts/v2 v2.0.0 // indirect github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect From 12b5d1fb4929c7b1d610ecbdc062343caeec8692 Mon Sep 17 00:00:00 2001 From: Steve McDaniel Date: Mon, 8 May 2023 12:42:35 -0600 Subject: [PATCH 4/4] lint fix Signed-off-by: Steve McDaniel --- internal/bloblang/parser/mapping_parser.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/bloblang/parser/mapping_parser.go b/internal/bloblang/parser/mapping_parser.go index c3f47e14b7..ef35e9a763 100644 --- a/internal/bloblang/parser/mapping_parser.go +++ b/internal/bloblang/parser/mapping_parser.go @@ -476,9 +476,9 @@ func pathParser() Func { var ( dotPathSlice []string ok bool - pathString string = p.(string) ) + pathString := p.(string) dotPathSlice, ok = dotPathCache.Get(pathString) if !ok {