if err != nil {
logger.Errorf("unable to print debug expression for '%s': %s", expression.Str, err)
}
- logger.Debugf(" %s = '%s'", expression.Str, debug)
+ logger.Debugf(" %s = '%v'", expression.Str, debug)
}
}
package parser
import (
- "plugin"
- "time"
-
"github.com/crowdsecurity/crowdsec/pkg/types"
log "github.com/sirupsen/logrus"
)
type InitFunc func(map[string]string) (interface{}, error)
type EnricherCtx struct {
- Funcs map[string]EnrichFunc
- Init InitFunc
- Plugin *plugin.Plugin //pointer to the actual plugin
+ Registered map[string]*Enricher
+}
+
+type Enricher struct {
Name string
- Path string //path to .so ?
- RuntimeCtx interface{} //the internal context of plugin, given back over every call
- initiated bool
+ InitFunc InitFunc
+ EnrichFunc EnrichFunc
+ Ctx interface{}
}
/* mimic plugin loading */
-// TODO fix this shit with real plugin loading
-func Loadplugin(path string) ([]EnricherCtx, error) {
- var err error
+func Loadplugin(path string) (EnricherCtx, error) {
+ enricherCtx := EnricherCtx{}
+ enricherCtx.Registered = make(map[string]*Enricher)
- c := EnricherCtx{}
- c.Name = path
- c.Path = path
- /* we don't want to deal with plugin loading for now :p */
- c.Funcs = map[string]EnrichFunc{
- "GeoIpASN": GeoIpASN,
- "GeoIpCity": GeoIpCity,
- "reverse_dns": reverse_dns,
- "ParseDate": ParseDate,
- "IpToRange": IpToRange,
- }
- c.Init = GeoIpInit
+ enricherConfig := map[string]string{"datadir": path}
- c.RuntimeCtx, err = c.Init(map[string]string{"datadir": path})
- if err != nil {
- log.Warningf("load (fake) plugin load : %v", err)
- c.initiated = false
+ EnrichersList := []*Enricher{
+ {
+ Name: "GeoIpCity",
+ InitFunc: GeoIPCityInit,
+ EnrichFunc: GeoIpCity,
+ },
+ {
+ Name: "GeoIpASN",
+ InitFunc: GeoIPASNInit,
+ EnrichFunc: GeoIpASN,
+ },
+ {
+ Name: "IpToRange",
+ InitFunc: IpToRangeInit,
+ EnrichFunc: IpToRange,
+ },
+ {
+ Name: "reverse_dns",
+ InitFunc: reverseDNSInit,
+ EnrichFunc: reverse_dns,
+ },
+ {
+ Name: "ParseDate",
+ InitFunc: parseDateInit,
+ EnrichFunc: ParseDate,
+ },
}
- c.initiated = true
- return []EnricherCtx{c}, nil
-}
-func GenDateParse(date string) (string, time.Time) {
- var retstr string
- var layouts = [...]string{
- time.RFC3339,
- "02/Jan/2006:15:04:05 -0700",
- "Mon Jan 2 15:04:05 2006",
- "02-Jan-2006 15:04:05 europe/paris",
- "01/02/2006 15:04:05",
- "2006-01-02 15:04:05.999999999 -0700 MST",
- //Jan 5 06:25:11
- "Jan 2 15:04:05",
- "Mon Jan 02 15:04:05.000000 2006",
- "2006-01-02T15:04:05Z07:00",
- "2006/01/02",
- "2006/01/02 15:04",
- "2006-01-02",
- "2006-01-02 15:04",
- }
-
- for _, dateFormat := range layouts {
- t, err := time.Parse(dateFormat, date)
- if err == nil && !t.IsZero() {
- //if the year isn't set, set it to current date :)
- if t.Year() == 0 {
- t = t.AddDate(time.Now().Year(), 0, 0)
- }
- retstr, err := t.MarshalText()
- if err != nil {
- log.Warningf("Failed marshaling '%v'", t)
- continue
- }
- return string(retstr), t
+ for _, enricher := range EnrichersList {
+ log.Debugf("Initiating enricher '%s'", enricher.Name)
+ pluginCtx, err := enricher.InitFunc(enricherConfig)
+ if err != nil {
+ log.Errorf("unable to register plugin '%s': %v", enricher.Name, err)
+ continue
}
+ enricher.Ctx = pluginCtx
+ log.Infof("Successfully registered enricher '%s'", enricher.Name)
+ enricherCtx.Registered[enricher.Name] = enricher
}
- return retstr, time.Time{}
-}
-
-func ParseDate(in string, p *types.Event, x interface{}) (map[string]string, error) {
- var ret map[string]string = make(map[string]string)
-
- tstr, tbin := GenDateParse(in)
- if !tbin.IsZero() {
- ret["MarshaledTime"] = string(tstr)
- return ret, nil
- }
- return nil, nil
+ return enricherCtx, nil
}
--- /dev/null
+package parser
+
+import (
+ "time"
+
+ "github.com/crowdsecurity/crowdsec/pkg/types"
+ log "github.com/sirupsen/logrus"
+)
+
+func GenDateParse(date string) (string, time.Time) {
+ var (
+ layouts = [...]string{
+ time.RFC3339,
+ "02/Jan/2006:15:04:05 -0700",
+ "Mon Jan 2 15:04:05 2006",
+ "02-Jan-2006 15:04:05 europe/paris",
+ "01/02/2006 15:04:05",
+ "2006-01-02 15:04:05.999999999 -0700 MST",
+ "Jan 2 15:04:05",
+ "Mon Jan 02 15:04:05.000000 2006",
+ "2006-01-02T15:04:05Z07:00",
+ "2006/01/02",
+ "2006/01/02 15:04",
+ "2006-01-02",
+ "2006-01-02 15:04",
+ "2006/01/02 15:04:05",
+ "2006-01-02 15:04:05",
+ }
+ )
+
+ for _, dateFormat := range layouts {
+ t, err := time.Parse(dateFormat, date)
+ if err == nil && !t.IsZero() {
+ //if the year isn't set, set it to current date :)
+ if t.Year() == 0 {
+ t = t.AddDate(time.Now().Year(), 0, 0)
+ }
+ retstr, err := t.MarshalText()
+ if err != nil {
+ log.Warningf("Failed marshaling '%v'", t)
+ continue
+ }
+ return string(retstr), t
+ }
+ }
+
+ now := time.Now()
+ retstr, err := now.MarshalText()
+ if err != nil {
+ log.Warningf("Failed marshaling current time")
+ return "", time.Time{}
+ }
+ return string(retstr), now
+}
+
+func ParseDate(in string, p *types.Event, x interface{}) (map[string]string, error) {
+
+ var ret map[string]string = make(map[string]string)
+ tstr, tbin := GenDateParse(in)
+ if !tbin.IsZero() {
+ ret["MarshaledTime"] = string(tstr)
+ return ret, nil
+ }
+
+ return nil, nil
+}
+
+func parseDateInit(cfg map[string]string) (interface{}, error) {
+ return nil, nil
+}
ret["reverse_dns"] = rets[0]
return ret, nil
}
+
+func reverseDNSInit(cfg map[string]string) (interface{}, error) {
+ return nil, nil
+}
//"github.com/crowdsecurity/crowdsec/pkg/parser"
)
-type GeoIpEnricherCtx struct {
- dbc *geoip2.Reader
- dba *geoip2.Reader
- dbraw *maxminddb.Reader
-}
-
-/* All plugins must export a list of function pointers for exported symbols */
-var ExportedFuncs = []string{"GeoIpASN", "GeoIpCity"}
-
func IpToRange(field string, p *types.Event, ctx interface{}) (map[string]string, error) {
var dummy interface{}
ret := make(map[string]string)
log.Infof("Can't parse ip %s, no range enrich", field)
return nil, nil
}
- net, ok, err := ctx.(GeoIpEnricherCtx).dbraw.LookupNetwork(ip, &dummy)
+ net, ok, err := ctx.(*maxminddb.Reader).LookupNetwork(ip, &dummy)
if err != nil {
log.Errorf("Failed to fetch network for %s : %v", ip.String(), err)
return nil, nil
log.Infof("Can't parse ip %s, no ASN enrich", ip)
return nil, nil
}
- record, err := ctx.(GeoIpEnricherCtx).dba.ASN(ip)
+ record, err := ctx.(*geoip2.Reader).ASN(ip)
if err != nil {
log.Errorf("Unable to enrich ip '%s'", field)
return nil, nil
}
ret["ASNNumber"] = fmt.Sprintf("%d", record.AutonomousSystemNumber)
ret["ASNOrg"] = record.AutonomousSystemOrganization
+
log.Tracef("geoip ASN %s -> %s, %s", field, ret["ASNNumber"], ret["ASNOrg"])
+
return ret, nil
}
log.Infof("Can't parse ip %s, no City enrich", ip)
return nil, nil
}
- record, err := ctx.(GeoIpEnricherCtx).dbc.City(ip)
+ record, err := ctx.(*geoip2.Reader).City(ip)
if err != nil {
log.Debugf("Unable to enrich ip '%s'", ip)
return nil, nil
return ret, nil
}
-/* All plugins must export an Init function */
-func GeoIpInit(cfg map[string]string) (interface{}, error) {
- var ctx GeoIpEnricherCtx
- var err error
- ctx.dbc, err = geoip2.Open(cfg["datadir"] + "/GeoLite2-City.mmdb")
+func GeoIPCityInit(cfg map[string]string) (interface{}, error) {
+ dbCityReader, err := geoip2.Open(cfg["datadir"] + "/GeoLite2-City.mmdb")
if err != nil {
log.Debugf("couldn't open geoip : %v", err)
return nil, err
}
- ctx.dba, err = geoip2.Open(cfg["datadir"] + "/GeoLite2-ASN.mmdb")
+
+ return dbCityReader, nil
+}
+
+func GeoIPASNInit(cfg map[string]string) (interface{}, error) {
+ dbASReader, err := geoip2.Open(cfg["datadir"] + "/GeoLite2-ASN.mmdb")
if err != nil {
log.Debugf("couldn't open geoip : %v", err)
return nil, err
}
- ctx.dbraw, err = maxminddb.Open(cfg["datadir"] + "/GeoLite2-ASN.mmdb")
+ return dbASReader, nil
+}
+
+func IpToRangeInit(cfg map[string]string) (interface{}, error) {
+ ipToRangeReader, err := maxminddb.Open(cfg["datadir"] + "/GeoLite2-ASN.mmdb")
if err != nil {
log.Debugf("couldn't open geoip : %v", err)
return nil, err
}
- return ctx, nil
+ return ipToRangeReader, nil
}
//If node has leafs, execute all of them until one asks for a 'break'
LeavesNodes []Node `yaml:"nodes,omitempty"`
//Flag used to describe when to 'break' or return an 'error'
- EnrichFunctions []EnricherCtx
+ EnrichFunctions EnricherCtx
/* If the node is actually a leaf, it can have : grok, enrich, statics */
//pattern_syntax are named grok patterns that are re-utilised over several grok patterns
Data []*types.DataSource `yaml:"data,omitempty"`
}
-func (n *Node) validate(pctx *UnixParserCtx, ectx []EnricherCtx) error {
+func (n *Node) validate(pctx *UnixParserCtx, ectx EnricherCtx) error {
//stage is being set automagically
if n.Stage == "" {
if static.ExpValue == "" {
return fmt.Errorf("static %d : when method is set, expression must be present", idx)
}
- method_found := false
- for _, enricherCtx := range ectx {
- if _, ok := enricherCtx.Funcs[static.Method]; ok && enricherCtx.initiated {
- method_found = true
- break
- }
- }
- if !method_found {
- return fmt.Errorf("the method '%s' doesn't exist or the plugin has not been initialized", static.Method)
+ if _, ok := ectx.Registered[static.Method]; !ok {
+ log.Warningf("the method '%s' doesn't exist or the plugin has not been initialized", static.Method)
}
} else {
if static.Meta == "" && static.Parsed == "" && static.TargetByName == "" {
return NodeState, nil
}
-func (n *Node) compile(pctx *UnixParserCtx, ectx []EnricherCtx) error {
+func (n *Node) compile(pctx *UnixParserCtx, ectx EnricherCtx) error {
var err error
var valid bool
//{&Node{Debug: true, Grok: []GrokPattern{ GrokPattern{}, }}, false},
}
for idx := range CfgTests {
- err := CfgTests[idx].NodeCfg.compile(pctx, []EnricherCtx{})
+ err := CfgTests[idx].NodeCfg.compile(pctx, EnricherCtx{})
if CfgTests[idx].Compiles == true && err != nil {
t.Fatalf("Compile: (%d/%d) expected valid, got : %s", idx+1, len(CfgTests), err)
}
t.Fatalf("Compile: (%d/%d) expected errror", idx+1, len(CfgTests))
}
- err = CfgTests[idx].NodeCfg.validate(pctx, []EnricherCtx{})
+ err = CfgTests[idx].NodeCfg.validate(pctx, EnricherCtx{})
if CfgTests[idx].Valid == true && err != nil {
t.Fatalf("Valid: (%d/%d) expected valid, got : %s", idx+1, len(CfgTests), err)
}
}
}
-func testOneParser(pctx *UnixParserCtx, ectx []EnricherCtx, dir string, b *testing.B) error {
+func testOneParser(pctx *UnixParserCtx, ectx EnricherCtx, dir string, b *testing.B) error {
var (
err error
}
//prepTests is going to do the initialisation of parser : it's going to load enrichment plugins and load the patterns. This is done here so that we don't redo it for each test
-func prepTests() (*UnixParserCtx, []EnricherCtx, error) {
+func prepTests() (*UnixParserCtx, EnricherCtx, error) {
var (
err error
pctx *UnixParserCtx
- ectx []EnricherCtx
+ ectx EnricherCtx
)
err = exprhelpers.Init()
// Init the parser
pctx, err = Init(map[string]interface{}{"patterns": cfgdir + string("/patterns/"), "data": "./tests/"})
if err != nil {
- return nil, nil, fmt.Errorf("failed to initialize parser : %v", err)
+ return nil, ectx, fmt.Errorf("failed to initialize parser : %v", err)
}
return pctx, ectx, nil
}
if static.Method != "" {
processed := false
/*still way too hackish, but : inject all the results in enriched, and */
- for _, x := range n.EnrichFunctions {
- if fptr, ok := x.Funcs[static.Method]; ok && x.initiated {
- clog.Tracef("Found method '%s'", static.Method)
- ret, err := fptr(value, event, x.RuntimeCtx)
- if err != nil {
- clog.Fatalf("plugin function error : %v", err)
- }
- processed = true
- clog.Debugf("+ Method %s('%s') returned %d entries to merge in .Enriched\n", static.Method, value, len(ret))
- if len(ret) == 0 {
- clog.Debugf("+ Method '%s' empty response on '%s'", static.Method, value)
- }
- for k, v := range ret {
- clog.Debugf("\t.Enriched[%s] = '%s'\n", k, v)
- event.Enriched[k] = v
- }
- break
- } else {
- clog.Warningf("method '%s' doesn't exist or plugin not initialized", static.Method)
+ if enricherPlugin, ok := n.EnrichFunctions.Registered[static.Method]; ok {
+ clog.Tracef("Found method '%s'", static.Method)
+ ret, err := enricherPlugin.EnrichFunc(value, event, enricherPlugin.Ctx)
+ if err != nil {
+ clog.Errorf("method '%s' returned an error : %v", static.Method, err)
}
+ processed = true
+ clog.Debugf("+ Method %s('%s') returned %d entries to merge in .Enriched\n", static.Method, value, len(ret))
+ if len(ret) == 0 {
+ clog.Debugf("+ Method '%s' empty response on '%s'", static.Method, value)
+ }
+ for k, v := range ret {
+ clog.Debugf("\t.Enriched[%s] = '%s'\n", k, v)
+ event.Enriched[k] = v
+ }
+ } else {
+ clog.Debugf("method '%s' doesn't exist or plugin not initialized", static.Method)
}
if !processed {
- clog.Warningf("method '%s' doesn't exist", static.Method)
+ clog.Debugf("method '%s' doesn't exist", static.Method)
}
} else if static.Parsed != "" {
clog.Debugf(".Parsed[%s] = '%s'", static.Parsed, value)
Stage string `yaml:"stage"`
}
-func LoadStages(stageFiles []Stagefile, pctx *UnixParserCtx, ectx []EnricherCtx) ([]Node, error) {
+func LoadStages(stageFiles []Stagefile, pctx *UnixParserCtx, ectx EnricherCtx) ([]Node, error) {
var nodes []Node
tmpstages := make(map[string]bool)
pctx.Stages = []string{}
PovfwStageFiles []Stagefile
Nodes []Node
Povfwnodes []Node
- EnricherCtx []EnricherCtx
+ EnricherCtx EnricherCtx
}
func Init(c map[string]interface{}) (*UnixParserCtx, error) {
projects / crowdsec.git / commitdiff