[PATCH] non-fatal error if some datasource can't be run (i.e. journalctl but systemd...

This on the other hand, gives a new fatal error when there are no valid datasources.
In the previous version, crowdsec kept running with just a warning if no
acquisition yaml or dir were specified.

Gbp-Pq: Name 0018-non-fatal-errors-for-invalid-datasources.patch

diff --git a/cmd/crowdsec/crowdsec.go b/cmd/crowdsec/crowdsec.go
index 84cf0838bf548b2f4d6da4f0af3fbc1978ce3085..a3b095ae0c26a1f29df91423533132e309489eed 100644 (file)
--- a/cmd/crowdsec/crowdsec.go
+++ b/cmd/crowdsec/crowdsec.go
@@ -23,22 +23,22 @@ func initCrowdsec(cConfig *csconfig.Config) (*parser.Parsers, error) {
        var err error
 
        // Populate cwhub package tools
-       if err := cwhub.GetHubIdx(cConfig.Hub); err != nil {
-               return &parser.Parsers{}, fmt.Errorf("Failed to load hub index : %s", err)
+       if err = cwhub.GetHubIdx(cConfig.Hub); err != nil {
+               return &parser.Parsers{}, fmt.Errorf("while loading hub index : %s", err)
        }
 
        // Start loading configs
        csParsers := newParsers()
        if csParsers, err = parser.LoadParsers(cConfig, csParsers); err != nil {
-               return &parser.Parsers{}, fmt.Errorf("Failed to load parsers: %s", err)
+               return nil, fmt.Errorf("while loading parsers: %s", err)
        }
 
        if err := LoadBuckets(cConfig); err != nil {
-               return &parser.Parsers{}, fmt.Errorf("Failed to load scenarios: %s", err)
+               return nil, fmt.Errorf("while loading scenarios: %s", err)
        }
 
        if err := LoadAcquisition(cConfig); err != nil {
-               return &parser.Parsers{}, fmt.Errorf("Error while loading acquisition config : %s", err)
+               return nil, fmt.Errorf("while loading acquisition config: %s", err)
        }
        return csParsers, nil
 }

diff --git a/cmd/crowdsec/main.go b/cmd/crowdsec/main.go
index 43eb63ec482185de8ffca8cbf159653d6e645405..a25c372fa1338240d3c85044d598cea40dc8f349 100644 (file)
--- a/cmd/crowdsec/main.go
+++ b/cmd/crowdsec/main.go
@@ -157,6 +157,10 @@ func LoadAcquisition(cConfig *csconfig.Config) error {
                }
        }
 
+       if len(dataSources) == 0 {
+               return fmt.Errorf("no datasource enabled")
+       }
+
        return nil
 }
 

diff --git a/pkg/acquisition/acquisition.go b/pkg/acquisition/acquisition.go
index db37fc953171f778d5eb5c1768b606ad7ebbb5d5..3ad38e135b569a9b708003760c0a80a42a6f1cb8 100644 (file)
--- a/pkg/acquisition/acquisition.go
+++ b/pkg/acquisition/acquisition.go
@@ -24,6 +24,20 @@ import (
        tomb "gopkg.in/tomb.v2"
 )
 
+type DataSourceUnavailableError struct {
+       Name string
+       Err error
+}
+
+func (e *DataSourceUnavailableError) Error() string {
+       return fmt.Sprintf("datasource '%s' is not available: %v", e.Name, e.Err)
+}
+
+func (e *DataSourceUnavailableError) Unwrap() error {
+       return e.Err
+}
+
+
 // The interface each datasource must implement
 type DataSource interface {
        GetMetrics() []prometheus.Collector                         // Returns pointers to metrics that are managed by the module
@@ -81,8 +95,11 @@ func GetDataSourceIface(dataSourceType string) DataSource {
        return nil
 }
 
+// DataSourceConfigure creates and returns a DataSource object from a configuration,
+// if the configuration is not valid it returns an error.
+// If the datasource can't be run (eg. journalctl not available), it still returns an error which
+// can be checked for the appropriate action.
 func DataSourceConfigure(commonConfig configuration.DataSourceCommonCfg) (*DataSource, error) {
-
        //we dump it back to []byte, because we want to decode the yaml blob twice :
        //once to DataSourceCommonCfg, and then later to the dedicated type of the datasource
        yamlConfig, err := yaml.Marshal(commonConfig)
@@ -107,7 +124,7 @@ func DataSourceConfigure(commonConfig configuration.DataSourceCommonCfg) (*DataS
                subLogger := clog.WithFields(customLog)
                /* check eventual dependencies are satisfied (ie. journald will check journalctl availability) */
                if err := dataSrc.CanRun(); err != nil {
-                       return nil, errors.Wrapf(err, "datasource %s cannot be run", commonConfig.Source)
+                       return nil, &DataSourceUnavailableError{Name: commonConfig.Source, Err: err}
                }
                /* configure the actual datasource */
                if err := dataSrc.Configure(yamlConfig, subLogger); err != nil {
@@ -174,10 +191,11 @@ func LoadAcquisitionFromFile(config *csconfig.CrowdsecServiceCfg) ([]DataSource,
                }
                dec := yaml.NewDecoder(yamlFile)
                dec.SetStrict(true)
+               idx := -1
                for {
                        var sub configuration.DataSourceCommonCfg
-                       var idx int
                        err = dec.Decode(&sub)
+                       idx += 1
                        if err != nil {
                                if ! errors.Is(err, io.EOF) {
                                        return nil, errors.Wrapf(err, "failed to yaml decode %s", acquisFile)
@@ -194,7 +212,6 @@ func LoadAcquisitionFromFile(config *csconfig.CrowdsecServiceCfg) ([]DataSource,
                        if len(sub.Labels) == 0 {
                                if sub.Source == "" {
                                        log.Debugf("skipping empty item in %s", acquisFile)
-                                       idx += 1
                                        continue
                                }
                                return nil, fmt.Errorf("missing labels in %s (position: %d)", acquisFile, idx)
@@ -207,10 +224,14 @@ func LoadAcquisitionFromFile(config *csconfig.CrowdsecServiceCfg) ([]DataSource,
                        }
                        src, err := DataSourceConfigure(sub)
                        if err != nil {
+                               var dserr *DataSourceUnavailableError
+                               if errors.As(err, &dserr) {
+                                       log.Error(err)
+                                       continue
+                               }
                                return nil, errors.Wrapf(err, "while configuring datasource of type %s from %s (position: %d)", sub.Source, acquisFile, idx)
                        }
                        sources = append(sources, *src)
-                       idx += 1
                }
        }
        return sources, nil

diff --git a/pkg/acquisition/acquisition_test.go b/pkg/acquisition/acquisition_test.go
index a547970a862e7d1cca3948b28eabcfd02bf0cb90..a96044ca618ea97b4517879d5d910034b5c2445d 100644 (file)
--- a/pkg/acquisition/acquisition_test.go
+++ b/pkg/acquisition/acquisition_test.go
@@ -171,7 +171,7 @@ log_level: debug
 source: mock_cant_run
 wowo: ajsajasjas
 `,
-                       ExpectedError: "datasource mock_cant_run cannot be run: can't run bro",
+                       ExpectedError: "datasource 'mock_cant_run' is not available: can't run bro",
                },
        }
 

diff --git a/tests/bats/01_crowdsec.bats b/tests/bats/01_crowdsec.bats
index a60b576dd9bf4f71a9efae4e02fde392e117529d..f8272eb90e5b34acfcafbca24ab46fa43f23a873 100644 (file)
--- a/tests/bats/01_crowdsec.bats
+++ b/tests/bats/01_crowdsec.bats
@@ -148,9 +148,10 @@ teardown() {
     rm -f "$ACQUIS_DIR"
 
     config_set '.common.log_media="stdout"'
-    run -124 --separate-stderr timeout 2s "${CROWDSEC}"
+    run -1 --separate-stderr timeout 2s "${CROWDSEC}"
     # check warning
-    assert_stderr_line --partial "no acquisition file found"
+    assert_stderr --partial "no acquisition file found"
+    assert_stderr --partial "crowdsec init: while loading acquisition config: no datasource enabled"
 }
 
 @test "crowdsec (error if acquisition_path and acquisition_dir are not defined)" {
@@ -163,19 +164,55 @@ teardown() {
     config_set '.crowdsec_service.acquisition_dir=""'
 
     config_set '.common.log_media="stdout"'
-    run -124 --separate-stderr timeout 2s "${CROWDSEC}"
+    run -1 --separate-stderr timeout 2s "${CROWDSEC}"
     # check warning
-    assert_stderr_line --partial "no acquisition_path or acquisition_dir specified"
+    assert_stderr --partial "no acquisition_path or acquisition_dir specified"
+    assert_stderr --partial "crowdsec init: while loading acquisition config: no datasource enabled"
 }
 
 @test "crowdsec (no error if acquisition_path is empty string but acquisition_dir is not empty)" {
     ACQUIS_YAML=$(config_get '.crowdsec_service.acquisition_path')
-    rm -f "$ACQUIS_YAML"
     config_set '.crowdsec_service.acquisition_path=""'
 
     ACQUIS_DIR=$(config_get '.crowdsec_service.acquisition_dir')
     mkdir -p "$ACQUIS_DIR"
-    touch "$ACQUIS_DIR"/foo.yaml
+    mv "$ACQUIS_YAML" "$ACQUIS_DIR"/foo.yaml
 
     run -124 --separate-stderr timeout 2s "${CROWDSEC}"
+
+    # now, if foo.yaml is empty instead, there won't be valid datasources.
+
+    cat /dev/null >"$ACQUIS_DIR"/foo.yaml
+
+    run --separate-stderr -1 timeout 2s "${CROWDSEC}"
+    assert_stderr --partial "crowdsec init: while loading acquisition config: no datasource enabled"
 }
+
+@test "crowdsec (disabled datasources)" {
+    config_set '.common.log_media="stdout"'
+
+    # a datasource cannot run - missing journalctl command
+
+    ACQUIS_DIR=$(config_get '.crowdsec_service.acquisition_dir')
+    mkdir -p "$ACQUIS_DIR"
+    cat >"$ACQUIS_DIR"/foo.yaml <<-EOT
+       source: journalctl
+       journalctl_filter:
+       - "_SYSTEMD_UNIT=ssh.service"
+       labels:
+         type: syslog
+       EOT
+
+    run --separate-stderr -124 timeout 2s env PATH='' "${CROWDSEC}"
+    #shellcheck disable=SC2016
+    assert_stderr --partial 'datasource '\''journalctl'\'' is not available: exec: "journalctl": executable file not found in $PATH'
+
+    # if all datasources are disabled, crowdsec should exit
+
+    ACQUIS_YAML=$(config_get '.crowdsec_service.acquisition_path')
+    rm -f "$ACQUIS_YAML"
+    config_set '.crowdsec_service.acquisition_path=""'
+
+    run --separate-stderr -1 timeout 2s env PATH='' "${CROWDSEC}"
+    assert_stderr --partial "crowdsec init: while loading acquisition config: no datasource enabled"
+ }

diff --git a/tests/bin/assert-crowdsec-not-running b/tests/bin/assert-crowdsec-not-running
index c6f381af93a86b1514aca5e10214fe68525aacff..d678cb4f263675fd7b303002824d723c18087a57 100755 (executable)
--- a/tests/bin/assert-crowdsec-not-running
+++ b/tests/bin/assert-crowdsec-not-running
@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 
 is_crowdsec_running() {
-    PIDS=$(pgrep -x 'crowdsec|crowdsec.test|crowdsec.cover')
+    PIDS=$(pgrep -x 'crowdsec|crowdsec.test|crowdsec.cover' 2>/dev/null)
 }
 
 # The process can be slow, especially on CI and during test coverage.