[PATCH] Download datafile (#895)

* add the ability to download datafile on cscli hub upgrade on files are missing
* fix stuff + lint
* fix error management

Co-authored-by: sabban <15465465+sabban@users.noreply.github.com>
Gbp-Pq: Name 0011-4dbbd4b3c4-automatically-download-files-when-needed.patch

diff --git a/cmd/crowdsec-cli/utils.go b/cmd/crowdsec-cli/utils.go
index 003181bc83c59a9391270cfae808638db032051f..925f7793e2a6023cef21f9c0fcfe9de8c1493b61 100644 (file)
--- a/cmd/crowdsec-cli/utils.go
+++ b/cmd/crowdsec-cli/utils.go
@@ -216,7 +216,11 @@ func UpgradeConfig(itemType string, name string, force bool) {
                found = true
                if v.UpToDate {
                        log.Infof("%s : up-to-date", v.Name)
+
                        if !force {
+                               if err = cwhub.DownloadDataIfNeeded(csConfig.Cscli.DataDir, csConfig.Cscli.HubDir, v, false); err != nil {
+                                       log.Fatalf("%s : download failed : %v", v.Name, err)
+                               }
                                continue
                        }
                }

diff --git a/pkg/cwhub/download.go b/pkg/cwhub/download.go
index 91fb8ecb65b37738c1645865bb215d73fca5e23f..64df7e81f491cdebe3687d91cfd3f1689409ade4 100644 (file)
--- a/pkg/cwhub/download.go
+++ b/pkg/cwhub/download.go
@@ -3,6 +3,7 @@ package cwhub
 import (
        "bytes"
        "crypto/sha256"
+       "path"
        "path/filepath"
 
        //"errors"
@@ -134,7 +135,7 @@ func DownloadItem(cscli *csconfig.CscliCfg, target Item, overwrite bool) (Item,
                }
                if target.UpToDate {
                        log.Debugf("%s : up-to-date, not updated", target.Name)
-                       return target, nil
+                       //  We still have to check if data files are present
                }
        }
        req, err := http.NewRequest("GET", fmt.Sprintf(RawFileURLTemplate, HubBranch, target.RemotePath), nil)
@@ -204,7 +205,34 @@ func DownloadItem(cscli *csconfig.CscliCfg, target Item, overwrite bool) (Item,
        target.Tainted = false
        target.UpToDate = true
 
-       dec := yaml.NewDecoder(bytes.NewReader(body))
+       if err = downloadData(dataFolder, overwrite, bytes.NewReader(body)); err != nil {
+               return target, errors.Wrapf(err, "while downloading data for %s", target.FileName)
+       }
+
+       hubIdx[target.Type][target.Name] = target
+       return target, nil
+}
+
+func DownloadDataIfNeeded(dataFolder string, hubdir string, target Item, force bool) error {
+       var (
+               itemFile *os.File
+               err      error
+       )
+       itemFilePath := fmt.Sprintf("%s/%s", hubdir, target.RemotePath)
+
+       if itemFile, err = os.Open(itemFilePath); err != nil {
+               return errors.Wrapf(err, "while opening %s", itemFilePath)
+       }
+       if err = downloadData(dataFolder, force, itemFile); err != nil {
+               return errors.Wrapf(err, "while downloading data for %s", itemFilePath)
+       }
+       return nil
+}
+
+func downloadData(dataFolder string, force bool, reader io.Reader) error {
+       var err error
+       dec := yaml.NewDecoder(reader)
+
        for {
                data := &types.DataSet{}
                err = dec.Decode(data)
@@ -212,14 +240,24 @@ func DownloadItem(cscli *csconfig.CscliCfg, target Item, overwrite bool) (Item,
                        if err == io.EOF {
                                break
                        } else {
-                               return target, errors.Wrap(err, "while reading file")
+                               return errors.Wrap(err, "while reading file")
                        }
                }
-               err = types.GetData(data.Data, dataFolder)
-               if err != nil {
-                       return target, errors.Wrap(err, "while getting data")
+
+               download := false
+               if !force {
+                       for _, dataS := range data.Data {
+                               if _, err := os.Stat(path.Join(dataFolder, dataS.DestPath)); os.IsNotExist(err) {
+                                       download = true
+                               }
+                       }
+               }
+               if download || force {
+                       err = types.GetData(data.Data, dataFolder)
+                       if err != nil {
+                               return errors.Wrap(err, "while getting data")
+                       }
                }
        }
-       hubIdx[target.Type][target.Name] = target
-       return target, nil
+       return nil
 }