add nvdmirror, including cache iface, jsoncache impl, Sync() func, and test data

1 files changed, 184 insertions, 0 deletions

diff --git a/nvdmirror/nvdmirror.go b/nvdmirror/nvdmirror.go
new file mode 100644
index 0000000..7b092b1
--- /dev/null
+++ b/nvdmirror/nvdmirror.go
@@ -0,0 +1,184 @@
+// mirror files from upstream NVD source
+package nvdmirror
+
+import (
+  "fmt"
+  "github.com/rs/zerolog/log"
+  "io"
+  "net/http"
+  "net/url"
+  "os"
+  "path/filepath"
+  "time"
+)
+
+// URLs
+type Urls struct {
+  // CVE 1.1 Base URL.  The full meta and JSON URLs are constructed from
+  // this appended to this base..
+  Cve11Base string
+
+  // CPE 1.0 match meta URL
+  Cpe10MatchMeta string
+
+  // CPE 1.0 match URL
+  Cpe10MatchJson string
+
+  // CPE 2.3 dictionary URL
+  Cpe23Dict string
+}
+
+// NVD URLs
+var DefaultUrls = Urls {
+  Cve11Base: "https://nvd.nist.gov/feeds/json/cve/1.1",
+  Cpe10MatchMeta: "https://nvd.nist.gov/feeds/json/cpematch/1.0/nvdcpematch-1.0.meta",
+  Cpe10MatchJson: "https://nvd.nist.gov/feeds/json/cpematch/1.0/nvdcpematch-1.0.gz",
+  Cpe23Dict: "https://nvd.nist.gov/feeds/xml/cpe/dictionary/official-cpe-dictionary_v2.3.xml.gz",
+}
+
+var extraFiles = []string {
+  "nvdcve-1.1-modified",
+  "nvdcve-1.1-recent",
+}
+
+// fetch result
+type fetchResult struct {
+  src string // source URL
+  err error // fetch result
+  notModified bool // was the result unmodified?
+  lastModified string // last modified response header
+  etag string // etag response header
+}
+
+// base CVE year
+const baseYear = 2002
+
+// default user agent (FIXME: make configurable)
+var userAgent = "cvez/0.1.0"
+
+func fetch(ch chan fetchResult, client *http.Client, cache Cache, srcUrl string, dstDir string) {
+  // parse source url
+  src, err := url.Parse(srcUrl)
+  if err != nil {
+    ch <- fetchResult { src: srcUrl, err: err }
+    return
+  }
+
+  // build destination path
+  path := filepath.Join(dstDir, filepath.Base(src.Path))
+  log.Debug().Str("url", srcUrl).Str("path", path).Send()
+
+  // create temporary output file
+  f, err := os.CreateTemp(filepath.Dir(path), "")
+  if err != nil {
+    ch <- fetchResult { src: srcUrl, err: err }
+    return
+  }
+  defer os.Remove(f.Name())
+
+  // create request
+  req, err := http.NewRequest("GET", srcUrl, nil)
+  if err != nil {
+    ch <- fetchResult { src: srcUrl, err: err }
+    return
+  }
+
+  // add request headers
+  req.Header.Add("user-agent", userAgent)
+  if headers, ok := cache.Get(srcUrl); ok {
+    for k, v := range(headers) {
+      req.Header.Add(k, v)
+    }
+  }
+
+  // send request
+  resp, err := client.Do(req)
+  if err != nil {
+    ch <- fetchResult { src: srcUrl, err: err }
+    return
+  }
+  defer resp.Body.Close()
+
+  switch resp.StatusCode {
+  case 200:
+    // copy body to result
+    if size, err := io.Copy(f, resp.Body); err != nil {
+      // copy failed
+      ch <- fetchResult { src: srcUrl, err: err }
+    } else if err = os.Rename(f.Name(), path); err != nil {
+      // rename failed
+      ch <- fetchResult { src: srcUrl, err: err }
+    } else {
+      log.Debug().Str("url", srcUrl).Int64("size", size).Send()
+      ch <- fetchResult {
+        src: srcUrl,
+        lastModified: resp.Header.Get("last-modified"),
+        etag: resp.Header.Get("etag"),
+      }
+    }
+  case 304:
+    ch <- fetchResult { src: srcUrl, notModified: true }
+  default:
+    code := resp.StatusCode
+    err := fmt.Errorf("%d: %s", code, http.StatusText(code))
+    ch <- fetchResult { src: srcUrl, err: err }
+  }
+}
+
+func Sync(urls Urls, cache Cache, dstDir string) error {
+  log.Debug().Str("dstDir", dstDir).Msg("Sync")
+
+  // create fetch result channel
+  ch := make(chan fetchResult)
+
+  // create shared transport and client
+  tr := &http.Transport{
+    // FIXME: make configurable
+	  MaxIdleConns:     10,
+	  IdleConnTimeout:  30 * time.Second,
+  }
+  client := &http.Client{Transport: tr}
+
+  // calculate total number of years
+  numYears := time.Now().Year() - 2002 + 1
+
+  // fetch metas
+  for year := baseYear; year < baseYear + numYears; year++ {
+    // build url
+    url := fmt.Sprintf("%s/nvdcve-1.1-%04d.meta", urls.Cve11Base, year)
+    log.Debug().Int("year", year).Str("url", url).Send()
+    go fetch(ch, client, cache, url, dstDir)
+  }
+
+  for _, s := range(extraFiles) {
+    url := fmt.Sprintf("%s/%s.meta", urls.Cve11Base, s)
+    log.Debug().Str("file", s).Str("url", url).Send()
+    go fetch(ch, client, cache, url, dstDir)
+  }
+
+  // read results
+  for i := 0; i < numYears + len(extraFiles); i++ {
+    if r := <-ch; r.err != nil {
+      log.Error().Str("url", r.src).Err(r.err).Send()
+      // FIXME: errs = append(errs, r)
+    } else {
+      log.Info().Str("url", r.src).Str("etag", r.etag).Msg("ok")
+
+      if !r.notModified {
+        // cache headers
+        err := cache.Set(r.src, map[string]string {
+          "if-none-match": r.etag,
+          "if-modified-since": r.lastModified,
+        })
+
+        if err != nil {
+          log.Error().Str("url", r.src).Err(r.err).Msg("Set")
+          return err
+        }
+      }
+    }
+  }
+
+  // return success
+  return nil
+}