add nvdmirror, including cache iface, jsoncache impl, Sync() func, and test data

28 files changed, 706 insertions, 0 deletions

diff --git a/nvdmirror/cache.go b/nvdmirror/cache.go
new file mode 100644
index 0000000..d65b2c7
--- /dev/null
+++ b/nvdmirror/cache.go
@@ -0,0 +1,22 @@
+package nvdmirror
+
+import (
+  "time"
+)
+
+// Cache entry
+type Entry struct {
+  // time that entry was set
+  Time time.Time `json:"time"`
+  Headers map[string]string `json:"headers"`
+}
+
+// URL cache
+type Cache interface {
+  Get(string) (map[string]string, bool)
+  Set(string, map[string]string) error
+
+  // delete cache entry 
+  Delete(string) error
+  Close() error // save and close cache file
+}
diff --git a/nvdmirror/jsoncache.go b/nvdmirror/jsoncache.go
new file mode 100644
index 0000000..7e84a94
--- /dev/null
+++ b/nvdmirror/jsoncache.go
@@ -0,0 +1,131 @@
+package nvdmirror
+
+import (
+  "compress/gzip"
+  "encoding/json"
+  "os"
+  "path/filepath"
+  "time"
+)
+
+// JSON file backed URL cache.
+type JsonCache struct {
+  dirty bool // has the cache been modified?
+  path string // backing JSON file
+  vals map[string]Entry // entries
+}
+
+// Create new JSON cache.
+func NewJsonCache(path string) (JsonCache, error) {
+  r := JsonCache {
+    path: path,
+  }
+
+  // open path
+  f, err := os.OpenFile(path, os.O_CREATE, 0640)
+  if err != nil {
+    return r, err
+  }
+  defer f.Close()
+
+  // stat file
+  st, err := f.Stat()
+  if err != nil {
+    return r, err
+  }
+
+  vals := make(map[string]Entry)
+  if st.Size() > 0 {
+    // create gzip reader
+    zr, err := gzip.NewReader(f)
+    if err != nil {
+      return r, err
+    }
+
+    // read/decode json
+    d := json.NewDecoder(zr)
+    if err := d.Decode(&vals); err != nil {
+      return r, err
+    }
+  }
+
+  // save path and values
+  r.path = path
+  r.vals = vals
+
+  // return success
+  return r, nil
+}
+
+// Get cache entry
+func (me JsonCache) Get(s string) (map[string]string, bool) {
+  if e, ok := me.vals[s]; ok {
+    return e.Headers, ok
+  } else {
+    return nil, false
+  }
+}
+
+// Set cache entry.
+func (me *JsonCache) Set(s string, headers map[string]string) error {
+  // mark cache as dirty
+  me.dirty = true
+
+  // copy map to internal storage
+  newHeaders := make(map[string]string)
+  for k, v := range(headers) {
+    newHeaders[k] = v
+  }
+
+  // update entry
+  me.vals[s] = Entry {
+    Time: time.Now(),
+    Headers: newHeaders,
+  }
+
+  // return success
+  return nil
+}
+
+// Remove cache entry, if it exists.
+func (me *JsonCache) Delete(s string) error {
+  me.dirty = true
+  delete(me.vals, s)
+  return nil
+}
+
+// Save and close cache.
+func (me *JsonCache) Close() error {
+  if !me.dirty {
+    // return success
+    return nil
+  }
+
+  // open temp output file
+  t, err := os.CreateTemp(filepath.Dir(me.path), "")
+  if err != nil {
+    return err
+  }
+  defer os.Remove(t.Name())
+
+  // encode/compress JSON
+  zw := gzip.NewWriter(t)
+  e := json.NewEncoder(zw)
+  if err = e.Encode(me.vals); err != nil {
+    return err
+  }
+
+  // flush changes
+  if err = zw.Flush(); err != nil {
+    return err
+  }
+
+  // rename to destination file
+  if err = os.Rename(t.Name(), me.path); err != nil {
+    return err
+  }
+
+  // Mark cache as clean, return success.
+  me.dirty = false
+  return nil
+}
diff --git a/nvdmirror/jsoncache_test.go b/nvdmirror/jsoncache_test.go
new file mode 100644
index 0000000..5dc0277
--- /dev/null
+++ b/nvdmirror/jsoncache_test.go
@@ -0,0 +1,151 @@
+package nvdmirror
+
+import (
+  "os"
+  "path/filepath"
+  "reflect"
+  "testing"
+)
+
+func TestNewJsonCache(t *testing.T) {
+  passTests := []struct {
+    name string
+    vals map[string]map[string]string
+  } {{
+    name: "empty",
+  }, {
+    name: "foobarbaz",
+    vals: map[string]map[string]string {
+      "foo": map[string]string {
+        "bar": "baz",
+      },
+    },
+  }}
+
+  // create temp dir
+  dir, err := os.MkdirTemp("", "TestNewJsonCache")
+	if err != nil {
+		t.Error(err)
+    return
+	}
+	defer os.RemoveAll(dir)
+  // dir = "testdata"
+
+  for _, test := range(passTests) {
+    t.Run(test.name, func(t *testing.T) {
+      // build cache path
+	    path := filepath.Join(dir, test.name)
+
+      {
+        // load cache
+        cache, err := NewJsonCache(path)
+        if err != nil {
+          t.Error(err)
+          return
+        }
+
+        // set values
+        for k, v := range(test.vals) {
+          if err = cache.Set(k, v); err != nil {
+            t.Error(err)
+            return
+          }
+        }
+
+        // close cache
+        if err = cache.Close(); err != nil {
+          t.Error(err)
+          return
+        }
+      }
+
+      {
+        // open previously saved cache
+        cache, err := NewJsonCache(path)
+        if err != nil {
+          t.Error(err)
+          return
+        }
+        defer cache.Close()
+
+        // check values
+        for k, exp := range(test.vals) {
+          if got, ok := cache.Get(k); !ok {
+            t.Errorf("missing key: %s", k)
+            return
+          } else if !reflect.DeepEqual(got, exp) {
+            t.Errorf("got %v, exp %v", got, exp)
+            return
+          }
+
+          // delete entry
+          if err = cache.Delete(k); err != nil {
+            t.Error(err)
+            return
+          }
+
+          // get entry again
+          if got, ok := cache.Get(k); ok {
+            t.Errorf("key: %s, got %v, exp missing", k, got)
+            return
+          }
+        }
+
+        // try accessing missing value
+        if got, ok := cache.Get("some key that doesn't exist"); ok {
+          t.Errorf("got %v, exp !ok", got)
+        }
+
+        // close cache, check for error
+        if err = cache.Close(); err != nil {
+          t.Error(err)
+          return
+        }
+
+        if cache.dirty {
+          t.Errorf("got dirty, exp clean")
+          return
+        }
+      }
+    })
+  }
+
+  failNewTests := []string {
+    "testdata/does/not/exist",
+    "testdata/bad-data.json",
+    "testdata/bad-stat.json",
+  }
+
+  for _, test := range(failNewTests) {
+    t.Run(test, func(t *testing.T) {
+      if got, err := NewJsonCache(test); err == nil {
+        t.Errorf("got %v, exp err", got)
+      }
+    })
+  }
+
+  failCloseTests := []string {
+    "/dev/null",
+  }
+
+  testVals := map[string]string { "bar": "baz" }
+
+  for _, test := range(failCloseTests) {
+    t.Run(test, func(t *testing.T) {
+      cache, err := NewJsonCache(test)
+      if err != nil {
+        t.Error(err)
+        return
+      }
+
+      if err = cache.Set("foo", testVals); err != nil {
+        t.Error(err)
+        return
+      }
+
+      if err = cache.Close(); err == nil {
+        t.Errorf("got success, exp error")
+      }
+    })
+  }
+}
diff --git a/nvdmirror/nvdmirror.go b/nvdmirror/nvdmirror.go
new file mode 100644
index 0000000..7b092b1
--- /dev/null
+++ b/nvdmirror/nvdmirror.go
@@ -0,0 +1,184 @@
+// mirror files from upstream NVD source
+package nvdmirror
+
+import (
+  "fmt"
+  "github.com/rs/zerolog/log"
+  "io"
+  "net/http"
+  "net/url"
+  "os"
+  "path/filepath"
+  "time"
+)
+
+// URLs
+type Urls struct {
+  // CVE 1.1 Base URL.  The full meta and JSON URLs are constructed from
+  // this appended to this base..
+  Cve11Base string
+
+  // CPE 1.0 match meta URL
+  Cpe10MatchMeta string
+
+  // CPE 1.0 match URL
+  Cpe10MatchJson string
+
+  // CPE 2.3 dictionary URL
+  Cpe23Dict string
+}
+
+// NVD URLs
+var DefaultUrls = Urls {
+  Cve11Base: "https://nvd.nist.gov/feeds/json/cve/1.1",
+  Cpe10MatchMeta: "https://nvd.nist.gov/feeds/json/cpematch/1.0/nvdcpematch-1.0.meta",
+  Cpe10MatchJson: "https://nvd.nist.gov/feeds/json/cpematch/1.0/nvdcpematch-1.0.gz",
+  Cpe23Dict: "https://nvd.nist.gov/feeds/xml/cpe/dictionary/official-cpe-dictionary_v2.3.xml.gz",
+}
+
+var extraFiles = []string {
+  "nvdcve-1.1-modified",
+  "nvdcve-1.1-recent",
+}
+
+// fetch result
+type fetchResult struct {
+  src string // source URL
+  err error // fetch result
+  notModified bool // was the result unmodified?
+  lastModified string // last modified response header
+  etag string // etag response header
+}
+
+// base CVE year
+const baseYear = 2002
+
+// default user agent (FIXME: make configurable)
+var userAgent = "cvez/0.1.0"
+
+func fetch(ch chan fetchResult, client *http.Client, cache Cache, srcUrl string, dstDir string) {
+  // parse source url
+  src, err := url.Parse(srcUrl)
+  if err != nil {
+    ch <- fetchResult { src: srcUrl, err: err }
+    return
+  }
+
+  // build destination path
+  path := filepath.Join(dstDir, filepath.Base(src.Path))
+  log.Debug().Str("url", srcUrl).Str("path", path).Send()
+
+  // create temporary output file
+  f, err := os.CreateTemp(filepath.Dir(path), "")
+  if err != nil {
+    ch <- fetchResult { src: srcUrl, err: err }
+    return
+  }
+  defer os.Remove(f.Name())
+
+  // create request
+  req, err := http.NewRequest("GET", srcUrl, nil)
+  if err != nil {
+    ch <- fetchResult { src: srcUrl, err: err }
+    return
+  }
+
+  // add request headers
+  req.Header.Add("user-agent", userAgent)
+  if headers, ok := cache.Get(srcUrl); ok {
+    for k, v := range(headers) {
+      req.Header.Add(k, v)
+    }
+  }
+
+  // send request
+  resp, err := client.Do(req)
+  if err != nil {
+    ch <- fetchResult { src: srcUrl, err: err }
+    return
+  }
+  defer resp.Body.Close()
+
+  switch resp.StatusCode {
+  case 200:
+    // copy body to result
+    if size, err := io.Copy(f, resp.Body); err != nil {
+      // copy failed
+      ch <- fetchResult { src: srcUrl, err: err }
+    } else if err = os.Rename(f.Name(), path); err != nil {
+      // rename failed
+      ch <- fetchResult { src: srcUrl, err: err }
+    } else {
+      log.Debug().Str("url", srcUrl).Int64("size", size).Send()
+      ch <- fetchResult {
+        src: srcUrl,
+        lastModified: resp.Header.Get("last-modified"),
+        etag: resp.Header.Get("etag"),
+      }
+    }
+  case 304:
+    ch <- fetchResult { src: srcUrl, notModified: true }
+  default:
+    code := resp.StatusCode
+    err := fmt.Errorf("%d: %s", code, http.StatusText(code))
+    ch <- fetchResult { src: srcUrl, err: err }
+  }
+}
+
+func Sync(urls Urls, cache Cache, dstDir string) error {
+  log.Debug().Str("dstDir", dstDir).Msg("Sync")
+
+  // create fetch result channel
+  ch := make(chan fetchResult)
+
+  // create shared transport and client
+  tr := &http.Transport{
+    // FIXME: make configurable
+	  MaxIdleConns:     10,
+	  IdleConnTimeout:  30 * time.Second,
+  }
+  client := &http.Client{Transport: tr}
+
+  // calculate total number of years
+  numYears := time.Now().Year() - 2002 + 1
+
+  // fetch metas
+  for year := baseYear; year < baseYear + numYears; year++ {
+    // build url
+    url := fmt.Sprintf("%s/nvdcve-1.1-%04d.meta", urls.Cve11Base, year)
+    log.Debug().Int("year", year).Str("url", url).Send()
+    go fetch(ch, client, cache, url, dstDir)
+  }
+
+  for _, s := range(extraFiles) {
+    url := fmt.Sprintf("%s/%s.meta", urls.Cve11Base, s)
+    log.Debug().Str("file", s).Str("url", url).Send()
+    go fetch(ch, client, cache, url, dstDir)
+  }
+
+  // read results
+  for i := 0; i < numYears + len(extraFiles); i++ {
+    if r := <-ch; r.err != nil {
+      log.Error().Str("url", r.src).Err(r.err).Send()
+      // FIXME: errs = append(errs, r)
+    } else {
+      log.Info().Str("url", r.src).Str("etag", r.etag).Msg("ok")
+
+      if !r.notModified {
+        // cache headers
+        err := cache.Set(r.src, map[string]string {
+          "if-none-match": r.etag,
+          "if-modified-since": r.lastModified,
+        })
+
+        if err != nil {
+          log.Error().Str("url", r.src).Err(r.err).Msg("Set")
+          return err
+        }
+      }
+    }
+  }
+
+  // return success
+  return nil
+}
diff --git a/nvdmirror/nvdmirror_test.go b/nvdmirror/nvdmirror_test.go
new file mode 100644
index 0000000..5b40dd8
--- /dev/null
+++ b/nvdmirror/nvdmirror_test.go
@@ -0,0 +1,111 @@
+package nvdmirror
+
+import (
+  "fmt"
+  "os"
+  "net/http"
+  "path/filepath"
+  "testing"
+)
+
+// serve on given port
+func serve(port int, ch chan bool) {
+  s := http.Server {
+    Addr: fmt.Sprintf(":%d", port),
+    Handler: http.FileServer(http.Dir("testdata/files")),
+  }
+
+  go (func() {
+    // block on channel
+    _ = <-ch
+
+    // shut down server
+    s.Close()
+  })()
+
+  // start server
+  s.ListenAndServe()
+}
+
+// close server immediately
+func stopServer(ch chan bool) {
+  ch <- false
+}
+
+func TestSync(t *testing.T) {
+  // 
+  port := 8888
+  ch := make(chan bool)
+  defer stopServer(ch)
+
+  // spin up local server
+  go serve(port, ch)
+
+  // create temp dir
+  dir, err := os.MkdirTemp("", "")
+  if err != nil {
+    t.Error(err)
+    return
+  }
+  defer os.RemoveAll(dir)
+  // dir = "testdata/out"
+
+  // create cache
+  cache, err := NewJsonCache(filepath.Join(dir, "cache.json.gz"))
+  if err != nil {
+    t.Error(err)
+    return
+  }
+  defer cache.Close()
+
+  // custom sync config
+  // FIXME: stand up custom server for this
+  urls := Urls {
+    Cve11Base: fmt.Sprintf("http://localhost:%d", port),
+  }
+
+  // sync data
+  if err := Sync(urls, &cache, dir); err != nil {
+    t.Error(err)
+  }
+
+  // sync data again (to test caching)
+  if err := Sync(urls, &cache, dir); err != nil {
+    t.Error(err)
+  }
+}
+
+func TestBadUrl(t *testing.T) {
+  // create temp dir
+  dir, err := os.MkdirTemp("", "")
+  if err != nil {
+    t.Error(err)
+    return
+  }
+  defer os.RemoveAll(dir)
+
+  // create cache
+  cache, err := NewJsonCache(filepath.Join(dir, "cache.json.gz"))
+  if err != nil {
+    t.Error(err)
+    return
+  }
+  defer cache.Close()
+
+  failTests := []string {
+    "httsldkfjasldkfjadp://localhost:0",
+    "http://localhost:0",
+  }
+
+  for _, test := range(failTests) {
+    t.Run(test, func(t *testing.T) {
+      // custom sync config
+      urls := Urls { Cve11Base: test }
+
+      // sync data
+      if err := Sync(urls, &cache, dir); err == nil {
+        t.Errorf("got success, exp error")
+      }
+    })
+  }
+}
diff --git a/nvdmirror/testdata/bad-data.json b/nvdmirror/testdata/bad-data.json
new file mode 100644
index 0000000..4117452
--- /dev/null
+++ b/nvdmirror/testdata/bad-data.json
@@ -0,0 +1 @@
+{"
diff --git a/nvdmirror/testdata/bad-stat.json b/nvdmirror/testdata/bad-stat.json
new file mode 120000
index 0000000..1eb768d
--- /dev/null
+++ b/nvdmirror/testdata/bad-stat.json
@@ -0,0 +1 @@
+does/not/exist
\ No newline at end of file
diff --git a/nvdmirror/testdata/files/nvdcve-1.1-2002.meta b/nvdmirror/testdata/files/nvdcve-1.1-2002.meta
new file mode 100644
index 0000000..4f79b34
--- /dev/null
+++ b/nvdmirror/testdata/files/nvdcve-1.1-2002.meta
@@ -0,0 +1,5 @@
+lastModifiedDate:2022-01-22T03:00:58-05:00

+size:20972695

+zipSize:1453971

+gzSize:1453835

+sha256:E2D7B3A7239C5DDCF9713E6A892782BC3E9B07A6023D1D4F345BCC9644D99FBA

diff --git a/nvdmirror/testdata/files/nvdcve-1.1-2003.meta b/nvdmirror/testdata/files/nvdcve-1.1-2003.meta
new file mode 100644
index 0000000..93ff803
--- /dev/null
+++ b/nvdmirror/testdata/files/nvdcve-1.1-2003.meta
@@ -0,0 +1,5 @@
+lastModifiedDate:2021-08-23T13:59:45-04:00

+size:6025730

+zipSize:434156

+gzSize:434020

+sha256:3BCA2121E3A72AACEA966C7B66F8E52D6EB55D65E5C97F6ACC1A5B7B48D503B3

diff --git a/nvdmirror/testdata/files/nvdcve-1.1-2004.meta b/nvdmirror/testdata/files/nvdcve-1.1-2004.meta
new file mode 100644
index 0000000..408fbd5
--- /dev/null
+++ b/nvdmirror/testdata/files/nvdcve-1.1-2004.meta
@@ -0,0 +1,5 @@
+lastModifiedDate:2022-02-08T03:02:48-05:00

+size:12598744

+zipSize:854973

+gzSize:854837

+sha256:AA4170920FBCCC7F3D7D681D4315321E685B7D9CCC12750F8D3D217A09252474

diff --git a/nvdmirror/testdata/files/nvdcve-1.1-2005.meta b/nvdmirror/testdata/files/nvdcve-1.1-2005.meta
new file mode 100644
index 0000000..7e540bc
--- /dev/null
+++ b/nvdmirror/testdata/files/nvdcve-1.1-2005.meta
@@ -0,0 +1,5 @@
+lastModifiedDate:2022-02-08T03:02:45-05:00

+size:19932851

+zipSize:1346296

+gzSize:1346160

+sha256:044A7F0C7E914E256FE2CC15B71F75F2B4D9F98EDC52F65B698FD1FC7C4C14E5

diff --git a/nvdmirror/testdata/files/nvdcve-1.1-2006.meta b/nvdmirror/testdata/files/nvdcve-1.1-2006.meta
new file mode 100644
index 0000000..81f20be
--- /dev/null
+++ b/nvdmirror/testdata/files/nvdcve-1.1-2006.meta
@@ -0,0 +1,5 @@
+lastModifiedDate:2022-02-08T03:02:40-05:00

+size:30464164

+zipSize:2131269

+gzSize:2131133

+sha256:7A6FD9344153D818C3198BF272C45A05C5798D9BE897BCD9D8B24AA438289B11

diff --git a/nvdmirror/testdata/files/nvdcve-1.1-2007.meta b/nvdmirror/testdata/files/nvdcve-1.1-2007.meta
new file mode 100644
index 0000000..8661b7d
--- /dev/null
+++ b/nvdmirror/testdata/files/nvdcve-1.1-2007.meta
@@ -0,0 +1,5 @@
+lastModifiedDate:2022-02-19T03:02:08-05:00

+size:28527928

+zipSize:2102361

+gzSize:2102225

+sha256:1EF12715AC9D4231938F3D01626E0563F2ADEF3CE69FF2F53FC78B60786583E4

diff --git a/nvdmirror/testdata/files/nvdcve-1.1-2008.meta b/nvdmirror/testdata/files/nvdcve-1.1-2008.meta
new file mode 100644
index 0000000..c6caf36
--- /dev/null
+++ b/nvdmirror/testdata/files/nvdcve-1.1-2008.meta
@@ -0,0 +1,5 @@
+lastModifiedDate:2022-02-10T03:01:59-05:00

+size:33340407

+zipSize:2160208

+gzSize:2160072

+sha256:9C61DCCA59E9875573C63B86EC6D6A3064FA045E83C32B0B17DD4C116370185E

diff --git a/nvdmirror/testdata/files/nvdcve-1.1-2009.meta b/nvdmirror/testdata/files/nvdcve-1.1-2009.meta
new file mode 100644
index 0000000..2666ef4
--- /dev/null
+++ b/nvdmirror/testdata/files/nvdcve-1.1-2009.meta
@@ -0,0 +1,5 @@
+lastModifiedDate:2022-02-11T03:02:10-05:00

+size:32580264

+zipSize:1974299

+gzSize:1974163

+sha256:C6BD6966A775A7115B4B2263BE2EFF863DF07D811C93AC65C5E1344AC3934EF0

diff --git a/nvdmirror/testdata/files/nvdcve-1.1-2010.meta b/nvdmirror/testdata/files/nvdcve-1.1-2010.meta
new file mode 100644
index 0000000..5192d80
--- /dev/null
+++ b/nvdmirror/testdata/files/nvdcve-1.1-2010.meta
@@ -0,0 +1,5 @@
+lastModifiedDate:2022-02-19T03:02:02-05:00

+size:34873319

+zipSize:1941377

+gzSize:1941241

+sha256:A51293FDE6446834BBCCCC854B5DB673A730309C6E0E4483090847DC7A7A4A4F

diff --git a/nvdmirror/testdata/files/nvdcve-1.1-2011.meta b/nvdmirror/testdata/files/nvdcve-1.1-2011.meta
new file mode 100644
index 0000000..cc83473
--- /dev/null
+++ b/nvdmirror/testdata/files/nvdcve-1.1-2011.meta
@@ -0,0 +1,5 @@
+lastModifiedDate:2022-02-08T03:02:08-05:00

+size:35005464

+zipSize:1835220

+gzSize:1835084

+sha256:B9FD38F830649B03756B2FB151064997ACD9E3B21330C743962D3CAC7FB211EB

diff --git a/nvdmirror/testdata/files/nvdcve-1.1-2012.meta b/nvdmirror/testdata/files/nvdcve-1.1-2012.meta
new file mode 100644
index 0000000..43df16e
--- /dev/null
+++ b/nvdmirror/testdata/files/nvdcve-1.1-2012.meta
@@ -0,0 +1,5 @@
+lastModifiedDate:2022-02-11T03:01:57-05:00

+size:40517537

+zipSize:2062242

+gzSize:2062106

+sha256:27CE6740CDCCE9086D6A18E3888B9D4AE606F513156CB8F17C75ACAFB92CA1CC

diff --git a/nvdmirror/testdata/files/nvdcve-1.1-2013.meta b/nvdmirror/testdata/files/nvdcve-1.1-2013.meta
new file mode 100644
index 0000000..7add80d
--- /dev/null
+++ b/nvdmirror/testdata/files/nvdcve-1.1-2013.meta
@@ -0,0 +1,5 @@
+lastModifiedDate:2022-02-19T03:01:54-05:00

+size:45815787

+zipSize:2455830

+gzSize:2455694

+sha256:C36F539B3B67943121D76AEFD182701EC14D1C65B83DD91E4690A68AA9D64643

diff --git a/nvdmirror/testdata/files/nvdcve-1.1-2014.meta b/nvdmirror/testdata/files/nvdcve-1.1-2014.meta
new file mode 100644
index 0000000..d15a85b
--- /dev/null
+++ b/nvdmirror/testdata/files/nvdcve-1.1-2014.meta
@@ -0,0 +1,5 @@
+lastModifiedDate:2022-02-19T03:01:45-05:00

+size:40764138

+zipSize:2353848

+gzSize:2353712

+sha256:FC17C65007A11D9E1045EF4C14174A99F911571C10361E7945E242AA7AD85F27

diff --git a/nvdmirror/testdata/files/nvdcve-1.1-2015.meta b/nvdmirror/testdata/files/nvdcve-1.1-2015.meta
new file mode 100644
index 0000000..36031de
--- /dev/null
+++ b/nvdmirror/testdata/files/nvdcve-1.1-2015.meta
@@ -0,0 +1,5 @@
+lastModifiedDate:2022-02-20T03:01:51-05:00

+size:40140856

+zipSize:2208094

+gzSize:2207958

+sha256:A63FCB0DCC45A25932E2D1CDF02A05231BAFB804E853D06400E0E31CF93B867A

diff --git a/nvdmirror/testdata/files/nvdcve-1.1-2016.meta b/nvdmirror/testdata/files/nvdcve-1.1-2016.meta
new file mode 100644
index 0000000..40a4a0e
--- /dev/null
+++ b/nvdmirror/testdata/files/nvdcve-1.1-2016.meta
@@ -0,0 +1,5 @@
+lastModifiedDate:2022-02-21T03:01:19-05:00

+size:51789260

+zipSize:2694233

+gzSize:2694097

+sha256:8AA3E55FE30B2592A090486D103805CCCFAD2EED32C5AA5AA8ED81A086754E0A

diff --git a/nvdmirror/testdata/files/nvdcve-1.1-2017.meta b/nvdmirror/testdata/files/nvdcve-1.1-2017.meta
new file mode 100644
index 0000000..ca4a00e
--- /dev/null
+++ b/nvdmirror/testdata/files/nvdcve-1.1-2017.meta
@@ -0,0 +1,5 @@
+lastModifiedDate:2022-02-20T03:01:23-05:00

+size:73254381

+zipSize:3757519

+gzSize:3757383

+sha256:6836C573B27DC8DC08CE077D6BEFD99C462FBA664502D965070A45B70EC674FD

diff --git a/nvdmirror/testdata/files/nvdcve-1.1-2018.meta b/nvdmirror/testdata/files/nvdcve-1.1-2018.meta
new file mode 100644
index 0000000..019a3ff
--- /dev/null
+++ b/nvdmirror/testdata/files/nvdcve-1.1-2018.meta
@@ -0,0 +1,5 @@
+lastModifiedDate:2022-02-21T03:01:07-05:00

+size:76729497

+zipSize:4097777

+gzSize:4097641

+sha256:AEF1FFDD7E31F4710B5B1C0D6639C99604674E16B5BB567961036BA0EB73485F

diff --git a/nvdmirror/testdata/files/nvdcve-1.1-2019.meta b/nvdmirror/testdata/files/nvdcve-1.1-2019.meta
new file mode 100644
index 0000000..1d11ed5
--- /dev/null
+++ b/nvdmirror/testdata/files/nvdcve-1.1-2019.meta
@@ -0,0 +1,5 @@
+lastModifiedDate:2022-02-21T03:00:51-05:00

+size:86200450

+zipSize:4585488

+gzSize:4585352

+sha256:89374F593C8D7C6A5ED9A7F9CD056CCF83DE132418F548E44436B8E349C9FF08

diff --git a/nvdmirror/testdata/files/nvdcve-1.1-2020.meta b/nvdmirror/testdata/files/nvdcve-1.1-2020.meta
new file mode 100644
index 0000000..9895d47
--- /dev/null
+++ b/nvdmirror/testdata/files/nvdcve-1.1-2020.meta
@@ -0,0 +1,5 @@
+lastModifiedDate:2022-02-21T03:00:34-05:00

+size:109758817

+zipSize:5408074

+gzSize:5407938

+sha256:95825EC8F1A6AAD18EEB7570188233FDBC02D1B779EC3A8540F6AA5ED21FAE44

diff --git a/nvdmirror/testdata/files/nvdcve-1.1-2021.meta b/nvdmirror/testdata/files/nvdcve-1.1-2021.meta
new file mode 100644
index 0000000..7a81d7e
--- /dev/null
+++ b/nvdmirror/testdata/files/nvdcve-1.1-2021.meta
@@ -0,0 +1,5 @@
+lastModifiedDate:2022-02-21T03:00:14-05:00

+size:104254008

+zipSize:5094678

+gzSize:5094542

+sha256:143EBF79D6A03F6FF2A96DF3173792B8968EDC3BB0D9C429422969D4D66121EA

diff --git a/nvdmirror/testdata/files/nvdcve-1.1-2022.meta b/nvdmirror/testdata/files/nvdcve-1.1-2022.meta
new file mode 100644
index 0000000..bc5bb55
--- /dev/null
+++ b/nvdmirror/testdata/files/nvdcve-1.1-2022.meta
@@ -0,0 +1,5 @@
+lastModifiedDate:2022-02-21T03:00:00-05:00

+size:6828897

+zipSize:368347

+gzSize:368211

+sha256:0B522A84B42E973672771F3AF4A9CA711708FA15D8EA3AE829310FE6C233BE6B