nvdmirror: break Sync() into logical chunks, move into syncContext

1 files changed, 30 insertions, 118 deletions

diff --git a/nvdmirror/sync.go b/nvdmirror/sync.go
index 6f0b4d4..8cdd4c7 100644
--- a/nvdmirror/sync.go
+++ b/nvdmirror/sync.go
@@ -1,127 +1,39 @@
 package nvdmirror
 
-import (
-  "github.com/rs/zerolog/log"
-  "path/filepath"
-)
-
-// Sync to destination directory and return an array of updated files.
+// Sync to destination directory based on given sync configuration and
+// cache.  Returns an array of file names which have been updated in the
+// destination directory.
+//
+// This function does the following:
+//
+// 1. Fetch the contents of the source meta URLs for CVE feeds and
+//    CPE matches.  All source URLs are fetched concurrently.
+//
+// 2. Check the size and hash from meta files against the existing
+//    CVE feed and CPE match files in the destination directory.  All
+//    file sizes and hashes are checked concurrently.
+//
+// 3. Fetch the contents of the changed CVE feeds, CPE match files, and
+//    the CPE dictionary.
+//
+// All HTTP requests are made with the following request headers:
+//
+// * if-modified-since (if the URL was queried previously and the
+//   previous successful response had a "last-modified" header).
+// * if-none-match (if the URL was queried previously and the
+//   previous successful response had an "etag" header).
+// * user-agent
+//
 func Sync(config SyncConfig, cache Cache, dstDir string) []string {
   // build sync context
   ctx := newSyncContext(config, cache, dstDir)
 
-  // get meta URL to full URL map
-  metaUrls := config.getMetaUrls()
-
-  // fetch meta URLs
-  for metaUrl, _ := range(metaUrls) {
-    log.Debug().Str("url", metaUrl).Msg("init")
-    go ctx.fetch(metaUrl)
-  }
-
-  // build list of metas to check
-  checkUrls := make([]string, 0, len(metaUrls))
-
-  // read meta results
-  for range(metaUrls) {
-    r := <-ctx.ch
-    sl := log.With().Str("url", r.fetch.src).Logger()
-
-    if r.fetch.err != nil {
-      // URL error
-      sl.Error().Err(r.fetch.err).Send()
-    } else if !r.fetch.modified {
-      // URL not modified
-      sl.Debug().Msg("not modified")
-    } else {
-      // URL updated
-      sl.Debug().Msg("update")
-
-      // build request headers
-      headers := map[string]string {
-        "if-none-match": r.fetch.headers.Get("etag"),
-        "if-modified-since": r.fetch.headers.Get("last-modified"),
-      }
-
-      // save headers to cache
-      if err := cache.Set(r.fetch.src, headers); err != nil {
-        sl.Error().Err(r.fetch.err).Msg("cache.Set")
-      } else {
-        // append to list of check URLs
-        checkUrls = append(checkUrls, r.fetch.src)
-      }
-    }
-  }
-
-  // check size and hash in updated metas
-  logArray("checkUrls", checkUrls)
-  for _, metaUrl := range(checkUrls) {
-    go ctx.check(metaUrl, metaUrls[metaUrl])
-  }
-
-  // build list of non-meta URLs to sync.
-  syncUrls := make([]string, 0, len(metaUrls))
-  syncUrls = append(syncUrls, config.GetCpeDictUrl())
-
-  for range(checkUrls) {
-    r := <-ctx.ch
-
-    // create sublogger
-    sl := log.With().
-      Str("metaUrl", r.check.metaUrl).
-      Str("metaPath", r.check.metaPath).
-      Str("fullPath", r.check.fullPath).
-      Logger()
-
-    if r.check.err != nil {
-      sl.Error().Err(r.check.err).Send()
-    } else if r.check.match {
-      sl.Debug().Msg("match")
-    } else {
-      syncUrls = append(syncUrls, metaUrls[r.check.metaUrl])
-    }
-  }
-
-  logArray("syncUrls", syncUrls)
-  for _, fullUrl := range(syncUrls) {
-    go ctx.fetch(fullUrl)
-  }
-
-  // build list of changed files
-  changed := make([]string, 0, len(syncUrls))
-
-  // read sync results
-  for range(syncUrls) {
-    r := <-ctx.ch
-    // build sublogger
-    sl := log.With().Str("url", r.fetch.src).Logger()
-
-    if r.fetch.err != nil {
-      sl.Error().Err(r.fetch.err).Send()
-    } else if !r.fetch.modified {
-      sl.Debug().Msg("not modified")
-    } else {
-      sl.Debug().Msg("update")
-
-      // build request headers
-      headers := map[string]string {
-        "if-none-match": r.fetch.headers.Get("etag"),
-        "if-modified-since": r.fetch.headers.Get("last-modified"),
-      }
-
-      // save headers to cache
-      if err := cache.Set(r.fetch.src, headers); err != nil {
-        sl.Error().Err(r.fetch.err).Msg("cache.Set")
-      } else {
-        // append to list of changed files
-        changed = append(changed, filepath.Base(r.fetch.src))
-      }
-    }
-  }
+  // fetch updated meta files
+  checks := ctx.fetchMetas()
 
-  // log changed files
-  logArray("changed", changed)
+  // get syncable URLs
+  urls := append(ctx.checkMetas(checks), config.GetCpeDictUrl())
 
-  // return success
-  return changed
+  // sync urls and return changed files
+  return ctx.syncUrls(urls)
 }