aboutsummaryrefslogtreecommitdiff
path: root/nvdmirror/sync.go
diff options
context:
space:
mode:
Diffstat (limited to 'nvdmirror/sync.go')
-rw-r--r--nvdmirror/sync.go148
1 files changed, 30 insertions, 118 deletions
diff --git a/nvdmirror/sync.go b/nvdmirror/sync.go
index 6f0b4d4..8cdd4c7 100644
--- a/nvdmirror/sync.go
+++ b/nvdmirror/sync.go
@@ -1,127 +1,39 @@
package nvdmirror
-import (
- "github.com/rs/zerolog/log"
- "path/filepath"
-)
-
-// Sync to destination directory and return an array of updated files.
+// Sync to destination directory based on given sync configuration and
+// cache. Returns an array of file names which have been updated in the
+// destination directory.
+//
+// This function does the following:
+//
+// 1. Fetch the contents of the source meta URLs for CVE feeds and
+// CPE matches. All source URLs are fetched concurrently.
+//
+// 2. Check the size and hash from meta files against the existing
+// CVE feed and CPE match files in the destination directory. All
+// file sizes and hashes are checked concurrently.
+//
+// 3. Fetch the contents of the changed CVE feeds, CPE match files, and
+// the CPE dictionary.
+//
+// All HTTP requests are made with the following request headers:
+//
+// * if-modified-since (if the URL was queried previously and the
+// previous successful response had a "last-modified" header).
+// * if-none-match (if the URL was queried previously and the
+// previous successful response had an "etag" header).
+// * user-agent
+//
func Sync(config SyncConfig, cache Cache, dstDir string) []string {
// build sync context
ctx := newSyncContext(config, cache, dstDir)
- // get meta URL to full URL map
- metaUrls := config.getMetaUrls()
-
- // fetch meta URLs
- for metaUrl, _ := range(metaUrls) {
- log.Debug().Str("url", metaUrl).Msg("init")
- go ctx.fetch(metaUrl)
- }
-
- // build list of metas to check
- checkUrls := make([]string, 0, len(metaUrls))
-
- // read meta results
- for range(metaUrls) {
- r := <-ctx.ch
- sl := log.With().Str("url", r.fetch.src).Logger()
-
- if r.fetch.err != nil {
- // URL error
- sl.Error().Err(r.fetch.err).Send()
- } else if !r.fetch.modified {
- // URL not modified
- sl.Debug().Msg("not modified")
- } else {
- // URL updated
- sl.Debug().Msg("update")
-
- // build request headers
- headers := map[string]string {
- "if-none-match": r.fetch.headers.Get("etag"),
- "if-modified-since": r.fetch.headers.Get("last-modified"),
- }
-
- // save headers to cache
- if err := cache.Set(r.fetch.src, headers); err != nil {
- sl.Error().Err(r.fetch.err).Msg("cache.Set")
- } else {
- // append to list of check URLs
- checkUrls = append(checkUrls, r.fetch.src)
- }
- }
- }
-
- // check size and hash in updated metas
- logArray("checkUrls", checkUrls)
- for _, metaUrl := range(checkUrls) {
- go ctx.check(metaUrl, metaUrls[metaUrl])
- }
-
- // build list of non-meta URLs to sync.
- syncUrls := make([]string, 0, len(metaUrls))
- syncUrls = append(syncUrls, config.GetCpeDictUrl())
-
- for range(checkUrls) {
- r := <-ctx.ch
-
- // create sublogger
- sl := log.With().
- Str("metaUrl", r.check.metaUrl).
- Str("metaPath", r.check.metaPath).
- Str("fullPath", r.check.fullPath).
- Logger()
-
- if r.check.err != nil {
- sl.Error().Err(r.check.err).Send()
- } else if r.check.match {
- sl.Debug().Msg("match")
- } else {
- syncUrls = append(syncUrls, metaUrls[r.check.metaUrl])
- }
- }
-
- logArray("syncUrls", syncUrls)
- for _, fullUrl := range(syncUrls) {
- go ctx.fetch(fullUrl)
- }
-
- // build list of changed files
- changed := make([]string, 0, len(syncUrls))
-
- // read sync results
- for range(syncUrls) {
- r := <-ctx.ch
- // build sublogger
- sl := log.With().Str("url", r.fetch.src).Logger()
-
- if r.fetch.err != nil {
- sl.Error().Err(r.fetch.err).Send()
- } else if !r.fetch.modified {
- sl.Debug().Msg("not modified")
- } else {
- sl.Debug().Msg("update")
-
- // build request headers
- headers := map[string]string {
- "if-none-match": r.fetch.headers.Get("etag"),
- "if-modified-since": r.fetch.headers.Get("last-modified"),
- }
-
- // save headers to cache
- if err := cache.Set(r.fetch.src, headers); err != nil {
- sl.Error().Err(r.fetch.err).Msg("cache.Set")
- } else {
- // append to list of changed files
- changed = append(changed, filepath.Base(r.fetch.src))
- }
- }
- }
+ // fetch updated meta files
+ checks := ctx.fetchMetas()
- // log changed files
- logArray("changed", changed)
+ // get syncable URLs
+ urls := append(ctx.checkMetas(checks), config.GetCpeDictUrl())
- // return success
- return changed
+ // sync urls and return changed files
+ return ctx.syncUrls(urls)
}