summaryrefslogtreecommitdiff
path: root/tests/cavp-tests/gen-main.rb
blob: 312262ecb38325f63e90cc7c0082faf6f0dd50c1 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
#!/usr/bin/env ruby
# frozen_string_literal: true

#
# gen-main.rb: Generate `main.c` for cavp-tests.  This script does the
# following:
#
# 1. Fetches the SHA-3 and SHAKE byte test vectors zip archives and
#    caches them in `./zip-cache`.
# 3. Verifies the SHA-256 digest of the downloaded archives.
# 3. Parses the response files (`.rsp`) in the downloaded zip archives
#    and extracts the SHA-3 and SHAKE test vectors.
# 4. Uses the test vectors to generate `main.c`.
# 5. Prints `main.c` to standard output.
#
# Usage:
#   cd tests/cavp-tests/
#
#   # download/cache archives, then generate main.c from them
#   ./gen-main.rb > main.c
#
#   # build and run tests
#   make && ./cavp-tests
#

# load libraries
require 'open-uri'
require 'openssl'
require 'uri'
require 'zip'

# number of elements per line of test data
PER_LINE = 128

# test vector sets
SETS = [{
  # URL to zip file containing test vectors for this set of algorithms
  url: 'https://csrc.nist.gov/CSRC/media/Projects/Cryptographic-Algorithm-Validation-Program/documents/sha3/sha-3bytetestvectors.zip',

  # expected sha256 digest of zip file
  hash: 'cd07701af2e47f5cc889d642528b4bf11f8b6eb55797c7307a96828ed8d8fc8c',

  # algorithms
  algos: [{
    name: 'SHA3-224', # algo name
    fn: 'sha3_224', # function prefix

    type: :hash, # function type
    size: 28, # output size, in bytes

    # response files
    rsp_files: %w{
      SHA3_224ShortMsg.rsp
      SHA3_224LongMsg.rsp
    },
  }, {
    name: 'SHA3-256', # algo name
    fn: 'sha3_256', # function prefix

    type: :hash, # function type
    size: 32, # output size, in bytes

    # response files
    rsp_files: %w{
      SHA3_256ShortMsg.rsp
      SHA3_256LongMsg.rsp
    },
  }, {
    name: 'SHA3-384', # algo name
    fn: 'sha3_384', # function prefix

    type: :hash, # function type
    size: 48, # output size, in bytes

    # response files
    rsp_files: %w{
      SHA3_384ShortMsg.rsp
      SHA3_384LongMsg.rsp
    },
  }, {
    name: 'SHA3-512', # algo name
    fn: 'sha3_512', # function prefix

    type: :hash, # function type
    size: 64, # output size, in bytes

    # response files
    rsp_files: %w{
      SHA3_512ShortMsg.rsp
      SHA3_512LongMsg.rsp
    },
  }],
}, {
  # URL to zip file containing test vectors for this set of algorithms
  url: 'https://csrc.nist.gov/CSRC/media/Projects/Cryptographic-Algorithm-Validation-Program/documents/sha3/shakebytetestvectors.zip',

  # expected sha256 digest of zip file
  hash: 'debfebc3157b3ceea002b84ca38476420389a3bf7e97dc5f53ea4689a16de4c7',

  # algorithms
  algos: [{
    name: 'SHAKE128', # algo name
    fn: 'shake128', # function prefix
    type: :xof, # function type

    # response files
    rsp_files: %w{
      SHAKE128ShortMsg.rsp
      SHAKE128LongMsg.rsp
      SHAKE128VariableOut.rsp
    },
  }, {
    name: 'SHAKE256', # algo name
    fn: 'shake256', # function prefix
    type: :xof, # function type

    # response files
    rsp_files: %w{
      SHAKE256ShortMsg.rsp
      SHAKE256LongMsg.rsp
      SHAKE256VariableOut.rsp
    },
  }],
}]

# main.c header
MAIN_HEAD = <<END_MAIN_HEAD
// Run test vectors for FIPS202 hash functions and XOFs.
// (generated by `gen-main.rb`)

#include <stdint.h> // uint8_t
#include <stdio.h> // printf()
#include <string.h> // memcmp()
#include "hex.h" // hex_write()
#include "sha3.h"

// test failure
typedef struct {
  const char *test_fn; // test function name
  size_t test_i; // test case index
  const char *rsp_file; // response file name
  size_t rsp_line; // start line of test in response file
  const uint8_t *got; // result
  size_t got_len; // result length
  const uint8_t *exp; // expected result
  size_t exp_len; // expected result length
} failure_t;

// print failure
static void fail(FILE *fh, const failure_t f) {
  fprintf(fh, "test_fn = %s\\n", f.test_fn);
  fprintf(fh, "rsp_file = %s\\n", f.rsp_file);
  fprintf(fh, "rsp_line = %zu\\n", f.rsp_line);
  fprintf(fh, "got (%zu) = ", f.got_len);
  hex_write(fh, f.got, f.got_len);
  fprintf(fh, "\\nexp (%zu) = ", f.exp_len);
  hex_write(fh, f.exp, f.exp_len);
  fprintf(fh, "\\n");
}
END_MAIN_HEAD

# main.c footer template
MAIN_TAIL_TMPL = <<END_MAIN_TAIL_TMPL
int main(void) {
%<fns>s
  return 0;
}
END_MAIN_TAIL_TMPL

# xof test vector template
XOF_TEST_TMPL = <<END_XOF_TEST_TMPL
    { %<file_pos>d, %<line>d, %<src_ofs>d, %<src_len>d, %<exp_ofs>d, %<exp_len>d },
END_XOF_TEST_TMPL

# xof test function template
XOF_FN_TMPL = <<END_XOF_FN_TMPL
// %<name>s tests from shakebytetestvectors.zip
static void test_%<fn>s(void) {
  // data for tests
  static const uint8_t DATA[] = {
%<data>s
  }; // total size = %<data_size>d

  // response file names
  static const char *RSP_FILES[] = { %<rsp_files>s };

  // test vectors
  static const struct {
    const size_t rsp_file_ofs, // offset of source file name in RSP_FILES
                 rsp_file_line, // line within rsp file
                 src_ofs, // message offset into DATA
                 src_len, // message length of data
                 exp_ofs, // expected output offset into DATA
                 exp_len; // expected output length
  } TESTS[] = {
%<tests>s
  };

  // run tests
  for (size_t i = 0; i < sizeof(TESTS) / sizeof(TESTS[0]); i++) {
    // get expected result
    const uint8_t * const exp = DATA + TESTS[i].exp_ofs;
    const size_t exp_len = TESTS[i].exp_len;

    // hash data into "got"
    uint8_t got[%<max_exp_len>d] = { 0 };
    %<fn>s(DATA + TESTS[i].src_ofs, TESTS[i].src_len, got, exp_len);

    // check for expected result
    if (memcmp(got, exp, exp_len)) {
      fail(stderr, (failure_t) {
        .test_fn = __func__, // test function
        .test_i = i, // test case index in TESTS
        .rsp_file = RSP_FILES[TESTS[i].rsp_file_ofs], // response file
        .rsp_line = TESTS[i].rsp_file_line, // start line of test case
        .got = got, // result
        .got_len = exp_len, // result length, in bytes
        .exp = exp, // expected result
        .exp_len = exp_len, // expected result length, in bytes
      });
    }
  }
}
END_XOF_FN_TMPL

# hash test vector template
HASH_TEST_TMPL = <<END_XOF_TEST_TMPL
    { %<file_pos>d, %<line>d, %<src_ofs>d, %<src_len>d, %<exp_ofs>d },
END_XOF_TEST_TMPL

# hash test function template
HASH_FN_TMPL = <<END_XOF_FN_TMPL
// %<name>s tests
static void test_%<fn>s(void) {
  // data for tests
  static const uint8_t DATA[] = {
%<data>s
  }; // total size = %<data_size>d

  // response file names
  static const char *RSP_FILES[] = { %<rsp_files>s };

  // test vectors
  static const struct {
    const size_t rsp_file_ofs, // offset of source file name in RSP_FILES
                 rsp_file_line, // line within rsp file
                 src_ofs, // message offset into DATA
                 src_len, // message length of data
                 exp_ofs; // expected output offset into DATA
  } TESTS[] = {
%<tests>s
  };

  // run tests
  for (size_t i = 0; i < sizeof(TESTS) / sizeof(TESTS[0]); i++) {
    // get expected result
    const uint8_t * const exp = DATA + TESTS[i].exp_ofs;

    // hash data into "got"
    uint8_t got[%<max_exp_len>d] = { 0 };
    %<fn>s(DATA + TESTS[i].src_ofs, TESTS[i].src_len, got);

    // check for expected result
    if (memcmp(got, exp, sizeof(got))) {
      fail(stderr, (failure_t) {
        .test_fn = __func__, // test function
        .test_i = i, // test case index in TESTS
        .rsp_file = RSP_FILES[TESTS[i].rsp_file_ofs], // response file
        .rsp_line = TESTS[i].rsp_file_line, // start line of test case
        .got = got, // result
        .got_len = sizeof(got), // result length, in bytes
        .exp = exp, // expected result
        .exp_len = sizeof(got), // expected result length, in bytes
      });
    }
  }
}
END_XOF_FN_TMPL

# parse hash of key/value pairs into test vector row
def parse(algo_type, file, h, data_size, &block)
  # parse hex-encoded message into array of bytes, then
  # truncate message to "Len" bytes, if "Len" is specified
  src = h[:Msg].scan(/../).map { |s| s.to_i(16) }
  src = src[0, h[:Len].to_i] if h.key?(:Len)
  src_ofs = data_size

  exp = case algo_type
  when :xof
    # parse hex-encoded expected output into array of bytes, then
    # truncate expected output to "OutputLen" bytes, if "OutputLen" is
    # specified
    exp = h[:Output].scan(/../).map { |s| s.to_i(16) }
    exp = exp[0, h[:OutputLen].to_i] if h.key?(:OutputLen)
    exp
  when :hash
    # get expected output
    h[:MD].scan(/../).map { |s| s.to_i(16) }
  else
    raise "unknown algorithm type: #{algo_type}"
  end
  exp_ofs = data_size + src.size

  # append message bytes and expected output bytes to data
  block.call(src + exp)

  # return parsed entry
  {
    file: file,
    line: h[:line],

    # message
    src_ofs: src_ofs,
    src_len: src.size,

    # expected output
    exp_ofs: exp_ofs,
    exp_len: exp.size,
  }
end

# build path to test vector archive cache
CACHE_DIR = File.join(__dir__, 'zip-cache')
Dir.mkdir(CACHE_DIR) unless Dir.exist?(CACHE_DIR)

# download test vector archives
SETS.map do |set|
  Thread.new(set) do |set|
    # parse url
    url = URI.parse(set[:url])

    # destination archive name
    zip_name = File.basename(url.path)

    # absolute path to destination archive
    zip_path = File.join(CACHE_DIR, zip_name)

    # download to destination path
    url.open { |src_io| IO.copy_stream(src_io, zip_path) } unless File.exist?(zip_path)

    # get sha256 hash of downloaded archive
    got_hash = ::OpenSSL::Digest::SHA256.new.file(zip_path).hexdigest

    # check against expected digest
    if got_hash != set[:hash]
      raise '%s: hash mismatch: got %s, exp %s' % [zip_name, got_hash, set[:hash]]
    end
  end
end.each { |t| t.join }

# print header
puts(MAIN_HEAD)

# generate test code
SETS.each do |set|
  # get absolute path to zip file
  zip_path = File.join(CACHE_DIR, File.basename(URI.parse(set[:url]).path))

  # open zip file
  Zip::File.open(zip_path, 'rb') do |zip|
    set[:algos].each do |algo|
      data = [] # test data (shared across all rsp files)

      # parse test vectors from rsp files
      rows = algo[:rsp_files].each_with_object([]) do |rsp_file, rows|
        curr = {} # current row

        # read lines from rsp file
        lines = zip.glob(rsp_file).first.get_input_stream.readlines.to_a.map { |line| line.strip }

        # parse lines into rows
        lines.size.times.each do |line_i|
          case lines[line_i]
          when /^(\w+) = (\w+)$/
            k, v = $1, $2 # extract key and value
            curr[k.intern] = v

            # cache line number
            curr[:line] = line_i unless curr[:line]
          when ''
            if curr.size > 0
              # parse current hash into test vector and add it to results
              rows << parse(algo[:type], rsp_file, curr, data.size) { |bytes| data += bytes }
            end

            curr = {} # clear current hash
          end
        end
      end

      # get maximum number of lines of emitted static DATA array
      num_lines = data.size / PER_LINE + ((data.size % PER_LINE) > 0 ? 1 : 0)

      # get maximum length of expected output
      # used to for size of "got" buffer in emitted code
      max_exp_len = case algo[:type]
      when :hash
        algo[:size]
      when :xof
        rows.reduce(0) { |r, row| row[:exp_len] > r ? row[:exp_len] : r }
      else
        raise "unknown algorithm type: #{algo[:type]}"
      end

      # get templates
      fn_tmpl, test_tmpl = case algo[:type]
      when :xof
        [XOF_FN_TMPL, XOF_TEST_TMPL]
      when :hash
        [HASH_FN_TMPL, HASH_TEST_TMPL]
      else
        raise "unknown algorithm type: #{algo[:type]}"
      end

      # generate test function
      puts(fn_tmpl % {
        name: algo[:name], # algorithm name
        fn: algo[:fn], # algorithm function prefix

        # test vector data
        data: num_lines.times.map { |ofs|
          "    %s,\n" % [data[PER_LINE * ofs, PER_LINE].join(', ')]
        }.join,

        # response file names
        rsp_files: algo[:rsp_files].map { |file| '"%s"' % [file] }.join(', '),

        data_size: data.size, # total size of emitted DATA array, in bytes
        max_exp_len: max_exp_len, # maximum expected output length, in bytes

        # test vectors
        tests: rows.map { |row|
          test_tmpl % row.merge({ file_pos: algo[:rsp_files].index(row[:file]) })
        }.join,
      })
    end
  end
end

# print main function body
puts(MAIN_TAIL_TMPL % {
  fns: SETS.each_with_object([]) do |set, r|
    set[:algos].each_with_object(r) do |algo, r|
      r << '  test_%<fn>s();' % algo
    end
  end.join("\n"),
})