From 98f1c3e7c38e94970a5a457a147c0fddf53fc026 Mon Sep 17 00:00:00 2001 From: Daira Hopwood Date: Thu, 29 Sep 2022 20:59:24 +0100 Subject: [PATCH 1/4] Move orchard_vesta.py to zcash_test_vectors/orchard/vesta.py and adjust imports. Signed-off-by: Daira Hopwood --- orchard_vesta.py => zcash_test_vectors/orchard/vesta.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename orchard_vesta.py => zcash_test_vectors/orchard/vesta.py (99%) diff --git a/orchard_vesta.py b/zcash_test_vectors/orchard/vesta.py similarity index 99% rename from orchard_vesta.py rename to zcash_test_vectors/orchard/vesta.py index e121866..881da40 100644 --- a/orchard_vesta.py +++ b/zcash_test_vectors/orchard/vesta.py @@ -2,8 +2,8 @@ # -*- coding: utf8 -*- import sys; assert sys.version_info[0] >= 3, "Python 3 required." -from sapling_jubjub import FieldElement -from utils import leos2ip +from ..sapling.jubjub import FieldElement +from ..utils import leos2ip q = 0x40000000000000000000000000000000224698fc0994a8dd8c46eb2100000001 p = 0x40000000000000000000000000000000224698fc094cf91b992d30ed00000001 From 9a39888f41d46ee4aeeb05bcf95717e76d24b3b6 Mon Sep 17 00:00:00 2001 From: Daira Hopwood Date: Thu, 29 Sep 2022 21:02:33 +0100 Subject: [PATCH 2/4] Add a pure Python implementation of RIPEMD-160 from Bitcoin: https://github.com/bitcoin/bitcoin/blob/ad3e9e1f214d739e098c6ebbd300da5df1026a44/test/functional/test_framework/ripemd160.py and use it instead of hashlib (which might not have RIPEMD-160 due to https://github.com/openssl/openssl/issues/16994). Signed-off-by: Daira Hopwood --- zcash_test_vectors/ripemd160.py | 130 +++++++++++++++++++++ zcash_test_vectors/transparent/bip_0032.py | 5 +- 2 files changed, 132 insertions(+), 3 deletions(-) create mode 100644 zcash_test_vectors/ripemd160.py diff --git a/zcash_test_vectors/ripemd160.py b/zcash_test_vectors/ripemd160.py new file mode 100644 index 0000000..eb9629f --- /dev/null +++ b/zcash_test_vectors/ripemd160.py @@ -0,0 +1,130 @@ +# Copyright (c) 2021 Pieter Wuille +# Distributed under the MIT software license, see the accompanying +# file COPYING or https://www.opensource.org/licenses/mit-license.php . +"""Test-only pure Python RIPEMD160 implementation.""" + +import unittest + +# Message schedule indexes for the left path. +ML = [ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 7, 4, 13, 1, 10, 6, 15, 3, 12, 0, 9, 5, 2, 14, 11, 8, + 3, 10, 14, 4, 9, 15, 8, 1, 2, 7, 0, 6, 13, 11, 5, 12, + 1, 9, 11, 10, 0, 8, 12, 4, 13, 3, 7, 15, 14, 5, 6, 2, + 4, 0, 5, 9, 7, 12, 2, 10, 14, 1, 3, 8, 11, 6, 15, 13 +] + +# Message schedule indexes for the right path. +MR = [ + 5, 14, 7, 0, 9, 2, 11, 4, 13, 6, 15, 8, 1, 10, 3, 12, + 6, 11, 3, 7, 0, 13, 5, 10, 14, 15, 8, 12, 4, 9, 1, 2, + 15, 5, 1, 3, 7, 14, 6, 9, 11, 8, 12, 2, 10, 0, 4, 13, + 8, 6, 4, 1, 3, 11, 15, 0, 5, 12, 2, 13, 9, 7, 10, 14, + 12, 15, 10, 4, 1, 5, 8, 7, 6, 2, 13, 14, 0, 3, 9, 11 +] + +# Rotation counts for the left path. +RL = [ + 11, 14, 15, 12, 5, 8, 7, 9, 11, 13, 14, 15, 6, 7, 9, 8, + 7, 6, 8, 13, 11, 9, 7, 15, 7, 12, 15, 9, 11, 7, 13, 12, + 11, 13, 6, 7, 14, 9, 13, 15, 14, 8, 13, 6, 5, 12, 7, 5, + 11, 12, 14, 15, 14, 15, 9, 8, 9, 14, 5, 6, 8, 6, 5, 12, + 9, 15, 5, 11, 6, 8, 13, 12, 5, 12, 13, 14, 11, 8, 5, 6 +] + +# Rotation counts for the right path. +RR = [ + 8, 9, 9, 11, 13, 15, 15, 5, 7, 7, 8, 11, 14, 14, 12, 6, + 9, 13, 15, 7, 12, 8, 9, 11, 7, 7, 12, 7, 6, 15, 13, 11, + 9, 7, 15, 11, 8, 6, 6, 14, 12, 13, 5, 14, 13, 13, 7, 5, + 15, 5, 8, 11, 14, 14, 6, 14, 6, 9, 12, 9, 12, 5, 15, 8, + 8, 5, 12, 9, 12, 5, 14, 6, 8, 13, 6, 5, 15, 13, 11, 11 +] + +# K constants for the left path. +KL = [0, 0x5a827999, 0x6ed9eba1, 0x8f1bbcdc, 0xa953fd4e] + +# K constants for the right path. +KR = [0x50a28be6, 0x5c4dd124, 0x6d703ef3, 0x7a6d76e9, 0] + + +def fi(x, y, z, i): + """The f1, f2, f3, f4, and f5 functions from the specification.""" + if i == 0: + return x ^ y ^ z + elif i == 1: + return (x & y) | (~x & z) + elif i == 2: + return (x | ~y) ^ z + elif i == 3: + return (x & z) | (y & ~z) + elif i == 4: + return x ^ (y | ~z) + else: + assert False + + +def rol(x, i): + """Rotate the bottom 32 bits of x left by i bits.""" + return ((x << i) | ((x & 0xffffffff) >> (32 - i))) & 0xffffffff + + +def compress(h0, h1, h2, h3, h4, block): + """Compress state (h0, h1, h2, h3, h4) with block.""" + # Left path variables. + al, bl, cl, dl, el = h0, h1, h2, h3, h4 + # Right path variables. + ar, br, cr, dr, er = h0, h1, h2, h3, h4 + # Message variables. + x = [int.from_bytes(block[4*i:4*(i+1)], 'little') for i in range(16)] + + # Iterate over the 80 rounds of the compression. + for j in range(80): + rnd = j >> 4 + # Perform left side of the transformation. + al = rol(al + fi(bl, cl, dl, rnd) + x[ML[j]] + KL[rnd], RL[j]) + el + al, bl, cl, dl, el = el, al, bl, rol(cl, 10), dl + # Perform right side of the transformation. + ar = rol(ar + fi(br, cr, dr, 4 - rnd) + x[MR[j]] + KR[rnd], RR[j]) + er + ar, br, cr, dr, er = er, ar, br, rol(cr, 10), dr + + # Compose old state, left transform, and right transform into new state. + return h1 + cl + dr, h2 + dl + er, h3 + el + ar, h4 + al + br, h0 + bl + cr + + +def ripemd160(data): + """Compute the RIPEMD-160 hash of data.""" + # Initialize state. + state = (0x67452301, 0xefcdab89, 0x98badcfe, 0x10325476, 0xc3d2e1f0) + # Process full 64-byte blocks in the input. + for b in range(len(data) >> 6): + state = compress(*state, data[64*b:64*(b+1)]) + # Construct final blocks (with padding and size). + pad = b"\x80" + b"\x00" * ((119 - len(data)) & 63) + fin = data[len(data) & ~63:] + pad + (8 * len(data)).to_bytes(8, 'little') + # Process final blocks. + for b in range(len(fin) >> 6): + state = compress(*state, fin[64*b:64*(b+1)]) + # Produce output. + return b"".join((h & 0xffffffff).to_bytes(4, 'little') for h in state) + + +class TestFrameworkKey(unittest.TestCase): + def test_ripemd160(self): + """RIPEMD-160 test vectors.""" + # See https://homes.esat.kuleuven.be/~bosselae/ripemd160.html + for msg, hexout in [ + (b"", "9c1185a5c5e9fc54612808977ee8f548b2258d31"), + (b"a", "0bdc9d2d256b3ee9daae347be6f4dc835a467ffe"), + (b"abc", "8eb208f7e05d987a9b044a8e98c6b087f15a0bfc"), + (b"message digest", "5d0689ef49d2fae572b881b123a85ffa21595f36"), + (b"abcdefghijklmnopqrstuvwxyz", + "f71c27109c692c1b56bbdceb5b9d2865b3708dbc"), + (b"abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq", + "12a053384a9c0c88e405a06c27dcf49ada62eb2b"), + (b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789", + "b0e20b6e3116640286ed3a87a5713079b21f5189"), + (b"1234567890" * 8, "9b752e45573d4b39f4dbd3323cab82bf63326bfb"), + (b"a" * 1000000, "52783243c1697bdbe16d37f97f68f08325dc1528") + ]: + self.assertEqual(ripemd160(msg).hex(), hexout) diff --git a/zcash_test_vectors/transparent/bip_0032.py b/zcash_test_vectors/transparent/bip_0032.py index 8b0efd0..afd9089 100644 --- a/zcash_test_vectors/transparent/bip_0032.py +++ b/zcash_test_vectors/transparent/bip_0032.py @@ -12,6 +12,7 @@ from ..hd_common import ZCASH_MAIN_COINTYPE, hardened from ..output import render_args, render_tv +from ..ripemd160 import ripemd160 from ..utils import i2beosp @@ -76,9 +77,7 @@ def __bytes__(self): return self.chaincode + self.pubkey_bytes() def address(self): - ripemd160 = hashlib.new('ripemd160') - ripemd160.update(hashlib.sha256(self.pubkey_bytes()).digest()) - return ripemd160.digest() + return ripemd160(hashlib.sha256(self.pubkey_bytes()).digest()) def child(self, i): assert 0 <= i and i <= 0xFFFFFFFF From 826e67c0edeb23306cd66dadda553905baddaa65 Mon Sep 17 00:00:00 2001 From: Daira Hopwood Date: Thu, 29 Sep 2022 21:04:14 +0100 Subject: [PATCH 3/4] Add a runner for unit tests. (This is currently only the RIPEMD-160 test.) Signed-off-by: Daira Hopwood --- README.md | 2 ++ pyproject.toml | 2 ++ test.py | 11 +++++++++++ 3 files changed, 15 insertions(+) create mode 100644 test.py diff --git a/README.md b/README.md index 0fae3b8..0872f2b 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,8 @@ after adjusting: - `poetry run SCRIPT_NAME [-t json|rust|zcash]` - `SCRIPT_NAME` is one of the scripts listed in `pyproject.toml`. +To run unit tests, use ``poetry run test``. + ## License Licensed under either of diff --git a/pyproject.toml b/pyproject.toml index 290a759..d0f96b2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -62,3 +62,5 @@ orchard_note_encryption = "zcash_test_vectors.orchard.note_encryption:main" orchard_poseidon = "zcash_test_vectors.orchard.poseidon:main" orchard_poseidon_hash = "zcash_test_vectors.orchard.poseidon:hash_test_vectors" orchard_sinsemilla = "zcash_test_vectors.orchard.sinsemilla:main" + +test = "test:test" diff --git a/test.py b/test.py new file mode 100644 index 0000000..8bfb934 --- /dev/null +++ b/test.py @@ -0,0 +1,11 @@ +# Copyright (c) 2022 The Zcash developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or https://www.opensource.org/licenses/mit-license.php . + +import unittest + +def test(): + loader = unittest.TestLoader() + suite = loader.discover('.', pattern='*.py') + runner = unittest.TextTestRunner() + runner.run(suite) From c019a2903e575f80a80698a2504a09f8059fc05c Mon Sep 17 00:00:00 2001 From: Daira Hopwood Date: Thu, 29 Sep 2022 21:05:16 +0100 Subject: [PATCH 4/4] Improve the usability of `regenerate.sh` and document it. Signed-off-by: Daira Hopwood --- README.md | 12 +++++++++--- regenerate.sh | 14 ++++++++++++-- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 0872f2b..7b5c551 100644 --- a/README.md +++ b/README.md @@ -3,10 +3,11 @@ Code to generate test vectors for various parts of Zcash. The generated test vectors are checked into the repository: -- `test-vectors/json/`: JSON format. -- `test-vectors/rust/`: Rust format, suitable for copying into a Rust library or + +- `test-vectors/json/*.json`: JSON format. +- `test-vectors/rust/*.rs`: Rust format, suitable for copying into a Rust library or application to use from `#[cfg(test)]` code. -- `test-vectors/zcash/`: Bitcoin-flavoured JSON format (where 256-bit values are +- `test-vectors/zcash/*.json`: Bitcoin-flavoured JSON format (where 256-bit values are encoded as byte-reversed hex strings), for use in `zcashd` unit tests. To generate the test vectors yourself (for example, to generate a larger set @@ -14,6 +15,11 @@ after adjusting: - Install [`poetry`](https://python-poetry.org/). - `poetry install` +- `./regenerate.sh ` + - `` defaults to `rust json zcash`. + +To generate an individual output file: + - `poetry run SCRIPT_NAME [-t json|rust|zcash]` - `SCRIPT_NAME` is one of the scripts listed in `pyproject.toml`. diff --git a/regenerate.sh b/regenerate.sh index 880e326..f5e9609 100755 --- a/regenerate.sh +++ b/regenerate.sh @@ -1,5 +1,7 @@ #!/usr/bin/env bash +poetry install -q + tv_scripts=( bip_0032 f4jumble @@ -27,8 +29,16 @@ tv_scripts=( zip_0244 zip_0316) +formats="${*:-rust json zcash}" + for generator in "${tv_scripts[@]}" do - echo "# $generator" - poetry run $generator -t $1 >test-vectors/$1/$generator.$2 + for format in $formats + do + filetype="${format/rust/rs}" + filetype="${filetype/zcash/json}" + output_file="test-vectors/$format/$generator.$filetype" + echo $output_file + poetry run $generator -t $format >$output_file + done done