From 1f021905b3a6bcdd778b2d433626c3ac6de36c95 Mon Sep 17 00:00:00 2001 From: Lydia Garms Date: Wed, 27 Nov 2024 18:15:34 +0000 Subject: [PATCH 1/2] comments for hyperplonk --- plonkish_backend/src/backend.rs | 2 + plonkish_backend/src/backend/hyperplonk.rs | 24 ++++++++++- .../src/backend/hyperplonk/preprocessor.rs | 40 ++++++++++++++++++- .../src/backend/hyperplonk/prover.rs | 23 +++++++++-- plonkish_backend/src/util/expression.rs | 4 +- 5 files changed, 87 insertions(+), 6 deletions(-) diff --git a/plonkish_backend/src/backend.rs b/plonkish_backend/src/backend.rs index 462e0483..01a4a089 100644 --- a/plonkish_backend/src/backend.rs +++ b/plonkish_backend/src/backend.rs @@ -107,12 +107,14 @@ impl PlonkishCircuitInfo { .unwrap_or(true) } + //Total number of columns/ polynomials in the circuit. pub fn num_poly(&self) -> usize { self.num_instances.len() + self.preprocess_polys.len() + self.num_witness_polys.iter().sum::() } + //Return all index of columns that are used in the permutation, in order. pub fn permutation_polys(&self) -> Vec { self.permutations .iter() diff --git a/plonkish_backend/src/backend/hyperplonk.rs b/plonkish_backend/src/backend/hyperplonk.rs index 363ea533..48c8296a 100644 --- a/plonkish_backend/src/backend/hyperplonk.rs +++ b/plonkish_backend/src/backend/hyperplonk.rs @@ -71,6 +71,7 @@ where pub(crate) num_lookups: usize, pub(crate) num_permutation_z_polys: usize, pub(crate) num_vars: usize, + // Expression for the constraint system pub(crate) expression: Expression, pub(crate) preprocess_comms: Vec, pub(crate) permutation_comms: Vec<(usize, Pcs::Commitment)>, @@ -85,6 +86,7 @@ where type ProverParam = HyperPlonkProverParam; type VerifierParam = HyperPlonkVerifierParam; + //TO DO - for KZG we want to use a trusted setup from a ceremony as currently generated in nightfish/ nightfall fn setup( circuit_info: &PlonkishCircuitInfo, rng: impl RngCore, @@ -92,7 +94,9 @@ where assert!(circuit_info.is_well_formed()); let num_vars = circuit_info.k; + // number of variables in the multilinear polynomial let poly_size = 1 << num_vars; + // set batch size for polynomial commitment scheme let batch_size = batch_size(circuit_info); Pcs::setup(poly_size, batch_size, rng) } @@ -115,12 +119,14 @@ where ) -> Result<(), Error> { let instance_polys = { let instances = circuit.instances(); + //Check there is the correct amount of instances and write them to the transcript for (num_instances, instances) in pp.num_instances.iter().zip_eq(instances) { assert_eq!(instances.len(), *num_instances); for instance in instances.iter() { transcript.common_field_element(instance)?; } } + // Create multi-linear polynomials from the instance columns instance_polys::<_, BinaryField>(pp.num_vars, instances) }; @@ -129,6 +135,7 @@ where let mut witness_polys = Vec::with_capacity(pp.num_witness_polys.iter().sum()); let mut witness_comms = Vec::with_capacity(witness_polys.len()); let mut challenges = Vec::with_capacity(pp.num_challenges.iter().sum::() + 4); + // For each round, generate multi-linear polynomials from witness columns and commit for (round, (num_witness_polys, num_challenges)) in pp .num_witness_polys .iter() @@ -152,8 +159,10 @@ where // Round n + // beta is used to compress the polynomials in the lookup argument let beta = transcript.squeeze_challenge(); + // Generate a compressed multilinear polynomial for each lookup in the vector of lookups let timer = start_timer(|| format!("lookup_compressed_polys-{}", pp.lookups.len())); let lookup_compressed_polys = { let max_lookup_width = pp.lookups.iter().map(Vec::len).max().unwrap_or_default(); @@ -162,6 +171,8 @@ where }; end_timer(timer); + // m and h are the polynomials generated as part of the logup GKR protocol + // if the lookups are f (input), t (table), m[i] is |j \in 2^{numvars} s.t f[j] = t[i] |, i.e. the number of times t[i] appears in f let timer = start_timer(|| format!("lookup_m_polys-{}", pp.lookups.len())); let lookup_m_polys = lookup_m_polys(&lookup_compressed_polys)?; end_timer(timer); @@ -172,6 +183,7 @@ where let gamma = transcript.squeeze_challenge(); + //h = (gamma + input)^-1 - m * (gamma + table)^-1 let timer = start_timer(|| format!("lookup_h_polys-{}", pp.lookups.len())); let lookup_h_polys = lookup_h_polys(&lookup_compressed_polys, &lookup_m_polys, &gamma); end_timer(timer); @@ -186,6 +198,7 @@ where ); end_timer(timer); + // Commit to h polynomiald and permutation z polynomials let lookup_h_permutation_z_polys = chain![lookup_h_polys.iter(), permutation_z_polys.iter()].collect_vec(); let lookup_h_permutation_z_comms = @@ -196,6 +209,7 @@ where let alpha = transcript.squeeze_challenge(); let y = transcript.squeeze_challenges(pp.num_vars); + // All of the polynomials in the trace committed to in the transcript let polys = chain![ polys, pp.permutation_polys.iter().map(|(_, poly)| poly), @@ -204,6 +218,7 @@ where ] .collect_vec(); challenges.extend([beta, gamma, alpha]); + // Prove the zero check is satisfied for the expression wrt the polynomials let (points, evals) = prove_zero_check( pp.num_instances.len(), &pp.expression, @@ -226,9 +241,10 @@ where ] .collect_vec(); let timer = start_timer(|| format!("pcs_batch_open-{}", evals.len())); + // Open all polynomials at the points from the zero check and give the opening proofs Pcs::batch_open(&pp.pcs, polys, comms, &points, &evals, transcript)?; end_timer(timer); - + // Proof is saved in transcript Ok(()) } @@ -238,6 +254,7 @@ where transcript: &mut impl TranscriptRead, _: impl RngCore, ) -> Result<(), Error> { + //Check there is the correct amount of instances and write them to the transcript for (num_instances, instances) in vp.num_instances.iter().zip_eq(instances) { assert_eq!(instances.len(), *num_instances); for instance in instances.iter() { @@ -247,6 +264,7 @@ where // Round 0..n + // For each round, read the witness commitments from the transcript and generate the challenges let mut witness_comms = Vec::with_capacity(vp.num_witness_polys.iter().sum()); let mut challenges = Vec::with_capacity(vp.num_challenges.iter().sum::() + 4); for (num_polys, num_challenges) in @@ -260,12 +278,14 @@ where let beta = transcript.squeeze_challenge(); + // Read the commitments to the m polynomials from the lookup arguments let lookup_m_comms = Pcs::read_commitments(&vp.pcs, vp.num_lookups, transcript)?; // Round n+1 let gamma = transcript.squeeze_challenge(); + // Read the commitments to the h polynomials and permutation z polynomials let lookup_h_permutation_z_comms = Pcs::read_commitments( &vp.pcs, vp.num_lookups + vp.num_permutation_z_polys, @@ -278,6 +298,7 @@ where let y = transcript.squeeze_challenges(vp.num_vars); challenges.extend([beta, gamma, alpha]); + // Verify the zero check for the constraints defined in the expression let (points, evals) = verify_zero_check( vp.num_vars, &vp.expression, @@ -299,6 +320,7 @@ where &lookup_h_permutation_z_comms, ] .collect_vec(); + // Verify the opening proofs for the polynomials commitments Pcs::batch_verify(&vp.pcs, comms, &points, &evals, transcript)?; Ok(()) diff --git a/plonkish_backend/src/backend/hyperplonk/preprocessor.rs b/plonkish_backend/src/backend/hyperplonk/preprocessor.rs index 5ce57289..6aa7bc05 100644 --- a/plonkish_backend/src/backend/hyperplonk/preprocessor.rs +++ b/plonkish_backend/src/backend/hyperplonk/preprocessor.rs @@ -46,7 +46,9 @@ pub(crate) fn preprocess>( let num_vars = circuit_info.k; let poly_size = 1 << num_vars; + // Batch size for the polynomial commitment scheme let batch_size = batch_size(circuit_info); + // Trim the parameters for the PCS to those necessary for the size of the circuit let (pcs_pp, pcs_vp) = Pcs::trim(param, poly_size, batch_size)?; // Compute preprocesses comms @@ -56,6 +58,7 @@ pub(crate) fn preprocess>( .cloned() .map(MultilinearPolynomial::new) .collect_vec(); + // Batch commit to all pre-processing polynomials - i.e. fixed colummns/ selector columns let (preprocess_polys, preprocess_comms) = batch_commit(&pcs_pp, preprocess_polys)?; // Compute permutation polys and comms @@ -66,8 +69,9 @@ pub(crate) fn preprocess>( ); let (permutation_polys, permutation_comms) = batch_commit(&pcs_pp, permutation_polys)?; - // Compose expression + // Compose an expression for all the constraints let (num_permutation_z_polys, expression) = compose(circuit_info); + // Setup parameters for verifier and prover let vp = HyperPlonkVerifierParam { pcs: pcs_vp, num_instances: circuit_info.num_instances.clone(), @@ -105,16 +109,21 @@ pub(crate) fn preprocess>( Ok((pp, vp)) } +// compose all constraints pub(crate) fn compose( circuit_info: &PlonkishCircuitInfo, ) -> (usize, Expression) { + //total number of challenges let challenge_offset = circuit_info.num_challenges.iter().sum::(); + // Generates three extra challenges beta, gamma, alpha let [beta, gamma, alpha] = &array::from_fn(|idx| Expression::::Challenge(challenge_offset + idx)); + // lookup_zero_checks are the sumcheck constraints in the logup GKR protocol let (lookup_constraints, lookup_zero_checks) = lookup_constraints(circuit_info, beta, gamma); let max_degree = max_degree(circuit_info, Some(&lookup_constraints)); + // Generate constraints for the permuation argument let (num_permutation_z_polys, permutation_constraints) = permutation_constraints( circuit_info, max_degree, @@ -125,8 +134,11 @@ pub(crate) fn compose( let expression = { let constraints = chain![ + // constraints from halo2 frontend , i.e. custom gates circuit_info.constraints.iter(), + // constraints from lookup argument lookup_constraints.iter(), + // constraints from permutation argument permutation_constraints.iter(), ] .collect_vec(); @@ -159,11 +171,13 @@ pub(super) fn max_degree( .unwrap() } +//generate lookup constraints using logup GKR pub(super) fn lookup_constraints( circuit_info: &PlonkishCircuitInfo, beta: &Expression, gamma: &Expression, ) -> (Vec>, Vec>) { + // Define indices where m and h polynomials begin in the trace let m_offset = circuit_info.num_poly() + circuit_info.permutation_polys().len(); let h_offset = m_offset + circuit_info.lookups.len(); let constraints = circuit_info @@ -172,18 +186,23 @@ pub(super) fn lookup_constraints( .zip(m_offset..) .zip(h_offset..) .flat_map(|((lookup, m), h)| { + // make m and h into polynomials, these are created during proving let [m, h] = &[m, h] .map(|poly| Query::new(poly, Rotation::cur())) .map(Expression::::Polynomial); + // separate the input and tables from the lookup let (inputs, tables) = lookup .iter() .map(|(input, table)| (input, table)) .unzip::<_, _, Vec<_>, Vec<_>>(); + // Returns a distributed power expression for the input and table, with base beta, i.e. inputs[0] + \beta inputs[1] + \beta^2 inputs[2] + ... let input = &Expression::distribute_powers(inputs, beta); let table = &Expression::distribute_powers(tables, beta); + // h[i] = (gamma + input[i])^-1 - m[i] * (gamma + table[i])^-1 [h * (input + gamma) * (table + gamma) - (table + gamma) + m * (input + gamma)] }) .collect_vec(); + // Every expression that must be proved in the sum check argument let sum_check = (h_offset..) .take(circuit_info.lookups.len()) .map(|h| Query::new(h, Rotation::cur()).into()) @@ -191,6 +210,7 @@ pub(super) fn lookup_constraints( (constraints, sum_check) } +// create constraints for the permutation argument pub(crate) fn permutation_constraints( circuit_info: &PlonkishCircuitInfo, max_degree: usize, @@ -198,40 +218,53 @@ pub(crate) fn permutation_constraints( gamma: &Expression, num_builtin_witness_polys: usize, ) -> (usize, Vec>) { + // get index of all columns used in the permutation let permutation_polys = circuit_info.permutation_polys(); let chunk_size = max_degree - 1; + // If there are more columns than max degree split into chunks, num_chunks corresponds to b in halo2 gitbook let num_chunks = div_ceil(permutation_polys.len(), chunk_size); + // The offset is set to the total number of instance columns in the circuit let permutation_offset = circuit_info.num_poly(); let z_offset = permutation_offset + permutation_polys.len() + num_builtin_witness_polys; + // Represent all columns in permutation argument with polynomials let polys = permutation_polys .iter() .map(|idx| Expression::Polynomial(Query::new(*idx, Rotation::cur()))) .collect_vec(); + //ids_i(X) = i 2^k + X let ids = (0..polys.len()) .map(|idx| { let offset = F::from((idx << circuit_info.k) as u64); Expression::Constant(offset) + Expression::identity() }) .collect_vec(); + // Create the polynomials to represent the permutation columns let permutations = (permutation_offset..) .map(|idx| Expression::Polynomial(Query::new(idx, Rotation::cur()))) .take(permutation_polys.len()) .collect_vec(); + // Represents Z polynomials from the permutation argument let zs = (z_offset..) .map(|idx| Expression::Polynomial(Query::new(idx, Rotation::cur()))) .take(num_chunks) .collect_vec(); + // Z_0(shift(X)) let z_0_next = Expression::::Polynomial(Query::new(z_offset, Rotation::next())); let l_0 = &Expression::::lagrange(0); let one = &Expression::one(); + // Create the constraints for the permutation argument + // The contraints here are the like those from the halo2 gitbook but the matrix Z_0 Z_1 ... Z_{b-1} is transposed let constraints = chain![ zs.first().map(|z_0| l_0 * (z_0 - one)), polys + //iterating over b elements which are vectors of length m .chunks(chunk_size) .zip(ids.chunks(chunk_size)) .zip(permutations.chunks(chunk_size)) .zip(zs.iter()) .zip(zs.iter().skip(1).chain([&z_0_next])) + // z_a prod_{am)}^{(a+1)m-1}(poly_i + beta * id_i + gamma) - z_{a+1} prod_{am)}^{(a+1)m-1}(poly_i + beta * permutation_i + gamma) + // z_{b-1} prod_{(b-1)m)}^{bm-1}(poly_{b-1} + beta * id_{b-1} + gamma) - z_0(shift(X)) prod_{(b-1)m)}^{bm-1}(poly_{b-1} + beta * permutation_{b-1} + gamma) .map(|((((polys, ids), permutations), z_lhs), z_rhs)| { z_lhs * polys @@ -251,11 +284,13 @@ pub(crate) fn permutation_constraints( (num_chunks, constraints) } +// Generate multi-linear permutation polynomials for permutation argument pub(crate) fn permutation_polys( num_vars: usize, permutation_polys: &[usize], cycles: &[Vec<(usize, usize)>], ) -> Vec> { + // The index of an element in permutation_polys let poly_index = { let mut poly_index = vec![0; permutation_polys.last().map(|poly| 1 + poly).unwrap_or(0)]; for (idx, poly) in permutation_polys.iter().enumerate() { @@ -263,6 +298,7 @@ pub(crate) fn permutation_polys( } poly_index }; + // permutations will be the matrix defining all permutation polynomials. As we start with the identity permutation, all entries have value of the index within the matrix. let mut permutations = (0..permutation_polys.len() as u64) .map(|idx| { steps(F::from(idx << num_vars)) @@ -270,6 +306,7 @@ pub(crate) fn permutation_polys( .collect_vec() }) .collect_vec(); + // For each cycle we update the permutation matrix. For each entry in the matrix, we have the location of the next element in the cycle. for cycle in cycles.iter() { let (i0, j0) = cycle[0]; let mut last = permutations[poly_index[i0]][j0]; @@ -277,6 +314,7 @@ pub(crate) fn permutation_polys( mem::swap(&mut permutations[poly_index[i]][j], &mut last); } } + // We generate a multilinear polynomial from each column of the permutation matrix. permutations .into_iter() .map(MultilinearPolynomial::new) diff --git a/plonkish_backend/src/backend/hyperplonk/prover.rs b/plonkish_backend/src/backend/hyperplonk/prover.rs index 72ffeb56..18c1fa24 100644 --- a/plonkish_backend/src/backend/hyperplonk/prover.rs +++ b/plonkish_backend/src/backend/hyperplonk/prover.rs @@ -26,6 +26,7 @@ use std::{ hash::Hash, }; +// Generate multi-linear polynomial from the instance columns pub(crate) fn instance_polys<'a, F: PrimeField, R: Rotatable + From>( num_vars: usize, instances: impl IntoIterator>, @@ -56,6 +57,7 @@ pub(crate) fn lookup_compressed_polys> let polys = polys.iter().map(Borrow::borrow).collect_vec(); let num_vars = polys[0].num_vars(); + // This is the sum of all elements in the input and table let expression = lookups .iter() .flat_map(|lookup| lookup.iter().map(|(input, table)| (input + table))) @@ -91,6 +93,7 @@ pub(super) fn lookup_compressed_poly>( let mut compressed = vec![F::ZERO; 1 << num_vars]; parallelize(&mut compressed, |(compressed, start)| { for (b, compressed) in (start..).zip(compressed) { + // evaluation expression on b *compressed = expression.evaluate( &|constant| constant, &|common_poly| match common_poly { @@ -113,11 +116,14 @@ pub(super) fn lookup_compressed_poly>( ); } }); + // Generate a multi-linear polynomial from each expression MultilinearPolynomial::new(compressed) })) + // Generate a compressed multi-linear polynomial by combining all the multi-linear polynomials, scaled with powers of beta .sum::>() }; + // split inputs and tables into separate vectors let (inputs, tables) = lookup .iter() .map(|(input, table)| (input, table)) @@ -146,6 +152,7 @@ pub(super) fn lookup_m_poly( let [input, table] = compressed_polys; let counts = { + // Maps each element from the table to its index let indice_map = table.iter().zip(0..).collect::>(); let chunk_size = div_ceil(input.evals().len(), num_threads()); @@ -159,12 +166,14 @@ pub(super) fn lookup_m_poly( .zip((0..).step_by(chunk_size)), |((count, valid), start)| { for input in input[start..].iter().take(chunk_size) { + // Finds index of the input in the table if let Some(idx) = indice_map.get(input) { count .entry(*idx) .and_modify(|count| *count += 1) .or_insert(1); } else { + // If the input is not found in the table, the lookup is invalid *valid = false; break; } @@ -210,6 +219,7 @@ pub(super) fn lookup_h_poly( let mut h_input = vec![F::ZERO; 1 << input.num_vars()]; let mut h_table = vec![F::ZERO; 1 << input.num_vars()]; + // set h_input and h_table to gamma + input and gamma + table parallelize(&mut h_input, |(h_input, start)| { for (h_input, input) in h_input.iter_mut().zip(input[start..].iter()) { *h_input = *gamma + input; @@ -221,6 +231,7 @@ pub(super) fn lookup_h_poly( } }); + // invert every element in h_input and h_table let chunk_size = div_ceil(2 * h_input.len(), num_threads()); parallelize_iter( chain![ @@ -232,6 +243,7 @@ pub(super) fn lookup_h_poly( }, ); + //h[i] = (gamma + input[i])^-1 - m[i] * (gamma + table[i])^-1 parallelize(&mut h_input, |(h_input, start)| { for (h_input, (h_table, m)) in h_input .iter_mut() @@ -248,6 +260,7 @@ pub(super) fn lookup_h_poly( MultilinearPolynomial::new(h_input) } +// Generates equality constraint polynomials pub(crate) fn permutation_z_polys>( num_chunks: usize, permutation_polys: &[(usize, MultilinearPolynomial)], @@ -258,7 +271,7 @@ pub(crate) fn permutation_z_polys>( if permutation_polys.is_empty() { return Vec::new(); } - + // permutation polys are split into chunks due to maximum constraint degree let chunk_size = div_ceil(permutation_polys.len(), num_chunks); let polys = polys.iter().map(Borrow::borrow).collect_vec(); let num_vars = polys[0].num_vars(); @@ -270,6 +283,7 @@ pub(crate) fn permutation_z_polys>( .map(|(chunk_idx, permutation_polys)| { let mut product = vec![F::ONE; 1 << num_vars]; + //product poly = product over i of beta permutation_i + gamma + value_i, where i is in that chunk, where permutation is the permutation polynomial and value is the polynomial for the relevant instance/ witness column for (poly, permutation_poly) in permutation_polys.iter() { parallelize(&mut product, |(product, start)| { for ((product, value), permutation) in product @@ -281,11 +295,11 @@ pub(crate) fn permutation_z_polys>( } }); } - + //product = product over i of (beta permutation_i + gamma + value_i)^-1, where i is in that chunk parallelize(&mut product, |(product, _)| { product.batch_invert(); }); - + //product = product over i of (beta * id_i + gamma + value_i) (beta permutation_i + gamma + value_i)^-1, where id_i polynomial returns label for ((poly, _), idx) in permutation_polys.iter().zip(chunk_idx * chunk_size..) { let id_offset = idx << num_vars; parallelize(&mut product, |(product, start)| { @@ -309,6 +323,8 @@ pub(crate) fn permutation_z_polys>( let usable_indices = R::from(num_vars).usable_indices(); let first_idx = usable_indices[0]; + // Generate polynomisals z_0, .., z_{b-1} such that z_i(X) = z_{i-1}(X) * product_i(X) and z_0(wX) = z_{b-1}(X) * product_{b-1}(X) + // The is the transpose of the permuation argument from the halo2 gitbook z[0][first_idx] = F::ONE; for chunk_idx in 1..num_chunks { z[chunk_idx][first_idx] = z[chunk_idx - 1][first_idx] * products[chunk_idx - 1][first_idx]; @@ -372,6 +388,7 @@ pub(crate) fn prove_sum_check( transcript, )?; + // Set of all polynomial queries in the expression let pcs_query = pcs_query(expression, num_instance_poly); let point_offset = point_offset(&pcs_query); diff --git a/plonkish_backend/src/util/expression.rs b/plonkish_backend/src/util/expression.rs index 775b70be..ff727eb7 100644 --- a/plonkish_backend/src/util/expression.rs +++ b/plonkish_backend/src/util/expression.rs @@ -15,6 +15,8 @@ pub mod rotate; pub use rotate::Rotation; #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] + +// This describes a query to a polynomial given by the index of the correponding column in the trace. pub struct Query { poly: usize, rotation: Rotation, @@ -116,7 +118,7 @@ impl Expression { ) }; match self { - Expression::Constant(scalar) => constant(scalar.clone()), + Expression::Constant(scalar) => constant(scalar.clone()), Expression::CommonPolynomial(poly) => common_poly(*poly), Expression::Polynomial(query) => poly(*query), Expression::Challenge(index) => challenge(*index), From c2515b08be41e61aea11bb444f2187e4324b7b56 Mon Sep 17 00:00:00 2001 From: Lydia Garms Date: Thu, 28 Nov 2024 14:50:17 +0000 Subject: [PATCH 2/2] comment --- plonkish_backend/src/backend/hyperplonk/prover.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plonkish_backend/src/backend/hyperplonk/prover.rs b/plonkish_backend/src/backend/hyperplonk/prover.rs index 18c1fa24..52fd4462 100644 --- a/plonkish_backend/src/backend/hyperplonk/prover.rs +++ b/plonkish_backend/src/backend/hyperplonk/prover.rs @@ -323,7 +323,7 @@ pub(crate) fn permutation_z_polys>( let usable_indices = R::from(num_vars).usable_indices(); let first_idx = usable_indices[0]; - // Generate polynomisals z_0, .., z_{b-1} such that z_i(X) = z_{i-1}(X) * product_i(X) and z_0(wX) = z_{b-1}(X) * product_{b-1}(X) + // Generate polynomisals z_0, .., z_{b-1} such that z_i(X) = z_{i-1}(X) * product_i(X) and z_0(shift(X) = z_{b-1}(X) * product_{b-1}(X) // The is the transpose of the permuation argument from the halo2 gitbook z[0][first_idx] = F::ONE; for chunk_idx in 1..num_chunks {