diff --git a/crates/pallet-subspace/src/mock.rs b/crates/pallet-subspace/src/mock.rs index 9404740f834..334629e8778 100644 --- a/crates/pallet-subspace/src/mock.rs +++ b/crates/pallet-subspace/src/mock.rs @@ -39,11 +39,11 @@ use std::num::NonZeroU64; use std::sync::Once; use subspace_archiving::archiver::{ArchivedSegment, Archiver}; use subspace_core_primitives::crypto::kzg::{embedded_kzg_settings, Kzg, Witness}; -use subspace_core_primitives::crypto::{blake2b_256_254_hash, kzg}; +use subspace_core_primitives::crypto::{blake2b_256_254_hash_to_scalar, kzg, ScalarLegacy}; use subspace_core_primitives::{ ArchivedBlockProgress, Blake2b256Hash, LastArchivedBlock, Piece, Randomness, RecordsRoot, - RootBlock, Scalar, SegmentIndex, Solution, SolutionRange, PIECE_SIZE, - RECORDED_HISTORY_SEGMENT_SIZE, RECORD_SIZE, + RootBlock, SegmentIndex, Solution, SolutionRange, PIECE_SIZE, RECORDED_HISTORY_SEGMENT_SIZE, + RECORD_SIZE, }; use subspace_solving::{create_chunk_signature, derive_global_challenge, REWARD_SIGNING_CONTEXT}; @@ -263,12 +263,12 @@ pub fn generate_equivocation_proof( let current_slot = CurrentSlot::::get(); let chunk = { - let mut chunk_bytes = [0; Scalar::SAFE_BYTES]; + let mut chunk_bytes = [0; ScalarLegacy::SAFE_BYTES]; chunk_bytes.as_mut().iter_mut().for_each(|byte| { *byte = (current_block % 8) as u8; }); - Scalar::from(&chunk_bytes) + ScalarLegacy::from(&chunk_bytes) }; let public_key = FarmerPublicKey::unchecked_from(keypair.public.to_bytes()); @@ -383,7 +383,7 @@ pub fn create_signed_vote( sector_index: 0, total_pieces: NonZeroU64::new(1).unwrap(), piece_offset: 0, - piece_record_hash: blake2b_256_254_hash(&piece.record()), + piece_record_hash: blake2b_256_254_hash_to_scalar(&piece.record()), piece_witness: Witness::try_from_bytes(&piece.witness()).unwrap(), chunk_offset: 0, chunk, diff --git a/crates/subspace-archiving/src/archiver.rs b/crates/subspace-archiving/src/archiver.rs index 21ebfd7f3c2..be902d89e29 100644 --- a/crates/subspace-archiving/src/archiver.rs +++ b/crates/subspace-archiving/src/archiver.rs @@ -29,14 +29,14 @@ use alloc::vec::Vec; use core::cmp::Ordering; use parity_scale_codec::{Compact, CompactLen, Decode, Encode, Input, Output}; use reed_solomon_erasure::galois_16::ReedSolomon; -use subspace_core_primitives::crypto::blake2b_256_254_hash; use subspace_core_primitives::crypto::kzg::{Kzg, Witness}; +use subspace_core_primitives::crypto::{blake2b_256_254_hash_to_scalar, Scalar}; use subspace_core_primitives::objects::{ BlockObject, BlockObjectMapping, PieceObject, PieceObjectMapping, }; use subspace_core_primitives::{ ArchivedBlockProgress, Blake2b256Hash, BlockNumber, FlatPieces, LastArchivedBlock, PieceRef, - RecordsRoot, RootBlock, BLAKE2B_256_HASH_SIZE, RECORDED_HISTORY_SEGMENT_SIZE, + RecordsRoot, RootBlock, RECORDED_HISTORY_SEGMENT_SIZE, }; const INITIAL_LAST_ARCHIVED_BLOCK: LastArchivedBlock = LastArchivedBlock { @@ -758,24 +758,13 @@ impl Archiver { .as_ref() .chunks_exact(self.record_size as usize) .skip(self.data_shards as usize) - .map(blake2b_256_254_hash), + .map(blake2b_256_254_hash_to_scalar), ) .collect::>(); - let data = { - let mut data = Vec::with_capacity( - (self.data_shards + self.parity_shards) as usize * BLAKE2B_256_HASH_SIZE, - ); - - for shard_commitment in record_commitments { - data.extend_from_slice(&shard_commitment); - } - - data - }; let polynomial = self .kzg - .poly(&data) + .poly(&record_commitments) .expect("Internally produced values must never fail; qed"); let commitment = self .kzg @@ -846,7 +835,7 @@ pub fn is_piece_valid( return false; } }; - let leaf_hash = blake2b_256_254_hash(&record); + let leaf_hash = blake2b_256_254_hash_to_scalar(&record); kzg.verify( &commitment, @@ -861,7 +850,7 @@ pub fn is_piece_valid( pub fn is_piece_record_hash_valid( kzg: &Kzg, num_pieces_in_segment: usize, - piece_record_hash: &Blake2b256Hash, + piece_record_hash: &Scalar, commitment: &RecordsRoot, witness: &Witness, position: u32, diff --git a/crates/subspace-archiving/src/archiver/incremental_record_commitments.rs b/crates/subspace-archiving/src/archiver/incremental_record_commitments.rs index faab8aad063..60abbc71ef0 100644 --- a/crates/subspace-archiving/src/archiver/incremental_record_commitments.rs +++ b/crates/subspace-archiving/src/archiver/incremental_record_commitments.rs @@ -7,6 +7,7 @@ use blake2::digest::{FixedOutput, Update}; use blake2::Blake2b; use core::mem; use parity_scale_codec::{Encode, Output}; +use subspace_core_primitives::crypto::Scalar; use subspace_core_primitives::Blake2b256Hash; /// State of incremental record commitments, encapsulated to hide implementation details and @@ -17,7 +18,7 @@ pub(super) struct IncrementalRecordCommitmentsState { /// /// NOTE: Until full segment is processed, this will not contain commitment to the first record /// since it is not ready yet. This in turn means all commitments will be at `-1` offset. - state: VecDeque, + state: VecDeque, } impl IncrementalRecordCommitmentsState { @@ -28,7 +29,7 @@ impl IncrementalRecordCommitmentsState { } } - pub(super) fn drain(&mut self) -> impl Iterator + '_ { + pub(super) fn drain(&mut self) -> impl Iterator + '_ { self.state.drain(..) } } @@ -60,7 +61,7 @@ struct IncrementalRecordCommitmentsProcessor<'a> { full: bool, /// Intermediate hashing state that computes Blake2-256-254. /// - /// See [`subspace_core_primitives::crypto::blake2b_256_254_hash`] for details. + /// See [`subspace_core_primitives::crypto::blake2b_256_254_hash_to_scalar`] for details. hashing_state: Blake2b, } @@ -162,7 +163,10 @@ impl<'a> IncrementalRecordCommitmentsProcessor<'a> { // little-endian) hash[31] &= 0b00111111; - self.incremental_record_commitments.state.push_back(hash); + self.incremental_record_commitments.state.push_back( + Scalar::try_from(hash) + .expect("Last bit erased, thus hash is guaranteed to fit into scalar; qed"), + ); } } } diff --git a/crates/subspace-archiving/src/piece_reconstructor.rs b/crates/subspace-archiving/src/piece_reconstructor.rs index 735b4f75641..a0d844eee20 100644 --- a/crates/subspace-archiving/src/piece_reconstructor.rs +++ b/crates/subspace-archiving/src/piece_reconstructor.rs @@ -4,9 +4,9 @@ use crate::utils; use alloc::vec; use alloc::vec::Vec; use reed_solomon_erasure::galois_16::ReedSolomon; -use subspace_core_primitives::crypto::blake2b_256_254_hash; use subspace_core_primitives::crypto::kzg::{Kzg, Polynomial}; -use subspace_core_primitives::{FlatPieces, Piece, BLAKE2B_256_HASH_SIZE, PIECES_IN_SEGMENT}; +use subspace_core_primitives::crypto::{blake2b_256_254_hash_to_scalar, Scalar}; +use subspace_core_primitives::{FlatPieces, Piece, PIECES_IN_SEGMENT}; /// Reconstructor-related instantiation error. #[derive(Debug, Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Hash)] @@ -103,26 +103,26 @@ impl PiecesReconstructor { .map_err(ReconstructorError::DataShardsReconstruction)?; let mut reconstructed_record_shards = FlatPieces::new(shards.len()); - let mut polynomial_data = - vec![0u8; (self.data_shards + self.parity_shards) as usize * BLAKE2B_256_HASH_SIZE]; + let mut record_commitments = + vec![Scalar::default(); (self.data_shards + self.parity_shards) as usize]; //TODO: Parity hashes will be erasure coded instead in the future //TODO: reuse already present commitments from segment_pieces, so we don't re-derive what // we already have reconstructed_record_shards .as_pieces_mut() - .zip(polynomial_data.chunks_exact_mut(BLAKE2B_256_HASH_SIZE)) + .zip(record_commitments.iter_mut()) .zip(shards) .for_each(|((mut piece, polynomial_data), record)| { let record = record.expect("Reconstruction just happened and all records are present; qed"); let record = record.flatten(); piece.record_mut().as_mut().copy_from_slice(record); - polynomial_data.copy_from_slice(&blake2b_256_254_hash(record)); + *polynomial_data = blake2b_256_254_hash_to_scalar(record); }); let polynomial = self .kzg - .poly(&polynomial_data) + .poly(&record_commitments) .expect("Internally produced values must never fail; qed"); Ok((reconstructed_record_shards, polynomial)) diff --git a/crates/subspace-core-primitives/benches/kzg.rs b/crates/subspace-core-primitives/benches/kzg.rs index bf2f95f93a1..dcf901a56b2 100644 --- a/crates/subspace-core-primitives/benches/kzg.rs +++ b/crates/subspace-core-primitives/benches/kzg.rs @@ -1,37 +1,29 @@ use criterion::{black_box, criterion_group, criterion_main, Criterion}; use subspace_core_primitives::crypto::kzg::{embedded_kzg_settings, Kzg}; -use subspace_core_primitives::Scalar; +use subspace_core_primitives::crypto::Scalar; fn criterion_benchmark(c: &mut Criterion) { - let data = { - // Multiple of 32 - let mut data = rand::random::<[u8; 256]>(); - - // We can only store 254 bits, set last byte to zero because of that - data.chunks_exact_mut(Scalar::FULL_BYTES) - .flat_map(|chunk| chunk.iter_mut().last()) - .for_each(|last_byte| *last_byte = 0); - - data - }; + let values = (0..8) + .map(|_| Scalar::from(rand::random::<[u8; Scalar::SAFE_BYTES]>())) + .collect::>(); let kzg = Kzg::new(embedded_kzg_settings()); c.bench_function("create-polynomial", |b| { b.iter(|| { - kzg.poly(black_box(&data)).unwrap(); + kzg.poly(black_box(&values)).unwrap(); }) }); c.bench_function("commit", |b| { - let polynomial = kzg.poly(&data).unwrap(); + let polynomial = kzg.poly(&values).unwrap(); b.iter(|| { kzg.commit(black_box(&polynomial)).unwrap(); }) }); c.bench_function("create-witness", |b| { - let polynomial = kzg.poly(&data).unwrap(); + let polynomial = kzg.poly(&values).unwrap(); b.iter(|| { kzg.create_witness(black_box(&polynomial), black_box(0)) @@ -40,13 +32,12 @@ fn criterion_benchmark(c: &mut Criterion) { }); c.bench_function("verify", |b| { - let polynomial = kzg.poly(&data).unwrap(); + let polynomial = kzg.poly(&values).unwrap(); let commitment = kzg.commit(&polynomial).unwrap(); let index = 0; let witness = kzg.create_witness(&polynomial, index).unwrap(); - let values = data.chunks_exact(Scalar::FULL_BYTES); let num_values = values.len(); - let value = values.into_iter().next().unwrap(); + let value = values.first().unwrap(); b.iter(|| { kzg.verify( diff --git a/crates/subspace-core-primitives/src/crypto.rs b/crates/subspace-core-primitives/src/crypto.rs index b84ce920812..669a25c2931 100644 --- a/crates/subspace-core-primitives/src/crypto.rs +++ b/crates/subspace-core-primitives/src/crypto.rs @@ -15,12 +15,24 @@ //! Various cryptographic utilities used across Subspace Network. +extern crate alloc; + pub mod kzg; use crate::Blake2b256Hash; +use alloc::format; +use alloc::string::String; +use alloc::vec::Vec; +use ark_bls12_381::Fr; +use ark_ff::{BigInteger, PrimeField}; use blake2::digest::typenum::U32; use blake2::digest::{FixedOutput, Update}; use blake2::{Blake2b, Blake2bMac, Digest}; +use blst_from_scratch::types::fr::FsFr; +use core::mem; +use derive_more::{AsMut, AsRef, Deref, DerefMut, From, Into}; +use parity_scale_codec::{Decode, Encode, EncodeLike, Input}; +use scale_info::{Type, TypeInfo}; /// BLAKE2b-256 hashing of a single value. pub fn blake2b_256_hash(data: &[u8]) -> Blake2b256Hash { @@ -39,6 +51,17 @@ pub fn blake2b_256_254_hash(data: &[u8]) -> Blake2b256Hash { hash } +/// BLAKE2b-256 hashing of a single value truncated to 254 bits. +/// +/// TODO: We probably wouldn't need this eventually +pub fn blake2b_256_254_hash_to_scalar(data: &[u8]) -> Scalar { + let mut hash = blake2b_256_hash(data); + // Erase last 2 bits to effectively truncate the hash (number is interpreted as little-endian) + hash[31] &= 0b00111111; + Scalar::try_from(hash) + .expect("Last bit erased, thus hash is guaranteed to fit into scalar; qed") +} + /// BLAKE2b-256 keyed hashing of a single value. /// /// PANIC: Panics if key is longer than 64 bytes. @@ -60,3 +83,397 @@ pub fn blake2b_256_hash_list(data: &[&[u8]]) -> Blake2b256Hash { .try_into() .expect("Initialized with correct length; qed") } + +// TODO: Remove once not used +/// Representation of a single BLS12-381 scalar value. +#[derive(Debug, Default, Copy, Clone, Eq, PartialEq)] +pub struct ScalarLegacy(pub(crate) Fr); + +impl Encode for ScalarLegacy { + fn size_hint(&self) -> usize { + Self::FULL_BYTES + } + + fn using_encoded R>(&self, f: F) -> R { + f(&self.to_bytes()) + } + + fn encoded_size(&self) -> usize { + Self::FULL_BYTES + } +} + +impl EncodeLike for ScalarLegacy {} + +impl Decode for ScalarLegacy { + fn decode(input: &mut I) -> Result { + Ok(Self::from(&<[u8; Self::FULL_BYTES]>::decode(input)?)) + } + + fn encoded_fixed_size() -> Option { + Some(Self::FULL_BYTES) + } +} + +impl TypeInfo for ScalarLegacy { + type Identity = Self; + + fn type_info() -> Type { + Type::builder() + .path(scale_info::Path::new(stringify!(Scalar), module_path!())) + .docs(&["BLS12-381 scalar"]) + .composite(scale_info::build::Fields::named().field(|f| { + f.ty::<[u8; Self::FULL_BYTES]>() + .name(stringify!(inner)) + .type_name("Fr") + })) + } +} + +#[cfg(feature = "serde")] +mod scalar_legacy_serde { + use serde::{Deserialize, Deserializer, Serialize, Serializer}; + + // Custom wrapper so we don't have to write serialization/deserialization code manually + #[derive(Serialize, Deserialize)] + struct Scalar(#[serde(with = "hex::serde")] pub(super) [u8; super::ScalarLegacy::FULL_BYTES]); + + impl Serialize for super::ScalarLegacy { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + Scalar(self.to_bytes()).serialize(serializer) + } + } + + impl<'de> Deserialize<'de> for super::ScalarLegacy { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let Scalar(bytes) = Scalar::deserialize(deserializer)?; + Ok(Self::from(&bytes)) + } + } +} + +impl From<&[u8; Self::SAFE_BYTES]> for ScalarLegacy { + fn from(value: &[u8; Self::SAFE_BYTES]) -> Self { + ScalarLegacy(Fr::from_le_bytes_mod_order(value)) + } +} + +impl From<[u8; Self::SAFE_BYTES]> for ScalarLegacy { + fn from(value: [u8; Self::SAFE_BYTES]) -> Self { + Self::from(&value) + } +} + +impl From<&[u8; Self::FULL_BYTES]> for ScalarLegacy { + fn from(value: &[u8; Self::FULL_BYTES]) -> Self { + ScalarLegacy(Fr::from_le_bytes_mod_order(value)) + } +} + +impl From<[u8; Self::FULL_BYTES]> for ScalarLegacy { + fn from(value: [u8; Self::FULL_BYTES]) -> Self { + Self::from(&value) + } +} + +impl From<&ScalarLegacy> for [u8; ScalarLegacy::FULL_BYTES] { + fn from(value: &ScalarLegacy) -> Self { + let mut bytes = Self::default(); + value.write_to_bytes(&mut bytes); + bytes + } +} + +impl From for [u8; ScalarLegacy::FULL_BYTES] { + fn from(value: ScalarLegacy) -> Self { + Self::from(&value) + } +} + +impl ScalarLegacy { + /// How many full bytes can be stored in BLS12-381 scalar (for instance before encoding). It is + /// actually 254 bits, but bits are mut harder to work with and likely not worth it. + /// + /// NOTE: After encoding more bytes can be used, so don't rely on this as the max number of + /// bytes stored within at all times! + pub const SAFE_BYTES: usize = 31; + /// How many bytes Scalar contains physically, use [`Self::SAFE_BYTES`] for the amount of data + /// that you can put into it safely (for instance before encoding). + pub const FULL_BYTES: usize = 32; + + /// Convert scalar into bytes + pub fn to_bytes(&self) -> [u8; ScalarLegacy::FULL_BYTES] { + self.into() + } + + /// Converts scalar to bytes that will be written to `bytes`. + pub fn write_to_bytes(&self, bytes: &mut [u8; Self::FULL_BYTES]) { + self.0 + .into_repr() + .write_le(&mut bytes.as_mut()) + .expect("Correct length input was provided; qed"); + } +} + +/// Representation of a single BLS12-381 scalar value. +#[derive(Debug, Default, Copy, Clone, Eq, PartialEq, From, Into, AsRef, AsMut, Deref, DerefMut)] +#[repr(transparent)] +pub struct Scalar(FsFr); + +impl Encode for Scalar { + fn size_hint(&self) -> usize { + Self::FULL_BYTES + } + + fn using_encoded R>(&self, f: F) -> R { + f(&self.to_bytes()) + } + + fn encoded_size(&self) -> usize { + Self::FULL_BYTES + } +} + +impl EncodeLike for Scalar {} + +impl Decode for Scalar { + fn decode(input: &mut I) -> Result { + Self::try_from(&<[u8; Self::FULL_BYTES]>::decode(input)?).map_err(|error_code| { + parity_scale_codec::Error::from("Failed to create scalar from bytes") + .chain(format!("Error code: {error_code}")) + }) + } + + fn encoded_fixed_size() -> Option { + Some(Self::FULL_BYTES) + } +} + +impl TypeInfo for Scalar { + type Identity = Self; + + fn type_info() -> Type { + Type::builder() + .path(scale_info::Path::new(stringify!(Scalar), module_path!())) + .docs(&["BLS12-381 scalar"]) + .composite(scale_info::build::Fields::named().field(|f| { + f.ty::<[u8; Self::FULL_BYTES]>() + .name(stringify!(inner)) + .type_name("FsFr") + })) + } +} + +#[cfg(feature = "serde")] +mod scalar_serde { + use serde::de::Error; + use serde::{Deserialize, Deserializer, Serialize, Serializer}; + + // Custom wrapper so we don't have to write serialization/deserialization code manually + #[derive(Serialize, Deserialize)] + struct Scalar(#[serde(with = "hex::serde")] pub(super) [u8; super::Scalar::FULL_BYTES]); + + impl Serialize for super::Scalar { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + Scalar(self.to_bytes()).serialize(serializer) + } + } + + impl<'de> Deserialize<'de> for super::Scalar { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let Scalar(bytes) = Scalar::deserialize(deserializer)?; + Self::try_from(bytes).map_err(D::Error::custom) + } + } +} + +impl From<&[u8; Self::SAFE_BYTES]> for Scalar { + fn from(value: &[u8; Self::SAFE_BYTES]) -> Self { + let mut bytes = [0u8; Self::FULL_BYTES]; + bytes[..Self::SAFE_BYTES].copy_from_slice(value); + Self::try_from(bytes).expect("Safe bytes always fit into scalar and thus succeed; qed") + } +} + +impl From<[u8; Self::SAFE_BYTES]> for Scalar { + fn from(value: [u8; Self::SAFE_BYTES]) -> Self { + Self::from(&value) + } +} + +impl TryFrom<&[u8; Self::FULL_BYTES]> for Scalar { + type Error = String; + + fn try_from(value: &[u8; Self::FULL_BYTES]) -> Result { + Self::try_from(*value) + } +} + +impl TryFrom<[u8; Self::FULL_BYTES]> for Scalar { + type Error = String; + + fn try_from(value: [u8; Self::FULL_BYTES]) -> Result { + FsFr::from_scalar(value) + .map_err(|error_code| { + format!("Failed to create scalar from bytes with code: {error_code}") + }) + .map(Scalar) + } +} + +impl From<&Scalar> for [u8; Scalar::FULL_BYTES] { + fn from(value: &Scalar) -> Self { + value.0.to_scalar() + } +} + +impl From for [u8; Scalar::FULL_BYTES] { + fn from(value: Scalar) -> Self { + Self::from(&value) + } +} + +impl Scalar { + /// How many full bytes can be stored in BLS12-381 scalar (for instance before encoding). It is + /// actually 254 bits, but bits are mut harder to work with and likely not worth it. + /// + /// NOTE: After encoding more bytes can be used, so don't rely on this as the max number of + /// bytes stored within at all times! + pub const SAFE_BYTES: usize = 31; + /// How many bytes Scalar contains physically, use [`Self::SAFE_BYTES`] for the amount of data + /// that you can put into it safely (for instance before encoding). + pub const FULL_BYTES: usize = 32; + + /// Convert scalar into bytes + pub fn to_bytes(&self) -> [u8; Scalar::FULL_BYTES] { + self.into() + } + + /// Convenient conversion from slice of scalar to underlying representation for efficiency + /// purposes. + pub fn slice_to_repr(value: &[Self]) -> &[FsFr] { + // SAFETY: `Scalar` is `#[repr(transparent)]` and guaranteed to have the same memory layout + unsafe { mem::transmute(value) } + } + + /// Convenient conversion from slice of underlying representation to scalar for efficiency + /// purposes. + pub fn slice_from_repr(value: &[FsFr]) -> &[Self] { + // SAFETY: `Scalar` is `#[repr(transparent)]` and guaranteed to have the same memory layout + unsafe { mem::transmute(value) } + } + + /// Convenient conversion from slice of optional scalar to underlying representation for efficiency + /// purposes. + pub fn slice_option_to_repr(value: &[Option]) -> &[Option] { + // SAFETY: `Scalar` is `#[repr(transparent)]` and guaranteed to have the same memory layout + unsafe { mem::transmute(value) } + } + + /// Convenient conversion from slice of optional underlying representation to scalar for efficiency + /// purposes. + pub fn slice_option_from_repr(value: &[Option]) -> &[Option] { + // SAFETY: `Scalar` is `#[repr(transparent)]` and guaranteed to have the same memory layout + unsafe { mem::transmute(value) } + } + + /// Convenient conversion from mutable slice of scalar to underlying representation for + /// efficiency purposes. + pub fn slice_mut_to_repr(value: &mut [Self]) -> &mut [FsFr] { + // SAFETY: `Scalar` is `#[repr(transparent)]` and guaranteed to have the same memory layout + unsafe { mem::transmute(value) } + } + + /// Convenient conversion from mutable slice of underlying representation to scalar for + /// efficiency purposes. + pub fn slice_mut_from_repr(value: &mut [FsFr]) -> &mut [Self] { + // SAFETY: `Scalar` is `#[repr(transparent)]` and guaranteed to have the same memory layout + unsafe { mem::transmute(value) } + } + + /// Convenient conversion from optional mutable slice of scalar to underlying representation for + /// efficiency purposes. + pub fn slice_option_mut_to_repr(value: &mut [Option]) -> &mut [Option] { + // SAFETY: `Scalar` is `#[repr(transparent)]` and guaranteed to have the same memory layout + unsafe { mem::transmute(value) } + } + + /// Convenient conversion from optional mutable slice of underlying representation to scalar for + /// efficiency purposes. + pub fn slice_option_mut_from_repr(value: &mut [Option]) -> &mut [Option] { + // SAFETY: `Scalar` is `#[repr(transparent)]` and guaranteed to have the same memory layout + unsafe { mem::transmute(value) } + } + + /// Convenient conversion from vector of scalar to underlying representation for efficiency + /// purposes. + pub fn vec_to_repr(value: Vec) -> Vec { + // SAFETY: `Scalar` is `#[repr(transparent)]` and guaranteed to have the same memory + // layout, original vector is not dropped + unsafe { + let mut value = mem::ManuallyDrop::new(value); + Vec::from_raw_parts( + value.as_mut_ptr() as *mut FsFr, + value.len(), + value.capacity(), + ) + } + } + + /// Convenient conversion from vector of underlying representation to scalar for efficiency + /// purposes. + pub fn vec_from_repr(value: Vec) -> Vec { + // SAFETY: `Scalar` is `#[repr(transparent)]` and guaranteed to have the same memory + // layout, original vector is not dropped + unsafe { + let mut value = mem::ManuallyDrop::new(value); + Vec::from_raw_parts( + value.as_mut_ptr() as *mut Self, + value.len(), + value.capacity(), + ) + } + } + + /// Convenient conversion from vector of optional scalar to underlying representation for + /// efficiency purposes. + pub fn vec_option_to_repr(value: Vec>) -> Vec> { + // SAFETY: `Scalar` is `#[repr(transparent)]` and guaranteed to have the same memory + // layout, original vector is not dropped + unsafe { + let mut value = mem::ManuallyDrop::new(value); + Vec::from_raw_parts( + value.as_mut_ptr() as *mut Option, + value.len(), + value.capacity(), + ) + } + } + + /// Convenient conversion from vector of optional underlying representation to scalar for + /// efficiency purposes. + pub fn vec_option_from_repr(value: Vec>) -> Vec> { + // SAFETY: `Scalar` is `#[repr(transparent)]` and guaranteed to have the same memory + // layout, original vector is not dropped + unsafe { + let mut value = mem::ManuallyDrop::new(value); + Vec::from_raw_parts( + value.as_mut_ptr() as *mut Option, + value.len(), + value.capacity(), + ) + } + } +} diff --git a/crates/subspace-core-primitives/src/crypto/kzg.rs b/crates/subspace-core-primitives/src/crypto/kzg.rs index da77734c643..ec248d66f84 100644 --- a/crates/subspace-core-primitives/src/crypto/kzg.rs +++ b/crates/subspace-core-primitives/src/crypto/kzg.rs @@ -7,16 +7,14 @@ mod tests; extern crate alloc; -use crate::Scalar; +use crate::crypto::Scalar; use alloc::collections::btree_map::Entry; use alloc::collections::BTreeMap; -use alloc::format; use alloc::string::{String, ToString}; use alloc::sync::Arc; use alloc::vec::Vec; use blst_from_scratch::eip_4844::{bytes_from_g1_rust, bytes_to_g1_rust, bytes_to_g2_rust}; use blst_from_scratch::types::fft_settings::FsFFTSettings; -use blst_from_scratch::types::fr::FsFr; use blst_from_scratch::types::g1::FsG1; use blst_from_scratch::types::kzg_settings::FsKZGSettings; use blst_from_scratch::types::poly::FsPoly; @@ -318,22 +316,11 @@ impl Kzg { /// bits of information. /// /// The resulting polynomial is in coefficient form. - pub fn poly(&self, data: &[u8]) -> Result { - let evals = data - .chunks(Scalar::FULL_BYTES) - .map(|scalar| { - FsFr::from_scalar( - scalar - .try_into() - .map_err(|_| "Failed to convert value to scalar".to_string())?, - ) - .map_err(|error_code| { - format!("Failed to create scalar from bytes with code: {error_code}") - }) - }) - .collect::, String>>()?; + pub fn poly(&self, data: &[Scalar]) -> Result { let poly = FsPoly { - coeffs: self.get_fft_settings(evals.len())?.fft_fr(&evals, true)?, + coeffs: self + .get_fft_settings(data.len())? + .fft_fr(Scalar::slice_to_repr(data), true)?, }; Ok(Polynomial(poly)) } @@ -362,7 +349,7 @@ impl Kzg { commitment: &Commitment, num_values: usize, index: u32, - value: &[u8], + value: &Scalar, witness: &Witness, ) -> bool { let fft_settings = match self.get_fft_settings(num_values) { @@ -373,24 +360,10 @@ impl Kzg { } }; let x = fft_settings.get_expanded_roots_of_unity_at(index as usize); - let value = match value.try_into() { - Ok(value) => value, - Err(_) => { - debug!("Failed to convert value to scalar"); - return false; - } - }; - let value = match FsFr::from_scalar(value) { - Ok(value) => value, - Err(error_code) => { - debug!(error_code, "Failed to create scalar from bytes with code"); - return false; - } - }; match self .kzg_settings - .check_proof_single(&commitment.0, &witness.0, &x, &value) + .check_proof_single(&commitment.0, &witness.0, &x, value) { Ok(result) => result, Err(error) => { diff --git a/crates/subspace-core-primitives/src/crypto/kzg/tests.rs b/crates/subspace-core-primitives/src/crypto/kzg/tests.rs index dbf289a6337..871c5f9d183 100644 --- a/crates/subspace-core-primitives/src/crypto/kzg/tests.rs +++ b/crates/subspace-core-primitives/src/crypto/kzg/tests.rs @@ -1,5 +1,5 @@ use crate::crypto::kzg::{embedded_kzg_settings, Kzg}; -use crate::Scalar; +use crate::crypto::Scalar; use blst_from_scratch::consts::{G1_GENERATOR, G2_GENERATOR}; use blst_from_scratch::eip_4844::{bytes_from_g1_rust, bytes_from_g2_rust}; use blst_from_scratch::types::fft_settings::FsFFTSettings; @@ -11,26 +11,17 @@ use rand_core::SeedableRng; #[test] fn basic() { - let data = { - // Multiple of 32 - let mut data = rand::random::<[u8; 256]>(); - - // We can only store 254 bits, set last byte to zero because of that - data.chunks_exact_mut(Scalar::FULL_BYTES) - .flat_map(|chunk| chunk.iter_mut().last()) - .for_each(|last_byte| *last_byte = 0); - - data - }; + let values = (0..8) + .map(|_| Scalar::from(rand::random::<[u8; Scalar::SAFE_BYTES]>())) + .collect::>(); let kzg = Kzg::new(embedded_kzg_settings()); - let polynomial = kzg.poly(&data).unwrap(); + let polynomial = kzg.poly(&values).unwrap(); let commitment = kzg.commit(&polynomial).unwrap(); - let values = data.chunks_exact(Scalar::FULL_BYTES); let num_values = values.len(); - for (index, value) in values.enumerate() { + for (index, value) in values.iter().enumerate() { let index = index.try_into().unwrap(); let witness = kzg.create_witness(&polynomial, index).unwrap(); diff --git a/crates/subspace-core-primitives/src/lib.rs b/crates/subspace-core-primitives/src/lib.rs index 73ec0b7847e..a9632aeae82 100644 --- a/crates/subspace-core-primitives/src/lib.rs +++ b/crates/subspace-core-primitives/src/lib.rs @@ -29,20 +29,18 @@ mod tests; extern crate alloc; use crate::crypto::kzg::{Commitment, Witness}; -use crate::crypto::{blake2b_256_hash, blake2b_256_hash_with_key}; -use ark_bls12_381::Fr; -use ark_ff::{BigInteger, PrimeField}; +use crate::crypto::{blake2b_256_hash, blake2b_256_hash_with_key, Scalar, ScalarLegacy}; use core::convert::AsRef; use core::fmt; use core::num::NonZeroU64; use derive_more::{Add, Deref, Display, Div, From, Into, Mul, Rem, Sub}; use num_traits::{WrappingAdd, WrappingSub}; -use parity_scale_codec::{Decode, Encode, EncodeLike, Input}; +use parity_scale_codec::{Decode, Encode}; pub use pieces::{ FlatPieces, Piece, PieceRef, PieceRefMut, RecordRef, RecordRefMut, WitnessRef, WitnessRefMut, PIECE_SIZE, RECORD_SIZE, WITNESS_SIZE, }; -use scale_info::{Type, TypeInfo}; +use scale_info::TypeInfo; #[cfg(feature = "serde")] use serde::{Deserialize, Serialize}; use uint::static_assertions::const_assert; @@ -148,142 +146,6 @@ impl PosProof { pub const SIZE: usize = POS_PROOF_LENGTH; } -/// Representation of a single BLS12-381 scalar value. -#[derive(Debug, Default, Copy, Clone, Eq, PartialEq)] -pub struct Scalar(Fr); - -impl Encode for Scalar { - fn size_hint(&self) -> usize { - 48 - } - - fn using_encoded R>(&self, f: F) -> R { - f(&self.to_bytes()) - } - - fn encoded_size(&self) -> usize { - 48 - } -} - -impl EncodeLike for Scalar {} - -impl Decode for Scalar { - fn decode(input: &mut I) -> Result { - Ok(Self::from(&<[u8; Self::FULL_BYTES]>::decode(input)?)) - } - - fn encoded_fixed_size() -> Option { - Some(Self::FULL_BYTES) - } -} - -impl TypeInfo for Scalar { - type Identity = Self; - - fn type_info() -> Type { - Type::builder() - .path(scale_info::Path::new(stringify!(Scalar), module_path!())) - .docs(&["BLS12-381 scalar"]) - .composite(scale_info::build::Fields::named().field(|f| { - f.ty::<[u8; Self::FULL_BYTES]>() - .name(stringify!(inner)) - .type_name("Fr") - })) - } -} - -#[cfg(feature = "serde")] -mod scalar_serde { - use serde::{Deserialize, Deserializer, Serialize, Serializer}; - - // Custom wrapper so we don't have to write serialization/deserialization code manually - #[derive(Serialize, Deserialize)] - struct Scalar(#[serde(with = "hex::serde")] pub(super) [u8; super::Scalar::FULL_BYTES]); - - impl Serialize for super::Scalar { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - Scalar(self.to_bytes()).serialize(serializer) - } - } - - impl<'de> Deserialize<'de> for super::Scalar { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - let Scalar(bytes) = Scalar::deserialize(deserializer)?; - Ok(Self::from(&bytes)) - } - } -} - -impl From<&[u8; Self::SAFE_BYTES]> for Scalar { - fn from(value: &[u8; Self::SAFE_BYTES]) -> Self { - Scalar(Fr::from_le_bytes_mod_order(value)) - } -} - -impl From<[u8; Self::SAFE_BYTES]> for Scalar { - fn from(value: [u8; Self::SAFE_BYTES]) -> Self { - Self::from(&value) - } -} - -impl From<&[u8; Self::FULL_BYTES]> for Scalar { - fn from(value: &[u8; Self::FULL_BYTES]) -> Self { - Scalar(Fr::from_le_bytes_mod_order(value)) - } -} - -impl From<[u8; Self::FULL_BYTES]> for Scalar { - fn from(value: [u8; Self::FULL_BYTES]) -> Self { - Self::from(&value) - } -} - -impl From<&Scalar> for [u8; Scalar::FULL_BYTES] { - fn from(value: &Scalar) -> Self { - let mut bytes = Self::default(); - value.write_to_bytes(&mut bytes); - bytes - } -} - -impl From for [u8; Scalar::FULL_BYTES] { - fn from(value: Scalar) -> Self { - Self::from(&value) - } -} - -impl Scalar { - /// How many full bytes can be stored in BLS12-381 scalar (for instance before encoding). It is - /// actually 254 bits, but bits are mut harder to work with and likely not worth it. - /// - /// NOTE: After encoding more bytes can be used, so don't rely on this as the max number of - /// bytes stored within at all times! - pub const SAFE_BYTES: usize = 31; - /// How many bytes Scalar contains physically, use [`Self::SAFE_BYTES`] for the amount of data - /// that you can put into it safely (for instance before encoding). - pub const FULL_BYTES: usize = 32; - - /// Convert scalar into bytes - pub fn to_bytes(&self) -> [u8; Scalar::FULL_BYTES] { - self.into() - } - - /// Converts scalar to bytes that will be written to `bytes`. - pub fn write_to_bytes(&self, bytes: &mut [u8; Self::FULL_BYTES]) { - self.0 - .into_repr() - .write_le(&mut bytes.as_mut()) - .expect("Correct length input was provided; qed"); - } -} - /// A Ristretto Schnorr public key as bytes produced by `schnorrkel` crate. #[derive( Debug, @@ -528,13 +390,13 @@ pub struct Solution { /// Pieces offset within sector pub piece_offset: PieceIndex, /// Piece commitment that can use used to verify that piece was included in blockchain history - pub piece_record_hash: Blake2b256Hash, + pub piece_record_hash: Scalar, /// Witness for above piece commitment pub piece_witness: Witness, /// Chunk offset within a piece pub chunk_offset: u32, /// Chunk at above offset - pub chunk: Scalar, + pub chunk: ScalarLegacy, /// VRF signature of expanded version of the above chunk pub chunk_signature: ChunkSignature, } @@ -589,10 +451,10 @@ where sector_index: 0, total_pieces: NonZeroU64::new(1).expect("1 is not 0; qed"), piece_offset: 0, - piece_record_hash: Blake2b256Hash::default(), + piece_record_hash: Scalar::default(), piece_witness: Witness::default(), chunk_offset: 0, - chunk: Scalar::default(), + chunk: ScalarLegacy::default(), chunk_signature: ChunkSignature { output: [0; 32], proof: [0; 64], diff --git a/crates/subspace-core-primitives/src/sector_codec.rs b/crates/subspace-core-primitives/src/sector_codec.rs index 3f1c443c98c..5a84684fe03 100644 --- a/crates/subspace-core-primitives/src/sector_codec.rs +++ b/crates/subspace-core-primitives/src/sector_codec.rs @@ -3,7 +3,7 @@ #[cfg(test)] mod tests; -use crate::Scalar; +use crate::crypto::ScalarLegacy; use alloc::vec::Vec; use ark_bls12_381::Fr; use ark_poly::{EvaluationDomain, GeneralEvaluationDomain}; @@ -46,11 +46,11 @@ pub struct SectorCodec { impl SectorCodec { /// Create new instance for sector size (in bytes) pub fn new(sector_size: usize) -> Result { - if sector_size % Scalar::FULL_BYTES != 0 { + if sector_size % ScalarLegacy::FULL_BYTES != 0 { return Err(SectorCodecError::WrongSectorSize); } - let sector_size_in_scalars = sector_size / Scalar::FULL_BYTES; + let sector_size_in_scalars = sector_size / ScalarLegacy::FULL_BYTES; if sector_size_in_scalars == 0 || !sector_size_in_scalars.is_power_of_two() { return Err(SectorCodecError::WrongSectorSize); @@ -72,11 +72,11 @@ impl SectorCodec { /// Data layout is expected to be flat pieces one after another, each piece is a column. The /// size of the sector should be equal to the global protocol parameters or else encoding will /// fail. - pub fn encode(&self, sector: &mut [Scalar]) -> Result<(), SectorCodecError> { + pub fn encode(&self, sector: &mut [ScalarLegacy]) -> Result<(), SectorCodecError> { if sector.len() != self.sector_size_in_scalars { return Err(SectorCodecError::WrongInputSectorSize { - expected: self.sector_size_in_scalars * Scalar::FULL_BYTES, - actual: sector.len() * Scalar::FULL_BYTES, + expected: self.sector_size_in_scalars * ScalarLegacy::FULL_BYTES, + actual: sector.len() * ScalarLegacy::FULL_BYTES, }); } @@ -103,7 +103,7 @@ impl SectorCodec { .skip(row_index) .step_by(self.sector_side_size_in_scalars) .zip(row.iter()) - .for_each(|(output, input)| *output = Scalar(*input)); + .for_each(|(output, input)| *output = ScalarLegacy(*input)); // Clear for next iteration of the loop row.clear(); @@ -115,11 +115,11 @@ impl SectorCodec { /// Decode sector in place. /// /// Data layout is the same as in encoding. - pub fn decode(&self, sector: &mut [Scalar]) -> Result<(), SectorCodecError> { + pub fn decode(&self, sector: &mut [ScalarLegacy]) -> Result<(), SectorCodecError> { if sector.len() != self.sector_size_in_scalars { return Err(SectorCodecError::WrongInputSectorSize { - expected: self.sector_size_in_scalars * Scalar::FULL_BYTES, - actual: sector.len() * Scalar::FULL_BYTES, + expected: self.sector_size_in_scalars * ScalarLegacy::FULL_BYTES, + actual: sector.len() * ScalarLegacy::FULL_BYTES, }); } @@ -148,7 +148,7 @@ impl SectorCodec { .skip(row_index) .step_by(self.sector_side_size_in_scalars) .zip(row.iter()) - .for_each(|(output, input)| *output = Scalar(*input)); + .for_each(|(output, input)| *output = ScalarLegacy(*input)); // Clear for next iteration of the loop row.clear(); @@ -183,7 +183,7 @@ impl SectorCodec { .skip(row_index) .step_by(self.sector_side_size_in_scalars) .zip(row) - .for_each(|(output, input)| *output = Scalar(input)); + .for_each(|(output, input)| *output = ScalarLegacy(input)); } } diff --git a/crates/subspace-core-primitives/src/sector_codec/tests.rs b/crates/subspace-core-primitives/src/sector_codec/tests.rs index d1459e00771..9261afbb337 100644 --- a/crates/subspace-core-primitives/src/sector_codec/tests.rs +++ b/crates/subspace-core-primitives/src/sector_codec/tests.rs @@ -1,16 +1,18 @@ +use crate::crypto::ScalarLegacy; use crate::sector_codec::SectorCodec; -use crate::Scalar; #[test] fn basic() { let rows_columns_count = 128_usize; - let sector_size = rows_columns_count.pow(2) * Scalar::FULL_BYTES; + let sector_size = rows_columns_count.pow(2) * ScalarLegacy::FULL_BYTES; let sector = { - let mut sector = Vec::with_capacity(sector_size / Scalar::FULL_BYTES); + let mut sector = Vec::with_capacity(sector_size / ScalarLegacy::FULL_BYTES); for _ in 0..sector.capacity() { - sector.push(Scalar::try_from(&rand::random::<[u8; Scalar::SAFE_BYTES]>()).unwrap()); + sector.push( + ScalarLegacy::try_from(&rand::random::<[u8; ScalarLegacy::SAFE_BYTES]>()).unwrap(), + ); } sector @@ -32,13 +34,15 @@ fn basic() { #[test] fn two_way_transformation_works() { let rows_columns_count = 4_usize; - let sector_size = rows_columns_count.pow(2) * Scalar::FULL_BYTES; + let sector_size = rows_columns_count.pow(2) * ScalarLegacy::FULL_BYTES; let mut scalars = { - let mut sector = Vec::with_capacity(sector_size / Scalar::FULL_BYTES); + let mut sector = Vec::with_capacity(sector_size / ScalarLegacy::FULL_BYTES); for _ in 0..sector.capacity() { - sector.push(Scalar::try_from(&rand::random::<[u8; Scalar::SAFE_BYTES]>()).unwrap()); + sector.push( + ScalarLegacy::try_from(&rand::random::<[u8; ScalarLegacy::SAFE_BYTES]>()).unwrap(), + ); } sector @@ -50,7 +54,7 @@ fn two_way_transformation_works() { let new_scalars = scalars .iter() - .map(|scalar| Scalar::from(&scalar.to_bytes())) + .map(|scalar| ScalarLegacy::from(&scalar.to_bytes())) .collect::>(); assert_eq!(scalars, new_scalars); diff --git a/crates/subspace-core-primitives/src/tests.rs b/crates/subspace-core-primitives/src/tests.rs index e01f2f23f11..fe2a154b11a 100644 --- a/crates/subspace-core-primitives/src/tests.rs +++ b/crates/subspace-core-primitives/src/tests.rs @@ -1,4 +1,5 @@ -use crate::{Scalar, PIECE_SIZE, PLOT_SECTOR_SIZE, U256}; +use crate::crypto::Scalar; +use crate::{PIECE_SIZE, PLOT_SECTOR_SIZE, U256}; use num_integer::Roots; use rand::thread_rng; use rand_core::RngCore; @@ -30,13 +31,12 @@ fn bytes_scalars_conversion() { let scalars = bytes .chunks_exact(Scalar::SAFE_BYTES) .map(|bytes| { - Scalar::try_from( + Scalar::from( <&[u8; Scalar::SAFE_BYTES]>::try_from(bytes) .expect("Chunked into correct size; qed"), ) }) - .collect::, _>>() - .unwrap(); + .collect::>(); { let mut decoded_bytes = vec![0u8; bytes.len()]; @@ -44,9 +44,7 @@ fn bytes_scalars_conversion() { .chunks_exact_mut(Scalar::SAFE_BYTES) .zip(scalars.iter()) .for_each(|(bytes, scalar)| { - let mut tmp = [0u8; Scalar::FULL_BYTES]; - scalar.write_to_bytes(&mut tmp); - bytes.copy_from_slice(&tmp[..Scalar::SAFE_BYTES]); + bytes.copy_from_slice(&scalar.to_bytes()[..Scalar::SAFE_BYTES]); }); assert_eq!(bytes, decoded_bytes); @@ -78,11 +76,5 @@ fn bytes_scalars_conversion() { assert_eq!(bytes, scalar.to_bytes()); } - - { - let scalar = Scalar::from(&bytes); - - assert_eq!(bytes, scalar.to_bytes()); - } } } diff --git a/crates/subspace-erasure-coding/src/lib.rs b/crates/subspace-erasure-coding/src/lib.rs index 0b58f6eea27..0176499854b 100644 --- a/crates/subspace-erasure-coding/src/lib.rs +++ b/crates/subspace-erasure-coding/src/lib.rs @@ -1,11 +1,10 @@ #![cfg_attr(not(feature = "std"), no_std)] -extern crate alloc; - -#[cfg(all(test, features = "std"))] +#[cfg(test)] mod tests; -use alloc::format; +extern crate alloc; + use alloc::string::{String, ToString}; use alloc::vec::Vec; use blst_from_scratch::types::fft_settings::FsFFTSettings; @@ -13,7 +12,7 @@ use blst_from_scratch::types::fr::FsFr; use blst_from_scratch::types::poly::FsPoly; use core::num::NonZeroUsize; use kzg::{FFTSettings, PolyRecover, DAS}; -use subspace_core_primitives::Scalar; +use subspace_core_primitives::crypto::Scalar; /// Erasure coding abstraction. /// @@ -38,28 +37,11 @@ impl ErasureCoding { /// /// Returns parity data. pub fn extend(&self, source: &[Scalar]) -> Result, String> { - // TODO: Once our scalars are based on `blst_from_scratch` we can use a bit of transmute to - // avoid allocation here // TODO: das_fft_extension modifies buffer internally, it needs to change to use // pre-allocated buffer instead of allocating a new one - let source = source - .iter() - .map(|scalar| { - FsFr::from_scalar(scalar.to_bytes()) - .map_err(|error| format!("Failed to convert scalar: {error}")) - }) - .collect::, String>>()?; - let parity = self - .fft_settings - .das_fft_extension(&source)? - .into_iter() - .map(|scalar| { - // This is fine, scalar is guaranteed to be correct here - Scalar::from(scalar.to_scalar()) - }) - .collect(); - - Ok(parity) + self.fft_settings + .das_fft_extension(Scalar::slice_to_repr(source)) + .map(Scalar::vec_from_repr) } /// Recovery of missing shards from given shards (at least 1/2 should be `Some`). @@ -73,31 +55,11 @@ impl ErasureCoding { { return Err("Impossible to recover, too many shards are missing".to_string()); } - // TODO: Once our scalars are based on `blst_from_scratch` we can use a bit of transmute to - // avoid allocation here - let shards = shards - .iter() - .map(|maybe_scalar| { - maybe_scalar - .map(|scalar| { - FsFr::from_scalar(scalar.into()) - .map_err(|error| format!("Failed to convert scalar: {error}")) - }) - .transpose() - }) - .collect::, _>>()?; let poly = >::recover_poly_from_samples( - &shards, + Scalar::slice_option_to_repr(shards), &self.fft_settings, )?; - Ok(poly - .coeffs - .iter() - .map(|scalar| { - // This is fine, scalar is guaranteed to be correct here - Scalar::from(scalar.to_scalar()) - }) - .collect()) + Ok(Scalar::vec_from_repr(poly.coeffs)) } } diff --git a/crates/subspace-erasure-coding/src/tests.rs b/crates/subspace-erasure-coding/src/tests.rs index 3a80a018c22..aba58d7f9dd 100644 --- a/crates/subspace-erasure-coding/src/tests.rs +++ b/crates/subspace-erasure-coding/src/tests.rs @@ -1,7 +1,7 @@ use crate::ErasureCoding; use std::iter; use std::num::NonZeroUsize; -use subspace_core_primitives::Scalar; +use subspace_core_primitives::crypto::Scalar; // TODO: This could have been done in-place, once implemented can be exposed as a utility fn concatenated_to_interleaved(input: Vec) -> Vec diff --git a/crates/subspace-farmer-components/src/farming.rs b/crates/subspace-farmer-components/src/farming.rs index ec18fcd7ea5..d3520fa6986 100644 --- a/crates/subspace-farmer-components/src/farming.rs +++ b/crates/subspace-farmer-components/src/farming.rs @@ -3,12 +3,12 @@ use parity_scale_codec::{Decode, IoReader}; use schnorrkel::Keypair; use std::io; use std::io::SeekFrom; -use subspace_core_primitives::crypto::blake2b_256_254_hash; use subspace_core_primitives::crypto::kzg::Witness; +use subspace_core_primitives::crypto::{blake2b_256_254_hash_to_scalar, ScalarLegacy}; use subspace_core_primitives::sector_codec::{SectorCodec, SectorCodecError}; use subspace_core_primitives::{ - Blake2b256Hash, Piece, PieceIndex, PublicKey, Scalar, SectorId, SectorIndex, Solution, - SolutionRange, PIECES_IN_SECTOR, PIECE_SIZE, PLOT_SECTOR_SIZE, + Blake2b256Hash, Piece, PieceIndex, PublicKey, SectorId, SectorIndex, Solution, SolutionRange, + PIECES_IN_SECTOR, PIECE_SIZE, PLOT_SECTOR_SIZE, }; use subspace_solving::create_chunk_signature; use subspace_verification::{derive_audit_chunk, is_within_solution_range}; @@ -39,7 +39,7 @@ pub struct EligibleChunk { /// Offset of the chunk within piece pub offset: u32, /// Chunk itself - pub chunk: Scalar, + pub chunk: ScalarLegacy, } /// Sector that can be used to create a solution that is within desired solution range @@ -80,14 +80,14 @@ impl EligibleSector { sector.read_exact(&mut sector_bytes)?; sector_bytes - .chunks_exact(Scalar::FULL_BYTES) + .chunks_exact(ScalarLegacy::FULL_BYTES) .map(|bytes| { - Scalar::from( - <&[u8; Scalar::FULL_BYTES]>::try_from(bytes) + ScalarLegacy::from( + <&[u8; ScalarLegacy::FULL_BYTES]>::try_from(bytes) .expect("Chunked into scalar full bytes above; qed"), ) }) - .collect::>() + .collect::>() }; let sector_metadata = SectorMetadata::decode(&mut IoReader(sector_metadata)) @@ -98,9 +98,9 @@ impl EligibleSector { .map_err(FarmingError::FailedToDecodeSector)?; let mut piece = Piece::default(); - let scalars_in_piece = PIECE_SIZE / Scalar::SAFE_BYTES; + let scalars_in_piece = PIECE_SIZE / ScalarLegacy::SAFE_BYTES; piece - .chunks_exact_mut(Scalar::SAFE_BYTES) + .chunks_exact_mut(ScalarLegacy::SAFE_BYTES) .zip( sector_scalars .into_iter() @@ -111,7 +111,7 @@ impl EligibleSector { // After decoding we get piece scalar bytes padded with zero byte, so we can read // the whole thing first and then copy just first `Scalar::SAFE_BYTES` we actually // care about - output.copy_from_slice(&input.to_bytes()[..Scalar::SAFE_BYTES]); + output.copy_from_slice(&input.to_bytes()[..ScalarLegacy::SAFE_BYTES]); }); let (record, witness) = piece.split(); @@ -122,7 +122,7 @@ impl EligibleSector { .sector_id .derive_piece_index(self.audit_piece_offset, sector_metadata.total_pieces); let audit_piece_bytes_offset = self.audit_piece_offset - * (PIECE_SIZE / Scalar::SAFE_BYTES * Scalar::FULL_BYTES) as u64; + * (PIECE_SIZE / ScalarLegacy::SAFE_BYTES * ScalarLegacy::FULL_BYTES) as u64; error!( ?error, sector_id = ?self.sector_id, @@ -143,7 +143,7 @@ impl EligibleSector { sector_index: self.sector_index, total_pieces: sector_metadata.total_pieces, piece_offset: self.audit_piece_offset, - piece_record_hash: blake2b_256_254_hash(&record), + piece_record_hash: blake2b_256_254_hash_to_scalar(&record), piece_witness, chunk_offset: offset, chunk, @@ -173,15 +173,15 @@ where let audit_piece_offset: PieceIndex = local_challenge % PIECES_IN_SECTOR; // Offset of the piece in sector (in bytes, accounts for the fact that encoded piece has its // chunks expanded with zero byte padding) - let audit_piece_bytes_offset = - audit_piece_offset * (PIECE_SIZE / Scalar::SAFE_BYTES * Scalar::FULL_BYTES) as u64; + let audit_piece_bytes_offset = audit_piece_offset + * (PIECE_SIZE / ScalarLegacy::SAFE_BYTES * ScalarLegacy::FULL_BYTES) as u64; let mut piece = Piece::default(); sector.seek(SeekFrom::Current(audit_piece_bytes_offset as i64))?; sector.read_exact(&mut piece)?; let chunks = piece - .chunks_exact(Scalar::FULL_BYTES) + .chunks_exact(ScalarLegacy::FULL_BYTES) .enumerate() .filter_map(|(offset, chunk_bytes)| { let chunk_bytes = chunk_bytes @@ -195,7 +195,7 @@ where ) .then(|| EligibleChunk { offset: offset as u32, - chunk: Scalar::from(&chunk_bytes), + chunk: ScalarLegacy::from(&chunk_bytes), }) }) .collect::>(); diff --git a/crates/subspace-farmer-components/src/plotting.rs b/crates/subspace-farmer-components/src/plotting.rs index 1d6bf910b89..35a56051008 100644 --- a/crates/subspace-farmer-components/src/plotting.rs +++ b/crates/subspace-farmer-components/src/plotting.rs @@ -9,9 +9,10 @@ use std::error::Error; use std::io; use std::sync::Arc; use subspace_core_primitives::crypto::kzg::{Commitment, Kzg}; +use subspace_core_primitives::crypto::{Scalar, ScalarLegacy}; use subspace_core_primitives::sector_codec::{SectorCodec, SectorCodecError}; use subspace_core_primitives::{ - Piece, PieceIndex, PieceIndexHash, PublicKey, Scalar, SectorId, SectorIndex, PIECE_SIZE, + Piece, PieceIndex, PieceIndexHash, PublicKey, SectorId, SectorIndex, PIECE_SIZE, PLOT_SECTOR_SIZE, }; use thiserror::Error; @@ -188,11 +189,15 @@ where // 32-byte chunks that have up to 254 bits of data in them and in sector encoding we're // dealing with 31-byte chunks instead. This workaround will not be necessary once we // change `kzg.poly()` API to use 31-byte chunks as well. - let mut expanded_piece = Vec::with_capacity(PIECE_SIZE / Scalar::SAFE_BYTES * 32); - piece.chunks_exact(Scalar::SAFE_BYTES).for_each(|chunk| { - expanded_piece.extend(chunk); - expanded_piece.extend([0]); - }); + let expanded_piece = piece + .chunks_exact(Scalar::SAFE_BYTES) + .map(|bytes| { + Scalar::from( + <&[u8; Scalar::SAFE_BYTES]>::try_from(bytes) + .expect("Chunked into correctly sized bytes; qed"), + ) + }) + .collect::>(); let polynomial = kzg .poly(&expanded_piece) .map_err(PlottingError::FailedToCommit)?; @@ -218,7 +223,7 @@ where } async fn plot_pieces_in_batches_non_blocking( - in_memory_sector_scalars: &mut Vec, + in_memory_sector_scalars: &mut Vec, sector_index: u64, piece_getter: &PG, piece_getter_retry_policy: PieceGetterRetryPolicy, @@ -256,7 +261,7 @@ async fn plot_pieces_in_batches_non_blocking( .ok_or(PlottingError::PieceNotFound { piece_index })?; in_memory_sector_scalars.extend(piece.chunks_exact(Scalar::SAFE_BYTES).map(|bytes| { - Scalar::from( + ScalarLegacy::from( <&[u8; Scalar::SAFE_BYTES]>::try_from(bytes) .expect("Chunked into scalar safe bytes above; qed"), ) diff --git a/crates/subspace-farmer/src/identity.rs b/crates/subspace-farmer/src/identity.rs index e6ef3518987..88ca809b3df 100644 --- a/crates/subspace-farmer/src/identity.rs +++ b/crates/subspace-farmer/src/identity.rs @@ -5,7 +5,8 @@ use schnorrkel::{ExpansionMode, Keypair, PublicKey, SecretKey, Signature}; use std::fs; use std::ops::Deref; use std::path::Path; -use subspace_core_primitives::{ChunkSignature, Scalar}; +use subspace_core_primitives::crypto::ScalarLegacy; +use subspace_core_primitives::ChunkSignature; use subspace_solving::{create_chunk_signature, REWARD_SIGNING_CONTEXT}; use substrate_bip39::mini_secret_from_entropy; use tracing::debug; @@ -130,7 +131,10 @@ impl Identity { &self.entropy } - pub fn create_chunk_signature(&self, chunk_bytes: &[u8; Scalar::FULL_BYTES]) -> ChunkSignature { + pub fn create_chunk_signature( + &self, + chunk_bytes: &[u8; ScalarLegacy::FULL_BYTES], + ) -> ChunkSignature { create_chunk_signature(&self.keypair, chunk_bytes) } diff --git a/crates/subspace-farmer/src/single_disk_plot/piece_reader.rs b/crates/subspace-farmer/src/single_disk_plot/piece_reader.rs index f1d985f663a..948ba2789f1 100644 --- a/crates/subspace-farmer/src/single_disk_plot/piece_reader.rs +++ b/crates/subspace-farmer/src/single_disk_plot/piece_reader.rs @@ -1,7 +1,8 @@ use futures::channel::{mpsc, oneshot}; use futures::SinkExt; +use subspace_core_primitives::crypto::ScalarLegacy; use subspace_core_primitives::sector_codec::SectorCodec; -use subspace_core_primitives::{Piece, Scalar, SectorIndex, PIECE_SIZE, PLOT_SECTOR_SIZE}; +use subspace_core_primitives::{Piece, SectorIndex, PIECE_SIZE, PLOT_SECTOR_SIZE}; use tracing::warn; #[derive(Debug)] @@ -95,29 +96,29 @@ pub(super) fn read_piece( [..PLOT_SECTOR_SIZE as usize]; let mut sector_bytes_scalars = sector_bytes - .chunks_exact(Scalar::FULL_BYTES) + .chunks_exact(ScalarLegacy::FULL_BYTES) .map(|bytes| { - Scalar::from( - <&[u8; Scalar::FULL_BYTES]>::try_from(bytes) + ScalarLegacy::from( + <&[u8; ScalarLegacy::FULL_BYTES]>::try_from(bytes) .expect("Chunked into scalar full bytes above; qed"), ) }) .collect::>(); sector_codec.decode(&mut sector_bytes_scalars).ok()?; - let scalars_in_piece = PIECE_SIZE / Scalar::SAFE_BYTES; + let scalars_in_piece = PIECE_SIZE / ScalarLegacy::SAFE_BYTES; let piece_scalars = §or_bytes_scalars[piece_offset as usize * scalars_in_piece..][..scalars_in_piece]; let mut piece = Piece::default(); piece - .chunks_exact_mut(Scalar::SAFE_BYTES) + .chunks_exact_mut(ScalarLegacy::SAFE_BYTES) .zip(piece_scalars) .for_each(|(output, input)| { // After decoding we get piece scalar bytes padded with zero byte, so we can read // the whole thing first and then copy just first `Scalar::SAFE_BYTES` we actually // care about - output.copy_from_slice(&input.to_bytes()[..Scalar::SAFE_BYTES]); + output.copy_from_slice(&input.to_bytes()[..ScalarLegacy::SAFE_BYTES]); }); piece diff --git a/crates/subspace-solving/src/lib.rs b/crates/subspace-solving/src/lib.rs index cad9a4c558e..426c1d74a12 100644 --- a/crates/subspace-solving/src/lib.rs +++ b/crates/subspace-solving/src/lib.rs @@ -23,8 +23,8 @@ use merlin::Transcript; use schnorrkel::vrf::{VRFInOut, VRFOutput, VRFProof}; use schnorrkel::{Keypair, PublicKey, SignatureResult}; -use subspace_core_primitives::crypto::blake2b_256_hash_list; -use subspace_core_primitives::{Blake2b256Hash, ChunkSignature, Randomness, Scalar}; +use subspace_core_primitives::crypto::{blake2b_256_hash_list, ScalarLegacy}; +use subspace_core_primitives::{Blake2b256Hash, ChunkSignature, Randomness}; const CHUNK_SIGNATURE_LABEL: &[u8] = b"subspace_chunk_signature"; @@ -38,7 +38,9 @@ pub fn derive_global_challenge(global_randomness: &Randomness, slot: u64) -> Bla } /// Transcript used for creation and verification of VRF signatures for chunks. -pub fn create_chunk_signature_transcript(chunk_bytes: &[u8; Scalar::FULL_BYTES]) -> Transcript { +pub fn create_chunk_signature_transcript( + chunk_bytes: &[u8; ScalarLegacy::FULL_BYTES], +) -> Transcript { let mut transcript = Transcript::new(CHUNK_SIGNATURE_LABEL); transcript.append_message(b"chunk", chunk_bytes); transcript @@ -47,7 +49,7 @@ pub fn create_chunk_signature_transcript(chunk_bytes: &[u8; Scalar::FULL_BYTES]) /// Create tag signature using farmer's keypair. pub fn create_chunk_signature( keypair: &Keypair, - chunk_bytes: &[u8; Scalar::FULL_BYTES], + chunk_bytes: &[u8; ScalarLegacy::FULL_BYTES], ) -> ChunkSignature { let (in_out, proof, _) = keypair.vrf_sign(create_chunk_signature_transcript(chunk_bytes)); @@ -59,7 +61,7 @@ pub fn create_chunk_signature( /// Verify that chunk signature was created correctly. pub fn verify_chunk_signature( - chunk_bytes: &[u8; Scalar::FULL_BYTES], + chunk_bytes: &[u8; ScalarLegacy::FULL_BYTES], chunk_signature: &ChunkSignature, public_key: &PublicKey, ) -> SignatureResult { diff --git a/crates/subspace-verification/src/lib.rs b/crates/subspace-verification/src/lib.rs index 9f0421052c8..343d6c9af28 100644 --- a/crates/subspace-verification/src/lib.rs +++ b/crates/subspace-verification/src/lib.rs @@ -25,11 +25,11 @@ use schnorrkel::vrf::VRFOutput; use schnorrkel::{SignatureError, SignatureResult}; use sp_arithmetic::traits::SaturatedConversion; use subspace_archiving::archiver; -use subspace_core_primitives::crypto::blake2b_256_hash; use subspace_core_primitives::crypto::kzg::Kzg; +use subspace_core_primitives::crypto::{blake2b_256_hash, ScalarLegacy}; use subspace_core_primitives::{ BlockNumber, ChunkSignature, PieceIndex, PublicKey, Randomness, RecordsRoot, RewardSignature, - Scalar, SectorId, SlotNumber, Solution, SolutionRange, PIECES_IN_SECTOR, RANDOMNESS_CONTEXT, + SectorId, SlotNumber, Solution, SolutionRange, PIECES_IN_SECTOR, RANDOMNESS_CONTEXT, }; use subspace_solving::{ create_chunk_signature_transcript, derive_global_challenge, verify_chunk_signature, @@ -96,7 +96,7 @@ where } /// Derive audit chunk from scalar bytes contained within plotted piece -pub fn derive_audit_chunk(chunk_bytes: &[u8; Scalar::FULL_BYTES]) -> SolutionRange { +pub fn derive_audit_chunk(chunk_bytes: &[u8; ScalarLegacy::FULL_BYTES]) -> SolutionRange { let hash = blake2b_256_hash(chunk_bytes); SolutionRange::from_le_bytes([ hash[0], hash[1], hash[2], hash[3], hash[4], hash[5], hash[6], hash[7], @@ -216,7 +216,7 @@ where /// function. pub fn derive_randomness( public_key: &PublicKey, - chunk_bytes: &[u8; Scalar::FULL_BYTES], + chunk_bytes: &[u8; ScalarLegacy::FULL_BYTES], chunk_signature: &ChunkSignature, ) -> SignatureResult { let in_out = VRFOutput(chunk_signature.output).attach_input_hash(