improve block propagation (massalabs#4252)

* improve block propagation (init) * improve block propagation * improve block propagation * improve block propagation: config and some errors * improved block retrieval * improve block propag * block propagation updates * undo merkle renaming * unite operation note_operations_from_peer * markers * update ask blocks * add max peer load * comments * simplify retrieval * fix retrieval * improve caches * debug endorsement propagation * improve endorsement propag * integrate endorsement handler * formatting * make tests compile * fix tests * clippy * correct root hash
bardiyanftt · Jul 24, 2023 · d935fc8 · d935fc8
1 parent ff9a431
commit d935fc8
Show file tree

Hide file tree

Showing 41 changed files with 1,966 additions and 2,378 deletions.
diff --git a/massa-factory-worker/src/block_factory.rs b/massa-factory-worker/src/block_factory.rs
@@ -2,19  2,17 @@
 
 use massa_channel::receiver::MassaReceiver;
 use massa_factory_exports::{FactoryChannels, FactoryConfig};
-use massa_hash::Hash;
 use massa_models::{
     block::{Block, BlockSerializer},
     block_header::{BlockHeader, BlockHeaderSerializer, SecuredHeader},
     block_id::BlockId,
     endorsement::SecureShareEndorsement,
-    operation::OperationIdSerializer,
     operation::{compute_operations_hash, OperationIdSerializer},
     prehash::PreHashSet,
     secure_share::SecureShareContent,
     slot::Slot,
     timeslots::{get_block_slot_timestamp, get_closest_slot_to_timestamp},
 };
-use massa_serialization::Serializer;
 use massa_time::MassaTime;
 use massa_versioning::versioning::MipStore;
 use massa_wallet::Wallet;
@@ -224,17  222,6 @@ impl BlockFactoryWorker {
         }
 
         block_storage.extend(op_storage);
-        let global_operations_hash = Hash::compute_from(
-            &op_ids
-                .iter()
-                .flat_map(|op_id| {
-                    let mut buffer = Vec::new();
-                    //It was a to_bytes() there before, we know the op is valid because it comes from the pool
-                    self.op_id_serializer.serialize(op_id, &mut buffer).unwrap();
-                    buffer
-                })
-                .collect::<Vec<u8>>(),
-        );
 
         // create header
         let current_version = self.mip_store.get_network_version_current();
@@ -245,7  232,7 @@ impl BlockFactoryWorker {
                 announced_version,
                 slot,
                 parents: parents.into_iter().map(|(id, _period)| id).collect(),
-                operation_merkle_root: global_operations_hash,
                 operation_merkle_root: compute_operations_hash(&op_ids, &self.op_id_serializer),
                 endorsements,
                 denunciations: self.channels.pool.get_block_denunciations(&slot),
             },

diff --git a/massa-final-state/src/tests/scenarios.rs b/massa-final-state/src/tests/scenarios.rs
@@ -96,9  96,8 @@ fn create_final_state(temp_dir: &TempDir, reset_final_state: bool) -> Arc<RwLock
     };
 
     // start proof-of-stake selectors
-    let (mut _selector_manager, selector_controller) =
-        start_selector_worker(selector_local_config.clone())
-            .expect("could not start server selector controller");
     let (mut _selector_manager, selector_controller) = start_selector_worker(selector_local_config)
         .expect("could not start server selector controller");
 
     // MIP store
     let mip_store = MipStore::try_from((
@@ -114,19  113,17 @@ fn create_final_state(temp_dir: &TempDir, reset_final_state: bool) -> Arc<RwLock
 
     let ledger = FinalLedger::new(final_state_local_config.ledger_config.clone(), db.clone());
 
-    let final_state = Arc::new(RwLock::new(
     Arc::new(RwLock::new(
         FinalState::new(
             db.clone(),
-            final_state_local_config.clone(),
             final_state_local_config,
             Box::new(ledger),
             selector_controller,
             mip_store,
             reset_final_state,
         )
         .unwrap(),
-    ));
-
-    final_state
     ))
 }
 
 use massa_versioning::versioning::{MipStatsConfig, MipStore};
@@ -186,10  183,9 @@ fn test_final_state() {
             None,
         );
         let mut async_pool_changes = AsyncPoolChanges::default();
-        async_pool_changes.0.insert(
-            message.compute_id(),
-            SetUpdateOrDelete::Set(message.clone()),
-        );
         async_pool_changes
             .0
             .insert(message.compute_id(), SetUpdateOrDelete::Set(message));
         state_changes.async_pool_changes = async_pool_changes;
 
         let amount = Amount::from_str("1").unwrap();
@@ -213,7  209,7 @@ fn test_final_state() {
         fs.write().db.write().flush().unwrap();
     }
 
-    copy_dir_all(temp_dir.path(), &temp_dir2.path()).unwrap();
     copy_dir_all(temp_dir.path(), temp_dir2.path()).unwrap();
 
     let fs2 = create_final_state(&temp_dir2, false);
     let hash2 = fs2.read().db.read().get_xof_db_hash();

diff --git a/massa-models/src/address.rs b/massa-models/src/address.rs
@@ -697,7  697,7 @@ mod test {
 
     #[test]
     fn test_address() {
-        let hash = massa_hash::Hash::compute_from(&"ADDR".as_bytes());
         let hash = massa_hash::Hash::compute_from("ADDR".as_bytes());
 
         let user_addr_0 = Address::User(UserAddress::UserAddressV0(UserAddressV0(hash)));
         let user_addr_1 = Address::User(UserAddress::UserAddressV1(UserAddressV1(hash)));
@@ -724,11  724,11 @@ mod test {
 
     #[test]
     fn test_address_get_thread() {
-        let hash = massa_hash::Hash::compute_from(&"ADDR".as_bytes());
         let hash = massa_hash::Hash::compute_from("ADDR".as_bytes());
 
         let user_addr_0 = Address::User(UserAddress::UserAddressV0(UserAddressV0(hash)));
         let thread_addr_0 = user_addr_0.get_thread(THREAD_COUNT);
-        let hash = massa_hash::Hash::compute_from(&"ADDR2".as_bytes());
         let hash = massa_hash::Hash::compute_from("ADDR2".as_bytes());
 
         let user_addr_1 = Address::User(UserAddress::UserAddressV0(UserAddressV0(hash)));
         let thread_addr_1 = user_addr_1.get_thread(THREAD_COUNT);

diff --git a/massa-models/src/block.rs b/massa-models/src/block.rs
@@ -1109,13  1109,13 @@ mod test {
 
         // create block
         let orig_block = Block {
-            header: orig_header.clone(),
             header: orig_header,
             operations: Default::default(),
         };
 
         // serialize block
         let secured_block: SecureShareBlock =
-            Block::new_verifiable(orig_block.clone(), BlockSerializer::new(), &keypair).unwrap();
             Block::new_verifiable(orig_block, BlockSerializer::new(), &keypair).unwrap();
         let mut ser_block = Vec::new();
         SecureShareSerializer::new()
             .serialize(&secured_block, &mut ser_block)
@@ -1195,13  1195,13 @@ mod test {
 
         // create block
         let orig_block = Block {
-            header: orig_header.clone(),
             header: orig_header,
             operations: Default::default(),
         };
 
         // serialize block
         let secured_block: SecureShareBlock =
-            Block::new_verifiable(orig_block.clone(), BlockSerializer::new(), &keypair).unwrap();
             Block::new_verifiable(orig_block, BlockSerializer::new(), &keypair).unwrap();
         let mut ser_block = Vec::new();
         SecureShareSerializer::new()
             .serialize(&secured_block, &mut ser_block)

diff --git a/massa-models/src/block_header.rs b/massa-models/src/block_header.rs
@@ -635,9  635,9 @@ mod test {
             .collect();
 
         let endorsement_1 = Endorsement {
-            slot: slot.clone(),
             slot,
             index: 1,
-            endorsed_block: parents_1[1].clone(),
             endorsed_block: parents_1[1],
         };
 
         assert_eq!(parents_1[1], endorsement_1.endorsed_block);
@@ -659,8  659,8 @@ mod test {
             slot,
             parents: parents_1,
             operation_merkle_root: Hash::compute_from("mno".as_bytes()),
-            endorsements: vec![s_endorsement_1.clone()],
-            denunciations: vec![de_a.clone(), de_b.clone()],
             endorsements: vec![s_endorsement_1],
             denunciations: vec![de_a, de_b],
         };
 
         let mut buffer = Vec::new();
@@ -675,7  675,7 @@ mod test {
 
         let (rem, block_header_der) = der.deserialize::<DeserializeError>(&buffer).unwrap();
 
-        assert_eq!(rem.is_empty(), true);
         assert!(rem.is_empty());
         assert_eq!(block_header_1, block_header_der);
     }
 

diff --git a/massa-models/src/config/constants.rs b/massa-models/src/config/constants.rs
@@ -34,8  34,6 @@ pub const SIGNATURE_DESER_SIZE: usize = 64   1;
 pub const MAX_ADVERTISE_LENGTH: u32 = 10000;
 /// Maximum message length in bytes
 pub const MAX_MESSAGE_SIZE: u32 = 1048576000;
-/// Max number of hash in the message `AskForBlocks`
-pub const MAX_ASK_BLOCKS_PER_MESSAGE: u32 = 128;
 /// Max number of operations per message
 pub const MAX_OPERATIONS_PER_MESSAGE: u32 = 1024;
 /// Length of the handshake random signature

diff --git a/massa-models/src/denunciation.rs b/massa-models/src/denunciation.rs
@@ -1208,8  1208,8 @@ mod tests {
             gen_endorsements_for_denunciation(None, None);
         let denunciation: Denunciation = (&s_endorsement_1, &s_endorsement_2).try_into().unwrap();
 
-        assert_eq!(denunciation.is_for_endorsement(), true);
-        assert_eq!(denunciation.is_valid(), true);
         assert!(denunciation.is_for_endorsement());
         assert!(denunciation.is_valid());
     }
 
     #[test]
@@ -1250,8  1250,8 @@ mod tests {
 
         let denunciation: Denunciation = (&s_endorsement_1, &s_endorsement_2).try_into().unwrap();
 
-        assert_eq!(denunciation.is_for_endorsement(), true);
-        assert_eq!(denunciation.is_valid(), true);
         assert!(denunciation.is_for_endorsement());
         assert!(denunciation.is_valid());
 
         // Try to create a denunciation from 2 endorsements @ != index
         let endorsement_4 = Endorsement {
@@ -1263,19  1263,13 @@ mod tests {
             Endorsement::new_verifiable(endorsement_4, EndorsementSerializer::new(), &keypair)
                 .unwrap();
 
-        assert_eq!(
-            denunciation
-                .is_also_for_endorsement(&s_endorsement_4)
-                .unwrap(),
-            false
-        );
-        assert_eq!(
-            denunciation
-                .is_also_for_endorsement(&s_endorsement_3)
-                .unwrap(),
-            true
-        );
-        assert_eq!(denunciation.is_valid(), true);
         assert!(!denunciation
             .is_also_for_endorsement(&s_endorsement_4)
             .unwrap());
         assert!(denunciation
             .is_also_for_endorsement(&s_endorsement_3)
             .unwrap());
         assert!(denunciation.is_valid());
     }
 
     #[test]
@@ -1285,14  1279,11 @@ mod tests {
             gen_block_headers_for_denunciation(None, None);
         let denunciation: Denunciation = (&s_block_header_1, &s_block_header_2).try_into().unwrap();
 
-        assert_eq!(denunciation.is_for_block_header(), true);
-        assert_eq!(denunciation.is_valid(), true);
-        assert_eq!(
-            denunciation
-                .is_also_for_block_header(&s_block_header_3)
-                .unwrap(),
-            true
-        );
         assert!(denunciation.is_for_block_header());
         assert!(denunciation.is_valid());
         assert!(denunciation
             .is_also_for_block_header(&s_block_header_3)
             .unwrap());
     }
 
     #[test]
@@ -1333,7  1324,7 @@ mod tests {
         });
 
         // hash_1 == hash_2 -> this is invalid
-        assert_eq!(de_forged_1.is_valid(), false);
         assert!(!de_forged_1.is_valid());
 
         // from an attacker - building manually a Denunciation object
         let de_forged_2 = Denunciation::Endorsement(EndorsementDenunciation {
@@ -1348,7  1339,7 @@ mod tests {
 
         // An attacker uses an old s_endorsement_1 to forge a Denunciation object @ slot_2
         // This has to be detected if Denunciation are send via the network
-        assert_eq!(de_forged_2.is_valid(), false);
         assert!(!de_forged_2.is_valid());
     }
 
     // SER / DER
@@ -1370,7  1361,7 @@ mod tests {
 
                 let (rem, de_der_res) = de_der.deserialize::<DeserializeError>(&buffer).unwrap();
 
-                assert_eq!(rem.is_empty(), true);
                 assert!(rem.is_empty());
                 assert_eq!(de, de_der_res);
             }
             Denunciation::BlockHeader(_) => {
@@ -1398,7  1389,7 @@ mod tests {
 
                 let (rem, de_der_res) = de_der.deserialize::<DeserializeError>(&buffer).unwrap();
 
-                assert_eq!(rem.is_empty(), true);
                 assert!(rem.is_empty());
                 assert_eq!(de, de_der_res);
             }
         }
@@ -1418,7  1409,7 @@ mod tests {
 
         let (rem, de_der_res) = de_der.deserialize::<DeserializeError>(&buffer).unwrap();
 
-        assert_eq!(rem.is_empty(), true);
         assert!(rem.is_empty());
         assert_eq!(denunciation, de_der_res);
 
         let (_, _, s_endorsement_1, s_endorsement_2, _) =
@@ -1428,7  1419,7 @@ mod tests {
 
         de_ser.serialize(&denunciation, &mut buffer).unwrap();
         let (rem, de_der_res) = de_der.deserialize::<DeserializeError>(&buffer).unwrap();
-        assert_eq!(rem.is_empty(), true);
         assert!(rem.is_empty());
         assert_eq!(denunciation, de_der_res);
     }
 

diff --git a/massa-models/src/operation.rs b/massa-models/src/operation.rs
@@ -1439,6  1439,29 @@ impl Deserializer<Vec<SecureShareOperation>> for OperationsDeserializer {
     }
 }
 
 /// Compute the hash of a list of operations(used typically in block headers)
 pub fn compute_operations_hash(
     op_ids: &[OperationId],
     op_id_serializer: &OperationIdSerializer,
 ) -> Hash {
     let op_ids = op_ids
         .iter()
         .map(|op_id| {
             let mut serialized = Vec::new();
             op_id_serializer
                 .serialize(op_id, &mut serialized)
                 .expect("serialization of operation id should not fail");
             serialized
         })
         .collect::<Vec<Vec<u8>>>();
     massa_hash::Hash::compute_from_tuple(
         &op_ids
             .iter()
             .map(|data| data.as_slice())
             .collect::<Vec<_>>(),
     )
 }
 
 #[cfg(test)]
 mod tests {
     use crate::config::{

diff --git a/massa-models/src/test_exports/data.rs b/massa-models/src/test_exports/data.rs
@@ -50,13  50,13 @@ pub fn gen_endorsements_for_denunciation(
     let v_endorsement_3 =
         Endorsement::new_verifiable(endorsement_3, EndorsementSerializer::new(), &keypair).unwrap();
 
-    return (
     (
         slot,
         keypair,
         v_endorsement1,
         v_endorsement2,
         v_endorsement_3,
-    );
     )
 }
 
 /// Helper to generate block headers ready for denunciation
@@ -128,7  128,7 @@ pub fn gen_block_headers_for_denunciation(
         slot,
         parents: parents_3,
         operation_merkle_root: Hash::compute_from("mno".as_bytes()),
-        endorsements: vec![s_endorsement_1.clone()],
         endorsements: vec![s_endorsement_1],
         denunciations: vec![],
     };
 
@@ -140,11  140,11 @@ pub fn gen_block_headers_for_denunciation(
     )
     .expect("error while producing block header");
 
-    return (
     (
         slot,
         keypair,
-        s_block_header_1.clone(),
         s_block_header_1,
         s_block_header_2,
         s_block_header_3,
-    );
     )
 }