network startup fixes
This commit is contained in:
parent
b843afe353
commit
7ca01b9e69
|
|
@ -56,8 +56,7 @@ fn online_node_count(response: &[u8]) -> Result<usize, String> {
|
||||||
.map_err(|_| "node deletion timestamp was invalid")?,
|
.map_err(|_| "node deletion timestamp was invalid")?,
|
||||||
);
|
);
|
||||||
let monitor_count = u16::from_le_bytes(
|
let monitor_count = u16::from_le_bytes(
|
||||||
response[offset + NODE_MONITOR_COUNT_OFFSET
|
response[offset + NODE_MONITOR_COUNT_OFFSET..offset + NODE_MONITOR_COUNT_OFFSET + 2]
|
||||||
..offset + NODE_MONITOR_COUNT_OFFSET + 2]
|
|
||||||
.try_into()
|
.try_into()
|
||||||
.map_err(|_| "node monitor count was invalid")?,
|
.map_err(|_| "node monitor count was invalid")?,
|
||||||
) as usize;
|
) as usize;
|
||||||
|
|
|
||||||
|
|
@ -6,7 +6,7 @@ use crate::records::memory::network_mapping::monitor::{MONITOR_ACTION_ADD, MONIT
|
||||||
use crate::records::memory::network_mapping::structs::{MonitorAddressParams, SignedMonitorEdit};
|
use crate::records::memory::network_mapping::structs::{MonitorAddressParams, SignedMonitorEdit};
|
||||||
use crate::records::memory::network_mapping::NodeInfo;
|
use crate::records::memory::network_mapping::NodeInfo;
|
||||||
use crate::records::memory::response_channels::{
|
use crate::records::memory::response_channels::{
|
||||||
delete_entry, reserve_entry_with_context, retire_peer_entries, Command,
|
delete_entry, reserve_entry_with_context, Command,
|
||||||
};
|
};
|
||||||
use crate::records::memory::structs::{Connection, ConnectionHealth, StoreConnectionParams};
|
use crate::records::memory::structs::{Connection, ConnectionHealth, StoreConnectionParams};
|
||||||
use crate::rpc::client::handshake::connect_and_handshake;
|
use crate::rpc::client::handshake::connect_and_handshake;
|
||||||
|
|
@ -131,13 +131,6 @@ pub(crate) fn spawn_outage_recovery_from_runtime() {
|
||||||
spawn_outage_recovery(context.db, context.wallet, context.map);
|
spawn_outage_recovery(context.db, context.wallet, context.map);
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) async fn retire_peer_requests_from_runtime(peer: &str) {
|
|
||||||
let Some(context) = node_runtime_context() else {
|
|
||||||
return;
|
|
||||||
};
|
|
||||||
retire_peer_entries(context.map, peer).await;
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn reconnect_replacement_inner(excluded_ip: &str) {
|
async fn reconnect_replacement_inner(excluded_ip: &str) {
|
||||||
// When an outgoing peer disappears, try to replace it with another
|
// When an outgoing peer disappears, try to replace it with another
|
||||||
// active node that is not already connected and is not the failed IP.
|
// active node that is not already connected and is not the failed IP.
|
||||||
|
|
@ -206,7 +199,7 @@ async fn retry_dropped_outgoing(ip: String, port: u16) {
|
||||||
|
|
||||||
// Losing every miner peer is process-level outage recovery, not an
|
// Losing every miner peer is process-level outage recovery, not an
|
||||||
// independent retry of each socket that happened to time out.
|
// independent retry of each socket that happened to time out.
|
||||||
if peer_connection_count().await == 0 {
|
if miner_connection_count().await == 0 {
|
||||||
spawn_outage_recovery(context.db, context.wallet, context.map);
|
spawn_outage_recovery(context.db, context.wallet, context.map);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
@ -532,7 +525,6 @@ impl Connection {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
drop(guard);
|
drop(guard);
|
||||||
retire_peer_entries(command_map.clone(), &format!("{ip}:{port}")).await;
|
|
||||||
if connection_type == ConnectionType::Outgoing {
|
if connection_type == ConnectionType::Outgoing {
|
||||||
spawn_retry_dropped_outgoing(ip.clone(), port);
|
spawn_retry_dropped_outgoing(ip.clone(), port);
|
||||||
}
|
}
|
||||||
|
|
@ -576,13 +568,20 @@ impl Connection {
|
||||||
.count()
|
.count()
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn mark_wallet_registry_synced(&mut self, key: &str) -> bool {
|
pub fn mark_wallet_registry_synced(
|
||||||
|
&mut self,
|
||||||
|
key: &str,
|
||||||
|
expected_stream: &Arc<Mutex<TcpStream>>,
|
||||||
|
) -> bool {
|
||||||
let Some((ip, port)) = split_ip_port_key(key) else {
|
let Some((ip, port)) = split_ip_port_key(key) else {
|
||||||
return false;
|
return false;
|
||||||
};
|
};
|
||||||
let ip_bytes = ip_to_binary(&ip);
|
let ip_bytes = ip_to_binary(&ip);
|
||||||
for (connection_key, info) in self.connection_map.iter_mut() {
|
for (connection_key, info) in self.connection_map.iter_mut() {
|
||||||
if connection_key.ip == ip_bytes && connection_key.port == port {
|
if connection_key.ip == ip_bytes
|
||||||
|
&& connection_key.port == port
|
||||||
|
&& Arc::ptr_eq(&info.stream, expected_stream)
|
||||||
|
{
|
||||||
info.wallet_registry_synced = true;
|
info.wallet_registry_synced = true;
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
@ -590,13 +589,20 @@ impl Connection {
|
||||||
false
|
false
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn mark_network_map_synced(&mut self, key: &str) -> bool {
|
pub fn mark_network_map_synced(
|
||||||
|
&mut self,
|
||||||
|
key: &str,
|
||||||
|
expected_stream: &Arc<Mutex<TcpStream>>,
|
||||||
|
) -> bool {
|
||||||
let Some((ip, port)) = split_ip_port_key(key) else {
|
let Some((ip, port)) = split_ip_port_key(key) else {
|
||||||
return false;
|
return false;
|
||||||
};
|
};
|
||||||
let ip_bytes = ip_to_binary(&ip);
|
let ip_bytes = ip_to_binary(&ip);
|
||||||
for (connection_key, info) in self.connection_map.iter_mut() {
|
for (connection_key, info) in self.connection_map.iter_mut() {
|
||||||
if connection_key.ip == ip_bytes && connection_key.port == port {
|
if connection_key.ip == ip_bytes
|
||||||
|
&& connection_key.port == port
|
||||||
|
&& Arc::ptr_eq(&info.stream, expected_stream)
|
||||||
|
{
|
||||||
info.network_map_synced = true;
|
info.network_map_synced = true;
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
@ -638,6 +644,7 @@ impl Connection {
|
||||||
pub fn mark_local_setup_complete(
|
pub fn mark_local_setup_complete(
|
||||||
&mut self,
|
&mut self,
|
||||||
key: &str,
|
key: &str,
|
||||||
|
expected_stream: &Arc<Mutex<TcpStream>>,
|
||||||
command_map: Arc<Mutex<Command>>,
|
command_map: Arc<Mutex<Command>>,
|
||||||
) -> Option<(Arc<Mutex<TcpStream>>, bool)> {
|
) -> Option<(Arc<Mutex<TcpStream>>, bool)> {
|
||||||
let Some((ip, port)) = split_ip_port_key(key) else {
|
let Some((ip, port)) = split_ip_port_key(key) else {
|
||||||
|
|
@ -645,7 +652,10 @@ impl Connection {
|
||||||
};
|
};
|
||||||
let ip_bytes = ip_to_binary(&ip);
|
let ip_bytes = ip_to_binary(&ip);
|
||||||
for (connection_key, info) in self.connection_map.iter_mut() {
|
for (connection_key, info) in self.connection_map.iter_mut() {
|
||||||
if connection_key.ip == ip_bytes && connection_key.port == port {
|
if connection_key.ip == ip_bytes
|
||||||
|
&& connection_key.port == port
|
||||||
|
&& Arc::ptr_eq(&info.stream, expected_stream)
|
||||||
|
{
|
||||||
if ClientType::from_bytes(&info.client_type) != Some(ClientType::Miner)
|
if ClientType::from_bytes(&info.client_type) != Some(ClientType::Miner)
|
||||||
|| !info.wallet_registry_synced
|
|| !info.wallet_registry_synced
|
||||||
|| !info.network_map_synced
|
|| !info.network_map_synced
|
||||||
|
|
@ -660,9 +670,27 @@ impl Connection {
|
||||||
None
|
None
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn local_setup_acknowledged(
|
||||||
|
&self,
|
||||||
|
key: &str,
|
||||||
|
expected_stream: &Arc<Mutex<TcpStream>>,
|
||||||
|
) -> bool {
|
||||||
|
let Some((ip, port)) = split_ip_port_key(key) else {
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
let ip_bytes = ip_to_binary(&ip);
|
||||||
|
self.connection_map.iter().any(|(connection_key, info)| {
|
||||||
|
connection_key.ip == ip_bytes
|
||||||
|
&& connection_key.port == port
|
||||||
|
&& Arc::ptr_eq(&info.stream, expected_stream)
|
||||||
|
&& info.local_setup_acknowledged
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
pub fn mark_remote_setup_complete(
|
pub fn mark_remote_setup_complete(
|
||||||
&mut self,
|
&mut self,
|
||||||
key: &str,
|
key: &str,
|
||||||
|
expected_stream: &Arc<Mutex<TcpStream>>,
|
||||||
command_map: Arc<Mutex<Command>>,
|
command_map: Arc<Mutex<Command>>,
|
||||||
) -> bool {
|
) -> bool {
|
||||||
let Some((ip, port)) = split_ip_port_key(key) else {
|
let Some((ip, port)) = split_ip_port_key(key) else {
|
||||||
|
|
@ -670,7 +698,10 @@ impl Connection {
|
||||||
};
|
};
|
||||||
let ip_bytes = ip_to_binary(&ip);
|
let ip_bytes = ip_to_binary(&ip);
|
||||||
for (connection_key, info) in self.connection_map.iter_mut() {
|
for (connection_key, info) in self.connection_map.iter_mut() {
|
||||||
if connection_key.ip == ip_bytes && connection_key.port == port {
|
if connection_key.ip == ip_bytes
|
||||||
|
&& connection_key.port == port
|
||||||
|
&& Arc::ptr_eq(&info.stream, expected_stream)
|
||||||
|
{
|
||||||
info.remote_setup_complete = true;
|
info.remote_setup_complete = true;
|
||||||
return Self::finalize_setup_if_complete(info, ip, port, command_map);
|
return Self::finalize_setup_if_complete(info, ip, port, command_map);
|
||||||
}
|
}
|
||||||
|
|
@ -681,6 +712,7 @@ impl Connection {
|
||||||
pub fn mark_local_setup_acknowledged(
|
pub fn mark_local_setup_acknowledged(
|
||||||
&mut self,
|
&mut self,
|
||||||
key: &str,
|
key: &str,
|
||||||
|
expected_stream: &Arc<Mutex<TcpStream>>,
|
||||||
command_map: Arc<Mutex<Command>>,
|
command_map: Arc<Mutex<Command>>,
|
||||||
) -> bool {
|
) -> bool {
|
||||||
let Some((ip, port)) = split_ip_port_key(key) else {
|
let Some((ip, port)) = split_ip_port_key(key) else {
|
||||||
|
|
@ -688,7 +720,10 @@ impl Connection {
|
||||||
};
|
};
|
||||||
let ip_bytes = ip_to_binary(&ip);
|
let ip_bytes = ip_to_binary(&ip);
|
||||||
for (connection_key, info) in self.connection_map.iter_mut() {
|
for (connection_key, info) in self.connection_map.iter_mut() {
|
||||||
if connection_key.ip == ip_bytes && connection_key.port == port {
|
if connection_key.ip == ip_bytes
|
||||||
|
&& connection_key.port == port
|
||||||
|
&& Arc::ptr_eq(&info.stream, expected_stream)
|
||||||
|
{
|
||||||
info.local_setup_acknowledged = true;
|
info.local_setup_acknowledged = true;
|
||||||
return Self::finalize_setup_if_complete(info, ip, port, command_map);
|
return Self::finalize_setup_if_complete(info, ip, port, command_map);
|
||||||
}
|
}
|
||||||
|
|
@ -1233,21 +1268,21 @@ pub async fn miner_connection_count() -> usize {
|
||||||
.unwrap_or(0)
|
.unwrap_or(0)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn mark_peer_wallet_registry_synced(key: &str) -> bool {
|
pub async fn mark_peer_wallet_registry_synced(key: &str, stream: &Arc<Mutex<TcpStream>>) -> bool {
|
||||||
CONNECTIONS
|
CONNECTIONS
|
||||||
.write()
|
.write()
|
||||||
.await
|
.await
|
||||||
.as_mut()
|
.as_mut()
|
||||||
.map(|connection| connection.mark_wallet_registry_synced(key))
|
.map(|connection| connection.mark_wallet_registry_synced(key, stream))
|
||||||
.unwrap_or(false)
|
.unwrap_or(false)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn mark_peer_network_map_synced(key: &str) -> bool {
|
pub async fn mark_peer_network_map_synced(key: &str, stream: &Arc<Mutex<TcpStream>>) -> bool {
|
||||||
CONNECTIONS
|
CONNECTIONS
|
||||||
.write()
|
.write()
|
||||||
.await
|
.await
|
||||||
.as_mut()
|
.as_mut()
|
||||||
.map(|connection| connection.mark_network_map_synced(key))
|
.map(|connection| connection.mark_network_map_synced(key, stream))
|
||||||
.unwrap_or(false)
|
.unwrap_or(false)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -1300,7 +1335,11 @@ pub async fn peer_catch_up_target(key: &str) -> Option<u32> {
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn mark_peer_operational(key: &str, map: Arc<Mutex<Command>>) -> bool {
|
pub async fn mark_peer_operational(
|
||||||
|
key: &str,
|
||||||
|
stream: &Arc<Mutex<TcpStream>>,
|
||||||
|
map: Arc<Mutex<Command>>,
|
||||||
|
) -> bool {
|
||||||
if peer_is_operational(key).await {
|
if peer_is_operational(key).await {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
@ -1309,11 +1348,18 @@ pub async fn mark_peer_operational(key: &str, map: Arc<Mutex<Command>>) -> bool
|
||||||
.write()
|
.write()
|
||||||
.await
|
.await
|
||||||
.as_mut()
|
.as_mut()
|
||||||
.and_then(|connection| connection.mark_local_setup_complete(key, map.clone()));
|
.and_then(|connection| connection.mark_local_setup_complete(key, stream, map.clone()));
|
||||||
let Some((stream, _already_acknowledged)) = setup else {
|
let Some((stored_stream, already_acknowledged)) = setup else {
|
||||||
return false;
|
return false;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
// The peer already acknowledged our setup-complete message. It may still
|
||||||
|
// be performing a long chain sync before sending its own setup-complete
|
||||||
|
// message, so do not repeatedly send setup or treat that wait as failure.
|
||||||
|
if already_acknowledged {
|
||||||
|
return peer_is_operational(key).await;
|
||||||
|
}
|
||||||
|
|
||||||
// Setup completion is idempotent. Keep sending it while the peer exists
|
// Setup completion is idempotent. Keep sending it while the peer exists
|
||||||
// until both sides have received and acknowledged the other's setup state.
|
// until both sides have received and acknowledged the other's setup state.
|
||||||
// A previous local acknowledgement alone does not make the peer operational.
|
// A previous local acknowledgement alone does not make the peer operational.
|
||||||
|
|
@ -1328,7 +1374,7 @@ pub async fn mark_peer_operational(key: &str, map: Arc<Mutex<Command>>) -> bool
|
||||||
message.push(RPC_SETUP_COMPLETE);
|
message.push(RPC_SETUP_COMPLETE);
|
||||||
message.extend_from_slice(&uid);
|
message.extend_from_slice(&uid);
|
||||||
|
|
||||||
if !RpcResponse::send_raw(&stream, Some(key), &message).await {
|
if !RpcResponse::send_raw(&stored_stream, Some(key), &message).await {
|
||||||
delete_entry(map.clone(), uid).await;
|
delete_entry(map.clone(), uid).await;
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
@ -1343,7 +1389,9 @@ pub async fn mark_peer_operational(key: &str, map: Arc<Mutex<Command>>) -> bool
|
||||||
.write()
|
.write()
|
||||||
.await
|
.await
|
||||||
.as_mut()
|
.as_mut()
|
||||||
.map(|connection| connection.mark_local_setup_acknowledged(key, map.clone()))
|
.map(|connection| {
|
||||||
|
connection.mark_local_setup_acknowledged(key, stream, map.clone())
|
||||||
|
})
|
||||||
.unwrap_or(false);
|
.unwrap_or(false);
|
||||||
if operational {
|
if operational {
|
||||||
return true;
|
return true;
|
||||||
|
|
@ -1358,31 +1406,69 @@ pub async fn mark_peer_operational(key: &str, map: Arc<Mutex<Command>>) -> bool
|
||||||
false
|
false
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn mark_peer_remote_setup_complete(key: &str, map: Arc<Mutex<Command>>) -> bool {
|
pub async fn mark_peer_remote_setup_complete(
|
||||||
|
key: &str,
|
||||||
|
stream: &Arc<Mutex<TcpStream>>,
|
||||||
|
map: Arc<Mutex<Command>>,
|
||||||
|
) -> bool {
|
||||||
CONNECTIONS
|
CONNECTIONS
|
||||||
.write()
|
.write()
|
||||||
.await
|
.await
|
||||||
.as_mut()
|
.as_mut()
|
||||||
.map(|connection| connection.mark_remote_setup_complete(key, map))
|
.map(|connection| connection.mark_remote_setup_complete(key, stream, map))
|
||||||
.unwrap_or(false)
|
.unwrap_or(false)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn spawn_peer_setup_retry(key: String, map: Arc<Mutex<Command>>) {
|
pub fn spawn_peer_setup_retry(
|
||||||
|
key: String,
|
||||||
|
stream: Arc<Mutex<TcpStream>>,
|
||||||
|
map: Arc<Mutex<Command>>,
|
||||||
|
) {
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
let Some(_retry_guard) = try_start_setup_retry(key.clone()) else {
|
let retry_key = format!("{key}:{:p}", Arc::as_ptr(&stream));
|
||||||
|
let Some(_retry_guard) = try_start_setup_retry(retry_key) else {
|
||||||
return;
|
return;
|
||||||
};
|
};
|
||||||
|
|
||||||
loop {
|
for attempt in 1..=3 {
|
||||||
sleep(Duration::from_secs(10)).await;
|
sleep(Duration::from_secs(10)).await;
|
||||||
if Connection::get_stream_from_memory(&key).await.is_none()
|
let current_stream = Connection::get_stream_from_memory(&key).await;
|
||||||
|
if current_stream
|
||||||
|
.as_ref()
|
||||||
|
.map(|current| !Arc::ptr_eq(current, &stream))
|
||||||
|
.unwrap_or(true)
|
||||||
|| peer_is_operational(&key).await
|
|| peer_is_operational(&key).await
|
||||||
{
|
{
|
||||||
break;
|
return;
|
||||||
}
|
}
|
||||||
if mark_peer_operational(&key, map.clone()).await {
|
if mark_peer_operational(&key, &stream, map.clone()).await {
|
||||||
break;
|
return;
|
||||||
}
|
}
|
||||||
|
let local_setup_acknowledged = CONNECTIONS
|
||||||
|
.read()
|
||||||
|
.await
|
||||||
|
.as_ref()
|
||||||
|
.map(|connection| connection.local_setup_acknowledged(&key, &stream))
|
||||||
|
.unwrap_or(false);
|
||||||
|
if local_setup_acknowledged {
|
||||||
|
// The connection remains valid while the remote peer finishes
|
||||||
|
// synchronization. Its eventual setup-complete message will
|
||||||
|
// finish promotion without another local retry.
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
warn!("[handshake_state] peer setup remains incomplete: peer={key} retry={attempt}/3");
|
||||||
|
}
|
||||||
|
|
||||||
|
if !peer_is_operational(&key).await
|
||||||
|
&& Connection::get_stream_from_memory(&key)
|
||||||
|
.await
|
||||||
|
.map(|current| Arc::ptr_eq(¤t, &stream))
|
||||||
|
.unwrap_or(false)
|
||||||
|
{
|
||||||
|
warn!(
|
||||||
|
"[handshake_state] removing incomplete peer after bounded setup retries: peer={key}"
|
||||||
|
);
|
||||||
|
remove_stream_from_memory(&stream).await;
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
@ -1599,4 +1685,32 @@ mod tests {
|
||||||
assert!(try_start_setup_retry("test-setup:first".to_string()).is_some());
|
assert!(try_start_setup_retry("test-setup:first".to_string()).is_some());
|
||||||
drop(second);
|
drop(second);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn setup_progress_can_only_be_recorded_by_the_stored_stream() {
|
||||||
|
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||||
|
let endpoint = listener.local_addr().unwrap();
|
||||||
|
let client = tokio::net::TcpStream::connect(endpoint).await.unwrap();
|
||||||
|
let (server, _) = listener.accept().await.unwrap();
|
||||||
|
let stored_stream = Arc::new(Mutex::new(server));
|
||||||
|
let stale_stream = Arc::new(Mutex::new(client));
|
||||||
|
let mut connections = Connection::new();
|
||||||
|
let command_map = Arc::new(Mutex::new(Command::new()));
|
||||||
|
let key = format!("127.0.0.1:{}", endpoint.port());
|
||||||
|
|
||||||
|
assert!(connections.store_connection(StoreConnectionParams {
|
||||||
|
connection_type: ConnectionType::Incoming,
|
||||||
|
ip: "127.0.0.1".to_string(),
|
||||||
|
port: endpoint.port(),
|
||||||
|
stream: stored_stream.clone(),
|
||||||
|
client_type: ClientType::Miner,
|
||||||
|
wallet_short_address: "ab13318c26250b048db92920a80a86127c933b0c.cltc".to_string(),
|
||||||
|
command_map: command_map.clone(),
|
||||||
|
}));
|
||||||
|
|
||||||
|
assert!(!connections.mark_wallet_registry_synced(&key, &stale_stream));
|
||||||
|
assert!(!connections.mark_network_map_synced(&key, &stale_stream));
|
||||||
|
assert!(connections.mark_wallet_registry_synced(&key, &stored_stream));
|
||||||
|
assert!(connections.mark_network_map_synced(&key, &stored_stream));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,6 @@
|
||||||
use super::*;
|
use super::*;
|
||||||
use crate::common::check_genesis::genesis_checkup;
|
use crate::common::check_genesis::genesis_checkup;
|
||||||
|
use crate::miner::flag::is_normal_mode;
|
||||||
use crate::records::unpack_block::unpack_header::load_block_header;
|
use crate::records::unpack_block::unpack_header::load_block_header;
|
||||||
use crate::sled::Batch;
|
use crate::sled::Batch;
|
||||||
|
|
||||||
|
|
@ -65,6 +66,13 @@ impl NodeInfo {
|
||||||
};
|
};
|
||||||
|
|
||||||
if let Some(count) = count {
|
if let Some(count) = count {
|
||||||
|
// Startup sync and orphan replay can accept thousands of blocks in
|
||||||
|
// seconds. Keep those counts in memory and persist the complete
|
||||||
|
// cache once the operation finishes instead of forcing a Sled
|
||||||
|
// write for every historical block.
|
||||||
|
if !is_normal_mode() {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
let mut batch = Batch::default();
|
let mut batch = Batch::default();
|
||||||
batch.insert(address.as_bytes(), &[count]);
|
batch.insert(address.as_bytes(), &[count]);
|
||||||
batch.insert(MINED_COUNTS_HEIGHT_KEY, &get_height(db).to_le_bytes());
|
batch.insert(MINED_COUNTS_HEIGHT_KEY, &get_height(db).to_le_bytes());
|
||||||
|
|
|
||||||
|
|
@ -65,6 +65,35 @@ mod tests {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn reconciliation_excludes_events_involving_deleted_nodes() {
|
||||||
|
let mut map = HashMap::new();
|
||||||
|
map.insert("active-a".to_string(), node(&[]));
|
||||||
|
map.insert("active-b".to_string(), node(&[]));
|
||||||
|
let mut deleted = node(&[]);
|
||||||
|
deleted.deleted_timestamp = 100;
|
||||||
|
map.insert("deleted".to_string(), deleted);
|
||||||
|
|
||||||
|
let event = |monitored: &str, monitoring: &str| SignedMonitorEdit {
|
||||||
|
action: MONITOR_ACTION_ADD,
|
||||||
|
monitored_address: monitored.to_string(),
|
||||||
|
monitoring_address: monitoring.to_string(),
|
||||||
|
target_ip: "1.2.3.4".to_string(),
|
||||||
|
modified_timestamp: 50,
|
||||||
|
modified_block: 10,
|
||||||
|
modified_signature: "signature".to_string(),
|
||||||
|
};
|
||||||
|
let events = HashMap::from([
|
||||||
|
("active".to_string(), event("active-a", "active-b")),
|
||||||
|
("deleted-target".to_string(), event("deleted", "active-b")),
|
||||||
|
("deleted-signer".to_string(), event("active-a", "deleted")),
|
||||||
|
]);
|
||||||
|
|
||||||
|
let reconciled = NodeInfo::reconcilable_monitor_events_from(&map, &events);
|
||||||
|
assert_eq!(reconciled.len(), 1);
|
||||||
|
assert!(reconciled.contains_key("active"));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn remove_wins_an_equal_timestamp_monitor_race() {
|
fn remove_wins_an_equal_timestamp_monitor_race() {
|
||||||
let add = SignedMonitorEdit {
|
let add = SignedMonitorEdit {
|
||||||
|
|
@ -186,6 +215,24 @@ mod tests {
|
||||||
}
|
}
|
||||||
|
|
||||||
impl NodeInfo {
|
impl NodeInfo {
|
||||||
|
pub(super) fn reconcilable_monitor_events_from(
|
||||||
|
address_map: &HashMap<String, NodeInfo>,
|
||||||
|
event_state: &HashMap<String, SignedMonitorEdit>,
|
||||||
|
) -> HashMap<String, SignedMonitorEdit> {
|
||||||
|
event_state
|
||||||
|
.iter()
|
||||||
|
.filter(|(_, event)| {
|
||||||
|
address_map
|
||||||
|
.get(&event.monitored_address)
|
||||||
|
.is_some_and(|node| node.deleted_timestamp == 0)
|
||||||
|
&& address_map
|
||||||
|
.get(&event.monitoring_address)
|
||||||
|
.is_some_and(|node| node.deleted_timestamp == 0)
|
||||||
|
})
|
||||||
|
.map(|(key, event)| (key.clone(), event.clone()))
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
pub async fn monitor_signature(
|
pub async fn monitor_signature(
|
||||||
action: u8,
|
action: u8,
|
||||||
monitored_address: &str,
|
monitored_address: &str,
|
||||||
|
|
@ -579,7 +626,9 @@ impl NodeInfo {
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) async fn signed_monitor_state_bytes() -> Vec<u8> {
|
pub(crate) async fn signed_monitor_state_bytes() -> Vec<u8> {
|
||||||
|
let map = ADDRESS_MAP.lock().await;
|
||||||
let state = MONITOR_EVENT_STATE.lock().await;
|
let state = MONITOR_EVENT_STATE.lock().await;
|
||||||
|
let state = Self::reconcilable_monitor_events_from(&map, &state);
|
||||||
Self::signed_monitor_state_bytes_from(&state)
|
Self::signed_monitor_state_bytes_from(&state)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -95,6 +95,7 @@ impl NodeInfo {
|
||||||
pub(crate) async fn signed_mapping_state() -> (Vec<SyncedNodeState>, Vec<SignedMonitorEdit>) {
|
pub(crate) async fn signed_mapping_state() -> (Vec<SyncedNodeState>, Vec<SignedMonitorEdit>) {
|
||||||
let map = ADDRESS_MAP.lock().await;
|
let map = ADDRESS_MAP.lock().await;
|
||||||
let monitor_events = MONITOR_EVENT_STATE.lock().await;
|
let monitor_events = MONITOR_EVENT_STATE.lock().await;
|
||||||
|
let monitor_events = Self::reconcilable_monitor_events_from(&map, &monitor_events);
|
||||||
|
|
||||||
let mut memberships: Vec<SyncedNodeState> = map
|
let mut memberships: Vec<SyncedNodeState> = map
|
||||||
.iter()
|
.iter()
|
||||||
|
|
@ -221,6 +222,7 @@ impl NodeInfo {
|
||||||
pub async fn canonical_mapping_digest() -> Result<Vec<u8>, String> {
|
pub async fn canonical_mapping_digest() -> Result<Vec<u8>, String> {
|
||||||
let map = ADDRESS_MAP.lock().await;
|
let map = ADDRESS_MAP.lock().await;
|
||||||
let monitor_events = MONITOR_EVENT_STATE.lock().await;
|
let monitor_events = MONITOR_EVENT_STATE.lock().await;
|
||||||
|
let monitor_events = Self::reconcilable_monitor_events_from(&map, &monitor_events);
|
||||||
let mut canonical = Self::canonical_mapping_records(&map)?;
|
let mut canonical = Self::canonical_mapping_records(&map)?;
|
||||||
let monitor_state = Self::signed_monitor_state_bytes_from(&monitor_events);
|
let monitor_state = Self::signed_monitor_state_bytes_from(&monitor_events);
|
||||||
canonical.extend_from_slice(&(monitor_state.len() as u32).to_le_bytes());
|
canonical.extend_from_slice(&(monitor_state.len() as u32).to_le_bytes());
|
||||||
|
|
@ -425,6 +427,7 @@ impl NodeInfo {
|
||||||
// node to validate blocks and avoid dialing deleted peers.
|
// node to validate blocks and avoid dialing deleted peers.
|
||||||
let map = ADDRESS_MAP.lock().await;
|
let map = ADDRESS_MAP.lock().await;
|
||||||
let monitor_events = MONITOR_EVENT_STATE.lock().await;
|
let monitor_events = MONITOR_EVENT_STATE.lock().await;
|
||||||
|
let monitor_events = Self::reconcilable_monitor_events_from(&map, &monitor_events);
|
||||||
let mut mapping: Vec<u8> = Vec::with_capacity(map.len() * NODE_RECORD_FIXED_BYTES);
|
let mut mapping: Vec<u8> = Vec::with_capacity(map.len() * NODE_RECORD_FIXED_BYTES);
|
||||||
|
|
||||||
for (address, node_info) in map.iter() {
|
for (address, node_info) in map.iter() {
|
||||||
|
|
|
||||||
|
|
@ -10,14 +10,36 @@ use crate::IpAddr;
|
||||||
use crate::SocketAddr;
|
use crate::SocketAddr;
|
||||||
use crate::TcpStream;
|
use crate::TcpStream;
|
||||||
use crate::{sleep, timeout, AsyncReadExt, AsyncWriteExt, Duration};
|
use crate::{sleep, timeout, AsyncReadExt, AsyncWriteExt, Duration};
|
||||||
|
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||||
use tokio::net::TcpSocket;
|
use tokio::net::TcpSocket;
|
||||||
|
|
||||||
const OUTBOUND_CONNECT_TIMEOUT_SECONDS: u64 = 2;
|
const OUTBOUND_CONNECT_TIMEOUT_SECONDS: u64 = 2;
|
||||||
const HANDSHAKE_RESPONSE_TIMEOUT_SECONDS: u64 = 2;
|
const HANDSHAKE_RESPONSE_TIMEOUT_SECONDS: u64 = 2;
|
||||||
|
static OUTBOUND_HANDSHAKES_ACTIVE: AtomicUsize = AtomicUsize::new(0);
|
||||||
|
|
||||||
|
struct OutboundHandshakeGuard;
|
||||||
|
|
||||||
|
impl OutboundHandshakeGuard {
|
||||||
|
fn acquire() -> Self {
|
||||||
|
OUTBOUND_HANDSHAKES_ACTIVE.fetch_add(1, Ordering::SeqCst);
|
||||||
|
Self
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Drop for OutboundHandshakeGuard {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
OUTBOUND_HANDSHAKES_ACTIVE.fetch_sub(1, Ordering::SeqCst);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn outbound_handshake_in_progress() -> bool {
|
||||||
|
OUTBOUND_HANDSHAKES_ACTIVE.load(Ordering::SeqCst) != 0
|
||||||
|
}
|
||||||
|
|
||||||
pub async fn connect_and_handshake(
|
pub async fn connect_and_handshake(
|
||||||
params: Connect,
|
params: Connect,
|
||||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||||
|
let _handshake_guard = OutboundHandshakeGuard::acquire();
|
||||||
let original_params = params.clone();
|
let original_params = params.clone();
|
||||||
let err = match connect_and_handshake_once(params).await {
|
let err = match connect_and_handshake_once(params).await {
|
||||||
Ok(()) => return Ok(()),
|
Ok(()) => return Ok(()),
|
||||||
|
|
@ -87,7 +109,7 @@ async fn connect_and_handshake_once(
|
||||||
.map_err(|err| Box::new(err) as Box<dyn std::error::Error + Send + Sync>)
|
.map_err(|err| Box::new(err) as Box<dyn std::error::Error + Send + Sync>)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn sponsor_endpoint_from_error(error: &str) -> Option<String> {
|
pub(crate) fn sponsor_endpoint_from_error(error: &str) -> Option<String> {
|
||||||
let (_, sponsor) = error.split_once("sponsor=")?;
|
let (_, sponsor) = error.split_once("sponsor=")?;
|
||||||
let endpoint = sponsor
|
let endpoint = sponsor
|
||||||
.split_whitespace()
|
.split_whitespace()
|
||||||
|
|
|
||||||
|
|
@ -15,7 +15,7 @@ use crate::records::memory::connections::{
|
||||||
spawn_peer_setup_retry, startup_synced_peer_streams, CONNECTIONS,
|
spawn_peer_setup_retry, startup_synced_peer_streams, CONNECTIONS,
|
||||||
};
|
};
|
||||||
use crate::records::memory::network_mapping::NodeInfo;
|
use crate::records::memory::network_mapping::NodeInfo;
|
||||||
use crate::records::memory::response_channels::{retire_peer_entries, Command};
|
use crate::records::memory::response_channels::Command;
|
||||||
use crate::records::memory::structs::Connection;
|
use crate::records::memory::structs::Connection;
|
||||||
use crate::rpc::client::genesis_compat::ensure_compatible_genesis;
|
use crate::rpc::client::genesis_compat::ensure_compatible_genesis;
|
||||||
use crate::rpc::client::governance_state_sync::sync_governance_state;
|
use crate::rpc::client::governance_state_sync::sync_governance_state;
|
||||||
|
|
@ -77,23 +77,32 @@ pub struct BootstrapParams {
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn spawn_bootstrap_peer_discovery(params: BootstrapParams) {
|
pub fn spawn_bootstrap_peer_discovery(params: BootstrapParams) {
|
||||||
let Some(startup_guard) = try_start_startup_discovery() else {
|
|
||||||
info!("[startup] startup synchronization is already active; coalescing duplicate request");
|
|
||||||
spawn_peer_setup_retry(params.connections_key, params.map);
|
|
||||||
return;
|
|
||||||
};
|
|
||||||
|
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
|
let startup_guard = loop {
|
||||||
|
if let Some(guard) = try_start_startup_discovery() {
|
||||||
|
break guard;
|
||||||
|
}
|
||||||
|
|
||||||
|
let current_stream = Connection::get_stream_from_memory(¶ms.connections_key).await;
|
||||||
|
if current_stream
|
||||||
|
.as_ref()
|
||||||
|
.map(|current| !Arc::ptr_eq(current, ¶ms.stream))
|
||||||
|
.unwrap_or(true)
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
sleep(Duration::from_millis(250)).await;
|
||||||
|
};
|
||||||
let _startup_guard = startup_guard;
|
let _startup_guard = startup_guard;
|
||||||
let failed_stream = params.stream.clone();
|
let failed_stream = params.stream.clone();
|
||||||
let failed_key = params.connections_key.clone();
|
let failed_key = params.connections_key.clone();
|
||||||
let command_map = params.map.clone();
|
|
||||||
if let Err(e) = bootstrap_peer_discovery(params).await {
|
if let Err(e) = bootstrap_peer_discovery(params).await {
|
||||||
warn!("[bootstrap] asynchronous peer setup failed for {failed_key}: {e}");
|
warn!("[bootstrap] asynchronous peer setup failed for {failed_key}: {e}");
|
||||||
|
// Request UIDs are allowed to expire independently. Retiring by
|
||||||
|
// endpoint here could cancel requests owned by a replacement
|
||||||
|
// stream connected to the same peer.
|
||||||
remove_stream_from_memory(&failed_stream).await;
|
remove_stream_from_memory(&failed_stream).await;
|
||||||
// The stream may already have been removed by the failing setup
|
|
||||||
// path. Retire its request waiters explicitly in either case.
|
|
||||||
retire_peer_entries(command_map, &failed_key).await;
|
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
@ -281,9 +290,9 @@ pub async fn bootstrap_peer_discovery(params: BootstrapParams) -> Result<(), Str
|
||||||
}
|
}
|
||||||
|
|
||||||
info!("[sync] post-sync checks complete, mining grace period started");
|
info!("[sync] post-sync checks complete, mining grace period started");
|
||||||
for (peer_key, _) in startup_synced_peer_streams().await {
|
for (peer_key, peer_stream) in startup_synced_peer_streams().await {
|
||||||
if !mark_peer_operational(&peer_key, params.map.clone()).await {
|
if !mark_peer_operational(&peer_key, &peer_stream, params.map.clone()).await {
|
||||||
spawn_peer_setup_retry(peer_key, params.map.clone());
|
spawn_peer_setup_retry(peer_key, peer_stream, params.map.clone());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
sleep(Duration::from_secs(15)).await;
|
sleep(Duration::from_secs(15)).await;
|
||||||
|
|
@ -477,7 +486,12 @@ pub async fn process_handshake_response(
|
||||||
"Wallet registry sync failed after handshake: {err}"
|
"Wallet registry sync failed after handshake: {err}"
|
||||||
)));
|
)));
|
||||||
}
|
}
|
||||||
mark_peer_wallet_registry_synced(&connections_key).await;
|
if !mark_peer_wallet_registry_synced(&connections_key, &stream).await {
|
||||||
|
remove_stream_from_memory(&stream).await;
|
||||||
|
return Err(io::Error::other(
|
||||||
|
"Wallet registry sync completed for a stale or missing connection",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
if let Err(err) = register_connected_wallet(
|
if let Err(err) = register_connected_wallet(
|
||||||
Arc::clone(&stream),
|
Arc::clone(&stream),
|
||||||
|
|
@ -556,33 +570,52 @@ pub async fn process_handshake_response(
|
||||||
})?;
|
})?;
|
||||||
|
|
||||||
if !params.first {
|
if !params.first {
|
||||||
let matched = compare_network_mapping_digest(
|
let matched = match compare_network_mapping_digest(
|
||||||
broadcast_stream.clone(),
|
broadcast_stream.clone(),
|
||||||
params.map.clone(),
|
params.map.clone(),
|
||||||
&connections_key,
|
&connections_key,
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
.map_err(|err| io::Error::other(format!("Network-map digest comparison failed: {err}")))?;
|
{
|
||||||
|
Ok(matched) => matched,
|
||||||
|
Err(err) => {
|
||||||
|
remove_stream_from_memory(&stream).await;
|
||||||
|
return Err(io::Error::other(format!(
|
||||||
|
"Network-map digest comparison failed: {err}"
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
if !matched {
|
if !matched {
|
||||||
info!("[network_map] reconciling signed mapping state with peer {connections_key}");
|
info!("[network_map] reconciling signed mapping state with peer {connections_key}");
|
||||||
reconcile_network_mapping_with_peer(
|
if let Err(err) = reconcile_network_mapping_with_peer(
|
||||||
broadcast_stream.clone(),
|
broadcast_stream.clone(),
|
||||||
params.map.clone(),
|
params.map.clone(),
|
||||||
&connections_key,
|
&connections_key,
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
.map_err(|err| io::Error::other(format!("Network-map reconciliation failed: {err}")))?;
|
{
|
||||||
|
remove_stream_from_memory(&stream).await;
|
||||||
|
return Err(io::Error::other(format!(
|
||||||
|
"Network-map reconciliation failed: {err}"
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
|
||||||
let reconciled = compare_network_mapping_digest(
|
let reconciled = match compare_network_mapping_digest(
|
||||||
broadcast_stream.clone(),
|
broadcast_stream.clone(),
|
||||||
params.map.clone(),
|
params.map.clone(),
|
||||||
&connections_key,
|
&connections_key,
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
.map_err(|err| {
|
{
|
||||||
io::Error::other(format!("Reconciled network-map comparison failed: {err}"))
|
Ok(reconciled) => reconciled,
|
||||||
})?;
|
Err(err) => {
|
||||||
|
remove_stream_from_memory(&stream).await;
|
||||||
|
return Err(io::Error::other(format!(
|
||||||
|
"Reconciled network-map comparison failed: {err}"
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
};
|
||||||
if !reconciled {
|
if !reconciled {
|
||||||
remove_stream_from_memory(&stream).await;
|
remove_stream_from_memory(&stream).await;
|
||||||
return Err(io::Error::other(format!(
|
return Err(io::Error::other(format!(
|
||||||
|
|
@ -591,7 +624,12 @@ pub async fn process_handshake_response(
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
mark_peer_network_map_synced(&connections_key).await;
|
if !mark_peer_network_map_synced(&connections_key, &stream).await {
|
||||||
|
remove_stream_from_memory(&stream).await;
|
||||||
|
return Err(io::Error::other(
|
||||||
|
"Network map sync completed for a stale or missing connection",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
if params.first {
|
if params.first {
|
||||||
let bsparams = BootstrapParams {
|
let bsparams = BootstrapParams {
|
||||||
|
|
@ -605,14 +643,14 @@ pub async fn process_handshake_response(
|
||||||
|
|
||||||
spawn_bootstrap_peer_discovery(bsparams);
|
spawn_bootstrap_peer_discovery(bsparams);
|
||||||
} else if is_normal_mode() {
|
} else if is_normal_mode() {
|
||||||
if !mark_peer_operational(&connections_key, params.map.clone()).await {
|
if !mark_peer_operational(&connections_key, &stream, params.map.clone()).await {
|
||||||
spawn_peer_setup_retry(connections_key.clone(), params.map.clone());
|
spawn_peer_setup_retry(connections_key.clone(), stream.clone(), params.map.clone());
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
// A concurrently discovered peer may finish setup after the startup
|
// A concurrently discovered peer may finish setup after the startup
|
||||||
// promotion snapshot. Retry until the canonical sync owner releases
|
// promotion snapshot. Retry until the canonical sync owner releases
|
||||||
// rather than leaving that valid peer passive indefinitely.
|
// rather than leaving that valid peer passive indefinitely.
|
||||||
spawn_peer_setup_retry(connections_key.clone(), params.map.clone());
|
spawn_peer_setup_retry(connections_key.clone(), stream.clone(), params.map.clone());
|
||||||
}
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -4,6 +4,7 @@ use crate::log::{error, info, warn};
|
||||||
use crate::orphans::structs::OrphanCheckup2;
|
use crate::orphans::structs::OrphanCheckup2;
|
||||||
use crate::orphans::sync_check::sync_checkup;
|
use crate::orphans::sync_check::sync_checkup;
|
||||||
use crate::records::block_height::get_block_height::get_height;
|
use crate::records::block_height::get_block_height::get_height;
|
||||||
|
use crate::records::memory::network_mapping::NodeInfo;
|
||||||
use crate::records::memory::response_channels::reserve_entry_with_context;
|
use crate::records::memory::response_channels::reserve_entry_with_context;
|
||||||
use crate::records::memory::response_channels::Command;
|
use crate::records::memory::response_channels::Command;
|
||||||
use crate::records::memory::torrent_status::{set_torrent_status, TorrentStatus};
|
use crate::records::memory::torrent_status::{set_torrent_status, TorrentStatus};
|
||||||
|
|
@ -34,9 +35,20 @@ fn is_mapping_sync_race(err: &str) -> bool {
|
||||||
|| err.contains("Miner wallet address is not registered")
|
|| err.contains("Miner wallet address is not registered")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn is_sync_transport_error(err: &str) -> bool {
|
||||||
|
err.contains("No connected miner peers available")
|
||||||
|
|| err.contains("No available peer could provide remaining pieces")
|
||||||
|
|| err.contains("Timed out waiting for torrent response")
|
||||||
|
|| err.contains("No torrent response received")
|
||||||
|
|| err.contains("Timed out waiting for piece reply")
|
||||||
|
|| err.contains("Piece reply channel closed")
|
||||||
|
|| err.contains("Peer could not provide piece")
|
||||||
|
|| err.contains("Failed to send torrent request")
|
||||||
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::is_mapping_sync_race;
|
use super::{is_mapping_sync_race, is_sync_transport_error};
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn missing_or_not_yet_eligible_miner_is_a_mapping_race() {
|
fn missing_or_not_yet_eligible_miner_is_a_mapping_race() {
|
||||||
|
|
@ -49,6 +61,17 @@ mod tests {
|
||||||
assert!(!is_mapping_sync_race("Invalid miner proof."));
|
assert!(!is_mapping_sync_race("Invalid miner proof."));
|
||||||
assert!(!is_mapping_sync_race("Incorrect previous_block_hash."));
|
assert!(!is_mapping_sync_race("Incorrect previous_block_hash."));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn peer_loss_is_not_sent_to_orphan_recovery() {
|
||||||
|
assert!(is_sync_transport_error(
|
||||||
|
"No connected miner peers available for block 15070"
|
||||||
|
));
|
||||||
|
assert!(is_sync_transport_error(
|
||||||
|
"Timed out waiting for torrent response at height 15070"
|
||||||
|
));
|
||||||
|
assert!(!is_sync_transport_error("Incorrect previous_block_hash."));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn is_transient_sync_error(err: &str) -> bool {
|
fn is_transient_sync_error(err: &str) -> bool {
|
||||||
|
|
@ -144,6 +167,15 @@ async fn handle_sync_error(
|
||||||
) -> io::Result<()> {
|
) -> io::Result<()> {
|
||||||
warn!("[sync] error saving block: height={height} err={err}");
|
warn!("[sync] error saving block: height={height} err={err}");
|
||||||
|
|
||||||
|
// Transport loss is not evidence of a fork. Release this sync owner so
|
||||||
|
// a replacement connection can resume from the persisted chain height.
|
||||||
|
if is_sync_transport_error(err) {
|
||||||
|
return Err(io::Error::new(
|
||||||
|
io::ErrorKind::ConnectionAborted,
|
||||||
|
format!("sync transport failed at height {height}: {err}"),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
if err.contains("Invalid reward for the Rewards Transaction")
|
if err.contains("Invalid reward for the Rewards Transaction")
|
||||||
|| err.contains("This address is not eligable to mine")
|
|| err.contains("This address is not eligable to mine")
|
||||||
|| err.contains("This miner address is not registered")
|
|| err.contains("This miner address is not registered")
|
||||||
|
|
@ -329,6 +361,9 @@ pub async fn node_syncing(
|
||||||
retry_counts.remove(&(next_to_save - 1));
|
retry_counts.remove(&(next_to_save - 1));
|
||||||
}
|
}
|
||||||
abort_pending_downloads(&mut pending);
|
abort_pending_downloads(&mut pending);
|
||||||
|
NodeInfo::persist_mined_counts(db)
|
||||||
|
.await
|
||||||
|
.map_err(io::Error::other)?;
|
||||||
info!("[sync] node syncing complete, awaiting post-sync checks before mining resumes");
|
info!("[sync] node syncing complete, awaiting post-sync checks before mining resumes");
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
|
use crate::records::memory::enums::ClientType;
|
||||||
use crate::records::memory::network_mapping::structs::{AddAddressParams, SignedNodeEdit};
|
use crate::records::memory::network_mapping::structs::{AddAddressParams, SignedNodeEdit};
|
||||||
use crate::records::memory::network_mapping::NodeInfo;
|
use crate::records::memory::network_mapping::NodeInfo;
|
||||||
use crate::records::memory::enums::ClientType;
|
|
||||||
use crate::records::memory::response_channels::Command;
|
use crate::records::memory::response_channels::Command;
|
||||||
use crate::rpc::read_bytes_from_stream;
|
use crate::rpc::read_bytes_from_stream;
|
||||||
use crate::rpc::responses::RpcResponse;
|
use crate::rpc::responses::RpcResponse;
|
||||||
|
|
|
||||||
|
|
@ -5,9 +5,9 @@ use crate::records::memory::connections::{
|
||||||
};
|
};
|
||||||
use crate::records::memory::enums::ClientType;
|
use crate::records::memory::enums::ClientType;
|
||||||
use crate::rpc::command_maps::{
|
use crate::rpc::command_maps::{
|
||||||
RPC_BLOCK_HEIGHT, RPC_BLOCK_PIECE, RPC_NETWORK_MEMBERSHIP_RECONCILE,
|
RPC_BLOCK_HEIGHT, RPC_BLOCK_PIECE, RPC_NETWORK_MEMBERSHIP_RECONCILE, RPC_NETWORK_MONITOR_ADD,
|
||||||
RPC_NETWORK_MONITOR_ADD, RPC_NETWORK_MONITOR_REMOVE, RPC_REPLY, RPC_SUBMIT_TORRENT,
|
RPC_NETWORK_MONITOR_REMOVE, RPC_REPLY, RPC_SUBMIT_TORRENT, RPC_SUBMIT_TRANSACTION,
|
||||||
RPC_SUBMIT_TRANSACTION, RPC_TORRENT_BY_HEIGHT,
|
RPC_TORRENT_BY_HEIGHT,
|
||||||
};
|
};
|
||||||
use crate::rpc::server::connection_memory_manager::remove_stream_from_memory;
|
use crate::rpc::server::connection_memory_manager::remove_stream_from_memory;
|
||||||
use crate::rpc::server::flood_protection::check_request_frequency_with_client_type;
|
use crate::rpc::server::flood_protection::check_request_frequency_with_client_type;
|
||||||
|
|
@ -108,9 +108,7 @@ pub async fn next_incoming_command(
|
||||||
|
|
||||||
let accepts_relay = if matches!(
|
let accepts_relay = if matches!(
|
||||||
command,
|
command,
|
||||||
RPC_NETWORK_MONITOR_ADD
|
RPC_NETWORK_MONITOR_ADD | RPC_NETWORK_MONITOR_REMOVE | RPC_NETWORK_MEMBERSHIP_RECONCILE
|
||||||
| RPC_NETWORK_MONITOR_REMOVE
|
|
||||||
| RPC_NETWORK_MEMBERSHIP_RECONCILE
|
|
||||||
) {
|
) {
|
||||||
peer_accepts_mapping_relay(connections_key).await
|
peer_accepts_mapping_relay(connections_key).await
|
||||||
} else {
|
} else {
|
||||||
|
|
|
||||||
|
|
@ -1,12 +1,12 @@
|
||||||
use crate::common::binary_conversions::{binary_to_ip, ip_to_binary};
|
use crate::common::binary_conversions::{binary_to_ip, ip_to_binary};
|
||||||
use crate::log::warn;
|
use crate::log::warn;
|
||||||
use crate::records::memory::connections::{
|
use crate::records::memory::connections::{
|
||||||
miner_connection_count, retire_peer_requests_from_runtime, set_stream_health,
|
miner_connection_count, set_stream_health, spawn_outage_recovery_from_runtime,
|
||||||
spawn_outage_recovery_from_runtime, spawn_retry_dropped_outgoing, CONNECTIONS,
|
spawn_retry_dropped_outgoing, CONNECTIONS,
|
||||||
};
|
};
|
||||||
use crate::records::memory::enums::{ClientType, ConnectionType};
|
use crate::records::memory::enums::{ClientType, ConnectionType};
|
||||||
use crate::records::memory::response_channels::{
|
use crate::records::memory::response_channels::{
|
||||||
delete_entry, reserve_entry_with_context, retire_peer_entries, Command,
|
delete_entry, reserve_entry_with_context, Command,
|
||||||
};
|
};
|
||||||
use crate::records::memory::structs::{ConnectionHealth, StoreConnectionParams};
|
use crate::records::memory::structs::{ConnectionHealth, StoreConnectionParams};
|
||||||
use crate::rpc::command_maps::RPC_BLOCK_HEIGHT;
|
use crate::rpc::command_maps::RPC_BLOCK_HEIGHT;
|
||||||
|
|
@ -167,7 +167,6 @@ async fn purge_stale_duplicate_miner(ip: &str, command_map: Arc<Mutex<Command>>)
|
||||||
);
|
);
|
||||||
connection.drop_connection(connection_type, duplicate_ip, duplicate_port);
|
connection.drop_connection(connection_type, duplicate_ip, duplicate_port);
|
||||||
drop(guard);
|
drop(guard);
|
||||||
retire_peer_entries(command_map, &duplicate_key).await;
|
|
||||||
true
|
true
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -337,7 +336,6 @@ pub async fn remove_stream_from_memory(stream: &Arc<Mutex<TcpStream>>) {
|
||||||
let ip = crate::common::binary_conversions::binary_to_ip(ip_bytes);
|
let ip = crate::common::binary_conversions::binary_to_ip(ip_bytes);
|
||||||
let dropped = connection.drop_connection(connection_type, ip.clone(), port);
|
let dropped = connection.drop_connection(connection_type, ip.clone(), port);
|
||||||
drop(connection_instance);
|
drop(connection_instance);
|
||||||
retire_peer_requests_from_runtime(&format!("{ip}:{port}")).await;
|
|
||||||
if connection_type == ConnectionType::Outgoing
|
if connection_type == ConnectionType::Outgoing
|
||||||
&& dropped
|
&& dropped
|
||||||
.as_ref()
|
.as_ref()
|
||||||
|
|
|
||||||
|
|
@ -2,8 +2,8 @@ use crate::records::ip_score::enums::InfractionType;
|
||||||
use crate::records::ip_score::score::update_ip_score;
|
use crate::records::ip_score::score::update_ip_score;
|
||||||
use crate::records::memory::enums::ClientType;
|
use crate::records::memory::enums::ClientType;
|
||||||
use crate::rpc::command_maps::{
|
use crate::rpc::command_maps::{
|
||||||
RPC_ADD_NETWORK_NODE, RPC_NETWORK_MAPPING_HASH, RPC_NETWORK_MONITOR_ADD,
|
RPC_ADD_NETWORK_NODE, RPC_NETWORK_MAPPING_HASH, RPC_NETWORK_MEMBERSHIP_RECONCILE,
|
||||||
RPC_NETWORK_MEMBERSHIP_RECONCILE, RPC_NETWORK_MONITOR_REMOVE, RPC_NETWORK_MONITOR_STATE,
|
RPC_NETWORK_MONITOR_ADD, RPC_NETWORK_MONITOR_REMOVE, RPC_NETWORK_MONITOR_STATE,
|
||||||
RPC_SETUP_COMPLETE,
|
RPC_SETUP_COMPLETE,
|
||||||
};
|
};
|
||||||
use crate::rpc::server::structs::RpcFloodState;
|
use crate::rpc::server::structs::RpcFloodState;
|
||||||
|
|
|
||||||
|
|
@ -8,7 +8,7 @@ use crate::orphans::torrent_candidates::hydrate_torrent_candidates;
|
||||||
use crate::records::block_height::get_block_height::get_height;
|
use crate::records::block_height::get_block_height::get_height;
|
||||||
use crate::records::memory::connections::{
|
use crate::records::memory::connections::{
|
||||||
mark_peer_catching_up, mark_peer_network_map_synced, mark_peer_operational,
|
mark_peer_catching_up, mark_peer_network_map_synced, mark_peer_operational,
|
||||||
mark_peer_wallet_registry_synced, spawn_peer_setup_retry,
|
mark_peer_wallet_registry_synced, peer_connection_count, spawn_peer_setup_retry,
|
||||||
};
|
};
|
||||||
use crate::records::memory::enums::ClientType;
|
use crate::records::memory::enums::ClientType;
|
||||||
use crate::records::memory::response_channels::generate_uid;
|
use crate::records::memory::response_channels::generate_uid;
|
||||||
|
|
@ -16,7 +16,9 @@ use crate::records::memory::response_channels::Command;
|
||||||
use crate::records::memory::structs::Connection;
|
use crate::records::memory::structs::Connection;
|
||||||
use crate::rpc::client::genesis_compat::ensure_compatible_genesis;
|
use crate::rpc::client::genesis_compat::ensure_compatible_genesis;
|
||||||
use crate::rpc::client::governance_state_sync::sync_governance_state;
|
use crate::rpc::client::governance_state_sync::sync_governance_state;
|
||||||
|
use crate::rpc::client::handshake::{connect_and_handshake, sponsor_endpoint_from_error};
|
||||||
use crate::rpc::client::register_wallet::register_connected_wallet;
|
use crate::rpc::client::register_wallet::register_connected_wallet;
|
||||||
|
use crate::rpc::client::structs::Connect;
|
||||||
use crate::rpc::client::syncing::node_syncing;
|
use crate::rpc::client::syncing::node_syncing;
|
||||||
use crate::rpc::client::wallet_registry_sync::sync_wallet_registry_with_retries;
|
use crate::rpc::client::wallet_registry_sync::sync_wallet_registry_with_retries;
|
||||||
use crate::rpc::responses::RpcResponse;
|
use crate::rpc::responses::RpcResponse;
|
||||||
|
|
@ -28,7 +30,7 @@ use crate::rpc::server::tests::{endpoint_port, is_port_open};
|
||||||
use crate::sled::Db;
|
use crate::sled::Db;
|
||||||
use crate::sleep;
|
use crate::sleep;
|
||||||
use crate::startup::network_broadcast::{
|
use crate::startup::network_broadcast::{
|
||||||
announce_self_to_network, get_network_mapping_for_address,
|
announce_self_to_network, get_network_mapping_for_address, reconcile_network_mapping_with_peer,
|
||||||
};
|
};
|
||||||
use crate::startup::remote_height::request_remote_height;
|
use crate::startup::remote_height::request_remote_height;
|
||||||
use crate::wallets::structures::Wallet;
|
use crate::wallets::structures::Wallet;
|
||||||
|
|
@ -41,7 +43,6 @@ use crate::TcpStream;
|
||||||
use crate::Utc;
|
use crate::Utc;
|
||||||
|
|
||||||
const LIVE_ORPHAN_WINDOW: u32 = 10;
|
const LIVE_ORPHAN_WINDOW: u32 = 10;
|
||||||
const PASSIVE_CATCH_UP_BLOCK_LIMIT: u32 = 20;
|
|
||||||
|
|
||||||
fn peer_reached_catch_up_target(
|
fn peer_reached_catch_up_target(
|
||||||
local_height: u32,
|
local_height: u32,
|
||||||
|
|
@ -52,11 +53,6 @@ fn peer_reached_catch_up_target(
|
||||||
&& local_height.saturating_sub(remote_height) <= LIVE_ORPHAN_WINDOW
|
&& local_height.saturating_sub(remote_height) <= LIVE_ORPHAN_WINDOW
|
||||||
}
|
}
|
||||||
|
|
||||||
fn passive_catch_up_expired(local_height: u32, remote_height: u32, catch_up_target: u32) -> bool {
|
|
||||||
local_height.saturating_sub(catch_up_target) > PASSIVE_CATCH_UP_BLOCK_LIMIT
|
|
||||||
&& local_height.saturating_sub(remote_height) > LIVE_ORPHAN_WINDOW
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn drop_failed_handshake(stream: &Arc<Mutex<TcpStream>>) {
|
async fn drop_failed_handshake(stream: &Arc<Mutex<TcpStream>>) {
|
||||||
// Failed handshakes are never stored in connection memory, but the
|
// Failed handshakes are never stored in connection memory, but the
|
||||||
// accepted TCP socket should still be closed immediately.
|
// accepted TCP socket should still be closed immediately.
|
||||||
|
|
@ -234,43 +230,32 @@ fn spawn_incoming_peer_promotion_watcher(
|
||||||
};
|
};
|
||||||
|
|
||||||
if peer_reached_catch_up_target(local_height, remote_height, catch_up_target) {
|
if peer_reached_catch_up_target(local_height, remote_height, catch_up_target) {
|
||||||
if mark_peer_operational(&connections_key, map.clone()).await {
|
if mark_peer_operational(&connections_key, &stream, map.clone()).await {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
if passive_catch_up_expired(local_height, remote_height, catch_up_target) {
|
|
||||||
warn!(
|
|
||||||
"[startup] incoming peer failed to catch up before passive limit; disconnecting: peer={connections_key} target_height={catch_up_target} local_height={local_height} remote_height={remote_height}"
|
|
||||||
);
|
|
||||||
remove_stream_from_memory(&stream).await;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
fn needs_reverse_bootstrap_announcement(
|
fn needs_reverse_membership_announcement(
|
||||||
self_add_allowed: bool,
|
|
||||||
local_membership_active: bool,
|
local_membership_active: bool,
|
||||||
|
has_operational_peer: bool,
|
||||||
) -> bool {
|
) -> bool {
|
||||||
self_add_allowed && !local_membership_active
|
!local_membership_active || !has_operational_peer
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::{
|
use super::{needs_reverse_membership_announcement, peer_reached_catch_up_target};
|
||||||
needs_reverse_bootstrap_announcement, passive_catch_up_expired,
|
|
||||||
peer_reached_catch_up_target,
|
|
||||||
};
|
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn reverse_bootstrap_announcement_is_only_for_an_unregistered_first_node() {
|
fn inactive_local_membership_requires_peer_sponsorship_at_every_height() {
|
||||||
assert!(needs_reverse_bootstrap_announcement(true, false));
|
assert!(needs_reverse_membership_announcement(false, false));
|
||||||
assert!(!needs_reverse_bootstrap_announcement(true, true));
|
assert!(needs_reverse_membership_announcement(false, true));
|
||||||
assert!(!needs_reverse_bootstrap_announcement(false, false));
|
assert!(needs_reverse_membership_announcement(true, false));
|
||||||
assert!(!needs_reverse_bootstrap_announcement(false, true));
|
assert!(!needs_reverse_membership_announcement(true, true));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|
@ -283,12 +268,6 @@ mod tests {
|
||||||
assert!(!peer_reached_catch_up_target(66_091, 66_090, 66_091));
|
assert!(!peer_reached_catch_up_target(66_091, 66_090, 66_091));
|
||||||
assert!(!peer_reached_catch_up_target(66_120, 66_091, 66_091));
|
assert!(!peer_reached_catch_up_target(66_120, 66_091, 66_091));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn stale_passive_peer_expires_only_after_falling_outside_live_window() {
|
|
||||||
assert!(!passive_catch_up_expired(66_112, 66_102, 66_091));
|
|
||||||
assert!(passive_catch_up_expired(66_112, 66_100, 66_091));
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn complete_incoming_miner_setup(
|
async fn complete_incoming_miner_setup(
|
||||||
|
|
@ -319,7 +298,13 @@ async fn complete_incoming_miner_setup(
|
||||||
);
|
);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
mark_peer_wallet_registry_synced(connections_key).await;
|
if !mark_peer_wallet_registry_synced(connections_key, &stream).await {
|
||||||
|
warn!(
|
||||||
|
"[startup] incoming peer wallet sync completed for a stale or missing stream: peer={connections_key}"
|
||||||
|
);
|
||||||
|
remove_stream_from_memory(&stream).await;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
if let Err(err) = register_connected_wallet(
|
if let Err(err) = register_connected_wallet(
|
||||||
stream.clone(),
|
stream.clone(),
|
||||||
|
|
@ -371,20 +356,16 @@ async fn complete_incoming_miner_setup(
|
||||||
remove_stream_from_memory(&stream).await;
|
remove_stream_from_memory(&stream).await;
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
let self_add_allowed =
|
|
||||||
crate::records::memory::network_mapping::NodeInfo::self_add_allowed_at_height(
|
|
||||||
remote_height,
|
|
||||||
);
|
|
||||||
let short_address = wallet.saved.short_address.clone();
|
let short_address = wallet.saved.short_address.clone();
|
||||||
let local_membership_active =
|
let local_membership_active =
|
||||||
crate::records::memory::network_mapping::NodeInfo::is_active_address(&short_address).await;
|
crate::records::memory::network_mapping::NodeInfo::is_active_address(&short_address).await;
|
||||||
|
let has_operational_peer = peer_connection_count().await > 0;
|
||||||
|
|
||||||
if needs_reverse_bootstrap_announcement(self_add_allowed, local_membership_active) {
|
if needs_reverse_membership_announcement(local_membership_active, has_operational_peer) {
|
||||||
// Before the sponsored-add gate, reverse announcement lets tiny early
|
// An inactive local node announces itself to the connected peer at
|
||||||
// networks bootstrap from one live peer. It is only valid while this
|
// every chain height. The peer sponsors and signs the membership; this
|
||||||
// receiving node has no active membership of its own. An established
|
// node never adds itself. The receiver's normal sponsor-eligibility
|
||||||
// node must never import even its own record from a newly connected
|
// rules still apply after the 10,000-block boundary.
|
||||||
// peer.
|
|
||||||
if let Err(err) = announce_self_to_network(
|
if let Err(err) = announce_self_to_network(
|
||||||
stream.clone(),
|
stream.clone(),
|
||||||
&short_address,
|
&short_address,
|
||||||
|
|
@ -395,9 +376,43 @@ async fn complete_incoming_miner_setup(
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
|
let Some(sponsor_endpoint) = sponsor_endpoint_from_error(&err) else {
|
||||||
error!("[startup] incoming peer network map sync failed: {err}");
|
error!("[startup] incoming peer network map sync failed: {err}");
|
||||||
remove_stream_from_memory(&stream).await;
|
remove_stream_from_memory(&stream).await;
|
||||||
return;
|
return;
|
||||||
|
};
|
||||||
|
let Ok(sponsor_addr) = sponsor_endpoint.parse() else {
|
||||||
|
error!(
|
||||||
|
"[startup] incoming peer returned an invalid sponsor endpoint: {sponsor_endpoint}"
|
||||||
|
);
|
||||||
|
remove_stream_from_memory(&stream).await;
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
|
||||||
|
warn!(
|
||||||
|
"[startup] incoming peer cannot sponsor local membership; connecting to eligible sponsor {sponsor_endpoint}"
|
||||||
|
);
|
||||||
|
let sponsor = Connect {
|
||||||
|
addr: sponsor_addr,
|
||||||
|
node_ip: sponsor_endpoint.clone(),
|
||||||
|
db: db.clone(),
|
||||||
|
wallet: wallet.clone(),
|
||||||
|
map: map.clone(),
|
||||||
|
first: true,
|
||||||
|
};
|
||||||
|
if let Err(sponsor_err) = connect_and_handshake(sponsor).await {
|
||||||
|
error!(
|
||||||
|
"[startup] eligible sponsor connection failed: original={err}; sponsor={sponsor_endpoint}; error={sponsor_err}"
|
||||||
|
);
|
||||||
|
remove_stream_from_memory(&stream).await;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// A sponsor rejection is a terminal response on the original
|
||||||
|
// connection. The redirected outgoing connection now owns local
|
||||||
|
// startup and mapping synchronization.
|
||||||
|
remove_stream_from_memory(&stream).await;
|
||||||
|
return;
|
||||||
}
|
}
|
||||||
if let Err(err) = get_network_mapping_for_address(
|
if let Err(err) = get_network_mapping_for_address(
|
||||||
stream.clone(),
|
stream.clone(),
|
||||||
|
|
@ -413,7 +428,27 @@ async fn complete_incoming_miner_setup(
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
mark_peer_network_map_synced(connections_key).await;
|
|
||||||
|
// The server side never replaces its established map with a connecting
|
||||||
|
// peer's snapshot. It does, however, send its signed membership and
|
||||||
|
// monitor state to that peer. On additional connections the outgoing side
|
||||||
|
// performs the reciprocal push, allowing deterministic record validation
|
||||||
|
// to converge both maps without assigning either connection direction
|
||||||
|
// unconditional authority.
|
||||||
|
if let Err(err) =
|
||||||
|
reconcile_network_mapping_with_peer(stream.clone(), map.clone(), connections_key).await
|
||||||
|
{
|
||||||
|
error!("[startup] incoming peer network-map reconciliation failed: {err}");
|
||||||
|
remove_stream_from_memory(&stream).await;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if !mark_peer_network_map_synced(connections_key, &stream).await {
|
||||||
|
warn!(
|
||||||
|
"[startup] incoming peer mapping sync completed for a stale or missing stream: peer={connections_key}"
|
||||||
|
);
|
||||||
|
remove_stream_from_memory(&stream).await;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
let (operational, chain_sync_guard, catch_up_target) =
|
let (operational, chain_sync_guard, catch_up_target) =
|
||||||
match sync_incoming_peer_before_operational(
|
match sync_incoming_peer_before_operational(
|
||||||
|
|
@ -434,8 +469,8 @@ async fn complete_incoming_miner_setup(
|
||||||
};
|
};
|
||||||
|
|
||||||
if operational {
|
if operational {
|
||||||
if !mark_peer_operational(connections_key, map.clone()).await {
|
if !mark_peer_operational(connections_key, &stream, map.clone()).await {
|
||||||
spawn_peer_setup_retry(connections_key.to_string(), map.clone());
|
spawn_peer_setup_retry(connections_key.to_string(), stream.clone(), map.clone());
|
||||||
}
|
}
|
||||||
sleep(Duration::from_secs(15)).await;
|
sleep(Duration::from_secs(15)).await;
|
||||||
if let Some(guard) = chain_sync_guard {
|
if let Some(guard) = chain_sync_guard {
|
||||||
|
|
|
||||||
|
|
@ -3,7 +3,7 @@ use crate::records::memory::enums::ClientType;
|
||||||
use crate::records::memory::response_channels::{generate_uid, Command};
|
use crate::records::memory::response_channels::{generate_uid, Command};
|
||||||
use crate::rpc::responses::RpcResponse;
|
use crate::rpc::responses::RpcResponse;
|
||||||
use crate::rpc::server::structs::HandshakeTestParams;
|
use crate::rpc::server::structs::HandshakeTestParams;
|
||||||
use crate::rpc::server::tests::{ip_test, is_within_one_second};
|
use crate::rpc::server::tests::{ip_test, is_within_two_seconds};
|
||||||
use crate::wallets::structures::Wallet;
|
use crate::wallets::structures::Wallet;
|
||||||
use crate::Arc;
|
use crate::Arc;
|
||||||
use crate::Mutex;
|
use crate::Mutex;
|
||||||
|
|
@ -57,7 +57,7 @@ pub async fn verify_timestamp(
|
||||||
peer_time: u32,
|
peer_time: u32,
|
||||||
timestamp: u32,
|
timestamp: u32,
|
||||||
) -> bool {
|
) -> bool {
|
||||||
if !is_within_one_second(timestamp, peer_time) {
|
if !is_within_two_seconds(timestamp, peer_time) {
|
||||||
// Tight timestamp checks keep peers from replaying stale signed
|
// Tight timestamp checks keep peers from replaying stale signed
|
||||||
// handshakes after the connection attempt has passed.
|
// handshakes after the connection attempt has passed.
|
||||||
let hashmap_key = generate_uid();
|
let hashmap_key = generate_uid();
|
||||||
|
|
@ -66,7 +66,7 @@ pub async fn verify_timestamp(
|
||||||
|
|
||||||
let response_bytes = RpcResponse::Binary({
|
let response_bytes = RpcResponse::Binary({
|
||||||
let msg = format!(
|
let msg = format!(
|
||||||
"error: Handshake Failed: The time on your computer must be within 1 second of the time of our server. Your local time: {peer_time}. Our server time: {timestamp}. Please consider installing NTP to ensure proper timestamps."
|
"error: Handshake Failed: The time on your computer must be within 2 seconds of the time of our server. Your local time: {peer_time}. Our server time: {timestamp}. Please consider installing NTP to ensure proper timestamps."
|
||||||
)
|
)
|
||||||
.as_bytes()
|
.as_bytes()
|
||||||
.to_vec();
|
.to_vec();
|
||||||
|
|
|
||||||
|
|
@ -640,7 +640,8 @@ pub async fn start_loop(
|
||||||
stream_locked.clone(),
|
stream_locked.clone(),
|
||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
mark_peer_remote_setup_complete(&connections_key, map.clone()).await;
|
mark_peer_remote_setup_complete(&connections_key, &stream_locked, map.clone())
|
||||||
|
.await;
|
||||||
let result = responses::RpcResponse::Binary(b"setup_complete_ack".to_vec());
|
let result = responses::RpcResponse::Binary(b"setup_complete_ack".to_vec());
|
||||||
result
|
result
|
||||||
.send(&stream_locked, Some(&connections_key), uid)
|
.send(&stream_locked, Some(&connections_key), uid)
|
||||||
|
|
|
||||||
|
|
@ -48,8 +48,21 @@ pub async fn is_port_open(ip_port: &str) -> Result<bool, Box<dyn std::error::Err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// verify connecting client time is within 1 second of our own.
|
// Verify connecting client time is within two seconds of our own.
|
||||||
pub fn is_within_one_second(timestamp1: u32, timestamp2: u32) -> bool {
|
pub fn is_within_two_seconds(timestamp1: u32, timestamp2: u32) -> bool {
|
||||||
let duration = timestamp1.wrapping_sub(timestamp2);
|
let duration = timestamp1.wrapping_sub(timestamp2);
|
||||||
duration <= 1 || timestamp2.wrapping_sub(timestamp1) <= 1
|
duration <= 2 || timestamp2.wrapping_sub(timestamp1) <= 2
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod timestamp_tests {
|
||||||
|
use super::is_within_two_seconds;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn handshake_clock_tolerance_includes_two_seconds_only() {
|
||||||
|
assert!(is_within_two_seconds(100, 98));
|
||||||
|
assert!(is_within_two_seconds(98, 100));
|
||||||
|
assert!(!is_within_two_seconds(100, 97));
|
||||||
|
assert!(!is_within_two_seconds(97, 100));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -3,12 +3,13 @@ use crate::common::network_startup::{get_ip_and_port, get_node_connections};
|
||||||
use crate::log::{error, info, warn};
|
use crate::log::{error, info, warn};
|
||||||
use crate::miner::flag::{is_mining_stop_requested, is_normal_mode};
|
use crate::miner::flag::{is_mining_stop_requested, is_normal_mode};
|
||||||
use crate::records::memory::connections::{
|
use crate::records::memory::connections::{
|
||||||
operational_peer_wallets, peer_connection_count, ready_outgoing_connection_count,
|
miner_connection_count, operational_peer_wallets, peer_connection_count,
|
||||||
refill_outgoing_connections_once,
|
ready_outgoing_connection_count, refill_outgoing_connections_once,
|
||||||
};
|
};
|
||||||
use crate::records::memory::network_mapping::NodeInfo;
|
use crate::records::memory::network_mapping::NodeInfo;
|
||||||
use crate::records::memory::response_channels::Command;
|
use crate::records::memory::response_channels::Command;
|
||||||
use crate::rpc::client::handshake::connect_and_handshake;
|
use crate::rpc::client::handshake::connect_and_handshake;
|
||||||
|
use crate::rpc::client::handshake::outbound_handshake_in_progress;
|
||||||
use crate::rpc::client::structs::Connect;
|
use crate::rpc::client::structs::Connect;
|
||||||
use crate::sled::Db;
|
use crate::sled::Db;
|
||||||
use crate::sleep;
|
use crate::sleep;
|
||||||
|
|
@ -65,6 +66,7 @@ pub async fn handle_connections(
|
||||||
// dialing the list. Only the first two nodes establishing genesis require
|
// dialing the list. Only the first two nodes establishing genesis require
|
||||||
// reciprocal sponsorship; mature restarts need an operational peer.
|
// reciprocal sponsorship; mature restarts need an operational peer.
|
||||||
info!("No existing network peer is available. Waiting for connections.");
|
info!("No existing network peer is available. Waiting for connections.");
|
||||||
|
let mut retry_seconds = 0_u64;
|
||||||
loop {
|
loop {
|
||||||
let peer_wallets = operational_peer_wallets().await;
|
let peer_wallets = operational_peer_wallets().await;
|
||||||
let sponsorship_ready = if first_two_node_startup {
|
let sponsorship_ready = if first_two_node_startup {
|
||||||
|
|
@ -80,6 +82,17 @@ pub async fn handle_connections(
|
||||||
}
|
}
|
||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// The normal isolated-recovery task is started only after this
|
||||||
|
// function returns. While startup is still waiting, periodically run
|
||||||
|
// that same single-owner recovery path so a temporarily unavailable
|
||||||
|
// configured peer cannot strand the node forever. A live or
|
||||||
|
// still-setting-up miner socket suppresses the attempt.
|
||||||
|
retry_seconds += 1;
|
||||||
|
if outgoing_connections > 0 && retry_seconds >= 60 {
|
||||||
|
retry_seconds = 0;
|
||||||
|
spawn_outage_recovery(db.clone(), wallet.clone(), map.clone());
|
||||||
|
}
|
||||||
sleep(Duration::from_secs(1)).await;
|
sleep(Duration::from_secs(1)).await;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -198,7 +211,7 @@ pub fn spawn_outage_recovery(db: Db, wallet: Arc<Wallet>, map: Arc<Mutex<Command
|
||||||
// A raw socket that never completed setup is not network recovery.
|
// A raw socket that never completed setup is not network recovery.
|
||||||
// Only an operational miner peer should suppress configured-peer
|
// Only an operational miner peer should suppress configured-peer
|
||||||
// outage recovery.
|
// outage recovery.
|
||||||
if peer_connection_count().await != 0 {
|
if outbound_handshake_in_progress() || miner_connection_count().await != 0 {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -273,11 +273,10 @@ pub async fn reconcile_network_mapping_with_peer(
|
||||||
.await?;
|
.await?;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Reconciliation on an additional/discovered connection is intentionally
|
// This operation is deliberately a signed-state push, never a snapshot
|
||||||
// one-way. This node already joined through its bootstrap owner, so the
|
// replacement. Incoming setup performs the corresponding push in the
|
||||||
// newly connected peer may merge our signed state but must never replace
|
// other direction, so each node validates and merges the peer's records
|
||||||
// this node's established mapping. Missing records continue to arrive
|
// using the same deterministic rules.
|
||||||
// through the normal redundant network broadcasts.
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -186,7 +186,6 @@ pub async fn download_block_pieces(params: DownloadSave) -> Result<(), String> {
|
||||||
.try_into()
|
.try_into()
|
||||||
.map_err(|_| "Invalid torrent info_hash length".to_string())?,
|
.map_err(|_| "Invalid torrent info_hash length".to_string())?,
|
||||||
);
|
);
|
||||||
|
|
||||||
// Keep polling until every piece has either been downloaded or the
|
// Keep polling until every piece has either been downloaded or the
|
||||||
// target block becomes obsolete because the chain tip advanced.
|
// target block becomes obsolete because the chain tip advanced.
|
||||||
loop {
|
loop {
|
||||||
|
|
@ -322,8 +321,8 @@ pub async fn download_block_pieces(params: DownloadSave) -> Result<(), String> {
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// Poll more frequently so a fully downloaded block moves into
|
// Many block downloads can be active concurrently. Keep this poll
|
||||||
// combine/verify/save without an extra 1-second stall.
|
// bounded so waiting tasks do not spin and starve network processing.
|
||||||
sleep(Duration::from_millis(50)).await;
|
sleep(Duration::from_millis(50)).await;
|
||||||
}
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
|
|
|
||||||
|
|
@ -30,8 +30,14 @@ pub async fn send_request_torrent_message(
|
||||||
message.extend(request_torrent_binary);
|
message.extend(request_torrent_binary);
|
||||||
message.extend(hashmap_key);
|
message.extend(hashmap_key);
|
||||||
message.extend(get_height_binary);
|
message.extend(get_height_binary);
|
||||||
RpcResponse::send_raw(&stream, Some(&connections_key), &message).await;
|
if RpcResponse::send_raw(&stream, Some(&connections_key), &message).await {
|
||||||
Ok(())
|
Ok(())
|
||||||
|
} else {
|
||||||
|
Err(io::Error::new(
|
||||||
|
io::ErrorKind::ConnectionAborted,
|
||||||
|
format!("Failed to send torrent request to {connections_key}"),
|
||||||
|
))
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn handle_response_and_save_torrent(
|
pub async fn handle_response_and_save_torrent(
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue