sdk%refac: consolidate exports, types, add utf8 serde codec to process domain names, preserve unknown values, drop legacy macro

contrib/codeql/decl.ql

@@ -5,7 +5,7 @@
  *
  * @id base-sdk/decl-rules
  * @name Rules for definition orders
- * @description Enforces order of definitions for enums, impls and structs.
+ * @description Enforces definition order and enum variant structure.
  * @kind problem
  * @precision high
  * @problem.severity warning
@@ -36,19 +36,66 @@ predicate outOfOrder(
   )
 }
 
-from TypeItem t, Locatable badItem, string name, string message
+/** Holds if `v` is a bare `Unknown` variant without associated data. */
+predicate bareUnknownVariant(Enum e, Variant v) {
+  isSourceType(e) and
+  v = e.getVariantList().getAVariant() and
+  v.getName().getText() = "Unknown" and
+  not exists(v.getFieldList())
+}
+
+/** Gets the NumCodec type parameter for enum `e`. */
+string numCodecType(Enum e) {
+  exists(Impl i |
+    fileOf(i) = fileOf(e) and
+    implSelfName(i) = e.getName().getText() and
+    implTraitName(i) = "NumCodec" and
+    result =
+      i.getTrait()
+          .(PathTypeRepr)
+          .getPath()
+          .getSegment()
+          .getGenericArgList()
+          .getGenericArg(0)
+          .(TypeArg)
+          .getTypeRepr()
+          .(PathTypeRepr)
+          .getPath()
+          .getSegment()
+          .getIdentifier()
+          .getText()
+  )
+}
+
+from Locatable item, string message
 where
-  isSourceType(t) and
-  (t instanceof Struct or t instanceof Enum) and
-  not isSerdeInternalType(t) and
-  not isNotEncodable(t) and
-  isEvaluatedCrate(fileOf(t)) and
-  name = t.getName().getText() and
-  exists(DeclSlot badSlot, int badLine, DeclSlot priorSlot |
-    outOfOrder(t, badSlot, badLine, priorSlot, badItem) and
-    message =
-      fmt("{0} {1} appears after {2}", name,
-        fmt("{0} (slot {1})", badSlot.toString(), badSlot.getOrder().toString()),
-        fmt("{0} (slot {1})", priorSlot.toString(), priorSlot.getOrder().toString()))
+  exists(TypeItem t, string name |
+    isSourceType(t) and
+    (t instanceof Struct or t instanceof Enum) and
+    not isSerdeInternalType(t) and
+    not isNotEncodable(t) and
+    isEvaluatedCrate(fileOf(t)) and
+    name = t.getName().getText() and
+    exists(DeclSlot badSlot, int badLine, DeclSlot priorSlot |
+      outOfOrder(t, badSlot, badLine, priorSlot, item) and
+      message =
+        fmt("{0} {1} appears after {2}", name,
+          fmt("{0} (slot {1})", badSlot.toString(), badSlot.getOrder().toString()),
+          fmt("{0} (slot {1})", priorSlot.toString(), priorSlot.getOrder().toString()))
+    )
+  )
+  or
+  exists(Enum e |
+    bareUnknownVariant(e, item) and
+    (
+      exists(string ty |
+        ty = numCodecType(e) and
+        message =
+          fmt("{0}::Unknown must carry the raw value (e.g. Unknown({1}))", e.getName().getText(), ty)
+      )
+      or
+      not exists(numCodecType(e)) and
+      message = fmt("{0}::Unknown must carry the raw value", e.getName().getText())
+    )
   )
-select badItem, message
+select item, message

contrib/semgrep/types.yml

@@ -0,0 +1,9 @@
+rules:
+  - id: no-newtype-in-types
+    message: "do not define newtypes in dash-types; use the owning crate instead"
+    severity: ERROR
+    languages: [rust]
+    paths:
+      include: [/pkgs/types/src/**/*.rs]
+      exclude: [/pkgs/types/src/hex.rs, /pkgs/types/src/uint.rs]
+    pattern-regex: '\b(?:make|impl)_(?:bytes|num|type)!\s*\{'

pkgs/dev/src/lambda.rs

@@ -62,7 +62,7 @@ where
 ///
 /// Panics on txid mismatch.
 fn assert_txid(raw: &[u8], label: &str) {
-  let computed = dash_primitives::hash::tx_hash(raw);
+  let computed = dash_primitives::tx_hash(raw);
   let expected = TxHash::from_hex(label).unwrap_or_else(|e| panic!("{label}: bad txid hex: {e}"));
   assert_eq!(computed, expected, "{label}: txid mismatch");
 }

pkgs/num/src/hash.rs

@@ -6,14 +6,40 @@
 
 //! Fixed-size opaque hash blob types.
 
-use crate::ParseHexError;
-
 use core::fmt;
 use core::hash::Hash;
 use core::str::FromStr;
 
 pub(crate) const HEX_LOWER: [u8; 16] = *b"0123456789abcdef";
 
+/// Error returned when parsing a hex string fails.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub enum ParseHexError {
+  /// The hex string has an odd number of characters.
+  OddLength,
+  /// The decoded byte count does not match the expected length.
+  InvalidLength { expected: usize, got: usize },
+  /// A non-hex character was encountered.
+  InvalidChar(u8),
+}
+
+impl fmt::Display for ParseHexError {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    match self {
+      Self::OddLength => write!(f, "hex string has odd length"),
+      Self::InvalidLength { expected, got } => {
+        write!(f, "expected {expected} hex chars, got {got}")
+      }
+      Self::InvalidChar(c) => {
+        write!(f, "invalid hex character: {:#04x}", c)
+      }
+    }
+  }
+}
+
+#[cfg(feature = "std")]
+impl std::error::Error for ParseHexError {}
+
 pub(crate) fn hex_val(c: u8) -> Result<u8, ParseHexError> {
   match c {
     b'0'..=b'9' => Ok(c - b'0'),

pkgs/num/src/lib.rs

@@ -18,7 +18,6 @@ extern crate std;
 mod arith;
 mod arith256;
 mod compact;
-mod error;
 mod hash;
 #[allow(unused_imports, reason = "ergonomic shim, exports may be unused")]
 mod prelude;
@@ -34,5 +33,4 @@ pub mod __private {
 pub use arith::ArithInt;
 pub use arith256::Arith256;
 pub use compact::{CompactTarget, DecodedTarget};
-pub use error::ParseHexError;
-pub use hash::{Hash160, Hash256, Hash512, HashBlob};
+pub use hash::{Hash160, Hash256, Hash512, HashBlob, ParseHexError};

pkgs/num/src/util.rs

@@ -141,11 +141,3 @@ macro_rules! make_hash {
     $crate::impl_hash!($base, $name);
   };
 }
-
-/// Convenience alias: generates a `Hash256`-based newtype.
-#[macro_export]
-macro_rules! make_hash256 {
-  ($(#[$attr:meta])* $name:ident) => {
-    $crate::make_hash!($crate::Hash256, $(#[$attr])* $name);
-  };
-}

pkgs/p2p_core/src/lib.rs

@@ -13,26 +13,22 @@ extern crate alloc;
 extern crate std;
 
 mod codec;
+mod error;
+mod msg;
 #[allow(unused_imports, reason = "ergonomic shim, exports may be unused")]
 mod prelude;
-
-pub mod error;
-pub mod msg;
-pub mod primitives;
-pub mod v2;
+mod primitives;
+mod v2;
 
 pub use error::P2pDecodeError;
-pub use msg::DashNetworkMessage;
+pub use msg::{
+  Addr, AddrV2Entry, AddrV2Msg, CFCheckpt, CFHeaders, CFilter, DashNetworkMessage, FilterType, GetCFCheckpt,
+  GetCFHeaders, GetCFilters, GetData, GetHeaders, GetHeaders2, GovSync, Headers, Headers2, Inv, NotFound, Ping, Pong,
+  TimestampedAddr, Version, VersionAddr,
+};
 pub use primitives::{
-  command::CommandString,
-  compressed_header::CompressionState,
-  filter_type::FilterType,
-  inventory::{InvType, Inventory},
-  magic::Magic,
-  mn_list::{MnListDiffPayload, SimplifiedMnListEntry},
-  net_addr::{AddrV2, AddrV2Entry, NetAddr, TimestampedAddr},
-  protocol_version::ProtocolVersion,
-  service_flags::ServiceFlags,
-  short_id::ShortId,
-  user_agent::UserAgent,
+  CommandString, CompressionState, DeletedQuorum, GetMnListDiff, InvType, Inventory, Magic, MnListDiff,
+  MnListDiffPayload, ProtocolVersion, QuorumClSig, ServiceFlags, ShortId, SimplifiedMnListEntry, UserAgent,
+  UserAgentTooLong,
 };
+pub use v2::{decode_v2, encode_v2};

pkgs/p2p_core/src/msg/addr.rs

@@ -6,9 +6,72 @@
 
 //! Address messages: addr, addrv2 (getaddr and sendaddrv2 are empty).
 
-use crate::codec::codec_p2p;
+use crate::codec::{codec_p2p, impl_p2p};
 use crate::prelude::*;
-use crate::primitives::net_addr::{AddrV2Entry, TimestampedAddr};
+use crate::primitives::ServiceFlags;
+
+use dash_primitives::{AddrV2, ServiceV1};
+use dash_types::codec::{self, BaseCodec, DecodeError};
+
+use core::fmt;
+
+/// Timestamped v1 address entry used in `addr` messages.
+#[derive(Clone, Debug, Eq, Hash, PartialEq)]
+#[cfg_attr(feature = "serde", derive(::serde::Serialize, ::serde::Deserialize))]
+pub struct TimestampedAddr {
+  /// Seconds since Unix epoch.
+  pub time: u32,
+  /// Advertised services.
+  pub services: ServiceFlags,
+  /// IPv4-mapped IPv6 address + port.
+  pub addr: ServiceV1,
+}
+
+codec_p2p!(TimestampedAddr { time, services, addr });
+
+/// BIP155 timestamped v2 address entry used in `addrv2` messages.
+#[derive(Clone, Debug, Eq, Hash, PartialEq)]
+#[cfg_attr(feature = "serde", derive(::serde::Serialize, ::serde::Deserialize))]
+pub struct AddrV2Entry {
+  /// Seconds since Unix epoch.
+  pub time: u32,
+  /// Advertised services (CompactSize-encoded on wire).
+  pub services: ServiceFlags,
+  /// Network address.
+  pub addr: AddrV2,
+  /// Port number (big-endian on wire).
+  pub port: u16,
+}
+
+impl BaseCodec for AddrV2Entry {
+  fn decode(data: &mut &[u8]) -> Result<Self, DecodeError> {
+    let time = u32::decode(data)?;
+    let services = ServiceFlags(codec::read_compact_u64(data)?);
+    let addr = AddrV2::decode(data)?;
+    let port = codec::read_u16_be(data)?;
+    Ok(Self {
+      time,
+      services,
+      addr,
+      port,
+    })
+  }
+
+  fn encode(&self, buf: &mut Vec<u8>) {
+    self.time.encode(buf);
+    codec::write_compact_u64(self.services.0, buf);
+    self.addr.encode(buf);
+    buf.extend_from_slice(&self.port.to_be_bytes());
+  }
+}
+
+impl_p2p!(AddrV2Entry);
+
+impl fmt::Display for AddrV2Entry {
+  fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    write!(f, "{:?}:{}", self.addr.network, self.port)
+  }
+}
 
 /// V1 address announcement carrying timestamped addresses.
 #[derive(Clone, Debug, Eq, Hash, PartialEq)]