From dbbefa4680376f0e3d3b984e7d3b76d48ea0baa4 Mon Sep 17 00:00:00 2001 From: Yacov Manevich Date: Tue, 9 Jun 2026 16:16:29 +0200 Subject: [PATCH] Add handling for Auxiliary info for epoch changes Introduce an AuxiliaryInfoApp interface abstraction that lets an application piggyback application-specific data on Simplex epoch changes (e.g. threshold distributed key generation), and gate the epoch transition on that data being final. Final doesn't mean finalized as in Simplex, but rather that the data is "good enough" to be used for the next epoch. Metadata & encoding: - Add AuxiliaryInfo (Info, PrevAuxInfoSeq, ApplicationID) to block metadata and an AppID type, with Canoto (de)serialization. - Each block's PrevAuxInfoSeq back-points to the most recent block with a non-empty Info, skipping empty-Info blocks. collectAuxiliaryInfo walks these pointers to rebuild the epoch's aux info history. The digest of the last AuxInfo in a final history is signed by the node, rather than the entire history of non-empty Aux Info. This is done for flexibility, as some applications like publicly verifiable DKG have commutative histories. State machine: - AuxiliaryInfoApp drives the flow: GenerateAuxInfo contributes to the history until IsFinalAuxInfoHistory reports it ready; only then are next-epoch approvals collected. IsLegalAuxInfoAppend validates a proposed append on the verify path. Each method also receives the next epoch's validator set (NodeBLSMappings). - The builder carries forward / extends aux info, and the verifier reconstructs the expected AuxiliaryInfo (ApplicationID, PrevAuxInfoSeq) and enforces it via the block-digest comparison. Approvals: - Approvals now bind to the auxiliary info: the signed payload is (NextPChainReferenceHeight, auxInfoDigest), where auxInfoDigest is the sha256 of the last aux info element in the history. - The ApprovalStore is now keyed by (NodeID, PChainHeight, AuxInfoSeqDigest) so approvals for different digests coexist, and sanitizeApprovals only aggregates approvals matching the candidate height and digest. Tests: - Verify approvals with real ECDSA signatures over the signed payload (the test signature verifier now actually checks signatures and (NextPChainReferenceHeight, auxInfoDigest), where auxInfoDigest is the sha256 of the final aux info history. assembleApprovalToBeSigned replaces the previous height-only payload. The switch to ECDSA is because this way we make sure we really sign the correct digest and not an empty one because of a bug. - The ApprovalStore is keyed by (NodeID, PChainHeight, AuxInfoSeqDigest) so approvals for different digests coexist, and sanitizeApprovals only aggregates approvals matching the candidate height and digest. Signed-off-by: Yacov Manevich --- msm/approvals.go | 46 ++-- msm/approvals_test.go | 67 +++--- msm/encoding.canoto.go | 263 +++++++++++++++++++++- msm/encoding.go | 45 +++- msm/fake_node_test.go | 26 ++- msm/fuzz_test.go | 22 +- msm/msm.go | 394 +++++++++++++++++++++++++++------ msm/msm_test.go | 480 +++++++++++++++++++++++++++++++++++++++-- msm/util_test.go | 202 +++++++++++++---- 9 files changed, 1361 insertions(+), 184 deletions(-) diff --git a/msm/approvals.go b/msm/approvals.go index 566ba409..5c6fe3cb 100644 --- a/msm/approvals.go +++ b/msm/approvals.go @@ -4,15 +4,18 @@ package metadata import ( - "encoding/asn1" - "encoding/binary" "fmt" "github.com/ava-labs/simplex/common" "go.uber.org/zap" ) -type approvalsByPChainHeight map[uint64]*approvalAndTimestamp +type approvalKey struct { + pChainHeight uint64 + auxInfoDigest [32]byte +} + +type approvalsByPChainHeightAndAuxInfoDigest map[approvalKey]*approvalAndTimestamp type approvalAndTimestamp struct { ValidatorSetApproval @@ -24,7 +27,7 @@ type ApprovalStore struct { validators NodeBLSMappings logger common.Logger pkByNodeID map[nodeID][]byte - approvalsByNodes map[nodeID]approvalsByPChainHeight + approvalsByNodes map[nodeID]approvalsByPChainHeightAndAuxInfoDigest storedCount int } @@ -34,9 +37,9 @@ func NewApprovalStore(signatureVerifier SignatureVerifier, validators NodeBLSMap pkByNodeID[vdr.NodeID] = vdr.BLSKey } - approvalsByNodes := make(map[nodeID]approvalsByPChainHeight, len(validators)) + approvalsByNodes := make(map[nodeID]approvalsByPChainHeightAndAuxInfoDigest, len(validators)) for _, vdr := range validators { - approvalsByNodes[vdr.NodeID] = make(approvalsByPChainHeight) + approvalsByNodes[vdr.NodeID] = make(approvalsByPChainHeightAndAuxInfoDigest) } return &ApprovalStore{ @@ -82,9 +85,14 @@ func (as *ApprovalStore) HandleApproval(approval *ValidatorSetApproval, timestam return nil } + key := approvalKey{ + pChainHeight: approval.PChainHeight, + auxInfoDigest: approval.AuxInfoDigest, + } + // Store the approval. - oldApproval := as.approvalsByNodes[approval.NodeID][approval.PChainHeight] - as.approvalsByNodes[approval.NodeID][approval.PChainHeight] = &approvalAndTimestamp{ + oldApproval := as.approvalsByNodes[approval.NodeID][key] + as.approvalsByNodes[approval.NodeID][key] = &approvalAndTimestamp{ ValidatorSetApproval: *approval, Timestamp: timestamp, } @@ -113,22 +121,22 @@ func (as *ApprovalStore) maybePruneOldApprovals(approval *ValidatorSetApproval) } if oldestApproval != nil { + key := approvalKey{ + pChainHeight: oldestApproval.PChainHeight, + auxInfoDigest: oldestApproval.AuxInfoDigest, + } + as.logger.Debug("Deleting old approval from node", zap.String("nodeID", fmt.Sprintf("%x", oldestApproval.NodeID)), zap.String("oldestApprovalPChainHeight", fmt.Sprintf("%d", oldestApproval.PChainHeight)), zap.Uint64("oldestApprovalTimestamp", oldestApproval.Timestamp)) - delete(as.approvalsByNodes[approval.NodeID], oldestApproval.PChainHeight) + delete(as.approvalsByNodes[approval.NodeID], key) as.storedCount-- } } func (as *ApprovalStore) checkApprovalSignature(approval *ValidatorSetApproval, pk []byte) error { - pChainHeight := approval.PChainHeight - pChainHeightBuff := make([]byte, 8) - binary.BigEndian.PutUint64(pChainHeightBuff, pChainHeight) - - signedMsg := common.SignedMessage{Payload: pChainHeightBuff, Context: signatureContext} - toBeSigned, err := asn1.Marshal(signedMsg) + toBeSigned, err := assembleApprovalToBeSigned(approval.PChainHeight, approval.AuxInfoDigest) if err != nil { return err } @@ -146,7 +154,13 @@ func (as *ApprovalStore) approvalExistsAndUpToDate(approval *ValidatorSetApprova if as.approvalsByNodes[approval.NodeID] == nil { return false } - existingApproval := as.approvalsByNodes[approval.NodeID][approval.PChainHeight] + + key := approvalKey{ + pChainHeight: approval.PChainHeight, + auxInfoDigest: approval.AuxInfoDigest, + } + + existingApproval := as.approvalsByNodes[approval.NodeID][key] if existingApproval == nil { return false } diff --git a/msm/approvals_test.go b/msm/approvals_test.go index c457ca5f..c24e5f37 100644 --- a/msm/approvals_test.go +++ b/msm/approvals_test.go @@ -61,7 +61,7 @@ func TestApprovalStoreHandleApproval(t *testing.T) { validators: 3, sigErr: errors.New("bad sig"), approvals: []approvalAndTimestamp{ - {ValidatorSetApproval{NodeID: makeNodeID(1), PChainHeight: 1, Signature: []byte{0xAA}}, 1}, + {ValidatorSetApproval{NodeID: makeNodeID(1), PChainHeight: 1, Signature: signApproval(1, [32]byte{})}, 1}, }, verify: func(t *testing.T, as *ApprovalStore, _ []approvalAndTimestamp) { require.Empty(t, as.Approvals()) @@ -74,7 +74,7 @@ func TestApprovalStoreHandleApproval(t *testing.T) { name: "valid approval is stored", validators: 3, approvals: []approvalAndTimestamp{ - {ValidatorSetApproval{NodeID: makeNodeID(1), PChainHeight: 7, Signature: []byte{0x01}}, 100}, + {ValidatorSetApproval{NodeID: makeNodeID(1), PChainHeight: 7, Signature: signApproval(7, [32]byte{})}, 100}, }, verify: func(t *testing.T, as *ApprovalStore, sent []approvalAndTimestamp) { got := as.Approvals() @@ -89,8 +89,8 @@ func TestApprovalStoreHandleApproval(t *testing.T) { name: "duplicate approval with same timestamp is a no-op", validators: 3, approvals: []approvalAndTimestamp{ - {ValidatorSetApproval{NodeID: makeNodeID(1), PChainHeight: 7, Signature: []byte{0x01}}, 100}, - {ValidatorSetApproval{NodeID: makeNodeID(1), PChainHeight: 7, Signature: []byte{0x01}}, 100}, + {ValidatorSetApproval{NodeID: makeNodeID(1), PChainHeight: 7, Signature: signApproval(7, [32]byte{})}, 100}, + {ValidatorSetApproval{NodeID: makeNodeID(1), PChainHeight: 7, Signature: signApproval(7, [32]byte{})}, 100}, }, verify: func(t *testing.T, as *ApprovalStore, _ []approvalAndTimestamp) { require.Len(t, as.Approvals(), 1) @@ -103,8 +103,8 @@ func TestApprovalStoreHandleApproval(t *testing.T) { name: "older timestamp is ignored", validators: 3, approvals: []approvalAndTimestamp{ - {ValidatorSetApproval{NodeID: makeNodeID(1), PChainHeight: 7, Signature: []byte{0x02}}, 200}, // newer, stored first - {ValidatorSetApproval{NodeID: makeNodeID(1), PChainHeight: 7, Signature: []byte{0x01}}, 100}, // older, dropped + {ValidatorSetApproval{NodeID: makeNodeID(1), PChainHeight: 7, Signature: signApproval(7, [32]byte{})}, 200}, // newer, stored first + {ValidatorSetApproval{NodeID: makeNodeID(1), PChainHeight: 7, Signature: signApproval(7, [32]byte{})}, 100}, // older, dropped }, verify: func(t *testing.T, as *ApprovalStore, sent []approvalAndTimestamp) { got := as.Approvals() @@ -119,8 +119,8 @@ func TestApprovalStoreHandleApproval(t *testing.T) { name: "newer timestamp replaces", validators: 3, approvals: []approvalAndTimestamp{ - {ValidatorSetApproval{NodeID: makeNodeID(1), PChainHeight: 7, Signature: []byte{0x01}}, 100}, // older, stored first - {ValidatorSetApproval{NodeID: makeNodeID(1), PChainHeight: 7, Signature: []byte{0x02}}, 200}, // newer, replaces + {ValidatorSetApproval{NodeID: makeNodeID(1), PChainHeight: 7, Signature: signApproval(7, [32]byte{})}, 100}, // older, stored first + {ValidatorSetApproval{NodeID: makeNodeID(1), PChainHeight: 7, Signature: signApproval(7, [32]byte{})}, 200}, // newer, replaces }, verify: func(t *testing.T, as *ApprovalStore, sent []approvalAndTimestamp) { got := as.Approvals() @@ -136,12 +136,12 @@ func TestApprovalStoreHandleApproval(t *testing.T) { validators: 3, // 3 validators x 2 heights, with timestamp == i*10 + h and Signature == {i} per validator. approvals: []approvalAndTimestamp{ - {ValidatorSetApproval{NodeID: makeNodeID(1), PChainHeight: 1, Signature: []byte{0}}, 1}, - {ValidatorSetApproval{NodeID: makeNodeID(1), PChainHeight: 2, Signature: []byte{0}}, 2}, - {ValidatorSetApproval{NodeID: makeNodeID(2), PChainHeight: 1, Signature: []byte{1}}, 11}, - {ValidatorSetApproval{NodeID: makeNodeID(2), PChainHeight: 2, Signature: []byte{1}}, 12}, - {ValidatorSetApproval{NodeID: makeNodeID(3), PChainHeight: 1, Signature: []byte{2}}, 21}, - {ValidatorSetApproval{NodeID: makeNodeID(3), PChainHeight: 2, Signature: []byte{2}}, 22}, + {ValidatorSetApproval{NodeID: makeNodeID(1), PChainHeight: 1, Signature: signApproval(1, [32]byte{})}, 1}, + {ValidatorSetApproval{NodeID: makeNodeID(1), PChainHeight: 2, Signature: signApproval(2, [32]byte{})}, 2}, + {ValidatorSetApproval{NodeID: makeNodeID(2), PChainHeight: 1, Signature: signApproval(1, [32]byte{})}, 11}, + {ValidatorSetApproval{NodeID: makeNodeID(2), PChainHeight: 2, Signature: signApproval(2, [32]byte{})}, 12}, + {ValidatorSetApproval{NodeID: makeNodeID(3), PChainHeight: 1, Signature: signApproval(1, [32]byte{})}, 21}, + {ValidatorSetApproval{NodeID: makeNodeID(3), PChainHeight: 2, Signature: signApproval(2, [32]byte{})}, 22}, }, verify: func(t *testing.T, as *ApprovalStore, sent []approvalAndTimestamp) { require.Len(t, as.Approvals(), len(sent)) @@ -154,9 +154,9 @@ func TestApprovalStoreHandleApproval(t *testing.T) { name: "prunes oldest when over cap", validators: 2, approvals: []approvalAndTimestamp{ - {ValidatorSetApproval{NodeID: makeNodeID(1), PChainHeight: 1, Signature: []byte{1}}, 10}, - {ValidatorSetApproval{NodeID: makeNodeID(1), PChainHeight: 2, Signature: []byte{2}}, 20}, - {ValidatorSetApproval{NodeID: makeNodeID(1), PChainHeight: 3, Signature: []byte{3}}, 30}, + {ValidatorSetApproval{NodeID: makeNodeID(1), PChainHeight: 1, Signature: signApproval(1, [32]byte{})}, 10}, + {ValidatorSetApproval{NodeID: makeNodeID(1), PChainHeight: 2, Signature: signApproval(2, [32]byte{})}, 20}, + {ValidatorSetApproval{NodeID: makeNodeID(1), PChainHeight: 3, Signature: signApproval(3, [32]byte{})}, 30}, }, verify: func(t *testing.T, as *ApprovalStore, _ []approvalAndTimestamp) { got := as.Approvals() @@ -174,6 +174,23 @@ func TestApprovalStoreHandleApproval(t *testing.T) { require.True(t, heights[3]) }, }, + { + // Verifies that the store keys on (NodeID, PChainHeight, AuxInfoDigest): + // two approvals from the same node at the same P-chain height but with different + // auxiliary info digests are kept as independent entries. + name: "same height different aux info digest coexist", + validators: 3, + approvals: []approvalAndTimestamp{ + {ValidatorSetApproval{NodeID: makeNodeID(1), PChainHeight: 7, AuxInfoDigest: [32]byte{0xAA}, Signature: signApproval(7, [32]byte{0xAA})}, 100}, + {ValidatorSetApproval{NodeID: makeNodeID(1), PChainHeight: 7, AuxInfoDigest: [32]byte{0xBB}, Signature: signApproval(7, [32]byte{0xBB})}, 100}, + }, + verify: func(t *testing.T, as *ApprovalStore, sent []approvalAndTimestamp) { + got := as.Approvals() + require.Len(t, got, 2) + require.Equal(t, 2, as.storedCount) + require.ElementsMatch(t, []ValidatorSetApproval{sent[0].ValidatorSetApproval, sent[1].ValidatorSetApproval}, got) + }, + }, { // Verifies that an approval with the maximum uint64 timestamp is stored, // and that a subsequent approval at the same (NodeID, PChainHeight) with any @@ -181,8 +198,8 @@ func TestApprovalStoreHandleApproval(t *testing.T) { name: "max uint64 timestamp is kept over any smaller timestamp", validators: 3, approvals: []approvalAndTimestamp{ - {ValidatorSetApproval{NodeID: makeNodeID(1), PChainHeight: 7, Signature: []byte{0xFF}}, math.MaxUint64}, // maxTS - {ValidatorSetApproval{NodeID: makeNodeID(1), PChainHeight: 7, Signature: []byte{0x01}}, math.MaxUint64 - 1}, // older + {ValidatorSetApproval{NodeID: makeNodeID(1), PChainHeight: 7, Signature: signApproval(7, [32]byte{})}, math.MaxUint64}, // maxTS + {ValidatorSetApproval{NodeID: makeNodeID(1), PChainHeight: 7, Signature: signApproval(7, [32]byte{})}, math.MaxUint64 - 1}, // older }, verify: func(t *testing.T, as *ApprovalStore, sent []approvalAndTimestamp) { got := as.Approvals() @@ -213,11 +230,11 @@ func TestApprovalStoreHandleApprovalStoredCountStaysConsistent(t *testing.T) { node := vdrs[0].NodeID for _, a := range []approvalAndTimestamp{ - {ValidatorSetApproval{NodeID: node, PChainHeight: 1}, 10}, - {ValidatorSetApproval{NodeID: node, PChainHeight: 1}, 10}, // duplicate - {ValidatorSetApproval{NodeID: node, PChainHeight: 1}, 20}, // replaces - {ValidatorSetApproval{NodeID: node, PChainHeight: 2}, 30}, // new height - {ValidatorSetApproval{NodeID: node, PChainHeight: 3}, 40}, // triggers prune + {ValidatorSetApproval{NodeID: node, PChainHeight: 1, Signature: signApproval(1, [32]byte{})}, 10}, + {ValidatorSetApproval{NodeID: node, PChainHeight: 1, Signature: signApproval(1, [32]byte{})}, 10}, // duplicate + {ValidatorSetApproval{NodeID: node, PChainHeight: 1, Signature: signApproval(1, [32]byte{})}, 20}, // replaces + {ValidatorSetApproval{NodeID: node, PChainHeight: 2, Signature: signApproval(2, [32]byte{})}, 30}, // new height + {ValidatorSetApproval{NodeID: node, PChainHeight: 3, Signature: signApproval(3, [32]byte{})}, 40}, // triggers prune } { require.NoError(t, as.HandleApproval(&a.ValidatorSetApproval, a.Timestamp)) require.Len(t, as.Approvals(), as.storedCount) @@ -235,12 +252,14 @@ func TestApprovalStoreHandleApprovalPruningIsPerNode(t *testing.T) { require.NoError(t, as.HandleApproval(&ValidatorSetApproval{ NodeID: vdrs[1].NodeID, PChainHeight: 1, + Signature: signApproval(1, [32]byte{}), }, 100)) for h := uint64(1); h <= 10; h++ { require.NoError(t, as.HandleApproval(&ValidatorSetApproval{ NodeID: vdrs[0].NodeID, PChainHeight: h, + Signature: signApproval(h, [32]byte{}), }, h)) } require.Len(t, as.Approvals().UniqueByNodeID(), 2) diff --git a/msm/encoding.canoto.go b/msm/encoding.canoto.go index d2651feb..d76a53e7 100644 --- a/msm/encoding.canoto.go +++ b/msm/encoding.canoto.go @@ -29,6 +29,7 @@ const ( canotoNumber_StateMachineMetadata__PChainHeight = 4 canotoNumber_StateMachineMetadata__Timestamp = 5 canotoNumber_StateMachineMetadata__ICMEpochInfo = 6 + canotoNumber_StateMachineMetadata__AuxiliaryInfo = 7 canotoTag_StateMachineMetadata__SimplexEpochInfo = "\x0a" // canoto.Tag(canotoNumber_StateMachineMetadata__SimplexEpochInfo, canoto.Len) canotoTag_StateMachineMetadata__SimplexProtocolMetadata = "\x12" // canoto.Tag(canotoNumber_StateMachineMetadata__SimplexProtocolMetadata, canoto.Len) @@ -36,6 +37,7 @@ const ( canotoTag_StateMachineMetadata__PChainHeight = "\x20" // canoto.Tag(canotoNumber_StateMachineMetadata__PChainHeight, canoto.Varint) canotoTag_StateMachineMetadata__Timestamp = "\x28" // canoto.Tag(canotoNumber_StateMachineMetadata__Timestamp, canoto.Varint) canotoTag_StateMachineMetadata__ICMEpochInfo = "\x32" // canoto.Tag(canotoNumber_StateMachineMetadata__ICMEpochInfo, canoto.Len) + canotoTag_StateMachineMetadata__AuxiliaryInfo = "\x3a" // canoto.Tag(canotoNumber_StateMachineMetadata__AuxiliaryInfo, canoto.Len) ) type canotoData_StateMachineMetadata struct { @@ -93,6 +95,16 @@ func (*StateMachineMetadata) CanotoSpec(types ...reflect.Type) *canoto.Spec { /*Pointer: */ false, /*types: */ types, ), + canoto.FieldTypeFromField( + /*type inference:*/ (zero.AuxiliaryInfo), + /*FieldNumber: */ canotoNumber_StateMachineMetadata__AuxiliaryInfo, + /*Name: */ "AuxiliaryInfo", + /*FixedLength: */ 0, + /*Repeated: */ false, + /*OneOf: */ "", + /*Pointer: */ true, + /*types: */ types, + ), }, } s.CalculateCanotoCache() @@ -223,6 +235,28 @@ func (c *StateMachineMetadata) UnmarshalCanotoFrom(r canoto.Reader) error { return err } r.B = remainingBytes + case canotoNumber_StateMachineMetadata__AuxiliaryInfo: + if wireType != canoto.Len { + return canoto.ErrUnexpectedWireType + } + + // Read the bytes for the field. + originalUnsafe := r.Unsafe + r.Unsafe = true + var msgBytes []byte + if err := canoto.ReadBytes(&r, &msgBytes); err != nil { + return err + } + r.Unsafe = originalUnsafe + + // Unmarshal the field from the bytes. + remainingBytes := r.B + r.B = msgBytes + c.AuxiliaryInfo = canoto.MakePointer(c.AuxiliaryInfo) + if err := (c.AuxiliaryInfo).UnmarshalCanotoFrom(r); err != nil { + return err + } + r.B = remainingBytes default: return canoto.ErrUnknownField } @@ -246,6 +280,9 @@ func (c *StateMachineMetadata) ValidCanoto() bool { if !(&c.ICMEpochInfo).ValidCanoto() { return false } + if c.AuxiliaryInfo != nil && !(c.AuxiliaryInfo).ValidCanoto() { + return false + } return true } @@ -275,6 +312,11 @@ func (c *StateMachineMetadata) CalculateCanotoCache() { if fieldSize := (&c.ICMEpochInfo).CachedCanotoSize(); fieldSize != 0 { size += uint64(len(canotoTag_StateMachineMetadata__ICMEpochInfo)) + canoto.SizeUint(fieldSize) + fieldSize } + if c.AuxiliaryInfo != nil { + (c.AuxiliaryInfo).CalculateCanotoCache() + fieldSize := (c.AuxiliaryInfo).CachedCanotoSize() + size += uint64(len(canotoTag_StateMachineMetadata__AuxiliaryInfo)) + canoto.SizeUint(fieldSize) + fieldSize + } atomic.StoreUint64(&c.canotoData.size, size) } @@ -339,6 +381,12 @@ func (c *StateMachineMetadata) MarshalCanotoInto(w canoto.Writer) canoto.Writer canoto.AppendUint(&w, fieldSize) w = (&c.ICMEpochInfo).MarshalCanotoInto(w) } + if c.AuxiliaryInfo != nil { + fieldSize := (c.AuxiliaryInfo).CachedCanotoSize() + canoto.Append(&w, canotoTag_StateMachineMetadata__AuxiliaryInfo) + canoto.AppendUint(&w, fieldSize) + w = (c.AuxiliaryInfo).MarshalCanotoInto(w) + } return w } @@ -539,6 +587,203 @@ func (c *ICMEpochInfo) MarshalCanotoInto(w canoto.Writer) canoto.Writer { return w } +const ( + canotoNumber_AuxiliaryInfo__Info = 1 + canotoNumber_AuxiliaryInfo__PrevAuxInfoSeq = 2 + canotoNumber_AuxiliaryInfo__VersionID = 3 + + canotoTag_AuxiliaryInfo__Info = "\x0a" // canoto.Tag(canotoNumber_AuxiliaryInfo__Info, canoto.Len) + canotoTag_AuxiliaryInfo__PrevAuxInfoSeq = "\x10" // canoto.Tag(canotoNumber_AuxiliaryInfo__PrevAuxInfoSeq, canoto.Varint) + canotoTag_AuxiliaryInfo__VersionID = "\x18" // canoto.Tag(canotoNumber_AuxiliaryInfo__VersionID, canoto.Varint) +) + +type canotoData_AuxiliaryInfo struct { + size uint64 +} + +// CanotoSpec returns the specification of this canoto message. +func (*AuxiliaryInfo) CanotoSpec(...reflect.Type) *canoto.Spec { + var zero AuxiliaryInfo + s := &canoto.Spec{ + Name: "AuxiliaryInfo", + Fields: []canoto.FieldType{ + { + FieldNumber: canotoNumber_AuxiliaryInfo__Info, + Name: "Info", + OneOf: "", + TypeBytes: true, + }, + { + FieldNumber: canotoNumber_AuxiliaryInfo__PrevAuxInfoSeq, + Name: "PrevAuxInfoSeq", + OneOf: "", + TypeUint: canoto.SizeOf(zero.PrevAuxInfoSeq), + }, + { + FieldNumber: canotoNumber_AuxiliaryInfo__VersionID, + Name: "VersionID", + OneOf: "", + TypeUint: canoto.SizeOf(zero.VersionID), + }, + }, + } + s.CalculateCanotoCache() + return s +} + +// UnmarshalCanoto unmarshals a Canoto-encoded byte slice into the struct. +// +// During parsing, the canoto cache is saved. +func (c *AuxiliaryInfo) UnmarshalCanoto(bytes []byte) error { + r := canoto.Reader{ + B: bytes, + } + return c.UnmarshalCanotoFrom(r) +} + +// UnmarshalCanotoFrom populates the struct from a [canoto.Reader]. Most users +// should just use UnmarshalCanoto. +// +// During parsing, the canoto cache is saved. +// +// This function enables configuration of reader options. +func (c *AuxiliaryInfo) UnmarshalCanotoFrom(r canoto.Reader) error { + // Zero the struct before unmarshaling. + *c = AuxiliaryInfo{} + atomic.StoreUint64(&c.canotoData.size, uint64(len(r.B))) + + var minField uint32 + for canoto.HasNext(&r) { + field, wireType, err := canoto.ReadTag(&r) + if err != nil { + return err + } + if field < minField { + return canoto.ErrInvalidFieldOrder + } + + switch field { + case canotoNumber_AuxiliaryInfo__Info: + if wireType != canoto.Len { + return canoto.ErrUnexpectedWireType + } + + if err := canoto.ReadBytes(&r, &c.Info); err != nil { + return err + } + if len(c.Info) == 0 { + return canoto.ErrZeroValue + } + case canotoNumber_AuxiliaryInfo__PrevAuxInfoSeq: + if wireType != canoto.Varint { + return canoto.ErrUnexpectedWireType + } + + if err := canoto.ReadUint(&r, &c.PrevAuxInfoSeq); err != nil { + return err + } + if canoto.IsZero(c.PrevAuxInfoSeq) { + return canoto.ErrZeroValue + } + case canotoNumber_AuxiliaryInfo__VersionID: + if wireType != canoto.Varint { + return canoto.ErrUnexpectedWireType + } + + if err := canoto.ReadUint(&r, &c.VersionID); err != nil { + return err + } + if canoto.IsZero(c.VersionID) { + return canoto.ErrZeroValue + } + default: + return canoto.ErrUnknownField + } + + minField = field + 1 + } + return nil +} + +// ValidCanoto validates that the struct can be correctly marshaled into the +// Canoto format. +// +// Specifically, ValidCanoto ensures: +// 1. All OneOfs are specified at most once. +// 2. All strings are valid utf-8. +// 3. All custom fields are ValidCanoto. +func (c *AuxiliaryInfo) ValidCanoto() bool { + return true +} + +// CalculateCanotoCache populates size and OneOf caches based on the current +// values in the struct. +// +// It is not safe to copy this struct concurrently. +func (c *AuxiliaryInfo) CalculateCanotoCache() { + var size uint64 + if len(c.Info) != 0 { + size += uint64(len(canotoTag_AuxiliaryInfo__Info)) + canoto.SizeBytes(c.Info) + } + if !canoto.IsZero(c.PrevAuxInfoSeq) { + size += uint64(len(canotoTag_AuxiliaryInfo__PrevAuxInfoSeq)) + canoto.SizeUint(c.PrevAuxInfoSeq) + } + if !canoto.IsZero(c.VersionID) { + size += uint64(len(canotoTag_AuxiliaryInfo__VersionID)) + canoto.SizeUint(c.VersionID) + } + atomic.StoreUint64(&c.canotoData.size, size) +} + +// CachedCanotoSize returns the previously calculated size of the Canoto +// representation from CalculateCanotoCache. +// +// If CalculateCanotoCache has not yet been called, it will return 0. +// +// If the struct has been modified since the last call to CalculateCanotoCache, +// the returned size may be incorrect. +func (c *AuxiliaryInfo) CachedCanotoSize() uint64 { + return atomic.LoadUint64(&c.canotoData.size) +} + +// MarshalCanoto returns the Canoto representation of this struct. +// +// It is assumed that this struct is ValidCanoto. +// +// It is not safe to copy this struct concurrently. +func (c *AuxiliaryInfo) MarshalCanoto() []byte { + c.CalculateCanotoCache() + w := canoto.Writer{ + B: make([]byte, 0, c.CachedCanotoSize()), + } + w = c.MarshalCanotoInto(w) + return w.B +} + +// MarshalCanotoInto writes the struct into a [canoto.Writer] and returns the +// resulting [canoto.Writer]. Most users should just use MarshalCanoto. +// +// It is assumed that CalculateCanotoCache has been called since the last +// modification to this struct. +// +// It is assumed that this struct is ValidCanoto. +// +// It is not safe to copy this struct concurrently. +func (c *AuxiliaryInfo) MarshalCanotoInto(w canoto.Writer) canoto.Writer { + if len(c.Info) != 0 { + canoto.Append(&w, canotoTag_AuxiliaryInfo__Info) + canoto.AppendBytes(&w, c.Info) + } + if !canoto.IsZero(c.PrevAuxInfoSeq) { + canoto.Append(&w, canotoTag_AuxiliaryInfo__PrevAuxInfoSeq) + canoto.AppendUint(&w, c.PrevAuxInfoSeq) + } + if !canoto.IsZero(c.VersionID) { + canoto.Append(&w, canotoTag_AuxiliaryInfo__VersionID) + canoto.AppendUint(&w, c.VersionID) + } + return w +} + const ( canotoNumber_SimplexEpochInfo__PChainReferenceHeight = 1 canotoNumber_SimplexEpochInfo__EpochNumber = 2 @@ -1712,9 +1957,9 @@ func (*ValidatorSetApproval) CanotoSpec(...reflect.Type) *canoto.Spec { }, { FieldNumber: canotoNumber_ValidatorSetApproval__AuxInfoSeqDigest, - Name: "AuxInfoSeqDigest", + Name: "AuxInfoDigest", OneOf: "", - TypeFixedBytes: uint64(len(zero.AuxInfoSeqDigest)), + TypeFixedBytes: uint64(len(zero.AuxInfoDigest)), }, { FieldNumber: canotoNumber_ValidatorSetApproval__PChainHeight, @@ -1797,7 +2042,7 @@ func (c *ValidatorSetApproval) UnmarshalCanotoFrom(r canoto.Reader) error { } const ( - expectedLength = len(c.AuxInfoSeqDigest) + expectedLength = len(c.AuxInfoDigest) expectedLengthUint64 = uint64(expectedLength) ) var length uint64 @@ -1811,8 +2056,8 @@ func (c *ValidatorSetApproval) UnmarshalCanotoFrom(r canoto.Reader) error { return io.ErrUnexpectedEOF } - copy((&c.AuxInfoSeqDigest)[:], r.B) - if canoto.IsZero(c.AuxInfoSeqDigest) { + copy((&c.AuxInfoDigest)[:], r.B) + if canoto.IsZero(c.AuxInfoDigest) { return canoto.ErrZeroValue } r.B = r.B[expectedLength:] @@ -1867,8 +2112,8 @@ func (c *ValidatorSetApproval) CalculateCanotoCache() { if !canoto.IsZero(c.NodeID) { size += uint64(len(canotoTag_ValidatorSetApproval__NodeID)) + canoto.SizeBytes((&c.NodeID)[:]) } - if !canoto.IsZero(c.AuxInfoSeqDigest) { - size += uint64(len(canotoTag_ValidatorSetApproval__AuxInfoSeqDigest)) + canoto.SizeBytes((&c.AuxInfoSeqDigest)[:]) + if !canoto.IsZero(c.AuxInfoDigest) { + size += uint64(len(canotoTag_ValidatorSetApproval__AuxInfoSeqDigest)) + canoto.SizeBytes((&c.AuxInfoDigest)[:]) } if !canoto.IsZero(c.PChainHeight) { size += uint64(len(canotoTag_ValidatorSetApproval__PChainHeight)) + canoto.SizeUint(c.PChainHeight) @@ -1918,9 +2163,9 @@ func (c *ValidatorSetApproval) MarshalCanotoInto(w canoto.Writer) canoto.Writer canoto.Append(&w, canotoTag_ValidatorSetApproval__NodeID) canoto.AppendBytes(&w, (&c.NodeID)[:]) } - if !canoto.IsZero(c.AuxInfoSeqDigest) { + if !canoto.IsZero(c.AuxInfoDigest) { canoto.Append(&w, canotoTag_ValidatorSetApproval__AuxInfoSeqDigest) - canoto.AppendBytes(&w, (&c.AuxInfoSeqDigest)[:]) + canoto.AppendBytes(&w, (&c.AuxInfoDigest)[:]) } if !canoto.IsZero(c.PChainHeight) { canoto.Append(&w, canotoTag_ValidatorSetApproval__PChainHeight) diff --git a/msm/encoding.go b/msm/encoding.go index 2a08d4ed..9659e5d6 100644 --- a/msm/encoding.go +++ b/msm/encoding.go @@ -32,6 +32,9 @@ type StateMachineMetadata struct { Timestamp uint64 `canoto:"uint,5"` // ICMEpochInfo is the metadata that the StateMachine uses for ICM epoching. ICMEpochInfo ICMEpochInfo `canoto:"value,6"` + // AuxiliaryInfo is application-specific information that the StateMachine doesn't need to understand, + // but can be used by applications that care about epoch changes, such as threshold distributed public key generation. + AuxiliaryInfo *AuxiliaryInfo `canoto:"pointer,7"` canotoData canotoData_StateMachineMetadata } @@ -60,6 +63,40 @@ func (ei *ICMEpochInfo) Equal(other *ICMEpochInfo) bool { return ei.EpochStartTime == other.EpochStartTime && ei.EpochNumber == other.EpochNumber && ei.PChainEpochHeight == other.PChainEpochHeight } +// VersionID is an identifier for applications that care about epoch changes. +type VersionID uint32 + +// AuxiliaryInfo defines application-specific information for applications that might care about epoch change, +// such as threshold distributed public key generation. +type AuxiliaryInfo struct { + // Info is opaque bytes that can be used by applications to encode any information that describes + // the current state for the application. + Info []byte `canoto:"bytes,1"` + // PrevAuxInfoSeq is a sequence number that applications can use to find previous AuxiliaryInfo in the chain. + // It is zero if this is the first AuxiliaryInfo for this epoch. + PrevAuxInfoSeq uint64 `canoto:"uint,2"` + // VersionID is an identifier that identifies the application. + // Can be used for backward-compatibility and upgrade purposes. + VersionID VersionID `canoto:"uint,3"` + + canotoData canotoData_AuxiliaryInfo +} + +func (ai *AuxiliaryInfo) IsZero() bool { + var zero AuxiliaryInfo + return ai.Equal(&zero) +} + +func (ai *AuxiliaryInfo) Equal(a *AuxiliaryInfo) bool { + if ai == nil { + return a == nil + } + if a == nil { + return ai == nil + } + return bytes.Equal(ai.Info, a.Info) && ai.PrevAuxInfoSeq == a.PrevAuxInfoSeq && ai.VersionID == a.VersionID +} + // SimplexEpochInfo is metadata used by the StateMachine. type SimplexEpochInfo struct { // PChainReferenceHeight is the P-Chain height that the StateMachine uses as a reference for the current epoch. @@ -321,10 +358,10 @@ func (nbms NodeBLSMappings) Equal(other NodeBLSMappings) bool { } type ValidatorSetApproval struct { - NodeID nodeID `canoto:"fixed bytes,1"` - AuxInfoSeqDigest [32]byte `canoto:"fixed bytes,2"` - PChainHeight uint64 `canoto:"uint,3"` - Signature []byte `canoto:"bytes,4"` + NodeID nodeID `canoto:"fixed bytes,1"` + AuxInfoDigest [32]byte `canoto:"fixed bytes,2"` + PChainHeight uint64 `canoto:"uint,3"` + Signature []byte `canoto:"bytes,4"` canotoData canotoData_ValidatorSetApproval } diff --git a/msm/fake_node_test.go b/msm/fake_node_test.go index f55a1900..9e8c69b9 100644 --- a/msm/fake_node_test.go +++ b/msm/fake_node_test.go @@ -6,6 +6,7 @@ package metadata import ( "context" "crypto/rand" + "crypto/sha256" "fmt" "sync/atomic" "testing" @@ -15,6 +16,8 @@ import ( "github.com/stretchr/testify/require" ) +var emptyAuxInfoDigest = sha256.Sum256(nil) + func TestFakeNodeEpochChangesDespiteEmptyMempool(t *testing.T) { validatorSetRetriever := validatorSetRetriever{ resultMap: map[uint64]NodeBLSMappings{ @@ -30,6 +33,7 @@ func TestFakeNodeEpochChangesDespiteEmptyMempool(t *testing.T) { var pChainHeight atomic.Uint64 pChainHeight.Store(100) node := newFakeNode(t) + node.sm.AuxiliaryInfoApp = &noopTestAuxInfoApp{} node.sm.GetValidatorSet = validatorSetRetriever.getValidatorSet node.sm.GetPChainHeightForProposing = func() uint64 { return pChainHeight.Load() @@ -57,11 +61,11 @@ func TestFakeNodeEpochChangesDespiteEmptyMempool(t *testing.T) { } if flipCoin() { node.sm.ApprovalsRetriever = &approvalsRetriever{ - result: []ValidatorSetApproval{{NodeID: [20]byte{1}, PChainHeight: 200, Signature: []byte{1}, AuxInfoSeqDigest: [32]byte{}}}, + result: []ValidatorSetApproval{{NodeID: [20]byte{1}, PChainHeight: 200, Signature: signApproval(200, emptyAuxInfoDigest), AuxInfoDigest: emptyAuxInfoDigest}}, } } else { node.sm.ApprovalsRetriever = &approvalsRetriever{ - result: []ValidatorSetApproval{{NodeID: [20]byte{2}, PChainHeight: 200, Signature: []byte{2}, AuxInfoSeqDigest: [32]byte{}}}, + result: []ValidatorSetApproval{{NodeID: [20]byte{2}, PChainHeight: 200, Signature: signApproval(200, emptyAuxInfoDigest), AuxInfoDigest: emptyAuxInfoDigest}}, } } @@ -86,6 +90,7 @@ func TestFakeNode(t *testing.T) { var pChainHeight atomic.Uint64 pChainHeight.Store(100) node := newFakeNode(t) + node.sm.AuxiliaryInfoApp = &noopTestAuxInfoApp{} node.sm.GetValidatorSet = validatorSetRetriever.getValidatorSet node.sm.GetPChainHeightForProposing = func() uint64 { return pChainHeight.Load() @@ -107,11 +112,11 @@ func TestFakeNode(t *testing.T) { node.act() if flipCoin() { node.sm.ApprovalsRetriever = &approvalsRetriever{ - result: []ValidatorSetApproval{{NodeID: [20]byte{1}, PChainHeight: 200, Signature: []byte{1}, AuxInfoSeqDigest: [32]byte{}}}, + result: []ValidatorSetApproval{{NodeID: [20]byte{1}, PChainHeight: 200, Signature: signApproval(200, emptyAuxInfoDigest), AuxInfoDigest: emptyAuxInfoDigest}}, } } else { node.sm.ApprovalsRetriever = &approvalsRetriever{ - result: []ValidatorSetApproval{{NodeID: [20]byte{2}, PChainHeight: 200, Signature: []byte{2}, AuxInfoSeqDigest: [32]byte{}}}, + result: []ValidatorSetApproval{{NodeID: [20]byte{2}, PChainHeight: 200, Signature: signApproval(200, emptyAuxInfoDigest), AuxInfoDigest: emptyAuxInfoDigest}}, } } } @@ -128,11 +133,11 @@ func TestFakeNode(t *testing.T) { node.act() if flipCoin() { node.sm.ApprovalsRetriever = &approvalsRetriever{ - result: []ValidatorSetApproval{{NodeID: [20]byte{2}, PChainHeight: 300, Signature: []byte{2}, AuxInfoSeqDigest: [32]byte{}}}, + result: []ValidatorSetApproval{{NodeID: [20]byte{2}, PChainHeight: 300, Signature: signApproval(300, emptyAuxInfoDigest), AuxInfoDigest: emptyAuxInfoDigest}}, } } else { node.sm.ApprovalsRetriever = &approvalsRetriever{ - result: []ValidatorSetApproval{{NodeID: [20]byte{3}, PChainHeight: 300, Signature: []byte{3}, AuxInfoSeqDigest: [32]byte{}}}, + result: []ValidatorSetApproval{{NodeID: [20]byte{3}, PChainHeight: 300, Signature: signApproval(300, emptyAuxInfoDigest), AuxInfoDigest: emptyAuxInfoDigest}}, } } } @@ -155,6 +160,7 @@ func TestFakeNodeEmptyMempool(t *testing.T) { var pChainHeight uint64 = 100 node := newFakeNode(t) + node.sm.AuxiliaryInfoApp = &noopTestAuxInfoApp{} node.sm.MaxBlockBuildingWaitTime = 100 * time.Millisecond node.sm.GetValidatorSet = validatorSetRetriever.getValidatorSet node.sm.GetPChainHeightForProposing = func() uint64 { @@ -180,11 +186,11 @@ func TestFakeNodeEmptyMempool(t *testing.T) { node.act() if flipCoin() { node.sm.ApprovalsRetriever = &approvalsRetriever{ - result: []ValidatorSetApproval{{NodeID: [20]byte{1}, PChainHeight: 200, Signature: []byte{1}, AuxInfoSeqDigest: [32]byte{}}}, + result: []ValidatorSetApproval{{NodeID: [20]byte{1}, PChainHeight: 200, Signature: signApproval(200, emptyAuxInfoDigest), AuxInfoDigest: emptyAuxInfoDigest}}, } } else { node.sm.ApprovalsRetriever = &approvalsRetriever{ - result: []ValidatorSetApproval{{NodeID: [20]byte{2}, PChainHeight: 200, Signature: []byte{2}, AuxInfoSeqDigest: [32]byte{}}}, + result: []ValidatorSetApproval{{NodeID: [20]byte{2}, PChainHeight: 200, Signature: signApproval(200, emptyAuxInfoDigest), AuxInfoDigest: emptyAuxInfoDigest}}, } } } @@ -213,11 +219,11 @@ func TestFakeNodeEmptyMempool(t *testing.T) { node.act() if flipCoin() { node.sm.ApprovalsRetriever = &approvalsRetriever{ - result: []ValidatorSetApproval{{NodeID: [20]byte{2}, PChainHeight: 300, Signature: []byte{2}, AuxInfoSeqDigest: [32]byte{}}}, + result: []ValidatorSetApproval{{NodeID: [20]byte{2}, PChainHeight: 300, Signature: signApproval(300, emptyAuxInfoDigest), AuxInfoDigest: emptyAuxInfoDigest}}, } } else { node.sm.ApprovalsRetriever = &approvalsRetriever{ - result: []ValidatorSetApproval{{NodeID: [20]byte{3}, PChainHeight: 300, Signature: []byte{3}, AuxInfoSeqDigest: [32]byte{}}}, + result: []ValidatorSetApproval{{NodeID: [20]byte{3}, PChainHeight: 300, Signature: signApproval(300, emptyAuxInfoDigest), AuxInfoDigest: emptyAuxInfoDigest}}, } } } diff --git a/msm/fuzz_test.go b/msm/fuzz_test.go index eb96e4d2..12a4e489 100644 --- a/msm/fuzz_test.go +++ b/msm/fuzz_test.go @@ -6,6 +6,7 @@ package metadata import ( "bytes" "context" + "crypto/sha256" "testing" "time" @@ -114,6 +115,11 @@ func FuzzVerifyBlock(f *testing.F) { // Mutating an authoritative field must make the block fail verification. fuzzedMD := block.Metadata field.set(&fuzzedMD, value) + + if fieldIdx%2 == 1 && block.Metadata.AuxiliaryInfo == nil { + fuzzedMD.AuxiliaryInfo = &AuxiliaryInfo{PrevAuxInfoSeq: value} + } + if bytes.Equal(fuzzedMD.MarshalCanoto(), block.Metadata.MarshalCanoto()) { t.Skip() // no-op mutation } @@ -184,6 +190,10 @@ func buildEpochChain(tb testing.TB, logger common.Logger) ([]*StateMachineBlock, currentTime := startTime sm, tc := newStateMachineWithLogger(tb, logger) + // This chain exercises the epoch lifecycle, not auxiliary info. Use an app whose + // history is always final so approvals are collected from the first collecting + // round (the builder only collects approvals once the aux info history is ready). + sm.AuxiliaryInfoApp = &noopTestAuxInfoApp{} sm.GetValidatorSet = getValidatorSet sm.GetPChainHeightForProposing = func() uint64 { return currentPChainHeight } sm.GetPChainHeightForVerifying = func() uint64 { return currentPChainHeight } @@ -244,21 +254,26 @@ func buildEpochChain(tb testing.TB, logger common.Logger) ([]*StateMachineBlock, block3 := build(3, 2, 1, block2) addBlock(3, block3, nil) + // The noopTestAuxInfoApp is always "ready" with an empty aux info history, so the candidate + // aux info digest the builder signs over is sha256 of the empty history. Peer approvals must + // carry the same digest to survive sanitizeApprovals' digest filter. + auxInfoDigest := sha256.Sum256(nil) + // block4 & block5: collecting-approvals blocks (1/3 then 2/3, not enough to seal). - approvalsResult = ValidatorSetApprovals{{NodeID: node1, PChainHeight: pChainHeight2, Signature: []byte("sig1")}} + approvalsResult = ValidatorSetApprovals{{NodeID: node1, PChainHeight: pChainHeight2, AuxInfoDigest: auxInfoDigest, Signature: signApproval(pChainHeight2, auxInfoDigest)}} currentTime = startTime.Add(time.Second + 4*time.Millisecond) tc.blockBuilder.block = nextInner(4) block4 := build(4, 3, 1, block3) addBlock(4, block4, nil) - approvalsResult = ValidatorSetApprovals{{NodeID: node2, PChainHeight: pChainHeight2, Signature: []byte("sig2")}} + approvalsResult = ValidatorSetApprovals{{NodeID: node2, PChainHeight: pChainHeight2, AuxInfoDigest: auxInfoDigest, Signature: signApproval(pChainHeight2, auxInfoDigest)}} currentTime = startTime.Add(time.Second + 5*time.Millisecond) tc.blockBuilder.block = nextInner(5) block5 := build(5, 4, 1, block4) addBlock(5, block5, nil) // block6: the sealing block (3/3 approvals). Its successor is in stateBuildBlockEpochSealed. - approvalsResult = ValidatorSetApprovals{{NodeID: node3, PChainHeight: pChainHeight2, Signature: []byte("sig3")}} + approvalsResult = ValidatorSetApprovals{{NodeID: node3, PChainHeight: pChainHeight2, AuxInfoDigest: auxInfoDigest, Signature: signApproval(pChainHeight2, auxInfoDigest)}} currentTime = startTime.Add(time.Second + 6*time.Millisecond) tc.blockBuilder.block = nextInner(6) block6 := build(6, 5, 1, block5) @@ -286,6 +301,7 @@ func buildEpochChain(tb testing.TB, logger common.Logger) ([]*StateMachineBlock, // Build a separate verifier MSM with its own copy of the fully populated store. verifier, vtc := newStateMachineWithLogger(tb, logger) + verifier.AuxiliaryInfoApp = &noopTestAuxInfoApp{} vtc.blockStore = tc.blockStore.clone() verifier.GetBlock = vtc.blockStore.getBlock verifier.GetValidatorSet = getValidatorSet diff --git a/msm/msm.go b/msm/msm.go index 20507935..bb7797a2 100644 --- a/msm/msm.go +++ b/msm/msm.go @@ -11,6 +11,7 @@ import ( "errors" "fmt" "math" + "slices" "time" "github.com/ava-labs/simplex/common" @@ -87,6 +88,7 @@ var ( errTimestampTooBig = errors.New("invalid timestamp: exceeds maximum int64 value") errTimestampDecreasing = errors.New("invalid timestamp: proposed timestamp is before parent block's timestamp") errTimestampTooFarInFuture = errors.New("invalid timestamp: proposed timestamp is too far in the future compared to current time") + errAuxInfoBlockRetrieval = errors.New("failed to retrieve block while collecting auxiliary info") signatureContext = "MSM approval" ) @@ -162,6 +164,30 @@ type BlockBuilder interface { WaitForPendingBlock(ctx context.Context) } +// AuxiliaryInfoGenVerifier abstracts the application-specific logic for generating and verifying auxiliary information +// that is piggybacked on epoch transitions. +type AuxiliaryInfoGenVerifier interface { + // IsLegalAppend checks whether the given auxiliary information byte slice [x] + // can be appended to the history of auxiliary information for the given versionID, according to the app's rules. + // Returns nil if the append is legal, or an error if the append is not legal or if any error occurs during the check. + IsLegalAppend(versionID VersionID, nodes NodeBLSMappings, history [][]byte, x []byte) error + + // IsSufficient checks whether the given history of auxiliary information for the given versionID is sufficient + // to start the epoch transition process. + IsSufficient(versionID VersionID, nodes NodeBLSMappings, history [][]byte) (bool, error) + + // Generate generates an auxiliary information encoded as a byte slice based on the history of auxiliary information + // for the given versionID in the current epoch so far. + // If this is the first invocation in the epoch, DefaultversionID() should be passed as the VersionID. + // Otherwise, the versionID from previous blocks in the epoch should be used. + // If the application deems the given history to be sufficient for the epoch change, it can return a nil byte slice, + // in which case it will not be appended to the history. + Generate(versionID VersionID, nodes NodeBLSMappings, history [][]byte) ([]byte, error) + + // DefaultVersionID returns the default VersionID that should be used for epochs that don't have any any auxiliary information yet. + DefaultVersionID() VersionID +} + // StateMachine manages block building and verification across epoch transitions. type StateMachine struct { *Config @@ -213,6 +239,8 @@ type Config struct { Signer common.Signer // ComputeICMEpoch computes the ICM epoch information in order to know which P-chain height to encode. ComputeICMEpoch ICMEpochTransition + // AuxiliaryInfoApp abstracts an application that piggybacks on epoch changes. + AuxiliaryInfoApp AuxiliaryInfoGenVerifier } type state uint8 @@ -454,7 +482,7 @@ func (sm *StateMachine) buildBlockOrTransitionEpoch(ctx context.Context, parentB } } - return wrapBlock(innerBlock, newSimplexEpochInfo, decisionToBuildBlock.pChainHeight, simplexMetadata, simplexBlacklist, now, icmEpochInfo), nil + return wrapBlock(innerBlock, newSimplexEpochInfo, decisionToBuildBlock.pChainHeight, simplexMetadata, simplexBlacklist, now, icmEpochInfo, nil), nil } func computeICMEpochInfo(parentBlock StateMachineBlock, computeICMEpoch ICMEpochTransition, childTimestamp time.Time) ICMEpochInfo { @@ -481,6 +509,7 @@ func verifyAgainstExpected( nextBlock *StateMachineBlock, timestamp time.Time, expectedIcmEpochInfo ICMEpochInfo, + auxInfo *AuxiliaryInfo, ) error { if innerBlock != nil { if err := innerBlock.Verify(ctx, expectedIcmEpochInfo.PChainEpochHeight); err != nil { @@ -489,7 +518,7 @@ func verifyAgainstExpected( } expectedBlock := wrapBlock( innerBlock, expectedSimplexEpochInfo, expectedPChainHeight, - nextBlock.Metadata.SimplexProtocolMetadata, nextBlock.Metadata.SimplexBlacklist, timestamp, expectedIcmEpochInfo) + nextBlock.Metadata.SimplexProtocolMetadata, nextBlock.Metadata.SimplexBlacklist, timestamp, expectedIcmEpochInfo, auxInfo) if expectedBlock.Digest() != nextBlock.Digest() { return fmt.Errorf("expected block digest %s does not match proposed block digest %s: %w", expectedBlock.Digest(), @@ -517,7 +546,7 @@ func (sm *StateMachine) verifyNormalBlock(ctx context.Context, parentBlock State } newSimplexEpochInfo.NextPChainReferenceHeight = nextBlock.Metadata.SimplexEpochInfo.NextPChainReferenceHeight - return verifyAgainstExpected(ctx, nextBlock.InnerBlock, newSimplexEpochInfo, proposedPChainHeight, nextBlock, timestamp, icmEpochInfo) + return verifyAgainstExpected(ctx, nextBlock.InnerBlock, newSimplexEpochInfo, proposedPChainHeight, nextBlock, timestamp, icmEpochInfo, nil) } func verifyPChainHeight(proposedPChainHeight uint64, currentPChainHeight uint64, prevPChainHeight uint64) error { @@ -755,6 +784,7 @@ func (sm *StateMachine) buildBlockZero(parentBlock StateMachineBlock, simplexMet simplexBlacklist, time.UnixMilli(timestamp), icmEpochInfo, + nil, ), nil } @@ -799,6 +829,7 @@ func (sm *StateMachine) verifyBlockZero(block *StateMachineBlock, prevBlock Stat block.Metadata.SimplexBlacklist, time.UnixMilli(int64(block.Metadata.Timestamp)), expectedICMEpochInfo, + nil, ) if expectedBlock.Digest() != block.Digest() { @@ -840,11 +871,31 @@ func (sm *StateMachine) verifyBlockZero(block *StateMachineBlock, prevBlock Stat // └──────────────────┘ └────────────────────┘ └────────────────────────────┘ // → stays Collecting → BuildBlockEpochSealed func (sm *StateMachine) buildBlockCollectingApprovals(ctx context.Context, parentBlock StateMachineBlock, simplexMetadata, simplexBlacklist []byte, prevBlockSeq uint64) (*StateMachineBlock, error) { - newApprovals, err := sm.computeNewApprovals(parentBlock) + // We prepare information that is needed to compute the approvals for the new epoch, + // such as the validator set for the next epoch, and the approvals from peers. + prevBlockNextPChainReferenceHeight := parentBlock.Metadata.SimplexEpochInfo.NextPChainReferenceHeight + validators, err := sm.GetValidatorSet(prevBlockNextPChainReferenceHeight) if err != nil { return nil, err } + auxInfo, isAuxInfoReadyForEpochTransition, auxInfoDigest, err := sm.computeAuxInfo(parentBlock, prevBlockSeq, validators) + if err != nil { + return nil, fmt.Errorf("failed to compute auxiliary info: %w", err) + } + + var newApprovals *approvals + if isAuxInfoReadyForEpochTransition { + newApprovals, err = sm.computeNewApprovals(parentBlock, validators, auxInfoDigest) + if err != nil { + return nil, err + } + } else { + // We're not ready for epoch transition yet, so putting a zero-value approvals here + // makes us stay in the collecting approvals state without contributing any approvals. + newApprovals = &approvals{} + } + newSimplexEpochInfo := computeSimplexEpochInfoForCollectingApprovalsBlock(parentBlock, prevBlockSeq, newApprovals) pChainHeight := parentBlock.Metadata.PChainHeight @@ -857,25 +908,16 @@ func (sm *StateMachine) buildBlockCollectingApprovals(ctx context.Context, paren // so that eventually we'll have enough approvals to seal the epoch. if !newApprovals.canSeal { sm.Logger.Debug("Not enough approvals to seal epoch, building block without sealing the epoch") - return sm.buildBlockImpatiently(ctx, parentBlock, simplexMetadata, simplexBlacklist, newSimplexEpochInfo, pChainHeight, icmEpochInfo) + return sm.buildBlockImpatiently(ctx, now, simplexMetadata, simplexBlacklist, newSimplexEpochInfo, pChainHeight, icmEpochInfo, auxInfo) } sm.Logger.Debug("Have enough approvals to seal epoch, building sealing block") // Else, we have enough approvals to seal the epoch, so we create the sealing block. - return sm.createSealingBlock(ctx, parentBlock, simplexMetadata, simplexBlacklist, newSimplexEpochInfo, pChainHeight, icmEpochInfo) + return sm.createSealingBlock(ctx, now, simplexMetadata, simplexBlacklist, newSimplexEpochInfo, pChainHeight, icmEpochInfo, auxInfo) } func (sm *StateMachine) verifyCollectingApprovalsBlock(ctx context.Context, parentBlock StateMachineBlock, nextBlock *StateMachineBlock, prevBlockSeq uint64) error { - nextMD := nextBlock.Metadata - newApprovals := nextMD.SimplexEpochInfo.NextEpochApprovals - - // The block builder should at least include its own approval in the block it builds, - // so we should have some approvals in the proposed block. - if newApprovals == nil || len(newApprovals.NodeIDs) == 0 || len(newApprovals.Signature) == 0 { - return errEmptyNextEpochApprovals - } - prevEpochInfo := parentBlock.Metadata.SimplexEpochInfo nextEpochInfo := nextBlock.Metadata.SimplexEpochInfo @@ -884,15 +926,32 @@ func (sm *StateMachine) verifyCollectingApprovalsBlock(ctx context.Context, pare return err } - err = sm.verifyNextEpochApprovalsSignature(prevEpochInfo, nextEpochInfo, validators) + newApprovals := nextBlock.Metadata.SimplexEpochInfo.NextEpochApprovals + + expectedAuxInfo, auxInfoDigest, isAuxInfoReady, err := sm.computeExpectedAuxInfoForApprovalCollection(parentBlock, nextBlock, prevBlockSeq, validators) if err != nil { - return err + return fmt.Errorf("failed to compute expected auxiliary info for approval collection: %w", err) } - // A node cannot remove other nodes' approvals, only add its own approval if it wasn't included in the previous block. - // So the set of signers in next.NextEpochApprovals should be a superset of the set of signers in prev.NextEpochApprovals. - if err := areNextEpochApprovalsSignersSupersetOfApprovalsOfPrevBlock(prevEpochInfo, nextEpochInfo); err != nil { - return err + // If the Auxiliary info is ready for epoch change, the block builder should at least include its own approval in the block it builds, + // so we should have some approvals in the proposed block. + if newApprovals == nil || len(newApprovals.NodeIDs) == 0 || len(newApprovals.Signature) == 0 { + if isAuxInfoReady { + return errEmptyNextEpochApprovals + } + } + + // If we aren't ready for epoch transition, we cannot collect approvals just yet. + // So just make an empty NextEpochApprovals. + if !isAuxInfoReady { + if newApprovals != nil && (len(newApprovals.NodeIDs) > 0 || len(newApprovals.Signature) > 0) { + return fmt.Errorf("expected no approvals when auxiliary info is not ready for epoch transition, but got some") + } + newSimplexEpochInfo := computeSimplexEpochInfoForCollectingApprovalsBlock(parentBlock, prevBlockSeq, &approvals{}) + timestamp := time.UnixMilli(int64(nextBlock.Metadata.Timestamp)) + icmEpochInfo := computeICMEpochInfo(parentBlock, sm.ComputeICMEpoch, timestamp) + + return verifyAgainstExpected(ctx, nextBlock.InnerBlock, newSimplexEpochInfo, nextBlock.Metadata.PChainHeight, nextBlock, timestamp, icmEpochInfo, expectedAuxInfo) } newSimplexEpochInfo := computeSimplexEpochInfoForCollectingApprovalsBlock(parentBlock, prevBlockSeq, &approvals{ @@ -900,6 +959,17 @@ func (sm *StateMachine) verifyCollectingApprovalsBlock(ctx context.Context, pare signature: newApprovals.Signature, }) + err = sm.verifyNextEpochApprovalsSignature(parentBlock.Metadata, nextBlock.Metadata, validators, auxInfoDigest) + if err != nil { + return err + } + + // A node cannot remove other nodes' approvals, only add its own approval if it wasn't included in the previous block. + // So the set of signers in next.NextEpochApprovals should be a superset of the set of signers in prev.NextEpochApprovals. + if err := areNextEpochApprovalsSignersSupersetOfApprovalsOfPrevBlock(prevEpochInfo, nextEpochInfo); err != nil { + return err + } + sigAggr := sm.SignatureAggregatorCreator(validators.NodeWeights()) approvals := bitmaskFromBytes(newApprovals.NodeIDs) canSeal := sigAggr.IsQuorum(validators.SelectSubset(approvals)) @@ -911,14 +981,16 @@ func (sm *StateMachine) verifyCollectingApprovalsBlock(ctx context.Context, pare } } - timestamp := time.UnixMilli(int64(nextMD.Timestamp)) - + timestamp := time.UnixMilli(int64(nextBlock.Metadata.Timestamp)) icmEpochInfo := computeICMEpochInfo(parentBlock, sm.ComputeICMEpoch, timestamp) - return verifyAgainstExpected(ctx, nextBlock.InnerBlock, newSimplexEpochInfo, nextMD.PChainHeight, nextBlock, timestamp, icmEpochInfo) + return verifyAgainstExpected(ctx, nextBlock.InnerBlock, newSimplexEpochInfo, nextBlock.Metadata.PChainHeight, nextBlock, timestamp, icmEpochInfo, expectedAuxInfo) } -func (sm *StateMachine) verifyNextEpochApprovalsSignature(prev SimplexEpochInfo, next SimplexEpochInfo, validators NodeBLSMappings) error { +func (sm *StateMachine) verifyNextEpochApprovalsSignature(prevMD StateMachineMetadata, nextMD StateMachineMetadata, validators NodeBLSMappings, auxInfoDigest [32]byte) error { + prev := prevMD.SimplexEpochInfo + next := nextMD.SimplexEpochInfo + // First figure out which validators are approving the next epoch by looking at the bitmask of approving nodes, // and then aggregate their public keys together to verify the signature. @@ -929,11 +1001,8 @@ func (sm *StateMachine) verifyNextEpochApprovalsSignature(prev SimplexEpochInfo, } pChainHeight := prev.NextPChainReferenceHeight - pChainHeightBuff := make([]byte, 8) - binary.BigEndian.PutUint64(pChainHeightBuff, pChainHeight) - signedMsg := common.SignedMessage{Payload: pChainHeightBuff, Context: signatureContext} - toBeSigned, err := asn1.Marshal(signedMsg) + toBeSigned, err := assembleApprovalToBeSigned(pChainHeight, auxInfoDigest) if err != nil { return err } @@ -944,6 +1013,17 @@ func (sm *StateMachine) verifyNextEpochApprovalsSignature(prev SimplexEpochInfo, return nil } +// assembleApprovalToBeSigned assembles the payload that is signed when approving an epoch transition. +// It consists of the P-chain reference height and the aux info digest (zeroed if not applicable). +func assembleApprovalToBeSigned(pChainHeight uint64, auxInfoDigest [32]byte) ([]byte, error) { + payloadToSignBuff := make([]byte, 8+32) // 8 bytes for the P-chain height and 32 bytes for the aux info digest if applicable. + binary.BigEndian.PutUint64(payloadToSignBuff[:8], pChainHeight) + copy(payloadToSignBuff[8:], auxInfoDigest[:]) + + signedMsg := common.SignedMessage{Payload: payloadToSignBuff, Context: signatureContext} + return asn1.Marshal(signedMsg) +} + func (sm *StateMachine) aggregatePubKeysForBitmask(nodeIDsBitmask []byte, validators NodeBLSMappings) ([]byte, error) { approvingNodes := bitmaskFromBytes(nodeIDsBitmask) publicKeys := make([][]byte, 0, len(validators)) @@ -983,14 +1063,8 @@ func computeSimplexEpochInfoForCollectingApprovalsBlock(parentBlock StateMachine return newSimplexEpochInfo } -func (sm *StateMachine) computeNewApprovals(parentBlock StateMachineBlock) (*approvals, error) { - // We prepare information that is needed to compute the approvals for the new epoch, - // such as the validator set for the next epoch, and the approvals from peers. - validators, err := sm.GetValidatorSet(parentBlock.Metadata.SimplexEpochInfo.NextPChainReferenceHeight) - if err != nil { - return nil, err - } - +func (sm *StateMachine) computeNewApprovals(parentBlock StateMachineBlock, validators NodeBLSMappings, auxInfoDigest [32]byte) (*approvals, error) { + prevBlockNextPChainReferenceHeight := parentBlock.Metadata.SimplexEpochInfo.NextPChainReferenceHeight sigAggr := sm.SignatureAggregatorCreator(validators.NodeWeights()) // We retrieve approvals that validators have sent us for the next epoch. @@ -1000,33 +1074,115 @@ func (sm *StateMachine) computeNewApprovals(parentBlock StateMachineBlock) (*app // Optimistically sign the epoch transition even if we have already did so in a previous round. // We'll just deduplicate this approval later on. - pChainHeightBuff := make([]byte, 8) - binary.BigEndian.PutUint64(pChainHeightBuff, parentBlock.Metadata.SimplexEpochInfo.NextPChainReferenceHeight) - sig, err := sm.Signer.Sign(pChainHeightBuff) + + sig, err := sm.createSelfApproval(prevBlockNextPChainReferenceHeight, auxInfoDigest) if err != nil { - return nil, fmt.Errorf("failed to sign approval: %w", err) + return nil, err } approvalsFromPeers = append(approvalsFromPeers, ValidatorSetApproval{ - NodeID: nodeID(sm.MyNodeID), - PChainHeight: parentBlock.Metadata.SimplexEpochInfo.NextPChainReferenceHeight, - Signature: sig, + NodeID: nodeID(sm.MyNodeID), + PChainHeight: prevBlockNextPChainReferenceHeight, + AuxInfoDigest: auxInfoDigest, + Signature: sig, }) - nextPChainHeight := parentBlock.Metadata.SimplexEpochInfo.NextPChainReferenceHeight + nextPChainHeight := prevBlockNextPChainReferenceHeight prevNextEpochApprovals := parentBlock.Metadata.SimplexEpochInfo.NextEpochApprovals - newApprovals, err := computeNewApprovals(prevNextEpochApprovals, approvalsFromPeers, nextPChainHeight, sigAggr, validators, sm.Logger) + newApprovals, err := computeNewApprovals(prevNextEpochApprovals, approvalsFromPeers, nextPChainHeight, auxInfoDigest, sigAggr, validators, sm.Logger) if err != nil { return nil, err } return newApprovals, nil } +func (sm *StateMachine) createSelfApproval(nextPChainReferenceHeight uint64, auxInfoDigest [32]byte) ([]byte, error) { + toBeSigned, err := assembleApprovalToBeSigned(nextPChainReferenceHeight, auxInfoDigest) + if err != nil { + return nil, err + } + + sig, err := sm.Signer.Sign(toBeSigned) + if err != nil { + return nil, fmt.Errorf("failed to sign approval: %w", err) + } + return sig, nil +} + +type auxInfoHistory struct { + data [][]byte + lastSeq uint64 +} + +func (aih *auxInfoHistory) lastHistory() []byte { + if len(aih.data) == 0 { + return nil + } + return aih.data[len(aih.data)-1] +} + +// collectAuxiliaryInfo traverses backwards starting from the given block and collects the AuxiliaryInfo of all blocks in the chain. +// returns the collected AuxiliaryInfo, the corresponding sequences of the blocks they were collected from, +// and the application ID of the oldest block that contains a non empty Info (or defaultVersionID if there was none). +func collectAuxiliaryInfo(block StateMachineBlock, startSeq uint64, getBlock BlockRetriever, defaultVersionID VersionID) (auxInfoHistory, VersionID, error) { + var lastSeq *uint64 + var history [][]byte + var versionID = defaultVersionID + + // We traverse the chain of blocks backwards in the following manner: + // (1) Every block that doesn't have AuxiliaryInfo, its parents also do not have AuxiliaryInfo. + // (2) Every block that has AuxiliaryInfo, its descendants also have AuxiliaryInfo. + // (3) A block that has AuxiliaryInfo may have an empty Info field, but its PrevAuxInfoSeq field must point + // to a block that its AuxiliaryInfo isn't nil, and its Info field is also non-nil. + // (4) When a block with an empty Info field is built on a parent block that has AuxiliaryInfo, + // if its parent block has a non-empty Info field, then the block's PrevAuxInfoSeq points to its parent block. + // Else, its parent block has an empty Info field, then the block's PrevAuxInfoSeq is inherited from its parent block's PrevAuxInfoSeq. + + auxInfo := block.Metadata.AuxiliaryInfo + currentSeq := startSeq + for auxInfo != nil { + if len(auxInfo.Info) > 0 { + history = append(history, auxInfo.Info) + if lastSeq == nil { + lastSeq = new(uint64) + *lastSeq = currentSeq + } + versionID = auxInfo.VersionID + } + if auxInfo.PrevAuxInfoSeq == 0 { + // This is the first auxiliary info of the epoch, we can stop traversing back. + break + } + currentSeq = auxInfo.PrevAuxInfoSeq + prevBlock, _, err := getBlock(auxInfo.PrevAuxInfoSeq, [32]byte{}) + if err != nil { + return auxInfoHistory{}, 0, fmt.Errorf("%w: at sequence %d: %w", errAuxInfoBlockRetrieval, auxInfo.PrevAuxInfoSeq, err) + } + auxInfo = prevBlock.Metadata.AuxiliaryInfo + } + + if lastSeq == nil { + lastSeq = new(uint64) + *lastSeq = 0 + } + + // Reverse so the history (and the matching seqs) are ordered from oldest to newest. + slices.Reverse(history) + return auxInfoHistory{data: history, lastSeq: *lastSeq}, versionID, nil +} + // buildBlockImpatiently builds a block by waiting for the VM to build a block until MaxBlockBuildingWaitTime. // If the VM fails to build a block within that time, we build a block without an inner block, // so that we can continue making progress and not get stuck waiting for the VM. -func (sm *StateMachine) buildBlockImpatiently(ctx context.Context, parentBlock StateMachineBlock, simplexMetadata []byte, simplexBlacklist []byte, simplexEpochInfo SimplexEpochInfo, pChainHeight uint64, icmEpochInfo ICMEpochInfo) (*StateMachineBlock, error) { +func (sm *StateMachine) buildBlockImpatiently(ctx context.Context, + timestamp time.Time, + simplexMetadata []byte, + simplexBlacklist []byte, + simplexEpochInfo SimplexEpochInfo, + pChainHeight uint64, + icmEpochInfo ICMEpochInfo, + auxInfo *AuxiliaryInfo) (*StateMachineBlock, error) { impatientContext, cancel := context.WithTimeout(ctx, sm.MaxBlockBuildingWaitTime) defer cancel() @@ -1043,23 +1199,22 @@ func (sm *StateMachine) buildBlockImpatiently(ctx context.Context, parentBlock S zap.Duration("elapsed", time.Since(start)), zap.Duration("maxBlockBuildingWaitTime", sm.MaxBlockBuildingWaitTime)) } - now := sm.GetTime() - icmEpochInfo = computeICMEpochInfo(parentBlock, sm.ComputeICMEpoch, now) - return wrapBlock(innerBlock, simplexEpochInfo, pChainHeight, simplexMetadata, simplexBlacklist, now, icmEpochInfo), nil + return wrapBlock(innerBlock, simplexEpochInfo, pChainHeight, simplexMetadata, simplexBlacklist, timestamp, icmEpochInfo, auxInfo), nil } func (sm *StateMachine) createSealingBlock(ctx context.Context, - parentBlock StateMachineBlock, + timestamp time.Time, simplexMetadata []byte, simplexBlacklist []byte, simplexEpochInfo SimplexEpochInfo, pChainHeight uint64, - icmEpochInfo ICMEpochInfo) (*StateMachineBlock, error) { + icmEpochInfo ICMEpochInfo, + auxInfo *AuxiliaryInfo) (*StateMachineBlock, error) { simplexEpochInfo, err := sm.computeSimplexEpochInfoForSealingBlock(simplexEpochInfo) if err != nil { return nil, fmt.Errorf("failed to compute simplex epoch info for sealing block: %w", err) } - return sm.buildBlockImpatiently(ctx, parentBlock, simplexMetadata, simplexBlacklist, simplexEpochInfo, pChainHeight, icmEpochInfo) + return sm.buildBlockImpatiently(ctx, timestamp, simplexMetadata, simplexBlacklist, simplexEpochInfo, pChainHeight, icmEpochInfo, auxInfo) } func (sm *StateMachine) computeSimplexEpochInfoForSealingBlock(simplexEpochInfo SimplexEpochInfo) (SimplexEpochInfo, error) { @@ -1094,7 +1249,8 @@ func wrapBlock( simplexMetadata, simplexBlacklist []byte, timestamp time.Time, - icmEpochInfo ICMEpochInfo) *StateMachineBlock { + icmEpochInfo ICMEpochInfo, + auxiliaryInfo *AuxiliaryInfo) *StateMachineBlock { return &StateMachineBlock{ InnerBlock: childBlock, @@ -1105,6 +1261,7 @@ func wrapBlock( SimplexEpochInfo: newSimplexEpochInfo, PChainHeight: pChainHeight, ICMEpochInfo: icmEpochInfo, + AuxiliaryInfo: auxiliaryInfo, }, } } @@ -1158,7 +1315,7 @@ func (sm *StateMachine) buildBlockEpochSealed(ctx context.Context, parentBlock S icmEpochInfo := computeICMEpochInfo(parentBlock, sm.ComputeICMEpoch, now) newSimplexEpochInfo := computeSimplexEpochInfoForTelock(parentBlock, sealingBlockSeq, prevBlockSeq) pChainHeight := parentBlock.Metadata.PChainHeight - return wrapBlock(nil, newSimplexEpochInfo, pChainHeight, simplexMetadata, simplexBlacklist, now, icmEpochInfo), nil + return wrapBlock(nil, newSimplexEpochInfo, pChainHeight, simplexMetadata, simplexBlacklist, now, icmEpochInfo, nil), nil } // Else, we build a block for the new epoch. @@ -1198,13 +1355,12 @@ func (sm *StateMachine) verifyBlockEpochSealed(ctx context.Context, parentBlock timestamp := time.UnixMilli(int64(nextBlock.Metadata.Timestamp)) - now := sm.GetTime() - icmEpochInfo := computeICMEpochInfo(parentBlock, sm.ComputeICMEpoch, now) + icmEpochInfo := computeICMEpochInfo(parentBlock, sm.ComputeICMEpoch, timestamp) newSimplexEpochInfo := computeSimplexEpochInfoForTelock(parentBlock, sealingBlockSeq, prevBlockSeq) if !isSealingBlockFinalized { - return verifyAgainstExpected(ctx, nil, newSimplexEpochInfo, nextBlock.Metadata.PChainHeight, nextBlock, timestamp, icmEpochInfo) + return verifyAgainstExpected(ctx, nil, newSimplexEpochInfo, nextBlock.Metadata.PChainHeight, nextBlock, timestamp, icmEpochInfo, nil) } // Else, it's a new epoch. @@ -1225,7 +1381,111 @@ func (sm *StateMachine) verifyBlockEpochSealed(ctx context.Context, parentBlock } newSimplexEpochInfo.NextPChainReferenceHeight = nextBlock.Metadata.SimplexEpochInfo.NextPChainReferenceHeight - return verifyAgainstExpected(ctx, nextBlock.InnerBlock, newSimplexEpochInfo, proposedPChainHeight, nextBlock, timestamp, icmEpochInfo) + return verifyAgainstExpected(ctx, nextBlock.InnerBlock, newSimplexEpochInfo, proposedPChainHeight, nextBlock, timestamp, icmEpochInfo, nil) +} + +// computeExpectedAuxInfoForApprovalCollection computes the expected AuxiliaryInfo that should be included in the proposed block +// for approval collection, and returns the auxiliary info digest, and whether the auxiliary info history is ready for epoch transition. +func (sm *StateMachine) computeExpectedAuxInfoForApprovalCollection(parentBlock StateMachineBlock, nextBlock *StateMachineBlock, prevBlockSeq uint64, validators NodeBLSMappings) (*AuxiliaryInfo, [32]byte, bool, error) { + nextMD := nextBlock.Metadata + prevMD := parentBlock.Metadata + + auxInfoHistory, versionID, err := collectAuxiliaryInfo(parentBlock, prevBlockSeq, sm.GetBlock, sm.AuxiliaryInfoApp.DefaultVersionID()) + if err != nil { + return nil, [32]byte{}, false, err + } + + if len(auxInfoHistory.data) > 0 && nextMD.AuxiliaryInfo == nil { + // If we have auxiliary info history but the proposed block doesn't include any auxiliary info, + // it means the block builder has dropped the auxiliary info, which is not allowed. + return nil, [32]byte{}, false, fmt.Errorf("expected auxiliary info for application %d with history length %d, but got nil", versionID, len(auxInfoHistory.data)) + } + + // Else, either len(auxInfoHistory) == 0, + // or the proposed block includes auxiliary info. + // Both of these cases are fine, because a node doesn't have to include Auxiliary information. + // We will verify the legality of the proposed auxiliary info (if any) in the next step. + + var expectedAuxInfo *AuxiliaryInfo + var proposedAuxInf []byte + + if nextMD.AuxiliaryInfo != nil { + proposedAuxInf = nextMD.AuxiliaryInfo.Info + expectedAuxInfo = &AuxiliaryInfo{ + VersionID: versionID, + Info: proposedAuxInf, + } + if prevMD.AuxiliaryInfo != nil { + expectedAuxInfo.PrevAuxInfoSeq = auxInfoHistory.lastSeq + } + } + + if err := sm.AuxiliaryInfoApp.IsLegalAppend(versionID, validators, auxInfoHistory.data, proposedAuxInf); err != nil { + return nil, [32]byte{}, false, fmt.Errorf("proposed auxiliary info is not a legal append to the history for application %d: %w", versionID, err) + } + + auxInfoReady, err := sm.AuxiliaryInfoApp.IsSufficient(versionID, validators, auxInfoHistory.data) + if err != nil { + return nil, [32]byte{}, false, fmt.Errorf("failed to check if auxiliary info history is final for application %d: %w", versionID, err) + } + + var digest [32]byte + if auxInfoReady { + digest = sha256.Sum256(auxInfoHistory.lastHistory()) + } + + return expectedAuxInfo, digest, auxInfoReady, nil +} + +// computeAuxInfo computes the AuxiliaryInfo that should be included in the block being built, and whether the auxiliary info history is ready for epoch transition, +func (sm *StateMachine) computeAuxInfo(parentBlock StateMachineBlock, prevBlockSeq uint64, validators NodeBLSMappings) (*AuxiliaryInfo, bool, common.Digest, error) { + auxInfoHistory, versionID, err := collectAuxiliaryInfo(parentBlock, prevBlockSeq, sm.GetBlock, sm.AuxiliaryInfoApp.DefaultVersionID()) + if err != nil { + return nil, false, common.Digest{}, err + } + + isAuxInfoReadyForEpochTransition, err := sm.AuxiliaryInfoApp.IsSufficient(versionID, validators, auxInfoHistory.data) + if err != nil { + return nil, false, common.Digest{}, fmt.Errorf("failed to check if auxiliary info history is final: %w", err) + } + + var auxInfo *AuxiliaryInfo + parentAuxInfo := parentBlock.Metadata.AuxiliaryInfo + if parentAuxInfo != nil { + auxInfo = &AuxiliaryInfo{ + VersionID: parentAuxInfo.VersionID, + PrevAuxInfoSeq: auxInfoHistory.lastSeq, + } + } + + if !isAuxInfoReadyForEpochTransition { + // If the auxiliary info isn't ready for epoch transition, + // we should focus on contributing to finalizing it before collecting approvals for the epoch transition, + // as without it being ready, we won't be able to transition epochs anyway. + auxInf, err := sm.AuxiliaryInfoApp.Generate(versionID, validators, auxInfoHistory.data) + if err != nil { + return nil, false, common.Digest{}, fmt.Errorf("failed to generate auxiliary info: %w", err) + } + if auxInfo == nil { + // This is the first auxiliary info we're generating for this epoch, + // so we need to initialize it. + auxInfo = &AuxiliaryInfo{ + VersionID: versionID, + Info: auxInf, + } + } else { + // Otherwise, we already have auxiliary info from the parent block, + // so we just update the Info field and carry over the VersionID and PrevAuxInfoSeq. + auxInfo.Info = auxInf + } + } + + var auxInfoDigest common.Digest + if isAuxInfoReadyForEpochTransition { + auxInfoDigest = sha256.Sum256(auxInfoHistory.lastHistory()) + } + + return auxInfo, isAuxInfoReadyForEpochTransition, auxInfoDigest, nil } // constructSimplexZeroBlockSimplexEpochInfo constructs the SimplexEpochInfo for the zero block, which is the first ever block built by Simplex. @@ -1253,6 +1513,7 @@ func computeNewApprovals( prevNextEpochApprovals *NextEpochApprovals, approvalsFromPeers ValidatorSetApprovals, pChainHeight uint64, + auxInfoDigest [32]byte, sigAggr common.SignatureAggregator, validators NodeBLSMappings, logger common.Logger, @@ -1269,7 +1530,7 @@ func computeNewApprovals( // We have the approvals obtained from peers, but we need to sanitize them by filtering out approvals that are not valid, // such as approvals that do not agree with our candidate auxiliary info digest and P-Chain height, // and approvals that are from nodes that are not in the validator set or have already approved in prior blocks. - approvalsFromPeers = sanitizeApprovals(approvalsFromPeers, pChainHeight, nodeID2ValidatorIndex, oldApprovingNodes, logger) + approvalsFromPeers = sanitizeApprovals(approvalsFromPeers, pChainHeight, auxInfoDigest, nodeID2ValidatorIndex, oldApprovingNodes, logger) logger.Debug("Sanitized approvals after filtering out invalid approvals", zap.Int("numApprovalsBefore", oldApprovalFromPeersCount), zap.Int("numApprovalsAfter", len(approvalsFromPeers))) // Next we aggregate both previous and new approvals to compute the new aggregated signatures and the new bitmask of approving nodes. @@ -1344,21 +1605,22 @@ func computeNewApproverSignaturesAndSigners( // sanitizeApprovals filters out approvals that are not valid by checking if they agree with our candidate auxiliary info digest and P-Chain height, // and if they are from the validator set and haven't already been approved. -func sanitizeApprovals(approvals ValidatorSetApprovals, pChainHeight uint64, nodeID2ValidatorIndex map[nodeID]int, oldApprovingNodes bitmask, logger common.Logger) ValidatorSetApprovals { - filter1 := approvalsThatAgreeWithPChainHeight(pChainHeight) +func sanitizeApprovals(approvals ValidatorSetApprovals, pChainHeight uint64, auxInfoDigest [32]byte, nodeID2ValidatorIndex map[nodeID]int, oldApprovingNodes bitmask, logger common.Logger) ValidatorSetApprovals { + filter1 := approvalsThatAgreeWithPChainHeightAndAuxInfoDigest(pChainHeight, auxInfoDigest) filter2 := approvalsThatAreInValidatorSetAndHaveNotAlreadyApproved(oldApprovingNodes.Clone(), nodeID2ValidatorIndex) return approvals.Filter(filter1, logger).Filter(filter2, logger).UniqueByNodeID() } -func approvalsThatAgreeWithPChainHeight(pChainHeight uint64) func(approval ValidatorSetApproval, logger common.Logger) bool { +func approvalsThatAgreeWithPChainHeightAndAuxInfoDigest(pChainHeight uint64, auxInfoDigest [32]byte) func(approval ValidatorSetApproval, logger common.Logger) bool { return func(approval ValidatorSetApproval, logger common.Logger) bool { // Pick only approvals that agree with our P-Chain height - ok := approval.PChainHeight == pChainHeight + ok := approval.PChainHeight == pChainHeight && approval.AuxInfoDigest == auxInfoDigest if !ok { - logger.Debug("Filtering out approval that does not agree with our P-Chain height", + logger.Debug("Filtering out approval that does not agree with our P-Chain height or auxiliary info digest", zap.String("nodeID", fmt.Sprintf("%x", approval.NodeID)), zap.Uint64("approvalPChainHeight", approval.PChainHeight), - zap.Uint64("expectedPChainHeight", pChainHeight)) + zap.Uint64("expectedPChainHeight", pChainHeight), + zap.String("approvalAuxInfoSeqDigest", fmt.Sprintf("%x", approval.AuxInfoDigest))) } return ok } diff --git a/msm/msm_test.go b/msm/msm_test.go index 08959583..6d7fca94 100644 --- a/msm/msm_test.go +++ b/msm/msm_test.go @@ -6,6 +6,8 @@ package metadata import ( "context" "crypto/rand" + "crypto/sha256" + "errors" "fmt" "math" "testing" @@ -553,6 +555,13 @@ func TestMSMFullEpochLifecycle(t *testing.T) { sm.GetValidatorSet = getValidatorSet sm.GetTime = fixedTime + + // This test exercises the epoch/approval/seal lifecycle, not auxiliary info. + // Uses an app (noopTestAuxInfoApp) whose history is always final so approvals are collected from the + // first collecting round and no auxiliary info is generated. Auxiliary info + // behavior is covered by TestVerifyCollectingApprovalsNotReady and + // TestCollectAuxiliaryInfo. + sm.AuxiliaryInfoApp = &noopTestAuxInfoApp{} tc.blockStore[0] = &outerBlock{block: genesis} tc.blockStore[42] = &outerBlock{block: notGenesis} @@ -561,6 +570,7 @@ func TestMSMFullEpochLifecycle(t *testing.T) { sm.LastNonSimplexBlockPChainHeight = pChainHeight1 smVerify, tcVerify := newStateMachine(t) + smVerify.AuxiliaryInfoApp = &noopTestAuxInfoApp{} smVerify.GetValidatorSet = getValidatorSet // sm only ever builds blocks and smVerify only ever verifies them. @@ -695,17 +705,19 @@ func TestMSMFullEpochLifecycle(t *testing.T) { var approvalsResult ValidatorSetApprovals sm.ApprovalsRetriever = &dynamicApprovalsRetriever{approvals: &approvalsResult} + sig1 := signApproval(pChainHeight2, emptyAuxInfoDigest) approvalsResult = ValidatorSetApprovals{ { - NodeID: node1, - PChainHeight: pChainHeight2, - Signature: []byte("sig1"), + NodeID: node1, + PChainHeight: pChainHeight2, + AuxInfoDigest: emptyAuxInfoDigest, + Signature: sig1, }, } // node1 is at index 0 in validatorSet2 → bitmask bit 0 → {1} bitmask := []byte{1} - sig, err := aggr.AppendSignatures(nil, []byte("sig1")) + sig, err := aggr.AppendSignatures(nil, sig1) require.NoError(t, err) currentTime = startTime.Add(time.Second + 4*time.Millisecond) @@ -737,16 +749,18 @@ func TestMSMFullEpochLifecycle(t *testing.T) { require.NoError(t, smVerify.VerifyBlock(context.Background(), block4)) // ----- Step 5: Second collecting block (2/3 approvals, still not enough since threshold is strictly > 2/3) ----- + sig2 := signApproval(pChainHeight2, emptyAuxInfoDigest) approvalsResult = ValidatorSetApprovals{ { - NodeID: node2, - PChainHeight: pChainHeight2, - Signature: []byte("sig2"), + NodeID: node2, + PChainHeight: pChainHeight2, + AuxInfoDigest: emptyAuxInfoDigest, + Signature: sig2, }, } // node2 is at index 1 → bitmask bits 0,1 → {3} - sig, err = aggr.AppendSignatures(sig, []byte("sig2")) + sig, err = aggr.AppendSignatures(sig, sig2) require.NoError(t, err) bitmask = []byte{3} @@ -779,16 +793,18 @@ func TestMSMFullEpochLifecycle(t *testing.T) { require.NoError(t, smVerify.VerifyBlock(context.Background(), block5)) // ----- Step 6: Sealing block (3/3 approvals, enough to seal) ----- + sig3 := signApproval(pChainHeight2, emptyAuxInfoDigest) approvalsResult = ValidatorSetApprovals{ { - NodeID: node3, - PChainHeight: pChainHeight2, - Signature: []byte("sig3"), + NodeID: node3, + PChainHeight: pChainHeight2, + AuxInfoDigest: emptyAuxInfoDigest, + Signature: sig3, }, } // node3 is at index 2 → bitmask bits 0,1,2 → {7} - sig6, err := aggr.AppendSignatures(sig, []byte("sig3")) + sig6, err := aggr.AppendSignatures(sig, sig3) require.NoError(t, err) bitmask = []byte{7} @@ -1275,7 +1291,18 @@ func TestSanitizeApprovals(t *testing.T) { {NodeID: node1, PChainHeight: 200}, } oldApproving := bitmaskFromBytes(nil) - result := sanitizeApprovals(approvals, 100, nodeID2Index, oldApproving, logger) + result := sanitizeApprovals(approvals, 100, [32]byte{}, nodeID2Index, oldApproving, logger) + require.Len(t, result, 1) + require.Equal(t, node0, result[0].NodeID) + }) + + t.Run("filters by aux info digest", func(t *testing.T) { + approvals := ValidatorSetApprovals{ + {NodeID: node0, PChainHeight: 100, AuxInfoDigest: [32]byte{0xAA}}, + {NodeID: node1, PChainHeight: 100, AuxInfoDigest: [32]byte{0xBB}}, + } + oldApproving := bitmaskFromBytes(nil) + result := sanitizeApprovals(approvals, 100, [32]byte{0xAA}, nodeID2Index, oldApproving, logger) require.Len(t, result, 1) require.Equal(t, node0, result[0].NodeID) }) @@ -1286,7 +1313,7 @@ func TestSanitizeApprovals(t *testing.T) { {NodeID: node1, PChainHeight: 100}, } oldApproving := bitmaskFromBytes([]byte{1}) - result := sanitizeApprovals(approvals, 100, nodeID2Index, oldApproving, logger) + result := sanitizeApprovals(approvals, 100, [32]byte{}, nodeID2Index, oldApproving, logger) require.Len(t, result, 1) require.Equal(t, node1, result[0].NodeID) }) @@ -1297,7 +1324,7 @@ func TestSanitizeApprovals(t *testing.T) { {NodeID: node2, PChainHeight: 100}, } oldApproving := bitmaskFromBytes(nil) - result := sanitizeApprovals(approvals, 100, nodeID2Index, oldApproving, logger) + result := sanitizeApprovals(approvals, 100, [32]byte{}, nodeID2Index, oldApproving, logger) require.Len(t, result, 1) require.Equal(t, node2, result[0].NodeID) }) @@ -1308,7 +1335,7 @@ func TestSanitizeApprovals(t *testing.T) { {NodeID: node0, PChainHeight: 100}, } oldApproving := bitmaskFromBytes(nil) - result := sanitizeApprovals(approvals, 100, nodeID2Index, oldApproving, logger) + result := sanitizeApprovals(approvals, 100, [32]byte{}, nodeID2Index, oldApproving, logger) require.Len(t, result, 1) }) } @@ -1447,6 +1474,11 @@ func TestComputeNewApproverSignaturesAndSigners(t *testing.T) { func TestBuildBlockCollectingApprovalsDedupsOwnApprovalAcrossRounds(t *testing.T) { sm, tc := newStateMachine(t) + // This test is about approval dedup, not auxiliary info. Use an app whose history + // is always final so approvals are collected from the first collecting round + // (the builder only collects approvals once the aux info history is ready). + sm.AuxiliaryInfoApp = &noopTestAuxInfoApp{} + // Use concatAggregator so that AppendSignatures(existing) with zero new // signatures returns `existing` verbatim. This makes signature equality // a direct witness that no new signature was aggregated in. @@ -1522,3 +1554,419 @@ func TestBuildBlockCollectingApprovalsDedupsOwnApprovalAcrossRounds(t *testing.T require.Equal(t, firstSig, block2.Metadata.SimplexEpochInfo.NextEpochApprovals.Signature, "aggregated signature must be unchanged when no new approvals were aggregated") } + +func TestVerifyCollectingApprovalsNotReady(t *testing.T) { + // Tests collecting-approvals state while the auxiliary info history is not yet final. + // In that state the builder does not collect approvals, + // so a block legitimately carries no NextEpochApprovals. The verifier must + // (1) accept such a block, (2) not panic when NextEpochApprovals is nil, and + // (3) reject a block that carries approvals before the aux info is ready. + + const ( + pChainRefHeight = uint64(100) + nextPChainRefHeight = uint64(200) + parentSeq = uint64(10) + ) + + newSM := func(t *testing.T) (*StateMachine, *testConfig, StateMachineBlock) { + sm, tc := newStateMachine(t) + // Default app (AuxiliaryInfoGenVerifier) for newStateMachine has threshold 2 + // so IsSufficient returns false: the aux info is not ready. + sm.GetPChainHeightForProposing = func() uint64 { return nextPChainRefHeight } + sm.GetPChainHeightForVerifying = func() uint64 { return nextPChainRefHeight } + + // Parent block: epoch transition in progress (NextPChainReferenceHeight > 0), + // not yet sealed, so NextState() is stateBuildCollectingApprovals. + parent := StateMachineBlock{ + InnerBlock: &InnerBlock{TS: time.Now(), BlockHeight: 1, Bytes: []byte{0xAA}}, + Metadata: StateMachineMetadata{ + PChainHeight: nextPChainRefHeight, + SimplexProtocolMetadata: (&common.ProtocolMetadata{ + Seq: parentSeq, Round: 5, Epoch: 1, + }).Bytes(), + SimplexEpochInfo: SimplexEpochInfo{ + PChainReferenceHeight: pChainRefHeight, + EpochNumber: 1, + NextPChainReferenceHeight: nextPChainRefHeight, + PrevVMBlockSeq: parentSeq - 1, + }, + }, + } + tc.blockStore[parentSeq] = &outerBlock{block: parent} + require.Equal(t, stateBuildCollectingApprovals, parent.Metadata.SimplexEpochInfo.NextState()) + return sm, tc, parent + } + + build := func(t *testing.T, sm *StateMachine, tc *testConfig, parent StateMachineBlock) *StateMachineBlock { + tc.blockBuilder.block = &InnerBlock{TS: time.Now(), BlockHeight: 2, Bytes: []byte{0x01}} + md := common.ProtocolMetadata{Seq: parentSeq + 1, Round: 6, Epoch: 1, Prev: parent.Digest()} + block, err := sm.BuildBlock(context.Background(), md, nil) + require.NoError(t, err) + return block + } + + t.Run("built not-ready block verifies and carries no approvals", func(t *testing.T) { + sm, tc, parent := newSM(t) + block := build(t, sm, tc, parent) + + // The builder generated auxiliary info but collected no approvals. + require.NotNil(t, block.Metadata.AuxiliaryInfo) + require.Empty(t, block.Metadata.SimplexEpochInfo.NextEpochApprovals.NodeIDs) + require.Empty(t, block.Metadata.SimplexEpochInfo.NextEpochApprovals.Signature) + + require.NoError(t, sm.VerifyBlock(context.Background(), block)) + }) + + t.Run("nil NextEpochApprovals does not panic", func(t *testing.T) { + sm, tc, parent := newSM(t) + block := build(t, sm, tc, parent) + block.Metadata.SimplexEpochInfo.NextEpochApprovals = nil + + // The regression: verifying a not-ready block with nil approvals must not panic. + // (The digest no longer matches, so an error is expected — just not a panic.) + require.NotPanics(t, func() { + _ = sm.verifyCollectingApprovalsBlock(context.Background(), parent, block, parentSeq) + }) + }) + + t.Run("approvals before aux info is ready are rejected", func(t *testing.T) { + sm, tc, parent := newSM(t) + block := build(t, sm, tc, parent) + block.Metadata.SimplexEpochInfo.NextEpochApprovals = &NextEpochApprovals{ + NodeIDs: []byte{1}, + Signature: []byte("sig"), + } + + err := sm.verifyCollectingApprovalsBlock(context.Background(), parent, block, parentSeq) + require.ErrorContains(t, err, "expected no approvals") + }) +} + +func TestCollectingApprovalsAuxInfoGating(t *testing.T) { + // Walks a chain through the collecting-approvals state while the auxiliary info history fills up. + // With a threshold of 2, the history only becomes final after two distinct votes, + // so the first two collecting blocks carry auxiliary info but no approvals, + // and approvals are collected only once the history is ready. + // Each built block must verify. + + const ( + pChainRefHeight = uint64(100) + nextPChainRefHeight = uint64(200) + parentSeq = uint64(10) + ) + + sm, tc := newStateMachine(t) + sm.GetPChainHeightForProposing = func() uint64 { return nextPChainRefHeight } + sm.GetPChainHeightForVerifying = func() uint64 { return nextPChainRefHeight } + + // Deterministic votes so the built auxiliary info is predictable. + vote1 := []byte("vote-1") + vote2 := []byte("vote-2") + votes := [][]byte{vote1, vote2} + sm.AuxiliaryInfoApp = &voteCountingAuxInfoApp{ + threshold: 2, + randomTape: func() []byte { + next := votes[0] + votes = votes[1:] + return next + }, + } + + // A 3-node validator set including MyNodeID at index 0, so the optimistic self-approval + // is retained once approvals are collected, but a single approval is below quorum (the + // block stays in the collecting state rather than sealing). + validators := NodeBLSMappings{ + {NodeID: nodeID(sm.MyNodeID), BLSKey: []byte{1}, Weight: 1}, + {NodeID: nodeID{0xBB}, BLSKey: []byte{2}, Weight: 1}, + {NodeID: nodeID{0xCC}, BLSKey: []byte{3}, Weight: 1}, + } + tc.validatorSetRetriever.result = validators + + parent := StateMachineBlock{ + InnerBlock: &InnerBlock{TS: time.Now(), BlockHeight: 1, Bytes: []byte{0xAA}}, + Metadata: StateMachineMetadata{ + PChainHeight: nextPChainRefHeight, + SimplexProtocolMetadata: (&common.ProtocolMetadata{ + Seq: parentSeq, Round: 5, Epoch: 1, + }).Bytes(), + SimplexEpochInfo: SimplexEpochInfo{ + PChainReferenceHeight: pChainRefHeight, + EpochNumber: 1, + NextPChainReferenceHeight: nextPChainRefHeight, + PrevVMBlockSeq: parentSeq - 1, + }, + }, + } + tc.blockStore[parentSeq] = &outerBlock{block: parent} + + // build constructs the next collecting block on top of prev, stores it so it can serve + // as a parent (and as a back-pointer target for the aux info history), and verifies it. + build := func(seq uint64, prev StateMachineBlock) *StateMachineBlock { + tc.blockBuilder.block = &InnerBlock{TS: time.Now(), BlockHeight: seq, Bytes: []byte{byte(seq)}} + md := common.ProtocolMetadata{Seq: seq, Round: seq, Epoch: 1, Prev: prev.Digest()} + block, err := sm.BuildBlock(context.Background(), md, nil) + require.NoError(t, err) + require.NoError(t, sm.VerifyBlock(context.Background(), block)) + tc.blockStore[seq] = &outerBlock{block: *block} + return block + } + + approvals := func(b *StateMachineBlock) *NextEpochApprovals { + return b.Metadata.SimplexEpochInfo.NextEpochApprovals + } + // requireAuxInfo compares the meaningful fields, ignoring the cached canoto size. + requireAuxInfo := func(want, got *AuxiliaryInfo) { + require.True(t, want.Equal(got), "expected aux info %+v, got %+v", want, got) + } + + // block1: history empty, not final -> generates vote1, collects no approvals. + block1 := build(parentSeq+1, parent) + requireAuxInfo(&AuxiliaryInfo{Info: vote1, VersionID: 1}, block1.Metadata.AuxiliaryInfo) + require.Empty(t, approvals(block1).NodeIDs) + + // block2: history [vote1], still not final -> generates vote2, collects no approvals. + block2 := build(parentSeq+2, *block1) + requireAuxInfo(&AuxiliaryInfo{Info: vote2, PrevAuxInfoSeq: parentSeq + 1, VersionID: 1}, block2.Metadata.AuxiliaryInfo) + require.Empty(t, approvals(block2).NodeIDs) + + // block3: history [vote1, vote2] is now final -> no new vote, and approvals are + // collected (the optimistic self-approval sets MyNodeID's bit). block3 is the first + // empty-Info block; it points at block2, the last non-empty Info block. + block3 := build(parentSeq+3, *block2) + requireAuxInfo(&AuxiliaryInfo{PrevAuxInfoSeq: parentSeq + 2, VersionID: 1}, block3.Metadata.AuxiliaryInfo) + require.Equal(t, []byte{1}, approvals(block3).NodeIDs, "self-approval bit should be set once aux info is ready") + + // The collected approval must be signed over the epoch-transition payload for the + //mnext epoch's P-chain reference height (200) and the digest + // of the final auxiliary info history, which is sha256 of the last vote (vote2). + wantSigned, err := assembleApprovalToBeSigned(nextPChainRefHeight, sha256.Sum256(vote2)) + require.NoError(t, err) + require.NoError(t, (&signatureVerifier{}).VerifySignature(approvals(block3).Signature, wantSigned, nil), + "NextEpochApprovals signature must verify against P-chain height 200 and the digest of vote2") + + // block4: built on the empty-Info block3 while still collecting approvals (1/3 is below + // quorum). Its PrevAuxInfoSeq must SKIP the empty block3 and point at block2 (parentSeq+2), + // the most recent non-empty Info block -- not at its immediate parent block3 (parentSeq+3). + // This is the case the rest of the chain never reaches and where "skip" differs from "successive". + block4 := build(parentSeq+4, *block3) + require.NotEqual(t, parentSeq+3, block4.Metadata.AuxiliaryInfo.PrevAuxInfoSeq, + "PrevAuxInfoSeq must not point at the empty-Info parent block3") + requireAuxInfo(&AuxiliaryInfo{PrevAuxInfoSeq: parentSeq + 2, VersionID: 1}, block4.Metadata.AuxiliaryInfo) + + // block5: another empty-Info block on top of the empty block4. The back-pointer still skips + // the whole empty run and points at block2, confirming the skip persists across consecutive + // empty-Info blocks (collectAuxiliaryInfo finds the same most-recent non-empty block each time). + block5 := build(parentSeq+5, *block4) + requireAuxInfo(&AuxiliaryInfo{PrevAuxInfoSeq: parentSeq + 2, VersionID: 1}, block5.Metadata.AuxiliaryInfo) +} + +func TestCollectingApprovalsAuxInfoVersionIDIsBackwardCompatible(t *testing.T) { + // Backward compatibility: once an epoch has a VersionID set on its auxiliary info, that + // VersionID must be reused for the rest of the epoch -- for both building AND verifying + // subsequent blocks -- even if the application's DefaultVersionID() later changes. + // + // collectAuxiliaryInfo only consults DefaultVersionID() when the auxiliary info history is + // empty; once a block carries a VersionID, every later buildAndVerify and verify reads that VersionID + // back from the chain instead. So we seed the epoch's parent with auxiliary info stamped with + // VersionID 1, then flip DefaultVersionID() to 2 right after the first Generate(). Because the + // epoch already has a VersionID on-chain, every Generate()/IsLegalAppend()/IsSufficient() + // invocation -- on the buildAndVerify path and the verify path -- must keep using VersionID 1, never 2. + // The app asserts that internally: it requires the VersionID it receives to equal + // expectedVersionID, which we hold at 1 throughout. + + const ( + pChainRefHeight = uint64(100) + nextPChainRefHeight = uint64(200) + parentSeq = uint64(10) + ) + + sm, tc := newStateMachine(t) + sm.GetPChainHeightForVerifying = func() uint64 { return nextPChainRefHeight } + sm.GetPChainHeightForProposing = func() uint64 { return nextPChainRefHeight } + + // threshold 4 so Generate() runs for the first three collecting blocks built on top of the + // pre-seeded parent (history not yet sufficient), giving us one "first" and several "later" + // Generate() invocations. defaultVersionID starts at 1 (the original default); expectedVersionID + // stays 1 for the whole test -- the app asserts every invocation uses it. + app := &versionRecordingAuxInfoApp{ + t: t, + threshold: 4, + votes: [][]byte{[]byte("vote-1"), []byte("vote-2"), []byte("vote-3")}, + defaultVersionID: 1, + expectedVersionID: 1, + } + sm.AuxiliaryInfoApp = app + + validators := NodeBLSMappings{ + {NodeID: nodeID(sm.MyNodeID), BLSKey: []byte{1}, Weight: 1}, + {NodeID: nodeID{0xBB}, BLSKey: []byte{2}, Weight: 1}, + {NodeID: nodeID{0xCC}, BLSKey: []byte{3}, Weight: 1}, + } + tc.validatorSetRetriever.result = validators + + // The parent already carries auxiliary info for this epoch, stamped with VersionID 1. + // This is the backward-compatibility precondition: the epoch's VersionID is already set. + parent := StateMachineBlock{ + InnerBlock: &InnerBlock{TS: time.Now(), BlockHeight: 1, Bytes: []byte{0xAA}}, + Metadata: StateMachineMetadata{ + PChainHeight: nextPChainRefHeight, + SimplexProtocolMetadata: (&common.ProtocolMetadata{ + Seq: parentSeq, Round: 5, Epoch: 1, + }).Bytes(), + SimplexEpochInfo: SimplexEpochInfo{ + PChainReferenceHeight: pChainRefHeight, + EpochNumber: 1, + NextPChainReferenceHeight: nextPChainRefHeight, + PrevVMBlockSeq: parentSeq - 1, + }, + AuxiliaryInfo: &AuxiliaryInfo{ + VersionID: 1, + Info: []byte("vote-0"), + PrevAuxInfoSeq: 0, + }, + }, + } + tc.blockStore[parentSeq] = &outerBlock{block: parent} + + // buildAndVerify constructs the next collecting block on top of prev, verifies it, and stores it so it + // can serve as the next parent (and as a back-pointer target for the aux info history). + buildAndVerify := func(seq uint64, prev StateMachineBlock) *StateMachineBlock { + tc.blockBuilder.block = &InnerBlock{TS: time.Now(), BlockHeight: seq, Bytes: []byte{byte(seq)}} + md := common.ProtocolMetadata{Seq: seq, Round: seq, Epoch: 1, Prev: prev.Digest()} + block, err := sm.BuildBlock(context.Background(), md, nil) + require.NoError(t, err) + require.NoError(t, sm.VerifyBlock(context.Background(), block)) + tc.blockStore[seq] = &outerBlock{block: *block} + return block + } + + // block1: the epoch already has VersionID 1 (from the parent), so the buildAndVerify reads 1 from the + // chain and generates vote-1 under VersionID 1. Being the first Generate(), we now flip the + // application's default to 2. Verifying block1 also reads VersionID 1 from the parent's aux + // info, so it passes despite the changed default. + block1 := buildAndVerify(parentSeq+1, parent) + require.Equal(t, VersionID(1), block1.Metadata.AuxiliaryInfo.VersionID) + app.defaultVersionID = 2 + + // block2, block3: the default is now 2, but each block's buildAndVerify and verify still read VersionID + // 1 back from the chain and ignore the changed default. + block2 := buildAndVerify(parentSeq+2, *block1) + require.Equal(t, VersionID(1), block2.Metadata.AuxiliaryInfo.VersionID) + + block3 := buildAndVerify(parentSeq+3, *block2) + require.Equal(t, VersionID(1), block3.Metadata.AuxiliaryInfo.VersionID) + + // block4: history [vote-0, vote-1, vote-2, vote-3] is now sufficient, so no further vote is + // generated and approvals are collected -- still under VersionID 1. + block4 := buildAndVerify(parentSeq+4, *block3) + require.Equal(t, VersionID(1), block4.Metadata.AuxiliaryInfo.VersionID) +} + +func TestCollectAuxiliaryInfo(t *testing.T) { + const versionID = VersionID(7) + + blockWithAuxInfo := func(info []byte, prevAuxInfoSeq uint64) StateMachineBlock { + return StateMachineBlock{ + Metadata: StateMachineMetadata{ + AuxiliaryInfo: &AuxiliaryInfo{ + Info: info, + PrevAuxInfoSeq: prevAuxInfoSeq, + VersionID: versionID, + }, + }, + } + } + + errRetrieval := errors.New("retrieval failed") + + // startSeq is the sequence of tt.block itself (the block collectAuxiliaryInfo starts from). + const startSeq = uint64(10) + + tests := []struct { + name string + block StateMachineBlock + blocks map[uint64]StateMachineBlock + getBlockErr error + expectedHistory [][]byte + expectedLastSeq uint64 + expectedversionID VersionID + expectedErr error + }{ + { + name: "block without auxiliary info", + block: StateMachineBlock{}, + }, + { + name: "empty info, first of epoch", + block: blockWithAuxInfo(nil, 0), + }, + { + name: "non-empty info, first of epoch", + block: blockWithAuxInfo([]byte{1}, 0), + expectedHistory: [][]byte{{1}}, + expectedLastSeq: startSeq, + expectedversionID: versionID, + }, + { + name: "empty info pointing back to non-empty info", + block: blockWithAuxInfo(nil, 3), + blocks: map[uint64]StateMachineBlock{ + 3: blockWithAuxInfo([]byte{1}, 0), + }, + expectedHistory: [][]byte{{1}}, + expectedLastSeq: 3, + expectedversionID: versionID, + }, + { + name: "history is ordered from oldest to newest", + block: blockWithAuxInfo([]byte{3}, 5), + blocks: map[uint64]StateMachineBlock{ + 5: blockWithAuxInfo([]byte{2}, 2), + 2: blockWithAuxInfo([]byte{1}, 0), + }, + expectedHistory: [][]byte{{1}, {2}, {3}}, + expectedLastSeq: startSeq, + expectedversionID: versionID, + }, + { + name: "traversal stops at a block without auxiliary info", + block: blockWithAuxInfo([]byte{2}, 4), + blocks: map[uint64]StateMachineBlock{ + 4: {}, + }, + expectedHistory: [][]byte{{2}}, + expectedLastSeq: startSeq, + expectedversionID: versionID, + }, + { + name: "block retrieval failure", + block: blockWithAuxInfo([]byte{2}, 4), + getBlockErr: errRetrieval, + expectedErr: errRetrieval, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + getBlock := func(seq uint64, _ [32]byte) (StateMachineBlock, *common.Finalization, error) { + if tt.getBlockErr != nil { + return StateMachineBlock{}, nil, tt.getBlockErr + } + block, ok := tt.blocks[seq] + require.True(t, ok, "unexpected retrieval of block at sequence %d", seq) + return block, nil, nil + } + + history, gotversionID, err := collectAuxiliaryInfo(tt.block, startSeq, getBlock, 0) + if tt.expectedErr != nil { + require.ErrorIs(t, err, tt.expectedErr) + require.ErrorIs(t, err, errAuxInfoBlockRetrieval) + return + } + require.NoError(t, err) + require.Equal(t, tt.expectedHistory, history.data) + require.Equal(t, tt.expectedLastSeq, history.lastSeq) + require.Equal(t, tt.expectedversionID, gotversionID) + }) + } +} diff --git a/msm/util_test.go b/msm/util_test.go index 5217ff25..c0465a12 100644 --- a/msm/util_test.go +++ b/msm/util_test.go @@ -6,6 +6,8 @@ package metadata import ( "bytes" "context" + "crypto/ecdsa" + "crypto/elliptic" "crypto/rand" "crypto/sha256" "encoding/asn1" @@ -22,6 +24,21 @@ import ( // Test helpers +var ( + testSK *ecdsa.PrivateKey + testPK *ecdsa.PublicKey +) + +func init() { + // We generate this key-pair to test that auxiliary info signs the right. + sk, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + panic(fmt.Sprintf("failed to generate test key: %v", err)) + } + testSK = sk + testPK = &sk.PublicKey +} + type InnerBlock struct { TS time.Time BlockHeight uint64 @@ -83,12 +100,55 @@ func (a approvalsRetriever) Approvals() ValidatorSetApprovals { return a.result } +type signer struct { +} + +func (s *signer) Sign(digest []byte) ([]byte, error) { + return testSK.Sign(rand.Reader, digest, nil) +} + +// signApproval produces a real ECDSA signature over the exact payload the production code signs +// for an epoch-transition approval (assembleApprovalToBeSigned), using the shared test key. The +// signatureVerifier above accepts it. Use this instead of placeholder signature bytes for any +// approval fixture that is expected to pass signature verification. +func signApproval(pChainHeight uint64, auxInfoDigest [32]byte) []byte { + toBeSigned, err := assembleApprovalToBeSigned(pChainHeight, auxInfoDigest) + if err != nil { + panic(fmt.Sprintf("failed to assemble approval payload: %v", err)) + } + sig, err := testSK.Sign(rand.Reader, toBeSigned, nil) + if err != nil { + panic(fmt.Sprintf("failed to sign approval: %v", err)) + } + return sig +} + type signatureVerifier struct { err error } -func (sv *signatureVerifier) VerifySignature(signature []byte, message []byte, publicKey []byte) error { - return sv.err +func (sv *signatureVerifier) VerifySignature(signature []byte, message []byte, _ []byte) error { + if sv.err != nil { + return sv.err + } + if ecdsa.VerifyASN1(testPK, message, signature) { + return nil + } + + // Maybe it's an aggregated signature? + var aggSig aggregatedSignature + _, err := asn1.Unmarshal(signature, &aggSig) + if err != nil { + return fmt.Errorf("invalid signature format: %w", err) + } + + for _, sig := range aggSig.Signatures { + if !ecdsa.VerifyASN1(testPK, message, sig) { + return fmt.Errorf("invalid signature in aggregate") + } + } + + return nil } type signatureAggregator struct { @@ -96,7 +156,7 @@ type signatureAggregator struct { totalWeight uint64 } -type aggregatrdSignature struct { +type aggregatedSignature struct { Signatures [][]byte } @@ -108,9 +168,16 @@ func (sv *signatureAggregator) AppendSignatures(existing []byte, sigs ...[]byte) all := make([][]byte, 0, len(sigs)+1) all = append(all, sigs...) if len(existing) > 0 { - all = append(all, existing) + // existing is itself a marshaled aggregate from a previous round. Flatten it into the + // component signatures instead of nesting the blob, so the aggregate stays a single level + // of individual signatures that signatureVerifier can validate one by one. + var prev aggregatedSignature + if _, err := asn1.Unmarshal(existing, &prev); err != nil { + return nil, err + } + all = append(all, prev.Signatures...) } - return asn1.Marshal(aggregatrdSignature{Signatures: all}) + return asn1.Marshal(aggregatedSignature{Signatures: all}) } func (sv *signatureAggregator) IsQuorum(signers []common.NodeID) bool { @@ -317,7 +384,10 @@ func newStateMachineWithLogger(tb testing.TB, logger common.Logger) (*StateMachi PChainProgressListener: &noOpPChainListener{}, LastNonSimplexInnerBlock: genesisBlock.InnerBlock, MyNodeID: myNodeID[:], - Signer: &testutil.TestSigner{}, + AuxiliaryInfoApp: &voteCountingAuxInfoApp{ + threshold: 2, + }, + Signer: &signer{}, ComputeICMEpoch: func(input ICMEpochInput) ICMEpochInfo { // This is just the ACP-181 implementation from avalanchego var zeroEpoch ICMEpochInfo @@ -378,56 +448,116 @@ func (failingAggregator) IsQuorum([]common.NodeID) bool { return false } -type testBlockStore map[uint64]StateMachineBlock +type noopTestAuxInfoApp struct { +} -func (bs testBlockStore) getBlock(seq uint64, _ [32]byte) (StateMachineBlock, *common.Finalization, error) { - blk, ok := bs[seq] - if !ok { - return StateMachineBlock{}, nil, fmt.Errorf("%w: block %d", common.ErrBlockNotFound, seq) - } - return blk, nil, nil +func (t *noopTestAuxInfoApp) IsLegalAppend(VersionID, NodeBLSMappings, [][]byte, []byte) error { + return nil } -type testVMBlock struct { - bytes []byte - height uint64 +func (t *noopTestAuxInfoApp) IsSufficient(VersionID, NodeBLSMappings, [][]byte) (bool, error) { + return true, nil } -func (b *testVMBlock) Digest() [32]byte { - return sha256.Sum256(b.bytes) +func (t *noopTestAuxInfoApp) Generate(VersionID, NodeBLSMappings, [][]byte) ([]byte, error) { + return nil, nil } -func (b *testVMBlock) Height() uint64 { - return b.height +func (t *noopTestAuxInfoApp) DefaultVersionID() VersionID { + return 1 } -func (b *testVMBlock) Timestamp() time.Time { - return time.Now() +type voteCountingAuxInfoApp struct { + threshold int + randomTape func() []byte } -func (b *testVMBlock) Verify(_ context.Context, _ uint64) error { +func (t *voteCountingAuxInfoApp) IsLegalAppend(_ VersionID, _ NodeBLSMappings, history [][]byte, addition []byte) error { + set := make(map[string]struct{}) + for _, item := range history { + set[string(item)] = struct{}{} + } + if _, exists := set[string(addition)]; exists { + return fmt.Errorf("duplicate addition: %s", string(addition)) + } return nil } -type testSigVerifier struct { - err error +func (t *voteCountingAuxInfoApp) IsSufficient(appID VersionID, nodes NodeBLSMappings, history [][]byte) (bool, error) { + if len(history) == 0 { + return t.threshold == 0, nil + } + addition := history[len(history)-1] + history = history[:len(history)-1] + if err := t.IsLegalAppend(appID, nodes, history, addition); err != nil { + return false, err + } + + history = append(history, addition) + set := make(map[string]struct{}) + for _, item := range history { + set[string(item)] = struct{}{} + } + final := len(set) >= t.threshold + return final, nil } -func (sv *testSigVerifier) VerifySignature(_, _, _ []byte) error { - return sv.err +func (t *voteCountingAuxInfoApp) Generate(VersionID, NodeBLSMappings, [][]byte) ([]byte, error) { + // Simulate a random node voting + if t.randomTape != nil { + return t.randomTape(), nil + } + var nodeID nodeID + rand.Read(nodeID[:]) + return nodeID[:], nil } -type testKeyAggregator struct { - err error +func (t *voteCountingAuxInfoApp) DefaultVersionID() VersionID { + return 1 +} + +// versionRecordingAuxInfoApp behaves like voteCountingAuxInfoApp (a vote is appended to the +// history on every Generate, and the history becomes sufficient once it holds `threshold` +// distinct votes), but it asserts that every method is invoked with expectedVersionID, and its +// DefaultVersionID() returns whatever defaultVersionID is currently set to. A test drives both +// fields directly to prove backward compatibility: once an epoch's auxiliary info carries a +// VersionID, every invocation keeps using that VersionID even after the default changes. +type versionRecordingAuxInfoApp struct { + t *testing.T + threshold int + votes [][]byte + defaultVersionID VersionID + expectedVersionID VersionID +} + +func (a *versionRecordingAuxInfoApp) DefaultVersionID() VersionID { + return a.defaultVersionID } -func (ka *testKeyAggregator) AggregateKeys(keys ...[]byte) ([]byte, error) { - if ka.err != nil { - return nil, ka.err +func (a *versionRecordingAuxInfoApp) IsLegalAppend(versionID VersionID, _ NodeBLSMappings, history [][]byte, addition []byte) error { + require.Equal(a.t, a.expectedVersionID, versionID) + set := make(map[string]struct{}) + for _, item := range history { + set[string(item)] = struct{}{} } - var agg []byte - for _, k := range keys { - agg = append(agg, k...) + if _, exists := set[string(addition)]; exists { + return fmt.Errorf("duplicate addition: %s", string(addition)) } - return agg, nil + return nil +} + +func (a *versionRecordingAuxInfoApp) IsSufficient(versionID VersionID, _ NodeBLSMappings, history [][]byte) (bool, error) { + require.Equal(a.t, a.expectedVersionID, versionID) + set := make(map[string]struct{}) + for _, item := range history { + set[string(item)] = struct{}{} + } + return len(set) >= a.threshold, nil +} + +func (a *versionRecordingAuxInfoApp) Generate(versionID VersionID, _ NodeBLSMappings, _ [][]byte) ([]byte, error) { + require.Equal(a.t, a.expectedVersionID, versionID) + next := a.votes[0] + a.votes = a.votes[1:] + return next, nil }