diff --git a/msm/approvals.go b/msm/approvals.go index 566ba409..5c6fe3cb 100644 --- a/msm/approvals.go +++ b/msm/approvals.go @@ -4,15 +4,18 @@ package metadata import ( - "encoding/asn1" - "encoding/binary" "fmt" "github.com/ava-labs/simplex/common" "go.uber.org/zap" ) -type approvalsByPChainHeight map[uint64]*approvalAndTimestamp +type approvalKey struct { + pChainHeight uint64 + auxInfoDigest [32]byte +} + +type approvalsByPChainHeightAndAuxInfoDigest map[approvalKey]*approvalAndTimestamp type approvalAndTimestamp struct { ValidatorSetApproval @@ -24,7 +27,7 @@ type ApprovalStore struct { validators NodeBLSMappings logger common.Logger pkByNodeID map[nodeID][]byte - approvalsByNodes map[nodeID]approvalsByPChainHeight + approvalsByNodes map[nodeID]approvalsByPChainHeightAndAuxInfoDigest storedCount int } @@ -34,9 +37,9 @@ func NewApprovalStore(signatureVerifier SignatureVerifier, validators NodeBLSMap pkByNodeID[vdr.NodeID] = vdr.BLSKey } - approvalsByNodes := make(map[nodeID]approvalsByPChainHeight, len(validators)) + approvalsByNodes := make(map[nodeID]approvalsByPChainHeightAndAuxInfoDigest, len(validators)) for _, vdr := range validators { - approvalsByNodes[vdr.NodeID] = make(approvalsByPChainHeight) + approvalsByNodes[vdr.NodeID] = make(approvalsByPChainHeightAndAuxInfoDigest) } return &ApprovalStore{ @@ -82,9 +85,14 @@ func (as *ApprovalStore) HandleApproval(approval *ValidatorSetApproval, timestam return nil } + key := approvalKey{ + pChainHeight: approval.PChainHeight, + auxInfoDigest: approval.AuxInfoDigest, + } + // Store the approval. - oldApproval := as.approvalsByNodes[approval.NodeID][approval.PChainHeight] - as.approvalsByNodes[approval.NodeID][approval.PChainHeight] = &approvalAndTimestamp{ + oldApproval := as.approvalsByNodes[approval.NodeID][key] + as.approvalsByNodes[approval.NodeID][key] = &approvalAndTimestamp{ ValidatorSetApproval: *approval, Timestamp: timestamp, } @@ -113,22 +121,22 @@ func (as *ApprovalStore) maybePruneOldApprovals(approval *ValidatorSetApproval) } if oldestApproval != nil { + key := approvalKey{ + pChainHeight: oldestApproval.PChainHeight, + auxInfoDigest: oldestApproval.AuxInfoDigest, + } + as.logger.Debug("Deleting old approval from node", zap.String("nodeID", fmt.Sprintf("%x", oldestApproval.NodeID)), zap.String("oldestApprovalPChainHeight", fmt.Sprintf("%d", oldestApproval.PChainHeight)), zap.Uint64("oldestApprovalTimestamp", oldestApproval.Timestamp)) - delete(as.approvalsByNodes[approval.NodeID], oldestApproval.PChainHeight) + delete(as.approvalsByNodes[approval.NodeID], key) as.storedCount-- } } func (as *ApprovalStore) checkApprovalSignature(approval *ValidatorSetApproval, pk []byte) error { - pChainHeight := approval.PChainHeight - pChainHeightBuff := make([]byte, 8) - binary.BigEndian.PutUint64(pChainHeightBuff, pChainHeight) - - signedMsg := common.SignedMessage{Payload: pChainHeightBuff, Context: signatureContext} - toBeSigned, err := asn1.Marshal(signedMsg) + toBeSigned, err := assembleApprovalToBeSigned(approval.PChainHeight, approval.AuxInfoDigest) if err != nil { return err } @@ -146,7 +154,13 @@ func (as *ApprovalStore) approvalExistsAndUpToDate(approval *ValidatorSetApprova if as.approvalsByNodes[approval.NodeID] == nil { return false } - existingApproval := as.approvalsByNodes[approval.NodeID][approval.PChainHeight] + + key := approvalKey{ + pChainHeight: approval.PChainHeight, + auxInfoDigest: approval.AuxInfoDigest, + } + + existingApproval := as.approvalsByNodes[approval.NodeID][key] if existingApproval == nil { return false } diff --git a/msm/approvals_test.go b/msm/approvals_test.go index c457ca5f..c24e5f37 100644 --- a/msm/approvals_test.go +++ b/msm/approvals_test.go @@ -61,7 +61,7 @@ func TestApprovalStoreHandleApproval(t *testing.T) { validators: 3, sigErr: errors.New("bad sig"), approvals: []approvalAndTimestamp{ - {ValidatorSetApproval{NodeID: makeNodeID(1), PChainHeight: 1, Signature: []byte{0xAA}}, 1}, + {ValidatorSetApproval{NodeID: makeNodeID(1), PChainHeight: 1, Signature: signApproval(1, [32]byte{})}, 1}, }, verify: func(t *testing.T, as *ApprovalStore, _ []approvalAndTimestamp) { require.Empty(t, as.Approvals()) @@ -74,7 +74,7 @@ func TestApprovalStoreHandleApproval(t *testing.T) { name: "valid approval is stored", validators: 3, approvals: []approvalAndTimestamp{ - {ValidatorSetApproval{NodeID: makeNodeID(1), PChainHeight: 7, Signature: []byte{0x01}}, 100}, + {ValidatorSetApproval{NodeID: makeNodeID(1), PChainHeight: 7, Signature: signApproval(7, [32]byte{})}, 100}, }, verify: func(t *testing.T, as *ApprovalStore, sent []approvalAndTimestamp) { got := as.Approvals() @@ -89,8 +89,8 @@ func TestApprovalStoreHandleApproval(t *testing.T) { name: "duplicate approval with same timestamp is a no-op", validators: 3, approvals: []approvalAndTimestamp{ - {ValidatorSetApproval{NodeID: makeNodeID(1), PChainHeight: 7, Signature: []byte{0x01}}, 100}, - {ValidatorSetApproval{NodeID: makeNodeID(1), PChainHeight: 7, Signature: []byte{0x01}}, 100}, + {ValidatorSetApproval{NodeID: makeNodeID(1), PChainHeight: 7, Signature: signApproval(7, [32]byte{})}, 100}, + {ValidatorSetApproval{NodeID: makeNodeID(1), PChainHeight: 7, Signature: signApproval(7, [32]byte{})}, 100}, }, verify: func(t *testing.T, as *ApprovalStore, _ []approvalAndTimestamp) { require.Len(t, as.Approvals(), 1) @@ -103,8 +103,8 @@ func TestApprovalStoreHandleApproval(t *testing.T) { name: "older timestamp is ignored", validators: 3, approvals: []approvalAndTimestamp{ - {ValidatorSetApproval{NodeID: makeNodeID(1), PChainHeight: 7, Signature: []byte{0x02}}, 200}, // newer, stored first - {ValidatorSetApproval{NodeID: makeNodeID(1), PChainHeight: 7, Signature: []byte{0x01}}, 100}, // older, dropped + {ValidatorSetApproval{NodeID: makeNodeID(1), PChainHeight: 7, Signature: signApproval(7, [32]byte{})}, 200}, // newer, stored first + {ValidatorSetApproval{NodeID: makeNodeID(1), PChainHeight: 7, Signature: signApproval(7, [32]byte{})}, 100}, // older, dropped }, verify: func(t *testing.T, as *ApprovalStore, sent []approvalAndTimestamp) { got := as.Approvals() @@ -119,8 +119,8 @@ func TestApprovalStoreHandleApproval(t *testing.T) { name: "newer timestamp replaces", validators: 3, approvals: []approvalAndTimestamp{ - {ValidatorSetApproval{NodeID: makeNodeID(1), PChainHeight: 7, Signature: []byte{0x01}}, 100}, // older, stored first - {ValidatorSetApproval{NodeID: makeNodeID(1), PChainHeight: 7, Signature: []byte{0x02}}, 200}, // newer, replaces + {ValidatorSetApproval{NodeID: makeNodeID(1), PChainHeight: 7, Signature: signApproval(7, [32]byte{})}, 100}, // older, stored first + {ValidatorSetApproval{NodeID: makeNodeID(1), PChainHeight: 7, Signature: signApproval(7, [32]byte{})}, 200}, // newer, replaces }, verify: func(t *testing.T, as *ApprovalStore, sent []approvalAndTimestamp) { got := as.Approvals() @@ -136,12 +136,12 @@ func TestApprovalStoreHandleApproval(t *testing.T) { validators: 3, // 3 validators x 2 heights, with timestamp == i*10 + h and Signature == {i} per validator. approvals: []approvalAndTimestamp{ - {ValidatorSetApproval{NodeID: makeNodeID(1), PChainHeight: 1, Signature: []byte{0}}, 1}, - {ValidatorSetApproval{NodeID: makeNodeID(1), PChainHeight: 2, Signature: []byte{0}}, 2}, - {ValidatorSetApproval{NodeID: makeNodeID(2), PChainHeight: 1, Signature: []byte{1}}, 11}, - {ValidatorSetApproval{NodeID: makeNodeID(2), PChainHeight: 2, Signature: []byte{1}}, 12}, - {ValidatorSetApproval{NodeID: makeNodeID(3), PChainHeight: 1, Signature: []byte{2}}, 21}, - {ValidatorSetApproval{NodeID: makeNodeID(3), PChainHeight: 2, Signature: []byte{2}}, 22}, + {ValidatorSetApproval{NodeID: makeNodeID(1), PChainHeight: 1, Signature: signApproval(1, [32]byte{})}, 1}, + {ValidatorSetApproval{NodeID: makeNodeID(1), PChainHeight: 2, Signature: signApproval(2, [32]byte{})}, 2}, + {ValidatorSetApproval{NodeID: makeNodeID(2), PChainHeight: 1, Signature: signApproval(1, [32]byte{})}, 11}, + {ValidatorSetApproval{NodeID: makeNodeID(2), PChainHeight: 2, Signature: signApproval(2, [32]byte{})}, 12}, + {ValidatorSetApproval{NodeID: makeNodeID(3), PChainHeight: 1, Signature: signApproval(1, [32]byte{})}, 21}, + {ValidatorSetApproval{NodeID: makeNodeID(3), PChainHeight: 2, Signature: signApproval(2, [32]byte{})}, 22}, }, verify: func(t *testing.T, as *ApprovalStore, sent []approvalAndTimestamp) { require.Len(t, as.Approvals(), len(sent)) @@ -154,9 +154,9 @@ func TestApprovalStoreHandleApproval(t *testing.T) { name: "prunes oldest when over cap", validators: 2, approvals: []approvalAndTimestamp{ - {ValidatorSetApproval{NodeID: makeNodeID(1), PChainHeight: 1, Signature: []byte{1}}, 10}, - {ValidatorSetApproval{NodeID: makeNodeID(1), PChainHeight: 2, Signature: []byte{2}}, 20}, - {ValidatorSetApproval{NodeID: makeNodeID(1), PChainHeight: 3, Signature: []byte{3}}, 30}, + {ValidatorSetApproval{NodeID: makeNodeID(1), PChainHeight: 1, Signature: signApproval(1, [32]byte{})}, 10}, + {ValidatorSetApproval{NodeID: makeNodeID(1), PChainHeight: 2, Signature: signApproval(2, [32]byte{})}, 20}, + {ValidatorSetApproval{NodeID: makeNodeID(1), PChainHeight: 3, Signature: signApproval(3, [32]byte{})}, 30}, }, verify: func(t *testing.T, as *ApprovalStore, _ []approvalAndTimestamp) { got := as.Approvals() @@ -174,6 +174,23 @@ func TestApprovalStoreHandleApproval(t *testing.T) { require.True(t, heights[3]) }, }, + { + // Verifies that the store keys on (NodeID, PChainHeight, AuxInfoDigest): + // two approvals from the same node at the same P-chain height but with different + // auxiliary info digests are kept as independent entries. + name: "same height different aux info digest coexist", + validators: 3, + approvals: []approvalAndTimestamp{ + {ValidatorSetApproval{NodeID: makeNodeID(1), PChainHeight: 7, AuxInfoDigest: [32]byte{0xAA}, Signature: signApproval(7, [32]byte{0xAA})}, 100}, + {ValidatorSetApproval{NodeID: makeNodeID(1), PChainHeight: 7, AuxInfoDigest: [32]byte{0xBB}, Signature: signApproval(7, [32]byte{0xBB})}, 100}, + }, + verify: func(t *testing.T, as *ApprovalStore, sent []approvalAndTimestamp) { + got := as.Approvals() + require.Len(t, got, 2) + require.Equal(t, 2, as.storedCount) + require.ElementsMatch(t, []ValidatorSetApproval{sent[0].ValidatorSetApproval, sent[1].ValidatorSetApproval}, got) + }, + }, { // Verifies that an approval with the maximum uint64 timestamp is stored, // and that a subsequent approval at the same (NodeID, PChainHeight) with any @@ -181,8 +198,8 @@ func TestApprovalStoreHandleApproval(t *testing.T) { name: "max uint64 timestamp is kept over any smaller timestamp", validators: 3, approvals: []approvalAndTimestamp{ - {ValidatorSetApproval{NodeID: makeNodeID(1), PChainHeight: 7, Signature: []byte{0xFF}}, math.MaxUint64}, // maxTS - {ValidatorSetApproval{NodeID: makeNodeID(1), PChainHeight: 7, Signature: []byte{0x01}}, math.MaxUint64 - 1}, // older + {ValidatorSetApproval{NodeID: makeNodeID(1), PChainHeight: 7, Signature: signApproval(7, [32]byte{})}, math.MaxUint64}, // maxTS + {ValidatorSetApproval{NodeID: makeNodeID(1), PChainHeight: 7, Signature: signApproval(7, [32]byte{})}, math.MaxUint64 - 1}, // older }, verify: func(t *testing.T, as *ApprovalStore, sent []approvalAndTimestamp) { got := as.Approvals() @@ -213,11 +230,11 @@ func TestApprovalStoreHandleApprovalStoredCountStaysConsistent(t *testing.T) { node := vdrs[0].NodeID for _, a := range []approvalAndTimestamp{ - {ValidatorSetApproval{NodeID: node, PChainHeight: 1}, 10}, - {ValidatorSetApproval{NodeID: node, PChainHeight: 1}, 10}, // duplicate - {ValidatorSetApproval{NodeID: node, PChainHeight: 1}, 20}, // replaces - {ValidatorSetApproval{NodeID: node, PChainHeight: 2}, 30}, // new height - {ValidatorSetApproval{NodeID: node, PChainHeight: 3}, 40}, // triggers prune + {ValidatorSetApproval{NodeID: node, PChainHeight: 1, Signature: signApproval(1, [32]byte{})}, 10}, + {ValidatorSetApproval{NodeID: node, PChainHeight: 1, Signature: signApproval(1, [32]byte{})}, 10}, // duplicate + {ValidatorSetApproval{NodeID: node, PChainHeight: 1, Signature: signApproval(1, [32]byte{})}, 20}, // replaces + {ValidatorSetApproval{NodeID: node, PChainHeight: 2, Signature: signApproval(2, [32]byte{})}, 30}, // new height + {ValidatorSetApproval{NodeID: node, PChainHeight: 3, Signature: signApproval(3, [32]byte{})}, 40}, // triggers prune } { require.NoError(t, as.HandleApproval(&a.ValidatorSetApproval, a.Timestamp)) require.Len(t, as.Approvals(), as.storedCount) @@ -235,12 +252,14 @@ func TestApprovalStoreHandleApprovalPruningIsPerNode(t *testing.T) { require.NoError(t, as.HandleApproval(&ValidatorSetApproval{ NodeID: vdrs[1].NodeID, PChainHeight: 1, + Signature: signApproval(1, [32]byte{}), }, 100)) for h := uint64(1); h <= 10; h++ { require.NoError(t, as.HandleApproval(&ValidatorSetApproval{ NodeID: vdrs[0].NodeID, PChainHeight: h, + Signature: signApproval(h, [32]byte{}), }, h)) } require.Len(t, as.Approvals().UniqueByNodeID(), 2) diff --git a/msm/encoding.canoto.go b/msm/encoding.canoto.go index d2651feb..d76a53e7 100644 --- a/msm/encoding.canoto.go +++ b/msm/encoding.canoto.go @@ -29,6 +29,7 @@ const ( canotoNumber_StateMachineMetadata__PChainHeight = 4 canotoNumber_StateMachineMetadata__Timestamp = 5 canotoNumber_StateMachineMetadata__ICMEpochInfo = 6 + canotoNumber_StateMachineMetadata__AuxiliaryInfo = 7 canotoTag_StateMachineMetadata__SimplexEpochInfo = "\x0a" // canoto.Tag(canotoNumber_StateMachineMetadata__SimplexEpochInfo, canoto.Len) canotoTag_StateMachineMetadata__SimplexProtocolMetadata = "\x12" // canoto.Tag(canotoNumber_StateMachineMetadata__SimplexProtocolMetadata, canoto.Len) @@ -36,6 +37,7 @@ const ( canotoTag_StateMachineMetadata__PChainHeight = "\x20" // canoto.Tag(canotoNumber_StateMachineMetadata__PChainHeight, canoto.Varint) canotoTag_StateMachineMetadata__Timestamp = "\x28" // canoto.Tag(canotoNumber_StateMachineMetadata__Timestamp, canoto.Varint) canotoTag_StateMachineMetadata__ICMEpochInfo = "\x32" // canoto.Tag(canotoNumber_StateMachineMetadata__ICMEpochInfo, canoto.Len) + canotoTag_StateMachineMetadata__AuxiliaryInfo = "\x3a" // canoto.Tag(canotoNumber_StateMachineMetadata__AuxiliaryInfo, canoto.Len) ) type canotoData_StateMachineMetadata struct { @@ -93,6 +95,16 @@ func (*StateMachineMetadata) CanotoSpec(types ...reflect.Type) *canoto.Spec { /*Pointer: */ false, /*types: */ types, ), + canoto.FieldTypeFromField( + /*type inference:*/ (zero.AuxiliaryInfo), + /*FieldNumber: */ canotoNumber_StateMachineMetadata__AuxiliaryInfo, + /*Name: */ "AuxiliaryInfo", + /*FixedLength: */ 0, + /*Repeated: */ false, + /*OneOf: */ "", + /*Pointer: */ true, + /*types: */ types, + ), }, } s.CalculateCanotoCache() @@ -223,6 +235,28 @@ func (c *StateMachineMetadata) UnmarshalCanotoFrom(r canoto.Reader) error { return err } r.B = remainingBytes + case canotoNumber_StateMachineMetadata__AuxiliaryInfo: + if wireType != canoto.Len { + return canoto.ErrUnexpectedWireType + } + + // Read the bytes for the field. + originalUnsafe := r.Unsafe + r.Unsafe = true + var msgBytes []byte + if err := canoto.ReadBytes(&r, &msgBytes); err != nil { + return err + } + r.Unsafe = originalUnsafe + + // Unmarshal the field from the bytes. + remainingBytes := r.B + r.B = msgBytes + c.AuxiliaryInfo = canoto.MakePointer(c.AuxiliaryInfo) + if err := (c.AuxiliaryInfo).UnmarshalCanotoFrom(r); err != nil { + return err + } + r.B = remainingBytes default: return canoto.ErrUnknownField } @@ -246,6 +280,9 @@ func (c *StateMachineMetadata) ValidCanoto() bool { if !(&c.ICMEpochInfo).ValidCanoto() { return false } + if c.AuxiliaryInfo != nil && !(c.AuxiliaryInfo).ValidCanoto() { + return false + } return true } @@ -275,6 +312,11 @@ func (c *StateMachineMetadata) CalculateCanotoCache() { if fieldSize := (&c.ICMEpochInfo).CachedCanotoSize(); fieldSize != 0 { size += uint64(len(canotoTag_StateMachineMetadata__ICMEpochInfo)) + canoto.SizeUint(fieldSize) + fieldSize } + if c.AuxiliaryInfo != nil { + (c.AuxiliaryInfo).CalculateCanotoCache() + fieldSize := (c.AuxiliaryInfo).CachedCanotoSize() + size += uint64(len(canotoTag_StateMachineMetadata__AuxiliaryInfo)) + canoto.SizeUint(fieldSize) + fieldSize + } atomic.StoreUint64(&c.canotoData.size, size) } @@ -339,6 +381,12 @@ func (c *StateMachineMetadata) MarshalCanotoInto(w canoto.Writer) canoto.Writer canoto.AppendUint(&w, fieldSize) w = (&c.ICMEpochInfo).MarshalCanotoInto(w) } + if c.AuxiliaryInfo != nil { + fieldSize := (c.AuxiliaryInfo).CachedCanotoSize() + canoto.Append(&w, canotoTag_StateMachineMetadata__AuxiliaryInfo) + canoto.AppendUint(&w, fieldSize) + w = (c.AuxiliaryInfo).MarshalCanotoInto(w) + } return w } @@ -539,6 +587,203 @@ func (c *ICMEpochInfo) MarshalCanotoInto(w canoto.Writer) canoto.Writer { return w } +const ( + canotoNumber_AuxiliaryInfo__Info = 1 + canotoNumber_AuxiliaryInfo__PrevAuxInfoSeq = 2 + canotoNumber_AuxiliaryInfo__VersionID = 3 + + canotoTag_AuxiliaryInfo__Info = "\x0a" // canoto.Tag(canotoNumber_AuxiliaryInfo__Info, canoto.Len) + canotoTag_AuxiliaryInfo__PrevAuxInfoSeq = "\x10" // canoto.Tag(canotoNumber_AuxiliaryInfo__PrevAuxInfoSeq, canoto.Varint) + canotoTag_AuxiliaryInfo__VersionID = "\x18" // canoto.Tag(canotoNumber_AuxiliaryInfo__VersionID, canoto.Varint) +) + +type canotoData_AuxiliaryInfo struct { + size uint64 +} + +// CanotoSpec returns the specification of this canoto message. +func (*AuxiliaryInfo) CanotoSpec(...reflect.Type) *canoto.Spec { + var zero AuxiliaryInfo + s := &canoto.Spec{ + Name: "AuxiliaryInfo", + Fields: []canoto.FieldType{ + { + FieldNumber: canotoNumber_AuxiliaryInfo__Info, + Name: "Info", + OneOf: "", + TypeBytes: true, + }, + { + FieldNumber: canotoNumber_AuxiliaryInfo__PrevAuxInfoSeq, + Name: "PrevAuxInfoSeq", + OneOf: "", + TypeUint: canoto.SizeOf(zero.PrevAuxInfoSeq), + }, + { + FieldNumber: canotoNumber_AuxiliaryInfo__VersionID, + Name: "VersionID", + OneOf: "", + TypeUint: canoto.SizeOf(zero.VersionID), + }, + }, + } + s.CalculateCanotoCache() + return s +} + +// UnmarshalCanoto unmarshals a Canoto-encoded byte slice into the struct. +// +// During parsing, the canoto cache is saved. +func (c *AuxiliaryInfo) UnmarshalCanoto(bytes []byte) error { + r := canoto.Reader{ + B: bytes, + } + return c.UnmarshalCanotoFrom(r) +} + +// UnmarshalCanotoFrom populates the struct from a [canoto.Reader]. Most users +// should just use UnmarshalCanoto. +// +// During parsing, the canoto cache is saved. +// +// This function enables configuration of reader options. +func (c *AuxiliaryInfo) UnmarshalCanotoFrom(r canoto.Reader) error { + // Zero the struct before unmarshaling. + *c = AuxiliaryInfo{} + atomic.StoreUint64(&c.canotoData.size, uint64(len(r.B))) + + var minField uint32 + for canoto.HasNext(&r) { + field, wireType, err := canoto.ReadTag(&r) + if err != nil { + return err + } + if field < minField { + return canoto.ErrInvalidFieldOrder + } + + switch field { + case canotoNumber_AuxiliaryInfo__Info: + if wireType != canoto.Len { + return canoto.ErrUnexpectedWireType + } + + if err := canoto.ReadBytes(&r, &c.Info); err != nil { + return err + } + if len(c.Info) == 0 { + return canoto.ErrZeroValue + } + case canotoNumber_AuxiliaryInfo__PrevAuxInfoSeq: + if wireType != canoto.Varint { + return canoto.ErrUnexpectedWireType + } + + if err := canoto.ReadUint(&r, &c.PrevAuxInfoSeq); err != nil { + return err + } + if canoto.IsZero(c.PrevAuxInfoSeq) { + return canoto.ErrZeroValue + } + case canotoNumber_AuxiliaryInfo__VersionID: + if wireType != canoto.Varint { + return canoto.ErrUnexpectedWireType + } + + if err := canoto.ReadUint(&r, &c.VersionID); err != nil { + return err + } + if canoto.IsZero(c.VersionID) { + return canoto.ErrZeroValue + } + default: + return canoto.ErrUnknownField + } + + minField = field + 1 + } + return nil +} + +// ValidCanoto validates that the struct can be correctly marshaled into the +// Canoto format. +// +// Specifically, ValidCanoto ensures: +// 1. All OneOfs are specified at most once. +// 2. All strings are valid utf-8. +// 3. All custom fields are ValidCanoto. +func (c *AuxiliaryInfo) ValidCanoto() bool { + return true +} + +// CalculateCanotoCache populates size and OneOf caches based on the current +// values in the struct. +// +// It is not safe to copy this struct concurrently. +func (c *AuxiliaryInfo) CalculateCanotoCache() { + var size uint64 + if len(c.Info) != 0 { + size += uint64(len(canotoTag_AuxiliaryInfo__Info)) + canoto.SizeBytes(c.Info) + } + if !canoto.IsZero(c.PrevAuxInfoSeq) { + size += uint64(len(canotoTag_AuxiliaryInfo__PrevAuxInfoSeq)) + canoto.SizeUint(c.PrevAuxInfoSeq) + } + if !canoto.IsZero(c.VersionID) { + size += uint64(len(canotoTag_AuxiliaryInfo__VersionID)) + canoto.SizeUint(c.VersionID) + } + atomic.StoreUint64(&c.canotoData.size, size) +} + +// CachedCanotoSize returns the previously calculated size of the Canoto +// representation from CalculateCanotoCache. +// +// If CalculateCanotoCache has not yet been called, it will return 0. +// +// If the struct has been modified since the last call to CalculateCanotoCache, +// the returned size may be incorrect. +func (c *AuxiliaryInfo) CachedCanotoSize() uint64 { + return atomic.LoadUint64(&c.canotoData.size) +} + +// MarshalCanoto returns the Canoto representation of this struct. +// +// It is assumed that this struct is ValidCanoto. +// +// It is not safe to copy this struct concurrently. +func (c *AuxiliaryInfo) MarshalCanoto() []byte { + c.CalculateCanotoCache() + w := canoto.Writer{ + B: make([]byte, 0, c.CachedCanotoSize()), + } + w = c.MarshalCanotoInto(w) + return w.B +} + +// MarshalCanotoInto writes the struct into a [canoto.Writer] and returns the +// resulting [canoto.Writer]. Most users should just use MarshalCanoto. +// +// It is assumed that CalculateCanotoCache has been called since the last +// modification to this struct. +// +// It is assumed that this struct is ValidCanoto. +// +// It is not safe to copy this struct concurrently. +func (c *AuxiliaryInfo) MarshalCanotoInto(w canoto.Writer) canoto.Writer { + if len(c.Info) != 0 { + canoto.Append(&w, canotoTag_AuxiliaryInfo__Info) + canoto.AppendBytes(&w, c.Info) + } + if !canoto.IsZero(c.PrevAuxInfoSeq) { + canoto.Append(&w, canotoTag_AuxiliaryInfo__PrevAuxInfoSeq) + canoto.AppendUint(&w, c.PrevAuxInfoSeq) + } + if !canoto.IsZero(c.VersionID) { + canoto.Append(&w, canotoTag_AuxiliaryInfo__VersionID) + canoto.AppendUint(&w, c.VersionID) + } + return w +} + const ( canotoNumber_SimplexEpochInfo__PChainReferenceHeight = 1 canotoNumber_SimplexEpochInfo__EpochNumber = 2 @@ -1712,9 +1957,9 @@ func (*ValidatorSetApproval) CanotoSpec(...reflect.Type) *canoto.Spec { }, { FieldNumber: canotoNumber_ValidatorSetApproval__AuxInfoSeqDigest, - Name: "AuxInfoSeqDigest", + Name: "AuxInfoDigest", OneOf: "", - TypeFixedBytes: uint64(len(zero.AuxInfoSeqDigest)), + TypeFixedBytes: uint64(len(zero.AuxInfoDigest)), }, { FieldNumber: canotoNumber_ValidatorSetApproval__PChainHeight, @@ -1797,7 +2042,7 @@ func (c *ValidatorSetApproval) UnmarshalCanotoFrom(r canoto.Reader) error { } const ( - expectedLength = len(c.AuxInfoSeqDigest) + expectedLength = len(c.AuxInfoDigest) expectedLengthUint64 = uint64(expectedLength) ) var length uint64 @@ -1811,8 +2056,8 @@ func (c *ValidatorSetApproval) UnmarshalCanotoFrom(r canoto.Reader) error { return io.ErrUnexpectedEOF } - copy((&c.AuxInfoSeqDigest)[:], r.B) - if canoto.IsZero(c.AuxInfoSeqDigest) { + copy((&c.AuxInfoDigest)[:], r.B) + if canoto.IsZero(c.AuxInfoDigest) { return canoto.ErrZeroValue } r.B = r.B[expectedLength:] @@ -1867,8 +2112,8 @@ func (c *ValidatorSetApproval) CalculateCanotoCache() { if !canoto.IsZero(c.NodeID) { size += uint64(len(canotoTag_ValidatorSetApproval__NodeID)) + canoto.SizeBytes((&c.NodeID)[:]) } - if !canoto.IsZero(c.AuxInfoSeqDigest) { - size += uint64(len(canotoTag_ValidatorSetApproval__AuxInfoSeqDigest)) + canoto.SizeBytes((&c.AuxInfoSeqDigest)[:]) + if !canoto.IsZero(c.AuxInfoDigest) { + size += uint64(len(canotoTag_ValidatorSetApproval__AuxInfoSeqDigest)) + canoto.SizeBytes((&c.AuxInfoDigest)[:]) } if !canoto.IsZero(c.PChainHeight) { size += uint64(len(canotoTag_ValidatorSetApproval__PChainHeight)) + canoto.SizeUint(c.PChainHeight) @@ -1918,9 +2163,9 @@ func (c *ValidatorSetApproval) MarshalCanotoInto(w canoto.Writer) canoto.Writer canoto.Append(&w, canotoTag_ValidatorSetApproval__NodeID) canoto.AppendBytes(&w, (&c.NodeID)[:]) } - if !canoto.IsZero(c.AuxInfoSeqDigest) { + if !canoto.IsZero(c.AuxInfoDigest) { canoto.Append(&w, canotoTag_ValidatorSetApproval__AuxInfoSeqDigest) - canoto.AppendBytes(&w, (&c.AuxInfoSeqDigest)[:]) + canoto.AppendBytes(&w, (&c.AuxInfoDigest)[:]) } if !canoto.IsZero(c.PChainHeight) { canoto.Append(&w, canotoTag_ValidatorSetApproval__PChainHeight) diff --git a/msm/encoding.go b/msm/encoding.go index 2a08d4ed..9659e5d6 100644 --- a/msm/encoding.go +++ b/msm/encoding.go @@ -32,6 +32,9 @@ type StateMachineMetadata struct { Timestamp uint64 `canoto:"uint,5"` // ICMEpochInfo is the metadata that the StateMachine uses for ICM epoching. ICMEpochInfo ICMEpochInfo `canoto:"value,6"` + // AuxiliaryInfo is application-specific information that the StateMachine doesn't need to understand, + // but can be used by applications that care about epoch changes, such as threshold distributed public key generation. + AuxiliaryInfo *AuxiliaryInfo `canoto:"pointer,7"` canotoData canotoData_StateMachineMetadata } @@ -60,6 +63,40 @@ func (ei *ICMEpochInfo) Equal(other *ICMEpochInfo) bool { return ei.EpochStartTime == other.EpochStartTime && ei.EpochNumber == other.EpochNumber && ei.PChainEpochHeight == other.PChainEpochHeight } +// VersionID is an identifier for applications that care about epoch changes. +type VersionID uint32 + +// AuxiliaryInfo defines application-specific information for applications that might care about epoch change, +// such as threshold distributed public key generation. +type AuxiliaryInfo struct { + // Info is opaque bytes that can be used by applications to encode any information that describes + // the current state for the application. + Info []byte `canoto:"bytes,1"` + // PrevAuxInfoSeq is a sequence number that applications can use to find previous AuxiliaryInfo in the chain. + // It is zero if this is the first AuxiliaryInfo for this epoch. + PrevAuxInfoSeq uint64 `canoto:"uint,2"` + // VersionID is an identifier that identifies the application. + // Can be used for backward-compatibility and upgrade purposes. + VersionID VersionID `canoto:"uint,3"` + + canotoData canotoData_AuxiliaryInfo +} + +func (ai *AuxiliaryInfo) IsZero() bool { + var zero AuxiliaryInfo + return ai.Equal(&zero) +} + +func (ai *AuxiliaryInfo) Equal(a *AuxiliaryInfo) bool { + if ai == nil { + return a == nil + } + if a == nil { + return ai == nil + } + return bytes.Equal(ai.Info, a.Info) && ai.PrevAuxInfoSeq == a.PrevAuxInfoSeq && ai.VersionID == a.VersionID +} + // SimplexEpochInfo is metadata used by the StateMachine. type SimplexEpochInfo struct { // PChainReferenceHeight is the P-Chain height that the StateMachine uses as a reference for the current epoch. @@ -321,10 +358,10 @@ func (nbms NodeBLSMappings) Equal(other NodeBLSMappings) bool { } type ValidatorSetApproval struct { - NodeID nodeID `canoto:"fixed bytes,1"` - AuxInfoSeqDigest [32]byte `canoto:"fixed bytes,2"` - PChainHeight uint64 `canoto:"uint,3"` - Signature []byte `canoto:"bytes,4"` + NodeID nodeID `canoto:"fixed bytes,1"` + AuxInfoDigest [32]byte `canoto:"fixed bytes,2"` + PChainHeight uint64 `canoto:"uint,3"` + Signature []byte `canoto:"bytes,4"` canotoData canotoData_ValidatorSetApproval } diff --git a/msm/fake_node_test.go b/msm/fake_node_test.go index f55a1900..9e8c69b9 100644 --- a/msm/fake_node_test.go +++ b/msm/fake_node_test.go @@ -6,6 +6,7 @@ package metadata import ( "context" "crypto/rand" + "crypto/sha256" "fmt" "sync/atomic" "testing" @@ -15,6 +16,8 @@ import ( "github.com/stretchr/testify/require" ) +var emptyAuxInfoDigest = sha256.Sum256(nil) + func TestFakeNodeEpochChangesDespiteEmptyMempool(t *testing.T) { validatorSetRetriever := validatorSetRetriever{ resultMap: map[uint64]NodeBLSMappings{ @@ -30,6 +33,7 @@ func TestFakeNodeEpochChangesDespiteEmptyMempool(t *testing.T) { var pChainHeight atomic.Uint64 pChainHeight.Store(100) node := newFakeNode(t) + node.sm.AuxiliaryInfoApp = &noopTestAuxInfoApp{} node.sm.GetValidatorSet = validatorSetRetriever.getValidatorSet node.sm.GetPChainHeightForProposing = func() uint64 { return pChainHeight.Load() @@ -57,11 +61,11 @@ func TestFakeNodeEpochChangesDespiteEmptyMempool(t *testing.T) { } if flipCoin() { node.sm.ApprovalsRetriever = &approvalsRetriever{ - result: []ValidatorSetApproval{{NodeID: [20]byte{1}, PChainHeight: 200, Signature: []byte{1}, AuxInfoSeqDigest: [32]byte{}}}, + result: []ValidatorSetApproval{{NodeID: [20]byte{1}, PChainHeight: 200, Signature: signApproval(200, emptyAuxInfoDigest), AuxInfoDigest: emptyAuxInfoDigest}}, } } else { node.sm.ApprovalsRetriever = &approvalsRetriever{ - result: []ValidatorSetApproval{{NodeID: [20]byte{2}, PChainHeight: 200, Signature: []byte{2}, AuxInfoSeqDigest: [32]byte{}}}, + result: []ValidatorSetApproval{{NodeID: [20]byte{2}, PChainHeight: 200, Signature: signApproval(200, emptyAuxInfoDigest), AuxInfoDigest: emptyAuxInfoDigest}}, } } @@ -86,6 +90,7 @@ func TestFakeNode(t *testing.T) { var pChainHeight atomic.Uint64 pChainHeight.Store(100) node := newFakeNode(t) + node.sm.AuxiliaryInfoApp = &noopTestAuxInfoApp{} node.sm.GetValidatorSet = validatorSetRetriever.getValidatorSet node.sm.GetPChainHeightForProposing = func() uint64 { return pChainHeight.Load() @@ -107,11 +112,11 @@ func TestFakeNode(t *testing.T) { node.act() if flipCoin() { node.sm.ApprovalsRetriever = &approvalsRetriever{ - result: []ValidatorSetApproval{{NodeID: [20]byte{1}, PChainHeight: 200, Signature: []byte{1}, AuxInfoSeqDigest: [32]byte{}}}, + result: []ValidatorSetApproval{{NodeID: [20]byte{1}, PChainHeight: 200, Signature: signApproval(200, emptyAuxInfoDigest), AuxInfoDigest: emptyAuxInfoDigest}}, } } else { node.sm.ApprovalsRetriever = &approvalsRetriever{ - result: []ValidatorSetApproval{{NodeID: [20]byte{2}, PChainHeight: 200, Signature: []byte{2}, AuxInfoSeqDigest: [32]byte{}}}, + result: []ValidatorSetApproval{{NodeID: [20]byte{2}, PChainHeight: 200, Signature: signApproval(200, emptyAuxInfoDigest), AuxInfoDigest: emptyAuxInfoDigest}}, } } } @@ -128,11 +133,11 @@ func TestFakeNode(t *testing.T) { node.act() if flipCoin() { node.sm.ApprovalsRetriever = &approvalsRetriever{ - result: []ValidatorSetApproval{{NodeID: [20]byte{2}, PChainHeight: 300, Signature: []byte{2}, AuxInfoSeqDigest: [32]byte{}}}, + result: []ValidatorSetApproval{{NodeID: [20]byte{2}, PChainHeight: 300, Signature: signApproval(300, emptyAuxInfoDigest), AuxInfoDigest: emptyAuxInfoDigest}}, } } else { node.sm.ApprovalsRetriever = &approvalsRetriever{ - result: []ValidatorSetApproval{{NodeID: [20]byte{3}, PChainHeight: 300, Signature: []byte{3}, AuxInfoSeqDigest: [32]byte{}}}, + result: []ValidatorSetApproval{{NodeID: [20]byte{3}, PChainHeight: 300, Signature: signApproval(300, emptyAuxInfoDigest), AuxInfoDigest: emptyAuxInfoDigest}}, } } } @@ -155,6 +160,7 @@ func TestFakeNodeEmptyMempool(t *testing.T) { var pChainHeight uint64 = 100 node := newFakeNode(t) + node.sm.AuxiliaryInfoApp = &noopTestAuxInfoApp{} node.sm.MaxBlockBuildingWaitTime = 100 * time.Millisecond node.sm.GetValidatorSet = validatorSetRetriever.getValidatorSet node.sm.GetPChainHeightForProposing = func() uint64 { @@ -180,11 +186,11 @@ func TestFakeNodeEmptyMempool(t *testing.T) { node.act() if flipCoin() { node.sm.ApprovalsRetriever = &approvalsRetriever{ - result: []ValidatorSetApproval{{NodeID: [20]byte{1}, PChainHeight: 200, Signature: []byte{1}, AuxInfoSeqDigest: [32]byte{}}}, + result: []ValidatorSetApproval{{NodeID: [20]byte{1}, PChainHeight: 200, Signature: signApproval(200, emptyAuxInfoDigest), AuxInfoDigest: emptyAuxInfoDigest}}, } } else { node.sm.ApprovalsRetriever = &approvalsRetriever{ - result: []ValidatorSetApproval{{NodeID: [20]byte{2}, PChainHeight: 200, Signature: []byte{2}, AuxInfoSeqDigest: [32]byte{}}}, + result: []ValidatorSetApproval{{NodeID: [20]byte{2}, PChainHeight: 200, Signature: signApproval(200, emptyAuxInfoDigest), AuxInfoDigest: emptyAuxInfoDigest}}, } } } @@ -213,11 +219,11 @@ func TestFakeNodeEmptyMempool(t *testing.T) { node.act() if flipCoin() { node.sm.ApprovalsRetriever = &approvalsRetriever{ - result: []ValidatorSetApproval{{NodeID: [20]byte{2}, PChainHeight: 300, Signature: []byte{2}, AuxInfoSeqDigest: [32]byte{}}}, + result: []ValidatorSetApproval{{NodeID: [20]byte{2}, PChainHeight: 300, Signature: signApproval(300, emptyAuxInfoDigest), AuxInfoDigest: emptyAuxInfoDigest}}, } } else { node.sm.ApprovalsRetriever = &approvalsRetriever{ - result: []ValidatorSetApproval{{NodeID: [20]byte{3}, PChainHeight: 300, Signature: []byte{3}, AuxInfoSeqDigest: [32]byte{}}}, + result: []ValidatorSetApproval{{NodeID: [20]byte{3}, PChainHeight: 300, Signature: signApproval(300, emptyAuxInfoDigest), AuxInfoDigest: emptyAuxInfoDigest}}, } } } diff --git a/msm/fuzz_test.go b/msm/fuzz_test.go index eb96e4d2..12a4e489 100644 --- a/msm/fuzz_test.go +++ b/msm/fuzz_test.go @@ -6,6 +6,7 @@ package metadata import ( "bytes" "context" + "crypto/sha256" "testing" "time" @@ -114,6 +115,11 @@ func FuzzVerifyBlock(f *testing.F) { // Mutating an authoritative field must make the block fail verification. fuzzedMD := block.Metadata field.set(&fuzzedMD, value) + + if fieldIdx%2 == 1 && block.Metadata.AuxiliaryInfo == nil { + fuzzedMD.AuxiliaryInfo = &AuxiliaryInfo{PrevAuxInfoSeq: value} + } + if bytes.Equal(fuzzedMD.MarshalCanoto(), block.Metadata.MarshalCanoto()) { t.Skip() // no-op mutation } @@ -184,6 +190,10 @@ func buildEpochChain(tb testing.TB, logger common.Logger) ([]*StateMachineBlock, currentTime := startTime sm, tc := newStateMachineWithLogger(tb, logger) + // This chain exercises the epoch lifecycle, not auxiliary info. Use an app whose + // history is always final so approvals are collected from the first collecting + // round (the builder only collects approvals once the aux info history is ready). + sm.AuxiliaryInfoApp = &noopTestAuxInfoApp{} sm.GetValidatorSet = getValidatorSet sm.GetPChainHeightForProposing = func() uint64 { return currentPChainHeight } sm.GetPChainHeightForVerifying = func() uint64 { return currentPChainHeight } @@ -244,21 +254,26 @@ func buildEpochChain(tb testing.TB, logger common.Logger) ([]*StateMachineBlock, block3 := build(3, 2, 1, block2) addBlock(3, block3, nil) + // The noopTestAuxInfoApp is always "ready" with an empty aux info history, so the candidate + // aux info digest the builder signs over is sha256 of the empty history. Peer approvals must + // carry the same digest to survive sanitizeApprovals' digest filter. + auxInfoDigest := sha256.Sum256(nil) + // block4 & block5: collecting-approvals blocks (1/3 then 2/3, not enough to seal). - approvalsResult = ValidatorSetApprovals{{NodeID: node1, PChainHeight: pChainHeight2, Signature: []byte("sig1")}} + approvalsResult = ValidatorSetApprovals{{NodeID: node1, PChainHeight: pChainHeight2, AuxInfoDigest: auxInfoDigest, Signature: signApproval(pChainHeight2, auxInfoDigest)}} currentTime = startTime.Add(time.Second + 4*time.Millisecond) tc.blockBuilder.block = nextInner(4) block4 := build(4, 3, 1, block3) addBlock(4, block4, nil) - approvalsResult = ValidatorSetApprovals{{NodeID: node2, PChainHeight: pChainHeight2, Signature: []byte("sig2")}} + approvalsResult = ValidatorSetApprovals{{NodeID: node2, PChainHeight: pChainHeight2, AuxInfoDigest: auxInfoDigest, Signature: signApproval(pChainHeight2, auxInfoDigest)}} currentTime = startTime.Add(time.Second + 5*time.Millisecond) tc.blockBuilder.block = nextInner(5) block5 := build(5, 4, 1, block4) addBlock(5, block5, nil) // block6: the sealing block (3/3 approvals). Its successor is in stateBuildBlockEpochSealed. - approvalsResult = ValidatorSetApprovals{{NodeID: node3, PChainHeight: pChainHeight2, Signature: []byte("sig3")}} + approvalsResult = ValidatorSetApprovals{{NodeID: node3, PChainHeight: pChainHeight2, AuxInfoDigest: auxInfoDigest, Signature: signApproval(pChainHeight2, auxInfoDigest)}} currentTime = startTime.Add(time.Second + 6*time.Millisecond) tc.blockBuilder.block = nextInner(6) block6 := build(6, 5, 1, block5) @@ -286,6 +301,7 @@ func buildEpochChain(tb testing.TB, logger common.Logger) ([]*StateMachineBlock, // Build a separate verifier MSM with its own copy of the fully populated store. verifier, vtc := newStateMachineWithLogger(tb, logger) + verifier.AuxiliaryInfoApp = &noopTestAuxInfoApp{} vtc.blockStore = tc.blockStore.clone() verifier.GetBlock = vtc.blockStore.getBlock verifier.GetValidatorSet = getValidatorSet diff --git a/msm/msm.go b/msm/msm.go index 20507935..bb7797a2 100644 --- a/msm/msm.go +++ b/msm/msm.go @@ -11,6 +11,7 @@ import ( "errors" "fmt" "math" + "slices" "time" "github.com/ava-labs/simplex/common" @@ -87,6 +88,7 @@ var ( errTimestampTooBig = errors.New("invalid timestamp: exceeds maximum int64 value") errTimestampDecreasing = errors.New("invalid timestamp: proposed timestamp is before parent block's timestamp") errTimestampTooFarInFuture = errors.New("invalid timestamp: proposed timestamp is too far in the future compared to current time") + errAuxInfoBlockRetrieval = errors.New("failed to retrieve block while collecting auxiliary info") signatureContext = "MSM approval" ) @@ -162,6 +164,30 @@ type BlockBuilder interface { WaitForPendingBlock(ctx context.Context) } +// AuxiliaryInfoGenVerifier abstracts the application-specific logic for generating and verifying auxiliary information +// that is piggybacked on epoch transitions. +type AuxiliaryInfoGenVerifier interface { + // IsLegalAppend checks whether the given auxiliary information byte slice [x] + // can be appended to the history of auxiliary information for the given versionID, according to the app's rules. + // Returns nil if the append is legal, or an error if the append is not legal or if any error occurs during the check. + IsLegalAppend(versionID VersionID, nodes NodeBLSMappings, history [][]byte, x []byte) error + + // IsSufficient checks whether the given history of auxiliary information for the given versionID is sufficient + // to start the epoch transition process. + IsSufficient(versionID VersionID, nodes NodeBLSMappings, history [][]byte) (bool, error) + + // Generate generates an auxiliary information encoded as a byte slice based on the history of auxiliary information + // for the given versionID in the current epoch so far. + // If this is the first invocation in the epoch, DefaultversionID() should be passed as the VersionID. + // Otherwise, the versionID from previous blocks in the epoch should be used. + // If the application deems the given history to be sufficient for the epoch change, it can return a nil byte slice, + // in which case it will not be appended to the history. + Generate(versionID VersionID, nodes NodeBLSMappings, history [][]byte) ([]byte, error) + + // DefaultVersionID returns the default VersionID that should be used for epochs that don't have any any auxiliary information yet. + DefaultVersionID() VersionID +} + // StateMachine manages block building and verification across epoch transitions. type StateMachine struct { *Config @@ -213,6 +239,8 @@ type Config struct { Signer common.Signer // ComputeICMEpoch computes the ICM epoch information in order to know which P-chain height to encode. ComputeICMEpoch ICMEpochTransition + // AuxiliaryInfoApp abstracts an application that piggybacks on epoch changes. + AuxiliaryInfoApp AuxiliaryInfoGenVerifier } type state uint8 @@ -454,7 +482,7 @@ func (sm *StateMachine) buildBlockOrTransitionEpoch(ctx context.Context, parentB } } - return wrapBlock(innerBlock, newSimplexEpochInfo, decisionToBuildBlock.pChainHeight, simplexMetadata, simplexBlacklist, now, icmEpochInfo), nil + return wrapBlock(innerBlock, newSimplexEpochInfo, decisionToBuildBlock.pChainHeight, simplexMetadata, simplexBlacklist, now, icmEpochInfo, nil), nil } func computeICMEpochInfo(parentBlock StateMachineBlock, computeICMEpoch ICMEpochTransition, childTimestamp time.Time) ICMEpochInfo { @@ -481,6 +509,7 @@ func verifyAgainstExpected( nextBlock *StateMachineBlock, timestamp time.Time, expectedIcmEpochInfo ICMEpochInfo, + auxInfo *AuxiliaryInfo, ) error { if innerBlock != nil { if err := innerBlock.Verify(ctx, expectedIcmEpochInfo.PChainEpochHeight); err != nil { @@ -489,7 +518,7 @@ func verifyAgainstExpected( } expectedBlock := wrapBlock( innerBlock, expectedSimplexEpochInfo, expectedPChainHeight, - nextBlock.Metadata.SimplexProtocolMetadata, nextBlock.Metadata.SimplexBlacklist, timestamp, expectedIcmEpochInfo) + nextBlock.Metadata.SimplexProtocolMetadata, nextBlock.Metadata.SimplexBlacklist, timestamp, expectedIcmEpochInfo, auxInfo) if expectedBlock.Digest() != nextBlock.Digest() { return fmt.Errorf("expected block digest %s does not match proposed block digest %s: %w", expectedBlock.Digest(), @@ -517,7 +546,7 @@ func (sm *StateMachine) verifyNormalBlock(ctx context.Context, parentBlock State } newSimplexEpochInfo.NextPChainReferenceHeight = nextBlock.Metadata.SimplexEpochInfo.NextPChainReferenceHeight - return verifyAgainstExpected(ctx, nextBlock.InnerBlock, newSimplexEpochInfo, proposedPChainHeight, nextBlock, timestamp, icmEpochInfo) + return verifyAgainstExpected(ctx, nextBlock.InnerBlock, newSimplexEpochInfo, proposedPChainHeight, nextBlock, timestamp, icmEpochInfo, nil) } func verifyPChainHeight(proposedPChainHeight uint64, currentPChainHeight uint64, prevPChainHeight uint64) error { @@ -755,6 +784,7 @@ func (sm *StateMachine) buildBlockZero(parentBlock StateMachineBlock, simplexMet simplexBlacklist, time.UnixMilli(timestamp), icmEpochInfo, + nil, ), nil } @@ -799,6 +829,7 @@ func (sm *StateMachine) verifyBlockZero(block *StateMachineBlock, prevBlock Stat block.Metadata.SimplexBlacklist, time.UnixMilli(int64(block.Metadata.Timestamp)), expectedICMEpochInfo, + nil, ) if expectedBlock.Digest() != block.Digest() { @@ -840,11 +871,31 @@ func (sm *StateMachine) verifyBlockZero(block *StateMachineBlock, prevBlock Stat // └──────────────────┘ └────────────────────┘ └────────────────────────────┘ // → stays Collecting → BuildBlockEpochSealed func (sm *StateMachine) buildBlockCollectingApprovals(ctx context.Context, parentBlock StateMachineBlock, simplexMetadata, simplexBlacklist []byte, prevBlockSeq uint64) (*StateMachineBlock, error) { - newApprovals, err := sm.computeNewApprovals(parentBlock) + // We prepare information that is needed to compute the approvals for the new epoch, + // such as the validator set for the next epoch, and the approvals from peers. + prevBlockNextPChainReferenceHeight := parentBlock.Metadata.SimplexEpochInfo.NextPChainReferenceHeight + validators, err := sm.GetValidatorSet(prevBlockNextPChainReferenceHeight) if err != nil { return nil, err } + auxInfo, isAuxInfoReadyForEpochTransition, auxInfoDigest, err := sm.computeAuxInfo(parentBlock, prevBlockSeq, validators) + if err != nil { + return nil, fmt.Errorf("failed to compute auxiliary info: %w", err) + } + + var newApprovals *approvals + if isAuxInfoReadyForEpochTransition { + newApprovals, err = sm.computeNewApprovals(parentBlock, validators, auxInfoDigest) + if err != nil { + return nil, err + } + } else { + // We're not ready for epoch transition yet, so putting a zero-value approvals here + // makes us stay in the collecting approvals state without contributing any approvals. + newApprovals = &approvals{} + } + newSimplexEpochInfo := computeSimplexEpochInfoForCollectingApprovalsBlock(parentBlock, prevBlockSeq, newApprovals) pChainHeight := parentBlock.Metadata.PChainHeight @@ -857,25 +908,16 @@ func (sm *StateMachine) buildBlockCollectingApprovals(ctx context.Context, paren // so that eventually we'll have enough approvals to seal the epoch. if !newApprovals.canSeal { sm.Logger.Debug("Not enough approvals to seal epoch, building block without sealing the epoch") - return sm.buildBlockImpatiently(ctx, parentBlock, simplexMetadata, simplexBlacklist, newSimplexEpochInfo, pChainHeight, icmEpochInfo) + return sm.buildBlockImpatiently(ctx, now, simplexMetadata, simplexBlacklist, newSimplexEpochInfo, pChainHeight, icmEpochInfo, auxInfo) } sm.Logger.Debug("Have enough approvals to seal epoch, building sealing block") // Else, we have enough approvals to seal the epoch, so we create the sealing block. - return sm.createSealingBlock(ctx, parentBlock, simplexMetadata, simplexBlacklist, newSimplexEpochInfo, pChainHeight, icmEpochInfo) + return sm.createSealingBlock(ctx, now, simplexMetadata, simplexBlacklist, newSimplexEpochInfo, pChainHeight, icmEpochInfo, auxInfo) } func (sm *StateMachine) verifyCollectingApprovalsBlock(ctx context.Context, parentBlock StateMachineBlock, nextBlock *StateMachineBlock, prevBlockSeq uint64) error { - nextMD := nextBlock.Metadata - newApprovals := nextMD.SimplexEpochInfo.NextEpochApprovals - - // The block builder should at least include its own approval in the block it builds, - // so we should have some approvals in the proposed block. - if newApprovals == nil || len(newApprovals.NodeIDs) == 0 || len(newApprovals.Signature) == 0 { - return errEmptyNextEpochApprovals - } - prevEpochInfo := parentBlock.Metadata.SimplexEpochInfo nextEpochInfo := nextBlock.Metadata.SimplexEpochInfo @@ -884,15 +926,32 @@ func (sm *StateMachine) verifyCollectingApprovalsBlock(ctx context.Context, pare return err } - err = sm.verifyNextEpochApprovalsSignature(prevEpochInfo, nextEpochInfo, validators) + newApprovals := nextBlock.Metadata.SimplexEpochInfo.NextEpochApprovals + + expectedAuxInfo, auxInfoDigest, isAuxInfoReady, err := sm.computeExpectedAuxInfoForApprovalCollection(parentBlock, nextBlock, prevBlockSeq, validators) if err != nil { - return err + return fmt.Errorf("failed to compute expected auxiliary info for approval collection: %w", err) } - // A node cannot remove other nodes' approvals, only add its own approval if it wasn't included in the previous block. - // So the set of signers in next.NextEpochApprovals should be a superset of the set of signers in prev.NextEpochApprovals. - if err := areNextEpochApprovalsSignersSupersetOfApprovalsOfPrevBlock(prevEpochInfo, nextEpochInfo); err != nil { - return err + // If the Auxiliary info is ready for epoch change, the block builder should at least include its own approval in the block it builds, + // so we should have some approvals in the proposed block. + if newApprovals == nil || len(newApprovals.NodeIDs) == 0 || len(newApprovals.Signature) == 0 { + if isAuxInfoReady { + return errEmptyNextEpochApprovals + } + } + + // If we aren't ready for epoch transition, we cannot collect approvals just yet. + // So just make an empty NextEpochApprovals. + if !isAuxInfoReady { + if newApprovals != nil && (len(newApprovals.NodeIDs) > 0 || len(newApprovals.Signature) > 0) { + return fmt.Errorf("expected no approvals when auxiliary info is not ready for epoch transition, but got some") + } + newSimplexEpochInfo := computeSimplexEpochInfoForCollectingApprovalsBlock(parentBlock, prevBlockSeq, &approvals{}) + timestamp := time.UnixMilli(int64(nextBlock.Metadata.Timestamp)) + icmEpochInfo := computeICMEpochInfo(parentBlock, sm.ComputeICMEpoch, timestamp) + + return verifyAgainstExpected(ctx, nextBlock.InnerBlock, newSimplexEpochInfo, nextBlock.Metadata.PChainHeight, nextBlock, timestamp, icmEpochInfo, expectedAuxInfo) } newSimplexEpochInfo := computeSimplexEpochInfoForCollectingApprovalsBlock(parentBlock, prevBlockSeq, &approvals{ @@ -900,6 +959,17 @@ func (sm *StateMachine) verifyCollectingApprovalsBlock(ctx context.Context, pare signature: newApprovals.Signature, }) + err = sm.verifyNextEpochApprovalsSignature(parentBlock.Metadata, nextBlock.Metadata, validators, auxInfoDigest) + if err != nil { + return err + } + + // A node cannot remove other nodes' approvals, only add its own approval if it wasn't included in the previous block. + // So the set of signers in next.NextEpochApprovals should be a superset of the set of signers in prev.NextEpochApprovals. + if err := areNextEpochApprovalsSignersSupersetOfApprovalsOfPrevBlock(prevEpochInfo, nextEpochInfo); err != nil { + return err + } + sigAggr := sm.SignatureAggregatorCreator(validators.NodeWeights()) approvals := bitmaskFromBytes(newApprovals.NodeIDs) canSeal := sigAggr.IsQuorum(validators.SelectSubset(approvals)) @@ -911,14 +981,16 @@ func (sm *StateMachine) verifyCollectingApprovalsBlock(ctx context.Context, pare } } - timestamp := time.UnixMilli(int64(nextMD.Timestamp)) - + timestamp := time.UnixMilli(int64(nextBlock.Metadata.Timestamp)) icmEpochInfo := computeICMEpochInfo(parentBlock, sm.ComputeICMEpoch, timestamp) - return verifyAgainstExpected(ctx, nextBlock.InnerBlock, newSimplexEpochInfo, nextMD.PChainHeight, nextBlock, timestamp, icmEpochInfo) + return verifyAgainstExpected(ctx, nextBlock.InnerBlock, newSimplexEpochInfo, nextBlock.Metadata.PChainHeight, nextBlock, timestamp, icmEpochInfo, expectedAuxInfo) } -func (sm *StateMachine) verifyNextEpochApprovalsSignature(prev SimplexEpochInfo, next SimplexEpochInfo, validators NodeBLSMappings) error { +func (sm *StateMachine) verifyNextEpochApprovalsSignature(prevMD StateMachineMetadata, nextMD StateMachineMetadata, validators NodeBLSMappings, auxInfoDigest [32]byte) error { + prev := prevMD.SimplexEpochInfo + next := nextMD.SimplexEpochInfo + // First figure out which validators are approving the next epoch by looking at the bitmask of approving nodes, // and then aggregate their public keys together to verify the signature. @@ -929,11 +1001,8 @@ func (sm *StateMachine) verifyNextEpochApprovalsSignature(prev SimplexEpochInfo, } pChainHeight := prev.NextPChainReferenceHeight - pChainHeightBuff := make([]byte, 8) - binary.BigEndian.PutUint64(pChainHeightBuff, pChainHeight) - signedMsg := common.SignedMessage{Payload: pChainHeightBuff, Context: signatureContext} - toBeSigned, err := asn1.Marshal(signedMsg) + toBeSigned, err := assembleApprovalToBeSigned(pChainHeight, auxInfoDigest) if err != nil { return err } @@ -944,6 +1013,17 @@ func (sm *StateMachine) verifyNextEpochApprovalsSignature(prev SimplexEpochInfo, return nil } +// assembleApprovalToBeSigned assembles the payload that is signed when approving an epoch transition. +// It consists of the P-chain reference height and the aux info digest (zeroed if not applicable). +func assembleApprovalToBeSigned(pChainHeight uint64, auxInfoDigest [32]byte) ([]byte, error) { + payloadToSignBuff := make([]byte, 8+32) // 8 bytes for the P-chain height and 32 bytes for the aux info digest if applicable. + binary.BigEndian.PutUint64(payloadToSignBuff[:8], pChainHeight) + copy(payloadToSignBuff[8:], auxInfoDigest[:]) + + signedMsg := common.SignedMessage{Payload: payloadToSignBuff, Context: signatureContext} + return asn1.Marshal(signedMsg) +} + func (sm *StateMachine) aggregatePubKeysForBitmask(nodeIDsBitmask []byte, validators NodeBLSMappings) ([]byte, error) { approvingNodes := bitmaskFromBytes(nodeIDsBitmask) publicKeys := make([][]byte, 0, len(validators)) @@ -983,14 +1063,8 @@ func computeSimplexEpochInfoForCollectingApprovalsBlock(parentBlock StateMachine return newSimplexEpochInfo } -func (sm *StateMachine) computeNewApprovals(parentBlock StateMachineBlock) (*approvals, error) { - // We prepare information that is needed to compute the approvals for the new epoch, - // such as the validator set for the next epoch, and the approvals from peers. - validators, err := sm.GetValidatorSet(parentBlock.Metadata.SimplexEpochInfo.NextPChainReferenceHeight) - if err != nil { - return nil, err - } - +func (sm *StateMachine) computeNewApprovals(parentBlock StateMachineBlock, validators NodeBLSMappings, auxInfoDigest [32]byte) (*approvals, error) { + prevBlockNextPChainReferenceHeight := parentBlock.Metadata.SimplexEpochInfo.NextPChainReferenceHeight sigAggr := sm.SignatureAggregatorCreator(validators.NodeWeights()) // We retrieve approvals that validators have sent us for the next epoch. @@ -1000,33 +1074,115 @@ func (sm *StateMachine) computeNewApprovals(parentBlock StateMachineBlock) (*app // Optimistically sign the epoch transition even if we have already did so in a previous round. // We'll just deduplicate this approval later on. - pChainHeightBuff := make([]byte, 8) - binary.BigEndian.PutUint64(pChainHeightBuff, parentBlock.Metadata.SimplexEpochInfo.NextPChainReferenceHeight) - sig, err := sm.Signer.Sign(pChainHeightBuff) + + sig, err := sm.createSelfApproval(prevBlockNextPChainReferenceHeight, auxInfoDigest) if err != nil { - return nil, fmt.Errorf("failed to sign approval: %w", err) + return nil, err } approvalsFromPeers = append(approvalsFromPeers, ValidatorSetApproval{ - NodeID: nodeID(sm.MyNodeID), - PChainHeight: parentBlock.Metadata.SimplexEpochInfo.NextPChainReferenceHeight, - Signature: sig, + NodeID: nodeID(sm.MyNodeID), + PChainHeight: prevBlockNextPChainReferenceHeight, + AuxInfoDigest: auxInfoDigest, + Signature: sig, }) - nextPChainHeight := parentBlock.Metadata.SimplexEpochInfo.NextPChainReferenceHeight + nextPChainHeight := prevBlockNextPChainReferenceHeight prevNextEpochApprovals := parentBlock.Metadata.SimplexEpochInfo.NextEpochApprovals - newApprovals, err := computeNewApprovals(prevNextEpochApprovals, approvalsFromPeers, nextPChainHeight, sigAggr, validators, sm.Logger) + newApprovals, err := computeNewApprovals(prevNextEpochApprovals, approvalsFromPeers, nextPChainHeight, auxInfoDigest, sigAggr, validators, sm.Logger) if err != nil { return nil, err } return newApprovals, nil } +func (sm *StateMachine) createSelfApproval(nextPChainReferenceHeight uint64, auxInfoDigest [32]byte) ([]byte, error) { + toBeSigned, err := assembleApprovalToBeSigned(nextPChainReferenceHeight, auxInfoDigest) + if err != nil { + return nil, err + } + + sig, err := sm.Signer.Sign(toBeSigned) + if err != nil { + return nil, fmt.Errorf("failed to sign approval: %w", err) + } + return sig, nil +} + +type auxInfoHistory struct { + data [][]byte + lastSeq uint64 +} + +func (aih *auxInfoHistory) lastHistory() []byte { + if len(aih.data) == 0 { + return nil + } + return aih.data[len(aih.data)-1] +} + +// collectAuxiliaryInfo traverses backwards starting from the given block and collects the AuxiliaryInfo of all blocks in the chain. +// returns the collected AuxiliaryInfo, the corresponding sequences of the blocks they were collected from, +// and the application ID of the oldest block that contains a non empty Info (or defaultVersionID if there was none). +func collectAuxiliaryInfo(block StateMachineBlock, startSeq uint64, getBlock BlockRetriever, defaultVersionID VersionID) (auxInfoHistory, VersionID, error) { + var lastSeq *uint64 + var history [][]byte + var versionID = defaultVersionID + + // We traverse the chain of blocks backwards in the following manner: + // (1) Every block that doesn't have AuxiliaryInfo, its parents also do not have AuxiliaryInfo. + // (2) Every block that has AuxiliaryInfo, its descendants also have AuxiliaryInfo. + // (3) A block that has AuxiliaryInfo may have an empty Info field, but its PrevAuxInfoSeq field must point + // to a block that its AuxiliaryInfo isn't nil, and its Info field is also non-nil. + // (4) When a block with an empty Info field is built on a parent block that has AuxiliaryInfo, + // if its parent block has a non-empty Info field, then the block's PrevAuxInfoSeq points to its parent block. + // Else, its parent block has an empty Info field, then the block's PrevAuxInfoSeq is inherited from its parent block's PrevAuxInfoSeq. + + auxInfo := block.Metadata.AuxiliaryInfo + currentSeq := startSeq + for auxInfo != nil { + if len(auxInfo.Info) > 0 { + history = append(history, auxInfo.Info) + if lastSeq == nil { + lastSeq = new(uint64) + *lastSeq = currentSeq + } + versionID = auxInfo.VersionID + } + if auxInfo.PrevAuxInfoSeq == 0 { + // This is the first auxiliary info of the epoch, we can stop traversing back. + break + } + currentSeq = auxInfo.PrevAuxInfoSeq + prevBlock, _, err := getBlock(auxInfo.PrevAuxInfoSeq, [32]byte{}) + if err != nil { + return auxInfoHistory{}, 0, fmt.Errorf("%w: at sequence %d: %w", errAuxInfoBlockRetrieval, auxInfo.PrevAuxInfoSeq, err) + } + auxInfo = prevBlock.Metadata.AuxiliaryInfo + } + + if lastSeq == nil { + lastSeq = new(uint64) + *lastSeq = 0 + } + + // Reverse so the history (and the matching seqs) are ordered from oldest to newest. + slices.Reverse(history) + return auxInfoHistory{data: history, lastSeq: *lastSeq}, versionID, nil +} + // buildBlockImpatiently builds a block by waiting for the VM to build a block until MaxBlockBuildingWaitTime. // If the VM fails to build a block within that time, we build a block without an inner block, // so that we can continue making progress and not get stuck waiting for the VM. -func (sm *StateMachine) buildBlockImpatiently(ctx context.Context, parentBlock StateMachineBlock, simplexMetadata []byte, simplexBlacklist []byte, simplexEpochInfo SimplexEpochInfo, pChainHeight uint64, icmEpochInfo ICMEpochInfo) (*StateMachineBlock, error) { +func (sm *StateMachine) buildBlockImpatiently(ctx context.Context, + timestamp time.Time, + simplexMetadata []byte, + simplexBlacklist []byte, + simplexEpochInfo SimplexEpochInfo, + pChainHeight uint64, + icmEpochInfo ICMEpochInfo, + auxInfo *AuxiliaryInfo) (*StateMachineBlock, error) { impatientContext, cancel := context.WithTimeout(ctx, sm.MaxBlockBuildingWaitTime) defer cancel() @@ -1043,23 +1199,22 @@ func (sm *StateMachine) buildBlockImpatiently(ctx context.Context, parentBlock S zap.Duration("elapsed", time.Since(start)), zap.Duration("maxBlockBuildingWaitTime", sm.MaxBlockBuildingWaitTime)) } - now := sm.GetTime() - icmEpochInfo = computeICMEpochInfo(parentBlock, sm.ComputeICMEpoch, now) - return wrapBlock(innerBlock, simplexEpochInfo, pChainHeight, simplexMetadata, simplexBlacklist, now, icmEpochInfo), nil + return wrapBlock(innerBlock, simplexEpochInfo, pChainHeight, simplexMetadata, simplexBlacklist, timestamp, icmEpochInfo, auxInfo), nil } func (sm *StateMachine) createSealingBlock(ctx context.Context, - parentBlock StateMachineBlock, + timestamp time.Time, simplexMetadata []byte, simplexBlacklist []byte, simplexEpochInfo SimplexEpochInfo, pChainHeight uint64, - icmEpochInfo ICMEpochInfo) (*StateMachineBlock, error) { + icmEpochInfo ICMEpochInfo, + auxInfo *AuxiliaryInfo) (*StateMachineBlock, error) { simplexEpochInfo, err := sm.computeSimplexEpochInfoForSealingBlock(simplexEpochInfo) if err != nil { return nil, fmt.Errorf("failed to compute simplex epoch info for sealing block: %w", err) } - return sm.buildBlockImpatiently(ctx, parentBlock, simplexMetadata, simplexBlacklist, simplexEpochInfo, pChainHeight, icmEpochInfo) + return sm.buildBlockImpatiently(ctx, timestamp, simplexMetadata, simplexBlacklist, simplexEpochInfo, pChainHeight, icmEpochInfo, auxInfo) } func (sm *StateMachine) computeSimplexEpochInfoForSealingBlock(simplexEpochInfo SimplexEpochInfo) (SimplexEpochInfo, error) { @@ -1094,7 +1249,8 @@ func wrapBlock( simplexMetadata, simplexBlacklist []byte, timestamp time.Time, - icmEpochInfo ICMEpochInfo) *StateMachineBlock { + icmEpochInfo ICMEpochInfo, + auxiliaryInfo *AuxiliaryInfo) *StateMachineBlock { return &StateMachineBlock{ InnerBlock: childBlock, @@ -1105,6 +1261,7 @@ func wrapBlock( SimplexEpochInfo: newSimplexEpochInfo, PChainHeight: pChainHeight, ICMEpochInfo: icmEpochInfo, + AuxiliaryInfo: auxiliaryInfo, }, } } @@ -1158,7 +1315,7 @@ func (sm *StateMachine) buildBlockEpochSealed(ctx context.Context, parentBlock S icmEpochInfo := computeICMEpochInfo(parentBlock, sm.ComputeICMEpoch, now) newSimplexEpochInfo := computeSimplexEpochInfoForTelock(parentBlock, sealingBlockSeq, prevBlockSeq) pChainHeight := parentBlock.Metadata.PChainHeight - return wrapBlock(nil, newSimplexEpochInfo, pChainHeight, simplexMetadata, simplexBlacklist, now, icmEpochInfo), nil + return wrapBlock(nil, newSimplexEpochInfo, pChainHeight, simplexMetadata, simplexBlacklist, now, icmEpochInfo, nil), nil } // Else, we build a block for the new epoch. @@ -1198,13 +1355,12 @@ func (sm *StateMachine) verifyBlockEpochSealed(ctx context.Context, parentBlock timestamp := time.UnixMilli(int64(nextBlock.Metadata.Timestamp)) - now := sm.GetTime() - icmEpochInfo := computeICMEpochInfo(parentBlock, sm.ComputeICMEpoch, now) + icmEpochInfo := computeICMEpochInfo(parentBlock, sm.ComputeICMEpoch, timestamp) newSimplexEpochInfo := computeSimplexEpochInfoForTelock(parentBlock, sealingBlockSeq, prevBlockSeq) if !isSealingBlockFinalized { - return verifyAgainstExpected(ctx, nil, newSimplexEpochInfo, nextBlock.Metadata.PChainHeight, nextBlock, timestamp, icmEpochInfo) + return verifyAgainstExpected(ctx, nil, newSimplexEpochInfo, nextBlock.Metadata.PChainHeight, nextBlock, timestamp, icmEpochInfo, nil) } // Else, it's a new epoch. @@ -1225,7 +1381,111 @@ func (sm *StateMachine) verifyBlockEpochSealed(ctx context.Context, parentBlock } newSimplexEpochInfo.NextPChainReferenceHeight = nextBlock.Metadata.SimplexEpochInfo.NextPChainReferenceHeight - return verifyAgainstExpected(ctx, nextBlock.InnerBlock, newSimplexEpochInfo, proposedPChainHeight, nextBlock, timestamp, icmEpochInfo) + return verifyAgainstExpected(ctx, nextBlock.InnerBlock, newSimplexEpochInfo, proposedPChainHeight, nextBlock, timestamp, icmEpochInfo, nil) +} + +// computeExpectedAuxInfoForApprovalCollection computes the expected AuxiliaryInfo that should be included in the proposed block +// for approval collection, and returns the auxiliary info digest, and whether the auxiliary info history is ready for epoch transition. +func (sm *StateMachine) computeExpectedAuxInfoForApprovalCollection(parentBlock StateMachineBlock, nextBlock *StateMachineBlock, prevBlockSeq uint64, validators NodeBLSMappings) (*AuxiliaryInfo, [32]byte, bool, error) { + nextMD := nextBlock.Metadata + prevMD := parentBlock.Metadata + + auxInfoHistory, versionID, err := collectAuxiliaryInfo(parentBlock, prevBlockSeq, sm.GetBlock, sm.AuxiliaryInfoApp.DefaultVersionID()) + if err != nil { + return nil, [32]byte{}, false, err + } + + if len(auxInfoHistory.data) > 0 && nextMD.AuxiliaryInfo == nil { + // If we have auxiliary info history but the proposed block doesn't include any auxiliary info, + // it means the block builder has dropped the auxiliary info, which is not allowed. + return nil, [32]byte{}, false, fmt.Errorf("expected auxiliary info for application %d with history length %d, but got nil", versionID, len(auxInfoHistory.data)) + } + + // Else, either len(auxInfoHistory) == 0, + // or the proposed block includes auxiliary info. + // Both of these cases are fine, because a node doesn't have to include Auxiliary information. + // We will verify the legality of the proposed auxiliary info (if any) in the next step. + + var expectedAuxInfo *AuxiliaryInfo + var proposedAuxInf []byte + + if nextMD.AuxiliaryInfo != nil { + proposedAuxInf = nextMD.AuxiliaryInfo.Info + expectedAuxInfo = &AuxiliaryInfo{ + VersionID: versionID, + Info: proposedAuxInf, + } + if prevMD.AuxiliaryInfo != nil { + expectedAuxInfo.PrevAuxInfoSeq = auxInfoHistory.lastSeq + } + } + + if err := sm.AuxiliaryInfoApp.IsLegalAppend(versionID, validators, auxInfoHistory.data, proposedAuxInf); err != nil { + return nil, [32]byte{}, false, fmt.Errorf("proposed auxiliary info is not a legal append to the history for application %d: %w", versionID, err) + } + + auxInfoReady, err := sm.AuxiliaryInfoApp.IsSufficient(versionID, validators, auxInfoHistory.data) + if err != nil { + return nil, [32]byte{}, false, fmt.Errorf("failed to check if auxiliary info history is final for application %d: %w", versionID, err) + } + + var digest [32]byte + if auxInfoReady { + digest = sha256.Sum256(auxInfoHistory.lastHistory()) + } + + return expectedAuxInfo, digest, auxInfoReady, nil +} + +// computeAuxInfo computes the AuxiliaryInfo that should be included in the block being built, and whether the auxiliary info history is ready for epoch transition, +func (sm *StateMachine) computeAuxInfo(parentBlock StateMachineBlock, prevBlockSeq uint64, validators NodeBLSMappings) (*AuxiliaryInfo, bool, common.Digest, error) { + auxInfoHistory, versionID, err := collectAuxiliaryInfo(parentBlock, prevBlockSeq, sm.GetBlock, sm.AuxiliaryInfoApp.DefaultVersionID()) + if err != nil { + return nil, false, common.Digest{}, err + } + + isAuxInfoReadyForEpochTransition, err := sm.AuxiliaryInfoApp.IsSufficient(versionID, validators, auxInfoHistory.data) + if err != nil { + return nil, false, common.Digest{}, fmt.Errorf("failed to check if auxiliary info history is final: %w", err) + } + + var auxInfo *AuxiliaryInfo + parentAuxInfo := parentBlock.Metadata.AuxiliaryInfo + if parentAuxInfo != nil { + auxInfo = &AuxiliaryInfo{ + VersionID: parentAuxInfo.VersionID, + PrevAuxInfoSeq: auxInfoHistory.lastSeq, + } + } + + if !isAuxInfoReadyForEpochTransition { + // If the auxiliary info isn't ready for epoch transition, + // we should focus on contributing to finalizing it before collecting approvals for the epoch transition, + // as without it being ready, we won't be able to transition epochs anyway. + auxInf, err := sm.AuxiliaryInfoApp.Generate(versionID, validators, auxInfoHistory.data) + if err != nil { + return nil, false, common.Digest{}, fmt.Errorf("failed to generate auxiliary info: %w", err) + } + if auxInfo == nil { + // This is the first auxiliary info we're generating for this epoch, + // so we need to initialize it. + auxInfo = &AuxiliaryInfo{ + VersionID: versionID, + Info: auxInf, + } + } else { + // Otherwise, we already have auxiliary info from the parent block, + // so we just update the Info field and carry over the VersionID and PrevAuxInfoSeq. + auxInfo.Info = auxInf + } + } + + var auxInfoDigest common.Digest + if isAuxInfoReadyForEpochTransition { + auxInfoDigest = sha256.Sum256(auxInfoHistory.lastHistory()) + } + + return auxInfo, isAuxInfoReadyForEpochTransition, auxInfoDigest, nil } // constructSimplexZeroBlockSimplexEpochInfo constructs the SimplexEpochInfo for the zero block, which is the first ever block built by Simplex. @@ -1253,6 +1513,7 @@ func computeNewApprovals( prevNextEpochApprovals *NextEpochApprovals, approvalsFromPeers ValidatorSetApprovals, pChainHeight uint64, + auxInfoDigest [32]byte, sigAggr common.SignatureAggregator, validators NodeBLSMappings, logger common.Logger, @@ -1269,7 +1530,7 @@ func computeNewApprovals( // We have the approvals obtained from peers, but we need to sanitize them by filtering out approvals that are not valid, // such as approvals that do not agree with our candidate auxiliary info digest and P-Chain height, // and approvals that are from nodes that are not in the validator set or have already approved in prior blocks. - approvalsFromPeers = sanitizeApprovals(approvalsFromPeers, pChainHeight, nodeID2ValidatorIndex, oldApprovingNodes, logger) + approvalsFromPeers = sanitizeApprovals(approvalsFromPeers, pChainHeight, auxInfoDigest, nodeID2ValidatorIndex, oldApprovingNodes, logger) logger.Debug("Sanitized approvals after filtering out invalid approvals", zap.Int("numApprovalsBefore", oldApprovalFromPeersCount), zap.Int("numApprovalsAfter", len(approvalsFromPeers))) // Next we aggregate both previous and new approvals to compute the new aggregated signatures and the new bitmask of approving nodes. @@ -1344,21 +1605,22 @@ func computeNewApproverSignaturesAndSigners( // sanitizeApprovals filters out approvals that are not valid by checking if they agree with our candidate auxiliary info digest and P-Chain height, // and if they are from the validator set and haven't already been approved. -func sanitizeApprovals(approvals ValidatorSetApprovals, pChainHeight uint64, nodeID2ValidatorIndex map[nodeID]int, oldApprovingNodes bitmask, logger common.Logger) ValidatorSetApprovals { - filter1 := approvalsThatAgreeWithPChainHeight(pChainHeight) +func sanitizeApprovals(approvals ValidatorSetApprovals, pChainHeight uint64, auxInfoDigest [32]byte, nodeID2ValidatorIndex map[nodeID]int, oldApprovingNodes bitmask, logger common.Logger) ValidatorSetApprovals { + filter1 := approvalsThatAgreeWithPChainHeightAndAuxInfoDigest(pChainHeight, auxInfoDigest) filter2 := approvalsThatAreInValidatorSetAndHaveNotAlreadyApproved(oldApprovingNodes.Clone(), nodeID2ValidatorIndex) return approvals.Filter(filter1, logger).Filter(filter2, logger).UniqueByNodeID() } -func approvalsThatAgreeWithPChainHeight(pChainHeight uint64) func(approval ValidatorSetApproval, logger common.Logger) bool { +func approvalsThatAgreeWithPChainHeightAndAuxInfoDigest(pChainHeight uint64, auxInfoDigest [32]byte) func(approval ValidatorSetApproval, logger common.Logger) bool { return func(approval ValidatorSetApproval, logger common.Logger) bool { // Pick only approvals that agree with our P-Chain height - ok := approval.PChainHeight == pChainHeight + ok := approval.PChainHeight == pChainHeight && approval.AuxInfoDigest == auxInfoDigest if !ok { - logger.Debug("Filtering out approval that does not agree with our P-Chain height", + logger.Debug("Filtering out approval that does not agree with our P-Chain height or auxiliary info digest", zap.String("nodeID", fmt.Sprintf("%x", approval.NodeID)), zap.Uint64("approvalPChainHeight", approval.PChainHeight), - zap.Uint64("expectedPChainHeight", pChainHeight)) + zap.Uint64("expectedPChainHeight", pChainHeight), + zap.String("approvalAuxInfoSeqDigest", fmt.Sprintf("%x", approval.AuxInfoDigest))) } return ok } diff --git a/msm/msm_test.go b/msm/msm_test.go index 08959583..6d7fca94 100644 --- a/msm/msm_test.go +++ b/msm/msm_test.go @@ -6,6 +6,8 @@ package metadata import ( "context" "crypto/rand" + "crypto/sha256" + "errors" "fmt" "math" "testing" @@ -553,6 +555,13 @@ func TestMSMFullEpochLifecycle(t *testing.T) { sm.GetValidatorSet = getValidatorSet sm.GetTime = fixedTime + + // This test exercises the epoch/approval/seal lifecycle, not auxiliary info. + // Uses an app (noopTestAuxInfoApp) whose history is always final so approvals are collected from the + // first collecting round and no auxiliary info is generated. Auxiliary info + // behavior is covered by TestVerifyCollectingApprovalsNotReady and + // TestCollectAuxiliaryInfo. + sm.AuxiliaryInfoApp = &noopTestAuxInfoApp{} tc.blockStore[0] = &outerBlock{block: genesis} tc.blockStore[42] = &outerBlock{block: notGenesis} @@ -561,6 +570,7 @@ func TestMSMFullEpochLifecycle(t *testing.T) { sm.LastNonSimplexBlockPChainHeight = pChainHeight1 smVerify, tcVerify := newStateMachine(t) + smVerify.AuxiliaryInfoApp = &noopTestAuxInfoApp{} smVerify.GetValidatorSet = getValidatorSet // sm only ever builds blocks and smVerify only ever verifies them. @@ -695,17 +705,19 @@ func TestMSMFullEpochLifecycle(t *testing.T) { var approvalsResult ValidatorSetApprovals sm.ApprovalsRetriever = &dynamicApprovalsRetriever{approvals: &approvalsResult} + sig1 := signApproval(pChainHeight2, emptyAuxInfoDigest) approvalsResult = ValidatorSetApprovals{ { - NodeID: node1, - PChainHeight: pChainHeight2, - Signature: []byte("sig1"), + NodeID: node1, + PChainHeight: pChainHeight2, + AuxInfoDigest: emptyAuxInfoDigest, + Signature: sig1, }, } // node1 is at index 0 in validatorSet2 → bitmask bit 0 → {1} bitmask := []byte{1} - sig, err := aggr.AppendSignatures(nil, []byte("sig1")) + sig, err := aggr.AppendSignatures(nil, sig1) require.NoError(t, err) currentTime = startTime.Add(time.Second + 4*time.Millisecond) @@ -737,16 +749,18 @@ func TestMSMFullEpochLifecycle(t *testing.T) { require.NoError(t, smVerify.VerifyBlock(context.Background(), block4)) // ----- Step 5: Second collecting block (2/3 approvals, still not enough since threshold is strictly > 2/3) ----- + sig2 := signApproval(pChainHeight2, emptyAuxInfoDigest) approvalsResult = ValidatorSetApprovals{ { - NodeID: node2, - PChainHeight: pChainHeight2, - Signature: []byte("sig2"), + NodeID: node2, + PChainHeight: pChainHeight2, + AuxInfoDigest: emptyAuxInfoDigest, + Signature: sig2, }, } // node2 is at index 1 → bitmask bits 0,1 → {3} - sig, err = aggr.AppendSignatures(sig, []byte("sig2")) + sig, err = aggr.AppendSignatures(sig, sig2) require.NoError(t, err) bitmask = []byte{3} @@ -779,16 +793,18 @@ func TestMSMFullEpochLifecycle(t *testing.T) { require.NoError(t, smVerify.VerifyBlock(context.Background(), block5)) // ----- Step 6: Sealing block (3/3 approvals, enough to seal) ----- + sig3 := signApproval(pChainHeight2, emptyAuxInfoDigest) approvalsResult = ValidatorSetApprovals{ { - NodeID: node3, - PChainHeight: pChainHeight2, - Signature: []byte("sig3"), + NodeID: node3, + PChainHeight: pChainHeight2, + AuxInfoDigest: emptyAuxInfoDigest, + Signature: sig3, }, } // node3 is at index 2 → bitmask bits 0,1,2 → {7} - sig6, err := aggr.AppendSignatures(sig, []byte("sig3")) + sig6, err := aggr.AppendSignatures(sig, sig3) require.NoError(t, err) bitmask = []byte{7} @@ -1275,7 +1291,18 @@ func TestSanitizeApprovals(t *testing.T) { {NodeID: node1, PChainHeight: 200}, } oldApproving := bitmaskFromBytes(nil) - result := sanitizeApprovals(approvals, 100, nodeID2Index, oldApproving, logger) + result := sanitizeApprovals(approvals, 100, [32]byte{}, nodeID2Index, oldApproving, logger) + require.Len(t, result, 1) + require.Equal(t, node0, result[0].NodeID) + }) + + t.Run("filters by aux info digest", func(t *testing.T) { + approvals := ValidatorSetApprovals{ + {NodeID: node0, PChainHeight: 100, AuxInfoDigest: [32]byte{0xAA}}, + {NodeID: node1, PChainHeight: 100, AuxInfoDigest: [32]byte{0xBB}}, + } + oldApproving := bitmaskFromBytes(nil) + result := sanitizeApprovals(approvals, 100, [32]byte{0xAA}, nodeID2Index, oldApproving, logger) require.Len(t, result, 1) require.Equal(t, node0, result[0].NodeID) }) @@ -1286,7 +1313,7 @@ func TestSanitizeApprovals(t *testing.T) { {NodeID: node1, PChainHeight: 100}, } oldApproving := bitmaskFromBytes([]byte{1}) - result := sanitizeApprovals(approvals, 100, nodeID2Index, oldApproving, logger) + result := sanitizeApprovals(approvals, 100, [32]byte{}, nodeID2Index, oldApproving, logger) require.Len(t, result, 1) require.Equal(t, node1, result[0].NodeID) }) @@ -1297,7 +1324,7 @@ func TestSanitizeApprovals(t *testing.T) { {NodeID: node2, PChainHeight: 100}, } oldApproving := bitmaskFromBytes(nil) - result := sanitizeApprovals(approvals, 100, nodeID2Index, oldApproving, logger) + result := sanitizeApprovals(approvals, 100, [32]byte{}, nodeID2Index, oldApproving, logger) require.Len(t, result, 1) require.Equal(t, node2, result[0].NodeID) }) @@ -1308,7 +1335,7 @@ func TestSanitizeApprovals(t *testing.T) { {NodeID: node0, PChainHeight: 100}, } oldApproving := bitmaskFromBytes(nil) - result := sanitizeApprovals(approvals, 100, nodeID2Index, oldApproving, logger) + result := sanitizeApprovals(approvals, 100, [32]byte{}, nodeID2Index, oldApproving, logger) require.Len(t, result, 1) }) } @@ -1447,6 +1474,11 @@ func TestComputeNewApproverSignaturesAndSigners(t *testing.T) { func TestBuildBlockCollectingApprovalsDedupsOwnApprovalAcrossRounds(t *testing.T) { sm, tc := newStateMachine(t) + // This test is about approval dedup, not auxiliary info. Use an app whose history + // is always final so approvals are collected from the first collecting round + // (the builder only collects approvals once the aux info history is ready). + sm.AuxiliaryInfoApp = &noopTestAuxInfoApp{} + // Use concatAggregator so that AppendSignatures(existing) with zero new // signatures returns `existing` verbatim. This makes signature equality // a direct witness that no new signature was aggregated in. @@ -1522,3 +1554,419 @@ func TestBuildBlockCollectingApprovalsDedupsOwnApprovalAcrossRounds(t *testing.T require.Equal(t, firstSig, block2.Metadata.SimplexEpochInfo.NextEpochApprovals.Signature, "aggregated signature must be unchanged when no new approvals were aggregated") } + +func TestVerifyCollectingApprovalsNotReady(t *testing.T) { + // Tests collecting-approvals state while the auxiliary info history is not yet final. + // In that state the builder does not collect approvals, + // so a block legitimately carries no NextEpochApprovals. The verifier must + // (1) accept such a block, (2) not panic when NextEpochApprovals is nil, and + // (3) reject a block that carries approvals before the aux info is ready. + + const ( + pChainRefHeight = uint64(100) + nextPChainRefHeight = uint64(200) + parentSeq = uint64(10) + ) + + newSM := func(t *testing.T) (*StateMachine, *testConfig, StateMachineBlock) { + sm, tc := newStateMachine(t) + // Default app (AuxiliaryInfoGenVerifier) for newStateMachine has threshold 2 + // so IsSufficient returns false: the aux info is not ready. + sm.GetPChainHeightForProposing = func() uint64 { return nextPChainRefHeight } + sm.GetPChainHeightForVerifying = func() uint64 { return nextPChainRefHeight } + + // Parent block: epoch transition in progress (NextPChainReferenceHeight > 0), + // not yet sealed, so NextState() is stateBuildCollectingApprovals. + parent := StateMachineBlock{ + InnerBlock: &InnerBlock{TS: time.Now(), BlockHeight: 1, Bytes: []byte{0xAA}}, + Metadata: StateMachineMetadata{ + PChainHeight: nextPChainRefHeight, + SimplexProtocolMetadata: (&common.ProtocolMetadata{ + Seq: parentSeq, Round: 5, Epoch: 1, + }).Bytes(), + SimplexEpochInfo: SimplexEpochInfo{ + PChainReferenceHeight: pChainRefHeight, + EpochNumber: 1, + NextPChainReferenceHeight: nextPChainRefHeight, + PrevVMBlockSeq: parentSeq - 1, + }, + }, + } + tc.blockStore[parentSeq] = &outerBlock{block: parent} + require.Equal(t, stateBuildCollectingApprovals, parent.Metadata.SimplexEpochInfo.NextState()) + return sm, tc, parent + } + + build := func(t *testing.T, sm *StateMachine, tc *testConfig, parent StateMachineBlock) *StateMachineBlock { + tc.blockBuilder.block = &InnerBlock{TS: time.Now(), BlockHeight: 2, Bytes: []byte{0x01}} + md := common.ProtocolMetadata{Seq: parentSeq + 1, Round: 6, Epoch: 1, Prev: parent.Digest()} + block, err := sm.BuildBlock(context.Background(), md, nil) + require.NoError(t, err) + return block + } + + t.Run("built not-ready block verifies and carries no approvals", func(t *testing.T) { + sm, tc, parent := newSM(t) + block := build(t, sm, tc, parent) + + // The builder generated auxiliary info but collected no approvals. + require.NotNil(t, block.Metadata.AuxiliaryInfo) + require.Empty(t, block.Metadata.SimplexEpochInfo.NextEpochApprovals.NodeIDs) + require.Empty(t, block.Metadata.SimplexEpochInfo.NextEpochApprovals.Signature) + + require.NoError(t, sm.VerifyBlock(context.Background(), block)) + }) + + t.Run("nil NextEpochApprovals does not panic", func(t *testing.T) { + sm, tc, parent := newSM(t) + block := build(t, sm, tc, parent) + block.Metadata.SimplexEpochInfo.NextEpochApprovals = nil + + // The regression: verifying a not-ready block with nil approvals must not panic. + // (The digest no longer matches, so an error is expected — just not a panic.) + require.NotPanics(t, func() { + _ = sm.verifyCollectingApprovalsBlock(context.Background(), parent, block, parentSeq) + }) + }) + + t.Run("approvals before aux info is ready are rejected", func(t *testing.T) { + sm, tc, parent := newSM(t) + block := build(t, sm, tc, parent) + block.Metadata.SimplexEpochInfo.NextEpochApprovals = &NextEpochApprovals{ + NodeIDs: []byte{1}, + Signature: []byte("sig"), + } + + err := sm.verifyCollectingApprovalsBlock(context.Background(), parent, block, parentSeq) + require.ErrorContains(t, err, "expected no approvals") + }) +} + +func TestCollectingApprovalsAuxInfoGating(t *testing.T) { + // Walks a chain through the collecting-approvals state while the auxiliary info history fills up. + // With a threshold of 2, the history only becomes final after two distinct votes, + // so the first two collecting blocks carry auxiliary info but no approvals, + // and approvals are collected only once the history is ready. + // Each built block must verify. + + const ( + pChainRefHeight = uint64(100) + nextPChainRefHeight = uint64(200) + parentSeq = uint64(10) + ) + + sm, tc := newStateMachine(t) + sm.GetPChainHeightForProposing = func() uint64 { return nextPChainRefHeight } + sm.GetPChainHeightForVerifying = func() uint64 { return nextPChainRefHeight } + + // Deterministic votes so the built auxiliary info is predictable. + vote1 := []byte("vote-1") + vote2 := []byte("vote-2") + votes := [][]byte{vote1, vote2} + sm.AuxiliaryInfoApp = &voteCountingAuxInfoApp{ + threshold: 2, + randomTape: func() []byte { + next := votes[0] + votes = votes[1:] + return next + }, + } + + // A 3-node validator set including MyNodeID at index 0, so the optimistic self-approval + // is retained once approvals are collected, but a single approval is below quorum (the + // block stays in the collecting state rather than sealing). + validators := NodeBLSMappings{ + {NodeID: nodeID(sm.MyNodeID), BLSKey: []byte{1}, Weight: 1}, + {NodeID: nodeID{0xBB}, BLSKey: []byte{2}, Weight: 1}, + {NodeID: nodeID{0xCC}, BLSKey: []byte{3}, Weight: 1}, + } + tc.validatorSetRetriever.result = validators + + parent := StateMachineBlock{ + InnerBlock: &InnerBlock{TS: time.Now(), BlockHeight: 1, Bytes: []byte{0xAA}}, + Metadata: StateMachineMetadata{ + PChainHeight: nextPChainRefHeight, + SimplexProtocolMetadata: (&common.ProtocolMetadata{ + Seq: parentSeq, Round: 5, Epoch: 1, + }).Bytes(), + SimplexEpochInfo: SimplexEpochInfo{ + PChainReferenceHeight: pChainRefHeight, + EpochNumber: 1, + NextPChainReferenceHeight: nextPChainRefHeight, + PrevVMBlockSeq: parentSeq - 1, + }, + }, + } + tc.blockStore[parentSeq] = &outerBlock{block: parent} + + // build constructs the next collecting block on top of prev, stores it so it can serve + // as a parent (and as a back-pointer target for the aux info history), and verifies it. + build := func(seq uint64, prev StateMachineBlock) *StateMachineBlock { + tc.blockBuilder.block = &InnerBlock{TS: time.Now(), BlockHeight: seq, Bytes: []byte{byte(seq)}} + md := common.ProtocolMetadata{Seq: seq, Round: seq, Epoch: 1, Prev: prev.Digest()} + block, err := sm.BuildBlock(context.Background(), md, nil) + require.NoError(t, err) + require.NoError(t, sm.VerifyBlock(context.Background(), block)) + tc.blockStore[seq] = &outerBlock{block: *block} + return block + } + + approvals := func(b *StateMachineBlock) *NextEpochApprovals { + return b.Metadata.SimplexEpochInfo.NextEpochApprovals + } + // requireAuxInfo compares the meaningful fields, ignoring the cached canoto size. + requireAuxInfo := func(want, got *AuxiliaryInfo) { + require.True(t, want.Equal(got), "expected aux info %+v, got %+v", want, got) + } + + // block1: history empty, not final -> generates vote1, collects no approvals. + block1 := build(parentSeq+1, parent) + requireAuxInfo(&AuxiliaryInfo{Info: vote1, VersionID: 1}, block1.Metadata.AuxiliaryInfo) + require.Empty(t, approvals(block1).NodeIDs) + + // block2: history [vote1], still not final -> generates vote2, collects no approvals. + block2 := build(parentSeq+2, *block1) + requireAuxInfo(&AuxiliaryInfo{Info: vote2, PrevAuxInfoSeq: parentSeq + 1, VersionID: 1}, block2.Metadata.AuxiliaryInfo) + require.Empty(t, approvals(block2).NodeIDs) + + // block3: history [vote1, vote2] is now final -> no new vote, and approvals are + // collected (the optimistic self-approval sets MyNodeID's bit). block3 is the first + // empty-Info block; it points at block2, the last non-empty Info block. + block3 := build(parentSeq+3, *block2) + requireAuxInfo(&AuxiliaryInfo{PrevAuxInfoSeq: parentSeq + 2, VersionID: 1}, block3.Metadata.AuxiliaryInfo) + require.Equal(t, []byte{1}, approvals(block3).NodeIDs, "self-approval bit should be set once aux info is ready") + + // The collected approval must be signed over the epoch-transition payload for the + //mnext epoch's P-chain reference height (200) and the digest + // of the final auxiliary info history, which is sha256 of the last vote (vote2). + wantSigned, err := assembleApprovalToBeSigned(nextPChainRefHeight, sha256.Sum256(vote2)) + require.NoError(t, err) + require.NoError(t, (&signatureVerifier{}).VerifySignature(approvals(block3).Signature, wantSigned, nil), + "NextEpochApprovals signature must verify against P-chain height 200 and the digest of vote2") + + // block4: built on the empty-Info block3 while still collecting approvals (1/3 is below + // quorum). Its PrevAuxInfoSeq must SKIP the empty block3 and point at block2 (parentSeq+2), + // the most recent non-empty Info block -- not at its immediate parent block3 (parentSeq+3). + // This is the case the rest of the chain never reaches and where "skip" differs from "successive". + block4 := build(parentSeq+4, *block3) + require.NotEqual(t, parentSeq+3, block4.Metadata.AuxiliaryInfo.PrevAuxInfoSeq, + "PrevAuxInfoSeq must not point at the empty-Info parent block3") + requireAuxInfo(&AuxiliaryInfo{PrevAuxInfoSeq: parentSeq + 2, VersionID: 1}, block4.Metadata.AuxiliaryInfo) + + // block5: another empty-Info block on top of the empty block4. The back-pointer still skips + // the whole empty run and points at block2, confirming the skip persists across consecutive + // empty-Info blocks (collectAuxiliaryInfo finds the same most-recent non-empty block each time). + block5 := build(parentSeq+5, *block4) + requireAuxInfo(&AuxiliaryInfo{PrevAuxInfoSeq: parentSeq + 2, VersionID: 1}, block5.Metadata.AuxiliaryInfo) +} + +func TestCollectingApprovalsAuxInfoVersionIDIsBackwardCompatible(t *testing.T) { + // Backward compatibility: once an epoch has a VersionID set on its auxiliary info, that + // VersionID must be reused for the rest of the epoch -- for both building AND verifying + // subsequent blocks -- even if the application's DefaultVersionID() later changes. + // + // collectAuxiliaryInfo only consults DefaultVersionID() when the auxiliary info history is + // empty; once a block carries a VersionID, every later buildAndVerify and verify reads that VersionID + // back from the chain instead. So we seed the epoch's parent with auxiliary info stamped with + // VersionID 1, then flip DefaultVersionID() to 2 right after the first Generate(). Because the + // epoch already has a VersionID on-chain, every Generate()/IsLegalAppend()/IsSufficient() + // invocation -- on the buildAndVerify path and the verify path -- must keep using VersionID 1, never 2. + // The app asserts that internally: it requires the VersionID it receives to equal + // expectedVersionID, which we hold at 1 throughout. + + const ( + pChainRefHeight = uint64(100) + nextPChainRefHeight = uint64(200) + parentSeq = uint64(10) + ) + + sm, tc := newStateMachine(t) + sm.GetPChainHeightForVerifying = func() uint64 { return nextPChainRefHeight } + sm.GetPChainHeightForProposing = func() uint64 { return nextPChainRefHeight } + + // threshold 4 so Generate() runs for the first three collecting blocks built on top of the + // pre-seeded parent (history not yet sufficient), giving us one "first" and several "later" + // Generate() invocations. defaultVersionID starts at 1 (the original default); expectedVersionID + // stays 1 for the whole test -- the app asserts every invocation uses it. + app := &versionRecordingAuxInfoApp{ + t: t, + threshold: 4, + votes: [][]byte{[]byte("vote-1"), []byte("vote-2"), []byte("vote-3")}, + defaultVersionID: 1, + expectedVersionID: 1, + } + sm.AuxiliaryInfoApp = app + + validators := NodeBLSMappings{ + {NodeID: nodeID(sm.MyNodeID), BLSKey: []byte{1}, Weight: 1}, + {NodeID: nodeID{0xBB}, BLSKey: []byte{2}, Weight: 1}, + {NodeID: nodeID{0xCC}, BLSKey: []byte{3}, Weight: 1}, + } + tc.validatorSetRetriever.result = validators + + // The parent already carries auxiliary info for this epoch, stamped with VersionID 1. + // This is the backward-compatibility precondition: the epoch's VersionID is already set. + parent := StateMachineBlock{ + InnerBlock: &InnerBlock{TS: time.Now(), BlockHeight: 1, Bytes: []byte{0xAA}}, + Metadata: StateMachineMetadata{ + PChainHeight: nextPChainRefHeight, + SimplexProtocolMetadata: (&common.ProtocolMetadata{ + Seq: parentSeq, Round: 5, Epoch: 1, + }).Bytes(), + SimplexEpochInfo: SimplexEpochInfo{ + PChainReferenceHeight: pChainRefHeight, + EpochNumber: 1, + NextPChainReferenceHeight: nextPChainRefHeight, + PrevVMBlockSeq: parentSeq - 1, + }, + AuxiliaryInfo: &AuxiliaryInfo{ + VersionID: 1, + Info: []byte("vote-0"), + PrevAuxInfoSeq: 0, + }, + }, + } + tc.blockStore[parentSeq] = &outerBlock{block: parent} + + // buildAndVerify constructs the next collecting block on top of prev, verifies it, and stores it so it + // can serve as the next parent (and as a back-pointer target for the aux info history). + buildAndVerify := func(seq uint64, prev StateMachineBlock) *StateMachineBlock { + tc.blockBuilder.block = &InnerBlock{TS: time.Now(), BlockHeight: seq, Bytes: []byte{byte(seq)}} + md := common.ProtocolMetadata{Seq: seq, Round: seq, Epoch: 1, Prev: prev.Digest()} + block, err := sm.BuildBlock(context.Background(), md, nil) + require.NoError(t, err) + require.NoError(t, sm.VerifyBlock(context.Background(), block)) + tc.blockStore[seq] = &outerBlock{block: *block} + return block + } + + // block1: the epoch already has VersionID 1 (from the parent), so the buildAndVerify reads 1 from the + // chain and generates vote-1 under VersionID 1. Being the first Generate(), we now flip the + // application's default to 2. Verifying block1 also reads VersionID 1 from the parent's aux + // info, so it passes despite the changed default. + block1 := buildAndVerify(parentSeq+1, parent) + require.Equal(t, VersionID(1), block1.Metadata.AuxiliaryInfo.VersionID) + app.defaultVersionID = 2 + + // block2, block3: the default is now 2, but each block's buildAndVerify and verify still read VersionID + // 1 back from the chain and ignore the changed default. + block2 := buildAndVerify(parentSeq+2, *block1) + require.Equal(t, VersionID(1), block2.Metadata.AuxiliaryInfo.VersionID) + + block3 := buildAndVerify(parentSeq+3, *block2) + require.Equal(t, VersionID(1), block3.Metadata.AuxiliaryInfo.VersionID) + + // block4: history [vote-0, vote-1, vote-2, vote-3] is now sufficient, so no further vote is + // generated and approvals are collected -- still under VersionID 1. + block4 := buildAndVerify(parentSeq+4, *block3) + require.Equal(t, VersionID(1), block4.Metadata.AuxiliaryInfo.VersionID) +} + +func TestCollectAuxiliaryInfo(t *testing.T) { + const versionID = VersionID(7) + + blockWithAuxInfo := func(info []byte, prevAuxInfoSeq uint64) StateMachineBlock { + return StateMachineBlock{ + Metadata: StateMachineMetadata{ + AuxiliaryInfo: &AuxiliaryInfo{ + Info: info, + PrevAuxInfoSeq: prevAuxInfoSeq, + VersionID: versionID, + }, + }, + } + } + + errRetrieval := errors.New("retrieval failed") + + // startSeq is the sequence of tt.block itself (the block collectAuxiliaryInfo starts from). + const startSeq = uint64(10) + + tests := []struct { + name string + block StateMachineBlock + blocks map[uint64]StateMachineBlock + getBlockErr error + expectedHistory [][]byte + expectedLastSeq uint64 + expectedversionID VersionID + expectedErr error + }{ + { + name: "block without auxiliary info", + block: StateMachineBlock{}, + }, + { + name: "empty info, first of epoch", + block: blockWithAuxInfo(nil, 0), + }, + { + name: "non-empty info, first of epoch", + block: blockWithAuxInfo([]byte{1}, 0), + expectedHistory: [][]byte{{1}}, + expectedLastSeq: startSeq, + expectedversionID: versionID, + }, + { + name: "empty info pointing back to non-empty info", + block: blockWithAuxInfo(nil, 3), + blocks: map[uint64]StateMachineBlock{ + 3: blockWithAuxInfo([]byte{1}, 0), + }, + expectedHistory: [][]byte{{1}}, + expectedLastSeq: 3, + expectedversionID: versionID, + }, + { + name: "history is ordered from oldest to newest", + block: blockWithAuxInfo([]byte{3}, 5), + blocks: map[uint64]StateMachineBlock{ + 5: blockWithAuxInfo([]byte{2}, 2), + 2: blockWithAuxInfo([]byte{1}, 0), + }, + expectedHistory: [][]byte{{1}, {2}, {3}}, + expectedLastSeq: startSeq, + expectedversionID: versionID, + }, + { + name: "traversal stops at a block without auxiliary info", + block: blockWithAuxInfo([]byte{2}, 4), + blocks: map[uint64]StateMachineBlock{ + 4: {}, + }, + expectedHistory: [][]byte{{2}}, + expectedLastSeq: startSeq, + expectedversionID: versionID, + }, + { + name: "block retrieval failure", + block: blockWithAuxInfo([]byte{2}, 4), + getBlockErr: errRetrieval, + expectedErr: errRetrieval, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + getBlock := func(seq uint64, _ [32]byte) (StateMachineBlock, *common.Finalization, error) { + if tt.getBlockErr != nil { + return StateMachineBlock{}, nil, tt.getBlockErr + } + block, ok := tt.blocks[seq] + require.True(t, ok, "unexpected retrieval of block at sequence %d", seq) + return block, nil, nil + } + + history, gotversionID, err := collectAuxiliaryInfo(tt.block, startSeq, getBlock, 0) + if tt.expectedErr != nil { + require.ErrorIs(t, err, tt.expectedErr) + require.ErrorIs(t, err, errAuxInfoBlockRetrieval) + return + } + require.NoError(t, err) + require.Equal(t, tt.expectedHistory, history.data) + require.Equal(t, tt.expectedLastSeq, history.lastSeq) + require.Equal(t, tt.expectedversionID, gotversionID) + }) + } +} diff --git a/msm/util_test.go b/msm/util_test.go index 5217ff25..c0465a12 100644 --- a/msm/util_test.go +++ b/msm/util_test.go @@ -6,6 +6,8 @@ package metadata import ( "bytes" "context" + "crypto/ecdsa" + "crypto/elliptic" "crypto/rand" "crypto/sha256" "encoding/asn1" @@ -22,6 +24,21 @@ import ( // Test helpers +var ( + testSK *ecdsa.PrivateKey + testPK *ecdsa.PublicKey +) + +func init() { + // We generate this key-pair to test that auxiliary info signs the right. + sk, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + panic(fmt.Sprintf("failed to generate test key: %v", err)) + } + testSK = sk + testPK = &sk.PublicKey +} + type InnerBlock struct { TS time.Time BlockHeight uint64 @@ -83,12 +100,55 @@ func (a approvalsRetriever) Approvals() ValidatorSetApprovals { return a.result } +type signer struct { +} + +func (s *signer) Sign(digest []byte) ([]byte, error) { + return testSK.Sign(rand.Reader, digest, nil) +} + +// signApproval produces a real ECDSA signature over the exact payload the production code signs +// for an epoch-transition approval (assembleApprovalToBeSigned), using the shared test key. The +// signatureVerifier above accepts it. Use this instead of placeholder signature bytes for any +// approval fixture that is expected to pass signature verification. +func signApproval(pChainHeight uint64, auxInfoDigest [32]byte) []byte { + toBeSigned, err := assembleApprovalToBeSigned(pChainHeight, auxInfoDigest) + if err != nil { + panic(fmt.Sprintf("failed to assemble approval payload: %v", err)) + } + sig, err := testSK.Sign(rand.Reader, toBeSigned, nil) + if err != nil { + panic(fmt.Sprintf("failed to sign approval: %v", err)) + } + return sig +} + type signatureVerifier struct { err error } -func (sv *signatureVerifier) VerifySignature(signature []byte, message []byte, publicKey []byte) error { - return sv.err +func (sv *signatureVerifier) VerifySignature(signature []byte, message []byte, _ []byte) error { + if sv.err != nil { + return sv.err + } + if ecdsa.VerifyASN1(testPK, message, signature) { + return nil + } + + // Maybe it's an aggregated signature? + var aggSig aggregatedSignature + _, err := asn1.Unmarshal(signature, &aggSig) + if err != nil { + return fmt.Errorf("invalid signature format: %w", err) + } + + for _, sig := range aggSig.Signatures { + if !ecdsa.VerifyASN1(testPK, message, sig) { + return fmt.Errorf("invalid signature in aggregate") + } + } + + return nil } type signatureAggregator struct { @@ -96,7 +156,7 @@ type signatureAggregator struct { totalWeight uint64 } -type aggregatrdSignature struct { +type aggregatedSignature struct { Signatures [][]byte } @@ -108,9 +168,16 @@ func (sv *signatureAggregator) AppendSignatures(existing []byte, sigs ...[]byte) all := make([][]byte, 0, len(sigs)+1) all = append(all, sigs...) if len(existing) > 0 { - all = append(all, existing) + // existing is itself a marshaled aggregate from a previous round. Flatten it into the + // component signatures instead of nesting the blob, so the aggregate stays a single level + // of individual signatures that signatureVerifier can validate one by one. + var prev aggregatedSignature + if _, err := asn1.Unmarshal(existing, &prev); err != nil { + return nil, err + } + all = append(all, prev.Signatures...) } - return asn1.Marshal(aggregatrdSignature{Signatures: all}) + return asn1.Marshal(aggregatedSignature{Signatures: all}) } func (sv *signatureAggregator) IsQuorum(signers []common.NodeID) bool { @@ -317,7 +384,10 @@ func newStateMachineWithLogger(tb testing.TB, logger common.Logger) (*StateMachi PChainProgressListener: &noOpPChainListener{}, LastNonSimplexInnerBlock: genesisBlock.InnerBlock, MyNodeID: myNodeID[:], - Signer: &testutil.TestSigner{}, + AuxiliaryInfoApp: &voteCountingAuxInfoApp{ + threshold: 2, + }, + Signer: &signer{}, ComputeICMEpoch: func(input ICMEpochInput) ICMEpochInfo { // This is just the ACP-181 implementation from avalanchego var zeroEpoch ICMEpochInfo @@ -378,56 +448,116 @@ func (failingAggregator) IsQuorum([]common.NodeID) bool { return false } -type testBlockStore map[uint64]StateMachineBlock +type noopTestAuxInfoApp struct { +} -func (bs testBlockStore) getBlock(seq uint64, _ [32]byte) (StateMachineBlock, *common.Finalization, error) { - blk, ok := bs[seq] - if !ok { - return StateMachineBlock{}, nil, fmt.Errorf("%w: block %d", common.ErrBlockNotFound, seq) - } - return blk, nil, nil +func (t *noopTestAuxInfoApp) IsLegalAppend(VersionID, NodeBLSMappings, [][]byte, []byte) error { + return nil } -type testVMBlock struct { - bytes []byte - height uint64 +func (t *noopTestAuxInfoApp) IsSufficient(VersionID, NodeBLSMappings, [][]byte) (bool, error) { + return true, nil } -func (b *testVMBlock) Digest() [32]byte { - return sha256.Sum256(b.bytes) +func (t *noopTestAuxInfoApp) Generate(VersionID, NodeBLSMappings, [][]byte) ([]byte, error) { + return nil, nil } -func (b *testVMBlock) Height() uint64 { - return b.height +func (t *noopTestAuxInfoApp) DefaultVersionID() VersionID { + return 1 } -func (b *testVMBlock) Timestamp() time.Time { - return time.Now() +type voteCountingAuxInfoApp struct { + threshold int + randomTape func() []byte } -func (b *testVMBlock) Verify(_ context.Context, _ uint64) error { +func (t *voteCountingAuxInfoApp) IsLegalAppend(_ VersionID, _ NodeBLSMappings, history [][]byte, addition []byte) error { + set := make(map[string]struct{}) + for _, item := range history { + set[string(item)] = struct{}{} + } + if _, exists := set[string(addition)]; exists { + return fmt.Errorf("duplicate addition: %s", string(addition)) + } return nil } -type testSigVerifier struct { - err error +func (t *voteCountingAuxInfoApp) IsSufficient(appID VersionID, nodes NodeBLSMappings, history [][]byte) (bool, error) { + if len(history) == 0 { + return t.threshold == 0, nil + } + addition := history[len(history)-1] + history = history[:len(history)-1] + if err := t.IsLegalAppend(appID, nodes, history, addition); err != nil { + return false, err + } + + history = append(history, addition) + set := make(map[string]struct{}) + for _, item := range history { + set[string(item)] = struct{}{} + } + final := len(set) >= t.threshold + return final, nil } -func (sv *testSigVerifier) VerifySignature(_, _, _ []byte) error { - return sv.err +func (t *voteCountingAuxInfoApp) Generate(VersionID, NodeBLSMappings, [][]byte) ([]byte, error) { + // Simulate a random node voting + if t.randomTape != nil { + return t.randomTape(), nil + } + var nodeID nodeID + rand.Read(nodeID[:]) + return nodeID[:], nil } -type testKeyAggregator struct { - err error +func (t *voteCountingAuxInfoApp) DefaultVersionID() VersionID { + return 1 +} + +// versionRecordingAuxInfoApp behaves like voteCountingAuxInfoApp (a vote is appended to the +// history on every Generate, and the history becomes sufficient once it holds `threshold` +// distinct votes), but it asserts that every method is invoked with expectedVersionID, and its +// DefaultVersionID() returns whatever defaultVersionID is currently set to. A test drives both +// fields directly to prove backward compatibility: once an epoch's auxiliary info carries a +// VersionID, every invocation keeps using that VersionID even after the default changes. +type versionRecordingAuxInfoApp struct { + t *testing.T + threshold int + votes [][]byte + defaultVersionID VersionID + expectedVersionID VersionID +} + +func (a *versionRecordingAuxInfoApp) DefaultVersionID() VersionID { + return a.defaultVersionID } -func (ka *testKeyAggregator) AggregateKeys(keys ...[]byte) ([]byte, error) { - if ka.err != nil { - return nil, ka.err +func (a *versionRecordingAuxInfoApp) IsLegalAppend(versionID VersionID, _ NodeBLSMappings, history [][]byte, addition []byte) error { + require.Equal(a.t, a.expectedVersionID, versionID) + set := make(map[string]struct{}) + for _, item := range history { + set[string(item)] = struct{}{} } - var agg []byte - for _, k := range keys { - agg = append(agg, k...) + if _, exists := set[string(addition)]; exists { + return fmt.Errorf("duplicate addition: %s", string(addition)) } - return agg, nil + return nil +} + +func (a *versionRecordingAuxInfoApp) IsSufficient(versionID VersionID, _ NodeBLSMappings, history [][]byte) (bool, error) { + require.Equal(a.t, a.expectedVersionID, versionID) + set := make(map[string]struct{}) + for _, item := range history { + set[string(item)] = struct{}{} + } + return len(set) >= a.threshold, nil +} + +func (a *versionRecordingAuxInfoApp) Generate(versionID VersionID, _ NodeBLSMappings, _ [][]byte) ([]byte, error) { + require.Equal(a.t, a.expectedVersionID, versionID) + next := a.votes[0] + a.votes = a.votes[1:] + return next, nil }