Skip to content

Commit 7342d84

Browse files
vfuscovfusco
committed
feat(prt): drain foreclosed apps to INOPERABLE
1 parent 4230d3c commit 7342d84

4 files changed

Lines changed: 485 additions & 6 deletions

File tree

‎internal/prt/handle_foreclosed_test.go‎

Lines changed: 261 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,261 @@
1+
// (c) Cartesi and individual authors (see AUTHORS)
2+
// SPDX-License-Identifier: Apache-2.0 (see LICENSE)
3+
4+
package prt
5+
6+
import (
7+
"context"
8+
"errors"
9+
"log/slog"
10+
"os"
11+
"testing"
12+
"time"
13+
14+
"github.com/cartesi/rollups-node/internal/model"
15+
"github.com/cartesi/rollups-node/internal/repository"
16+
"github.com/cartesi/rollups-node/pkg/service"
17+
"github.com/ethereum/go-ethereum/common"
18+
"github.com/stretchr/testify/assert"
19+
"github.com/stretchr/testify/mock"
20+
"github.com/stretchr/testify/require"
21+
)
22+
23+
// prtRepositoryMock is a hand-written mock for the prtRepository interface,
24+
// stubbing only the methods used by handleForeclosedApp. Unused methods
25+
// keep zero-value Return signatures so the surface compiles; if a test
26+
// accidentally invokes them, testify/mock reports an unexpected call.
27+
type prtRepositoryMock struct {
28+
mock.Mock
29+
}
30+
31+
func (m *prtRepositoryMock) HasUndrainedEpochsBeforeBlock(
32+
ctx context.Context, appID int64, blockBound uint64,
33+
) (bool, error) {
34+
args := m.Called(ctx, appID, blockBound)
35+
return args.Bool(0), args.Error(1)
36+
}
37+
38+
func (m *prtRepositoryMock) UpdateApplicationStatus(
39+
ctx context.Context, appID int64, status model.ApplicationStatus, reason *string,
40+
) error {
41+
args := m.Called(ctx, appID, status, reason)
42+
return args.Error(0)
43+
}
44+
45+
// Unused-by-this-suite methods. We satisfy the interface but each panics
46+
// loudly if invoked — handleForeclosedApp must not reach for them.
47+
func (m *prtRepositoryMock) ListApplications(
48+
ctx context.Context, f repository.ApplicationFilter, p repository.Pagination, descending bool,
49+
) ([]*model.Application, uint64, error) {
50+
args := m.Called(ctx, f, p, descending)
51+
return args.Get(0).([]*model.Application), args.Get(1).(uint64), args.Error(2)
52+
}
53+
func (m *prtRepositoryMock) ListEpochs(
54+
context.Context, string, repository.EpochFilter, repository.Pagination, bool,
55+
) ([]*model.Epoch, uint64, error) {
56+
panic("unexpected ListEpochs")
57+
}
58+
func (m *prtRepositoryMock) GetEpoch(context.Context, string, uint64) (*model.Epoch, error) {
59+
panic("unexpected GetEpoch")
60+
}
61+
func (m *prtRepositoryMock) UpdateEpochStatus(context.Context, string, *model.Epoch) error {
62+
panic("unexpected UpdateEpochStatus")
63+
}
64+
func (m *prtRepositoryMock) CreateTournament(context.Context, string, *model.Tournament) error {
65+
panic("unexpected CreateTournament")
66+
}
67+
func (m *prtRepositoryMock) GetTournament(context.Context, string, string) (*model.Tournament, error) {
68+
panic("unexpected GetTournament")
69+
}
70+
func (m *prtRepositoryMock) UpdateTournament(context.Context, string, *model.Tournament) error {
71+
panic("unexpected UpdateTournament")
72+
}
73+
func (m *prtRepositoryMock) ListTournaments(
74+
context.Context, string, repository.TournamentFilter, repository.Pagination, bool,
75+
) ([]*model.Tournament, uint64, error) {
76+
panic("unexpected ListTournaments")
77+
}
78+
func (m *prtRepositoryMock) StoreTournamentEvents(
79+
context.Context, int64, []*model.Commitment, []*model.Match,
80+
[]*model.MatchAdvanced, []*model.Match, uint64,
81+
) error {
82+
panic("unexpected StoreTournamentEvents")
83+
}
84+
func (m *prtRepositoryMock) GetCommitment(context.Context, string, uint64, string, string) (*model.Commitment, error) {
85+
panic("unexpected GetCommitment")
86+
}
87+
func (m *prtRepositoryMock) SaveNodeConfigRaw(context.Context, string, []byte) error {
88+
panic("unexpected SaveNodeConfigRaw")
89+
}
90+
func (m *prtRepositoryMock) LoadNodeConfigRaw(context.Context, string) ([]byte, time.Time, time.Time, error) {
91+
panic("unexpected LoadNodeConfigRaw")
92+
}
93+
94+
// newPRTServiceMock builds a minimal Service wired to a prtRepositoryMock.
95+
// Only the fields handleForeclosedApp reaches for are populated.
96+
func newPRTServiceMock() (*Service, *prtRepositoryMock) {
97+
repo := &prtRepositoryMock{}
98+
s := &Service{
99+
Service: service.Service{
100+
Logger: slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelDebug})),
101+
},
102+
repository: repo,
103+
}
104+
return s, repo
105+
}
106+
107+
func prtForeclosedApp(id int64, block uint64) *model.Application {
108+
txHash := common.HexToHash("0xcafe")
109+
return &model.Application{
110+
ID: id,
111+
Name: "prt-app",
112+
IApplicationAddress: common.BigToAddress(common.Big1),
113+
ConsensusType: model.Consensus_PRT,
114+
Enabled: true,
115+
Status: model.ApplicationStatus_Foreclosed,
116+
ForecloseBlock: block,
117+
ForecloseTransaction: &txHash,
118+
// LastEpochCheckBlock defaults to the foreclose block so callers
119+
// who don't care about the bootstrap guard skip past it. Tests
120+
// that exercise the guard override this field explicitly.
121+
LastEpochCheckBlock: block,
122+
}
123+
}
124+
125+
// TestHandleForeclosedApp_NoOpWhenForecloseBlockZero verifies the guard at
126+
// the top of handleForeclosedApp. The PRT Tick passes every running app
127+
// through this function; only those with a non-zero ForecloseBlock should
128+
// drive any work.
129+
func TestHandleForeclosedApp_NoOpWhenForecloseBlockZero(t *testing.T) {
130+
s, r := newPRTServiceMock()
131+
defer r.AssertExpectations(t)
132+
133+
app := &model.Application{ID: 1, ConsensusType: model.Consensus_PRT}
134+
require.NoError(t, s.handleForeclosedApp(context.Background(), app))
135+
}
136+
137+
func TestGetAllRunningApplications_UsesPRTTickFilter(t *testing.T) {
138+
r := &prtRepositoryMock{}
139+
r.On("ListApplications",
140+
mock.Anything,
141+
mock.MatchedBy(func(f repository.ApplicationFilter) bool {
142+
return f.Enabled != nil && *f.Enabled &&
143+
f.ConsensusType != nil && *f.ConsensusType == model.Consensus_PRT &&
144+
assert.ElementsMatch(t,
145+
[]model.ApplicationStatus{model.ApplicationStatus_OK, model.ApplicationStatus_Foreclosed},
146+
f.Statuses,
147+
)
148+
}),
149+
repository.Pagination{},
150+
false,
151+
).Return([]*model.Application{}, uint64(0), nil).Once()
152+
153+
_, _, err := getAllRunningApplications(context.Background(), r)
154+
require.NoError(t, err)
155+
r.AssertExpectations(t)
156+
}
157+
158+
// TestHandleForeclosedApp_DefersWhenUndrained verifies the
159+
// pre-foreclosure-work guard. While the advancer/validator have epochs to
160+
// process before the foreclose block, the PRT app must keep its current
161+
// status. Marking it INOPERABLE early would lose the last machine state needed
162+
// to settle any in-flight tournament.
163+
func TestHandleForeclosedApp_DefersWhenUndrained(t *testing.T) {
164+
s, r := newPRTServiceMock()
165+
defer r.AssertExpectations(t)
166+
167+
app := prtForeclosedApp(1, 100)
168+
r.On("HasUndrainedEpochsBeforeBlock",
169+
mock.Anything, app.ID, app.ForecloseBlock,
170+
).Return(true, nil).Once()
171+
// No UpdateApplicationStatus expectation — see TestProcessForeclosedApps_DefersWhenUndrained
172+
// in the claimer suite for the equivalent reasoning.
173+
174+
require.NoError(t, s.handleForeclosedApp(context.Background(), app))
175+
}
176+
177+
// TestHandleForeclosedApp_NoTransitionWhenDrained verifies that once the
178+
// narrow drain gate clears, handleForeclosedApp is a no-op. No
179+
// UpdateApplicationStatus call fires — the PRT app keeps status FORECLOSED
180+
// with foreclose_block set. evmreader picks up the post-foreclosure
181+
// observation work from here.
182+
//
183+
// The mock has no UpdateApplicationStatus expectation registered;
184+
// testify/mock fails the test on an unexpected call, so any regression that
185+
// re-introduces a terminal-state transition trips this test loudly.
186+
func TestHandleForeclosedApp_NoTransitionWhenDrained(t *testing.T) {
187+
s, r := newPRTServiceMock()
188+
defer r.AssertExpectations(t)
189+
190+
app := prtForeclosedApp(1, 100)
191+
r.On("HasUndrainedEpochsBeforeBlock",
192+
mock.Anything, app.ID, app.ForecloseBlock,
193+
).Return(false, nil).Once()
194+
195+
// No UpdateApplicationStatus expectation — the assertion is by negation.
196+
197+
require.NoError(t, s.handleForeclosedApp(context.Background(), app))
198+
}
199+
200+
// TestHandleForeclosedApp_SurfacesDrainCheckError verifies the surrounding
201+
// behavior on transient repository failures: the error must propagate so
202+
// the Tick's err slice marks the app as in trouble; the app keeps its current
203+
// status for retry on the next tick.
204+
func TestHandleForeclosedApp_SurfacesDrainCheckError(t *testing.T) {
205+
s, r := newPRTServiceMock()
206+
defer r.AssertExpectations(t)
207+
208+
app := prtForeclosedApp(1, 100)
209+
dbErr := errors.New("connection refused")
210+
r.On("HasUndrainedEpochsBeforeBlock",
211+
mock.Anything, app.ID, app.ForecloseBlock,
212+
).Return(false, dbErr).Once()
213+
214+
err := s.handleForeclosedApp(context.Background(), app)
215+
require.Error(t, err)
216+
assert.ErrorIs(t, err, dbErr)
217+
}
218+
219+
// TestHandleForeclosedApp_DefersWhenStillBackfilling verifies the
220+
// bootstrap-readiness guard. When a freshly registered PRT app encounters
221+
// an already-foreclosed contract, evmreader sets ForecloseBlock before
222+
// checkForEpochsAndInputs has ingested any historical sealed epochs. The
223+
// drain gate would then see an empty input table and incorrectly return
224+
// false, making the app look drained before any pre-foreclosure epoch is
225+
// observed locally. The guard must defer the drain check until
226+
// LastEpochCheckBlock >= ForecloseBlock.
227+
//
228+
// The mock has no HasUndrainedEpochsBeforeBlock or UpdateApplicationStatus
229+
// expectation registered; testify/mock panics on an unexpected call, so
230+
// either reach attempt fails the test loudly.
231+
func TestHandleForeclosedApp_DefersWhenStillBackfilling(t *testing.T) {
232+
s, r := newPRTServiceMock()
233+
defer r.AssertExpectations(t)
234+
235+
app := prtForeclosedApp(1, 100)
236+
app.LastEpochCheckBlock = 50 // scanner is well below the foreclose block
237+
238+
require.NoError(t, s.handleForeclosedApp(context.Background(), app))
239+
}
240+
241+
// TestHandleForeclosedApp_ProceedsAfterBackfillCatchesUp verifies the
242+
// guard does not over-defer. Once LastEpochCheckBlock reaches the
243+
// foreclose block, the gate is consulted normally; on a "drained=false"
244+
// response the function returns nil silently (no terminal action — see
245+
// TestHandleForeclosedApp_NoTransitionWhenDrained).
246+
func TestHandleForeclosedApp_ProceedsAfterBackfillCatchesUp(t *testing.T) {
247+
s, r := newPRTServiceMock()
248+
defer r.AssertExpectations(t)
249+
250+
app := prtForeclosedApp(1, 100)
251+
app.LastEpochCheckBlock = app.ForecloseBlock // exact-boundary case: caught up
252+
253+
r.On("HasUndrainedEpochsBeforeBlock",
254+
mock.Anything, app.ID, app.ForecloseBlock,
255+
).Return(false, nil).Once()
256+
257+
// No UpdateApplicationStatus expectation — the gate has cleared but the
258+
// function does not transition the app.
259+
260+
require.NoError(t, s.handleForeclosedApp(context.Background(), app))
261+
}

‎internal/prt/prt.go‎

Lines changed: 43 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,8 @@ import (
2727
type prtRepository interface {
2828
ListApplications(ctx context.Context, f repository.ApplicationFilter,
2929
p repository.Pagination, descending bool) ([]*Application, uint64, error)
30-
UpdateApplicationState(ctx context.Context, appID int64, state ApplicationState, reason *string) error
30+
UpdateApplicationStatus(ctx context.Context, appID int64, status ApplicationStatus, reason *string) error
31+
HasUndrainedEpochsBeforeBlock(ctx context.Context, appID int64, blockBound uint64) (bool, error)
3132

3233
ListEpochs(ctx context.Context, nameOrAddress string, f repository.EpochFilter,
3334
p repository.Pagination, descending bool) ([]*Epoch, uint64, error)
@@ -78,8 +79,15 @@ func (f *DefaultAdapterFactory) CreateDaveConsensusAdapter(addr common.Address)
7879
}
7980

8081
func getAllRunningApplications(ctx context.Context, r prtRepository) ([]*Application, uint64, error) {
81-
f := repository.ApplicationFilter{State: Pointer(ApplicationState_Enabled), ConsensusType: Pointer(Consensus_PRT)}
82-
return r.ListApplications(ctx, f, repository.Pagination{}, false)
82+
return r.ListApplications(ctx, prtTickApplicationsFilter(), repository.Pagination{}, false)
83+
}
84+
85+
func prtTickApplicationsFilter() repository.ApplicationFilter {
86+
return repository.ApplicationFilter{
87+
Enabled: new(true),
88+
Statuses: []ApplicationStatus{ApplicationStatus_OK, ApplicationStatus_Foreclosed},
89+
ConsensusType: new(Consensus_PRT),
90+
}
8391
}
8492

8593
func getAllClaimComputedEpochs(ctx context.Context, r prtRepository, nameOrAddress string) ([]*Epoch, uint64, error) {
@@ -448,7 +456,7 @@ func (s *Service) checkEpochs(ctx context.Context, app *Application, mostRecentB
448456
"application", app.Name,
449457
"epoch", epoch.Index,
450458
"event_block_number", event.Raw.BlockNumber,
451-
"claim_hash", fmt.Sprintf("%x", event.OutputsMerkleRoot),
459+
"outputs_merkle_root", fmt.Sprintf("%x", event.OutputsMerkleRoot),
452460
"tx", epoch.ClaimTransactionHash,
453461
)
454462

@@ -712,6 +720,22 @@ func (s *Service) trySettle(ctx context.Context, app *Application, mostRecentBlo
712720
"epoch_index", result.EpochNumber.Uint64())
713721
return nil
714722
}
723+
// Transient broadcast race: the chain has already mined a tx with
724+
// this EOA's nonce, so this attempt is rejected before execution.
725+
// Most commonly hit straddling a node restart — the prior process
726+
// broadcast Settle (or some other tx) that landed, but the
727+
// post-restart PendingNonceAt has not yet caught up. The next tick's
728+
// IsEpochSettled check reads chain state at a fresh block and
729+
// short-circuits if our prior Settle actually mined; otherwise a
730+
// new broadcast goes out with a fresh nonce.
731+
if ethutil.IsNonceTooLowError(err) {
732+
s.Logger.Info(
733+
"Settle broadcast rejected with 'nonce too low'; "+
734+
"deferring to the next tick's IsEpochSettled reconciliation",
735+
"application", app.Name,
736+
"epoch_index", result.EpochNumber.Uint64())
737+
return nil
738+
}
715739
s.Logger.Error("failed to send Settle transaction", "application", app.Name,
716740
"epoch_index", result.EpochNumber.Uint64(), "error", err)
717741
return err
@@ -853,6 +877,21 @@ func (s *Service) reactToTournament(ctx context.Context, app *Application, mostR
853877
"tournament", epoch.TournamentAddress.Hex(), "commitment", epoch.Commitment.Hex())
854878
return nil
855879
}
880+
// Transient broadcast race: a tx with this EOA's nonce is already
881+
// mined. The next tick's IsCommitmentJoined check will reconcile
882+
// against the propagated chain state and short-circuit if our prior
883+
// JoinTournament landed; otherwise a new broadcast goes out with a
884+
// fresh nonce.
885+
if ethutil.IsNonceTooLowError(err) {
886+
s.Logger.Info(
887+
"JoinTournament broadcast rejected with 'nonce too low'; "+
888+
"deferring to the next tick's IsCommitmentJoined reconciliation",
889+
"application", app.Name,
890+
"epoch_index", currentEpochIndex,
891+
"tournament", epoch.TournamentAddress.Hex(),
892+
"commitment", epoch.Commitment.Hex())
893+
return nil
894+
}
856895
s.Logger.Error("failed to send join tournament transaction", "application", app.Name,
857896
"epoch_index", currentEpochIndex, "error", err)
858897
return err

0 commit comments

Comments
 (0)