Skip to content

DAOS-19036 dtx: handle DTX race issues #18428

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
daltonbohning merged 2 commits into from
Jun 18, 2026
Merged

DAOS-19036 dtx: handle DTX race issues #18428

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
9a20cab
DAOS-19036 dtx: handle DTX race issues
Nasf-Fan Jun 3, 2026
213f1fa
Merge remote-tracking branch 'origin/master' into Nasf-Fan/DAOS-19036_1
daltonbohning Jun 16, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 13 additions & 10 deletions src/dtx/dtx_coll.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,12 +38,13 @@
*/

struct dtx_coll_local_args {
uuid_t dcla_po_uuid;
uuid_t dcla_co_uuid;
struct dtx_id dcla_xid;
daos_epoch_t dcla_epoch;
uint32_t dcla_opc;
int *dcla_results;
uuid_t dcla_po_uuid;
uuid_t dcla_co_uuid;
struct dtx_id dcla_xid;
daos_epoch_t dcla_epoch;
uint32_t dcla_ver;
uint32_t dcla_opc;
int *dcla_results;
};

void
Expand Down Expand Up @@ -361,7 +362,7 @@ dtx_coll_local_one(void *args)
rc = vos_dtx_commit(cont->sc_hdl, &dcla->dcla_xid, 1, false, NULL);
break;
case DTX_COLL_ABORT:
rc = vos_dtx_abort(cont->sc_hdl, &dcla->dcla_xid, dcla->dcla_epoch);
rc = vos_dtx_abort(cont->sc_hdl, &dcla->dcla_xid, dcla->dcla_epoch, dcla->dcla_ver);
break;
case DTX_COLL_CHECK:
rc = vos_dtx_check(cont->sc_hdl, &dcla->dcla_xid, NULL, NULL, NULL, false);
Expand Down Expand Up @@ -397,7 +398,8 @@ dtx_coll_local_one(void *args)

int
dtx_coll_local_exec(uuid_t po_uuid, uuid_t co_uuid, struct dtx_id *xid, daos_epoch_t epoch,
uint32_t opc, uint32_t bitmap_sz, uint8_t *bitmap, int **p_results)
uint32_t version, uint32_t opc, uint32_t bitmap_sz, uint8_t *bitmap,
int **p_results)
{
struct dtx_coll_local_args dcla = { 0 };
struct dss_coll_ops coll_ops = { 0 };
Expand All @@ -410,9 +412,10 @@ dtx_coll_local_exec(uuid_t po_uuid, uuid_t co_uuid, struct dtx_id *xid, daos_epo

uuid_copy(dcla.dcla_po_uuid, po_uuid);
uuid_copy(dcla.dcla_co_uuid, co_uuid);
dcla.dcla_xid = *xid;
dcla.dcla_xid = *xid;
dcla.dcla_epoch = epoch;
dcla.dcla_opc = opc;
dcla.dcla_ver = version;
dcla.dcla_opc = opc;

coll_ops.co_func = dtx_coll_local_one;
coll_args.ca_func_args = &dcla;
Expand Down
3 changes: 2 additions & 1 deletion src/dtx/dtx_internal.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -304,7 +304,8 @@ int dtx_coll_prep(uuid_t po_uuid, daos_unit_oid_t oid, struct dtx_id *xid,
struct dtx_memberships *mbs, uint32_t my_tgtid, uint32_t dtx_ver,
uint32_t pm_ver, bool for_check, bool need_hint, struct dtx_coll_entry **p_dce);
int dtx_coll_local_exec(uuid_t po_uuid, uuid_t co_uuid, struct dtx_id *xid, daos_epoch_t epoch,
uint32_t opc, uint32_t bitmap_sz, uint8_t *bitmap, int **p_results);
uint32_t version, uint32_t opc, uint32_t bitmap_sz, uint8_t *bitmap,
int **p_results);
/* clang-format on */

enum dtx_status_handle_result {
Expand Down
37 changes: 27 additions & 10 deletions src/dtx/dtx_resync.c
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
/**
* (C) Copyright 2019-2024 Intel Corporation.
* (C) Copyright 2025 Hewlett Packard Enterprise Development LP
* (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP
*
* SPDX-License-Identifier: BSD-2-Clause-Patent
*/
Expand Down Expand Up @@ -36,11 +36,12 @@ struct dtx_resync_head {
};

struct dtx_resync_args {
struct ds_cont_child *cont;
struct dtx_resync_head tables;
daos_epoch_t epoch;
uint32_t resync_version;
uint32_t discard_version;
struct ds_cont_child *cont;
struct dtx_resync_head tables;
daos_epoch_t epoch;
uint32_t resync_version;
uint32_t discard_version;
bool for_all;
};

static inline void
Expand Down Expand Up @@ -329,8 +330,8 @@ dtx_status_handle_one(struct ds_cont_child *cont, struct dtx_entry *dte, daos_un
else
rc = dtx_abort(cont, dte, epoch);

D_DEBUG(DB_TRACE, "As new leader for DTX "DF_DTI", abort it (2): "DF_RC"\n",
DP_DTI(&dte->dte_xid), DP_RC(rc));
DL_CDEBUG(rc != 0, DLOG_ERR, DB_TRACE, rc,
"As new leader for DTX " DF_DTI ", abort it (2)", DP_DTI(&dte->dte_xid));

if (rc < 0) {
if (err != NULL)
Expand Down Expand Up @@ -392,7 +393,8 @@ dtx_status_handle(struct dtx_resync_args *dra)
again:
d_list_for_each_entry_safe(dre, next, &drh->drh_list, dre_link) {
if (dre->dre_dte.dte_ver < dra->discard_version) {
err = vos_dtx_abort(cont->sc_hdl, &dre->dre_xid, dre->dre_epoch);
err = vos_dtx_abort(cont->sc_hdl, &dre->dre_xid, dre->dre_epoch,
dre->dre_dte.dte_ver);
if (err == -DER_NONEXIST)
err = 0;
if (err != 0)
Expand Down Expand Up @@ -538,7 +540,17 @@ dtx_iter_cb(uuid_t co_uuid, vos_iter_entry_t *ent, void *args)
if (dra->resync_version == dra->discard_version)
return 0;

/* Skip unprepared entry which version is at least not older than discard version. */
/*
* The DTX version maybe refreshed via obj_handle_resend(). It means that either the
* DTX is generated against the latest pool map or related IO RPC is resent by client
* after pool map changed. Under both cases, the DTX resync that is triggered for pool
* map changes (@for_all is false) should not handle such DTX to avoid making conflict
* commit/abort decision (against regular IO handler) by race.
*/
if ((ent->ie_dtx_ver > dra->resync_version) ||
(ent->ie_dtx_ver == dra->resync_version && !dra->for_all))

@liuxuezhao liuxuezhao Jun 15, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

just confirm,
L551 looks a little bit confuse, for example pool map change to ver 10, and do an update dtx@pm_ver_10, and then dtx_resync @pm_ver_10. Will above code ignore the update dtx@pm_ver_10? if ignore it, how to ensure dtx_resync()'s semantics?
why dtx_resync()'s "block" parameter is true for rebuild, and false for ds_cont_local_open()?

@Nasf-Fan Nasf-Fan Jun 16, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

For the update with pm_ver@10, since the latest pm_ver is also 10, means that the update is against the latest pm_ver, so the DTX leader is alive and will be in charge of committing or aborting related DTX entry.

The parameter block is used to distinguish whether for rebuild or for regular container open. For rebuild case, the DTX (new) leader will handle related DTX entry; for container open case, since related DTX leader ULT is already exit and nobody will sponsor commit or abort action, then DTX resync needs to do that.

return 0;

if (ent->ie_dtx_tgt_cnt == 0)
return 0;

Expand Down Expand Up @@ -665,6 +677,11 @@ dtx_resync(daos_handle_t po_hdl, struct ds_cont_child *cont, uint32_t ver, bool
D_INIT_LIST_HEAD(&dra.tables.drh_list);
dra.tables.drh_count = 0;

if (block)
dra.for_all = false;
else
dra.for_all = true;

/*
* Trigger DTX reindex. That will avoid DTX_CHECK from others being blocked.
* It is harmless even if (committed) DTX entries have already been re-indexed.
Expand Down
19 changes: 10 additions & 9 deletions src/dtx/dtx_rpc.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -963,7 +963,7 @@ dtx_abort(struct ds_cont_child *cont, struct dtx_entry *dte, daos_epoch_t epoch)
* to resend sometime later.
*/
if (epoch != 0)
rc1 = vos_dtx_abort(cont->sc_hdl, &dte->dte_xid, epoch);
rc1 = vos_dtx_abort(cont->sc_hdl, &dte->dte_xid, epoch, dte->dte_ver);
else
rc1 = vos_dtx_set_flags(cont->sc_hdl, &dte->dte_xid, 1, DTE_CORRUPTED);
if (rc1 > 0 || rc1 == -DER_NONEXIST)
Expand Down Expand Up @@ -1235,7 +1235,8 @@ dtx_refresh_internal(struct ds_cont_child *cont, int *check_count, d_list_t *che
d_list_del(&dsp->dsp_link);
dtx_dsp_free(dsp);
} else {
rc1 = vos_dtx_abort(cont->sc_hdl, &dsp->dsp_xid, dsp->dsp_epoch);
rc1 = vos_dtx_abort(cont->sc_hdl, &dsp->dsp_xid, dsp->dsp_epoch,
dsp->dsp_version);
D_ASSERT(rc1 != -DER_NO_PERM);

if (rc1 == 0 || !for_io) {
Expand Down Expand Up @@ -1655,8 +1656,8 @@ dtx_coll_commit(struct ds_cont_child *cont, struct dtx_coll_entry *dce, struct d
if (dce->dce_bitmap != NULL) {
clrbit(dce->dce_bitmap, dss_get_module_info()->dmi_tgt_id);
len = dtx_coll_local_exec(cont->sc_pool_uuid, cont->sc_uuid, &dce->dce_xid, 0,
DTX_COLL_COMMIT, dce->dce_bitmap_sz, dce->dce_bitmap,
&results);
dce->dce_ver, DTX_COLL_COMMIT, dce->dce_bitmap_sz,
dce->dce_bitmap, &results);
if (len < 0) {
rc1 = len;
} else {
Expand Down Expand Up @@ -1738,8 +1739,8 @@ dtx_coll_abort(struct ds_cont_child *cont, struct dtx_coll_entry *dce, daos_epoc
if (dce->dce_bitmap != NULL) {
clrbit(dce->dce_bitmap, dss_get_module_info()->dmi_tgt_id);
len = dtx_coll_local_exec(cont->sc_pool_uuid, cont->sc_uuid, &dce->dce_xid, epoch,
DTX_COLL_ABORT, dce->dce_bitmap_sz, dce->dce_bitmap,
&results);
dce->dce_ver, DTX_COLL_ABORT, dce->dce_bitmap_sz,
dce->dce_bitmap, &results);
if (len < 0) {
rc1 = len;
} else {
Expand All @@ -1759,7 +1760,7 @@ dtx_coll_abort(struct ds_cont_child *cont, struct dtx_coll_entry *dce, daos_epoc
}

if (epoch != 0)
rc2 = vos_dtx_abort(cont->sc_hdl, &dce->dce_xid, epoch);
rc2 = vos_dtx_abort(cont->sc_hdl, &dce->dce_xid, epoch, dce->dce_ver);
else
rc2 = vos_dtx_set_flags(cont->sc_hdl, &dce->dce_xid, 1, DTE_CORRUPTED);
if (rc2 > 0 || rc2 == -DER_NONEXIST)
Expand Down Expand Up @@ -1795,8 +1796,8 @@ dtx_coll_check(struct ds_cont_child *cont, struct dtx_coll_entry *dce, daos_epoc

if (dce->dce_bitmap != NULL) {
len = dtx_coll_local_exec(cont->sc_pool_uuid, cont->sc_uuid, &dce->dce_xid, epoch,
DTX_COLL_CHECK, dce->dce_bitmap_sz, dce->dce_bitmap,
&results);
dce->dce_ver, DTX_COLL_CHECK, dce->dce_bitmap_sz,
dce->dce_bitmap, &results);
if (len < 0) {
rc1 = len;
} else {
Expand Down
4 changes: 2 additions & 2 deletions src/dtx/dtx_srv.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -239,7 +239,7 @@ dtx_handler(crt_rpc_t *rpc)

rc = vos_dtx_abort(cont->sc_hdl,
(struct dtx_id *)din->di_dtx_array.ca_arrays,
din->di_epoch);
din->di_epoch, din->di_version);
} else {
rc = vos_dtx_set_flags(cont->sc_hdl,
(struct dtx_id *)din->di_dtx_array.ca_arrays,
Expand Down Expand Up @@ -462,7 +462,7 @@ dtx_coll_handler(crt_rpc_t *rpc)
}

len = dtx_coll_local_exec(dci->dci_po_uuid, dci->dci_co_uuid, &dci->dci_xid, dci->dci_epoch,
opc, bitmap_sz, bitmap, &results);
dci->dci_version, opc, bitmap_sz, bitmap, &results);
if (len < 0)
D_GOTO(out, rc = len);

Expand Down
9 changes: 5 additions & 4 deletions src/include/daos_srv/vos.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -199,14 +199,15 @@ vos_dtx_commit(daos_handle_t coh, struct dtx_id dtis[], int count, bool keep_act
/**
* Abort the specified DTXs.
*
* \param coh [IN] Container open handle.
* \param dti [IN] The DTX identifiers to be aborted.
* \param epoch [IN] The max epoch for the DTX to be aborted.
* \param coh [IN] Container open handle.
* \param dti [IN] The DTX identifiers to be aborted.
* \param epoch [IN] The max epoch for the DTX to be aborted.
* \param version [IN] The max version for the DTX to be aborted.
*
* \return Zero on success, negative value if error.
*/
int
vos_dtx_abort(daos_handle_t coh, struct dtx_id *dti, daos_epoch_t epoch);
vos_dtx_abort(daos_handle_t coh, struct dtx_id *dti, daos_epoch_t epoch, uint32_t version);

/**
* Discard the active DTX entry's records if invalid.
Expand Down
Loading
Loading