target: encapsulate smp_mb__after_atomic()

The target code has a rather generous helping of smp_mb__after_atomic()
throughout the code base.  Most atomic operations were followed by one
and none were preceded by smp_mb__before_atomic(), nor accompanied by a
comment explaining the need for a barrier.

Instead of trying to prove for every case whether or not it is needed,
this patch introduces atomic_inc_mb() and atomic_dec_mb(), which
explicitly include the memory barriers before and after the atomic
operation.  For now they are defined in a target header, although they
could be of general use.

Most of the existing atomic/mb combinations were replaced by the new
helpers.  In a few cases the atomic was sandwiched in
spin_lock/spin_unlock and I simply removed the barrier.

I suspect that in most cases the correct conversion would have been to
drop the barrier.  I also suspect that a few cases exist where a) the
barrier was necessary and b) a second barrier before the atomic would
have been necessary and got added by this patch.

Signed-off-by: Joern Engel <joern@logfs.org>
Signed-off-by: Nicholas Bellinger <nab@linux-iscsi.org>

diff --git a/drivers/target/loopback/tcm_loop.c b/drivers/target/loopback/tcm_loop.c
index 340de9d..a7f6dc6 100644 (file)
--- a/drivers/target/loopback/tcm_loop.c
+++ b/drivers/target/loopback/tcm_loop.c
@@ -960,8 +960,7 @@ static int tcm_loop_port_link(
                                struct tcm_loop_tpg, tl_se_tpg);
        struct tcm_loop_hba *tl_hba = tl_tpg->tl_hba;
 
-       atomic_inc(&tl_tpg->tl_tpg_port_count);
-       smp_mb__after_atomic();
+       atomic_inc_mb(&tl_tpg->tl_tpg_port_count);
        /*
         * Add Linux/SCSI struct scsi_device by HCTL
         */
@@ -995,8 +994,7 @@ static void tcm_loop_port_unlink(
        scsi_remove_device(sd);
        scsi_device_put(sd);
 
-       atomic_dec(&tl_tpg->tl_tpg_port_count);
-       smp_mb__after_atomic();
+       atomic_dec_mb(&tl_tpg->tl_tpg_port_count);
 
        pr_debug("TCM_Loop_ConfigFS: Port Unlink Successful\n");
 }

diff --git a/drivers/target/target_core_alua.c b/drivers/target/target_core_alua.c
index fbc5ebb..fb87780 100644 (file)
--- a/drivers/target/target_core_alua.c
+++ b/drivers/target/target_core_alua.c
@@ -392,8 +392,7 @@ target_emulate_set_target_port_groups(struct se_cmd *cmd)
                                if (tg_pt_id != tg_pt_gp->tg_pt_gp_id)
                                        continue;
 
-                               atomic_inc(&tg_pt_gp->tg_pt_gp_ref_cnt);
-                               smp_mb__after_atomic();
+                               atomic_inc_mb(&tg_pt_gp->tg_pt_gp_ref_cnt);
 
                                spin_unlock(&dev->t10_alua.tg_pt_gps_lock);
 
@@ -403,8 +402,7 @@ target_emulate_set_target_port_groups(struct se_cmd *cmd)
                                        found = true;
 
                                spin_lock(&dev->t10_alua.tg_pt_gps_lock);
-                               atomic_dec(&tg_pt_gp->tg_pt_gp_ref_cnt);
-                               smp_mb__after_atomic();
+                               atomic_dec_mb(&tg_pt_gp->tg_pt_gp_ref_cnt);
                                break;
                        }
                        spin_unlock(&dev->t10_alua.tg_pt_gps_lock);
@@ -998,8 +996,7 @@ static void core_alua_do_transition_tg_pt_work(struct work_struct *work)
                 * every I_T nexus other than the I_T nexus on which the SET
                 * TARGET PORT GROUPS command
                 */
-               atomic_inc(&mem->tg_pt_gp_mem_ref_cnt);
-               smp_mb__after_atomic();
+               atomic_inc_mb(&mem->tg_pt_gp_mem_ref_cnt);
                spin_unlock(&tg_pt_gp->tg_pt_gp_lock);
 
                spin_lock_bh(&port->sep_alua_lock);
@@ -1028,8 +1025,7 @@ static void core_alua_do_transition_tg_pt_work(struct work_struct *work)
                spin_unlock_bh(&port->sep_alua_lock);
 
                spin_lock(&tg_pt_gp->tg_pt_gp_lock);
-               atomic_dec(&mem->tg_pt_gp_mem_ref_cnt);
-               smp_mb__after_atomic();
+               atomic_dec_mb(&mem->tg_pt_gp_mem_ref_cnt);
        }
        spin_unlock(&tg_pt_gp->tg_pt_gp_lock);
        /*
@@ -1063,7 +1059,6 @@ static void core_alua_do_transition_tg_pt_work(struct work_struct *work)
                core_alua_dump_state(tg_pt_gp->tg_pt_gp_alua_pending_state));
        spin_lock(&dev->t10_alua.tg_pt_gps_lock);
        atomic_dec(&tg_pt_gp->tg_pt_gp_ref_cnt);
-       smp_mb__after_atomic();
        spin_unlock(&dev->t10_alua.tg_pt_gps_lock);
 
        if (tg_pt_gp->tg_pt_gp_transition_complete)
@@ -1125,7 +1120,6 @@ static int core_alua_do_transition_tg_pt(
         */
        spin_lock(&dev->t10_alua.tg_pt_gps_lock);
        atomic_inc(&tg_pt_gp->tg_pt_gp_ref_cnt);
-       smp_mb__after_atomic();
        spin_unlock(&dev->t10_alua.tg_pt_gps_lock);
 
        if (!explicit && tg_pt_gp->tg_pt_gp_implicit_trans_secs) {
@@ -1168,7 +1162,6 @@ int core_alua_do_port_transition(
        spin_lock(&local_lu_gp_mem->lu_gp_mem_lock);
        lu_gp = local_lu_gp_mem->lu_gp;
        atomic_inc(&lu_gp->lu_gp_ref_cnt);
-       smp_mb__after_atomic();
        spin_unlock(&local_lu_gp_mem->lu_gp_mem_lock);
        /*
         * For storage objects that are members of the 'default_lu_gp',
@@ -1184,8 +1177,7 @@ int core_alua_do_port_transition(
                l_tg_pt_gp->tg_pt_gp_alua_nacl = l_nacl;
                rc = core_alua_do_transition_tg_pt(l_tg_pt_gp,
                                                   new_state, explicit);
-               atomic_dec(&lu_gp->lu_gp_ref_cnt);
-               smp_mb__after_atomic();
+               atomic_dec_mb(&lu_gp->lu_gp_ref_cnt);
                return rc;
        }
        /*
@@ -1198,8 +1190,7 @@ int core_alua_do_port_transition(
                                lu_gp_mem_list) {
 
                dev = lu_gp_mem->lu_gp_mem_dev;
-               atomic_inc(&lu_gp_mem->lu_gp_mem_ref_cnt);
-               smp_mb__after_atomic();
+               atomic_inc_mb(&lu_gp_mem->lu_gp_mem_ref_cnt);
                spin_unlock(&lu_gp->lu_gp_lock);
 
                spin_lock(&dev->t10_alua.tg_pt_gps_lock);
@@ -1227,8 +1218,7 @@ int core_alua_do_port_transition(
                                tg_pt_gp->tg_pt_gp_alua_port = NULL;
                                tg_pt_gp->tg_pt_gp_alua_nacl = NULL;
                        }
-                       atomic_inc(&tg_pt_gp->tg_pt_gp_ref_cnt);
-                       smp_mb__after_atomic();
+                       atomic_inc_mb(&tg_pt_gp->tg_pt_gp_ref_cnt);
                        spin_unlock(&dev->t10_alua.tg_pt_gps_lock);
                        /*
                         * core_alua_do_transition_tg_pt() will always return
@@ -1238,16 +1228,14 @@ int core_alua_do_port_transition(
                                        new_state, explicit);
 
                        spin_lock(&dev->t10_alua.tg_pt_gps_lock);
-                       atomic_dec(&tg_pt_gp->tg_pt_gp_ref_cnt);
-                       smp_mb__after_atomic();
+                       atomic_dec_mb(&tg_pt_gp->tg_pt_gp_ref_cnt);
                        if (rc)
                                break;
                }
                spin_unlock(&dev->t10_alua.tg_pt_gps_lock);
 
                spin_lock(&lu_gp->lu_gp_lock);
-               atomic_dec(&lu_gp_mem->lu_gp_mem_ref_cnt);
-               smp_mb__after_atomic();
+               atomic_dec_mb(&lu_gp_mem->lu_gp_mem_ref_cnt);
        }
        spin_unlock(&lu_gp->lu_gp_lock);
 
@@ -1260,8 +1248,7 @@ int core_alua_do_port_transition(
                         core_alua_dump_state(new_state));
        }
 
-       atomic_dec(&lu_gp->lu_gp_ref_cnt);
-       smp_mb__after_atomic();
+       atomic_dec_mb(&lu_gp->lu_gp_ref_cnt);
        return rc;
 }
 

diff --git a/drivers/target/target_core_device.c b/drivers/target/target_core_device.c
index e784284..f5057a2 100644 (file)
--- a/drivers/target/target_core_device.c
+++ b/drivers/target/target_core_device.c
@@ -224,8 +224,7 @@ struct se_dev_entry *core_get_se_deve_from_rtpi(
                if (port->sep_rtpi != rtpi)
                        continue;
 
-               atomic_inc(&deve->pr_ref_count);
-               smp_mb__after_atomic();
+               atomic_inc_mb(&deve->pr_ref_count);
                spin_unlock_irq(&nacl->device_list_lock);
 
                return deve;
@@ -1388,8 +1387,7 @@ int core_dev_add_initiator_node_lun_acl(
 
        spin_lock(&lun->lun_acl_lock);
        list_add_tail(&lacl->lacl_list, &lun->lun_acl_list);
-       atomic_inc(&lun->lun_acl_count);
-       smp_mb__after_atomic();
+       atomic_inc_mb(&lun->lun_acl_count);
        spin_unlock(&lun->lun_acl_lock);
 
        pr_debug("%s_TPG[%hu]_LUN[%u->%u] - Added %s ACL for "
@@ -1422,8 +1420,7 @@ int core_dev_del_initiator_node_lun_acl(
 
        spin_lock(&lun->lun_acl_lock);
        list_del(&lacl->lacl_list);
-       atomic_dec(&lun->lun_acl_count);
-       smp_mb__after_atomic();
+       atomic_dec_mb(&lun->lun_acl_count);
        spin_unlock(&lun->lun_acl_lock);
 
        core_disable_device_list_for_node(lun, NULL, lacl->mapped_lun,

diff --git a/drivers/target/target_core_pr.c b/drivers/target/target_core_pr.c
index 281d52e..48a8010 100644 (file)
--- a/drivers/target/target_core_pr.c
+++ b/drivers/target/target_core_pr.c
@@ -674,8 +674,7 @@ static struct t10_pr_registration *__core_scsi3_alloc_registration(
         */
        spin_lock(&dev->se_port_lock);
        list_for_each_entry_safe(port, port_tmp, &dev->dev_sep_list, sep_list) {
-               atomic_inc(&port->sep_tg_pt_ref_cnt);
-               smp_mb__after_atomic();
+               atomic_inc_mb(&port->sep_tg_pt_ref_cnt);
                spin_unlock(&dev->se_port_lock);
 
                spin_lock_bh(&port->sep_alua_lock);
@@ -709,8 +708,7 @@ static struct t10_pr_registration *__core_scsi3_alloc_registration(
                        if (strcmp(nacl->initiatorname, nacl_tmp->initiatorname))
                                continue;
 
-                       atomic_inc(&deve_tmp->pr_ref_count);
-                       smp_mb__after_atomic();
+                       atomic_inc_mb(&deve_tmp->pr_ref_count);
                        spin_unlock_bh(&port->sep_alua_lock);
                        /*
                         * Grab a configfs group dependency that is released
@@ -722,10 +720,8 @@ static struct t10_pr_registration *__core_scsi3_alloc_registration(
                        if (ret < 0) {
                                pr_err("core_scsi3_lunacl_depend"
                                                "_item() failed\n");
-                               atomic_dec(&port->sep_tg_pt_ref_cnt);
-                               smp_mb__after_atomic();
-                               atomic_dec(&deve_tmp->pr_ref_count);
-                               smp_mb__after_atomic();
+                               atomic_dec_mb(&port->sep_tg_pt_ref_cnt);
+                               atomic_dec_mb(&deve_tmp->pr_ref_count);
                                goto out;
                        }
                        /*
@@ -739,10 +735,8 @@ static struct t10_pr_registration *__core_scsi3_alloc_registration(
                                                nacl_tmp, deve_tmp, NULL,
                                                sa_res_key, all_tg_pt, aptpl);
                        if (!pr_reg_atp) {
-                               atomic_dec(&port->sep_tg_pt_ref_cnt);
-                               smp_mb__after_atomic();
-                               atomic_dec(&deve_tmp->pr_ref_count);
-                               smp_mb__after_atomic();
+                               atomic_dec_mb(&port->sep_tg_pt_ref_cnt);
+                               atomic_dec_mb(&deve_tmp->pr_ref_count);
                                core_scsi3_lunacl_undepend_item(deve_tmp);
                                goto out;
                        }
@@ -754,8 +748,7 @@ static struct t10_pr_registration *__core_scsi3_alloc_registration(
                spin_unlock_bh(&port->sep_alua_lock);
 
                spin_lock(&dev->se_port_lock);
-               atomic_dec(&port->sep_tg_pt_ref_cnt);
-               smp_mb__after_atomic();
+               atomic_dec_mb(&port->sep_tg_pt_ref_cnt);
        }
        spin_unlock(&dev->se_port_lock);
 
@@ -1109,8 +1102,7 @@ static struct t10_pr_registration *__core_scsi3_locate_pr_reg(
                                if (dev->dev_attrib.enforce_pr_isids)
                                        continue;
                        }
-                       atomic_inc(&pr_reg->pr_res_holders);
-                       smp_mb__after_atomic();
+                       atomic_inc_mb(&pr_reg->pr_res_holders);
                        spin_unlock(&pr_tmpl->registration_lock);
                        return pr_reg;
                }
@@ -1124,8 +1116,7 @@ static struct t10_pr_registration *__core_scsi3_locate_pr_reg(
                if (strcmp(isid, pr_reg->pr_reg_isid))
                        continue;
 
-               atomic_inc(&pr_reg->pr_res_holders);
-               smp_mb__after_atomic();
+               atomic_inc_mb(&pr_reg->pr_res_holders);
                spin_unlock(&pr_tmpl->registration_lock);
                return pr_reg;
        }
@@ -1154,8 +1145,7 @@ static struct t10_pr_registration *core_scsi3_locate_pr_reg(
 
 static void core_scsi3_put_pr_reg(struct t10_pr_registration *pr_reg)
 {
-       atomic_dec(&pr_reg->pr_res_holders);
-       smp_mb__after_atomic();
+       atomic_dec_mb(&pr_reg->pr_res_holders);
 }
 
 static int core_scsi3_check_implicit_release(
@@ -1348,8 +1338,7 @@ static void core_scsi3_tpg_undepend_item(struct se_portal_group *tpg)
        configfs_undepend_item(tpg->se_tpg_tfo->tf_subsys,
                        &tpg->tpg_group.cg_item);
 
-       atomic_dec(&tpg->tpg_pr_ref_count);
-       smp_mb__after_atomic();
+       atomic_dec_mb(&tpg->tpg_pr_ref_count);
 }
 
 static int core_scsi3_nodeacl_depend_item(struct se_node_acl *nacl)
@@ -1368,16 +1357,14 @@ static void core_scsi3_nodeacl_undepend_item(struct se_node_acl *nacl)
        struct se_portal_group *tpg = nacl->se_tpg;
 
        if (nacl->dynamic_node_acl) {
-               atomic_dec(&nacl->acl_pr_ref_count);
-               smp_mb__after_atomic();
+               atomic_dec_mb(&nacl->acl_pr_ref_count);
                return;
        }
 
        configfs_undepend_item(tpg->se_tpg_tfo->tf_subsys,
                        &nacl->acl_group.cg_item);
 
-       atomic_dec(&nacl->acl_pr_ref_count);
-       smp_mb__after_atomic();
+       atomic_dec_mb(&nacl->acl_pr_ref_count);
 }
 
 static int core_scsi3_lunacl_depend_item(struct se_dev_entry *se_deve)
@@ -1407,8 +1394,7 @@ static void core_scsi3_lunacl_undepend_item(struct se_dev_entry *se_deve)
         * For nacl->dynamic_node_acl=1
         */
        if (!lun_acl) {
-               atomic_dec(&se_deve->pr_ref_count);
-               smp_mb__after_atomic();
+               atomic_dec_mb(&se_deve->pr_ref_count);
                return;
        }
        nacl = lun_acl->se_lun_nacl;
@@ -1417,8 +1403,7 @@ static void core_scsi3_lunacl_undepend_item(struct se_dev_entry *se_deve)
        configfs_undepend_item(tpg->se_tpg_tfo->tf_subsys,
                        &lun_acl->se_lun_group.cg_item);
 
-       atomic_dec(&se_deve->pr_ref_count);
-       smp_mb__after_atomic();
+       atomic_dec_mb(&se_deve->pr_ref_count);
 }
 
 static sense_reason_t
@@ -1551,15 +1536,13 @@ core_scsi3_decode_spec_i_port(
                        if (!i_str)
                                continue;
 
-                       atomic_inc(&tmp_tpg->tpg_pr_ref_count);
-                       smp_mb__after_atomic();
+                       atomic_inc_mb(&tmp_tpg->tpg_pr_ref_count);
                        spin_unlock(&dev->se_port_lock);
 
                        if (core_scsi3_tpg_depend_item(tmp_tpg)) {
                                pr_err(" core_scsi3_tpg_depend_item()"
                                        " for tmp_tpg\n");
-                               atomic_dec(&tmp_tpg->tpg_pr_ref_count);
-                               smp_mb__after_atomic();
+                               atomic_dec_mb(&tmp_tpg->tpg_pr_ref_count);
                                ret = TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
                                goto out_unmap;
                        }
@@ -1571,10 +1554,8 @@ core_scsi3_decode_spec_i_port(
                        spin_lock_irq(&tmp_tpg->acl_node_lock);
                        dest_node_acl = __core_tpg_get_initiator_node_acl(
                                                tmp_tpg, i_str);
-                       if (dest_node_acl) {
-                               atomic_inc(&dest_node_acl->acl_pr_ref_count);
-                               smp_mb__after_atomic();
-                       }
+                       if (dest_node_acl)
+                               atomic_inc_mb(&dest_node_acl->acl_pr_ref_count);
                        spin_unlock_irq(&tmp_tpg->acl_node_lock);
 
                        if (!dest_node_acl) {
@@ -1586,8 +1567,7 @@ core_scsi3_decode_spec_i_port(
                        if (core_scsi3_nodeacl_depend_item(dest_node_acl)) {
                                pr_err("configfs_depend_item() failed"
                                        " for dest_node_acl->acl_group\n");
-                               atomic_dec(&dest_node_acl->acl_pr_ref_count);
-                               smp_mb__after_atomic();
+                               atomic_dec_mb(&dest_node_acl->acl_pr_ref_count);
                                core_scsi3_tpg_undepend_item(tmp_tpg);
                                ret = TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
                                goto out_unmap;
@@ -1646,8 +1626,7 @@ core_scsi3_decode_spec_i_port(
                if (core_scsi3_lunacl_depend_item(dest_se_deve)) {
                        pr_err("core_scsi3_lunacl_depend_item()"
                                        " failed\n");
-                       atomic_dec(&dest_se_deve->pr_ref_count);
-                       smp_mb__after_atomic();
+                       atomic_dec_mb(&dest_se_deve->pr_ref_count);
                        core_scsi3_nodeacl_undepend_item(dest_node_acl);
                        core_scsi3_tpg_undepend_item(dest_tpg);
                        ret = TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
@@ -3167,15 +3146,13 @@ core_scsi3_emulate_pro_register_and_move(struct se_cmd *cmd, u64 res_key,
                if (!dest_tf_ops)
                        continue;
 
-               atomic_inc(&dest_se_tpg->tpg_pr_ref_count);
-               smp_mb__after_atomic();
+               atomic_inc_mb(&dest_se_tpg->tpg_pr_ref_count);
                spin_unlock(&dev->se_port_lock);
 
                if (core_scsi3_tpg_depend_item(dest_se_tpg)) {
                        pr_err("core_scsi3_tpg_depend_item() failed"
                                " for dest_se_tpg\n");
-                       atomic_dec(&dest_se_tpg->tpg_pr_ref_count);
-                       smp_mb__after_atomic();
+                       atomic_dec_mb(&dest_se_tpg->tpg_pr_ref_count);
                        ret = TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
                        goto out_put_pr_reg;
                }
@@ -3271,10 +3248,8 @@ after_iport_check:
        spin_lock_irq(&dest_se_tpg->acl_node_lock);
        dest_node_acl = __core_tpg_get_initiator_node_acl(dest_se_tpg,
                                initiator_str);
-       if (dest_node_acl) {
-               atomic_inc(&dest_node_acl->acl_pr_ref_count);
-               smp_mb__after_atomic();
-       }
+       if (dest_node_acl)
+               atomic_inc_mb(&dest_node_acl->acl_pr_ref_count);
        spin_unlock_irq(&dest_se_tpg->acl_node_lock);
 
        if (!dest_node_acl) {
@@ -3288,8 +3263,7 @@ after_iport_check:
        if (core_scsi3_nodeacl_depend_item(dest_node_acl)) {
                pr_err("core_scsi3_nodeacl_depend_item() for"
                        " dest_node_acl\n");
-               atomic_dec(&dest_node_acl->acl_pr_ref_count);
-               smp_mb__after_atomic();
+               atomic_dec_mb(&dest_node_acl->acl_pr_ref_count);
                dest_node_acl = NULL;
                ret = TCM_INVALID_PARAMETER_LIST;
                goto out;
@@ -3313,8 +3287,7 @@ after_iport_check:
 
        if (core_scsi3_lunacl_depend_item(dest_se_deve)) {
                pr_err("core_scsi3_lunacl_depend_item() failed\n");
-               atomic_dec(&dest_se_deve->pr_ref_count);
-               smp_mb__after_atomic();
+               atomic_dec_mb(&dest_se_deve->pr_ref_count);
                dest_se_deve = NULL;
                ret = TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
                goto out;
@@ -3879,8 +3852,7 @@ core_scsi3_pri_read_full_status(struct se_cmd *cmd)
                se_tpg = pr_reg->pr_reg_nacl->se_tpg;
                add_desc_len = 0;
 
-               atomic_inc(&pr_reg->pr_res_holders);
-               smp_mb__after_atomic();
+               atomic_inc_mb(&pr_reg->pr_res_holders);
                spin_unlock(&pr_tmpl->registration_lock);
                /*
                 * Determine expected length of $FABRIC_MOD specific
@@ -3893,8 +3865,7 @@ core_scsi3_pri_read_full_status(struct se_cmd *cmd)
                        pr_warn("SPC-3 PRIN READ_FULL_STATUS ran"
                                " out of buffer: %d\n", cmd->data_length);
                        spin_lock(&pr_tmpl->registration_lock);
-                       atomic_dec(&pr_reg->pr_res_holders);
-                       smp_mb__after_atomic();
+                       atomic_dec_mb(&pr_reg->pr_res_holders);
                        break;
                }
                /*
@@ -3955,8 +3926,7 @@ core_scsi3_pri_read_full_status(struct se_cmd *cmd)
                                se_nacl, pr_reg, &format_code, &buf[off+4]);
 
                spin_lock(&pr_tmpl->registration_lock);
-               atomic_dec(&pr_reg->pr_res_holders);
-               smp_mb__after_atomic();
+               atomic_dec_mb(&pr_reg->pr_res_holders);
                /*
                 * Set the ADDITIONAL DESCRIPTOR LENGTH
                 */

diff --git a/drivers/target/target_core_transport.c b/drivers/target/target_core_transport.c
index 0b43761..115632e 100644 (file)
--- a/drivers/target/target_core_transport.c
+++ b/drivers/target/target_core_transport.c
@@ -752,8 +752,7 @@ void target_qf_do_work(struct work_struct *work)
 
        list_for_each_entry_safe(cmd, cmd_tmp, &qf_cmd_list, se_qf_node) {
                list_del(&cmd->se_qf_node);
-               atomic_dec(&dev->dev_qf_count);
-               smp_mb__after_atomic();
+               atomic_dec_mb(&dev->dev_qf_count);
 
                pr_debug("Processing %s cmd: %p QUEUE_FULL in work queue"
                        " context: %s\n", cmd->se_tfo->get_fabric_name(), cmd,
@@ -1721,8 +1720,7 @@ static bool target_handle_task_attr(struct se_cmd *cmd)
                         cmd->t_task_cdb[0], cmd->se_ordered_id);
                return false;
        case MSG_ORDERED_TAG:
-               atomic_inc(&dev->dev_ordered_sync);
-               smp_mb__after_atomic();
+               atomic_inc_mb(&dev->dev_ordered_sync);
 
                pr_debug("Added ORDERED for CDB: 0x%02x to ordered list, "
                         " se_ordered_id: %u\n",
@@ -1739,8 +1737,7 @@ static bool target_handle_task_attr(struct se_cmd *cmd)
                /*
                 * For SIMPLE and UNTAGGED Task Attribute commands
                 */
-               atomic_inc(&dev->simple_cmds);
-               smp_mb__after_atomic();
+               atomic_inc_mb(&dev->simple_cmds);
                break;
        }
 
@@ -1844,8 +1841,7 @@ static void transport_complete_task_attr(struct se_cmd *cmd)
                return;
 
        if (cmd->sam_task_attr == MSG_SIMPLE_TAG) {
-               atomic_dec(&dev->simple_cmds);
-               smp_mb__after_atomic();
+               atomic_dec_mb(&dev->simple_cmds);
                dev->dev_cur_ordered_id++;
                pr_debug("Incremented dev->dev_cur_ordered_id: %u for"
                        " SIMPLE: %u\n", dev->dev_cur_ordered_id,
@@ -1856,8 +1852,7 @@ static void transport_complete_task_attr(struct se_cmd *cmd)
                        " HEAD_OF_QUEUE: %u\n", dev->dev_cur_ordered_id,
                        cmd->se_ordered_id);
        } else if (cmd->sam_task_attr == MSG_ORDERED_TAG) {
-               atomic_dec(&dev->dev_ordered_sync);
-               smp_mb__after_atomic();
+               atomic_dec_mb(&dev->dev_ordered_sync);
 
                dev->dev_cur_ordered_id++;
                pr_debug("Incremented dev_cur_ordered_id: %u for ORDERED:"
@@ -1915,8 +1910,7 @@ static void transport_handle_queue_full(
 {
        spin_lock_irq(&dev->qf_cmd_lock);
        list_add_tail(&cmd->se_qf_node, &cmd->se_dev->qf_cmd_list);
-       atomic_inc(&dev->dev_qf_count);
-       smp_mb__after_atomic();
+       atomic_inc_mb(&dev->dev_qf_count);
        spin_unlock_irq(&cmd->se_dev->qf_cmd_lock);
 
        schedule_work(&cmd->se_dev->qf_work_queue);

diff --git a/drivers/target/target_core_ua.c b/drivers/target/target_core_ua.c
index 101858e..1738b16 100644 (file)
--- a/drivers/target/target_core_ua.c
+++ b/drivers/target/target_core_ua.c
@@ -161,8 +161,7 @@ int core_scsi3_ua_allocate(
                spin_unlock(&deve->ua_lock);
                spin_unlock_irq(&nacl->device_list_lock);
 
-               atomic_inc(&deve->ua_count);
-               smp_mb__after_atomic();
+               atomic_inc_mb(&deve->ua_count);
                return 0;
        }
        list_add_tail(&ua->ua_nacl_list, &deve->ua_list);
@@ -174,8 +173,7 @@ int core_scsi3_ua_allocate(
                nacl->se_tpg->se_tpg_tfo->get_fabric_name(), unpacked_lun,
                asc, ascq);
 
-       atomic_inc(&deve->ua_count);
-       smp_mb__after_atomic();
+       atomic_inc_mb(&deve->ua_count);
        return 0;
 }
 
@@ -189,8 +187,7 @@ void core_scsi3_ua_release_all(
                list_del(&ua->ua_nacl_list);
                kmem_cache_free(se_ua_cache, ua);
 
-               atomic_dec(&deve->ua_count);
-               smp_mb__after_atomic();
+               atomic_dec_mb(&deve->ua_count);
        }
        spin_unlock(&deve->ua_lock);
 }
@@ -250,8 +247,7 @@ void core_scsi3_ua_for_check_condition(
                list_del(&ua->ua_nacl_list);
                kmem_cache_free(se_ua_cache, ua);
 
-               atomic_dec(&deve->ua_count);
-               smp_mb__after_atomic();
+               atomic_dec_mb(&deve->ua_count);
        }
        spin_unlock(&deve->ua_lock);
        spin_unlock_irq(&nacl->device_list_lock);
@@ -309,8 +305,7 @@ int core_scsi3_ua_clear_for_request_sense(
                list_del(&ua->ua_nacl_list);
                kmem_cache_free(se_ua_cache, ua);
 
-               atomic_dec(&deve->ua_count);
-               smp_mb__after_atomic();
+               atomic_dec_mb(&deve->ua_count);
        }
        spin_unlock(&deve->ua_lock);
        spin_unlock_irq(&nacl->device_list_lock);

diff --git a/include/target/target_core_base.h b/include/target/target_core_base.h
index 9ec9864..b106240 100644 (file)
--- a/include/target/target_core_base.h
+++ b/include/target/target_core_base.h
@@ -903,4 +903,18 @@ struct se_wwn {
        struct config_group     fabric_stat_group;
 };
 
+static inline void atomic_inc_mb(atomic_t *v)
+{
+       smp_mb__before_atomic();
+       atomic_inc(v);
+       smp_mb__after_atomic();
+}
+
+static inline void atomic_dec_mb(atomic_t *v)
+{
+       smp_mb__before_atomic();
+       atomic_dec(v);
+       smp_mb__after_atomic();
+}
+
 #endif /* TARGET_CORE_BASE_H */