diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum.c
index fd6c40ead96a8fcd3da3adba5d43dfd3c099f3f6..cb0e77ad4c0aacf2e2c982fc14a5a35e69a52214 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/spectrum.c
+++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum.c
@@ -3392,16 +3392,16 @@ void mlxsw_sp_port_dev_put(struct mlxsw_sp_port *mlxsw_sp_port)
 }
 
 static bool mlxsw_sp_rif_should_config(struct mlxsw_sp_rif *r,
+				       const struct in_device *in_dev,
 				       unsigned long event)
 {
 	switch (event) {
 	case NETDEV_UP:
 		if (!r)
 			return true;
-		r->ref_count++;
 		return false;
 	case NETDEV_DOWN:
-		if (r && --r->ref_count == 0)
+		if (r && !in_dev->ifa_list)
 			return true;
 		/* It is possible we already removed the RIF ourselves
 		 * if it was assigned to a netdev that is now a bridge
@@ -3484,7 +3484,6 @@ mlxsw_sp_rif_alloc(u16 rif, struct net_device *l3_dev, struct mlxsw_sp_fid *f)
 	INIT_LIST_HEAD(&r->neigh_list);
 	ether_addr_copy(r->addr, l3_dev->dev_addr);
 	r->mtu = l3_dev->mtu;
-	r->ref_count = 1;
 	r->dev = l3_dev;
 	r->rif = rif;
 	r->f = f;
@@ -3858,7 +3857,7 @@ static int mlxsw_sp_inetaddr_event(struct notifier_block *unused,
 		goto out;
 
 	r = mlxsw_sp_rif_find_by_dev(mlxsw_sp, dev);
-	if (!mlxsw_sp_rif_should_config(r, event))
+	if (!mlxsw_sp_rif_should_config(r, ifa->ifa_dev, event))
 		goto out;
 
 	if (mlxsw_sp_port_dev_check(dev))
diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum.h b/drivers/net/ethernet/mellanox/mlxsw/spectrum.h
index ac445d810eb366ddc6f5cb04a785a07b8ebd8a78..7805430e5d4f0751e66efd013d0585967bf7cbee 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/spectrum.h
+++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum.h
@@ -111,7 +111,6 @@ struct mlxsw_sp_rif {
 	struct list_head nexthop_list;
 	struct list_head neigh_list;
 	struct net_device *dev;
-	unsigned int ref_count;
 	struct mlxsw_sp_fid *f;
 	unsigned char addr[ETH_ALEN];
 	int mtu;