diff --git a/kernel/irq/matrix.c b/kernel/irq/matrix.c
index 30cc217b863184989987f2fa76c76c8bc9a082b4..0a5b2e26fd544d495d23001ebf774561b247df24 100644
--- a/kernel/irq/matrix.c
+++ b/kernel/irq/matrix.c
@@ -416,7 +416,9 @@ void irq_matrix_free(struct irq_matrix *m, unsigned int cpu,
 	if (WARN_ON_ONCE(bit < m->alloc_start || bit >= m->alloc_end))
 		return;
 
-	clear_bit(bit, cm->alloc_map);
+	if (WARN_ON_ONCE(!test_and_clear_bit(bit, cm->alloc_map)))
+		return;
+
 	cm->allocated--;
 	if(managed)
 		cm->managed_allocated--;