x86/bitops: Introduce variable/constant pairs for __{set,clear,change}_bit()

Just as with test_bit, the non-atomic set/clear/change helpers can be better
optimised by the compiler in the case that the nr parameter is constant, and
it often is.

This results in a general replacement of `mov $imm, %reg; bt* %reg, mem` with
the shorter and more simple `op $imm, mem`, also reducing register pressure.

The net diffstat is:
  add/remove: 0/1 grow/shrink: 5/17 up/down: 90/-301 (-211)

As a piece of minor cleanup, drop unnecessary brackets in the test_bit()
macro, and fix the indentation.

Signed-off-by: Andrew Cooper <andrew.cooper3@citrix.com>
Reviewed-by: Dario Faggioli <dfaggioli@suse.com>
Reviewed-by: Jan Beulich <jbeulich@suse.com>

diff --git a/xen/include/asm-x86/bitops.h b/xen/include/asm-x86/bitops.h
index 440abb7ec99eebfe8b00220c7cac3e03cac8a156..e66d86108081af485dc3cf8f225bbfd8db467c62 100644 (file)
--- a/xen/include/asm-x86/bitops.h
+++ b/xen/include/asm-x86/bitops.h
@@ -51,13 +51,19 @@ static inline void set_bit(int nr, volatile void *addr)
  * If it's called on the same region of memory simultaneously, the effect
  * may be that only one operation succeeds.
  */
-static inline void __set_bit(int nr, void *addr)
+static inline void variable_set_bit(int nr, void *addr)
 {
     asm volatile ( "btsl %1,%0" : "+m" (*(int *)addr) : "Ir" (nr) : "memory" );
 }
+static inline void constant_set_bit(int nr, void *addr)
+{
+    ((unsigned int *)addr)[nr >> 5] |= (1u << (nr & 31));
+}
 #define __set_bit(nr, addr) ({                          \
     if ( bitop_bad_size(addr) ) __bitop_bad_size();     \
-    __set_bit(nr, addr);                                \
+    __builtin_constant_p(nr) ?                          \
+        constant_set_bit(nr, addr) :                    \
+        variable_set_bit(nr, addr);                     \
 })
 
 /**
@@ -86,13 +92,19 @@ static inline void clear_bit(int nr, volatile void *addr)
  * If it's called on the same region of memory simultaneously, the effect
  * may be that only one operation succeeds.
  */
-static inline void __clear_bit(int nr, void *addr)
+static inline void variable_clear_bit(int nr, void *addr)
 {
     asm volatile ( "btrl %1,%0" : "+m" (*(int *)addr) : "Ir" (nr) : "memory" );
 }
+static inline void constant_clear_bit(int nr, void *addr)
+{
+    ((unsigned int *)addr)[nr >> 5] &= ~(1u << (nr & 31));
+}
 #define __clear_bit(nr, addr) ({                        \
     if ( bitop_bad_size(addr) ) __bitop_bad_size();     \
-    __clear_bit(nr, addr);                              \
+    __builtin_constant_p(nr) ?                          \
+        constant_clear_bit(nr, addr) :                  \
+        variable_clear_bit(nr, addr);                   \
 })
 
 /**
@@ -104,13 +116,19 @@ static inline void __clear_bit(int nr, void *addr)
  * If it's called on the same region of memory simultaneously, the effect
  * may be that only one operation succeeds.
  */
-static inline void __change_bit(int nr, void *addr)
+static inline void variable_change_bit(int nr, void *addr)
 {
     asm volatile ( "btcl %1,%0" : "+m" (*(int *)addr) : "Ir" (nr) : "memory" );
 }
+static inline void constant_change_bit(int nr, void *addr)
+{
+    ((unsigned int *)addr)[nr >> 5] ^= (1u << (nr & 31));
+}
 #define __change_bit(nr, addr) ({                       \
     if ( bitop_bad_size(addr) ) __bitop_bad_size();     \
-    __change_bit(nr, addr);                             \
+    __builtin_constant_p(nr) ?                          \
+        constant_change_bit(nr, addr) :                 \
+        variable_change_bit(nr, addr);                  \
 })
 
 /**
@@ -291,9 +309,9 @@ static inline int variable_test_bit(int nr, const volatile void *addr)
 
 #define test_bit(nr, addr) ({                           \
     if ( bitop_bad_size(addr) ) __bitop_bad_size();     \
-    (__builtin_constant_p(nr) ?                         \
-     constant_test_bit((nr),(addr)) :                   \
-     variable_test_bit((nr),(addr)));                   \
+    __builtin_constant_p(nr) ?                          \
+        constant_test_bit(nr, addr) :                   \
+        variable_test_bit(nr, addr);                    \
 })
 
 extern unsigned int __find_first_bit(