#ifndef __ASSEMBLY__
+#include <linux/compiler.h>
#include <asm/hazards.h>
-__asm__ (
+__asm__(
" .macro raw_local_irq_enable \n"
" .set push \n"
" .set reorder \n"
*
* Workaround: mask EXL bit of the result or place a nop before mfc0.
*/
-__asm__ (
+__asm__(
" .macro raw_local_irq_disable\n"
" .set push \n"
" .set noat \n"
: "memory");
}
-__asm__ (
+__asm__(
" .macro raw_local_save_flags flags \n"
" .set push \n"
" .set reorder \n"
"raw_local_save_flags %0" \
: "=r" (x))
-__asm__ (
+__asm__(
" .macro raw_local_irq_save result \n"
" .set push \n"
" .set reorder \n"
: /* no inputs */ \
: "memory")
-__asm__ (
+__asm__(
" .macro raw_local_irq_restore flags \n"
" .set push \n"
" .set noreorder \n"
" .set pop \n"
" .endm \n");
-#define raw_local_irq_restore(flags) \
-do { \
- unsigned long __tmp1; \
- \
- __asm__ __volatile__( \
- "raw_local_irq_restore\t%0" \
- : "=r" (__tmp1) \
- : "0" (flags) \
- : "memory"); \
-} while(0)
+extern void smtc_ipi_replay(void);
+
+static inline void raw_local_irq_restore(unsigned long flags)
+{
+ unsigned long __tmp1;
+
+#ifdef CONFIG_MIPS_MT_SMTC_INSTANT_REPLAY
+ /*
+ * CONFIG_MIPS_MT_SMTC_INSTANT_REPLAY does prompt replay of deferred
+ * IPIs, at the cost of branch and call overhead on each
+ * local_irq_restore()
+ */
+ if (unlikely(!(flags & 0x0400)))
+ smtc_ipi_replay();
+#endif
+
+ __asm__ __volatile__(
+ "raw_local_irq_restore\t%0"
+ : "=r" (__tmp1)
+ : "0" (flags)
+ : "memory");
+}
static inline int raw_irqs_disabled_flags(unsigned long flags)
{
* Do the CPU's IRQ-state tracing from assembly code.
*/
#ifdef CONFIG_TRACE_IRQFLAGS
+/* Reload some registers clobbered by trace_hardirqs_on */
+#ifdef CONFIG_64BIT
+# define TRACE_IRQS_RELOAD_REGS \
+ LONG_L $11, PT_R11(sp); \
+ LONG_L $10, PT_R10(sp); \
+ LONG_L $9, PT_R9(sp); \
+ LONG_L $8, PT_R8(sp); \
+ LONG_L $7, PT_R7(sp); \
+ LONG_L $6, PT_R6(sp); \
+ LONG_L $5, PT_R5(sp); \
+ LONG_L $4, PT_R4(sp); \
+ LONG_L $2, PT_R2(sp)
+#else
+# define TRACE_IRQS_RELOAD_REGS \
+ LONG_L $7, PT_R7(sp); \
+ LONG_L $6, PT_R6(sp); \
+ LONG_L $5, PT_R5(sp); \
+ LONG_L $4, PT_R4(sp); \
+ LONG_L $2, PT_R2(sp)
+#endif
# define TRACE_IRQS_ON \
+ CLI; /* make sure trace_hardirqs_on() is called in kernel level */ \
jal trace_hardirqs_on
+# define TRACE_IRQS_ON_RELOAD \
+ TRACE_IRQS_ON; \
+ TRACE_IRQS_RELOAD_REGS
# define TRACE_IRQS_OFF \
jal trace_hardirqs_off
#else
# define TRACE_IRQS_ON
+# define TRACE_IRQS_ON_RELOAD
# define TRACE_IRQS_OFF
#endif