This patch adds a menuconfig option that allows controlling
the lazy interrupt disabling feature implemented by this
commit:
commit d04c56f73c30a5e593202ecfcf25ed43d42363a2
Author: Paul Mackerras
Date: Wed Oct 4 16:47:49 2006 +1000
[POWERPC] Lazy interrupt disabling for 64-bit machines
The code in 'powerpc/include/asm/hw_irq.h' was rearranged and
cleaned-up a bit in order to reduce the number of needed #ifdef's.
Signed-off-by: Laurentiu Tudor <laurentiu.tu...@freescale.com>
---
Patch is against
git://git.kernel.org/pub/scm/linux/kernel/git/benh/powerpc.git next
arch/powerpc/Kconfig | 16 ++++++
arch/powerpc/include/asm/hw_irq.h | 86 ++++++++++++++--------------------
arch/powerpc/include/asm/irqflags.h | 22 +++++++++
arch/powerpc/include/asm/paca.h | 4 ++
arch/powerpc/kernel/asm-offsets.c | 3 +
arch/powerpc/kernel/entry_64.S | 4 ++
arch/powerpc/kernel/exceptions-64e.S | 36 +++++++++++---
arch/powerpc/kernel/head_64.S | 10 ++++
arch/powerpc/kernel/idle_book3e.S | 7 +++
arch/powerpc/kernel/irq.c | 5 ++
arch/powerpc/kernel/setup_64.c | 2 +
11 files changed, 138 insertions(+), 57 deletions(-)
diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig
index ce5e045..2792278 100644
--- a/arch/powerpc/Kconfig
+++ b/arch/powerpc/Kconfig
@@ -609,6 +609,22 @@ config SECCOMP
If unsure, say Y. Only embedded should say N here.
+config PPC_LAZY_EE
+ bool
+ prompt "Lazy interrupt disabling" if PPC_BOOK3E_64
+ default y if PPC_BOOK3S_64
+ help
+ Local interrupt disabling functions don't disable
+ interrupts right away and instead just clear an
+ internal 'interrupts are enabled' flag. If an
+ interrupt is triggered during this time, the
+ interrupt handling code checks the flag, and if
+ interrupts are supposed to be off, disables them
+ for real and returns, skipping interrupt handling.
+ When interrupts are enabled back, the interrupt
+ fires again and this time gets handled.
+
+ If unsure, say N.
endmenu
config ISA_DMA_API
diff --git a/arch/powerpc/include/asm/hw_irq.h
b/arch/powerpc/include/asm/hw_irq.h
index bb712c9..6f32593 100644
--- a/arch/powerpc/include/asm/hw_irq.h
+++ b/arch/powerpc/include/asm/hw_irq.h
@@ -13,7 +13,20 @@
extern void timer_interrupt(struct pt_regs *);
+#ifdef CONFIG_BOOKE
+#define __hard_irq_enable() asm volatile("wrteei 1" : : : "memory");
+#define __hard_irq_disable() asm volatile("wrteei 0" : : : "memory");
+#else
#ifdef CONFIG_PPC64
+#define __hard_irq_enable() __mtmsrd(mfmsr() | MSR_EE, 1)
+#define __hard_irq_disable() __mtmsrd(mfmsr() & ~MSR_EE, 1)
+#else
+#define __hard_irq_enable() mtmsr(mfmsr() | MSR_EE)
+#define __hard_irq_disable() mtmsr(mfmsr() & ~MSR_EE)
+#endif
+#endif /* CONFIG_BOOKE */
+
+#if defined(CONFIG_PPC64) && defined(CONFIG_PPC_LAZY_EE)
#include <asm/paca.h>
static inline unsigned long arch_local_save_flags(void)
@@ -49,9 +62,11 @@ static inline void arch_local_irq_enable(void)
arch_local_irq_restore(1);
}
-static inline unsigned long arch_local_irq_save(void)
+static inline void hard_irq_disable(void)
{
- return arch_local_irq_disable();
+ __hard_irq_disable();
+ get_paca()->soft_enabled = 0;
+ get_paca()->hard_enabled = 0;
}
static inline bool arch_irqs_disabled_flags(unsigned long flags)
@@ -59,29 +74,7 @@ static inline bool arch_irqs_disabled_flags(unsigned long
flags)
return flags == 0;
}
-static inline bool arch_irqs_disabled(void)
-{
- return arch_irqs_disabled_flags(arch_local_save_flags());
-}
-
-#ifdef CONFIG_PPC_BOOK3E
-#define __hard_irq_enable() asm volatile("wrteei 1" : : : "memory");
-#define __hard_irq_disable() asm volatile("wrteei 0" : : : "memory");
-#else
-#define __hard_irq_enable() __mtmsrd(mfmsr() | MSR_EE, 1)
-#define __hard_irq_disable() __mtmsrd(mfmsr() & ~MSR_EE, 1)
-#endif
-
-#define hard_irq_disable() \
- do { \
- __hard_irq_disable(); \
- get_paca()->soft_enabled = 0; \
- get_paca()->hard_enabled = 0; \
- } while(0)
-
-#else /* CONFIG_PPC64 */
-
-#define SET_MSR_EE(x) mtmsr(x)
+#else /* CONFIG_PPC64 && CONFIG_PPC_LAZY_EE */
static inline unsigned long arch_local_save_flags(void)
{
@@ -97,34 +90,24 @@ static inline void arch_local_irq_restore(unsigned long
flags)
#endif
}
-static inline unsigned long arch_local_irq_save(void)
+static inline void arch_local_irq_enable(void)
{
- unsigned long flags = arch_local_save_flags();
-#ifdef CONFIG_BOOKE
- asm volatile("wrteei 0" : : : "memory");
-#else
- SET_MSR_EE(flags & ~MSR_EE);
-#endif
- return flags;
+ __hard_irq_enable();
}
-static inline void arch_local_irq_disable(void)
+static inline unsigned long arch_local_irq_disable(void)
{
-#ifdef CONFIG_BOOKE
- asm volatile("wrteei 0" : : : "memory");
-#else
- arch_local_irq_save();
-#endif
+ unsigned long flags;
+
+ flags = arch_local_save_flags();
+ __hard_irq_disable();
+
+ return flags;
}
-static inline void arch_local_irq_enable(void)
+static inline void hard_irq_disable(void)
{
-#ifdef CONFIG_BOOKE
- asm volatile("wrteei 1" : : : "memory");
-#else
- unsigned long msr = mfmsr();
- SET_MSR_EE(msr | MSR_EE);
-#endif
+ __hard_irq_disable();
}
static inline bool arch_irqs_disabled_flags(unsigned long flags)
@@ -132,15 +115,18 @@ static inline bool arch_irqs_disabled_flags(unsigned long
flags)
return (flags & MSR_EE) == 0;
}
+#endif /* CONFIG_PPC64 && CONFIG_PPC_LAZY_EE */
+
+static inline unsigned long arch_local_irq_save(void)
+{
+ return arch_local_irq_disable();
+}
+
static inline bool arch_irqs_disabled(void)
{
return arch_irqs_disabled_flags(arch_local_save_flags());
}
-#define hard_irq_disable() arch_local_irq_disable()
-
-#endif /* CONFIG_PPC64 */
-
#define ARCH_IRQ_INIT_FLAGS IRQ_NOREQUEST
/*
diff --git a/arch/powerpc/include/asm/irqflags.h
b/arch/powerpc/include/asm/irqflags.h
index b0b06d8..9685b75c 100644
--- a/arch/powerpc/include/asm/irqflags.h
+++ b/arch/powerpc/include/asm/irqflags.h
@@ -39,6 +39,8 @@
#define TRACE_ENABLE_INTS TRACE_WITH_FRAME_BUFFER(.trace_hardirqs_on)
#define TRACE_DISABLE_INTS TRACE_WITH_FRAME_BUFFER(.trace_hardirqs_off)
+#ifdef CONFIG_PPC_LAZY_EE
+
#define TRACE_AND_RESTORE_IRQ_PARTIAL(en,skip) \
cmpdi en,0; \
bne 95f; \
@@ -51,12 +53,32 @@
TRACE_AND_RESTORE_IRQ_PARTIAL(en,96f); \
stb en,PACASOFTIRQEN(r13); \
96:
+
+#else /* CONFIG_PPC_LAZY_EE */
+
+#define TRACE_AND_RESTORE_IRQ_PARTIAL(en,skip) \
+ cmpdi en,0; \
+ bne 95f; \
+ TRACE_WITH_FRAME_BUFFER(.trace_hardirqs_off) \
+ b skip; \
+95: TRACE_WITH_FRAME_BUFFER(.trace_hardirqs_on) \
+ li en,1;
+#define TRACE_AND_RESTORE_IRQ(en) \
+ TRACE_AND_RESTORE_IRQ_PARTIAL(en,96f); \
+96:
+
+#endif /* CONFIG_PPC_LAZY_EE */
+
#else
#define TRACE_ENABLE_INTS
#define TRACE_DISABLE_INTS
#define TRACE_AND_RESTORE_IRQ_PARTIAL(en,skip)
+#ifdef CONFIG_PPC_LAZY_EE
#define TRACE_AND_RESTORE_IRQ(en) \
stb en,PACASOFTIRQEN(r13)
+#else
+#define TRACE_AND_RESTORE_IRQ(en)
+#endif
#endif
#endif
diff --git a/arch/powerpc/include/asm/paca.h b/arch/powerpc/include/asm/paca.h
index 269c05a..b74c2b1 100644
--- a/arch/powerpc/include/asm/paca.h
+++ b/arch/powerpc/include/asm/paca.h
@@ -131,8 +131,12 @@ struct paca_struct {
u64 saved_r1; /* r1 save for RTAS calls or PM */
u64 saved_msr; /* MSR saved here by enter_rtas */
u16 trap_save; /* Used when bad stack is encountered */
+
+#ifdef CONFIG_PPC_LAZY_EE
u8 soft_enabled; /* irq soft-enable flag */
u8 hard_enabled; /* set if irqs are enabled in MSR */
+#endif
+
u8 io_sync; /* writel() needs spin_unlock sync */
u8 irq_work_pending; /* IRQ_WORK interrupt while
soft-disable */
u8 nap_state_lost; /* NV GPR values lost in power7_idle */
diff --git a/arch/powerpc/kernel/asm-offsets.c
b/arch/powerpc/kernel/asm-offsets.c
index 04caee7..5b1fac9 100644
--- a/arch/powerpc/kernel/asm-offsets.c
+++ b/arch/powerpc/kernel/asm-offsets.c
@@ -146,8 +146,11 @@ int main(void)
DEFINE(PACATOC, offsetof(struct paca_struct, kernel_toc));
DEFINE(PACAKBASE, offsetof(struct paca_struct, kernelbase));
DEFINE(PACAKMSR, offsetof(struct paca_struct, kernel_msr));
+#ifdef CONFIG_PPC_LAZY_EE
DEFINE(PACASOFTIRQEN, offsetof(struct paca_struct, soft_enabled));
DEFINE(PACAHARDIRQEN, offsetof(struct paca_struct, hard_enabled));
+#endif
+
DEFINE(PACACONTEXTID, offsetof(struct paca_struct, context.id));
#ifdef CONFIG_PPC_MM_SLICES
DEFINE(PACALOWSLICESPSIZE, offsetof(struct paca_struct,
diff --git a/arch/powerpc/kernel/entry_64.S b/arch/powerpc/kernel/entry_64.S
index d834425..ce7620b 100644
--- a/arch/powerpc/kernel/entry_64.S
+++ b/arch/powerpc/kernel/entry_64.S
@@ -124,8 +124,10 @@ END_FW_FTR_SECTION_IFSET(FW_FEATURE_SPLPAR)
ld r12,_MSR(r1)
#endif /* CONFIG_TRACE_IRQFLAGS */
li r10,1
+#ifdef CONFIG_PPC_LAZY_EE
stb r10,PACASOFTIRQEN(r13)
stb r10,PACAHARDIRQEN(r13)
+#endif
std r10,SOFTE(r1)
#ifdef CONFIG_PPC_ISERIES
BEGIN_FW_FTR_SECTION
@@ -602,10 +604,12 @@ ALT_FW_FTR_SECTION_END_IFCLR(FW_FEATURE_ISERIES)
2:
TRACE_AND_RESTORE_IRQ(r5);
+#ifdef CONFIG_PPC_LAZY_EE
/* extract EE bit and use it to restore paca->hard_enabled */
ld r3,_MSR(r1)
rldicl r4,r3,49,63 /* r0 = (r3 >> 15) & 1 */
stb r4,PACAHARDIRQEN(r13)
+#endif
#ifdef CONFIG_PPC_BOOK3E
b .exception_return_book3e
diff --git a/arch/powerpc/kernel/exceptions-64e.S
b/arch/powerpc/kernel/exceptions-64e.S
index 429983c..d17abb3 100644
--- a/arch/powerpc/kernel/exceptions-64e.S
+++ b/arch/powerpc/kernel/exceptions-64e.S
@@ -96,11 +96,6 @@
#define PROLOG_ADDITION_NONE_DBG
#define PROLOG_ADDITION_NONE_MC
-#define PROLOG_ADDITION_MASKABLE_GEN \
- lbz r11,PACASOFTIRQEN(r13); /* are irqs soft-disabled ? */ \
- cmpwi cr0,r11,0; /* yes -> go out of line */ \
- beq masked_interrupt_book3e;
-
#define PROLOG_ADDITION_2REGS_GEN \
std r14,PACA_EXGEN+EX_R14(r13); \
std r15,PACA_EXGEN+EX_R15(r13)
@@ -120,11 +115,29 @@
std r14,PACA_EXMC+EX_R14(r13); \
std r15,PACA_EXMC+EX_R15(r13)
+#ifdef CONFIG_PPC_LAZY_EE
+#define PROLOG_ADDITION_MASKABLE_GEN \
+ lbz r11,PACASOFTIRQEN(r13); /* are irqs soft-disabled ? */ \
+ cmpwi cr0,r11,0; /* yes -> go out of line */ \
+ beq masked_interrupt_book3e;
+
#define PROLOG_ADDITION_DOORBELL_GEN \
lbz r11,PACASOFTIRQEN(r13); /* are irqs soft-disabled ? */ \
cmpwi cr0,r11,0; /* yes -> go out of line */ \
beq masked_doorbell_book3e
-
+#else /* CONFIG_PPC_LAZY_EE */
+#define PROLOG_ADDITION_MASKABLE_GEN
+#define PROLOG_ADDITION_DOORBELL_GEN
+#endif /* CONFIG_PPC_LAZY_EE */
+
+#ifdef CONFIG_PPC_LAZY_EE
+#define GET_IRQ_SOFT_ENABLED \
+ lbz r11,PACASOFTIRQEN(r13)
+#else
+#define GET_IRQ_SOFT_ENABLED \
+ mfmsr r11; \
+ rlwimi r11,r11,0,16,16
+#endif
/* Core exception code for all exceptions except TLB misses.
* XXX: Needs to make SPRN_SPRG_GEN depend on exception type
@@ -148,7 +161,7 @@
mfspr r8,SPRN_XER; /* save XER in stackframe */ \
ld r9,excf+EX_R1(r13); /* load orig r1 back from PACA */ \
lwz r10,excf+EX_CR(r13); /* load orig CR back from PACA */ \
- lbz r11,PACASOFTIRQEN(r13); /* get current IRQ softe */ \
+ GET_IRQ_SOFT_ENABLED; /* get current IRQ softe */ \
ld r12,exception_marker@toc(r2); \
li r0,0; \
std r3,GPR10(r1); /* save r10 to stackframe */ \
@@ -169,11 +182,18 @@
/* Variants for the "ints" argument */
#define INTS_KEEP
+
+#ifdef CONFIG_PPC_LAZY_EE
#define INTS_DISABLE_SOFT \
stb r0,PACASOFTIRQEN(r13); /* mark interrupts soft-disabled */ \
TRACE_DISABLE_INTS;
#define INTS_DISABLE_HARD \
stb r0,PACAHARDIRQEN(r13); /* and hard disabled */
+#else
+#define INTS_DISABLE_SOFT
+#define INTS_DISABLE_HARD
+#endif
+
#define INTS_DISABLE_ALL \
INTS_DISABLE_SOFT \
INTS_DISABLE_HARD
@@ -553,6 +573,7 @@ kernel_dbg_exc:
MASKABLE_EXCEPTION(0x320, ehpriv, .unknown_exception, ACK_NONE)
+#ifdef CONFIG_PPC_LAZY_EE
/*
* An interrupt came in while soft-disabled; clear EE in SRR1,
* clear paca->hard_enabled and return.
@@ -577,6 +598,7 @@ masked_interrupt_book3e_common:
mfspr r13,SPRN_SPRG_GEN_SCRATCH;
rfi
b .
+#endif /* CONFIG_PPC_LAZY_EE */
/*
* This is called from 0x300 and 0x400 handlers after the prologs with
diff --git a/arch/powerpc/kernel/head_64.S b/arch/powerpc/kernel/head_64.S
index 06c7251..017d669 100644
--- a/arch/powerpc/kernel/head_64.S
+++ b/arch/powerpc/kernel/head_64.S
@@ -559,12 +559,14 @@ _GLOBAL(pmac_secondary_start)
add r13,r13,r4 /* for this processor. */
SET_PACA(r13) /* Save vaddr of paca in an SPRG*/
+#ifdef CONFIG_PPC_LAZY_EE
/* Mark interrupts soft and hard disabled (they might be enabled
* in the PACA when doing hotplug)
*/
li r0,0
stb r0,PACASOFTIRQEN(r13)
stb r0,PACAHARDIRQEN(r13)
+#endif
/* Create a temp kernel stack for use before relocation is on. */
ld r1,PACAEMERGSP(r13)
@@ -621,14 +623,18 @@ __secondary_start:
#ifdef CONFIG_PPC_ISERIES
BEGIN_FW_FTR_SECTION
ori r4,r4,MSR_EE
+#ifdef CONFIG_PPC_LAZY_EE
li r8,1
stb r8,PACAHARDIRQEN(r13)
+#endif
END_FW_FTR_SECTION_IFSET(FW_FEATURE_ISERIES)
#endif
+#ifdef CONFIG_PPC_LAZY_EE
BEGIN_FW_FTR_SECTION
stb r7,PACAHARDIRQEN(r13)
END_FW_FTR_SECTION_IFCLR(FW_FEATURE_ISERIES)
stb r7,PACASOFTIRQEN(r13)
+#endif
mtspr SPRN_SRR0,r3
mtspr SPRN_SRR1,r4
@@ -775,8 +781,10 @@ _INIT_GLOBAL(start_here_common)
/* Load up the kernel context */
5:
+#ifdef CONFIG_PPC_LAZY_EE
li r5,0
stb r5,PACASOFTIRQEN(r13) /* Soft Disabled */
+#endif
#ifdef CONFIG_PPC_ISERIES
BEGIN_FW_FTR_SECTION
mfmsr r5
@@ -785,7 +793,9 @@ BEGIN_FW_FTR_SECTION
li r5,1
END_FW_FTR_SECTION_IFSET(FW_FEATURE_ISERIES)
#endif
+#ifdef CONFIG_PPC_LAZY_EE
stb r5,PACAHARDIRQEN(r13) /* Hard Disabled on others */
+#endif
bl .start_kernel
diff --git a/arch/powerpc/kernel/idle_book3e.S
b/arch/powerpc/kernel/idle_book3e.S
index 16c002d..34b09e0 100644
--- a/arch/powerpc/kernel/idle_book3e.S
+++ b/arch/powerpc/kernel/idle_book3e.S
@@ -28,6 +28,7 @@ _GLOBAL(book3e_idle)
/* Hard disable interrupts */
wrteei 0
+#ifdef CONFIG_PPC_LAZY_EE
/* Now check if an interrupt came in while we were soft disabled
* since we may otherwise lose it (doorbells etc...). We know
* that since PACAHARDIRQEN will have been cleared in that case.
@@ -35,6 +36,7 @@ _GLOBAL(book3e_idle)
lbz r3,PACAHARDIRQEN(r13)
cmpwi cr0,r3,0
beqlr
+#endif
/* Now we are going to mark ourselves as soft and hard enables in
* order to be able to take interrupts while asleep. We inform lockdep
@@ -44,9 +46,12 @@ _GLOBAL(book3e_idle)
stdu r1,-128(r1)
bl .trace_hardirqs_on
#endif
+
+#ifdef CONFIG_PPC_LAZY_EE
li r0,1
stb r0,PACASOFTIRQEN(r13)
stb r0,PACAHARDIRQEN(r13)
+#endif
/* Interrupts will make use return to LR, so get something we want
* in there
@@ -56,10 +61,12 @@ _GLOBAL(book3e_idle)
/* Hard disable interrupts again */
wrteei 0
+#ifdef CONFIG_PPC_LAZY_EE
/* Mark them off again in the PACA as well */
li r0,0
stb r0,PACASOFTIRQEN(r13)
stb r0,PACAHARDIRQEN(r13)
+#endif
/* Tell lockdep about it */
#ifdef CONFIG_TRACE_IRQFLAGS
diff --git a/arch/powerpc/kernel/irq.c b/arch/powerpc/kernel/irq.c
index 701d4ac..e2d1784 100644
--- a/arch/powerpc/kernel/irq.c
+++ b/arch/powerpc/kernel/irq.c
@@ -99,6 +99,8 @@ EXPORT_SYMBOL(irq_desc);
int distribute_irqs = 1;
+#ifdef CONFIG_PPC_LAZY_EE
+
static inline notrace unsigned long get_hard_enabled(void)
{
unsigned long enabled;
@@ -193,6 +195,9 @@ notrace void arch_local_irq_restore(unsigned long en)
__hard_irq_enable();
}
EXPORT_SYMBOL(arch_local_irq_restore);
+
+#endif /* CONFIG_PPC_LAZY_EE */
+
#endif /* CONFIG_PPC64 */
int arch_show_interrupts(struct seq_file *p, int prec)
diff --git a/arch/powerpc/kernel/setup_64.c b/arch/powerpc/kernel/setup_64.c
index 4cb8f1e..06f5b19 100644
--- a/arch/powerpc/kernel/setup_64.c
+++ b/arch/powerpc/kernel/setup_64.c
@@ -233,8 +233,10 @@ void __init early_setup(unsigned long dt_ptr)
#ifdef CONFIG_SMP
void early_setup_secondary(void)
{
+#ifdef CONFIG_PPC_LAZY_EE
/* Mark interrupts enabled in PACA */
get_paca()->soft_enabled = 0;
+#endif
/* Initialize the hash table or TLB handling */
early_init_mmu_secondary();
--
1.7.5.1
_______________________________________________
Linuxppc-dev mailing list
Linuxppc-dev@lists.ozlabs.org
https://lists.ozlabs.org/listinfo/linuxppc-dev
