This does increase kernel text size by about 0.4%, but code is often
improved by putting the interrupt-replay call out of line, and gcc
function "shrink wrapping" can more often avoid setting up a stack
frame, e.g., _raw_spin_unlock_irqrestore fastpath before:
<_raw_spin_unlock_irqrestore>:
addis r2,r12,63
addi r2,r2,24688
mflr r0
andi. r9,r14,256
mr r9,r3
std r0,16(r1)
stdu r1,-32(r1)
bne c0000000009fd1e0 <_raw_spin_unlock_irqrestore+0x50>
lwsync
li r10,0
mr r3,r4
stw r10,0(r9)
bl c000000000013f98 <arch_local_irq_restore+0x8>
<arch_local_irq_restore>:
addis r2,r12,222
addi r2,r2,-3472
rldimi r14,r3,0,62
cmpdi cr7,r3,0
bnelr cr7
andi. r9,r14,252
beqlr
nop
addi r1,r1,32
ld r0,16(r1)
mtlr r0
blr
And after:
<_raw_spin_unlock_irqrestore>:
addis r2,r12,64
addi r2,r2,-15200
andi. r9,r14,256
bne c000000000a06dd0 <_raw_spin_unlock_irqrestore+0x70>
lwsync
li r9,0
stw r9,0(r3)
rldimi r14,r4,0,62
cmpdi cr7,r4,0
bne cr7,c000000000a06d90 <_raw_spin_unlock_irqrestore+0x30>
andi. r9,r14,252
bne c000000000a06da0 <_raw_spin_unlock_irqrestore+0x40>
blr
GCC can still improve code size for the slow paths by avoiding aligning
branch targets too, so there is room to reduce the text size cost there.
---
arch/powerpc/include/asm/hw_irq.h | 15 +++++++++++++--
arch/powerpc/kernel/irq.c | 28 ++++++----------------------
2 files changed, 19 insertions(+), 24 deletions(-)
diff --git a/arch/powerpc/include/asm/hw_irq.h
b/arch/powerpc/include/asm/hw_irq.h
index f492a7779ea3..8690e0d5605d 100644
--- a/arch/powerpc/include/asm/hw_irq.h
+++ b/arch/powerpc/include/asm/hw_irq.h
@@ -132,11 +132,22 @@ static inline void arch_local_irq_disable(void)
irq_soft_mask_set(IRQ_SOFT_MASK_STD);
}
-extern void arch_local_irq_restore(unsigned long);
+extern void __arch_local_irq_enable(void);
static inline void arch_local_irq_enable(void)
{
- arch_local_irq_restore(0);
+ __irq_soft_mask_clear(IRQ_SOFT_MASK_ALL);
+ if (unlikely(local_r14 & R14_BIT_IRQ_HAPPENED_MASK))
+ __arch_local_irq_enable();
+}
+
+static inline void arch_local_irq_restore(unsigned long flags)
+{
+ __irq_soft_mask_insert(flags);
+ if (!flags) {
+ if (unlikely(local_r14 & R14_BIT_IRQ_HAPPENED_MASK))
+ __arch_local_irq_enable();
+ }
}
static inline unsigned long arch_local_irq_save(void)
diff --git a/arch/powerpc/kernel/irq.c b/arch/powerpc/kernel/irq.c
index ebaf210a7406..e2ff0210477e 100644
--- a/arch/powerpc/kernel/irq.c
+++ b/arch/powerpc/kernel/irq.c
@@ -97,11 +97,6 @@ extern int tau_interrupts(int);
int distribute_irqs = 1;
-static inline notrace unsigned long get_irq_happened(void)
-{
- return local_r14 & R14_BIT_IRQ_HAPPENED_MASK;
-}
-
static inline notrace int decrementer_check_overflow(void)
{
u64 now = get_tb_or_rtc();
@@ -210,19 +205,10 @@ notrace unsigned int __check_irq_replay(void)
return 0;
}
-notrace void arch_local_irq_restore(unsigned long mask)
+notrace void __arch_local_irq_enable(void)
{
- unsigned char irq_happened;
unsigned int replay;
- /* Write the new soft-enabled value */
- __irq_soft_mask_insert(mask);
- /* any bits still disabled */
- if (mask)
- return;
-
- barrier();
-
/*
* From this point onward, we can take interrupts, preempt,
* etc... unless we got hard-disabled. We check if an event
@@ -236,9 +222,6 @@ notrace void arch_local_irq_restore(unsigned long mask)
* be hard-disabled, so there is no problem, we
* cannot have preempted.
*/
- irq_happened = get_irq_happened();
- if (!irq_happened)
- return;
/*
* We need to hard disable to get a trusted value from
@@ -252,10 +235,11 @@ notrace void arch_local_irq_restore(unsigned long mask)
* (expensive) mtmsrd.
* XXX: why not test & IRQ_HARD_DIS?
*/
- if (unlikely(irq_happened != PACA_IRQ_HARD_DIS))
+ if (unlikely((local_r14 & R14_BIT_IRQ_HAPPENED_MASK) !=
+ PACA_IRQ_HARD_DIS)) {
__hard_irq_disable();
#ifdef CONFIG_PPC_IRQ_SOFT_MASK_DEBUG
- else {
+ } else {
/*
* We should already be hard disabled here. We had bugs
* where that wasn't the case so let's dbl check it and
@@ -264,8 +248,8 @@ notrace void arch_local_irq_restore(unsigned long mask)
*/
if (WARN_ON(mfmsr() & MSR_EE))
__hard_irq_disable();
- }
#endif
+ }
__irq_soft_mask_set(IRQ_SOFT_MASK_ALL);
trace_hardirqs_off();
@@ -293,7 +277,7 @@ notrace void arch_local_irq_restore(unsigned long mask)
/* Finally, let's ensure we are hard enabled */
__hard_irq_enable();
}
-EXPORT_SYMBOL(arch_local_irq_restore);
+EXPORT_SYMBOL(__arch_local_irq_enable);
/*
* This is specifically called by assembly code to re-enable interrupts
--
2.15.0
