mpic.c

/*
 *  arch/powerpc/kernel/mpic.c
 *
 *  Driver for interrupt controllers following the OpenPIC standard, the
 *  common implementation beeing IBM's MPIC. This driver also can deal
 *  with various broken implementations of this HW.
 *
 *  Copyright (C) 2004 Benjamin Herrenschmidt, IBM Corp.
 *  Copyright 2010-2011 Freescale Semiconductor, Inc.
 *
 *  This file is subject to the terms and conditions of the GNU General Public
 *  License.  See the file COPYING in the main directory of this archive
 *  for more details.
 */

#undef DEBUG
#undef DEBUG_IPI
#undef DEBUG_IRQ
#undef DEBUG_LOW

#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/irq.h>
#include <linux/smp.h>
#include <linux/interrupt.h>
#include <linux/bootmem.h>
#include <linux/spinlock.h>
#include <linux/pci.h>
#include <linux/slab.h>
#include <linux/syscore_ops.h>
#include <linux/ratelimit.h>

#include <asm/ptrace.h>
#include <asm/signal.h>
#include <asm/io.h>
#include <asm/pgtable.h>
#include <asm/irq.h>
#include <asm/machdep.h>
#include <asm/mpic.h>
#include <asm/smp.h>

#include "mpic.h"

#ifdef DEBUG
#define DBG(fmt...) printk(fmt)
#else
#define DBG(fmt...)
#endif

static struct mpic *mpics;
static struct mpic *mpic_primary;
static DEFINE_RAW_SPINLOCK(mpic_lock);

#ifdef CONFIG_PPC32	/* XXX for now */
#ifdef CONFIG_IRQ_ALL_CPUS
#define distribute_irqs	(1)
#else
#define distribute_irqs	(0)
#endif
#endif

#ifdef CONFIG_MPIC_WEIRD
static u32 mpic_infos[][MPIC_IDX_END] = {
	[0] = {	/* Original OpenPIC compatible MPIC */
		MPIC_GREG_BASE,
		MPIC_GREG_FEATURE_0,
		MPIC_GREG_GLOBAL_CONF_0,
		MPIC_GREG_VENDOR_ID,
		MPIC_GREG_IPI_VECTOR_PRI_0,
		MPIC_GREG_IPI_STRIDE,
		MPIC_GREG_SPURIOUS,
		MPIC_GREG_TIMER_FREQ,

		MPIC_TIMER_BASE,
		MPIC_TIMER_STRIDE,
		MPIC_TIMER_CURRENT_CNT,
		MPIC_TIMER_BASE_CNT,
		MPIC_TIMER_VECTOR_PRI,
		MPIC_TIMER_DESTINATION,

		MPIC_CPU_BASE,
		MPIC_CPU_STRIDE,
		MPIC_CPU_IPI_DISPATCH_0,
		MPIC_CPU_IPI_DISPATCH_STRIDE,
		MPIC_CPU_CURRENT_TASK_PRI,
		MPIC_CPU_WHOAMI,
		MPIC_CPU_INTACK,
		MPIC_CPU_EOI,
		MPIC_CPU_MCACK,

		MPIC_IRQ_BASE,
		MPIC_IRQ_STRIDE,
		MPIC_IRQ_VECTOR_PRI,
		MPIC_VECPRI_VECTOR_MASK,
		MPIC_VECPRI_POLARITY_POSITIVE,
		MPIC_VECPRI_POLARITY_NEGATIVE,
		MPIC_VECPRI_SENSE_LEVEL,
		MPIC_VECPRI_SENSE_EDGE,
		MPIC_VECPRI_POLARITY_MASK,
		MPIC_VECPRI_SENSE_MASK,
		MPIC_IRQ_DESTINATION
	},
	[1] = {	/* Tsi108/109 PIC */
		TSI108_GREG_BASE,
		TSI108_GREG_FEATURE_0,
		TSI108_GREG_GLOBAL_CONF_0,
		TSI108_GREG_VENDOR_ID,
		TSI108_GREG_IPI_VECTOR_PRI_0,
		TSI108_GREG_IPI_STRIDE,
		TSI108_GREG_SPURIOUS,
		TSI108_GREG_TIMER_FREQ,

		TSI108_TIMER_BASE,
		TSI108_TIMER_STRIDE,
		TSI108_TIMER_CURRENT_CNT,
		TSI108_TIMER_BASE_CNT,
		TSI108_TIMER_VECTOR_PRI,
		TSI108_TIMER_DESTINATION,

		TSI108_CPU_BASE,
		TSI108_CPU_STRIDE,
		TSI108_CPU_IPI_DISPATCH_0,
		TSI108_CPU_IPI_DISPATCH_STRIDE,
		TSI108_CPU_CURRENT_TASK_PRI,
		TSI108_CPU_WHOAMI,
		TSI108_CPU_INTACK,
		TSI108_CPU_EOI,
		TSI108_CPU_MCACK,

		TSI108_IRQ_BASE,
		TSI108_IRQ_STRIDE,
		TSI108_IRQ_VECTOR_PRI,
		TSI108_VECPRI_VECTOR_MASK,
		TSI108_VECPRI_POLARITY_POSITIVE,
		TSI108_VECPRI_POLARITY_NEGATIVE,
		TSI108_VECPRI_SENSE_LEVEL,
		TSI108_VECPRI_SENSE_EDGE,
		TSI108_VECPRI_POLARITY_MASK,
		TSI108_VECPRI_SENSE_MASK,
		TSI108_IRQ_DESTINATION
	},
};

#define MPIC_INFO(name) mpic->hw_set[MPIC_IDX_##name]

#else /* CONFIG_MPIC_WEIRD */

#define MPIC_INFO(name) MPIC_##name

#endif /* CONFIG_MPIC_WEIRD */

static inline unsigned int mpic_processor_id(struct mpic *mpic)
{
	unsigned int cpu = 0;

	if (!(mpic->flags & MPIC_SECONDARY))
		cpu = hard_smp_processor_id();

	return cpu;
}

/*
 * Register accessor functions
 */


static inline u32 _mpic_read(enum mpic_reg_type type,
			     struct mpic_reg_bank *rb,
			     unsigned int reg)
{
	switch(type) {
#ifdef CONFIG_PPC_DCR
	case mpic_access_dcr:
		return dcr_read(rb->dhost, reg);
#endif
	case mpic_access_mmio_be:
		return in_be32(rb->base + (reg >> 2));
	case mpic_access_mmio_le:
	default:
		return in_le32(rb->base + (reg >> 2));
	}
}

static inline void _mpic_write(enum mpic_reg_type type,
			       struct mpic_reg_bank *rb,
 			       unsigned int reg, u32 value)
{
	switch(type) {
#ifdef CONFIG_PPC_DCR
	case mpic_access_dcr:
		dcr_write(rb->dhost, reg, value);
		break;
#endif
	case mpic_access_mmio_be:
		out_be32(rb->base + (reg >> 2), value);
		break;
	case mpic_access_mmio_le:
	default:
		out_le32(rb->base + (reg >> 2), value);
		break;
	}
}

static inline u32 _mpic_ipi_read(struct mpic *mpic, unsigned int ipi)
{
	enum mpic_reg_type type = mpic->reg_type;
	unsigned int offset = MPIC_INFO(GREG_IPI_VECTOR_PRI_0) +
			      (ipi * MPIC_INFO(GREG_IPI_STRIDE));

	if ((mpic->flags & MPIC_BROKEN_IPI) && type == mpic_access_mmio_le)
		type = mpic_access_mmio_be;
	return _mpic_read(type, &mpic->gregs, offset);
}

static inline void _mpic_ipi_write(struct mpic *mpic, unsigned int ipi, u32 value)
{
	unsigned int offset = MPIC_INFO(GREG_IPI_VECTOR_PRI_0) +
			      (ipi * MPIC_INFO(GREG_IPI_STRIDE));

	_mpic_write(mpic->reg_type, &mpic->gregs, offset, value);
}

static inline u32 _mpic_tm_read(struct mpic *mpic, unsigned int tm)
{
	unsigned int offset = MPIC_INFO(TIMER_VECTOR_PRI) +
			      ((tm & 3) * MPIC_INFO(TIMER_STRIDE));

	if (tm >= 4)
		offset += 0x1000 / 4;

	return _mpic_read(mpic->reg_type, &mpic->tmregs, offset);
}

static inline void _mpic_tm_write(struct mpic *mpic, unsigned int tm, u32 value)
{
	unsigned int offset = MPIC_INFO(TIMER_VECTOR_PRI) +
			      ((tm & 3) * MPIC_INFO(TIMER_STRIDE));

	if (tm >= 4)
		offset += 0x1000 / 4;

	_mpic_write(mpic->reg_type, &mpic->tmregs, offset, value);
}

static inline u32 _mpic_cpu_read(struct mpic *mpic, unsigned int reg)
{
	unsigned int cpu = mpic_processor_id(mpic);

	return _mpic_read(mpic->reg_type, &mpic->cpuregs[cpu], reg);
}

static inline void _mpic_cpu_write(struct mpic *mpic, unsigned int reg, u32 value)
{
	unsigned int cpu = mpic_processor_id(mpic);

	_mpic_write(mpic->reg_type, &mpic->cpuregs[cpu], reg, value);
}

static inline u32 _mpic_irq_read(struct mpic *mpic, unsigned int src_no, unsigned int reg)
{
	unsigned int	isu = src_no >> mpic->isu_shift;
	unsigned int	idx = src_no & mpic->isu_mask;
	unsigned int	val;

	val = _mpic_read(mpic->reg_type, &mpic->isus[isu],
			 reg + (idx * MPIC_INFO(IRQ_STRIDE)));
#ifdef CONFIG_MPIC_BROKEN_REGREAD
	if (reg == 0)
		val = (val & (MPIC_VECPRI_MASK | MPIC_VECPRI_ACTIVITY)) |
			mpic->isu_reg0_shadow[src_no];
#endif
	return val;
}

static inline void _mpic_irq_write(struct mpic *mpic, unsigned int src_no,
				   unsigned int reg, u32 value)
{
	unsigned int	isu = src_no >> mpic->isu_shift;
	unsigned int	idx = src_no & mpic->isu_mask;

	_mpic_write(mpic->reg_type, &mpic->isus[isu],
		    reg + (idx * MPIC_INFO(IRQ_STRIDE)), value);

#ifdef CONFIG_MPIC_BROKEN_REGREAD
	if (reg == 0)
		mpic->isu_reg0_shadow[src_no] =
			value & ~(MPIC_VECPRI_MASK | MPIC_VECPRI_ACTIVITY);
#endif
}

#define mpic_read(b,r)		_mpic_read(mpic->reg_type,&(b),(r))
#define mpic_write(b,r,v)	_mpic_write(mpic->reg_type,&(b),(r),(v))
#define mpic_ipi_read(i)	_mpic_ipi_read(mpic,(i))
#define mpic_ipi_write(i,v)	_mpic_ipi_write(mpic,(i),(v))
#define mpic_tm_read(i)		_mpic_tm_read(mpic,(i))
#define mpic_tm_write(i,v)	_mpic_tm_write(mpic,(i),(v))
#define mpic_cpu_read(i)	_mpic_cpu_read(mpic,(i))
#define mpic_cpu_write(i,v)	_mpic_cpu_write(mpic,(i),(v))
#define mpic_irq_read(s,r)	_mpic_irq_read(mpic,(s),(r))
#define mpic_irq_write(s,r,v)	_mpic_irq_write(mpic,(s),(r),(v))


/*
 * Low level utility functions
 */


static void _mpic_map_mmio(struct mpic *mpic, phys_addr_t phys_addr,
			   struct mpic_reg_bank *rb, unsigned int offset,
			   unsigned int size)
{
	rb->base = ioremap(phys_addr + offset, size);
	BUG_ON(rb->base == NULL);
}

#ifdef CONFIG_PPC_DCR
static void _mpic_map_dcr(struct mpic *mpic, struct mpic_reg_bank *rb,
			  unsigned int offset, unsigned int size)
{
	phys_addr_t phys_addr = dcr_resource_start(mpic->node, 0);
	rb->dhost = dcr_map(mpic->node, phys_addr + offset, size);
	BUG_ON(!DCR_MAP_OK(rb->dhost));
}

static inline void mpic_map(struct mpic *mpic,
			    phys_addr_t phys_addr, struct mpic_reg_bank *rb,
			    unsigned int offset, unsigned int size)
{
	if (mpic->flags & MPIC_USES_DCR)
		_mpic_map_dcr(mpic, rb, offset, size);
	else
		_mpic_map_mmio(mpic, phys_addr, rb, offset, size);
}
#else /* CONFIG_PPC_DCR */
#define mpic_map(m,p,b,o,s)	_mpic_map_mmio(m,p,b,o,s)
#endif /* !CONFIG_PPC_DCR */



/* Check if we have one of those nice broken MPICs with a flipped endian on
 * reads from IPI registers
 */
static void __init mpic_test_broken_ipi(struct mpic *mpic)
{
	u32 r;

	mpic_write(mpic->gregs, MPIC_INFO(GREG_IPI_VECTOR_PRI_0), MPIC_VECPRI_MASK);
	r = mpic_read(mpic->gregs, MPIC_INFO(GREG_IPI_VECTOR_PRI_0));

	if (r == le32_to_cpu(MPIC_VECPRI_MASK)) {
		printk(KERN_INFO "mpic: Detected reversed IPI registers\n");
		mpic->flags |= MPIC_BROKEN_IPI;
	}
}

#ifdef CONFIG_MPIC_U3_HT_IRQS

/* Test if an interrupt is sourced from HyperTransport (used on broken U3s)
 * to force the edge setting on the MPIC and do the ack workaround.
 */
static inline int mpic_is_ht_interrupt(struct mpic *mpic, unsigned int source)
{
	if (source >= 128 || !mpic->fixups)
		return 0;
	return mpic->fixups[source].base != NULL;
}


static inline void mpic_ht_end_irq(struct mpic *mpic, unsigned int source)
{
	struct mpic_irq_fixup *fixup = &mpic->fixups[source];

	if (fixup->applebase) {
		unsigned int soff = (fixup->index >> 3) & ~3;
		unsigned int mask = 1U << (fixup->index & 0x1f);
		writel(mask, fixup->applebase + soff);
	} else {
		raw_spin_lock(&mpic->fixup_lock);
		writeb(0x11 + 2 * fixup->index, fixup->base + 2);
		writel(fixup->data, fixup->base + 4);
		raw_spin_unlock(&mpic->fixup_lock);
	}
}

static void mpic_startup_ht_interrupt(struct mpic *mpic, unsigned int source,
				      bool level)
{
	struct mpic_irq_fixup *fixup = &mpic->fixups[source];
	unsigned long flags;
	u32 tmp;

	if (fixup->base == NULL)
		return;

	DBG("startup_ht_interrupt(0x%x) index: %d\n",
	    source, fixup->index);
	raw_spin_lock_irqsave(&mpic->fixup_lock, flags);
	/* Enable and configure */
	writeb(0x10 + 2 * fixup->index, fixup->base + 2);
	tmp = readl(fixup->base + 4);
	tmp &= ~(0x23U);
	if (level)
		tmp |= 0x22;
	writel(tmp, fixup->base + 4);
	raw_spin_unlock_irqrestore(&mpic->fixup_lock, flags);

#ifdef CONFIG_PM
	/* use the lowest bit inverted to the actual HW,
	 * set if this fixup was enabled, clear otherwise */
	mpic->save_data[source].fixup_data = tmp | 1;
#endif
}

static void mpic_shutdown_ht_interrupt(struct mpic *mpic, unsigned int source)
{
	struct mpic_irq_fixup *fixup = &mpic->fixups[source];
	unsigned long flags;
	u32 tmp;

	if (fixup->base == NULL)
		return;

	DBG("shutdown_ht_interrupt(0x%x)\n", source);

	/* Disable */
	raw_spin_lock_irqsave(&mpic->fixup_lock, flags);
	writeb(0x10 + 2 * fixup->index, fixup->base + 2);
	tmp = readl(fixup->base + 4);
	tmp |= 1;
	writel(tmp, fixup->base + 4);
	raw_spin_unlock_irqrestore(&mpic->fixup_lock, flags);

#ifdef CONFIG_PM
	/* use the lowest bit inverted to the actual HW,
	 * set if this fixup was enabled, clear otherwise */
	mpic->save_data[source].fixup_data = tmp & ~1;
#endif
}

#ifdef CONFIG_PCI_MSI
static void __init mpic_scan_ht_msi(struct mpic *mpic, u8 __iomem *devbase,
				    unsigned int devfn)
{
	u8 __iomem *base;
	u8 pos, flags;
	u64 addr = 0;

	for (pos = readb(devbase + PCI_CAPABILITY_LIST); pos != 0;
	     pos = readb(devbase + pos + PCI_CAP_LIST_NEXT)) {
		u8 id = readb(devbase + pos + PCI_CAP_LIST_ID);
		if (id == PCI_CAP_ID_HT) {
			id = readb(devbase + pos + 3);
			if ((id & HT_5BIT_CAP_MASK) == HT_CAPTYPE_MSI_MAPPING)
				break;
		}
	}

	if (pos == 0)
		return;

	base = devbase + pos;

	flags = readb(base + HT_MSI_FLAGS);
	if (!(flags & HT_MSI_FLAGS_FIXED)) {
		addr = readl(base + HT_MSI_ADDR_LO) & HT_MSI_ADDR_LO_MASK;
		addr = addr | ((u64)readl(base + HT_MSI_ADDR_HI) << 32);
	}

	printk(KERN_DEBUG "mpic:   - HT:%02x.%x %s MSI mapping found @ 0x%llx\n",
		PCI_SLOT(devfn), PCI_FUNC(devfn),
		flags & HT_MSI_FLAGS_ENABLE ? "enabled" : "disabled", addr);

	if (!(flags & HT_MSI_FLAGS_ENABLE))
		writeb(flags | HT_MSI_FLAGS_ENABLE, base + HT_MSI_FLAGS);
}
#else
static void __init mpic_scan_ht_msi(struct mpic *mpic, u8 __iomem *devbase,
				    unsigned int devfn)
{
	return;
}
#endif

static void __init mpic_scan_ht_pic(struct mpic *mpic, u8 __iomem *devbase,
				    unsigned int devfn, u32 vdid)
{
	int i, irq, n;
	u8 __iomem *base;
	u32 tmp;
	u8 pos;

	for (pos = readb(devbase + PCI_CAPABILITY_LIST); pos != 0;
	     pos = readb(devbase + pos + PCI_CAP_LIST_NEXT)) {
		u8 id = readb(devbase + pos + PCI_CAP_LIST_ID);
		if (id == PCI_CAP_ID_HT) {
			id = readb(devbase + pos + 3);
			if ((id & HT_5BIT_CAP_MASK) == HT_CAPTYPE_IRQ)
				break;
		}
	}
	if (pos == 0)
		return;

	base = devbase + pos;
	writeb(0x01, base + 2);
	n = (readl(base + 4) >> 16) & 0xff;

	printk(KERN_INFO "mpic:   - HT:%02x.%x [0x%02x] vendor %04x device %04x"
	       " has %d irqs\n",
	       devfn >> 3, devfn & 0x7, pos, vdid & 0xffff, vdid >> 16, n + 1);

	for (i = 0; i <= n; i++) {
		writeb(0x10 + 2 * i, base + 2);
		tmp = readl(base + 4);
		irq = (tmp >> 16) & 0xff;
		DBG("HT PIC index 0x%x, irq 0x%x, tmp: %08x\n", i, irq, tmp);
		/* mask it , will be unmasked later */
		tmp |= 0x1;
		writel(tmp, base + 4);
		mpic->fixups[irq].index = i;
		mpic->fixups[irq].base = base;
		/* Apple HT PIC has a non-standard way of doing EOIs */
		if ((vdid & 0xffff) == 0x106b)
			mpic->fixups[irq].applebase = devbase + 0x60;
		else
			mpic->fixups[irq].applebase = NULL;
		writeb(0x11 + 2 * i, base + 2);
		mpic->fixups[irq].data = readl(base + 4) | 0x80000000;
	}
}
 

static void __init mpic_scan_ht_pics(struct mpic *mpic)
{
	unsigned int devfn;
	u8 __iomem *cfgspace;

	printk(KERN_INFO "mpic: Setting up HT PICs workarounds for U3/U4\n");

	/* Allocate fixups array */
	mpic->fixups = kzalloc(128 * sizeof(*mpic->fixups), GFP_KERNEL);
	BUG_ON(mpic->fixups == NULL);

	/* Init spinlock */
	raw_spin_lock_init(&mpic->fixup_lock);

	/* Map U3 config space. We assume all IO-APICs are on the primary bus
	 * so we only need to map 64kB.
	 */
	cfgspace = ioremap(0xf2000000, 0x10000);
	BUG_ON(cfgspace == NULL);

	/* Now we scan all slots. We do a very quick scan, we read the header
	 * type, vendor ID and device ID only, that's plenty enough
	 */
	for (devfn = 0; devfn < 0x100; devfn++) {
		u8 __iomem *devbase = cfgspace + (devfn << 8);
		u8 hdr_type = readb(devbase + PCI_HEADER_TYPE);
		u32 l = readl(devbase + PCI_VENDOR_ID);
		u16 s;

		DBG("devfn %x, l: %x\n", devfn, l);

		/* If no device, skip */
		if (l == 0xffffffff || l == 0x00000000 ||
		    l == 0x0000ffff || l == 0xffff0000)
			goto next;
		/* Check if is supports capability lists */
		s = readw(devbase + PCI_STATUS);
		if (!(s & PCI_STATUS_CAP_LIST))
			goto next;

		mpic_scan_ht_pic(mpic, devbase, devfn, l);
		mpic_scan_ht_msi(mpic, devbase, devfn);

	next:
		/* next device, if function 0 */
		if (PCI_FUNC(devfn) == 0 && (hdr_type & 0x80) == 0)
			devfn += 7;
	}
}

#else /* CONFIG_MPIC_U3_HT_IRQS */

static inline int mpic_is_ht_interrupt(struct mpic *mpic, unsigned int source)
{
	return 0;
}

static void __init mpic_scan_ht_pics(struct mpic *mpic)
{
}

#endif /* CONFIG_MPIC_U3_HT_IRQS */

/* Find an mpic associated with a given linux interrupt */
static struct mpic *mpic_find(unsigned int irq)
{
	if (irq < NUM_ISA_INTERRUPTS)
		return NULL;

	return irq_get_chip_data(irq);
}

/* Determine if the linux irq is an IPI */
static unsigned int mpic_is_ipi(struct mpic *mpic, unsigned int src)
{
	return (src >= mpic->ipi_vecs[0] && src <= mpic->ipi_vecs[3]);
}

/* Determine if the linux irq is a timer */
static unsigned int mpic_is_tm(struct mpic *mpic, unsigned int src)
{
	return (src >= mpic->timer_vecs[0] && src <= mpic->timer_vecs[7]);
}

/* Convert a cpu mask from logical to physical cpu numbers. */
static inline u32 mpic_physmask(u32 cpumask)
{
	int i;
	u32 mask = 0;

	for (i = 0; i < min(32, NR_CPUS); ++i, cpumask >>= 1)
		mask |= (cpumask & 1) << get_hard_smp_processor_id(i);
	return mask;
}

#ifdef CONFIG_SMP
/* Get the mpic structure from the IPI number */
static inline struct mpic * mpic_from_ipi(struct irq_data *d)
{
	return irq_data_get_irq_chip_data(d);
}
#endif

/* Get the mpic structure from the irq number */
static inline struct mpic * mpic_from_irq(unsigned int irq)
{
	return irq_get_chip_data(irq);
}

/* Get the mpic structure from the irq data */
static inline struct mpic * mpic_from_irq_data(struct irq_data *d)
{
	return irq_data_get_irq_chip_data(d);
}

/* Send an EOI */
static inline void mpic_eoi(struct mpic *mpic)
{
	mpic_cpu_write(MPIC_INFO(CPU_EOI), 0);
	(void)mpic_cpu_read(MPIC_INFO(CPU_WHOAMI));
}

/*
 * Linux descriptor level callbacks
 */


void mpic_unmask_irq(struct irq_data *d)
{
	unsigned int loops = 100000;
	struct mpic *mpic = mpic_from_irq_data(d);
	unsigned int src = irqd_to_hwirq(d);

	DBG("%p: %s: enable_irq: %d (src %d)\n", mpic, mpic->name, d->irq, src);

	mpic_irq_write(src, MPIC_INFO(IRQ_VECTOR_PRI),
		       mpic_irq_read(src, MPIC_INFO(IRQ_VECTOR_PRI)) &
		       ~MPIC_VECPRI_MASK);
	/* make sure mask gets to controller before we return to user */
	do {
		if (!loops--) {
			printk(KERN_ERR "%s: timeout on hwirq %u\n",
			       __func__, src);
			break;
		}
	} while(mpic_irq_read(src, MPIC_INFO(IRQ_VECTOR_PRI)) & MPIC_VECPRI_MASK);
}

void mpic_mask_irq(struct irq_data *d)
{
	unsigned int loops = 100000;
	struct mpic *mpic = mpic_from_irq_data(d);
	unsigned int src = irqd_to_hwirq(d);

	DBG("%s: disable_irq: %d (src %d)\n", mpic->name, d->irq, src);

	mpic_irq_write(src, MPIC_INFO(IRQ_VECTOR_PRI),
		       mpic_irq_read(src, MPIC_INFO(IRQ_VECTOR_PRI)) |
		       MPIC_VECPRI_MASK);

	/* make sure mask gets to controller before we return to user */
	do {
		if (!loops--) {
			printk(KERN_ERR "%s: timeout on hwirq %u\n",
			       __func__, src);
			break;
		}
	} while(!(mpic_irq_read(src, MPIC_INFO(IRQ_VECTOR_PRI)) & MPIC_VECPRI_MASK));
}

void mpic_end_irq(struct irq_data *d)
{
	struct mpic *mpic = mpic_from_irq_data(d);

#ifdef DEBUG_IRQ
	DBG("%s: end_irq: %d\n", mpic->name, d->irq);
#endif
	/* We always EOI on end_irq() even for edge interrupts since that
	 * should only lower the priority, the MPIC should have properly
	 * latched another edge interrupt coming in anyway
	 */

	mpic_eoi(mpic);
}

#ifdef CONFIG_MPIC_U3_HT_IRQS

static void mpic_unmask_ht_irq(struct irq_data *d)
{
	struct mpic *mpic = mpic_from_irq_data(d);
	unsigned int src = irqd_to_hwirq(d);

	mpic_unmask_irq(d);

	if (irqd_is_level_type(d))
		mpic_ht_end_irq(mpic, src);
}

static unsigned int mpic_startup_ht_irq(struct irq_data *d)
{
	struct mpic *mpic = mpic_from_irq_data(d);
	unsigned int src = irqd_to_hwirq(d);

	mpic_unmask_irq(d);
	mpic_startup_ht_interrupt(mpic, src, irqd_is_level_type(d));

	return 0;
}

static void mpic_shutdown_ht_irq(struct irq_data *d)
{
	struct mpic *mpic = mpic_from_irq_data(d);
	unsigned int src = irqd_to_hwirq(d);

	mpic_shutdown_ht_interrupt(mpic, src);
	mpic_mask_irq(d);
}

static void mpic_end_ht_irq(struct irq_data *d)
{
	struct mpic *mpic = mpic_from_irq_data(d);
	unsigned int src = irqd_to_hwirq(d);

#ifdef DEBUG_IRQ
	DBG("%s: end_irq: %d\n", mpic->name, d->irq);
#endif
	/* We always EOI on end_irq() even for edge interrupts since that
	 * should only lower the priority, the MPIC should have properly
	 * latched another edge interrupt coming in anyway
	 */

	if (irqd_is_level_type(d))
		mpic_ht_end_irq(mpic, src);
	mpic_eoi(mpic);
}
#endif /* !CONFIG_MPIC_U3_HT_IRQS */

#ifdef CONFIG_SMP

static void mpic_unmask_ipi(struct irq_data *d)
{
	struct mpic *mpic = mpic_from_ipi(d);
	unsigned int src = virq_to_hw(d->irq) - mpic->ipi_vecs[0];

	DBG("%s: enable_ipi: %d (ipi %d)\n", mpic->name, d->irq, src);
	mpic_ipi_write(src, mpic_ipi_read(src) & ~MPIC_VECPRI_MASK);
}

static void mpic_mask_ipi(struct irq_data *d)
{
	/* NEVER disable an IPI... that's just plain wrong! */
}

static void mpic_end_ipi(struct irq_data *d)
{
	struct mpic *mpic = mpic_from_ipi(d);

	/*
	 * IPIs are marked IRQ_PER_CPU. This has the side effect of
	 * preventing the IRQ_PENDING/IRQ_INPROGRESS logic from
	 * applying to them. We EOI them late to avoid re-entering.
	 */
	mpic_eoi(mpic);
}

#endif /* CONFIG_SMP */

static void mpic_unmask_tm(struct irq_data *d)
{
	struct mpic *mpic = mpic_from_irq_data(d);
	unsigned int src = virq_to_hw(d->irq) - mpic->timer_vecs[0];

	DBG("%s: enable_tm: %d (tm %d)\n", mpic->name, d->irq, src);
	mpic_tm_write(src, mpic_tm_read(src) & ~MPIC_VECPRI_MASK);
	mpic_tm_read(src);
}

static void mpic_mask_tm(struct irq_data *d)
{
	struct mpic *mpic = mpic_from_irq_data(d);
	unsigned int src = virq_to_hw(d->irq) - mpic->timer_vecs[0];

	mpic_tm_write(src, mpic_tm_read(src) | MPIC_VECPRI_MASK);
	mpic_tm_read(src);
}

int mpic_set_affinity(struct irq_data *d, const struct cpumask *cpumask,
		      bool force)
{
	struct mpic *mpic = mpic_from_irq_data(d);
	unsigned int src = irqd_to_hwirq(d);

	if (mpic->flags & MPIC_SINGLE_DEST_CPU) {
		int cpuid = irq_choose_cpu(cpumask);

		mpic_irq_write(src, MPIC_INFO(IRQ_DESTINATION), 1 << cpuid);
	} else {
		u32 mask = cpumask_bits(cpumask)[0];

		mask &= cpumask_bits(cpu_online_mask)[0];

		mpic_irq_write(src, MPIC_INFO(IRQ_DESTINATION),
			       mpic_physmask(mask));
	}

	return 0;
}

static unsigned int mpic_type_to_vecpri(struct mpic *mpic, unsigned int type)
{
	/* Now convert sense value */
	switch(type & IRQ_TYPE_SENSE_MASK) {
	case IRQ_TYPE_EDGE_RISING:
		return MPIC_INFO(VECPRI_SENSE_EDGE) |
		       MPIC_INFO(VECPRI_POLARITY_POSITIVE);
	case IRQ_TYPE_EDGE_FALLING:
	case IRQ_TYPE_EDGE_BOTH:
		return MPIC_INFO(VECPRI_SENSE_EDGE) |
		       MPIC_INFO(VECPRI_POLARITY_NEGATIVE);
	case IRQ_TYPE_LEVEL_HIGH:
		return MPIC_INFO(VECPRI_SENSE_LEVEL) |
		       MPIC_INFO(VECPRI_POLARITY_POSITIVE);
	case IRQ_TYPE_LEVEL_LOW:
	default:
		return MPIC_INFO(VECPRI_SENSE_LEVEL) |
		       MPIC_INFO(VECPRI_POLARITY_NEGATIVE);
	}
}

int mpic_set_irq_type(struct irq_data *d, unsigned int flow_type)
{
	struct mpic *mpic = mpic_from_irq_data(d);
	unsigned int src = irqd_to_hwirq(d);
	unsigned int vecpri, vold, vnew;

	DBG("mpic: set_irq_type(mpic:@%p,virq:%d,src:0x%x,type:0x%x)\n",
	    mpic, d->irq, src, flow_type);

	if (src >= mpic->num_sources)
		return -EINVAL;

	vold = mpic_irq_read(src, MPIC_INFO(IRQ_VECTOR_PRI));

	/* We don't support "none" type */
	if (flow_type == IRQ_TYPE_NONE)
		flow_type = IRQ_TYPE_DEFAULT;

	/* Default: read HW settings */
	if (flow_type == IRQ_TYPE_DEFAULT) {
		switch(vold & (MPIC_INFO(VECPRI_POLARITY_MASK) |
			       MPIC_INFO(VECPRI_SENSE_MASK))) {
			case MPIC_INFO(VECPRI_SENSE_EDGE) |
			     MPIC_INFO(VECPRI_POLARITY_POSITIVE):
				flow_type = IRQ_TYPE_EDGE_RISING;
				break;
			case MPIC_INFO(VECPRI_SENSE_EDGE) |
			     MPIC_INFO(VECPRI_POLARITY_NEGATIVE):
				flow_type = IRQ_TYPE_EDGE_FALLING;
				break;
			case MPIC_INFO(VECPRI_SENSE_LEVEL) |
			     MPIC_INFO(VECPRI_POLARITY_POSITIVE):
				flow_type = IRQ_TYPE_LEVEL_HIGH;
				break;
			case MPIC_INFO(VECPRI_SENSE_LEVEL) |
			     MPIC_INFO(VECPRI_POLARITY_NEGATIVE):
				flow_type = IRQ_TYPE_LEVEL_LOW;
				break;
		}
	}

	/* Apply to irq desc */
	irqd_set_trigger_type(d, flow_type);

	/* Apply to HW */
	if (mpic_is_ht_interrupt(mpic, src))
		vecpri = MPIC_VECPRI_POLARITY_POSITIVE |
			MPIC_VECPRI_SENSE_EDGE;
	else
		vecpri = mpic_type_to_vecpri(mpic, flow_type);

	vnew = vold & ~(MPIC_INFO(VECPRI_POLARITY_MASK) |
			MPIC_INFO(VECPRI_SENSE_MASK));
	vnew |= vecpri;
	if (vold != vnew)
		mpic_irq_write(src, MPIC_INFO(IRQ_VECTOR_PRI), vnew);

	return IRQ_SET_MASK_OK_NOCOPY;
}

void mpic_set_vector(unsigned int virq, unsigned int vector)
{
	struct mpic *mpic = mpic_from_irq(virq);
	unsigned int src = virq_to_hw(virq);
	unsigned int vecpri;

	DBG("mpic: set_vector(mpic:@%p,virq:%d,src:%d,vector:0x%x)\n",
	    mpic, virq, src, vector);

	if (src >= mpic->num_sources)
		return;

	vecpri = mpic_irq_read(src, MPIC_INFO(IRQ_VECTOR_PRI));
	vecpri = vecpri & ~MPIC_INFO(VECPRI_VECTOR_MASK);
	vecpri |= vector;
	mpic_irq_write(src, MPIC_INFO(IRQ_VECTOR_PRI), vecpri);
}

void mpic_set_destination(unsigned int virq, unsigned int cpuid)
{
	struct mpic *mpic = mpic_from_irq(virq);
	unsigned int src = virq_to_hw(virq);

	DBG("mpic: set_destination(mpic:@%p,virq:%d,src:%d,cpuid:0x%x)\n",
	    mpic, virq, src, cpuid);

	if (src >= mpic->num_sources)
		return;

	mpic_irq_write(src, MPIC_INFO(IRQ_DESTINATION), 1 << cpuid);
}

static struct irq_chip mpic_irq_chip = {
	.irq_mask	= mpic_mask_irq,
	.irq_unmask	= mpic_unmask_irq,
	.irq_eoi	= mpic_end_irq,
	.irq_set_type	= mpic_set_irq_type,
};

#ifdef CONFIG_SMP
static struct irq_chip mpic_ipi_chip = {
	.irq_mask	= mpic_mask_ipi,
	.irq_unmask	= mpic_unmask_ipi,
	.irq_eoi	= mpic_end_ipi,
};
#endif /* CONFIG_SMP */

static struct irq_chip mpic_tm_chip = {
	.irq_mask	= mpic_mask_tm,
	.irq_unmask	= mpic_unmask_tm,
	.irq_eoi	= mpic_end_irq,
};

#ifdef CONFIG_MPIC_U3_HT_IRQS
static struct irq_chip mpic_irq_ht_chip = {
	.irq_startup	= mpic_startup_ht_irq,
	.irq_shutdown	= mpic_shutdown_ht_irq,
	.irq_mask	= mpic_mask_irq,
	.irq_unmask	= mpic_unmask_ht_irq,
	.irq_eoi	= mpic_end_ht_irq,
	.irq_set_type	= mpic_set_irq_type,
};
#endif /* CONFIG_MPIC_U3_HT_IRQS */


static int mpic_host_match(struct irq_domain *h, struct device_node *node)
{
	/* Exact match, unless mpic node is NULL */
	return h->of_node == NULL || h->of_node == node;
}

static int mpic_host_map(struct irq_domain *h, unsigned int virq,
			 irq_hw_number_t hw)
{
	struct mpic *mpic = h->host_data;
	struct irq_chip *chip;

	DBG("mpic: map virq %d, hwirq 0x%lx\n", virq, hw);