// SPDX-License-Identifier: Apache-2.0
/*
* XIVE2: eXternal Interrupt Virtualization Engine. POWER10 interrupt
* controller
*
* Copyright (c) 2016-2019, IBM Corporation.
*/
#define pr_fmt(fmt) "XIVE: " fmt
#include <skiboot.h>
#include <xscom.h>
#include <chip.h>
#include <io.h>
#include <xive.h>
#include <xive2-regs.h>
#include <xscom-p10-regs.h>
#include <interrupts.h>
#include <timebase.h>
#include <bitmap.h>
#include <buddy.h>
#include <phys-map.h>
#include <p10_stop_api.H>
/* Verbose debug */
#undef XIVE_VERBOSE_DEBUG
#undef DEBUG
/* Extra debug options used in debug builds */
#ifdef DEBUG
#define XIVE_CHECK_LOCKS
#define XIVE_DEBUG_INIT_CACHE_UPDATES
#define XIVE_EXTRA_CHECK_INIT_CACHE
#else
#undef XIVE_CHECK_LOCKS
#undef XIVE_DEBUG_INIT_CACHE_UPDATES
#undef XIVE_EXTRA_CHECK_INIT_CACHE
#endif
/*
* VSDs, blocks, set translation etc...
*
* For the following data structures, the XIVE use a mechanism called
* Virtualization Structure Tables (VST) to manage the memory layout
* and access: ESBs (Event State Buffers), EAS (Event assignment
* structures), ENDs (Event Notification Descriptors) and NVT/NVP
* (Notification Virtual Targets/Processors).
*
* These structures divide those tables into 16 "blocks". Each XIVE
* instance has a definition for all 16 blocks that can either represent
* an actual table in memory or a remote XIVE MMIO port to access a
* block that is owned by that remote XIVE.
*
* Our SW design will consist of allocating one block per chip (and thus
* per XIVE instance) for now, thus giving us up to 16 supported chips in
* the system. We may have to revisit that if we ever support systems with
* more than 16 chips but that isn't on our radar at the moment or if we
* want to do like pHyp on some machines and dedicate 2 blocks per chip
* for some structures.
*
* Thus we need to be careful that we never expose to Linux the concept
* of block and block boundaries, but instead we provide full number ranges
* so that consecutive blocks can be supported.
*
* Similarily, for MMIO access, the BARs support what is called "set
* translation" which allows the BAR to be devided into a certain
* number of sets. Each "set" can be routed to a specific block and
* offset within a block.
*/
#define XIVE_MAX_BLOCKS 16
#define XIVE_VSD_SIZE 8
/*
* Max number of ESBs. (direct table)
*
* The max number of ESBs supported in the P10 MMIO space is 1TB/128K: 8M.
*
* 1M is our current top limit of ESB entries and EAS entries
* pre-allocated per chip. That allocates 256KB per chip for the state
* bits and 8M per chip for the EAS.
*/
#define XIVE_INT_ORDER 20 /* 1M interrupts */
#define XIVE_INT_COUNT (1ul << XIVE_INT_ORDER)
/*
* First interrupt number, also the first logical interrupt number
* allocated by Linux (maximum ISA interrupt number + 1)
*/
#define XIVE_INT_FIRST 0x10
/* Corresponding direct table sizes */
#define XIVE_ESB_SIZE (XIVE_INT_COUNT / 4)
#define XIVE_EAT_SIZE (XIVE_INT_COUNT * 8)
/* Use 64K for everything by default */
#define XIVE_ESB_SHIFT (16 + 1) /* trigger + mgmt pages */
#define XIVE_ESB_PAGE_SIZE (1ul << XIVE_ESB_SHIFT) /* 2 pages */
/*
* Max number of ENDs. (indirect table)
*
* The max number of ENDs supported in the P10 MMIO space is 2TB/128K: 16M.
* Since one END is 32 bytes, a 64K indirect subpage can hold 2K ENDs.
* We need 8192 subpages, ie, 64K of memory for the indirect table.
*/
#define END_PER_PAGE (PAGE_SIZE / sizeof(struct xive_end))
#define XIVE_END_ORDER 23 /* 8M ENDs */
#define XIVE_END_COUNT (1ul << XIVE_END_ORDER)
#define XIVE_END_TABLE_SIZE ((XIVE_END_COUNT / END_PER_PAGE) * XIVE_VSD_SIZE)
#define XIVE_END_SHIFT (16 + 1) /* ESn + ESe pages */
/* One bit per number of priorities configured */
#define xive_end_bitmap_size(x) (XIVE_END_COUNT >> xive_cfg_vp_prio_shift(x))
/* Number of priorities (and thus ENDs) we allocate for each VP */
#define xive_cfg_vp_prio_shift(x) GETFIELD(CQ_XIVE_CFG_VP_INT_PRIO, (x)->config)
#define xive_cfg_vp_prio(x) (1 << xive_cfg_vp_prio_shift(x))
/* Max priority number */
#define xive_max_prio(x) (xive_cfg_vp_prio(x) - 1)
/* Priority used for gather/silent escalation (KVM) */
#define xive_escalation_prio(x) xive_max_prio(x)
/*
* Max number of VPs. (indirect table)
*
* The max number of NVPs we support in our MMIO space is 1TB/128K: 8M.
* Since one NVP is 32 bytes, a 64K indirect subpage can hold 2K NVPs.
* We need 4096 pointers, ie, 32K of memory for the indirect table.
*
* However, we use 8 priorities (by default) per NVP and the number of
* ENDs is configured to 8M. Therefore, our VP space is limited to 1M.
*/
#define VP_PER_PAGE (PAGE_SIZE / sizeof(struct xive_nvp))
#define XIVE_VP_ORDER(x) (XIVE_END_ORDER - xive_cfg_vp_prio_shift(x))
#define XIVE_VP_COUNT(x) (1ul << XIVE_VP_ORDER(x))
#define XIVE_VP_TABLE_SIZE(x) ((XIVE_VP_COUNT(x) / VP_PER_PAGE) * XIVE_VSD_SIZE)
#define XIVE_NVP_SHIFT 17 /* NVPG BAR: two pages, even NVP, odd NVG */
/* VP Space maximums in Gen1 and Gen2 modes */
#define VP_SHIFT_GEN1 19 /* in sync with END_W6_VP_OFFSET_GEN1 */
#define VP_SHIFT_GEN2 24 /* in sync with END_W6_VP_OFFSET */
/*
* VP ids for HW threads.
*
* Depends on the thread id bits configuration of the IC. 8bit is the
* default for P10 and 7bit for p9.
*
* These values are global because they should be common to all chips
*/
static uint32_t xive_threadid_shift;
static uint32_t xive_hw_vp_base;
static uint32_t xive_hw_vp_count;
/*
* The XIVE operation mode indicates the active "API" and corresponds
* to the "version/mode" parameter of the opal_xive_reset() call
*/
static enum {
/* No XICS emulation */
XIVE_MODE_EXPL = OPAL_XIVE_MODE_EXPL, /* default */
XIVE_MODE_NONE,
} xive_mode = XIVE_MODE_NONE;
/*
* The XIVE exploitation mode options indicates the active features and
* is part of the mode parameter of the opal_xive_reset() call
*/
static uint64_t xive_expl_options;
#define XIVE_EXPL_ALL_OPTIONS 0
/*
* Each source controller has one of these. There's one embedded in
* the XIVE struct for IPIs
*/
struct xive_src {
struct irq_source is;
const struct irq_source_ops *orig_ops;
struct xive *xive;
void *esb_mmio;
uint32_t esb_base;
uint32_t esb_shift;
uint32_t flags;
};
struct xive_cpu_state {
struct xive *xive;
void *tm_ring1;
/* Base HW VP and associated queues */
uint32_t vp_blk;
uint32_t vp_idx;
uint32_t end_blk;
uint32_t end_idx; /* Base end index of a block of 8 */
struct lock lock;
};
enum xive_generation {
XIVE_GEN1 = 1, /* P9 compat mode */
XIVE_GEN2 = 2, /* P10 default */
};
enum xive_quirks {
/* HW527671 - 8bits Hardwired Thread Id range not implemented */
XIVE_QUIRK_THREADID_7BITS = 0x00000001,
/* HW542974 - interrupt command priority checker not working properly */
XIVE_QUIRK_BROKEN_PRIO_CHECK = 0x00000002,
};
struct xive {
uint32_t chip_id;
uint32_t block_id;
struct dt_node *x_node;
enum xive_generation generation;
uint64_t capabilities;
uint64_t config;
uint64_t xscom_base;
/* MMIO regions */
void *ic_base;
uint64_t ic_size;
uint32_t ic_shift;
void *ic_tm_direct_base;
void *tm_base;
uint64_t tm_size;
uint32_t tm_shift;
void *nvp_base;
uint64_t nvp_size;
void *esb_base;
uint64_t esb_size;
void *end_base;
uint64_t end_size;
/* Set on XSCOM register access error */
bool last_reg_error;
/* Per-XIVE mutex */
struct lock lock;
/* Pre-allocated tables.
*
* We setup all the VDS for actual tables (ie, by opposition to
* forwarding ports) as either direct pre-allocated or indirect
* and partially populated.
*
* Currently, the ESB and the EAS tables are direct and fully
* pre-allocated based on XIVE_INT_COUNT.
*
* The other tables are indirect, we thus pre-allocate the indirect
* table (ie, pages of pointers) and populate enough of the pages
* for our basic setup using 64K subpages.
*
* The size of the indirect tables are driven by XIVE_VP_COUNT
* and XIVE_END_COUNT. The number of pre-allocated ones are
* driven by xive_hw_vp_count for the HW threads. The number
* of END depends on number of VP.
*/
/* Direct SBE and EAT tables */
void *sbe_base;
void *eat_base;
/* Indirect END table. NULL entries are unallocated, count is
* the numbre of pointers (ie, sub page placeholders).
*/
beint64_t *end_ind_base;
uint32_t end_ind_count;
uint64_t end_ind_size;
/* END allocation bitmap. Each bit represent #priority ENDs */
bitmap_t *end_map;
/* Indirect NVT/VP table. NULL entries are unallocated, count is
* the numbre of pointers (ie, sub page placeholders).
*/
beint64_t *vp_ind_base;
uint32_t vp_ind_count;
uint64_t vp_ind_size;
/* VP space size. Depends on Gen1/2 mode */
uint32_t vp_shift;
/* Pool of donated pages for provisioning indirect END and VP pages */
struct list_head donated_pages;
/* To ease a possible change to supporting more than one block of
* interrupts per chip, we store here the "base" global number
* and max number of interrupts for this chip. The global number
* encompass the block number and index.
*/
uint32_t int_base;
uint32_t int_count;
/* Due to the overlap between IPIs and HW sources in the EAS table,
* we keep some kind of top-down allocator. It is used for HW sources
* to "allocate" interrupt entries and will limit what can be handed
* out as IPIs. Of course this assumes we "allocate" all HW sources
* before we start handing out IPIs.
*
* Note: The numbers here are global interrupt numbers so that we can
* potentially handle more than one block per chip in the future.
*/
uint32_t int_hw_bot; /* Bottom of HW allocation */
uint32_t int_ipi_top; /* Highest IPI handed out so far + 1 */
/* The IPI allocation bitmap */
bitmap_t *ipi_alloc_map;
/* We keep track of which interrupts were ever enabled to
* speed up xive_reset
*/
bitmap_t *int_enabled_map;
/* Embedded source IPIs */
struct xive_src ipis;
/* Embedded escalation interrupts */
struct xive_src esc_irqs;
/* In memory queue overflow */
void *q_ovf;
/* Cache/sync injection */
uint64_t sync_inject_size;
void *sync_inject;
/* INT HW Errata */
uint64_t quirks;
};
/* First XIVE unit configured on the system */
static struct xive *one_xive;
/* Global DT node */
static struct dt_node *xive_dt_node;
/* Block <-> Chip conversions.
*
* As chipIDs may not be within the range of 16 block IDs supported by XIVE,
* we have a 2 way conversion scheme.
*
* From block to chip, use the global table below.
*
* From chip to block, a field in struct proc_chip contains the first block
* of that chip. For now we only support one block per chip but that might
* change in the future
*/
#define XIVE_INVALID_CHIP 0xffffffff
#define XIVE_MAX_CHIPS 16
static uint32_t xive_block_to_chip[XIVE_MAX_CHIPS];
static uint32_t xive_block_count;
static uint32_t xive_chip_to_block(uint32_t chip_id)
{
struct proc_chip *c = get_chip(chip_id);
assert(c);
assert(c->xive);
return c->xive->block_id;
}
/*
* Conversion between GIRQ and block/index.
*
* ------------------------------------
* |000E|BLOC| INDEX|
* ------------------------------------
* 4 4 24
*
* the E bit indicates that this is an escalation interrupt, in
* that case, the BLOC/INDEX represents the END containing the
* corresponding escalation descriptor.
*
* Global interrupt numbers for non-escalation interrupts are thus
* limited to 28 bits.
*/
#define INT_SHIFT 24
#define INT_ESC_SHIFT (INT_SHIFT + 4) /* 4bits block id */
#if XIVE_INT_ORDER > INT_SHIFT
#error "Too many ESBs for IRQ encoding"
#endif
#if XIVE_END_ORDER > INT_SHIFT
#error "Too many ENDs for escalation IRQ number encoding"
#endif
#define GIRQ_TO_BLK(__g) (((__g) >> INT_SHIFT) & 0xf)
#define GIRQ_TO_IDX(__g) ((__g) & ((1 << INT_SHIFT) - 1))
#define BLKIDX_TO_GIRQ(__b,__i) (((uint32_t)(__b)) << INT_SHIFT | (__i))
#define GIRQ_IS_ESCALATION(__g) ((__g) & (1 << INT_ESC_SHIFT))
#define MAKE_ESCALATION_GIRQ(__b,__i)(BLKIDX_TO_GIRQ(__b,__i) | (1 << INT_ESC_SHIFT))
/* Block/IRQ to chip# conversions */
#define PC_BLK_TO_CHIP(__b) (xive_block_to_chip[__b])
#define VC_BLK_TO_CHIP(__b) (xive_block_to_chip[__b])
#define GIRQ_TO_CHIP(__isn) (VC_BLK_TO_CHIP(GIRQ_TO_BLK(__isn)))
/* Routing of physical processors to VPs */
#define PIR2VP_IDX( __pir) (xive_hw_vp_base | P10_PIR2LOCALCPU(__pir))
#define PIR2VP_BLK(__pir) (xive_chip_to_block(P10_PIR2GCID(__pir)))
#define VP2PIR(__blk, __idx) (P10_PIRFROMLOCALCPU(VC_BLK_TO_CHIP(__blk), (__idx) & 0xff))
/* Decoding of OPAL API VP IDs. The VP IDs are encoded as follow
*
* Block group mode:
*
* -----------------------------------
* |GVEOOOOO| INDEX|
* -----------------------------------
* || |
* || Order
* |Virtual
* Group
*
* G (Group) : Set to 1 for a group VP (not currently supported)
* V (Virtual) : Set to 1 for an allocated VP (vs. a physical processor ID)
* E (Error) : Should never be 1, used internally for errors
* O (Order) : Allocation order of the VP block
*
* The conversion is thus done as follow (groups aren't implemented yet)
*
* If V=0, O must be 0 and 24-bit INDEX value is the PIR
* If V=1, the order O group is allocated such that if N is the number of
* chip bits considered for allocation (*)
* then the INDEX is constructed as follow (bit numbers such as 0=LSB)
* - bottom O-N bits is the index within the "VP block"
* - next N bits is the XIVE blockID of the VP
* - the remaining bits is the per-chip "base"
* so the conversion consists of "extracting" the block ID and moving
* down the upper bits by N bits.
*
* In non-block-group mode, the difference is that the blockID is
* on the left of the index (the entire VP block is in a single
* block ID)
*/
#define VP_GROUP_SHIFT 31
#define VP_VIRTUAL_SHIFT 30
#define VP_ERROR_SHIFT 29
#define VP_ORDER_SHIFT 24
#define vp_group(vp) (((vp) >> VP_GROUP_SHIFT) & 1)
#define vp_virtual(vp) (((vp) >> VP_VIRTUAL_SHIFT) & 1)
#define vp_order(vp) (((vp) >> VP_ORDER_SHIFT) & 0x1f)
#define vp_index(vp) ((vp) & ((1 << VP_ORDER_SHIFT) - 1))
/* VP allocation */
static uint32_t xive_chips_alloc_bits = 0;
static struct buddy *xive_vp_buddy;
static struct lock xive_buddy_lock = LOCK_UNLOCKED;
/* VP# decoding/encoding */
static bool xive_decode_vp(uint32_t vp, uint32_t *blk, uint32_t *idx,
uint8_t *order, bool *group)
{
uint32_t o = vp_order(vp);
uint32_t n = xive_chips_alloc_bits;
uint32_t index = vp_index(vp);
uint32_t imask = (1 << (o - n)) - 1;
/* Groups not supported yet */
if (vp_group(vp))
return false;
if (group)
*group = false;
/* PIR case */
if (!vp_virtual(vp)) {
if (find_cpu_by_pir(index) == NULL)
return false;
if (blk)
*blk = PIR2VP_BLK(index);
if (idx)
*idx = PIR2VP_IDX(index);
return true;
}
/* Ensure o > n, we have *at least* 2 VPs per block */
if (o <= n)
return false;
/* Combine the index base and index */
if (idx)
*idx = ((index >> n) & ~imask) | (index & imask);
/* Extract block ID */
if (blk)
*blk = (index >> (o - n)) & ((1 << n) - 1);
/* Return order as well if asked for */
if (order)
*order = o;
return true;
}
static uint32_t xive_encode_vp(uint32_t blk, uint32_t idx, uint32_t order)
{
uint32_t vp = (1 << VP_VIRTUAL_SHIFT) | (order << VP_ORDER_SHIFT);
uint32_t n = xive_chips_alloc_bits;
uint32_t imask = (1 << (order - n)) - 1;
vp |= (idx & ~imask) << n;
vp |= blk << (order - n);
vp |= idx & imask;
return vp;
}
/*
* XSCOM/MMIO helpers
*/
#define XIVE_NO_MMIO -1
#define xive_regw(__x, __r, __v) \
__xive_regw(__x, __r, X_##__r, __v, #__r)
#define xive_regr(__x, __r) \
__xive_regr(__x, __r, X_##__r, #__r)
#define xive_regwx(__x, __r, __v) \
__xive_regw(__x, XIVE_NO_MMIO, X_##__r, __v, #__r)
#define xive_regrx(__x, __r) \
__xive_regr(__x, XIVE_NO_MMIO, X_##__r, #__r)
#ifdef XIVE_VERBOSE_DEBUG
#define xive_vdbg(__x,__fmt,...) prlog(PR_DEBUG,"[ IC %02x ] " __fmt, (__x)->chip_id, ##__VA_ARGS__)
#define xive_cpu_vdbg(__c,__fmt,...) prlog(PR_DEBUG,"[CPU %04x] " __fmt, (__c)->pir, ##__VA_ARGS__)
#else
#define xive_vdbg(x,fmt,...) do { } while(0)
#define xive_cpu_vdbg(x,fmt,...) do { } while(0)
#endif
#define xive_dbg(__x,__fmt,...) prlog(PR_DEBUG,"[ IC %02x ] " __fmt, (__x)->chip_id, ##__VA_ARGS__)
#define xive_cpu_dbg(__c,__fmt,...) prlog(PR_DEBUG,"[CPU %04x] " __fmt, (__c)->pir, ##__VA_ARGS__)
#define xive_notice(__x,__fmt,...) prlog(PR_NOTICE,"[ IC %02x ] " __fmt, (__x)->chip_id, ##__VA_ARGS__)
#define xive_cpu_notice(__c,__fmt,...) prlog(PR_NOTICE,"[CPU %04x] " __fmt, (__c)->pir, ##__VA_ARGS__)
#define xive_warn(__x,__fmt,...) prlog(PR_WARNING,"[ IC %02x ] " __fmt, (__x)->chip_id, ##__VA_ARGS__)
#define xive_cpu_warn(__c,__fmt,...) prlog(PR_WARNING,"[CPU %04x] " __fmt, (__c)->pir, ##__VA_ARGS__)
#define xive_err(__x,__fmt,...) prlog(PR_ERR,"[ IC %02x ] " __fmt, (__x)->chip_id, ##__VA_ARGS__)
#define xive_cpu_err(__c,__fmt,...) prlog(PR_ERR,"[CPU %04x] " __fmt, (__c)->pir, ##__VA_ARGS__)
/*
* The XIVE subengine being accessed can be deduced from the XSCOM
* reg, and from there, the page offset in the IC BAR.
*/
static void* xive_ic_page(struct xive *x, uint32_t x_reg)
{
uint64_t pgoff = (x_reg >> 8) & 0x3;
return x->ic_base + (pgoff << x->ic_shift);
}
static void __xive_regw(struct xive *x, uint32_t m_reg, uint32_t x_reg, uint64_t v,
const char *rname)
{
bool use_xscom = (m_reg == XIVE_NO_MMIO) || !x->ic_base;
int64_t rc;
x->last_reg_error = false;
assert(x_reg != 0);
if (use_xscom) {
rc = xscom_write(x->chip_id, x->xscom_base + x_reg, v);
if (rc) {
if (!rname)
rname = "???";
xive_err(x, "Error writing register %s\n", rname);
/* Anything else we can do here ? */
x->last_reg_error = true;
}
} else {
out_be64(xive_ic_page(x, x_reg) + m_reg, v);
}
}
static uint64_t __xive_regr(struct xive *x, uint32_t m_reg, uint32_t x_reg,
const char *rname)
{
bool use_xscom = (m_reg == XIVE_NO_MMIO) || !x->ic_base;
int64_t rc;
uint64_t val;
x->last_reg_error = false;
assert(x_reg != 0);
if (use_xscom) {
rc = xscom_read(x->chip_id, x->xscom_base + x_reg, &val);
if (rc) {
if (!rname)
rname = "???";
xive_err(x, "Error reading register %s\n", rname);
/* Anything else we can do here ? */
x->last_reg_error = true;
return -1ull;
}
} else {
val = in_be64(xive_ic_page(x, x_reg) + m_reg);
}
return val;
}
/* Locate a controller from an IRQ number */
static struct xive *xive_from_isn(uint32_t isn)
{
uint32_t chip_id = GIRQ_TO_CHIP(isn);
struct proc_chip *c = get_chip(chip_id);
if (!c)
return NULL;
return c->xive;
}
static struct xive *xive_from_pc_blk(uint32_t blk)
{
uint32_t chip_id = PC_BLK_TO_CHIP(blk);
struct proc_chip *c = get_chip(chip_id);
if (!c)
return NULL;
return c->xive;
}
static struct xive *xive_from_vc_blk(uint32_t blk)
{
uint32_t chip_id = VC_BLK_TO_CHIP(blk);
struct proc_chip *c = get_chip(chip_id);
if (!c)
return NULL;
return c->xive;
}
static struct xive_end *xive_get_end(struct xive *x, unsigned int idx)
{
struct xive_end *p;
if (idx >= (x->end_ind_count * END_PER_PAGE))
return NULL;
p = (struct xive_end *)(be64_to_cpu(x->end_ind_base[idx / END_PER_PAGE]) &
VSD_ADDRESS_MASK);
if (!p)
return NULL;
return &p[idx % END_PER_PAGE];
}
static struct xive_eas *xive_get_eas(struct xive *x, unsigned int isn)
{
struct xive_eas *eat;
uint32_t idx = GIRQ_TO_IDX(isn);
if (GIRQ_IS_ESCALATION(isn)) {
/* Allright, an escalation EAS is buried inside an END, let's
* try to find it
*/
struct xive_end *end;
if (x->chip_id != VC_BLK_TO_CHIP(GIRQ_TO_BLK(isn))) {
xive_err(x, "%s, ESC ISN 0x%x not on right chip\n",
__func__, isn);
return NULL;
}
end = xive_get_end(x, idx);
if (!end) {
xive_err(x, "%s, ESC ISN 0x%x END not found\n",
__func__, isn);
return NULL;
}
/* If using single-escalation, don't let anybody get
* to the individual escalation interrupts
*/
if (xive_get_field32(END_W0_UNCOND_ESCALATE, end->w0))
return NULL;
/* Grab the escalation END */
return (struct xive_eas *)(char *)&end->w4;
} else {
/* Check the block matches */
if (isn < x->int_base || isn >= x->int_count) {
xive_err(x, "%s, ISN 0x%x not on right chip\n",
__func__, isn);
return NULL;
}
assert (idx < XIVE_INT_COUNT);
/* If we support >1 block per chip, this should still
* work as we are likely to make the table contiguous
* anyway
*/
eat = x->eat_base;
assert(eat);
return eat + idx;
}
}
static struct xive_nvp *xive_get_vp(struct xive *x, unsigned int idx)
{
struct xive_nvp *p;
assert(idx < (x->vp_ind_count * VP_PER_PAGE));
p = (struct xive_nvp *)(be64_to_cpu(x->vp_ind_base[idx / VP_PER_PAGE]) &
VSD_ADDRESS_MASK);
if (!p)
return NULL;
return &p[idx % VP_PER_PAGE];
}
/*
* Store the END base of the VP in W5, using the new architected field
* in P10. Used to be the pressure relief interrupt field on P9.
*/
static void xive_vp_set_end_base(struct xive_nvp *vp,
uint32_t end_blk, uint32_t end_idx)
{
vp->w5 = xive_set_field32(NVP_W5_VP_END_BLOCK, 0, end_blk) |
xive_set_field32(NVP_W5_VP_END_INDEX, 0, end_idx);
/* This is the criteria to know if a VP was allocated */
assert(vp->w5 != 0);
}
static void xive_init_default_vp(struct xive_nvp *vp,
uint32_t end_blk, uint32_t end_idx)
{
memset(vp, 0, sizeof(struct xive_nvp));
xive_vp_set_end_base(vp, end_blk, end_idx);
vp->w0 = xive_set_field32(NVP_W0_VALID, 0, 1);
}
/*
* VPs of the HW threads have their own set of ENDs which is allocated
* when XIVE is initialized. These are tagged with a FIRMWARE bit so
* that they can be identified when the driver is reset (kexec).
*/
static void xive_init_hw_end(struct xive_end *end)
{
memset(end, 0, sizeof(struct xive_end));
end->w0 = xive_set_field32(END_W0_FIRMWARE1, 0, 1);
}
static void *xive_get_donated_page(struct xive *x)
{
return (void *)list_pop_(&x->donated_pages, 0);
}
#define XIVE_ALLOC_IS_ERR(_idx) ((_idx) >= 0xfffffff0)
#define XIVE_ALLOC_NO_SPACE 0xffffffff /* No possible space */
#define XIVE_ALLOC_NO_IND 0xfffffffe /* Indirect need provisioning */
#define XIVE_ALLOC_NO_MEM 0xfffffffd /* Local allocation failed */
static uint32_t xive_alloc_end_set(struct xive *x, bool alloc_indirect)
{
uint32_t ind_idx;
int idx;
int end_base_idx;
xive_vdbg(x, "Allocating END set...\n");
assert(x->end_map);
/* Allocate from the END bitmap. Each bit is 8 ENDs */
idx = bitmap_find_zero_bit(*x->end_map, 0, xive_end_bitmap_size(x));
if (idx < 0) {
xive_dbg(x, "Allocation from END bitmap failed !\n");
return XIVE_ALLOC_NO_SPACE;
}
end_base_idx = idx << xive_cfg_vp_prio_shift(x);
xive_vdbg(x, "Got ENDs 0x%x..0x%x\n", end_base_idx,
end_base_idx + xive_max_prio(x));
/* Calculate the indirect page where the ENDs reside */
ind_idx = end_base_idx / END_PER_PAGE;
/* Is there an indirect page ? If not, check if we can provision it */
if (!x->end_ind_base[ind_idx]) {
/* Default flags */
uint64_t vsd_flags = SETFIELD(VSD_TSIZE, 0ull, 4) |
SETFIELD(VSD_MODE, 0ull, VSD_MODE_EXCLUSIVE);
void *page;
/* If alloc_indirect is set, allocate the memory from OPAL own,
* otherwise try to provision from the donated pool
*/
if (alloc_indirect) {
/* Allocate/provision indirect page during boot only */
xive_vdbg(x, "Indirect empty, provisioning from local pool\n");
page = local_alloc(x->chip_id, PAGE_SIZE, PAGE_SIZE);
if (!page) {
xive_dbg(x, "provisioning failed !\n");
return XIVE_ALLOC_NO_MEM;
}
vsd_flags |= VSD_FIRMWARE;
} else {
xive_vdbg(x, "Indirect empty, provisioning from donated pages\n");
page = xive_get_donated_page(x);
if (!page) {
xive_vdbg(x, "no idirect pages available !\n");
return XIVE_ALLOC_NO_IND;
}
}
memset(page, 0, PAGE_SIZE);
x->end_ind_base[ind_idx] = cpu_to_be64(vsd_flags |
(((uint64_t)page) & VSD_ADDRESS_MASK));
/* Any cache scrub needed ? */
}
bitmap_set_bit(*x->end_map, idx);
return end_base_idx;
}
static void xive_free_end_set(struct xive *x, uint32_t ends)
{
uint32_t idx;
uint8_t prio_mask = xive_max_prio(x);
xive_vdbg(x, "Freeing END 0x%x..0x%x\n", ends, ends + xive_max_prio(x));
assert((ends & prio_mask) == 0);
assert(x->end_map);
idx = ends >> xive_cfg_vp_prio_shift(x);
bitmap_clr_bit(*x->end_map, idx);
}
static bool xive_provision_vp_ind(struct xive *x, uint32_t vp_idx, uint32_t order)
{
uint32_t pbase, pend, i;
pbase = vp_idx / VP_PER_PAGE;
pend = (vp_idx + (1 << order)) / VP_PER_PAGE;
for (i = pbase; i <= pend; i++) {
void *page;
u64 vsd;
/* Already provisioned ? */
if (x->vp_ind_base[i])
continue;
/* Try to grab a donated page */
page = xive_get_donated_page(x);
if (!page)
return false;
/* Install the page */
memset(page, 0, PAGE_SIZE);
vsd = ((uint64_t)page) & VSD_ADDRESS_MASK;
vsd |= SETFIELD(VSD_TSIZE, 0ull, 4);
vsd |= SETFIELD(VSD_MODE, 0ull, VSD_MODE_EXCLUSIVE);
x->vp_ind_base[i] = cpu_to_be64(vsd);
}
return true;
}
static void xive_init_vp_allocator(void)
{
/* Initialize chip alloc bits */
xive_chips_alloc_bits = ilog2(xive_block_count);
prlog(PR_INFO, "%d chips considered for VP allocations\n",
1 << xive_chips_alloc_bits);
/* Allocate a buddy big enough for XIVE_VP_ORDER allocations.
*
* each bit in the buddy represents 1 << xive_chips_alloc_bits
* VPs.
*/
xive_vp_buddy = buddy_create(XIVE_VP_ORDER(one_xive));
assert(xive_vp_buddy);
/*
* We reserve the whole range of VP ids representing HW threads.
*/
assert(buddy_reserve(xive_vp_buddy, xive_hw_vp_base,
xive_threadid_shift));
}
static uint32_t xive_alloc_vps(uint32_t order)
{
uint32_t local_order, i;
int vp;
/* The minimum order is 2 VPs per chip */
if (order < (xive_chips_alloc_bits + 1))
order = xive_chips_alloc_bits + 1;
/* We split the allocation */
local_order = order - xive_chips_alloc_bits;
/* We grab that in the global buddy */
assert(xive_vp_buddy);
lock(&xive_buddy_lock);
vp = buddy_alloc(xive_vp_buddy, local_order);
unlock(&xive_buddy_lock);
if (vp < 0)
return XIVE_ALLOC_NO_SPACE;
/* Provision on every chip considered for allocation */
for (i = 0; i < (1 << xive_chips_alloc_bits); i++) {
struct xive *x = xive_from_pc_blk(i);
bool success;
/* Return internal error & log rather than assert ? */
assert(x);
lock(&x->lock);
success = xive_provision_vp_ind(x, vp, local_order);
unlock(&x->lock);
if (!success) {
lock(&xive_buddy_lock);
buddy_free(xive_vp_buddy, vp, local_order);
unlock(&xive_buddy_lock);
return XIVE_ALLOC_NO_IND;
}
}
/* Encode the VP number. "blk" is 0 as this represents
* all blocks and the allocation always starts at 0
*/
return xive_encode_vp(0, vp, order);
}
static void xive_free_vps(uint32_t vp)
{
uint32_t idx;
uint8_t order, local_order;
assert(xive_decode_vp(vp, NULL, &idx, &order, NULL));
/* We split the allocation */
local_order = order - xive_chips_alloc_bits;
/* Free that in the buddy */
lock(&xive_buddy_lock);
buddy_free(xive_vp_buddy, idx, local_order);
unlock(&xive_buddy_lock);
}
enum xive_cache_type {
xive_cache_easc,
xive_cache_esbc,
xive_cache_endc,
xive_cache_nxc,
};
/*
* Cache update
*/
#define FLUSH_CTRL_POLL_VALID PPC_BIT(0) /* POLL bit is the same for all */
static int64_t __xive_cache_scrub(struct xive *x,
enum xive_cache_type ctype,
uint64_t block, uint64_t idx,
bool want_inval __unused, bool want_disable __unused)
{
uint64_t ctrl_reg, x_ctrl_reg;
uint64_t poll_val, ctrl_val;
#ifdef XIVE_CHECK_LOCKS
assert(lock_held_by_me(&x->lock));
#endif
switch (ctype) {
case xive_cache_easc:
poll_val =
SETFIELD(VC_EASC_FLUSH_POLL_BLOCK_ID, 0ll, block) |
SETFIELD(VC_EASC_FLUSH_POLL_OFFSET, 0ll, idx) |
VC_EASC_FLUSH_POLL_BLOCK_ID_MASK |
VC_EASC_FLUSH_POLL_OFFSET_MASK;
xive_regw(x, VC_EASC_FLUSH_POLL, poll_val);
ctrl_reg = VC_EASC_FLUSH_CTRL;
x_ctrl_reg = X_VC_EASC_FLUSH_CTRL;
break;
case xive_cache_esbc:
poll_val =
SETFIELD(VC_ESBC_FLUSH_POLL_BLOCK_ID, 0ll, block) |
SETFIELD(VC_ESBC_FLUSH_POLL_OFFSET, 0ll, idx) |
VC_ESBC_FLUSH_POLL_BLOCK_ID_MASK |
VC_ESBC_FLUSH_POLL_OFFSET_MASK;
xive_regw(x, VC_ESBC_FLUSH_POLL, poll_val);
ctrl_reg = VC_ESBC_FLUSH_CTRL;
x_ctrl_reg = X_VC_ESBC_FLUSH_CTRL;
break;
case xive_cache_endc:
poll_val =
SETFIELD(VC_ENDC_FLUSH_POLL_BLOCK_ID, 0ll, block) |
SETFIELD(VC_ENDC_FLUSH_POLL_OFFSET, 0ll, idx) |
VC_ENDC_FLUSH_POLL_BLOCK_ID_MASK |
VC_ENDC_FLUSH_POLL_OFFSET_MASK;
xive_regw(x, VC_ENDC_FLUSH_POLL, poll_val);
ctrl_reg = VC_ENDC_FLUSH_CTRL;
x_ctrl_reg = X_VC_ENDC_FLUSH_CTRL;
break;
case xive_cache_nxc:
poll_val =
SETFIELD(PC_NXC_FLUSH_POLL_BLOCK_ID, 0ll, block) |
SETFIELD(PC_NXC_FLUSH_POLL_OFFSET, 0ll, idx) |
PC_NXC_FLUSH_POLL_BLOCK_ID_MASK |
PC_NXC_FLUSH_POLL_OFFSET_MASK;
xive_regw(x, PC_NXC_FLUSH_POLL, poll_val);
ctrl_reg = PC_NXC_FLUSH_CTRL;
x_ctrl_reg = X_PC_NXC_FLUSH_CTRL;
break;
default:
return OPAL_INTERNAL_ERROR;
}
/* XXX Add timeout !!! */
for (;;) {
ctrl_val = __xive_regr(x, ctrl_reg, x_ctrl_reg, NULL);
if (!(ctrl_val & FLUSH_CTRL_POLL_VALID))
break;
/* Small delay */
time_wait(100);
}
sync();
return 0;
}
static int64_t xive_easc_scrub(struct xive *x, uint64_t block, uint64_t idx)
{
return __xive_cache_scrub(x, xive_cache_easc, block, idx, false, false);
}
static int64_t xive_nxc_scrub(struct xive *x, uint64_t block, uint64_t idx)
{
return __xive_cache_scrub(x, xive_cache_nxc, block, idx, false, false);
}
static int64_t xive_nxc_scrub_clean(struct xive *x, uint64_t block, uint64_t idx)
{
return __xive_cache_scrub(x, xive_cache_nxc, block, idx, true, false);
}
static int64_t xive_endc_scrub(struct xive *x, uint64_t block, uint64_t idx)
{
return __xive_cache_scrub(x, xive_cache_endc, block, idx, false, false);
}
#define XIVE_CACHE_WATCH_MAX_RETRIES 10
static int64_t __xive_cache_watch(struct xive *x, enum xive_cache_type ctype,
uint64_t block, uint64_t idx,
uint32_t start_dword, uint32_t dword_count,
beint64_t *new_data, bool light_watch,
bool synchronous)
{
uint64_t sreg, sregx, dreg0, dreg0x;
uint64_t dval0, sval, status;
int64_t i;
int retries = 0;
#ifdef XIVE_CHECK_LOCKS
assert(lock_held_by_me(&x->lock));
#endif
switch (ctype) {
case xive_cache_endc:
sreg = VC_ENDC_WATCH0_SPEC;
sregx = X_VC_ENDC_WATCH0_SPEC;
dreg0 = VC_ENDC_WATCH0_DATA0;
dreg0x = X_VC_ENDC_WATCH0_DATA0;
sval = SETFIELD(VC_ENDC_WATCH_BLOCK_ID, idx, block);
break;
case xive_cache_nxc:
sreg = PC_NXC_WATCH0_SPEC;
sregx = X_PC_NXC_WATCH0_SPEC;
dreg0 = PC_NXC_WATCH0_DATA0;
dreg0x = X_PC_NXC_WATCH0_DATA0;
sval = SETFIELD(PC_NXC_WATCH_BLOCK_ID, idx, block);
break;
default:
return OPAL_INTERNAL_ERROR;
}
/* The full bit is in the same position for ENDC and NXC */
if (!light_watch)
sval |= VC_ENDC_WATCH_FULL;
for (;;) {
/* Write the cache watch spec */
__xive_regw(x, sreg, sregx, sval, NULL);
/* Load data0 register to populate the watch */
dval0 = __xive_regr(x, dreg0, dreg0x, NULL);
/* If new_data is NULL, this is a dummy watch used as a
* workaround for a HW bug
*/
if (!new_data) {
__xive_regw(x, dreg0, dreg0x, dval0, NULL);
return 0;
}
/* Write the words into the watch facility. We write in reverse
* order in case word 0 is part of it as it must be the last
* one written.
*/
for (i = start_dword + dword_count - 1; i >= start_dword ;i--) {
uint64_t dw = be64_to_cpu(new_data[i - start_dword]);
__xive_regw(x, dreg0 + i * 8, dreg0x + i, dw, NULL);
}
/* Write data0 register to trigger the update if word 0 wasn't
* written above
*/
if (start_dword > 0)
__xive_regw(x, dreg0, dreg0x, dval0, NULL);
/* This may not be necessary for light updates (it's possible
* that a sync in sufficient, TBD). Ensure the above is
* complete and check the status of the watch.
*/
status = __xive_regr(x, sreg, sregx, NULL);
/* Bits FULL and CONFLICT are in the same position in
* ENDC and NXC
*/
if (!(status & VC_ENDC_WATCH_FULL) ||
!(status & VC_ENDC_WATCH_CONFLICT))
break;
if (!synchronous)
return OPAL_BUSY;
if (++retries == XIVE_CACHE_WATCH_MAX_RETRIES) {
xive_err(x, "Reached maximum retries %d when doing "
"a %s cache update\n", retries,
ctype == xive_cache_endc ? "ENDC" : "NXC");
return OPAL_BUSY;
}
}
/* Perform a scrub with "want_invalidate" set to false to push the
* cache updates to memory as well
*/
return __xive_cache_scrub(x, ctype, block, idx, false, false);
}
#ifdef XIVE_DEBUG_INIT_CACHE_UPDATES
static bool xive_check_endc_update(struct xive *x, uint32_t idx, struct xive_end *end)
{
struct xive_end *end_p = xive_get_end(x, idx);
struct xive_end end2;
assert(end_p);
end2 = *end_p;
if (memcmp(end, &end2, sizeof(struct xive_end)) != 0) {
xive_err(x, "END update mismatch idx %d\n", idx);
xive_err(x, "want: %08x %08x %08x %08x\n",
end->w0, end->w1, end->w2, end->w3);
xive_err(x, " %08x %08x %08x %08x\n",
end->w4, end->w5, end->w6, end->w7);
xive_err(x, "got : %08x %08x %08x %08x\n",
end2.w0, end2.w1, end2.w2, end2.w3);
xive_err(x, " %08x %08x %08x %08x\n",
end2.w4, end2.w5, end2.w6, end2.w7);
return false;
}
return true;
}
static bool xive_check_nxc_update(struct xive *x, uint32_t idx, struct xive_nvp *vp)
{
struct xive_nvp *vp_p = xive_get_vp(x, idx);
struct xive_nvp vp2;
assert(vp_p);
vp2 = *vp_p;
if (memcmp(vp, &vp2, sizeof(struct xive_nvp)) != 0) {
xive_err(x, "VP update mismatch idx %d\n", idx);
xive_err(x, "want: %08x %08x %08x %08x\n",
vp->w0, vp->w1, vp->w2, vp->w3);
xive_err(x, " %08x %08x %08x %08x\n",
vp->w4, vp->w5, vp->w6, vp->w7);
xive_err(x, "got : %08x %08x %08x %08x\n",
vp2.w0, vp2.w1, vp2.w2, vp2.w3);
xive_err(x, " %08x %08x %08x %08x\n",
vp2.w4, vp2.w5, vp2.w6, vp2.w7);
return false;
}
return true;
}
#else
static inline bool xive_check_endc_update(struct xive *x __unused,
uint32_t idx __unused,
struct xive_end *end __unused)
{
return true;
}
static inline bool xive_check_nxc_update(struct xive *x __unused,
uint32_t idx __unused,
struct xive_nvp *vp __unused)
{
return true;
}
#endif
static int64_t xive_escalation_ive_cache_update(struct xive *x, uint64_t block,
uint64_t idx, struct xive_eas *eas,
bool synchronous)
{
return __xive_cache_watch(x, xive_cache_endc, block, idx,
2, 1, &eas->w, true, synchronous);
}
static int64_t xive_endc_cache_update(struct xive *x, uint64_t block,
uint64_t idx, struct xive_end *end,
bool synchronous)
{
int64_t ret;
ret = __xive_cache_watch(x, xive_cache_endc, block, idx,
0, 4, (beint64_t *)end, false, synchronous);
xive_check_endc_update(x, idx, end);
return ret;
}
static int64_t xive_nxc_cache_update(struct xive *x, uint64_t block,
uint64_t idx, struct xive_nvp *vp,
bool synchronous)
{
int64_t ret;
ret = __xive_cache_watch(x, xive_cache_nxc, block, idx,
0, 4, (beint64_t *)vp, false, synchronous);
xive_check_nxc_update(x, idx, vp);
return ret;
}
/*
* VSD
*/
static bool xive_set_vsd(struct xive *x, uint32_t tbl, uint32_t idx, uint64_t v)
{
/* Set VC subengine */
xive_regw(x, VC_VSD_TABLE_ADDR,
SETFIELD(VC_VSD_TABLE_SELECT, 0ull, tbl) |
SETFIELD(VC_VSD_TABLE_ADDRESS, 0ull, idx));
if (x->last_reg_error)
return false;
xive_regw(x, VC_VSD_TABLE_DATA, v);
if (x->last_reg_error)
return false;
/* also set PC subengine if table is used */
if (tbl == VST_EAS || tbl == VST_ERQ || tbl == VST_IC)
return true;
xive_regw(x, PC_VSD_TABLE_ADDR,
SETFIELD(PC_VSD_TABLE_SELECT, 0ull, tbl) |
SETFIELD(PC_VSD_TABLE_ADDRESS, 0ull, idx));
if (x->last_reg_error)
return false;
xive_regw(x, PC_VSD_TABLE_DATA, v);
if (x->last_reg_error)
return false;
return true;
}
static bool xive_set_local_tables(struct xive *x)
{
uint64_t base, i;
/* These have to be power of 2 sized */
assert(is_pow2(XIVE_ESB_SIZE));
assert(is_pow2(XIVE_EAT_SIZE));
/* All tables set as exclusive */
base = SETFIELD(VSD_MODE, 0ull, VSD_MODE_EXCLUSIVE);
/* ESB: direct mode */
if (!xive_set_vsd(x, VST_ESB, x->block_id, base |
(((uint64_t)x->sbe_base) & VSD_ADDRESS_MASK) |
SETFIELD(VSD_TSIZE, 0ull, ilog2(XIVE_ESB_SIZE) - 12)))
return false;
/* EAS: direct mode */
if (!xive_set_vsd(x, VST_EAS, x->block_id, base |
(((uint64_t)x->eat_base) & VSD_ADDRESS_MASK) |
SETFIELD(VSD_TSIZE, 0ull, ilog2(XIVE_EAT_SIZE) - 12)))
return false;
/* END: indirect mode with 64K subpages */
if (!xive_set_vsd(x, VST_END, x->block_id, base |
(((uint64_t)x->end_ind_base) & VSD_ADDRESS_MASK) |
VSD_INDIRECT | SETFIELD(VSD_TSIZE, 0ull,
ilog2(x->end_ind_size) - 12)))
return false;
/* NVP: indirect mode with 64K subpages */
if (!xive_set_vsd(x, VST_NVP, x->block_id, base |
(((uint64_t)x->vp_ind_base) & VSD_ADDRESS_MASK) |
VSD_INDIRECT | SETFIELD(VSD_TSIZE, 0ull,
ilog2(x->vp_ind_size) - 12)))
return false;
/* NVG: not used */
/* NVC: not used */
/* INT and SYNC: indexed with the Topology# */
if (!xive_set_vsd(x, VST_IC, x->chip_id, base |
(((uint64_t)x->ic_base) & VSD_ADDRESS_MASK) |
SETFIELD(VSD_TSIZE, 0ull, ilog2(x->ic_size) - 12)))
return false;
if (!xive_set_vsd(x, VST_SYNC, x->chip_id, base |
(((uint64_t)x->sync_inject) & VSD_ADDRESS_MASK) |
SETFIELD(VSD_TSIZE, 0ull, ilog2(x->sync_inject_size) - 12)))
return false;
/*
* ERQ: one 64K page for each queue overflow. Indexed with :
*
* 0:IPI, 1:HWD, 2:NxC, 3:INT, 4:OS-Queue, 5:Pool-Queue, 6:Hard-Queue
*/
for (i = 0; i < VC_QUEUE_COUNT; i++) {
u64 addr = ((uint64_t)x->q_ovf) + i * PAGE_SIZE;
u64 cfg, sreg, sregx;
if (!xive_set_vsd(x, VST_ERQ, i, base |
(addr & VSD_ADDRESS_MASK) |
SETFIELD(VSD_TSIZE, 0ull, 4)))
return false;
sreg = VC_QUEUES_CFG_REM0 + i * 8;
sregx = X_VC_QUEUES_CFG_REM0 + i;
cfg = __xive_regr(x, sreg, sregx, NULL);
cfg |= VC_QUEUES_CFG_MEMB_EN;
cfg = SETFIELD(VC_QUEUES_CFG_MEMB_SZ, cfg, 4);
__xive_regw(x, sreg, sregx, cfg, NULL);
}
return true;
}
/*
* IC BAR layout
*
* Page 0: Internal CQ register accesses (reads & writes)
* Page 1: Internal PC register accesses (reads & writes)
* Page 2: Internal VC register accesses (reads & writes)
* Page 3: Internal TCTXT (TIMA) reg accesses (read & writes)
* Page 4: Notify Port page (writes only, w/data),
* Page 5: Reserved
* Page 6: Sync Poll page (writes only, dataless)
* Page 7: Sync Inject page (writes only, dataless)
* Page 8: LSI Trigger page (writes only, dataless)
* Page 9: LSI SB Management page (reads & writes dataless)
* Pages 10-255: Reserved
* Pages 256-383: Direct mapped Thread Context Area (reads & writes)
* covering the 128 threads in P10.
* Pages 384-511: Reserved
*/
#define XIVE_IC_CQ_PGOFF 0
#define XIVE_IC_PC_PGOFF 1
#define XIVE_IC_VC_PGOFF 2
#define XIVE_IC_TCTXT_PGOFF 3
#define XIVE_NOTIFY_PGOFF 4
#define XIVE_SYNC_POLL_PGOFF 6
#define XIVE_SYNC_INJECT_PGOFF 7
#define XIVE_LSI_TRIGGER_PGOFF 8
#define XIVE_LSI_MGMT_PGOFF 9
#define XIVE_IC_TM_DIRECT_PGOFF 256
static bool xive_configure_ic_bars(struct xive *x)
{
uint64_t chip_id = x->chip_id;
uint64_t val;
/* Reset all bars to zero */
xive_regwx(x, CQ_RST_CTL, CQ_RST_PB_BAR_RESET);
/* IC BAR */
phys_map_get(chip_id, XIVE_IC, 0, (uint64_t *)&x->ic_base, &x->ic_size);
val = (uint64_t)x->ic_base | CQ_IC_BAR_VALID | CQ_IC_BAR_64K;
x->ic_shift = 16;
xive_regwx(x, CQ_IC_BAR, val);
if (x->last_reg_error)
return false;
/*
* TM BAR, same address for each chip. Hence we create a fake
* chip 0 and use that for all phys_map_get(XIVE_TM) calls.
*/
phys_map_get(0, XIVE_TM, 0, (uint64_t *)&x->tm_base, &x->tm_size);
val = (uint64_t)x->tm_base | CQ_TM_BAR_VALID | CQ_TM_BAR_64K;
x->tm_shift = 16;
xive_regwx(x, CQ_TM_BAR, val);
if (x->last_reg_error)
return false;
/* IC BAR sub-pages shortcuts */
x->ic_tm_direct_base = x->ic_base +
(XIVE_IC_TM_DIRECT_PGOFF << x->ic_shift);
return true;
}
/*
* NVPG, NVC, ESB, END BARs have common attributes: 64k page and only
* one set covering the whole BAR.
*/
static bool xive_configure_bars(struct xive *x)
{
uint64_t chip_id = x->chip_id;
uint64_t val;
uint64_t esb_size;
uint64_t end_size;
uint64_t nvp_size;
x->nvp_size = XIVE_VP_COUNT(x) << XIVE_NVP_SHIFT;
x->esb_size = XIVE_INT_COUNT << XIVE_ESB_SHIFT;
x->end_size = XIVE_END_COUNT << XIVE_END_SHIFT;
/*
* NVC BAR is not configured because we do not use the XIVE2
* Crowd capability.
*/
/* NVPG BAR: two pages, even NVP, odd NVG */
phys_map_get(chip_id, XIVE_NVPG, 0, (uint64_t *)&x->nvp_base, &nvp_size);
if (x->nvp_size > nvp_size) {
xive_err(x, "NVP table is larger than default: "
"0x%012llx > 0x%012llx\n", x->nvp_size, nvp_size);
return false;
}
val = (uint64_t)x->nvp_base | CQ_BAR_VALID | CQ_BAR_64K |
SETFIELD(CQ_BAR_RANGE, 0ull, ilog2(x->nvp_size) - 24);
xive_regwx(x, CQ_NVPG_BAR, val);
if (x->last_reg_error)
return false;
/* ESB BAR */
phys_map_get(chip_id, XIVE_ESB, 0, (uint64_t *)&x->esb_base, &esb_size);
if (x->esb_size > esb_size) {
xive_err(x, "ESB table is larger than default: "
"0x%012llx > 0x%012llx\n", x->esb_size, esb_size);
return false;
}
val = (uint64_t)x->esb_base | CQ_BAR_VALID | CQ_BAR_64K |
SETFIELD(CQ_BAR_RANGE, 0ull, ilog2(x->esb_size) - 24);
xive_regwx(x, CQ_ESB_BAR, val);
if (x->last_reg_error)
return false;
/* END BAR */
phys_map_get(chip_id, XIVE_END, 0, (uint64_t *)&x->end_base, &end_size);
if (x->end_size > end_size) {
xive_err(x, "END table is larger than default: "
"0x%012llx > 0x%012llx\n", x->end_size, end_size);
return false;
}
val = (uint64_t)x->end_base | CQ_BAR_VALID | CQ_BAR_64K |
SETFIELD(CQ_BAR_RANGE, 0ull, ilog2(x->end_size) - 24);
xive_regwx(x, CQ_END_BAR, val);
if (x->last_reg_error)
return false;
xive_dbg(x, "IC: %14p [0x%012llx]\n", x->ic_base, x->ic_size);
xive_dbg(x, "TM: %14p [0x%012llx]\n", x->tm_base, x->tm_size);
xive_dbg(x, "NVP: %14p [0x%012llx]\n", x->nvp_base, x->nvp_size);
xive_dbg(x, "ESB: %14p [0x%012llx]\n", x->esb_base, x->esb_size);
xive_dbg(x, "END: %14p [0x%012llx]\n", x->end_base, x->end_size);
xive_dbg(x, "OVF: %14p [0x%012x]\n", x->q_ovf,
VC_QUEUE_COUNT * PAGE_SIZE);
return true;
}
static void xive_dump_mmio(struct xive *x)
{
prlog(PR_DEBUG, " CQ_CFG_PB_GEN = %016llx\n",
in_be64(x->ic_base + CQ_CFG_PB_GEN));
prlog(PR_DEBUG, " CQ_MSGSND = %016llx\n",
in_be64(x->ic_base + CQ_MSGSND));
}
static const struct {
uint64_t bitmask;
const char *name;
} xive_capabilities[] = {
{ CQ_XIVE_CAP_PHB_PQ_DISABLE, "PHB PQ disable mode support" },
{ CQ_XIVE_CAP_PHB_ABT, "PHB address based trigger mode support" },
{ CQ_XIVE_CAP_EXPLOITATION_MODE, "Exploitation mode" },
{ CQ_XIVE_CAP_STORE_EOI, "StoreEOI mode support" },
{ CQ_XIVE_CAP_VP_SAVE_RESTORE, "VP Context Save and Restore" },
};
static void xive_dump_capabilities(struct xive *x, uint64_t cap_val)
{
int i;
xive_dbg(x, "capabilities: %016llx\n", cap_val);
xive_dbg(x, "\tVersion: %lld\n",
GETFIELD(CQ_XIVE_CAP_VERSION, cap_val));
xive_dbg(x, "\tUser interrupt priorities: [ 1 - %d ]\n",
1 << GETFIELD(CQ_XIVE_CAP_USER_INT_PRIO, cap_val));
xive_dbg(x, "\tVP interrupt priorities: [ %d - 8 ]\n",
1 << GETFIELD(CQ_XIVE_CAP_VP_INT_PRIO, cap_val));
xive_dbg(x, "\tExtended Blockid bits: %lld\n",
4 + GETFIELD(CQ_XIVE_CAP_BLOCK_ID_WIDTH, cap_val));
for (i = 0; i < ARRAY_SIZE(xive_capabilities); i++) {
if (xive_capabilities[i].bitmask & cap_val)
xive_dbg(x, "\t%s\n", xive_capabilities[i].name);
}
}
static const struct {
uint64_t bitmask;
const char *name;
} xive_configs[] = {
{ CQ_XIVE_CFG_GEN1_TIMA_OS, "Gen1 mode TIMA OS" },
{ CQ_XIVE_CFG_GEN1_TIMA_HYP, "Gen1 mode TIMA Hyp" },
{ CQ_XIVE_CFG_GEN1_TIMA_HYP_BLK0, "Gen1 mode TIMA General Hypervisor Block0" },
{ CQ_XIVE_CFG_GEN1_TIMA_CROWD_DIS, "Gen1 mode TIMA Crowd disable" },
{ CQ_XIVE_CFG_GEN1_END_ESX, "Gen1 mode END ESx" },
{ CQ_XIVE_CFG_EN_VP_SAVE_RESTORE, "VP Context Save and Restore" },
{ CQ_XIVE_CFG_EN_VP_SAVE_REST_STRICT, "VP Context Save and Restore strict" },
};
static void xive_dump_configuration(struct xive *x, const char *prefix,
uint64_t cfg_val)
{
int i ;
xive_dbg(x, "%s configuration: %016llx\n", prefix, cfg_val);
xive_dbg(x, "\tHardwired Thread Id range: %lld bits\n",
7 + GETFIELD(CQ_XIVE_CFG_HYP_HARD_RANGE, cfg_val));
xive_dbg(x, "\tUser Interrupt priorities: [ 1 - %d ]\n",
1 << GETFIELD(CQ_XIVE_CFG_USER_INT_PRIO, cfg_val));
xive_dbg(x, "\tVP Interrupt priorities: [ 0 - %d ]\n", xive_max_prio(x));
xive_dbg(x, "\tBlockId bits: %lld bits\n",
4 + GETFIELD(CQ_XIVE_CFG_BLOCK_ID_WIDTH, cfg_val));
if (CQ_XIVE_CFG_HYP_HARD_BLKID_OVERRIDE & cfg_val)
xive_dbg(x, "\tHardwired BlockId: %lld\n",
GETFIELD(CQ_XIVE_CFG_HYP_HARD_BLOCK_ID, cfg_val));
for (i = 0; i < ARRAY_SIZE(xive_configs); i++) {
if (xive_configs[i].bitmask & cfg_val)
xive_dbg(x, "\t%s\n", xive_configs[i].name);
}
}
/*
* Default XIVE configuration
*/
#define XIVE_CONFIGURATION \
(SETFIELD(CQ_XIVE_CFG_HYP_HARD_RANGE, 0ull, CQ_XIVE_CFG_THREADID_8BITS) | \
SETFIELD(CQ_XIVE_CFG_VP_INT_PRIO, 0ull, CQ_XIVE_CFG_INT_PRIO_8))
/*
* Gen1 configuration for tests (QEMU)
*/
#define XIVE_CONFIGURATION_GEN1 \
(SETFIELD(CQ_XIVE_CFG_HYP_HARD_RANGE, 0ull, CQ_XIVE_CFG_THREADID_7BITS) | \
SETFIELD(CQ_XIVE_CFG_VP_INT_PRIO, 0ull, CQ_XIVE_CFG_INT_PRIO_8) | \
CQ_XIVE_CFG_GEN1_TIMA_OS | \
CQ_XIVE_CFG_GEN1_TIMA_HYP | \
CQ_XIVE_CFG_GEN1_TIMA_HYP_BLK0 | \
CQ_XIVE_CFG_GEN1_TIMA_CROWD_DIS | \
CQ_XIVE_CFG_GEN1_END_ESX)
static bool xive_has_cap(struct xive *x, uint64_t cap)
{
return !!x && !!(x->capabilities & cap);
}
#define XIVE_CAN_STORE_EOI(x) xive_has_cap(x, CQ_XIVE_CAP_STORE_EOI)
static bool xive_cfg_save_restore(struct xive *x)
{
return !!(x->config & CQ_XIVE_CFG_EN_VP_SAVE_RESTORE);
}
/*
* When PQ_disable is available, configure the ESB cache to improve
* performance for PHB ESBs.
*
* split_mode :
* 1/3rd of the cache is reserved for PHB ESBs and the rest to
* IPIs. This is sufficient to keep all the PHB ESBs in cache and
* avoid ESB cache misses during IO interrupt processing.
*
* hash_array_enable :
* Internal cache hashing optimization. The hash_array tracks for
* ESBs where the original trigger came from so that we avoid
* getting the EAS into the cache twice.
*/
static void xive_config_esb_cache(struct xive *x)
{
uint64_t val = xive_regr(x, VC_ESBC_CFG);
if (xive_has_cap(x, CQ_XIVE_CAP_PHB_PQ_DISABLE)) {
val |= VC_ESBC_CFG_SPLIT_MODE | VC_ESBC_CFG_HASH_ARRAY_ENABLE;
val = SETFIELD(VC_ESBC_CFG_MAX_ENTRIES_IN_MODIFIED, val, 0xE);
xive_dbg(x, "ESB cache configured with split mode "
"and hash array. VC_ESBC_CFG=%016llx\n", val);
} else
val &= ~VC_ESBC_CFG_SPLIT_MODE;
xive_regw(x, VC_ESBC_CFG, val);
}
static void xive_config_fused_core(struct xive *x)
{
uint64_t val = xive_regr(x, TCTXT_CFG);
if (this_cpu()->is_fused_core) {
val |= TCTXT_CFG_FUSE_CORE_EN;
xive_dbg(x, "configured for fused cores. "
"PC_TCTXT_CFG=%016llx\n", val);
} else
val &= ~TCTXT_CFG_FUSE_CORE_EN;
xive_regw(x, TCTXT_CFG, val);
}
static void xive_config_reduced_priorities_fixup(struct xive *x)
{
if (xive_cfg_vp_prio_shift(x) < CQ_XIVE_CFG_INT_PRIO_8 &&
x->quirks & XIVE_QUIRK_BROKEN_PRIO_CHECK) {
uint64_t val = xive_regr(x, PC_ERR1_CFG1);
val &= ~PC_ERR1_CFG1_INTERRUPT_INVALID_PRIO;
xive_dbg(x, "workaround for reduced priorities. "
"PC_ERR1_CFG1=%016llx\n", val);
xive_regw(x, PC_ERR1_CFG1, val);
}
}
static bool xive_config_init(struct xive *x)
{
x->capabilities = xive_regr(x, CQ_XIVE_CAP);
xive_dump_capabilities(x, x->capabilities);
x->generation = GETFIELD(CQ_XIVE_CAP_VERSION, x->capabilities);
/*
* Allow QEMU to override version for tests
*/
if (x->generation != XIVE_GEN2 && !chip_quirk(QUIRK_QEMU)) {
xive_err(x, "Invalid XIVE controller version %d\n",
x->generation);
return false;
}
x->config = xive_regr(x, CQ_XIVE_CFG);
xive_dump_configuration(x, "default", x->config);
/* Start with default settings */
x->config = x->generation == XIVE_GEN1 ? XIVE_CONFIGURATION_GEN1 :
XIVE_CONFIGURATION;
if (x->quirks & XIVE_QUIRK_THREADID_7BITS)
x->config = SETFIELD(CQ_XIVE_CFG_HYP_HARD_RANGE, x->config,
CQ_XIVE_CFG_THREADID_7BITS);
/*
* Hardwire the block ID. The default value is the topology ID
* of the chip which is different from the block.
*/
x->config |= CQ_XIVE_CFG_HYP_HARD_BLKID_OVERRIDE |
SETFIELD(CQ_XIVE_CFG_HYP_HARD_BLOCK_ID, 0ull, x->block_id);
/*
* Enable "VP Context Save and Restore" by default. it is
* compatible with KVM which currently does the context
* save&restore in the entry/exit path of the vCPU
*/
if (x->capabilities & CQ_XIVE_CAP_VP_SAVE_RESTORE)
x->config |= CQ_XIVE_CFG_EN_VP_SAVE_RESTORE;
xive_dump_configuration(x, "new", x->config);
xive_regw(x, CQ_XIVE_CFG, x->config);
if (xive_regr(x, CQ_XIVE_CFG) != x->config) {
xive_err(x, "configuration setting failed\n");
}
/*
* Disable error reporting in the FIR for info errors from the VC.
*/
xive_regw(x, CQ_FIRMASK_OR, CQ_FIR_VC_INFO_ERROR_0_2);
/*
* Mask CI Load and Store to bad location, as IPI trigger
* pages may be mapped to user space, and a read on the
* trigger page causes a checkstop
*/
xive_regw(x, CQ_FIRMASK_OR, CQ_FIR_PB_RCMDX_CI_ERR1);
/*
* VP space settings. P9 mode is 19bits.
*/
x->vp_shift = x->generation == XIVE_GEN1 ?
VP_SHIFT_GEN1 : VP_SHIFT_GEN2;
/*
* VP ids for HW threads. These values are hardcoded in the
* CAM line of the HW context
*
* POWER10 |chip|0000000000000001|threadid|
* 28bits 4 16 8
*
* POWER9 |chip|000000000001|thrdid |
* 23bits 4 12 7
*/
/* TODO (cosmetic): set VP ids for HW threads only once */
xive_threadid_shift = 7 + GETFIELD(CQ_XIVE_CFG_HYP_HARD_RANGE,
x->config);
xive_hw_vp_base = 1 << xive_threadid_shift;
xive_hw_vp_count = 1 << xive_threadid_shift;
xive_dbg(x, "store EOI is %savailable\n",
XIVE_CAN_STORE_EOI(x) ? "" : "not ");
xive_config_fused_core(x);
xive_config_esb_cache(x);
xive_config_reduced_priorities_fixup(x);
return true;
}
/* Set Translation tables : 1 block per chip */
static bool xive_setup_set_xlate(struct xive *x)
{
unsigned int i;
/* Configure ESBs */
xive_regw(x, CQ_TAR,
CQ_TAR_AUTOINC | SETFIELD(CQ_TAR_SELECT, 0ull, CQ_TAR_ESB));
if (x->last_reg_error)
return false;
for (i = 0; i < XIVE_MAX_BLOCKS; i++) {
xive_regw(x, CQ_TDR, CQ_TDR_VALID |
SETFIELD(CQ_TDR_BLOCK_ID, 0ull, x->block_id));
if (x->last_reg_error)
return false;
}
/* Configure ENDs */
xive_regw(x, CQ_TAR,
CQ_TAR_AUTOINC | SETFIELD(CQ_TAR_SELECT, 0ull, CQ_TAR_END));
if (x->last_reg_error)
return false;
for (i = 0; i < XIVE_MAX_BLOCKS; i++) {
xive_regw(x, CQ_TDR, CQ_TDR_VALID |
SETFIELD(CQ_TDR_BLOCK_ID, 0ull, x->block_id));
if (x->last_reg_error)
return false;
}
/* Configure NVPs */
xive_regw(x, CQ_TAR,
CQ_TAR_AUTOINC | SETFIELD(CQ_TAR_SELECT, 0ull, CQ_TAR_NVPG));
if (x->last_reg_error)
return false;
for (i = 0; i < XIVE_MAX_BLOCKS; i++) {
xive_regw(x, CQ_TDR, CQ_TDR_VALID |
SETFIELD(CQ_TDR_BLOCK_ID, 0ull, x->block_id));
if (x->last_reg_error)
return false;
}
return true;
}
static bool xive_prealloc_tables(struct xive *x)
{
uint32_t i;
uint32_t pbase, pend;
/* ESB has 4 entries per byte */
x->sbe_base = local_alloc(x->chip_id, XIVE_ESB_SIZE, XIVE_ESB_SIZE);
if (!x->sbe_base) {
xive_err(x, "Failed to allocate SBE\n");
return false;
}
/* PQs are initialized to 0b01 which corresponds to "ints off" */
memset(x->sbe_base, 0x55, XIVE_ESB_SIZE);
xive_dbg(x, "SBE at %p size 0x%lx\n", x->sbe_base, XIVE_ESB_SIZE);
/* EAS entries are 8 bytes */
x->eat_base = local_alloc(x->chip_id, XIVE_EAT_SIZE, XIVE_EAT_SIZE);
if (!x->eat_base) {
xive_err(x, "Failed to allocate EAS\n");
return false;
}
/*
* We clear the entries (non-valid). They will be initialized
* when actually used
*/
memset(x->eat_base, 0, XIVE_EAT_SIZE);
xive_dbg(x, "EAT at %p size 0x%lx\n", x->eat_base, XIVE_EAT_SIZE);
/* Indirect END table. Limited to one top page. */
x->end_ind_size = ALIGN_UP(XIVE_END_TABLE_SIZE, PAGE_SIZE);
if (x->end_ind_size > PAGE_SIZE) {
xive_err(x, "END indirect table is too big !\n");
return false;
}
x->end_ind_base = local_alloc(x->chip_id, x->end_ind_size,
x->end_ind_size);
if (!x->end_ind_base) {
xive_err(x, "Failed to allocate END indirect table\n");
return false;
}
memset(x->end_ind_base, 0, x->end_ind_size);
xive_dbg(x, "ENDi at %p size 0x%llx #%ld entries\n", x->end_ind_base,
x->end_ind_size, XIVE_END_COUNT);
x->end_ind_count = XIVE_END_TABLE_SIZE / XIVE_VSD_SIZE;
/* Indirect VP table. Limited to one top page. */
x->vp_ind_size = ALIGN_UP(XIVE_VP_TABLE_SIZE(x), PAGE_SIZE);
if (x->vp_ind_size > PAGE_SIZE) {
xive_err(x, "VP indirect table is too big !\n");
return false;
}
x->vp_ind_base = local_alloc(x->chip_id, x->vp_ind_size,
x->vp_ind_size);
if (!x->vp_ind_base) {
xive_err(x, "Failed to allocate VP indirect table\n");
return false;
}
xive_dbg(x, "VPi at %p size 0x%llx #%ld entries\n", x->vp_ind_base,
x->vp_ind_size, XIVE_VP_COUNT(x));
x->vp_ind_count = XIVE_VP_TABLE_SIZE(x) / XIVE_VSD_SIZE;
memset(x->vp_ind_base, 0, x->vp_ind_size);
/* Allocate pages for the VP ids representing HW threads */
pbase = xive_hw_vp_base / VP_PER_PAGE;
pend = (xive_hw_vp_base + xive_hw_vp_count) / VP_PER_PAGE;
xive_dbg(x, "Allocating pages %d to %d of VPs (for %d VPs)\n",
pbase, pend, xive_hw_vp_count);
for (i = pbase; i <= pend; i++) {
void *page;
u64 vsd;
/* Indirect entries have a VSD format */
page = local_alloc(x->chip_id, PAGE_SIZE, PAGE_SIZE);
if (!page) {
xive_err(x, "Failed to allocate VP page\n");
return false;
}
xive_dbg(x, "VP%d at %p size 0x%x\n", i, page, PAGE_SIZE);
memset(page, 0, PAGE_SIZE);
vsd = ((uint64_t)page) & VSD_ADDRESS_MASK;
vsd |= SETFIELD(VSD_TSIZE, 0ull, 4);
vsd |= SETFIELD(VSD_MODE, 0ull, VSD_MODE_EXCLUSIVE);
vsd |= VSD_FIRMWARE;
x->vp_ind_base[i] = cpu_to_be64(vsd);
}
/*
* Allocate page for cache and sync injection (512 * 128 hw
* threads) + one extra page for future use
*/
x->sync_inject_size = PAGE_SIZE + PAGE_SIZE;
x->sync_inject = local_alloc(x->chip_id, x->sync_inject_size,
x->sync_inject_size);
if (!x->sync_inject) {
xive_err(x, "Failed to allocate sync pages\n");
return false;
}
/*
* The Memory Coherence Directory uses 16M "granule" to track
* shared copies of a cache line. If any cache line within the
* 16M range gets touched by someone outside of the group, the
* MCD forces accesses to any cache line within the range to
* include everyone that might have a shared copy.
*/
#define QUEUE_OVF_ALIGN (16 << 20) /* MCD granule size */
/*
* Allocate the queue overflow pages and use a 16M alignment
* to avoid sharing with other structures and reduce traffic
* on the PowerBus.
*/
x->q_ovf = local_alloc(x->chip_id, VC_QUEUE_COUNT * PAGE_SIZE,
QUEUE_OVF_ALIGN);
if (!x->q_ovf) {
xive_err(x, "Failed to allocate queue overflow\n");
return false;
}
return true;
}
static void xive_add_provisioning_properties(void)
{
beint32_t chips[XIVE_MAX_CHIPS];
uint32_t i, count;
dt_add_property_cells(xive_dt_node,
"ibm,xive-provision-page-size", PAGE_SIZE);
count = 1 << xive_chips_alloc_bits;
for (i = 0; i < count; i++)
chips[i] = cpu_to_be32(xive_block_to_chip[i]);
dt_add_property(xive_dt_node, "ibm,xive-provision-chips",
chips, 4 * count);
}
static void xive_create_mmio_dt_node(struct xive *x)
{
uint64_t tb = (uint64_t)x->tm_base;
uint32_t stride = 1u << x->tm_shift;
xive_dt_node = dt_new_addr(dt_root, "interrupt-controller", tb);
assert(xive_dt_node);
dt_add_property_u64s(xive_dt_node, "reg",
tb + 0 * stride, stride,
tb + 1 * stride, stride,
tb + 2 * stride, stride,
tb + 3 * stride, stride);
dt_add_property_strings(xive_dt_node, "compatible",
"ibm,opal-xive-pe", "ibm,opal-intc");
dt_add_property_cells(xive_dt_node, "ibm,xive-eq-sizes",
12, 16, 21, 24);
dt_add_property_cells(xive_dt_node, "ibm,xive-#priorities",
xive_cfg_vp_prio(x));
dt_add_property(xive_dt_node, "single-escalation-support", NULL, 0);
if (XIVE_CAN_STORE_EOI(x))
dt_add_property(xive_dt_node, "store-eoi", NULL, 0);
if (xive_cfg_save_restore(x))
dt_add_property(xive_dt_node, "vp-save-restore", NULL, 0);
xive_add_provisioning_properties();
}
static void xive_setup_forward_ports(struct xive *x, struct proc_chip *remote_chip)
{
struct xive *remote_xive = remote_chip->xive;
uint64_t base = SETFIELD(VSD_MODE, 0ull, VSD_MODE_FORWARD);
if (!xive_set_vsd(x, VST_ESB, remote_xive->block_id,
base | ((uint64_t)remote_xive->esb_base) |
SETFIELD(VSD_TSIZE, 0ull, ilog2(x->esb_size) - 12)))
goto error;
/* EAS: No remote */
if (!xive_set_vsd(x, VST_END, remote_xive->block_id,
base | ((uint64_t)remote_xive->end_base) |
SETFIELD(VSD_TSIZE, 0ull, ilog2(x->end_size) - 12)))
goto error;
if (!xive_set_vsd(x, VST_NVP, remote_xive->block_id,
base | ((uint64_t)remote_xive->nvp_base) |
SETFIELD(VSD_TSIZE, 0ull, ilog2(x->nvp_size) - 12)))
goto error;
/* NVG: not used */
/* NVC: not used */
if (!xive_set_vsd(x, VST_IC, remote_xive->chip_id,
base | ((uint64_t)remote_xive->ic_base) |
SETFIELD(VSD_TSIZE, 0ull, ilog2(x->ic_size) - 12)))
goto error;
if (!xive_set_vsd(x, VST_SYNC, remote_xive->chip_id,
base | ((uint64_t)remote_xive->sync_inject) |
SETFIELD(VSD_TSIZE, 0ull, ilog2(x->sync_inject_size) - 12)))
goto error;
/* ERQ: No remote */
return;
error:
xive_err(x, "Failure configuring forwarding ports\n");
}
static void late_init_one_xive(struct xive *x)
{
struct proc_chip *chip;
/* We need to setup the cross-chip forward ports. Let's
* iterate all chip and set them up accordingly
*/
for_each_chip(chip) {
/* We skip ourselves or chips without a xive */
if (chip->xive == x || !chip->xive)
continue;
/* Setup our forward ports to that chip */
xive_setup_forward_ports(x, chip);
}
}
static bool xive_check_ipi_free(struct xive *x, uint32_t irq, uint32_t count)
{
uint32_t i, idx = GIRQ_TO_IDX(irq);
for (i = 0; i < count; i++)
if (bitmap_tst_bit(*x->ipi_alloc_map, idx + i))
return false;
return true;
}
uint32_t xive2_alloc_hw_irqs(uint32_t chip_id, uint32_t count,
uint32_t align)
{
struct proc_chip *chip = get_chip(chip_id);
struct xive *x;
uint32_t base, i;
assert(chip);
assert(is_pow2(align));
x = chip->xive;
assert(x);
lock(&x->lock);
/* Allocate the HW interrupts */
base = x->int_hw_bot - count;
base &= ~(align - 1);
if (base < x->int_ipi_top) {
xive_err(x,
"HW alloc request for %d interrupts aligned to %d failed\n",
count, align);
unlock(&x->lock);
return XIVE_IRQ_ERROR;
}
if (!xive_check_ipi_free(x, base, count)) {
xive_err(x, "HWIRQ boot allocator request overlaps dynamic allocator\n");
unlock(&x->lock);
return XIVE_IRQ_ERROR;
}
x->int_hw_bot = base;
/* Initialize the corresponding EAS entries to sane defaults,
* IE entry is valid, not routed and masked, EQ data is set
* to the GIRQ number.
*/
for (i = 0; i < count; i++) {
struct xive_eas *eas = xive_get_eas(x, base + i);
eas->w = xive_set_field64(EAS_VALID, 0, 1) |
xive_set_field64(EAS_MASKED, 0, 1) |
xive_set_field64(EAS_END_DATA, 0, base + i);
}
unlock(&x->lock);
return base;
}
uint32_t xive2_alloc_ipi_irqs(uint32_t chip_id, uint32_t count,
uint32_t align)
{
struct proc_chip *chip = get_chip(chip_id);
struct xive *x;
uint32_t base, i;
assert(chip);
assert(is_pow2(align));
x = chip->xive;
assert(x);
lock(&x->lock);
/* Allocate the IPI interrupts */
base = x->int_ipi_top + (align - 1);
base &= ~(align - 1);
if (base >= x->int_hw_bot) {
xive_err(x,
"IPI alloc request for %d interrupts aligned to %d failed\n",
count, align);
unlock(&x->lock);
return XIVE_IRQ_ERROR;
}
if (!xive_check_ipi_free(x, base, count)) {
xive_err(x, "IPI boot allocator request overlaps dynamic allocator\n");
unlock(&x->lock);
return XIVE_IRQ_ERROR;
}
x->int_ipi_top = base + count;
/* Initialize the corresponding EAS entries to sane defaults,
* IE entry is valid, not routed and masked, END data is set
* to the GIRQ number.
*/
for (i = 0; i < count; i++) {
struct xive_eas *eas = xive_get_eas(x, base + i);
eas->w = xive_set_field64(EAS_VALID, 0, 1) |
xive_set_field64(EAS_MASKED, 0, 1) |
xive_set_field64(EAS_END_DATA, 0, base + i);
}
unlock(&x->lock);
return base;
}
void *xive2_get_trigger_port(uint32_t girq)
{
uint32_t idx = GIRQ_TO_IDX(girq);
struct xive *x;
/* Find XIVE on which the EAS resides */
x = xive_from_isn(girq);
if (!x)
return NULL;
if (GIRQ_IS_ESCALATION(girq)) {
/* There is no trigger page for escalation interrupts */
return NULL;
} else {
/* Make sure it's an IPI on that chip */
if (girq < x->int_base ||
girq >= x->int_ipi_top)
return NULL;
return x->esb_base + idx * XIVE_ESB_PAGE_SIZE;
}
}
/*
* Notify Port page (writes only, w/data), separated into two
* categories, both sent to VC:
* - IPI queue (Addr bit 52 = 0) (for NPU)
* - HW queue (Addr bit 52 = 1)
*/
uint64_t xive2_get_notify_port(uint32_t chip_id, uint32_t ent)
{
struct proc_chip *chip = get_chip(chip_id);
struct xive *x;
uint32_t offset = 0;
assert(chip);
x = chip->xive;
assert(x);
/* This is where we can assign a different HW queue to a different
* source by offsetting into the cache lines of the notify port
*
* For now we keep it very basic, this will have to be looked at
* again on real HW with some proper performance analysis.
*
* Here's what Florian says on the matter:
*
* <<
* The first 2k of the notify port page can all be used for PCIe triggers
*
* However the idea would be that we try to use the first 4 cache lines to
* balance the PCIe Interrupt requests to use the least used snoop buses
* (we went from 2 to 4 snoop buses for P9). snoop 0 is heavily used
* (I think TLBIs are using that in addition to the normal addresses),
* snoop 3 is used for all Int commands, so I think snoop 2 (CL 2 in the
* page) is the least used overall. So we probably should that one for
* the Int commands from PCIe.
*
* In addition, our EAS cache supports hashing to provide "private" cache
* areas for the PHBs in the shared 1k EAS cache. This allows e.g. to avoid
* that one "thrashing" PHB thrashes the EAS cache for everyone, or provide
* a PHB with a private area that would allow high cache hits in case of a
* device using very few interrupts. The hashing is based on the offset within
* the cache line. So using that, you can e.g. set the EAS cache up so that
* IPIs use 512 entries, the x16 PHB uses 256 entries and the x8 PHBs 128
* entries each - or IPIs using all entries and sharing with PHBs, so PHBs
* would use 512 entries and 256 entries respectively.
*
* This is a tuning we would probably do later in the lab, but as a "prep"
* we should set up the different PHBs such that they are using different
* 8B-aligned offsets within the cache line, so e.g.
* PH4_0 addr 0x100 (CL 2 DW0
* PH4_1 addr 0x108 (CL 2 DW1)
* PH4_2 addr 0x110 (CL 2 DW2)
* etc.
* >>
*
* I'm using snoop1 for PHB0 and snoop2 for everybody else.
*/
/* Florian adds :
*
* we just set them up for a start to have different offsets
* within the cache line so that we could use the allocation
* restrictions that can be enforced in the interrupt
* controller
*
* P10 might now be randomizing the cache line bits in HW to
* balance snoop bus usage
*/
switch(ent) {
case XIVE_HW_SRC_PHBn(0):
offset = 0x800;
break;
case XIVE_HW_SRC_PHBn(1):
offset = 0x908;
break;
case XIVE_HW_SRC_PHBn(2):
offset = 0x910;
break;
case XIVE_HW_SRC_PHBn(3):
offset = 0x918;
break;
case XIVE_HW_SRC_PHBn(4):
offset = 0x920;
break;
case XIVE_HW_SRC_PHBn(5):
offset = 0x928;
break;
case XIVE_HW_SRC_PSI:
offset = 0x930;
break;
default:
assert(false);
return 0;
}
return ((uint64_t)x->ic_base) +
(XIVE_NOTIFY_PGOFF << x->ic_shift) + offset;
}
/* Manufacture the powerbus packet bits 32:63 */
__attrconst uint32_t xive2_get_notify_base(uint32_t girq)
{
return (GIRQ_TO_BLK(girq) << 28) | GIRQ_TO_IDX(girq);
}
static bool xive_get_irq_targetting(uint32_t isn, uint32_t *out_target,
uint8_t *out_prio, uint32_t *out_lirq)
{
struct xive_eas *eas;
struct xive *x, *end_x;
struct xive_end *end;
uint32_t end_blk, end_idx;
uint32_t vp_blk, vp_idx;
uint32_t prio, server;
bool is_escalation = GIRQ_IS_ESCALATION(isn);
/* Find XIVE on which the EAS resides */
x = xive_from_isn(isn);
if (!x)
return false;
/* Grab the EAS */
eas = xive_get_eas(x, isn);
if (!eas)
return false;
if (!xive_get_field64(EAS_VALID, eas->w) && !is_escalation) {
xive_err(x, "ISN %x lead to invalid EAS !\n", isn);
return false;
}
if (out_lirq)
*out_lirq = xive_get_field64(EAS_END_DATA, eas->w);
/* Find the END and its xive instance */
end_blk = xive_get_field64(EAS_END_BLOCK, eas->w);
end_idx = xive_get_field64(EAS_END_INDEX, eas->w);
end_x = xive_from_vc_blk(end_blk);
/* This can fail if the interrupt hasn't been initialized yet
* but it should also be masked, so fail silently
*/
if (!end_x)
goto pick_default;
end = xive_get_end(end_x, end_idx);
if (!end)
goto pick_default;
/* XXX Check valid and format 0 */
/* No priority conversion, return the actual one ! */
if (xive_get_field64(EAS_MASKED, eas->w))
prio = 0xff;
else
prio = xive_get_field32(END_W7_F0_PRIORITY, end->w7);
if (out_prio)
*out_prio = prio;
vp_blk = xive_get_field32(END_W6_VP_BLOCK, end->w6);
vp_idx = xive_get_field32(END_W6_VP_OFFSET, end->w6);
server = VP2PIR(vp_blk, vp_idx);
if (out_target)
*out_target = server;
xive_vdbg(end_x, "END info for ISN %x: prio=%d, server=0x%x (VP %x/%x)\n",
isn, prio, server, vp_blk, vp_idx);
return true;
pick_default:
xive_vdbg(end_x, "END info for ISN %x: Using masked defaults\n", isn);
if (out_prio)
*out_prio = 0xff;
/* Pick a random default, me will be fine ... */
if (out_target)
*out_target = mfspr(SPR_PIR);
return true;
}
static inline bool xive_end_for_target(uint32_t target, uint8_t prio,
uint32_t *out_end_blk,
uint32_t *out_end_idx)
{
struct xive *x;
struct xive_nvp *vp;
uint32_t vp_blk, vp_idx;
uint32_t end_blk, end_idx;
if (prio > xive_max_prio(one_xive))
return false;
/* Get the VP block/index from the target word */
if (!xive_decode_vp(target, &vp_blk, &vp_idx, NULL, NULL))
return false;
/* Grab the target VP's XIVE */
x = xive_from_pc_blk(vp_blk);
if (!x)
return false;
/* Find the VP structrure where we stashed the END number */
vp = xive_get_vp(x, vp_idx);
if (!vp)
return false;
end_blk = xive_get_field32(NVP_W5_VP_END_BLOCK, vp->w5);
end_idx = xive_get_field32(NVP_W5_VP_END_INDEX, vp->w5);
/* Currently the END block and VP block should be the same */
if (end_blk != vp_blk) {
xive_err(x, "end_blk != vp_blk (%d vs. %d) for target 0x%08x/%d\n",
end_blk, vp_blk, target, prio);
assert(false);
}
if (out_end_blk)
*out_end_blk = end_blk;
if (out_end_idx)
*out_end_idx = end_idx + prio;
return true;
}
static int64_t xive_set_irq_targetting(uint32_t isn, uint32_t target,
uint8_t prio, uint32_t lirq,
bool synchronous)
{
struct xive *x;
struct xive_eas *eas, new_eas;
uint32_t end_blk, end_idx;
bool is_escalation = GIRQ_IS_ESCALATION(isn);
int64_t rc;
/* Find XIVE on which the EAS resides */
x = xive_from_isn(isn);
if (!x)
return OPAL_PARAMETER;
/* Grab the EAS */
eas = xive_get_eas(x, isn);
if (!eas)
return OPAL_PARAMETER;
if (!xive_get_field64(EAS_VALID, eas->w) && !is_escalation) {
xive_err(x, "ISN %x lead to invalid EAS !\n", isn);
return OPAL_PARAMETER;
}
lock(&x->lock);
/* Read existing EAS */
new_eas = *eas;
/* Are we masking ? */
if (prio == 0xff && !is_escalation) {
new_eas.w = xive_set_field64(EAS_MASKED, new_eas.w, 1);
xive_vdbg(x, "ISN %x masked !\n", isn);
/* Put prio 7 in the END */
prio = xive_max_prio(x);
} else {
/* Unmasking */
new_eas.w = xive_set_field64(EAS_MASKED, new_eas.w, 0);
xive_vdbg(x, "ISN %x unmasked !\n", isn);
/* For normal interrupt sources, keep track of which ones
* we ever enabled since the last reset
*/
if (!is_escalation)
bitmap_set_bit(*x->int_enabled_map, GIRQ_TO_IDX(isn));
}
/* If prio isn't 0xff, re-target the EAS. First find the END
* correponding to the target
*/
if (prio != 0xff) {
if (!xive_end_for_target(target, prio, &end_blk, &end_idx)) {
xive_err(x, "Can't find END for target/prio 0x%x/%d\n",
target, prio);
unlock(&x->lock);
return OPAL_PARAMETER;
}
/* Try to update it atomically to avoid an intermediary
* stale state
*/
new_eas.w = xive_set_field64(EAS_END_BLOCK, new_eas.w, end_blk);
new_eas.w = xive_set_field64(EAS_END_INDEX, new_eas.w, end_idx);
}
new_eas.w = xive_set_field64(EAS_END_DATA, new_eas.w, lirq);
xive_vdbg(x,"ISN %x routed to end %x/%x lirq=%08x EAS=%016llx !\n",
isn, end_blk, end_idx, lirq, new_eas.w);
/* Updating the cache differs between real EAS and escalation
* EAS inside an END
*/
if (is_escalation) {
rc = xive_escalation_ive_cache_update(x, x->block_id,
GIRQ_TO_IDX(isn), &new_eas, synchronous);
} else {
sync();
*eas = new_eas;
rc = xive_easc_scrub(x, x->block_id, GIRQ_TO_IDX(isn));
}
unlock(&x->lock);
return rc;
}
static void xive_update_irq_mask(struct xive_src *s, uint32_t idx, bool masked)
{
void *mmio_base = s->esb_mmio + (1ul << s->esb_shift) * idx;
uint32_t offset;
/* XXX FIXME: A quick mask/umask can make us shoot an interrupt
* more than once to a queue. We need to keep track better
*/
if (s->flags & XIVE_SRC_EOI_PAGE1)
mmio_base += 1ull << (s->esb_shift - 1);
if (masked)
offset = XIVE_ESB_SET_PQ_01;
else
offset = XIVE_ESB_SET_PQ_00;
in_be64(mmio_base + offset);
}
#define XIVE_SYNC_IPI 0x000
#define XIVE_SYNC_HW 0x080
#define XIVE_SYNC_NxC 0x100
#define XIVE_SYNC_INT 0x180
#define XIVE_SYNC_OS_ESC 0x200
#define XIVE_SYNC_POOL_ESC 0x280
#define XIVE_SYNC_HARD_ESC 0x300
static int64_t xive_sync(struct xive *x __unused)
{
uint64_t r;
void *sync_base;
lock(&x->lock);
sync_base = x->ic_base + (XIVE_SYNC_POLL_PGOFF << x->ic_shift);
out_be64(sync_base + XIVE_SYNC_IPI, 0);
out_be64(sync_base + XIVE_SYNC_HW, 0);
out_be64(sync_base + XIVE_SYNC_NxC, 0);
out_be64(sync_base + XIVE_SYNC_INT, 0);
out_be64(sync_base + XIVE_SYNC_OS_ESC, 0);
out_be64(sync_base + XIVE_SYNC_POOL_ESC, 0);
out_be64(sync_base + XIVE_SYNC_HARD_ESC, 0);
/* XXX Add timeout */
for (;;) {
r = xive_regr(x, VC_ENDC_SYNC_DONE);
if ((r & VC_ENDC_SYNC_POLL_DONE) == VC_ENDC_SYNC_POLL_DONE)
break;
cpu_relax();
}
xive_regw(x, VC_ENDC_SYNC_DONE, r & ~VC_ENDC_SYNC_POLL_DONE);
/*
* Do a read after clearing the sync done bit to prevent any
* race between CI write and next sync command
*/
xive_regr(x, VC_ENDC_SYNC_DONE);
unlock(&x->lock);
return 0;
}
static int64_t __xive_set_irq_config(struct irq_source *is, uint32_t girq,
uint64_t vp, uint8_t prio, uint32_t lirq,
bool update_esb, bool sync)
{
struct xive_src *s = container_of(is, struct xive_src, is);
uint32_t old_target, vp_blk;
u8 old_prio;
int64_t rc;
/* Grab existing target */
if (!xive_get_irq_targetting(girq, &old_target, &old_prio, NULL))
return OPAL_PARAMETER;
/* Let XIVE configure the END. We do the update without the
* synchronous flag, thus a cache update failure will result
* in us returning OPAL_BUSY
*/
rc = xive_set_irq_targetting(girq, vp, prio, lirq, false);
if (rc)
return rc;
/* Do we need to update the mask ? */
if (old_prio != prio && (old_prio == 0xff || prio == 0xff)) {
/* The source has special variants of masking/unmasking */
if (update_esb) {
/* Ensure it's enabled/disabled in the source
* controller
*/
xive_update_irq_mask(s, girq - s->esb_base,
prio == 0xff);
}
}
/*
* Synchronize the source and old target XIVEs to ensure that
* all pending interrupts to the old target have reached their
* respective queue.
*
* WARNING: This assumes the VP and it's queues are on the same
* XIVE instance !
*/
if (!sync)
return OPAL_SUCCESS;
xive_sync(s->xive);
if (xive_decode_vp(old_target, &vp_blk, NULL, NULL, NULL)) {
struct xive *x = xive_from_pc_blk(vp_blk);
if (x)
xive_sync(x);
}
return OPAL_SUCCESS;
}
static int64_t xive_set_irq_config(uint32_t girq, uint64_t vp, uint8_t prio,
uint32_t lirq, bool update_esb)
{
struct irq_source *is = irq_find_source(girq);
return __xive_set_irq_config(is, girq, vp, prio, lirq, update_esb,
true);
}
static void xive_source_interrupt(struct irq_source *is, uint32_t isn)
{
struct xive_src *s = container_of(is, struct xive_src, is);
if (!s->orig_ops || !s->orig_ops->interrupt)
return;
s->orig_ops->interrupt(is, isn);
}
static uint64_t xive_source_attributes(struct irq_source *is, uint32_t isn)
{
struct xive_src *s = container_of(is, struct xive_src, is);
if (!s->orig_ops || !s->orig_ops->attributes)
return IRQ_ATTR_TARGET_LINUX;
return s->orig_ops->attributes(is, isn);
}
static char *xive_source_name(struct irq_source *is, uint32_t isn)
{
struct xive_src *s = container_of(is, struct xive_src, is);
if (!s->orig_ops || !s->orig_ops->name)
return NULL;
return s->orig_ops->name(is, isn);
}
void xive2_source_mask(struct irq_source *is, uint32_t isn)
{
struct xive_src *s = container_of(is, struct xive_src, is);
xive_update_irq_mask(s, isn - s->esb_base, true);
}
static const struct irq_source_ops xive_irq_source_ops = {
.interrupt = xive_source_interrupt,
.attributes = xive_source_attributes,
.name = xive_source_name,
};
static void __xive_register_source(struct xive *x, struct xive_src *s,
uint32_t base, uint32_t count,
uint32_t shift, void *mmio, uint32_t flags,
bool secondary, void *data,
const struct irq_source_ops *orig_ops)
{
s->esb_base = base;
s->esb_shift = shift;
s->esb_mmio = mmio;
s->flags = flags;
s->orig_ops = orig_ops;
s->xive = x;
s->is.start = base;
s->is.end = base + count;
s->is.ops = &xive_irq_source_ops;
s->is.data = data;
__register_irq_source(&s->is, secondary);
}
void xive2_register_hw_source(uint32_t base, uint32_t count, uint32_t shift,
void *mmio, uint32_t flags, void *data,
const struct irq_source_ops *ops)
{
struct xive_src *s;
struct xive *x = xive_from_isn(base);
assert(x);
s = malloc(sizeof(struct xive_src));
assert(s);
__xive_register_source(x, s, base, count, shift, mmio, flags,
false, data, ops);
}
static void __xive2_register_esb_source(uint32_t base, uint32_t count,
void *data, const struct irq_source_ops *ops)
{
struct xive_src *s;
struct xive *x = xive_from_isn(base);
uint32_t base_idx = GIRQ_TO_IDX(base);
void *mmio_base;
uint32_t flags = XIVE_SRC_EOI_PAGE1 | XIVE_SRC_TRIGGER_PAGE;
assert(x);
s = malloc(sizeof(struct xive_src));
assert(s);
if (XIVE_CAN_STORE_EOI(x))
flags |= XIVE_SRC_STORE_EOI;
/* Callbacks assume the MMIO base corresponds to the first
* interrupt of that source structure so adjust it
*/
mmio_base = x->esb_base + (1ul << XIVE_ESB_SHIFT) * base_idx;
__xive_register_source(x, s, base, count, XIVE_ESB_SHIFT, mmio_base,
flags, false, data, ops);
}
/*
* Check that IPI sources have interrupt numbers in the IPI interrupt
* number range
*/
void xive2_register_ipi_source(uint32_t base, uint32_t count, void *data,
const struct irq_source_ops *ops)
{
struct xive *x = xive_from_isn(base);
assert(x);
assert(base >= x->int_base && (base + count) <= x->int_ipi_top);
__xive2_register_esb_source(base, count, data, ops);
}
/*
* Some HW sources (PHB) can disable the use of their own ESB pages
* and offload all the checks on ESB pages of the IC. The interrupt
* numbers are not necessarily in the IPI range.
*/
void xive2_register_esb_source(uint32_t base, uint32_t count)
{
__xive2_register_esb_source(base, count, NULL, NULL);
}
uint64_t xive2_get_esb_base(uint32_t base)
{
struct xive *x = xive_from_isn(base);
uint32_t base_idx = GIRQ_TO_IDX(base);
assert(x);
return (uint64_t) x->esb_base + (1ul << XIVE_ESB_SHIFT) * base_idx;
}
static void xive_set_quirks(struct xive *x, struct proc_chip *chip __unused)
{
uint64_t quirks = 0;
/* This extension is dropped for P10 */
if (proc_gen == proc_gen_p10)
quirks |= XIVE_QUIRK_THREADID_7BITS;
/* Broken check on invalid priority when reduced priorities is in use */
if (proc_gen == proc_gen_p10)
quirks |= XIVE_QUIRK_BROKEN_PRIO_CHECK;
xive_dbg(x, "setting XIVE quirks to %016llx\n", quirks);
x->quirks = quirks;
}
static struct xive *init_one_xive(struct dt_node *np)
{
struct xive *x;
struct proc_chip *chip;
uint32_t flags;
x = zalloc(sizeof(struct xive));
assert(x);
x->x_node = np;
x->xscom_base = dt_get_address(np, 0, NULL);
x->chip_id = dt_get_chip_id(np);
/* "Allocate" a new block ID for the chip */
x->block_id = xive_block_count++;
assert (x->block_id < XIVE_MAX_CHIPS);
xive_block_to_chip[x->block_id] = x->chip_id;
init_lock(&x->lock);
chip = get_chip(x->chip_id);
assert(chip);
xive_notice(x, "Initializing XIVE block ID %d...\n", x->block_id);
chip->xive = x;
xive_set_quirks(x, chip);
list_head_init(&x->donated_pages);
/* Base interrupt numbers and allocator init */
x->int_base = BLKIDX_TO_GIRQ(x->block_id, 0);
x->int_count = x->int_base + XIVE_INT_COUNT;
x->int_hw_bot = x->int_count;
x->int_ipi_top = x->int_base;
if (x->int_ipi_top < XIVE_INT_FIRST)
x->int_ipi_top = XIVE_INT_FIRST;
/* Allocate a few bitmaps */
x->end_map = local_alloc(x->chip_id, BITMAP_BYTES(xive_end_bitmap_size(x)), PAGE_SIZE);
assert(x->end_map);
memset(x->end_map, 0, BITMAP_BYTES(xive_end_bitmap_size(x)));
/*
* Allocate END index 0 to make sure it can not be used as an
* END base for a VP. This is the criteria to know if a VP was
* allocated.
*/
bitmap_set_bit(*x->end_map, 0);
x->int_enabled_map = local_alloc(x->chip_id, BITMAP_BYTES(XIVE_INT_COUNT), PAGE_SIZE);
assert(x->int_enabled_map);
memset(x->int_enabled_map, 0, BITMAP_BYTES(XIVE_INT_COUNT));
x->ipi_alloc_map = local_alloc(x->chip_id, BITMAP_BYTES(XIVE_INT_COUNT), PAGE_SIZE);
assert(x->ipi_alloc_map);
memset(x->ipi_alloc_map, 0, BITMAP_BYTES(XIVE_INT_COUNT));
xive_dbg(x, "Handling interrupts [%08x..%08x]\n",
x->int_base, x->int_count - 1);
/* Setup the IC BARs */
if (!xive_configure_ic_bars(x))
goto fail;
/* Some basic global inits such as page sizes etc... */
if (!xive_config_init(x))
goto fail;
/* Configure the set translations for MMIO */
if (!xive_setup_set_xlate(x))
goto fail;
/* Dump some MMIO registers for diagnostics */
xive_dump_mmio(x);
/* Pre-allocate a number of tables */
if (!xive_prealloc_tables(x))
goto fail;
/* Setup the XIVE structures BARs */
if (!xive_configure_bars(x))
goto fail;
/*
* Configure local tables in VSDs (forward ports will be
* handled later)
*/
if (!xive_set_local_tables(x))
goto fail;
/* Register built-in source controllers (aka IPIs) */
flags = XIVE_SRC_EOI_PAGE1 | XIVE_SRC_TRIGGER_PAGE;
if (XIVE_CAN_STORE_EOI(x))
flags |= XIVE_SRC_STORE_EOI;
__xive_register_source(x, &x->ipis, x->int_base,
x->int_hw_bot - x->int_base, XIVE_ESB_SHIFT,
x->esb_base, flags, true, NULL, NULL);
/* Register escalation sources (ENDs)
*
* The ESe PQ bits are used for coalescing and the END ESB for
* interrupt management. The word 4&5 of the END is the EAS
* for the escalation source and the indexing is the same as
* the END.
*
* This is an OPAL primary source, IPIs are secondary.
*/
__xive_register_source(x, &x->esc_irqs,
MAKE_ESCALATION_GIRQ(x->block_id, 0),
XIVE_END_COUNT, XIVE_END_SHIFT,
x->end_base, XIVE_SRC_EOI_PAGE1,
false, NULL, NULL);
return x;
fail:
xive_err(x, "Initialization failed...\n");
/* Should this be fatal ? */
//assert(false);
return NULL;
}
static void xive_reset_enable_thread(struct cpu_thread *c)
{
struct proc_chip *chip = get_chip(c->chip_id);
struct xive *x = chip->xive;
uint32_t fc, bit;
uint64_t enable;
/* Get fused core number */
fc = (c->pir >> 3) & 0xf;
/* Get bit in register */
bit = c->pir & 0x3f;
/* Get which register to access */
if (fc < 8) {
xive_regw(x, TCTXT_EN0_RESET, PPC_BIT(bit));
xive_regw(x, TCTXT_EN0_SET, PPC_BIT(bit));
enable = xive_regr(x, TCTXT_EN0);
if (!(enable & PPC_BIT(bit)))
xive_cpu_err(c, "Failed to enable thread\n");
} else {
xive_regw(x, TCTXT_EN1_RESET, PPC_BIT(bit));
xive_regw(x, TCTXT_EN1_SET, PPC_BIT(bit));
enable = xive_regr(x, TCTXT_EN1);
if (!(enable & PPC_BIT(bit)))
xive_cpu_err(c, "Failed to enable thread\n");
}
}
void xive2_cpu_callin(struct cpu_thread *cpu)
{
struct xive_cpu_state *xs = cpu->xstate;
uint8_t old_w2 __unused, w2 __unused;
if (!xs)
return;
/* Reset the HW thread context and enable it */
xive_reset_enable_thread(cpu);
/* Set VT to 1 */
old_w2 = in_8(xs->tm_ring1 + TM_QW3_HV_PHYS + TM_WORD2);
out_8(xs->tm_ring1 + TM_QW3_HV_PHYS + TM_WORD2, 0x80);
w2 = in_8(xs->tm_ring1 + TM_QW3_HV_PHYS + TM_WORD2);
xive_cpu_vdbg(cpu, "Initialized TIMA VP=%x/%x W01=%016llx W2=%02x->%02x\n",
xs->vp_blk, xs->vp_idx,
in_be64(xs->tm_ring1 + TM_QW3_HV_PHYS),
old_w2, w2);
}
#ifdef XIVE_EXTRA_CHECK_INIT_CACHE
#define CHECK_INIT_CACHE_LOOP 0x100
static void xive_special_cache_check(struct xive *x, uint32_t blk, uint32_t idx)
{
struct xive_nvp vp = {0};
uint32_t i;
/*
* SIMICS checks the value of reserved fields
*/
if (chip_quirk(QUIRK_SIMICS))
return;
for (i = 0; i < CHECK_INIT_CACHE_LOOP; i++) {
struct xive_nvp *vp_m = xive_get_vp(x, idx);
memset(vp_m, (~i) & 0xff, sizeof(*vp_m));
sync();
vp.w1 = (i << 16) | i;
assert(!xive_nxc_cache_update(x, blk, idx, &vp, true));
if (!xive_check_nxc_update(x, idx, &vp)) {
xive_dbg(x, "NXC update test failed at %d iterations\n", i);
return;
}
}
xive_dbg(x, "NXC update test passed for %d/0x%x\n", blk, idx);
}
#else
static inline void xive_special_cache_check(struct xive *x __unused,
uint32_t blk __unused,
uint32_t idx __unused)
{
}
#endif
static void xive_init_cpu_exploitation(struct xive_cpu_state *xs)
{
struct xive_end end;
struct xive_nvp vp;
struct xive *x_vp, *x_end;
int i;
/* Grab the XIVE where the VP resides. It could be different from
* the local chip XIVE if not using block group mode
*/
x_vp = xive_from_pc_blk(xs->vp_blk);
assert(x_vp);
/* Grab the XIVE where the END resides. It should be the same
* as the VP.
*/
x_end = xive_from_vc_blk(xs->end_blk);
assert(x_end);
xive_init_hw_end(&end);
/* Use the cache watch to update all ENDs reserved for HW VPs */
lock(&x_end->lock);
for (i = 0; i < xive_cfg_vp_prio(x_end); i++)
xive_endc_cache_update(x_end, xs->end_blk, xs->end_idx + i,
&end, true);
unlock(&x_end->lock);
/* Initialize/enable the VP */
xive_init_default_vp(&vp, xs->end_blk, xs->end_idx);
/* Use the cache watch to write it out */
lock(&x_vp->lock);
xive_special_cache_check(x_vp, xs->vp_blk, xs->vp_idx);
xive_nxc_cache_update(x_vp, xs->vp_blk, xs->vp_idx, &vp, true);
unlock(&x_vp->lock);
}
static void xive_configure_ex_special_bar(struct xive *x, struct cpu_thread *c)
{
uint64_t xa, val;
int64_t rc;
xive_cpu_vdbg(c, "Setting up special BAR\n");
xa = XSCOM_ADDR_P10_NCU(pir_to_core_id(c->pir), P10_NCU_SPEC_BAR);
val = (uint64_t)x->tm_base | P10_NCU_SPEC_BAR_ENABLE;
if (x->tm_shift == 16)
val |= P10_NCU_SPEC_BAR_256K;
xive_cpu_vdbg(c, "NCU_SPEC_BAR_XA[%08llx]=%016llx\n", xa, val);
rc = xscom_write(c->chip_id, xa, val);
if (rc) {
xive_cpu_err(c, "Failed to setup NCU_SPEC_BAR\n");
/* XXXX what do do now ? */
}
}
void xive2_late_init(void)
{
struct cpu_thread *c;
prlog(PR_INFO, "SLW: Configuring self-restore for NCU_SPEC_BAR\n");
for_each_present_cpu(c) {
if(cpu_is_thread0(c)) {
struct proc_chip *chip = get_chip(c->chip_id);
struct xive *x = chip->xive;
uint64_t xa, val, rc;
xa = XSCOM_ADDR_P10_NCU(pir_to_core_id(c->pir), P10_NCU_SPEC_BAR);
val = (uint64_t)x->tm_base | P10_NCU_SPEC_BAR_ENABLE;
/* Bail out if wakeup engine has already failed */
if (wakeup_engine_state != WAKEUP_ENGINE_PRESENT) {
prlog(PR_ERR, "XIVE proc_stop_api fail detected\n");
break;
}
rc = proc_stop_save_scom((void *)chip->homer_base, xa, val,
PROC_STOP_SCOM_REPLACE, PROC_STOP_SECTION_L3);
if (rc) {
xive_cpu_err(c, "proc_stop_save_scom failed for NCU_SPEC_BAR rc=%lld\n",
rc);
wakeup_engine_state = WAKEUP_ENGINE_FAILED;
}
}
}
}
static void xive_provision_cpu(struct xive_cpu_state *xs, struct cpu_thread *c)
{
struct xive *x;
/* VP ids for HW threads are pre-allocated */
xs->vp_blk = PIR2VP_BLK(c->pir);
xs->vp_idx = PIR2VP_IDX(c->pir);
/* For now we use identical block IDs for VC and PC but that might
* change. We allocate the ENDs on the same XIVE as the VP.
*/
xs->end_blk = xs->vp_blk;
/* Grab the XIVE where the END resides. It could be different from
* the local chip XIVE if not using block group mode
*/
x = xive_from_vc_blk(xs->end_blk);
assert(x);
/* Allocate a set of ENDs for that VP */
xs->end_idx = xive_alloc_end_set(x, true);
assert(!XIVE_ALLOC_IS_ERR(xs->end_idx));
}
static void xive_init_cpu(struct cpu_thread *c)
{
struct proc_chip *chip = get_chip(c->chip_id);
struct xive *x = chip->xive;
struct xive_cpu_state *xs;
if (!x)
return;
/*
* Each core pair (EX) needs this special BAR setup to have the
* right powerbus cycle for the TM area (as it has the same address
* on all chips so it's somewhat special).
*
* Because we don't want to bother trying to figure out which core
* of a pair is present we just do the setup for each of them, which
* is harmless.
*/
if (cpu_is_thread0(c) || cpu_is_core_chiplet_primary(c))
xive_configure_ex_special_bar(x, c);
/* Initialize the state structure */
c->xstate = xs = local_alloc(c->chip_id, sizeof(struct xive_cpu_state), 1);
assert(xs);
memset(xs, 0, sizeof(struct xive_cpu_state));
xs->xive = x;
init_lock(&xs->lock);
/* Shortcut to TM HV ring */
xs->tm_ring1 = x->tm_base + (1u << x->tm_shift);
/* Provision a VP id and some ENDs for a HW thread */
xive_provision_cpu(xs, c);
xive_init_cpu_exploitation(xs);
}
static uint64_t xive_convert_irq_flags(uint64_t iflags)
{
uint64_t oflags = 0;
if (iflags & XIVE_SRC_STORE_EOI)
oflags |= OPAL_XIVE_IRQ_STORE_EOI2;
/* OPAL_XIVE_IRQ_TRIGGER_PAGE is only meant to be set if
* the interrupt has a *separate* trigger page.
*/
if ((iflags & XIVE_SRC_EOI_PAGE1) &&
(iflags & XIVE_SRC_TRIGGER_PAGE))
oflags |= OPAL_XIVE_IRQ_TRIGGER_PAGE;
if (iflags & XIVE_SRC_LSI)
oflags |= OPAL_XIVE_IRQ_LSI;
return oflags;
}
static int64_t opal_xive_get_irq_info(uint32_t girq,
beint64_t *out_flags,
beint64_t *out_eoi_page,
beint64_t *out_trig_page,
beint32_t *out_esb_shift,
beint32_t *out_src_chip)
{
struct irq_source *is = irq_find_source(girq);
struct xive_src *s = container_of(is, struct xive_src, is);
uint32_t idx;
uint64_t mm_base;
uint64_t eoi_page = 0, trig_page = 0;
if (xive_mode != XIVE_MODE_EXPL)
return OPAL_WRONG_STATE;
if (is == NULL || out_flags == NULL)
return OPAL_PARAMETER;
assert(is->ops == &xive_irq_source_ops);
if (out_flags)
*out_flags = cpu_to_be64(xive_convert_irq_flags(s->flags));
idx = girq - s->esb_base;
if (out_esb_shift)
*out_esb_shift = cpu_to_be32(s->esb_shift);
mm_base = (uint64_t)s->esb_mmio + (1ull << s->esb_shift) * idx;
/* The EOI page can either be the first or second page */
if (s->flags & XIVE_SRC_EOI_PAGE1) {
uint64_t p1off = 1ull << (s->esb_shift - 1);
eoi_page = mm_base + p1off;
} else
eoi_page = mm_base;
/* The trigger page, if it exists, is always the first page */
if (s->flags & XIVE_SRC_TRIGGER_PAGE)
trig_page = mm_base;
if (out_eoi_page)
*out_eoi_page = cpu_to_be64(eoi_page);
if (out_trig_page)
*out_trig_page = cpu_to_be64(trig_page);
if (out_src_chip)
*out_src_chip = cpu_to_be32(GIRQ_TO_CHIP(girq));
return OPAL_SUCCESS;
}
static int64_t opal_xive_get_irq_config(uint32_t girq,
beint64_t *out_vp,
uint8_t *out_prio,
beint32_t *out_lirq)
{
uint32_t vp;
uint32_t lirq;
uint8_t prio;
if (xive_mode != XIVE_MODE_EXPL)
return OPAL_WRONG_STATE;
if (xive_get_irq_targetting(girq, &vp, &prio, &lirq)) {
*out_vp = cpu_to_be64(vp);
*out_prio = prio;
*out_lirq = cpu_to_be32(lirq);
return OPAL_SUCCESS;
} else
return OPAL_PARAMETER;
}
static int64_t opal_xive_set_irq_config(uint32_t girq,
uint64_t vp,
uint8_t prio,
uint32_t lirq)
{
/*
* This variant is meant for a XIVE-aware OS, thus it will
* *not* affect the ESB state of the interrupt. If used with
* a prio of FF, the EAS will be masked. In that case the
* races have to be handled by the OS.
*/
if (xive_mode != XIVE_MODE_EXPL)
return OPAL_WRONG_STATE;
return xive_set_irq_config(girq, vp, prio, lirq, false);
}
static int64_t opal_xive_get_queue_info(uint64_t vp, uint32_t prio,
beint64_t *out_qpage,
beint64_t *out_qsize,
beint64_t *out_qeoi_page,
beint32_t *out_escalate_irq,
beint64_t *out_qflags)
{
uint32_t blk, idx;
struct xive *x;
struct xive_end *end;
if (xive_mode != XIVE_MODE_EXPL)
return OPAL_WRONG_STATE;
if (!xive_end_for_target(vp, prio, &blk, &idx))
return OPAL_PARAMETER;
x = xive_from_vc_blk(blk);
if (!x)
return OPAL_PARAMETER;
end = xive_get_end(x, idx);
if (!end)
return OPAL_PARAMETER;
if (out_escalate_irq) {
uint32_t esc_idx = idx;
/* If escalations are routed to a single queue, fix up
* the escalation interrupt number here.
*/
if (xive_get_field32(END_W0_UNCOND_ESCALATE, end->w0))
esc_idx |= xive_escalation_prio(x);
*out_escalate_irq =
cpu_to_be32(MAKE_ESCALATION_GIRQ(blk, esc_idx));
}
/* If this is a single-escalation gather queue, that's all
* there is to return
*/
if (xive_get_field32(END_W0_SILENT_ESCALATE, end->w0)) {
if (out_qflags)
*out_qflags = 0;
if (out_qpage)
*out_qpage = 0;
if (out_qsize)
*out_qsize = 0;
if (out_qeoi_page)
*out_qeoi_page = 0;
return OPAL_SUCCESS;
}
if (out_qpage) {
if (xive_get_field32(END_W0_ENQUEUE, end->w0))
*out_qpage = cpu_to_be64(
((uint64_t)xive_get_field32(END_W2_EQ_ADDR_HI, end->w2) << 32) |
xive_get_field32(END_W3_EQ_ADDR_LO, end->w3));
else
*out_qpage = 0;
}
if (out_qsize) {
if (xive_get_field32(END_W0_ENQUEUE, end->w0))
*out_qsize = cpu_to_be64(xive_get_field32(END_W3_QSIZE, end->w3) + 12);
else
*out_qsize = 0;
}
if (out_qeoi_page) {
*out_qeoi_page = cpu_to_be64(
(uint64_t)x->end_base + idx * XIVE_ESB_PAGE_SIZE);
}
if (out_qflags) {
*out_qflags = 0;
if (xive_get_field32(END_W0_VALID, end->w0))
*out_qflags |= cpu_to_be64(OPAL_XIVE_EQ_ENABLED);
if (xive_get_field32(END_W0_UCOND_NOTIFY, end->w0))
*out_qflags |= cpu_to_be64(OPAL_XIVE_EQ_ALWAYS_NOTIFY);
if (xive_get_field32(END_W0_ESCALATE_CTL, end->w0))
*out_qflags |= cpu_to_be64(OPAL_XIVE_EQ_ESCALATE);
}
return OPAL_SUCCESS;
}
static void xive_cleanup_end(struct xive_end *end)
{
end->w0 = xive_set_field32(END_W0_FIRMWARE1, 0, xive_end_is_firmware1(end));
end->w1 = xive_set_field32(END_W1_ESe_Q, 0, 1) |
xive_set_field32(END_W1_ESn_Q, 0, 1);
end->w2 = end->w3 = end->w4 = end->w5 = end->w6 = end->w7 = 0;
}
static int64_t opal_xive_set_queue_info(uint64_t vp, uint32_t prio,
uint64_t qpage,
uint64_t qsize,
uint64_t qflags)
{
uint32_t blk, idx;
struct xive *x;
struct xive_end *old_end;
struct xive_end end;
uint32_t vp_blk, vp_idx;
bool group;
int64_t rc;
if (!xive_end_for_target(vp, prio, &blk, &idx))
return OPAL_PARAMETER;
x = xive_from_vc_blk(blk);
if (!x)
return OPAL_PARAMETER;
old_end = xive_get_end(x, idx);
if (!old_end)
return OPAL_PARAMETER;
/* If this is a silent escalation queue, it cannot be
* configured directly
*/
if (xive_get_field32(END_W0_SILENT_ESCALATE, old_end->w0))
return OPAL_PARAMETER;
/* This shouldn't fail or xive_end_for_target would have
* failed already
*/
if (!xive_decode_vp(vp, &vp_blk, &vp_idx, NULL, &group))
return OPAL_PARAMETER;
/*
* Make a local copy which we will later try to commit using
* the cache watch facility
*/
end = *old_end;
if (qflags & OPAL_XIVE_EQ_ENABLED) {
switch(qsize) {
/* Supported sizes */
case 12:
case 16:
case 21:
case 24:
end.w3 = cpu_to_be32(qpage & END_W3_EQ_ADDR_LO);
end.w2 = cpu_to_be32((qpage >> 32) & END_W2_EQ_ADDR_HI);
end.w3 = xive_set_field32(END_W3_QSIZE, end.w3, qsize - 12);
end.w0 = xive_set_field32(END_W0_ENQUEUE, end.w0, 1);
break;
case 0:
end.w2 = end.w3 = 0;
end.w0 = xive_set_field32(END_W0_ENQUEUE, end.w0, 0);
break;
default:
return OPAL_PARAMETER;
}
/* Ensure the priority and target are correctly set (they will
* not be right after allocation
*/
end.w6 = xive_set_field32(END_W6_VP_BLOCK, 0, vp_blk) |
xive_set_field32(END_W6_VP_OFFSET, 0, vp_idx);
end.w7 = xive_set_field32(END_W7_F0_PRIORITY, 0, prio);
/* XXX Handle group i bit when needed */
/* Always notify flag */
if (qflags & OPAL_XIVE_EQ_ALWAYS_NOTIFY)
end.w0 = xive_set_field32(END_W0_UCOND_NOTIFY, end.w0, 1);
else
end.w0 = xive_set_field32(END_W0_UCOND_NOTIFY, end.w0, 0);
/* Escalation flag */
if (qflags & OPAL_XIVE_EQ_ESCALATE)
end.w0 = xive_set_field32(END_W0_ESCALATE_CTL, end.w0, 1);
else
end.w0 = xive_set_field32(END_W0_ESCALATE_CTL, end.w0, 0);
/* Unconditionally clear the current queue pointer, set
* generation to 1 and disable escalation interrupts.
*/
end.w1 = xive_set_field32(END_W1_GENERATION, 0, 1) |
xive_set_field32(END_W1_ES, 0, xive_get_field32(END_W1_ES, old_end->w1));
/* Enable. We always enable backlog for an enabled queue
* otherwise escalations won't work.
*/
end.w0 = xive_set_field32(END_W0_VALID, end.w0, 1);
end.w0 = xive_set_field32(END_W0_BACKLOG, end.w0, 1);
} else
xive_cleanup_end(&end);
/* Update END, non-synchronous */
lock(&x->lock);
rc = xive_endc_cache_update(x, blk, idx, &end, false);
unlock(&x->lock);
return rc;
}
static int64_t opal_xive_get_queue_state(uint64_t vp, uint32_t prio,
beint32_t *out_qtoggle,
beint32_t *out_qindex)
{
uint32_t blk, idx;
struct xive *x;
struct xive_end *end;
int64_t rc;
if (xive_mode != XIVE_MODE_EXPL)
return OPAL_WRONG_STATE;
if (!out_qtoggle || !out_qindex ||
!xive_end_for_target(vp, prio, &blk, &idx))
return OPAL_PARAMETER;
x = xive_from_vc_blk(blk);
if (!x)
return OPAL_PARAMETER;
end = xive_get_end(x, idx);
if (!end)
return OPAL_PARAMETER;
/* Scrub the queue */
lock(&x->lock);
rc = xive_endc_scrub(x, blk, idx);
unlock(&x->lock);
if (rc)
return rc;
/* We don't do disable queues */
if (!xive_get_field32(END_W0_VALID, end->w0))
return OPAL_WRONG_STATE;
*out_qtoggle = cpu_to_be32(xive_get_field32(END_W1_GENERATION, end->w1));
*out_qindex = cpu_to_be32(xive_get_field32(END_W1_PAGE_OFF, end->w1));
return OPAL_SUCCESS;
}
static int64_t opal_xive_set_queue_state(uint64_t vp, uint32_t prio,
uint32_t qtoggle, uint32_t qindex)
{
uint32_t blk, idx;
struct xive *x;
struct xive_end *end, new_end;
int64_t rc;
if (xive_mode != XIVE_MODE_EXPL)
return OPAL_WRONG_STATE;
if (!xive_end_for_target(vp, prio, &blk, &idx))
return OPAL_PARAMETER;
x = xive_from_vc_blk(blk);
if (!x)
return OPAL_PARAMETER;
end = xive_get_end(x, idx);
if (!end)
return OPAL_PARAMETER;
/* We don't do disable queues */
if (!xive_get_field32(END_W0_VALID, end->w0))
return OPAL_WRONG_STATE;
new_end = *end;
new_end.w1 = xive_set_field32(END_W1_GENERATION, new_end.w1, qtoggle);
new_end.w1 = xive_set_field32(END_W1_PAGE_OFF, new_end.w1, qindex);
lock(&x->lock);
rc = xive_endc_cache_update(x, blk, idx, &new_end, false);
unlock(&x->lock);
return rc;
}
static int64_t opal_xive_donate_page(uint32_t chip_id, uint64_t addr)
{
struct proc_chip *c = get_chip(chip_id);
struct list_node *n;
if (xive_mode != XIVE_MODE_EXPL)
return OPAL_WRONG_STATE;
if (!c)
return OPAL_PARAMETER;
if (!c->xive)
return OPAL_PARAMETER;
if (addr & 0xffff)
return OPAL_PARAMETER;
n = (struct list_node *)addr;
lock(&c->xive->lock);
list_add(&c->xive->donated_pages, n);
unlock(&c->xive->lock);
return OPAL_SUCCESS;
}
static int64_t opal_xive_get_vp_info(uint64_t vp_id,
beint64_t *out_flags,
beint64_t *out_cam_value,
beint64_t *out_report_cl_pair,
beint32_t *out_chip_id)
{
struct xive *x;
struct xive_nvp *vp;
uint32_t blk, idx;
bool group;
if (!xive_decode_vp(vp_id, &blk, &idx, NULL, &group))
return OPAL_PARAMETER;
/* We don't do groups yet */
if (group)
return OPAL_PARAMETER;
x = xive_from_pc_blk(blk);
if (!x)
return OPAL_PARAMETER;
vp = xive_get_vp(x, idx);
if (!vp)
return OPAL_PARAMETER;
if (out_flags) {
uint32_t end_blk, end_idx;
struct xive_end *end;
struct xive *end_x;
*out_flags = 0;
/*
* We would like to a way to stash a SW bit in the VP
* to know whether silent escalation is enabled or
* not, but unlike what happens with ENDs, the PC
* cache watch doesn't implement the reserved bit in
* the VPs... so we have to go look at END 7 instead.
*/
/* Grab END for prio 7 to check for silent escalation */
if (!xive_end_for_target(vp_id, xive_escalation_prio(x),
&end_blk, &end_idx))
return OPAL_PARAMETER;
end_x = xive_from_vc_blk(end_blk);
if (!end_x)
return OPAL_PARAMETER;
end = xive_get_end(x, end_idx);
if (!end)
return OPAL_PARAMETER;
if (xive_get_field32(NVP_W0_VALID, vp->w0))
*out_flags |= cpu_to_be64(OPAL_XIVE_VP_ENABLED);
if (xive_cfg_save_restore(x))
*out_flags |= cpu_to_be64(OPAL_XIVE_VP_SAVE_RESTORE);
if (xive_get_field32(END_W0_SILENT_ESCALATE, end->w0))
*out_flags |= cpu_to_be64(OPAL_XIVE_VP_SINGLE_ESCALATION);
}
if (out_cam_value) {
uint64_t cam_value;
cam_value = (blk << x->vp_shift) | idx;
/*
* If save-restore is enabled, force the CAM line
* value with the H bit.
*/
if (xive_cfg_save_restore(x))
cam_value |= TM10_QW1W2_HO;
*out_cam_value = cpu_to_be64(cam_value);
}
if (out_report_cl_pair) {
uint64_t report_cl_pair;
report_cl_pair = ((uint64_t)(be32_to_cpu(vp->w6) & 0x0fffffff)) << 32;
report_cl_pair |= be32_to_cpu(vp->w7) & 0xffffff00;
*out_report_cl_pair = cpu_to_be64(report_cl_pair);
}
if (out_chip_id)
*out_chip_id = cpu_to_be32(xive_block_to_chip[blk]);
return OPAL_SUCCESS;
}
static int64_t xive_setup_silent_gather(uint64_t vp_id, bool enable)
{
uint32_t blk, idx, i;
struct xive_end *end_orig;
struct xive_end end;
struct xive *x;
int64_t rc;
/* Get base END block */
if (!xive_end_for_target(vp_id, 0, &blk, &idx)) {
prlog(PR_ERR, "%s: Invalid VP 0x%08llx\n", __func__, vp_id);
return OPAL_PARAMETER;
}
x = xive_from_vc_blk(blk);
if (!x) {
prlog(PR_ERR, "%s: VP 0x%08llx has invalid block %d\n", __func__,
vp_id, blk);
return OPAL_PARAMETER;
}
/* Grab prio 7 */
end_orig = xive_get_end(x, idx + xive_escalation_prio(x));
if (!end_orig) {
xive_err(x, "Failed to get silent gather END 0x%x for VP 0x%08llx\n",
idx + xive_escalation_prio(x), vp_id);
return OPAL_PARAMETER;
}
/* If trying to enable silent gather, make sure prio 7 is not
* already enabled as a normal queue
*/
if (enable && xive_get_field32(END_W0_VALID, end_orig->w0) &&
!xive_get_field32(END_W0_SILENT_ESCALATE, end_orig->w0)) {
xive_err(x, "silent gather END 0x%x already in use\n",
idx + xive_escalation_prio(x));
return OPAL_PARAMETER;
}
end = *end_orig;
if (enable) {
/* W0: Enabled and "s" set, no other bit */
end.w0 = xive_set_field32(END_W0_FIRMWARE1, end.w0, 0);
end.w0 = xive_set_field32(END_W0_VALID, end.w0, 1);
end.w0 = xive_set_field32(END_W0_SILENT_ESCALATE, end.w0, 1);
end.w0 = xive_set_field32(END_W0_ESCALATE_CTL, end.w0, 1);
end.w0 = xive_set_field32(END_W0_BACKLOG, end.w0, 1);
/* Set new "N" for END escalation (vs. ESB) */
end.w0 = xive_set_field32(END_W0_ESCALATE_END, end.w0, 1);
/* W1: Mark ESn as 01, ESe as 00 */
end.w1 = xive_set_field32(END_W1_ESn_P, end.w1, 0);
end.w1 = xive_set_field32(END_W1_ESn_Q, end.w1, 1);
end.w1 = xive_set_field32(END_W1_ESe, end.w1, 0);
} else if (xive_get_field32(END_W0_SILENT_ESCALATE, end.w0))
xive_cleanup_end(&end);
if (!memcmp(end_orig, &end, sizeof(end)))
rc = 0;
else
rc = xive_endc_cache_update(x, blk, idx + xive_escalation_prio(x),
&end, false);
if (rc)
return rc;
/* Mark/unmark all other prios with the new "u" bit and update
* escalation
*/
for (i = 0; i < xive_cfg_vp_prio(x); i++) {
if (i == xive_escalation_prio(x))
continue;
end_orig = xive_get_end(x, idx + i);
if (!end_orig)
continue;
end = *end_orig;
if (enable) {
/* Set "u" bit */
end.w0 = xive_set_field32(END_W0_UNCOND_ESCALATE, end.w0, 1);
/* Set new "N" for END escalation (vs. ESB) */
/* TODO (Gen2+) : use ESB escalation configuration */
end.w0 = xive_set_field32(END_W0_ESCALATE_END, end.w0, 1);
/* Re-route escalation interrupt (previous
* route is lost !) to the gather queue
*/
end.w4 = xive_set_field32(END_W4_END_BLOCK, end.w4, blk);
end.w4 = xive_set_field32(END_W4_ESC_END_INDEX,
end.w4, idx + xive_escalation_prio(x));
} else if (xive_get_field32(END_W0_UNCOND_ESCALATE, end.w0)) {
/* Clear the "u" bit, disable escalations if it was set */
end.w0 = xive_set_field32(END_W0_UNCOND_ESCALATE, end.w0, 0);
end.w0 = xive_set_field32(END_W0_ESCALATE_CTL, end.w0, 0);
}
if (!memcmp(end_orig, &end, sizeof(end)))
continue;
rc = xive_endc_cache_update(x, blk, idx + i, &end, false);
if (rc)
break;
}
return rc;
}
static int64_t opal_xive_set_vp_info(uint64_t vp_id,
uint64_t flags,
uint64_t report_cl_pair)
{
struct xive *x;
struct xive_nvp *vp, vp_new;
uint32_t blk, idx;
bool group;
int64_t rc;
if (!xive_decode_vp(vp_id, &blk, &idx, NULL, &group))
return OPAL_PARAMETER;
/* We don't do groups yet */
if (group)
return OPAL_PARAMETER;
if (report_cl_pair & 0xff)
return OPAL_PARAMETER;
x = xive_from_pc_blk(blk);
if (!x)
return OPAL_PARAMETER;
vp = xive_get_vp(x, idx);
if (!vp)
return OPAL_PARAMETER;
/* Consistency check. */
if ((flags & OPAL_XIVE_VP_SAVE_RESTORE) && !xive_cfg_save_restore(x))
return OPAL_PARAMETER;
lock(&x->lock);
vp_new = *vp;
if (flags & OPAL_XIVE_VP_ENABLED) {
vp_new.w0 = xive_set_field32(NVP_W0_VALID, vp_new.w0, 1);
vp_new.w6 = cpu_to_be32(report_cl_pair >> 32);
vp_new.w7 = cpu_to_be32(report_cl_pair & 0xffffffff);
if (flags & OPAL_XIVE_VP_SINGLE_ESCALATION)
rc = xive_setup_silent_gather(vp_id, true);
else
rc = xive_setup_silent_gather(vp_id, false);
/*
* Prepare NVP to be HW owned for automatic save-restore
*/
if (xive_cfg_save_restore(x)) {
/*
* Set NVP privilege level. Default to OS.
* This check only makes sense for KVM guests
* currently. We would need an extra flag to
* distinguish from pool level.
*/
vp_new.w0 = xive_set_field32(NVP_W0_VPRIV, vp_new.w0, 0);
vp_new.w2 = xive_set_field32(NVP_W2_CPPR, vp_new.w2, 0xFF);
vp_new.w0 = xive_set_field32(NVP_W0_HW, vp_new.w0, 1);
}
} else {
/*
* TODO (kvm): disabling a VP invalidates the associated ENDs.
*
* The loads then return all 1s which can be an issue for the
* Linux code to handle.
*/
vp_new.w0 = vp_new.w6 = vp_new.w7 = 0;
rc = xive_setup_silent_gather(vp_id, false);
}
if (rc) {
if (rc != OPAL_BUSY)
xive_dbg(x, "Silent gather setup failed with err %lld\n", rc);
goto bail;
}
rc = xive_nxc_cache_update(x, blk, idx, &vp_new, false);
if (rc)
goto bail;
/* When disabling, we scrub clean (invalidate the entry) so
* we can avoid cache ops in alloc/free
*/
if (!(flags & OPAL_XIVE_VP_ENABLED))
xive_nxc_scrub_clean(x, blk, idx);
bail:
unlock(&x->lock);
return rc;
}
static int64_t opal_xive_get_vp_state(uint64_t vp_id, beint64_t *out_state)
{
struct xive *x;
struct xive_nvp *vp;
uint32_t blk, idx;
int64_t rc;
bool group;
if (!out_state || !xive_decode_vp(vp_id, &blk, &idx, NULL, &group))
return OPAL_PARAMETER;
if (group)
return OPAL_PARAMETER;
x = xive_from_pc_blk(blk);
if (!x)
return OPAL_PARAMETER;
vp = xive_get_vp(x, idx);
if (!vp)
return OPAL_PARAMETER;
/* Scrub the vp */
lock(&x->lock);
rc = xive_nxc_scrub(x, blk, idx);
unlock(&x->lock);
if (rc)
return rc;
if (!xive_get_field32(NVP_W0_VALID, vp->w0))
return OPAL_WRONG_STATE;
/*
* return a state matching the layout of WORD 0-1 of the TIMA
* as this is expected by current implementation.
*/
*out_state = cpu_to_be64(((uint64_t) 0x0) << 54 |
(uint64_t)xive_get_field32(NVP_W2_CPPR, vp->w2) << 48 |
(uint64_t)xive_get_field32(NVP_W2_IPB, vp->w2) << 40 |
(uint64_t)xive_get_field32(NVP_W2_LSMFB, vp->w2) << 32);
return OPAL_SUCCESS;
}
static void *xive_cpu_get_tima(struct cpu_thread *c)
{
struct xive_cpu_state *xs = c->xstate;
struct xive *x = xs->xive;
return x->ic_tm_direct_base + ((c->pir & 0xff) << x->ic_shift);
}
static void xive_cleanup_cpu_tima(struct cpu_thread *c)
{
struct xive_cpu_state *xs __unused = c->xstate;
void *cpu_tm_base = xive_cpu_get_tima(c);
uint8_t old_w2 __unused, w2 __unused;
/* Reset the HW context */
xive_reset_enable_thread(c);
/* Set VT to 1 */
old_w2 = in_8(cpu_tm_base + TM_QW3_HV_PHYS + TM_WORD2);
out_8(cpu_tm_base + TM_QW3_HV_PHYS + TM_WORD2, 0x80);
w2 = in_8(cpu_tm_base + TM_QW3_HV_PHYS + TM_WORD2);
/* Dump HV state */
xive_cpu_vdbg(c, "[reset] VP TIMA VP=%x/%x W01=%016llx W2=%02x->%02x\n",
xs->vp_blk, xs->vp_idx,
in_be64(cpu_tm_base + TM_QW3_HV_PHYS),
old_w2, w2);
}
static int64_t xive_vc_ind_cache_kill(struct xive *x, uint64_t type)
{
uint64_t val;
/* We clear the whole thing */
xive_regw(x, VC_AT_MACRO_KILL_MASK, 0);
xive_regw(x, VC_AT_MACRO_KILL, VC_AT_MACRO_KILL_VALID |
SETFIELD(VC_AT_MACRO_KILL_VSD, 0ull, type));
/* XXX Add timeout */
for (;;) {
val = xive_regr(x, VC_AT_MACRO_KILL);
if (!(val & VC_AT_MACRO_KILL_VALID))
break;
}
return 0;
}
static int64_t xive_pc_ind_cache_kill(struct xive *x)
{
uint64_t val;
/* We clear the whole thing */
xive_regw(x, PC_AT_KILL_MASK, 0);
xive_regw(x, PC_AT_KILL, PC_AT_KILL_VALID |
SETFIELD(VC_AT_MACRO_KILL_VSD, 0ull, VST_NVP));
/* XXX Add timeout */
for (;;) {
val = xive_regr(x, PC_AT_KILL);
if (!(val & PC_AT_KILL_VALID))
break;
}
return 0;
}
static void xive_cleanup_vp_ind(struct xive *x)
{
int i;
xive_dbg(x, "Cleaning up %d VP ind entries...\n", x->vp_ind_count);
for (i = 0; i < x->vp_ind_count; i++) {
if (be64_to_cpu(x->vp_ind_base[i]) & VSD_FIRMWARE) {
xive_dbg(x, " %04x ... skip (firmware)\n", i);
continue;
}
if (x->vp_ind_base[i] != 0) {
x->vp_ind_base[i] = 0;
xive_dbg(x, " %04x ... cleaned\n", i);
}
}
xive_pc_ind_cache_kill(x);
}
static void xive_cleanup_end_ind(struct xive *x)
{
int i;
xive_dbg(x, "Cleaning up %d END ind entries...\n", x->end_ind_count);
for (i = 0; i < x->end_ind_count; i++) {
if (be64_to_cpu(x->end_ind_base[i]) & VSD_FIRMWARE) {
xive_dbg(x, " %04x ... skip (firmware)\n", i);
continue;
}
if (x->end_ind_base[i] != 0) {
x->end_ind_base[i] = 0;
xive_dbg(x, " %04x ... cleaned\n", i);
}
}
xive_vc_ind_cache_kill(x, VST_END);
}
static void xive_reset_one(struct xive *x)
{
struct cpu_thread *c;
bool end_firmware;
int i;
xive_notice(x, "Resetting one xive...\n");
lock(&x->lock);
/* Check all interrupts are disabled */
i = bitmap_find_one_bit(*x->int_enabled_map, 0, XIVE_INT_COUNT);
if (i >= 0)
xive_warn(x, "Interrupt %d (and maybe more) not disabled"
" at reset !\n", i);
/* Reset IPI allocation */
xive_dbg(x, "freeing alloc map %p/%p\n",
x->ipi_alloc_map, *x->ipi_alloc_map);
memset(x->ipi_alloc_map, 0, BITMAP_BYTES(XIVE_INT_COUNT));
xive_dbg(x, "Resetting ENDs...\n");
/* Reset all allocated ENDs and free the user ones */
bitmap_for_each_one(*x->end_map, xive_end_bitmap_size(x), i) {
struct xive_end end0;
struct xive_end *end;
int j;
if (i == 0)
continue;
end_firmware = false;
for (j = 0; j < xive_cfg_vp_prio(x); j++) {
uint32_t idx = (i << xive_cfg_vp_prio_shift(x)) | j;
end = xive_get_end(x, idx);
if (!end)
continue;
/* We need to preserve the firmware bit, otherwise
* we will incorrectly free the ENDs that are reserved
* for the physical CPUs
*/
if (xive_get_field32(END_W0_VALID, end->w0)) {
if (!xive_end_is_firmware1(end))
xive_dbg(x, "END 0x%x:0x%x is valid at reset: %08x %08x\n",
x->block_id, idx, end->w0, end->w1);
end0 = *end;
xive_cleanup_end(&end0);
xive_endc_cache_update(x, x->block_id, idx, &end0, true);
}
if (xive_end_is_firmware1(end))
end_firmware = true;
}
if (!end_firmware)
bitmap_clr_bit(*x->end_map, i);
}
/* Take out all VPs from HW and reset all CPPRs to 0 */
for_each_present_cpu(c) {
if (c->chip_id != x->chip_id)
continue;
if (!c->xstate)
continue;
xive_cleanup_cpu_tima(c);
}
/* Reset all user-allocated VPs. This is inefficient, we should
* either keep a bitmap of allocated VPs or add an iterator to
* the buddy which is trickier but doable.
*/
for (i = 0; i < XIVE_VP_COUNT(x); i++) {
struct xive_nvp *vp;
struct xive_nvp vp0 = {0};
/* Ignore the physical CPU VPs */
if (i >= xive_hw_vp_count &&
i < (xive_hw_vp_base + xive_hw_vp_count))
continue;
/* Is the VP valid ? */
vp = xive_get_vp(x, i);
if (!vp || !xive_get_field32(NVP_W0_VALID, vp->w0))
continue;
/* Clear it */
xive_dbg(x, "VP 0x%x:0x%x is valid at reset\n", x->block_id, i);
xive_nxc_cache_update(x, x->block_id, i, &vp0, true);
}
/* Forget about remaining donated pages */
list_head_init(&x->donated_pages);
/* And cleanup donated indirect VP and END pages */
xive_cleanup_vp_ind(x);
xive_cleanup_end_ind(x);
/* The rest must not be called with the lock held */
unlock(&x->lock);
/* Re-configure VPs */
for_each_present_cpu(c) {
struct xive_cpu_state *xs = c->xstate;
if (c->chip_id != x->chip_id || !xs)
continue;
xive_init_cpu_exploitation(xs);
}
}
static void xive_reset_mask_source_cb(struct irq_source *is,
void *data __unused)
{
struct xive_src *s = container_of(is, struct xive_src, is);
struct xive *x;
uint32_t isn;
if (is->ops != &xive_irq_source_ops)
return;
/* Skip escalation sources */
if (GIRQ_IS_ESCALATION(is->start))
return;
x = s->xive;
/* Iterate all interrupts */
for (isn = is->start; isn < is->end; isn++) {
/* Has it ever been enabled ? */
if (!bitmap_tst_bit(*x->int_enabled_map, GIRQ_TO_IDX(isn)))
continue;
/* Mask it and clear the enabled map bit */
xive_vdbg(x, "[reset] disabling source 0x%x\n", isn);
__xive_set_irq_config(is, isn, 0, 0xff, isn, true, false);
bitmap_clr_bit(*x->int_enabled_map, GIRQ_TO_IDX(isn));
}
}
void xive2_cpu_reset(void)
{
struct cpu_thread *c = this_cpu();
struct xive_cpu_state *xs = c->xstate;
out_8(xs->tm_ring1 + TM_QW3_HV_PHYS + TM_CPPR, 0);
in_be64(xs->tm_ring1 + TM_SPC_PULL_POOL_CTX);
}
static int64_t __xive_reset(uint64_t mode)
{
struct proc_chip *chip;
xive_mode = mode;
/* Mask all interrupt sources */
irq_for_each_source(xive_reset_mask_source_cb, NULL);
/* For each XIVE do a sync... */
for_each_chip(chip) {
if (!chip->xive)
continue;
xive_sync(chip->xive);
}
/* For each XIVE reset everything else... */
for_each_chip(chip) {
if (!chip->xive)
continue;
xive_reset_one(chip->xive);
}
/* Cleanup global VP allocator */
buddy_reset(xive_vp_buddy);
/*
* We reserve the whole range of VP ids for HW threads.
*/
assert(buddy_reserve(xive_vp_buddy, xive_hw_vp_base, xive_threadid_shift));
return OPAL_SUCCESS;
}
/* Called by fast reboot */
int64_t xive2_reset(void)
{
if (xive_mode == XIVE_MODE_NONE)
return OPAL_SUCCESS;
return __xive_reset(XIVE_MODE_EXPL);
}
static int64_t opal_xive_reset(uint64_t mode)
{
prlog(PR_DEBUG, "XIVE reset. mode = %llx\n", mode);
if (!(mode & XIVE_MODE_EXPL)) {
prlog(PR_NOTICE, "No emulation mode. XIVE exploitation mode "
"is the default\n");
}
xive_expl_options = mode & ~XIVE_MODE_EXPL;
if (xive_expl_options & ~XIVE_EXPL_ALL_OPTIONS) {
prerror("invalid XIVE exploitation mode option %016llx\n",
xive_expl_options);
return OPAL_PARAMETER;
}
return __xive_reset(XIVE_MODE_EXPL);
}
static int64_t opal_xive_free_vp_block(uint64_t vp_base)
{
uint32_t blk, idx, i, j, count;
uint8_t order;
bool group;
if (xive_mode != XIVE_MODE_EXPL)
return OPAL_WRONG_STATE;
if (!xive_decode_vp(vp_base, &blk, &idx, &order, &group))
return OPAL_PARAMETER;
if (group)
return OPAL_PARAMETER;
if (blk)
return OPAL_PARAMETER;
if (order < (xive_chips_alloc_bits + 1))
return OPAL_PARAMETER;
if (idx & ((1 << (order - xive_chips_alloc_bits)) - 1))
return OPAL_PARAMETER;
count = 1 << order;
for (i = 0; i < count; i++) {
uint32_t vp_id = vp_base + i;
uint32_t blk, idx, end_blk, end_idx;
struct xive *x;
struct xive_nvp *vp;
if (!xive_decode_vp(vp_id, &blk, &idx, NULL, NULL)) {
prerror("Couldn't decode VP id %u\n", vp_id);
return OPAL_INTERNAL_ERROR;
}
x = xive_from_pc_blk(blk);
if (!x) {
prerror("Instance not found for deallocated VP"
" block %d\n", blk);
return OPAL_INTERNAL_ERROR;
}
vp = xive_get_vp(x, idx);
if (!vp) {
prerror("VP not found for deallocation !");
return OPAL_INTERNAL_ERROR;
}
/* VP must be disabled */
if (xive_get_field32(NVP_W0_VALID, vp->w0)) {
prlog(PR_ERR, "freeing active VP %d\n", vp_id);
return OPAL_XIVE_FREE_ACTIVE;
}
/* Not populated */
if (vp->w5 == 0)
continue;
end_blk = xive_get_field32(NVP_W5_VP_END_BLOCK, vp->w5);
end_idx = xive_get_field32(NVP_W5_VP_END_INDEX, vp->w5);
lock(&x->lock);
/* Ensure ENDs are disabled and cleaned up. Ideally the caller
* should have done it but we double check it here
*/
for (j = 0; j < xive_cfg_vp_prio(x); j++) {
struct xive *end_x = xive_from_vc_blk(end_blk);
struct xive_end end, *orig_end = xive_get_end(end_x, end_idx + j);
if (!xive_get_field32(END_W0_VALID, orig_end->w0))
continue;
prlog(PR_WARNING, "freeing VP %d with queue %d active\n",
vp_id, j);
end = *orig_end;
xive_cleanup_end(&end);
xive_endc_cache_update(x, end_blk, end_idx + j, &end, true);
}
/* Mark it not populated so we don't try to free it again */
vp->w5 = 0;
if (end_blk != blk) {
prerror("Block mismatch trying to free ENDs\n");
unlock(&x->lock);
return OPAL_INTERNAL_ERROR;
}
xive_free_end_set(x, end_idx);
unlock(&x->lock);
}
xive_free_vps(vp_base);
return OPAL_SUCCESS;
}
static int64_t opal_xive_alloc_vp_block(uint32_t alloc_order)
{
uint32_t vp_base, ends, count, i;
int64_t rc;
if (xive_mode != XIVE_MODE_EXPL)
return OPAL_WRONG_STATE;
prlog(PR_TRACE, "opal_xive_alloc_vp_block(%d)\n", alloc_order);
vp_base = xive_alloc_vps(alloc_order);
if (XIVE_ALLOC_IS_ERR(vp_base)) {
if (vp_base == XIVE_ALLOC_NO_IND)
return OPAL_XIVE_PROVISIONING;
return OPAL_RESOURCE;
}
/* Allocate ENDs and initialize VPs */
count = 1 << alloc_order;
for (i = 0; i < count; i++) {
uint32_t vp_id = vp_base + i;
uint32_t blk, idx;
struct xive *x;
struct xive_nvp *vp;
if (!xive_decode_vp(vp_id, &blk, &idx, NULL, NULL)) {
prerror("Couldn't decode VP id %u\n", vp_id);
return OPAL_INTERNAL_ERROR;
}
x = xive_from_pc_blk(blk);
if (!x) {
prerror("Instance not found for allocated VP"
" block %d\n", blk);
rc = OPAL_INTERNAL_ERROR;
goto fail;
}
vp = xive_get_vp(x, idx);
if (!vp) {
prerror("VP not found after allocation !");
rc = OPAL_INTERNAL_ERROR;
goto fail;
}
/* Allocate ENDs, if fails, free the VPs and return */
lock(&x->lock);
ends = xive_alloc_end_set(x, false);
unlock(&x->lock);
if (XIVE_ALLOC_IS_ERR(ends)) {
if (ends == XIVE_ALLOC_NO_IND)
rc = OPAL_XIVE_PROVISIONING;
else
rc = OPAL_RESOURCE;
goto fail;
}
/* Initialize the VP structure. We don't use a cache watch
* as we have made sure when freeing the entries to scrub
* it out of the cache.
*/
memset(vp, 0, sizeof(*vp));
/* Store the END base of the VP in W5 (new in p10) */
xive_vp_set_end_base(vp, blk, ends);
}
return vp_base;
fail:
opal_xive_free_vp_block(vp_base);
return rc;
}
static int64_t xive_try_allocate_irq(struct xive *x)
{
int idx, base_idx, max_count, girq;
struct xive_eas *eas;
lock(&x->lock);
base_idx = x->int_ipi_top - x->int_base;
max_count = x->int_hw_bot - x->int_ipi_top;
idx = bitmap_find_zero_bit(*x->ipi_alloc_map, base_idx, max_count);
if (idx < 0) {
unlock(&x->lock);
return OPAL_RESOURCE;
}
bitmap_set_bit(*x->ipi_alloc_map, idx);
girq = x->int_base + idx;
/* Mark the EAS valid. Don't bother with the HW cache, it's
* still masked anyway, the cache will be updated when unmasked
* and configured.
*/
eas = xive_get_eas(x, girq);
if (!eas) {
bitmap_clr_bit(*x->ipi_alloc_map, idx);
unlock(&x->lock);
return OPAL_PARAMETER;
}
eas->w = xive_set_field64(EAS_VALID, 0, 1) |
xive_set_field64(EAS_MASKED, 0, 1) |
xive_set_field64(EAS_END_DATA, 0, girq);
unlock(&x->lock);
return girq;
}
static int64_t opal_xive_allocate_irq(uint32_t chip_id)
{
struct proc_chip *chip;
bool try_all = false;
int64_t rc;
if (xive_mode != XIVE_MODE_EXPL)
return OPAL_WRONG_STATE;
if (chip_id == OPAL_XIVE_ANY_CHIP) {
try_all = true;
chip_id = this_cpu()->chip_id;
}
chip = get_chip(chip_id);
if (!chip)
return OPAL_PARAMETER;
/* Try initial target chip */
if (!chip->xive)
rc = OPAL_PARAMETER;
else
rc = xive_try_allocate_irq(chip->xive);
if (rc >= 0 || !try_all)
return rc;
/* Failed and we try all... do so */
for_each_chip(chip) {
if (!chip->xive)
continue;
rc = xive_try_allocate_irq(chip->xive);
if (rc >= 0)
break;
}
return rc;
}
static int64_t opal_xive_free_irq(uint32_t girq)
{
struct irq_source *is = irq_find_source(girq);
struct xive_src *s = container_of(is, struct xive_src, is);
struct xive *x = xive_from_isn(girq);
struct xive_eas *eas;
uint32_t idx;
if (xive_mode != XIVE_MODE_EXPL)
return OPAL_WRONG_STATE;
if (!x || !is)
return OPAL_PARAMETER;
idx = GIRQ_TO_IDX(girq);
lock(&x->lock);
eas = xive_get_eas(x, girq);
if (!eas) {
unlock(&x->lock);
return OPAL_PARAMETER;
}
/* Mask the interrupt source */
xive_update_irq_mask(s, girq - s->esb_base, true);
/* Mark the EAS masked and invalid */
eas->w = xive_set_field64(EAS_VALID, 0, 1) |
xive_set_field64(EAS_MASKED, 0, 1);
xive_easc_scrub(x, x->block_id, idx);
/* Free it */
if (!bitmap_tst_bit(*x->ipi_alloc_map, idx)) {
unlock(&x->lock);
return OPAL_PARAMETER;
}
bitmap_clr_bit(*x->ipi_alloc_map, idx);
bitmap_clr_bit(*x->int_enabled_map, idx);
unlock(&x->lock);
return OPAL_SUCCESS;
}
static int64_t opal_xive_dump_tm(uint32_t offset, const char *n, uint32_t pir)
{
struct cpu_thread *c = find_cpu_by_pir(pir);
struct xive_cpu_state *xs;
struct xive *x;
void *cpu_tm_base;
uint64_t v0,v1;
if (!c)
return OPAL_PARAMETER;
xs = c->xstate;
if (!xs || !xs->tm_ring1)
return OPAL_INTERNAL_ERROR;
x = xs->xive;
cpu_tm_base = xive_cpu_get_tima(c);
lock(&x->lock);
v0 = in_be64(cpu_tm_base + offset);
if (offset == TM_QW3_HV_PHYS) {
v1 = in_8(cpu_tm_base + offset + 8);
v1 <<= 56;
} else {
v1 = in_be32(cpu_tm_base + offset + 8);
v1 <<= 32;
}
prlog(PR_INFO, "CPU[%04x]: TM state for QW %s\n", pir, n);
prlog(PR_INFO, "CPU[%04x]: NSR CPPR IPB LSMFB ACK# INC AGE PIPR"
" W2 W3\n", pir);
prlog(PR_INFO, "CPU[%04x]: %02x %02x %02x %02x %02x "
"%02x %02x %02x %08x %08x\n", pir,
(uint8_t)(v0 >> 58) & 0xff, (uint8_t)(v0 >> 48) & 0xff,
(uint8_t)(v0 >> 40) & 0xff, (uint8_t)(v0 >> 32) & 0xff,
(uint8_t)(v0 >> 24) & 0xff, (uint8_t)(v0 >> 16) & 0xff,
(uint8_t)(v0 >> 8) & 0xff, (uint8_t)(v0 ) & 0xff,
(uint32_t)(v1 >> 32) & 0xffffffff,
(uint32_t)(v1 & 0xffffffff));
unlock(&x->lock);
return OPAL_SUCCESS;
}
static int64_t opal_xive_dump_vp(uint32_t vp_id)
{
uint32_t blk, idx;
uint8_t order;
bool group;
struct xive *x;
struct xive_nvp *vp;
uint32_t *vpw;
if (!xive_decode_vp(vp_id, &blk, &idx, &order, &group))
return OPAL_PARAMETER;
x = xive_from_vc_blk(blk);
if (!x)
return OPAL_PARAMETER;
vp = xive_get_vp(x, idx);
if (!vp)
return OPAL_PARAMETER;
lock(&x->lock);
xive_nxc_scrub_clean(x, blk, idx);
vpw = ((uint32_t *)vp) + (group ? 8 : 0);
prlog(PR_INFO, "VP[%08x]: 0..3: %08x %08x %08x %08x\n", vp_id,
vpw[0], vpw[1], vpw[2], vpw[3]);
prlog(PR_INFO, "VP[%08x]: 4..7: %08x %08x %08x %08x\n", vp_id,
vpw[4], vpw[5], vpw[6], vpw[7]);
unlock(&x->lock);
return OPAL_SUCCESS;
}
static int64_t opal_xive_sync_irq_src(uint32_t girq)
{
struct xive *x = xive_from_isn(girq);
if (!x)
return OPAL_PARAMETER;
return xive_sync(x);
}
static int64_t opal_xive_sync_irq_target(uint32_t girq)
{
uint32_t target, vp_blk;
struct xive *x;
if (!xive_get_irq_targetting(girq, &target, NULL, NULL))
return OPAL_PARAMETER;
if (!xive_decode_vp(target, &vp_blk, NULL, NULL, NULL))
return OPAL_PARAMETER;
x = xive_from_pc_blk(vp_blk);
if (!x)
return OPAL_PARAMETER;
return xive_sync(x);
}
static int64_t opal_xive_sync(uint32_t type, uint32_t id)
{
int64_t rc = OPAL_SUCCESS;;
if (type & XIVE_SYNC_EAS)
rc = opal_xive_sync_irq_src(id);
if (rc)
return rc;
if (type & XIVE_SYNC_QUEUE)
rc = opal_xive_sync_irq_target(id);
if (rc)
return rc;
/* Add more ... */
return rc;
}
static int64_t opal_xive_dump(uint32_t type, uint32_t id)
{
switch (type) {
case XIVE_DUMP_TM_HYP:
return opal_xive_dump_tm(TM_QW3_HV_PHYS, "PHYS", id);
case XIVE_DUMP_TM_POOL:
return opal_xive_dump_tm(TM_QW2_HV_POOL, "POOL", id);
case XIVE_DUMP_TM_OS:
return opal_xive_dump_tm(TM_QW1_OS, "OS ", id);
case XIVE_DUMP_TM_USER:
return opal_xive_dump_tm(TM_QW0_USER, "USER", id);
case XIVE_DUMP_VP:
return opal_xive_dump_vp(id);
default:
return OPAL_PARAMETER;
}
}
static void xive_init_globals(void)
{
uint32_t i;
for (i = 0; i < XIVE_MAX_CHIPS; i++)
xive_block_to_chip[i] = XIVE_INVALID_CHIP;
}
/*
* The global availability of some capabilities used in other drivers
* (PHB, PSI) is deduced from the capabilities of the first XIVE chip
* of the system. It should be common to all chips.
*/
bool xive2_cap_phb_pq_disable(void)
{
return xive_has_cap(one_xive, CQ_XIVE_CAP_PHB_PQ_DISABLE);
}
bool xive2_cap_phb_abt(void)
{
if (!xive_has_cap(one_xive, CQ_XIVE_CAP_PHB_ABT))
return false;
/*
* We need 'PQ disable' to use ABT mode, else the OS will use
* two different sets of ESB pages (PHB and IC) to control the
* interrupt sources. Can not work.
*/
if (!xive2_cap_phb_pq_disable()) {
prlog_once(PR_ERR, "ABT mode is set without PQ disable. "
"Ignoring bogus configuration\n");
return false;
}
return true;
}
bool xive2_cap_store_eoi(void)
{
return xive_has_cap(one_xive, CQ_XIVE_CAP_STORE_EOI);
}
void xive2_init(void)
{
struct dt_node *np;
struct proc_chip *chip;
struct cpu_thread *cpu;
bool first = true;
/* Look for xive nodes and do basic inits */
dt_for_each_compatible(dt_root, np, "ibm,power10-xive-x") {
struct xive *x;
/* Initialize some global stuff */
if (first)
xive_init_globals();
/* Create/initialize the xive instance */
x = init_one_xive(np);
if (first)
one_xive = x;
first = false;
}
if (first)
return;
/*
* P8 emulation is not supported on P10 anymore. Exploitation
* is the default XIVE mode. We might introduce a GEN2 mode.
*/
xive_mode = XIVE_MODE_EXPL;
/* Init VP allocator */
xive_init_vp_allocator();
/* Create a device-tree node for Linux use */
xive_create_mmio_dt_node(one_xive);
/* Some inits must be done after all xive have been created
* such as setting up the forwarding ports
*/
for_each_chip(chip) {
if (chip->xive)
late_init_one_xive(chip->xive);
}
/* Initialize per-cpu structures */
for_each_present_cpu(cpu) {
xive_init_cpu(cpu);
}
/* Calling boot CPU */
xive2_cpu_callin(this_cpu());
/* Register XIVE exploitation calls */
opal_register(OPAL_XIVE_RESET, opal_xive_reset, 1);
opal_register(OPAL_XIVE_GET_IRQ_INFO, opal_xive_get_irq_info, 6);
opal_register(OPAL_XIVE_GET_IRQ_CONFIG, opal_xive_get_irq_config, 4);
opal_register(OPAL_XIVE_SET_IRQ_CONFIG, opal_xive_set_irq_config, 4);
opal_register(OPAL_XIVE_GET_QUEUE_INFO, opal_xive_get_queue_info, 7);
opal_register(OPAL_XIVE_SET_QUEUE_INFO, opal_xive_set_queue_info, 5);
opal_register(OPAL_XIVE_DONATE_PAGE, opal_xive_donate_page, 2);
opal_register(OPAL_XIVE_ALLOCATE_IRQ, opal_xive_allocate_irq, 1);
opal_register(OPAL_XIVE_FREE_IRQ, opal_xive_free_irq, 1);
opal_register(OPAL_XIVE_ALLOCATE_VP_BLOCK, opal_xive_alloc_vp_block, 1);
opal_register(OPAL_XIVE_FREE_VP_BLOCK, opal_xive_free_vp_block, 1);
opal_register(OPAL_XIVE_GET_VP_INFO, opal_xive_get_vp_info, 5);
opal_register(OPAL_XIVE_SET_VP_INFO, opal_xive_set_vp_info, 3);
opal_register(OPAL_XIVE_SYNC, opal_xive_sync, 2);
opal_register(OPAL_XIVE_DUMP, opal_xive_dump, 2);
opal_register(OPAL_XIVE_GET_QUEUE_STATE, opal_xive_get_queue_state, 4);
opal_register(OPAL_XIVE_SET_QUEUE_STATE, opal_xive_set_queue_state, 4);
opal_register(OPAL_XIVE_GET_VP_STATE, opal_xive_get_vp_state, 2);
}