summaryrefslogtreecommitdiff
path: root/drivers/misc
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/misc')
-rw-r--r--drivers/misc/bcm-vk/bcm_vk_msg.c6
-rw-r--r--drivers/misc/bcm-vk/bcm_vk_msg.h2
-rw-r--r--drivers/misc/cardreader/alcor_pci.c8
-rw-r--r--drivers/misc/cxl/file.c5
-rw-r--r--drivers/misc/eeprom/Kconfig5
-rw-r--r--drivers/misc/eeprom/at25.c158
-rw-r--r--drivers/misc/eeprom/ee1004.c223
-rw-r--r--drivers/misc/eeprom/eeprom_93xx46.c90
-rw-r--r--drivers/misc/eeprom/idt_89hpesx.c41
-rw-r--r--drivers/misc/habanalabs/common/command_submission.c81
-rw-r--r--drivers/misc/habanalabs/common/context.c9
-rw-r--r--drivers/misc/habanalabs/common/debugfs.c5
-rw-r--r--drivers/misc/habanalabs/common/device.c82
-rw-r--r--drivers/misc/habanalabs/common/firmware_if.c1792
-rw-r--r--drivers/misc/habanalabs/common/habanalabs.h280
-rw-r--r--drivers/misc/habanalabs/common/habanalabs_drv.c24
-rw-r--r--drivers/misc/habanalabs/common/habanalabs_ioctl.c23
-rw-r--r--drivers/misc/habanalabs/common/hw_queue.c42
-rw-r--r--drivers/misc/habanalabs/common/irq.c24
-rw-r--r--drivers/misc/habanalabs/common/memory.c22
-rw-r--r--drivers/misc/habanalabs/common/mmu/mmu.c14
-rw-r--r--drivers/misc/habanalabs/common/pci/pci.c34
-rw-r--r--drivers/misc/habanalabs/common/sysfs.c2
-rw-r--r--drivers/misc/habanalabs/gaudi/gaudi.c1013
-rw-r--r--drivers/misc/habanalabs/gaudi/gaudiP.h1
-rw-r--r--drivers/misc/habanalabs/gaudi/gaudi_coresight.c6
-rw-r--r--drivers/misc/habanalabs/gaudi/gaudi_security.c15
-rw-r--r--drivers/misc/habanalabs/goya/goya.c251
-rw-r--r--drivers/misc/habanalabs/goya/goyaP.h2
-rw-r--r--drivers/misc/habanalabs/goya/goya_coresight.c2
-rw-r--r--drivers/misc/habanalabs/include/common/cpucp_if.h45
-rw-r--r--drivers/misc/habanalabs/include/common/hl_boot_if.h184
-rw-r--r--drivers/misc/habanalabs/include/gaudi/gaudi_async_events.h14
-rw-r--r--drivers/misc/habanalabs/include/gaudi/gaudi_async_ids_map_extended.h31
-rw-r--r--drivers/misc/habanalabs/include/gaudi/gaudi_fw_if.h46
-rw-r--r--drivers/misc/habanalabs/include/gaudi/gaudi_masks.h15
-rw-r--r--drivers/misc/habanalabs/include/gaudi/gaudi_reg_map.h10
-rw-r--r--drivers/misc/hpilo.c10
-rw-r--r--drivers/misc/hpilo.h3
-rw-r--r--drivers/misc/ibmasm/module.c5
-rw-r--r--drivers/misc/ibmasm/remote.h2
-rw-r--r--drivers/misc/lkdtm/bugs.c11
-rw-r--r--drivers/misc/lkdtm/cfi.c3
-rw-r--r--drivers/misc/lkdtm/core.c58
-rw-r--r--drivers/misc/lkdtm/fortify.c3
-rw-r--r--drivers/misc/lkdtm/heap.c97
-rw-r--r--drivers/misc/lkdtm/lkdtm.h46
-rw-r--r--drivers/misc/lkdtm/stackleak.c4
-rw-r--r--drivers/misc/lkdtm/usercopy.c7
-rw-r--r--drivers/misc/mei/bus-fixup.c2
-rw-r--r--drivers/misc/mei/client.c22
-rw-r--r--drivers/misc/mei/hbm.c2
-rw-r--r--drivers/misc/mei/hdcp/Kconfig1
-rw-r--r--drivers/misc/mei/hw-me.c4
-rw-r--r--drivers/misc/mei/hw.h28
-rw-r--r--drivers/misc/mei/interrupt.c23
-rw-r--r--drivers/misc/mei/main.c4
-rw-r--r--drivers/misc/mei/pci-txe.c2
-rw-r--r--drivers/misc/pvpanic/pvpanic-mmio.c17
-rw-r--r--drivers/misc/pvpanic/pvpanic-pci.c22
-rw-r--r--drivers/misc/pvpanic/pvpanic.c33
-rw-r--r--drivers/misc/pvpanic/pvpanic.h3
-rw-r--r--drivers/misc/uacce/uacce.c11
-rw-r--r--drivers/misc/vmw_vmci/vmci_context.c2
-rw-r--r--drivers/misc/xilinx_sdfec.c3
65 files changed, 3799 insertions, 1236 deletions
diff --git a/drivers/misc/bcm-vk/bcm_vk_msg.c b/drivers/misc/bcm-vk/bcm_vk_msg.c
index f40cf08a6192..066b9ef7fcd7 100644
--- a/drivers/misc/bcm-vk/bcm_vk_msg.c
+++ b/drivers/misc/bcm-vk/bcm_vk_msg.c
@@ -354,8 +354,7 @@ static void bcm_vk_drain_all_pend(struct device *dev,
for (num = 0; num < chan->q_nr; num++) {
list_for_each_entry_safe(entry, tmp, &chan->pendq[num], node) {
if ((!ctx) || (entry->ctx->idx == ctx->idx)) {
- list_del(&entry->node);
- list_add_tail(&entry->node, &del_q);
+ list_move_tail(&entry->node, &del_q);
}
}
}
@@ -701,8 +700,7 @@ int bcm_vk_send_shutdown_msg(struct bcm_vk *vk, u32 shut_type,
return -EINVAL;
}
- entry = kzalloc(sizeof(*entry) +
- sizeof(struct vk_msg_blk), GFP_KERNEL);
+ entry = kzalloc(struct_size(entry, to_v_msg, 1), GFP_KERNEL);
if (!entry)
return -ENOMEM;
diff --git a/drivers/misc/bcm-vk/bcm_vk_msg.h b/drivers/misc/bcm-vk/bcm_vk_msg.h
index 4eaad84825d6..56784c8896d8 100644
--- a/drivers/misc/bcm-vk/bcm_vk_msg.h
+++ b/drivers/misc/bcm-vk/bcm_vk_msg.h
@@ -116,7 +116,7 @@ struct bcm_vk_wkent {
u32 usr_msg_id;
u32 to_v_blks;
u32 seq_num;
- struct vk_msg_blk to_v_msg[0];
+ struct vk_msg_blk to_v_msg[];
};
/* queue stats counters */
diff --git a/drivers/misc/cardreader/alcor_pci.c b/drivers/misc/cardreader/alcor_pci.c
index cd402c89189e..de6d44a158bb 100644
--- a/drivers/misc/cardreader/alcor_pci.c
+++ b/drivers/misc/cardreader/alcor_pci.c
@@ -139,7 +139,13 @@ static void alcor_pci_init_check_aspm(struct alcor_pci_priv *priv)
u32 val32;
priv->pdev_cap_off = alcor_pci_find_cap_offset(priv, priv->pdev);
- priv->parent_cap_off = alcor_pci_find_cap_offset(priv,
+ /*
+ * A device might be attached to root complex directly and
+ * priv->parent_pdev will be NULL. In this case we don't check its
+ * capability and disable ASPM completely.
+ */
+ if (priv->parent_pdev)
+ priv->parent_cap_off = alcor_pci_find_cap_offset(priv,
priv->parent_pdev);
if ((priv->pdev_cap_off == 0) || (priv->parent_cap_off == 0)) {
diff --git a/drivers/misc/cxl/file.c b/drivers/misc/cxl/file.c
index bd3bd32333c5..3dbdce96fae0 100644
--- a/drivers/misc/cxl/file.c
+++ b/drivers/misc/cxl/file.c
@@ -569,7 +569,8 @@ static int cxl_add_chardev(struct cxl_afu *afu, dev_t devt, struct cdev *cdev,
int rc;
cdev_init(cdev, fops);
- if ((rc = cdev_add(cdev, devt, 1))) {
+ rc = cdev_add(cdev, devt, 1);
+ if (rc) {
dev_err(&afu->dev, "Unable to add %s chardev: %i\n", desc, rc);
return rc;
}
@@ -577,8 +578,8 @@ static int cxl_add_chardev(struct cxl_afu *afu, dev_t devt, struct cdev *cdev,
dev = device_create(cxl_class, &afu->dev, devt, afu,
"afu%i.%i%s", afu->adapter->adapter_num, afu->slice, postfix);
if (IS_ERR(dev)) {
- dev_err(&afu->dev, "Unable to create %s chardev in sysfs: %i\n", desc, rc);
rc = PTR_ERR(dev);
+ dev_err(&afu->dev, "Unable to create %s chardev in sysfs: %i\n", desc, rc);
goto err;
}
diff --git a/drivers/misc/eeprom/Kconfig b/drivers/misc/eeprom/Kconfig
index 0f791bfdc1f5..f0a7531f354c 100644
--- a/drivers/misc/eeprom/Kconfig
+++ b/drivers/misc/eeprom/Kconfig
@@ -32,12 +32,13 @@ config EEPROM_AT24
will be called at24.
config EEPROM_AT25
- tristate "SPI EEPROMs from most vendors"
+ tristate "SPI EEPROMs (FRAMs) from most vendors"
depends on SPI && SYSFS
select NVMEM
select NVMEM_SYSFS
help
- Enable this driver to get read/write support to most SPI EEPROMs,
+ Enable this driver to get read/write support to most SPI EEPROMs
+ and Cypress FRAMs,
after you configure the board init code to know about each eeprom
on your target board.
diff --git a/drivers/misc/eeprom/at25.c b/drivers/misc/eeprom/at25.c
index b76e4901b4a4..4d09b672ac3c 100644
--- a/drivers/misc/eeprom/at25.c
+++ b/drivers/misc/eeprom/at25.c
@@ -1,6 +1,7 @@
// SPDX-License-Identifier: GPL-2.0-or-later
/*
* at25.c -- support most SPI EEPROMs, such as Atmel AT25 models
+ * and Cypress FRAMs FM25 models
*
* Copyright (C) 2006 David Brownell
*/
@@ -16,6 +17,9 @@
#include <linux/spi/spi.h>
#include <linux/spi/eeprom.h>
#include <linux/property.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
+#include <linux/math.h>
/*
* NOTE: this is an *EEPROM* driver. The vagaries of product naming
@@ -27,6 +31,7 @@
* AT25M02, AT25128B
*/
+#define FM25_SN_LEN 8 /* serial number length */
struct at25_data {
struct spi_device *spi;
struct mutex lock;
@@ -34,6 +39,7 @@ struct at25_data {
unsigned addrlen;
struct nvmem_config nvmem_config;
struct nvmem_device *nvmem;
+ u8 sernum[FM25_SN_LEN];
};
#define AT25_WREN 0x06 /* latch the write enable */
@@ -42,6 +48,9 @@ struct at25_data {
#define AT25_WRSR 0x01 /* write status register */
#define AT25_READ 0x03 /* read byte(s) */
#define AT25_WRITE 0x02 /* write byte(s)/sector */
+#define FM25_SLEEP 0xb9 /* enter sleep mode */
+#define FM25_RDID 0x9f /* read device ID */
+#define FM25_RDSN 0xc3 /* read S/N */
#define AT25_SR_nRDY 0x01 /* nRDY = write-in-progress */
#define AT25_SR_WEN 0x02 /* write enable (latched) */
@@ -51,6 +60,8 @@ struct at25_data {
#define AT25_INSTR_BIT3 0x08 /* Additional address bit in instr */
+#define FM25_ID_LEN 9 /* ID length */
+
#define EE_MAXADDRLEN 3 /* 24 bit addresses, up to 2 MBytes */
/* Specs often allow 5 msec for a page write, sometimes 20 msec;
@@ -129,6 +140,51 @@ static int at25_ee_read(void *priv, unsigned int offset,
return status;
}
+/*
+ * read extra registers as ID or serial number
+ */
+static int fm25_aux_read(struct at25_data *at25, u8 *buf, uint8_t command,
+ int len)
+{
+ int status;
+ struct spi_transfer t[2];
+ struct spi_message m;
+
+ spi_message_init(&m);
+ memset(t, 0, sizeof(t));
+
+ t[0].tx_buf = &command;
+ t[0].len = 1;
+ spi_message_add_tail(&t[0], &m);
+
+ t[1].rx_buf = buf;
+ t[1].len = len;
+ spi_message_add_tail(&t[1], &m);
+
+ mutex_lock(&at25->lock);
+
+ status = spi_sync(at25->spi, &m);
+ dev_dbg(&at25->spi->dev, "read %d aux bytes --> %d\n", len, status);
+
+ mutex_unlock(&at25->lock);
+ return status;
+}
+
+static ssize_t sernum_show(struct device *dev, struct device_attribute *attr, char *buf)
+{
+ struct at25_data *at25;
+
+ at25 = dev_get_drvdata(dev);
+ return sysfs_emit(buf, "%*ph\n", (int)sizeof(at25->sernum), at25->sernum);
+}
+static DEVICE_ATTR_RO(sernum);
+
+static struct attribute *sernum_attrs[] = {
+ &dev_attr_sernum.attr,
+ NULL,
+};
+ATTRIBUTE_GROUPS(sernum);
+
static int at25_ee_write(void *priv, unsigned int off, void *val, size_t count)
{
struct at25_data *at25 = priv;
@@ -303,34 +359,39 @@ static int at25_fw_to_chip(struct device *dev, struct spi_eeprom *chip)
return 0;
}
+static const struct of_device_id at25_of_match[] = {
+ { .compatible = "atmel,at25",},
+ { .compatible = "cypress,fm25",},
+ { }
+};
+MODULE_DEVICE_TABLE(of, at25_of_match);
+
static int at25_probe(struct spi_device *spi)
{
struct at25_data *at25 = NULL;
struct spi_eeprom chip;
int err;
int sr;
- int addrlen;
+ u8 id[FM25_ID_LEN];
+ u8 sernum[FM25_SN_LEN];
+ int i;
+ const struct of_device_id *match;
+ bool is_fram = 0;
+
+ match = of_match_device(of_match_ptr(at25_of_match), &spi->dev);
+ if (match && !strcmp(match->compatible, "cypress,fm25"))
+ is_fram = 1;
/* Chip description */
if (!spi->dev.platform_data) {
- err = at25_fw_to_chip(&spi->dev, &chip);
- if (err)
- return err;
+ if (!is_fram) {
+ err = at25_fw_to_chip(&spi->dev, &chip);
+ if (err)
+ return err;
+ }
} else
chip = *(struct spi_eeprom *)spi->dev.platform_data;
- /* For now we only support 8/16/24 bit addressing */
- if (chip.flags & EE_ADDR1)
- addrlen = 1;
- else if (chip.flags & EE_ADDR2)
- addrlen = 2;
- else if (chip.flags & EE_ADDR3)
- addrlen = 3;
- else {
- dev_dbg(&spi->dev, "unsupported address type\n");
- return -EINVAL;
- }
-
/* Ping the chip ... the status register is pretty portable,
* unlike probing manufacturer IDs. We do expect that system
* firmware didn't write it in the past few milliseconds!
@@ -349,9 +410,51 @@ static int at25_probe(struct spi_device *spi)
at25->chip = chip;
at25->spi = spi;
spi_set_drvdata(spi, at25);
- at25->addrlen = addrlen;
- at25->nvmem_config.type = NVMEM_TYPE_EEPROM;
+ if (is_fram) {
+ /* Get ID of chip */
+ fm25_aux_read(at25, id, FM25_RDID, FM25_ID_LEN);
+ if (id[6] != 0xc2) {
+ dev_err(&spi->dev,
+ "Error: no Cypress FRAM (id %02x)\n", id[6]);
+ return -ENODEV;
+ }
+ /* set size found in ID */
+ if (id[7] < 0x21 || id[7] > 0x26) {
+ dev_err(&spi->dev, "Error: unsupported size (id %02x)\n", id[7]);
+ return -ENODEV;
+ }
+ chip.byte_len = int_pow(2, id[7] - 0x21 + 4) * 1024;
+
+ if (at25->chip.byte_len > 64 * 1024)
+ at25->chip.flags |= EE_ADDR3;
+ else
+ at25->chip.flags |= EE_ADDR2;
+
+ if (id[8]) {
+ fm25_aux_read(at25, sernum, FM25_RDSN, FM25_SN_LEN);
+ /* swap byte order */
+ for (i = 0; i < FM25_SN_LEN; i++)
+ at25->sernum[i] = sernum[FM25_SN_LEN - 1 - i];
+ }
+
+ at25->chip.page_size = PAGE_SIZE;
+ strncpy(at25->chip.name, "fm25", sizeof(at25->chip.name));
+ }
+
+ /* For now we only support 8/16/24 bit addressing */
+ if (at25->chip.flags & EE_ADDR1)
+ at25->addrlen = 1;
+ else if (at25->chip.flags & EE_ADDR2)
+ at25->addrlen = 2;
+ else if (at25->chip.flags & EE_ADDR3)
+ at25->addrlen = 3;
+ else {
+ dev_dbg(&spi->dev, "unsupported address type\n");
+ return -EINVAL;
+ }
+
+ at25->nvmem_config.type = is_fram ? NVMEM_TYPE_FRAM : NVMEM_TYPE_EEPROM;
at25->nvmem_config.name = dev_name(&spi->dev);
at25->nvmem_config.dev = &spi->dev;
at25->nvmem_config.read_only = chip.flags & EE_READONLY;
@@ -370,27 +473,22 @@ static int at25_probe(struct spi_device *spi)
if (IS_ERR(at25->nvmem))
return PTR_ERR(at25->nvmem);
- dev_info(&spi->dev, "%d %s %s eeprom%s, pagesize %u\n",
- (chip.byte_len < 1024) ? chip.byte_len : (chip.byte_len / 1024),
- (chip.byte_len < 1024) ? "Byte" : "KByte",
- at25->chip.name,
- (chip.flags & EE_READONLY) ? " (readonly)" : "",
- at25->chip.page_size);
+ dev_info(&spi->dev, "%d %s %s %s%s, pagesize %u\n",
+ (chip.byte_len < 1024) ? chip.byte_len : (chip.byte_len / 1024),
+ (chip.byte_len < 1024) ? "Byte" : "KByte",
+ at25->chip.name, is_fram ? "fram" : "eeprom",
+ (chip.flags & EE_READONLY) ? " (readonly)" : "",
+ at25->chip.page_size);
return 0;
}
/*-------------------------------------------------------------------------*/
-static const struct of_device_id at25_of_match[] = {
- { .compatible = "atmel,at25", },
- { }
-};
-MODULE_DEVICE_TABLE(of, at25_of_match);
-
static struct spi_driver at25_driver = {
.driver = {
.name = "at25",
.of_match_table = at25_of_match,
+ .dev_groups = sernum_groups,
},
.probe = at25_probe,
};
diff --git a/drivers/misc/eeprom/ee1004.c b/drivers/misc/eeprom/ee1004.c
index 252e15ba65e1..bb9c4512c968 100644
--- a/drivers/misc/eeprom/ee1004.c
+++ b/drivers/misc/eeprom/ee1004.c
@@ -32,16 +32,17 @@
*/
#define EE1004_ADDR_SET_PAGE 0x36
-#define EE1004_EEPROM_SIZE 512
+#define EE1004_NUM_PAGES 2
#define EE1004_PAGE_SIZE 256
#define EE1004_PAGE_SHIFT 8
+#define EE1004_EEPROM_SIZE (EE1004_PAGE_SIZE * EE1004_NUM_PAGES)
/*
* Mutex protects ee1004_set_page and ee1004_dev_count, and must be held
* from page selection to end of read.
*/
static DEFINE_MUTEX(ee1004_bus_lock);
-static struct i2c_client *ee1004_set_page[2];
+static struct i2c_client *ee1004_set_page[EE1004_NUM_PAGES];
static unsigned int ee1004_dev_count;
static int ee1004_current_page;
@@ -71,40 +72,58 @@ static int ee1004_get_current_page(void)
return 0;
}
+static int ee1004_set_current_page(struct device *dev, int page)
+{
+ int ret;
+
+ if (page == ee1004_current_page)
+ return 0;
+
+ /* Data is ignored */
+ ret = i2c_smbus_write_byte(ee1004_set_page[page], 0x00);
+ /*
+ * Don't give up just yet. Some memory modules will select the page
+ * but not ack the command. Check which page is selected now.
+ */
+ if (ret == -ENXIO && ee1004_get_current_page() == page)
+ ret = 0;
+ if (ret < 0) {
+ dev_err(dev, "Failed to select page %d (%d)\n", page, ret);
+ return ret;
+ }
+
+ dev_dbg(dev, "Selected page %d\n", page);
+ ee1004_current_page = page;
+
+ return 0;
+}
+
static ssize_t ee1004_eeprom_read(struct i2c_client *client, char *buf,
unsigned int offset, size_t count)
{
- int status;
+ int status, page;
+
+ page = offset >> EE1004_PAGE_SHIFT;
+ offset &= (1 << EE1004_PAGE_SHIFT) - 1;
+
+ status = ee1004_set_current_page(&client->dev, page);
+ if (status)
+ return status;
- if (count > I2C_SMBUS_BLOCK_MAX)
- count = I2C_SMBUS_BLOCK_MAX;
/* Can't cross page boundaries */
- if (unlikely(offset + count > EE1004_PAGE_SIZE))
+ if (offset + count > EE1004_PAGE_SIZE)
count = EE1004_PAGE_SIZE - offset;
- status = i2c_smbus_read_i2c_block_data_or_emulated(client, offset,
- count, buf);
- dev_dbg(&client->dev, "read %zu@%d --> %d\n", count, offset, status);
-
- return status;
+ return i2c_smbus_read_i2c_block_data_or_emulated(client, offset, count, buf);
}
-static ssize_t ee1004_read(struct file *filp, struct kobject *kobj,
+static ssize_t eeprom_read(struct file *filp, struct kobject *kobj,
struct bin_attribute *bin_attr,
char *buf, loff_t off, size_t count)
{
- struct device *dev = kobj_to_dev(kobj);
- struct i2c_client *client = to_i2c_client(dev);
+ struct i2c_client *client = kobj_to_i2c_client(kobj);
size_t requested = count;
- int page;
-
- if (unlikely(!count))
- return count;
-
- page = off >> EE1004_PAGE_SHIFT;
- if (unlikely(page > 1))
- return 0;
- off &= (1 << EE1004_PAGE_SHIFT) - 1;
+ int ret = 0;
/*
* Read data from chip, protecting against concurrent access to
@@ -113,139 +132,85 @@ static ssize_t ee1004_read(struct file *filp, struct kobject *kobj,
mutex_lock(&ee1004_bus_lock);
while (count) {
- int status;
-
- /* Select page */
- if (page != ee1004_current_page) {
- /* Data is ignored */
- status = i2c_smbus_write_byte(ee1004_set_page[page],
- 0x00);
- if (status == -ENXIO) {
- /*
- * Don't give up just yet. Some memory
- * modules will select the page but not
- * ack the command. Check which page is
- * selected now.
- */
- if (ee1004_get_current_page() == page)
- status = 0;
- }
- if (status < 0) {
- dev_err(dev, "Failed to select page %d (%d)\n",
- page, status);
- mutex_unlock(&ee1004_bus_lock);
- return status;
- }
- dev_dbg(dev, "Selected page %d\n", page);
- ee1004_current_page = page;
- }
-
- status = ee1004_eeprom_read(client, buf, off, count);
- if (status < 0) {
- mutex_unlock(&ee1004_bus_lock);
- return status;
- }
- buf += status;
- off += status;
- count -= status;
+ ret = ee1004_eeprom_read(client, buf, off, count);
+ if (ret < 0)
+ goto out;
- if (off == EE1004_PAGE_SIZE) {
- page++;
- off = 0;
- }
+ buf += ret;
+ off += ret;
+ count -= ret;
}
-
+out:
mutex_unlock(&ee1004_bus_lock);
- return requested;
+ return ret < 0 ? ret : requested;
}
-static const struct bin_attribute eeprom_attr = {
- .attr = {
- .name = "eeprom",
- .mode = 0444,
- },
- .size = EE1004_EEPROM_SIZE,
- .read = ee1004_read,
+static BIN_ATTR_RO(eeprom, EE1004_EEPROM_SIZE);
+
+static struct bin_attribute *ee1004_attrs[] = {
+ &bin_attr_eeprom,
+ NULL
};
-static int ee1004_probe(struct i2c_client *client,
- const struct i2c_device_id *id)
+BIN_ATTRIBUTE_GROUPS(ee1004);
+
+static void ee1004_cleanup(int idx)
+{
+ if (--ee1004_dev_count == 0)
+ while (--idx >= 0) {
+ i2c_unregister_device(ee1004_set_page[idx]);
+ ee1004_set_page[idx] = NULL;
+ }
+}
+
+static int ee1004_probe(struct i2c_client *client)
{
int err, cnr = 0;
- const char *slow = NULL;
/* Make sure we can operate on this adapter */
if (!i2c_check_functionality(client->adapter,
- I2C_FUNC_SMBUS_READ_BYTE |
- I2C_FUNC_SMBUS_READ_I2C_BLOCK)) {
- if (i2c_check_functionality(client->adapter,
- I2C_FUNC_SMBUS_READ_BYTE |
- I2C_FUNC_SMBUS_READ_WORD_DATA))
- slow = "word";
- else if (i2c_check_functionality(client->adapter,
- I2C_FUNC_SMBUS_READ_BYTE |
- I2C_FUNC_SMBUS_READ_BYTE_DATA))
- slow = "byte";
- else
- return -EPFNOSUPPORT;
- }
+ I2C_FUNC_SMBUS_BYTE | I2C_FUNC_SMBUS_READ_I2C_BLOCK) &&
+ !i2c_check_functionality(client->adapter,
+ I2C_FUNC_SMBUS_BYTE | I2C_FUNC_SMBUS_READ_BYTE_DATA))
+ return -EPFNOSUPPORT;
/* Use 2 dummy devices for page select command */
mutex_lock(&ee1004_bus_lock);
if (++ee1004_dev_count == 1) {
- for (cnr = 0; cnr < 2; cnr++) {
- ee1004_set_page[cnr] = i2c_new_dummy_device(client->adapter,
- EE1004_ADDR_SET_PAGE + cnr);
- if (IS_ERR(ee1004_set_page[cnr])) {
- dev_err(&client->dev,
- "address 0x%02x unavailable\n",
- EE1004_ADDR_SET_PAGE + cnr);
- err = PTR_ERR(ee1004_set_page[cnr]);
+ for (cnr = 0; cnr < EE1004_NUM_PAGES; cnr++) {
+ struct i2c_client *cl;
+
+ cl = i2c_new_dummy_device(client->adapter, EE1004_ADDR_SET_PAGE + cnr);
+ if (IS_ERR(cl)) {
+ err = PTR_ERR(cl);
goto err_clients;
}
+ ee1004_set_page[cnr] = cl;
}
- } else if (i2c_adapter_id(client->adapter) !=
- i2c_adapter_id(ee1004_set_page[0]->adapter)) {
+
+ /* Remember current page to avoid unneeded page select */
+ err = ee1004_get_current_page();
+ if (err < 0)
+ goto err_clients;
+ dev_dbg(&client->dev, "Currently selected page: %d\n", err);
+ ee1004_current_page = err;
+ } else if (client->adapter != ee1004_set_page[0]->adapter) {
dev_err(&client->dev,
"Driver only supports devices on a single I2C bus\n");
err = -EOPNOTSUPP;
goto err_clients;
}
-
- /* Remember current page to avoid unneeded page select */
- err = ee1004_get_current_page();
- if (err < 0)
- goto err_clients;
- ee1004_current_page = err;
- dev_dbg(&client->dev, "Currently selected page: %d\n",
- ee1004_current_page);
mutex_unlock(&ee1004_bus_lock);
- /* Create the sysfs eeprom file */
- err = sysfs_create_bin_file(&client->dev.kobj, &eeprom_attr);
- if (err)
- goto err_clients_lock;
-
dev_info(&client->dev,
"%u byte EE1004-compliant SPD EEPROM, read-only\n",
EE1004_EEPROM_SIZE);
- if (slow)
- dev_notice(&client->dev,
- "Falling back to %s reads, performance will suffer\n",
- slow);
return 0;
- err_clients_lock:
- mutex_lock(&ee1004_bus_lock);
err_clients:
- if (--ee1004_dev_count == 0) {
- for (cnr--; cnr >= 0; cnr--) {
- i2c_unregister_device(ee1004_set_page[cnr]);
- ee1004_set_page[cnr] = NULL;
- }
- }
+ ee1004_cleanup(cnr);
mutex_unlock(&ee1004_bus_lock);
return err;
@@ -253,18 +218,9 @@ static int ee1004_probe(struct i2c_client *client,
static int ee1004_remove(struct i2c_client *client)
{
- int i;
-
- sysfs_remove_bin_file(&client->dev.kobj, &eeprom_attr);
-
/* Remove page select clients if this is the last device */
mutex_lock(&ee1004_bus_lock);
- if (--ee1004_dev_count == 0) {
- for (i = 0; i < 2; i++) {
- i2c_unregister_device(ee1004_set_page[i]);
- ee1004_set_page[i] = NULL;
- }
- }
+ ee1004_cleanup(EE1004_NUM_PAGES);
mutex_unlock(&ee1004_bus_lock);
return 0;
@@ -275,8 +231,9 @@ static int ee1004_remove(struct i2c_client *client)
static struct i2c_driver ee1004_driver = {
.driver = {
.name = "ee1004",
+ .dev_groups = ee1004_groups,
},
- .probe = ee1004_probe,
+ .probe_new = ee1004_probe,
.remove = ee1004_remove,
.id_table = ee1004_ids,
};
diff --git a/drivers/misc/eeprom/eeprom_93xx46.c b/drivers/misc/eeprom/eeprom_93xx46.c
index 80114f4c80ad..29d8971ec558 100644
--- a/drivers/misc/eeprom/eeprom_93xx46.c
+++ b/drivers/misc/eeprom/eeprom_93xx46.c
@@ -9,6 +9,7 @@
#include <linux/device.h>
#include <linux/gpio/consumer.h>
#include <linux/kernel.h>
+#include <linux/log2.h>
#include <linux/module.h>
#include <linux/mutex.h>
#include <linux/of.h>
@@ -28,14 +29,29 @@
struct eeprom_93xx46_devtype_data {
unsigned int quirks;
+ unsigned char flags;
+};
+
+static const struct eeprom_93xx46_devtype_data at93c46_data = {
+ .flags = EE_SIZE1K,
+};
+
+static const struct eeprom_93xx46_devtype_data at93c56_data = {
+ .flags = EE_SIZE2K,
+};
+
+static const struct eeprom_93xx46_devtype_data at93c66_data = {
+ .flags = EE_SIZE4K,
};
static const struct eeprom_93xx46_devtype_data atmel_at93c46d_data = {
+ .flags = EE_SIZE1K,
.quirks = EEPROM_93XX46_QUIRK_SINGLE_WORD_READ |
EEPROM_93XX46_QUIRK_INSTRUCTION_LENGTH,
};
static const struct eeprom_93xx46_devtype_data microchip_93lc46b_data = {
+ .flags = EE_SIZE1K,
.quirks = EEPROM_93XX46_QUIRK_EXTRA_READ_CYCLE,
};
@@ -70,6 +86,7 @@ static int eeprom_93xx46_read(void *priv, unsigned int off,
struct eeprom_93xx46_dev *edev = priv;
char *buf = val;
int err = 0;
+ int bits;
if (unlikely(off >= edev->size))
return 0;
@@ -83,21 +100,21 @@ static int eeprom_93xx46_read(void *priv, unsigned int off,
if (edev->pdata->prepare)
edev->pdata->prepare(edev);
+ /* The opcode in front of the address is three bits. */
+ bits = edev->addrlen + 3;
+
while (count) {
struct spi_message m;
struct spi_transfer t[2] = { { 0 } };
u16 cmd_addr = OP_READ << edev->addrlen;
size_t nbytes = count;
- int bits;
- if (edev->addrlen == 7) {
- cmd_addr |= off & 0x7f;
- bits = 10;
+ if (edev->pdata->flags & EE_ADDR8) {
+ cmd_addr |= off;
if (has_quirk_single_word_read(edev))
nbytes = 1;
} else {
- cmd_addr |= (off >> 1) & 0x3f;
- bits = 9;
+ cmd_addr |= (off >> 1);
if (has_quirk_single_word_read(edev))
nbytes = 2;
}
@@ -152,14 +169,14 @@ static int eeprom_93xx46_ew(struct eeprom_93xx46_dev *edev, int is_on)
int bits, ret;
u16 cmd_addr;
+ /* The opcode in front of the address is three bits. */
+ bits = edev->addrlen + 3;
+
cmd_addr = OP_START << edev->addrlen;
- if (edev->addrlen == 7) {
+ if (edev->pdata->flags & EE_ADDR8)
cmd_addr |= (is_on ? ADDR_EWEN : ADDR_EWDS) << 1;
- bits = 10;
- } else {
+ else
cmd_addr |= (is_on ? ADDR_EWEN : ADDR_EWDS);
- bits = 9;
- }
if (has_quirk_instruction_length(edev)) {
cmd_addr <<= 2;
@@ -205,15 +222,19 @@ eeprom_93xx46_write_word(struct eeprom_93xx46_dev *edev,
int bits, data_len, ret;
u16 cmd_addr;
+ if (unlikely(off >= edev->size))
+ return -EINVAL;
+
+ /* The opcode in front of the address is three bits. */
+ bits = edev->addrlen + 3;
+
cmd_addr = OP_WRITE << edev->addrlen;
- if (edev->addrlen == 7) {
- cmd_addr |= off & 0x7f;
- bits = 10;
+ if (edev->pdata->flags & EE_ADDR8) {
+ cmd_addr |= off;
data_len = 1;
} else {
- cmd_addr |= (off >> 1) & 0x3f;
- bits = 9;
+ cmd_addr |= (off >> 1);
data_len = 2;
}
@@ -253,7 +274,7 @@ static int eeprom_93xx46_write(void *priv, unsigned int off,
return count;
/* only write even number of bytes on 16-bit devices */
- if (edev->addrlen == 6) {
+ if (edev->pdata->flags & EE_ADDR16) {
step = 2;
count &= ~1;
}
@@ -295,14 +316,14 @@ static int eeprom_93xx46_eral(struct eeprom_93xx46_dev *edev)
int bits, ret;
u16 cmd_addr;
+ /* The opcode in front of the address is three bits. */
+ bits = edev->addrlen + 3;
+
cmd_addr = OP_START << edev->addrlen;
- if (edev->addrlen == 7) {
+ if (edev->pdata->flags & EE_ADDR8)
cmd_addr |= ADDR_ERAL << 1;
- bits = 10;
- } else {
+ else
cmd_addr |= ADDR_ERAL;
- bits = 9;
- }
if (has_quirk_instruction_length(edev)) {
cmd_addr <<= 2;
@@ -375,8 +396,11 @@ static void select_deassert(void *context)
}
static const struct of_device_id eeprom_93xx46_of_table[] = {
- { .compatible = "eeprom-93xx46", },
+ { .compatible = "eeprom-93xx46", .data = &at93c46_data, },
+ { .compatible = "atmel,at93c46", .data = &at93c46_data, },
{ .compatible = "atmel,at93c46d", .data = &atmel_at93c46d_data, },
+ { .compatible = "atmel,at93c56", .data = &at93c56_data, },
+ { .compatible = "atmel,at93c66", .data = &at93c66_data, },
{ .compatible = "microchip,93lc46b", .data = &microchip_93lc46b_data, },
{}
};
@@ -426,6 +450,7 @@ static int eeprom_93xx46_probe_dt(struct spi_device *spi)
const struct eeprom_93xx46_devtype_data *data = of_id->data;
pd->quirks = data->quirks;
+ pd->flags |= data->flags;
}
spi->dev.platform_data = pd;
@@ -455,10 +480,21 @@ static int eeprom_93xx46_probe(struct spi_device *spi)
if (!edev)
return -ENOMEM;
+ if (pd->flags & EE_SIZE1K)
+ edev->size = 128;
+ else if (pd->flags & EE_SIZE2K)
+ edev->size = 256;
+ else if (pd->flags & EE_SIZE4K)
+ edev->size = 512;
+ else {
+ dev_err(&spi->dev, "unspecified size\n");
+ return -EINVAL;
+ }
+
if (pd->flags & EE_ADDR8)
- edev->addrlen = 7;
+ edev->addrlen = ilog2(edev->size);
else if (pd->flags & EE_ADDR16)
- edev->addrlen = 6;
+ edev->addrlen = ilog2(edev->size) - 1;
else {
dev_err(&spi->dev, "unspecified address type\n");
return -EINVAL;
@@ -469,7 +505,6 @@ static int eeprom_93xx46_probe(struct spi_device *spi)
edev->spi = spi;
edev->pdata = pd;
- edev->size = 128;
edev->nvmem_config.type = NVMEM_TYPE_EEPROM;
edev->nvmem_config.name = dev_name(&spi->dev);
edev->nvmem_config.dev = &spi->dev;
@@ -489,8 +524,9 @@ static int eeprom_93xx46_probe(struct spi_device *spi)
if (IS_ERR(edev->nvmem))
return PTR_ERR(edev->nvmem);
- dev_info(&spi->dev, "%d-bit eeprom %s\n",
+ dev_info(&spi->dev, "%d-bit eeprom containing %d bytes %s\n",
(pd->flags & EE_ADDR8) ? 8 : 16,
+ edev->size,
(pd->flags & EE_READONLY) ? "(readonly)" : "");
if (!(pd->flags & EE_READONLY)) {
diff --git a/drivers/misc/eeprom/idt_89hpesx.c b/drivers/misc/eeprom/idt_89hpesx.c
index 81c70e5bc168..b0cff4b152da 100644
--- a/drivers/misc/eeprom/idt_89hpesx.c
+++ b/drivers/misc/eeprom/idt_89hpesx.c
@@ -1,38 +1,7 @@
+// SPDX-License-Identifier: GPL-2.0-only
/*
- * This file is provided under a GPLv2 license. When using or
- * redistributing this file, you may do so under that license.
- *
- * GPL LICENSE SUMMARY
- *
* Copyright (C) 2016 T-Platforms. All Rights Reserved.
*
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, it can be found <http://www.gnu.org/licenses/>.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
* IDT PCIe-switch NTB Linux driver
*
* Contact Information:
@@ -1126,11 +1095,10 @@ static void idt_get_fw_data(struct idt_89hpesx_dev *pdev)
device_for_each_child_node(dev, fwnode) {
ee_id = idt_ee_match_id(fwnode);
- if (!ee_id) {
- dev_warn(dev, "Skip unsupported EEPROM device");
- continue;
- } else
+ if (ee_id)
break;
+
+ dev_warn(dev, "Skip unsupported EEPROM device %pfw\n", fwnode);
}
/* If there is no fwnode EEPROM device, then set zero size */
@@ -1161,6 +1129,7 @@ static void idt_get_fw_data(struct idt_89hpesx_dev *pdev)
else /* if (!fwnode_property_read_bool(node, "read-only")) */
pdev->eero = false;
+ fwnode_handle_put(fwnode);
dev_info(dev, "EEPROM of %d bytes found by 0x%x",
pdev->eesize, pdev->eeaddr);
}
diff --git a/drivers/misc/habanalabs/common/command_submission.c b/drivers/misc/habanalabs/common/command_submission.c
index af3c497defb1..80c60fb41bbc 100644
--- a/drivers/misc/habanalabs/common/command_submission.c
+++ b/drivers/misc/habanalabs/common/command_submission.c
@@ -556,6 +556,13 @@ out:
else if (!cs->submitted)
cs->fence->error = -EBUSY;
+ if (unlikely(cs->skip_reset_on_timeout)) {
+ dev_err(hdev->dev,
+ "Command submission %llu completed after %llu (s)\n",
+ cs->sequence,
+ div_u64(jiffies - cs->submission_time_jiffies, HZ));
+ }
+
if (cs->timestamp)
cs->fence->timestamp = ktime_get();
complete_all(&cs->fence->completion);
@@ -571,6 +578,8 @@ static void cs_timedout(struct work_struct *work)
int rc;
struct hl_cs *cs = container_of(work, struct hl_cs,
work_tdr.work);
+ bool skip_reset_on_timeout = cs->skip_reset_on_timeout;
+
rc = cs_get_unless_zero(cs);
if (!rc)
return;
@@ -581,7 +590,8 @@ static void cs_timedout(struct work_struct *work)
}
/* Mark the CS is timed out so we won't try to cancel its TDR */
- cs->timedout = true;
+ if (likely(!skip_reset_on_timeout))
+ cs->timedout = true;
hdev = cs->ctx->hdev;
@@ -613,10 +623,12 @@ static void cs_timedout(struct work_struct *work)
cs_put(cs);
- if (hdev->reset_on_lockup)
- hl_device_reset(hdev, 0);
- else
- hdev->needs_reset = true;
+ if (likely(!skip_reset_on_timeout)) {
+ if (hdev->reset_on_lockup)
+ hl_device_reset(hdev, HL_RESET_TDR);
+ else
+ hdev->needs_reset = true;
+ }
}
static int allocate_cs(struct hl_device *hdev, struct hl_ctx *ctx,
@@ -650,6 +662,10 @@ static int allocate_cs(struct hl_device *hdev, struct hl_ctx *ctx,
cs->type = cs_type;
cs->timestamp = !!(flags & HL_CS_FLAGS_TIMESTAMP);
cs->timeout_jiffies = timeout;
+ cs->skip_reset_on_timeout =
+ hdev->skip_reset_on_timeout ||
+ !!(flags & HL_CS_FLAGS_SKIP_RESET_ON_TIMEOUT);
+ cs->submission_time_jiffies = jiffies;
INIT_LIST_HEAD(&cs->job_list);
INIT_DELAYED_WORK(&cs->work_tdr, cs_timedout);
kref_init(&cs->refcount);
@@ -1481,6 +1497,61 @@ out:
return rc;
}
+/*
+ * hl_cs_signal_sob_wraparound_handler: handle SOB value wrapaound case.
+ * if the SOB value reaches the max value move to the other SOB reserved
+ * to the queue.
+ * Note that this function must be called while hw_queues_lock is taken.
+ */
+int hl_cs_signal_sob_wraparound_handler(struct hl_device *hdev, u32 q_idx,
+ struct hl_hw_sob **hw_sob, u32 count)
+{
+ struct hl_sync_stream_properties *prop;
+ struct hl_hw_sob *sob = *hw_sob, *other_sob;
+ u8 other_sob_offset;
+
+ prop = &hdev->kernel_queues[q_idx].sync_stream_prop;
+
+ kref_get(&sob->kref);
+
+ /* check for wraparound */
+ if (prop->next_sob_val + count >= HL_MAX_SOB_VAL) {
+ /*
+ * Decrement as we reached the max value.
+ * The release function won't be called here as we've
+ * just incremented the refcount right before calling this
+ * function.
+ */
+ kref_put(&sob->kref, hl_sob_reset_error);
+
+ /*
+ * check the other sob value, if it still in use then fail
+ * otherwise make the switch
+ */
+ other_sob_offset = (prop->curr_sob_offset + 1) % HL_RSVD_SOBS;
+ other_sob = &prop->hw_sob[other_sob_offset];
+
+ if (kref_read(&other_sob->kref) != 1) {
+ dev_err(hdev->dev, "error: Cannot switch SOBs q_idx: %d\n",
+ q_idx);
+ return -EINVAL;
+ }
+
+ prop->next_sob_val = 1;
+
+ /* only two SOBs are currently in use */
+ prop->curr_sob_offset = other_sob_offset;
+ *hw_sob = other_sob;
+
+ dev_dbg(hdev->dev, "switched to SOB %d, q_idx: %d\n",
+ prop->curr_sob_offset, q_idx);
+ } else {
+ prop->next_sob_val += count;
+ }
+
+ return 0;
+}
+
static int cs_ioctl_extract_signal_seq(struct hl_device *hdev,
struct hl_cs_chunk *chunk, u64 *signal_seq, struct hl_ctx *ctx)
{
diff --git a/drivers/misc/habanalabs/common/context.c b/drivers/misc/habanalabs/common/context.c
index 62d705889ca8..19b6b045219e 100644
--- a/drivers/misc/habanalabs/common/context.c
+++ b/drivers/misc/habanalabs/common/context.c
@@ -12,7 +12,6 @@
static void hl_ctx_fini(struct hl_ctx *ctx)
{
struct hl_device *hdev = ctx->hdev;
- u64 idle_mask[HL_BUSY_ENGINES_MASK_EXT_SIZE] = {0};
int i;
/* Release all allocated pending cb's, those cb's were never
@@ -57,14 +56,6 @@ static void hl_ctx_fini(struct hl_ctx *ctx)
/* Scrub both SRAM and DRAM */
hdev->asic_funcs->scrub_device_mem(hdev, 0, 0);
-
- if ((!hdev->pldm) && (hdev->pdev) &&
- (!hdev->asic_funcs->is_device_idle(hdev,
- idle_mask,
- HL_BUSY_ENGINES_MASK_EXT_SIZE, NULL)))
- dev_notice(hdev->dev,
- "device not idle after user context is closed (0x%llx, 0x%llx)\n",
- idle_mask[0], idle_mask[1]);
} else {
dev_dbg(hdev->dev, "closing kernel context\n");
hdev->asic_funcs->ctx_fini(ctx);
diff --git a/drivers/misc/habanalabs/common/debugfs.c b/drivers/misc/habanalabs/common/debugfs.c
index 8381155578a0..703d79fb6f3f 100644
--- a/drivers/misc/habanalabs/common/debugfs.c
+++ b/drivers/misc/habanalabs/common/debugfs.c
@@ -1278,6 +1278,11 @@ void hl_debugfs_add_device(struct hl_device *hdev)
dev_entry->root,
&dev_entry->blob_desc);
+ debugfs_create_x8("skip_reset_on_timeout",
+ 0644,
+ dev_entry->root,
+ &hdev->skip_reset_on_timeout);
+
for (i = 0, entry = dev_entry->entry_arr ; i < count ; i++, entry++) {
debugfs_create_file(hl_debugfs_list[i].name,
0444,
diff --git a/drivers/misc/habanalabs/common/device.c b/drivers/misc/habanalabs/common/device.c
index 00e92b678828..ff4cbde289c0 100644
--- a/drivers/misc/habanalabs/common/device.c
+++ b/drivers/misc/habanalabs/common/device.c
@@ -51,6 +51,8 @@ bool hl_device_operational(struct hl_device *hdev,
static void hpriv_release(struct kref *ref)
{
+ u64 idle_mask[HL_BUSY_ENGINES_MASK_EXT_SIZE] = {0};
+ bool device_is_idle = true;
struct hl_fpriv *hpriv;
struct hl_device *hdev;
@@ -71,8 +73,20 @@ static void hpriv_release(struct kref *ref)
kfree(hpriv);
- if (hdev->reset_upon_device_release)
- hl_device_reset(hdev, 0);
+ if ((!hdev->pldm) && (hdev->pdev) &&
+ (!hdev->asic_funcs->is_device_idle(hdev,
+ idle_mask,
+ HL_BUSY_ENGINES_MASK_EXT_SIZE, NULL))) {
+ dev_err(hdev->dev,
+ "device not idle after user context is closed (0x%llx_%llx)\n",
+ idle_mask[1], idle_mask[0]);
+
+ device_is_idle = false;
+ }
+
+ if ((hdev->reset_if_device_not_idle && !device_is_idle)
+ || hdev->reset_upon_device_release)
+ hl_device_reset(hdev, HL_RESET_DEVICE_RELEASE);
}
void hl_hpriv_get(struct hl_fpriv *hpriv)
@@ -118,6 +132,9 @@ static int hl_device_release(struct inode *inode, struct file *filp)
dev_warn(hdev->dev,
"Device is still in use because there are live CS and/or memory mappings\n");
+ hdev->last_open_session_duration_jif =
+ jiffies - hdev->last_successful_open_jif;
+
return 0;
}
@@ -868,7 +885,7 @@ static void device_disable_open_processes(struct hl_device *hdev)
int hl_device_reset(struct hl_device *hdev, u32 flags)
{
u64 idle_mask[HL_BUSY_ENGINES_MASK_EXT_SIZE] = {0};
- bool hard_reset, from_hard_reset_thread;
+ bool hard_reset, from_hard_reset_thread, hard_instead_soft = false;
int i, rc;
if (!hdev->init_done) {
@@ -880,11 +897,28 @@ int hl_device_reset(struct hl_device *hdev, u32 flags)
hard_reset = (flags & HL_RESET_HARD) != 0;
from_hard_reset_thread = (flags & HL_RESET_FROM_RESET_THREAD) != 0;
- if ((!hard_reset) && (!hdev->supports_soft_reset)) {
- dev_dbg(hdev->dev, "Doing hard-reset instead of soft-reset\n");
+ if (!hard_reset && !hdev->supports_soft_reset) {
+ hard_instead_soft = true;
+ hard_reset = true;
+ }
+
+ if (hdev->reset_upon_device_release &&
+ (flags & HL_RESET_DEVICE_RELEASE)) {
+ dev_dbg(hdev->dev,
+ "Perform %s-reset upon device release\n",
+ hard_reset ? "hard" : "soft");
+ goto do_reset;
+ }
+
+ if (!hard_reset && !hdev->allow_external_soft_reset) {
+ hard_instead_soft = true;
hard_reset = true;
}
+ if (hard_instead_soft)
+ dev_dbg(hdev->dev, "Doing hard-reset instead of soft-reset\n");
+
+do_reset:
/* Re-entry of reset thread */
if (from_hard_reset_thread && hdev->process_kill_trial_cnt)
goto kill_processes;
@@ -901,6 +935,19 @@ int hl_device_reset(struct hl_device *hdev, u32 flags)
return 0;
/*
+ * 'reset cause' is being updated here, because getting here
+ * means that it's the 1st time and the last time we're here
+ * ('in_reset' makes sure of it). This makes sure that
+ * 'reset_cause' will continue holding its 1st recorded reason!
+ */
+ if (flags & HL_RESET_HEARTBEAT)
+ hdev->curr_reset_cause = HL_RESET_CAUSE_HEARTBEAT;
+ else if (flags & HL_RESET_TDR)
+ hdev->curr_reset_cause = HL_RESET_CAUSE_TDR;
+ else
+ hdev->curr_reset_cause = HL_RESET_CAUSE_UNKNOWN;
+
+ /*
* if reset is due to heartbeat, device CPU is no responsive in
* which case no point sending PCI disable message to it
*/
@@ -943,9 +990,8 @@ again:
hdev->process_kill_trial_cnt = 0;
/*
- * Because the reset function can't run from interrupt or
- * from heartbeat work, we need to call the reset function
- * from a dedicated work
+ * Because the reset function can't run from heartbeat work,
+ * we need to call the reset function from a dedicated work.
*/
queue_delayed_work(hdev->device_reset_work.wq,
&hdev->device_reset_work.reset_work, 0);
@@ -1096,8 +1142,8 @@ kill_processes:
if (!hdev->asic_funcs->is_device_idle(hdev, idle_mask,
HL_BUSY_ENGINES_MASK_EXT_SIZE, NULL)) {
dev_err(hdev->dev,
- "device is not idle (mask %#llx %#llx) after reset\n",
- idle_mask[0], idle_mask[1]);
+ "device is not idle (mask 0x%llx_%llx) after reset\n",
+ idle_mask[1], idle_mask[0]);
rc = -EIO;
goto out_err;
}
@@ -1334,8 +1380,9 @@ int hl_device_init(struct hl_device *hdev, struct class *hclass)
}
/*
- * From this point, in case of an error, add char devices and create
- * sysfs nodes as part of the error flow, to allow debugging.
+ * From this point, override rc (=0) in case of an error to allow
+ * debugging (by adding char devices and create sysfs nodes as part of
+ * the error flow).
*/
add_cdev_sysfs_on_err = true;
@@ -1369,7 +1416,7 @@ int hl_device_init(struct hl_device *hdev, struct class *hclass)
dev_info(hdev->dev, "Found %s device with %lluGB DRAM\n",
hdev->asic_name,
- hdev->asic_prop.dram_size / 1024 / 1024 / 1024);
+ hdev->asic_prop.dram_size / SZ_1G);
rc = hl_vm_init(hdev);
if (rc) {
@@ -1475,6 +1522,7 @@ out_disabled:
void hl_device_fini(struct hl_device *hdev)
{
ktime_t timeout;
+ u64 reset_sec;
int i, rc;
dev_info(hdev->dev, "Removing device\n");
@@ -1482,6 +1530,11 @@ void hl_device_fini(struct hl_device *hdev)
hdev->device_fini_pending = 1;
flush_delayed_work(&hdev->device_reset_work.reset_work);
+ if (hdev->pldm)
+ reset_sec = HL_PLDM_HARD_RESET_MAX_TIMEOUT;
+ else
+ reset_sec = HL_HARD_RESET_MAX_TIMEOUT;
+
/*
* This function is competing with the reset function, so try to
* take the reset atomic and if we are already in middle of reset,
@@ -1490,8 +1543,7 @@ void hl_device_fini(struct hl_device *hdev)
* ports, the hard reset could take between 10-30 seconds
*/
- timeout = ktime_add_us(ktime_get(),
- HL_HARD_RESET_MAX_TIMEOUT * 1000 * 1000);
+ timeout = ktime_add_us(ktime_get(), reset_sec * 1000 * 1000);
rc = atomic_cmpxchg(&hdev->in_reset, 0, 1);
while (rc) {
usleep_range(50, 200);
diff --git a/drivers/misc/habanalabs/common/firmware_if.c b/drivers/misc/habanalabs/common/firmware_if.c
index 0713b2c12d54..2e4d04ec6b53 100644
--- a/drivers/misc/habanalabs/common/firmware_if.c
+++ b/drivers/misc/habanalabs/common/firmware_if.c
@@ -9,42 +9,63 @@
#include "../include/common/hl_boot_if.h"
#include <linux/firmware.h>
+#include <linux/crc32.h>
#include <linux/slab.h>
+#include <linux/ctype.h>
-#define FW_FILE_MAX_SIZE 0x1400000 /* maximum size of 20MB */
-/**
- * hl_fw_load_fw_to_device() - Load F/W code to device's memory.
- *
- * @hdev: pointer to hl_device structure.
- * @fw_name: the firmware image name
- * @dst: IO memory mapped address space to copy firmware to
- * @src_offset: offset in src FW to copy from
- * @size: amount of bytes to copy (0 to copy the whole binary)
- *
- * Copy fw code from firmware file to device memory.
- *
- * Return: 0 on success, non-zero for failure.
- */
-int hl_fw_load_fw_to_device(struct hl_device *hdev, const char *fw_name,
- void __iomem *dst, u32 src_offset, u32 size)
+#define FW_FILE_MAX_SIZE 0x1400000 /* maximum size of 20MB */
+
+#define FW_CPU_STATUS_POLL_INTERVAL_USEC 10000
+
+static char *extract_fw_ver_from_str(const char *fw_str)
+{
+ char *str, *fw_ver, *whitespace;
+
+ fw_ver = kmalloc(16, GFP_KERNEL);
+ if (!fw_ver)
+ return NULL;
+
+ str = strnstr(fw_str, "fw-", VERSION_MAX_LEN);
+ if (!str)
+ goto free_fw_ver;
+
+ /* Skip the fw- part */
+ str += 3;
+
+ /* Copy until the next whitespace */
+ whitespace = strnstr(str, " ", 15);
+ if (!whitespace)
+ goto free_fw_ver;
+
+ strscpy(fw_ver, str, whitespace - str + 1);
+
+ return fw_ver;
+
+free_fw_ver:
+ kfree(fw_ver);
+ return NULL;
+}
+
+static int hl_request_fw(struct hl_device *hdev,
+ const struct firmware **firmware_p,
+ const char *fw_name)
{
- const struct firmware *fw;
- const void *fw_data;
size_t fw_size;
int rc;
- rc = request_firmware(&fw, fw_name, hdev->dev);
+ rc = request_firmware(firmware_p, fw_name, hdev->dev);
if (rc) {
- dev_err(hdev->dev, "Firmware file %s is not found!\n", fw_name);
+ dev_err(hdev->dev, "Firmware file %s is not found! (error %d)\n",
+ fw_name, rc);
goto out;
}
- fw_size = fw->size;
+ fw_size = (*firmware_p)->size;
if ((fw_size % 4) != 0) {
dev_err(hdev->dev, "Illegal %s firmware size %zu\n",
- fw_name, fw_size);
+ fw_name, fw_size);
rc = -EINVAL;
- goto out;
+ goto release_fw;
}
dev_dbg(hdev->dev, "%s firmware size == %zu\n", fw_name, fw_size);
@@ -54,26 +75,125 @@ int hl_fw_load_fw_to_device(struct hl_device *hdev, const char *fw_name,
"FW file size %zu exceeds maximum of %u bytes\n",
fw_size, FW_FILE_MAX_SIZE);
rc = -EINVAL;
- goto out;
+ goto release_fw;
}
- if (size - src_offset > fw_size) {
+ return 0;
+
+release_fw:
+ release_firmware(*firmware_p);
+out:
+ return rc;
+}
+
+/**
+ * hl_release_firmware() - release FW
+ *
+ * @fw: fw descriptor
+ *
+ * note: this inline function added to serve as a comprehensive mirror for the
+ * hl_request_fw function.
+ */
+static inline void hl_release_firmware(const struct firmware *fw)
+{
+ release_firmware(fw);
+}
+
+/**
+ * hl_fw_copy_fw_to_device() - copy FW to device
+ *
+ * @hdev: pointer to hl_device structure.
+ * @fw: fw descriptor
+ * @dst: IO memory mapped address space to copy firmware to
+ * @src_offset: offset in src FW to copy from
+ * @size: amount of bytes to copy (0 to copy the whole binary)
+ *
+ * actual copy of FW binary data to device, shared by static and dynamic loaders
+ */
+static int hl_fw_copy_fw_to_device(struct hl_device *hdev,
+ const struct firmware *fw, void __iomem *dst,
+ u32 src_offset, u32 size)
+{
+ const void *fw_data;
+
+ /* size 0 indicates to copy the whole file */
+ if (!size)
+ size = fw->size;
+
+ if (src_offset + size > fw->size) {
dev_err(hdev->dev,
"size to copy(%u) and offset(%u) are invalid\n",
size, src_offset);
- rc = -EINVAL;
- goto out;
+ return -EINVAL;
}
- if (size)
- fw_size = size;
-
fw_data = (const void *) fw->data;
- memcpy_toio(dst, fw_data + src_offset, fw_size);
+ memcpy_toio(dst, fw_data + src_offset, size);
+ return 0;
+}
-out:
- release_firmware(fw);
+/**
+ * hl_fw_copy_msg_to_device() - copy message to device
+ *
+ * @hdev: pointer to hl_device structure.
+ * @msg: message
+ * @dst: IO memory mapped address space to copy firmware to
+ * @src_offset: offset in src message to copy from
+ * @size: amount of bytes to copy (0 to copy the whole binary)
+ *
+ * actual copy of message data to device.
+ */
+static int hl_fw_copy_msg_to_device(struct hl_device *hdev,
+ struct lkd_msg_comms *msg, void __iomem *dst,
+ u32 src_offset, u32 size)
+{
+ void *msg_data;
+
+ /* size 0 indicates to copy the whole file */
+ if (!size)
+ size = sizeof(struct lkd_msg_comms);
+
+ if (src_offset + size > sizeof(struct lkd_msg_comms)) {
+ dev_err(hdev->dev,
+ "size to copy(%u) and offset(%u) are invalid\n",
+ size, src_offset);
+ return -EINVAL;
+ }
+
+ msg_data = (void *) msg;
+
+ memcpy_toio(dst, msg_data + src_offset, size);
+
+ return 0;
+}
+
+/**
+ * hl_fw_load_fw_to_device() - Load F/W code to device's memory.
+ *
+ * @hdev: pointer to hl_device structure.
+ * @fw_name: the firmware image name
+ * @dst: IO memory mapped address space to copy firmware to
+ * @src_offset: offset in src FW to copy from
+ * @size: amount of bytes to copy (0 to copy the whole binary)
+ *
+ * Copy fw code from firmware file to device memory.
+ *
+ * Return: 0 on success, non-zero for failure.
+ */
+int hl_fw_load_fw_to_device(struct hl_device *hdev, const char *fw_name,
+ void __iomem *dst, u32 src_offset, u32 size)
+{
+ const struct firmware *fw;
+ int rc;
+
+ rc = hl_request_fw(hdev, &fw, fw_name);
+ if (rc)
+ return rc;
+
+ rc = hl_fw_copy_fw_to_device(hdev, fw, dst, src_offset, size);
+
+ hl_release_firmware(fw);
return rc;
}
@@ -91,6 +211,7 @@ int hl_fw_send_cpu_message(struct hl_device *hdev, u32 hw_queue_id, u32 *msg,
u16 len, u32 timeout, u64 *result)
{
struct hl_hw_queue *queue = &hdev->kernel_queues[hw_queue_id];
+ struct asic_fixed_properties *prop = &hdev->asic_prop;
struct cpucp_packet *pkt;
dma_addr_t pkt_dma_addr;
u32 tmp, expected_ack_val;
@@ -117,7 +238,7 @@ int hl_fw_send_cpu_message(struct hl_device *hdev, u32 hw_queue_id, u32 *msg,
}
/* set fence to a non valid value */
- pkt->fence = UINT_MAX;
+ pkt->fence = cpu_to_le32(UINT_MAX);
rc = hl_hw_queue_send_cb_no_cmpl(hdev, hw_queue_id, len, pkt_dma_addr);
if (rc) {
@@ -125,8 +246,7 @@ int hl_fw_send_cpu_message(struct hl_device *hdev, u32 hw_queue_id, u32 *msg,
goto out;
}
- if (hdev->asic_prop.fw_app_security_map &
- CPU_BOOT_DEV_STS0_PKT_PI_ACK_EN)
+ if (prop->fw_app_cpu_boot_dev_sts0 & CPU_BOOT_DEV_STS0_PKT_PI_ACK_EN)
expected_ack_val = queue->pi;
else
expected_ack_val = CPUCP_PACKET_FENCE_VAL;
@@ -272,10 +392,11 @@ void hl_fw_cpu_accessible_dma_pool_free(struct hl_device *hdev, size_t size,
int hl_fw_send_heartbeat(struct hl_device *hdev)
{
- struct cpucp_packet hb_pkt = {};
+ struct cpucp_packet hb_pkt;
u64 result;
int rc;
+ memset(&hb_pkt, 0, sizeof(hb_pkt));
hb_pkt.ctl = cpu_to_le32(CPUCP_PACKET_TEST <<
CPUCP_PKT_CTL_OPCODE_SHIFT);
hb_pkt.value = cpu_to_le64(CPUCP_PACKET_FENCE_VAL);
@@ -284,29 +405,24 @@ int hl_fw_send_heartbeat(struct hl_device *hdev)
sizeof(hb_pkt), 0, &result);
if ((rc) || (result != CPUCP_PACKET_FENCE_VAL))
+ return -EIO;
+
+ if (le32_to_cpu(hb_pkt.status_mask) &
+ CPUCP_PKT_HB_STATUS_EQ_FAULT_MASK) {
+ dev_warn(hdev->dev, "FW reported EQ fault during heartbeat\n");
rc = -EIO;
+ }
return rc;
}
-static int fw_read_errors(struct hl_device *hdev, u32 boot_err0_reg,
- u32 cpu_security_boot_status_reg)
+static bool fw_report_boot_dev0(struct hl_device *hdev, u32 err_val,
+ u32 sts_val)
{
- u32 err_val, security_val;
bool err_exists = false;
- /* Some of the firmware status codes are deprecated in newer f/w
- * versions. In those versions, the errors are reported
- * in different registers. Therefore, we need to check those
- * registers and print the exact errors. Moreover, there
- * may be multiple errors, so we need to report on each error
- * separately. Some of the error codes might indicate a state
- * that is not an error per-se, but it is an error in production
- * environment
- */
- err_val = RREG32(boot_err0_reg);
if (!(err_val & CPU_BOOT_ERR0_ENABLED))
- return 0;
+ return false;
if (err_val & CPU_BOOT_ERR0_DRAM_INIT_FAIL) {
dev_err(hdev->dev,
@@ -377,44 +493,120 @@ static int fw_read_errors(struct hl_device *hdev, u32 boot_err0_reg,
err_exists = true;
}
+ if (err_val & CPU_BOOT_ERR0_PRI_IMG_VER_FAIL) {
+ dev_warn(hdev->dev,
+ "Device boot warning - Failed to load preboot primary image\n");
+ /* This is a warning so we don't want it to disable the
+ * device as we have a secondary preboot image
+ */
+ err_val &= ~CPU_BOOT_ERR0_PRI_IMG_VER_FAIL;
+ }
+
+ if (err_val & CPU_BOOT_ERR0_SEC_IMG_VER_FAIL) {
+ dev_err(hdev->dev, "Device boot error - Failed to load preboot secondary image\n");
+ err_exists = true;
+ }
+
if (err_val & CPU_BOOT_ERR0_PLL_FAIL) {
dev_err(hdev->dev, "Device boot error - PLL failure\n");
err_exists = true;
}
if (err_val & CPU_BOOT_ERR0_DEVICE_UNUSABLE_FAIL) {
- dev_err(hdev->dev,
- "Device boot error - device unusable\n");
- err_exists = true;
+ /* Ignore this bit, don't prevent driver loading */
+ dev_dbg(hdev->dev, "device unusable status is set\n");
+ err_val &= ~CPU_BOOT_ERR0_DEVICE_UNUSABLE_FAIL;
}
- security_val = RREG32(cpu_security_boot_status_reg);
- if (security_val & CPU_BOOT_DEV_STS0_ENABLED)
- dev_dbg(hdev->dev, "Device security status %#x\n",
- security_val);
+ if (sts_val & CPU_BOOT_DEV_STS0_ENABLED)
+ dev_dbg(hdev->dev, "Device status0 %#x\n", sts_val);
if (!err_exists && (err_val & ~CPU_BOOT_ERR0_ENABLED)) {
dev_err(hdev->dev,
- "Device boot error - unknown error 0x%08x\n",
- err_val);
+ "Device boot error - unknown ERR0 error 0x%08x\n", err_val);
err_exists = true;
}
+ /* return error only if it's in the predefined mask */
if (err_exists && ((err_val & ~CPU_BOOT_ERR0_ENABLED) &
lower_32_bits(hdev->boot_error_status_mask)))
+ return true;
+
+ return false;
+}
+
+/* placeholder for ERR1 as no errors defined there yet */
+static bool fw_report_boot_dev1(struct hl_device *hdev, u32 err_val,
+ u32 sts_val)
+{
+ /*
+ * keep this variable to preserve the logic of the function.
+ * this way it would require less modifications when error will be
+ * added to DEV_ERR1
+ */
+ bool err_exists = false;
+
+ if (!(err_val & CPU_BOOT_ERR1_ENABLED))
+ return false;
+
+ if (sts_val & CPU_BOOT_DEV_STS1_ENABLED)
+ dev_dbg(hdev->dev, "Device status1 %#x\n", sts_val);
+
+ if (!err_exists && (err_val & ~CPU_BOOT_ERR1_ENABLED)) {
+ dev_err(hdev->dev,
+ "Device boot error - unknown ERR1 error 0x%08x\n",
+ err_val);
+ err_exists = true;
+ }
+
+ /* return error only if it's in the predefined mask */
+ if (err_exists && ((err_val & ~CPU_BOOT_ERR1_ENABLED) &
+ upper_32_bits(hdev->boot_error_status_mask)))
+ return true;
+
+ return false;
+}
+
+static int fw_read_errors(struct hl_device *hdev, u32 boot_err0_reg,
+ u32 boot_err1_reg, u32 cpu_boot_dev_status0_reg,
+ u32 cpu_boot_dev_status1_reg)
+{
+ u32 err_val, status_val;
+ bool err_exists = false;
+
+ /* Some of the firmware status codes are deprecated in newer f/w
+ * versions. In those versions, the errors are reported
+ * in different registers. Therefore, we need to check those
+ * registers and print the exact errors. Moreover, there
+ * may be multiple errors, so we need to report on each error
+ * separately. Some of the error codes might indicate a state
+ * that is not an error per-se, but it is an error in production
+ * environment
+ */
+ err_val = RREG32(boot_err0_reg);
+ status_val = RREG32(cpu_boot_dev_status0_reg);
+ err_exists = fw_report_boot_dev0(hdev, err_val, status_val);
+
+ err_val = RREG32(boot_err1_reg);
+ status_val = RREG32(cpu_boot_dev_status1_reg);
+ err_exists |= fw_report_boot_dev1(hdev, err_val, status_val);
+
+ if (err_exists)
return -EIO;
return 0;
}
int hl_fw_cpucp_info_get(struct hl_device *hdev,
- u32 cpu_security_boot_status_reg,
- u32 boot_err0_reg)
+ u32 sts_boot_dev_sts0_reg,
+ u32 sts_boot_dev_sts1_reg, u32 boot_err0_reg,
+ u32 boot_err1_reg)
{
struct asic_fixed_properties *prop = &hdev->asic_prop;
struct cpucp_packet pkt = {};
- void *cpucp_info_cpu_addr;
dma_addr_t cpucp_info_dma_addr;
+ void *cpucp_info_cpu_addr;
+ char *kernel_ver;
u64 result;
int rc;
@@ -443,7 +635,8 @@ int hl_fw_cpucp_info_get(struct hl_device *hdev,
goto out;
}
- rc = fw_read_errors(hdev, boot_err0_reg, cpu_security_boot_status_reg);
+ rc = fw_read_errors(hdev, boot_err0_reg, boot_err1_reg,
+ sts_boot_dev_sts0_reg, sts_boot_dev_sts1_reg);
if (rc) {
dev_err(hdev->dev, "Errors in device boot\n");
goto out;
@@ -460,10 +653,27 @@ int hl_fw_cpucp_info_get(struct hl_device *hdev,
goto out;
}
+ kernel_ver = extract_fw_ver_from_str(prop->cpucp_info.kernel_version);
+ if (kernel_ver) {
+ dev_info(hdev->dev, "Linux version %s", kernel_ver);
+ kfree(kernel_ver);
+ }
+
+ /* assume EQ code doesn't need to check eqe index */
+ hdev->event_queue.check_eqe_index = false;
+
/* Read FW application security bits again */
- if (hdev->asic_prop.fw_security_status_valid)
- hdev->asic_prop.fw_app_security_map =
- RREG32(cpu_security_boot_status_reg);
+ if (hdev->asic_prop.fw_cpu_boot_dev_sts0_valid) {
+ hdev->asic_prop.fw_app_cpu_boot_dev_sts0 =
+ RREG32(sts_boot_dev_sts0_reg);
+ if (hdev->asic_prop.fw_app_cpu_boot_dev_sts0 &
+ CPU_BOOT_DEV_STS0_EQ_INDEX_EN)
+ hdev->event_queue.check_eqe_index = true;
+ }
+
+ if (hdev->asic_prop.fw_cpu_boot_dev_sts1_valid)
+ hdev->asic_prop.fw_app_cpu_boot_dev_sts1 =
+ RREG32(sts_boot_dev_sts1_reg);
out:
hdev->asic_funcs->cpu_accessible_dma_pool_free(hdev,
@@ -501,7 +711,8 @@ static int hl_fw_send_msi_info_msg(struct hl_device *hdev)
pkt->length = cpu_to_le32(CPUCP_NUM_OF_MSI_TYPES);
- hdev->asic_funcs->get_msi_info((u32 *)&pkt->data);
+ memset((void *) &pkt->data, 0xFF, data_size);
+ hdev->asic_funcs->get_msi_info(pkt->data);
pkt->cpucp_pkt.ctl = cpu_to_le32(CPUCP_PACKET_MSI_INFO_SET <<
CPUCP_PKT_CTL_OPCODE_SHIFT);
@@ -526,13 +737,15 @@ static int hl_fw_send_msi_info_msg(struct hl_device *hdev)
}
int hl_fw_cpucp_handshake(struct hl_device *hdev,
- u32 cpu_security_boot_status_reg,
- u32 boot_err0_reg)
+ u32 sts_boot_dev_sts0_reg,
+ u32 sts_boot_dev_sts1_reg, u32 boot_err0_reg,
+ u32 boot_err1_reg)
{
int rc;
- rc = hl_fw_cpucp_info_get(hdev, cpu_security_boot_status_reg,
- boot_err0_reg);
+ rc = hl_fw_cpucp_info_get(hdev, sts_boot_dev_sts0_reg,
+ sts_boot_dev_sts1_reg, boot_err0_reg,
+ boot_err1_reg);
if (rc)
return rc;
@@ -667,8 +880,8 @@ int get_used_pll_index(struct hl_device *hdev, u32 input_pll_index,
bool dynamic_pll;
int fw_pll_idx;
- dynamic_pll = prop->fw_security_status_valid &&
- (prop->fw_app_security_map & CPU_BOOT_DEV_STS0_DYN_PLL_EN);
+ dynamic_pll = !!(prop->fw_app_cpu_boot_dev_sts0 &
+ CPU_BOOT_DEV_STS0_DYN_PLL_EN);
if (!dynamic_pll) {
/*
@@ -759,6 +972,47 @@ int hl_fw_cpucp_power_get(struct hl_device *hdev, u64 *power)
return rc;
}
+void hl_fw_ask_hard_reset_without_linux(struct hl_device *hdev)
+{
+ struct static_fw_load_mgr *static_loader =
+ &hdev->fw_loader.static_loader;
+ int rc;
+
+ if (hdev->asic_prop.dynamic_fw_load) {
+ rc = hl_fw_dynamic_send_protocol_cmd(hdev, &hdev->fw_loader,
+ COMMS_RST_DEV, 0, false,
+ hdev->fw_loader.cpu_timeout);
+ if (rc)
+ dev_warn(hdev->dev, "Failed sending COMMS_RST_DEV\n");
+ } else {
+ WREG32(static_loader->kmd_msg_to_cpu_reg, KMD_MSG_RST_DEV);
+ }
+}
+
+void hl_fw_ask_halt_machine_without_linux(struct hl_device *hdev)
+{
+ struct static_fw_load_mgr *static_loader =
+ &hdev->fw_loader.static_loader;
+ int rc;
+
+ if (hdev->device_cpu_is_halted)
+ return;
+
+ /* Stop device CPU to make sure nothing bad happens */
+ if (hdev->asic_prop.dynamic_fw_load) {
+ rc = hl_fw_dynamic_send_protocol_cmd(hdev, &hdev->fw_loader,
+ COMMS_GOTO_WFE, 0, true,
+ hdev->fw_loader.cpu_timeout);
+ if (rc)
+ dev_warn(hdev->dev, "Failed sending COMMS_GOTO_WFE\n");
+ } else {
+ WREG32(static_loader->kmd_msg_to_cpu_reg, KMD_MSG_GOTO_WFE);
+ msleep(static_loader->cpu_reset_wait_msec);
+ }
+
+ hdev->device_cpu_is_halted = true;
+}
+
static void detect_cpu_boot_status(struct hl_device *hdev, u32 status)
{
/* Some of the status codes below are deprecated in newer f/w
@@ -767,63 +1021,59 @@ static void detect_cpu_boot_status(struct hl_device *hdev, u32 status)
switch (status) {
case CPU_BOOT_STATUS_NA:
dev_err(hdev->dev,
- "Device boot error - BTL did NOT run\n");
+ "Device boot progress - BTL did NOT run\n");
break;
case CPU_BOOT_STATUS_IN_WFE:
dev_err(hdev->dev,
- "Device boot error - Stuck inside WFE loop\n");
+ "Device boot progress - Stuck inside WFE loop\n");
break;
case CPU_BOOT_STATUS_IN_BTL:
dev_err(hdev->dev,
- "Device boot error - Stuck in BTL\n");
+ "Device boot progress - Stuck in BTL\n");
break;
case CPU_BOOT_STATUS_IN_PREBOOT:
dev_err(hdev->dev,
- "Device boot error - Stuck in Preboot\n");
+ "Device boot progress - Stuck in Preboot\n");
break;
case CPU_BOOT_STATUS_IN_SPL:
dev_err(hdev->dev,
- "Device boot error - Stuck in SPL\n");
+ "Device boot progress - Stuck in SPL\n");
break;
case CPU_BOOT_STATUS_IN_UBOOT:
dev_err(hdev->dev,
- "Device boot error - Stuck in u-boot\n");
+ "Device boot progress - Stuck in u-boot\n");
break;
case CPU_BOOT_STATUS_DRAM_INIT_FAIL:
dev_err(hdev->dev,
- "Device boot error - DRAM initialization failed\n");
+ "Device boot progress - DRAM initialization failed\n");
break;
case CPU_BOOT_STATUS_UBOOT_NOT_READY:
dev_err(hdev->dev,
- "Device boot error - u-boot stopped by user\n");
+ "Device boot progress - Cannot boot\n");
break;
case CPU_BOOT_STATUS_TS_INIT_FAIL:
dev_err(hdev->dev,
- "Device boot error - Thermal Sensor initialization failed\n");
+ "Device boot progress - Thermal Sensor initialization failed\n");
break;
default:
dev_err(hdev->dev,
- "Device boot error - Invalid status code %d\n",
+ "Device boot progress - Invalid status code %d\n",
status);
break;
}
}
-int hl_fw_read_preboot_status(struct hl_device *hdev, u32 cpu_boot_status_reg,
- u32 cpu_security_boot_status_reg, u32 boot_err0_reg,
- u32 timeout)
+static int hl_fw_read_preboot_caps(struct hl_device *hdev,
+ u32 cpu_boot_status_reg,
+ u32 sts_boot_dev_sts0_reg,
+ u32 sts_boot_dev_sts1_reg,
+ u32 boot_err0_reg, u32 boot_err1_reg,
+ u32 timeout)
{
struct asic_fixed_properties *prop = &hdev->asic_prop;
- u32 status, security_status;
+ u32 status, reg_val;
int rc;
- /* pldm was added for cases in which we use preboot on pldm and want
- * to load boot fit, but we can't wait for preboot because it runs
- * very slowly
- */
- if (!(hdev->fw_components & FW_TYPE_PREBOOT_CPU) || hdev->pldm)
- return 0;
-
/* Need to check two possible scenarios:
*
* CPU_BOOT_STATUS_WAITING_FOR_BOOT_FIT - for newer firmwares where
@@ -842,29 +1092,145 @@ int hl_fw_read_preboot_status(struct hl_device *hdev, u32 cpu_boot_status_reg,
(status == CPU_BOOT_STATUS_READY_TO_BOOT) ||
(status == CPU_BOOT_STATUS_SRAM_AVAIL) ||
(status == CPU_BOOT_STATUS_WAITING_FOR_BOOT_FIT),
- 10000,
+ FW_CPU_STATUS_POLL_INTERVAL_USEC,
timeout);
if (rc) {
- dev_err(hdev->dev, "Failed to read preboot version\n");
+ dev_err(hdev->dev, "CPU boot ready status timeout\n");
detect_cpu_boot_status(hdev, status);
/* If we read all FF, then something is totally wrong, no point
* of reading specific errors
*/
if (status != -1)
- fw_read_errors(hdev, boot_err0_reg,
- cpu_security_boot_status_reg);
+ fw_read_errors(hdev, boot_err0_reg, boot_err1_reg,
+ sts_boot_dev_sts0_reg,
+ sts_boot_dev_sts1_reg);
return -EIO;
}
- rc = hdev->asic_funcs->read_device_fw_version(hdev, FW_COMP_PREBOOT);
- if (rc)
- return rc;
+ /*
+ * the registers DEV_STS* contain FW capabilities/features.
+ * We can rely on this registers only if bit CPU_BOOT_DEV_STS*_ENABLED
+ * is set.
+ * In the first read of this register we store the value of this
+ * register ONLY if the register is enabled (which will be propagated
+ * to next stages) and also mark the register as valid.
+ * In case it is not enabled the stored value will be left 0- all
+ * caps/features are off
+ */
+ reg_val = RREG32(sts_boot_dev_sts0_reg);
+ if (reg_val & CPU_BOOT_DEV_STS0_ENABLED) {
+ prop->fw_cpu_boot_dev_sts0_valid = true;
+ prop->fw_preboot_cpu_boot_dev_sts0 = reg_val;
+ }
+
+ reg_val = RREG32(sts_boot_dev_sts1_reg);
+ if (reg_val & CPU_BOOT_DEV_STS1_ENABLED) {
+ prop->fw_cpu_boot_dev_sts1_valid = true;
+ prop->fw_preboot_cpu_boot_dev_sts1 = reg_val;
+ }
- security_status = RREG32(cpu_security_boot_status_reg);
+ prop->dynamic_fw_load = !!(prop->fw_preboot_cpu_boot_dev_sts0 &
+ CPU_BOOT_DEV_STS0_FW_LD_COM_EN);
+
+ /* initialize FW loader once we know what load protocol is used */
+ hdev->asic_funcs->init_firmware_loader(hdev);
+
+ dev_dbg(hdev->dev, "Attempting %s FW load\n",
+ prop->dynamic_fw_load ? "dynamic" : "legacy");
+ return 0;
+}
- /* We read security status multiple times during boot:
+static int hl_fw_static_read_device_fw_version(struct hl_device *hdev,
+ enum hl_fw_component fwc)
+{
+ struct asic_fixed_properties *prop = &hdev->asic_prop;
+ struct fw_load_mgr *fw_loader = &hdev->fw_loader;
+ struct static_fw_load_mgr *static_loader;
+ char *dest, *boot_ver, *preboot_ver;
+ u32 ver_off, limit;
+ const char *name;
+ char btl_ver[32];
+
+ static_loader = &hdev->fw_loader.static_loader;
+
+ switch (fwc) {
+ case FW_COMP_BOOT_FIT:
+ ver_off = RREG32(static_loader->boot_fit_version_offset_reg);
+ dest = prop->uboot_ver;
+ name = "Boot-fit";
+ limit = static_loader->boot_fit_version_max_off;
+ break;
+ case FW_COMP_PREBOOT:
+ ver_off = RREG32(static_loader->preboot_version_offset_reg);
+ dest = prop->preboot_ver;
+ name = "Preboot";
+ limit = static_loader->preboot_version_max_off;
+ break;
+ default:
+ dev_warn(hdev->dev, "Undefined FW component: %d\n", fwc);
+ return -EIO;
+ }
+
+ ver_off &= static_loader->sram_offset_mask;
+
+ if (ver_off < limit) {
+ memcpy_fromio(dest,
+ hdev->pcie_bar[fw_loader->sram_bar_id] + ver_off,
+ VERSION_MAX_LEN);
+ } else {
+ dev_err(hdev->dev, "%s version offset (0x%x) is above SRAM\n",
+ name, ver_off);
+ strscpy(dest, "unavailable", VERSION_MAX_LEN);
+ return -EIO;
+ }
+
+ if (fwc == FW_COMP_BOOT_FIT) {
+ boot_ver = extract_fw_ver_from_str(prop->uboot_ver);
+ if (boot_ver) {
+ dev_info(hdev->dev, "boot-fit version %s\n", boot_ver);
+ kfree(boot_ver);
+ }
+ } else if (fwc == FW_COMP_PREBOOT) {
+ preboot_ver = strnstr(prop->preboot_ver, "Preboot",
+ VERSION_MAX_LEN);
+ if (preboot_ver && preboot_ver != prop->preboot_ver) {
+ strscpy(btl_ver, prop->preboot_ver,
+ min((int) (preboot_ver - prop->preboot_ver),
+ 31));
+ dev_info(hdev->dev, "%s\n", btl_ver);
+ }
+
+ preboot_ver = extract_fw_ver_from_str(prop->preboot_ver);
+ if (preboot_ver) {
+ dev_info(hdev->dev, "preboot version %s\n",
+ preboot_ver);
+ kfree(preboot_ver);
+ }
+ }
+
+ return 0;
+}
+
+/**
+ * hl_fw_preboot_update_state - update internal data structures during
+ * handshake with preboot
+ *
+ *
+ * @hdev: pointer to the habanalabs device structure
+ *
+ * @return 0 on success, otherwise non-zero error code
+ */
+static void hl_fw_preboot_update_state(struct hl_device *hdev)
+{
+ struct asic_fixed_properties *prop = &hdev->asic_prop;
+ u32 cpu_boot_dev_sts0, cpu_boot_dev_sts1;
+
+ cpu_boot_dev_sts0 = prop->fw_preboot_cpu_boot_dev_sts0;
+ cpu_boot_dev_sts1 = prop->fw_preboot_cpu_boot_dev_sts1;
+
+ /* We read boot_dev_sts registers multiple times during boot:
* 1. preboot - a. Check whether the security status bits are valid
* b. Check whether fw security is enabled
* c. Check whether hard reset is done by preboot
@@ -874,48 +1240,1121 @@ int hl_fw_read_preboot_status(struct hl_device *hdev, u32 cpu_boot_status_reg,
* b. Check whether hard reset is done by fw app
*
* Preboot:
- * Check security status bit (CPU_BOOT_DEV_STS0_ENABLED), if it is set
+ * Check security status bit (CPU_BOOT_DEV_STS0_ENABLED). If set, then-
* check security enabled bit (CPU_BOOT_DEV_STS0_SECURITY_EN)
+ * If set, then mark GIC controller to be disabled.
*/
- if (security_status & CPU_BOOT_DEV_STS0_ENABLED) {
- prop->fw_security_status_valid = 1;
+ prop->hard_reset_done_by_fw =
+ !!(cpu_boot_dev_sts0 & CPU_BOOT_DEV_STS0_FW_HARD_RST_EN);
- /* FW security should be derived from PCI ID, we keep this
- * check for backward compatibility
- */
- if (security_status & CPU_BOOT_DEV_STS0_SECURITY_EN)
- prop->fw_security_disabled = false;
+ dev_dbg(hdev->dev, "Firmware preboot boot device status0 %#x\n",
+ cpu_boot_dev_sts0);
+
+ dev_dbg(hdev->dev, "Firmware preboot boot device status1 %#x\n",
+ cpu_boot_dev_sts1);
+
+ dev_dbg(hdev->dev, "Firmware preboot hard-reset is %s\n",
+ prop->hard_reset_done_by_fw ? "enabled" : "disabled");
+
+ dev_dbg(hdev->dev, "firmware-level security is %s\n",
+ prop->fw_security_enabled ? "enabled" : "disabled");
+
+ dev_dbg(hdev->dev, "GIC controller is %s\n",
+ prop->gic_interrupts_enable ? "enabled" : "disabled");
+}
+
+static int hl_fw_static_read_preboot_status(struct hl_device *hdev)
+{
+ int rc;
+
+ rc = hl_fw_static_read_device_fw_version(hdev, FW_COMP_PREBOOT);
+ if (rc)
+ return rc;
+
+ return 0;
+}
+
+int hl_fw_read_preboot_status(struct hl_device *hdev, u32 cpu_boot_status_reg,
+ u32 sts_boot_dev_sts0_reg,
+ u32 sts_boot_dev_sts1_reg, u32 boot_err0_reg,
+ u32 boot_err1_reg, u32 timeout)
+{
+ int rc;
+
+ /* pldm was added for cases in which we use preboot on pldm and want
+ * to load boot fit, but we can't wait for preboot because it runs
+ * very slowly
+ */
+ if (!(hdev->fw_components & FW_TYPE_PREBOOT_CPU) || hdev->pldm)
+ return 0;
+
+ /*
+ * In order to determine boot method (static VS dymanic) we need to
+ * read the boot caps register
+ */
+ rc = hl_fw_read_preboot_caps(hdev, cpu_boot_status_reg,
+ sts_boot_dev_sts0_reg,
+ sts_boot_dev_sts1_reg, boot_err0_reg,
+ boot_err1_reg, timeout);
+ if (rc)
+ return rc;
+
+ hl_fw_preboot_update_state(hdev);
- if (security_status & CPU_BOOT_DEV_STS0_FW_HARD_RST_EN)
+ /* no need to read preboot status in dynamic load */
+ if (hdev->asic_prop.dynamic_fw_load)
+ return 0;
+
+ return hl_fw_static_read_preboot_status(hdev);
+}
+
+/* associate string with COMM status */
+static char *hl_dynamic_fw_status_str[COMMS_STS_INVLD_LAST] = {
+ [COMMS_STS_NOOP] = "NOOP",
+ [COMMS_STS_ACK] = "ACK",
+ [COMMS_STS_OK] = "OK",
+ [COMMS_STS_ERR] = "ERR",
+ [COMMS_STS_VALID_ERR] = "VALID_ERR",
+ [COMMS_STS_TIMEOUT_ERR] = "TIMEOUT_ERR",
+};
+
+/**
+ * hl_fw_dynamic_report_error_status - report error status
+ *
+ * @hdev: pointer to the habanalabs device structure
+ * @status: value of FW status register
+ * @expected_status: the expected status
+ */
+static void hl_fw_dynamic_report_error_status(struct hl_device *hdev,
+ u32 status,
+ enum comms_sts expected_status)
+{
+ enum comms_sts comm_status =
+ FIELD_GET(COMMS_STATUS_STATUS_MASK, status);
+
+ if (comm_status < COMMS_STS_INVLD_LAST)
+ dev_err(hdev->dev, "Device status %s, expected status: %s\n",
+ hl_dynamic_fw_status_str[comm_status],
+ hl_dynamic_fw_status_str[expected_status]);
+ else
+ dev_err(hdev->dev, "Device status unknown %d, expected status: %s\n",
+ comm_status,
+ hl_dynamic_fw_status_str[expected_status]);
+}
+
+/**
+ * hl_fw_dynamic_send_cmd - send LKD to FW cmd
+ *
+ * @hdev: pointer to the habanalabs device structure
+ * @fw_loader: managing structure for loading device's FW
+ * @cmd: LKD to FW cmd code
+ * @size: size of next FW component to be loaded (0 if not necessary)
+ *
+ * LDK to FW exact command layout is defined at struct comms_command.
+ * note: the size argument is used only when the next FW component should be
+ * loaded, otherwise it shall be 0. the size is used by the FW in later
+ * protocol stages and when sending only indicating the amount of memory
+ * to be allocated by the FW to receive the next boot component.
+ */
+static void hl_fw_dynamic_send_cmd(struct hl_device *hdev,
+ struct fw_load_mgr *fw_loader,
+ enum comms_cmd cmd, unsigned int size)
+{
+ struct cpu_dyn_regs *dyn_regs;
+ u32 val;
+
+ dyn_regs = &fw_loader->dynamic_loader.comm_desc.cpu_dyn_regs;
+
+ val = FIELD_PREP(COMMS_COMMAND_CMD_MASK, cmd);
+ val |= FIELD_PREP(COMMS_COMMAND_SIZE_MASK, size);
+
+ WREG32(le32_to_cpu(dyn_regs->kmd_msg_to_cpu), val);
+}
+
+/**
+ * hl_fw_dynamic_extract_fw_response - update the FW response
+ *
+ * @hdev: pointer to the habanalabs device structure
+ * @fw_loader: managing structure for loading device's FW
+ * @response: FW response
+ * @status: the status read from CPU status register
+ *
+ * @return 0 on success, otherwise non-zero error code
+ */
+static int hl_fw_dynamic_extract_fw_response(struct hl_device *hdev,
+ struct fw_load_mgr *fw_loader,
+ struct fw_response *response,
+ u32 status)
+{
+ response->status = FIELD_GET(COMMS_STATUS_STATUS_MASK, status);
+ response->ram_offset = FIELD_GET(COMMS_STATUS_OFFSET_MASK, status) <<
+ COMMS_STATUS_OFFSET_ALIGN_SHIFT;
+ response->ram_type = FIELD_GET(COMMS_STATUS_RAM_TYPE_MASK, status);
+
+ if ((response->ram_type != COMMS_SRAM) &&
+ (response->ram_type != COMMS_DRAM)) {
+ dev_err(hdev->dev, "FW status: invalid RAM type %u\n",
+ response->ram_type);
+ return -EIO;
+ }
+
+ return 0;
+}
+
+/**
+ * hl_fw_dynamic_wait_for_status - wait for status in dynamic FW load
+ *
+ * @hdev: pointer to the habanalabs device structure
+ * @fw_loader: managing structure for loading device's FW
+ * @expected_status: expected status to wait for
+ * @timeout: timeout for status wait
+ *
+ * @return 0 on success, otherwise non-zero error code
+ *
+ * waiting for status from FW include polling the FW status register until
+ * expected status is received or timeout occurs (whatever occurs first).
+ */
+static int hl_fw_dynamic_wait_for_status(struct hl_device *hdev,
+ struct fw_load_mgr *fw_loader,
+ enum comms_sts expected_status,
+ u32 timeout)
+{
+ struct cpu_dyn_regs *dyn_regs;
+ u32 status;
+ int rc;
+
+ dyn_regs = &fw_loader->dynamic_loader.comm_desc.cpu_dyn_regs;
+
+ /* Wait for expected status */
+ rc = hl_poll_timeout(
+ hdev,
+ le32_to_cpu(dyn_regs->cpu_cmd_status_to_host),
+ status,
+ FIELD_GET(COMMS_STATUS_STATUS_MASK, status) == expected_status,
+ FW_CPU_STATUS_POLL_INTERVAL_USEC,
+ timeout);
+
+ if (rc) {
+ hl_fw_dynamic_report_error_status(hdev, status,
+ expected_status);
+ return -EIO;
+ }
+
+ /*
+ * skip storing FW response for NOOP to preserve the actual desired
+ * FW status
+ */
+ if (expected_status == COMMS_STS_NOOP)
+ return 0;
+
+ rc = hl_fw_dynamic_extract_fw_response(hdev, fw_loader,
+ &fw_loader->dynamic_loader.response,
+ status);
+ return rc;
+}
+
+/**
+ * hl_fw_dynamic_send_clear_cmd - send clear command to FW
+ *
+ * @hdev: pointer to the habanalabs device structure
+ * @fw_loader: managing structure for loading device's FW
+ *
+ * @return 0 on success, otherwise non-zero error code
+ *
+ * after command cycle between LKD to FW CPU (i.e. LKD got an expected status
+ * from FW) we need to clear the CPU status register in order to avoid garbage
+ * between command cycles.
+ * This is done by sending clear command and polling the CPU to LKD status
+ * register to hold the status NOOP
+ */
+static int hl_fw_dynamic_send_clear_cmd(struct hl_device *hdev,
+ struct fw_load_mgr *fw_loader)
+{
+ hl_fw_dynamic_send_cmd(hdev, fw_loader, COMMS_CLR_STS, 0);
+
+ return hl_fw_dynamic_wait_for_status(hdev, fw_loader, COMMS_STS_NOOP,
+ fw_loader->cpu_timeout);
+}
+
+/**
+ * hl_fw_dynamic_send_protocol_cmd - send LKD to FW cmd and wait for ACK
+ *
+ * @hdev: pointer to the habanalabs device structure
+ * @fw_loader: managing structure for loading device's FW
+ * @cmd: LKD to FW cmd code
+ * @size: size of next FW component to be loaded (0 if not necessary)
+ * @wait_ok: if true also wait for OK response from FW
+ * @timeout: timeout for status wait
+ *
+ * @return 0 on success, otherwise non-zero error code
+ *
+ * brief:
+ * when sending protocol command we have the following steps:
+ * - send clear (clear command and verify clear status register)
+ * - send the actual protocol command
+ * - wait for ACK on the protocol command
+ * - send clear
+ * - send NOOP
+ * if, in addition, the specific protocol command should wait for OK then:
+ * - wait for OK
+ * - send clear
+ * - send NOOP
+ *
+ * NOTES:
+ * send clear: this is necessary in order to clear the status register to avoid
+ * leftovers between command
+ * NOOP command: necessary to avoid loop on the clear command by the FW
+ */
+int hl_fw_dynamic_send_protocol_cmd(struct hl_device *hdev,
+ struct fw_load_mgr *fw_loader,
+ enum comms_cmd cmd, unsigned int size,
+ bool wait_ok, u32 timeout)
+{
+ int rc;
+
+ /* first send clear command to clean former commands */
+ rc = hl_fw_dynamic_send_clear_cmd(hdev, fw_loader);
+
+ /* send the actual command */
+ hl_fw_dynamic_send_cmd(hdev, fw_loader, cmd, size);
+
+ /* wait for ACK for the command */
+ rc = hl_fw_dynamic_wait_for_status(hdev, fw_loader, COMMS_STS_ACK,
+ timeout);
+ if (rc)
+ return rc;
+
+ /* clear command to prepare for NOOP command */
+ rc = hl_fw_dynamic_send_clear_cmd(hdev, fw_loader);
+ if (rc)
+ return rc;
+
+ /* send the actual NOOP command */
+ hl_fw_dynamic_send_cmd(hdev, fw_loader, COMMS_NOOP, 0);
+
+ if (!wait_ok)
+ return 0;
+
+ rc = hl_fw_dynamic_wait_for_status(hdev, fw_loader, COMMS_STS_OK,
+ timeout);
+ if (rc)
+ return rc;
+
+ /* clear command to prepare for NOOP command */
+ rc = hl_fw_dynamic_send_clear_cmd(hdev, fw_loader);
+ if (rc)
+ return rc;
+
+ /* send the actual NOOP command */
+ hl_fw_dynamic_send_cmd(hdev, fw_loader, COMMS_NOOP, 0);
+
+ return 0;
+}
+
+/**
+ * hl_fw_compat_crc32 - CRC compatible with FW
+ *
+ * @data: pointer to the data
+ * @size: size of the data
+ *
+ * @return the CRC32 result
+ *
+ * NOTE: kernel's CRC32 differ's from standard CRC32 calculation.
+ * in order to be aligned we need to flip the bits of both the input
+ * initial CRC and kernel's CRC32 result.
+ * in addition both sides use initial CRC of 0,
+ */
+static u32 hl_fw_compat_crc32(u8 *data, size_t size)
+{
+ return ~crc32_le(~((u32)0), data, size);
+}
+
+/**
+ * hl_fw_dynamic_validate_memory_bound - validate memory bounds for memory
+ * transfer (image or descriptor) between
+ * host and FW
+ *
+ * @hdev: pointer to the habanalabs device structure
+ * @addr: device address of memory transfer
+ * @size: memory transter size
+ * @region: PCI memory region
+ *
+ * @return 0 on success, otherwise non-zero error code
+ */
+static int hl_fw_dynamic_validate_memory_bound(struct hl_device *hdev,
+ u64 addr, size_t size,
+ struct pci_mem_region *region)
+{
+ u64 end_addr;
+
+ /* now make sure that the memory transfer is within region's bounds */
+ end_addr = addr + size;
+ if (end_addr >= region->region_base + region->region_size) {
+ dev_err(hdev->dev,
+ "dynamic FW load: memory transfer end address out of memory region bounds. addr: %llx\n",
+ end_addr);
+ return -EIO;
+ }
+
+ /*
+ * now make sure memory transfer is within predefined BAR bounds.
+ * this is to make sure we do not need to set the bar (e.g. for DRAM
+ * memory transfers)
+ */
+ if (end_addr >= region->region_base - region->offset_in_bar +
+ region->bar_size) {
+ dev_err(hdev->dev,
+ "FW image beyond PCI BAR bounds\n");
+ return -EIO;
+ }
+
+ return 0;
+}
+
+/**
+ * hl_fw_dynamic_validate_descriptor - validate FW descriptor
+ *
+ * @hdev: pointer to the habanalabs device structure
+ * @fw_loader: managing structure for loading device's FW
+ * @fw_desc: the descriptor form FW
+ *
+ * @return 0 on success, otherwise non-zero error code
+ */
+static int hl_fw_dynamic_validate_descriptor(struct hl_device *hdev,
+ struct fw_load_mgr *fw_loader,
+ struct lkd_fw_comms_desc *fw_desc)
+{
+ struct pci_mem_region *region;
+ enum pci_region region_id;
+ size_t data_size;
+ u32 data_crc32;
+ u8 *data_ptr;
+ u64 addr;
+ int rc;
+
+ if (le32_to_cpu(fw_desc->header.magic) != HL_COMMS_DESC_MAGIC) {
+ dev_err(hdev->dev, "Invalid magic for dynamic FW descriptor (%x)\n",
+ fw_desc->header.magic);
+ return -EIO;
+ }
+
+ if (fw_desc->header.version != HL_COMMS_DESC_VER) {
+ dev_err(hdev->dev, "Invalid version for dynamic FW descriptor (%x)\n",
+ fw_desc->header.version);
+ return -EIO;
+ }
+
+ /*
+ * calc CRC32 of data without header.
+ * note that no alignment/stride address issues here as all structures
+ * are 64 bit padded
+ */
+ data_size = sizeof(struct lkd_fw_comms_desc) -
+ sizeof(struct comms_desc_header);
+ data_ptr = (u8 *)fw_desc + sizeof(struct comms_desc_header);
+
+ if (le16_to_cpu(fw_desc->header.size) != data_size) {
+ dev_err(hdev->dev,
+ "Invalid descriptor size 0x%x, expected size 0x%zx\n",
+ le16_to_cpu(fw_desc->header.size), data_size);
+ return -EIO;
+ }
+
+ data_crc32 = hl_fw_compat_crc32(data_ptr, data_size);
+
+ if (data_crc32 != le32_to_cpu(fw_desc->header.crc32)) {
+ dev_err(hdev->dev,
+ "CRC32 mismatch for dynamic FW descriptor (%x:%x)\n",
+ data_crc32, fw_desc->header.crc32);
+ return -EIO;
+ }
+
+ /* find memory region to which to copy the image */
+ addr = le64_to_cpu(fw_desc->img_addr);
+ region_id = hl_get_pci_memory_region(hdev, addr);
+ if ((region_id != PCI_REGION_SRAM) &&
+ ((region_id != PCI_REGION_DRAM))) {
+ dev_err(hdev->dev,
+ "Invalid region to copy FW image address=%llx\n", addr);
+ return -EIO;
+ }
+
+ region = &hdev->pci_mem_region[region_id];
+
+ /* store the region for the copy stage */
+ fw_loader->dynamic_loader.image_region = region;
+
+ /*
+ * here we know that the start address is valid, now make sure that the
+ * image is within region's bounds
+ */
+ rc = hl_fw_dynamic_validate_memory_bound(hdev, addr,
+ fw_loader->dynamic_loader.fw_image_size,
+ region);
+ if (rc) {
+ dev_err(hdev->dev,
+ "invalid mem transfer request for FW image\n");
+ return rc;
+ }
+
+ return 0;
+}
+
+static int hl_fw_dynamic_validate_response(struct hl_device *hdev,
+ struct fw_response *response,
+ struct pci_mem_region *region)
+{
+ u64 device_addr;
+ int rc;
+
+ device_addr = region->region_base + response->ram_offset;
+
+ /*
+ * validate that the descriptor is within region's bounds
+ * Note that as the start address was supplied according to the RAM
+ * type- testing only the end address is enough
+ */
+ rc = hl_fw_dynamic_validate_memory_bound(hdev, device_addr,
+ sizeof(struct lkd_fw_comms_desc),
+ region);
+ return rc;
+}
+
+/**
+ * hl_fw_dynamic_read_and_validate_descriptor - read and validate FW descriptor
+ *
+ * @hdev: pointer to the habanalabs device structure
+ * @fw_loader: managing structure for loading device's FW
+ *
+ * @return 0 on success, otherwise non-zero error code
+ */
+static int hl_fw_dynamic_read_and_validate_descriptor(struct hl_device *hdev,
+ struct fw_load_mgr *fw_loader)
+{
+ struct lkd_fw_comms_desc *fw_desc;
+ struct pci_mem_region *region;
+ struct fw_response *response;
+ enum pci_region region_id;
+ void __iomem *src;
+ int rc;
+
+ fw_desc = &fw_loader->dynamic_loader.comm_desc;
+ response = &fw_loader->dynamic_loader.response;
+
+ region_id = (response->ram_type == COMMS_SRAM) ?
+ PCI_REGION_SRAM : PCI_REGION_DRAM;
+
+ region = &hdev->pci_mem_region[region_id];
+
+ rc = hl_fw_dynamic_validate_response(hdev, response, region);
+ if (rc) {
+ dev_err(hdev->dev,
+ "invalid mem transfer request for FW descriptor\n");
+ return rc;
+ }
+
+ /* extract address copy the descriptor from */
+ src = hdev->pcie_bar[region->bar_id] + region->offset_in_bar +
+ response->ram_offset;
+ memcpy_fromio(fw_desc, src, sizeof(struct lkd_fw_comms_desc));
+
+ return hl_fw_dynamic_validate_descriptor(hdev, fw_loader, fw_desc);
+}
+
+/**
+ * hl_fw_dynamic_request_descriptor - handshake with CPU to get FW descriptor
+ *
+ * @hdev: pointer to the habanalabs device structure
+ * @fw_loader: managing structure for loading device's FW
+ * @next_image_size: size to allocate for next FW component
+ *
+ * @return 0 on success, otherwise non-zero error code
+ */
+static int hl_fw_dynamic_request_descriptor(struct hl_device *hdev,
+ struct fw_load_mgr *fw_loader,
+ size_t next_image_size)
+{
+ int rc;
+
+ rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_PREP_DESC,
+ next_image_size, true,
+ fw_loader->cpu_timeout);
+ if (rc)
+ return rc;
+
+ return hl_fw_dynamic_read_and_validate_descriptor(hdev, fw_loader);
+}
+
+/**
+ * hl_fw_dynamic_read_device_fw_version - read FW version to exposed properties
+ *
+ * @hdev: pointer to the habanalabs device structure
+ * @fwc: the firmware component
+ * @fw_version: fw component's version string
+ */
+static void hl_fw_dynamic_read_device_fw_version(struct hl_device *hdev,
+ enum hl_fw_component fwc,
+ const char *fw_version)
+{
+ struct asic_fixed_properties *prop = &hdev->asic_prop;
+ char *preboot_ver, *boot_ver;
+ char btl_ver[32];
+
+ switch (fwc) {
+ case FW_COMP_BOOT_FIT:
+ strscpy(prop->uboot_ver, fw_version, VERSION_MAX_LEN);
+ boot_ver = extract_fw_ver_from_str(prop->uboot_ver);
+ if (boot_ver) {
+ dev_info(hdev->dev, "boot-fit version %s\n", boot_ver);
+ kfree(boot_ver);
+ }
+
+ break;
+ case FW_COMP_PREBOOT:
+ strscpy(prop->preboot_ver, fw_version, VERSION_MAX_LEN);
+ preboot_ver = strnstr(prop->preboot_ver, "Preboot",
+ VERSION_MAX_LEN);
+ if (preboot_ver && preboot_ver != prop->preboot_ver) {
+ strscpy(btl_ver, prop->preboot_ver,
+ min((int) (preboot_ver - prop->preboot_ver),
+ 31));
+ dev_info(hdev->dev, "%s\n", btl_ver);
+ }
+
+ preboot_ver = extract_fw_ver_from_str(prop->preboot_ver);
+ if (preboot_ver) {
+ dev_info(hdev->dev, "preboot version %s\n",
+ preboot_ver);
+ kfree(preboot_ver);
+ }
+
+ break;
+ default:
+ dev_warn(hdev->dev, "Undefined FW component: %d\n", fwc);
+ return;
+ }
+}
+
+/**
+ * hl_fw_dynamic_copy_image - copy image to memory allocated by the FW
+ *
+ * @hdev: pointer to the habanalabs device structure
+ * @fw: fw descriptor
+ * @fw_loader: managing structure for loading device's FW
+ */
+static int hl_fw_dynamic_copy_image(struct hl_device *hdev,
+ const struct firmware *fw,
+ struct fw_load_mgr *fw_loader)
+{
+ struct lkd_fw_comms_desc *fw_desc;
+ struct pci_mem_region *region;
+ void __iomem *dest;
+ u64 addr;
+ int rc;
+
+ fw_desc = &fw_loader->dynamic_loader.comm_desc;
+ addr = le64_to_cpu(fw_desc->img_addr);
+
+ /* find memory region to which to copy the image */
+ region = fw_loader->dynamic_loader.image_region;
+
+ dest = hdev->pcie_bar[region->bar_id] + region->offset_in_bar +
+ (addr - region->region_base);
+
+ rc = hl_fw_copy_fw_to_device(hdev, fw, dest,
+ fw_loader->boot_fit_img.src_off,
+ fw_loader->boot_fit_img.copy_size);
+
+ return rc;
+}
+
+/**
+ * hl_fw_dynamic_copy_msg - copy msg to memory allocated by the FW
+ *
+ * @hdev: pointer to the habanalabs device structure
+ * @msg: message
+ * @fw_loader: managing structure for loading device's FW
+ */
+static int hl_fw_dynamic_copy_msg(struct hl_device *hdev,
+ struct lkd_msg_comms *msg, struct fw_load_mgr *fw_loader)
+{
+ struct lkd_fw_comms_desc *fw_desc;
+ struct pci_mem_region *region;
+ void __iomem *dest;
+ u64 addr;
+ int rc;
+
+ fw_desc = &fw_loader->dynamic_loader.comm_desc;
+ addr = le64_to_cpu(fw_desc->img_addr);
+
+ /* find memory region to which to copy the image */
+ region = fw_loader->dynamic_loader.image_region;
+
+ dest = hdev->pcie_bar[region->bar_id] + region->offset_in_bar +
+ (addr - region->region_base);
+
+ rc = hl_fw_copy_msg_to_device(hdev, msg, dest, 0, 0);
+
+ return rc;
+}
+
+/**
+ * hl_fw_boot_fit_update_state - update internal data structures after boot-fit
+ * is loaded
+ *
+ * @hdev: pointer to the habanalabs device structure
+ * @cpu_boot_dev_sts0_reg: register holding CPU boot dev status 0
+ * @cpu_boot_dev_sts1_reg: register holding CPU boot dev status 1
+ *
+ * @return 0 on success, otherwise non-zero error code
+ */
+static void hl_fw_boot_fit_update_state(struct hl_device *hdev,
+ u32 cpu_boot_dev_sts0_reg,
+ u32 cpu_boot_dev_sts1_reg)
+{
+ struct asic_fixed_properties *prop = &hdev->asic_prop;
+
+ /* Clear reset status since we need to read it again from boot CPU */
+ prop->hard_reset_done_by_fw = false;
+
+ /* Read boot_cpu status bits */
+ if (prop->fw_preboot_cpu_boot_dev_sts0 & CPU_BOOT_DEV_STS0_ENABLED) {
+ prop->fw_bootfit_cpu_boot_dev_sts0 =
+ RREG32(cpu_boot_dev_sts0_reg);
+
+ if (prop->fw_bootfit_cpu_boot_dev_sts0 &
+ CPU_BOOT_DEV_STS0_FW_HARD_RST_EN)
prop->hard_reset_done_by_fw = true;
- } else {
- prop->fw_security_status_valid = 0;
+
+ dev_dbg(hdev->dev, "Firmware boot CPU status0 %#x\n",
+ prop->fw_bootfit_cpu_boot_dev_sts0);
}
- dev_dbg(hdev->dev, "Firmware preboot security status %#x\n",
- security_status);
+ if (prop->fw_cpu_boot_dev_sts1_valid) {
+ prop->fw_bootfit_cpu_boot_dev_sts1 =
+ RREG32(cpu_boot_dev_sts1_reg);
- dev_dbg(hdev->dev, "Firmware preboot hard-reset is %s\n",
+ dev_dbg(hdev->dev, "Firmware boot CPU status1 %#x\n",
+ prop->fw_bootfit_cpu_boot_dev_sts1);
+ }
+
+ dev_dbg(hdev->dev, "Firmware boot CPU hard-reset is %s\n",
prop->hard_reset_done_by_fw ? "enabled" : "disabled");
+}
+
+static void hl_fw_dynamic_update_linux_interrupt_if(struct hl_device *hdev)
+{
+ struct cpu_dyn_regs *dyn_regs =
+ &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs;
+
+ /* Check whether all 3 interrupt interfaces are set, if not use a
+ * single interface
+ */
+ if (!hdev->asic_prop.gic_interrupts_enable &&
+ !(hdev->asic_prop.fw_app_cpu_boot_dev_sts0 &
+ CPU_BOOT_DEV_STS0_MULTI_IRQ_POLL_EN)) {
+ dyn_regs->gic_host_halt_irq = dyn_regs->gic_host_irq_ctrl;
+ dyn_regs->gic_host_ints_irq = dyn_regs->gic_host_irq_ctrl;
+
+ dev_warn(hdev->dev,
+ "Using a single interrupt interface towards cpucp");
+ }
+}
+/**
+ * hl_fw_dynamic_load_image - load FW image using dynamic protocol
+ *
+ * @hdev: pointer to the habanalabs device structure
+ * @fw_loader: managing structure for loading device's FW
+ * @load_fwc: the FW component to be loaded
+ * @img_ld_timeout: image load timeout
+ *
+ * @return 0 on success, otherwise non-zero error code
+ */
+static int hl_fw_dynamic_load_image(struct hl_device *hdev,
+ struct fw_load_mgr *fw_loader,
+ enum hl_fw_component load_fwc,
+ u32 img_ld_timeout)
+{
+ enum hl_fw_component cur_fwc;
+ const struct firmware *fw;
+ char *fw_name;
+ int rc = 0;
+
+ /*
+ * when loading image we have one of 2 scenarios:
+ * 1. current FW component is preboot and we want to load boot-fit
+ * 2. current FW component is boot-fit and we want to load linux
+ */
+ if (load_fwc == FW_COMP_BOOT_FIT) {
+ cur_fwc = FW_COMP_PREBOOT;
+ fw_name = fw_loader->boot_fit_img.image_name;
+ } else {
+ cur_fwc = FW_COMP_BOOT_FIT;
+ fw_name = fw_loader->linux_img.image_name;
+ }
+
+ /* request FW in order to communicate to FW the size to be allocated */
+ rc = hl_request_fw(hdev, &fw, fw_name);
+ if (rc)
+ return rc;
+
+ /* store the image size for future validation */
+ fw_loader->dynamic_loader.fw_image_size = fw->size;
+
+ rc = hl_fw_dynamic_request_descriptor(hdev, fw_loader, fw->size);
+ if (rc)
+ goto release_fw;
+
+ /* read preboot version */
+ hl_fw_dynamic_read_device_fw_version(hdev, cur_fwc,
+ fw_loader->dynamic_loader.comm_desc.cur_fw_ver);
+
+
+ /* update state according to boot stage */
+ if (cur_fwc == FW_COMP_BOOT_FIT) {
+ struct cpu_dyn_regs *dyn_regs;
+
+ dyn_regs = &fw_loader->dynamic_loader.comm_desc.cpu_dyn_regs;
+ hl_fw_boot_fit_update_state(hdev,
+ le32_to_cpu(dyn_regs->cpu_boot_dev_sts0),
+ le32_to_cpu(dyn_regs->cpu_boot_dev_sts1));
+ }
+
+ /* copy boot fit to space allocated by FW */
+ rc = hl_fw_dynamic_copy_image(hdev, fw, fw_loader);
+ if (rc)
+ goto release_fw;
- dev_info(hdev->dev, "firmware-level security is %s\n",
- prop->fw_security_disabled ? "disabled" : "enabled");
+ rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_DATA_RDY,
+ 0, true,
+ fw_loader->cpu_timeout);
+ if (rc)
+ goto release_fw;
+
+ rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_EXEC,
+ 0, false,
+ img_ld_timeout);
+release_fw:
+ hl_release_firmware(fw);
+ return rc;
+}
+
+static int hl_fw_dynamic_wait_for_boot_fit_active(struct hl_device *hdev,
+ struct fw_load_mgr *fw_loader)
+{
+ struct dynamic_fw_load_mgr *dyn_loader;
+ u32 status;
+ int rc;
+
+ dyn_loader = &fw_loader->dynamic_loader;
+
+ /* Make sure CPU boot-loader is running */
+ rc = hl_poll_timeout(
+ hdev,
+ le32_to_cpu(dyn_loader->comm_desc.cpu_dyn_regs.cpu_boot_status),
+ status,
+ (status == CPU_BOOT_STATUS_NIC_FW_RDY) ||
+ (status == CPU_BOOT_STATUS_READY_TO_BOOT),
+ FW_CPU_STATUS_POLL_INTERVAL_USEC,
+ dyn_loader->wait_for_bl_timeout);
+ if (rc) {
+ dev_err(hdev->dev, "failed to wait for boot\n");
+ return rc;
+ }
+
+ dev_dbg(hdev->dev, "uboot status = %d\n", status);
return 0;
}
-int hl_fw_init_cpu(struct hl_device *hdev, u32 cpu_boot_status_reg,
- u32 msg_to_cpu_reg, u32 cpu_msg_status_reg,
- u32 cpu_security_boot_status_reg, u32 boot_err0_reg,
- bool skip_bmc, u32 cpu_timeout, u32 boot_fit_timeout)
+static int hl_fw_dynamic_wait_for_linux_active(struct hl_device *hdev,
+ struct fw_load_mgr *fw_loader)
{
- struct asic_fixed_properties *prop = &hdev->asic_prop;
+ struct dynamic_fw_load_mgr *dyn_loader;
u32 status;
int rc;
+ dyn_loader = &fw_loader->dynamic_loader;
+
+ /* Make sure CPU boot-loader is running */
+
+ rc = hl_poll_timeout(
+ hdev,
+ le32_to_cpu(dyn_loader->comm_desc.cpu_dyn_regs.cpu_boot_status),
+ status,
+ (status == CPU_BOOT_STATUS_SRAM_AVAIL),
+ FW_CPU_STATUS_POLL_INTERVAL_USEC,
+ fw_loader->cpu_timeout);
+ if (rc) {
+ dev_err(hdev->dev, "failed to wait for Linux\n");
+ return rc;
+ }
+
+ dev_dbg(hdev->dev, "Boot status = %d\n", status);
+ return 0;
+}
+
+/**
+ * hl_fw_linux_update_state - update internal data structures after Linux
+ * is loaded.
+ * Note: Linux initialization is comprised mainly
+ * of two stages - loading kernel (SRAM_AVAIL)
+ * & loading ARMCP.
+ * Therefore reading boot device status in any of
+ * these stages might result in different values.
+ *
+ * @hdev: pointer to the habanalabs device structure
+ * @cpu_boot_dev_sts0_reg: register holding CPU boot dev status 0
+ * @cpu_boot_dev_sts1_reg: register holding CPU boot dev status 1
+ *
+ * @return 0 on success, otherwise non-zero error code
+ */
+static void hl_fw_linux_update_state(struct hl_device *hdev,
+ u32 cpu_boot_dev_sts0_reg,
+ u32 cpu_boot_dev_sts1_reg)
+{
+ struct asic_fixed_properties *prop = &hdev->asic_prop;
+
+ hdev->fw_loader.linux_loaded = true;
+
+ /* Clear reset status since we need to read again from app */
+ prop->hard_reset_done_by_fw = false;
+
+ /* Read FW application security bits */
+ if (prop->fw_cpu_boot_dev_sts0_valid) {
+ prop->fw_app_cpu_boot_dev_sts0 =
+ RREG32(cpu_boot_dev_sts0_reg);
+
+ if (prop->fw_app_cpu_boot_dev_sts0 &
+ CPU_BOOT_DEV_STS0_FW_HARD_RST_EN)
+ prop->hard_reset_done_by_fw = true;
+
+ if (prop->fw_app_cpu_boot_dev_sts0 &
+ CPU_BOOT_DEV_STS0_GIC_PRIVILEGED_EN)
+ prop->gic_interrupts_enable = false;
+
+ dev_dbg(hdev->dev,
+ "Firmware application CPU status0 %#x\n",
+ prop->fw_app_cpu_boot_dev_sts0);
+
+ dev_dbg(hdev->dev, "GIC controller is %s\n",
+ prop->gic_interrupts_enable ?
+ "enabled" : "disabled");
+ }
+
+ if (prop->fw_cpu_boot_dev_sts1_valid) {
+ prop->fw_app_cpu_boot_dev_sts1 =
+ RREG32(cpu_boot_dev_sts1_reg);
+
+ dev_dbg(hdev->dev,
+ "Firmware application CPU status1 %#x\n",
+ prop->fw_app_cpu_boot_dev_sts1);
+ }
+
+ dev_dbg(hdev->dev, "Firmware application CPU hard-reset is %s\n",
+ prop->hard_reset_done_by_fw ? "enabled" : "disabled");
+
+ dev_info(hdev->dev, "Successfully loaded firmware to device\n");
+}
+
+/**
+ * hl_fw_dynamic_report_reset_cause - send a COMMS message with the cause
+ * of the newly triggered hard reset
+ *
+ * @hdev: pointer to the habanalabs device structure
+ * @fw_loader: managing structure for loading device's FW
+ * @reset_cause: enumerated cause for the recent hard reset
+ *
+ * @return 0 on success, otherwise non-zero error code
+ */
+static int hl_fw_dynamic_report_reset_cause(struct hl_device *hdev,
+ struct fw_load_mgr *fw_loader,
+ enum comms_reset_cause reset_cause)
+{
+ struct lkd_msg_comms msg;
+ int rc;
+
+ memset(&msg, 0, sizeof(msg));
+
+ /* create message to be sent */
+ msg.header.type = HL_COMMS_RESET_CAUSE_TYPE;
+ msg.header.size = cpu_to_le16(sizeof(struct comms_msg_header));
+ msg.header.magic = cpu_to_le32(HL_COMMS_MSG_MAGIC);
+
+ msg.reset_cause = reset_cause;
+
+ rc = hl_fw_dynamic_request_descriptor(hdev, fw_loader,
+ sizeof(struct lkd_msg_comms));
+ if (rc)
+ return rc;
+
+ /* copy message to space allocated by FW */
+ rc = hl_fw_dynamic_copy_msg(hdev, &msg, fw_loader);
+ if (rc)
+ return rc;
+
+ rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_DATA_RDY,
+ 0, true,
+ fw_loader->cpu_timeout);
+ if (rc)
+ return rc;
+
+ rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_EXEC,
+ 0, true,
+ fw_loader->cpu_timeout);
+ if (rc)
+ return rc;
+
+ return 0;
+}
+
+/**
+ * hl_fw_dynamic_init_cpu - initialize the device CPU using dynamic protocol
+ *
+ * @hdev: pointer to the habanalabs device structure
+ * @fw_loader: managing structure for loading device's FW
+ *
+ * @return 0 on success, otherwise non-zero error code
+ *
+ * brief: the dynamic protocol is master (LKD) slave (FW CPU) protocol.
+ * the communication is done using registers:
+ * - LKD command register
+ * - FW status register
+ * the protocol is race free. this goal is achieved by splitting the requests
+ * and response to known synchronization points between the LKD and the FW.
+ * each response to LKD request is known and bound to a predefined timeout.
+ * in case of timeout expiration without the desired status from FW- the
+ * protocol (and hence the boot) will fail.
+ */
+static int hl_fw_dynamic_init_cpu(struct hl_device *hdev,
+ struct fw_load_mgr *fw_loader)
+{
+ struct cpu_dyn_regs *dyn_regs;
+ int rc;
+
+ dev_info(hdev->dev,
+ "Loading firmware to device, may take some time...\n");
+
+ dyn_regs = &fw_loader->dynamic_loader.comm_desc.cpu_dyn_regs;
+
+ rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader, COMMS_RST_STATE,
+ 0, true,
+ fw_loader->cpu_timeout);
+ if (rc)
+ goto protocol_err;
+
+ if (hdev->curr_reset_cause) {
+ rc = hl_fw_dynamic_report_reset_cause(hdev, fw_loader,
+ hdev->curr_reset_cause);
+ if (rc)
+ goto protocol_err;
+
+ /* Clear current reset cause */
+ hdev->curr_reset_cause = HL_RESET_CAUSE_UNKNOWN;
+ }
+
+ if (!(hdev->fw_components & FW_TYPE_BOOT_CPU)) {
+ rc = hl_fw_dynamic_request_descriptor(hdev, fw_loader, 0);
+ if (rc)
+ goto protocol_err;
+
+ /* read preboot version */
+ hl_fw_dynamic_read_device_fw_version(hdev, FW_COMP_PREBOOT,
+ fw_loader->dynamic_loader.comm_desc.cur_fw_ver);
+ return 0;
+ }
+
+ /* load boot fit to FW */
+ rc = hl_fw_dynamic_load_image(hdev, fw_loader, FW_COMP_BOOT_FIT,
+ fw_loader->boot_fit_timeout);
+ if (rc) {
+ dev_err(hdev->dev, "failed to load boot fit\n");
+ goto protocol_err;
+ }
+
+ rc = hl_fw_dynamic_wait_for_boot_fit_active(hdev, fw_loader);
+ if (rc)
+ goto protocol_err;
+
+ /* Enable DRAM scrambling before Linux boot and after successful
+ * UBoot
+ */
+ hdev->asic_funcs->init_cpu_scrambler_dram(hdev);
+
+ if (!(hdev->fw_components & FW_TYPE_LINUX)) {
+ dev_info(hdev->dev, "Skip loading Linux F/W\n");
+ return 0;
+ }
+
+ if (fw_loader->skip_bmc) {
+ rc = hl_fw_dynamic_send_protocol_cmd(hdev, fw_loader,
+ COMMS_SKIP_BMC, 0,
+ true,
+ fw_loader->cpu_timeout);
+ if (rc) {
+ dev_err(hdev->dev, "failed to load boot fit\n");
+ goto protocol_err;
+ }
+ }
+
+ /* load Linux image to FW */
+ rc = hl_fw_dynamic_load_image(hdev, fw_loader, FW_COMP_LINUX,
+ fw_loader->cpu_timeout);
+ if (rc) {
+ dev_err(hdev->dev, "failed to load Linux\n");
+ goto protocol_err;
+ }
+
+ rc = hl_fw_dynamic_wait_for_linux_active(hdev, fw_loader);
+ if (rc)
+ goto protocol_err;
+
+ hl_fw_linux_update_state(hdev, le32_to_cpu(dyn_regs->cpu_boot_dev_sts0),
+ le32_to_cpu(dyn_regs->cpu_boot_dev_sts1));
+
+ hl_fw_dynamic_update_linux_interrupt_if(hdev);
+
+ return 0;
+
+protocol_err:
+ fw_read_errors(hdev, le32_to_cpu(dyn_regs->cpu_boot_err0),
+ le32_to_cpu(dyn_regs->cpu_boot_err1),
+ le32_to_cpu(dyn_regs->cpu_boot_dev_sts0),
+ le32_to_cpu(dyn_regs->cpu_boot_dev_sts1));
+ return rc;
+}
+
+/**
+ * hl_fw_static_init_cpu - initialize the device CPU using static protocol
+ *
+ * @hdev: pointer to the habanalabs device structure
+ * @fw_loader: managing structure for loading device's FW
+ *
+ * @return 0 on success, otherwise non-zero error code
+ */
+static int hl_fw_static_init_cpu(struct hl_device *hdev,
+ struct fw_load_mgr *fw_loader)
+{
+ u32 cpu_msg_status_reg, cpu_timeout, msg_to_cpu_reg, status;
+ u32 cpu_boot_dev_status0_reg, cpu_boot_dev_status1_reg;
+ struct static_fw_load_mgr *static_loader;
+ u32 cpu_boot_status_reg;
+ int rc;
+
if (!(hdev->fw_components & FW_TYPE_BOOT_CPU))
return 0;
+ /* init common loader parameters */
+ cpu_timeout = fw_loader->cpu_timeout;
+
+ /* init static loader parameters */
+ static_loader = &fw_loader->static_loader;
+ cpu_msg_status_reg = static_loader->cpu_cmd_status_to_host_reg;
+ msg_to_cpu_reg = static_loader->kmd_msg_to_cpu_reg;
+ cpu_boot_dev_status0_reg = static_loader->cpu_boot_dev_status0_reg;
+ cpu_boot_dev_status1_reg = static_loader->cpu_boot_dev_status1_reg;
+ cpu_boot_status_reg = static_loader->cpu_boot_status_reg;
+
dev_info(hdev->dev, "Going to wait for device boot (up to %lds)\n",
cpu_timeout / USEC_PER_SEC);
@@ -925,8 +2364,8 @@ int hl_fw_init_cpu(struct hl_device *hdev, u32 cpu_boot_status_reg,
cpu_boot_status_reg,
status,
status == CPU_BOOT_STATUS_WAITING_FOR_BOOT_FIT,
- 10000,
- boot_fit_timeout);
+ FW_CPU_STATUS_POLL_INTERVAL_USEC,
+ fw_loader->boot_fit_timeout);
if (rc) {
dev_dbg(hdev->dev,
@@ -948,8 +2387,8 @@ int hl_fw_init_cpu(struct hl_device *hdev, u32 cpu_boot_status_reg,
cpu_msg_status_reg,
status,
status == CPU_MSG_OK,
- 10000,
- boot_fit_timeout);
+ FW_CPU_STATUS_POLL_INTERVAL_USEC,
+ fw_loader->boot_fit_timeout);
if (rc) {
dev_err(hdev->dev,
@@ -970,33 +2409,17 @@ int hl_fw_init_cpu(struct hl_device *hdev, u32 cpu_boot_status_reg,
(status == CPU_BOOT_STATUS_NIC_FW_RDY) ||
(status == CPU_BOOT_STATUS_READY_TO_BOOT) ||
(status == CPU_BOOT_STATUS_SRAM_AVAIL),
- 10000,
+ FW_CPU_STATUS_POLL_INTERVAL_USEC,
cpu_timeout);
dev_dbg(hdev->dev, "uboot status = %d\n", status);
/* Read U-Boot version now in case we will later fail */
- hdev->asic_funcs->read_device_fw_version(hdev, FW_COMP_UBOOT);
-
- /* Clear reset status since we need to read it again from boot CPU */
- prop->hard_reset_done_by_fw = false;
-
- /* Read boot_cpu security bits */
- if (prop->fw_security_status_valid) {
- prop->fw_boot_cpu_security_map =
- RREG32(cpu_security_boot_status_reg);
-
- if (prop->fw_boot_cpu_security_map &
- CPU_BOOT_DEV_STS0_FW_HARD_RST_EN)
- prop->hard_reset_done_by_fw = true;
-
- dev_dbg(hdev->dev,
- "Firmware boot CPU security status %#x\n",
- prop->fw_boot_cpu_security_map);
- }
+ hl_fw_static_read_device_fw_version(hdev, FW_COMP_BOOT_FIT);
- dev_dbg(hdev->dev, "Firmware boot CPU hard-reset is %s\n",
- prop->hard_reset_done_by_fw ? "enabled" : "disabled");
+ /* update state according to boot stage */
+ hl_fw_boot_fit_update_state(hdev, cpu_boot_dev_status0_reg,
+ cpu_boot_dev_status1_reg);
if (rc) {
detect_cpu_boot_status(hdev, status);
@@ -1004,13 +2427,21 @@ int hl_fw_init_cpu(struct hl_device *hdev, u32 cpu_boot_status_reg,
goto out;
}
+ /* Enable DRAM scrambling before Linux boot and after successful
+ * UBoot
+ */
+ hdev->asic_funcs->init_cpu_scrambler_dram(hdev);
+
if (!(hdev->fw_components & FW_TYPE_LINUX)) {
dev_info(hdev->dev, "Skip loading Linux F/W\n");
+ rc = 0;
goto out;
}
- if (status == CPU_BOOT_STATUS_SRAM_AVAIL)
+ if (status == CPU_BOOT_STATUS_SRAM_AVAIL) {
+ rc = 0;
goto out;
+ }
dev_info(hdev->dev,
"Loading firmware to device, may take some time...\n");
@@ -1019,7 +2450,7 @@ int hl_fw_init_cpu(struct hl_device *hdev, u32 cpu_boot_status_reg,
if (rc)
goto out;
- if (skip_bmc) {
+ if (fw_loader->skip_bmc) {
WREG32(msg_to_cpu_reg, KMD_MSG_SKIP_BMC);
rc = hl_poll_timeout(
@@ -1027,7 +2458,7 @@ int hl_fw_init_cpu(struct hl_device *hdev, u32 cpu_boot_status_reg,
cpu_boot_status_reg,
status,
(status == CPU_BOOT_STATUS_BMC_WAITING_SKIPPED),
- 10000,
+ FW_CPU_STATUS_POLL_INTERVAL_USEC,
cpu_timeout);
if (rc) {
@@ -1047,7 +2478,7 @@ int hl_fw_init_cpu(struct hl_device *hdev, u32 cpu_boot_status_reg,
cpu_boot_status_reg,
status,
(status == CPU_BOOT_STATUS_SRAM_AVAIL),
- 10000,
+ FW_CPU_STATUS_POLL_INTERVAL_USEC,
cpu_timeout);
/* Clear message */
@@ -1066,36 +2497,43 @@ int hl_fw_init_cpu(struct hl_device *hdev, u32 cpu_boot_status_reg,
goto out;
}
- rc = fw_read_errors(hdev, boot_err0_reg, cpu_security_boot_status_reg);
+ rc = fw_read_errors(hdev, fw_loader->static_loader.boot_err0_reg,
+ fw_loader->static_loader.boot_err1_reg,
+ cpu_boot_dev_status0_reg,
+ cpu_boot_dev_status1_reg);
if (rc)
return rc;
- /* Clear reset status since we need to read again from app */
- prop->hard_reset_done_by_fw = false;
-
- /* Read FW application security bits */
- if (prop->fw_security_status_valid) {
- prop->fw_app_security_map =
- RREG32(cpu_security_boot_status_reg);
-
- if (prop->fw_app_security_map &
- CPU_BOOT_DEV_STS0_FW_HARD_RST_EN)
- prop->hard_reset_done_by_fw = true;
-
- dev_dbg(hdev->dev,
- "Firmware application CPU security status %#x\n",
- prop->fw_app_security_map);
- }
-
- dev_dbg(hdev->dev, "Firmware application CPU hard-reset is %s\n",
- prop->hard_reset_done_by_fw ? "enabled" : "disabled");
-
- dev_info(hdev->dev, "Successfully loaded firmware to device\n");
+ hl_fw_linux_update_state(hdev, cpu_boot_dev_status0_reg,
+ cpu_boot_dev_status1_reg);
return 0;
out:
- fw_read_errors(hdev, boot_err0_reg, cpu_security_boot_status_reg);
+ fw_read_errors(hdev, fw_loader->static_loader.boot_err0_reg,
+ fw_loader->static_loader.boot_err1_reg,
+ cpu_boot_dev_status0_reg,
+ cpu_boot_dev_status1_reg);
return rc;
}
+
+/**
+ * hl_fw_init_cpu - initialize the device CPU
+ *
+ * @hdev: pointer to the habanalabs device structure
+ *
+ * @return 0 on success, otherwise non-zero error code
+ *
+ * perform necessary initializations for device's CPU. takes into account if
+ * init protocol is static or dynamic.
+ */
+int hl_fw_init_cpu(struct hl_device *hdev)
+{
+ struct asic_fixed_properties *prop = &hdev->asic_prop;
+ struct fw_load_mgr *fw_loader = &hdev->fw_loader;
+
+ return prop->dynamic_fw_load ?
+ hl_fw_dynamic_init_cpu(hdev, fw_loader) :
+ hl_fw_static_init_cpu(hdev, fw_loader);
+}
diff --git a/drivers/misc/habanalabs/common/habanalabs.h b/drivers/misc/habanalabs/common/habanalabs.h
index 6579f8767abd..6b3cdd7e068a 100644
--- a/drivers/misc/habanalabs/common/habanalabs.h
+++ b/drivers/misc/habanalabs/common/habanalabs.h
@@ -48,6 +48,7 @@
#define HL_PENDING_RESET_LONG_SEC 60
#define HL_HARD_RESET_MAX_TIMEOUT 120
+#define HL_PLDM_HARD_RESET_MAX_TIMEOUT (HL_HARD_RESET_MAX_TIMEOUT * 3)
#define HL_DEVICE_TIMEOUT_USEC 1000000 /* 1 s */
@@ -115,10 +116,18 @@ enum hl_mmu_page_table_location {
*
* - HL_RESET_HEARTBEAT
* Set if reset is due to heartbeat
+ *
+ * - HL_RESET_TDR
+ * Set if reset is due to TDR
+ *
+ * - HL_RESET_DEVICE_RELEASE
+ * Set if reset is due to device release
*/
#define HL_RESET_HARD (1 << 0)
#define HL_RESET_FROM_RESET_THREAD (1 << 1)
#define HL_RESET_HEARTBEAT (1 << 2)
+#define HL_RESET_TDR (1 << 3)
+#define HL_RESET_DEVICE_RELEASE (1 << 4)
#define HL_MAX_SOBS_PER_MONITOR 8
@@ -178,12 +187,14 @@ enum hl_pci_match_mode {
/**
* enum hl_fw_component - F/W components to read version through registers.
- * @FW_COMP_UBOOT: u-boot.
+ * @FW_COMP_BOOT_FIT: boot fit.
* @FW_COMP_PREBOOT: preboot.
+ * @FW_COMP_LINUX: linux.
*/
enum hl_fw_component {
- FW_COMP_UBOOT,
- FW_COMP_PREBOOT
+ FW_COMP_BOOT_FIT,
+ FW_COMP_PREBOOT,
+ FW_COMP_LINUX,
};
/**
@@ -420,12 +431,24 @@ struct hl_mmu_properties {
* @cb_pool_cb_size: size of each CB in the CB pool.
* @max_pending_cs: maximum of concurrent pending command submissions
* @max_queues: maximum amount of queues in the system
- * @fw_boot_cpu_security_map: bitmap representation of boot cpu security status
- * reported by FW, bit description can be found in
- * CPU_BOOT_DEV_STS*
- * @fw_app_security_map: bitmap representation of application security status
- * reported by FW, bit description can be found in
- * CPU_BOOT_DEV_STS*
+ * @fw_preboot_cpu_boot_dev_sts0: bitmap representation of preboot cpu
+ * capabilities reported by FW, bit description
+ * can be found in CPU_BOOT_DEV_STS0
+ * @fw_preboot_cpu_boot_dev_sts1: bitmap representation of preboot cpu
+ * capabilities reported by FW, bit description
+ * can be found in CPU_BOOT_DEV_STS1
+ * @fw_bootfit_cpu_boot_dev_sts0: bitmap representation of boot cpu security
+ * status reported by FW, bit description can be
+ * found in CPU_BOOT_DEV_STS0
+ * @fw_bootfit_cpu_boot_dev_sts1: bitmap representation of boot cpu security
+ * status reported by FW, bit description can be
+ * found in CPU_BOOT_DEV_STS1
+ * @fw_app_cpu_boot_dev_sts0: bitmap representation of application security
+ * status reported by FW, bit description can be
+ * found in CPU_BOOT_DEV_STS0
+ * @fw_app_cpu_boot_dev_sts1: bitmap representation of application security
+ * status reported by FW, bit description can be
+ * found in CPU_BOOT_DEV_STS1
* @collective_first_sob: first sync object available for collective use
* @collective_first_mon: first monitor available for collective use
* @sync_stream_first_sob: first sync object available for sync stream use
@@ -438,14 +461,19 @@ struct hl_mmu_properties {
* @user_interrupt_count: number of user interrupts.
* @tpc_enabled_mask: which TPCs are enabled.
* @completion_queues_count: number of completion queues.
- * @fw_security_disabled: true if security measures are disabled in firmware,
- * false otherwise
- * @fw_security_status_valid: security status bits are valid and can be fetched
- * from BOOT_DEV_STS0
+ * @fw_security_enabled: true if security measures are enabled in firmware,
+ * false otherwise
+ * @fw_cpu_boot_dev_sts0_valid: status bits are valid and can be fetched from
+ * BOOT_DEV_STS0
+ * @fw_cpu_boot_dev_sts1_valid: status bits are valid and can be fetched from
+ * BOOT_DEV_STS1
* @dram_supports_virtual_memory: is there an MMU towards the DRAM
* @hard_reset_done_by_fw: true if firmware is handling hard reset flow
* @num_functional_hbms: number of functional HBMs in each DCORE.
* @iatu_done_by_fw: true if iATU configuration is being done by FW.
+ * @dynamic_fw_load: is dynamic FW load is supported.
+ * @gic_interrupts_enable: true if FW is not blocking GIC controller,
+ * false otherwise.
*/
struct asic_fixed_properties {
struct hw_queue_properties *hw_queues_props;
@@ -491,8 +519,12 @@ struct asic_fixed_properties {
u32 cb_pool_cb_size;
u32 max_pending_cs;
u32 max_queues;
- u32 fw_boot_cpu_security_map;
- u32 fw_app_security_map;
+ u32 fw_preboot_cpu_boot_dev_sts0;
+ u32 fw_preboot_cpu_boot_dev_sts1;
+ u32 fw_bootfit_cpu_boot_dev_sts0;
+ u32 fw_bootfit_cpu_boot_dev_sts1;
+ u32 fw_app_cpu_boot_dev_sts0;
+ u32 fw_app_cpu_boot_dev_sts1;
u16 collective_first_sob;
u16 collective_first_mon;
u16 sync_stream_first_sob;
@@ -504,12 +536,15 @@ struct asic_fixed_properties {
u16 user_interrupt_count;
u8 tpc_enabled_mask;
u8 completion_queues_count;
- u8 fw_security_disabled;
- u8 fw_security_status_valid;
+ u8 fw_security_enabled;
+ u8 fw_cpu_boot_dev_sts0_valid;
+ u8 fw_cpu_boot_dev_sts1_valid;
u8 dram_supports_virtual_memory;
u8 hard_reset_done_by_fw;
u8 num_functional_hbms;
u8 iatu_done_by_fw;
+ u8 dynamic_fw_load;
+ u8 gic_interrupts_enable;
};
/**
@@ -750,12 +785,19 @@ struct hl_user_pending_interrupt {
* @kernel_address: holds the queue's kernel virtual address
* @bus_address: holds the queue's DMA address
* @ci: ci inside the queue
+ * @prev_eqe_index: the index of the previous event queue entry. The index of
+ * the current entry's index must be +1 of the previous one.
+ * @check_eqe_index: do we need to check the index of the current entry vs. the
+ * previous one. This is for backward compatibility with older
+ * firmwares
*/
struct hl_eq {
struct hl_device *hdev;
void *kernel_address;
dma_addr_t bus_address;
u32 ci;
+ u32 prev_eqe_index;
+ bool check_eqe_index;
};
@@ -812,6 +854,132 @@ enum div_select_defs {
DIV_SEL_DIVIDED_PLL = 3,
};
+enum pci_region {
+ PCI_REGION_CFG,
+ PCI_REGION_SRAM,
+ PCI_REGION_DRAM,
+ PCI_REGION_SP_SRAM,
+ PCI_REGION_NUMBER,
+};
+
+/**
+ * struct pci_mem_region - describe memory region in a PCI bar
+ * @region_base: region base address
+ * @region_size: region size
+ * @bar_size: size of the BAR
+ * @offset_in_bar: region offset into the bar
+ * @bar_id: bar ID of the region
+ * @used: if used 1, otherwise 0
+ */
+struct pci_mem_region {
+ u64 region_base;
+ u64 region_size;
+ u64 bar_size;
+ u32 offset_in_bar;
+ u8 bar_id;
+ u8 used;
+};
+
+/**
+ * struct static_fw_load_mgr - static FW load manager
+ * @preboot_version_max_off: max offset to preboot version
+ * @boot_fit_version_max_off: max offset to boot fit version
+ * @kmd_msg_to_cpu_reg: register address for KDM->CPU messages
+ * @cpu_cmd_status_to_host_reg: register address for CPU command status response
+ * @cpu_boot_status_reg: boot status register
+ * @cpu_boot_dev_status0_reg: boot device status register 0
+ * @cpu_boot_dev_status1_reg: boot device status register 1
+ * @boot_err0_reg: boot error register 0
+ * @boot_err1_reg: boot error register 1
+ * @preboot_version_offset_reg: SRAM offset to preboot version register
+ * @boot_fit_version_offset_reg: SRAM offset to boot fit version register
+ * @sram_offset_mask: mask for getting offset into the SRAM
+ * @cpu_reset_wait_msec: used when setting WFE via kmd_msg_to_cpu_reg
+ */
+struct static_fw_load_mgr {
+ u64 preboot_version_max_off;
+ u64 boot_fit_version_max_off;
+ u32 kmd_msg_to_cpu_reg;
+ u32 cpu_cmd_status_to_host_reg;
+ u32 cpu_boot_status_reg;
+ u32 cpu_boot_dev_status0_reg;
+ u32 cpu_boot_dev_status1_reg;
+ u32 boot_err0_reg;
+ u32 boot_err1_reg;
+ u32 preboot_version_offset_reg;
+ u32 boot_fit_version_offset_reg;
+ u32 sram_offset_mask;
+ u32 cpu_reset_wait_msec;
+};
+
+/**
+ * struct fw_response - FW response to LKD command
+ * @ram_offset: descriptor offset into the RAM
+ * @ram_type: RAM type containing the descriptor (SRAM/DRAM)
+ * @status: command status
+ */
+struct fw_response {
+ u32 ram_offset;
+ u8 ram_type;
+ u8 status;
+};
+
+/**
+ * struct dynamic_fw_load_mgr - dynamic FW load manager
+ * @response: FW to LKD response
+ * @comm_desc: the communication descriptor with FW
+ * @image_region: region to copy the FW image to
+ * @fw_image_size: size of FW image to load
+ * @wait_for_bl_timeout: timeout for waiting for boot loader to respond
+ */
+struct dynamic_fw_load_mgr {
+ struct fw_response response;
+ struct lkd_fw_comms_desc comm_desc;
+ struct pci_mem_region *image_region;
+ size_t fw_image_size;
+ u32 wait_for_bl_timeout;
+};
+
+/**
+ * struct fw_image_props - properties of FW image
+ * @image_name: name of the image
+ * @src_off: offset in src FW to copy from
+ * @copy_size: amount of bytes to copy (0 to copy the whole binary)
+ */
+struct fw_image_props {
+ char *image_name;
+ u32 src_off;
+ u32 copy_size;
+};
+
+/**
+ * struct fw_load_mgr - manager FW loading process
+ * @dynamic_loader: specific structure for dynamic load
+ * @static_loader: specific structure for static load
+ * @boot_fit_img: boot fit image properties
+ * @linux_img: linux image properties
+ * @cpu_timeout: CPU response timeout in usec
+ * @boot_fit_timeout: Boot fit load timeout in usec
+ * @skip_bmc: should BMC be skipped
+ * @sram_bar_id: SRAM bar ID
+ * @dram_bar_id: DRAM bar ID
+ * @linux_loaded: true if linux was loaded so far
+ */
+struct fw_load_mgr {
+ union {
+ struct dynamic_fw_load_mgr dynamic_loader;
+ struct static_fw_load_mgr static_loader;
+ };
+ struct fw_image_props boot_fit_img;
+ struct fw_image_props linux_img;
+ u32 cpu_timeout;
+ u32 boot_fit_timeout;
+ u8 skip_bmc;
+ u8 sram_bar_id;
+ u8 dram_bar_id;
+ u8 linux_loaded;
+};
+
/**
* struct hl_asic_funcs - ASIC specific functions that are can be called from
* common code.
@@ -901,8 +1069,6 @@ enum div_select_defs {
* @ctx_fini: context dependent cleanup.
* @get_clk_rate: Retrieve the ASIC current and maximum clock rate in MHz
* @get_queue_id_for_cq: Get the H/W queue id related to the given CQ index.
- * @read_device_fw_version: read the device's firmware versions that are
- * contained in registers
* @load_firmware_to_device: load the firmware to the device's memory
* @load_boot_fit_to_device: load boot fit to device's memory
* @get_signal_cb_size: Get signal CB size.
@@ -933,6 +1099,8 @@ enum div_select_defs {
* @get_msi_info: Retrieve asic-specific MSI ID of the f/w async event
* @map_pll_idx_to_fw_idx: convert driver specific per asic PLL index to
* generic f/w compatible PLL Indexes
+ * @init_firmware_loader: initialize data for FW loader.
+ * @init_cpu_scrambler_dram: Enable CPU specific DRAM scrambling
*/
struct hl_asic_funcs {
int (*early_init)(struct hl_device *hdev);
@@ -1006,7 +1174,7 @@ struct hl_asic_funcs {
int (*mmu_invalidate_cache)(struct hl_device *hdev, bool is_hard,
u32 flags);
int (*mmu_invalidate_cache_range)(struct hl_device *hdev, bool is_hard,
- u32 asid, u64 va, u64 size);
+ u32 flags, u32 asid, u64 va, u64 size);
int (*send_heartbeat)(struct hl_device *hdev);
void (*set_clock_gating)(struct hl_device *hdev);
void (*disable_clock_gating)(struct hl_device *hdev);
@@ -1030,8 +1198,6 @@ struct hl_asic_funcs {
void (*ctx_fini)(struct hl_ctx *ctx);
int (*get_clk_rate)(struct hl_device *hdev, u32 *cur_clk, u32 *max_clk);
u32 (*get_queue_id_for_cq)(struct hl_device *hdev, u32 cq_idx);
- int (*read_device_fw_version)(struct hl_device *hdev,
- enum hl_fw_component fwc);
int (*load_firmware_to_device)(struct hl_device *hdev);
int (*load_boot_fit_to_device)(struct hl_device *hdev);
u32 (*get_signal_cb_size)(struct hl_device *hdev);
@@ -1056,8 +1222,10 @@ struct hl_asic_funcs {
int (*hw_block_mmap)(struct hl_device *hdev, struct vm_area_struct *vma,
u32 block_id, u32 block_size);
void (*enable_events_from_fw)(struct hl_device *hdev);
- void (*get_msi_info)(u32 *table);
+ void (*get_msi_info)(__le32 *table);
int (*map_pll_idx_to_fw_idx)(u32 pll_idx);
+ void (*init_firmware_loader)(struct hl_device *hdev);
+ void (*init_cpu_scrambler_dram)(struct hl_device *hdev);
};
@@ -1262,6 +1430,7 @@ struct hl_userptr {
* @staged_sequence: the sequence of the staged submission this CS is part of,
* relevant only if staged_cs is set.
* @timeout_jiffies: cs timeout in jiffies.
+ * @submission_time_jiffies: submission time of the cs
* @type: CS_TYPE_*.
* @submitted: true if CS was submitted to H/W.
* @completed: true if CS was completed by device.
@@ -1274,6 +1443,8 @@ struct hl_userptr {
* @staged_first: true if this is the first staged CS and we need to receive
* timeout for this CS.
* @staged_cs: true if this CS is part of a staged submission.
+ * @skip_reset_on_timeout: true if we shall not reset the device in case
+ * timeout occurs (debug scenario).
*/
struct hl_cs {
u16 *jobs_in_queue_cnt;
@@ -1291,6 +1462,7 @@ struct hl_cs {
u64 sequence;
u64 staged_sequence;
u64 timeout_jiffies;
+ u64 submission_time_jiffies;
enum hl_cs_type type;
u8 submitted;
u8 completed;
@@ -1301,6 +1473,7 @@ struct hl_cs {
u8 staged_last;
u8 staged_first;
u8 staged_cs;
+ u8 skip_reset_on_timeout;
};
/**
@@ -1922,7 +2095,7 @@ struct hl_mmu_funcs {
* @kernel_queues: array of hl_hw_queue.
* @cs_mirror_list: CS mirror list for TDR.
* @cs_mirror_lock: protects cs_mirror_list.
- * @kernel_cb_mgr: command buffer manager for creating/destroying/handling CGs.
+ * @kernel_cb_mgr: command buffer manager for creating/destroying/handling CBs.
* @event_queue: event queue for IRQ from CPU-CP.
* @dma_pool: DMA pool for small allocations.
* @cpu_accessible_dma_mem: Host <-> CPU-CP shared memory CPU address.
@@ -1954,6 +2127,8 @@ struct hl_mmu_funcs {
* @aggregated_cs_counters: aggregated cs counters among all contexts
* @mmu_priv: device-specific MMU data.
* @mmu_func: device-related MMU functions.
+ * @fw_loader: FW loader manager.
+ * @pci_mem_region: array of memory regions in the PCI
* @dram_used_mem: current DRAM memory consumption.
* @timeout_jiffies: device CS timeout value.
* @max_power: the max power of the device, as configured by the sysadmin. This
@@ -1968,6 +2143,11 @@ struct hl_mmu_funcs {
* the error will be ignored by the driver during
* device initialization. Mainly used to debug and
* workaround firmware bugs
+ * @last_successful_open_jif: timestamp (jiffies) of the last successful
+ * device open.
+ * @last_open_session_duration_jif: duration (jiffies) of the last device open
+ * session.
+ * @open_counter: number of successful device open operations.
* @in_reset: is device in reset flow.
* @curr_pll_profile: current PLL profile.
* @card_type: Various ASICs have several card types. This indicates the card
@@ -2007,6 +2187,8 @@ struct hl_mmu_funcs {
* @collective_mon_idx: helper index for collective initialization
* @supports_coresight: is CoreSight supported.
* @supports_soft_reset: is soft reset supported.
+ * @allow_external_soft_reset: true if soft reset initiated by user or TDR is
+ * allowed.
* @supports_cb_mapping: is mapping a CB to the device's MMU supported.
* @needs_reset: true if reset_on_lockup is false and device should be reset
* due to lockup.
@@ -2015,6 +2197,14 @@ struct hl_mmu_funcs {
* @device_fini_pending: true if device_fini was called and might be
* waiting for the reset thread to finish
* @supports_staged_submission: true if staged submissions are supported
+ * @curr_reset_cause: saves an enumerated reset cause when a hard reset is
+ * triggered, and cleared after it is shared with preboot.
+ * @skip_reset_on_timeout: Skip device reset if CS has timed out, wait for it to
+ * complete instead.
+ * @device_cpu_is_halted: Flag to indicate whether the device CPU was already
+ * halted. We can't halt it again because the COMMS
+ * protocol will throw an error. Relevant only for
+ * cases where Linux was not loaded to device CPU
*/
struct hl_device {
struct pci_dev *pdev;
@@ -2079,11 +2269,18 @@ struct hl_device {
struct hl_mmu_priv mmu_priv;
struct hl_mmu_funcs mmu_func[MMU_NUM_PGT_LOCATIONS];
+ struct fw_load_mgr fw_loader;
+
+ struct pci_mem_region pci_mem_region[PCI_REGION_NUMBER];
+
atomic64_t dram_used_mem;
u64 timeout_jiffies;
u64 max_power;
u64 clock_gating_mask;
u64 boot_error_status_mask;
+ u64 last_successful_open_jif;
+ u64 last_open_session_duration_jif;
+ u64 open_counter;
atomic_t in_reset;
enum hl_pll_frequency curr_pll_profile;
enum cpucp_card_types card_type;
@@ -2116,11 +2313,15 @@ struct hl_device {
u8 collective_mon_idx;
u8 supports_coresight;
u8 supports_soft_reset;
+ u8 allow_external_soft_reset;
u8 supports_cb_mapping;
u8 needs_reset;
u8 process_kill_trial_cnt;
u8 device_fini_pending;
u8 supports_staged_submission;
+ u8 curr_reset_cause;
+ u8 skip_reset_on_timeout;
+ u8 device_cpu_is_halted;
/* Parameters for bring-up */
u64 nic_ports_mask;
@@ -2138,6 +2339,7 @@ struct hl_device {
u8 rl_enable;
u8 reset_on_preboot_fail;
u8 reset_upon_device_release;
+ u8 reset_if_device_not_idle;
};
@@ -2384,11 +2586,13 @@ void hl_fw_cpu_accessible_dma_pool_free(struct hl_device *hdev, size_t size,
void *vaddr);
int hl_fw_send_heartbeat(struct hl_device *hdev);
int hl_fw_cpucp_info_get(struct hl_device *hdev,
- u32 cpu_security_boot_status_reg,
- u32 boot_err0_reg);
+ u32 sts_boot_dev_sts0_reg,
+ u32 sts_boot_dev_sts1_reg, u32 boot_err0_reg,
+ u32 boot_err1_reg);
int hl_fw_cpucp_handshake(struct hl_device *hdev,
- u32 cpu_security_boot_status_reg,
- u32 boot_err0_reg);
+ u32 sts_boot_dev_sts0_reg,
+ u32 sts_boot_dev_sts1_reg, u32 boot_err0_reg,
+ u32 boot_err1_reg);
int hl_fw_get_eeprom_data(struct hl_device *hdev, void *data, size_t max_size);
int hl_fw_cpucp_pci_counters_get(struct hl_device *hdev,
struct hl_info_pci_counters *counters);
@@ -2399,14 +2603,17 @@ int get_used_pll_index(struct hl_device *hdev, u32 input_pll_index,
int hl_fw_cpucp_pll_info_get(struct hl_device *hdev, u32 pll_index,
u16 *pll_freq_arr);
int hl_fw_cpucp_power_get(struct hl_device *hdev, u64 *power);
-int hl_fw_init_cpu(struct hl_device *hdev, u32 cpu_boot_status_reg,
- u32 msg_to_cpu_reg, u32 cpu_msg_status_reg,
- u32 cpu_security_boot_status_reg, u32 boot_err0_reg,
- bool skip_bmc, u32 cpu_timeout, u32 boot_fit_timeout);
+void hl_fw_ask_hard_reset_without_linux(struct hl_device *hdev);
+void hl_fw_ask_halt_machine_without_linux(struct hl_device *hdev);
+int hl_fw_init_cpu(struct hl_device *hdev);
int hl_fw_read_preboot_status(struct hl_device *hdev, u32 cpu_boot_status_reg,
- u32 cpu_security_boot_status_reg, u32 boot_err0_reg,
- u32 timeout);
-
+ u32 sts_boot_dev_sts0_reg,
+ u32 sts_boot_dev_sts1_reg, u32 boot_err0_reg,
+ u32 boot_err1_reg, u32 timeout);
+int hl_fw_dynamic_send_protocol_cmd(struct hl_device *hdev,
+ struct fw_load_mgr *fw_loader,
+ enum comms_cmd cmd, unsigned int size,
+ bool wait_ok, u32 timeout);
int hl_pci_bars_map(struct hl_device *hdev, const char * const name[3],
bool is_wc[3]);
int hl_pci_elbi_read(struct hl_device *hdev, u64 addr, u32 *data);
@@ -2415,6 +2622,7 @@ int hl_pci_set_inbound_region(struct hl_device *hdev, u8 region,
struct hl_inbound_pci_region *pci_region);
int hl_pci_set_outbound_region(struct hl_device *hdev,
struct hl_outbound_pci_region *pci_region);
+enum pci_region hl_get_pci_memory_region(struct hl_device *hdev, u64 addr);
int hl_pci_init(struct hl_device *hdev);
void hl_pci_fini(struct hl_device *hdev);
@@ -2443,6 +2651,8 @@ int hl_set_voltage(struct hl_device *hdev,
int hl_set_current(struct hl_device *hdev,
int sensor_index, u32 attr, long value);
void hl_release_pending_user_interrupts(struct hl_device *hdev);
+int hl_cs_signal_sob_wraparound_handler(struct hl_device *hdev, u32 q_idx,
+ struct hl_hw_sob **hw_sob, u32 count);
#ifdef CONFIG_DEBUG_FS
diff --git a/drivers/misc/habanalabs/common/habanalabs_drv.c b/drivers/misc/habanalabs/common/habanalabs_drv.c
index 64d1530db985..4194cda2d04c 100644
--- a/drivers/misc/habanalabs/common/habanalabs_drv.c
+++ b/drivers/misc/habanalabs/common/habanalabs_drv.c
@@ -29,7 +29,7 @@ static DEFINE_MUTEX(hl_devs_idr_lock);
static int timeout_locked = 30;
static int reset_on_lockup = 1;
-static int memory_scrub = 1;
+static int memory_scrub;
static ulong boot_error_status_mask = ULONG_MAX;
module_param(timeout_locked, int, 0444);
@@ -42,7 +42,7 @@ MODULE_PARM_DESC(reset_on_lockup,
module_param(memory_scrub, int, 0444);
MODULE_PARM_DESC(memory_scrub,
- "Scrub device memory in various states (0 = no, 1 = yes, default yes)");
+ "Scrub device memory in various states (0 = no, 1 = yes, default no)");
module_param(boot_error_status_mask, ulong, 0444);
MODULE_PARM_DESC(boot_error_status_mask,
@@ -187,6 +187,9 @@ int hl_device_open(struct inode *inode, struct file *filp)
hl_debugfs_add_file(hpriv);
+ hdev->open_counter++;
+ hdev->last_successful_open_jif = jiffies;
+
return 0;
out_err:
@@ -264,6 +267,7 @@ static void set_driver_behavior_per_device(struct hl_device *hdev)
hdev->bmc_enable = 1;
hdev->hard_reset_on_fw_events = 1;
hdev->reset_on_preboot_fail = 1;
+ hdev->reset_if_device_not_idle = 1;
hdev->reset_pcilink = 0;
hdev->axi_drain = 0;
@@ -308,10 +312,10 @@ int create_hdev(struct hl_device **dev, struct pci_dev *pdev,
}
if (pdev)
- hdev->asic_prop.fw_security_disabled =
- !is_asic_secured(pdev->device);
+ hdev->asic_prop.fw_security_enabled =
+ is_asic_secured(hdev->asic_type);
else
- hdev->asic_prop.fw_security_disabled = true;
+ hdev->asic_prop.fw_security_enabled = false;
/* Assign status description string */
strncpy(hdev->status[HL_DEVICE_STATUS_MALFUNCTION],
@@ -325,11 +329,14 @@ int create_hdev(struct hl_device **dev, struct pci_dev *pdev,
hdev->reset_on_lockup = reset_on_lockup;
hdev->memory_scrub = memory_scrub;
hdev->boot_error_status_mask = boot_error_status_mask;
+ hdev->stop_on_err = true;
hdev->pldm = 0;
set_driver_behavior_per_device(hdev);
+ hdev->curr_reset_cause = HL_RESET_CAUSE_UNKNOWN;
+
if (timeout_locked)
hdev->timeout_jiffies = msecs_to_jiffies(timeout_locked * 1000);
else
@@ -464,6 +471,7 @@ static int hl_pci_probe(struct pci_dev *pdev,
return 0;
disable_device:
+ pci_disable_pcie_error_reporting(pdev);
pci_set_drvdata(pdev, NULL);
destroy_hdev(hdev);
@@ -572,7 +580,11 @@ static struct pci_driver hl_pci_driver = {
.probe = hl_pci_probe,
.remove = hl_pci_remove,
.shutdown = hl_pci_remove,
- .driver.pm = &hl_pm_ops,
+ .driver = {
+ .name = HL_NAME,
+ .pm = &hl_pm_ops,
+ .probe_type = PROBE_PREFER_ASYNCHRONOUS,
+ },
.err_handler = &hl_pci_err_handler,
};
diff --git a/drivers/misc/habanalabs/common/habanalabs_ioctl.c b/drivers/misc/habanalabs/common/habanalabs_ioctl.c
index 33841c272eb6..f4dda7b4acdd 100644
--- a/drivers/misc/habanalabs/common/habanalabs_ioctl.c
+++ b/drivers/misc/habanalabs/common/habanalabs_ioctl.c
@@ -95,7 +95,7 @@ static int hw_ip_info(struct hl_device *hdev, struct hl_info_args *args)
hw_ip.first_available_interrupt_id =
prop->first_available_user_msix_interrupt;
return copy_to_user(out, &hw_ip,
- min((size_t)size, sizeof(hw_ip))) ? -EFAULT : 0;
+ min((size_t) size, sizeof(hw_ip))) ? -EFAULT : 0;
}
static int hw_events_info(struct hl_device *hdev, bool aggregate,
@@ -460,6 +460,24 @@ static int power_info(struct hl_fpriv *hpriv, struct hl_info_args *args)
min((size_t) max_size, sizeof(power_info))) ? -EFAULT : 0;
}
+static int open_stats_info(struct hl_fpriv *hpriv, struct hl_info_args *args)
+{
+ struct hl_device *hdev = hpriv->hdev;
+ u32 max_size = args->return_size;
+ struct hl_open_stats_info open_stats_info = {0};
+ void __user *out = (void __user *) (uintptr_t) args->return_pointer;
+
+ if ((!max_size) || (!out))
+ return -EINVAL;
+
+ open_stats_info.last_open_period_ms = jiffies64_to_msecs(
+ hdev->last_open_session_duration_jif);
+ open_stats_info.open_counter = hdev->open_counter;
+
+ return copy_to_user(out, &open_stats_info,
+ min((size_t) max_size, sizeof(open_stats_info))) ? -EFAULT : 0;
+}
+
static int _hl_info_ioctl(struct hl_fpriv *hpriv, void *data,
struct device *dev)
{
@@ -543,6 +561,9 @@ static int _hl_info_ioctl(struct hl_fpriv *hpriv, void *data,
case HL_INFO_POWER:
return power_info(hpriv, args);
+ case HL_INFO_OPEN_STATS:
+ return open_stats_info(hpriv, args);
+
default:
dev_err(dev, "Invalid request %d\n", args->op);
rc = -ENOTTY;
diff --git a/drivers/misc/habanalabs/common/hw_queue.c b/drivers/misc/habanalabs/common/hw_queue.c
index 173438461835..bcabfdbf1e01 100644
--- a/drivers/misc/habanalabs/common/hw_queue.c
+++ b/drivers/misc/habanalabs/common/hw_queue.c
@@ -410,19 +410,20 @@ static void hw_queue_schedule_job(struct hl_cs_job *job)
ext_and_hw_queue_submit_bd(hdev, q, ctl, len, ptr);
}
-static void init_signal_cs(struct hl_device *hdev,
+static int init_signal_cs(struct hl_device *hdev,
struct hl_cs_job *job, struct hl_cs_compl *cs_cmpl)
{
struct hl_sync_stream_properties *prop;
struct hl_hw_sob *hw_sob;
u32 q_idx;
+ int rc = 0;
q_idx = job->hw_queue_id;
prop = &hdev->kernel_queues[q_idx].sync_stream_prop;
hw_sob = &prop->hw_sob[prop->curr_sob_offset];
cs_cmpl->hw_sob = hw_sob;
- cs_cmpl->sob_val = prop->next_sob_val++;
+ cs_cmpl->sob_val = prop->next_sob_val;
dev_dbg(hdev->dev,
"generate signal CB, sob_id: %d, sob val: 0x%x, q_idx: %d\n",
@@ -434,24 +435,9 @@ static void init_signal_cs(struct hl_device *hdev,
hdev->asic_funcs->gen_signal_cb(hdev, job->patched_cb,
cs_cmpl->hw_sob->sob_id, 0, true);
- kref_get(&hw_sob->kref);
+ rc = hl_cs_signal_sob_wraparound_handler(hdev, q_idx, &hw_sob, 1);
- /* check for wraparound */
- if (prop->next_sob_val == HL_MAX_SOB_VAL) {
- /*
- * Decrement as we reached the max value.
- * The release function won't be called here as we've
- * just incremented the refcount.
- */
- kref_put(&hw_sob->kref, hl_sob_reset_error);
- prop->next_sob_val = 1;
- /* only two SOBs are currently in use */
- prop->curr_sob_offset =
- (prop->curr_sob_offset + 1) % HL_RSVD_SOBS;
-
- dev_dbg(hdev->dev, "switched to SOB %d, q_idx: %d\n",
- prop->curr_sob_offset, q_idx);
- }
+ return rc;
}
static void init_wait_cs(struct hl_device *hdev, struct hl_cs *cs,
@@ -504,22 +490,25 @@ static void init_wait_cs(struct hl_device *hdev, struct hl_cs *cs,
*
* H/W queues spinlock should be taken before calling this function
*/
-static void init_signal_wait_cs(struct hl_cs *cs)
+static int init_signal_wait_cs(struct hl_cs *cs)
{
struct hl_ctx *ctx = cs->ctx;
struct hl_device *hdev = ctx->hdev;
struct hl_cs_job *job;
struct hl_cs_compl *cs_cmpl =
container_of(cs->fence, struct hl_cs_compl, base_fence);
+ int rc = 0;
/* There is only one job in a signal/wait CS */
job = list_first_entry(&cs->job_list, struct hl_cs_job,
cs_node);
if (cs->type & CS_TYPE_SIGNAL)
- init_signal_cs(hdev, job, cs_cmpl);
+ rc = init_signal_cs(hdev, job, cs_cmpl);
else if (cs->type & CS_TYPE_WAIT)
init_wait_cs(hdev, cs, job, cs_cmpl);
+
+ return rc;
}
/*
@@ -590,11 +579,16 @@ int hl_hw_queue_schedule_cs(struct hl_cs *cs)
}
}
- if ((cs->type == CS_TYPE_SIGNAL) || (cs->type == CS_TYPE_WAIT))
- init_signal_wait_cs(cs);
- else if (cs->type == CS_TYPE_COLLECTIVE_WAIT)
+ if ((cs->type == CS_TYPE_SIGNAL) || (cs->type == CS_TYPE_WAIT)) {
+ rc = init_signal_wait_cs(cs);
+ if (rc) {
+ dev_err(hdev->dev, "Failed to submit signal cs\n");
+ goto unroll_cq_resv;
+ }
+ } else if (cs->type == CS_TYPE_COLLECTIVE_WAIT)
hdev->asic_funcs->collective_wait_init_cs(cs);
+
spin_lock(&hdev->cs_mirror_lock);
/* Verify staged CS exists and add to the staged list */
diff --git a/drivers/misc/habanalabs/common/irq.c b/drivers/misc/habanalabs/common/irq.c
index 27129868c711..39b14a933393 100644
--- a/drivers/misc/habanalabs/common/irq.c
+++ b/drivers/misc/habanalabs/common/irq.c
@@ -207,17 +207,33 @@ irqreturn_t hl_irq_handler_eq(int irq, void *arg)
struct hl_eq_entry *eq_entry;
struct hl_eq_entry *eq_base;
struct hl_eqe_work *handle_eqe_work;
+ bool entry_ready;
+ u32 cur_eqe;
+ u16 cur_eqe_index;
eq_base = eq->kernel_address;
while (1) {
- bool entry_ready =
- ((le32_to_cpu(eq_base[eq->ci].hdr.ctl) &
- EQ_CTL_READY_MASK) >> EQ_CTL_READY_SHIFT);
+ cur_eqe = le32_to_cpu(eq_base[eq->ci].hdr.ctl);
+ entry_ready = !!FIELD_GET(EQ_CTL_READY_MASK, cur_eqe);
if (!entry_ready)
break;
+ cur_eqe_index = FIELD_GET(EQ_CTL_INDEX_MASK, cur_eqe);
+ if ((hdev->event_queue.check_eqe_index) &&
+ (((eq->prev_eqe_index + 1) & EQ_CTL_INDEX_MASK)
+ != cur_eqe_index)) {
+ dev_dbg(hdev->dev,
+ "EQE 0x%x in queue is ready but index does not match %d!=%d",
+ eq_base[eq->ci].hdr.ctl,
+ ((eq->prev_eqe_index + 1) & EQ_CTL_INDEX_MASK),
+ cur_eqe_index);
+ break;
+ }
+
+ eq->prev_eqe_index++;
+
eq_entry = &eq_base[eq->ci];
/*
@@ -341,6 +357,7 @@ int hl_eq_init(struct hl_device *hdev, struct hl_eq *q)
q->hdev = hdev;
q->kernel_address = p;
q->ci = 0;
+ q->prev_eqe_index = 0;
return 0;
}
@@ -365,6 +382,7 @@ void hl_eq_fini(struct hl_device *hdev, struct hl_eq *q)
void hl_eq_reset(struct hl_device *hdev, struct hl_eq *q)
{
q->ci = 0;
+ q->prev_eqe_index = 0;
/*
* It's not enough to just reset the PI/CI because the H/W may have
diff --git a/drivers/misc/habanalabs/common/memory.c b/drivers/misc/habanalabs/common/memory.c
index 2938cbbafbbc..af339ce1ab4f 100644
--- a/drivers/misc/habanalabs/common/memory.c
+++ b/drivers/misc/habanalabs/common/memory.c
@@ -570,8 +570,10 @@ static u64 get_va_block(struct hl_device *hdev,
if ((is_align_pow_2 && (hint_addr & (va_block_align - 1))) ||
(!is_align_pow_2 &&
do_div(tmp_hint_addr, va_range->page_size))) {
- dev_info(hdev->dev, "Hint address 0x%llx will be ignored\n",
- hint_addr);
+
+ dev_dbg(hdev->dev,
+ "Hint address 0x%llx will be ignored because it is not aligned\n",
+ hint_addr);
hint_addr = 0;
}
@@ -1117,7 +1119,8 @@ static int map_device_va(struct hl_ctx *ctx, struct hl_mem_in *args,
goto map_err;
}
- rc = hdev->asic_funcs->mmu_invalidate_cache(hdev, false, *vm_type);
+ rc = hdev->asic_funcs->mmu_invalidate_cache_range(hdev, false,
+ *vm_type, ctx->asid, ret_vaddr, phys_pg_pack->total_size);
mutex_unlock(&ctx->mmu_lock);
@@ -1261,8 +1264,9 @@ static int unmap_device_va(struct hl_ctx *ctx, struct hl_mem_in *args,
* at the loop end rather than for each iteration
*/
if (!ctx_free)
- rc = hdev->asic_funcs->mmu_invalidate_cache(hdev, true,
- *vm_type);
+ rc = hdev->asic_funcs->mmu_invalidate_cache_range(hdev, true,
+ *vm_type, ctx->asid, vaddr,
+ phys_pg_pack->total_size);
mutex_unlock(&ctx->mmu_lock);
@@ -1369,12 +1373,7 @@ int hl_hw_block_mmap(struct hl_fpriv *hpriv, struct vm_area_struct *vma)
/* Driver only allows mapping of a complete HW block */
block_size = vma->vm_end - vma->vm_start;
-#ifdef _HAS_TYPE_ARG_IN_ACCESS_OK
- if (!access_ok(VERIFY_WRITE,
- (void __user *) (uintptr_t) vma->vm_start, block_size)) {
-#else
if (!access_ok((void __user *) (uintptr_t) vma->vm_start, block_size)) {
-#endif
dev_err(hdev->dev,
"user pointer is invalid - 0x%lx\n",
vma->vm_start);
@@ -1608,7 +1607,8 @@ static int get_user_memory(struct hl_device *hdev, u64 addr, u64 size,
if (rc != npages) {
dev_err(hdev->dev,
- "Failed to map host memory, user ptr probably wrong\n");
+ "Failed (%d) to pin host memory with user ptr 0x%llx, size 0x%llx, npages %d\n",
+ rc, addr, size, npages);
if (rc < 0)
goto destroy_pages;
npages = rc;
diff --git a/drivers/misc/habanalabs/common/mmu/mmu.c b/drivers/misc/habanalabs/common/mmu/mmu.c
index b37189956b14..792d25b79ea6 100644
--- a/drivers/misc/habanalabs/common/mmu/mmu.c
+++ b/drivers/misc/habanalabs/common/mmu/mmu.c
@@ -501,12 +501,20 @@ static void hl_mmu_pa_page_with_offset(struct hl_ctx *ctx, u64 virt_addr,
if ((hops->range_type == HL_VA_RANGE_TYPE_DRAM) &&
!is_power_of_2(prop->dram_page_size)) {
- u32 bit;
+ unsigned long dram_page_size = prop->dram_page_size;
u64 page_offset_mask;
u64 phys_addr_mask;
+ u32 bit;
- bit = __ffs64((u64)prop->dram_page_size);
- page_offset_mask = ((1ull << bit) - 1);
+ /*
+ * find last set bit in page_size to cover all bits of page
+ * offset. note that 1 has to be added to bit index.
+ * note that the internal ulong variable is used to avoid
+ * alignment issue.
+ */
+ bit = find_last_bit(&dram_page_size,
+ sizeof(dram_page_size) * BITS_PER_BYTE) + 1;
+ page_offset_mask = (BIT_ULL(bit) - 1);
phys_addr_mask = ~page_offset_mask;
*phys_addr = (tmp_phys_addr & phys_addr_mask) |
(virt_addr & page_offset_mask);
diff --git a/drivers/misc/habanalabs/common/pci/pci.c b/drivers/misc/habanalabs/common/pci/pci.c
index e941b7eef346..d5bedf5ba011 100644
--- a/drivers/misc/habanalabs/common/pci/pci.c
+++ b/drivers/misc/habanalabs/common/pci/pci.c
@@ -10,7 +10,7 @@
#include <linux/pci.h>
-#define HL_PLDM_PCI_ELBI_TIMEOUT_MSEC (HL_PCI_ELBI_TIMEOUT_MSEC * 10)
+#define HL_PLDM_PCI_ELBI_TIMEOUT_MSEC (HL_PCI_ELBI_TIMEOUT_MSEC * 100)
#define IATU_REGION_CTRL_REGION_EN_MASK BIT(31)
#define IATU_REGION_CTRL_MATCH_MODE_MASK BIT(30)
@@ -360,6 +360,32 @@ int hl_pci_set_outbound_region(struct hl_device *hdev,
}
/**
+ * hl_get_pci_memory_region() - get PCI region for given address
+ * @hdev: Pointer to hl_device structure.
+ * @addr: device address
+ *
+ * @return region index on success, otherwise PCI_REGION_NUMBER (invalid
+ * region index)
+ */
+enum pci_region hl_get_pci_memory_region(struct hl_device *hdev, u64 addr)
+{
+ int i;
+
+ for (i = 0 ; i < PCI_REGION_NUMBER ; i++) {
+ struct pci_mem_region *region = &hdev->pci_mem_region[i];
+
+ if (!region->used)
+ continue;
+
+ if ((addr >= region->region_base) &&
+ (addr < region->region_base + region->region_size))
+ return i;
+ }
+
+ return PCI_REGION_NUMBER;
+}
+
+/**
* hl_pci_init() - PCI initialization code.
* @hdev: Pointer to hl_device structure.
*
@@ -395,6 +421,12 @@ int hl_pci_init(struct hl_device *hdev)
goto unmap_pci_bars;
}
+ /* Driver must sleep in order for FW to finish the iATU configuration */
+ if (hdev->asic_prop.iatu_done_by_fw) {
+ usleep_range(2000, 3000);
+ hdev->asic_funcs->set_dma_mask_from_fw(hdev);
+ }
+
rc = dma_set_mask_and_coherent(&pdev->dev,
DMA_BIT_MASK(hdev->dma_mask));
if (rc) {
diff --git a/drivers/misc/habanalabs/common/sysfs.c b/drivers/misc/habanalabs/common/sysfs.c
index c9f649b31e3a..db72df282ef8 100644
--- a/drivers/misc/habanalabs/common/sysfs.c
+++ b/drivers/misc/habanalabs/common/sysfs.c
@@ -208,7 +208,7 @@ static ssize_t soft_reset_store(struct device *dev,
goto out;
}
- if (!hdev->supports_soft_reset) {
+ if (!hdev->allow_external_soft_reset) {
dev_err(hdev->dev, "Device does not support soft-reset\n");
goto out;
}
diff --git a/drivers/misc/habanalabs/gaudi/gaudi.c b/drivers/misc/habanalabs/gaudi/gaudi.c
index 9e4a6bb3acd1..aa8a0ca5aca2 100644
--- a/drivers/misc/habanalabs/gaudi/gaudi.c
+++ b/drivers/misc/habanalabs/gaudi/gaudi.c
@@ -78,6 +78,7 @@
#define GAUDI_PLDM_TPC_KERNEL_WAIT_USEC (HL_DEVICE_TIMEOUT_USEC * 30)
#define GAUDI_BOOT_FIT_REQ_TIMEOUT_USEC 1000000 /* 1s */
#define GAUDI_MSG_TO_CPU_TIMEOUT_USEC 4000000 /* 4s */
+#define GAUDI_WAIT_FOR_BL_TIMEOUT_USEC 15000000 /* 15s */
#define GAUDI_QMAN0_FENCE_VAL 0x72E91AB9
@@ -409,7 +410,7 @@ static inline void set_default_power_values(struct hl_device *hdev)
}
}
-static int gaudi_get_fixed_properties(struct hl_device *hdev)
+static int gaudi_set_fixed_properties(struct hl_device *hdev)
{
struct asic_fixed_properties *prop = &hdev->asic_prop;
u32 num_sync_stream_queues = 0;
@@ -545,8 +546,10 @@ static int gaudi_get_fixed_properties(struct hl_device *hdev)
for (i = 0 ; i < HL_MAX_DCORES ; i++)
prop->first_available_cq[i] = USHRT_MAX;
- prop->fw_security_status_valid = false;
+ prop->fw_cpu_boot_dev_sts0_valid = false;
+ prop->fw_cpu_boot_dev_sts1_valid = false;
prop->hard_reset_done_by_fw = false;
+ prop->gic_interrupts_enable = true;
return 0;
}
@@ -577,6 +580,9 @@ static u64 gaudi_set_hbm_bar_base(struct hl_device *hdev, u64 addr)
if ((gaudi) && (gaudi->hbm_bar_cur_addr == addr))
return old_addr;
+ if (hdev->asic_prop.iatu_done_by_fw)
+ return U64_MAX;
+
/* Inbound Region 2 - Bar 4 - Point to HBM */
pci_region.mode = PCI_BAR_MATCH_MODE;
pci_region.bar = HBM_BAR_ID;
@@ -599,10 +605,8 @@ static int gaudi_init_iatu(struct hl_device *hdev)
struct hl_outbound_pci_region outbound_region;
int rc;
- if (hdev->asic_prop.iatu_done_by_fw) {
- hdev->asic_funcs->set_dma_mask_from_fw(hdev);
+ if (hdev->asic_prop.iatu_done_by_fw)
return 0;
- }
/* Inbound Region 0 - Bar 0 - Point to SRAM + CFG */
inbound_region.mode = PCI_BAR_MATCH_MODE;
@@ -651,9 +655,9 @@ static int gaudi_early_init(struct hl_device *hdev)
u32 fw_boot_status;
int rc;
- rc = gaudi_get_fixed_properties(hdev);
+ rc = gaudi_set_fixed_properties(hdev);
if (rc) {
- dev_err(hdev->dev, "Failed to get fixed properties\n");
+ dev_err(hdev->dev, "Failed setting fixed properties\n");
return rc;
}
@@ -683,8 +687,14 @@ static int gaudi_early_init(struct hl_device *hdev)
prop->dram_pci_bar_size = pci_resource_len(pdev, HBM_BAR_ID);
/* If FW security is enabled at this point it means no access to ELBI */
- if (!hdev->asic_prop.fw_security_disabled) {
+ if (hdev->asic_prop.fw_security_enabled) {
hdev->asic_prop.iatu_done_by_fw = true;
+
+ /*
+ * GIC-security-bit can ONLY be set by CPUCP, so in this stage
+ * decision can only be taken based on PCI ID security.
+ */
+ hdev->asic_prop.gic_interrupts_enable = false;
goto pci_init;
}
@@ -707,8 +717,10 @@ pci_init:
* version to determine whether we run with a security-enabled firmware
*/
rc = hl_fw_read_preboot_status(hdev, mmPSOC_GLOBAL_CONF_CPU_BOOT_STATUS,
- mmCPU_BOOT_DEV_STS0, mmCPU_BOOT_ERR0,
- GAUDI_BOOT_FIT_REQ_TIMEOUT_USEC);
+ mmCPU_BOOT_DEV_STS0,
+ mmCPU_BOOT_DEV_STS1, mmCPU_BOOT_ERR0,
+ mmCPU_BOOT_ERR1,
+ GAUDI_BOOT_FIT_REQ_TIMEOUT_USEC);
if (rc) {
if (hdev->reset_on_preboot_fail)
hdev->asic_funcs->hw_fini(hdev, true);
@@ -751,7 +763,14 @@ static int gaudi_fetch_psoc_frequency(struct hl_device *hdev)
u16 pll_freq_arr[HL_PLL_NUM_OUTPUTS], freq;
int rc;
- if (hdev->asic_prop.fw_security_disabled) {
+ if (hdev->asic_prop.fw_security_enabled) {
+ rc = hl_fw_cpucp_pll_info_get(hdev, HL_GAUDI_CPU_PLL, pll_freq_arr);
+
+ if (rc)
+ return rc;
+
+ freq = pll_freq_arr[2];
+ } else {
/* Backward compatibility */
div_fctr = RREG32(mmPSOC_CPU_PLL_DIV_FACTOR_2);
div_sel = RREG32(mmPSOC_CPU_PLL_DIV_SEL_2);
@@ -779,13 +798,6 @@ static int gaudi_fetch_psoc_frequency(struct hl_device *hdev)
div_sel);
freq = 0;
}
- } else {
- rc = hl_fw_cpucp_pll_info_get(hdev, HL_GAUDI_CPU_PLL, pll_freq_arr);
-
- if (rc)
- return rc;
-
- freq = pll_freq_arr[2];
}
prop->psoc_timestamp_frequency = freq;
@@ -988,9 +1000,27 @@ static void gaudi_sob_group_reset_error(struct kref *ref)
hw_sob_group->base_sob_id);
}
+static void gaudi_collective_mstr_sob_mask_set(struct gaudi_device *gaudi)
+{
+ struct gaudi_collective_properties *prop;
+ int i;
+
+ prop = &gaudi->collective_props;
+
+ memset(prop->mstr_sob_mask, 0, sizeof(prop->mstr_sob_mask));
+
+ for (i = 0 ; i < NIC_NUMBER_OF_ENGINES ; i++)
+ if (gaudi->hw_cap_initialized & BIT(HW_CAP_NIC_SHIFT + i))
+ prop->mstr_sob_mask[i / HL_MAX_SOBS_PER_MONITOR] |=
+ BIT(i % HL_MAX_SOBS_PER_MONITOR);
+ /* Set collective engine bit */
+ prop->mstr_sob_mask[i / HL_MAX_SOBS_PER_MONITOR] |=
+ BIT(i % HL_MAX_SOBS_PER_MONITOR);
+}
+
static int gaudi_collective_init(struct hl_device *hdev)
{
- u32 i, master_monitor_sobs, sob_id, reserved_sobs_per_group;
+ u32 i, sob_id, reserved_sobs_per_group;
struct gaudi_collective_properties *prop;
struct gaudi_device *gaudi;
@@ -1016,22 +1046,7 @@ static int gaudi_collective_init(struct hl_device *hdev)
gaudi_collective_map_sobs(hdev, i);
}
- prop->mstr_sob_mask[0] = 0;
- master_monitor_sobs = HL_MAX_SOBS_PER_MONITOR;
- for (i = 0 ; i < master_monitor_sobs ; i++)
- if (gaudi->hw_cap_initialized & BIT(HW_CAP_NIC_SHIFT + i))
- prop->mstr_sob_mask[0] |= BIT(i);
-
- prop->mstr_sob_mask[1] = 0;
- master_monitor_sobs =
- NIC_NUMBER_OF_ENGINES - HL_MAX_SOBS_PER_MONITOR;
- for (i = 0 ; i < master_monitor_sobs; i++) {
- if (gaudi->hw_cap_initialized & BIT(HW_CAP_NIC_SHIFT + i))
- prop->mstr_sob_mask[1] |= BIT(i);
- }
-
- /* Set collective engine bit */
- prop->mstr_sob_mask[1] |= BIT(i);
+ gaudi_collective_mstr_sob_mask_set(gaudi);
return 0;
}
@@ -1513,7 +1528,7 @@ static int gaudi_alloc_cpu_accessible_dma_mem(struct hl_device *hdev)
hdev->cpu_pci_msb_addr =
GAUDI_CPU_PCI_MSB_ADDR(hdev->cpu_accessible_dma_address);
- if (hdev->asic_prop.fw_security_disabled)
+ if (!hdev->asic_prop.fw_security_enabled)
GAUDI_PCI_TO_CPU_ADDR(hdev->cpu_accessible_dma_address);
free_dma_mem_arr:
@@ -1590,6 +1605,48 @@ free_internal_qmans_pq_mem:
return rc;
}
+static void gaudi_set_pci_memory_regions(struct hl_device *hdev)
+{
+ struct asic_fixed_properties *prop = &hdev->asic_prop;
+ struct pci_mem_region *region;
+
+ /* CFG */
+ region = &hdev->pci_mem_region[PCI_REGION_CFG];
+ region->region_base = CFG_BASE;
+ region->region_size = CFG_SIZE;
+ region->offset_in_bar = CFG_BASE - SPI_FLASH_BASE_ADDR;
+ region->bar_size = CFG_BAR_SIZE;
+ region->bar_id = CFG_BAR_ID;
+ region->used = 1;
+
+ /* SRAM */
+ region = &hdev->pci_mem_region[PCI_REGION_SRAM];
+ region->region_base = SRAM_BASE_ADDR;
+ region->region_size = SRAM_SIZE;
+ region->offset_in_bar = 0;
+ region->bar_size = SRAM_BAR_SIZE;
+ region->bar_id = SRAM_BAR_ID;
+ region->used = 1;
+
+ /* DRAM */
+ region = &hdev->pci_mem_region[PCI_REGION_DRAM];
+ region->region_base = DRAM_PHYS_BASE;
+ region->region_size = hdev->asic_prop.dram_size;
+ region->offset_in_bar = 0;
+ region->bar_size = prop->dram_pci_bar_size;
+ region->bar_id = HBM_BAR_ID;
+ region->used = 1;
+
+ /* SP SRAM */
+ region = &hdev->pci_mem_region[PCI_REGION_SP_SRAM];
+ region->region_base = PSOC_SCRATCHPAD_ADDR;
+ region->region_size = PSOC_SCRATCHPAD_SIZE;
+ region->offset_in_bar = PSOC_SCRATCHPAD_ADDR - SPI_FLASH_BASE_ADDR;
+ region->bar_size = CFG_BAR_SIZE;
+ region->bar_id = CFG_BAR_ID;
+ region->used = 1;
+}
+
static int gaudi_sw_init(struct hl_device *hdev)
{
struct gaudi_device *gaudi;
@@ -1664,12 +1721,14 @@ static int gaudi_sw_init(struct hl_device *hdev)
hdev->supports_coresight = true;
hdev->supports_staged_submission = true;
+ gaudi_set_pci_memory_regions(hdev);
+
return 0;
free_cpu_accessible_dma_pool:
gen_pool_destroy(hdev->cpu_accessible_dma_pool);
free_cpu_dma_mem:
- if (hdev->asic_prop.fw_security_disabled)
+ if (!hdev->asic_prop.fw_security_enabled)
GAUDI_CPU_TO_PCI_ADDR(hdev->cpu_accessible_dma_address,
hdev->cpu_pci_msb_addr);
hdev->asic_funcs->asic_dma_free_coherent(hdev,
@@ -1691,7 +1750,7 @@ static int gaudi_sw_fini(struct hl_device *hdev)
gen_pool_destroy(hdev->cpu_accessible_dma_pool);
- if (hdev->asic_prop.fw_security_disabled)
+ if (!hdev->asic_prop.fw_security_enabled)
GAUDI_CPU_TO_PCI_ADDR(hdev->cpu_accessible_dma_address,
hdev->cpu_pci_msb_addr);
@@ -1879,12 +1938,11 @@ static void gaudi_init_scrambler_sram(struct hl_device *hdev)
{
struct gaudi_device *gaudi = hdev->asic_specific;
- if (!hdev->asic_prop.fw_security_disabled)
+ if (hdev->asic_prop.fw_security_enabled)
return;
- if (hdev->asic_prop.fw_security_status_valid &&
- (hdev->asic_prop.fw_app_security_map &
- CPU_BOOT_DEV_STS0_SRAM_SCR_EN))
+ if (hdev->asic_prop.fw_app_cpu_boot_dev_sts0 &
+ CPU_BOOT_DEV_STS0_SRAM_SCR_EN)
return;
if (gaudi->hw_cap_initialized & HW_CAP_SRAM_SCRAMBLER)
@@ -1951,12 +2009,11 @@ static void gaudi_init_scrambler_hbm(struct hl_device *hdev)
{
struct gaudi_device *gaudi = hdev->asic_specific;
- if (!hdev->asic_prop.fw_security_disabled)
+ if (hdev->asic_prop.fw_security_enabled)
return;
- if (hdev->asic_prop.fw_security_status_valid &&
- (hdev->asic_prop.fw_boot_cpu_security_map &
- CPU_BOOT_DEV_STS0_DRAM_SCR_EN))
+ if (hdev->asic_prop.fw_bootfit_cpu_boot_dev_sts0 &
+ CPU_BOOT_DEV_STS0_DRAM_SCR_EN)
return;
if (gaudi->hw_cap_initialized & HW_CAP_HBM_SCRAMBLER)
@@ -2021,12 +2078,11 @@ static void gaudi_init_scrambler_hbm(struct hl_device *hdev)
static void gaudi_init_e2e(struct hl_device *hdev)
{
- if (!hdev->asic_prop.fw_security_disabled)
+ if (hdev->asic_prop.fw_security_enabled)
return;
- if (hdev->asic_prop.fw_security_status_valid &&
- (hdev->asic_prop.fw_boot_cpu_security_map &
- CPU_BOOT_DEV_STS0_E2E_CRED_EN))
+ if (hdev->asic_prop.fw_bootfit_cpu_boot_dev_sts0 &
+ CPU_BOOT_DEV_STS0_E2E_CRED_EN)
return;
WREG32(mmSIF_RTR_CTRL_0_E2E_HBM_WR_SIZE, 247 >> 3);
@@ -2396,12 +2452,11 @@ static void gaudi_init_hbm_cred(struct hl_device *hdev)
{
uint32_t hbm0_wr, hbm1_wr, hbm0_rd, hbm1_rd;
- if (!hdev->asic_prop.fw_security_disabled)
+ if (hdev->asic_prop.fw_security_enabled)
return;
- if (hdev->asic_prop.fw_security_status_valid &&
- (hdev->asic_prop.fw_boot_cpu_security_map &
- CPU_BOOT_DEV_STS0_HBM_CRED_EN))
+ if (hdev->asic_prop.fw_bootfit_cpu_boot_dev_sts0 &
+ CPU_BOOT_DEV_STS0_HBM_CRED_EN)
return;
hbm0_wr = 0x33333333;
@@ -2487,10 +2542,12 @@ static void gaudi_init_golden_registers(struct hl_device *hdev)
static void gaudi_init_pci_dma_qman(struct hl_device *hdev, int dma_id,
int qman_id, dma_addr_t qman_pq_addr)
{
+ struct cpu_dyn_regs *dyn_regs =
+ &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs;
u32 mtr_base_en_lo, mtr_base_en_hi, mtr_base_ws_lo, mtr_base_ws_hi;
u32 so_base_en_lo, so_base_en_hi, so_base_ws_lo, so_base_ws_hi;
u32 q_off, dma_qm_offset;
- u32 dma_qm_err_cfg;
+ u32 dma_qm_err_cfg, irq_handler_offset;
dma_qm_offset = dma_id * DMA_QMAN_OFFSET;
@@ -2539,20 +2596,23 @@ static void gaudi_init_pci_dma_qman(struct hl_device *hdev, int dma_id,
/* The following configuration is needed only once per QMAN */
if (qman_id == 0) {
+ irq_handler_offset = hdev->asic_prop.gic_interrupts_enable ?
+ mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR :
+ le32_to_cpu(dyn_regs->gic_dma_qm_irq_ctrl);
+
/* Configure RAZWI IRQ */
dma_qm_err_cfg = PCI_DMA_QMAN_GLBL_ERR_CFG_MSG_EN_MASK;
- if (hdev->stop_on_err) {
+ if (hdev->stop_on_err)
dma_qm_err_cfg |=
PCI_DMA_QMAN_GLBL_ERR_CFG_STOP_ON_ERR_EN_MASK;
- }
WREG32(mmDMA0_QM_GLBL_ERR_CFG + dma_qm_offset, dma_qm_err_cfg);
+
WREG32(mmDMA0_QM_GLBL_ERR_ADDR_LO + dma_qm_offset,
- lower_32_bits(CFG_BASE +
- mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR));
+ lower_32_bits(CFG_BASE + irq_handler_offset));
WREG32(mmDMA0_QM_GLBL_ERR_ADDR_HI + dma_qm_offset,
- upper_32_bits(CFG_BASE +
- mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR));
+ upper_32_bits(CFG_BASE + irq_handler_offset));
+
WREG32(mmDMA0_QM_GLBL_ERR_WDATA + dma_qm_offset,
gaudi_irq_map_table[GAUDI_EVENT_DMA0_QM].cpu_id +
dma_id);
@@ -2573,8 +2633,11 @@ static void gaudi_init_pci_dma_qman(struct hl_device *hdev, int dma_id,
static void gaudi_init_dma_core(struct hl_device *hdev, int dma_id)
{
- u32 dma_offset = dma_id * DMA_CORE_OFFSET;
+ struct cpu_dyn_regs *dyn_regs =
+ &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs;
u32 dma_err_cfg = 1 << DMA0_CORE_ERR_CFG_ERR_MSG_EN_SHIFT;
+ u32 dma_offset = dma_id * DMA_CORE_OFFSET;
+ u32 irq_handler_offset;
/* Set to maximum possible according to physical size */
WREG32(mmDMA0_CORE_RD_MAX_OUTSTAND + dma_offset, 0);
@@ -2588,10 +2651,16 @@ static void gaudi_init_dma_core(struct hl_device *hdev, int dma_id)
dma_err_cfg |= 1 << DMA0_CORE_ERR_CFG_STOP_ON_ERR_SHIFT;
WREG32(mmDMA0_CORE_ERR_CFG + dma_offset, dma_err_cfg);
+
+ irq_handler_offset = hdev->asic_prop.gic_interrupts_enable ?
+ mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR :
+ le32_to_cpu(dyn_regs->gic_dma_core_irq_ctrl);
+
WREG32(mmDMA0_CORE_ERRMSG_ADDR_LO + dma_offset,
- lower_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR));
+ lower_32_bits(CFG_BASE + irq_handler_offset));
WREG32(mmDMA0_CORE_ERRMSG_ADDR_HI + dma_offset,
- upper_32_bits(CFG_BASE + mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR));
+ upper_32_bits(CFG_BASE + irq_handler_offset));
+
WREG32(mmDMA0_CORE_ERRMSG_WDATA + dma_offset,
gaudi_irq_map_table[GAUDI_EVENT_DMA0_CORE].cpu_id + dma_id);
WREG32(mmDMA0_CORE_PROT + dma_offset,
@@ -2654,10 +2723,12 @@ static void gaudi_init_pci_dma_qmans(struct hl_device *hdev)
static void gaudi_init_hbm_dma_qman(struct hl_device *hdev, int dma_id,
int qman_id, u64 qman_base_addr)
{
+ struct cpu_dyn_regs *dyn_regs =
+ &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs;
u32 mtr_base_en_lo, mtr_base_en_hi, mtr_base_ws_lo, mtr_base_ws_hi;
u32 so_base_en_lo, so_base_en_hi, so_base_ws_lo, so_base_ws_hi;
+ u32 dma_qm_err_cfg, irq_handler_offset;
u32 q_off, dma_qm_offset;
- u32 dma_qm_err_cfg;
dma_qm_offset = dma_id * DMA_QMAN_OFFSET;
@@ -2697,6 +2768,10 @@ static void gaudi_init_hbm_dma_qman(struct hl_device *hdev, int dma_id,
WREG32(mmDMA0_QM_CP_LDMA_DST_BASE_LO_OFFSET_0 + q_off,
QMAN_CPDMA_DST_OFFSET);
} else {
+ irq_handler_offset = hdev->asic_prop.gic_interrupts_enable ?
+ mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR :
+ le32_to_cpu(dyn_regs->gic_dma_qm_irq_ctrl);
+
WREG32(mmDMA0_QM_CP_LDMA_TSIZE_OFFSET_0 + q_off,
QMAN_LDMA_SIZE_OFFSET);
WREG32(mmDMA0_QM_CP_LDMA_SRC_BASE_LO_OFFSET_0 + q_off,
@@ -2706,18 +2781,17 @@ static void gaudi_init_hbm_dma_qman(struct hl_device *hdev, int dma_id,
/* Configure RAZWI IRQ */
dma_qm_err_cfg = HBM_DMA_QMAN_GLBL_ERR_CFG_MSG_EN_MASK;
- if (hdev->stop_on_err) {
+ if (hdev->stop_on_err)
dma_qm_err_cfg |=
HBM_DMA_QMAN_GLBL_ERR_CFG_STOP_ON_ERR_EN_MASK;
- }
+
WREG32(mmDMA0_QM_GLBL_ERR_CFG + dma_qm_offset, dma_qm_err_cfg);
WREG32(mmDMA0_QM_GLBL_ERR_ADDR_LO + dma_qm_offset,
- lower_32_bits(CFG_BASE +
- mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR));
+ lower_32_bits(CFG_BASE + irq_handler_offset));
WREG32(mmDMA0_QM_GLBL_ERR_ADDR_HI + dma_qm_offset,
- upper_32_bits(CFG_BASE +
- mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR));
+ upper_32_bits(CFG_BASE + irq_handler_offset));
+
WREG32(mmDMA0_QM_GLBL_ERR_WDATA + dma_qm_offset,
gaudi_irq_map_table[GAUDI_EVENT_DMA0_QM].cpu_id +
dma_id);
@@ -2792,8 +2866,11 @@ static void gaudi_init_hbm_dma_qmans(struct hl_device *hdev)
static void gaudi_init_mme_qman(struct hl_device *hdev, u32 mme_offset,
int qman_id, u64 qman_base_addr)
{
+ struct cpu_dyn_regs *dyn_regs =
+ &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs;
u32 mtr_base_lo, mtr_base_hi;
u32 so_base_lo, so_base_hi;
+ u32 irq_handler_offset;
u32 q_off, mme_id;
u32 mme_qm_err_cfg;
@@ -2825,6 +2902,10 @@ static void gaudi_init_mme_qman(struct hl_device *hdev, u32 mme_offset,
WREG32(mmMME0_QM_CP_LDMA_DST_BASE_LO_OFFSET_0 + q_off,
QMAN_CPDMA_DST_OFFSET);
} else {
+ irq_handler_offset = hdev->asic_prop.gic_interrupts_enable ?
+ mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR :
+ le32_to_cpu(dyn_regs->gic_mme_qm_irq_ctrl);
+
WREG32(mmMME0_QM_CP_LDMA_TSIZE_OFFSET_0 + q_off,
QMAN_LDMA_SIZE_OFFSET);
WREG32(mmMME0_QM_CP_LDMA_SRC_BASE_LO_OFFSET_0 + q_off,
@@ -2834,20 +2915,20 @@ static void gaudi_init_mme_qman(struct hl_device *hdev, u32 mme_offset,
/* Configure RAZWI IRQ */
mme_id = mme_offset /
- (mmMME1_QM_GLBL_CFG0 - mmMME0_QM_GLBL_CFG0);
+ (mmMME1_QM_GLBL_CFG0 - mmMME0_QM_GLBL_CFG0) / 2;
mme_qm_err_cfg = MME_QMAN_GLBL_ERR_CFG_MSG_EN_MASK;
- if (hdev->stop_on_err) {
+ if (hdev->stop_on_err)
mme_qm_err_cfg |=
MME_QMAN_GLBL_ERR_CFG_STOP_ON_ERR_EN_MASK;
- }
+
WREG32(mmMME0_QM_GLBL_ERR_CFG + mme_offset, mme_qm_err_cfg);
+
WREG32(mmMME0_QM_GLBL_ERR_ADDR_LO + mme_offset,
- lower_32_bits(CFG_BASE +
- mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR));
+ lower_32_bits(CFG_BASE + irq_handler_offset));
WREG32(mmMME0_QM_GLBL_ERR_ADDR_HI + mme_offset,
- upper_32_bits(CFG_BASE +
- mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR));
+ upper_32_bits(CFG_BASE + irq_handler_offset));
+
WREG32(mmMME0_QM_GLBL_ERR_WDATA + mme_offset,
gaudi_irq_map_table[GAUDI_EVENT_MME0_QM].cpu_id +
mme_id);
@@ -2912,10 +2993,12 @@ static void gaudi_init_mme_qmans(struct hl_device *hdev)
static void gaudi_init_tpc_qman(struct hl_device *hdev, u32 tpc_offset,
int qman_id, u64 qman_base_addr)
{
+ struct cpu_dyn_regs *dyn_regs =
+ &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs;
u32 mtr_base_en_lo, mtr_base_en_hi, mtr_base_ws_lo, mtr_base_ws_hi;
u32 so_base_en_lo, so_base_en_hi, so_base_ws_lo, so_base_ws_hi;
+ u32 tpc_qm_err_cfg, irq_handler_offset;
u32 q_off, tpc_id;
- u32 tpc_qm_err_cfg;
mtr_base_en_lo = lower_32_bits(CFG_BASE +
mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0);
@@ -2956,6 +3039,10 @@ static void gaudi_init_tpc_qman(struct hl_device *hdev, u32 tpc_offset,
WREG32(mmTPC0_QM_CP_LDMA_DST_BASE_LO_OFFSET_0 + q_off,
QMAN_CPDMA_DST_OFFSET);
} else {
+ irq_handler_offset = hdev->asic_prop.gic_interrupts_enable ?
+ mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR :
+ le32_to_cpu(dyn_regs->gic_tpc_qm_irq_ctrl);
+
WREG32(mmTPC0_QM_CP_LDMA_TSIZE_OFFSET_0 + q_off,
QMAN_LDMA_SIZE_OFFSET);
WREG32(mmTPC0_QM_CP_LDMA_SRC_BASE_LO_OFFSET_0 + q_off,
@@ -2965,18 +3052,17 @@ static void gaudi_init_tpc_qman(struct hl_device *hdev, u32 tpc_offset,
/* Configure RAZWI IRQ */
tpc_qm_err_cfg = TPC_QMAN_GLBL_ERR_CFG_MSG_EN_MASK;
- if (hdev->stop_on_err) {
+ if (hdev->stop_on_err)
tpc_qm_err_cfg |=
TPC_QMAN_GLBL_ERR_CFG_STOP_ON_ERR_EN_MASK;
- }
WREG32(mmTPC0_QM_GLBL_ERR_CFG + tpc_offset, tpc_qm_err_cfg);
+
WREG32(mmTPC0_QM_GLBL_ERR_ADDR_LO + tpc_offset,
- lower_32_bits(CFG_BASE +
- mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR));
+ lower_32_bits(CFG_BASE + irq_handler_offset));
WREG32(mmTPC0_QM_GLBL_ERR_ADDR_HI + tpc_offset,
- upper_32_bits(CFG_BASE +
- mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR));
+ upper_32_bits(CFG_BASE + irq_handler_offset));
+
WREG32(mmTPC0_QM_GLBL_ERR_WDATA + tpc_offset,
gaudi_irq_map_table[GAUDI_EVENT_TPC0_QM].cpu_id +
tpc_id);
@@ -3059,10 +3145,12 @@ static void gaudi_init_tpc_qmans(struct hl_device *hdev)
static void gaudi_init_nic_qman(struct hl_device *hdev, u32 nic_offset,
int qman_id, u64 qman_base_addr, int nic_id)
{
+ struct cpu_dyn_regs *dyn_regs =
+ &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs;
u32 mtr_base_en_lo, mtr_base_en_hi, mtr_base_ws_lo, mtr_base_ws_hi;
u32 so_base_en_lo, so_base_en_hi, so_base_ws_lo, so_base_ws_hi;
+ u32 nic_qm_err_cfg, irq_handler_offset;
u32 q_off;
- u32 nic_qm_err_cfg;
mtr_base_en_lo = lower_32_bits(CFG_BASE +
mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0);
@@ -3109,20 +3197,23 @@ static void gaudi_init_nic_qman(struct hl_device *hdev, u32 nic_offset,
WREG32(mmNIC0_QM0_CP_MSG_BASE3_ADDR_HI_0 + q_off, so_base_ws_hi);
if (qman_id == 0) {
+ irq_handler_offset = hdev->asic_prop.gic_interrupts_enable ?
+ mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR :
+ le32_to_cpu(dyn_regs->gic_nic_qm_irq_ctrl);
+
/* Configure RAZWI IRQ */
nic_qm_err_cfg = NIC_QMAN_GLBL_ERR_CFG_MSG_EN_MASK;
- if (hdev->stop_on_err) {
+ if (hdev->stop_on_err)
nic_qm_err_cfg |=
NIC_QMAN_GLBL_ERR_CFG_STOP_ON_ERR_EN_MASK;
- }
WREG32(mmNIC0_QM0_GLBL_ERR_CFG + nic_offset, nic_qm_err_cfg);
+
WREG32(mmNIC0_QM0_GLBL_ERR_ADDR_LO + nic_offset,
- lower_32_bits(CFG_BASE +
- mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR));
+ lower_32_bits(CFG_BASE + irq_handler_offset));
WREG32(mmNIC0_QM0_GLBL_ERR_ADDR_HI + nic_offset,
- upper_32_bits(CFG_BASE +
- mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR));
+ upper_32_bits(CFG_BASE + irq_handler_offset));
+
WREG32(mmNIC0_QM0_GLBL_ERR_WDATA + nic_offset,
gaudi_irq_map_table[GAUDI_EVENT_NIC0_QM0].cpu_id +
nic_id);
@@ -3475,7 +3566,7 @@ static void gaudi_set_clock_gating(struct hl_device *hdev)
if (hdev->in_debug)
return;
- if (!hdev->asic_prop.fw_security_disabled)
+ if (hdev->asic_prop.fw_security_enabled)
return;
for (i = GAUDI_PCI_DMA_1, qman_offset = 0 ; i < GAUDI_HBM_DMA_1 ; i++) {
@@ -3535,7 +3626,7 @@ static void gaudi_disable_clock_gating(struct hl_device *hdev)
u32 qman_offset;
int i;
- if (!hdev->asic_prop.fw_security_disabled)
+ if (hdev->asic_prop.fw_security_enabled)
return;
for (i = 0, qman_offset = 0 ; i < DMA_NUMBER_OF_CHANNELS ; i++) {
@@ -3674,9 +3765,6 @@ static int gaudi_load_firmware_to_device(struct hl_device *hdev)
{
void __iomem *dst;
- /* HBM scrambler must be initialized before pushing F/W to HBM */
- gaudi_init_scrambler_hbm(hdev);
-
dst = hdev->pcie_bar[HBM_BAR_ID] + LINUX_FW_OFFSET;
return hl_fw_load_fw_to_device(hdev, GAUDI_LINUX_FW_FILE, dst, 0, 0);
@@ -3691,42 +3779,71 @@ static int gaudi_load_boot_fit_to_device(struct hl_device *hdev)
return hl_fw_load_fw_to_device(hdev, GAUDI_BOOT_FIT_FILE, dst, 0, 0);
}
-static int gaudi_read_device_fw_version(struct hl_device *hdev,
- enum hl_fw_component fwc)
+static void gaudi_init_dynamic_firmware_loader(struct hl_device *hdev)
{
- const char *name;
- u32 ver_off;
- char *dest;
+ struct dynamic_fw_load_mgr *dynamic_loader;
+ struct cpu_dyn_regs *dyn_regs;
- switch (fwc) {
- case FW_COMP_UBOOT:
- ver_off = RREG32(mmUBOOT_VER_OFFSET);
- dest = hdev->asic_prop.uboot_ver;
- name = "U-Boot";
- break;
- case FW_COMP_PREBOOT:
- ver_off = RREG32(mmPREBOOT_VER_OFFSET);
- dest = hdev->asic_prop.preboot_ver;
- name = "Preboot";
- break;
- default:
- dev_warn(hdev->dev, "Undefined FW component: %d\n", fwc);
- return -EIO;
- }
+ dynamic_loader = &hdev->fw_loader.dynamic_loader;
- ver_off &= ~((u32)SRAM_BASE_ADDR);
+ /*
+ * here we update initial values for few specific dynamic regs (as
+ * before reading the first descriptor from FW those value has to be
+ * hard-coded) in later stages of the protocol those values will be
+ * updated automatically by reading the FW descriptor so data there
+ * will always be up-to-date
+ */
+ dyn_regs = &dynamic_loader->comm_desc.cpu_dyn_regs;
+ dyn_regs->kmd_msg_to_cpu =
+ cpu_to_le32(mmPSOC_GLOBAL_CONF_KMD_MSG_TO_CPU);
+ dyn_regs->cpu_cmd_status_to_host =
+ cpu_to_le32(mmCPU_CMD_STATUS_TO_HOST);
- if (ver_off < SRAM_SIZE - VERSION_MAX_LEN) {
- memcpy_fromio(dest, hdev->pcie_bar[SRAM_BAR_ID] + ver_off,
- VERSION_MAX_LEN);
- } else {
- dev_err(hdev->dev, "%s version offset (0x%x) is above SRAM\n",
- name, ver_off);
- strcpy(dest, "unavailable");
- return -EIO;
- }
+ dynamic_loader->wait_for_bl_timeout = GAUDI_WAIT_FOR_BL_TIMEOUT_USEC;
+}
- return 0;
+static void gaudi_init_static_firmware_loader(struct hl_device *hdev)
+{
+ struct static_fw_load_mgr *static_loader;
+
+ static_loader = &hdev->fw_loader.static_loader;
+
+ static_loader->preboot_version_max_off = SRAM_SIZE - VERSION_MAX_LEN;
+ static_loader->boot_fit_version_max_off = SRAM_SIZE - VERSION_MAX_LEN;
+ static_loader->kmd_msg_to_cpu_reg = mmPSOC_GLOBAL_CONF_KMD_MSG_TO_CPU;
+ static_loader->cpu_cmd_status_to_host_reg = mmCPU_CMD_STATUS_TO_HOST;
+ static_loader->cpu_boot_status_reg = mmPSOC_GLOBAL_CONF_CPU_BOOT_STATUS;
+ static_loader->cpu_boot_dev_status0_reg = mmCPU_BOOT_DEV_STS0;
+ static_loader->cpu_boot_dev_status1_reg = mmCPU_BOOT_DEV_STS1;
+ static_loader->boot_err0_reg = mmCPU_BOOT_ERR0;
+ static_loader->boot_err1_reg = mmCPU_BOOT_ERR1;
+ static_loader->preboot_version_offset_reg = mmPREBOOT_VER_OFFSET;
+ static_loader->boot_fit_version_offset_reg = mmUBOOT_VER_OFFSET;
+ static_loader->sram_offset_mask = ~(lower_32_bits(SRAM_BASE_ADDR));
+ static_loader->cpu_reset_wait_msec = hdev->pldm ?
+ GAUDI_PLDM_RESET_WAIT_MSEC :
+ GAUDI_CPU_RESET_WAIT_MSEC;
+}
+
+static void gaudi_init_firmware_loader(struct hl_device *hdev)
+{
+ struct asic_fixed_properties *prop = &hdev->asic_prop;
+ struct fw_load_mgr *fw_loader = &hdev->fw_loader;
+
+ /* fill common fields */
+ fw_loader->linux_loaded = false;
+ fw_loader->boot_fit_img.image_name = GAUDI_BOOT_FIT_FILE;
+ fw_loader->linux_img.image_name = GAUDI_LINUX_FW_FILE;
+ fw_loader->cpu_timeout = GAUDI_CPU_TIMEOUT_USEC;
+ fw_loader->boot_fit_timeout = GAUDI_BOOT_FIT_REQ_TIMEOUT_USEC;
+ fw_loader->skip_bmc = !hdev->bmc_enable;
+ fw_loader->sram_bar_id = SRAM_BAR_ID;
+ fw_loader->dram_bar_id = HBM_BAR_ID;
+
+ if (prop->dynamic_fw_load)
+ gaudi_init_dynamic_firmware_loader(hdev);
+ else
+ gaudi_init_static_firmware_loader(hdev);
}
static int gaudi_init_cpu(struct hl_device *hdev)
@@ -3744,15 +3861,10 @@ static int gaudi_init_cpu(struct hl_device *hdev)
* The device CPU works with 40 bits addresses.
* This register sets the extension to 50 bits.
*/
- if (hdev->asic_prop.fw_security_disabled)
+ if (!hdev->asic_prop.fw_security_enabled)
WREG32(mmCPU_IF_CPU_MSB_ADDR, hdev->cpu_pci_msb_addr);
- rc = hl_fw_init_cpu(hdev, mmPSOC_GLOBAL_CONF_CPU_BOOT_STATUS,
- mmPSOC_GLOBAL_CONF_KMD_MSG_TO_CPU,
- mmCPU_CMD_STATUS_TO_HOST,
- mmCPU_BOOT_DEV_STS0, mmCPU_BOOT_ERR0,
- !hdev->bmc_enable, GAUDI_CPU_TIMEOUT_USEC,
- GAUDI_BOOT_FIT_REQ_TIMEOUT_USEC);
+ rc = hl_fw_init_cpu(hdev);
if (rc)
return rc;
@@ -3764,10 +3876,12 @@ static int gaudi_init_cpu(struct hl_device *hdev)
static int gaudi_init_cpu_queues(struct hl_device *hdev, u32 cpu_timeout)
{
- struct gaudi_device *gaudi = hdev->asic_specific;
+ struct cpu_dyn_regs *dyn_regs =
+ &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs;
struct asic_fixed_properties *prop = &hdev->asic_prop;
+ struct gaudi_device *gaudi = hdev->asic_specific;
+ u32 status, irq_handler_offset;
struct hl_eq *eq;
- u32 status;
struct hl_hw_queue *cpu_pq =
&hdev->kernel_queues[GAUDI_QUEUE_ID_CPU_PQ];
int err;
@@ -3806,7 +3920,12 @@ static int gaudi_init_cpu_queues(struct hl_device *hdev, u32 cpu_timeout)
WREG32(mmCPU_IF_QUEUE_INIT,
PQ_INIT_STATUS_READY_FOR_CP_SINGLE_MSI);
- WREG32(mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR, GAUDI_EVENT_PI_UPDATE);
+ irq_handler_offset = prop->gic_interrupts_enable ?
+ mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR :
+ le32_to_cpu(dyn_regs->gic_host_pi_upd_irq);
+
+ WREG32(irq_handler_offset,
+ gaudi_irq_map_table[GAUDI_EVENT_PI_UPDATE].cpu_id);
err = hl_poll_timeout(
hdev,
@@ -3823,8 +3942,10 @@ static int gaudi_init_cpu_queues(struct hl_device *hdev, u32 cpu_timeout)
}
/* update FW application security bits */
- if (prop->fw_security_status_valid)
- prop->fw_app_security_map = RREG32(mmCPU_BOOT_DEV_STS0);
+ if (prop->fw_cpu_boot_dev_sts0_valid)
+ prop->fw_app_cpu_boot_dev_sts0 = RREG32(mmCPU_BOOT_DEV_STS0);
+ if (prop->fw_cpu_boot_dev_sts1_valid)
+ prop->fw_app_cpu_boot_dev_sts1 = RREG32(mmCPU_BOOT_DEV_STS1);
gaudi->hw_cap_initialized |= HW_CAP_CPU_Q;
return 0;
@@ -3835,7 +3956,7 @@ static void gaudi_pre_hw_init(struct hl_device *hdev)
/* Perform read from the device to make sure device is up */
RREG32(mmHW_STATE);
- if (hdev->asic_prop.fw_security_disabled) {
+ if (!hdev->asic_prop.fw_security_enabled) {
/* Set the access through PCI bars (Linux driver only) as
* secured
*/
@@ -3860,13 +3981,27 @@ static void gaudi_pre_hw_init(struct hl_device *hdev)
static int gaudi_hw_init(struct hl_device *hdev)
{
+ struct gaudi_device *gaudi = hdev->asic_specific;
int rc;
gaudi_pre_hw_init(hdev);
- gaudi_init_pci_dma_qmans(hdev);
+ /* If iATU is done by FW, the HBM bar ALWAYS points to DRAM_PHYS_BASE.
+ * So we set it here and if anyone tries to move it later to
+ * a different address, there will be an error
+ */
+ if (hdev->asic_prop.iatu_done_by_fw)
+ gaudi->hbm_bar_cur_addr = DRAM_PHYS_BASE;
- gaudi_init_hbm_dma_qmans(hdev);
+ /*
+ * Before pushing u-boot/linux to device, need to set the hbm bar to
+ * base address of dram
+ */
+ if (gaudi_set_hbm_bar_base(hdev, DRAM_PHYS_BASE) == U64_MAX) {
+ dev_err(hdev->dev,
+ "failed to map HBM bar to DRAM base address\n");
+ return -EIO;
+ }
rc = gaudi_init_cpu(hdev);
if (rc) {
@@ -3895,6 +4030,10 @@ static int gaudi_hw_init(struct hl_device *hdev)
gaudi_init_security(hdev);
+ gaudi_init_pci_dma_qmans(hdev);
+
+ gaudi_init_hbm_dma_qmans(hdev);
+
gaudi_init_mme_qmans(hdev);
gaudi_init_tpc_qmans(hdev);
@@ -3934,8 +4073,11 @@ disable_queues:
static void gaudi_hw_fini(struct hl_device *hdev, bool hard_reset)
{
+ struct cpu_dyn_regs *dyn_regs =
+ &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs;
+ u32 status, reset_timeout_ms, cpu_timeout_ms, irq_handler_offset;
struct gaudi_device *gaudi = hdev->asic_specific;
- u32 status, reset_timeout_ms, cpu_timeout_ms;
+ bool driver_performs_reset;
if (!hard_reset) {
dev_err(hdev->dev, "GAUDI doesn't support soft-reset\n");
@@ -3950,26 +4092,35 @@ static void gaudi_hw_fini(struct hl_device *hdev, bool hard_reset)
cpu_timeout_ms = GAUDI_CPU_RESET_WAIT_MSEC;
}
+ driver_performs_reset = !!(!hdev->asic_prop.fw_security_enabled &&
+ !hdev->asic_prop.hard_reset_done_by_fw);
+
/* Set device to handle FLR by H/W as we will put the device CPU to
* halt mode
*/
- if (hdev->asic_prop.fw_security_disabled &&
- !hdev->asic_prop.hard_reset_done_by_fw)
+ if (driver_performs_reset)
WREG32(mmPCIE_AUX_FLR_CTRL, (PCIE_AUX_FLR_CTRL_HW_CTRL_MASK |
PCIE_AUX_FLR_CTRL_INT_MASK_MASK));
- /* I don't know what is the state of the CPU so make sure it is
- * stopped in any means necessary
+ /* If linux is loaded in the device CPU we need to communicate with it
+ * via the GIC. Otherwise, we need to use COMMS or the MSG_TO_CPU
+ * registers in case of old F/Ws
*/
- if (hdev->asic_prop.hard_reset_done_by_fw)
- WREG32(mmPSOC_GLOBAL_CONF_KMD_MSG_TO_CPU, KMD_MSG_RST_DEV);
- else
- WREG32(mmPSOC_GLOBAL_CONF_KMD_MSG_TO_CPU, KMD_MSG_GOTO_WFE);
+ if (hdev->fw_loader.linux_loaded) {
+ irq_handler_offset = hdev->asic_prop.gic_interrupts_enable ?
+ mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR :
+ le32_to_cpu(dyn_regs->gic_host_halt_irq);
- WREG32(mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR, GAUDI_EVENT_HALT_MACHINE);
+ WREG32(irq_handler_offset,
+ gaudi_irq_map_table[GAUDI_EVENT_HALT_MACHINE].cpu_id);
+ } else {
+ if (hdev->asic_prop.hard_reset_done_by_fw)
+ hl_fw_ask_hard_reset_without_linux(hdev);
+ else
+ hl_fw_ask_halt_machine_without_linux(hdev);
+ }
- if (hdev->asic_prop.fw_security_disabled &&
- !hdev->asic_prop.hard_reset_done_by_fw) {
+ if (driver_performs_reset) {
/* Configure the reset registers. Must be done as early as
* possible in case we fail during H/W initialization
@@ -4003,8 +4154,7 @@ static void gaudi_hw_fini(struct hl_device *hdev, bool hard_reset)
WREG32(mmPREBOOT_PCIE_EN, LKD_HARD_RESET_MAGIC);
/* Restart BTL/BLR upon hard-reset */
- if (hdev->asic_prop.fw_security_disabled)
- WREG32(mmPSOC_GLOBAL_CONF_BOOT_SEQ_RE_START, 1);
+ WREG32(mmPSOC_GLOBAL_CONF_BOOT_SEQ_RE_START, 1);
WREG32(mmPSOC_GLOBAL_CONF_SW_ALL_RST,
1 << PSOC_GLOBAL_CONF_SW_ALL_RST_IND_SHIFT);
@@ -4041,6 +4191,8 @@ static void gaudi_hw_fini(struct hl_device *hdev, bool hard_reset)
HW_CAP_CLK_GATE);
memset(gaudi->events_stat, 0, sizeof(gaudi->events_stat));
+
+ hdev->device_cpu_is_halted = false;
}
}
@@ -4078,10 +4230,12 @@ static int gaudi_cb_mmap(struct hl_device *hdev, struct vm_area_struct *vma,
static void gaudi_ring_doorbell(struct hl_device *hdev, u32 hw_queue_id, u32 pi)
{
+ struct cpu_dyn_regs *dyn_regs =
+ &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs;
+ u32 db_reg_offset, db_value, dma_qm_offset, q_off, irq_handler_offset;
struct gaudi_device *gaudi = hdev->asic_specific;
- u32 db_reg_offset, db_value, dma_qm_offset, q_off;
- int dma_id;
bool invalid_queue = false;
+ int dma_id;
switch (hw_queue_id) {
case GAUDI_QUEUE_ID_DMA_0_0...GAUDI_QUEUE_ID_DMA_0_3:
@@ -4307,164 +4461,84 @@ static void gaudi_ring_doorbell(struct hl_device *hdev, u32 hw_queue_id, u32 pi)
db_reg_offset = mmTPC7_QM_PQ_PI_3;
break;
- case GAUDI_QUEUE_ID_NIC_0_0:
- db_reg_offset = mmNIC0_QM0_PQ_PI_0;
- break;
-
- case GAUDI_QUEUE_ID_NIC_0_1:
- db_reg_offset = mmNIC0_QM0_PQ_PI_1;
- break;
-
- case GAUDI_QUEUE_ID_NIC_0_2:
- db_reg_offset = mmNIC0_QM0_PQ_PI_2;
- break;
-
- case GAUDI_QUEUE_ID_NIC_0_3:
- db_reg_offset = mmNIC0_QM0_PQ_PI_3;
- break;
-
- case GAUDI_QUEUE_ID_NIC_1_0:
- db_reg_offset = mmNIC0_QM1_PQ_PI_0;
- break;
-
- case GAUDI_QUEUE_ID_NIC_1_1:
- db_reg_offset = mmNIC0_QM1_PQ_PI_1;
- break;
-
- case GAUDI_QUEUE_ID_NIC_1_2:
- db_reg_offset = mmNIC0_QM1_PQ_PI_2;
- break;
-
- case GAUDI_QUEUE_ID_NIC_1_3:
- db_reg_offset = mmNIC0_QM1_PQ_PI_3;
- break;
-
- case GAUDI_QUEUE_ID_NIC_2_0:
- db_reg_offset = mmNIC1_QM0_PQ_PI_0;
- break;
-
- case GAUDI_QUEUE_ID_NIC_2_1:
- db_reg_offset = mmNIC1_QM0_PQ_PI_1;
- break;
-
- case GAUDI_QUEUE_ID_NIC_2_2:
- db_reg_offset = mmNIC1_QM0_PQ_PI_2;
- break;
-
- case GAUDI_QUEUE_ID_NIC_2_3:
- db_reg_offset = mmNIC1_QM0_PQ_PI_3;
- break;
-
- case GAUDI_QUEUE_ID_NIC_3_0:
- db_reg_offset = mmNIC1_QM1_PQ_PI_0;
- break;
-
- case GAUDI_QUEUE_ID_NIC_3_1:
- db_reg_offset = mmNIC1_QM1_PQ_PI_1;
- break;
-
- case GAUDI_QUEUE_ID_NIC_3_2:
- db_reg_offset = mmNIC1_QM1_PQ_PI_2;
- break;
-
- case GAUDI_QUEUE_ID_NIC_3_3:
- db_reg_offset = mmNIC1_QM1_PQ_PI_3;
- break;
-
- case GAUDI_QUEUE_ID_NIC_4_0:
- db_reg_offset = mmNIC2_QM0_PQ_PI_0;
- break;
-
- case GAUDI_QUEUE_ID_NIC_4_1:
- db_reg_offset = mmNIC2_QM0_PQ_PI_1;
- break;
-
- case GAUDI_QUEUE_ID_NIC_4_2:
- db_reg_offset = mmNIC2_QM0_PQ_PI_2;
- break;
-
- case GAUDI_QUEUE_ID_NIC_4_3:
- db_reg_offset = mmNIC2_QM0_PQ_PI_3;
- break;
-
- case GAUDI_QUEUE_ID_NIC_5_0:
- db_reg_offset = mmNIC2_QM1_PQ_PI_0;
- break;
+ case GAUDI_QUEUE_ID_NIC_0_0...GAUDI_QUEUE_ID_NIC_0_3:
+ if (!(gaudi->hw_cap_initialized & HW_CAP_NIC0))
+ invalid_queue = true;
- case GAUDI_QUEUE_ID_NIC_5_1:
- db_reg_offset = mmNIC2_QM1_PQ_PI_1;
+ q_off = ((hw_queue_id - 1) & 0x3) * 4;
+ db_reg_offset = mmNIC0_QM0_PQ_PI_0 + q_off;
break;
- case GAUDI_QUEUE_ID_NIC_5_2:
- db_reg_offset = mmNIC2_QM1_PQ_PI_2;
- break;
+ case GAUDI_QUEUE_ID_NIC_1_0...GAUDI_QUEUE_ID_NIC_1_3:
+ if (!(gaudi->hw_cap_initialized & HW_CAP_NIC1))
+ invalid_queue = true;
- case GAUDI_QUEUE_ID_NIC_5_3:
- db_reg_offset = mmNIC2_QM1_PQ_PI_3;
+ q_off = ((hw_queue_id - 1) & 0x3) * 4;
+ db_reg_offset = mmNIC0_QM1_PQ_PI_0 + q_off;
break;
- case GAUDI_QUEUE_ID_NIC_6_0:
- db_reg_offset = mmNIC3_QM0_PQ_PI_0;
- break;
+ case GAUDI_QUEUE_ID_NIC_2_0...GAUDI_QUEUE_ID_NIC_2_3:
+ if (!(gaudi->hw_cap_initialized & HW_CAP_NIC2))
+ invalid_queue = true;
- case GAUDI_QUEUE_ID_NIC_6_1:
- db_reg_offset = mmNIC3_QM0_PQ_PI_1;
+ q_off = ((hw_queue_id - 1) & 0x3) * 4;
+ db_reg_offset = mmNIC1_QM0_PQ_PI_0 + q_off;
break;
- case GAUDI_QUEUE_ID_NIC_6_2:
- db_reg_offset = mmNIC3_QM0_PQ_PI_2;
- break;
+ case GAUDI_QUEUE_ID_NIC_3_0...GAUDI_QUEUE_ID_NIC_3_3:
+ if (!(gaudi->hw_cap_initialized & HW_CAP_NIC3))
+ invalid_queue = true;
- case GAUDI_QUEUE_ID_NIC_6_3:
- db_reg_offset = mmNIC3_QM0_PQ_PI_3;
+ q_off = ((hw_queue_id - 1) & 0x3) * 4;
+ db_reg_offset = mmNIC1_QM1_PQ_PI_0 + q_off;
break;
- case GAUDI_QUEUE_ID_NIC_7_0:
- db_reg_offset = mmNIC3_QM1_PQ_PI_0;
- break;
+ case GAUDI_QUEUE_ID_NIC_4_0...GAUDI_QUEUE_ID_NIC_4_3:
+ if (!(gaudi->hw_cap_initialized & HW_CAP_NIC4))
+ invalid_queue = true;
- case GAUDI_QUEUE_ID_NIC_7_1:
- db_reg_offset = mmNIC3_QM1_PQ_PI_1;
+ q_off = ((hw_queue_id - 1) & 0x3) * 4;
+ db_reg_offset = mmNIC2_QM0_PQ_PI_0 + q_off;
break;
- case GAUDI_QUEUE_ID_NIC_7_2:
- db_reg_offset = mmNIC3_QM1_PQ_PI_2;
- break;
+ case GAUDI_QUEUE_ID_NIC_5_0...GAUDI_QUEUE_ID_NIC_5_3:
+ if (!(gaudi->hw_cap_initialized & HW_CAP_NIC5))
+ invalid_queue = true;
- case GAUDI_QUEUE_ID_NIC_7_3:
- db_reg_offset = mmNIC3_QM1_PQ_PI_3;
+ q_off = ((hw_queue_id - 1) & 0x3) * 4;
+ db_reg_offset = mmNIC2_QM1_PQ_PI_0 + q_off;
break;
- case GAUDI_QUEUE_ID_NIC_8_0:
- db_reg_offset = mmNIC4_QM0_PQ_PI_0;
- break;
+ case GAUDI_QUEUE_ID_NIC_6_0...GAUDI_QUEUE_ID_NIC_6_3:
+ if (!(gaudi->hw_cap_initialized & HW_CAP_NIC6))
+ invalid_queue = true;
- case GAUDI_QUEUE_ID_NIC_8_1:
- db_reg_offset = mmNIC4_QM0_PQ_PI_1;
+ q_off = ((hw_queue_id - 1) & 0x3) * 4;
+ db_reg_offset = mmNIC3_QM0_PQ_PI_0 + q_off;
break;
- case GAUDI_QUEUE_ID_NIC_8_2:
- db_reg_offset = mmNIC4_QM0_PQ_PI_2;
- break;
+ case GAUDI_QUEUE_ID_NIC_7_0...GAUDI_QUEUE_ID_NIC_7_3:
+ if (!(gaudi->hw_cap_initialized & HW_CAP_NIC7))
+ invalid_queue = true;
- case GAUDI_QUEUE_ID_NIC_8_3:
- db_reg_offset = mmNIC4_QM0_PQ_PI_3;
+ q_off = ((hw_queue_id - 1) & 0x3) * 4;
+ db_reg_offset = mmNIC3_QM1_PQ_PI_0 + q_off;
break;
- case GAUDI_QUEUE_ID_NIC_9_0:
- db_reg_offset = mmNIC4_QM1_PQ_PI_0;
- break;
+ case GAUDI_QUEUE_ID_NIC_8_0...GAUDI_QUEUE_ID_NIC_8_3:
+ if (!(gaudi->hw_cap_initialized & HW_CAP_NIC8))
+ invalid_queue = true;
- case GAUDI_QUEUE_ID_NIC_9_1:
- db_reg_offset = mmNIC4_QM1_PQ_PI_1;
+ q_off = ((hw_queue_id - 1) & 0x3) * 4;
+ db_reg_offset = mmNIC4_QM0_PQ_PI_0 + q_off;
break;
- case GAUDI_QUEUE_ID_NIC_9_2:
- db_reg_offset = mmNIC4_QM1_PQ_PI_2;
- break;
+ case GAUDI_QUEUE_ID_NIC_9_0...GAUDI_QUEUE_ID_NIC_9_3:
+ if (!(gaudi->hw_cap_initialized & HW_CAP_NIC9))
+ invalid_queue = true;
- case GAUDI_QUEUE_ID_NIC_9_3:
- db_reg_offset = mmNIC4_QM1_PQ_PI_3;
+ q_off = ((hw_queue_id - 1) & 0x3) * 4;
+ db_reg_offset = mmNIC4_QM1_PQ_PI_0 + q_off;
break;
default:
@@ -4486,8 +4560,13 @@ static void gaudi_ring_doorbell(struct hl_device *hdev, u32 hw_queue_id, u32 pi)
if (hw_queue_id == GAUDI_QUEUE_ID_CPU_PQ) {
/* make sure device CPU will read latest data from host */
mb();
- WREG32(mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR,
- GAUDI_EVENT_PI_UPDATE);
+
+ irq_handler_offset = hdev->asic_prop.gic_interrupts_enable ?
+ mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR :
+ le32_to_cpu(dyn_regs->gic_host_pi_upd_irq);
+
+ WREG32(irq_handler_offset,
+ gaudi_irq_map_table[GAUDI_EVENT_PI_UPDATE].cpu_id);
}
}
@@ -4934,6 +5013,7 @@ already_pinned:
return 0;
unpin_memory:
+ list_del(&userptr->job_node);
hl_unpin_host_memory(hdev, userptr);
free_userptr:
kfree(userptr);
@@ -6513,7 +6593,7 @@ static void gaudi_mmu_prepare(struct hl_device *hdev, u32 asid)
gaudi_mmu_prepare_reg(hdev, mmMME2_ACC_WBC, asid);
gaudi_mmu_prepare_reg(hdev, mmMME3_ACC_WBC, asid);
- if (hdev->nic_ports_mask & GAUDI_NIC_MASK_NIC0) {
+ if (gaudi->hw_cap_initialized & HW_CAP_NIC0) {
gaudi_mmu_prepare_reg(hdev, mmNIC0_QM0_GLBL_NON_SECURE_PROPS_0,
asid);
gaudi_mmu_prepare_reg(hdev, mmNIC0_QM0_GLBL_NON_SECURE_PROPS_1,
@@ -6526,7 +6606,7 @@ static void gaudi_mmu_prepare(struct hl_device *hdev, u32 asid)
asid);
}
- if (hdev->nic_ports_mask & GAUDI_NIC_MASK_NIC1) {
+ if (gaudi->hw_cap_initialized & HW_CAP_NIC1) {
gaudi_mmu_prepare_reg(hdev, mmNIC0_QM1_GLBL_NON_SECURE_PROPS_0,
asid);
gaudi_mmu_prepare_reg(hdev, mmNIC0_QM1_GLBL_NON_SECURE_PROPS_1,
@@ -6539,7 +6619,7 @@ static void gaudi_mmu_prepare(struct hl_device *hdev, u32 asid)
asid);
}
- if (hdev->nic_ports_mask & GAUDI_NIC_MASK_NIC2) {
+ if (gaudi->hw_cap_initialized & HW_CAP_NIC2) {
gaudi_mmu_prepare_reg(hdev, mmNIC1_QM0_GLBL_NON_SECURE_PROPS_0,
asid);
gaudi_mmu_prepare_reg(hdev, mmNIC1_QM0_GLBL_NON_SECURE_PROPS_1,
@@ -6552,7 +6632,7 @@ static void gaudi_mmu_prepare(struct hl_device *hdev, u32 asid)
asid);
}
- if (hdev->nic_ports_mask & GAUDI_NIC_MASK_NIC3) {
+ if (gaudi->hw_cap_initialized & HW_CAP_NIC3) {
gaudi_mmu_prepare_reg(hdev, mmNIC1_QM1_GLBL_NON_SECURE_PROPS_0,
asid);
gaudi_mmu_prepare_reg(hdev, mmNIC1_QM1_GLBL_NON_SECURE_PROPS_1,
@@ -6565,7 +6645,7 @@ static void gaudi_mmu_prepare(struct hl_device *hdev, u32 asid)
asid);
}
- if (hdev->nic_ports_mask & GAUDI_NIC_MASK_NIC4) {
+ if (gaudi->hw_cap_initialized & HW_CAP_NIC4) {
gaudi_mmu_prepare_reg(hdev, mmNIC2_QM0_GLBL_NON_SECURE_PROPS_0,
asid);
gaudi_mmu_prepare_reg(hdev, mmNIC2_QM0_GLBL_NON_SECURE_PROPS_1,
@@ -6578,7 +6658,7 @@ static void gaudi_mmu_prepare(struct hl_device *hdev, u32 asid)
asid);
}
- if (hdev->nic_ports_mask & GAUDI_NIC_MASK_NIC5) {
+ if (gaudi->hw_cap_initialized & HW_CAP_NIC5) {
gaudi_mmu_prepare_reg(hdev, mmNIC2_QM1_GLBL_NON_SECURE_PROPS_0,
asid);
gaudi_mmu_prepare_reg(hdev, mmNIC2_QM1_GLBL_NON_SECURE_PROPS_1,
@@ -6591,7 +6671,7 @@ static void gaudi_mmu_prepare(struct hl_device *hdev, u32 asid)
asid);
}
- if (hdev->nic_ports_mask & GAUDI_NIC_MASK_NIC6) {
+ if (gaudi->hw_cap_initialized & HW_CAP_NIC6) {
gaudi_mmu_prepare_reg(hdev, mmNIC3_QM0_GLBL_NON_SECURE_PROPS_0,
asid);
gaudi_mmu_prepare_reg(hdev, mmNIC3_QM0_GLBL_NON_SECURE_PROPS_1,
@@ -6604,7 +6684,7 @@ static void gaudi_mmu_prepare(struct hl_device *hdev, u32 asid)
asid);
}
- if (hdev->nic_ports_mask & GAUDI_NIC_MASK_NIC7) {
+ if (gaudi->hw_cap_initialized & HW_CAP_NIC7) {
gaudi_mmu_prepare_reg(hdev, mmNIC3_QM1_GLBL_NON_SECURE_PROPS_0,
asid);
gaudi_mmu_prepare_reg(hdev, mmNIC3_QM1_GLBL_NON_SECURE_PROPS_1,
@@ -6617,7 +6697,7 @@ static void gaudi_mmu_prepare(struct hl_device *hdev, u32 asid)
asid);
}
- if (hdev->nic_ports_mask & GAUDI_NIC_MASK_NIC8) {
+ if (gaudi->hw_cap_initialized & HW_CAP_NIC8) {
gaudi_mmu_prepare_reg(hdev, mmNIC4_QM0_GLBL_NON_SECURE_PROPS_0,
asid);
gaudi_mmu_prepare_reg(hdev, mmNIC4_QM0_GLBL_NON_SECURE_PROPS_1,
@@ -6630,7 +6710,7 @@ static void gaudi_mmu_prepare(struct hl_device *hdev, u32 asid)
asid);
}
- if (hdev->nic_ports_mask & GAUDI_NIC_MASK_NIC9) {
+ if (gaudi->hw_cap_initialized & HW_CAP_NIC9) {
gaudi_mmu_prepare_reg(hdev, mmNIC4_QM1_GLBL_NON_SECURE_PROPS_0,
asid);
gaudi_mmu_prepare_reg(hdev, mmNIC4_QM1_GLBL_NON_SECURE_PROPS_1,
@@ -7044,14 +7124,158 @@ enable_clk_gate:
return rc;
}
+/*
+ * gaudi_queue_idx_dec - decrement queue index (pi/ci) and handle wrap
+ *
+ * @idx: the current pi/ci value
+ * @q_len: the queue length (power of 2)
+ *
+ * @return the cyclically decremented index
+ */
+static inline u32 gaudi_queue_idx_dec(u32 idx, u32 q_len)
+{
+ u32 mask = q_len - 1;
+
+ /*
+ * modular decrement is equivalent to adding (queue_size -1)
+ * later we take LSBs to make sure the value is in the
+ * range [0, queue_len - 1]
+ */
+ return (idx + q_len - 1) & mask;
+}
+
+/**
+ * gaudi_print_sw_config_stream_data - print SW config stream data
+ *
+ * @hdev: pointer to the habanalabs device structure
+ * @stream: the QMAN's stream
+ * @qman_base: base address of QMAN registers block
+ */
+static void gaudi_print_sw_config_stream_data(struct hl_device *hdev, u32 stream,
+ u64 qman_base)
+{
+ u64 cq_ptr_lo, cq_ptr_hi, cq_tsize, cq_ptr;
+ u32 cq_ptr_lo_off, size;
+
+ cq_ptr_lo_off = mmTPC0_QM_CQ_PTR_LO_1 - mmTPC0_QM_CQ_PTR_LO_0;
+
+ cq_ptr_lo = qman_base + (mmTPC0_QM_CQ_PTR_LO_0 - mmTPC0_QM_BASE) +
+ stream * cq_ptr_lo_off;
+ cq_ptr_hi = cq_ptr_lo +
+ (mmTPC0_QM_CQ_PTR_HI_0 - mmTPC0_QM_CQ_PTR_LO_0);
+ cq_tsize = cq_ptr_lo +
+ (mmTPC0_QM_CQ_TSIZE_0 - mmTPC0_QM_CQ_PTR_LO_0);
+
+ cq_ptr = (((u64) RREG32(cq_ptr_hi)) << 32) | RREG32(cq_ptr_lo);
+ size = RREG32(cq_tsize);
+ dev_info(hdev->dev, "stop on err: stream: %u, addr: %#llx, size: %x\n",
+ stream, cq_ptr, size);
+}
+
+/**
+ * gaudi_print_last_pqes_on_err - print last PQEs on error
+ *
+ * @hdev: pointer to the habanalabs device structure
+ * @qid_base: first QID of the QMAN (out of 4 streams)
+ * @stream: the QMAN's stream
+ * @qman_base: base address of QMAN registers block
+ * @pr_sw_conf: if true print the SW config stream data (CQ PTR and SIZE)
+ */
+static void gaudi_print_last_pqes_on_err(struct hl_device *hdev, u32 qid_base,
+ u32 stream, u64 qman_base,
+ bool pr_sw_conf)
+{
+ u32 ci, qm_ci_stream_off, queue_len;
+ struct hl_hw_queue *q;
+ u64 pq_ci;
+ int i;
+
+ q = &hdev->kernel_queues[qid_base + stream];
+
+ qm_ci_stream_off = mmTPC0_QM_PQ_CI_1 - mmTPC0_QM_PQ_CI_0;
+ pq_ci = qman_base + (mmTPC0_QM_PQ_CI_0 - mmTPC0_QM_BASE) +
+ stream * qm_ci_stream_off;
+
+ queue_len = (q->queue_type == QUEUE_TYPE_INT) ?
+ q->int_queue_len : HL_QUEUE_LENGTH;
+
+ hdev->asic_funcs->hw_queues_lock(hdev);
+
+ if (pr_sw_conf)
+ gaudi_print_sw_config_stream_data(hdev, stream, qman_base);
+
+ ci = RREG32(pq_ci);
+
+ /* we should start printing form ci -1 */
+ ci = gaudi_queue_idx_dec(ci, queue_len);
+
+ for (i = 0; i < PQ_FETCHER_CACHE_SIZE; i++) {
+ struct hl_bd *bd;
+ u64 addr;
+ u32 len;
+
+ bd = q->kernel_address;
+ bd += ci;
+
+ len = le32_to_cpu(bd->len);
+ /* len 0 means uninitialized entry- break */
+ if (!len)
+ break;
+
+ addr = le64_to_cpu(bd->ptr);
+
+ dev_info(hdev->dev, "stop on err PQE(stream %u): ci: %u, addr: %#llx, size: %x\n",
+ stream, ci, addr, len);
+
+ /* get previous ci, wrap if needed */
+ ci = gaudi_queue_idx_dec(ci, queue_len);
+ }
+
+ hdev->asic_funcs->hw_queues_unlock(hdev);
+}
+
+/**
+ * print_qman_data_on_err - extract QMAN data on error
+ *
+ * @hdev: pointer to the habanalabs device structure
+ * @qid_base: first QID of the QMAN (out of 4 streams)
+ * @stream: the QMAN's stream
+ * @qman_base: base address of QMAN registers block
+ *
+ * This function attempt to exatract as much data as possible on QMAN error.
+ * On upper CP print the SW config stream data and last 8 PQEs.
+ * On lower CP print SW config data and last PQEs of ALL 4 upper CPs
+ */
+static void print_qman_data_on_err(struct hl_device *hdev, u32 qid_base,
+ u32 stream, u64 qman_base)
+{
+ u32 i;
+
+ if (stream != QMAN_STREAMS) {
+ gaudi_print_last_pqes_on_err(hdev, qid_base, stream, qman_base,
+ true);
+ return;
+ }
+
+ gaudi_print_sw_config_stream_data(hdev, stream, qman_base);
+
+ for (i = 0; i < QMAN_STREAMS; i++)
+ gaudi_print_last_pqes_on_err(hdev, qid_base, i, qman_base,
+ false);
+}
+
static void gaudi_handle_qman_err_generic(struct hl_device *hdev,
const char *qm_name,
- u64 glbl_sts_addr,
- u64 arb_err_addr)
+ u64 qman_base,
+ u32 qid_base)
{
u32 i, j, glbl_sts_val, arb_err_val, glbl_sts_clr_val;
+ u64 glbl_sts_addr, arb_err_addr;
char reg_desc[32];
+ glbl_sts_addr = qman_base + (mmTPC0_QM_GLBL_STS1_0 - mmTPC0_QM_BASE);
+ arb_err_addr = qman_base + (mmTPC0_QM_ARB_ERR_CAUSE - mmTPC0_QM_BASE);
+
/* Iterate through all stream GLBL_STS1 registers + Lower CP */
for (i = 0 ; i < QMAN_STREAMS + 1 ; i++) {
glbl_sts_clr_val = 0;
@@ -7078,6 +7302,8 @@ static void gaudi_handle_qman_err_generic(struct hl_device *hdev,
/* Write 1 clear errors */
if (!hdev->stop_on_err)
WREG32(glbl_sts_addr + 4 * i, glbl_sts_clr_val);
+ else
+ print_qman_data_on_err(hdev, qid_base, i, qman_base);
}
arb_err_val = RREG32(arb_err_addr);
@@ -7222,90 +7448,88 @@ static void gaudi_handle_ecc_event(struct hl_device *hdev, u16 event_type,
static void gaudi_handle_qman_err(struct hl_device *hdev, u16 event_type)
{
- u64 glbl_sts_addr, arb_err_addr;
- u8 index;
+ u64 qman_base;
char desc[32];
+ u32 qid_base;
+ u8 index;
switch (event_type) {
case GAUDI_EVENT_TPC0_QM ... GAUDI_EVENT_TPC7_QM:
index = event_type - GAUDI_EVENT_TPC0_QM;
- glbl_sts_addr =
- mmTPC0_QM_GLBL_STS1_0 + index * TPC_QMAN_OFFSET;
- arb_err_addr =
- mmTPC0_QM_ARB_ERR_CAUSE + index * TPC_QMAN_OFFSET;
+ qid_base = GAUDI_QUEUE_ID_TPC_0_0 + index * QMAN_STREAMS;
+ qman_base = mmTPC0_QM_BASE + index * TPC_QMAN_OFFSET;
snprintf(desc, ARRAY_SIZE(desc), "%s%d", "TPC_QM", index);
break;
case GAUDI_EVENT_MME0_QM ... GAUDI_EVENT_MME2_QM:
index = event_type - GAUDI_EVENT_MME0_QM;
- glbl_sts_addr =
- mmMME0_QM_GLBL_STS1_0 + index * MME_QMAN_OFFSET;
- arb_err_addr =
- mmMME0_QM_ARB_ERR_CAUSE + index * MME_QMAN_OFFSET;
+ qid_base = GAUDI_QUEUE_ID_MME_0_0 + index * QMAN_STREAMS;
+ qman_base = mmMME0_QM_BASE + index * MME_QMAN_OFFSET;
snprintf(desc, ARRAY_SIZE(desc), "%s%d", "MME_QM", index);
break;
case GAUDI_EVENT_DMA0_QM ... GAUDI_EVENT_DMA7_QM:
index = event_type - GAUDI_EVENT_DMA0_QM;
- glbl_sts_addr =
- mmDMA0_QM_GLBL_STS1_0 + index * DMA_QMAN_OFFSET;
- arb_err_addr =
- mmDMA0_QM_ARB_ERR_CAUSE + index * DMA_QMAN_OFFSET;
+ qid_base = GAUDI_QUEUE_ID_DMA_0_0 + index * QMAN_STREAMS;
+ /* skip GAUDI_QUEUE_ID_CPU_PQ if necessary */
+ if (index > 1)
+ qid_base++;
+ qman_base = mmDMA0_QM_BASE + index * DMA_QMAN_OFFSET;
snprintf(desc, ARRAY_SIZE(desc), "%s%d", "DMA_QM", index);
break;
case GAUDI_EVENT_NIC0_QM0:
- glbl_sts_addr = mmNIC0_QM0_GLBL_STS1_0;
- arb_err_addr = mmNIC0_QM0_ARB_ERR_CAUSE;
+ qid_base = GAUDI_QUEUE_ID_NIC_0_0;
+ qman_base = mmNIC0_QM0_BASE;
snprintf(desc, ARRAY_SIZE(desc), "NIC0_QM0");
break;
case GAUDI_EVENT_NIC0_QM1:
- glbl_sts_addr = mmNIC0_QM1_GLBL_STS1_0;
- arb_err_addr = mmNIC0_QM1_ARB_ERR_CAUSE;
+ qid_base = GAUDI_QUEUE_ID_NIC_1_0;
+ qman_base = mmNIC0_QM1_BASE;
snprintf(desc, ARRAY_SIZE(desc), "NIC0_QM1");
break;
case GAUDI_EVENT_NIC1_QM0:
- glbl_sts_addr = mmNIC1_QM0_GLBL_STS1_0;
- arb_err_addr = mmNIC1_QM0_ARB_ERR_CAUSE;
+ qid_base = GAUDI_QUEUE_ID_NIC_2_0;
+ qman_base = mmNIC1_QM0_BASE;
snprintf(desc, ARRAY_SIZE(desc), "NIC1_QM0");
break;
case GAUDI_EVENT_NIC1_QM1:
- glbl_sts_addr = mmNIC1_QM1_GLBL_STS1_0;
- arb_err_addr = mmNIC1_QM1_ARB_ERR_CAUSE;
+ qid_base = GAUDI_QUEUE_ID_NIC_3_0;
+ qman_base = mmNIC1_QM1_BASE;
snprintf(desc, ARRAY_SIZE(desc), "NIC1_QM1");
break;
case GAUDI_EVENT_NIC2_QM0:
- glbl_sts_addr = mmNIC2_QM0_GLBL_STS1_0;
- arb_err_addr = mmNIC2_QM0_ARB_ERR_CAUSE;
+ qid_base = GAUDI_QUEUE_ID_NIC_4_0;
+ qman_base = mmNIC2_QM0_BASE;
snprintf(desc, ARRAY_SIZE(desc), "NIC2_QM0");
break;
case GAUDI_EVENT_NIC2_QM1:
- glbl_sts_addr = mmNIC2_QM1_GLBL_STS1_0;
- arb_err_addr = mmNIC2_QM1_ARB_ERR_CAUSE;
+ qid_base = GAUDI_QUEUE_ID_NIC_5_0;
+ qman_base = mmNIC2_QM1_BASE;
snprintf(desc, ARRAY_SIZE(desc), "NIC2_QM1");
break;
case GAUDI_EVENT_NIC3_QM0:
- glbl_sts_addr = mmNIC3_QM0_GLBL_STS1_0;
- arb_err_addr = mmNIC3_QM0_ARB_ERR_CAUSE;
+ qid_base = GAUDI_QUEUE_ID_NIC_6_0;
+ qman_base = mmNIC3_QM0_BASE;
snprintf(desc, ARRAY_SIZE(desc), "NIC3_QM0");
break;
case GAUDI_EVENT_NIC3_QM1:
- glbl_sts_addr = mmNIC3_QM1_GLBL_STS1_0;
- arb_err_addr = mmNIC3_QM1_ARB_ERR_CAUSE;
+ qid_base = GAUDI_QUEUE_ID_NIC_7_0;
+ qman_base = mmNIC3_QM1_BASE;
snprintf(desc, ARRAY_SIZE(desc), "NIC3_QM1");
break;
case GAUDI_EVENT_NIC4_QM0:
- glbl_sts_addr = mmNIC4_QM0_GLBL_STS1_0;
- arb_err_addr = mmNIC4_QM0_ARB_ERR_CAUSE;
+ qid_base = GAUDI_QUEUE_ID_NIC_8_0;
+ qman_base = mmNIC4_QM0_BASE;
snprintf(desc, ARRAY_SIZE(desc), "NIC4_QM0");
break;
case GAUDI_EVENT_NIC4_QM1:
- glbl_sts_addr = mmNIC4_QM1_GLBL_STS1_0;
- arb_err_addr = mmNIC4_QM1_ARB_ERR_CAUSE;
+ qid_base = GAUDI_QUEUE_ID_NIC_9_0;
+ qman_base = mmNIC4_QM1_BASE;
snprintf(desc, ARRAY_SIZE(desc), "NIC4_QM1");
break;
default:
return;
}
- gaudi_handle_qman_err_generic(hdev, desc, glbl_sts_addr, arb_err_addr);
+ gaudi_handle_qman_err_generic(hdev, desc, qman_base, qid_base);
}
static void gaudi_print_irq_info(struct hl_device *hdev, u16 event_type,
@@ -7332,6 +7556,16 @@ static void gaudi_print_out_of_sync_info(struct hl_device *hdev,
sync_err->pi, sync_err->ci, q->pi, atomic_read(&q->ci));
}
+static void gaudi_print_fw_alive_info(struct hl_device *hdev,
+ struct hl_eq_fw_alive *fw_alive)
+{
+ dev_err(hdev->dev,
+ "FW alive report: severity=%s, process_id=%u, thread_id=%u, uptime=%llu seconds\n",
+ (fw_alive->severity == FW_ALIVE_SEVERITY_MINOR) ?
+ "Minor" : "Critical", fw_alive->process_id,
+ fw_alive->thread_id, fw_alive->uptime_seconds);
+}
+
static int gaudi_soft_reset_late_init(struct hl_device *hdev)
{
struct gaudi_device *gaudi = hdev->asic_specific;
@@ -7346,11 +7580,10 @@ static int gaudi_hbm_read_interrupts(struct hl_device *hdev, int device,
struct hl_eq_hbm_ecc_data *hbm_ecc_data)
{
u32 base, val, val2, wr_par, rd_par, ca_par, derr, serr, type, ch;
- int err = 0;
+ int rc = 0;
- if (hdev->asic_prop.fw_security_status_valid &&
- (hdev->asic_prop.fw_app_security_map &
- CPU_BOOT_DEV_STS0_HBM_ECC_EN)) {
+ if (hdev->asic_prop.fw_app_cpu_boot_dev_sts0 &
+ CPU_BOOT_DEV_STS0_HBM_ECC_EN) {
if (!hbm_ecc_data) {
dev_err(hdev->dev, "No FW ECC data");
return 0;
@@ -7379,13 +7612,10 @@ static int gaudi_hbm_read_interrupts(struct hl_device *hdev, int device,
device, ch, hbm_ecc_data->first_addr, type,
hbm_ecc_data->sec_cont_cnt, hbm_ecc_data->sec_cnt,
hbm_ecc_data->dec_cnt);
-
- err = 1;
-
return 0;
}
- if (!hdev->asic_prop.fw_security_disabled) {
+ if (hdev->asic_prop.fw_security_enabled) {
dev_info(hdev->dev, "Cannot access MC regs for ECC data while security is enabled\n");
return 0;
}
@@ -7395,7 +7625,7 @@ static int gaudi_hbm_read_interrupts(struct hl_device *hdev, int device,
val = RREG32_MASK(base + ch * 0x1000 + 0x06C, 0x0000FFFF);
val = (val & 0xFF) | ((val >> 8) & 0xFF);
if (val) {
- err = 1;
+ rc = -EIO;
dev_err(hdev->dev,
"HBM%d pc%d interrupts info: WR_PAR=%d, RD_PAR=%d, CA_PAR=%d, SERR=%d, DERR=%d\n",
device, ch * 2, val & 0x1, (val >> 1) & 0x1,
@@ -7415,7 +7645,7 @@ static int gaudi_hbm_read_interrupts(struct hl_device *hdev, int device,
val = RREG32_MASK(base + ch * 0x1000 + 0x07C, 0x0000FFFF);
val = (val & 0xFF) | ((val >> 8) & 0xFF);
if (val) {
- err = 1;
+ rc = -EIO;
dev_err(hdev->dev,
"HBM%d pc%d interrupts info: WR_PAR=%d, RD_PAR=%d, CA_PAR=%d, SERR=%d, DERR=%d\n",
device, ch * 2 + 1, val & 0x1, (val >> 1) & 0x1,
@@ -7444,7 +7674,7 @@ static int gaudi_hbm_read_interrupts(struct hl_device *hdev, int device,
val = RREG32(base + 0x8F30);
val2 = RREG32(base + 0x8F34);
if (val | val2) {
- err = 1;
+ rc = -EIO;
dev_err(hdev->dev,
"HBM %d MC SRAM SERR info: Reg 0x8F30=0x%x, Reg 0x8F34=0x%x\n",
device, val, val2);
@@ -7452,13 +7682,13 @@ static int gaudi_hbm_read_interrupts(struct hl_device *hdev, int device,
val = RREG32(base + 0x8F40);
val2 = RREG32(base + 0x8F44);
if (val | val2) {
- err = 1;
+ rc = -EIO;
dev_err(hdev->dev,
"HBM %d MC SRAM DERR info: Reg 0x8F40=0x%x, Reg 0x8F44=0x%x\n",
device, val, val2);
}
- return err;
+ return rc;
}
static int gaudi_hbm_event_to_dev(u16 hbm_event_type)
@@ -7604,6 +7834,7 @@ static void gaudi_handle_eqe(struct hl_device *hdev,
case GAUDI_EVENT_DMA_IF0_DERR ... GAUDI_EVENT_DMA_IF3_DERR:
case GAUDI_EVENT_HBM_0_DERR ... GAUDI_EVENT_HBM_3_DERR:
case GAUDI_EVENT_MMU_DERR:
+ case GAUDI_EVENT_NIC0_CS_DBG_DERR ... GAUDI_EVENT_NIC4_CS_DBG_DERR:
gaudi_print_irq_info(hdev, event_type, true);
gaudi_handle_ecc_event(hdev, event_type, &eq_entry->ecc_data);
goto reset_device;
@@ -7786,6 +8017,11 @@ static void gaudi_handle_eqe(struct hl_device *hdev,
gaudi_print_out_of_sync_info(hdev, &eq_entry->pkt_sync_err);
goto reset_device;
+ case GAUDI_EVENT_FW_ALIVE_S:
+ gaudi_print_irq_info(hdev, event_type, false);
+ gaudi_print_fw_alive_info(hdev, &eq_entry->fw_alive);
+ goto reset_device;
+
default:
dev_err(hdev->dev, "Received invalid H/W interrupt %d\n",
event_type);
@@ -7856,52 +8092,13 @@ static int gaudi_mmu_invalidate_cache(struct hl_device *hdev, bool is_hard,
}
static int gaudi_mmu_invalidate_cache_range(struct hl_device *hdev,
- bool is_hard, u32 asid, u64 va, u64 size)
+ bool is_hard, u32 flags,
+ u32 asid, u64 va, u64 size)
{
- struct gaudi_device *gaudi = hdev->asic_specific;
- u32 status, timeout_usec;
- u32 inv_data;
- u32 pi;
- int rc;
-
- if (!(gaudi->hw_cap_initialized & HW_CAP_MMU) ||
- hdev->hard_reset_pending)
- return 0;
-
- if (hdev->pldm)
- timeout_usec = GAUDI_PLDM_MMU_TIMEOUT_USEC;
- else
- timeout_usec = MMU_CONFIG_TIMEOUT_USEC;
-
- /*
- * TODO: currently invalidate entire L0 & L1 as in regular hard
- * invalidation. Need to apply invalidation of specific cache
- * lines with mask of ASID & VA & size.
- * Note that L1 with be flushed entirely in any case.
+ /* Treat as invalidate all because there is no range invalidation
+ * in Gaudi
*/
-
- /* L0 & L1 invalidation */
- inv_data = RREG32(mmSTLB_CACHE_INV);
- /* PI is 8 bit */
- pi = ((inv_data & STLB_CACHE_INV_PRODUCER_INDEX_MASK) + 1) & 0xFF;
- WREG32(mmSTLB_CACHE_INV,
- (inv_data & STLB_CACHE_INV_INDEX_MASK_MASK) | pi);
-
- rc = hl_poll_timeout(
- hdev,
- mmSTLB_INV_CONSUMER_INDEX,
- status,
- status == pi,
- 1000,
- timeout_usec);
-
- if (rc) {
- dev_err_ratelimited(hdev->dev,
- "MMU cache invalidation timeout\n");
- hl_device_reset(hdev, HL_RESET_HARD);
- }
-
- return rc;
+ return hdev->asic_funcs->mmu_invalidate_cache(hdev, is_hard, flags);
}
static int gaudi_mmu_update_asid_hop0_addr(struct hl_device *hdev,
@@ -7956,7 +8153,9 @@ static int gaudi_cpucp_info_get(struct hl_device *hdev)
if (!(gaudi->hw_cap_initialized & HW_CAP_CPU_Q))
return 0;
- rc = hl_fw_cpucp_handshake(hdev, mmCPU_BOOT_DEV_STS0, mmCPU_BOOT_ERR0);
+ rc = hl_fw_cpucp_handshake(hdev, mmCPU_BOOT_DEV_STS0,
+ mmCPU_BOOT_DEV_STS1, mmCPU_BOOT_ERR0,
+ mmCPU_BOOT_ERR1);
if (rc)
return rc;
@@ -8077,7 +8276,7 @@ static bool gaudi_is_device_idle(struct hl_device *hdev, u64 *mask_arr,
for (i = 0 ; i < (NIC_NUMBER_OF_ENGINES / 2) ; i++) {
offset = i * NIC_MACRO_QMAN_OFFSET;
port = 2 * i;
- if (hdev->nic_ports_mask & BIT(port)) {
+ if (gaudi->hw_cap_initialized & BIT(HW_CAP_NIC_SHIFT + port)) {
qm_glbl_sts0 = RREG32(mmNIC0_QM0_GLBL_STS0 + offset);
qm_cgm_sts = RREG32(mmNIC0_QM0_CGM_STS + offset);
is_eng_idle = IS_QM_IDLE(qm_glbl_sts0, qm_cgm_sts);
@@ -8092,7 +8291,7 @@ static bool gaudi_is_device_idle(struct hl_device *hdev, u64 *mask_arr,
}
port = 2 * i + 1;
- if (hdev->nic_ports_mask & BIT(port)) {
+ if (gaudi->hw_cap_initialized & BIT(HW_CAP_NIC_SHIFT + port)) {
qm_glbl_sts0 = RREG32(mmNIC0_QM1_GLBL_STS0 + offset);
qm_cgm_sts = RREG32(mmNIC0_QM1_CGM_STS + offset);
is_eng_idle = IS_QM_IDLE(qm_glbl_sts0, qm_cgm_sts);
@@ -8306,8 +8505,10 @@ static int gaudi_internal_cb_pool_init(struct hl_device *hdev,
HL_VA_RANGE_TYPE_HOST, HOST_SPACE_INTERNAL_CB_SZ,
HL_MMU_VA_ALIGNMENT_NOT_NEEDED);
- if (!hdev->internal_cb_va_base)
+ if (!hdev->internal_cb_va_base) {
+ rc = -ENOMEM;
goto destroy_internal_cb_pool;
+ }
mutex_lock(&ctx->mmu_lock);
rc = hl_mmu_map_contiguous(ctx, hdev->internal_cb_va_base,
@@ -8749,7 +8950,14 @@ static int gaudi_block_mmap(struct hl_device *hdev,
static void gaudi_enable_events_from_fw(struct hl_device *hdev)
{
- WREG32(mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR, GAUDI_EVENT_INTS_REGISTER);
+ struct cpu_dyn_regs *dyn_regs =
+ &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs;
+ u32 irq_handler_offset = hdev->asic_prop.gic_interrupts_enable ?
+ mmGIC_DISTRIBUTOR__5_GICD_SETSPI_NSR :
+ le32_to_cpu(dyn_regs->gic_host_ints_irq);
+
+ WREG32(irq_handler_offset,
+ gaudi_irq_map_table[GAUDI_EVENT_INTS_REGISTER].cpu_id);
}
static int gaudi_map_pll_idx_to_fw_idx(u32 pll_idx)
@@ -8834,7 +9042,6 @@ static const struct hl_asic_funcs gaudi_funcs = {
.ctx_fini = gaudi_ctx_fini,
.get_clk_rate = gaudi_get_clk_rate,
.get_queue_id_for_cq = gaudi_get_queue_id_for_cq,
- .read_device_fw_version = gaudi_read_device_fw_version,
.load_firmware_to_device = gaudi_load_firmware_to_device,
.load_boot_fit_to_device = gaudi_load_boot_fit_to_device,
.get_signal_cb_size = gaudi_get_signal_cb_size,
@@ -8853,7 +9060,9 @@ static const struct hl_asic_funcs gaudi_funcs = {
.get_hw_block_id = gaudi_get_hw_block_id,
.hw_block_mmap = gaudi_block_mmap,
.enable_events_from_fw = gaudi_enable_events_from_fw,
- .map_pll_idx_to_fw_idx = gaudi_map_pll_idx_to_fw_idx
+ .map_pll_idx_to_fw_idx = gaudi_map_pll_idx_to_fw_idx,
+ .init_firmware_loader = gaudi_init_firmware_loader,
+ .init_cpu_scrambler_dram = gaudi_init_scrambler_hbm
};
/**
diff --git a/drivers/misc/habanalabs/gaudi/gaudiP.h b/drivers/misc/habanalabs/gaudi/gaudiP.h
index 5929be81ec23..957bf3720f70 100644
--- a/drivers/misc/habanalabs/gaudi/gaudiP.h
+++ b/drivers/misc/habanalabs/gaudi/gaudiP.h
@@ -82,6 +82,7 @@
QMAN_STREAMS)
#define QMAN_STREAMS 4
+#define PQ_FETCHER_CACHE_SIZE 8
#define DMA_QMAN_OFFSET (mmDMA1_QM_BASE - mmDMA0_QM_BASE)
#define TPC_QMAN_OFFSET (mmTPC1_QM_BASE - mmTPC0_QM_BASE)
diff --git a/drivers/misc/habanalabs/gaudi/gaudi_coresight.c b/drivers/misc/habanalabs/gaudi/gaudi_coresight.c
index 6e56fa1c6c69..c2a27ed1c4d1 100644
--- a/drivers/misc/habanalabs/gaudi/gaudi_coresight.c
+++ b/drivers/misc/habanalabs/gaudi/gaudi_coresight.c
@@ -424,7 +424,7 @@ static int gaudi_config_stm(struct hl_device *hdev,
if (frequency == 0)
frequency = input->frequency;
WREG32(base_reg + 0xE8C, frequency);
- WREG32(base_reg + 0xE90, 0x7FF);
+ WREG32(base_reg + 0xE90, 0x1F00);
/* SW-2176 - SW WA for HW bug */
if ((CFG_BASE + base_reg) >= mmDMA_CH_0_CS_STM_BASE &&
@@ -434,7 +434,7 @@ static int gaudi_config_stm(struct hl_device *hdev,
WREG32(base_reg + 0xE6C, 0x0);
}
- WREG32(base_reg + 0xE80, 0x27 | (input->id << 16));
+ WREG32(base_reg + 0xE80, 0x23 | (input->id << 16));
} else {
WREG32(base_reg + 0xE80, 4);
WREG32(base_reg + 0xD64, 0);
@@ -634,7 +634,7 @@ static int gaudi_config_etr(struct hl_device *hdev,
WREG32(mmPSOC_ETR_BUFWM, 0x3FFC);
WREG32(mmPSOC_ETR_RSZ, input->buffer_size);
WREG32(mmPSOC_ETR_MODE, input->sink_mode);
- if (hdev->asic_prop.fw_security_disabled) {
+ if (!hdev->asic_prop.fw_security_enabled) {
/* make ETR not privileged */
val = FIELD_PREP(
PSOC_ETR_AXICTL_PROTCTRLBIT0_MASK, 0);
diff --git a/drivers/misc/habanalabs/gaudi/gaudi_security.c b/drivers/misc/habanalabs/gaudi/gaudi_security.c
index 9a706c5980ef..0d3240f1f7d7 100644
--- a/drivers/misc/habanalabs/gaudi/gaudi_security.c
+++ b/drivers/misc/habanalabs/gaudi/gaudi_security.c
@@ -1448,7 +1448,7 @@ static void gaudi_init_dma_protection_bits(struct hl_device *hdev)
u32 pb_addr, mask;
u8 word_offset;
- if (hdev->asic_prop.fw_security_disabled) {
+ if (!hdev->asic_prop.fw_security_enabled) {
gaudi_pb_set_block(hdev, mmDMA_IF_E_S_BASE);
gaudi_pb_set_block(hdev, mmDMA_IF_E_S_DOWN_CH0_BASE);
gaudi_pb_set_block(hdev, mmDMA_IF_E_S_DOWN_CH1_BASE);
@@ -9135,7 +9135,7 @@ static void gaudi_init_tpc_protection_bits(struct hl_device *hdev)
u32 pb_addr, mask;
u8 word_offset;
- if (hdev->asic_prop.fw_security_disabled) {
+ if (!hdev->asic_prop.fw_security_enabled) {
gaudi_pb_set_block(hdev, mmTPC0_E2E_CRED_BASE);
gaudi_pb_set_block(hdev, mmTPC1_E2E_CRED_BASE);
gaudi_pb_set_block(hdev, mmTPC2_E2E_CRED_BASE);
@@ -12818,7 +12818,7 @@ static void gaudi_init_protection_bits(struct hl_device *hdev)
* secured
*/
- if (hdev->asic_prop.fw_security_disabled) {
+ if (!hdev->asic_prop.fw_security_enabled) {
gaudi_pb_set_block(hdev, mmIF_E_PLL_BASE);
gaudi_pb_set_block(hdev, mmMESH_W_PLL_BASE);
gaudi_pb_set_block(hdev, mmSRAM_W_PLL_BASE);
@@ -13023,7 +13023,7 @@ void gaudi_init_security(struct hl_device *hdev)
* property configuration of MME SBAB and ACC to be non-privileged and
* non-secured
*/
- if (hdev->asic_prop.fw_security_disabled) {
+ if (!hdev->asic_prop.fw_security_enabled) {
WREG32(mmMME0_SBAB_PROT, 0x2);
WREG32(mmMME0_ACC_PROT, 0x2);
WREG32(mmMME1_SBAB_PROT, 0x2);
@@ -13032,11 +13032,12 @@ void gaudi_init_security(struct hl_device *hdev)
WREG32(mmMME2_ACC_PROT, 0x2);
WREG32(mmMME3_SBAB_PROT, 0x2);
WREG32(mmMME3_ACC_PROT, 0x2);
- }
- /* On RAZWI, 0 will be returned from RR and 0xBABA0BAD from PB */
- if (hdev->asic_prop.fw_security_disabled)
+ /*
+ * On RAZWI, 0 will be returned from RR and 0xBABA0BAD from PB
+ */
WREG32(0xC01B28, 0x1);
+ }
gaudi_init_range_registers_lbw(hdev);
diff --git a/drivers/misc/habanalabs/goya/goya.c b/drivers/misc/habanalabs/goya/goya.c
index e0ad2a269779..755e08cf2ecc 100644
--- a/drivers/misc/habanalabs/goya/goya.c
+++ b/drivers/misc/habanalabs/goya/goya.c
@@ -87,6 +87,7 @@
#define GOYA_PLDM_QMAN0_TIMEOUT_USEC (HL_DEVICE_TIMEOUT_USEC * 30)
#define GOYA_BOOT_FIT_REQ_TIMEOUT_USEC 1000000 /* 1s */
#define GOYA_MSG_TO_CPU_TIMEOUT_USEC 4000000 /* 4s */
+#define GOYA_WAIT_FOR_BL_TIMEOUT_USEC 15000000 /* 15s */
#define GOYA_QMAN0_FENCE_VAL 0xD169B243
@@ -354,7 +355,7 @@ static int goya_mmu_set_dram_default_page(struct hl_device *hdev);
static int goya_mmu_add_mappings_for_device_cpu(struct hl_device *hdev);
static void goya_mmu_prepare(struct hl_device *hdev, u32 asid);
-int goya_get_fixed_properties(struct hl_device *hdev)
+int goya_set_fixed_properties(struct hl_device *hdev)
{
struct asic_fixed_properties *prop = &hdev->asic_prop;
int i;
@@ -460,8 +461,10 @@ int goya_get_fixed_properties(struct hl_device *hdev)
for (i = 0 ; i < HL_MAX_DCORES ; i++)
prop->first_available_cq[i] = USHRT_MAX;
- prop->fw_security_status_valid = false;
+ prop->fw_cpu_boot_dev_sts0_valid = false;
+ prop->fw_cpu_boot_dev_sts1_valid = false;
prop->hard_reset_done_by_fw = false;
+ prop->gic_interrupts_enable = true;
return 0;
}
@@ -531,10 +534,8 @@ static int goya_init_iatu(struct hl_device *hdev)
struct hl_outbound_pci_region outbound_region;
int rc;
- if (hdev->asic_prop.iatu_done_by_fw) {
- hdev->asic_funcs->set_dma_mask_from_fw(hdev);
+ if (hdev->asic_prop.iatu_done_by_fw)
return 0;
- }
/* Inbound Region 0 - Bar 0 - Point to SRAM and CFG */
inbound_region.mode = PCI_BAR_MATCH_MODE;
@@ -586,7 +587,7 @@ static int goya_early_init(struct hl_device *hdev)
u32 fw_boot_status, val;
int rc;
- rc = goya_get_fixed_properties(hdev);
+ rc = goya_set_fixed_properties(hdev);
if (rc) {
dev_err(hdev->dev, "Failed to get fixed properties\n");
return rc;
@@ -618,7 +619,7 @@ static int goya_early_init(struct hl_device *hdev)
prop->dram_pci_bar_size = pci_resource_len(pdev, DDR_BAR_ID);
/* If FW security is enabled at this point it means no access to ELBI */
- if (!hdev->asic_prop.fw_security_disabled) {
+ if (hdev->asic_prop.fw_security_enabled) {
hdev->asic_prop.iatu_done_by_fw = true;
goto pci_init;
}
@@ -642,8 +643,10 @@ pci_init:
* version to determine whether we run with a security-enabled firmware
*/
rc = hl_fw_read_preboot_status(hdev, mmPSOC_GLOBAL_CONF_CPU_BOOT_STATUS,
- mmCPU_BOOT_DEV_STS0, mmCPU_BOOT_ERR0,
- GOYA_BOOT_FIT_REQ_TIMEOUT_USEC);
+ mmCPU_BOOT_DEV_STS0,
+ mmCPU_BOOT_DEV_STS1, mmCPU_BOOT_ERR0,
+ mmCPU_BOOT_ERR1,
+ GOYA_BOOT_FIT_REQ_TIMEOUT_USEC);
if (rc) {
if (hdev->reset_on_preboot_fail)
hdev->asic_funcs->hw_fini(hdev, true);
@@ -723,7 +726,15 @@ static void goya_fetch_psoc_frequency(struct hl_device *hdev)
u16 pll_freq_arr[HL_PLL_NUM_OUTPUTS], freq;
int rc;
- if (hdev->asic_prop.fw_security_disabled) {
+ if (hdev->asic_prop.fw_security_enabled) {
+ rc = hl_fw_cpucp_pll_info_get(hdev, HL_GOYA_PCI_PLL,
+ pll_freq_arr);
+
+ if (rc)
+ return;
+
+ freq = pll_freq_arr[1];
+ } else {
div_fctr = RREG32(mmPSOC_PCI_PLL_DIV_FACTOR_1);
div_sel = RREG32(mmPSOC_PCI_PLL_DIV_SEL_1);
nr = RREG32(mmPSOC_PCI_PLL_NR);
@@ -750,14 +761,6 @@ static void goya_fetch_psoc_frequency(struct hl_device *hdev)
div_sel);
freq = 0;
}
- } else {
- rc = hl_fw_cpucp_pll_info_get(hdev, HL_GOYA_PCI_PLL,
- pll_freq_arr);
-
- if (rc)
- return;
-
- freq = pll_freq_arr[1];
}
prop->psoc_timestamp_frequency = freq;
@@ -849,6 +852,39 @@ void goya_late_fini(struct hl_device *hdev)
hdev->hl_chip_info->info = NULL;
}
+static void goya_set_pci_memory_regions(struct hl_device *hdev)
+{
+ struct asic_fixed_properties *prop = &hdev->asic_prop;
+ struct pci_mem_region *region;
+
+ /* CFG */
+ region = &hdev->pci_mem_region[PCI_REGION_CFG];
+ region->region_base = CFG_BASE;
+ region->region_size = CFG_SIZE;
+ region->offset_in_bar = CFG_BASE - SRAM_BASE_ADDR;
+ region->bar_size = CFG_BAR_SIZE;
+ region->bar_id = SRAM_CFG_BAR_ID;
+ region->used = 1;
+
+ /* SRAM */
+ region = &hdev->pci_mem_region[PCI_REGION_SRAM];
+ region->region_base = SRAM_BASE_ADDR;
+ region->region_size = SRAM_SIZE;
+ region->offset_in_bar = 0;
+ region->bar_size = CFG_BAR_SIZE;
+ region->bar_id = SRAM_CFG_BAR_ID;
+ region->used = 1;
+
+ /* DRAM */
+ region = &hdev->pci_mem_region[PCI_REGION_DRAM];
+ region->region_base = DRAM_PHYS_BASE;
+ region->region_size = hdev->asic_prop.dram_size;
+ region->offset_in_bar = 0;
+ region->bar_size = prop->dram_pci_bar_size;
+ region->bar_id = DDR_BAR_ID;
+ region->used = 1;
+}
+
/*
* goya_sw_init - Goya software initialization code
*
@@ -918,6 +954,9 @@ static int goya_sw_init(struct hl_device *hdev)
spin_lock_init(&goya->hw_queues_lock);
hdev->supports_coresight = true;
hdev->supports_soft_reset = true;
+ hdev->allow_external_soft_reset = true;
+
+ goya_set_pci_memory_regions(hdev);
return 0;
@@ -1263,8 +1302,11 @@ int goya_init_cpu_queues(struct hl_device *hdev)
}
/* update FW application security bits */
- if (prop->fw_security_status_valid)
- prop->fw_app_security_map = RREG32(mmCPU_BOOT_DEV_STS0);
+ if (prop->fw_cpu_boot_dev_sts0_valid)
+ prop->fw_app_cpu_boot_dev_sts0 = RREG32(mmCPU_BOOT_DEV_STS0);
+
+ if (prop->fw_cpu_boot_dev_sts1_valid)
+ prop->fw_app_cpu_boot_dev_sts1 = RREG32(mmCPU_BOOT_DEV_STS1);
goya->hw_cap_initialized |= HW_CAP_CPU_Q;
return 0;
@@ -2402,47 +2444,67 @@ static int goya_load_boot_fit_to_device(struct hl_device *hdev)
return hl_fw_load_fw_to_device(hdev, GOYA_BOOT_FIT_FILE, dst, 0, 0);
}
-/*
- * FW component passes an offset from SRAM_BASE_ADDR in SCRATCHPAD_xx.
- * The version string should be located by that offset.
- */
-static int goya_read_device_fw_version(struct hl_device *hdev,
- enum hl_fw_component fwc)
-{
- const char *name;
- u32 ver_off;
- char *dest;
-
- switch (fwc) {
- case FW_COMP_UBOOT:
- ver_off = RREG32(mmUBOOT_VER_OFFSET);
- dest = hdev->asic_prop.uboot_ver;
- name = "U-Boot";
- break;
- case FW_COMP_PREBOOT:
- ver_off = RREG32(mmPREBOOT_VER_OFFSET);
- dest = hdev->asic_prop.preboot_ver;
- name = "Preboot";
- break;
- default:
- dev_warn(hdev->dev, "Undefined FW component: %d\n", fwc);
- return -EIO;
- }
+static void goya_init_dynamic_firmware_loader(struct hl_device *hdev)
+{
+ struct dynamic_fw_load_mgr *dynamic_loader;
+ struct cpu_dyn_regs *dyn_regs;
- ver_off &= ~((u32)SRAM_BASE_ADDR);
+ dynamic_loader = &hdev->fw_loader.dynamic_loader;
- if (ver_off < SRAM_SIZE - VERSION_MAX_LEN) {
- memcpy_fromio(dest, hdev->pcie_bar[SRAM_CFG_BAR_ID] + ver_off,
- VERSION_MAX_LEN);
- } else {
- dev_err(hdev->dev, "%s version offset (0x%x) is above SRAM\n",
- name, ver_off);
- strcpy(dest, "unavailable");
+ /*
+ * here we update initial values for few specific dynamic regs (as
+ * before reading the first descriptor from FW those value has to be
+ * hard-coded) in later stages of the protocol those values will be
+ * updated automatically by reading the FW descriptor so data there
+ * will always be up-to-date
+ */
+ dyn_regs = &dynamic_loader->comm_desc.cpu_dyn_regs;
+ dyn_regs->kmd_msg_to_cpu =
+ cpu_to_le32(mmPSOC_GLOBAL_CONF_KMD_MSG_TO_CPU);
+ dyn_regs->cpu_cmd_status_to_host =
+ cpu_to_le32(mmCPU_CMD_STATUS_TO_HOST);
- return -EIO;
- }
+ dynamic_loader->wait_for_bl_timeout = GOYA_WAIT_FOR_BL_TIMEOUT_USEC;
+}
- return 0;
+static void goya_init_static_firmware_loader(struct hl_device *hdev)
+{
+ struct static_fw_load_mgr *static_loader;
+
+ static_loader = &hdev->fw_loader.static_loader;
+
+ static_loader->preboot_version_max_off = SRAM_SIZE - VERSION_MAX_LEN;
+ static_loader->boot_fit_version_max_off = SRAM_SIZE - VERSION_MAX_LEN;
+ static_loader->kmd_msg_to_cpu_reg = mmPSOC_GLOBAL_CONF_KMD_MSG_TO_CPU;
+ static_loader->cpu_cmd_status_to_host_reg = mmCPU_CMD_STATUS_TO_HOST;
+ static_loader->cpu_boot_status_reg = mmPSOC_GLOBAL_CONF_CPU_BOOT_STATUS;
+ static_loader->cpu_boot_dev_status0_reg = mmCPU_BOOT_DEV_STS0;
+ static_loader->cpu_boot_dev_status1_reg = mmCPU_BOOT_DEV_STS1;
+ static_loader->boot_err0_reg = mmCPU_BOOT_ERR0;
+ static_loader->boot_err1_reg = mmCPU_BOOT_ERR1;
+ static_loader->preboot_version_offset_reg = mmPREBOOT_VER_OFFSET;
+ static_loader->boot_fit_version_offset_reg = mmUBOOT_VER_OFFSET;
+ static_loader->sram_offset_mask = ~(lower_32_bits(SRAM_BASE_ADDR));
+}
+
+static void goya_init_firmware_loader(struct hl_device *hdev)
+{
+ struct asic_fixed_properties *prop = &hdev->asic_prop;
+ struct fw_load_mgr *fw_loader = &hdev->fw_loader;
+
+ /* fill common fields */
+ fw_loader->boot_fit_img.image_name = GOYA_BOOT_FIT_FILE;
+ fw_loader->linux_img.image_name = GOYA_LINUX_FW_FILE;
+ fw_loader->cpu_timeout = GOYA_CPU_TIMEOUT_USEC;
+ fw_loader->boot_fit_timeout = GOYA_BOOT_FIT_REQ_TIMEOUT_USEC;
+ fw_loader->skip_bmc = false;
+ fw_loader->sram_bar_id = SRAM_CFG_BAR_ID;
+ fw_loader->dram_bar_id = DDR_BAR_ID;
+
+ if (prop->dynamic_fw_load)
+ goya_init_dynamic_firmware_loader(hdev);
+ else
+ goya_init_static_firmware_loader(hdev);
}
static int goya_init_cpu(struct hl_device *hdev)
@@ -2466,12 +2528,7 @@ static int goya_init_cpu(struct hl_device *hdev)
return -EIO;
}
- rc = hl_fw_init_cpu(hdev, mmPSOC_GLOBAL_CONF_CPU_BOOT_STATUS,
- mmPSOC_GLOBAL_CONF_UBOOT_MAGIC,
- mmCPU_CMD_STATUS_TO_HOST,
- mmCPU_BOOT_DEV_STS0, mmCPU_BOOT_ERR0,
- false, GOYA_CPU_TIMEOUT_USEC,
- GOYA_BOOT_FIT_REQ_TIMEOUT_USEC);
+ rc = hl_fw_init_cpu(hdev);
if (rc)
return rc;
@@ -2881,7 +2938,7 @@ void *goya_get_int_queue_base(struct hl_device *hdev, u32 queue_id,
*dma_handle = hdev->asic_prop.sram_base_address;
- base = (void *) hdev->pcie_bar[SRAM_CFG_BAR_ID];
+ base = (__force void *) hdev->pcie_bar[SRAM_CFG_BAR_ID];
switch (queue_id) {
case GOYA_QUEUE_ID_MME:
@@ -3270,6 +3327,7 @@ already_pinned:
return 0;
unpin_memory:
+ list_del(&userptr->job_node);
hl_unpin_host_memory(hdev, userptr);
free_userptr:
kfree(userptr);
@@ -5169,54 +5227,13 @@ static int goya_mmu_invalidate_cache(struct hl_device *hdev, bool is_hard,
}
static int goya_mmu_invalidate_cache_range(struct hl_device *hdev,
- bool is_hard, u32 asid, u64 va, u64 size)
+ bool is_hard, u32 flags,
+ u32 asid, u64 va, u64 size)
{
- struct goya_device *goya = hdev->asic_specific;
- u32 status, timeout_usec, inv_data, pi;
- int rc;
-
- if (!(goya->hw_cap_initialized & HW_CAP_MMU) ||
- hdev->hard_reset_pending)
- return 0;
-
- /* no need in L1 only invalidation in Goya */
- if (!is_hard)
- return 0;
-
- if (hdev->pldm)
- timeout_usec = GOYA_PLDM_MMU_TIMEOUT_USEC;
- else
- timeout_usec = MMU_CONFIG_TIMEOUT_USEC;
-
- /*
- * TODO: currently invalidate entire L0 & L1 as in regular hard
- * invalidation. Need to apply invalidation of specific cache lines with
- * mask of ASID & VA & size.
- * Note that L1 with be flushed entirely in any case.
+ /* Treat as invalidate all because there is no range invalidation
+ * in Goya
*/
-
- /* L0 & L1 invalidation */
- inv_data = RREG32(mmSTLB_CACHE_INV);
- /* PI is 8 bit */
- pi = ((inv_data & STLB_CACHE_INV_PRODUCER_INDEX_MASK) + 1) & 0xFF;
- WREG32(mmSTLB_CACHE_INV,
- (inv_data & STLB_CACHE_INV_INDEX_MASK_MASK) | pi);
-
- rc = hl_poll_timeout(
- hdev,
- mmSTLB_INV_CONSUMER_INDEX,
- status,
- status == pi,
- 1000,
- timeout_usec);
-
- if (rc) {
- dev_err_ratelimited(hdev->dev,
- "MMU cache invalidation timeout\n");
- hl_device_reset(hdev, HL_RESET_HARD);
- }
-
- return rc;
+ return hdev->asic_funcs->mmu_invalidate_cache(hdev, is_hard, flags);
}
int goya_send_heartbeat(struct hl_device *hdev)
@@ -5239,7 +5256,9 @@ int goya_cpucp_info_get(struct hl_device *hdev)
if (!(goya->hw_cap_initialized & HW_CAP_CPU_Q))
return 0;
- rc = hl_fw_cpucp_handshake(hdev, mmCPU_BOOT_DEV_STS0, mmCPU_BOOT_ERR0);
+ rc = hl_fw_cpucp_handshake(hdev, mmCPU_BOOT_DEV_STS0,
+ mmCPU_BOOT_DEV_STS1, mmCPU_BOOT_ERR0,
+ mmCPU_BOOT_ERR1);
if (rc)
return rc;
@@ -5385,6 +5404,11 @@ static int goya_get_eeprom_data(struct hl_device *hdev, void *data,
return hl_fw_get_eeprom_data(hdev, data, max_size);
}
+static void goya_cpu_init_scrambler_dram(struct hl_device *hdev)
+{
+
+}
+
static int goya_ctx_init(struct hl_ctx *ctx)
{
if (ctx->asid != HL_KERNEL_ASID_ID)
@@ -5565,7 +5589,6 @@ static const struct hl_asic_funcs goya_funcs = {
.ctx_fini = goya_ctx_fini,
.get_clk_rate = goya_get_clk_rate,
.get_queue_id_for_cq = goya_get_queue_id_for_cq,
- .read_device_fw_version = goya_read_device_fw_version,
.load_firmware_to_device = goya_load_firmware_to_device,
.load_boot_fit_to_device = goya_load_boot_fit_to_device,
.get_signal_cb_size = goya_get_signal_cb_size,
@@ -5584,7 +5607,9 @@ static const struct hl_asic_funcs goya_funcs = {
.get_hw_block_id = goya_get_hw_block_id,
.hw_block_mmap = goya_block_mmap,
.enable_events_from_fw = goya_enable_events_from_fw,
- .map_pll_idx_to_fw_idx = goya_map_pll_idx_to_fw_idx
+ .map_pll_idx_to_fw_idx = goya_map_pll_idx_to_fw_idx,
+ .init_firmware_loader = goya_init_firmware_loader,
+ .init_cpu_scrambler_dram = goya_cpu_init_scrambler_dram
};
/*
diff --git a/drivers/misc/habanalabs/goya/goyaP.h b/drivers/misc/habanalabs/goya/goyaP.h
index ef8c6c8b5c8d..0b05da614729 100644
--- a/drivers/misc/habanalabs/goya/goyaP.h
+++ b/drivers/misc/habanalabs/goya/goyaP.h
@@ -168,7 +168,7 @@ struct goya_device {
u8 device_cpu_mmu_mappings_done;
};
-int goya_get_fixed_properties(struct hl_device *hdev);
+int goya_set_fixed_properties(struct hl_device *hdev);
int goya_mmu_init(struct hl_device *hdev);
void goya_init_dma_qmans(struct hl_device *hdev);
void goya_init_mme_qmans(struct hl_device *hdev);
diff --git a/drivers/misc/habanalabs/goya/goya_coresight.c b/drivers/misc/habanalabs/goya/goya_coresight.c
index 6b7445cca580..c55c100fdd24 100644
--- a/drivers/misc/habanalabs/goya/goya_coresight.c
+++ b/drivers/misc/habanalabs/goya/goya_coresight.c
@@ -434,7 +434,7 @@ static int goya_config_etr(struct hl_device *hdev,
WREG32(mmPSOC_ETR_BUFWM, 0x3FFC);
WREG32(mmPSOC_ETR_RSZ, input->buffer_size);
WREG32(mmPSOC_ETR_MODE, input->sink_mode);
- if (hdev->asic_prop.fw_security_disabled) {
+ if (!hdev->asic_prop.fw_security_enabled) {
/* make ETR not privileged */
val = FIELD_PREP(PSOC_ETR_AXICTL_PROTCTRLBIT0_MASK, 0);
/* make ETR non-secured (inverted logic) */
diff --git a/drivers/misc/habanalabs/include/common/cpucp_if.h b/drivers/misc/habanalabs/include/common/cpucp_if.h
index 27cd0ba99aa3..80b1d5a9d9f1 100644
--- a/drivers/misc/habanalabs/include/common/cpucp_if.h
+++ b/drivers/misc/habanalabs/include/common/cpucp_if.h
@@ -84,6 +84,20 @@ struct hl_eq_sm_sei_data {
__u8 pad[3];
};
+enum hl_fw_alive_severity {
+ FW_ALIVE_SEVERITY_MINOR,
+ FW_ALIVE_SEVERITY_CRITICAL
+};
+
+struct hl_eq_fw_alive {
+ __le64 uptime_seconds;
+ __le32 process_id;
+ __le32 thread_id;
+ /* enum hl_fw_alive_severity */
+ __u8 severity;
+ __u8 pad[7];
+};
+
struct hl_eq_entry {
struct hl_eq_header hdr;
union {
@@ -91,6 +105,7 @@ struct hl_eq_entry {
struct hl_eq_hbm_ecc_data hbm_ecc_data;
struct hl_eq_sm_sei_data sm_sei_data;
struct cpucp_pkt_sync_err pkt_sync_err;
+ struct hl_eq_fw_alive fw_alive;
__le64 data[7];
};
};
@@ -103,11 +118,16 @@ struct hl_eq_entry {
#define EQ_CTL_EVENT_TYPE_SHIFT 16
#define EQ_CTL_EVENT_TYPE_MASK 0x03FF0000
+#define EQ_CTL_INDEX_SHIFT 0
+#define EQ_CTL_INDEX_MASK 0x0000FFFF
+
enum pq_init_status {
PQ_INIT_STATUS_NA = 0,
PQ_INIT_STATUS_READY_FOR_CP,
PQ_INIT_STATUS_READY_FOR_HOST,
- PQ_INIT_STATUS_READY_FOR_CP_SINGLE_MSI
+ PQ_INIT_STATUS_READY_FOR_CP_SINGLE_MSI,
+ PQ_INIT_STATUS_LEN_NOT_POWER_OF_TWO_ERR,
+ PQ_INIT_STATUS_ILLEGAL_Q_ADDR_ERR
};
/*
@@ -384,6 +404,20 @@ enum cpucp_packet_id {
#define CPUCP_PKT_RES_PLL_OUT3_SHIFT 48
#define CPUCP_PKT_RES_PLL_OUT3_MASK 0xFFFF000000000000ull
+#define CPUCP_PKT_VAL_PFC_IN1_SHIFT 0
+#define CPUCP_PKT_VAL_PFC_IN1_MASK 0x0000000000000001ull
+#define CPUCP_PKT_VAL_PFC_IN2_SHIFT 1
+#define CPUCP_PKT_VAL_PFC_IN2_MASK 0x000000000000001Eull
+
+#define CPUCP_PKT_VAL_LPBK_IN1_SHIFT 0
+#define CPUCP_PKT_VAL_LPBK_IN1_MASK 0x0000000000000001ull
+#define CPUCP_PKT_VAL_LPBK_IN2_SHIFT 1
+#define CPUCP_PKT_VAL_LPBK_IN2_MASK 0x000000000000001Eull
+
+/* heartbeat status bits */
+#define CPUCP_PKT_HB_STATUS_EQ_FAULT_SHIFT 0
+#define CPUCP_PKT_HB_STATUS_EQ_FAULT_MASK 0x00000001
+
struct cpucp_packet {
union {
__le64 value; /* For SET packets */
@@ -425,6 +459,12 @@ struct cpucp_packet {
/* For get CpuCP info/EEPROM data/NIC info */
__le32 data_max_size;
+
+ /*
+ * For any general status bitmask. Shall be used whenever the
+ * result cannot be used to hold general purpose data.
+ */
+ __le32 status_mask;
};
__le32 reserved;
@@ -629,6 +669,8 @@ struct cpucp_security_info {
* @card_name: card name that will be displayed in HWMON subsystem on the host
* @sec_info: security information
* @pll_map: Bit map of supported PLLs for current ASIC version.
+ * @mme_binning_mask: MME binning mask,
+ * (0 = functional, 1 = binned)
*/
struct cpucp_info {
struct cpucp_sensor sensors[CPUCP_MAX_SENSORS];
@@ -651,6 +693,7 @@ struct cpucp_info {
struct cpucp_security_info sec_info;
__le32 reserved6;
__u8 pll_map[PLL_MAP_LEN];
+ __le64 mme_binning_mask;
};
struct cpucp_mac_addr {
diff --git a/drivers/misc/habanalabs/include/common/hl_boot_if.h b/drivers/misc/habanalabs/include/common/hl_boot_if.h
index e0a259e0495c..fa8a5ad2d438 100644
--- a/drivers/misc/habanalabs/include/common/hl_boot_if.h
+++ b/drivers/misc/habanalabs/include/common/hl_boot_if.h
@@ -8,7 +8,7 @@
#ifndef HL_BOOT_IF_H
#define HL_BOOT_IF_H
-#define LKD_HARD_RESET_MAGIC 0xED7BD694
+#define LKD_HARD_RESET_MAGIC 0xED7BD694 /* deprecated - do not use */
#define HL_POWER9_HOST_MAGIC 0x1DA30009
#define BOOT_FIT_SRAM_OFFSET 0x200000
@@ -99,6 +99,7 @@
#define CPU_BOOT_ERR0_PLL_FAIL (1 << 12)
#define CPU_BOOT_ERR0_DEVICE_UNUSABLE_FAIL (1 << 13)
#define CPU_BOOT_ERR0_ENABLED (1 << 31)
+#define CPU_BOOT_ERR1_ENABLED (1 << 31)
/*
* BOOT DEVICE STATUS bits in BOOT_DEVICE_STS registers
@@ -190,6 +191,24 @@
* PLLs.
* Initialized in: linux
*
+ * CPU_BOOT_DEV_STS0_GIC_PRIVILEGED_EN GIC access permission only from
+ * previleged entity. FW sets this status
+ * bit for host. If this bit is set then
+ * GIC can not be accessed from host.
+ * Initialized in: linux
+ *
+ * CPU_BOOT_DEV_STS0_EQ_INDEX_EN Event Queue (EQ) index is a running
+ * index for each new event sent to host.
+ * This is used as a method in host to
+ * identify that the waiting event in
+ * queue is actually a new event which
+ * was not served before.
+ * Initialized in: linux
+ *
+ * CPU_BOOT_DEV_STS0_MULTI_IRQ_POLL_EN Use multiple scratchpad interfaces to
+ * prevent IRQs overriding each other.
+ * Initialized in: linux
+ *
* CPU_BOOT_DEV_STS0_ENABLED Device status register enabled.
* This is a main indication that the
* running FW populates the device status
@@ -218,7 +237,11 @@
#define CPU_BOOT_DEV_STS0_FW_LD_COM_EN (1 << 16)
#define CPU_BOOT_DEV_STS0_FW_IATU_CONF_EN (1 << 17)
#define CPU_BOOT_DEV_STS0_DYN_PLL_EN (1 << 19)
+#define CPU_BOOT_DEV_STS0_GIC_PRIVILEGED_EN (1 << 20)
+#define CPU_BOOT_DEV_STS0_EQ_INDEX_EN (1 << 21)
+#define CPU_BOOT_DEV_STS0_MULTI_IRQ_POLL_EN (1 << 22)
#define CPU_BOOT_DEV_STS0_ENABLED (1 << 31)
+#define CPU_BOOT_DEV_STS1_ENABLED (1 << 31)
enum cpu_boot_status {
CPU_BOOT_STATUS_NA = 0, /* Default value after reset of chip */
@@ -264,46 +287,98 @@ enum cpu_msg_status {
/* communication registers mapping - consider ABI when changing */
struct cpu_dyn_regs {
- uint32_t cpu_pq_base_addr_low;
- uint32_t cpu_pq_base_addr_high;
- uint32_t cpu_pq_length;
- uint32_t cpu_pq_init_status;
- uint32_t cpu_eq_base_addr_low;
- uint32_t cpu_eq_base_addr_high;
- uint32_t cpu_eq_length;
- uint32_t cpu_eq_ci;
- uint32_t cpu_cq_base_addr_low;
- uint32_t cpu_cq_base_addr_high;
- uint32_t cpu_cq_length;
- uint32_t cpu_pf_pq_pi;
- uint32_t cpu_boot_dev_sts0;
- uint32_t cpu_boot_dev_sts1;
- uint32_t cpu_boot_err0;
- uint32_t cpu_boot_err1;
- uint32_t cpu_boot_status;
- uint32_t fw_upd_sts;
- uint32_t fw_upd_cmd;
- uint32_t fw_upd_pending_sts;
- uint32_t fuse_ver_offset;
- uint32_t preboot_ver_offset;
- uint32_t uboot_ver_offset;
- uint32_t hw_state;
- uint32_t kmd_msg_to_cpu;
- uint32_t cpu_cmd_status_to_host;
- uint32_t reserved1[32]; /* reserve for future use */
+ __le32 cpu_pq_base_addr_low;
+ __le32 cpu_pq_base_addr_high;
+ __le32 cpu_pq_length;
+ __le32 cpu_pq_init_status;
+ __le32 cpu_eq_base_addr_low;
+ __le32 cpu_eq_base_addr_high;
+ __le32 cpu_eq_length;
+ __le32 cpu_eq_ci;
+ __le32 cpu_cq_base_addr_low;
+ __le32 cpu_cq_base_addr_high;
+ __le32 cpu_cq_length;
+ __le32 cpu_pf_pq_pi;
+ __le32 cpu_boot_dev_sts0;
+ __le32 cpu_boot_dev_sts1;
+ __le32 cpu_boot_err0;
+ __le32 cpu_boot_err1;
+ __le32 cpu_boot_status;
+ __le32 fw_upd_sts;
+ __le32 fw_upd_cmd;
+ __le32 fw_upd_pending_sts;
+ __le32 fuse_ver_offset;
+ __le32 preboot_ver_offset;
+ __le32 uboot_ver_offset;
+ __le32 hw_state;
+ __le32 kmd_msg_to_cpu;
+ __le32 cpu_cmd_status_to_host;
+ union {
+ __le32 gic_host_irq_ctrl;
+ __le32 gic_host_pi_upd_irq;
+ };
+ __le32 gic_tpc_qm_irq_ctrl;
+ __le32 gic_mme_qm_irq_ctrl;
+ __le32 gic_dma_qm_irq_ctrl;
+ __le32 gic_nic_qm_irq_ctrl;
+ __le32 gic_dma_core_irq_ctrl;
+ __le32 gic_host_halt_irq;
+ __le32 gic_host_ints_irq;
+ __le32 reserved1[24]; /* reserve for future use */
};
+/* TODO: remove the desc magic after the code is updated to use message */
/* HCDM - Habana Communications Descriptor Magic */
#define HL_COMMS_DESC_MAGIC 0x4843444D
#define HL_COMMS_DESC_VER 1
+/* HCMv - Habana Communications Message + header version */
+#define HL_COMMS_MSG_MAGIC_VALUE 0x48434D00
+#define HL_COMMS_MSG_MAGIC_MASK 0xFFFFFF00
+#define HL_COMMS_MSG_MAGIC_VER_MASK 0xFF
+
+#define HL_COMMS_MSG_MAGIC_VER(ver) (HL_COMMS_MSG_MAGIC_VALUE | \
+ ((ver) & HL_COMMS_MSG_MAGIC_VER_MASK))
+#define HL_COMMS_MSG_MAGIC_V0 HL_COMMS_DESC_MAGIC
+#define HL_COMMS_MSG_MAGIC_V1 HL_COMMS_MSG_MAGIC_VER(1)
+
+#define HL_COMMS_MSG_MAGIC HL_COMMS_MSG_MAGIC_V1
+
+#define HL_COMMS_MSG_MAGIC_VALIDATE_MAGIC(magic) \
+ (((magic) & HL_COMMS_MSG_MAGIC_MASK) == \
+ HL_COMMS_MSG_MAGIC_VALUE)
+
+#define HL_COMMS_MSG_MAGIC_VALIDATE_VERSION(magic, ver) \
+ (((magic) & HL_COMMS_MSG_MAGIC_VER_MASK) >= \
+ ((ver) & HL_COMMS_MSG_MAGIC_VER_MASK))
+
+#define HL_COMMS_MSG_MAGIC_VALIDATE(magic, ver) \
+ (HL_COMMS_MSG_MAGIC_VALIDATE_MAGIC((magic)) && \
+ HL_COMMS_MSG_MAGIC_VALIDATE_VERSION((magic), (ver)))
+
+enum comms_msg_type {
+ HL_COMMS_DESC_TYPE = 0,
+ HL_COMMS_RESET_CAUSE_TYPE = 1,
+};
+
+/* TODO: remove this struct after the code is updated to use message */
/* this is the comms descriptor header - meta data */
struct comms_desc_header {
- uint32_t magic; /* magic for validation */
- uint32_t crc32; /* CRC32 of the descriptor w/o header */
- uint16_t size; /* size of the descriptor w/o header */
- uint8_t version; /* descriptor version */
- uint8_t reserved[5]; /* pad to 64 bit */
+ __le32 magic; /* magic for validation */
+ __le32 crc32; /* CRC32 of the descriptor w/o header */
+ __le16 size; /* size of the descriptor w/o header */
+ __u8 version; /* descriptor version */
+ __u8 reserved[5]; /* pad to 64 bit */
+};
+
+/* this is the comms message header - meta data */
+struct comms_msg_header {
+ __le32 magic; /* magic for validation */
+ __le32 crc32; /* CRC32 of the message w/o header */
+ __le16 size; /* size of the message w/o header */
+ __u8 version; /* message payload version */
+ __u8 type; /* message type */
+ __u8 reserved[4]; /* pad to 64 bit */
};
/* this is the main FW descriptor - consider ABI when changing */
@@ -314,7 +389,36 @@ struct lkd_fw_comms_desc {
char cur_fw_ver[VERSION_MAX_LEN];
/* can be used for 1 more version w/o ABI change */
char reserved0[VERSION_MAX_LEN];
- uint64_t img_addr; /* address for next FW component load */
+ __le64 img_addr; /* address for next FW component load */
+};
+
+enum comms_reset_cause {
+ HL_RESET_CAUSE_UNKNOWN = 0,
+ HL_RESET_CAUSE_HEARTBEAT = 1,
+ HL_RESET_CAUSE_TDR = 2,
+};
+
+/* TODO: remove define after struct name is aligned on all projects */
+#define lkd_msg_comms lkd_fw_comms_msg
+
+/* this is the comms message descriptor */
+struct lkd_fw_comms_msg {
+ struct comms_msg_header header;
+ /* union for future expantions of new messages */
+ union {
+ struct {
+ struct cpu_dyn_regs cpu_dyn_regs;
+ char fuse_ver[VERSION_MAX_LEN];
+ char cur_fw_ver[VERSION_MAX_LEN];
+ /* can be used for 1 more version w/o ABI change */
+ char reserved0[VERSION_MAX_LEN];
+ /* address for next FW component load */
+ __le64 img_addr;
+ };
+ struct {
+ __u8 reset_cause;
+ };
+ };
};
/*
@@ -386,11 +490,11 @@ enum comms_cmd {
struct comms_command {
union { /* bit fields are only for FW use */
struct {
- unsigned int size :25; /* 32MB max. */
- unsigned int reserved :2;
+ u32 size :25; /* 32MB max. */
+ u32 reserved :2;
enum comms_cmd cmd :5; /* 32 commands */
};
- unsigned int val;
+ __le32 val;
};
};
@@ -449,11 +553,11 @@ enum comms_ram_types {
struct comms_status {
union { /* bit fields are only for FW use */
struct {
- unsigned int offset :26;
- unsigned int ram_type :2;
+ u32 offset :26;
+ enum comms_ram_types ram_type :2;
enum comms_sts status :4; /* 16 statuses */
};
- unsigned int val;
+ __le32 val;
};
};
diff --git a/drivers/misc/habanalabs/include/gaudi/gaudi_async_events.h b/drivers/misc/habanalabs/include/gaudi/gaudi_async_events.h
index e8651abf84f2..d966bd4dfea6 100644
--- a/drivers/misc/habanalabs/include/gaudi/gaudi_async_events.h
+++ b/drivers/misc/habanalabs/include/gaudi/gaudi_async_events.h
@@ -252,10 +252,11 @@ enum gaudi_async_event_id {
GAUDI_EVENT_HBM3_SPI_0 = 407,
GAUDI_EVENT_HBM3_SPI_1 = 408,
GAUDI_EVENT_PSOC_GPIO_U16_0 = 421,
- GAUDI_EVENT_PI_UPDATE = 484,
- GAUDI_EVENT_HALT_MACHINE = 485,
- GAUDI_EVENT_INTS_REGISTER = 486,
- GAUDI_EVENT_SOFT_RESET = 487,
+ GAUDI_EVENT_NIC0_CS_DBG_DERR = 483,
+ GAUDI_EVENT_NIC1_CS_DBG_DERR = 487,
+ GAUDI_EVENT_NIC2_CS_DBG_DERR = 491,
+ GAUDI_EVENT_NIC3_CS_DBG_DERR = 495,
+ GAUDI_EVENT_NIC4_CS_DBG_DERR = 499,
GAUDI_EVENT_RAZWI_OR_ADC = 548,
GAUDI_EVENT_TPC0_QM = 572,
GAUDI_EVENT_TPC1_QM = 573,
@@ -303,6 +304,11 @@ enum gaudi_async_event_id {
GAUDI_EVENT_NIC3_QP1 = 619,
GAUDI_EVENT_NIC4_QP0 = 620,
GAUDI_EVENT_NIC4_QP1 = 621,
+ GAUDI_EVENT_PI_UPDATE = 635,
+ GAUDI_EVENT_HALT_MACHINE = 636,
+ GAUDI_EVENT_INTS_REGISTER = 637,
+ GAUDI_EVENT_SOFT_RESET = 638,
+ GAUDI_EVENT_FW_ALIVE_S = 645,
GAUDI_EVENT_DEV_RESET_REQ = 646,
GAUDI_EVENT_PKT_QUEUE_OUT_SYNC = 647,
GAUDI_EVENT_FIX_POWER_ENV_S = 658,
diff --git a/drivers/misc/habanalabs/include/gaudi/gaudi_async_ids_map_extended.h b/drivers/misc/habanalabs/include/gaudi/gaudi_async_ids_map_extended.h
index 3dc79c131805..479b6b038254 100644
--- a/drivers/misc/habanalabs/include/gaudi/gaudi_async_ids_map_extended.h
+++ b/drivers/misc/habanalabs/include/gaudi/gaudi_async_ids_map_extended.h
@@ -507,23 +507,28 @@ static struct gaudi_async_events_ids_map gaudi_irq_map_table[] = {
{ .fc_id = 480, .cpu_id = 329, .valid = 0, .name = "" },
{ .fc_id = 481, .cpu_id = 330, .valid = 0, .name = "" },
{ .fc_id = 482, .cpu_id = 331, .valid = 0, .name = "" },
- { .fc_id = 483, .cpu_id = 332, .valid = 0, .name = "" },
- { .fc_id = 484, .cpu_id = 333, .valid = 1, .name = "PI_UPDATE" },
- { .fc_id = 485, .cpu_id = 334, .valid = 1, .name = "HALT_MACHINE" },
- { .fc_id = 486, .cpu_id = 335, .valid = 1, .name = "INTS_REGISTER" },
- { .fc_id = 487, .cpu_id = 336, .valid = 1, .name = "SOFT_RESET" },
+ { .fc_id = 483, .cpu_id = 332, .valid = 1,
+ .name = "NIC0_CS_DBG_DERR" },
+ { .fc_id = 484, .cpu_id = 333, .valid = 0, .name = "" },
+ { .fc_id = 485, .cpu_id = 334, .valid = 0, .name = "" },
+ { .fc_id = 486, .cpu_id = 335, .valid = 0, .name = "" },
+ { .fc_id = 487, .cpu_id = 336, .valid = 1,
+ .name = "NIC1_CS_DBG_DERR" },
{ .fc_id = 488, .cpu_id = 337, .valid = 0, .name = "" },
{ .fc_id = 489, .cpu_id = 338, .valid = 0, .name = "" },
{ .fc_id = 490, .cpu_id = 339, .valid = 0, .name = "" },
- { .fc_id = 491, .cpu_id = 340, .valid = 0, .name = "" },
+ { .fc_id = 491, .cpu_id = 340, .valid = 1,
+ .name = "NIC2_CS_DBG_DERR" },
{ .fc_id = 492, .cpu_id = 341, .valid = 0, .name = "" },
{ .fc_id = 493, .cpu_id = 342, .valid = 0, .name = "" },
{ .fc_id = 494, .cpu_id = 343, .valid = 0, .name = "" },
- { .fc_id = 495, .cpu_id = 344, .valid = 0, .name = "" },
+ { .fc_id = 495, .cpu_id = 344, .valid = 1,
+ .name = "NIC3_CS_DBG_DERR" },
{ .fc_id = 496, .cpu_id = 345, .valid = 0, .name = "" },
{ .fc_id = 497, .cpu_id = 346, .valid = 0, .name = "" },
{ .fc_id = 498, .cpu_id = 347, .valid = 0, .name = "" },
- { .fc_id = 499, .cpu_id = 348, .valid = 0, .name = "" },
+ { .fc_id = 499, .cpu_id = 348, .valid = 1,
+ .name = "NIC4_CS_DBG_DERR" },
{ .fc_id = 500, .cpu_id = 349, .valid = 0, .name = "" },
{ .fc_id = 501, .cpu_id = 350, .valid = 0, .name = "" },
{ .fc_id = 502, .cpu_id = 351, .valid = 0, .name = "" },
@@ -659,17 +664,17 @@ static struct gaudi_async_events_ids_map gaudi_irq_map_table[] = {
{ .fc_id = 632, .cpu_id = 481, .valid = 0, .name = "" },
{ .fc_id = 633, .cpu_id = 482, .valid = 0, .name = "" },
{ .fc_id = 634, .cpu_id = 483, .valid = 0, .name = "" },
- { .fc_id = 635, .cpu_id = 484, .valid = 0, .name = "" },
- { .fc_id = 636, .cpu_id = 485, .valid = 0, .name = "" },
- { .fc_id = 637, .cpu_id = 486, .valid = 0, .name = "" },
- { .fc_id = 638, .cpu_id = 487, .valid = 0, .name = "" },
+ { .fc_id = 635, .cpu_id = 484, .valid = 1, .name = "PI_UPDATE" },
+ { .fc_id = 636, .cpu_id = 485, .valid = 1, .name = "HALT_MACHINE" },
+ { .fc_id = 637, .cpu_id = 486, .valid = 1, .name = "INTS_REGISTER" },
+ { .fc_id = 638, .cpu_id = 487, .valid = 1, .name = "SOFT_RESET" },
{ .fc_id = 639, .cpu_id = 488, .valid = 0, .name = "" },
{ .fc_id = 640, .cpu_id = 489, .valid = 0, .name = "" },
{ .fc_id = 641, .cpu_id = 490, .valid = 0, .name = "" },
{ .fc_id = 642, .cpu_id = 491, .valid = 0, .name = "" },
{ .fc_id = 643, .cpu_id = 492, .valid = 0, .name = "" },
{ .fc_id = 644, .cpu_id = 493, .valid = 0, .name = "" },
- { .fc_id = 645, .cpu_id = 494, .valid = 0, .name = "" },
+ { .fc_id = 645, .cpu_id = 494, .valid = 1, .name = "FW_ALIVE_S" },
{ .fc_id = 646, .cpu_id = 495, .valid = 1, .name = "DEV_RESET_REQ" },
{ .fc_id = 647, .cpu_id = 496, .valid = 1,
.name = "PKT_QUEUE_OUT_SYNC" },
diff --git a/drivers/misc/habanalabs/include/gaudi/gaudi_fw_if.h b/drivers/misc/habanalabs/include/gaudi/gaudi_fw_if.h
index a9f51f9f9e92..34ca4fe50d91 100644
--- a/drivers/misc/habanalabs/include/gaudi/gaudi_fw_if.h
+++ b/drivers/misc/habanalabs/include/gaudi/gaudi_fw_if.h
@@ -20,6 +20,9 @@
#define UBOOT_FW_OFFSET 0x100000 /* 1MB in SRAM */
#define LINUX_FW_OFFSET 0x800000 /* 8MB in HBM */
+/* HBM thermal delta in [Deg] added to composite (CTemp) */
+#define HBM_TEMP_ADJUST_COEFF 6
+
enum gaudi_nic_axi_error {
RXB,
RXE,
@@ -27,6 +30,7 @@ enum gaudi_nic_axi_error {
TXE,
QPC_RESP,
NON_AXI_ERR,
+ TMR,
};
/*
@@ -42,6 +46,48 @@ struct eq_nic_sei_event {
__u8 pad[6];
};
+/*
+ * struct gaudi_nic_status - describes the status of a NIC port.
+ * @port: NIC port index.
+ * @bad_format_cnt: e.g. CRC.
+ * @responder_out_of_sequence_psn_cnt: e.g NAK.
+ * @high_ber_reinit_cnt: link reinit due to high BER.
+ * @correctable_err_cnt: e.g. bit-flip.
+ * @uncorrectable_err_cnt: e.g. MAC errors.
+ * @retraining_cnt: re-training counter.
+ * @up: is port up.
+ * @pcs_link: has PCS link.
+ * @phy_ready: is PHY ready.
+ * @auto_neg: is Autoneg enabled.
+ * @timeout_retransmission_cnt: timeout retransmission events
+ * @high_ber_cnt: high ber events
+ */
+struct gaudi_nic_status {
+ __u32 port;
+ __u32 bad_format_cnt;
+ __u32 responder_out_of_sequence_psn_cnt;
+ __u32 high_ber_reinit;
+ __u32 correctable_err_cnt;
+ __u32 uncorrectable_err_cnt;
+ __u32 retraining_cnt;
+ __u8 up;
+ __u8 pcs_link;
+ __u8 phy_ready;
+ __u8 auto_neg;
+ __u32 timeout_retransmission_cnt;
+ __u32 high_ber_cnt;
+};
+
+struct gaudi_flops_2_data {
+ union {
+ struct {
+ __u32 spsram_init_done : 1;
+ __u32 reserved : 31;
+ };
+ __u32 data;
+ };
+};
+
#define GAUDI_PLL_FREQ_LOW 200000000 /* 200 MHz */
#endif /* GAUDI_FW_IF_H */
diff --git a/drivers/misc/habanalabs/include/gaudi/gaudi_masks.h b/drivers/misc/habanalabs/include/gaudi/gaudi_masks.h
index b53aeda9a982..9aea7e996654 100644
--- a/drivers/misc/habanalabs/include/gaudi/gaudi_masks.h
+++ b/drivers/misc/habanalabs/include/gaudi/gaudi_masks.h
@@ -66,7 +66,8 @@
#define PCI_DMA_QMAN_GLBL_ERR_CFG_STOP_ON_ERR_EN_MASK (\
(FIELD_PREP(DMA0_QM_GLBL_ERR_CFG_PQF_STOP_ON_ERR_MASK, 0xF)) | \
(FIELD_PREP(DMA0_QM_GLBL_ERR_CFG_CQF_STOP_ON_ERR_MASK, 0xF)) | \
- (FIELD_PREP(DMA0_QM_GLBL_ERR_CFG_CP_STOP_ON_ERR_MASK, 0xF)))
+ (FIELD_PREP(DMA0_QM_GLBL_ERR_CFG_CP_STOP_ON_ERR_MASK, 0xF)) | \
+ (FIELD_PREP(DMA0_QM_GLBL_ERR_CFG_ARB_STOP_ON_ERR_MASK, 0x1)))
#define HBM_DMA_QMAN_GLBL_ERR_CFG_MSG_EN_MASK (\
(FIELD_PREP(DMA0_QM_GLBL_ERR_CFG_PQF_ERR_MSG_EN_MASK, 0xF)) | \
@@ -76,7 +77,8 @@
#define HBM_DMA_QMAN_GLBL_ERR_CFG_STOP_ON_ERR_EN_MASK (\
(FIELD_PREP(DMA0_QM_GLBL_ERR_CFG_PQF_STOP_ON_ERR_MASK, 0xF)) | \
(FIELD_PREP(DMA0_QM_GLBL_ERR_CFG_CQF_STOP_ON_ERR_MASK, 0x1F)) | \
- (FIELD_PREP(DMA0_QM_GLBL_ERR_CFG_CP_STOP_ON_ERR_MASK, 0x1F)))
+ (FIELD_PREP(DMA0_QM_GLBL_ERR_CFG_CP_STOP_ON_ERR_MASK, 0x1F)) | \
+ (FIELD_PREP(DMA0_QM_GLBL_ERR_CFG_ARB_STOP_ON_ERR_MASK, 0x1)))
#define TPC_QMAN_GLBL_ERR_CFG_MSG_EN_MASK (\
(FIELD_PREP(TPC0_QM_GLBL_ERR_CFG_PQF_ERR_MSG_EN_MASK, 0xF)) | \
@@ -86,7 +88,8 @@
#define TPC_QMAN_GLBL_ERR_CFG_STOP_ON_ERR_EN_MASK (\
(FIELD_PREP(TPC0_QM_GLBL_ERR_CFG_PQF_STOP_ON_ERR_MASK, 0xF)) | \
(FIELD_PREP(TPC0_QM_GLBL_ERR_CFG_CQF_STOP_ON_ERR_MASK, 0x1F)) | \
- (FIELD_PREP(TPC0_QM_GLBL_ERR_CFG_CP_STOP_ON_ERR_MASK, 0x1F)))
+ (FIELD_PREP(TPC0_QM_GLBL_ERR_CFG_CP_STOP_ON_ERR_MASK, 0x1F)) | \
+ (FIELD_PREP(TPC0_QM_GLBL_ERR_CFG_ARB_STOP_ON_ERR_MASK, 0x1)))
#define MME_QMAN_GLBL_ERR_CFG_MSG_EN_MASK (\
(FIELD_PREP(MME0_QM_GLBL_ERR_CFG_PQF_ERR_MSG_EN_MASK, 0xF)) | \
@@ -96,7 +99,8 @@
#define MME_QMAN_GLBL_ERR_CFG_STOP_ON_ERR_EN_MASK (\
(FIELD_PREP(MME0_QM_GLBL_ERR_CFG_PQF_STOP_ON_ERR_MASK, 0xF)) | \
(FIELD_PREP(MME0_QM_GLBL_ERR_CFG_CQF_STOP_ON_ERR_MASK, 0x1F)) | \
- (FIELD_PREP(MME0_QM_GLBL_ERR_CFG_CP_STOP_ON_ERR_MASK, 0x1F)))
+ (FIELD_PREP(MME0_QM_GLBL_ERR_CFG_CP_STOP_ON_ERR_MASK, 0x1F)) | \
+ (FIELD_PREP(MME0_QM_GLBL_ERR_CFG_ARB_STOP_ON_ERR_MASK, 0x1)))
#define NIC_QMAN_GLBL_ERR_CFG_MSG_EN_MASK (\
(FIELD_PREP(NIC0_QM0_GLBL_ERR_CFG_PQF_ERR_MSG_EN_MASK, 0xF)) | \
@@ -106,7 +110,8 @@
#define NIC_QMAN_GLBL_ERR_CFG_STOP_ON_ERR_EN_MASK (\
(FIELD_PREP(NIC0_QM0_GLBL_ERR_CFG_PQF_STOP_ON_ERR_MASK, 0xF)) | \
(FIELD_PREP(NIC0_QM0_GLBL_ERR_CFG_CQF_STOP_ON_ERR_MASK, 0xF)) | \
- (FIELD_PREP(NIC0_QM0_GLBL_ERR_CFG_CP_STOP_ON_ERR_MASK, 0xF)))
+ (FIELD_PREP(NIC0_QM0_GLBL_ERR_CFG_CP_STOP_ON_ERR_MASK, 0xF)) | \
+ (FIELD_PREP(NIC0_QM0_GLBL_ERR_CFG_ARB_STOP_ON_ERR_MASK, 0x1)))
#define QMAN_CGM1_PWR_GATE_EN (FIELD_PREP(DMA0_QM_CGM_CFG1_MASK_TH_MASK, 0xA))
diff --git a/drivers/misc/habanalabs/include/gaudi/gaudi_reg_map.h b/drivers/misc/habanalabs/include/gaudi/gaudi_reg_map.h
index 137afedf5f15..d95d4162ae2c 100644
--- a/drivers/misc/habanalabs/include/gaudi/gaudi_reg_map.h
+++ b/drivers/misc/habanalabs/include/gaudi/gaudi_reg_map.h
@@ -12,6 +12,16 @@
* PSOC scratch-pad registers
*/
#define mmHW_STATE mmPSOC_GLOBAL_CONF_SCRATCHPAD_0
+/* TODO: remove mmGIC_HOST_IRQ_CTRL_POLL_REG */
+#define mmGIC_HOST_IRQ_CTRL_POLL_REG mmPSOC_GLOBAL_CONF_SCRATCHPAD_1
+#define mmGIC_HOST_PI_UPD_IRQ_POLL_REG mmPSOC_GLOBAL_CONF_SCRATCHPAD_1
+#define mmGIC_TPC_QM_IRQ_CTRL_POLL_REG mmPSOC_GLOBAL_CONF_SCRATCHPAD_2
+#define mmGIC_MME_QM_IRQ_CTRL_POLL_REG mmPSOC_GLOBAL_CONF_SCRATCHPAD_3
+#define mmGIC_DMA_QM_IRQ_CTRL_POLL_REG mmPSOC_GLOBAL_CONF_SCRATCHPAD_4
+#define mmGIC_NIC_QM_IRQ_CTRL_POLL_REG mmPSOC_GLOBAL_CONF_SCRATCHPAD_5
+#define mmGIC_DMA_CR_IRQ_CTRL_POLL_REG mmPSOC_GLOBAL_CONF_SCRATCHPAD_6
+#define mmGIC_HOST_HALT_IRQ_POLL_REG mmPSOC_GLOBAL_CONF_SCRATCHPAD_7
+#define mmGIC_HOST_INTS_IRQ_POLL_REG mmPSOC_GLOBAL_CONF_SCRATCHPAD_8
#define mmCPU_BOOT_DEV_STS0 mmPSOC_GLOBAL_CONF_SCRATCHPAD_20
#define mmCPU_BOOT_DEV_STS1 mmPSOC_GLOBAL_CONF_SCRATCHPAD_21
#define mmFUSE_VER_OFFSET mmPSOC_GLOBAL_CONF_SCRATCHPAD_22
diff --git a/drivers/misc/hpilo.c b/drivers/misc/hpilo.c
index fea3ae9d8686..8d00df9243c4 100644
--- a/drivers/misc/hpilo.c
+++ b/drivers/misc/hpilo.c
@@ -693,6 +693,8 @@ static int ilo_map_device(struct pci_dev *pdev, struct ilo_hwinfo *hw)
{
int bar;
unsigned long off;
+ u8 pci_rev_id;
+ int rc;
/* map the memory mapped i/o registers */
hw->mmio_vaddr = pci_iomap(pdev, 1, 0);
@@ -702,7 +704,13 @@ static int ilo_map_device(struct pci_dev *pdev, struct ilo_hwinfo *hw)
}
/* map the adapter shared memory region */
- if (pdev->subsystem_device == 0x00E4) {
+ rc = pci_read_config_byte(pdev, PCI_REVISION_ID, &pci_rev_id);
+ if (rc != 0) {
+ dev_err(&pdev->dev, "Error reading PCI rev id: %d\n", rc);
+ goto out;
+ }
+
+ if (pci_rev_id >= PCI_REV_ID_NECHES) {
bar = 5;
/* Last 8k is reserved for CCBs */
off = pci_resource_len(pdev, bar) - 0x2000;
diff --git a/drivers/misc/hpilo.h b/drivers/misc/hpilo.h
index f69ff645cac9..d57c34680b09 100644
--- a/drivers/misc/hpilo.h
+++ b/drivers/misc/hpilo.h
@@ -10,6 +10,9 @@
#define ILO_NAME "hpilo"
+/* iLO ASIC PCI revision id */
+#define PCI_REV_ID_NECHES 7
+
/* max number of open channel control blocks per device, hw limited to 32 */
#define MAX_CCB 24
/* min number of open channel control blocks per device, hw limited to 32 */
diff --git a/drivers/misc/ibmasm/module.c b/drivers/misc/ibmasm/module.c
index 4edad6c445d3..dc8a06c06c63 100644
--- a/drivers/misc/ibmasm/module.c
+++ b/drivers/misc/ibmasm/module.c
@@ -111,7 +111,7 @@ static int ibmasm_init_one(struct pci_dev *pdev, const struct pci_device_id *id)
result = ibmasm_init_remote_input_dev(sp);
if (result) {
dev_err(sp->dev, "Failed to initialize remote queue\n");
- goto error_send_message;
+ goto error_init_remote;
}
result = ibmasm_send_driver_vpd(sp);
@@ -131,8 +131,9 @@ static int ibmasm_init_one(struct pci_dev *pdev, const struct pci_device_id *id)
return 0;
error_send_message:
- disable_sp_interrupts(sp->base_address);
ibmasm_free_remote_input_dev(sp);
+error_init_remote:
+ disable_sp_interrupts(sp->base_address);
free_irq(sp->irq, (void *)sp);
error_request_irq:
iounmap(sp->base_address);
diff --git a/drivers/misc/ibmasm/remote.h b/drivers/misc/ibmasm/remote.h
index 8d364462aeea..ec4e78ec5a67 100644
--- a/drivers/misc/ibmasm/remote.h
+++ b/drivers/misc/ibmasm/remote.h
@@ -43,7 +43,7 @@
#define REMOTE_BUTTON_MIDDLE 0x02
#define REMOTE_BUTTON_RIGHT 0x04
-/* size of keysym/keycode translation matricies */
+/* size of keysym/keycode translation matrices */
#define XLATE_SIZE 256
struct mouse_input {
diff --git a/drivers/misc/lkdtm/bugs.c b/drivers/misc/lkdtm/bugs.c
index a164896dc6d4..88c218a9f8b3 100644
--- a/drivers/misc/lkdtm/bugs.c
+++ b/drivers/misc/lkdtm/bugs.c
@@ -161,6 +161,9 @@ void lkdtm_UNALIGNED_LOAD_STORE_WRITE(void)
if (*p == 0)
val = 0x87654321;
*p = val;
+
+ if (IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS))
+ pr_err("XFAIL: arch has CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS\n");
}
void lkdtm_SOFTLOCKUP(void)
@@ -300,8 +303,10 @@ void lkdtm_CORRUPT_LIST_ADD(void)
if (target[0] == NULL && target[1] == NULL)
pr_err("Overwrite did not happen, but no BUG?!\n");
- else
+ else {
pr_err("list_add() corruption not detected!\n");
+ pr_expected_config(CONFIG_DEBUG_LIST);
+ }
}
void lkdtm_CORRUPT_LIST_DEL(void)
@@ -325,8 +330,10 @@ void lkdtm_CORRUPT_LIST_DEL(void)
if (target[0] == NULL && target[1] == NULL)
pr_err("Overwrite did not happen, but no BUG?!\n");
- else
+ else {
pr_err("list_del() corruption not detected!\n");
+ pr_expected_config(CONFIG_DEBUG_LIST);
+ }
}
/* Test that VMAP_STACK is actually allocating with a leading guard page */
diff --git a/drivers/misc/lkdtm/cfi.c b/drivers/misc/lkdtm/cfi.c
index e73ebdbfa806..c9aeddef1044 100644
--- a/drivers/misc/lkdtm/cfi.c
+++ b/drivers/misc/lkdtm/cfi.c
@@ -38,5 +38,6 @@ void lkdtm_CFI_FORWARD_PROTO(void)
func = (void *)lkdtm_increment_int;
func(&called_count);
- pr_info("Fail: survived mismatched prototype function call!\n");
+ pr_err("FAIL: survived mismatched prototype function call!\n");
+ pr_expected_config(CONFIG_CFI_CLANG);
}
diff --git a/drivers/misc/lkdtm/core.c b/drivers/misc/lkdtm/core.c
index 8024b6a5cc7f..9dda87c6b54a 100644
--- a/drivers/misc/lkdtm/core.c
+++ b/drivers/misc/lkdtm/core.c
@@ -26,6 +26,7 @@
#include <linux/init.h>
#include <linux/slab.h>
#include <linux/debugfs.h>
+#include <linux/init.h>
#define DEFAULT_COUNT 10
@@ -120,11 +121,14 @@ static const struct crashtype crashtypes[] = {
CRASHTYPE(UNALIGNED_LOAD_STORE_WRITE),
CRASHTYPE(FORTIFY_OBJECT),
CRASHTYPE(FORTIFY_SUBOBJECT),
- CRASHTYPE(OVERWRITE_ALLOCATION),
+ CRASHTYPE(SLAB_LINEAR_OVERFLOW),
+ CRASHTYPE(VMALLOC_LINEAR_OVERFLOW),
CRASHTYPE(WRITE_AFTER_FREE),
CRASHTYPE(READ_AFTER_FREE),
CRASHTYPE(WRITE_BUDDY_AFTER_FREE),
CRASHTYPE(READ_BUDDY_AFTER_FREE),
+ CRASHTYPE(SLAB_INIT_ON_ALLOC),
+ CRASHTYPE(BUDDY_INIT_ON_ALLOC),
CRASHTYPE(SLAB_FREE_DOUBLE),
CRASHTYPE(SLAB_FREE_CROSS),
CRASHTYPE(SLAB_FREE_PAGE),
@@ -177,9 +181,7 @@ static const struct crashtype crashtypes[] = {
CRASHTYPE(STACKLEAK_ERASING),
CRASHTYPE(CFI_FORWARD_PROTO),
CRASHTYPE(FORTIFIED_STRSCPY),
-#ifdef CONFIG_X86_32
CRASHTYPE(DOUBLE_FAULT),
-#endif
#ifdef CONFIG_PPC_BOOK3S_64
CRASHTYPE(PPC_SLB_MULTIHIT),
#endif
@@ -399,6 +401,56 @@ static ssize_t direct_entry(struct file *f, const char __user *user_buf,
return count;
}
+#ifndef MODULE
+/*
+ * To avoid needing to export parse_args(), just don't use this code
+ * when LKDTM is built as a module.
+ */
+struct check_cmdline_args {
+ const char *param;
+ int value;
+};
+
+static int lkdtm_parse_one(char *param, char *val,
+ const char *unused, void *arg)
+{
+ struct check_cmdline_args *args = arg;
+
+ /* short circuit if we already found a value. */
+ if (args->value != -ESRCH)
+ return 0;
+ if (strncmp(param, args->param, strlen(args->param)) == 0) {
+ bool bool_result;
+ int ret;
+
+ ret = kstrtobool(val, &bool_result);
+ if (ret == 0)
+ args->value = bool_result;
+ }
+ return 0;
+}
+
+int lkdtm_check_bool_cmdline(const char *param)
+{
+ char *command_line;
+ struct check_cmdline_args args = {
+ .param = param,
+ .value = -ESRCH,
+ };
+
+ command_line = kstrdup(saved_command_line, GFP_KERNEL);
+ if (!command_line)
+ return -ENOMEM;
+
+ parse_args("Setting sysctl args", command_line,
+ NULL, 0, -1, -1, &args, lkdtm_parse_one);
+
+ kfree(command_line);
+
+ return args.value;
+}
+#endif
+
static struct dentry *lkdtm_debugfs_root;
static int __init lkdtm_module_init(void)
diff --git a/drivers/misc/lkdtm/fortify.c b/drivers/misc/lkdtm/fortify.c
index faf29cf04baa..0f51d31b57ca 100644
--- a/drivers/misc/lkdtm/fortify.c
+++ b/drivers/misc/lkdtm/fortify.c
@@ -76,7 +76,8 @@ void lkdtm_FORTIFIED_STRSCPY(void)
*/
strscpy(dst, src, strlen(src));
- pr_warn("FAIL: No overflow in above strscpy()\n");
+ pr_err("FAIL: strscpy() overflow not detected!\n");
+ pr_expected_config(CONFIG_FORTIFY_SOURCE);
kfree(src);
}
diff --git a/drivers/misc/lkdtm/heap.c b/drivers/misc/lkdtm/heap.c
index 1323bc16f113..3d9aae5821a0 100644
--- a/drivers/misc/lkdtm/heap.c
+++ b/drivers/misc/lkdtm/heap.c
@@ -5,6 +5,7 @@
*/
#include "lkdtm.h"
#include <linux/slab.h>
+#include <linux/vmalloc.h>
#include <linux/sched.h>
static struct kmem_cache *double_free_cache;
@@ -12,17 +13,36 @@ static struct kmem_cache *a_cache;
static struct kmem_cache *b_cache;
/*
+ * If there aren't guard pages, it's likely that a consecutive allocation will
+ * let us overflow into the second allocation without overwriting something real.
+ */
+void lkdtm_VMALLOC_LINEAR_OVERFLOW(void)
+{
+ char *one, *two;
+
+ one = vzalloc(PAGE_SIZE);
+ two = vzalloc(PAGE_SIZE);
+
+ pr_info("Attempting vmalloc linear overflow ...\n");
+ memset(one, 0xAA, PAGE_SIZE + 1);
+
+ vfree(two);
+ vfree(one);
+}
+
+/*
* This tries to stay within the next largest power-of-2 kmalloc cache
* to avoid actually overwriting anything important if it's not detected
* correctly.
*/
-void lkdtm_OVERWRITE_ALLOCATION(void)
+void lkdtm_SLAB_LINEAR_OVERFLOW(void)
{
size_t len = 1020;
u32 *data = kmalloc(len, GFP_KERNEL);
if (!data)
return;
+ pr_info("Attempting slab linear overflow ...\n");
data[1024 / sizeof(u32)] = 0x12345678;
kfree(data);
}
@@ -89,9 +109,10 @@ void lkdtm_READ_AFTER_FREE(void)
if (saw != *val) {
/* Good! Poisoning happened, so declare a win. */
pr_info("Memory correctly poisoned (%x)\n", saw);
- BUG();
+ } else {
+ pr_err("FAIL: Memory was not poisoned!\n");
+ pr_expected_config_param(CONFIG_INIT_ON_FREE_DEFAULT_ON, "init_on_free");
}
- pr_info("Memory was not poisoned\n");
kfree(val);
}
@@ -145,13 +166,79 @@ void lkdtm_READ_BUDDY_AFTER_FREE(void)
if (saw != *val) {
/* Good! Poisoning happened, so declare a win. */
pr_info("Memory correctly poisoned (%x)\n", saw);
- BUG();
+ } else {
+ pr_err("FAIL: Buddy page was not poisoned!\n");
+ pr_expected_config_param(CONFIG_INIT_ON_FREE_DEFAULT_ON, "init_on_free");
+ }
+
+ kfree(val);
+}
+
+void lkdtm_SLAB_INIT_ON_ALLOC(void)
+{
+ u8 *first;
+ u8 *val;
+
+ first = kmalloc(512, GFP_KERNEL);
+ if (!first) {
+ pr_info("Unable to allocate 512 bytes the first time.\n");
+ return;
+ }
+
+ memset(first, 0xAB, 512);
+ kfree(first);
+
+ val = kmalloc(512, GFP_KERNEL);
+ if (!val) {
+ pr_info("Unable to allocate 512 bytes the second time.\n");
+ return;
+ }
+ if (val != first) {
+ pr_warn("Reallocation missed clobbered memory.\n");
}
- pr_info("Buddy page was not poisoned\n");
+ if (memchr(val, 0xAB, 512) == NULL) {
+ pr_info("Memory appears initialized (%x, no earlier values)\n", *val);
+ } else {
+ pr_err("FAIL: Slab was not initialized\n");
+ pr_expected_config_param(CONFIG_INIT_ON_ALLOC_DEFAULT_ON, "init_on_alloc");
+ }
kfree(val);
}
+void lkdtm_BUDDY_INIT_ON_ALLOC(void)
+{
+ u8 *first;
+ u8 *val;
+
+ first = (u8 *)__get_free_page(GFP_KERNEL);
+ if (!first) {
+ pr_info("Unable to allocate first free page\n");
+ return;
+ }
+
+ memset(first, 0xAB, PAGE_SIZE);
+ free_page((unsigned long)first);
+
+ val = (u8 *)__get_free_page(GFP_KERNEL);
+ if (!val) {
+ pr_info("Unable to allocate second free page\n");
+ return;
+ }
+
+ if (val != first) {
+ pr_warn("Reallocation missed clobbered memory.\n");
+ }
+
+ if (memchr(val, 0xAB, PAGE_SIZE) == NULL) {
+ pr_info("Memory appears initialized (%x, no earlier values)\n", *val);
+ } else {
+ pr_err("FAIL: Slab was not initialized\n");
+ pr_expected_config_param(CONFIG_INIT_ON_ALLOC_DEFAULT_ON, "init_on_alloc");
+ }
+ free_page((unsigned long)val);
+}
+
void lkdtm_SLAB_FREE_DOUBLE(void)
{
int *val;
diff --git a/drivers/misc/lkdtm/lkdtm.h b/drivers/misc/lkdtm/lkdtm.h
index 99f90d3e5e9c..6a30b60519f3 100644
--- a/drivers/misc/lkdtm/lkdtm.h
+++ b/drivers/misc/lkdtm/lkdtm.h
@@ -6,6 +6,47 @@
#include <linux/kernel.h>
+#define pr_expected_config(kconfig) \
+{ \
+ if (IS_ENABLED(kconfig)) \
+ pr_err("Unexpected! This kernel was built with " #kconfig "=y\n"); \
+ else \
+ pr_warn("This is probably expected, since this kernel was built *without* " #kconfig "=y\n"); \
+}
+
+#ifndef MODULE
+int lkdtm_check_bool_cmdline(const char *param);
+#define pr_expected_config_param(kconfig, param) \
+{ \
+ if (IS_ENABLED(kconfig)) { \
+ switch (lkdtm_check_bool_cmdline(param)) { \
+ case 0: \
+ pr_warn("This is probably expected, since this kernel was built with " #kconfig "=y but booted with '" param "=N'\n"); \
+ break; \
+ case 1: \
+ pr_err("Unexpected! This kernel was built with " #kconfig "=y and booted with '" param "=Y'\n"); \
+ break; \
+ default: \
+ pr_err("Unexpected! This kernel was built with " #kconfig "=y (and booted without '" param "' specified)\n"); \
+ } \
+ } else { \
+ switch (lkdtm_check_bool_cmdline(param)) { \
+ case 0: \
+ pr_warn("This is probably expected, as kernel was built *without* " #kconfig "=y and booted with '" param "=N'\n"); \
+ break; \
+ case 1: \
+ pr_err("Unexpected! This kernel was built *without* " #kconfig "=y but booted with '" param "=Y'\n"); \
+ break; \
+ default: \
+ pr_err("This is probably expected, since this kernel was built *without* " #kconfig "=y (and booted without '" param "' specified)\n"); \
+ break; \
+ } \
+ } \
+}
+#else
+#define pr_expected_config_param(kconfig, param) pr_expected_config(kconfig)
+#endif
+
/* bugs.c */
void __init lkdtm_bugs_init(int *recur_param);
void lkdtm_PANIC(void);
@@ -39,11 +80,14 @@ void lkdtm_FORTIFY_SUBOBJECT(void);
/* heap.c */
void __init lkdtm_heap_init(void);
void __exit lkdtm_heap_exit(void);
-void lkdtm_OVERWRITE_ALLOCATION(void);
+void lkdtm_VMALLOC_LINEAR_OVERFLOW(void);
+void lkdtm_SLAB_LINEAR_OVERFLOW(void);
void lkdtm_WRITE_AFTER_FREE(void);
void lkdtm_READ_AFTER_FREE(void);
void lkdtm_WRITE_BUDDY_AFTER_FREE(void);
void lkdtm_READ_BUDDY_AFTER_FREE(void);
+void lkdtm_SLAB_INIT_ON_ALLOC(void);
+void lkdtm_BUDDY_INIT_ON_ALLOC(void);
void lkdtm_SLAB_FREE_DOUBLE(void);
void lkdtm_SLAB_FREE_CROSS(void);
void lkdtm_SLAB_FREE_PAGE(void);
diff --git a/drivers/misc/lkdtm/stackleak.c b/drivers/misc/lkdtm/stackleak.c
index d1a5c0705be3..00db21ff115e 100644
--- a/drivers/misc/lkdtm/stackleak.c
+++ b/drivers/misc/lkdtm/stackleak.c
@@ -74,8 +74,8 @@ void lkdtm_STACKLEAK_ERASING(void)
end:
if (test_failed) {
- pr_err("FAIL: the thread stack is NOT properly erased\n");
- dump_stack();
+ pr_err("FAIL: the thread stack is NOT properly erased!\n");
+ pr_expected_config(CONFIG_GCC_PLUGIN_STACKLEAK);
} else {
pr_info("OK: the rest of the thread stack is properly erased\n");
}
diff --git a/drivers/misc/lkdtm/usercopy.c b/drivers/misc/lkdtm/usercopy.c
index 15d220ef35a5..9161ce7ed47a 100644
--- a/drivers/misc/lkdtm/usercopy.c
+++ b/drivers/misc/lkdtm/usercopy.c
@@ -173,6 +173,8 @@ static void do_usercopy_heap_size(bool to_user)
goto free_user;
}
}
+ pr_err("FAIL: bad usercopy not detected!\n");
+ pr_expected_config_param(CONFIG_HARDENED_USERCOPY, "hardened_usercopy");
free_user:
vm_munmap(user_addr, PAGE_SIZE);
@@ -248,6 +250,8 @@ static void do_usercopy_heap_whitelist(bool to_user)
goto free_user;
}
}
+ pr_err("FAIL: bad usercopy not detected!\n");
+ pr_expected_config_param(CONFIG_HARDENED_USERCOPY, "hardened_usercopy");
free_user:
vm_munmap(user_alloc, PAGE_SIZE);
@@ -319,7 +323,8 @@ void lkdtm_USERCOPY_KERNEL(void)
pr_warn("copy_to_user failed, but lacked Oops\n");
goto free_user;
}
- pr_err("FAIL: survived bad copy_to_user()\n");
+ pr_err("FAIL: bad copy_to_user() not detected!\n");
+ pr_expected_config_param(CONFIG_HARDENED_USERCOPY, "hardened_usercopy");
free_user:
vm_munmap(user_addr, PAGE_SIZE);
diff --git a/drivers/misc/mei/bus-fixup.c b/drivers/misc/mei/bus-fixup.c
index d8e760b11ae3..67844089db21 100644
--- a/drivers/misc/mei/bus-fixup.c
+++ b/drivers/misc/mei/bus-fixup.c
@@ -498,7 +498,7 @@ static struct mei_fixup {
};
/**
- * mei_cldev_fixup - run fixup handlers
+ * mei_cl_bus_dev_fixup - run fixup handlers
*
* @cldev: me client device
*/
diff --git a/drivers/misc/mei/client.c b/drivers/misc/mei/client.c
index 2cc370adb238..96f4e59c32a5 100644
--- a/drivers/misc/mei/client.c
+++ b/drivers/misc/mei/client.c
@@ -326,7 +326,7 @@ void mei_io_cb_free(struct mei_cl_cb *cb)
}
/**
- * mei_tx_cb_queue - queue tx callback
+ * mei_tx_cb_enqueue - queue tx callback
*
* Locking: called under "dev->device_lock" lock
*
@@ -1726,12 +1726,15 @@ nortpm:
return rets;
}
-static inline u8 mei_ext_hdr_set_vtag(struct mei_ext_hdr *ext, u8 vtag)
+static inline u8 mei_ext_hdr_set_vtag(void *ext, u8 vtag)
{
- ext->type = MEI_EXT_HDR_VTAG;
- ext->ext_payload[0] = vtag;
- ext->length = mei_data2slots(sizeof(*ext));
- return ext->length;
+ struct mei_ext_hdr_vtag *vtag_hdr = ext;
+
+ vtag_hdr->hdr.type = MEI_EXT_HDR_VTAG;
+ vtag_hdr->hdr.length = mei_data2slots(sizeof(*vtag_hdr));
+ vtag_hdr->vtag = vtag;
+ vtag_hdr->reserved = 0;
+ return vtag_hdr->hdr.length;
}
/**
@@ -1745,7 +1748,6 @@ static struct mei_msg_hdr *mei_msg_hdr_init(const struct mei_cl_cb *cb)
{
size_t hdr_len;
struct mei_ext_meta_hdr *meta;
- struct mei_ext_hdr *ext;
struct mei_msg_hdr *mei_hdr;
bool is_ext, is_vtag;
@@ -1764,7 +1766,7 @@ static struct mei_msg_hdr *mei_msg_hdr_init(const struct mei_cl_cb *cb)
hdr_len += sizeof(*meta);
if (is_vtag)
- hdr_len += sizeof(*ext);
+ hdr_len += sizeof(struct mei_ext_hdr_vtag);
setup_hdr:
mei_hdr = kzalloc(hdr_len, GFP_KERNEL);
@@ -2250,7 +2252,7 @@ static void mei_cl_dma_free(struct mei_cl *cl)
}
/**
- * mei_cl_alloc_and_map - send client dma map request
+ * mei_cl_dma_alloc_and_map - send client dma map request
*
* @cl: host client
* @fp: pointer to file structure
@@ -2349,7 +2351,7 @@ out:
}
/**
- * mei_cl_unmap_and_free - send client dma unmap request
+ * mei_cl_dma_unmap - send client dma unmap request
*
* @cl: host client
* @fp: pointer to file structure
diff --git a/drivers/misc/mei/hbm.c b/drivers/misc/mei/hbm.c
index d0277c7fed10..99b5c1ecc444 100644
--- a/drivers/misc/mei/hbm.c
+++ b/drivers/misc/mei/hbm.c
@@ -853,7 +853,7 @@ out:
}
/**
- * mei_hbm_cl_flow_control_res - flow control response from me
+ * mei_hbm_cl_tx_flow_ctrl_creds_res - flow control response from me
*
* @dev: the device structure
* @fctrl: flow control response bus message
diff --git a/drivers/misc/mei/hdcp/Kconfig b/drivers/misc/mei/hdcp/Kconfig
index 95b2d6d37f10..54e1c9526909 100644
--- a/drivers/misc/mei/hdcp/Kconfig
+++ b/drivers/misc/mei/hdcp/Kconfig
@@ -1,4 +1,3 @@
-
# SPDX-License-Identifier: GPL-2.0
# Copyright (c) 2019, Intel Corporation. All rights reserved.
#
diff --git a/drivers/misc/mei/hw-me.c b/drivers/misc/mei/hw-me.c
index cda0829ac589..d3a6c0728645 100644
--- a/drivers/misc/mei/hw-me.c
+++ b/drivers/misc/mei/hw-me.c
@@ -1380,7 +1380,7 @@ static bool mei_me_fw_type_nm(const struct pci_dev *pdev)
.quirk_probe = mei_me_fw_type_nm
/**
- * mei_me_fw_sku_sps_4() - check for sps 4.0 sku
+ * mei_me_fw_type_sps_4() - check for sps 4.0 sku
*
* Read ME FW Status register to check for SPS Firmware.
* The SPS FW is only signaled in the PCI function 0.
@@ -1405,7 +1405,7 @@ static bool mei_me_fw_type_sps_4(const struct pci_dev *pdev)
.quirk_probe = mei_me_fw_type_sps_4
/**
- * mei_me_fw_sku_sps() - check for sps sku
+ * mei_me_fw_type_sps() - check for sps sku
*
* Read ME FW Status register to check for SPS Firmware.
* The SPS FW is only signaled in pci function 0
diff --git a/drivers/misc/mei/hw.h b/drivers/misc/mei/hw.h
index b10606550613..dfd60c916da0 100644
--- a/drivers/misc/mei/hw.h
+++ b/drivers/misc/mei/hw.h
@@ -235,9 +235,8 @@ enum mei_ext_hdr_type {
struct mei_ext_hdr {
u8 type;
u8 length;
- u8 ext_payload[2];
- u8 hdr[];
-};
+ u8 data[];
+} __packed;
/**
* struct mei_ext_meta_hdr - extend header meta data
@@ -250,8 +249,21 @@ struct mei_ext_meta_hdr {
u8 count;
u8 size;
u8 reserved[2];
- struct mei_ext_hdr hdrs[];
-};
+ u8 hdrs[];
+} __packed;
+
+/**
+ * struct mei_ext_hdr_vtag - extend header for vtag
+ *
+ * @hdr: standard extend header
+ * @vtag: virtual tag
+ * @reserved: reserved
+ */
+struct mei_ext_hdr_vtag {
+ struct mei_ext_hdr hdr;
+ u8 vtag;
+ u8 reserved;
+} __packed;
/*
* Extended header iterator functions
@@ -266,7 +278,7 @@ struct mei_ext_meta_hdr {
*/
static inline struct mei_ext_hdr *mei_ext_begin(struct mei_ext_meta_hdr *meta)
{
- return meta->hdrs;
+ return (struct mei_ext_hdr *)meta->hdrs;
}
/**
@@ -284,7 +296,7 @@ static inline bool mei_ext_last(struct mei_ext_meta_hdr *meta,
}
/**
- *mei_ext_next - following extended header on the TLV list
+ * mei_ext_next - following extended header on the TLV list
*
* @ext: current extend header
*
@@ -295,7 +307,7 @@ static inline bool mei_ext_last(struct mei_ext_meta_hdr *meta,
*/
static inline struct mei_ext_hdr *mei_ext_next(struct mei_ext_hdr *ext)
{
- return (struct mei_ext_hdr *)(ext->hdr + (ext->length * 4));
+ return (struct mei_ext_hdr *)((u8 *)ext + (ext->length * 4));
}
/**
diff --git a/drivers/misc/mei/interrupt.c b/drivers/misc/mei/interrupt.c
index aab3ebfa9fc4..a67f4f2d33a9 100644
--- a/drivers/misc/mei/interrupt.c
+++ b/drivers/misc/mei/interrupt.c
@@ -123,13 +123,13 @@ static int mei_cl_irq_read_msg(struct mei_cl *cl,
if (mei_hdr->extended) {
struct mei_ext_hdr *ext;
- struct mei_ext_hdr *vtag = NULL;
+ struct mei_ext_hdr_vtag *vtag_hdr = NULL;
ext = mei_ext_begin(meta);
do {
switch (ext->type) {
case MEI_EXT_HDR_VTAG:
- vtag = ext;
+ vtag_hdr = (struct mei_ext_hdr_vtag *)ext;
break;
case MEI_EXT_HDR_NONE:
fallthrough;
@@ -141,20 +141,20 @@ static int mei_cl_irq_read_msg(struct mei_cl *cl,
ext = mei_ext_next(ext);
} while (!mei_ext_last(meta, ext));
- if (!vtag) {
+ if (!vtag_hdr) {
cl_dbg(dev, cl, "vtag not found in extended header.\n");
cb->status = -EPROTO;
goto discard;
}
- cl_dbg(dev, cl, "vtag: %d\n", vtag->ext_payload[0]);
- if (cb->vtag && cb->vtag != vtag->ext_payload[0]) {
+ cl_dbg(dev, cl, "vtag: %d\n", vtag_hdr->vtag);
+ if (cb->vtag && cb->vtag != vtag_hdr->vtag) {
cl_err(dev, cl, "mismatched tag: %d != %d\n",
- cb->vtag, vtag->ext_payload[0]);
+ cb->vtag, vtag_hdr->vtag);
cb->status = -EPROTO;
goto discard;
}
- cb->vtag = vtag->ext_payload[0];
+ cb->vtag = vtag_hdr->vtag;
}
if (!mei_cl_is_connected(cl)) {
@@ -331,7 +331,6 @@ int mei_irq_read_handler(struct mei_device *dev,
struct mei_ext_meta_hdr *meta_hdr = NULL;
struct mei_cl *cl;
int ret;
- u32 ext_meta_hdr_u32;
u32 hdr_size_left;
u32 hdr_size_ext;
int i;
@@ -367,14 +366,12 @@ int mei_irq_read_handler(struct mei_device *dev,
if (mei_hdr->extended) {
if (!dev->rd_msg_hdr[1]) {
- ext_meta_hdr_u32 = mei_read_hdr(dev);
- dev->rd_msg_hdr[1] = ext_meta_hdr_u32;
+ dev->rd_msg_hdr[1] = mei_read_hdr(dev);
dev->rd_msg_hdr_count++;
(*slots)--;
- dev_dbg(dev->dev, "extended header is %08x\n",
- ext_meta_hdr_u32);
+ dev_dbg(dev->dev, "extended header is %08x\n", dev->rd_msg_hdr[1]);
}
- meta_hdr = ((struct mei_ext_meta_hdr *)dev->rd_msg_hdr + 1);
+ meta_hdr = ((struct mei_ext_meta_hdr *)&dev->rd_msg_hdr[1]);
if (check_add_overflow((u32)sizeof(*meta_hdr),
mei_slots2data(meta_hdr->size),
&hdr_size_ext)) {
diff --git a/drivers/misc/mei/main.c b/drivers/misc/mei/main.c
index 28937b6e7e0c..786f7c8f7f61 100644
--- a/drivers/misc/mei/main.c
+++ b/drivers/misc/mei/main.c
@@ -50,8 +50,6 @@ static int mei_open(struct inode *inode, struct file *file)
int err;
dev = container_of(inode->i_cdev, struct mei_device, cdev);
- if (!dev)
- return -ENODEV;
mutex_lock(&dev->device_lock);
@@ -1104,7 +1102,7 @@ static ssize_t dev_state_show(struct device *device,
static DEVICE_ATTR_RO(dev_state);
/**
- * dev_set_devstate: set to new device state and notify sysfs file.
+ * mei_set_devstate: set to new device state and notify sysfs file.
*
* @dev: mei_device
* @state: new device state
diff --git a/drivers/misc/mei/pci-txe.c b/drivers/misc/mei/pci-txe.c
index 4bf26ce61044..aec0483b8e72 100644
--- a/drivers/misc/mei/pci-txe.c
+++ b/drivers/misc/mei/pci-txe.c
@@ -156,7 +156,7 @@ end:
}
/**
- * mei_txe_remove - Device Shutdown Routine
+ * mei_txe_shutdown- Device Shutdown Routine
*
* @pdev: PCI device structure
*
diff --git a/drivers/misc/pvpanic/pvpanic-mmio.c b/drivers/misc/pvpanic/pvpanic-mmio.c
index 4c0841776087..be4016084979 100644
--- a/drivers/misc/pvpanic/pvpanic-mmio.c
+++ b/drivers/misc/pvpanic/pvpanic-mmio.c
@@ -93,7 +93,7 @@ static int pvpanic_mmio_probe(struct platform_device *pdev)
return -EINVAL;
}
- pi = kmalloc(sizeof(*pi), GFP_ATOMIC);
+ pi = devm_kmalloc(dev, sizeof(*pi), GFP_KERNEL);
if (!pi)
return -ENOMEM;
@@ -104,19 +104,7 @@ static int pvpanic_mmio_probe(struct platform_device *pdev)
pi->capability &= ioread8(base);
pi->events = pi->capability;
- dev_set_drvdata(dev, pi);
-
- return pvpanic_probe(pi);
-}
-
-static int pvpanic_mmio_remove(struct platform_device *pdev)
-{
- struct pvpanic_instance *pi = dev_get_drvdata(&pdev->dev);
-
- pvpanic_remove(pi);
- kfree(pi);
-
- return 0;
+ return devm_pvpanic_probe(dev, pi);
}
static const struct of_device_id pvpanic_mmio_match[] = {
@@ -139,6 +127,5 @@ static struct platform_driver pvpanic_mmio_driver = {
.dev_groups = pvpanic_mmio_dev_groups,
},
.probe = pvpanic_mmio_probe,
- .remove = pvpanic_mmio_remove,
};
module_platform_driver(pvpanic_mmio_driver);
diff --git a/drivers/misc/pvpanic/pvpanic-pci.c b/drivers/misc/pvpanic/pvpanic-pci.c
index 9ecc4e8559d5..a43c401017ae 100644
--- a/drivers/misc/pvpanic/pvpanic-pci.c
+++ b/drivers/misc/pvpanic/pvpanic-pci.c
@@ -73,20 +73,19 @@ ATTRIBUTE_GROUPS(pvpanic_pci_dev);
static int pvpanic_pci_probe(struct pci_dev *pdev,
const struct pci_device_id *ent)
{
- struct device *dev = &pdev->dev;
struct pvpanic_instance *pi;
void __iomem *base;
int ret;
- ret = pci_enable_device(pdev);
+ ret = pcim_enable_device(pdev);
if (ret < 0)
return ret;
- base = pci_iomap(pdev, 0, 0);
+ base = pcim_iomap(pdev, 0, 0);
if (!base)
return -ENOMEM;
- pi = kmalloc(sizeof(*pi), GFP_ATOMIC);
+ pi = devm_kmalloc(&pdev->dev, sizeof(*pi), GFP_KERNEL);
if (!pi)
return -ENOMEM;
@@ -97,26 +96,13 @@ static int pvpanic_pci_probe(struct pci_dev *pdev,
pi->capability &= ioread8(base);
pi->events = pi->capability;
- dev_set_drvdata(dev, pi);
-
- return pvpanic_probe(pi);
-}
-
-static void pvpanic_pci_remove(struct pci_dev *pdev)
-{
- struct pvpanic_instance *pi = dev_get_drvdata(&pdev->dev);
-
- pvpanic_remove(pi);
- iounmap(pi->base);
- kfree(pi);
- pci_disable_device(pdev);
+ return devm_pvpanic_probe(&pdev->dev, pi);
}
static struct pci_driver pvpanic_pci_driver = {
.name = "pvpanic-pci",
.id_table = pvpanic_pci_id_tbl,
.probe = pvpanic_pci_probe,
- .remove = pvpanic_pci_remove,
.driver = {
.dev_groups = pvpanic_pci_dev_groups,
},
diff --git a/drivers/misc/pvpanic/pvpanic.c b/drivers/misc/pvpanic/pvpanic.c
index 793ea0c01193..02b807c788c9 100644
--- a/drivers/misc/pvpanic/pvpanic.c
+++ b/drivers/misc/pvpanic/pvpanic.c
@@ -61,25 +61,10 @@ static struct notifier_block pvpanic_panic_nb = {
.priority = 1, /* let this called before broken drm_fb_helper */
};
-int pvpanic_probe(struct pvpanic_instance *pi)
-{
- if (!pi || !pi->base)
- return -EINVAL;
-
- spin_lock(&pvpanic_lock);
- list_add(&pi->list, &pvpanic_list);
- spin_unlock(&pvpanic_lock);
-
- return 0;
-}
-EXPORT_SYMBOL_GPL(pvpanic_probe);
-
-void pvpanic_remove(struct pvpanic_instance *pi)
+static void pvpanic_remove(void *param)
{
struct pvpanic_instance *pi_cur, *pi_next;
-
- if (!pi)
- return;
+ struct pvpanic_instance *pi = param;
spin_lock(&pvpanic_lock);
list_for_each_entry_safe(pi_cur, pi_next, &pvpanic_list, list) {
@@ -90,7 +75,19 @@ void pvpanic_remove(struct pvpanic_instance *pi)
}
spin_unlock(&pvpanic_lock);
}
-EXPORT_SYMBOL_GPL(pvpanic_remove);
+
+int devm_pvpanic_probe(struct device *dev, struct pvpanic_instance *pi)
+{
+ if (!pi || !pi->base)
+ return -EINVAL;
+
+ spin_lock(&pvpanic_lock);
+ list_add(&pi->list, &pvpanic_list);
+ spin_unlock(&pvpanic_lock);
+
+ return devm_add_action_or_reset(dev, pvpanic_remove, pi);
+}
+EXPORT_SYMBOL_GPL(devm_pvpanic_probe);
static int pvpanic_init(void)
{
diff --git a/drivers/misc/pvpanic/pvpanic.h b/drivers/misc/pvpanic/pvpanic.h
index 1afccc2e9fec..493545951754 100644
--- a/drivers/misc/pvpanic/pvpanic.h
+++ b/drivers/misc/pvpanic/pvpanic.h
@@ -15,7 +15,6 @@ struct pvpanic_instance {
struct list_head list;
};
-int pvpanic_probe(struct pvpanic_instance *pi);
-void pvpanic_remove(struct pvpanic_instance *pi);
+int devm_pvpanic_probe(struct device *dev, struct pvpanic_instance *pi);
#endif /* PVPANIC_H_ */
diff --git a/drivers/misc/uacce/uacce.c b/drivers/misc/uacce/uacce.c
index bae18ef03dcb..488eeb2811ae 100644
--- a/drivers/misc/uacce/uacce.c
+++ b/drivers/misc/uacce/uacce.c
@@ -387,15 +387,22 @@ static void uacce_release(struct device *dev)
static unsigned int uacce_enable_sva(struct device *parent, unsigned int flags)
{
+ int ret;
+
if (!(flags & UACCE_DEV_SVA))
return flags;
flags &= ~UACCE_DEV_SVA;
- if (iommu_dev_enable_feature(parent, IOMMU_DEV_FEAT_IOPF))
+ ret = iommu_dev_enable_feature(parent, IOMMU_DEV_FEAT_IOPF);
+ if (ret) {
+ dev_err(parent, "failed to enable IOPF feature! ret = %pe\n", ERR_PTR(ret));
return flags;
+ }
- if (iommu_dev_enable_feature(parent, IOMMU_DEV_FEAT_SVA)) {
+ ret = iommu_dev_enable_feature(parent, IOMMU_DEV_FEAT_SVA);
+ if (ret) {
+ dev_err(parent, "failed to enable SVA feature! ret = %pe\n", ERR_PTR(ret));
iommu_dev_disable_feature(parent, IOMMU_DEV_FEAT_IOPF);
return flags;
}
diff --git a/drivers/misc/vmw_vmci/vmci_context.c b/drivers/misc/vmw_vmci/vmci_context.c
index 26ff49fdf0f7..c0b5e339d5a1 100644
--- a/drivers/misc/vmw_vmci/vmci_context.c
+++ b/drivers/misc/vmw_vmci/vmci_context.c
@@ -107,7 +107,7 @@ struct vmci_ctx *vmci_ctx_create(u32 cid, u32 priv_flags,
context = kzalloc(sizeof(*context), GFP_KERNEL);
if (!context) {
pr_warn("Failed to allocate memory for VMCI context\n");
- error = -EINVAL;
+ error = -ENOMEM;
goto err_out;
}
diff --git a/drivers/misc/xilinx_sdfec.c b/drivers/misc/xilinx_sdfec.c
index 23c8448a9c3b..d6e3c650bd11 100644
--- a/drivers/misc/xilinx_sdfec.c
+++ b/drivers/misc/xilinx_sdfec.c
@@ -1013,9 +1013,6 @@ static __poll_t xsdfec_poll(struct file *file, poll_table *wait)
xsdfec = container_of(file->private_data, struct xsdfec_dev, miscdev);
- if (!xsdfec)
- return EPOLLNVAL | EPOLLHUP;
-
poll_wait(file, &xsdfec->waitq, wait);
/* XSDFEC ISR detected an error */