403 lines
11 KiB
C
403 lines
11 KiB
C
/*
|
|
* Copyright (c) 2019, Intel Corporation. All rights reserved.
|
|
*
|
|
* SPDX-License-Identifier: BSD-3-Clause
|
|
*/
|
|
|
|
#include <errno.h>
|
|
#include <lib/mmio.h>
|
|
#include <lib/utils.h>
|
|
#include <common/debug.h>
|
|
#include <drivers/delay_timer.h>
|
|
#include <platform_def.h>
|
|
|
|
#include "agilex_memory_controller.h"
|
|
|
|
#define ALT_CCU_NOC_DI_SET_MSK 0x10
|
|
|
|
#define DDR_READ_LATENCY_DELAY 40
|
|
#define MAX_MEM_CAL_RETRY 3
|
|
#define PRE_CALIBRATION_DELAY 1
|
|
#define POST_CALIBRATION_DELAY 1
|
|
#define TIMEOUT_EMIF_CALIBRATION 1000
|
|
#define CLEAR_EMIF_DELAY 50000
|
|
#define CLEAR_EMIF_TIMEOUT 0x100000
|
|
#define TIMEOUT_INT_RESP 10000
|
|
|
|
#define DDR_CONFIG(A, B, C, R) (((A) << 24) | ((B) << 16) | ((C) << 8) | (R))
|
|
#define DDR_CONFIG_ELEMENTS (sizeof(ddr_config)/sizeof(uint32_t))
|
|
|
|
/* tWR = Min. 15ns constant, see JEDEC standard eg. DDR4 is JESD79-4.pdf */
|
|
#define tWR_IN_NS 15
|
|
|
|
void configure_hmc_adaptor_regs(void);
|
|
void configure_ddr_sched_ctrl_regs(void);
|
|
|
|
/* The followring are the supported configurations */
|
|
uint32_t ddr_config[] = {
|
|
/* DDR_CONFIG(Address order,Bank,Column,Row) */
|
|
/* List for DDR3 or LPDDR3 (pinout order > chip, row, bank, column) */
|
|
DDR_CONFIG(0, 3, 10, 12),
|
|
DDR_CONFIG(0, 3, 9, 13),
|
|
DDR_CONFIG(0, 3, 10, 13),
|
|
DDR_CONFIG(0, 3, 9, 14),
|
|
DDR_CONFIG(0, 3, 10, 14),
|
|
DDR_CONFIG(0, 3, 10, 15),
|
|
DDR_CONFIG(0, 3, 11, 14),
|
|
DDR_CONFIG(0, 3, 11, 15),
|
|
DDR_CONFIG(0, 3, 10, 16),
|
|
DDR_CONFIG(0, 3, 11, 16),
|
|
DDR_CONFIG(0, 3, 12, 15), /* 0xa */
|
|
/* List for DDR4 only (pinout order > chip, bank, row, column) */
|
|
DDR_CONFIG(1, 3, 10, 14),
|
|
DDR_CONFIG(1, 4, 10, 14),
|
|
DDR_CONFIG(1, 3, 10, 15),
|
|
DDR_CONFIG(1, 4, 10, 15),
|
|
DDR_CONFIG(1, 3, 10, 16),
|
|
DDR_CONFIG(1, 4, 10, 16),
|
|
DDR_CONFIG(1, 3, 10, 17),
|
|
DDR_CONFIG(1, 4, 10, 17),
|
|
};
|
|
|
|
static int match_ddr_conf(uint32_t ddr_conf)
|
|
{
|
|
int i;
|
|
|
|
for (i = 0; i < DDR_CONFIG_ELEMENTS; i++) {
|
|
if (ddr_conf == ddr_config[i])
|
|
return i;
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
static int check_hmc_clk(void)
|
|
{
|
|
unsigned long timeout = 0;
|
|
uint32_t hmc_clk;
|
|
|
|
do {
|
|
hmc_clk = mmio_read_32(AGX_SYSMGR_CORE_HMC_CLK);
|
|
if (hmc_clk & AGX_SYSMGR_CORE_HMC_CLK_STATUS)
|
|
break;
|
|
udelay(1);
|
|
} while (++timeout < 1000);
|
|
if (timeout >= 1000)
|
|
return -ETIMEDOUT;
|
|
|
|
return 0;
|
|
}
|
|
|
|
static int clear_emif(void)
|
|
{
|
|
uint32_t data;
|
|
unsigned long timeout;
|
|
|
|
mmio_write_32(AGX_MPFE_HMC_ADP_RSTHANDSHAKECTRL, 0);
|
|
|
|
timeout = 0;
|
|
do {
|
|
data = mmio_read_32(AGX_MPFE_HMC_ADP_RSTHANDSHAKESTAT);
|
|
if ((data & AGX_MPFE_HMC_ADP_RSTHANDSHAKESTAT_SEQ2CORE) == 0)
|
|
break;
|
|
udelay(CLEAR_EMIF_DELAY);
|
|
} while (++timeout < CLEAR_EMIF_TIMEOUT);
|
|
if (timeout >= CLEAR_EMIF_TIMEOUT)
|
|
return -ETIMEDOUT;
|
|
|
|
return 0;
|
|
}
|
|
|
|
static int mem_calibration(void)
|
|
{
|
|
int status;
|
|
uint32_t data;
|
|
unsigned long timeout;
|
|
unsigned long retry = 0;
|
|
|
|
udelay(PRE_CALIBRATION_DELAY);
|
|
|
|
do {
|
|
if (retry != 0)
|
|
INFO("DDR: Retrying DRAM calibration\n");
|
|
|
|
timeout = 0;
|
|
do {
|
|
data = mmio_read_32(AGX_MPFE_HMC_ADP_DDRCALSTAT);
|
|
if (AGX_MPFE_HMC_ADP_DDRCALSTAT_CAL(data) == 1)
|
|
break;
|
|
mdelay(1);
|
|
} while (++timeout < TIMEOUT_EMIF_CALIBRATION);
|
|
|
|
if (AGX_MPFE_HMC_ADP_DDRCALSTAT_CAL(data) == 0) {
|
|
status = clear_emif();
|
|
if (status)
|
|
ERROR("Failed to clear Emif\n");
|
|
} else {
|
|
break;
|
|
}
|
|
} while (++retry < MAX_MEM_CAL_RETRY);
|
|
|
|
if (AGX_MPFE_HMC_ADP_DDRCALSTAT_CAL(data) == 0) {
|
|
ERROR("DDR: DRAM calibration failed.\n");
|
|
status = -EIO;
|
|
} else {
|
|
INFO("DDR: DRAM calibration success.\n");
|
|
status = 0;
|
|
}
|
|
|
|
udelay(POST_CALIBRATION_DELAY);
|
|
|
|
return status;
|
|
}
|
|
|
|
int init_hard_memory_controller(void)
|
|
{
|
|
int status;
|
|
|
|
status = check_hmc_clk();
|
|
if (status) {
|
|
ERROR("DDR: Error, HMC clock not running\n");
|
|
return status;
|
|
}
|
|
|
|
/* mmio_clrbits_32(AGX_RSTMGR_BRGMODRST, AGX_RSTMGR_BRGMODRST_DDRSCH);*/
|
|
|
|
status = mem_calibration();
|
|
if (status) {
|
|
ERROR("DDR: Memory Calibration Failed\n");
|
|
return status;
|
|
}
|
|
|
|
configure_hmc_adaptor_regs();
|
|
/* configure_ddr_sched_ctrl_regs();*/
|
|
|
|
return 0;
|
|
}
|
|
|
|
void configure_ddr_sched_ctrl_regs(void)
|
|
{
|
|
uint32_t data, dram_addr_order, ddr_conf, bank, row, col,
|
|
rd_to_miss, wr_to_miss, burst_len, burst_len_ddr_clk,
|
|
burst_len_sched_clk, act_to_act, rd_to_wr, wr_to_rd, bw_ratio,
|
|
t_rtp, t_rp, t_rcd, rd_latency, tw_rin_clk_cycles,
|
|
bw_ratio_extended, auto_precharge = 0, act_to_act_bank, faw,
|
|
faw_bank, bus_rd_to_rd, bus_rd_to_wr, bus_wr_to_rd;
|
|
|
|
INFO("Init HPS NOC's DDR Scheduler.\n");
|
|
|
|
data = mmio_read_32(AGX_MPFE_IOHMC_CTRLCFG1);
|
|
dram_addr_order = AGX_MPFE_IOHMC_CTRLCFG1_CFG_ADDR_ORDER(data);
|
|
|
|
data = mmio_read_32(AGX_MPFE_IOHMC_DRAMADDRW);
|
|
|
|
col = IOHMC_DRAMADDRW_COL_ADDR_WIDTH(data);
|
|
row = IOHMC_DRAMADDRW_ROW_ADDR_WIDTH(data);
|
|
bank = IOHMC_DRAMADDRW_BANK_ADDR_WIDTH(data) +
|
|
IOHMC_DRAMADDRW_BANK_GRP_ADDR_WIDTH(data);
|
|
|
|
ddr_conf = match_ddr_conf(DDR_CONFIG(dram_addr_order, bank, col, row));
|
|
|
|
if (ddr_conf) {
|
|
mmio_clrsetbits_32(
|
|
AGX_MPFE_DDR_MAIN_SCHED_DDRCONF,
|
|
AGX_MPFE_DDR_MAIN_SCHED_DDRCONF_SET_MSK,
|
|
AGX_MPFE_DDR_MAIN_SCHED_DDRCONF_SET(ddr_conf));
|
|
} else {
|
|
ERROR("DDR: Cannot find predefined ddrConf configuration.\n");
|
|
}
|
|
|
|
mmio_write_32(AGX_MPFE_HMC_ADP(ADP_DRAMADDRWIDTH), data);
|
|
|
|
data = mmio_read_32(AGX_MPFE_IOHMC_DRAMTIMING0);
|
|
rd_latency = AGX_MPFE_IOHMC_REG_DRAMTIMING0_CFG_TCL(data);
|
|
|
|
data = mmio_read_32(AGX_MPFE_IOHMC_CALTIMING0);
|
|
act_to_act = ACT_TO_ACT(data);
|
|
t_rcd = ACT_TO_RDWR(data);
|
|
act_to_act_bank = ACT_TO_ACT_DIFF_BANK(data);
|
|
|
|
data = mmio_read_32(AGX_MPFE_IOHMC_CALTIMING1);
|
|
rd_to_wr = RD_TO_WR(data);
|
|
bus_rd_to_rd = RD_TO_RD_DIFF_CHIP(data);
|
|
bus_rd_to_wr = RD_TO_WR_DIFF_CHIP(data);
|
|
|
|
data = mmio_read_32(AGX_MPFE_IOHMC_CALTIMING2);
|
|
t_rtp = RD_TO_PCH(data);
|
|
|
|
data = mmio_read_32(AGX_MPFE_IOHMC_CALTIMING3);
|
|
wr_to_rd = CALTIMING3_WR_TO_RD(data);
|
|
bus_wr_to_rd = CALTIMING3_WR_TO_RD_DIFF_CHIP(data);
|
|
|
|
data = mmio_read_32(AGX_MPFE_IOHMC_CALTIMING4);
|
|
t_rp = PCH_TO_VALID(data);
|
|
|
|
data = mmio_read_32(AGX_MPFE_HMC_ADP(HMC_ADP_DDRIOCTRL));
|
|
bw_ratio = ((HMC_ADP_DDRIOCTRL_IO_SIZE(data) == 0) ? 0 : 1);
|
|
|
|
data = mmio_read_32(AGX_MPFE_IOHMC_CTRLCFG0);
|
|
burst_len = HMC_ADP_DDRIOCTRL_CTRL_BURST_LENGTH(data);
|
|
burst_len_ddr_clk = burst_len / 2;
|
|
burst_len_sched_clk = ((burst_len/2) / 2);
|
|
|
|
data = mmio_read_32(AGX_MPFE_IOHMC_CTRLCFG0);
|
|
switch (AGX_MPFE_IOHMC_REG_CTRLCFG0_CFG_MEM_TYPE(data)) {
|
|
case 1:
|
|
/* DDR4 - 1333MHz */
|
|
/* 20 (19.995) clock cycles = 15ns */
|
|
/* Calculate with rounding */
|
|
tw_rin_clk_cycles = (((tWR_IN_NS * 1333) % 1000) >= 500) ?
|
|
((tWR_IN_NS * 1333) / 1000) + 1 :
|
|
((tWR_IN_NS * 1333) / 1000);
|
|
break;
|
|
default:
|
|
/* Others - 1066MHz or slower */
|
|
/* 16 (15.990) clock cycles = 15ns */
|
|
/* Calculate with rounding */
|
|
tw_rin_clk_cycles = (((tWR_IN_NS * 1066) % 1000) >= 500) ?
|
|
((tWR_IN_NS * 1066) / 1000) + 1 :
|
|
((tWR_IN_NS * 1066) / 1000);
|
|
break;
|
|
}
|
|
|
|
rd_to_miss = t_rtp + t_rp + t_rcd - burst_len_sched_clk;
|
|
wr_to_miss = ((rd_latency + burst_len_ddr_clk + 2 + tw_rin_clk_cycles)
|
|
/ 2) - rd_to_wr + t_rp + t_rcd;
|
|
|
|
mmio_write_32(AGX_MPFE_DDR_MAIN_SCHED_DDRTIMING,
|
|
bw_ratio << DDRTIMING_BWRATIO_OFST |
|
|
wr_to_rd << DDRTIMING_WRTORD_OFST|
|
|
rd_to_wr << DDRTIMING_RDTOWR_OFST |
|
|
burst_len_sched_clk << DDRTIMING_BURSTLEN_OFST |
|
|
wr_to_miss << DDRTIMING_WRTOMISS_OFST |
|
|
rd_to_miss << DDRTIMING_RDTOMISS_OFST |
|
|
act_to_act << DDRTIMING_ACTTOACT_OFST);
|
|
|
|
data = mmio_read_32(AGX_MPFE_HMC_ADP(HMC_ADP_DDRIOCTRL));
|
|
bw_ratio_extended = ((ADP_DDRIOCTRL_IO_SIZE(data) == 0) ? 1 : 0);
|
|
|
|
mmio_write_32(AGX_MPFE_DDR_MAIN_SCHED_DDRMODE,
|
|
bw_ratio_extended << DDRMODE_BWRATIOEXTENDED_OFST |
|
|
auto_precharge << DDRMODE_AUTOPRECHARGE_OFST);
|
|
|
|
mmio_write_32(AGX_MPFE_DDR_MAIN_SCHED_READLATENCY,
|
|
(rd_latency / 2) + DDR_READ_LATENCY_DELAY);
|
|
|
|
data = mmio_read_32(AGX_MPFE_IOHMC_CALTIMING9);
|
|
faw = AGX_MPFE_IOHMC_CALTIMING9_ACT_TO_ACT(data);
|
|
|
|
faw_bank = 1; // always 1 because we always have 4 bank DDR.
|
|
|
|
mmio_write_32(AGX_MPFE_DDR_MAIN_SCHED_ACTIVATE,
|
|
faw_bank << AGX_MPFE_DDR_MAIN_SCHED_ACTIVATE_FAWBANK_OFST |
|
|
faw << AGX_MPFE_DDR_MAIN_SCHED_ACTIVATE_FAW_OFST |
|
|
act_to_act_bank << AGX_MPFE_DDR_MAIN_SCHED_ACTIVATE_RRD_OFST);
|
|
|
|
mmio_write_32(AGX_MPFE_DDR_MAIN_SCHED_DEVTODEV,
|
|
((bus_rd_to_rd
|
|
<< AGX_MPFE_DDR_MAIN_SCHED_DEVTODEV_BUSRDTORD_OFST)
|
|
& AGX_MPFE_DDR_MAIN_SCHED_DEVTODEV_BUSRDTORD_MSK) |
|
|
((bus_rd_to_wr
|
|
<< AGX_MPFE_DDR_MAIN_SCHED_DEVTODEV_BUSRDTOWR_OFST)
|
|
& AGX_MPFE_DDR_MAIN_SCHED_DEVTODEV_BUSRDTOWR_MSK) |
|
|
((bus_wr_to_rd
|
|
<< AGX_MPFE_DDR_MAIN_SCHED_DEVTODEV_BUSWRTORD_OFST)
|
|
& AGX_MPFE_DDR_MAIN_SCHED_DEVTODEV_BUSWRTORD_MSK));
|
|
|
|
}
|
|
|
|
unsigned long get_physical_dram_size(void)
|
|
{
|
|
uint32_t data;
|
|
unsigned long ram_addr_width, ram_ext_if_io_width;
|
|
|
|
data = mmio_read_32(AGX_MPFE_HMC_ADP_DDRIOCTRL);
|
|
switch (AGX_MPFE_HMC_ADP_DDRIOCTRL_IO_SIZE(data)) {
|
|
case 0:
|
|
ram_ext_if_io_width = 16;
|
|
break;
|
|
case 1:
|
|
ram_ext_if_io_width = 32;
|
|
break;
|
|
case 2:
|
|
ram_ext_if_io_width = 64;
|
|
break;
|
|
default:
|
|
ram_ext_if_io_width = 0;
|
|
break;
|
|
}
|
|
|
|
data = mmio_read_32(AGX_MPFE_IOHMC_REG_DRAMADDRW);
|
|
ram_addr_width = IOHMC_DRAMADDRW_CFG_COL_ADDR_WIDTH(data) +
|
|
IOHMC_DRAMADDRW_CFG_ROW_ADDR_WIDTH(data) +
|
|
IOHMC_DRAMADDRW_CFG_BANK_ADDR_WIDTH(data) +
|
|
IOHMC_DRAMADDRW_CFG_BANK_GROUP_ADDR_WIDTH(data) +
|
|
IOHMC_DRAMADDRW_CFG_CS_ADDR_WIDTH(data);
|
|
|
|
return (1 << ram_addr_width) * (ram_ext_if_io_width / 8);
|
|
}
|
|
|
|
|
|
|
|
void configure_hmc_adaptor_regs(void)
|
|
{
|
|
uint32_t data;
|
|
uint32_t dram_io_width;
|
|
|
|
/* Configure DDR data rate */
|
|
dram_io_width = AGX_MPFE_IOHMC_NIOSRESERVE0_NIOS_RESERVE0(
|
|
mmio_read_32(AGX_MPFE_IOHMC_REG_NIOSRESERVE0_OFST));
|
|
dram_io_width = (dram_io_width & 0xFF) >> 5;
|
|
|
|
data = mmio_read_32(AGX_MPFE_IOHMC_CTRLCFG3);
|
|
|
|
dram_io_width |= (data & 0x4);
|
|
|
|
mmio_write_32(AGX_MPFE_HMC_ADP_DDRIOCTRL, dram_io_width);
|
|
|
|
/* Copy dram addr width from IOHMC to HMC ADP */
|
|
data = mmio_read_32(AGX_MPFE_IOHMC_DRAMADDRW);
|
|
mmio_write_32(AGX_MPFE_HMC_ADP(ADP_DRAMADDRWIDTH), data);
|
|
|
|
/* Enable nonsecure access to DDR */
|
|
mmio_write_32(AGX_NOC_FW_DDR_SCR_MPUREGION0ADDR_LIMIT,
|
|
AGX_DDR_SIZE - 1);
|
|
mmio_write_32(AGX_NOC_FW_DDR_SCR_MPUREGION0ADDR_LIMITEXT,
|
|
0x1f);
|
|
|
|
mmio_write_32(AGX_NOC_FW_DDR_SCR_NONMPUREGION0ADDR_LIMIT,
|
|
AGX_DDR_SIZE - 1);
|
|
|
|
mmio_write_32(AGX_SOC_NOC_FW_DDR_SCR_ENABLESET, BIT(0) | BIT(8));
|
|
|
|
|
|
/* ECC enablement */
|
|
data = mmio_read_32(AGX_MPFE_IOHMC_REG_CTRLCFG1);
|
|
if (data & (1 << AGX_IOHMC_CTRLCFG1_ENABLE_ECC_OFST)) {
|
|
mmio_clrsetbits_32(AGX_MPFE_HMC_ADP_ECCCTRL1,
|
|
AGX_MPFE_HMC_ADP_ECCCTRL1_AUTOWB_CNT_RST_SET_MSK |
|
|
AGX_MPFE_HMC_ADP_ECCCTRL1_CNT_RST_SET_MSK |
|
|
AGX_MPFE_HMC_ADP_ECCCTRL1_ECC_EN_SET_MSK,
|
|
AGX_MPFE_HMC_ADP_ECCCTRL1_AUTOWB_CNT_RST_SET_MSK |
|
|
AGX_MPFE_HMC_ADP_ECCCTRL1_CNT_RST_SET_MSK);
|
|
|
|
mmio_clrsetbits_32(AGX_MPFE_HMC_ADP_ECCCTRL2,
|
|
AGX_MPFE_HMC_ADP_ECCCTRL2_OVRW_RB_ECC_EN_SET_MSK |
|
|
AGX_MPFE_HMC_ADP_ECCCTRL2_RMW_EN_SET_MSK |
|
|
AGX_MPFE_HMC_ADP_ECCCTRL2_AUTOWB_EN_SET_MSK,
|
|
AGX_MPFE_HMC_ADP_ECCCTRL2_RMW_EN_SET_MSK |
|
|
AGX_MPFE_HMC_ADP_ECCCTRL2_AUTOWB_EN_SET_MSK);
|
|
|
|
mmio_clrsetbits_32(AGX_MPFE_HMC_ADP_ECCCTRL1,
|
|
AGX_MPFE_HMC_ADP_ECCCTRL1_AUTOWB_CNT_RST_SET_MSK |
|
|
AGX_MPFE_HMC_ADP_ECCCTRL1_CNT_RST_SET_MSK |
|
|
AGX_MPFE_HMC_ADP_ECCCTRL1_ECC_EN_SET_MSK,
|
|
AGX_MPFE_HMC_ADP_ECCCTRL1_ECC_EN_SET_MSK);
|
|
INFO("Scrubbing ECC\n");
|
|
|
|
/* ECC Scrubbing */
|
|
zeromem(DRAM_BASE, DRAM_SIZE);
|
|
} else {
|
|
INFO("ECC is disabled.\n");
|
|
}
|
|
}
|