This patch includes the sequence for controlling the rate changing for
EMC frequency and the dynamic training mechanism when the rate reaches
the higher rates of EMC rate.
And historically there have been different sequences to change the EMC
clock. The sequence to be used is specified in the scaling data.
However, for the currently supported upstreaming platform, only the most
recent sequence is used. So only support that in this patch.
Based on the work of Peter De Schrijver <pdeschrijver@xxxxxxxxxx>.
Signed-off-by: Joseph Lo <josephl@xxxxxxxxxx>
---
drivers/memory/tegra/Makefile | 2 +-
drivers/memory/tegra/tegra210-emc-cc-r21021.c | 1962 +++++++++++++++++
drivers/memory/tegra/tegra210-emc-reg.h | 134 ++
drivers/memory/tegra/tegra210-emc.c | 5 +
4 files changed, 2102 insertions(+), 1 deletion(-)
create mode 100644 drivers/memory/tegra/tegra210-emc-cc-r21021.c
diff --git a/drivers/memory/tegra/Makefile b/drivers/memory/tegra/Makefile
index 36a835620bbd..dcc245b2ef45 100644
--- a/drivers/memory/tegra/Makefile
+++ b/drivers/memory/tegra/Makefile
@@ -12,5 +12,5 @@ obj-$(CONFIG_TEGRA_MC) += tegra-mc.o
obj-$(CONFIG_TEGRA20_EMC) += tegra20-emc.o
obj-$(CONFIG_TEGRA124_EMC) += tegra124-emc.o
-obj-$(CONFIG_TEGRA210_EMC) += tegra210-emc.o tegra210-dt-parse.o
+obj-$(CONFIG_TEGRA210_EMC) += tegra210-emc.o tegra210-dt-parse.o tegra210-emc-cc-r21021.o
obj-$(CONFIG_ARCH_TEGRA_186_SOC) += tegra186.o
diff --git a/drivers/memory/tegra/tegra210-emc-cc-r21021.c b/drivers/memory/tegra/tegra210-emc-cc-r21021.c
new file mode 100644
index 000000000000..f577a8c3aa95
--- /dev/null
+++ b/drivers/memory/tegra/tegra210-emc-cc-r21021.c
@@ -0,0 +1,1962 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2014-2019, NVIDIA CORPORATION. All rights reserved.
+ */
+
+#include <linux/kernel.h>
+#include <linux/io.h>
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/of.h>
+#include <soc/tegra/mc.h>
+
+#include "mc.h"
+#include "tegra210-emc-reg.h"
+
+#define DVFS_CLOCK_CHANGE_VERSION 21021
+#define EMC_PRELOCK_VERSION 2101
+
+#define emc_cc_dbg(t, ...) pr_debug(__VA_ARGS__)
+
+/*
+ * Enable flags for specifying verbosity.
+ */
+#define INFO (1 << 0)
+#define STEPS (1 << 1)
+#define SUB_STEPS (1 << 2)
+#define PRELOCK (1 << 3)
+#define PRELOCK_STEPS (1 << 4)
+#define ACTIVE_EN (1 << 5)
+#define PRAMP_UP (1 << 6)
+#define PRAMP_DN (1 << 7)
+#define EMA_WRITES (1 << 10)
+#define EMA_UPDATES (1 << 11)
+#define PER_TRAIN (1 << 16)
+#define CC_PRINT (1 << 17)
+#define CCFIFO (1 << 29)
+#define REGS (1 << 30)
+#define REG_LISTS (1 << 31)
+
+enum {
+ DVFS_SEQUENCE = 1,
+ WRITE_TRAINING_SEQUENCE = 2,
+ PERIODIC_TRAINING_SEQUENCE = 3,
+ DVFS_PT1 = 10,
+ DVFS_UPDATE = 11,
+ TRAINING_PT1 = 12,
+ TRAINING_UPDATE = 13,
+ PERIODIC_TRAINING_UPDATE = 14
+};
+
+/*
+ * PTFV defines - basically just indexes into the per table PTFV array.
+ */
+#define PTFV_DQSOSC_MOVAVG_C0D0U0_INDEX 0
+#define PTFV_DQSOSC_MOVAVG_C0D0U1_INDEX 1
+#define PTFV_DQSOSC_MOVAVG_C0D1U0_INDEX 2
+#define PTFV_DQSOSC_MOVAVG_C0D1U1_INDEX 3
+#define PTFV_DQSOSC_MOVAVG_C1D0U0_INDEX 4
+#define PTFV_DQSOSC_MOVAVG_C1D0U1_INDEX 5
+#define PTFV_DQSOSC_MOVAVG_C1D1U0_INDEX 6
+#define PTFV_DQSOSC_MOVAVG_C1D1U1_INDEX 7
+#define PTFV_DVFS_SAMPLES_INDEX 9
+#define PTFV_MOVAVG_WEIGHT_INDEX 10
+#define PTFV_CONFIG_CTRL_INDEX 11
+
+#define PTFV_CONFIG_CTRL_USE_PREVIOUS_EMA (1 << 0)
+
+/*
+ * Do arithmetic in fixed point.
+ */
+#define MOVAVG_PRECISION_FACTOR 100
+
+/*
+ * The division portion of the average operation.
+ */
+#define __AVERAGE_PTFV(dev) \
+ ({ next_timing->ptfv_list[PTFV_DQSOSC_MOVAVG_ ## dev ## _INDEX] = \
+ next_timing->ptfv_list[PTFV_DQSOSC_MOVAVG_ ## dev ## _INDEX] / \
+ next_timing->ptfv_list[PTFV_DVFS_SAMPLES_INDEX]; })
+
+/*
+ * Convert val to fixed point and add it to the temporary average.
+ */
+#define __INCREMENT_PTFV(dev, val) \
+ ({ next_timing->ptfv_list[PTFV_DQSOSC_MOVAVG_ ## dev ## _INDEX] += \
+ ((val) * MOVAVG_PRECISION_FACTOR); })
+
+/*
+ * Convert a moving average back to integral form and return the value.
+ */
+#define __MOVAVG_AC(timing, dev) \
+ ((timing)->ptfv_list[PTFV_DQSOSC_MOVAVG_ ## dev ## _INDEX] / \
+ MOVAVG_PRECISION_FACTOR)
+
+/* Weighted update. */
+#define __WEIGHTED_UPDATE_PTFV(dev, nval) \
+ do { \
+ int w = PTFV_MOVAVG_WEIGHT_INDEX; \
+ int dqs = PTFV_DQSOSC_MOVAVG_ ## dev ## _INDEX; \
+ \
+ next_timing->ptfv_list[dqs] = \
+ ((nval * MOVAVG_PRECISION_FACTOR) + \
+ (next_timing->ptfv_list[dqs] * \
+ next_timing->ptfv_list[w])) / \
+ (next_timing->ptfv_list[w] + 1); \
+ \
+ emc_cc_dbg(EMA_UPDATES, "%s: (s=%u) EMA: %u\n", \
+ __stringify(dev), nval, \
+ next_timing->ptfv_list[dqs]); \
+ } while (0)
+
+/* Access a particular average. */
+#define __MOVAVG(timing, dev) \
+ ((timing)->ptfv_list[PTFV_DQSOSC_MOVAVG_ ## dev ## _INDEX])
+
+static u32 update_clock_tree_delay(struct tegra_emc *emc,
+ u32 dram_dev_num, u32 channel_mode, int type)
+{
+ u32 mrr_req = 0, mrr_data = 0;
+ u32 temp0_0 = 0, temp0_1 = 0, temp1_0 = 0, temp1_1 = 0;
+ s32 tdel = 0, tmdel = 0, adel = 0;
+ u32 cval = 0;
+ struct emc_table *last_timing = emc->current_timing;
+ struct emc_table *next_timing = emc->next_timing;
+ u32 last_timing_rate_mhz = last_timing->rate / 1000;
+ u32 next_timing_rate_mhz = next_timing->rate / 1000;
+ int dvfs_pt1 = type == DVFS_PT1;
+ int dvfs_update = type == DVFS_UPDATE;
+ int periodic_training_update = type == PERIODIC_TRAINING_UPDATE;
+
+ /*
+ * Dev0 MSB.
+ */
+ if (dvfs_pt1 || periodic_training_update) {
+ mrr_req = (2 << EMC_MRR_DEV_SEL_SHIFT) |
+ (19 << EMC_MRR_MA_SHIFT);
+ emc_writel(emc, mrr_req, EMC_MRR);
+
+ WARN(wait_for_update(emc, EMC_EMC_STATUS,
+ EMC_EMC_STATUS_MRR_DIVLD, 1, REG_EMC),
+ "Timed out waiting for MRR 19 (ch=0)\n");
+ if (channel_mode == DUAL_CHANNEL)
+ WARN(wait_for_update(emc, EMC_EMC_STATUS,
+ EMC_EMC_STATUS_MRR_DIVLD, 1,
+ REG_EMC1),
+ "Timed out waiting for MRR 19 (ch=1)\n");
+
+ mrr_data = (emc_readl(emc, EMC_MRR) & EMC_MRR_DATA_MASK) <<
+ EMC_MRR_DATA_SHIFT;
+
+ temp0_0 = (mrr_data & 0xff) << 8;
+ temp0_1 = mrr_data & 0xff00;
+
+ if (channel_mode == DUAL_CHANNEL) {
+ mrr_data = (emc1_readl(emc, EMC_MRR) &
+ EMC_MRR_DATA_MASK) <<
+ EMC_MRR_DATA_SHIFT;
+ temp1_0 = (mrr_data & 0xff) << 8;
+ temp1_1 = mrr_data & 0xff00;
+ }
+
+ /*
+ * Dev0 LSB.
+ */
+ mrr_req = (mrr_req & ~EMC_MRR_MA_MASK) |
+ (18 << EMC_MRR_MA_SHIFT);
+ emc_writel(emc, mrr_req, EMC_MRR);
+
+ WARN(wait_for_update(emc, EMC_EMC_STATUS,
+ EMC_EMC_STATUS_MRR_DIVLD, 1, REG_EMC),
+ "Timed out waiting for MRR 18 (ch=0)\n");
+ if (channel_mode == DUAL_CHANNEL)
+ WARN(wait_for_update(emc, EMC_EMC_STATUS,
+ EMC_EMC_STATUS_MRR_DIVLD, 1,
+ REG_EMC1),
+ "Timed out waiting for MRR 18 (ch=1)\n");
+
+ mrr_data = (emc_readl(emc, EMC_MRR) & EMC_MRR_DATA_MASK) <<
+ EMC_MRR_DATA_SHIFT;
+
+ temp0_0 |= mrr_data & 0xff;
+ temp0_1 |= (mrr_data & 0xff00) >> 8;
+
+ if (channel_mode == DUAL_CHANNEL) {
+ mrr_data = (emc1_readl(emc, EMC_MRR) &
+ EMC_MRR_DATA_MASK) <<
+ EMC_MRR_DATA_SHIFT;
+ temp1_0 |= (mrr_data & 0xff);
+ temp1_1 |= (mrr_data & 0xff00) >> 8;
+ }
+ }
+
+ if (dvfs_pt1 || periodic_training_update)
+ cval = (1000000 * tegra210_actual_osc_clocks(
+ last_timing->run_clocks)) /
+ (last_timing_rate_mhz * 2 * temp0_0);
+
+ if (dvfs_pt1)
+ __INCREMENT_PTFV(C0D0U0, cval);
+ else if (dvfs_update)
+ __AVERAGE_PTFV(C0D0U0);
+ else if (periodic_training_update)
+ __WEIGHTED_UPDATE_PTFV(C0D0U0, cval);
+
+ if (dvfs_update || periodic_training_update) {
+ tdel = next_timing->current_dram_clktree_c0d0u0 -
+ __MOVAVG_AC(next_timing, C0D0U0);
+ tmdel = (tdel < 0) ? -1 * tdel : tdel;
+ adel = tmdel;
+ if (tmdel * 128 * next_timing_rate_mhz / 1000000 >
+ next_timing->tree_margin)
+ next_timing->current_dram_clktree_c0d0u0 =
+ __MOVAVG_AC(next_timing, C0D0U0);
+ }
+
+ if (dvfs_pt1 || periodic_training_update)
+ cval = (1000000 * tegra210_actual_osc_clocks(
+ last_timing->run_clocks)) /
+ (last_timing_rate_mhz * 2 * temp0_1);
+
+ if (dvfs_pt1)
+ __INCREMENT_PTFV(C0D0U1, cval);
+ else if (dvfs_update)
+ __AVERAGE_PTFV(C0D0U1);
+ else if (periodic_training_update)
+ __WEIGHTED_UPDATE_PTFV(C0D0U1, cval);
+
+ if (dvfs_update || periodic_training_update) {
+ tdel = next_timing->current_dram_clktree_c0d0u1 -
+ __MOVAVG_AC(next_timing, C0D0U1);
+ tmdel = (tdel < 0) ? -1 * tdel : tdel;
+
+ if (tmdel > adel)
+ adel = tmdel;
+
+ if (tmdel * 128 * next_timing_rate_mhz / 1000000 >
+ next_timing->tree_margin)
+ next_timing->current_dram_clktree_c0d0u1 =
+ __MOVAVG_AC(next_timing, C0D0U1);
+ }
+
+ if (channel_mode == DUAL_CHANNEL) {
+ if (dvfs_pt1 || periodic_training_update)
+ cval = (1000000 * tegra210_actual_osc_clocks(
+ last_timing->run_clocks)) /
+ (last_timing_rate_mhz * 2 * temp1_0);
+ if (dvfs_pt1)
+ __INCREMENT_PTFV(C1D0U0, cval);
+ else if (dvfs_update)
+ __AVERAGE_PTFV(C1D0U0);
+ else if (periodic_training_update)
+ __WEIGHTED_UPDATE_PTFV(C1D0U0, cval);
+
+ if (dvfs_update || periodic_training_update) {
+ tdel = next_timing->current_dram_clktree_c1d0u0 -
+ __MOVAVG_AC(next_timing, C1D0U0);
+ tmdel = (tdel < 0) ? -1 * tdel : tdel;
+
+ if (tmdel > adel)
+ adel = tmdel;
+
+ if (tmdel * 128 * next_timing_rate_mhz / 1000000 >
+ next_timing->tree_margin)
+ next_timing->current_dram_clktree_c1d0u0 =
+ __MOVAVG_AC(next_timing, C1D0U0);
+ }
+
+ if (dvfs_pt1 || periodic_training_update)
+ cval = (1000000 * tegra210_actual_osc_clocks(
+ last_timing->run_clocks)) /
+ (last_timing_rate_mhz * 2 * temp1_1);
+ if (dvfs_pt1)
+ __INCREMENT_PTFV(C1D0U1, cval);
+ else if (dvfs_update)
+ __AVERAGE_PTFV(C1D0U1);
+ else if (periodic_training_update)
+ __WEIGHTED_UPDATE_PTFV(C1D0U1, cval);
+
+ if (dvfs_update || periodic_training_update) {
+ tdel = next_timing->current_dram_clktree_c1d0u1 -
+ __MOVAVG_AC(next_timing, C1D0U1);
+ tmdel = (tdel < 0) ? -1 * tdel : tdel;
+
+ if (tmdel > adel)
+ adel = tmdel;
+
+ if (tmdel * 128 * next_timing_rate_mhz / 1000000 >
+ next_timing->tree_margin)
+ next_timing->current_dram_clktree_c1d0u1 =
+ __MOVAVG_AC(next_timing, C1D0U1);
+ }
+ }
+
+ if (emc->dram_dev_num != TWO_RANK)
+ goto done;
+
+ /*
+ * Dev1 MSB.
+ */
+ if (dvfs_pt1 || periodic_training_update) {
+ mrr_req = (1 << EMC_MRR_DEV_SEL_SHIFT) |
+ (19 << EMC_MRR_MA_SHIFT);
+ emc_writel(emc, mrr_req, EMC_MRR);
+
+ WARN(wait_for_update(emc, EMC_EMC_STATUS,
+ EMC_EMC_STATUS_MRR_DIVLD, 1, REG_EMC),
+ "Timed out waiting for MRR 19 (ch=0)\n");
+ if (channel_mode == DUAL_CHANNEL)
+ WARN(wait_for_update(emc, EMC_EMC_STATUS,
+ EMC_EMC_STATUS_MRR_DIVLD, 1,
+ REG_EMC1),
+ "Timed out waiting for MRR 19 (ch=1)\n");
+
+ mrr_data = (emc_readl(emc, EMC_MRR) & EMC_MRR_DATA_MASK) <<
+ EMC_MRR_DATA_SHIFT;
+
+ temp0_0 = (mrr_data & 0xff) << 8;
+ temp0_1 = mrr_data & 0xff00;
+
+ if (channel_mode == DUAL_CHANNEL) {
+ mrr_data = (emc1_readl(emc, EMC_MRR) &
+ EMC_MRR_DATA_MASK) <<
+ EMC_MRR_DATA_SHIFT;
+ temp1_0 = (mrr_data & 0xff) << 8;
+ temp1_1 = mrr_data & 0xff00;
+ }
+
+ /*
+ * Dev1 LSB.
+ */
+ mrr_req = (mrr_req & ~EMC_MRR_MA_MASK) |
+ (18 << EMC_MRR_MA_SHIFT);
+ emc_writel(emc, mrr_req, EMC_MRR);
+
+ WARN(wait_for_update(emc, EMC_EMC_STATUS,
+ EMC_EMC_STATUS_MRR_DIVLD, 1, REG_EMC),
+ "Timed out waiting for MRR 18 (ch=0)\n");
+ if (channel_mode == DUAL_CHANNEL)
+ WARN(wait_for_update(emc, EMC_EMC_STATUS,
+ EMC_EMC_STATUS_MRR_DIVLD, 1,
+ REG_EMC1),
+ "Timed out waiting for MRR 18 (ch=1)\n");
+
+ mrr_data = (emc_readl(emc, EMC_MRR) & EMC_MRR_DATA_MASK) <<
+ EMC_MRR_DATA_SHIFT;
+
+ temp0_0 |= mrr_data & 0xff;
+ temp0_1 |= (mrr_data & 0xff00) >> 8;
+
+ if (channel_mode == DUAL_CHANNEL) {
+ mrr_data = (emc1_readl(emc, EMC_MRR) &
+ EMC_MRR_DATA_MASK) <<
+ EMC_MRR_DATA_SHIFT;
+ temp1_0 |= (mrr_data & 0xff);
+ temp1_1 |= (mrr_data & 0xff00) >> 8;
+ }
+ }
+
+ if (dvfs_pt1 || periodic_training_update)
+ cval = (1000000 * tegra210_actual_osc_clocks(
+ last_timing->run_clocks)) /
+ (last_timing_rate_mhz * 2 * temp0_0);
+
+ if (dvfs_pt1)
+ __INCREMENT_PTFV(C0D1U0, cval);
+ else if (dvfs_update)
+ __AVERAGE_PTFV(C0D1U0);
+ else if (periodic_training_update)
+ __WEIGHTED_UPDATE_PTFV(C0D1U0, cval);
+
+ if (dvfs_update || periodic_training_update) {
+ tdel = next_timing->current_dram_clktree_c0d1u0 -
+ __MOVAVG_AC(next_timing, C0D1U0);
+ tmdel = (tdel < 0) ? -1 * tdel : tdel;
+ if (tmdel > adel)
+ adel = tmdel;
+
+ if (tmdel * 128 * next_timing_rate_mhz / 1000000 >
+ next_timing->tree_margin)
+ next_timing->current_dram_clktree_c0d1u0 =
+ __MOVAVG_AC(next_timing, C0D1U0);
+ }
+
+ if (dvfs_pt1 || periodic_training_update)
+ cval = (1000000 * tegra210_actual_osc_clocks(
+ last_timing->run_clocks)) /
+ (last_timing_rate_mhz * 2 * temp0_1);
+
+ if (dvfs_pt1)
+ __INCREMENT_PTFV(C0D1U1, cval);
+ else if (dvfs_update)
+ __AVERAGE_PTFV(C0D1U1);
+ else if (periodic_training_update)
+ __WEIGHTED_UPDATE_PTFV(C0D1U1, cval);
+
+ if (dvfs_update || periodic_training_update) {
+ tdel = next_timing->current_dram_clktree_c0d1u1 -
+ __MOVAVG_AC(next_timing, C0D1U1);
+ tmdel = (tdel < 0) ? -1 * tdel : tdel;
+ if (tmdel > adel)
+ adel = tmdel;
+
+ if (tmdel * 128 * next_timing_rate_mhz / 1000000 >
+ next_timing->tree_margin)
+ next_timing->current_dram_clktree_c0d1u1 =
+ __MOVAVG_AC(next_timing, C0D1U1);
+ }
+
+ if (channel_mode == DUAL_CHANNEL) {
+ if (dvfs_pt1 || periodic_training_update)
+ cval = (1000000 * tegra210_actual_osc_clocks(
+ last_timing->run_clocks)) /
+ (last_timing_rate_mhz * 2 * temp1_0);
+
+ if (dvfs_pt1)
+ __INCREMENT_PTFV(C1D1U0, cval);
+ else if (dvfs_update)
+ __AVERAGE_PTFV(C1D1U0);
+ else if (periodic_training_update)
+ __WEIGHTED_UPDATE_PTFV(C1D1U0, cval);
+
+ if (dvfs_update || periodic_training_update) {
+ tdel = next_timing->current_dram_clktree_c1d1u0 -
+ __MOVAVG_AC(next_timing, C1D1U0);
+ tmdel = (tdel < 0) ? -1 * tdel : tdel;
+ if (tmdel > adel)
+ adel = tmdel;
+
+ if (tmdel * 128 * next_timing_rate_mhz / 1000000 >
+ next_timing->tree_margin)
+ next_timing->current_dram_clktree_c1d1u0 =
+ __MOVAVG_AC(next_timing, C1D1U0);
+ }
+
+ if (dvfs_pt1 || periodic_training_update)
+ cval = (1000000 * tegra210_actual_osc_clocks(
+ last_timing->run_clocks)) /
+ (last_timing_rate_mhz * 2 * temp1_1);
+
+ if (dvfs_pt1)
+ __INCREMENT_PTFV(C1D1U1, cval);
+ else if (dvfs_update)
+ __AVERAGE_PTFV(C1D1U1);
+ else if (periodic_training_update)
+ __WEIGHTED_UPDATE_PTFV(C1D1U1, cval);
+
+ if (dvfs_update || periodic_training_update) {
+ tdel = next_timing->current_dram_clktree_c1d1u1 -
+ __MOVAVG_AC(next_timing, C1D1U1);
+ tmdel = (tdel < 0) ? -1 * tdel : tdel;
+ if (tmdel > adel)
+ adel = tmdel;
+
+ if (tmdel * 128 * next_timing_rate_mhz / 1000000 >
+ next_timing->tree_margin)
+ next_timing->current_dram_clktree_c1d1u1 =
+ __MOVAVG_AC(next_timing, C1D1U1);
+ }
+ }
+
+done:
+ return adel;
+}
+
+static u32 periodic_compensation_handler(struct tegra_emc *emc, u32 type,
+ u32 dram_dev_num,
+ u32 channel_mode,
+ struct emc_table *last_timing,
+ struct emc_table *next_timing)
+{
+#define __COPY_EMA(nt, lt, dev) \
+ ({ __MOVAVG(nt, dev) = __MOVAVG(lt, dev) * \
+ (nt)->ptfv_list[PTFV_DVFS_SAMPLES_INDEX]; })
+
+ u32 i;
+ u32 adel = 0;
+ u32 samples = next_timing->ptfv_list[PTFV_DVFS_SAMPLES_INDEX];
+ u32 delay = 2 +
+ (1000 * tegra210_actual_osc_clocks(last_timing->run_clocks) /
+ last_timing->rate);
+
+ if (!next_timing->periodic_training)
+ return 0;
+
+ if (type == DVFS_SEQUENCE) {
+ if (last_timing->periodic_training &&
+ (next_timing->ptfv_list[PTFV_CONFIG_CTRL_INDEX] &
+ PTFV_CONFIG_CTRL_USE_PREVIOUS_EMA)) {
+ /*
+ * If the previous frequency was using periodic
+ * calibration then we can reuse the previous
+ * frequencies EMA data.
+ */
+ __COPY_EMA(next_timing, last_timing, C0D0U0);
+ __COPY_EMA(next_timing, last_timing, C0D0U1);
+ __COPY_EMA(next_timing, last_timing, C1D0U0);
+ __COPY_EMA(next_timing, last_timing, C1D0U1);
+ __COPY_EMA(next_timing, last_timing, C0D1U0);
+ __COPY_EMA(next_timing, last_timing, C0D1U1);
+ __COPY_EMA(next_timing, last_timing, C1D1U0);
+ __COPY_EMA(next_timing, last_timing, C1D1U1);
+ } else {
+ /* Reset the EMA.*/
+ __MOVAVG(next_timing, C0D0U0) = 0;
+ __MOVAVG(next_timing, C0D0U1) = 0;
+ __MOVAVG(next_timing, C1D0U0) = 0;
+ __MOVAVG(next_timing, C1D0U1) = 0;
+ __MOVAVG(next_timing, C0D1U0) = 0;
+ __MOVAVG(next_timing, C0D1U1) = 0;
+ __MOVAVG(next_timing, C1D1U0) = 0;
+ __MOVAVG(next_timing, C1D1U1) = 0;
+
+ for (i = 0; i < samples; i++) {
+ tegra210_start_periodic_compensation(emc);
+ udelay(delay);
+
+ /*
+ * Generate next sample of data.
+ */
+ adel = update_clock_tree_delay(emc,
+ emc->dram_dev_num,
+ channel_mode,
+ DVFS_PT1);
+ }
+ }
+
+ /*
+ * Seems like it should be part of the
+ * 'if (last_timing->periodic_training)' conditional
+ * since is already done for the else clause.
+ */
+ adel = update_clock_tree_delay(emc,
+ emc->dram_dev_num,
+ channel_mode,
+ DVFS_UPDATE);
+ }
+
+ if (type == PERIODIC_TRAINING_SEQUENCE) {
+ tegra210_start_periodic_compensation(emc);
+ udelay(delay);
+
+ adel = update_clock_tree_delay(emc,
+ emc->dram_dev_num,
+ channel_mode,
+ PERIODIC_TRAINING_UPDATE);
+ }
+
+ return adel;
+}
+
+u32 __do_periodic_emc_compensation_r21021(struct tegra_emc *emc)
+{
+ u32 dram_dev_num;
+ u32 channel_mode;
+ u32 emc_cfg, emc_cfg_o;
+ u32 emc_dbg_o;
+ u32 del, i;
+ u32 list[] = {
+ EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_0,
+ EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_1,
+ EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_2,
+ EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_3,
+ EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_0,
+ EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_1,
+ EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_2,
+ EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_3,
+ EMC_DATA_BRLSHFT_0,
+ EMC_DATA_BRLSHFT_1
+ };
+ u32 items = ARRAY_SIZE(list);
+ u32 emc_cfg_update;
+ struct emc_table *current_timing = emc->current_timing;
+
+ if (current_timing->periodic_training) {
+ channel_mode =
+ !!(current_timing->burst_regs[EMC_FBIO_CFG7_INDEX] &
+ (1 << 2));
+ dram_dev_num = 1 + (mc_readl(emc->mc, MC_EMEM_ADR_CFG) & 0x1);
+
+ emc_cc_dbg(PER_TRAIN, "Periodic training starting\n");
+
+ emc_dbg_o = emc_readl(emc, EMC_DBG);
+ emc_cfg_o = emc_readl(emc, EMC_CFG);
+ emc_cfg = emc_cfg_o & ~(EMC_CFG_DYN_SELF_REF |
+ EMC_CFG_DRAM_ACPD |
+ EMC_CFG_DRAM_CLKSTOP_PD |
+ EMC_CFG_DRAM_CLKSTOP_PD);
+
+
+ /*
+ * 1. Power optimizations should be off.
+ */
+ emc_writel(emc, emc_cfg, EMC_CFG);
+
+ /* Does emc_timing_update() for above changes. */
+ tegra210_dll_disable(emc, channel_mode);
+
+ wait_for_update(emc, EMC_EMC_STATUS,
+ EMC_EMC_STATUS_DRAM_IN_POWERDOWN_MASK, 0,
+ REG_EMC);
+ if (channel_mode)
+ wait_for_update(emc, EMC_EMC_STATUS,
+ EMC_EMC_STATUS_DRAM_IN_POWERDOWN_MASK,
+ 0, REG_EMC1);
+
+ wait_for_update(emc, EMC_EMC_STATUS,
+ EMC_EMC_STATUS_DRAM_IN_SELF_REFRESH_MASK, 0,
+ REG_EMC);
+ if (channel_mode)
+ wait_for_update(emc, EMC_EMC_STATUS,
+ EMC_EMC_STATUS_DRAM_IN_SELF_REFRESH_MASK, 0,
+ REG_EMC1);
+
+ emc_cfg_update = emc_readl(emc, EMC_CFG_UPDATE);
+ emc_writel(emc, (emc_cfg_update &
+ ~EMC_CFG_UPDATE_UPDATE_DLL_IN_UPDATE_MASK) |
+ (2 << EMC_CFG_UPDATE_UPDATE_DLL_IN_UPDATE_SHIFT),
+ EMC_CFG_UPDATE);
+
+ /*
+ * 2. osc kick off - this assumes training and dvfs have set
+ * correct MR23.
+ */
+ tegra210_start_periodic_compensation(emc);
+
+ /*
+ * 3. Let dram capture its clock tree delays.
+ */
+ udelay((tegra210_actual_osc_clocks(current_timing->run_clocks) *
+ 1000) /
+ current_timing->rate + 1);
+
+ /*
+ * 4. Check delta wrt previous values (save value if margin
+ * exceeds what is set in table).
+ */
+ del = periodic_compensation_handler(emc,
+ PERIODIC_TRAINING_SEQUENCE,
+ dram_dev_num,
+ channel_mode,
+ current_timing,
+ current_timing);
+
+ /*
+ * 5. Apply compensation w.r.t. trained values (if clock tree
+ * has drifted more than the set margin).
+ */
+ if (current_timing->tree_margin <
+ ((del * 128 * (current_timing->rate / 1000)) / 1000000)) {
+ for (i = 0; i < items; i++) {
+ u32 tmp =
+ tegra210_apply_periodic_compensation_trimmer(
+ current_timing, list[i]);
+
+ emc_cc_dbg(EMA_WRITES, "0x%08x <= 0x%08x\n",
+ list[i], tmp);
+ emc_writel(emc, tmp, list[i]);
+ }
+ }
+
+ emc_writel(emc, emc_cfg_o, EMC_CFG);
+
+ /*
+ * 6. Timing update actally applies the new trimmers.
+ */
+ emc_timing_update(emc, channel_mode);
+
+ /* 6.1. Restore the UPDATE_DLL_IN_UPDATE field. */
+ emc_writel(emc, emc_cfg_update, EMC_CFG_UPDATE);
+
+ /* 6.2. Restore the DLL. */
+ tegra210_dll_enable(emc, channel_mode);
+
+ /*
+ * 7. Copy over the periodic training registers that we updated
+ * here to the corresponding derated/non-derated table.
+ */
+ tegra210_update_emc_alt_timing(emc, current_timing);
+ }
+
+ return 0;
+}
+
+/*
+ * Do the clock change sequence.
+ */
+void emc_set_clock_r21021(struct tegra_emc *emc, u32 clksrc)
+{
+ /*
+ * This is the timing table for the source frequency. It does _not_
+ * necessarily correspond to the actual timing values in the EMC at the
+ * moment. If the boot BCT differs from the table then this can happen.
+ * However, we need it for accessing the dram_timings (which are not
+ * really registers) array for the current frequency.
+ */
+ struct emc_table *fake_timing;
+ struct emc_table *last_timing = emc->current_timing;
+ struct emc_table *next_timing = emc->next_timing;
+
+ u32 i, tmp;
+
+ u32 cya_allow_ref_cc = 0, ref_b4_sref_en = 0, cya_issue_pc_ref = 0;
+
+ u32 zqcal_before_cc_cutoff = 2400; /* In picoseconds */
+ u32 ref_delay_mult;
+ u32 ref_delay;
+ s32 zq_latch_dvfs_wait_time;
+ s32 tZQCAL_lpddr4_fc_adj;
+ /* Scaled by x1000 */
+ u32 tFC_lpddr4 = 1000 * next_timing->dram_timings[T_FC_LPDDR4];
+ u32 tZQCAL_lpddr4 = 1000000;
+
+ u32 dram_type, dram_dev_num, shared_zq_resistor;
+ u32 channel_mode;
+ u32 is_lpddr3;
+
+ u32 emc_cfg, emc_sel_dpd_ctrl, emc_cfg_reg;
+
+ u32 emc_dbg;
+ u32 emc_zcal_interval;
+ u32 emc_zcal_wait_cnt_old;
+ u32 emc_zcal_wait_cnt_new;
+ u32 emc_dbg_active;
+ u32 zq_op;
+ u32 zcal_wait_time_clocks;
+ u32 zcal_wait_time_ps;
+
+ u32 emc_auto_cal_config;
+ u32 auto_cal_en;
+
+ u32 mr13_catr_enable;
+
+ u32 ramp_up_wait = 0, ramp_down_wait = 0;
+
+ /* In picoseconds. */
+ u32 source_clock_period;
+ u32 destination_clock_period;
+
+ u32 emc_dbg_o;
+ u32 emc_cfg_pipe_clk_o;
+ u32 emc_pin_o;
+
+ u32 mr13_flip_fspwr;
+ u32 mr13_flip_fspop;
+
+ u32 opt_zcal_en_cc;
+ u32 opt_do_sw_qrst = 1;
+ u32 opt_dvfs_mode;
+ u32 opt_dll_mode;
+ u32 opt_cc_short_zcal = 1;
+ u32 opt_short_zcal = 1;
+ u32 save_restore_clkstop_pd = 1;
+
+ u32 prelock_dll_en = 0, dll_out;
+
+ int next_push, next_dq_e_ivref, next_dqs_e_ivref;
+
+ u32 opt_war_200024907;
+ u32 zq_wait_long;
+ u32 zq_wait_short;
+
+ u32 bg_regulator_switch_complete_wait_clks;
+ u32 bg_regulator_mode_change;
+ u32 enable_bglp_regulator;
+ u32 enable_bg_regulator;
+
+ u32 tRTM;
+ u32 RP_war;
+ u32 R2P_war;
+ u32 TRPab_war;
+ s32 nRTP;
+ u32 deltaTWATM;
+ u32 W2P_war;
+ u32 tRPST;
+
+ u32 mrw_req;
+ u32 adel = 0, compensate_trimmer_applicable = 0;
+ u32 next_timing_rate_mhz = next_timing->rate / 1000;
+
+ static u32 fsp_for_next_freq;
+
+ emc_cc_dbg(INFO, "Running clock change.\n");
+
+ fake_timing = get_timing_from_freq(emc, last_timing->rate);
+
+ fsp_for_next_freq = !fsp_for_next_freq;
+
+ dram_type = emc_readl(emc, EMC_FBIO_CFG5) &
+ EMC_FBIO_CFG5_DRAM_TYPE_MASK >>
+ EMC_FBIO_CFG5_DRAM_TYPE_SHIFT;
+ shared_zq_resistor = last_timing->burst_regs[EMC_ZCAL_WAIT_CNT_INDEX] &
+ 1 << 31;
+ channel_mode = !!(last_timing->burst_regs[EMC_FBIO_CFG7_INDEX] &
+ 1 << 2);
+ opt_zcal_en_cc = (next_timing->burst_regs[EMC_ZCAL_INTERVAL_INDEX] &&
+ !last_timing->burst_regs[EMC_ZCAL_INTERVAL_INDEX]) ||
+ dram_type == DRAM_TYPE_LPDDR4;
+ opt_dll_mode = (dram_type == DRAM_TYPE_DDR3) ?
+ get_dll_state(next_timing) : DLL_OFF;
+ is_lpddr3 = (dram_type == DRAM_TYPE_LPDDR2) &&
+ next_timing->burst_regs[EMC_FBIO_CFG5_INDEX] &
+ 1 << 25;
+ opt_war_200024907 = (dram_type == DRAM_TYPE_LPDDR4);
+ opt_dvfs_mode = MAN_SR;
+ dram_dev_num = (mc_readl(emc->mc, MC_EMEM_ADR_CFG) & 0x1) + 1;
+
+ emc_cfg_reg = emc_readl(emc, EMC_CFG);
+ emc_auto_cal_config = emc_readl(emc, EMC_AUTO_CAL_CONFIG);
+
+ source_clock_period = 1000000000 / last_timing->rate;
+ destination_clock_period = 1000000000 / next_timing->rate;
+
+ tZQCAL_lpddr4_fc_adj = (destination_clock_period >
+ zqcal_before_cc_cutoff) ?
+ tZQCAL_lpddr4 / destination_clock_period :
+ (tZQCAL_lpddr4 - tFC_lpddr4) / destination_clock_period;
+ emc_dbg_o = emc_readl(emc, EMC_DBG);
+ emc_pin_o = emc_readl(emc, EMC_PIN);
+ emc_cfg_pipe_clk_o = emc_readl(emc, EMC_CFG_PIPE_CLK);
+ emc_dbg = emc_dbg_o;
+
+ emc_cfg = next_timing->burst_regs[EMC_CFG_INDEX];
+ emc_cfg &= ~(EMC_CFG_DYN_SELF_REF | EMC_CFG_DRAM_ACPD |
+ EMC_CFG_DRAM_CLKSTOP_SR | EMC_CFG_DRAM_CLKSTOP_PD);
+ emc_sel_dpd_ctrl = next_timing->emc_sel_dpd_ctrl;
+ emc_sel_dpd_ctrl &= ~(EMC_SEL_DPD_CTRL_CLK_SEL_DPD_EN |
+ EMC_SEL_DPD_CTRL_CA_SEL_DPD_EN |
+ EMC_SEL_DPD_CTRL_RESET_SEL_DPD_EN |
+ EMC_SEL_DPD_CTRL_ODT_SEL_DPD_EN |
+ EMC_SEL_DPD_CTRL_DATA_SEL_DPD_EN);
+
+ emc_cc_dbg(INFO, "Clock change version: %d\n",
+ DVFS_CLOCK_CHANGE_VERSION);
+ emc_cc_dbg(INFO, "DRAM type = %d\n", emc->dram_type);
+ emc_cc_dbg(INFO, "DRAM dev #: %d\n", dram_dev_num);
+ emc_cc_dbg(INFO, "Next EMC clksrc: 0x%08x\n", clksrc);
+ emc_cc_dbg(INFO, "DLL clksrc: 0x%08x\n", next_timing->dll_clk_src);
+ emc_cc_dbg(INFO, "last rate: %u, next rate %u\n", last_timing->rate,
+ next_timing->rate);
+ emc_cc_dbg(INFO, "last period: %u, next period: %u\n",
+ source_clock_period, destination_clock_period);
+ emc_cc_dbg(INFO, " shared_zq_resistor: %d\n", !!shared_zq_resistor);
+ emc_cc_dbg(INFO, " channel_mode: %d\n", channel_mode);
+ emc_cc_dbg(INFO, " opt_dll_mode: %d\n", opt_dll_mode);
+
+ /*
+ * Step 1:
+ * Pre DVFS SW sequence.
+ */
+ emc_cc_dbg(STEPS, "Step 1\n");
+ emc_cc_dbg(STEPS, "Step 1.1: Disable DLL temporarily.\n");
+ tmp = emc_readl(emc, EMC_CFG_DIG_DLL);
+ tmp &= ~EMC_CFG_DIG_DLL_CFG_DLL_EN;
+ emc_writel(emc, tmp, EMC_CFG_DIG_DLL);
+
+ emc_timing_update(emc, channel_mode);
+ wait_for_update(emc, EMC_CFG_DIG_DLL,
+ EMC_CFG_DIG_DLL_CFG_DLL_EN, 0, REG_EMC);
+ if (channel_mode)
+ wait_for_update(emc, EMC_CFG_DIG_DLL,
+ EMC_CFG_DIG_DLL_CFG_DLL_EN, 0, REG_EMC1);
+
+ emc_cc_dbg(STEPS, "Step 1.2: Disable AUTOCAL temporarily.\n");
+ emc_auto_cal_config = next_timing->emc_auto_cal_config;
+ auto_cal_en = emc_auto_cal_config & EMC_AUTO_CAL_CONFIG_AUTO_CAL_ENABLE;
+ emc_auto_cal_config &= ~EMC_AUTO_CAL_CONFIG_AUTO_CAL_START;
+ emc_auto_cal_config |= EMC_AUTO_CAL_CONFIG_AUTO_CAL_MEASURE_STALL;
+ emc_auto_cal_config |= EMC_AUTO_CAL_CONFIG_AUTO_CAL_UPDATE_STALL;
+ emc_auto_cal_config |= auto_cal_en;
+ emc_writel(emc, emc_auto_cal_config, EMC_AUTO_CAL_CONFIG);
+ emc_readl(emc, EMC_AUTO_CAL_CONFIG); /* Flush write. */
+
+ emc_cc_dbg(STEPS, "Step 1.3: Disable other power features.\n");
+ emc_set_shadow_bypass(emc, ACTIVE);
+ emc_writel(emc, emc_cfg, EMC_CFG);
+ emc_writel(emc, emc_sel_dpd_ctrl, EMC_SEL_DPD_CTRL);
+ emc_set_shadow_bypass(emc, ASSEMBLY);
+
+ if (next_timing->periodic_training) {
+ tegra210_reset_dram_clktree_values(next_timing);
+
+ wait_for_update(emc, EMC_EMC_STATUS,
+ EMC_EMC_STATUS_DRAM_IN_POWERDOWN_MASK, 0,
+ REG_EMC);
+ if (channel_mode)
+ wait_for_update(emc, EMC_EMC_STATUS,
+ EMC_EMC_STATUS_DRAM_IN_POWERDOWN_MASK,
+ 0, REG_EMC1);
+
+ wait_for_update(emc, EMC_EMC_STATUS,
+ EMC_EMC_STATUS_DRAM_IN_SELF_REFRESH_MASK, 0,
+ REG_EMC);
+ if (channel_mode)
+ wait_for_update(emc, EMC_EMC_STATUS,
+ EMC_EMC_STATUS_DRAM_IN_SELF_REFRESH_MASK, 0,
+ REG_EMC1);
+
+ tegra210_start_periodic_compensation(emc);
+
+ udelay(((1000 *
+ tegra210_actual_osc_clocks(last_timing->run_clocks)) /
+ last_timing->rate) + 2);
+ adel = periodic_compensation_handler(emc, DVFS_SEQUENCE,
+ dram_dev_num,
+ channel_mode,
+ fake_timing, next_timing);
+ compensate_trimmer_applicable =
+ next_timing->periodic_training &&
+ ((adel * 128 * next_timing_rate_mhz) / 1000000) >
+ next_timing->tree_margin;
+ }
+
+ emc_writel(emc, EMC_INTSTATUS_CLKCHANGE_COMPLETE, EMC_INTSTATUS);
+ emc_set_shadow_bypass(emc, ACTIVE);
+ emc_writel(emc, emc_cfg, EMC_CFG);
+ emc_writel(emc, emc_sel_dpd_ctrl, EMC_SEL_DPD_CTRL);
+ emc_writel(emc, emc_cfg_pipe_clk_o | EMC_CFG_PIPE_CLK_CLK_ALWAYS_ON,
+ EMC_CFG_PIPE_CLK);
+ emc_writel(emc, next_timing->emc_fdpd_ctrl_cmd_no_ramp &
+ ~EMC_FDPD_CTRL_CMD_NO_RAMP_CMD_DPD_NO_RAMP_ENABLE,
+ EMC_FDPD_CTRL_CMD_NO_RAMP);
+
+ bg_regulator_mode_change =
+ ((next_timing->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] &
+ EMC_PMACRO_BG_BIAS_CTRL_0_BGLP_E_PWRD) ^
+ (last_timing->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] &
+ EMC_PMACRO_BG_BIAS_CTRL_0_BGLP_E_PWRD)) ||
+ ((next_timing->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] &
+ EMC_PMACRO_BG_BIAS_CTRL_0_BG_E_PWRD) ^
+ (last_timing->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] &
+ EMC_PMACRO_BG_BIAS_CTRL_0_BG_E_PWRD));
+ enable_bglp_regulator =
+ (next_timing->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] &
+ EMC_PMACRO_BG_BIAS_CTRL_0_BGLP_E_PWRD) == 0;
+ enable_bg_regulator =
+ (next_timing->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] &
+ EMC_PMACRO_BG_BIAS_CTRL_0_BG_E_PWRD) == 0;
+
+ if (bg_regulator_mode_change) {
+ if (enable_bg_regulator)
+ emc_writel(emc, last_timing->burst_regs
+ [EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] &
+ ~EMC_PMACRO_BG_BIAS_CTRL_0_BG_E_PWRD,
+ EMC_PMACRO_BG_BIAS_CTRL_0);
+ else
+ emc_writel(emc, last_timing->burst_regs
+ [EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] &
+ ~EMC_PMACRO_BG_BIAS_CTRL_0_BGLP_E_PWRD,
+ EMC_PMACRO_BG_BIAS_CTRL_0);
+ }
+
+ /* Check if we need to turn on VREF generator. */
+ if ((((last_timing->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX] &
+ EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_E_IVREF) == 0) &&
+ ((next_timing->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX] &
+ EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_E_IVREF) == 1)) ||
+ (((last_timing->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX] &
+ EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQS_E_IVREF) == 0) &&
+ ((next_timing->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX] &
+ EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQS_E_IVREF) == 1))) {
+ u32 pad_tx_ctrl =
+ next_timing->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX];
+ u32 last_pad_tx_ctrl =
+ last_timing->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX];
+
+ next_dqs_e_ivref = pad_tx_ctrl &
+ EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQS_E_IVREF;
+ next_dq_e_ivref = pad_tx_ctrl &
+ EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_E_IVREF;
+ next_push = (last_pad_tx_ctrl &
+ ~EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_E_IVREF &
+ ~EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQS_E_IVREF) |
+ next_dq_e_ivref | next_dqs_e_ivref;
+ emc_writel(emc, next_push, EMC_PMACRO_DATA_PAD_TX_CTRL);
+ udelay(1);
+ } else if (bg_regulator_mode_change) {
+ udelay(1);
+ }
+
+ emc_set_shadow_bypass(emc, ASSEMBLY);
+
+ /*
+ * Step 2:
+ * Prelock the DLL.
+ */
+ emc_cc_dbg(STEPS, "Step 2\n");
+ if (next_timing->burst_regs[EMC_CFG_DIG_DLL_INDEX] &
+ EMC_CFG_DIG_DLL_CFG_DLL_EN) {
+ emc_cc_dbg(INFO, "Prelock enabled for target frequency.\n");
+ dll_out = tegra210_dll_prelock(emc, 0, clksrc);
+ emc_cc_dbg(INFO, "DLL out: 0x%03x\n", dll_out);
+ prelock_dll_en = 1;
+ } else {
+ emc_cc_dbg(INFO, "Disabling DLL for target frequency.\n");
+ tegra210_dll_disable(emc, channel_mode);
+ }
+
+ /*
+ * Step 3:
+ * Prepare autocal for the clock change.
+ */
+ emc_cc_dbg(STEPS, "Step 3\n");
+ emc_set_shadow_bypass(emc, ACTIVE);
+ emc_writel(emc, next_timing->emc_auto_cal_config2,
+ EMC_AUTO_CAL_CONFIG2);
+ emc_writel(emc, next_timing->emc_auto_cal_config3,
+ EMC_AUTO_CAL_CONFIG3);
+ emc_writel(emc, next_timing->emc_auto_cal_config4,
+ EMC_AUTO_CAL_CONFIG4);
+ emc_writel(emc, next_timing->emc_auto_cal_config5,
+ EMC_AUTO_CAL_CONFIG5);
+ emc_writel(emc, next_timing->emc_auto_cal_config6,
+ EMC_AUTO_CAL_CONFIG6);
+ emc_writel(emc, next_timing->emc_auto_cal_config7,
+ EMC_AUTO_CAL_CONFIG7);
+ emc_writel(emc, next_timing->emc_auto_cal_config8,
+ EMC_AUTO_CAL_CONFIG8);
+ emc_set_shadow_bypass(emc, ASSEMBLY);
+
+ emc_auto_cal_config |= (EMC_AUTO_CAL_CONFIG_AUTO_CAL_COMPUTE_START |
+ auto_cal_en);
+ emc_writel(emc, emc_auto_cal_config, EMC_AUTO_CAL_CONFIG);
+
+ /*
+ * Step 4:
+ * Update EMC_CFG. (??)
+ */
+ emc_cc_dbg(STEPS, "Step 4\n");
+ if (source_clock_period > 50000 && dram_type == DRAM_TYPE_LPDDR4)
+ ccfifo_writel(emc, 1, EMC_SELF_REF, 0);
+ else
+ emc_writel(emc, next_timing->emc_cfg_2, EMC_CFG_2);
+
+ /*
+ * Step 5:
+ * Prepare reference variables for ZQCAL regs.
+ */
+ emc_cc_dbg(STEPS, "Step 5\n");
+ emc_zcal_interval = 0;
+ emc_zcal_wait_cnt_old =
+ last_timing->burst_regs[EMC_ZCAL_WAIT_CNT_INDEX];
+ emc_zcal_wait_cnt_new =
+ next_timing->burst_regs[EMC_ZCAL_WAIT_CNT_INDEX];
+ emc_zcal_wait_cnt_old &= ~EMC_ZCAL_WAIT_CNT_ZCAL_WAIT_CNT_MASK;
+ emc_zcal_wait_cnt_new &= ~EMC_ZCAL_WAIT_CNT_ZCAL_WAIT_CNT_MASK;
+
+ if (dram_type == DRAM_TYPE_LPDDR4)
+ zq_wait_long = max((u32)1,
+ div_o3(1000000, destination_clock_period));
+ else if (dram_type == DRAM_TYPE_LPDDR2 || is_lpddr3)
+ zq_wait_long = max(next_timing->min_mrs_wait,
+ div_o3(360000, destination_clock_period)) +
+ 4;
+ else if (dram_type == DRAM_TYPE_DDR3)
+ zq_wait_long = max((u32)256,
+ div_o3(320000, destination_clock_period) +
+ 2);
+ else
+ zq_wait_long = 0;
+
+ if (dram_type == DRAM_TYPE_LPDDR2 || is_lpddr3)
+ zq_wait_short = max(max(next_timing->min_mrs_wait, (u32)6),
+ div_o3(90000, destination_clock_period)) +
+ 4;
+ else if (dram_type == DRAM_TYPE_DDR3)
+ zq_wait_short = max((u32)64,
+ div_o3(80000, destination_clock_period)) +
+ 2;
+ else
+ zq_wait_short = 0;
+
+ /*
+ * Step 6:
+ * Training code - removed.
+ */
+ emc_cc_dbg(STEPS, "Step 6\n");
+
+ /*
+ * Step 7:
+ * Program FSP reference registers and send MRWs to new FSPWR.
+ */
+ emc_cc_dbg(STEPS, "Step 7\n");
+ emc_cc_dbg(SUB_STEPS, "Step 7.1: Bug 200024907 - Patch RP R2P");
+ if (opt_war_200024907) {
+ nRTP = 16;
+ if (source_clock_period >= 1000000/1866) /* 535.91 ps */
+ nRTP = 14;
+ if (source_clock_period >= 1000000/1600) /* 625.00 ps */
+ nRTP = 12;
+ if (source_clock_period >= 1000000/1333) /* 750.19 ps */
+ nRTP = 10;
+ if (source_clock_period >= 1000000/1066) /* 938.09 ps */
+ nRTP = 8;
+
+ deltaTWATM = max_t(u32, div_o3(7500, source_clock_period), 8);
+
+ /*
+ * Originally there was a + .5 in the tRPST calculation.
+ * However since we can't do FP in the kernel and the tRTM
+ * computation was in a floating point ceiling function, adding
+ * one to tRTP should be ok. There is no other source of non
+ * integer values, so the result was always going to be
+ * something for the form: f_ceil(N + .5) = N + 1;
+ */
+ tRPST = ((last_timing->emc_mrw & 0x80) >> 7);
+ tRTM = fake_timing->dram_timings[RL] +
+ div_o3(3600, source_clock_period) +
+ max_t(u32, div_o3(7500, source_clock_period), 8) +
+ tRPST + 1 + nRTP;
+
+ emc_cc_dbg(INFO, "tRTM = %u, EMC_RP = %u\n", tRTM,
+ next_timing->burst_regs[EMC_RP_INDEX]);
+
+ if (last_timing->burst_regs[EMC_RP_INDEX] < tRTM) {
+ if (tRTM > (last_timing->burst_regs[EMC_R2P_INDEX] +
+ last_timing->burst_regs[EMC_RP_INDEX])) {
+ R2P_war = tRTM -
+ last_timing->burst_regs[EMC_RP_INDEX];
+ RP_war = last_timing->burst_regs[EMC_RP_INDEX];
+ TRPab_war = last_timing->burst_regs[
+ EMC_TRPAB_INDEX];
+ if (R2P_war > 63) {
+ RP_war = R2P_war +
+ last_timing->burst_regs[
+ EMC_RP_INDEX] - 63;
+ if (TRPab_war < RP_war)
+ TRPab_war = RP_war;
+ R2P_war = 63;
+ }
+ } else {
+ R2P_war = last_timing->burst_regs[
+ EMC_R2P_INDEX];
+ RP_war = last_timing->burst_regs[EMC_RP_INDEX];
+ TRPab_war = last_timing->burst_regs[
+ EMC_TRPAB_INDEX];
+ }
+
+ if (RP_war < deltaTWATM) {
+ W2P_war = last_timing->burst_regs[EMC_W2P_INDEX]
+ + deltaTWATM - RP_war;
+ if (W2P_war > 63) {
+ RP_war = RP_war + W2P_war - 63;
+ if (TRPab_war < RP_war)
+ TRPab_war = RP_war;
+ W2P_war = 63;
+ }
+ } else {
+ W2P_war = last_timing->burst_regs[
+ EMC_W2P_INDEX];
+ }
+
+ if ((last_timing->burst_regs[EMC_W2P_INDEX] ^
+ W2P_war) ||
+ (last_timing->burst_regs[EMC_R2P_INDEX] ^
+ R2P_war) ||
+ (last_timing->burst_regs[EMC_RP_INDEX] ^
+ RP_war) ||
+ (last_timing->burst_regs[EMC_TRPAB_INDEX] ^
+ TRPab_war)) {
+ emc_writel(emc, RP_war, EMC_RP);
+ emc_writel(emc, R2P_war, EMC_R2P);
+ emc_writel(emc, W2P_war, EMC_W2P);
+ emc_writel(emc, TRPab_war, EMC_TRPAB);
+ }
+ emc_timing_update(emc, DUAL_CHANNEL);
+ } else {
+ emc_cc_dbg(INFO, "Skipped WAR\n");
+ }
+ }
+
+ if (!fsp_for_next_freq) {
+ mr13_flip_fspwr = (next_timing->emc_mrw3 & 0xffffff3f) | 0x80;
+ mr13_flip_fspop = (next_timing->emc_mrw3 & 0xffffff3f) | 0x00;
+ } else {
+ mr13_flip_fspwr = (next_timing->emc_mrw3 & 0xffffff3f) | 0x40;
+ mr13_flip_fspop = (next_timing->emc_mrw3 & 0xffffff3f) | 0xc0;
+ }
+
+ mr13_catr_enable = (mr13_flip_fspwr & 0xFFFFFFFE) | 0x01;
+ if (dram_dev_num == TWO_RANK)
+ mr13_catr_enable = (mr13_catr_enable & 0x3fffffff) | 0x80000000;
+
+ if (dram_type == DRAM_TYPE_LPDDR4) {
+ emc_writel(emc, mr13_flip_fspwr, EMC_MRW3);
+ emc_writel(emc, next_timing->emc_mrw, EMC_MRW);
+ emc_writel(emc, next_timing->emc_mrw2, EMC_MRW2);
+ }
+
+ /*
+ * Step 8:
+ * Program the shadow registers.
+ */
+ emc_cc_dbg(STEPS, "Step 8\n");
+ emc_cc_dbg(SUB_STEPS, "Writing burst_regs\n");
+ for (i = 0; i < next_timing->num_burst; i++) {
+ u32 var;
+ u32 wval;
+
+ if (!burst_regs_off[i])
+ continue;
+
+ var = burst_regs_off[i];
+ wval = next_timing->burst_regs[i];
+
+ if (dram_type != DRAM_TYPE_LPDDR4 &&
+ (var == EMC_MRW6 || var == EMC_MRW7 ||
+ var == EMC_MRW8 || var == EMC_MRW9 ||
+ var == EMC_MRW10 || var == EMC_MRW11 ||
+ var == EMC_MRW12 || var == EMC_MRW13 ||
+ var == EMC_MRW14 || var == EMC_MRW15 ||
+ var == EMC_TRAINING_CTRL))
+ continue;
+
+ /* Pain... And suffering. */
+ if (var == EMC_CFG) {
+ wval &= ~EMC_CFG_DRAM_ACPD;
+ wval &= ~EMC_CFG_DYN_SELF_REF;
+ if (dram_type == DRAM_TYPE_LPDDR4) {
+ wval &= ~EMC_CFG_DRAM_CLKSTOP_SR;
+ wval &= ~EMC_CFG_DRAM_CLKSTOP_PD;
+ }
+ } else if (var == EMC_MRS_WAIT_CNT &&
+ dram_type == DRAM_TYPE_LPDDR2 &&
+ opt_zcal_en_cc && !opt_cc_short_zcal &&
+ opt_short_zcal) {
+ wval = (wval & ~(EMC_MRS_WAIT_CNT_SHORT_WAIT_MASK <<
+ EMC_MRS_WAIT_CNT_SHORT_WAIT_SHIFT)) |
+ ((zq_wait_long & EMC_MRS_WAIT_CNT_SHORT_WAIT_MASK) <<
+ EMC_MRS_WAIT_CNT_SHORT_WAIT_SHIFT);
+ } else if (var == EMC_ZCAL_WAIT_CNT &&
+ dram_type == DRAM_TYPE_DDR3 && opt_zcal_en_cc &&
+ !opt_cc_short_zcal && opt_short_zcal) {
+ wval = (wval & ~(EMC_ZCAL_WAIT_CNT_ZCAL_WAIT_CNT_MASK <<
+ EMC_ZCAL_WAIT_CNT_ZCAL_WAIT_CNT_SHIFT))
+ | ((zq_wait_long &
+ EMC_ZCAL_WAIT_CNT_ZCAL_WAIT_CNT_MASK) <<
+ EMC_MRS_WAIT_CNT_SHORT_WAIT_SHIFT);
+ } else if (var == EMC_ZCAL_INTERVAL && opt_zcal_en_cc) {
+ wval = 0; /* EMC_ZCAL_INTERVAL reset value. */
+ } else if (var == EMC_PMACRO_AUTOCAL_CFG_COMMON) {
+ wval |= EMC_PMACRO_AUTOCAL_CFG_COMMON_E_CAL_BYPASS_DVFS;
+ } else if (var == EMC_PMACRO_DATA_PAD_TX_CTRL) {
+ wval &=
+ ~(EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSP_TX_E_DCC |
+ EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSN_TX_E_DCC |
+ EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_TX_E_DCC |
+ EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_CMD_TX_E_DCC);
+ } else if (var == EMC_PMACRO_CMD_PAD_TX_CTRL) {
+ wval |= EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_DRVFORCEON;
+ wval &= ~(EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSP_TX_E_DCC |
+ EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSN_TX_E_DCC |
+ EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_E_DCC |
+ EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_CMD_TX_E_DCC);
+ } else if (var == EMC_PMACRO_BRICK_CTRL_RFU1) {
+ wval &= 0xf800f800;
+ } else if (var == EMC_PMACRO_COMMON_PAD_TX_CTRL) {
+ wval &= 0xfffffff0;
+ }
+
+ emc_writel(emc, wval, var);
+ }
+
+ /* SW addition: do EMC refresh adjustment here. */
+ set_over_temp_timing(emc, next_timing, dram_over_temp_state);
+
+ if (dram_type == DRAM_TYPE_LPDDR4) {
+ mrw_req = (23 << EMC_MRW_MRW_MA_SHIFT) |
+ (next_timing->run_clocks & EMC_MRW_MRW_OP_MASK);
+ emc_writel(emc, mrw_req, EMC_MRW);
+ }
+
+ /* Per channel burst registers. */
+ emc_cc_dbg(SUB_STEPS, "Writing burst_regs_per_ch\n");
+ for (i = 0; i < next_timing->num_burst_per_ch; i++) {
+ if (!burst_regs_per_ch_off[i])
+ continue;
+
+ if (dram_type != DRAM_TYPE_LPDDR4 &&
+ (burst_regs_per_ch_off[i] == EMC_MRW6 ||
+ burst_regs_per_ch_off[i] == EMC_MRW7 ||
+ burst_regs_per_ch_off[i] == EMC_MRW8 ||
+ burst_regs_per_ch_off[i] == EMC_MRW9 ||
+ burst_regs_per_ch_off[i] == EMC_MRW10 ||
+ burst_regs_per_ch_off[i] == EMC_MRW11 ||
+ burst_regs_per_ch_off[i] == EMC_MRW12 ||
+ burst_regs_per_ch_off[i] == EMC_MRW13 ||
+ burst_regs_per_ch_off[i] == EMC_MRW14 ||
+ burst_regs_per_ch_off[i] == EMC_MRW15))
+ continue;
+
+ /* Filter out second channel if not in DUAL_CHANNEL mode. */
+ if (channel_mode != DUAL_CHANNEL &&
+ burst_regs_per_ch_type[i] >= REG_EMC1)
+ continue;
+
+ emc_cc_dbg(REG_LISTS, "(%u) 0x%08x => 0x%08x\n",
+ i, next_timing->burst_reg_per_ch[i],
+ burst_regs_per_ch_off[i]);
+ emc_writel_per_ch(emc, next_timing->burst_reg_per_ch[i],
+ burst_regs_per_ch_type[i],
+ burst_regs_per_ch_off[i]);
+ }
+
+ /* Vref regs. */
+ emc_cc_dbg(SUB_STEPS, "Writing vref_regs\n");
+ for (i = 0; i < next_timing->vref_num; i++) {
+ if (!vref_regs_per_ch_off[i])
+ continue;
+
+ if (channel_mode != DUAL_CHANNEL &&
+ vref_regs_per_ch_type[i] >= REG_EMC1)
+ continue;
+
+ emc_cc_dbg(REG_LISTS, "(%u) 0x%08x => 0x%08x\n",
+ i, next_timing->vref_perch_regs[i],
+ vref_regs_per_ch_off[i]);
+ emc_writel_per_ch(emc, next_timing->vref_perch_regs[i],
+ vref_regs_per_ch_type[i],
+ vref_regs_per_ch_off[i]);
+ }
+
+ /* Trimmers. */
+ emc_cc_dbg(SUB_STEPS, "Writing trim_regs\n");
+ for (i = 0; i < next_timing->num_trim; i++) {
+ u64 trim_reg;
+
+ if (!trim_regs_off[i])
+ continue;
+
+ trim_reg = trim_regs_off[i];
+ if (compensate_trimmer_applicable &&
+ (trim_reg == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_0 ||
+ trim_reg == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_1 ||
+ trim_reg == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_2 ||
+ trim_reg == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_3 ||
+ trim_reg == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_0 ||
+ trim_reg == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_1 ||
+ trim_reg == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_2 ||
+ trim_reg == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_3 ||
+ trim_reg == EMC_DATA_BRLSHFT_0 ||
+ trim_reg == EMC_DATA_BRLSHFT_1)) {
+ u32 reg = tegra210_apply_periodic_compensation_trimmer(
+ next_timing, trim_reg);
+ emc_cc_dbg(REG_LISTS, "(%u) 0x%08x => 0x%08x\n", i, reg,
+ trim_regs_off[i]);
+ emc_cc_dbg(EMA_WRITES, "0x%08x <= 0x%08x\n",
+ (u32)(u64)trim_regs_off[i], reg);
+ emc_writel(emc, reg, trim_regs_off[i]);
+ } else {
+ emc_cc_dbg(REG_LISTS, "(%u) 0x%08x => 0x%08x\n",
+ i, next_timing->trim_regs[i],
+ trim_regs_off[i]);
+ emc_writel(emc, next_timing->trim_regs[i],
+ trim_regs_off[i]);
+ }
+ }
+
+ /* Per channel trimmers. */
+ emc_cc_dbg(SUB_STEPS, "Writing trim_regs_per_ch\n");
+ for (i = 0; i < next_timing->num_trim_per_ch; i++) {
+ u32 trim_reg;
+
+ if (!trim_regs_per_ch_off[i])
+ continue;
+
+ if (channel_mode != DUAL_CHANNEL &&
+ trim_regs_per_ch_type[i] >= REG_EMC1)
+ continue;
+
+ trim_reg = trim_regs_per_ch_off[i];
+ if (compensate_trimmer_applicable &&
+ (trim_reg == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_0 ||
+ trim_reg == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_1 ||
+ trim_reg == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_2 ||
+ trim_reg == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_3 ||
+ trim_reg == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_0 ||
+ trim_reg == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_1 ||
+ trim_reg == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_2 ||
+ trim_reg == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_3 ||
+ trim_reg == EMC_DATA_BRLSHFT_0 ||
+ trim_reg == EMC_DATA_BRLSHFT_1)) {
+ u32 reg =
+ tegra210_apply_periodic_compensation_trimmer(
+ next_timing, trim_reg);
+ emc_cc_dbg(REG_LISTS, "(%u) 0x%08x => 0x%08x\n",
+ i, reg, trim_regs_per_ch_off[i]);
+ emc_cc_dbg(EMA_WRITES, "0x%08x <= 0x%08x\n",
+ trim_regs_per_ch_off[i], reg);
+ emc_writel_per_ch(emc, reg, trim_regs_per_ch_type[i],
+ trim_regs_per_ch_off[i]);
+ } else {
+ emc_cc_dbg(REG_LISTS, "(%u) 0x%08x => 0x%08x\n",
+ i, next_timing->trim_perch_regs[i],
+ trim_regs_per_ch_off[i]);
+ emc_writel_per_ch(emc, next_timing->trim_perch_regs[i],
+ trim_regs_per_ch_type[i],
+ trim_regs_per_ch_off[i]);
+ }
+ }
+
+ emc_cc_dbg(SUB_STEPS, "Writing burst_mc_regs\n");
+ for (i = 0; i < next_timing->num_mc_regs; i++) {
+ emc_cc_dbg(REG_LISTS, "(%u) 0x%08x => 0x%08x\n",
+ i, next_timing->burst_mc_regs[i],
+ burst_mc_regs_off[i]);
+ mc_writel(emc->mc, next_timing->burst_mc_regs[i],
+ burst_mc_regs_off[i]);
+ }
+
+ /* Registers to be programmed on the faster clock. */
+ if (next_timing->rate < last_timing->rate) {
+ emc_cc_dbg(SUB_STEPS, "Writing la_scale_regs\n");
+ for (i = 0; i < next_timing->num_up_down; i++) {
+ emc_cc_dbg(REG_LISTS, "(%u) 0x%08x => 0x%08x\n",
+ i, next_timing->la_scale_regs[i],
+ la_scale_regs_off[i]);
+ mc_writel(emc->mc, next_timing->la_scale_regs[i],
+ la_scale_regs_off[i]);
+ }
+ }
+
+ /* Flush all the burst register writes. */
+ wmb();
