summaryrefslogtreecommitdiffstats
path: root/arch/arm/mach-mvebu/coherency.c
blob: 044b51185fccb2e68c1f89c4efb3822704d28488 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
/*
 * Coherency fabric (Aurora) support for Armada 370 and XP platforms.
 *
 * Copyright (C) 2012 Marvell
 *
 * Yehuda Yitschak <yehuday@marvell.com>
 * Gregory Clement <gregory.clement@free-electrons.com>
 * Thomas Petazzoni <thomas.petazzoni@free-electrons.com>
 *
 * This file is licensed under the terms of the GNU General Public
 * License version 2.  This program is licensed "as is" without any
 * warranty of any kind, whether express or implied.
 *
 * The Armada 370 and Armada XP SOCs have a coherency fabric which is
 * responsible for ensuring hardware coherency between all CPUs and between
 * CPUs and I/O masters. This file initializes the coherency fabric and
 * supplies basic routines for configuring and controlling hardware coherency
 */

#define pr_fmt(fmt) "mvebu-coherency: " fmt

#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/of_address.h>
#include <linux/io.h>
#include <linux/smp.h>
#include <linux/dma-mapping.h>
#include <linux/platform_device.h>
#include <linux/slab.h>
#include <linux/mbus.h>
#include <linux/clk.h>
#include <linux/pci.h>
#include <asm/smp_plat.h>
#include <asm/cacheflush.h>
#include <asm/mach/map.h>
#include "armada-370-xp.h"
#include "coherency.h"
#include "mvebu-soc-id.h"

unsigned long coherency_phys_base;
void __iomem *coherency_base;
static void __iomem *coherency_cpu_base;

/* Coherency fabric registers */
#define COHERENCY_FABRIC_CFG_OFFSET		   0x4

#define IO_SYNC_BARRIER_CTL_OFFSET		   0x0

enum {
	COHERENCY_FABRIC_TYPE_NONE,
	COHERENCY_FABRIC_TYPE_ARMADA_370_XP,
	COHERENCY_FABRIC_TYPE_ARMADA_375,
	COHERENCY_FABRIC_TYPE_ARMADA_380,
};

static struct of_device_id of_coherency_table[] = {
	{.compatible = "marvell,coherency-fabric",
	 .data = (void *) COHERENCY_FABRIC_TYPE_ARMADA_370_XP },
	{.compatible = "marvell,armada-375-coherency-fabric",
	 .data = (void *) COHERENCY_FABRIC_TYPE_ARMADA_375 },
	{.compatible = "marvell,armada-380-coherency-fabric",
	 .data = (void *) COHERENCY_FABRIC_TYPE_ARMADA_380 },
	{ /* end of list */ },
};

/* Functions defined in coherency_ll.S */
int ll_enable_coherency(void);
void ll_add_cpu_to_smp_group(void);

int set_cpu_coherent(void)
{
	if (!coherency_base) {
		pr_warn("Can't make current CPU cache coherent.\n");
		pr_warn("Coherency fabric is not initialized\n");
		return 1;
	}

	ll_add_cpu_to_smp_group();
	return ll_enable_coherency();
}

/*
 * The below code implements the I/O coherency workaround on Armada
 * 375. This workaround consists in using the two channels of the
 * first XOR engine to trigger a XOR transaction that serves as the
 * I/O coherency barrier.
 */

static void __iomem *xor_base, *xor_high_base;
static dma_addr_t coherency_wa_buf_phys[CONFIG_NR_CPUS];
static void *coherency_wa_buf[CONFIG_NR_CPUS];
static bool coherency_wa_enabled;

#define XOR_CONFIG(chan)            (0x10 + (chan * 4))
#define XOR_ACTIVATION(chan)        (0x20 + (chan * 4))
#define WINDOW_BAR_ENABLE(chan)     (0x240 + ((chan) << 2))
#define WINDOW_BASE(w)              (0x250 + ((w) << 2))
#define WINDOW_SIZE(w)              (0x270 + ((w) << 2))
#define WINDOW_REMAP_HIGH(w)        (0x290 + ((w) << 2))
#define WINDOW_OVERRIDE_CTRL(chan)  (0x2A0 + ((chan) << 2))
#define XOR_DEST_POINTER(chan)      (0x2B0 + (chan * 4))
#define XOR_BLOCK_SIZE(chan)        (0x2C0 + (chan * 4))
#define XOR_INIT_VALUE_LOW           0x2E0
#define XOR_INIT_VALUE_HIGH          0x2E4

static inline void mvebu_hwcc_armada375_sync_io_barrier_wa(void)
{
	int idx = smp_processor_id();

	/* Write '1' to the first word of the buffer */
	writel(0x1, coherency_wa_buf[idx]);

	/* Wait until the engine is idle */
	while ((readl(xor_base + XOR_ACTIVATION(idx)) >> 4) & 0x3)
		;

	dmb();

	/* Trigger channel */
	writel(0x1, xor_base + XOR_ACTIVATION(idx));

	/* Poll the data until it is cleared by the XOR transaction */
	while (readl(coherency_wa_buf[idx]))
		;
}

static void __init armada_375_coherency_init_wa(void)
{
	const struct mbus_dram_target_info *dram;
	struct device_node *xor_node;
	struct property *xor_status;
	struct clk *xor_clk;
	u32 win_enable = 0;
	int i;

	pr_warn("enabling coherency workaround for Armada 375 Z1, one XOR engine disabled\n");

	/*
	 * Since the workaround uses one XOR engine, we grab a
	 * reference to its Device Tree node first.
	 */
	xor_node = of_find_compatible_node(NULL, NULL, "marvell,orion-xor");
	BUG_ON(!xor_node);

	/*
	 * Then we mark it as disabled so that the real XOR driver
	 * will not use it.
	 */
	xor_status = kzalloc(sizeof(struct property), GFP_KERNEL);
	BUG_ON(!xor_status);

	xor_status->value = kstrdup("disabled", GFP_KERNEL);
	BUG_ON(!xor_status->value);

	xor_status->length = 8;
	xor_status->name = kstrdup("status", GFP_KERNEL);
	BUG_ON(!xor_status->name);

	of_update_property(xor_node, xor_status);

	/*
	 * And we remap the registers, get the clock, and do the
	 * initial configuration of the XOR engine.
	 */
	xor_base = of_iomap(xor_node, 0);
	xor_high_base = of_iomap(xor_node, 1);

	xor_clk = of_clk_get_by_name(xor_node, NULL);
	BUG_ON(!xor_clk);

	clk_prepare_enable(xor_clk);

	dram = mv_mbus_dram_info();

	for (i = 0; i < 8; i++) {
		writel(0, xor_base + WINDOW_BASE(i));
		writel(0, xor_base + WINDOW_SIZE(i));
		if (i < 4)
			writel(0, xor_base + WINDOW_REMAP_HIGH(i));
	}

	for (i = 0; i < dram->num_cs; i++) {
		const struct mbus_dram_window *cs = dram->cs + i;
		writel((cs->base & 0xffff0000) |
		       (cs->mbus_attr << 8) |
		       dram->mbus_dram_target_id, xor_base + WINDOW_BASE(i));
		writel((cs->size - 1) & 0xffff0000, xor_base + WINDOW_SIZE(i));

		win_enable |= (1 << i);
		win_enable |= 3 << (16 + (2 * i));
	}

	writel(win_enable, xor_base + WINDOW_BAR_ENABLE(0));
	writel(win_enable, xor_base + WINDOW_BAR_ENABLE(1));
	writel(0, xor_base + WINDOW_OVERRIDE_CTRL(0));
	writel(0, xor_base + WINDOW_OVERRIDE_CTRL(1));

	for (i = 0; i < CONFIG_NR_CPUS; i++) {
		coherency_wa_buf[i] = kzalloc(PAGE_SIZE, GFP_KERNEL);
		BUG_ON(!coherency_wa_buf[i]);

		/*
		 * We can't use the DMA mapping API, since we don't
		 * have a valid 'struct device' pointer
		 */
		coherency_wa_buf_phys[i] =
			virt_to_phys(coherency_wa_buf[i]);
		BUG_ON(!coherency_wa_buf_phys[i]);

		/*
		 * Configure the XOR engine for memset operation, with
		 * a 128 bytes block size
		 */
		writel(0x444, xor_base + XOR_CONFIG(i));
		writel(128, xor_base + XOR_BLOCK_SIZE(i));
		writel(coherency_wa_buf_phys[i],
		       xor_base + XOR_DEST_POINTER(i));
	}

	writel(0x0, xor_base + XOR_INIT_VALUE_LOW);
	writel(0x0, xor_base + XOR_INIT_VALUE_HIGH);

	coherency_wa_enabled = true;
}

static inline void mvebu_hwcc_sync_io_barrier(void)
{
	if (coherency_wa_enabled) {
		mvebu_hwcc_armada375_sync_io_barrier_wa();
		return;
	}

	writel(0x1, coherency_cpu_base + IO_SYNC_BARRIER_CTL_OFFSET);
	while (readl(coherency_cpu_base + IO_SYNC_BARRIER_CTL_OFFSET) & 0x1);
}

static dma_addr_t mvebu_hwcc_dma_map_page(struct device *dev, struct page *page,
				  unsigned long offset, size_t size,
				  enum dma_data_direction dir,
				  struct dma_attrs *attrs)
{
	if (dir != DMA_TO_DEVICE)
		mvebu_hwcc_sync_io_barrier();
	return pfn_to_dma(dev, page_to_pfn(page)) + offset;
}


static void mvebu_hwcc_dma_unmap_page(struct device *dev, dma_addr_t dma_handle,
			      size_t size, enum dma_data_direction dir,
			      struct dma_attrs *attrs)
{
	if (dir != DMA_TO_DEVICE)
		mvebu_hwcc_sync_io_barrier();
}

static void mvebu_hwcc_dma_sync(struct device *dev, dma_addr_t dma_handle,
			size_t size, enum dma_data_direction dir)
{
	if (dir != DMA_TO_DEVICE)
		mvebu_hwcc_sync_io_barrier();
}

static struct dma_map_ops mvebu_hwcc_dma_ops = {
	.alloc			= arm_dma_alloc,
	.free			= arm_dma_free,
	.mmap			= arm_dma_mmap,
	.map_page		= mvebu_hwcc_dma_map_page,
	.unmap_page		= mvebu_hwcc_dma_unmap_page,
	.get_sgtable		= arm_dma_get_sgtable,
	.map_sg			= arm_dma_map_sg,
	.unmap_sg		= arm_dma_unmap_sg,
	.sync_single_for_cpu	= mvebu_hwcc_dma_sync,
	.sync_single_for_device	= mvebu_hwcc_dma_sync,
	.sync_sg_for_cpu	= arm_dma_sync_sg_for_cpu,
	.sync_sg_for_device	= arm_dma_sync_sg_for_device,
	.set_dma_mask		= arm_dma_set_mask,
};

static int mvebu_hwcc_notifier(struct notifier_block *nb,
			       unsigned long event, void *__dev)
{
	struct device *dev = __dev;

	if (event != BUS_NOTIFY_ADD_DEVICE)
		return NOTIFY_DONE;
	set_dma_ops(dev, &mvebu_hwcc_dma_ops);

	return NOTIFY_OK;
}

static struct notifier_block mvebu_hwcc_nb = {
	.notifier_call = mvebu_hwcc_notifier,
};

static struct notifier_block mvebu_hwcc_pci_nb = {
	.notifier_call = mvebu_hwcc_notifier,
};

static void __init armada_370_coherency_init(struct device_node *np)
{
	struct resource res;

	of_address_to_resource(np, 0, &res);
	coherency_phys_base = res.start;
	/*
	 * Ensure secondary CPUs will see the updated value,
	 * which they read before they join the coherency
	 * fabric, and therefore before they are coherent with
	 * the boot CPU cache.
	 */
	sync_cache_w(&coherency_phys_base);
	coherency_base = of_iomap(np, 0);
	coherency_cpu_base = of_iomap(np, 1);
	set_cpu_coherent();
}

/*
 * This ioremap hook is used on Armada 375/38x to ensure that PCIe
 * memory areas are mapped as MT_UNCACHED instead of MT_DEVICE. This
 * is needed as a workaround for a deadlock issue between the PCIe
 * interface and the cache controller.
 */
static void __iomem *
armada_pcie_wa_ioremap_caller(phys_addr_t phys_addr, size_t size,
			      unsigned int mtype, void *caller)
{
	struct resource pcie_mem;

	mvebu_mbus_get_pcie_mem_aperture(&pcie_mem);

	if (pcie_mem.start <= phys_addr && (phys_addr + size) <= pcie_mem.end)
		mtype = MT_UNCACHED;

	return __arm_ioremap_caller(phys_addr, size, mtype, caller);
}

static void __init armada_375_380_coherency_init(struct device_node *np)
{
	struct device_node *cache_dn;

	coherency_cpu_base = of_iomap(np, 0);
	arch_ioremap_caller = armada_pcie_wa_ioremap_caller;

	/*
	 * Add the PL310 property "arm,io-coherent". This makes sure the
	 * outer sync operation is not used, which allows to
	 * workaround the system erratum that causes deadlocks when
	 * doing PCIe in an SMP situation on Armada 375 and Armada
	 * 38x.
	 */
	for_each_compatible_node(cache_dn, NULL, "arm,pl310-cache") {
		struct property *p;

		p = kzalloc(sizeof(*p), GFP_KERNEL);
		p->name = kstrdup("arm,io-coherent", GFP_KERNEL);
		of_add_property(cache_dn, p);
	}
}

static int coherency_type(void)
{
	struct device_node *np;
	const struct of_device_id *match;

	np = of_find_matching_node_and_match(NULL, of_coherency_table, &match);
	if (np) {
		int type = (int) match->data;

		/* Armada 370/XP coherency works in both UP and SMP */
		if (type == COHERENCY_FABRIC_TYPE_ARMADA_370_XP)
			return type;

		/* Armada 375 coherency works only on SMP */
		else if (type == COHERENCY_FABRIC_TYPE_ARMADA_375 && is_smp())
			return type;

		/* Armada 380 coherency works only on SMP */
		else if (type == COHERENCY_FABRIC_TYPE_ARMADA_380 && is_smp())
			return type;
	}

	return COHERENCY_FABRIC_TYPE_NONE;
}

int coherency_available(void)
{
	return coherency_type() != COHERENCY_FABRIC_TYPE_NONE;
}

int __init coherency_init(void)
{
	int type = coherency_type();
	struct device_node *np;

	np = of_find_matching_node(NULL, of_coherency_table);

	if (type == COHERENCY_FABRIC_TYPE_ARMADA_370_XP)
		armada_370_coherency_init(np);
	else if (type == COHERENCY_FABRIC_TYPE_ARMADA_375 ||
		 type == COHERENCY_FABRIC_TYPE_ARMADA_380)
		armada_375_380_coherency_init(np);

	of_node_put(np);

	return 0;
}

static int __init coherency_late_init(void)
{
	int type = coherency_type();

	if (type == COHERENCY_FABRIC_TYPE_NONE)
		return 0;

	if (type == COHERENCY_FABRIC_TYPE_ARMADA_375) {
		u32 dev, rev;

		if (mvebu_get_soc_id(&dev, &rev) == 0 &&
		    rev == ARMADA_375_Z1_REV)
			armada_375_coherency_init_wa();
	}

	bus_register_notifier(&platform_bus_type,
			      &mvebu_hwcc_nb);

	return 0;
}

postcore_initcall(coherency_late_init);

#if IS_ENABLED(CONFIG_PCI)
static int __init coherency_pci_init(void)
{
	if (coherency_available())
		bus_register_notifier(&pci_bus_type,
				       &mvebu_hwcc_pci_nb);
	return 0;
}

arch_initcall(coherency_pci_init);
#endif