1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
|
From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
From: Profpatsch <mail@profpatsch.de>
Subject: [PATCH 10/17] disp: e-ink buffers from the DMA API, not ION
disp allocated its e-ink buffers (image, index, working, mode and wave
data buffers, two copies of the waveform) from ION's carveout heap,
cached, and synced them with sunxi_ion_cache_sync(), i.e.
dmac_flush_range(). ION is gone from current kernels.
dma_alloc_coherent() would be the obvious replacement, but on ARM it is
uncached, and the CPU does real work on these buffers (index calculation,
libeink's REGAL processing): with it, the index step of a page turn went
from ~30 ms to 400-700 ms. So the pages and their IOMMU mapping come
from dma_alloc_attrs(DMA_ATTR_NO_KERNEL_MAPPING), the kernel gets a
cached mapping with vmap(), and disp_eink_cache_sync() cleans and
invalidates the pages in range with dma_sync_sg_for_device(...,
DMA_BIDIRECTIONAL), which on ARMv7 is one pass, like the vendor's flush.
The buffer list is under a spinlock, since the e-ink interrupt syncs too.
Measured with injected page turns, 32 each: index step 24-35 ms with
ION, 27-39 ms now. A full refresh of the same page is pixel-identical.
---
.../fbdev/sunxi/disp2/disp/de/disp_eink.h | 2 +-
.../video/fbdev/sunxi/disp2/disp/dev_disp.c | 340 +++++++-----------
.../video/fbdev/sunxi/disp2/disp/dev_disp.h | 25 +-
3 files changed, 132 insertions(+), 235 deletions(-)
diff --git a/drivers/video/fbdev/sunxi/disp2/disp/de/disp_eink.h b/drivers/video/fbdev/sunxi/disp2/disp/de/disp_eink.h
index 1ff759c2..09b566bb 100644
--- a/drivers/video/fbdev/sunxi/disp2/disp/de/disp_eink.h
+++ b/drivers/video/fbdev/sunxi/disp2/disp/de/disp_eink.h
@@ -13,7 +13,7 @@
#include "disp_private.h"
#include "include.h"
#include <video/sunxi_display2.h>
-#include <linux/ion_sunxi.h>
+#include <linux/dma-mapping.h>
//#define TEST_EDMA
diff --git a/drivers/video/fbdev/sunxi/disp2/disp/dev_disp.c b/drivers/video/fbdev/sunxi/disp2/disp/dev_disp.c
index 0b519653..e78d57ab 100755
--- a/drivers/video/fbdev/sunxi/disp2/disp/dev_disp.c
+++ b/drivers/video/fbdev/sunxi/disp2/disp/dev_disp.c
@@ -91,236 +91,150 @@ int hwc_dump(char *buf);
extern int gSleep_Mode_Suspend;
-#if defined(CONFIG_ION_SUNXI) && defined(DISP_ION_MEM)
-static int init_disp_ion_mgr(struct disp_ion_mgr *ion_mgr)
+/*
+ * Memory for the e-ink pipeline (image, index, working, mode and wave data
+ * buffers, the waveform): cached in the kernel, behind the IOMMU for the
+ * display engine, cache maintenance explicit via disp_eink_cache_sync().
+ *
+ * The vendor allocated these from ION's carveout heap with
+ * ION_FLAG_CACHED and synced with dmac_flush_range(). ION is gone from
+ * current kernels. dma_alloc_coherent() would do, but is uncached on ARM:
+ * the CPU-heavy steps (libeink's REGAL processing, index calculation) run
+ * on these buffers, and were measured 15-20x slower that way (25 ms to
+ * 400-700 ms per page turn). So: pages and IOMMU mapping from the DMA API
+ * without a kernel mapping, a cached kernel mapping by vmap(), and syncs
+ * through the DMA API on the same pages (disp_eink_mem_sync()).
+ */
+static void disp_eink_mem_init(struct disp_eink_mem_mgr *mgr)
{
- if (NULL == ion_mgr) {
- __wrn("input param is null\n");
- return -EINVAL;
- }
-
- mutex_init(&(ion_mgr->mlock));
-
- mutex_lock(&(ion_mgr->mlock));
- INIT_LIST_HEAD(&(ion_mgr->ion_list));
- ion_mgr->client = sunxi_ion_client_create((const char *)DISP_ION_NAME);
- if (IS_ERR_OR_NULL(ion_mgr->client)) {
- mutex_unlock(&(ion_mgr->mlock));
- __wrn("disp_ion client create failed!!");
- return -ENOMEM;
- } else {
- __debug("init ion manager for disp ok\n");
- }
- mutex_unlock(&(ion_mgr->mlock));
-
- return 0;
+ spin_lock_init(&mgr->lock);
+ INIT_LIST_HEAD(&mgr->list);
}
-static int __disp_ion_alloc_coherent(struct ion_client *client, struct disp_ion_mem *mem)
+static void *disp_eink_mem_alloc(u32 num_bytes, void *phys_addr)
{
- unsigned int flags = ION_FLAG_CACHED | ION_FLAG_CACHED_NEEDS_SYNC;
- struct scatterlist *sgl = NULL;
- int ret = -1;
-
- if (IS_ERR_OR_NULL(client) || (NULL == mem)) {
- __wrn("input param is null\n");
- return -1;
- }
+ struct disp_eink_mem_mgr *mgr = &g_disp_drv.eink_mem;
+ struct device *dev = g_disp_drv.dev;
+ struct disp_eink_mem *mem;
+ dma_addr_t iova;
+ unsigned int count;
+ unsigned long flags;
- mem->handle = ion_alloc(client, mem->size, PAGE_SIZE,
- (1<<ION_HEAP_TYPE_CARVEOUT) , flags);
- if (IS_ERR_OR_NULL(mem->handle)) {
- __wrn("ion_alloc failed, size=%d!\n", mem->size);
- return -2;
- }
+ mem = kzalloc(sizeof(*mem), GFP_KERNEL);
+ if (!mem)
+ return NULL;
+ mem->size = PAGE_ALIGN(num_bytes);
+ count = mem->size >> PAGE_SHIFT;
- mem->vaddr = ion_map_kernel(client, mem->handle);
- if (IS_ERR_OR_NULL(mem->vaddr)) {
- __wrn("ion_map_kernel failed!!\n");
- goto err_map_kernel;
+ mem->pages = dma_alloc_attrs(dev, mem->size, &iova, GFP_KERNEL,
+ DMA_ATTR_NO_KERNEL_MAPPING);
+ if (!mem->pages) {
+ __wrn("dma_alloc_attrs failed, size=%zu\n", mem->size);
+ goto err_mem;
}
+ mem->paddr = iova;
- #ifndef CONFIG_IOMMU_SUPPORT
- if(ion_phys(client, mem->handle, (ion_phys_addr_t *)&mem->paddr, &mem->size )){
- __wrn("ion_phys failed!!\n");
- goto err_phys;
- }
- #else
- sgl = mem->handle->buffer->sg_table->sgl;
- ret = dma_map_sg(g_disp_drv.dev, sgl, 1, DMA_BIDIRECTIONAL);
- if (1 != ret) {
- __wrn("dma map sg fail, ret=%d\n", ret);
- goto err_phys;
+ mem->vaddr = vmap(mem->pages, count, VM_MAP, PAGE_KERNEL);
+ if (!mem->vaddr) {
+ __wrn("vmap failed, size=%zu\n", mem->size);
+ goto err_dma;
}
- mem->paddr = sg_dma_address(sgl);
- #endif
-
- __debug("ion malloc size=0x%x, vaddr=0x%08x, paddr=0x%08x\n",
- mem->size, mem->vaddr, mem->paddr);
-
- return 0;
-
-err_phys:
- ion_unmap_kernel(client, mem->handle);
-err_map_kernel:
- ion_free(client, mem->handle);
- return -ENOMEM;
-}
-static void __disp_ion_free_coherent(struct ion_client *client, struct disp_ion_mem *mem)
-{
- if (IS_ERR_OR_NULL(client )||IS_ERR_OR_NULL(mem->handle)||IS_ERR_OR_NULL(mem->vaddr)) {
- __wrn("input param is null\n");
- return;
- }
-
- ion_unmap_kernel(client , mem->handle);
- ion_free(client , mem->handle);
- return;
-}
+ spin_lock_irqsave(&mgr->lock, flags);
+ list_add_tail(&mem->node, &mgr->list);
+ spin_unlock_irqrestore(&mgr->lock, flags);
-static void __disp_ion_dump(void)
-{
- struct disp_ion_mgr *ion_mgr = &(g_disp_drv.ion_mgr);
- struct ion_client *client = NULL;
- struct disp_ion_list_node *ion_node = NULL, *tmp_ion_node = NULL;
- struct disp_ion_mem *mem = NULL;
- unsigned int i = 0;
-
- if (NULL == ion_mgr) {
- __wrn("disp ion mgr is not initial yet\n");
- return;
- }
-
- client = ion_mgr->client;
-
- mutex_lock(&(ion_mgr->mlock));
- list_for_each_entry_safe(ion_node, tmp_ion_node, &ion_mgr->ion_list, node) {
- if (NULL != ion_node) {
- mem = &ion_node->mem;
- printk("ion node %d: vaddr=0x%08x, paddr=0x%08x, size=%d\n",
- i, mem->vaddr, mem->paddr, mem->size);
- i++;
- }
- }
- mutex_unlock(&(ion_mgr->mlock));
+ *(u32 *)phys_addr = (u32)mem->paddr;
+ return mem->vaddr;
- return ;
+err_dma:
+ dma_free_attrs(dev, mem->size, mem->pages, iova,
+ DMA_ATTR_NO_KERNEL_MAPPING);
+err_mem:
+ kfree(mem);
+ return NULL;
}
-static void *disp_ion_malloc(u32 num_bytes, void *phys_addr)
+static void __disp_eink_mem_free(struct disp_eink_mem *mem)
{
- struct disp_ion_mgr *ion_mgr = &(g_disp_drv.ion_mgr);
- struct ion_client *client = NULL;
- struct disp_ion_list_node *ion_node = NULL;
- struct disp_ion_mem *mem = NULL;
- u32 *paddr = NULL;
- int ret = -1;
-
- if (NULL == ion_mgr) {
- __wrn("disp ion manager has not initial yet\n");
- return NULL;
- }
-
- ion_node = kmalloc(sizeof(*ion_node), GFP_KERNEL);
- if (NULL == ion_node) {
- __wrn("fail to alloc ion node, size=%d\n", sizeof(*ion_node));
- return NULL;
- }
-
- mutex_lock(&(ion_mgr->mlock));
- client = ion_mgr->client;
- mem = &ion_node->mem;
- mem->size = MY_BYTE_ALIGN(num_bytes);
- ret = __disp_ion_alloc_coherent(client, mem);
- if (0 != ret) {
- __wrn("fail to alloc ion, ret=%d\n", ret);
- goto err_hdl;
- }
-
- paddr = (u32 *)phys_addr;
- *paddr = (u32)mem->paddr;
- list_add_tail(&(ion_node->node), &(ion_mgr->ion_list));
-
- mutex_unlock(&(ion_mgr->mlock));
-
- //for debug
- //__disp_ion_dump();
-
- return mem->vaddr;
-
-err_hdl:
- if (ion_node)
- kfree(ion_node);
- mutex_unlock(&(ion_mgr->mlock));
-
- return NULL;
+ vunmap(mem->vaddr);
+ dma_free_attrs(g_disp_drv.dev, mem->size, mem->pages, mem->paddr,
+ DMA_ATTR_NO_KERNEL_MAPPING);
+ kfree(mem);
}
-static void disp_ion_free(void *virt_addr, void* phys_addr, u32 num_bytes)
+static void disp_eink_mem_free(void *virt_addr, void *phys_addr, u32 num_bytes)
{
- struct disp_ion_mgr *ion_mgr = &(g_disp_drv.ion_mgr);
- struct ion_client *client = NULL;
- struct disp_ion_list_node *ion_node = NULL, *tmp_ion_node = NULL;
- struct disp_ion_mem *mem = NULL;
- bool found = false;
- int ret = -1;
-
- if (NULL == ion_mgr) {
- __wrn("disp ion manager has not initial yet\n");
- return;
- }
+ struct disp_eink_mem_mgr *mgr = &g_disp_drv.eink_mem;
+ struct disp_eink_mem *mem, *tmp;
+ unsigned long flags;
- client = ion_mgr->client;
-
- mutex_lock(&(ion_mgr->mlock));
- list_for_each_entry_safe(ion_node, tmp_ion_node, &ion_mgr->ion_list, node) {
- if (NULL != ion_node) {
- mem = &ion_node->mem;
- if ((((unsigned long)mem->paddr) == ((unsigned long)phys_addr)) &&
- (((unsigned long)mem->vaddr) == ((unsigned long)virt_addr))) {
- __disp_ion_free_coherent(client, mem);
- __list_del_entry(&(ion_node->node));
- found = true;
- break;
- }
+ spin_lock_irqsave(&mgr->lock, flags);
+ list_for_each_entry_safe(mem, tmp, &mgr->list, node) {
+ if (mem->vaddr == virt_addr &&
+ mem->paddr == (unsigned long)phys_addr) {
+ list_del(&mem->node);
+ spin_unlock_irqrestore(&mgr->lock, flags);
+ __disp_eink_mem_free(mem);
+ return;
}
}
- mutex_unlock(&(ion_mgr->mlock));
-
- if (false == found) {
- __wrn("vaddr=0x%08x, paddr=0x%08x is not found in ion\n", virt_addr, phys_addr);
- }
-
- return ;
+ spin_unlock_irqrestore(&mgr->lock, flags);
+ __wrn("vaddr=%p, paddr=%p is not an e-ink buffer\n", virt_addr,
+ phys_addr);
}
-static void deinit_disp_ion_mgr(struct disp_ion_mgr *ion_mgr)
+/*
+ * Write back and invalidate [start, end) of an e-ink buffer, as the
+ * vendor's dmac_flush_range() did: the pipeline calls this both after the
+ * CPU wrote a buffer and before it reads one the hardware wrote, without
+ * saying which, so both. On ARMv7, syncing for the device with
+ * DMA_BIDIRECTIONAL is clean + invalidate in one pass (dma_map_area with
+ * dir != DMA_FROM_DEVICE cleans, then invalidates lines that are clean
+ * anyway); syncing for the CPU as well would walk the range a second
+ * time. Only the pages in range, through a one-entry scatterlist on the
+ * stack: the interrupt handler syncs too.
+ *
+ * The DMA API maintains the cache by physical page (the kernel's linear
+ * map, or a kmap for highmem), not through vmap()'s alias the pipeline
+ * writes through; the Cortex-A7's data cache is physically indexed, so
+ * those are the same lines.
+ */
+static void disp_eink_mem_sync(const void *start, const void *end)
{
- struct disp_ion_list_node *ion_node = NULL, *tmp_ion_node = NULL;
- struct disp_ion_mem *mem = NULL;
- struct ion_client *client = NULL;
-
- if (NULL == ion_mgr) {
- __wrn("input param is null\n");
- return;
- }
+ struct disp_eink_mem_mgr *mgr = &g_disp_drv.eink_mem;
+ struct device *dev = g_disp_drv.dev;
+ struct disp_eink_mem *mem;
+ unsigned long flags;
- client = ion_mgr->client;
- mutex_lock(&(ion_mgr->mlock));
- list_for_each_entry_safe(ion_node, tmp_ion_node, &ion_mgr->ion_list, node) {
- if (NULL != ion_node) {
- //free all ion node
- mem = &ion_node->mem;
- __disp_ion_free_coherent(client, mem);
- __list_del_entry(&(ion_node->node));
- kfree(ion_node);
+ spin_lock_irqsave(&mgr->lock, flags);
+ list_for_each_entry(mem, &mgr->list, node) {
+ unsigned long off, len, first, last, i;
+ struct scatterlist sg;
+
+ if (start < mem->vaddr || start >= mem->vaddr + mem->size)
+ continue;
+ if (end > mem->vaddr + mem->size)
+ end = mem->vaddr + mem->size;
+ if (end <= start)
+ break;
+ off = start - mem->vaddr;
+ len = end - start;
+ first = off >> PAGE_SHIFT;
+ last = (off + len - 1) >> PAGE_SHIFT;
+ for (i = first; i <= last; i++) {
+ unsigned long pstart = i == first ? off & ~PAGE_MASK : 0;
+ unsigned long pend = i == last ?
+ ((off + len - 1) & ~PAGE_MASK) + 1 : PAGE_SIZE;
+
+ sg_init_table(&sg, 1);
+ sg_set_page(&sg, mem->pages[i], pend - pstart, pstart);
+ dma_sync_sg_for_device(dev, &sg, 1, DMA_BIDIRECTIONAL);
}
+ break;
}
- ion_client_destroy(client);
- mutex_unlock(&(ion_mgr->mlock));
+ spin_unlock_irqrestore(&mgr->lock, flags);
}
-#endif
static void disp_shutdown(struct platform_device *pdev);
static ssize_t disp_sys_show(struct device *dev,
@@ -1814,31 +1728,19 @@ void disp_free(void *virt_addr, void *phys_addr, u32 num_bytes)
void *disp_eink_malloc(u32 num_bytes, void *phys_addr)
{
-#if defined(CONFIG_ION_SUNXI) && defined(DISP_ION_MEM)
- return disp_ion_malloc(num_bytes, phys_addr);
-#else
- return disp_malloc(num_bytes, phys_addr);
-#endif
+ return disp_eink_mem_alloc(num_bytes, phys_addr);
}
EXPORT_SYMBOL(disp_eink_malloc);
void disp_eink_free(void *virt_addr, void* phys_addr, u32 num_bytes)
{
-#if defined(CONFIG_ION_SUNXI) && defined(DISP_ION_MEM)
- return disp_ion_free(virt_addr, phys_addr, num_bytes);
-#else
- return disp_free(virt_addr, phys_addr, num_bytes);
-#endif
+ disp_eink_mem_free(virt_addr, phys_addr, num_bytes);
}
EXPORT_SYMBOL(disp_eink_free);
void disp_eink_cache_sync(const void *start_addr, const void *end_addr)
{
-#if defined(CONFIG_ION_SUNXI) && defined(DISP_ION_MEM)
- return sunxi_ion_cache_sync(start_addr, end_addr);
-#else
- return;
-#endif
+ disp_eink_mem_sync(start_addr, end_addr);
}
EXPORT_SYMBOL(disp_eink_cache_sync);
@@ -2756,9 +2658,7 @@ static int disp_probe(struct platform_device *pdev)
counter++;
#endif
-#if defined(CONFIG_ION_SUNXI) && defined(DISP_ION_MEM)
- init_disp_ion_mgr(&g_disp_drv.ion_mgr);
-#endif
+ disp_eink_mem_init(&g_disp_drv.eink_mem);
disp_init(pdev);
ret = sysfs_create_group(&display_dev->kobj, &disp_attribute_group);
diff --git a/drivers/video/fbdev/sunxi/disp2/disp/dev_disp.h b/drivers/video/fbdev/sunxi/disp2/disp/dev_disp.h
index 23f00a64..1651bd04 100755
--- a/drivers/video/fbdev/sunxi/disp2/disp/dev_disp.h
+++ b/drivers/video/fbdev/sunxi/disp2/disp/dev_disp.h
@@ -21,7 +21,6 @@
#ifdef CONFIG_ION_SUNXI
//#include "../../../../../staging/android/ion/ion.h"
-#include <linux/ion_sunxi.h> //for import global variable "sunxi_ion_client_create"
#include <linux/dma-mapping.h> //just include"PAGE_SIZE" macro
#define FB_RESERVED_MEM
#define DISP_ION_NAME "disp_ion"
@@ -86,23 +85,23 @@ struct disp_init_para {
unsigned int chn_cfg_mode;
};
-struct disp_ion_mem {
+/*
+ * A buffer the e-ink pipeline shares with the hardware: pages mapped cached
+ * into the kernel (vaddr) and through the IOMMU for the display engine
+ * (paddr, an IO virtual address). See disp_eink_malloc().
+ */
+struct disp_eink_mem {
void *vaddr;
unsigned long paddr;
size_t size;
- struct ion_handle *handle;
-};
-
-struct disp_ion_list_node {
- struct disp_ion_mem mem;
+ struct page **pages;
struct list_head node;
};
-struct disp_ion_mgr {
- struct mutex mlock;
- struct ion_client *client;
- struct list_head ion_list;
+struct disp_eink_mem_mgr {
+ spinlock_t lock; /* syncs run in the e-ink interrupt */
+ struct list_head list;
};
@@ -128,9 +127,7 @@ struct disp_drv_info {
bool b_lcd_enabled[DISP_SCREEN_NUM];
bool inited; /* indicate driver if init */
struct disp_bsp_init_para para;
-#if defined(CONFIG_ION_SUNXI) && defined(DISP_ION_MEM)
- struct disp_ion_mgr ion_mgr;
-#endif
+ struct disp_eink_mem_mgr eink_mem;
};
|