1 |
In case you missed it. |
2 |
|
3 |
-------- Forwarded Message -------- |
4 |
From: David Miller <davem@×××××××××.net> |
5 |
To: joel.bertrand@××××××××.fr |
6 |
Cc: mt1@××××××××.fr, sparclinux@×××××××××××.org |
7 |
Subject: Re: [Repost] Ultra2 SMP freezes with heavy disk I/O |
8 |
Date: Wed, 11 Apr 2007 21:55:11 -0700 (PDT) |
9 |
|
10 |
From: David Miller <davem@×××××××××.net> |
11 |
Date: Tue, 10 Apr 2007 14:04:58 -0700 (PDT) |
12 |
|
13 |
> From: BERTRAND Joël <joel.bertrand@××××××××.fr> |
14 |
> Date: Tue, 20 Mar 2007 21:53:26 +0100 |
15 |
> |
16 |
> > I have tried the last 2.6.21-rc4. Same bug. |
17 |
> |
18 |
> The good news is that I can reproduce this problem now on my |
19 |
> Ultra2, I'll try to figure out what's wrong. |
20 |
> |
21 |
> Running "dbench 32" a few time is enough to trigger it. |
22 |
|
23 |
Ok, I think I might have killed at least the bug I was able to hit. |
24 |
|
25 |
I make no promises that this will fix the issues everyone else was |
26 |
seeing, but let's cross our fingers :-))) |
27 |
|
28 |
Please give the following patch some testing if you can. And please |
29 |
be careful, messing with the DMA mapping code can do things like eat |
30 |
your disk :-) |
31 |
|
32 |
If all goes well I'll push this bug fix around, and I have a secret |
33 |
treat for your poor sparc64 SBUS users once we get past this. |
34 |
|
35 |
Thanks! |
36 |
|
37 |
commit af822e612593c1feef5052e685bca0f8d087d120 |
38 |
Author: David S. Miller <davem@××××××××××××××××.net> |
39 |
Date: Wed Apr 11 21:38:45 2007 -0700 |
40 |
|
41 |
[SPARC64]: Fix SBUS IOMMU allocation code. |
42 |
|
43 |
There are several IOMMU allocator bugs. Instead of trying to fix this |
44 |
overly complicated code, just mirror the PCI IOMMU arena allocator |
45 |
which is very stable and well stress tested. |
46 |
|
47 |
I tried to make the code as identical as possible so we can switch |
48 |
sun4u PCI and SBUS over to a common piece of IOMMU code. All that |
49 |
will be need are two callbacks, one to do a full IOMMU flush and one |
50 |
to do a streaming buffer flush. |
51 |
|
52 |
This patch gets rid of a lot of hangs and mysterious crashes on SBUS |
53 |
sparc64 systems, at least for me. |
54 |
|
55 |
Signed-off-by: David S. Miller <davem@×××××××××.net> |
56 |
|
57 |
diff --git a/arch/sparc64/kernel/sbus.c b/arch/sparc64/kernel/sbus.c |
58 |
index 01d6d86..14f78fb 100644 |
59 |
--- a/arch/sparc64/kernel/sbus.c |
60 |
+++ b/arch/sparc64/kernel/sbus.c |
61 |
@@ -24,48 +24,25 @@ |
62 |
|
63 |
#include "iommu_common.h" |
64 |
|
65 |
-/* These should be allocated on an SMP_CACHE_BYTES |
66 |
- * aligned boundary for optimal performance. |
67 |
- * |
68 |
- * On SYSIO, using an 8K page size we have 1GB of SBUS |
69 |
- * DMA space mapped. We divide this space into equally |
70 |
- * sized clusters. We allocate a DMA mapping from the |
71 |
- * cluster that matches the order of the allocation, or |
72 |
- * if the order is greater than the number of clusters, |
73 |
- * we try to allocate from the last cluster. |
74 |
- */ |
75 |
- |
76 |
-#define NCLUSTERS 8UL |
77 |
-#define ONE_GIG (1UL * 1024UL * 1024UL * 1024UL) |
78 |
-#define CLUSTER_SIZE (ONE_GIG / NCLUSTERS) |
79 |
-#define CLUSTER_MASK (CLUSTER_SIZE - 1) |
80 |
-#define CLUSTER_NPAGES (CLUSTER_SIZE >> IO_PAGE_SHIFT) |
81 |
#define MAP_BASE ((u32)0xc0000000) |
82 |
|
83 |
+struct sbus_iommu_arena { |
84 |
+ unsigned long *map; |
85 |
+ unsigned int hint; |
86 |
+ unsigned int limit; |
87 |
+}; |
88 |
+ |
89 |
struct sbus_iommu { |
90 |
-/*0x00*/spinlock_t lock; |
91 |
+ spinlock_t lock; |
92 |
|
93 |
-/*0x08*/iopte_t *page_table; |
94 |
-/*0x10*/unsigned long strbuf_regs; |
95 |
-/*0x18*/unsigned long iommu_regs; |
96 |
-/*0x20*/unsigned long sbus_control_reg; |
97 |
+ struct sbus_iommu_arena arena; |
98 |
|
99 |
-/*0x28*/volatile unsigned long strbuf_flushflag; |
100 |
+ iopte_t *page_table; |
101 |
+ unsigned long strbuf_regs; |
102 |
+ unsigned long iommu_regs; |
103 |
+ unsigned long sbus_control_reg; |
104 |
|
105 |
- /* If NCLUSTERS is ever decresed to 4 or lower, |
106 |
- * you must increase the size of the type of |
107 |
- * these counters. You have been duly warned. -DaveM |
108 |
- */ |
109 |
-/*0x30*/struct { |
110 |
- u16 next; |
111 |
- u16 flush; |
112 |
- } alloc_info[NCLUSTERS]; |
113 |
- |
114 |
- /* The lowest used consistent mapping entry. Since |
115 |
- * we allocate consistent maps out of cluster 0 this |
116 |
- * is relative to the beginning of closter 0. |
117 |
- */ |
118 |
-/*0x50*/u32 lowest_consistent_map; |
119 |
+ volatile unsigned long strbuf_flushflag; |
120 |
}; |
121 |
|
122 |
/* Offsets from iommu_regs */ |
123 |
@@ -91,19 +68,6 @@ static void __iommu_flushall(struct sbus_iommu *iommu) |
124 |
tag += 8UL; |
125 |
} |
126 |
upa_readq(iommu->sbus_control_reg); |
127 |
- |
128 |
- for (entry = 0; entry < NCLUSTERS; entry++) { |
129 |
- iommu->alloc_info[entry].flush = |
130 |
- iommu->alloc_info[entry].next; |
131 |
- } |
132 |
-} |
133 |
- |
134 |
-static void iommu_flush(struct sbus_iommu *iommu, u32 base, unsigned long npages) |
135 |
-{ |
136 |
- while (npages--) |
137 |
- upa_writeq(base + (npages << IO_PAGE_SHIFT), |
138 |
- iommu->iommu_regs + IOMMU_FLUSH); |
139 |
- upa_readq(iommu->sbus_control_reg); |
140 |
} |
141 |
|
142 |
/* Offsets from strbuf_regs */ |
143 |
@@ -156,178 +120,115 @@ static void sbus_strbuf_flush(struct sbus_iommu *iommu, u32 base, unsigned long |
144 |
base, npages); |
145 |
} |
146 |
|
147 |
-static iopte_t *alloc_streaming_cluster(struct sbus_iommu *iommu, unsigned long npages) |
148 |
+/* Based largely upon the ppc64 iommu allocator. */ |
149 |
+static long sbus_arena_alloc(struct sbus_iommu *iommu, unsigned long npages) |
150 |
{ |
151 |
- iopte_t *iopte, *limit, *first, *cluster; |
152 |
- unsigned long cnum, ent, nent, flush_point, found; |
153 |
- |
154 |
- cnum = 0; |
155 |
- nent = 1; |
156 |
- while ((1UL << cnum) < npages) |
157 |
- cnum++; |
158 |
- if(cnum >= NCLUSTERS) { |
159 |
- nent = 1UL << (cnum - NCLUSTERS); |
160 |
- cnum = NCLUSTERS - 1; |
161 |
- } |
162 |
- iopte = iommu->page_table + (cnum * CLUSTER_NPAGES); |
163 |
- |
164 |
- if (cnum == 0) |
165 |
- limit = (iommu->page_table + |
166 |
- iommu->lowest_consistent_map); |
167 |
- else |
168 |
- limit = (iopte + CLUSTER_NPAGES); |
169 |
- |
170 |
- iopte += ((ent = iommu->alloc_info[cnum].next) << cnum); |
171 |
- flush_point = iommu->alloc_info[cnum].flush; |
172 |
- |
173 |
- first = iopte; |
174 |
- cluster = NULL; |
175 |
- found = 0; |
176 |
- for (;;) { |
177 |
- if (iopte_val(*iopte) == 0UL) { |
178 |
- found++; |
179 |
- if (!cluster) |
180 |
- cluster = iopte; |
181 |
+ struct sbus_iommu_arena *arena = &iommu->arena; |
182 |
+ unsigned long n, i, start, end, limit; |
183 |
+ int pass; |
184 |
+ |
185 |
+ limit = arena->limit; |
186 |
+ start = arena->hint; |
187 |
+ pass = 0; |
188 |
+ |
189 |
+again: |
190 |
+ n = find_next_zero_bit(arena->map, limit, start); |
191 |
+ end = n + npages; |
192 |
+ if (unlikely(end >= limit)) { |
193 |
+ if (likely(pass < 1)) { |
194 |
+ limit = start; |
195 |
+ start = 0; |
196 |
+ __iommu_flushall(iommu); |
197 |
+ pass++; |
198 |
+ goto again; |
199 |
} else { |
200 |
- /* Used cluster in the way */ |
201 |
- cluster = NULL; |
202 |
- found = 0; |
203 |
+ /* Scanned the whole thing, give up. */ |
204 |
+ return -1; |
205 |
} |
206 |
+ } |
207 |
|
208 |
- if (found == nent) |
209 |
- break; |
210 |
- |
211 |
- iopte += (1 << cnum); |
212 |
- ent++; |
213 |
- if (iopte >= limit) { |
214 |
- iopte = (iommu->page_table + (cnum * CLUSTER_NPAGES)); |
215 |
- ent = 0; |
216 |
- |
217 |
- /* Multiple cluster allocations must not wrap */ |
218 |
- cluster = NULL; |
219 |
- found = 0; |
220 |
+ for (i = n; i < end; i++) { |
221 |
+ if (test_bit(i, arena->map)) { |
222 |
+ start = i + 1; |
223 |
+ goto again; |
224 |
} |
225 |
- if (ent == flush_point) |
226 |
- __iommu_flushall(iommu); |
227 |
- if (iopte == first) |
228 |
- goto bad; |
229 |
} |
230 |
|
231 |
- /* ent/iopte points to the last cluster entry we're going to use, |
232 |
- * so save our place for the next allocation. |
233 |
- */ |
234 |
- if ((iopte + (1 << cnum)) >= limit) |
235 |
- ent = 0; |
236 |
- else |
237 |
- ent = ent + 1; |
238 |
- iommu->alloc_info[cnum].next = ent; |
239 |
- if (ent == flush_point) |
240 |
- __iommu_flushall(iommu); |
241 |
- |
242 |
- /* I've got your streaming cluster right here buddy boy... */ |
243 |
- return cluster; |
244 |
- |
245 |
-bad: |
246 |
- printk(KERN_EMERG "sbus: alloc_streaming_cluster of npages(%ld) failed!\n", |
247 |
- npages); |
248 |
- return NULL; |
249 |
+ for (i = n; i < end; i++) |
250 |
+ __set_bit(i, arena->map); |
251 |
+ |
252 |
+ arena->hint = end; |
253 |
+ |
254 |
+ return n; |
255 |
} |
256 |
|
257 |
-static void free_streaming_cluster(struct sbus_iommu *iommu, u32 base, unsigned long npages) |
258 |
+static void sbus_arena_free(struct sbus_iommu_arena *arena, unsigned long base, unsigned long npages) |
259 |
{ |
260 |
- unsigned long cnum, ent, nent; |
261 |
- iopte_t *iopte; |
262 |
+ unsigned long i; |
263 |
|
264 |
- cnum = 0; |
265 |
- nent = 1; |
266 |
- while ((1UL << cnum) < npages) |
267 |
- cnum++; |
268 |
- if(cnum >= NCLUSTERS) { |
269 |
- nent = 1UL << (cnum - NCLUSTERS); |
270 |
- cnum = NCLUSTERS - 1; |
271 |
- } |
272 |
- ent = (base & CLUSTER_MASK) >> (IO_PAGE_SHIFT + cnum); |
273 |
- iopte = iommu->page_table + ((base - MAP_BASE) >> IO_PAGE_SHIFT); |
274 |
- do { |
275 |
- iopte_val(*iopte) = 0UL; |
276 |
- iopte += 1 << cnum; |
277 |
- } while(--nent); |
278 |
- |
279 |
- /* If the global flush might not have caught this entry, |
280 |
- * adjust the flush point such that we will flush before |
281 |
- * ever trying to reuse it. |
282 |
- */ |
283 |
-#define between(X,Y,Z) (((Z) - (Y)) >= ((X) - (Y))) |
284 |
- if (between(ent, iommu->alloc_info[cnum].next, iommu->alloc_info[cnum].flush)) |
285 |
- iommu->alloc_info[cnum].flush = ent; |
286 |
-#undef between |
287 |
+ for (i = base; i < (base + npages); i++) |
288 |
+ __clear_bit(i, arena->map); |
289 |
} |
290 |
|
291 |
-/* We allocate consistent mappings from the end of cluster zero. */ |
292 |
-static iopte_t *alloc_consistent_cluster(struct sbus_iommu *iommu, unsigned long npages) |
293 |
+static void sbus_iommu_table_init(struct sbus_iommu *iommu, unsigned int tsbsize) |
294 |
{ |
295 |
- iopte_t *iopte; |
296 |
+ unsigned long tsbbase, order, sz, num_tsb_entries; |
297 |
|
298 |
- iopte = iommu->page_table + (1 * CLUSTER_NPAGES); |
299 |
- while (iopte > iommu->page_table) { |
300 |
- iopte--; |
301 |
- if (!(iopte_val(*iopte) & IOPTE_VALID)) { |
302 |
- unsigned long tmp = npages; |
303 |
+ num_tsb_entries = tsbsize / sizeof(iopte_t); |
304 |
|
305 |
- while (--tmp) { |
306 |
- iopte--; |
307 |
- if (iopte_val(*iopte) & IOPTE_VALID) |
308 |
- break; |
309 |
- } |
310 |
- if (tmp == 0) { |
311 |
- u32 entry = (iopte - iommu->page_table); |
312 |
+ /* Setup initial software IOMMU state. */ |
313 |
+ spin_lock_init(&iommu->lock); |
314 |
|
315 |
- if (entry < iommu->lowest_consistent_map) |
316 |
- iommu->lowest_consistent_map = entry; |
317 |
- return iopte; |
318 |
- } |
319 |
- } |
320 |
+ /* Allocate and initialize the free area map. */ |
321 |
+ sz = num_tsb_entries / 8; |
322 |
+ sz = (sz + 7UL) & ~7UL; |
323 |
+ iommu->arena.map = kzalloc(sz, GFP_KERNEL); |
324 |
+ if (!iommu->arena.map) { |
325 |
+ prom_printf("PCI_IOMMU: Error, kmalloc(arena.map) failed.\n"); |
326 |
+ prom_halt(); |
327 |
+ } |
328 |
+ iommu->arena.limit = num_tsb_entries; |
329 |
+ |
330 |
+ /* Now allocate and setup the IOMMU page table itself. */ |
331 |
+ order = get_order(tsbsize); |
332 |
+ tsbbase = __get_free_pages(GFP_KERNEL, order); |
333 |
+ if (!tsbbase) { |
334 |
+ prom_printf("IOMMU: Error, gfp(tsb) failed.\n"); |
335 |
+ prom_halt(); |
336 |
} |
337 |
- return NULL; |
338 |
+ iommu->page_table = (iopte_t *)tsbbase; |
339 |
+ memset(iommu->page_table, 0, tsbsize); |
340 |
} |
341 |
|
342 |
-static void free_consistent_cluster(struct sbus_iommu *iommu, u32 base, unsigned long npages) |
343 |
+static inline iopte_t *alloc_npages(struct sbus_iommu *iommu, unsigned long npages) |
344 |
{ |
345 |
- iopte_t *iopte = iommu->page_table + ((base - MAP_BASE) >> IO_PAGE_SHIFT); |
346 |
+ long entry; |
347 |
|
348 |
- if ((iopte - iommu->page_table) == iommu->lowest_consistent_map) { |
349 |
- iopte_t *walk = iopte + npages; |
350 |
- iopte_t *limit; |
351 |
+ entry = sbus_arena_alloc(iommu, npages); |
352 |
+ if (unlikely(entry < 0)) |
353 |
+ return NULL; |
354 |
|
355 |
- limit = iommu->page_table + CLUSTER_NPAGES; |
356 |
- while (walk < limit) { |
357 |
- if (iopte_val(*walk) != 0UL) |
358 |
- break; |
359 |
- walk++; |
360 |
- } |
361 |
- iommu->lowest_consistent_map = |
362 |
- (walk - iommu->page_table); |
363 |
- } |
364 |
+ return iommu->page_table + entry; |
365 |
+} |
366 |
|
367 |
- while (npages--) |
368 |
- *iopte++ = __iopte(0UL); |
369 |
+static inline void free_npages(struct sbus_iommu *iommu, dma_addr_t base, unsigned long npages) |
370 |
+{ |
371 |
+ sbus_arena_free(&iommu->arena, base >> IO_PAGE_SHIFT, npages); |
372 |
} |
373 |
|
374 |
void *sbus_alloc_consistent(struct sbus_dev *sdev, size_t size, dma_addr_t *dvma_addr) |
375 |
{ |
376 |
- unsigned long order, first_page, flags; |
377 |
struct sbus_iommu *iommu; |
378 |
iopte_t *iopte; |
379 |
+ unsigned long flags, order, first_page; |
380 |
void *ret; |
381 |
int npages; |
382 |
|
383 |
- if (size <= 0 || sdev == NULL || dvma_addr == NULL) |
384 |
- return NULL; |
385 |
- |
386 |
size = IO_PAGE_ALIGN(size); |
387 |
order = get_order(size); |
388 |
if (order >= 10) |
389 |
return NULL; |
390 |
+ |
391 |
first_page = __get_free_pages(GFP_KERNEL|__GFP_COMP, order); |
392 |
if (first_page == 0UL) |
393 |
return NULL; |
394 |
@@ -336,108 +237,121 @@ void *sbus_alloc_consistent(struct sbus_dev *sdev, size_t size, dma_addr_t *dvma |
395 |
iommu = sdev->bus->iommu; |
396 |
|
397 |
spin_lock_irqsave(&iommu->lock, flags); |
398 |
- iopte = alloc_consistent_cluster(iommu, size >> IO_PAGE_SHIFT); |
399 |
- if (iopte == NULL) { |
400 |
- spin_unlock_irqrestore(&iommu->lock, flags); |
401 |
+ iopte = alloc_npages(iommu, size >> IO_PAGE_SHIFT); |
402 |
+ spin_unlock_irqrestore(&iommu->lock, flags); |
403 |
+ |
404 |
+ if (unlikely(iopte == NULL)) { |
405 |
free_pages(first_page, order); |
406 |
return NULL; |
407 |
} |
408 |
|
409 |
- /* Ok, we're committed at this point. */ |
410 |
- *dvma_addr = MAP_BASE + ((iopte - iommu->page_table) << IO_PAGE_SHIFT); |
411 |
+ *dvma_addr = (MAP_BASE + |
412 |
+ ((iopte - iommu->page_table) << IO_PAGE_SHIFT)); |
413 |
ret = (void *) first_page; |
414 |
npages = size >> IO_PAGE_SHIFT; |
415 |
+ first_page = __pa(first_page); |
416 |
while (npages--) { |
417 |
- *iopte++ = __iopte(IOPTE_VALID | IOPTE_CACHE | IOPTE_WRITE | |
418 |
- (__pa(first_page) & IOPTE_PAGE)); |
419 |
+ iopte_val(*iopte) = (IOPTE_VALID | IOPTE_CACHE | |
420 |
+ IOPTE_WRITE | |
421 |
+ (first_page & IOPTE_PAGE)); |
422 |
+ iopte++; |
423 |
first_page += IO_PAGE_SIZE; |
424 |
} |
425 |
- iommu_flush(iommu, *dvma_addr, size >> IO_PAGE_SHIFT); |
426 |
- spin_unlock_irqrestore(&iommu->lock, flags); |
427 |
|
428 |
return ret; |
429 |
} |
430 |
|
431 |
void sbus_free_consistent(struct sbus_dev *sdev, size_t size, void *cpu, dma_addr_t dvma) |
432 |
{ |
433 |
- unsigned long order, npages; |
434 |
struct sbus_iommu *iommu; |
435 |
- |
436 |
- if (size <= 0 || sdev == NULL || cpu == NULL) |
437 |
- return; |
438 |
+ iopte_t *iopte; |
439 |
+ unsigned long flags, order, npages; |
440 |
|
441 |
npages = IO_PAGE_ALIGN(size) >> IO_PAGE_SHIFT; |
442 |
iommu = sdev->bus->iommu; |
443 |
+ iopte = iommu->page_table + |
444 |
+ ((dvma - MAP_BASE) >> IO_PAGE_SHIFT); |
445 |
+ |
446 |
+ spin_lock_irqsave(&iommu->lock, flags); |
447 |
+ |
448 |
+ free_npages(iommu, dvma - MAP_BASE, npages); |
449 |
|
450 |
- spin_lock_irq(&iommu->lock); |
451 |
- free_consistent_cluster(iommu, dvma, npages); |
452 |
- iommu_flush(iommu, dvma, npages); |
453 |
- spin_unlock_irq(&iommu->lock); |
454 |
+ spin_unlock_irqrestore(&iommu->lock, flags); |
455 |
|
456 |
order = get_order(size); |
457 |
if (order < 10) |
458 |
free_pages((unsigned long)cpu, order); |
459 |
} |
460 |
|
461 |
-dma_addr_t sbus_map_single(struct sbus_dev *sdev, void *ptr, size_t size, int dir) |
462 |
+dma_addr_t sbus_map_single(struct sbus_dev *sdev, void *ptr, size_t sz, int direction) |
463 |
{ |
464 |
- struct sbus_iommu *iommu = sdev->bus->iommu; |
465 |
- unsigned long npages, pbase, flags; |
466 |
- iopte_t *iopte; |
467 |
- u32 dma_base, offset; |
468 |
- unsigned long iopte_bits; |
469 |
+ struct sbus_iommu *iommu; |
470 |
+ iopte_t *base; |
471 |
+ unsigned long flags, npages, oaddr; |
472 |
+ unsigned long i, base_paddr; |
473 |
+ u32 bus_addr, ret; |
474 |
+ unsigned long iopte_protection; |
475 |
+ |
476 |
+ iommu = sdev->bus->iommu; |
477 |
|
478 |
- if (dir == SBUS_DMA_NONE) |
479 |
+ if (unlikely(direction == SBUS_DMA_NONE)) |
480 |
BUG(); |
481 |
|
482 |
- pbase = (unsigned long) ptr; |
483 |
- offset = (u32) (pbase & ~IO_PAGE_MASK); |
484 |
- size = (IO_PAGE_ALIGN(pbase + size) - (pbase & IO_PAGE_MASK)); |
485 |
- pbase = (unsigned long) __pa(pbase & IO_PAGE_MASK); |
486 |
+ oaddr = (unsigned long)ptr; |
487 |
+ npages = IO_PAGE_ALIGN(oaddr + sz) - (oaddr & IO_PAGE_MASK); |
488 |
+ npages >>= IO_PAGE_SHIFT; |
489 |
|
490 |
spin_lock_irqsave(&iommu->lock, flags); |
491 |
- npages = size >> IO_PAGE_SHIFT; |
492 |
- iopte = alloc_streaming_cluster(iommu, npages); |
493 |
- if (iopte == NULL) |
494 |
- goto bad; |
495 |
- dma_base = MAP_BASE + ((iopte - iommu->page_table) << IO_PAGE_SHIFT); |
496 |
- npages = size >> IO_PAGE_SHIFT; |
497 |
- iopte_bits = IOPTE_VALID | IOPTE_STBUF | IOPTE_CACHE; |
498 |
- if (dir != SBUS_DMA_TODEVICE) |
499 |
- iopte_bits |= IOPTE_WRITE; |
500 |
- while (npages--) { |
501 |
- *iopte++ = __iopte(iopte_bits | (pbase & IOPTE_PAGE)); |
502 |
- pbase += IO_PAGE_SIZE; |
503 |
- } |
504 |
- npages = size >> IO_PAGE_SHIFT; |
505 |
+ base = alloc_npages(iommu, npages); |
506 |
spin_unlock_irqrestore(&iommu->lock, flags); |
507 |
|
508 |
- return (dma_base | offset); |
509 |
+ if (unlikely(!base)) |
510 |
+ BUG(); |
511 |
|
512 |
-bad: |
513 |
- spin_unlock_irqrestore(&iommu->lock, flags); |
514 |
- BUG(); |
515 |
- return 0; |
516 |
+ bus_addr = (MAP_BASE + |
517 |
+ ((base - iommu->page_table) << IO_PAGE_SHIFT)); |
518 |
+ ret = bus_addr | (oaddr & ~IO_PAGE_MASK); |
519 |
+ base_paddr = __pa(oaddr & IO_PAGE_MASK); |
520 |
+ |
521 |
+ iopte_protection = IOPTE_VALID | IOPTE_STBUF | IOPTE_CACHE; |
522 |
+ if (direction != SBUS_DMA_TODEVICE) |
523 |
+ iopte_protection |= IOPTE_WRITE; |
524 |
+ |
525 |
+ for (i = 0; i < npages; i++, base++, base_paddr += IO_PAGE_SIZE) |
526 |
+ iopte_val(*base) = iopte_protection | base_paddr; |
527 |
+ |
528 |
+ return ret; |
529 |
} |
530 |
|
531 |
-void sbus_unmap_single(struct sbus_dev *sdev, dma_addr_t dma_addr, size_t size, int direction) |
532 |
+void sbus_unmap_single(struct sbus_dev *sdev, dma_addr_t bus_addr, size_t sz, int direction) |
533 |
{ |
534 |
struct sbus_iommu *iommu = sdev->bus->iommu; |
535 |
- u32 dma_base = dma_addr & IO_PAGE_MASK; |
536 |
- unsigned long flags; |
537 |
+ iopte_t *base; |
538 |
+ unsigned long flags, npages, i; |
539 |
+ |
540 |
+ if (unlikely(direction == SBUS_DMA_NONE)) |
541 |
+ BUG(); |
542 |
+ |
543 |
+ npages = IO_PAGE_ALIGN(bus_addr + sz) - (bus_addr & IO_PAGE_MASK); |
544 |
+ npages >>= IO_PAGE_SHIFT; |
545 |
+ base = iommu->page_table + |
546 |
+ ((bus_addr - MAP_BASE) >> IO_PAGE_SHIFT); |
547 |
|
548 |
- size = (IO_PAGE_ALIGN(dma_addr + size) - dma_base); |
549 |
+ bus_addr &= IO_PAGE_MASK; |
550 |
|
551 |
spin_lock_irqsave(&iommu->lock, flags); |
552 |
- free_streaming_cluster(iommu, dma_base, size >> IO_PAGE_SHIFT); |
553 |
- sbus_strbuf_flush(iommu, dma_base, size >> IO_PAGE_SHIFT, direction); |
554 |
+ sbus_strbuf_flush(iommu, bus_addr, npages, direction); |
555 |
+ for (i = 0; i < npages; i++) |
556 |
+ iopte_val(base[i]) = 0UL; |
557 |
+ free_npages(iommu, bus_addr - MAP_BASE, npages); |
558 |
spin_unlock_irqrestore(&iommu->lock, flags); |
559 |
} |
560 |
|
561 |
#define SG_ENT_PHYS_ADDRESS(SG) \ |
562 |
(__pa(page_address((SG)->page)) + (SG)->offset) |
563 |
|
564 |
-static inline void fill_sg(iopte_t *iopte, struct scatterlist *sg, int nused, int nelems, unsigned long iopte_bits) |
565 |
+static inline void fill_sg(iopte_t *iopte, struct scatterlist *sg, |
566 |
+ int nused, int nelems, unsigned long iopte_protection) |
567 |
{ |
568 |
struct scatterlist *dma_sg = sg; |
569 |
struct scatterlist *sg_end = sg + nelems; |
570 |
@@ -462,7 +376,7 @@ static inline void fill_sg(iopte_t *iopte, struct scatterlist *sg, int nused, in |
571 |
for (;;) { |
572 |
unsigned long tmp; |
573 |
|
574 |
- tmp = (unsigned long) SG_ENT_PHYS_ADDRESS(sg); |
575 |
+ tmp = SG_ENT_PHYS_ADDRESS(sg); |
576 |
len = sg->length; |
577 |
if (((tmp ^ pteval) >> IO_PAGE_SHIFT) != 0UL) { |
578 |
pteval = tmp & IO_PAGE_MASK; |
579 |
@@ -478,7 +392,7 @@ static inline void fill_sg(iopte_t *iopte, struct scatterlist *sg, int nused, in |
580 |
sg++; |
581 |
} |
582 |
|
583 |
- pteval = ((pteval & IOPTE_PAGE) | iopte_bits); |
584 |
+ pteval = iopte_protection | (pteval & IOPTE_PAGE); |
585 |
while (len > 0) { |
586 |
*iopte++ = __iopte(pteval); |
587 |
pteval += IO_PAGE_SIZE; |
588 |
@@ -509,103 +423,111 @@ static inline void fill_sg(iopte_t *iopte, struct scatterlist *sg, int nused, in |
589 |
} |
590 |
} |
591 |
|
592 |
-int sbus_map_sg(struct sbus_dev *sdev, struct scatterlist *sg, int nents, int dir) |
593 |
+int sbus_map_sg(struct sbus_dev *sdev, struct scatterlist *sglist, int nelems, int direction) |
594 |
{ |
595 |
- struct sbus_iommu *iommu = sdev->bus->iommu; |
596 |
- unsigned long flags, npages; |
597 |
- iopte_t *iopte; |
598 |
+ struct sbus_iommu *iommu; |
599 |
+ unsigned long flags, npages, iopte_protection; |
600 |
+ iopte_t *base; |
601 |
u32 dma_base; |
602 |
struct scatterlist *sgtmp; |
603 |
int used; |
604 |
- unsigned long iopte_bits; |
605 |
- |
606 |
- if (dir == SBUS_DMA_NONE) |
607 |
- BUG(); |
608 |
|
609 |
/* Fast path single entry scatterlists. */ |
610 |
- if (nents == 1) { |
611 |
- sg->dma_address = |
612 |
+ if (nelems == 1) { |
613 |
+ sglist->dma_address = |
614 |
sbus_map_single(sdev, |
615 |
- (page_address(sg->page) + sg->offset), |
616 |
- sg->length, dir); |
617 |
- sg->dma_length = sg->length; |
618 |
+ (page_address(sglist->page) + sglist->offset), |
619 |
+ sglist->length, direction); |
620 |
+ sglist->dma_length = sglist->length; |
621 |
return 1; |
622 |
} |
623 |
|
624 |
- npages = prepare_sg(sg, nents); |
625 |
+ iommu = sdev->bus->iommu; |
626 |
+ |
627 |
+ if (unlikely(direction == SBUS_DMA_NONE)) |
628 |
+ BUG(); |
629 |
+ |
630 |
+ npages = prepare_sg(sglist, nelems); |
631 |
|
632 |
spin_lock_irqsave(&iommu->lock, flags); |
633 |
- iopte = alloc_streaming_cluster(iommu, npages); |
634 |
- if (iopte == NULL) |
635 |
- goto bad; |
636 |
- dma_base = MAP_BASE + ((iopte - iommu->page_table) << IO_PAGE_SHIFT); |
637 |
+ base = alloc_npages(iommu, npages); |
638 |
+ spin_unlock_irqrestore(&iommu->lock, flags); |
639 |
+ |
640 |
+ if (unlikely(base == NULL)) |
641 |
+ BUG(); |
642 |
+ |
643 |
+ dma_base = MAP_BASE + |
644 |
+ ((base - iommu->page_table) << IO_PAGE_SHIFT); |
645 |
|
646 |
/* Normalize DVMA addresses. */ |
647 |
- sgtmp = sg; |
648 |
- used = nents; |
649 |
+ used = nelems; |
650 |
|
651 |
+ sgtmp = sglist; |
652 |
while (used && sgtmp->dma_length) { |
653 |
sgtmp->dma_address += dma_base; |
654 |
sgtmp++; |
655 |
used--; |
656 |
} |
657 |
- used = nents - used; |
658 |
+ used = nelems - used; |
659 |
|
660 |
- iopte_bits = IOPTE_VALID | IOPTE_STBUF | IOPTE_CACHE; |
661 |
- if (dir != SBUS_DMA_TODEVICE) |
662 |
- iopte_bits |= IOPTE_WRITE; |
663 |
+ iopte_protection = IOPTE_VALID | IOPTE_STBUF | IOPTE_CACHE; |
664 |
+ if (direction != SBUS_DMA_TODEVICE) |
665 |
+ iopte_protection |= IOPTE_WRITE; |
666 |
+ |
667 |
+ fill_sg(base, sglist, used, nelems, iopte_protection); |
668 |
|
669 |
- fill_sg(iopte, sg, used, nents, iopte_bits); |
670 |
#ifdef VERIFY_SG |
671 |
- verify_sglist(sg, nents, iopte, npages); |
672 |
+ verify_sglist(sglist, nelems, base, npages); |
673 |
#endif |
674 |
- spin_unlock_irqrestore(&iommu->lock, flags); |
675 |
|
676 |
return used; |
677 |
- |
678 |
-bad: |
679 |
- spin_unlock_irqrestore(&iommu->lock, flags); |
680 |
- BUG(); |
681 |
- return 0; |
682 |
} |
683 |
|
684 |
-void sbus_unmap_sg(struct sbus_dev *sdev, struct scatterlist *sg, int nents, int direction) |
685 |
+void sbus_unmap_sg(struct sbus_dev *sdev, struct scatterlist *sglist, int nelems, int direction) |
686 |
{ |
687 |
- unsigned long size, flags; |
688 |
struct sbus_iommu *iommu; |
689 |
- u32 dvma_base; |
690 |
- int i; |
691 |
+ iopte_t *base; |
692 |
+ unsigned long flags, i, npages; |
693 |
+ u32 bus_addr; |
694 |
|
695 |
- /* Fast path single entry scatterlists. */ |
696 |
- if (nents == 1) { |
697 |
- sbus_unmap_single(sdev, sg->dma_address, sg->dma_length, direction); |
698 |
- return; |
699 |
- } |
700 |
+ if (unlikely(direction == SBUS_DMA_NONE)) |
701 |
+ BUG(); |
702 |
+ |
703 |
+ iommu = sdev->bus->iommu; |
704 |
+ |
705 |
+ bus_addr = sglist->dma_address & IO_PAGE_MASK; |
706 |
|
707 |
- dvma_base = sg[0].dma_address & IO_PAGE_MASK; |
708 |
- for (i = 0; i < nents; i++) { |
709 |
- if (sg[i].dma_length == 0) |
710 |
+ for (i = 1; i < nelems; i++) |
711 |
+ if (sglist[i].dma_length == 0) |
712 |
break; |
713 |
- } |
714 |
i--; |
715 |
- size = IO_PAGE_ALIGN(sg[i].dma_address + sg[i].dma_length) - dvma_base; |
716 |
+ npages = (IO_PAGE_ALIGN(sglist[i].dma_address + sglist[i].dma_length) - |
717 |
+ bus_addr) >> IO_PAGE_SHIFT; |
718 |
+ |
719 |
+ base = iommu->page_table + |
720 |
+ ((bus_addr - MAP_BASE) >> IO_PAGE_SHIFT); |
721 |
|
722 |
- iommu = sdev->bus->iommu; |
723 |
spin_lock_irqsave(&iommu->lock, flags); |
724 |
- free_streaming_cluster(iommu, dvma_base, size >> IO_PAGE_SHIFT); |
725 |
- sbus_strbuf_flush(iommu, dvma_base, size >> IO_PAGE_SHIFT, direction); |
726 |
+ sbus_strbuf_flush(iommu, bus_addr, npages, direction); |
727 |
+ for (i = 0; i < npages; i++) |
728 |
+ iopte_val(base[i]) = 0UL; |
729 |
+ free_npages(iommu, bus_addr - MAP_BASE, npages); |
730 |
spin_unlock_irqrestore(&iommu->lock, flags); |
731 |
} |
732 |
|
733 |
-void sbus_dma_sync_single_for_cpu(struct sbus_dev *sdev, dma_addr_t base, size_t size, int direction) |
734 |
+void sbus_dma_sync_single_for_cpu(struct sbus_dev *sdev, dma_addr_t bus_addr, size_t sz, int direction) |
735 |
{ |
736 |
- struct sbus_iommu *iommu = sdev->bus->iommu; |
737 |
- unsigned long flags; |
738 |
+ struct sbus_iommu *iommu; |
739 |
+ unsigned long flags, npages; |
740 |
+ |
741 |
+ iommu = sdev->bus->iommu; |
742 |
|
743 |
- size = (IO_PAGE_ALIGN(base + size) - (base & IO_PAGE_MASK)); |
744 |
+ npages = IO_PAGE_ALIGN(bus_addr + sz) - (bus_addr & IO_PAGE_MASK); |
745 |
+ npages >>= IO_PAGE_SHIFT; |
746 |
+ bus_addr &= IO_PAGE_MASK; |
747 |
|
748 |
spin_lock_irqsave(&iommu->lock, flags); |
749 |
- sbus_strbuf_flush(iommu, base & IO_PAGE_MASK, size >> IO_PAGE_SHIFT, direction); |
750 |
+ sbus_strbuf_flush(iommu, bus_addr, npages, direction); |
751 |
spin_unlock_irqrestore(&iommu->lock, flags); |
752 |
} |
753 |
|
754 |
@@ -613,23 +535,25 @@ void sbus_dma_sync_single_for_device(struct sbus_dev *sdev, dma_addr_t base, siz |
755 |
{ |
756 |
} |
757 |
|
758 |
-void sbus_dma_sync_sg_for_cpu(struct sbus_dev *sdev, struct scatterlist *sg, int nents, int direction) |
759 |
+void sbus_dma_sync_sg_for_cpu(struct sbus_dev *sdev, struct scatterlist *sglist, int nelems, int direction) |
760 |
{ |
761 |
- struct sbus_iommu *iommu = sdev->bus->iommu; |
762 |
- unsigned long flags, size; |
763 |
- u32 base; |
764 |
- int i; |
765 |
+ struct sbus_iommu *iommu; |
766 |
+ unsigned long flags, npages, i; |
767 |
+ u32 bus_addr; |
768 |
+ |
769 |
+ iommu = sdev->bus->iommu; |
770 |
|
771 |
- base = sg[0].dma_address & IO_PAGE_MASK; |
772 |
- for (i = 0; i < nents; i++) { |
773 |
- if (sg[i].dma_length == 0) |
774 |
+ bus_addr = sglist[0].dma_address & IO_PAGE_MASK; |
775 |
+ for (i = 0; i < nelems; i++) { |
776 |
+ if (!sglist[i].dma_length) |
777 |
break; |
778 |
} |
779 |
i--; |
780 |
- size = IO_PAGE_ALIGN(sg[i].dma_address + sg[i].dma_length) - base; |
781 |
+ npages = (IO_PAGE_ALIGN(sglist[i].dma_address + sglist[i].dma_length) |
782 |
+ - bus_addr) >> IO_PAGE_SHIFT; |
783 |
|
784 |
spin_lock_irqsave(&iommu->lock, flags); |
785 |
- sbus_strbuf_flush(iommu, base, size >> IO_PAGE_SHIFT, direction); |
786 |
+ sbus_strbuf_flush(iommu, bus_addr, npages, direction); |
787 |
spin_unlock_irqrestore(&iommu->lock, flags); |
788 |
} |
789 |
|
790 |
@@ -1104,7 +1028,7 @@ static void __init sbus_iommu_init(int __node, struct sbus_bus *sbus) |
791 |
struct linux_prom64_registers *pr; |
792 |
struct device_node *dp; |
793 |
struct sbus_iommu *iommu; |
794 |
- unsigned long regs, tsb_base; |
795 |
+ unsigned long regs; |
796 |
u64 control; |
797 |
int i; |
798 |
|
799 |
@@ -1132,14 +1056,6 @@ static void __init sbus_iommu_init(int __node, struct sbus_bus *sbus) |
800 |
|
801 |
memset(iommu, 0, sizeof(*iommu)); |
802 |
|
803 |
- /* We start with no consistent mappings. */ |
804 |
- iommu->lowest_consistent_map = CLUSTER_NPAGES; |
805 |
- |
806 |
- for (i = 0; i < NCLUSTERS; i++) { |
807 |
- iommu->alloc_info[i].flush = 0; |
808 |
- iommu->alloc_info[i].next = 0; |
809 |
- } |
810 |
- |
811 |
/* Setup spinlock. */ |
812 |
spin_lock_init(&iommu->lock); |
813 |
|
814 |
@@ -1159,25 +1075,13 @@ static void __init sbus_iommu_init(int __node, struct sbus_bus *sbus) |
815 |
sbus->portid, regs); |
816 |
|
817 |
/* Setup for TSB_SIZE=7, TBW_SIZE=0, MMU_DE=1, MMU_EN=1 */ |
818 |
+ sbus_iommu_table_init(iommu, IO_TSB_SIZE); |
819 |
+ |
820 |
control = upa_readq(iommu->iommu_regs + IOMMU_CONTROL); |
821 |
control = ((7UL << 16UL) | |
822 |
(0UL << 2UL) | |
823 |
(1UL << 1UL) | |
824 |
(1UL << 0UL)); |
825 |
- |
826 |
- /* Using the above configuration we need 1MB iommu page |
827 |
- * table (128K ioptes * 8 bytes per iopte). This is |
828 |
- * page order 7 on UltraSparc. |
829 |
- */ |
830 |
- tsb_base = __get_free_pages(GFP_ATOMIC, get_order(IO_TSB_SIZE)); |
831 |
- if (tsb_base == 0UL) { |
832 |
- prom_printf("sbus_iommu_init: Fatal error, cannot alloc TSB table.\n"); |
833 |
- prom_halt(); |
834 |
- } |
835 |
- |
836 |
- iommu->page_table = (iopte_t *) tsb_base; |
837 |
- memset(iommu->page_table, 0, IO_TSB_SIZE); |
838 |
- |
839 |
upa_writeq(control, iommu->iommu_regs + IOMMU_CONTROL); |
840 |
|
841 |
/* Clean out any cruft in the IOMMU using |
842 |
@@ -1195,7 +1099,7 @@ static void __init sbus_iommu_init(int __node, struct sbus_bus *sbus) |
843 |
upa_readq(iommu->sbus_control_reg); |
844 |
|
845 |
/* Give the TSB to SYSIO. */ |
846 |
- upa_writeq(__pa(tsb_base), iommu->iommu_regs + IOMMU_TSBBASE); |
847 |
+ upa_writeq(__pa(iommu->page_table), iommu->iommu_regs + IOMMU_TSBBASE); |
848 |
|
849 |
/* Setup streaming buffer, DE=1 SB_EN=1 */ |
850 |
control = (1UL << 1UL) | (1UL << 0UL); |
851 |
- |
852 |
To unsubscribe from this list: send the line "unsubscribe sparclinux" in |
853 |
the body of a message to majordomo@×××××××××××.org |
854 |
More majordomo info at http://vger.kernel.org/majordomo-info.html |
855 |
|
856 |
Regards, |
857 |
-- |
858 |
Ferris McCormick (P44646, MI) <fmccor@g.o> |
859 |
Developer, Gentoo Linux (Devrel, Sparc) |