1 |
Just saw that 2.6.21-gentoo was released, so I'm walking |
2 |
through the build on my U2. |
3 |
|
4 |
I'll post a report after some more testing. |
5 |
|
6 |
|
7 |
|
8 |
> -----Original Message----- |
9 |
> From: Ferris McCormick [mailto:fmccor@g.o] |
10 |
> Sent: Thursday, April 12, 2007 4:41 AM |
11 |
> To: Gentoo Sparc |
12 |
> Subject: [gentoo-sparc] [Fwd: Re: [Repost] Ultra2 SMP freezes |
13 |
> with heavy disk I/O] |
14 |
> |
15 |
> In case you missed it. |
16 |
> |
17 |
> -------- Forwarded Message -------- |
18 |
> From: David Miller <davem@×××××××××.net> |
19 |
> To: joel.bertrand@××××××××.fr |
20 |
> Cc: mt1@××××××××.fr, sparclinux@×××××××××××.org |
21 |
> Subject: Re: [Repost] Ultra2 SMP freezes with heavy disk I/O |
22 |
> Date: Wed, 11 Apr 2007 21:55:11 -0700 (PDT) |
23 |
> |
24 |
> From: David Miller <davem@×××××××××.net> |
25 |
> Date: Tue, 10 Apr 2007 14:04:58 -0700 (PDT) |
26 |
> |
27 |
> > From: BERTRAND Joël <joel.bertrand@××××××××.fr> |
28 |
> > Date: Tue, 20 Mar 2007 21:53:26 +0100 |
29 |
> > |
30 |
> > > I have tried the last 2.6.21-rc4. Same bug. |
31 |
> > |
32 |
> > The good news is that I can reproduce this problem now on |
33 |
> my Ultra2, |
34 |
> > I'll try to figure out what's wrong. |
35 |
> > |
36 |
> > Running "dbench 32" a few time is enough to trigger it. |
37 |
> |
38 |
> Ok, I think I might have killed at least the bug I was able to hit. |
39 |
> |
40 |
> I make no promises that this will fix the issues everyone |
41 |
> else was seeing, but let's cross our fingers :-))) |
42 |
> |
43 |
> Please give the following patch some testing if you can. And |
44 |
> please be careful, messing with the DMA mapping code can do |
45 |
> things like eat your disk :-) |
46 |
> |
47 |
> If all goes well I'll push this bug fix around, and I have a |
48 |
> secret treat for your poor sparc64 SBUS users once we get past this. |
49 |
> |
50 |
> Thanks! |
51 |
> |
52 |
> commit af822e612593c1feef5052e685bca0f8d087d120 |
53 |
> Author: David S. Miller <davem@××××××××××××××××.net> |
54 |
> Date: Wed Apr 11 21:38:45 2007 -0700 |
55 |
> |
56 |
> [SPARC64]: Fix SBUS IOMMU allocation code. |
57 |
> |
58 |
> There are several IOMMU allocator bugs. Instead of |
59 |
> trying to fix this |
60 |
> overly complicated code, just mirror the PCI IOMMU arena allocator |
61 |
> which is very stable and well stress tested. |
62 |
> |
63 |
> I tried to make the code as identical as possible so we can switch |
64 |
> sun4u PCI and SBUS over to a common piece of IOMMU code. All that |
65 |
> will be need are two callbacks, one to do a full IOMMU |
66 |
> flush and one |
67 |
> to do a streaming buffer flush. |
68 |
> |
69 |
> This patch gets rid of a lot of hangs and mysterious |
70 |
> crashes on SBUS |
71 |
> sparc64 systems, at least for me. |
72 |
> |
73 |
> Signed-off-by: David S. Miller <davem@×××××××××.net> |
74 |
> |
75 |
> diff --git a/arch/sparc64/kernel/sbus.c |
76 |
> b/arch/sparc64/kernel/sbus.c index 01d6d86..14f78fb 100644 |
77 |
> --- a/arch/sparc64/kernel/sbus.c |
78 |
> +++ b/arch/sparc64/kernel/sbus.c |
79 |
> @@ -24,48 +24,25 @@ |
80 |
> |
81 |
> #include "iommu_common.h" |
82 |
> |
83 |
> -/* These should be allocated on an SMP_CACHE_BYTES |
84 |
> - * aligned boundary for optimal performance. |
85 |
> - * |
86 |
> - * On SYSIO, using an 8K page size we have 1GB of SBUS |
87 |
> - * DMA space mapped. We divide this space into equally |
88 |
> - * sized clusters. We allocate a DMA mapping from the |
89 |
> - * cluster that matches the order of the allocation, or |
90 |
> - * if the order is greater than the number of clusters, |
91 |
> - * we try to allocate from the last cluster. |
92 |
> - */ |
93 |
> - |
94 |
> -#define NCLUSTERS 8UL |
95 |
> -#define ONE_GIG (1UL * 1024UL * 1024UL * 1024UL) |
96 |
> -#define CLUSTER_SIZE (ONE_GIG / NCLUSTERS) |
97 |
> -#define CLUSTER_MASK (CLUSTER_SIZE - 1) |
98 |
> -#define CLUSTER_NPAGES (CLUSTER_SIZE >> IO_PAGE_SHIFT) |
99 |
> #define MAP_BASE ((u32)0xc0000000) |
100 |
> |
101 |
> +struct sbus_iommu_arena { |
102 |
> + unsigned long *map; |
103 |
> + unsigned int hint; |
104 |
> + unsigned int limit; |
105 |
> +}; |
106 |
> + |
107 |
> struct sbus_iommu { |
108 |
> -/*0x00*/spinlock_t lock; |
109 |
> + spinlock_t lock; |
110 |
> |
111 |
> -/*0x08*/iopte_t *page_table; |
112 |
> -/*0x10*/unsigned long strbuf_regs; |
113 |
> -/*0x18*/unsigned long iommu_regs; |
114 |
> -/*0x20*/unsigned long sbus_control_reg; |
115 |
> + struct sbus_iommu_arena arena; |
116 |
> |
117 |
> -/*0x28*/volatile unsigned long strbuf_flushflag; |
118 |
> + iopte_t *page_table; |
119 |
> + unsigned long strbuf_regs; |
120 |
> + unsigned long iommu_regs; |
121 |
> + unsigned long sbus_control_reg; |
122 |
> |
123 |
> - /* If NCLUSTERS is ever decresed to 4 or lower, |
124 |
> - * you must increase the size of the type of |
125 |
> - * these counters. You have been duly warned. -DaveM |
126 |
> - */ |
127 |
> -/*0x30*/struct { |
128 |
> - u16 next; |
129 |
> - u16 flush; |
130 |
> - } alloc_info[NCLUSTERS]; |
131 |
> - |
132 |
> - /* The lowest used consistent mapping entry. Since |
133 |
> - * we allocate consistent maps out of cluster 0 this |
134 |
> - * is relative to the beginning of closter 0. |
135 |
> - */ |
136 |
> -/*0x50*/u32 lowest_consistent_map; |
137 |
> + volatile unsigned long strbuf_flushflag; |
138 |
> }; |
139 |
> |
140 |
> /* Offsets from iommu_regs */ |
141 |
> @@ -91,19 +68,6 @@ static void __iommu_flushall(struct |
142 |
> sbus_iommu *iommu) |
143 |
> tag += 8UL; |
144 |
> } |
145 |
> upa_readq(iommu->sbus_control_reg); |
146 |
> - |
147 |
> - for (entry = 0; entry < NCLUSTERS; entry++) { |
148 |
> - iommu->alloc_info[entry].flush = |
149 |
> - iommu->alloc_info[entry].next; |
150 |
> - } |
151 |
> -} |
152 |
> - |
153 |
> -static void iommu_flush(struct sbus_iommu *iommu, u32 base, |
154 |
> unsigned long npages) -{ |
155 |
> - while (npages--) |
156 |
> - upa_writeq(base + (npages << IO_PAGE_SHIFT), |
157 |
> - iommu->iommu_regs + IOMMU_FLUSH); |
158 |
> - upa_readq(iommu->sbus_control_reg); |
159 |
> } |
160 |
> |
161 |
> /* Offsets from strbuf_regs */ |
162 |
> @@ -156,178 +120,115 @@ static void sbus_strbuf_flush(struct |
163 |
> sbus_iommu *iommu, u32 base, unsigned long |
164 |
> base, npages); |
165 |
> } |
166 |
> |
167 |
> -static iopte_t *alloc_streaming_cluster(struct sbus_iommu |
168 |
> *iommu, unsigned long npages) |
169 |
> +/* Based largely upon the ppc64 iommu allocator. */ static long |
170 |
> +sbus_arena_alloc(struct sbus_iommu *iommu, unsigned long npages) |
171 |
> { |
172 |
> - iopte_t *iopte, *limit, *first, *cluster; |
173 |
> - unsigned long cnum, ent, nent, flush_point, found; |
174 |
> - |
175 |
> - cnum = 0; |
176 |
> - nent = 1; |
177 |
> - while ((1UL << cnum) < npages) |
178 |
> - cnum++; |
179 |
> - if(cnum >= NCLUSTERS) { |
180 |
> - nent = 1UL << (cnum - NCLUSTERS); |
181 |
> - cnum = NCLUSTERS - 1; |
182 |
> - } |
183 |
> - iopte = iommu->page_table + (cnum * CLUSTER_NPAGES); |
184 |
> - |
185 |
> - if (cnum == 0) |
186 |
> - limit = (iommu->page_table + |
187 |
> - iommu->lowest_consistent_map); |
188 |
> - else |
189 |
> - limit = (iopte + CLUSTER_NPAGES); |
190 |
> - |
191 |
> - iopte += ((ent = iommu->alloc_info[cnum].next) << cnum); |
192 |
> - flush_point = iommu->alloc_info[cnum].flush; |
193 |
> - |
194 |
> - first = iopte; |
195 |
> - cluster = NULL; |
196 |
> - found = 0; |
197 |
> - for (;;) { |
198 |
> - if (iopte_val(*iopte) == 0UL) { |
199 |
> - found++; |
200 |
> - if (!cluster) |
201 |
> - cluster = iopte; |
202 |
> + struct sbus_iommu_arena *arena = &iommu->arena; |
203 |
> + unsigned long n, i, start, end, limit; |
204 |
> + int pass; |
205 |
> + |
206 |
> + limit = arena->limit; |
207 |
> + start = arena->hint; |
208 |
> + pass = 0; |
209 |
> + |
210 |
> +again: |
211 |
> + n = find_next_zero_bit(arena->map, limit, start); |
212 |
> + end = n + npages; |
213 |
> + if (unlikely(end >= limit)) { |
214 |
> + if (likely(pass < 1)) { |
215 |
> + limit = start; |
216 |
> + start = 0; |
217 |
> + __iommu_flushall(iommu); |
218 |
> + pass++; |
219 |
> + goto again; |
220 |
> } else { |
221 |
> - /* Used cluster in the way */ |
222 |
> - cluster = NULL; |
223 |
> - found = 0; |
224 |
> + /* Scanned the whole thing, give up. */ |
225 |
> + return -1; |
226 |
> } |
227 |
> + } |
228 |
> |
229 |
> - if (found == nent) |
230 |
> - break; |
231 |
> - |
232 |
> - iopte += (1 << cnum); |
233 |
> - ent++; |
234 |
> - if (iopte >= limit) { |
235 |
> - iopte = (iommu->page_table + (cnum * |
236 |
> CLUSTER_NPAGES)); |
237 |
> - ent = 0; |
238 |
> - |
239 |
> - /* Multiple cluster allocations must not wrap */ |
240 |
> - cluster = NULL; |
241 |
> - found = 0; |
242 |
> + for (i = n; i < end; i++) { |
243 |
> + if (test_bit(i, arena->map)) { |
244 |
> + start = i + 1; |
245 |
> + goto again; |
246 |
> } |
247 |
> - if (ent == flush_point) |
248 |
> - __iommu_flushall(iommu); |
249 |
> - if (iopte == first) |
250 |
> - goto bad; |
251 |
> } |
252 |
> |
253 |
> - /* ent/iopte points to the last cluster entry we're |
254 |
> going to use, |
255 |
> - * so save our place for the next allocation. |
256 |
> - */ |
257 |
> - if ((iopte + (1 << cnum)) >= limit) |
258 |
> - ent = 0; |
259 |
> - else |
260 |
> - ent = ent + 1; |
261 |
> - iommu->alloc_info[cnum].next = ent; |
262 |
> - if (ent == flush_point) |
263 |
> - __iommu_flushall(iommu); |
264 |
> - |
265 |
> - /* I've got your streaming cluster right here buddy boy... */ |
266 |
> - return cluster; |
267 |
> - |
268 |
> -bad: |
269 |
> - printk(KERN_EMERG "sbus: alloc_streaming_cluster of |
270 |
> npages(%ld) failed!\n", |
271 |
> - npages); |
272 |
> - return NULL; |
273 |
> + for (i = n; i < end; i++) |
274 |
> + __set_bit(i, arena->map); |
275 |
> + |
276 |
> + arena->hint = end; |
277 |
> + |
278 |
> + return n; |
279 |
> } |
280 |
> |
281 |
> -static void free_streaming_cluster(struct sbus_iommu *iommu, |
282 |
> u32 base, unsigned long npages) |
283 |
> +static void sbus_arena_free(struct sbus_iommu_arena *arena, unsigned |
284 |
> +long base, unsigned long npages) |
285 |
> { |
286 |
> - unsigned long cnum, ent, nent; |
287 |
> - iopte_t *iopte; |
288 |
> + unsigned long i; |
289 |
> |
290 |
> - cnum = 0; |
291 |
> - nent = 1; |
292 |
> - while ((1UL << cnum) < npages) |
293 |
> - cnum++; |
294 |
> - if(cnum >= NCLUSTERS) { |
295 |
> - nent = 1UL << (cnum - NCLUSTERS); |
296 |
> - cnum = NCLUSTERS - 1; |
297 |
> - } |
298 |
> - ent = (base & CLUSTER_MASK) >> (IO_PAGE_SHIFT + cnum); |
299 |
> - iopte = iommu->page_table + ((base - MAP_BASE) >> |
300 |
> IO_PAGE_SHIFT); |
301 |
> - do { |
302 |
> - iopte_val(*iopte) = 0UL; |
303 |
> - iopte += 1 << cnum; |
304 |
> - } while(--nent); |
305 |
> - |
306 |
> - /* If the global flush might not have caught this entry, |
307 |
> - * adjust the flush point such that we will flush before |
308 |
> - * ever trying to reuse it. |
309 |
> - */ |
310 |
> -#define between(X,Y,Z) (((Z) - (Y)) >= ((X) - (Y))) |
311 |
> - if (between(ent, iommu->alloc_info[cnum].next, |
312 |
> iommu->alloc_info[cnum].flush)) |
313 |
> - iommu->alloc_info[cnum].flush = ent; |
314 |
> -#undef between |
315 |
> + for (i = base; i < (base + npages); i++) |
316 |
> + __clear_bit(i, arena->map); |
317 |
> } |
318 |
> |
319 |
> -/* We allocate consistent mappings from the end of cluster |
320 |
> zero. */ -static iopte_t *alloc_consistent_cluster(struct |
321 |
> sbus_iommu *iommu, unsigned long npages) |
322 |
> +static void sbus_iommu_table_init(struct sbus_iommu *iommu, unsigned |
323 |
> +int tsbsize) |
324 |
> { |
325 |
> - iopte_t *iopte; |
326 |
> + unsigned long tsbbase, order, sz, num_tsb_entries; |
327 |
> |
328 |
> - iopte = iommu->page_table + (1 * CLUSTER_NPAGES); |
329 |
> - while (iopte > iommu->page_table) { |
330 |
> - iopte--; |
331 |
> - if (!(iopte_val(*iopte) & IOPTE_VALID)) { |
332 |
> - unsigned long tmp = npages; |
333 |
> + num_tsb_entries = tsbsize / sizeof(iopte_t); |
334 |
> |
335 |
> - while (--tmp) { |
336 |
> - iopte--; |
337 |
> - if (iopte_val(*iopte) & IOPTE_VALID) |
338 |
> - break; |
339 |
> - } |
340 |
> - if (tmp == 0) { |
341 |
> - u32 entry = (iopte - iommu->page_table); |
342 |
> + /* Setup initial software IOMMU state. */ |
343 |
> + spin_lock_init(&iommu->lock); |
344 |
> |
345 |
> - if (entry < |
346 |
> iommu->lowest_consistent_map) |
347 |
> - |
348 |
> iommu->lowest_consistent_map = entry; |
349 |
> - return iopte; |
350 |
> - } |
351 |
> - } |
352 |
> + /* Allocate and initialize the free area map. */ |
353 |
> + sz = num_tsb_entries / 8; |
354 |
> + sz = (sz + 7UL) & ~7UL; |
355 |
> + iommu->arena.map = kzalloc(sz, GFP_KERNEL); |
356 |
> + if (!iommu->arena.map) { |
357 |
> + prom_printf("PCI_IOMMU: Error, |
358 |
> kmalloc(arena.map) failed.\n"); |
359 |
> + prom_halt(); |
360 |
> + } |
361 |
> + iommu->arena.limit = num_tsb_entries; |
362 |
> + |
363 |
> + /* Now allocate and setup the IOMMU page table itself. */ |
364 |
> + order = get_order(tsbsize); |
365 |
> + tsbbase = __get_free_pages(GFP_KERNEL, order); |
366 |
> + if (!tsbbase) { |
367 |
> + prom_printf("IOMMU: Error, gfp(tsb) failed.\n"); |
368 |
> + prom_halt(); |
369 |
> } |
370 |
> - return NULL; |
371 |
> + iommu->page_table = (iopte_t *)tsbbase; |
372 |
> + memset(iommu->page_table, 0, tsbsize); |
373 |
> } |
374 |
> |
375 |
> -static void free_consistent_cluster(struct sbus_iommu |
376 |
> *iommu, u32 base, unsigned long npages) |
377 |
> +static inline iopte_t *alloc_npages(struct sbus_iommu |
378 |
> *iommu, unsigned |
379 |
> +long npages) |
380 |
> { |
381 |
> - iopte_t *iopte = iommu->page_table + ((base - MAP_BASE) |
382 |
> >> IO_PAGE_SHIFT); |
383 |
> + long entry; |
384 |
> |
385 |
> - if ((iopte - iommu->page_table) == |
386 |
> iommu->lowest_consistent_map) { |
387 |
> - iopte_t *walk = iopte + npages; |
388 |
> - iopte_t *limit; |
389 |
> + entry = sbus_arena_alloc(iommu, npages); |
390 |
> + if (unlikely(entry < 0)) |
391 |
> + return NULL; |
392 |
> |
393 |
> - limit = iommu->page_table + CLUSTER_NPAGES; |
394 |
> - while (walk < limit) { |
395 |
> - if (iopte_val(*walk) != 0UL) |
396 |
> - break; |
397 |
> - walk++; |
398 |
> - } |
399 |
> - iommu->lowest_consistent_map = |
400 |
> - (walk - iommu->page_table); |
401 |
> - } |
402 |
> + return iommu->page_table + entry; |
403 |
> +} |
404 |
> |
405 |
> - while (npages--) |
406 |
> - *iopte++ = __iopte(0UL); |
407 |
> +static inline void free_npages(struct sbus_iommu *iommu, dma_addr_t |
408 |
> +base, unsigned long npages) { |
409 |
> + sbus_arena_free(&iommu->arena, base >> IO_PAGE_SHIFT, npages); |
410 |
> } |
411 |
> |
412 |
> void *sbus_alloc_consistent(struct sbus_dev *sdev, size_t |
413 |
> size, dma_addr_t *dvma_addr) { |
414 |
> - unsigned long order, first_page, flags; |
415 |
> struct sbus_iommu *iommu; |
416 |
> iopte_t *iopte; |
417 |
> + unsigned long flags, order, first_page; |
418 |
> void *ret; |
419 |
> int npages; |
420 |
> |
421 |
> - if (size <= 0 || sdev == NULL || dvma_addr == NULL) |
422 |
> - return NULL; |
423 |
> - |
424 |
> size = IO_PAGE_ALIGN(size); |
425 |
> order = get_order(size); |
426 |
> if (order >= 10) |
427 |
> return NULL; |
428 |
> + |
429 |
> first_page = __get_free_pages(GFP_KERNEL|__GFP_COMP, order); |
430 |
> if (first_page == 0UL) |
431 |
> return NULL; |
432 |
> @@ -336,108 +237,121 @@ void *sbus_alloc_consistent(struct |
433 |
> sbus_dev *sdev, size_t size, dma_addr_t *dvma |
434 |
> iommu = sdev->bus->iommu; |
435 |
> |
436 |
> spin_lock_irqsave(&iommu->lock, flags); |
437 |
> - iopte = alloc_consistent_cluster(iommu, size >> IO_PAGE_SHIFT); |
438 |
> - if (iopte == NULL) { |
439 |
> - spin_unlock_irqrestore(&iommu->lock, flags); |
440 |
> + iopte = alloc_npages(iommu, size >> IO_PAGE_SHIFT); |
441 |
> + spin_unlock_irqrestore(&iommu->lock, flags); |
442 |
> + |
443 |
> + if (unlikely(iopte == NULL)) { |
444 |
> free_pages(first_page, order); |
445 |
> return NULL; |
446 |
> } |
447 |
> |
448 |
> - /* Ok, we're committed at this point. */ |
449 |
> - *dvma_addr = MAP_BASE + ((iopte - iommu->page_table) << |
450 |
> IO_PAGE_SHIFT); |
451 |
> + *dvma_addr = (MAP_BASE + |
452 |
> + ((iopte - iommu->page_table) << IO_PAGE_SHIFT)); |
453 |
> ret = (void *) first_page; |
454 |
> npages = size >> IO_PAGE_SHIFT; |
455 |
> + first_page = __pa(first_page); |
456 |
> while (npages--) { |
457 |
> - *iopte++ = __iopte(IOPTE_VALID | IOPTE_CACHE | |
458 |
> IOPTE_WRITE | |
459 |
> - (__pa(first_page) & IOPTE_PAGE)); |
460 |
> + iopte_val(*iopte) = (IOPTE_VALID | IOPTE_CACHE | |
461 |
> + IOPTE_WRITE | |
462 |
> + (first_page & IOPTE_PAGE)); |
463 |
> + iopte++; |
464 |
> first_page += IO_PAGE_SIZE; |
465 |
> } |
466 |
> - iommu_flush(iommu, *dvma_addr, size >> IO_PAGE_SHIFT); |
467 |
> - spin_unlock_irqrestore(&iommu->lock, flags); |
468 |
> |
469 |
> return ret; |
470 |
> } |
471 |
> |
472 |
> void sbus_free_consistent(struct sbus_dev *sdev, size_t |
473 |
> size, void *cpu, dma_addr_t dvma) { |
474 |
> - unsigned long order, npages; |
475 |
> struct sbus_iommu *iommu; |
476 |
> - |
477 |
> - if (size <= 0 || sdev == NULL || cpu == NULL) |
478 |
> - return; |
479 |
> + iopte_t *iopte; |
480 |
> + unsigned long flags, order, npages; |
481 |
> |
482 |
> npages = IO_PAGE_ALIGN(size) >> IO_PAGE_SHIFT; |
483 |
> iommu = sdev->bus->iommu; |
484 |
> + iopte = iommu->page_table + |
485 |
> + ((dvma - MAP_BASE) >> IO_PAGE_SHIFT); |
486 |
> + |
487 |
> + spin_lock_irqsave(&iommu->lock, flags); |
488 |
> + |
489 |
> + free_npages(iommu, dvma - MAP_BASE, npages); |
490 |
> |
491 |
> - spin_lock_irq(&iommu->lock); |
492 |
> - free_consistent_cluster(iommu, dvma, npages); |
493 |
> - iommu_flush(iommu, dvma, npages); |
494 |
> - spin_unlock_irq(&iommu->lock); |
495 |
> + spin_unlock_irqrestore(&iommu->lock, flags); |
496 |
> |
497 |
> order = get_order(size); |
498 |
> if (order < 10) |
499 |
> free_pages((unsigned long)cpu, order); } |
500 |
> |
501 |
> -dma_addr_t sbus_map_single(struct sbus_dev *sdev, void *ptr, |
502 |
> size_t size, int dir) |
503 |
> +dma_addr_t sbus_map_single(struct sbus_dev *sdev, void *ptr, |
504 |
> size_t sz, |
505 |
> +int direction) |
506 |
> { |
507 |
> - struct sbus_iommu *iommu = sdev->bus->iommu; |
508 |
> - unsigned long npages, pbase, flags; |
509 |
> - iopte_t *iopte; |
510 |
> - u32 dma_base, offset; |
511 |
> - unsigned long iopte_bits; |
512 |
> + struct sbus_iommu *iommu; |
513 |
> + iopte_t *base; |
514 |
> + unsigned long flags, npages, oaddr; |
515 |
> + unsigned long i, base_paddr; |
516 |
> + u32 bus_addr, ret; |
517 |
> + unsigned long iopte_protection; |
518 |
> + |
519 |
> + iommu = sdev->bus->iommu; |
520 |
> |
521 |
> - if (dir == SBUS_DMA_NONE) |
522 |
> + if (unlikely(direction == SBUS_DMA_NONE)) |
523 |
> BUG(); |
524 |
> |
525 |
> - pbase = (unsigned long) ptr; |
526 |
> - offset = (u32) (pbase & ~IO_PAGE_MASK); |
527 |
> - size = (IO_PAGE_ALIGN(pbase + size) - (pbase & IO_PAGE_MASK)); |
528 |
> - pbase = (unsigned long) __pa(pbase & IO_PAGE_MASK); |
529 |
> + oaddr = (unsigned long)ptr; |
530 |
> + npages = IO_PAGE_ALIGN(oaddr + sz) - (oaddr & IO_PAGE_MASK); |
531 |
> + npages >>= IO_PAGE_SHIFT; |
532 |
> |
533 |
> spin_lock_irqsave(&iommu->lock, flags); |
534 |
> - npages = size >> IO_PAGE_SHIFT; |
535 |
> - iopte = alloc_streaming_cluster(iommu, npages); |
536 |
> - if (iopte == NULL) |
537 |
> - goto bad; |
538 |
> - dma_base = MAP_BASE + ((iopte - iommu->page_table) << |
539 |
> IO_PAGE_SHIFT); |
540 |
> - npages = size >> IO_PAGE_SHIFT; |
541 |
> - iopte_bits = IOPTE_VALID | IOPTE_STBUF | IOPTE_CACHE; |
542 |
> - if (dir != SBUS_DMA_TODEVICE) |
543 |
> - iopte_bits |= IOPTE_WRITE; |
544 |
> - while (npages--) { |
545 |
> - *iopte++ = __iopte(iopte_bits | (pbase & IOPTE_PAGE)); |
546 |
> - pbase += IO_PAGE_SIZE; |
547 |
> - } |
548 |
> - npages = size >> IO_PAGE_SHIFT; |
549 |
> + base = alloc_npages(iommu, npages); |
550 |
> spin_unlock_irqrestore(&iommu->lock, flags); |
551 |
> |
552 |
> - return (dma_base | offset); |
553 |
> + if (unlikely(!base)) |
554 |
> + BUG(); |
555 |
> |
556 |
> -bad: |
557 |
> - spin_unlock_irqrestore(&iommu->lock, flags); |
558 |
> - BUG(); |
559 |
> - return 0; |
560 |
> + bus_addr = (MAP_BASE + |
561 |
> + ((base - iommu->page_table) << IO_PAGE_SHIFT)); |
562 |
> + ret = bus_addr | (oaddr & ~IO_PAGE_MASK); |
563 |
> + base_paddr = __pa(oaddr & IO_PAGE_MASK); |
564 |
> + |
565 |
> + iopte_protection = IOPTE_VALID | IOPTE_STBUF | IOPTE_CACHE; |
566 |
> + if (direction != SBUS_DMA_TODEVICE) |
567 |
> + iopte_protection |= IOPTE_WRITE; |
568 |
> + |
569 |
> + for (i = 0; i < npages; i++, base++, base_paddr += IO_PAGE_SIZE) |
570 |
> + iopte_val(*base) = iopte_protection | base_paddr; |
571 |
> + |
572 |
> + return ret; |
573 |
> } |
574 |
> |
575 |
> -void sbus_unmap_single(struct sbus_dev *sdev, dma_addr_t |
576 |
> dma_addr, size_t size, int direction) |
577 |
> +void sbus_unmap_single(struct sbus_dev *sdev, dma_addr_t bus_addr, |
578 |
> +size_t sz, int direction) |
579 |
> { |
580 |
> struct sbus_iommu *iommu = sdev->bus->iommu; |
581 |
> - u32 dma_base = dma_addr & IO_PAGE_MASK; |
582 |
> - unsigned long flags; |
583 |
> + iopte_t *base; |
584 |
> + unsigned long flags, npages, i; |
585 |
> + |
586 |
> + if (unlikely(direction == SBUS_DMA_NONE)) |
587 |
> + BUG(); |
588 |
> + |
589 |
> + npages = IO_PAGE_ALIGN(bus_addr + sz) - (bus_addr & |
590 |
> IO_PAGE_MASK); |
591 |
> + npages >>= IO_PAGE_SHIFT; |
592 |
> + base = iommu->page_table + |
593 |
> + ((bus_addr - MAP_BASE) >> IO_PAGE_SHIFT); |
594 |
> |
595 |
> - size = (IO_PAGE_ALIGN(dma_addr + size) - dma_base); |
596 |
> + bus_addr &= IO_PAGE_MASK; |
597 |
> |
598 |
> spin_lock_irqsave(&iommu->lock, flags); |
599 |
> - free_streaming_cluster(iommu, dma_base, size >> IO_PAGE_SHIFT); |
600 |
> - sbus_strbuf_flush(iommu, dma_base, size >> |
601 |
> IO_PAGE_SHIFT, direction); |
602 |
> + sbus_strbuf_flush(iommu, bus_addr, npages, direction); |
603 |
> + for (i = 0; i < npages; i++) |
604 |
> + iopte_val(base[i]) = 0UL; |
605 |
> + free_npages(iommu, bus_addr - MAP_BASE, npages); |
606 |
> spin_unlock_irqrestore(&iommu->lock, flags); } |
607 |
> |
608 |
> #define SG_ENT_PHYS_ADDRESS(SG) \ |
609 |
> (__pa(page_address((SG)->page)) + (SG)->offset) |
610 |
> |
611 |
> -static inline void fill_sg(iopte_t *iopte, struct |
612 |
> scatterlist *sg, int nused, int nelems, unsigned long iopte_bits) |
613 |
> +static inline void fill_sg(iopte_t *iopte, struct scatterlist *sg, |
614 |
> + int nused, int nelems, unsigned long |
615 |
> iopte_protection) |
616 |
> { |
617 |
> struct scatterlist *dma_sg = sg; |
618 |
> struct scatterlist *sg_end = sg + nelems; @@ -462,7 |
619 |
> +376,7 @@ static inline void fill_sg(iopte_t *iopte, struct |
620 |
> scatterlist *sg, int nused, in |
621 |
> for (;;) { |
622 |
> unsigned long tmp; |
623 |
> |
624 |
> - tmp = (unsigned long) |
625 |
> SG_ENT_PHYS_ADDRESS(sg); |
626 |
> + tmp = SG_ENT_PHYS_ADDRESS(sg); |
627 |
> len = sg->length; |
628 |
> if (((tmp ^ pteval) >> |
629 |
> IO_PAGE_SHIFT) != 0UL) { |
630 |
> pteval = tmp & IO_PAGE_MASK; |
631 |
> @@ -478,7 +392,7 @@ static inline void fill_sg(iopte_t |
632 |
> *iopte, struct scatterlist *sg, int nused, in |
633 |
> sg++; |
634 |
> } |
635 |
> |
636 |
> - pteval = ((pteval & IOPTE_PAGE) | iopte_bits); |
637 |
> + pteval = iopte_protection | (pteval & |
638 |
> IOPTE_PAGE); |
639 |
> while (len > 0) { |
640 |
> *iopte++ = __iopte(pteval); |
641 |
> pteval += IO_PAGE_SIZE; |
642 |
> @@ -509,103 +423,111 @@ static inline void fill_sg(iopte_t |
643 |
> *iopte, struct scatterlist *sg, int nused, in |
644 |
> } |
645 |
> } |
646 |
> |
647 |
> -int sbus_map_sg(struct sbus_dev *sdev, struct scatterlist |
648 |
> *sg, int nents, int dir) |
649 |
> +int sbus_map_sg(struct sbus_dev *sdev, struct scatterlist |
650 |
> *sglist, int |
651 |
> +nelems, int direction) |
652 |
> { |
653 |
> - struct sbus_iommu *iommu = sdev->bus->iommu; |
654 |
> - unsigned long flags, npages; |
655 |
> - iopte_t *iopte; |
656 |
> + struct sbus_iommu *iommu; |
657 |
> + unsigned long flags, npages, iopte_protection; |
658 |
> + iopte_t *base; |
659 |
> u32 dma_base; |
660 |
> struct scatterlist *sgtmp; |
661 |
> int used; |
662 |
> - unsigned long iopte_bits; |
663 |
> - |
664 |
> - if (dir == SBUS_DMA_NONE) |
665 |
> - BUG(); |
666 |
> |
667 |
> /* Fast path single entry scatterlists. */ |
668 |
> - if (nents == 1) { |
669 |
> - sg->dma_address = |
670 |
> + if (nelems == 1) { |
671 |
> + sglist->dma_address = |
672 |
> sbus_map_single(sdev, |
673 |
> - (page_address(sg->page) |
674 |
> + sg->offset), |
675 |
> - sg->length, dir); |
676 |
> - sg->dma_length = sg->length; |
677 |
> + |
678 |
> (page_address(sglist->page) + sglist->offset), |
679 |
> + sglist->length, direction); |
680 |
> + sglist->dma_length = sglist->length; |
681 |
> return 1; |
682 |
> } |
683 |
> |
684 |
> - npages = prepare_sg(sg, nents); |
685 |
> + iommu = sdev->bus->iommu; |
686 |
> + |
687 |
> + if (unlikely(direction == SBUS_DMA_NONE)) |
688 |
> + BUG(); |
689 |
> + |
690 |
> + npages = prepare_sg(sglist, nelems); |
691 |
> |
692 |
> spin_lock_irqsave(&iommu->lock, flags); |
693 |
> - iopte = alloc_streaming_cluster(iommu, npages); |
694 |
> - if (iopte == NULL) |
695 |
> - goto bad; |
696 |
> - dma_base = MAP_BASE + ((iopte - iommu->page_table) << |
697 |
> IO_PAGE_SHIFT); |
698 |
> + base = alloc_npages(iommu, npages); |
699 |
> + spin_unlock_irqrestore(&iommu->lock, flags); |
700 |
> + |
701 |
> + if (unlikely(base == NULL)) |
702 |
> + BUG(); |
703 |
> + |
704 |
> + dma_base = MAP_BASE + |
705 |
> + ((base - iommu->page_table) << IO_PAGE_SHIFT); |
706 |
> |
707 |
> /* Normalize DVMA addresses. */ |
708 |
> - sgtmp = sg; |
709 |
> - used = nents; |
710 |
> + used = nelems; |
711 |
> |
712 |
> + sgtmp = sglist; |
713 |
> while (used && sgtmp->dma_length) { |
714 |
> sgtmp->dma_address += dma_base; |
715 |
> sgtmp++; |
716 |
> used--; |
717 |
> } |
718 |
> - used = nents - used; |
719 |
> + used = nelems - used; |
720 |
> |
721 |
> - iopte_bits = IOPTE_VALID | IOPTE_STBUF | IOPTE_CACHE; |
722 |
> - if (dir != SBUS_DMA_TODEVICE) |
723 |
> - iopte_bits |= IOPTE_WRITE; |
724 |
> + iopte_protection = IOPTE_VALID | IOPTE_STBUF | IOPTE_CACHE; |
725 |
> + if (direction != SBUS_DMA_TODEVICE) |
726 |
> + iopte_protection |= IOPTE_WRITE; |
727 |
> + |
728 |
> + fill_sg(base, sglist, used, nelems, iopte_protection); |
729 |
> |
730 |
> - fill_sg(iopte, sg, used, nents, iopte_bits); |
731 |
> #ifdef VERIFY_SG |
732 |
> - verify_sglist(sg, nents, iopte, npages); |
733 |
> + verify_sglist(sglist, nelems, base, npages); |
734 |
> #endif |
735 |
> - spin_unlock_irqrestore(&iommu->lock, flags); |
736 |
> |
737 |
> return used; |
738 |
> - |
739 |
> -bad: |
740 |
> - spin_unlock_irqrestore(&iommu->lock, flags); |
741 |
> - BUG(); |
742 |
> - return 0; |
743 |
> } |
744 |
> |
745 |
> -void sbus_unmap_sg(struct sbus_dev *sdev, struct scatterlist |
746 |
> *sg, int nents, int direction) |
747 |
> +void sbus_unmap_sg(struct sbus_dev *sdev, struct scatterlist |
748 |
> *sglist, |
749 |
> +int nelems, int direction) |
750 |
> { |
751 |
> - unsigned long size, flags; |
752 |
> struct sbus_iommu *iommu; |
753 |
> - u32 dvma_base; |
754 |
> - int i; |
755 |
> + iopte_t *base; |
756 |
> + unsigned long flags, i, npages; |
757 |
> + u32 bus_addr; |
758 |
> |
759 |
> - /* Fast path single entry scatterlists. */ |
760 |
> - if (nents == 1) { |
761 |
> - sbus_unmap_single(sdev, sg->dma_address, |
762 |
> sg->dma_length, direction); |
763 |
> - return; |
764 |
> - } |
765 |
> + if (unlikely(direction == SBUS_DMA_NONE)) |
766 |
> + BUG(); |
767 |
> + |
768 |
> + iommu = sdev->bus->iommu; |
769 |
> + |
770 |
> + bus_addr = sglist->dma_address & IO_PAGE_MASK; |
771 |
> |
772 |
> - dvma_base = sg[0].dma_address & IO_PAGE_MASK; |
773 |
> - for (i = 0; i < nents; i++) { |
774 |
> - if (sg[i].dma_length == 0) |
775 |
> + for (i = 1; i < nelems; i++) |
776 |
> + if (sglist[i].dma_length == 0) |
777 |
> break; |
778 |
> - } |
779 |
> i--; |
780 |
> - size = IO_PAGE_ALIGN(sg[i].dma_address + |
781 |
> sg[i].dma_length) - dvma_base; |
782 |
> + npages = (IO_PAGE_ALIGN(sglist[i].dma_address + |
783 |
> sglist[i].dma_length) - |
784 |
> + bus_addr) >> IO_PAGE_SHIFT; |
785 |
> + |
786 |
> + base = iommu->page_table + |
787 |
> + ((bus_addr - MAP_BASE) >> IO_PAGE_SHIFT); |
788 |
> |
789 |
> - iommu = sdev->bus->iommu; |
790 |
> spin_lock_irqsave(&iommu->lock, flags); |
791 |
> - free_streaming_cluster(iommu, dvma_base, size >> IO_PAGE_SHIFT); |
792 |
> - sbus_strbuf_flush(iommu, dvma_base, size >> |
793 |
> IO_PAGE_SHIFT, direction); |
794 |
> + sbus_strbuf_flush(iommu, bus_addr, npages, direction); |
795 |
> + for (i = 0; i < npages; i++) |
796 |
> + iopte_val(base[i]) = 0UL; |
797 |
> + free_npages(iommu, bus_addr - MAP_BASE, npages); |
798 |
> spin_unlock_irqrestore(&iommu->lock, flags); } |
799 |
> |
800 |
> -void sbus_dma_sync_single_for_cpu(struct sbus_dev *sdev, |
801 |
> dma_addr_t base, size_t size, int direction) |
802 |
> +void sbus_dma_sync_single_for_cpu(struct sbus_dev *sdev, dma_addr_t |
803 |
> +bus_addr, size_t sz, int direction) |
804 |
> { |
805 |
> - struct sbus_iommu *iommu = sdev->bus->iommu; |
806 |
> - unsigned long flags; |
807 |
> + struct sbus_iommu *iommu; |
808 |
> + unsigned long flags, npages; |
809 |
> + |
810 |
> + iommu = sdev->bus->iommu; |
811 |
> |
812 |
> - size = (IO_PAGE_ALIGN(base + size) - (base & IO_PAGE_MASK)); |
813 |
> + npages = IO_PAGE_ALIGN(bus_addr + sz) - (bus_addr & |
814 |
> IO_PAGE_MASK); |
815 |
> + npages >>= IO_PAGE_SHIFT; |
816 |
> + bus_addr &= IO_PAGE_MASK; |
817 |
> |
818 |
> spin_lock_irqsave(&iommu->lock, flags); |
819 |
> - sbus_strbuf_flush(iommu, base & IO_PAGE_MASK, size >> |
820 |
> IO_PAGE_SHIFT, direction); |
821 |
> + sbus_strbuf_flush(iommu, bus_addr, npages, direction); |
822 |
> spin_unlock_irqrestore(&iommu->lock, flags); } |
823 |
> |
824 |
> @@ -613,23 +535,25 @@ void |
825 |
> sbus_dma_sync_single_for_device(struct sbus_dev *sdev, |
826 |
> dma_addr_t base, siz { } |
827 |
> |
828 |
> -void sbus_dma_sync_sg_for_cpu(struct sbus_dev *sdev, struct |
829 |
> scatterlist *sg, int nents, int direction) |
830 |
> +void sbus_dma_sync_sg_for_cpu(struct sbus_dev *sdev, struct |
831 |
> scatterlist |
832 |
> +*sglist, int nelems, int direction) |
833 |
> { |
834 |
> - struct sbus_iommu *iommu = sdev->bus->iommu; |
835 |
> - unsigned long flags, size; |
836 |
> - u32 base; |
837 |
> - int i; |
838 |
> + struct sbus_iommu *iommu; |
839 |
> + unsigned long flags, npages, i; |
840 |
> + u32 bus_addr; |
841 |
> + |
842 |
> + iommu = sdev->bus->iommu; |
843 |
> |
844 |
> - base = sg[0].dma_address & IO_PAGE_MASK; |
845 |
> - for (i = 0; i < nents; i++) { |
846 |
> - if (sg[i].dma_length == 0) |
847 |
> + bus_addr = sglist[0].dma_address & IO_PAGE_MASK; |
848 |
> + for (i = 0; i < nelems; i++) { |
849 |
> + if (!sglist[i].dma_length) |
850 |
> break; |
851 |
> } |
852 |
> i--; |
853 |
> - size = IO_PAGE_ALIGN(sg[i].dma_address + |
854 |
> sg[i].dma_length) - base; |
855 |
> + npages = (IO_PAGE_ALIGN(sglist[i].dma_address + |
856 |
> sglist[i].dma_length) |
857 |
> + - bus_addr) >> IO_PAGE_SHIFT; |
858 |
> |
859 |
> spin_lock_irqsave(&iommu->lock, flags); |
860 |
> - sbus_strbuf_flush(iommu, base, size >> IO_PAGE_SHIFT, |
861 |
> direction); |
862 |
> + sbus_strbuf_flush(iommu, bus_addr, npages, direction); |
863 |
> spin_unlock_irqrestore(&iommu->lock, flags); } |
864 |
> |
865 |
> @@ -1104,7 +1028,7 @@ static void __init sbus_iommu_init(int |
866 |
> __node, struct sbus_bus *sbus) |
867 |
> struct linux_prom64_registers *pr; |
868 |
> struct device_node *dp; |
869 |
> struct sbus_iommu *iommu; |
870 |
> - unsigned long regs, tsb_base; |
871 |
> + unsigned long regs; |
872 |
> u64 control; |
873 |
> int i; |
874 |
> |
875 |
> @@ -1132,14 +1056,6 @@ static void __init sbus_iommu_init(int |
876 |
> __node, struct sbus_bus *sbus) |
877 |
> |
878 |
> memset(iommu, 0, sizeof(*iommu)); |
879 |
> |
880 |
> - /* We start with no consistent mappings. */ |
881 |
> - iommu->lowest_consistent_map = CLUSTER_NPAGES; |
882 |
> - |
883 |
> - for (i = 0; i < NCLUSTERS; i++) { |
884 |
> - iommu->alloc_info[i].flush = 0; |
885 |
> - iommu->alloc_info[i].next = 0; |
886 |
> - } |
887 |
> - |
888 |
> /* Setup spinlock. */ |
889 |
> spin_lock_init(&iommu->lock); |
890 |
> |
891 |
> @@ -1159,25 +1075,13 @@ static void __init |
892 |
> sbus_iommu_init(int __node, struct sbus_bus *sbus) |
893 |
> sbus->portid, regs); |
894 |
> |
895 |
> /* Setup for TSB_SIZE=7, TBW_SIZE=0, MMU_DE=1, MMU_EN=1 */ |
896 |
> + sbus_iommu_table_init(iommu, IO_TSB_SIZE); |
897 |
> + |
898 |
> control = upa_readq(iommu->iommu_regs + IOMMU_CONTROL); |
899 |
> control = ((7UL << 16UL) | |
900 |
> (0UL << 2UL) | |
901 |
> (1UL << 1UL) | |
902 |
> (1UL << 0UL)); |
903 |
> - |
904 |
> - /* Using the above configuration we need 1MB iommu page |
905 |
> - * table (128K ioptes * 8 bytes per iopte). This is |
906 |
> - * page order 7 on UltraSparc. |
907 |
> - */ |
908 |
> - tsb_base = __get_free_pages(GFP_ATOMIC, get_order(IO_TSB_SIZE)); |
909 |
> - if (tsb_base == 0UL) { |
910 |
> - prom_printf("sbus_iommu_init: Fatal error, |
911 |
> cannot alloc TSB table.\n"); |
912 |
> - prom_halt(); |
913 |
> - } |
914 |
> - |
915 |
> - iommu->page_table = (iopte_t *) tsb_base; |
916 |
> - memset(iommu->page_table, 0, IO_TSB_SIZE); |
917 |
> - |
918 |
> upa_writeq(control, iommu->iommu_regs + IOMMU_CONTROL); |
919 |
> |
920 |
> /* Clean out any cruft in the IOMMU using @@ -1195,7 |
921 |
> +1099,7 @@ static void __init sbus_iommu_init(int __node, |
922 |
> struct sbus_bus *sbus) |
923 |
> upa_readq(iommu->sbus_control_reg); |
924 |
> |
925 |
> /* Give the TSB to SYSIO. */ |
926 |
> - upa_writeq(__pa(tsb_base), iommu->iommu_regs + IOMMU_TSBBASE); |
927 |
> + upa_writeq(__pa(iommu->page_table), iommu->iommu_regs + |
928 |
> +IOMMU_TSBBASE); |
929 |
> |
930 |
> /* Setup streaming buffer, DE=1 SB_EN=1 */ |
931 |
> control = (1UL << 1UL) | (1UL << 0UL); |
932 |
> - |
933 |
> To unsubscribe from this list: send the line "unsubscribe |
934 |
> sparclinux" in the body of a message to |
935 |
> majordomo@×××××××××××.org More majordomo info at |
936 |
> http://vger.kernel.org/majordomo-info.html |
937 |
> |
938 |
> Regards, |
939 |
> -- |
940 |
> Ferris McCormick (P44646, MI) <fmccor@g.o> Developer, |
941 |
> Gentoo Linux (Devrel, Sparc) |
942 |
> |
943 |
> |
944 |
-- |
945 |
gentoo-sparc@g.o mailing list |