Gentoo Archives: gentoo-sparc

From: Leif Sawyer <lsawyer@×××.com>
To: gentoo-sparc@l.g.o
Subject: RE: [gentoo-sparc] [Fwd: Re: [Repost] Ultra2 SMP freezes with heavy disk I/O]
Date: Tue, 01 May 2007 17:45:19
Message-Id: 38D04BF3A4B7B2499D19EB1DB54285EA049C002D@FNB1EX01.gci.com
In Reply to: [gentoo-sparc] [Fwd: Re: [Repost] Ultra2 SMP freezes with heavy disk I/O] by Ferris McCormick
1 Just saw that 2.6.21-gentoo was released, so I'm walking
2 through the build on my U2.
3
4 I'll post a report after some more testing.
5
6
7
8 > -----Original Message-----
9 > From: Ferris McCormick [mailto:fmccor@g.o]
10 > Sent: Thursday, April 12, 2007 4:41 AM
11 > To: Gentoo Sparc
12 > Subject: [gentoo-sparc] [Fwd: Re: [Repost] Ultra2 SMP freezes
13 > with heavy disk I/O]
14 >
15 > In case you missed it.
16 >
17 > -------- Forwarded Message --------
18 > From: David Miller <davem@×××××××××.net>
19 > To: joel.bertrand@××××××××.fr
20 > Cc: mt1@××××××××.fr, sparclinux@×××××××××××.org
21 > Subject: Re: [Repost] Ultra2 SMP freezes with heavy disk I/O
22 > Date: Wed, 11 Apr 2007 21:55:11 -0700 (PDT)
23 >
24 > From: David Miller <davem@×××××××××.net>
25 > Date: Tue, 10 Apr 2007 14:04:58 -0700 (PDT)
26 >
27 > > From: BERTRAND Joël <joel.bertrand@××××××××.fr>
28 > > Date: Tue, 20 Mar 2007 21:53:26 +0100
29 > >
30 > > > I have tried the last 2.6.21-rc4. Same bug.
31 > >
32 > > The good news is that I can reproduce this problem now on
33 > my Ultra2,
34 > > I'll try to figure out what's wrong.
35 > >
36 > > Running "dbench 32" a few time is enough to trigger it.
37 >
38 > Ok, I think I might have killed at least the bug I was able to hit.
39 >
40 > I make no promises that this will fix the issues everyone
41 > else was seeing, but let's cross our fingers :-)))
42 >
43 > Please give the following patch some testing if you can. And
44 > please be careful, messing with the DMA mapping code can do
45 > things like eat your disk :-)
46 >
47 > If all goes well I'll push this bug fix around, and I have a
48 > secret treat for your poor sparc64 SBUS users once we get past this.
49 >
50 > Thanks!
51 >
52 > commit af822e612593c1feef5052e685bca0f8d087d120
53 > Author: David S. Miller <davem@××××××××××××××××.net>
54 > Date: Wed Apr 11 21:38:45 2007 -0700
55 >
56 > [SPARC64]: Fix SBUS IOMMU allocation code.
57 >
58 > There are several IOMMU allocator bugs. Instead of
59 > trying to fix this
60 > overly complicated code, just mirror the PCI IOMMU arena allocator
61 > which is very stable and well stress tested.
62 >
63 > I tried to make the code as identical as possible so we can switch
64 > sun4u PCI and SBUS over to a common piece of IOMMU code. All that
65 > will be need are two callbacks, one to do a full IOMMU
66 > flush and one
67 > to do a streaming buffer flush.
68 >
69 > This patch gets rid of a lot of hangs and mysterious
70 > crashes on SBUS
71 > sparc64 systems, at least for me.
72 >
73 > Signed-off-by: David S. Miller <davem@×××××××××.net>
74 >
75 > diff --git a/arch/sparc64/kernel/sbus.c
76 > b/arch/sparc64/kernel/sbus.c index 01d6d86..14f78fb 100644
77 > --- a/arch/sparc64/kernel/sbus.c
78 > +++ b/arch/sparc64/kernel/sbus.c
79 > @@ -24,48 +24,25 @@
80 >
81 > #include "iommu_common.h"
82 >
83 > -/* These should be allocated on an SMP_CACHE_BYTES
84 > - * aligned boundary for optimal performance.
85 > - *
86 > - * On SYSIO, using an 8K page size we have 1GB of SBUS
87 > - * DMA space mapped. We divide this space into equally
88 > - * sized clusters. We allocate a DMA mapping from the
89 > - * cluster that matches the order of the allocation, or
90 > - * if the order is greater than the number of clusters,
91 > - * we try to allocate from the last cluster.
92 > - */
93 > -
94 > -#define NCLUSTERS 8UL
95 > -#define ONE_GIG (1UL * 1024UL * 1024UL * 1024UL)
96 > -#define CLUSTER_SIZE (ONE_GIG / NCLUSTERS)
97 > -#define CLUSTER_MASK (CLUSTER_SIZE - 1)
98 > -#define CLUSTER_NPAGES (CLUSTER_SIZE >> IO_PAGE_SHIFT)
99 > #define MAP_BASE ((u32)0xc0000000)
100 >
101 > +struct sbus_iommu_arena {
102 > + unsigned long *map;
103 > + unsigned int hint;
104 > + unsigned int limit;
105 > +};
106 > +
107 > struct sbus_iommu {
108 > -/*0x00*/spinlock_t lock;
109 > + spinlock_t lock;
110 >
111 > -/*0x08*/iopte_t *page_table;
112 > -/*0x10*/unsigned long strbuf_regs;
113 > -/*0x18*/unsigned long iommu_regs;
114 > -/*0x20*/unsigned long sbus_control_reg;
115 > + struct sbus_iommu_arena arena;
116 >
117 > -/*0x28*/volatile unsigned long strbuf_flushflag;
118 > + iopte_t *page_table;
119 > + unsigned long strbuf_regs;
120 > + unsigned long iommu_regs;
121 > + unsigned long sbus_control_reg;
122 >
123 > - /* If NCLUSTERS is ever decresed to 4 or lower,
124 > - * you must increase the size of the type of
125 > - * these counters. You have been duly warned. -DaveM
126 > - */
127 > -/*0x30*/struct {
128 > - u16 next;
129 > - u16 flush;
130 > - } alloc_info[NCLUSTERS];
131 > -
132 > - /* The lowest used consistent mapping entry. Since
133 > - * we allocate consistent maps out of cluster 0 this
134 > - * is relative to the beginning of closter 0.
135 > - */
136 > -/*0x50*/u32 lowest_consistent_map;
137 > + volatile unsigned long strbuf_flushflag;
138 > };
139 >
140 > /* Offsets from iommu_regs */
141 > @@ -91,19 +68,6 @@ static void __iommu_flushall(struct
142 > sbus_iommu *iommu)
143 > tag += 8UL;
144 > }
145 > upa_readq(iommu->sbus_control_reg);
146 > -
147 > - for (entry = 0; entry < NCLUSTERS; entry++) {
148 > - iommu->alloc_info[entry].flush =
149 > - iommu->alloc_info[entry].next;
150 > - }
151 > -}
152 > -
153 > -static void iommu_flush(struct sbus_iommu *iommu, u32 base,
154 > unsigned long npages) -{
155 > - while (npages--)
156 > - upa_writeq(base + (npages << IO_PAGE_SHIFT),
157 > - iommu->iommu_regs + IOMMU_FLUSH);
158 > - upa_readq(iommu->sbus_control_reg);
159 > }
160 >
161 > /* Offsets from strbuf_regs */
162 > @@ -156,178 +120,115 @@ static void sbus_strbuf_flush(struct
163 > sbus_iommu *iommu, u32 base, unsigned long
164 > base, npages);
165 > }
166 >
167 > -static iopte_t *alloc_streaming_cluster(struct sbus_iommu
168 > *iommu, unsigned long npages)
169 > +/* Based largely upon the ppc64 iommu allocator. */ static long
170 > +sbus_arena_alloc(struct sbus_iommu *iommu, unsigned long npages)
171 > {
172 > - iopte_t *iopte, *limit, *first, *cluster;
173 > - unsigned long cnum, ent, nent, flush_point, found;
174 > -
175 > - cnum = 0;
176 > - nent = 1;
177 > - while ((1UL << cnum) < npages)
178 > - cnum++;
179 > - if(cnum >= NCLUSTERS) {
180 > - nent = 1UL << (cnum - NCLUSTERS);
181 > - cnum = NCLUSTERS - 1;
182 > - }
183 > - iopte = iommu->page_table + (cnum * CLUSTER_NPAGES);
184 > -
185 > - if (cnum == 0)
186 > - limit = (iommu->page_table +
187 > - iommu->lowest_consistent_map);
188 > - else
189 > - limit = (iopte + CLUSTER_NPAGES);
190 > -
191 > - iopte += ((ent = iommu->alloc_info[cnum].next) << cnum);
192 > - flush_point = iommu->alloc_info[cnum].flush;
193 > -
194 > - first = iopte;
195 > - cluster = NULL;
196 > - found = 0;
197 > - for (;;) {
198 > - if (iopte_val(*iopte) == 0UL) {
199 > - found++;
200 > - if (!cluster)
201 > - cluster = iopte;
202 > + struct sbus_iommu_arena *arena = &iommu->arena;
203 > + unsigned long n, i, start, end, limit;
204 > + int pass;
205 > +
206 > + limit = arena->limit;
207 > + start = arena->hint;
208 > + pass = 0;
209 > +
210 > +again:
211 > + n = find_next_zero_bit(arena->map, limit, start);
212 > + end = n + npages;
213 > + if (unlikely(end >= limit)) {
214 > + if (likely(pass < 1)) {
215 > + limit = start;
216 > + start = 0;
217 > + __iommu_flushall(iommu);
218 > + pass++;
219 > + goto again;
220 > } else {
221 > - /* Used cluster in the way */
222 > - cluster = NULL;
223 > - found = 0;
224 > + /* Scanned the whole thing, give up. */
225 > + return -1;
226 > }
227 > + }
228 >
229 > - if (found == nent)
230 > - break;
231 > -
232 > - iopte += (1 << cnum);
233 > - ent++;
234 > - if (iopte >= limit) {
235 > - iopte = (iommu->page_table + (cnum *
236 > CLUSTER_NPAGES));
237 > - ent = 0;
238 > -
239 > - /* Multiple cluster allocations must not wrap */
240 > - cluster = NULL;
241 > - found = 0;
242 > + for (i = n; i < end; i++) {
243 > + if (test_bit(i, arena->map)) {
244 > + start = i + 1;
245 > + goto again;
246 > }
247 > - if (ent == flush_point)
248 > - __iommu_flushall(iommu);
249 > - if (iopte == first)
250 > - goto bad;
251 > }
252 >
253 > - /* ent/iopte points to the last cluster entry we're
254 > going to use,
255 > - * so save our place for the next allocation.
256 > - */
257 > - if ((iopte + (1 << cnum)) >= limit)
258 > - ent = 0;
259 > - else
260 > - ent = ent + 1;
261 > - iommu->alloc_info[cnum].next = ent;
262 > - if (ent == flush_point)
263 > - __iommu_flushall(iommu);
264 > -
265 > - /* I've got your streaming cluster right here buddy boy... */
266 > - return cluster;
267 > -
268 > -bad:
269 > - printk(KERN_EMERG "sbus: alloc_streaming_cluster of
270 > npages(%ld) failed!\n",
271 > - npages);
272 > - return NULL;
273 > + for (i = n; i < end; i++)
274 > + __set_bit(i, arena->map);
275 > +
276 > + arena->hint = end;
277 > +
278 > + return n;
279 > }
280 >
281 > -static void free_streaming_cluster(struct sbus_iommu *iommu,
282 > u32 base, unsigned long npages)
283 > +static void sbus_arena_free(struct sbus_iommu_arena *arena, unsigned
284 > +long base, unsigned long npages)
285 > {
286 > - unsigned long cnum, ent, nent;
287 > - iopte_t *iopte;
288 > + unsigned long i;
289 >
290 > - cnum = 0;
291 > - nent = 1;
292 > - while ((1UL << cnum) < npages)
293 > - cnum++;
294 > - if(cnum >= NCLUSTERS) {
295 > - nent = 1UL << (cnum - NCLUSTERS);
296 > - cnum = NCLUSTERS - 1;
297 > - }
298 > - ent = (base & CLUSTER_MASK) >> (IO_PAGE_SHIFT + cnum);
299 > - iopte = iommu->page_table + ((base - MAP_BASE) >>
300 > IO_PAGE_SHIFT);
301 > - do {
302 > - iopte_val(*iopte) = 0UL;
303 > - iopte += 1 << cnum;
304 > - } while(--nent);
305 > -
306 > - /* If the global flush might not have caught this entry,
307 > - * adjust the flush point such that we will flush before
308 > - * ever trying to reuse it.
309 > - */
310 > -#define between(X,Y,Z) (((Z) - (Y)) >= ((X) - (Y)))
311 > - if (between(ent, iommu->alloc_info[cnum].next,
312 > iommu->alloc_info[cnum].flush))
313 > - iommu->alloc_info[cnum].flush = ent;
314 > -#undef between
315 > + for (i = base; i < (base + npages); i++)
316 > + __clear_bit(i, arena->map);
317 > }
318 >
319 > -/* We allocate consistent mappings from the end of cluster
320 > zero. */ -static iopte_t *alloc_consistent_cluster(struct
321 > sbus_iommu *iommu, unsigned long npages)
322 > +static void sbus_iommu_table_init(struct sbus_iommu *iommu, unsigned
323 > +int tsbsize)
324 > {
325 > - iopte_t *iopte;
326 > + unsigned long tsbbase, order, sz, num_tsb_entries;
327 >
328 > - iopte = iommu->page_table + (1 * CLUSTER_NPAGES);
329 > - while (iopte > iommu->page_table) {
330 > - iopte--;
331 > - if (!(iopte_val(*iopte) & IOPTE_VALID)) {
332 > - unsigned long tmp = npages;
333 > + num_tsb_entries = tsbsize / sizeof(iopte_t);
334 >
335 > - while (--tmp) {
336 > - iopte--;
337 > - if (iopte_val(*iopte) & IOPTE_VALID)
338 > - break;
339 > - }
340 > - if (tmp == 0) {
341 > - u32 entry = (iopte - iommu->page_table);
342 > + /* Setup initial software IOMMU state. */
343 > + spin_lock_init(&iommu->lock);
344 >
345 > - if (entry <
346 > iommu->lowest_consistent_map)
347 > -
348 > iommu->lowest_consistent_map = entry;
349 > - return iopte;
350 > - }
351 > - }
352 > + /* Allocate and initialize the free area map. */
353 > + sz = num_tsb_entries / 8;
354 > + sz = (sz + 7UL) & ~7UL;
355 > + iommu->arena.map = kzalloc(sz, GFP_KERNEL);
356 > + if (!iommu->arena.map) {
357 > + prom_printf("PCI_IOMMU: Error,
358 > kmalloc(arena.map) failed.\n");
359 > + prom_halt();
360 > + }
361 > + iommu->arena.limit = num_tsb_entries;
362 > +
363 > + /* Now allocate and setup the IOMMU page table itself. */
364 > + order = get_order(tsbsize);
365 > + tsbbase = __get_free_pages(GFP_KERNEL, order);
366 > + if (!tsbbase) {
367 > + prom_printf("IOMMU: Error, gfp(tsb) failed.\n");
368 > + prom_halt();
369 > }
370 > - return NULL;
371 > + iommu->page_table = (iopte_t *)tsbbase;
372 > + memset(iommu->page_table, 0, tsbsize);
373 > }
374 >
375 > -static void free_consistent_cluster(struct sbus_iommu
376 > *iommu, u32 base, unsigned long npages)
377 > +static inline iopte_t *alloc_npages(struct sbus_iommu
378 > *iommu, unsigned
379 > +long npages)
380 > {
381 > - iopte_t *iopte = iommu->page_table + ((base - MAP_BASE)
382 > >> IO_PAGE_SHIFT);
383 > + long entry;
384 >
385 > - if ((iopte - iommu->page_table) ==
386 > iommu->lowest_consistent_map) {
387 > - iopte_t *walk = iopte + npages;
388 > - iopte_t *limit;
389 > + entry = sbus_arena_alloc(iommu, npages);
390 > + if (unlikely(entry < 0))
391 > + return NULL;
392 >
393 > - limit = iommu->page_table + CLUSTER_NPAGES;
394 > - while (walk < limit) {
395 > - if (iopte_val(*walk) != 0UL)
396 > - break;
397 > - walk++;
398 > - }
399 > - iommu->lowest_consistent_map =
400 > - (walk - iommu->page_table);
401 > - }
402 > + return iommu->page_table + entry;
403 > +}
404 >
405 > - while (npages--)
406 > - *iopte++ = __iopte(0UL);
407 > +static inline void free_npages(struct sbus_iommu *iommu, dma_addr_t
408 > +base, unsigned long npages) {
409 > + sbus_arena_free(&iommu->arena, base >> IO_PAGE_SHIFT, npages);
410 > }
411 >
412 > void *sbus_alloc_consistent(struct sbus_dev *sdev, size_t
413 > size, dma_addr_t *dvma_addr) {
414 > - unsigned long order, first_page, flags;
415 > struct sbus_iommu *iommu;
416 > iopte_t *iopte;
417 > + unsigned long flags, order, first_page;
418 > void *ret;
419 > int npages;
420 >
421 > - if (size <= 0 || sdev == NULL || dvma_addr == NULL)
422 > - return NULL;
423 > -
424 > size = IO_PAGE_ALIGN(size);
425 > order = get_order(size);
426 > if (order >= 10)
427 > return NULL;
428 > +
429 > first_page = __get_free_pages(GFP_KERNEL|__GFP_COMP, order);
430 > if (first_page == 0UL)
431 > return NULL;
432 > @@ -336,108 +237,121 @@ void *sbus_alloc_consistent(struct
433 > sbus_dev *sdev, size_t size, dma_addr_t *dvma
434 > iommu = sdev->bus->iommu;
435 >
436 > spin_lock_irqsave(&iommu->lock, flags);
437 > - iopte = alloc_consistent_cluster(iommu, size >> IO_PAGE_SHIFT);
438 > - if (iopte == NULL) {
439 > - spin_unlock_irqrestore(&iommu->lock, flags);
440 > + iopte = alloc_npages(iommu, size >> IO_PAGE_SHIFT);
441 > + spin_unlock_irqrestore(&iommu->lock, flags);
442 > +
443 > + if (unlikely(iopte == NULL)) {
444 > free_pages(first_page, order);
445 > return NULL;
446 > }
447 >
448 > - /* Ok, we're committed at this point. */
449 > - *dvma_addr = MAP_BASE + ((iopte - iommu->page_table) <<
450 > IO_PAGE_SHIFT);
451 > + *dvma_addr = (MAP_BASE +
452 > + ((iopte - iommu->page_table) << IO_PAGE_SHIFT));
453 > ret = (void *) first_page;
454 > npages = size >> IO_PAGE_SHIFT;
455 > + first_page = __pa(first_page);
456 > while (npages--) {
457 > - *iopte++ = __iopte(IOPTE_VALID | IOPTE_CACHE |
458 > IOPTE_WRITE |
459 > - (__pa(first_page) & IOPTE_PAGE));
460 > + iopte_val(*iopte) = (IOPTE_VALID | IOPTE_CACHE |
461 > + IOPTE_WRITE |
462 > + (first_page & IOPTE_PAGE));
463 > + iopte++;
464 > first_page += IO_PAGE_SIZE;
465 > }
466 > - iommu_flush(iommu, *dvma_addr, size >> IO_PAGE_SHIFT);
467 > - spin_unlock_irqrestore(&iommu->lock, flags);
468 >
469 > return ret;
470 > }
471 >
472 > void sbus_free_consistent(struct sbus_dev *sdev, size_t
473 > size, void *cpu, dma_addr_t dvma) {
474 > - unsigned long order, npages;
475 > struct sbus_iommu *iommu;
476 > -
477 > - if (size <= 0 || sdev == NULL || cpu == NULL)
478 > - return;
479 > + iopte_t *iopte;
480 > + unsigned long flags, order, npages;
481 >
482 > npages = IO_PAGE_ALIGN(size) >> IO_PAGE_SHIFT;
483 > iommu = sdev->bus->iommu;
484 > + iopte = iommu->page_table +
485 > + ((dvma - MAP_BASE) >> IO_PAGE_SHIFT);
486 > +
487 > + spin_lock_irqsave(&iommu->lock, flags);
488 > +
489 > + free_npages(iommu, dvma - MAP_BASE, npages);
490 >
491 > - spin_lock_irq(&iommu->lock);
492 > - free_consistent_cluster(iommu, dvma, npages);
493 > - iommu_flush(iommu, dvma, npages);
494 > - spin_unlock_irq(&iommu->lock);
495 > + spin_unlock_irqrestore(&iommu->lock, flags);
496 >
497 > order = get_order(size);
498 > if (order < 10)
499 > free_pages((unsigned long)cpu, order); }
500 >
501 > -dma_addr_t sbus_map_single(struct sbus_dev *sdev, void *ptr,
502 > size_t size, int dir)
503 > +dma_addr_t sbus_map_single(struct sbus_dev *sdev, void *ptr,
504 > size_t sz,
505 > +int direction)
506 > {
507 > - struct sbus_iommu *iommu = sdev->bus->iommu;
508 > - unsigned long npages, pbase, flags;
509 > - iopte_t *iopte;
510 > - u32 dma_base, offset;
511 > - unsigned long iopte_bits;
512 > + struct sbus_iommu *iommu;
513 > + iopte_t *base;
514 > + unsigned long flags, npages, oaddr;
515 > + unsigned long i, base_paddr;
516 > + u32 bus_addr, ret;
517 > + unsigned long iopte_protection;
518 > +
519 > + iommu = sdev->bus->iommu;
520 >
521 > - if (dir == SBUS_DMA_NONE)
522 > + if (unlikely(direction == SBUS_DMA_NONE))
523 > BUG();
524 >
525 > - pbase = (unsigned long) ptr;
526 > - offset = (u32) (pbase & ~IO_PAGE_MASK);
527 > - size = (IO_PAGE_ALIGN(pbase + size) - (pbase & IO_PAGE_MASK));
528 > - pbase = (unsigned long) __pa(pbase & IO_PAGE_MASK);
529 > + oaddr = (unsigned long)ptr;
530 > + npages = IO_PAGE_ALIGN(oaddr + sz) - (oaddr & IO_PAGE_MASK);
531 > + npages >>= IO_PAGE_SHIFT;
532 >
533 > spin_lock_irqsave(&iommu->lock, flags);
534 > - npages = size >> IO_PAGE_SHIFT;
535 > - iopte = alloc_streaming_cluster(iommu, npages);
536 > - if (iopte == NULL)
537 > - goto bad;
538 > - dma_base = MAP_BASE + ((iopte - iommu->page_table) <<
539 > IO_PAGE_SHIFT);
540 > - npages = size >> IO_PAGE_SHIFT;
541 > - iopte_bits = IOPTE_VALID | IOPTE_STBUF | IOPTE_CACHE;
542 > - if (dir != SBUS_DMA_TODEVICE)
543 > - iopte_bits |= IOPTE_WRITE;
544 > - while (npages--) {
545 > - *iopte++ = __iopte(iopte_bits | (pbase & IOPTE_PAGE));
546 > - pbase += IO_PAGE_SIZE;
547 > - }
548 > - npages = size >> IO_PAGE_SHIFT;
549 > + base = alloc_npages(iommu, npages);
550 > spin_unlock_irqrestore(&iommu->lock, flags);
551 >
552 > - return (dma_base | offset);
553 > + if (unlikely(!base))
554 > + BUG();
555 >
556 > -bad:
557 > - spin_unlock_irqrestore(&iommu->lock, flags);
558 > - BUG();
559 > - return 0;
560 > + bus_addr = (MAP_BASE +
561 > + ((base - iommu->page_table) << IO_PAGE_SHIFT));
562 > + ret = bus_addr | (oaddr & ~IO_PAGE_MASK);
563 > + base_paddr = __pa(oaddr & IO_PAGE_MASK);
564 > +
565 > + iopte_protection = IOPTE_VALID | IOPTE_STBUF | IOPTE_CACHE;
566 > + if (direction != SBUS_DMA_TODEVICE)
567 > + iopte_protection |= IOPTE_WRITE;
568 > +
569 > + for (i = 0; i < npages; i++, base++, base_paddr += IO_PAGE_SIZE)
570 > + iopte_val(*base) = iopte_protection | base_paddr;
571 > +
572 > + return ret;
573 > }
574 >
575 > -void sbus_unmap_single(struct sbus_dev *sdev, dma_addr_t
576 > dma_addr, size_t size, int direction)
577 > +void sbus_unmap_single(struct sbus_dev *sdev, dma_addr_t bus_addr,
578 > +size_t sz, int direction)
579 > {
580 > struct sbus_iommu *iommu = sdev->bus->iommu;
581 > - u32 dma_base = dma_addr & IO_PAGE_MASK;
582 > - unsigned long flags;
583 > + iopte_t *base;
584 > + unsigned long flags, npages, i;
585 > +
586 > + if (unlikely(direction == SBUS_DMA_NONE))
587 > + BUG();
588 > +
589 > + npages = IO_PAGE_ALIGN(bus_addr + sz) - (bus_addr &
590 > IO_PAGE_MASK);
591 > + npages >>= IO_PAGE_SHIFT;
592 > + base = iommu->page_table +
593 > + ((bus_addr - MAP_BASE) >> IO_PAGE_SHIFT);
594 >
595 > - size = (IO_PAGE_ALIGN(dma_addr + size) - dma_base);
596 > + bus_addr &= IO_PAGE_MASK;
597 >
598 > spin_lock_irqsave(&iommu->lock, flags);
599 > - free_streaming_cluster(iommu, dma_base, size >> IO_PAGE_SHIFT);
600 > - sbus_strbuf_flush(iommu, dma_base, size >>
601 > IO_PAGE_SHIFT, direction);
602 > + sbus_strbuf_flush(iommu, bus_addr, npages, direction);
603 > + for (i = 0; i < npages; i++)
604 > + iopte_val(base[i]) = 0UL;
605 > + free_npages(iommu, bus_addr - MAP_BASE, npages);
606 > spin_unlock_irqrestore(&iommu->lock, flags); }
607 >
608 > #define SG_ENT_PHYS_ADDRESS(SG) \
609 > (__pa(page_address((SG)->page)) + (SG)->offset)
610 >
611 > -static inline void fill_sg(iopte_t *iopte, struct
612 > scatterlist *sg, int nused, int nelems, unsigned long iopte_bits)
613 > +static inline void fill_sg(iopte_t *iopte, struct scatterlist *sg,
614 > + int nused, int nelems, unsigned long
615 > iopte_protection)
616 > {
617 > struct scatterlist *dma_sg = sg;
618 > struct scatterlist *sg_end = sg + nelems; @@ -462,7
619 > +376,7 @@ static inline void fill_sg(iopte_t *iopte, struct
620 > scatterlist *sg, int nused, in
621 > for (;;) {
622 > unsigned long tmp;
623 >
624 > - tmp = (unsigned long)
625 > SG_ENT_PHYS_ADDRESS(sg);
626 > + tmp = SG_ENT_PHYS_ADDRESS(sg);
627 > len = sg->length;
628 > if (((tmp ^ pteval) >>
629 > IO_PAGE_SHIFT) != 0UL) {
630 > pteval = tmp & IO_PAGE_MASK;
631 > @@ -478,7 +392,7 @@ static inline void fill_sg(iopte_t
632 > *iopte, struct scatterlist *sg, int nused, in
633 > sg++;
634 > }
635 >
636 > - pteval = ((pteval & IOPTE_PAGE) | iopte_bits);
637 > + pteval = iopte_protection | (pteval &
638 > IOPTE_PAGE);
639 > while (len > 0) {
640 > *iopte++ = __iopte(pteval);
641 > pteval += IO_PAGE_SIZE;
642 > @@ -509,103 +423,111 @@ static inline void fill_sg(iopte_t
643 > *iopte, struct scatterlist *sg, int nused, in
644 > }
645 > }
646 >
647 > -int sbus_map_sg(struct sbus_dev *sdev, struct scatterlist
648 > *sg, int nents, int dir)
649 > +int sbus_map_sg(struct sbus_dev *sdev, struct scatterlist
650 > *sglist, int
651 > +nelems, int direction)
652 > {
653 > - struct sbus_iommu *iommu = sdev->bus->iommu;
654 > - unsigned long flags, npages;
655 > - iopte_t *iopte;
656 > + struct sbus_iommu *iommu;
657 > + unsigned long flags, npages, iopte_protection;
658 > + iopte_t *base;
659 > u32 dma_base;
660 > struct scatterlist *sgtmp;
661 > int used;
662 > - unsigned long iopte_bits;
663 > -
664 > - if (dir == SBUS_DMA_NONE)
665 > - BUG();
666 >
667 > /* Fast path single entry scatterlists. */
668 > - if (nents == 1) {
669 > - sg->dma_address =
670 > + if (nelems == 1) {
671 > + sglist->dma_address =
672 > sbus_map_single(sdev,
673 > - (page_address(sg->page)
674 > + sg->offset),
675 > - sg->length, dir);
676 > - sg->dma_length = sg->length;
677 > +
678 > (page_address(sglist->page) + sglist->offset),
679 > + sglist->length, direction);
680 > + sglist->dma_length = sglist->length;
681 > return 1;
682 > }
683 >
684 > - npages = prepare_sg(sg, nents);
685 > + iommu = sdev->bus->iommu;
686 > +
687 > + if (unlikely(direction == SBUS_DMA_NONE))
688 > + BUG();
689 > +
690 > + npages = prepare_sg(sglist, nelems);
691 >
692 > spin_lock_irqsave(&iommu->lock, flags);
693 > - iopte = alloc_streaming_cluster(iommu, npages);
694 > - if (iopte == NULL)
695 > - goto bad;
696 > - dma_base = MAP_BASE + ((iopte - iommu->page_table) <<
697 > IO_PAGE_SHIFT);
698 > + base = alloc_npages(iommu, npages);
699 > + spin_unlock_irqrestore(&iommu->lock, flags);
700 > +
701 > + if (unlikely(base == NULL))
702 > + BUG();
703 > +
704 > + dma_base = MAP_BASE +
705 > + ((base - iommu->page_table) << IO_PAGE_SHIFT);
706 >
707 > /* Normalize DVMA addresses. */
708 > - sgtmp = sg;
709 > - used = nents;
710 > + used = nelems;
711 >
712 > + sgtmp = sglist;
713 > while (used && sgtmp->dma_length) {
714 > sgtmp->dma_address += dma_base;
715 > sgtmp++;
716 > used--;
717 > }
718 > - used = nents - used;
719 > + used = nelems - used;
720 >
721 > - iopte_bits = IOPTE_VALID | IOPTE_STBUF | IOPTE_CACHE;
722 > - if (dir != SBUS_DMA_TODEVICE)
723 > - iopte_bits |= IOPTE_WRITE;
724 > + iopte_protection = IOPTE_VALID | IOPTE_STBUF | IOPTE_CACHE;
725 > + if (direction != SBUS_DMA_TODEVICE)
726 > + iopte_protection |= IOPTE_WRITE;
727 > +
728 > + fill_sg(base, sglist, used, nelems, iopte_protection);
729 >
730 > - fill_sg(iopte, sg, used, nents, iopte_bits);
731 > #ifdef VERIFY_SG
732 > - verify_sglist(sg, nents, iopte, npages);
733 > + verify_sglist(sglist, nelems, base, npages);
734 > #endif
735 > - spin_unlock_irqrestore(&iommu->lock, flags);
736 >
737 > return used;
738 > -
739 > -bad:
740 > - spin_unlock_irqrestore(&iommu->lock, flags);
741 > - BUG();
742 > - return 0;
743 > }
744 >
745 > -void sbus_unmap_sg(struct sbus_dev *sdev, struct scatterlist
746 > *sg, int nents, int direction)
747 > +void sbus_unmap_sg(struct sbus_dev *sdev, struct scatterlist
748 > *sglist,
749 > +int nelems, int direction)
750 > {
751 > - unsigned long size, flags;
752 > struct sbus_iommu *iommu;
753 > - u32 dvma_base;
754 > - int i;
755 > + iopte_t *base;
756 > + unsigned long flags, i, npages;
757 > + u32 bus_addr;
758 >
759 > - /* Fast path single entry scatterlists. */
760 > - if (nents == 1) {
761 > - sbus_unmap_single(sdev, sg->dma_address,
762 > sg->dma_length, direction);
763 > - return;
764 > - }
765 > + if (unlikely(direction == SBUS_DMA_NONE))
766 > + BUG();
767 > +
768 > + iommu = sdev->bus->iommu;
769 > +
770 > + bus_addr = sglist->dma_address & IO_PAGE_MASK;
771 >
772 > - dvma_base = sg[0].dma_address & IO_PAGE_MASK;
773 > - for (i = 0; i < nents; i++) {
774 > - if (sg[i].dma_length == 0)
775 > + for (i = 1; i < nelems; i++)
776 > + if (sglist[i].dma_length == 0)
777 > break;
778 > - }
779 > i--;
780 > - size = IO_PAGE_ALIGN(sg[i].dma_address +
781 > sg[i].dma_length) - dvma_base;
782 > + npages = (IO_PAGE_ALIGN(sglist[i].dma_address +
783 > sglist[i].dma_length) -
784 > + bus_addr) >> IO_PAGE_SHIFT;
785 > +
786 > + base = iommu->page_table +
787 > + ((bus_addr - MAP_BASE) >> IO_PAGE_SHIFT);
788 >
789 > - iommu = sdev->bus->iommu;
790 > spin_lock_irqsave(&iommu->lock, flags);
791 > - free_streaming_cluster(iommu, dvma_base, size >> IO_PAGE_SHIFT);
792 > - sbus_strbuf_flush(iommu, dvma_base, size >>
793 > IO_PAGE_SHIFT, direction);
794 > + sbus_strbuf_flush(iommu, bus_addr, npages, direction);
795 > + for (i = 0; i < npages; i++)
796 > + iopte_val(base[i]) = 0UL;
797 > + free_npages(iommu, bus_addr - MAP_BASE, npages);
798 > spin_unlock_irqrestore(&iommu->lock, flags); }
799 >
800 > -void sbus_dma_sync_single_for_cpu(struct sbus_dev *sdev,
801 > dma_addr_t base, size_t size, int direction)
802 > +void sbus_dma_sync_single_for_cpu(struct sbus_dev *sdev, dma_addr_t
803 > +bus_addr, size_t sz, int direction)
804 > {
805 > - struct sbus_iommu *iommu = sdev->bus->iommu;
806 > - unsigned long flags;
807 > + struct sbus_iommu *iommu;
808 > + unsigned long flags, npages;
809 > +
810 > + iommu = sdev->bus->iommu;
811 >
812 > - size = (IO_PAGE_ALIGN(base + size) - (base & IO_PAGE_MASK));
813 > + npages = IO_PAGE_ALIGN(bus_addr + sz) - (bus_addr &
814 > IO_PAGE_MASK);
815 > + npages >>= IO_PAGE_SHIFT;
816 > + bus_addr &= IO_PAGE_MASK;
817 >
818 > spin_lock_irqsave(&iommu->lock, flags);
819 > - sbus_strbuf_flush(iommu, base & IO_PAGE_MASK, size >>
820 > IO_PAGE_SHIFT, direction);
821 > + sbus_strbuf_flush(iommu, bus_addr, npages, direction);
822 > spin_unlock_irqrestore(&iommu->lock, flags); }
823 >
824 > @@ -613,23 +535,25 @@ void
825 > sbus_dma_sync_single_for_device(struct sbus_dev *sdev,
826 > dma_addr_t base, siz { }
827 >
828 > -void sbus_dma_sync_sg_for_cpu(struct sbus_dev *sdev, struct
829 > scatterlist *sg, int nents, int direction)
830 > +void sbus_dma_sync_sg_for_cpu(struct sbus_dev *sdev, struct
831 > scatterlist
832 > +*sglist, int nelems, int direction)
833 > {
834 > - struct sbus_iommu *iommu = sdev->bus->iommu;
835 > - unsigned long flags, size;
836 > - u32 base;
837 > - int i;
838 > + struct sbus_iommu *iommu;
839 > + unsigned long flags, npages, i;
840 > + u32 bus_addr;
841 > +
842 > + iommu = sdev->bus->iommu;
843 >
844 > - base = sg[0].dma_address & IO_PAGE_MASK;
845 > - for (i = 0; i < nents; i++) {
846 > - if (sg[i].dma_length == 0)
847 > + bus_addr = sglist[0].dma_address & IO_PAGE_MASK;
848 > + for (i = 0; i < nelems; i++) {
849 > + if (!sglist[i].dma_length)
850 > break;
851 > }
852 > i--;
853 > - size = IO_PAGE_ALIGN(sg[i].dma_address +
854 > sg[i].dma_length) - base;
855 > + npages = (IO_PAGE_ALIGN(sglist[i].dma_address +
856 > sglist[i].dma_length)
857 > + - bus_addr) >> IO_PAGE_SHIFT;
858 >
859 > spin_lock_irqsave(&iommu->lock, flags);
860 > - sbus_strbuf_flush(iommu, base, size >> IO_PAGE_SHIFT,
861 > direction);
862 > + sbus_strbuf_flush(iommu, bus_addr, npages, direction);
863 > spin_unlock_irqrestore(&iommu->lock, flags); }
864 >
865 > @@ -1104,7 +1028,7 @@ static void __init sbus_iommu_init(int
866 > __node, struct sbus_bus *sbus)
867 > struct linux_prom64_registers *pr;
868 > struct device_node *dp;
869 > struct sbus_iommu *iommu;
870 > - unsigned long regs, tsb_base;
871 > + unsigned long regs;
872 > u64 control;
873 > int i;
874 >
875 > @@ -1132,14 +1056,6 @@ static void __init sbus_iommu_init(int
876 > __node, struct sbus_bus *sbus)
877 >
878 > memset(iommu, 0, sizeof(*iommu));
879 >
880 > - /* We start with no consistent mappings. */
881 > - iommu->lowest_consistent_map = CLUSTER_NPAGES;
882 > -
883 > - for (i = 0; i < NCLUSTERS; i++) {
884 > - iommu->alloc_info[i].flush = 0;
885 > - iommu->alloc_info[i].next = 0;
886 > - }
887 > -
888 > /* Setup spinlock. */
889 > spin_lock_init(&iommu->lock);
890 >
891 > @@ -1159,25 +1075,13 @@ static void __init
892 > sbus_iommu_init(int __node, struct sbus_bus *sbus)
893 > sbus->portid, regs);
894 >
895 > /* Setup for TSB_SIZE=7, TBW_SIZE=0, MMU_DE=1, MMU_EN=1 */
896 > + sbus_iommu_table_init(iommu, IO_TSB_SIZE);
897 > +
898 > control = upa_readq(iommu->iommu_regs + IOMMU_CONTROL);
899 > control = ((7UL << 16UL) |
900 > (0UL << 2UL) |
901 > (1UL << 1UL) |
902 > (1UL << 0UL));
903 > -
904 > - /* Using the above configuration we need 1MB iommu page
905 > - * table (128K ioptes * 8 bytes per iopte). This is
906 > - * page order 7 on UltraSparc.
907 > - */
908 > - tsb_base = __get_free_pages(GFP_ATOMIC, get_order(IO_TSB_SIZE));
909 > - if (tsb_base == 0UL) {
910 > - prom_printf("sbus_iommu_init: Fatal error,
911 > cannot alloc TSB table.\n");
912 > - prom_halt();
913 > - }
914 > -
915 > - iommu->page_table = (iopte_t *) tsb_base;
916 > - memset(iommu->page_table, 0, IO_TSB_SIZE);
917 > -
918 > upa_writeq(control, iommu->iommu_regs + IOMMU_CONTROL);
919 >
920 > /* Clean out any cruft in the IOMMU using @@ -1195,7
921 > +1099,7 @@ static void __init sbus_iommu_init(int __node,
922 > struct sbus_bus *sbus)
923 > upa_readq(iommu->sbus_control_reg);
924 >
925 > /* Give the TSB to SYSIO. */
926 > - upa_writeq(__pa(tsb_base), iommu->iommu_regs + IOMMU_TSBBASE);
927 > + upa_writeq(__pa(iommu->page_table), iommu->iommu_regs +
928 > +IOMMU_TSBBASE);
929 >
930 > /* Setup streaming buffer, DE=1 SB_EN=1 */
931 > control = (1UL << 1UL) | (1UL << 0UL);
932 > -
933 > To unsubscribe from this list: send the line "unsubscribe
934 > sparclinux" in the body of a message to
935 > majordomo@×××××××××××.org More majordomo info at
936 > http://vger.kernel.org/majordomo-info.html
937 >
938 > Regards,
939 > --
940 > Ferris McCormick (P44646, MI) <fmccor@g.o> Developer,
941 > Gentoo Linux (Devrel, Sparc)
942 >
943 >
944 --
945 gentoo-sparc@g.o mailing list

Replies