RE: dcache BUG()

Brian,

I have also seen the dcache BUG, as well as bugs and warnings from other
parts of the kernel in the MontaVista 2.4.0 kernel.  They all seem to be
related to a inconsistency with reference counters, which led me to suspect
a problem with atomic instructions in our kernel.  I have replaced the
lwarx/stcrx pairs in include/asm-ppc/atomic.h with code that just turns off
and on interrupts, and that seemed to have made the error messages and BUGs
disappear.  Please try this patch and see if you still have the same
problems.  This is really just a work around for us until we find out what
is the real problem.

thanks,
Eli

diff -c -r1.1.1.2 atomic.h
*** atomic.h 2001/02/21 00:53:16 1.1.1.2

--- atomic.h 2001/05/07 18:35:10

***************
*** 5,10 ****

--- 5,25 ----
  #ifndef _ASM_PPC_ATOMIC_H_
  #define _ASM_PPC_ATOMIC_H_

+ struct int_control_struct
+ {
+         void (*int_cli)(void);
+         void (*int_sti)(void);
+         void (*int_restore_flags)(unsigned long);
+         void (*int_save_flags)(unsigned long *);
+         void (*int_set_lost)(unsigned long);
+ };
+
+ extern struct int_control_struct int_control;
+ #define __cli() int_control.int_cli()
+ #define __sti() int_control.int_sti()
+ #define __save_flags(flags) int_control.int_save_flags((unsigned long

*)&flags)
+ #define __restore_flags(flags) int_control.int_restore_flags((unsigned
long)flags)
+
  typedef struct { volatile int counter; } atomic_t;

  #define ATOMIC_INIT(i) { (i) }
***************
*** 17,80 ****

  static __inline__ int atomic_add_return(int a, atomic_t *v)
  {
!  int t;

!  __asm__ __volatile__("\n\
! 1: lwarx %0,0,%3\n\
!  add %0,%2,%0\n\
!  stwcx. %0,0,%3\n\
!  bne- 1b"
!  : "=&r" (t), "=m" (v->counter)
!  : "r" (a), "r" (v), "m" (v->counter)
!  : "cc");

   return t;
  }

  static __inline__ int atomic_sub_return(int a, atomic_t *v)
  {
!  int t;

!  __asm__ __volatile__("\n\
! 1: lwarx %0,0,%3\n\
!  subf %0,%2,%0\n\
!  stwcx. %0,0,%3\n\
!  bne- 1b"
!  : "=&r" (t), "=m" (v->counter)
!  : "r" (a), "r" (v), "m" (v->counter)
!  : "cc");

   return t;
  }

  static __inline__ int atomic_inc_return(atomic_t *v)
  {
!  int t;

!  __asm__ __volatile__("\n\
! 1: lwarx %0,0,%2\n\
!  addic %0,%0,1\n\
!  stwcx. %0,0,%2\n\
!  bne- 1b"
!  : "=&r" (t), "=m" (v->counter)
!  : "r" (v), "m" (v->counter)
!  : "cc");

   return t;
  }

  static __inline__ int atomic_dec_return(atomic_t *v)
  {
!  int t;

!  __asm__ __volatile__("\n\
! 1: lwarx %0,0,%2\n\
!  addic %0,%0,-1\n\
!  stwcx. %0,0,%2\n\
!  bne 1b"
!  : "=&r" (t), "=m" (v->counter)
!  : "r" (v), "m" (v->counter)
!  : "cc");

   return t;
  }

--- 32,91 ----

  static __inline__ int atomic_add_return(int a, atomic_t *v)
  {

!  int flags, t;
!
!  __save_flags(flags);__cli();

!  t = (v)->counter;
!  t = t + a;
!  (v)->counter = t;

+  __restore_flags(flags);
+
   return t;
  }

  static __inline__ int atomic_sub_return(int a, atomic_t *v)
  {
!  int flags, t;
!
!  __save_flags(flags);__cli();

!  t = (v)->counter;
!  t = t - a;
!  (v)->counter = t;

+  __restore_flags(flags);
+
   return t;
  }

  static __inline__ int atomic_inc_return(atomic_t *v)
  {
!  int flags, t;
!
!  __save_flags(flags);__cli();
!
!  t = (v)->counter;
!  t = t + 1;
!  (v)->counter = t;

!  __restore_flags(flags);

   return t;
  }

  static __inline__ int atomic_dec_return(atomic_t *v)
  {
!  int flags, t;
!
!  __save_flags(flags);__cli();
!
!  t = (v)->counter;
!  t = t - 1;
!  (v)->counter = t;

!  __restore_flags(flags);

   return t;
  }
diff -c -r1.1.1.2 hw_irq.h
*** hw_irq.h 2001/02/21 00:53:16 1.1.1.2

--- hw_irq.h 2001/05/07 18:35:27

***************
*** 7,12 ****

--- 7,13 ----
  #ifndef _PPC_HW_IRQ_H
  #define _PPC_HW_IRQ_H

+ #ifndef _ASM_PPC_ATOMIC_H_
  struct int_control_struct
  {
   void (*int_cli)(void);

***************
*** 15,20 ****

--- 16,23 ----
   void (*int_save_flags)(unsigned long *);
   void (*int_set_lost)(unsigned long);
  };
+ #endif
+
  extern struct int_control_struct int_control;
  extern unsigned long timer_interrupt_intercept;
  extern unsigned long do_IRQ_intercept;

** Sent via the linuxppc-embedded mail list. See http://lists.linuxppc.org/

Eli Chen wrote:

I have also seen the dcache BUG, as well as bugs and warnings from other
parts of the kernel in the MontaVista 2.4.0 kernel.

Again, I don't know what a "MontaVista 2.4.0" kernel would be.
MontaVista clearly names our software distributions and releases
them on a CD after a QA process.  If you can duplicate this
problem with the software that is on the CD, and then use the
proper reference name, we could all use the same baseline.  Of
course, it would be nicer if you would have purchased the subscription
with the CD and called your dedicated technical representative,
but we'll provide the best free help available now :-).



	-- Dan

** Sent via the linuxppc-embedded mail list. See http://lists.linuxppc.org/

Heh....the best free help wasn't available when I wrote that last
message, so I guess I'll take a stab at an answer :-).

Eli Chen wrote:

..........  They all seem to be
related to a inconsistency with reference counters, which led me to suspect
a problem with atomic instructions in our kernel.

That's an interesting piece of information......

There have been problems with the directory entry counters in
older 2.3.99/2.4 kernels.  The 4xx development has kind of been on
a planet all by it's lonesome for a long time, and I am now trying
to bring it back into the mainstream.  It could very well be there
were some generic kernel bug fixes that were missed in the 4xx
kernel.

..... I have replaced the
lwarx/stcrx pairs in include/asm-ppc/atomic.h with code that just turns off
and on interrupts,

What version of silicon do you have, and what platform are you using?

...  This is really just a work around for us until we find out what
is the real problem.

Is there some simple test I can use to trigger this problem?  It
would be nice if you could try a "newer" kernel from FSM Labs.  This
was originally in the linuxppc_2_5 tree, and we are merging/changing
trees at the moment.  I don't remember the URL......TOM, can you
provide some insight?


	-- Dan

** Sent via the linuxppc-embedded mail list. See http://lists.linuxppc.org/

What version of silicon do you have, and what platform are you using?

I am using the 405GP core, rev D.  My tree is based off of the February 26th
source from MontaVista.

Is there some simple test I can use to trigger this problem?

Besides Brian's one-liner test, you can try flood pinging your 405GP.  I
have been consistently receiving these error messages after letting it run a
while:

Freeing alive device (cxxxxxxx), ethx

and

Attempt to release alive inet socket cxxxxxxx

I have also occasionaly received other messages, which I have yet to receive
after changing atomic.h.

Eli


** Sent via the linuxppc-embedded mail list. See http://lists.linuxppc.org/

Eli Chen wrote:

I have also occasionaly received other messages, which I have yet to receive
after changing atomic.h.

I dunno..........The code can't be broken because it is used
everyday on bazillions of processors.....I can't believe these
instructions are "broken" on the 4xx because that would be too
obvious as well.  I suspect all you are doing is changing some
code timing and masking the real problem.  I'm not saying we
shouldn't fix it, just that it isn't likely the problem is here.


	-- Dan

** Sent via the linuxppc-embedded mail list. See http://lists.linuxppc.org/

On Mon, 7 May 2001, Eli Chen wrote:

quoted

What version of silicon do you have, and what platform are you using?

I am using the 405GP core, rev D.  My tree is based off of the February 26th
source from MontaVista.

quoted

Is there some simple test I can use to trigger this problem?

Besides Brian's one-liner test, you can try flood pinging your 405GP.  I
have been consistently receiving these error messages after letting it run a
while:

Freeing alive device (cxxxxxxx), ethx

and

Attempt to release alive inet socket cxxxxxxx

I have also occasionaly received other messages, which I have yet to receive
after changing atomic.h.

Hmm, consider what happens if a down_trylock in an interrupt handler
fails. Actually dec_if_positive will leave a dangling reservation, since
it will skip the stwcx. instruction.

I had not looked at the code for very long, so I may misss something or be
completely wrong but I see a stwcx. instruction in transfer_to_handler
which I think is useless since the handler will always execute a lwarx
before attempting a stwcx., thereby making the state of the reservation at
the interrupt enty irrelevant.

On the other hand, when an interrupt handler dows a down_trylock and
then returns because it failed, it will leave the reservation active
until returing to the caller (once again if I did not miss anything in the
return path).

So the sequence of events which can cause corruption is the following:

1) lwarx atomic_var,
   reservation set
2) interrrupt taken,
   reservation set
3) stwcx. in interrupt prologue (transfer_to_handler),
    reservation cleared
4) interrupt handler executes, talks to hardware
5) interupt handler modifies atomic_var,
   reservation set and cleared (hence step 3 was not necessary)
6) down_trylock() fails,
   reservation set
8) interrupt handler returns,
   reservation still set
9) interrupt epilogue restores state and returns between lwarx and stwcx.,
   reservation is still set!
10) swtcx. atomic_var, succeeds, but the variable has been modified in the
   meantime, chaos ensues

In short, I think that step 3) should be moved to the epilogue(s),
ret_from_intercept, etc... Note that spin_trylock() could produce the same
effect in step 6), but it's SMP only.

What do you think, am I completely off base ?

I try to avoid looking at entry.S/head.S/misc.S and had not done it for a
long time since I think it's dangerous for my mental health, so, once
again, I might be completely wrong.

So I'd suggest the following one liner:

===== arch/ppc/kernel/entry.S 1.7 vs edited =====

--- 1.7/arch/ppc/kernel/entry.S	Fri Apr 13 20:44:42 2001
+++ edited/arch/ppc/kernel/entry.S	Tue May  8 01:02:47 2001

@@ -382,6 +382,7 @@
 	CLR_TOP32(r8)
 	mtspr	SPRG2,r8		/* phys exception stack pointer */
 1:
+	stwcx.	r0,0,r1			/* Clear reservation - Gabriel. */
 	lwz	r3,_CTR(r1)
 	lwz	r0,_LINK(r1)
 	mtctr	r3

You can also try to remove the

	li r22,RESULT
	stwcx. r22,r22,r21

or similar lines in head.S, head_4xx.S, and head_8xx.S to check that my
theory is correct.

	Regards,
	Gabriel.


** Sent via the linuxppc-embedded mail list. See http://lists.linuxppc.org/

Gabriel Paubert wrote:

10) swtcx. atomic_var, succeeds, but the variable has been modified in the
   meantime, chaos ensues

How can this happen?  The reservation for the lwarx in 1) has
long been broken, so this swtcx. will fail.....


	-- Dan

** Sent via the linuxppc-embedded mail list. See http://lists.linuxppc.org/

On Mon, 7 May 2001, Dan Malek wrote:

Gabriel Paubert wrote:

quoted

10) swtcx. atomic_var, succeeds, but the variable has been modified in the
   meantime, chaos ensues

How can this happen?  The reservation for the lwarx in 1) has
long been broken, so this swtcx. will fail.....

Because the reservation has been set by the interrupt handler. Read the
scenario again, it's not the reservation from step 1), it's been cleared
twice, it's the stale reservation from an unmatched lwarx from step 6) in
a fainled down_trylock() in the interrupt handler.

Duh, I should go to bed, there is no step 7) in my mail :-)

What I claim is that the stwcx. in transfer_to_handler is useless (but
harmless) but we should always clear the reservation before an rfi in case
a dangling reservation was left by the handler and the return address of
the rfi is between a lwarx and its matching stwcx.

	Regards,
	Gabriel.


** Sent via the linuxppc-embedded mail list. See http://lists.linuxppc.org/

How can this happen?  The reservation for the lwarx in 1) has
long been broken, so this swtcx. will fail.....

-- Dan

Because reservation is held per processor in the "Reservation bit", and it
doesn't seem like the 405GP checks the reservation address.

From the PPC manual:

"Because the hardware doesn't compare reservation address when executing the
stwcx. instruction, operating systems software MUST reset the reservation if
an exception or other types of interrupt occurs to insure atomic memory
references of lwarx and stwcx. pairs."

-eli


** Sent via the linuxppc-embedded mail list. See http://lists.linuxppc.org/

Eli Chen wrote:

quoted

From the PPC manual:

"Because the hardware doesn't compare reservation address when executing the
stwcx.

F**K...that's what I was looking for.  What manual is that in?
Everything I have handy (older UISA books), state the granularity
is implementation dependent.  I couldn't find any 4xx manual that
stated the granularity of the reservation.  I thought 6xx/7xx at
least checked cache line granularity in addition to a single
reservation bit.

Gabriel is right................


	-- Dan

** Sent via the linuxppc-embedded mail list. See http://lists.linuxppc.org/

On Mon, 7 May 2001, Eli Chen wrote:

quoted

How can this happen?  The reservation for the lwarx in 1) has
long been broken, so this swtcx. will fail.....

-- Dan

Because reservation is held per processor in the "Reservation bit", and it
doesn't seem like the 405GP checks the reservation address.

I don't think any PPC checks the reservation address on stwcx.; the
reservation address is checked on snoops to clear the reservation
bit, but having the reservation bit set is a necessary and
sufficient condition for stwcx. to actually perform the store.

quoted

From the PPC manual:

"Because the hardware doesn't compare reservation address when executing the
stwcx. instruction, operating systems software MUST reset the reservation if
an exception or other types of interrupt occurs to insure atomic memory
references of lwarx and stwcx. pairs."

Indeed, but as I said this means that you have to clear the reservation on
_return_ from an interrupt in case a dangling reservation is left.
Clearing the reservation on entry is not necessary since the interrupt
handler will never execute a stwcx. without an earlier lwarx.

	Gabriel.


** Sent via the linuxppc-embedded mail list. See http://lists.linuxppc.org/

F**K...that's what I was looking for.  What manual is that in?
Everything I have handy (older UISA books), state the granularity
is implementation dependent.  I couldn't find any 4xx manual that
stated the granularity of the reservation.  I thought 6xx/7xx at
least checked cache line granularity in addition to a single
reservation bit.

The book is titled "PowerPC Microprocessor Family: The Programming
Environments".  It's greenish-blue, dated 3/21/2000.  The quote is from the
stwcx. instruction description.  In section 5-4 however, it has this note:

"When a reservation is made to a word in memory by the lwarx instruction, an
address is saved and a reservation is set.  Both of these are necessary for
the memory coherence mechanism, however, some processors do not implement
the address compare for the stwcx. instruction.  Only the reservation need
be established in order of the stwcx. to be successful.  This requires that
exception handlers clear reservations if control is passed to another
program.  Programmers should read the specifications for each individual
processor."

I searched through the 405GP user manual, and it makes no mention of if it
checks the reservation address or not, just like you said.

Eli


** Sent via the linuxppc-embedded mail list. See http://lists.linuxppc.org/

Eli Chen wrote:

The book is titled "PowerPC Microprocessor Family:....

I left that one at home today.

I searched through the 405GP user manual, and it makes no mention of if it
checks the reservation address or not, just like you said.

I grabbed my stack of Motorola manuals.  All of the 6xx/7xx/8xx manuals
have a UISA section that describes the behavior of all implementation
dependent instructions.  The lwarx/stwcx behavior is clearly defined,
along with either 16 or 32 word granularity.......I have Gabriel's
patch floating around on a couple of systems for testing.  If you
try it, let me know the results please.

Thanks.


	-- Dan

** Sent via the linuxppc-embedded mail list. See http://lists.linuxppc.org/

Eli Chen wrote:

.... Perhaps there are other places that returns
from an interrupt?

No, that's a common return path that should catch all cases.
There may be something else wrong with the Ethernet driver itself.
When I updated it to the 2.4_devel baseline, there were some weird
cache management calls that didn't make sense.  My updates were to
use the standard non-coherent cache management functions, and I
changed the logic to make sense (to me :-).  From this quick update,
I noticed it would be nice to make the transmit more efficient
and higher performance by handling multiple frames, but it should
function properly.


	-- Dan

** Sent via the linuxppc-embedded mail list. See http://lists.linuxppc.org/

I've been running Gabriel's patch for a while now, and I'm still seeing the
"de_put: entry net already free!" messages from running Brian's test
(although seemingly not as often).

I do believe Gabriel is right about clearing the reservation before the rfi,
not in transfer_to_handler.  Perhaps there are other places that returns
from an interrupt?

-eli


** Sent via the linuxppc-embedded mail list. See http://lists.linuxppc.org/

On Mon, 7 May 2001, Dan Malek wrote:

Eli Chen wrote:

quoted

quoted

From the PPC manual:

"Because the hardware doesn't compare reservation address when executing the
stwcx.

F**K...that's what I was looking for.  What manual is that in?
Everything I have handy (older UISA books), state the granularity
is implementation dependent.  I couldn't find any 4xx manual that
stated the granularity of the reservation.  I thought 6xx/7xx at
least checked cache line granularity in addition to a single
reservation bit.

Nope. The reservation address register sits on the bus side for snoops,
and is at least on 601 and 603/603e only used to clear the reservation bit
in case of snoop hit. The pem makes it clear that adress chcking is
implementation dependent.

Anyway on SMP the following scenario:

1) processor 1:  sem=1, down(sem) interrupted between lwarx and stwcx.
   reservation set, sem=1 in RAM, value to store 0
2) processor 2: down(sem), sem = 0, clears reservation on processor 1
3) processor 1: interrupt handler ends in down_trylock(sem) which fails
   but sets reservation, sem = -1
4) processor 1: down(sem) finishes succesfully and stores 0 since it
   misses the modification of step 2.
5) Now two processors access the data protected by the same semaphore,
   causing interferences, generally of the destructive kind. The value of
   the semaphore when nobody has acquired it is 2, it has become useless as
   an interlock mechanism.

does screw up with only using a single variable. I've not been able to
find such a scenario on UP, this does not mean it can't be built, however.

Fix: stwcx. at the return from interrupt, again.

Gabriel is right................

I'm myself more and more convinced myself that this is really a bug, and
a rather serious one. Now the question is whether the patch
I suggested is correct or not. It does not fix Eli's problems, so
either my patch is wrong or something else is going on.

	Gabriel.


** Sent via the linuxppc-embedded mail list. See http://lists.linuxppc.org/

Gabriel Paubert wrote:

I'm myself more and more convinced myself that this is really a bug,

I'm convinced, too, so thanks for the suggestion.  I think it is
much more likely to show up on the 4xx because of my (assumed)
implementation than on other processors that attempt to make some
use of the EA in the instructions.

.... It does not fix Eli's problems, so
either my patch is wrong or something else is going on.

We have many people testing on a variety of systems, so we will
know more shortly.

Thanks again.


	-- Dan

** Sent via the linuxppc-embedded mail list. See http://lists.linuxppc.org/

Dan Malek wrote:

There may be something else wrong with the Ethernet driver itself.
When I updated it to the 2.4_devel baseline, there were some weird
cache management calls that didn't make sense.  My updates were to
use the standard non-coherent cache management functions, and I
changed the logic to make sense (to me :-).  From this quick update,
I noticed it would be nice to make the transmit more efficient
and higher performance by handling multiple frames, but it should
function properly.

There are a large number of bugs in the 405 ethernet driver.  We (I)
were waiting until we had resolved this reference count problem, and had
a some workable solution to the starvation under packet floods problem
discussed a few weeks ago with other embedded processors before posting
a patch (i.e., have the driver stand up to reasonable stress tests).
Among the bugs are:

leaks of all the receive buffers, plus other memory on every device
close,
not checking for failed allocations in skb allocations,
poor choice of cache operations when manipulating buffers,
race conditions in data structure access between the rxde and rxeob
interrupt handlers

In either event we had proven to ourselves that this "reference count"
bug happened with other nic cards when used with the 405GP processor, so
we are reasonably certain that it is not specific to the 405 ethernet
driver.  As Eli mentioned ping flooding demonstrates the problem too so
we still believe that it is a generic atomic op problem.  However, we
can only make it happen on our 405GP walnut board(s) and not our
prototype 405GP board(s) (they both have rev D processors).

Just to refresh everyone's memory, the other reference count problems we
were seeing were "Freeing alive device" messages indicating the dev
reference count had gone to zero, and another one in the skb code when
ping flooding with large packet sizes (causing lots of fragments to be
generated).

	david

** Sent via the linuxppc-embedded mail list. See http://lists.linuxppc.org/

David Blythe wrote:

....  However, we
can only make it happen on our 405GP walnut board(s) and not our
prototype 405GP board(s) (they both have rev D processors).

Oh, great :-).....Are there any obvious differences, like clock
speed, memory type or configuration, etc.?

If someone gets a chance, would you give the FSM labs linuxppc_2_5
sources a whirl before they disappear?  I don't expect it to be
perfect, but it would sure be nice debugging something else for
a change :-).

I modified the Ethernet driver cache stuff at least.....

Thanks.


	-- Dan

** Sent via the linuxppc-embedded mail list. See http://lists.linuxppc.org/

Dan Malek wrote:

David Blythe wrote:

quoted

....  However, we
can only make it happen on our 405GP walnut board(s) and not our
prototype 405GP board(s) (they both have rev D processors).

Oh, great :-).....Are there any obvious differences, like clock
speed, memory type or configuration, etc.?

Yes, there are some differences.  Memory speed is slower at the moment
(board is still in bringup stage).  There are other subtle differences
as well.  I believe Brian also saw the bug on a non-walnut 405GP.  I was
kinda hoping it was specific to some hardware, but i don't see any
pattern.  It seems that the bug is very sensitive to timing, at least
from watching it seeminly disappear as Eli made small changes to the
atomic ops or related code, as well as Brian's comments about using the
non-inline form of the atomic ops.

If someone gets a chance, would you give the FSM labs linuxppc_2_5
sources a whirl before they disappear?  I don't expect it to be
perfect, but it would sure be nice debugging something else for
a change :-).

Should it run out of the box?  I can give it a try if you have some
confidence it should build and and run a shell on a walnut board.

	thanks
	david

** Sent via the linuxppc-embedded mail list. See http://lists.linuxppc.org/

David Blythe wrote:

Should it run out of the box?  I can give it a try if you have some
confidence it should build and and run a shell on a walnut board.

Oh, yes, it will get to a shell prompt.  I just don't know what
else you may need to run some of these tests, or what you may
discover later :-).  There have been lots of PCI updates that are
in the pipeline for the upcoming 2.4/2.4_devel that aren't part of
the linuxppc_2_5 tree.  It basically makes the PCI look like all
other PowerPC PCI systems.


	-- Dan

** Sent via the linuxppc-embedded mail list. See http://lists.linuxppc.org/

Dan Malek wrote:

Oh, yes, it will get to a shell prompt.  I just don't know what
else you may need to run some of these tests, or what you may
discover later :-).  There have been lots of PCI updates that are
in the pipeline for the upcoming 2.4/2.4_devel that aren't part of
the linuxppc_2_5 tree.  It basically makes the PCI look like all
other PowerPC PCI systems.

When is this comming out?  I heard 2.0 has been pushed back to June?

Ira Weiny


** Sent via the linuxppc-embedded mail list. See http://lists.linuxppc.org/

Ira Weiny wrote:

When is this comming out?  I heard 2.0 has been pushed back to June?

When is what coming out?  The changes for the 4xx/Walnut are in the
pipeline back to FSM Labs.  Once the 2.4/2.4_devel/linuxppc_2_5
stuff is straightened out it should pop up there.

You mean HHL 2.0 from MontaVista?  Creating a CD with supported
software, tools, and applications is a little different than
throwing a couple of 4xx PCI functions over the fence for you to
play with :-).  There are many people doing lots of software for that
release, but it doesn't change the way we work on software in the
public source trees.  There are some substantial changes among those
FSM Labs trees for the 4xx, and it takes a while to sort that out.
It doesn't do any good for me to put something there that doesn't
work, because then all I will do is answer those questions here :-).


	-- Dan

** Sent via the linuxppc-embedded mail list. See http://lists.linuxppc.org/