Re: UVM/genfs review

[Juergen Hannken-Illjes <hannken%eis.cs.tu-bs.de@localhost>]

On Wed, Jan 14, 2009 at 02:43:53PM -0600, Greg Oster wrote:
> Joerg Sonnenberger writes:
> > On Thu, Jan 08, 2009 at 09:12:55PM +0100, Juergen Hannken-Illjes wrote:
> > > Just for the records, still strong objection from me.  As I told you 
> > > before
> > > this would bring back a deadlock (see Rev. 1.12 of genfs_io.c).
> > 
> > >From testing, this isn't my biggest problem with this combination. On an
> > active filesystem, I got twice a panic when trying to configure fss. Not
> > sure what it was the first time, the second it was "snapacct: bad
> > block". Otherwise it seems to be good enough to run a backup during a
> > bulk build.
> 
> So where are we at on this?  Does the deadlock that Juergen refers to 
> occur only with fss+wapbl or with fss+other as well?  If only the 
> former, and we get the patch in, call fss+wapbl explicitly not 
> supported, and hope to get it fixed in -current (and pulled up for 
> 5.1 or something)?

Just tried to look again at this deadlock.  All I have is part of the ddb
log (attached).  It happened on my build-once-a-day machine once in ~ 4 weeks,
so it is not easy to repeat.  I never saw it again after the commit Joerg
wants to revert.

If I get it right it is one thread paging in for ffs_write and running

    ffs_write (wapbl locked) -> VOP_GETPAGES -> genfs_getpages -> uvn_findpage

here the page is already locked by a thread syncing and running

    ffs_full_fsync -> VOP_PUTPAGES -> genfs_do_putpages (page locked)
    -> ffs_gop_write

where ffs_gop_write just locked wapbl and called gop_write.  With the diff
presented this would now become

    -> gop_write -> genfs_do_io

where we need the wapbl lock already held by the first thread.

So this deadlock is independent of snapshots, fstrans_* etc are not
involved at all.

Hope I got it right,
-- 
Juergen Hannken-Illjes - hannken%eis.cs.tu-bs.de@localhost - TU Braunschweig 
(Germany)

ddb log:

 26187         1 3         4           d0e6d640   sparc64--netbsd- uvn_fp2
              38 3       204           ce55f560            ioflush tstile

OBJECT 0xd2119c3c: locked=0, pgops=0xc0844280, npages=84, refs=2
VNODE flags 4030<MPSAFE,LOCKSWORK,ONWORKLST>
mp 0xd0481000 numoutput 0 size 0x51b60 writesize 0x51b60
data 0xe327f058 writecount 0 holdcnt 2
tag VT_UFS(1) type VREG(1) mount 0xd0481000 typedata 0x0
v_lock 0xd2119cdc v_vnlock 0xd2119cdc

tag VT_UFS, ino 99488, on dev 4, 12 flags 0x1, effnlink 1, nlink 1
        mode 0100644, owner 0, group 0, size 334688

PAGES <pg,offset>:
  ...
  <0xc456d020,0x4e000> <0xc4f118e8,0x4f000> <0xc336f4b0,0x50000>
  <0xc37df8f8,0x51000> <0xc2d83f40,0x52000> <0xc41bab08,0x53000>
  <0xd01b5acc,0xffffffffffffffff> <0xd01b5a84,0xffffffffffffffff>

PAGE 0xc336f4b0:
  flags=f<BUSY,WANTED,TABLED,CLEAN>, pqflags=200<PRIVATE2>, wire_count=0, 
pa=0x8a804000
  uobject=0xd2119c3c, uanon=0x0, offset=0x50000 loan_count=0
  [page ownership tracking disabled]

PAGE 0xc37df8f8:
  flags=d<BUSY,TABLED,CLEAN>, pqflags=200<PRIVATE2>, wire_count=0, pa=0x9a485000
  uobject=0xd2119c3c, uanon=0x0, offset=0x51000 loan_count=0
  [page ownership tracking disabled]

PAGE 0xc2d83f40:
  flags=d<BUSY,TABLED,CLEAN>, pqflags=200<PRIVATE2>, wire_count=0, pa=0x75746000
  uobject=0xd2119c3c, uanon=0x0, offset=0x52000 loan_count=0
  [page ownership tracking disabled]

PAGE 0xc41bab08:
  flags=d<BUSY,TABLED,CLEAN>, pqflags=200<PRIVATE2>, wire_count=0, pa=0xbd537000
  uobject=0xd2119c3c, uanon=0x0, offset=0x53000 loan_count=0
  [page ownership tracking disabled]

trace: pid 26187 lid 1 at 0xe059047c
sleepq_block(0,0,c0a49b6c,c0abd6d8,4,c0b6ff30,74,c336f4b0,d2119c3c,0) at 
netbsd:sleepq_block+0x112
mtsleep(c336f4b0,204,c0a49b6c,0,d2119c3c,df637a10,e059051c,c0474bc9,ceab20d0,d0e6d7f4)
 at netbsd:mtsleep+0x12d
uvn_findpage(e059066c,0,e059055c,c04f1cb7,0,0,1,0,54000,e059067c) at 
netbsd:uvn_findpage+0x92
uvn_findpages(d2119c3c,50000,0,e05906cc,e059066c,0,40,d0e69900,1b71660,0) at 
netbsd:uvn_findpages+0x73
genfs_getpages(e0590710,0,51000,0,0,10,af000,c08481e0,d2119c3c,50000) at 
netbsd:genfs_getpages+0x763
VOP_GETPAGES(e30593a8,50000,0,e059089c,e05908a4,0,1,0,2,b5ba9067) at 
netbsd:VOP_GETPAGES+0x90
uvn_get(e30593a8,50000,0,e059089c,e05908a4,0,1,0,2,1) at netbsd:uvn_get+0x117
uvm_fault_internal(ce55cd00,bbae7000,1,0,c0b71660,0,c5477a48,0,0,d2f11780) at 
netbsd:uvm_fault_internal+0x106c
trap() at netbsd:trap+0x688
--- trap (number 6) ---
copyin(ce55cd00,bbae7000,cfc67000,1b60,ce55cd00,0,cfc67000,50000,0,cfc67000) at 
netbsd:copyin+0x33
uiomove(cfc67000,1b60,e0590c7c,e0590a5c,1,2,4000,c042e680,54000,0) at 
netbsd:uiomove+0xd9
ubc_uiomove(dcd202ec,e0590c7c,1b60,1,2,d0e69900,0,c042004e,0,0) at 
netbsd:ubc_uiomove+0xeb
ffs_write(e0590c04,0,e0590bcc,c52c4c00,ce55cd00,d048a000,c0847aa0,e0590c08,dcd2038c,1)
 at netbsd:ffs_write+0x9da
layer_bypass(e0590c04,1,c0847fa0,d36073a8,2,20002,e0590c1c,c04f25e8,c0847aa0,dcd202ec)
 at netbsd:layer_bypass+0xc8
VOP_WRITE(d36073a8,e0590c7c,10,d0e69900,6,d0e631c0,e30593a8,16,51b60,bba97000) 
at netbsd:VOP_WRITE+0x6c
vn_write(d0e63880,d0e63880,e0590c7c,d0e69900,1,d0163000,1,ce55cd00,bfbed000,1000)
 at netbsd:vn_write+0xb1
dofilewrite(5,d0e63880,bba97000,51b60,d0e63880,1,e0590d28,0,0,d0e6d640) at 
netbsd:dofilewrite+0x75
sys_write(d0e6d640,e0590d00,e0590d28,bfbeda84,bfbed000,ce55cd00,2,5,bba97000,51b60)
 at netbsd:sys_write+0x6f
syscall(e0590d48,b3,ab,1f,1f,bba97000,0,bfbfdba8,51b60,0) at netbsd:syscall+0xab

trace: pid 0 lid 38 at 0xd01b58bc
sleepq_block(0,0,c0a50be8,c0abd61c,6,d0e6d640,c0b705a0,dcfdc90c,7c,40) at 
netbsd:sleepq_block+0x112
turnstile_block(0,1,c5477a4c,c0abd61c,c41bab08,d01b597c,c5316500,54000,d2119c3c,13)
 at netbsd:turnstile_block+0x23c
rw_vector_enter(c5477a4c,1,d01b59ec,54000,11,d2119c3c,d01b5a0c,c03e469e,c5477a00,c0843780)
 at netbsd:rw_vector_enter+0x20e
wapbl_begin(c5477a00,c0843780,2bf,7,3,4,d01b5b14,c41bab08,c41bab08,5) at 
netbsd:wapbl_begin+0x6c
ffs_gop_write(d2119c3c,d01b5b14,4,11,d01b5b18,13,1,d01b5b14,fffff000,7fffffff) 
at netbsd:ffs_gop_write+0x7e
genfs_do_putpages(d2119c3c,0,0,0,0,11,0,c0474bc9,d9869ac8,d0481000) at 
netbsd:genfs_do_putpages+0x919
genfs_putpages(d01b5bbc,6,d01b5bcc,c0474bc9,ceab20d0,ce55f714,c0848220,d2119c3c,0,0)
 at netbsd:genfs_putpages+0x3d
VOP_PUTPAGES(d2119c3c,0,0,0,0,11,1,c04e7287,d0481000,0) at 
netbsd:VOP_PUTPAGES+0x7f
ffs_full_fsync(d2119c3c,8,1,c04f9ca0,d01b5c5c,e026e5e0,d01b5c6c,c04e7287,d2119cdc,1)
 at netbsd:ffs_full_fsync+0x159
ffs_fsync(d01b5cd8,12,d01b5cfc,c0502aff,d2119c3c,c0847c60,d2119c3c,ce546f00,8,0)
 at netbsd:ffs_fsync+0x6f
VOP_FSYNC(d2119c3c,ce546f00,8,0,0,0,0,ce55f560,c0502d30,ce55f560) at 
netbsd:VOP_FSYNC+0x7e
sched_sync(ce55f560,0,c01002cd,0,c01002cd,0,0,0,0,0) at netbsd:sched_sync+0x14a