99 "os"
1010 "path/filepath"
1111 "sync"
12- ratomic "sync/atomic"
1312 "time"
1413
1514 "github.com/NVIDIA/aistore/api/apc"
@@ -31,15 +30,15 @@ import (
3130
3231const (
3332 ivalInactive = 4 * time .Second
34- retrySleep = cos .PollSleepMedium
3533)
3634
3735type (
3836 Res struct {
39- xres ratomic.Pointer [xs.Resilver ]
40- end atomic.Int64
41- nbusy atomic.Int64
42- mu sync.Mutex
37+ xres * xs.Resilver
38+ jgroup * mpather.Jgroup
39+ end atomic.Int64
40+ nbusy atomic.Int64
41+ mu sync.Mutex
4342 }
4443 Args struct {
4544 UUID string
@@ -76,13 +75,33 @@ func (res *Res) IsActive(multiplier int64) bool {
7675 return mono .Since (end ) < time .Duration (multiplier )* ivalInactive
7776}
7877
79- func (res * Res ) GetXact () * xs.Resilver { return res .xres .Load () }
78+ func (res * Res ) GetXact () * xs.Resilver {
79+ res .mu .Lock ()
80+ xres := res .xres
81+ res .mu .Unlock ()
82+ return xres
83+ }
8084
8185func (res * Res ) Abort (err error ) (aborted bool ) {
82- if r := res .xres .Load (); r != nil {
83- aborted = r .Abort (err )
86+ res .mu .Lock ()
87+ xres := res .xres
88+ jgroup := res .jgroup
89+ if xres != nil {
90+ res .jgroup = nil
91+ }
92+ res .mu .Unlock ()
93+ if xres == nil {
94+ return false
95+ }
96+ aborted = xres .Abort (err )
97+ if ! aborted {
98+ return false
99+ }
100+
101+ if jgroup != nil {
102+ _ = jgroup .Stop ()
84103 }
85- return
104+ return true
86105}
87106
88107func (res * Res ) Run (args * Args , tstats cos.StatsUpdater ) {
@@ -120,24 +139,31 @@ func (res *Res) Run(args *Args, tstats cos.StatsUpdater) {
120139 debug .Assert (args .PostDD == nil || (args .Action == apc .ActMountpathDetach || args .Action == apc .ActMountpathDisable ))
121140
122141 if args .SingleRmiJogger {
123- jgroup = mpather .NewJoggerGroup (opts , args .Custom .Config , args .Rmi )
142+ jgroup = mpather .NewJgroup (opts , args .Custom .Config , args .Rmi )
124143 nlog .Infof ("%s, action %q, jogger->(%q)" , xres .Name (), args .Action , args .Rmi )
125144 } else {
126- jgroup = mpather .NewJoggerGroup (opts , args .Custom .Config , nil )
145+ jgroup = mpather .NewJgroup (opts , args .Custom .Config , nil )
127146 if args .Rmi != nil {
128147 nlog .Infof ("%s, action %q, rmi %s, num %d" , xres .Name (), args .Action , args .Rmi , jgroup .NumJ ())
129148 } else {
130149 nlog .Infof ("%s, num %d" , xres .Name (), jgroup .NumJ ())
131150 }
132151 }
133152
134- // run and block waiting
153+ res .mu .Lock () // 2nd --------------------------------------
154+ if xres != res .xres {
155+ // (unlikely)
156+ res .mu .Unlock ()
157+ _ = jgroup .Stop ()
158+ return
159+ }
135160 res .end .Store (0 )
161+ res .jgroup = jgroup
162+ res .mu .Unlock () // --------------------------------------
163+
164+ // run and block waiting
136165 jgroup .Run ()
137- err = wait (jgroup , xres , tstats )
138- if err != nil {
139- xres .AddErr (err )
140- }
166+ res .wait (jgroup , xres , tstats )
141167
142168 // callback to, finally, detach-disable
143169 if args .PostDD != nil {
@@ -147,8 +173,11 @@ func (res *Res) Run(args *Args, tstats cos.StatsUpdater) {
147173 xres .Finish ()
148174
149175 res .mu .Lock ()
150- res .end .Store (mono .NanoTime ())
151- res .xres .Store (nil )
176+ if xres == res .xres {
177+ res .end .Store (mono .NanoTime ())
178+ res .xres = nil
179+ res .jgroup = nil
180+ }
152181 res .mu .Unlock ()
153182
154183 if n := res .nbusy .Load (); n > 0 {
@@ -158,20 +187,17 @@ func (res *Res) Run(args *Args, tstats cos.StatsUpdater) {
158187
159188func (res * Res ) initRenew (args * Args ) * xs.Resilver {
160189 rns := xreg .RenewResilver (args .UUID , & args .Custom )
161- if rns .Err == nil && rns .IsRunning () {
162- // retry just once
163- time .Sleep (retrySleep )
164- rns = xreg .RenewResilver (args .UUID , & args .Custom )
165- }
166190 if rns .Err != nil {
191+ debug .Assertf (! cmn .IsErrXactUsePrev (rns .Err ), "not expecting %v(%T)" , rns .Err , rns .Err )
192+ nlog .Errorln ("failed to start" , args .Action , "renewal err:" , rns .Err )
167193 return nil
168194 }
195+ xctn := rns .Entry .Get ()
196+ xres := xctn .(* xs.Resilver )
169197 if rns .IsRunning () {
170- nlog .Warningf ("%s: failed to start/preempt for xid=%q: %s is still running " , args .Action , args .UUID , rns . Entry . Get () .Name ())
198+ nlog .Warningf ("%s: start/preempt race [%q, %s] " , args .Action , args .UUID , xres .Name ())
171199 return nil
172200 }
173- xctn := rns .Entry .Get ()
174- xres := xctn .(* xs.Resilver )
175201
176202 debug .Func (func () {
177203 if r := res .GetXact (); r != nil {
@@ -180,15 +206,15 @@ func (res *Res) initRenew(args *Args) *xs.Resilver {
180206 })
181207 debug .Assertf (xres .ID () == args .UUID , "res-id mismatch: %q vs %q" , xres .Name (), args .UUID )
182208
183- res .mu .Lock () // --------------------------------------
209+ res .mu .Lock () // 1st --------------------------------------
184210
185211 fatalErr , warnErr := fs .PersistMarker (fname .ResilverMarker , true /*quiet*/ )
186212 if fatalErr != nil {
187213 xres .Abort (fatalErr )
188214 res .mu .Unlock ()
189215 return nil
190216 }
191- res .xres . Store ( xres )
217+ res .xres = xres
192218
193219 res .mu .Unlock () // ------------------------------------
194220
@@ -203,16 +229,22 @@ func (res *Res) initRenew(args *Args) *xs.Resilver {
203229}
204230
205231// Wait for an abort or for resilvering joggers to finish.
206- func wait (jgroup * mpather.Jgroup , xres * xs.Resilver , tstats cos.StatsUpdater ) ( err error ) {
232+ func ( res * Res ) wait (jgroup * mpather.Jgroup , xres * xs.Resilver , tstats cos.StatsUpdater ) {
207233 for {
208234 select {
209235 case errCause := <- xres .ChanAbort ():
210- if err = jgroup .Stop (); err != nil {
211- xres .AddErr (err , 0 )
212- } else {
213- nlog .Infoln (core .T .String (), xres .Name (), "aborted, cause:" , errCause )
236+ err := jgroup .Stop ()
237+ res .mu .Lock ()
238+ if res .jgroup == jgroup {
239+ res .jgroup = nil
240+ }
241+ res .mu .Unlock ()
242+ if ! xres .IsAborted () {
243+ e := cos .Ternary (errCause == nil , err , errCause )
244+ xres .Abort (e )
245+ nlog .Warningln (xres .Name (), "stopped: [" , errCause , err , "]" )
214246 }
215- return cmn . NewErrAborted ( xres . Name (), "" , errCause )
247+ return
216248 case <- jgroup .ListenFinished ():
217249 if fs .RemoveMarker (fname .ResilverMarker , tstats , false /*stopping*/ ) {
218250 nlog .Infoln (core .T .String ()+ ":" , xres .Name (), "removed marker ok" )
@@ -257,6 +289,11 @@ func (j *jogger) visitObj(lom *core.LOM, buf []byte) (errHrw error) {
257289 size int64
258290 copied bool
259291 )
292+
293+ if j .xres .IsAborted () {
294+ return nil
295+ }
296+
260297 if ! lom .TryLock (true ) { // NOTE: skipping busy
261298 j .p .nbusy .Inc ()
262299 if cmn .Rom .V (4 , cos .ModReb ) {
@@ -419,6 +456,9 @@ func (j *jogger) visitECSlice(ct *core.CT, buf []byte) (err error) {
419456 if ! ct .Bck ().Props .EC .Enabled {
420457 return filepath .SkipDir
421458 }
459+ if j .xres .IsAborted () {
460+ return nil
461+ }
422462 j ._mvSlice (ct , buf )
423463 return nil
424464}
0 commit comments