🌐 US-Proxy
class="logged-out env-production page-responsive" style="word-wrap: break-word;" >
Skip to content

Commit 140c6e4

Browse files
committed
resilver: stop joggers; revise concurrent access to shared state
* res.Abort() now calls jgroup.Stop() to immediately stop all joggers * remove atomic pointer; use mutex to access (xres, jgroup) pair * add early IsAborted() checks in visitObj/visitECSlice - exit immediately * simplify preempt() - call xaction Abort() first, then check state * revise res.wait(); abort xres if needed * add race check in Run() - verify xres unchanged between initRenew and jgroup.Run ---- * with minor refactoring * part ten, prev. commit: 97f9bf6 Signed-off-by: Alex Aizman <alex.aizman@gmail.com>
1 parent b8e9990 commit 140c6e4

8 files changed

Lines changed: 98 additions & 58 deletions

File tree

‎ais/fspathrgrp.go‎

Lines changed: 10 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -177,28 +177,24 @@ func (g *fsprungroup) doDD(action string, flags uint64, mpath string, dontResilv
177177

178178
func (g *fsprungroup) preempt(action string, mi *fs.Mountpath) bool /*prev*/ {
179179
const (
180-
preemptSleep = cos.PollSleepShort
181-
)
182-
var (
183-
res = g.t.res
184-
xres = res.GetXact()
180+
sleep = cos.PollSleepShort
185181
)
182+
res := g.t.res
183+
// try to abort an active resilver, if any
184+
if res.Abort(fmt.Errorf("%v: %q, %s", cmn.ErrXactRenewAbort, action, mi)) {
185+
return true
186+
}
187+
xres := res.GetXact()
186188
if xres == nil { // finished ok or never ran
187189
return false
188190
}
189-
if xres.IsAborted() {
190-
return true
191-
}
192191
if xres.IsDone() {
193-
time.Sleep(preemptSleep) // just finished; Res atomic state
192+
time.Sleep(sleep) // just finished; Res atomic state
194193
return false
195194
}
196195

197-
err := fmt.Errorf("%s: %q %s: starting new resilver", g.t, action, mi)
198-
if res.Abort(err) {
199-
return true
200-
}
201-
time.Sleep(preemptSleep)
196+
// (unlikely)
197+
time.Sleep(sleep)
202198
debug.Assert(xres.IsAborted() || xres.IsDone(), xres.String())
203199
return true
204200
}

‎ec/bencodex.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -174,7 +174,7 @@ func (r *XactBckEncode) Run(gowg *sync.WaitGroup) {
174174
}
175175
}
176176

177-
jg := mpather.NewJoggerGroup(opts, r.config, nil)
177+
jg := mpather.NewJgroup(opts, r.config, nil)
178178
jg.Run()
179179

180180
select {

‎fs/mpather/jogger.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,7 @@ type (
7777
}
7878
)
7979

80-
func NewJoggerGroup(opts *JgroupOpts, config *cmn.Config, smi *fs.Mountpath) *Jgroup {
80+
func NewJgroup(opts *JgroupOpts, config *cmn.Config, smi *fs.Mountpath) *Jgroup {
8181
var (
8282
joggers map[string]*jogger
8383
avail = fs.GetAvail()

‎fs/mpather/jogger_test.go‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ func TestJoggerGroup(t *testing.T) {
4444
return nil
4545
},
4646
}
47-
jg := mpather.NewJoggerGroup(opts, cmn.GCO.Get(), nil)
47+
jg := mpather.NewJgroup(opts, cmn.GCO.Get(), nil)
4848
jg.Run()
4949
<-jg.ListenFinished()
5050

@@ -83,7 +83,7 @@ func TestJoggerGroupLoad(t *testing.T) {
8383
},
8484
DoLoad: mpather.Load,
8585
}
86-
jg := mpather.NewJoggerGroup(opts, cmn.GCO.Get(), nil)
86+
jg := mpather.NewJgroup(opts, cmn.GCO.Get(), nil)
8787

8888
jg.Run()
8989
<-jg.ListenFinished()
@@ -119,7 +119,7 @@ func TestJoggerGroupError(t *testing.T) {
119119
return errors.New("oops")
120120
},
121121
}
122-
jg := mpather.NewJoggerGroup(opts, cmn.GCO.Get(), nil)
122+
jg := mpather.NewJgroup(opts, cmn.GCO.Get(), nil)
123123
jg.Run()
124124
<-jg.ListenFinished()
125125

@@ -172,7 +172,7 @@ func TestJoggerGroupOneErrorStopsAll(t *testing.T) {
172172
return nil
173173
},
174174
}
175-
jg := mpather.NewJoggerGroup(opts, cmn.GCO.Get(), nil)
175+
jg := mpather.NewJgroup(opts, cmn.GCO.Get(), nil)
176176
jg.Run()
177177
<-jg.ListenFinished()
178178

@@ -227,7 +227,7 @@ func TestJoggerGroupMultiContentTypes(t *testing.T) {
227227
return nil
228228
},
229229
}
230-
jg := mpather.NewJoggerGroup(opts, cmn.GCO.Get(), nil)
230+
jg := mpather.NewJgroup(opts, cmn.GCO.Get(), nil)
231231
jg.Run()
232232
<-jg.ListenFinished()
233233

‎res/resilver.go‎

Lines changed: 75 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@ import (
99
"os"
1010
"path/filepath"
1111
"sync"
12-
ratomic "sync/atomic"
1312
"time"
1413

1514
"github.com/NVIDIA/aistore/api/apc"
@@ -31,15 +30,15 @@ import (
3130

3231
const (
3332
ivalInactive = 4 * time.Second
34-
retrySleep = cos.PollSleepMedium
3533
)
3634

3735
type (
3836
Res struct {
39-
xres ratomic.Pointer[xs.Resilver]
40-
end atomic.Int64
41-
nbusy atomic.Int64
42-
mu sync.Mutex
37+
xres *xs.Resilver
38+
jgroup *mpather.Jgroup
39+
end atomic.Int64
40+
nbusy atomic.Int64
41+
mu sync.Mutex
4342
}
4443
Args struct {
4544
UUID string
@@ -76,13 +75,33 @@ func (res *Res) IsActive(multiplier int64) bool {
7675
return mono.Since(end) < time.Duration(multiplier)*ivalInactive
7776
}
7877

79-
func (res *Res) GetXact() *xs.Resilver { return res.xres.Load() }
78+
func (res *Res) GetXact() *xs.Resilver {
79+
res.mu.Lock()
80+
xres := res.xres
81+
res.mu.Unlock()
82+
return xres
83+
}
8084

8185
func (res *Res) Abort(err error) (aborted bool) {
82-
if r := res.xres.Load(); r != nil {
83-
aborted = r.Abort(err)
86+
res.mu.Lock()
87+
xres := res.xres
88+
jgroup := res.jgroup
89+
if xres != nil {
90+
res.jgroup = nil
91+
}
92+
res.mu.Unlock()
93+
if xres == nil {
94+
return false
95+
}
96+
aborted = xres.Abort(err)
97+
if !aborted {
98+
return false
99+
}
100+
101+
if jgroup != nil {
102+
_ = jgroup.Stop()
84103
}
85-
return
104+
return true
86105
}
87106

88107
func (res *Res) Run(args *Args, tstats cos.StatsUpdater) {
@@ -120,24 +139,31 @@ func (res *Res) Run(args *Args, tstats cos.StatsUpdater) {
120139
debug.Assert(args.PostDD == nil || (args.Action == apc.ActMountpathDetach || args.Action == apc.ActMountpathDisable))
121140

122141
if args.SingleRmiJogger {
123-
jgroup = mpather.NewJoggerGroup(opts, args.Custom.Config, args.Rmi)
142+
jgroup = mpather.NewJgroup(opts, args.Custom.Config, args.Rmi)
124143
nlog.Infof("%s, action %q, jogger->(%q)", xres.Name(), args.Action, args.Rmi)
125144
} else {
126-
jgroup = mpather.NewJoggerGroup(opts, args.Custom.Config, nil)
145+
jgroup = mpather.NewJgroup(opts, args.Custom.Config, nil)
127146
if args.Rmi != nil {
128147
nlog.Infof("%s, action %q, rmi %s, num %d", xres.Name(), args.Action, args.Rmi, jgroup.NumJ())
129148
} else {
130149
nlog.Infof("%s, num %d", xres.Name(), jgroup.NumJ())
131150
}
132151
}
133152

134-
// run and block waiting
153+
res.mu.Lock() // 2nd --------------------------------------
154+
if xres != res.xres {
155+
// (unlikely)
156+
res.mu.Unlock()
157+
_ = jgroup.Stop()
158+
return
159+
}
135160
res.end.Store(0)
161+
res.jgroup = jgroup
162+
res.mu.Unlock() // --------------------------------------
163+
164+
// run and block waiting
136165
jgroup.Run()
137-
err = wait(jgroup, xres, tstats)
138-
if err != nil {
139-
xres.AddErr(err)
140-
}
166+
res.wait(jgroup, xres, tstats)
141167

142168
// callback to, finally, detach-disable
143169
if args.PostDD != nil {
@@ -147,8 +173,11 @@ func (res *Res) Run(args *Args, tstats cos.StatsUpdater) {
147173
xres.Finish()
148174

149175
res.mu.Lock()
150-
res.end.Store(mono.NanoTime())
151-
res.xres.Store(nil)
176+
if xres == res.xres {
177+
res.end.Store(mono.NanoTime())
178+
res.xres = nil
179+
res.jgroup = nil
180+
}
152181
res.mu.Unlock()
153182

154183
if n := res.nbusy.Load(); n > 0 {
@@ -158,20 +187,17 @@ func (res *Res) Run(args *Args, tstats cos.StatsUpdater) {
158187

159188
func (res *Res) initRenew(args *Args) *xs.Resilver {
160189
rns := xreg.RenewResilver(args.UUID, &args.Custom)
161-
if rns.Err == nil && rns.IsRunning() {
162-
// retry just once
163-
time.Sleep(retrySleep)
164-
rns = xreg.RenewResilver(args.UUID, &args.Custom)
165-
}
166190
if rns.Err != nil {
191+
debug.Assertf(!cmn.IsErrXactUsePrev(rns.Err), "not expecting %v(%T)", rns.Err, rns.Err)
192+
nlog.Errorln("failed to start", args.Action, "renewal err:", rns.Err)
167193
return nil
168194
}
195+
xctn := rns.Entry.Get()
196+
xres := xctn.(*xs.Resilver)
169197
if rns.IsRunning() {
170-
nlog.Warningf("%s: failed to start/preempt for xid=%q: %s is still running", args.Action, args.UUID, rns.Entry.Get().Name())
198+
nlog.Warningf("%s: start/preempt race [%q, %s]", args.Action, args.UUID, xres.Name())
171199
return nil
172200
}
173-
xctn := rns.Entry.Get()
174-
xres := xctn.(*xs.Resilver)
175201

176202
debug.Func(func() {
177203
if r := res.GetXact(); r != nil {
@@ -180,15 +206,15 @@ func (res *Res) initRenew(args *Args) *xs.Resilver {
180206
})
181207
debug.Assertf(xres.ID() == args.UUID, "res-id mismatch: %q vs %q", xres.Name(), args.UUID)
182208

183-
res.mu.Lock() // --------------------------------------
209+
res.mu.Lock() // 1st --------------------------------------
184210

185211
fatalErr, warnErr := fs.PersistMarker(fname.ResilverMarker, true /*quiet*/)
186212
if fatalErr != nil {
187213
xres.Abort(fatalErr)
188214
res.mu.Unlock()
189215
return nil
190216
}
191-
res.xres.Store(xres)
217+
res.xres = xres
192218

193219
res.mu.Unlock() // ------------------------------------
194220

@@ -203,16 +229,22 @@ func (res *Res) initRenew(args *Args) *xs.Resilver {
203229
}
204230

205231
// Wait for an abort or for resilvering joggers to finish.
206-
func wait(jgroup *mpather.Jgroup, xres *xs.Resilver, tstats cos.StatsUpdater) (err error) {
232+
func (res *Res) wait(jgroup *mpather.Jgroup, xres *xs.Resilver, tstats cos.StatsUpdater) {
207233
for {
208234
select {
209235
case errCause := <-xres.ChanAbort():
210-
if err = jgroup.Stop(); err != nil {
211-
xres.AddErr(err, 0)
212-
} else {
213-
nlog.Infoln(core.T.String(), xres.Name(), "aborted, cause:", errCause)
236+
err := jgroup.Stop()
237+
res.mu.Lock()
238+
if res.jgroup == jgroup {
239+
res.jgroup = nil
240+
}
241+
res.mu.Unlock()
242+
if !xres.IsAborted() {
243+
e := cos.Ternary(errCause == nil, err, errCause)
244+
xres.Abort(e)
245+
nlog.Warningln(xres.Name(), "stopped: [", errCause, err, "]")
214246
}
215-
return cmn.NewErrAborted(xres.Name(), "", errCause)
247+
return
216248
case <-jgroup.ListenFinished():
217249
if fs.RemoveMarker(fname.ResilverMarker, tstats, false /*stopping*/) {
218250
nlog.Infoln(core.T.String()+":", xres.Name(), "removed marker ok")
@@ -257,6 +289,11 @@ func (j *jogger) visitObj(lom *core.LOM, buf []byte) (errHrw error) {
257289
size int64
258290
copied bool
259291
)
292+
293+
if j.xres.IsAborted() {
294+
return nil
295+
}
296+
260297
if !lom.TryLock(true) { // NOTE: skipping busy
261298
j.p.nbusy.Inc()
262299
if cmn.Rom.V(4, cos.ModReb) {
@@ -419,6 +456,9 @@ func (j *jogger) visitECSlice(ct *core.CT, buf []byte) (err error) {
419456
if !ct.Bck().Props.EC.Enabled {
420457
return filepath.SkipDir
421458
}
459+
if j.xres.IsAborted() {
460+
return nil
461+
}
422462
j._mvSlice(ct, buf)
423463
return nil
424464
}

‎xact/bckjog.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ type BckJog struct {
1818

1919
func (r *BckJog) Init(id, kind string, bck *meta.Bck, opts *mpather.JgroupOpts, config *cmn.Config) {
2020
r.InitBase(id, kind, bck)
21-
r.joggers = mpather.NewJoggerGroup(opts, config, nil)
21+
r.joggers = mpather.NewJgroup(opts, config, nil)
2222
r.Config = config
2323
}
2424

‎xact/xreg/xreg.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -773,6 +773,10 @@ func (r *RenewBase) Str(kind string) string {
773773
// RenewRes //
774774
//////////////
775775

776+
// note:
777+
// for a newly constructed instance x-registry always returns RenewRes{Entry: entry}
778+
// with empty rns.UUID
779+
// in other words, IsRunning() can only be true when we are using xprev
776780
func (rns *RenewRes) IsRunning() bool {
777781
if rns.UUID == "" {
778782
return false

‎xact/xs/prune.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ func (rp *prune) init(config *cmn.Config) {
4848
// DoLoad: noLoad
4949
}
5050
rmopts.Bck.Copy(rp.bckTo.Bucket())
51-
rp.joggers = mpather.NewJoggerGroup(rmopts, config, nil)
51+
rp.joggers = mpather.NewJgroup(rmopts, config, nil)
5252
rp.filter = prob.NewDefaultFilter()
5353
rp.same = rp.bckTo.Equal(rp.bckFrom, true, true)
5454
}

0 commit comments

Comments
 (0)