🌐 US-Proxy
class="logged-out env-production page-responsive" style="word-wrap: break-word;" >
Skip to content

Commit 805f9ab

Browse files
alex-aizmanalex-aizman
committed
resilver: rewrite copy-recovering-restoring path; amend and refactor
* remove lom.ToMpathCopies and retry-based copying loop * deterministically select a single ("primary") copy to rebuild main - cleanup copy metadata - and missing copies - support both (copy => HRW) and vise versa * reduce contention: retry TryLock - but only in two distinct cases * def: ObjsAdd == num objects actually repaired (HRW/main restored) * minor cleanups and comments ----- * part thirteen, prev. commit: 5eaf263 Signed-off-by: Alex Aizman <alex.aizman@gmail.com>
1 parent 4253d7b commit 805f9ab

4 files changed

Lines changed: 165 additions & 97 deletions

File tree

‎core/lcopy.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -210,6 +210,7 @@ func (lom *LOM) _restore(fqn string, buf []byte) (dst *LOM, err error) {
210210

211211
// increment the object's num copies by (well) copying the former
212212
// (compare with lom.Copy2FQN below)
213+
// TODO: add T.FSHC() calls _after_ having disambiguated source vs destination (to blame)
213214
func (lom *LOM) Copy(mi *fs.Mountpath, buf []byte) error {
214215
debug.Assert(lom.bid() != 0, lom.String())
215216
if err := lom._checkBucket(); err != nil {
@@ -245,6 +246,7 @@ func (lom *LOM) Copy(mi *fs.Mountpath, buf []byte) error {
245246
return err
246247
}
247248
if err := cos.Rename(workFQN, copyFQN); err != nil {
249+
T.FSHC(err, mi, copyFQN)
248250
if errRemove := cos.RemoveFile(workFQN); errRemove != nil && !cos.IsNotExist(errRemove) {
249251
nlog.Errorln("nested err:", errRemove)
250252
}

‎core/lresil.go‎

Lines changed: 28 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ import (
1818
// - checks hrw location first, and
1919
// - checks copies (if any) against the current configuration and available mountpaths;
2020
// - does not check `fstat` in either case
21+
// (usage: resilvering)
2122
func (lom *LOM) ToMpath(avail fs.MPI) (*fs.Mountpath, bool /*fix HRW*/) {
2223
debug.Assert(lom.IsLocked() == apc.LockWrite, lom.Cname(), "expecting w-locked")
2324

@@ -51,40 +52,43 @@ func (lom *LOM) ToMpath(avail fs.MPI) (*fs.Mountpath, bool /*fix HRW*/) {
5152
return hrwMi, !ok
5253
}
5354

54-
func (lom *LOM) ToMpathCopies(avail fs.MPI) (*fs.Mountpath, bool /*fix HRW*/) {
55-
// given (verb => abort => new run) mountpaths should be stable during resilver
56-
// TODO: remove
57-
hrwMi, _, err := avail.Hrw(cos.UnsafeB(*lom.md.uname))
58-
if err != nil {
59-
nlog.Errorln(err)
60-
return nil, false
61-
}
62-
if lom.mi.Path != hrwMi.Path {
63-
return hrwMi, true
55+
// deterministically pick a copy to restore any missing ones
56+
// must be called under w-lock
57+
// (usage: resilvering)
58+
func (lom *LOM) IsPrimaryCopy(avail fs.MPI, hmi *fs.Mountpath, sentinel string) (isPrimary, mainExists bool) {
59+
debug.Assert(lom.IsCopy(), lom.Cname(), "must be a copy")
60+
selected := sentinel
61+
for fqn, mi := range lom.md.copies {
62+
if _, ok := avail[mi.Path]; !ok { // not skipping fs.FlagWaitingDD mountpaths
63+
continue
64+
}
65+
if err := cos.Stat(fqn); err != nil {
66+
continue
67+
}
68+
mainExists = mainExists || mi.Path == hmi.Path
69+
selected = min(fqn, selected)
6470
}
71+
return lom.FQN == selected, mainExists
72+
}
6573

66-
// check copies, if any
74+
// remove stale copy metadata entries (copies on unavailable or
75+
// disabled mountpaths) and return the expected number of copies per config
76+
// and the actual number of valid copies remaining;
77+
// must be called under w-lock
78+
// (usage: resilvering)
79+
func (lom *LOM) CleanupCopies(avail fs.MPI) (exp, got int) {
6780
mirror := lom.MirrorConf()
6881
if !mirror.Enabled || mirror.Copies < 2 {
69-
return nil, false
82+
return
7083
}
71-
72-
// count copies vs. configuration
73-
// take into account mountpath flags but stop short of `fstat`-ing
74-
expCopies, gotCopies := int(mirror.Copies), 0
84+
exp, got = int(mirror.Copies), 0
7585
for fqn, mpi := range lom.md.copies {
7686
mpathInfo, ok := avail[mpi.Path]
7787
if !ok || mpathInfo.IsAnySet(fs.FlagWaitingDD) {
7888
lom.delCopyMd(fqn)
7989
} else {
80-
gotCopies++
90+
got++
8191
}
8292
}
83-
if expCopies <= gotCopies {
84-
return nil, false
85-
}
86-
87-
// NOTE: nil when not enough mountpaths
88-
mi := lom.LeastUtilNoCopy()
89-
return mi, false
93+
return
9094
}

‎res/alt_test.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,9 @@ func TestResilver_DisableEnable(t *testing.T) {
8888

8989
// resilver and check
9090
runResilver(t, config, rmi, apc.ActMountpathDisable)
91+
92+
time.Sleep(time.Second) // TODO -- FIXME: remove (a rare, intermittent failure)
93+
9194
assertLocations(t, bck, objs)
9295

9396
// disable (as in: commit)

0 commit comments

Comments
 (0)