🌐 US-Proxy
class="logged-out env-production page-responsive" style="word-wrap: break-word;" >
Skip to content

Commit dd03cae

Browse files
alex-aizmanalex-aizman
committed
sys: bare-metal CPU usage parsing (fix)
* fix /proc/stat parsing: include specific enumerated fields - comments inline * add temporary minimum wall-clock interval guard to reject millisecond-range resampling noise - TODO inline * switch CPU load reporting to integer percentages - update all callers Signed-off-by: Alex Aizman <alex.aizman@gmail.com>
1 parent 07e5674 commit dd03cae

11 files changed

Lines changed: 54 additions & 38 deletions

File tree

‎ais/tgtbck.go‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -335,6 +335,11 @@ func (t *target) listObjects(w http.ResponseWriter, r *http.Request, bck *meta.B
335335

336336
// NOTE: blocking next-page request
337337
resp := xls.Do(lsmsg)
338+
if resp == nil {
339+
// (unlikely shutdown)
340+
w.WriteHeader(http.StatusNoContent)
341+
return false
342+
}
338343
if resp.Err != nil {
339344
t.writeErr(w, r, resp.Err, resp.Status)
340345
return false

‎cmn/load/npa.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -106,7 +106,7 @@ func CPU() Load {
106106
switch {
107107
case extreme:
108108
return Critical
109-
case load >= float64(high):
109+
case load >= high:
110110
return High
111111
default:
112112
return Low

‎docs/relnotes/4.3.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ Improvements include media-aware worker parallelism, global rebalance hardening,
1010

1111
Object HEAD v2, introduced as experimental in 4.2, is now stable.
1212

13-
This release includes over 300 commits since [4.2](https://github.com/NVIDIA/aistore/releases/tag/v4.2) and is fully backward compatible.
13+
This release includes over 300 commits since [4.2](https://github.com/NVIDIA/aistore/releases/tag/v1.4.2) and is fully backward compatible.
1414

1515
---
1616

‎memsys/gc.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,9 +33,9 @@ func (r *MMSA) freeMemToOS(mingc int64, p int, forces ...bool) {
3333
if !force {
3434
// too busy and not too "pressured"
3535
switch {
36-
case load >= float64(highLoad) && p <= PressureHigh:
36+
case load >= highLoad && p <= PressureHigh:
3737
return
38-
case load >= max(float64(highLoad>>1), 1.0) && p <= PressureModerate:
38+
case load >= max(highLoad>>1, 1) && p <= PressureModerate:
3939
return
4040
}
4141
}

‎memsys/housekeep_mm.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ func (r *MMSA) hkcb(now int64) time.Duration {
6767
highLoad = sys.HighLoadWM()
6868
)
6969
// too busy and not too "pressured"
70-
if load >= float64(highLoad) {
70+
if load >= highLoad {
7171
return r.hkIval(p, now)
7272
}
7373
r.refreshStats(now)

‎stats/common.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -572,10 +572,10 @@ func _load(sname string, flags, set, clr cos.NodeStateFlags) (cos.NodeStateFlags
572572
const tag = "CPU utilization:"
573573
var (
574574
load, isExtreme = sys.MaxLoad2()
575-
ncpu = sys.NumCPU()
575+
ncpu = int64(sys.NumCPU())
576576
)
577577
// 1. normal
578-
if load < float64(ncpu>>1) { // 50%
578+
if load < ncpu>>1 { // 50%
579579
if flags.IsAnySet(cos.LowCPU | cos.OOCPU) {
580580
clr |= cos.OOCPU | cos.LowCPU
581581
nlog.Infoln(sname, tag, "back to normal")
@@ -593,7 +593,7 @@ func _load(sname string, flags, set, clr cos.NodeStateFlags) (cos.NodeStateFlags
593593
}
594594
// 3. high
595595
highcpu := ncpu * sys.HighLoad / 100
596-
if load > float64(highcpu) {
596+
if load > highcpu {
597597
clr |= cos.OOCPU
598598
if !flags.IsSet(cos.LowCPU) {
599599
set |= cos.LowCPU

‎sys/cpu.go‎

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,12 @@ const (
2323
// - stats_time
2424
maxSampleAge = int64(13 * time.Second)
2525

26+
// TODO:
27+
// - temp workaround against millisecond-range resampling noise
28+
// - replace with cached/smoothed CPU-load state
29+
// (similar to disk utilization in `ios` package)
30+
minWallIval = int64(2 * time.Second)
31+
2632
// >10% wall-clock throttled => extreme starvation
2733
throttleExtremeThresh = 10
2834

@@ -86,14 +92,14 @@ func GoEnvMaxprocs() {
8692
// HighLoadWM: "high-load watermark" as a percentage.
8793
// For 8 CPUs: max(100 - 100/8, 1) = 88 - between HighLoad(82) and ExtremeLoad(92).
8894
// see also: (ExtremeLoad, HighLoad) defaults
89-
func HighLoadWM() int {
90-
ncpu := NumCPU()
95+
func HighLoadWM() int64 {
96+
ncpu := int64(NumCPU())
9197
return max(100-100/ncpu, 1)
9298
}
9399

94100
// MaxLoad returns CPU utilization percentage (0-100).
95101
// See also: README.md in this package.
96-
func MaxLoad() (load float64) {
102+
func MaxLoad() (load int64) {
97103
util, _, err := ctracker.get()
98104
if err != nil {
99105
return maxLoadFallback()
@@ -107,7 +113,7 @@ func MaxLoad() (load float64) {
107113
// is being throttled, that's CPU starvation regardless of utilization percentage.
108114
//
109115
// See also: README.md in this package.
110-
func MaxLoad2() (load float64, isExtreme bool) {
116+
func MaxLoad2() (load int64, isExtreme bool) {
111117
util, throttled, err := ctracker.get()
112118
if err != nil {
113119
load = maxLoadFallback()
@@ -126,7 +132,7 @@ func MaxLoad2() (load float64, isExtreme bool) {
126132

127133
// fallback when cpuTracker returns an error:
128134
// convert load average to percentage: (load / NumCPU)%
129-
func maxLoadFallback() float64 {
135+
func maxLoadFallback() int64 {
130136
avg, err := LoadAverage()
131137
if err != nil {
132138
nlog.ErrorDepth(1, err)
@@ -135,5 +141,5 @@ func maxLoadFallback() float64 {
135141
load := max(avg.One, avg.Five)
136142
ncpu := float64(NumCPU())
137143
pct := load * 100 / ncpu
138-
return min(pct, 100)
144+
return min(int64(pct+0.4), 100)
139145
}

‎sys/cpu_linux.go‎

Lines changed: 21 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -171,7 +171,7 @@ func LoadAverage() (avg LoadAvg, _ error) {
171171
// - percentage of wall-clock time the container was CPU-throttled since the last call
172172
// or:
173173
// - 0 if the prev. sample is old (or first call)
174-
func (t *cpuTracker) get() (util, throttled float64, _ error) {
174+
func (t *cpuTracker) get() (util, throttled int64, _ error) {
175175
cur, err := readCPUUsage()
176176
if err != nil {
177177
return 0, 0, err
@@ -190,21 +190,21 @@ func (t *cpuTracker) get() (util, throttled float64, _ error) {
190190
}
191191

192192
wallDt := cur.ts - prev.ts
193-
if wallDt <= 0 || wallDt > maxSampleAge {
193+
if wallDt < minWallIval || wallDt > maxSampleAge {
194194
t.mu.Unlock()
195195
return 0, 0, nil
196196
}
197197

198198
if containerized && cur.throttledUsec > 0 {
199199
dtUsec := max(cur.throttledUsec-prev.throttledUsec, 0)
200-
throttled = min(float64(dtUsec)*100_000/float64(wallDt), 100)
200+
throttled = min((dtUsec*100_000+wallDt/2)/wallDt, int64(100))
201201
}
202202

203203
t.mu.Unlock()
204204

205205
cpuDt := max(cur.usage-prev.usage, 0)
206-
ncpu := float64(NumCPU())
207-
util = float64(cpuDt) * 100 / float64(wallDt) / ncpu
206+
denom := wallDt * int64(NumCPU())
207+
util = (cpuDt*100 + denom/2) / denom
208208
return min(util, 100), throttled, nil
209209
}
210210

@@ -266,9 +266,11 @@ func readCgroupCPUUsage() (usageNs, throttledUsec int64, _ error) {
266266
return ns, 0, nil
267267
}
268268

269-
// aggregate "cpu" line from /proc/stat
270-
// - sum user+nice+system+irq+softirq+steal
271-
// - exclude idle and iowait
269+
// return total cumulative "active" (non-idle) CPU time in nanoseconds:
270+
// - parse the aggregate "cpu" line from /proc/stat
271+
// - require at least 9 fields
272+
// - sum up (user, nice, system, irq, softirq, and steal) jiffies
273+
// - explicitly exclude idle and iowait; ignor guest/guest_nice (already included)
272274
func readProcStatCPUUsage() (int64, error) {
273275
data, err := os.ReadFile(hostProcStat)
274276
if err != nil {
@@ -281,20 +283,20 @@ func readProcStatCPUUsage() (int64, error) {
281283
}
282284

283285
fields := strings.Fields(line)
284-
if len(fields) < 5 {
285-
return 0, fmt.Errorf("unexpected %s format: num-fields=%d", hostProcStat, len(fields))
286+
if len(fields) < 9 {
287+
return 0, fmt.Errorf("unexpected %s format: num-fields=%d (old kernel?)", hostProcStat, len(fields))
286288
}
287289

288-
var totalJiffies int64
289-
for i, f := range fields[1:] {
290-
if i == 3 || i == 4 { // idle, iowait
291-
continue
292-
}
293-
v, e := strconv.ParseInt(f, 10, 64)
294-
if e != nil {
295-
continue
290+
var (
291+
totalJiffies int64
292+
ii = [...]int{1, 2, 3, 6, 7, 8} // 1:user, 2:nice, 3:system, 6:irq, 7:softirq, 8:steal
293+
)
294+
for _, i := range ii {
295+
v, err := strconv.ParseInt(fields[i], 10, 64)
296+
if err != nil {
297+
return 0, fmt.Errorf("%s: parse field[%d]=%q: %w", hostProcStat, i, fields[i], err)
296298
}
297299
totalJiffies += v
298300
}
299-
return int64(float64(totalJiffies) * nsPerJiffy), nil
301+
return totalJiffies * nsPerJiffy, nil
300302
}

‎xact/xs/create_nbi.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -304,6 +304,10 @@ func (r *XactNBI) fini(smap *meta.Smap, ntotal int64) error {
304304
return errLoad // IO err
305305
}
306306

307+
if r.IsAborted() {
308+
return nil
309+
}
310+
307311
now := time.Now()
308312
if r.ufest.Count() != 0 {
309313
err := lom.CompleteUfest(r.ufest, true /*locked*/)

‎xact/xs/nwp.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -127,7 +127,7 @@ func TuneNumWorkers(xname string, numWorkers, numMpaths int) (int, error) {
127127
// 4. CPU load averages
128128
cpuLoad := load.CPU()
129129
if cpuLoad >= load.High {
130-
if lv, wm := sys.MaxLoad(), sys.HighLoadWM(); lv >= float64(wm) {
130+
if lv, wm := sys.MaxLoad(), sys.HighLoadWM(); lv >= wm {
131131
nlog.Warningln(xname, "high load [", lv, wm, "]")
132132
}
133133
if ngrLoad == load.High {

0 commit comments

Comments
 (0)