🌐 US-Proxy
class="logged-out env-production page-responsive" style="word-wrap: break-word;" >
Skip to content

Commit 28d55f6

Browse files
alex-aizmanalex-aizman
committed
global rebalance: cleanup-mode observability
* cli: add dedicated template to show rebalance run in cleanup mode - show removed objects/bytes instead of migration TX/RX counters - group output by rebalance generation and print per-generation totals - include deduplicated ctl messages above each generation * target/xreb: - count cleanup removals via generic xaction stats - carry cleanup force mode through log headers and ctl messages - simplify cleanup jogger state by keeping force/stats in clnArgs - with minor refactoring ---- * part two, prev. commit: d8d4f99 "introduce safe cleanup" Signed-off-by: Alex Aizman <alex.aizman@gmail.com>
1 parent d7b55dd commit 28d55f6

17 files changed

Lines changed: 266 additions & 104 deletions

‎cmd/cli/cli/cluster_hdlr.go‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
// Package cli provides easy-to-use commands to manage, monitor, and utilize AIS clusters.
2-
// This file handles commands that interact with the cluster.
32
/*
43
* Copyright (c) 2018-2026, NVIDIA CORPORATION. All rights reserved.
54
*/

‎cmd/cli/cli/config_hdlr.go‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
// Package cli provides easy-to-use commands to manage, monitor, and utilize AIS clusters.
2-
// This file handles commands that interact with the cluster.
32
/*
4-
* Copyright (c) 2021-2025, NVIDIA CORPORATION. All rights reserved.
3+
* Copyright (c) 2021-2026, NVIDIA CORPORATION. All rights reserved.
54
*/
65
package cli
76

‎cmd/cli/cli/dsort.go‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
// Package cli provides easy-to-use commands to manage, monitor, and utilize AIS clusters.
2-
// This file handles commands that interact with objects in the cluster
32
/*
43
* Copyright (c) 2018-2026, NVIDIA CORPORATION. All rights reserved.
54
*/

‎cmd/cli/cli/log_hdlr.go‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
// Package cli provides easy-to-use commands to manage, monitor, and utilize AIS clusters.
2-
// This file handles commands that interact with the cluster.
32
/*
4-
* Copyright (c) 2021-2025, NVIDIA CORPORATION. All rights reserved.
3+
* Copyright (c) 2021-2026, NVIDIA CORPORATION. All rights reserved.
54
*/
65
package cli
76

‎cmd/cli/cli/nbi_hdlr.go‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
// Package cli provides easy-to-use commands to manage, monitor, and utilize AIS clusters.
2-
// This file handles commands that interact with the cluster.
32
/*
43
* Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
54
*/

‎cmd/cli/cli/reb_hdlr.go‎

Lines changed: 190 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
// Package cli provides easy-to-use commands to manage, monitor, and utilize AIS clusters.
2-
// This file handles commands that interact with the cluster.
32
/*
43
* Copyright (c) 2021-2026, NVIDIA CORPORATION. All rights reserved.
54
*/
@@ -24,7 +23,14 @@ import (
2423
)
2524

2625
const (
26+
// migration mode: TX/RX split
2727
showRebHdr = "REB ID\t NODE\t OBJECTS RECV\t SIZE RECV\t OBJECTS SENT\t SIZE SENT\t START\t END\t STATE"
28+
29+
// cleanup mode: removed = Objs/Bytes (see reb/cleanup.go)
30+
showRebCleanupHdr = "REB ID\t NODE\t REMOVED OBJECTS\t REMOVED SIZE\t START\t END\t STATE"
31+
32+
rebColCount = 9 // showRebHdr column count (\t-separated)
33+
rebCleanupColCount = 7 // showRebCleanupHdr column count
2834
)
2935

3036
var showRebUsage = "Show global rebalance status and per-target stats.\n" +
@@ -33,7 +39,8 @@ var showRebUsage = "Show global rebalance status and per-target stats.\n" +
3339
indent1 + "\t- show rebalance g21\t- show a specific rebalance by ID;\n" +
3440
indent1 + "\t- show rebalance --refresh 10\t- continuous monitoring with 10s refresh (until Ctrl-C);\n" +
3541
indent1 + "\t- show rebalance --refresh 10 --count 5\t- same as above, limited to 5 iterations;\n" +
36-
indent1 + "\t- show rebalance --all\t- include all rebalances: running, finished, and aborted.\n" +
42+
indent1 + "\t- show rebalance --all\t- include all rebalances: running, finished, and aborted\n" +
43+
indent1 + "\t\t (note: 'rebalance --cleanup' generations render with a cleanup-mode template).\n" +
3744
indent1 + "for alternative views, see also:\n" +
3845
indent1 + "\t- 'ais show job'\t- all running jobs (possibly including rebalance) across the cluster;\n" +
3946
indent1 + "\t- 'ais performance intra-data'\t- live peer-to-peer RX/TX counters and sizes (rebalance generates intra-cluster traffic)."
@@ -55,6 +62,22 @@ var (
5562
}
5663
)
5764

65+
// TODO -- FIXME:
66+
// add Flags field to core.Snap (requires msgpack regen)
67+
// and replace with `snap.Flags & xact.FlagRemoveMisplaced`
68+
func _isRebCleanup(snap *core.Snap) bool {
69+
return snap.Kind == apc.ActRebalance && strings.Contains(snap.CtlMsg, ":cleanup")
70+
}
71+
72+
func isRebCleanupGroup(group []*targetRebSnap) bool {
73+
for _, sts := range group {
74+
if _isRebCleanup(sts.snap) {
75+
return true
76+
}
77+
}
78+
return false
79+
}
80+
5881
// (implemented over Go text/tabwriter directly w/ no templates)
5982
func showRebalanceHandler(c *cli.Context) error {
6083
var (
@@ -113,12 +136,7 @@ func showRebalanceHandler(c *cli.Context) error {
113136
return V(err)
114137
}
115138

116-
var (
117-
numMigratedObjs int64 // acknowledged migrations
118-
sizeMigratedBytes int64
119-
prevID string
120-
allSnaps = make([]*targetRebSnap, 0, 32)
121-
)
139+
allSnaps := make([]*targetRebSnap, 0, 32)
122140
for daemonID, daemonStats := range rebSnaps {
123141
if xargs.DaemonID != "" && xargs.DaemonID != daemonID {
124142
continue
@@ -131,6 +149,7 @@ func showRebalanceHandler(c *cli.Context) error {
131149
}
132150
}
133151

152+
var lastID string
134153
if len(allSnaps) > 0 {
135154
sort.Slice(allSnaps, func(i, j int) bool {
136155
if allSnaps[i].snap.ID != allSnaps[j].snap.ID {
@@ -142,40 +161,12 @@ func showRebalanceHandler(c *cli.Context) error {
142161
if printed {
143162
fmt.Fprintln(tw)
144163
}
145-
if !hideHeader {
146-
// NOTE: when/if changing header update `colCount` & `displayRebStats` as well
147-
fmt.Fprintln(tw, showRebHdr)
148-
}
149-
for _, sts := range allSnaps {
150-
if flagIsSet(c, allJobsFlag) {
151-
if prevID != "" && sts.snap.ID != prevID {
152-
fmt.Fprintln(tw, strings.Repeat("\t ", 9 /*colCount*/))
153-
numMigratedObjs, sizeMigratedBytes = 0, 0
154-
}
155-
displayRebStats(tw, sts, units, datedTime)
156-
} else {
157-
if prevID != "" && sts.snap.ID != prevID {
158-
break
159-
}
160-
latestAborted = latestAborted || sts.snap.AbortedX
161-
latestFinished = latestFinished || !sts.snap.EndTime.IsZero()
162-
displayRebStats(tw, sts, units, datedTime)
163-
}
164-
// migrated total, as in: objects successfully sent out
165-
numMigratedObjs += sts.snap.Stats.OutObjs
166-
sizeMigratedBytes += sts.snap.Stats.OutBytes
167-
prevID = sts.snap.ID
168-
}
169-
tw.Flush()
170-
printed = true
171-
}
172164

173-
id := fcyan(prevID)
174-
if numMigratedObjs > 0 {
175-
fmt.Fprintf(c.App.Writer, "%s: %d objects migrated (total size %s)\n",
176-
id, numMigratedObjs, teb.FmtSize(sizeMigratedBytes, units, 1))
165+
latestAborted, latestFinished, printed, lastID = renderSnaps(c, tw, allSnaps, units, datedTime, hideHeader)
177166
}
167+
178168
if !flagIsSet(c, allJobsFlag) {
169+
id := fcyan(lastID)
179170
if latestFinished && latestAborted {
180171
fmt.Fprintf(c.App.Writer, "\nRebalance %s aborted.\n", id)
181172
break
@@ -200,15 +191,146 @@ func showRebalanceHandler(c *cli.Context) error {
200191
return nil
201192
}
202193

203-
func displayRebStats(tw *tabwriter.Writer, st *targetRebSnap, units string, datedTime bool) {
204-
var startTime, endTime string
205-
if datedTime {
206-
startTime = teb.FmtDateTime(st.snap.StartTime)
207-
endTime = teb.FmtDateTime(st.snap.EndTime)
208-
} else {
209-
startTime = teb.FmtTime(st.snap.StartTime)
210-
endTime = teb.FmtTime(st.snap.EndTime)
194+
// walk sorted snaps, grouping by rebalance ID. Each ID-group renders
195+
// with its own header (migration vs cleanup mode), body, and per-group totals
196+
// footer; return aborted/finished flags and ID for the selected latest generation
197+
// when not showing all; with --all these return values are ignored by caller.
198+
func renderSnaps(c *cli.Context, tw *tabwriter.Writer, allSnaps []*targetRebSnap,
199+
units string, datedTime, hideHeader bool) (latestAborted, latestFinished, printed bool, lastID string) {
200+
showAll := flagIsSet(c, allJobsFlag)
201+
202+
// allSnaps is sorted newest-ID first above; groupByRebID preserves that order.
203+
groups := groupByRebID(allSnaps)
204+
if !showAll && len(groups) > 0 {
205+
groups = groups[len(groups)-1:] // only the most-recent generation; preserve old first-ID behavior
206+
}
207+
208+
for i, group := range groups {
209+
cleanup := isRebCleanupGroup(group)
210+
renderGroup(c, tw, group, units, datedTime, hideHeader, cleanup)
211+
212+
// generation transition separator (between groups only)
213+
if showAll && i < len(groups)-1 {
214+
fmt.Fprintln(tw, strings.Repeat("\t ", colCountFor(cleanup)))
215+
}
216+
217+
latestAborted, latestFinished = false, false
218+
for _, sts := range group {
219+
latestAborted = latestAborted || sts.snap.AbortedX
220+
latestFinished = latestFinished || !sts.snap.EndTime.IsZero()
221+
}
222+
lastID = group[0].snap.ID
223+
}
224+
225+
tw.Flush()
226+
return latestAborted, latestFinished, len(groups) > 0, lastID
227+
}
228+
229+
// groupByRebID partitions a slice of snaps (sorted by ID then tid) into per-ID
230+
// groups, preserving order. Returns nil for empty input.
231+
func groupByRebID(allSnaps []*targetRebSnap) [][]*targetRebSnap {
232+
if len(allSnaps) == 0 {
233+
return nil
234+
}
235+
groups := make([][]*targetRebSnap, 0, 4)
236+
start := 0
237+
for i := 1; i <= len(allSnaps); i++ {
238+
if i == len(allSnaps) || allSnaps[i].snap.ID != allSnaps[start].snap.ID {
239+
groups = append(groups, allSnaps[start:i])
240+
start = i
241+
}
242+
}
243+
return groups
244+
}
245+
246+
// print header + body + per-group totals footer for one rebalance generation
247+
func renderGroup(c *cli.Context, tw *tabwriter.Writer, snaps []*targetRebSnap, units string, datedTime, hideHeader, cleanup bool) {
248+
// caption: per-target ctlMsg lines (deduped) above the table
249+
tw.Flush()
250+
printGroupCtlMsgs(c, snaps)
251+
252+
if !hideHeader {
253+
if cleanup {
254+
fmt.Fprintln(tw, showRebCleanupHdr)
255+
} else {
256+
fmt.Fprintln(tw, showRebHdr)
257+
}
211258
}
259+
260+
var totalObjs, totalBytes int64
261+
for _, sts := range snaps {
262+
if cleanup {
263+
displayCleanupStats(tw, sts, units, datedTime)
264+
// cleanup: locally-processed counters (ObjsAdd => Stats.Objs/Bytes)
265+
totalObjs += sts.snap.Stats.Objs
266+
totalBytes += sts.snap.Stats.Bytes
267+
} else {
268+
displayMigrationStats(tw, sts, units, datedTime)
269+
totalObjs += sts.snap.Stats.OutObjs
270+
totalBytes += sts.snap.Stats.OutBytes
271+
}
272+
}
273+
274+
tw.Flush()
275+
if totalObjs == 0 || len(snaps) == 0 {
276+
return
277+
}
278+
id := fcyan(snaps[0].snap.ID)
279+
verb := "migrated"
280+
if cleanup {
281+
verb = "removed"
282+
}
283+
fmt.Fprintf(c.App.Writer, "%s: %d objects %s (total size %s)\n",
284+
id, totalObjs, verb, teb.FmtSize(totalBytes, units, 1))
285+
}
286+
287+
// printGroupCtlMsgs prints deduped, deterministically-ordered ctlMsg lines
288+
// above the per-generation table. Mirrors the caption block in `ais show job`
289+
// (see job_show_hdlr.go: seenCtl/ctlmsgs collection, jobCptn rendering).
290+
//
291+
// For regular rebalance (migration) mode, ctlMsg carries traversal phase timings:
292+
// <fin> trav:6s post-trav:2s fin:28s fin-streams:12s
293+
// For cleanup mode, it carries visits/loads/removed counters:
294+
// t[PAYt8083]:cleanup done visits=257259 loads=86170 removed=86170
295+
func printGroupCtlMsgs(c *cli.Context, snaps []*targetRebSnap) {
296+
if len(snaps) == 0 {
297+
return
298+
}
299+
var (
300+
seen = make(map[string]struct{}, len(snaps))
301+
msgs = make([]string, 0, len(snaps))
302+
)
303+
for _, sts := range snaps {
304+
m := sts.snap.CtlMsg
305+
if m == "" {
306+
continue
307+
}
308+
if _, ok := seen[m]; ok {
309+
continue
310+
}
311+
seen[m] = struct{}{}
312+
msgs = append(msgs, m)
313+
}
314+
if len(msgs) == 0 {
315+
return
316+
}
317+
sort.Strings(msgs) // deterministic regardless of map iteration order
318+
id := fcyan(snaps[0].snap.ID)
319+
fmt.Fprintf(c.App.Writer, "%s ctl:\n", id)
320+
for _, m := range msgs {
321+
fmt.Fprintf(c.App.Writer, " %s\n", m)
322+
}
323+
}
324+
325+
func colCountFor(cleanup bool) int {
326+
if cleanup {
327+
return rebCleanupColCount
328+
}
329+
return rebColCount
330+
}
331+
332+
func displayMigrationStats(tw *tabwriter.Writer, st *targetRebSnap, units string, datedTime bool) {
333+
startTime, endTime := fmtRebTimes(st.snap, datedTime)
212334
fmt.Fprintf(tw,
213335
"%s\t %s\t %d\t %s\t %d\t %s\t %s\t %s\t %s\n",
214336
st.snap.ID, st.tid,
@@ -217,3 +339,24 @@ func displayRebStats(tw *tabwriter.Writer, st *targetRebSnap, units string, date
217339
startTime, endTime, teb.FmtXactRunFinAbrt(st.snap),
218340
)
219341
}
342+
343+
func displayCleanupStats(tw *tabwriter.Writer, st *targetRebSnap, units string, datedTime bool) {
344+
startTime, endTime := fmtRebTimes(st.snap, datedTime)
345+
fmt.Fprintf(tw,
346+
"%s\t %s\t %d\t %s\t %s\t %s\t %s\n",
347+
st.snap.ID, st.tid,
348+
st.snap.Stats.Objs, teb.FmtSize(st.snap.Stats.Bytes, units, 2),
349+
startTime, endTime, teb.FmtXactRunFinAbrt(st.snap),
350+
)
351+
}
352+
353+
func fmtRebTimes(snap *core.Snap, datedTime bool) (startTime, endTime string) {
354+
if datedTime {
355+
startTime = teb.FmtDateTime(snap.StartTime)
356+
endTime = teb.FmtDateTime(snap.EndTime)
357+
} else {
358+
startTime = teb.FmtTime(snap.StartTime)
359+
endTime = teb.FmtTime(snap.EndTime)
360+
}
361+
return
362+
}

‎cmd/cli/cli/remcluster_hdlr.go‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
// Package cli provides easy-to-use commands to manage, monitor, and utilize AIS clusters.
2-
// This file handles commands that interact with the cluster.
32
/*
4-
* Copyright (c) 2021-2025, NVIDIA CORPORATION. All rights reserved.
3+
* Copyright (c) 2021-2026, NVIDIA CORPORATION. All rights reserved.
54
*/
65
package cli
76

‎cmd/cli/cli/scrub.go‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
// Package cli provides easy-to-use commands to manage, monitor, and utilize AIS clusters.
2-
// This file handles commands that interact with the cluster.
32
/*
4-
* Copyright (c) 2021-2025, NVIDIA CORPORATION. All rights reserved.
3+
* Copyright (c) 2021-2026, NVIDIA CORPORATION. All rights reserved.
54
*/
65
package cli
76

‎cmd/cli/cli/storage_hdlr.go‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
// Package cli provides easy-to-use commands to manage, monitor, and utilize AIS clusters.
2-
// This file handles commands that interact with the cluster.
32
/*
4-
* Copyright (c) 2021-2025, NVIDIA CORPORATION. All rights reserved.
3+
* Copyright (c) 2021-2026, NVIDIA CORPORATION. All rights reserved.
54
*/
65
package cli
76

‎cmd/cli/cli/x509.go‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
// Package cli provides easy-to-use commands to manage, monitor, and utilize AIS clusters.
2-
// This file handles commands that interact with the cluster.
32
/*
4-
* Copyright (c) 2024-2025, NVIDIA CORPORATION. All rights reserved.
3+
* Copyright (c) 2024-2026, NVIDIA CORPORATION. All rights reserved.
54
*/
65
package cli
76

0 commit comments

Comments
 (0)