🌐 US-Proxy
class="logged-out env-production page-responsive" style="word-wrap: break-word;" >
Skip to content

Commit ee7514c

Browse files
alex-aizmanalex-aizman
committed
feat: native bucket inventory (page sizing)
* merging proxy/gateway: - do not default NBI PageSize=0 to MaxPageSizeAIS * target/list-NBI: - treat inventory listing PageSize as best-effort and approximate - compute per-target page budget as an approximate share of requested size with a minimum floor and slight overfetch ---- * tests: - use api.ListObjectsPage in prefix permutation test - add empty invName coverage - add zero-page-size coverage - remove exact page-size assertion ---- * part thirty, prev. commit: ffefd96 Signed-off-by: Alex Aizman <alex.aizman@gmail.com>
1 parent 7241c93 commit ee7514c

8 files changed

Lines changed: 105 additions & 54 deletions

File tree

‎ais/plstcx.go‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -252,8 +252,9 @@ func (p *proxy) lsObjsA(bck *meta.Bck, lsmsg *apc.LsoMsg, hdr http.Header) (allE
252252
args *bcastArgs
253253
results sliceResults
254254
smap = p.owner.smap.get()
255+
isNBI = lsmsg.IsFlagSet(apc.LsNBI)
255256
)
256-
if lsmsg.PageSize == 0 {
257+
if lsmsg.PageSize == 0 && !isNBI {
257258
lsmsg.PageSize = apc.MaxPageSizeAIS
258259
}
259260

@@ -284,13 +285,13 @@ func (p *proxy) lsObjsA(bck *meta.Bck, lsmsg *apc.LsoMsg, hdr http.Header) (allE
284285
return nil, err
285286
}
286287
lst := res.v.(*cmn.LsoRes)
287-
if len(lst.Entries) > 0 || (lsmsg.IsFlagSet(apc.LsNBI) && lst.ContinuationToken != "") {
288+
if len(lst.Entries) > 0 || (isNBI && lst.ContinuationToken != "") {
288289
lists = append(lists, lst)
289290
}
290291
}
291292
freeBcastRes(results)
292293

293-
if lsmsg.IsFlagSet(apc.LsNBI) {
294+
if isNBI {
294295
page := finLsoNBI(lists, lsmsg)
295296
return page, nil
296297
}

‎ais/test/nbi_test.go‎

Lines changed: 52 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -324,7 +324,6 @@ func TestListInventoryPrefix(t *testing.T) {
324324
}
325325
}
326326

327-
// TODO -- FIXME: add test case(s) with empty invName
328327
func TestListInventoryPrefixPermute(t *testing.T) {
329328
tools.CheckSkip(t, &tools.SkipTestArgs{RemoteBck: true, Bck: cliBck})
330329

@@ -333,7 +332,7 @@ func TestListInventoryPrefixPermute(t *testing.T) {
333332
numA, numM, numZ int
334333
pageSize int64 // CreateNBIMsg.PageSize
335334
namesPerChunk int64
336-
listPageSize int64 // listing PageSize
335+
listPageSize int64 // listing PageSize (best-effort for NBI)
337336
invName string
338337
}
339338
tests := []test{
@@ -343,6 +342,11 @@ func TestListInventoryPrefixPermute(t *testing.T) {
343342
pageSize: 3, namesPerChunk: 6, listPageSize: 4,
344343
invName: "inv-sc-sp",
345344
},
345+
{
346+
name: "small-chunks-pages-empty-invName",
347+
numA: 40, numM: 30, numZ: 50,
348+
pageSize: 3, namesPerChunk: 6, listPageSize: 4,
349+
},
346350
{
347351
name: "small-chunks-large-pages",
348352
numA: 40, numM: 30, numZ: 50,
@@ -361,6 +365,12 @@ func TestListInventoryPrefixPermute(t *testing.T) {
361365
pageSize: 10, namesPerChunk: 8, listPageSize: 5,
362366
invName: "inv-maxent",
363367
},
368+
{
369+
name: "zero-page-size",
370+
numA: 40, numM: 30, numZ: 50,
371+
pageSize: 3, namesPerChunk: 6, listPageSize: 0,
372+
invName: "inv-zero-ps",
373+
},
364374
}
365375

366376
bck := cliBck
@@ -372,12 +382,10 @@ func TestListInventoryPrefixPermute(t *testing.T) {
372382
parent = "p-" + cos.GenTie() + "/"
373383
bp = tools.BaseAPIParams()
374384
)
375-
// One init/cleanup only
385+
376386
m0 := &ioContext{t: t, bck: bck, prefix: parent}
377387
m0.init(true /*cleanup*/)
378388

379-
// Three disjoint subtrees under parent: a/, m/, z/
380-
// Ensures: a/ < m/ < z/ in lexicographic order.
381389
ma := &ioContext{t: t, bck: bck, prefix: parent + "a/"}
382390
mm := &ioContext{t: t, bck: bck, prefix: parent + "m/"}
383391
mz := &ioContext{t: t, bck: bck, prefix: parent + "z/"}
@@ -386,12 +394,10 @@ func TestListInventoryPrefixPermute(t *testing.T) {
386394
mm.num = tc.numM
387395
mz.num = tc.numZ
388396

389-
// Per your note: pass-through writes, no remotePuts/evicting.
390397
ma.puts()
391398
mm.puts()
392399
mz.puts()
393400

394-
// Create inventory covering the entire parent prefix
395401
createMsg := &apc.CreateNBIMsg{
396402
Name: tc.invName,
397403
LsoMsg: apc.LsoMsg{
@@ -409,16 +415,16 @@ func TestListInventoryPrefixPermute(t *testing.T) {
409415
_, err = api.WaitForXactionIC(bp, &wargs)
410416
tassert.CheckFatal(t, err)
411417

412-
// Helper: list NBI with token-loop guard, strict order, and prefix check.
413-
list := func(prefix string, tc *test, invName string) (entries []*cmn.LsoEnt) {
418+
var args api.ListArgs
419+
if createMsg.Name != "" {
420+
args.Header = http.Header{apc.HdrInvName: []string{createMsg.Name}}
421+
}
422+
423+
listNBI := func(prefix string) (entries []*cmn.LsoEnt) {
414424
var (
415425
token string
416426
seenTok = make(cos.StrSet, 16)
417-
args api.ListArgs
418427
)
419-
if invName != "" {
420-
args.Header = http.Header{apc.HdrInvName: []string{invName}}
421-
}
422428
for {
423429
lsmsg := &apc.LsoMsg{
424430
Prefix: prefix,
@@ -427,14 +433,11 @@ func TestListInventoryPrefixPermute(t *testing.T) {
427433
Flags: apc.LsNBI,
428434
ContinuationToken: token,
429435
}
430-
lst, err := api.ListObjects(bp, bck, lsmsg, args)
436+
lst, err := api.ListObjectsPage(bp, bck, lsmsg, args)
431437
tassert.CheckFatal(t, err)
432438

433-
// page size sanity
434-
if lst.ContinuationToken != "" {
435-
tassert.Fatalf(t, len(lst.Entries) <= int(tc.listPageSize),
436-
"page too large: got %d, max %d", len(lst.Entries), tc.listPageSize)
437-
}
439+
tassert.Fatalf(t, len(lst.Entries) > 0 || lst.ContinuationToken == "",
440+
"empty page with continuation token %q for prefix %q", lst.ContinuationToken, prefix)
438441

439442
entries = append(entries, lst.Entries...)
440443
if lst.ContinuationToken == "" {
@@ -445,7 +448,7 @@ func TestListInventoryPrefixPermute(t *testing.T) {
445448
seenTok.Set(lst.ContinuationToken)
446449
token = lst.ContinuationToken
447450
}
448-
// strict order + prefix match
451+
449452
for i := range entries {
450453
tassert.Fatalf(t, strings.HasPrefix(entries[i].Name, prefix),
451454
"entry %q doesn't match prefix %q", entries[i].Name, prefix)
@@ -458,29 +461,31 @@ func TestListInventoryPrefixPermute(t *testing.T) {
458461
return entries
459462
}
460463

461-
// 1) list sub-prefix in the middle => exercises: skip a/ chunks + Search in-chunk + stop before z/
462-
gotM := list(mm.prefix, &tc, createMsg.Name)
463-
464-
// validate set equality with PUT names
465-
gotSet := make(cos.StrSet, len(gotM))
466-
for _, e := range gotM {
467-
gotSet.Set(e.Name)
468-
}
469-
var missed = make([]string, 0, 4)
470-
for _, name := range mm.objNames {
471-
if _, ok := gotSet[name]; !ok {
472-
missed = append(missed, name)
464+
missingFromEntries := func(expected []string, got []*cmn.LsoEnt) []string {
465+
gotSet := make(cos.StrSet, len(got))
466+
for _, e := range got {
467+
gotSet.Set(e.Name)
473468
}
469+
missed := make([]string, 0, 4)
470+
for _, name := range expected {
471+
if _, ok := gotSet[name]; !ok {
472+
missed = append(missed, name)
473+
}
474+
}
475+
return missed
474476
}
475-
tassert.Fatalf(t, len(missed) == 0, "missing %v for prefix %q - %d out of %d expected", missed, mm.prefix, len(missed), len(mm.objNames))
476-
tassert.Fatalf(t, len(gotM) == mm.num, "prefix %q: expected %d, got %d", mm.prefix, mm.num, len(gotM))
477477

478-
// 2) list parent => must include all
479-
gotParent := list(parent, &tc, createMsg.Name)
478+
gotM := listNBI(mm.prefix)
479+
missed := missingFromEntries(mm.objNames, gotM)
480+
tassert.Fatalf(t, len(missed) == 0,
481+
"missing %v for prefix %q - %d out of %d expected",
482+
missed, mm.prefix, len(missed), len(mm.objNames))
483+
tassert.Fatalf(t, len(gotM) == mm.num,
484+
"prefix %q: expected %d, got %d", mm.prefix, mm.num, len(gotM))
485+
486+
gotParent := listNBI(parent)
480487
expTotal := ma.num + mm.num + mz.num
481488

482-
clear(missed)
483-
missed = missed[:0]
484489
expSet := make(cos.StrSet, expTotal)
485490
expSet.Add(ma.objNames...)
486491
expSet.Add(mm.objNames...)
@@ -491,18 +496,22 @@ func TestListInventoryPrefixPermute(t *testing.T) {
491496
for _, en := range gotParent {
492497
gotParentSet.Set(en.Name)
493498
}
499+
missed = missed[:0]
494500
for name := range expSet {
495501
if _, ok := gotParentSet[name]; !ok {
496502
missed = append(missed, name)
497503
}
498504
}
499-
tassert.Fatalf(t, len(missed) == 0, "missing %v for parent prefix %q - %d out of %d expected", missed, parent, len(missed), expTotal)
500-
tassert.Fatalf(t, len(gotParent) == expTotal, "parent prefix %q: expected %d, got %d", parent, expTotal, len(gotParent))
505+
tassert.Fatalf(t, len(missed) == 0,
506+
"missing %v for parent prefix %q - %d out of %d expected",
507+
missed, parent, len(missed), expTotal)
508+
tassert.Fatalf(t, len(gotParent) == expTotal,
509+
"parent prefix %q: expected %d, got %d", parent, expTotal, len(gotParent))
501510

502-
// 3) list missing prefix => must be empty (and must not loop tokens)
503511
missing := parent + "zzzz/"
504-
gotMissing := list(missing, &tc, createMsg.Name)
505-
tassert.Fatalf(t, len(gotMissing) == 0, "prefix %q: expected 0, got %d", missing, len(gotMissing))
512+
gotMissing := listNBI(missing)
513+
tassert.Fatalf(t, len(gotMissing) == 0,
514+
"prefix %q: expected 0, got %d", missing, len(gotMissing))
506515
})
507516
}
508517
}

‎api/apc/lsmsg.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,10 @@ const (
9292
LsIsS3
9393

9494
// instead of (remote) bucket: list native bucket inventory (NBI) snapshot
95+
// NOTE on pagination:
96+
// in the context of inventory listing, lsmsg.PageSize (if non-zero) is best-effort and approximate:
97+
// each target delivers an approximate share of the requested page size,
98+
// subject to local chunking, minimum bounds, and slight overfetch
9599
LsNBI
96100
)
97101

‎cmd/cli/go.mod‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ module github.com/NVIDIA/aistore/cmd/cli
33
go 1.25
44

55
require (
6-
github.com/NVIDIA/aistore v1.4.3-0.20260317160545-bf3c128925d2
6+
github.com/NVIDIA/aistore v1.4.3-0.20260317162519-ffefd96db92f
77
github.com/fatih/color v1.18.0
88
github.com/json-iterator/go v1.1.12
99
github.com/onsi/ginkgo/v2 v2.27.5

‎cmd/cli/go.sum‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
github.com/BurntSushi/toml v1.5.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
22
github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0=
33
github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM=
4-
github.com/NVIDIA/aistore v1.4.3-0.20260317160545-bf3c128925d2 h1:viUKZ18MT0AE223hYU95kUwsYs3CoJ46Fk0K9A12cOQ=
5-
github.com/NVIDIA/aistore v1.4.3-0.20260317160545-bf3c128925d2/go.mod h1:LWOYptrTZyGfxqIg6M7IIO+8UNmHBQZM68CbYVyPVM8=
4+
github.com/NVIDIA/aistore v1.4.3-0.20260317162519-ffefd96db92f h1:SAH+HEj5m/2Dw/HYZ2uYvh+8jHqURDmvJwkNa5fe7U0=
5+
github.com/NVIDIA/aistore v1.4.3-0.20260317162519-ffefd96db92f/go.mod h1:LWOYptrTZyGfxqIg6M7IIO+8UNmHBQZM68CbYVyPVM8=
66
github.com/OneOfOne/xxhash v1.2.8 h1:31czK/TI9sNkxIKfaUfGlU47BAxQ0ztGgd9vPyqimf8=
77
github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q=
88
github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA=

‎cmd/ishard/go.mod‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ module github.com/NVIDIA/aistore/cmd/ishard
33
go 1.25
44

55
require (
6-
github.com/NVIDIA/aistore v1.4.3-0.20260311173632-f3472105d2ea
6+
github.com/NVIDIA/aistore v1.4.3-0.20260317162519-ffefd96db92f
77
github.com/json-iterator/go v1.1.12
88
github.com/vbauerster/mpb/v4 v4.12.2
99
)

‎cmd/ishard/go.sum‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0=
22
github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM=
3-
github.com/NVIDIA/aistore v1.4.3-0.20260311173632-f3472105d2ea h1:YBsuGoB+GBGDcCJeQAHRB3lADTKymwtiqPprLO6AjyE=
4-
github.com/NVIDIA/aistore v1.4.3-0.20260311173632-f3472105d2ea/go.mod h1:LWOYptrTZyGfxqIg6M7IIO+8UNmHBQZM68CbYVyPVM8=
3+
github.com/NVIDIA/aistore v1.4.3-0.20260317162519-ffefd96db92f h1:SAH+HEj5m/2Dw/HYZ2uYvh+8jHqURDmvJwkNa5fe7U0=
4+
github.com/NVIDIA/aistore v1.4.3-0.20260317162519-ffefd96db92f/go.mod h1:LWOYptrTZyGfxqIg6M7IIO+8UNmHBQZM68CbYVyPVM8=
55
github.com/OneOfOne/xxhash v1.2.8 h1:31czK/TI9sNkxIKfaUfGlU47BAxQ0ztGgd9vPyqimf8=
66
github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q=
77
github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA=

‎xact/xs/ls_nbi.go‎

Lines changed: 39 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,21 @@ import (
2525
"github.com/tinylib/msgp/msgp"
2626
)
2727

28+
// NBI pagination note:
29+
//
30+
// In the context of inventory listing, lsmsg.PageSize is best-effort and approximate:
31+
// - each target delivers an approximate share of the requested page size,
32+
// subject to local chunking, minimum bounds, and slight overfetch;
33+
// - proxy then merges, sorts, and truncates the combined result.
34+
// - the actual number of returned entries may differ from the requested PageSize
35+
// (if specified).
36+
// PageSize=0 is strongly recommended: lets the system optimize per-target
37+
// chunk reads and minimize the total number of distributed-merge roundtrips.
38+
39+
const (
40+
minPageSize = max(10, apc.MinInvNamesPerChunk)
41+
)
42+
2843
// TODO -- FIXME:
2944
// - corner case: very small inventory w/ not every target having chunks
3045
// - test list-objects with empty invName (target must resolve single)
@@ -50,6 +65,7 @@ type nbiCtx struct {
5065
buf []byte
5166
slab *memsys.Slab
5267
cksum *cos.CksumHash
68+
nat int
5369
}
5470

5571
func (nbi *nbiCtx) init(invName string) error {
@@ -77,6 +93,14 @@ func (nbi *nbiCtx) init(invName string) error {
7793
return err
7894
}
7995

96+
smap := core.T.Sowner().Get()
97+
nbi.nat = smap.CountActiveTs()
98+
99+
if nbi.nat == 0 {
100+
nbi.cleanup()
101+
return cmn.NewErrNoNodes(apc.Target, smap.CountTargets())
102+
}
103+
80104
return nil
81105
}
82106

@@ -191,6 +215,18 @@ func (nbi *nbiCtx) readChunk() error {
191215

192216
func (nbi *nbiCtx) skip() { nbi.nidx = len(nbi.entries) } // in re: prefix
193217

218+
func (nbi *nbiCtx) pageSize(msg *apc.LsoMsg) int {
219+
// chunk-driven (optimize local reads/merge roundtrips)
220+
n := max(len(nbi.entries), minPageSize)
221+
if msg.PageSize == 0 {
222+
return n
223+
}
224+
225+
// honor requested size approximately, per target
226+
share := cos.DivRound(int(msg.PageSize), nbi.nat)
227+
return max(minPageSize, cos.DivRound(share*5, 4)) // overshoot by 25%
228+
}
229+
194230
func (nbi *nbiCtx) nextPage(msg *apc.LsoMsg, lst *cmn.LsoRes) error {
195231
lst.Entries = lst.Entries[:0]
196232
lst.ContinuationToken = ""
@@ -225,8 +261,9 @@ func (nbi *nbiCtx) nextPage(msg *apc.LsoMsg, lst *cmn.LsoRes) error {
225261
}
226262
}
227263

228-
pageSize := cos.Ternary(msg.PageSize <= 0, apc.MaxPageSizeAIS, msg.PageSize)
229-
for len(lst.Entries) < int(pageSize) {
264+
pageSize := nbi.pageSize(msg) // apply msg.PageSize
265+
266+
for len(lst.Entries) < pageSize {
230267
// next chunk
231268
if nbi.nidx >= len(nbi.entries) {
232269
// clone entries from the current (soon previous) chunk

0 commit comments

Comments
 (0)