metabase: use meta bucket to mark containers with GC (#3561)

Closes #3521.

Author: roman-khimov

CHANGELOG.md

@@ -17,6 +17,7 @@ Changelog for NeoFS Node
 - `neofs-cli object range` command now creates file with `rw-r--r--` permissions (#3544)
 - Alphabet nodes send basic storage income based on the new Reports API from `container` contract (#3053)
 - Use stream API of FSTree for object service `Get` operation (#3466)
+- Use meta buckets to mark containers with GC (#3561)
 
 ### Removed
 - `neofs-cli object head --main-only` no-op flag (#3509)

pkg/local_object_storage/metabase/VERSION.md

@@ -16,10 +16,6 @@ The lowest not used bucket index: 20.
   - Name: `1`
   - Key: object address
   - Value: dummy value
-- Garbage containers bucket
-  - Name: `17`
-  - Key: container ID
-  - Value: dummy value
 - Bucket containing IDs of objects that are candidates for moving
    to another shard.
   - Name: `2`
@@ -51,6 +47,8 @@ The lowest not used bucket index: 20.
       Sign byte is 0 for negatives, 1 otherwise. Bits are inverted for negatives also.
     - `2` + attribute + `0x00` + value + `0x00` + object ID
     - `3` + object ID + attribute + `0x00` + value
+    - `4` — container-level GC mark. \
+      Presence means the whole container is scheduled for garbage collection.
 
 # History
 
@@ -59,6 +57,8 @@ The lowest not used bucket index: 20.
 Container statistic is now a bucket with multiple values. In the version number
 of objects was added
 
+Drop garbage containers index (17), replaced with mark in metadata bucket.
+
 ## Version 7
 
 Fixed version 6 which could store OID keys in garbage object bucket.

pkg/local_object_storage/metabase/containers.go

@@ -205,14 +205,6 @@ func (db *DB) DeleteContainer(cID cid.ID) error {
 			return fmt.Errorf("metadata bucket cleanup: %w", err)
 		}
 
-		cnrGCBkt := tx.Bucket(garbageContainersBucketName)
-		if cnrGCBkt != nil {
-			err = cnrGCBkt.Delete(cIDRaw)
-			if err != nil {
-				return fmt.Errorf("garbage containers cleanup: %w", err)
-			}
-		}
-
 		return nil
 	})
 }

pkg/local_object_storage/metabase/control.go

@@ -86,13 +86,12 @@ func (db *DB) init(reset bool) error {
 	}
 
 	mStaticBuckets := map[string]struct{}{
-		string(containerVolumeBucketName):   {},
-		string(graveyardBucketName):         {},
-		string(toMoveItBucketName):          {},
-		string(garbageObjectsBucketName):    {},
-		string(garbageContainersBucketName): {},
-		string(shardInfoBucket):             {},
-		string(bucketNameLocked):            {},
+		string(containerVolumeBucketName): {},
+		string(graveyardBucketName):       {},
+		string(toMoveItBucketName):        {},
+		string(garbageObjectsBucketName):  {},
+		string(shardInfoBucket):           {},
+		string(bucketNameLocked):          {},
 	}
 
 	if !reset {

pkg/local_object_storage/metabase/counter.go

@@ -140,7 +140,6 @@ func syncCounter(tx *bbolt.Tx, currEpoch uint64, force bool) error {
 
 	graveyardBKT := tx.Bucket(graveyardBucketName)
 	garbageObjectsBKT := tx.Bucket(garbageObjectsBucketName)
-	garbageContainersBKT := tx.Bucket(garbageContainersBucketName)
 	key := make([]byte, addressKeySize)
 
 	err = iteratePhyObjects(tx, func(cnr cid.ID, obj oid.ID) error {
@@ -157,7 +156,7 @@ func syncCounter(tx *bbolt.Tx, currEpoch uint64, force bool) error {
 
 		// check if an object is available: not with GCMark
 		// and not covered with a tombstone
-		if inGraveyardWithKey(metaCursor, addressKey(addr, key), graveyardBKT, garbageObjectsBKT, garbageContainersBKT) == statusAvailable {
+		if inGraveyardWithKey(metaCursor, addressKey(addr, key), graveyardBKT, garbageObjectsBKT) == statusAvailable {
 			logicCounter++
 		}
 

pkg/local_object_storage/metabase/delete.go

@@ -132,15 +132,14 @@ func (db *DB) delete(tx *bbolt.Tx, addr oid.Address, currEpoch uint64) (bool, bo
 	cID := addr.Container()
 	addrKey := addressKey(addr, key)
 	garbageObjectsBKT := tx.Bucket(garbageObjectsBucketName)
-	garbageContainersBKT := tx.Bucket(garbageContainersBucketName)
 	graveyardBKT := tx.Bucket(graveyardBucketName)
 	metaBucket := tx.Bucket(metaBucketKey(cID))
 	var metaCursor *bbolt.Cursor
 	if metaBucket != nil {
 		metaCursor = metaBucket.Cursor()
 	}
 
-	removeAvailableObject := inGraveyardWithKey(metaCursor, addrKey, graveyardBKT, garbageObjectsBKT, garbageContainersBKT) == statusAvailable
+	removeAvailableObject := inGraveyardWithKey(metaCursor, addrKey, graveyardBKT, garbageObjectsBKT) == statusAvailable
 
 	// remove record from the garbage bucket
 	if garbageObjectsBKT != nil {

pkg/local_object_storage/metabase/exists.go

@@ -145,15 +145,18 @@ func objectStatus(tx *bbolt.Tx, metaCursor *bbolt.Cursor, addr oid.Address, curr
 // where a single address needs to be checked.
 func inGraveyard(tx *bbolt.Tx, metaCursor *bbolt.Cursor, addr oid.Address) uint8 {
 	var (
-		addrKey              = addressKey(addr, make([]byte, addressKeySize))
-		garbageContainersBkt = tx.Bucket(garbageContainersBucketName)
-		garbageObjectsBkt    = tx.Bucket(garbageObjectsBucketName)
-		graveyardBkt         = tx.Bucket(graveyardBucketName)
+		addrKey           = addressKey(addr, make([]byte, addressKeySize))
+		garbageObjectsBkt = tx.Bucket(garbageObjectsBucketName)
+		graveyardBkt      = tx.Bucket(graveyardBucketName)
 	)
-	return inGraveyardWithKey(metaCursor, addrKey, graveyardBkt, garbageObjectsBkt, garbageContainersBkt)
+	return inGraveyardWithKey(metaCursor, addrKey, graveyardBkt, garbageObjectsBkt)
 }
 
-func inGraveyardWithKey(metaCursor *bbolt.Cursor, addrKey []byte, graveyard, garbageObjectsBCK, garbageContainersBCK *bbolt.Bucket) uint8 {
+func inGraveyardWithKey(metaCursor *bbolt.Cursor, addrKey []byte, graveyard, garbageObjectsBCK *bbolt.Bucket) uint8 {
+	if metaCursor != nil && containerMarkedGC(metaCursor) {
+		return statusGCMarked
+	}
+
 	if associatedWithTypedObject(0, metaCursor, oid.ID(addrKey[cid.Size:]), objectSDK.TypeTombstone) {
 		return statusTombstoned
 	}
@@ -171,11 +174,7 @@ func inGraveyardWithKey(metaCursor *bbolt.Cursor, addrKey []byte, graveyard, gar
 			return statusAvailable
 		}
 
-		val = garbageContainersBCK.Get(addrKey[:cidSize])
-		if val == nil {
-			val = garbageObjectsBCK.Get(addrKey)
-		}
-
+		val = garbageObjectsBCK.Get(addrKey)
 		if val != nil {
 			// object has been marked with GC
 			return statusGCMarked

pkg/local_object_storage/metabase/graveyard.go

@@ -288,7 +288,6 @@ func (db *DB) GetGarbage(limit int) ([]oid.Address, []cid.ID, error) {
 	initCap := min(limit, reasonableLimit)
 
 	var addrBuff oid.Address
-	var cidBuff cid.ID
 	alreadyHandledContainers := make(map[cid.ID]struct{})
 	resObjects := make([]oid.Address, 0, initCap)
 	resContainers := make([]cid.ID, 0)
@@ -299,26 +298,34 @@ func (db *DB) GetGarbage(limit int) ([]oid.Address, []cid.ID, error) {
 		// also be deleted as a part of non-existing
 		// container so no need to handle it twice
 
-		bkt := tx.Bucket(garbageContainersBucketName)
-		c := bkt.Cursor()
-
-		for k, _ := c.First(); k != nil; k, _ = c.Next() {
-			err := cidBuff.Decode(k)
-			if err != nil {
-				return fmt.Errorf("parsing raw CID: %w", err)
+		var inhumedCnrs []cid.ID
+		err := tx.ForEach(func(name []byte, b *bbolt.Bucket) error {
+			if name[0] == metadataPrefix && containerMarkedGC(b.Cursor()) {
+				var cnr cid.ID
+				cidRaw, prefix := parseContainerIDWithPrefix(&cnr, name)
+				if cidRaw == nil || prefix != metadataPrefix {
+					return nil
+				}
+				inhumedCnrs = append(inhumedCnrs, cnr)
 			}
+			return nil
+		})
+		if err != nil {
+			return fmt.Errorf("scanning inhumed containers: %w", err)
+		}
 
-			resObjects, err = listContainerObjects(tx, cidBuff, resObjects, limit)
+		for _, cnr := range inhumedCnrs {
+			resObjects, err = listContainerObjects(tx, cnr, resObjects, limit)
 			if err != nil {
-				return fmt.Errorf("listing objects for %s container: %w", cidBuff, err)
+				return fmt.Errorf("listing objects for %s container: %w", cnr, err)
 			}
 
-			alreadyHandledContainers[cidBuff] = struct{}{}
+			alreadyHandledContainers[cnr] = struct{}{}
 
 			if len(resObjects) < limit {
 				// all the objects from the container were listed,
 				// container can be removed
-				resContainers = append(resContainers, cidBuff)
+				resContainers = append(resContainers, cnr)
 			} else {
 				return nil
 			}
@@ -327,8 +334,8 @@ func (db *DB) GetGarbage(limit int) ([]oid.Address, []cid.ID, error) {
 		// deleted containers are not enough to reach the limit,
 		// check manually deleted objects then
 
-		bkt = tx.Bucket(garbageObjectsBucketName)
-		c = bkt.Cursor()
+		bkt := tx.Bucket(garbageObjectsBucketName)
+		c := bkt.Cursor()
 
 		for k, _ := c.First(); k != nil; k, _ = c.Next() {
 			err := decodeAddressFromKey(&addrBuff, k)

pkg/local_object_storage/metabase/inhume.go

@@ -58,7 +58,6 @@ func (db *DB) inhume(tombstone *oid.Address, tombExpiration uint64, force bool,
 	)
 	err = db.boltDB.Update(func(tx *bbolt.Tx) error {
 		garbageObjectsBKT := tx.Bucket(garbageObjectsBucketName)
-		garbageContainersBKT := tx.Bucket(garbageContainersBucketName)
 		graveyardBKT := tx.Bucket(graveyardBucketName)
 
 		var (
@@ -125,7 +124,7 @@ func (db *DB) inhume(tombstone *oid.Address, tombExpiration uint64, force bool,
 			obj, err := get(tx, addr, false, true, currEpoch)
 			targetKey := addressKey(addr, buf)
 			if err == nil {
-				if inGraveyardWithKey(metaCursor, targetKey, graveyardBKT, garbageObjectsBKT, garbageContainersBKT) == statusAvailable {
+				if inGraveyardWithKey(metaCursor, targetKey, graveyardBKT, garbageObjectsBKT) == statusAvailable {
 					// object is available, decrement the
 					// logical counter
 					inhumed++
@@ -219,13 +218,14 @@ func (db *DB) InhumeContainer(cID cid.ID) (uint64, error) {
 	}
 
 	var removedAvailable uint64
-	rawCID := cID[:]
 
 	err := db.boltDB.Update(func(tx *bbolt.Tx) error {
-		garbageContainersBKT := tx.Bucket(garbageContainersBucketName)
-		err := garbageContainersBKT.Put(rawCID, zeroValue)
+		metaBkt, err := tx.CreateBucketIfNotExists(metaBucketKey(cID))
 		if err != nil {
-			return fmt.Errorf("put GC mark for container: %w", err)
+			return fmt.Errorf("create meta bucket: %w", err)
+		}
+		if err := metaBkt.Put(containerGCMarkKey, nil); err != nil {
+			return fmt.Errorf("write container GC mark: %w", err)
 		}
 
 		info := db.containerInfo(tx, cID)

pkg/local_object_storage/metabase/list.go

@@ -64,7 +64,6 @@ func (db *DB) listWithCursor(tx *bbolt.Tx, currEpoch uint64, result []objectcore
 	var offset []byte
 	graveyardBkt := tx.Bucket(graveyardBucketName)
 	garbageObjectsBkt := tx.Bucket(garbageObjectsBucketName)
-	garbageContainersBkt := tx.Bucket(garbageContainersBucketName)
 
 	var rawAddr = make([]byte, cidSize, addressKeySize)
 
@@ -78,7 +77,7 @@ loop:
 		bkt := tx.Bucket(name)
 		if bkt != nil {
 			copy(rawAddr, cidRaw)
-			result, offset, cursor = selectNFromBucket(bkt, currEpoch, graveyardBkt, garbageObjectsBkt, garbageContainersBkt, rawAddr, containerID,
+			result, offset, cursor = selectNFromBucket(bkt, currEpoch, graveyardBkt, garbageObjectsBkt, rawAddr, containerID,
 				result, count, cursor, threshold)
 		}
 		bucketName = name
@@ -112,7 +111,7 @@ loop:
 // object to start selecting from. Ignores inhumed objects.
 func selectNFromBucket(bkt *bbolt.Bucket, // main bucket
 	currEpoch uint64,
-	graveyardBkt, garbageObjectsBkt, garbageContainersBkt *bbolt.Bucket, // cached graveyard buckets
+	graveyardBkt, garbageObjectsBkt *bbolt.Bucket, // cached graveyard buckets
 	cidRaw []byte, // container ID prefix, optimization
 	cnt cid.ID, // container ID
 	to []objectcore.AddressWithType, // listing result
@@ -132,6 +131,10 @@ func selectNFromBucket(bkt *bbolt.Bucket, // main bucket
 		typePrefix = make([]byte, metaIDTypePrefixSize)
 	)
 
+	if containerMarkedGC(c) {
+		return to, nil, cursor
+	}
+
 	fillIDTypePrefix(typePrefix)
 
 	if threshold {
@@ -157,7 +160,7 @@ func selectNFromBucket(bkt *bbolt.Bucket, // main bucket
 
 		mCursor := bkt.Cursor()
 		offset = k
-		if inGraveyardWithKey(mCursor, append(cidRaw, obj[:]...), graveyardBkt, garbageObjectsBkt, garbageContainersBkt) != statusAvailable {
+		if inGraveyardWithKey(mCursor, append(cidRaw, obj[:]...), graveyardBkt, garbageObjectsBkt) != statusAvailable {
 			continue
 		}