[main] building indexes on old files if have some (cp from 3.1) (erigontech#17018)

cp from erigontech#16984

---------

Co-authored-by: JkLondon <me@ilyamikheev.com>
Co-authored-by: alex <AskAlexSharov@gmail.com>

Author: 3 people

cmd/capcli/cli.go

@@ -22,6 +22,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"github.com/erigontech/erigon/db/snaptype"
 	"io"
 	"math"
 	"net/http"
@@ -64,7 +65,6 @@ import (
 	"github.com/erigontech/erigon/db/kv"
 	"github.com/erigontech/erigon/db/snapshotsync"
 	"github.com/erigontech/erigon/db/snapshotsync/freezeblocks"
-	"github.com/erigontech/erigon/db/snaptype"
 	"github.com/erigontech/erigon/eth/ethconfig"
 	"github.com/erigontech/erigon/turbo/debug"
 )

db/snapshotsync/snapshots.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"github.com/erigontech/erigon/db/version"
 	"math"
 	"os"
 	"path/filepath"
@@ -464,7 +465,19 @@ func (s *DirtySegment) openIdx(dir string) (err error) {
 		if s.indexes[i] != nil {
 			continue
 		}
-		index, err := recsplit.OpenIndex(filepath.Join(dir, fileName))
+		fPathMask, err := version.ReplaceVersionWithMask(filepath.Join(dir, fileName))
+		if err != nil {
+			return fmt.Errorf("[open index] can't replace with mask in file %s: %w", fileName, err)
+		}
+		fPath, _, ok, err := version.FindFilesWithVersionsByPattern(fPathMask)
+		if err != nil {
+			return fmt.Errorf("%w, fileName: %s", err, fileName)
+		}
+		if !ok {
+			_, fName := filepath.Split(fPath)
+			return fmt.Errorf("[open index] find files by pattern err %w fname %s", os.ErrNotExist, fName)
+		}
+		index, err := recsplit.OpenIndex(fPath)
 
 		if err != nil {
 			return fmt.Errorf("%w, fileName: %s", err, fileName)

db/snaptype/files.go

@@ -39,13 +39,24 @@ func FileName(version Version, from, to uint64, fileType string) string {
 	return fmt.Sprintf("%s-%06d-%06d-%s", version.String(), from/1_000, to/1_000, fileType)
 }
 
+func FileMask(from, to uint64, fileType string) string {
+	return fmt.Sprintf("*-%06d-%06d-%s", from/1_000, to/1_000, fileType)
+}
+
 func SegmentFileName(version Version, from, to uint64, t Enum) string {
 	return FileName(version, from, to, t.String()) + ".seg"
 }
 func IdxFileName(version Version, from, to uint64, fType string) string {
 	return FileName(version, from, to, fType) + ".idx"
 }
 
+func SegmentFileMask(from, to uint64, t Enum) string {
+	return FileMask(from, to, t.String()) + ".seg"
+}
+func IdxFileMask(from, to uint64, fType string) string {
+	return FileMask(from, to, fType) + ".idx"
+}
+
 func FilterExt(in []FileInfo, expectExt string) (out []FileInfo) {
 	for _, f := range in {
 		if f.Ext != expectExt { // filter out only compressed files

db/snaptype/type.go

@@ -184,6 +184,7 @@ type Type interface {
 	Name() string
 	FileName(version Version, from uint64, to uint64) string
 	FileInfo(dir string, from uint64, to uint64) FileInfo
+	FileInfoByMask(dir string, from uint64, to uint64) FileInfo
 	IdxFileName(version Version, from uint64, to uint64, index ...Index) string
 	IdxFileNames(version Version, from uint64, to uint64) []string
 	Indexes() []Index
@@ -254,11 +255,29 @@ func (s snapType) FileName(version Version, from uint64, to uint64) string {
 	return SegmentFileName(version, from, to, s.enum)
 }
 
+func (s snapType) FileMask(from uint64, to uint64) string {
+	return SegmentFileMask(from, to, s.enum)
+}
+
 func (s snapType) FileInfo(dir string, from uint64, to uint64) FileInfo {
 	f, _, _ := ParseFileName(dir, s.FileName(s.versions.Current, from, to))
 	return f
 }
 
+func (s snapType) FileInfoByMask(dir string, from uint64, to uint64) FileInfo {
+	fName, _, ok, err := version.FindFilesWithVersionsByPattern(filepath.Join(dir, s.FileName(s.versions.Current, from, to)))
+	if err != nil {
+		log.Debug("[snaptype] file mask error", "err", err, "fName", s.FileName(s.versions.Current, from, to))
+		return FileInfo{}
+	}
+	if !ok {
+		return FileInfo{}
+	}
+
+	f, _, _ := ParseFileName("", fName)
+	return f
+}
+
 func (s snapType) ExtractRange(ctx context.Context, info FileInfo, rangeExtractor RangeExtractor, indexBuilder IndexBuilder, firstKeyGetter FirstKeyGetter, db kv.RoDB, chainConfig *chain.Config, tmpDir string, workers int, lvl log.Lvl, logger log.Logger, hashResolver BlockHashResolver) (uint64, error) {
 	if rangeExtractor == nil {
 		rangeExtractor = s.rangeExtractor
@@ -429,6 +448,22 @@ func BuildIndex(ctx context.Context, info FileInfo, cfg recsplit.RecSplitArgs, l
 		}
 	}()
 
+	fPathMask, err := version.ReplaceVersionWithMask(info.Path)
+	if err != nil {
+		return fmt.Errorf("[build index] can't replace with mask in file %s: %w", info.Name(), err)
+	}
+	fPath, fileVer, ok, err := version.FindFilesWithVersionsByPattern(fPathMask)
+	if err != nil {
+		_, fName := filepath.Split(fPath)
+		return fmt.Errorf("build index err %w fname %s", err, fName)
+	}
+	if !ok {
+		_, fName := filepath.Split(fPath)
+		return fmt.Errorf("build index err %w fname %s", os.ErrNotExist, fName)
+	}
+	info.Version = fileVer
+	info.Path = fPath
+
 	d, err := seg.NewDecompressor(info.Path)
 	if err != nil {
 		return fmt.Errorf("can't open %s for indexing: %w", info.Name(), err)
@@ -441,7 +476,7 @@ func BuildIndex(ctx context.Context, info FileInfo, cfg recsplit.RecSplitArgs, l
 		p.Total.Store(uint64(d.Count()))
 	}
 	cfg.KeyCount = d.Count()
-	cfg.IndexFile = filepath.Join(info.Dir(), info.Type.IdxFileName(info.Version, info.From, info.To))
+	cfg.IndexFile = filepath.Join(info.Dir(), info.Type.IdxFileName(fileVer, info.From, info.To))
 	rs, err := recsplit.NewRecSplit(cfg, logger)
 	if err != nil {
 		return err

db/version/file_version.go

@@ -217,3 +217,12 @@ func (v *Version) UnmarshalYAML(node *yaml.Node) error {
 	*v = ver
 	return nil
 }
+
+func VersionTooLowPanic(filename string, version Versions) {
+	panic(fmt.Sprintf(
+		"Version is too low, try to run snapshot reset: `erigon --datadir $DATADIR --chain $CHAIN snapshots reset`. file=%s, min_supported=%s, current=%s",
+		filename,
+		version.MinSupported,
+		version.Current,
+	))
+}