|
| 1 | +# TokenLists Package |
| 2 | + |
| 3 | +The `tokenlists` package provides a comprehensive solution for managing and fetching token lists from various sources in a privacy-aware manner. It supports multiple token list formats, automatic refresh capabilities, and cross-chain token management. |
| 4 | + |
| 5 | +## Features |
| 6 | + |
| 7 | +- **Multi-source Support**: Fetch token lists from Status, Uniswap, CoinGecko, and custom sources |
| 8 | +- **Privacy-aware**: Respects privacy settings to prevent unwanted network requests |
| 9 | +- **Automatic Refresh**: Configurable automatic refresh intervals with ETag support |
| 10 | +- **Cross-chain Support**: Manage tokens across multiple blockchain networks |
| 11 | +- **Extensible**: Plugin-based parser system for custom token list formats |
| 12 | +- **Thread-safe**: Concurrent access support with proper synchronization |
| 13 | +- **Caching**: Built-in content caching with ETag support for efficient updates |
| 14 | + |
| 15 | +## Quick Start |
| 16 | + |
| 17 | +```go |
| 18 | +package main |
| 19 | + |
| 20 | +import ( |
| 21 | + "context" |
| 22 | + "log" |
| 23 | + "time" |
| 24 | + |
| 25 | + "github.com/status-im/go-wallet-sdk/pkg/tokenlists" |
| 26 | + "go.uber.org/zap" |
| 27 | +) |
| 28 | + |
| 29 | +func main() { |
| 30 | + // Create configuration |
| 31 | + config := tokenlists.DefaultConfig(). |
| 32 | + WithChains([]uint64{1, 10, 137}). // Ethereum, Polygon, BSC |
| 33 | + WithRemoteListOfTokenListsURL("https://example.com/token-lists.json"). |
| 34 | + WithAutoRefreshInterval(30*time.Minute, 3*time.Minute). |
| 35 | + WithLogger(zap.NewNop()) |
| 36 | + |
| 37 | + // Create token list manager |
| 38 | + tl, err := tokenlists.NewTokensList(config) |
| 39 | + if err != nil { |
| 40 | + log.Fatal(err) |
| 41 | + } |
| 42 | + |
| 43 | + // Start the service |
| 44 | + ctx := context.Background() |
| 45 | + notifyCh := make(chan struct{}, 1) |
| 46 | + |
| 47 | + if err := tl.Start(ctx, notifyCh); err != nil { |
| 48 | + log.Fatal(err) |
| 49 | + } |
| 50 | + defer tl.Stop(ctx) |
| 51 | + |
| 52 | + // Wait for initial fetch |
| 53 | + select { |
| 54 | + case <-notifyCh: |
| 55 | + log.Println("Token lists updated") |
| 56 | + case <-time.After(10 * time.Second): |
| 57 | + log.Println("Timeout waiting for token lists") |
| 58 | + } |
| 59 | + |
| 60 | + // Get all unique tokens |
| 61 | + tokens := tl.UniqueTokens() |
| 62 | + log.Printf("Found %d unique tokens", len(tokens)) |
| 63 | + |
| 64 | + // Get tokens for a specific chain |
| 65 | + ethereumTokens := tl.GetTokensByChain(1) |
| 66 | + log.Printf("Found %d tokens on Ethereum", len(ethereumTokens)) |
| 67 | +} |
| 68 | +``` |
| 69 | + |
| 70 | +## Configuration |
| 71 | + |
| 72 | +The package uses a builder pattern for configuration: |
| 73 | + |
| 74 | +```go |
| 75 | +config := &tokenlists.Config{ |
| 76 | + // Required fields |
| 77 | + MainList: []byte(`{"tokens": []}`), |
| 78 | + MainListID: "status", |
| 79 | + Chains: []uint64{1, 10, 137}, |
| 80 | + |
| 81 | + // Optional fields with defaults |
| 82 | + RemoteListOfTokenListsURL: "https://example.com/lists.json", |
| 83 | + AutoRefreshInterval: 30 * time.Minute, |
| 84 | + AutoRefreshCheckInterval: 3 * time.Minute, |
| 85 | + |
| 86 | + // Custom components |
| 87 | + PrivacyGuard: tokenlists.NewDefaultPrivacyGuard(false), |
| 88 | + LastTokenListsUpdateTimeStore: tokenlists.NewDefaultLastTokenListsUpdateTimeStore(), |
| 89 | + ContentStore: tokenlists.NewDefaultContentStore(), |
| 90 | + Parsers: make(map[string]tokenlists.Parser), |
| 91 | +} |
| 92 | +``` |
| 93 | + |
| 94 | +### Configuration Options |
| 95 | + |
| 96 | +| Option | Type | Default | Description | |
| 97 | +|--------|------|---------|-------------| |
| 98 | +| `MainList` | `[]byte` | - | Initial token list data | |
| 99 | +| `MainListID` | `string` | - | Identifier for the main token list | |
| 100 | +| `Chains` | `[]uint64` | - | Supported blockchain chain IDs | |
| 101 | +| `RemoteListOfTokenListsURL` | `string` | - | URL to fetch list of token lists | |
| 102 | +| `AutoRefreshInterval` | `time.Duration` | 30 min | How often to refresh token lists | |
| 103 | +| `AutoRefreshCheckInterval` | `time.Duration` | 3 min | How often to check if refresh is needed | |
| 104 | +| `PrivacyGuard` | `PrivacyGuard` | `NewDefaultPrivacyGuard(false)` | Privacy mode controller | |
| 105 | +| `ContentStore` | `ContentStore` | `NewDefaultContentStore()` | Content caching store | |
| 106 | +| `Parsers` | `map[string]Parser` | Built-in parsers | Token list format parsers | |
| 107 | + |
| 108 | +## API Reference |
| 109 | + |
| 110 | +### TokensList Interface |
| 111 | + |
| 112 | +```go |
| 113 | +type TokensList interface { |
| 114 | + // Lifecycle management |
| 115 | + Start(ctx context.Context, notifyCh chan struct{}) error |
| 116 | + Stop(ctx context.Context) error |
| 117 | + |
| 118 | + LastRefreshTime() (time.Time, error) |
| 119 | + RefreshNow(ctx context.Context) error |
| 120 | + |
| 121 | + // Privacy management |
| 122 | + PrivacyModeUpdated(ctx context.Context) error |
| 123 | + |
| 124 | + // Token queries |
| 125 | + UniqueTokens() []*Token |
| 126 | + GetTokenByChainAddress(chainID uint64, addr common.Address) (*Token, bool) |
| 127 | + GetTokensByChain(chainID uint64) []*Token |
| 128 | + |
| 129 | + // Token list queries |
| 130 | + TokenLists() []*TokenList |
| 131 | + TokenList(id string) (*TokenList, bool) |
| 132 | +} |
| 133 | +``` |
| 134 | + |
| 135 | +### Token Structure |
| 136 | + |
| 137 | +```go |
| 138 | +type Token struct { |
| 139 | + CrossChainID string `json:"crossChainId"` |
| 140 | + ChainID uint64 `json:"chainId"` |
| 141 | + Address common.Address `json:"address"` |
| 142 | + Decimals uint `json:"decimals"` |
| 143 | + Name string `json:"name"` |
| 144 | + Symbol string `json:"symbol"` |
| 145 | + LogoURI string `json:"logoUri"` |
| 146 | + Color string `json:"color"` |
| 147 | + CommunityID bool `json:"community"` |
| 148 | + CustomToken bool `json:"custom"` |
| 149 | +} |
| 150 | +``` |
| 151 | + |
| 152 | +### TokenList Structure |
| 153 | + |
| 154 | +```go |
| 155 | +type TokenList struct { |
| 156 | + Name string `json:"name"` |
| 157 | + Timestamp string `json:"timestamp"` |
| 158 | + FetchedTimestamp string `json:"fetchedTimestamp"` |
| 159 | + Source string `json:"source"` |
| 160 | + Version Version `json:"version"` |
| 161 | + Tags map[string]interface{} `json:"tags"` |
| 162 | + LogoURI string `json:"logoURI"` |
| 163 | + Keywords []string `json:"keywords"` |
| 164 | + Tokens []*Token `json:"tokens"` |
| 165 | +} |
| 166 | +``` |
| 167 | + |
| 168 | +## Privacy Mode |
| 169 | + |
| 170 | +The package respects privacy settings to prevent unwanted network requests: |
| 171 | + |
| 172 | +```go |
| 173 | +// Enable privacy mode |
| 174 | +config := tokenlists.DefaultConfig(). |
| 175 | + WithPrivacyGuard(tokenlists.NewDefaultPrivacyGuard(true)) |
| 176 | + |
| 177 | +// Privacy mode prevents: |
| 178 | +// - Automatic token list fetching |
| 179 | +// - RefreshNow() calls from making network requests |
| 180 | +// - Background refresh worker from running |
| 181 | +``` |
| 182 | + |
| 183 | +## Supported Token List Formats |
| 184 | + |
| 185 | +### Status Token List |
| 186 | +- **Parser**: `StatusTokenListParser` |
| 187 | +- **Format**: Status-specific JSON format |
| 188 | + |
| 189 | +### Standard Token List Formats (uniswap, platform specific coingecko list and others use this format) |
| 190 | +- **Parser**: `StandardTokenListParser` |
| 191 | +- **Format**: Standard Token List format |
| 192 | + |
| 193 | +### CoinGecko All Token List (doesn't contain decimals) |
| 194 | +- **Parser**: `CoinGeckoAllTokensParser` |
| 195 | +- **Format**: CoinGecko API format with chain mapping |
| 196 | + |
| 197 | +## Custom Parsers (if the list doesn't follow the standard token list format) |
| 198 | + |
| 199 | +Implement the `Parser` interface to support custom token list formats: |
| 200 | + |
| 201 | +```go |
| 202 | +type CustomParser struct{} |
| 203 | + |
| 204 | +func (p *CustomParser) Parse(raw []byte, sourceURL string, fetchedAt time.Time) (*TokenList, error) { |
| 205 | + // Parse your custom format |
| 206 | + var customFormat struct { |
| 207 | + Name string `json:"name"` |
| 208 | + Tokens []*Token `json:"tokens"` |
| 209 | + } |
| 210 | + |
| 211 | + if err := json.Unmarshal(raw, &customFormat); err != nil { |
| 212 | + return nil, err |
| 213 | + } |
| 214 | + |
| 215 | + return &TokenList{ |
| 216 | + Name: customFormat.Name, |
| 217 | + Timestamp: time.Now().Format(time.RFC3339), |
| 218 | + FetchedTimestamp: fetchedAt.Format(time.RFC3339), |
| 219 | + Source: sourceURL, |
| 220 | + Tokens: customFormat.Tokens, |
| 221 | + }, nil |
| 222 | +} |
| 223 | + |
| 224 | +// Register custom parser |
| 225 | +config := tokenlists.DefaultConfig(). |
| 226 | + WithParsers(map[string]tokenlists.Parser{ |
| 227 | + "custom": &CustomParser{}, |
| 228 | + }) |
| 229 | +``` |
| 230 | + |
| 231 | +## Error Handling |
| 232 | + |
| 233 | +The package provides comprehensive error handling: |
| 234 | + |
| 235 | +```go |
| 236 | +tl, err := tokenlists.NewTokensList(config) |
| 237 | +if err != nil { |
| 238 | + // Handle configuration errors |
| 239 | + log.Fatal(err) |
| 240 | +} |
| 241 | + |
| 242 | +if err := tl.Start(ctx, notifyCh); err != nil { |
| 243 | + // Handle startup errors |
| 244 | + log.Fatal(err) |
| 245 | +} |
| 246 | + |
| 247 | +// Handle refresh errors via notification channel |
| 248 | +go func() { |
| 249 | + for range notifyCh { |
| 250 | + // Token lists updated successfully |
| 251 | + log.Println("Token lists refreshed") |
| 252 | + } |
| 253 | +}() |
| 254 | +``` |
| 255 | + |
| 256 | +## Testing |
| 257 | + |
| 258 | +The package includes comprehensive tests: |
| 259 | + |
| 260 | +```bash |
| 261 | +# Run all tests |
| 262 | +go test ./pkg/tokenlists/... |
| 263 | + |
| 264 | +# Run specific test |
| 265 | +go test ./pkg/tokenlists -run TestTokensList_RefreshNow |
| 266 | + |
| 267 | +# Run with verbose output |
| 268 | +go test ./pkg/tokenlists/... -v |
| 269 | +``` |
0 commit comments