feat: add createGetCheckpointsRange

This function can be used to create getCheckpointsRange function
for provider with out-of-the-box cache.

Author: Sekhmet

src/providers/core/createGetCheckpointsRange.ts

@@ -0,0 +1,106 @@
+import { CheckpointRecord } from '../../stores/checkpoints';
+
+type CacheEntry = {
+  /**
+   * Block range for which cache record is valid. This is inclusive on both ends.
+   */
+  range: [number, number];
+  /**
+   * Checkpoint records for the source in the given range.
+   */
+  records: CheckpointRecord[];
+};
+
+/**
+ * Options for createGetCheckpointsRange.
+ */
+type Options<T> = {
+  /**
+   * Function to retrieve sources based on the block number.
+   *
+   * @param blockNumber Block number.
+   * @returns Array of sources.
+   */
+  sourcesFn: (blockNumber: number) => T[];
+  /**
+   * Function to extract a key from the source. This key is used for cache.
+   *
+   * @param source Source.
+   * @returns Key.
+   */
+  keyFn: (source: T) => string;
+  /**
+   * Function to query checkpoint records for a source within given block range.
+   *
+   * @param fromBlock Starting block number.
+   * @param toBlock Ending block number.
+   * @param source The source to query.
+   * @returns Promise resolving to an array of CheckpointRecords.
+   */
+  querySourceFn: (fromBlock: number, toBlock: number, source: T) => Promise<CheckpointRecord[]>;
+};
+
+/**
+ * Creates a getCheckpointsRange function.
+ *
+ * This function has a cache to avoid querying the same source for the same block range multiple times.
+ * This cache automatically evicts entries outside of the last queried range.
+ *
+ * @param options
+ * @returns A function that retrieves checkpoint records for a given block range.
+ */
+export function createGetCheckpointsRange<T>(options: Options<T>) {
+  const cache = new Map<string, CacheEntry>();
+
+  return async (fromBlock: number, toBlock: number): Promise<CheckpointRecord[]> => {
+    const sources = options.sourcesFn(fromBlock);
+
+    let events: CheckpointRecord[] = [];
+    for (const source of sources) {
+      let sourceEvents: CheckpointRecord[] = [];
+
+      const key = options.keyFn(source);
+
+      const cacheEntry = cache.get(key);
+      if (!cacheEntry) {
+        const events = await options.querySourceFn(fromBlock, toBlock, source);
+        sourceEvents = sourceEvents.concat(events);
+      } else {
+        const [cacheStart, cacheEnd] = cacheEntry.range;
+
+        const cacheEntries = cacheEntry.records.filter(
+          ({ blockNumber }) => blockNumber >= fromBlock && blockNumber <= toBlock
+        );
+
+        sourceEvents = sourceEvents.concat(cacheEntries);
+
+        const bottomHalfStart = fromBlock;
+        const bottomHalfEnd = Math.min(toBlock, cacheStart - 1);
+
+        const topHalfStart = Math.max(fromBlock, cacheEnd + 1);
+        const topHalfEnd = toBlock;
+
+        if (bottomHalfStart <= bottomHalfEnd) {
+          const events = await options.querySourceFn(bottomHalfStart, bottomHalfEnd, source);
+          sourceEvents = sourceEvents.concat(events);
+        }
+
+        if (topHalfStart <= topHalfEnd) {
+          const events = await options.querySourceFn(topHalfStart, topHalfEnd, source);
+          sourceEvents = sourceEvents.concat(events);
+        }
+      }
+
+      sourceEvents.sort((a, b) => a.blockNumber - b.blockNumber);
+
+      cache.set(key, {
+        range: [fromBlock, toBlock],
+        records: sourceEvents
+      });
+
+      events = events.concat(sourceEvents);
+    }
+
+    return events;
+  };
+}

src/providers/evm/provider.ts

@@ -4,10 +4,10 @@ import { Log, Provider, StaticJsonRpcProvider } from '@ethersproject/providers';
 import { Interface, LogDescription } from '@ethersproject/abi';
 import { keccak256 } from '@ethersproject/keccak256';
 import { toUtf8Bytes } from '@ethersproject/strings';
-import { CheckpointRecord } from '../../stores/checkpoints';
 import { Writer } from './types';
 import { ContractSourceConfig } from '../../types';
 import { sleep } from '../../utils/helpers';
+import { createGetCheckpointsRange } from '../core/createGetCheckpointsRange';
 
 type BlockWithTransactions = Awaited<ReturnType<Provider['getBlockWithTransactions']>>;
 type Transaction = BlockWithTransactions['transactions'][number];
@@ -32,6 +32,13 @@ export class EvmProvider extends BaseProvider {
 
     this.provider = new StaticJsonRpcProvider(this.instance.config.network_node_url);
     this.writers = writers;
+
+    this.getCheckpointsRange = createGetCheckpointsRange({
+      sourcesFn: blockNumber => this.instance.getCurrentSources(blockNumber),
+      keyFn: source => source.contract,
+      querySourceFn: async (fromBlock, toBlock, source) =>
+        this.getLogs(fromBlock, toBlock, source.contract)
+    });
   }
 
   formatAddresses(addresses: string[]): string[] {
@@ -274,17 +281,6 @@ export class EvmProvider extends BaseProvider {
     }));
   }
 
-  async getCheckpointsRange(fromBlock: number, toBlock: number): Promise<CheckpointRecord[]> {
-    let events: CheckpointRecord[] = [];
-
-    for (const source of this.instance.getCurrentSources(fromBlock)) {
-      const addressEvents = await this.getLogs(fromBlock, toBlock, source.contract);
-      events = events.concat(addressEvents);
-    }
-
-    return events;
-  }
-
   getEventHash(eventName: string) {
     if (!this.sourceHashes.has(eventName)) {
       this.sourceHashes.set(eventName, keccak256(toUtf8Bytes(eventName)));

src/providers/index.ts

@@ -1,3 +1,4 @@
 export * from './base';
 export * as starknet from './starknet';
 export * as evm from './evm';
+export * from './core/createGetCheckpointsRange';

src/providers/starknet/provider.ts

@@ -16,6 +16,7 @@ import {
 } from './types';
 import { ContractSourceConfig } from '../../types';
 import { sleep } from '../../utils/helpers';
+import { createGetCheckpointsRange } from '../core/createGetCheckpointsRange';
 
 export class StarknetProvider extends BaseProvider {
   private readonly provider: RpcProvider;
@@ -39,6 +40,18 @@ export class StarknetProvider extends BaseProvider {
       nodeUrl: this.instance.config.network_node_url
     });
     this.writers = writers;
+
+    this.getCheckpointsRangeForAddress = createGetCheckpointsRange({
+      sourcesFn: blockNumber => this.instance.getCurrentSources(blockNumber),
+      keyFn: source => source.contract,
+      querySourceFn: async (fromBlock, toBlock, source) =>
+        this.getCheckpointsRangeForAddress(
+          fromBlock,
+          toBlock,
+          source.contract,
+          source.events.map(event => event.name)
+        )
+    });
   }
 
   public async init() {

test/unit/providers/core/createGetCheckpointsRange.ts

@@ -0,0 +1,161 @@
+import { createGetCheckpointsRange } from '../../../../src/providers/core/createGetCheckpointsRange';
+
+it('should call querySourceFn for every source', async () => {
+  const sources = ['a', 'b', 'c'];
+
+  const mockFunction = jest.fn().mockReturnValue([]);
+
+  const getCheckpointsRange = createGetCheckpointsRange({
+    sourcesFn: () => sources,
+    keyFn: source => source,
+    querySourceFn: mockFunction
+  });
+
+  await getCheckpointsRange(10, 20);
+  expect(mockFunction).toHaveBeenCalledTimes(sources.length);
+  for (const source of sources) {
+    expect(mockFunction).toHaveBeenCalledWith(10, 20, source);
+  }
+});
+
+it('should return value for requested source', async () => {
+  let sources = ['a', 'b'];
+
+  const mockFunction = jest.fn().mockImplementation((fromBlock, toBlock, source) => {
+    return {
+      blockNumber: 14,
+      contractAddress: source
+    };
+  });
+
+  const getCheckpointsRange = createGetCheckpointsRange({
+    sourcesFn: () => sources,
+    keyFn: source => source,
+    querySourceFn: mockFunction
+  });
+
+  let result = await getCheckpointsRange(10, 20);
+  expect(result).toEqual([
+    {
+      blockNumber: 14,
+      contractAddress: 'a'
+    },
+    {
+      blockNumber: 14,
+      contractAddress: 'b'
+    }
+  ]);
+
+  sources = ['b'];
+  result = await getCheckpointsRange(10, 20);
+  expect(result).toEqual([
+    {
+      blockNumber: 14,
+      contractAddress: 'b'
+    }
+  ]);
+});
+
+describe('cache', () => {
+  const mockFunction = jest.fn().mockResolvedValue([]);
+
+  function getCheckpointQuery() {
+    return createGetCheckpointsRange({
+      sourcesFn: () => ['a'],
+      keyFn: source => source,
+      querySourceFn: mockFunction
+    });
+  }
+
+  beforeEach(() => {
+    mockFunction.mockClear();
+  });
+
+  // Case 1:
+  // Cache exists and we are fetching the same range again
+  // This triggers no queryFn calls
+  it('exact cache match', async () => {
+    const getCheckpointsRange = getCheckpointQuery();
+
+    await getCheckpointsRange(10, 20);
+    expect(mockFunction).toHaveBeenCalledTimes(1);
+
+    mockFunction.mockClear();
+
+    await getCheckpointsRange(10, 20);
+    expect(mockFunction).toHaveBeenCalledTimes(0);
+  });
+
+  // Case 2:
+  // Cache exists but we are fetching blocks outside of cache range
+  // This triggers single queryFn call
+  test('cache outside of the range', async () => {
+    const getCheckpointsRange = getCheckpointQuery();
+
+    await getCheckpointsRange(10, 20);
+    expect(mockFunction).toHaveBeenCalledTimes(1);
+
+    // Case 2a: Cache exists but we are fetching blocks further than the cache
+    mockFunction.mockClear();
+
+    await getCheckpointsRange(21, 31);
+    expect(mockFunction).toHaveBeenCalledTimes(1);
+    expect(mockFunction).toHaveBeenCalledWith(21, 31, 'a');
+
+    // Case 2b: Cache exists but we are fetching blocks before the cache
+    mockFunction.mockClear();
+
+    await getCheckpointsRange(0, 9);
+    expect(mockFunction).toHaveBeenCalledTimes(1);
+    expect(mockFunction).toHaveBeenCalledWith(0, 9, 'a');
+  });
+
+  // Case 3:
+  // Part of the range is cached and part of the range is not cached
+  // This triggers two queryFn calls (one to fetch block before cache and one to fetch block after cache)
+  test('cache is fully inside the range', async () => {
+    const getCheckpointsRange = getCheckpointQuery();
+
+    await getCheckpointsRange(10, 20);
+    expect(mockFunction).toHaveBeenCalledTimes(1);
+
+    mockFunction.mockClear();
+
+    await getCheckpointsRange(5, 25);
+    expect(mockFunction).toHaveBeenCalledTimes(2);
+    expect(mockFunction).toHaveBeenCalledWith(5, 9, 'a');
+    expect(mockFunction).toHaveBeenCalledWith(21, 25, 'a');
+  });
+
+  // Case 4:
+  // Cache covers bottom part of the range
+  // This triggers single queryFn call to fetch the top part of the range
+  test('cache covers bottom part of range', async () => {
+    const getCheckpointsRange = getCheckpointQuery();
+
+    await getCheckpointsRange(10, 20);
+    expect(mockFunction).toHaveBeenCalledTimes(1);
+
+    mockFunction.mockClear();
+
+    await getCheckpointsRange(15, 25);
+    expect(mockFunction).toHaveBeenCalledTimes(1);
+    expect(mockFunction).toHaveBeenCalledWith(21, 25, 'a');
+  });
+
+  // Case 5:
+  // Cache covers top part of the range
+  // This triggers single queryFn call to fetch the bottom part of the range
+  test('cache covers top part of range', async () => {
+    const getCheckpointsRange = getCheckpointQuery();
+
+    await getCheckpointsRange(10, 20);
+    expect(mockFunction).toHaveBeenCalledTimes(1);
+
+    mockFunction.mockClear();
+
+    await getCheckpointsRange(0, 15);
+    expect(mockFunction).toHaveBeenCalledTimes(1);
+    expect(mockFunction).toHaveBeenCalledWith(0, 9, 'a');
+  });
+});