diff --git a/src/Deeptable.ts b/src/Deeptable.ts index d6f705d73..337058b05 100644 --- a/src/Deeptable.ts +++ b/src/Deeptable.ts @@ -134,6 +134,9 @@ export class Deeptable { this.root_tile = new Tile(defaultManifest, null, this); const preProcessRootTile = this.root_tile.preprocessRootTileInfo(); + // At instantiation, the deeptable isn't ready; only once this + // async stuff is done can the deeptable be used. + // TODO: Add an async static method as the preferred initialization method. this.promise = preProcessRootTile.then(async () => { const batch = await this.root_tile.get_arrow(null); const schema = batch.schema; diff --git a/src/aesthetics/Aesthetic.ts b/src/aesthetics/Aesthetic.ts index 9dd1ae15d..4c1ef3b49 100644 --- a/src/aesthetics/Aesthetic.ts +++ b/src/aesthetics/Aesthetic.ts @@ -1,10 +1,11 @@ import type { TextureSet } from './AestheticSet'; import { isConstantChannel } from '../typing'; -import { Type, Vector } from 'apache-arrow'; +import { Struct, Type, Vector } from 'apache-arrow'; import { StructRowProxy } from 'apache-arrow/row/struct'; import { isNumber } from 'lodash'; import type * as DS from '../types'; import { Scatterplot } from '../scatterplot'; +import { Some } from '../utilityFunctions'; /** * An Aesthetic bundles all operations in mapping from user dataspace to webGL based aesthetics. @@ -26,6 +27,7 @@ export abstract class Aesthetic< public abstract default_range: [Output['rangeType'], Output['rangeType']]; public scatterplot: Scatterplot; public field: string | null = null; + public subfield: string[] = []; public _texture_buffer: Float32Array | Uint8Array | null = null; protected abstract _func?: (d: Input['domainType']) => Output['rangeType']; public aesthetic_map: TextureSet; @@ -76,9 +78,25 @@ export abstract class Aesthetic< this.field = null; } else { this.field = encoding.field; + if (encoding.subfield) { + this.subfield = Array.isArray(encoding.subfield) + ? encoding.subfield + : [encoding.subfield]; + } } } + /** + * Returns the keys that are used to access the data in the record batch, + * including with any nesting. + */ + get columnKeys(): null | Some { + if (this.field === null) { + return null; + } + return [this.field, ...this.subfield] as Some; + } + get deeptable() { return this.scatterplot.deeptable; } @@ -100,10 +118,14 @@ export abstract class Aesthetic< value_for(point: Datum): Input['domainType'] | null { if (this.field && point[this.field]) { - return point[this.field] as Input['domainType']; + let v = point[this.field] as Input['domainType']; + for (let i = 0; i < this.subfield.length; i++) { + v = v[this.subfield[i]] as Input['domainType']; + } + return v; + // Needs a default perhaps? + return null; } - // Needs a default perhaps? - return null; } get map_position() { @@ -136,9 +158,17 @@ export abstract class Aesthetic< if (this.field === null || this.field === undefined) { return (this.column = null); } - return (this.column = this.deeptable.root_tile.record_batch.getChild( - this.field, - ) as Vector); + let output: Vector | Vector | null = null; + for (const f of [this.field, ...this.subfield]) { + if (output === null) { + output = this.deeptable.root_tile.record_batch.getChild(f) as Vector< + Input['arrowType'] + >; + } else { + output = (output as Vector).getChild(f) as Vector; + } + } + return (this.column = output as Vector); } is_dictionary(): boolean { diff --git a/src/aesthetics/AestheticSet.ts b/src/aesthetics/AestheticSet.ts index 04daf719d..20273a172 100644 --- a/src/aesthetics/AestheticSet.ts +++ b/src/aesthetics/AestheticSet.ts @@ -6,6 +6,7 @@ import type { Deeptable } from '../Deeptable'; import { StatefulAesthetic } from './StatefulAesthetic'; import type { Encoding } from '../types'; import type * as DS from '../types'; +import { TupleSet } from '../utilityFunctions'; type AesMap = { [K in keyof typeof dimensions]: StatefulAesthetic< @@ -83,6 +84,12 @@ export class AestheticSet { } } + _neededFields: TupleSet = new TupleSet(); + + get neededFields(): string[][] { + return [...this._neededFields.values()]; + } + apply_encoding(encoding: Encoding) { if ( encoding['jitter_radius'] && @@ -107,6 +114,17 @@ export class AestheticSet { this.dim(k).update(encoding[k] as DS.ChannelType | null); } + // Update the needed fields. + this._neededFields.clear(); + + for (const v of Object.values(this.store)) { + if (v instanceof StatefulAesthetic) { + for (const f of v.neededFields) { + this._neededFields.add(f); + } + } + } + // Apply settings that are not full-on aesthetics. for (const setting of ['jitter_method'] as const) { this.options[setting].last = this.options[setting].current; diff --git a/src/aesthetics/StatefulAesthetic.ts b/src/aesthetics/StatefulAesthetic.ts index c631acb32..c1138965c 100644 --- a/src/aesthetics/StatefulAesthetic.ts +++ b/src/aesthetics/StatefulAesthetic.ts @@ -55,6 +55,7 @@ export type ConcreteScaledAesthetic = import type { Deeptable } from '../Deeptable'; import type { Regl } from 'regl'; import type { TextureSet } from './AestheticSet'; +import { Some } from '../utilityFunctions'; export class StatefulAesthetic { /** @@ -97,11 +98,12 @@ export class StatefulAesthetic { ] as [T, T]; } - get neededFields(): string[] { - return [this.current.field, this.last.field].filter( + get neededFields(): Some[] { + return [this.current.columnKeys, this.last.columnKeys].filter( (f) => f !== null, - ) as string[]; + ); } + get current() { return this.states[0]; } diff --git a/src/regl_rendering.ts b/src/regl_rendering.ts index 10f75f8c9..db2807881 100644 --- a/src/regl_rendering.ts +++ b/src/regl_rendering.ts @@ -32,6 +32,7 @@ import { Scatterplot } from './scatterplot'; import { Data, Dictionary, + Struct, StructRowProxy, Type, Utf8, @@ -41,8 +42,7 @@ import { Color } from './aesthetics/ColorAesthetic'; import { StatefulAesthetic } from './aesthetics/StatefulAesthetic'; import { Filter, Foreground } from './aesthetics/BooleanAesthetic'; import { ZoomTransform } from 'd3-zoom'; -import { TupleMap } from './utilityFunctions'; -import { buffer } from 'd3'; +import { Some, TupleMap, TupleSet } from './utilityFunctions'; // eslint-disable-next-line import/prefer-default-export export class ReglRenderer extends Renderer { public regl: Regl; @@ -66,6 +66,11 @@ export class ReglRenderer extends Renderer { public tick_num?: number; public reglframe?: REGL.Cancellable; public bufferManager: BufferManager; + + private aes_to_buffer_num?: Record; + private variable_to_buffer_num?: TupleMap; + private buffer_num_to_variable?: string[][]; + // public _renderer : Renderer; constructor( @@ -160,7 +165,7 @@ export class ReglRenderer extends Renderer { ] as [number, number]; const props: DS.GlobalDrawProps = { // Copy the aesthetic as a string. - aes: { encoding: this.aes.encoding }, + // aes: { encoding: this.aes.encoding }, colors_as_grid: 0, corners: this.zoom.current_corners(), zoom_balance: prefs.zoom_balance, @@ -225,12 +230,7 @@ export class ReglRenderer extends Renderer { props.background_draw_needed[0] || props.background_draw_needed[1]; for (const tile of this.visible_tiles()) { // Do the binding operation; returns truthy if it's already done. - if ( - !this.bufferManager.ready( - tile, - this.needeedFields.map((d) => [d]), - ) - ) { + if (!this.bufferManager.ready(tile, this.aes.neededFields)) { continue; } @@ -260,6 +260,7 @@ export class ReglRenderer extends Renderer { a.tile_id ); }); + // console.log({ prop_list }); this._renderer(prop_list); } @@ -283,7 +284,7 @@ export class ReglRenderer extends Renderer { this.zoom.current_corners(), this.props.max_ix, 5, - this.needeedFields, + this.aes.neededFields.map((x) => x[0]), 'high', ); } else { @@ -292,7 +293,7 @@ export class ReglRenderer extends Renderer { undefined, prefs.max_points, 5, - this.needeedFields, + this.aes.neededFields.map((x) => x[0]), 'high', ); } @@ -887,7 +888,7 @@ export class ReglRenderer extends Renderer { type BufferSummary = { aesthetic: keyof typeof dimensions; time: time; - field: string; + field: [string, ...string[]]; }; const buffers: BufferSummary[] = []; const priorities = [ @@ -911,7 +912,10 @@ export class ReglRenderer extends Renderer { buffers.push({ aesthetic, time, - field: this.aes.dim(aesthetic)[time].field, + field: [ + this.aes.dim(aesthetic)[time].field, + ...this.aes.dim(aesthetic)[time].subfield, + ], }); } } catch (error) { @@ -936,20 +940,20 @@ export class ReglRenderer extends Renderer { const aes_to_buffer_num: Record = {}; // eg 'x' => 3 // Pre-allocate the 'ix' buffer and the 'ix_in_tile' buffers. - const variable_to_buffer_num: Record = { - ix: 0, - ix_in_tile: 1, - }; // eg 'year' => 3 + const variable_to_buffer_num: TupleMap = new TupleMap([ + [['ix'], 0], + [['ix_in_tile'], 1], + ]); // eg 'year' => 3 let num = 1; for (const { aesthetic, time, field } of buffers) { const k = `${aesthetic}--${time}`; - if (variable_to_buffer_num[field] !== undefined) { - aes_to_buffer_num[k] = variable_to_buffer_num[field]; + if (variable_to_buffer_num.get(field) !== undefined) { + aes_to_buffer_num[k] = variable_to_buffer_num.get(field); continue; } if (num++ < 16) { aes_to_buffer_num[k] = num; - variable_to_buffer_num[field] = num; + variable_to_buffer_num.set(field, num); continue; } else { // Don't use the last value, use the current value. @@ -960,18 +964,12 @@ export class ReglRenderer extends Renderer { } } - const buffer_num_to_variable = [ - ...Object.keys(variable_to_buffer_num).map((k) => [k]), - ]; + const buffer_num_to_variable = [...variable_to_buffer_num.keys()]; this.aes_to_buffer_num = aes_to_buffer_num; this.variable_to_buffer_num = variable_to_buffer_num; this.buffer_num_to_variable = buffer_num_to_variable; } - aes_to_buffer_num?: Record; - variable_to_buffer_num?: Record; - buffer_num_to_variable?: string[][]; - get discard_share() { // If jitter is temporal, e.g., or filters are in place, // it may make sense to estimate the number of hidden points. @@ -1030,7 +1028,7 @@ export class BufferManager { return this._integer_buffer; } - get(k: (string | Tile)[]): DS.BufferLocation | null { + get(k: Some): DS.BufferLocation | null { const a = this.bufferMap.get(this.arrayMap.get(k)); return a; } @@ -1042,17 +1040,22 @@ export class BufferManager { */ ready(tile: Tile, needed_dimensions: Iterable): boolean { // We don't allocate buffers for dimensions until they're needed. + for (const keyset of [['ix'], ['ix_in_tile'], ...needed_dimensions] as Some< + string[] + >) { + const current = this.get([tile, ...keyset]); - for (const key of [['ix'], ['ix_in_tile'], ...needed_dimensions]) { - const current = this.get([tile, ...key]); if (current === null || current === undefined) { - if (tile.hasLoadedColumn(key[0])) { - this.create_regl_buffer(tile, key); + if (tile.hasLoadedColumn(keyset[0])) { + this.create_regl_buffer(tile, keyset); } else { - // console.log('not ready because of', key); - if (key[0] === 'ix_in_tile') { - this.create_regl_buffer(tile, key); + if (keyset[0] === 'ix_in_tile') { + this.create_regl_buffer(tile, keyset); } else { + if (tile.readyToUse) { + // tile.get_column(keyset[0]); + } else { + } return false; } } @@ -1146,11 +1149,10 @@ export class BufferManager { create_regl_buffer(tile: Tile, keys: string[]): void { const { renderer } = this; - const key = [tile, ...keys]; + const key = [tile, ...keys] as Some; if (this.arrayMap.has(key)) { return; } - // console.log({ keys }); if (keys[0] === 'ix_in_tile') { this.arrayMap.set(key, this.integer_array); if (!this.bufferMap.has(this.integer_array)) { @@ -1171,6 +1173,7 @@ export class BufferManager { const data_length = data.length; const buffer_desc = renderer.buffers.allocate_block(data_length, item_size); + buffer_desc.buffer.subdata(data, buffer_desc.offset); this.bufferMap.set(vector.data[0].values, buffer_desc); @@ -1204,9 +1207,8 @@ function getNestedVector( } } - let column: Vector = tile.record_batch.getChild( - key[0], - ); + let column: Vector | Vector = + tile.record_batch.getChild(key[0]); for (const k of key.slice(1)) { column = column.getChild(k); } @@ -1217,7 +1219,17 @@ function getNestedVector( if (!column.type || !column.type.typeId) { throw new Error(`Column ${key} has no type.`); } + function assertNotStruct( + value: Vector | Vector, + ): asserts value is Vector { + if (column.type.typeId === Type.Struct) { + throw new Error( + 'Structs are not supported for buffer data on column ' + key.join('->'), + ); + } + } + assertNotStruct(column); return column; } @@ -1335,16 +1347,24 @@ class MultipurposeBufferSet { * @param prefs The preferences object to be used. * * @returns The fields that need to be allocated in the buffers for - * a tile to be drawn. + * a tile to be drawn. Returns a map of columns and any subfields in them that are needed. */ -export function neededFieldsToPlot(prefs: DS.CompletePrefs): Set { - const needed_keys: Set = new Set(); - if (!prefs.encoding) { +export function neededFieldsToPlot( + encoding: DS.Encoding | undefined, +): TupleSet { + const needed_keys = new TupleSet([['ix']]); + if (!encoding) { return needed_keys; } - for (const [_, v] of Object.entries(prefs.encoding)) { + for (const [_, v] of Object.entries(encoding)) { if (v && typeof v !== 'string' && v['field'] !== undefined) { - needed_keys.add(v['field'] as string); + const needed_key: Some = [v['field']]; + if (v['subfield'] !== undefined) { + const subfield = v['subfield']; + const asArray = Array.isArray(subfield) ? [...subfield] : [subfield]; + needed_key.push(...asArray); + } + needed_keys.add(needed_key); } } return needed_keys; diff --git a/src/rendering.ts b/src/rendering.ts index 39674ab61..10a0262c2 100644 --- a/src/rendering.ts +++ b/src/rendering.ts @@ -168,21 +168,6 @@ export class Renderer { return this.render_props.alpha; } - get needeedFields(): string[] { - const { aes } = this; - const needed = new Set(); - if (aes) { - for (const v of Object.values(aes.store)) { - if (v instanceof StatefulAesthetic) { - for (const f of v.neededFields) { - needed.add(f); - } - } - } - } - return [...needed, 'ix']; - } - get optimal_alpha() { // This extends a formula suggested by Ricky Reusser to include // discard share. diff --git a/src/scatterplot.ts b/src/scatterplot.ts index f3698dde5..ae9bc294f 100644 --- a/src/scatterplot.ts +++ b/src/scatterplot.ts @@ -121,6 +121,7 @@ export class Scatterplot { this.ready = new Promise((resolve) => { this.mark_ready = resolve; }); + this.click_handler = new ClickFunction(this); this.tooltip_handler = new TooltipHTML(this); this.label_click_handler = new LabelClick(this); @@ -389,7 +390,7 @@ export class Scatterplot { async reinitialize() { const { prefs } = this; - await this.deeptable.ready; + await this.deeptable.promise; await this.deeptable.root_tile.get_column('x'); this._renderer = new ReglRenderer( '#container-for-webgl-canvas', @@ -417,8 +418,11 @@ export class Scatterplot { ctx.fillRect(0, 0, window.innerWidth * 2, window.innerHeight * 2); void this._renderer.initialize(); - void this.deeptable.promise.then(() => this.mark_ready()); - return this.ready; + await this.deeptable.promise.then(() => { + this.mark_ready(); + }); + this.mark_ready(); + return; } /* @@ -632,12 +636,29 @@ export class Scatterplot { } await this.plot_queue; + // Ensure that the deeptable exists. + if (this._root === undefined) { + const { source_url, arrow_table, arrow_buffer } = + prefs as DS.InitialAPICall; + const dataSpec = { source_url, arrow_table, arrow_buffer } as DS.DataSpec; + if (Object.values(dataSpec).filter((x) => x !== undefined).length !== 1) { + throw new Error( + 'The initial API call specify exactly one of source_url, arrow_table, or arrow_buffer', + ); + } + await this.load_deeptable(dataSpec); + } + this.update_prefs(prefs); + // Then ensure the renderer and interaction handlers exist. + if (this._zoom === undefined || this._renderer === undefined) { + await this.reinitialize(); + } if (prefs) { this.start_transformations(prefs); } - this.plot_queue = this.unsafe_plotAPI(prefs); - await this.plot_queue; + this.plot_queue = this.unsafe_plotAPI(prefs); + console.log('C'); // eslint-disable-next-line @typescript-eslint/no-unused-vars for (const [_, hook] of Object.entries(this.hooks)) { hook(); @@ -666,23 +687,23 @@ export class Scatterplot { if (this.prefs.duration < delay) { delay = this.prefs.duration; } - const needed_keys: Set = neededFieldsToPlot(this.prefs); if (!prefs.encoding) { resolve(); } - if (this._renderer) { - this.deeptable.root_tile.require_columns(needed_keys); - // Immediately start loading what we can onto the GPUs, too. - for (const tile of this.renderer.visible_tiles()) { - this._renderer.bufferManager.ready( - tile, - [...needed_keys].map((k) => [k]), - ); - } - resolve(); - } else { - resolve(); + if (!this._renderer) { + throw new Error('No renderer has been initialized'); + } + // + const needed_keys = neededFieldsToPlot(prefs.encoding); + this.deeptable.root_tile.require_columns( + [...needed_keys].map((k) => k[0]), + ); + // Immediately start loading what we can onto the GPUs, too. + for (const tile of this.renderer.visible_tiles()) { + this._renderer.bufferManager.ready(tile, needed_keys); } + // TODO: There should be a setTimeout here before the resolution + resolve(); }); } /** @@ -724,18 +745,6 @@ export class Scatterplot { this.update_prefs(prefs); - if (this._root === undefined) { - const { source_url, arrow_table, arrow_buffer } = - prefs as DS.InitialAPICall; - const dataSpec = { source_url, arrow_table, arrow_buffer } as DS.DataSpec; - if (Object.values(dataSpec).filter((x) => x !== undefined).length !== 1) { - throw new Error( - 'The initial API call specify exactly one of source_url, arrow_table, or arrow_buffer', - ); - } - await this.load_deeptable(dataSpec); - } - if (prefs.transformations) { for (const [k, func] of Object.entries(prefs.transformations)) { if (!this.deeptable.transformations[k]) { @@ -743,9 +752,7 @@ export class Scatterplot { } } } - if (this._zoom === undefined || this._renderer === undefined) { - await this.reinitialize(); - } + const renderer = this._renderer; const zoom = this._zoom; diff --git a/src/selection.ts b/src/selection.ts index 020b63dbc..1a7505c97 100644 --- a/src/selection.ts +++ b/src/selection.ts @@ -817,8 +817,8 @@ export class DataSelection { if (i === undefined) { i = this.cursor; } - if (i > this.selectionSize) { - undefined; + if (i >= this.selectionSize) { + throw new Error('Index out of bounds'); } let currentOffset = 0; let relevantTile: Tile | undefined = undefined; @@ -885,10 +885,14 @@ export class DataSelection { } } -function bigintmatcher(field: string, matches: bigint[]) { +function bigintmatcher( + field: string, + matches: bigint[], + subfield: string | string[] | null = null, +) { const matchings = new Set(matches); return async function (tile: Tile) { - const col = (await tile.get_column(field)).data[0]; + const col = (await tile.get_column(field, subfield)).data[0]; const values = col.values as bigint[]; const bitmask = new Bitmask(tile.record_batch.numRows); for (let i = 0; i < tile.record_batch.numRows; i++) { @@ -920,7 +924,11 @@ function bigintmatcher(field: string, matches: bigint[]) { * @param matches A list of strings to match in that column * @returns */ -function stringmatcher(field: string, matches: string[]) { +function stringmatcher( + field: string, + matches: string[], + subfield: string | string[] | null = null, +) { if (field === undefined) { throw new Error('Field must be defined'); } @@ -958,7 +966,8 @@ function stringmatcher(field: string, matches: string[]) { * The Deepscatter transformation function. */ return async function (tile: Tile) { - const col = ((await tile.get_column(field)) as Vector).data[0]; + const col = ((await tile.get_column(field, subfield)) as Vector) + .data[0]; const bytes = col.values; const offsets = col.valueOffsets; diff --git a/src/tile.ts b/src/tile.ts index 36b187d03..95e4c572d 100644 --- a/src/tile.ts +++ b/src/tile.ts @@ -64,6 +64,9 @@ export class Tile { private _partialManifest: Partial | Partial; private _manifest?: TileManifest | LazyTileManifest; + // Does the tile have a loaded manifest and other features sufficient to plot. + public readyToUse = false; + // A cache of fetchCalls for downloaded arrow tables, including any table schema metadata. // Tables may contain more than a single column, so this prevents multiple dispatch. //private _promiseOfChildren: Promise; @@ -134,22 +137,38 @@ export class Tile { * * * @param colname The name of the column to retrive. + * @param subfield If the column is a struct vector, the subfield to retrieve. When a string, retrieves a single + * subfield. When an array, treats it as a nesting order. * @returns An Arrow Vector of the column. */ - async get_column(colname: string): Promise { - const existing = this._batch?.getChild(colname); - if (existing) { - return existing; + async get_column( + colname: string, + subfield: string | string[] | null = null, + ): Promise { + const subfields = + subfield === null ? [] : Array.isArray(subfield) ? subfield : [subfield]; + let existing = this._batch?.getChild(colname); + + if (!existing) { + if (this.deeptable.transformations[colname]) { + await this.apply_transformation(colname); + existing = this.record_batch.getChild(colname); + if (existing === null) { + throw new Error(`Column ${colname} not found after transformation`); + } + } } - if (this.deeptable.transformations[colname]) { - await this.apply_transformation(colname); - const vector = this.record_batch.getChild(colname); - if (vector === null) { - throw new Error(`Column ${colname} not found after transformation`); + + // If subfields are passed, use them. + for (let i = 0; i < subfields.length; i++) { + existing = existing.getChild(subfields[i]); + if (existing === null) { + throw new Error( + `Column ${colname} lacks subfield ${subfields.slice(0, i).join(' >> ')}`, + ); } - return vector; } - throw new Error(`Column ${colname} not found`); + return existing; } /** @@ -286,6 +305,7 @@ export class Tile { }); this.highest_known_ix = manifest.max_ix; this._manifest = manifest; + this.readyToUse = true; } set highest_known_ix(val) { diff --git a/src/tixrixqid.ts b/src/tixrixqid.ts index 1522b6343..402a160fd 100644 --- a/src/tixrixqid.ts +++ b/src/tixrixqid.ts @@ -1,4 +1,11 @@ -import type { Bool, Data, Field, Struct, StructRowProxy, Vector } from 'apache-arrow'; +import type { + Bool, + Data, + Field, + Struct, + StructRowProxy, + Vector, +} from 'apache-arrow'; import type { Tile } from './deepscatter'; import { Bitmask, DataSelection, Deeptable } from './deepscatter'; @@ -102,7 +109,7 @@ export function tixToZxy(tix: Tix): [number, number, number] { */ export function getQidFromRow( row: StructRowProxy, - dataset: Deeptable + dataset: Deeptable, ): [number, number] { const tile = getTileFromRow(row, dataset); const rix = row[Symbol.for('rowIndex')] as number; @@ -110,7 +117,6 @@ export function getQidFromRow( } export function getTileFromRow(row: StructRowProxy, dataset: Deeptable): Tile { - const parent = row[Symbol.for('parent')] as Data; const parentsColumns = parent.children; @@ -119,8 +125,8 @@ export function getTileFromRow(row: StructRowProxy, dataset: Deeptable): Tile { // need to find the tile that matches the most columns, not assume // that every column matches exactly. let best_match: [Tile | null, number] = [null, 0]; - const parentNames : [string, Data][] = parent.type.children.map( - (d: Field, i: number) => [d.name, parentsColumns[i]] + const parentNames: [string, Data][] = parent.type.children.map( + (d: Field, i: number) => [d.name, parentsColumns[i]], ); dataset.map((t: Tile) => { @@ -144,7 +150,7 @@ export function getTileFromRow(row: StructRowProxy, dataset: Deeptable): Tile { }); if (best_match[0] === undefined) { throw new Error( - 'No tiles found for this row.' + JSON.stringify({ ...row }) + 'No tiles found for this row.' + JSON.stringify({ ...row }), ); } return best_match[0]; diff --git a/src/types.ts b/src/types.ts index 5bcd4ca6a..a173d0811 100644 --- a/src/types.ts +++ b/src/types.ts @@ -21,6 +21,7 @@ import { Scatterplot } from './scatterplot'; import { ZoomTransform } from 'd3-zoom'; import type { Tile } from './tile'; import type { Rectangle } from './tile'; +import { TupleMap } from './utilityFunctions'; export type { Renderer, Deeptable, ConcreteAesthetic }; /** @@ -294,6 +295,8 @@ export type NumericScaleChannel< > = { /** The name of a column in the data table to be encoded. */ field: string; + // If field is a struct, subfield indicates which child to extract. + subfield?: string | string[]; /** * A transformation to apply on the field. * 'literal' maps in the implied dataspace set by 'x', 'y', while @@ -312,6 +315,8 @@ export type LambdaChannel< > = { lambda?: (v: DomainType) => RangeType; field: string; + // If field is a struct, subfield indicates which child to extract. + subfield?: string | string[]; }; /** @@ -342,6 +347,8 @@ type TwoArgumentOp = { export type OpChannel = { field: string; + // If field is a struct, subfield indicates which child to extract. + subfield?: string | string[]; } & (OneArgumentOp | TwoArgumentOp); export type ConstantChannel = { @@ -376,12 +383,16 @@ export type ChannelType = export type CategoricalColorScale = { field: string; + // If field is a struct, subfield indicates which child to extract. + subfield?: string | string[]; domain: string | [string, string, ...string[]]; range: Colorname[]; }; export type LinearColorScale = { field: string; + // If field is a struct, subfield indicates which child to extract. + subfield?: string | string[]; domain: [number, number]; // TODO: | [number, number, number] // TODO: implement some codegen for these values range: 'viridis' | 'magma' | 'ylorrd'; @@ -605,7 +616,7 @@ export type RowFunction = ( // Props that are needed for all the draws in a single tick. export type GlobalDrawProps = { - aes: { encoding: Encoding }; + // aes: { encoding: Encoding }; colors_as_grid: 0 | 1; corners: Rectangle; zoom_balance: number; @@ -625,9 +636,9 @@ export type GlobalDrawProps = { last_webgl_scale: number[]; use_scale_for_tiles: boolean; grid_mode: 1 | 0; - buffer_num_to_variable: string[]; + buffer_num_to_variable: string[][]; aes_to_buffer_num: Record; - variable_to_buffer_num: Record; + variable_to_buffer_num: TupleMap; color_picker_mode: 0 | 1 | 2 | 3; zoom_matrix: [ number, diff --git a/src/utilityFunctions.ts b/src/utilityFunctions.ts index 3d82f9215..5e9a4bfb6 100644 --- a/src/utilityFunctions.ts +++ b/src/utilityFunctions.ts @@ -91,11 +91,23 @@ function createDictionaryWithVector( return returnval; } +// An array, but with a guarantee there is at least element. +export type Some = [T, ...T[]]; + +/** + * A Map that allows for tuples as keys with proper identity checks. + */ export class TupleMap { private map: Map> = new Map(); private value?: V; - set(keys: K[], value: V): void { + constructor(v: [Some, V][] = []) { + for (const [keys, value] of v) { + this.set(keys, value); + } + } + + set(keys: Some, value: V): void { let currentMap: TupleMap = this; for (const key of keys) { if (!currentMap.map.has(key)) { @@ -106,7 +118,30 @@ export class TupleMap { currentMap.value = value; } - get(keys: K[]): V | undefined { + *entries(): IterableIterator<[Some, V]> { + for (const [key, map] of this.map) { + for (const [keys, value] of map.entries()) { + yield [[key, ...keys], value]; + } + } + if (this.value !== undefined) { + // If this map has a value, yield it with an empty key array + // @ts-expect-error - the empty array gets around the type check + // but is OK because this only happens BELOW the first level; + // the first level is not allowed to have a value but I don't feel + // like explicitly classing the first level as a different type from + // the rest. + yield [[], this.value]; + } + } + + *keys(): IterableIterator> { + for (const [k] of this.entries()) { + yield k; + } + } + + get(keys: Some): V | undefined { let currentMap: TupleMap = this; for (const key of keys) { currentMap = currentMap.map.get(key) as TupleMap; @@ -117,7 +152,7 @@ export class TupleMap { return currentMap.value; } - has(keys: K[]): boolean { + has(keys: Some): boolean { let currentMap: TupleMap = this; for (const key of keys) { currentMap = currentMap.map.get(key) as TupleMap; @@ -128,7 +163,7 @@ export class TupleMap { return currentMap.value !== undefined; } - delete(keys: K[]): boolean { + delete(keys: Some): boolean { let currentMap: TupleMap = this; const stack: TupleMap[] = []; @@ -157,7 +192,42 @@ export class TupleMap { } } -// // finds the first set bit. -// function ffs(n: number): number { -// return Math.log2(n & -n); -// } +export class TupleSet { + private map = new TupleMap(); + + constructor(v: Some[] = []) { + for (const keys of v) { + this.add(keys); + } + } + + add(keys: Some): void { + this.map.set(keys, true); + } + + has(keys: Some): boolean { + return this.map.has(keys); + } + + delete(keys: Some): boolean { + return this.map.delete(keys); + } + + *values(): IterableIterator> { + for (const keys of this.map.keys()) { + yield keys; + } + } + + [Symbol.iterator](): IterableIterator> { + return this.values(); + } + + get size(): number { + return [...this.values()].length; + } + + clear(): void { + this.map = new TupleMap(); + } +}