vhad/partition-filter-visualization
1
0
Fork 0
mirror of https://github.com/gosticks/partition-filter-visualization.git synced 2026-08-11 20:30:23 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

feat: refactor rendering flow & add rendering options

Browse Source
This commit is contained in:
Wlad Meixner
2023-12-04 17:04:29 +01:00
parent 1ccc18bf0a
commit 7df4cc27f9
17 changed files with 805 additions and 575 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/components/graph/Minimal.svelte
+1 -2
View File
@@ -1,9 +1,8 @@
<script lang="ts">
import { getContext, onDestroy, onMount } from 'svelte';
import type { GraphService } from './types';
import { Minimap as MinimapRenderer } from '$lib/rendering/Minimap';
import { browser } from '$app/environment';
import { getGraphContext, type CameraState } from '../BasicGraph.svelte';
import { getGraphContext, type CameraState, type GraphService } from '../BasicGraph.svelte';
const graphService: GraphService = getGraphContext();
src/lib/components/graph/PlaneGraph.svelte
+5 -6
View File
@@ -8,7 +8,7 @@
type IPlaneData
} from '$lib/rendering/PlaneRenderer';
import Card from '../Card.svelte';
import type { PlaneGraphOptions } from '$lib/store/filterStore/graphs/plane';
import type { PlaneGraphModel } from '$lib/store/filterStore/graphs/plane';
import { writable, type Unsubscriber } from 'svelte/store';
import { dataStore as dbStore } from '$lib/store/dataStore/DataStore';
import Button from '../button/Button.svelte';
@@ -25,7 +25,7 @@
import { CopyIcon, LayersIcon, LockIcon } from 'svelte-feather-icons';
import notificationStore from '$lib/store/notificationStore';
export let options: PlaneGraphOptions;
export let options: PlaneGraphModel;
const graphService: GraphService = getGraphContext();
@@ -57,10 +57,10 @@
graphService.registerOnBeforeRender(dataRenderer.onBeforeRender.bind(dataRenderer));
};
const updateWithData = (data?: IPlaneRendererData) => {
const update = (data?: IPlaneRendererData) => {
if (!data || !dataRenderer) return;
dataRenderer.setAxisLabelRenderer(labelForAxis);
dataRenderer.updateWithData(data);
dataRenderer.update(data, options.renderSettings);
layerVisibility = dataRenderer.getLayerVisibility();
};
@@ -114,7 +114,7 @@
threeDomContainer.addEventListener('mousemove', onMouseMove);
let dataUnsub = options.dataStore.subscribe(updateWithData);
let dataUnsub = options.dataStore.subscribe(update);
unsubscriber = () => {
dataUnsub();
@@ -142,7 +142,6 @@
const range = $dataStore!.ranges[axis];
if (Axis.Y == axis && range) {
console.log({ range, segment });
return ((range[1] / numSegments) * segment).toFixed(2);
}
src/lib/components/graph/SliceGraph.svelte
+2 -3
View File
@@ -6,7 +6,7 @@
PlaneRenderer
} from '$lib/rendering/PlaneRenderer';
import Slider, { type SliderInputEvent } from '../slider/Slider.svelte';
import type { PlaneGraphOptions } from '$lib/store/filterStore/graphs/plane';
import type { PlaneGraphModel } from '$lib/store/filterStore/graphs/plane';
import SliceSelection from './SliceSelection.svelte';
import Dropdown, { getDropdownCtx } from '../Dropdown.svelte';
import Button from '../button/Button.svelte';
@@ -27,13 +27,12 @@
import type { LayerVisibilityList } from '../layerLegend/LayerGroup.svelte';
import { Axis } from '$lib/rendering/AxisRenderer';
import Dialog, { DialogSize } from '../dialog/Dialog.svelte';
import type { ITableReference } from '$lib/store/filterStore/types';
import type { ITiledDataRow } from '$lib/store/dataStore/filterActions';
import DropdownSelect, { type DropdownSelectionEvent } from '../DropdownSelect.svelte';
import BasicGraph from '../BasicGraph.svelte';
import { DataScaling } from '$lib/store/dataStore/types';
export let options: PlaneGraphOptions;
export let options: PlaneGraphModel;
export let layerVisibility: LayerVisibilityList;
export let axis: Axis = Axis.X;
// If enabled adds an expand button that rerenders the slice graph into a dialog
src/lib/rendering/BarRenderer.ts
+1 -1
View File
@@ -72,7 +72,7 @@ export class BarRenderer extends GraphRenderer<BarData> {
return raycaster.intersectObjects(this.bars, true);
}
updateWithData(data: BarData) {
update(data: BarData) {
if (this.barGroup) {
this.scene?.remove(this.barGroup);
}
src/lib/rendering/GraphRenderer.ts
+1 -1
View File
@@ -27,7 +27,7 @@ export abstract class GraphRenderer<T = unknown, InstanceMetaInfo = any> extends
* Used to update rendering based on data changes
* @param data
*/
abstract updateWithData(data: T, colorPalette?: THREE.ColorRepresentation[]): void;
abstract update(data: T, options: any, colorPalette?: THREE.ColorRepresentation[]): void;
abstract getInfoAtPoint(glPoint: Vector2): InstanceMetaInfo | undefined;
}
src/lib/rendering/PlaneRenderer.ts
+85 -103
View File
@@ -9,6 +9,7 @@ import { Theme } from '$lib/store/SettingsStore';
import { SparsePlaneGeometry, type Point3D } from './geometry/SparsePlaneGeometry';
import { SelectablePointCloud } from './geometry/PointCloudGeometry';
import { identity } from './geometry/transformers';
import { DensePlaneGeometry } from './geometry/DensePlaneGeometry';
export interface IPlaneData {
points: Point3D[];
@@ -48,15 +49,33 @@ export interface IPlaneRendererData {
};
}
export enum PlaneTriangulation {
grid = "grid",
delaunay = "delaunay"
};
export type IPlaneRenderOptions = {
triangulation: PlaneTriangulation,
showSelection: boolean,
}
export interface IPlaneSelection {
dataIndex: number;
layer: IPlaneData;
parent?: IPlaneData;
point: [number, number, number]
}
export class PlaneRenderer extends GraphRenderer<IPlaneRendererData, IPlaneSelection> {
public data?: IPlaneRendererData;
public static defaultRenderOptions():IPlaneRenderOptions {
return {
triangulation: PlaneTriangulation.delaunay,
showSelection: true,
}
}
private colorPalette: THREE.ColorRepresentation[] = []
private grids?: THREE.Group;
private dataDepth = 0;
private dataWidth = 0;
@@ -89,7 +108,7 @@ export class PlaneRenderer extends GraphRenderer<IPlaneRendererData, IPlaneSelec
private raycaster = new THREE.Raycaster();
private axisRenderer?: AxisRenderer;
constructor() {
constructor(private options: IPlaneRenderOptions = PlaneRenderer.defaultRenderOptions()) {
super();
console.log('Setup complete');
this.raycaster.layers.set(INTERSECTION_CHECK_LAYER);
@@ -117,12 +136,9 @@ export class PlaneRenderer extends GraphRenderer<IPlaneRendererData, IPlaneSelec
renderer: THREE.WebGLRenderer,
scene: THREE.Scene,
camera: THREE.Camera,
geometry: THREE.BufferGeometry<THREE.NormalBufferAttributes>,
material: THREE.Material,
group: THREE.Group
) => {
// Update axis renderer
this.axisRenderer?.onBeforeRender(renderer, scene, camera, geometry, material, group);
this.axisRenderer?.onBeforeRender(renderer, scene, camera);
if (!this.grids) {
return;
@@ -280,6 +296,7 @@ export class PlaneRenderer extends GraphRenderer<IPlaneRendererData, IPlaneSelec
this.currentSelection = {
layer: dataLayer,
mesh,
point,
dataIndex: instanceId,
parent: meshChildIndex ? this.data?.layers[meshIndex] : undefined
};
@@ -301,27 +318,33 @@ export class PlaneRenderer extends GraphRenderer<IPlaneRendererData, IPlaneSelec
return this.currentSelection;
}
private planeGeometry(plane: IPlaneData) {
switch (this.options.triangulation) {
case 'grid':
return new DensePlaneGeometry(plane.points);
case 'delaunay':
return new SparsePlaneGeometry(plane.points);
}
}
private renderPlane(
planeData: IPlaneData,
index: number,
color: THREE.Color,
width: number,
height: number,
childIndex?: number
childIndex?: number,
color?: THREE.Color,
) {
const plane = planeData.points;
const geo = new DataPlaneShapeGeometry(plane, undefined, true);
const geoSparse = new SparsePlaneGeometry(plane);
const geo = this.planeGeometry(planeData);
const mat = new THREE.MeshLambertMaterial({
color: color,
opacity: 0.5,
color: this.colorForPlane(planeData, childIndex ?? index),
depthWrite: true,
// clipIntersection: true,
// clipShadows: true,
side: THREE.DoubleSide
});
const mesh = new THREE.Mesh(geoSparse, mat);
const mesh = new THREE.Mesh(geo, mat);
mesh.scale.multiply(new THREE.Vector3(1/width, 1, 1/ height))
// Add metadata to mesh
mesh.userData = { index, name: planeData.name, meta: planeData.meta, childIndex };
@@ -331,78 +354,6 @@ export class PlaneRenderer extends GraphRenderer<IPlaneRendererData, IPlaneSelec
return mesh;
}
// /**
// * Renders visible Dots on data points and render invisible hit area
// * @param layerGeometry
// * @param index
// * @param color
// * @param subIndex
// * @returns
// */
// private renderPlaneDots(
// points: Point3D,
// index: number,
// childIndex?: number,
// color: THREE.ColorRepresentation = 0xeeeeff
// ): THREE.Group {
// const group = new THREE.Group();
// const sphereSize = 0.008;
// const sphereGeo = new THREE.SphereGeometry(sphereSize);
// const hitSphereGeo = new THREE.SphereGeometry(sphereSize * 2);
// const sphereMat = new THREE.MeshPhongMaterial({
// color: color,
// depthWrite: false,
// transparent: true,
// opacity: 0.4
// });
// const hitSphereMat = new THREE.MeshBasicMaterial({ color: color, depthWrite: true });
// const dotMesh = new THREE.InstancedMesh(sphereGeo, sphereMat, points.length);
// const hitDotMesh = new THREE.InstancedMesh(
// hitSphereGeo,
// hitSphereMat,
// points.length
// );
// // Set position of each dot
// const matrix = new THREE.Matrix4();
// const transparent = new THREE.Color(0x00000000);
// const mainColor = new THREE.Color(0xffffff);
// const yAxisScaleFactor = this.yAxisNormalizationFactor;
// for (let i = 0; i < layerGeometry.pointsPerPlane; i++) {
// const idx = i * DataPlaneShapeGeometry.pointComponentSize;
// // Apply scale
// // matrix.scale(one);
// // if (pointBuffer[idx + 1] == 0) {
// // matrix.scale(zero);
// // }
// matrix.setPosition(
// pointBuffer[idx],
// pointBuffer[idx + 1] * yAxisScaleFactor,
// pointBuffer[idx + 2]
// );
// if (pointBuffer[idx + 1] == 0) {
// dotMesh.setColorAt(i, transparent);
// continue;
// } else {
// dotMesh.setColorAt(i, mainColor);
// }
// dotMesh.setMatrixAt(i, matrix);
// hitDotMesh.setMatrixAt(i, matrix);
// }
// hitDotMesh.visible = false;
// hitDotMesh.layers.set(INTERSECTION_CHECK_LAYER);
// hitDotMesh.userData = { index, childIndex };
// group.add(hitDotMesh, dotMesh);
// return group;
// }
setupSelection() {
const geo = new THREE.SphereGeometry(0.02);
const mat = new THREE.MeshBasicMaterial({
@@ -415,46 +366,77 @@ export class PlaneRenderer extends GraphRenderer<IPlaneRendererData, IPlaneSelec
this.add(this.selectionMesh);
}
updateWithData(
// Y axis min max over all layers and sublayers
get globalYAxisRange():[number, number] {
if (!this.data) {
return [0, 0]
}
let min = Infinity;
let max = -Infinity
for (const l of this.data!.layers) {
const [childMin, childMax] = l.layers?.reduce(([min,max], l) => [Math.min(min, l.min), Math.max(max, l.max)], [min,max]) ?? [min, max];
min = Math.min(l.min, childMin);
max = Math.max(l.max, childMax);
}
return [min, max]
}
renderSelectionPoints(
points: Point3D[],
color: THREE.ColorRepresentation = 0xeeeeff,
):SelectablePointCloud | null {
if (!this.data) {
return null
}
// compute x and z scales since we cannot use
// non uniform scaling -> affects circle proportions
const xScaler = (x: number) => x / this.data!.tileRange.x
const yScaler = (y: number) => y / this.max
const zScaler = (z: number) => z / this.data!.tileRange.z
const visibleRadius = Math.max(Math.min(this.data.tileRange.x / 4, 0.0002), 0.0001);
return new SelectablePointCloud(points, new THREE.Color(color), visibleRadius, 1/this.data.tileRange.x, xScaler,yScaler, zScaler);
}
colorForPlane(planeData: IPlaneData, index: number): THREE.Color {
return new THREE.Color(planeData.color ?? this.colorPalette[index % this.colorPalette.length]);
}
update(
data: IPlaneRendererData,
options: IPlaneRenderOptions,
colorPalette: THREE.ColorRepresentation[] = graphColors
) {
this.options = options;
// Validate data
if (!data.layers.length) {
console.warn('No data provided');
return;
}
this.cleanup();
this.colorPalette = colorPalette;
this.data = data;
let [globalMin, globalMax] = this.globalYAxisRange;
this.min = globalMin;
this.max = globalMax;
this.planeGroup = new THREE.Group();
this.setupSelection();
this.data = data;
let globalMin = Infinity;
let globalMax = -Infinity;
const meshes: ReturnType<PlaneRenderer['renderPlane']>[] = new Array(data.layers.length);
const childLayers: ReturnType<PlaneRenderer['renderPlane']>[][] = new Array(data.layers.length);
for (const [index, planeData] of data.layers.entries()) {
globalMax = Math.max(globalMax, planeData.max);
globalMin = Math.min(globalMin, planeData.min);
const color = new THREE.Color(planeData.color ?? colorPalette[index % colorPalette.length]);
meshes[index] = this.renderPlane(planeData, index,color, data.tileRange.x, data.tileRange.z, );
meshes[index] = this.renderPlane(planeData, index, data.tileRange.x, data.tileRange.z, );
childLayers[index] =
planeData.layers?.map((childData, childIndex) => {
const color = new THREE.Color(
childData.color ?? colorPalette[index % colorPalette.length]
);
return this.renderPlane(childData, index, color, data.tileRange.x, data.tileRange.z, childIndex);
return this.renderPlane(childData, index, data.tileRange.x, data.tileRange.z, childIndex);
}) ?? [];
}
this.min = globalMin;
this.max = globalMax;
const dataScaleFactor = 1 / globalMax;
let xScaleFactor = 1/data.tileRange.x;
let zScaleFactor = 1/data.tileRange.z;
src/lib/rendering/geometry/DensePlaneGeometry.ts
+78
View File
@@ -0,0 +1,78 @@
import * as THREE from "three"
import { identity } from "./transformers";
export type Point2D = [number, number];
export type Point3D = [number, number, number];
export class DensePlaneGeometry extends THREE.BufferGeometry {
static readonly pointComponentSize = 3;
private pointBuffer: Float32Array;
constructor(
values: [number, number, number][],
scaleX: (x: number) => number = identity,
scaleY: (y: number) => number = identity,
scaleZ: (z: number) => number = identity,
skipsEmptyCells: boolean = false
) {
super();
const [width, depth] = values.reduce(([w, d], [x,z,_]) => [Math.max(w,x), Math.max(d,z)], [0,0])
console.debug(width, depth);
// Transform rows into a 2D array for display
// const data: number[][] = new Array(width).fill(-1).map(() => new Array(depth).fill(-1));
this.pointBuffer = new Float32Array(width * depth * 3).fill(-1);
// set points in dense arr
values.forEach(([x,z,y]) => {
this.pointBuffer[(x + z * depth) * DensePlaneGeometry.pointComponentSize] = scaleX(x);
this.pointBuffer[(x + z * depth) * DensePlaneGeometry.pointComponentSize + 1] = scaleY(y);
this.pointBuffer[(x + z * depth) * DensePlaneGeometry.pointComponentSize + 2] = scaleZ(z);
});
// TODO: interpolate holes in data
// construct polygons
const numPolygons = (width - 1) * (depth - 1) * 2 * 3;
const triangleIndexBuffer = new Uint32Array(numPolygons);
for (let z = 0; z < depth; z++) {
for (let x = 0; x < width; x++) {
const pointIdx = z * width + x;
const vertexIdx = pointIdx * DensePlaneGeometry.pointComponentSize;
const indexIdx = (z * (width - 1) + x) * 6;
// console.log({x, z, pointIdx, vertexIdx, indexIdx});
// Add top plane coordinates
this.pointBuffer[vertexIdx] = x; //x
// this.pointBuffer[vertexIdx + 1] = 5;//normalizedData[z][x]; //y
this.pointBuffer[vertexIdx + 2] = z; //z
if (this.pointBuffer[vertexIdx + 1] == -1) {
if (skipsEmptyCells) {
continue;
}
this.pointBuffer[vertexIdx + 1] = 0;
}
triangleIndexBuffer[indexIdx] = pointIdx + width;
triangleIndexBuffer[indexIdx + 1] = pointIdx + 1;
triangleIndexBuffer[indexIdx + 2] = pointIdx;
triangleIndexBuffer[indexIdx + 3] = pointIdx + width;
triangleIndexBuffer[indexIdx + 4] = pointIdx + width + 1;
triangleIndexBuffer[indexIdx + 5] = pointIdx + 1;
}
}
this.setAttribute("position", new THREE.BufferAttribute(this.pointBuffer, DensePlaneGeometry.pointComponentSize, true));
// Create buffers for rendering
this.setIndex(new THREE.Uint32BufferAttribute(triangleIndexBuffer, 1));
this.computeVertexNormals();
}
}
src/lib/rendering/geometry/SparsePlaneGeometry.ts
+5 -1
View File
@@ -4,10 +4,13 @@ import { identity } from "./transformers";
export type Point2D = [number, number];
export type Point3D = [number, number, number];
// Geometry that only renders polygons between data points
export class SparsePlaneGeometry extends THREE.BufferGeometry {
static readonly pointComponentSize = 3;
private d: Delaunay<Point2D>;
private d: Delaunay<Point3D>;
private pointBuffer: Float32Array;
constructor(
@@ -15,6 +18,7 @@ export class SparsePlaneGeometry extends THREE.BufferGeometry {
scaleX: (x: number) => number = identity,
scaleY: (y: number) => number = identity,
scaleZ: (z: number) => number = identity,
) {
super();
src/lib/store/dataStore/filterActions.ts
+17 -14
View File
@@ -13,6 +13,15 @@ export interface ITiledDataRow {
name: string;
}
export type IQueryResult = {
// data: number[][];
points: Point3D[];
min: number;
max: number;
tiles: [number, number];
queryResult?: ITiledDataRow[];
}
export type MinValue = number;
export type MaxValue = number;
export type ValueRange = [MinValue, MaxValue];
@@ -169,11 +178,14 @@ export const dataStoreFilterExtension = (store: BaseStoreType) => {
try {
const resp = await store.executeQuery(queryV2);
if (!resp) {
// TODO: fix/handle this
return [];
}
return resp.toArray();
// TODO: move deduplication to DB for now simply do this in place
return resp.toArray().filter((val, index, arr) => index === 0 ? true : val["x"] != arr[index-1]["x"] || val["z"] != arr[index-1]["z"] )
} catch (e) {
console.error(e);
return [];
@@ -187,13 +199,7 @@ export const dataStoreFilterExtension = (store: BaseStoreType) => {
yRange?: ValueRange,
zRange?: ValueRange,
where?: { columnName: string; value: string }
): Promise<{
data: number[][];
points: Point3D[];
min: number;
max: number;
queryResult?: ITiledDataRow[];
}> => {
): Promise<IQueryResult> => {
const options = {
...defaultTiledDataOptions,
..._options
@@ -205,8 +211,6 @@ export const dataStoreFilterExtension = (store: BaseStoreType) => {
const zDim = (options.zTileCount ?? options.tileCount) + 1;
const xDim = (options.xTileCount ?? options.tileCount) + 1;
// Transform rows into a 2D array for display
const data: number[][] = new Array(xDim).fill(-1).map(() => new Array(zDim).fill(-1));
let min = Number.MAX_VALUE;
let max = Number.MIN_VALUE;
@@ -217,24 +221,23 @@ export const dataStoreFilterExtension = (store: BaseStoreType) => {
}
min = Math.min(min, r.y);
max = Math.max(max, r.y);
data[r.z][r.x] = r.y;
});
return {
data,
points,
min,
max,
tiles: [xDim, zDim],
queryResult: rows
};
} catch (e) {
console.error('Failed to create tiled data:', e);
return {
data: [],
points: [],
min: 0,
max: 0
max: 0,
tiles: [0, 0],
};
}
};
src/lib/store/dataStore/loadActions.ts
+25 -21
View File
@@ -1,15 +1,12 @@
import { get, type Writable } from 'svelte/store';
import type { BaseStoreType } from './DataStore';
import type { IDataStore, ITableEntry, TableSchema } from './types';
import type { IDataStore, ILoadedTable, ITableBuildIn, TableSchema } from './types';
import {
TableSource,
type ITableReference,
type ITableRefList,
type ITableExternalUrl,
type ITableExternalFile
} from '../filterStore/types';
} from './types';
import notificationStore from '../notificationStore';
import { flatGroup } from 'd3';
// Store extension containing actions to load data, transform & drop data
export const dataStoreLoadExtension = (store: BaseStoreType, dataStore: Writable<IDataStore>) => {
@@ -92,12 +89,11 @@ export const dataStoreLoadExtension = (store: BaseStoreType, dataStore: Writable
shouldSetLoading = true,
createTable = true,
shouldUpdateTableList = true
): Promise<ITableEntry | undefined> => {
): Promise<ILoadedTable | undefined> => {
const conn = await store.getConnection();
if (!conn) {
// TODO: add error handling
return;
throw new Error("no database connection")
}
if (shouldSetLoading) {
@@ -119,6 +115,11 @@ export const dataStoreLoadExtension = (store: BaseStoreType, dataStore: Writable
break;
}
if (createTable) {
// remove old table with this name
await removeTable(ref.tableName);
}
await conn.insertCSVFromPath(url, {
name: ref.tableName,
detect: true,
@@ -126,23 +127,24 @@ export const dataStoreLoadExtension = (store: BaseStoreType, dataStore: Writable
});
const schema = await store.getTableSchema(ref.tableName);
const tableEntry: ITableEntry = {
const prevRefs = get(store).tables[ref.tableName]?.refs ?? [];
const loadedTableInfo: ILoadedTable = {
name: ref.tableName,
displayName: ref.displayName,
schema,
filterOptions: {},
ref: ref
refs: createTable ? [ref] : [...prevRefs, ref]
};
if (shouldUpdateTableList) {
// Update or replace table entry
dataStore.update((store) => {
store.tables[ref.tableName] = tableEntry;
store.tables[ref.tableName] = loadedTableInfo
return store;
});
}
return tableEntry;
return loadedTableInfo;
} catch (e) {
const msg = `Failed to load table ${ref.tableName} from path ${url}`;
console.error(msg, e);
@@ -161,7 +163,7 @@ export const dataStoreLoadExtension = (store: BaseStoreType, dataStore: Writable
const postProcessTable = async (
tableName: string,
refs: ITableRefList
): Promise<ITableEntry | undefined> => {
): Promise<ILoadedTable | undefined> => {
console.debug('Post process', { tableName, refs });
if (refs.length === 0) {
return;
@@ -171,17 +173,18 @@ export const dataStoreLoadExtension = (store: BaseStoreType, dataStore: Writable
try {
switch (refType) {
case TableSource.BUILD_IN: {
switch (refs[0].dataset.name) {
case 'experiments':
// FIXME: handle table parsing
// switch (refs[0].dataset.name) {
// case 'experiments':
await rewriteExperimentsEntries(tableName);
}
// }
}
}
// await addIndexColumn(tableName);
const schema = await store.getTableSchema(tableName);
const filterOptions = {}; //await getFiltersOptions(tableName, Object.keys(schema));
console.log('Rewrite response:', { schema, filterOptions, table: tableName });
return { schema, filterOptions, name: tableName, dataUrl: '' };
return { schema, filterOptions, name: tableName, refs };
} catch (e) {
console.error(`Failed to rewrite entries for table ${tableName}:`, e);
return undefined;
@@ -234,12 +237,13 @@ export const dataStoreLoadExtension = (store: BaseStoreType, dataStore: Writable
}, {} as TableSchema);
};
/**
* Loads all CSVs for the selected filters entries
* @param selected
* @returns
*/
const loadCsvsFromRefs = async (refs: ITableRefList): Promise<ITableEntry[]> =>
const loadCsvsFromRefs = async (refs: ITableReference[]): Promise<ILoadedTable[]> =>
withLoading(async () => {
if (refs.length === 0) {
return [];
@@ -268,7 +272,7 @@ export const dataStoreLoadExtension = (store: BaseStoreType, dataStore: Writable
return [];
}
const loadedTables: ITableEntry[] = [];
const loadedTables: ILoadedTable[] = [];
console.debug('loading table group:', { tableName, entries });
// Load grouped entries sequentially
@@ -294,7 +298,7 @@ export const dataStoreLoadExtension = (store: BaseStoreType, dataStore: Writable
const promiseResult = await promise;
const tableDefinitions = promiseResult.filter(
(t) => t !== undefined
) as unknown as ITableEntry[];
) as unknown as ILoadedTable[];
console.debug('loaded tables into db:', { tableDefinitions, promiseResult });
// Update data store
dataStore.update((store) => {
@@ -369,7 +373,7 @@ export const dataStoreLoadExtension = (store: BaseStoreType, dataStore: Writable
return {
// Add modifiers
loadEntriesFromFileList: async (fileList: FileList) => {
const promises: Promise<ITableEntry | undefined>[] = [];
const promises: Promise<ILoadedTable | undefined>[] = [];
for (const file of fileList) {
const tableName = file.name.replace('.csv', '');
src/lib/store/dataStore/types.ts
+41 -5
View File
@@ -1,5 +1,4 @@
import type { AsyncDuckDB, AsyncDuckDBConnection } from '@duckdb/duckdb-wasm';
import type { ITableReference } from '../filterStore/types';
export type FilterOptions = Record<string, { options: unknown[]; label?: string; type: string }>;
export type TableSchema = Record<string, 'number' | 'string'>;
@@ -16,11 +15,46 @@ export enum DataAggregation {
SUM = 'sum'
}
export interface ITableEntry {
export enum TableSource {
BUILD_IN,
URL,
FILE
}
interface ITableRef {
tableName: string;
displayName?: string;
}
export interface ITableBuildIn extends ITableRef {
source: TableSource.BUILD_IN;
url: string;
// build in tables are ordered in folders
// we call these folders datasets since they indicate
// comparable table structure
datasetName: string;
}
export interface ITableExternalUrl extends ITableRef {
source: TableSource.URL;
url: string;
}
export interface ITableExternalFile extends ITableRef {
source: TableSource.FILE;
file: File;
}
export type ITableReference = ITableBuildIn | ITableExternalFile | ITableExternalUrl;
export type ITableRefList = ITableBuildIn[] | ITableExternalFile[] | ITableExternalUrl[];
export interface ILoadedTable {
name: string;
displayName?: string;
schema: TableSchema;
ref: ITableReference;
refs: ITableReference[]; // can be multiple since a single table can be multiple files
filterOptions: FilterOptions;
}
@@ -28,9 +62,11 @@ export interface IDataStore {
db: AsyncDuckDB | null;
isLoading: boolean;
sharedConnection: AsyncDuckDBConnection | null;
// Property to keep track of which tables have been loaded
tables: Record<string, ITableEntry>;
// stores currently loaded tables and sources
tables: Record<string, ILoadedTable>;
// Table schema shared across all tables
combinedSchema: TableSchema;
// FIXME: hide behind debug flag
previousQueries: { query: string; success: boolean; executionTime: number }[];
}
src/lib/store/filterStore/FilterStore.ts
+198 -232
View File
@@ -1,109 +1,39 @@
import { get, writable } from 'svelte/store';
import { dataStore } from '../dataStore/DataStore';
import {
withUrlStorage,
type UrlEncoder,
defaultUrlEncoder,
type UrlDecoder,
defaultUrlDecoder
} from '../urlStorage';
import {
type IFilterStore,
TableSource,
GraphOptions,
GraphType,
type ITableRefList,
type ITableBuildIn,
type ITableExternalUrl,
type ITableExternalFile,
type ITableReference
} from './types';
import { PlaneGraphOptions } from './graphs/plane';
import { withSingleKeyUrlStorage } from '../urlStorage';
import { type IFilterStore, GraphOptions, GraphType, type GraphStateConfig } from './types';
import { PlaneGraphModel } from './graphs/plane';
import { defaultLogOptions, withLogMiddleware } from '../logMiddleware';
import notificationStore from '../notificationStore';
import type { Dataset, DatasetItem } from '../../../dataset/types';
type UrlTableSelection = {
source: TableSource;
tableName: string;
datasetName?: string; // Only set for source == build_in
};
import { browser } from '$app/environment';
import {
TableSource,
type ITableExternalUrl,
type ITableExternalFile,
type ITableBuildIn
} from '../dataStore/types';
import { toStateObject, urlEncodeFilterState } from './restore';
const initialStore: IFilterStore = {
isLoading: true,
preloadedDatasets: [],
selectedTables: []
preloadedDatasets: []
};
// Hacky way to create a new store with a new base object
const baseStore = writable<IFilterStore>(JSON.parse(JSON.stringify(initialStore)));
const storeEncodeSelectedTables = (tables: IFilterStore['selectedTables']) =>
tables.map(
(el) =>
({
source: el.source,
tableName: el.tableName,
datasetName: el.source == TableSource.BUILD_IN ? el.dataset.name : undefined
} as UrlTableSelection)
);
const _filterStore = () => {
const urlEncoder: UrlEncoder = (key, type, value) => {
if (key === 'graphOptions') {
console.log('Encoding graph options', value);
if (!value) {
return undefined;
}
return (value as GraphOptions).getType();
}
if (key === 'selectedTables') {
const val = value as ITableReference[];
return defaultUrlEncoder(key, type, storeEncodeSelectedTables(val));
}
const encodedValue = defaultUrlEncoder(key, type, value);
// console.log('Encoding', key, encodedValue, value, JSON.stringify(value));
return encodedValue;
};
// Store renderer temporarily globally and
// Set it after database init was completed
let urlRestoredGraphOptions: GraphOptions | null = null;
let urlRestoredTableSelection: UrlTableSelection[] = [];
const urlDecoder: UrlDecoder = (key, type, value) => {
switch (key) {
case 'graphOptions': {
const graphType = value as GraphType;
switch (graphType) {
case GraphType.PLANE: {
urlRestoredGraphOptions = new PlaneGraphOptions();
return undefined;
}
}
break;
}
case 'selectedTables': {
urlRestoredTableSelection = defaultUrlDecoder(key, type, value) as UrlTableSelection[];
return [];
}
}
return defaultUrlDecoder(key, type, value);
};
// let urlRestoredTableSelection: UrlTableSelection[] = [];
const store = withLogMiddleware(
withUrlStorage(
withSingleKeyUrlStorage<IFilterStore>(
baseStore,
{
selectedTables: 'object',
graphOptions: 'object'
},
urlEncoder,
urlDecoder
'filter',
urlEncodeFilterState,
() => initialStore
),
'FilterStore',
{ ...defaultLogOptions, color: 'green' }
@@ -118,23 +48,6 @@ const _filterStore = () => {
});
};
const selectTables = async (tables: ITableRefList) => {
setIsLoading(true);
// Load tables into data store
try {
const loadedTables = await dataStore.loadCsvsFromRefs(tables);
store.update((store) => {
store.selectedTables = [...store.selectedTables, ...tables];
return store;
});
} catch (e) {
console.error('Failed to load tables:', e);
return;
} finally {
setIsLoading(false);
}
};
const reloadCurrentGraph = async () => {
// reload state with table changes
const state = get(store);
@@ -144,7 +57,14 @@ const _filterStore = () => {
}
};
const selectBuildInTables = async (dataset: Dataset, tablePaths: DatasetItem[]) => {
if (browser) {
// update filters every time data store changes
// dataStore.subscribe((state) => {
// reloadCurrentGraph();
// });
}
const loadBuildInTables = async (dataset: Dataset, tablePaths: DatasetItem[]) => {
// Convert filter options to table references
const tableReferences: ITableBuildIn[] = tablePaths.flatMap((item) => {
return item.files.map((file) => ({
@@ -153,13 +73,12 @@ const _filterStore = () => {
source: TableSource.BUILD_IN,
// FIXME: create correct path on server
url: '/' + file.dataURL,
dataset
datasetName: dataset.name
}));
});
try {
await selectTables(tableReferences);
await reloadCurrentGraph();
await dataStore.loadCsvsFromRefs(tableReferences);
return;
} catch {
notificationStore.error({
@@ -169,30 +88,127 @@ const _filterStore = () => {
}
};
const loadTableFromURL = async (url: URL, tableName?: string) => {
// Convert filter options to table references
const tableReferences: ITableExternalUrl[] = [
{
tableName: tableName ?? url.pathname.replaceAll('/', '-'),
source: TableSource.URL,
url: url.href
}
];
try {
await dataStore.loadCsvsFromRefs(tableReferences);
return;
} catch (err) {
notificationStore.error({
message: 'Failed to load tables from file',
description: `${tableReferences.map((table) => table.tableName).join(',')}, err=${
err ?? 'Unknown Error'
}`
});
}
};
const loadTablesFromFiles = async (fileList: FileList, tableName?: string) => {
// Convert filter options to table references
const tableReferences: ITableExternalFile[] = [];
for (const file of fileList) {
tableReferences.push({
tableName: tableName ?? file.name,
source: TableSource.FILE,
file
});
}
try {
await dataStore.loadCsvsFromRefs(tableReferences);
return;
} catch (err) {
notificationStore.error({
message: 'Failed to load tables from file',
description: `${tableReferences.map((table) => table.tableName).join(',')}, err=${
err ?? 'Unknown Error'
}`
});
}
};
const initWithConfig = async (config: GraphStateConfig) => {
const datasets = get(store).preloadedDatasets;
console.debug('init from config ', { config, datasets });
for (const table of config.selectedTables) {
let loadedDatasets: Record<string, DatasetItem[]> = {};
for (const ref of table.refs) {
switch (ref.source) {
case TableSource.BUILD_IN: {
const dataset = datasets.find((dataset) => dataset.name == ref.datasetName);
if (dataset) {
if (!loadedDatasets[dataset.name]) {
loadedDatasets[dataset.name] = [];
}
const datasetItem = dataset.items.find((item) => item.name == table.tableName);
if (datasetItem) {
loadedDatasets[dataset.name] = [...loadedDatasets[dataset.name], datasetItem];
}
}
break;
}
case TableSource.URL:
console.warn('restore from URL not supported yet');
break;
case TableSource.FILE:
console.warn('restore from FILE not supported yet');
break;
}
}
for (const [datasetName, items] of Object.entries(loadedDatasets)) {
await loadBuildInTables(datasets.find((dataset) => dataset.name === datasetName)!, items);
}
}
if (config.graphOption) {
switch (config.graphOption.type) {
case GraphType.PLANE: {
store.update((state) => {
state.graphOptions = new PlaneGraphModel(
config.graphOption?.data,
config.graphOption?.renderer
);
return state;
});
}
}
}
};
return {
set,
update,
subscribe,
selectTables,
// Util methods for loading collection of tables and handling reloads
loadBuildInTables,
loadTableFromURL,
loadTablesFromFiles,
removeTable: async (tableName: string) => {
try {
await dataStore.removeTable(tableName);
await reloadCurrentGraph();
} catch {
notificationStore.error({
message: `Failed to remove table "${tableName}"`
});
}
await reloadCurrentGraph();
},
toStateObject: () => {
const state = get(store);
return {
selectedTables: storeEncodeSelectedTables(state.selectedTables),
graphOptions: state.graphOptions?.toStateObject()
};
return toStateObject(state);
},
reset: () => {
@@ -209,140 +225,90 @@ const _filterStore = () => {
// initWithPreloadedDatasets:
// called after frontend completed mount of graph component and server defined Datasets are available to the client
// This is the perfect spot for restoring state since initial server and client states are available
initWithPreloadedDatasets: async (datasets: Dataset[], selectedGraph?: any) => {
initWithPreloadedDatasets: async (datasets: Dataset[], config?: GraphStateConfig) => {
update((store) => {
store.preloadedDatasets = datasets;
return store;
});
// restore selected tables not that we have the paths from the server
const restoredSelectedTables: [Dataset, DatasetItem][] = [];
// FIXME: cleanup & and handle edge cases
if (selectedGraph) {
console.log('using selected Graph', selectedGraph);
urlRestoredTableSelection = selectedGraph.selectedTables;
if (selectedGraph['graphOptions']) {
const graphType = selectedGraph.graphOptions.type as GraphType;
switch (graphType) {
case GraphType.PLANE: {
urlRestoredGraphOptions = new PlaneGraphOptions(selectedGraph.graphOptions.state);
}
}
}
// if we have a config it takes precedence over URL decoding
if (config) {
await initWithConfig(config);
}
await reloadCurrentGraph();
// // restore selected tables not that we have the paths from the server
// const restoredSelectedTables: [Dataset, DatasetItem][] = [];
urlRestoredTableSelection.forEach((selection) => {
switch (selection.source) {
case TableSource.BUILD_IN: {
const dataset = datasets.find((dataset) => dataset.name == selection.datasetName);
if (dataset) {
const datasetItem = dataset.items.find((item) => item.name == selection.tableName);
if (datasetItem) {
console.log(datasetItem);
restoredSelectedTables.push([dataset, datasetItem]);
}
}
break;
}
case TableSource.URL:
console.warn('restore from URL not supported yet');
break;
case TableSource.FILE:
console.warn('restore from FILE not supported yet');
break;
}
});
// remove temporary url values
urlRestoredTableSelection = [];
// load all tables
for (const [dataset, item] of restoredSelectedTables) {
await selectBuildInTables(dataset, [item]);
}
// // FIXME: cleanup & and handle edge cases
// if (selectedGraph) {
// console.log('using selected Graph', selectedGraph);
// urlRestoredTableSelection = selectedGraph.selectedTables;
// Attempt reloading selected tables
if (get(store).selectedTables.length !== 0) {
try {
if (urlRestoredGraphOptions && urlRestoredGraphOptions !== null) {
update((store) => {
store.graphOptions = urlRestoredGraphOptions ?? undefined;
return store;
});
await reloadCurrentGraph();
}
} catch (e) {
console.error('Failed to load selected tables:', e);
}
}
// remove temporary url values
urlRestoredGraphOptions = null;
// if (selectedGraph['graphOptions']) {
// const graphType = selectedGraph.graphOptions.type as GraphType;
// switch (graphType) {
// case GraphType.PLANE: {
// urlRestoredGraphOptions = new PlaneGraphModel(selectedGraph.graphOptions.state);
// }
// }
// }
// }
// urlRestoredTableSelection.forEach((selection) => {
// switch (selection.source) {
// case TableSource.BUILD_IN: {
// const dataset = datasets.find((dataset) => dataset.name == selection.datasetName);
// if (dataset) {
// const datasetItem = dataset.items.find((item) => item.name == selection.tableName);
// if (datasetItem) {
// console.log(datasetItem);
// restoredSelectedTables.push([dataset, datasetItem]);
// }
// }
// break;
// }
// case TableSource.URL:
// console.warn('restore from URL not supported yet');
// break;
// case TableSource.FILE:
// console.warn('restore from FILE not supported yet');
// break;
// }
// });
// // remove temporary url values
// urlRestoredTableSelection = [];
// // load all tables
// for (const [dataset, item] of restoredSelectedTables) {
// await dataStore.loadBuildInTables(dataset, [item]);
// }
// // Attempt reloading selected tables
// if (get(dataStore).tables.length) {
// try {
// if (urlRestoredGraphOptions && urlRestoredGraphOptions !== null) {
// update((store) => {
// store.graphOptions = urlRestoredGraphOptions ?? undefined;
// return store;
// });
// await reloadCurrentGraph();
// }
// } catch (e) {
// console.error('Failed to load selected tables:', e);
// }
// }
// // remove temporary url values
// urlRestoredGraphOptions = null;
setIsLoading(false);
return;
},
selectBuildInTables,
selectDataset: (dataset?: Dataset) => {
update((state) => {
state.selectedDataset = dataset;
return state;
});
},
selectTableFromURL: async (url: URL) => {
// Convert filter options to table references
const tableReferences: ITableExternalUrl[] = [
{
tableName: url.pathname.replaceAll('/', '-'),
source: TableSource.URL,
url: url.href
}
];
try {
await selectTables(tableReferences);
await reloadCurrentGraph();
return;
} catch (err) {
notificationStore.error({
message: 'Failed to load tables from file',
description: `${tableReferences.map((table) => table.tableName).join(',')}, err=${
err ?? 'Unknown Error'
}`
});
}
},
selectTablesFromFiles: async (fileList: FileList) => {
// Convert filter options to table references
const tableReferences: ITableExternalFile[] = [];
for (const file of fileList) {
tableReferences.push({
tableName: file.name,
source: TableSource.FILE,
file
});
}
try {
await selectTables(tableReferences);
await reloadCurrentGraph();
return;
} catch (err) {
notificationStore.error({
message: 'Failed to load tables from file',
description: `${tableReferences.map((table) => table.tableName).join(',')}, err=${
err ?? 'Unknown Error'
}`
});
}
},
selectGraphType: async (graphType: GraphType) => {
switch (graphType) {
case GraphType.PLANE: {
update((store) => {
store.graphOptions = new PlaneGraphOptions();
const state = store.graphOptions?.toStateObject();
store.graphOptions = new PlaneGraphModel(state?.data, state?.render);
return store;
});
}
src/lib/store/filterStore/graphs/plane.ts
+222 -138
View File
@@ -1,16 +1,21 @@
import type { IPlaneChildData, IPlaneRendererData } from '$lib/rendering/PlaneRenderer';
import {
PlaneRenderer,
type IPlaneChildData,
type IPlaneRenderOptions,
type IPlaneRendererData,
PlaneTriangulation
} from '$lib/rendering/PlaneRenderer';
import { dataStore } from '$lib/store/dataStore/DataStore';
import { get, readonly, writable, type Readable, type Writable } from 'svelte/store';
import { GraphOptions, GraphType } from '../types';
import { GraphOptions, GraphType, type GraphFilterOptions } from '../types';
import { DataAggregation, DataScaling } from '$lib/store/dataStore/types';
import { colorBrewer, graphColors } from '$lib/rendering/colors';
import type { ITiledDataOptions, ValueRange } from '$lib/store/dataStore/filterActions';
import { urlDecodeObject, urlEncodeObject, withSingleKeyUrlStorage } from '$lib/store/urlStorage';
import { withLogMiddleware } from '$lib/store/logMiddleware';
import notificationStore from '$lib/store/notificationStore';
type RequiredOptions = ITiledDataOptions & {
groupBy: string;
groupBy?: string;
};
export type IPlaneGraphState = (
@@ -22,8 +27,11 @@ export type IPlaneGraphState = (
} & Partial<RequiredOptions>)
) & { isRendered: boolean };
export class PlaneGraphOptions extends GraphOptions<
const defaultInitialState = {};
export class PlaneGraphModel extends GraphOptions<
Partial<RequiredOptions>,
IPlaneRenderOptions,
IPlaneRendererData | undefined
> {
private _dataStore: Writable<IPlaneRendererData | undefined>;
@@ -32,11 +40,44 @@ export class PlaneGraphOptions extends GraphOptions<
public dataStore: Readable<IPlaneRendererData | undefined>;
public optionsStore: Readable<Partial<RequiredOptions>>;
constructor(initialState: Partial<RequiredOptions> = {}) {
private _renderOptions: IPlaneRenderOptions;
public get renderSettings() {
return this._renderOptions as Readonly<IPlaneRenderOptions>;
}
private renderOptionFields: GraphFilterOptions<IPlaneRenderOptions> = {
showSelection: {
type: 'boolean',
label: 'Data points',
default: true,
required: true
},
triangulation: {
type: 'string',
label: 'Triangulation',
options: Object.values(PlaneTriangulation),
required: true
}
};
public getRenderOptionFields() {
return this.renderOptionFields;
}
constructor(
initialState: Partial<RequiredOptions> = defaultInitialState,
renderSettings: Partial<IPlaneRenderOptions> = {}
) {
super({});
this._dataStore = writable(undefined);
this.dataStore = readonly(this._dataStore);
this._renderOptions = {
...PlaneRenderer.defaultRenderOptions(),
...renderSettings
};
// Check if options are initially valid
const initialOptions = {
...initialState,
@@ -44,48 +85,25 @@ export class PlaneGraphOptions extends GraphOptions<
isValid: this.isValid(initialState)
} as IPlaneGraphState;
this._optionsStore = withLogMiddleware(
withSingleKeyUrlStorage(
writable(initialOptions),
'filterStore',
(state) => {
return urlEncodeObject(state);
},
(value) => {
if (!value || value === 'undefined') {
return initialOptions;
}
const state = urlDecodeObject(value);
return {
isRendered: false,
isValid: this.isValid(state),
...state
} as IPlaneGraphState;
}
),
'PlaneGraphOptions',
{
color: 'orange'
}
);
this._optionsStore = withLogMiddleware(writable(initialOptions), 'PlaneGraphModel', {
color: 'orange'
});
this.optionsStore = readonly(this._optionsStore);
this.reloadFilterOptions();
// if no default init was passed in attempt to init with some data based defaults
if (initialState === defaultInitialState) {
this.resetOptions();
}
this.applyOptionsIfValid();
}
public toString(): string {
const state = get(this._optionsStore);
return urlEncodeObject({
type: this.getType(),
state
});
}
public toStateObject() {
const state = get(this._optionsStore);
return {
type: this.getType(),
state
data: this.getCurrentOptions(),
render: this._renderOptions
};
}
@@ -109,12 +127,25 @@ export class PlaneGraphOptions extends GraphOptions<
this.applyOptionsIfValid();
};
public setRenderOption = <K extends keyof IPlaneRenderOptions>(
key: K,
value: IPlaneRenderOptions[K]
) => {
this._renderOptions[key] = value;
this.applyOptionsIfValid();
};
// Utility method to check if current user input results in a valid graph state
private isValid(state: Partial<RequiredOptions> | undefined): boolean {
if (!state) {
return false;
}
const isValid = Object.entries(this.filterOptions).every(([key, value]) => {
if (Object.keys(state).length === 0) {
return false;
}
const isValid = Object.entries(this.filterOptionFields).every(([key, value]) => {
if (value.type === 'row') {
return value.keys.every((key, index) =>
value.items[index].required ? state[key] !== undefined : true
@@ -124,11 +155,25 @@ export class PlaneGraphOptions extends GraphOptions<
return value.required ? state[key as keyof RequiredOptions] !== undefined : true;
});
console.log({ isValid });
return isValid;
}
private resetOptions() {
this._optionsStore.update((state) => {
for (const [k, v] of Object.entries(this.filterOptionFields)) {
if (v.type === 'row') {
v.keys.forEach((key, index) => {
(state as any)[key as keyof RequiredOptions] = v.items[index].default;
});
} else {
(state as any)[k as keyof RequiredOptions] = v.default;
}
}
state.isValid = this.isValid(state);
return state;
});
}
// performs a DB query and constructs UI Options that will be used for dropdowns and other components
// to configure a given graph
public reloadFilterOptions() {
@@ -139,7 +184,7 @@ export class PlaneGraphOptions extends GraphOptions<
const numberTableColumns = Object.entries(data.combinedSchema)
.filter(([, type]) => type === 'number')
.map(([column]) => column);
this.filterOptions = {
this.filterOptionFields = {
groupBy: {
type: 'string',
options: stringTableColumns,
@@ -161,13 +206,15 @@ export class PlaneGraphOptions extends GraphOptions<
type: 'string',
options: numberTableColumns,
label: 'X Axis',
required: true
required: true,
default: numberTableColumns[Math.floor(Math.random() * numberTableColumns.length)]
},
{
type: 'string',
options: Object.values(DataScaling),
label: 'X Scale',
required: true
required: true,
default: DataScaling.LINEAR
}
]
},
@@ -180,13 +227,15 @@ export class PlaneGraphOptions extends GraphOptions<
type: 'string',
options: numberTableColumns,
label: 'Y Axis',
required: true
required: true,
default: numberTableColumns[Math.floor(Math.random() * numberTableColumns.length)]
},
{
type: 'string',
options: Object.values(DataScaling),
label: 'Y Scale',
required: true
required: true,
default: DataScaling.LINEAR
}
]
},
@@ -199,13 +248,15 @@ export class PlaneGraphOptions extends GraphOptions<
type: 'string',
options: numberTableColumns,
label: 'Z Axis',
required: true
required: true,
default: numberTableColumns[Math.floor(Math.random() * numberTableColumns.length)]
},
{
type: 'string',
options: [DataScaling.LINEAR, DataScaling.LOG],
options: Object.values(DataScaling),
label: 'Z Scale',
required: true
required: true,
default: DataScaling.LINEAR
}
]
},
@@ -213,7 +264,8 @@ export class PlaneGraphOptions extends GraphOptions<
type: 'number',
options: [2, 128],
label: 'Tile Count',
required: true
required: true,
default: 24
}
};
}
@@ -225,65 +277,28 @@ export class PlaneGraphOptions extends GraphOptions<
public getCurrentOptions() {
return get(this._optionsStore);
}
public async getGlobalRange(
columnName: string,
scaling: DataScaling
): Promise<ValueRange | null> {
const data = get(dataStore);
const tables = Object.keys(data.tables);
try {
const result = await Promise.all(
tables.map((table) => dataStore.getMinMax(table, columnName, scaling))
);
return result.reduce(
(acc, [min, max]) => [Math.min(acc[0], min), Math.max(acc[1], max)],
[0, -Infinity]
);
} catch {
return null;
}
}
// if options are valid dataStore will be updated with new values
// dataStore is used for actual rendering
public async applyOptionsIfValid() {
const state = get(this._optionsStore);
console.log('applying store', state);
if (state.isValid !== true) {
return;
}
const ranges = await this.getGlobalRanges();
if (ranges === null) {
return;
}
const [xAxisRange, yAxisRange, zAxisRange] = ranges;
// Get available tables
const data = get(dataStore);
const tables = Object.keys(data.tables);
const hasGroupBy = state.groupBy !== null;
// Get all layers
try {
const tables = Object.keys(data.tables);
console.debug('loading data from tables', tables);
const xAxisRange = await this.getGlobalRange(state.xColumnName, state.scaleX);
const yAxisRange = await this.getGlobalRange(state.yColumnName, state.scaleY);
const zAxisRange = await this.getGlobalRange(state.zColumnName, state.scaleZ);
if (!xAxisRange || !yAxisRange || !zAxisRange) {
notificationStore.error({
message: 'Data range invalid',
description: JSON.stringify({
column: state.xColumnName,
xAxisRange,
yAxisRange,
zAxisRange
})
});
return;
}
console.debug('axis ranges', {
xAxisRange,
yAxisRange,
zAxisRange
});
const promise = await Promise.all(
tables.map((table) =>
dataStore.getTiledData(
@@ -297,48 +312,15 @@ export class PlaneGraphOptions extends GraphOptions<
);
// If group by set also query groupBy data
const childLayers = await Promise.all<IPlaneChildData[]>(
tables.map(async (table) => {
const values = await dataStore.getDistinctValues(table, state.groupBy);
if (values.length > 30) {
const error = `Too many options returned by group by number=${values.length} (limit 10)`;
notificationStore.error({
message: error
});
throw new Error(error);
}
const data = await Promise.all(
values.map((value) =>
dataStore.getTiledData(
table,
state as RequiredOptions,
xAxisRange,
yAxisRange,
zAxisRange,
{ columnName: state.groupBy, value: value as string }
)
)
);
return data.map((value, index) => ({
points: value.points,
min: value.min,
max: value.max,
isChild: true,
name: values[index] as string,
color: colorBrewer.Set2[8][index % colorBrewer.Set2[8].length],
meta: {
rows: value.queryResult
}
}));
})
);
const childLayers = hasGroupBy
? await Promise.all(
tables.map((table) => this.queryGroupedLayers(table, state.groupBy!, ranges))
)
: null;
const layers = promise.map((data, index) => ({
points: data.points,
layers: childLayers[index],
layers: childLayers?.[index],
min: data.min,
max: data.max,
name: tables[index] as string,
@@ -370,4 +352,106 @@ export class PlaneGraphOptions extends GraphOptions<
return;
}
}
private async getGlobalRange(
columnName: string,
scaling: DataScaling
): Promise<ValueRange | null> {
const data = get(dataStore);
const tables = Object.keys(data.tables);
try {
const result = await Promise.all(
tables.map((table) => dataStore.getMinMax(table, columnName, scaling))
);
return result.reduce(
(acc, [min, max]) => [Math.min(acc[0], min), Math.max(acc[1], max)],
[0, -Infinity]
);
} catch {
return null;
}
}
// Get ranges along all selected columns and all tables
private async getGlobalRanges(): Promise<[ValueRange, ValueRange, ValueRange] | null> {
const state = get(this._optionsStore);
if (state.isValid !== true) {
return null;
}
// Get all layers
try {
const xAxisRange = await this.getGlobalRange(state.xColumnName, state.scaleX);
const yAxisRange = await this.getGlobalRange(state.yColumnName, state.scaleY);
const zAxisRange = await this.getGlobalRange(state.zColumnName, state.scaleZ);
if (!xAxisRange || !yAxisRange || !zAxisRange) {
throw Error(`Data ranges empty, x:${xAxisRange}, y:${yAxisRange}, z:${zAxisRange}`);
}
return [xAxisRange, yAxisRange, zAxisRange];
} catch (err) {
notificationStore.error({
message: `Could not compute data range: ${err}`,
description: 'Verify databases are loaded correctly and contain the selected columns'
});
console.error({ msg: 'getGlobalRanges:', state });
return null;
}
}
private async queryGroupedLayers(
tableName: string,
groupColumn: string,
ranges: [ValueRange, ValueRange, ValueRange],
groupValueLimit: number = 40
): Promise<IPlaneChildData[]> {
const state = get(this._optionsStore);
if (state.isValid !== true) {
return [];
}
const values = await dataStore.getDistinctValues(tableName, groupColumn);
if (values.length > groupValueLimit) {
throw Error(
`Too many options for group by: col:${groupColumn} has ${values.length} distinct values`
);
}
try {
const data = await Promise.all(
values.map((value) =>
dataStore.getTiledData(
tableName,
state as RequiredOptions,
ranges[0],
ranges[1],
ranges[2],
{
columnName: groupColumn,
value: value as string
}
)
)
);
return data.map((value, index) => ({
points: value.points,
min: value.min,
max: value.max,
isChild: true,
name: values[index] as string,
color: colorBrewer.Set2[8][index % colorBrewer.Set2[8].length],
meta: {
rows: value.queryResult
}
}));
} catch (err) {
notificationStore.error({
message: `${err}`
});
return [];
}
}
}
src/lib/store/filterStore/restore.ts
+67
View File
@@ -0,0 +1,67 @@
import { get } from "svelte/store";
import { dataStore } from "../dataStore/DataStore";
import { defaultUrlDecoder, defaultUrlEncoder, urlDecodeObject, urlEncodeObject } from "../urlStorage";
import type { GraphOptions, GraphStateConfig, GraphType, IFilterStore, IMinimalTableRef } from "./types";
import { TableSource } from "../dataStore/types";
export const toStateObject = (state: IFilterStore): GraphStateConfig => {
const tables:IMinimalTableRef[] = Object.entries(get(dataStore).tables).map(
([tableName, table]) => ({
tableName,
// build in only need one of the sources
// since a table in a dataset can be composed from multiple parts
refs: table.refs.length > 0 ? [{
source: table.refs[0].source,
datasetName: table.refs[0].source == TableSource.BUILD_IN ? table.refs[0].datasetName : undefined
}] : []})
);
// ({
// source: el.source,
// tableName: el.tableName,
// datasetName: el.source == TableSource.BUILD_IN ? el.dataset.name : undefined
// } as UrlTableSelection)
return {selectedTables: tables,
// Probably defer this?
graphOption: state.graphOptions && {
type: state.graphOptions!.getType(),
state: state.graphOptions!.toStateObject()
}
};
}
// Stores currently loaded DB state and filter selections
export const urlEncodeFilterState = (state: IFilterStore):string => {
return urlEncodeObject(toStateObject(state))
}
// export const urlDecodeFilterState = (param: string): IFilterStore {
// const value = urlDecodeObject(param);
// const graphType = value as GraphType;
// switch (graphType) {
// case GraphType.PLANE: {
// urlRestoredGraphOptions = new PlaneGraphModel();
// return undefined;
// }
// }
// break;
// }
// case 'selectedTables': {
// urlRestoredTableSelection = defaultUrlDecoder(key, type, value) as UrlTableSelection[];
// return [];
// }
// }
// return defaultUrlDecoder(key, type, value);
// };
// }
src/lib/store/filterStore/types.ts
+52 -46
View File
@@ -1,6 +1,9 @@
import type { FilterEntry } from '$routes/graph/[slug]/+page.server';
import type { Readable } from 'svelte/store';
import type { Dataset } from '../../../dataset/types';
import type { TableSource } from '../dataStore/types';
import type { IPlaneRenderOptions, IPlaneRendererData } from '$lib/rendering/PlaneRenderer';
import type { IPlaneGraphState } from './graphs/plane';
export enum GraphType {
PLANE = 'plane'
@@ -10,6 +13,45 @@ export type DeepPartial<T> = {
[P in keyof T]?: DeepPartial<T[P]>;
};
// Minimal definition of a loaded table
// omits full file paths
// should be matched agains preloaded dataset at
// init time
export type IMinimalTableRef = {
refs: {
source: TableSource;
// name: string;
url?: string;
datasetName?: string; // Only set for source == build_in
}[]
tableName: string;
};
export type GraphStateConfig = {
name?: string,
description?: string,
selectedTables: IMinimalTableRef[],
graphOption?: {
type: GraphType.PLANE,
data: IPlaneGraphState,
renderer: IPlaneRenderOptions,
}
ui?: {
rotation?: {
x: number,
y: number,
z: number
},
position?: {
x: number,
y: number,
z: number
}
}
}
// Filter options used to render UI components for dynamic configuration
export type SimpleGraphFilterOption =
| (
| {
@@ -46,32 +88,29 @@ export type GraphFilterOptions<T> = Partial<Record<keyof T, GraphFilterOption<T>
export abstract class GraphOptions<
Options extends Record<string, unknown> = Record<string, unknown>,
RenderOptions extends Record<string, unknown> = Record<string, unknown>,
Data = unknown,
K extends keyof Options = keyof Options
K extends keyof Options = keyof Options,
RenderKey extends keyof RenderOptions = keyof RenderOptions,
> {
public active = false;
public filterOptions: GraphFilterOptions<Options>;
public filterOptionFields: GraphFilterOptions<Options>;
constructor(filterOptions: GraphFilterOptions<Options>) {
this.filterOptions = filterOptions;
constructor(filterOptionFields: GraphFilterOptions<Options>) {
this.filterOptionFields = filterOptionFields;
}
public abstract getType(): GraphType;
public abstract applyOptionsIfValid(): Promise<void>;
public abstract reloadFilterOptions(): void;
public abstract getRenderOptionFields(): GraphFilterOptions<RenderOptions>;
public abstract setFilterOption(key: K, value: Options[K]): void;
public abstract setRenderOption(key: RenderKey, value: RenderOptions[RenderKey]): void;
public abstract dataStore: Readable<Data | undefined>;
public abstract optionsStore: Readable<Options | undefined>;
public abstract toString(): string;
public abstract toStateObject(): {
type: GraphType;
state: any;
};
public static fromString(str: string): GraphOptions | null {
return null;
}
public abstract toStateObject(): {data:Options, render: RenderOptions};
public abstract description(): string | null;
}
@@ -79,38 +118,5 @@ export abstract class GraphOptions<
export interface IFilterStore {
isLoading: boolean;
preloadedDatasets: Dataset[];
selectedDataset?: Dataset;
selectedTables: ITableReference[];
graphOptions?: GraphOptions;
}
export enum TableSource {
BUILD_IN,
URL,
FILE
}
interface ITableRef {
tableName: string;
displayName?: string;
}
export interface ITableBuildIn extends ITableRef {
source: TableSource.BUILD_IN;
url: string;
dataset: Dataset;
}
export interface ITableExternalUrl extends ITableRef {
source: TableSource.URL;
url: string;
}
export interface ITableExternalFile extends ITableRef {
source: TableSource.FILE;
file: File;
}
export type ITableReference = ITableBuildIn | ITableExternalFile | ITableExternalUrl;
export type ITableRefList = ITableBuildIn[] | ITableExternalFile[] | ITableExternalUrl[];
src/lib/store/logMiddleware.ts
+1 -2
View File
@@ -1,5 +1,4 @@
import { get, type Subscriber, type Writable } from 'svelte/store';
import { detailedDiff } from 'deep-object-diff';
import type{ Subscriber, Writable } from 'svelte/store';
export const defaultLogOptions = {
color: 'blue'
};
src/lib/store/urlStorage.ts
+4
View File
@@ -60,6 +60,10 @@ export const withSingleKeyUrlStorage = <S>(
encoder: (state: S) => string | null,
decoder: (value?: string | null) => S
) => {
if (!browser) {
return store;
}
// Restore state from storage
const params = new URLSearchParams(location.search);