From 6a7589599efd232e077703e25557d4e0656276f4 Mon Sep 17 00:00:00 2001
From: Wlad <9556979+gosticks@users.noreply.github.com>
Date: Thu, 4 Jan 2024 16:48:44 +0100
Subject: [PATCH] wip: cleanup and refactor

---
 src/lib/components/CodeEditor.svelte          |   2 +-
 src/lib/components/Input.svelte               |  11 +
 src/lib/components/MiniEditor.svelte          |  39 +++
 src/lib/components/graph/AxisRenderer.svelte  |   6 +-
 src/lib/components/graph/Grid.svelte          |   6 +-
 src/lib/components/graph/Minimap.svelte       |  10 +-
 src/lib/components/graph/PlaneGraph.svelte    |   2 +-
 src/lib/components/graph/Selection3D.svelte   |   6 +-
 src/lib/components/graph/SliceGraph.svelte    |  31 +--
 .../components/graph/SliceSelection.svelte    |  25 +-
 src/lib/components/graph/Stats.svelte         |   8 +-
 src/lib/rendering/Minimap.ts                  |   2 +-
 src/lib/store/dataStore/DataStore.ts          |  13 +-
 src/lib/store/dataStore/loadActions.ts        | 235 +++++-----------
 src/lib/store/dataStore/postProcessing.ts     | 261 ++++++++++++++++++
 src/lib/store/dataStore/types.ts              |  34 ++-
 src/lib/store/filterStore/restore.ts          |   2 +-
 .../CoreGraph.svelte}                         |   9 +-
 .../FilterSidebar.svelte                      |  37 +--
 .../{components => views}/QueryEditor.svelte  |  14 +-
 src/lib/views/SchemaMatter.svelte             | 131 +++++++++
 src/lib/views/TransformerEditor.svelte        |  74 +++++
 .../tableSelection/TableSelection.svelte      |  18 +-
 src/routes/graph/[slug]/+page.svelte          |  31 +--
 24 files changed, 726 insertions(+), 281 deletions(-)
 create mode 100644 src/lib/components/Input.svelte
 create mode 100644 src/lib/components/MiniEditor.svelte
 create mode 100644 src/lib/store/dataStore/postProcessing.ts
 rename src/lib/{components/BasicGraph.svelte => views/CoreGraph.svelte} (97%)
 rename src/lib/{components => views}/FilterSidebar.svelte (87%)
 rename src/lib/{components => views}/QueryEditor.svelte (92%)
 create mode 100644 src/lib/views/SchemaMatter.svelte
 create mode 100644 src/lib/views/TransformerEditor.svelte
 rename src/lib/{components => views}/tableSelection/TableSelection.svelte (85%)

diff --git a/src/lib/components/CodeEditor.svelte b/src/lib/components/CodeEditor.svelte
index ebb0476..70f0348 100644
--- a/src/lib/components/CodeEditor.svelte
+++ b/src/lib/components/CodeEditor.svelte
@@ -22,7 +22,7 @@
 		if (!browser) return;
 
 		self.MonacoEnvironment = {
-			getWorker: async function (workerId, label) {
+			getWorker: async function (_, label) {
 				// We only support SQL for now
 				let worker: any;
 				switch (label) {
diff --git a/src/lib/components/Input.svelte b/src/lib/components/Input.svelte
new file mode 100644
index 0000000..0425852
--- /dev/null
+++ b/src/lib/components/Input.svelte
@@ -0,0 +1,11 @@
+<script lang="ts">
+	export let value: string | undefined;
+	let className = '';
+	export { className as class };
+</script>
+
+<input
+	bind:value
+	class="bg-gray-50 border border-gray-300 text-gray-900 text-sm rounded-lg focus:ring-blue-500 focus:border-blue-500 block w-full p-2.5 dark:bg-gray-700 dark:border-gray-600 dark:placeholder-gray-400 dark:text-white dark:focus:ring-blue-500 dark:focus:border-blue-500 {className}"
+	{...$$restProps}
+/>
diff --git a/src/lib/components/MiniEditor.svelte b/src/lib/components/MiniEditor.svelte
new file mode 100644
index 0000000..5a0c405
--- /dev/null
+++ b/src/lib/components/MiniEditor.svelte
@@ -0,0 +1,39 @@
+<script lang="ts" context="module">
+	import type { editor } from 'monaco-editor';
+
+	export type EditorContentChanged = CustomEvent<{
+		editorEvent: editor.IModelContentChangedEvent;
+		content: string;
+	}>;
+</script>
+
+<script lang="ts">
+	import { createEventDispatcher, onMount } from 'svelte';
+	import CodeEditor from './CodeEditor.svelte';
+
+	interface $$Events {
+		change: EditorContentChanged;
+	}
+
+	export let initialValue = '';
+	let className: string = '';
+	export { className as class };
+
+	const eventDispatcher = createEventDispatcher();
+	let editor: editor.IStandaloneCodeEditor;
+
+	const onContentChanged = (evt: editor.IModelContentChangedEvent) =>
+		eventDispatcher('change', {
+			editorEvent: evt,
+			content: editor.getValue()
+		});
+
+	$: if (editor) {
+		editor.layout();
+		editor.setValue(initialValue);
+
+		editor.onDidChangeModelContent(onContentChanged);
+	}
+</script>
+
+<CodeEditor bind:editor class={className} />
diff --git a/src/lib/components/graph/AxisRenderer.svelte b/src/lib/components/graph/AxisRenderer.svelte
index a02a725..57701a6 100644
--- a/src/lib/components/graph/AxisRenderer.svelte
+++ b/src/lib/components/graph/AxisRenderer.svelte
@@ -1,7 +1,11 @@
 <script lang="ts">
 	import { onDestroy, onMount } from 'svelte';
 	import { browser } from '$app/environment';
-	import { getGraphContext, type GraphService, type GraphUnsubscribe } from '../BasicGraph.svelte';
+	import {
+		getGraphContext,
+		type GraphService,
+		type GraphUnsubscribe
+	} from '$lib/views/CoreGraph.svelte';
 	import { AxesRenderer } from '$lib/rendering/AxesRenderer';
 	import { Axis, type AxisLabelRenderer } from '$lib/rendering/AxisRenderer';
 
diff --git a/src/lib/components/graph/Grid.svelte b/src/lib/components/graph/Grid.svelte
index ac5fcb5..9c96406 100644
--- a/src/lib/components/graph/Grid.svelte
+++ b/src/lib/components/graph/Grid.svelte
@@ -87,7 +87,11 @@
 <script lang="ts">
 	import { onDestroy, onMount } from 'svelte';
 	import { browser } from '$app/environment';
-	import { getGraphContext, type GraphService, type GraphUnsubscribe } from '../BasicGraph.svelte';
+	import {
+		getGraphContext,
+		type GraphService,
+		type GraphUnsubscribe
+	} from '$lib/views/CoreGraph.svelte';
 
 	export let scale = 1;
 	export let xDivisions: number = 10;
diff --git a/src/lib/components/graph/Minimap.svelte b/src/lib/components/graph/Minimap.svelte
index 589e126..3d9801c 100644
--- a/src/lib/components/graph/Minimap.svelte
+++ b/src/lib/components/graph/Minimap.svelte
@@ -2,8 +2,12 @@
 	import { onDestroy, onMount } from 'svelte';
 	import { Minimap as MinimapRenderer } from '$lib/rendering/Minimap';
 	import { browser } from '$app/environment';
-	import { getGraphContext, type CameraState, type GraphService } from '../BasicGraph.svelte';
-	import SettingsStore, { Theme } from '$lib/store/SettingsStore';
+	import {
+		getGraphContext,
+		type CameraState,
+		type GraphService
+	} from '$lib/views/CoreGraph.svelte';
+	import SettingsStore from '$lib/store/SettingsStore';
 
 	const graphService: GraphService = getGraphContext();
 	let minimalRenderer: MinimapRenderer | undefined;
@@ -36,8 +40,6 @@
 	onDestroy(() => {
 		minimalRenderer?.destroy();
 	});
-
-	export let setCameraState = (state: CameraState) => minimalRenderer?.setCameraState(state);
 </script>
 
 <div
diff --git a/src/lib/components/graph/PlaneGraph.svelte b/src/lib/components/graph/PlaneGraph.svelte
index 44d0f98..529094a 100644
--- a/src/lib/components/graph/PlaneGraph.svelte
+++ b/src/lib/components/graph/PlaneGraph.svelte
@@ -19,7 +19,7 @@
 		type LayerColorSelectionEvent,
 		type LayerSelectionEvent
 	} from '../layerLegend/LayerGroup.svelte';
-	import { getGraphContext, type GraphService } from '../BasicGraph.svelte';
+	import { getGraphContext, type GraphService } from '$lib/views/CoreGraph.svelte';
 	import SliceGraph from './SliceGraph.svelte';
 	import { LayersIcon, LockIcon } from 'svelte-feather-icons';
 
diff --git a/src/lib/components/graph/Selection3D.svelte b/src/lib/components/graph/Selection3D.svelte
index 37414bd..2bb1f51 100644
--- a/src/lib/components/graph/Selection3D.svelte
+++ b/src/lib/components/graph/Selection3D.svelte
@@ -130,7 +130,11 @@
 </script>
 
 <script lang="ts">
-	import { getGraphContext, type GraphService, type GraphUnsubscribe } from '../BasicGraph.svelte';
+	import {
+		getGraphContext,
+		type GraphService,
+		type GraphUnsubscribe
+	} from '$lib/views/CoreGraph.svelte';
 	import { browser } from '$app/environment';
 	import { onDestroy, onMount } from 'svelte';
 	import type { ThemeColors } from '$lib/store/SettingsStore';
diff --git a/src/lib/components/graph/SliceGraph.svelte b/src/lib/components/graph/SliceGraph.svelte
index 29291a1..87c1bdb 100644
--- a/src/lib/components/graph/SliceGraph.svelte
+++ b/src/lib/components/graph/SliceGraph.svelte
@@ -1,16 +1,14 @@
 <script lang="ts">
+	import { PortalPlacement } from '$lib/actions/portal';
+	import { Axis } from '$lib/rendering/AxisRenderer';
 	import type {
 		IPlaneChildData,
 		IPlaneData,
-		IPlaneRendererData,
-		PlaneRenderer
+		IPlaneRendererData
 	} from '$lib/rendering/PlaneRenderer';
-	import Slider, { type SliderInputEvent } from '../slider/Slider.svelte';
+	import { DataScaling } from '$lib/store/dataStore/types';
 	import type { PlaneGraphModel } from '$lib/store/filterStore/graphs/plane';
-	import SliceSelection from './SliceSelection.svelte';
-	import Dropdown, { getDropdownCtx } from '../Dropdown.svelte';
-	import Button from '../button/Button.svelte';
-	import { ButtonColor, ButtonSize, ButtonVariant } from '../button/type';
+	import { scaleDecoder } from '$lib/util';
 	import {
 		ChevronDownIcon,
 		ChevronUpIcon,
@@ -18,21 +16,18 @@
 		EyeOffIcon,
 		InfoIcon,
 		Maximize2Icon,
-		MehIcon,
 		MoveIcon
 	} from 'svelte-feather-icons';
-	import { PortalPlacement } from '$lib/actions/portal';
+	import Dropdown, { getDropdownCtx } from '../Dropdown.svelte';
+	import { type DropdownSelectionEvent } from '../DropdownSelect.svelte';
+	import Button from '../button/Button.svelte';
+	import { ButtonColor, ButtonSize, ButtonVariant } from '../button/type';
+	import Dialog, { DialogSize } from '../dialog/Dialog.svelte';
+	import type { LayerVisibilityList } from '../layerLegend/LayerGroup.svelte';
+	import Slider, { type SliderInputEvent } from '../slider/Slider.svelte';
 	import type { IGraph2dData } from './Graph2D.svelte';
 	import Graph2D from './Graph2D.svelte';
-	import type { LayerVisibilityList } from '../layerLegend/LayerGroup.svelte';
-	import { Axis } from '$lib/rendering/AxisRenderer';
-	import Dialog, { DialogSize } from '../dialog/Dialog.svelte';
-	import type { ITiledDataRow } from '$lib/store/dataStore/filterActions';
-	import DropdownSelect, { type DropdownSelectionEvent } from '../DropdownSelect.svelte';
-	import BasicGraph from '../BasicGraph.svelte';
-	import { DataScaling } from '$lib/store/dataStore/types';
-	import type { Point3D } from '$lib/rendering/geometry/SparsePlaneGeometry';
-	import { scaleDecoder } from '$lib/util';
+	import SliceSelection from './SliceSelection.svelte';
 
 	export let options: PlaneGraphModel;
 	export let layerVisibility: LayerVisibilityList;
diff --git a/src/lib/components/graph/SliceSelection.svelte b/src/lib/components/graph/SliceSelection.svelte
index 7c5781d..4a04aa8 100644
--- a/src/lib/components/graph/SliceSelection.svelte
+++ b/src/lib/components/graph/SliceSelection.svelte
@@ -1,22 +1,15 @@
 <script lang="ts" context="module">
 	import {
-		AmbientLight,
-		Camera,
-		DirectionalLight,
-		OrthographicCamera,
-		PerspectiveCamera,
-		Scene,
-		Vector2,
-		Mesh,
-		Group,
-		PlaneGeometry,
-		MeshBasicMaterial,
-		Vector3,
-		WebGLRenderer,
-		DoubleSide,
 		AddEquation,
+		DoubleSide,
+		Group,
+		Mesh,
+		MeshBasicMaterial,
+		PlaneGeometry,
+		Vector3,
 		type ColorRepresentation
 	} from 'three';
+
 	class SliceSelectionRenderer extends Group {
 		private meshes = new Map<Axis, Mesh<PlaneGeometry, MeshBasicMaterial>>();
 		private colors: Record<Axis, ColorRepresentation> = {
@@ -84,12 +77,12 @@
 </script>
 
 <script lang="ts">
-	import { getContext, onDestroy, onMount } from 'svelte';
+	import { onDestroy, onMount } from 'svelte';
 
 	import { browser } from '$app/environment';
 	import { Axis } from '$lib/rendering/AxisRenderer';
 	import { colorBrewer } from '$lib/rendering/colors';
-	import { getGraphContext, type GraphService } from '../BasicGraph.svelte';
+	import { getGraphContext, type GraphService } from '$lib/views/CoreGraph.svelte';
 
 	const graphService: GraphService = getGraphContext();
 
diff --git a/src/lib/components/graph/Stats.svelte b/src/lib/components/graph/Stats.svelte
index 194252d..7cb397e 100644
--- a/src/lib/components/graph/Stats.svelte
+++ b/src/lib/components/graph/Stats.svelte
@@ -1,8 +1,12 @@
 <script lang="ts">
-	import { getContext, onDestroy, onMount } from 'svelte';
+	import { onDestroy, onMount } from 'svelte';
 	import { browser } from '$app/environment';
 	import Stats from 'stats.js';
-	import { getGraphContext, type GraphService, type GraphUnsubscribe } from '../BasicGraph.svelte';
+	import {
+		getGraphContext,
+		type GraphService,
+		type GraphUnsubscribe
+	} from '$lib/views/CoreGraph.svelte';
 	const graphService: GraphService = getGraphContext();
 
 	let stats: Stats;
diff --git a/src/lib/rendering/Minimap.ts b/src/lib/rendering/Minimap.ts
index eb25302..832a648 100644
--- a/src/lib/rendering/Minimap.ts
+++ b/src/lib/rendering/Minimap.ts
@@ -21,7 +21,7 @@ import {
 
 import { OrbitControls } from 'three/addons/controls/OrbitControls.js';
 import { Easing, Tween } from '@tweenjs/tween.js';
-import type { CameraState } from '$lib/components/BasicGraph.svelte';
+import type { CameraState } from '$lib/views/CoreGraph.svelte';
 import { TextTexture } from './textures/TextTexture';
 import type { ThemeColors } from '$lib/store/SettingsStore';
 
diff --git a/src/lib/store/dataStore/DataStore.ts b/src/lib/store/dataStore/DataStore.ts
index f122a16..07e18fa 100644
--- a/src/lib/store/dataStore/DataStore.ts
+++ b/src/lib/store/dataStore/DataStore.ts
@@ -1,5 +1,6 @@
 import { writable, get } from 'svelte/store';
 import { dataStoreLoadExtension } from './loadActions';
+import { dataStorePostProcessingExtension } from './postProcessing';
 import type { IDataStore, TableSchema } from './types';
 import { browser } from '$app/environment';
 import { dataStoreFilterExtension } from './filterActions';
@@ -17,8 +18,8 @@ const initialStore: IDataStore = {
 
 const _baseStore = () => {
 	console.log('Initializing data store');
-	// const store = withLogMiddleware(writable<IDataStore>(initialStore), 'DataStore');
-	const store = writable<IDataStore>(initialStore);
+	const store = withLogMiddleware(writable<IDataStore>(initialStore), 'DataStore');
+	// const store = writable<IDataStore>(initialStore);
 
 	const { set, update, subscribe } = store;
 
@@ -207,16 +208,22 @@ const _baseStore = () => {
 };
 
 export type BaseStoreType = ReturnType<typeof _baseStore>;
+export type DataStoreType = ReturnType<typeof _dataStore>;
 
 const _dataStore = () => {
 	const store = _baseStore();
 
 	// Execute extension
-	return {
+	const extendedStore = {
 		...store,
 		...dataStoreLoadExtension(store, store.rawStore),
 		...dataStoreFilterExtension(store)
+		// FIXME: due to tigh coupling between load and post processing this
+		// is loaded by the load module. Probably should be changed
+		//...dataStorePostProcessingExtension(store, store.rawStore)
 	};
+
+	return extendedStore;
 };
 
 export const dataStore = _dataStore();
diff --git a/src/lib/store/dataStore/loadActions.ts b/src/lib/store/dataStore/loadActions.ts
index 6f53edb..b640dc0 100644
--- a/src/lib/store/dataStore/loadActions.ts
+++ b/src/lib/store/dataStore/loadActions.ts
@@ -1,11 +1,14 @@
 import { get, type Writable } from 'svelte/store';
-import type { BaseStoreType } from './DataStore';
-import type { IDataStore, ILoadedTable, ITableBuildIn, TableSchema } from './types';
-import { TableSource, type ITableReference, type ITableRefList } from './types';
 import notificationStore from '../notificationStore';
+import type { BaseStoreType } from './DataStore';
+import { dataStorePostProcessingExtension } from './postProcessing';
+import type { IDataStore, ILoadedTable, ITableExternalFile, TableSchema } from './types';
+import { TableSource, type ITableReference, type ITableRefList } from './types';
 
 // Store extension containing actions to load data, transform & drop data
 export const dataStoreLoadExtension = (store: BaseStoreType, dataStore: Writable<IDataStore>) => {
+	const postProcessingExtension = dataStorePostProcessingExtension(store, dataStore);
+
 	// Wrapper utility to set loading state
 	const withLoading = async <T>(fn: () => Promise<T>) => {
 		store.setIsLoading(true);
@@ -16,63 +19,6 @@ export const dataStoreLoadExtension = (store: BaseStoreType, dataStore: Writable
 		}
 	};
 
-	const rewriteExperimentsEntries = async (tableName: string) => {
-		console.log('Rewriting entries for table:', tableName);
-		const rewriteQuery = `
-	ALTER TABLE "${tableName}" ADD COLUMN family TEXT;
-	ALTER TABLE "${tableName}" ADD COLUMN mode TEXT;
-	ALTER TABLE "${tableName}" ADD COLUMN vectorization TEXT;
-	ALTER TABLE "${tableName}" ADD COLUMN fixture TEXT;
-	ALTER TABLE "${tableName}" ADD COLUMN s FLOAT;
-	ALTER TABLE "${tableName}" ADD COLUMN n_threads INTEGER;
-	ALTER TABLE "${tableName}" ADD COLUMN n_partitions INTEGER;
-	ALTER TABLE "${tableName}" ADD COLUMN n_elements_build INTEGER;
-	ALTER TABLE "${tableName}" ADD COLUMN n_elements_lookup INTEGER;
-	ALTER TABLE "${tableName}" ADD COLUMN shared_elements FLOAT;
-	ALTER TABLE "${tableName}" ADD COLUMN construction_throughput FLOAT;
-	ALTER TABLE "${tableName}" ADD COLUMN lookup_throughput FLOAT;
-
-	WITH SplitValues AS (
-		SELECT
-		name,
-		SPLIT_PART(name, '_', 1) AS family,
-		SPLIT_PART(name, '_', 2) AS mode,
-		SPLIT_PART(name, '_', 4) AS vectorization,
-		SPLIT_PART(name, '/', 2) AS fixture,
-		CAST(SPLIT_PART(name, '/', 3) AS FLOAT) / 100 AS s,
-		CAST(SPLIT_PART(name, '/', 4) AS INTEGER) AS n_threads,
-		CAST(SPLIT_PART(name, '/', 5) AS INTEGER) AS n_partitions,
-		CAST(SPLIT_PART(name, '/', 6) AS INTEGER) AS n_elements_build,
-		CAST(SPLIT_PART(name, '/', 7) AS INTEGER) AS n_elements_lookup,
-		CAST(SPLIT_PART(name, '/', 8) AS FLOAT) / 100 AS shared_elements
-		FROM "${tableName}")
-	UPDATE "${tableName}" AS t
-		SET
-			family = sv.family,
-			mode = sv.mode,
-			vectorization = sv.vectorization,
-			fixture = sv.fixture,
-			s = sv.s,
-			n_threads = sv.n_threads,
-			n_partitions = sv.n_partitions,
-			n_elements_build = sv.n_elements_build,
-			n_elements_lookup = sv.n_elements_lookup,
-	shared_elements = sv.shared_elements
-	FROM SplitValues AS sv
-	WHERE t.name = sv.name;
-
-	UPDATE "${tableName}"
-	SET "construction_throughput" = (n_elements_build * 1000.0) / real_time
-	WHERE real_time IS NOT NULL AND real_time != 0;
-
-	UPDATE "${tableName}" as t
-		SET fpr = 'NaN'
-		WHERE fpr = -1;
-		`;
-
-		return store.executeQuery(rewriteQuery);
-	};
-
 	const bindFileToDuckDB = async (dbPath: string, file: File) => {
 		const { db } = get(dataStore);
 		const conn = await store.getConnection();
@@ -86,12 +32,17 @@ export const dataStoreLoadExtension = (store: BaseStoreType, dataStore: Writable
 		await db.registerFileHandle(dbPath, file, DuckDBDataProtocol.BROWSER_FILEREADER, true);
 	};
 
+	const getTableDisplayName = (rawTableName: string) => {
+		// NOTE: inefficient string replacement with multiple loops, deemed acceptable due to
+		// low call frequency. At most once per table creation.
+		return rawTableName.replaceAll('_', ' ').replaceAll('-', ' ').trim();
+	};
+
 	const loadCsvFromRef = async (
 		ref: ITableReference,
 		shouldSetLoading = true,
-		createTable = true,
-		shouldUpdateTableList = true
-	): Promise<ILoadedTable | undefined> => {
+		createTable = true
+	): Promise<ITableReference | undefined> => {
 		const conn = await store.getConnection();
 
 		if (!conn) {
@@ -128,25 +79,7 @@ export const dataStoreLoadExtension = (store: BaseStoreType, dataStore: Writable
 				create: createTable
 			});
 
-			const schema = await store.getTableSchema(ref.tableName);
-			const prevRefs = get(store).tables[ref.tableName]?.refs ?? [];
-			const loadedTableInfo: ILoadedTable = {
-				name: ref.tableName,
-				displayName: ref.displayName,
-				schema,
-				filterOptions: {},
-				refs: createTable ? [ref] : [...prevRefs, ref]
-			};
-
-			if (shouldUpdateTableList) {
-				// Update or replace table entry
-				dataStore.update((store) => {
-					store.tables[ref.tableName] = loadedTableInfo;
-					return store;
-				});
-			}
-
-			return loadedTableInfo;
+			return ref;
 		} catch (e) {
 			const msg = `Failed to load table ${ref.tableName} from path ${url}`;
 			console.error(msg, e);
@@ -162,56 +95,6 @@ export const dataStoreLoadExtension = (store: BaseStoreType, dataStore: Writable
 		}
 	};
 
-	const addIdsToTable = async (tableName: string) => {
-		const tempName = `${tableName}-temp`;
-		const rename = `ALTER TABLE "${tableName}" RENAME TO "${tempName}";`;
-		await store.executeQuery(rename);
-
-		const addIdQuery = `
-			CREATE TABLE "${tableName}" AS
-			SELECT
-				ROW_NUMBER() OVER () AS id, *
-			FROM "${tempName}";
-
-			DROP TABLE "${tempName}";
-		`;
-
-		await store.executeQuery(addIdQuery);
-	};
-
-	const postProcessTable = async (
-		tableName: string,
-		refs: ITableRefList
-	): Promise<ILoadedTable | undefined> => {
-		console.debug('Post process', { tableName, refs });
-		await addIdsToTable(tableName);
-		if (refs.length === 0) {
-			return;
-		}
-		// get list type
-		const refType = refs[0].source;
-		try {
-			await rewriteExperimentsEntries(tableName);
-			switch (refType) {
-				case TableSource.BUILD_IN: {
-					// FIXME: handle table parsing
-					// switch (refs[0].dataset.name) {
-					// case 'experiments':
-					// await rewriteExperimentsEntries(tableName);
-					// }
-				}
-			}
-			// await addIndexColumn(tableName);
-			const schema = await store.getTableSchema(tableName);
-			const filterOptions = {}; //await getFiltersOptions(tableName, Object.keys(schema));
-			console.log('Rewrite response:', { schema, filterOptions, table: tableName });
-			return { schema, filterOptions, name: tableName, refs };
-		} catch (e) {
-			console.error(`Failed to rewrite entries for table ${tableName}:`, e);
-			return undefined;
-		}
-	};
-
 	/**
 	 * Returns the common table schema for all tables (e.g. all columns that are present in all tables with the same type)
 	 * @param onlyCheckFields Only check these fields
@@ -258,6 +141,22 @@ export const dataStoreLoadExtension = (store: BaseStoreType, dataStore: Writable
 		}, {} as TableSchema);
 	};
 
+	/**
+	 * group table references by their table name
+	 */
+	const groupTableReferences = (refs: ITableReference[]): Record<string, ITableRefList> =>
+		refs.reduce(
+			(acc, ref) => {
+				if (!acc[ref.tableName]) {
+					acc[ref.tableName] = [];
+				}
+				// type
+				acc[ref.tableName].push(ref as any);
+				return acc;
+			},
+			{} as Record<string, ITableRefList>
+		);
+
 	/**
 	 * Loads all CSVs for the selected filters entries
 	 * @param selected
@@ -272,46 +171,43 @@ export const dataStoreLoadExtension = (store: BaseStoreType, dataStore: Writable
 			console.debug('Loading table references:', refs);
 
 			// Group tables by name for later processing
-			const grouped = refs.reduce(
-				(acc, ref) => {
-					if (!acc[ref.tableName]) {
-						acc[ref.tableName] = [];
-					}
-					// type
-					acc[ref.tableName].push(ref as any);
-					return acc;
-				},
-				{} as Record<string, ITableRefList>
-			);
-
+			const grouped = groupTableReferences(refs);
 			console.debug('Grouped table references:', grouped);
 
-			// Load multiple tables at once but all linked to the same tableName sequentially
-			// This is required since we need to rewrite the entries for each table
-			// and a table should only be created once
+			// Load multiple tables at once, with all references for one table loaded in sequentially
+			// Otherwise parts of one table would be loaded and processed incorrectly
 			const promise = Promise.all(
 				Object.entries(grouped).flatMap(async ([tableName, entries]) => {
 					if (entries.length === 0) {
-						return [];
+						return undefined;
 					}
 
-					const loadedTables: ILoadedTable[] = [];
-
 					console.debug('loading table group:', { tableName, entries });
 					// Load grouped entries sequentially
 					for (const [i, entry] of entries.entries()) {
 						// Only create table for first entry
 						// do not update store table list, this will be done after post processing
-						const loadedTable = await loadCsvFromRef(entry, false, i === 0, false);
-						if (loadedTable) {
-							loadedTables.push(loadedTable);
-						}
+						await loadCsvFromRef(entry, false, i === 0);
 						console.debug('loaded:', { tableName, entry });
 					}
-					console.debug('applying post processing:', { tableName, entries });
+
+					const sourceSchema = await store.getTableSchema(tableName);
+
+					const loadedTableInfo: ILoadedTable = {
+						tableName: tableName, // when no transformations were applied use the default table
+						displayName: getTableDisplayName(tableName),
+						sourceTableName: tableName,
+						sourceSchema,
+						transformations: [],
+						schema: sourceSchema,
+						filterOptions: {},
+						refs: entries
+					};
 
 					// Post process tables
-					return postProcessTable(tableName, entries) ?? [];
+					await postProcessingExtension.applyPostProcessing(loadedTableInfo);
+
+					return loadedTableInfo;
 				})
 			);
 
@@ -319,14 +215,12 @@ export const dataStoreLoadExtension = (store: BaseStoreType, dataStore: Writable
 			try {
 				// Filter out undefined values
 				const promiseResult = await promise;
-				const tableDefinitions = promiseResult.filter(
-					(t) => t !== undefined
-				) as unknown as ILoadedTable[];
+				const tableDefinitions = promiseResult.filter((t) => t !== undefined) as ILoadedTable[];
 				console.debug('loaded tables into db:', { tableDefinitions, promiseResult });
 				// Update data store
 				dataStore.update((store) => {
 					tableDefinitions.forEach((table) => {
-						store.tables[table.name] = table;
+						store.tables[table.sourceTableName] = table;
 					});
 					store.combinedSchema = computeCombinedTableSchema();
 					return store;
@@ -394,24 +288,23 @@ export const dataStoreLoadExtension = (store: BaseStoreType, dataStore: Writable
 	// Export public API
 
 	return {
+		...postProcessingExtension,
 		// Add modifiers
 		loadEntriesFromFileList: async (fileList: FileList) => {
-			const promises: Promise<ILoadedTable | undefined>[] = [];
+			// TODO: handle non CSV files
+
+			let refs: ITableExternalFile[] = [];
 
 			for (const file of fileList) {
-				const tableName = file.name.replace('.csv', '');
-				promises.push(
-					loadCsvFromRef({
-						source: TableSource.FILE,
-						file: file,
-						tableName: tableName
-					})
-				);
+				refs.push({
+					source: TableSource.FILE,
+					file,
+					tableName: file.name.replace('.csv', '')
+				});
 			}
 
-			await Promise.all(promises);
+			await loadCsvsFromRefs(refs);
 		},
-		loadCsvFromRef,
 		loadCsvsFromRefs,
 		resetDatabase,
 		removeTable
diff --git a/src/lib/store/dataStore/postProcessing.ts b/src/lib/store/dataStore/postProcessing.ts
new file mode 100644
index 0000000..ff64a43
--- /dev/null
+++ b/src/lib/store/dataStore/postProcessing.ts
@@ -0,0 +1,261 @@
+import type { Writable } from 'svelte/store';
+import type { BaseStoreType } from './DataStore';
+import {
+	TransformationType,
+	type IDataStore,
+	type ILoadedTable,
+	type ITableReference,
+	TableSource,
+	type TableTransformation,
+	type JsTransformation
+} from './types';
+import notificationStore from '../notificationStore';
+
+type Jsonable =
+	| string
+	| number
+	| boolean
+	| null
+	| undefined
+	| readonly Jsonable[]
+	| { readonly [key: string]: Jsonable }
+	| { toJSON(): Jsonable };
+export class PostProeccingError extends Error {
+	public readonly context?: Jsonable;
+
+	constructor(message: string, options: { cause?: unknown; context?: Jsonable } = {}) {
+		const { cause, context } = options;
+
+		super(message, { cause });
+		this.name = this.constructor.name;
+		this.context = context;
+	}
+}
+
+// returns the name of the unmodified table for a table reference
+const getOutputTableName = (table: ILoadedTable): string => `${table.sourceTableName}-output`;
+
+// post processing SQL query that parses extra columns from GoogleBenchmark name column
+const getExperimentRewriteQuerty = (tableName: string): string => `
+	ALTER TABLE "${tableName}" ADD COLUMN family TEXT;
+	ALTER TABLE "${tableName}" ADD COLUMN mode TEXT;
+	ALTER TABLE "${tableName}" ADD COLUMN vectorization TEXT;
+	ALTER TABLE "${tableName}" ADD COLUMN fixture TEXT;
+	ALTER TABLE "${tableName}" ADD COLUMN s FLOAT;
+	ALTER TABLE "${tableName}" ADD COLUMN n_threads INTEGER;
+	ALTER TABLE "${tableName}" ADD COLUMN n_partitions INTEGER;
+	ALTER TABLE "${tableName}" ADD COLUMN n_elements_build INTEGER;
+	ALTER TABLE "${tableName}" ADD COLUMN n_elements_lookup INTEGER;
+	ALTER TABLE "${tableName}" ADD COLUMN shared_elements FLOAT;
+	ALTER TABLE "${tableName}" ADD COLUMN construction_throughput FLOAT;
+	ALTER TABLE "${tableName}" ADD COLUMN lookup_throughput FLOAT;
+
+	WITH SplitValues AS (
+		SELECT
+		name,
+		SPLIT_PART(name, '_', 1) AS family,
+		SPLIT_PART(name, '_', 2) AS mode,
+		SPLIT_PART(name, '_', 4) AS vectorization,
+		SPLIT_PART(name, '/', 2) AS fixture,
+		CAST(SPLIT_PART(name, '/', 3) AS FLOAT) / 100 AS s,
+		CAST(SPLIT_PART(name, '/', 4) AS INTEGER) AS n_threads,
+		CAST(SPLIT_PART(name, '/', 5) AS INTEGER) AS n_partitions,
+		CAST(SPLIT_PART(name, '/', 6) AS INTEGER) AS n_elements_build,
+		CAST(SPLIT_PART(name, '/', 7) AS INTEGER) AS n_elements_lookup,
+		CAST(SPLIT_PART(name, '/', 8) AS FLOAT) / 100 AS shared_elements
+		FROM "${tableName}")
+	UPDATE "${tableName}" AS t
+		SET
+			family = sv.family,
+			mode = sv.mode,
+			vectorization = sv.vectorization,
+			fixture = sv.fixture,
+			s = sv.s,
+			n_threads = sv.n_threads,
+			n_partitions = sv.n_partitions,
+			n_elements_build = sv.n_elements_build,
+			n_elements_lookup = sv.n_elements_lookup,
+	shared_elements = sv.shared_elements
+	FROM SplitValues AS sv
+	WHERE t.name = sv.name;
+
+	UPDATE "${tableName}"
+	SET "construction_throughput" = (n_elements_build * 1000.0) / real_time
+	WHERE real_time IS NOT NULL AND real_time != 0;
+
+	UPDATE "${tableName}" as t
+		SET fpr = 'NaN'
+		WHERE fpr = -1;
+		`;
+
+export const dataStorePostProcessingExtension = (
+	store: BaseStoreType,
+	dataStore: Writable<IDataStore>
+) => {
+	const deleteTable = (tableName: string) => store.executeQuery(`DROP TABLE "${tableName}"`);
+	const renameTable = (tableName: string, newTableName: string) =>
+		store.executeQuery(`ALTER TABLE "${tableName}" RENAME TO "${newTableName}"`);
+
+	// Adds an Id column in the table
+	const createIdColumn = async (targetTable: string, info: ILoadedTable) => {
+		const replace = info.sourceTableName == targetTable;
+		const intermediaryTableName = replace ? `id-inject-${targetTable}-temp` : targetTable;
+		const query = `
+			CREATE TABLE "${intermediaryTableName}" AS
+			SELECT
+				ROW_NUMBER() OVER () AS id, *
+			FROM "${info.sourceTableName}";
+		`;
+
+		await store.executeQuery(query);
+
+		if (replace) {
+			await deleteTable(info.sourceTableName);
+			await renameTable(intermediaryTableName, info.sourceTableName);
+		}
+		return true;
+	};
+
+	// some default transformations
+	const idTransformation: JsTransformation = {
+		name: 'Add Row ID',
+		type: TransformationType.JS,
+		description: 'Adds a row ID for faster query operations',
+		method: createIdColumn,
+		required: true
+	};
+
+	const experimentTransformation: JsTransformation = {
+		name: 'Gtest parser',
+		type: TransformationType.JS,
+		description: 'Splits Gtest name column into relevant values',
+		method: async (tableName: string, info: ILoadedTable) => {
+			const query = getExperimentRewriteQuerty(tableName);
+			await store.executeQuery(query);
+			return true;
+		},
+		required: true
+	};
+
+	const processSqlStringLiteral = (
+		tableName: string,
+		info: ILoadedTable,
+		queryTemplate: string
+	) => {
+		return queryTemplate
+			.replaceAll(/\${tableName}/g, tableName)
+			.replaceAll(/\${sourceTableName}/g, info.sourceTableName);
+	};
+
+	const applyPostProcessing = async (table: ILoadedTable) => {
+		// clear output table
+		if (table.sourceTableName != table.tableName) {
+			await deleteTable(table.tableName);
+		}
+
+		const outputTableName = getOutputTableName(table);
+		// ensure we at least add ID injection
+		if (
+			typeof table.sourceSchema['id'] === 'undefined' &&
+			table.transformations.indexOf(idTransformation) === -1
+		) {
+			table.transformations.push(idTransformation);
+		}
+
+		// handle internal database rewrites
+		// TODO: we should probably add a mixed or pure type to ILoadedTable to check if refs have same sourcetype
+		if (
+			table.refs[0].source === TableSource.BUILD_IN &&
+			table.transformations.indexOf(experimentTransformation) === -1
+		) {
+			if (table.sourceTableName.indexOf('count') !== -1) {
+				table.transformations.push(experimentTransformation);
+			}
+		}
+
+		table.tableName = outputTableName;
+
+		for (const transformation of table.transformations) {
+			try {
+				switch (transformation.type) {
+					case TransformationType.JS:
+						await transformation.method(outputTableName, table);
+						break;
+
+					case TransformationType.SQL:
+						const query = processSqlStringLiteral(table.tableName, table, transformation.query);
+						await store.executeQuery(query);
+						break;
+				}
+
+				// NOTE: remove if this becomes an issue
+				// collect intermediary table schemas
+				transformation.resultSchema = await store.getTableSchema(table.tableName);
+			} catch (err) {
+				console.error(err);
+				throw new PostProeccingError('Transformation failed', {
+					cause: err,
+					context: {
+						transformationName: transformation.name,
+						type: transformation.type,
+						sourceTableName: table.sourceTableName
+					}
+				});
+			}
+		}
+
+		table.schema = await store.getTableSchema(table.tableName);
+
+		// trigger UI-update
+		store.update((state) => state);
+	};
+
+	const addPostProcessingTransformer = async (
+		table: ILoadedTable,
+		transformer: TableTransformation
+	) => {
+		store.setIsLoading(true);
+		table.transformations.push(transformer);
+		try {
+			await applyPostProcessing(table);
+		} catch (err) {
+			if (err instanceof PostProeccingError) {
+				notificationStore.error({
+					message: err.name,
+					description: err.message
+				});
+			}
+			return err;
+		} finally {
+			store.setIsLoading(false);
+		}
+	};
+
+	const removePostProcessingTransformer = async (
+		table: ILoadedTable,
+		transformer: TableTransformation
+	) => {
+		store.setIsLoading(true);
+		table.transformations = table.transformations.filter((tr) => tr !== transformer);
+		try {
+			await applyPostProcessing(table);
+		} catch (err) {
+			if (err instanceof PostProeccingError) {
+				notificationStore.error({
+					message: err.name,
+					description: err.message
+				});
+			}
+			return err;
+		} finally {
+			store.setIsLoading(false);
+		}
+	};
+
+	return {
+		createIdColumn,
+		applyPostProcessing,
+		addPostProcessingTransformer,
+		removePostProcessingTransformer
+	};
+};
diff --git a/src/lib/store/dataStore/types.ts b/src/lib/store/dataStore/types.ts
index 0c6cf6c..385d9f1 100644
--- a/src/lib/store/dataStore/types.ts
+++ b/src/lib/store/dataStore/types.ts
@@ -49,10 +49,38 @@ export type ITableReference = ITableBuildIn | ITableExternalFile | ITableExterna
 
 export type ITableRefList = ITableBuildIn[] | ITableExternalFile[] | ITableExternalUrl[];
 
-export interface ILoadedTable {
+export enum TransformationType {
+	SQL,
+	JS
+}
+
+export type BaseTableTransformation = {
 	name: string;
-	displayName?: string;
-	schema: TableSchema;
+	description: string;
+	type: TransformationType;
+	resultSchema?: TableSchema;
+	required?: boolean;
+};
+
+export type SqlTransformation = BaseTableTransformation & {
+	type: TransformationType.SQL;
+	query: string;
+};
+
+export type JsTransformation = BaseTableTransformation & {
+	type: TransformationType.JS;
+	method: (tableName: string, info: ILoadedTable) => Promise<boolean>;
+};
+
+export type TableTransformation = SqlTransformation | JsTransformation;
+
+export interface ILoadedTable {
+	tableName: string; // Table name that should be used for query operations with transformations applied
+	displayName: string; // Defaults to table name
+	schema: TableSchema; // Schema after all transformations were applied
+	sourceSchema: TableSchema; // Schema of raw/unmodified table
+	sourceTableName: Readonly<string>; // Unmodified table loaded by user
+	transformations: TableTransformation[];
 	refs: ITableReference[]; // can be multiple since a single table can be multiple files
 	filterOptions: FilterOptions;
 }
diff --git a/src/lib/store/filterStore/restore.ts b/src/lib/store/filterStore/restore.ts
index 58903da..43ff081 100644
--- a/src/lib/store/filterStore/restore.ts
+++ b/src/lib/store/filterStore/restore.ts
@@ -12,7 +12,7 @@ export const toStateObject = (state: IFilterStore): GraphStateConfig => {
 			// build in only need one of the sources
 			// since a table in a dataset can be composed from multiple parts
 			refs:
-				table.refs.length > 0
+				(table.refs ?? []).length > 0
 					? [
 							{
 								source: table.refs[0].source,
diff --git a/src/lib/components/BasicGraph.svelte b/src/lib/views/CoreGraph.svelte
similarity index 97%
rename from src/lib/components/BasicGraph.svelte
rename to src/lib/views/CoreGraph.svelte
index 10269fc..204906f 100644
--- a/src/lib/components/BasicGraph.svelte
+++ b/src/lib/views/CoreGraph.svelte
@@ -67,7 +67,6 @@
 	let outlinePass: OutlinePass;
 	let composer: EffectComposer;
 	let isSetupComplete = false;
-	let stats: Stats;
 
 	function setupControls() {
 		controls = new OrbitControls(camera, renderer.domElement);
@@ -149,11 +148,9 @@
 		renderer = new WebGLRenderer({
 			alpha: false,
 			antialias: true,
-			// powerPreference: 'high-performance'
-			logarithmicDepthBuffer: true,
-			preserveDrawingBuffer: true
-			// failIfMajorPerformanceCaveat: true
-			// precision: 'lowp'
+			preserveDrawingBuffer: true,
+			// TODO: disable for production builds
+			failIfMajorPerformanceCaveat: true
 		});
 		renderer.domElement.setAttribute('id', 'basic-graph');
 		renderer.setPixelRatio(window.devicePixelRatio || 1);
diff --git a/src/lib/components/FilterSidebar.svelte b/src/lib/views/FilterSidebar.svelte
similarity index 87%
rename from src/lib/components/FilterSidebar.svelte
rename to src/lib/views/FilterSidebar.svelte
index a78f8b9..debb5dd 100644
--- a/src/lib/components/FilterSidebar.svelte
+++ b/src/lib/views/FilterSidebar.svelte
@@ -2,20 +2,16 @@
 	import { dataStore } from '$lib/store/dataStore/DataStore';
 	import filterStore from '$lib/store/filterStore/FilterStore';
 	import settingsStore, { Theme } from '$lib/store/SettingsStore';
-	import { getContext, onMount } from 'svelte';
-	import Button from './button/Button.svelte';
-	import Card from './Card.svelte';
+	import Button from '$lib/components/button/Button.svelte';
+	import Card from '$lib/components/Card.svelte';
 	import { GraphOptions, GraphType } from '$lib/store/filterStore/types';
-	import OptionRenderer from './OptionRenderer.svelte';
-	import Divider from './base/Divider.svelte';
+	import OptionRenderer from '$lib/components/OptionRenderer.svelte';
+	import Divider from '$lib/components/base/Divider.svelte';
 	import {
 		CameraIcon,
-		CodeIcon,
-		CpuIcon,
 		DatabaseIcon,
-		InfoIcon,
+		Edit2Icon,
 		LayersIcon,
-		MehIcon,
 		MoonIcon,
 		PlusIcon,
 		RefreshCcwIcon,
@@ -24,18 +20,18 @@
 		SunIcon,
 		XIcon
 	} from 'svelte-feather-icons';
-	import { ButtonColor, ButtonSize, ButtonVariant } from './button/type';
-	import Dialog, { DialogSize, getDialogContext } from './dialog/Dialog.svelte';
+	import { ButtonColor, ButtonSize, ButtonVariant } from '$lib/components/button/type';
+	import Dialog, { DialogSize, getDialogContext } from '$lib/components/dialog/Dialog.svelte';
 	import TableSelection, {
-		type DatasetSelectionEvent,
 		type TableSelectionEvent
-	} from './tableSelection/TableSelection.svelte';
-	import QueryEditor from './QueryEditor.svelte';
+	} from '$lib/views/tableSelection/TableSelection.svelte';
+	import QueryEditor from '$lib/views/QueryEditor.svelte';
 	import { fadeSlide } from '$lib/transitions/fadeSlide';
 	import { get } from 'svelte/store';
 	import notificationStore from '$lib/store/notificationStore';
-	import { getGraphContext, type GraphService } from './BasicGraph.svelte';
+	import { getGraphContext, type GraphService } from '$lib/views/CoreGraph.svelte';
 	import { imageFromGlContext } from '$lib/rendering/screenshot';
+	import SchemaMatter from './SchemaMatter.svelte';
 
 	const graphService: GraphService = getGraphContext();
 	let optionsStore: GraphOptions['optionsStore'] | undefined;
@@ -179,7 +175,16 @@
 				<ul>
 					{#each Object.entries($dataStore.tables) as [tableName, table]}
 						<li class="flex py-1 justify-between items-center">
-							<div>{table.displayName ?? table.name}</div>
+							<div class="flex gap-2">
+								<span>{table.displayName}</span>
+								<Dialog>
+									<Button slot="trigger" variant={ButtonVariant.DEFAULT} size={ButtonSize.SM}>
+										<Edit2Icon size="15" />
+									</Button>
+
+									<SchemaMatter initiallySelectedTable={table} />
+								</Dialog>
+							</div>
 							<Button
 								on:click={() => filterStore.removeTable(tableName)}
 								variant={ButtonVariant.LINK}
diff --git a/src/lib/components/QueryEditor.svelte b/src/lib/views/QueryEditor.svelte
similarity index 92%
rename from src/lib/components/QueryEditor.svelte
rename to src/lib/views/QueryEditor.svelte
index 1d86f79..59eae53 100644
--- a/src/lib/components/QueryEditor.svelte
+++ b/src/lib/views/QueryEditor.svelte
@@ -1,17 +1,17 @@
 <script lang="ts">
 	import { onMount, onDestroy } from 'svelte';
 
-	import CodeEditor from './CodeEditor.svelte';
+	import CodeEditor from '$lib/components/CodeEditor.svelte';
 	import type { editor } from 'monaco-editor';
-	import Button from './button/Button.svelte';
+	import Button from '$lib/components/button/Button.svelte';
 	import { dataStore } from '$lib/store/dataStore/DataStore';
-	import { ButtonColor } from './button/type';
+	import { ButtonColor } from '$lib/components/button/type';
 	import DropdownSelect, {
 		type DropdownSelectionEvent,
 		type OptionConstructor
-	} from './DropdownSelect.svelte';
+	} from '$lib/components/DropdownSelect.svelte';
 	import type { DbQueryHistoryItem } from '$lib/store/dataStore/types';
-	import Tag from './base/Tag.svelte';
+	import Tag from '$lib/components/base/Tag.svelte';
 
 	let currentQuery: ReturnType<typeof dataStore.executeQuery> | undefined = undefined;
 
@@ -20,7 +20,7 @@
 	export let storageKey = 'query-editor';
 	export let initialQuery = '';
 
-	onMount(async () => {
+	onMount(() => {
 		// Reload query if it was saved in the store
 		initialQuery = sessionStorage.getItem(storageKey) || initialQuery;
 
@@ -78,7 +78,7 @@
 				<p class="flex gap-2 mb-2">
 					Tables:
 					{#each Object.values($dataStore.tables) as table}
-						<Tag>{table.name}</Tag>
+						<Tag>{table.tableName}</Tag>
 					{/each}
 				</p>
 			</div>
diff --git a/src/lib/views/SchemaMatter.svelte b/src/lib/views/SchemaMatter.svelte
new file mode 100644
index 0000000..bac8345
--- /dev/null
+++ b/src/lib/views/SchemaMatter.svelte
@@ -0,0 +1,131 @@
+<script lang="ts">
+	import Card from '$lib/components/Card.svelte';
+	import DropdownSelect, {
+		type DropdownSelectionEvent,
+		type OptionConstructor
+	} from '$lib/components/DropdownSelect.svelte';
+	import H2 from '$lib/components/base/H2.svelte';
+	import H3 from '$lib/components/base/H3.svelte';
+	import Tag, { TagColor } from '$lib/components/base/Tag.svelte';
+	import Button from '$lib/components/button/Button.svelte';
+	import { ButtonSize, ButtonVariant } from '$lib/components/button/type';
+	import { dataStore } from '$lib/store/dataStore/DataStore';
+	import type { ILoadedTable, TableTransformation } from '$lib/store/dataStore/types';
+	import { PlusIcon, XIcon } from 'svelte-feather-icons';
+	import TransformerEditor, { type TransformerCreated } from './TransformerEditor.svelte';
+	export let initiallySelectedTable: ILoadedTable;
+
+	let table = initiallySelectedTable;
+
+	const typeToColor = (typename: string): TagColor => {
+		switch (typename) {
+			case 'string':
+				return TagColor.orange;
+			case 'number':
+				return TagColor.blue;
+			default:
+				return TagColor.default;
+		}
+	};
+
+	const isColumnNew = (columnName: string) =>
+		table.schema[columnName] != table.sourceSchema[columnName];
+
+	const addTransformer = (evt: TransformerCreated) => {
+		// NOTE: hack to force rerender
+		dataStore
+			.addPostProcessingTransformer(table, evt.detail.transformation)
+			.finally(() => (table = table));
+	};
+
+	const removeTransformer = (transformer: TableTransformation) => {
+		// NOTE: hack to force rerender
+		dataStore.removePostProcessingTransformer(table, transformer).finally(() => (table = table));
+	};
+
+	const tableSelectionOptionConstructor: OptionConstructor<ILoadedTable, ILoadedTable> = (
+		value,
+		index,
+		meta
+	) => {
+		return {
+			label: value.displayName,
+			value: value,
+			id: index,
+			initiallySelected: value === table
+		};
+	};
+
+	const onTableSelected = (evt: DropdownSelectionEvent<ILoadedTable>) => {
+		if (evt.detail.selected.length === 0) {
+			return;
+		}
+		console.log({ table, new: evt.detail.selected });
+		table = evt.detail.selected[0].value;
+	};
+</script>
+
+<div class="mb-4 flex gap-2 items-center">
+	<H2>Schema Mapper</H2><DropdownSelect
+		values={Object.values($dataStore.tables)}
+		singular
+		required
+		optionConstructor={tableSelectionOptionConstructor}
+		on:select={onTableSelected}
+	/>
+</div>
+<div class="grid md:grid-cols-3 gap-4">
+	<div>
+		<div class="flex mb-2 gap-4 items-center">
+			<H3>Transformations</H3>
+
+			<TransformerEditor on:created={addTransformer}>
+				<Button slot="trigger" size={ButtonSize.SM}>
+					<svelte:fragment slot="leading">
+						<PlusIcon size="14" />
+					</svelte:fragment>
+					Add Transformer</Button
+				>
+			</TransformerEditor>
+		</div>
+		<div class="max-h-[60vh] overflow-auto">
+			{#each table.transformations as transformation}
+				<Card
+					><h3 class="flex items-center font-bold mr-8">
+						<span class="pr-2">{transformation.name}</span>
+						{#if transformation.required}
+							<Tag color={TagColor.red}>required</Tag>
+						{/if}
+					</h3>
+
+					<p>{transformation.description}</p>
+					<div class="absolute right-0 top-2">
+						<Button
+							disabled={transformation.required}
+							on:click={() => removeTransformer(transformation)}
+							variant={ButtonVariant.LINK}><XIcon /></Button
+						>
+					</div>
+				</Card>
+			{/each}
+		</div>
+	</div>
+	<div>
+		<H3 class="mb-2">Input Schema</H3>
+		<div class="md:max-h-[60vh] overflow-auto">
+			{#each Object.entries(table.sourceSchema) as [key, value]}
+				<Tag color={typeToColor(value)}>{key}:<b>{value}</b></Tag>
+			{/each}
+		</div>
+	</div>
+	<div>
+		<H3 class="mb-2">Output Schema</H3>
+		<div class="md:max-h-[60vh] overflow-auto">
+			{#each Object.entries(table.schema) as [key, value]}
+				<Tag color={isColumnNew(key) ? TagColor.green : typeToColor(value)}
+					>{key}:<b>{value}</b></Tag
+				>
+			{/each}
+		</div>
+	</div>
+</div>
diff --git a/src/lib/views/TransformerEditor.svelte b/src/lib/views/TransformerEditor.svelte
new file mode 100644
index 0000000..4475585
--- /dev/null
+++ b/src/lib/views/TransformerEditor.svelte
@@ -0,0 +1,74 @@
+<script lang="ts" context="module">
+	import type { TableTransformation } from '$lib/store/dataStore/types';
+	import { TransformationType } from '$lib/store/dataStore/types';
+	export type TransformerCreated = CustomEvent<{
+		transformation: TableTransformation;
+	}>;
+</script>
+
+<script lang="ts">
+	import Input from '$lib/components/Input.svelte';
+	import MiniEditor from '$lib/components/MiniEditor.svelte';
+	import Button from '$lib/components/button/Button.svelte';
+	import { ButtonColor } from '$lib/components/button/type';
+	import Dialog from '$lib/components/dialog/Dialog.svelte';
+	import { createEventDispatcher } from 'svelte';
+
+	interface $$Events {
+		created: TransformerCreated;
+	}
+
+	const initialExample =
+		'ALTER TABLE "${tableName}" ADD COLUMN example TEXT; \nUPDATE "${tableName}" SET example = \'foobar\'';
+	const eventDispatcher = createEventDispatcher();
+
+	let query = initialExample;
+
+	let isOpen = false;
+
+	let defaultName = 'Untitled Transformation';
+
+	let transformerName = defaultName;
+
+	const createTransformation = () => {
+		eventDispatcher('created', {
+			transformation: {
+				type: TransformationType.SQL,
+				name: transformerName,
+				description: 'Custom SQL Transformer',
+				query
+			}
+		});
+
+		isOpen = false;
+	};
+
+	const onClose = () => {
+		isOpen = false;
+		transformerName = defaultName;
+	};
+</script>
+
+<button on:click={() => (isOpen = true)}>
+	<slot name="trigger" />
+</button>
+<Dialog dialogOpen={isOpen}>
+	<h2>Create custom transformer</h2>
+	<Input bind:value={transformerName} placeholder="Name" />
+	{transformerName}
+	<p class="mb-2">
+		Use DuckDB/SQL syntax, instead of table name use <span style="font-family: monospace;"
+			>$&#123;tableName&#125;</span
+		>
+	</p>
+	<MiniEditor
+		initialValue={initialExample}
+		on:change={(evt) => (query = evt.detail.content)}
+		class="min-h-[50vh]"
+	/>
+	<hr />
+	<div class="flex justify-end mt-2 gap-2">
+		<Button on:click={onClose}>Cancel</Button>
+		<Button color={ButtonColor.PRIMARY} on:click={createTransformation}>Create</Button>
+	</div>
+</Dialog>
diff --git a/src/lib/components/tableSelection/TableSelection.svelte b/src/lib/views/tableSelection/TableSelection.svelte
similarity index 85%
rename from src/lib/components/tableSelection/TableSelection.svelte
rename to src/lib/views/tableSelection/TableSelection.svelte
index 8ed4dc2..572f1e9 100644
--- a/src/lib/components/tableSelection/TableSelection.svelte
+++ b/src/lib/views/tableSelection/TableSelection.svelte
@@ -16,16 +16,17 @@
 <script lang="ts">
 	import filterStore from '$lib/store/filterStore/FilterStore';
 	import { createEventDispatcher } from 'svelte';
-	import DropZone from '../DropZone.svelte';
+	import DropZone from '$lib/components/DropZone.svelte';
+	import Input from '$lib/components/Input.svelte';
 	import DropdownSelect, {
 		type DropdownSelectionEvent,
 		type Option,
 		type OptionConstructor
-	} from '../DropdownSelect.svelte';
-	import Divider from '../base/Divider.svelte';
-	import Button from '../button/Button.svelte';
+	} from '$lib/components/DropdownSelect.svelte';
+	import Divider from '$lib/components/base/Divider.svelte';
+	import Button from '$lib/components/button/Button.svelte';
+	// FIXME: add new alias for types
 	import type { Dataset, DatasetItem } from '../../../dataset/types';
-	import { get } from 'svelte/store';
 
 	interface $$Events {
 		selectDataset: DatasetSelectionEvent;
@@ -138,11 +139,6 @@
 </div>
 <p class="mb-2">a CSV dataset from url</p>
 <form class="flex gap-2" on:submit={onUrlLoad}>
-	<input
-		bind:value={urlInput}
-		type="url"
-		placeholder="URL"
-		class="bg-gray-50 border border-gray-300 text-gray-900 text-sm rounded-lg focus:ring-blue-500 focus:border-blue-500 block w-full p-2.5 dark:bg-gray-700 dark:border-gray-600 dark:placeholder-gray-400 dark:text-white dark:focus:ring-blue-500 dark:focus:border-blue-500"
-	/>
+	<Input bind:value={urlInput} type="url" placeholder="URL" class="" />
 	<Button>Load</Button>
 </form>
diff --git a/src/routes/graph/[slug]/+page.svelte b/src/routes/graph/[slug]/+page.svelte
index 9274a2f..3f3e9e0 100644
--- a/src/routes/graph/[slug]/+page.svelte
+++ b/src/routes/graph/[slug]/+page.svelte
@@ -1,25 +1,22 @@
-<script context="module">
-</script>
-
 <script lang="ts">
-	import type { PageServerData } from './$types';
-	import BasicGraph, { type CameraState } from '$lib/components/BasicGraph.svelte';
-	import LoadingOverlay from '$lib/components/LoadingOverlay.svelte';
-	import { dataStore } from '$lib/store/dataStore/DataStore';
 	import { browser } from '$app/environment';
-	import { onMount } from 'svelte';
 	import { page } from '$app/stores';
-	import FilterSidebar from '$lib/components/FilterSidebar.svelte';
+	import CoreGraph, { type CameraState } from '$lib/views/CoreGraph.svelte';
 	import GridBackground from '$lib/components/GridBackground.svelte';
-	import filterStore from '$lib/store/filterStore/FilterStore';
-	import Minimap from '$lib/components/graph/Minimap.svelte';
-	import PlaneGraph from '$lib/components/graph/PlaneGraph.svelte';
-	import { PlaneGraphModel } from '$lib/store/filterStore/graphs/plane';
-	import { Euler, Vector3 } from 'three';
-	import type { GraphStateConfig } from '$lib/store/filterStore/types';
+	import LoadingOverlay from '$lib/components/LoadingOverlay.svelte';
 	import Button from '$lib/components/button/Button.svelte';
 	import { ButtonVariant } from '$lib/components/button/type';
+	import Minimap from '$lib/components/graph/Minimap.svelte';
+	import PlaneGraph from '$lib/components/graph/PlaneGraph.svelte';
+	import { dataStore } from '$lib/store/dataStore/DataStore';
+	import filterStore from '$lib/store/filterStore/FilterStore';
+	import { PlaneGraphModel } from '$lib/store/filterStore/graphs/plane';
+	import type { GraphStateConfig } from '$lib/store/filterStore/types';
+	import FilterSidebar from '$lib/views/FilterSidebar.svelte';
+	import { onMount } from 'svelte';
 	import { ArrowLeftCircleIcon } from 'svelte-feather-icons';
+	import { Euler, Vector3 } from 'three';
+	import type { PageServerData } from './$types';
 
 	export let data: PageServerData;
 	let loadedGraph: GraphStateConfig | undefined = undefined;
@@ -56,7 +53,7 @@
 </script>
 
 <div class="h-screen w-full relative">
-	<BasicGraph>
+	<CoreGraph>
 		<svelte:fragment slot="inner">
 			{#if $filterStore.graphOptions}
 				{#if $filterStore.graphOptions instanceof PlaneGraphModel}
@@ -68,7 +65,7 @@
 			{/if}
 		</svelte:fragment>
 		<FilterSidebar />
-	</BasicGraph>
+	</CoreGraph>
 	<div class="absolute left-3 top-4 text-lg font-bold flex gap-2 items-center">
 		<a href="/"><Button variant={ButtonVariant.LINK}><ArrowLeftCircleIcon /></Button></a>
 		<div>