From df587c53cef3c1e8dc8fd1d04064aa6ac31d69f3 Mon Sep 17 00:00:00 2001 From: snstrauss Date: Tue, 11 Mar 2025 12:13:49 +0200 Subject: [PATCH 01/16] removed empty useEffect, and unified practically duplicated useEffects that call layout service --- spark-ui/src/components/SqlFlow/SqlFlow.tsx | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/spark-ui/src/components/SqlFlow/SqlFlow.tsx b/spark-ui/src/components/SqlFlow/SqlFlow.tsx index 6a859af..385b2b4 100644 --- a/spark-ui/src/components/SqlFlow/SqlFlow.tsx +++ b/spark-ui/src/components/SqlFlow/SqlFlow.tsx @@ -31,16 +31,6 @@ const SqlFlow: FC<{ sparkSQL: EnrichedSparkSQL }> = ({ const graphFilter = useAppSelector((state) => state.general.sqlMode); const selectedStage = useAppSelector((state) => state.general.selectedStage); - React.useEffect(() => { - if (!sparkSQL) return; - const { layoutNodes, layoutEdges } = SqlLayoutService.SqlElementsToLayout( - sparkSQL, - graphFilter, - ); - - setNodes(layoutNodes); - }, [sparkSQL.metricUpdateId]); - useEffect(() => { if (!sparkSQL) return; const { layoutNodes, layoutEdges } = SqlLayoutService.SqlElementsToLayout( @@ -50,7 +40,7 @@ const SqlFlow: FC<{ sparkSQL: EnrichedSparkSQL }> = ({ setNodes(layoutNodes); setEdges(layoutEdges); - }, [sparkSQL.uniqueId, graphFilter]); + }, [sparkSQL.metricUpdateId, sparkSQL.uniqueId, graphFilter]); useEffect(() => { if (instance) { @@ -58,8 +48,6 @@ const SqlFlow: FC<{ sparkSQL: EnrichedSparkSQL }> = ({ } }, [instance, edges]); - useEffect(() => { }, [nodes]); - const onConnect = useCallback( (params: any) => setEdges((eds) => From c69e07e3a1c3f17399b3699c1e68c41ca7736b5c Mon Sep 17 00:00:00 2001 From: snstrauss Date: Wed, 12 Mar 2025 17:33:55 +0200 Subject: [PATCH 02/16] create common flow elements builders for sharing between flat layout and grouped layout. added logic for creating grouped nodes and edges --- .../SqlLayoutService/layoutServiceBuilders.ts | 108 ++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100644 spark-ui/src/components/SqlFlow/SqlLayoutService/layoutServiceBuilders.ts diff --git a/spark-ui/src/components/SqlFlow/SqlLayoutService/layoutServiceBuilders.ts b/spark-ui/src/components/SqlFlow/SqlLayoutService/layoutServiceBuilders.ts new file mode 100644 index 0000000..40fd422 --- /dev/null +++ b/spark-ui/src/components/SqlFlow/SqlLayoutService/layoutServiceBuilders.ts @@ -0,0 +1,108 @@ +import { Node } from "reactflow"; +import { v4 as uuidv4 } from "uuid"; +import { + EnrichedSparkSQL, + EnrichedSqlEdge, + EnrichedSqlNode, + GraphFilter, +} from "../../../interfaces/AppStore"; +import { StageNodeName } from "../StageNode"; + +const getPosition = (x = 0, y = 0) => ({ x, y }); +const getStageIdString = (id = "") => `stage-${id}`; + +export const getFlowNodes = ( + sql: EnrichedSparkSQL, + graphFilter: GraphFilter, +) => { + const { nodes } = sql; + const { nodesIds } = sql.filters[graphFilter]; + + return nodes + .filter((node) => nodesIds.includes(node.nodeId)) + .map((node: EnrichedSqlNode) => ({ + id: node.nodeId.toString(), + data: { sqlId: sql.id, node: node }, + type: StageNodeName, + position: getPosition(), + })); +}; + +export const getGroupNodes = (flowNodes: Node[]) => { + const allGroupsWithNodes = flowNodes.reduce((stageGroups, node) => { + const stageId = node.data.node.stage?.stageId; + + if (!stageId) return stageGroups; + + if (!stageGroups.has(stageId)) { + stageGroups.set(stageId, []); + } + + stageGroups.get(stageId)?.push(node); + + return stageGroups; + }, new Map()); + + return Array.from(allGroupsWithNodes) + .filter(([_, nodes]) => nodes.length > 1) + .map(([stageId, nodes]) => ({ + id: getStageIdString(stageId.toString()), + data: { + nodes, + }, + position: getPosition(), + })); +}; + +interface ToFlowEdgeParams { + fromId: string | number; + toId: string | number; +} +export const toFlowEdge = ({ fromId, toId }: ToFlowEdgeParams) => ({ + id: uuidv4(), + source: fromId.toString(), + animated: true, + target: toId.toString(), +}); + +const getNodeToGroupMap = (flowNodes: Node[]) => + flowNodes.reduce>((nodeToStageMap, node) => { + const stageId = node.data.node.stage?.stageId; + nodeToStageMap[node.id] = getStageIdString(stageId); + return nodeToStageMap; + }, {}); + +export const transformEdgesToGroupEdges = ( + flowNodes: Node[], + groupNodes: Node[], + originalEdges: EnrichedSqlEdge[], +) => { + const nodeIdToStageGroupId = getNodeToGroupMap(flowNodes); + + return originalEdges + .map(({ fromId, toId }) => { + const fromStageGroupId = nodeIdToStageGroupId[fromId]; + const toStageGroupId = nodeIdToStageGroupId[toId]; + + if (fromStageGroupId === toStageGroupId) return; + const fromStageGroup = groupNodes.find( + (node) => node.id === fromStageGroupId, + ); + const toStageGroup = groupNodes.find( + (node) => node.id === toStageGroupId, + ); + + const finalFromId = + fromStageGroup && fromStageGroup.data.nodes.length > 1 + ? fromStageGroupId + : String(fromId); + + const finalToId = + toStageGroup && toStageGroup.data.nodes.length > 1 + ? toStageGroupId + : String(toId); + + return toFlowEdge({ fromId: finalFromId, toId: finalToId }); + }) + .filter(Boolean); +}; \ No newline at end of file From 12d60c7a9adf5883a369d2cab262fb21f4fc11f8 Mon Sep 17 00:00:00 2001 From: snstrauss Date: Wed, 12 Mar 2025 17:34:36 +0200 Subject: [PATCH 03/16] moved dagre layout helper into it's own file, as preparation for addition of grouped layout helper --- .../SqlFlow/SqlLayoutService/dagreLayouts.ts | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 spark-ui/src/components/SqlFlow/SqlLayoutService/dagreLayouts.ts diff --git a/spark-ui/src/components/SqlFlow/SqlLayoutService/dagreLayouts.ts b/spark-ui/src/components/SqlFlow/SqlLayoutService/dagreLayouts.ts new file mode 100644 index 0000000..4758561 --- /dev/null +++ b/spark-ui/src/components/SqlFlow/SqlLayoutService/dagreLayouts.ts @@ -0,0 +1,41 @@ +import dagre from "dagre"; +import { Edge, Node, Position } from "reactflow"; + +const nodeWidth = 280; +const nodeHeight = 280; + +export const getFlatElementsLayout = ( + nodes: Node[], + edges: Edge[], +): { layoutNodes: Node[]; layoutEdges: Edge[] } => { + const dagreGraph = new dagre.graphlib.Graph(); + dagreGraph.setDefaultEdgeLabel(() => ({})); + dagreGraph.setGraph({ rankdir: "LR" }); + + nodes.forEach((node) => { + dagreGraph.setNode(node.id, { width: nodeWidth, height: nodeHeight }); + }); + + edges.forEach((edge) => { + dagreGraph.setEdge(edge.source, edge.target); + }); + + dagre.layout(dagreGraph); + + nodes.forEach((node) => { + const nodeWithPosition = dagreGraph.node(node.id); + node.targetPosition = Position.Left; + node.sourcePosition = Position.Right; + + // We are shifting the dagre node position (anchor=center center) to the top left + // so it matches the React Flow node anchor point (top left). + node.position = { + x: nodeWithPosition.x - nodeWidth / 2, + y: nodeWithPosition.y - nodeHeight / 2, + }; + + return node; + }); + + return { layoutNodes: nodes, layoutEdges: edges }; +}; \ No newline at end of file From eec6a45a70698ae33d5a7e8ca04cce94ae69bbbc Mon Sep 17 00:00:00 2001 From: snstrauss Date: Wed, 12 Mar 2025 17:35:41 +0200 Subject: [PATCH 04/16] moved SqlLayoutService into it's own folder, and it is now using the external builders for clarity. added method for getting grouped layout --- .../components/SqlFlow/SqlLayoutService.ts | 85 ------------------- .../SqlLayoutService/SqlLayoutService.ts | 48 +++++++++++ 2 files changed, 48 insertions(+), 85 deletions(-) delete mode 100644 spark-ui/src/components/SqlFlow/SqlLayoutService.ts create mode 100644 spark-ui/src/components/SqlFlow/SqlLayoutService/SqlLayoutService.ts diff --git a/spark-ui/src/components/SqlFlow/SqlLayoutService.ts b/spark-ui/src/components/SqlFlow/SqlLayoutService.ts deleted file mode 100644 index 54f3c65..0000000 --- a/spark-ui/src/components/SqlFlow/SqlLayoutService.ts +++ /dev/null @@ -1,85 +0,0 @@ -import dagre from "dagre"; -import { Edge, Node, Position } from "reactflow"; -import { v4 as uuidv4 } from "uuid"; -import { - EnrichedSparkSQL, - EnrichedSqlEdge, - EnrichedSqlNode, - GraphFilter, -} from "../../interfaces/AppStore"; -import { StageNodeName } from "./StageNode"; - -const nodeWidth = 280; -const nodeHeight = 280; - -const getLayoutedElements = ( - nodes: Node[], - edges: Edge[], -): { layoutNodes: Node[]; layoutEdges: Edge[] } => { - const dagreGraph = new dagre.graphlib.Graph(); - dagreGraph.setDefaultEdgeLabel(() => ({})); - dagreGraph.setGraph({ rankdir: "LR" }); - - nodes.forEach((node) => { - dagreGraph.setNode(node.id, { width: nodeWidth, height: nodeHeight }); - }); - - edges.forEach((edge) => { - dagreGraph.setEdge(edge.source, edge.target); - }); - - dagre.layout(dagreGraph); - - nodes.forEach((node) => { - const nodeWithPosition = dagreGraph.node(node.id); - node.targetPosition = Position.Left; - node.sourcePosition = Position.Right; - - // We are shifting the dagre node position (anchor=center center) to the top left - // so it matches the React Flow node anchor point (top left). - node.position = { - x: nodeWithPosition.x - nodeWidth / 2, - y: nodeWithPosition.y - nodeHeight / 2, - }; - - return node; - }); - - return { layoutNodes: nodes, layoutEdges: edges }; -}; - -class SqlLayoutService { - static SqlElementsToLayout( - sql: EnrichedSparkSQL, - graphFilter: GraphFilter, - ): { layoutNodes: Node[]; layoutEdges: Edge[] } { - const { nodesIds, edges } = sql.filters[graphFilter]; - - const flowNodes: Node[] = sql.nodes - .filter((node) => nodesIds.includes(node.nodeId)) - .map((node: EnrichedSqlNode) => { - return { - id: node.nodeId.toString(), - data: { sqlId: sql.id, node: node }, - type: StageNodeName, - position: { x: 0, y: 0 }, - }; - }); - const flowEdges: Edge[] = edges.map((edge: EnrichedSqlEdge) => { - return { - id: uuidv4(), - source: edge.fromId.toString(), - animated: true, - target: edge.toId.toString(), - }; - }); - - const { layoutNodes, layoutEdges } = getLayoutedElements( - flowNodes, - flowEdges, - ); - return { layoutNodes: layoutNodes, layoutEdges: layoutEdges }; - } -} - -export default SqlLayoutService; diff --git a/spark-ui/src/components/SqlFlow/SqlLayoutService/SqlLayoutService.ts b/spark-ui/src/components/SqlFlow/SqlLayoutService/SqlLayoutService.ts new file mode 100644 index 0000000..9ad662a --- /dev/null +++ b/spark-ui/src/components/SqlFlow/SqlLayoutService/SqlLayoutService.ts @@ -0,0 +1,48 @@ +import { Edge, Node } from "reactflow"; +import { EnrichedSparkSQL, GraphFilter } from "../../../interfaces/AppStore"; +import { getFlatElementsLayout } from "./dagreLayouts"; +import { + getFlowNodes, + getGroupNodes, + toFlowEdge, + transformEdgesToGroupEdges, +} from "./layoutServiceBuilders"; + +class SqlLayoutService { + static SqlElementsToLayout( + sql: EnrichedSparkSQL, + graphFilter: GraphFilter, + ): { layoutNodes: Node[]; layoutEdges: Edge[] } { + const { edges } = sql.filters[graphFilter]; + + const flowNodes = getFlowNodes(sql, graphFilter); + const flowEdges = edges.map(toFlowEdge); + + const { layoutNodes, layoutEdges } = getFlatElementsLayout( + flowNodes, + flowEdges, + ); + return { layoutNodes: layoutNodes, layoutEdges: layoutEdges }; + } + static SqlElementsToGroupedLayout( + sql: EnrichedSparkSQL, + graphFilter: GraphFilter, + ): { layoutNodes: Node[]; layoutEdges: Edge[] | [] } { + const { edges } = sql.filters[graphFilter]; + + const flowNodes = getFlowNodes(sql, graphFilter); + const flowGroupNodes = getGroupNodes(flowNodes); + const nodeAndGroupEdges = transformEdgesToGroupEdges( + flowNodes, + flowGroupNodes, + edges, + ); + + return { + layoutNodes: [...flowNodes, ...flowGroupNodes], + layoutEdges: nodeAndGroupEdges as Edge[], + }; + } +} + +export default SqlLayoutService; From 42d0c3ec1ac00e229b28249b3330ee5a6d6f5742 Mon Sep 17 00:00:00 2001 From: snstrauss Date: Wed, 12 Mar 2025 17:37:42 +0200 Subject: [PATCH 05/16] SqlFlow can now call for either flat or grouped nodes and edges layouts, based on mock configuration --- spark-ui/src/components/SqlFlow/SqlFlow.tsx | 34 +++++++++++++------ .../SqlLayoutService/SqlLayoutService.ts | 2 +- spark-ui/src/interfaces/AppStore.ts | 1 + spark-ui/src/reducers/SQLNodeStageReducer.ts | 1 + 4 files changed, 26 insertions(+), 12 deletions(-) diff --git a/spark-ui/src/components/SqlFlow/SqlFlow.tsx b/spark-ui/src/components/SqlFlow/SqlFlow.tsx index 385b2b4..37d881f 100644 --- a/spark-ui/src/components/SqlFlow/SqlFlow.tsx +++ b/spark-ui/src/components/SqlFlow/SqlFlow.tsx @@ -1,24 +1,37 @@ import React, { FC, useCallback, useEffect, useState } from "react"; import ReactFlow, { + addEdge, ConnectionLineType, Controls, ReactFlowInstance, - addEdge, useEdgesState, useNodesState, } from "reactflow"; -import { Box, Drawer, FormControl, InputLabel, MenuItem, Select } from "@mui/material"; +import { + Box, + Drawer, + FormControl, + InputLabel, + MenuItem, + Select, +} from "@mui/material"; import "reactflow/dist/style.css"; import { useAppDispatch, useAppSelector } from "../../Hooks"; import { EnrichedSparkSQL, GraphFilter } from "../../interfaces/AppStore"; -import { setSQLMode, setSelectedStage } from "../../reducers/GeneralSlice"; -import SqlLayoutService from "./SqlLayoutService"; +import { setSelectedStage, setSQLMode } from "../../reducers/GeneralSlice"; +import SqlLayoutService from "./SqlLayoutService/SqlLayoutService"; import StageIconDrawer from "./StageIconDrawer"; import { StageNode, StageNodeName } from "./StageNode"; const options = { hideAttribution: true }; -const nodeTypes = { [StageNodeName]: StageNode }; +const nodeTypes = { + [StageNodeName]: StageNode, +}; + +// this is a mock to allow for use of external configuration +// const shouldUseGroupedLayout = true; +const shouldUseGroupedLayout = false; const SqlFlow: FC<{ sparkSQL: EnrichedSparkSQL }> = ({ sparkSQL, @@ -33,10 +46,10 @@ const SqlFlow: FC<{ sparkSQL: EnrichedSparkSQL }> = ({ useEffect(() => { if (!sparkSQL) return; - const { layoutNodes, layoutEdges } = SqlLayoutService.SqlElementsToLayout( - sparkSQL, - graphFilter, - ); + + const { layoutNodes, layoutEdges } = shouldUseGroupedLayout + ? SqlLayoutService.SqlElementsToGroupedLayout(sparkSQL, graphFilter) + : SqlLayoutService.SqlElementsToFlatLayout(sparkSQL, graphFilter); setNodes(layoutNodes); setEdges(layoutEdges); @@ -109,8 +122,7 @@ const SqlFlow: FC<{ sparkSQL: EnrichedSparkSQL }> = ({ open={selectedStage !== undefined} onClose={() => dispatch(setSelectedStage({ selectedStage: undefined }))} > - + diff --git a/spark-ui/src/components/SqlFlow/SqlLayoutService/SqlLayoutService.ts b/spark-ui/src/components/SqlFlow/SqlLayoutService/SqlLayoutService.ts index 9ad662a..fae943e 100644 --- a/spark-ui/src/components/SqlFlow/SqlLayoutService/SqlLayoutService.ts +++ b/spark-ui/src/components/SqlFlow/SqlLayoutService/SqlLayoutService.ts @@ -9,7 +9,7 @@ import { } from "./layoutServiceBuilders"; class SqlLayoutService { - static SqlElementsToLayout( + static SqlElementsToFlatLayout( sql: EnrichedSparkSQL, graphFilter: GraphFilter, ): { layoutNodes: Node[]; layoutEdges: Edge[] } { diff --git a/spark-ui/src/interfaces/AppStore.ts b/spark-ui/src/interfaces/AppStore.ts index 20e5248..56bac5d 100644 --- a/spark-ui/src/interfaces/AppStore.ts +++ b/spark-ui/src/interfaces/AppStore.ts @@ -274,6 +274,7 @@ export interface EnrichedSqlNode { export interface SQLNodeExchangeStageData { type: "exchange"; + stageId: number; writeStage: number; readStage: number; status: string; diff --git a/spark-ui/src/reducers/SQLNodeStageReducer.ts b/spark-ui/src/reducers/SQLNodeStageReducer.ts index f816080..d91be0b 100644 --- a/spark-ui/src/reducers/SQLNodeStageReducer.ts +++ b/spark-ui/src/reducers/SQLNodeStageReducer.ts @@ -63,6 +63,7 @@ export function calculateSQLNodeStage(sql: EnrichedSparkSQL): EnrichedSparkSQL { ...node, stage: { type: "exchange", + stageId: node.nodeId, writeStage: previousNode.stage?.type === "onestage" ? previousNode.stage.stageId From f4e84bee651147781be46ed8a1637a07e84f556d Mon Sep 17 00:00:00 2001 From: snstrauss Date: Wed, 12 Mar 2025 17:55:59 +0200 Subject: [PATCH 06/16] added memoization to SummaryTab to prevent redundant re-renders --- spark-ui/src/tabs/SummaryTab.tsx | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/spark-ui/src/tabs/SummaryTab.tsx b/spark-ui/src/tabs/SummaryTab.tsx index 67e5717..30bdeac 100644 --- a/spark-ui/src/tabs/SummaryTab.tsx +++ b/spark-ui/src/tabs/SummaryTab.tsx @@ -1,7 +1,7 @@ import ArrowBackIcon from "@mui/icons-material/ArrowBack"; import BuildIcon from "@mui/icons-material/Build"; import { Box, Fade, IconButton, Tooltip, Typography } from "@mui/material"; -import * as React from "react"; +import React, { useEffect, useMemo, useState } from "react"; import SqlFlow from "../components/SqlFlow/SqlFlow"; import SqlTable from "../components/SqlTable/SqlTable"; import SummaryBar from "../components/SummaryBar"; @@ -13,19 +13,20 @@ import { getBaseAppUrl } from "../utils/UrlUtils"; export default function SummaryTab() { const sql = useAppSelector((state) => state.spark.sql); - const [selectedSqlId, setSelectedSqlId] = React.useState( + const [selectedSqlId, setSelectedSqlId] = useState( undefined, ); - const selectedSql = - selectedSqlId === undefined - ? undefined - : sql?.sqls.find((sql) => sql.id === selectedSqlId); - React.useEffect(() => { + const selectedSql = useMemo( + () => sql?.sqls.find((sql) => sql.id === selectedSqlId), + [selectedSqlId], + ); + + useEffect(() => { MixpanelService.TrackPageView(); }, []); - React.useEffect(() => { + useEffect(() => { function handleEscapeKey(event: KeyboardEvent) { if (event.code === "Escape") { setSelectedSqlId(undefined); From e0dac334947a717c5e394636bfb26be51ed05192 Mon Sep 17 00:00:00 2001 From: snstrauss Date: Wed, 12 Mar 2025 18:35:15 +0200 Subject: [PATCH 07/16] separated SqlLayoutService class into stand alone functions - no real need for class when using modules --- spark-ui/src/components/SqlFlow/SqlFlow.tsx | 13 ++-- .../SqlLayoutService/SqlLayoutService.ts | 68 ++++++++++--------- 2 files changed, 44 insertions(+), 37 deletions(-) diff --git a/spark-ui/src/components/SqlFlow/SqlFlow.tsx b/spark-ui/src/components/SqlFlow/SqlFlow.tsx index 37d881f..a91cedd 100644 --- a/spark-ui/src/components/SqlFlow/SqlFlow.tsx +++ b/spark-ui/src/components/SqlFlow/SqlFlow.tsx @@ -20,7 +20,10 @@ import "reactflow/dist/style.css"; import { useAppDispatch, useAppSelector } from "../../Hooks"; import { EnrichedSparkSQL, GraphFilter } from "../../interfaces/AppStore"; import { setSelectedStage, setSQLMode } from "../../reducers/GeneralSlice"; -import SqlLayoutService from "./SqlLayoutService/SqlLayoutService"; +import { + sqlElementsToFlatLayout, + sqlElementsToGroupedLayout, +} from "./SqlLayoutService/SqlLayoutService"; import StageIconDrawer from "./StageIconDrawer"; import { StageNode, StageNodeName } from "./StageNode"; @@ -48,8 +51,8 @@ const SqlFlow: FC<{ sparkSQL: EnrichedSparkSQL }> = ({ if (!sparkSQL) return; const { layoutNodes, layoutEdges } = shouldUseGroupedLayout - ? SqlLayoutService.SqlElementsToGroupedLayout(sparkSQL, graphFilter) - : SqlLayoutService.SqlElementsToFlatLayout(sparkSQL, graphFilter); + ? sqlElementsToGroupedLayout(sparkSQL, graphFilter) + : sqlElementsToFlatLayout(sparkSQL, graphFilter); setNodes(layoutNodes); setEdges(layoutEdges); @@ -120,7 +123,9 @@ const SqlFlow: FC<{ sparkSQL: EnrichedSparkSQL }> = ({ dispatch(setSelectedStage({ selectedStage: undefined }))} + onClose={() => + dispatch(setSelectedStage({ selectedStage: undefined })) + } > diff --git a/spark-ui/src/components/SqlFlow/SqlLayoutService/SqlLayoutService.ts b/spark-ui/src/components/SqlFlow/SqlLayoutService/SqlLayoutService.ts index fae943e..3cfbf54 100644 --- a/spark-ui/src/components/SqlFlow/SqlLayoutService/SqlLayoutService.ts +++ b/spark-ui/src/components/SqlFlow/SqlLayoutService/SqlLayoutService.ts @@ -8,41 +8,43 @@ import { transformEdgesToGroupEdges, } from "./layoutServiceBuilders"; -class SqlLayoutService { - static SqlElementsToFlatLayout( - sql: EnrichedSparkSQL, - graphFilter: GraphFilter, - ): { layoutNodes: Node[]; layoutEdges: Edge[] } { - const { edges } = sql.filters[graphFilter]; +export function sqlElementsToFlatLayout( + sql: EnrichedSparkSQL, + graphFilter: GraphFilter, +): { layoutNodes: Node[]; layoutEdges: Edge[] } { + const { edges } = sql.filters[graphFilter]; - const flowNodes = getFlowNodes(sql, graphFilter); - const flowEdges = edges.map(toFlowEdge); + const flowNodes = getFlowNodes(sql, graphFilter); + const flowEdges = edges.map(toFlowEdge); - const { layoutNodes, layoutEdges } = getFlatElementsLayout( - flowNodes, - flowEdges, - ); - return { layoutNodes: layoutNodes, layoutEdges: layoutEdges }; - } - static SqlElementsToGroupedLayout( - sql: EnrichedSparkSQL, - graphFilter: GraphFilter, - ): { layoutNodes: Node[]; layoutEdges: Edge[] | [] } { - const { edges } = sql.filters[graphFilter]; + const { layoutNodes, layoutEdges } = getFlatElementsLayout( + flowNodes, + flowEdges, + ); + return { layoutNodes: layoutNodes, layoutEdges: layoutEdges }; +} - const flowNodes = getFlowNodes(sql, graphFilter); - const flowGroupNodes = getGroupNodes(flowNodes); - const nodeAndGroupEdges = transformEdgesToGroupEdges( - flowNodes, - flowGroupNodes, - edges, - ); +export function sqlElementsToGroupedLayout( + sql: EnrichedSparkSQL, + graphFilter: GraphFilter, +): { layoutNodes: Node[]; layoutEdges: Edge[] | [] } { + const { edges } = sql.filters[graphFilter]; - return { - layoutNodes: [...flowNodes, ...flowGroupNodes], - layoutEdges: nodeAndGroupEdges as Edge[], - }; - } -} + const flowNodes = getFlowNodes(sql, graphFilter); + const flowGroupNodes = getGroupNodes(flowNodes); + const nodeAndGroupEdges = transformEdgesToGroupEdges( + flowNodes, + flowGroupNodes, + edges, + ); -export default SqlLayoutService; + const { layoutNodes, layoutEdges } = getFlatElementsLayout( + [...flowNodes, ...flowGroupNodes], + nodeAndGroupEdges as Edge[], + ); + + return { + layoutNodes: layoutNodes, + layoutEdges: layoutEdges, + }; +} From adeec6c9f2035b3bf0c858c22d2db515b6bb7fb6 Mon Sep 17 00:00:00 2001 From: snstrauss Date: Wed, 12 Mar 2025 19:16:35 +0200 Subject: [PATCH 08/16] adding internal edges --- spark-ui/src/components/SqlFlow/SqlFlow.tsx | 4 +-- .../SqlLayoutService/SqlLayoutService.ts | 5 +++- .../SqlLayoutService/layoutServiceBuilders.ts | 25 ++++++++++++++++++- 3 files changed, 30 insertions(+), 4 deletions(-) diff --git a/spark-ui/src/components/SqlFlow/SqlFlow.tsx b/spark-ui/src/components/SqlFlow/SqlFlow.tsx index a91cedd..b2db4a5 100644 --- a/spark-ui/src/components/SqlFlow/SqlFlow.tsx +++ b/spark-ui/src/components/SqlFlow/SqlFlow.tsx @@ -33,8 +33,8 @@ const nodeTypes = { }; // this is a mock to allow for use of external configuration -// const shouldUseGroupedLayout = true; -const shouldUseGroupedLayout = false; +const shouldUseGroupedLayout = true; +// const shouldUseGroupedLayout = false; const SqlFlow: FC<{ sparkSQL: EnrichedSparkSQL }> = ({ sparkSQL, diff --git a/spark-ui/src/components/SqlFlow/SqlLayoutService/SqlLayoutService.ts b/spark-ui/src/components/SqlFlow/SqlLayoutService/SqlLayoutService.ts index 3cfbf54..045c1a8 100644 --- a/spark-ui/src/components/SqlFlow/SqlLayoutService/SqlLayoutService.ts +++ b/spark-ui/src/components/SqlFlow/SqlLayoutService/SqlLayoutService.ts @@ -4,6 +4,7 @@ import { getFlatElementsLayout } from "./dagreLayouts"; import { getFlowNodes, getGroupNodes, + getInternalEdges, toFlowEdge, transformEdgesToGroupEdges, } from "./layoutServiceBuilders"; @@ -21,6 +22,7 @@ export function sqlElementsToFlatLayout( flowNodes, flowEdges, ); + return { layoutNodes: layoutNodes, layoutEdges: layoutEdges }; } @@ -32,6 +34,7 @@ export function sqlElementsToGroupedLayout( const flowNodes = getFlowNodes(sql, graphFilter); const flowGroupNodes = getGroupNodes(flowNodes); + const internalEdges = getInternalEdges(flowGroupNodes, edges); const nodeAndGroupEdges = transformEdgesToGroupEdges( flowNodes, flowGroupNodes, @@ -40,7 +43,7 @@ export function sqlElementsToGroupedLayout( const { layoutNodes, layoutEdges } = getFlatElementsLayout( [...flowNodes, ...flowGroupNodes], - nodeAndGroupEdges as Edge[], + [...nodeAndGroupEdges, ...internalEdges] as Edge[], ); return { diff --git a/spark-ui/src/components/SqlFlow/SqlLayoutService/layoutServiceBuilders.ts b/spark-ui/src/components/SqlFlow/SqlLayoutService/layoutServiceBuilders.ts index 40fd422..82cc42e 100644 --- a/spark-ui/src/components/SqlFlow/SqlLayoutService/layoutServiceBuilders.ts +++ b/spark-ui/src/components/SqlFlow/SqlLayoutService/layoutServiceBuilders.ts @@ -105,4 +105,27 @@ export const transformEdgesToGroupEdges = ( return toFlowEdge({ fromId: finalFromId, toId: finalToId }); }) .filter(Boolean); -}; \ No newline at end of file +}; + +export function getInternalEdges( + flowGroupNodes: Node[], + originalEdges: EnrichedSqlEdge[], +) { + return flowGroupNodes.map((node) => { + const groupNodes = node.data.nodes; + const groupNodeIds = new Set(groupNodes.map((node: Node) => node.id)); + + return originalEdges + .filter( + ({ fromId, toId }) => + groupNodeIds.has(fromId.toString()) && + groupNodeIds.has(toId.toString()), + ) + .map(({ fromId, toId }) => + toFlowEdge({ + fromId, + toId, + }), + ) + }).flat(); +} From 58e3c0c2aba79db297c853babb4029a15297260d Mon Sep 17 00:00:00 2001 From: snstrauss Date: Wed, 12 Mar 2025 19:19:45 +0200 Subject: [PATCH 09/16] added fix for when groupd id is 0 --- .../SqlLayoutService/layoutServiceBuilders.ts | 38 ++++++++++--------- 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/spark-ui/src/components/SqlFlow/SqlLayoutService/layoutServiceBuilders.ts b/spark-ui/src/components/SqlFlow/SqlLayoutService/layoutServiceBuilders.ts index 82cc42e..74c7c49 100644 --- a/spark-ui/src/components/SqlFlow/SqlLayoutService/layoutServiceBuilders.ts +++ b/spark-ui/src/components/SqlFlow/SqlLayoutService/layoutServiceBuilders.ts @@ -32,7 +32,7 @@ export const getGroupNodes = (flowNodes: Node[]) => { const allGroupsWithNodes = flowNodes.reduce((stageGroups, node) => { const stageId = node.data.node.stage?.stageId; - if (!stageId) return stageGroups; + if (stageId === undefined) return stageGroups; if (!stageGroups.has(stageId)) { stageGroups.set(stageId, []); @@ -111,21 +111,23 @@ export function getInternalEdges( flowGroupNodes: Node[], originalEdges: EnrichedSqlEdge[], ) { - return flowGroupNodes.map((node) => { - const groupNodes = node.data.nodes; - const groupNodeIds = new Set(groupNodes.map((node: Node) => node.id)); - - return originalEdges - .filter( - ({ fromId, toId }) => - groupNodeIds.has(fromId.toString()) && - groupNodeIds.has(toId.toString()), - ) - .map(({ fromId, toId }) => - toFlowEdge({ - fromId, - toId, - }), - ) - }).flat(); + return flowGroupNodes + .map((node) => { + const groupNodes = node.data.nodes; + const groupNodeIds = new Set(groupNodes.map((node: Node) => node.id)); + + return originalEdges + .filter( + ({ fromId, toId }) => + groupNodeIds.has(fromId.toString()) && + groupNodeIds.has(toId.toString()), + ) + .map(({ fromId, toId }) => + toFlowEdge({ + fromId, + toId, + }), + ); + }) + .flat(); } From efd2cfb61d13ea726e6ee853007237a99f6eed65 Mon Sep 17 00:00:00 2001 From: snstrauss Date: Thu, 13 Mar 2025 09:18:21 +0200 Subject: [PATCH 10/16] encapsulated logic for resolving groups and nodes edges in a function --- .../SqlLayoutService/layoutServiceBuilders.ts | 47 +++++++++++-------- 1 file changed, 28 insertions(+), 19 deletions(-) diff --git a/spark-ui/src/components/SqlFlow/SqlLayoutService/layoutServiceBuilders.ts b/spark-ui/src/components/SqlFlow/SqlLayoutService/layoutServiceBuilders.ts index 74c7c49..7cb92d7 100644 --- a/spark-ui/src/components/SqlFlow/SqlLayoutService/layoutServiceBuilders.ts +++ b/spark-ui/src/components/SqlFlow/SqlLayoutService/layoutServiceBuilders.ts @@ -72,6 +72,19 @@ const getNodeToGroupMap = (flowNodes: Node[]) => return nodeToStageMap; }, {}); +const getResolvedEdgeConnection = ( + nodeId: number, + nodeIdToStageGroupId: Record, + groupNodes: Node[], +) => { + const stageGroupId = nodeIdToStageGroupId[nodeId]; + const stageGroup = groupNodes.find((node) => node.id === stageGroupId); + + return stageGroup && stageGroup.data.nodes.length > 1 + ? stageGroupId + : String(nodeId); +}; + export const transformEdgesToGroupEdges = ( flowNodes: Node[], groupNodes: Node[], @@ -80,31 +93,27 @@ export const transformEdgesToGroupEdges = ( const nodeIdToStageGroupId = getNodeToGroupMap(flowNodes); return originalEdges - .map(({ fromId, toId }) => { + .filter(({ fromId, toId }) => { const fromStageGroupId = nodeIdToStageGroupId[fromId]; const toStageGroupId = nodeIdToStageGroupId[toId]; - if (fromStageGroupId === toStageGroupId) return; - const fromStageGroup = groupNodes.find( - (node) => node.id === fromStageGroupId, - ); - const toStageGroup = groupNodes.find( - (node) => node.id === toStageGroupId, + return fromStageGroupId !== toStageGroupId; + }) + .map(({ fromId, toId }) => { + const resolvedFromId = getResolvedEdgeConnection( + fromId, + nodeIdToStageGroupId, + groupNodes, ); - const finalFromId = - fromStageGroup && fromStageGroup.data.nodes.length > 1 - ? fromStageGroupId - : String(fromId); - - const finalToId = - toStageGroup && toStageGroup.data.nodes.length > 1 - ? toStageGroupId - : String(toId); + const resolvedToId = getResolvedEdgeConnection( + toId, + nodeIdToStageGroupId, + groupNodes, + ); - return toFlowEdge({ fromId: finalFromId, toId: finalToId }); - }) - .filter(Boolean); + return toFlowEdge({ fromId: resolvedFromId, toId: resolvedToId }); + }); }; export function getInternalEdges( From 4ebda5194c5f841ee4d436db34f32d22532701c8 Mon Sep 17 00:00:00 2001 From: snstrauss Date: Thu, 13 Mar 2025 17:32:38 +0200 Subject: [PATCH 11/16] added dagre layout function for getting positions of groups and inner nodes --- .../SqlLayoutService/SqlLayoutService.ts | 30 +++-- .../SqlFlow/SqlLayoutService/dagreLayouts.ts | 111 ++++++++++++++++-- .../SqlLayoutService/layoutServiceBuilders.ts | 77 +++++++----- spark-ui/src/components/SqlFlow/StageNode.tsx | 10 +- 4 files changed, 173 insertions(+), 55 deletions(-) diff --git a/spark-ui/src/components/SqlFlow/SqlLayoutService/SqlLayoutService.ts b/spark-ui/src/components/SqlFlow/SqlLayoutService/SqlLayoutService.ts index 045c1a8..8890c8d 100644 --- a/spark-ui/src/components/SqlFlow/SqlLayoutService/SqlLayoutService.ts +++ b/spark-ui/src/components/SqlFlow/SqlLayoutService/SqlLayoutService.ts @@ -1,10 +1,13 @@ import { Edge, Node } from "reactflow"; import { EnrichedSparkSQL, GraphFilter } from "../../../interfaces/AppStore"; -import { getFlatElementsLayout } from "./dagreLayouts"; +import { + getFlatElementsLayout, + getGroupedElementsLayout, +} from "./dagreLayouts"; import { getFlowNodes, - getGroupNodes, getInternalEdges, + getTopLevelNodes, toFlowEdge, transformEdgesToGroupEdges, } from "./layoutServiceBuilders"; @@ -23,7 +26,7 @@ export function sqlElementsToFlatLayout( flowEdges, ); - return { layoutNodes: layoutNodes, layoutEdges: layoutEdges }; + return { layoutNodes, layoutEdges }; } export function sqlElementsToGroupedLayout( @@ -33,21 +36,22 @@ export function sqlElementsToGroupedLayout( const { edges } = sql.filters[graphFilter]; const flowNodes = getFlowNodes(sql, graphFilter); - const flowGroupNodes = getGroupNodes(flowNodes); - const internalEdges = getInternalEdges(flowGroupNodes, edges); - const nodeAndGroupEdges = transformEdgesToGroupEdges( + const topLevelNodes = getTopLevelNodes(flowNodes); + const innerLevelEdges = getInternalEdges(topLevelNodes, edges); + const topLevelEdges = transformEdgesToGroupEdges( flowNodes, - flowGroupNodes, + topLevelNodes, edges, ); - const { layoutNodes, layoutEdges } = getFlatElementsLayout( - [...flowNodes, ...flowGroupNodes], - [...nodeAndGroupEdges, ...internalEdges] as Edge[], - ); + const { layoutNodes, layoutEdges } = getGroupedElementsLayout({ + topLevelNodes, + topLevelEdges, + innerLevelEdges, + }); return { - layoutNodes: layoutNodes, - layoutEdges: layoutEdges, + layoutNodes, + layoutEdges, }; } diff --git a/spark-ui/src/components/SqlFlow/SqlLayoutService/dagreLayouts.ts b/spark-ui/src/components/SqlFlow/SqlLayoutService/dagreLayouts.ts index 4758561..7a48a0f 100644 --- a/spark-ui/src/components/SqlFlow/SqlLayoutService/dagreLayouts.ts +++ b/spark-ui/src/components/SqlFlow/SqlLayoutService/dagreLayouts.ts @@ -1,16 +1,29 @@ import dagre from "dagre"; import { Edge, Node, Position } from "reactflow"; -const nodeWidth = 280; -const nodeHeight = 280; +const buildDagreGraph = (rankdir: "LR" | "TB" = "LR") => { + const dagreGraph = new dagre.graphlib.Graph(); + dagreGraph.setDefaultEdgeLabel(() => ({})); + dagreGraph.setGraph({ rankdir }); + + return dagreGraph; +}; + +const isNodeAGroup = (node: Node) => node.type === "group"; + +const nodeSize = 280; +const nodeWidth = nodeSize; +const nodeHeight = nodeSize; +const groupPadding = 40; +const nodeMargin = 30; +const groupWidth = nodeWidth + groupPadding * 2; +const groupHeight = nodeHeight + groupPadding * 2; export const getFlatElementsLayout = ( nodes: Node[], edges: Edge[], ): { layoutNodes: Node[]; layoutEdges: Edge[] } => { - const dagreGraph = new dagre.graphlib.Graph(); - dagreGraph.setDefaultEdgeLabel(() => ({})); - dagreGraph.setGraph({ rankdir: "LR" }); + const dagreGraph = buildDagreGraph(); nodes.forEach((node) => { dagreGraph.setNode(node.id, { width: nodeWidth, height: nodeHeight }); @@ -33,9 +46,91 @@ export const getFlatElementsLayout = ( x: nodeWithPosition.x - nodeWidth / 2, y: nodeWithPosition.y - nodeHeight / 2, }; - - return node; }); return { layoutNodes: nodes, layoutEdges: edges }; -}; \ No newline at end of file +}; + +interface GroupedElementsLayoutParams { + topLevelNodes: Node[]; + topLevelEdges: Edge[]; + innerLevelEdges: Edge[]; +} +export const getGroupedElementsLayout = ({ + topLevelNodes, + topLevelEdges, + innerLevelEdges, +}: GroupedElementsLayoutParams): { + layoutNodes: Node[]; + layoutEdges: Edge[]; +} => { + const topLevelGraph = buildDagreGraph(); + const innerLevelGraph = buildDagreGraph("TB"); + + topLevelNodes.forEach((node) => { + const nodeIsGroup = isNodeAGroup(node); + + topLevelGraph.setNode(node.id, { + width: nodeIsGroup ? groupWidth : nodeWidth, + height: nodeIsGroup ? groupHeight : nodeHeight, + }); + + if (nodeIsGroup) { + node.data.nodes.forEach((childNode: Node) => { + innerLevelGraph.setNode(childNode.id, { + width: nodeWidth, + height: nodeHeight, + }); + }); + } + }); + + const innerLevelNodes = topLevelNodes + .filter(isNodeAGroup) + .map((node) => node.data.nodes) + .flat(); + + topLevelEdges.forEach((edge) => { + topLevelGraph.setEdge(edge.source, edge.target); + }); + + innerLevelEdges.forEach((edge) => { + innerLevelGraph.setEdge(edge.source, edge.target); + }); + + dagre.layout(topLevelGraph); + dagre.layout(innerLevelGraph); + + topLevelNodes.forEach((topLevelNode) => { + const nodeWithPosition = topLevelGraph.node(topLevelNode.id); + topLevelNode.targetPosition = Position.Left; + topLevelNode.sourcePosition = Position.Right; + + topLevelNode.position = { + x: nodeWithPosition.x - nodeWidth / 2, + y: nodeWithPosition.y - nodeHeight / 2, + }; + + if (isNodeAGroup(topLevelNode)) { + topLevelNode.data.nodes.forEach((childNode: Node, index: number) => { + childNode.data.isChildNode = true; + + childNode.targetPosition = Position.Top; + childNode.sourcePosition = Position.Bottom; + + childNode.position = { + x: topLevelNode.position.x + groupPadding, + y: + topLevelNode.position.y + + index * (nodeHeight + nodeMargin) + + groupPadding, + }; + }); + } + }); + + return { + layoutNodes: [...topLevelNodes, ...innerLevelNodes], + layoutEdges: [...topLevelEdges, ...innerLevelEdges], + }; +}; diff --git a/spark-ui/src/components/SqlFlow/SqlLayoutService/layoutServiceBuilders.ts b/spark-ui/src/components/SqlFlow/SqlLayoutService/layoutServiceBuilders.ts index 7cb92d7..f0f7848 100644 --- a/spark-ui/src/components/SqlFlow/SqlLayoutService/layoutServiceBuilders.ts +++ b/spark-ui/src/components/SqlFlow/SqlLayoutService/layoutServiceBuilders.ts @@ -11,6 +11,33 @@ import { StageNodeName } from "../StageNode"; const getPosition = (x = 0, y = 0) => ({ x, y }); const getStageIdString = (id = "") => `stage-${id}`; +const toFlowNode = (node: EnrichedSqlNode, sqlId: string) => ({ + id: node.nodeId.toString(), + data: { sqlId, node: node }, + type: StageNodeName, + position: getPosition(), +}); + +const toFlowGroupNode = (nodes: Node[], stageId: number) => ({ + type: "group", + id: getStageIdString(stageId.toString()), + data: { + nodes, + }, + position: getPosition(), +}); + +interface ToFlowEdgeParams { + fromId: string | number; + toId: string | number; +} +export const toFlowEdge = ({ fromId, toId }: ToFlowEdgeParams) => ({ + id: uuidv4(), + source: fromId.toString(), + animated: true, + target: toId.toString(), +}); + export const getFlowNodes = ( sql: EnrichedSparkSQL, graphFilter: GraphFilter, @@ -20,15 +47,10 @@ export const getFlowNodes = ( return nodes .filter((node) => nodesIds.includes(node.nodeId)) - .map((node: EnrichedSqlNode) => ({ - id: node.nodeId.toString(), - data: { sqlId: sql.id, node: node }, - type: StageNodeName, - position: getPosition(), - })); + .map((node: EnrichedSqlNode) => toFlowNode(node, sql.id)); }; -export const getGroupNodes = (flowNodes: Node[]) => { +const buildAllNodeGroups = (flowNodes: Node[]): Map => { const allGroupsWithNodes = flowNodes.reduce((stageGroups, node) => { const stageId = node.data.node.stage?.stageId; @@ -43,27 +65,20 @@ export const getGroupNodes = (flowNodes: Node[]) => { return stageGroups; }, new Map()); - return Array.from(allGroupsWithNodes) - .filter(([_, nodes]) => nodes.length > 1) - .map(([stageId, nodes]) => ({ - id: getStageIdString(stageId.toString()), - data: { - nodes, - }, - position: getPosition(), - })); + return allGroupsWithNodes; }; -interface ToFlowEdgeParams { - fromId: string | number; - toId: string | number; -} -export const toFlowEdge = ({ fromId, toId }: ToFlowEdgeParams) => ({ - id: uuidv4(), - source: fromId.toString(), - animated: true, - target: toId.toString(), -}); +export const getTopLevelNodes = (flowNodes: Node[]): Node[] => { + const allGroupsWithNodes = buildAllNodeGroups(flowNodes); + + return Array.from(allGroupsWithNodes).map(([stageId, nodes]) => { + if (nodes.length === 1) { + return nodes[0]; + } else { + return toFlowGroupNode(nodes, stageId); + } + }); +}; const getNodeToGroupMap = (flowNodes: Node[]) => flowNodes.reduce>((nodeToStageMap, node) => { @@ -116,11 +131,12 @@ export const transformEdgesToGroupEdges = ( }); }; -export function getInternalEdges( - flowGroupNodes: Node[], +export const getInternalEdges = ( + topLevelNodes: Node[], originalEdges: EnrichedSqlEdge[], -) { - return flowGroupNodes +) => + topLevelNodes + .filter((node) => node.type === "group") .map((node) => { const groupNodes = node.data.nodes; const groupNodeIds = new Set(groupNodes.map((node: Node) => node.id)); @@ -139,4 +155,3 @@ export function getInternalEdges( ); }) .flat(); -} diff --git a/spark-ui/src/components/SqlFlow/StageNode.tsx b/spark-ui/src/components/SqlFlow/StageNode.tsx index 6cfc61f..68db44f 100644 --- a/spark-ui/src/components/SqlFlow/StageNode.tsx +++ b/spark-ui/src/components/SqlFlow/StageNode.tsx @@ -131,7 +131,7 @@ function handleAddedRemovedMetrics( } export const StageNode: FC<{ - data: { sqlId: string; node: EnrichedSqlNode }; + data: { sqlId: string; node: EnrichedSqlNode; isChildNode: boolean }; }> = ({ data }): JSX.Element => { const dispatch = useAppDispatch(); const alerts = useAppSelector((state) => state.spark.alerts); @@ -516,7 +516,11 @@ export const StageNode: FC<{ return ( <> - +
@@ -610,7 +614,7 @@ export const StageNode: FC<{ - + ); }; From 11229c4270414a48e1200869c7e01bd4dc9c9977 Mon Sep 17 00:00:00 2001 From: snstrauss Date: Thu, 13 Mar 2025 17:40:08 +0200 Subject: [PATCH 12/16] moved the various components of SqlFlow into directories --- spark-ui/src/components/SqlFlow/SqlFlow.tsx | 4 ++-- .../SqlLayoutService/layoutServiceBuilders.ts | 2 +- .../BytesDistributionChart.tsx | 0 .../DurationDistributionChart.tsx | 2 +- .../NumbersDistributionChart.tsx | 0 .../{ => drawerComponents}/StageIconDrawer.tsx | 12 ++++++------ .../StageNode}/StageIcon.tsx | 0 .../StageNode}/StageNode.tsx | 18 +++++++++--------- 8 files changed, 19 insertions(+), 19 deletions(-) rename spark-ui/src/components/SqlFlow/{ => drawerComponents}/BytesDistributionChart.tsx (100%) rename spark-ui/src/components/SqlFlow/{ => drawerComponents}/DurationDistributionChart.tsx (95%) rename spark-ui/src/components/SqlFlow/{ => drawerComponents}/NumbersDistributionChart.tsx (100%) rename spark-ui/src/components/SqlFlow/{ => drawerComponents}/StageIconDrawer.tsx (97%) rename spark-ui/src/components/SqlFlow/{ => flowComponents/StageNode}/StageIcon.tsx (100%) rename spark-ui/src/components/SqlFlow/{ => flowComponents/StageNode}/StageNode.tsx (97%) diff --git a/spark-ui/src/components/SqlFlow/SqlFlow.tsx b/spark-ui/src/components/SqlFlow/SqlFlow.tsx index b2db4a5..ff8a480 100644 --- a/spark-ui/src/components/SqlFlow/SqlFlow.tsx +++ b/spark-ui/src/components/SqlFlow/SqlFlow.tsx @@ -24,8 +24,8 @@ import { sqlElementsToFlatLayout, sqlElementsToGroupedLayout, } from "./SqlLayoutService/SqlLayoutService"; -import StageIconDrawer from "./StageIconDrawer"; -import { StageNode, StageNodeName } from "./StageNode"; +import StageIconDrawer from "./drawerComponents/StageIconDrawer"; +import { StageNode, StageNodeName } from "./flowComponents/StageNode/StageNode"; const options = { hideAttribution: true }; const nodeTypes = { diff --git a/spark-ui/src/components/SqlFlow/SqlLayoutService/layoutServiceBuilders.ts b/spark-ui/src/components/SqlFlow/SqlLayoutService/layoutServiceBuilders.ts index f0f7848..5627994 100644 --- a/spark-ui/src/components/SqlFlow/SqlLayoutService/layoutServiceBuilders.ts +++ b/spark-ui/src/components/SqlFlow/SqlLayoutService/layoutServiceBuilders.ts @@ -6,7 +6,7 @@ import { EnrichedSqlNode, GraphFilter, } from "../../../interfaces/AppStore"; -import { StageNodeName } from "../StageNode"; +import { StageNodeName } from "../flowComponents/StageNode/StageNode"; const getPosition = (x = 0, y = 0) => ({ x, y }); const getStageIdString = (id = "") => `stage-${id}`; diff --git a/spark-ui/src/components/SqlFlow/BytesDistributionChart.tsx b/spark-ui/src/components/SqlFlow/drawerComponents/BytesDistributionChart.tsx similarity index 100% rename from spark-ui/src/components/SqlFlow/BytesDistributionChart.tsx rename to spark-ui/src/components/SqlFlow/drawerComponents/BytesDistributionChart.tsx diff --git a/spark-ui/src/components/SqlFlow/DurationDistributionChart.tsx b/spark-ui/src/components/SqlFlow/drawerComponents/DurationDistributionChart.tsx similarity index 95% rename from spark-ui/src/components/SqlFlow/DurationDistributionChart.tsx rename to spark-ui/src/components/SqlFlow/drawerComponents/DurationDistributionChart.tsx index e437c6b..b673d4d 100644 --- a/spark-ui/src/components/SqlFlow/DurationDistributionChart.tsx +++ b/spark-ui/src/components/SqlFlow/drawerComponents/DurationDistributionChart.tsx @@ -2,7 +2,7 @@ import { ApexOptions } from "apexcharts"; import { duration } from "moment"; import React from "react"; import ReactApexChart from "react-apexcharts"; -import { humanizeTimeDiff } from "../../utils/FormatUtils"; +import { humanizeTimeDiff } from "../../../utils/FormatUtils"; export default function DurationDistributionChart({ durationDist, diff --git a/spark-ui/src/components/SqlFlow/NumbersDistributionChart.tsx b/spark-ui/src/components/SqlFlow/drawerComponents/NumbersDistributionChart.tsx similarity index 100% rename from spark-ui/src/components/SqlFlow/NumbersDistributionChart.tsx rename to spark-ui/src/components/SqlFlow/drawerComponents/NumbersDistributionChart.tsx diff --git a/spark-ui/src/components/SqlFlow/StageIconDrawer.tsx b/spark-ui/src/components/SqlFlow/drawerComponents/StageIconDrawer.tsx similarity index 97% rename from spark-ui/src/components/SqlFlow/StageIconDrawer.tsx rename to spark-ui/src/components/SqlFlow/drawerComponents/StageIconDrawer.tsx index ba8129f..d04efe6 100644 --- a/spark-ui/src/components/SqlFlow/StageIconDrawer.tsx +++ b/spark-ui/src/components/SqlFlow/drawerComponents/StageIconDrawer.tsx @@ -7,18 +7,18 @@ import { } from "@mui/material"; import { duration } from "moment"; import React from "react"; -import { useAppSelector } from "../../Hooks"; +import { useAppSelector } from "../../../Hooks"; import { SQLNodeExchangeStageData, SQLNodeStageData, SparkStageStore, -} from "../../interfaces/AppStore"; -import { humanFileSize, humanizeTimeDiff } from "../../utils/FormatUtils"; -import { BASE_CURRENT_PAGE } from "../../utils/UrlConsts"; -import { getBaseAppUrl } from "../../utils/UrlUtils"; +} from "../../../interfaces/AppStore"; +import { humanFileSize, humanizeTimeDiff } from "../../../utils/FormatUtils"; +import { BASE_CURRENT_PAGE } from "../../../utils/UrlConsts"; +import { getBaseAppUrl } from "../../../utils/UrlUtils"; import BytesDistributionChart from "./BytesDistributionChart"; -import DurationDistributionChart from "./DurationDistributionChart"; import NumbersDistributionChart from "./NumbersDistributionChart"; +import DurationDistributionChart from "./DurationDistributionChart"; const linkToStage = (stageId: number) => { window.open( diff --git a/spark-ui/src/components/SqlFlow/StageIcon.tsx b/spark-ui/src/components/SqlFlow/flowComponents/StageNode/StageIcon.tsx similarity index 100% rename from spark-ui/src/components/SqlFlow/StageIcon.tsx rename to spark-ui/src/components/SqlFlow/flowComponents/StageNode/StageIcon.tsx diff --git a/spark-ui/src/components/SqlFlow/StageNode.tsx b/spark-ui/src/components/SqlFlow/flowComponents/StageNode/StageNode.tsx similarity index 97% rename from spark-ui/src/components/SqlFlow/StageNode.tsx rename to spark-ui/src/components/SqlFlow/flowComponents/StageNode/StageNode.tsx index 68db44f..c25d8c2 100644 --- a/spark-ui/src/components/SqlFlow/StageNode.tsx +++ b/spark-ui/src/components/SqlFlow/flowComponents/StageNode/StageNode.tsx @@ -5,20 +5,20 @@ import React, { FC } from "react"; import SyntaxHighlighter from "react-syntax-highlighter"; import { a11yDark } from "react-syntax-highlighter/dist/esm/styles/hljs"; import { Handle, Position } from "reactflow"; -import { useAppDispatch, useAppSelector } from "../../Hooks"; -import { EnrichedSqlNode } from "../../interfaces/AppStore"; -import { SqlMetric } from "../../interfaces/SparkSQLs"; -import { setSelectedStage } from '../../reducers/GeneralSlice'; -import { truncateMiddle } from "../../reducers/PlanParsers/PlanParserUtils"; +import { useAppDispatch, useAppSelector } from "../../../../Hooks"; +import { EnrichedSqlNode } from "../../../../interfaces/AppStore"; +import { SqlMetric } from "../../../../interfaces/SparkSQLs"; +import { setSelectedStage } from '../../../../reducers/GeneralSlice'; +import { truncateMiddle } from "../../../../reducers/PlanParsers/PlanParserUtils"; import { calculatePercentage, humanFileSize, humanizeTimeDiff, parseBytesString, -} from "../../utils/FormatUtils"; -import AlertBadge, { TransperantTooltip } from "../AlertBadge/AlertBadge"; -import { ConditionalWrapper } from "../InfoBox/InfoBox"; -import StageIcon from "./StageIcon"; +} from "../../../../utils/FormatUtils"; +import AlertBadge, { TransperantTooltip } from "../../../AlertBadge/AlertBadge"; +import { ConditionalWrapper } from "../../../InfoBox/InfoBox"; +import StageIcon from './StageIcon'; import styles from "./node-style.module.css"; export const StageNodeName: string = "stageNode"; From 0fa57ef547c3b862e193dc78dad664fed54c99eb Mon Sep 17 00:00:00 2001 From: snstrauss Date: Thu, 13 Mar 2025 18:50:44 +0200 Subject: [PATCH 13/16] some fixes after moving of files --- .../SqlFlow/drawerComponents/BytesDistributionChart.tsx | 2 +- .../SqlFlow/flowComponents/StageNode/StageIcon.tsx | 6 +++--- .../{ => flowComponents/StageNode}/node-style.module.css | 0 3 files changed, 4 insertions(+), 4 deletions(-) rename spark-ui/src/components/SqlFlow/{ => flowComponents/StageNode}/node-style.module.css (100%) diff --git a/spark-ui/src/components/SqlFlow/drawerComponents/BytesDistributionChart.tsx b/spark-ui/src/components/SqlFlow/drawerComponents/BytesDistributionChart.tsx index 1573449..47b6f27 100644 --- a/spark-ui/src/components/SqlFlow/drawerComponents/BytesDistributionChart.tsx +++ b/spark-ui/src/components/SqlFlow/drawerComponents/BytesDistributionChart.tsx @@ -1,7 +1,7 @@ import { ApexOptions } from "apexcharts"; import React from "react"; import ReactApexChart from "react-apexcharts"; -import { humanFileSize } from "../../utils/FormatUtils"; +import { humanFileSize } from "../../../utils/FormatUtils"; export default function BytesDistributionChart({ bytesDist, diff --git a/spark-ui/src/components/SqlFlow/flowComponents/StageNode/StageIcon.tsx b/spark-ui/src/components/SqlFlow/flowComponents/StageNode/StageIcon.tsx index 5f3bff2..41e96da 100644 --- a/spark-ui/src/components/SqlFlow/flowComponents/StageNode/StageIcon.tsx +++ b/spark-ui/src/components/SqlFlow/flowComponents/StageNode/StageIcon.tsx @@ -7,12 +7,12 @@ import { Typography } from "@mui/material"; import React from "react"; -import { useAppSelector } from "../../Hooks"; +import { useAppSelector } from "../../../../Hooks"; import { SQLNodeExchangeStageData, SQLNodeStageData -} from "../../interfaces/AppStore"; -import ExceptionIcon from "../ExceptionIcon"; +} from '../../../../interfaces/AppStore'; +import ExceptionIcon from "../../../ExceptionIcon"; function CircularProgressWithLabel( props: CircularProgressProps & { value: number }, diff --git a/spark-ui/src/components/SqlFlow/node-style.module.css b/spark-ui/src/components/SqlFlow/flowComponents/StageNode/node-style.module.css similarity index 100% rename from spark-ui/src/components/SqlFlow/node-style.module.css rename to spark-ui/src/components/SqlFlow/flowComponents/StageNode/node-style.module.css From 84b90c83cabc2c1e08591f51b1d785240497abb3 Mon Sep 17 00:00:00 2001 From: snstrauss Date: Thu, 13 Mar 2025 18:51:38 +0200 Subject: [PATCH 14/16] added StageGroupNode components. since they are the ones that render their own child node, there no really anymore need for all the 'inner edges' logic --- spark-ui/src/components/SqlFlow/SqlFlow.tsx | 8 ++- .../SqlLayoutService/SqlLayoutService.ts | 3 - .../SqlFlow/SqlLayoutService/dagreLayouts.ts | 56 +++---------------- .../SqlLayoutService/layoutServiceBuilders.ts | 29 +--------- .../StageGroupNode/StageGroupNode.module.css | 15 +++++ .../StageGroupNode/StageGroupNode.tsx | 30 ++++++++++ .../flowComponents/StageNode/StageNode.tsx | 6 +- 7 files changed, 64 insertions(+), 83 deletions(-) create mode 100644 spark-ui/src/components/SqlFlow/flowComponents/StageGroupNode/StageGroupNode.module.css create mode 100644 spark-ui/src/components/SqlFlow/flowComponents/StageGroupNode/StageGroupNode.tsx diff --git a/spark-ui/src/components/SqlFlow/SqlFlow.tsx b/spark-ui/src/components/SqlFlow/SqlFlow.tsx index ff8a480..dcdd562 100644 --- a/spark-ui/src/components/SqlFlow/SqlFlow.tsx +++ b/spark-ui/src/components/SqlFlow/SqlFlow.tsx @@ -20,16 +20,20 @@ import "reactflow/dist/style.css"; import { useAppDispatch, useAppSelector } from "../../Hooks"; import { EnrichedSparkSQL, GraphFilter } from "../../interfaces/AppStore"; import { setSelectedStage, setSQLMode } from "../../reducers/GeneralSlice"; +import StageIconDrawer from "./drawerComponents/StageIconDrawer"; +import StageGroupNode, { + StageGroupNodeName, +} from "./flowComponents/StageGroupNode/StageGroupNode"; +import { StageNode, StageNodeName } from "./flowComponents/StageNode/StageNode"; import { sqlElementsToFlatLayout, sqlElementsToGroupedLayout, } from "./SqlLayoutService/SqlLayoutService"; -import StageIconDrawer from "./drawerComponents/StageIconDrawer"; -import { StageNode, StageNodeName } from "./flowComponents/StageNode/StageNode"; const options = { hideAttribution: true }; const nodeTypes = { [StageNodeName]: StageNode, + [StageGroupNodeName]: StageGroupNode, }; // this is a mock to allow for use of external configuration diff --git a/spark-ui/src/components/SqlFlow/SqlLayoutService/SqlLayoutService.ts b/spark-ui/src/components/SqlFlow/SqlLayoutService/SqlLayoutService.ts index 8890c8d..45db796 100644 --- a/spark-ui/src/components/SqlFlow/SqlLayoutService/SqlLayoutService.ts +++ b/spark-ui/src/components/SqlFlow/SqlLayoutService/SqlLayoutService.ts @@ -6,7 +6,6 @@ import { } from "./dagreLayouts"; import { getFlowNodes, - getInternalEdges, getTopLevelNodes, toFlowEdge, transformEdgesToGroupEdges, @@ -37,7 +36,6 @@ export function sqlElementsToGroupedLayout( const flowNodes = getFlowNodes(sql, graphFilter); const topLevelNodes = getTopLevelNodes(flowNodes); - const innerLevelEdges = getInternalEdges(topLevelNodes, edges); const topLevelEdges = transformEdgesToGroupEdges( flowNodes, topLevelNodes, @@ -47,7 +45,6 @@ export function sqlElementsToGroupedLayout( const { layoutNodes, layoutEdges } = getGroupedElementsLayout({ topLevelNodes, topLevelEdges, - innerLevelEdges, }); return { diff --git a/spark-ui/src/components/SqlFlow/SqlLayoutService/dagreLayouts.ts b/spark-ui/src/components/SqlFlow/SqlLayoutService/dagreLayouts.ts index 7a48a0f..ca1a7cf 100644 --- a/spark-ui/src/components/SqlFlow/SqlLayoutService/dagreLayouts.ts +++ b/spark-ui/src/components/SqlFlow/SqlLayoutService/dagreLayouts.ts @@ -1,5 +1,6 @@ import dagre from "dagre"; import { Edge, Node, Position } from "reactflow"; +import { isNodeAGroup } from "../flowComponents/StageGroupNode/StageGroupNode"; const buildDagreGraph = (rankdir: "LR" | "TB" = "LR") => { const dagreGraph = new dagre.graphlib.Graph(); @@ -9,15 +10,11 @@ const buildDagreGraph = (rankdir: "LR" | "TB" = "LR") => { return dagreGraph; }; -const isNodeAGroup = (node: Node) => node.type === "group"; - const nodeSize = 280; -const nodeWidth = nodeSize; -const nodeHeight = nodeSize; +export const nodeWidth = nodeSize; +export const nodeHeight = nodeSize; const groupPadding = 40; -const nodeMargin = 30; const groupWidth = nodeWidth + groupPadding * 2; -const groupHeight = nodeHeight + groupPadding * 2; export const getFlatElementsLayout = ( nodes: Node[], @@ -54,12 +51,10 @@ export const getFlatElementsLayout = ( interface GroupedElementsLayoutParams { topLevelNodes: Node[]; topLevelEdges: Edge[]; - innerLevelEdges: Edge[]; } export const getGroupedElementsLayout = ({ topLevelNodes, topLevelEdges, - innerLevelEdges, }: GroupedElementsLayoutParams): { layoutNodes: Node[]; layoutEdges: Edge[]; @@ -68,36 +63,16 @@ export const getGroupedElementsLayout = ({ const innerLevelGraph = buildDagreGraph("TB"); topLevelNodes.forEach((node) => { - const nodeIsGroup = isNodeAGroup(node); - topLevelGraph.setNode(node.id, { - width: nodeIsGroup ? groupWidth : nodeWidth, - height: nodeIsGroup ? groupHeight : nodeHeight, + width: isNodeAGroup(node) ? groupWidth : nodeWidth, + height: nodeHeight, }); - - if (nodeIsGroup) { - node.data.nodes.forEach((childNode: Node) => { - innerLevelGraph.setNode(childNode.id, { - width: nodeWidth, - height: nodeHeight, - }); - }); - } }); - const innerLevelNodes = topLevelNodes - .filter(isNodeAGroup) - .map((node) => node.data.nodes) - .flat(); - topLevelEdges.forEach((edge) => { topLevelGraph.setEdge(edge.source, edge.target); }); - innerLevelEdges.forEach((edge) => { - innerLevelGraph.setEdge(edge.source, edge.target); - }); - dagre.layout(topLevelGraph); dagre.layout(innerLevelGraph); @@ -110,27 +85,10 @@ export const getGroupedElementsLayout = ({ x: nodeWithPosition.x - nodeWidth / 2, y: nodeWithPosition.y - nodeHeight / 2, }; - - if (isNodeAGroup(topLevelNode)) { - topLevelNode.data.nodes.forEach((childNode: Node, index: number) => { - childNode.data.isChildNode = true; - - childNode.targetPosition = Position.Top; - childNode.sourcePosition = Position.Bottom; - - childNode.position = { - x: topLevelNode.position.x + groupPadding, - y: - topLevelNode.position.y + - index * (nodeHeight + nodeMargin) + - groupPadding, - }; - }); - } }); return { - layoutNodes: [...topLevelNodes, ...innerLevelNodes], - layoutEdges: [...topLevelEdges, ...innerLevelEdges], + layoutNodes: topLevelNodes, + layoutEdges: topLevelEdges, }; }; diff --git a/spark-ui/src/components/SqlFlow/SqlLayoutService/layoutServiceBuilders.ts b/spark-ui/src/components/SqlFlow/SqlLayoutService/layoutServiceBuilders.ts index 5627994..3e6f692 100644 --- a/spark-ui/src/components/SqlFlow/SqlLayoutService/layoutServiceBuilders.ts +++ b/spark-ui/src/components/SqlFlow/SqlLayoutService/layoutServiceBuilders.ts @@ -6,6 +6,7 @@ import { EnrichedSqlNode, GraphFilter, } from "../../../interfaces/AppStore"; +import { StageGroupNodeName } from "../flowComponents/StageGroupNode/StageGroupNode"; import { StageNodeName } from "../flowComponents/StageNode/StageNode"; const getPosition = (x = 0, y = 0) => ({ x, y }); @@ -19,9 +20,10 @@ const toFlowNode = (node: EnrichedSqlNode, sqlId: string) => ({ }); const toFlowGroupNode = (nodes: Node[], stageId: number) => ({ - type: "group", + type: StageGroupNodeName, id: getStageIdString(stageId.toString()), data: { + friendlyName: `Stage ${stageId}`, nodes, }, position: getPosition(), @@ -130,28 +132,3 @@ export const transformEdgesToGroupEdges = ( return toFlowEdge({ fromId: resolvedFromId, toId: resolvedToId }); }); }; - -export const getInternalEdges = ( - topLevelNodes: Node[], - originalEdges: EnrichedSqlEdge[], -) => - topLevelNodes - .filter((node) => node.type === "group") - .map((node) => { - const groupNodes = node.data.nodes; - const groupNodeIds = new Set(groupNodes.map((node: Node) => node.id)); - - return originalEdges - .filter( - ({ fromId, toId }) => - groupNodeIds.has(fromId.toString()) && - groupNodeIds.has(toId.toString()), - ) - .map(({ fromId, toId }) => - toFlowEdge({ - fromId, - toId, - }), - ); - }) - .flat(); diff --git a/spark-ui/src/components/SqlFlow/flowComponents/StageGroupNode/StageGroupNode.module.css b/spark-ui/src/components/SqlFlow/flowComponents/StageGroupNode/StageGroupNode.module.css new file mode 100644 index 0000000..de0a35e --- /dev/null +++ b/spark-ui/src/components/SqlFlow/flowComponents/StageGroupNode/StageGroupNode.module.css @@ -0,0 +1,15 @@ +.groupContainer { + background-color: #ffffff50; + border-radius: 15px; + padding: 20px; + + display: flex; + flex-direction: column; + align-items: center; + justify-content: center; + gap: 20px; +} + +.groupName { + text-decoration: underline; +} diff --git a/spark-ui/src/components/SqlFlow/flowComponents/StageGroupNode/StageGroupNode.tsx b/spark-ui/src/components/SqlFlow/flowComponents/StageGroupNode/StageGroupNode.tsx new file mode 100644 index 0000000..5b2ae83 --- /dev/null +++ b/spark-ui/src/components/SqlFlow/flowComponents/StageGroupNode/StageGroupNode.tsx @@ -0,0 +1,30 @@ +import React from "react"; +import { Handle, Node, Position } from "reactflow"; +import { StageNode } from "../StageNode/StageNode"; +import styles from "./StageGroupNode.module.css"; + +export const StageGroupNodeName: string = "stageGroupNode"; + +export const isNodeAGroup = (node: Node) => node.type === StageGroupNodeName; + +interface StageGroupNodeProps { + data: { + nodes: Node[]; + friendlyName: string; + }; +} + +export default function StageGroupNode({ data }: StageGroupNodeProps) { + const { friendlyName, nodes } = data; + + return ( +
+ +

{friendlyName}

+ {nodes.map((node) => { + return ; + })} + +
+ ); +} diff --git a/spark-ui/src/components/SqlFlow/flowComponents/StageNode/StageNode.tsx b/spark-ui/src/components/SqlFlow/flowComponents/StageNode/StageNode.tsx index c25d8c2..902b411 100644 --- a/spark-ui/src/components/SqlFlow/flowComponents/StageNode/StageNode.tsx +++ b/spark-ui/src/components/SqlFlow/flowComponents/StageNode/StageNode.tsx @@ -131,7 +131,7 @@ function handleAddedRemovedMetrics( } export const StageNode: FC<{ - data: { sqlId: string; node: EnrichedSqlNode; isChildNode: boolean }; + data: { sqlId: string; node: EnrichedSqlNode; }; }> = ({ data }): JSX.Element => { const dispatch = useAppDispatch(); const alerts = useAppSelector((state) => state.spark.alerts); @@ -518,7 +518,7 @@ export const StageNode: FC<{ <> @@ -614,7 +614,7 @@ export const StageNode: FC<{ - + ); }; From 5ffad1b3ce231e12058f272ad57c01c297042715 Mon Sep 17 00:00:00 2001 From: snstrauss Date: Thu, 13 Mar 2025 19:05:27 +0200 Subject: [PATCH 15/16] no more need for duplications of sqlToLayoutElements since both functions basically do the same now, only with different building of nodes and edges --- spark-ui/src/components/SqlFlow/SqlFlow.tsx | 13 +++-- .../SqlLayoutService/SqlLayoutService.ts | 44 +++++++--------- .../SqlFlow/SqlLayoutService/dagreLayouts.ts | 52 ++----------------- 3 files changed, 30 insertions(+), 79 deletions(-) diff --git a/spark-ui/src/components/SqlFlow/SqlFlow.tsx b/spark-ui/src/components/SqlFlow/SqlFlow.tsx index dcdd562..de84dab 100644 --- a/spark-ui/src/components/SqlFlow/SqlFlow.tsx +++ b/spark-ui/src/components/SqlFlow/SqlFlow.tsx @@ -25,10 +25,7 @@ import StageGroupNode, { StageGroupNodeName, } from "./flowComponents/StageGroupNode/StageGroupNode"; import { StageNode, StageNodeName } from "./flowComponents/StageNode/StageNode"; -import { - sqlElementsToFlatLayout, - sqlElementsToGroupedLayout, -} from "./SqlLayoutService/SqlLayoutService"; +import { sqlElementsToLayout } from "./SqlLayoutService/SqlLayoutService"; const options = { hideAttribution: true }; const nodeTypes = { @@ -54,9 +51,11 @@ const SqlFlow: FC<{ sparkSQL: EnrichedSparkSQL }> = ({ useEffect(() => { if (!sparkSQL) return; - const { layoutNodes, layoutEdges } = shouldUseGroupedLayout - ? sqlElementsToGroupedLayout(sparkSQL, graphFilter) - : sqlElementsToFlatLayout(sparkSQL, graphFilter); + const { layoutNodes, layoutEdges } = sqlElementsToLayout( + sparkSQL, + graphFilter, + shouldUseGroupedLayout, + ); setNodes(layoutNodes); setEdges(layoutEdges); diff --git a/spark-ui/src/components/SqlFlow/SqlLayoutService/SqlLayoutService.ts b/spark-ui/src/components/SqlFlow/SqlLayoutService/SqlLayoutService.ts index 45db796..f4e3daa 100644 --- a/spark-ui/src/components/SqlFlow/SqlLayoutService/SqlLayoutService.ts +++ b/spark-ui/src/components/SqlFlow/SqlLayoutService/SqlLayoutService.ts @@ -1,9 +1,5 @@ -import { Edge, Node } from "reactflow"; import { EnrichedSparkSQL, GraphFilter } from "../../../interfaces/AppStore"; -import { - getFlatElementsLayout, - getGroupedElementsLayout, -} from "./dagreLayouts"; +import { getLayoutElements } from "./dagreLayouts"; import { getFlowNodes, getTopLevelNodes, @@ -11,27 +7,21 @@ import { transformEdgesToGroupEdges, } from "./layoutServiceBuilders"; -export function sqlElementsToFlatLayout( +const getFlatFlowElements = ( sql: EnrichedSparkSQL, graphFilter: GraphFilter, -): { layoutNodes: Node[]; layoutEdges: Edge[] } { +) => { const { edges } = sql.filters[graphFilter]; - const flowNodes = getFlowNodes(sql, graphFilter); const flowEdges = edges.map(toFlowEdge); - const { layoutNodes, layoutEdges } = getFlatElementsLayout( - flowNodes, - flowEdges, - ); - - return { layoutNodes, layoutEdges }; -} + return { flowNodes, flowEdges }; +}; -export function sqlElementsToGroupedLayout( +const getGroupedFlowElements = ( sql: EnrichedSparkSQL, graphFilter: GraphFilter, -): { layoutNodes: Node[]; layoutEdges: Edge[] | [] } { +) => { const { edges } = sql.filters[graphFilter]; const flowNodes = getFlowNodes(sql, graphFilter); @@ -42,13 +32,17 @@ export function sqlElementsToGroupedLayout( edges, ); - const { layoutNodes, layoutEdges } = getGroupedElementsLayout({ - topLevelNodes, - topLevelEdges, - }); + return { flowNodes: topLevelNodes, flowEdges: topLevelEdges }; +}; + +export function sqlElementsToLayout( + sql: EnrichedSparkSQL, + graphFilter: GraphFilter, + useGroupedLayout: boolean, +) { + const { flowNodes, flowEdges } = useGroupedLayout + ? getGroupedFlowElements(sql, graphFilter) + : getFlatFlowElements(sql, graphFilter); - return { - layoutNodes, - layoutEdges, - }; + return getLayoutElements(flowNodes, flowEdges); } diff --git a/spark-ui/src/components/SqlFlow/SqlLayoutService/dagreLayouts.ts b/spark-ui/src/components/SqlFlow/SqlLayoutService/dagreLayouts.ts index ca1a7cf..51386ac 100644 --- a/spark-ui/src/components/SqlFlow/SqlLayoutService/dagreLayouts.ts +++ b/spark-ui/src/components/SqlFlow/SqlLayoutService/dagreLayouts.ts @@ -16,14 +16,17 @@ export const nodeHeight = nodeSize; const groupPadding = 40; const groupWidth = nodeWidth + groupPadding * 2; -export const getFlatElementsLayout = ( +export const getLayoutElements = ( nodes: Node[], edges: Edge[], ): { layoutNodes: Node[]; layoutEdges: Edge[] } => { const dagreGraph = buildDagreGraph(); nodes.forEach((node) => { - dagreGraph.setNode(node.id, { width: nodeWidth, height: nodeHeight }); + dagreGraph.setNode(node.id, { + width: isNodeAGroup(node) ? groupWidth : nodeWidth, + height: nodeHeight, + }); }); edges.forEach((edge) => { @@ -47,48 +50,3 @@ export const getFlatElementsLayout = ( return { layoutNodes: nodes, layoutEdges: edges }; }; - -interface GroupedElementsLayoutParams { - topLevelNodes: Node[]; - topLevelEdges: Edge[]; -} -export const getGroupedElementsLayout = ({ - topLevelNodes, - topLevelEdges, -}: GroupedElementsLayoutParams): { - layoutNodes: Node[]; - layoutEdges: Edge[]; -} => { - const topLevelGraph = buildDagreGraph(); - const innerLevelGraph = buildDagreGraph("TB"); - - topLevelNodes.forEach((node) => { - topLevelGraph.setNode(node.id, { - width: isNodeAGroup(node) ? groupWidth : nodeWidth, - height: nodeHeight, - }); - }); - - topLevelEdges.forEach((edge) => { - topLevelGraph.setEdge(edge.source, edge.target); - }); - - dagre.layout(topLevelGraph); - dagre.layout(innerLevelGraph); - - topLevelNodes.forEach((topLevelNode) => { - const nodeWithPosition = topLevelGraph.node(topLevelNode.id); - topLevelNode.targetPosition = Position.Left; - topLevelNode.sourcePosition = Position.Right; - - topLevelNode.position = { - x: nodeWithPosition.x - nodeWidth / 2, - y: nodeWithPosition.y - nodeHeight / 2, - }; - }); - - return { - layoutNodes: topLevelNodes, - layoutEdges: topLevelEdges, - }; -}; From d46bc51c1636ca7a9d0d098a4c1b0d56865bf175 Mon Sep 17 00:00:00 2001 From: snstrauss Date: Sun, 16 Mar 2025 10:30:31 +0200 Subject: [PATCH 16/16] turn off 'grouping by stage' mock configuration --- spark-ui/src/components/SqlFlow/SqlFlow.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spark-ui/src/components/SqlFlow/SqlFlow.tsx b/spark-ui/src/components/SqlFlow/SqlFlow.tsx index de84dab..729782c 100644 --- a/spark-ui/src/components/SqlFlow/SqlFlow.tsx +++ b/spark-ui/src/components/SqlFlow/SqlFlow.tsx @@ -34,8 +34,8 @@ const nodeTypes = { }; // this is a mock to allow for use of external configuration -const shouldUseGroupedLayout = true; -// const shouldUseGroupedLayout = false; +// const shouldUseGroupedLayout = true; +const shouldUseGroupedLayout = false; const SqlFlow: FC<{ sparkSQL: EnrichedSparkSQL }> = ({ sparkSQL,