Support graphics functions with side effects! (#1111)

* feat: graphics output options within dependencies query * feat: basic link to dependency resolution * feat(r-graphics-pkg): rudimentary side effect support
flowr-analysis · Oct 27, 2024 · dfb2e2a · dfb2e2a
1 parent 6580164
commit dfb2e2a
Show file tree

Hide file tree

Showing 26 changed files with 335 additions and 135 deletions.
diff --git a/src/core/steps/all/core/20-dataflow.ts b/src/core/steps/all/core/20-dataflow.ts
@@ -26,15 +26,15 @@ const staticDataflowCommon = {
 	dependencies: [ 'normalize' ],
 } as const;
 
-function legacyProcessor(results: { normalize?: NormalizedAst }, input: { request?: RParseRequests }) {
+function processor(results: { normalize?: NormalizedAst }, input: { request?: RParseRequests }) {
 	return produceDataFlowGraph(input.request as RParseRequests, results.normalize as NormalizedAst);
 }
 
 export const STATIC_DATAFLOW = {
 	...staticDataflowCommon,
 	humanReadableName: 'dataflow',
-	processor:         legacyProcessor,
+	processor,
 	requiredInput:     {
 	}
-} as const satisfies DeepReadonly<IPipelineStep<'dataflow', typeof legacyProcessor>>;
+} as const satisfies DeepReadonly<IPipelineStep<'dataflow', typeof processor>>;
 
diff --git a/src/dataflow/environments/built-in.ts b/src/dataflow/environments/built-in.ts
@@ -28,6 +28,10 @@ import type { ForceArguments } from '../internal/process/functions/call/common';
 import { processApply } from '../internal/process/functions/call/built-in/built-in-apply';
 import { registerBuiltInDefinitions } from './built-in-config';
 import { DefaultBuiltinConfig } from './default-builtin-config';
+import type { LinkTo } from '../../queries/catalog/call-context-query/call-context-query-format';
+
+
+
 
 export const BuiltIn = 'built-in';
 
@@ -58,12 +62,19 @@ export interface BuiltInIdentifierConstant<T = unknown> extends IdentifierRefere
 	value:     T
 }
 
+export interface DefaultBuiltInProcessorConfiguration extends ForceArguments {
+	readonly returnsNthArgument?:    number | 'last',
+	readonly cfg?:                   ExitPointType,
+	readonly readAllArguments?:      boolean,
+	readonly hasUnknownSideEffects?: boolean | LinkTo<RegExp | string>
+}
+
 function defaultBuiltInProcessor<OtherInfo>(
 	name: RSymbol<OtherInfo & ParentInformation>,
 	args: readonly RFunctionArgument<OtherInfo & ParentInformation>[],
 	rootId: NodeId,
 	data: DataflowProcessorInformation<OtherInfo & ParentInformation>,
-	config: { returnsNthArgument?: number | 'last', cfg?: ExitPointType, readAllArguments?: boolean, hasUnknownSideEffects?: boolean } & ForceArguments
+	config: DefaultBuiltInProcessorConfiguration
 ): DataflowInformation {
 	const { information: res, processedArguments } = processKnownFunctionCall({ name, args, rootId, data, forceArgs: config.forceArgs });
 	if(config.returnsNthArgument !== undefined) {
@@ -81,12 +92,17 @@ function defaultBuiltInProcessor<OtherInfo>(
 	}
 
 	if(config.hasUnknownSideEffects) {
-		res.graph.markIdForUnknownSideEffects(rootId);
+		if(typeof config.hasUnknownSideEffects !== 'boolean') {
+			res.graph.markIdForUnknownSideEffects(rootId, config.hasUnknownSideEffects);
+		} else {
+			res.graph.markIdForUnknownSideEffects(rootId);
+		}
 	}
 
 	if(config.cfg !== undefined) {
 		res.exitPoints = [...res.exitPoints, { type: config.cfg, nodeId: rootId, controlDependencies: data.controlDependencies }];
 	}
+
 	return res;
 }
 

diff --git a/src/dataflow/environments/default-builtin-config.ts b/src/dataflow/environments/default-builtin-config.ts
@@ -11,12 +11,22 @@ export const DefaultBuiltinConfig: BuiltInDefinitions = [
 	{
 		type:  'function',
 		names: [
-			'~', '+', '-', '*', '/', '^', '!', '?', '**', '==', '!=', '>', '<', '>=', '<=', '%%', '%/%', '%*%', '%in%', ':', 'list', 'c',
+			'~', '+', '-', '*', '/', '^', '!', '?', '**', '==', '!=', '>', '<', '>=', '<=', '%%', '%/%', '%*%', '%in%', ':', 'list',
 			'rep', 'seq', 'seq_len', 'seq_along', 'seq.int', 'gsub', 'which', 'class', 'dimnames', 'min', 'max',
 			'intersect', 'subset', 'match', 'sqrt', 'abs', 'round', 'floor', 'ceiling', 'signif', 'trunc', 'log', 'log10', 'log2', 'sum', 'mean',
-			'unique', 'paste', 'paste0', 'read.csv', 'stop', 'is.null', 'plot', 'numeric', 'as.character', 'as.integer', 'as.logical', 'as.numeric', 'as.matrix',
+			'unique', 'paste', 'paste0', 'read.csv', 'stop', 'is.null', 'numeric', 'as.character', 'as.integer', 'as.logical', 'as.numeric', 'as.matrix',
 			'rbind', 'nrow', 'ncol', 'tryCatch', 'expression', 'factor',
-			'missing', 'as.data.frame', 'data.frame', 'na.omit', 'rownames', 'names', 'order', 'length', 'any', 'dim', 'matrix', 'cbind', 'nchar', 't'
+			'missing', 'as.data.frame', 'data.frame', 'na.omit', 'rownames', 'names', 'order', 'length', 'any', 'dim', 'matrix', 'cbind', 'nchar',
+			'pdf', 'jpeg', 'png', 'windows', 'postscript', 'xfig', 'bitmap', 'pictex', 'cairo_pdf', 'svg', 'bmp', 'tiff', 'X11', 'quartz'
+		],
+		processor:       'builtin:default',
+		config:          { readAllArguments: true },
+		assumePrimitive: true
+	},
+	{
+		type:  'function',
+		names: [
+			'c', 't'
 		],
 		processor:       'builtin:default',
 		config:          { readAllArguments: true },
@@ -27,11 +37,17 @@ export const DefaultBuiltinConfig: BuiltInDefinitions = [
 	{ type: 'function', names: ['lapply', 'sapply', 'vapply'],                 processor: 'builtin:apply',               config: { indexOfFunction: 1, nameOfFunctionArgument: 'FUN' },                        assumePrimitive: false },
 	{ type: 'function', names: ['Lapply', 'Sapply', 'Vapply'],                 processor: 'builtin:apply',               config: { indexOfFunction: 1, nameOfFunctionArgument: 'FUN' },                        assumePrimitive: false }, /* functool wrappers */
 	{ type: 'function', names: ['apply', 'tapply', 'Tapply'],                  processor: 'builtin:apply',               config: { indexOfFunction: 2, nameOfFunctionArgument: 'FUN' },                        assumePrimitive: false },
-	{ type: 'function', names: ['print'],                                      processor: 'builtin:default',             config: { returnsNthArgument: 0, forceArgs: 'all' },                                  assumePrimitive: false },
+	{ type: 'function', names: ['print', 'message', 'warning'],                processor: 'builtin:default',             config: { returnsNthArgument: 0, forceArgs: 'all', hasUnknownSideEffects: { type: 'link-to-last-call', callName: /^sink$/ } },                                  assumePrimitive: false },
+	// graphics base
+	{ type:            'function', names:           ['plot', 'map', 'image', 'boxplot', 'barplot', 'matplot', 'hist', 'stem', 'density', 'smoothScatter', 'contour', 'persp'],
+		processor:       'builtin:default',             config:          { forceArgs: 'all', hasUnknownSideEffects: { type: 'link-to-last-call', callName: /^pdf|jpeg|png|windows|postscript|xfig|bitmap|pictex|cairo_pdf|svg|bmp|tiff|X11|quartz$/ } }, assumePrimitive: true },
+	// graphics addons
+	{ type:            'function', names:           ['points', 'abline', 'lines', 'text', 'legend', 'title', 'axis', 'polygon', 'polypath', 'pie', 'rect', 'segments', 'arrows', 'symbols', 'tiplabels'],
+		processor:       'builtin:default',             config:          { forceArgs: 'all', hasUnknownSideEffects: { type: 'link-to-last-call', callName: /^dev\.new|dev\.copy|plot|map|image|boxplot|barplot|matplot|hist|stem|density|smoothScatter|contour|persp$/ } }, assumePrimitive: true },
 	{ type: 'function', names: ['('],                                          processor: 'builtin:default',             config: { returnsNthArgument: 0 },                                                    assumePrimitive: true  },
 	{ type: 'function', names: ['load', 'load_all', 'setwd', 'set.seed'],      processor: 'builtin:default',             config: { hasUnknownSideEffects: true, forceArgs: [true] },                           assumePrimitive: false },
 	{ type: 'function', names: ['eval', 'body', 'formals', 'environment'],     processor: 'builtin:default',             config: { hasUnknownSideEffects: true, forceArgs: [true] },                           assumePrimitive: false },
-	{ type: 'function', names: ['cat'],                                        processor: 'builtin:default',             config: { forceArgs: 'all' },                                                         assumePrimitive: false },
+	{ type: 'function', names: ['cat'],                                        processor: 'builtin:default',             config: { forceArgs: 'all', hasUnknownSideEffects: { type: 'link-to-last-call', callName: /^sink$/ } },                                                         assumePrimitive: false },
 	{ type: 'function', names: ['switch'],                                     processor: 'builtin:default',             config: { forceArgs: [true] },                                                        assumePrimitive: false },
 	{ type: 'function', names: ['return'],                                     processor: 'builtin:default',             config: { returnsNthArgument: 0, cfg: ExitPointType.Return },                         assumePrimitive: false },
 	{ type: 'function', names: ['break'],                                      processor: 'builtin:default',             config: { cfg: ExitPointType.Break },                                                 assumePrimitive: false },

diff --git a/src/dataflow/extractor.ts b/src/dataflow/extractor.ts
@@ -17,6 +17,13 @@ import type { RParseRequest, RParseRequests } from '../r-bridge/retriever';
 import { requestFingerprint } from '../r-bridge/retriever';
 import { initializeCleanEnvironments } from './environments/environment';
 import { standaloneSourceFile } from './internal/process/functions/call/built-in/built-in-source';
+import type { DataflowGraph } from './graph/graph';
+import type { ControlFlowGraph } from '../util/cfg/cfg';
+import { extractCFG } from '../util/cfg/cfg';
+import { EdgeType } from './graph/edge';
+import {
+	identifyLinkToLastCallRelation
+} from '../queries/catalog/call-context-query/identify-link-to-last-call-relation';
 
 export const processors: DataflowProcessors<ParentInformation> = {
 	[RType.Number]:             processValue,
@@ -49,6 +56,27 @@ export const processors: DataflowProcessors<ParentInformation> = {
 	}, wrapArgumentsUnnamed(n.children, d.completeAst.idMap), n.info.id, d)
 };
 
+
+function resolveLinkToSideEffects(ast: NormalizedAst, graph: DataflowGraph) {
+	let cfg: ControlFlowGraph | undefined = undefined;
+	for(const s of graph.unknownSideEffects) {
+		if(typeof s !== 'object') {
+			continue;
+		}
+		if(!cfg) {
+			cfg = extractCFG(ast).graph;
+		}
+		/* this has to change whenever we add a new link to relations because we currently offer no abstraction for the type */
+		const potentials = identifyLinkToLastCallRelation(s.id, cfg, graph, s.linkTo.callName);
+		for(const pot of potentials) {
+			graph.addEdge(s.id, pot, EdgeType.Reads);
+		}
+		if(potentials.length > 0) {
+			graph.unknownSideEffects.delete(s);
+		}
+	}
+}
+
 export function produceDataFlowGraph<OtherInfo>(
 	request: RParseRequests,
 	ast:     NormalizedAst<OtherInfo & ParentInformation>
@@ -76,5 +104,6 @@ export function produceDataFlowGraph<OtherInfo>(
 		}
 	}
 
+	resolveLinkToSideEffects(ast, df.graph);
 	return df;
 }
diff --git a/src/dataflow/graph/dataflowgraph-builder.ts b/src/dataflow/graph/dataflowgraph-builder.ts
@@ -12,6 +12,8 @@ import { EmptyArgument } from '../../r-bridge/lang-4.x/ast/model/nodes/r-functio
 import { BuiltIn } from '../environments/built-in';
 import { EdgeType } from './edge';
 import type { ControlDependency } from '../info';
+import type { LinkTo } from '../../queries/catalog/call-context-query/call-context-query-format';
+import { DefaultBuiltinConfig } from '../environments/default-builtin-config';
 
 export function emptyGraph(idMap?: AstIdMap) {
 	return new DataflowGraphBuilder(idMap);
@@ -277,3 +279,11 @@ export class DataflowGraphBuilder extends DataflowGraph {
 		return this;
 	}
 }
+
+export function getBuiltInSideEffect(name: string): LinkTo<RegExp> | undefined {
+	const got = DefaultBuiltinConfig.find(e => e.names.includes(name));
+	if(got?.type !== 'function') {
+		return undefined;
+	}
+	return (got?.config as { hasUnknownSideEffects: LinkTo<RegExp> | undefined }).hasUnknownSideEffects;
+}
diff --git a/src/dataflow/graph/diff.ts b/src/dataflow/graph/diff.ts
@@ -130,7 +130,10 @@ function diffOutgoingEdges(ctx: DataflowDiffContext): void {
 
 function diffRootVertices(ctx: DataflowDiffContext): void {
 	setDifference(ctx.left.rootIds(), ctx.right.rootIds(), { ...ctx, position: `${ctx.position}Root vertices differ in graphs. ` });
-	setDifference(ctx.left.unknownSideEffects, ctx.right.unknownSideEffects, { ...ctx, position: `${ctx.position}Unknown side effects differ in graphs. ` });
+	setDifference(
+		new Set([...ctx.left.unknownSideEffects].map(n => typeof n === 'object' ? n.id : n)),
+		new Set([...ctx.right.unknownSideEffects].map(n => typeof n === 'object' ? n.id : n)),
+		{ ...ctx, position: `${ctx.position}Unknown side effects differ in graphs. ` });
 }
 
 

diff --git a/src/dataflow/graph/graph.ts b/src/dataflow/graph/graph.ts
@@ -24,6 +24,7 @@ import { cloneEnvironmentInformation } from '../environments/clone';
 import { jsonReplacer } from '../../util/json';
 import { BuiltIn } from '../environments/built-in';
 import { dataflowLogger } from '../logger';
+import type { LinkTo } from '../../queries/catalog/call-context-query/call-context-query-format';
 
 export type DataflowFunctionFlowInformation = Omit<DataflowInformation, 'graph' | 'exitPoints'>  & { graph: Set<NodeId> }
 
@@ -89,6 +90,8 @@ export interface DataflowGraphJson {
 	readonly edgeInformation:   [NodeId, [NodeId, DataflowGraphEdge][]][]
 }
 
+export type UnknownSidEffect = NodeId | { id: NodeId, linkTo: LinkTo<RegExp> }
+
 /**
  * The dataflow graph holds the dataflow information found within the given AST.
  * We differentiate the directed edges in {@link EdgeType} and the vertices indicated by {@link DataflowGraphVertexArgument}
@@ -106,8 +109,12 @@ export class DataflowGraph<
 > {
 	private static DEFAULT_ENVIRONMENT: REnvironmentInformation | undefined = undefined;
 	private _idMap:                     AstIdMap | undefined;
-	/* Set of vertices which have sideEffects that we do not know anything about */
-	private readonly _unknownSideEffects = new Set<NodeId>();
+	/*
+	 * Set of vertices which have sideEffects that we do not know anything about.
+	 * As a (temporary) solution until we have FD edges, a side effect may also store known target links
+	 * that have to be/should be resolved (as globals) as a separate pass before the df analysis ends.
+	 */
+	private readonly _unknownSideEffects = new Set<UnknownSidEffect>();
 
 	constructor(idMap: AstIdMap | undefined) {
 		DataflowGraph.DEFAULT_ENVIRONMENT ??= initializeCleanEnvironments();
@@ -173,7 +180,7 @@ export class DataflowGraph<
 	/**
 	 * Retrieves the set of vertices which have side effects that we do not know anything about.
 	 */
-	public get unknownSideEffects(): ReadonlySet<NodeId> {
+	public get unknownSideEffects(): Set<UnknownSidEffect> {
 		return this._unknownSideEffects;
 	}
 
@@ -401,7 +408,11 @@ export class DataflowGraph<
 	}
 
 	/** Marks the given node as having unknown side effects */
-	public markIdForUnknownSideEffects(id: NodeId): this {
+	public markIdForUnknownSideEffects(id: NodeId, target?: LinkTo<RegExp | string>): this {
+		if(target) {
+			this._unknownSideEffects.add({ id: normalizeIdToNumberIfPossible(id), linkTo: typeof target.callName === 'string' ? { ...target, callName: new RegExp(target.callName) } : target as LinkTo<RegExp> });
+			return this;
+		}
 		this._unknownSideEffects.add(normalizeIdToNumberIfPossible(id));
 		return this;
 	}

diff --git a/src/dataflow/processor.ts b/src/dataflow/processor.ts
@@ -12,7 +12,7 @@ import type { RParseRequest } from '../r-bridge/retriever';
 import type { RNode } from '../r-bridge/lang-4.x/ast/model/model';
 
 export interface DataflowProcessorInformation<OtherInfo> {
-	/**
+   /**
    * Initial and frozen ast-information
    */
 	readonly completeAst:         NormalizedAst<OtherInfo>
@@ -35,7 +35,7 @@ export interface DataflowProcessorInformation<OtherInfo> {
 	 */
 	readonly referenceChain:      string[]
 	/**
-	 * The chain of control-flow {@link NodeId}s that lead to the current node (e.g. of known ifs).
+	 * The chain of control-flow {@link NodeId}s that lead to the current node (e.g., of known ifs).
 	 */
 	readonly controlDependencies: ControlDependency[] | undefined
 }

diff --git a/src/documentation/data/server/doc-data-server-messages.ts b/src/documentation/data/server/doc-data-server-messages.ts
@@ -26,8 +26,8 @@ import {
 	responseQueryMessage
 } from '../../../cli/repl/server/messages/message-query';
 import { exampleQueryCode } from '../query/example-query-code';
-import { CallTargets } from '../../../queries/catalog/call-context-query/call-context-query-format';
 import { requestLineageMessage, responseLineageMessage } from '../../../cli/repl/server/messages/message-lineage';
+import { CallTargets } from '../../../queries/catalog/call-context-query/identify-link-to-last-call-relation';
 
 export function documentAllServerMessages() {
 
@@ -229,7 +229,7 @@ While the context is derived from the \`filename\`, we currently offer no way to
 	});
 
 	const deprecatedByQuery = `(<a href="${FlowrWikiBaseRef}/Query%20API">deprecated</a>)`;
-	
+
 	documentServerMessage({
 		title:                  'Slice',
 		type:                   'request',

diff --git a/src/documentation/print-query-wiki.ts b/src/documentation/print-query-wiki.ts
@@ -11,7 +11,6 @@ import {
 	showQuery,
 	tocForQueryType
 } from './doc-util/doc-query';
-import { CallTargets } from '../queries/catalog/call-context-query/call-context-query-format';
 import { describeSchema } from '../util/schema';
 import { markdownFormatter } from '../util/ansi';
 import { executeCallContextQueries } from '../queries/catalog/call-context-query/call-context-query-executor';
@@ -30,6 +29,7 @@ import { executeDependenciesQuery } from '../queries/catalog/dependencies-query/
 import { getReplCommand } from './doc-util/doc-cli-option';
 import { NewIssueUrl } from './doc-util/doc-issue';
 import { executeLocationMapQuery } from '../queries/catalog/location-map-query/location-map-query-executor';
+import { CallTargets } from '../queries/catalog/call-context-query/identify-link-to-last-call-relation';
 
 
 registerQueryDocumentation('call-context', {