Skip to content

Free ForAll and Self Modifying Code: ForAll #3043

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
gregli-msft wants to merge 8 commits into
base: gregli/free-self-base
Choose a base branch
from
Open

Free ForAll and Self Modifying Code: ForAll #3043

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
5b8922c
Updates
gregli-msft Mar 14, 2026
70f18e2
Merge branch 'gregli/free-self-base' of https://github.com/microsoft/…
gregli-msft Mar 17, 2026
8cec647
Merge branch 'gregli/free-self-base' of https://github.com/microsoft/…
gregli-msft Mar 17, 2026
745bfcf
Merge branch 'gregli/free-self-base' of https://github.com/microsoft/…
gregli-msft Mar 18, 2026
2dfb819
Updates
gregli-msft Mar 20, 2026
ef758dc
Merge branch 'gregli/free-self-base' of https://github.com/microsoft/…
gregli-msft Mar 20, 2026
7910a82
Merge branch 'gregli/free-self-base' of https://github.com/microsoft/…
gregli-msft Apr 24, 2026
29935df
Merge branch 'gregli/free-self-base' of https://github.com/microsoft/…
gregli-msft May 2, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 12 additions & 4 deletions src/libraries/Microsoft.PowerFx.Core/Functions/FunctionScopeInfo.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,13 @@ internal class FunctionScopeInfo
/// the scope data source, applying the lambda. This is used to determine what
/// default behavior to block (such as refusing lambdas that modify the scope).
/// </summary>
public bool IteratesOverScope { get; }
public bool IteratesOverScope { get; }

/// <summary>
/// True indicates that the function performs iteration in a deterministic manner,
/// for example ForAll with one record after another in order, evaluated exactly once per record.
/// </summary>
public bool IteratesOnceEachInOrder { get; }

/// <summary>
/// Null if this is a row scope, but if it's a constant row scope this will
Expand All @@ -66,7 +72,7 @@ internal class FunctionScopeInfo
// True indicates that this function cannot guarantee that it will iterate over the datasource in order.
// This means it should not allow lambdas that operate on the same data multiple times, as this will
// cause nondeterministic behavior.
public bool HasNondeterministicOperationOrder => IteratesOverScope && SupportsAsyncLambdas;
public bool HasNondeterministicOperationOrder => IteratesOverScope && !IteratesOnceEachInOrder && SupportsAsyncLambdas;

public FunctionScopeInfo(
TexlFunction function,
Expand All @@ -76,7 +82,8 @@ public FunctionScopeInfo(
bool iteratesOverScope = true,
DType scopeType = null,
Func<int, bool> appliesToArgument = null,
bool canBeCreatedByRecord = false)
bool canBeCreatedByRecord = false,
bool iteratesOnceEachInOrder = false)
{
UsesAllFieldsInScope = usesAllFieldsInScope;
SupportsAsyncLambdas = supportsAsyncLambdas;
Expand All @@ -85,7 +92,8 @@ public FunctionScopeInfo(
ScopeType = scopeType;
_function = function;
AppliesToArgument = appliesToArgument ?? (i => i > 0);
CanBeCreatedByRecord = canBeCreatedByRecord;
CanBeCreatedByRecord = canBeCreatedByRecord;
IteratesOnceEachInOrder = iteratesOnceEachInOrder;
}

/// <summary>
Expand Down
8 changes: 7 additions & 1 deletion src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/ForAll.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,13 @@ internal sealed class ForAllFunction : FunctionWithTableInput
public ForAllFunction()
: base(ForAllInvariantFunctionName, TexlStrings.AboutForAll, FunctionCategories.Table, DType.Unknown, 0x2, 2, 2, DType.EmptyTable)
{
ScopeInfo = new FunctionScopeInfo(this);
// ForAll is now defined to always iterate over its table in record order.
// Initially, the function was meant to be parallelizable. But an attempt was made in Canvas
// to use parallel execution that did not go well as despite our best efforts to block makers from
// taking an order dependency (no Clear, no Set, etc) they found a way anyway and we would have broken formulas.
// We also have had numerous requests over the years to remove the blocks that made ForAll harder to use.
// A Map function will one day be introduced which does not allow any side effects and that can be parallelized.
ScopeInfo = new FunctionScopeInfo(this, iteratesOnceEachInOrder: true);
}

public override IEnumerable<TexlStrings.StringGetter[]> GetSignatures()
Expand Down
2 changes: 2 additions & 0 deletions src/libraries/Microsoft.PowerFx.Interpreter/Functions/Library.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2683,6 +2683,8 @@ public static async ValueTask<FormulaValue> ForAll(EvalVisitor runner, EvalVisit

var rows = new List<FormulaValue>();

// ForAll semantics now call for a sequential processing of each row. Do not parallelize this loop.
// This is declared with "iteratesOnceEachInOrder: true" in the compiler's function definition.
foreach (var row in rowsAsync)
{
runner.CheckCancel();
Expand Down
4 changes: 3 additions & 1 deletion src/libraries/Microsoft.PowerFx.Interpreter/Functions/LibraryUntypedObject.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -593,7 +593,9 @@ public static async ValueTask<FormulaValue> ForAll_UO(EvalVisitor runner, EvalVi
var rowsAsync = LazyForAll(runner, context, items, arg1);

var rows = new List<FormulaValue>();


// ForAll semantics now call for a sequential processing of each row. Do not parallelize this loop.
// This is declared with "iteratesOnceEachInOrder: true" in the compiler's function definition.
foreach (var row in rowsAsync)
{
runner.CheckCancel();
Expand Down
28 changes: 14 additions & 14 deletions src/tests/Microsoft.PowerFx.Core.Tests.Shared/TexlTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4626,8 +4626,8 @@ public void TexlFunctionTypeSemanticsTable_PageableInputs(string script, string
[InlineData("Max(DS, Collect(TblVar, {Id:1}); Id) ", "NonDelWarn", "NonDelWarn", "NonDelWnOp", "NonDelWnOp", " Ok", " Ok")]
[InlineData("VarP(DS, Collect(TblVar, {Id:1}); Id) ", " Ok", " Ok", " Ok", " Ok", " Ok", " Ok")]
[InlineData("StdevP(DS, Collect(TblVar, {Id:1}); Id) ", " Ok", " Ok", " Ok", " Ok", " Ok", " Ok")]
[InlineData("ForAll(DS, ClearCollect(TblVar, {Id:1})) ", " Unordered", " Unordered", " Unordered", " Unordered", " Unordered", " Unordered")]
[InlineData("ForAll(DS, Clear(TblVar)) ", " Unordered", " Unordered", " Unordered", " Unordered", " Unordered", " Unordered")]
[InlineData("ForAll(DS, ClearCollect(TblVar, {Id:1})) ", " Ok", " Ok", " Ok", " Ok", " Ok", " Ok")]
[InlineData("ForAll(DS, Clear(TblVar)) ", " Ok", " Ok", " Ok", " Ok", " Ok", " Ok")]
[InlineData("AddColumns(DS, Num, ClearCollect(TblVar, {Id:1}); 2) ", " Unordered", " Unordered", " Unordered", " Unordered", " Unordered", " Unordered")]
[InlineData("Concat(DS, Clear(TblVar); Text(Id)) ", " Unordered", " Unordered", " Unordered", " Unordered", " Unordered", " Unordered")]
[InlineData("Distinct(DS, ClearCollect(TblVar, {Id:1}); Id) ", " Unordered", " Unordered", " Unordered", " Unordered", " Unordered", " Unordered")]
Expand Down Expand Up @@ -4656,8 +4656,8 @@ public void TexlFunctionTypeSemanticsTable_PageableInputs(string script, string
[InlineData("Max(DS, Collect(DS, {Id:1}); Id) ", " SelfMod", " SelfMod", " SelfMod", " SelfMod", " Ok", " Ok")]
[InlineData("VarP(DS, Collect(DS, {Id:1}); Id) ", " SelfMod", " SelfMod", " SelfMod", " SelfMod", " Ok", " Ok")]
[InlineData("StdevP(DS, Collect(DS, {Id:1}); Id) ", " SelfMod", " SelfMod", " SelfMod", " SelfMod", " Ok", " Ok")]
[InlineData("ForAll(DS, ClearCollect(DS, {Id:1})) ", " SelfMod", " SelfMod", " SelfMod", " SelfMod", " Unordered", " Unordered")]
[InlineData("ForAll(DS, Clear(DS)) ", " SelfMod", " SelfMod", " SelfMod", " SelfMod", " Unordered", " Unordered")]
[InlineData("ForAll(DS, ClearCollect(DS, {Id:1})) ", " SelfMod", " SelfMod", " SelfMod", " SelfMod", " Ok", " Ok")]
[InlineData("ForAll(DS, Clear(DS)) ", " SelfMod", " SelfMod", " SelfMod", " SelfMod", " Ok", " Ok")]
[InlineData("AddColumns(DS, Num, ClearCollect(DS, {Id:1}); 2) ", " SelfMod", " SelfMod", " SelfMod", " SelfMod", " Unordered", " Unordered")]
[InlineData("Concat(DS, Clear(DS); Text(Id)) ", " SelfMod", " SelfMod", " SelfMod", " SelfMod", " Unordered", " Unordered")]
[InlineData("Distinct(DS, ClearCollect(DS, {Id:1}); Id) ", " SelfMod", " SelfMod", " SelfMod", " SelfMod", " Unordered", " Unordered")]
Expand All @@ -4682,8 +4682,8 @@ public void TexlFunctionTypeSemanticsTable_PageableInputs(string script, string
[InlineData("Max(Filter(DS,1=1), Collect(DS, {Id:1}); Id) ", " SelfMod", " SelfMod", " Ok", " Ok", " Ok", " Ok")]
[InlineData("VarP(Filter(DS,1=1), Collect(DS, {Id:1}); Id) ", " SelfMod", " SelfMod", " Ok", " Ok", " Ok", " Ok")]
[InlineData("StdevP(Filter(DS,1=1), Collect(DS, {Id:1}); Id) ", " SelfMod", " SelfMod", " Ok", " Ok", " Ok", " Ok")]
[InlineData("ForAll(Filter(DS,1=1), ClearCollect(DS, {Id:1})) ", " SelfMod", " SelfMod", " Unordered", " Unordered", " Unordered", " Unordered")]
[InlineData("ForAll(Filter(DS,1=1), Clear(DS)) ", " SelfMod", " SelfMod", " Unordered", " Unordered", " Unordered", " Unordered")]
[InlineData("ForAll(Filter(DS,1=1), ClearCollect(DS, {Id:1})) ", " SelfMod", " SelfMod", " Ok", " Ok", " Ok", " Ok")]
[InlineData("ForAll(Filter(DS,1=1), Clear(DS)) ", " SelfMod", " SelfMod", " Ok", " Ok", " Ok", " Ok")]
[InlineData("AddColumns(Filter(DS,1=1), Num, Clear(DS); 2) ", " SelfMod", " SelfMod", " Unordered", " Unordered", " Unordered", " Unordered")]
[InlineData("Concat(Filter(DS,1=1), Clear(DS); Text(Id)) ", " SelfMod", " SelfMod", " Unordered", " Unordered", " Unordered", " Unordered")]
[InlineData("Distinct(Filter(DS,1=1), Clear(DS); Id) ", " SelfMod", " SelfMod", " Unordered", " Unordered", " Unordered", " Unordered")]
Expand All @@ -4708,8 +4708,8 @@ public void TexlFunctionTypeSemanticsTable_PageableInputs(string script, string
[InlineData("Max(Sort(DS,Id), Collect(DS, {Id:1}); Id) ", " SelfMod", " SelfMod", " Ok", " Ok", " Ok", " Ok")]
[InlineData("VarP(Sort(DS,Id), Collect(DS, {Id:1}); Id) ", " SelfMod", " SelfMod", " Ok", " Ok", " Ok", " Ok")]
[InlineData("StdevP(Sort(DS,Id), Collect(DS, {Id:1}); Id) ", " SelfMod", " SelfMod", " Ok", " Ok", " Ok", " Ok")]
[InlineData("ForAll(Sort(DS,Id), ClearCollect(DS, {Id:1})) ", " SelfMod", " SelfMod", " Unordered", " Unordered", " Unordered", " Unordered")]
[InlineData("ForAll(Sort(DS,Id), Clear(DS)) ", " SelfMod", " SelfMod", " Unordered", " Unordered", " Unordered", " Unordered")]
[InlineData("ForAll(Sort(DS,Id), ClearCollect(DS, {Id:1})) ", " SelfMod", " SelfMod", " Ok", " Ok", " Ok", " Ok")]
[InlineData("ForAll(Sort(DS,Id), Clear(DS)) ", " SelfMod", " SelfMod", " Ok", " Ok", " Ok", " Ok")]
[InlineData("AddColumns(Sort(DS,Id), Num, ClearCollect(DS, {Id:1}); 2) ", " SelfMod", " SelfMod", " Unordered", " Unordered", " Unordered", " Unordered")]
[InlineData("Concat(Sort(DS,Id), Clear(DS); Text(Id)) ", " SelfMod", " SelfMod", " Unordered", " Unordered", " Unordered", " Unordered")]
[InlineData("Distinct(Sort(DS,Id), Clear(DS); Id) ", " SelfMod", " SelfMod", " Unordered", " Unordered", " Unordered", " Unordered")]
Expand All @@ -4734,8 +4734,8 @@ public void TexlFunctionTypeSemanticsTable_PageableInputs(string script, string
[InlineData("Max(Sort(Filter(DS,1=1),Id), Collect(DS, {Id:1}); Id) ", " SelfMod", " SelfMod", " Ok", " Ok", " Ok", " Ok")]
[InlineData("VarP(Sort(Filter(DS,1=1),Id), Collect(DS, {Id:1}); Id) ", " SelfMod", " SelfMod", " Ok", " Ok", " Ok", " Ok")]
[InlineData("StdevP(Sort(Filter(DS,1=1),Id), Collect(DS, {Id:1}); Id) ", " SelfMod", " SelfMod", " Ok", " Ok", " Ok", " Ok")]
[InlineData("ForAll(Sort(Filter(DS,1=1),Id), ClearCollect(DS, {Id:1})) ", " SelfMod", " SelfMod", " Unordered", " Unordered", " Unordered", " Unordered")]
[InlineData("ForAll(Sort(Filter(DS,1=1),Id), Clear(DS)) ", " SelfMod", " SelfMod", " Unordered", " Unordered", " Unordered", " Unordered")]
[InlineData("ForAll(Sort(Filter(DS,1=1),Id), ClearCollect(DS, {Id:1})) ", " SelfMod", " SelfMod", " Ok", " Ok", " Ok", " Ok")]
[InlineData("ForAll(Sort(Filter(DS,1=1),Id), Clear(DS)) ", " SelfMod", " SelfMod", " Ok", " Ok", " Ok", " Ok")]
[InlineData("AddColumns(Sort(Filter(DS,1=1),Id), Num, Clear(DS); 2) ", " SelfMod", " SelfMod", " Unordered", " Unordered", " Unordered", " Unordered")]
[InlineData("Concat(Sort(Filter(DS,1=1),Id), Clear(DS); Text(Id)) ", " SelfMod", " SelfMod", " Unordered", " Unordered", " Unordered", " Unordered")]
[InlineData("Distinct(Sort(Filter(DS,1=1),Id), Clear(DS); Id) ", " SelfMod", " SelfMod", " Unordered", " Unordered", " Unordered", " Unordered")]
Expand All @@ -4760,8 +4760,8 @@ public void TexlFunctionTypeSemanticsTable_PageableInputs(string script, string
[InlineData("Max(SortByColumns(DS,\"Id\"), Collect(DS, {Id:1}); Id) ", "NonDelWarn", "NonDelWarn", " Ok", " Ok", " Ok", " Ok")]
[InlineData("VarP(SortByColumns(DS,\"Id\"), Collect(DS, {Id:1}); Id) ", "NonDelWarn", "NonDelWarn", " Ok", " Ok", " Ok", " Ok")]
[InlineData("StdevP(SortByColumns(DS,\"Id\"), Collect(DS, {Id:1}); Id) ", "NonDelWarn", "NonDelWarn", " Ok", " Ok", " Ok", " Ok")]
[InlineData("ForAll(SortByColumns(DS,\"Id\"), ClearCollect(DS, {Id:1})) ", " Unordered", " Unordered", " Unordered", " Unordered", " Unordered", " Unordered")]
[InlineData("ForAll(SortByColumns(DS,\"Id\"), Clear(DS)) ", " Unordered", " Unordered", " Unordered", " Unordered", " Unordered", " Unordered")]
[InlineData("ForAll(SortByColumns(DS,\"Id\"), ClearCollect(DS, {Id:1})) ", "NonDelWarn", "NonDelWarn", " Ok", " Ok", " Ok", " Ok")]
[InlineData("ForAll(SortByColumns(DS,\"Id\"), Clear(DS)) ", "NonDelWarn", "NonDelWarn", " Ok", " Ok", " Ok", " Ok")]
[InlineData("AddColumns(SortByColumns(DS,\"Id\"), Num, Clear(DS); 2) ", " Unordered", " Unordered", " Unordered", " Unordered", " Unordered", " Unordered")]
[InlineData("Concat(SortByColumns(DS,\"Id\"), Clear(DS); Text(Id)) ", " Unordered", " Unordered", " Unordered", " Unordered", " Unordered", " Unordered")]
[InlineData("Distinct(SortByColumns(DS,\"Id\"), Clear(DS); Id) ", " Unordered", " Unordered", " Unordered", " Unordered", " Unordered", " Unordered")]
Expand All @@ -4786,8 +4786,8 @@ public void TexlFunctionTypeSemanticsTable_PageableInputs(string script, string
[InlineData("Max(FirstN(DS,10), Collect(DS, {Id:1}); Id) ", " Ok", " Ok", " Ok", " Ok", " Ok", " Ok")]
[InlineData("VarP(FirstN(DS,10), Collect(DS, {Id:1}); Id) ", " Ok", " Ok", " Ok", " Ok", " Ok", " Ok")]
[InlineData("StdevP(FirstN(DS,10), Collect(DS, {Id:1}); Id) ", " Ok", " Ok", " Ok", " Ok", " Ok", " Ok")]
[InlineData("ForAll(FirstN(DS,10), ClearCollect(DS, {Id:1})) ", " Unordered", " Unordered", " Unordered", " Unordered", " Unordered", " Unordered")]
[InlineData("ForAll(FirstN(DS,10), Clear(DS)) ", " Unordered", " Unordered", " Unordered", " Unordered", " Unordered", " Unordered")]
[InlineData("ForAll(FirstN(DS,10), ClearCollect(DS, {Id:1})) ", " Ok", " Ok", " Ok", " Ok", " Ok", " Ok")]
[InlineData("ForAll(FirstN(DS,10), Clear(DS)) ", " Ok", " Ok", " Ok", " Ok", " Ok", " Ok")]
[InlineData("AddColumns(FirstN(DS,10), Num, ClearCollect(DS, {Id:1}); 2) ", " Unordered", " Unordered", " Unordered", " Unordered", " Unordered", " Unordered")]
[InlineData("Concat(FirstN(DS,10), Clear(DS); Text(Id)) ", " Unordered", " Unordered", " Unordered", " Unordered", " Unordered", " Unordered")]
[InlineData("Distinct(FirstN(DS,10), Clear(DS); Id) ", " Unordered", " Unordered", " Unordered", " Unordered", " Unordered", " Unordered")]
Expand Down