From 52f5fd855ef5cfa617bc63c06c4f437641fd3d14 Mon Sep 17 00:00:00 2001 From: ywcb00 Date: Tue, 21 Jul 2026 09:34:51 +0200 Subject: [PATCH 1/6] fix(runtime/matrix/data/MatrixBlock.java): use the average parameter when picking multiple values for quantile computation feat(test/functions/binary/matrix/QuantileTest.java): test the quantile function with an array as percentiles argument --- .../runtime/matrix/data/MatrixBlock.java | 3 +- .../functions/binary/matrix/QuantileTest.java | 13 ++++++++ .../functions/binary/matrix/QuartileArray.R | 33 +++++++++++++++++++ .../functions/binary/matrix/QuartileArray.dml | 24 ++++++++++++++ 4 files changed, 71 insertions(+), 2 deletions(-) create mode 100644 src/test/scripts/functions/binary/matrix/QuartileArray.R create mode 100644 src/test/scripts/functions/binary/matrix/QuartileArray.dml diff --git a/src/main/java/org/apache/sysds/runtime/matrix/data/MatrixBlock.java b/src/main/java/org/apache/sysds/runtime/matrix/data/MatrixBlock.java index 18bc30094c6..8f26468ad7b 100644 --- a/src/main/java/org/apache/sysds/runtime/matrix/data/MatrixBlock.java +++ b/src/main/java/org/apache/sysds/runtime/matrix/data/MatrixBlock.java @@ -4784,8 +4784,7 @@ public MatrixBlock pickValues(MatrixValue quantiles, MatrixValue ret, boolean av output.reset(qs.rlen, qs.clen, false); for ( int i=0; i < qs.rlen; i++ ) { - // FIXME: [SYSTEMDS-3953] consider the average parameter here - output.set(i, 0, this.pickValue(qs.get(i,0)) ); + output.set(i, 0, this.pickValue(qs.get(i,0), average)); } return output; diff --git a/src/test/java/org/apache/sysds/test/functions/binary/matrix/QuantileTest.java b/src/test/java/org/apache/sysds/test/functions/binary/matrix/QuantileTest.java index 489b31080ef..f0b4c2f88c3 100644 --- a/src/test/java/org/apache/sysds/test/functions/binary/matrix/QuantileTest.java +++ b/src/test/java/org/apache/sysds/test/functions/binary/matrix/QuantileTest.java @@ -36,6 +36,7 @@ public class QuantileTest extends AutomatedTestBase private final static String TEST_NAME3 = "IQM"; private final static String TEST_NAME4 = "QuantileBug"; private final static String TEST_NAME5 = "MedianBug"; + private final static String TEST_NAME6 = "QuartileArray"; private final static String TEST_DIR = "functions/binary/matrix/"; private final static String TEST_CLASS_DIR = TEST_DIR + QuantileTest.class.getSimpleName() + "/"; @@ -60,6 +61,8 @@ public void setUp() new TestConfiguration(TEST_CLASS_DIR, TEST_NAME4, new String[] { "R" }) ); addTestConfiguration(TEST_NAME5, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME5, new String[] { "R" }) ); + addTestConfiguration(TEST_NAME6, + new TestConfiguration(TEST_CLASS_DIR, TEST_NAME6, new String[] { "R" }) ); } @Test @@ -182,6 +185,16 @@ public void testMedianBugSP() { runQuantileTest(TEST_NAME5, -1, false, ExecType.SPARK); } + @Test + public void testQuartileArrayCP() { + runQuantileTest(TEST_NAME6, 0, false, ExecType.CP); + } + + @Test + public void testQuartileArraySP() { + runQuantileTest(TEST_NAME6, 0, false, ExecType.SPARK); + } + private void runQuantileTest( String TEST_NAME, double p, boolean sparse, ExecType et) { ExecMode platformOld = setExecMode(et); diff --git a/src/test/scripts/functions/binary/matrix/QuartileArray.R b/src/test/scripts/functions/binary/matrix/QuartileArray.R new file mode 100644 index 00000000000..3f4cff52597 --- /dev/null +++ b/src/test/scripts/functions/binary/matrix/QuartileArray.R @@ -0,0 +1,33 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +args <- commandArgs(TRUE) +options(digits=22) + +library("Matrix") + +A = as.matrix(c(1,5,7,10)) + +m = quantile(A, c(0.25, 0.5, 0.75)); + +writeMM(as(m, "CsparseMatrix"), paste(args[3], "R", sep="")); + + diff --git a/src/test/scripts/functions/binary/matrix/QuartileArray.dml b/src/test/scripts/functions/binary/matrix/QuartileArray.dml new file mode 100644 index 00000000000..a63ac8f0cbc --- /dev/null +++ b/src/test/scripts/functions/binary/matrix/QuartileArray.dml @@ -0,0 +1,24 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +A = as.matrix(list(1,5,7,10)); +m = quantile(A, as.matrix(list(0.25, 0.5, 0.75))); +write(m, $3, format="text"); From a95f7b3223e0851b5ae45a51d1073658ef1532a2 Mon Sep 17 00:00:00 2001 From: ywcb00 Date: Tue, 21 Jul 2026 14:57:08 +0200 Subject: [PATCH 2/6] chore(test/functions/binary/matrix/QuantileTest.java): ignore the quartile test cases since they still fail due to the second issue reported in SYSTEMDS-3953 --- .../sysds/test/functions/binary/matrix/QuantileTest.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/test/java/org/apache/sysds/test/functions/binary/matrix/QuantileTest.java b/src/test/java/org/apache/sysds/test/functions/binary/matrix/QuantileTest.java index f0b4c2f88c3..0522d6d4733 100644 --- a/src/test/java/org/apache/sysds/test/functions/binary/matrix/QuantileTest.java +++ b/src/test/java/org/apache/sysds/test/functions/binary/matrix/QuantileTest.java @@ -21,6 +21,7 @@ import java.util.HashMap; +import org.junit.Ignore; import org.junit.Test; import org.apache.sysds.common.Types.ExecMode; import org.apache.sysds.common.Types.ExecType; @@ -186,11 +187,13 @@ public void testMedianBugSP() { } @Test + @Ignore // FIXME: fix SYSTEMDS-3953 Issue 2 public void testQuartileArrayCP() { runQuantileTest(TEST_NAME6, 0, false, ExecType.CP); } @Test + @Ignore // FIXME: fix SYSTEMDS-3953 Issue 2 public void testQuartileArraySP() { runQuantileTest(TEST_NAME6, 0, false, ExecType.SPARK); } From f57bea3dfae4ab75789e707ed541b6ed93674d97 Mon Sep 17 00:00:00 2001 From: ywcb00 Date: Wed, 22 Jul 2026 23:26:18 +0200 Subject: [PATCH 3/6] chore(runtime/matrix/data/MatrixBlock.java): minor formatting chore(test/functions/binary/matrix/QuantileTest.java): minor formatting --- .../sysds/runtime/matrix/data/MatrixBlock.java | 6 +++--- .../test/functions/binary/matrix/QuantileTest.java | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/main/java/org/apache/sysds/runtime/matrix/data/MatrixBlock.java b/src/main/java/org/apache/sysds/runtime/matrix/data/MatrixBlock.java index 8f26468ad7b..05dfecbdce9 100644 --- a/src/main/java/org/apache/sysds/runtime/matrix/data/MatrixBlock.java +++ b/src/main/java/org/apache/sysds/runtime/matrix/data/MatrixBlock.java @@ -4782,9 +4782,9 @@ public MatrixBlock pickValues(MatrixValue quantiles, MatrixValue ret, boolean av output=new MatrixBlock(qs.rlen, qs.clen, false); // resulting matrix is mostly likely be dense else output.reset(qs.rlen, qs.clen, false); - - for ( int i=0; i < qs.rlen; i++ ) { - output.set(i, 0, this.pickValue(qs.get(i,0), average)); + + for(int i = 0; i < qs.rlen; i++) { + output.set(i, 0, this.pickValue(qs.get(i, 0), average)); } return output; diff --git a/src/test/java/org/apache/sysds/test/functions/binary/matrix/QuantileTest.java b/src/test/java/org/apache/sysds/test/functions/binary/matrix/QuantileTest.java index 0522d6d4733..c371c126c89 100644 --- a/src/test/java/org/apache/sysds/test/functions/binary/matrix/QuantileTest.java +++ b/src/test/java/org/apache/sysds/test/functions/binary/matrix/QuantileTest.java @@ -53,17 +53,17 @@ public void setUp() { TestUtils.clearAssertionInformation(); addTestConfiguration(TEST_NAME1, - new TestConfiguration(TEST_CLASS_DIR, TEST_NAME1, new String[] { "R" }) ); + new TestConfiguration(TEST_CLASS_DIR, TEST_NAME1, new String[] {"R"})); addTestConfiguration(TEST_NAME2, - new TestConfiguration(TEST_CLASS_DIR, TEST_NAME2, new String[] { "R" }) ); + new TestConfiguration(TEST_CLASS_DIR, TEST_NAME2, new String[] {"R"})); addTestConfiguration(TEST_NAME3, - new TestConfiguration(TEST_CLASS_DIR, TEST_NAME3, new String[] { "R" }) ); + new TestConfiguration(TEST_CLASS_DIR, TEST_NAME3, new String[] {"R"})); addTestConfiguration(TEST_NAME4, - new TestConfiguration(TEST_CLASS_DIR, TEST_NAME4, new String[] { "R" }) ); + new TestConfiguration(TEST_CLASS_DIR, TEST_NAME4, new String[] {"R"})); addTestConfiguration(TEST_NAME5, - new TestConfiguration(TEST_CLASS_DIR, TEST_NAME5, new String[] { "R" }) ); + new TestConfiguration(TEST_CLASS_DIR, TEST_NAME5, new String[] {"R"})); addTestConfiguration(TEST_NAME6, - new TestConfiguration(TEST_CLASS_DIR, TEST_NAME6, new String[] { "R" }) ); + new TestConfiguration(TEST_CLASS_DIR, TEST_NAME6, new String[] {"R"})); } @Test From 1d38b9d6670d4b743bbba04b920575eae51a94ee Mon Sep 17 00:00:00 2001 From: ywcb00 Date: Wed, 22 Jul 2026 23:32:30 +0200 Subject: [PATCH 4/6] chore(test/functions/federated/primitives/FederatedQuantileTest.java): ignore the test case calling the quantile function with an array of percentiles and refer to SystemDS-3953 --- .../federated/primitives/part2/FederatedQuantileTest.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/test/java/org/apache/sysds/test/functions/federated/primitives/part2/FederatedQuantileTest.java b/src/test/java/org/apache/sysds/test/functions/federated/primitives/part2/FederatedQuantileTest.java index 20ad6eb1f01..06dfb9f5fc5 100644 --- a/src/test/java/org/apache/sysds/test/functions/federated/primitives/part2/FederatedQuantileTest.java +++ b/src/test/java/org/apache/sysds/test/functions/federated/primitives/part2/FederatedQuantileTest.java @@ -30,6 +30,7 @@ import org.apache.sysds.test.TestConfiguration; import org.apache.sysds.test.TestUtils; import org.junit.Assert; +import org.junit.Ignore; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; @@ -93,6 +94,7 @@ public void federatedIQRCP() { } @Test + @Ignore // FIXME: fix SYSTEMDS-3953 public void federatedQuantilesCP() { federatedQuartile(Types.ExecMode.SINGLE_NODE, TEST_NAME4, -1); } @@ -123,6 +125,7 @@ public void federatedIQRSP() { } @Test + @Ignore // FIXME: fix SYSTEMDS-3953 public void federatedQuantilesSP() { federatedQuartile(Types.ExecMode.SPARK, TEST_NAME4, -1); } From f06d9998abab50e72dc9248ab86603dcfba71abe Mon Sep 17 00:00:00 2001 From: ywcb00 Date: Thu, 23 Jul 2026 09:01:47 +0200 Subject: [PATCH 5/6] Revert "chore(test/functions/federated/primitives/FederatedQuantileTest.java): ignore the test case calling the quantile function with an array of percentiles and refer to SystemDS-3953" This reverts commit 1d38b9d6670d4b743bbba04b920575eae51a94ee. --- .../federated/primitives/part2/FederatedQuantileTest.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/test/java/org/apache/sysds/test/functions/federated/primitives/part2/FederatedQuantileTest.java b/src/test/java/org/apache/sysds/test/functions/federated/primitives/part2/FederatedQuantileTest.java index 06dfb9f5fc5..20ad6eb1f01 100644 --- a/src/test/java/org/apache/sysds/test/functions/federated/primitives/part2/FederatedQuantileTest.java +++ b/src/test/java/org/apache/sysds/test/functions/federated/primitives/part2/FederatedQuantileTest.java @@ -30,7 +30,6 @@ import org.apache.sysds.test.TestConfiguration; import org.apache.sysds.test.TestUtils; import org.junit.Assert; -import org.junit.Ignore; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; @@ -94,7 +93,6 @@ public void federatedIQRCP() { } @Test - @Ignore // FIXME: fix SYSTEMDS-3953 public void federatedQuantilesCP() { federatedQuartile(Types.ExecMode.SINGLE_NODE, TEST_NAME4, -1); } @@ -125,7 +123,6 @@ public void federatedIQRSP() { } @Test - @Ignore // FIXME: fix SYSTEMDS-3953 public void federatedQuantilesSP() { federatedQuartile(Types.ExecMode.SPARK, TEST_NAME4, -1); } From bb09de1a441553e6003e9d5e6eb32d9b554303c6 Mon Sep 17 00:00:00 2001 From: ywcb00 Date: Thu, 23 Jul 2026 09:09:19 +0200 Subject: [PATCH 6/6] fix(runtime/matrix/data/MatrixBlock.java): remove the averaging parameter again due to several failing test cases add a comment pointing to the jira issue SystemDS-3953 instead (test cases failing due to the second problem reported in this issue) --- .../org/apache/sysds/runtime/matrix/data/MatrixBlock.java | 3 ++- .../sysds/test/functions/binary/matrix/QuantileTest.java | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/apache/sysds/runtime/matrix/data/MatrixBlock.java b/src/main/java/org/apache/sysds/runtime/matrix/data/MatrixBlock.java index 05dfecbdce9..7525dab2f7f 100644 --- a/src/main/java/org/apache/sysds/runtime/matrix/data/MatrixBlock.java +++ b/src/main/java/org/apache/sysds/runtime/matrix/data/MatrixBlock.java @@ -4784,7 +4784,8 @@ public MatrixBlock pickValues(MatrixValue quantiles, MatrixValue ret, boolean av output.reset(qs.rlen, qs.clen, false); for(int i = 0; i < qs.rlen; i++) { - output.set(i, 0, this.pickValue(qs.get(i, 0), average)); + // FIXME: include the average parameter here to fix SYSTEMDS-3953 + output.set(i, 0, this.pickValue(qs.get(i, 0))); } return output; diff --git a/src/test/java/org/apache/sysds/test/functions/binary/matrix/QuantileTest.java b/src/test/java/org/apache/sysds/test/functions/binary/matrix/QuantileTest.java index c371c126c89..810e2614e7c 100644 --- a/src/test/java/org/apache/sysds/test/functions/binary/matrix/QuantileTest.java +++ b/src/test/java/org/apache/sysds/test/functions/binary/matrix/QuantileTest.java @@ -187,13 +187,13 @@ public void testMedianBugSP() { } @Test - @Ignore // FIXME: fix SYSTEMDS-3953 Issue 2 + @Ignore // FIXME: fix SYSTEMDS-3953 public void testQuartileArrayCP() { runQuantileTest(TEST_NAME6, 0, false, ExecType.CP); } @Test - @Ignore // FIXME: fix SYSTEMDS-3953 Issue 2 + @Ignore // FIXME: fix SYSTEMDS-3953 public void testQuartileArraySP() { runQuantileTest(TEST_NAME6, 0, false, ExecType.SPARK); }