[
https://issues.apache.org/jira/browse/FLINK-10674?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16684785#comment-16684785
]
ASF GitHub Bot commented on FLINK-10674:
----------------------------------------
wenhuitang closed pull request #7076: [FLINK-10674] [table] Fix
DistinctAccumulator.remove lead to NPE.
URL: https://github.com/apache/flink/pull/7076
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/docs/build_docs.sh b/docs/build_docs.sh
old mode 100755
new mode 100644
diff --git a/docs/check_links.sh b/docs/check_links.sh
old mode 100755
new mode 100644
diff --git a/docs/docker/run.sh b/docs/docker/run.sh
old mode 100755
new mode 100644
diff --git a/docs/fig/ssl_internal_external.svg
b/docs/fig/ssl_internal_external.svg
old mode 100755
new mode 100644
diff --git a/flink-container/docker/build.sh b/flink-container/docker/build.sh
old mode 100755
new mode 100644
diff --git a/flink-container/docker/docker-entrypoint.sh
b/flink-container/docker/docker-entrypoint.sh
old mode 100755
new mode 100644
diff --git a/flink-contrib/docker-flink/bluemix-docker-compose.sh
b/flink-contrib/docker-flink/bluemix-docker-compose.sh
old mode 100755
new mode 100644
diff --git a/flink-contrib/docker-flink/build.sh
b/flink-contrib/docker-flink/build.sh
old mode 100755
new mode 100644
diff --git a/flink-contrib/docker-flink/create-docker-swarm-service.sh
b/flink-contrib/docker-flink/create-docker-swarm-service.sh
old mode 100755
new mode 100644
diff --git a/flink-contrib/docker-flink/docker-entrypoint.sh
b/flink-contrib/docker-flink/docker-entrypoint.sh
old mode 100755
new mode 100644
diff --git a/flink-contrib/docker-flink/remove-docker-swarm-service.sh
b/flink-contrib/docker-flink/remove-docker-swarm-service.sh
old mode 100755
new mode 100644
diff --git a/flink-dist/src/main/flink-bin/bin/config.sh
b/flink-dist/src/main/flink-bin/bin/config.sh
old mode 100755
new mode 100644
diff --git a/flink-dist/src/main/flink-bin/bin/jobmanager.sh
b/flink-dist/src/main/flink-bin/bin/jobmanager.sh
old mode 100755
new mode 100644
diff --git a/flink-dist/src/main/flink-bin/bin/start-cluster.sh
b/flink-dist/src/main/flink-bin/bin/start-cluster.sh
old mode 100755
new mode 100644
diff --git a/flink-dist/src/main/flink-bin/bin/start-zookeeper-quorum.sh
b/flink-dist/src/main/flink-bin/bin/start-zookeeper-quorum.sh
old mode 100755
new mode 100644
diff --git a/flink-dist/src/main/flink-bin/bin/stop-cluster.sh
b/flink-dist/src/main/flink-bin/bin/stop-cluster.sh
old mode 100755
new mode 100644
diff --git a/flink-dist/src/main/flink-bin/bin/stop-zookeeper-quorum.sh
b/flink-dist/src/main/flink-bin/bin/stop-zookeeper-quorum.sh
old mode 100755
new mode 100644
diff --git a/flink-dist/src/main/flink-bin/bin/taskmanager.sh
b/flink-dist/src/main/flink-bin/bin/taskmanager.sh
old mode 100755
new mode 100644
diff --git a/flink-dist/src/main/flink-bin/bin/zookeeper.sh
b/flink-dist/src/main/flink-bin/bin/zookeeper.sh
old mode 100755
new mode 100644
diff --git a/flink-dist/src/main/flink-bin/mesos-bin/mesos-appmaster-job.sh
b/flink-dist/src/main/flink-bin/mesos-bin/mesos-appmaster-job.sh
old mode 100755
new mode 100644
diff --git a/flink-dist/src/main/flink-bin/mesos-bin/mesos-appmaster.sh
b/flink-dist/src/main/flink-bin/mesos-bin/mesos-appmaster.sh
old mode 100755
new mode 100644
diff --git a/flink-dist/src/main/flink-bin/mesos-bin/mesos-taskmanager.sh
b/flink-dist/src/main/flink-bin/mesos-bin/mesos-taskmanager.sh
old mode 100755
new mode 100644
diff --git a/flink-dist/src/main/flink-bin/yarn-bin/yarn-session.sh
b/flink-dist/src/main/flink-bin/yarn-bin/yarn-session.sh
old mode 100755
new mode 100644
diff --git
a/flink-end-to-end-tests/flink-parent-child-classloading-test/src/main/resources/.version.properties
b/flink-end-to-end-tests/flink-parent-child-classloading-test/src/main/resources/.version.properties
deleted file mode 100644
index 76f9c5aa859..00000000000
---
a/flink-end-to-end-tests/flink-parent-child-classloading-test/src/main/resources/.version.properties
+++ /dev/null
@@ -1 +0,0 @@
-git.commit.id.abbrev=hello-there-42
diff --git
a/flink-end-to-end-tests/test-scripts/test_resume_externalized_checkpoints.sh
b/flink-end-to-end-tests/test-scripts/test_resume_externalized_checkpoints.sh
old mode 100755
new mode 100644
diff --git a/flink-end-to-end-tests/test-scripts/test_resume_savepoint.sh
b/flink-end-to-end-tests/test-scripts/test_resume_savepoint.sh
old mode 100755
new mode 100644
diff --git a/flink-end-to-end-tests/test-scripts/test_yarn_kerberos_docker.sh
b/flink-end-to-end-tests/test-scripts/test_yarn_kerberos_docker.sh
old mode 100755
new mode 100644
diff --git a/flink-jepsen/docker/up.sh b/flink-jepsen/docker/up.sh
old mode 100755
new mode 100644
diff --git
a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/aggfunctions/DistinctAccumulator.scala
b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/aggfunctions/DistinctAccumulator.scala
index 3427c9c96f7..91c2f923b72 100644
---
a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/aggfunctions/DistinctAccumulator.scala
+++
b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/functions/aggfunctions/DistinctAccumulator.scala
@@ -101,13 +101,18 @@ class DistinctAccumulator[ACC](
* @return true if no instances of the parameters remain in the map, false
otherwise.
*/
def remove(params: Row): Boolean = {
- val currentCnt = distinctValueMap.get(params)
- if (currentCnt == 1) {
- distinctValueMap.remove(params)
+ if (!distinctValueMap.contains(params)) {
true
} else {
- distinctValueMap.put(params, currentCnt - 1L)
- false
+ val currentCnt = distinctValueMap.get(params)
+
+ if (currentCnt == null || currentCnt <= 1) {
+ distinctValueMap.remove(params)
+ true
+ } else {
+ distinctValueMap.put(params, currentCnt - 1L)
+ false
+ }
}
}
diff --git
a/flink-mesos/src/main/java/org/apache/flink/mesos/entrypoint/MesosEntrypointUtils.java
b/flink-mesos/src/main/java/org/apache/flink/mesos/entrypoint/MesosEntrypointUtils.java
old mode 100755
new mode 100644
diff --git
a/flink-mesos/src/main/java/org/apache/flink/mesos/entrypoint/MesosJobClusterEntrypoint.java
b/flink-mesos/src/main/java/org/apache/flink/mesos/entrypoint/MesosJobClusterEntrypoint.java
old mode 100755
new mode 100644
diff --git
a/flink-mesos/src/main/java/org/apache/flink/mesos/entrypoint/MesosSessionClusterEntrypoint.java
b/flink-mesos/src/main/java/org/apache/flink/mesos/entrypoint/MesosSessionClusterEntrypoint.java
old mode 100755
new mode 100644
diff --git
a/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosApplicationMasterRunner.java
b/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosApplicationMasterRunner.java
old mode 100755
new mode 100644
diff --git
a/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/ClusterEntrypoint.java
b/flink-runtime/src/main/java/org/apache/flink/runtime/entrypoint/ClusterEntrypoint.java
old mode 100755
new mode 100644
diff --git a/tools/list_deps.py b/tools/list_deps.py
old mode 100755
new mode 100644
diff --git a/tools/merge_flink_pr.py b/tools/merge_flink_pr.py
old mode 100755
new mode 100644
diff --git a/tools/merge_pull_request.sh.template
b/tools/merge_pull_request.sh.template
old mode 100755
new mode 100644
diff --git a/tools/releasing/create_binary_release.sh
b/tools/releasing/create_binary_release.sh
old mode 100755
new mode 100644
diff --git a/tools/releasing/create_release_branch.sh
b/tools/releasing/create_release_branch.sh
old mode 100755
new mode 100644
diff --git a/tools/releasing/create_source_release.sh
b/tools/releasing/create_source_release.sh
old mode 100755
new mode 100644
diff --git a/tools/releasing/deploy_staging_jars.sh
b/tools/releasing/deploy_staging_jars.sh
old mode 100755
new mode 100644
diff --git a/tools/releasing/update_branch_version.sh
b/tools/releasing/update_branch_version.sh
old mode 100755
new mode 100644
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
> DistinctAccumulator.remove lead to NPE
> --------------------------------------
>
> Key: FLINK-10674
> URL: https://issues.apache.org/jira/browse/FLINK-10674
> Project: Flink
> Issue Type: Bug
> Components: Table API & SQL
> Affects Versions: 1.6.1
> Environment: Flink 1.6.0
> Reporter: ambition
> Assignee: winifredtang
> Priority: Minor
> Labels: pull-request-available
> Attachments: image-2018-10-25-14-46-03-373.png
>
>
> Our online Flink Job run about a week，job contain sql ：
> {code:java}
> select `time`,
> lower(trim(os_type)) as os_type,
> count(distinct feed_id) as feed_total_view
> from my_table
> group by `time`, lower(trim(os_type)){code}
>
> then occur NPE:
>
> {code:java}
> java.lang.NullPointerException
> at scala.Predef$.Long2long(Predef.scala:363)
> at
> org.apache.flink.table.functions.aggfunctions.DistinctAccumulator.remove(DistinctAccumulator.scala:109)
> at NonWindowedAggregationHelper$894.retract(Unknown Source)
> at
> org.apache.flink.table.runtime.aggregate.GroupAggProcessFunction.processElement(GroupAggProcessFunction.scala:124)
> at
> org.apache.flink.table.runtime.aggregate.GroupAggProcessFunction.processElement(GroupAggProcessFunction.scala:39)
> at
> org.apache.flink.streaming.api.operators.LegacyKeyedProcessOperator.processElement(LegacyKeyedProcessOperator.java:88)
> at
> org.apache.flink.streaming.runtime.io.StreamInputProcessor.processInput(StreamInputProcessor.java:202)
> at
> org.apache.flink.streaming.runtime.tasks.OneInputStreamTask.run(OneInputStreamTask.java:105)
> at
> org.apache.flink.streaming.runtime.tasks.StreamTask.invoke(StreamTask.java:300)
> at org.apache.flink.runtime.taskmanager.Task.run(Task.java:711)
> at java.lang.Thread.run(Thread.java:745)
> {code}
>
>
> View DistinctAccumulator.remove
> !image-2018-10-25-14-46-03-373.png!
>
> this NPE should currentCnt = null lead to, so we simple handle like :
> {code:java}
> def remove(params: Row): Boolean = {
> if(!distinctValueMap.contains(params)){
> true
> }else{
> val currentCnt = distinctValueMap.get(params)
> //
> if (currentCnt == null || currentCnt == 1) {
> distinctValueMap.remove(params)
> true
> } else {
> var value = currentCnt - 1L
> if(value < 0){
> value = 1
> }
> distinctValueMap.put(params, value)
> false
> }
> }
> }{code}
>
>
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
