Apache Flink 中文用户邮件列表

PyFlink UDTF 运行一段时间后报 NullPointerException

Classic

List

Threaded

2 messages Options

梁丁文

PyFlink UDTF 运行一段时间后报 NullPointerException

测试PyFlink UDTF，运行数秒后异常退出。下面是UDTF函数

class Mac(TableFunction):
    def eval(self, body_data):
        mac_arr = [
                {"mac":"44:c7:fc:24:0f:91","rssi":"-92","range":"100.0"},
                {"mac":"44:c7:fc:24:0f:91","rssi":"-92","range":"100.0"},
                {"mac":"44:c7:fc:24:0f:91","rssi":"-92","range":"100.0"},
                {"mac":"44:c7:fc:24:0f:91","rssi":"-92","range":"100.0"},
                {"mac":"44:c7:fc:24:0f:91","rssi":"-92","range":"100.0"},
                {"mac":"44:c7:fc:24:0f:91","rssi":"-92","range":"100.0"},
                {"mac":"44:c7:fc:24:0f:91","rssi":"-92","range":"100.0"},
                {"mac":"44:c7:fc:24:0f:91","rssi":"-92","range":"100.0"},
                {"mac":"44:c7:fc:24:0f:91","rssi":"-92","range":"100.0"},
                {"mac":"44:c7:fc:24:0f:91","rssi":"-92","range":"100.0"},
                {"mac":"44:c7:fc:24:0f:91","rssi":"-92","range":"100.0"},
                {"mac":"44:c7:fc:24:0f:91","rssi":"-92","range":"100.0"},
                {"mac":"44:c7:fc:24:0f:91","rssi":"-92","range":"100.0"},
                {"mac":"44:c7:fc:24:0f:91","rssi":"-92","range":"100.0"},
                {"mac":"44:c7:fc:24:0f:91","rssi":"-92","range":"100.0"},
                {"mac":"44:c7:fc:24:0f:91","rssi":"-92","range":"100.0"},
                {"mac":"44:c7:fc:24:0f:91","rssi":"-92","range":"100.0"},
                {"mac":"44:c7:fc:24:0f:91","rssi":"-92","range":"100.0"},
                {"mac":"44:c7:fc:24:0f:91","rssi":"-92","range":"100.0"},
                {"mac":"44:c7:fc:24:0f:91","rssi":"-92","range":"100.0"},
                {"mac":"44:c7:fc:24:0f:91","rssi":"-92","range":"100.0"},
                {"mac":"44:c7:fc:24:0f:91","rssi":"-92","range":"100.0"},
                {"mac":"44:c7:fc:24:0f:91","rssi":"-92","range":"100.0"},
                {"mac":"44:c7:fc:24:0f:91","rssi":"-92","range":"100.0"},
                {"mac":"44:c7:fc:24:0f:91","rssi":"-92","range":"100.0"},
                {"mac":"44:c7:fc:24:0f:91","rssi":"-92","range":"100.0"},
                {"mac":"44:c7:fc:24:0f:91","rssi":"-92","range":"100.0"},
                {"mac":"44:c7:fc:24:0f:91","rssi":"-92","range":"100.0"},
                {"mac":"08:10:79:67:16:1b","rssi":"-85","router":"Netcore_67161B","range":"55.0"}
                ]
        return mac_arr

table_env.create_temporary_function("mac", udtf(Mac(), result_types=[DataTypes.MAP(DataTypes.STRING(),DataTypes.STRING())]))

执行语句：

INSERT INTO MacTableSink select data['mac'] mac, data['rssi'] rssi, data['router'] router, data['range'] distance from KafkaMacTable, lateral table(mac(body_data)) as T(data)

异常日志：

2021-03-09 17:19:29
org.apache.flink.runtime.JobException: Recovery is suppressed by NoRestartBackoffTimeStrategy
    at org.apache.flink.runtime.executiongraph.failover.flip1.ExecutionFailureHandler.handleFailure(ExecutionFailureHandler.java:118)
    at org.apache.flink.runtime.executiongraph.failover.flip1.ExecutionFailureHandler.getFailureHandlingResult(ExecutionFailureHandler.java:80)
    at org.apache.flink.runtime.scheduler.DefaultScheduler.handleTaskFailure(DefaultScheduler.java:233)
    at org.apache.flink.runtime.scheduler.DefaultScheduler.maybeHandleTaskFailure(DefaultScheduler.java:224)
    at org.apache.flink.runtime.scheduler.DefaultScheduler.updateTaskExecutionStateInternal(DefaultScheduler.java:215)
    at org.apache.flink.runtime.scheduler.SchedulerBase.updateTaskExecutionState(SchedulerBase.java:669)
    at org.apache.flink.runtime.scheduler.SchedulerNG.updateTaskExecutionState(SchedulerNG.java:89)
    at org.apache.flink.runtime.jobmaster.JobMaster.updateTaskExecutionState(JobMaster.java:447)
    at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.lang.reflect.Method.invoke(Method.java:498)
    at org.apache.flink.runtime.rpc.akka.AkkaRpcActor.handleRpcInvocation(AkkaRpcActor.java:305)
    at org.apache.flink.runtime.rpc.akka.AkkaRpcActor.handleRpcMessage(AkkaRpcActor.java:212)
    at org.apache.flink.runtime.rpc.akka.FencedAkkaRpcActor.handleRpcMessage(FencedAkkaRpcActor.java:77)
    at org.apache.flink.runtime.rpc.akka.AkkaRpcActor.handleMessage(AkkaRpcActor.java:158)
    at akka.japi.pf.UnitCaseStatement.apply(CaseStatements.scala:26)
    at akka.japi.pf.UnitCaseStatement.apply(CaseStatements.scala:21)
    at scala.PartialFunction$class.applyOrElse(PartialFunction.scala:123)
    at akka.japi.pf.UnitCaseStatement.applyOrElse(CaseStatements.scala:21)
    at scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:170)
    at scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:171)
    at scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:171)
    at akka.actor.Actor$class.aroundReceive(Actor.scala:517)
    at akka.actor.AbstractActor.aroundReceive(AbstractActor.scala:225)
    at akka.actor.ActorCell.receiveMessage(ActorCell.scala:592)
    at akka.actor.ActorCell.invoke(ActorCell.scala:561)
    at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:258)
    at akka.dispatch.Mailbox.run(Mailbox.scala:225)
    at akka.dispatch.Mailbox.exec(Mailbox.scala:235)
    at akka.dispatch.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260)
    at akka.dispatch.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)
    at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
    at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
Caused by: org.apache.flink.streaming.runtime.tasks.AsynchronousException: Caught exception while processing timer.
    at org.apache.flink.streaming.runtime.tasks.StreamTask$StreamTaskAsyncExceptionHandler.handleAsyncException(StreamTask.java:1205)
    at org.apache.flink.streaming.runtime.tasks.StreamTask.handleAsyncException(StreamTask.java:1181)
    at org.apache.flink.streaming.runtime.tasks.StreamTask.invokeProcessingTimeCallback(StreamTask.java:1320)
    at org.apache.flink.streaming.runtime.tasks.StreamTask.lambda$null$17(StreamTask.java:1309)
    at org.apache.flink.streaming.runtime.tasks.StreamTaskActionExecutor$SynchronizedStreamTaskActionExecutor.runThrowing(StreamTaskActionExecutor.java:93)
    at org.apache.flink.streaming.runtime.tasks.mailbox.Mail.run(Mail.java:90)
    at org.apache.flink.streaming.runtime.tasks.mailbox.MailboxProcessor.processMail(MailboxProcessor.java:317)
    at org.apache.flink.streaming.runtime.tasks.mailbox.MailboxProcessor.runMailboxLoop(MailboxProcessor.java:189)
    at org.apache.flink.streaming.runtime.tasks.StreamTask.runMailboxLoop(StreamTask.java:617)
    at org.apache.flink.streaming.runtime.tasks.StreamTask.invoke(StreamTask.java:581)
    at org.apache.flink.runtime.taskmanager.Task.doRun(Task.java:755)
    at org.apache.flink.runtime.taskmanager.Task.run(Task.java:570)
    at java.lang.Thread.run(Thread.java:748)
Caused by: TimerException{java.lang.NullPointerException}
    ... 11 more
Caused by: java.lang.NullPointerException
    at org.apache.flink.table.runtime.operators.python.table.RowDataPythonTableFunctionOperator.emitResult(RowDataPythonTableFunctionOperator.java:138)
    at org.apache.flink.streaming.api.operators.python.AbstractPythonFunctionOperator.emitResults(AbstractPythonFunctionOperator.java:250)
    at org.apache.flink.streaming.api.operators.python.AbstractPythonFunctionOperator.invokeFinishBundle(AbstractPythonFunctionOperator.java:273)
    at org.apache.flink.streaming.api.operators.python.AbstractPythonFunctionOperator.checkInvokeFinishBundleByTime(AbstractPythonFunctionOperator.java:265)
    at org.apache.flink.streaming.api.operators.python.AbstractPythonFunctionOperator.lambda$open$0(AbstractPythonFunctionOperator.java:125)
    at org.apache.flink.streaming.runtime.tasks.StreamTask.invokeProcessingTimeCallback(StreamTask.java:1318)
    ... 10 more

Dian Fu

Re: PyFlink UDTF 运行一段时间后报 NullPointerException

这个问题应该和这个JIRA有关系：https://issues.apache.org/jira/browse/FLINK-21434

目前已经在master和1.12.3上修复了，但是1.12.3还没有release。你要不cherry-pick这个fix，自己build一个版本，验证一下？

On Tue, Mar 9, 2021 at 5:27 PM 梁丁文 <[hidden email]> wrote:

> 测试PyFlink UDTF，运行数秒后异常退出。下面是UDTF函数
>
>
> class Mac(TableFunction):
>     def eval(self, body_data):
>         mac_arr = [
>                
> {"mac":"44:c7:fc:24:0f:91","rssi":"-92","range":"100.0"},
>                
> {"mac":"44:c7:fc:24:0f:91","rssi":"-92","range":"100.0"},
>                
> {"mac":"44:c7:fc:24:0f:91","rssi":"-92","range":"100.0"},
>                
> {"mac":"44:c7:fc:24:0f:91","rssi":"-92","range":"100.0"},
>                
> {"mac":"44:c7:fc:24:0f:91","rssi":"-92","range":"100.0"},
>                
> {"mac":"44:c7:fc:24:0f:91","rssi":"-92","range":"100.0"},
>                
> {"mac":"44:c7:fc:24:0f:91","rssi":"-92","range":"100.0"},
>                
> {"mac":"44:c7:fc:24:0f:91","rssi":"-92","range":"100.0"},
>                
> {"mac":"44:c7:fc:24:0f:91","rssi":"-92","range":"100.0"},
>                
> {"mac":"44:c7:fc:24:0f:91","rssi":"-92","range":"100.0"},
>                
> {"mac":"44:c7:fc:24:0f:91","rssi":"-92","range":"100.0"},
>                
> {"mac":"44:c7:fc:24:0f:91","rssi":"-92","range":"100.0"},
>                
> {"mac":"44:c7:fc:24:0f:91","rssi":"-92","range":"100.0"},
>                
> {"mac":"44:c7:fc:24:0f:91","rssi":"-92","range":"100.0"},
>                
> {"mac":"44:c7:fc:24:0f:91","rssi":"-92","range":"100.0"},
>                
> {"mac":"44:c7:fc:24:0f:91","rssi":"-92","range":"100.0"},
>                
> {"mac":"44:c7:fc:24:0f:91","rssi":"-92","range":"100.0"},
>                
> {"mac":"44:c7:fc:24:0f:91","rssi":"-92","range":"100.0"},
>                
> {"mac":"44:c7:fc:24:0f:91","rssi":"-92","range":"100.0"},
>                
> {"mac":"44:c7:fc:24:0f:91","rssi":"-92","range":"100.0"},
>                
> {"mac":"44:c7:fc:24:0f:91","rssi":"-92","range":"100.0"},
>                
> {"mac":"44:c7:fc:24:0f:91","rssi":"-92","range":"100.0"},
>                
> {"mac":"44:c7:fc:24:0f:91","rssi":"-92","range":"100.0"},
>                
> {"mac":"44:c7:fc:24:0f:91","rssi":"-92","range":"100.0"},
>                
> {"mac":"44:c7:fc:24:0f:91","rssi":"-92","range":"100.0"},
>                
> {"mac":"44:c7:fc:24:0f:91","rssi":"-92","range":"100.0"},
>                
> {"mac":"44:c7:fc:24:0f:91","rssi":"-92","range":"100.0"},
>                
> {"mac":"44:c7:fc:24:0f:91","rssi":"-92","range":"100.0"},
>                
> {"mac":"08:10:79:67:16:1b","rssi":"-85","router":"Netcore_67161B","range":"55.0"}
>                 ]
>         return mac_arr
>
>
> table_env.create_temporary_function("mac", udtf(Mac(),
> result_types=[DataTypes.MAP(DataTypes.STRING(),DataTypes.STRING())]))
>
>
>
> 执行语句：
>
>
> INSERT INTO MacTableSink select data['mac'] mac, data['rssi'] rssi,
> data['router'] router, data['range'] distance from KafkaMacTable, lateral
> table(mac(body_data)) as T(data)
>
>
> 异常日志：
>
>
> 2021-03-09 17:19:29
> org.apache.flink.runtime.JobException: Recovery is suppressed by
> NoRestartBackoffTimeStrategy
>     at
> org.apache.flink.runtime.executiongraph.failover.flip1.ExecutionFailureHandler.handleFailure(ExecutionFailureHandler.java:118)
>     at
> org.apache.flink.runtime.executiongraph.failover.flip1.ExecutionFailureHandler.getFailureHandlingResult(ExecutionFailureHandler.java:80)
>     at
> org.apache.flink.runtime.scheduler.DefaultScheduler.handleTaskFailure(DefaultScheduler.java:233)
>     at
> org.apache.flink.runtime.scheduler.DefaultScheduler.maybeHandleTaskFailure(DefaultScheduler.java:224)
>     at
> org.apache.flink.runtime.scheduler.DefaultScheduler.updateTaskExecutionStateInternal(DefaultScheduler.java:215)
>     at
> org.apache.flink.runtime.scheduler.SchedulerBase.updateTaskExecutionState(SchedulerBase.java:669)
>     at
> org.apache.flink.runtime.scheduler.SchedulerNG.updateTaskExecutionState(SchedulerNG.java:89)
>     at
> org.apache.flink.runtime.jobmaster.JobMaster.updateTaskExecutionState(JobMaster.java:447)
>     at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
> Method)
>     at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
>     at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>     at java.lang.reflect.Method.invoke(Method.java:498)
>     at
> org.apache.flink.runtime.rpc.akka.AkkaRpcActor.handleRpcInvocation(AkkaRpcActor.java:305)
>     at
> org.apache.flink.runtime.rpc.akka.AkkaRpcActor.handleRpcMessage(AkkaRpcActor.java:212)
>     at
> org.apache.flink.runtime.rpc.akka.FencedAkkaRpcActor.handleRpcMessage(FencedAkkaRpcActor.java:77)
>     at
> org.apache.flink.runtime.rpc.akka.AkkaRpcActor.handleMessage(AkkaRpcActor.java:158)
>     at akka.japi.pf
> .UnitCaseStatement.apply(CaseStatements.scala:26)
>     at akka.japi.pf
> .UnitCaseStatement.apply(CaseStatements.scala:21)
>     at
> scala.PartialFunction$class.applyOrElse(PartialFunction.scala:123)
>     at akka.japi.pf
> .UnitCaseStatement.applyOrElse(CaseStatements.scala:21)
>     at
> scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:170)
>     at
> scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:171)
>     at
> scala.PartialFunction$OrElse.applyOrElse(PartialFunction.scala:171)
>     at akka.actor.Actor$class.aroundReceive(Actor.scala:517)
>     at
> akka.actor.AbstractActor.aroundReceive(AbstractActor.scala:225)
>     at
> akka.actor.ActorCell.receiveMessage(ActorCell.scala:592)
>     at akka.actor.ActorCell.invoke(ActorCell.scala:561)
>     at
> akka.dispatch.Mailbox.processMailbox(Mailbox.scala:258)
>     at akka.dispatch.Mailbox.run(Mailbox.scala:225)
>     at akka.dispatch.Mailbox.exec(Mailbox.scala:235)
>     at
> akka.dispatch.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260)
>     at
> akka.dispatch.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)
>     at
> akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
>     at
> akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
> Caused by: org.apache.flink.streaming.runtime.tasks.AsynchronousException:
> Caught exception while processing timer.
>     at
> org.apache.flink.streaming.runtime.tasks.StreamTask$StreamTaskAsyncExceptionHandler.handleAsyncException(StreamTask.java:1205)
>     at
> org.apache.flink.streaming.runtime.tasks.StreamTask.handleAsyncException(StreamTask.java:1181)
>     at
> org.apache.flink.streaming.runtime.tasks.StreamTask.invokeProcessingTimeCallback(StreamTask.java:1320)
>     at
> org.apache.flink.streaming.runtime.tasks.StreamTask.lambda$null$17(StreamTask.java:1309)
>     at
> org.apache.flink.streaming.runtime.tasks.StreamTaskActionExecutor$SynchronizedStreamTaskActionExecutor.runThrowing(StreamTaskActionExecutor.java:93)
>     at
> org.apache.flink.streaming.runtime.tasks.mailbox.Mail.run(Mail.java:90)
>     at
> org.apache.flink.streaming.runtime.tasks.mailbox.MailboxProcessor.processMail(MailboxProcessor.java:317)
>     at
> org.apache.flink.streaming.runtime.tasks.mailbox.MailboxProcessor.runMailboxLoop(MailboxProcessor.java:189)
>     at
> org.apache.flink.streaming.runtime.tasks.StreamTask.runMailboxLoop(StreamTask.java:617)
>     at
> org.apache.flink.streaming.runtime.tasks.StreamTask.invoke(StreamTask.java:581)
>     at
> org.apache.flink.runtime.taskmanager.Task.doRun(Task.java:755)
>     at
> org.apache.flink.runtime.taskmanager.Task.run(Task.java:570)
>     at java.lang.Thread.run(Thread.java:748)
> Caused by: TimerException{java.lang.NullPointerException}
>     ... 11 more
> Caused by: java.lang.NullPointerException
>     at
> org.apache.flink.table.runtime.operators.python.table.RowDataPythonTableFunctionOperator.emitResult(RowDataPythonTableFunctionOperator.java:138)
>     at
> org.apache.flink.streaming.api.operators.python.AbstractPythonFunctionOperator.emitResults(AbstractPythonFunctionOperator.java:250)
>     at
> org.apache.flink.streaming.api.operators.python.AbstractPythonFunctionOperator.invokeFinishBundle(AbstractPythonFunctionOperator.java:273)
>     at
> org.apache.flink.streaming.api.operators.python.AbstractPythonFunctionOperator.checkInvokeFinishBundleByTime(AbstractPythonFunctionOperator.java:265)
>     at
> org.apache.flink.streaming.api.operators.python.AbstractPythonFunctionOperator.lambda$open$0(AbstractPythonFunctionOperator.java:125)
>     at
> org.apache.flink.streaming.runtime.tasks.StreamTask.invokeProcessingTimeCallback(StreamTask.java:1318)
>     ... 10 more