[
https://issues.apache.org/jira/browse/FLINK-15798?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17128546#comment-17128546
]
Robert Metzger edited comment on FLINK-15798 at 6/8/20, 7:02 PM:
-----------------------------------------------------------------
I got this error also on job submission. Logging in DEBUG level does not reveal
any helpful messages
{code}
2020-06-08 20:56:52,865 INFO
org.apache.flink.configuration.GlobalConfiguration [] - Loading
configuration property: jobmanager.execution.failover-strategy, region
2020-06-08 20:56:52,923 DEBUG io.fabric8.kubernetes.client.Config
[] - Trying to configure client from Kubernetes config...
2020-06-08 20:56:52,924 DEBUG io.fabric8.kubernetes.client.Config
[] - Found for Kubernetes config at: [/Users/robert/.kube/config].
2020-06-08 20:56:53,170 DEBUG io.fabric8.kubernetes.client.Config
[] - Trying to configure client from Kubernetes config...
2020-06-08 20:56:53,170 DEBUG io.fabric8.kubernetes.client.Config
[] - Found for Kubernetes config at: [/Users/robert/.kube/config].
2020-06-08 20:56:53,563 ERROR org.apache.flink.client.cli.CliFrontend
[] - Error while running the command.
org.apache.flink.client.program.ProgramInvocationException: The main method
caused an error: Failure executing: GET at:
https://35.205.14.250/api/v1/namespaces/default/services/rob-test0-rest.
Message: Unauthorized! Token may have expired! Please log-in again.
Unauthorized.
at
org.apache.flink.client.program.PackagedProgram.callMainMethod(PackagedProgram.java:302)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
at
org.apache.flink.client.program.PackagedProgram.invokeInteractiveModeForExecution(PackagedProgram.java:198)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
at
org.apache.flink.client.ClientUtils.executeProgram(ClientUtils.java:148)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
at
org.apache.flink.client.cli.CliFrontend.executeProgram(CliFrontend.java:699)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
at org.apache.flink.client.cli.CliFrontend.run(CliFrontend.java:232)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
at
org.apache.flink.client.cli.CliFrontend.parseParameters(CliFrontend.java:916)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
at
org.apache.flink.client.cli.CliFrontend.lambda$main$10(CliFrontend.java:992)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
at
org.apache.flink.runtime.security.contexts.NoOpSecurityContext.runSecured(NoOpSecurityContext.java:30)
[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
at org.apache.flink.client.cli.CliFrontend.main(CliFrontend.java:992)
[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
Caused by: io.fabric8.kubernetes.client.KubernetesClientException: Failure
executing: GET at:
https://35.205.14.250/api/v1/namespaces/default/services/rob-test0-rest.
Message: Unauthorized! Token may have expired! Please log-in again.
Unauthorized.
at
io.fabric8.kubernetes.client.dsl.base.OperationSupport.requestFailure(OperationSupport.java:568)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
at
io.fabric8.kubernetes.client.dsl.base.OperationSupport.assertResponseCode(OperationSupport.java:505)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
at
io.fabric8.kubernetes.client.dsl.base.OperationSupport.handleResponse(OperationSupport.java:471)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
at
io.fabric8.kubernetes.client.dsl.base.OperationSupport.handleResponse(OperationSupport.java:430)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
at
io.fabric8.kubernetes.client.dsl.base.OperationSupport.handleGet(OperationSupport.java:395)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
at
io.fabric8.kubernetes.client.dsl.base.OperationSupport.handleGet(OperationSupport.java:376)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
at
io.fabric8.kubernetes.client.dsl.base.BaseOperation.handleGet(BaseOperation.java:845)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
at
io.fabric8.kubernetes.client.dsl.base.BaseOperation.getMandatory(BaseOperation.java:214)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
at
io.fabric8.kubernetes.client.dsl.base.BaseOperation.get(BaseOperation.java:168)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
at
org.apache.flink.kubernetes.kubeclient.Fabric8FlinkKubeClient.getRestService(Fabric8FlinkKubeClient.java:202)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
at
org.apache.flink.kubernetes.kubeclient.Fabric8FlinkKubeClient.getRestEndpoint(Fabric8FlinkKubeClient.java:144)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
at
org.apache.flink.kubernetes.KubernetesClusterDescriptor.lambda$createClusterClientProvider$0(KubernetesClusterDescriptor.java:96)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
at
org.apache.flink.kubernetes.KubernetesClusterDescriptor.retrieve(KubernetesClusterDescriptor.java:126)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
at
org.apache.flink.kubernetes.KubernetesClusterDescriptor.retrieve(KubernetesClusterDescriptor.java:67)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
at
org.apache.flink.client.deployment.executors.AbstractSessionClusterExecutor.execute(AbstractSessionClusterExecutor.java:63)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
at
org.apache.flink.api.java.ExecutionEnvironment.executeAsync(ExecutionEnvironment.java:962)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
at
org.apache.flink.client.program.ContextEnvironment.executeAsync(ContextEnvironment.java:124)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
at
org.apache.flink.client.program.ContextEnvironment.execute(ContextEnvironment.java:72)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
at
org.apache.flink.examples.java.wordcount.WordCount.main(WordCount.java:93)
~[?:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
~[?:1.8.0_252]
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
~[?:1.8.0_252]
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
~[?:1.8.0_252]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_252]
at
org.apache.flink.client.program.PackagedProgram.callMainMethod(PackagedProgram.java:288)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
... 8 more
{code}
I was able to reproduce the issue until I did a {{kubectl get pods}} on my
command line. Maybe there's some token somewhere that expires quickly and only
{{kubectl}} can refresh it.
was (Author: rmetzger):
I got this error also on job submission. Logging in DEBUG level does not reveal
any helpful messages
{code}
2020-06-08 20:56:52,865 INFO
org.apache.flink.configuration.GlobalConfiguration [] - Loading
configuration property: jobmanager.execution.failover-strategy, region
2020-06-08 20:56:52,923 DEBUG io.fabric8.kubernetes.client.Config
[] - Trying to configure client from Kubernetes config...
2020-06-08 20:56:52,924 DEBUG io.fabric8.kubernetes.client.Config
[] - Found for Kubernetes config at: [/Users/robert/.kube/config].
2020-06-08 20:56:53,170 DEBUG io.fabric8.kubernetes.client.Config
[] - Trying to configure client from Kubernetes config...
2020-06-08 20:56:53,170 DEBUG io.fabric8.kubernetes.client.Config
[] - Found for Kubernetes config at: [/Users/robert/.kube/config].
2020-06-08 20:56:53,563 ERROR org.apache.flink.client.cli.CliFrontend
[] - Error while running the command.
org.apache.flink.client.program.ProgramInvocationException: The main method
caused an error: Failure executing: GET at:
https://35.205.14.250/api/v1/namespaces/default/services/rob-test0-rest.
Message: Unauthorized! Token may have expired! Please log-in again.
Unauthorized.
at
org.apache.flink.client.program.PackagedProgram.callMainMethod(PackagedProgram.java:302)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
at
org.apache.flink.client.program.PackagedProgram.invokeInteractiveModeForExecution(PackagedProgram.java:198)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
at
org.apache.flink.client.ClientUtils.executeProgram(ClientUtils.java:148)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
at
org.apache.flink.client.cli.CliFrontend.executeProgram(CliFrontend.java:699)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
at org.apache.flink.client.cli.CliFrontend.run(CliFrontend.java:232)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
at
org.apache.flink.client.cli.CliFrontend.parseParameters(CliFrontend.java:916)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
at
org.apache.flink.client.cli.CliFrontend.lambda$main$10(CliFrontend.java:992)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
at
org.apache.flink.runtime.security.contexts.NoOpSecurityContext.runSecured(NoOpSecurityContext.java:30)
[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
at org.apache.flink.client.cli.CliFrontend.main(CliFrontend.java:992)
[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
Caused by: io.fabric8.kubernetes.client.KubernetesClientException: Failure
executing: GET at:
https://35.205.14.250/api/v1/namespaces/default/services/rob-test0-rest.
Message: Unauthorized! Token may have expired! Please log-in again.
Unauthorized.
at
io.fabric8.kubernetes.client.dsl.base.OperationSupport.requestFailure(OperationSupport.java:568)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
at
io.fabric8.kubernetes.client.dsl.base.OperationSupport.assertResponseCode(OperationSupport.java:505)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
at
io.fabric8.kubernetes.client.dsl.base.OperationSupport.handleResponse(OperationSupport.java:471)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
at
io.fabric8.kubernetes.client.dsl.base.OperationSupport.handleResponse(OperationSupport.java:430)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
at
io.fabric8.kubernetes.client.dsl.base.OperationSupport.handleGet(OperationSupport.java:395)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
at
io.fabric8.kubernetes.client.dsl.base.OperationSupport.handleGet(OperationSupport.java:376)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
at
io.fabric8.kubernetes.client.dsl.base.BaseOperation.handleGet(BaseOperation.java:845)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
at
io.fabric8.kubernetes.client.dsl.base.BaseOperation.getMandatory(BaseOperation.java:214)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
at
io.fabric8.kubernetes.client.dsl.base.BaseOperation.get(BaseOperation.java:168)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
at
org.apache.flink.kubernetes.kubeclient.Fabric8FlinkKubeClient.getRestService(Fabric8FlinkKubeClient.java:202)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
at
org.apache.flink.kubernetes.kubeclient.Fabric8FlinkKubeClient.getRestEndpoint(Fabric8FlinkKubeClient.java:144)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
at
org.apache.flink.kubernetes.KubernetesClusterDescriptor.lambda$createClusterClientProvider$0(KubernetesClusterDescriptor.java:96)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
at
org.apache.flink.kubernetes.KubernetesClusterDescriptor.retrieve(KubernetesClusterDescriptor.java:126)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
at
org.apache.flink.kubernetes.KubernetesClusterDescriptor.retrieve(KubernetesClusterDescriptor.java:67)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
at
org.apache.flink.client.deployment.executors.AbstractSessionClusterExecutor.execute(AbstractSessionClusterExecutor.java:63)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
at
org.apache.flink.api.java.ExecutionEnvironment.executeAsync(ExecutionEnvironment.java:962)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
at
org.apache.flink.client.program.ContextEnvironment.executeAsync(ContextEnvironment.java:124)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
at
org.apache.flink.client.program.ContextEnvironment.execute(ContextEnvironment.java:72)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
at
org.apache.flink.examples.java.wordcount.WordCount.main(WordCount.java:93)
~[?:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
~[?:1.8.0_252]
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
~[?:1.8.0_252]
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
~[?:1.8.0_252]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_252]
at
org.apache.flink.client.program.PackagedProgram.callMainMethod(PackagedProgram.java:288)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
... 8 more
{code}
> Running ./bin/kubernetes-session.sh -Dkubernetes.cluster-id=<CLUSTER-ID>
> -Dexecution.attached=true fails with exception
> -----------------------------------------------------------------------------------------------------------------------
>
> Key: FLINK-15798
> URL: https://issues.apache.org/jira/browse/FLINK-15798
> Project: Flink
> Issue Type: Sub-task
> Components: Deployment / Kubernetes
> Affects Versions: 1.10.0
> Reporter: Till Rohrmann
> Priority: Major
>
> Running {{./bin/kubernetes-session.sh -Dkubernetes.cluster-id=<CLUSTER-ID>
> -Dexecution.attached=true}} fails with
> {code}
> 2020-01-28 15:04:28,669 ERROR
> org.apache.flink.kubernetes.cli.KubernetesSessionCli - Error while
> running the Flink session.
> io.fabric8.kubernetes.client.KubernetesClientException: Failure executing:
> GET at: https://35.234.77.125/api/v1/namespaces/default/services/testing.
> Message: Unauthorized! Token may have expired! Please log-in again.
> Unauthorized.
> at
> io.fabric8.kubernetes.client.dsl.base.OperationSupport.requestFailure(OperationSupport.java:510)
> at
> io.fabric8.kubernetes.client.dsl.base.OperationSupport.assertResponseCode(OperationSupport.java:447)
> at
> io.fabric8.kubernetes.client.dsl.base.OperationSupport.handleResponse(OperationSupport.java:413)
> at
> io.fabric8.kubernetes.client.dsl.base.OperationSupport.handleResponse(OperationSupport.java:372)
> at
> io.fabric8.kubernetes.client.dsl.base.OperationSupport.handleGet(OperationSupport.java:337)
> at
> io.fabric8.kubernetes.client.dsl.base.OperationSupport.handleGet(OperationSupport.java:318)
> at
> io.fabric8.kubernetes.client.dsl.base.BaseOperation.handleGet(BaseOperation.java:812)
> at
> io.fabric8.kubernetes.client.dsl.base.BaseOperation.getMandatory(BaseOperation.java:220)
> at
> io.fabric8.kubernetes.client.dsl.base.BaseOperation.get(BaseOperation.java:164)
> at
> org.apache.flink.kubernetes.kubeclient.Fabric8FlinkKubeClient.getService(Fabric8FlinkKubeClient.java:330)
> at
> org.apache.flink.kubernetes.kubeclient.Fabric8FlinkKubeClient.getInternalService(Fabric8FlinkKubeClient.java:243)
> at
> org.apache.flink.kubernetes.cli.KubernetesSessionCli.run(KubernetesSessionCli.java:104)
> at
> org.apache.flink.kubernetes.cli.KubernetesSessionCli.lambda$main$0(KubernetesSessionCli.java:185)
> at
> org.apache.flink.runtime.security.NoOpSecurityContext.runSecured(NoOpSecurityContext.java:30)
> at
> org.apache.flink.kubernetes.cli.KubernetesSessionCli.main(KubernetesSessionCli.java:185)
> {code}
> even though {{echo "stop" | ./bin/kubernetes-session.sh
> -Dkubernetes.cluster-id=<CLUSTER-ID> -Dexecution.attached=true}} succeeds
> with my setup. This is strange as I would expect that the former call should
> do exactly the same as the second except for sending the "stop" command right
> away. I think we should check whether this is a real problem or only specific
> to my setup.
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
