I attempted to serialize a "QueryPlan (ql/QueryPlan.java)" using Utilities.serializePlan() function, and encountered errors (pasted below); I had "hive.plan.serialization.format=javaXML". No such exception was thrown while setting serialization format to "kryo". However, I was not successful in deserializing the object after writing the serialized form to disk. So,
1. Does it make sense to serialize QueryPlan ? 2. If yes, what are the correct configurations ? 3. If not, which is the ideal data structure to serialize after the query compilation stage ? Thanks, Raajay ps: Apologies for including long log output, but perhaps it is helpful. 15/09/03 19:21:51 [main]: INFO exec.Utilities: Serializing QueryPlan via javaXML 15/09/03 19:21:51 [main]: INFO exec.Utilities: Raajay: Serializing using JAVA-XML 15/09/03 19:21:51 [main]: WARN exec.Utilities: java.lang.InstantiationException: org.apache.hadoop.hive.ql.plan.TezWork at java.lang.Class.newInstance(Class.java:427) at sun.reflect.GeneratedMethodAccessor54.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:497) at sun.reflect.misc.Trampoline.invoke(MethodUtil.java:71) at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:497) at sun.reflect.misc.MethodUtil.invoke(MethodUtil.java:275) at java.beans.Statement.invokeInternal(Statement.java:292) at java.beans.Statement.access$000(Statement.java:58) at java.beans.Statement$2.run(Statement.java:185) at java.security.AccessController.doPrivileged(Native Method) at java.beans.Statement.invoke(Statement.java:182) at java.beans.Expression.getValue(Expression.java:155) at java.beans.Encoder.getValue(Encoder.java:105) at java.beans.Encoder.get(Encoder.java:252) at java.beans.PersistenceDelegate.writeObject(PersistenceDelegate.java:112) at java.beans.Encoder.writeObject(Encoder.java:74) at java.beans.XMLEncoder.writeObject(XMLEncoder.java:327) at java.beans.Encoder.writeExpression(Encoder.java:330) at java.beans.XMLEncoder.writeExpression(XMLEncoder.java:454) at java.beans.PersistenceDelegate.writeObject(PersistenceDelegate.java:115) at java.beans.Encoder.writeObject(Encoder.java:74) at java.beans.XMLEncoder.writeObject(XMLEncoder.java:327) at java.beans.Encoder.writeExpression(Encoder.java:330) at java.beans.XMLEncoder.writeExpression(XMLEncoder.java:454) at java.beans.DefaultPersistenceDelegate.doProperty(DefaultPersistenceDelegate.java:196) at java.beans.DefaultPersistenceDelegate.initBean(DefaultPersistenceDelegate.java:258) at java.beans.DefaultPersistenceDelegate.initialize(DefaultPersistenceDelegate.java:406) at java.beans.PersistenceDelegate.writeObject(PersistenceDelegate.java:118) at java.beans.Encoder.writeObject(Encoder.java:74) at java.beans.XMLEncoder.writeObject(XMLEncoder.java:327) at java.beans.Encoder.writeExpression(Encoder.java:330) at java.beans.XMLEncoder.writeExpression(XMLEncoder.java:454) at java.beans.PersistenceDelegate.writeObject(PersistenceDelegate.java:115) at java.beans.Encoder.writeObject(Encoder.java:74) at java.beans.XMLEncoder.writeObject(XMLEncoder.java:327) at java.beans.Encoder.writeObject1(Encoder.java:258) at java.beans.Encoder.cloneStatement(Encoder.java:271) at java.beans.Encoder.writeStatement(Encoder.java:301) at java.beans.XMLEncoder.writeStatement(XMLEncoder.java:400) at java.beans.DefaultPersistenceDelegate.invokeStatement(DefaultPersistenceDelegate.java:219) at java.beans.MetaData$java_util_List_PersistenceDelegate.initialize(MetaData.java:655) at java.beans.PersistenceDelegate.initialize(PersistenceDelegate.java:214) at java.beans.DefaultPersistenceDelegate.initialize(DefaultPersistenceDelegate.java:404) at java.beans.PersistenceDelegate.writeObject(PersistenceDelegate.java:118) at java.beans.Encoder.writeObject(Encoder.java:74) at java.beans.XMLEncoder.writeObject(XMLEncoder.java:327) at java.beans.Encoder.writeExpression(Encoder.java:330) at java.beans.XMLEncoder.writeExpression(XMLEncoder.java:454) at java.beans.PersistenceDelegate.writeObject(PersistenceDelegate.java:115) at java.beans.Encoder.writeObject(Encoder.java:74) at java.beans.XMLEncoder.writeObject(XMLEncoder.java:327) at java.beans.Encoder.writeExpression(Encoder.java:330) at java.beans.XMLEncoder.writeExpression(XMLEncoder.java:454) at java.beans.DefaultPersistenceDelegate.doProperty(DefaultPersistenceDelegate.java:196) at java.beans.DefaultPersistenceDelegate.initBean(DefaultPersistenceDelegate.java:258) at java.beans.DefaultPersistenceDelegate.initialize(DefaultPersistenceDelegate.java:406) at java.beans.PersistenceDelegate.writeObject(PersistenceDelegate.java:118) at java.beans.Encoder.writeObject(Encoder.java:74) at java.beans.XMLEncoder.writeObject(XMLEncoder.java:327) at java.beans.Encoder.writeExpression(Encoder.java:330) at java.beans.XMLEncoder.writeExpression(XMLEncoder.java:454) at java.beans.PersistenceDelegate.writeObject(PersistenceDelegate.java:115) at java.beans.Encoder.writeObject(Encoder.java:74) at java.beans.XMLEncoder.writeObject(XMLEncoder.java:327) at java.beans.Encoder.writeObject1(Encoder.java:258) at java.beans.Encoder.cloneStatement(Encoder.java:271) at java.beans.Encoder.writeStatement(Encoder.java:301) at java.beans.XMLEncoder.writeStatement(XMLEncoder.java:400) at java.beans.XMLEncoder.writeObject(XMLEncoder.java:330) at org.apache.hadoop.hive.ql.exec.Utilities.serializeObjectByJavaXML(Utilities.java:1066) at org.apache.hadoop.hive.ql.exec.Utilities.serializePlan(Utilities.java:946) at org.apache.hadoop.hive.ql.exec.Utilities.serializePlan(Utilities.java:966) at org.apache.hadoop.hive.ql.Driver.execute(Driver.java:1511) at org.apache.hadoop.hive.ql.Driver.runInternal(Driver.java:1284) at org.apache.hadoop.hive.ql.Driver.run(Driver.java:1100) at org.apache.hadoop.hive.ql.Driver.run(Driver.java:1090) at org.apache.hadoop.hive.cli.CliDriver.processLocalCmd(CliDriver.java:213) at org.apache.hadoop.hive.cli.CliDriver.processCmd(CliDriver.java:165) at org.apache.hadoop.hive.cli.CliDriver.processLine(CliDriver.java:376) at org.apache.hadoop.hive.cli.CliDriver.processLine(CliDriver.java:311) at org.apache.hadoop.hive.cli.CliDriver.processReader(CliDriver.java:409) at org.apache.hadoop.hive.cli.CliDriver.processFile(CliDriver.java:425) at org.apache.hadoop.hive.cli.CliDriver.executeDriver(CliDriver.java:714) at org.apache.hadoop.hive.cli.CliDriver.run(CliDriver.java:681) at org.apache.hadoop.hive.cli.CliDriver.main(CliDriver.java:621) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:497) at org.apache.hadoop.util.RunJar.run(RunJar.java:221) at org.apache.hadoop.util.RunJar.main(RunJar.java:136) Caused by: java.lang.NoSuchMethodException: org.apache.hadoop.hive.ql.plan.TezWork.<init>() at java.lang.Class.getConstructor0(Class.java:3082) at java.lang.Class.newInstance(Class.java:412) ... 93 more
