Guarav, it looks like you're trying to use Windows but don't have your Hadoop environment set up correctly. I'm not expert on Hadoop with Windows, so you might want to either run in a Docker container or ask for help in the Hadoop community.
On Wed, May 10, 2023 at 7:13 AM Gaurav Agarwal <[email protected]> wrote: > HI All > > I am trying to set up the workspace and running the test case to > understand the flow of delete functionality with different mode copyonwrite > and mergeonread. > the test case is failing locally. > > TestStoragePartitionedJoinsInRowLevelOperations.java in spark 3.3_2.12 > > I tried setting up hadoop winutils.exe 2.7 in TestHiveMetastore static > block > > static { > try { > HIVE_LOCAL_DIR = > createTempDirectory("hive").toFile(); > DERBY_PATH = new File(HIVE_LOCAL_DIR, "metastore_db").getPath(); > File derbyLogFile = new File(HIVE_LOCAL_DIR, "derby.log"); > System.setProperty("derby.stream.error.file", > derbyLogFile.getAbsolutePath()); > > System.setProperty("hadoop.home.dir","E:\\workspace\\iceberg-apache\\win"); > > still the issue is not resolved, kindly help if possible > > getting this exception: > > > org.apache.hadoop.io.nativeio.NativeIO$Windows.access0(Ljava/lang/String;I)Z > java.lang.UnsatisfiedLinkError: > org.apache.hadoop.io.nativeio.NativeIO$Windows.access0(Ljava/lang/String;I)Z > at org.apache.hadoop.io.nativeio.NativeIO$Windows.access0(Native Method) > at org.apache.hadoop.io.nativeio.NativeIO$Windows.access(NativeIO.java:793) > at org.apache.hadoop.fs.FileUtil.canRead(FileUtil.java:1218) > at org.apache.hadoop.fs.FileUtil.list(FileUtil.java:1423) > at > org.apache.hadoop.fs.RawLocalFileSystem.listStatus(RawLocalFileSystem.java:601) > at org.apache.hadoop.fs.FileSystem.listStatus(FileSystem.java:1972) > at org.apache.hadoop.fs.FileSystem.listStatus(FileSystem.java:2014) > at > org.apache.hadoop.fs.ChecksumFileSystem.listStatus(ChecksumFileSystem.java:761) > at > org.apache.iceberg.hive.TestHiveMetastore.reset(TestHiveMetastore.java:227) > at > org.apache.iceberg.hive.TestHiveMetastore.stop(TestHiveMetastore.java:178) > at > org.apache.iceberg.spark.SparkTestBase.stopMetastoreAndSpark(SparkTestBase.java:101) > at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native > Method) > at > java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) > at > java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > at java.base/java.lang.reflect.Method.invoke(Method.java:566) > at > org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) > at > org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) > at > org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) > at > org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) > at > org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) > at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) > at org.junit.runners.ParentRunner.run(ParentRunner.java:413) > at > org.gradle.api.internal.tasks.testing.junit.JUnitTestClassExecutor.runTestClass(JUnitTestClassExecutor.java:108) > at > org.gradle.api.internal.tasks.testing.junit.JUnitTestClassExecutor.execute(JUnitTestClassExecutor.java:58) > at > org.gradle.api.internal.tasks.testing.junit.JUnitTestClassExecutor.execute(JUnitTestClassExecutor.java:40) > at > org.gradle.api.internal.tasks.testing.junit.AbstractJUnitTestClassProcessor.processTestClass(AbstractJUnitTestClassProcessor.java:60) > at > org.gradle.api.internal.tasks.testing.SuiteTestClassProcessor.processTestClass(SuiteTestClassProcessor.java:52) > at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native > Method) > at > java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) > at > java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > at java.base/java.lang.reflect.Method.invoke(Method.java:566) > at > org.gradle.internal.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:36) > at > org.gradle.internal.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:24) > at > org.gradle.internal.dispatch.ContextClassLoaderDispatch.dispatch(ContextClassLoaderDispatch.java:33) > at > org.gradle.internal.dispatch.ProxyDispatchAdapter$DispatchingInvocationHandler.invoke(ProxyDispatchAdapter.java:94) > at com.sun.proxy.$Proxy2.processTestClass(Unknown Source) > at > org.gradle.api.internal.tasks.testing.worker.TestWorker$2.run(TestWorker.java:176) > at > org.gradle.api.internal.tasks.testing.worker.TestWorker.executeAndMaintainThreadName(TestWorker.java:129) > at > org.gradle.api.internal.tasks.testing.worker.TestWorker.execute(TestWorker.java:100) > at > org.gradle.api.internal.tasks.testing.worker.TestWorker.execute(TestWorker.java:60) > at > org.gradle.process.internal.worker.child.ActionExecutionWorker.execute(ActionExecutionWorker.java:56) > at > org.gradle.process.internal.worker.child.SystemApplicationClassLoaderWorker.call(SystemApplicationClassLoaderWorker.java:113) > at > org.gradle.process.internal.worker.child.SystemApplicationClassLoaderWorker.call(SystemApplicationClassLoaderWorker.java:65) > at > worker.org.gradle.process.internal.worker.GradleWorkerMain.run(GradleWorkerMain.java:69) > at > worker.org.gradle.process.internal.worker.GradleWorkerMain.main(GradleWorkerMain.java:74) > > > org.apache.iceberg.spark.extensions.TestStoragePartitionedJoinsInRowLevelOperations > > classMethod FAILED > java.lang.RuntimeException: Cannot start TestHiveMetastore > at > org.apache.iceberg.hive.TestHiveMetastore.start(TestHiveMetastore.java:173) > at > org.apache.iceberg.hive.TestHiveMetastore.start(TestHiveMetastore.java:136) > at > org.apache.iceberg.spark.extensions.SparkExtensionsTestBase.startMetastoreAndSpark(SparkExtensionsTestBase.java:48) > at > java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native > Method) > at > java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) > at > java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > at java.base/java.lang.reflect.Method.invoke(Method.java:566) > at > org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) > at > org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) > at > org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) > at > org.junit.internal.runners.statements.RunBefores.invokeMethod(RunBefores.java:33) > at > org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:24) > at > org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) > at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) > at org.junit.runners.ParentRunner.run(ParentRunner.java:413) > at > org.gradle.api.internal.tasks.testing.junit.JUnitTestClassExecutor.runTestClass(JUnitTestClassExecutor.java:108) > at > org.gradle.api.internal.tasks.testing.junit.JUnitTestClassExecutor.execute(JUnitTestClassExecutor.java:58) > at > org.gradle.api.internal.tasks.testing.junit.JUnitTestClassExecutor.execute(JUnitTestClassExecutor.java:40) > at > org.gradle.api.internal.tasks.testing.junit.AbstractJUnitTestClassProcessor.processTestClass(AbstractJUnitTestClassProcessor.java:60) > at > org.gradle.api.internal.tasks.testing.SuiteTestClassProcessor.processTestClass(SuiteTestClassProcessor.java:52) > at > java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native > Method) > at > java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) > at > java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > at java.base/java.lang.reflect.Method.invoke(Method.java:566) > at > org.gradle.internal.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:36) > at > org.gradle.internal.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:24) > at > org.gradle.internal.dispatch.ContextClassLoaderDispatch.dispatch(ContextClassLoaderDispatch.java:33) > at > org.gradle.internal.dispatch.ProxyDispatchAdapter$DispatchingInvocationHandler.invoke(ProxyDispatchAdapter.java:94) > at com.sun.proxy.$Proxy2.processTestClass(Unknown Source) > at > org.gradle.api.internal.tasks.testing.worker.TestWorker$2.run(TestWorker.java:176) > at > org.gradle.api.internal.tasks.testing.worker.TestWorker.executeAndMaintainThreadName(TestWorker.java:129) > at > org.gradle.api.internal.tasks.testing.worker.TestWorker.execute(TestWorker.java:100) > at > org.gradle.api.internal.tasks.testing.worker.TestWorker.execute(TestWorker.java:60) > at > org.gradle.process.internal.worker.child.ActionExecutionWorker.execute(ActionExecutionWorker.java:56) > at > org.gradle.process.internal.worker.child.SystemApplicationClassLoaderWorker.call(SystemApplicationClassLoaderWorker.java:113) > at > org.gradle.process.internal.worker.child.SystemApplicationClassLoaderWorker.call(SystemApplicationClassLoaderWorker.java:65) > at > worker.org.gradle.process.internal.worker.GradleWorkerMain.run(GradleWorkerMain.java:69) > at > worker.org.gradle.process.internal.worker.GradleWorkerMain.main(GradleWorkerMain.java:74) > Caused by: > java.lang.IllegalArgumentException: java.net.URISyntaxException: > Relative path in absolute URI: > file:C:/Users/gaurav/AppData/Local/Temp/hive18306566538493121237 > at org.apache.hadoop.fs.Path.initialize(Path.java:263) > at org.apache.hadoop.fs.Path.<init>(Path.java:221) > at > org.apache.hadoop.hive.metastore.Warehouse.getWhRoot(Warehouse.java:150) > at > org.apache.hadoop.hive.metastore.Warehouse.getDefaultDatabasePath(Warehouse.java:163) > at > org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB_core(HiveMetaStore.java:636) > at > org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.createDefaultDB(HiveMetaStore.java:655) > at > org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:431) > at > org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.<init>(HiveMetaStore.java:380) > at > org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.<init>(HiveMetaStore.java:363) > at > java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native > Method) > at > java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) > at > java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) > at > java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:490) > at > org.apache.iceberg.common.DynConstructors$Ctor.newInstanceChecked(DynConstructors.java:55) > at > org.apache.iceberg.common.DynConstructors$Ctor.newInstance(DynConstructors.java:68) > at > org.apache.iceberg.hive.TestHiveMetastore.newThriftServer(TestHiveMetastore.java:249) > at > org.apache.iceberg.hive.TestHiveMetastore.start(TestHiveMetastore.java:162) > ... 37 more > Caused by: > java.net.URISyntaxException: Relative path in absolute URI: > file:C:/Users/gaurav/AppData/Local/Temp/hive18306566538493121237 > at java.base/java.net.URI.checkPath(URI.java:1940) > at java.base/java.net.URI.<init>(URI.java:757) > at org.apache.hadoop.fs.Path.initialize(Path.java:260) > ... 53 more > org.apache.iceberg.spark.extensions.TestStoragePartitionedJoinsInRowLevelOperations > > classMethod FAILED > java.lang.UnsatisfiedLinkError: > org.apache.hadoop.io.nativeio.NativeIO$Windows.access0(Ljava/lang/String;I)Z > at org.apache.hadoop.io.nativeio.NativeIO$Windows.access0(Native > Method) > at > org.apache.hadoop.io.nativeio.NativeIO$Windows.access(NativeIO.java:793) > at org.apache.hadoop.fs.FileUtil.canRead(FileUtil.java:1218) > at org.apache.hadoop.fs.FileUtil.list(FileUtil.java:1423) > at > org.apache.hadoop.fs.RawLocalFileSystem.listStatus(RawLocalFileSystem.java:601) > at org.apache.hadoop.fs.FileSystem.listStatus(FileSystem.java:1972) > at org.apache.hadoop.fs.FileSystem.listStatus(FileSystem.java:2014) > at > org.apache.hadoop.fs.ChecksumFileSystem.listStatus(ChecksumFileSystem.java:761) > at > org.apache.iceberg.hive.TestHiveMetastore.reset(TestHiveMetastore.java:227) > at > org.apache.iceberg.hive.TestHiveMetastore.stop(TestHiveMetastore.java:178) > at > org.apache.iceberg.spark.SparkTestBase.stopMetastoreAndSpark(SparkTestBase.java:101) > at > java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native > Method) > at > java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) > at > java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > at java.base/java.lang.reflect.Method.invoke(Method.java:566) > at > org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) > at > org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) > at > org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) > at > org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) > at > org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) > at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) > at org.junit.runners.ParentRunner.run(ParentRunner.java:413) > at > org.gradle.api.internal.tasks.testing.junit.JUnitTestClassExecutor.runTestClass(JUnitTestClassExecutor.java:108) > at > org.gradle.api.internal.tasks.testing.junit.JUnitTestClassExecutor.execute(JUnitTestClassExecutor.java:58) > at > org.gradle.api.internal.tasks.testing.junit.JUnitTestClassExecutor.execute(JUnitTestClassExecutor.java:40) > at > org.gradle.api.internal.tasks.testing.junit.AbstractJUnitTestClassProcessor.processTestClass(AbstractJUnitTestClassProcessor.java:60) > at > org.gradle.api.internal.tasks.testing.SuiteTestClassProcessor.processTestClass(SuiteTestClassProcessor.java:52) > at > java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native > Method) > at > java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) > at > java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > at java.base/java.lang.reflect.Method.invoke(Method.java:566) > at > org.gradle.internal.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:36) > at > org.gradle.internal.dispatch.ReflectionDispatch.dispatch(ReflectionDispatch.java:24) > at > org.gradle.internal.dispatch.ContextClassLoaderDispatch.dispatch(ContextClassLoaderDispatch.java:33) > at > org.gradle.internal.dispatch.ProxyDispatchAdapter$DispatchingInvocationHandler.invoke(ProxyDispatchAdapter.java:94) > at com.sun.proxy.$Proxy2.processTestClass(Unknown Source) > at > org.gradle.api.internal.tasks.testing.worker.TestWorker$2.run(TestWorker.java:176) > at > org.gradle.api.internal.tasks.testing.worker.TestWorker.executeAndMaintainThreadName(TestWorker.java:129) > at > org.gradle.api.internal.tasks.testing.worker.TestWorker.execute(TestWorker.java:100) > at > org.gradle.api.internal.tasks.testing.worker.TestWorker.execute(TestWorker.java:60) > at > org.gradle.process.internal.worker.child.ActionExecutionWorker.execute(ActionExecutionWorker.java:56) > at > org.gradle.process.internal.worker.child.SystemApplicationClassLoaderWorker.call(SystemApplicationClassLoaderWorker.java:113) > at > org.gradle.process.internal.worker.child.SystemApplicationClassLoaderWorker.call(SystemApplicationClassLoaderWorker.java:65) > at > worker.org.gradle.process.internal.worker.GradleWorkerMain.run(GradleWorkerMain.java:69) > at > worker.org.gradle.process.internal.worker.GradleWorkerMain.main(GradleWorkerMain.java:74) > -- Ryan Blue Tabular
