Hello.
We are trying to use the Ignite Memory File System and sometimes Ignite can't write file to IGFS and can't read. What is this happens? Below is an example for Cloudera Quick Start VM 5.10.0 and error, also configuration and full log in attachments. This problems arise on our cluster with CentOS 7 and CDH 5.11.1 too. In-Memory Hadoop Accelerator: Version 2.1.0 Date 2017-07-27 File http://apache-mirror.rbc.ru/pub/apache//ignite/2.1.0/apache-ignite-hadoop-2. 1.0-bin.zip <http://apache-mirror.rbc.ru/pub/apache/ignite/2.1.0/apache-ignite-hadoop-2. 1.0-bin.zip> [cloudera@quickstart ~]$ ls -l dtm_ekp_scoring_plan_oper75.csv -rw-r--r-- 1 cloudera cloudera 19579883 Aug 16 03:53 dtm_ekp_scoring_plan_oper75.csv [cloudera@quickstart ~]$ hdfs dfs -mkdir -p igfs://igfs@/user/cloudera/dtm_ekp_scoring_plan_oper/ [cloudera@quickstart ~]$ hdfs dfs -put dtm_ekp_scoring_plan_oper75.csv igfs://igfs@/user/cloudera/dtm_ekp_scoring_plan_oper/ put: Failed to flush data during stream close [path=/user/cloudera/dtm_ekp_scoring_plan_oper/dtm_ekp_scoring_plan_oper75.c sv._COPYING_, fileInfo=IgfsFileInfo [len=0, blockSize=65536, lockId=4600eafed51-15b0cff9-0c6e-459c-8c1e-1d8f59d102e6, affKey=null, fileMap=IgfsFileMap [ranges=null], evictExclude=true]] [2017-08-17 03:13:07,951][ERROR][igfs-#47%null%][GridNearTxLocal] Heuristic transaction failure. class org.apache.ignite.internal.transactions.IgniteTxHeuristicCheckedException: Failed to locally write to cache (all transaction entries will be invalidated, however there was a window when entries for this transaction were visible to others): GridNearTxLocal [mappings=IgniteTxMappingsSingleImpl [mapping=GridDistributedTxMapping [entries=[IgniteTxEntry [key=KeyCacheObjectImpl [part=954, val=IgfsBlockKey [fileId=1600eafed51-cd651f8d-10b5-4cc3-9c14-e74963c7c2be, blockId=130, affKey=null, evictExclude=true], hasValBytes=true], cacheId=-313790114, txKey=IgniteTxKey [key=KeyCacheObjectImpl [part=954, val=IgfsBlockKey [fileId=1600eafed51-cd651f8d-10b5-4cc3-9c14-e74963c7c2be, blockId=130, affKey=null, evictExclude=true], hasValBytes=true], cacheId=-313790114], val=[op=CREATE, val=CacheObjectByteArrayImpl [arrLen=65536]], prevVal=[op=NOOP, val=null], oldVal=[op=NOOP, val=null], entryProcessorsCol=null, ttl=-1, conflictExpireTime=-1, conflictVer=null, explicitVer=null, dhtVer=null, filters=[], filtersPassed=false, filtersSet=true, entry=GridDhtCacheEntry [rdrs=[], part=954, super=GridDistributedCacheEntry [super=GridCacheMapEntry [key=KeyCacheObjectImpl [part=954, val=IgfsBlockKey [fileId=1600eafed51-cd651f8d-10b5-4cc3-9c14-e74963c7c2be, blockId=130, affKey=null, evictExclude=true], hasValBytes=true], val=null, startVer=1502964754897, ver=GridCacheVersion [topVer=114444755, order=1502964754897, nodeOrder=1], hash=236544549, extras=GridCacheMvccEntryExtras [mvcc=GridCacheMvcc [locs=[GridCacheMvccCandidate [nodeId=15b0cff9-0c6e-459c-8c1e-1d8f59d102e6, ver=GridCacheVersion [topVer=114444755, order=1502964754896, nodeOrder=1], threadId=69, id=152, topVer=AffinityTopologyVersion [topVer=1, minorTopVer=0], reentry=null, otherNodeId=15b0cff9-0c6e-459c-8c1e-1d8f59d102e6, otherVer=GridCacheVersion [topVer=114444755, order=1502964754896, nodeOrder=1], mappedDhtNodes=null, mappedNearNodes=null, ownerVer=null, serOrder=null, key=KeyCacheObjectImpl [part=954, val=IgfsBlockKey [fileId=1600eafed51-cd651f8d-10b5-4cc3-9c14-e74963c7c2be, blockId=130, affKey=null, evictExclude=true], hasValBytes=true], masks=local=1|owner=1|ready=1|reentry=0|used=0|tx=1|single_implicit=1|dht_lo cal=1|near_local=0|removed=0|read=0, prevVer=null, nextVer=null]], rmts=null]], flags=2]]], prepared=1, locked=false, nodeId=15b0cff9-0c6e-459c-8c1e-1d8f59d102e6, locMapped=false, expiryPlc=null, transferExpiryPlc=false, flags=0, partUpdateCntr=0, serReadVer=null, xidVer=GridCacheVersion [topVer=114444755, order=1502964754896, nodeOrder=1]]], explicitLock=false, dhtVer=null, last=false, nearEntries=0, clientFirst=false, node=15b0cff9-0c6e-459c-8c1e-1d8f59d102e6]], nearLocallyMapped=false, colocatedLocallyMapped=true, needCheckBackup=null, hasRemoteLocks=false, thread=igfs-#47%null%, mappings=IgniteTxMappingsSingleImpl [mapping=GridDistributedTxMapping [entries=[IgniteTxEntry [key=KeyCacheObjectImpl [part=954, val=IgfsBlockKey [fileId=1600eafed51-cd651f8d-10b5-4cc3-9c14-e74963c7c2be, blockId=130, affKey=null, evictExclude=true], hasValBytes=true], cacheId=-313790114, txKey=IgniteTxKey [key=KeyCacheObjectImpl [part=954, val=IgfsBlockKey [fileId=1600eafed51-cd651f8d-10b5-4cc3-9c14-e74963c7c2be, blockId=130, affKey=null, evictExclude=true], hasValBytes=true], cacheId=-313790114], val=[op=CREATE, val=CacheObjectByteArrayImpl [arrLen=65536]], prevVal=[op=NOOP, val=null], oldVal=[op=NOOP, val=null], entryProcessorsCol=null, ttl=-1, conflictExpireTime=-1, conflictVer=null, explicitVer=null, dhtVer=null, filters=[], filtersPassed=false, filtersSet=true, entry=GridDhtCacheEntry [rdrs=[], part=954, super=GridDistributedCacheEntry [super=GridCacheMapEntry [key=KeyCacheObjectImpl [part=954, val=IgfsBlockKey [fileId=1600eafed51-cd651f8d-10b5-4cc3-9c14-e74963c7c2be, blockId=130, affKey=null, evictExclude=true], hasValBytes=true], val=null, startVer=1502964754897, ver=GridCacheVersion [topVer=114444755, order=1502964754897, nodeOrder=1], hash=236544549, extras=GridCacheMvccEntryExtras [mvcc=GridCacheMvcc [locs=[GridCacheMvccCandidate [nodeId=15b0cff9-0c6e-459c-8c1e-1d8f59d102e6, ver=GridCacheVersion [topVer=114444755, order=1502964754896, nodeOrder=1], threadId=69, id=152, topVer=AffinityTopologyVersion [topVer=1, minorTopVer=0], reentry=null, otherNodeId=15b0cff9-0c6e-459c-8c1e-1d8f59d102e6, otherVer=GridCacheVersion [topVer=114444755, order=1502964754896, nodeOrder=1], mappedDhtNodes=null, mappedNearNodes=null, ownerVer=null, serOrder=null, key=KeyCacheObjectImpl [part=954, val=IgfsBlockKey [fileId=1600eafed51-cd651f8d-10b5-4cc3-9c14-e74963c7c2be, blockId=130, affKey=null, evictExclude=true], hasValBytes=true], masks=local=1|owner=1|ready=1|reentry=0|used=0|tx=1|single_implicit=1|dht_lo cal=1|near_local=0|removed=0|read=0, prevVer=null, nextVer=null]], rmts=null]], flags=2]]], prepared=1, locked=false, nodeId=15b0cff9-0c6e-459c-8c1e-1d8f59d102e6, locMapped=false, expiryPlc=null, transferExpiryPlc=false, flags=0, partUpdateCntr=0, serReadVer=null, xidVer=GridCacheVersion [topVer=114444755, order=1502964754896, nodeOrder=1]]], explicitLock=false, dhtVer=null, last=false, nearEntries=0, clientFirst=false, node=15b0cff9-0c6e-459c-8c1e-1d8f59d102e6]], super=GridDhtTxLocalAdapter [nearOnOriginatingNode=false, nearNodes=[], dhtNodes=[], explicitLock=false, super=IgniteTxLocalAdapter [completedBase=null, sndTransformedVals=false, depEnabled=false, txState=IgniteTxImplicitSingleStateImpl [init=true, recovery=false], super=IgniteTxAdapter [xidVer=GridCacheVersion [topVer=114444755, order=1502964754896, nodeOrder=1], writeVer=GridCacheVersion [topVer=114444755, order=1502964754898, nodeOrder=1], implicit=true, loc=true, threadId=69, startTime=1502964787910, nodeId=15b0cff9-0c6e-459c-8c1e-1d8f59d102e6, startVer=GridCacheVersion [topVer=114444755, order=1502964754896, nodeOrder=1], endVer=null, isolation=READ_COMMITTED, concurrency=OPTIMISTIC, timeout=0, sysInvalidate=false, sys=true, plc=2, commitVer=GridCacheVersion [topVer=114444755, order=1502964754896, nodeOrder=1], finalizing=NONE, invalidParts=null, state=COMMITTING, timedOut=false, topVer=AffinityTopologyVersion [topVer=1, minorTopVer=0], duration=0ms, onePhaseCommit=true], size=1]]] at org.apache.ignite.internal.processors.cache.transactions.IgniteTxLocalAdapte r.userCommit(IgniteTxLocalAdapter.java:834) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal .localFinish(GridNearTxLocal.java:3030) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinis hFuture.finish(GridNearTxFinishFuture.java:405) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal $16.apply(GridNearTxLocal.java:3180) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal $16.apply(GridNearTxLocal.java:3172) at org.apache.ignite.internal.util.future.GridFutureAdapter.notifyListener(Grid FutureAdapter.java:382) at org.apache.ignite.internal.util.future.GridFutureAdapter.listen(GridFutureAd apter.java:352) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal .commitNearTxLocalAsync(GridNearTxLocal.java:3172) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal .optimisticPutFuture(GridNearTxLocal.java:2382) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal .putAllAsync0(GridNearTxLocal.java:803) at org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal .putAllAsync(GridNearTxLocal.java:362) at org.apache.ignite.internal.processors.cache.GridCacheAdapter$35.op(GridCache Adapter.java:2814) at org.apache.ignite.internal.processors.cache.GridCacheAdapter$AsyncOp.op(Grid CacheAdapter.java:5076) at org.apache.ignite.internal.processors.cache.GridCacheAdapter.asyncOp(GridCac heAdapter.java:4225) at org.apache.ignite.internal.processors.cache.GridCacheAdapter$AsyncOpRetryFut ure.execute(GridCacheAdapter.java:4797) at org.apache.ignite.internal.processors.cache.GridCacheAdapter.asyncOp(GridCac heAdapter.java:4154) at org.apache.ignite.internal.processors.cache.GridCacheAdapter.putAllAsync0(Gr idCacheAdapter.java:2812) at org.apache.ignite.internal.processors.cache.GridCacheAdapter.putAllAsync(Gri dCacheAdapter.java:2804) at org.apache.ignite.internal.processors.cache.GridCacheProxyImpl.putAllAsync(G ridCacheProxyImpl.java:827) at org.apache.ignite.internal.processors.igfs.IgfsDataManager.storeBlocksAsync( IgfsDataManager.java:1084) at org.apache.ignite.internal.processors.igfs.IgfsDataManager.access$1500(IgfsD ataManager.java:96) at org.apache.ignite.internal.processors.igfs.IgfsDataManager$5.run(IgfsDataMan ager.java:1000) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:11 45) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:6 15) at java.lang.Thread.run(Thread.java:745) Caused by: class org.apache.ignite.IgniteException: Runtime failure on search row: org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManagerImpl$Se archRow@7b30c9d0 at org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.invok e(BPlusTree.java:1632) at org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManagerImpl$Ca cheDataStoreImpl.invoke(IgniteCacheOffheapManagerImpl.java:1204) at org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManagerImpl.in voke(IgniteCacheOffheapManagerImpl.java:352) at org.apache.ignite.internal.processors.cache.GridCacheMapEntry.storeValue(Gri dCacheMapEntry.java:3179) at org.apache.ignite.internal.processors.cache.GridCacheMapEntry.innerSet(GridC acheMapEntry.java:979) at org.apache.ignite.internal.processors.cache.transactions.IgniteTxLocalAdapte r.userCommit(IgniteTxLocalAdapter.java:666) ... 24 more Caused by: java.lang.IllegalMonitorStateException: Attempted to release write lock while not holding it [lock=00007f1bd4be0270, state=000103ba000025c3 at org.apache.ignite.internal.util.OffheapReadWriteLock.writeUnlock(OffheapRead WriteLock.java:263) at org.apache.ignite.internal.pagemem.impl.PageMemoryNoStoreImpl.writeUnlock(Pa geMemoryNoStoreImpl.java:490) at org.apache.ignite.internal.processors.cache.persistence.tree.util.PageHandle r.writeUnlock(PageHandler.java:377) at org.apache.ignite.internal.processors.cache.persistence.tree.util.PageHandle r.writePage(PageHandler.java:287) at org.apache.ignite.internal.processors.cache.persistence.DataStructure.write( DataStructure.java:281) at org.apache.ignite.internal.processors.cache.persistence.freelist.FreeListImp l.insertDataRow(FreeListImpl.java:493) at org.apache.ignite.internal.processors.cache.persistence.RowStore.addRow(RowS tore.java:84) at org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManagerImpl$Ca cheDataStoreImpl.createRow(IgniteCacheOffheapManagerImpl.java:1258) at org.apache.ignite.internal.processors.cache.GridCacheMapEntry$UpdateClosure. call(GridCacheMapEntry.java:3875) at org.apache.ignite.internal.processors.cache.GridCacheMapEntry$UpdateClosure. call(GridCacheMapEntry.java:3833) at org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree$Invok e.invokeClosure(BPlusTree.java:2988) at org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree$Invok e.access$6200(BPlusTree.java:2882) at org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.invok eDown(BPlusTree.java:1713) at org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.invok e(BPlusTree.java:1602) ... 29 more Best Regards, Usein Faradzhev Chief Software Engineer EPAM Systems Togliatti office, Russia GMT+3 (Standard) / GMT+4 (Daylight) Mobile phone: +7 919 815 00 32 Skype: usein.faradgev E-mail: <mailto:usein_faradz...@epam.com> usein_faradz...@epam.com < <http://www.epam.com/> http://www.epam.com> CONFIDENTIALITY CAUTION AND DISCLAIMER This message is intended only for the use of the individual(s) or entity(ies) to which it is addressed and contains information that is legally privileged and confidential. If you are not the intended recipient, or the person responsible for delivering the message to the intended recipient, you are hereby notified that any dissemination, distribution or copying of this communication is strictly prohibited. All unintended recipients are obliged to delete this message and destroy any printed copies.
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<!--
Ignite Spring configuration file.
When starting a standalone Ignite node, you need to execute the following command:
{IGNITE_HOME}/bin/ignite.{bat|sh} path-to-this-file/default-config.xml
When starting Ignite from Java IDE, pass path to this file into Ignition:
Ignition.start("path-to-this-file/default-config.xml");
-->
<beans xmlns="http://www.springframework.org/schema/beans";
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xmlns:util="http://www.springframework.org/schema/util";
xsi:schemaLocation="http://www.springframework.org/schema/beans
http://www.springframework.org/schema/beans/spring-beans.xsd
http://www.springframework.org/schema/util
http://www.springframework.org/schema/util/spring-util.xsd";>
<!-- Optional description. -->
<description>
Spring file for Ignite node configuration with IGFS and Apache Hadoop map-reduce support enabled.
Ignite node will start with this configuration by default.
</description>
<!-- Initialize property configurer so we can reference environment variables. -->
<bean id="propertyConfigurer" class="org.springframework.beans.factory.config.PropertyPlaceholderConfigurer">
<property name="systemPropertiesModeName" value="SYSTEM_PROPERTIES_MODE_FALLBACK"/>
<property name="searchSystemEnvironment" value="true"/>
</bean>
<!-- Configuration of Ignite node. -->
<bean id="grid.cfg" class="org.apache.ignite.configuration.IgniteConfiguration">
<property name="clientMode" value="false"/>
<property name="networkTimeout" value="150000" />
<!-- This port will be used by Apache Hadoop client to connect to Ignite node as if it was a job tracker. -->
<property name="connectorConfiguration">
<bean class="org.apache.ignite.configuration.ConnectorConfiguration">
<property name="port" value="11211"/>
</bean>
</property>
<!-- Changing total RAM size to be used by Ignite Node -->
<property name="memoryConfiguration">
<bean class="org.apache.ignite.configuration.MemoryConfiguration">
<property name="systemCacheInitialSize" value="#{50L * 1024 * 1024}"/>
<property name="systemCacheMaxSize" value="#{200L * 1024 * 1024}"/>
<property name="defaultMemoryPolicyName" value="dfltPlc"/>
<property name="memoryPolicies">
<list>
<bean class="org.apache.ignite.configuration.MemoryPolicyConfiguration">
<property name="name" value="dfltPlc"/>
<property name="initialSize" value="#{50L * 1024 * 1024}"/>
<property name="maxSize" value="#{200L * 1024 * 1024}"/>
</bean>
</list>
</property>
</bean>
</property>
<!-- Configure one IGFS file system instance named "igfs" on this node. -->
<property name="fileSystemConfiguration">
<list>
<bean class="org.apache.ignite.configuration.FileSystemConfiguration">
<!-- IGFS name you will use to access IGFS through Hadoop API. -->
<property name="name" value="igfs"/>
<!-- Configure TCP endpoint for communication with the file system instance. -->
<property name="ipcEndpointConfiguration">
<bean class="org.apache.ignite.igfs.IgfsIpcEndpointConfiguration">
<property name="type" value="TCP" />
<property name="host" value="0.0.0.0" />
<property name="port" value="10500" />
</bean>
</property>
</bean>
</list>
</property>
<!-- TCP discovery SPI can be configured with list of addresses if multicast is not available. -->
<property name="discoverySpi">
<bean class="org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi">
<property name="ipFinder">
<bean class="org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder">
<property name="addresses">
<list>
<value>127.0.0.1:47500..47509</value>
</list>
</property>
</bean>
</property>
</bean>
</property>
</bean>
</beans>
ignite-15b0cff9.log
Description: Binary data
smime.p7s
Description: S/MIME cryptographic signature
