Author: jbellis
Date: Mon Mar 28 19:05:32 2011
New Revision: 1086343
URL: http://svn.apache.org/viewvc?rev=1086343&view=rev
Log:
fix migration race vs flush
patch by jbellis; reviewed by gdusbabek for CASSANDRA-2381
Modified:
cassandra/branches/cassandra-0.7/CHANGES.txt
cassandra/branches/cassandra-0.7/NEWS.txt
cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/ConsistencyLevel.java
cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/Memtable.java
cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/RowIteratorFactory.java
cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/Table.java
cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/migration/DropColumnFamily.java
cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/migration/DropKeyspace.java
cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/migration/Migration.java
cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/migration/UpdateColumnFamily.java
Modified: cassandra/branches/cassandra-0.7/CHANGES.txt
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.7/CHANGES.txt?rev=1086343&r1=1086342&r2=1086343&view=diff
==============================================================================
--- cassandra/branches/cassandra-0.7/CHANGES.txt (original)
+++ cassandra/branches/cassandra-0.7/CHANGES.txt Mon Mar 28 19:05:32 2011
@@ -21,6 +21,8 @@
* fsync statistics component on write (CASSANDRA-2382)
* fix incorrect truncation of long to int when reading columns via block
index (CASSANDRA-2376)
+ * fix race condition that could leave orphaned data files when
+ dropping CF or KS (CASSANDRA-2381)
0.7.4
Modified: cassandra/branches/cassandra-0.7/NEWS.txt
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.7/NEWS.txt?rev=1086343&r1=1086342&r2=1086343&view=diff
==============================================================================
--- cassandra/branches/cassandra-0.7/NEWS.txt (original)
+++ cassandra/branches/cassandra-0.7/NEWS.txt Mon Mar 28 19:05:32 2011
@@ -1,3 +1,18 @@
+0.7.5
+=====
+
+Upgrading
+---------
+ - Nothing specific to 0.7.4, but see 0.7.3 Upgrading if upgrading
+ from earlier than 0.7.1.
+
+Changes
+-------
+ - system_update_column_family no longer snapshots before applying
+ the schema change. (_update_keyspace never did. _drop_keyspace
+ and _drop_column_family continue to snapshot.)
+
+
0.7.4
=====
Modified:
cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/ConsistencyLevel.java
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/ConsistencyLevel.java?rev=1086343&r1=1086342&r2=1086343&view=diff
==============================================================================
---
cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/ConsistencyLevel.java
(original)
+++
cassandra/branches/cassandra-0.7/interface/thrift/gen-java/org/apache/cassandra/thrift/ConsistencyLevel.java
Mon Mar 28 19:05:32 2011
@@ -32,13 +32,26 @@ import java.util.HashMap;
import org.apache.thrift.TEnum;
/**
- * The ConsistencyLevel is an enum that controls both read and write behavior
based on the ReplicationFactor of the keyspace.
- * The different consistency levels have different meanings, depending on if
you're doing a write or read
- * operation. Note that if W + R > ReplicationFactor, where W is the number of
nodes to block for on write, and R
- * the number to block for on reads, you will have strongly consistent
behavior; that is, readers will always see the most
- * recent write. Of these, the most interesting is to do QUORUM reads and
writes, which gives you consistency while still
- * allowing availability in the face of node failures up to half of
<ReplicationFactor>. Of course if latency is more
- * important than consistency then you can use lower values for either or both.
+ * The ConsistencyLevel is an enum that controls both read and write
+ * behavior based on the ReplicationFactor of the keyspace. The
+ * different consistency levels have different meanings, depending on
+ * if you're doing a write or read operation.
+ *
+ * If W + R > ReplicationFactor, where W is the number of nodes to
+ * block for on write, and R the number to block for on reads, you
+ * will have strongly consistent behavior; that is, readers will
+ * always see the most recent write. Of these, the most interesting is
+ * to do QUORUM reads and writes, which gives you consistency while
+ * still allowing availability in the face of node failures up to half
+ * of <ReplicationFactor>. Of course if latency is more important than
+ * consistency then you can use lower values for either or both.
+ *
+ * Some ConsistencyLevels (ONE, TWO, THREE) refer to a specific number
+ * of replicas rather than a logical concept that adjusts
+ * automatically with the replication factor. Of these, only ONE is
+ * commonly used; TWO and (even more rarely) THREE are only useful
+ * when you care more about guaranteeing a certain level of
+ * durability, than consistency.
*
* Write consistency levels make the following guarantees before reporting
success to the client:
* ANY Ensure that the write has been written once somewhere,
including possibly being hinted in a non-target node.
Modified:
cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/ColumnFamilyStore.java?rev=1086343&r1=1086342&r2=1086343&view=diff
==============================================================================
---
cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
(original)
+++
cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
Mon Mar 28 19:05:32 2011
@@ -25,6 +25,8 @@ import java.util.*;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
import java.util.regex.Pattern;
import javax.management.MBeanServer;
import javax.management.ObjectName;
@@ -136,6 +138,9 @@ public class ColumnFamilyStore implement
private volatile DefaultInteger rowCacheSaveInSeconds;
private volatile DefaultInteger keyCacheSaveInSeconds;
+ /** Lock to allow migrations to block all flushing, so we can be sure not
to write orphaned data files */
+ public final Lock flushLock = new ReentrantLock();
+
// Locally held row/key cache scheduled tasks
private volatile ScheduledFuture<?> saveRowCacheTask;
private volatile ScheduledFuture<?> saveKeyCacheTask;
@@ -695,7 +700,7 @@ public class ColumnFamilyStore implement
* contexts (commitlog position) were read, even though the flush
executor
* is multithreaded.
*/
- Table.flusherLock.writeLock().lock();
+ Table.switchLock.writeLock().lock();
try
{
if (oldMemtable.isFrozen())
@@ -759,7 +764,7 @@ public class ColumnFamilyStore implement
}
finally
{
- Table.flusherLock.writeLock().unlock();
+ Table.switchLock.writeLock().unlock();
}
}
@@ -1084,14 +1089,14 @@ public class ColumnFamilyStore implement
*/
private Memtable getMemtableThreadSafe()
{
- Table.flusherLock.readLock().lock();
+ Table.switchLock.readLock().lock();
try
{
return memtable;
}
finally
{
- Table.flusherLock.readLock().unlock();
+ Table.switchLock.readLock().unlock();
}
}
@@ -1138,7 +1143,7 @@ public class ColumnFamilyStore implement
// TODO this actually isn't a good meature of pending tasks
public int getPendingTasks()
{
- return Table.flusherLock.getQueueLength();
+ return Table.switchLock.getQueueLength();
}
public long getWriteCount()
Modified:
cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/Memtable.java
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/Memtable.java?rev=1086343&r1=1086342&r2=1086343&view=diff
==============================================================================
---
cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/Memtable.java
(original)
+++
cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/Memtable.java
Mon Mar 28 19:05:32 2011
@@ -37,7 +37,6 @@ import com.google.common.collect.Peeking
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.cassandra.config.ConfigurationException;
import org.apache.cassandra.db.columniterator.IColumnIterator;
import org.apache.cassandra.db.columniterator.SimpleAbstractColumnIterator;
import org.apache.cassandra.db.filter.AbstractColumnIterator;
@@ -173,8 +172,16 @@ public class Memtable implements Compara
{
public void runMayThrow() throws IOException
{
- cfs.addSSTable(writeSortedContents());
- cfs.getMemtablesPendingFlush().remove(Memtable.this);
+ cfs.flushLock.lock();
+ try
+ {
+ cfs.addSSTable(writeSortedContents());
+ cfs.getMemtablesPendingFlush().remove(Memtable.this);
+ }
+ finally
+ {
+ cfs.flushLock.unlock();
+ }
latch.countDown();
}
});
Modified:
cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/RowIteratorFactory.java
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/RowIteratorFactory.java?rev=1086343&r1=1086342&r2=1086343&view=diff
==============================================================================
---
cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/RowIteratorFactory.java
(original)
+++
cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/RowIteratorFactory.java
Mon Mar 28 19:05:32 2011
@@ -169,14 +169,14 @@ public class RowIteratorFactory
*/
private static Iterator<Map.Entry<DecoratedKey, ColumnFamily>>
memtableEntryIterator(Memtable memtable, DecoratedKey startWith)
{
- Table.flusherLock.readLock().lock();
+ Table.switchLock.readLock().lock();
try
{
return memtable.getEntryIterator(startWith);
}
finally
{
- Table.flusherLock.readLock().unlock();
+ Table.switchLock.readLock().unlock();
}
}
Modified:
cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/Table.java
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/Table.java?rev=1086343&r1=1086342&r2=1086343&view=diff
==============================================================================
---
cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/Table.java
(original)
+++
cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/Table.java
Mon Mar 28 19:05:32 2011
@@ -24,7 +24,6 @@ import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.*;
import java.util.concurrent.*;
-import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import com.google.common.base.Function;
@@ -61,7 +60,7 @@ public class Table
* accesses to CFS.memtable should acquire this for thread safety.
* Table.maybeSwitchMemtable should aquire the writeLock; see that method
for the full explanation.
*/
- static final ReentrantReadWriteLock flusherLock = new
ReentrantReadWriteLock();
+ static final ReentrantReadWriteLock switchLock = new
ReentrantReadWriteLock();
// It is possible to call Table.open without a running daemon, so it makes
sense to ensure
// proper directories here as well as in CassandraDaemon.
@@ -115,11 +114,6 @@ public class Table
}
return tableInstance;
}
-
- public static Lock getFlushLock()
- {
- return flusherLock.writeLock();
- }
public static Table clear(String table) throws IOException
{
@@ -343,7 +337,7 @@ public class Table
logger.debug("applying mutation of row {}",
ByteBufferUtil.bytesToHex(mutation.key()));
// write the mutation to the commitlog and memtables
- flusherLock.readLock().lock();
+ switchLock.readLock().lock();
try
{
if (writeCommitLog)
@@ -407,7 +401,7 @@ public class Table
}
finally
{
- flusherLock.readLock().unlock();
+ switchLock.readLock().unlock();
}
// flush memtables that got filled up outside the readlock
(maybeSwitchMemtable acquires writeLock).
@@ -562,7 +556,7 @@ public class Table
DecoratedKey key = iter.next();
logger.debug("Indexing row {} ", key);
List<Memtable> memtablesToFlush = Collections.emptyList();
- flusherLock.readLock().lock();
+ switchLock.readLock().lock();
try
{
synchronized (indexLockFor(key.key))
@@ -574,7 +568,7 @@ public class Table
}
finally
{
- flusherLock.readLock().unlock();
+ switchLock.readLock().unlock();
}
// during index build, we do flush index memtables separately
from master; otherwise we could OOM
Modified:
cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/migration/DropColumnFamily.java
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/migration/DropColumnFamily.java?rev=1086343&r1=1086342&r2=1086343&view=diff
==============================================================================
---
cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/migration/DropColumnFamily.java
(original)
+++
cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/migration/DropColumnFamily.java
Mon Mar 28 19:05:32 2011
@@ -9,6 +9,7 @@ import org.apache.cassandra.config.Confi
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.config.KSMetaData;
import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.CompactionManager;
import org.apache.cassandra.db.Table;
import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.UUIDGen;
@@ -66,36 +67,31 @@ public class DropColumnFamily extends Mi
return new KSMetaData(ksm.name, ksm.strategyClass,
ksm.strategyOptions, ksm.replicationFactor, newCfs.toArray(new
CFMetaData[newCfs.size()]));
}
- @Override
- public void beforeApplyModels()
+ public void applyModels() throws IOException
{
- if (clientMode)
- return;
ColumnFamilyStore cfs =
Table.open(tableName).getColumnFamilyStore(cfName);
- cfs.snapshot(Table.getTimestampedSnapshotName(null));
- }
- @Override
- public void applyModels() throws IOException
- {
- acquireLocks();
- try
+ // reinitialize the table.
+ KSMetaData existing = DatabaseDescriptor.getTableDefinition(tableName);
+ CFMetaData cfm = existing.cfMetaData().get(cfName);
+ KSMetaData ksm = makeNewKeyspaceDefinition(existing);
+ CFMetaData.purge(cfm);
+ DatabaseDescriptor.setTableDefinition(ksm, newVersion);
+
+ if (!clientMode)
{
- // reinitialize the table.
- KSMetaData existing =
DatabaseDescriptor.getTableDefinition(tableName);
- CFMetaData cfm = existing.cfMetaData().get(cfName);
- KSMetaData ksm = makeNewKeyspaceDefinition(existing);
- CFMetaData.purge(cfm);
- DatabaseDescriptor.setTableDefinition(ksm, newVersion);
-
- if (!clientMode)
+ CompactionManager.instance.getCompactionLock().lock();
+ cfs.flushLock.lock();
+ try
{
+ cfs.snapshot(Table.getTimestampedSnapshotName(null));
Table.open(ksm.name).dropCf(cfm.cfId);
}
- }
- finally
- {
- releaseLocks();
+ finally
+ {
+ cfs.flushLock.unlock();
+ CompactionManager.instance.getCompactionLock().unlock();
+ }
}
}
Modified:
cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/migration/DropKeyspace.java
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/migration/DropKeyspace.java?rev=1086343&r1=1086342&r2=1086343&view=diff
==============================================================================
---
cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/migration/DropKeyspace.java
(original)
+++
cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/migration/DropKeyspace.java
Mon Mar 28 19:05:32 2011
@@ -24,6 +24,8 @@ import org.apache.cassandra.config.CFMet
import org.apache.cassandra.config.ConfigurationException;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.config.KSMetaData;
+import org.apache.cassandra.db.ColumnFamilyStore;
+import org.apache.cassandra.db.CompactionManager;
import org.apache.cassandra.db.HintedHandOffManager;
import org.apache.cassandra.db.Table;
import org.apache.cassandra.utils.FBUtilities;
@@ -46,35 +48,37 @@ public class DropKeyspace extends Migrat
rm = makeDefinitionMutation(null, ksm, newVersion);
}
- @Override
- public void beforeApplyModels()
- {
- if (!clientMode)
- Table.open(name).snapshot(null);
- }
-
- @Override
public void applyModels() throws IOException
{
- acquireLocks();
+ String snapshotName = Table.getTimestampedSnapshotName(null);
+ CompactionManager.instance.getCompactionLock().lock();
try
{
KSMetaData ksm = DatabaseDescriptor.getTableDefinition(name);
- // remove the table from the static instances.
- Table table = Table.clear(ksm.name);
- if (table == null)
- throw new IOException("Table is not active. " + ksm.name);
-
+
// remove all cfs from the table instance.
for (CFMetaData cfm : ksm.cfMetaData().values())
{
+ ColumnFamilyStore cfs =
Table.open(ksm.name).getColumnFamilyStore(cfm.cfName);
CFMetaData.purge(cfm);
if (!clientMode)
{
- table.dropCf(cfm.cfId);
+ cfs.flushLock.lock();
+ try
+ {
+ cfs.snapshot(snapshotName);
+ Table.open(ksm.name).dropCf(cfm.cfId);
+ }
+ finally
+ {
+ cfs.flushLock.unlock();
+ }
}
}
+ // remove the table from the static instances.
+ Table table = Table.clear(ksm.name);
+ assert table != null;
// reset defs.
DatabaseDescriptor.clearTableDefinition(ksm, newVersion);
@@ -86,7 +90,7 @@ public class DropKeyspace extends Migrat
}
finally
{
- releaseLocks();
+ CompactionManager.instance.getCompactionLock().unlock();
}
}
Modified:
cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/migration/Migration.java
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/migration/Migration.java?rev=1086343&r1=1086342&r2=1086343&view=diff
==============================================================================
---
cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/migration/Migration.java
(original)
+++
cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/migration/Migration.java
Mon Mar 28 19:05:32 2011
@@ -92,22 +92,6 @@ public abstract class Migration
this.lastVersion = lastVersion;
}
- // block compactions and flushing.
- protected final void acquireLocks()
- {
- CompactionManager.instance.getCompactionLock().lock();
- Table.getFlushLock().lock();
- }
-
- protected final void releaseLocks()
- {
- Table.getFlushLock().unlock();
- CompactionManager.instance.getCompactionLock().unlock();
- }
-
- /** override this to perform logic before writing the migration or
applying it. defaults to nothing. */
- public void beforeApplyModels() {}
-
/** apply changes */
public final void apply() throws IOException, ConfigurationException
{
@@ -119,8 +103,6 @@ public abstract class Migration
if (!clientMode)
rm.apply();
- beforeApplyModels();
-
// write migration.
if (!clientMode)
{
Modified:
cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/migration/UpdateColumnFamily.java
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/migration/UpdateColumnFamily.java?rev=1086343&r1=1086342&r2=1086343&view=diff
==============================================================================
---
cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/migration/UpdateColumnFamily.java
(original)
+++
cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/migration/UpdateColumnFamily.java
Mon Mar 28 19:05:32 2011
@@ -65,14 +65,6 @@ public class UpdateColumnFamily extends
newKsMeta.cfMetaData().get(cf_def.name.toString()).apply(cf_def);
rm = Migration.makeDefinitionMutation(newKsMeta, null, newVersion);
}
-
- public void beforeApplyModels()
- {
- if (clientMode)
- return;
- ColumnFamilyStore cfs =
Table.open(metadata.tableName).getColumnFamilyStore(metadata.cfName);
- cfs.snapshot(Table.getTimestampedSnapshotName(null));
- }
void applyModels() throws IOException
{
