merge from 1.2
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/52b4fc39 Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/52b4fc39 Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/52b4fc39 Branch: refs/heads/cassandra-2.0 Commit: 52b4fc393350e85f44034094f323f2f06a347968 Parents: fb55849 2267c20 Author: Jonathan Ellis <jbel...@apache.org> Authored: Mon Sep 23 13:33:38 2013 -0500 Committer: Jonathan Ellis <jbel...@apache.org> Committed: Mon Sep 23 13:33:38 2013 -0500 ---------------------------------------------------------------------- CHANGES.txt | 1 + conf/cassandra.yaml | 5 ++++ .../org/apache/cassandra/config/Config.java | 2 ++ .../cassandra/config/DatabaseDescriptor.java | 14 +++++++++ .../apache/cassandra/db/ColumnFamilyStore.java | 31 ++++++++++++++++---- .../cassandra/db/ColumnFamilyStoreMBean.java | 9 ++++++ .../cassandra/db/filter/SliceQueryFilter.java | 20 +++++++++---- .../cassandra/metrics/ColumnFamilyMetrics.java | 8 +++++ .../cassandra/service/StorageService.java | 10 +++++++ .../cassandra/service/StorageServiceMBean.java | 5 ++++ 10 files changed, 93 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/52b4fc39/CHANGES.txt ---------------------------------------------------------------------- diff --cc CHANGES.txt index 88daf35,9ed00e2..aef45ba --- a/CHANGES.txt +++ b/CHANGES.txt @@@ -7,43 -3,10 +7,44 @@@ Merged from 1.2 * Allow where clause conditions to be in parenthesis (CASSANDRA-6037) * Do not open non-ssl storage port if encryption option is all (CASSANDRA-3916) * Improve memory usage of metadata min/max column names (CASSANDRA-6077) + * Add tombstone debug threshold and histogram (CASSANDRA-6042, 6057) -1.2.10 +2.0.1 + * Fix bug that could allow reading deleted data temporarily (CASSANDRA-6025) + * Improve memory use defaults (CASSANDRA-5069) + * Make ThriftServer more easlly extensible (CASSANDRA-6058) + * Remove Hadoop dependency from ITransportFactory (CASSANDRA-6062) + * add file_cache_size_in_mb setting (CASSANDRA-5661) + * Improve error message when yaml contains invalid properties (CASSANDRA-5958) + * Improve leveled compaction's ability to find non-overlapping L0 compactions + to work on concurrently (CASSANDRA-5921) + * Notify indexer of columns shadowed by range tombstones (CASSANDRA-5614) + * Log Merkle tree stats (CASSANDRA-2698) + * Switch from crc32 to adler32 for compressed sstable checksums (CASSANDRA-5862) + * Improve offheap memcpy performance (CASSANDRA-5884) + * Use a range aware scanner for cleanup (CASSANDRA-2524) + * Cleanup doesn't need to inspect sstables that contain only local data + (CASSANDRA-5722) + * Add ability for CQL3 to list partition keys (CASSANDRA-4536) + * Improve native protocol serialization (CASSANDRA-5664) + * Upgrade Thrift to 0.9.1 (CASSANDRA-5923) + * Require superuser status for adding triggers (CASSANDRA-5963) + * Make standalone scrubber handle old and new style leveled manifest + (CASSANDRA-6005) + * Fix paxos bugs (CASSANDRA-6012, 6013, 6023) + * Fix paged ranges with multiple replicas (CASSANDRA-6004) + * Fix potential AssertionError during tracing (CASSANDRA-6041) + * Fix NPE in sstablesplit (CASSANDRA-6027) + * Migrate pre-2.0 key/value/column aliases to system.schema_columns + (CASSANDRA-6009) + * Paging filter empty rows too agressively (CASSANDRA-6040) + * Support variadic parameters for IN clauses (CASSANDRA-4210) + * cqlsh: return the result of CAS writes (CASSANDRA-5796) + * Fix validation of IN clauses with 2ndary indexes (CASSANDRA-6050) + * Support named bind variables in CQL (CASSANDRA-6033) +Merged from 1.2: + * Allow cache-keys-to-save to be set at runtime (CASSANDRA-5980) * Avoid second-guessing out-of-space state (CASSANDRA-5605) * Tuning knobs for dealing with large blobs and many CFs (CASSANDRA-5982) * (Hadoop) Fix CQLRW for thrift tables (CASSANDRA-6002) http://git-wip-us.apache.org/repos/asf/cassandra/blob/52b4fc39/conf/cassandra.yaml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/52b4fc39/src/java/org/apache/cassandra/config/Config.java ---------------------------------------------------------------------- diff --cc src/java/org/apache/cassandra/config/Config.java index 392e8c3,292161b..131c9ef --- a/src/java/org/apache/cassandra/config/Config.java +++ b/src/java/org/apache/cassandra/config/Config.java @@@ -177,8 -173,11 +177,10 @@@ public class Confi public String memtable_allocator = "SlabAllocator"; - private static boolean loadYaml = true; private static boolean outboundBindAny = false; + public volatile int tombstone_debug_threshold = 10000; + public static boolean getOutboundBindAny() { return outboundBindAny; http://git-wip-us.apache.org/repos/asf/cassandra/blob/52b4fc39/src/java/org/apache/cassandra/config/DatabaseDescriptor.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/52b4fc39/src/java/org/apache/cassandra/db/ColumnFamilyStore.java ---------------------------------------------------------------------- diff --cc src/java/org/apache/cassandra/db/ColumnFamilyStore.java index 1ff4832,a41c157..ec9570a --- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java +++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java @@@ -28,11 -28,9 +28,12 @@@ import java.util.concurrent.atomic.Atom import java.util.regex.Pattern; import javax.management.*; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Function; import com.google.common.collect.*; import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.Uninterruptibles; + import org.cliffc.high_scale_lib.NonBlockingHashMap; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@@ -48,12 -45,11 +49,8 @@@ import org.apache.cassandra.config.Sche import org.apache.cassandra.db.columniterator.OnDiskAtomIterator; import org.apache.cassandra.db.commitlog.CommitLog; import org.apache.cassandra.db.commitlog.ReplayPosition; -import org.apache.cassandra.db.compaction.AbstractCompactionStrategy; -import org.apache.cassandra.db.compaction.CompactionManager; -import org.apache.cassandra.db.compaction.LeveledCompactionStrategy; -import org.apache.cassandra.db.compaction.OperationType; +import org.apache.cassandra.db.compaction.*; - import org.apache.cassandra.db.filter.ColumnSlice; - import org.apache.cassandra.db.filter.ExtendedFilter; - import org.apache.cassandra.db.filter.IDiskAtomFilter; - import org.apache.cassandra.db.filter.QueryFilter; - import org.apache.cassandra.db.filter.SliceQueryFilter; + import org.apache.cassandra.db.filter.*; import org.apache.cassandra.db.index.SecondaryIndex; import org.apache.cassandra.db.index.SecondaryIndexManager; import org.apache.cassandra.db.marshal.AbstractType; @@@ -1296,10 -1213,18 +1292,17 @@@ public class ColumnFamilyStore implemen if (cf == null) return null; - // TODO this is necessary because when we collate supercolumns together, we don't check - // their subcolumns for relevance, so we need to do a second prune post facto here. - result = cf.isSuper() ? removeDeleted(cf, gcBefore) : removeDeletedCF(cf, gcBefore); - + result = removeDeletedCF(cf, gcBefore); } + removeDroppedColumns(result); ++ + if (filter.filter instanceof SliceQueryFilter) + { + // Log the number of tombstones scanned on single key queries + metric.tombstoneScannedHistogram.update(((SliceQueryFilter) filter.filter).lastIgnored()); + metric.liveScannedHistogram.update(((SliceQueryFilter) filter.filter).lastLive()); + } } finally { @@@ -2164,15 -1901,32 +2167,31 @@@ private void validateCompactionThresholds(int minThreshold, int maxThreshold) { - if (minThreshold > maxThreshold && maxThreshold != 0) + if (minThreshold > maxThreshold) throw new RuntimeException(String.format("The min_compaction_threshold cannot be larger than the max_compaction_threshold. " + "Min is '%d', Max is '%d'.", minThreshold, maxThreshold)); - } - public boolean isCompactionDisabled() - { - return getMinimumCompactionThreshold() <= 0 || getMaximumCompactionThreshold() <= 0; + if (maxThreshold == 0 || minThreshold == 0) + throw new RuntimeException("Disabling compaction by setting min_compaction_threshold or max_compaction_threshold to 0 " + + "is deprecated, set the compaction strategy option 'enabled' to 'false' instead or use the nodetool command 'disableautocompaction'."); } + public long getTombstonesPerLastRead() + { + return metric.tombstoneScannedHistogram.count(); + } + + public float getPercentageTombstonesPerLastRead() + { + long total = metric.tombstoneScannedHistogram.count() + metric.liveScannedHistogram.count(); + return (metric.tombstoneScannedHistogram.count() / total); + } + + public long getLiveCellsPerLastRead() + { + return metric.liveScannedHistogram.count(); + } + // End JMX get/set. public long estimateKeys() http://git-wip-us.apache.org/repos/asf/cassandra/blob/52b4fc39/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java ---------------------------------------------------------------------- diff --cc src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java index e810dc6,f937032..6105714 --- a/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java +++ b/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java @@@ -281,8 -281,20 +281,17 @@@ public interface ColumnFamilyStoreMBea */ public void setCrcCheckChance(double crcCheckChance); - /** - * Disable automatic compaction. - */ - public void disableAutoCompaction(); + public boolean isAutoCompactionDisabled(); + /** Number of tombstoned cells retreived during the last slicequery */ + public long getTombstonesPerLastRead(); + + /** Percentage of tombstoned cells retreived during the last slicequery */ + public float getPercentageTombstonesPerLastRead(); + + /** Number of live cells retreived during the last slicequery */ + public long getLiveCellsPerLastRead(); + public long estimateKeys(); /** http://git-wip-us.apache.org/repos/asf/cassandra/blob/52b4fc39/src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java ---------------------------------------------------------------------- diff --cc src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java index 11d7438,7e1ec6b..443ff8e --- a/src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java +++ b/src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java @@@ -21,17 -21,16 +21,13 @@@ import java.io.DataInput import java.io.DataOutput; import java.io.IOException; import java.nio.ByteBuffer; - import java.util.Arrays; - import java.util.ArrayList; - import java.util.Collection; - import java.util.Comparator; - import java.util.Iterator; - import java.util.List; + import java.util.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.*; -import org.apache.cassandra.db.columniterator.ISSTableColumnIterator; import org.apache.cassandra.db.columniterator.OnDiskAtomIterator; import org.apache.cassandra.db.columniterator.SSTableSliceIterator; import org.apache.cassandra.db.marshal.AbstractType; @@@ -202,48 -175,62 +198,50 @@@ public class SliceQueryFilter implement } Tracing.trace("Read {} live and {} tombstoned cells", columnCounter.live(), columnCounter.ignored()); + if (columnCounter.ignored() > DatabaseDescriptor.getTombstoneDebugThreshold()) + logger.debug("Read {} live and {} tombstoned cells", columnCounter.live(), columnCounter.ignored()); } - public int getLiveCount(ColumnFamily cf) + public int getLiveCount(ColumnFamily cf, long now) { - ColumnCounter counter = getColumnCounter(cf); - for (IColumn column : cf) - counter.count(column, cf); - return counter.live(); + return columnCounter(cf.getComparator(), now).countAll(cf).live(); } - private ColumnCounter getColumnCounter(IColumnContainer container) + public ColumnCounter columnCounter(AbstractType<?> comparator, long now) { - AbstractType<?> comparator = container.getComparator(); if (compositesToGroup < 0) - return new ColumnCounter(); + return new ColumnCounter(now); else if (compositesToGroup == 0) - return new ColumnCounter.GroupByPrefix(null, 0); + return new ColumnCounter.GroupByPrefix(now, null, 0); else - return new ColumnCounter.GroupByPrefix((CompositeType)comparator, compositesToGroup); + return new ColumnCounter.GroupByPrefix(now, (CompositeType)comparator, compositesToGroup); } - public void trim(ColumnFamily cf, int trimTo) + public void trim(ColumnFamily cf, int trimTo, long now) { - ColumnCounter counter = getColumnCounter(cf); + ColumnCounter counter = columnCounter(cf.getComparator(), now); - Collection<ByteBuffer> toRemove = null; - boolean trimRemaining = false; + Collection<Column> columns = reversed + ? cf.getReverseSortedColumns() + : cf.getSortedColumns(); - Collection<IColumn> columns = reversed - ? cf.getReverseSortedColumns() - : cf.getSortedColumns(); + DeletionInfo.InOrderTester tester = cf.deletionInfo().inOrderTester(reversed); - for (IColumn column : columns) + for (Iterator<Column> iter = columns.iterator(); iter.hasNext(); ) { - if (trimRemaining) - { - toRemove.add(column.name()); - continue; - } + Column column = iter.next(); + counter.count(column, tester); - counter.count(column, cf); if (counter.live() > trimTo) { - toRemove = new HashSet<ByteBuffer>(); - toRemove.add(column.name()); - trimRemaining = true; + iter.remove(); + while (iter.hasNext()) + { + iter.next(); + iter.remove(); + } } } - - if (toRemove != null) - { - for (ByteBuffer columnName : toRemove) - cf.remove(columnName); - } } public ByteBuffer start() http://git-wip-us.apache.org/repos/asf/cassandra/blob/52b4fc39/src/java/org/apache/cassandra/metrics/ColumnFamilyMetrics.java ---------------------------------------------------------------------- diff --cc src/java/org/apache/cassandra/metrics/ColumnFamilyMetrics.java index 0f9f87d,9bd90a9..b63bbfb --- a/src/java/org/apache/cassandra/metrics/ColumnFamilyMetrics.java +++ b/src/java/org/apache/cassandra/metrics/ColumnFamilyMetrics.java @@@ -334,7 -337,8 +340,9 @@@ public class ColumnFamilyMetric Metrics.defaultRegistry().removeMetric(factory.createMetricName("RecentBloomFilterFalseRatio")); Metrics.defaultRegistry().removeMetric(factory.createMetricName("BloomFilterDiskSpaceUsed")); Metrics.defaultRegistry().removeMetric(factory.createMetricName("KeyCacheHitRate")); + Metrics.defaultRegistry().removeMetric(factory.createMetricName("SpeculativeRetry")); + Metrics.defaultRegistry().removeMetric(factory.createMetricName("TombstoneScannedHistogram")); + Metrics.defaultRegistry().removeMetric(factory.createMetricName("LiveScannedHistogram")); } class ColumnFamilyMetricNameFactory implements MetricNameFactory http://git-wip-us.apache.org/repos/asf/cassandra/blob/52b4fc39/src/java/org/apache/cassandra/service/StorageService.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/52b4fc39/src/java/org/apache/cassandra/service/StorageServiceMBean.java ----------------------------------------------------------------------
