On Mon, Oct 10, 2011 at 4:03 PM, Norman Maurer <norman.mau...@googlemail.com> wrote: > Hi there, > > did you alter the snappy-java version by mistake or was this part of > the commit ?
Sorry, that was intentional but I should probably not have been lazy and made a separate commit. I just updated the version referenced in the pom so that it matches what we are using. i.e, I haven't updated the lib, I've just made it so that user of maven central will get the 'right' version. -- Sylvain > > Bye, > Norman > > > 2011/10/10 <slebre...@apache.org>: >> Author: slebresne >> Date: Mon Oct 10 13:56:27 2011 >> New Revision: 1180970 >> >> URL: http://svn.apache.org/viewvc?rev=1180970&view=rev >> Log: >> Fix places where uncompressed sstable size is used in place of the >> compressed one. >> patch by slebresne; reviewed by jbellis for CASSANDRA-3338 >> >> Modified: >> cassandra/branches/cassandra-1.0.0/CHANGES.txt >> cassandra/branches/cassandra-1.0.0/build.xml >> >> cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/ColumnFamilyStore.java >> >> cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/CompactionManager.java >> >> cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategy.java >> >> cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTable.java >> >> cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTableReader.java >> >> cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/CompressedSegmentedFile.java >> >> cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/SegmentedFile.java >> >> Modified: cassandra/branches/cassandra-1.0.0/CHANGES.txt >> URL: >> http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/CHANGES.txt?rev=1180970&r1=1180969&r2=1180970&view=diff >> ============================================================================== >> --- cassandra/branches/cassandra-1.0.0/CHANGES.txt (original) >> +++ cassandra/branches/cassandra-1.0.0/CHANGES.txt Mon Oct 10 13:56:27 2011 >> @@ -12,6 +12,8 @@ >> * run compaction and hinted handoff threads at MIN_PRIORITY (CASSANDRA-3308) >> * default hsha thrift server to cpu core count in rpc pool (CASSANDRA-3329) >> * add bin\daemon to binary tarball for Windows service (CASSANDRA-3331) >> + * Fix places where uncompressed size of sstables was use in place of the >> + compressed one (CASSANDRA-3338) >> Fixes merged from 0.8 below: >> * Fix tool .bat files when CASSANDRA_HOME contains spaces (CASSANDRA-3258) >> * Force flush of status table when removing/updating token (CASSANDRA-3243) >> >> Modified: cassandra/branches/cassandra-1.0.0/build.xml >> URL: >> http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/build.xml?rev=1180970&r1=1180969&r2=1180970&view=diff >> ============================================================================== >> --- cassandra/branches/cassandra-1.0.0/build.xml (original) >> +++ cassandra/branches/cassandra-1.0.0/build.xml Mon Oct 10 13:56:27 2011 >> @@ -350,7 +350,7 @@ url=${svn.entry.url}?pathrev=${svn.entry >> <license name="The Apache Software License, Version 2.0" >> url="http://www.apache.org/licenses/LICENSE-2.0.txt"/> >> <scm connection="${scm.connection}" >> developerConnection="${scm.developerConnection}" url="${scm.url}"/> >> <dependencyManagement> >> - <dependency groupId="org.xerial.snappy" artifactId="snappy-java" >> version="1.0.3.3"/> >> + <dependency groupId="org.xerial.snappy" artifactId="snappy-java" >> version="1.0.3"/> >> <dependency groupId="com.ning" artifactId="compress-lzf" >> version="0.8.4"/> >> <dependency groupId="com.google.guava" artifactId="guava" >> version="r08"/> >> <dependency groupId="commons-cli" artifactId="commons-cli" >> version="1.1"/> >> >> Modified: >> cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/ColumnFamilyStore.java >> URL: >> http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/ColumnFamilyStore.java?rev=1180970&r1=1180969&r2=1180970&view=diff >> ============================================================================== >> --- >> cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/ColumnFamilyStore.java >> (original) >> +++ >> cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/ColumnFamilyStore.java >> Mon Oct 10 13:56:27 2011 >> @@ -915,7 +915,7 @@ public class ColumnFamilyStore implement >> long expectedFileSize = 0; >> for (SSTableReader sstable : sstables) >> { >> - long size = sstable.length(); >> + long size = sstable.onDiskLength(); >> expectedFileSize = expectedFileSize + size; >> } >> return expectedFileSize; >> @@ -930,9 +930,9 @@ public class ColumnFamilyStore implement >> SSTableReader maxFile = null; >> for (SSTableReader sstable : sstables) >> { >> - if (sstable.length() > maxSize) >> + if (sstable.onDiskLength() > maxSize) >> { >> - maxSize = sstable.length(); >> + maxSize = sstable.onDiskLength(); >> maxFile = sstable; >> } >> } >> >> Modified: >> cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/CompactionManager.java >> URL: >> http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/CompactionManager.java?rev=1180970&r1=1180969&r2=1180970&view=diff >> ============================================================================== >> --- >> cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/CompactionManager.java >> (original) >> +++ >> cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/CompactionManager.java >> Mon Oct 10 13:56:27 2011 >> @@ -472,7 +472,7 @@ public class CompactionManager implement >> boolean isCommutative = >> cfs.metadata.getDefaultValidator().isCommutative(); >> >> // Calculate the expected compacted filesize >> - String compactionFileLocation = >> cfs.table.getDataFileLocation(sstable.length()); >> + String compactionFileLocation = >> cfs.table.getDataFileLocation(sstable.onDiskLength()); >> if (compactionFileLocation == null) >> throw new IOException("disk full"); >> int expectedBloomFilterSize = >> Math.max(DatabaseDescriptor.getIndexInterval(), >> @@ -765,8 +765,8 @@ public class CompactionManager implement >> >> String format = "Cleaned up to %s. %,d to %,d (~%d%% of >> original) bytes for %,d keys. Time: %,dms."; >> long dTime = System.currentTimeMillis() - startTime; >> - long startsize = sstable.length(); >> - long endsize = newSstable.length(); >> + long startsize = sstable.onDiskLength(); >> + long endsize = newSstable.onDiskLength(); >> double ratio = (double)endsize / (double)startsize; >> logger.info(String.format(format, writer.getFilename(), >> startsize, endsize, (int)(ratio*100), totalkeysWritten, dTime)); >> } >> >> Modified: >> cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategy.java >> URL: >> http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategy.java?rev=1180970&r1=1180969&r2=1180970&view=diff >> ============================================================================== >> --- >> cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategy.java >> (original) >> +++ >> cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/db/compaction/SizeTieredCompactionStrategy.java >> Mon Oct 10 13:56:27 2011 >> @@ -100,7 +100,7 @@ public class SizeTieredCompactionStrateg >> { >> List<Pair<SSTableReader, Long>> tableLengthPairs = new >> ArrayList<Pair<SSTableReader, Long>>(); >> for(SSTableReader table: collection) >> - tableLengthPairs.add(new Pair<SSTableReader, Long>(table, >> table.length())); >> + tableLengthPairs.add(new Pair<SSTableReader, Long>(table, >> table.onDiskLength())); >> return tableLengthPairs; >> } >> >> >> Modified: >> cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTable.java >> URL: >> http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTable.java?rev=1180970&r1=1180969&r2=1180970&view=diff >> ============================================================================== >> --- >> cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTable.java >> (original) >> +++ >> cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTable.java >> Mon Oct 10 13:56:27 2011 >> @@ -257,7 +257,7 @@ public abstract class SSTable >> long sum = 0; >> for (SSTableReader sstable : sstables) >> { >> - sum += sstable.length(); >> + sum += sstable.onDiskLength(); >> } >> return sum; >> } >> >> Modified: >> cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTableReader.java >> URL: >> http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTableReader.java?rev=1180970&r1=1180969&r2=1180970&view=diff >> ============================================================================== >> --- >> cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTableReader.java >> (original) >> +++ >> cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/sstable/SSTableReader.java >> Mon Oct 10 13:56:27 2011 >> @@ -555,7 +555,7 @@ public class SSTableReader extends SSTab >> long right = getPosition(new DecoratedKey(range.right, null), >> Operator.GT); >> if (right == -1 || Range.isWrapAround(range.left, range.right)) >> // right is past the end of the file, or it wraps >> - right = length(); >> + right = uncompressedLength(); >> if (left == right) >> // empty range >> continue; >> @@ -669,13 +669,25 @@ public class SSTableReader extends SSTab >> } >> >> /** >> - * @return The length in bytes of the data file for this SSTable. >> + * @return The length in bytes of the data for this SSTable. For >> + * compressed files, this is not the same thing as the on disk size (see >> + * onDiskLength()) >> */ >> - public long length() >> + public long uncompressedLength() >> { >> return dfile.length; >> } >> >> + /** >> + * @return The length in bytes of the on disk size for this SSTable. For >> + * compressed files, this is not the same thing as the data length (see >> + * length()) >> + */ >> + public long onDiskLength() >> + { >> + return dfile.onDiskLength; >> + } >> + >> public boolean acquireReference() >> { >> while (true) >> >> Modified: >> cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/CompressedSegmentedFile.java >> URL: >> http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/CompressedSegmentedFile.java?rev=1180970&r1=1180969&r2=1180970&view=diff >> ============================================================================== >> --- >> cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/CompressedSegmentedFile.java >> (original) >> +++ >> cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/CompressedSegmentedFile.java >> Mon Oct 10 13:56:27 2011 >> @@ -30,7 +30,7 @@ public class CompressedSegmentedFile ext >> >> public CompressedSegmentedFile(String path, CompressionMetadata metadata) >> { >> - super(path, metadata.dataLength); >> + super(path, metadata.dataLength, metadata.compressedFileLength); >> this.metadata = metadata; >> } >> >> >> Modified: >> cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/SegmentedFile.java >> URL: >> http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/SegmentedFile.java?rev=1180970&r1=1180969&r2=1180970&view=diff >> ============================================================================== >> --- >> cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/SegmentedFile.java >> (original) >> +++ >> cassandra/branches/cassandra-1.0.0/src/java/org/apache/cassandra/io/util/SegmentedFile.java >> Mon Oct 10 13:56:27 2011 >> @@ -42,13 +42,23 @@ public abstract class SegmentedFile >> public final String path; >> public final long length; >> >> + // This differs from length for compressed files (but we still need >> length for >> + // SegmentIterator because offsets in the file are relative to the >> uncompressed size) >> + public final long onDiskLength; >> + >> /** >> * Use getBuilder to get a Builder to construct a SegmentedFile. >> */ >> SegmentedFile(String path, long length) >> { >> + this(path, length, length); >> + } >> + >> + protected SegmentedFile(String path, long length, long onDiskLength) >> + { >> this.path = path; >> this.length = length; >> + this.onDiskLength = onDiskLength; >> } >> >> /** >> >> >> >
