Merge branch cassandra-3.11 into trunk
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/05d7661d Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/05d7661d Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/05d7661d Branch: refs/heads/trunk Commit: 05d7661d04b2cd47b29a767986d4b6d283419db2 Parents: 09f3c96 95cfee6 Author: Benjamin Lerer <[email protected]> Authored: Tue Apr 17 12:28:16 2018 +0200 Committer: Benjamin Lerer <[email protected]> Committed: Tue Apr 17 12:30:53 2018 +0200 ---------------------------------------------------------------------- CHANGES.txt | 6 +- .../cassandra/cql3/selection/Selection.java | 66 +++++++++++++++++--- .../cql3/statements/SelectStatement.java | 23 +------ .../cql3/validation/entities/JsonTest.java | 39 ++++++++++++ 4 files changed, 103 insertions(+), 31 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/05d7661d/CHANGES.txt ---------------------------------------------------------------------- diff --cc CHANGES.txt index 71d947a,42ea3b4..4c557b4 --- a/CHANGES.txt +++ b/CHANGES.txt @@@ -1,229 -1,7 +1,228 @@@ +4.0 + * Use standard Amazon naming for datacenter and rack in Ec2Snitch (CASSANDRA-7839) + * Fix junit failure for SSTableReaderTest (CASSANDRA-14387) + * Abstract write path for pluggable storage (CASSANDRA-14118) + * nodetool describecluster should be more informative (CASSANDRA-13853) + * Compaction performance improvements (CASSANDRA-14261) + * Refactor Pair usage to avoid boxing ints/longs (CASSANDRA-14260) + * Add options to nodetool tablestats to sort and limit output (CASSANDRA-13889) + * Rename internals to reflect CQL vocabulary (CASSANDRA-14354) + * Add support for hybrid MIN(), MAX() speculative retry policies + (CASSANDRA-14293, CASSANDRA-14338, CASSANDRA-14352) + * Fix some regressions caused by 14058 (CASSANDRA-14353) + * Abstract repair for pluggable storage (CASSANDRA-14116) + * Add meaningful toString() impls (CASSANDRA-13653) + * Add sstableloader option to accept target keyspace name (CASSANDRA-13884) + * Move processing of EchoMessage response to gossip stage (CASSANDRA-13713) + * Add coordinator write metric per CF (CASSANDRA-14232) + * Correct and clarify SSLFactory.getSslContext method and call sites (CASSANDRA-14314) + * Handle static and partition deletion properly on ThrottledUnfilteredIterator (CASSANDRA-14315) + * NodeTool clientstats should show SSL Cipher (CASSANDRA-14322) + * Add ability to specify driver name and version (CASSANDRA-14275) + * Abstract streaming for pluggable storage (CASSANDRA-14115) + * Forced incremental repairs should promote sstables if they can (CASSANDRA-14294) + * Use Murmur3 for validation compactions (CASSANDRA-14002) + * Comma at the end of the seed list is interpretated as localhost (CASSANDRA-14285) + * Refactor read executor and response resolver, abstract read repair (CASSANDRA-14058) + * Add optional startup delay to wait until peers are ready (CASSANDRA-13993) + * Add a few options to nodetool verify (CASSANDRA-14201) + * CVE-2017-5929 Security vulnerability and redefine default log rotation policy (CASSANDRA-14183) + * Use JVM default SSL validation algorithm instead of custom default (CASSANDRA-13259) + * Better document in code InetAddressAndPort usage post 7544, incorporate port into UUIDGen node (CASSANDRA-14226) + * Fix sstablemetadata date string for minLocalDeletionTime (CASSANDRA-14132) + * Make it possible to change neverPurgeTombstones during runtime (CASSANDRA-14214) + * Remove GossipDigestSynVerbHandler#doSort() (CASSANDRA-14174) + * Add nodetool clientlist (CASSANDRA-13665) + * Revert ProtocolVersion changes from CASSANDRA-7544 (CASSANDRA-14211) + * Non-disruptive seed node list reload (CASSANDRA-14190) + * Nodetool tablehistograms to print statics for all the tables (CASSANDRA-14185) + * Migrate dtests to use pytest and python3 (CASSANDRA-14134) + * Allow storage port to be configurable per node (CASSANDRA-7544) + * Make sub-range selection for non-frozen collections return null instead of empty (CASSANDRA-14182) + * BloomFilter serialization format should not change byte ordering (CASSANDRA-9067) + * Remove unused on-heap BloomFilter implementation (CASSANDRA-14152) + * Delete temp test files on exit (CASSANDRA-14153) + * Make PartitionUpdate and Mutation immutable (CASSANDRA-13867) + * Fix CommitLogReplayer exception for CDC data (CASSANDRA-14066) + * Fix cassandra-stress startup failure (CASSANDRA-14106) + * Remove initialDirectories from CFS (CASSANDRA-13928) + * Fix trivial log format error (CASSANDRA-14015) + * Allow sstabledump to do a json object per partition (CASSANDRA-13848) + * Add option to optimise merkle tree comparison across replicas (CASSANDRA-3200) + * Remove unused and deprecated methods from AbstractCompactionStrategy (CASSANDRA-14081) + * Fix Distribution.average in cassandra-stress (CASSANDRA-14090) + * Support a means of logging all queries as they were invoked (CASSANDRA-13983) + * Presize collections (CASSANDRA-13760) + * Add GroupCommitLogService (CASSANDRA-13530) + * Parallelize initial materialized view build (CASSANDRA-12245) + * Fix flaky SecondaryIndexManagerTest.assert[Not]MarkedAsBuilt (CASSANDRA-13965) + * Make LWTs send resultset metadata on every request (CASSANDRA-13992) + * Fix flaky indexWithFailedInitializationIsNotQueryableAfterPartialRebuild (CASSANDRA-13963) + * Introduce leaf-only iterator (CASSANDRA-9988) + * Upgrade Guava to 23.3 and Airline to 0.8 (CASSANDRA-13997) + * Allow only one concurrent call to StatusLogger (CASSANDRA-12182) + * Refactoring to specialised functional interfaces (CASSANDRA-13982) + * Speculative retry should allow more friendly params (CASSANDRA-13876) + * Throw exception if we send/receive repair messages to incompatible nodes (CASSANDRA-13944) + * Replace usages of MessageDigest with Guava's Hasher (CASSANDRA-13291) + * Add nodetool cmd to print hinted handoff window (CASSANDRA-13728) + * Fix some alerts raised by static analysis (CASSANDRA-13799) + * Checksum sstable metadata (CASSANDRA-13321, CASSANDRA-13593) + * Add result set metadata to prepared statement MD5 hash calculation (CASSANDRA-10786) + * Refactor GcCompactionTest to avoid boxing (CASSANDRA-13941) + * Expose recent histograms in JmxHistograms (CASSANDRA-13642) + * Fix buffer length comparison when decompressing in netty-based streaming (CASSANDRA-13899) + * Properly close StreamCompressionInputStream to release any ByteBuf (CASSANDRA-13906) + * Add SERIAL and LOCAL_SERIAL support for cassandra-stress (CASSANDRA-13925) + * LCS needlessly checks for L0 STCS candidates multiple times (CASSANDRA-12961) + * Correctly close netty channels when a stream session ends (CASSANDRA-13905) + * Update lz4 to 1.4.0 (CASSANDRA-13741) + * Optimize Paxos prepare and propose stage for local requests (CASSANDRA-13862) + * Throttle base partitions during MV repair streaming to prevent OOM (CASSANDRA-13299) + * Use compaction threshold for STCS in L0 (CASSANDRA-13861) + * Fix problem with min_compress_ratio: 1 and disallow ratio < 1 (CASSANDRA-13703) + * Add extra information to SASI timeout exception (CASSANDRA-13677) + * Add incremental repair support for --hosts, --force, and subrange repair (CASSANDRA-13818) + * Rework CompactionStrategyManager.getScanners synchronization (CASSANDRA-13786) + * Add additional unit tests for batch behavior, TTLs, Timestamps (CASSANDRA-13846) + * Add keyspace and table name in schema validation exception (CASSANDRA-13845) + * Emit metrics whenever we hit tombstone failures and warn thresholds (CASSANDRA-13771) + * Make netty EventLoopGroups daemon threads (CASSANDRA-13837) + * Race condition when closing stream sessions (CASSANDRA-13852) + * NettyFactoryTest is failing in trunk on macOS (CASSANDRA-13831) + * Allow changing log levels via nodetool for related classes (CASSANDRA-12696) + * Add stress profile yaml with LWT (CASSANDRA-7960) + * Reduce memory copies and object creations when acting on ByteBufs (CASSANDRA-13789) + * Simplify mx4j configuration (Cassandra-13578) + * Fix trigger example on 4.0 (CASSANDRA-13796) + * Force minumum timeout value (CASSANDRA-9375) + * Use netty for streaming (CASSANDRA-12229) + * Use netty for internode messaging (CASSANDRA-8457) + * Add bytes repaired/unrepaired to nodetool tablestats (CASSANDRA-13774) + * Don't delete incremental repair sessions if they still have sstables (CASSANDRA-13758) + * Fix pending repair manager index out of bounds check (CASSANDRA-13769) + * Don't use RangeFetchMapCalculator when RF=1 (CASSANDRA-13576) + * Don't optimise trivial ranges in RangeFetchMapCalculator (CASSANDRA-13664) + * Use an ExecutorService for repair commands instead of new Thread(..).start() (CASSANDRA-13594) + * Fix race / ref leak in anticompaction (CASSANDRA-13688) + * Expose tasks queue length via JMX (CASSANDRA-12758) + * Fix race / ref leak in PendingRepairManager (CASSANDRA-13751) + * Enable ppc64le runtime as unsupported architecture (CASSANDRA-13615) + * Improve sstablemetadata output (CASSANDRA-11483) + * Support for migrating legacy users to roles has been dropped (CASSANDRA-13371) + * Introduce error metrics for repair (CASSANDRA-13387) + * Refactoring to primitive functional interfaces in AuthCache (CASSANDRA-13732) + * Update metrics to 3.1.5 (CASSANDRA-13648) + * batch_size_warn_threshold_in_kb can now be set at runtime (CASSANDRA-13699) + * Avoid always rebuilding secondary indexes at startup (CASSANDRA-13725) + * Upgrade JMH from 1.13 to 1.19 (CASSANDRA-13727) + * Upgrade SLF4J from 1.7.7 to 1.7.25 (CASSANDRA-12996) + * Default for start_native_transport now true if not set in config (CASSANDRA-13656) + * Don't add localhost to the graph when calculating where to stream from (CASSANDRA-13583) + * Make CDC availability more deterministic via hard-linking (CASSANDRA-12148) + * Allow skipping equality-restricted clustering columns in ORDER BY clause (CASSANDRA-10271) + * Use common nowInSec for validation compactions (CASSANDRA-13671) + * Improve handling of IR prepare failures (CASSANDRA-13672) + * Send IR coordinator messages synchronously (CASSANDRA-13673) + * Flush system.repair table before IR finalize promise (CASSANDRA-13660) + * Fix column filter creation for wildcard queries (CASSANDRA-13650) + * Add 'nodetool getbatchlogreplaythrottle' and 'nodetool setbatchlogreplaythrottle' (CASSANDRA-13614) + * fix race condition in PendingRepairManager (CASSANDRA-13659) + * Allow noop incremental repair state transitions (CASSANDRA-13658) + * Run repair with down replicas (CASSANDRA-10446) + * Added started & completed repair metrics (CASSANDRA-13598) + * Added started & completed repair metrics (CASSANDRA-13598) + * Improve secondary index (re)build failure and concurrency handling (CASSANDRA-10130) + * Improve calculation of available disk space for compaction (CASSANDRA-13068) + * Change the accessibility of RowCacheSerializer for third party row cache plugins (CASSANDRA-13579) + * Allow sub-range repairs for a preview of repaired data (CASSANDRA-13570) + * NPE in IR cleanup when columnfamily has no sstables (CASSANDRA-13585) + * Fix Randomness of stress values (CASSANDRA-12744) + * Allow selecting Map values and Set elements (CASSANDRA-7396) + * Fast and garbage-free Streaming Histogram (CASSANDRA-13444) + * Update repairTime for keyspaces on completion (CASSANDRA-13539) + * Add configurable upper bound for validation executor threads (CASSANDRA-13521) + * Bring back maxHintTTL propery (CASSANDRA-12982) + * Add testing guidelines (CASSANDRA-13497) + * Add more repair metrics (CASSANDRA-13531) + * RangeStreamer should be smarter when picking endpoints for streaming (CASSANDRA-4650) + * Avoid rewrapping an exception thrown for cache load functions (CASSANDRA-13367) + * Log time elapsed for each incremental repair phase (CASSANDRA-13498) + * Add multiple table operation support to cassandra-stress (CASSANDRA-8780) + * Fix incorrect cqlsh results when selecting same columns multiple times (CASSANDRA-13262) + * Fix WriteResponseHandlerTest is sensitive to test execution order (CASSANDRA-13421) + * Improve incremental repair logging (CASSANDRA-13468) + * Start compaction when incremental repair finishes (CASSANDRA-13454) + * Add repair streaming preview (CASSANDRA-13257) + * Cleanup isIncremental/repairedAt usage (CASSANDRA-13430) + * Change protocol to allow sending key space independent of query string (CASSANDRA-10145) + * Make gc_log and gc_warn settable at runtime (CASSANDRA-12661) + * Take number of files in L0 in account when estimating remaining compaction tasks (CASSANDRA-13354) + * Skip building views during base table streams on range movements (CASSANDRA-13065) + * Improve error messages for +/- operations on maps and tuples (CASSANDRA-13197) + * Remove deprecated repair JMX APIs (CASSANDRA-11530) + * Fix version check to enable streaming keep-alive (CASSANDRA-12929) + * Make it possible to monitor an ideal consistency level separate from actual consistency level (CASSANDRA-13289) + * Outbound TCP connections ignore internode authenticator (CASSANDRA-13324) + * Upgrade junit from 4.6 to 4.12 (CASSANDRA-13360) + * Cleanup ParentRepairSession after repairs (CASSANDRA-13359) + * Upgrade snappy-java to 1.1.2.6 (CASSANDRA-13336) + * Incremental repair not streaming correct sstables (CASSANDRA-13328) + * Upgrade the jna version to 4.3.0 (CASSANDRA-13300) + * Add the currentTimestamp, currentDate, currentTime and currentTimeUUID functions (CASSANDRA-13132) + * Remove config option index_interval (CASSANDRA-10671) + * Reduce lock contention for collection types and serializers (CASSANDRA-13271) + * Make it possible to override MessagingService.Verb ids (CASSANDRA-13283) + * Avoid synchronized on prepareForRepair in ActiveRepairService (CASSANDRA-9292) + * Adds the ability to use uncompressed chunks in compressed files (CASSANDRA-10520) + * Don't flush sstables when streaming for incremental repair (CASSANDRA-13226) + * Remove unused method (CASSANDRA-13227) + * Fix minor bugs related to #9143 (CASSANDRA-13217) + * Output warning if user increases RF (CASSANDRA-13079) + * Remove pre-3.0 streaming compatibility code for 4.0 (CASSANDRA-13081) + * Add support for + and - operations on dates (CASSANDRA-11936) + * Fix consistency of incrementally repaired data (CASSANDRA-9143) + * Increase commitlog version (CASSANDRA-13161) + * Make TableMetadata immutable, optimize Schema (CASSANDRA-9425) + * Refactor ColumnCondition (CASSANDRA-12981) + * Parallelize streaming of different keyspaces (CASSANDRA-4663) + * Improved compactions metrics (CASSANDRA-13015) + * Speed-up start-up sequence by avoiding un-needed flushes (CASSANDRA-13031) + * Use Caffeine (W-TinyLFU) for on-heap caches (CASSANDRA-10855) + * Thrift removal (CASSANDRA-11115) + * Remove pre-3.0 compatibility code for 4.0 (CASSANDRA-12716) + * Add column definition kind to dropped columns in schema (CASSANDRA-12705) + * Add (automate) Nodetool Documentation (CASSANDRA-12672) + * Update bundled cqlsh python driver to 3.7.0 (CASSANDRA-12736) + * Reject invalid replication settings when creating or altering a keyspace (CASSANDRA-12681) + * Clean up the SSTableReader#getScanner API wrt removal of RateLimiter (CASSANDRA-12422) + * Use new token allocation for non bootstrap case as well (CASSANDRA-13080) + * Avoid byte-array copy when key cache is disabled (CASSANDRA-13084) + * Require forceful decommission if number of nodes is less than replication factor (CASSANDRA-12510) + * Allow IN restrictions on column families with collections (CASSANDRA-12654) + * Log message size in trace message in OutboundTcpConnection (CASSANDRA-13028) + * Add timeUnit Days for cassandra-stress (CASSANDRA-13029) + * Add mutation size and batch metrics (CASSANDRA-12649) + * Add method to get size of endpoints to TokenMetadata (CASSANDRA-12999) + * Expose time spent waiting in thread pool queue (CASSANDRA-8398) + * Conditionally update index built status to avoid unnecessary flushes (CASSANDRA-12969) + * cqlsh auto completion: refactor definition of compaction strategy options (CASSANDRA-12946) + * Add support for arithmetic operators (CASSANDRA-11935) + * Add histogram for delay to deliver hints (CASSANDRA-13234) + * Fix cqlsh automatic protocol downgrade regression (CASSANDRA-13307) + * Changing `max_hint_window_in_ms` at runtime (CASSANDRA-11720) + * Trivial format error in StorageProxy (CASSANDRA-13551) + * Nodetool repair can hang forever if we lose the notification for the repair completing/failing (CASSANDRA-13480) + * Anticompaction can cause noisy log messages (CASSANDRA-13684) + * Switch to client init for sstabledump (CASSANDRA-13683) + * CQLSH: Don't pause when capturing data (CASSANDRA-13743) + * nodetool clearsnapshot requires --all to clear all snapshots (CASSANDRA-13391) + * Correctly count range tombstones in traces and tombstone thresholds (CASSANDRA-8527) + * cqlshrc.sample uses incorrect option for time formatting (CASSANDRA-14243) + + 3.11.3 - * Fix cqlsh to read connection.ssl cqlshrc option again (CASSANDRA-14299) * Downgrade log level to trace for CommitLogSegmentManager (CASSANDRA-14370) -- * CQL fromJson(null) throws NullPointerException (CASSANDRA-13891) * Serialize empty buffer as empty string for json output format (CASSANDRA-14245) * Allow logging implementation to be interchanged for embedded testing (CASSANDRA-13396) * SASI tokenizer for simple delimiter based entries (CASSANDRA-14247) @@@ -246,8 -24,12 +245,11 @@@ Merged from 3.0 * Chain commit log marker potential performance regression in batch commit mode (CASSANDRA-14194) * Fully utilise specified compaction threads (CASSANDRA-14210) * Pre-create deletion log records to finish compactions quicker (CASSANDRA-12763) - Merged from 2.1: + Merged from 2.2: + * Fix JSON queries with IN restrictions and ORDER BY clause (CASSANDRA-14286) - * Backport circleci yaml (CASSANDRA-14240) ++ * CQL fromJson(null) throws NullPointerException (CASSANDRA-13891) + Merged from 2.1: * Check checksum before decompressing data (CASSANDRA-14284) - * CVE-2017-5929 Security vulnerability in Logback warning in NEWS.txt (CASSANDRA-14183) 3.11.2 http://git-wip-us.apache.org/repos/asf/cassandra/blob/05d7661d/src/java/org/apache/cassandra/cql3/selection/Selection.java ---------------------------------------------------------------------- diff --cc src/java/org/apache/cassandra/cql3/selection/Selection.java index 38cde1e,fb0b60c..1849b22 --- a/src/java/org/apache/cassandra/cql3/selection/Selection.java +++ b/src/java/org/apache/cassandra/cql3/selection/Selection.java @@@ -40,33 -46,35 +40,38 @@@ public abstract class Selectio /** * A predicate that returns <code>true</code> for static columns. */ - private static final Predicate<ColumnDefinition> STATIC_COLUMN_FILTER = new Predicate<ColumnDefinition>() - { - public boolean apply(ColumnDefinition def) - { - return def.isStatic(); - } - }; + private static final Predicate<ColumnMetadata> STATIC_COLUMN_FILTER = (column) -> column.isStatic(); - private final CFMetaData cfm; - private final List<ColumnDefinition> columns; + private final TableMetadata table; + private final List<ColumnMetadata> columns; private final SelectionColumnMapping columnMapping; - private final ResultSet.ResultMetadata metadata; - private final boolean collectTimestamps; - private final boolean collectTTLs; - // Columns used to order the result set for multi-partition queries - private Map<ColumnDefinition, Integer> orderingIndex; - - protected Selection(CFMetaData cfm, - List<ColumnDefinition> columns, + protected final ResultSet.ResultMetadata metadata; + protected final ColumnFilterFactory columnFilterFactory; + protected final boolean isJson; + ++ // Columns used to order the result set for JSON queries with post ordering. ++ protected final List<ColumnMetadata> orderingColumns; ++ + protected Selection(TableMetadata table, + List<ColumnMetadata> selectedColumns, + Set<ColumnMetadata> orderingColumns, SelectionColumnMapping columnMapping, - boolean collectTimestamps, - boolean collectTTLs) + ColumnFilterFactory columnFilterFactory, + boolean isJson) { - this.cfm = cfm; - this.columns = columns; + this.table = table; + this.columns = selectedColumns; this.columnMapping = columnMapping; this.metadata = new ResultSet.ResultMetadata(columnMapping.getColumnSpecifications()); - this.collectTimestamps = collectTimestamps; - this.collectTTLs = collectTTLs; + this.columnFilterFactory = columnFilterFactory; + this.isJson = isJson; + + // If we order post-query, the sorted column needs to be in the ResultSet for sorting, + // even if we don't ultimately ship them to the client (CASSANDRA-4911). + this.columns.addAll(orderingColumns); + this.metadata.addNonSerializedColumns(orderingColumns); ++ ++ this.orderingColumns = orderingColumns.isEmpty() ? Collections.emptyList() : new ArrayList<>(orderingColumns); } // Overriden by SimpleSelection when appropriate. @@@ -90,39 -98,117 +95,59 @@@ return !Iterables.isEmpty(Iterables.filter(columns, STATIC_COLUMN_FILTER)); } + /** - * Checks if this selection contains only static columns. - * @return <code>true</code> if this selection contains only static columns, <code>false</code> otherwise; ++ * Returns the corresponding column index used for post query ordering ++ * @param c ordering column ++ * @return + */ - public boolean containsOnlyStaticColumns() - { - if (!containsStaticColumns()) - return false; - - if (isWildcard()) - return false; - - for (ColumnDefinition def : getColumns()) - { - if (!def.isPartitionKey() && !def.isStatic()) - return false; - } - - return true; - } - - /** - * Checks if this selection contains a complex column. - * - * @return <code>true</code> if this selection contains a multicell collection or UDT, <code>false</code> otherwise. - */ - public boolean containsAComplexColumn() - { - for (ColumnDefinition def : getColumns()) - if (def.isComplex()) - return true; - - return false; - } - - public Map<ColumnDefinition, Integer> getOrderingIndex(boolean isJson) ++ public Integer getOrderingIndex(ColumnMetadata c) + { + if (!isJson) - return orderingIndex; ++ return getResultSetIndex(c); + + // If we order post-query in json, the first and only column that we ship to the client is the json column. + // In that case, we should keep ordering columns around to perform the ordering, then these columns will + // be placed after the json column. As a consequence of where the colums are placed, we should give the + // ordering index a value based on their position in the json encoding and discard the original index. + // (CASSANDRA-14286) - int columnIndex = 1; - Map<ColumnDefinition, Integer> jsonOrderingIndex = new LinkedHashMap<>(orderingIndex.size()); - for (ColumnDefinition column : orderingIndex.keySet()) - jsonOrderingIndex.put(column, columnIndex++); - - return jsonOrderingIndex; ++ return orderingColumns.indexOf(c) + 1; + } + - public ResultSet.ResultMetadata getResultMetadata(boolean isJson) + public ResultSet.ResultMetadata getResultMetadata() { if (!isJson) return metadata; ColumnSpecification firstColumn = metadata.names.get(0); ColumnSpecification jsonSpec = new ColumnSpecification(firstColumn.ksName, firstColumn.cfName, Json.JSON_COLUMN_ID, UTF8Type.instance); - return new ResultSet.ResultMetadata(Arrays.asList(jsonSpec)); + ResultSet.ResultMetadata resultMetadata = new ResultSet.ResultMetadata(Lists.newArrayList(jsonSpec)); - if (orderingIndex != null) - { - for (ColumnDefinition orderingColumn : orderingIndex.keySet()) - resultMetadata.addNonSerializedColumn(orderingColumn); - } ++ resultMetadata.addNonSerializedColumns(orderingColumns); + return resultMetadata; } - public static Selection wildcard(CFMetaData cfm) + public static Selection wildcard(TableMetadata table, boolean isJson) { - List<ColumnDefinition> all = new ArrayList<>(cfm.allColumns().size()); - Iterators.addAll(all, cfm.allColumnsInSelectOrder()); - return new SimpleSelection(cfm, all, true); + List<ColumnMetadata> all = new ArrayList<>(table.columns().size()); + Iterators.addAll(all, table.allColumnsInSelectOrder()); + return new SimpleSelection(table, all, Collections.emptySet(), true, isJson); } - public static Selection wildcardWithGroupBy(CFMetaData cfm, VariableSpecifications boundNames) + public static Selection wildcardWithGroupBy(TableMetadata table, + VariableSpecifications boundNames, + boolean isJson) { - List<RawSelector> rawSelectors = new ArrayList<>(cfm.allColumns().size()); - Iterator<ColumnDefinition> iter = cfm.allColumnsInSelectOrder(); - while (iter.hasNext()) - { - ColumnDefinition.Raw raw = ColumnDefinition.Raw.forColumn(iter.next()); - rawSelectors.add(new RawSelector(raw, null)); - } - return fromSelectors(cfm, rawSelectors, boundNames, true); - } - - public static Selection forColumns(CFMetaData cfm, List<ColumnDefinition> columns) - { - return new SimpleSelection(cfm, columns, false); - } - - public void addColumnForOrdering(ColumnDefinition c) - { - if (orderingIndex == null) - orderingIndex = new LinkedHashMap<>(); - - int index = getResultSetIndex(c); - - if (index < 0) - index = addOrderingColumn(c); - - orderingIndex.put(c, index); + return fromSelectors(table, + Lists.newArrayList(table.allColumnsInSelectOrder()), + boundNames, + Collections.emptySet(), + Collections.emptySet(), + true, + isJson); } - protected int addOrderingColumn(ColumnDefinition c) + public static Selection forColumns(TableMetadata table, List<ColumnMetadata> columns) { - columns.add(c); - metadata.addNonSerializedColumn(c); - return columns.size() - 1; + return new SimpleSelection(table, columns, Collections.emptySet(), false, false); } public void addFunctionsTo(List<Function> functions) @@@ -139,62 -225,17 +164,67 @@@ return false; } - public static Selection fromSelectors(CFMetaData cfm, List<RawSelector> rawSelectors, VariableSpecifications boundNames, boolean hasGroupBy) + public static Selection fromSelectors(TableMetadata table, + List<Selectable> selectables, + VariableSpecifications boundNames, + Set<ColumnMetadata> orderingColumns, + Set<ColumnMetadata> nonPKRestrictedColumns, + boolean hasGroupBy, + boolean isJson) { - List<ColumnDefinition> defs = new ArrayList<>(); + List<ColumnMetadata> selectedColumns = new ArrayList<>(); SelectorFactories factories = - SelectorFactories.createFactoriesAndCollectColumnDefinitions(RawSelector.toSelectables(rawSelectors, cfm), null, cfm, defs, boundNames); - SelectionColumnMapping mapping = collectColumnMappings(cfm, rawSelectors, factories); + SelectorFactories.createFactoriesAndCollectColumnDefinitions(selectables, null, table, selectedColumns, boundNames); + SelectionColumnMapping mapping = collectColumnMappings(table, factories); + + Set<ColumnMetadata> filteredOrderingColumns = filterOrderingColumns(orderingColumns, + selectedColumns, - factories); ++ factories, ++ isJson); + + return (processesSelection(selectables) || selectables.size() != selectedColumns.size() || hasGroupBy) + ? new SelectionWithProcessing(table, + selectedColumns, + filteredOrderingColumns, + nonPKRestrictedColumns, + mapping, + factories, + isJson) + : new SimpleSelection(table, + selectedColumns, + filteredOrderingColumns, + nonPKRestrictedColumns, + mapping, + isJson); + } + + /** + * Removes the ordering columns that are already selected. + * + * @param orderingColumns the columns used to order the results + * @param selectedColumns the selected columns + * @param factories the factory used to create the selectors + * @return the ordering columns that are not part of the selection + */ + private static Set<ColumnMetadata> filterOrderingColumns(Set<ColumnMetadata> orderingColumns, + List<ColumnMetadata> selectedColumns, - SelectorFactories factories) ++ SelectorFactories factories, ++ boolean isJson) + { ++ // CASSANDRA-14286 ++ if (isJson) ++ return orderingColumns; + Set<ColumnMetadata> filteredOrderingColumns = new LinkedHashSet<>(orderingColumns.size()); + for (ColumnMetadata orderingColumn : orderingColumns) + { + int index = selectedColumns.indexOf(orderingColumn); + if (index >= 0 && factories.indexOfSimpleSelectorFactory(index) >= 0) + continue; - return (processesSelection(rawSelectors) || rawSelectors.size() != defs.size() || hasGroupBy) - ? new SelectionWithProcessing(cfm, defs, mapping, factories) - : new SimpleSelection(cfm, defs, mapping, false); + filteredOrderingColumns.add(orderingColumn); + } + return filteredOrderingColumns; } /** @@@ -259,20 -318,22 +289,34 @@@ .toString(); } - private static List<ByteBuffer> rowToJson(List<ByteBuffer> row, ProtocolVersion protocolVersion, ResultSet.ResultMetadata metadata) - public static List<ByteBuffer> rowToJson(List<ByteBuffer> row, ProtocolVersion protocolVersion, ResultSet.ResultMetadata metadata) ++ private static List<ByteBuffer> rowToJson(List<ByteBuffer> row, ++ ProtocolVersion protocolVersion, ++ ResultSet.ResultMetadata metadata, ++ List<ColumnMetadata> orderingColumns) { ++ ByteBuffer[] jsonRow = new ByteBuffer[orderingColumns.size() + 1]; StringBuilder sb = new StringBuilder("{"); - for (int i = 0; i < metadata.getColumnCount(); i++) + for (int i = 0; i < metadata.names.size(); i++) { ++ ColumnSpecification spec = metadata.names.get(i); ++ ByteBuffer buffer = row.get(i); ++ ++ // If it is an ordering column we need to keep it in case we need it for post ordering ++ int index = orderingColumns.indexOf(spec); ++ if (index >= 0) ++ jsonRow[index + 1] = buffer; ++ ++ // If the column is only used for ordering we can stop here. ++ if (i >= metadata.getColumnCount()) ++ continue; ++ if (i > 0) sb.append(", "); -- ColumnSpecification spec = metadata.names.get(i); String columnName = spec.name.toString(); if (!columnName.equals(columnName.toLowerCase(Locale.US))) columnName = "\"" + columnName + "\""; -- ByteBuffer buffer = row.get(i); sb.append('"'); sb.append(Json.quoteAsJsonString(columnName)); sb.append("\": "); @@@ -284,40 -345,171 +328,42 @@@ sb.append(spec.type.toJSONString(buffer, protocolVersion)); } sb.append("}"); - return Collections.singletonList(UTF8Type.instance.getSerializer().serialize(sb.toString())); - List<ByteBuffer> jsonRow = new ArrayList<>(); - jsonRow.add(UTF8Type.instance.getSerializer().serialize(sb.toString())); - return jsonRow; ++ ++ jsonRow[0] = UTF8Type.instance.getSerializer().serialize(sb.toString()); ++ return Arrays.asList(jsonRow); } - public class ResultSetBuilder + public static interface Selectors { - private final ResultSet resultSet; - private final ProtocolVersion protocolVersion; - /** - * As multiple thread can access a <code>Selection</code> instance each <code>ResultSetBuilder</code> will use - * its own <code>Selectors</code> instance. + * Returns the {@code ColumnFilter} corresponding to those selectors + * @return the {@code ColumnFilter} corresponding to those selectors */ - private final Selectors selectors; + public ColumnFilter getColumnFilter(); /** - * The <code>GroupMaker</code> used to build the aggregates. + * Checks if one of the selectors perform some aggregations. + * @return {@code true} if one of the selectors perform some aggregations, {@code false} otherwise. */ - private final GroupMaker groupMaker; - - /* - * We'll build CQL3 row one by one. - * The currentRow is the values for the (CQL3) columns we've fetched. - * We also collect timestamps and ttls for the case where the writetime and - * ttl functions are used. Note that we might collect timestamp and/or ttls - * we don't care about, but since the array below are allocated just once, - * it doesn't matter performance wise. - */ - List<ByteBuffer> current; - final long[] timestamps; - final int[] ttls; - - private final boolean isJson; - - private ResultSetBuilder(QueryOptions options, boolean isJson) - { - this(options, isJson, null); - } - - private ResultSetBuilder(QueryOptions options, boolean isJson, GroupMaker groupMaker) - { - this.resultSet = new ResultSet(getResultMetadata(isJson).copy(), new ArrayList<List<ByteBuffer>>()); - this.protocolVersion = options.getProtocolVersion(); - this.selectors = newSelectors(options); - this.groupMaker = groupMaker; - this.timestamps = collectTimestamps ? new long[columns.size()] : null; - this.ttls = collectTTLs ? new int[columns.size()] : null; - this.isJson = isJson; - - // We use MIN_VALUE to indicate no timestamp and -1 for no ttl - if (timestamps != null) - Arrays.fill(timestamps, Long.MIN_VALUE); - if (ttls != null) - Arrays.fill(ttls, -1); - } - - public void add(ByteBuffer v) - { - current.add(v); - } - - public void add(Cell c, int nowInSec) - { - if (c == null) - { - current.add(null); - return; - } - - current.add(value(c)); - - if (timestamps != null) - timestamps[current.size() - 1] = c.timestamp(); - - if (ttls != null) - ttls[current.size() - 1] = remainingTTL(c, nowInSec); - } - - private int remainingTTL(Cell c, int nowInSec) - { - if (!c.isExpiring()) - return -1; - - int remaining = c.localDeletionTime() - nowInSec; - return remaining >= 0 ? remaining : -1; - } - - private ByteBuffer value(Cell c) - { - return c.isCounterCell() - ? ByteBufferUtil.bytes(CounterContext.instance().total(c.value())) - : c.value(); - } + public boolean isAggregate(); /** - * Notifies this <code>Builder</code> that a new row is being processed. - * - * @param partitionKey the partition key of the new row - * @param clustering the clustering of the new row + * Returns the number of fetched columns + * @return the number of fetched columns */ - public void newRow(DecoratedKey partitionKey, Clustering clustering) - { - // The groupMaker needs to be called for each row - boolean isNewAggregate = groupMaker == null || groupMaker.isNewGroup(partitionKey, clustering); - if (current != null) - { - selectors.addInputRow(protocolVersion, this); - if (isNewAggregate) - { - resultSet.addRow(getOutputRow()); - selectors.reset(); - } - } - current = new ArrayList<>(columns.size()); - - // Timestamps and TTLs are arrays per row, we must null them out between rows - if (timestamps != null) - Arrays.fill(timestamps, Long.MIN_VALUE); - if (ttls != null) - Arrays.fill(ttls, -1); - } + public int numberOfFetchedColumns(); /** - * Builds the <code>ResultSet</code> + * Checks if one of the selectors collect TTLs. + * @return {@code true} if one of the selectors collect TTLs, {@code false} otherwise. */ - public ResultSet build() - { - if (current != null) - { - selectors.addInputRow(protocolVersion, this); - resultSet.addRow(getOutputRow()); - selectors.reset(); - current = null; - } - - // For aggregates we need to return a row even it no records have been found - if (resultSet.isEmpty() && groupMaker != null && groupMaker.returnAtLeastOneRow()) - resultSet.addRow(getOutputRow()); - return resultSet; - } + public boolean collectTTLs(); - private List<ByteBuffer> getOutputRow() - { - List<ByteBuffer> outputRow = selectors.getOutputRow(protocolVersion); - if (isJson) - { - // Keep all columns around for possible post-query ordering. (CASSANDRA-14286) - List<ByteBuffer> jsonRow = rowToJson(outputRow, protocolVersion, metadata); - - // Keep ordering columns around for possible post-query ordering. (CASSANDRA-14286) - if (orderingIndex != null) - { - for (Integer orderingColumnIndex : orderingIndex.values()) - jsonRow.add(outputRow.get(orderingColumnIndex)); - } - outputRow = jsonRow; - } - return outputRow; - } - } - - private static interface Selectors - { - public boolean isAggregate(); + /** + * Checks if one of the selectors collect timestamps. + * @return {@code true} if one of the selectors collect timestamps, {@code false} otherwise. + */ + public boolean collectTimestamps(); /** * Adds the current row of the specified <code>ResultSetBuilder</code>. @@@ -408,12 -571,12 +454,14 @@@ current = null; } - public List<ByteBuffer> getOutputRow(ProtocolVersion protocolVersion) + public List<ByteBuffer> getOutputRow() { - return isJson ? rowToJson(current, options.getProtocolVersion(), metadata) : current; ++ if (isJson) ++ return rowToJson(current, options.getProtocolVersion(), metadata, orderingColumns); + return current; } - public void addInputRow(ProtocolVersion protocolVersion, ResultSetBuilder rs) throws InvalidRequestException + public void addInputRow(ResultSetBuilder rs) throws InvalidRequestException { current = rs.current; } @@@ -522,39 -663,15 +570,39 @@@ List<ByteBuffer> outputRow = new ArrayList<>(selectors.size()); for (Selector selector: selectors) - outputRow.add(selector.getOutput(protocolVersion)); + outputRow.add(selector.getOutput(options.getProtocolVersion())); - return isJson ? rowToJson(outputRow, options.getProtocolVersion(), metadata) : outputRow; - return outputRow; ++ return isJson ? rowToJson(outputRow, options.getProtocolVersion(), metadata, orderingColumns) : outputRow; } - public void addInputRow(ProtocolVersion protocolVersion, ResultSetBuilder rs) throws InvalidRequestException + public void addInputRow(ResultSetBuilder rs) throws InvalidRequestException { for (Selector selector : selectors) - selector.addInput(protocolVersion, rs); + selector.addInput(options.getProtocolVersion(), rs); + } + + @Override + public int numberOfFetchedColumns() + { + return getColumns().size(); + } + + @Override + public boolean collectTTLs() + { + return collectTTLs; + } + + @Override + public boolean collectTimestamps() + { + return collectTimestamps; + } + + @Override + public ColumnFilter getColumnFilter() + { + return columnFilterFactory.newInstance(selectors); } }; } http://git-wip-us.apache.org/repos/asf/cassandra/blob/05d7661d/src/java/org/apache/cassandra/cql3/statements/SelectStatement.java ---------------------------------------------------------------------- diff --cc src/java/org/apache/cassandra/cql3/statements/SelectStatement.java index 8272eb5,348fa52..84fa1d7 --- a/src/java/org/apache/cassandra/cql3/statements/SelectStatement.java +++ b/src/java/org/apache/cassandra/cql3/statements/SelectStatement.java @@@ -18,10 -18,17 +18,9 @@@ package org.apache.cassandra.cql3.statements; import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.Comparator; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.NavigableSet; -import java.util.SortedSet; +import java.util.*; import com.google.common.base.MoreObjects; - import com.google.common.collect.Maps; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@@ -979,8 -982,8 +978,8 @@@ public class SelectStatement implement { assert !forView; verifyOrderingIsAllowed(restrictions); - orderingComparator = getOrderingComparator(table, selection, restrictions, orderingColumns); - orderingComparator = getOrderingComparator(cfm, selection, restrictions, parameters.isJson); - isReversed = isReversed(cfm); ++ orderingComparator = getOrderingComparator(selection, restrictions, orderingColumns); + isReversed = isReversed(table, orderingColumns, restrictions); if (isReversed) orderingComparator = Collections.reverseOrder(orderingComparator); } @@@ -1178,52 -1142,54 +1177,36 @@@ checkFalse(clusteringPrefixSize > 0 && isDistinct, "Grouping on clustering columns is not allowed for SELECT DISTINCT queries"); - return AggregationSpecification.aggregatePkPrefix(cfm.comparator, clusteringPrefixSize); + return AggregationSpecification.aggregatePkPrefix(metadata.comparator, clusteringPrefixSize); } - private Comparator<List<ByteBuffer>> getOrderingComparator(TableMetadata metadata, - private Comparator<List<ByteBuffer>> getOrderingComparator(CFMetaData cfm, -- Selection selection, ++ private Comparator<List<ByteBuffer>> getOrderingComparator(Selection selection, StatementRestrictions restrictions, - boolean isJson) + Map<ColumnMetadata, Boolean> orderingColumns) throws InvalidRequestException { if (!restrictions.keyIsInRelation()) return null; - Map<ColumnIdentifier, Integer> orderingIndexes = getOrderingIndex(metadata, selection, orderingColumns); - - Map<ColumnDefinition, Integer> orderingIndexes = getOrderingIndex(cfm, selection, isJson); + List<Integer> idToSort = new ArrayList<Integer>(orderingColumns.size()); + List<Comparator<ByteBuffer>> sorters = new ArrayList<Comparator<ByteBuffer>>(orderingColumns.size()); - List<Integer> idToSort = new ArrayList<Integer>(); - List<Comparator<ByteBuffer>> sorters = new ArrayList<Comparator<ByteBuffer>>(); - - for (ColumnDefinition.Raw raw : parameters.orderings.keySet()) + for (ColumnMetadata orderingColumn : orderingColumns.keySet()) { - idToSort.add(orderingIndexes.get(orderingColumn.name)); - ColumnDefinition orderingColumn = raw.prepare(cfm); - idToSort.add(orderingIndexes.get(orderingColumn)); ++ idToSort.add(selection.getOrderingIndex(orderingColumn)); sorters.add(orderingColumn.type); } return idToSort.size() == 1 ? new SingleColumnComparator(idToSort.get(0), sorters.get(0)) : new CompositeComparator(sorters, idToSort); } - private Map<ColumnIdentifier, Integer> getOrderingIndex(TableMetadata table, - Selection selection, - Map<ColumnMetadata, Boolean> orderingColumns) - { - Map<ColumnIdentifier, Integer> orderingIndexes = Maps.newHashMapWithExpectedSize(orderingColumns.size()); - for (ColumnMetadata def : orderingColumns.keySet()) - { - int index = selection.getResultSetIndex(def); - orderingIndexes.put(def.name, index); - } - return orderingIndexes; - } - - private Map<ColumnDefinition, Integer> getOrderingIndex(CFMetaData cfm, Selection selection, boolean isJson) - throws InvalidRequestException + private boolean isReversed(TableMetadata table, Map<ColumnMetadata, Boolean> orderingColumns, StatementRestrictions restrictions) throws InvalidRequestException { - // If we order post-query (see orderResults), the sorted column needs to be in the ResultSet for sorting, - // even if we don't - // ultimately ship them to the client (CASSANDRA-4911). - for (ColumnDefinition.Raw raw : parameters.orderings.keySet()) - { - final ColumnDefinition def = raw.prepare(cfm); - selection.addColumnForOrdering(def); - } - return selection.getOrderingIndex(isJson); - } - - private boolean isReversed(CFMetaData cfm) throws InvalidRequestException - { - Boolean[] reversedMap = new Boolean[cfm.clusteringColumns().size()]; + Boolean[] reversedMap = new Boolean[table.clusteringColumns().size()]; int i = 0; - for (Map.Entry<ColumnDefinition.Raw, Boolean> entry : parameters.orderings.entrySet()) + for (Map.Entry<ColumnMetadata, Boolean> entry : orderingColumns.entrySet()) { - ColumnDefinition def = entry.getKey().prepare(cfm); + ColumnMetadata def = entry.getKey(); boolean reversed = entry.getValue(); checkTrue(def.isClusteringColumn(), http://git-wip-us.apache.org/repos/asf/cassandra/blob/05d7661d/test/unit/org/apache/cassandra/cql3/validation/entities/JsonTest.java ---------------------------------------------------------------------- diff --cc test/unit/org/apache/cassandra/cql3/validation/entities/JsonTest.java index e696899,16df6e1..ec42f5d --- a/test/unit/org/apache/cassandra/cql3/validation/entities/JsonTest.java +++ b/test/unit/org/apache/cassandra/cql3/validation/entities/JsonTest.java @@@ -1377,4 -1369,43 +1377,43 @@@ public class JsonTest extends CQLTeste row("{\"id\": 2, \"name\": \"\"}"), row("{\"id\": 3, \"name\": null}")); } + + // CASSANDRA-14286 + @Test + public void testJsonOrdering() throws Throwable + { - createTable("CREATE TABLE %s( PRIMARY KEY (a, b), a INT, b INT);"); ++ createTable("CREATE TABLE %s(a INT, b INT, PRIMARY KEY (a, b))"); + execute("INSERT INTO %s(a, b) VALUES (20, 30);"); + execute("INSERT INTO %s(a, b) VALUES (100, 200);"); + + assertRows(execute("SELECT JSON a, b FROM %s WHERE a IN (20, 100) ORDER BY b"), + row("{\"a\": 20, \"b\": 30}"), + row("{\"a\": 100, \"b\": 200}")); + + assertRows(execute("SELECT JSON a, b FROM %s WHERE a IN (20, 100) ORDER BY b DESC"), + row("{\"a\": 100, \"b\": 200}"), + row("{\"a\": 20, \"b\": 30}")); + + assertRows(execute("SELECT JSON a FROM %s WHERE a IN (20, 100) ORDER BY b DESC"), + row("{\"a\": 100}"), + row("{\"a\": 20}")); + - // Check ordering with alias ++ // Check ordering with alias + assertRows(execute("SELECT JSON a, b as c FROM %s WHERE a IN (20, 100) ORDER BY b"), + row("{\"a\": 20, \"c\": 30}"), + row("{\"a\": 100, \"c\": 200}")); + + assertRows(execute("SELECT JSON a, b as c FROM %s WHERE a IN (20, 100) ORDER BY b DESC"), + row("{\"a\": 100, \"c\": 200}"), + row("{\"a\": 20, \"c\": 30}")); + - // Check ordering with CAST ++ // Check ordering with CAST + assertRows(execute("SELECT JSON a, CAST(b AS FLOAT) FROM %s WHERE a IN (20, 100) ORDER BY b"), + row("{\"a\": 20, \"cast(b as float)\": 30.0}"), + row("{\"a\": 100, \"cast(b as float)\": 200.0}")); + + assertRows(execute("SELECT JSON a, CAST(b AS FLOAT) FROM %s WHERE a IN (20, 100) ORDER BY b DESC"), + row("{\"a\": 100, \"cast(b as float)\": 200.0}"), + row("{\"a\": 20, \"cast(b as float)\": 30.0}")); + } } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
