This is an automated email from the ASF dual-hosted git repository.
vjasani pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/phoenix-adapters.git
The following commit(s) were added to refs/heads/main by this push:
new 14b6b0d Skip last evaluated key when items to return are less than
limit
14b6b0d is described below
commit 14b6b0dffaccaaa248451bcaaa9d873cae8b6dc7
Author: Palash Chauhan <[email protected]>
AuthorDate: Fri Jan 23 11:41:32 2026 -0800
Skip last evaluated key when items to return are less than limit
---
.../apache/phoenix/ddb/service/QueryService.java | 4 +-
.../apache/phoenix/ddb/service/ScanService.java | 6 +--
.../apache/phoenix/ddb/service/utils/DQLUtils.java | 7 ++-
.../test/java/org/apache/phoenix/ddb/QueryIT.java | 63 +++++++++++++++++-----
.../java/org/apache/phoenix/ddb/QueryIndex1IT.java | 18 ++-----
.../phoenix/ddb/ScanExclusiveStartKeyIT.java | 4 +-
.../java/org/apache/phoenix/ddb/ScanTableIT.java | 48 +++++++++++++++++
7 files changed, 113 insertions(+), 37 deletions(-)
diff --git
a/phoenix-ddb-rest/src/main/java/org/apache/phoenix/ddb/service/QueryService.java
b/phoenix-ddb-rest/src/main/java/org/apache/phoenix/ddb/service/QueryService.java
index 7a7c116..141a289 100644
---
a/phoenix-ddb-rest/src/main/java/org/apache/phoenix/ddb/service/QueryService.java
+++
b/phoenix-ddb-rest/src/main/java/org/apache/phoenix/ddb/service/QueryService.java
@@ -75,9 +75,11 @@ public class QueryService {
PreparedStatement stmt = pairVal.getFirst();
boolean isSingleRowExpected = pairVal.getSecond();
boolean countOnly =
ApiMetadata.SELECT_COUNT.equals(request.get(ApiMetadata.SELECT));
+ Integer limit = (Integer) request.get(ApiMetadata.LIMIT);
+ int effectiveLimit = limit == null ? MAX_QUERY_LIMIT :
Math.min(limit, MAX_QUERY_LIMIT);
return DQLUtils.executeStatementReturnResult(stmt,
getProjectionAttributes(request), useIndex, tablePKCols,
indexPKCols, tableName,
- isSingleRowExpected, false, countOnly);
+ isSingleRowExpected, false, countOnly, effectiveLimit);
} catch (SQLException e) {
throw new PhoenixServiceException(e);
}
diff --git
a/phoenix-ddb-rest/src/main/java/org/apache/phoenix/ddb/service/ScanService.java
b/phoenix-ddb-rest/src/main/java/org/apache/phoenix/ddb/service/ScanService.java
index 9a086c5..67e6ef9 100644
---
a/phoenix-ddb-rest/src/main/java/org/apache/phoenix/ddb/service/ScanService.java
+++
b/phoenix-ddb-rest/src/main/java/org/apache/phoenix/ddb/service/ScanService.java
@@ -150,7 +150,7 @@ public class ScanService {
PreparedStatement stmt = buildQuery(connection, request, config);
return DQLUtils.executeStatementReturnResult(stmt,
getProjectionAttributes(request),
config.useIndex(), config.getTablePKCols(),
config.getIndexPKCols(), config.getTableName(),
- false, false, config.isCountOnly());
+ false, false, config.isCountOnly(), config.getLimit());
}
/**
@@ -164,7 +164,7 @@ public class ScanService {
PreparedStatement firstStmt = buildQuery(connection, request, config);
Map<String, Object> firstResult =
DQLUtils.executeStatementReturnResult(firstStmt,
getProjectionAttributes(request), config.useIndex(),
config.getTablePKCols(), config.getIndexPKCols(),
- config.getTableName(), false, true, config.isCountOnly());
+ config.getTableName(), false, true, config.isCountOnly(),
config.getLimit());
List<Map<String, Object>> allItems = config.isCountOnly()
? new ArrayList<>()
@@ -180,7 +180,7 @@ public class ScanService {
PreparedStatement secondStmt = buildQuery(connection, request,
secondConfig);
Map<String, Object> secondResult =
DQLUtils.executeStatementReturnResult(secondStmt,
getProjectionAttributes(request), config.useIndex(),
config.getTablePKCols(), config.getIndexPKCols(),
- config.getTableName(), false, false, config.isCountOnly());
+ config.getTableName(), false, false, config.isCountOnly(),
secondConfig.getLimit());
if (!config.isCountOnly()) {
List<Map<String, Object>> secondItems = (List<Map<String,
Object>>) secondResult.get(ApiMetadata.ITEMS);
diff --git
a/phoenix-ddb-rest/src/main/java/org/apache/phoenix/ddb/service/utils/DQLUtils.java
b/phoenix-ddb-rest/src/main/java/org/apache/phoenix/ddb/service/utils/DQLUtils.java
index c45246a..447eba1 100644
---
a/phoenix-ddb-rest/src/main/java/org/apache/phoenix/ddb/service/utils/DQLUtils.java
+++
b/phoenix-ddb-rest/src/main/java/org/apache/phoenix/ddb/service/utils/DQLUtils.java
@@ -37,7 +37,8 @@ public class DQLUtils {
public static Map<String, Object>
executeStatementReturnResult(PreparedStatement stmt,
List<String> projectionAttributes, boolean useIndex,
List<PColumn> tablePKCols, List<PColumn> indexPKCols, String
tableName,
- boolean isSingleRowExpected, boolean isScanFirstQuery, boolean
countOnly) throws SQLException {
+ boolean isSingleRowExpected, boolean isScanFirstQuery, boolean
countOnly,
+ int effectiveLimit) throws SQLException {
int count = 0;
int bytesSize = 0;
List<Map<String, Object>> items = new ArrayList<>();
@@ -65,7 +66,9 @@ public class DQLUtils {
response.put(ApiMetadata.ITEMS, items);
}
response.put(ApiMetadata.COUNT, count);
- response.put(ApiMetadata.LAST_EVALUATED_KEY, lastKey);
+ if (count == effectiveLimit || sizeLimitReached) {
+ response.put(ApiMetadata.LAST_EVALUATED_KEY, lastKey);
+ }
response.put(ApiMetadata.SCANNED_COUNT, countRowsScanned);
response.put(ApiMetadata.CONSUMED_CAPACITY,
CommonServiceUtils.getConsumedCapacity(tableName));
diff --git a/phoenix-ddb-rest/src/test/java/org/apache/phoenix/ddb/QueryIT.java
b/phoenix-ddb-rest/src/test/java/org/apache/phoenix/ddb/QueryIT.java
index dbd0562..8ed4582 100644
--- a/phoenix-ddb-rest/src/test/java/org/apache/phoenix/ddb/QueryIT.java
+++ b/phoenix-ddb-rest/src/test/java/org/apache/phoenix/ddb/QueryIT.java
@@ -453,20 +453,8 @@ public class QueryIT {
Assert.assertTrue(phoenixResult.count() == 1);
// check last key
- lastKey = phoenixResult.lastEvaluatedKey();
- Assert.assertEquals("B", lastKey.get("attr_0").s());
- Assert.assertEquals(4, Integer.parseInt(lastKey.get("attr_1").n()));
-
- // note that dynamo's last evaluated key will be null here
- // sdkv2 returns empty item
+ Assert.assertTrue(phoenixResult.lastEvaluatedKey().isEmpty());
Assert.assertTrue(dynamoResult.lastEvaluatedKey().isEmpty());
-
- // provide lastEvaluatedKey as exclusiveStartKey, no items should be
returned
- qr.exclusiveStartKey(lastKey);
- phoenixResult = phoenixDBClientV2.query(qr.build());
- dynamoResult = dynamoDbClient.query(qr.build());
- Assert.assertEquals(dynamoResult.count(), phoenixResult.count());
- Assert.assertTrue(phoenixResult.count() == 0);
}
@Test(timeout = 120000)
@@ -827,6 +815,55 @@ public class QueryIT {
} while (!lastEvaluatedKey.isEmpty());
}
+ @Test(timeout = 120000)
+ public void queryLastEvaluatedKeyNotSetWhenItemsLessThanLimit() {
+ final String tableName = testName.getMethodName();
+ CreateTableRequest createTableRequest =
+ DDLTestUtils.getCreateTableRequest(tableName, "attr_0",
+ ScalarAttributeType.S, "attr_1",
ScalarAttributeType.N);
+ phoenixDBClientV2.createTable(createTableRequest);
+ dynamoDbClient.createTable(createTableRequest);
+
+ PutItemRequest putItemRequest1 =
PutItemRequest.builder().tableName(tableName).item(getItem1()).build();
+ PutItemRequest putItemRequest2 =
PutItemRequest.builder().tableName(tableName).item(getItem2()).build();
+ PutItemRequest putItemRequest3 =
PutItemRequest.builder().tableName(tableName).item(getItem3()).build();
+ phoenixDBClientV2.putItem(putItemRequest1);
+ phoenixDBClientV2.putItem(putItemRequest2);
+ phoenixDBClientV2.putItem(putItemRequest3);
+ dynamoDbClient.putItem(putItemRequest1);
+ dynamoDbClient.putItem(putItemRequest2);
+ dynamoDbClient.putItem(putItemRequest3);
+
+ QueryRequest.Builder qr = QueryRequest.builder().tableName(tableName);
+ qr.keyConditionExpression("#0 = :v0");
+ Map<String, String> exprAttrNames = new HashMap<>();
+ exprAttrNames.put("#0", "attr_0");
+ qr.expressionAttributeNames(exprAttrNames);
+ Map<String, AttributeValue> exprAttrVal = new HashMap<>();
+ exprAttrVal.put(":v0", AttributeValue.builder().s("B").build());
+ qr.expressionAttributeValues(exprAttrVal);
+ qr.limit(10);
+
+ QueryResponse phoenixResult = phoenixDBClientV2.query(qr.build());
+ QueryResponse dynamoResult = dynamoDbClient.query(qr.build());
+
+ Assert.assertEquals(dynamoResult.count(), phoenixResult.count());
+ Assert.assertEquals(2, phoenixResult.count().intValue());
+ Assert.assertTrue(phoenixResult.lastEvaluatedKey().isEmpty());
+ Assert.assertTrue(dynamoResult.lastEvaluatedKey().isEmpty());
+ Assert.assertEquals(dynamoResult.items(), phoenixResult.items());
+
+ qr.limit(1);
+ phoenixResult = phoenixDBClientV2.query(qr.build());
+ dynamoResult = dynamoDbClient.query(qr.build());
+
+ Assert.assertEquals(dynamoResult.count(), phoenixResult.count());
+ Assert.assertEquals(1, phoenixResult.count().intValue());
+ Assert.assertFalse(phoenixResult.lastEvaluatedKey().isEmpty());
+ Assert.assertFalse(dynamoResult.lastEvaluatedKey().isEmpty());
+ Assert.assertEquals(dynamoResult.lastEvaluatedKey(),
phoenixResult.lastEvaluatedKey());
+ }
+
public static Map<String, AttributeValue> getItem1() {
Map<String, AttributeValue> item = new HashMap<>();
item.put("attr_0", AttributeValue.builder().s("A").build());
diff --git
a/phoenix-ddb-rest/src/test/java/org/apache/phoenix/ddb/QueryIndex1IT.java
b/phoenix-ddb-rest/src/test/java/org/apache/phoenix/ddb/QueryIndex1IT.java
index 14e9888..65810a3 100644
--- a/phoenix-ddb-rest/src/test/java/org/apache/phoenix/ddb/QueryIndex1IT.java
+++ b/phoenix-ddb-rest/src/test/java/org/apache/phoenix/ddb/QueryIndex1IT.java
@@ -259,11 +259,7 @@ public class QueryIndex1IT {
Assert.assertEquals(dynamoResult.count(), phoenixResult.count());
Assert.assertEquals(dynamoResult.items().get(0),
phoenixResult.items().get(0));
Assert.assertEquals(dynamoResult.scannedCount(),
phoenixResult.scannedCount());
-
- // check last evaluated key
- Map<String, AttributeValue> lastKey = phoenixResult.lastEvaluatedKey();
- Assert.assertEquals("101.01", lastKey.get("IdS").s());
- Assert.assertEquals(1.1, Double.parseDouble(lastKey.get("Id2").n()),
0);
+ Assert.assertEquals(dynamoResult.lastEvaluatedKey(),
phoenixResult.lastEvaluatedKey());
// explain plan
TestUtils.validateIndexUsed(qr.build(), url);
@@ -315,11 +311,7 @@ public class QueryIndex1IT {
QueryResponse dynamoResult = dynamoDbClient.query(qr.build());
Assert.assertEquals(dynamoResult.count(), phoenixResult.count());
Assert.assertEquals(dynamoResult.items().get(0),
phoenixResult.items().get(0));
-
- // check last evaluated key
- Map<String, AttributeValue> lastKey = phoenixResult.lastEvaluatedKey();
- Assert.assertEquals("101.01", lastKey.get("IdS").s());
- Assert.assertEquals(1.1, Double.parseDouble(lastKey.get("Id2").n()),
0);
+ Assert.assertEquals(dynamoResult.lastEvaluatedKey(),
phoenixResult.lastEvaluatedKey());
// explain plan
TestUtils.validateIndexUsed(qr.build(), url);
@@ -371,11 +363,7 @@ public class QueryIndex1IT {
QueryResponse dynamoResult = dynamoDbClient.query(qr.build());
Assert.assertEquals(dynamoResult.count(), phoenixResult.count());
Assert.assertEquals(dynamoResult.items().get(0),
phoenixResult.items().get(0));
-
- // check last evaluated key
- Map<String, AttributeValue> lastKey = phoenixResult.lastEvaluatedKey();
- Assert.assertEquals("str_val_1", lastKey.get("attr_0").s());
- Assert.assertEquals("101.01", lastKey.get("IdS").s());
+ Assert.assertEquals(dynamoResult.lastEvaluatedKey(),
phoenixResult.lastEvaluatedKey());
// explain plan
TestUtils.validateIndexUsed(qr.build(), url);
diff --git
a/phoenix-ddb-rest/src/test/java/org/apache/phoenix/ddb/ScanExclusiveStartKeyIT.java
b/phoenix-ddb-rest/src/test/java/org/apache/phoenix/ddb/ScanExclusiveStartKeyIT.java
index 712b819..2fa34dc 100644
---
a/phoenix-ddb-rest/src/test/java/org/apache/phoenix/ddb/ScanExclusiveStartKeyIT.java
+++
b/phoenix-ddb-rest/src/test/java/org/apache/phoenix/ddb/ScanExclusiveStartKeyIT.java
@@ -262,9 +262,7 @@ public class ScanExclusiveStartKeyIT {
Assert.assertEquals("DynamoDB should return all 56 items", 56,
dynamoItems.size());
// Verify that both clients returned the same number of pagination
rounds
- // Note: The exact pagination behavior might differ slightly, but both
should complete
- Assert.assertTrue("Phoenix pagination should complete",
phoenixPaginationCount > 0);
- Assert.assertTrue("DynamoDB pagination should complete",
dynamoPaginationCount > 0);
+ Assert.assertEquals("Pagination count with same limit should be the
same", dynamoPaginationCount, phoenixPaginationCount);
// For limits smaller than 8, we should see multiple pagination rounds
if (scanLimit < 8) {
diff --git
a/phoenix-ddb-rest/src/test/java/org/apache/phoenix/ddb/ScanTableIT.java
b/phoenix-ddb-rest/src/test/java/org/apache/phoenix/ddb/ScanTableIT.java
index 18fc91e..01a9a8d 100644
--- a/phoenix-ddb-rest/src/test/java/org/apache/phoenix/ddb/ScanTableIT.java
+++ b/phoenix-ddb-rest/src/test/java/org/apache/phoenix/ddb/ScanTableIT.java
@@ -1771,4 +1771,52 @@ public class ScanTableIT {
Assert.assertEquals("Expected 400 status code for DynamoDB", 400,
e.statusCode());
}
}
+
+ @Test(timeout = 120000)
+ public void testScanLastEvaluatedKeyNotSetWhenItemsLessThanLimit() {
+ final String tableName = testName.getMethodName();
+ CreateTableRequest createTableRequest =
+ DDLTestUtils.getCreateTableRequest(tableName, "attr_0",
+ ScalarAttributeType.S, "attr_1",
ScalarAttributeType.N);
+ phoenixDBClientV2.createTable(createTableRequest);
+ dynamoDbClient.createTable(createTableRequest);
+
+ PutItemRequest putItemRequest1 =
PutItemRequest.builder().tableName(tableName).item(getItem1()).build();
+ PutItemRequest putItemRequest2 =
PutItemRequest.builder().tableName(tableName).item(getItem2()).build();
+ PutItemRequest putItemRequest3 =
PutItemRequest.builder().tableName(tableName).item(getItem3()).build();
+ phoenixDBClientV2.putItem(putItemRequest1);
+ phoenixDBClientV2.putItem(putItemRequest2);
+ phoenixDBClientV2.putItem(putItemRequest3);
+ dynamoDbClient.putItem(putItemRequest1);
+ dynamoDbClient.putItem(putItemRequest2);
+ dynamoDbClient.putItem(putItemRequest3);
+
+ ScanRequest.Builder sr = ScanRequest.builder().tableName(tableName);
+ sr.filterExpression("#0 = :v0");
+ Map<String, String> exprAttrNames = new HashMap<>();
+ exprAttrNames.put("#0", "title");
+ sr.expressionAttributeNames(exprAttrNames);
+ Map<String, AttributeValue> exprAttrVal = new HashMap<>();
+ exprAttrVal.put(":v0", AttributeValue.builder().s("Title3").build());
+ sr.expressionAttributeValues(exprAttrVal);
+ sr.limit(10);
+
+ ScanResponse phoenixResult = phoenixDBClientV2.scan(sr.build());
+ ScanResponse dynamoResult = dynamoDbClient.scan(sr.build());
+
+ Assert.assertEquals(dynamoResult.count(), phoenixResult.count());
+ Assert.assertEquals(1, phoenixResult.count().intValue());
+ Assert.assertTrue(phoenixResult.lastEvaluatedKey().isEmpty());
+ Assert.assertTrue(dynamoResult.lastEvaluatedKey().isEmpty());
+
+ ScanRequest.Builder sr2 = ScanRequest.builder().tableName(tableName);
+ sr2.limit(1);
+ phoenixResult = phoenixDBClientV2.scan(sr2.build());
+ dynamoResult = dynamoDbClient.scan(sr2.build());
+
+ Assert.assertEquals(dynamoResult.count(), phoenixResult.count());
+ Assert.assertEquals(1, phoenixResult.count().intValue());
+ Assert.assertFalse(phoenixResult.lastEvaluatedKey().isEmpty());
+ Assert.assertFalse(dynamoResult.lastEvaluatedKey().isEmpty());
+ }
}
