This is an automated email from the ASF dual-hosted git repository.
Aias00 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hertzbeat.git
The following commit(s) were added to refs/heads/master by this push:
new 33284bb955 [fix] strip NUL bytes before tdengine history insert (#4310)
33284bb955 is described below
commit 33284bb95553719782b8084161c23c8ecdaa89ae
Author: NekoPunch <[email protected]>
AuthorDate: Sun Aug 16 22:46:55 2026 -0700
[fix] strip NUL bytes before tdengine history insert (#4310)
Co-authored-by: aias00 <[email protected]>
---
.../history/tsdb/tdengine/TdEngineDataStorage.java | 10 +++++---
.../warehouse/store/TdEngineDataStorageTest.java | 29 +++++++++++++++++++---
2 files changed, 32 insertions(+), 7 deletions(-)
diff --git
a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/tdengine/TdEngineDataStorage.java
b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/tdengine/TdEngineDataStorage.java
index 361d45dbbb..94eb19413d 100644
---
a/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/tdengine/TdEngineDataStorage.java
+++
b/hertzbeat-warehouse/src/main/java/org/apache/hertzbeat/warehouse/store/history/tsdb/tdengine/TdEngineDataStorage.java
@@ -66,6 +66,7 @@ public class TdEngineDataStorage extends
AbstractHistoryDataStorage {
private static final String CONSTANTS_URL_PREFIX = "jdbc:TAOS-RS://";
private static final Pattern SQL_SPECIAL_STRING_PATTERN =
Pattern.compile("(\\\\)|(')");
+ private static final Pattern NUL_CHAR_PATTERN = Pattern.compile("\\u0000");
private static final String INSTANCE_NULL = "''";
private static final String CONSTANTS_CREATE_DATABASE = "CREATE DATABASE
IF NOT EXISTS %s";
private static final String INSERT_TABLE_DATA_SQL = "INSERT INTO `%s`
USING `%s` TAGS (%s) VALUES %s";
@@ -346,12 +347,13 @@ public class TdEngineDataStorage extends
AbstractHistoryDataStorage {
}
private String formatStringValue(String value) {
- String formatValue =
SQL_SPECIAL_STRING_PATTERN.matcher(value).replaceAll("\\\\$0");
- // bugfix Argument list too long
- if (formatValue != null && formatValue.length() >
tableStrColumnDefineMaxLength) {
+ // snmp octet strings may carry NUL padding that breaks the insert sql
+ String formatValue = NUL_CHAR_PATTERN.matcher(value).replaceAll("");
+ // truncate the logical value before escaping so the cut cannot split
an escape sequence
+ if (formatValue.length() > tableStrColumnDefineMaxLength) {
formatValue = formatValue.substring(0,
tableStrColumnDefineMaxLength);
}
- return formatValue;
+ return
SQL_SPECIAL_STRING_PATTERN.matcher(formatValue).replaceAll("\\\\$0");
}
@Override
diff --git
a/hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/store/TdEngineDataStorageTest.java
b/hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/store/TdEngineDataStorageTest.java
index 97f94090c6..2c3327f87e 100644
---
a/hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/store/TdEngineDataStorageTest.java
+++
b/hertzbeat-warehouse/src/test/java/org/apache/hertzbeat/warehouse/store/TdEngineDataStorageTest.java
@@ -115,6 +115,25 @@ class TdEngineDataStorageTest {
assertTrue(executedSql.matches(".*VALUES\\s+\\(\\d+.*68\\.7\\)"),
"Should contain timestamp and value 68.7");
}
+ @Test
+ void testSaveDataStripsControlCharacters() throws Exception {
+ tdEngineDataStorage = new TdEngineDataStorage(tdEngineProperties);
+ setPrivateField(tdEngineDataStorage, "hikariDataSource",
mockHikariDataSource);
+ setParentPrivateField(tdEngineDataStorage, "serverAvailable", true);
+
+ // snmp octet strings can carry NUL bytes (issue #1481); tabs are
legitimate data and stay
+ CollectRep.MetricsData metricsData =
generateMockedMetricsData("Loopback\tInterface 1\u0000");
+ tdEngineDataStorage.saveData(metricsData);
+
+ ArgumentCaptor<String> sqlCaptor =
ArgumentCaptor.forClass(String.class);
+ verify(mockStatement, atLeastOnce()).execute(sqlCaptor.capture());
+ String executedSql = sqlCaptor.getValue();
+
+ assertTrue(executedSql.contains("Loopback\tInterface 1'"), "NUL
stripped, tab preserved");
+ assertTrue(executedSql.indexOf('\u0000') < 0, "no raw NUL byte may
reach the sql text");
+ assertTrue(!executedSql.contains("\\u0000"), "no textual NUL escape
may survive in the labels json");
+ }
+
@Test
void destroy() {
}
@@ -146,6 +165,10 @@ class TdEngineDataStorageTest {
}
public static CollectRep.MetricsData generateMockedMetricsData() {
+ return generateMockedMetricsData("test-%server-01");
+ }
+
+ public static CollectRep.MetricsData generateMockedMetricsData(String
instanceValue) {
CollectRep.MetricsData mockMetricsData =
Mockito.mock(CollectRep.MetricsData.class);
when(mockMetricsData.getId()).thenReturn(0L);
@@ -156,9 +179,9 @@ class TdEngineDataStorageTest {
when(mockMetricsData.getInstance()).thenReturn("test-%server-01");
CollectRep.ValueRow mockValueRow =
Mockito.mock(CollectRep.ValueRow.class);
- List<String> columnValues = List.of("test-%server-01", "68.7");
+ List<String> columnValues = List.of(instanceValue, "68.7");
when(mockValueRow.getColumnsList()).thenReturn(columnValues);
- when(mockValueRow.getColumns(0)).thenReturn("test-%server-01");
+ when(mockValueRow.getColumns(0)).thenReturn(instanceValue);
when(mockValueRow.getColumns(1)).thenReturn("68.7");
List<CollectRep.ValueRow> mockValueRowsList = List.of(mockValueRow);
when(mockMetricsData.getValues()).thenReturn(mockValueRowsList);
@@ -177,7 +200,7 @@ class TdEngineDataStorageTest {
Field instanceArrowField = new Field("instance", instanceFieldType,
null);
ArrowCell instanceCell = Mockito.mock(ArrowCell.class);
when(instanceCell.getField()).thenReturn(instanceArrowField);
- when(instanceCell.getValue()).thenReturn("test-%server-01");
+ when(instanceCell.getValue()).thenReturn(instanceValue);
when(instanceCell.getMetadataAsBoolean(MetricDataConstants.LABEL)).thenReturn(true);
when(instanceCell.getMetadataAsByte(MetricDataConstants.TYPE)).thenReturn(CommonConstants.TYPE_STRING);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
