Package: release.debian.org
Severity: normal
User: release.debian....@packages.debian.org
Usertags: unblock
Please unblock package hyperscan. The 4.4.1 minor update in unstable
fixes a bug that might be affecting users by delivering incorrect
matches. This could especially be undesirable in contexts where
Hyperscan is used as a matching engine in security relevant
applications. Intrusion detection/prevention systems come to mind,
such as Suricata which is also in Debian and makes use of libhyperscan4.
IMHO the update is suitable for stretch because the new version only
adds two tests and fixes a bug without introducing or retiring
features that could cause further breakage. It has also already been
suggested that the update could be suitable (see #858887 [1]).
If there are any more questions, please just let us know.
unblock hyperscan/4.4.1-1
Thanks
Sascha
[1] https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=858887#20
diff -Nru hyperscan-4.4.0/CHANGELOG.md hyperscan-4.4.1/CHANGELOG.md
--- hyperscan-4.4.0/CHANGELOG.md 2017-01-20 04:16:41.000000000 +0100
+++ hyperscan-4.4.1/CHANGELOG.md 2017-03-01 03:09:48.000000000 +0100
@@ -2,6 +2,12 @@
This is a list of notable changes to Hyperscan, in reverse chronological order.
+## [4.4.1] 2017-02-28
+- Bugfixes to fix issues where stale data was being referenced in scratch
+ memory. In particular this may have resulted in hs_close_stream()
+ referencing data from other previously scanned streams. This may result in
+ incorrect matches being been reported.
+
## [4.4.0] 2017-01-20
- Introduce the "fat runtime" build. This will build several variants of the
Hyperscan scanning engine specialised for different processor feature sets,
@@ -136,7 +142,9 @@
supplied with a NULL scratch pointer if no matches are required. This is in
line with the behaviour of `hs_close_stream()`.
- Disallow bounded repeats with a very large minimum repeat but no maximum,
- i.e. {N,} for very large N.
+ i.e. {
+ N,
+} for very large N.
- Reduce compile memory usage in literal set explansion for some large cases.
## [4.0.0] 2015-10-20
diff -Nru hyperscan-4.4.0/CMakeLists.txt hyperscan-4.4.1/CMakeLists.txt
--- hyperscan-4.4.0/CMakeLists.txt 2017-01-20 04:16:41.000000000 +0100
+++ hyperscan-4.4.1/CMakeLists.txt 2017-03-01 03:09:48.000000000 +0100
@@ -3,7 +3,7 @@
set (HS_MAJOR_VERSION 4)
set (HS_MINOR_VERSION 4)
-set (HS_PATCH_VERSION 0)
+set (HS_PATCH_VERSION 1)
set (HS_VERSION ${HS_MAJOR_VERSION}.${HS_MINOR_VERSION}.${HS_PATCH_VERSION})
set(CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake)
diff -Nru hyperscan-4.4.0/debian/changelog hyperscan-4.4.1/debian/changelog
--- hyperscan-4.4.0/debian/changelog 2017-01-20 10:08:50.000000000 +0100
+++ hyperscan-4.4.1/debian/changelog 2017-03-04 17:06:09.000000000 +0100
@@ -1,7 +1,16 @@
+hyperscan (4.4.1-1) unstable; urgency=medium
+
+ * Fix overlong changelog line to please lintian
+ * Remove explicit FAT_RUNTIME cmake directive
+ * New upstream version 4.4.1
+
+ -- Robert Haist <rha...@mailbox.org> Sat, 04 Mar 2017 17:06:09 +0100
+
hyperscan (4.4.0-1) unstable; urgency=medium
[ Robert Haist ]
- * Add brazilian / portuguese translations thanks to Adriano Rafael Gomes
(Closes: #846528)
+ * Add brazilian / portuguese translations thanks to Adriano Rafael Gomes
+ (Closes: #846528)
* New upstream version 4.4.0
[ Sascha Steinbiss ]
diff -Nru hyperscan-4.4.0/debian/rules hyperscan-4.4.1/debian/rules
--- hyperscan-4.4.0/debian/rules 2017-01-20 10:08:50.000000000 +0100
+++ hyperscan-4.4.1/debian/rules 2017-03-04 17:06:09.000000000 +0100
@@ -15,8 +15,7 @@
override_dh_auto_configure:
dh_auto_configure -- \
-DBUILD_STATIC_AND_SHARED=1 \
- -DCMAKE_BUILD_TYPE=RelWithDebInfo \
- -DFAT_RUNTIME=1
+ -DCMAKE_BUILD_TYPE=RelWithDebInfo
override_dh_install:
dh_install --fail-missing
diff -Nru hyperscan-4.4.0/src/nfa/lbr.c hyperscan-4.4.1/src/nfa/lbr.c
--- hyperscan-4.4.0/src/nfa/lbr.c 2017-01-20 04:16:41.000000000 +0100
+++ hyperscan-4.4.1/src/nfa/lbr.c 2017-03-01 03:09:48.000000000 +0100
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2015-2016, Intel Corporation
+ * Copyright (c) 2015-2017, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
@@ -77,6 +77,7 @@
const struct RepeatInfo *info = getRepeatInfo(l);
repeatUnpack(stream_state, info, offset, &lstate->ctrl);
+ lstate->lastEscape = 0;
}
static really_inline
diff -Nru hyperscan-4.4.0/src/nfa/limex_compile.cpp
hyperscan-4.4.1/src/nfa/limex_compile.cpp
--- hyperscan-4.4.0/src/nfa/limex_compile.cpp 2017-01-20 04:16:41.000000000
+0100
+++ hyperscan-4.4.1/src/nfa/limex_compile.cpp 2017-03-01 03:09:48.000000000
+0100
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2015-2016, Intel Corporation
+ * Copyright (c) 2015-2017, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
@@ -1803,6 +1803,12 @@
assert(cyclic != NO_STATE);
maskSetBit(limex->repeatCyclicMask, cyclic);
}
+ /* also include tugs in repeat cyclic mask */
+ for (NFAVertex v : args.tugs) {
+ u32 v_state = args.state_ids.at(v);
+ assert(v_state != NO_STATE);
+ maskSetBit(limex->repeatCyclicMask, v_state);
+ }
}
static
diff -Nru hyperscan-4.4.0/src/nfa/limex_internal.h
hyperscan-4.4.1/src/nfa/limex_internal.h
--- hyperscan-4.4.0/src/nfa/limex_internal.h 2017-01-20 04:16:41.000000000
+0100
+++ hyperscan-4.4.1/src/nfa/limex_internal.h 2017-03-01 03:09:48.000000000
+0100
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2015-2016, Intel Corporation
+ * Copyright (c) 2015-2017, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
@@ -151,7 +151,7 @@
* followers */ \
u_##size compressMask; /**< switch off before compress */ \
u_##size exceptionMask; \
- u_##size repeatCyclicMask; \
+ u_##size repeatCyclicMask; /**< also includes tug states */ \
u_##size zombieMask; /**< zombie if in any of the set states */ \
u_##size shift[MAX_SHIFT_COUNT]; \
u32 shiftCount; /**< number of shift masks used */ \
diff -Nru hyperscan-4.4.0/src/nfa/limex_runtime_impl.h
hyperscan-4.4.1/src/nfa/limex_runtime_impl.h
--- hyperscan-4.4.0/src/nfa/limex_runtime_impl.h 2017-01-20
04:16:41.000000000 +0100
+++ hyperscan-4.4.1/src/nfa/limex_runtime_impl.h 2017-03-01
03:09:48.000000000 +0100
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2015-2016, Intel Corporation
+ * Copyright (c) 2015-2017, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
@@ -393,19 +393,16 @@
DEBUG_PRINTF("repeat %u\n", i);
const struct NFARepeatInfo *info = GET_NFA_REPEAT_INFO_FN(limex, i);
- if (!TESTBIT_STATE(s, info->cyclicState)) {
+ const ENG_STATE_T *tug_mask =
+ (const ENG_STATE_T *)((const char *)info + info->tugMaskOffset);
+ /* repeat may still be inspected if its tug state is on */
+ if (!TESTBIT_STATE(s, info->cyclicState)
+ && ISZERO_STATE(AND_STATE(s, LOAD_FROM_ENG(tug_mask)))) {
DEBUG_PRINTF("is dead\n");
continue;
}
const struct RepeatInfo *repeat = getRepeatInfo(info);
- if (repeatHasMatch(repeat, &ctrl[i], state_base + info->stateOffset,
- offset) == REPEAT_STALE) {
- DEBUG_PRINTF("is stale, clearing state\n");
- CLEARBIT_STATE(&s, info->cyclicState);
- continue;
- }
-
DEBUG_PRINTF("packing state (packedCtrlOffset=%u)\n",
info->packedCtrlOffset);
repeatPack(state_base + info->packedCtrlOffset, repeat, &ctrl[i],
@@ -448,8 +445,11 @@
for (u32 i = 0; i < limex->repeatCount; i++) {
DEBUG_PRINTF("repeat %u\n", i);
const struct NFARepeatInfo *info = GET_NFA_REPEAT_INFO_FN(limex, i);
+ const ENG_STATE_T *tug_mask =
+ (const ENG_STATE_T *)((const char *)info + info->tugMaskOffset);
- if (!TESTBIT_STATE(cyclics, info->cyclicState)) {
+ if (!TESTBIT_STATE(cyclics, info->cyclicState)
+ && ISZERO_STATE(AND_STATE(cyclics, LOAD_FROM_ENG(tug_mask)))) {
DEBUG_PRINTF("is dead\n");
continue;
}
diff -Nru hyperscan-4.4.0/src/nfa/repeat.c hyperscan-4.4.1/src/nfa/repeat.c
--- hyperscan-4.4.0/src/nfa/repeat.c 2017-01-20 04:16:41.000000000 +0100
+++ hyperscan-4.4.1/src/nfa/repeat.c 2017-03-01 03:09:48.000000000 +0100
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2015-2016, Intel Corporation
+ * Copyright (c) 2015-2017, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
@@ -177,6 +177,10 @@
u64a repeatLastTopBitmap(const union RepeatControl *ctrl) {
const struct RepeatBitmapControl *xs = &ctrl->bitmap;
+ if (!xs->bitmap) {
+ /* last top was too long ago */
+ return 0;
+ }
return xs->offset + 63 - clz64(xs->bitmap);
}
diff -Nru hyperscan-4.4.0/src/rose/program_runtime.h
hyperscan-4.4.1/src/rose/program_runtime.h
--- hyperscan-4.4.0/src/rose/program_runtime.h 2017-01-20 04:16:41.000000000
+0100
+++ hyperscan-4.4.1/src/rose/program_runtime.h 2017-03-01 03:09:48.000000000
+0100
@@ -1209,6 +1209,8 @@
const u8 *aa = getActiveLeafArray(rose, scratch->core_info.state);
const u32 aaCount = rose->activeArrayCount;
+ const u32 qCount = rose->queueCount;
+ struct fatbit *aqa = scratch->aqa;
const struct mmbit_sparse_iter *it = getByOffset(rose, iter_offset);
assert(ISALIGNED(it));
@@ -1221,6 +1223,10 @@
qi = mmbit_sparse_iter_next(aa, aaCount, qi, &idx, it, si_state)) {
DEBUG_PRINTF("checking nfa %u\n", qi);
struct mq *q = scratch->queues + qi;
+ if (!fatbit_set(aqa, qCount, qi)) {
+ initQueue(q, qi, rose, scratch);
+ }
+
assert(q->nfa == getNfaByQueue(rose, qi));
assert(nfaAcceptsEod(q->nfa));
diff -Nru hyperscan-4.4.0/src/runtime.c hyperscan-4.4.1/src/runtime.c
--- hyperscan-4.4.0/src/runtime.c 2017-01-20 04:16:41.000000000 +0100
+++ hyperscan-4.4.1/src/runtime.c 2017-03-01 03:09:48.000000000 +0100
@@ -187,6 +187,18 @@
}
static really_inline
+void pureLiteralInitScratch(struct hs_scratch *scratch, u64a offset) {
+ // Some init has already been done.
+ assert(offset == scratch->core_info.buf_offset);
+
+ scratch->tctxt.lit_offset_adjust = offset + 1;
+ scratch->tctxt.lastEndOffset = offset;
+ scratch->tctxt.delayLastEndOffset = offset;
+ scratch->tctxt.filledDelayedSlots = 0;
+ scratch->al_log_sum = 0;
+}
+
+static really_inline
void pureLiteralBlockExec(const struct RoseEngine *rose,
struct hs_scratch *scratch) {
assert(rose);
@@ -198,9 +210,8 @@
size_t length = scratch->core_info.len;
DEBUG_PRINTF("rose engine %d\n", rose->runtimeImpl);
- // RoseContext values that need to be set for use by roseCallback.
+ pureLiteralInitScratch(scratch, 0);
scratch->tctxt.groups = rose->initialGroups;
- scratch->tctxt.lit_offset_adjust = 1;
hwlmExec(ftable, buffer, length, 0, roseCallback, scratch,
rose->initialGroups);
@@ -743,9 +754,8 @@
DEBUG_PRINTF("::: streaming rose ::: offset = %llu len = %zu\n",
stream_state->offset, scratch->core_info.len);
- // RoseContext values that need to be set for use by roseCallback.
+ pureLiteralInitScratch(scratch, stream_state->offset);
scratch->tctxt.groups = loadGroups(rose, scratch->core_info.state);
- scratch->tctxt.lit_offset_adjust = scratch->core_info.buf_offset + 1;
// Pure literal cases don't have floatingMinDistance set, so we always
// start the match region at zero.
diff -Nru hyperscan-4.4.0/unit/hyperscan/behaviour.cpp
hyperscan-4.4.1/unit/hyperscan/behaviour.cpp
--- hyperscan-4.4.0/unit/hyperscan/behaviour.cpp 2017-01-20
04:16:41.000000000 +0100
+++ hyperscan-4.4.1/unit/hyperscan/behaviour.cpp 2017-03-01
03:09:48.000000000 +0100
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2015-2016, Intel Corporation
+ * Copyright (c) 2015-2017, Intel Corporation
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
@@ -1166,6 +1166,106 @@
ASSERT_EQ(HS_SUCCESS, err);
EXPECT_EQ(1U, matchCount);
+ // teardown
+ err = hs_free_scratch(scratch);
+ ASSERT_EQ(HS_SUCCESS, err);
+ hs_free_database(db);
+}
+
+TEST(HyperscanTestBehaviour, MultiStream1) {
+ hs_error_t err;
+
+ // build a database
+ hs_database_t *db = buildDB("foo.*bar.*\\b", 0, 0, HS_MODE_STREAM);
+ ASSERT_TRUE(db != nullptr);
+
+ // alloc some scratch
+ hs_scratch_t *scratch = nullptr;
+ err = hs_alloc_scratch(db, &scratch);
+ ASSERT_EQ(HS_SUCCESS, err);
+ EXPECT_TRUE(scratch != nullptr);
+
+ hs_stream_t *stream = nullptr;
+ err = hs_open_stream(db, 0, &stream);
+ ASSERT_EQ(HS_SUCCESS, err);
+ ASSERT_TRUE(stream != nullptr);
+
+ hs_stream_t *stream2 = nullptr;
+ err = hs_open_stream(db, 0, &stream2);
+ ASSERT_EQ(HS_SUCCESS, err);
+ ASSERT_TRUE(stream2 != nullptr);
+
+ matchCount = 0;
+ const string data("foo bara");
+ err = hs_scan_stream(stream, data.c_str(), data.size(), 0, scratch,
+ countHandler, nullptr);
+ ASSERT_EQ(HS_SUCCESS, err);
+ EXPECT_EQ(0U, matchCount); // hasn't matched until stream end
+
+ const string data2("foo bar ");
+ err = hs_scan_stream(stream2, data2.c_str(), data2.size(), 0, scratch,
+ nullptr, nullptr);
+ ASSERT_EQ(HS_SUCCESS, err);
+ EXPECT_EQ(0U, matchCount);
+
+ err = hs_close_stream(stream, scratch, countHandler, nullptr);
+ ASSERT_EQ(HS_SUCCESS, err);
+ EXPECT_EQ(1U, matchCount);
+
+ err = hs_close_stream(stream2, scratch, countHandler, nullptr);
+ ASSERT_EQ(HS_SUCCESS, err);
+ EXPECT_EQ(1U, matchCount);
+
+ // teardown
+ err = hs_free_scratch(scratch);
+ ASSERT_EQ(HS_SUCCESS, err);
+ hs_free_database(db);
+}
+
+TEST(HyperscanTestBehaviour, MultiStream2) {
+ hs_error_t err;
+
+ // build a database
+ hs_database_t *db = buildDB("foo.*bar.*\\b", 0, 0, HS_MODE_STREAM);
+ ASSERT_TRUE(db != nullptr);
+
+ // alloc some scratch
+ hs_scratch_t *scratch = nullptr;
+ err = hs_alloc_scratch(db, &scratch);
+ ASSERT_EQ(HS_SUCCESS, err);
+ EXPECT_TRUE(scratch != nullptr);
+
+ hs_stream_t *stream = nullptr;
+ err = hs_open_stream(db, 0, &stream);
+ ASSERT_EQ(HS_SUCCESS, err);
+ ASSERT_TRUE(stream != nullptr);
+
+ hs_stream_t *stream2 = nullptr;
+ err = hs_open_stream(db, 0, &stream2);
+ ASSERT_EQ(HS_SUCCESS, err);
+ ASSERT_TRUE(stream2 != nullptr);
+
+ matchCount = 0;
+ const string data2("foo bar ");
+ err = hs_scan_stream(stream2, data2.c_str(), data2.size(), 0, scratch,
+ nullptr, nullptr);
+ ASSERT_EQ(HS_SUCCESS, err);
+ EXPECT_EQ(0U, matchCount);
+
+ const string data("foo bara");
+ err = hs_scan_stream(stream, data.c_str(), data.size(), 0, scratch,
+ countHandler, nullptr);
+ ASSERT_EQ(HS_SUCCESS, err);
+ EXPECT_EQ(0U, matchCount); // hasn't matched until stream end
+
+ err = hs_close_stream(stream2, scratch, countHandler, nullptr);
+ ASSERT_EQ(HS_SUCCESS, err);
+ EXPECT_EQ(0U, matchCount);
+
+ err = hs_close_stream(stream, scratch, countHandler, nullptr);
+ ASSERT_EQ(HS_SUCCESS, err);
+ EXPECT_EQ(1U, matchCount);
+
// teardown
err = hs_free_scratch(scratch);
ASSERT_EQ(HS_SUCCESS, err);
