The branch main has been updated by dougm: URL: https://cgit.FreeBSD.org/src/commit/?id=7f2ec173e4613a57732ca162572d25b0a546689f
commit 7f2ec173e4613a57732ca162572d25b0a546689f
Author: Doug Moore <do...@freebsd.org>
AuthorDate: 2022-08-06 18:26:18 +0000
Commit: Doug Moore <do...@freebsd.org>
CommitDate: 2022-08-06 18:26:18 +0000
iommu_gas: avoid pointless augmentation
iommu_gas_augment_entry updates a map entry element. Invoked as
RB_AUGMENT in RB tree code, it is applied from the point where the
tree is modified, all the way up to the root, and is also applied when
rotation moves a node down in the tree.
There are several opportunities to invoke it less. The automatic
augmentation with every rotation is a mistake. Delaying these
augmentations until RB_INSERT_COLOR or RB_REMOVE_COLOR are finishing
allows the augmentation code to be duplicated less, to work when there
is less register pressure, and to be skipped when conditions allow it:
In the double-rotate case of RB_INSERT_COLOR, augmentation after
the first rotation is not necessary when the element being moved
down the tree becomes a leaf. It was in the tree, and was a leaf,
before the RB_INSERT operation began, and so recomputing
augmentation for it would do nothing.
In the final (possibly only) rotation of RB_REMOVE_COLOR, both the
elements - the one moving up and the one moving down - end up in
the path from the deletion point to the tree root, so there's no
need to augment either of them immediately.
In RB_REMOVE, when the right child of the removed node replaces it
in tree, it began with a null left child. Replacement creates a
non-NULL left child, and then rotation may put a NULL node back in
that place. If that happens, start the augmenting-up-to-root with
the parent of that node, since augmentation would do nothing.
Adjust to avoid these needless augmentations.
Reviewed by: alc
MFC after: 3 weeks
Differential Revision: https://reviews.freebsd.org/D35502
---
sys/sys/tree.h | 43 +++++++++++++++++++++++++++++--------------
1 file changed, 29 insertions(+), 14 deletions(-)
diff --git a/sys/sys/tree.h b/sys/sys/tree.h
index 45791e08c947..d82c341049c6 100644
--- a/sys/sys/tree.h
+++ b/sys/sys/tree.h
@@ -372,7 +372,7 @@ struct {
\
#endif
#define RB_UPDATE_AUGMENT(elm, field) do { \
- __typeof(elm) rb_update_tmp = (elm); \
+ __typeof(elm) rb_update_tmp = (elm); \
do { \
RB_AUGMENT(rb_update_tmp); \
} while ((rb_update_tmp = RB_PARENT(rb_update_tmp, field)) != NULL); \
@@ -395,7 +395,6 @@ struct {
\
RB_SWAP_CHILD(head, elm, tmp, field); \
RB_LEFT(tmp, field) = (elm); \
RB_SET_PARENT(elm, tmp, field); \
- RB_AUGMENT(elm); \
} while (/*CONSTCOND*/ 0)
#define RB_ROTATE_RIGHT(head, elm, tmp, field) do { \
@@ -406,7 +405,6 @@ struct {
\
RB_SWAP_CHILD(head, elm, tmp, field); \
RB_RIGHT(tmp, field) = (elm); \
RB_SET_PARENT(elm, tmp, field); \
- RB_AUGMENT(elm); \
} while (/*CONSTCOND*/ 0)
/* Generates prototypes and inline functions */
@@ -494,7 +492,9 @@ name##_RB_INSERT_COLOR(struct name *head, struct type *elm)
\
elm = parent; \
continue; \
} \
- if (!RB_RED_RIGHT(elm, field)) { \
+ if (RB_RED_RIGHT(elm, field)) \
+ child = elm; \
+ else { \
/* coverity[uninit_use] */ \
RB_ROTATE_LEFT(head, elm, child, field);\
if (RB_RED_RIGHT(child, field)) \
@@ -503,9 +503,11 @@ name##_RB_INSERT_COLOR(struct name *head, struct type
*elm) \
RB_FLIP_ALL(elm, field); \
else \
RB_FLIP_LEFT(elm, field); \
- elm = child; \
+ if ((RB_BITS(child, field) & \
+ RB_RED_MASK) == 0) \
+ elm = child; \
} \
- RB_ROTATE_RIGHT(head, parent, elm, field); \
+ RB_ROTATE_RIGHT(head, parent, child, field); \
} else { \
if (RB_RED_RIGHT(parent, field)) { \
RB_FLIP_RIGHT(parent, field); \
@@ -517,7 +519,9 @@ name##_RB_INSERT_COLOR(struct name *head, struct type *elm)
\
elm = parent; \
continue; \
} \
- if (!RB_RED_LEFT(elm, field)) { \
+ if (RB_RED_LEFT(elm, field)) \
+ child = elm; \
+ else { \
/* coverity[uninit_use] */ \
RB_ROTATE_RIGHT(head, elm, child, field);\
if (RB_RED_LEFT(child, field)) \
@@ -526,11 +530,16 @@ name##_RB_INSERT_COLOR(struct name *head, struct type
*elm) \
RB_FLIP_ALL(elm, field); \
else \
RB_FLIP_RIGHT(elm, field); \
- elm = child; \
+ if ((RB_BITS(child, field) & \
+ RB_RED_MASK) == 0) \
+ elm = child; \
} \
- RB_ROTATE_LEFT(head, parent, elm, field); \
+ RB_ROTATE_LEFT(head, parent, child, field); \
} \
- RB_BITS(elm, field) &= ~RB_RED_MASK; \
+ RB_BITS(child, field) &= ~RB_RED_MASK; \
+ if (elm != child) \
+ RB_AUGMENT(elm); \
+ RB_AUGMENT(parent); \
break; \
} \
}
@@ -589,21 +598,22 @@ name##_RB_REMOVE_COLOR(struct name *head,
\
else \
RB_FLIP_RIGHT(sib, field); \
RB_BITS(elm, field) |= RB_RED_MASK; \
- sib = elm; \
break; \
case RB_RED_L: \
if (RB_STRICT_HST && elm != NULL) { \
RB_FLIP_RIGHT(parent, field); \
RB_FLIP_ALL(sib, field); \
+ elm = sib; \
break; \
} \
RB_FLIP_LEFT(parent, field); \
/* FALLTHROUGH */ \
default: \
RB_FLIP_RIGHT(sib, field); \
+ elm = sib; \
break; \
} \
- RB_ROTATE_LEFT(head, parent, sib, field); \
+ RB_ROTATE_LEFT(head, parent, elm, field); \
} else { \
if (!RB_RED_RIGHT(parent, field)) { \
RB_FLIP_RIGHT(parent, field); \
@@ -632,22 +642,25 @@ name##_RB_REMOVE_COLOR(struct name *head,
\
else \
RB_FLIP_LEFT(sib, field); \
RB_BITS(elm, field) |= RB_RED_MASK; \
- sib = elm; \
break; \
case RB_RED_R: \
if (RB_STRICT_HST && elm != NULL) { \
RB_FLIP_LEFT(parent, field); \
RB_FLIP_ALL(sib, field); \
+ elm = sib; \
break; \
} \
RB_FLIP_RIGHT(parent, field); \
/* FALLTHROUGH */ \
default: \
RB_FLIP_LEFT(sib, field); \
+ elm = sib; \
break; \
} \
- RB_ROTATE_RIGHT(head, parent, sib, field); \
+ RB_ROTATE_RIGHT(head, parent, elm, field); \
} \
+ if (sib != elm) \
+ RB_AUGMENT(sib); \
break; \
} while ((parent = RB_PARENT(elm, field)) != NULL); \
}
@@ -687,6 +700,8 @@ name##_RB_REMOVE(struct name *head, struct type *elm)
\
RB_SET_PARENT(child, parent, field); \
if (parent != NULL) { \
name##_RB_REMOVE_COLOR(head, parent, child); \
+ if (parent == elm && RB_LEFT(parent, field) == NULL) \
+ parent = RB_PARENT(parent, field); \
RB_UPDATE_AUGMENT(parent, field); \
} \
return (old); \
