The branch main has been updated by sjg:
URL:
https://cgit.FreeBSD.org/src/commit/?id=bf7aa99a55a7692da6e55864098fd085392542b0
commit bf7aa99a55a7692da6e55864098fd085392542b0
Author: Simon J. Gerraty <s...@freebsd.org>
AuthorDate: 2023-04-17 01:40:53 +0000
Commit: Simon J. Gerraty <s...@freebsd.org>
CommitDate: 2023-04-17 01:40:53 +0000
Update meta mode makefiles
meta2deps - add checks to detect truncated/corrupted filemon data
(only known to happen on Linux hosts), to ensure we do not auto
update dependencies based on incomplete data.
meta.stage.mk adds STAGE_SHLIB_LINKS_FILTER and STAGE_LINK_AS_*
We also allow for hosts where egrep is deprecated for grep -E
Reviewed by: stevek
---
share/mk/dirdeps-options.mk | 9 +++++++-
share/mk/gendirdeps.mk | 8 ++++---
share/mk/meta.autodep.mk | 5 ++---
share/mk/meta.stage.mk | 21 ++++++++++--------
share/mk/meta2deps.py | 52 ++++++++++++++++++++++++++++++++++++++-----
share/mk/meta2deps.sh | 54 ++++++++++++++++++++++++++++++++++++++-------
6 files changed, 119 insertions(+), 30 deletions(-)
diff --git a/share/mk/dirdeps-options.mk b/share/mk/dirdeps-options.mk
index b31d2033ae98..9a97615bbeb8 100644
--- a/share/mk/dirdeps-options.mk
+++ b/share/mk/dirdeps-options.mk
@@ -1,4 +1,4 @@
-# $Id: dirdeps-options.mk,v 1.20 2022/03/17 20:11:36 sjg Exp $
+# $Id: dirdeps-options.mk,v 1.21 2022/09/06 22:18:45 sjg Exp $
#
# @(#) Copyright (c) 2018-2022, Simon J. Gerraty
#
@@ -42,6 +42,13 @@
# so we qualify MK_FOO with .${TARGET_SPEC} and each component
# TARGET_SPEC_VAR (in reverse order) before using MK_FOO.
#
+# Because Makefile.depend.options are processed at both level 0 (when
+# computing DIRDEPS to build) and higher (when updating
+# Makefile.depend* after successful build), it is important that
+# all references to TARGET_SPEC_VARs should use the syntax
+# ${DEP_${TARGET_SPEC_VAR}:U${TARGET_SPEC_VAR}} to ensure correct
+# behavior.
+#
# This should have been set by Makefile.depend.options
# before including us
diff --git a/share/mk/gendirdeps.mk b/share/mk/gendirdeps.mk
index 6d26b3d308b1..1ff2036237ed 100644
--- a/share/mk/gendirdeps.mk
+++ b/share/mk/gendirdeps.mk
@@ -1,5 +1,4 @@
-# $FreeBSD$
-# $Id: gendirdeps.mk,v 1.46 2020/08/19 17:51:53 sjg Exp $
+# $Id: gendirdeps.mk,v 1.48 2022/09/09 17:44:29 sjg Exp $
# Copyright (c) 2011-2020, Simon J. Gerraty
# Copyright (c) 2010-2018, Juniper Networks, Inc.
@@ -89,7 +88,7 @@ META_FILES := ${META_FILES:T:O:u}
# they should all be absolute paths
SKIP_GENDIRDEPS ?=
.if !empty(SKIP_GENDIRDEPS)
-_skip_gendirdeps = egrep -v '^(${SKIP_GENDIRDEPS:O:u:ts|})' |
+_skip_gendirdeps = ${EGREP:Uegrep} -v '^(${SKIP_GENDIRDEPS:O:u:ts|})' |
.else
_skip_gendirdeps =
.endif
@@ -340,6 +339,8 @@ CAT_DEPEND ?= .depend
.PHONY: ${_DEPENDFILE}
.endif
+# set this to 'no' and we will not capture any
+# local depends
LOCAL_DEPENDS_GUARD ?= _{.MAKE.LEVEL} > 0
# 'cat .depend' should suffice, but if we are mixing build modes
@@ -352,6 +353,7 @@ ${_DEPENDFILE}: .NOMETA ${CAT_DEPEND:M.depend}
${META_FILES:O:u:@m@${exists($m):
echo '${DIRDEPS:@d@ $d \\${.newline}@}'; echo; \
${_include_src_dirdeps} \
echo '.include <dirdeps.mk>'; \
+ [ "${LOCAL_DEPENDS_GUARD:[1]:tl}" != no ] || exit 0; \
echo; \
echo '.if ${LOCAL_DEPENDS_GUARD}'; \
echo '# local dependencies - needed for -jN in clean tree'; \
diff --git a/share/mk/meta.autodep.mk b/share/mk/meta.autodep.mk
index 842554beea45..cd08ac3b3520 100644
--- a/share/mk/meta.autodep.mk
+++ b/share/mk/meta.autodep.mk
@@ -1,5 +1,4 @@
-# $FreeBSD$
-# $Id: meta.autodep.mk,v 1.55 2021/12/13 08:12:01 sjg Exp $
+# $Id: meta.autodep.mk,v 1.56 2022/09/09 17:44:29 sjg Exp $
#
# @(#) Copyright (c) 2010, Simon J. Gerraty
@@ -175,7 +174,7 @@ DEPEND_SUFFIXES += .c .h .cpp .hpp .cxx .hxx .cc .hh
.endif
.depend: .NOMETA $${.MAKE.META.CREATED} ${_this}
@echo "Updating $@: ${.OODATE:T:[1..8]}"
- @egrep -i '^R .*\.(${DEPEND_SUFFIXES:tl:O:u:S,^.,,:ts|})$$' /dev/null
${.MAKE.META.FILES:T:O:u:${META_FILE_FILTER:ts:}:M*o.meta} | \
+ @${EGREP:Uegrep} -i '^R .*\.(${DEPEND_SUFFIXES:tl:O:u:S,^.,,:ts|})$$'
/dev/null ${.MAKE.META.FILES:T:O:u:${META_FILE_FILTER:ts:}:M*o.meta} | \
sed -e 's, \./, ,${OBJDIR_REFS:O:u:@d@;s, $d/, ,@};/\//d' \
-e 's,^\([^/][^/]*\).meta...[0-9]* ,\1: ,' | \
sort -u | \
diff --git a/share/mk/meta.stage.mk b/share/mk/meta.stage.mk
index 9f54f3b1f9c5..168e46d22a82 100644
--- a/share/mk/meta.stage.mk
+++ b/share/mk/meta.stage.mk
@@ -1,5 +1,4 @@
-# $FreeBSD$
-# $Id: meta.stage.mk,v 1.60 2020/08/19 17:51:53 sjg Exp $
+# $Id: meta.stage.mk,v 1.67 2023/04/17 01:22:10 sjg Exp $
#
# @(#) Copyright (c) 2011-2017, Simon J. Gerraty
#
@@ -31,8 +30,11 @@ _dirdep ?= ${RELDIR}
CLEANFILES+= .dirdep
# this allows us to trace dependencies back to their src dir
-.dirdep: .NOPATH
+.dirdep: .NOPATH
+.if !commands(.dirdep)
+.dirdep:
@echo '${_dirdep}' > $@
+.endif
.if defined(NO_POSIX_SHELL) || ${type printf:L:sh:Mbuiltin} == ""
_stage_file_basename = `basename $$f`
@@ -64,7 +66,7 @@ GENDIRDEPS_FILTER += Nnot-empty-is-important \
LN_CP_SCRIPT = LnCp() { \
rm -f $$2 2> /dev/null; \
{ [ -z "$$mode" ] && ${LN:Uln} $$1 $$2 2> /dev/null; } || \
- cp -p $$1 $$2; }
+ cp -p $$1 $$2 2> /dev/null || cp $$1 $$2; }
# a staging conflict should cause an error
# a warning is handy when bootstapping different options.
@@ -171,7 +173,7 @@ stage_libs: .dirdep
.if !defined(NO_SHLIB_LINKS)
.if !empty(SHLIB_LINKS)
@${STAGE_LINKS_SCRIPT}; StageLinks -s
${STAGE_LIBDIR:${STAGE_DIR_FILTER}} \
- ${SHLIB_LINKS:@t@${STAGE_LIBS:T:M$t.*} $t@}
+ ${SHLIB_LINKS:@t@${STAGE_LIBS:T:M$t.*:${STAGE_SHLIB_LINKS_FILTER:U}}
$t@}
.elif !empty(SHLIB_LINK) && !empty(SHLIB_NAME)
@${STAGE_LINKS_SCRIPT}; StageLinks -s
${STAGE_LIBDIR:${STAGE_DIR_FILTER}} ${SHLIB_NAME} ${SHLIB_LINK}
.endif
@@ -262,7 +264,8 @@ CLEANFILES += ${STAGE_AS_SETS:@s@stage*$s@}
# sometimes things need to be renamed as they are staged
# each ${file} will be staged as ${STAGE_AS_${file:T}}
# one could achieve the same with SYMLINKS
-# stage_as_and_symlink makes the original name a symlink to the new name
+# stage_as_and_symlink makes the original name (or ${STAGE_LINK_AS_${name}})
+# a symlink to the new name
# it is the same as using stage_as and stage_symlinks but ensures
# both operations happen together
.for s in ${STAGE_AS_SETS:O:u}
@@ -292,7 +295,7 @@ STAGE_AS_AND_SYMLINK.$s ?= ${.ALLSRC:N.dirdep:Nstage_*}
stage_as_and_symlink: stage_as_and_symlink.$s
stage_as_and_symlink.$s: .dirdep
@${STAGE_AS_SCRIPT}; StageAs ${FLAGS.$@}
${STAGE_FILES_DIR.$s:U${STAGE_DIR.$s}:${STAGE_DIR_FILTER}}
${STAGE_AS_AND_SYMLINK.$s:O:@f@$f
${STAGE_AS_${f:tA}:U${STAGE_AS_${f:T}:U${f:T}}}@}
- @${STAGE_LINKS_SCRIPT}; StageLinks -s
${STAGE_FILES_DIR.$s:U${STAGE_DIR.$s}:${STAGE_DIR_FILTER}}
${STAGE_AS_AND_SYMLINK.$s:O:@f@${STAGE_AS_${f:tA}:U${STAGE_AS_${f:T}:U${f:T}}}
$f@}
+ @${STAGE_LINKS_SCRIPT}; StageLinks -s
${STAGE_FILES_DIR.$s:U${STAGE_DIR.$s}:${STAGE_DIR_FILTER}}
${STAGE_AS_AND_SYMLINK.$s:O:@f@${STAGE_AS_${f:tA}:U${STAGE_AS_${f:T}:U${f:T}}}
${STAGE_LINK_AS_${f}:U$f}@}
@touch $@
.endif
.endif
@@ -304,7 +307,7 @@ CLEANFILES += ${STAGE_TARGETS} stage_incs stage_includes
# this lot also only makes sense the first time...
.if !target(__${.PARSEFILE}__)
-__${.PARSEFILE}__:
+__${.PARSEFILE}__: .NOTMAIN
# stage_*links usually needs to follow any others.
# for non-jobs mode the order here matters
@@ -351,7 +354,7 @@ all: stale_staged
# get a list of paths that we have previously staged to those same dirs
# anything in the 2nd list but not the first is stale - remove it.
stale_staged: staging .NOMETA
- @egrep '^[WL] .*${STAGE_OBJTOP}' /dev/null
${.MAKE.META.FILES:M*stage_*} | \
+ @${EGREP:Uegrep} '^[WL] .*${STAGE_OBJTOP}' /dev/null
${.MAKE.META.FILES:M*stage_*} | \
sed "/\.dirdep/d;s,.* '*\(${STAGE_OBJTOP}/[^ '][^ ']*\).*,\1," | \
sort > ${.TARGET}.staged1
@grep -l '${_dirdep}' /dev/null
${_STAGED_DIRS:M${STAGE_OBJTOP}*:O:u:@d@$d/*.dirdep@} | \
diff --git a/share/mk/meta2deps.py b/share/mk/meta2deps.py
index 1bcf1baedaaa..d7820ec71b5b 100755
--- a/share/mk/meta2deps.py
+++ b/share/mk/meta2deps.py
@@ -37,8 +37,7 @@ We only pay attention to a subset of the information in the
"""
RCSid:
- $FreeBSD$
- $Id: meta2deps.py,v 1.40 2021/12/13 19:32:46 sjg Exp $
+ $Id: meta2deps.py,v 1.45 2023/01/18 01:35:24 sjg Exp $
Copyright (c) 2011-2020, Simon J. Gerraty
Copyright (c) 2011-2017, Juniper Networks, Inc.
@@ -67,7 +66,10 @@ RCSid:
"""
-import os, re, sys
+import os
+import re
+import sys
+import stat
def resolve(path, cwd, last_dir=None, debug=0, debug_out=sys.stderr):
"""
@@ -245,6 +247,7 @@ class MetaFile:
self.curdir = conf.get('CURDIR')
self.reldir = conf.get('RELDIR')
self.dpdeps = conf.get('DPDEPS')
+ self.pids = {}
self.line = 0
if not self.conf:
@@ -445,12 +448,13 @@ class MetaFile:
pid_cwd = {}
pid_last_dir = {}
last_pid = 0
+ eof_token = False
self.line = 0
if self.curdir:
self.seenit(self.curdir) # we ignore this
- interesting = 'CEFLRV'
+ interesting = '#CEFLRVX'
for line in f:
self.line += 1
# ignore anything we don't care about
@@ -477,6 +481,12 @@ class MetaFile:
print("%s: CWD=%s" % (self.name, cwd),
file=self.debug_out)
continue
+ if w[0] == '#':
+ # check the file has not been truncated
+ if line.find('Bye') > 0:
+ eof_token = True
+ continue
+
pid = int(w[1])
if pid != last_pid:
if last_pid:
@@ -506,6 +516,13 @@ class MetaFile:
print("cwd=", cwd, file=self.debug_out)
continue
+ if w[0] == 'X':
+ try:
+ del self.pids[pid]
+ except KeyError:
+ pass
+ continue
+
if w[2] in self.seen:
if self.debug > 2:
print("seen:", w[2], file=self.debug_out)
@@ -519,11 +536,34 @@ class MetaFile:
continue
elif w[0] in 'ERWS':
path = w[2]
- if path == '.':
+ if w[0] == 'E':
+ self.pids[pid] = path
+ elif path == '.':
continue
self.parse_path(path, cwd, w[0], w)
- assert(version > 0)
+ if version == 0:
+ raise AssertionError('missing filemon data')
+ if not eof_token:
+ raise AssertionError('truncated filemon data')
+
+ setid_pids = []
+ # self.pids should be empty!
+ for pid,path in self.pids.items():
+ try:
+ # no guarantee that path is still valid
+ if os.stat(path).st_mode & (stat.S_ISUID|stat.S_ISGID):
+ # we do not expect anything after Exec
+ setid_pids.append(pid)
+ continue
+ except:
+ # we do not care why the above fails,
+ # we do not want to miss the ERROR below.
+ pass
+ print("ERROR: missing eXit for {} pid {}".format(path, pid))
+ for pid in setid_pids:
+ del self.pids[pid]
+ assert(len(self.pids) == 0)
if not file:
f.close()
diff --git a/share/mk/meta2deps.sh b/share/mk/meta2deps.sh
index 4767b83f4343..56367e0105f4 100755
--- a/share/mk/meta2deps.sh
+++ b/share/mk/meta2deps.sh
@@ -49,8 +49,10 @@
# The output, is a set of absolute paths with "SB" like:
#.nf
#
+# $SB/obj-i386/bsd/gnu/lib/csu
+# $SB/obj-i386/bsd/gnu/lib/libgcc
# $SB/obj-i386/bsd/include
-# $SB/obj-i386/bsd/lib/csu/i386
+# $SB/obj-i386/bsd/lib/csu/i386-elf
# $SB/obj-i386/bsd/lib/libc
# $SB/src/bsd/include
# $SB/src/bsd/sys/i386/include
@@ -75,8 +77,7 @@
# RCSid:
-# $FreeBSD$
-# $Id: meta2deps.sh,v 1.14 2020/10/02 03:11:17 sjg Exp $
+# $Id: meta2deps.sh,v 1.20 2023/01/18 01:35:24 sjg Exp $
# Copyright (c) 2010-2013, Juniper Networks, Inc.
# All rights reserved.
@@ -138,6 +139,13 @@ add_list() {
eval "$name=\"$list\""
}
+# some Linux systems have deprecated egrep in favor of grep -E
+# but not everyone supports that
+case "`echo bmake | egrep 'a|b' 2>&1`" in
+bmake) ;;
+*) egrep() { grep -E "$@"; }
+esac
+
_excludes_f() {
egrep -v "$EXCLUDES"
}
@@ -240,8 +248,8 @@ meta2deps() {
;;
*) cat /dev/null "$@";;
esac 2> /dev/null |
- sed -e 's,^CWD,C C,;/^[CREFLMV] /!d' -e "s,',,g" |
- $_excludes | ( version=no
+ sed -e 's,^CWD,C C,;/^[#CREFLMVX] /!d' -e "s,',,g" |
+ $_excludes | ( version=no epids= xpids= eof_token=no
while read op pid path junk
do
: op=$op pid=$pid path=$path
@@ -259,10 +267,15 @@ meta2deps() {
*) ;;
esac
version=0
+ case "$eof_token" in
+ no) ;; # ignore
+ 0) error "truncated filemon data";;
+ esac
+ eof_token=0
continue
;;
$pid,$pid) ;;
- *)
+ [1-9]*)
case "$lpid" in
"") ;;
*) eval ldir_$lpid=$ldir;;
@@ -272,8 +285,9 @@ meta2deps() {
;;
esac
+ : op=$op path=$path
case "$op,$path" in
- V,*) version=$path; continue;;
+ V,*) version=$pid; continue;;
W,*srcrel|*.dirdep) continue;;
C,*)
case "$path" in
@@ -289,7 +303,18 @@ meta2deps() {
eval cwd_$path=$cwd ldir_$path=$ldir
continue
;;
+ \#,bye) eof_token=1; continue;;
+ \#*) continue;;
*) dir=${path%/*}
+ case "$op" in
+ E) # setid apps get no tracing so we won't see eXit
+ case `'ls' -l $path 2> /dev/null | sed 's, .*,,'` in
+ *s*) ;;
+ *) epids="$epids $pid";;
+ esac
+ ;;
+ X) xpids="$xpids $pid"; continue;;
+ esac
case "$path" in
$src_re|$obj_re) ;;
/*/stage/*) ;;
@@ -379,9 +404,22 @@ meta2deps() {
echo $dir;;
esac
done > $tf.dirdep
+ : version=$version
case "$version" in
0) error "no filemon data";;
- esac ) || exit 1
+ esac
+ : eof_token=$eof_token
+ case "$eof_token" in
+ 0) error "truncated filemon data";;
+ esac
+ for p in $epids
+ do
+ : p=$p
+ case " $xpids " in
+ *" $p "*) ;;
+ *) error "missing eXit for pid $p";;
+ esac
+ done ) || exit 1
_nl=echo
for f in $tf.dirdep $tf.qual $tf.srcdep
do
