The branch main has been updated by kevans:
URL:
https://cgit.FreeBSD.org/src/commit/?id=8378665ff9176677c2ed117756afceb601a1dca0
commit 8378665ff9176677c2ed117756afceb601a1dca0
Author: Kyle Evans <kev...@freebsd.org>
AuthorDate: 2025-06-04 17:06:25 +0000
Commit: Kyle Evans <kev...@freebsd.org>
CommitDate: 2025-06-04 17:06:29 +0000
tools: build: add a rewrite of makeman in lua
The primary benefit of this rewrite is that it parallelizes a number of
the make(1) jobs that it needs to do. It does so with a very naive
forking model that could likely be improved, but is sufficient for our
purposes. This version also doesn't assume that CWD is sane, and
instead operates relative to the directory the script resides in.
Note that this initial version is only intended to match the output of
the legacy script. Some work is planned afterward to refactor the
script out into various components to improve maintainability after we
have switched over to it.
In my horribly performing dev environment, this version runs in 40s
rather than the original ~2 minutes. On a Mt. Snow machine, this
version runs in ~15s rather than the original ~1m40s.
This change does not yet switch the top-level `makeman` target over to
the new version.
Reviewed by: bapt (earlier version), emaste
Differential Revision: https://reviews.freebsd.org/D39084
---
tools/build/options/makeman.lua | 791 ++++++++++++++++++++++++++++++++++++++++
1 file changed, 791 insertions(+)
diff --git a/tools/build/options/makeman.lua b/tools/build/options/makeman.lua
new file mode 100644
index 000000000000..e96e6f50174b
--- /dev/null
+++ b/tools/build/options/makeman.lua
@@ -0,0 +1,791 @@
+--
+-- Copyright (c) 2023 Kyle Evans <kev...@freebsd.org>
+--
+-- SPDX-License-Identifier: BSD-2-Clause
+--
+
+local libgen = require('posix.libgen')
+local lfs = require('lfs')
+local stdlib = require('posix.stdlib')
+local unistd = require('posix.unistd')
+local sys_wait = require('posix.sys.wait')
+
+local curdate = os.date("%B %e, %Y")
+
+local output_head <const> = ".\\\" DO NOT EDIT-- this file is @" ..
[[generated by tools/build/options/makeman.
+.Dd ]] .. curdate .. [[
+
+.Dt SRC.CONF 5
+.Os
+.Sh NAME
+.Nm src.conf
+.Nd "source build options"
+.Sh DESCRIPTION
+The
+.Nm
+file contains variables that control what components will be generated during
+the build process of the
+.Fx
+source tree; see
+.Xr build 7 .
+.Pp
+The
+.Nm
+file uses the standard makefile syntax.
+However,
+.Nm
+should not specify any dependencies to
+.Xr make 1 .
+Instead,
+.Nm
+is to set
+.Xr make 1
+variables that control the aspects of how the system builds.
+.Pp
+The default location of
+.Nm
+is
+.Pa /etc/src.conf ,
+though an alternative location can be specified in the
+.Xr make 1
+variable
+.Va SRCCONF .
+Overriding the location of
+.Nm
+may be necessary if the system-wide settings are not suitable
+for a particular build.
+For instance, setting
+.Va SRCCONF
+to
+.Pa /dev/null
+effectively resets all build controls to their defaults.
+.Pp
+The only purpose of
+.Nm
+is to control the compilation of the
+.Fx
+source code, which is usually located in
+.Pa /usr/src .
+As a rule, the system administrator creates
+.Nm
+when the values of certain control variables need to be changed
+from their defaults.
+.Pp
+In addition, control variables can be specified
+for a particular build via the
+.Fl D
+option of
+.Xr make 1
+or in its environment; see
+.Xr environ 7 .
+.Pp
+The environment of
+.Xr make 1
+for the build can be controlled via the
+.Va SRC_ENV_CONF
+variable, which defaults to
+.Pa /etc/src-env.conf .
+Some examples that may only be set in this file are
+.Va WITH_DIRDEPS_BUILD ,
+and
+.Va WITH_META_MODE ,
+and
+.Va MAKEOBJDIRPREFIX
+as they are environment-only variables.
+.Pp
+The values of
+.Va WITH_
+and
+.Va WITHOUT_
+variables are ignored regardless of their setting;
+even if they would be set to
+.Dq Li FALSE
+or
+.Dq Li NO .
+The presence of an option causes
+it to be honored by
+.Xr make 1 .
+.Pp
+This list provides a name and short description for variables
+that can be used for source builds.
+.Bl -tag -width indent
+]]
+
+local output_tail <const> = [[.El
+.Sh FILES
+.Bl -tag -compact -width Pa
+.It Pa /etc/src.conf
+.It Pa /etc/src-env.conf
+.It Pa /usr/share/mk/bsd.own.mk
+.El
+.Sh SEE ALSO
+.Xr make 1 ,
+.Xr make.conf 5 ,
+.Xr build 7 ,
+.Xr ports 7
+.Sh HISTORY
+The
+.Nm
+file appeared in
+.Fx 7.0 .
+.Sh AUTHORS
+This manual page was autogenerated by
+.An tools/build/options/makeman .
+]]
+
+local scriptdir <const> = libgen.dirname(stdlib.realpath(arg[0]))
+local srcdir <const> = stdlib.realpath(scriptdir .. "/../../../")
+local makesysdir <const> = srcdir .. "/share/mk"
+
+local make_envvar = os.getenv("MAKE")
+local make_cmd_override = {}
+if make_envvar then
+ for word in make_envvar:gmatch("[^%s]+") do
+ make_cmd_override[#make_cmd_override + 1] = word
+ end
+end
+
+-- Lifted from bsdinstall/scripts/pkgbase.in (read_all)
+local function read_pipe(pipe)
+ local ret = ""
+ repeat
+ local buffer = assert(unistd.read(pipe, 1024))
+ ret = ret .. buffer
+ until buffer == ""
+ return ret
+end
+local function run_make(args)
+ local cmd_args = {"env", "-i", "make", "-C", srcdir, "-m", makesysdir,
+ "__MAKE_CONF=/dev/null", "SRCCONF=/dev/null"}
+
+ if #make_cmd_override > 0 then
+ cmd_args[3] = make_cmd_override[1]
+ for k = 2, #make_cmd_override do
+ local val = make_cmd_override[k]
+
+ table.insert(cmd_args, 3 + (k - 1), val)
+ end
+ end
+ for k, v in ipairs(args) do
+ cmd_args[#cmd_args + 1] = v
+ end
+
+ local r, w = assert(unistd.pipe())
+ local pid = assert(unistd.fork())
+ if pid == 0 then
+ -- Child
+ assert(unistd.close(r))
+ assert(unistd.dup2(w, 1))
+ assert(unistd.dup2(w, 2))
+ assert(unistd.execp("env", cmd_args))
+ unistd._exit()
+ end
+
+ -- Parent
+ assert(unistd.close(w))
+
+ local output = read_pipe(r)
+ assert(unistd.close(r))
+
+ local _, exit_type, exit_code = assert(sys_wait.wait(pid))
+ assert(exit_type == "exited", "make exited with wrong status")
+ assert(exit_code == 0, "make exited unsuccessfully")
+ return output
+end
+
+local function native_target()
+ local output = run_make({"MK_AUTO_OBJ=NO", "-V", "MACHINE",
+ "-V", "MACHINE_ARCH"})
+
+ local arch, machine_arch
+ for x in output:gmatch("[^\n]+") do
+ if not arch then
+ arch = x
+ elseif not machine_arch then
+ machine_arch = x
+ end
+ end
+
+ return arch .. "/" .. machine_arch
+end
+
+local function src_targets()
+ local targets = {}
+ targets[native_target()] = true
+
+ local output = run_make({"MK_AUTO_OBJ=no", "targets"})
+ local curline = 0
+
+ for line in output:gmatch("[^\n]+") do
+ curline = curline + 1
+ if curline ~= 1 then
+ local arch = line:match("[^%s]+/[^%s]+")
+
+ -- Make sure we don't roll over our default arch
+ if arch and not targets[arch] then
+ targets[arch] = false
+ end
+ end
+ end
+
+ return targets
+end
+
+local function config_options(srcconf, env, take_dupes, linting)
+ srcconf = srcconf or "/dev/null"
+ env = env or {}
+
+ local option_args = {".MAKE.MODE=normal", "showconfig",
+ "SRC_ENV_CONF=" .. srcconf}
+
+ for _, val in ipairs(env) do
+ option_args[#option_args + 1] = val
+ end
+
+ local output = run_make(option_args)
+
+ local options = {}
+ local known_dupes = {}
+
+ local function warn_on_dupe(option, val)
+ if not linting or known_dupes[option] then
+ return false
+ end
+ if not option:match("^OPT_") then
+ val = val == "yes"
+ end
+
+ known_dupes[option] = true
+ return val ~= options[val]
+ end
+
+ for opt in output:gmatch("[^\n]+") do
+ if opt:match("^MK_[%a%d_]+%s+=%s+.+") then
+ local name = opt:match("MK_[%a%d_]+")
+ local val = opt:match("= .+"):sub(3)
+
+ -- Some settings, e.g., MK_INIT_ALL_ZERO, may end up
+ -- output twice for some reason that I haven't dug into;
+ -- take the first value. In some circumstances, though,
+ -- we do make an exception and actually want to take the
+ -- latest.
+ if take_dupes or options[name] == nil then
+ options[name] = val == "yes"
+ elseif warn_on_dupe(name, val) then
+ io.stderr:write("ignoring duplicate option " ..
+ name .. "\n")
+ end
+ elseif opt:match("^OPT_[%a%d_]+%s+=%s+.+") then
+ local name = opt:match("OPT_[%a%d_]+")
+ local val = opt:match("= .+"):sub(3)
+
+ -- Multi-value options will arbitrarily use a table here
+ -- to indicate the difference.
+ if take_dupes or options[name] == nil then
+ options[name] = val
+ elseif warn_on_dupe(name, val) then
+ io.stderr:write("ignoring duplicate option " ..
+ name .. "\n")
+ end
+ end
+ end
+
+ return options
+end
+
+local function env_only_options()
+ local output = run_make({"MK_AUTO_OBJ=no", "-V", "__ENV_ONLY_OPTIONS"})
+ local options = {}
+
+ for opt in output:gmatch("[^%s]+") do
+ options["MK_" .. opt] = true
+ end
+
+ return options
+end
+
+local function required_options()
+ local output = run_make({"-f", "share/mk/src.opts.mk", "-V",
+ "__REQUIRED_OPTIONS"})
+ local options = {}
+
+ for opt in output:gmatch("[^%s]+") do
+ options["MK_" .. opt] = true
+ end
+
+ return options
+end
+
+local function config_description(option_name)
+ local fh = io.open(scriptdir .. "/" .. option_name)
+ local desc
+
+ if fh then
+ desc = ""
+ for line in fh:lines() do
+ if not line:match("%$FreeBSD%$") then
+ desc = desc .. line .. "\n"
+ end
+ end
+
+ assert(fh:close())
+ end
+
+ return desc
+end
+
+local function config_descriptions(options)
+ local desc = {}
+ for name, _ in pairs(options) do
+ if name:match("^MK_") then
+ local basename = name:gsub("^MK_", "")
+ local with_name = "WITH_" .. basename
+ local without_name = "WITHOUT_" .. basename
+
+ desc[with_name] = config_description(with_name)
+ desc[without_name] = config_description(without_name)
+ elseif name:match("^OPT_") then
+ local basename = name:gsub("^OPT_", "")
+
+ desc[name] = config_description(basename)
+ end
+ end
+ return desc
+end
+
+local function dependent_options(tmpdir, option_name, all_opts, omit_others)
+ local opt_sense = not not option_name:match("^WITH_")
+ local base_option_name = option_name:gsub("^[^_]+_", "")
+ local prefix = (opt_sense and "WITHOUT_") or "WITH_"
+
+ local srcconf = tmpdir .. "/src-" ..prefix .. "ALL_" ..
+ option_name .. ".conf"
+ local fh = assert(io.open(srcconf, "w+"))
+
+ fh:write(option_name .. "=\"YES\"\n")
+ if not omit_others then
+ for opt, value in pairs(all_opts) do
+ local base_opt = opt:gsub("^MK_", "")
+
+ if base_opt ~= base_option_name then
+ local opt_prefix = (value and "WITH_") or
"WITHOUT_"
+ fh:write(opt_prefix .. base_opt .. "=\"YES\"\n")
+ end
+ end
+ end
+ assert(fh:close())
+
+ local option_name_key = "MK_" .. base_option_name
+ local options = config_options(srcconf, nil, omit_others)
+ for name, value in pairs(options) do
+ if name == option_name_key or value == all_opts[name] then
+ options[name] = nil
+ elseif name:match("^OPT_") then
+ -- Strip out multi-option values at the moment, they do
+ -- not really make sense.
+ options[name] = nil
+ end
+ end
+
+ return options
+end
+
+local function export_option_table(fd, name, options)
+ unistd.write(fd, name .. " = {")
+ for k, v in pairs(options) do
+ v = (v and "true") or "false"
+ unistd.write(fd, "['" .. k .. "'] = " .. v .. ",")
+ end
+ unistd.write(fd, "}")
+end
+
+local function all_dependent_options(tmpdir, options, default_opts,
+ with_all_opts, without_all_opts)
+ local all_enforced_options = {}
+ local all_effect_options = {}
+ local children = {}
+
+ for _, name in ipairs(options) do
+ local rfd, wfd = assert(unistd.pipe())
+ local pid = assert(unistd.fork())
+
+ if pid == 0 then
+ -- We need to pcall() this so that errors bubble up to
+ -- our _exit() call rather than the main exit.
+ local ret, errobj = pcall(function()
+ unistd.close(rfd)
+
+ local compare_table
+ if name:match("^WITHOUT") then
+ compare_table = with_all_opts
+ else
+ compare_table = without_all_opts
+ end
+
+ -- List of knobs forced on by this one
+ local enforced_options =
dependent_options(tmpdir, name,
+ compare_table)
+ -- List of knobs implied by this by one (once
additionally
+ -- filtered based on enforced_options values)
+ local effect_options =
dependent_options(tmpdir, name,
+ default_opts, true)
+
+ export_option_table(wfd, "enforced_options",
+ enforced_options)
+ export_option_table(wfd, "effect_options",
+ effect_options)
+ end)
+
+ io.stderr:write(".")
+
+ if ret then
+ unistd._exit(0)
+ else
+ unistd.write(wfd, errobj)
+ unistd._exit(1)
+ end
+ end
+
+ unistd.close(wfd)
+ children[pid] = {name, rfd}
+ end
+
+ while next(children) ~= nil do
+::again::
+ local pid, status, exitcode = sys_wait.wait(-1)
+
+ if status ~= "exited" then
+ goto again
+ end
+
+ local info = children[pid]
+ children[pid] = nil
+
+ local name = info[1]
+ local rfd = info[2]
+ local buf = ''
+ local rbuf, sz
+
+ -- Drain the pipe
+ rbuf = unistd.read(rfd, 512)
+ while #rbuf ~= 0 do
+ buf = buf .. rbuf
+ rbuf = unistd.read(rfd, 512)
+ end
+
+ unistd.close(rfd)
+
+ if exitcode ~= 0 then
+ error("Child " .. pid .. " failed, buf: " .. buf)
+ end
+
+ -- The child has written a pair of tables named enforced_options
+ -- and effect_options to the pipe. We'll load the pipe buffer
+ -- as a string and then yank these out of the clean environment
+ -- that we execute the chunk in.
+ local child_env = {}
+ local res, err = pcall(load(buf, "child", "t", child_env))
+
+ all_enforced_options[name] = child_env["enforced_options"]
+ all_effect_options[name] = child_env["effect_options"]
+ end
+
+ io.stderr:write("\n")
+ return all_enforced_options, all_effect_options
+end
+
+local function get_defaults(target_archs, native_default_opts)
+ local target_defaults = {}
+ -- Set of options with differing defaults in some archs
+ local different_defaults = {}
+
+ for tgt, dflt in pairs(target_archs) do
+ if dflt then
+ local native_copy = {}
+ for opt, val in pairs(native_default_opts) do
+ native_copy[opt] = val
+ end
+ target_defaults[tgt] = native_copy
+ goto skip
+ end
+
+ local target = tgt:gsub("/.+$", "")
+ local target_arch = tgt:gsub("^.+/", "")
+
+ local target_opts = config_options(nil, {"TARGET=" .. target,
+ "TARGET_ARCH=" .. target_arch})
+
+ for opt, val in pairs(target_opts) do
+ if val ~= native_default_opts[opt] then
+ different_defaults[opt] = true
+ end
+ end
+
+ target_defaults[tgt] = target_opts
+::skip::
+ end
+
+ for opt in pairs(native_default_opts) do
+ if different_defaults[opt] == nil then
+ for _, opts in pairs(target_defaults) do
+ opts[opt] = nil
+ end
+ end
+ end
+
+ for tgt, opts in pairs(target_defaults) do
+ local val = opts["MK_ACPI"]
+
+ if val ~= nil then
+ print(" - " .. tgt .. ": " .. ((val and "yes") or "no"))
+ end
+ end
+
+ return target_defaults, different_defaults
+end
+
+local function option_comparator(lhs, rhs)
+ -- Convert both options to the base name, compare that instead unless
+ -- they're the same option. For the same option, we just want to get
+ -- ordering between WITH_/WITHOUT_ correct.
+ local base_lhs = lhs:gsub("^[^_]+_", "")
+ local base_rhs = rhs:gsub("^[^_]+_", "")
+
+ if base_lhs == base_rhs then
+ return lhs < rhs
+ else
+ return base_lhs < base_rhs
+ end
+end
+
+local function main(tmpdir)
+ io.stderr:write("building src.conf.5 man page from files in " ..
+ scriptdir .. "\n")
+
+ local env_only_opts <const> = env_only_options()
+ local default_opts = config_options(nil, nil, nil, true)
+ local opt_descriptions = config_descriptions(default_opts)
+ local srcconf_all <const> = tmpdir .. "/src-all-enabled.conf"
+ local fh = io.open(srcconf_all, "w+")
+ local all_targets = src_targets()
+ local target_defaults, different_defaults = get_defaults(all_targets,
+ default_opts)
+ local options = {}
+ local without_all_opts = {}
+
+ for name, value in pairs(default_opts) do
+ if name:match("^MK_") then
+ local base_name = name:gsub("^MK_", "")
+ local with_name = "WITH_" .. base_name
+ local without_name = "WITHOUT_" .. base_name
+ -- If it's differently defaulted on some architectures,
+ -- we'll split it into WITH_/WITHOUT_ just to simplify
+ -- some later bits.
+ if different_defaults[name] ~= nil then
+ options[#options + 1] = with_name
+ options[#options + 1] = without_name
+ elseif value then
+ options[#options + 1] = without_name
+ else
+ options[#options + 1] = with_name
+ end
+
+ without_all_opts[name] = false
+ assert(fh:write(with_name .. '="YES"\n'))
+ else
+ options[#options + 1] = name
+ end
+ end
+
+ assert(fh:close())
+
+ local with_all_opts = config_options(srcconf_all)
+ local all_enforced_options, all_effect_options
+ local all_required_options = required_options()
+
+ all_enforced_options, all_effect_options = all_dependent_options(tmpdir,
+ options, default_opts, with_all_opts, without_all_opts)
+
+ table.sort(options, option_comparator)
+ io.stdout:write(output_head)
+ for _, name in ipairs(options) do
+ local value
+
+ if name:match("^OPT_") then
+ goto skip
+ end
+ assert(name:match("^WITH"), "Name looks wrong: " .. name)
+ local describe_option = name
+
+ value = not not name:match("^WITHOUT")
+
+ -- Normalize name to MK_ for indexing into various other
+ -- arrays
+ name = "MK_" .. name:gsub("^[^_]+_", "")
+
+ print(".It Va " .. describe_option:gsub("^OPT_", ""))
+ if opt_descriptions[describe_option] then
+ io.stdout:write(opt_descriptions[describe_option])
+ else
+ io.stderr:write("Missing description for " ..
+ describe_option .. "\n")
+ end
+
+ local enforced_options = all_enforced_options[describe_option]
+ local effect_options = all_effect_options[describe_option]
+
+ if different_defaults[name] ~= nil then
+ print([[.Pp
+This is a default setting on]])
+
+ local which_targets = {}
+ for tgt, tgt_options in pairs(target_defaults) do
+ if tgt_options[name] ~= value then
+ which_targets[#which_targets + 1] = tgt
+ end
+ end
+
+ table.sort(which_targets)
+ for idx, tgt in ipairs(which_targets) do
+ io.stdout:write(tgt)
+ if idx < #which_targets - 1 then
+ io.stdout:write(", ")
+ elseif idx == #which_targets - 1 then
+ io.stdout:write(" and ")
+ end
+ end
+ print(".")
+ end
+
+ -- Unset any implied options that are actually required.
+ for dep_opt in pairs(enforced_options) do
+ if all_required_options[dep_opt] then
+ enforced_options[dep_opt] = nil
+ end
+ end
+ if next(enforced_options) ~= nil then
+ print([[When set, it enforces these options:
+.Pp
+.Bl -item -compact]])
+
+ local sorted_dep_opt = {}
+ for dep_opt in pairs(enforced_options) do
+ sorted_dep_opt[#sorted_dep_opt + 1] = dep_opt
+ end
+
+ table.sort(sorted_dep_opt)
+ for _, dep_opt in ipairs(sorted_dep_opt) do
+ local dep_val = enforced_options[dep_opt]
+ local dep_prefix = (dep_val and "WITH_") or
+ "WITHOUT_"
+ local dep_name = dep_opt:gsub("^MK_",
+ dep_prefix)
+ print(".It")
+ print(".Va " .. dep_name)
+ end
+
+ print(".El")
+ end
+
+ if next(effect_options) ~= nil then
+ if next(enforced_options) ~= nil then
+ -- Remove any options that were previously
+ -- noted as enforced...
+ for opt, val in pairs(effect_options) do
+ if enforced_options[opt] == val then
+ effect_options[opt] = nil
+ end
+ end
+
+ -- ... and this could leave us with an empty
+ -- set.
+ if next(effect_options) == nil then
+ goto noenforce
+ end
+
+ print(".Pp")
+ end
+
+ print([[When set, these options are also in effect:
+.Pp
+.Bl -inset -compact]])
+
+ local sorted_dep_opt = {}
+ for dep_opt in pairs(effect_options) do
+ sorted_dep_opt[#sorted_dep_opt + 1] = dep_opt
+ end
+
+ table.sort(sorted_dep_opt)
+ for _, dep_opt in ipairs(sorted_dep_opt) do
+ local dep_val = effect_options[dep_opt]
+ local dep_prefix = (dep_val and "WITH_") or
+ "WITHOUT_"
+ local not_dep_prefix = ((not dep_val) and
"WITH_") or
+ "WITHOUT_"
+ local dep_name = dep_opt:gsub("^MK_",
+ dep_prefix)
+ local not_dep_name = dep_opt:gsub("^MK_",
+ not_dep_prefix)
+
+ print(".It Va " .. dep_name)
+ print("(unless")
+ print(".Va " .. not_dep_name)
+ print("is set explicitly)")
+ end
+
+ print(".El")
+::noenforce::
+ end
+
+ if env_only_opts[name] ~= nil then
+ print([[.Pp
+This must be set in the environment, make command line, or
+.Pa /etc/src-env.conf ,
+not
+.Pa /etc/src.conf .]])
+ end
+ ::skip::
+ end
+ print([[.El
+.Pp
+The following options accept a single value from a list of valid values.
+.Bl -tag -width indent]])
+ for _, name in ipairs(options) do
+ if name:match("^OPT_") then
+ local desc = opt_descriptions[name]
+
+ print(".It Va " .. name:gsub("^OPT_", ""))
+ if desc then
+ io.stdout:write(desc)
+ else
+ io.stderr:write("Missing description for " ..
+ name .. "\n")
+ end
+ end
+ end
+ io.stdout:write(output_tail)
+end
+
+local tmpdir = "/tmp/makeman." .. unistd.getpid()
+
+if not lfs.mkdir(tmpdir) then
+ error("Failed to create tempdir " .. tmpdir)
+end
+
+-- Catch any errors so that we can properly clean up, then re-throw it.
+local ret, errobj = pcall(main, tmpdir)
+
+for fname in lfs.dir(tmpdir) do
+ if fname ~= "." and fname ~= ".." then
+ assert(os.remove(tmpdir .. "/" .. fname))
+ end
+end
+
+if not lfs.rmdir(tmpdir) then
+ assert(io.stderr:write("Failed to clean up tmpdir: " .. tmpdir .. "\n"))
+end
+
+if not ret then
+ io.stderr:write(errobj .. "\n")
+ os.exit(1)
+end
