#!/bin/sh -e # vim: fdm=marker ############################################################################## # dman comes originally from Ubuntu's manpage repository generator and # interface. The original copyright notice is: # # Copyright (C) 2008 Canonical Ltd. # # This code was originally written by Dustin Kirkland , # based on a framework by Kees Cook . # # It was later modified by others, including the maintainers of the Debian # package "debian-goodies". # # In 2017 it was extended by astian . # # License: GPLv3+ ############################################################################## PROG_NAME=$(basename -- "$0") # Default configuration. {{{ BASE_URL='https://dyn.manpages.debian.org' # Cache refresh policy. CACHE_VALID_REFRESH_POLICIES='stale miss never always' CACHE_REFRESH_POLICY='stale' # Store cached copies under this directory. CACHE_DIR=${XDG_CACHE_HOME:-~/.cache}/dman # Consider cached files "fresh" for a period this long. In refresh policy # "stale" a fresh cache will not be refreshed. # # This mechanism is a hack for quantising/coalescing the remote checks for # cache "up-to-dateness" in order to reduce the number of cases in which one # needs to wait for network roundtrips. It is unrelated to real cache # "up-to-dateness": a cached file outside this window may still be up to date # with respect to the remote file; similarly, a cached file inside this window # may nonetheless be out of date. # # Set to 0 to disable the quantisation and check for update every time. This # makes "stale" the same as "always" with the exception that "stale" will # still fall back on an existing cache if refreshing fails. # # In essence, the complete cache refreshing process is: # # Every time a file is requested: # - if the cache does not exist, create it along with the tuple (cached # file, current time); # - if the cache does exist, check whether the current time is within the # window of freshness that starts at the stored time associated with the # cached file and spans the number of seconds indicated by this variable: # - if it is, use the cached file (without caring whether it really is up # to date or not); # - if it isn't, refresh the cache: check whether the remote file is newer # than the cached copy: # - if it is, recreate the cache; # - if it isn't, leave the cached copy untouched; # - in any case, if this process completed succesfully, update the # time associated with the cached copy to the current time. # # TODO: On the --help message, show this in a human readable form. CACHE_FRESHNESS_PERIOD=$((14 * 24 * 3600)) # 14 days # Option --help turns this on. HELP_AND_EXIT=0 if [ -r /etc/lsb-release ] ; then . /etc/lsb-release else DISTRIB_CODENAME=$(lsb_release -c -s 2>/dev/null) fi # Default to stable if LSB fails or is absent. RELEASE=${DISTRIB_CODENAME:-stable} # Additional curl options. CURL_EXTRA_OPTIONS='' # }}} # Utilities. {{{ usage() { cat <<-EOF Usage: $PROG_NAME [OPTIONS] ... OPTIONS: Mandatory arguments to long options are mandatory for their short forms as well. -h, --help Show this help message and exit. -r, --release= Fetch manual pages for the given distribution release. -p, --refresh-policy= Policy for refreshing cached files, i.e., checking for and downloading an updated copy: stale Refresh whenever the freshness period elapses and create the cache if it's missing. If refreshing fails, fall back on an existing cache. miss Don't refresh, but create the cache if it's missing. never Never refresh, fail if there's no cache. always Always refresh, fail if refreshing fails. The default is "$CACHE_REFRESH_POLICY". -f, --freshness-period= Consider cached files "fresh" for a period this long. In refresh policy "stale" a fresh cache will not be refreshed. The default is "$CACHE_FRESHNESS_PERIOD". -m, --missing Equivalent to --refresh-policy=miss. -c, --cached Equivalent to --refresh-policy=never. -d, --download Equivalent to --refresh-policy=always. --debug-network Print TLS/HTTP debugging information to stderr. Courtesy of Curl's "verbose" option. EOF } die_usage() { usage | head -n1 >&2 exit 1 } msg() { printf '%s: %s\n' "$PROG_NAME" "$*" } err() { msg "$*" >&2 } die() { err "$*" exit 1 } # in_wordlist [...] in_wordlist() { local what i what=$1 shift for i; do if [ "$what" = "$i" ]; then return 0 fi done return 1 } # }}} # Argument parsing. {{{ QUOTED_ARGS=$(getopt -n "$PROG_NAME" -o hr:p:f:mcd -l help \ -l release:,refresh-policy:,freshness-period: \ -l missing,cached,download \ -l debug-network \ -- "$@") || die_usage eval set -- "$QUOTED_ARGS" unset QUOTED_ARGS while true; do case "$1" in -h|--help) HELP_AND_EXIT=1 shift ;; -r|--release) RELEASE=$2 shift 2 ;; -p|--refresh-policy) in_wordlist "$2" $CACHE_VALID_REFRESH_POLICIES || die "Invalid refresh policy: $2" CACHE_REFRESH_POLICY=$2 shift 2 ;; -f|--freshness-period) [ "$2" -ge 0 ] || die "Invalid freshness period value: $2" CACHE_FRESHNESS_PERIOD=$2 shift 2 ;; -m|--missing) CACHE_REFRESH_POLICY='miss' shift ;; -c|--cached) CACHE_REFRESH_POLICY='never' shift ;; -d|--download) CACHE_REFRESH_POLICY='always' shift ;; --debug-network) if ! in_wordlist '-v' $CURL_EXTRA_OPTIONS; then CURL_EXTRA_OPTIONS="$CURL_EXTRA_OPTIONS -v" fi shift ;; --) shift break ;; esac done if [ "$HELP_AND_EXIT" -eq 1 ]; then usage exit 0 fi in_wordlist "$CACHE_REFRESH_POLICY" $CACHE_VALID_REFRESH_POLICIES || die "Internal error: Invalid refresh policy: $CACHE_REFRESH_POLICY" [ $# -gt 0 ] || die_usage # }}} # Helpers. {{{ curl_dl() { # Curl BUG: saying --no-tlsv1.1 actually results in using TLS 1.1. # For this server, saying nothing gives me TLS 1.2. # # See also: # - https://github.com/curl/curl/issues/1453 # - https://github.com/curl/curl/commit/913c3c8f5476bd7bc4d8d00509396bd4b525b8fc # # --no-sslv2 --no-sslv3 --no-tlsv1.0 --no-tlsv1.1 # Note: It's probably enough just saying "--proto =https". curl --proto '=https' --proto-redir '=https' --proto-default 'https' \ --fail --referer ';auto' --location \ --compressed --remote-time --progress-bar \ $CURL_EXTRA_OPTIONS "$@" } # make_alternative_filepath make_alternative_filepath() { local filepath suffix dir name filepath=$1 suffix=$2 dir=$(dirname -- "$filepath") name=$(basename -- "$filepath") printf '%s/.%s.%s' "$dir" "$name" "$suffix" } # cache_is_fresh # # Exit successfully if the last refresh for file is within the # window of "cache freshness". Otherwise, error out. cache_is_fresh() { local cached_file cached_file_meta last_refresh_time now cached_file=$1 cached_file_meta=$(make_alternative_filepath "$cached_file" 'last-refresh') now=$(date +%s) [ -f "$cached_file_meta" ] || return 1 last_refresh_time=$(stat -c %Y -- "$cached_file_meta") test "$((last_refresh_time + CACHE_FRESHNESS_PERIOD))" -ge "$now" } # do_refresh # # Download the resource at , place it at , but only if # is older or does not exist. If sucessful, also record the # current time in a "last-refresh" metadata file. Error out upon failure. do_refresh() { local url dst_file dst_file_meta tmp_file url=$1 dst_file=$2 dst_file_meta=$(make_alternative_filepath "$dst_file" 'last-refresh') # Note: Curl could be instructed to update the file directly (if it # exists), without a mediating temporary. The only reason for using # one is to avoid clobbering the file with an incomplete download. tmp_file=$(mktemp --tmpdir 'dman.XXXXXXXX') && trap "rm -f -- '$tmp_file'" EXIT && if [ -f "$dst_file" ]; then # If a cached copy exists, only try to download newer copies. curl_dl --time-cond "$dst_file" --output "$tmp_file" -- "$url" && # Check if we did in fact download a new copy. if [ "$(stat -c %s -- "$tmp_file")" -gt 0 ]; then mv -f -- "$tmp_file" "$dst_file" fi else # Otherwise, download whatever is available. curl_dl --output "$tmp_file" -- "$url" && mkdir -p -- "$(dirname -- "$dst_file")" && mv -f -- "$tmp_file" "$dst_file" fi && # Note last cache refresh time. touch "$dst_file_meta" } # }}} # Mirror support of man's languages. if [ ! -z "$LANG" ]; then LOCALE=$(echo $LANG | sed 's/_.*$//') LOCDOT=".$LOCALE" fi if [ ! -z "$LC_MESSAGES" ]; then LOCALE=$LC_MESSAGES LOCDOT=".$LOCALE" fi if echo $LOCALE | grep -Eq '^(C|en)'; then LOCALE='' LOCDOT='.en' fi DID_ERROR=0 for MANPAGE_NAME; do # TODO: Why the .gz suffix? The files I tried do exist but don't seem # to be compressed. Are only some compressed? Does the server ignore # the .gz suffix? In any case, should I make sure to compress the # cached copy? MANPAGE_PATH="$RELEASE/$MANPAGE_NAME$LOCDOT.gz" MANPAGE_URL="$BASE_URL/$MANPAGE_PATH" MANPAGE_CACHE_FILE="$CACHE_DIR/$MANPAGE_PATH" if [ -f "$MANPAGE_CACHE_FILE" ]; then CACHE_EXISTS=1 CACHE_MTIME=$(stat -c %Y -- "$MANPAGE_CACHE_FILE") else CACHE_EXISTS=0 fi # TODO: Avoid discarding open connections: pipeline all necessary # downloads on the same connection (http 2), or serialise them over # the same connection (http 1.1). Also, show them in the requested # order but download them asynchronously (show the first page as soon # as possible, keep downoading in the background). if [ "$CACHE_REFRESH_POLICY" = 'stale' -a "$CACHE_EXISTS" -eq 1 ] && cache_is_fresh "$MANPAGE_CACHE_FILE" || in_wordlist "$CACHE_REFRESH_POLICY" miss never && [ "$CACHE_EXISTS" -eq 1 ]; then # If refresh policy is "stale" and cache is fresh; or refresh # policy is "never" or "miss", and there's a cache. msg "$MANPAGE_NAME: Using cached copy." elif [ "$CACHE_REFRESH_POLICY" = 'never' ]; then # If refresh policy is "never" but there's no cache. err "$MANPAGE_NAME: Not found in cache." DID_ERROR=1 continue else # If refresh policy is "always", or there's no cache or must # be refreshed. if [ "$CACHE_REFRESH_POLICY" != 'always' ]; then if [ "$CACHE_EXISTS" -eq 0 ]; then msg "$MANPAGE_NAME: Not found in cache." else msg "$MANPAGE_NAME: Cache freshness period elapsed." fi fi msg "$MANPAGE_NAME: Refreshing cache..." if do_refresh "$MANPAGE_URL" "$MANPAGE_CACHE_FILE"; then NEW_CACHE_MTIME=$(stat -c %Y -- "$MANPAGE_CACHE_FILE") if [ "$CACHE_EXISTS" -eq 1 ] && [ "$NEW_CACHE_MTIME" -eq "$CACHE_MTIME" ]; then msg "$MANPAGE_NAME: Cached copy is up to date." else msg "$MANPAGE_NAME: Cache refreshed with an updated copy." fi else err "$MANPAGE_NAME: Failed to fetch '$MANPAGE_URL'." DID_ERROR=1 [ "$CACHE_EXISTS" -eq 1 ] || continue if [ "$CACHE_REFRESH_POLICY" = 'always' ]; then err "$MANPAGE_NAME: Note: A copy from $(date -Rd "@$CACHE_MTIME") exists in cache." continue fi msg "$MANPAGE_NAME: Falling back to cached copy from $(date -Rd "@$CACHE_MTIME")." fi fi man -l -- "$MANPAGE_CACHE_FILE" done if [ "$DID_ERROR" -eq 1 ]; then err 'Some errors occurred.' exit 1 fi