On Fri, Apr 17, 2020 at 05:26:58PM -0600, Aaron Bieber wrote:
> On Wed, 15 Apr 2020 at 18:56:09 -0600, Aaron Bieber wrote:
> > On Mon, 06 Apr 2020 at 17:15:17 -0600, Aaron Bieber wrote:
> > > Round two! This time with espie@'s bsd.port.mk fix:
> > > https://marc.info/?l=openbsd-ports-cvs&m=158618354824687&w=2
> > >
> > > This resolves the issue with the conversion of [A-Z] letters to ![a-z] in
> > > the
> > > package path.
> > >
> > > With this diff I can generate ports for the following without issue:
> > >
> > > github.com/jrick/domain
> > > github.com/jrick/ss
> > > github.com/junegunn/fzf
> > > github.com/qbit/gavin
> > > golang.zx2c4.com/wireguard
> > > humungus.tedunangst.com/r/honk
> > > suah.dev/ogvt
> > >
> > > There are still some issues with things like github.com/restic/restic and
> > > github.com/gohugoio/hugo . For some reason the build looks for some files
> > > that
> > > 'go mod graph' doesn't list.
> > >
> > > For most go apps that follow the module guidelines:
> > > https://github.com/golang/go/wiki/Modules
> > >
> > > This method of building things should work pretty well. One will have to
> > > go in
> > > and add a "do-install" to grab other files like man pages, etc.
> > >
> >
> > OK, this fixes some issues with determining the latest version of a given
> > port.
> >
> > With this, I can build a port for restic! (it still needs
> > 'ALL_TARGET=github.com/restic/restic/...' set manually)
> >
> > hugo still results in some missing mod/zip files.
> >
>
> Round... 10?
A few super tiny comments inline, only one I would fail the review for :-)
>
> Changes:
> - Handle determine latest version in a manner that is more inline with how Go
> does it.
> - Error out when we are asked to gen a port for an incompatible module.
> - Detect licenses from pkg.go.dev.
> - Update ALL_TARGET to include "/cmd/..." when a "cmd" directory is present in
> a module.
>
> diff --git a/infrastructure/bin/portgen b/infrastructure/bin/portgen
> index ad5ab17f3cf..b7316d42b64 100755
> --- a/infrastructure/bin/portgen
> +++ b/infrastructure/bin/portgen
> @@ -32,6 +32,7 @@ use lib ( "$portdir/infrastructure/lib",
> "$FindBin::Bin/../lib" );
> use OpenBSD::PortGen::Port::CPAN;
> use OpenBSD::PortGen::Port::PyPI;
> use OpenBSD::PortGen::Port::Ruby;
> +use OpenBSD::PortGen::Port::Go;
>
> my ( $type, $module ) = @ARGV;
>
> @@ -44,6 +45,8 @@ if ( $type eq 'p5' ) {
> $o = OpenBSD::PortGen::Port::PyPI->new();
> } elsif ( $type eq 'ruby' ) {
> $o = OpenBSD::PortGen::Port::Ruby->new();
> +} elsif ( $type eq 'go' ) {
> + $o = OpenBSD::PortGen::Port::Go->new();
> } else {
> die "unknown module type\n";
> }
> diff --git a/infrastructure/lib/OpenBSD/PortGen/Port/Go.pm
> b/infrastructure/lib/OpenBSD/PortGen/Port/Go.pm
> new file mode 100644
> index 00000000000..34668ea6fa7
> --- /dev/null
> +++ b/infrastructure/lib/OpenBSD/PortGen/Port/Go.pm
> @@ -0,0 +1,288 @@
> +# $OpenBSD: Go.pm,v 1.16 2019/05/16 16:01:10 afresh1 Exp $
> +#
> +# Copyright (c) 2019 Aaron Bieber <[email protected]>
> +#
> +# Permission to use, copy, modify, and distribute this software for any
> +# purpose with or without fee is hereby granted, provided that the above
> +# copyright notice and this permission notice appear in all copies.
> +#
> +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
> +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
> +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
> +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
> +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
> +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
> +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
> +
> +package OpenBSD::PortGen::Port::Go;
> +
> +use 5.028;
> +use utf8;
> +use warnings;
> +use strict;
> +use warnings qw(FATAL utf8); # fatalize encoding glitches
> +use open qw(:std :encoding(UTF-8)); # undeclared streams in UTF-8
> +use OpenBSD::PackageName;
> +use OpenBSD::PortGen::Utils qw( fetch );
> +
> +use parent 'OpenBSD::PortGen::Port';
> +
> +use Carp;
> +use Cwd;
> +use File::Temp qw/ tempdir /;
> +use Data::Dumper;
> +
> +use OpenBSD::PortGen::Dependency;
> +
> +my $license_map = {
> + 'BSD-2-Clause' => 'BSD',
> + 'BSD-3-Clause' => 'BSD3',
> +};
This could just go into "%good_licenses" in OpenBSD::PortGen::License.
https://github.com/openbsd/ports/blob/master/infrastructure/lib/OpenBSD/PortGen/License.pm
> +
> +sub ecosystem_prefix
> +{
> + my $self = shift;
> + return '';
> +}
> +
> +sub base_url
> +{
> + my $self = shift;
> + return 'https://proxy.golang.org/';
> +}
> +
> +sub _go_lic_info
> +{
> + my ( $self, $module ) = @_;
> + my $html = fetch("https://pkg.go.dev/mod/"; . $module);
> + my $license = "unknown";
> + if ($html =~ m/<a.+tab=licenses.+>(.+)<\/a>/) {
> + $license = $1;
> + $license = $license_map->{$license} || $license;
> + }
> + return $license;
> +}
> +
> +sub _go_determine_name
> +{
> + # Some modules end in "v1" or "v2", if we find one of these, we need
> + # to set PKGNAME to something up a level
> + my ( $self, $module ) = @_;
> + my $json = {};
> +
> + $json = $self->get_json( $module . '/@latest' );
totally fine to just do:
my $json = $self->get_json( $module . '/@latest' );
> +
> + if ($json->{Version} =~ m/incompatible/) {
> + my $msg = "${module} $json->{Version} is incompatible with Go
> modules.";
> + warn "$msg\n";
> + croak $msg;
> + }
croak should be printing out this $msg, so this would end up with
duplicate output. That seems not great.
> +
> + if ($module =~ m/v\d$/) {
> + $json->{Name} = ( split '/', $module )[-2];
> + } else {
> + $json->{Name} = ( split '/', $module )[-1];
> + }
> +
> + $json->{Module} = $module;
> +
> + return $json;
> +}
> +
> +sub get_dist_info
> +{
> + my ( $self, $module ) = @_;
> +
> + my $json = $self->_go_determine_name($module);
> +
> + my %mods;
> + for ( $self->_go_mod_graph($json) ) {
> + my ($direct, $ephemeral) = @{$_};
> +
> + for my $d ( $direct, $ephemeral ) {
> + next unless $d->{Version};
> + $mods{ $d->{Module} }{ $d->{Version} } ||= $d;
> + }
> + }
> +
> + # Filter dependencies to the one with the highest version
> + foreach my $mod ( sort keys %mods ) {
> + # Sort semver numerically then the rest alphanumeric.
> + # This is overkill for sorting, but I already wrote it
> + #
> + # XXX this chokes on the +incompatible likes in some modules
> + my @versions =
> + map { $_->[0] }
> + sort {
> + $a->[1] <=> $b->[1]
> + || $a->[2] <=> $b->[2]
> + || $a->[3] <=> $b->[3]
> + || $a->[4] cmp $b->[4]
> + }
> + map { $_->[1] =~ s/^v//; $_ }
> + map { [ $_, split /[\.-]/, $_, 4 ] }
If you add the `+` to this character class along with `\.-` that may
solve the choking on +incompatible. Although I'm not 100% sure what
those versions look like.
> + keys %{ $mods{$mod} };
> +
> + push @{ $json->{Dist} }, $mods{$mod}{ $versions[-1] };
> + push @{ $json->{Mods} }, map { $mods{$mod}{$_} } @versions;
> + }
> +
> + $json->{License} = $self->_go_lic_info($module);
> +
> + return $json;
> +}
> +
> +sub _go_mod_graph
> +{
> + my ($self, $json) = @_;
> + my $dir = tempdir(CLEANUP => 0);
> +
> + my $mod = $self->get("$json->{Module}/\@v/$json->{Version}.mod");
> + my ($module) = $mod =~ /\bmodule\s+(.*)\n/;
> + $module =~ s/\s+$//;
or
my ($module) = $mod =~ /\bmodule\s+(.*?)\s/;
> + unless ( $json->{Module} eq $module ) {
> + my $msg = "Module $json->{Module} doesn't match $module";
> + warn "$msg\n";
> + croak $msg;
> + }
> +
> + {
> + open my $fh, '>', $dir . "/go.mod" or die $!;
> + print $fh $mod;
> + close $fh;
> + }
> +
> + my $old_cwd = getcwd();
> + chdir $dir or die "Unable to chdir '$dir': $!";
> +
> + my @mods;
> + {
> + # Outputs: "direct_dep ephemeral_dep"
> + local $ENV{GOPATH} = "$dir/go";
> + open my $fh, '-|', qw< go mod graph >
> + or die "Unable to spawn 'go mod path': $!";
> + @mods = readline $fh;
> + close $fh
> + or die "Error closing pipe to 'go mod path': $!";
> + }
> +
> + chdir $old_cwd or die "Unable to chdir '$old_cwd': $!";
> +
> + chomp @mods;
> +
> + # parse the graph into pairs of hashrefs
> + return map { [
> + map {
> + my ($m, $v) = split /@/;
> + { Module => $m, Version => $v };
> + } split /\s/
> + ] } grep { $_ } @mods;
> +}
> +
> +sub get_ver_info
> +{
> + my ( $self, $module ) = @_;
> + my $version_list = $self->get( $module . '/@v/list' );
> + my $version = "v0.0.0";
> + my $ret;
> +
> + # If list isn't populated, it's likely that upstream doesn't follow
> + # semver, this means we will have to fallback to @latest to get the
> + # version information.
> + if ($version_list eq "") {
> + $ret = $self->get_json( $module . '/@latest' );
> + } else {
> + my @parts = split("\n", $version_list);
> + for my $v ( @parts ) {
> + my $a =
> OpenBSD::PackageName::version->from_string($version);
> + my $b = OpenBSD::PackageName::version->from_string($v);
> + if ($a->compare($b)) {
> + $version = $v;
> + }
> + }
> + $ret = { Module => $module, Version => $version };
> + }
> +
> + return $ret;
> +}
> +
> +sub name_new_port
> +{
> + my ( $self, $di ) = @_;
> +
> + my $name = $di->{Name};
> + $name = $self->SUPER::name_new_port($name);
> + $name = "go/$name" unless $name =~ m{/};
> +
> + return $name;
> +}
> +
> +sub fill_in_makefile
> +{
> + my ( $self, $di, $vi ) = @_;
> +
> + $self->set_modules('lang/go');
> + $self->set_comment("todo");
> + $self->set_descr("TODO");
> +
> + $self->set_license($di->{License});
> +
> + $self->set_other( MODGO_MODNAME => $di->{Module} );
> + $self->set_other( MODGO_VERSION => $di->{Version} );
> + $self->set_distname($di->{Name} . '-${MODGO_VERSION}');
> +
> + $self->set_pkgname($di->{PkgName}) if defined $di->{PkgName};
> +
> + my @parts = split("-", $di->{Version});
> + if (@parts > 1) {
> + $self->set_pkgname($di->{Name} . "-" . $parts[1])
> + if $parts[1] =~ m/\d{6}/;
> + } else {
> + $parts[0] =~ s/^v//;
> + $self->set_pkgname($di->{Name} . "-" . $parts[0]);
> + }
> +
> + my @dist = map { [ $_->{Module}, $_->{Version} ] }
> + @{ $di->{Dist} || [] };
> + my @mods = map { [ $_->{Module}, $_->{Version} ] }
> + @{ $di->{Mods} };
> +
> + # Turn the deps into tab separated columns
> + foreach my $s ( \@dist, \@mods ) {
> + next unless @{$s}; # if there aren't any, don't try
> + my ($length) = sort { $b <=> $a } map { length $_->[0] } @$s;
> + my $n = ( 1 + int $length / 8 );
> + @{$s} = map {
> + ( my $l = $_->[0] ) =~ s/\p{Upper}/!\L$&/g;
> + my $tabs = "\t" x ( $n - int( length($l) / 8 ) );
> + "$l$tabs $_->[1]"
> + } @{$s};
> + }
> +
> + $self->set_other( MODGO_MODULES => \@dist ) if @dist;
> + $self->set_other( MODGO_MODFILES => \@mods ) if @mods;
> +}
> +
> +sub try_building
> +{
> + my $self = shift;
> + $self->make_fake();
> +}
> +
> +sub postextract
> +{
> +}
> +
> +sub get_deps
> +{
> + my ( $self, $di, $wrksrc ) = @_;
> + my $deps = OpenBSD::PortGen::Dependency->new();
> +
> + return $deps->format;
> +}
> +
> +sub get_config_style
> +{
> +}
> +
> +1;
No idea on the Makefile stuff :-)
> diff --git a/infrastructure/templates/Makefile.template
> b/infrastructure/templates/Makefile.template
> index 6be8b86b3ea..2ebf2d37880 100644
> --- a/infrastructure/templates/Makefile.template
> +++ b/infrastructure/templates/Makefile.template
> @@ -22,7 +22,13 @@ COMMENT = ???
> #
> #MODPY_EGG_VERSION = ???
>
> +# MODGO_MODNAME should be set to the 'module' specified in the 'go.mod' file.
> +#MODGO_MODNAME = github.com/test/app
> #
> +# Version of port if using lang/go and MODGO_MODULES
> +#
> +#MODGO_VERSION = 0.1.1
> +
> # What port/package will be created
> #
> # DISTNAME should not include suffix (like .tar.gz .tgz .tar.bz2 etc.)
> @@ -125,6 +131,15 @@ MASTER_SITES = ???
> # If port is python3 only
> #MODPY_VERSION = ${MODPY_DEFAULT_VERSION_3}
>
> +#
> +# MODGO_ settings for when using lang/go module
> +#
> +# Get source from proxy.golang.org
> +#MODGO_MODULES = modulename version
> +# These are needed for dependency resolution. We don't actually need the
> +# coresponding code
> +#MODGO_MODFILES = modulename version
> +
> # Dependencies
> #BUILD_DEPENDS = ???
> #RUN_DEPENDS = ???
> diff --git a/lang/go/go.port.mk b/lang/go/go.port.mk
> index 983c3990706..e6b135ddb1f 100644
> --- a/lang/go/go.port.mk
> +++ b/lang/go/go.port.mk
> @@ -4,6 +4,13 @@ ONLY_FOR_ARCHS ?= ${GO_ARCHS}
>
> MODGO_BUILDDEP ?= Yes
>
> +MODGO_DIST_SUBDIR ?= go_modules
> +
> +MASTER_SITE_ATHENS = https://proxy.golang.org/
> +
> +MODGO_MASTER_SITESN = 9
> +MASTER_SITES${MODGO_MASTER_SITESN} ?= ${MASTER_SITE_ATHENS}
> +
> MODGO_RUN_DEPENDS = lang/go
> MODGO_BUILD_DEPENDS = lang/go
>
> @@ -33,17 +40,12 @@ MODGO_TYPE ?= bin
> MODGO_WORKSPACE ?= ${WRKDIR}/go
> MODGO_GOCACHE ?= ${WRKDIR}/go-cache
> MODGO_GOPATH ?= ${MODGO_WORKSPACE}:${MODGO_PACKAGE_PATH}
> -MAKE_ENV += GOCACHE="${MODGO_GOCACHE}" \
> - GOPATH="${MODGO_GOPATH}" \
> - GO111MODULE=off
> # We cannot assume that the maching running the built code will have SSE,
> # even though the machine building the package has SSE. As such, we need
> # to explicitly disable SSE on i386 builds.
> MAKE_ENV += GO386=387
> -# Ports are not allowed to fetch from the network at build time; point
> -# GOPROXY at an unreachable host so that failures are also visible to
> -# developers who don't have PORTS_PRIVSEP and a "deny .. _pbuild" PF rule.
> -MAKE_ENV += GOPROXY=invalid://ports.should.not.fetch.at.buildtime/
> +MAKE_ENV += GOCACHE="${MODGO_GOCACHE}"
> +
> MODGO_CMD ?= ${SETENV} ${MAKE_ENV} go
> MODGO_BUILD_CMD = ${MODGO_CMD} install ${MODGO_FLAGS}
> MODGO_TEST_CMD = ${MODGO_CMD} test ${MODGO_FLAGS} ${MODGO_TEST_FLAGS}
> @@ -54,19 +56,38 @@ MODGO_BUILD_CMD += -ldflags="${MODGO_LDFLAGS}"
> MODGO_TEST_CMD += -ldflags="${MODGO_LDFLAGS}"
> .endif
>
> -.if defined(GH_ACCOUNT) && defined(GH_PROJECT)
> -ALL_TARGET ?= github.com/${GH_ACCOUNT}/${GH_PROJECT}
> +.if defined(MODGO_MODNAME)
> +EXTRACT_SUFX ?= .zip
> +PKGNAME ?= ${DISTNAME:S/-v/-/}
> +ALL_TARGET ?= ${MODGO_MODNAME}
> +DISTFILES =
> ${DISTNAME}${EXTRACT_SUFX}{${MODGO_VERSION}${EXTRACT_SUFX}}
> +MASTER_SITES ?= ${MASTER_SITE_ATHENS}${MODGO_MODNAME}/@v/
> +. for _modpath _modver in ${MODGO_MODULES}
> +SUPDISTFILES +=
> ${MODGO_DIST_SUBDIR}/${_modpath}/@v/${_modver}.zip{${_modpath}/@v/${_modver}.zip}:${MODGO_MASTER_SITESN}
> +. endfor
> +. for _modpath _modver in ${MODGO_MODFILES}
> +SUPDISTFILES +=
> ${MODGO_DIST_SUBDIR}/${_modpath}/@v/${_modver}.mod{${_modpath}/@v/${_modver}.mod}:${MODGO_MASTER_SITESN}
> +. endfor
> +MAKE_ENV += GOPROXY=file://${DISTDIR}/${MODGO_DIST_SUBDIR}
> +MAKE_ENV += GO111MODULE=on GOPATH="${MODGO_GOPATH}"
> +.else
> +# ports are not allowed to fetch from the network at build time; point
> +# GOPROXY at an unreachable host so that failures are also visible to
> +# developers who don't have PORTS_PRIVSEP and a "deny .. _pbuild" PF rule.
> +MAKE_ENV += GOPROXY=invalid://ports.should.not.fetch.at.buildtime/
> +MAKE_ENV += GO111MODULE=off GOPATH="${MODGO_GOPATH}"
> +. if defined(GH_ACCOUNT) && defined(GH_PROJECT)
> +ALL_TARGET ?= github.com/${GH_ACCOUNT}/${GH_PROJECT}
> +. endif
> .endif
> -TEST_TARGET ?= ${ALL_TARGET}
>
> -SEPARATE_BUILD ?= Yes
> -WRKSRC ?= ${MODGO_WORKSPACE}/src/${ALL_TARGET}
> +MODGO_TEST_TARGET ?= cd ${WRKSRC} && ${MODGO_CMD} test ${ALL_TARGET}
>
> -MODGO_SETUP_WORKSPACE = mkdir -p ${WRKSRC:H}; mv ${MODGO_SUBDIR}
> ${WRKSRC};
> +SEPARATE_BUILD ?= Yes
>
> CATEGORIES += lang/go
>
> -MODGO_BUILD_TARGET = ${MODGO_BUILD_CMD} ${ALL_TARGET}
> +#MODGO_BUILD_TARGET = ${MODGO_BUILD_CMD}
> MODGO_FLAGS += -v -p ${MAKE_JOBS}
>
> .if empty(DEBUG)
> @@ -76,6 +97,14 @@ MODGO_LDFLAGS += -s -w
> MODGO_FLAGS += -x
> .endif
>
> +.if empty(MODGO_MODNAME)
> +WRKSRC ?= ${MODGO_WORKSPACE}/src/${ALL_TARGET}
> +MODGO_SETUP_WORKSPACE = mkdir -p ${WRKSRC:H}; mv ${MODGO_SUBDIR}
> ${WRKSRC};
> +.else
> +WRKSRC ?= ${WRKDIR}/${MODGO_MODNAME}@${MODGO_VERSION}
> +MODGO_SETUP_WORKSPACE = ln -sf ${WRKSRC} ${WRKDIR}/${MODGO_MODNAME}
> +.endif
> +
> INSTALL_STRIP =
> .if ${MODGO_TYPE:L:Mbin}
> MODGO_INSTALL_TARGET = ${INSTALL_PROGRAM_DIR}
> ${PREFIX}/${MODGO_BINDIR} && \
> @@ -99,14 +128,18 @@ MODGO_INSTALL_TARGET += ${INSTALL_DATA_DIR}
> ${MODGO_PACKAGE_PATH} && \
> RUN_DEPENDS += ${MODGO_RUN_DEPENDS}
> .endif
>
> -MODGO_TEST_TARGET = ${MODGO_TEST_CMD} ${TEST_TARGET}
> -
> .if empty(CONFIGURE_STYLE)
> MODGO_pre-configure += ${MODGO_SETUP_WORKSPACE}
>
> -. if !target(do-build)
> +. if !target(do-install)
> do-build:
> - ${MODGO_BUILD_TARGET}
> + if [ -d ${WRKSRC}/cmd ]; then \
> + cd ${WRKSRC} && \
> + ${MODGO_BUILD_CMD} ${ALL_TARGET}/cmd/... ; \
> + else \
> + cd ${WRKSRC} && \
> + ${MODGO_BUILD_CMD} ${ALL_TARGET}; \
> + fi;
> . endif
>
> . if !target(do-install)
>
--
andrew - http://afresh1.com
Hey! It compiles! Ship it!
