Umoci unpack fix

CONTRIBUTORS.md

@@ -12,6 +12,7 @@
 # Contributors:
 
     - Adam Hughes <[email protected]>, <[email protected]>
+    - Adam Simpson <[email protected]>, <[email protected]>
     - Afif Elghraoui <[email protected]>
     - Amanda Duffy <[email protected]>
     - Ángel Bejarano <[email protected]>

go.mod

@@ -7,6 +7,7 @@ require (
 	github.com/Microsoft/hcsshim v0.8.6 // indirect
 	github.com/Netflix/go-expect v0.0.0-20180928190340-9d1f4485533b
 	github.com/alexflint/go-filemutex v0.0.0-20171028004239-d358565f3c3f // indirect
+	github.com/apex/log v1.1.1 // indirect
 	github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 // indirect
 	github.com/bitly/go-simplejson v0.5.0 // indirect
 	github.com/blang/semver v3.5.1+incompatible
@@ -24,6 +25,7 @@ require (
 	github.com/containers/storage v0.0.0-20180604200230-88d80428f9b1 // indirect
 	github.com/coreos/go-iptables v0.3.0 // indirect
 	github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7 // indirect
+	github.com/cyphar/filepath-securejoin v0.2.2 // indirect
 	github.com/d2g/dhcp4 v0.0.0-20170904100407-a1d1b6c41b1c // indirect
 	github.com/d2g/dhcp4client v0.0.0-20180611075603-e61299896203 // indirect
 	github.com/d2g/dhcp4server v0.0.0-20181031114812-7d4a0a7f59a5 // indirect
@@ -56,21 +58,21 @@ require (
 	github.com/juju/loggo v0.0.0-20190526231331-6e530bcce5d8 // indirect
 	github.com/juju/testing v0.0.0-20190613124551-e81189438503 // indirect
 	github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect
+	github.com/klauspost/pgzip v1.2.1 // indirect
 	github.com/kr/pty v1.1.8
 	github.com/kubernetes-sigs/cri-o v0.0.0-20180917213123-8afc34092907
-	github.com/mattn/go-colorable v0.1.1 // indirect
 	github.com/mattn/go-runewidth v0.0.2 // indirect
 	github.com/mattn/go-shellwords v1.0.3 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
 	github.com/mtrmac/gpgme v0.0.0-20170102180018-b2432428689c // indirect
 	github.com/onsi/ginkgo v1.8.0 // indirect
-	github.com/onsi/gomega v1.5.0 // indirect
+	github.com/openSUSE/umoci v0.4.2
 	github.com/opencontainers/go-digest v1.0.0-rc1
 	github.com/opencontainers/image-spec v0.0.0-20180411145040-e562b0440392
 	github.com/opencontainers/image-tools v0.0.0-20180129025323-c95f76cbae74
 	github.com/opencontainers/runc v0.1.1 // indirect
 	github.com/opencontainers/runtime-spec v0.0.0-20180913141938-5806c3563733
-	github.com/opencontainers/runtime-tools v0.6.0
+	github.com/opencontainers/runtime-tools v0.7.0
 	github.com/opencontainers/selinux v1.3.0
 	github.com/pelletier/go-toml v1.4.0
 	github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 // indirect
@@ -80,6 +82,7 @@ require (
 	github.com/prometheus/client_model v0.0.0-20171117100541-99fa1f4be8e5 // indirect
 	github.com/prometheus/common v0.0.0-20180518154759-7600349dcfe1 // indirect
 	github.com/prometheus/procfs v0.0.0-20180612222113-7d6f385de8be // indirect
+	github.com/rootless-containers/proto v0.1.0 // indirect
 	github.com/safchain/ethtool v0.0.0-20180504150752-6e3f4faa84e1 // indirect
 	github.com/satori/go.uuid v1.2.0
 	github.com/seccomp/libseccomp-golang v0.9.1
@@ -93,6 +96,8 @@ require (
 	github.com/sylabs/scs-library-client v0.3.0
 	github.com/sylabs/sif v1.0.7
 	github.com/syndtr/gocapability v0.0.0-20180223013746-33e07d32887e // indirect
+	github.com/urfave/cli v1.21.0 // indirect
+	github.com/vbatts/go-mtree v0.4.4 // indirect
 	github.com/vishvananda/netlink v1.0.1-0.20190618143317-99a56c251ae6 // indirect
 	github.com/vishvananda/netns v0.0.0-20171111001504-be1fbeda1936 // indirect
 	github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect
@@ -103,7 +108,7 @@ require (
 	github.com/yvasiyarov/gorelic v0.0.6 // indirect
 	github.com/yvasiyarov/newrelic_platform_go v0.0.0-20160601141957-9c099fbc30e9 // indirect
 	go4.org v0.0.0-20180417224846-9599cf28b011 // indirect
-	golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2
+	golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734
 	golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223
 	golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 // indirect
 	google.golang.org/grpc v1.20.0 // indirect

internal/pkg/build/sources/conveyorPacker_oci.go

@@ -28,7 +28,6 @@ import (
 	"github.com/containers/image/signature"
 	"github.com/containers/image/types"
 	imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1"
-	imagetools "github.com/opencontainers/image-tools/image"
 	ociclient "github.com/sylabs/singularity/internal/pkg/client/oci"
 	"github.com/sylabs/singularity/internal/pkg/sylog"
 	"github.com/sylabs/singularity/internal/pkg/util/shell"
@@ -290,10 +289,8 @@ func (cp *OCIConveyorPacker) extractArchive(src string, dst string) error {
 	}
 }
 
-func (cp *OCIConveyorPacker) unpackTmpfs() (err error) {
-	refs := []string{"name=tmp"}
-	err = imagetools.UnpackLayout(cp.b.Path, cp.b.Rootfs(), "amd64", refs)
-	return err
+func (cp *OCIConveyorPacker) unpackTmpfs() error {
+	return unpackRootfs(cp.b, cp.tmpfsRef, cp.sysCtx)
 }
 
 func (cp *OCIConveyorPacker) insertBaseEnv() (err error) {

internal/pkg/build/sources/oci_unpack_darwin.go

@@ -0,0 +1,23 @@
+// Copyright (c) 2019, Sylabs Inc. All rights reserved.
+// This software is licensed under a 3-clause BSD license. Please consult the
+// LICENSE.md file distributed with the sources of this project regarding your
+// rights to use or distribute this software.
+
+// TODO(ian): The build package should be refactored to make each conveyorpacker
+// its own separate package. With that change, this file should be grouped with the
+// OCIConveyorPacker code
+
+package sources
+
+import (
+	"github.com/containers/image/types"
+	imagetools "github.com/opencontainers/image-tools/image"
+	sytypes "github.com/sylabs/singularity/pkg/build/types"
+)
+
+// unpackRootfs extracts all of the layers of the given image reference into the rootfs of the provided bundle
+func unpackRootfs(b *sytypes.Bundle, _ types.ImageReference, _ *types.SystemContext) (err error) {
+	refs := []string{"name=tmp"}
+	err = imagetools.UnpackLayout(b.Path, b.Rootfs(), "amd64", refs)
+	return err
+}

internal/pkg/build/sources/oci_unpack_linux.go

@@ -0,0 +1,72 @@
+// Copyright (c) 2019, Sylabs Inc. All rights reserved.
+// This software is licensed under a 3-clause BSD license. Please consult the
+// LICENSE.md file distributed with the sources of this project regarding your
+// rights to use or distribute this software.
+
+// TODO(ian): The build package should be refactored to make each conveyorpacker
+// its own separate package. With that change, this file should be grouped with the
+// OCIConveyorPacker code
+
+package sources
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+
+	"github.com/containers/image/types"
+	"github.com/openSUSE/umoci"
+	umocilayer "github.com/openSUSE/umoci/oci/layer"
+	"github.com/openSUSE/umoci/pkg/idtools"
+	imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1"
+	sytypes "github.com/sylabs/singularity/pkg/build/types"
+)
+
+// unpackRootfs extracts all of the layers of the given image reference into the rootfs of the provided bundle
+func unpackRootfs(b *sytypes.Bundle, tmpfsRef types.ImageReference, sysCtx *types.SystemContext) (err error) {
+	var mapOptions umocilayer.MapOptions
+
+	// Allow unpacking as non-root
+	if os.Geteuid() != 0 {
+		mapOptions.Rootless = true
+
+		uidMap, err := idtools.ParseMapping(fmt.Sprintf("0:%d:1", os.Geteuid()))
+		if err != nil {
+			return fmt.Errorf("error parsing uidmap: %s", err)
+		}
+		mapOptions.UIDMappings = append(mapOptions.UIDMappings, uidMap)
+
+		gidMap, err := idtools.ParseMapping(fmt.Sprintf("0:%d:1", os.Getegid()))
+		if err != nil {
+			return fmt.Errorf("error parsing gidmap: %s", err)
+		}
+		mapOptions.GIDMappings = append(mapOptions.GIDMappings, gidMap)
+	}
+
+	engineExt, err := umoci.OpenLayout(b.Path)
+	if err != nil {
+		return fmt.Errorf("error opening layout: %s", err)
+	}
+
+	// Obtain the manifest
+	imageSource, err := tmpfsRef.NewImageSource(context.Background(), sysCtx)
+	if err != nil {
+		return fmt.Errorf("error creating image source: %s", err)
+	}
+	manifestData, mediaType, err := imageSource.GetManifest(context.Background(), nil)
+	if err != nil {
+		return fmt.Errorf("error obtaining manifest source: %s", err)
+	}
+	if mediaType != imgspecv1.MediaTypeImageManifest {
+		return fmt.Errorf("error verifying manifest media type: %s", mediaType)
+	}
+	var manifest imgspecv1.Manifest
+	json.Unmarshal(manifestData, &manifest)
+
+	// UnpackRootfs from umoci v0.4.2 expects a path to a non-existing directory
+	os.RemoveAll(b.Rootfs())
+
+	// Unpack root filesystem
+	return umocilayer.UnpackRootfs(context.Background(), engineExt, b.Rootfs(), manifest, &mapOptions)
+}