From 4bd71828b1c9340062b07547863d098e760ec0a4 Mon Sep 17 00:00:00 2001 From: Roi Martin Date: Thu, 23 Nov 2023 20:13:11 +0100 Subject: [PATCH] internal/urlutil: support downloading OCI artifacts --- go.mod | 17 +-- go.sum | 46 ++++--- internal/urlutil/urlutil.go | 91 ++++++++++++-- internal/urlutil/urlutil_test.go | 209 ++++++++++++++++++++++++++----- 4 files changed, 291 insertions(+), 72 deletions(-) diff --git a/go.mod b/go.mod index 2375231..0c8f0e2 100644 --- a/go.mod +++ b/go.mod @@ -6,17 +6,20 @@ require ( github.com/adevinta/vulcan-agent v1.2.0 github.com/adevinta/vulcan-check-catalog v0.0.0-20230511151135-4f1b3329ba4c github.com/adevinta/vulcan-report v1.0.0 - github.com/adevinta/vulcan-types v1.2.5 + github.com/adevinta/vulcan-types v1.2.7 github.com/docker/cli v24.0.7+incompatible github.com/docker/docker v24.0.7+incompatible github.com/docker/go-connections v0.4.0 github.com/fatih/color v1.16.0 github.com/google/go-cmp v0.6.0 + github.com/google/go-containerregistry v0.16.1 github.com/google/uuid v1.4.0 github.com/jroimartin/clilog v0.1.1 github.com/jroimartin/proxy v0.4.3 + github.com/opencontainers/image-spec v1.1.0-rc5 golang.org/x/mod v0.14.0 gopkg.in/yaml.v3 v3.0.1 + oras.land/oras-go/v2 v2.3.1 ) require ( @@ -26,7 +29,7 @@ require ( github.com/Microsoft/go-winio v0.6.1 // indirect github.com/Microsoft/hcsshim v0.11.0 // indirect github.com/adevinta/vulcan-metrics-client v1.0.1 // indirect - github.com/aws/aws-sdk-go v1.47.0 // indirect + github.com/aws/aws-sdk-go v1.48.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/containerd/containerd v1.7.5 // indirect @@ -45,18 +48,18 @@ require ( github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/julienschmidt/httprouter v1.3.0 // indirect github.com/klauspost/compress v1.16.7 // indirect + github.com/kr/pretty v0.2.1 // indirect github.com/lestrrat-go/backoff v1.0.1 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect - github.com/miekg/dns v1.1.56 // indirect + github.com/miekg/dns v1.1.57 // indirect github.com/miekg/pkcs11 v1.1.1 // indirect github.com/moby/patternmatcher v0.6.0 // indirect github.com/moby/sys/sequential v0.5.0 // indirect - github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect + github.com/moby/term v0.0.0-20221205130635-1aeaba878587 // indirect github.com/morikuni/aec v1.0.0 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect - github.com/opencontainers/image-spec v1.1.0-rc2.0.20221005185240-3a7f492d3f1b // indirect github.com/opencontainers/runc v1.1.9 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.14.0 // indirect @@ -69,9 +72,9 @@ require ( github.com/theupdateframework/notary v0.7.0 // indirect golang.org/x/crypto v0.14.0 // indirect golang.org/x/net v0.17.0 // indirect + golang.org/x/sync v0.4.0 // indirect golang.org/x/sys v0.14.0 // indirect golang.org/x/term v0.13.0 // indirect golang.org/x/tools v0.13.0 // indirect - google.golang.org/protobuf v1.29.1 // indirect - gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect + google.golang.org/protobuf v1.30.0 // indirect ) diff --git a/go.sum b/go.sum index e540fce..872af86 100644 --- a/go.sum +++ b/go.sum @@ -33,7 +33,6 @@ cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 h1:bvDV9vkmnHYOMsOr4WLk+Vo07yKIzd94sVoIqshQ4bU= github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= -github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= @@ -56,15 +55,15 @@ github.com/adevinta/vulcan-metrics-client v1.0.1 h1:BAugnnRWvkA3vnuCX77W04PWhneZ github.com/adevinta/vulcan-metrics-client v1.0.1/go.mod h1:we8vxfPMYQqZtOy42PJxsWwv2DwruSaT/wwNMxkum8I= github.com/adevinta/vulcan-report v1.0.0 h1:44aICPZ+4svucgCSA5KmjlT3ZGzrvZXiSnkbnj6AC2k= github.com/adevinta/vulcan-report v1.0.0/go.mod h1:k34KaeoXc3H77WNMwI9F4F1G28hBjB95PeMUp9oHbEE= -github.com/adevinta/vulcan-types v1.2.5 h1:gYJI1OsQ+MQAX1tXBH3vMrBuDNmwMlAlRGX0zdXph+A= -github.com/adevinta/vulcan-types v1.2.5/go.mod h1:/QZcpEwdRahgwVMXVnP2Zi0z95JuOKApZcPSAi2vKoI= +github.com/adevinta/vulcan-types v1.2.7 h1:fWctG6zPtxb5/Y+JPJ3GuwyRtVClzVUB7C8Gp0Bz3nw= +github.com/adevinta/vulcan-types v1.2.7/go.mod h1:uyadEXIS7SfnCgxMSEV2GgHchmtXGov+DqOQjbQ0xqk= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= -github.com/aws/aws-sdk-go v1.47.0 h1:/JUg9V1+xh+qBn8A6ec/l15ETPaMaBqxkjz+gg63dNk= -github.com/aws/aws-sdk-go v1.47.0/go.mod h1:DlEaEbWKZmsITVbqlSVvekPARM1HzeV9PMYg15ymSDA= +github.com/aws/aws-sdk-go v1.48.0 h1:1SeJ8agckRDQvnSCt1dGZYAwUaoD2Ixj6IaXB4LCv8Q= +github.com/aws/aws-sdk-go v1.48.0/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= github.com/beorn7/perks v0.0.0-20150223135152-b965b613227f/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= @@ -93,10 +92,12 @@ github.com/cloudflare/cfssl v0.0.0-20180223231731-4e2dcbde5004/go.mod h1:yMWuSON github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/containerd/containerd v1.7.5 h1:i9T9XpAWMe11BHMN7pu1BZqOGjXaKTPyz2v+KYOZgkY= github.com/containerd/containerd v1.7.5/go.mod h1:ieJNCSzASw2shSGYLHx8NAE7WsZ/gEigo5fQ78W5Zvw= +github.com/containerd/stargz-snapshotter/estargz v0.14.3 h1:OqlDCK3ZVUO6C3B/5FSkDwbkEETK84kQgEeFwDC+62k= +github.com/containerd/stargz-snapshotter/estargz v0.14.3/go.mod h1:KY//uOCIkSuNAHhJogcZtrNHdKrA99/FCCRjE3HD36o= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw= -github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= +github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -197,6 +198,8 @@ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-containerregistry v0.16.1 h1:rUEt426sR6nyrL3gt+18ibRcvYpKYdpsa5ZW7MA08dQ= +github.com/google/go-containerregistry v0.16.1/go.mod h1:u0qB2l7mvtWVR5kNcbFIhFY1hLbf8eeGapA+vbFDCtQ= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -280,19 +283,21 @@ github.com/mattn/go-sqlite3 v1.6.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOq github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= -github.com/miekg/dns v1.1.56 h1:5imZaSeoRNvpM9SzWNhEcP9QliKiz20/dA2QabIGVnE= -github.com/miekg/dns v1.1.56/go.mod h1:cRm6Oo2C8TY9ZS/TqsSrseAcncm74lfK5G+ikN2SWWY= +github.com/miekg/dns v1.1.57 h1:Jzi7ApEIzwEPLHWRcafCN9LZSBbqQpxjt/wpgvg7wcM= +github.com/miekg/dns v1.1.57/go.mod h1:uqRjCRUuEAA6qsOiJvDd+CFo/vW+y5WR6SNmHE55hZk= github.com/miekg/pkcs11 v1.0.2/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= github.com/miekg/pkcs11 v1.1.1 h1:Ugu9pdy6vAYku5DEpVWVFPYnzV+bxB+iRdbuFSu7TvU= github.com/miekg/pkcs11 v1.1.1/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v0.0.0-20150613213606-2caf8efc9366 h1:1ypTpKUfEOyX1YsJru6lLq7hrmK+QGECpJQ1PHUHuGo= github.com/mitchellh/mapstructure v0.0.0-20150613213606-2caf8efc9366/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk= github.com/moby/patternmatcher v0.6.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= github.com/moby/sys/sequential v0.5.0 h1:OPvI35Lzn9K04PBbCLW0g4LcFAJgHsvXsRyewg5lXtc= github.com/moby/sys/sequential v0.5.0/go.mod h1:tH2cOOs5V9MlPiXcQzRC+eEyab644PWKGRYaaV5ZZlo= -github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 h1:dcztxKSvZ4Id8iPpHERQBbIJfabdt4wUm5qy3wOL2Zc= -github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6/go.mod h1:E2VnQOmVuvZB6UYnnDB0qG5Nq/1tD9acaOpo6xmt0Kw= +github.com/moby/term v0.0.0-20221205130635-1aeaba878587 h1:HfkjXDfhgVaN5rmueG8cL8KKeFNecRCXFhaJ2qZ5SKA= +github.com/moby/term v0.0.0-20221205130635-1aeaba878587/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= @@ -311,8 +316,8 @@ github.com/opencontainers/go-digest v0.0.0-20170106003457-a6d0ee40d420/go.mod h1 github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= -github.com/opencontainers/image-spec v1.1.0-rc2.0.20221005185240-3a7f492d3f1b h1:YWuSjZCQAPM8UUBLkYUk1e+rZcvWHJmFb6i6rM44Xs8= -github.com/opencontainers/image-spec v1.1.0-rc2.0.20221005185240-3a7f492d3f1b/go.mod h1:3OVijpioIKYWTqjiG0zfF6wvoJ4fAXGbjdZuI2NgsRQ= +github.com/opencontainers/image-spec v1.1.0-rc5 h1:Ygwkfw9bpDvs+c9E34SdgGOj41dX/cbdlwvlWt0pnFI= +github.com/opencontainers/image-spec v1.1.0-rc5/go.mod h1:X4pATf0uXsnn3g5aiGIsVnJBR4mxhKzfwmvK/B2NTm8= github.com/opencontainers/runc v1.1.9 h1:XR0VIHTGce5eWPkaPesqTBrhW2yAcaraWfsEalNwQLM= github.com/opencontainers/runc v1.1.9/go.mod h1:CbUumNnWCuTGFukNXahoo/RFBZvDAgRh/smNYNOhA50= github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= @@ -373,7 +378,6 @@ github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRM github.com/spf13/jwalterweatherman v0.0.0-20141219030609-3d60171a6431 h1:XTHrT015sxHyJ5FnQ0AeemSspZWaDq7DoTRW0EVsDCE= github.com/spf13/jwalterweatherman v0.0.0-20141219030609-3d60171a6431/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v1.0.0/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v0.0.0-20150530192845-be5ff3e4840c h1:2EejZtjFjKJGk71ANb+wtFK5EjUzUkEM3R0xnp559xg= @@ -393,6 +397,8 @@ github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/theupdateframework/notary v0.7.0 h1:QyagRZ7wlSpjT5N2qQAh/pN+DVqgekv4DzbAiAiEL3c= github.com/theupdateframework/notary v0.7.0/go.mod h1:c9DRxcmhHmVLDay4/2fUYdISnHqbFDGRSlXPO0AhYWw= +github.com/vbatts/tar-split v0.11.3 h1:hLFqsOLQ1SsppQNTMpkpPXClLDfC2A3Zgy9OUU+RVck= +github.com/vbatts/tar-split v0.11.3/go.mod h1:9QlHN18E+fEH7RdG+QAJJcuya3rqT7eXSTY7wGrAokY= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -497,8 +503,8 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= -golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ= +golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -576,7 +582,6 @@ golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBn golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -690,8 +695,8 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.29.1 h1:7QBf+IK2gx70Ap/hDsOmam3GE0v9HicjfEdAxE62UoM= -google.golang.org/protobuf v1.29.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= +google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/cenkalti/backoff.v2 v2.2.1 h1:eJ9UAg01/HIHG987TwxvnzK2MgxXq97YY6rYDpY9aII= @@ -719,7 +724,6 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= gotest.tools/v3 v3.2.0 h1:I0DwBVMGAx26dttAj1BtJLAkVGncrkkUXfJLC4Flt/I= gotest.tools/v3 v3.2.0/go.mod h1:Mcr9QNxkg0uMvy/YElmo4SpXgJKWgQvYrT7Kw5RzJ1A= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -729,6 +733,8 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +oras.land/oras-go/v2 v2.3.1 h1:lUC6q8RkeRReANEERLfH86iwGn55lbSWP20egdFHVec= +oras.land/oras-go/v2 v2.3.1/go.mod h1:5AQXVEu1X/FKp1F9DMOb5ZItZBOa0y5dha0yCm4NR9c= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/internal/urlutil/urlutil.go b/internal/urlutil/urlutil.go index 472047e..a4254df 100644 --- a/internal/urlutil/urlutil.go +++ b/internal/urlutil/urlutil.go @@ -4,12 +4,20 @@ package urlutil import ( + "context" "errors" "fmt" "io" "net/http" "net/url" "os" + "strings" + + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "oras.land/oras-go/v2" + "oras.land/oras-go/v2/content" + "oras.land/oras-go/v2/content/memory" + "oras.land/oras-go/v2/registry/remote" ) var ( @@ -26,35 +34,92 @@ var ( // if the URL is not valid or if it is not possible to get the // contents. // -// It supports the following schemes: http, https. If the provided URL -// does not specify a scheme, it is considered a file path. In the -// case of http and https, the contents are retrieved issuing an HTTP -// GET request. +// It supports the following schemes: http, https and oci. If the +// provided URL does not specify a scheme, it is considered a file +// path. In the case of http and https, the contents are retrieved +// issuing an HTTP GET request. In the case of oci, an OCI artifact is +// retrieved. func Get(rawURL string) ([]byte, error) { - parsedURL, err := url.Parse(rawURL) + u, err := url.Parse(rawURL) if err != nil { return nil, fmt.Errorf("%w: %w", ErrInvalidURL, err) } - switch parsedURL.Scheme { + switch u.Scheme { case "http", "https": - return getHTTP(parsedURL) + return getHTTP(u) + case "oci": + return getOCI(u) case "": - return os.ReadFile(parsedURL.Path) + return os.ReadFile(u.Path) } - return nil, fmt.Errorf("%w: %v", ErrInvalidScheme, parsedURL.Scheme) + return nil, fmt.Errorf("%w: %v", ErrInvalidScheme, u.Scheme) } // getHTTP retrieves the contents of a given HTTP URL. -func getHTTP(parsedURL *url.URL) ([]byte, error) { - resp, err := http.Get(parsedURL.String()) +func getHTTP(u *url.URL) ([]byte, error) { + resp, err := http.Get(u.String()) if err != nil { - return nil, fmt.Errorf("get %q: %w", parsedURL, err) + return nil, fmt.Errorf("get %q: %w", u, err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("get %q: %w", parsedURL, err) + return nil, fmt.Errorf("get %q: %w", u, err) } return io.ReadAll(resp.Body) } + +// ociPlainHTTP is set by the tests to force plain HTTP connections +// when retrieving OCI artifacts. +var ociPlainHTTP bool + +// getOCI retrieves the contents of a given OCI artifact. It expects +// one single file. +func getOCI(u *url.URL) ([]byte, error) { + if u.Host == "" { + return nil, fmt.Errorf("%w: empty host", ErrInvalidURL) + } + + parts := strings.Split(u.Path, ":") + if len(parts) != 2 || parts[0] == "" || parts[1] == "" { + return nil, fmt.Errorf("%w: malformed path: %q", ErrInvalidURL, u.Path) + } + repo := u.Host + parts[0] + tag := parts[1] + + src, err := remote.NewRepository(repo) + if err != nil { + return nil, fmt.Errorf("%w: new repository: %w", ErrInvalidURL, err) + } + src.PlainHTTP = ociPlainHTTP + + dst := memory.New() + + desc, err := oras.Copy(context.Background(), src, tag, dst, tag, oras.DefaultCopyOptions) + if err != nil { + return nil, fmt.Errorf("artifact copy: %w", err) + } + + successors, err := content.Successors(context.Background(), dst, desc) + if err != nil { + return nil, fmt.Errorf("artifact successors: %w", err) + } + + for _, s := range successors { + // The title annotation is used to store the file + // name. We are only interested in regular files, so + // we can skip successors with no title. + if s.Annotations[ocispec.AnnotationTitle] == "" { + continue + } + + data, err := content.FetchAll(context.Background(), dst, s) + if err != nil { + return nil, fmt.Errorf("artifact fetch: %w", err) + } + return data, nil + } + + return nil, errors.New("malformed artifact") +} diff --git a/internal/urlutil/urlutil_test.go b/internal/urlutil/urlutil_test.go index 101714c..62ccbfd 100644 --- a/internal/urlutil/urlutil_test.go +++ b/internal/urlutil/urlutil_test.go @@ -3,17 +3,71 @@ package urlutil import ( + "context" "errors" "fmt" + "io" + "log" "net/http" "net/http/httptest" - "os" "testing" "github.com/google/go-cmp/cmp" + "github.com/google/go-containerregistry/pkg/registry" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "oras.land/oras-go/v2" + "oras.land/oras-go/v2/content/file" + "oras.land/oras-go/v2/registry/remote" ) -func TestGet_HTTP(t *testing.T) { +func TestGet_file(t *testing.T) { + tests := []struct { + name string + rawURL string + want []byte + wantNilErr bool + }{ + { + name: "valid", + rawURL: "testdata/content.txt", + want: []byte("file with content\n"), + wantNilErr: true, + }, + { + name: "empty", + rawURL: "testdata/empty.txt", + want: []byte{}, + wantNilErr: true, + }, + { + name: "not exist", + rawURL: "testdata/not_exist", + want: nil, + wantNilErr: false, + }, + { + name: "empty path", + rawURL: "", + want: nil, + wantNilErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := Get(tt.rawURL) + if (err == nil) != tt.wantNilErr { + t.Fatalf("unexpected error: %v", err) + } + + if diff := cmp.Diff(tt.want, got); diff != "" { + t.Errorf("content mismatch (-want +got):\n%v", diff) + } + }) + } +} + +func TestGet_http(t *testing.T) { tests := []struct { name string handlerFunc func(http.ResponseWriter, *http.Request) @@ -51,7 +105,69 @@ func TestGet_HTTP(t *testing.T) { got, err := Get(ts.URL) if (err == nil) != tt.wantNilErr { - t.Fatalf("unexpected error: want nil: %v, got: %v", tt.wantNilErr, err) + t.Fatalf("unexpected error: %v", err) + } + + if diff := cmp.Diff(tt.want, got); diff != "" { + t.Errorf("content mismatch (-want +got):\n%v", diff) + } + }) + } +} + +func TestGet_oci(t *testing.T) { + oldOCIPlainHTTP := ociPlainHTTP + defer func() { ociPlainHTTP = oldOCIPlainHTTP }() + + tests := []struct { + name string + file string + tag string + want []byte + wantNilErr bool + }{ + { + name: "valid", + file: "testdata/content.txt", + tag: "v1", + want: []byte("file with content\n"), + wantNilErr: true, + }, + { + name: "empty", + file: "testdata/empty.txt", + tag: "v1", + want: []byte{}, + wantNilErr: true, + }, + { + name: "not found", + file: "", + tag: "v1", + want: nil, + wantNilErr: false, + }, + } + + ociPlainHTTP = true + nopLogger := log.New(io.Discard, "", 0) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ts := httptest.NewServer(registry.New(registry.Logger(nopLogger))) + defer ts.Close() + + reg := ts.Listener.Addr().String() + + if tt.file != "" { + if err := pushArtifact(reg, tt.file, tt.tag); err != nil { + t.Fatalf("error pushing artifact: %v", err) + } + } + + repo := fmt.Sprintf("oci://%v/%v:%v", reg, tt.file, tt.tag) + got, err := Get(repo) + if (err == nil) != tt.wantNilErr { + t.Fatalf("unexpected error: %v", err) } if diff := cmp.Diff(tt.want, got); diff != "" { @@ -61,61 +177,90 @@ func TestGet_HTTP(t *testing.T) { } } -func TestGet_URL(t *testing.T) { +func TestGet_invalid_url(t *testing.T) { tests := []struct { name string - url string - want []byte + rawURL string wantErr error }{ { - name: "file", - url: "testdata/content.txt", - want: []byte("file with content\n"), - wantErr: nil, + name: "invalid URL scheme", + rawURL: "invalid://example.com/file.json", + wantErr: ErrInvalidScheme, }, { - name: "empty file", - url: "testdata/empty.txt", - want: []byte{}, - wantErr: nil, + name: "invalid URL", + rawURL: "1http://example.com/file.json", + wantErr: ErrInvalidURL, }, { - name: "file does not exist", - url: "testdata/not_exist", - want: nil, - wantErr: os.ErrNotExist, + name: "empty OCI host", + rawURL: "oci:///test/repo:tag", + wantErr: ErrInvalidURL, }, { - name: "empty file path", - url: "", - want: nil, - wantErr: os.ErrNotExist, + name: "empty OCI repo", + rawURL: "oci://registry/:tag", + wantErr: ErrInvalidURL, }, { - name: "invalid scheme", - url: "invalid://example.com/file.json", - want: nil, - wantErr: ErrInvalidScheme, + name: "empty OCI tag", + rawURL: "oci://registry/test/repo:", + wantErr: ErrInvalidURL, }, { - name: "invalid URL", - url: "1http://example.com/file.json", - want: nil, + name: "malformed OCI path", + rawURL: "oci:///test/repo", wantErr: ErrInvalidURL, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := Get(tt.url) + got, err := Get(tt.rawURL) if !errors.Is(err, tt.wantErr) { t.Errorf("unexpected error: want: %v, got: %v", tt.wantErr, err) } - if diff := cmp.Diff(tt.want, got); diff != "" { - t.Errorf("content mismatch (-want +got):\n%v", diff) + if got != nil { + t.Errorf("unexpected value: %v", got) } }) } } + +func pushArtifact(reg, name, tag string) error { + fs, err := file.New("") + if err != nil { + return fmt.Errorf("create file store: %w", err) + } + defer fs.Close() + + desc, err := fs.Add(context.Background(), name, "application/vnd.test.file", "") + if err != nil { + return fmt.Errorf("add file: %w", err) + } + + opts := oras.PackManifestOptions{ + Layers: []ocispec.Descriptor{desc}, + } + descManifest, err := oras.PackManifest(context.Background(), fs, oras.PackManifestVersion1_1_RC4, "application/vnd.test.manifest", opts) + if err != nil { + return fmt.Errorf("pack manifest: %w", err) + } + + if err := fs.Tag(context.Background(), descManifest, tag); err != nil { + return fmt.Errorf("tag artifact: %w", err) + } + + repo, err := remote.NewRepository(reg + "/" + name) + if err != nil { + return fmt.Errorf("new repository: %w", err) + } + repo.PlainHTTP = true + + if _, err := oras.Copy(context.Background(), fs, tag, repo, tag, oras.DefaultCopyOptions); err != nil { + return fmt.Errorf("copy artifact: %w", err) + } + return nil +}