mirror of
git://git.yoctoproject.org/poky
synced 2026-04-02 02:49:11 +00:00
bitbake: fetch2: Add gomod fetcher
Add a go module fetcher for downloading module dependencies to the module cache from a module proxy. The fetcher can be used with the go-mod class in OE-Core. A module dependency can be specified with: SRC_URI += "gomod://golang.org/x/net;version=v0.9.0;sha256sum=..." (Bitbake rev: 5ff4694bf305e266ebf0abab5d9745c6b6d07d67) Signed-off-by: Christian Lindeberg <christian.lindeberg@axis.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
This commit is contained in:
Christian Lindeberg
Richard Purdie
parent
4fc8427a6c
commit
dd7631426c
@ -1317,7 +1317,7 @@ class FetchData(object):
|
||||
|
||||
if checksum_name in self.parm:
|
||||
checksum_expected = self.parm[checksum_name]
|
||||
elif self.type not in ["http", "https", "ftp", "ftps", "sftp", "s3", "az", "crate", "gs"]:
|
||||
elif self.type not in ["http", "https", "ftp", "ftps", "sftp", "s3", "az", "crate", "gs", "gomod"]:
|
||||
checksum_expected = None
|
||||
else:
|
||||
checksum_expected = d.getVarFlag("SRC_URI", checksum_name)
|
||||
@ -2088,6 +2088,7 @@ from . import npmsw
|
||||
from . import az
|
||||
from . import crate
|
||||
from . import gcp
|
||||
from . import gomod
|
||||
|
||||
methods.append(local.Local())
|
||||
methods.append(wget.Wget())
|
||||
@ -2110,3 +2111,4 @@ methods.append(npmsw.NpmShrinkWrap())
|
||||
methods.append(az.Az())
|
||||
methods.append(crate.Crate())
|
||||
methods.append(gcp.GCP())
|
||||
methods.append(gomod.GoMod())
|
||||
|
||||
128
bitbake/lib/bb/fetch2/gomod.py
Normal file
128
bitbake/lib/bb/fetch2/gomod.py
Normal file
@ -0,0 +1,128 @@
|
||||
"""
|
||||
BitBake 'Fetch' implementation for Go modules
|
||||
|
||||
The gomod fetcher is used to download Go modules to the module cache from a
|
||||
module proxy.
|
||||
|
||||
Example SRC_URI:
|
||||
|
||||
SRC_URI += "gomod://golang.org/x/net;version=v0.9.0;sha256sum=..."
|
||||
|
||||
Required SRC_URI parameters:
|
||||
|
||||
- version
|
||||
The version of the module.
|
||||
|
||||
Optional SRC_URI parameters:
|
||||
|
||||
- mod
|
||||
Fetch and unpack the go.mod file only instead of the complete module.
|
||||
The go command may need to download go.mod files for many different modules
|
||||
when computing the build list, and go.mod files are much smaller than
|
||||
module zip files.
|
||||
The default is "0", set mod=1 for the go.mod file only.
|
||||
|
||||
- sha256sum
|
||||
The checksum of the module zip file, or the go.mod file in case of fetching
|
||||
only the go.mod file. Alternatively, set the SRC_URI varible flag for
|
||||
"module@version.sha256sum".
|
||||
|
||||
Related variables:
|
||||
|
||||
- GO_MOD_PROXY
|
||||
The module proxy used by the fetcher.
|
||||
|
||||
- GO_MOD_CACHE_DIR
|
||||
The directory where the module cache is located.
|
||||
This must match the exported GOMODCACHE variable for the go command to find
|
||||
the downloaded modules.
|
||||
|
||||
See the Go modules reference, https://go.dev/ref/mod, for more information
|
||||
about the module cache, module proxies and version control systems.
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import zipfile
|
||||
|
||||
import bb
|
||||
from bb.fetch2 import FetchError
|
||||
from bb.fetch2 import MissingParameterError
|
||||
from bb.fetch2.wget import Wget
|
||||
|
||||
|
||||
def escape(path):
|
||||
"""Escape capital letters using exclamation points."""
|
||||
return re.sub(r'([A-Z])', lambda m: '!' + m.group(1).lower(), path)
|
||||
|
||||
|
||||
class GoMod(Wget):
|
||||
"""Class to fetch Go modules from a Go module proxy via wget"""
|
||||
|
||||
def supports(self, ud, d):
|
||||
"""Check to see if a given URL is for this fetcher."""
|
||||
return ud.type == 'gomod'
|
||||
|
||||
def urldata_init(self, ud, d):
|
||||
"""Set up to download the module from the module proxy.
|
||||
|
||||
Set up to download the module zip file to the module cache directory
|
||||
and unpack the go.mod file (unless downloading only the go.mod file):
|
||||
|
||||
cache/download/<module>/@v/<version>.zip: The module zip file.
|
||||
cache/download/<module>/@v/<version>.mod: The go.mod file.
|
||||
"""
|
||||
|
||||
proxy = d.getVar('GO_MOD_PROXY') or 'proxy.golang.org'
|
||||
moddir = d.getVar('GO_MOD_CACHE_DIR') or 'pkg/mod'
|
||||
|
||||
if 'version' not in ud.parm:
|
||||
raise MissingParameterError('version', ud.url)
|
||||
|
||||
module = ud.host + ud.path
|
||||
ud.parm['module'] = module
|
||||
|
||||
# Set URL and filename for wget download
|
||||
path = escape(module + '/@v/' + ud.parm['version'])
|
||||
if ud.parm.get('mod', '0') == '1':
|
||||
path += '.mod'
|
||||
else:
|
||||
path += '.zip'
|
||||
ud.parm['unpack'] = '0'
|
||||
ud.url = bb.fetch2.encodeurl(
|
||||
('https', proxy, '/' + path, None, None, None))
|
||||
ud.parm['downloadfilename'] = path
|
||||
|
||||
# Set name parameter if sha256sum is set in recipe
|
||||
name = f"{module}@{ud.parm['version']}"
|
||||
if d.getVarFlag('SRC_URI', name + '.sha256sum'):
|
||||
ud.parm['name'] = name
|
||||
|
||||
# Set subdir for unpack
|
||||
ud.parm['subdir'] = os.path.join(moddir, 'cache/download',
|
||||
os.path.dirname(path))
|
||||
|
||||
super().urldata_init(ud, d)
|
||||
|
||||
def unpack(self, ud, rootdir, d):
|
||||
"""Unpack the module in the module cache."""
|
||||
|
||||
# Unpack the module zip file or go.mod file
|
||||
super().unpack(ud, rootdir, d)
|
||||
|
||||
if ud.localpath.endswith('.zip'):
|
||||
# Unpack the go.mod file from the zip file
|
||||
module = ud.parm['module']
|
||||
unpackdir = os.path.join(rootdir, ud.parm['subdir'])
|
||||
name = os.path.basename(ud.localpath).rsplit('.', 1)[0] + '.mod'
|
||||
bb.note(f"Unpacking {name} to {unpackdir}/")
|
||||
with zipfile.ZipFile(ud.localpath) as zf:
|
||||
with open(os.path.join(unpackdir, name), mode='wb') as mf:
|
||||
try:
|
||||
f = module + '@' + ud.parm['version'] + '/go.mod'
|
||||
shutil.copyfileobj(zf.open(f), mf)
|
||||
except KeyError:
|
||||
# If the module does not have a go.mod file, synthesize
|
||||
# one containing only a module statement.
|
||||
mf.write(f'module {module}\n'.encode())
|
||||
@ -3390,3 +3390,68 @@ class FetchPremirroronlyBrokenTarball(FetcherTest):
|
||||
fetcher.download()
|
||||
output = "".join(logs.output)
|
||||
self.assertFalse(" not a git repository (or any parent up to mount point /)" in output)
|
||||
|
||||
class GoModTest(FetcherTest):
|
||||
|
||||
@skipIfNoNetwork()
|
||||
def test_gomod_url(self):
|
||||
urls = ['gomod://github.com/Azure/azure-sdk-for-go/sdk/storage/azblob;version=v1.0.0;'
|
||||
'sha256sum=9bb69aea32f1d59711701f9562d66432c9c0374205e5009d1d1a62f03fb4fdad']
|
||||
|
||||
fetcher = bb.fetch2.Fetch(urls, self.d)
|
||||
ud = fetcher.ud[urls[0]]
|
||||
self.assertEqual(ud.url, 'https://proxy.golang.org/github.com/%21azure/azure-sdk-for-go/sdk/storage/azblob/%40v/v1.0.0.zip')
|
||||
self.assertNotIn('name', ud.parm)
|
||||
|
||||
fetcher.download()
|
||||
fetcher.unpack(self.unpackdir)
|
||||
downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download')
|
||||
self.assertTrue(os.path.exists(os.path.join(downloaddir, 'github.com/!azure/azure-sdk-for-go/sdk/storage/azblob/@v/v1.0.0.zip')))
|
||||
self.assertTrue(os.path.exists(os.path.join(downloaddir, 'github.com/!azure/azure-sdk-for-go/sdk/storage/azblob/@v/v1.0.0.mod')))
|
||||
|
||||
@skipIfNoNetwork()
|
||||
def test_gomod_url_go_mod_only(self):
|
||||
urls = ['gomod://github.com/Azure/azure-sdk-for-go/sdk/storage/azblob;version=v1.0.0;mod=1;'
|
||||
'sha256sum=7873b8544842329b4f385a3aa6cf82cc2bc8defb41a04fa5291c35fd5900e873']
|
||||
|
||||
fetcher = bb.fetch2.Fetch(urls, self.d)
|
||||
ud = fetcher.ud[urls[0]]
|
||||
self.assertEqual(ud.url, 'https://proxy.golang.org/github.com/%21azure/azure-sdk-for-go/sdk/storage/azblob/%40v/v1.0.0.mod')
|
||||
self.assertNotIn('name', ud.parm)
|
||||
|
||||
fetcher.download()
|
||||
fetcher.unpack(self.unpackdir)
|
||||
downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download')
|
||||
self.assertTrue(os.path.exists(os.path.join(downloaddir, 'github.com/!azure/azure-sdk-for-go/sdk/storage/azblob/@v/v1.0.0.mod')))
|
||||
|
||||
@skipIfNoNetwork()
|
||||
def test_gomod_url_sha256sum_varflag(self):
|
||||
urls = ['gomod://gopkg.in/ini.v1;version=v1.67.0']
|
||||
self.d.setVarFlag('SRC_URI', 'gopkg.in/ini.v1@v1.67.0.sha256sum', 'bd845dfc762a87a56e5a32a07770dc83e86976db7705d7f89c5dbafdc60b06c6')
|
||||
|
||||
fetcher = bb.fetch2.Fetch(urls, self.d)
|
||||
ud = fetcher.ud[urls[0]]
|
||||
self.assertEqual(ud.url, 'https://proxy.golang.org/gopkg.in/ini.v1/%40v/v1.67.0.zip')
|
||||
self.assertEqual(ud.parm['name'], 'gopkg.in/ini.v1@v1.67.0')
|
||||
|
||||
fetcher.download()
|
||||
fetcher.unpack(self.unpackdir)
|
||||
downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download')
|
||||
self.assertTrue(os.path.exists(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.zip')))
|
||||
self.assertTrue(os.path.exists(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.mod')))
|
||||
|
||||
@skipIfNoNetwork()
|
||||
def test_gomod_url_no_go_mod_in_module(self):
|
||||
urls = ['gomod://gopkg.in/ini.v1;version=v1.67.0;'
|
||||
'sha256sum=bd845dfc762a87a56e5a32a07770dc83e86976db7705d7f89c5dbafdc60b06c6']
|
||||
|
||||
fetcher = bb.fetch2.Fetch(urls, self.d)
|
||||
ud = fetcher.ud[urls[0]]
|
||||
self.assertEqual(ud.url, 'https://proxy.golang.org/gopkg.in/ini.v1/%40v/v1.67.0.zip')
|
||||
self.assertNotIn('name', ud.parm)
|
||||
|
||||
fetcher.download()
|
||||
fetcher.unpack(self.unpackdir)
|
||||
downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download')
|
||||
self.assertTrue(os.path.exists(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.zip')))
|
||||
self.assertTrue(os.path.exists(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.mod')))
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user