diff options
Diffstat (limited to 'external/poky/bitbake/lib/bb/fetch2')
19 files changed, 914 insertions, 749 deletions
diff --git a/external/poky/bitbake/lib/bb/fetch2/__init__.py b/external/poky/bitbake/lib/bb/fetch2/__init__.py index 03e56471..eb112f06 100644 --- a/external/poky/bitbake/lib/bb/fetch2/__init__.py +++ b/external/poky/bitbake/lib/bb/fetch2/__init__.py @@ -1,5 +1,3 @@ -# ex:ts=4:sw=4:sts=4:et -# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- """ BitBake 'Fetch' implementations @@ -10,18 +8,7 @@ BitBake build tools. # Copyright (C) 2003, 2004 Chris Larson # Copyright (C) 2012 Intel Corporation # -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# SPDX-License-Identifier: GPL-2.0-only # # Based on functions from the base bb module, Copyright 2003 Holger Schurig @@ -46,6 +33,9 @@ _checksum_cache = bb.checksum.FileChecksumCache() logger = logging.getLogger("BitBake.Fetcher") +CHECKSUM_LIST = [ "md5", "sha256", "sha1", "sha384", "sha512" ] +SHOWN_CHECKSUM_LIST = ["sha256"] + class BBFetchException(Exception): """Class all fetch exceptions inherit from""" def __init__(self, message): @@ -144,10 +134,9 @@ class NonLocalMethod(Exception): Exception.__init__(self) class MissingChecksumEvent(bb.event.Event): - def __init__(self, url, md5sum, sha256sum): + def __init__(self, url, **checksums): self.url = url - self.checksums = {'md5sum': md5sum, - 'sha256sum': sha256sum} + self.checksums = checksums bb.event.Event.__init__(self) @@ -497,17 +486,22 @@ def fetcher_init(d): Called to initialize the fetchers once the configuration data is known. Calls before this must not hit the cache. """ + + revs = bb.persist_data.persist('BB_URI_HEADREVS', d) + try: + # fetcher_init is called multiple times, so make sure we only save the + # revs the first time it is called. + if not bb.fetch2.saved_headrevs: + bb.fetch2.saved_headrevs = dict(revs) + except: + pass + # When to drop SCM head revisions controlled by user policy srcrev_policy = d.getVar('BB_SRCREV_POLICY') or "clear" if srcrev_policy == "cache": logger.debug(1, "Keeping SRCREV cache due to cache policy of: %s", srcrev_policy) elif srcrev_policy == "clear": logger.debug(1, "Clearing SRCREV cache due to cache policy of: %s", srcrev_policy) - revs = bb.persist_data.persist('BB_URI_HEADREVS', d) - try: - bb.fetch2.saved_headrevs = revs.items() - except: - pass revs.clear() else: raise FetchError("Invalid SRCREV cache policy of: %s" % srcrev_policy) @@ -524,24 +518,14 @@ def fetcher_parse_save(): def fetcher_parse_done(): _checksum_cache.save_merge() -def fetcher_compare_revisions(): +def fetcher_compare_revisions(d): """ - Compare the revisions in the persistant cache with current values and - return true/false on whether they've changed. + Compare the revisions in the persistent cache with the saved values from + when bitbake was started and return true if they have changed. """ - data = bb.persist_data.persist('BB_URI_HEADREVS', d).items() - data2 = bb.fetch2.saved_headrevs - - changed = False - for key in data: - if key not in data2 or data2[key] != data[key]: - logger.debug(1, "%s changed", key) - changed = True - return True - else: - logger.debug(2, "%s did not change", key) - return False + headrevs = dict(bb.persist_data.persist('BB_URI_HEADREVS', d)) + return headrevs != bb.fetch2.saved_headrevs def mirror_from_string(data): mirrors = (data or "").replace('\\n',' ').split() @@ -565,71 +549,84 @@ def verify_checksum(ud, d, precomputed={}): downloading. See https://bugzilla.yoctoproject.org/show_bug.cgi?id=5571. """ - _MD5_KEY = "md5" - _SHA256_KEY = "sha256" - if ud.ignore_checksums or not ud.method.supports_checksum(ud): return {} - if _MD5_KEY in precomputed: - md5data = precomputed[_MD5_KEY] - else: - md5data = bb.utils.md5_file(ud.localpath) + def compute_checksum_info(checksum_id): + checksum_name = getattr(ud, "%s_name" % checksum_id) - if _SHA256_KEY in precomputed: - sha256data = precomputed[_SHA256_KEY] - else: - sha256data = bb.utils.sha256_file(ud.localpath) + if checksum_id in precomputed: + checksum_data = precomputed[checksum_id] + else: + checksum_data = getattr(bb.utils, "%s_file" % checksum_id)(ud.localpath) - if ud.method.recommends_checksum(ud) and not ud.md5_expected and not ud.sha256_expected: - # If strict checking enabled and neither sum defined, raise error + checksum_expected = getattr(ud, "%s_expected" % checksum_id) + + return { + "id": checksum_id, + "name": checksum_name, + "data": checksum_data, + "expected": checksum_expected + } + + checksum_infos = [] + for checksum_id in CHECKSUM_LIST: + checksum_infos.append(compute_checksum_info(checksum_id)) + + checksum_dict = {ci["id"] : ci["data"] for ci in checksum_infos} + checksum_event = {"%ssum" % ci["id"] : ci["data"] for ci in checksum_infos} + + for ci in checksum_infos: + if ci["id"] in SHOWN_CHECKSUM_LIST: + checksum_lines = ["SRC_URI[%s] = \"%s\"" % (ci["name"], ci["data"])] + + # If no checksum has been provided + if ud.method.recommends_checksum(ud) and all(ci["expected"] is None for ci in checksum_infos): + messages = [] strict = d.getVar("BB_STRICT_CHECKSUM") or "0" + + # If strict checking enabled and neither sum defined, raise error if strict == "1": - logger.error('No checksum specified for %s, please add at least one to the recipe:\n' - 'SRC_URI[%s] = "%s"\nSRC_URI[%s] = "%s"' % - (ud.localpath, ud.md5_name, md5data, - ud.sha256_name, sha256data)) - raise NoChecksumError('Missing SRC_URI checksum', ud.url) + messages.append("No checksum specified for '%s', please add at " \ + "least one to the recipe:" % ud.localpath) + messages.extend(checksum_lines) + logger.error("\n".join(messages)) + raise NoChecksumError("Missing SRC_URI checksum", ud.url) - bb.event.fire(MissingChecksumEvent(ud.url, md5data, sha256data), d) + bb.event.fire(MissingChecksumEvent(ud.url, **checksum_event), d) if strict == "ignore": - return { - _MD5_KEY: md5data, - _SHA256_KEY: sha256data - } + return checksum_dict # Log missing sums so user can more easily add them - logger.warning('Missing md5 SRC_URI checksum for %s, consider adding to the recipe:\n' - 'SRC_URI[%s] = "%s"', - ud.localpath, ud.md5_name, md5data) - logger.warning('Missing sha256 SRC_URI checksum for %s, consider adding to the recipe:\n' - 'SRC_URI[%s] = "%s"', - ud.localpath, ud.sha256_name, sha256data) + messages.append("Missing checksum for '%s', consider adding at " \ + "least one to the recipe:" % ud.localpath) + messages.extend(checksum_lines) + logger.warning("\n".join(messages)) # We want to alert the user if a checksum is defined in the recipe but # it does not match. - msg = "" - mismatch = False - if ud.md5_expected and ud.md5_expected != md5data: - msg = msg + "\nFile: '%s' has %s checksum %s when %s was expected" % (ud.localpath, 'md5', md5data, ud.md5_expected) - mismatch = True; - - if ud.sha256_expected and ud.sha256_expected != sha256data: - msg = msg + "\nFile: '%s' has %s checksum %s when %s was expected" % (ud.localpath, 'sha256', sha256data, ud.sha256_expected) - mismatch = True; - - if mismatch: - msg = msg + '\nIf this change is expected (e.g. you have upgraded to a new version without updating the checksums) then you can use these lines within the recipe:\nSRC_URI[%s] = "%s"\nSRC_URI[%s] = "%s"\nOtherwise you should retry the download and/or check with upstream to determine if the file has become corrupted or otherwise unexpectedly modified.\n' % (ud.md5_name, md5data, ud.sha256_name, sha256data) - - if len(msg): - raise ChecksumError('Checksum mismatch!%s' % msg, ud.url, md5data) - - return { - _MD5_KEY: md5data, - _SHA256_KEY: sha256data - } - + messages = [] + messages.append("Checksum mismatch!") + bad_checksum = None + + for ci in checksum_infos: + if ci["expected"] and ci["expected"] != ci["data"]: + messages.append("File: '%s' has %s checksum %s when %s was " \ + "expected" % (ud.localpath, ci["id"], ci["data"], ci["expected"])) + bad_checksum = ci["data"] + + if bad_checksum: + messages.append("If this change is expected (e.g. you have upgraded " \ + "to a new version without updating the checksums) " \ + "then you can use these lines within the recipe:") + messages.extend(checksum_lines) + messages.append("Otherwise you should retry the download and/or " \ + "check with upstream to determine if the file has " \ + "become corrupted or otherwise unexpectedly modified.") + raise ChecksumError("\n".join(messages), ud.url, bad_checksum) + + return checksum_dict def verify_donestamp(ud, d, origud=None): """ @@ -777,7 +774,8 @@ def get_srcrev(d, method_name='sortable_revision'): # format = d.getVar('SRCREV_FORMAT') if not format: - raise FetchError("The SRCREV_FORMAT variable must be set when multiple SCMs are used.") + raise FetchError("The SRCREV_FORMAT variable must be set when multiple SCMs are used.\n"\ + "The SCMs are:\n%s" % '\n'.join(scms)) name_to_rev = {} seenautoinc = False @@ -855,10 +853,18 @@ def runfetchcmd(cmd, d, quiet=False, cleanup=None, log=None, workdir=None): if val: cmd = 'export ' + var + '=\"%s\"; %s' % (val, cmd) + # Ensure that a _PYTHON_SYSCONFIGDATA_NAME value set by a recipe + # (for example via python3native.bbclass since warrior) is not set for + # host Python (otherwise tools like git-make-shallow will fail) + cmd = 'unset _PYTHON_SYSCONFIGDATA_NAME; ' + cmd + # Disable pseudo as it may affect ssh, potentially causing it to hang. cmd = 'export PSEUDO_DISABLED=1; ' + cmd - logger.debug(1, "Running %s", cmd) + if workdir: + logger.debug(1, "Running '%s' in %s" % (cmd, workdir)) + else: + logger.debug(1, "Running %s", cmd) success = False error_message = "" @@ -894,7 +900,7 @@ def check_network_access(d, info, url): log remote network access, and error if BB_NO_NETWORK is set or the given URI is untrusted """ - if d.getVar("BB_NO_NETWORK") == "1": + if bb.utils.to_boolean(d.getVar("BB_NO_NETWORK")): raise NetworkAccess(url, info) elif not trusted_network(d, url): raise UntrustedUrl(url, info) @@ -1028,7 +1034,7 @@ def try_mirror_url(fetch, origud, ud, ld, check = False): raise except IOError as e: - if e.errno in [os.errno.ESTALE]: + if e.errno in [errno.ESTALE]: logger.warning("Stale Error Observed %s." % ud.url) return False raise @@ -1085,7 +1091,7 @@ def try_mirrors(fetch, d, origud, mirrors, check = False): for index, uri in enumerate(uris): ret = try_mirror_url(fetch, origud, uds[index], ld, check) - if ret != False: + if ret: return ret return None @@ -1095,7 +1101,7 @@ def trusted_network(d, url): BB_ALLOWED_NETWORKS is set globally or for a specific recipe. Note: modifies SRC_URI & mirrors. """ - if d.getVar('BB_NO_NETWORK') == "1": + if bb.utils.to_boolean(d.getVar("BB_NO_NETWORK")): return True pkgname = d.expand(d.getVar('PN', False)) @@ -1201,14 +1207,14 @@ def get_checksum_file_list(d): return " ".join(filelist) -def get_file_checksums(filelist, pn): +def get_file_checksums(filelist, pn, localdirsexclude): """Get a list of the checksums for a list of local files Returns the checksums for a list of local files, caching the results as it proceeds """ - return _checksum_cache.get_checksums(filelist, pn) + return _checksum_cache.get_checksums(filelist, pn, localdirsexclude) class FetchData(object): @@ -1234,24 +1240,26 @@ class FetchData(object): self.pswd = self.parm["pswd"] self.setup = False - if "name" in self.parm: - self.md5_name = "%s.md5sum" % self.parm["name"] - self.sha256_name = "%s.sha256sum" % self.parm["name"] - else: - self.md5_name = "md5sum" - self.sha256_name = "sha256sum" - if self.md5_name in self.parm: - self.md5_expected = self.parm[self.md5_name] - elif self.type not in ["http", "https", "ftp", "ftps", "sftp", "s3"]: - self.md5_expected = None - else: - self.md5_expected = d.getVarFlag("SRC_URI", self.md5_name) - if self.sha256_name in self.parm: - self.sha256_expected = self.parm[self.sha256_name] - elif self.type not in ["http", "https", "ftp", "ftps", "sftp", "s3"]: - self.sha256_expected = None - else: - self.sha256_expected = d.getVarFlag("SRC_URI", self.sha256_name) + def configure_checksum(checksum_id): + if "name" in self.parm: + checksum_name = "%s.%ssum" % (self.parm["name"], checksum_id) + else: + checksum_name = "%ssum" % checksum_id + + setattr(self, "%s_name" % checksum_id, checksum_name) + + if checksum_name in self.parm: + checksum_expected = self.parm[checksum_name] + elif self.type not in ["http", "https", "ftp", "ftps", "sftp", "s3"]: + checksum_expected = None + else: + checksum_expected = d.getVarFlag("SRC_URI", checksum_name) + + setattr(self, "%s_expected" % checksum_id, checksum_expected) + + for checksum_id in CHECKSUM_LIST: + configure_checksum(checksum_id) + self.ignore_checksums = False self.names = self.parm.get("name",'default').split(',') @@ -1355,7 +1363,7 @@ class FetchMethod(object): """ # We cannot compute checksums for directories - if os.path.isdir(urldata.localpath) == True: + if os.path.isdir(urldata.localpath): return False if urldata.localpath.find("*") != -1: return False @@ -1369,6 +1377,18 @@ class FetchMethod(object): """ return False + def verify_donestamp(self, ud, d): + """ + Verify the donestamp file + """ + return verify_donestamp(ud, d) + + def update_donestamp(self, ud, d): + """ + Update the donestamp file + """ + update_stamp(ud, d) + def _strip_leading_slashes(self, relpath): """ Remove leading slash as os.path.join can't cope @@ -1404,7 +1424,7 @@ class FetchMethod(object): Fetch urls Assumes localpath was called first """ - raise NoMethodError(url) + raise NoMethodError(urldata.url) def unpack(self, urldata, rootdir, data): iterate = False @@ -1543,12 +1563,18 @@ class FetchMethod(object): """ return True + def try_mirrors(self, fetch, urldata, d, mirrors, check=False): + """ + Try to use a mirror + """ + return bool(try_mirrors(fetch, d, urldata, mirrors, check)) + def checkstatus(self, fetch, urldata, d): """ Check the status of a URL Assumes localpath was called first """ - logger.info("URL %s could not be checked for status since no method exists.", url) + logger.info("URL %s could not be checked for status since no method exists.", urldata.url) return True def latest_revision(self, ud, d, name): @@ -1556,7 +1582,7 @@ class FetchMethod(object): Look in the cache for the latest revision, if not present ask the SCM. """ if not hasattr(self, "_latest_revision"): - raise ParameterError("The fetcher for this URL does not support _latest_revision", url) + raise ParameterError("The fetcher for this URL does not support _latest_revision", ud.url) revs = bb.persist_data.persist('BB_URI_HEADREVS', d) key = self.generate_revision_key(ud, d, name) @@ -1571,8 +1597,7 @@ class FetchMethod(object): return True, str(latest_rev) def generate_revision_key(self, ud, d, name): - key = self._revision_key(ud, d, name) - return "%s-%s" % (key, d.getVar("PN") or "") + return self._revision_key(ud, d, name) def latest_versionstring(self, ud, d): """ @@ -1582,6 +1607,16 @@ class FetchMethod(object): """ return ('', '') + def done(self, ud, d): + """ + Is the download done ? + """ + if os.path.exists(ud.localpath): + return True + if ud.localpath.find("*") != -1: + return True + return False + class Fetch(object): def __init__(self, urls, d, cache = True, localonly = False, connection_cache = None): if localonly and cache: @@ -1596,8 +1631,11 @@ class Fetch(object): fn = d.getVar('FILE') mc = d.getVar('__BBMULTICONFIG') or "" - if cache and fn and mc + fn in urldata_cache: - self.ud = urldata_cache[mc + fn + str(id(d))] + key = None + if cache and fn: + key = mc + fn + str(id(d)) + if key in urldata_cache: + self.ud = urldata_cache[key] for url in urls: if url not in self.ud: @@ -1608,8 +1646,8 @@ class Fetch(object): self.ud[url] = None pass - if fn and cache: - urldata_cache[mc + fn + str(id(d))] = self.ud + if key: + urldata_cache[key] = self.ud def localpath(self, url): if url not in self.urls: @@ -1639,13 +1677,13 @@ class Fetch(object): urls = self.urls network = self.d.getVar("BB_NO_NETWORK") - premirroronly = (self.d.getVar("BB_FETCH_PREMIRRORONLY") == "1") + premirroronly = bb.utils.to_boolean(self.d.getVar("BB_FETCH_PREMIRRORONLY")) for u in urls: ud = self.ud[u] ud.setup_localpath(self.d) m = ud.method - localpath = "" + done = False if ud.lockfile: lf = bb.utils.lockfile(ud.lockfile) @@ -1653,28 +1691,28 @@ class Fetch(object): try: self.d.setVar("BB_NO_NETWORK", network) - if verify_donestamp(ud, self.d) and not m.need_update(ud, self.d): - localpath = ud.localpath + if m.verify_donestamp(ud, self.d) and not m.need_update(ud, self.d): + done = True elif m.try_premirror(ud, self.d): logger.debug(1, "Trying PREMIRRORS") mirrors = mirror_from_string(self.d.getVar('PREMIRRORS')) - localpath = try_mirrors(self, self.d, ud, mirrors, False) - if localpath: + done = m.try_mirrors(self, ud, self.d, mirrors) + if done: try: # early checksum verification so that if the checksum of the premirror # contents mismatch the fetcher can still try upstream and mirrors - update_stamp(ud, self.d) + m.update_donestamp(ud, self.d) except ChecksumError as e: logger.warning("Checksum failure encountered with premirror download of %s - will attempt other sources." % u) logger.debug(1, str(e)) - localpath = "" + done = False if premirroronly: self.d.setVar("BB_NO_NETWORK", "1") firsterr = None - verified_stamp = verify_donestamp(ud, self.d) - if not localpath and (not verified_stamp or m.need_update(ud, self.d)): + verified_stamp = m.verify_donestamp(ud, self.d) + if not done and (not verified_stamp or m.need_update(ud, self.d)): try: if not trusted_network(self.d, ud.url): raise UntrustedUrl(ud.url) @@ -1682,10 +1720,10 @@ class Fetch(object): m.download(ud, self.d) if hasattr(m, "build_mirror_data"): m.build_mirror_data(ud, self.d) - localpath = ud.localpath + done = True # early checksum verify, so that if checksum mismatched, # fetcher still have chance to fetch from mirror - update_stamp(ud, self.d) + m.update_donestamp(ud, self.d) except bb.fetch2.NetworkAccess: raise @@ -1707,17 +1745,17 @@ class Fetch(object): m.clean(ud, self.d) logger.debug(1, "Trying MIRRORS") mirrors = mirror_from_string(self.d.getVar('MIRRORS')) - localpath = try_mirrors(self, self.d, ud, mirrors) + done = m.try_mirrors(self, ud, self.d, mirrors) - if not localpath or ((not os.path.exists(localpath)) and localpath.find("*") == -1): + if not done or not m.done(ud, self.d): if firsterr: logger.error(str(firsterr)) raise FetchError("Unable to fetch URL from any source.", u) - update_stamp(ud, self.d) + m.update_donestamp(ud, self.d) except IOError as e: - if e.errno in [os.errno.ESTALE]: + if e.errno in [errno.ESTALE]: logger.error("Stale Error Observed %s." % u) raise ChecksumError("Stale Error Detected") @@ -1745,14 +1783,14 @@ class Fetch(object): logger.debug(1, "Testing URL %s", u) # First try checking uri, u, from PREMIRRORS mirrors = mirror_from_string(self.d.getVar('PREMIRRORS')) - ret = try_mirrors(self, self.d, ud, mirrors, True) + ret = m.try_mirrors(self, ud, self.d, mirrors, True) if not ret: # Next try checking from the original uri, u ret = m.checkstatus(self, ud, self.d) if not ret: # Finally, try checking uri, u, from MIRRORS mirrors = mirror_from_string(self.d.getVar('MIRRORS')) - ret = try_mirrors(self, self.d, ud, mirrors, True) + ret = m.try_mirrors(self, ud, self.d, mirrors, True) if not ret: raise FetchError("URL %s doesn't work" % u, u) @@ -1787,7 +1825,7 @@ class Fetch(object): for url in urls: if url not in self.ud: - self.ud[url] = FetchData(url, d) + self.ud[url] = FetchData(url, self.d) ud = self.ud[url] ud.setup_localpath(self.d) @@ -1857,6 +1895,7 @@ from . import osc from . import repo from . import clearcase from . import npm +from . import npmsw methods.append(local.Local()) methods.append(wget.Wget()) @@ -1875,3 +1914,4 @@ methods.append(osc.Osc()) methods.append(repo.Repo()) methods.append(clearcase.ClearCase()) methods.append(npm.Npm()) +methods.append(npmsw.NpmShrinkWrap()) diff --git a/external/poky/bitbake/lib/bb/fetch2/bzr.py b/external/poky/bitbake/lib/bb/fetch2/bzr.py index 658502f9..566ace9f 100644 --- a/external/poky/bitbake/lib/bb/fetch2/bzr.py +++ b/external/poky/bitbake/lib/bb/fetch2/bzr.py @@ -10,22 +10,10 @@ BitBake 'Fetch' implementation for bzr. # BitBake build tools. # Copyright (C) 2003, 2004 Chris Larson # -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. +# SPDX-License-Identifier: GPL-2.0-only # -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. import os -import sys -import logging import bb from bb.fetch2 import FetchMethod from bb.fetch2 import FetchError diff --git a/external/poky/bitbake/lib/bb/fetch2/clearcase.py b/external/poky/bitbake/lib/bb/fetch2/clearcase.py index 3a6573d0..49d7ae1b 100644 --- a/external/poky/bitbake/lib/bb/fetch2/clearcase.py +++ b/external/poky/bitbake/lib/bb/fetch2/clearcase.py @@ -1,5 +1,3 @@ -# ex:ts=4:sw=4:sts=4:et -# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- """ BitBake 'Fetch' clearcase implementation @@ -47,26 +45,16 @@ User credentials: """ # Copyright (C) 2014 Siemens AG # -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# SPDX-License-Identifier: GPL-2.0-only # import os -import sys import shutil import bb from bb.fetch2 import FetchMethod from bb.fetch2 import FetchError +from bb.fetch2 import MissingParameterError +from bb.fetch2 import ParameterError from bb.fetch2 import runfetchcmd from bb.fetch2 import logger @@ -92,7 +80,7 @@ class ClearCase(FetchMethod): if 'protocol' in ud.parm: ud.proto = ud.parm['protocol'] if not ud.proto in ('http', 'https'): - raise fetch2.ParameterError("Invalid protocol type", ud.url) + raise ParameterError("Invalid protocol type", ud.url) ud.vob = '' if 'vob' in ud.parm: @@ -156,18 +144,18 @@ class ClearCase(FetchMethod): basecmd = "%s %s" % (ud.basecmd, command) - if command is 'mkview': + if command == 'mkview': if not "rcleartool" in ud.basecmd: # Cleartool needs a -snapshot view options.append("-snapshot") options.append("-tag %s" % ud.viewname) options.append(ud.viewdir) - elif command is 'rmview': + elif command == 'rmview': options.append("-force") options.append("%s" % ud.viewdir) - elif command is 'setcs': + elif command == 'setcs': options.append("-overwrite") options.append(ud.configspecfile) @@ -249,7 +237,7 @@ class ClearCase(FetchMethod): # Clean clearcase meta-data before tar - runfetchcmd('tar -czf "%s" .' % (ud.localpath), d, cleanup = [ud.localpath]) + runfetchcmd('tar -czf "%s" .' % (ud.localpath), d, cleanup = [ud.localpath], workdir = ud.viewdir) # Clean up so we can create a new view next time self.clean(ud, d); diff --git a/external/poky/bitbake/lib/bb/fetch2/cvs.py b/external/poky/bitbake/lib/bb/fetch2/cvs.py index 0e0a3196..29123a48 100644 --- a/external/poky/bitbake/lib/bb/fetch2/cvs.py +++ b/external/poky/bitbake/lib/bb/fetch2/cvs.py @@ -1,5 +1,3 @@ -# ex:ts=4:sw=4:sts=4:et -# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- """ BitBake 'Fetch' implementations @@ -10,24 +8,12 @@ BitBake build tools. # Copyright (C) 2003, 2004 Chris Larson # -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. +# SPDX-License-Identifier: GPL-2.0-only # -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# -#Based on functions from the base bb module, Copyright 2003 Holger Schurig +# Based on functions from the base bb module, Copyright 2003 Holger Schurig # import os -import logging import bb from bb.fetch2 import FetchMethod, FetchError, MissingParameterError, logger from bb.fetch2 import runfetchcmd diff --git a/external/poky/bitbake/lib/bb/fetch2/git.py b/external/poky/bitbake/lib/bb/fetch2/git.py index 59a2ee8f..2ce9395f 100644 --- a/external/poky/bitbake/lib/bb/fetch2/git.py +++ b/external/poky/bitbake/lib/bb/fetch2/git.py @@ -1,5 +1,3 @@ -# ex:ts=4:sw=4:sts=4:et -# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- """ BitBake 'Fetch' git implementation @@ -55,20 +53,10 @@ Supported SRC_URI options are: """ -#Copyright (C) 2005 Richard Purdie +# Copyright (C) 2005 Richard Purdie # -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. +# SPDX-License-Identifier: GPL-2.0-only # -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. import collections import errno @@ -199,7 +187,7 @@ class Git(FetchMethod): depth_default = 1 ud.shallow_depths = collections.defaultdict(lambda: depth_default) - revs_default = d.getVar("BB_GIT_SHALLOW_REVS", True) + revs_default = d.getVar("BB_GIT_SHALLOW_REVS") ud.shallow_revs = [] ud.branches = {} for pos, name in enumerate(ud.names): @@ -304,11 +292,21 @@ class Git(FetchMethod): def clonedir_need_update(self, ud, d): if not os.path.exists(ud.clonedir): return True + if ud.shallow and ud.write_shallow_tarballs and self.clonedir_need_shallow_revs(ud, d): + return True for name in ud.names: if not self._contains_ref(ud, d, name, ud.clonedir): return True return False + def clonedir_need_shallow_revs(self, ud, d): + for rev in ud.shallow_revs: + try: + runfetchcmd('%s rev-parse -q --verify %s' % (ud.basecmd, rev), d, quiet=True, workdir=ud.clonedir) + except bb.fetch2.FetchError: + return rev + return None + def shallow_tarball_need_update(self, ud): return ud.shallow and ud.write_shallow_tarballs and not os.path.exists(ud.fullshallow) @@ -318,7 +316,7 @@ class Git(FetchMethod): def try_premirror(self, ud, d): # If we don't do this, updating an existing checkout with only premirrors # is not possible - if d.getVar("BB_FETCH_PREMIRRORONLY") is not None: + if bb.utils.to_boolean(d.getVar("BB_FETCH_PREMIRRORONLY")): return True if os.path.exists(ud.clonedir): return False @@ -351,19 +349,13 @@ class Git(FetchMethod): runfetchcmd(clone_cmd, d, log=progresshandler) # Update the checkout if needed - needupdate = False - for name in ud.names: - if not self._contains_ref(ud, d, name, ud.clonedir): - needupdate = True - break - - if needupdate: + if self.clonedir_need_update(ud, d): output = runfetchcmd("%s remote" % ud.basecmd, d, quiet=True, workdir=ud.clonedir) if "origin" in output: runfetchcmd("%s remote rm origin" % ud.basecmd, d, workdir=ud.clonedir) runfetchcmd("%s remote add --mirror=fetch origin %s" % (ud.basecmd, repourl), d, workdir=ud.clonedir) - fetch_cmd = "LANG=C %s fetch -f --prune --progress %s refs/*:refs/*" % (ud.basecmd, repourl) + fetch_cmd = "LANG=C %s fetch -f --progress %s refs/*:refs/*" % (ud.basecmd, repourl) if ud.proto.lower() != 'file': bb.fetch2.check_network_access(d, fetch_cmd, ud.url) progresshandler = GitProgressHandler(d) @@ -381,6 +373,11 @@ class Git(FetchMethod): if not self._contains_ref(ud, d, name, ud.clonedir): raise bb.fetch2.FetchError("Unable to find revision %s in branch %s even from upstream" % (ud.revisions[name], ud.branches[name])) + if ud.shallow and ud.write_shallow_tarballs: + missing_rev = self.clonedir_need_shallow_revs(ud, d) + if missing_rev: + raise bb.fetch2.FetchError("Unable to find revision %s even from upstream" % missing_rev) + def build_mirror_data(self, ud, d): if ud.shallow and ud.write_shallow_tarballs: if not os.path.exists(ud.fullshallow): @@ -476,6 +473,8 @@ class Git(FetchMethod): if os.path.exists(destdir): bb.utils.prunedir(destdir) + need_lfs = ud.parm.get("lfs", "1") == "1" + source_found = False source_error = [] @@ -503,6 +502,13 @@ class Git(FetchMethod): repourl = self._get_repo_url(ud) runfetchcmd("%s remote set-url origin %s" % (ud.basecmd, repourl), d, workdir=destdir) + + if self._contains_lfs(ud, d, destdir): + if need_lfs and not self._find_git_lfs(d): + raise bb.fetch2.FetchError("Repository %s has LFS content, install git-lfs on host to download (or set lfs=0 to ignore it)" % (repourl)) + else: + bb.note("Repository %s has LFS content but it is not being fetched" % (repourl)) + if not ud.nocheckout: if subdir != "": runfetchcmd("%s read-tree %s%s" % (ud.basecmd, ud.revisions[ud.names[0]], readpathspec), d, @@ -522,9 +528,17 @@ class Git(FetchMethod): def clean(self, ud, d): """ clean the git directory """ - bb.utils.remove(ud.localpath, True) - bb.utils.remove(ud.fullmirror) - bb.utils.remove(ud.fullmirror + ".done") + to_remove = [ud.localpath, ud.fullmirror, ud.fullmirror + ".done"] + # The localpath is a symlink to clonedir when it is cloned from a + # mirror, so remove both of them. + if os.path.islink(ud.localpath): + clonedir = os.path.realpath(ud.localpath) + to_remove.append(clonedir) + + for r in to_remove: + if os.path.exists(r): + bb.note('Removing %s' % r) + bb.utils.remove(r, True) def supports_srcrev(self): return True @@ -545,6 +559,27 @@ class Git(FetchMethod): raise bb.fetch2.FetchError("The command '%s' gave output with more then 1 line unexpectedly, output: '%s'" % (cmd, output)) return output.split()[0] != "0" + def _contains_lfs(self, ud, d, wd): + """ + Check if the repository has 'lfs' (large file) content + """ + cmd = "%s grep lfs HEAD:.gitattributes | wc -l" % ( + ud.basecmd) + try: + output = runfetchcmd(cmd, d, quiet=True, workdir=wd) + if int(output) > 0: + return True + except (bb.fetch2.FetchError,ValueError): + pass + return False + + def _find_git_lfs(self, d): + """ + Return True if git-lfs can be found, False otherwise. + """ + import shutil + return shutil.which("git-lfs", path=d.getVar('PATH')) is not None + def _get_repo_url(self, ud): """ Return the repository URL @@ -559,7 +594,9 @@ class Git(FetchMethod): """ Return a unique key for the url """ - return "git:" + ud.host + ud.path.replace('/', '.') + ud.unresolvedrev[name] + # Collapse adjacent slashes + slash_re = re.compile(r"/+") + return "git:" + ud.host + slash_re.sub(".", ud.path) + ud.unresolvedrev[name] def _lsremote(self, ud, d, search): """ @@ -615,7 +652,7 @@ class Git(FetchMethod): """ pupver = ('', '') - tagregex = re.compile(d.getVar('UPSTREAM_CHECK_GITTAGREGEX') or "(?P<pver>([0-9][\.|_]?)+)") + tagregex = re.compile(d.getVar('UPSTREAM_CHECK_GITTAGREGEX') or r"(?P<pver>([0-9][\.|_]?)+)") try: output = self._lsremote(ud, d, "refs/tags/*") except (bb.fetch2.FetchError, bb.fetch2.NetworkAccess) as e: @@ -630,13 +667,13 @@ class Git(FetchMethod): tag_head = line.split("/")[-1] # Ignore non-released branches - m = re.search("(alpha|beta|rc|final)+", tag_head) + m = re.search(r"(alpha|beta|rc|final)+", tag_head) if m: continue # search for version in the line tag = tagregex.search(tag_head) - if tag == None: + if tag is None: continue tag = tag.group('pver') diff --git a/external/poky/bitbake/lib/bb/fetch2/gitannex.py b/external/poky/bitbake/lib/bb/fetch2/gitannex.py index a9b69caa..80a808d8 100644 --- a/external/poky/bitbake/lib/bb/fetch2/gitannex.py +++ b/external/poky/bitbake/lib/bb/fetch2/gitannex.py @@ -1,5 +1,3 @@ -# ex:ts=4:sw=4:sts=4:et -# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- """ BitBake 'Fetch' git annex implementation """ @@ -7,24 +5,12 @@ BitBake 'Fetch' git annex implementation # Copyright (C) 2014 Otavio Salvador # Copyright (C) 2014 O.S. Systems Software LTDA. # -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. +# SPDX-License-Identifier: GPL-2.0-only # -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -import os import bb from bb.fetch2.git import Git from bb.fetch2 import runfetchcmd -from bb.fetch2 import logger class GitANNEX(Git): def supports(self, ud, d): diff --git a/external/poky/bitbake/lib/bb/fetch2/gitsm.py b/external/poky/bitbake/lib/bb/fetch2/gitsm.py index a07eb7e7..e7083001 100644 --- a/external/poky/bitbake/lib/bb/fetch2/gitsm.py +++ b/external/poky/bitbake/lib/bb/fetch2/gitsm.py @@ -1,5 +1,3 @@ -# ex:ts=4:sw=4:sts=4:et -# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- """ BitBake 'Fetch' git submodules implementation @@ -16,27 +14,18 @@ NOTE: Switching a SRC_URI from "git://" to "gitsm://" requires a clean of your r # Copyright (C) 2013 Richard Purdie # -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. +# SPDX-License-Identifier: GPL-2.0-only # -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. import os import bb import copy +import shutil +import tempfile from bb.fetch2.git import Git from bb.fetch2 import runfetchcmd from bb.fetch2 import logger from bb.fetch2 import Fetch -from bb.fetch2 import BBFetchException class GitSM(Git): def supports(self, ud, d): @@ -143,7 +132,7 @@ class GitSM(Git): ld.setVar('SRCPV', d.getVar('SRCPV')) ld.setVar('SRCREV_FORMAT', module) - function(ud, url, module, paths[module], ld) + function(ud, url, module, paths[module], workdir, ld) return submodules != [] @@ -165,7 +154,7 @@ class GitSM(Git): return False def download(self, ud, d): - def download_submodule(ud, url, module, modpath, d): + def download_submodule(ud, url, module, modpath, workdir, d): url += ";bareclone=1;nobranch=1" # Is the following still needed? @@ -176,16 +165,25 @@ class GitSM(Git): newfetch.download() # Drop a nugget to add each of the srcrevs we've fetched (used by need_update) runfetchcmd("%s config --add bitbake.srcrev %s" % \ - (ud.basecmd, ud.revisions[ud.names[0]]), d, workdir=ud.clonedir) + (ud.basecmd, ud.revisions[ud.names[0]]), d, workdir=workdir) except Exception as e: logger.error('gitsm: submodule download failed: %s %s' % (type(e).__name__, str(e))) raise Git.download(self, ud, d) - self.process_submodules(ud, ud.clonedir, download_submodule, d) + + # If we're using a shallow mirror tarball it needs to be unpacked + # temporarily so that we can examine the .gitmodules file + if ud.shallow and os.path.exists(ud.fullshallow) and self.need_update(ud, d): + tmpdir = tempfile.mkdtemp(dir=d.getVar("DL_DIR")) + runfetchcmd("tar -xzf %s" % ud.fullshallow, d, workdir=tmpdir) + self.process_submodules(ud, tmpdir, download_submodule, d) + shutil.rmtree(tmpdir) + else: + self.process_submodules(ud, ud.clonedir, download_submodule, d) def unpack(self, ud, destdir, d): - def unpack_submodules(ud, url, module, modpath, d): + def unpack_submodules(ud, url, module, modpath, workdir, d): url += ";bareclone=1;nobranch=1" # Figure out where we clone over the bare submodules... diff --git a/external/poky/bitbake/lib/bb/fetch2/hg.py b/external/poky/bitbake/lib/bb/fetch2/hg.py index 936d0431..8f503701 100644 --- a/external/poky/bitbake/lib/bb/fetch2/hg.py +++ b/external/poky/bitbake/lib/bb/fetch2/hg.py @@ -1,5 +1,3 @@ -# ex:ts=4:sw=4:sts=4:et -# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- """ BitBake 'Fetch' implementation for mercurial DRCS (hg). @@ -9,24 +7,12 @@ BitBake 'Fetch' implementation for mercurial DRCS (hg). # Copyright (C) 2004 Marcin Juszkiewicz # Copyright (C) 2007 Robert Schuster # -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# SPDX-License-Identifier: GPL-2.0-only # # Based on functions from the base bb module, Copyright 2003 Holger Schurig +# import os -import sys -import logging import bb import errno from bb.fetch2 import FetchMethod @@ -66,13 +52,6 @@ class Hg(FetchMethod): else: ud.proto = "hg" - ud.setup_revisions(d) - - if 'rev' in ud.parm: - ud.revision = ud.parm['rev'] - elif not ud.revision: - ud.revision = self.latest_revision(ud, d) - # Create paths to mercurial checkouts hgsrcname = '%s_%s_%s' % (ud.module.replace('/', '.'), \ ud.host, ud.path.replace('/', '.')) @@ -86,6 +65,13 @@ class Hg(FetchMethod): ud.localfile = ud.moddir ud.basecmd = d.getVar("FETCHCMD_hg") or "/usr/bin/env hg" + ud.setup_revisions(d) + + if 'rev' in ud.parm: + ud.revision = ud.parm['rev'] + elif not ud.revision: + ud.revision = self.latest_revision(ud, d) + ud.write_tarballs = d.getVar("BB_GENERATE_MIRROR_TARBALLS") def need_update(self, ud, d): @@ -99,7 +85,7 @@ class Hg(FetchMethod): def try_premirror(self, ud, d): # If we don't do this, updating an existing checkout with only premirrors # is not possible - if d.getVar("BB_FETCH_PREMIRRORONLY") is not None: + if bb.utils.to_boolean(d.getVar("BB_FETCH_PREMIRRORONLY")): return True if os.path.exists(ud.moddir): return False @@ -151,7 +137,7 @@ class Hg(FetchMethod): cmd = "%s --config auth.default.prefix=* --config auth.default.username=%s --config auth.default.password=%s --config \"auth.default.schemes=%s\" pull" % (ud.basecmd, ud.user, ud.pswd, proto) else: cmd = "%s pull" % (ud.basecmd) - elif command == "update": + elif command == "update" or command == "up": if ud.user and ud.pswd: cmd = "%s --config auth.default.prefix=* --config auth.default.username=%s --config auth.default.password=%s --config \"auth.default.schemes=%s\" update -C %s" % (ud.basecmd, ud.user, ud.pswd, proto, " ".join(options)) else: @@ -259,12 +245,19 @@ class Hg(FetchMethod): scmdata = ud.parm.get("scmdata", "") if scmdata != "nokeep": + proto = ud.parm.get('protocol', 'http') if not os.access(os.path.join(codir, '.hg'), os.R_OK): logger.debug(2, "Unpack: creating new hg repository in '" + codir + "'") runfetchcmd("%s init %s" % (ud.basecmd, codir), d) logger.debug(2, "Unpack: updating source in '" + codir + "'") - runfetchcmd("%s pull %s" % (ud.basecmd, ud.moddir), d, workdir=codir) - runfetchcmd("%s up -C %s" % (ud.basecmd, revflag), d, workdir=codir) + if ud.user and ud.pswd: + runfetchcmd("%s --config auth.default.prefix=* --config auth.default.username=%s --config auth.default.password=%s --config \"auth.default.schemes=%s\" pull %s" % (ud.basecmd, ud.user, ud.pswd, proto, ud.moddir), d, workdir=codir) + else: + runfetchcmd("%s pull %s" % (ud.basecmd, ud.moddir), d, workdir=codir) + if ud.user and ud.pswd: + runfetchcmd("%s --config auth.default.prefix=* --config auth.default.username=%s --config auth.default.password=%s --config \"auth.default.schemes=%s\" up -C %s" % (ud.basecmd, ud.user, ud.pswd, proto, revflag), d, workdir=codir) + else: + runfetchcmd("%s up -C %s" % (ud.basecmd, revflag), d, workdir=codir) else: logger.debug(2, "Unpack: extracting source to '" + codir + "'") runfetchcmd("%s archive -t files %s %s" % (ud.basecmd, revflag, codir), d, workdir=ud.moddir) diff --git a/external/poky/bitbake/lib/bb/fetch2/local.py b/external/poky/bitbake/lib/bb/fetch2/local.py index a114ac12..01d9ff9f 100644 --- a/external/poky/bitbake/lib/bb/fetch2/local.py +++ b/external/poky/bitbake/lib/bb/fetch2/local.py @@ -1,5 +1,3 @@ -# ex:ts=4:sw=4:sts=4:et -# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- """ BitBake 'Fetch' implementations @@ -10,20 +8,10 @@ BitBake build tools. # Copyright (C) 2003, 2004 Chris Larson # -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# SPDX-License-Identifier: GPL-2.0-only # # Based on functions from the base bb module, Copyright 2003 Holger Schurig +# import os import urllib.request, urllib.parse, urllib.error diff --git a/external/poky/bitbake/lib/bb/fetch2/npm.py b/external/poky/bitbake/lib/bb/fetch2/npm.py index 408dfc3d..47898509 100644 --- a/external/poky/bitbake/lib/bb/fetch2/npm.py +++ b/external/poky/bitbake/lib/bb/fetch2/npm.py @@ -1,308 +1,296 @@ -# ex:ts=4:sw=4:sts=4:et -# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# Copyright (C) 2020 Savoir-Faire Linux +# +# SPDX-License-Identifier: GPL-2.0-only +# """ -BitBake 'Fetch' NPM implementation +BitBake 'Fetch' npm implementation -The NPM fetcher is used to retrieve files from the npmjs repository +npm fetcher support the SRC_URI with format of: +SRC_URI = "npm://some.registry.url;OptionA=xxx;OptionB=xxx;..." -Usage in the recipe: +Supported SRC_URI options are: - SRC_URI = "npm://registry.npmjs.org/;name=${PN};version=${PV}" - Suported SRC_URI options are: +- package + The npm package name. This is a mandatory parameter. - - name - - version +- version + The npm package version. This is a mandatory parameter. - npm://registry.npmjs.org/${PN}/-/${PN}-${PV}.tgz would become npm://registry.npmjs.org;name=${PN};version=${PV} - The fetcher all triggers off the existence of ud.localpath. If that exists and has the ".done" stamp, its assumed the fetch is good/done +- downloadfilename + Specifies the filename used when storing the downloaded file. +- destsuffix + Specifies the directory to use to unpack the package (default: npm). """ -import os -import sys -import urllib.request, urllib.parse, urllib.error +import base64 import json -import subprocess -import signal +import os +import re +import shlex +import tempfile import bb -from bb.fetch2 import FetchMethod -from bb.fetch2 import FetchError -from bb.fetch2 import ChecksumError -from bb.fetch2 import runfetchcmd -from bb.fetch2 import logger -from bb.fetch2 import UnpackError -from bb.fetch2 import ParameterError - -def subprocess_setup(): - # Python installs a SIGPIPE handler by default. This is usually not what - # non-Python subprocesses expect. - # SIGPIPE errors are known issues with gzip/bash - signal.signal(signal.SIGPIPE, signal.SIG_DFL) +from bb.fetch2 import Fetch +from bb.fetch2 import FetchError +from bb.fetch2 import FetchMethod +from bb.fetch2 import MissingParameterError +from bb.fetch2 import ParameterError +from bb.fetch2 import URI +from bb.fetch2 import check_network_access +from bb.fetch2 import runfetchcmd +from bb.utils import is_semver + +def npm_package(package): + """Convert the npm package name to remove unsupported character""" + # Scoped package names (with the @) use the same naming convention + # as the 'npm pack' command. + if package.startswith("@"): + return re.sub("/", "-", package[1:]) + return package + +def npm_filename(package, version): + """Get the filename of a npm package""" + return npm_package(package) + "-" + version + ".tgz" + +def npm_localfile(package, version): + """Get the local filename of a npm package""" + return os.path.join("npm2", npm_filename(package, version)) + +def npm_integrity(integrity): + """ + Get the checksum name and expected value from the subresource integrity + https://www.w3.org/TR/SRI/ + """ + algo, value = integrity.split("-", maxsplit=1) + return "%ssum" % algo, base64.b64decode(value).hex() + +def npm_unpack(tarball, destdir, d): + """Unpack a npm tarball""" + bb.utils.mkdirhier(destdir) + cmd = "tar --extract --gzip --file=%s" % shlex.quote(tarball) + cmd += " --no-same-owner" + cmd += " --strip-components=1" + runfetchcmd(cmd, d, workdir=destdir) + +class NpmEnvironment(object): + """ + Using a npm config file seems more reliable than using cli arguments. + This class allows to create a controlled environment for npm commands. + """ + def __init__(self, d, configs=None): + self.d = d + self.configs = configs + + def run(self, cmd, args=None, configs=None, workdir=None): + """Run npm command in a controlled environment""" + with tempfile.TemporaryDirectory() as tmpdir: + d = bb.data.createCopy(self.d) + d.setVar("HOME", tmpdir) + + cfgfile = os.path.join(tmpdir, "npmrc") + + if not workdir: + workdir = tmpdir + + def _run(cmd): + cmd = "NPM_CONFIG_USERCONFIG=%s " % cfgfile + cmd + cmd = "NPM_CONFIG_GLOBALCONFIG=%s " % cfgfile + cmd + return runfetchcmd(cmd, d, workdir=workdir) + + if self.configs: + for key, value in self.configs: + _run("npm config set %s %s" % (key, shlex.quote(value))) + + if configs: + for key, value in configs: + _run("npm config set %s %s" % (key, shlex.quote(value))) + + if args: + for key, value in args: + cmd += " --%s=%s" % (key, shlex.quote(value)) + + return _run(cmd) class Npm(FetchMethod): - - """Class to fetch urls via 'npm'""" - def init(self, d): - pass + """Class to fetch a package from a npm registry""" def supports(self, ud, d): - """ - Check to see if a given url can be fetched with npm - """ - return ud.type in ['npm'] + """Check if a given url can be fetched with npm""" + return ud.type in ["npm"] + + def urldata_init(self, ud, d): + """Init npm specific variables within url data""" + ud.package = None + ud.version = None + ud.registry = None - def debug(self, msg): - logger.debug(1, "NpmFetch: %s", msg) + # Get the 'package' parameter + if "package" in ud.parm: + ud.package = ud.parm.get("package") - def clean(self, ud, d): - logger.debug(2, "Calling cleanup %s" % ud.pkgname) - bb.utils.remove(ud.localpath, False) - bb.utils.remove(ud.pkgdatadir, True) - bb.utils.remove(ud.fullmirror, False) + if not ud.package: + raise MissingParameterError("Parameter 'package' required", ud.url) + + # Get the 'version' parameter + if "version" in ud.parm: + ud.version = ud.parm.get("version") - def urldata_init(self, ud, d): - """ - init NPM specific variable within url data - """ - if 'downloadfilename' in ud.parm: - ud.basename = ud.parm['downloadfilename'] - else: - ud.basename = os.path.basename(ud.path) - - # can't call it ud.name otherwise fetcher base class will start doing sha1stuff - # TODO: find a way to get an sha1/sha256 manifest of pkg & all deps - ud.pkgname = ud.parm.get("name", None) - if not ud.pkgname: - raise ParameterError("NPM fetcher requires a name parameter", ud.url) - ud.version = ud.parm.get("version", None) if not ud.version: - raise ParameterError("NPM fetcher requires a version parameter", ud.url) - ud.bbnpmmanifest = "%s-%s.deps.json" % (ud.pkgname, ud.version) - ud.bbnpmmanifest = ud.bbnpmmanifest.replace('/', '-') - ud.registry = "http://%s" % (ud.url.replace('npm://', '', 1).split(';'))[0] - prefixdir = "npm/%s" % ud.pkgname - ud.pkgdatadir = d.expand("${DL_DIR}/%s" % prefixdir) - if not os.path.exists(ud.pkgdatadir): - bb.utils.mkdirhier(ud.pkgdatadir) - ud.localpath = d.expand("${DL_DIR}/npm/%s" % ud.bbnpmmanifest) - - self.basecmd = d.getVar("FETCHCMD_wget") or "/usr/bin/env wget -O -t 2 -T 30 -nv --passive-ftp --no-check-certificate " - ud.prefixdir = prefixdir - - ud.write_tarballs = ((d.getVar("BB_GENERATE_MIRROR_TARBALLS") or "0") != "0") - mirrortarball = 'npm_%s-%s.tar.xz' % (ud.pkgname, ud.version) - mirrortarball = mirrortarball.replace('/', '-') - ud.fullmirror = os.path.join(d.getVar("DL_DIR"), mirrortarball) - ud.mirrortarballs = [mirrortarball] + raise MissingParameterError("Parameter 'version' required", ud.url) - def need_update(self, ud, d): - if os.path.exists(ud.localpath): - return False - return True - - def _runwget(self, ud, d, command, quiet): - logger.debug(2, "Fetching %s using command '%s'" % (ud.url, command)) - bb.fetch2.check_network_access(d, command, ud.url) - dldir = d.getVar("DL_DIR") - runfetchcmd(command, d, quiet, workdir=dldir) - - def _unpackdep(self, ud, pkg, data, destdir, dldir, d): - file = data[pkg]['tgz'] - logger.debug(2, "file to extract is %s" % file) - if file.endswith('.tgz') or file.endswith('.tar.gz') or file.endswith('.tar.Z'): - cmd = 'tar xz --strip 1 --no-same-owner --warning=no-unknown-keyword -f %s/%s' % (dldir, file) - else: - bb.fatal("NPM package %s downloaded not a tarball!" % file) - - # Change to subdir before executing command - if not os.path.exists(destdir): - os.makedirs(destdir) - path = d.getVar('PATH') - if path: - cmd = "PATH=\"%s\" %s" % (path, cmd) - bb.note("Unpacking %s to %s/" % (file, destdir)) - ret = subprocess.call(cmd, preexec_fn=subprocess_setup, shell=True, cwd=destdir) - - if ret != 0: - raise UnpackError("Unpack command %s failed with return value %s" % (cmd, ret), ud.url) - - if 'deps' not in data[pkg]: - return - for dep in data[pkg]['deps']: - self._unpackdep(ud, dep, data[pkg]['deps'], "%s/node_modules/%s" % (destdir, dep), dldir, d) - - - def unpack(self, ud, destdir, d): - dldir = d.getVar("DL_DIR") - with open("%s/npm/%s" % (dldir, ud.bbnpmmanifest)) as datafile: - workobj = json.load(datafile) - dldir = "%s/%s" % (os.path.dirname(ud.localpath), ud.pkgname) - - if 'subdir' in ud.parm: - unpackdir = '%s/%s' % (destdir, ud.parm.get('subdir')) + if not is_semver(ud.version) and not ud.version == "latest": + raise ParameterError("Invalid 'version' parameter", ud.url) + + # Extract the 'registry' part of the url + ud.registry = re.sub(r"^npm://", "http://", ud.url.split(";")[0]) + + # Using the 'downloadfilename' parameter as local filename + # or the npm package name. + if "downloadfilename" in ud.parm: + ud.localfile = d.expand(ud.parm["downloadfilename"]) else: - unpackdir = '%s/npmpkg' % destdir - - self._unpackdep(ud, ud.pkgname, workobj, unpackdir, dldir, d) - - def _parse_view(self, output): - ''' - Parse the output of npm view --json; the last JSON result - is assumed to be the one that we're interested in. - ''' - pdata = None - outdeps = {} - datalines = [] - bracelevel = 0 - for line in output.splitlines(): - if bracelevel: - datalines.append(line) - elif '{' in line: - datalines = [] - datalines.append(line) - bracelevel = bracelevel + line.count('{') - line.count('}') - if datalines: - pdata = json.loads('\n'.join(datalines)) - return pdata - - def _getdependencies(self, pkg, data, version, d, ud, optional=False, fetchedlist=None): - if fetchedlist is None: - fetchedlist = [] - pkgfullname = pkg - if version != '*' and not '/' in version: - pkgfullname += "@'%s'" % version - logger.debug(2, "Calling getdeps on %s" % pkg) - fetchcmd = "npm view %s --json --registry %s" % (pkgfullname, ud.registry) - output = runfetchcmd(fetchcmd, d, True) - pdata = self._parse_view(output) - if not pdata: - raise FetchError("The command '%s' returned no output" % fetchcmd) - if optional: - pkg_os = pdata.get('os', None) - if pkg_os: - if not isinstance(pkg_os, list): - pkg_os = [pkg_os] - blacklist = False - for item in pkg_os: - if item.startswith('!'): - blacklist = True - break - if (not blacklist and 'linux' not in pkg_os) or '!linux' in pkg_os: - logger.debug(2, "Skipping %s since it's incompatible with Linux" % pkg) - return - #logger.debug(2, "Output URL is %s - %s - %s" % (ud.basepath, ud.basename, ud.localfile)) - outputurl = pdata['dist']['tarball'] - data[pkg] = {} - data[pkg]['tgz'] = os.path.basename(outputurl) - if outputurl in fetchedlist: - return - - self._runwget(ud, d, "%s --directory-prefix=%s %s" % (self.basecmd, ud.prefixdir, outputurl), False) - fetchedlist.append(outputurl) - - dependencies = pdata.get('dependencies', {}) - optionalDependencies = pdata.get('optionalDependencies', {}) - dependencies.update(optionalDependencies) - depsfound = {} - optdepsfound = {} - data[pkg]['deps'] = {} - for dep in dependencies: - if dep in optionalDependencies: - optdepsfound[dep] = dependencies[dep] + ud.localfile = npm_localfile(ud.package, ud.version) + + # Get the base 'npm' command + ud.basecmd = d.getVar("FETCHCMD_npm") or "npm" + + # This fetcher resolves a URI from a npm package name and version and + # then forwards it to a proxy fetcher. A resolve file containing the + # resolved URI is created to avoid unwanted network access (if the file + # already exists). The management of the donestamp file, the lockfile + # and the checksums are forwarded to the proxy fetcher. + ud.proxy = None + ud.needdonestamp = False + ud.resolvefile = self.localpath(ud, d) + ".resolved" + + def _resolve_proxy_url(self, ud, d): + def _npm_view(): + configs = [] + configs.append(("json", "true")) + configs.append(("registry", ud.registry)) + pkgver = shlex.quote(ud.package + "@" + ud.version) + cmd = ud.basecmd + " view %s" % pkgver + env = NpmEnvironment(d) + check_network_access(d, cmd, ud.registry) + view_string = env.run(cmd, configs=configs) + + if not view_string: + raise FetchError("Unavailable package %s" % pkgver, ud.url) + + try: + view = json.loads(view_string) + + error = view.get("error") + if error is not None: + raise FetchError(error.get("summary"), ud.url) + + if ud.version == "latest": + bb.warn("The npm package %s is using the latest " \ + "version available. This could lead to " \ + "non-reproducible builds." % pkgver) + elif ud.version != view.get("version"): + raise ParameterError("Invalid 'version' parameter", ud.url) + + return view + + except Exception as e: + raise FetchError("Invalid view from npm: %s" % str(e), ud.url) + + def _get_url(view): + tarball_url = view.get("dist", {}).get("tarball") + + if tarball_url is None: + raise FetchError("Invalid 'dist.tarball' in view", ud.url) + + uri = URI(tarball_url) + uri.params["downloadfilename"] = ud.localfile + + integrity = view.get("dist", {}).get("integrity") + shasum = view.get("dist", {}).get("shasum") + + if integrity is not None: + checksum_name, checksum_expected = npm_integrity(integrity) + uri.params[checksum_name] = checksum_expected + elif shasum is not None: + uri.params["sha1sum"] = shasum else: - depsfound[dep] = dependencies[dep] - for dep, version in optdepsfound.items(): - self._getdependencies(dep, data[pkg]['deps'], version, d, ud, optional=True, fetchedlist=fetchedlist) - for dep, version in depsfound.items(): - self._getdependencies(dep, data[pkg]['deps'], version, d, ud, fetchedlist=fetchedlist) - - def _getshrinkeddependencies(self, pkg, data, version, d, ud, lockdown, manifest, toplevel=True): - logger.debug(2, "NPM shrinkwrap file is %s" % data) - if toplevel: - name = data.get('name', None) - if name and name != pkg: - for obj in data.get('dependencies', []): - if obj == pkg: - self._getshrinkeddependencies(obj, data['dependencies'][obj], data['dependencies'][obj]['version'], d, ud, lockdown, manifest, False) - return - outputurl = "invalid" - if ('resolved' not in data) or (not data['resolved'].startswith('http')): - # will be the case for ${PN} - fetchcmd = "npm view %s@%s dist.tarball --registry %s" % (pkg, version, ud.registry) - logger.debug(2, "Found this matching URL: %s" % str(fetchcmd)) - outputurl = runfetchcmd(fetchcmd, d, True) - else: - outputurl = data['resolved'] - self._runwget(ud, d, "%s --directory-prefix=%s %s" % (self.basecmd, ud.prefixdir, outputurl), False) - manifest[pkg] = {} - manifest[pkg]['tgz'] = os.path.basename(outputurl).rstrip() - manifest[pkg]['deps'] = {} - - if pkg in lockdown: - sha1_expected = lockdown[pkg][version] - sha1_data = bb.utils.sha1_file("npm/%s/%s" % (ud.pkgname, manifest[pkg]['tgz'])) - if sha1_expected != sha1_data: - msg = "\nFile: '%s' has %s checksum %s when %s was expected" % (manifest[pkg]['tgz'], 'sha1', sha1_data, sha1_expected) - raise ChecksumError('Checksum mismatch!%s' % msg) - else: - logger.debug(2, "No lockdown data for %s@%s" % (pkg, version)) + raise FetchError("Invalid 'dist.integrity' in view", ud.url) - if 'dependencies' in data: - for obj in data['dependencies']: - logger.debug(2, "Found dep is %s" % str(obj)) - self._getshrinkeddependencies(obj, data['dependencies'][obj], data['dependencies'][obj]['version'], d, ud, lockdown, manifest[pkg]['deps'], False) + return str(uri) + + url = _get_url(_npm_view()) + + bb.utils.mkdirhier(os.path.dirname(ud.resolvefile)) + with open(ud.resolvefile, "w") as f: + f.write(url) + + def _setup_proxy(self, ud, d): + if ud.proxy is None: + if not os.path.exists(ud.resolvefile): + self._resolve_proxy_url(ud, d) + + with open(ud.resolvefile, "r") as f: + url = f.read() + + # Avoid conflicts between the environment data and: + # - the proxy url checksum + data = bb.data.createCopy(d) + data.delVarFlags("SRC_URI") + ud.proxy = Fetch([url], data) + + def _get_proxy_method(self, ud, d): + self._setup_proxy(ud, d) + proxy_url = ud.proxy.urls[0] + proxy_ud = ud.proxy.ud[proxy_url] + proxy_d = ud.proxy.d + proxy_ud.setup_localpath(proxy_d) + return proxy_ud.method, proxy_ud, proxy_d + + def verify_donestamp(self, ud, d): + """Verify the donestamp file""" + proxy_m, proxy_ud, proxy_d = self._get_proxy_method(ud, d) + return proxy_m.verify_donestamp(proxy_ud, proxy_d) + + def update_donestamp(self, ud, d): + """Update the donestamp file""" + proxy_m, proxy_ud, proxy_d = self._get_proxy_method(ud, d) + proxy_m.update_donestamp(proxy_ud, proxy_d) + + def need_update(self, ud, d): + """Force a fetch, even if localpath exists ?""" + if not os.path.exists(ud.resolvefile): + return True + if ud.version == "latest": + return True + proxy_m, proxy_ud, proxy_d = self._get_proxy_method(ud, d) + return proxy_m.need_update(proxy_ud, proxy_d) + + def try_mirrors(self, fetch, ud, d, mirrors): + """Try to use a mirror""" + proxy_m, proxy_ud, proxy_d = self._get_proxy_method(ud, d) + return proxy_m.try_mirrors(fetch, proxy_ud, proxy_d, mirrors) def download(self, ud, d): """Fetch url""" - jsondepobj = {} - shrinkobj = {} - lockdown = {} - - if not os.listdir(ud.pkgdatadir) and os.path.exists(ud.fullmirror): - dest = d.getVar("DL_DIR") - bb.utils.mkdirhier(dest) - runfetchcmd("tar -xJf %s" % (ud.fullmirror), d, workdir=dest) - return - - if ud.parm.get("noverify", None) != '1': - shwrf = d.getVar('NPM_SHRINKWRAP') - logger.debug(2, "NPM shrinkwrap file is %s" % shwrf) - if shwrf: - try: - with open(shwrf) as datafile: - shrinkobj = json.load(datafile) - except Exception as e: - raise FetchError('Error loading NPM_SHRINKWRAP file "%s" for %s: %s' % (shwrf, ud.pkgname, str(e))) - elif not ud.ignore_checksums: - logger.warning('Missing shrinkwrap file in NPM_SHRINKWRAP for %s, this will lead to unreliable builds!' % ud.pkgname) - lckdf = d.getVar('NPM_LOCKDOWN') - logger.debug(2, "NPM lockdown file is %s" % lckdf) - if lckdf: - try: - with open(lckdf) as datafile: - lockdown = json.load(datafile) - except Exception as e: - raise FetchError('Error loading NPM_LOCKDOWN file "%s" for %s: %s' % (lckdf, ud.pkgname, str(e))) - elif not ud.ignore_checksums: - logger.warning('Missing lockdown file in NPM_LOCKDOWN for %s, this will lead to unreproducible builds!' % ud.pkgname) - - if ('name' not in shrinkobj): - self._getdependencies(ud.pkgname, jsondepobj, ud.version, d, ud) - else: - self._getshrinkeddependencies(ud.pkgname, shrinkobj, ud.version, d, ud, lockdown, jsondepobj) - - with open(ud.localpath, 'w') as outfile: - json.dump(jsondepobj, outfile) - - def build_mirror_data(self, ud, d): - # Generate a mirror tarball if needed - if ud.write_tarballs and not os.path.exists(ud.fullmirror): - # it's possible that this symlink points to read-only filesystem with PREMIRROR - if os.path.islink(ud.fullmirror): - os.unlink(ud.fullmirror) - - dldir = d.getVar("DL_DIR") - logger.info("Creating tarball of npm data") - runfetchcmd("tar -cJf %s npm/%s npm/%s" % (ud.fullmirror, ud.bbnpmmanifest, ud.pkgname), d, - workdir=dldir) - runfetchcmd("touch %s.done" % (ud.fullmirror), d, workdir=dldir) + self._setup_proxy(ud, d) + ud.proxy.download() + + def unpack(self, ud, rootdir, d): + """Unpack the downloaded archive""" + destsuffix = ud.parm.get("destsuffix", "npm") + destdir = os.path.join(rootdir, destsuffix) + npm_unpack(ud.localpath, destdir, d) + + def clean(self, ud, d): + """Clean any existing full or partial download""" + if os.path.exists(ud.resolvefile): + self._setup_proxy(ud, d) + ud.proxy.clean() + bb.utils.remove(ud.resolvefile) + + def done(self, ud, d): + """Is the download done ?""" + if not os.path.exists(ud.resolvefile): + return False + proxy_m, proxy_ud, proxy_d = self._get_proxy_method(ud, d) + return proxy_m.done(proxy_ud, proxy_d) diff --git a/external/poky/bitbake/lib/bb/fetch2/npmsw.py b/external/poky/bitbake/lib/bb/fetch2/npmsw.py new file mode 100644 index 00000000..0c3511d8 --- /dev/null +++ b/external/poky/bitbake/lib/bb/fetch2/npmsw.py @@ -0,0 +1,255 @@ +# Copyright (C) 2020 Savoir-Faire Linux +# +# SPDX-License-Identifier: GPL-2.0-only +# +""" +BitBake 'Fetch' npm shrinkwrap implementation + +npm fetcher support the SRC_URI with format of: +SRC_URI = "npmsw://some.registry.url;OptionA=xxx;OptionB=xxx;..." + +Supported SRC_URI options are: + +- dev + Set to 1 to also install devDependencies. + +- destsuffix + Specifies the directory to use to unpack the dependencies (default: ${S}). +""" + +import json +import os +import re +import bb +from bb.fetch2 import Fetch +from bb.fetch2 import FetchMethod +from bb.fetch2 import ParameterError +from bb.fetch2 import URI +from bb.fetch2.npm import npm_integrity +from bb.fetch2.npm import npm_localfile +from bb.fetch2.npm import npm_unpack +from bb.utils import is_semver + +def foreach_dependencies(shrinkwrap, callback=None, dev=False): + """ + Run a callback for each dependencies of a shrinkwrap file. + The callback is using the format: + callback(name, params, deptree) + with: + name = the package name (string) + params = the package parameters (dictionary) + deptree = the package dependency tree (array of strings) + """ + def _walk_deps(deps, deptree): + for name in deps: + subtree = [*deptree, name] + _walk_deps(deps[name].get("dependencies", {}), subtree) + if callback is not None: + if deps[name].get("dev", False) and not dev: + continue + elif deps[name].get("bundled", False): + continue + callback(name, deps[name], subtree) + + _walk_deps(shrinkwrap.get("dependencies", {}), []) + +class NpmShrinkWrap(FetchMethod): + """Class to fetch all package from a shrinkwrap file""" + + def supports(self, ud, d): + """Check if a given url can be fetched with npmsw""" + return ud.type in ["npmsw"] + + def urldata_init(self, ud, d): + """Init npmsw specific variables within url data""" + + # Get the 'shrinkwrap' parameter + ud.shrinkwrap_file = re.sub(r"^npmsw://", "", ud.url.split(";")[0]) + + # Get the 'dev' parameter + ud.dev = bb.utils.to_boolean(ud.parm.get("dev"), False) + + # Resolve the dependencies + ud.deps = [] + + def _resolve_dependency(name, params, deptree): + url = None + localpath = None + extrapaths = [] + destsubdirs = [os.path.join("node_modules", dep) for dep in deptree] + destsuffix = os.path.join(*destsubdirs) + + integrity = params.get("integrity", None) + resolved = params.get("resolved", None) + version = params.get("version", None) + + # Handle registry sources + if is_semver(version) and resolved and integrity: + localfile = npm_localfile(name, version) + + uri = URI(resolved) + uri.params["downloadfilename"] = localfile + + checksum_name, checksum_expected = npm_integrity(integrity) + uri.params[checksum_name] = checksum_expected + + url = str(uri) + + localpath = os.path.join(d.getVar("DL_DIR"), localfile) + + # Create a resolve file to mimic the npm fetcher and allow + # re-usability of the downloaded file. + resolvefile = localpath + ".resolved" + + bb.utils.mkdirhier(os.path.dirname(resolvefile)) + with open(resolvefile, "w") as f: + f.write(url) + + extrapaths.append(resolvefile) + + # Handle http tarball sources + elif version.startswith("http") and integrity: + localfile = os.path.join("npm2", os.path.basename(version)) + + uri = URI(version) + uri.params["downloadfilename"] = localfile + + checksum_name, checksum_expected = npm_integrity(integrity) + uri.params[checksum_name] = checksum_expected + + url = str(uri) + + localpath = os.path.join(d.getVar("DL_DIR"), localfile) + + # Handle git sources + elif version.startswith("git"): + regex = re.compile(r""" + ^ + git\+ + (?P<protocol>[a-z]+) + :// + (?P<url>[^#]+) + \# + (?P<rev>[0-9a-f]+) + $ + """, re.VERBOSE) + + match = regex.match(version) + + if not match: + raise ParameterError("Invalid git url: %s" % version, ud.url) + + groups = match.groupdict() + + uri = URI("git://" + str(groups["url"])) + uri.params["protocol"] = str(groups["protocol"]) + uri.params["rev"] = str(groups["rev"]) + uri.params["destsuffix"] = destsuffix + + url = str(uri) + + # local tarball sources and local link sources are unsupported + else: + raise ParameterError("Unsupported dependency: %s" % name, ud.url) + + ud.deps.append({ + "url": url, + "localpath": localpath, + "extrapaths": extrapaths, + "destsuffix": destsuffix, + }) + + try: + with open(ud.shrinkwrap_file, "r") as f: + shrinkwrap = json.load(f) + except Exception as e: + raise ParameterError("Invalid shrinkwrap file: %s" % str(e), ud.url) + + foreach_dependencies(shrinkwrap, _resolve_dependency, ud.dev) + + # Avoid conflicts between the environment data and: + # - the proxy url revision + # - the proxy url checksum + data = bb.data.createCopy(d) + data.delVar("SRCREV") + data.delVarFlags("SRC_URI") + + # This fetcher resolves multiple URIs from a shrinkwrap file and then + # forwards it to a proxy fetcher. The management of the donestamp file, + # the lockfile and the checksums are forwarded to the proxy fetcher. + ud.proxy = Fetch([dep["url"] for dep in ud.deps], data) + ud.needdonestamp = False + + @staticmethod + def _foreach_proxy_method(ud, handle): + returns = [] + for proxy_url in ud.proxy.urls: + proxy_ud = ud.proxy.ud[proxy_url] + proxy_d = ud.proxy.d + proxy_ud.setup_localpath(proxy_d) + returns.append(handle(proxy_ud.method, proxy_ud, proxy_d)) + return returns + + def verify_donestamp(self, ud, d): + """Verify the donestamp file""" + def _handle(m, ud, d): + return m.verify_donestamp(ud, d) + return all(self._foreach_proxy_method(ud, _handle)) + + def update_donestamp(self, ud, d): + """Update the donestamp file""" + def _handle(m, ud, d): + m.update_donestamp(ud, d) + self._foreach_proxy_method(ud, _handle) + + def need_update(self, ud, d): + """Force a fetch, even if localpath exists ?""" + def _handle(m, ud, d): + return m.need_update(ud, d) + return all(self._foreach_proxy_method(ud, _handle)) + + def try_mirrors(self, fetch, ud, d, mirrors): + """Try to use a mirror""" + def _handle(m, ud, d): + return m.try_mirrors(fetch, ud, d, mirrors) + return all(self._foreach_proxy_method(ud, _handle)) + + def download(self, ud, d): + """Fetch url""" + ud.proxy.download() + + def unpack(self, ud, rootdir, d): + """Unpack the downloaded dependencies""" + destdir = d.getVar("S") + destsuffix = ud.parm.get("destsuffix") + if destsuffix: + destdir = os.path.join(rootdir, destsuffix) + + bb.utils.mkdirhier(destdir) + bb.utils.copyfile(ud.shrinkwrap_file, + os.path.join(destdir, "npm-shrinkwrap.json")) + + auto = [dep["url"] for dep in ud.deps if not dep["localpath"]] + manual = [dep for dep in ud.deps if dep["localpath"]] + + if auto: + ud.proxy.unpack(destdir, auto) + + for dep in manual: + depdestdir = os.path.join(destdir, dep["destsuffix"]) + npm_unpack(dep["localpath"], depdestdir, d) + + def clean(self, ud, d): + """Clean any existing full or partial download""" + ud.proxy.clean() + + # Clean extra files + for dep in ud.deps: + for path in dep["extrapaths"]: + bb.utils.remove(path) + + def done(self, ud, d): + """Is the download done ?""" + def _handle(m, ud, d): + return m.done(ud, d) + return all(self._foreach_proxy_method(ud, _handle)) diff --git a/external/poky/bitbake/lib/bb/fetch2/osc.py b/external/poky/bitbake/lib/bb/fetch2/osc.py index 6c60456b..8f091efd 100644 --- a/external/poky/bitbake/lib/bb/fetch2/osc.py +++ b/external/poky/bitbake/lib/bb/fetch2/osc.py @@ -1,13 +1,12 @@ -# ex:ts=4:sw=4:sts=4:et -# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- +# +# SPDX-License-Identifier: GPL-2.0-only +# """ Bitbake "Fetch" implementation for osc (Opensuse build service client). Based on the svn "Fetch" implementation. """ -import os -import sys import logging import bb from bb.fetch2 import FetchMethod @@ -42,7 +41,7 @@ class Osc(FetchMethod): else: pv = d.getVar("PV", False) rev = bb.fetch2.srcrev_internal_helper(ud, d) - if rev and rev != True: + if rev: ud.revision = rev else: ud.revision = "" diff --git a/external/poky/bitbake/lib/bb/fetch2/perforce.py b/external/poky/bitbake/lib/bb/fetch2/perforce.py index 903a8e61..f57c2a4f 100644 --- a/external/poky/bitbake/lib/bb/fetch2/perforce.py +++ b/external/poky/bitbake/lib/bb/fetch2/perforce.py @@ -1,5 +1,3 @@ -# ex:ts=4:sw=4:sts=4:et -# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- """ BitBake 'Fetch' implementation for perforce @@ -8,23 +6,11 @@ BitBake 'Fetch' implementation for perforce # Copyright (C) 2003, 2004 Chris Larson # Copyright (C) 2016 Kodak Alaris, Inc. # -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# SPDX-License-Identifier: GPL-2.0-only # # Based on functions from the base bb module, Copyright 2003 Holger Schurig import os -import logging import bb from bb.fetch2 import FetchMethod from bb.fetch2 import FetchError @@ -118,7 +104,7 @@ class Perforce(FetchMethod): if command == 'changes': p4cmd = '%s%s changes -m 1 //%s' % (ud.basecmd, p4opt, pathnrev) elif command == 'print': - if depot_filename != None: + if depot_filename is not None: p4cmd = '%s%s print -o "p4/%s" "%s"' % (ud.basecmd, p4opt, filename, depot_filename) else: raise FetchError('No depot file name provided to p4 %s' % command, ud.url) diff --git a/external/poky/bitbake/lib/bb/fetch2/repo.py b/external/poky/bitbake/lib/bb/fetch2/repo.py index 8c7e8185..2bdbbd40 100644 --- a/external/poky/bitbake/lib/bb/fetch2/repo.py +++ b/external/poky/bitbake/lib/bb/fetch2/repo.py @@ -1,5 +1,3 @@ -# ex:ts=4:sw=4:sts=4:et -# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- """ BitBake "Fetch" repo (git) implementation @@ -8,20 +6,10 @@ BitBake "Fetch" repo (git) implementation # Copyright (C) 2009 Tom Rini <trini@embeddedalley.com> # # Based on git.py which is: -#Copyright (C) 2005 Richard Purdie +# Copyright (C) 2005 Richard Purdie # -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. +# SPDX-License-Identifier: GPL-2.0-only # -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. import os import bb diff --git a/external/poky/bitbake/lib/bb/fetch2/s3.py b/external/poky/bitbake/lib/bb/fetch2/s3.py index 16292886..ffca73c8 100644 --- a/external/poky/bitbake/lib/bb/fetch2/s3.py +++ b/external/poky/bitbake/lib/bb/fetch2/s3.py @@ -1,5 +1,3 @@ -# ex:ts=4:sw=4:sts=4:et -# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- """ BitBake 'Fetch' implementation for Amazon AWS S3. @@ -13,18 +11,7 @@ The aws tool must be correctly installed and configured prior to use. # Based in part on bb.fetch2.wget: # Copyright (C) 2003, 2004 Chris Larson # -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# SPDX-License-Identifier: GPL-2.0-only # # Based on functions from the base bb module, Copyright 2003 Holger Schurig diff --git a/external/poky/bitbake/lib/bb/fetch2/sftp.py b/external/poky/bitbake/lib/bb/fetch2/sftp.py index 81884a6a..f87f292e 100644 --- a/external/poky/bitbake/lib/bb/fetch2/sftp.py +++ b/external/poky/bitbake/lib/bb/fetch2/sftp.py @@ -1,5 +1,3 @@ -# ex:ts=4:sw=4:sts=4:et -# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- """ BitBake SFTP Fetch implementation @@ -44,18 +42,7 @@ SRC_URI = "sftp://user@host.example.com/dir/path.file.txt" # Based in part on bb.fetch2.wget: # Copyright (C) 2003, 2004 Chris Larson # -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# SPDX-License-Identifier: GPL-2.0-only # # Based on functions from the base bb module, Copyright 2003 Holger Schurig diff --git a/external/poky/bitbake/lib/bb/fetch2/ssh.py b/external/poky/bitbake/lib/bb/fetch2/ssh.py index 6047ee41..5e982ecf 100644 --- a/external/poky/bitbake/lib/bb/fetch2/ssh.py +++ b/external/poky/bitbake/lib/bb/fetch2/ssh.py @@ -1,5 +1,3 @@ -# ex:ts=4:sw=4:sts=4:et -# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- ''' BitBake 'Fetch' implementations @@ -29,23 +27,11 @@ IETF secsh internet draft: # Copyright 2003 Holger Schurig # # -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. +# SPDX-License-Identifier: GPL-2.0-only # -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. import re, os from bb.fetch2 import FetchMethod -from bb.fetch2 import FetchError -from bb.fetch2 import logger from bb.fetch2 import runfetchcmd @@ -72,7 +58,7 @@ class SSH(FetchMethod): '''Class to fetch a module or modules via Secure Shell''' def supports(self, urldata, d): - return __pattern__.match(urldata.url) != None + return __pattern__.match(urldata.url) is not None def supports_checksum(self, urldata): return False diff --git a/external/poky/bitbake/lib/bb/fetch2/svn.py b/external/poky/bitbake/lib/bb/fetch2/svn.py index 9dcf3eb0..971a5add 100644 --- a/external/poky/bitbake/lib/bb/fetch2/svn.py +++ b/external/poky/bitbake/lib/bb/fetch2/svn.py @@ -1,5 +1,3 @@ -# ex:ts=4:sw=4:sts=4:et -# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- """ BitBake 'Fetch' implementation for svn. @@ -8,24 +6,11 @@ BitBake 'Fetch' implementation for svn. # Copyright (C) 2003, 2004 Chris Larson # Copyright (C) 2004 Marcin Juszkiewicz # -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# SPDX-License-Identifier: GPL-2.0-only # # Based on functions from the base bb module, Copyright 2003 Holger Schurig import os -import sys -import logging import bb import re from bb.fetch2 import FetchMethod @@ -62,7 +47,7 @@ class Svn(FetchMethod): svndir = d.getVar("SVNDIR") or (d.getVar("DL_DIR") + "/svn") relpath = self._strip_leading_slashes(ud.path) ud.pkgdir = os.path.join(svndir, ud.host, relpath) - ud.moddir = os.path.join(ud.pkgdir, ud.module) + ud.moddir = os.path.join(ud.pkgdir, ud.path_spec) # Protects the repository from concurrent updates, e.g. from two # recipes fetching different revisions at the same time ud.svnlock = os.path.join(ud.pkgdir, "svn.lock") @@ -104,6 +89,13 @@ class Svn(FetchMethod): svncmd = "%s log --limit 1 %s %s://%s/%s/" % (ud.basecmd, " ".join(options), proto, svnroot, ud.module) else: suffix = "" + + # externals may be either 'allowed' or 'nowarn', but not both. Allowed + # will not issue a warning, but will log to the debug buffer what has likely + # been downloaded by SVN. + if not ("externals" in ud.parm and ud.parm["externals"] == "allowed"): + options.append("--ignore-externals") + if ud.revision: options.append("-r %s" % ud.revision) suffix = "@%s" % (ud.revision) @@ -130,24 +122,36 @@ class Svn(FetchMethod): try: if os.access(os.path.join(ud.moddir, '.svn'), os.R_OK): - svnupdatecmd = self._buildsvncommand(ud, d, "update") + svncmd = self._buildsvncommand(ud, d, "update") logger.info("Update " + ud.url) # We need to attempt to run svn upgrade first in case its an older working format try: runfetchcmd(ud.basecmd + " upgrade", d, workdir=ud.moddir) except FetchError: pass - logger.debug(1, "Running %s", svnupdatecmd) - bb.fetch2.check_network_access(d, svnupdatecmd, ud.url) - runfetchcmd(svnupdatecmd, d, workdir=ud.moddir) + logger.debug(1, "Running %s", svncmd) + bb.fetch2.check_network_access(d, svncmd, ud.url) + runfetchcmd(svncmd, d, workdir=ud.moddir) else: - svnfetchcmd = self._buildsvncommand(ud, d, "fetch") + svncmd = self._buildsvncommand(ud, d, "fetch") logger.info("Fetch " + ud.url) # check out sources there bb.utils.mkdirhier(ud.pkgdir) - logger.debug(1, "Running %s", svnfetchcmd) - bb.fetch2.check_network_access(d, svnfetchcmd, ud.url) - runfetchcmd(svnfetchcmd, d, workdir=ud.pkgdir) + logger.debug(1, "Running %s", svncmd) + bb.fetch2.check_network_access(d, svncmd, ud.url) + runfetchcmd(svncmd, d, workdir=ud.pkgdir) + + if not ("externals" in ud.parm and ud.parm["externals"] == "nowarn"): + # Warn the user if this had externals (won't catch them all) + output = runfetchcmd("svn propget svn:externals || true", d, workdir=ud.moddir) + if output: + if "--ignore-externals" in svncmd.split(): + bb.warn("%s contains svn:externals." % ud.url) + bb.warn("These should be added to the recipe SRC_URI as necessary.") + bb.warn("svn fetch has ignored externals:\n%s" % output) + bb.warn("To disable this warning add ';externals=nowarn' to the url.") + else: + bb.debug(1, "svn repository has externals:\n%s" % output) scmdata = ud.parm.get("scmdata", "") if scmdata == "keep": diff --git a/external/poky/bitbake/lib/bb/fetch2/wget.py b/external/poky/bitbake/lib/bb/fetch2/wget.py index 8f505b6d..f7d1de26 100644 --- a/external/poky/bitbake/lib/bb/fetch2/wget.py +++ b/external/poky/bitbake/lib/bb/fetch2/wget.py @@ -1,5 +1,3 @@ -# ex:ts=4:sw=4:sts=4:et -# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- """ BitBake 'Fetch' implementations @@ -10,29 +8,19 @@ BitBake build tools. # Copyright (C) 2003, 2004 Chris Larson # -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# SPDX-License-Identifier: GPL-2.0-only # # Based on functions from the base bb module, Copyright 2003 Holger Schurig +import shlex import re import tempfile -import subprocess import os -import logging import errno import bb import bb.progress +import socket +import http.client import urllib.request, urllib.parse, urllib.error from bb.fetch2 import FetchMethod from bb.fetch2 import FetchError @@ -104,9 +92,9 @@ class Wget(FetchMethod): fetchcmd = self.basecmd if 'downloadfilename' in ud.parm: - dldir = d.getVar("DL_DIR") - bb.utils.mkdirhier(os.path.dirname(dldir + os.sep + ud.localfile)) - fetchcmd += " -O " + dldir + os.sep + ud.localfile + localpath = os.path.join(d.getVar("DL_DIR"), ud.localfile) + bb.utils.mkdirhier(os.path.dirname(localpath)) + fetchcmd += " -O %s" % shlex.quote(localpath) if ud.user and ud.pswd: fetchcmd += " --user=%s --password=%s --auth-no-challenge" % (ud.user, ud.pswd) @@ -132,10 +120,6 @@ class Wget(FetchMethod): return True def checkstatus(self, fetch, ud, d, try_again=True): - import urllib.request, urllib.error, urllib.parse, socket, http.client - from urllib.response import addinfourl - from bb.fetch2 import FetchConnectionCache - class HTTPConnectionCache(http.client.HTTPConnection): if fetch.connection_cache: def connect(self): @@ -168,7 +152,7 @@ class Wget(FetchMethod): """ host = req.host if not host: - raise urlllib2.URLError('no host given') + raise urllib.error.URLError('no host given') h = http_class(host, timeout=req.timeout) # will parse host:port h.set_debuglevel(self._debuglevel) @@ -185,7 +169,7 @@ class Wget(FetchMethod): # request. # Don't close connection when connection_cache is enabled, - if fetch.connection_cache is None: + if fetch.connection_cache is None: headers["Connection"] = "close" else: headers["Connection"] = "Keep-Alive" # Works for HTTP/1.0 @@ -252,7 +236,7 @@ class Wget(FetchMethod): pass closed = False - resp = addinfourl(fp_dummy(), r.msg, req.get_full_url()) + resp = urllib.response.addinfourl(fp_dummy(), r.msg, req.get_full_url()) resp.code = r.status resp.msg = r.reason @@ -271,17 +255,18 @@ class Wget(FetchMethod): fp.read() fp.close() - newheaders = dict((k,v) for k,v in list(req.headers.items()) - if k.lower() not in ("content-length", "content-type")) - return self.parent.open(urllib.request.Request(req.get_full_url(), - headers=newheaders, - origin_req_host=req.origin_req_host, - unverifiable=True)) + if req.get_method() != 'GET': + newheaders = dict((k, v) for k, v in list(req.headers.items()) + if k.lower() not in ("content-length", "content-type")) + return self.parent.open(urllib.request.Request(req.get_full_url(), + headers=newheaders, + origin_req_host=req.origin_req_host, + unverifiable=True)) - """ - Some servers (e.g. GitHub archives, hosted on Amazon S3) return 403 - Forbidden when they actually mean 405 Method Not Allowed. - """ + raise urllib.request.HTTPError(req, code, msg, headers, None) + + # Some servers (e.g. GitHub archives, hosted on Amazon S3) return 403 + # Forbidden when they actually mean 405 Method Not Allowed. http_error_403 = http_error_405 @@ -292,15 +277,15 @@ class Wget(FetchMethod): """ def redirect_request(self, req, fp, code, msg, headers, newurl): newreq = urllib.request.HTTPRedirectHandler.redirect_request(self, req, fp, code, msg, headers, newurl) - newreq.get_method = lambda: req.get_method() + newreq.get_method = req.get_method return newreq exported_proxies = export_proxies(d) handlers = [FixedHTTPRedirectHandler, HTTPMethodFallback] - if export_proxies: + if exported_proxies: handlers.append(urllib.request.ProxyHandler()) handlers.append(CacheHTTPHandler()) - # XXX: Since Python 2.7.9 ssl cert validation is enabled by default + # Since Python 2.7.9 ssl cert validation is enabled by default # see PEP-0476, this causes verification errors on some https servers # so disable by default. import ssl @@ -315,23 +300,24 @@ class Wget(FetchMethod): # Some servers (FusionForge, as used on Alioth) require that the # optional Accept header is set. r.add_header("Accept", "*/*") + r.add_header("User-Agent", "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.2.12) Gecko/20101027 Ubuntu/9.10 (karmic) Firefox/3.6.12") def add_basic_auth(login_str, request): '''Adds Basic auth to http request, pass in login:password as string''' import base64 encodeuser = base64.b64encode(login_str.encode('utf-8')).decode("utf-8") - authheader = "Basic %s" % encodeuser + authheader = "Basic %s" % encodeuser r.add_header("Authorization", authheader) - if ud.user: - add_basic_auth(ud.user, r) + if ud.user and ud.pswd: + add_basic_auth(ud.user + ':' + ud.pswd, r) try: - import netrc, urllib.parse + import netrc n = netrc.netrc() login, unused, password = n.authenticators(urllib.parse.urlparse(uri).hostname) add_basic_auth("%s:%s" % (login, password), r) except (TypeError, ImportError, IOError, netrc.NetrcParseError): - pass + pass with opener.open(r) as response: pass @@ -396,18 +382,14 @@ class Wget(FetchMethod): (oldpn, oldpv, oldsuffix) = old (newpn, newpv, newsuffix) = new - """ - Check for a new suffix type that we have never heard of before - """ - if (newsuffix): + # Check for a new suffix type that we have never heard of before + if newsuffix: m = self.suffix_regex_comp.search(newsuffix) if not m: bb.warn("%s has a possible unknown suffix: %s" % (newpn, newsuffix)) return False - """ - Not our package so ignore it - """ + # Not our package so ignore it if oldpn != newpn: return False @@ -473,15 +455,14 @@ class Wget(FetchMethod): return "" - def _check_latest_version_by_dir(self, dirver, package, package_regex, - current_version, ud, d): + def _check_latest_version_by_dir(self, dirver, package, package_regex, current_version, ud, d): """ - Scan every directory in order to get upstream version. + Scan every directory in order to get upstream version. """ version_dir = ['', '', ''] version = ['', '', ''] - dirver_regex = re.compile("(?P<pfx>\D*)(?P<ver>(\d+[\.\-_])+(\d+))") + dirver_regex = re.compile(r"(?P<pfx>\D*)(?P<ver>(\d+[\.\-_])+(\d+))") s = dirver_regex.search(dirver) if s: version_dir[1] = s.group('ver') @@ -541,26 +522,26 @@ class Wget(FetchMethod): gst-fluendo-mp3 """ # match most patterns which uses "-" as separator to version digits - pn_prefix1 = "[a-zA-Z][a-zA-Z0-9]*([-_][a-zA-Z]\w+)*\+?[-_]" + pn_prefix1 = r"[a-zA-Z][a-zA-Z0-9]*([-_][a-zA-Z]\w+)*\+?[-_]" # a loose pattern such as for unzip552.tar.gz - pn_prefix2 = "[a-zA-Z]+" + pn_prefix2 = r"[a-zA-Z]+" # a loose pattern such as for 80325-quicky-0.4.tar.gz - pn_prefix3 = "[0-9]+[-]?[a-zA-Z]+" + pn_prefix3 = r"[0-9]+[-]?[a-zA-Z]+" # Save the Package Name (pn) Regex for use later - pn_regex = "(%s|%s|%s)" % (pn_prefix1, pn_prefix2, pn_prefix3) + pn_regex = r"(%s|%s|%s)" % (pn_prefix1, pn_prefix2, pn_prefix3) # match version - pver_regex = "(([A-Z]*\d+[a-zA-Z]*[\.\-_]*)+)" + pver_regex = r"(([A-Z]*\d+[a-zA-Z]*[\.\-_]*)+)" # match arch parch_regex = "-source|_all_" # src.rpm extension was added only for rpm package. Can be removed if the rpm # packaged will always be considered as having to be manually upgraded - psuffix_regex = "(tar\.gz|tgz|tar\.bz2|zip|xz|tar\.lz|rpm|bz2|orig\.tar\.gz|tar\.xz|src\.tar\.gz|src\.tgz|svnr\d+\.tar\.bz2|stable\.tar\.gz|src\.rpm)" + psuffix_regex = r"(tar\.gz|tgz|tar\.bz2|zip|xz|tar\.lz|rpm|bz2|orig\.tar\.gz|tar\.xz|src\.tar\.gz|src\.tgz|svnr\d+\.tar\.bz2|stable\.tar\.gz|src\.rpm)" # match name, version and archive type of a package - package_regex_comp = re.compile("(?P<name>%s?\.?v?)(?P<pver>%s)(?P<arch>%s)?[\.-](?P<type>%s$)" + package_regex_comp = re.compile(r"(?P<name>%s?\.?v?)(?P<pver>%s)(?P<arch>%s)?[\.-](?P<type>%s$)" % (pn_regex, pver_regex, parch_regex, psuffix_regex)) self.suffix_regex_comp = re.compile(psuffix_regex) @@ -572,7 +553,7 @@ class Wget(FetchMethod): version = self._parse_path(package_regex_comp, package) if version: package_custom_regex_comp = re.compile( - "(?P<name>%s)(?P<pver>%s)(?P<arch>%s)?[\.-](?P<type>%s)" % + r"(?P<name>%s)(?P<pver>%s)(?P<arch>%s)?[\.-](?P<type>%s)" % (re.escape(version[0]), pver_regex, parch_regex, psuffix_regex)) else: package_custom_regex_comp = None @@ -589,7 +570,7 @@ class Wget(FetchMethod): current_version = ['', d.getVar('PV'), ''] """possible to have no version in pkg name, such as spectrum-fw""" - if not re.search("\d+", package): + if not re.search(r"\d+", package): current_version[1] = re.sub('_', '.', current_version[1]) current_version[1] = re.sub('-', '.', current_version[1]) return (current_version[1], '') @@ -607,13 +588,13 @@ class Wget(FetchMethod): # search for version matches on folders inside the path, like: # "5.7" in http://download.gnome.org/sources/${PN}/5.7/${PN}-${PV}.tar.gz - dirver_regex = re.compile("(?P<dirver>[^/]*(\d+\.)*\d+([-_]r\d+)*)/") + dirver_regex = re.compile(r"(?P<dirver>[^/]*(\d+\.)*\d+([-_]r\d+)*)/") m = dirver_regex.search(path) if m: pn = d.getVar('PN') dirver = m.group('dirver') - dirver_pn_regex = re.compile("%s\d?" % (re.escape(pn))) + dirver_pn_regex = re.compile(r"%s\d?" % (re.escape(pn))) if not dirver_pn_regex.search(dirver): return (self._check_latest_version_by_dir(dirver, package, package_regex, current_version, ud, d), '') |