diff options
Diffstat (limited to 'external/poky/bitbake/lib/bb/tests/fetch.py')
| -rw-r--r-- | external/poky/bitbake/lib/bb/tests/fetch.py | 757 |
1 files changed, 717 insertions, 40 deletions
diff --git a/external/poky/bitbake/lib/bb/tests/fetch.py b/external/poky/bitbake/lib/bb/tests/fetch.py index 57376c44..29c96b2b 100644 --- a/external/poky/bitbake/lib/bb/tests/fetch.py +++ b/external/poky/bitbake/lib/bb/tests/fetch.py @@ -1,33 +1,20 @@ -# ex:ts=4:sw=4:sts=4:et -# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- # # BitBake Tests for the Fetcher (fetch2/) # # Copyright (C) 2012 Richard Purdie # -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# SPDX-License-Identifier: GPL-2.0-only # import unittest import hashlib import tempfile -import subprocess import collections import os from bb.fetch2 import URI from bb.fetch2 import FetchMethod import bb +from bb.tests.support.httpserver import HTTPService def skipIfNoNetwork(): if os.environ.get("BB_SKIP_NETTESTS") == "yes": @@ -912,6 +899,7 @@ class FetcherNetworkTest(FetcherTest): if os.path.exists(os.path.join(repo_path, 'bitbake-gitsm-test1')): self.assertTrue(os.path.exists(os.path.join(repo_path, 'bitbake-gitsm-test1', 'bitbake')), msg='submodule of submodule missing') + @skipIfNoNetwork() def test_git_submodule_dbus_broker(self): # The following external repositories have show failures in fetch and unpack operations # We want to avoid regressions! @@ -929,6 +917,7 @@ class FetcherNetworkTest(FetcherTest): self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-sundry/config')), msg='Missing submodule config "subprojects/c-sundry"') self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/subprojects/c-utf8/config')), msg='Missing submodule config "subprojects/c-utf8"') + @skipIfNoNetwork() def test_git_submodule_CLI11(self): url = "gitsm://github.com/CLIUtils/CLI11;protocol=git;rev=bd4dc911847d0cde7a6b41dfa626a85aab213baf" fetcher = bb.fetch.Fetch([url], self.d) @@ -942,6 +931,7 @@ class FetcherNetworkTest(FetcherTest): self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/json/config')), msg='Missing submodule config "extern/json"') self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/sanitizers/config')), msg='Missing submodule config "extern/sanitizers"') + @skipIfNoNetwork() def test_git_submodule_update_CLI11(self): """ Prevent regression on update detection not finding missing submodule, or modules without needed commits """ url = "gitsm://github.com/CLIUtils/CLI11;protocol=git;rev=cf6a99fa69aaefe477cc52e3ef4a7d2d7fa40714" @@ -961,6 +951,7 @@ class FetcherNetworkTest(FetcherTest): self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/json/config')), msg='Missing submodule config "extern/json"') self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/extern/sanitizers/config')), msg='Missing submodule config "extern/sanitizers"') + @skipIfNoNetwork() def test_git_submodule_aktualizr(self): url = "gitsm://github.com/advancedtelematic/aktualizr;branch=master;protocol=git;rev=d00d1a04cc2366d1a5f143b84b9f507f8bd32c44" fetcher = bb.fetch.Fetch([url], self.d) @@ -977,6 +968,7 @@ class FetcherNetworkTest(FetcherTest): self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/third_party/googletest/config')), msg='Missing submodule config "third_party/googletest/config"') self.assertTrue(os.path.exists(os.path.join(repo_path, '.git/modules/third_party/HdrHistogram_c/config')), msg='Missing submodule config "third_party/HdrHistogram_c/config"') + @skipIfNoNetwork() def test_git_submodule_iotedge(self): """ Prevent regression on deeply nested submodules not being checked out properly, even though they were fetched. """ @@ -1004,6 +996,86 @@ class FetcherNetworkTest(FetcherTest): self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/testtools/umock-c/deps/ctest/README.md')), msg='Missing submodule checkout') self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/testtools/umock-c/deps/testrunner/readme.md')), msg='Missing submodule checkout') +class SVNTest(FetcherTest): + def skipIfNoSvn(): + import shutil + if not shutil.which("svn"): + return unittest.skip("svn not installed, tests being skipped") + + if not shutil.which("svnadmin"): + return unittest.skip("svnadmin not installed, tests being skipped") + + return lambda f: f + + @skipIfNoSvn() + def setUp(self): + """ Create a local repository """ + + super(SVNTest, self).setUp() + + # Create something we can fetch + src_dir = tempfile.mkdtemp(dir=self.tempdir, + prefix='svnfetch_srcdir_') + src_dir = os.path.abspath(src_dir) + bb.process.run("echo readme > README.md", cwd=src_dir) + + # Store it in a local SVN repository + repo_dir = tempfile.mkdtemp(dir=self.tempdir, + prefix='svnfetch_localrepo_') + repo_dir = os.path.abspath(repo_dir) + bb.process.run("svnadmin create project", cwd=repo_dir) + + self.repo_url = "file://%s/project" % repo_dir + bb.process.run("svn import --non-interactive -m 'Initial import' %s %s/trunk" % (src_dir, self.repo_url), + cwd=repo_dir) + + bb.process.run("svn co %s svnfetch_co" % self.repo_url, cwd=self.tempdir) + # Github will emulate SVN. Use this to check if we're downloding... + bb.process.run("svn propset svn:externals 'bitbake svn://vcs.pcre.org/pcre2/code' .", + cwd=os.path.join(self.tempdir, 'svnfetch_co', 'trunk')) + bb.process.run("svn commit --non-interactive -m 'Add external'", + cwd=os.path.join(self.tempdir, 'svnfetch_co', 'trunk')) + + self.src_dir = src_dir + self.repo_dir = repo_dir + + @skipIfNoSvn() + def tearDown(self): + os.chdir(self.origdir) + if os.environ.get("BB_TMPDIR_NOCLEAN") == "yes": + print("Not cleaning up %s. Please remove manually." % self.tempdir) + else: + bb.utils.prunedir(self.tempdir) + + @skipIfNoSvn() + @skipIfNoNetwork() + def test_noexternal_svn(self): + # Always match the rev count from setUp (currently rev 2) + url = "svn://%s;module=trunk;protocol=file;rev=2" % self.repo_url.replace('file://', '') + fetcher = bb.fetch.Fetch([url], self.d) + fetcher.download() + os.chdir(os.path.dirname(self.unpackdir)) + fetcher.unpack(self.unpackdir) + + self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk')), msg="Missing trunk") + self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk', 'README.md')), msg="Missing contents") + self.assertFalse(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/trunk')), msg="External dir should NOT exist") + self.assertFalse(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/trunk', 'README')), msg="External README should NOT exit") + + @skipIfNoSvn() + def test_external_svn(self): + # Always match the rev count from setUp (currently rev 2) + url = "svn://%s;module=trunk;protocol=file;externals=allowed;rev=2" % self.repo_url.replace('file://', '') + fetcher = bb.fetch.Fetch([url], self.d) + fetcher.download() + os.chdir(os.path.dirname(self.unpackdir)) + fetcher.unpack(self.unpackdir) + + self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk')), msg="Missing trunk") + self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk', 'README.md')), msg="Missing contents") + self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/trunk')), msg="External dir should exist") + self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/trunk', 'README')), msg="External README should exit") + class TrustedNetworksTest(FetcherTest): def test_trusted_network(self): # Ensure trusted_network returns False when the host IS in the list. @@ -1080,7 +1152,8 @@ class FetchLatestVersionTest(FetcherTest): ("mx-1.0", "git://github.com/clutter-project/mx.git;branch=mx-1.4", "9b1db6b8060bd00b121a692f942404a24ae2960f", "") : "1.99.4", # version pattern "vX.Y" - ("mtd-utils", "git://git.infradead.org/mtd-utils.git", "ca39eb1d98e736109c64ff9c1aa2a6ecca222d8f", "") + # mirror of git.infradead.org since network issues interfered with testing + ("mtd-utils", "git://git.yoctoproject.org/mtd-utils.git", "ca39eb1d98e736109c64ff9c1aa2a6ecca222d8f", "") : "1.5.0", # version pattern "pkg_name-X.Y" ("presentproto", "git://anongit.freedesktop.org/git/xorg/proto/presentproto", "24f3a56e541b0a9e6c6ee76081f441221a120ef9", "") @@ -1108,27 +1181,46 @@ class FetchLatestVersionTest(FetcherTest): } test_wget_uris = { + # # packages with versions inside directory name - ("util-linux", "http://kernel.org/pub/linux/utils/util-linux/v2.23/util-linux-2.24.2.tar.bz2", "", "") + # + # http://kernel.org/pub/linux/utils/util-linux/v2.23/util-linux-2.24.2.tar.bz2 + ("util-linux", "/pub/linux/utils/util-linux/v2.23/util-linux-2.24.2.tar.bz2", "", "") : "2.24.2", - ("enchant", "http://www.abisource.com/downloads/enchant/1.6.0/enchant-1.6.0.tar.gz", "", "") + # http://www.abisource.com/downloads/enchant/1.6.0/enchant-1.6.0.tar.gz + ("enchant", "/downloads/enchant/1.6.0/enchant-1.6.0.tar.gz", "", "") : "1.6.0", - ("cmake", "http://www.cmake.org/files/v2.8/cmake-2.8.12.1.tar.gz", "", "") + # http://www.cmake.org/files/v2.8/cmake-2.8.12.1.tar.gz + ("cmake", "/files/v2.8/cmake-2.8.12.1.tar.gz", "", "") : "2.8.12.1", + # # packages with versions only in current directory - ("eglic", "http://downloads.yoctoproject.org/releases/eglibc/eglibc-2.18-svnr23787.tar.bz2", "", "") + # + # http://downloads.yoctoproject.org/releases/eglibc/eglibc-2.18-svnr23787.tar.bz2 + ("eglic", "/releases/eglibc/eglibc-2.18-svnr23787.tar.bz2", "", "") : "2.19", - ("gnu-config", "http://downloads.yoctoproject.org/releases/gnu-config/gnu-config-20120814.tar.bz2", "", "") + # http://downloads.yoctoproject.org/releases/gnu-config/gnu-config-20120814.tar.bz2 + ("gnu-config", "/releases/gnu-config/gnu-config-20120814.tar.bz2", "", "") : "20120814", + # # packages with "99" in the name of possible version - ("pulseaudio", "http://freedesktop.org/software/pulseaudio/releases/pulseaudio-4.0.tar.xz", "", "") + # + # http://freedesktop.org/software/pulseaudio/releases/pulseaudio-4.0.tar.xz + ("pulseaudio", "/software/pulseaudio/releases/pulseaudio-4.0.tar.xz", "", "") : "5.0", - ("xserver-xorg", "http://xorg.freedesktop.org/releases/individual/xserver/xorg-server-1.15.1.tar.bz2", "", "") + # http://xorg.freedesktop.org/releases/individual/xserver/xorg-server-1.15.1.tar.bz2 + ("xserver-xorg", "/releases/individual/xserver/xorg-server-1.15.1.tar.bz2", "", "") : "1.15.1", + # # packages with valid UPSTREAM_CHECK_URI and UPSTREAM_CHECK_REGEX - ("cups", "http://www.cups.org/software/1.7.2/cups-1.7.2-source.tar.bz2", "https://github.com/apple/cups/releases", "(?P<name>cups\-)(?P<pver>((\d+[\.\-_]*)+))\-source\.tar\.gz") + # + # http://www.cups.org/software/1.7.2/cups-1.7.2-source.tar.bz2 + # https://github.com/apple/cups/releases + ("cups", "/software/1.7.2/cups-1.7.2-source.tar.bz2", "/apple/cups/releases", "(?P<name>cups\-)(?P<pver>((\d+[\.\-_]*)+))\-source\.tar\.gz") : "2.0.0", - ("db", "http://download.oracle.com/berkeley-db/db-5.3.21.tar.gz", "http://ftp.debian.org/debian/pool/main/d/db5.3/", "(?P<name>db5\.3_)(?P<pver>\d+(\.\d+)+).+\.orig\.tar\.xz") + # http://download.oracle.com/berkeley-db/db-5.3.21.tar.gz + # http://ftp.debian.org/debian/pool/main/d/db5.3/ + ("db", "/berkeley-db/db-5.3.21.tar.gz", "/debian/pool/main/d/db5.3/", "(?P<name>db5\.3_)(?P<pver>\d+(\.\d+)+).+\.orig\.tar\.xz") : "5.3.10", } @@ -1145,18 +1237,28 @@ class FetchLatestVersionTest(FetcherTest): r = bb.utils.vercmp_string(v, verstring) self.assertTrue(r == -1 or r == 0, msg="Package %s, version: %s <= %s" % (k[0], v, verstring)) - @skipIfNoNetwork() def test_wget_latest_versionstring(self): - for k, v in self.test_wget_uris.items(): - self.d.setVar("PN", k[0]) - self.d.setVar("UPSTREAM_CHECK_URI", k[2]) - self.d.setVar("UPSTREAM_CHECK_REGEX", k[3]) - ud = bb.fetch2.FetchData(k[1], self.d) - pupver = ud.method.latest_versionstring(ud, self.d) - verstring = pupver[0] - self.assertTrue(verstring, msg="Could not find upstream version for %s" % k[0]) - r = bb.utils.vercmp_string(v, verstring) - self.assertTrue(r == -1 or r == 0, msg="Package %s, version: %s <= %s" % (k[0], v, verstring)) + testdata = os.path.dirname(os.path.abspath(__file__)) + "/fetch-testdata" + server = HTTPService(testdata) + server.start() + port = server.port + try: + for k, v in self.test_wget_uris.items(): + self.d.setVar("PN", k[0]) + checkuri = "" + if k[2]: + checkuri = "http://localhost:%s/" % port + k[2] + self.d.setVar("UPSTREAM_CHECK_URI", checkuri) + self.d.setVar("UPSTREAM_CHECK_REGEX", k[3]) + url = "http://localhost:%s/" % port + k[1] + ud = bb.fetch2.FetchData(url, self.d) + pupver = ud.method.latest_versionstring(ud, self.d) + verstring = pupver[0] + self.assertTrue(verstring, msg="Could not find upstream version for %s" % k[0]) + r = bb.utils.vercmp_string(v, verstring) + self.assertTrue(r == -1 or r == 0, msg="Package %s, version: %s <= %s" % (k[0], v, verstring)) + finally: + server.stop() class FetchCheckStatusTest(FetcherTest): @@ -1384,7 +1486,7 @@ class GitShallowTest(FetcherTest): def fetch(self, uri=None): if uri is None: - uris = self.d.getVar('SRC_URI', True).split() + uris = self.d.getVar('SRC_URI').split() uri = uris[0] d = self.d else: @@ -1443,7 +1545,7 @@ class GitShallowTest(FetcherTest): srcrev = self.git('rev-parse HEAD', cwd=self.srcdir).strip() self.d.setVar('SRCREV', srcrev) - uri = self.d.getVar('SRC_URI', True).split()[0] + uri = self.d.getVar('SRC_URI').split()[0] uri = '%s;nobranch=1;bare=1' % uri self.fetch_shallow(uri) @@ -1590,6 +1692,47 @@ class GitShallowTest(FetcherTest): # Verify the submodule is also shallow self.assertRevCount(1, cwd=os.path.join(self.gitdir, 'gitsubmodule')) + def test_shallow_submodule_mirrors(self): + self.add_empty_file('a') + self.add_empty_file('b') + + smdir = os.path.join(self.tempdir, 'gitsubmodule') + bb.utils.mkdirhier(smdir) + self.git('init', cwd=smdir) + # Make this look like it was cloned from a remote... + self.git('config --add remote.origin.url "%s"' % smdir, cwd=smdir) + self.git('config --add remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*"', cwd=smdir) + self.add_empty_file('asub', cwd=smdir) + self.add_empty_file('bsub', cwd=smdir) + + self.git('submodule init', cwd=self.srcdir) + self.git('submodule add file://%s' % smdir, cwd=self.srcdir) + self.git('submodule update', cwd=self.srcdir) + self.git('commit -m submodule -a', cwd=self.srcdir) + + uri = 'gitsm://%s;protocol=file;subdir=${S}' % self.srcdir + + # Fetch once to generate the shallow tarball + fetcher, ud = self.fetch(uri) + + # Set up the mirror + mirrordir = os.path.join(self.tempdir, 'mirror') + os.rename(self.dldir, mirrordir) + self.d.setVar('PREMIRRORS', 'gitsm://.*/.* file://%s/\n' % mirrordir) + + # Fetch from the mirror + bb.utils.remove(self.dldir, recurse=True) + bb.utils.remove(self.gitdir, recurse=True) + self.fetch_and_unpack(uri) + + # Verify the main repository is shallow + self.assertRevCount(1) + + # Verify the gitsubmodule directory is present + assert os.listdir(os.path.join(self.gitdir, 'gitsubmodule')) + + # Verify the submodule is also shallow + self.assertRevCount(1, cwd=os.path.join(self.gitdir, 'gitsubmodule')) if any(os.path.exists(os.path.join(p, 'git-annex')) for p in os.environ.get('PATH').split(':')): def test_shallow_annex(self): @@ -1622,7 +1765,7 @@ class GitShallowTest(FetcherTest): self.add_empty_file('f') self.assertRevCount(7, cwd=self.srcdir) - uri = self.d.getVar('SRC_URI', True).split()[0] + uri = self.d.getVar('SRC_URI').split()[0] uri = '%s;branch=master,a_branch;name=master,a_branch' % uri self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0') @@ -1648,7 +1791,7 @@ class GitShallowTest(FetcherTest): self.add_empty_file('f') self.assertRevCount(7, cwd=self.srcdir) - uri = self.d.getVar('SRC_URI', True).split()[0] + uri = self.d.getVar('SRC_URI').split()[0] uri = '%s;branch=master,a_branch;name=master,a_branch' % uri self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0') @@ -1791,6 +1934,26 @@ class GitShallowTest(FetcherTest): with self.assertRaises(bb.fetch2.FetchError): self.fetch() + def test_shallow_fetch_missing_revs(self): + self.add_empty_file('a') + self.add_empty_file('b') + fetcher, ud = self.fetch(self.d.getVar('SRC_URI')) + self.git('tag v0.0 master', cwd=self.srcdir) + self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0') + self.d.setVar('BB_GIT_SHALLOW_REVS', 'v0.0') + self.fetch_shallow() + + def test_shallow_fetch_missing_revs_fails(self): + self.add_empty_file('a') + self.add_empty_file('b') + fetcher, ud = self.fetch(self.d.getVar('SRC_URI')) + self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0') + self.d.setVar('BB_GIT_SHALLOW_REVS', 'v0.0') + + with self.assertRaises(bb.fetch2.FetchError), self.assertLogs("BitBake.Fetcher", level="ERROR") as cm: + self.fetch_shallow() + self.assertIn("Unable to find revision v0.0 even from upstream", cm.output[0]) + @skipIfNoNetwork() def test_bitbake(self): self.git('remote add --mirror=fetch origin git://github.com/openembedded/bitbake', cwd=self.srcdir) @@ -1836,3 +1999,517 @@ class GitShallowTest(FetcherTest): dir = os.listdir(self.unpackdir + "/git/") self.assertIn("fstests.doap", dir) + +class GitLfsTest(FetcherTest): + def setUp(self): + FetcherTest.setUp(self) + + self.gitdir = os.path.join(self.tempdir, 'git') + self.srcdir = os.path.join(self.tempdir, 'gitsource') + + self.d.setVar('WORKDIR', self.tempdir) + self.d.setVar('S', self.gitdir) + self.d.delVar('PREMIRRORS') + self.d.delVar('MIRRORS') + + self.d.setVar('SRCREV', '${AUTOREV}') + self.d.setVar('AUTOREV', '${@bb.fetch2.get_autorev(d)}') + + bb.utils.mkdirhier(self.srcdir) + self.git('init', cwd=self.srcdir) + with open(os.path.join(self.srcdir, '.gitattributes'), 'wt') as attrs: + attrs.write('*.mp3 filter=lfs -text') + self.git(['add', '.gitattributes'], cwd=self.srcdir) + self.git(['commit', '-m', "attributes", '.gitattributes'], cwd=self.srcdir) + + def git(self, cmd, cwd=None): + if isinstance(cmd, str): + cmd = 'git ' + cmd + else: + cmd = ['git'] + cmd + if cwd is None: + cwd = self.gitdir + return bb.process.run(cmd, cwd=cwd)[0] + + def fetch(self, uri=None): + uris = self.d.getVar('SRC_URI').split() + uri = uris[0] + d = self.d + + fetcher = bb.fetch2.Fetch(uris, d) + fetcher.download() + ud = fetcher.ud[uri] + return fetcher, ud + + def test_lfs_enabled(self): + import shutil + + uri = 'git://%s;protocol=file;subdir=${S};lfs=1' % self.srcdir + self.d.setVar('SRC_URI', uri) + + fetcher, ud = self.fetch() + self.assertIsNotNone(ud.method._find_git_lfs) + + # If git-lfs can be found, the unpack should be successful + ud.method._find_git_lfs = lambda d: True + shutil.rmtree(self.gitdir, ignore_errors=True) + fetcher.unpack(self.d.getVar('WORKDIR')) + + # If git-lfs cannot be found, the unpack should throw an error + with self.assertRaises(bb.fetch2.FetchError): + ud.method._find_git_lfs = lambda d: False + shutil.rmtree(self.gitdir, ignore_errors=True) + fetcher.unpack(self.d.getVar('WORKDIR')) + + def test_lfs_disabled(self): + import shutil + + uri = 'git://%s;protocol=file;subdir=${S};lfs=0' % self.srcdir + self.d.setVar('SRC_URI', uri) + + fetcher, ud = self.fetch() + self.assertIsNotNone(ud.method._find_git_lfs) + + # If git-lfs can be found, the unpack should be successful + ud.method._find_git_lfs = lambda d: True + shutil.rmtree(self.gitdir, ignore_errors=True) + fetcher.unpack(self.d.getVar('WORKDIR')) + + # If git-lfs cannot be found, the unpack should be successful + ud.method._find_git_lfs = lambda d: False + shutil.rmtree(self.gitdir, ignore_errors=True) + fetcher.unpack(self.d.getVar('WORKDIR')) + +class NPMTest(FetcherTest): + def skipIfNoNpm(): + import shutil + if not shutil.which('npm'): + return unittest.skip('npm not installed, tests being skipped') + return lambda f: f + + @skipIfNoNpm() + @skipIfNoNetwork() + def test_npm(self): + url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0' + fetcher = bb.fetch.Fetch([url], self.d) + ud = fetcher.ud[fetcher.urls[0]] + fetcher.download() + self.assertTrue(os.path.exists(ud.localpath)) + self.assertTrue(os.path.exists(ud.localpath + '.done')) + self.assertTrue(os.path.exists(ud.resolvefile)) + fetcher.unpack(self.unpackdir) + unpackdir = os.path.join(self.unpackdir, 'npm') + self.assertTrue(os.path.exists(os.path.join(unpackdir, 'package.json'))) + + @skipIfNoNpm() + @skipIfNoNetwork() + def test_npm_bad_checksum(self): + url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0' + # Fetch once to get a tarball + fetcher = bb.fetch.Fetch([url], self.d) + ud = fetcher.ud[fetcher.urls[0]] + fetcher.download() + self.assertTrue(os.path.exists(ud.localpath)) + # Modify the tarball + bad = b'bad checksum' + with open(ud.localpath, 'wb') as f: + f.write(bad) + # Verify that the tarball is fetched again + fetcher.download() + badsum = hashlib.sha512(bad).hexdigest() + self.assertTrue(os.path.exists(ud.localpath + '_bad-checksum_' + badsum)) + self.assertTrue(os.path.exists(ud.localpath)) + + @skipIfNoNpm() + @skipIfNoNetwork() + def test_npm_premirrors(self): + url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0' + # Fetch once to get a tarball + fetcher = bb.fetch.Fetch([url], self.d) + ud = fetcher.ud[fetcher.urls[0]] + fetcher.download() + self.assertTrue(os.path.exists(ud.localpath)) + # Setup the mirror + mirrordir = os.path.join(self.tempdir, 'mirror') + bb.utils.mkdirhier(mirrordir) + os.replace(ud.localpath, os.path.join(mirrordir, os.path.basename(ud.localpath))) + self.d.setVar('PREMIRRORS', 'https?$://.*/.* file://%s/\n' % mirrordir) + self.d.setVar('BB_FETCH_PREMIRRORONLY', '1') + # Fetch again + self.assertFalse(os.path.exists(ud.localpath)) + fetcher.download() + self.assertTrue(os.path.exists(ud.localpath)) + + @skipIfNoNpm() + @skipIfNoNetwork() + def test_npm_mirrors(self): + # Fetch once to get a tarball + url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0' + fetcher = bb.fetch.Fetch([url], self.d) + ud = fetcher.ud[fetcher.urls[0]] + fetcher.download() + self.assertTrue(os.path.exists(ud.localpath)) + # Setup the mirror + mirrordir = os.path.join(self.tempdir, 'mirror') + bb.utils.mkdirhier(mirrordir) + os.replace(ud.localpath, os.path.join(mirrordir, os.path.basename(ud.localpath))) + self.d.setVar('MIRRORS', 'https?$://.*/.* file://%s/\n' % mirrordir) + # Update the resolved url to an invalid url + with open(ud.resolvefile, 'r') as f: + url = f.read() + uri = URI(url) + uri.path = '/invalid' + with open(ud.resolvefile, 'w') as f: + f.write(str(uri)) + # Fetch again + self.assertFalse(os.path.exists(ud.localpath)) + fetcher.download() + self.assertTrue(os.path.exists(ud.localpath)) + + @skipIfNoNpm() + @skipIfNoNetwork() + def test_npm_destsuffix_downloadfilename(self): + url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0;destsuffix=foo/bar;downloadfilename=foo-bar.tgz' + fetcher = bb.fetch.Fetch([url], self.d) + fetcher.download() + self.assertTrue(os.path.exists(os.path.join(self.dldir, 'foo-bar.tgz'))) + fetcher.unpack(self.unpackdir) + unpackdir = os.path.join(self.unpackdir, 'foo', 'bar') + self.assertTrue(os.path.exists(os.path.join(unpackdir, 'package.json'))) + + def test_npm_no_network_no_tarball(self): + url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0' + self.d.setVar('BB_NO_NETWORK', '1') + fetcher = bb.fetch.Fetch([url], self.d) + with self.assertRaises(bb.fetch2.NetworkAccess): + fetcher.download() + + @skipIfNoNpm() + @skipIfNoNetwork() + def test_npm_no_network_with_tarball(self): + url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0' + # Fetch once to get a tarball + fetcher = bb.fetch.Fetch([url], self.d) + fetcher.download() + # Disable network access + self.d.setVar('BB_NO_NETWORK', '1') + # Fetch again + fetcher.download() + fetcher.unpack(self.unpackdir) + unpackdir = os.path.join(self.unpackdir, 'npm') + self.assertTrue(os.path.exists(os.path.join(unpackdir, 'package.json'))) + + @skipIfNoNpm() + @skipIfNoNetwork() + def test_npm_registry_alternate(self): + url = 'npm://registry.freajs.org;package=@savoirfairelinux/node-server-example;version=1.0.0' + fetcher = bb.fetch.Fetch([url], self.d) + fetcher.download() + fetcher.unpack(self.unpackdir) + unpackdir = os.path.join(self.unpackdir, 'npm') + self.assertTrue(os.path.exists(os.path.join(unpackdir, 'package.json'))) + + @skipIfNoNpm() + @skipIfNoNetwork() + def test_npm_version_latest(self): + url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=latest' + fetcher = bb.fetch.Fetch([url], self.d) + fetcher.download() + fetcher.unpack(self.unpackdir) + unpackdir = os.path.join(self.unpackdir, 'npm') + self.assertTrue(os.path.exists(os.path.join(unpackdir, 'package.json'))) + + @skipIfNoNpm() + @skipIfNoNetwork() + def test_npm_registry_invalid(self): + url = 'npm://registry.invalid.org;package=@savoirfairelinux/node-server-example;version=1.0.0' + fetcher = bb.fetch.Fetch([url], self.d) + with self.assertRaises(bb.fetch2.FetchError): + fetcher.download() + + @skipIfNoNpm() + @skipIfNoNetwork() + def test_npm_package_invalid(self): + url = 'npm://registry.npmjs.org;package=@savoirfairelinux/invalid;version=1.0.0' + fetcher = bb.fetch.Fetch([url], self.d) + with self.assertRaises(bb.fetch2.FetchError): + fetcher.download() + + @skipIfNoNpm() + @skipIfNoNetwork() + def test_npm_version_invalid(self): + url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=invalid' + with self.assertRaises(bb.fetch2.ParameterError): + fetcher = bb.fetch.Fetch([url], self.d) + + @skipIfNoNpm() + @skipIfNoNetwork() + def test_npm_registry_none(self): + url = 'npm://;package=@savoirfairelinux/node-server-example;version=1.0.0' + with self.assertRaises(bb.fetch2.MalformedUrl): + fetcher = bb.fetch.Fetch([url], self.d) + + @skipIfNoNpm() + @skipIfNoNetwork() + def test_npm_package_none(self): + url = 'npm://registry.npmjs.org;version=1.0.0' + with self.assertRaises(bb.fetch2.MissingParameterError): + fetcher = bb.fetch.Fetch([url], self.d) + + @skipIfNoNpm() + @skipIfNoNetwork() + def test_npm_version_none(self): + url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example' + with self.assertRaises(bb.fetch2.MissingParameterError): + fetcher = bb.fetch.Fetch([url], self.d) + + def create_shrinkwrap_file(self, data): + import json + datadir = os.path.join(self.tempdir, 'data') + swfile = os.path.join(datadir, 'npm-shrinkwrap.json') + bb.utils.mkdirhier(datadir) + with open(swfile, 'w') as f: + json.dump(data, f) + # Also configure the S directory + self.sdir = os.path.join(self.unpackdir, 'S') + self.d.setVar('S', self.sdir) + return swfile + + @skipIfNoNpm() + @skipIfNoNetwork() + def test_npmsw(self): + swfile = self.create_shrinkwrap_file({ + 'dependencies': { + 'array-flatten': { + 'version': '1.1.1', + 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz', + 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=', + 'dependencies': { + 'content-type': { + 'version': 'https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz', + 'integrity': 'sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==', + 'dependencies': { + 'cookie': { + 'version': 'git+https://github.com/jshttp/cookie.git#aec1177c7da67e3b3273df96cf476824dbc9ae09', + 'from': 'git+https://github.com/jshttp/cookie.git' + } + } + } + } + } + } + }) + fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) + fetcher.download() + self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'array-flatten-1.1.1.tgz'))) + self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'content-type-1.0.4.tgz'))) + self.assertTrue(os.path.exists(os.path.join(self.dldir, 'git2', 'github.com.jshttp.cookie.git'))) + fetcher.unpack(self.unpackdir) + self.assertTrue(os.path.exists(os.path.join(self.sdir, 'npm-shrinkwrap.json'))) + self.assertTrue(os.path.exists(os.path.join(self.sdir, 'node_modules', 'array-flatten', 'package.json'))) + self.assertTrue(os.path.exists(os.path.join(self.sdir, 'node_modules', 'array-flatten', 'node_modules', 'content-type', 'package.json'))) + self.assertTrue(os.path.exists(os.path.join(self.sdir, 'node_modules', 'array-flatten', 'node_modules', 'content-type', 'node_modules', 'cookie', 'package.json'))) + + @skipIfNoNpm() + @skipIfNoNetwork() + def test_npmsw_dev(self): + swfile = self.create_shrinkwrap_file({ + 'dependencies': { + 'array-flatten': { + 'version': '1.1.1', + 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz', + 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=' + }, + 'content-type': { + 'version': '1.0.4', + 'resolved': 'https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz', + 'integrity': 'sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==', + 'dev': True + } + } + }) + # Fetch with dev disabled + fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) + fetcher.download() + self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'array-flatten-1.1.1.tgz'))) + self.assertFalse(os.path.exists(os.path.join(self.dldir, 'npm2', 'content-type-1.0.4.tgz'))) + # Fetch with dev enabled + fetcher = bb.fetch.Fetch(['npmsw://' + swfile + ';dev=1'], self.d) + fetcher.download() + self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'array-flatten-1.1.1.tgz'))) + self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'content-type-1.0.4.tgz'))) + + @skipIfNoNpm() + @skipIfNoNetwork() + def test_npmsw_destsuffix(self): + swfile = self.create_shrinkwrap_file({ + 'dependencies': { + 'array-flatten': { + 'version': '1.1.1', + 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz', + 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=' + } + } + }) + fetcher = bb.fetch.Fetch(['npmsw://' + swfile + ';destsuffix=foo/bar'], self.d) + fetcher.download() + fetcher.unpack(self.unpackdir) + self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'foo', 'bar', 'node_modules', 'array-flatten', 'package.json'))) + + def test_npmsw_no_network_no_tarball(self): + swfile = self.create_shrinkwrap_file({ + 'dependencies': { + 'array-flatten': { + 'version': '1.1.1', + 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz', + 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=' + } + } + }) + self.d.setVar('BB_NO_NETWORK', '1') + fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) + with self.assertRaises(bb.fetch2.NetworkAccess): + fetcher.download() + + @skipIfNoNpm() + @skipIfNoNetwork() + def test_npmsw_no_network_with_tarball(self): + # Fetch once to get a tarball + fetcher = bb.fetch.Fetch(['npm://registry.npmjs.org;package=array-flatten;version=1.1.1'], self.d) + fetcher.download() + # Disable network access + self.d.setVar('BB_NO_NETWORK', '1') + # Fetch again + swfile = self.create_shrinkwrap_file({ + 'dependencies': { + 'array-flatten': { + 'version': '1.1.1', + 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz', + 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=' + } + } + }) + fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) + fetcher.download() + fetcher.unpack(self.unpackdir) + self.assertTrue(os.path.exists(os.path.join(self.sdir, 'node_modules', 'array-flatten', 'package.json'))) + + @skipIfNoNpm() + @skipIfNoNetwork() + def test_npmsw_npm_reusability(self): + # Fetch once with npmsw + swfile = self.create_shrinkwrap_file({ + 'dependencies': { + 'array-flatten': { + 'version': '1.1.1', + 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz', + 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=' + } + } + }) + fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) + fetcher.download() + # Disable network access + self.d.setVar('BB_NO_NETWORK', '1') + # Fetch again with npm + fetcher = bb.fetch.Fetch(['npm://registry.npmjs.org;package=array-flatten;version=1.1.1'], self.d) + fetcher.download() + fetcher.unpack(self.unpackdir) + self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'npm', 'package.json'))) + + @skipIfNoNpm() + @skipIfNoNetwork() + def test_npmsw_bad_checksum(self): + # Try to fetch with bad checksum + swfile = self.create_shrinkwrap_file({ + 'dependencies': { + 'array-flatten': { + 'version': '1.1.1', + 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz', + 'integrity': 'sha1-gfNEp2hqgLTFKT6P3AsBYMgsBqg=' + } + } + }) + fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) + with self.assertRaises(bb.fetch2.FetchError): + fetcher.download() + # Fetch correctly to get a tarball + swfile = self.create_shrinkwrap_file({ + 'dependencies': { + 'array-flatten': { + 'version': '1.1.1', + 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz', + 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=' + } + } + }) + fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) + fetcher.download() + localpath = os.path.join(self.dldir, 'npm2', 'array-flatten-1.1.1.tgz') + self.assertTrue(os.path.exists(localpath)) + # Modify the tarball + bad = b'bad checksum' + with open(localpath, 'wb') as f: + f.write(bad) + # Verify that the tarball is fetched again + fetcher.download() + badsum = hashlib.sha1(bad).hexdigest() + self.assertTrue(os.path.exists(localpath + '_bad-checksum_' + badsum)) + self.assertTrue(os.path.exists(localpath)) + + @skipIfNoNpm() + @skipIfNoNetwork() + def test_npmsw_premirrors(self): + # Fetch once to get a tarball + fetcher = bb.fetch.Fetch(['npm://registry.npmjs.org;package=array-flatten;version=1.1.1'], self.d) + ud = fetcher.ud[fetcher.urls[0]] + fetcher.download() + self.assertTrue(os.path.exists(ud.localpath)) + # Setup the mirror + mirrordir = os.path.join(self.tempdir, 'mirror') + bb.utils.mkdirhier(mirrordir) + os.replace(ud.localpath, os.path.join(mirrordir, os.path.basename(ud.localpath))) + self.d.setVar('PREMIRRORS', 'https?$://.*/.* file://%s/\n' % mirrordir) + self.d.setVar('BB_FETCH_PREMIRRORONLY', '1') + # Fetch again + self.assertFalse(os.path.exists(ud.localpath)) + swfile = self.create_shrinkwrap_file({ + 'dependencies': { + 'array-flatten': { + 'version': '1.1.1', + 'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz', + 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=' + } + } + }) + fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) + fetcher.download() + self.assertTrue(os.path.exists(ud.localpath)) + + @skipIfNoNpm() + @skipIfNoNetwork() + def test_npmsw_mirrors(self): + # Fetch once to get a tarball + fetcher = bb.fetch.Fetch(['npm://registry.npmjs.org;package=array-flatten;version=1.1.1'], self.d) + ud = fetcher.ud[fetcher.urls[0]] + fetcher.download() + self.assertTrue(os.path.exists(ud.localpath)) + # Setup the mirror + mirrordir = os.path.join(self.tempdir, 'mirror') + bb.utils.mkdirhier(mirrordir) + os.replace(ud.localpath, os.path.join(mirrordir, os.path.basename(ud.localpath))) + self.d.setVar('MIRRORS', 'https?$://.*/.* file://%s/\n' % mirrordir) + # Fetch again with invalid url + self.assertFalse(os.path.exists(ud.localpath)) + swfile = self.create_shrinkwrap_file({ + 'dependencies': { + 'array-flatten': { + 'version': '1.1.1', + 'resolved': 'https://invalid', + 'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=' + } + } + }) + fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d) + fetcher.download() + self.assertTrue(os.path.exists(ud.localpath)) |