From a8bf787b4232beef67a3643d858166f776bb829d Mon Sep 17 00:00:00 2001 From: Tadao Tanikawa Date: Fri, 27 Jul 2018 03:54:43 +0000 Subject: Fix do_rootfs eats huge time on docker environment RPM is causing a serious performance regression on Docker. Docker can set ulimit -n to 1048576 but it causes huge time consumption when thousands of packages are installed like bitbake agl-demo-platform. This issue is already resolved in upstream of RPM and yocto follows it at sumo, so backporting it into Flounder. (From OE-Core rev: 6ecb10e3952af4a77bc79160ecd81117e97d022a) Bug-AGL: SPEC-1622 Change-Id: Ia8d97daea663f9682928a14ab84199ed6fda6d61 Signed-off-by: Tadao Tanikawa --- .../files/0002-Optimize-rpmSetCloseOnExec.patch | 102 +++++++++++++++++++++ 1 file changed, 102 insertions(+) create mode 100644 meta-agl-profile-core/recipes-devtools/rpm/files/0002-Optimize-rpmSetCloseOnExec.patch (limited to 'meta-agl-profile-core/recipes-devtools/rpm/files/0002-Optimize-rpmSetCloseOnExec.patch') diff --git a/meta-agl-profile-core/recipes-devtools/rpm/files/0002-Optimize-rpmSetCloseOnExec.patch b/meta-agl-profile-core/recipes-devtools/rpm/files/0002-Optimize-rpmSetCloseOnExec.patch new file mode 100644 index 000000000..9354dd6e0 --- /dev/null +++ b/meta-agl-profile-core/recipes-devtools/rpm/files/0002-Optimize-rpmSetCloseOnExec.patch @@ -0,0 +1,102 @@ +From 6f56aa01a357bbcf9401d79a378ae380f5f939d4 Mon Sep 17 00:00:00 2001 +From: Kir Kolyshkin +Date: Tue, 29 May 2018 17:52:56 -0700 +Subject: [PATCH 2/3] Optimize rpmSetCloseOnExec + +In case maximum number of open files limit is set too high, both +luaext/Pexec() and lib/doScriptExec() spend way too much time +trying to set FD_CLOEXEC flag for all those file descriptors, +resulting in severe increase of time it takes to execute say +rpm or dnf. + +This becomes increasingly noticeable when running with e.g. under +Docker, the reason being: + +> $ docker run fedora ulimit -n +> 1048576 + +One obvious fix is to use procfs to get the actual list of opened fds +and iterate over it. My quick-n-dirty benchmark shows the /proc approach +is about 10x faster than iterating through a list of just 1024 fds, +so it's an improvement even for default ulimit values. + +Note that the old method is still used in case /proc is not available. + +While at it, + + 1. fix the function by making sure we modify (rather than set) + the existing flags. As the only known flag is FD_CLOEXEC, + this change is currently purely aesthetical, but in case + other flags will appear it will become a real bug fix. + + 2. get rid of magic number 3; use STDERR_FILENO + +Signed-off-by: Kir Kolyshkin + +Fixes #444 + +(cherry picked from commit 5e6f05cd8dad6c1ee6bd1e6e43f176976c9c3416) +--- + rpmio/rpmio.c | 43 ++++++++++++++++++++++++++++++++++--------- + 1 file changed, 34 insertions(+), 9 deletions(-) + +diff --git a/rpmio/rpmio.c b/rpmio/rpmio.c +index 747bf3c..8148aa2 100644 +--- a/rpmio/rpmio.c ++++ b/rpmio/rpmio.c +@@ -1477,18 +1477,43 @@ void fdFiniDigest(FD_t fd, int hashalgo, + } + } + ++static void set_cloexec(int fd) ++{ ++ int flags = fcntl(fd, F_GETFD); ++ ++ if (flags == -1 || (flags & FD_CLOEXEC)) ++ return; ++ ++ fcntl(fd, F_SETFD, flags | FD_CLOEXEC); ++} ++ + void rpmSetCloseOnExec(void) + { +- int flag, fdno, open_max; ++ const int min_fd = STDERR_FILENO; /* don't touch stdin/out/err */ ++ int fd; ++ ++ DIR *dir = opendir("/proc/self/fd"); ++ if (dir == NULL) { /* /proc not available */ ++ /* iterate over all possible fds, might be slow */ ++ int open_max = sysconf(_SC_OPEN_MAX); ++ if (open_max == -1) ++ open_max = 1024; + +- open_max = sysconf(_SC_OPEN_MAX); +- if (open_max == -1) { +- open_max = 1024; ++ for (fd = min_fd + 1; fd < open_max; fd++) ++ set_cloexec(fd); ++ ++ return; + } +- for (fdno = 3; fdno < open_max; fdno++) { +- flag = fcntl(fdno, F_GETFD); +- if (flag == -1 || (flag & FD_CLOEXEC)) +- continue; +- fcntl(fdno, F_SETFD, FD_CLOEXEC); ++ ++ /* iterate over fds obtained from /proc */ ++ struct dirent *entry; ++ while ((entry = readdir(dir)) != NULL) { ++ fd = atoi(entry->d_name); ++ if (fd > min_fd) ++ set_cloexec(fd); + } ++ ++ closedir(dir); ++ ++ return; + } +-- +2.7.4 + -- cgit 1.2.3-korg