Gentoo Archives: gentoo-portage-dev

From: "Michał Górny" <mgorny@g.o>
To: gentoo-portage-dev@l.g.o
Cc: "Michał Górny" <mgorny@g.o>
Subject: [gentoo-portage-dev] [PATCH 2/3] portage.dbapi.vartree: Move INSTALL_MASK handling into merging
Date: Thu, 15 Mar 2018 19:22:35
Message-Id: 20180315192212.13454-3-mgorny@gentoo.org
In Reply to: [gentoo-portage-dev] [PATCH 0/3] INSTALL_MASK refurbishing resubmit by "Michał Górny"
1 Introduce a new logic for INSTALL_MASK handling in merging code,
2 replacing the old code that removed matching files and directories
3 from imagedir in bash. The new code actually ignores matching files
4 on-the-fly while testing for file collisions and merging files.
5 The files are still written to CONTENTS, and output using "###" zing
6 to indicate being masked, yet are not actually merged to the filesystem.
7 ---
8 bin/misc-functions.sh | 17 ------
9 pym/portage/dbapi/vartree.py | 102 ++++++++++++++++++++++-------------
10 pym/portage/package/ebuild/config.py | 3 +-
11 3 files changed, 66 insertions(+), 56 deletions(-)
12
13 diff --git a/bin/misc-functions.sh b/bin/misc-functions.sh
14 index 6a5c2ea05..59391816b 100755
15 --- a/bin/misc-functions.sh
16 +++ b/bin/misc-functions.sh
17 @@ -361,23 +361,6 @@ install_mask() {
18 set -${shopts}
19 }
20
21 -preinst_mask() {
22 - if [ -z "${D}" ]; then
23 - eerror "${FUNCNAME}: D is unset"
24 - return 1
25 - fi
26 -
27 - if ! ___eapi_has_prefix_variables; then
28 - local ED=${D}
29 - fi
30 -
31 - # Make sure $PWD is not ${D} so that we don't leave gmon.out files
32 - # in there in case any tools were built with -pg in CFLAGS.
33 - cd "${T}"
34 -
35 - install_mask "${ED}" "${INSTALL_MASK}"
36 -}
37 -
38 preinst_sfperms() {
39 if [ -z "${D}" ]; then
40 eerror "${FUNCNAME}: D is unset"
41 diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py
42 index 8b1b77f7d..21904edca 100644
43 --- a/pym/portage/dbapi/vartree.py
44 +++ b/pym/portage/dbapi/vartree.py
45 @@ -2491,7 +2491,7 @@ class dblink(object):
46 (statobj.st_dev, statobj.st_ino),
47 []).append(relative_path)
48
49 - if is_owned:
50 + if is_owned and not self._is_install_masked(relative_path[1:]):
51 show_unmerge("---", unmerge_desc["replaced"], file_type, obj)
52 continue
53 elif relative_path in cfgfiledict:
54 @@ -3689,6 +3689,24 @@ class dblink(object):
55 def _emerge_log(self, msg):
56 emergelog(False, msg)
57
58 + def _is_install_masked(self, relative_path):
59 + ret = False
60 + for pattern in self.settings.install_mask:
61 + # absolute path pattern
62 + if pattern.startswith('/'):
63 + # match either exact path or one of parent dirs
64 + # the latter is done via matching pattern/*
65 + if (fnmatch.fnmatch(relative_path, pattern[1:])
66 + or fnmatch.fnmatch(relative_path, pattern[1:] + '/*')):
67 + ret = True
68 + break
69 + # filename
70 + else:
71 + if fnmatch.fnmatch(os.path.basename(relative_path), pattern):
72 + ret = True
73 + break
74 + return ret
75 +
76 def treewalk(self, srcroot, destroot, inforoot, myebuild, cleanup=0,
77 mydbapi=None, prev_mtimes=None, counter=None):
78 """
79 @@ -3848,16 +3866,6 @@ class dblink(object):
80 max_dblnk = dblnk
81 self._installed_instance = max_dblnk
82
83 - # Apply INSTALL_MASK before collision-protect, since it may
84 - # be useful to avoid collisions in some scenarios.
85 - # We cannot detect if this is needed or not here as INSTALL_MASK can be
86 - # modified by bashrc files.
87 - phase = MiscFunctionsProcess(background=False,
88 - commands=["preinst_mask"], phase="preinst",
89 - scheduler=self._scheduler, settings=self.settings)
90 - phase.start()
91 - phase.wait()
92 -
93 # We check for unicode encoding issues after src_install. However,
94 # the check must be repeated here for binary packages (it's
95 # inexpensive since we call os.walk() here anyway).
96 @@ -3929,6 +3937,10 @@ class dblink(object):
97
98 relative_path = fpath[srcroot_len:]
99
100 + # filter on INSTALL_MASK
101 + if self._is_install_masked(relative_path):
102 + continue
103 +
104 if line_ending_re.search(relative_path) is not None:
105 paths_with_newlines.append(relative_path)
106
107 @@ -4658,6 +4670,7 @@ class dblink(object):
108 while mergelist:
109
110 relative_path = mergelist.pop()
111 + instmasked = self._is_install_masked(relative_path)
112 mysrc = join(srcroot, relative_path)
113 mydest = join(destroot, relative_path)
114 # myrealdest is mydest without the $ROOT prefix (makes a difference if ROOT!="/")
115 @@ -4744,7 +4757,7 @@ class dblink(object):
116 destmd5 = None
117
118 moveme = True
119 - if protected:
120 + if protected and not instmasked:
121 mydest, protected, moveme = self._protect(cfgfiledict,
122 protect_if_modified, mymd5, myto, mydest,
123 myrealdest, mydmode, destmd5, mydest_link)
124 @@ -4772,7 +4785,7 @@ class dblink(object):
125 # we can simply test for existence of this file to see if the target has been merged yet
126 myrealto = normalize_path(os.path.join(destroot, myabsto))
127 if mydmode is not None and stat.S_ISDIR(mydmode):
128 - if not protected:
129 + if not protected and not instmasked:
130 # we can't merge a symlink over a directory
131 newdest = self._new_backup_path(mydest)
132 msg = []
133 @@ -4792,26 +4805,32 @@ class dblink(object):
134 # it later.
135 secondhand.append(mysrc[len(srcroot):])
136 continue
137 - # unlinking no longer necessary; "movefile" will overwrite symlinks atomically and correctly
138 - if moveme:
139 - zing = ">>>"
140 - mymtime = movefile(mysrc, mydest, newmtime=thismtime,
141 - sstat=mystat, mysettings=self.settings,
142 - encoding=_encodings['merge'])
143
144 - try:
145 - self._merged_path(mydest, os.lstat(mydest))
146 - except OSError:
147 - pass
148 + if instmasked:
149 + zing = "###"
150 + # pass mymtime through from initial stat
151 + else:
152 + # unlinking no longer necessary; "movefile" will overwrite symlinks atomically and correctly
153 + if moveme:
154 + zing = ">>>"
155 + mymtime = movefile(mysrc, mydest, newmtime=thismtime,
156 + sstat=mystat, mysettings=self.settings,
157 + encoding=_encodings['merge'])
158 +
159 + try:
160 + self._merged_path(mydest, os.lstat(mydest))
161 + except OSError:
162 + pass
163
164 if mymtime != None:
165 - # Use lexists, since if the target happens to be a broken
166 - # symlink then that should trigger an independent warning.
167 - if not (os.path.lexists(myrealto) or
168 - os.path.lexists(join(srcroot, myabsto))):
169 - self._eqawarn('preinst',
170 - [_("QA Notice: Symbolic link /%s points to /%s which does not exist.")
171 - % (relative_path, myabsto)])
172 + if not instmasked:
173 + # Use lexists, since if the target happens to be a broken
174 + # symlink then that should trigger an independent warning.
175 + if not (os.path.lexists(myrealto) or
176 + os.path.lexists(join(srcroot, myabsto))):
177 + self._eqawarn('preinst',
178 + [_("QA Notice: Symbolic link /%s points to /%s which does not exist.")
179 + % (relative_path, myabsto)])
180
181 showMessage("%s %s -> %s\n" % (zing, mydest, myto))
182 if sys.hexversion >= 0x3030000:
183 @@ -4826,7 +4845,9 @@ class dblink(object):
184 return 1
185 elif stat.S_ISDIR(mymode):
186 # we are merging a directory
187 - if mydmode != None:
188 + if instmasked:
189 + showMessage("### %s/\n" % mydest)
190 + elif mydmode != None:
191 # destination exists
192
193 if bsd_chflags:
194 @@ -4913,10 +4934,11 @@ class dblink(object):
195 os.chown(mydest, mystat[4], mystat[5])
196 showMessage(">>> %s/\n" % mydest)
197
198 - try:
199 - self._merged_path(mydest, os.lstat(mydest))
200 - except OSError:
201 - pass
202 + if not instmasked:
203 + try:
204 + self._merged_path(mydest, os.lstat(mydest))
205 + except OSError:
206 + pass
207
208 outfile.write("dir "+myrealdest+"\n")
209 # recurse and merge this directory
210 @@ -4925,7 +4947,7 @@ class dblink(object):
211
212 elif stat.S_ISREG(mymode):
213 # we are merging a regular file
214 - if not protected and \
215 + if not protected and not instmasked and \
216 mydmode is not None and stat.S_ISDIR(mydmode):
217 # install of destination is blocked by an existing directory with the same name
218 newdest = self._new_backup_path(mydest)
219 @@ -4939,9 +4961,11 @@ class dblink(object):
220 self._eerror("preinst", msg)
221 mydest = newdest
222
223 + if instmasked:
224 + zing = "###"
225 # whether config protection or not, we merge the new file the
226 # same way. Unless moveme=0 (blocking directory)
227 - if moveme:
228 + elif moveme:
229 # Create hardlinks only for source files that already exist
230 # as hardlinks (having identical st_dev and st_ino).
231 hardlink_key = (mystat.st_dev, mystat.st_ino)
232 @@ -4974,7 +4998,9 @@ class dblink(object):
233 else:
234 # we are merging a fifo or device node
235 zing = "!!!"
236 - if mydmode is None:
237 + if instmasked:
238 + zing = "###"
239 + elif mydmode is None:
240 # destination doesn't exist
241 if movefile(mysrc, mydest, newmtime=thismtime,
242 sstat=mystat, mysettings=self.settings,
243 diff --git a/pym/portage/package/ebuild/config.py b/pym/portage/package/ebuild/config.py
244 index 3f575fcaf..d04baacf9 100644
245 --- a/pym/portage/package/ebuild/config.py
246 +++ b/pym/portage/package/ebuild/config.py
247 @@ -271,6 +271,7 @@ class config(object):
248 self.mycpv = clone.mycpv
249 self._setcpv_args_hash = clone._setcpv_args_hash
250 self._soname_provided = clone._soname_provided
251 + self.install_mask = clone.install_mask
252
253 # immutable attributes (internal policy ensures lack of mutation)
254 self._locations_manager = clone._locations_manager
255 @@ -2473,7 +2474,7 @@ class config(object):
256 install_mask.append("/usr/share/info")
257 if 'noman' in self.features:
258 install_mask.append("/usr/share/man")
259 - self["INSTALL_MASK"] = ' '.join(install_mask)
260 + self.install_mask = tuple(install_mask)
261
262 if self.mycpv is None:
263 # Generate global USE_EXPAND variables settings that are
264 --
265 2.16.2