1 |
Since the Manifest "stable mtime" behavior could have undiscovered |
2 |
bugs, disable it by default, and add a corresponding egencache option. |
3 |
|
4 |
Suggested-by: Michał Górny <mgorny@g.o> |
5 |
--- |
6 |
[PATCH v3] fixes the commit message and adds a comment block to |
7 |
Manifest.write in order to document subtle issues involving the mtime |
8 |
of self.pkgdir that must be accounted for. |
9 |
|
10 |
bin/egencache | 6 +++++- |
11 |
man/egencache.1 | 3 +++ |
12 |
pym/portage/manifest.py | 23 +++++++++++++++++----- |
13 |
.../ebuild/_parallel_manifest/ManifestProcess.py | 6 ++++-- |
14 |
.../ebuild/_parallel_manifest/ManifestScheduler.py | 7 +++++-- |
15 |
.../ebuild/_parallel_manifest/ManifestTask.py | 8 +++++--- |
16 |
6 files changed, 40 insertions(+), 13 deletions(-) |
17 |
|
18 |
diff --git a/bin/egencache b/bin/egencache |
19 |
index 7e3387e..07665e8 100755 |
20 |
--- a/bin/egencache |
21 |
+++ b/bin/egencache |
22 |
@@ -120,6 +120,9 @@ def parse_args(args): |
23 |
choices=('y', 'n'), |
24 |
metavar="<y|n>", |
25 |
help="manually override layout.conf sign-manifests setting") |
26 |
+ common.add_argument("--stable-mtime", |
27 |
+ action="store_true", |
28 |
+ help="apply stable mtime to generated manifests (for rsync)") |
29 |
common.add_argument("--strict-manifests", |
30 |
choices=('y', 'n'), |
31 |
metavar="<y|n>", |
32 |
@@ -1151,7 +1154,8 @@ def egencache_main(args): |
33 |
force_sign_key=force_sign_key, |
34 |
max_jobs=options.jobs, |
35 |
max_load=options.load_average, |
36 |
- event_loop=event_loop) |
37 |
+ event_loop=event_loop, |
38 |
+ manifest_kwargs=dict(stable_mtime=options.stable_mtime)) |
39 |
|
40 |
signum = run_main_scheduler(scheduler) |
41 |
if signum is not None: |
42 |
diff --git a/man/egencache.1 b/man/egencache.1 |
43 |
index 7fd17c2..081e8c1 100644 |
44 |
--- a/man/egencache.1 |
45 |
+++ b/man/egencache.1 |
46 |
@@ -100,6 +100,9 @@ Manually override layout.conf sign-manifests setting. |
47 |
.BR "\-\-strict\-manifests< y | n >" |
48 |
Manually override "strict" FEATURES setting. |
49 |
.TP |
50 |
+.BR "\-\-stable\-mtime" |
51 |
+Apply stable mtime to generated manifests (for rsync). |
52 |
+.TP |
53 |
.BR "\-\-thin\-manifests< y | n >" |
54 |
Manually override layout.conf thin-manifests setting. |
55 |
.TP |
56 |
diff --git a/pym/portage/manifest.py b/pym/portage/manifest.py |
57 |
index f696f84..6efd6e5 100644 |
58 |
--- a/pym/portage/manifest.py |
59 |
+++ b/pym/portage/manifest.py |
60 |
@@ -128,7 +128,7 @@ class Manifest(object): |
61 |
def __init__(self, pkgdir, distdir=None, fetchlist_dict=None, |
62 |
manifest1_compat=DeprecationWarning, from_scratch=False, thin=False, |
63 |
allow_missing=False, allow_create=True, hashes=None, |
64 |
- find_invalid_path_char=None): |
65 |
+ find_invalid_path_char=None, stable_mtime=False): |
66 |
""" Create new Manifest instance for package in pkgdir. |
67 |
Do not parse Manifest file if from_scratch == True (only for internal use) |
68 |
The fetchlist_dict parameter is required only for generation of |
69 |
@@ -145,6 +145,7 @@ class Manifest(object): |
70 |
find_invalid_path_char = _find_invalid_path_char |
71 |
self._find_invalid_path_char = find_invalid_path_char |
72 |
self.pkgdir = _unicode_decode(pkgdir).rstrip(os.sep) + os.sep |
73 |
+ self.stable_mtime = stable_mtime |
74 |
self.fhashdict = {} |
75 |
self.hashes = set() |
76 |
|
77 |
@@ -283,7 +284,16 @@ class Manifest(object): |
78 |
myentries = list(self._createManifestEntries()) |
79 |
update_manifest = True |
80 |
preserved_stats = {} |
81 |
- preserved_stats[self.pkgdir.rstrip(os.sep)] = os.stat(self.pkgdir) |
82 |
+ if self.stable_mtime: |
83 |
+ # The pre-existing mtime of self.pkgdir is included in the |
84 |
+ # max mtime calculation in order to account for anything |
85 |
+ # that may have been renamed or removed in this directory |
86 |
+ # (including the Manifest itself). Note that the mtime of |
87 |
+ # this directory will be always be bumped as a side-effect |
88 |
+ # of writing the Manifest (since write_atomic uses a rename |
89 |
+ # operation for atomicity), therefore it must be preserved |
90 |
+ # before the Manifest is written. |
91 |
+ preserved_stats[self.pkgdir.rstrip(os.sep)] = os.stat(self.pkgdir) |
92 |
if myentries and not force: |
93 |
try: |
94 |
f = io.open(_unicode_encode(self.getFullname(), |
95 |
@@ -291,7 +301,8 @@ class Manifest(object): |
96 |
mode='r', encoding=_encodings['repo.content'], |
97 |
errors='replace') |
98 |
oldentries = list(self._parseManifestLines(f)) |
99 |
- preserved_stats[self.getFullname()] = os.fstat(f.fileno()) |
100 |
+ if self.stable_mtime: |
101 |
+ preserved_stats[self.getFullname()] = os.fstat(f.fileno()) |
102 |
f.close() |
103 |
if len(oldentries) == len(myentries): |
104 |
update_manifest = False |
105 |
@@ -313,7 +324,8 @@ class Manifest(object): |
106 |
# non-empty for all currently known use cases. |
107 |
write_atomic(self.getFullname(), "".join("%s\n" % |
108 |
_unicode(myentry) for myentry in myentries)) |
109 |
- self._apply_max_mtime(preserved_stats, myentries) |
110 |
+ if self.stable_mtime: |
111 |
+ self._apply_max_mtime(preserved_stats, myentries) |
112 |
rval = True |
113 |
else: |
114 |
# With thin manifest, there's no need to have |
115 |
@@ -450,7 +462,8 @@ class Manifest(object): |
116 |
fetchlist_dict=self.fetchlist_dict, from_scratch=True, |
117 |
thin=self.thin, allow_missing=self.allow_missing, |
118 |
allow_create=self.allow_create, hashes=self.hashes, |
119 |
- find_invalid_path_char=self._find_invalid_path_char) |
120 |
+ find_invalid_path_char=self._find_invalid_path_char, |
121 |
+ stable_mtime=self.stable_mtime) |
122 |
pn = os.path.basename(self.pkgdir.rstrip(os.path.sep)) |
123 |
cat = self._pkgdir_category() |
124 |
|
125 |
diff --git a/pym/portage/package/ebuild/_parallel_manifest/ManifestProcess.py b/pym/portage/package/ebuild/_parallel_manifest/ManifestProcess.py |
126 |
index 44e2576..01595a3 100644 |
127 |
--- a/pym/portage/package/ebuild/_parallel_manifest/ManifestProcess.py |
128 |
+++ b/pym/portage/package/ebuild/_parallel_manifest/ManifestProcess.py |
129 |
@@ -10,14 +10,16 @@ from portage.util._async.ForkProcess import ForkProcess |
130 |
|
131 |
class ManifestProcess(ForkProcess): |
132 |
|
133 |
- __slots__ = ("cp", "distdir", "fetchlist_dict", "repo_config") |
134 |
+ __slots__ = ("cp", "distdir", "fetchlist_dict", "manifest_kwargs", |
135 |
+ "repo_config") |
136 |
|
137 |
MODIFIED = 16 |
138 |
|
139 |
def _run(self): |
140 |
mf = self.repo_config.load_manifest( |
141 |
os.path.join(self.repo_config.location, self.cp), |
142 |
- self.distdir, fetchlist_dict=self.fetchlist_dict) |
143 |
+ self.distdir, fetchlist_dict=self.fetchlist_dict, |
144 |
+ **(self.manifest_kwargs or {})) |
145 |
|
146 |
try: |
147 |
mf.create(assumeDistHashesAlways=True) |
148 |
diff --git a/pym/portage/package/ebuild/_parallel_manifest/ManifestScheduler.py b/pym/portage/package/ebuild/_parallel_manifest/ManifestScheduler.py |
149 |
index 38ac482..8a1c1d0 100644 |
150 |
--- a/pym/portage/package/ebuild/_parallel_manifest/ManifestScheduler.py |
151 |
+++ b/pym/portage/package/ebuild/_parallel_manifest/ManifestScheduler.py |
152 |
@@ -12,11 +12,13 @@ from .ManifestTask import ManifestTask |
153 |
class ManifestScheduler(AsyncScheduler): |
154 |
|
155 |
def __init__(self, portdb, cp_iter=None, |
156 |
- gpg_cmd=None, gpg_vars=None, force_sign_key=None, **kwargs): |
157 |
+ gpg_cmd=None, gpg_vars=None, force_sign_key=None, |
158 |
+ manifest_kwargs=None, **kwargs): |
159 |
|
160 |
AsyncScheduler.__init__(self, **kwargs) |
161 |
|
162 |
self._portdb = portdb |
163 |
+ self._manifest_kwargs = manifest_kwargs |
164 |
|
165 |
if cp_iter is None: |
166 |
cp_iter = self._iter_every_cp() |
167 |
@@ -79,7 +81,8 @@ class ManifestScheduler(AsyncScheduler): |
168 |
yield ManifestTask(cp=cp, distdir=distdir, |
169 |
fetchlist_dict=fetchlist_dict, repo_config=repo_config, |
170 |
gpg_cmd=self._gpg_cmd, gpg_vars=self._gpg_vars, |
171 |
- force_sign_key=self._force_sign_key) |
172 |
+ force_sign_key=self._force_sign_key, |
173 |
+ manifest_kwargs=self._manifest_kwargs) |
174 |
|
175 |
def _task_exit(self, task): |
176 |
|
177 |
diff --git a/pym/portage/package/ebuild/_parallel_manifest/ManifestTask.py b/pym/portage/package/ebuild/_parallel_manifest/ManifestTask.py |
178 |
index 0ee2b91..fb5e16e 100644 |
179 |
--- a/pym/portage/package/ebuild/_parallel_manifest/ManifestTask.py |
180 |
+++ b/pym/portage/package/ebuild/_parallel_manifest/ManifestTask.py |
181 |
@@ -18,8 +18,8 @@ from .ManifestProcess import ManifestProcess |
182 |
|
183 |
class ManifestTask(CompositeTask): |
184 |
|
185 |
- __slots__ = ("cp", "distdir", "fetchlist_dict", "gpg_cmd", |
186 |
- "gpg_vars", "repo_config", "force_sign_key", "_manifest_path") |
187 |
+ __slots__ = ("cp", "distdir", "fetchlist_dict", "force_sign_key", |
188 |
+ "gpg_cmd", "gpg_vars", "manifest_kwargs", "repo_config", "_manifest_path") |
189 |
|
190 |
_PGP_HEADER = b"BEGIN PGP SIGNED MESSAGE" |
191 |
_manifest_line_re = re.compile(r'^(%s) ' % "|".join(MANIFEST2_IDENTIFIERS)) |
192 |
@@ -30,7 +30,9 @@ class ManifestTask(CompositeTask): |
193 |
self._manifest_path = os.path.join(self.repo_config.location, |
194 |
self.cp, "Manifest") |
195 |
manifest_proc = ManifestProcess(cp=self.cp, distdir=self.distdir, |
196 |
- fetchlist_dict=self.fetchlist_dict, repo_config=self.repo_config, |
197 |
+ fetchlist_dict=self.fetchlist_dict, |
198 |
+ manifest_kwargs=self.manifest_kwargs, |
199 |
+ repo_config=self.repo_config, |
200 |
scheduler=self.scheduler) |
201 |
self._start_task(manifest_proc, self._manifest_proc_exit) |
202 |
|
203 |
-- |
204 |
2.4.10 |