| 1 |
commit: 8bdcac12a5c3a41d8321a55b8fef36355dc0f6e7 |
| 2 |
Author: Magnus Granberg <zorry <AT> gentoo <DOT> org> |
| 3 |
AuthorDate: Fri Apr 2 22:11:49 2021 +0000 |
| 4 |
Commit: Magnus Granberg <zorry <AT> gentoo <DOT> org> |
| 5 |
CommitDate: Fri Apr 2 22:11:49 2021 +0000 |
| 6 |
URL: https://gitweb.gentoo.org/proj/tinderbox-cluster.git/commit/?id=8bdcac12 |
| 7 |
|
| 8 |
Add GetAuxMetadata and use that to get aux metadata |
| 9 |
|
| 10 |
Signed-off-by: Magnus Granberg <zorry <AT> gentoo.org> |
| 11 |
|
| 12 |
buildbot_gentoo_ci/steps/portage.py | 63 ++++++++++++++++++++++++++++++++--- |
| 13 |
buildbot_gentoo_ci/steps/update_db.py | 17 +++++++++- |
| 14 |
buildbot_gentoo_ci/steps/version.py | 12 ++++--- |
| 15 |
3 files changed, 83 insertions(+), 9 deletions(-) |
| 16 |
|
| 17 |
diff --git a/buildbot_gentoo_ci/steps/portage.py b/buildbot_gentoo_ci/steps/portage.py |
| 18 |
index 293b862..d66c952 100644 |
| 19 |
--- a/buildbot_gentoo_ci/steps/portage.py |
| 20 |
+++ b/buildbot_gentoo_ci/steps/portage.py |
| 21 |
@@ -10,6 +10,7 @@ from portage import _encodings |
| 22 |
from portage import _unicode_encode |
| 23 |
from portage import _parse_eapi_ebuild_head, eapi_is_supported |
| 24 |
from portage.versions import cpv_getversion, pkgsplit, catpkgsplit |
| 25 |
+from portage import portdbapi |
| 26 |
|
| 27 |
from twisted.internet import defer |
| 28 |
from twisted.python import log |
| 29 |
@@ -249,7 +250,7 @@ class SetMakeConf(BuildStep): |
| 30 |
makeconf_list.append('DISTDIR="/var/cache/portage/distfiles"') |
| 31 |
makeconf_list.append('PORTAGE_ELOG_CLASSES="*"') |
| 32 |
# We need echo:info to get the logfile name |
| 33 |
- makeconf_list.append('PORTAGE_ELOG_SYSTEM="save echo:info"') |
| 34 |
+ makeconf_list.append('PORTAGE_ELOG_SYSTEM="save:* echo:info"') |
| 35 |
# add ACCEPT_KEYWORDS from the project_data info |
| 36 |
keyword_data = yield self.gentooci.db.keywords.getKeywordById(project_data['keyword_id']) |
| 37 |
if project_data['status'] == 'unstable': |
| 38 |
@@ -395,7 +396,8 @@ class SetReposConfLocal(BuildStep): |
| 39 |
repos_conf_path = yield os.path.join('etc', 'portage', 'repos.conf') |
| 40 |
repos_conf_default_path = yield os.path.join(repos_conf_path, 'default.conf') |
| 41 |
self.gentooci = self.master.namedServices['services'].namedServices['gentooci'] |
| 42 |
- self.repository_basedir = self.gentooci.config.project['repository_basedir'] |
| 43 |
+ # the path should be set in the confg |
| 44 |
+ self.repository_basedir2 = '/home/repos2/' |
| 45 |
if not os.path.isfile(repos_conf_default_path): |
| 46 |
# setup the default.conf |
| 47 |
repos_conf_data = yield self.gentooci.db.projects.getProjectPortageByUuidAndDirectory(self.getProperty('project_data')['uuid'], 'repos.conf') |
| 48 |
@@ -409,7 +411,7 @@ class SetReposConfLocal(BuildStep): |
| 49 |
yield WriteTextToFile(repos_conf_default_path, default_conf) |
| 50 |
repos_conf_repository_path = yield os.path.join(repos_conf_path, self.getProperty("repository_data")['name'] + '.conf') |
| 51 |
if not os.path.isfile(repos_conf_repository_path): |
| 52 |
- repository_path = yield os.path.join(self.getProperty("builddir"), self.repository_basedir, self.getProperty("repository_data")['name']) |
| 53 |
+ repository_path = yield os.path.join(self.repository_basedir2, self.getProperty("repository_data")['name']) |
| 54 |
repository_conf = [] |
| 55 |
repository_conf.append('[' + self.getProperty("repository_data")['name'] + ']') |
| 56 |
repository_conf.append('location = ' + repository_path) |
| 57 |
@@ -444,7 +446,7 @@ class SetMakeConfLocal(BuildStep): |
| 58 |
makeconf_list.append('ACCEPT_KEYWORDS="~amd64 amd64"') |
| 59 |
makeconf_list.append('EMERGE_DEFAULT_OPTS=""') |
| 60 |
makeconf_list.append('ABI_X86="32 64"') |
| 61 |
- makeconf_list.append('FEATURES=""') |
| 62 |
+ makeconf_list.append('FEATURES="sandbox"') |
| 63 |
yield WriteTextToFile(make_conf_path, makeconf_list) |
| 64 |
return SUCCESS |
| 65 |
|
| 66 |
@@ -530,3 +532,56 @@ class SetEnvForEbuildSH(BuildStep): |
| 67 |
)) |
| 68 |
yield self.build.addStepsAfterCurrentStep(addStepEbuildSH) |
| 69 |
return SUCCESS |
| 70 |
+ |
| 71 |
+class GetAuxMetadata(BuildStep): |
| 72 |
+ |
| 73 |
+ name = 'GetAuxMetadata' |
| 74 |
+ description = 'Running' |
| 75 |
+ descriptionDone = 'Ran' |
| 76 |
+ descriptionSuffix = None |
| 77 |
+ haltOnFailure = True |
| 78 |
+ flunkOnFailure = True |
| 79 |
+ |
| 80 |
+ def __init__(self, **kwargs): |
| 81 |
+ super().__init__(**kwargs) |
| 82 |
+ |
| 83 |
+ #@defer.inlineCallbacks |
| 84 |
+ def thd_getAuxDbKeys(self): |
| 85 |
+ auxdbs = self.myportdb.aux_get(self.getProperty("cpv"), auxdbkeys, myrepo=self.getProperty("repository_data")['name']) |
| 86 |
+ return auxdbs |
| 87 |
+ |
| 88 |
+ @defer.inlineCallbacks |
| 89 |
+ def run(self): |
| 90 |
+ config_root = yield os.path.join(self.getProperty("builddir"), '') |
| 91 |
+ # setup mysettings and myportdb |
| 92 |
+ mysettings = yield portage_config(config_root = config_root) |
| 93 |
+ self.myportdb = yield portdbapi(mysettings=mysettings) |
| 94 |
+ auxdbs = yield self.thd_getAuxDbKeys() |
| 95 |
+ metadata = None |
| 96 |
+ NoSplit = [] |
| 97 |
+ NoSplit.append('DESCRIPTION') |
| 98 |
+ # should have 22 lines |
| 99 |
+ if len(auxdbkeys) != len(auxdbs) or not isinstance(auxdbs, list): |
| 100 |
+ # number of lines is incorrect or not a list. |
| 101 |
+ print("Lines don't match or not a list") |
| 102 |
+ yield self.myportdb.close_caches() |
| 103 |
+ yield portdbapi.portdbapi_instances.remove(self.myportdb) |
| 104 |
+ #self.setProperty('auxdb', metadata, 'auxdb') |
| 105 |
+ return FAILURE |
| 106 |
+ # split all keys to list instead of speces |
| 107 |
+ metadata = {} |
| 108 |
+ i = 0 |
| 109 |
+ for key in auxdbkeys: |
| 110 |
+ if auxdbs[i] == '': |
| 111 |
+ metadata[key] = None |
| 112 |
+ else: |
| 113 |
+ if ' ' in auxdbs[i] and key not in NoSplit: |
| 114 |
+ metadata[key] = auxdbs[i].split(' ') |
| 115 |
+ else: |
| 116 |
+ metadata[key] = [] |
| 117 |
+ metadata[key].append(auxdbs[i]) |
| 118 |
+ i = i + 1 |
| 119 |
+ self.setProperty('auxdb', metadata, 'auxdb') |
| 120 |
+ yield self.myportdb.close_caches() |
| 121 |
+ yield portdbapi.portdbapi_instances.remove(self.myportdb) |
| 122 |
+ return SUCCESS |
| 123 |
|
| 124 |
diff --git a/buildbot_gentoo_ci/steps/update_db.py b/buildbot_gentoo_ci/steps/update_db.py |
| 125 |
index b84e256..3594c1b 100644 |
| 126 |
--- a/buildbot_gentoo_ci/steps/update_db.py |
| 127 |
+++ b/buildbot_gentoo_ci/steps/update_db.py |
| 128 |
@@ -140,11 +140,20 @@ class UpdateRepos(BuildStep): |
| 129 |
else: |
| 130 |
raise AssertionError('Unknown merge analysis result') |
| 131 |
|
| 132 |
+ @defer.inlineCallbacks |
| 133 |
+ def setchmod(self, path): |
| 134 |
+ for root, dirs, files in os.walk(path): |
| 135 |
+ for d in dirs: |
| 136 |
+ yield os.chmod(os.path.join(root, d), 0o0755) |
| 137 |
+ for f in files: |
| 138 |
+ yield os.chmod(os.path.join(root, f), 0o0644) |
| 139 |
+ |
| 140 |
@defer.inlineCallbacks |
| 141 |
def run(self): |
| 142 |
#FIXME check HEAD agenst local and worker local tree so we don't gitpull evrytime |
| 143 |
self.gentooci = self.master.namedServices['services'].namedServices['gentooci'] |
| 144 |
- self.repository_basedir = self.gentooci.config.project['repository_basedir'] |
| 145 |
+ #self.repository_basedir = self.gentooci.config.project['repository_basedir'] |
| 146 |
+ self.repository_basedir = yield os.path.join('/home', 'repos2') |
| 147 |
self.profile_repository_path = yield os.path.join(self.repository_basedir, self.getProperty("profile_repository_data")['name']) |
| 148 |
repo_path = yield pygit2.discover_repository(self.profile_repository_path) |
| 149 |
print(repo_path) |
| 150 |
@@ -153,6 +162,9 @@ class UpdateRepos(BuildStep): |
| 151 |
else: |
| 152 |
repo = yield pygit2.Repository(repo_path) |
| 153 |
yield self.gitPull(repo) |
| 154 |
+ # chmod and chown |
| 155 |
+ yield self.setchmod(self.profile_repository_path) |
| 156 |
+ #yield os.chown(self.profile_repository_path, 'buildbot', 'portage') |
| 157 |
if self.getProperty("profile_repository_data")['name'] != self.getProperty("repository_data")['name']: |
| 158 |
self.repository_path = yield os.path.join(self.repository_basedir, self.getProperty("repository_data")['name']) |
| 159 |
repo_path = yield pygit2.discover_repository(self.repository_path) |
| 160 |
@@ -161,6 +173,9 @@ class UpdateRepos(BuildStep): |
| 161 |
else: |
| 162 |
repo = yield pygit2.Repository(repo_path) |
| 163 |
yield self.gitPull(repo) |
| 164 |
+ # chmod and chown |
| 165 |
+ yield self.setchmod(self.profile_repository_path) |
| 166 |
+ #yield os.chown(self.repository_path, 'buildbot', 'portage') |
| 167 |
return SUCCESS |
| 168 |
|
| 169 |
class TriggerCheckForCPV(BuildStep): |
| 170 |
|
| 171 |
diff --git a/buildbot_gentoo_ci/steps/version.py b/buildbot_gentoo_ci/steps/version.py |
| 172 |
index 4c35914..426ddd4 100644 |
| 173 |
--- a/buildbot_gentoo_ci/steps/version.py |
| 174 |
+++ b/buildbot_gentoo_ci/steps/version.py |
| 175 |
@@ -131,7 +131,6 @@ class AddVersionKeyword(BuildStep): |
| 176 |
return SUCCESS |
| 177 |
print(auxdb) |
| 178 |
for keyword in auxdb: |
| 179 |
- print(keyword) |
| 180 |
status = 'stable' |
| 181 |
if keyword[0] in ["~"]: |
| 182 |
keyword = keyword[1:] |
| 183 |
@@ -170,7 +169,8 @@ class CheckPathHash(BuildStep): |
| 184 |
def run(self): |
| 185 |
self.gentooci = self.master.namedServices['services'].namedServices['gentooci'] |
| 186 |
self.repository_basedir = self.gentooci.config.project['repository_basedir'] |
| 187 |
- self.repository_path = yield os.path.join(self.repository_basedir, self.getProperty("repository_data")['name']) |
| 188 |
+ self.repository_path = yield os.path.join('/home', 'repos2', self.getProperty("repository_data")['name']) |
| 189 |
+ #self.repository_path = yield os.path.join(self.repository_basedir, self.getProperty("repository_data")['name']) |
| 190 |
self.cp_path = yield pkgsplit(self.getProperty("cpv"))[0] |
| 191 |
self.file_name = yield self.getProperty("package_data")['name'] + '-' + self.getProperty("version") + '.ebuild' |
| 192 |
self.ebuild_file = yield os.path.join(self.repository_path, self.cp_path, self.file_name) |
| 193 |
@@ -256,7 +256,9 @@ class CheckV(BuildStep): |
| 194 |
if self.getProperty("ebuild_file") is not None and self.getProperty("old_version_data") is not None: |
| 195 |
if self.getProperty("ebuild_file_hash") != self.getProperty("old_version_data")['file_hash']: |
| 196 |
addStepVData.append(GetCommitdata()) |
| 197 |
- addStepVData.append(portage_steps.SetEnvForEbuildSH()) |
| 198 |
+ #FIXME: use GetAuxMetadata insted of bugy SetEnvForEbuildSH |
| 199 |
+ #addStepVData.append(portage_steps.SetEnvForEbuildSH()) |
| 200 |
+ addStepVData.append(portage_steps.GetAuxMetadata()) |
| 201 |
addStepVData.append(AddVersion()) |
| 202 |
addStepVData.append(AddVersionKeyword()) |
| 203 |
addStepVData.append(TriggerBuildCheck()) |
| 204 |
@@ -265,7 +267,9 @@ class CheckV(BuildStep): |
| 205 |
return SUCCESS |
| 206 |
if self.getProperty("ebuild_file") is not None and self.getProperty("old_version_data") is None: |
| 207 |
addStepVData.append(GetCommitdata()) |
| 208 |
- addStepVData.append(portage_steps.SetEnvForEbuildSH()) |
| 209 |
+ #FIXME: use GetAuxMetadata insted of bugy SetEnvForEbuildSH |
| 210 |
+ #addStepVData.append(portage_steps.SetEnvForEbuildSH()) |
| 211 |
+ addStepVData.append(portage_steps.GetAuxMetadata()) |
| 212 |
addStepVData.append(AddVersion()) |
| 213 |
addStepVData.append(AddVersionKeyword()) |
| 214 |
addStepVData.append(TriggerBuildCheck()) |
Archives