[gentoo-commits] proj/catalyst:master commit in: targets/stage3/, catalyst/base/, etc/, targets/support/, targets/stage2/, ...

[Brian Dolbec]

commit:     b30dd97d672d61bb40340799330ef7863b173b0c
Author:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
AuthorDate: Sun Mar 19 22:04:49 2017 +0000
Commit:     Brian Dolbec <dolsen <AT> gentoo <DOT> org>
CommitDate: Wed Nov 22 01:16:22 2017 +0000
URL:        https://gitweb.gentoo.org/proj/catalyst.git/commit/?id=b30dd97d

Unify all make.conf settings and writing

This excludes stage1 builds which needs to modify the seed stage's make.conf for
initial stage building.

 catalyst/base/stagebase.py          | 221 ++++++++++++++++++------------------
 etc/catalyst.conf                   |   9 ++
 targets/stage2/stage2-controller.sh |   2 -
 targets/stage3/stage3-controller.sh |   2 -
 targets/stage4/stage4-controller.sh |   2 -
 targets/support/chroot-functions.sh |  16 +--
 targets/support/functions.sh        |  12 --
 7 files changed, 130 insertions(+), 134 deletions(-)

diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py
index d44db985..a7aa7f41 100644
--- a/catalyst/base/stagebase.py
+++ b/catalyst/base/stagebase.py
@@ -548,27 +548,29 @@ class StageBase(TargetBase, ClearBase, GenBase):
 		return
 
 	def set_use(self):
-		if self.settings["spec_prefix"] + "/use" in self.settings:
-			self.settings["use"] = \
-				self.settings[self.settings["spec_prefix"] + "/use"]
-			del self.settings[self.settings["spec_prefix"] + "/use"]
-		if "use" not in self.settings:
-			self.settings["use"] = ""
-		if isinstance(self.settings['use'], str):
-			self.settings["use"] = self.settings["use"].split()
+		use = self.settings["spec_prefix"] + "/use"
+		if use in self.settings:
+			if isinstance(self.settings[use], str):
+				self.settings["use"] = self.settings[use].split()
+			self.settings["use"] = self.settings[use]
+			del self.settings[use]
+		else:
+			self.settings["use"] = []
 
 	def set_catalyst_use(self):
-		if self.settings["spec_prefix"] + "/catalyst_use" in self.settings:
-			self.settings["catalyst_use"] = \
-				self.settings[self.settings["spec_prefix"]+"/catalyst_use"]
-			del self.settings[self.settings["spec_prefix"]+"/catalyst_use"]
-		if "catalyst_use" not in self.settings:
-			self.settings["catalyst_use"] = ""
-		if isinstance(self.settings['catalyst_use'], str):
-			self.settings["catalyst_use"] = self.settings["catalyst_use"].split()
+		catalyst_use = self.settings["spec_prefix"] + "/catalyst_use"
+		if catalyst_use in self.settings:
+			if isinstance(self.settings[catalyst_use], str):
+				self.settings["catalyst_use"] = self.settings[catalyst_use].split()
+			else:
+				self.settings["catalyst_use"] = self.settings[catalyst_use]
+			del self.settings[catalyst_use]
+		else:
+			self.settings["catalyst_use"] = []
 
 		# Force bindist when options ask for it
-		if "BINDIST" in self.settings:
+		if "bindist" in self.settings["options"]:
+			log.debug("Enabling bindist USE flag")
 			self.settings["catalyst_use"].append("bindist")
 
 	def set_stage_path(self):
@@ -1071,92 +1073,98 @@ class StageBase(TargetBase, ClearBase, GenBase):
 		makepath = normpath(self.settings["chroot_path"] +
 			self.settings["make_conf"])
 		clear_path(makepath)
-		myf = open(makepath, "w")
-		myf.write("# These settings were set by the catalyst build script "
-				"that automatically\n# built this stage.\n")
-		myf.write("# Please consult "
-				"/usr/share/portage/config/make.conf.example "
-				"for a more\n# detailed example.\n")
-
-		for flags in ["CFLAGS", "CXXFLAGS", "FCFLAGS", "FFLAGS", "LDFLAGS",
-					"ASFLAGS"]:
-			if not flags in self.settings:
-				continue
-			if flags in ["LDFLAGS", "ASFLAGS"]:
-				myf.write("# %s is unsupported.  USE AT YOUR OWN RISK!\n"
-						% flags)
-			if (flags is not "CFLAGS" and
-				self.settings[flags] == self.settings["CFLAGS"]):
-				myf.write('%s="${CFLAGS}"\n' % flags)
-			elif isinstance(self.settings[flags], list):
-				myf.write('%s="%s"\n'
-						% (flags, ' '.join(self.settings[flags])))
-			else:
-				myf.write('%s="%s"\n'
-						% (flags, self.settings[flags]))
-
-		if "CBUILD" in self.settings:
-			myf.write("# This should not be changed unless you know exactly"
-				" what you are doing.  You\n# should probably be "
-				"using a different stage, instead.\n")
-			myf.write('CBUILD="' + self.settings["CBUILD"] + '"\n')
-
-		if "CHOST" in self.settings:
-			myf.write("# WARNING: Changing your CHOST is not something "
-				"that should be done lightly.\n# Please consult "
-				"https://wiki.gentoo.org/wiki/Changing_the_CHOST_variable "
-				"before changing.\n")
-			myf.write('CHOST="' + self.settings["CHOST"] + '"\n')
-
-		# Figure out what our USE vars are for building
-		myusevars = []
-		if "HOSTUSE" in self.settings:
-			myusevars.extend(self.settings["HOSTUSE"])
-
-		if "use" in self.settings:
-			myusevars.extend(self.settings["use"])
-
-		if myusevars:
-			myf.write("# These are the USE and USE_EXPAND flags that were "
-					"used for\n# building in addition to what is provided "
-					"by the profile.\n")
-			myusevars = sorted(set(myusevars))
-			myf.write('USE="' + ' '.join(myusevars) + '"\n')
-			if '-*' in myusevars:
-				log.warning(
-					'The use of -* in %s/use will cause portage to ignore\n'
-					'package.use in the profile and portage_confdir.\n'
-					"You've been warned!", self.settings['spec_prefix'])
-
-		myuseexpandvars = {}
-		if "HOSTUSEEXPAND" in self.settings:
-			for hostuseexpand in self.settings["HOSTUSEEXPAND"]:
-				myuseexpandvars.update(
-					{hostuseexpand:self.settings["HOSTUSEEXPAND"][hostuseexpand]})
-
-		if myuseexpandvars:
-			for hostuseexpand in myuseexpandvars:
-				myf.write(hostuseexpand + '="' +
-					' '.join(myuseexpandvars[hostuseexpand]) + '"\n')
-		# write out a shipable version
-		target_portdir = normpath(self.settings["repo_basedir"] + "/" +
-			self.settings["repo_name"])
-
-		myf.write('PORTDIR="%s"\n' % target_portdir)
-		myf.write('DISTDIR="%s"\n' % self.settings['target_distdir'])
-		myf.write('PKGDIR="%s"\n' % self.settings['target_pkgdir'])
-		if setup:
-			# Setup the portage overlay
-			if "portage_overlay" in self.settings:
-				myf.write('PORTDIR_OVERLAY="%s"\n' %  self.settings["local_overlay"])
-
-		# Set default locale for system responses. #478382
-		myf.write(
-			'\n'
-			'# This sets the language of build output to English.\n'
-			'# Please keep this setting intact when reporting bugs.\n'
-			'LC_MESSAGES=C\n')
-		myf.close()
+		with open(makepath, "w") as myf:
+			log.notice("Writing the stage make.conf to: %s" % makepath)
+			myf.write("# These settings were set by the catalyst build script "
+					"that automatically\n# built this stage.\n")
+			myf.write("# Please consult "
+					"/usr/share/portage/config/make.conf.example "
+					"for a more\n# detailed example.\n")
+
+			for flags in ["CFLAGS", "CXXFLAGS", "FCFLAGS", "FFLAGS", "LDFLAGS",
+						"ASFLAGS"]:
+				if not flags in self.settings:
+					continue
+				if flags in ["LDFLAGS", "ASFLAGS"]:
+					myf.write("# %s is unsupported.  USE AT YOUR OWN RISK!\n"
+							% flags)
+				if (flags is not "CFLAGS" and
+					self.settings[flags] == self.settings["CFLAGS"]):
+					myf.write('%s="${CFLAGS}"\n' % flags)
+				elif isinstance(self.settings[flags], list):
+					myf.write('%s="%s"\n'
+							% (flags, ' '.join(self.settings[flags])))
+				else:
+					myf.write('%s="%s"\n'
+							% (flags, self.settings[flags]))
+
+			if "CBUILD" in self.settings:
+				myf.write("# This should not be changed unless you know exactly"
+					" what you are doing.  You\n# should probably be "
+					"using a different stage, instead.\n")
+				myf.write('CBUILD="' + self.settings["CBUILD"] + '"\n')
+
+			if "CHOST" in self.settings:
+				myf.write("# WARNING: Changing your CHOST is not something "
+					"that should be done lightly.\n# Please consult "
+					"https://wiki.gentoo.org/wiki/Changing_the_CHOST_variable "
+					"before changing.\n")
+				myf.write('CHOST="' + self.settings["CHOST"] + '"\n')
+
+			# Figure out what our USE vars are for building
+			myusevars = []
+			if "bindist" in self.settings["options"]:
+				myf.write("\n# NOTE: This stage was built with the bindist Use flag enabled\n")
+			if setup or "sticky-config" in self.settings["options"]:
+				myusevars.extend(self.settings["catalyst_use"])
+				log.notice("STICKY-CONFIG is enabled")
+			if "HOSTUSE" in self.settings:
+				myusevars.extend(self.settings["HOSTUSE"])
+
+			if "use" in self.settings:
+				myusevars.extend(self.settings["use"])
+
+			if myusevars:
+				myf.write("# These are the USE and USE_EXPAND flags that were "
+						"used for\n# building in addition to what is provided "
+						"by the profile.\n")
+				myusevars = sorted(set(myusevars))
+				myf.write('USE="' + ' '.join(myusevars) + '"\n')
+				if '-*' in myusevars:
+					log.warning(
+						'The use of -* in %s/use will cause portage to ignore\n'
+						'package.use in the profile and portage_confdir.\n'
+						"You've been warned!", self.settings['spec_prefix'])
+
+			myuseexpandvars = {}
+			if "HOSTUSEEXPAND" in self.settings:
+				for hostuseexpand in self.settings["HOSTUSEEXPAND"]:
+					myuseexpandvars.update(
+						{hostuseexpand:self.settings["HOSTUSEEXPAND"][hostuseexpand]})
+
+			if myuseexpandvars:
+				for hostuseexpand in myuseexpandvars:
+					myf.write(hostuseexpand + '="' +
+						' '.join(myuseexpandvars[hostuseexpand]) + '"\n')
+			# write out a shipable version
+			target_portdir = normpath(self.settings["repo_basedir"] + "/" +
+				self.settings["repo_name"])
+
+			myf.write('PORTDIR="%s"\n' % target_portdir)
+			myf.write('DISTDIR="%s"\n' % self.settings['target_distdir'])
+			myf.write('PKGDIR="%s"\n' % self.settings['target_pkgdir'])
+			if setup:
+				# Setup the portage overlay
+				if "portage_overlay" in self.settings:
+					myf.write('PORTDIR_OVERLAY="%s"\n' %  self.settings["local_overlay"])
+
+			# Set default locale for system responses. #478382
+			myf.write(
+				'\n'
+				'# This sets the language of build output to English.\n'
+				'# Please keep this setting intact when reporting bugs.\n'
+				'LC_MESSAGES=C\n')
+
 
 	def fsscript(self):
 		if "autoresume" in self.settings["options"] \
@@ -1197,11 +1205,8 @@ class StageBase(TargetBase, ClearBase, GenBase):
 		if os.path.exists(self.settings["chroot_path"] + self.settings["local_overlay"]):
 			clear_path(self.settings["chroot_path"] + self.settings["local_overlay"])
 
-			make_conf = self.settings['chroot_path'] + self.settings['make_conf']
-			try:
-				self.write_make_conf(setup=False)
-			except OSError as e:
-				raise CatalystError('Could not update %s: %s' % (make_conf, e))
+		# re-write the make.conf to be sure it is clean
+		self.write_make_conf(setup=False)
 
 		# Clean up old and obsoleted files in /etc
 		if os.path.exists(self.settings["stage_path"]+"/etc"):

diff --git a/etc/catalyst.conf b/etc/catalyst.conf
index b4db063c..c22cd439 100644
--- a/etc/catalyst.conf
+++ b/etc/catalyst.conf
@@ -53,6 +53,9 @@ hash_function="crc32"
 #	( This option is not fully tested, bug reports welcome )
 # bindist = enables the bindist USE flag, please see package specific definition,
 #	however, it is suggested to enable this if redistributing builds.
+#	This optional USE flag is normally cleaned from the make.conf file on
+#	completion of the stage.  For a non-cleaned version,
+#	use sticky-config also (see below)
 # ccache = enables build time ccache support
 # distcc = enable distcc support for building. You have to set distcc_hosts in
 #	your spec file.
@@ -70,6 +73,12 @@ hash_function="crc32"
 # snapcache = cache the snapshot so that it can be bind-mounted into the chroot.
 #	WARNING: moving parts of the portage tree from within fsscript *will* break
 #	your cache. The cache is unlinked before any empty or rm processing, though.
+# sticky-config = enables the code that will keep any internal 'catalyst_use' flags
+#	added to the USE= for building the stage.  These ae usually added for legal
+#	or specific needs in building the the early stage.  Mostly it is the
+#	'bindist' USE flag option that is used for legal reasons, please see its
+#	specific definition.  It will also keep any /etc/portage/package.*
+#	files or directories.
 #
 # (These options can be used together)
 options="autoresume bindist kerncache pkgcache seedcache snapcache"

diff --git a/targets/stage2/stage2-controller.sh b/targets/stage2/stage2-controller.sh
index 41bd43bb..2eee79f3 100755
--- a/targets/stage2/stage2-controller.sh
+++ b/targets/stage2/stage2-controller.sh
@@ -10,8 +10,6 @@ case $1 in
 	;;
 
 	run)
-		prepare_portage
-
 		shift
 		export clst_packages="$*"
 		exec_in_chroot \

diff --git a/targets/stage3/stage3-controller.sh b/targets/stage3/stage3-controller.sh
index eaa40b3d..2d415e40 100755
--- a/targets/stage3/stage3-controller.sh
+++ b/targets/stage3/stage3-controller.sh
@@ -10,8 +10,6 @@ case $1 in
 	;;
 
 	run)
-		prepare_portage
-
 		shift
 		export clst_packages="$*"
 		exec_in_chroot ${clst_shdir}/${clst_target}/${clst_target}-chroot.sh

diff --git a/targets/stage4/stage4-controller.sh b/targets/stage4/stage4-controller.sh
index 4c5d5a06..6a876c82 100755
--- a/targets/stage4/stage4-controller.sh
+++ b/targets/stage4/stage4-controller.sh
@@ -10,8 +10,6 @@ case $1 in
 	;;
 
 	pre-kmerge)
-		prepare_portage
-
 		# Sets up the build environment before any kernels are compiled
 		exec_in_chroot ${clst_shdir}/support/pre-kmerge.sh
 	;;

diff --git a/targets/support/chroot-functions.sh b/targets/support/chroot-functions.sh
index b01bbbbd..e44de61b 100755
--- a/targets/support/chroot-functions.sh
+++ b/targets/support/chroot-functions.sh
@@ -183,14 +183,14 @@ setup_gcc(){
 setup_pkgmgr(){
 	# Set bindist USE flag if clst_BINDIST is set
 	# this is handled independantly in stage2, changes here should be mirrored there
-	if [ "${clst_target}" != "stage1" ] && [ -e "${clst_make_conf}" ] \
-		&& [ -n "${clst_BINDIST}" ]; then
-		if grep -q ^USE "${clst_make_conf}"; then
-			echo "USE=\"\${USE} bindist\"" >> "${clst_make_conf}"
-		else
-			echo "USE=\"bindist\"" >> "${clst_make_conf}"
-		fi
-	fi
+	#if [ "${clst_target}" != "stage1" ] && [ -e "${clst_make_conf}" ] \
+	#	&& [ -n "${clst_BINDIST}" ]; then
+	#	if grep -q ^USE "${clst_make_conf}"; then
+	#		echo "USE=\"\${USE} bindist\"" >> "${clst_make_conf}"
+	#	else
+	#		echo "USE=\"bindist\"" >> "${clst_make_conf}"
+	#	fi
+	#fi
 
 	# We need to merge our package manager with USE="build" set in case it is
 	# portage to avoid frying our /etc/portage/make.conf file.  Otherwise, we could

diff --git a/targets/support/functions.sh b/targets/support/functions.sh
index f743d419..ac4ec6c7 100755
--- a/targets/support/functions.sh
+++ b/targets/support/functions.sh
@@ -16,19 +16,7 @@ delete_from_chroot(){
 	fi
 }
 
-prepare_portage() {
-
-	echo "CATALYST_USE=\"${clst_CATALYST_USE}\"" >> ${clst_chroot_path}${clst_make_conf}
-	sed -i -e "/^USE=\"/s//\${CATALYST_USE} ${USE} /" ${clst_chroot_path}${clst_make_conf}
-}
-
 clear_portage() {
-	# Clean-up USE again
-	[ -e ${clst_chroot_path}${clst_make_conf} ] && echo "Drop \$CATALYST_USE from USE on ${clst_chroot_path}${clst_make_conf}"
-	[ -e ${clst_chroot_path}${clst_make_conf} ] && sed -i -e "/^USE=\"/s/\${CATALYST_USE} //" ${clst_chroot_path}${clst_make_conf}
-	[ -e ${clst_chroot_path}${clst_make_conf} ] && echo "Remove \$CATALYST_USE on ${clst_chroot_path}${clst_make_conf}"
-	[ -e ${clst_chroot_path}${clst_make_conf} ] && sed -i -e "/^CATALYST_USE/d" ${clst_chroot_path}${clist_make_conf}
-
 	if [ -n "${clst_portage_prefix}" ]; then
 		for dir in "keywords", "mask", "unmask", "use"; do
 			[ -d ${clst_chroot_path}/etc/portage/package.${dir}/${clst_portage_prefix} ] &&