From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from lists.gentoo.org (pigeon.gentoo.org [208.92.234.80]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by finch.gentoo.org (Postfix) with ESMTPS id EFC631382C5 for ; Wed, 20 May 2020 03:43:10 +0000 (UTC) Received: from pigeon.gentoo.org (localhost [127.0.0.1]) by pigeon.gentoo.org (Postfix) with SMTP id 471BEE08F4; Wed, 20 May 2020 03:43:10 +0000 (UTC) Received: from mail-pl1-f173.google.com (mail-pl1-f173.google.com [209.85.214.173]) (using TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 (128/128 bits)) (No client certificate requested) by pigeon.gentoo.org (Postfix) with ESMTPS id 1EAB7E08F4 for ; Wed, 20 May 2020 03:43:10 +0000 (UTC) Received: by mail-pl1-f173.google.com with SMTP id f15so777954plr.3 for ; Tue, 19 May 2020 20:43:10 -0700 (PDT) X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20161025; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references:mime-version:content-transfer-encoding; bh=IwyteRsWEKHalTv+4nHwKYM4ov6pxLDerMtG6SAuvuY=; b=q9aojnVFLCmY4iDRAhozmV30K62ldALvtkpu0f7m5+dXSMXmn4TpcgNOIPP19pQSwi HtVQqWys/mjd9y6mi4r3SkHhcBwo4uZTqyUcxPqquHTK1fYNOiA8q1Z7Y1no9GfH7JKK 6nocVskSerivjpfWabOUbCUzwAgTo09t8VFSxd4gr3J61IAFA3R7fbZrvtRZWwIrAkHi UIRxWMSDhDP+DCPUHy9+QT2wJ3nFU93NT92rsI8seWogkc7Yy63QggdTGugMNz7WhU9F oExh5Sbg/GGdkfi+bA/REn1bCCE+OCyLFKfcnWn+sodDQtzS1vth2ffkNYmlt6dOpXcS q5XQ== X-Gm-Message-State: AOAM532YtyQEjEdpxaplNzBhsdSVgKGdP2+grwwQp2zYb01aLw0oxbbb fOyNuQgpMRgX20yXdXeuLRy6rPHI X-Google-Smtp-Source: ABdhPJyZSqN2//qyyfR6uXzsVUhMR2uHF67WZkbL1Hi8dmOeBH8VSNfenLu5exX5KfPOABnG5Ohnig== X-Received: by 2002:a17:902:e903:: with SMTP id k3mr2506523pld.317.1589946188605; Tue, 19 May 2020 20:43:08 -0700 (PDT) Received: from localhost ([134.134.137.77]) by smtp.gmail.com with ESMTPSA id a6sm768191pfa.111.2020.05.19.20.43.07 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Tue, 19 May 2020 20:43:07 -0700 (PDT) From: Matt Turner To: gentoo-catalyst@lists.gentoo.org Cc: Matt Turner Subject: [gentoo-catalyst] [PATCH 14/21] catalyst: Convert catalyst.conf to TOML Date: Tue, 19 May 2020 20:42:19 -0700 Message-Id: <20200520034226.2870937-14-mattst88@gentoo.org> X-Mailer: git-send-email 2.26.2 In-Reply-To: <20200520034226.2870937-1-mattst88@gentoo.org> References: <20200520034226.2870937-1-mattst88@gentoo.org> Precedence: bulk List-Post: List-Help: List-Unsubscribe: List-Subscribe: List-Id: Gentoo Linux mail X-BeenThere: gentoo-catalyst@lists.gentoo.org Reply-to: gentoo-catalyst@lists.gentoo.org X-Auto-Response-Suppress: DR, RN, NRN, OOF, AutoReply MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Archives-Salt: 94aa5c14-d1f9-4cc8-87a5-41958eebe58d X-Archives-Hash: 1eba4e3bd3110aba2ee7fb46b3d3430b Signed-off-by: Matt Turner --- catalyst/base/stagebase.py | 4 +- catalyst/main.py | 47 ++++++-------------- etc/catalyst.conf | 88 ++++++++++++++++++++++---------------- 3 files changed, 67 insertions(+), 72 deletions(-) diff --git a/catalyst/base/stagebase.py b/catalyst/base/stagebase.py index febaf969..9410f151 100644 --- a/catalyst/base/stagebase.py +++ b/catalyst/base/stagebase.py @@ -857,8 +857,8 @@ class StageBase(TargetBase, ClearBase, GenBase): if 'var_tmpfs_portage' not in self.settings: continue - mount += ['-t', 'tmpfs', '-o', 'size=' + - self.settings['var_tmpfs_portage'] + 'G'] + mount += ['-t', 'tmpfs', '-o', + f"size={self.settings['var_tmpfs_portage']}G"] elif source == 'tmpfs': mount += ['-t', 'tmpfs'] elif source == 'shm': diff --git a/catalyst/main.py b/catalyst/main.py index be06ccd7..159fe454 100644 --- a/catalyst/main.py +++ b/catalyst/main.py @@ -5,6 +5,8 @@ import os import sys import textwrap +import toml + from snakeoil.process import namespaces from DeComp.definitions import (COMPRESS_DEFINITIONS, DECOMPRESS_DEFINITIONS, @@ -17,8 +19,7 @@ from catalyst.defaults import confdefaults, option_messages, DEFAULT_CONFIG_FILE from catalyst.support import CatalystError from catalyst.version import get_version - -conf_values = {} +conf_values = confdefaults def version(): @@ -30,42 +31,20 @@ def version(): def parse_config(config_files): - # search a couple of different areas for the main config file - myconf = {} - - # try and parse the config file "config_file" for config_file in config_files: log.notice('Loading configuration file: %s', config_file) try: - config = catalyst.config.ConfigParser(config_file) - myconf.update(config.get_values()) + conf_values.update(toml.load(config_file)) except Exception as e: log.critical('Could not find parse configuration file: %s: %s', config_file, e) - # now, load up the values into conf_values so that we can use them - for x in list(confdefaults): - if x in myconf: - if x == 'options': - conf_values[x] = set(myconf[x].split()) - elif x in ["decompressor_search_order"]: - conf_values[x] = myconf[x].split() - else: - conf_values[x] = myconf[x] - else: - conf_values[x] = confdefaults[x] - # print out any options messages for opt in conf_values['options']: if opt in option_messages: log.info(option_messages[opt]) - for key in ["digests", "envscript", "var_tmpfs_portage", "port_logdir", - "local_overlay", "repos"]: - if key in myconf: - conf_values[key] = myconf[key] - - if "envscript" in myconf: + if "envscript" in conf_values: log.info('Envscript support enabled.') # take care of any variable substitutions that may be left @@ -297,17 +276,17 @@ def _main(parser, opts): conf_values['DEBUG'] = opts.debug conf_values['VERBOSE'] = opts.debug or opts.verbose - options = set() + options = [] if opts.fetchonly: - options.add('fetch') + options.append('fetch') if opts.purge: - options.add('purge') + options.append('purge') if opts.purgeonly: - options.add('purgeonly') + options.append('purgeonly') if opts.purgetmponly: - options.add('purgetmponly') + options.append('purgetmponly') if opts.clear_autoresume: - options.add('clear-autoresume') + options.append('clear-autoresume') # Make sure we have some work before moving further. if not myspecfile and not mycmdline: @@ -318,7 +297,7 @@ def _main(parser, opts): # import configuration file and import our main module using those settings parse_config(myconfigs) - conf_values["options"].update(options) + conf_values["options"].extend(options) log.notice('conf_values[options] = %s', conf_values['options']) # initialize our contents generator @@ -335,7 +314,7 @@ def _main(parser, opts): if "digests" in conf_values: valid_digests = hashlib.algorithms_available - digests = set(conf_values['digests'].split()) + digests = set(conf_values['digests']) conf_values['digests'] = digests # First validate all the requested digests are valid keys. diff --git a/etc/catalyst.conf b/etc/catalyst.conf index d33be15f..2272cb86 100644 --- a/etc/catalyst.conf +++ b/etc/catalyst.conf @@ -10,53 +10,69 @@ # # $ python3 -c 'import hashlib; print(hashlib.algorithms_available)' # -digests="blake2b sha512" +digests = ["blake2b", "sha512"] # envscript allows users to set options such as http proxies, MAKEOPTS, # GENTOO_MIRRORS, or any other environment variables needed for building. # The envscript file sets environment variables like so: # export FOO="bar" -envscript="/etc/catalyst/catalystrc" - -# options set different build-time options for catalyst. Some examples are: -# autoresume = Attempt to resume a failed build, clear the autoresume flags with -# the -a option to the catalyst cmdline. -p will clear the autoresume flags -# as well as your pkgcache and kerncache -# ( This option is not fully tested, bug reports welcome ) -# bindist = enables the bindist USE flag, please see package specific definition, -# however, it is suggested to enable this if redistributing builds. -# This optional USE flag is normally cleaned from the make.conf file on -# completion of the stage. For a non-cleaned version, -# use sticky-config also (see below) -# ccache = enables build time ccache support -# distcc = enable distcc support for building. You have to set distcc_hosts in -# your spec file. -# icecream = enables icecream compiler cluster support for building -# keepwork = Prevents the removal of the working chroot path and any autoresume -# files or points. -# kerncache = keeps a tbz2 of your built kernel and modules (useful if your -# build stops in livecd-stage2) -# pkgcache = keeps a tbz2 of every built package (useful if your build stops -# prematurely) -# seedcache = use the build output of a previous target if it exists to speed up -# the copy -# sticky-config = enables the code that will keep any internal 'catalyst_use' flags -# added to the USE= for building the stage. These are usually added for legal -# or specific needs in building the the early stage. Mostly it is the -# 'bindist' USE flag option that is used for legal reasons, please see its -# specific definition. It will also keep any /etc/portage/package.* -# files or directories. -# -# (These options can be used together) -options="autoresume bindist kerncache pkgcache seedcache" +envscript = "/etc/catalyst/catalystrc" + +# options set different build-time options for catalyst. +options = [ + # Attempt to resume a failed build, clear the autoresume flags with the + # -a option to the catalyst cmdline. -p will clear the autoresume + # flags as well as your pkgcache and kerncache + "autoresume", + + # Enables the bindist USE flag, please see package specific definition, + # however, it is suggested to enable this if redistributing builds. + # This optional USE flag is normally cleaned from the make.conf file on + # completion of the stage. For a non-cleaned version, use + # sticky-config also (see below) + "bindist", + + # Enable FEATURES=ccache + # "ccache", + + # Enable FEATURES=distcc. You have to set distcc_hosts in your spec + # file. + # "distcc", + + # Enable FEATURES=icecream + # "icecream", + + # Prevents the removal of the working chroot path and any autoresume + # files or points. + # "keepwork", + + # keeps a tbz2 of your built kernel and modules (useful if your + # build stops in livecd-stage2) + "kerncache", + + # Build and use binary packages + "pkgcache", + + # Use the build output of a previous target if it exists rather than + # the tarball + "seedcache", + + # enables the code that will keep any internal 'catalyst_use' flags + # added to the USE= for building the stage. These are usually added + # for legal or specific needs in building the the early stage. Mostly + # it is the 'bindist' USE flag option that is used for legal reasons, + # please see its specific definition. It will also keep any + # /etc/portage/package.* files or directories. + # "sticky-config", +] # port_logdir is where all build logs will be kept. This dir will be automatically cleaned # of all logs over 30 days old. If left undefined the logs will remain in the build directory # as usual and get cleaned every time a stage build is restarted. -# port_logdir="/var/tmp/catalyst/tmp" +# port_logdir = "/var/tmp/catalyst/tmp" # var_tmpfs_portage will mount a tmpfs for /var/tmp/portage so building takes place in RAM # this feature requires a pretty large tmpfs ({open,libre}office needs ~8GB to build) # WARNING: If you use too much RAM everything will fail horribly and it is not our fault. # set size of /var/tmp/portage tmpfs in gigabytes -# var_tmpfs_portage=16 +# var_tmpfs_portage = 16 -- 2.26.2