atomic-desktops/ci-test
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

comps-sync.py: Fix a few Python lints

Browse source
This commit is contained in:
Timothée Ravier 2023-08-14 13:06:18 +02:00
parent a17aa8e553
commit 4522a99c07
1 changed files with 41 additions and 33 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

72
comps-sync.py
View file

@ -1,41 +1,55 @@
#!/usr/bin/python3 #!/usr/bin/python3
# Usage: ./comps-sync.py /path/to/comps-f39.xml.in
#
# Can both remove packages from the manifest
# which are not mentioned in comps, and add packages from
# comps.
import os, sys, subprocess, argparse, shlex, json, yaml, re '''
Usage: ./comps-sync.py /path/to/comps-f39.xml.in
Can both remove packages from the manifest which are not mentioned in comps,
and add packages from comps.
'''
import argparse
import re
import sys
import yaml
import libcomps import libcomps
ARCHES = ("x86_64", "aarch64", "ppc64le") ARCHES = ("x86_64", "aarch64", "ppc64le")
def fatal(msg): def fatal(msg):
print >>sys.stderr, msg '''Print the error message and exit.'''
print(msg, file = sys.stderr)
sys.exit(1) sys.exit(1)
def format_pkgtype(n): def format_pkgtype(pkgtype):
if n == libcomps.PACKAGE_TYPE_DEFAULT: '''Return a printable string from a libcomps package type.'''
if pkgtype == libcomps.PACKAGE_TYPE_DEFAULT:
return 'default' return 'default'
elif n == libcomps.PACKAGE_TYPE_MANDATORY: if pkgtype == libcomps.PACKAGE_TYPE_MANDATORY:
return 'mandatory' return 'mandatory'
else:
assert False assert False
def write_manifest(fpath, pkgs, include=None): def write_manifest(fpath, pkgs, include=None):
with open(fpath, 'w') as f: '''Write the package list in a manifest.'''
with open(fpath, 'w', encoding='UTF-8') as f:
f.write("# DO NOT EDIT! This content is generated from comps-sync.py\n") f.write("# DO NOT EDIT! This content is generated from comps-sync.py\n")
if include is not None: if include is not None:
f.write("include: {}\n".format(include)) f.write(f'include: {include}\n')
f.write("packages:\n") f.write("packages:\n")
for pkg in sorted(pkgs['all']): for pkg in sorted(pkgs['all']):
f.write(" - {}\n".format(pkg)) f.write(f' - {pkg}\n')
for arch in ARCHES: for arch in ARCHES:
if pkgs[arch]: if pkgs[arch]:
f.write(f"packages-{arch}:\n") f.write(f"packages-{arch}:\n")
for pkg in sorted(pkgs[arch]): for pkg in sorted(pkgs[arch]):
f.write(" - {}\n".format(pkg)) f.write(f' - {pkg}\n')
print("Wrote {}".format(fpath)) print(f'Wrote {fpath}')
def is_exclude_listed(pkgname):
'''Check if pkgname is in the exclude list.'''
for br in comps_exclude_list_all:
if br.match(pkgname):
return True
return False
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
parser.add_argument("--save", help="Write changes", action='store_true') parser.add_argument("--save", help="Write changes", action='store_true')
@ -46,7 +60,7 @@ args = parser.parse_args()
print("Syncing packages common to all desktops:") print("Syncing packages common to all desktops:")
base_pkgs_path = 'fedora-common-ostree-pkgs.yaml' base_pkgs_path = 'fedora-common-ostree-pkgs.yaml'
with open(base_pkgs_path) as f: with open(base_pkgs_path, encoding='UTF-8') as f:
manifest = yaml.safe_load(f) manifest = yaml.safe_load(f)
manifest_packages = {} manifest_packages = {}
manifest_packages['all'] = set(manifest['packages']) manifest_packages['all'] = set(manifest['packages'])
@ -56,7 +70,7 @@ for arch in ARCHES:
else: else:
manifest_packages[arch] = set() manifest_packages[arch] = set()
with open('comps-sync-exclude-list.yml') as f: with open('comps-sync-exclude-list.yml', encoding='UTF-8') as f:
doc = yaml.safe_load(f) doc = yaml.safe_load(f)
comps_exclude_list = doc['exclude_list'] comps_exclude_list = doc['exclude_list']
comps_include_list = doc['include_list'] comps_include_list = doc['include_list']
@ -64,12 +78,6 @@ with open('comps-sync-exclude-list.yml') as f:
comps_desktop_exclude_list = doc['desktop_exclude_list'] comps_desktop_exclude_list = doc['desktop_exclude_list']
comps_exclude_list_all = [re.compile(x) for x in doc['exclude_list_all_regexp']] comps_exclude_list_all = [re.compile(x) for x in doc['exclude_list_all_regexp']]
def is_exclude_listed(pkgname):
for br in comps_exclude_list_all:
if br.match(pkgname):
return True
return False
# Parse comps, and build up a set of all packages so we # Parse comps, and build up a set of all packages so we
# can find packages not listed in comps *at all*, beyond # can find packages not listed in comps *at all*, beyond
# just the workstation environment. # just the workstation environment.
@ -128,9 +136,9 @@ n_manifest_new = len(comps_unknown)
if n_manifest_new == 0: if n_manifest_new == 0:
print(" - All manifest packages are already listed in comps.") print(" - All manifest packages are already listed in comps.")
else: else:
print(" - {} packages not in {}:".format(n_manifest_new, ws_env_name)) print(f' - {n_manifest_new} packages not in {ws_env_name}:')
for (pkg, arch) in sorted(comps_unknown, key = lambda x: x[0]): for (pkg, arch) in sorted(comps_unknown, key = lambda x: x[0]):
print(' {} (arch: {})'.format(pkg, arch)) print(f' {pkg} (arch: {arch})')
manifest_packages[arch].remove(pkg) manifest_packages[arch].remove(pkg)
# Look for packages in workstation but not in the manifest # Look for packages in workstation but not in the manifest
@ -153,7 +161,7 @@ n_comps_new = len(ws_added)
if n_comps_new == 0: if n_comps_new == 0:
print(" - All comps packages are already listed in manifest.") print(" - All comps packages are already listed in manifest.")
else: else:
print(" - {} packages not in manifest:".format(n_comps_new)) print(f' - {n_comps_new} packages not in manifest:')
for pkg in sorted(ws_added): for pkg in sorted(ws_added):
(req, groups, arches) = ws_added[pkg] (req, groups, arches) = ws_added[pkg]
print(' {} ({}, groups: {}, arches: {})'.format(pkg, format_pkgtype(req), ', '.join(groups), ', '.join(arches))) print(' {} ({}, groups: {}, arches: {})'.format(pkg, format_pkgtype(req), ', '.join(groups), ', '.join(arches)))
@ -177,9 +185,9 @@ desktops_comps_groups = {
# Generate treefiles for all desktops # Generate treefiles for all desktops
for desktop, groups in desktops_comps_groups.items(): for desktop, groups in desktops_comps_groups.items():
print() print()
print("Syncing packages for {}:".format(desktop)) print(f'Syncing packages for {desktop}:')
manifest_path = '{}-desktop-pkgs.yaml'.format(desktop) manifest_path = f'{desktop}-desktop-pkgs.yaml'
with open(manifest_path, encoding='UTF-8') as f: with open(manifest_path, encoding='UTF-8') as f:
manifest = yaml.safe_load(f) manifest = yaml.safe_load(f)
manifest_packages = {} manifest_packages = {}
@ -221,9 +229,9 @@ for desktop, groups in desktops_comps_groups.items():
if n_manifest_new == 0: if n_manifest_new == 0:
print(" - All manifest packages are already listed in comps.") print(" - All manifest packages are already listed in comps.")
else: else:
print(" - {} packages not in {}:".format(n_manifest_new, ws_env_name)) print(f' - {n_manifest_new} packages not in {ws_env_name}:')
for (pkg, arch) in sorted(comps_unknown, key = lambda x: x[0]): for (pkg, arch) in sorted(comps_unknown, key = lambda x: x[0]):
print(' {} (arch: {})'.format(pkg, arch)) print(f' {pkg} (arch: {arch})')
manifest_packages[arch].remove(pkg) manifest_packages[arch].remove(pkg)
@ -247,7 +255,7 @@ for desktop, groups in desktops_comps_groups.items():
if n_comps_new == 0: if n_comps_new == 0:
print(" - All comps packages are already listed in manifest.") print(" - All comps packages are already listed in manifest.")
else: else:
print(" - {} packages not in {} manifest:".format(n_comps_new, desktop)) print(f' - {n_comps_new} packages not in {desktop} manifest:')
for pkg in sorted(desktop_pkgs_added): for pkg in sorted(desktop_pkgs_added):
arches = desktop_pkgs_added[pkg] arches = desktop_pkgs_added[pkg]
print(' {} (arches: {})'.format(pkg, ', '.join(arches))) print(' {} (arches: {})'.format(pkg, ', '.join(arches)))