atomic-desktops/ci-test
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

comps-sync.py: handle arch-dependent package specs (#311)

Browse source 
comps allows for a package to be included in a group only on
specified arches, with a line like this:

      <packagereq arch="aarch64,x86_64" type="default">kio-gdrive</packagereq>

up till now, `comps-sync.py` effectively ignored this, treating
such a line exactly the same as this:

      <packagereq type="default">kio-gdrive</packagereq>

and attempting to include the package even when building on e.g.
ppc64le.

Solving this is unfortunately tricky due to exactly how libcomps
handles these arch restrictions. It does not expose the list of
arches as a property of each returned 'package' object when you
do a group search. Instead you're supposed to filter the group
search down to the arch(es) you are interested in, and it only
includes the appropriate packages for that arch. So all we can
do is run our queries multiple times, once for each arch we may
wish to build on, keep track of the results per arch as we go,
and do some fancy footwork to keep track of the mapping between
"archful" comps entries and the lists in the YAML files.

Signed-off-by: Adam Williamson <awilliam@redhat.com>
This commit is contained in:
Adam Williamson 2022-10-31 16:16:03 -07:00
parent 0e38562e28
commit fa798595cf
1 changed files with 111 additions and 52 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

163
comps-sync.py
View file

@ -8,6 +8,8 @@
import os, sys, subprocess, argparse, shlex, json, yaml, re
import libcomps
ARCHES = ("x86_64", "aarch64", "ppc64le")
def fatal(msg):
print >>sys.stderr, msg
sys.exit(1)
@ -26,8 +28,13 @@ def write_manifest(fpath, pkgs, include=None):
if include is not None:
f.write("include: {}\n".format(include))
f.write("packages:\n")
for pkg in sorted(pkgs):
for pkg in sorted(pkgs['all']):
f.write(" - {}\n".format(pkg))
for arch in ARCHES:
if pkgs[arch]:
f.write(f"packages-{arch}:\n")
for pkg in sorted(pkgs[arch]):
f.write(" - {}\n".format(pkg))
print("Wrote {}".format(fpath))
parser = argparse.ArgumentParser()
@ -41,7 +48,13 @@ print("Syncing packages common to all desktops:")
base_pkgs_path = 'fedora-common-ostree-pkgs.yaml'
with open(base_pkgs_path) as f:
manifest = yaml.safe_load(f)
manifest_packages = set(manifest['packages'])
manifest_packages = {}
manifest_packages['all'] = set(manifest['packages'])
for arch in ARCHES:
if f'packages-{arch}' in manifest:
manifest_packages[arch] = set(manifest[f'packages-{arch}'])
else:
manifest_packages[arch] = set()
with open('comps-sync-exclude-list.yml') as f:
doc = yaml.safe_load(f)
@ -70,24 +83,27 @@ ws_ostree_name = 'workstation-ostree-support'
ws_environ = comps.environments[ws_env_name]
ws_pkgs = {}
for gid in ws_environ.group_ids:
group = comps.groups_match(id=gid.name)[0]
if gid.name in comps_exclude_list_groups:
continue
exclude_list = comps_exclude_list.get(gid.name, set())
for pkg in group.packages:
pkgname = pkg.name
if pkg.type not in (libcomps.PACKAGE_TYPE_DEFAULT,
libcomps.PACKAGE_TYPE_MANDATORY):
continue
if pkgname in exclude_list or is_exclude_listed(pkgname):
continue
pkgdata = ws_pkgs.get(pkgname)
if pkgdata is None:
ws_pkgs[pkgname] = pkgdata = (pkg.type, set([gid.name]))
if (pkgdata[0] == libcomps.PACKAGE_TYPE_DEFAULT and
pkg.type == libcomps.PACKAGE_TYPE_MANDATORY):
ws_pkgs[pkgname] = pkgdata = (pkg.type, pkgdata[1])
pkgdata[1].add(gid.name)
for arch in ARCHES:
filtered = comps.arch_filter([arch])
group = filtered.groups_match(id=gid.name)[0]
for pkg in group.packages:
pkgname = pkg.name
if pkg.type not in (libcomps.PACKAGE_TYPE_DEFAULT,
libcomps.PACKAGE_TYPE_MANDATORY):
continue
if pkgname in exclude_list or is_exclude_listed(pkgname):
continue
pkgdata = ws_pkgs.get(pkgname)
if pkgdata is None:
ws_pkgs[pkgname] = pkgdata = (pkg.type, set([gid.name]), set([arch]))
if (pkgdata[0] == libcomps.PACKAGE_TYPE_DEFAULT and
pkg.type == libcomps.PACKAGE_TYPE_MANDATORY):
ws_pkgs[pkgname] = pkgdata = (pkg.type, pkgdata[1], pkgdata[2])
pkgdata[1].add(gid.name)
pkgdata[2].add(arch)
ws_ostree_pkgs = set()
for pkg in comps.groups_match(id=ws_ostree_name)[0].packages:
@ -95,11 +111,17 @@ for pkg in comps.groups_match(id=ws_ostree_name)[0].packages:
ws_ostree_pkgs.add(pkg.name)
comps_unknown = set()
for pkg in manifest_packages:
if (pkg not in comps_include_list and
pkg not in ws_pkgs and
pkg not in ws_ostree_pkgs):
comps_unknown.add(pkg)
for arch in manifest_packages:
for pkg in manifest_packages[arch]:
if arch == "all":
if pkg in ws_pkgs and set(ws_pkgs[pkg][2]) == set(ARCHES):
continue
else:
if pkg in ws_pkgs and arch in ws_pkgs[pkg][2]:
continue
if (pkg not in comps_include_list and
pkg not in ws_ostree_pkgs):
comps_unknown.add((pkg, arch))
# Look for packages in the manifest but not in comps at all
n_manifest_new = len(comps_unknown)
@ -107,16 +129,25 @@ if n_manifest_new == 0:
print(" - All manifest packages are already listed in comps.")
else:
print(" - {} packages not in {}:".format(n_manifest_new, ws_env_name))
for pkg in sorted(comps_unknown):
print(' {}'.format(pkg))
manifest_packages.remove(pkg)
for (pkg, arch) in sorted(comps_unknown, key = lambda x: x[0]):
print(' {} (arch: {})'.format(pkg, arch))
manifest_packages[arch].remove(pkg)
# Look for packages in workstation but not in the manifest
ws_added = {}
for (pkg,data) in ws_pkgs.items():
if pkg not in manifest_packages:
ws_added[pkg] = data
manifest_packages.add(pkg)
if set(ARCHES) == set(data[2]):
if pkg not in manifest_packages['all']:
ws_added[pkg] = data
manifest_packages['all'].add(pkg)
else:
for arch in data[2]:
if pkg not in manifest_packages[arch]:
manifest_packages[arch].add(pkg)
if pkg not in ws_added:
ws_added[pkg] = (data[0], data[1], set([arch]))
else:
ws_added[pkg][2].add(arch)
n_comps_new = len(ws_added)
if n_comps_new == 0:
@ -124,8 +155,8 @@ if n_comps_new == 0:
else:
print(" - {} packages not in manifest:".format(n_comps_new))
for pkg in sorted(ws_added):
(req, groups) = ws_added[pkg]
print(' {} ({}, groups: {})'.format(pkg, format_pkgtype(req), ', '.join(groups)))
(req, groups, arches) = ws_added[pkg]
print(' {} ({}, groups: {}, arches: {})'.format(pkg, format_pkgtype(req), ', '.join(groups), ', '.join(arches)))
if (n_manifest_new > 0 or n_comps_new > 0) and args.save:
write_manifest(base_pkgs_path, manifest_packages)
@ -139,37 +170,65 @@ for desktop in [ 'gnome-desktop', 'kde-desktop', 'xfce-desktop',
manifest_path = '{}-pkgs.yaml'.format(desktop)
with open(manifest_path) as f:
manifest = yaml.safe_load(f)
manifest_packages = set(manifest['packages'])
manifest_packages = {}
manifest_packages['all'] = set(manifest['packages'])
for arch in ARCHES:
if f'packages-{arch}' in manifest:
manifest_packages[arch] = set(manifest[f'packages-{arch}'])
else:
manifest_packages[arch] = set()
# Filter packages in the comps desktop group using the exclude_list
comps_group_pkgs = set()
for pkg in comps.groups_match(id=desktop)[0].packages:
pkgname = pkg.name
exclude_list = comps_desktop_exclude_list.get(desktop, set())
if pkgname in exclude_list or is_exclude_listed(pkgname):
continue
comps_group_pkgs.add(pkg.name)
comps_group_pkgs = {}
for arch in ARCHES:
filtered = comps.arch_filter([arch])
for pkg in filtered.groups_match(id=desktop)[0].packages:
pkgname = pkg.name
exclude_list = comps_desktop_exclude_list.get(desktop, set())
if pkgname in exclude_list or is_exclude_listed(pkgname):
continue
if pkgname in comps_group_pkgs:
comps_group_pkgs[pkgname].add(arch)
else:
comps_group_pkgs[pkgname] = set([arch])
# Look for packages in the manifest but not in comps group
comps_unknown = set()
for pkg in manifest_packages:
if pkg not in comps_group_pkgs:
comps_unknown.add(pkg)
for arch in manifest_packages:
for pkg in manifest_packages[arch]:
if arch == "all":
if pkg in comps_group_pkgs and set(comps_group_pkgs[pkg]) == set(ARCHES):
continue
else:
if pkg in comps_group_pkgs and arch in comps_group_pkgs[pkg]:
continue
comps_unknown.add((pkg, arch))
# Look for packages in the manifest but not in comps at all
n_manifest_new = len(comps_unknown)
if n_manifest_new == 0:
print(" - All manifest packages are already listed in comps.")
else:
print(" - {} packages not in {} comps group:".format(n_manifest_new, desktop))
for pkg in sorted(comps_unknown):
print(' {}'.format(pkg))
manifest_packages.remove(pkg)
print(" - {} packages not in {}:".format(n_manifest_new, ws_env_name))
for (pkg, arch) in sorted(comps_unknown, key = lambda x: x[0]):
print(' {} (arch: {})'.format(pkg, arch))
manifest_packages[arch].remove(pkg)
# Look for packages in comps but not in the manifest
desktop_pkgs_added = set()
for pkg in comps_group_pkgs:
if pkg not in manifest_packages:
desktop_pkgs_added.add(pkg)
desktop_pkgs_added = {}
for (pkg, parches) in comps_group_pkgs.items():
if set(ARCHES) == set(parches):
if pkg not in manifest_packages['all']:
desktop_pkgs_added[pkg] = parches
manifest_packages['all'].add(pkg)
else:
for arch in parches:
if pkg not in manifest_packages[arch]:
manifest_packages[arch].add(pkg)
if pkg not in desktop_pkgs_added:
desktop_pkgs_added[pkg] = set([arch])
else:
desktop_pkgs_added[pkg].add(arch)
n_comps_new = len(desktop_pkgs_added)
if n_comps_new == 0:
@ -177,8 +236,8 @@ for desktop in [ 'gnome-desktop', 'kde-desktop', 'xfce-desktop',
else:
print(" - {} packages not in {} manifest:".format(n_comps_new, desktop))
for pkg in sorted(desktop_pkgs_added):
print(' {}'.format(pkg))
manifest_packages.add(pkg)
arches = desktop_pkgs_added[pkg]
print(' {} (arches: {})'.format(pkg, ', '.join(arches)))
# Update manifest
if (n_manifest_new > 0 or n_comps_new > 0) and args.save: