AnacondaGenHdList

When installing RPMs, Anaconda needs information about package dependencies to ensure that RPMS get installed in the proper order.

This information is contained in the 'hdlist' and 'hdlist2' files within the base/ directory of the install tree (i.e. Fedora/base or RedHat/base). These files are in a packed, binary format, and generating or re-generating them is surprisingly annoying and very poorly documented. Hopefully, this page will help explain things a little bit.

Each RPM file contains a header section that outlines what software or facility the package provides, what software or facilities it requires, and any software or facility conflicts or incompatibilities. In order for Anaconda to install a series of packages smoothly, all the RPMS in the install tree need to be examined for this header information, and the information must be collected and collated. This is normally a two stage process, requiring the use of two tools that are part of the anaconda-runtime RPM - 'pkgorder', which extracts RPM header information and saves it, and 'genhdlist', which gathers it and packs it into the hdlist and hdlist2 files used during installation. Neither of these tools comes with a manual or usage instructions, which makes them difficult to use!

If you want to add or update RPMS in your install tree (to incorporate new or different packages into your base install), you need to regenerate the hdlist files. Below is a Python script that attempts as much as possible to update this process and make it a little less painful.

#!/usr/bin/python
#
#
#
#
#


import os
import sys
import getopt
import commands
import rpm
import string
import shutil

def usage():
print "Usage: %s [-h]  | [[-n]  [-v]  [-d dir]  [-i dir]  [-u dir]  [-b branch]  [-c]  [-H]  [-G]  " % sys.argv[0] 
print "  -h|--help                print this message"
print "  -n|--dryrun              dry run mode (show a trace of what would be done)"
print "  -v|--verbose             print more information when running"
print "  -d dir|--basedir=dir     base dir (current dir by default)"
print "  -i dir|--installdir=dir  installation dir (updated_release/RedHat/RPMS/ by default)"
print "  -u dir|--updatesdir=dir  updates dir (updates/ by default)"
print "  -b branch|--branch=vers  RedHat version (i.e. Fedora, RedHat)"
print "  -c|--checkonly           check for dependencies/conflicts in the installation dir"
print "  -G|--genhdlist              regenerate hdlists"
print "  -H|--nohdlist            don't regenerate hdlists"


def formatRequire (name, version, flags):
string = name

if flags:
if flags & (rpm.RPMSENSE_LESS | rpm.RPMSENSE_GREATER | rpm.RPMSENSE_EQUAL):
string = string + " "
if flags & rpm.RPMSENSE_LESS:
string = string + "<"
if flags & rpm.RPMSENSE_GREATER:
string = string + ">"
if flags & rpm.RPMSENSE_EQUAL:
string = string + "="
string = string + " %s" % version
return string


def getRpmsHeaders(rpms):
headers = {}
bad_rpms = [] 
ts = rpm.TransactionSet("/", (rpm._RPMVSF_NOSIGNATURES))
ts.closeDB()
for rpm_file in rpms:
fdno = os.open(rpm_file, os.O_RDONLY)
header = ts.hdrFromFdno(fdno)
os.close(fdno)

if string.find(rpm_file, "debuginfo") >= 0:
bad_rpms.append(rpm_file)
if verbose_flag:
print " skip %s which is a debug package" % os.path.basename(rpm_file)

elif header[rpm.RPMTAG_SOURCEPACKAGE] :
bad_rpms.append(rpm_file)
if verbose_flag:
print " skip %s which is a source package" % os.path.basename(rpm_file)
else:
key = (header[rpm.RPMTAG_NAME] , header[rpm.RPMTAG_ARCH] )
if headers.has_key(key):
cmp = rpm.versionCompare(headers[key] [0] , header)
if cmp == 1:
bad_rpms.append(rpm_file)
if verbose_flag:
print " skip %s which is older than %s" % (os.path.basename(rpm_file), os.path.basename(headers[key] [1] ))
elif cmp == 0:
if verbose_flag:
print "skip %s which is the same as %s" % (os.path.basename(headers[key] [1] ), os.path.basename(rpm_file))
elif cmp == -1:
bad_rpms.append(headers[key] [1] )
if verbose_flag:
print " use %s instead of %s" % (os.path.basename(rpm_file), os.path.basename(headers[key] [1] ))
headers[key]  = (header, rpm_file)
else:
print " rpm.versionCompare() returned error code %d while comparing %s and %s" % (cmp, os.path.basename(headers[key] [1] ), os.path.basename(rpm_file))
sys.exit(cmp)
else:
headers[key]  = (header, rpm_file)

return (headers, bad_rpms)


#	None
def skipRpms(rpms_list, skip_list):
for item in skip_list:
del rpms_list[rpms_list.index(item)] 


def checkDepsAndConflicts(headers):
ts = rpm.TransactionSet("/", ~(rpm._RPMVSF_NOSIGNATURES))
ts.closeDB()
for key in headers.keys():
ts.addInstall(headers[key] [0] , key, 'i')
return ts.check()


def main():
try:
opts, args = getopt.getopt(sys.argv[1:] , 'hnvd:i:u:b:cHG', \
["help", "dryrun", "verbose", "basedir=", "installdir=", \
]"updatesdir=", "branch=", "checkonly", "nohdlist", "genhdlist" )
except getopt.GetoptError:
usage()
sys.exit(1)

if args != [] :
usage()
sys.exit(1)

global verbose_flag
verbose_flag = 0
dryrun_flag = 0
base_dir = os.getcwd()
updated_release_dir = "Fedora/RPMS"
updates_dir = "updates"
branch = "Fedora"
checkonly_flag = 0
nohdlist_flag = 0
genhdlist_flag = 0

for o, a in opts:
if o in ("-h", "--help"):
usage()
sys.exit()
if o in ("-n", "--dryrun"):
dryrun_flag = 1
if o in ("-v", "--verbose"):
verbose_flag = 1
if o in ("-d", "--basedir"):
base_dir = a
if o in ("-i", "--installdir"):
updated_release_dir = a
if o in ("-u", "--updatesdir"):
updates_dir = a
if o in ("-b", "--branch"):
branch = a
if o in ("-c", "--checkonly"):
checkonly_flag = 1
if o in ("-H", "--nohdlist"):
nohdlist_flag = 1
if o in ("-G", "--genhdlist"):
genhdlist_flag = 1

updated_release_dir = os.path.join(base_dir, updated_release_dir)
updates_dir = os.path.join(base_dir, updates_dir)
old_rpms_dir = os.path.join(base_dir, "old_rpms")
regenerate_hdlist = 0
if dryrun_flag:
verbose_flag = 1

if not os.path.isdir(updated_release_dir):
print "can't access %s, nothing to do" % updated_release_dir
sys.exit(2)

if verbose_flag:
print "processing rpms under %s ..." % updated_release_dir

exit_status, output = commands.getstatusoutput("find %s -name '*\.rpm' | grep -v '.src.rpm'" % updated_release_dir)
if exit_status != 0:
sys.exit(3)
updated_release_rpms = output.split()

updated_release_headers, bad_rpms = getRpmsHeaders(updated_release_rpms)
if bad_rpms != [] :
print " ERROR: the following rpms are either source rpms or older rpms. Please remove them:"
for rpm_file in bad_rpms:
print " %s" % rpm_file
sys.exit(4)

errors = checkDepsAndConflicts(updated_release_headers)
if errors:
print "ERROR: the installation tree already contains conflicts and/or dependencies problems"
for ((name, version, release), (reqname, reqversion), \
flags, suggest, sense) in errors:
if sense==rpm.RPMDEP_SENSE_REQUIRES:
print " depcheck: package %s needs %s" % ( name, formatRequire(reqname, reqversion, flags))
elif sense==rpm.RPMDEP_SENSE_CONFLICTS:
print " depcheck: package %s conflicts with %s" % (name, reqname)
sys.exit(4)

if verbose_flag:
print "done"

if checkonly_flag:
if errors:
print "dependencies/conflicts problems exist in %s" % updated_release_dir
else:
print "no dependencies/conflicts problems in %s" % updated_release_dir
sys.exit()


if not os.path.isdir(updates_dir):
print "can't access %s, nothing to do" % updates_dir
sys.exit(2)

if not os.path.isdir(old_rpms_dir):
os.mkdir(old_rpms_dir)

if verbose_flag:
print "processing rpms under %s ..." % updates_dir

exit_status, output = commands.getstatusoutput("find %s -follow -name '*\.rpm' | grep -v '.src.rpm'" % updates_dir)
if exit_status != 0:
sys.exit(3)
updates_rpms = output.split()

updates_headers, bad_rpms = getRpmsHeaders(updates_rpms)

skipRpms(updates_rpms, bad_rpms)
if bad_rpms != [] :
print " WARNING: the following rpms are either source rpms or older rpms. I will ignore them:"
for rpm_file in bad_rpms:
print " %s" % rpm_file

if verbose_flag:
print "done"

old_rpms_headers = {}
for key in updates_headers.keys():
if updated_release_headers.has_key(key):
cmp = rpm.versionCompare(updated_release_headers[key] [0] , updates_headers[key] [0] )
if cmp == 1:
if verbose_flag:
print "skip %s which is older than %s" % (os.path.basename(updates_headers[key] [1] ), os.path.basename(updated_release_headers[key] [1] ))
elif cmp == 0:
if verbose_flag:
print "skip %s which is the same as %s" % (os.path.basename(updates_headers[key] [1] ), os.path.basename(updated_release_headers[key] [1] ))
elif cmp == -1:
if verbose_flag:
print "exchange %s with %s" % (os.path.basename(updated_release_headers[key] [1] ), os.path.basename(updates_headers[key] [1] ))
old_rpms_headers[key]  = updated_release_headers[key] 
updated_release_headers[key]  = updates_headers[key] 
else:
print " rpm.versionCompare() returned error code %d while comparing %s and %s" % (cmp, os.path.basename(updated_release_headers[key] [1] ), os.path.basename(updates_headers[key] [1] ))
sys.exit(cmp)
else:
updated_release_headers[key]  = updates_headers[key] 
if verbose_flag:
print "add %s" % os.path.basename(updates_headers[key] [1] )

while 1:
deps_and_conflicts = 0

errors = checkDepsAndConflicts(updated_release_headers)
if errors:
if verbose_flag:
print "merging will result in the following conflicts and/or dependencies problems:"
name_list = [] 
for ((name, version, release), (reqname, reqversion), \
flags, suggest, sense) in errors:
if sense==rpm.RPMDEP_SENSE_REQUIRES:
if verbose_flag:
print " depcheck: package %s needs %s" % ( name, formatRequire(reqname, reqversion, flags))
deps_and_conflicts = 1
if not name in name_list:
name_list.append(name)
elif sense==rpm.RPMDEP_SENSE_CONFLICTS:
if verbose_flag:
print " depcheck: package %s conflicts with %s" % (name, reqname)
deps_and_conflicts = 1
if not name in name_list:
name_list.append(name)

if verbose_flag:
print "backtracking:"
for (name, arch) in updated_release_headers.keys():
if name in name_list :
if os.path.dirname(updated_release_headers[(name, arch)] [1] ) == updated_release_dir:
print "CRITICAL ERROR: attempt to discard %s" % updated_release_headers[(name, arch)] [1] 
sys.exit(4)
if verbose_flag:
print " discard %s" % updated_release_headers[(name, arch)] [1] 
del updated_release_headers[(name, arch)] 

for (name, arch) in old_rpms_headers.keys():
if name in name_list :
if verbose_flag:
print " reuse %s" % old_rpms_headers[(name, arch)] [1] 
updated_release_headers[(name, arch)]  = old_rpms_headers[(name, arch)] 
del old_rpms_headers[(name, arch)] 

if deps_and_conflicts == 0:
break

for key in updated_release_headers.keys():
if os.path.dirname(updated_release_headers[key] [1] ) == updated_release_dir:
continue
regenerate_hdlist = 1
if not dryrun_flag:
shutil.copy2(updated_release_headers[key] [1] , \
os.path.join(updated_release_dir, os.path.basename(updated_release_headers[key] [1] )))
else:
print "should",
if key in old_rpms_headers.keys():
print "update %s with %s" % (os.path.basename(old_rpms_headers[key] [1] ), \
os.path.basename(updated_release_headers[key] [1] ))
if not dryrun_flag:
os.rename(old_rpms_headers[key] [1] , \
os.path.join(old_rpms_dir, os.path.basename(old_rpms_headers[key] [1] )))
else:
print "add %s" % os.path.basename(updated_release_headers[key] [1] )

if genhdlist_flag or ((not nohdlist_flag) and regenerate_hdlist):
if not dryrun_flag:
os.system("PYTHONPATH=/usr/lib/anaconda /usr/lib/anaconda-runtime/genhdlist --withnumbers --productpath %s %s/../.." % (branch, updated_release_dir) )
os.system("PYTHONPATH=/usr/lib/anaconda /usr/lib/anaconda-runtime/pkgorder %s/../.. i386 %s > pkgorder" % (updated_release_dir, branch) )
os.system("PYTHONPATH=/usr/lib/anaconda /usr/lib/anaconda-runtime/genhdlist --withnumbers --productpath %s --fileorder pkgorder %s/../.." % (branch, updated_release_dir) )
else:
if verbose_flag:
print "should",
if verbose_flag:
print "regenerate hdlist"


if __name__ == "__main__":
main()

The author of this page makes no assertions about the fitness or maintainability of this code, nor does he claim any right of ownership or authorship.

In my site's configuration, I use the above script to merge RPMs into my install trees and maintain them. My directory structure for my Fedora Core 4 install tree looks like this:

FC4/
Fedora/
RPMS/
base/
updates/

Which means that when adding an RPM, I will save it in the 'updates' directory, and (assuming I am in the directory above 'FC4'), run 'updatehdlist' like this:

updatehdlist -v -G -d FC4 -u updates -i Fedora/RPMS -b Fedora

This invocation, if everything proceeds smoothly, will update the hdlist and hdlist2 files contained in the 'Fedora/base' directory, and allow me to install my added RPMS the same way I install any other RPM from the '%packages' section of my Kickstart script.

Much credit and many thanks are due to the original author of the Python script shown above. It saved me more time and hassle than you may ever know.

---