classes/packaged-staging.bbclass

   1 #
   2 # Populate builds using prebuilt packages where possible to speed up builds
   3 # and allow staging to be reconstructed.
   4 #
   5 # To use it add that line to conf/local.conf:
   6 #
   7 # INHERIT += "packaged-staging"
   8 #
   9
  10
  11 #
  12 # bitbake.conf set PSTAGING_ACTIVE = "0", this class sets to "1" if we're active
  13 #
  14 PSTAGE_PKGVERSION = "${PV}-${PR}"
  15 PSTAGE_PKGARCH    = "${BUILD_SYS}"
  16 PSTAGE_EXTRAPATH  ?= "/${OELAYOUT_ABI}/${DISTRO_PR}/"
  17 PSTAGE_PKGPATH    = "${DISTRO}${PSTAGE_EXTRAPATH}"
  18 PSTAGE_PKGPN      = "${@bb.data.expand('staging-${PN}-${MULTIMACH_ARCH}${TARGET_VENDOR}-${TARGET_OS}', d).replace('_', '-')}"
  19 PSTAGE_PKGNAME    = "${PSTAGE_PKGPN}_${PSTAGE_PKGVERSION}_${PSTAGE_PKGARCH}.ipk"
  20 PSTAGE_PKG        ?= "${DEPLOY_DIR_PSTAGE}/${PSTAGE_PKGPATH}/${PSTAGE_PKGNAME}"
  21
  22 PSTAGE_NATIVEDEPENDS = "\
  23     shasum-native \
  24     stagemanager-native \
  25     "
  26
  27 BB_STAMP_WHITELIST = "${PSTAGE_NATIVEDEPENDS}"
  28
  29 python () {
  30     pstage_allowed = True
  31
  32     # These classes encode staging paths into the binary data so can only be
  33     # reused if the path doesn't change/
  34     if bb.data.inherits_class('native', d) or bb.data.inherits_class('cross', d) or bb.data.inherits_class('sdk', d):
  35         path = bb.data.getVar('PSTAGE_PKGPATH', d, 1)
  36         path = path + bb.data.getVar('TMPDIR', d, 1).replace('/', '-')
  37         bb.data.setVar('PSTAGE_PKGPATH', path, d)
  38
  39     # PSTAGE_NATIVEDEPENDS lists the packages we need before we can use packaged
  40     # staging. There will always be some packages we depend on.
  41     if bb.data.inherits_class('native', d):
  42         pn = bb.data.getVar('PN', d, True)
  43         nativedeps = bb.data.getVar('PSTAGE_NATIVEDEPENDS', d, True).split()
  44         if pn in nativedeps:
  45             pstage_allowed = False
  46
  47     # Images aren't of interest to us
  48     if bb.data.inherits_class('image', d):
  49         pstage_allowed = False
  50
  51     if bb.data.getVar('PSTAGING_DISABLED', d, True) == "1":
  52         pstage_allowed = False
  53
  54     # Add task dependencies if we're active, otherwise mark packaged staging
  55     # as inactive.
  56     if pstage_allowed:
  57         deps = bb.data.getVarFlag('do_setscene', 'depends', d) or ""
  58         deps += " stagemanager-native:do_populate_staging"
  59         bb.data.setVarFlag('do_setscene', 'depends', deps, d)
  60
  61         policy = bb.data.getVar("BB_STAMP_POLICY", d, True)
  62         if policy == "whitelist" or policy == "full":
  63            deps = bb.data.getVarFlag('do_setscene', 'recrdeptask', d) or ""
  64            deps += " do_setscene"
  65            bb.data.setVarFlag('do_setscene', 'recrdeptask', deps, d)
  66
  67         bb.data.setVar("PSTAGING_ACTIVE", "1", d)
  68     else:
  69         bb.data.setVar("PSTAGING_ACTIVE", "0", d)
  70 }
  71
  72 DEPLOY_DIR_PSTAGE   ?= "${DEPLOY_DIR}/pstage"
  73 PSTAGE_MACHCONFIG   = "${DEPLOY_DIR_PSTAGE}/opkg.conf"
  74
  75 PSTAGE_PKGMANAGER = "stage-manager-ipkg"
  76
  77 PSTAGE_BUILD_CMD        = "stage-manager-ipkg-build -o 0 -g 0"
  78 PSTAGE_INSTALL_CMD      = "${PSTAGE_PKGMANAGER} -f ${PSTAGE_MACHCONFIG} -force-depends -o ${TMPDIR} install"
  79 PSTAGE_UPDATE_CMD       = "${PSTAGE_PKGMANAGER} -f ${PSTAGE_MACHCONFIG} -o ${TMPDIR} update"
  80 PSTAGE_REMOVE_CMD       = "${PSTAGE_PKGMANAGER} -f ${PSTAGE_MACHCONFIG} -force-depends -o ${TMPDIR} remove"
  81 PSTAGE_LIST_CMD         = "${PSTAGE_PKGMANAGER} -f ${PSTAGE_MACHCONFIG} -o ${TMPDIR} list_installed"
  82
  83 PSTAGE_TMPDIR_STAGE     = "${WORKDIR}/staging-pkg"
  84
  85 def pstage_manualclean(srcname, destvarname, d):
  86         src = os.path.join(bb.data.getVar('PSTAGE_TMPDIR_STAGE', d, True), srcname)
  87         dest = bb.data.getVar(destvarname, d, True)
  88
  89         for walkroot, dirs, files in os.walk(src):
  90                 for file in files:
  91                         filepath = os.path.join(walkroot, file).replace(src, dest)
  92                         bb.note("rm %s" % filepath)
  93                         os.system("rm %s" % filepath)
  94
  95 def pstage_set_pkgmanager(d):
  96     path = bb.data.getVar("PATH", d, 1)
  97     pkgmanager = bb.which(path, 'opkg-cl')
  98     if pkgmanager == "":
  99         pkgmanager = bb.which(path, 'ipkg-cl')
 100     if pkgmanager != "":
 101         bb.data.setVar("PSTAGE_PKGMANAGER", pkgmanager, d)
 102
 103
 104 def pstage_cleanpackage(pkgname, d):
 105         path = bb.data.getVar("PATH", d, 1)
 106         pstage_set_pkgmanager(d)
 107         list_cmd = bb.data.getVar("PSTAGE_LIST_CMD", d, True)
 108
 109         bb.debug(2, "Checking if staging package installed")
 110         lf = bb.utils.lockfile(bb.data.expand("${SYSROOT_LOCK}", d))
 111         ret = os.system("PATH=\"%s\" %s | grep %s" % (path, list_cmd, pkgname))
 112         if ret == 0:
 113                 bb.debug(1, "Uninstalling package from staging...")
 114                 removecmd = bb.data.getVar("PSTAGE_REMOVE_CMD", d, 1)
 115                 ret = os.system("PATH=\"%s\" %s %s" % (path, removecmd, pkgname))
 116                 if ret != 0:
 117                         bb.note("Failure removing staging package")
 118         else:
 119                 bb.debug(1, "Manually removing any installed files from staging...")
 120                 pstage_manualclean("staging", "STAGING_DIR", d)
 121                 pstage_manualclean("cross", "CROSS_DIR", d)
 122                 pstage_manualclean("deploy", "DEPLOY_DIR", d)
 123
 124         bb.utils.unlockfile(lf)
 125
 126 do_clean_prepend() {
 127         """
 128         Clear the build and temp directories
 129         """
 130
 131         removepkg = bb.data.expand("${PSTAGE_PKGPN}", d)
 132         pstage_cleanpackage(removepkg, d)
 133
 134         stagepkg = bb.data.expand("${PSTAGE_PKG}", d)
 135         bb.note("Removing staging package %s" % base_path_out(stagepkg, d))
 136         os.system('rm -rf ' + stagepkg)
 137 }
 138
 139 staging_helper () {
 140         # Assemble appropriate opkg.conf
 141         conffile=${PSTAGE_MACHCONFIG}
 142         mkdir -p ${DEPLOY_DIR_PSTAGE}/pstaging_lists
 143         if [ ! -e $conffile ]; then
 144                 ipkgarchs="${BUILD_SYS}"
 145                 priority=1
 146                 for arch in $ipkgarchs; do
 147                         echo "arch $arch $priority" >> $conffile
 148                         priority=$(expr $priority + 5)
 149                 done
 150                 echo "dest root /" >> $conffile
 151         fi
 152         if [ ! -e ${TMPDIR}${libdir_native}/opkg/info/ ]; then
 153                 mkdir -p ${TMPDIR}${libdir_native}/opkg/info/
 154         fi
 155         if [ ! -e ${TMPDIR}${libdir_native}/ipkg/ ]; then
 156                 ln -sf opkg/ ${TMPDIR}${libdir_native}/ipkg || true
 157         fi
 158 }
 159
 160 PSTAGE_TASKS_COVERED = "fetch unpack munge patch configure qa_configure rig_locales compile sizecheck install deploy package populate_staging package_write_deb package_write_ipk package_write package_stage qa_staging"
 161
 162 SCENEFUNCS += "packagestage_scenefunc"
 163
 164 python packagestage_scenefunc () {
 165     if bb.data.getVar("PSTAGING_ACTIVE", d, 1) == "0":
 166         return
 167
 168     bb.build.exec_func("staging_helper", d)
 169
 170     removepkg = bb.data.expand("${PSTAGE_PKGPN}", d)
 171
 172     pstage_cleanpackage(removepkg, d)
 173
 174     stagepkg = bb.data.expand("${PSTAGE_PKG}", d)
 175
 176     if os.path.exists(stagepkg):
 177         path = bb.data.getVar("PATH", d, 1)
 178         pstage_set_pkgmanager(d)
 179         file = bb.data.getVar("FILE", d, True)
 180         bb.debug(2, "Packaged staging active for %s\n" % file)
 181
 182         #
 183         # Install the staging package somewhere temporarily so we can extract the stamp files
 184         #
 185         bb.mkdirhier(bb.data.expand("${WORKDIR}/tstage/${libdir_native}/opkg/info/ ", d))
 186         cmd = bb.data.expand("${PSTAGE_PKGMANAGER} -f ${PSTAGE_MACHCONFIG} -force-depends -o ${WORKDIR}/tstage install", d)
 187         ret = os.system("PATH=\"%s\" %s %s" % (path, cmd, stagepkg))
 188         if ret != 0:
 189             bb.fatal("Couldn't install the staging package to a temp directory")
 190
 191         #
 192         # Grab the staging lock now so that we don't have other threads try and
 193         # validate or install based on these stamps being valid.  This is a
 194         # potential issue for certain BB_STAMP_POLICY values and enough
 195         # concurrent threads.
 196         #
 197         lf = bb.utils.lockfile(bb.data.expand("${SYSROOT_LOCK}", d))
 198
 199         #
 200         # Copy the stamp files into the main stamps directoy
 201         #
 202         cmd = bb.data.expand("cp -dpR ${WORKDIR}/tstage/stamps/* ${TMPDIR}/stamps/", d)
 203         ret = os.system(cmd)
 204         if ret != 0:
 205             bb.utils.unlockfile(lf)
 206             bb.fatal("Couldn't copy the staging package stamp files")
 207
 208         #
 209         # Iterate over the stamps seeing if they're valid. If we find any that
 210         # are invalid or the task wasn't in the taskgraph, assume caution and
 211         # do a rebuild.
 212         #
 213         # FIXME - some tasks are safe to ignore in the task graph. e.g. package_write_*
 214         stageok = True
 215         taskscovered = bb.data.getVar("PSTAGE_TASKS_COVERED", d, True).split()
 216         stamp = bb.data.getVar("STAMP", d, True)
 217         for task in taskscovered:
 218             task = 'do_' + task
 219             stampfn = "%s.%s" % (stamp, task)
 220             bb.debug(1, "Checking %s" % (stampfn))
 221             if os.path.exists(stampfn):
 222                 stageok = bb.runqueue.check_stamp_fn(file, task, d)
 223                 bb.debug(1, "Result %s" % (stageok))
 224                 if not stageok:
 225                     bb.utils.unlockfile(lf)
 226                     break
 227
 228         # Remove the stamps and files we added above
 229         # FIXME - we should really only remove the stamps we added
 230         os.system('rm -f ' + stamp + '.*')
 231         os.system(bb.data.expand("rm -rf ${WORKDIR}/tstage", d))
 232
 233         if stageok:
 234             bb.note("Staging package found, using it for %s." % file)
 235             installcmd = bb.data.getVar("PSTAGE_INSTALL_CMD", d, 1)
 236             ret = os.system("PATH=\"%s\" %s %s" % (path, installcmd, stagepkg))
 237             bb.utils.unlockfile(lf)
 238             if ret != 0:
 239                 bb.note("Failure installing prestage package")
 240
 241             bb.build.make_stamp("do_stage_package_populated", d)
 242         else:
 243             bb.note("Staging package found but invalid for %s" % file)
 244
 245 }
 246 packagestage_scenefunc[cleandirs] = "${PSTAGE_TMPDIR_STAGE}"
 247 packagestage_scenefunc[dirs] = "${STAGING_DIR}"
 248
 249 addhandler packagedstage_stampfixing_eventhandler
 250 python packagedstage_stampfixing_eventhandler() {
 251     if bb.event.getName(e) == "StampUpdate":
 252         taskscovered = bb.data.getVar("PSTAGE_TASKS_COVERED", e.data, 1).split()
 253         for (fn, task) in e.targets:
 254             # strip off 'do_'
 255             task = task[3:]
 256             if task in taskscovered:
 257                 stamp = "%s.do_stage_package_populated" % e.stampPrefix[fn]
 258                 if os.path.exists(stamp):
 259                     # We're targetting a task which was skipped with packaged staging
 260                     # so we need to remove the autogenerated stamps.
 261                     for task in taskscovered:
 262                         dir = "%s.do_%s" % (e.stampPrefix[fn], task)
 263                         os.system('rm -f ' + dir)
 264                     os.system('rm -f ' + stamp)
 265
 266     return NotHandled
 267 }
 268
 269 populate_staging_preamble () {
 270         if [ "$PSTAGING_ACTIVE" = "1" ]; then
 271                 stage-manager -p ${STAGING_DIR} -c ${DEPLOY_DIR_PSTAGE}/stamp-cache-staging -u || true
 272                 stage-manager -p ${CROSS_DIR} -c ${DEPLOY_DIR_PSTAGE}/stamp-cache-cross -u || true
 273         fi
 274 }
 275
 276 populate_staging_postamble () {
 277         if [ "$PSTAGING_ACTIVE" = "1" ]; then
 278                 # list the packages currently installed in staging
 279                 # ${PSTAGE_LIST_CMD} | awk '{print $1}' > ${DEPLOY_DIR_PSTAGE}/installed-list
 280
 281                 # exitcode == 5 is ok, it means the files change
 282                 set +e
 283                 stage-manager -p ${STAGING_DIR} -c ${DEPLOY_DIR_PSTAGE}/stamp-cache-staging -u -d ${PSTAGE_TMPDIR_STAGE}/staging
 284                 exitcode=$?
 285                 if [ "$exitcode" != "5" -a "$exitcode" != "0" ]; then
 286                         exit $exitcode
 287                 fi
 288                 stage-manager -p ${CROSS_DIR} -c ${DEPLOY_DIR_PSTAGE}/stamp-cache-cross -u -d ${PSTAGE_TMPDIR_STAGE}/cross/${BASE_PACKAGE_ARCH}
 289                 if [ "$exitcode" != "5" -a "$exitcode" != "0" ]; then
 290                         exit $exitcode
 291                 fi
 292                 set -e
 293         fi
 294 }
 295
 296 packagedstaging_fastpath () {
 297         if [ "$PSTAGING_ACTIVE" = "1" ]; then
 298                 mkdir -p ${PSTAGE_TMPDIR_STAGE}/staging/
 299                 mkdir -p ${PSTAGE_TMPDIR_STAGE}/cross/
 300                 cp -fpPR ${SYSROOT_DESTDIR}${STAGING_DIR}/* ${PSTAGE_TMPDIR_STAGE}/staging/ || /bin/true
 301                 cp -fpPR ${SYSROOT_DESTDIR}${CROSS_DIR}/* ${PSTAGE_TMPDIR_STAGE}/cross/${BASE_PACKAGE_ARCH}/ || /bin/true
 302         fi
 303 }
 304
 305 do_populate_staging[dirs] =+ "${DEPLOY_DIR_PSTAGE}"
 306 python populate_staging_prehook() {
 307     bb.build.exec_func("populate_staging_preamble", d)
 308 }
 309
 310 python populate_staging_posthook() {
 311     bb.build.exec_func("populate_staging_postamble", d)
 312 }
 313
 314
 315 staging_packager () {
 316
 317         mkdir -p ${PSTAGE_TMPDIR_STAGE}/CONTROL
 318         mkdir -p ${DEPLOY_DIR_PSTAGE}/${PSTAGE_PKGPATH}
 319
 320         echo "Package: ${PSTAGE_PKGPN}"         >  ${PSTAGE_TMPDIR_STAGE}/CONTROL/control
 321         echo "Version: ${PSTAGE_PKGVERSION}"    >> ${PSTAGE_TMPDIR_STAGE}/CONTROL/control
 322         echo "Description: ${DESCRIPTION}"      >> ${PSTAGE_TMPDIR_STAGE}/CONTROL/control
 323         echo "Section: ${SECTION}"              >> ${PSTAGE_TMPDIR_STAGE}/CONTROL/control
 324         echo "Priority: Optional"               >> ${PSTAGE_TMPDIR_STAGE}/CONTROL/control
 325         echo "Maintainer: ${MAINTAINER}"        >> ${PSTAGE_TMPDIR_STAGE}/CONTROL/control
 326         echo "Architecture: ${PSTAGE_PKGARCH}"  >> ${PSTAGE_TMPDIR_STAGE}/CONTROL/control
 327
 328         # Protect against empty SRC_URI
 329         if [ "${SRC_URI}" != "" ] ; then
 330                 echo "Source: ${SRC_URI}"               >> ${PSTAGE_TMPDIR_STAGE}/CONTROL/control
 331         else
 332                 echo "Source: OpenEmbedded"               >> ${PSTAGE_TMPDIR_STAGE}/CONTROL/control
 333         fi
 334
 335         ${PSTAGE_BUILD_CMD} ${PSTAGE_TMPDIR_STAGE} ${DEPLOY_DIR_PSTAGE}/${PSTAGE_PKGPATH}
 336 }
 337
 338 staging_package_installer () {
 339         #${PSTAGE_INSTALL_CMD} ${PSTAGE_PKG}
 340
 341         STATUSFILE=${TMPDIR}${libdir_native}/opkg/status
 342         echo "Package: ${PSTAGE_PKGPN}"        >> $STATUSFILE
 343         echo "Version: ${PSTAGE_PKGVERSION}"   >> $STATUSFILE
 344         echo "Status: install user installed"  >> $STATUSFILE
 345         echo "Architecture: ${PSTAGE_PKGARCH}" >> $STATUSFILE
 346         echo "" >> $STATUSFILE
 347
 348         CTRLFILE=${TMPDIR}${libdir_native}/opkg/info/${PSTAGE_PKGPN}.control
 349         echo "Package: ${PSTAGE_PKGPN}"        > $CTRLFILE
 350         echo "Version: ${PSTAGE_PKGVERSION}"   >> $CTRLFILE
 351         echo "Architecture: ${PSTAGE_PKGARCH}" >> $CTRLFILE
 352
 353         cd ${PSTAGE_TMPDIR_STAGE}
 354         find -type f | grep -v ./CONTROL | sed -e 's/^\.//' > ${TMPDIR}${libdir_native}/opkg/info/${PSTAGE_PKGPN}.list
 355 }
 356
 357 python do_package_stage () {
 358     if bb.data.getVar("PSTAGING_ACTIVE", d, 1) != "1":
 359         return
 360
 361     #
 362     # Handle deploy/ packages
 363     #
 364     bb.build.exec_func("read_subpackage_metadata", d)
 365     stagepath = bb.data.getVar("PSTAGE_TMPDIR_STAGE", d, 1)
 366     tmpdir = bb.data.getVar("TMPDIR", d, True)
 367     packages = (bb.data.getVar('PACKAGES', d, 1) or "").split()
 368     if len(packages) > 0:
 369         if bb.data.inherits_class('package_ipk', d):
 370             ipkpath = bb.data.getVar('DEPLOY_DIR_IPK', d, True).replace(tmpdir, stagepath)
 371         if bb.data.inherits_class('package_deb', d):
 372             debpath = bb.data.getVar('DEPLOY_DIR_DEB', d, True).replace(tmpdir, stagepath)
 373         if bb.data.inherits_class('package_rpm', d):
 374             rpmpath = bb.data.getVar('DEPLOY_DIR_RPM', d, True).replace(tmpdir, stagepath)
 375
 376         for pkg in packages:
 377             pkgname = bb.data.getVar('PKG_%s' % pkg, d, 1)
 378             if not pkgname:
 379                 pkgname = pkg
 380             arch = bb.data.getVar('PACKAGE_ARCH_%s' % pkg, d, 1)
 381             if not arch:
 382                 arch = bb.data.getVar('PACKAGE_ARCH', d, 1)
 383             pr = bb.data.getVar('PR_%s' % pkg, d, 1)
 384             if not pr:
 385                 pr = bb.data.getVar('PR', d, 1)
 386             if not packaged(pkg, d):
 387                 continue
 388             if bb.data.inherits_class('package_ipk', d):
 389                 srcname = bb.data.expand(pkgname + "_${PV}-" + pr + "${DISTRO_PR}" + "_" + arch + ".ipk", d)
 390                 srcfile = bb.data.expand("${DEPLOY_DIR_IPK}/" + arch + "/" + srcname, d)
 391                 if os.path.exists(srcfile):
 392                     destpath = ipkpath + "/" + arch + "/"
 393                     bb.mkdirhier(destpath)
 394                     print destpath
 395                     bb.copyfile(srcfile, destpath + srcname)
 396
 397             if bb.data.inherits_class('package_deb', d):
 398                 if arch == 'all':
 399                     srcname = bb.data.expand(pkgname + "_${PV}-" + pr + "${DISTRO_PR}" + "_all.deb", d)
 400                 else:
 401                     srcname = bb.data.expand(pkgname + "_${PV}-" + pr + "${DISTRO_PR}" + "_${DPKG_ARCH}.deb", d)
 402                 srcfile = bb.data.expand("${DEPLOY_DIR_DEB}/" + arch + "/" + srcname, d)
 403                 if os.path.exists(srcfile):
 404                     destpath = debpath + "/" + arch + "/"
 405                     bb.mkdirhier(destpath)
 406                     bb.copyfile(srcfile, destpath + srcname)
 407
 408             if bb.data.inherits_class('package_rpm', d):
 409                 version = bb.data.getVar('PV', d, 1)
 410                 version = version.replace('-', '+')
 411                 bb.data.setVar('RPMPV', version, d)
 412                 srcname = bb.data.expand(pkgname + "-${RPMPV}-" + pr + "${DISTRO_PR}" + ".${TARGET_ARCH}.rpm", d)
 413                 srcfile = bb.data.expand("${DEPLOY_DIR_RPM}/" + arch + "/" + srcname, d)
 414                 if os.path.exists(srcfile):
 415                     destpath = rpmpath + "/" + arch + "/"
 416                     bb.mkdirhier(destpath)
 417                     bb.copyfile(srcfile, destpath + srcname)
 418
 419
 420     #
 421     # Handle stamps/ files
 422     #
 423     stampfn = bb.data.getVar("STAMP", d, True)
 424     destdir = os.path.dirname(stampfn.replace(tmpdir, stagepath))
 425     bb.mkdirhier(destdir)
 426     # We need to include the package_stage stamp in the staging package so create one
 427     bb.build.make_stamp("do_package_stage", d)
 428     os.system("cp -dpR %s.do_* %s/" % (stampfn, destdir))
 429
 430     pstage_set_pkgmanager(d)
 431     bb.build.exec_func("staging_helper", d)
 432     bb.build.exec_func("staging_packager", d)
 433     lf = bb.utils.lockfile(bb.data.expand("${SYSROOT_LOCK}", d))
 434     bb.build.exec_func("staging_package_installer", d)
 435     bb.utils.unlockfile(lf)
 436 }
 437
 438 #
 439 # Note an assumption here is that do_deploy runs before do_package_write/do_populate_staging
 440 #
 441 addtask package_stage after do_package_write do_populate_staging before do_build
 442
 443 do_package_stage_all () {
 444         :
 445 }
 446 do_package_stage_all[recrdeptask] = "do_package_stage"
 447 addtask package_stage_all after do_package_stage before do_build