2 # Populate builds using prebuilt packages where possible to speed up builds
3 # and allow staging to be reconstructed.
5 # To use it add that line to conf/local.conf:
7 # INHERIT += "packaged-staging"
9 # To use the prebuilt pstage packages, save them away..
10 # $ rsync --delete -a tmp/pstage/ /there/oe/build/pstage_mirror
11 # .. and use it as mirror in your conf/local.conf:
12 # PSTAGE_MIRROR = "file:///there/oe/build/pstage_mirror"
16 # bitbake.conf set PSTAGING_ACTIVE = "0", this class sets to "1" if we're active
18 PSTAGE_PKGVERSION = "${PV}-${PR}"
19 PSTAGE_PKGARCH = "${PACKAGE_ARCH}-${HOST_OS}"
20 PSTAGE_EXTRAPATH ?= "/${OELAYOUT_ABI}/${DISTRO_PR}/"
21 PSTAGE_PKGPATH = "${DISTRO}${PSTAGE_EXTRAPATH}"
22 PSTAGE_PKGPN = "${@bb.data.expand('staging-${PN}-${MULTIMACH_ARCH}${TARGET_VENDOR}-${TARGET_OS}', d).replace('_', '-')}"
23 PSTAGE_PKGNAME = "${PSTAGE_PKGPN}_${PSTAGE_PKGVERSION}_${PSTAGE_PKGARCH}.ipk"
24 PSTAGE_PKG = "${PSTAGE_DIR}/${PSTAGE_PKGPATH}/${PSTAGE_PKGNAME}"
25 PSTAGE_WORKDIR = "${TMPDIR}/pstage"
26 PSTAGE_SCAN_CMD ?= "find ${PSTAGE_TMPDIR_STAGE} \( -name "*.la" -o -name "*-config" \) -type f"
28 PSTAGE_NATIVEDEPENDS = "\
32 BB_STAMP_WHITELIST = "${PSTAGE_NATIVEDEPENDS}"
34 python __anonymous() {
36 pstage_host_mangle = False
37 pstage_is_cross = False
39 # PSTAGE_NATIVEDEPENDS lists the packages we need before we can use
40 # packaged staging. There will always be some packages we depend on.
41 if bb.data.inherits_class('native', d):
42 pn = bb.data.getVar('PN', d, True)
43 nativedeps = bb.data.getVar('PSTAGE_NATIVEDEPENDS', d, True).split()
45 pstage_allowed = False
47 # Images aren't of interest to us
48 if bb.data.inherits_class('image', d):
49 pstage_allowed = False
51 # We need OVERRIDES to be evaluated and applied.
52 localdata = d.createCopy()
53 bb.data.update_data(localdata)
54 if localdata.getVar('PSTAGING_DISABLED', True) == "1":
55 pstage_allowed = False
57 # libtool-cross does not inherit cross.
58 if localdata.getVar('PN', True).startswith("libtool-cross"):
59 pstage_host_mangle = True
60 pstage_is_cross = True
62 # We need PSTAGE_PKGARCH to contain information about the target.
63 if bb.data.inherits_class('cross', d):
64 pstage_host_mangle = True
66 if pstage_host_mangle:
67 bb.data.setVar('PSTAGE_PKGARCH', "${HOST_SYS}-${PACKAGE_ARCH}-${TARGET_OS}", d)
69 # These classes encode staging paths data files so we must mangle them
71 if bb.data.inherits_class('native', d) or bb.data.inherits_class('nativesdk', d) or bb.data.inherits_class('cross', d) or bb.data.inherits_class('crosssdk', d) or bb.data.inherits_class('sdk', d):
72 pstage_is_cross = True
75 scan_cmd = "grep -Irl ${TMPDIR} ${PSTAGE_TMPDIR_STAGE}"
76 bb.data.setVar('PSTAGE_SCAN_CMD', scan_cmd, d)
78 # Add task dependencies if we're active, otherwise mark packaged staging
81 policy = bb.data.getVar("BB_STAMP_POLICY", d, True)
82 if policy == "whitelist" or policy == "full":
83 deps = bb.data.getVarFlag('do_setscene', 'recrdeptask', d) or ""
84 deps += " do_setscene"
85 bb.data.setVarFlag('do_setscene', 'recrdeptask', deps, d)
87 bb.data.setVar("PSTAGING_ACTIVE", "1", d)
89 bb.data.setVar("PSTAGING_ACTIVE", "0", d)
92 PSTAGE_MACHCONFIG = "${PSTAGE_WORKDIR}/opkg.conf"
94 PSTAGE_PKGMANAGER = "stage-manager-opkg"
96 PSTAGE_BUILD_CMD = "stage-manager-opkg-build -o 0 -g 0"
97 PSTAGE_INSTALL_CMD = "${PSTAGE_PKGMANAGER} -f ${PSTAGE_MACHCONFIG} -force-depends -o ${TMPDIR} install"
98 PSTAGE_UPDATE_CMD = "${PSTAGE_PKGMANAGER} -f ${PSTAGE_MACHCONFIG} -o ${TMPDIR} update"
99 PSTAGE_REMOVE_CMD = "${PSTAGE_PKGMANAGER} -f ${PSTAGE_MACHCONFIG} -force-depends -o ${TMPDIR} remove"
100 PSTAGE_LIST_CMD = "${PSTAGE_PKGMANAGER} -f ${PSTAGE_MACHCONFIG} -o ${TMPDIR} list_installed"
102 PSTAGE_TMPDIR_STAGE = "${WORKDIR}/staging-pkg"
104 def pstage_manualclean(srcname, destvarname, d):
105 src = os.path.join(bb.data.getVar('PSTAGE_TMPDIR_STAGE', d, True), srcname)
106 dest = bb.data.getVar(destvarname, d, True)
108 for walkroot, dirs, files in os.walk(src):
109 bb.debug(2, "rm %s" % walkroot)
111 # Avoid breaking the held lock
112 if (file == "staging.lock"):
114 filepath = os.path.join(walkroot, file).replace(src, dest)
115 oe.path.remove(filepath)
117 def pstage_set_pkgmanager(d):
118 path = bb.data.getVar("PATH", d, 1)
119 pkgmanager = bb.which(path, 'opkg-cl')
121 pkgmanager = bb.which(path, 'ipkg-cl')
123 bb.data.setVar("PSTAGE_PKGMANAGER", pkgmanager, d)
126 def pstage_cleanpackage(pkgname, d):
127 pstage_set_pkgmanager(d)
128 list_cmd = bb.data.getVar("PSTAGE_LIST_CMD", d, True)
130 bb.debug(2, "Checking if staging package installed")
131 lf = bb.utils.lockfile(bb.data.expand("${SYSROOT_LOCK}", d))
133 oe_run(d, "%s | grep %s" % (list_cmd, pkgname))
135 bb.debug(1, "Manually removing any installed files from staging...")
136 pstage_manualclean("sysroots", "STAGING_DIR", d)
137 pstage_manualclean("deploy", "DEPLOY_DIR", d)
139 bb.debug(1, "Uninstalling package from staging...")
140 removecmd = bb.data.getVar("PSTAGE_REMOVE_CMD", d, 1)
142 oe_run(d, "%s %s" % (removecmd, pkgname))
144 bb.note("Failure removing staging package")
146 bb.utils.unlockfile(lf)
150 Clear the build and temp directories
153 removepkg = bb.data.expand("${PSTAGE_PKGPN}", d)
154 pstage_cleanpackage(removepkg, d)
156 stagepkg = bb.data.expand("${PSTAGE_PKG}", d)
157 if os.path.exists(stagepkg):
158 bb.note("Removing staging package %s" % base_path_out(stagepkg, d))
159 oe.path.remove(stagepkg)
160 oe.path.remove(stagepkg + ".md5")
164 # Assemble appropriate opkg.conf
165 conffile=${PSTAGE_MACHCONFIG}
166 mkdir -p ${PSTAGE_WORKDIR}/pstaging_lists
167 arch="${PSTAGE_PKGARCH}"
168 if [ ! -e $conffile ]; then
169 echo "arch $arch 1" > $conffile
170 echo "dest root /" >> $conffile
171 elif [ `grep -c " $arch " $conffile` -eq 0 ]; then
172 priority=$(expr `grep -cE "^arch" $conffile` + 1)
173 sed -i -e "/dest/iarch $arch $priority" $conffile
175 if [ ! -e ${TMPDIR}${libdir_native}/opkg/info/ ]; then
176 mkdir -p ${TMPDIR}${libdir_native}/opkg/info/
178 if [ ! -e ${TMPDIR}${libdir_native}/ipkg/ ]; then
179 ln -sf opkg/ ${TMPDIR}${libdir_native}/ipkg || true
183 def staging_fetch(stagepkg, d):
186 # only try and fetch if the user has configured a mirror
187 if bb.data.getVar('PSTAGE_MIRROR', d) != "":
188 # Copy the data object and override DL_DIR and SRC_URI
190 dldir = bb.data.expand("${PSTAGE_DIR}/${PSTAGE_PKGPATH}", pd)
191 mirror = bb.data.expand("${PSTAGE_MIRROR}/${PSTAGE_PKGPATH}/", pd)
192 bn = os.path.basename(stagepkg)
194 bb.data.setVar('DL_DIR', dldir, pd)
195 bb.data.setVar('SRC_URI', srcuri, pd)
197 # Try a fetch from the pstage mirror, if it fails just return and
198 # we will build the package
199 bb.debug(1, "Attempting to fetch staging package %s" % (bn))
201 bb.fetch.init([srcuri], pd)
202 bb.fetch.go(pd, [srcuri])
203 except Exception, ex:
204 bb.debug(1, "Failed to fetch staging package %s: %s" % (bn, ex))
206 bb.debug(1, "Fetched staging package %s" % bn)
208 PSTAGE_TASKS_COVERED = "fetch unpack munge patch configure qa_configure rig_locales compile sizecheck install deploy package populate_sysroot package_write_deb package_write_ipk package_write package_stage qa_staging"
210 SCENEFUNCS += "packagestage_scenefunc"
212 python packagestage_scenefunc () {
214 if bb.data.getVar("PSTAGING_ACTIVE", d, 1) == "0":
217 bb.build.exec_func("staging_helper", d)
219 removepkg = bb.data.expand("${PSTAGE_PKGPN}", d)
221 pstage_cleanpackage(removepkg, d)
223 stagepkg = bb.data.expand("${PSTAGE_PKG}", d)
224 if not os.path.exists(stagepkg):
225 staging_fetch(stagepkg, d)
227 if os.path.exists(stagepkg):
228 pstage_set_pkgmanager(d)
229 file = bb.data.getVar("FILE", d, True)
230 bb.debug(2, "Packaged staging active for %s\n" % file)
233 # Install the staging package somewhere temporarily so we can extract the stamp files
235 bb.mkdirhier(bb.data.expand("${WORKDIR}/tstage/${libdir_native}/opkg/info/", d))
236 cmd = bb.data.expand("${PSTAGE_PKGMANAGER} -f ${PSTAGE_MACHCONFIG} -force-depends -o ${WORKDIR}/tstage install", d)
238 oe_run(d, "%s %s" % (cmd, stagepkg))
239 except RuntimeError, exc:
240 bb.fatal("Couldn't install the staging package to a temp directory: %s" % exc)
243 # Grab the staging lock now so that we don't have other threads try and
244 # validate or install based on these stamps being valid. This is a
245 # potential issue for certain BB_STAMP_POLICY values and enough
246 # concurrent threads.
248 lf = bb.utils.lockfile(bb.data.expand("${SYSROOT_LOCK}", d))
251 # Copy the stamp files into the main stamps directoy
253 cmd = bb.data.expand("cp -PpR ${WORKDIR}/tstage/stamps/* ${TMPDIR}/stamps/", d)
256 except RuntimeError, exc:
257 bb.utils.unlockfile(lf)
258 bb.fatal("Couldn't copy the staging package stamp files: %s" % exc)
261 # Iterate over the stamps seeing if they're valid. If we find any that
262 # are invalid or the task wasn't in the taskgraph, assume caution and
265 # FIXME - some tasks are safe to ignore in the task graph. e.g. package_write_*
267 taskscovered = bb.data.getVar("PSTAGE_TASKS_COVERED", d, True).split()
268 stamp = bb.data.getVar("STAMP", d, True)
269 for task in taskscovered:
271 stampfn = "%s.%s" % (stamp, task)
272 bb.debug(1, "Checking %s" % (stampfn))
273 if os.path.exists(stampfn):
274 stageok = bb.runqueue.check_stamp_fn(file, task, d)
275 bb.debug(1, "Result %s" % (stageok))
277 bb.utils.unlockfile(lf)
280 # Remove the stamps and files we added above
281 # FIXME - we should really only remove the stamps we added
282 for fname in glob.glob(stamp + '.*'):
283 oe.path.remove(fname)
285 oe.path.remove(bb.data.expand("${WORKDIR}/tstage", d))
287 bb.note("Staging package found, using it for %s." % file)
288 installcmd = bb.data.getVar("PSTAGE_INSTALL_CMD", d, 1)
290 ret = oe_run(d, "%s %s" % (installcmd, stagepkg))
292 bb.note("Failure installing prestage package")
294 bb.utils.unlockfile(lf)
296 bb.build.exec_func("staging_package_libtoolhack", d)
298 bb.build.make_stamp("do_stage_package_populated", d)
300 bb.note("Staging package found but invalid for %s" % file)
303 packagestage_scenefunc[cleandirs] = "${PSTAGE_TMPDIR_STAGE}"
304 packagestage_scenefunc[dirs] = "${STAGING_DIR}"
306 addhandler packagedstage_stampfixing_eventhandler
307 python packagedstage_stampfixing_eventhandler() {
308 if bb.event.getName(e) == "StampUpdate":
309 taskscovered = bb.data.getVar("PSTAGE_TASKS_COVERED", e.data, 1).split()
310 for (fn, task) in e.targets:
313 if task in taskscovered:
314 stamp = "%s.do_stage_package_populated" % e.stampPrefix[fn]
315 if os.path.exists(stamp):
316 # We're targetting a task which was skipped with packaged staging
317 # so we need to remove the autogenerated stamps.
318 for task in taskscovered:
319 covered = "%s.do_%s" % (e.stampPrefix[fn], task)
320 oe.path.remove(covered)
321 oe.path.remove(stamp)
324 populate_sysroot_preamble () {
325 if [ "$PSTAGING_ACTIVE" = "1" ]; then
326 stage-manager -p ${STAGING_DIR} -c ${PSTAGE_WORKDIR}/stamp-cache-staging -u || true
330 populate_sysroot_postamble () {
331 if [ "$PSTAGING_ACTIVE" = "1" ]; then
332 # list the packages currently installed in staging
333 # ${PSTAGE_LIST_CMD} | awk '{print $1}' > ${PSTAGE_WORKDIR}/installed-list
335 # exitcode == 5 is ok, it means the files change
337 stage-manager -p ${STAGING_DIR} -c ${PSTAGE_WORKDIR}/stamp-cache-staging -u -d ${PSTAGE_TMPDIR_STAGE}/sysroots
339 if [ "$exitcode" != "5" -a "$exitcode" != "0" ]; then
346 packagedstaging_fastpath () {
347 mkdir -p ${PSTAGE_TMPDIR_STAGE}/sysroots/
348 cp -fpPR ${SYSROOT_DESTDIR}/${STAGING_DIR}/* ${PSTAGE_TMPDIR_STAGE}/sysroots/ || /bin/true
351 do_populate_sysroot[dirs] =+ "${PSTAGE_DIR}"
352 python populate_sysroot_prehook() {
353 bb.build.exec_func("populate_sysroot_preamble", d)
356 python populate_sysroot_posthook() {
357 bb.build.exec_func("populate_sysroot_postamble", d)
361 staging_packager () {
363 mkdir -p ${PSTAGE_TMPDIR_STAGE}/CONTROL
364 mkdir -p ${PSTAGE_DIR}/${PSTAGE_PKGPATH}
366 echo "Package: ${PSTAGE_PKGPN}" > ${PSTAGE_TMPDIR_STAGE}/CONTROL/control
367 echo "Version: ${PSTAGE_PKGVERSION}" >> ${PSTAGE_TMPDIR_STAGE}/CONTROL/control
368 echo "Description: ${DESCRIPTION}" >> ${PSTAGE_TMPDIR_STAGE}/CONTROL/control
369 echo "Section: ${SECTION}" >> ${PSTAGE_TMPDIR_STAGE}/CONTROL/control
370 echo "Priority: Optional" >> ${PSTAGE_TMPDIR_STAGE}/CONTROL/control
371 echo "Maintainer: ${MAINTAINER}" >> ${PSTAGE_TMPDIR_STAGE}/CONTROL/control
372 echo "Architecture: ${PSTAGE_PKGARCH}" >> ${PSTAGE_TMPDIR_STAGE}/CONTROL/control
374 # Protect against empty SRC_URI
376 if [ "$srcuri" = "" ]; then
377 srcuri="OpenEmbedded"
379 echo "Source: $srcuri" >> ${PSTAGE_TMPDIR_STAGE}/CONTROL/control
381 # Deal with libtool not supporting sysroots
382 # Need to remove hardcoded paths and fix these when we install the
384 # Could someone please add sysroot support to libtool!
385 for i in `${PSTAGE_SCAN_CMD}` ; do \
386 sed -i -e s:${STAGING_DIR}:FIXMESTAGINGDIR:g $i
387 echo $i | sed -e 's:${PSTAGE_TMPDIR_STAGE}/::' >> ${PSTAGE_TMPDIR_STAGE}/sysroots/fixmepath
390 ${PSTAGE_BUILD_CMD} ${PSTAGE_TMPDIR_STAGE} ${PSTAGE_DIR}/${PSTAGE_PKGPATH}
393 staging_package_installer () {
394 #${PSTAGE_INSTALL_CMD} ${PSTAGE_PKG}
396 STATUSFILE=${TMPDIR}${libdir_native}/opkg/status
397 echo "Package: ${PSTAGE_PKGPN}" >> $STATUSFILE
398 echo "Version: ${PSTAGE_PKGVERSION}" >> $STATUSFILE
399 echo "Status: install user installed" >> $STATUSFILE
400 echo "Architecture: ${PSTAGE_PKGARCH}" >> $STATUSFILE
401 echo "" >> $STATUSFILE
403 CTRLFILE=${TMPDIR}${libdir_native}/opkg/info/${PSTAGE_PKGPN}.control
404 echo "Package: ${PSTAGE_PKGPN}" > $CTRLFILE
405 echo "Version: ${PSTAGE_PKGVERSION}" >> $CTRLFILE
406 echo "Architecture: ${PSTAGE_PKGARCH}" >> $CTRLFILE
408 cd ${PSTAGE_TMPDIR_STAGE}
409 find -type f | grep -v ./CONTROL | sed -e 's/^\.//' > ${TMPDIR}${libdir_native}/opkg/info/${PSTAGE_PKGPN}.list
412 python staging_package_libtoolhack () {
413 # Deal with libtool not supporting sysroots and add our new
415 tmpdir = bb.data.getVar('TMPDIR', d, True)
416 staging = bb.data.getVar('STAGING_DIR', d, True)
417 fixmefn = staging + "/fixmepath"
419 fixmefd = open(fixmefn,"r")
420 fixmefiles = fixmefd.readlines()
422 oe.path.remove(fixmefn)
423 for file in fixmefiles:
424 oe_run(d, "sed -i -e s:FIXMESTAGINGDIR:%s:g %s" % (staging, tmpdir + '/' + file))
429 python do_package_stage () {
430 import oe.packagedata
432 if bb.data.getVar("PSTAGING_ACTIVE", d, 1) != "1":
436 # Handle deploy/ packages
438 bb.build.exec_func("read_subpackage_metadata", d)
439 stagepath = bb.data.getVar("PSTAGE_TMPDIR_STAGE", d, 1)
440 tmpdir = bb.data.getVar("TMPDIR", d, True)
441 packages = (bb.data.getVar('PACKAGES', d, 1) or "").split()
442 if len(packages) > 0:
443 if bb.data.inherits_class('package_ipk', d):
444 ipkpath = bb.data.getVar('DEPLOY_DIR_IPK', d, True).replace(tmpdir, stagepath)
445 if bb.data.inherits_class('package_deb', d):
446 debpath = bb.data.getVar('DEPLOY_DIR_DEB', d, True).replace(tmpdir, stagepath)
447 if bb.data.inherits_class('package_rpm', d):
448 rpmpath = bb.data.getVar('DEPLOY_DIR_RPM', d, True).replace(tmpdir, stagepath)
451 pkgname = bb.data.getVar('PKG_%s' % pkg, d, 1)
454 arch = bb.data.getVar('PACKAGE_ARCH_%s' % pkg, d, 1)
456 arch = bb.data.getVar('PACKAGE_ARCH', d, 1)
457 pr = bb.data.getVar('PR_%s' % pkg, d, 1)
459 pr = bb.data.getVar('PR', d, 1)
460 pkgv = bb.data.getVar('PKGV_%s' % pkg, d, 1)
462 pkgv = bb.data.getVar('PKGV', d, 1)
463 if not oe.packagedata.packaged(pkg, d):
465 if bb.data.inherits_class('package_ipk', d):
466 srcname = bb.data.expand(pkgname + "_" + pkgv + "-" + pr + "${DISTRO_PR}" + "_" + arch + ".ipk", d)
467 srcfile = bb.data.expand("${DEPLOY_DIR_IPK}/" + arch + "/" + srcname, d)
468 if os.path.exists(srcfile):
469 destpath = ipkpath + "/" + arch + "/"
470 bb.mkdirhier(destpath)
471 bb.copyfile(srcfile, destpath + srcname)
473 if bb.data.inherits_class('package_deb', d):
475 srcname = bb.data.expand(pkgname + "_${PV}-" + pr + "${DISTRO_PR}" + "_all.deb", d)
477 srcname = bb.data.expand(pkgname + "_${PV}-" + pr + "${DISTRO_PR}" + "_${DPKG_ARCH}.deb", d)
478 srcfile = bb.data.expand("${DEPLOY_DIR_DEB}/" + arch + "/" + srcname, d)
479 if os.path.exists(srcfile):
480 destpath = debpath + "/" + arch + "/"
481 bb.mkdirhier(destpath)
482 bb.copyfile(srcfile, destpath + srcname)
484 if bb.data.inherits_class('package_rpm', d):
485 version = bb.data.getVar('PV', d, 1)
486 version = version.replace('-', '+')
487 bb.data.setVar('RPMPV', version, d)
488 srcname = bb.data.expand(pkgname + "-${RPMPV}-" + pr + "${DISTRO_PR}" + ".${TARGET_ARCH}.rpm", d)
489 srcfile = bb.data.expand("${DEPLOY_DIR_RPM}/" + arch + "/" + srcname, d)
490 if os.path.exists(srcfile):
491 destpath = rpmpath + "/" + arch + "/"
492 bb.mkdirhier(destpath)
493 bb.copyfile(srcfile, destpath + srcname)
497 # Handle stamps/ files
499 stampfn = bb.data.getVar("STAMP", d, True)
500 destdir = os.path.dirname(stampfn.replace(tmpdir, stagepath))
501 bb.mkdirhier(destdir)
502 # We need to include the package_stage stamp in the staging package so create one
503 bb.build.make_stamp("do_package_stage", d)
504 oe_run(d, "cp -PpR %s.do_* %s/" % (stampfn, destdir))
506 pstage_set_pkgmanager(d)
507 bb.build.exec_func("staging_helper", d)
508 bb.build.exec_func("staging_packager", d)
509 lf = bb.utils.lockfile(bb.data.expand("${SYSROOT_LOCK}", d))
510 bb.build.exec_func("staging_package_installer", d)
511 bb.utils.unlockfile(lf)
515 # Note an assumption here is that do_deploy runs before do_package_write/do_populate_sysroot
517 addtask package_stage after do_package_write do_populate_sysroot before do_build
519 do_package_stage_all () {
522 do_package_stage_all[recrdeptask] = "do_package_stage"
523 addtask package_stage_all after do_package_stage before do_build