2 # Populate builds using prebuilt packages where possible to speed up builds
3 # and allow staging to be reconstructed.
5 # To use it add that line to conf/local.conf:
7 # INHERIT += "packaged-staging"
12 # bitbake.conf set PSTAGING_ACTIVE = "0", this class sets to "1" if we're active
14 PSTAGE_PKGVERSION = "${PV}-${PR}"
15 PSTAGE_PKGARCH = "${BUILD_SYS}"
16 PSTAGE_EXTRAPATH ?= "/${OELAYOUT_ABI}/${DISTRO_PR}/"
17 PSTAGE_PKGPATH = "${DISTRO}${PSTAGE_EXTRAPATH}"
18 PSTAGE_PKGPN = "${@bb.data.expand('staging-${PN}-${MULTIMACH_ARCH}${TARGET_VENDOR}-${TARGET_OS}', d).replace('_', '-')}"
19 PSTAGE_PKGNAME = "${PSTAGE_PKGPN}_${PSTAGE_PKGVERSION}_${PSTAGE_PKGARCH}.ipk"
20 PSTAGE_PKG ?= "${DEPLOY_DIR_PSTAGE}/${PSTAGE_PKGPATH}/${PSTAGE_PKGNAME}"
22 PSTAGE_NATIVEDEPENDS = "\
27 BB_STAMP_WHITELIST = "${PSTAGE_NATIVEDEPENDS}"
32 # These classes encode staging paths into the binary data so can only be
33 # reused if the path doesn't change/
34 if bb.data.inherits_class('native', d) or bb.data.inherits_class('cross', d) or bb.data.inherits_class('sdk', d):
35 path = bb.data.getVar('PSTAGE_PKGPATH', d, 1)
36 path = path + bb.data.getVar('TMPDIR', d, 1).replace('/', '-')
37 bb.data.setVar('PSTAGE_PKGPATH', path, d)
39 # PSTAGE_NATIVEDEPENDS lists the packages we need before we can use packaged
40 # staging. There will always be some packages we depend on.
41 if bb.data.inherits_class('native', d):
42 pn = bb.data.getVar('PN', d, True)
43 nativedeps = bb.data.getVar('PSTAGE_NATIVEDEPENDS', d, True).split()
45 pstage_allowed = False
47 # Images aren't of interest to us
48 if bb.data.inherits_class('image', d):
49 pstage_allowed = False
51 if bb.data.getVar('PSTAGING_DISABLED', d, True) == "1":
52 pstage_allowed = False
54 # Add task dependencies if we're active, otherwise mark packaged staging
57 deps = bb.data.getVarFlag('do_setscene', 'depends', d) or ""
58 deps += " stagemanager-native:do_populate_staging"
59 bb.data.setVarFlag('do_setscene', 'depends', deps, d)
61 policy = bb.data.getVar("BB_STAMP_POLICY", d, True)
62 if policy == "whitelist" or policy == "full":
63 deps = bb.data.getVarFlag('do_setscene', 'recrdeptask', d) or ""
64 deps += " do_setscene"
65 bb.data.setVarFlag('do_setscene', 'recrdeptask', deps, d)
67 bb.data.setVar("PSTAGING_ACTIVE", "1", d)
69 bb.data.setVar("PSTAGING_ACTIVE", "0", d)
72 DEPLOY_DIR_PSTAGE ?= "${DEPLOY_DIR}/pstage"
73 PSTAGE_MACHCONFIG = "${DEPLOY_DIR_PSTAGE}/opkg.conf"
75 PSTAGE_PKGMANAGER = "stage-manager-ipkg"
77 PSTAGE_BUILD_CMD = "stage-manager-ipkg-build -o 0 -g 0"
78 PSTAGE_INSTALL_CMD = "${PSTAGE_PKGMANAGER} -f ${PSTAGE_MACHCONFIG} -force-depends -o ${TMPDIR} install"
79 PSTAGE_UPDATE_CMD = "${PSTAGE_PKGMANAGER} -f ${PSTAGE_MACHCONFIG} -o ${TMPDIR} update"
80 PSTAGE_REMOVE_CMD = "${PSTAGE_PKGMANAGER} -f ${PSTAGE_MACHCONFIG} -force-depends -o ${TMPDIR} remove"
81 PSTAGE_LIST_CMD = "${PSTAGE_PKGMANAGER} -f ${PSTAGE_MACHCONFIG} -o ${TMPDIR} list_installed"
83 PSTAGE_TMPDIR_STAGE = "${WORKDIR}/staging-pkg"
85 def pstage_manualclean(srcname, destvarname, d):
86 src = os.path.join(bb.data.getVar('PSTAGE_TMPDIR_STAGE', d, True), srcname)
87 dest = bb.data.getVar(destvarname, d, True)
89 for walkroot, dirs, files in os.walk(src):
91 filepath = os.path.join(walkroot, file).replace(src, dest)
92 bb.note("rm %s" % filepath)
93 os.system("rm %s" % filepath)
95 def pstage_set_pkgmanager(d):
96 path = bb.data.getVar("PATH", d, 1)
97 pkgmanager = bb.which(path, 'opkg-cl')
99 pkgmanager = bb.which(path, 'ipkg-cl')
101 bb.data.setVar("PSTAGE_PKGMANAGER", pkgmanager, d)
104 def pstage_cleanpackage(pkgname, d):
105 path = bb.data.getVar("PATH", d, 1)
106 pstage_set_pkgmanager(d)
107 list_cmd = bb.data.getVar("PSTAGE_LIST_CMD", d, True)
109 bb.debug(2, "Checking if staging package installed")
110 lf = bb.utils.lockfile(bb.data.expand("${SYSROOT_LOCK}", d))
111 ret = os.system("PATH=\"%s\" %s | grep %s" % (path, list_cmd, pkgname))
113 bb.debug(1, "Uninstalling package from staging...")
114 removecmd = bb.data.getVar("PSTAGE_REMOVE_CMD", d, 1)
115 ret = os.system("PATH=\"%s\" %s %s" % (path, removecmd, pkgname))
117 bb.note("Failure removing staging package")
119 bb.debug(1, "Manually removing any installed files from staging...")
120 pstage_manualclean("staging", "STAGING_DIR", d)
121 pstage_manualclean("cross", "CROSS_DIR", d)
122 pstage_manualclean("deploy", "DEPLOY_DIR", d)
124 bb.utils.unlockfile(lf)
128 Clear the build and temp directories
131 removepkg = bb.data.expand("${PSTAGE_PKGPN}", d)
132 pstage_cleanpackage(removepkg, d)
134 stagepkg = bb.data.expand("${PSTAGE_PKG}", d)
135 bb.note("Removing staging package %s" % base_path_out(stagepkg, d))
136 os.system('rm -rf ' + stagepkg)
140 # Assemble appropriate opkg.conf
141 conffile=${PSTAGE_MACHCONFIG}
142 mkdir -p ${DEPLOY_DIR_PSTAGE}/pstaging_lists
143 if [ ! -e $conffile ]; then
144 ipkgarchs="${BUILD_SYS}"
146 for arch in $ipkgarchs; do
147 echo "arch $arch $priority" >> $conffile
148 priority=$(expr $priority + 5)
150 echo "dest root /" >> $conffile
152 if [ ! -e ${TMPDIR}${libdir_native}/opkg/info/ ]; then
153 mkdir -p ${TMPDIR}${libdir_native}/opkg/info/
155 if [ ! -e ${TMPDIR}${libdir_native}/ipkg/ ]; then
156 ln -sf opkg/ ${TMPDIR}${libdir_native}/ipkg || true
160 PSTAGE_TASKS_COVERED = "fetch unpack munge patch configure qa_configure rig_locales compile sizecheck install deploy package populate_staging package_write_deb package_write_ipk package_write package_stage qa_staging"
162 SCENEFUNCS += "packagestage_scenefunc"
164 python packagestage_scenefunc () {
165 if bb.data.getVar("PSTAGING_ACTIVE", d, 1) == "0":
168 bb.build.exec_func("staging_helper", d)
170 removepkg = bb.data.expand("${PSTAGE_PKGPN}", d)
172 pstage_cleanpackage(removepkg, d)
174 stagepkg = bb.data.expand("${PSTAGE_PKG}", d)
176 if os.path.exists(stagepkg):
177 path = bb.data.getVar("PATH", d, 1)
178 pstage_set_pkgmanager(d)
179 file = bb.data.getVar("FILE", d, True)
180 bb.debug(2, "Packaged staging active for %s\n" % file)
183 # Install the staging package somewhere temporarily so we can extract the stamp files
185 bb.mkdirhier(bb.data.expand("${WORKDIR}/tstage/${libdir_native}/opkg/info/ ", d))
186 cmd = bb.data.expand("${PSTAGE_PKGMANAGER} -f ${PSTAGE_MACHCONFIG} -force-depends -o ${WORKDIR}/tstage install", d)
187 ret = os.system("PATH=\"%s\" %s %s" % (path, cmd, stagepkg))
189 bb.fatal("Couldn't install the staging package to a temp directory")
192 # Grab the staging lock now so that we don't have other threads try and
193 # validate or install based on these stamps being valid. This is a
194 # potential issue for certain BB_STAMP_POLICY values and enough
195 # concurrent threads.
197 lf = bb.utils.lockfile(bb.data.expand("${SYSROOT_LOCK}", d))
200 # Copy the stamp files into the main stamps directoy
202 cmd = bb.data.expand("cp -dpR ${WORKDIR}/tstage/stamps/* ${TMPDIR}/stamps/", d)
205 bb.utils.unlockfile(lf)
206 bb.fatal("Couldn't copy the staging package stamp files")
209 # Iterate over the stamps seeing if they're valid. If we find any that
210 # are invalid or the task wasn't in the taskgraph, assume caution and
213 # FIXME - some tasks are safe to ignore in the task graph. e.g. package_write_*
215 taskscovered = bb.data.getVar("PSTAGE_TASKS_COVERED", d, True).split()
216 stamp = bb.data.getVar("STAMP", d, True)
217 for task in taskscovered:
219 stampfn = "%s.%s" % (stamp, task)
220 bb.debug(1, "Checking %s" % (stampfn))
221 if os.path.exists(stampfn):
222 stageok = bb.runqueue.check_stamp_fn(file, task, d)
223 bb.debug(1, "Result %s" % (stageok))
225 bb.utils.unlockfile(lf)
228 # Remove the stamps and files we added above
229 # FIXME - we should really only remove the stamps we added
230 os.system('rm -f ' + stamp + '.*')
231 os.system(bb.data.expand("rm -rf ${WORKDIR}/tstage", d))
234 bb.note("Staging package found, using it for %s." % file)
235 installcmd = bb.data.getVar("PSTAGE_INSTALL_CMD", d, 1)
236 ret = os.system("PATH=\"%s\" %s %s" % (path, installcmd, stagepkg))
237 bb.utils.unlockfile(lf)
239 bb.note("Failure installing prestage package")
241 bb.build.make_stamp("do_stage_package_populated", d)
243 bb.note("Staging package found but invalid for %s" % file)
246 packagestage_scenefunc[cleandirs] = "${PSTAGE_TMPDIR_STAGE}"
247 packagestage_scenefunc[dirs] = "${STAGING_DIR}"
249 addhandler packagedstage_stampfixing_eventhandler
250 python packagedstage_stampfixing_eventhandler() {
251 if bb.event.getName(e) == "StampUpdate":
252 taskscovered = bb.data.getVar("PSTAGE_TASKS_COVERED", e.data, 1).split()
253 for (fn, task) in e.targets:
256 if task in taskscovered:
257 stamp = "%s.do_stage_package_populated" % e.stampPrefix[fn]
258 if os.path.exists(stamp):
259 # We're targetting a task which was skipped with packaged staging
260 # so we need to remove the autogenerated stamps.
261 for task in taskscovered:
262 dir = "%s.do_%s" % (e.stampPrefix[fn], task)
263 os.system('rm -f ' + dir)
264 os.system('rm -f ' + stamp)
269 populate_staging_preamble () {
270 if [ "$PSTAGING_ACTIVE" = "1" ]; then
271 stage-manager -p ${STAGING_DIR} -c ${DEPLOY_DIR_PSTAGE}/stamp-cache-staging -u || true
272 stage-manager -p ${CROSS_DIR} -c ${DEPLOY_DIR_PSTAGE}/stamp-cache-cross -u || true
276 populate_staging_postamble () {
277 if [ "$PSTAGING_ACTIVE" = "1" ]; then
278 # list the packages currently installed in staging
279 # ${PSTAGE_LIST_CMD} | awk '{print $1}' > ${DEPLOY_DIR_PSTAGE}/installed-list
281 # exitcode == 5 is ok, it means the files change
283 stage-manager -p ${STAGING_DIR} -c ${DEPLOY_DIR_PSTAGE}/stamp-cache-staging -u -d ${PSTAGE_TMPDIR_STAGE}/staging
285 if [ "$exitcode" != "5" -a "$exitcode" != "0" ]; then
288 stage-manager -p ${CROSS_DIR} -c ${DEPLOY_DIR_PSTAGE}/stamp-cache-cross -u -d ${PSTAGE_TMPDIR_STAGE}/cross/${BASE_PACKAGE_ARCH}
289 if [ "$exitcode" != "5" -a "$exitcode" != "0" ]; then
296 packagedstaging_fastpath () {
297 if [ "$PSTAGING_ACTIVE" = "1" ]; then
298 mkdir -p ${PSTAGE_TMPDIR_STAGE}/staging/
299 mkdir -p ${PSTAGE_TMPDIR_STAGE}/cross/
300 cp -fpPR ${SYSROOT_DESTDIR}${STAGING_DIR}/* ${PSTAGE_TMPDIR_STAGE}/staging/ || /bin/true
301 cp -fpPR ${SYSROOT_DESTDIR}${CROSS_DIR}/* ${PSTAGE_TMPDIR_STAGE}/cross/${BASE_PACKAGE_ARCH}/ || /bin/true
305 do_populate_staging[dirs] =+ "${DEPLOY_DIR_PSTAGE}"
306 python populate_staging_prehook() {
307 bb.build.exec_func("populate_staging_preamble", d)
310 python populate_staging_posthook() {
311 bb.build.exec_func("populate_staging_postamble", d)
315 staging_packager () {
317 mkdir -p ${PSTAGE_TMPDIR_STAGE}/CONTROL
318 mkdir -p ${DEPLOY_DIR_PSTAGE}/${PSTAGE_PKGPATH}
320 echo "Package: ${PSTAGE_PKGPN}" > ${PSTAGE_TMPDIR_STAGE}/CONTROL/control
321 echo "Version: ${PSTAGE_PKGVERSION}" >> ${PSTAGE_TMPDIR_STAGE}/CONTROL/control
322 echo "Description: ${DESCRIPTION}" >> ${PSTAGE_TMPDIR_STAGE}/CONTROL/control
323 echo "Section: ${SECTION}" >> ${PSTAGE_TMPDIR_STAGE}/CONTROL/control
324 echo "Priority: Optional" >> ${PSTAGE_TMPDIR_STAGE}/CONTROL/control
325 echo "Maintainer: ${MAINTAINER}" >> ${PSTAGE_TMPDIR_STAGE}/CONTROL/control
326 echo "Architecture: ${PSTAGE_PKGARCH}" >> ${PSTAGE_TMPDIR_STAGE}/CONTROL/control
328 # Protect against empty SRC_URI
329 if [ "${SRC_URI}" != "" ] ; then
330 echo "Source: ${SRC_URI}" >> ${PSTAGE_TMPDIR_STAGE}/CONTROL/control
332 echo "Source: OpenEmbedded" >> ${PSTAGE_TMPDIR_STAGE}/CONTROL/control
335 ${PSTAGE_BUILD_CMD} ${PSTAGE_TMPDIR_STAGE} ${DEPLOY_DIR_PSTAGE}/${PSTAGE_PKGPATH}
338 staging_package_installer () {
339 #${PSTAGE_INSTALL_CMD} ${PSTAGE_PKG}
341 STATUSFILE=${TMPDIR}${libdir_native}/opkg/status
342 echo "Package: ${PSTAGE_PKGPN}" >> $STATUSFILE
343 echo "Version: ${PSTAGE_PKGVERSION}" >> $STATUSFILE
344 echo "Status: install user installed" >> $STATUSFILE
345 echo "Architecture: ${PSTAGE_PKGARCH}" >> $STATUSFILE
346 echo "" >> $STATUSFILE
348 CTRLFILE=${TMPDIR}${libdir_native}/opkg/info/${PSTAGE_PKGPN}.control
349 echo "Package: ${PSTAGE_PKGPN}" > $CTRLFILE
350 echo "Version: ${PSTAGE_PKGVERSION}" >> $CTRLFILE
351 echo "Architecture: ${PSTAGE_PKGARCH}" >> $CTRLFILE
353 cd ${PSTAGE_TMPDIR_STAGE}
354 find -type f | grep -v ./CONTROL | sed -e 's/^\.//' > ${TMPDIR}${libdir_native}/opkg/info/${PSTAGE_PKGPN}.list
357 python do_package_stage () {
358 if bb.data.getVar("PSTAGING_ACTIVE", d, 1) != "1":
362 # Handle deploy/ packages
364 bb.build.exec_func("read_subpackage_metadata", d)
365 stagepath = bb.data.getVar("PSTAGE_TMPDIR_STAGE", d, 1)
366 tmpdir = bb.data.getVar("TMPDIR", d, True)
367 packages = (bb.data.getVar('PACKAGES', d, 1) or "").split()
368 if len(packages) > 0:
369 if bb.data.inherits_class('package_ipk', d):
370 ipkpath = bb.data.getVar('DEPLOY_DIR_IPK', d, True).replace(tmpdir, stagepath)
371 if bb.data.inherits_class('package_deb', d):
372 debpath = bb.data.getVar('DEPLOY_DIR_DEB', d, True).replace(tmpdir, stagepath)
373 if bb.data.inherits_class('package_rpm', d):
374 rpmpath = bb.data.getVar('DEPLOY_DIR_RPM', d, True).replace(tmpdir, stagepath)
377 pkgname = bb.data.getVar('PKG_%s' % pkg, d, 1)
380 arch = bb.data.getVar('PACKAGE_ARCH_%s' % pkg, d, 1)
382 arch = bb.data.getVar('PACKAGE_ARCH', d, 1)
383 pr = bb.data.getVar('PR_%s' % pkg, d, 1)
385 pr = bb.data.getVar('PR', d, 1)
386 if not packaged(pkg, d):
388 if bb.data.inherits_class('package_ipk', d):
389 srcname = bb.data.expand(pkgname + "_${PV}-" + pr + "${DISTRO_PR}" + "_" + arch + ".ipk", d)
390 srcfile = bb.data.expand("${DEPLOY_DIR_IPK}/" + arch + "/" + srcname, d)
391 if os.path.exists(srcfile):
392 destpath = ipkpath + "/" + arch + "/"
393 bb.mkdirhier(destpath)
395 bb.copyfile(srcfile, destpath + srcname)
397 if bb.data.inherits_class('package_deb', d):
399 srcname = bb.data.expand(pkgname + "_${PV}-" + pr + "${DISTRO_PR}" + "_all.deb", d)
401 srcname = bb.data.expand(pkgname + "_${PV}-" + pr + "${DISTRO_PR}" + "_${DPKG_ARCH}.deb", d)
402 srcfile = bb.data.expand("${DEPLOY_DIR_DEB}/" + arch + "/" + srcname, d)
403 if os.path.exists(srcfile):
404 destpath = debpath + "/" + arch + "/"
405 bb.mkdirhier(destpath)
406 bb.copyfile(srcfile, destpath + srcname)
408 if bb.data.inherits_class('package_rpm', d):
409 version = bb.data.getVar('PV', d, 1)
410 version = version.replace('-', '+')
411 bb.data.setVar('RPMPV', version, d)
412 srcname = bb.data.expand(pkgname + "-${RPMPV}-" + pr + "${DISTRO_PR}" + ".${TARGET_ARCH}.rpm", d)
413 srcfile = bb.data.expand("${DEPLOY_DIR_RPM}/" + arch + "/" + srcname, d)
414 if os.path.exists(srcfile):
415 destpath = rpmpath + "/" + arch + "/"
416 bb.mkdirhier(destpath)
417 bb.copyfile(srcfile, destpath + srcname)
421 # Handle stamps/ files
423 stampfn = bb.data.getVar("STAMP", d, True)
424 destdir = os.path.dirname(stampfn.replace(tmpdir, stagepath))
425 bb.mkdirhier(destdir)
426 # We need to include the package_stage stamp in the staging package so create one
427 bb.build.make_stamp("do_package_stage", d)
428 os.system("cp -dpR %s.do_* %s/" % (stampfn, destdir))
430 pstage_set_pkgmanager(d)
431 bb.build.exec_func("staging_helper", d)
432 bb.build.exec_func("staging_packager", d)
433 lf = bb.utils.lockfile(bb.data.expand("${SYSROOT_LOCK}", d))
434 bb.build.exec_func("staging_package_installer", d)
435 bb.utils.unlockfile(lf)
439 # Note an assumption here is that do_deploy runs before do_package_write/do_populate_staging
441 addtask package_stage after do_package_write do_populate_staging before do_build
443 do_package_stage_all () {
446 do_package_stage_all[recrdeptask] = "do_package_stage"
447 addtask package_stage_all after do_package_stage before do_build