--- /dev/null
+#!/bin/sh
+
+# TestBuilder setup script for OpenEmbedded
+# Copyright (c) 2010 Yuri Bushmelev <jay4mail@gmail.com>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+
+TB_DIR=`dirname $0`
+CFG_FILE="${TB_DIR}/testbuilder.conf"
+
+if [ "$#" -lt "2" -o "$1" = "-h" -o "$1" = "--help" ]; then
+ echo "Usage: `basename $0` <OE branch> <bitbake branch>"
+ exit 1
+fi
+
+OE_BRANCH=$1
+BB_BRANCH=$2
+
+. "${CFG_FILE}"
+
+cd "${TB_BASE}"
+
+echo "OE branch: $OE_BRANCH"
+echo "BB branch: $BB_BRANCH"
+
+## Checkout bitbake
+echo "= Fetching bitbake"
+git clone -n git://git.openembedded.org/bitbake "${BB_DIR}"
+cd "${BB_DIR}"
+
+if [ -n "${BB_BRANCH}" ]; then
+ git checkout -b "${BB_BRANCH}" "origin/${BB_BRANCH}"
+elif [ -n "${BB_TAG}" ]; then
+ git checkout -b "${BB_TAG}" "${BB_TAG}"
+else
+ echo "You should specify BB_BRANCH or BB_TAG in ${CFG_FILE}"
+fi
+
+echo "= Fetching OE"
+cd "${TB_BASE}"
+
+git clone -n git://git.openembedded.org/openembedded "${OE_DIR}"
+cd "${OE_DIR}"
+
+if [ -n "${OE_BRANCH}" ]; then
+ git checkout -b "${OE_BRANCH}" "origin/${OE_BRANCH}"
+else
+ echo "You should specify OE_BRANCH in ${CFG_FILE}"
+fi
+
+cd "${TB_BASE}"
+mkdir -p "${BLD_DIR}/conf"
+
+echo "= Creating bitbake config"
+cat >> "${BLD_DIR}/conf/local.conf" << "EOF"
+# Testbuilder bitbake local configuration file
+
+DISTRO ?= "${@bb.fatal('Set DISTRO in your config')}"
+MACHINE ?= "${@bb.fatal('Set MACHINE in your config')}"
+
+DL_DIR ?= "${@bb.fatal('Set DL_DIR in your config')}"
+TMPDIR = "${TMP_DIR}"
+
+BBFILES ?= "${OE_DIR}/recipes/*/*.bb"
+
+# ENABLE_BINARY_LOCALE_GENERATION = "0"
+# GLIBC_GENERATE_LOCALES = "en_US.UTF-8 en_GB.UTF-8 de_DE.UTF-8 fr_FR.UTF-8 ru_RU.UTF-8"
+# IMAGE_LINGUAS="en-us en-gb de-de fr-fr ru-ru"
+
+# jffs2, tar(.gz|bz2), cpio(.gz), cramfs, ext2(.gz), ext3(.gz)
+# squashfs, squashfs-lzma
+IMAGE_FSTYPES = "jffs2 tar.gz"
+
+# CCACHE = "${@bb.which(bb.data.getVar('PATH', d, 1), 'ccache') and 'ccache '}"
+PARALLEL_MAKE = "-j ${MAKE_NUMBER_THREADS}"
+BB_NUMBER_THREADS ?= "${@bb.fatal('Set BB_NUMBER_THREADS in your config')}"
+
+INHERIT += "rm_work"
+#INHERIT += "devshell"
+#TERMCMD = ${SCREEN_TERMCMD}
+
+BBINCLUDELOGS = "yes"
+
+# OE stats client (make troubleshooting easier)
+INHERIT += "oestats-client"
+OESTATS_BUILDER = "${TB_BUILDER}"
+OESTATS_SERVER = "tinderbox.openembedded.net"
+
+EOF
+
+echo "= All is done. Run ${TB_DIR}/testbuilder now"
+
--- /dev/null
+#!/bin/sh
+
+# TestBuilder for OpenEmbedded
+# Copyright (c) 2010 Yuri Bushmelev <jay4mail@gmail.com>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+VERSION="0.3"
+PACKAGE="TestBuilder"
+
+TB_DIR=`dirname $0`
+TB_NAME=`basename $0`
+cd "${TB_DIR}"
+
+tb_usage() {
+ echo "${PACKAGE} v${VERSION}"
+ # Run only specified build
+ echo "Usage: ${TB_NAME} -Vh"
+ echo " or ${TB_NAME} [-B build ]"
+ # Behave like DISTRO=distro MACHINE=machine bitbake images
+ # with -S testbuilder will only setup env and run interactive sub-shell
+ echo " or ${TB_NAME} [-D distro] [-M machine] [-T /full/path/to/tmpdir] [-S | images]"
+}
+
+CFG_FILE="${TB_DIR}/testbuilder.conf"
+. "${CFG_FILE}"
+
+# Sanity cleanup
+ORIG_PATH=${PATH}
+ORIG_LD_LIBRARY_PATH=${LD_LIBRARY_PATH}
+ORIG_LANG=${LANG}
+export ORIG_PATH ORIG_LD_LIBRARY_PATH ORIG_LANG
+
+# clean LD_LIBRARY_PATH
+unset LD_LIBRARY_PATH
+
+# clean locale
+LANG=C
+
+# include bitbake into PATH
+PATH=${BB_DIR}/bin:${PATH}
+
+BBPATH="${BLD_DIR}:${OE_DIR}"
+
+TB_LOG="${LOG_DIR}/tb.log"
+
+#export LD_LIBRARY_PATH LANG PATH
+export LD_LIBRARY_PATH LANG
+
+BB_ENV_EXTRAWHITE="MACHINE DISTRO TB_BUILDER OE_DIR TMP_DIR DL_DIR MAKE_NUMBER_THREADS BB_NUMBER_THREADS"
+export BB_ENV_EXTRAWHITE BBPATH TB_BUILDER OE_DIR TMP_DIR DL_DIR MAKE_NUMBER_THREADS BB_NUMBER_THREADS
+
+mkdir -p ${LOG_DIR}
+
+### Functions
+
+# log message
+tb_log() {
+ local _dt=`date "+%F %X (%s)"`
+ echo "[${_dt}] $@" >&2
+ echo "[${_dt}] $@" >> $TB_LOG
+}
+
+# Do vmstat accounting
+tb_accounting_vmstat() {
+ exec vmstat -n "${ACCOUNTING_INTERVAL}" >> "${LOG_DIR}/A.vmstat.log"
+}
+
+# Do accounting by commands (df/ps)
+tb_accounting_cmd() {
+ local _dir=$1; shift
+ while true; do
+ [ -n "${DO_ACCOUNTING_DF}" ] && ( df -Pk "${_dir}" | awk 'END {OFS="\t"; print systime(), $4, $6;}' ) >> "${LOG_DIR}/A.df.log"
+ [ -n "${DO_ACCOUNTING_PS}" ] && ( ps ax | awk 'END {OFS="\t"; print systime(), NR-1;}' ) >> "${LOG_DIR}/A.ps.log"
+ sleep "${ACCOUNTING_INTERVAL}"
+ done
+}
+
+# Stop all remaining tasks
+tb_kill_all() {
+ for proc in `jobs | cut -c 2- | cut -d] -f1`; do
+ kill "%${proc}"
+ done
+}
+
+
+# update git tree
+tb_git_update() {
+ local _dir=$1
+
+ cd "${_dir}"
+ tb_log "Updating git tree ${_dir}"
+ git pull
+ cd "${TB_DIR}"
+}
+
+# checkout branch from git tree
+tb_git_branch() {
+ local _dir=$1
+ local _branch=$2
+
+ cd "${_dir}"
+ tb_log "Checking out branch ${_branch} from tree ${_dir}"
+ git checkout -f ${_branch}
+ cd "${TB_DIR}"
+}
+
+
+# Do build of image-set $bbimages for machine-set $MLIST
+# tb_build_machines "$MLIST" $bbimages
+tb_build_machines() {
+ local _MLIST=$1; shift
+ local _bbimages=$@
+ local _machine
+ local _rc
+ local _image
+ local _log
+
+ # Start accounting
+ trap "tb_kill_all" RETURN
+ [ -n "${DO_ACCOUNTING_VMSTAT}" ] && tb_accounting_vmstat &
+ [ -n "${DO_ACCOUNTING_DF}" -o -n "${DO_ACCOUNTING_PS}" ] && tb_accounting_cmd "${TMP_DIR}" &
+
+ for _machine in $_MLIST; do
+ [ "${_machine}" != "${_machine#!}" ] && continue
+
+ tb_log "${_machine} build started, images: ${_bbimages}"
+
+ export MACHINE=$_machine
+
+ case "${SEPARATE_BUILD}" in
+ [Yy][Ee][Ss])
+ # Build images separately
+ for _image in $_bbimages; do
+ tb_log "${_image} build started"
+ _log="${LOG_DIR}/B.${build}.${DISTRO}.${_machine}.${_image}.log"
+ [ -f "${_log}" ] && mv "${_log}" "${_log}.0"
+ bitbake ${BB_OPTS} ${_image} 2>&1 | tee "${_log}"
+ _rc=$?
+ tb_log "${_image} build finished. Exit code: ${_rc}."
+ done
+ ;;
+ *)
+ # Build all images with one bitbake invocation
+ _log="${LOG_DIR}/B.${build}.${DISTRO}.${_machine}.log"
+ [ -f "${_log}" ] && mv "${_log}" "${_log}.0"
+ bitbake ${BB_OPTS} ${_bbimages} 2>&1 | tee "${_log}"
+ _rc=$?
+ ;;
+ esac
+
+ tb_log "${_machine} build finished. Exit code:${_rc}."
+ done
+
+}
+
+
+### Main code
+
+# Parse stages
+unset DO_RUN_BUILDS DO_UPDATE_OE DO_UPDATE_BB
+for stage in $STAGES; do
+ [ "${stage}" != "${stage#!}" ] && continue
+ case ${stage} in
+ 'run_builds')
+ DO_RUN_BUILDS=y;;
+ 'update_oe')
+ DO_UPDATE_OE=y;;
+ 'update_bb')
+ DO_UPDATE_BB=y;;
+ *)
+ tb_log "Unknown stage ${stage}.";;
+ esac
+done
+
+# Parse accounting
+unset DO_ACCOUNTING_DF DO_ACCOUNTING_PS DO_ACCOUNTING_VMSTAT
+for count in $ACCOUNTING; do
+[ "${count}" != "${count#!}" ] && continue
+ case ${count} in
+ 'df')
+ DO_ACCOUNTING_DF=y;;
+ 'ps')
+ DO_ACCOUNTING_PS=y;;
+ 'vmstat')
+ DO_ACCOUNTING_VMSTAT=y;;
+ *)
+ tb_log "Unknown accounting type ${count}.";;
+ esac
+done
+
+# Parse cmdline
+unset ODISTRO OBUILD OMACHINE OIMAGES OTMPDIR DO_RUN_SHELL
+while getopts VhSB:D:M:T: opt; do
+ case $opt in
+ B) OBUILD="${OPTARG}";;
+ D) ODISTRO="${OPTARG}";;
+ M) OMACHINE="${OPTARG}";;
+ T) OTMPDIR="${OPTARG}";;
+ S) DO_RUN_SHELL=y;;
+ *)
+ tb_usage
+ exit 2
+ ;;
+ esac
+done
+
+shift $(($OPTIND - 1))
+OIMAGES=$@
+
+# Kill all tasks on exit
+trap "tb_kill_all" EXIT
+
+# Check for 'direct' mode
+if [ -n "${ODISTRO}" ]; then
+ DISTRO="${ODISTRO}"
+ MACHINE="${OMACHINE}"
+ TMP_DIR=${OTMPDIR:-${CLN_DIR}}
+ export DISTRO MACHINE TMP_DIR
+
+ if [ -n "${DO_RUN_SHELL}" ]; then
+ # Run shell if -s option was given
+ echo "Distro: $DISTRO"
+ echo "Machine: $MACHINE"
+ echo "TMPDIR: $TMP_DIR"
+ echo "Starting shell $TB_SHELL. Use 'exit' or ^D to quit"
+ [ -n "${TB_PS1}" ] && export PS1=${TB_PS1}
+ eval $TB_SHELL
+ else
+ tb_build_machines "${MACHINE}" ${OIMAGES}
+ fi
+
+ exit $rc
+fi
+
+# Do stages
+[ -n "${DO_UPDATE_BB}" ] && tb_git_update ${BB_DIR}
+[ -n "${DO_UPDATE_OE}" ] && tb_git_update ${OE_DIR}
+[ -z "${DO_RUN_BUILDS}" ] && exit 0
+
+# Use build from cmdline if any
+[ -n "$OBUILD" ] && BUILDS=$OBUILD
+
+# Auto-building mode
+tb_log "${PACKAGE} v${VERSION}"
+
+for build in $BUILDS; do
+ [ "${build}" != "${build#!}" ] && continue
+
+ eval BUILD="\$BUILD_${build}"
+ tb_log "= Started build '$build'"
+ [ -n "$BUILD" ] && tb_log "= $BUILD"
+
+ # change BB branch
+ eval BB_BRANCH="\$BB_BRANCH_${build}"
+ if [ -n "${BB_BRANCH}" ]; then
+ tb_git_branch "$BB_DIR" "$BB_BRANCH"
+ fi
+
+ # change OE branch
+ eval OE_BRANCH="\$OE_BRANCH_${build}"
+ if [ -n "${OE_BRANCH}" ]; then
+ tb_git_branch "$OE_DIR" "$OE_BRANCH"
+ fi
+
+ # parse tasks
+ eval TASKS="\$TASKS_${build}"
+ unset DO_BUILD_CLEAN DO_BUILD_INCREMENTAL
+ for task in $TASKS; do
+ [ "${task}" != "${task#!}" ] && continue
+ case ${task} in
+ 'build_clean')
+ DO_BUILD_CLEAN=y;;
+ 'build_incremental')
+ DO_BUILD_INCREMENTAL=y;;
+ *)
+ tb_log "Unknown task ${task}.";;
+ esac
+ done
+
+ # distros
+ eval DISTROS="\$DISTROS_${build}"
+ for distro in $DISTROS; do
+ [ "${distro}" != "${distro#!}" ] && continue
+
+ export DISTRO=$distro
+
+ # Get build sets
+ eval BSLIST="\$BUILD_SETS_${build}"
+
+ # Empty $BUILD_SETS is exception - we should use $MACHINES
+ # to build every machine in separate TMPDIRs
+ if [ -z "${BSLIST}" ]; then
+ # Use every machine as separate build set
+ eval MACHINES="\$MACHINES_${build}"
+ for machine in $MACHINES; do
+ [ "${machine}" != "${machine#!}" ] && continue
+ eval "MACHINE_${machine}=${machine}"
+ BSLIST="${BSLIST} ${machine}"
+ done
+ fi
+
+ tb_log "== Processing distro ${DISTRO}, build set: ${BSLIST}"
+
+ for bset in $BSLIST; do
+ [ "${bset}" != "${bset#!}" ] && continue
+
+ eval MLIST="\$MACHINES_${build}_${bset}"
+ [ -z "$MLIST" ] && eval MLIST="\$MACHINES_${build}"
+
+ eval ILIST="\$IMAGES_${build}"
+
+ tb_log "=== Processing set ${bset}, machines: ${MLIST}"
+
+ INC_TMP_DIR="${INC_DIR}/${build}/${distro}/${bset}"
+
+
+ # collect images to build in $bbimages
+ bbimages=''
+ for image in $ILIST; do
+ [ "${image}" != "${image#!}" ] && continue
+ bbimages="${bbimages} ${image}"
+ done
+
+ # If there is no previous incremental builddir and we are requested
+ # to do both incremental and clean builds
+ # then skip clean build and do only incremental because it will be clean build then :)
+ if [ "${DO_BUILD_INCREMENTAL}" = 'y' \
+ -a "${DO_BUILD_CLEAN}" = 'y' \
+ -a ! -d "${INC_TMP_DIR}/work" \
+ ]; then
+ tb_log "Will do incremental build instead of clean to populate TMPDIR"
+ DO_BUILD_CLEAN='skip-once'
+ fi
+
+ ## Do clean builds
+ case "${DO_BUILD_CLEAN}" in
+ 'y')
+ export TMP_DIR="${CLN_DIR}"
+ mkdir -p "${TMP_DIR}"
+
+ tb_log "=== Cleaning ${TMP_DIR}"
+ #rm -rf ${TMP_DIR}/*
+ find ${TMP_DIR}/* -delete
+
+ tb_log "=== Starting clean builds for machines {$MLIST}"
+ tb_build_machines "$MLIST" $bbimages
+
+ tb_log "=== Cleaning ${TMP_DIR} again"
+ find ${TMP_DIR}/* -delete
+ ;;
+ 'skip-once')
+ # Re-enable skipped clean build
+ DO_BUILD_CLEAN=y
+ ;;
+ esac
+
+ ## Do incremental build
+ case "${DO_BUILD_INCREMENTAL}" in
+ 'y')
+ # Switch tmpdir to archive
+ export TMP_DIR="${INC_TMP_DIR}"
+ mkdir -p "${TMP_DIR}"
+
+ tb_log "=== Starting incremental builds for machine-set {$MLIST}"
+ tb_build_machines "$MLIST" $bbimages
+ ;;
+ esac
+ done
+ done
+done
+
+tb_log "All done."
+
+exit 0
+
--- /dev/null
+# Testbuilder config
+
+### NOTES
+# 0. For first time setup look into end of file for "Misc dirs"
+# 1. All names in BUILDS and BUILD_SETS variables should
+# consists only of alphanumeric characters and underscores,
+# and beginning with an alphabetic character or an underscore.
+# i.e. [a-zA-Z_][a-zA-Z0-9_]*
+# My recommendation is not to use underscore as well
+# 2. You can prefix any word in following 'list' variables
+# with '!' do disable it.
+# 3. All lists items will be processed in order they appears in list
+
+### BUILDS: Define here your build profiles
+#
+BUILDS="testing master my"
+
+## BUILD: Just some text comment to show in build log
+BUILD_testing="Regular build of testing branch"
+## OE_BRANCH: OE branch to use for this build
+OE_BRANCH_testing="testing-next"
+## BB_BRANCH: bitbake branch to use for this build
+BB_BRANCH_testing="1.10"
+## DISTROS: OE distros
+DISTROS_testing="angstrom-2008.1 angstrom-2010.x"
+## IMAGES: OE images
+IMAGES_testing="console-image x11-image opie-image"
+## BUILD_SETS: build all machines in set within same TMPDIR
+BUILD_SETS_testing="armv5 mips32 mips64 ppc sh4 i686"
+## MACHINES: OE machines per build set
+MACHINES_testing_armv5="qemuarm"
+MACHINES_testing_mips32="qemumips qemumipsel"
+MACHINES_testing_mips64="qemumips64"
+MACHINES_testing_ppc="qemuppc"
+MACHINES_testing_sh4="qemush4"
+MACHINES_testing_i686="qemux86"
+## TASKS: What to do exactly
+# build_clean - do clean builds
+# build_incremental - do incremental builds
+TASKS_testing="!build_clean build_incremental"
+
+# Other sample build profile
+BUILD_master="Build from master branch"
+OE_BRANCH_master="master"
+BB_BRANCH_master="master"
+# Note that angstrom-2010.x is disabled below ('!' before)
+DISTROS_master="!angstrom-2010.x minimal"
+IMAGES_master="console-image"
+BUILD_SETS_master="zauruses qemux86"
+MACHINES_master_zauruses="collie tosa akita"
+MACHINES_master_qemu="qemux86"
+TASKS_master="build_clean !build_incremental"
+
+# Sample build from local branch
+BUILD_my="Build my local branch"
+OE_BRANCH_my="my/work"
+BB_BRANCH_my="master"
+DISTROS_my="minimal"
+IMAGES_my="console-image !my-own-image"
+# Note BUILD_SETS_my absence
+# Every machine will be built in own TMPDIR
+MACHINES_my="tosa akita spitz"
+TASKS_my="build_clean !build_incremental"
+
+
+### Common options
+
+## STAGES: what TB should do
+# update_bb - update bitbake git tree
+# update_oe - update OE git tree
+# run_builds - run builds
+STAGES="update_bb update_oe !run_builds"
+
+## ACCOUNTING: account some useful things
+# df - log free disk space
+# ps - log processes count
+# vmstat - log vm statistics
+ACCOUNTING="!df !ps !vmstat"
+# Accounting interval (seconds)
+ACCOUNTING_INTERVAL="10"
+
+# Build every image by separate bitbake invocation (lower peak disk usage)
+#SEPARATE_BUILD="yes"
+# Build all images by single bitbake invocation (slightly faster)
+SEPARATE_BUILD="no"
+
+# Number of parallel bitbake threads
+BB_NUMBER_THREADS=2
+# Number of parallel make threads inside one bitbake thread (-j)
+MAKE_NUMBER_THREADS=2
+
+# Bitbake cmdline options
+#BB_OPTS='-k'
+
+# Direct build invocation shell variables (-S option)
+# PS1 for shell mode
+TB_PS1="[TB] "
+# Use bash with --norc to disable overriding PS1
+TB_SHELL='/bin/bash --norc'
+# Use $SHELL variable or fallback to /bin/sh
+#TB_SHELL=${SHELL:-'/bin/sh'}
+
+# Name for oestats-client
+TB_BUILDER="You_should_change_this"
+
+# Misc dirs
+TB_BASE=`pwd`
+BB_DIR="${TB_BASE}/bitbake"
+OE_DIR="${TB_BASE}/openembedded"
+DL_DIR="${TB_BASE}/sources"
+BLD_DIR="${TB_BASE}/build"
+LOG_DIR="${TB_BASE}/logs"
+CLN_DIR="/var/tmp/tb-clean-build" # TMPDIR for clean builds
+INC_DIR="${BLD_DIR}" # Top dir for incremental builds archive
+
git.openpandora.org / openembedded.git / commitdiff