2015-04-24 build Vyos lithium from source

This document describes one mechanism to build a bootable Vyos iso from the Vyos github source code repository.

Start with a Centos 6 workstation with the virtualization package and tools. Really, anything that can do KVM virtualization should work.

Download debian-6.0.7-amd64-netinst.iso (sha256sum is 28b2070d54b73d24f6d66d4223a0ca5e975e2555478c89ee8b48cbb4b89ee88d) or debian-6.0.7-i386-netinst.iso (sha256sum is 26a8c9de527ad59d473e10dffc8883c0dc095cf95ed3439bfdae952276dbc0df) from http://www.debian.org/releases/squeeze/debian-installer/. Use that in virt-manager to create a VM.

type linux, version debian squeeze
2G memory, 20G disk
advanced, type KVM, arch x86_64
graphic install
install std system utilities, and ssh server

On your workstation:

# define build machine
target=root@host46
targetdir=/home/carl/vyos/updates

# copy ssh key
ssh-copy-id -i $target

# copy the script to your build machine
scp build.vyos $target:/tmp

# run phase 1 to finish debian setup
ssh $target 'cd /home/carl/vyos; bash /tmp/build.vyos phase1'

# you could save the vm disk at this point

# copy patches
ssh $target mkdir -p $targetdir
for p in *.git.patch; do
    i=$(basename $p .git.patch)
    logger "copy modified $i"
    scp -rpq $p $target:/$targetdir
done
ssh $target ls -al $targetdir

# run phase 2 to build iso from source
ssh $target 'cd /home/carl/vyos; bash /tmp/build.vyos phase2'

# fetch the iso and build log
f=$(ssh $target "echo /home/carl/vyos/build-iso/livecd/VyOS-livecd*.iso")
b=$(basename "$f")
scp $target:$f .
scp $target:/tmp/phase2.log.txt $b.log.txt

build.vyos

function logger() {
    d=$(date)
    echo " "
    echo "*** $d $1"
}

function phase1 {
    logger "import debian keys"
    apt-get install debian-archive-keyring

    logger "import vyos keys"
    wget http://vyos.net/so3group_maintainers.key
    gpg --import so3group_maintainers.key
    gpg --list-keys
    apt-key add ./so3group_maintainers.key

    logger "install squashfs from backports"
    echo "deb http://backports.debian.org/debian-backports squeeze-backports main" >>/etc/apt/sources.list
    apt-get update
    apt-get -y -t squeeze-backports install squashfs-tools

    logger "install more build dependencies"
    apt-get -y install git autoconf automake dpkg-dev live-helper syslinux genisoimage

    logger "install undocumented dependencies, debclean needed by submod-clean"
    apt-get -y install devscripts

    logger "install undocumented dependencies found by submod-clean"
    apt-get -y install autogen bison cdbs flex gawk gcc-4.4-multilib \
        hardening-wrapper indent iptables-dev libapt-pkg-dev libatm1-dev \
        libattr1-dev libboost-filesystem1.42-dev libcap-dev \
        libc-ares-dev libcurl4-openssl-dev \
        libdaemon-dev libdb4.8-dev libdb-dev libdevmapper-dev libedit-dev \
        libexpat1-dev libfreetype6-dev libglib2.0-dev libgmp3-dev libkrb5-dev \
        libldap2-dev libncurses5-dev libnetfilter-conntrack-dev \
        libnfnetlink-dev libpam0g-dev libpcap0.8-dev libpci-dev \
        libperl-dev libpgm-dev libpopt-dev libreadline-dev libsensors4-dev \
        libsnmp-dev libssl-dev libtool libusb-dev \
        libwrap0-dev libxml2-dev libzmq-dev lynx pkg-config python-all-dev \
        python-setuptools quilt ruby uuid-dev xfonts-unifont zlib1g-dev

    logger "install undocumented dependencies found by build attempts"
    apt-get -y install kernel-package dkms doxygen libcunit1-dev libdumbnet-dev \
        libfuse-dev libgtk2.0-dev libgtkmm-2.4-dev libicu-dev libnotify-dev \
        libx11-dev libxinerama-dev libxss-dev libxtst-dev dh-autoreconf \
        xmlto mscgen graphviz python-pygments xmlstarlet asciidoc source-highlight

    logger "install more dependencies found by lithium build attempts"
    apt-get -y install libcluster-glue-dev cluster-glue-dev libbz2-dev swig \
        libgnutls-dev libopenhpi-dev libopenipmi-dev liblzo2-dev \
        libpkcs11-helper1-dev libsqlite3-dev

    logger "look for pending upgrades"
    apt-get upgrade --dry-run
}

function phase2 {
    arch=$(dpkg --print-architecture)
    flavor=amd64-vyos
    [ $arch == "i386" ] && flavor=586-vyos

    logger "install more dependencies found by lithium build attempts"
    apt-get -y install libsysfs-dev libpcsclite-dev

    logger "look for pending upgrades for arch $arch"
    apt-get upgrade --dry-run

    logger "setup git clone, building flavor $flavor"
    git clone https://github.com/vyos/build-iso.git
    cd build-iso
    #git branch $branch --track origin/$branch
    git checkout $branch
    git submodule init
    git submodule update
    for i in pkgs/*; do
        if [ -d "$i/.git" ]; then
            logger "select branch $branch for $i"
            pushd "$i"
            p=$(basename "$i")
            #git branch $branch --track origin/$branch
            git checkout $branch
            if [ $? -eq 1 ]; then
                git checkout master
            fi
            if [ -f ../../../updates/$p.git.patch ]; then
                #git am --ignore-whitespace ../../../updates/$p.git.patch
                patch -p1 <../../../updates/$p.git.patch
            fi
            popd
        fi
    done

    logger "show active branches"
    for i in pkgs/*; do
        if [ -d "$i/.git" ]; then
            (cd $i; b=$(git branch | grep '^\*'); echo $i "$b")
        fi
    done

    logger "fetch source from vyos"
    git pull
    git submodule init
    tools/submod-git pull
    (
        logger "linux-image missing debian/control file"
        cd pkgs/linux-image
        debian/bin/build-flavour.sh $flavor
    )
    (
        logger "vyos-open-vm-tools missing config/guess file, source too deep"
        cd pkgs/vyos-open-vm-tools
        mv -f open-vm-tools/* .
        automake --add-missing
    )

    logger "cleanup all the subpackages"
    tools/submod-clean -d

    logger "rebuild some packages needed to build the rest"
    for i in pkgs/{net-snmp,libmnl,ppp}; do
        p=$(basename $i)
        if [ -d $i/.git ]; then
            logger "building source package $p"
            tools/submod-mk $p >$i/vyos.prebuild.log 2>&1
        fi
    done

    logger "install some rebuilt packages"
    PKGS="
        pkgs/libsnmp*.deb
        pkgs/libmnl*.deb
        pkgs/vyatta-ppp*.deb
    "
    dpkg -i $PKGS

    logger "rebuild more packages needed to build conntrack-tools"
    for i in pkgs/libnl3 pkgs/libnfnetlink pkgs/libnetfilter-{conntrack,cthelper,cttimeout,queue}; do
        p=$(basename $i)
        if [ -d $i/.git ]; then
            logger "building source package $p"
            tools/submod-mk $p >$i/vyos.prebuild.log 2>&1
        fi
    done

    logger "install more rebuilt packages"
    PKGS="
        pkgs/libnl*.deb pkgs/libnfnetlink*.deb pkgs/libnetfilter*.deb
    "
    dpkg -i $PKGS

    logger "again cleanup all the subpackages"
    tools/submod-clean -d

    (
        logger "updated strongswan 5.2 was setup on a newer system, regenerate aclocal and friends"
        cd pkgs/vyatta-strongswan
        # aclocal
        # automake --add-missing
        autoreconf
    )

    logger "rebuild all packages from source"
    for i in pkgs/*; do
        p=$(basename $i)
        if [ -d $i/.git ]; then
            logger "building source package $p"
            tools/submod-mk $p >$i/vyos.build.log 2>&1
            cat $i/vyos.build.log
        fi
    done

    logger "build the new iso"
    export PATH=/sbin:/usr/sbin:$PATH
    autoreconf -i
    ./configure
    cat >>livecd/config.vyatta/chroot_local-packageslists/vyatta-full.list <<EOF
libstrongswan
strongswan
strongswan-tools
EOF
    make iso

    (
        logger "find packages that were built from source but the iso used the version from dev.packages.vyos.net"
        cd livecd
        fn=config.vyatta/chroot_local-packageslists/vyatta-full.list
        for p in ../pkgs/*.deb; do
            pp=${p%%_*}
            pkg=${pp#../pkgs/}
            c=$(find cache/packages_chroot -name "${pkg}_*.deb" | wc -l)
            if [ $c -gt 0 ]; then
                a=$(basename ../pkgs/${pkg}_*.deb)
                b=$(basename cache/packages_chroot/${pkg}_*.deb)
                [ "$a" != "$b" ] && echo "$a $b"
                echo "$pkg" >>$fn
            fi
        done

        logger "remove duplicate package pppoe from rp-pppoe, which conflicts with vyatta-ppp"
        sed -i -e '/^pppoe$/d' $fn
    )

    logger "rebuild the new iso"
    rm -rf livecd/{auto,binary*,cache,chroot,config,*.iso,.stage}
    make iso

    logger "show packages still pulled from dev.packages.vyos.net"
    ls -al livecd/cache/packages_chroot

    logger "done, iso in $(pwd)/livecd"
    ls -al $(pwd)/livecd/*iso
}


case "$1" in
    phase*)
        branch=lithium
        $1 2>&1 | tee /tmp/$1.log.txt
        ;;
esac