diff options
Diffstat (limited to 'nix/default.nix')
| -rw-r--r-- | nix/default.nix | 321 |
1 files changed, 236 insertions, 85 deletions
diff --git a/nix/default.nix b/nix/default.nix index 228047f3d..f021146ff 100644 --- a/nix/default.nix +++ b/nix/default.nix @@ -1,48 +1,173 @@ +# -*- mode: nix; indent-tabs-mode: nil -*- { nixpkgs ? <nixpkgs>, lvm2Src, release ? false, rawhide32 ? "" , rawhide64 ? "" , + fc20_32_updates ? "", fc20_64_updates ? "", fc19_32_updates ? "", fc19_64_updates ? "", fc18_32_updates ? "", fc18_64_updates ? "", - lvm2Nix ? lvm2Src, T ? "" }: + T ? "", ENV ? "", timeout ? 60 }: let pkgs = import nixpkgs {}; - mkVM = { VM, extras ? [], diskFun, kernel }: - VM rec { - inherit kernel; - name = "lvm2"; - fullName = "LVM2"; - src = jobs.tarball; + lib = pkgs.lib; + + mkTest = args: pkgs.stdenv.mkDerivation rec { + name = "lvm2-test-${(args.diskFun {}).name}"; + + builder = pkgs.writeScript "lvm2-collect-results" '' + #!${pkgs.bash}/bin/bash + . $stdenv/setup + mkdir -p $out/test-results + for i in ${lib.concatStringsSep " " buildInputs}; do + cat $i/test-results/list >> $out/test-results/list + cp $i/test-results'/'*.txt $out/test-results/ || true + done + mkdir -p $out/nix-support + grep '\<failed\>' $out/test-results/list && touch $out/nix-support/failed || true + ''; + + buildInputs = map (x: runTest (args // { flavour = x; })) + [ "ndev-vanilla" "ndev-lvmetad" "ndev-cluster" "udev-vanilla" "udev-lvmetad" "udev-cluster" ]; + }; + + runTest = { build, diskFun, extras ? [], kernel, vmtools, flavour, ... }: pkgs.stdenv.mkDerivation rec { diskImage = diskFun { extraPackages = extras; }; - memSize = 768; - - # fc16 lcov is broken and el6 has none... be creative - prepareImagePhase = '' - rpm -Uv ${pkgs.fetchurl { - url = "http://archives.fedoraproject.org/pub/archive/fedora/linux/updates/16/i386/lcov-1.9-2.fc16.noarch.rpm"; - sha256 = "0ycdh5mb7p5ll76mqk0p6gpnjskvxxgh3a3bfr1crh94nvpwhp4z"; }} - dmesg -n 1 # avoid spilling dmesg into the main log, we capture it in harness + name = "lvm2-test-${diskImage.name}-${flavour}"; + + # this is the builder that runs in the guest + origBuilder = pkgs.writeScript "vm-test-guest" '' + #!/bin/bash + export PATH=/usr/bin:/bin:/usr/sbin:/sbin + + # we always run in a fresh image, so need to install everything again + ls ${build}/rpms/*/*.rpm | grep -v sysvinit | xargs rpm -Uv # */ + rpm -Uv ${pkgs.fetchurl { + url = "http://archives.fedoraproject.org/pub/archive/fedora/linux/updates/16/i386/lcov-1.9-2.fc16.noarch.rpm"; + sha256 = "0ycdh5mb7p5ll76mqk0p6gpnjskvxxgh3a3bfr1crh94nvpwhp4z"; }} + + mkdir -p /xchg/results + touch /xchg/booted + + dmsetup targets + + export LVM_TEST_BACKING_DEVICE=/dev/sdb + + watch= + if echo ${flavour} | grep -q udev; then + (/usr/lib/systemd/systemd-udevd || /usr/lib/udev/udevd || /sbin/udevd || \ + find / -xdev -name \*udevd) >> /xchg/udevd.log 2>&1 & + watch="--watch /xchg/udevd.log" + fi + + export ${ENV} + lvm2-testsuite --batch --outdir /xchg/results --continue \ + --timeout ${toString timeout} --fatal-timeouts --heartbeat /xchg/heartbeat \ + --flavours ${flavour} $watch --kmsg ${if lib.eqStrings T "" then "" else "--only ${T}"} + + # TODO: coverage reports + # make lcov || true + # cp -R lcov_reports $out/coverage && \ + # echo "report coverage $out/coverage" >> $out/nix-support/hydra-build-products || \ + # true # not really fatal, although kinda disappointing ''; - postBuild = '' - mkdir -p $out/nix-support - cd `cat /tmp/build-location` - mv test/results/list test/results/list-rpm - ls /tmp/rpmout/RPMS/*/*.rpm | grep -v sysvinit | xargs rpm -Uvh # */ - (/usr/lib/systemd/systemd-udevd || /usr/lib/udev/udevd || /sbin/udevd || find / -xdev -name \*udevd) & - make check_system QUIET=1 T=${T} || touch $out/nix-support/failed - mv test/results/list test/results/list-system - cat test/results/list-* > test/results/list - cp -R test/results $out/test-results && \ - echo "report tests $out/test-results" >> $out/nix-support/hydra-build-products || \ - true - make lcov || true - cp -R lcov_reports $out/coverage && \ - echo "report coverage $out/coverage" >> $out/nix-support/hydra-build-products || \ - true # not really fatal, although kinda disappointing + buildInputs = [ pkgs.coreutils pkgs.bash pkgs.utillinux ]; + + # make a qcow copy of the main image + preVM = '' + diskImage=$(pwd)/disk-image.qcow2 + origImage=${diskImage} + if test -d "$origImage"; then origImage="$origImage/disk-image.qcow2"; fi + ${vmtools.qemu}/bin/qemu-img create -b "$origImage" -f qcow2 $diskImage + ''; + + builder = pkgs.writeScript "vm-test" '' + #!${pkgs.bash}/bin/bash + . $stdenv/setup + + export QEMU_OPTS="-drive file=/dev/shm/testdisk.img,if=ide -m 256M" + export QEMU_DRIVE_OPTS=",if=ide" + export KERNEL_OPTS="log_buf_len=131072 loglevel=1" + export mountDisk=1 + + mkdir -p $out/test-results $out/nix-support + touch $out/nix-support/failed + + monitor() { + set +e + counter=0 + rm -f j.current j.last t.current t.last + while true; do + if ! test -f pid; then + counter=0 + sleep 60 + continue + fi + + cat xchg/results/journal > j.current 2> /dev/null + cat xchg/heartbeat > hb.current 2> /dev/null + if diff j.current j.last >& /dev/null; then + counter=$(($counter + 1)); + else + counter=0 + fi + if test $counter -eq 10 || test $(cat hb.current | wc -c) -eq $(cat hb.last | wc -c); then + echo + echo "VM got stuck; heartbeat: $(cat hb.current | wc -c) $(cat hb.last | wc -c), counter = $counter." + echo "last journal entry: $(tail -n 1 j.current), previously $(tail -n 1 j.last)" + kill -- -$(cat pid) + fi + sleep 60 + mv j.current j.last >& /dev/null + mv hb.current hb.last >& /dev/null + done + } + + monitor & + + for i in `seq 1 20`; do # we allow up to 20 VM restarts + rm -f xchg/booted + ${vmtools.qemu}/bin/qemu-img create -f qcow2 /dev/shm/testdisk.img 4G + setsid bash -e ${vmtools.vmRunCommand (vmtools.qemuCommandLinux kernel)} & + pid=$! + + # give the VM some time to get up and running + slept=0 + while test $slept -le 180 && test ! -e xchg/booted; do + sleep 10 + slept=$(($slept + 10)) + done + echo $pid > pid # monitor go + wait $pid || true + rm -f pid # disarm the monitor process + + # if we have any new results, stash them + mv xchg/results'/'*.txt $out/test-results/ || true + + if test -n "$(cat xchg/in-vm-exit)"; then # the VM is done + test 0 -eq "$(cat xchg/in-vm-exit)" && rm -f $out/nix-support/failed + break + fi + + sleep 10 # wait for the VM to clean up before starting up a new one + done + + cat xchg/results/list > $out/test-results/list || true ''; + }; + + mkBuild = { VM, extras ? [], diskFun, ... }: + VM rec { + name = "lvm2-build-${diskImage.name}"; + fullName = "lvm2-build-${diskImage.name}"; + + src = jobs.tarball; + diskImage = diskFun { extraPackages = extras; }; + memSize = 512; + checkPhase = ":"; postInstall = '' - for i in $out/rpms/*/*.rpm; do + mkdir -p $out/nix-support + for i in $out/rpms/*/*.rpm; do # */ if echo $i | grep -vq "\.src\.rpm$"; then echo "file rpm $i" >> $out/nix-support/hydra-build-products else @@ -53,16 +178,17 @@ let }; rootmods = [ "virtio_pci" "virtio_blk" "virtio_balloon" "ext4" "unix" - "cifs" "virtio_net" "unix" "hmac" "md4" "ecb" "des_generic" "sha256" ]; + "cifs" "virtio_net" "unix" "hmac" "md4" "ecb" "des_generic" "sha256" + "ata_piix" "sd_mod" ]; centos_url = ver: arch: if ver == "6.5" then "http://ftp.fi.muni.cz/pub/linux/centos/${ver}/os/${arch}/" else "http://vault.centos.org/${ver}/os/${arch}/"; - fedora_url = ver: arch: if pkgs.lib.eqStrings ver "rawhide" || pkgs.lib.eqStrings ver "19" + fedora_url = ver: arch: if lib.eqStrings ver "rawhide" || lib.eqStrings ver "19" then "ftp://ftp.fi.muni.cz/pub/linux/fedora/linux/development/${ver}/${arch}/os/" else "mirror://fedora/linux/releases/${ver}/Everything/${arch}/os/"; fedora_update_url = ver: arch: "mirror://fedora/linux/updates/${ver}/${arch}"; - extra_distros = with pkgs.lib; let + extra_distros = with lib; let centos = { version, sha, arch }: { name = "centos-${version}-${arch}"; fullName = "CentOS ${version} (${arch})"; @@ -105,6 +231,8 @@ let in { rawhidex86_64 = rawhide "rawhide" "x86_64" rawhide64; rawhidei386 = rawhide "rawhide" "i386" rawhide32; + fedora20ux86_64 = update "20" "x86_64" fc20_64_updates pkgs.vmTools.rpmDistros.fedora20x86_64; + fedora20ui386 = update "20" "i386" fc20_32_updates pkgs.vmTools.rpmDistros.fedora20i386; fedora19ux86_64 = update "19" "x86_64" fc19_64_updates pkgs.vmTools.rpmDistros.fedora19x86_64; fedora19ui386 = update "19" "i386" fc19_32_updates pkgs.vmTools.rpmDistros.fedora19i386; fedora18ux86_64 = update "18" "x86_64" fc18_64_updates pkgs.vmTools.rpmDistros.fedora18x86_64; @@ -141,10 +269,11 @@ let }; }; - vm = pkgs: xmods: with pkgs.lib; rec { + vm = { pkgs, xmods, dmmods ? false }: with lib; rec { tools = import "${nixpkgs}/pkgs/build-support/vm/default.nix" { inherit pkgs; rootModules = rootmods ++ xmods ++ - [ "loop" "dm_mod" "dm_snapshot" "dm_mirror" "dm_zero" "dm_raid" "dm_thin_pool" ]; }; + (if dmmods then [ "loop" "dm_mod" "dm_snapshot" "dm_mirror" "dm_zero" "dm_raid" "dm_thin_pool" ] + else []); }; release = import "${nixpkgs}/pkgs/build-support/release/default.nix" { pkgs = pkgs // { vmTools = tools; }; }; imgs = tools.diskImageFuns // @@ -156,6 +285,7 @@ let extra_rpms = rec { common = [ "libselinux-devel" "libsepol-devel" "ncurses-devel" "readline-devel" "corosynclib-devel" + "valgrind" "valgrind-devel" "strace" "redhat-rpm-config" # needed for rpmbuild of lvm "which" "e2fsprogs" # needed for fsadm "perl-GD" # for lcov @@ -166,25 +296,67 @@ let fedora16 = [ "clusterlib-devel" "openaislib-devel" "cman" "systemd-devel" "libudev-devel" ]; fedora17 = [ "dlm-devel" "corosynclib-devel" "device-mapper-persistent-data" "dlm" "systemd-devel" "perl-Digest-MD5" "libudev-devel" ]; + fedora18 = [ "dlm-devel" "corosynclib-devel" "device-mapper-persistent-data" "dlm" "systemd-devel" "perl-Digest-MD5" ]; fedora18u = fedora18; + fedora19 = [ "dlm-devel" "dlm" "corosynclib-devel" "perl-Digest-MD5" "systemd-devel" "procps-ng" ]; fedora19u = fedora19; - rawhide = fedora19; + + fedora20 = fedora19; + fedora20u = fedora20; + + rawhide = fedora20; }; - mkRPM = { arch, image }: with pkgs.lib; - let use = vm (if eqStrings arch "i386" then pkgs.pkgsi686Linux else pkgs) - (if image == "centos64" || image == "centos65" then [] else [ "9p" "9pnet_virtio" ]); - in mkVM { + wrapper = fun: { arch, image, build ? {}, istest ? false }: with lib; + let use = vm { pkgs = if eqStrings arch "i386" then pkgs.pkgsi686Linux else pkgs; + xmods = if istest && (image == "centos64" || image == "centos65") + then [] else [ "9p" "9pnet_virtio" ]; + dmmods = istest; }; + in fun { + inherit build istest; VM = use.rpmbuild; diskFun = builtins.getAttr "${image}${arch}" use.imgs; extras = extra_rpms.common ++ builtins.getAttr image extra_rpms; + vmtools = use.tools; kernel = use.tools.makeKernelFromRPMDist (builtins.getAttr "${image}${arch}" use.rpmdistros); }; - jobs = rec { + configs = { + fc20_x86_64 = { arch = "x86_64"; image = "fedora20"; }; + fc20_i386 = { arch = "i386" ; image = "fedora20"; }; + fc19_x86_64 = { arch = "x86_64"; image = "fedora19"; }; + fc19_i386 = { arch = "i386" ; image = "fedora19"; }; + fc18_x86_64 = { arch = "x86_64"; image = "fedora18"; }; + fc18_i386 = { arch = "i386" ; image = "fedora18"; }; + fc17_x86_64 = { arch = "x86_64"; image = "fedora17"; }; + fc17_i386 = { arch = "i386" ; image = "fedora17"; }; + fc16_x86_64 = { arch = "x86_64"; image = "fedora16"; }; + fc16_i386 = { arch = "i386" ; image = "fedora16"; }; + + fc18u_x86_64 = { arch = "x86_64"; image = "fedora18u"; }; + fc18u_i386 = { arch = "i386"; image = "fedora18u"; }; + fc19u_x86_64 = { arch = "x86_64"; image = "fedora19u"; }; + fc19u_i386 = { arch = "i386"; image = "fedora19u"; }; + + #centos63_i386 = { arch = "i386" ; image = "centos63"; }; + #centos63_x86_64 = { arch = "x86_64" ; image = "centos63"; }; + centos64_i386 = { arch = "i386" ; image = "centos64"; }; + centos64_x86_64 = { arch = "x86_64" ; image = "centos64"; }; + centos65_i386 = { arch = "i386" ; image = "centos65"; }; + centos65_x86_64 = { arch = "x86_64" ; image = "centos65"; }; + + rawhide_i386 = { arch = "i386" ; image = "rawhide"; }; + rawhide_x86_64 = { arch = "x86_64" ; image = "rawhide"; }; + }; + + rpms = lib.mapAttrs (n: v: wrapper mkBuild v) configs; + tests = let make = n: v: wrapper mkTest (v // { build = builtins.getAttr n rpms; istest = true; }); + in lib.mapAttrs make configs; + + jobs = tests // { tarball = pkgs.releaseTools.sourceTarball rec { name = "lvm2-tarball"; versionSuffix = if lvm2Src ? revCount @@ -193,31 +365,34 @@ let src = lvm2Src; autoconfPhase = ":"; distPhase = '' - set -x make distclean + version=`cat VERSION | cut "-d(" -f1`${versionSuffix} version_dm=`cat VERSION_DM | cut "-d-" -f1`${versionSuffix} - sed -e s,-git,${versionSuffix}, -i VERSION VERSION_DM - rm -rf spec; cp -R ${lvm2Nix}/spec/* . + chmod u+w * - (echo "%define enable_profiling 1"; - echo "%define check_commands \\"; - echo "make lcov-reset \\"; - echo "dmsetup targets\\"; - echo "mkdir -p \$out/nix-support \\"; - echo "make check QUIET=1 T=${T} || touch \$out/nix-support/failed \\" - echo "pwd > /tmp/build-location \\" - echo "touch rpm-no-clean") >> source.inc + + # set up versions + sed -e s,-git,${versionSuffix}, -i VERSION VERSION_DM sed -e "s,\(device_mapper_version\) [0-9.]*$,\1 $version_dm," \ -e "s,^\(Version:[^0-9%]*\)[0-9.]*$,\1 $version," \ -e "s,^\(Release:[^0-9%]*\)[0-9.]\+,\1 0.HYDRA," \ - -e "s:%with clvmd corosync:%with clvmd corosync,singlenode:" \ - -i source.inc - sed -e '/^%changelog/,$d' \ - -i lvm2.spec - echo "%changelog" >> lvm2.spec; - echo "* `date +"%a %b %d %Y"` Petr Rockai <prockai@redhat.com> - $version" >> lvm2.spec; - echo "- AUTOMATED BUILD BY Hydra" >> lvm2.spec + -i spec/source.inc + + # tweak RPM configuration + echo "%define enable_profiling 1" >> spec/source.inc + echo "%define enable_testsuite 1" >> spec/source.inc + sed -e "s:%with clvmd corosync:%with clvmd corosync,singlenode:" -i spec/source.inc + + # synthesize a changelog + sed -e '/^%changelog/,$d' -i spec/lvm2.spec + (echo "%changelog"; + echo "* `date +"%a %b %d %Y"` Petr Rockai <prockai@redhat.com> - $version"; + echo "- AUTOMATED BUILD BY Hydra") >> spec/lvm2.spec + + mv spec/* . && rmdir spec # */ # RPM needs the spec file in the source root + + # make a tarball mkdir ../LVM2.$version mv * ../LVM2.$version ensureDir $out/tarballs @@ -225,29 +400,5 @@ let tar cvzf $out/tarballs/LVM2.$version.tgz LVM2.$version ''; }; - - fc19_x86_64 = mkRPM { arch = "x86_64"; image = "fedora19"; }; - fc19_i386 = mkRPM { arch = "i386" ; image = "fedora19"; }; - fc18_x86_64 = mkRPM { arch = "x86_64"; image = "fedora18"; }; - fc18_i386 = mkRPM { arch = "i386" ; image = "fedora18"; }; - fc17_x86_64 = mkRPM { arch = "x86_64"; image = "fedora17"; }; - fc17_i386 = mkRPM { arch = "i386" ; image = "fedora17"; }; - fc16_x86_64 = mkRPM { arch = "x86_64"; image = "fedora16"; }; - fc16_i386 = mkRPM { arch = "i386" ; image = "fedora16"; }; - - fc18u_x86_64 = mkRPM { arch = "x86_64"; image = "fedora18u"; }; - fc18u_i386 = mkRPM { arch = "i386"; image = "fedora18u"; }; - fc19u_x86_64 = mkRPM { arch = "x86_64"; image = "fedora19u"; }; - fc19u_i386 = mkRPM { arch = "i386"; image = "fedora19u"; }; - - #centos63_i386 = mkRPM { arch = "i386" ; image = "centos63"; }; - #centos63_x86_64 = mkRPM { arch = "x86_64" ; image = "centos63"; }; - centos64_i386 = mkRPM { arch = "i386" ; image = "centos64"; }; - centos64_x86_64 = mkRPM { arch = "x86_64" ; image = "centos64"; }; - centos65_i386 = mkRPM { arch = "i386" ; image = "centos65"; }; - centos65_x86_64 = mkRPM { arch = "x86_64" ; image = "centos65"; }; - - rawhide_i386 = mkRPM { arch = "i386" ; image = "rawhide"; }; - rawhide_x86_64 = mkRPM { arch = "x86_64" ; image = "rawhide"; }; }; in jobs |