travis: build debian packages and documentation

The resulting packages are pushed to Debian repositories hosted on bintray.com. This is the first step to move away from our custom buildbot setup (see https://i3wm.org/docs/buildbot.html for details on that) towards infrastructure which is more standard (travis) and in the open.
tdemin · Apr 8, 2016 · bd29745 · bd29745
1 parent ea55729
commit bd29745
Show file tree

Hide file tree

Showing 18 changed files with 345 additions and 4 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -6,18 +6,67 @@ language: c
 compiler:
   - gcc
   - clang
+addons:
+  apt:
+    packages:
+    # For https support in HTTP::Tiny.
+    - libio-socket-ssl-perl
 env:
   global:
-  - BASENAME="i3wm/travis-base:$(date +'%Y-%m')-$(./travis/ha.sh)"
+  - BASENAME="i3wm/travis-base:$(date +'%Y-%m')-$(./travis/ha.sh travis/travis-base.Dockerfile)"
+  - BASENAME_386="i3wm/travis-base-386:$(date +'%Y-%m')-$(./travis/ha.sh travis/travis-base-386.Dockerfile)"
+  - BASENAME_UBUNTU="i3wm/travis-base-ubuntu:$(date +'%Y-%m')-$(./travis/ha.sh travis/travis-base-ubuntu.Dockerfile)"
+  - BASENAME_UBUNTU_386="i3wm/travis-base-ubuntu-386:$(date +'%Y-%m')-$(./travis/ha.sh travis/travis-base-ubuntu-386.Dockerfile)"
   - secure: "B5IICA8MPx/FKaB50rTPqL8P1NU+Q0yuWl+lElL4+a9xSyLikfm3NzUPHoVwx8lNw2AVK6br7p0OmF7vMFjqAgrgc1cajTtEae5uFRKNUrWLpXM046YgNEYLLIHsQOjInxE+S4O6EFVzsUqsu8aeo2Xhq4sm4iUocG7e5isYgYo=" # DOCKER_PASS
   - secure: "EIvrq8PG7lRjidppG0RCv4F0X4GP3DT9F5+ixVuGPfhK/hZT3jYC2AVY9G+NnUcXVwQEpW92rlqpftQ/qZ13FoyWokC8ZyoyD06fr5FPCfoFF3OczZwAJzZYkObI/hE9+/hXcylx/Os6N4INd2My1ntGk3JPsWL9riopod5EjSg=" # DOCKER_EMAIL
   - secure: "hvhBunS4xXTgnIOsk/BPT7I7FrJhvVwCSt5PfxxvMqNaztOJI9BuK7ZrZ5Cy38KyHwlh3VHAH5AaCygJcPauoSQCV3bpnlbaWn3ruq2F0Q697Q5uNf73liXzyUqb9/Zvfvge4y4WWOhP5tVz1C6ZBe/NfhU7pqKLMA+6ads+99c=" # DOCKER_USER
+  - secure: "uJuuefmnJUuEH15ZD8xQilibx7EeBvMHBLoIZ8bgGHeleEImBD0XbD1ypvhYJKpviOmw5BkZmc9bVO8DGDEHYbSlIa2xDlF6vGrwgCEaxcMIhOAhv+dW9C/maJVieLOEPM01/fK2qdKESZaLvlopkWmxZwDyMObI9L7AMW9zQD8=" # BINTRAY_USER
+  - secure: "L3aPSNLySPXtWCW+xf8h/AAdquwNgxyTQpYOwexJmTPav82Qx8uQlp1yJkUmt+a+FLZDFfQeMivaHq0311RvuQVmkAJx49DjaddrwqOJut2UPsoVDn1WeuAcSHIXOq/0H+zgFMr/PGY0HXIsw1mTMhgheGJNqg09BvYWROCEAcA=" # BINTRAY_KEY
+  - secure: "sBMVn4C/WRWgoAytEFGx4CC5O55Q63h02AcuBnb1jXcBm0RenoBpzUPtxSseJwDPUA1o/UkuEDDjm3PosT5NF+dvED01VDFMsPVE11K0u6+avYy3jYXqyUEDW3G2o6Wo/2aqNjmd++8jskBdS9+Cx9gaFbgxfzSp0Yfu3oJm/4c=" # GH_TOKEN
 install:
   - if [ -a .git/shallow ]; then git fetch --unshallow; fi
-  - docker pull ${BASENAME} || ./travis/docker-build-and-push.sh
+  - docker pull ${BASENAME} || ./travis/docker-build-and-push.sh ${BASENAME} travis/travis-base.Dockerfile
+  - ./travis/skip-pkg.sh || docker pull ${BASENAME_UBUNTU} || ./travis/docker-build-and-push.sh ${BASENAME_UBUNTU} travis/travis-base-ubuntu.Dockerfile
+  - ./travis/skip-pkg.sh || docker pull ${BASENAME_386} || ./travis/docker-build-and-push.sh ${BASENAME_386} travis/travis-base-386.Dockerfile
+  - ./travis/skip-pkg.sh || docker pull ${BASENAME_UBUNTU_386} || ./travis/docker-build-and-push.sh ${BASENAME_UBUNTU_386} travis/travis-base-ubuntu-386.Dockerfile
 script:
   - docker run -v $PWD:/usr/src/i3/ -w /usr/src/i3 ${BASENAME} ./travis/check-safe-wrappers.sh
   - docker run -v $PWD:/usr/src/i3/ -w /usr/src/i3 ${BASENAME} ./travis/check-formatting.sh
   - docker run -v $PWD:/usr/src/i3/ -w /usr/src/i3 -e CC -e CFLAGS="-Wformat -Wformat-security -Wextra -Wno-unused-parameter -Werror" ${BASENAME} make all mans -j ASAN=1
   - docker run -v $PWD:/usr/src/i3/ -w /usr/src/i3 ${BASENAME} ./travis/check-spelling.pl
   - docker run -v $PWD:/usr/src/i3/ -w /usr/src/i3 ${BASENAME} ./travis/run-tests.sh
+  - ./travis/skip-pkg.sh || docker run -v $PWD:/usr/src/i3/ -w /usr/src/i3 ${BASENAME} ./travis/debian-build.sh deb/debian-amd64/DIST
+  - ./travis/skip-pkg.sh || docker run -v $PWD:/usr/src/i3/ -w /usr/src/i3 ${BASENAME_UBUNTU} ./travis/debian-build.sh deb/ubuntu-amd64/DIST
+  - ./travis/skip-pkg.sh || docker run -v $PWD:/usr/src/i3/ -w /usr/src/i3 ${BASENAME_386} linux32 ./travis/debian-build.sh deb/debian-i386/DIST
+  - ./travis/skip-pkg.sh || docker run -v $PWD:/usr/src/i3/ -w /usr/src/i3 ${BASENAME_UBUNTU_386} linux32 ./travis/debian-build.sh deb/ubuntu-i386/DIST
+  - ./travis/skip-pkg.sh || docker run -v $PWD:/usr/src/i3/ -w /usr/src/i3 ${BASENAME} ./travis/clang-analyze.sh
+  - ./travis/skip-pkg.sh || docker run -v $PWD:/usr/src/i3/ -w /usr/src/i3 ${BASENAME} ./travis/docs.sh
+  - ./travis/skip-pkg.sh || travis/prep-bintray.sh
+
+deploy:
+  - provider: bintray
+    file: travis/bintray-autobuild-debian.json
+    user: $BINTRAY_USER
+    key: $BINTRAY_KEY
+    skip_cleanup: true
+    on:
+      branch: next
+      condition: $CC = gcc
+  - provider: bintray
+    file: travis/bintray-autobuild-ubuntu.json
+    user: $BINTRAY_USER
+    key: $BINTRAY_KEY
+    skip_cleanup: true
+    on:
+      branch: next
+      condition: $CC = gcc
+  - provider: script
+    script: travis/deploy-github-pages.sh
+    skip_cleanup: true
+    on:
+      branch: next
+      condition: $CC = gcc
+
+after_deploy:
+  - travis/cleanup-bintray.pl i3-autobuild
+  - travis/cleanup-bintray.pl i3-autobuild-ubuntu
diff --git a/travis/bintray-autobuild-debian.json b/travis/bintray-autobuild-debian.json
@@ -0,0 +1,36 @@
+{
+   "package": {
+        "name": "i3-wm",
+        "repo": "i3-autobuild",
+        "subject": "i3"
+    },
+
+    "version": {
+        "name": "%version%",
+        "desc": "TODO",
+        "gpgSign": false
+    },
+
+    "files": [
+        {
+            "includePattern": "deb/debian-amd64/(.*\\.deb)$",
+	    "matrixParams": {
+                "deb_distribution": "sid",
+                "deb_component": "main",
+                "deb_architecture": "amd64"
+	    },
+	    "uploadPattern": "$1"
+        },
+        {
+            "includePattern": "deb/debian-i386/(.*\\.deb)$",
+	    "matrixParams": {
+                "deb_distribution": "sid",
+                "deb_component": "main",
+                "deb_architecture": "i386"
+	    },
+	    "uploadPattern": "$1"
+        }
+    ],
+
+    "publish": true
+}
diff --git a/travis/bintray-autobuild-ubuntu.json b/travis/bintray-autobuild-ubuntu.json
@@ -0,0 +1,36 @@
+{
+   "package": {
+        "name": "i3-wm",
+        "repo": "i3-autobuild-ubuntu",
+        "subject": "i3"
+    },
+
+    "version": {
+        "name": "%version%",
+        "desc": "TODO",
+        "gpgSign": false
+    },
+
+    "files": [
+        {
+            "includePattern": "deb/ubuntu-amd64/(.*\\.deb)$",
+	    "matrixParams": {
+                "deb_distribution": "wily",
+                "deb_component": "main",
+                "deb_architecture": "amd64"
+	    },
+	    "uploadPattern": "$1"
+        },
+        {
+            "includePattern": "deb/ubuntu-i386/(.*\\.deb)$",
+	    "matrixParams": {
+                "deb_distribution": "wily",
+                "deb_component": "main",
+                "deb_architecture": "i386"
+	    },
+	    "uploadPattern": "$1"
+        }
+    ],
+
+    "publish": true
+}
diff --git a/travis/check-formatting.sh b/travis/check-formatting.sh
@@ -1,2 +1,6 @@
 #!/bin/sh
+
+set -e
+set -x
+
 clang-format-3.5 -i $(find . -name "*.[ch]" | tr '\n' ' ') && git diff --exit-code || (echo 'Code was not formatted using clang-format!'; false)
diff --git a/travis/clang-analyze.sh b/travis/clang-analyze.sh
@@ -0,0 +1,9 @@
+#!/bin/sh
+
+set -e
+set -x
+
+mkdir -p deb/DIST-clang
+tar xf *.tar.bz2 -C deb/DIST-clang --strip-components=1
+(cd deb/DIST-clang && scan-build -o ../CLANG --html-title="Analysis of i3 v$(git describe --tags)" make -j8)
+mv deb/CLANG/*/* deb/CLANG
diff --git a/travis/cleanup-bintray.pl b/travis/cleanup-bintray.pl
@@ -0,0 +1,37 @@
+#!/usr/bin/env perl
+# vim:ts=4:sw=4:expandtab
+
+use strict;
+use warnings;
+use Data::Dumper;
+use HTTP::Tiny; # in core since v5.13.9
+use JSON::PP; # in core since v5.13.9
+use MIME::Base64; # in core since v5.7
+use v5.13;
+
+my $repo = shift;
+
+my $auth = $ENV{'BINTRAY_USER'} . ':' . $ENV{'BINTRAY_KEY'};
+die "BINTRAY_USER and/or BINTRAY_KEY environment variables not set" if $auth eq ':';
+# TODO(stapelberg): switch to putting $auth into the URL once perl-modules ≥
+# 5.20 is available on travis (Ubuntu Wily or newer).
+my $auth_header = 'Basic ' . MIME::Base64::encode_base64($auth, "");
+my $apiurl = 'https://api.bintray.com/packages/i3/' . $repo . '/i3-wm';
+my $client = HTTP::Tiny->new(
+    verify_SSL => 1,
+    default_headers => {
+        'authorization' => $auth_header,
+    });
+my $resp = $client->get($apiurl);
+die "Getting versions failed: HTTP status $resp->{status} (content: $resp->{content})" unless $resp->{success};
+my $decoded = decode_json($resp->{content});
+my @versions = sort @{$decoded->{versions}};
+
+# Keep the most recent 5 versions.
+splice(@versions, 0, 5);
+
+for my $version (@versions) {
+    say "Deleting old version $version";
+    $resp = $client->request('DELETE', "$apiurl/versions/$version");
+    die "Deletion of version $version failed: HTTP status $resp->{status} (content: $resp->{content})" unless $resp->{success};
+}
diff --git a/travis/debian-build.sh b/travis/debian-build.sh
@@ -0,0 +1,15 @@
+#!/bin/sh
+
+set -e
+set -x
+
+DEST=$1
+
+make dist
+# unpack dist tarball
+mkdir -p "${DEST}"
+tar xf *.tar.bz2 -C "${DEST}" --strip-components=1
+cp -r debian "${DEST}"
+cd "${DEST}"
+debchange -m -l+g$(git describe --tags) 'Automatically built'
+dpkg-buildpackage -b
diff --git a/travis/deploy-github-pages.sh b/travis/deploy-github-pages.sh
@@ -0,0 +1,23 @@
+#!/bin/sh
+
+set -e
+set -x
+
+GITVERSION=$(git describe --tags)
+
+mkdir build.i3wm.org
+cp -r deb/CLANG build.i3wm.org/clang-analyze
+cp -r deb/COPY-DOCS build.i3wm.org/docs
+cd build.i3wm.org
+echo build.i3wm.org > CNAME
+git init
+
+git config user.name "Travis CI"
+git config user.email "i3bot@i3wm.org"
+git add .
+git commit -m "Publish docs/static analysis for github.com/i3/i3 v${GITVERSION}"
+
+# Hide stdout/stderr because it might contain sensitive info.
+set +x
+echo "git push"
+git push --force --quiet "https://${GH_TOKEN}@github.com/i3/build.i3wm.org.git" master:gh-pages >/dev/null 2>&1
diff --git a/travis/docker-build-and-push.sh b/travis/docker-build-and-push.sh
@@ -2,11 +2,14 @@
 
 set -e
 
+BASENAME=$1
+DOCKERFILE=$2
+
 # .dockerignore is created on demand so that release.sh and other scripts are
 # not influenced by our travis setup.
 echo .git > .dockerignore
 
-docker build --pull --no-cache --rm -t=${BASENAME} -f travis-build.Dockerfile .
+docker build --pull --no-cache --rm -t=${BASENAME} -f ${DOCKERFILE} .
 # For pull requests, travis does not add secure environment variables to the
 # environment (because pull requests could then steal their values), so skip
 # the login+push step when the variable isn’t set.

diff --git a/travis/docs.sh b/travis/docs.sh
@@ -0,0 +1,17 @@
+#!/bin/sh
+
+set -e
+set -x
+
+make -C docs ASCIIDOC="asciidoc -a linkcss -a stylesdir=http://i3wm.org/css -a scriptsdir=http://i3wm.org/js --backend=xhtml11 -f docs/asciidoc-git.conf"
+./docs/i3-pod2html i3-dmenu-desktop man/i3-dmenu-desktop.html
+./docs/i3-pod2html i3-save-tree man/i3-save-tree.html
+for file in $(sed 's/\.1$/.man/g' debian/i3-wm.manpages)
+do
+	[ -f "$file" ] && asciidoc -a linkcss -a stylesdir=http://i3wm.org/css -a scriptsdir=http://i3wm.org/js --backend=xhtml11 -f docs/asciidoc-git.conf "$file"
+done
+
+mkdir -p deb/COPY-DOCS
+
+cp $(tr "\n" ' ' < debian/i3-wm.docs) deb/COPY-DOCS/
+cp $(sed 's/\.1$/.html/g' debian/i3-wm.manpages | tr "\n" ' ') deb/COPY-DOCS/
diff --git a/travis/ha.sh b/travis/ha.sh
@@ -4,4 +4,4 @@
 # installed in the container, so that any changes in what needs to be installed
 # will result in a cache invalidation.
 
-cat debian/control travis-build.Dockerfile | sha256sum | dd bs=1 count=8 status=none
+cat debian/control "$1" | sha256sum | dd bs=1 count=8 status=none
diff --git a/travis/prep-bintray.sh b/travis/prep-bintray.sh
@@ -0,0 +1,6 @@
+#!/bin/sh
+
+set -e
+set -x
+
+sed -i "s,%version%,$(git describe --tags),g" travis/bintray-autobuild-*.json
diff --git a/travis/run-tests.sh b/travis/run-tests.sh
@@ -1,4 +1,8 @@
 #!/bin/sh
+
+set -e
+set -x
+
 cd testcases
 # Try running the tests in parallel so that the common case (tests pass) is
 # quick, but fall back to running them in sequence to make debugging easier.

diff --git a/travis/skip-pkg.sh b/travis/skip-pkg.sh
@@ -0,0 +1,14 @@
+#!/bin/sh
+# Returns true if Debian/Ubuntu packages should be skipped because this CI run
+# was triggered by a pull request.
+
+# Verify BINTRAY_USER is present (only set on github.com/i3/i3),
+# otherwise the CI run was triggered by a pull request.
+# Verify CC=gcc so that we only build packages once for each commit,
+# not twice (with gcc and clang).
+if [ ! -z "$BINTRAY_USER" ] && [ "$CC" = "gcc" ]
+then
+	exit 1
+fi
+
+exit 0
diff --git a/travis/travis-base-386.Dockerfile b/travis/travis-base-386.Dockerfile
@@ -0,0 +1,29 @@
+# vim:ft=Dockerfile
+# Same as travis-base.Dockerfile, but without the test suite dependencies since
+# we only build Debian packages on i386, we don’t run the tests.
+FROM i386/debian:sid
+
+RUN echo force-unsafe-io > /etc/dpkg/dpkg.cfg.d/docker-apt-speedup
+# Paper over occasional network flakiness of some mirrors.
+RUN echo 'APT::Acquire::Retries "5";' > /etc/apt/apt.conf.d/80retry
+
+# NOTE: I tried exclusively using gce_debian_mirror.storage.googleapis.com
+# instead of httpredir.debian.org, but the results (Fetched 123 MB in 36s (3357
+# kB/s)) are not any better than httpredir.debian.org (Fetched 123 MB in 34s
+# (3608 kB/s)). Hence, let’s stick with httpredir.debian.org (default) for now.
+
+# Install mk-build-deps (for installing the i3 build dependencies),
+# clang and clang-format-3.5 (for checking formatting and building with clang),
+# lintian (for checking spelling errors),
+RUN linux32 apt-get update && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+    dpkg-dev devscripts git equivs \
+    clang clang-format-3.5 \
+    lintian && \
+    rm -rf /var/lib/apt/lists/*
+
+# Install i3 build dependencies.
+COPY debian/control /usr/src/i3-debian-packaging/control
+RUN linux32 apt-get update && \
+    DEBIAN_FRONTEND=noninteractive mk-build-deps --install --remove --tool 'apt-get --no-install-recommends -y' /usr/src/i3-debian-packaging/control && \
+    rm -rf /var/lib/apt/lists/*
diff --git a/travis/travis-base-ubuntu-386.Dockerfile b/travis/travis-base-ubuntu-386.Dockerfile
@@ -0,0 +1,29 @@
+# vim:ft=Dockerfile
+# Same as travis-base.Dockerfile, but without the test suite dependencies since
+# we only build Debian packages on Ubuntu i386, we don’t run the tests.
+FROM i386/ubuntu:wily
+
+RUN echo force-unsafe-io > /etc/dpkg/dpkg.cfg.d/docker-apt-speedup
+# Paper over occasional network flakiness of some mirrors.
+RUN echo 'APT::Acquire::Retries "5";' > /etc/apt/apt.conf.d/80retry
+
+# NOTE: I tried exclusively using gce_debian_mirror.storage.googleapis.com
+# instead of httpredir.debian.org, but the results (Fetched 123 MB in 36s (3357
+# kB/s)) are not any better than httpredir.debian.org (Fetched 123 MB in 34s
+# (3608 kB/s)). Hence, let’s stick with httpredir.debian.org (default) for now.
+
+# Install mk-build-deps (for installing the i3 build dependencies),
+# clang and clang-format-3.5 (for checking formatting and building with clang),
+# lintian (for checking spelling errors),
+RUN linux32 apt-get update && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+    dpkg-dev devscripts git equivs \
+    clang clang-format-3.5 \
+    lintian && \
+    rm -rf /var/lib/apt/lists/*
+
+# Install i3 build dependencies.
+COPY debian/control /usr/src/i3-debian-packaging/control
+RUN linux32 apt-get update && \
+    DEBIAN_FRONTEND=noninteractive mk-build-deps --install --remove --tool 'apt-get --no-install-recommends -y' /usr/src/i3-debian-packaging/control && \
+    rm -rf /var/lib/apt/lists/*