Add Ubuntu 16.04 packaging. Fix and improve some packaging things.

GUI · GUI · commit 09f8f3cf72e5 · 2016-06-25T17:05:56.000-06:00
The packaging makefile is now more flexible with specific targets for
building a single distro's packages. The download packages script is
also improved to only download the necessary packages.

Also fix an erroneous --rpm-dist flag when building .deb packages.

We also need a newer version of librdkafka for Ubuntu 16.04
compatibility, but the newer v0.9.1 seems to compile fine against
rsyslog.
diff --git a/build/cmake/rsyslog.cmake b/build/cmake/rsyslog.cmake
@@ -35,6 +35,10 @@ ExternalProject_Add(
 )
 
 if(ENABLE_HADOOP_ANALYTICS)
+  # There's a small dependency on Python for librdkafka's Makefile:
+  # https://github.com/edenhill/librdkafka/blob/0.9.1/Makefile#L8
+  find_package(PythonInterp REQUIRED)
+
   ExternalProject_Add(
     librdkafka
     URL https://github.com/edenhill/librdkafka/archive/${LIBRDKAFKA_VERSION}.tar.gz
@@ -45,6 +49,7 @@ if(ENABLE_HADOOP_ANALYTICS)
   )
 endif()
 
+list(APPEND RSYSLOG_DEPENDS json-c)
 list(APPEND RSYSLOG_DEPENDS libestr)
 if(ENABLE_HADOOP_ANALYTICS)
   list(APPEND RSYSLOG_DEPENDS librdkafka)
diff --git a/build/cmake/versions.cmake b/build/cmake/versions.cmake
@@ -20,9 +20,8 @@ set(LIBESTR_VERSION 0.1.10)
 set(LIBESTR_HASH bd655e126e750edd18544b88eb1568d200a424a0c23f665eb14bbece07ac703c)
 set(LIBGEOIP_VERSION 1.6.9)
 set(LIBGEOIP_HASH 7475942dc8155046dddb4846f587a7e6)
-# Hold at 0.8 - 0.9 doesn't seem to be compatible with rsyslog.
-set(LIBRDKAFKA_VERSION 0.8.6)
-set(LIBRDKAFKA_HASH 1b77543f9be82d3f700c0ef98f494990)
+set(LIBRDKAFKA_VERSION 0.9.1)
+set(LIBRDKAFKA_HASH feb25faed02815f60ff363b2f40ba1b9)
 set(LUAROCKS_VERSION 2.3.0)
 set(LUAROCKS_HASH a38126684cf42b7d0e7a3c7cf485defb)
 set(LUAROCK_ARGPARSE_VERSION 0.5.0-1)
diff --git a/build/package/Makefile b/build/package/Makefile
@@ -4,74 +4,112 @@ LOG_DIR:=$(WORK_DIR)/log
 
 .PHONY: \
 	docker \
+	docker_build \
+	docker_verify \
 	docker_centos6 \
-	docker_centos6_verify_only \
+	docker_centos6_build \
+	docker_centos6_verify \
 	docker_centos7 \
-	docker_centos7_verify_only \
+	docker_centos7_build \
+	docker_centos7_verify \
 	docker_ubuntu1204 \
-	docker_ubuntu1204_verify_only \
+	docker_ubuntu1204_build \
+	docker_ubuntu1204_verify \
 	docker_ubuntu1404 \
-	docker_ubuntu1404_verify_only \
+	docker_ubuntu1404_build \
+	docker_ubuntu1404_verify \
+	docker_ubuntu1604 \
+	docker_ubuntu1604_build \
+	docker_ubuntu1604_verify \
 	docker_debian7 \
-	docker_debian7_verify_only \
+	docker_debian7_build \
+	docker_debian7_verify \
 	docker_debian8 \
-	docker_debian8_verify_only \
+	docker_debian8_build \
+	docker_debian8_verify \
+	docker_all
 
 $(LOG_DIR):
 	mkdir -p $@
 
-$(WORK_DIR)/download_previous_packages.stamp: $(SOURCE_DIR)/build/package/verify/download_previous_packages
-	$(SOURCE_DIR)/build/package/verify/download_previous_packages
-	mkdir -p $(WORK_DIR)
-	touch $@
+docker: | $(LOG_DIR)
+	$(MAKE) docker_build
+	$(MAKE) docker_verify
 
-docker: $(LOG_DIR)
+docker_build: | $(LOG_DIR)
 	$(SOURCE_DIR)/build/package/docker_run > $(LOG_DIR)/$(DIST).log 2>&1
-	$(SOURCE_DIR)/build/package/verify/docker_run >> $(LOG_DIR)/$(DIST).log 2>&1
 
-docker_verify_only: $(WORK_DIR)/download_previous_packages.stamp | $(LOG_DIR)
+docker_verify: | $(LOG_DIR)
+	$(SOURCE_DIR)/build/package/verify/download_previous_packages >> $(LOG_DIR)/$(DIST).log 2>&1
 	$(SOURCE_DIR)/build/package/verify/docker_run >> $(LOG_DIR)/$(DIST).log 2>&1
 
 docker_centos6:
 	DIST=centos-6 $(MAKE) docker
 
-docker_centos6_verify_only:
-	DIST=centos-6 $(MAKE) docker_verify_only
+docker_centos6_build:
+	DIST=centos-6 $(MAKE) docker_build
+
+docker_centos6_verify:
+	DIST=centos-6 $(MAKE) docker_verify
 
 docker_centos7:
 	DIST=centos-7 $(MAKE) docker
 
-docker_centos7_verify_only:
-	DIST=centos-7 $(MAKE) docker_verify_only
+docker_centos7_build:
+	DIST=centos-7 $(MAKE) docker_build
+
+docker_centos7_verify:
+	DIST=centos-7 $(MAKE) docker_verify
 
 docker_ubuntu1204:
 	DIST=ubuntu-12.04 $(MAKE) docker
 
-docker_ubuntu1204_verify_only:
-	DIST=ubuntu-12.04 $(MAKE) docker_verify_only
+docker_ubuntu1204_build:
+	DIST=ubuntu-12.04 $(MAKE) docker_build
+
+docker_ubuntu1204_verify:
+	DIST=ubuntu-12.04 $(MAKE) docker_verify
 
 docker_ubuntu1404:
 	DIST=ubuntu-14.04 $(MAKE) docker
 
-docker_ubuntu1404_verify_only:
-	DIST=ubuntu-14.04 $(MAKE) docker_verify_only
+docker_ubuntu1404_build:
+	DIST=ubuntu-14.04 $(MAKE) docker_build
+
+docker_ubuntu1404_verify:
+	DIST=ubuntu-14.04 $(MAKE) docker_verify
+
+docker_ubuntu1604:
+	DIST=ubuntu-16.04 $(MAKE) docker
+
+docker_ubuntu1604_build:
+	DIST=ubuntu-16.04 $(MAKE) docker_build
+
+docker_ubuntu1604_verify:
+	DIST=ubuntu-16.04 $(MAKE) docker_verify
 
 docker_debian7:
 	DIST=debian-7 $(MAKE) docker
 
-docker_debian7_verify_only:
-	DIST=debian-7 $(MAKE) docker_verify_only
+docker_debian7_build:
+	DIST=debian-7 $(MAKE) docker_build
+
+docker_debian7_verify:
+	DIST=debian-7 $(MAKE) docker_verify
 
 docker_debian8:
 	DIST=debian-8 $(MAKE) docker
 
-docker_debian8_verify_only:
-	DIST=debian-8 $(MAKE) docker_verify_only
+docker_debian8_build:
+	DIST=debian-8 $(MAKE) docker_build
 
-docker_all: \
-	docker_centos6 \
+docker_debian8_verify:
+	DIST=debian-8 $(MAKE) docker_verify
+
+docker_all: docker_centos6 \
 	docker_centos7 \
 	docker_ubuntu1204 \
 	docker_ubuntu1404 \
+	docker_ubuntu1604 \
 	docker_debian7 \
 	docker_debian8
diff --git a/build/package/build_package b/build/package/build_package
@@ -22,7 +22,6 @@ else
 fi
 
 fpm_args=()
-fpm_args+=("--rpm-dist" "$rpm_dist")
 fpm_args+=("-t" "$package_type")
 fpm_args+=("-s" "dir")
 fpm_args+=("--verbose")
diff --git a/build/package/verify/download_previous_packages b/build/package/verify/download_previous_packages
@@ -8,6 +8,10 @@ set -e -u
 
 source_dir="$(dirname "$(dirname "$(dirname "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)")")")"
 
+function compare_version {
+  echo "$@" | awk -F. '{ printf("%d%03d%03d%03d\n", $1,$2,$3,$4); }';
+}
+
 previous_versions=(
   0.8.0-1
   0.9.0-1
@@ -25,6 +29,10 @@ distros=(
   debian-8
 )
 
+if [ -n "${DIST:-}" ]; then
+  distros=($DIST)
+fi
+
 sourceforge_url_root="http://pilotfiber.dl.sourceforge.net/project/api-umbrella"
 bintray_url_root="https://bintray.com/artifact/download/nrel"
 checksums="$(cat "$source_dir/sha256sums.txt")"
@@ -38,56 +46,62 @@ for dist in "${distros[@]}"; do
     case "$dist" in
       centos-6)
         filename="api-umbrella-${version}.el6.x86_64.rpm"
+        url_prefix="$bintray_url_root/api-umbrella-el6"
         if [ "$version" == "0.8.0-1" ]; then
           url_prefix="$sourceforge_url_root/el/6"
-        else
-          url_prefix="$bintray_url_root/api-umbrella-el6"
         fi
         ;;
       centos-7)
         filename="api-umbrella-${version}.el7.x86_64.rpm"
+        url_prefix="$bintray_url_root/api-umbrella-el7"
         if [ "$version" == "0.8.0-1" ]; then
           url_prefix="$sourceforge_url_root/el/7"
-        else
-          url_prefix="$bintray_url_root/api-umbrella-el7"
         fi
         ;;
       ubuntu-12.04)
         filename="api-umbrella_${version}~precise_amd64.deb"
+        url_prefix="$bintray_url_root/api-umbrella-ubuntu/pool/main/a/api-umbrella"
         if [ "$version" == "0.8.0-1" ]; then
           url_filename="api-umbrella_${version}_amd64.deb"
           url_prefix="$sourceforge_url_root/ubuntu/12.04"
-        else
-          url_prefix="$bintray_url_root/api-umbrella-ubuntu/pool/main/a/api-umbrella"
         fi
         ;;
       ubuntu-14.04)
         filename="api-umbrella_${version}~trusty_amd64.deb"
+        url_prefix="$bintray_url_root/api-umbrella-ubuntu/pool/main/a/api-umbrella"
         if [ "$version" == "0.8.0-1" ]; then
           url_filename="api-umbrella_${version}_amd64.deb"
           url_prefix="$sourceforge_url_root/ubuntu/14.04"
-        else
-          url_prefix="$bintray_url_root/api-umbrella-ubuntu/pool/main/a/api-umbrella"
+        fi
+        ;;
+      ubuntu-16.04)
+        filename="api-umbrella_${version}~xenial_amd64.deb"
+        url_prefix="$bintray_url_root/api-umbrella-ubuntu/pool/main/a/api-umbrella"
+        if [ "$(compare_version "$version")" -lt "$(compare_version "0.12.0")" ]; then
+          # No Ubuntu 16.04 packages until v0.12
+          filename=""
         fi
         ;;
       debian-7)
         filename="api-umbrella_${version}~wheezy_amd64.deb"
+        url_prefix="$bintray_url_root/api-umbrella-debian/pool/main/a/api-umbrella"
         if [ "$version" == "0.8.0-1" ]; then
           url_filename="api-umbrella_${version}_amd64.deb"
           url_prefix="$sourceforge_url_root/debian/7"
-        else
-          url_prefix="$bintray_url_root/api-umbrella-debian/pool/main/a/api-umbrella"
         fi
         ;;
       debian-8)
         filename="api-umbrella_${version}~jessie_amd64.deb"
-        if [ "$version" == "0.8.0-1" ]; then
-          # No Debian 8 packages for API Umbrella v0.8.
+        url_prefix="$bintray_url_root/api-umbrella-debian/pool/main/a/api-umbrella"
+        if [ "$(compare_version "$version")" -lt "$(compare_version "0.9.0")" ]; then
+          # No Debian 8 packages until v0.9
           filename=""
-        else
-          url_prefix="$bintray_url_root/api-umbrella-debian/pool/main/a/api-umbrella"
         fi
         ;;
+      *)
+        echo "Unknown distribution: $dist"
+        exit 1
+        ;;
     esac
 
     if [ -n "$filename" ]; then
diff --git a/build/package/verify/spec/localhost/service_spec.rb b/build/package/verify/spec/localhost/service_spec.rb
@@ -100,6 +100,21 @@
 end
 
 RSpec.shared_examples("package upgrade") do |package_version|
+  # Skip testing upgrades if we don't have binary packages for certain distro
+  # and version combinations.
+  case(ENV["DIST"])
+  when "debian-8"
+    # No Debian 8 packages until v0.9
+    if(Gem::Version.new(package_version) < Gem::Version.new("0.9.0-1"))
+      next
+    end
+  when "ubuntu-16.04"
+    # No Ubuntu 16.04 packages until v0.12
+    if(Gem::Version.new(package_version) < Gem::Version.new("0.12.0-1"))
+      next
+    end
+  end
+
   def ensure_uninstalled
     command_result = command("/etc/init.d/api-umbrella stop")
     command_result.exit_status
@@ -520,12 +535,7 @@ def install_package(version)
     end
   end
 
-  # We don't have Debian 8 builds of API Umbrella v0.8, so skip testing that
-  # upgrade path for Debian 8.
-  if(ENV["DIST"] != "debian-8")
-    it_behaves_like "package upgrade", "0.8.0-1"
-  end
-
+  it_behaves_like "package upgrade", "0.8.0-1"
   it_behaves_like "package upgrade", "0.9.0-1"
   it_behaves_like "package upgrade", "0.10.0-1"
   it_behaves_like "package upgrade", "0.11.0-1"
diff --git a/build/package_dependencies.sh b/build/package_dependencies.sh
@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 
+set -e -u
+
 if [ -f /etc/redhat-release ]; then
   core_package_dependencies=(
     # General
@@ -54,6 +56,8 @@ if [ -f /etc/redhat-release ]; then
     openssl-devel
     patch
     pcre-devel
+    pkgconfig
+    python
     rpm-build
     rsync
     tar
@@ -65,10 +69,22 @@ if [ -f /etc/redhat-release ]; then
     java-1.8.0-openjdk-devel
   )
 elif [ -f /etc/debian_version ]; then
+  libffi_version=6
+  openjdk_version=7
+
+  # shellcheck disable=SC1091
+  source /etc/os-release
+  if [[ "$ID" == "debian" && "$VERSION_ID" == "7" ]]; then
+    libffi_version=5
+  elif [[ "$ID" == "ubuntu" && "$VERSION_ID" == "16.04" ]]; then
+    openjdk_version=8
+  fi
+
   core_package_dependencies=(
     # General
     bash
     libc6
+    libffi$libffi_version
     libncurses5
     libpcre3
     libuuid1
@@ -81,7 +97,7 @@ elif [ -f /etc/debian_version ]; then
     tcl
 
     # ElasticSearch
-    openjdk-7-jre-headless
+    openjdk-$openjdk_version-jre-headless
 
     # init.d script helpers
     sysvinit-utils
@@ -91,7 +107,7 @@ elif [ -f /etc/debian_version ]; then
     procps
   )
   hadoop_analytics_package_dependencies=(
-    openjdk-7-jre-headless
+    openjdk-$openjdk_version-jre-headless
   )
   core_build_dependencies=(
     autoconf
@@ -114,6 +130,8 @@ elif [ -f /etc/debian_version ]; then
     make
     openssl
     patch
+    pkg-config
+    python
     rsync
     tar
     tcl-dev
@@ -122,13 +140,11 @@ elif [ -f /etc/debian_version ]; then
     xz-utils
   )
   hadoop_analytics_build_dependencies=(
-    openjdk-7-jdk-headless
+    openjdk-$openjdk_version-jdk
   )
 
-  if lsb_release --codename --short | grep wheezy; then
-    core_package_dependencies+=("libffi5")
-  else
-    core_package_dependencies+=("libffi6")
+  if [[ "$ID" == "ubuntu" && "$VERSION_ID" == "16.04" ]]; then
+    core_build_dependencies+=("libtool-bin")
   fi
 else
   echo "Unknown build system"
diff --git a/build/scripts/outdated b/build/scripts/outdated
diff --git a/docs/developer/packaging.md b/docs/developer/packaging.md
