[Pkg-protobuf-devel] Bug#846343: protobuf FTCBFS: lots of issues

Wed Nov 30 13:08:58 UTC 2016

Source: protobuf
Version: 3.0.0-9
Tags: patch
User: helmutg at debian.org
Usertags: rebootstrap

protobuf fails to cross build from source for a variety of different
reasons:

 * Its dependency on g++ is unsatisfiable, because the host architecture
   g++ is requested and it conflicts with the build architecture g++
   requested by build-essential. In a distant future, you'd replace it
   with "g++-for-host (>= ...)", but for now that doesn't work.
   Thankfully, the dependency is satisfied in oldstable and can be
   removed.
 
 * Likewise the python-all and python3-all dependencies request Python
   for the host architecture. Installation of these packages simply
   fails in postinst (byte compilation). protobuf really wants to
   execute Python during build, so it should be requesting them for the
   build architecture (i.e. annotate those dependencies with :any).

 * python-google-apputils is Architecture: all and (implicitly)
   Multi-Arch: no. Thus it can never satisfy cross Build-Depends.
   Thankfully, it is nowhere used by protobuf anymore. The dependency
   can simply be dropped.

 * The build tries to run the compiled binary protoc. That fails,
   because protoc is compiled for the host architecture. Upstream
   advises that cross builds should pass --with-protoc=path/to/protoc.

 * After the build, protoc is used by generate_descriptor_proto.sh to
   regenerate sources. Of course, it cannot do that after a cross build.
   I propose doing the first build pass for the build architecture and
   then running generate_descriptor_proto.sh. When performing a cross
   build, another build passing --with-protoc is performed.

 * The python extension build is totally unaware of cross compilation.
   Setting the right environment variables solves that part.

 * Even then, due to bug #846326 in python2.7, the python2 extension
   build uses the build architecture linker. Explicitly overriding CXX
   fixes that as well.

The attached patch addresses all of the listed issues. After applying
it, protobuf cross builds successfully to e.g. arm64. Please consider
applying it even though it got a bit lengthy.

Helmut
-------------- next part --------------

--- protobuf-3.0.0/debian/changelog
+++ protobuf-3.0.0/debian/changelog
@@ -1,3 +1,18 @@
+protobuf (3.0.0-9.1) UNRELEASED; urgency=medium
+
+  * Non-maintainer upload.
+  * Fix FTCBFS: (Closes: #-1)
+    + Remove unused Build-Depends g++ (satisfied in oldstable) and
+      python-google-apputils.
+    + Annotate python Build-Depends with :any
+    + Run generate_descriptor_proto.sh on a native build before performing
+      the actual cross build
+    + Save the native protoc in debian/native_protoc/ and pass it to the
+      cross build via --with-protoc
+    + Set up environment variables for cross building python extensions
+
+ -- Helmut Grohne <helmut at subdivi.de>  Wed, 30 Nov 2016 12:58:07 +0100
+
 protobuf (3.0.0-9) unstable; urgency=medium
 
   * Backport upstream fix for Python 3 build failure (closes: #845686).
--- protobuf-3.0.0/debian/clean
+++ protobuf-3.0.0/debian/clean
@@ -1 +1,2 @@
 protoc.1
+debian/native_protoc/
--- protobuf-3.0.0/debian/control
+++ protobuf-3.0.0/debian/control
@@ -13,19 +13,17 @@
  , debhelper (>= 9)
  , dh-autoreconf
 # C/C++
- , g++ (>= 4:4.7)
  , zlib1g-dev
  , google-mock
  , libgtest-dev
 # Python
  , dh-python
- , python-all (>= 2.7)
+ , python-all:any (>= 2.7)
  , libpython-all-dev (>= 2.7)
- , python3-all (>= 3.3)
+ , python3-all:any (>= 3.3)
  , libpython3-all-dev (>= 3.3)
  , python-setuptools
  , python3-setuptools
- , python-google-apputils
  , python3-six
 # Manpage generator
  , xmlto
--- protobuf-3.0.0/debian/rules
+++ protobuf-3.0.0/debian/rules
@@ -1,9 +1,25 @@
 #!/usr/bin/make -f
 # -*- makefile -*-
 
+include /usr/share/dpkg/architecture.mk
+ifeq ($(origin CXX),default)
+CXX := $(DEB_HOST_GNU_TYPE)-g++
+endif
+
 %:
 	dh $@ --with autoreconf,python2,python3 --parallel
 
+ifneq ($(DEB_BUILD_ARCH),$(DEB_HOST_ARCH))
+override_dh_auto_configure:
+	dh_auto_configure -- --host=$(DEB_BUILD_GNU_TYPE)
+
+PYTHON_CROSS_VARS += PROTOC=$(CURDIR)/debian/run_protoc
+PYTHON_CROSS_VARS += PYTHONPATH=/usr/lib/python$$pv/plat-$(DEB_HOST_MULTIARCH)
+PYTHON_CROSS_VARS += _PYTHON_HOST_PLATFORM=$(DEB_HOST_ARCH_OS)-$(DEB_HOST_GNU_CPU)
+# work around #846326:
+PYTHON_CROSS_VARS += CXX=$(CXX)
+endif
+
 override_dh_auto_build-arch:
 ## Chicken<->Egg problem: protobuf requires self-generated .pb.go files to
 ## be built. First we build it normally; then "generate_descriptor_proto.sh"
@@ -12,13 +28,27 @@
 	dh_auto_build --arch
 	bash -x ./generate_descriptor_proto.sh
 
+ifneq ($(DEB_BUILD_ARCH),$(DEB_HOST_ARCH))
+	# save the compiler
+	cp -Rv src/.libs debian/native_protoc
+	# clean everything but regenerated .pb.{cc,go} files
+	$(MAKE) clean
+	# cross build
+	dh_auto_configure -- --with-protoc=$(CURDIR)/debian/run_protoc
+	dh_auto_build --arch
+endif
+
 	# Generate the manpage.
 	xmlto man debian/protoc.xml
 
 	# Python and Python3 build.
 	cp -rv python python3
-	cd python && python setup.py build --cpp_implementation
-	cd python3 && python3 setup.py build --cpp_implementation
+	set -e; cd python && for pv in $(shell pyversions -vr); do \
+		$(PYTHON_CROSS_VARS) python$$pv setup.py build --cpp_implementation; \
+	done
+	set -e; cd python3 && for pv in $(shell py3versions -vr); do \
+		$(PYTHON_CROSS_VARS) python$$pv setup.py build --cpp_implementation; \
+	done
 
 override_dh_auto_build-indep:
 	dh_auto_build --indep
@@ -82,16 +112,17 @@
 
 	# Python install.
 	set -e; \
-	cd python && for python in $(shell pyversions -r); do \
-		$$python setup.py install --cpp_implementation \
+	cd python && for pv in $(shell pyversions -vr); do \
+		$(PYTHON_CROSS_VARS) python$$pv setup.py install --cpp_implementation \
 			--install-layout=deb --no-compile \
 			--root=$(CURDIR)/debian/python-protobuf; \
 	done
 	find $(CURDIR)/debian/python-protobuf -name 'protobuf-*-nspkg.pth' -delete
 
 	# Python3 install.
-	cd python3 && for python in $(shell py3versions -r); do \
-		$$python setup.py install --cpp_implementation \
+	set -e; \
+	cd python3 && for pv in $(shell py3versions -vr); do \
+		$(PYTHON_CROSS_VARS) python$$pv setup.py install --cpp_implementation \
 			--install-layout=deb --no-compile \
 			--root=$(CURDIR)/debian/python3-protobuf; \
 	done
--- protobuf-3.0.0/debian/run_protoc
+++ protobuf-3.0.0/debian/run_protoc
@@ -0,0 +1,5 @@
+#!/bin/sh
+# see debian/rules for how debian/native_protoc gets populated during cross builds
+location=$(dirname "$0")
+export LD_LIBRARY_PATH="$location/native_protoc${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"
+exec "$location/native_protoc/protoc" "$@"
