diff --git a/debian/changelog b/debian/changelog
index fea2b6cd92..82076037c4 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,8 +1,13 @@
 gtk+3.0 (3.20.6-2) UNRELEASED; urgency=medium
 
+  [ Laurent Bigonville ]
   * Drop patches/015_default-fallback-icon-theme.patch: The default icon theme
     is already defined as Adwaita, not need to also define the fallback theme.
 
+  [ Simon McVittie ]
+  * d/rules: reset XDG_CONFIG_HOME, XDG_CACHE_HOME, XDG_DATA_HOME so that
+    they are based on the temporary HOME
+
  -- Laurent Bigonville <bigon@debian.org>  Sun, 05 Jun 2016 02:29:49 +0200
 
 gtk+3.0 (3.20.6-1) unstable; urgency=medium
diff --git a/debian/rules b/debian/rules
index 759c44745f..ba48407470 100644
--- a/debian/rules
+++ b/debian/rules
@@ -108,6 +108,12 @@ export HOME=$(CURDIR)/debian/build
 export XDG_RUNTIME_DIR=$(HOME)
 # So that gsettings can find the (uninstalled) gtk schemas
 export XDG_DATA_DIRS=/usr/share:$(CURDIR)/debian/build
+# Put these back to their defaults if we are not running with a clean
+# environment, so that they are based on the temporary $HOME above
+unexport XDG_CACHE_HOME
+unexport XDG_CONFIG_DIRS
+unexport XDG_CONFIG_HOME
+unexport XDG_DATA_HOME
 # Get failing tests' stdout/err so we have some information when a build fails
 export VERBOSE=1