diff --git a/configure.ac b/configure.ac
index 969090b94..268c82fda 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1237,7 +1237,7 @@ esac
 
 AM_CONDITIONAL(DRI3, test "x$DRI3" = xyes)
 
-if test "x$DRI" = xyes || test "x$DRI2" = xyes || test "x$DRI3" = xyes || test "x$CONFIG_UDEV_KMS" = xyes; then
+if test "x$DRI" = xyes || test "x$DRI2" = xyes || test "x$DRI3" = xyes || test "x$CONFIG_UDEV_KMS" = xyes || test "x$XORG" = xyes; then
 	if test "x$DRM" = xyes; then
 		AC_DEFINE(WITH_LIBDRM, 1, [Building with libdrm support])
 		PKG_CHECK_MODULES([LIBDRM], $LIBDRM)