diff --git a/configure.ac b/configure.ac
index 7db718748..220478a72 100644
--- a/configure.ac
+++ b/configure.ac
@@ -926,8 +926,6 @@ fi
 AM_CONDITIONAL(SYSTEMD_LOGIND, [test "x$SYSTEMD_LOGIND" = xyes])
 
 if test "x$SUID_WRAPPER" = xyes; then
-        dnl The wrapper uses libdrm headers, so ensure they are available
-        PKG_CHECK_MODULES([LIBDRM], $LIBDRM)
         dnl This is a define so that if some platforms want to put the wrapper
         dnl somewhere else this can be easily changed
         AC_DEFINE_DIR(SUID_WRAPPER_DIR, libexecdir, [Where to install the Xorg binary and Xorg.wrap])
diff --git a/hw/xfree86/xorg-wrapper.c b/hw/xfree86/xorg-wrapper.c
index 22e97ad5d..75d120ab6 100644
--- a/hw/xfree86/xorg-wrapper.c
+++ b/hw/xfree86/xorg-wrapper.c
@@ -39,8 +39,10 @@
 #include <sys/consio.h>
 #endif
 #include <unistd.h>
+#ifdef WITH_LIBDRM
 #include <drm.h>
 #include <xf86drm.h> /* For DRM_DEV_NAME */
+#endif
 
 #define CONFIG_FILE SYSCONFDIR "/X11/Xwrapper.config"
 
@@ -183,7 +185,9 @@ static int on_console(int fd)
 
 int main(int argc, char *argv[])
 {
+#ifdef WITH_LIBDRM
     struct drm_mode_card_res res;
+#endif
     char buf[PATH_MAX];
     int i, r, fd;
     int kms_cards = 0;
@@ -219,6 +223,7 @@ int main(int argc, char *argv[])
         }
     }
 
+#ifdef WITH_LIBDRM
     /* Detect if we need root rights, except when overriden by the config */
     if (needs_root_rights == -1) {
         for (i = 0; i < 16; i++) {
@@ -237,6 +242,7 @@ int main(int argc, char *argv[])
             close(fd);
         }
     }
+#endif
 
     /* If we've found cards, and all cards support kms, drop root rights */
     if (needs_root_rights == 0 || (total_cards && kms_cards == total_cards)) {