diff --git a/dri3/dri3.h b/dri3/dri3.h
index edc7fa267..7562352ff 100644
--- a/dri3/dri3.h
+++ b/dri3/dri3.h
@@ -23,8 +23,6 @@
 #ifndef _DRI3_H_
 #define _DRI3_H_
 
-#include <xorg-server.h>
-
 #ifdef DRI3
 
 #include <X11/extensions/dri3proto.h>
diff --git a/glamor/glamor_egl.c b/glamor/glamor_egl.c
index 54af2753e..182e2e8c9 100644
--- a/glamor/glamor_egl.c
+++ b/glamor/glamor_egl.c
@@ -30,7 +30,6 @@
 #include "dix-config.h"
 
 #define GLAMOR_FOR_XORG
-#include <xorg-server.h>
 #include <unistd.h>
 #include <fcntl.h>
 #include <sys/ioctl.h>
diff --git a/glamor/glamor_eglmodule.c b/glamor/glamor_eglmodule.c
index d7e183649..dd4664b22 100644
--- a/glamor/glamor_eglmodule.c
+++ b/glamor/glamor_eglmodule.c
@@ -29,7 +29,6 @@
 
 #include "dix-config.h"
 
-#include <xorg-server.h>
 #include <xf86.h>
 #define GLAMOR_FOR_XORG
 #include <xf86Module.h>
diff --git a/glamor/glamor_priv.h b/glamor/glamor_priv.h
index 57a46873c..385c0279d 100644
--- a/glamor/glamor_priv.h
+++ b/glamor/glamor_priv.h
@@ -29,7 +29,6 @@
 
 #include "dix-config.h"
 
-#include <xorg-server.h>
 #include "glamor.h"
 #include "xvdix.h"
 
diff --git a/hw/xwayland/xwayland.h b/hw/xwayland/xwayland.h
index fc6855044..60b0c2964 100644
--- a/hw/xwayland/xwayland.h
+++ b/hw/xwayland/xwayland.h
@@ -27,7 +27,6 @@
 #define XWAYLAND_H
 
 #include <dix-config.h>
-#include <xorg-server.h>
 
 #include <stdio.h>
 #include <unistd.h>
diff --git a/include/xorg-server.h.in b/include/xorg-server.h.in
index 8bf9d38f2..09b79b6e3 100644
--- a/include/xorg-server.h.in
+++ b/include/xorg-server.h.in
@@ -16,6 +16,10 @@
 #ifndef _XORG_SERVER_H_
 #define _XORG_SERVER_H_
 
+#ifdef HAVE_XORG_CONFIG_H
+#error Include xorg-config.h when building the X server
+#endif
+
 /* Support BigRequests extension */
 #undef BIGREQS