diff --git a/include/meson.build b/include/meson.build
index 3521fa3ef..6b4a69dd4 100644
--- a/include/meson.build
+++ b/include/meson.build
@@ -201,7 +201,9 @@ conf_data.set('UNIXCONN', host_machine.system() != 'windows' ? '1' : false)
 conf_data.set('IPv6', build_ipv6 ? '1' : false)
 
 conf_data.set('BIGREQS', '1')
-conf_data.set('COMPOSITE', '1')
+if build_composite
+    conf_data.set('COMPOSITE', '1')
+endif
 conf_data.set('DAMAGE', '1')
 conf_data.set('DBE', '1')
 conf_data.set('DGA', build_dga ? '1' : false)
diff --git a/meson.build b/meson.build
index a6afd5575..c09e8509c 100644
--- a/meson.build
+++ b/meson.build
@@ -235,8 +235,10 @@ else
     build_xquartz = get_option('xquartz') == 'true'
 endif
 
+build_composite = true
 build_rootless = false
 if build_xquartz
+    build_composite = false
     build_rootless = true
 endif
 
@@ -697,7 +699,9 @@ subdir('fb')
 subdir('mi')
 subdir('os')
 # X extensions
-subdir('composite')
+if build_composite
+    subdir('composite')
+endif
 subdir('damageext')
 subdir('dbe')
 subdir('miext/damage')
@@ -731,7 +735,6 @@ libxserver = [
     libxserver_mi,
     libxserver_dix,
 
-    libxserver_composite,
     libxserver_damageext,
     libxserver_dbe,
     libxserver_randr,
@@ -748,6 +751,10 @@ libxserver = [
     libxserver_os,
 ]
 
+if build_composite
+    libxserver += libxserver_composite
+endif
+
 libxserver += libxserver_dri3
 
 subdir('hw')