diff --git a/glamor/glamor_priv.h b/glamor/glamor_priv.h
index 610af5315..76570febd 100644
--- a/glamor/glamor_priv.h
+++ b/glamor/glamor_priv.h
@@ -945,6 +945,9 @@ typedef struct {
     RegionRec clip;
     PixmapPtr src_pix[3];       /* y, u, v for planar */
     int src_pix_w, src_pix_h;
+    /* Port optimization */
+    int prev_fmt;
+    glamor_program xv_prog;
 } glamor_port_private;
 
 extern XvAttributeRec glamor_xv_attributes[];
diff --git a/glamor/glamor_xv.c b/glamor/glamor_xv.c
index f68053e8c..eba756555 100644
--- a/glamor/glamor_xv.c
+++ b/glamor/glamor_xv.c
@@ -143,9 +143,8 @@ XvImageRec glamor_xv_images[] = {
 int glamor_xv_num_images = ARRAY_SIZE(glamor_xv_images);
 
 static void
-glamor_init_xv_shader(ScreenPtr screen, int id)
+glamor_init_xv_shader(ScreenPtr screen, glamor_port_private *port_priv, int id)
 {
-    glamor_screen_private *glamor_priv = glamor_get_screen_private(screen);
     GLint sampler_loc;
     const glamor_facet *glamor_facet_xv_planar = NULL;
 
@@ -162,10 +161,10 @@ glamor_init_xv_shader(ScreenPtr screen, int id)
     }
 
     glamor_build_program(screen,
-                         &glamor_priv->xv_prog,
+                         &port_priv->xv_prog,
                          glamor_facet_xv_planar, NULL, NULL, NULL);
 
-    glUseProgram(glamor_priv->xv_prog.prog);
+    glUseProgram(port_priv->xv_prog.prog);
     sampler_loc = glGetUniformLocation(glamor_priv->xv_prog.prog, "y_sampler");
     glUniform1i(sampler_loc, 0);
     sampler_loc = glGetUniformLocation(glamor_priv->xv_prog.prog, "u_sampler");
@@ -330,8 +329,8 @@ glamor_xv_render(glamor_port_private *port_priv, int id)
     char *vbo_offset;
     int dst_box_index;
 
-    if (!glamor_priv->xv_prog.prog)
-        glamor_init_xv_shader(screen, id);
+    if (!port_priv->xv_prog.prog)
+        glamor_init_xv_shader(screen, port_priv, id);
 
     cont = RTFContrast(port_priv->contrast);
     bright = RTFBrightness(port_priv->brightness);
@@ -365,13 +364,13 @@ glamor_xv_render(glamor_port_private *port_priv, int id)
         }
     }
     glamor_make_current(glamor_priv);
-    glUseProgram(glamor_priv->xv_prog.prog);
+    glUseProgram(port_priv->xv_prog.prog);
 
-    uloc = glGetUniformLocation(glamor_priv->xv_prog.prog, "offsetyco");
+    uloc = glGetUniformLocation(port_priv->xv_prog.prog, "offsetyco");
     glUniform4f(uloc, off[0], off[1], off[2], yco);
-    uloc = glGetUniformLocation(glamor_priv->xv_prog.prog, "ucogamma");
+    uloc = glGetUniformLocation(port_priv->xv_prog.prog, "ucogamma");
     glUniform4f(uloc, uco[0], uco[1], uco[2], gamma);
-    uloc = glGetUniformLocation(glamor_priv->xv_prog.prog, "vco");
+    uloc = glGetUniformLocation(port_priv->xv_prog.prog, "vco");
     glUniform4f(uloc, vco[0], vco[1], vco[2], 0);
 
     glActiveTexture(GL_TEXTURE0);
@@ -455,7 +454,7 @@ glamor_xv_render(glamor_port_private *port_priv, int id)
         glamor_set_destination_drawable(port_priv->pDraw,
                                         dst_box_index,
                                         FALSE, FALSE,
-                                        glamor_priv->xv_prog.matrix_uniform,
+                                        port_priv->xv_prog.matrix_uniform,
                                         &dst_off_x, &dst_off_y);
 
         for (i = 0; i < nBox; i++) {
@@ -476,8 +475,25 @@ glamor_xv_render(glamor_port_private *port_priv, int id)
     glDisableVertexAttribArray(GLAMOR_VERTEX_SOURCE);
 
     DamageDamageRegion(port_priv->pDraw, &port_priv->clip);
+}
 
-    glamor_xv_free_port_data(port_priv);
+static Bool
+glamor_xv_can_reuse_port(glamor_port_private *port_priv, int id, short w, short h)
+{
+    int ret = TRUE;
+
+    if (port_priv->prev_fmt != id)
+        ret = FALSE;
+
+    if (w != port_priv->src_pix_w || h != port_priv->src_pix_h)
+        ret = FALSE;
+
+    if (!port_priv->src_pix[0])
+        ret = FALSE;
+
+    port_priv->prev_fmt = id;
+
+    return ret;
 }
 
 int
@@ -495,7 +511,6 @@ glamor_xv_put_image(glamor_port_private *port_priv,
                     RegionPtr clipBoxes)
 {
     ScreenPtr pScreen = pDrawable->pScreen;
-    glamor_screen_private *glamor_priv = glamor_get_screen_private(pScreen);
     int srcPitch, srcPitch2;
     int top, nlines;
     int s2offset, s3offset, tmp;
@@ -503,15 +518,14 @@ glamor_xv_put_image(glamor_port_private *port_priv,
 
     s2offset = s3offset = srcPitch2 = 0;
 
-    if (!port_priv->src_pix[0] ||
-        (width != port_priv->src_pix_w || height != port_priv->src_pix_h) ||
-        (port_priv->src_pix[2] && id == FOURCC_NV12) ||
-        (!port_priv->src_pix[2] && id != FOURCC_NV12)) {
+    if (!glamor_xv_can_reuse_port(port_priv, id, width, height)) {
         int i;
 
-        if (glamor_priv->xv_prog.prog) {
-            glDeleteProgram(glamor_priv->xv_prog.prog);
-            glamor_priv->xv_prog.prog = 0;
+        glamor_xv_free_port_data(port_priv);
+
+        if (port_priv->xv_prog.prog) {
+            glDeleteProgram(port_priv->xv_prog.prog);
+            port_priv->xv_prog.prog = 0;
         }
 
         for (i = 0; i < 3; i++)