diff --git a/qrcode/image/styles/colormasks.py b/qrcode/image/styles/colormasks.py
index 43186b2..668be4f 100644
--- a/qrcode/image/styles/colormasks.py
+++ b/qrcode/image/styles/colormasks.py
@@ -27,24 +27,31 @@ class QRColorMask:
     def initialize(self, styledPilImage, image):
         self.paint_color = styledPilImage.paint_color
 
-    def apply_mask(self, image):
+    def apply_mask(self, image, use_cache=False):
         width, height = image.size
+        pixels = image.load()
+        fg_color_cache = {} if use_cache else None
         for x in range(width):
             for y in range(height):
-                norm = self.extrap_color(
-                    self.back_color, self.paint_color, image.getpixel((x, y))
-                )
+                current_color = pixels[x, y]
+                if current_color == self.back_color:
+                    continue
+                if use_cache and current_color in fg_color_cache:
+                    pixels[x, y] = fg_color_cache[current_color]
+                    continue
+                norm = self.extrap_color(self.back_color, self.paint_color, current_color)
                 if norm is not None:
-                    image.putpixel(
-                        (x, y),
-                        self.interp_color(
-                            self.get_bg_pixel(image, x, y),
-                            self.get_fg_pixel(image, x, y),
-                            norm,
-                        ),
+                    new_color = self.interp_color(
+                        self.get_bg_pixel(image, x, y),
+                        self.get_fg_pixel(image, x, y),
+                        norm
                     )
+                    pixels[x, y] = new_color
+
+                    if use_cache:
+                        fg_color_cache[current_color] = new_color
                 else:
-                    image.putpixel((x, y), self.get_bg_pixel(image, x, y))
+                    pixels[x, y] = self.get_bg_pixel(image, x, y)
 
     def get_fg_pixel(self, image, x, y):
         raise NotImplementedError("QRModuleDrawer.paint_fg_pixel")
@@ -103,7 +110,7 @@ def apply_mask(self, image):
             # the individual pixel comparisons that the base class uses, which
             # would be a lot faster. (In fact doing this would probably remove
             # the need for the B&W optimization above.)
-            QRColorMask.apply_mask(self, image)
+            QRColorMask.apply_mask(self, image, use_cache=True)
 
     def get_fg_pixel(self, image, x, y):
         return self.front_color