Bug 316479 - The Perspective Tool creates an empy image...

...instead of transforming it

Add gimp_matrix3_will_explode() which determines if a transform
matrix will blow up something in a rectangle to infinity, and use
the function so set both the GIMP and GEGL code paths to clip the
transform to the input size.

app/core/gimpdrawable-transform.c

@@ -116,6 +116,9 @@ gimp_drawable_transform_buffer_affine (GimpDrawable           *drawable,
       ! babl_format_has_alpha (gegl_buffer_get_format (orig_buffer)))
     clip_result = GIMP_TRANSFORM_RESIZE_CLIP;
 
+  if (gimp_matrix3_will_explode (&m, u1, v1, u2, v2))
+    clip_result = GIMP_TRANSFORM_RESIZE_CLIP;
+
   /*  Find the bounding coordinates of target */
   gimp_transform_resize_boundary (&m, clip_result,
                                   u1, v1, u2, v2,
@@ -133,6 +136,7 @@ gimp_drawable_transform_buffer_affine (GimpDrawable           *drawable,
   gimp_gegl_apply_transform (orig_buffer, progress, NULL,
                              new_buffer,
                              interpolation_type,
+                             clip_result,
                              &gegl_matrix);
 
   *new_offset_x = x1;

app/gegl/gimp-gegl-apply-operation.c

@@ -685,17 +685,22 @@ gimp_gegl_apply_transform (GeglBuffer            *src_buffer,
                            const gchar           *undo_desc,
                            GeglBuffer            *dest_buffer,
                            GimpInterpolationType  interpolation_type,
+                           GimpTransformResize    clip_result,
                            GimpMatrix3           *transform)
 {
   GeglNode *node;
+  gboolean  clip_to_input;
 
   g_return_if_fail (GEGL_IS_BUFFER (src_buffer));
   g_return_if_fail (progress == NULL || GIMP_IS_PROGRESS (progress));
   g_return_if_fail (GEGL_IS_BUFFER (dest_buffer));
 
+  clip_to_input = (clip_result == GIMP_TRANSFORM_RESIZE_CLIP);
+
   node = gegl_node_new_child (NULL,
-                              "operation", "gegl:transform",
-                              "sampler",   interpolation_type,
+                              "operation",     "gegl:transform",
+                              "sampler",       interpolation_type,
+                              "clip-to-input", clip_to_input,
                               NULL);
 
   gimp_gegl_node_set_matrix (node, transform);

app/gegl/gimp-gegl-apply-operation.h

@@ -153,6 +153,7 @@ void   gimp_gegl_apply_transform       (GeglBuffer             *src_buffer,
                                         const gchar            *undo_desc,
                                         GeglBuffer             *dest_buffer,
                                         GimpInterpolationType   interpolation_type,
+                                        GimpTransformResize     clip_result,
                                         GimpMatrix3            *transform);
 
 

libgimpmath/gimpmath.def

@@ -16,6 +16,7 @@ EXPORTS
 	gimp_matrix3_scale
 	gimp_matrix3_transform_point
 	gimp_matrix3_translate
+	gimp_matrix3_will_explode
 	gimp_matrix3_xshear
 	gimp_matrix3_yshear
 	gimp_matrix4_to_deg

libgimpmath/gimpmatrix.c

@@ -865,6 +865,42 @@ gimp_matrix3_is_simple (const GimpMatrix3 *matrix)
   return TRUE;
 }
 
+/**
+ * gimp_matrix3_will_explode:
+ * @m:  The matrix that is to be tested.
+ * @u1: The rectangle's left coordinate.
+ * @v1: The rectangle's top coordinate.
+ * @u2: The rectangle's right coordinate.
+ * @v2: The rectangle's bottom coordinate.
+ *
+ * Checks if the given transformation maps a point of the rectangle to
+ * infinity, or something equally stupid.
+ *
+ * Returns: %TRUE if the transformation will fail, %FALSE otherwise
+ *
+ * Since: 2.10
+ */
+gboolean
+gimp_matrix3_will_explode (const GimpMatrix3 *m,
+                           gdouble            u1,
+                           gdouble            v1,
+                           gdouble            u2,
+                           gdouble            v2)
+{
+  const gdouble a  = m->coeff[2][0];
+  const gdouble b  = m->coeff[2][1];
+  const gdouble c  = m->coeff[2][2];
+  const gdouble d1 = a * u1 + b * v1 + c;
+  const gdouble d2 = a * u1 + b * v2 + c;
+  const gdouble d3 = a * u2 + b * v1 + c;
+  const gdouble d4 = a * u2 + b * v2 + c;
+
+  /* We are safe if all 4 corners of the region are on the same side
+   * of the a.u+b.v+c=0 line, ie. if d1..d4 have the same sign.
+   */
+  return ! (d1 * d2 > 0 && d1 * d3 > 0 && d1 * d4 > 0);
+}
+
 /**
  * gimp_matrix4_to_deg:
  * @matrix:

libgimpmath/gimpmatrix.h

@@ -111,6 +111,12 @@ gboolean      gimp_matrix3_is_diagonal     (const GimpMatrix3 *matrix);
 gboolean      gimp_matrix3_is_affine       (const GimpMatrix3 *matrix);
 gboolean      gimp_matrix3_is_simple       (const GimpMatrix3 *matrix);
 
+gboolean      gimp_matrix3_will_explode    (const GimpMatrix3 *matrix,
+                                            gdouble            u1,
+                                            gdouble            v1,
+                                            gdouble            u2,
+                                            gdouble            v2);
+
 void          gimp_matrix3_transform_point (const GimpMatrix3 *matrix,
                                             gdouble            x,
                                             gdouble            y,