commit 270c27bcfaeab9cc1403e3001eb795551adac62b
Author: Leszek Koltunski <leszek@distoretedandroid.org>
Date:   Fri May 19 16:58:12 2017 +0100

    Progress with Stencils in postprocessing.

diff --git a/src/main/java/org/distorted/library/DistortedEffects.java b/src/main/java/org/distorted/library/DistortedEffects.java
index e2aef60..fb43385 100644
--- a/src/main/java/org/distorted/library/DistortedEffects.java
+++ b/src/main/java/org/distorted/library/DistortedEffects.java
@@ -245,7 +245,7 @@ public class DistortedEffects
 
 ///////////////////////////////////////////////////////////////////////////////////////////////////
 
-  void drawPriv(float halfW, float halfH, MeshObject mesh, DistortedOutputSurface surface, long currTime)
+  void drawPriv(float halfW, float halfH, MeshObject mesh, DistortedOutputSurface surface, long currTime, float marginInPixels)
     {
     mM.compute(currTime);
     mV.compute(currTime);
@@ -277,7 +277,7 @@ public class DistortedEffects
       GLES30.glVertexAttribPointer(mMainProgram.mAttribute[2], MeshObject.TEX_DATA_SIZE, GLES30.GL_FLOAT, false, MeshObject.VERTSIZE, mesh.mVertAttribs);
       }
 
-    mM.send(surface,halfW,halfH,halfZ);
+    mM.send(surface,halfW,halfH,halfZ,marginInPixels);
     mV.send(halfW,halfH,halfZ);
     mF.send(halfW,halfH);
 
diff --git a/src/main/java/org/distorted/library/DistortedNode.java b/src/main/java/org/distorted/library/DistortedNode.java
index 8366e20..38bdf11 100644
--- a/src/main/java/org/distorted/library/DistortedNode.java
+++ b/src/main/java/org/distorted/library/DistortedNode.java
@@ -257,6 +257,26 @@ public class DistortedNode implements DistortedSlave
     if( mParent!=null ) mParent.adjustIsomorphism();
     }
 
+///////////////////////////////////////////////////////////////////////////////////////////////////
+
+  int markStencilAndDraw(long currTime, DistortedOutputSurface surface, float marginInPixels)
+    {
+    DistortedInputSurface input = mNumChildren[0]==0 ? mSurface : mData.mFBO;
+
+    if( input.setAsInput() )
+      {
+      // Mark area of our object + marginInPixels pixels around with 1s in Stencil buffer
+      DistortedRenderState.setUpStencilMark();
+      mEffects.drawPriv(mSurface.getWidth()/2.0f, mSurface.getHeight()/2.0f, mMesh, surface, currTime, marginInPixels);
+
+      // Actually draw our object.
+      mState.apply();
+      mEffects.drawPriv(mSurface.getWidth()/2.0f, mSurface.getHeight()/2.0f, mMesh, surface, currTime, 0);
+      return 1;
+      }
+    return 0;
+    }
+
 ///////////////////////////////////////////////////////////////////////////////////////////////////
 // return the total number of render calls issued
 
@@ -267,7 +287,7 @@ public class DistortedNode implements DistortedSlave
     if( input.setAsInput() )
       {
       mState.apply();
-      mEffects.drawPriv(mSurface.getWidth()/2.0f, mSurface.getHeight()/2.0f, mMesh, surface, currTime);
+      mEffects.drawPriv(mSurface.getWidth()/2.0f, mSurface.getHeight()/2.0f, mMesh, surface, currTime, 0);
       return 1;
       }
 
diff --git a/src/main/java/org/distorted/library/DistortedOutputSurface.java b/src/main/java/org/distorted/library/DistortedOutputSurface.java
index 9897325..31f922c 100644
--- a/src/main/java/org/distorted/library/DistortedOutputSurface.java
+++ b/src/main/java/org/distorted/library/DistortedOutputSurface.java
@@ -80,9 +80,6 @@ abstract class DistortedOutputSurface extends DistortedSurface implements Distor
   private float mClearDepth;
   private int mClearStencil;
   private int mClear;
-
-//private String sNew="", sOld="";
-
   float mMipmap;
 
 ///////////////////////////////////////////////////////////////////////////////////////////////////
@@ -171,16 +168,12 @@ abstract class DistortedOutputSurface extends DistortedSurface implements Distor
     DistortedEffectsPostprocess lastP=null, currP;
     long lastB=0, currB;
 
-//sNew = "";
-
     for(int i=0; i<num; i++)
       {
       child = children.get(i);
       currP = child.getEffectsPostprocess();
       currB = currP==null ? 0 : currP.getBucket();
 
-//sNew += currB;
-
       if( lastB!=currB && lastB!=0 )
         {
         numRenders += lastP.postprocess(time,this);
@@ -204,10 +197,10 @@ abstract class DistortedOutputSurface extends DistortedSurface implements Distor
             mBuffer1[j].glClear(GLES30.GL_COLOR_BUFFER_BIT|GLES30.GL_DEPTH_BUFFER_BIT|GLES30.GL_STENCIL_BUFFER_BIT);
             mipmap *= EffectQuality.MULTIPLIER;
             }
-          DistortedObject.toDo();  // create immediately
+          DistortedObject.toDo(); // create the FBOs immediately. This is safe as we must be holding the OpenGL context now.
           }
 
-        numRenders += child.draw(time,mBuffer1[currP.getQuality()]);
+        numRenders += child.markStencilAndDraw(time,mBuffer1[currP.getQuality()],20);
 
         if( i==num-1 )
           {
@@ -218,13 +211,7 @@ abstract class DistortedOutputSurface extends DistortedSurface implements Distor
       lastP = currP;
       lastB = currB;
       }
-/*
-if( !sNew.equals(sOld) )
-  {
-  sOld = sNew;
-  android.util.Log.e("surface", "Surface"+getID()+": "+sOld);
-  }
-*/
+
     return numRenders;
     }
 
diff --git a/src/main/java/org/distorted/library/DistortedRenderState.java b/src/main/java/org/distorted/library/DistortedRenderState.java
index 97b803a..37311ab 100644
--- a/src/main/java/org/distorted/library/DistortedRenderState.java
+++ b/src/main/java/org/distorted/library/DistortedRenderState.java
@@ -56,6 +56,12 @@ class DistortedRenderState
   private int mBlendSrc, mBlendDst;                                        //
   private int mClear;                                                      // This does not have a 'static' compatriot
 
+  private static int rStencilTest;                                         //
+  private static int rStencilFuncFunc;                                     //
+  private static int rStencilFuncRef;                                      // Remember values of Stencil.
+  private static int rStencilFuncMask;                                     //
+  private static int rStencilMask;                                         //
+
 ///////////////////////////////////////////////////////////////////////////////////////////////////
 // default: color writes on, depth test and writes on, blending on, stencil off.
 
@@ -153,6 +159,112 @@ class DistortedRenderState
     GLES30.glDisable(GLES30.GL_SCISSOR_TEST);
     }
 
+///////////////////////////////////////////////////////////////////////////////////////////////////
+
+  static void setUpStencilMark()
+    {
+    if( sStencilTest!=1 )
+      {
+      sStencilTest = 1;
+      GLES30.glEnable(GLES30.GL_STENCIL_TEST);
+      }
+    if( sStencilFuncFunc!=GLES30.GL_ALWAYS || sStencilFuncRef!=1 || sStencilFuncMask!=STENCIL_MASK )
+      {
+      sStencilFuncFunc = GLES30.GL_ALWAYS;
+      sStencilFuncRef  = 1;
+      sStencilFuncMask = STENCIL_MASK;
+      GLES30.glStencilFunc(sStencilFuncFunc,sStencilFuncRef,sStencilFuncMask);
+      }
+    if( sStencilOpSfail!=GLES30.GL_KEEP || sStencilOpDpfail!=GLES30.GL_KEEP || sStencilOpDppass!=GLES30.GL_REPLACE )
+      {
+      sStencilOpSfail = GLES30.GL_KEEP;
+      sStencilOpDpfail= GLES30.GL_KEEP;
+      sStencilOpDppass= GLES30.GL_REPLACE;
+      GLES30.glStencilOp(sStencilOpSfail,sStencilOpDpfail,sStencilOpDppass);
+      }
+    if( sColorMaskR!=0 || sColorMaskG!=0 || sColorMaskB!=0 || sColorMaskA!=0 )
+      {
+      sColorMaskR = 0;
+      sColorMaskG = 0;
+      sColorMaskB = 0;
+      sColorMaskA = 0;
+      GLES30.glColorMask(false,false,false,false);
+      }
+    if( sDepthMask!=0 )
+      {
+      sDepthMask = 0;
+      GLES30.glDepthMask(false);
+      }
+    if( sStencilMask!= STENCIL_MASK )
+      {
+      sStencilMask = STENCIL_MASK;
+      GLES30.glStencilMask(sStencilMask);
+      }
+    }
+
+///////////////////////////////////////////////////////////////////////////////////////////////////
+
+  static void useStencilMark()
+    {
+    if( sStencilTest!=1 )
+      {
+      rStencilTest = sStencilTest;
+      sStencilTest = 1;
+      GLES30.glEnable(GLES30.GL_STENCIL_TEST);
+      }
+    if( sStencilFuncFunc!=GLES30.GL_EQUAL || sStencilFuncRef!=1 || sStencilFuncMask!=STENCIL_MASK )
+      {
+      rStencilFuncFunc = sStencilFuncFunc;
+      rStencilFuncRef  = sStencilFuncRef;
+      rStencilFuncMask = sStencilFuncMask;
+      sStencilFuncFunc = GLES30.GL_EQUAL;
+      sStencilFuncRef  = 1;
+      sStencilFuncMask = STENCIL_MASK;
+      GLES30.glStencilFunc(sStencilFuncFunc,sStencilFuncRef,sStencilFuncMask);
+      }
+
+    // TODO: Debatable
+    if( sStencilMask!= 0x00 )
+      {
+      rStencilMask = sStencilMask;
+      sStencilMask = 0x00;
+      GLES30.glStencilMask(sStencilMask);
+      }
+    }
+
+///////////////////////////////////////////////////////////////////////////////////////////////////
+
+  static void restoreStencilMark()
+    {
+    if( rStencilTest!=sStencilTest )
+      {
+      sStencilTest = rStencilTest;
+
+      if (sStencilTest == 0)
+        {
+        GLES30.glDisable(GLES30.GL_STENCIL_TEST);
+        }
+      else
+        {
+        GLES30.glEnable(GLES30.GL_STENCIL_TEST);
+        }
+      }
+
+    if( rStencilFuncFunc!=sStencilFuncFunc || rStencilFuncRef!=sStencilFuncRef || rStencilFuncMask!=sStencilFuncMask )
+      {
+      sStencilFuncFunc = rStencilFuncFunc;
+      sStencilFuncRef  = rStencilFuncRef ;
+      sStencilFuncMask = rStencilFuncMask;
+      GLES30.glStencilFunc(sStencilFuncFunc,sStencilFuncRef,sStencilFuncMask);
+      }
+
+    if( rStencilMask!=sStencilMask )
+      {
+      sStencilMask = rStencilMask;
+      GLES30.glStencilMask(sStencilMask);
+      }
+    }
+
 ///////////////////////////////////////////////////////////////////////////////////////////////////
 
   void apply()
diff --git a/src/main/java/org/distorted/library/EffectQueueMatrix.java b/src/main/java/org/distorted/library/EffectQueueMatrix.java
index 305d5bf..f6780d1 100644
--- a/src/main/java/org/distorted/library/EffectQueueMatrix.java
+++ b/src/main/java/org/distorted/library/EffectQueueMatrix.java
@@ -105,74 +105,99 @@ class EffectQueueMatrix extends EffectQueue
 
 ///////////////////////////////////////////////////////////////////////////////////////////////////
 
-  private void magnify(float halfX, float halfY, float halfZ, DistortedOutputSurface projection)
+  private void magnify(DistortedOutputSurface projection, float halfX, float halfY, float halfZ, float marginInPixels)
     {
-    float scale;
+    float scale, nx, ny;
     float[] result= new float[4];
     float[] point = new float[4];
     float[] matrix= new float[16];
-    double d1, d2, d3, d4, d5, d6, d7, d8;
-    float nx, ny;
-    float w = projection.mWidth/2;
-    float h = projection.mHeight/2;
-    double sum = 0.0;
-    point[3] = 0.0f;
+    float minx = Integer.MAX_VALUE;
+    float maxx = Integer.MIN_VALUE;
+    float miny = Integer.MAX_VALUE;
+    float maxy = Integer.MIN_VALUE;
+
+    point[3] = 1.0f;
 
     Matrix.multiplyMM(matrix, 0, projection.mProjectionMatrix, 0, mViewMatrix, 0);
 
     point[0] = +halfX; point[1] = +halfY; point[2] = +halfZ;
     Matrix.multiplyMV(result,0,matrix,0,point,0);
-    nx = w*result[0]/result[3];
-    ny = h*result[1]/result[3];
-    d1 = Math.sqrt(nx*nx+ny*ny);
+    nx = result[0]/result[3];
+    ny = result[1]/result[3];
+    if( nx<minx ) minx = nx;
+    if( nx>maxx ) maxx = nx;
+    if( ny<miny ) miny = ny;
+    if( ny>maxy ) maxy = ny;
 
     point[0] = +halfX; point[1] = +halfY; point[2] = -halfZ;
     Matrix.multiplyMV(result,0,matrix,0,point,0);
-    nx = w*result[0]/result[3];
-    ny = h*result[1]/result[3];
-    d2 = Math.sqrt(nx*nx+ny*ny);
+    nx = result[0]/result[3];
+    ny = result[1]/result[3];
+    if( nx<minx ) minx = nx;
+    if( nx>maxx ) maxx = nx;
+    if( ny<miny ) miny = ny;
+    if( ny>maxy ) maxy = ny;
 
     point[0] = +halfX; point[1] = -halfY; point[2] = +halfZ;
     Matrix.multiplyMV(result,0,matrix,0,point,0);
-    nx = w*result[0]/result[3];
-    ny = h*result[1]/result[3];
-    d3 = Math.sqrt(nx*nx+ny*ny);
+    nx = result[0]/result[3];
+    ny = result[1]/result[3];
+    if( nx<minx ) minx = nx;
+    if( nx>maxx ) maxx = nx;
+    if( ny<miny ) miny = ny;
+    if( ny>maxy ) maxy = ny;
 
     point[0] = +halfX; point[1] = -halfY; point[2] = -halfZ;
     Matrix.multiplyMV(result,0,matrix,0,point,0);
-    nx = w*result[0]/result[3];
-    ny = h*result[1]/result[3];
-    d4 = Math.sqrt(nx*nx+ny*ny);
+    nx = result[0]/result[3];
+    ny = result[1]/result[3];
+    if( nx<minx ) minx = nx;
+    if( nx>maxx ) maxx = nx;
+    if( ny<miny ) miny = ny;
+    if( ny>maxy ) maxy = ny;
 
     point[0] = -halfX; point[1] = +halfY; point[2] = +halfZ;
     Matrix.multiplyMV(result,0,matrix,0,point,0);
-    nx = w*result[0]/result[3];
-    ny = h*result[1]/result[3];
-    d5 = Math.sqrt(nx*nx+ny*ny);
+    nx = result[0]/result[3];
+    ny = result[1]/result[3];
+    if( nx<minx ) minx = nx;
+    if( nx>maxx ) maxx = nx;
+    if( ny<miny ) miny = ny;
+    if( ny>maxy ) maxy = ny;
 
     point[0] = -halfX; point[1] = +halfY; point[2] = -halfZ;
     Matrix.multiplyMV(result,0,matrix,0,point,0);
-    nx = w*result[0]/result[3];
-    ny = h*result[1]/result[3];
-    d6 = Math.sqrt(nx*nx+ny*ny);
+    nx = result[0]/result[3];
+    ny = result[1]/result[3];
+    if( nx<minx ) minx = nx;
+    if( nx>maxx ) maxx = nx;
+    if( ny<miny ) miny = ny;
+    if( ny>maxy ) maxy = ny;
 
     point[0] = -halfX; point[1] = -halfY; point[2] = +halfZ;
     Matrix.multiplyMV(result,0,matrix,0,point,0);
-    nx = w*result[0]/result[3];
-    ny = h*result[1]/result[3];
-    d7 = Math.sqrt(nx*nx+ny*ny);
+    nx = result[0]/result[3];
+    ny = result[1]/result[3];
+    if( nx<minx ) minx = nx;
+    if( nx>maxx ) maxx = nx;
+    if( ny<miny ) miny = ny;
+    if( ny>maxy ) maxy = ny;
 
     point[0] = -halfX; point[1] = -halfY; point[2] = -halfZ;
     Matrix.multiplyMV(result,0,matrix,0,point,0);
-    nx = w*result[0]/result[3];
-    ny = h*result[1]/result[3];
-    d8 = Math.sqrt(nx*nx+ny*ny);
+    nx = result[0]/result[3];
+    ny = result[1]/result[3];
+    if( nx<minx ) minx = nx;
+    if( nx>maxx ) maxx = nx;
+    if( ny<miny ) miny = ny;
+    if( ny>maxy ) maxy = ny;
 
-    sum = (d1+d2+d3+d4+d5+d6+d7+d8)/8;
+    float xlen = projection.mWidth *(maxx-minx)/2;
+    float ylen = projection.mHeight*(maxy-miny)/2;
 
-    scale = 1.0f + (float)(Distorted.mMargin/sum);
+    scale = 1.0f + marginInPixels/( xlen>ylen ? ylen:xlen );
 
-    android.util.Log.d("scale", "scale="+scale+" sum="+sum+" "+d1+" "+d2+" "+d3+" "+d4+" "+d5+" "+d6+" "+d7+" "+d8);
+   // android.util.Log.d("scale", "scale= "+scale+" xlen="+xlen+" ylen="+ylen);
 
     Matrix.scaleM(mViewMatrix, 0, scale, scale, scale);
     }
@@ -180,7 +205,7 @@ class EffectQueueMatrix extends EffectQueue
 ///////////////////////////////////////////////////////////////////////////////////////////////////
 // here construct the ModelView and the ModelViewProjection Matrices
 
-  private void constructMatrices(DistortedOutputSurface projection, float halfX, float halfY, float halfZ)
+  private void constructMatrices(DistortedOutputSurface projection, float halfX, float halfY, float halfZ, float marginInPixels)
     {
     Matrix.setIdentityM(mViewMatrix, 0);
     Matrix.translateM(mViewMatrix, 0, -projection.mWidth/2, projection.mHeight/2, -projection.mDistance);
@@ -261,7 +286,7 @@ class EffectQueueMatrix extends EffectQueue
 
     Matrix.translateM(mViewMatrix, 0, halfX,-halfY,-halfZ);
 
-    if( Distorted.mMargin!=0 ) magnify(halfX,halfY,halfZ,projection);
+    if( marginInPixels!=0 ) magnify(projection,halfX,halfY,halfZ, marginInPixels);
 
     Matrix.multiplyMM(mMVPMatrix, 0, projection.mProjectionMatrix, 0, mViewMatrix, 0);
     }
@@ -336,9 +361,9 @@ class EffectQueueMatrix extends EffectQueue
 
 ///////////////////////////////////////////////////////////////////////////////////////////////////
 
-  synchronized void send(DistortedOutputSurface projection, float halfX, float halfY, float halfZ)
+  synchronized void send(DistortedOutputSurface projection, float halfX, float halfY, float halfZ, float marginInPixels)
     {
-    constructMatrices(projection,halfX,halfY,halfZ);
+    constructMatrices(projection,halfX,halfY,halfZ, marginInPixels);
 
     GLES30.glUniform3f( mObjDH , halfX, halfY, halfZ);
     GLES30.glUniformMatrix4fv(mMVMatrixH , 1, false, mViewMatrix, 0);