use correct color in ascii ari shader

still slow as hell, but it looks much nicer

sorry neobrain :-P

Data/User/Shaders/asciiart.txt

@@ -4,6 +4,7 @@ uniform sampler2D samp9;
 const int char_width = 8;
 const int char_height = 13;
 const int char_count = 95;
+const int char_pixels = char_width*char_height;
 const vec2 char_dim = vec2(char_width, char_height);
 const vec2 font_scale = vec2(1.0/char_width/char_count, 1.0/char_height);
 
@@ -17,43 +18,78 @@ void main()
   vec2 char_pos = floor(uv0*resolution.xy/char_dim);
   vec2 pixel_offset = floor(uv0*resolution.xy) - char_pos*char_dim;
 
-  float minc = 0;
-  float mindiff = char_width*char_height*100;
-  float diffsum[char_count];
-  
-  vec4 color_avg = vec4(0.0, 0.0, 0.0, 0.0);
+  float mindiff = char_width*char_height*100; // just a big number
+  float minc = 0.0;
+  vec4 mina = vec4(0.0, 0.0, 0.0, 0.0);
+  vec4 minb = vec4(0.0, 0.0, 0.0, 0.0);
   
   for(int i=0; i<char_count; i++) {
-    diffsum[i] = 0.0;
-  }
-  for(int x=0; x<char_width; x++) {
-    for(int y=0; y<char_height; y++) {
-      vec2 tex_pos = char_pos*char_dim + vec2(x,y) + 0.5;
-      vec4 tex = texture(samp9, tex_pos * resolution.zw);
-      float tex_f = (tex.x + tex.y + tex.z)/3;
-      color_avg += tex;
+    vec4 ff = vec4(0.0, 0.0, 0.0, 0.0);
+    vec4 f = vec4(0.0, 0.0, 0.0, 0.0);
+    vec4 ft = vec4(0.0, 0.0, 0.0, 0.0);
+    vec4 t = vec4(0.0, 0.0, 0.0, 0.0);
+    vec4 tt = vec4(0.0, 0.0, 0.0, 0.0);
+    
+    for(int x=0; x<char_width; x++) {
+      for(int y=0; y<char_height; y++) {
+      
+	vec2 tex_pos = char_pos*char_dim + vec2(x,y) + 0.5;
+	vec4 tex = texture(samp9, tex_pos * resolution.zw);
       
-      for(int i=0; i<char_count; i++) {
 	vec2 font_pos = vec2(x+i*char_width, y) + 0.5;
-	
 	vec4 font = texture(samp8, font_pos * font_scale);
-	float font_f = font.x;
 	
-	float diff = (font_f-tex_f)*(font_f-tex_f);
-	diffsum[i] = diffsum[i] + diff;
+	// generates sum of texture and font and their squares
+	ff += font*font;
+	f += font;
+	ft += font*tex;
+	t += tex;
+	tt += tex*tex;
       }
     }
-  }
-  for(int i=0; i<char_count; i++) {
-    if(diffsum[i] < mindiff) {
-      mindiff = diffsum[i];
+    
+    /*
+      The next lines are a bit harder, hf :-)
+      
+      The idea is to find the perfect char with the perfect background color and the perfect font color.
+      As this is an equation with three unknowns, we can't just try all chars and color combinations.
+      
+      As criterion how "perfect" the selection is, we compare the "mean squared error" of the resulted colors of all chars.
+      So, now the big issue: how to calculate the MSE without knowing the two colors ...
+      
+      In the next steps, "a" is the font color, "b" is the background color, "f" is the font value at this pixel, "t" is the texture value
+      
+      So the square error of one pixel is: 
+        e = ( t - a⋅f - b⋅(1-f) ) ^ 2
+        
+      In longer:
+        e = a^2⋅f^2 - 2⋅a⋅b⋅f^2 + 2⋅a⋅b⋅f - 2⋅a⋅f⋅t + b^2⋅f^2 - 2⋅b^2⋅f + b^2 + 2⋅b⋅f⋅t - 2⋅b⋅t + t^2
+        
+      The sum of all errors is: (as shortcut, ff,f,ft,t,tt are now the sums like declared above, sum(1) is the count of pixels)
+        sum(e) = a^2⋅ff - 2⋅a^2⋅ff + 2⋅a⋅b⋅f - 2⋅a⋅ft + b^2⋅ff - 2⋅b^2⋅f + b^2⋅sum(1) + 2⋅b⋅ft - 2⋅b⋅t + tt
+        
+      To find the minimum, we have to derive this by "a" and "b":
+        d/da sum(e) = 2⋅a⋅ff + 2⋅b⋅f - 2⋅b⋅ff - 2⋅ft
+        d/db sum(e) = 2⋅a⋅f - 2⋅a⋅ff - 4⋅b⋅f + 2⋅b⋅ff + 2⋅b⋅sum(1) + 2⋅ft - 2⋅t
+      
+      So, both equations must be zero at minimum and there is only one solution.
+    */
+    vec4 a = (f*ft - ff*t + f*t - ft*char_pixels) / (f*f  - ff*char_pixels);
+    vec4 b = (f*ft - ff*t)                        / (f*f  - ff*char_pixels);
+    
+    vec4 diff = a*a*ff + 2.0*a*b*f - 2.0*a*b*ff - 2.0*a*ft + b*b *(-2.0*f + ff + char_pixels) + 2.0*b*ft - 2.0*b*t + tt;
+    float diff_f = dot(diff, vec4(1.0, 1.0, 1.0, 1.0));
+    
+    if(diff_f < mindiff) {
+      mindiff = diff_f;
       minc = i;
+      mina = a;
+      minb = b;
     }
   }
   
-  color_avg /= char_width*char_height;
-  
   vec2 font_pos_res = vec2(minc*char_width, 0) + pixel_offset + 0.5;
 
-  ocol0 = color_avg * texture(samp8, font_pos_res * font_scale);
+  vec4 col = texture(samp8, font_pos_res * font_scale);
+  ocol0 = mina * col + minb * (vec4(1.0,1.0,1.0,1.0) - col);
 }