1 /**
2 * A GUIGraphics is the interface between a plugin client and a IWindow.
3 *
4 * Copyright: Copyright Auburn Sounds 2015 and later.
5 * License:   $(LINK2 http://www.boost.org/LICENSE_1_0.txt, Boost License 1.0)
6 * Authors:   Guillaume Piolat
7 */
8 module dplug.gui.graphics;
9 
10 import std.math;
11 import std.algorithm.comparison;
12 import std.algorithm.sorting;
13 import std.algorithm.mutation;
14 
15 import dplug.core.math;
16 import dplug.core.thread;
17 
18 import dplug.client.client;
19 import dplug.client.graphics;
20 import dplug.client.daw;
21 
22 import dplug.window.window;
23 
24 import dplug.graphics.mipmap;
25 
26 import dplug.gui.boxlist;
27 import dplug.gui.context;
28 import dplug.gui.element;
29 import dplug.gui.compositor;
30 
31 /// In the whole package:
32 /// The diffuse maps contains:
33 ///   RGBA = red/green/blue/emissiveness
34 /// The depth maps contains depth.
35 /// The material map contains:
36 ///   RGBA = roughness / metalness / specular / physical (allows to bypass PBR)
37 
38 alias RMSP = RGBA; // reminder
39 
40 // Uncomment to benchmark compositing and devise optimizations.
41 //version = BenchmarkCompositing;
42 
43 // A GUIGraphics is the interface between a plugin client and a IWindow.
44 // It is also an UIElement and the root element of the plugin UI hierarchy.
45 // You have to derive it to have a GUI.
46 // It dispatches window events to the GUI hierarchy.
47 class GUIGraphics : UIElement, IGraphics
48 {
49 nothrow:
50 @nogc:
51 
52     ICompositor compositor;
53 
54     this(int initialWidth, int initialHeight)
55     {
56         _uiContext = mallocNew!UIContext();
57         super(_uiContext);
58 
59         // Don't like the default rendering? Make another compositor.
60         compositor = mallocNew!PBRCompositor();
61 
62         _windowListener = mallocNew!WindowListener(this);
63 
64         _window = null;
65         _askedWidth = initialWidth;
66         _askedHeight = initialHeight;
67 
68         _threadPool = mallocNew!ThreadPool();
69 
70         _areasToUpdateNonOverlapping = makeVec!box2i;
71         _areasToUpdateTemp = makeVec!box2i;
72 
73         _updateRectScratch[0] = makeVec!box2i;
74         _updateRectScratch[1] = makeVec!box2i;
75 
76         _areasToRender = makeVec!box2i;
77         _areasToRenderNonOverlapping = makeVec!box2i;
78         _areasToRenderNonOverlappingTiled = makeVec!box2i;
79 
80         _elemsToDraw = makeVec!UIElement;
81 
82         version(BenchmarkCompositing)
83         {
84             _compositingWatch = new StopWatch("Compositing = ");
85             _drawWatch = new StopWatch("Draw = ");
86             _mipmapWatch = new StopWatch("Mipmap = ");
87         }
88 
89         _diffuseMap = mallocNew!(Mipmap!RGBA)();
90         _materialMap = mallocNew!(Mipmap!RGBA)();
91         _depthMap = mallocNew!(Mipmap!L16)();
92     }
93 
94 
95     ~this()
96     {
97         closeUI();
98         _uiContext.destroyFree();
99 
100         _threadPool.destroyFree();
101 
102         compositor.destroyFree();
103         _diffuseMap.destroyFree();
104         _materialMap.destroyFree();
105         _depthMap.destroyFree();
106 
107         _windowListener.destroyFree();
108         alignedFree(_renderedBuffer, 16);
109     }
110 
111     // Graphics implementation
112 
113     override void* openUI(void* parentInfo, void* controlInfo, DAW daw, GraphicsBackend backend)
114     {
115         WindowBackend wbackend = void;
116         final switch(backend)
117         {
118             case GraphicsBackend.autodetect: wbackend = WindowBackend.autodetect; break;
119             case GraphicsBackend.win32: wbackend = WindowBackend.win32; break;
120             case GraphicsBackend.cocoa: wbackend = WindowBackend.cocoa; break;
121             case GraphicsBackend.carbon: wbackend = WindowBackend.carbon; break;
122             case GraphicsBackend.x11: wbackend = WindowBackend.x11; break;
123         }
124 
125         // We create this window each time.
126         _window = createWindow(WindowUsage.plugin, parentInfo, controlInfo, _windowListener, wbackend, _askedWidth, _askedHeight);
127 
128         reflow(box2i(0, 0, _askedWidth, _askedHeight));
129 
130         // Sets the whole UI dirty
131         setDirtyWhole();
132 
133         return _window.systemHandle();
134     }
135 
136     override void closeUI()
137     {
138         // Destroy window.
139         if (_window !is null)
140         {
141             _window.destroyFree();
142             _window = null;
143         }
144     }
145 
146     override void getGUISize(int* width, int* height)
147     {
148         *width = _askedWidth;
149         *height = _askedHeight;
150     }
151 
152     version(BenchmarkCompositing)
153     {
154         class StopWatch
155         {
156             this(string title)
157             {
158                 _title = title;
159             }
160 
161             void start()
162             {
163                 _lastTime = _window.getTimeMs();
164             }
165 
166             enum WARMUP = 30;
167 
168             void stop()
169             {
170                 uint now = _window.getTimeMs();
171                 int timeDiff = cast(int)(now - _lastTime);
172 
173                 if (times >= WARMUP)
174                     sum += timeDiff; // first samples are discarded
175 
176                 times++;
177                 string msg = _title ~ to!string(timeDiff) ~ " ms";
178                 _window.debugOutput(msg);
179             }
180 
181             void displayMean()
182             {
183                 if (times > WARMUP)
184                 {
185                     string msg = _title ~ to!string(sum / (times - WARMUP)) ~ " ms mean";
186                     _window.debugOutput(msg);
187                 }
188             }
189 
190             string _title;
191             uint _lastTime;
192             double sum = 0;
193             int times = 0;
194         }
195     }
196 
197 
198     // This class is only here to avoid name conflicts between
199     // UIElement and IWindowListener methods :|
200     // Explicit outer to avoid emplace crashing
201     static class WindowListener : IWindowListener
202     {
203     nothrow:
204     @nogc:
205         GUIGraphics outer;
206 
207         this(GUIGraphics outer)
208         {
209             this.outer = outer;
210         }
211 
212         override bool onMouseClick(int x, int y, MouseButton mb, bool isDoubleClick, MouseState mstate)
213         {
214             return outer.mouseClick(x, y, mb, isDoubleClick, mstate);
215         }
216 
217         override bool onMouseRelease(int x, int y, MouseButton mb, MouseState mstate)
218         {
219             outer.mouseRelease(x, y, mb, mstate);
220             return true;
221         }
222 
223         override bool onMouseWheel(int x, int y, int wheelDeltaX, int wheelDeltaY, MouseState mstate)
224         {
225             return outer.mouseWheel(x, y, wheelDeltaX, wheelDeltaY, mstate);
226         }
227 
228         override void onMouseMove(int x, int y, int dx, int dy, MouseState mstate)
229         {
230             outer.mouseMove(x, y, dx, dy, mstate);
231         }
232 
233         override void recomputeDirtyAreas()
234         {
235             return outer.recomputeDirtyAreas();
236         }
237 
238         override bool onKeyDown(Key key)
239         {
240             // Sends the event to the last clicked element first
241             if (outer._uiContext.focused !is null)
242                 if (outer._uiContext.focused.onKeyDown(key))
243                     return true;
244 
245             // else to all Elements
246             return outer.keyDown(key);
247         }
248 
249         override bool onKeyUp(Key key)
250         {
251             // Sends the event to the last clicked element first
252             if (outer._uiContext.focused !is null)
253                 if (outer._uiContext.focused.onKeyUp(key))
254                     return true;
255             // else to all Elements
256             return outer.keyUp(key);
257         }
258 
259         /// Returns areas affected by updates.
260         override box2i getDirtyRectangle() nothrow @nogc
261         {
262             return outer._areasToRender[].boundingBox();
263         }
264 
265         override ImageRef!RGBA onResized(int width, int height)
266         {
267             return outer.doResize(width, height);          
268         }
269 
270         // Redraw dirtied controls in depth and diffuse maps.
271         // Update composited cache.
272         override void onDraw(WindowPixelFormat pf) nothrow @nogc
273         {
274             return outer.doDraw(pf);
275         }
276 
277         override void onMouseCaptureCancelled()
278         {
279             // Stop an eventual drag operation
280             outer._uiContext.stopDragging();
281         }
282 
283         override void onAnimate(double dt, double time)
284         {
285             outer.animate(dt, time);
286         }
287     }
288 
289     /// Tune this to tune the trade-off between light quality and speed.
290     /// The default value was tuned by hand on very shiny light sources.
291     /// Too high and processing becomes very expensive.
292     /// Too little and the ligth decay doesn't feel natural.
293     void setUpdateMargin(int margin = 20) nothrow @nogc
294     {
295         _updateMargin = margin;
296     }
297 
298 protected:
299 
300     UIContext _uiContext;
301 
302     WindowListener _windowListener;
303 
304     // An interface to the underlying window
305     IWindow _window;
306 
307     // Task pool for multi-threaded image work
308     ThreadPool _threadPool;
309 
310     int _askedWidth = 0;
311     int _askedHeight = 0;
312 
313     // Diffuse color values for the whole UI.
314     Mipmap!RGBA _diffuseMap;
315 
316     // Depth values for the whole UI.
317     Mipmap!L16 _depthMap;
318 
319     // Depth values for the whole UI.
320     Mipmap!RGBA _materialMap;
321 
322     // The list of areas whose diffuse/depth data have been changed.
323     Vec!box2i _areasToUpdateNonOverlapping;
324 
325     // Used to maintain the _areasToUpdate invariant of no overlap
326     Vec!box2i _areasToUpdateTemp;
327 
328     // Same, but temporary variable for mipmap generation
329     Vec!box2i[2] _updateRectScratch;
330 
331     // The list of areas that must be effectively updated in the composite buffer
332     // (sligthly larger than _areasToUpdate).
333     Vec!box2i _areasToRender;
334 
335     // same list, but reorganized to avoid overlap
336     Vec!box2i _areasToRenderNonOverlapping;
337 
338     // same list, but separated in smaller tiles
339     Vec!box2i _areasToRenderNonOverlappingTiled;
340 
341     // The list of UIElement to draw
342     // Note: AlignedBuffer memory isn't scanned,
343     //       but this doesn't matter since UIElement are the UI hierarchy anyway.
344     Vec!UIElement _elemsToDraw;
345 
346     /// Amount of pixels dirty rectangles are extended with.
347     int _updateMargin = 20;
348 
349     // The fully rendered framebuffer.
350     // This should point into comitted virtual memory for faster (maybe) upload to device
351     ubyte* _renderedBuffer = null;
352 
353     version(BenchmarkCompositing)
354     {
355         StopWatch _compositingWatch;
356         StopWatch _mipmapWatch;
357         StopWatch _drawWatch;
358     }
359 
360     void doDraw(WindowPixelFormat pf) nothrow @nogc
361     {
362         ImageRef!RGBA wfb;
363         wfb.w = _askedWidth;
364         wfb.h = _askedHeight;
365         wfb.pitch = byteStride(_askedWidth);
366         wfb.pixels = cast(RGBA*)_renderedBuffer;
367 
368         // Composite GUI
369         // Most of the cost of rendering is here
370         version(BenchmarkCompositing)
371             _drawWatch.start();
372 
373         renderElements();
374 
375         version(BenchmarkCompositing)
376         {
377             _drawWatch.stop();
378             _drawWatch.displayMean();
379         }
380 
381         version(BenchmarkCompositing)
382             _mipmapWatch.start();
383 
384         // Split boxes to avoid overlapped work
385         // Note: this is done separately for update areas and render areas
386         _areasToRenderNonOverlapping.clearContents();
387         removeOverlappingAreas(_areasToRender, _areasToRenderNonOverlapping);
388 
389         regenerateMipmaps();
390 
391         version(BenchmarkCompositing)
392         {
393             _mipmapWatch.stop();
394             _mipmapWatch.displayMean();
395         }
396 
397         version(BenchmarkCompositing)
398             _compositingWatch.start();
399 
400         compositeGUI(wfb, pf);
401 
402         // only then is the list of rectangles to update cleared
403         _areasToUpdateNonOverlapping.clearContents();
404 
405         // wait for compositing completion
406         //_threadPool.waitForCompletion();
407 
408         version(BenchmarkCompositing)
409         {
410             _compositingWatch.stop();
411             _compositingWatch.displayMean();
412         }
413     }
414 
415     // Fills _areasToUpdate and _areasToRender
416     void recomputeDirtyAreas() nothrow @nogc
417     {
418         // Get areas to update
419         _areasToRender.clearContents();
420 
421         // First we pull dirty rectangles from the UI
422         context().dirtyList.pullAllRectangles(_areasToUpdateNonOverlapping);
423 
424         // TECHNICAL DEBT HERE
425         // The problem here is that if the window isn't shown there may be duplicates in
426         // _areasToUpdate, so we have to maintain unicity again
427         // The code with dirty rects is a big mess, it needs a severe rewrite.
428         //
429         // SOLUTION
430         // The fundamental problem is that dirtyList should probably be merged with
431         // _areasToUpdateNonOverlapping.
432         // _areasToRender should also be purely derived from _areasToUpdateNonOverlapping
433         // Finally the interface of IWindowListener is poorly defined, this ties the window
434         // to the renderer in a bad way.
435         {
436             _areasToUpdateTemp.clearContents();
437             removeOverlappingAreas(_areasToUpdateNonOverlapping, _areasToUpdateTemp);
438             _areasToUpdateNonOverlapping.clearContents();
439             _areasToUpdateNonOverlapping.pushBack(_areasToUpdateTemp);
440         }
441         assert(haveNoOverlap(_areasToUpdateNonOverlapping[]));
442 
443         foreach(dirtyRect; _areasToUpdateNonOverlapping)
444         {
445             assert(dirtyRect.isSorted);
446             assert(!dirtyRect.empty);
447             _areasToRender.pushBack( extendsDirtyRect(dirtyRect, _askedWidth, _askedHeight) );
448         }
449     }
450 
451     box2i extendsDirtyRect(box2i rect, int width, int height) nothrow @nogc
452     {
453         int xmin = rect.min.x - _updateMargin;
454         int ymin = rect.min.y - _updateMargin;
455         int xmax = rect.max.x + _updateMargin;
456         int ymax = rect.max.y + _updateMargin;
457 
458         if (xmin < 0) xmin = 0;
459         if (ymin < 0) ymin = 0;
460         if (xmax > width) xmax = width;
461         if (ymax > height) ymax = height;
462 
463         // This could also happen if an UIElement is moved quickly
464         if (xmax < 0) xmax = 0;
465         if (ymax < 0) ymax = 0;
466         if (xmin > width) xmin = width;
467         if (ymin > height) ymin = height;
468 
469         box2i result = box2i(xmin, ymin, xmax, ymax);
470         assert(result.isSorted);
471         return result;
472     }
473 
474     ImageRef!RGBA doResize(int width, int height) nothrow @nogc
475     {
476         _askedWidth = width;
477         _askedHeight = height;
478 
479         reflow(box2i(0, 0, _askedWidth, _askedHeight));
480 
481         // FUTURE: maybe not destroy the whole mipmap?
482         _diffuseMap.size(5, width, height);
483         _depthMap.size(4, width, height);
484         _materialMap.size(0, width, height);
485 
486         // Extends buffer
487         size_t sizeNeeded = byteStride(width) * height;
488         _renderedBuffer = cast(ubyte*) alignedRealloc(_renderedBuffer, sizeNeeded, 16);
489 
490         ImageRef!RGBA wfb;
491         wfb.w = _askedWidth;
492         wfb.h = _askedHeight;
493         wfb.pitch = byteStride(_askedWidth);
494         wfb.pixels = cast(RGBA*)(_renderedBuffer);
495         return wfb;
496     }
497 
498     /// Redraw UIElements
499     void renderElements() nothrow @nogc
500     {
501         // recompute draw list
502         _elemsToDraw.clearContents();
503         getDrawList(_elemsToDraw);
504 
505         // Sort by ascending z-order (high z-order gets drawn last)
506         // This sort must be stable to avoid messing with tree natural order.
507         int compareZOrder(in UIElement a, in UIElement b) nothrow @nogc
508         {
509             return a.zOrder() - b.zOrder();
510         }
511         grailSort!UIElement(_elemsToDraw[], &compareZOrder);
512 
513         enum bool parallelDraw = true;
514 
515         auto diffuseRef = _diffuseMap.levels[0].toRef();
516         auto depthRef = _depthMap.levels[0].toRef();
517         auto materialRef = _materialMap.levels[0].toRef();
518 
519         static if (parallelDraw)
520         {
521             int drawn = 0;
522             int maxParallelElements = 32;
523             int N = cast(int)_elemsToDraw.length;
524 
525             while(drawn < N)
526             {
527                 int canBeDrawn = 1; // at least one can be drawn without collision
528 
529                 // Search max number of parallelizable draws until the end of the list or a collision is found
530                 bool foundIntersection = false;
531                 for ( ; (canBeDrawn < maxParallelElements) && (drawn + canBeDrawn < N); ++canBeDrawn)
532                 {
533                     box2i candidate = _elemsToDraw[drawn + canBeDrawn].position;
534 
535                     for (int j = 0; j < canBeDrawn; ++j)
536                     {
537                         if (_elemsToDraw[drawn + j].position.intersects(candidate))
538                         {
539                             foundIntersection = true;
540                             break;
541                         }
542                     }
543                     if (foundIntersection)
544                         break;
545                 }
546 
547                 assert(canBeDrawn >= 1 && canBeDrawn <= maxParallelElements);
548  
549                 // Draw a number of UIElement in parallel
550                 void drawOneItem(int i) nothrow @nogc
551                 {
552                     _elemsToDraw[drawn + i].render(diffuseRef, depthRef, materialRef, _areasToUpdateNonOverlapping[]);
553                 }
554                 //_threadPool.waitForCompletion();
555                 _threadPool.parallelFor/*Async*/(canBeDrawn, &drawOneItem);
556 
557                 drawn += canBeDrawn;
558                 assert(drawn <= N);
559             }
560             assert(drawn == N);
561         }
562         else
563         {
564             // Render required areas in diffuse and depth maps, base level
565             foreach(elem; _elemsToDraw)
566                 elem.render(diffuseRef, depthRef, _areasToUpdate[]);
567         }
568     }
569 
570     /// Compose lighting effects from depth and diffuse into a result.
571     /// takes output image and non-overlapping areas as input
572     /// Useful multithreading code.
573     void compositeGUI(ImageRef!RGBA wfb, WindowPixelFormat pf) nothrow @nogc
574     {
575         // Was tuned for performance, maybe the tradeoff has changed now that we use LDC.
576         enum tileWidth = 64;
577         enum tileHeight = 32;
578 
579         _areasToRenderNonOverlappingTiled.clearContents();
580         tileAreas(_areasToRenderNonOverlapping[], tileWidth, tileHeight,_areasToRenderNonOverlappingTiled);
581 
582         int numAreas = cast(int)_areasToRenderNonOverlappingTiled.length;
583 
584         void compositeOneTile(int i) nothrow @nogc
585         {
586             compositor.compositeTile(wfb, pf, _areasToRenderNonOverlappingTiled[i],
587                                      _diffuseMap, _materialMap, _depthMap, context.skybox);
588         }
589         //_threadPool.waitForCompletion(); // wait for mipmaps to be generated
590         _threadPool.parallelFor/*Async*/(numAreas, &compositeOneTile);
591     }
592 
593     /// Compose lighting effects from depth and diffuse into a result.
594     /// takes output image and non-overlapping areas as input
595     /// Useful multithreading code.
596     void regenerateMipmaps() nothrow @nogc
597     {
598         int numAreas = cast(int)_areasToUpdateNonOverlapping.length;
599 
600         // Fill update rect buffer with the content of _areasToUpdateNonOverlapping
601         for (int i = 0; i < 2; ++i)
602         {
603             _updateRectScratch[i].clearContents();
604             _updateRectScratch[i].pushBack(_areasToUpdateNonOverlapping[]);
605         }
606 
607         // We can't use tiled parallelism for mipmapping here because there is overdraw beyond level 0
608         // So instead what we do is using up to 2 threads.
609         void processOneMipmap(int i) nothrow @nogc
610         {
611             if (i == 0)
612             {
613                 // diffuse
614                 Mipmap!RGBA mipmap = _diffuseMap;
615                 int levelMax = min(mipmap.numLevels(), 5);
616                 foreach(level; 1 .. mipmap.numLevels())
617                 {
618                     Mipmap!RGBA.Quality quality;
619                     if (level == 1)
620                         quality = Mipmap!RGBA.Quality.boxAlphaCovIntoPremul;
621                     else
622                         quality = Mipmap!RGBA.Quality.cubic;
623 
624                     foreach(ref area; _updateRectScratch[i])
625                     {
626                         area = mipmap.generateNextLevel(quality, area, level);
627                     }
628                 }
629             }
630             else
631             {
632                 // depth
633                 Mipmap!L16 mipmap = _depthMap;
634                 foreach(level; 1 .. mipmap.numLevels())
635                 {
636                     auto quality = level >= 3 ? Mipmap!L16.Quality.cubic : Mipmap!L16.Quality.box;
637                     foreach(ref area; _updateRectScratch[i])
638                     {
639                         area = mipmap.generateNextLevel(quality, area, level);
640                     }
641                 }
642             }
643         }
644         //_threadPool.waitForCompletion(); // wait for UI widgets to be drawn
645         _threadPool.parallelFor/*Async*/(2, &processOneMipmap);
646     }
647 }
648 
649 
650 enum scanLineAlignment = 4; // could be anything
651 
652 // given a width, how long in bytes should scanlines be
653 int byteStride(int width) pure nothrow @nogc
654 {
655     int widthInBytes = width * 4;
656     return (widthInBytes + (scanLineAlignment - 1)) & ~(scanLineAlignment-1);
657 }