1 /**
2 * Copyright: Copyright Auburn Sounds 2015 and later.
3 * License:   $(LINK2 http://www.boost.org/LICENSE_1_0.txt, Boost License 1.0)
4 * Authors:   Guillaume Piolat
5 */
6 module dplug.gui.graphics;
7 
8 import std.math;
9 import std.algorithm.comparison;
10 import std.algorithm.sorting;
11 import std.algorithm.mutation;
12 
13 import dplug.core.math;
14 import dplug.core.thread;
15 
16 import dplug.client.client;
17 import dplug.client.graphics;
18 import dplug.client.daw;
19 
20 import dplug.window.window;
21 
22 import dplug.graphics.mipmap;
23 
24 import dplug.gui.boxlist;
25 import dplug.gui.context;
26 import dplug.gui.element;
27 import dplug.gui.compositor;
28 
29 /// In the whole package:
30 /// The diffuse maps contains:
31 ///   RGBA = red/green/blue/emissiveness
32 /// The depth maps contains depth.
33 /// The material map contains:
34 ///   RGBA = roughness / metalness / specular / physical (allows to bypass PBR)
35 
36 alias RMSP = RGBA; // reminder
37 
38 // Uncomment to benchmark compositing and devise optimizations.
39 //version = BenchmarkCompositing;
40 
41 // A GUIGraphics is the interface between a plugin client and a IWindow.
42 // It is also an UIElement and the root element of the plugin UI hierarchy.
43 // You have to derive it to have a GUI.
44 // It dispatches window events to the GUI hierarchy.
45 class GUIGraphics : UIElement, IGraphics
46 {
47 nothrow:
48 @nogc:
49 
50     ICompositor compositor;
51 
52     this(int initialWidth, int initialHeight)
53     {
54         _uiContext = mallocEmplace!UIContext();
55         super(_uiContext);
56 
57         // Don't like the default rendering? Make another compositor.
58         compositor = mallocEmplace!PBRCompositor();
59 
60         _windowListener = mallocEmplace!WindowListener(this);
61 
62         _window = null;
63         _askedWidth = initialWidth;
64         _askedHeight = initialHeight;
65 
66         _threadPool = mallocEmplace!ThreadPool();
67 
68         _areasToUpdateNonOverlapping = makeAlignedBuffer!box2i;
69         _areasToUpdateTemp = makeAlignedBuffer!box2i;
70 
71         _updateRectScratch[0] = makeAlignedBuffer!box2i;
72         _updateRectScratch[1] = makeAlignedBuffer!box2i;
73 
74         _areasToRender = makeAlignedBuffer!box2i;
75         _areasToRenderNonOverlapping = makeAlignedBuffer!box2i;
76         _areasToRenderNonOverlappingTiled = makeAlignedBuffer!box2i;
77 
78         _elemsToDraw = makeAlignedBuffer!UIElement;
79         _elemsToDrawScratch = makeAlignedBuffer!UIElement;
80 
81         version(BenchmarkCompositing)
82         {
83             _compositingWatch = new StopWatch("Compositing = ");
84             _drawWatch = new StopWatch("Draw = ");
85             _mipmapWatch = new StopWatch("Mipmap = ");
86         }
87 
88         _diffuseMap = mallocEmplace!(Mipmap!RGBA)();
89         _materialMap = mallocEmplace!(Mipmap!RGBA)();
90         _depthMap = mallocEmplace!(Mipmap!L16)();
91     }
92 
93 
94     ~this()
95     {
96         closeUI();
97         _uiContext.destroyFree();
98 
99         _threadPool.destroyFree();
100 
101         compositor.destroyFree();
102         _diffuseMap.destroyFree();
103         _materialMap.destroyFree();
104         _depthMap.destroyFree();
105 
106         _windowListener.destroyFree();
107         alignedFree(_renderedBuffer, 16);
108     }
109 
110     // Graphics implementation
111 
112     override void* openUI(void* parentInfo, void* controlInfo, DAW daw, GraphicsBackend backend)
113     {
114         WindowBackend wbackend = void;
115         final switch(backend)
116         {
117             case GraphicsBackend.autodetect: wbackend = WindowBackend.autodetect; break;
118             case GraphicsBackend.win32: wbackend = WindowBackend.win32; break;
119             case GraphicsBackend.cocoa: wbackend = WindowBackend.cocoa; break;
120             case GraphicsBackend.carbon: wbackend = WindowBackend.carbon; break;
121         }
122 
123         // We create this window each time.
124         _window = createWindow(parentInfo, controlInfo, _windowListener, wbackend, _askedWidth, _askedHeight);
125 
126         reflow(box2i(0, 0, _askedWidth, _askedHeight));
127 
128         // Sets the whole UI dirty
129         setDirtyWhole();
130 
131         return _window.systemHandle();
132     }
133 
134     override void closeUI()
135     {
136         // Destroy window.
137         if (_window !is null)
138         {
139             _window.destroyFree();
140             _window = null;
141         }
142     }
143 
144     override void getGUISize(int* width, int* height)
145     {
146         *width = _askedWidth;
147         *height = _askedHeight;
148     }
149 
150     version(BenchmarkCompositing)
151     {
152         class StopWatch
153         {
154             this(string title)
155             {
156                 _title = title;
157             }
158 
159             void start()
160             {
161                 _lastTime = _window.getTimeMs();
162             }
163 
164             enum WARMUP = 30;
165 
166             void stop()
167             {
168                 uint now = _window.getTimeMs();
169                 int timeDiff = cast(int)(now - _lastTime);
170 
171                 if (times >= WARMUP)
172                     sum += timeDiff; // first samples are discarded
173 
174                 times++;
175                 string msg = _title ~ to!string(timeDiff) ~ " ms";
176                 _window.debugOutput(msg);
177             }
178 
179             void displayMean()
180             {
181                 if (times > WARMUP)
182                 {
183                     string msg = _title ~ to!string(sum / (times - WARMUP)) ~ " ms mean";
184                     _window.debugOutput(msg);
185                 }
186             }
187 
188             string _title;
189             uint _lastTime;
190             double sum = 0;
191             int times = 0;
192         }
193     }
194 
195 
196     // This class is only here to avoid name conflicts between
197     // UIElement and IWindowListener methods :|
198     // Explicit outer to avoid emplace crashing
199     static class WindowListener : IWindowListener
200     {
201     nothrow:
202     @nogc:
203         GUIGraphics outer;
204 
205         this(GUIGraphics outer)
206         {
207             this.outer = outer;
208         }
209 
210         override bool onMouseClick(int x, int y, MouseButton mb, bool isDoubleClick, MouseState mstate)
211         {
212             return outer.mouseClick(x, y, mb, isDoubleClick, mstate);
213         }
214 
215         override bool onMouseRelease(int x, int y, MouseButton mb, MouseState mstate)
216         {
217             outer.mouseRelease(x, y, mb, mstate);
218             return true;
219         }
220 
221         override bool onMouseWheel(int x, int y, int wheelDeltaX, int wheelDeltaY, MouseState mstate)
222         {
223             return outer.mouseWheel(x, y, wheelDeltaX, wheelDeltaY, mstate);
224         }
225 
226         override void onMouseMove(int x, int y, int dx, int dy, MouseState mstate)
227         {
228             outer.mouseMove(x, y, dx, dy, mstate);
229         }
230 
231         override void recomputeDirtyAreas()
232         {
233             return outer.recomputeDirtyAreas();
234         }
235 
236         override bool onKeyDown(Key key)
237         {
238             // Sends the event to the last clicked element first
239             if (outer._uiContext.focused !is null)
240                 if (outer._uiContext.focused.onKeyDown(key))
241                     return true;
242 
243             // else to all Elements
244             return outer.keyDown(key);
245         }
246 
247         override bool onKeyUp(Key key)
248         {
249             // Sends the event to the last clicked element first
250             if (outer._uiContext.focused !is null)
251                 if (outer._uiContext.focused.onKeyUp(key))
252                     return true;
253             // else to all Elements
254             return outer.keyUp(key);
255         }
256 
257         /// Returns areas affected by updates.
258         override box2i getDirtyRectangle() nothrow @nogc
259         {
260             return outer._areasToRender[].boundingBox();
261         }
262 
263         override ImageRef!RGBA onResized(int width, int height)
264         {
265             return outer.doResize(width, height);          
266         }
267 
268         // Redraw dirtied controls in depth and diffuse maps.
269         // Update composited cache.
270         override void onDraw(WindowPixelFormat pf) nothrow @nogc
271         {
272             return outer.doDraw(pf);
273         }
274 
275         override void onMouseCaptureCancelled()
276         {
277             // Stop an eventual drag operation
278             outer._uiContext.stopDragging();
279         }
280 
281         override void onAnimate(double dt, double time)
282         {
283             outer.animate(dt, time);
284         }
285     }
286 
287     /// Tune this to tune the trade-off between light quality and speed.
288     /// The default value was tuned by hand on very shiny light sources.
289     /// Too high and processing becomes very expensive.
290     /// Too little and the ligth decay doesn't feel natural.
291     void setUpdateMargin(int margin = 20) nothrow @nogc
292     {
293         _updateMargin = margin;
294     }
295 
296 protected:
297 
298     UIContext _uiContext;
299 
300     WindowListener _windowListener;
301 
302     // An interface to the underlying window
303     IWindow _window;
304 
305     // Task pool for multi-threaded image work
306     ThreadPool _threadPool;
307 
308     int _askedWidth = 0;
309     int _askedHeight = 0;
310 
311     // Diffuse color values for the whole UI.
312     Mipmap!RGBA _diffuseMap;
313 
314     // Depth values for the whole UI.
315     Mipmap!L16 _depthMap;
316 
317     // Depth values for the whole UI.
318     Mipmap!RGBA _materialMap;
319 
320     // The list of areas whose diffuse/depth data have been changed.
321     AlignedBuffer!box2i _areasToUpdateNonOverlapping;
322 
323     // Used to maintain the _areasToUpdate invariant of no overlap
324     AlignedBuffer!box2i _areasToUpdateTemp;
325 
326     // Same, but temporary variable for mipmap generation
327     AlignedBuffer!box2i[2] _updateRectScratch;
328 
329     // The list of areas that must be effectively updated in the composite buffer
330     // (sligthly larger than _areasToUpdate).
331     AlignedBuffer!box2i _areasToRender;
332 
333     // same list, but reorganized to avoid overlap
334     AlignedBuffer!box2i _areasToRenderNonOverlapping;
335 
336     // same list, but separated in smaller tiles
337     AlignedBuffer!box2i _areasToRenderNonOverlappingTiled;
338 
339     // The list of UIElement to draw
340     // Note: AlignedBuffer memory isn't scanned,
341     //       but this doesn't matter since UIElement are the UI hierarchy anyway.
342     AlignedBuffer!UIElement _elemsToDraw;
343 
344     /// Temporary buffer for stable sorting of _elemsToDraw
345     AlignedBuffer!UIElement _elemsToDrawScratch;
346 
347     /// Amount of pixels dirty rectangles are extended with.
348     int _updateMargin = 20;
349 
350     // The fully rendered framebuffer.
351     // This should point into comitted virtual memory for faster (maybe) upload to device
352     ubyte* _renderedBuffer = null;
353 
354     version(BenchmarkCompositing)
355     {
356         StopWatch _compositingWatch;
357         StopWatch _mipmapWatch;
358         StopWatch _drawWatch;
359     }
360 
361     void doDraw(WindowPixelFormat pf) nothrow @nogc
362     {
363         ImageRef!RGBA wfb;
364         wfb.w = _askedWidth;
365         wfb.h = _askedHeight;
366         wfb.pitch = byteStride(_askedWidth);
367         wfb.pixels = cast(RGBA*)_renderedBuffer;
368 
369         // Composite GUI
370         // Most of the cost of rendering is here
371         version(BenchmarkCompositing)
372             _drawWatch.start();
373 
374         renderElements();
375 
376         version(BenchmarkCompositing)
377         {
378             _drawWatch.stop();
379             _drawWatch.displayMean();
380         }
381 
382         version(BenchmarkCompositing)
383             _mipmapWatch.start();
384 
385         // Split boxes to avoid overlapped work
386         // Note: this is done separately for update areas and render areas
387         _areasToRenderNonOverlapping.clearContents();
388         removeOverlappingAreas(_areasToRender, _areasToRenderNonOverlapping);
389 
390         regenerateMipmaps();
391 
392         version(BenchmarkCompositing)
393         {
394             _mipmapWatch.stop();
395             _mipmapWatch.displayMean();
396         }
397 
398         version(BenchmarkCompositing)
399             _compositingWatch.start();
400 
401         compositeGUI(wfb, pf);
402 
403         // only then is the list of rectangles to update cleared
404         _areasToUpdateNonOverlapping.clearContents();
405 
406         // wait for compositing completion
407         //_threadPool.waitForCompletion();
408 
409         version(BenchmarkCompositing)
410         {
411             _compositingWatch.stop();
412             _compositingWatch.displayMean();
413         }
414     }
415 
416     // Fills _areasToUpdate and _areasToRender
417     void recomputeDirtyAreas() nothrow @nogc
418     {
419         // Get areas to update
420         _areasToRender.clearContents();
421 
422         // First we pull dirty rectangles from the UI
423         context().dirtyList.pullAllRectangles(_areasToUpdateNonOverlapping);
424 
425         // TECHNICAL DEBT HERE
426         // The problem here is that if the window isn't shown there may be duplicates in
427         // _areasToUpdate, so we have to maintain unicity again
428         // The code with dirty rects is a big mess, it needs a severe rewrite.
429         //
430         // SOLUTION
431         // The fundamental problem is that dirtyList should probably be merged with
432         // _areasToUpdateNonOverlapping.
433         // _areasToRender should also be purely derived from _areasToUpdateNonOverlapping
434         // Finally the interface of IWindowListener is poorly defined, this ties the window
435         // to the renderer in a bad way.
436         {
437             _areasToUpdateTemp.clearContents();
438             removeOverlappingAreas(_areasToUpdateNonOverlapping, _areasToUpdateTemp);
439             _areasToUpdateNonOverlapping.clearContents();
440             _areasToUpdateNonOverlapping.pushBack(_areasToUpdateTemp);
441         }
442         assert(haveNoOverlap(_areasToUpdateNonOverlapping[]));
443 
444         foreach(dirtyRect; _areasToUpdateNonOverlapping)
445         {
446             assert(dirtyRect.isSorted);
447             assert(!dirtyRect.empty);
448             _areasToRender.pushBack( extendsDirtyRect(dirtyRect, _askedWidth, _askedHeight) );
449         }
450     }
451 
452     box2i extendsDirtyRect(box2i rect, int width, int height) nothrow @nogc
453     {
454         int xmin = rect.min.x - _updateMargin;
455         int ymin = rect.min.y - _updateMargin;
456         int xmax = rect.max.x + _updateMargin;
457         int ymax = rect.max.y + _updateMargin;
458 
459         if (xmin < 0) xmin = 0;
460         if (ymin < 0) ymin = 0;
461         if (xmax > width) xmax = width;
462         if (ymax > height) ymax = height;
463 
464         // This could also happen if an UIElement is moved quickly
465         if (xmax < 0) xmax = 0;
466         if (ymax < 0) ymax = 0;
467         if (xmin > width) xmin = width;
468         if (ymin > height) ymin = height;
469 
470         box2i result = box2i(xmin, ymin, xmax, ymax);
471         assert(result.isSorted);
472         return result;
473     }
474 
475     ImageRef!RGBA doResize(int width, int height) nothrow @nogc
476     {
477         _askedWidth = width;
478         _askedHeight = height;
479 
480         reflow(box2i(0, 0, _askedWidth, _askedHeight));
481 
482         // FUTURE: maybe not destroy the whole mipmap?
483         _diffuseMap.size(5, width, height);
484         _depthMap.size(4, width, height);
485         _materialMap.size(0, width, height);
486 
487         // Extends buffer
488         size_t sizeNeeded = byteStride(width) * height;
489         _renderedBuffer = cast(ubyte*) alignedRealloc(_renderedBuffer, sizeNeeded, 16);
490 
491         ImageRef!RGBA wfb;
492         wfb.w = _askedWidth;
493         wfb.h = _askedHeight;
494         wfb.pitch = byteStride(_askedWidth);
495         wfb.pixels = cast(RGBA*)(_renderedBuffer);
496         return wfb;
497     }
498 
499     /// Redraw UIElements
500     void renderElements() nothrow @nogc
501     {
502         // recompute draw list
503         _elemsToDraw.clearContents();
504         getDrawList(_elemsToDraw);
505 
506         // Sort by ascending z-order (high z-order gets drawn last)
507         // This sort must be stable to avoid messing with tree natural order.
508         _elemsToDrawScratch.resize(_elemsToDraw.length);
509         int compareZOrder(in UIElement a, in UIElement b) nothrow @nogc
510         {
511             return a.zOrder() - b.zOrder();
512         }
513         mergeSort!UIElement(_elemsToDraw[], _elemsToDrawScratch[] , &compareZOrder);
514 
515         enum bool parallelDraw = true;
516 
517         auto diffuseRef = _diffuseMap.levels[0].toRef();
518         auto depthRef = _depthMap.levels[0].toRef();
519         auto materialRef = _materialMap.levels[0].toRef();
520 
521         static if (parallelDraw)
522         {
523             int drawn = 0;
524             int maxParallelElements = 32;
525             int N = cast(int)_elemsToDraw.length;
526 
527             while(drawn < N)
528             {
529                 int canBeDrawn = 1; // at least one can be drawn without collision
530 
531                 // Search max number of parallelizable draws until the end of the list or a collision is found
532                 bool foundIntersection = false;
533                 for ( ; (canBeDrawn < maxParallelElements) && (drawn + canBeDrawn < N); ++canBeDrawn)
534                 {
535                     box2i candidate = _elemsToDraw[drawn + canBeDrawn].position;
536 
537                     for (int j = 0; j < canBeDrawn; ++j)
538                     {
539                         if (_elemsToDraw[drawn + j].position.intersects(candidate))
540                         {
541                             foundIntersection = true;
542                             break;
543                         }
544                     }
545                     if (foundIntersection)
546                         break;
547                 }
548 
549                 assert(canBeDrawn >= 1 && canBeDrawn <= maxParallelElements);
550  
551                 // Draw a number of UIElement in parallel
552                 void drawOneItem(int i) nothrow @nogc
553                 {
554                     _elemsToDraw[drawn + i].render(diffuseRef, depthRef, materialRef, _areasToUpdateNonOverlapping[]);
555                 }
556                 //_threadPool.waitForCompletion();
557                 _threadPool.parallelFor/*Async*/(canBeDrawn, &drawOneItem);
558 
559                 drawn += canBeDrawn;
560                 assert(drawn <= N);
561             }
562             assert(drawn == N);
563         }
564         else
565         {
566             // Render required areas in diffuse and depth maps, base level
567             foreach(elem; _elemsToDraw)
568                 elem.render(diffuseRef, depthRef, _areasToUpdate[]);
569         }
570     }
571 
572     /// Compose lighting effects from depth and diffuse into a result.
573     /// takes output image and non-overlapping areas as input
574     /// Useful multithreading code.
575     void compositeGUI(ImageRef!RGBA wfb, WindowPixelFormat pf) nothrow @nogc
576     {
577         // Was tuned for performance, maybe the tradeoff has changed now that we use LDC.
578         enum tileWidth = 64;
579         enum tileHeight = 32;
580 
581         _areasToRenderNonOverlappingTiled.clearContents();
582         tileAreas(_areasToRenderNonOverlapping[], tileWidth, tileHeight,_areasToRenderNonOverlappingTiled);
583 
584         int numAreas = cast(int)_areasToRenderNonOverlappingTiled.length;
585 
586         void compositeOneTile(int i) nothrow @nogc
587         {
588             compositor.compositeTile(wfb, pf, _areasToRenderNonOverlappingTiled[i],
589                                      _diffuseMap, _materialMap, _depthMap, context.skybox);
590         }
591         //_threadPool.waitForCompletion(); // wait for mipmaps to be generated
592         _threadPool.parallelFor/*Async*/(numAreas, &compositeOneTile);
593     }
594 
595     /// Compose lighting effects from depth and diffuse into a result.
596     /// takes output image and non-overlapping areas as input
597     /// Useful multithreading code.
598     void regenerateMipmaps() nothrow @nogc
599     {
600         int numAreas = cast(int)_areasToUpdateNonOverlapping.length;
601 
602         // Fill update rect buffer with the content of _areasToUpdateNonOverlapping
603         for (int i = 0; i < 2; ++i)
604         {
605             _updateRectScratch[i].clearContents();
606             _updateRectScratch[i].pushBack(_areasToUpdateNonOverlapping[]);
607         }
608 
609         // We can't use tiled parallelism for mipmapping here because there is overdraw beyond level 0
610         // So instead what we do is using up to 2 threads.
611         void processOneMipmap(int i) nothrow @nogc
612         {
613             if (i == 0)
614             {
615                 // diffuse
616                 Mipmap!RGBA mipmap = _diffuseMap;
617                 int levelMax = min(mipmap.numLevels(), 5);
618                 foreach(level; 1 .. mipmap.numLevels())
619                 {
620                     Mipmap!RGBA.Quality quality;
621                     if (level == 1)
622                         quality = Mipmap!RGBA.Quality.boxAlphaCovIntoPremul;
623                     else
624                         quality = Mipmap!RGBA.Quality.cubic;
625 
626                     foreach(ref area; _updateRectScratch[i])
627                     {
628                         area = mipmap.generateNextLevel(quality, area, level);
629                     }
630                 }
631             }
632             else
633             {
634                 // depth
635                 Mipmap!L16 mipmap = _depthMap;
636                 foreach(level; 1 .. mipmap.numLevels())
637                 {
638                     auto quality = level >= 3 ? Mipmap!L16.Quality.cubic : Mipmap!L16.Quality.box;
639                     foreach(ref area; _updateRectScratch[i])
640                     {
641                         area = mipmap.generateNextLevel(quality, area, level);
642                     }
643                 }
644             }
645         }
646         //_threadPool.waitForCompletion(); // wait for UI widgets to be drawn
647         _threadPool.parallelFor/*Async*/(2, &processOneMipmap);
648     }
649 }
650 
651 
652 enum scanLineAlignment = 4; // could be anything
653 
654 // given a width, how long in bytes should scanlines be
655 int byteStride(int width) pure nothrow @nogc
656 {
657     int widthInBytes = width * 4;
658     return (widthInBytes + (scanLineAlignment - 1)) & ~(scanLineAlignment-1);
659 }