1 /**
2 * A GUIGraphics is the interface between a plugin client and a IWindow.
3 *
4 * Copyright: Copyright Auburn Sounds 2015 and later.
5 * License: $(LINK2 http://www.boost.org/LICENSE_1_0.txt, Boost License 1.0)
6 * Authors: Guillaume Piolat
7 */
8 module dplug.gui.graphics;
9
10 import std.math;
11 import std.algorithm.comparison;
12 import std.algorithm.sorting;
13 import std.algorithm.mutation;
14
15 import dplug.core.math;
16 import dplug.core.thread;
17
18 import dplug.client.client;
19 import dplug.client.graphics;
20 import dplug.client.daw;
21
22 import dplug.window.window;
23
24 import dplug.graphics.mipmap;
25
26 import dplug.gui.boxlist;
27 import dplug.gui.context;
28 import dplug.gui.element;
29 import dplug.gui.compositor;
30
31 /// In the whole package:
32 /// The diffuse maps contains:
33 /// RGBA = red/green/blue/emissiveness
34 /// The depth maps contains depth.
35 /// The material map contains:
36 /// RGBA = roughness / metalness / specular / physical (allows to bypass PBR)
37
38 alias RMSP = RGBA; // reminder
39
40 // Uncomment to benchmark compositing and devise optimizations.
41 //version = BenchmarkCompositing;
42
43 // A GUIGraphics is the interface between a plugin client and a IWindow.
44 // It is also an UIElement and the root element of the plugin UI hierarchy.
45 // You have to derive it to have a GUI.
46 // It dispatches window events to the GUI hierarchy.
47 class GUIGraphics : UIElement, IGraphics
48 {
49 nothrow:
50 @nogc:
51
52 ICompositor compositor;
53
54 this(int initialWidth, int initialHeight)
55 {
56 _uiContext = mallocNew!UIContext();
57 super(_uiContext);
58
59 // Don't like the default rendering? Make another compositor.
60 compositor = mallocNew!PBRCompositor();
61
62 _windowListener = mallocNew!WindowListener(this);
63
64 _window = null;
65 _askedWidth = initialWidth;
66 _askedHeight = initialHeight;
67
68 _threadPool = mallocNew!ThreadPool();
69
70 _areasToUpdateNonOverlapping = makeVec!box2i;
71 _areasToUpdateTemp = makeVec!box2i;
72
73 _updateRectScratch[0] = makeVec!box2i;
74 _updateRectScratch[1] = makeVec!box2i;
75
76 _areasToRender = makeVec!box2i;
77 _areasToRenderNonOverlapping = makeVec!box2i;
78 _areasToRenderNonOverlappingTiled = makeVec!box2i;
79
80 _elemsToDraw = makeVec!UIElement;
81
82 version(BenchmarkCompositing)
83 {
84 _compositingWatch = new StopWatch("Compositing = ");
85 _drawWatch = new StopWatch("Draw = ");
86 _mipmapWatch = new StopWatch("Mipmap = ");
87 }
88
89 _diffuseMap = mallocNew!(Mipmap!RGBA)();
90 _materialMap = mallocNew!(Mipmap!RGBA)();
91 _depthMap = mallocNew!(Mipmap!L16)();
92 }
93
94
95 ~this()
96 {
97 closeUI();
98 _uiContext.destroyFree();
99
100 _threadPool.destroyFree();
101
102 compositor.destroyFree();
103 _diffuseMap.destroyFree();
104 _materialMap.destroyFree();
105 _depthMap.destroyFree();
106
107 _windowListener.destroyFree();
108 alignedFree(_renderedBuffer, 16);
109 }
110
111 // Graphics implementation
112
113 override void* openUI(void* parentInfo, void* controlInfo, DAW daw, GraphicsBackend backend)
114 {
115 WindowBackend wbackend = void;
116 final switch(backend)
117 {
118 case GraphicsBackend.autodetect: wbackend = WindowBackend.autodetect; break;
119 case GraphicsBackend.win32: wbackend = WindowBackend.win32; break;
120 case GraphicsBackend.cocoa: wbackend = WindowBackend.cocoa; break;
121 case GraphicsBackend.carbon: wbackend = WindowBackend.carbon; break;
122 case GraphicsBackend.x11: wbackend = WindowBackend.x11; break;
123 }
124
125 // We create this window each time.
126 _window = createWindow(WindowUsage.plugin, parentInfo, controlInfo, _windowListener, wbackend, _askedWidth, _askedHeight);
127
128 reflow(box2i(0, 0, _askedWidth, _askedHeight));
129
130 // Sets the whole UI dirty
131 setDirtyWhole();
132
133 return _window.systemHandle();
134 }
135
136 override void closeUI()
137 {
138 // Destroy window.
139 if (_window !is null)
140 {
141 _window.destroyFree();
142 _window = null;
143 }
144 }
145
146 override void getGUISize(int* width, int* height)
147 {
148 *width = _askedWidth;
149 *height = _askedHeight;
150 }
151
152 version(BenchmarkCompositing)
153 {
154 class StopWatch
155 {
156 this(string title)
157 {
158 _title = title;
159 }
160
161 void start()
162 {
163 _lastTime = _window.getTimeMs();
164 }
165
166 enum WARMUP = 30;
167
168 void stop()
169 {
170 uint now = _window.getTimeMs();
171 int timeDiff = cast(int)(now - _lastTime);
172
173 if (times >= WARMUP)
174 sum += timeDiff; // first samples are discarded
175
176 times++;
177 string msg = _title ~ to!string(timeDiff) ~ " ms";
178 _window.debugOutput(msg);
179 }
180
181 void displayMean()
182 {
183 if (times > WARMUP)
184 {
185 string msg = _title ~ to!string(sum / (times - WARMUP)) ~ " ms mean";
186 _window.debugOutput(msg);
187 }
188 }
189
190 string _title;
191 uint _lastTime;
192 double sum = 0;
193 int times = 0;
194 }
195 }
196
197
198 // This class is only here to avoid name conflicts between
199 // UIElement and IWindowListener methods :|
200 // Explicit outer to avoid emplace crashing
201 static class WindowListener : IWindowListener
202 {
203 nothrow:
204 @nogc:
205 GUIGraphics outer;
206
207 this(GUIGraphics outer)
208 {
209 this.outer = outer;
210 }
211
212 override bool onMouseClick(int x, int y, MouseButton mb, bool isDoubleClick, MouseState mstate)
213 {
214 return outer.mouseClick(x, y, mb, isDoubleClick, mstate);
215 }
216
217 override bool onMouseRelease(int x, int y, MouseButton mb, MouseState mstate)
218 {
219 outer.mouseRelease(x, y, mb, mstate);
220 return true;
221 }
222
223 override bool onMouseWheel(int x, int y, int wheelDeltaX, int wheelDeltaY, MouseState mstate)
224 {
225 return outer.mouseWheel(x, y, wheelDeltaX, wheelDeltaY, mstate);
226 }
227
228 override void onMouseMove(int x, int y, int dx, int dy, MouseState mstate)
229 {
230 outer.mouseMove(x, y, dx, dy, mstate);
231 }
232
233 override void recomputeDirtyAreas()
234 {
235 return outer.recomputeDirtyAreas();
236 }
237
238 override bool onKeyDown(Key key)
239 {
240 // Sends the event to the last clicked element first
241 if (outer._uiContext.focused !is null)
242 if (outer._uiContext.focused.onKeyDown(key))
243 return true;
244
245 // else to all Elements
246 return outer.keyDown(key);
247 }
248
249 override bool onKeyUp(Key key)
250 {
251 // Sends the event to the last clicked element first
252 if (outer._uiContext.focused !is null)
253 if (outer._uiContext.focused.onKeyUp(key))
254 return true;
255 // else to all Elements
256 return outer.keyUp(key);
257 }
258
259 /// Returns areas affected by updates.
260 override box2i getDirtyRectangle() nothrow @nogc
261 {
262 return outer._areasToRender[].boundingBox();
263 }
264
265 override ImageRef!RGBA onResized(int width, int height)
266 {
267 return outer.doResize(width, height);
268 }
269
270 // Redraw dirtied controls in depth and diffuse maps.
271 // Update composited cache.
272 override void onDraw(WindowPixelFormat pf) nothrow @nogc
273 {
274 return outer.doDraw(pf);
275 }
276
277 override void onMouseCaptureCancelled()
278 {
279 // Stop an eventual drag operation
280 outer._uiContext.stopDragging();
281 }
282
283 override void onAnimate(double dt, double time)
284 {
285 outer.animate(dt, time);
286 }
287 }
288
289 /// Tune this to tune the trade-off between light quality and speed.
290 /// The default value was tuned by hand on very shiny light sources.
291 /// Too high and processing becomes very expensive.
292 /// Too little and the ligth decay doesn't feel natural.
293 void setUpdateMargin(int margin = 20) nothrow @nogc
294 {
295 _updateMargin = margin;
296 }
297
298 protected:
299
300 UIContext _uiContext;
301
302 WindowListener _windowListener;
303
304 // An interface to the underlying window
305 IWindow _window;
306
307 // Task pool for multi-threaded image work
308 ThreadPool _threadPool;
309
310 int _askedWidth = 0;
311 int _askedHeight = 0;
312
313 // Diffuse color values for the whole UI.
314 Mipmap!RGBA _diffuseMap;
315
316 // Depth values for the whole UI.
317 Mipmap!L16 _depthMap;
318
319 // Depth values for the whole UI.
320 Mipmap!RGBA _materialMap;
321
322 // The list of areas whose diffuse/depth data have been changed.
323 Vec!box2i _areasToUpdateNonOverlapping;
324
325 // Used to maintain the _areasToUpdate invariant of no overlap
326 Vec!box2i _areasToUpdateTemp;
327
328 // Same, but temporary variable for mipmap generation
329 Vec!box2i[2] _updateRectScratch;
330
331 // The list of areas that must be effectively updated in the composite buffer
332 // (sligthly larger than _areasToUpdate).
333 Vec!box2i _areasToRender;
334
335 // same list, but reorganized to avoid overlap
336 Vec!box2i _areasToRenderNonOverlapping;
337
338 // same list, but separated in smaller tiles
339 Vec!box2i _areasToRenderNonOverlappingTiled;
340
341 // The list of UIElement to draw
342 // Note: AlignedBuffer memory isn't scanned,
343 // but this doesn't matter since UIElement are the UI hierarchy anyway.
344 Vec!UIElement _elemsToDraw;
345
346 /// Amount of pixels dirty rectangles are extended with.
347 int _updateMargin = 20;
348
349 // The fully rendered framebuffer.
350 // This should point into comitted virtual memory for faster (maybe) upload to device
351 ubyte* _renderedBuffer = null;
352
353 version(BenchmarkCompositing)
354 {
355 StopWatch _compositingWatch;
356 StopWatch _mipmapWatch;
357 StopWatch _drawWatch;
358 }
359
360 void doDraw(WindowPixelFormat pf) nothrow @nogc
361 {
362 ImageRef!RGBA wfb;
363 wfb.w = _askedWidth;
364 wfb.h = _askedHeight;
365 wfb.pitch = byteStride(_askedWidth);
366 wfb.pixels = cast(RGBA*)_renderedBuffer;
367
368 // Composite GUI
369 // Most of the cost of rendering is here
370 version(BenchmarkCompositing)
371 _drawWatch.start();
372
373 renderElements();
374
375 version(BenchmarkCompositing)
376 {
377 _drawWatch.stop();
378 _drawWatch.displayMean();
379 }
380
381 version(BenchmarkCompositing)
382 _mipmapWatch.start();
383
384 // Split boxes to avoid overlapped work
385 // Note: this is done separately for update areas and render areas
386 _areasToRenderNonOverlapping.clearContents();
387 removeOverlappingAreas(_areasToRender, _areasToRenderNonOverlapping);
388
389 regenerateMipmaps();
390
391 version(BenchmarkCompositing)
392 {
393 _mipmapWatch.stop();
394 _mipmapWatch.displayMean();
395 }
396
397 version(BenchmarkCompositing)
398 _compositingWatch.start();
399
400 compositeGUI(wfb, pf);
401
402 // only then is the list of rectangles to update cleared
403 _areasToUpdateNonOverlapping.clearContents();
404
405 // wait for compositing completion
406 //_threadPool.waitForCompletion();
407
408 version(BenchmarkCompositing)
409 {
410 _compositingWatch.stop();
411 _compositingWatch.displayMean();
412 }
413 }
414
415 // Fills _areasToUpdate and _areasToRender
416 void recomputeDirtyAreas() nothrow @nogc
417 {
418 // Get areas to update
419 _areasToRender.clearContents();
420
421 // First we pull dirty rectangles from the UI
422 context().dirtyList.pullAllRectangles(_areasToUpdateNonOverlapping);
423
424 // TECHNICAL DEBT HERE
425 // The problem here is that if the window isn't shown there may be duplicates in
426 // _areasToUpdate, so we have to maintain unicity again
427 // The code with dirty rects is a big mess, it needs a severe rewrite.
428 //
429 // SOLUTION
430 // The fundamental problem is that dirtyList should probably be merged with
431 // _areasToUpdateNonOverlapping.
432 // _areasToRender should also be purely derived from _areasToUpdateNonOverlapping
433 // Finally the interface of IWindowListener is poorly defined, this ties the window
434 // to the renderer in a bad way.
435 {
436 _areasToUpdateTemp.clearContents();
437 removeOverlappingAreas(_areasToUpdateNonOverlapping, _areasToUpdateTemp);
438 _areasToUpdateNonOverlapping.clearContents();
439 _areasToUpdateNonOverlapping.pushBack(_areasToUpdateTemp);
440 }
441 assert(haveNoOverlap(_areasToUpdateNonOverlapping[]));
442
443 foreach(dirtyRect; _areasToUpdateNonOverlapping)
444 {
445 assert(dirtyRect.isSorted);
446 assert(!dirtyRect.empty);
447 _areasToRender.pushBack( extendsDirtyRect(dirtyRect, _askedWidth, _askedHeight) );
448 }
449 }
450
451 box2i extendsDirtyRect(box2i rect, int width, int height) nothrow @nogc
452 {
453 int xmin = rect.min.x - _updateMargin;
454 int ymin = rect.min.y - _updateMargin;
455 int xmax = rect.max.x + _updateMargin;
456 int ymax = rect.max.y + _updateMargin;
457
458 if (xmin < 0) xmin = 0;
459 if (ymin < 0) ymin = 0;
460 if (xmax > width) xmax = width;
461 if (ymax > height) ymax = height;
462
463 // This could also happen if an UIElement is moved quickly
464 if (xmax < 0) xmax = 0;
465 if (ymax < 0) ymax = 0;
466 if (xmin > width) xmin = width;
467 if (ymin > height) ymin = height;
468
469 box2i result = box2i(xmin, ymin, xmax, ymax);
470 assert(result.isSorted);
471 return result;
472 }
473
474 ImageRef!RGBA doResize(int width, int height) nothrow @nogc
475 {
476 _askedWidth = width;
477 _askedHeight = height;
478
479 reflow(box2i(0, 0, _askedWidth, _askedHeight));
480
481 // FUTURE: maybe not destroy the whole mipmap?
482 _diffuseMap.size(5, width, height);
483 _depthMap.size(4, width, height);
484 _materialMap.size(0, width, height);
485
486 // Extends buffer
487 size_t sizeNeeded = byteStride(width) * height;
488 _renderedBuffer = cast(ubyte*) alignedRealloc(_renderedBuffer, sizeNeeded, 16);
489
490 ImageRef!RGBA wfb;
491 wfb.w = _askedWidth;
492 wfb.h = _askedHeight;
493 wfb.pitch = byteStride(_askedWidth);
494 wfb.pixels = cast(RGBA*)(_renderedBuffer);
495 return wfb;
496 }
497
498 /// Redraw UIElements
499 void renderElements() nothrow @nogc
500 {
501 // recompute draw list
502 _elemsToDraw.clearContents();
503 getDrawList(_elemsToDraw);
504
505 // Sort by ascending z-order (high z-order gets drawn last)
506 // This sort must be stable to avoid messing with tree natural order.
507 int compareZOrder(in UIElement a, in UIElement b) nothrow @nogc
508 {
509 return a.zOrder() - b.zOrder();
510 }
511 grailSort!UIElement(_elemsToDraw[], &compareZOrder);
512
513 enum bool parallelDraw = true;
514
515 auto diffuseRef = _diffuseMap.levels[0].toRef();
516 auto depthRef = _depthMap.levels[0].toRef();
517 auto materialRef = _materialMap.levels[0].toRef();
518
519 static if (parallelDraw)
520 {
521 int drawn = 0;
522 int maxParallelElements = 32;
523 int N = cast(int)_elemsToDraw.length;
524
525 while(drawn < N)
526 {
527 int canBeDrawn = 1; // at least one can be drawn without collision
528
529 // Search max number of parallelizable draws until the end of the list or a collision is found
530 bool foundIntersection = false;
531 for ( ; (canBeDrawn < maxParallelElements) && (drawn + canBeDrawn < N); ++canBeDrawn)
532 {
533 box2i candidate = _elemsToDraw[drawn + canBeDrawn].position;
534
535 for (int j = 0; j < canBeDrawn; ++j)
536 {
537 if (_elemsToDraw[drawn + j].position.intersects(candidate))
538 {
539 foundIntersection = true;
540 break;
541 }
542 }
543 if (foundIntersection)
544 break;
545 }
546
547 assert(canBeDrawn >= 1 && canBeDrawn <= maxParallelElements);
548
549 // Draw a number of UIElement in parallel
550 void drawOneItem(int i) nothrow @nogc
551 {
552 _elemsToDraw[drawn + i].render(diffuseRef, depthRef, materialRef, _areasToUpdateNonOverlapping[]);
553 }
554 //_threadPool.waitForCompletion();
555 _threadPool.parallelFor/*Async*/(canBeDrawn, &drawOneItem);
556
557 drawn += canBeDrawn;
558 assert(drawn <= N);
559 }
560 assert(drawn == N);
561 }
562 else
563 {
564 // Render required areas in diffuse and depth maps, base level
565 foreach(elem; _elemsToDraw)
566 elem.render(diffuseRef, depthRef, _areasToUpdate[]);
567 }
568 }
569
570 /// Compose lighting effects from depth and diffuse into a result.
571 /// takes output image and non-overlapping areas as input
572 /// Useful multithreading code.
573 void compositeGUI(ImageRef!RGBA wfb, WindowPixelFormat pf) nothrow @nogc
574 {
575 // Was tuned for performance, maybe the tradeoff has changed now that we use LDC.
576 enum tileWidth = 64;
577 enum tileHeight = 32;
578
579 _areasToRenderNonOverlappingTiled.clearContents();
580 tileAreas(_areasToRenderNonOverlapping[], tileWidth, tileHeight,_areasToRenderNonOverlappingTiled);
581
582 int numAreas = cast(int)_areasToRenderNonOverlappingTiled.length;
583
584 void compositeOneTile(int i) nothrow @nogc
585 {
586 compositor.compositeTile(wfb, pf, _areasToRenderNonOverlappingTiled[i],
587 _diffuseMap, _materialMap, _depthMap, context.skybox);
588 }
589 //_threadPool.waitForCompletion(); // wait for mipmaps to be generated
590 _threadPool.parallelFor/*Async*/(numAreas, &compositeOneTile);
591 }
592
593 /// Compose lighting effects from depth and diffuse into a result.
594 /// takes output image and non-overlapping areas as input
595 /// Useful multithreading code.
596 void regenerateMipmaps() nothrow @nogc
597 {
598 int numAreas = cast(int)_areasToUpdateNonOverlapping.length;
599
600 // Fill update rect buffer with the content of _areasToUpdateNonOverlapping
601 for (int i = 0; i < 2; ++i)
602 {
603 _updateRectScratch[i].clearContents();
604 _updateRectScratch[i].pushBack(_areasToUpdateNonOverlapping[]);
605 }
606
607 // We can't use tiled parallelism for mipmapping here because there is overdraw beyond level 0
608 // So instead what we do is using up to 2 threads.
609 void processOneMipmap(int i) nothrow @nogc
610 {
611 if (i == 0)
612 {
613 // diffuse
614 Mipmap!RGBA mipmap = _diffuseMap;
615 int levelMax = min(mipmap.numLevels(), 5);
616 foreach(level; 1 .. mipmap.numLevels())
617 {
618 Mipmap!RGBA.Quality quality;
619 if (level == 1)
620 quality = Mipmap!RGBA.Quality.boxAlphaCovIntoPremul;
621 else
622 quality = Mipmap!RGBA.Quality.cubic;
623
624 foreach(ref area; _updateRectScratch[i])
625 {
626 area = mipmap.generateNextLevel(quality, area, level);
627 }
628 }
629 }
630 else
631 {
632 // depth
633 Mipmap!L16 mipmap = _depthMap;
634 foreach(level; 1 .. mipmap.numLevels())
635 {
636 auto quality = level >= 3 ? Mipmap!L16.Quality.cubic : Mipmap!L16.Quality.box;
637 foreach(ref area; _updateRectScratch[i])
638 {
639 area = mipmap.generateNextLevel(quality, area, level);
640 }
641 }
642 }
643 }
644 //_threadPool.waitForCompletion(); // wait for UI widgets to be drawn
645 _threadPool.parallelFor/*Async*/(2, &processOneMipmap);
646 }
647 }
648
649
650 enum scanLineAlignment = 4; // could be anything
651
652 // given a width, how long in bytes should scanlines be
653 int byteStride(int width) pure nothrow @nogc
654 {
655 int widthInBytes = width * 4;
656 return (widthInBytes + (scanLineAlignment - 1)) & ~(scanLineAlignment-1);
657 }