1 /** 2 * Copyright: Copyright Auburn Sounds 2015 and later. 3 * License: $(LINK2 http://www.boost.org/LICENSE_1_0.txt, Boost License 1.0) 4 * Authors: Guillaume Piolat 5 */ 6 module dplug.gui.graphics; 7 8 import std.math; 9 import std.algorithm.comparison; 10 import std.algorithm.sorting; 11 import std.algorithm.mutation; 12 13 import dplug.core.math; 14 import dplug.core.thread; 15 16 import dplug.client.client; 17 import dplug.client.graphics; 18 import dplug.client.daw; 19 20 import dplug.window.window; 21 22 import dplug.graphics.mipmap; 23 24 import dplug.gui.boxlist; 25 import dplug.gui.context; 26 import dplug.gui.element; 27 import dplug.gui.compositor; 28 29 /// In the whole package: 30 /// The diffuse maps contains: 31 /// RGBA = red/green/blue/emissiveness 32 /// The depth maps contains depth. 33 /// The material map contains: 34 /// RGBA = roughness / metalness / specular / physical (allows to bypass PBR) 35 36 alias RMSP = RGBA; // reminder 37 38 // Uncomment to benchmark compositing and devise optimizations. 39 //version = BenchmarkCompositing; 40 41 // A GUIGraphics is the interface between a plugin client and a IWindow. 42 // It is also an UIElement and the root element of the plugin UI hierarchy. 43 // You have to derive it to have a GUI. 44 // It dispatches window events to the GUI hierarchy. 45 class GUIGraphics : UIElement, IGraphics 46 { 47 nothrow: 48 @nogc: 49 50 ICompositor compositor; 51 52 this(int initialWidth, int initialHeight) 53 { 54 _uiContext = mallocEmplace!UIContext(); 55 super(_uiContext); 56 57 // Don't like the default rendering? Make another compositor. 58 compositor = mallocEmplace!PBRCompositor(); 59 60 _windowListener = mallocEmplace!WindowListener(this); 61 62 _window = null; 63 _askedWidth = initialWidth; 64 _askedHeight = initialHeight; 65 66 _threadPool = mallocEmplace!ThreadPool(); 67 68 _areasToUpdateNonOverlapping = makeAlignedBuffer!box2i; 69 _areasToUpdateTemp = makeAlignedBuffer!box2i; 70 71 _updateRectScratch[0] = makeAlignedBuffer!box2i; 72 _updateRectScratch[1] = makeAlignedBuffer!box2i; 73 74 _areasToRender = makeAlignedBuffer!box2i; 75 _areasToRenderNonOverlapping = makeAlignedBuffer!box2i; 76 _areasToRenderNonOverlappingTiled = makeAlignedBuffer!box2i; 77 78 _elemsToDraw = makeAlignedBuffer!UIElement; 79 _elemsToDrawScratch = makeAlignedBuffer!UIElement; 80 81 version(BenchmarkCompositing) 82 { 83 _compositingWatch = new StopWatch("Compositing = "); 84 _drawWatch = new StopWatch("Draw = "); 85 _mipmapWatch = new StopWatch("Mipmap = "); 86 } 87 88 _diffuseMap = mallocEmplace!(Mipmap!RGBA)(); 89 _materialMap = mallocEmplace!(Mipmap!RGBA)(); 90 _depthMap = mallocEmplace!(Mipmap!L16)(); 91 } 92 93 94 ~this() 95 { 96 closeUI(); 97 _uiContext.destroyFree(); 98 99 _threadPool.destroyFree(); 100 101 compositor.destroyFree(); 102 _diffuseMap.destroyFree(); 103 _materialMap.destroyFree(); 104 _depthMap.destroyFree(); 105 106 _windowListener.destroyFree(); 107 alignedFree(_renderedBuffer, 16); 108 } 109 110 // Graphics implementation 111 112 override void* openUI(void* parentInfo, void* controlInfo, DAW daw, GraphicsBackend backend) 113 { 114 WindowBackend wbackend = void; 115 final switch(backend) 116 { 117 case GraphicsBackend.autodetect: wbackend = WindowBackend.autodetect; break; 118 case GraphicsBackend.win32: wbackend = WindowBackend.win32; break; 119 case GraphicsBackend.cocoa: wbackend = WindowBackend.cocoa; break; 120 case GraphicsBackend.carbon: wbackend = WindowBackend.carbon; break; 121 } 122 123 // We create this window each time. 124 _window = createWindow(parentInfo, controlInfo, _windowListener, wbackend, _askedWidth, _askedHeight); 125 126 reflow(box2i(0, 0, _askedWidth, _askedHeight)); 127 128 // Sets the whole UI dirty 129 setDirtyWhole(); 130 131 return _window.systemHandle(); 132 } 133 134 override void closeUI() 135 { 136 // Destroy window. 137 if (_window !is null) 138 { 139 _window.destroyFree(); 140 _window = null; 141 } 142 } 143 144 override void getGUISize(int* width, int* height) 145 { 146 *width = _askedWidth; 147 *height = _askedHeight; 148 } 149 150 version(BenchmarkCompositing) 151 { 152 class StopWatch 153 { 154 this(string title) 155 { 156 _title = title; 157 } 158 159 void start() 160 { 161 _lastTime = _window.getTimeMs(); 162 } 163 164 enum WARMUP = 30; 165 166 void stop() 167 { 168 uint now = _window.getTimeMs(); 169 int timeDiff = cast(int)(now - _lastTime); 170 171 if (times >= WARMUP) 172 sum += timeDiff; // first samples are discarded 173 174 times++; 175 string msg = _title ~ to!string(timeDiff) ~ " ms"; 176 _window.debugOutput(msg); 177 } 178 179 void displayMean() 180 { 181 if (times > WARMUP) 182 { 183 string msg = _title ~ to!string(sum / (times - WARMUP)) ~ " ms mean"; 184 _window.debugOutput(msg); 185 } 186 } 187 188 string _title; 189 uint _lastTime; 190 double sum = 0; 191 int times = 0; 192 } 193 } 194 195 196 // This class is only here to avoid name conflicts between 197 // UIElement and IWindowListener methods :| 198 // Explicit outer to avoid emplace crashing 199 static class WindowListener : IWindowListener 200 { 201 nothrow: 202 @nogc: 203 GUIGraphics outer; 204 205 this(GUIGraphics outer) 206 { 207 this.outer = outer; 208 } 209 210 override bool onMouseClick(int x, int y, MouseButton mb, bool isDoubleClick, MouseState mstate) 211 { 212 return outer.mouseClick(x, y, mb, isDoubleClick, mstate); 213 } 214 215 override bool onMouseRelease(int x, int y, MouseButton mb, MouseState mstate) 216 { 217 outer.mouseRelease(x, y, mb, mstate); 218 return true; 219 } 220 221 override bool onMouseWheel(int x, int y, int wheelDeltaX, int wheelDeltaY, MouseState mstate) 222 { 223 return outer.mouseWheel(x, y, wheelDeltaX, wheelDeltaY, mstate); 224 } 225 226 override void onMouseMove(int x, int y, int dx, int dy, MouseState mstate) 227 { 228 outer.mouseMove(x, y, dx, dy, mstate); 229 } 230 231 override void recomputeDirtyAreas() 232 { 233 return outer.recomputeDirtyAreas(); 234 } 235 236 override bool onKeyDown(Key key) 237 { 238 // Sends the event to the last clicked element first 239 if (outer._uiContext.focused !is null) 240 if (outer._uiContext.focused.onKeyDown(key)) 241 return true; 242 243 // else to all Elements 244 return outer.keyDown(key); 245 } 246 247 override bool onKeyUp(Key key) 248 { 249 // Sends the event to the last clicked element first 250 if (outer._uiContext.focused !is null) 251 if (outer._uiContext.focused.onKeyUp(key)) 252 return true; 253 // else to all Elements 254 return outer.keyUp(key); 255 } 256 257 /// Returns areas affected by updates. 258 override box2i getDirtyRectangle() nothrow @nogc 259 { 260 return outer._areasToRender[].boundingBox(); 261 } 262 263 override ImageRef!RGBA onResized(int width, int height) 264 { 265 return outer.doResize(width, height); 266 } 267 268 // Redraw dirtied controls in depth and diffuse maps. 269 // Update composited cache. 270 override void onDraw(WindowPixelFormat pf) nothrow @nogc 271 { 272 return outer.doDraw(pf); 273 } 274 275 override void onMouseCaptureCancelled() 276 { 277 // Stop an eventual drag operation 278 outer._uiContext.stopDragging(); 279 } 280 281 override void onAnimate(double dt, double time) 282 { 283 outer.animate(dt, time); 284 } 285 } 286 287 /// Tune this to tune the trade-off between light quality and speed. 288 /// The default value was tuned by hand on very shiny light sources. 289 /// Too high and processing becomes very expensive. 290 /// Too little and the ligth decay doesn't feel natural. 291 void setUpdateMargin(int margin = 20) nothrow @nogc 292 { 293 _updateMargin = margin; 294 } 295 296 protected: 297 298 UIContext _uiContext; 299 300 WindowListener _windowListener; 301 302 // An interface to the underlying window 303 IWindow _window; 304 305 // Task pool for multi-threaded image work 306 ThreadPool _threadPool; 307 308 int _askedWidth = 0; 309 int _askedHeight = 0; 310 311 // Diffuse color values for the whole UI. 312 Mipmap!RGBA _diffuseMap; 313 314 // Depth values for the whole UI. 315 Mipmap!L16 _depthMap; 316 317 // Depth values for the whole UI. 318 Mipmap!RGBA _materialMap; 319 320 // The list of areas whose diffuse/depth data have been changed. 321 AlignedBuffer!box2i _areasToUpdateNonOverlapping; 322 323 // Used to maintain the _areasToUpdate invariant of no overlap 324 AlignedBuffer!box2i _areasToUpdateTemp; 325 326 // Same, but temporary variable for mipmap generation 327 AlignedBuffer!box2i[2] _updateRectScratch; 328 329 // The list of areas that must be effectively updated in the composite buffer 330 // (sligthly larger than _areasToUpdate). 331 AlignedBuffer!box2i _areasToRender; 332 333 // same list, but reorganized to avoid overlap 334 AlignedBuffer!box2i _areasToRenderNonOverlapping; 335 336 // same list, but separated in smaller tiles 337 AlignedBuffer!box2i _areasToRenderNonOverlappingTiled; 338 339 // The list of UIElement to draw 340 // Note: AlignedBuffer memory isn't scanned, 341 // but this doesn't matter since UIElement are the UI hierarchy anyway. 342 AlignedBuffer!UIElement _elemsToDraw; 343 344 /// Temporary buffer for stable sorting of _elemsToDraw 345 AlignedBuffer!UIElement _elemsToDrawScratch; 346 347 /// Amount of pixels dirty rectangles are extended with. 348 int _updateMargin = 20; 349 350 // The fully rendered framebuffer. 351 // This should point into comitted virtual memory for faster (maybe) upload to device 352 ubyte* _renderedBuffer = null; 353 354 version(BenchmarkCompositing) 355 { 356 StopWatch _compositingWatch; 357 StopWatch _mipmapWatch; 358 StopWatch _drawWatch; 359 } 360 361 void doDraw(WindowPixelFormat pf) nothrow @nogc 362 { 363 ImageRef!RGBA wfb; 364 wfb.w = _askedWidth; 365 wfb.h = _askedHeight; 366 wfb.pitch = byteStride(_askedWidth); 367 wfb.pixels = cast(RGBA*)_renderedBuffer; 368 369 // Composite GUI 370 // Most of the cost of rendering is here 371 version(BenchmarkCompositing) 372 _drawWatch.start(); 373 374 renderElements(); 375 376 version(BenchmarkCompositing) 377 { 378 _drawWatch.stop(); 379 _drawWatch.displayMean(); 380 } 381 382 version(BenchmarkCompositing) 383 _mipmapWatch.start(); 384 385 // Split boxes to avoid overlapped work 386 // Note: this is done separately for update areas and render areas 387 _areasToRenderNonOverlapping.clearContents(); 388 removeOverlappingAreas(_areasToRender, _areasToRenderNonOverlapping); 389 390 regenerateMipmaps(); 391 392 version(BenchmarkCompositing) 393 { 394 _mipmapWatch.stop(); 395 _mipmapWatch.displayMean(); 396 } 397 398 version(BenchmarkCompositing) 399 _compositingWatch.start(); 400 401 compositeGUI(wfb, pf); 402 403 // only then is the list of rectangles to update cleared 404 _areasToUpdateNonOverlapping.clearContents(); 405 406 // wait for compositing completion 407 //_threadPool.waitForCompletion(); 408 409 version(BenchmarkCompositing) 410 { 411 _compositingWatch.stop(); 412 _compositingWatch.displayMean(); 413 } 414 } 415 416 // Fills _areasToUpdate and _areasToRender 417 void recomputeDirtyAreas() nothrow @nogc 418 { 419 // Get areas to update 420 _areasToRender.clearContents(); 421 422 // First we pull dirty rectangles from the UI 423 context().dirtyList.pullAllRectangles(_areasToUpdateNonOverlapping); 424 425 // TECHNICAL DEBT HERE 426 // The problem here is that if the window isn't shown there may be duplicates in 427 // _areasToUpdate, so we have to maintain unicity again 428 // The code with dirty rects is a big mess, it needs a severe rewrite. 429 // 430 // SOLUTION 431 // The fundamental problem is that dirtyList should probably be merged with 432 // _areasToUpdateNonOverlapping. 433 // _areasToRender should also be purely derived from _areasToUpdateNonOverlapping 434 // Finally the interface of IWindowListener is poorly defined, this ties the window 435 // to the renderer in a bad way. 436 { 437 _areasToUpdateTemp.clearContents(); 438 removeOverlappingAreas(_areasToUpdateNonOverlapping, _areasToUpdateTemp); 439 _areasToUpdateNonOverlapping.clearContents(); 440 _areasToUpdateNonOverlapping.pushBack(_areasToUpdateTemp); 441 } 442 assert(haveNoOverlap(_areasToUpdateNonOverlapping[])); 443 444 foreach(dirtyRect; _areasToUpdateNonOverlapping) 445 { 446 assert(dirtyRect.isSorted); 447 assert(!dirtyRect.empty); 448 _areasToRender.pushBack( extendsDirtyRect(dirtyRect, _askedWidth, _askedHeight) ); 449 } 450 } 451 452 box2i extendsDirtyRect(box2i rect, int width, int height) nothrow @nogc 453 { 454 int xmin = rect.min.x - _updateMargin; 455 int ymin = rect.min.y - _updateMargin; 456 int xmax = rect.max.x + _updateMargin; 457 int ymax = rect.max.y + _updateMargin; 458 459 if (xmin < 0) xmin = 0; 460 if (ymin < 0) ymin = 0; 461 if (xmax > width) xmax = width; 462 if (ymax > height) ymax = height; 463 464 // This could also happen if an UIElement is moved quickly 465 if (xmax < 0) xmax = 0; 466 if (ymax < 0) ymax = 0; 467 if (xmin > width) xmin = width; 468 if (ymin > height) ymin = height; 469 470 box2i result = box2i(xmin, ymin, xmax, ymax); 471 assert(result.isSorted); 472 return result; 473 } 474 475 ImageRef!RGBA doResize(int width, int height) nothrow @nogc 476 { 477 _askedWidth = width; 478 _askedHeight = height; 479 480 reflow(box2i(0, 0, _askedWidth, _askedHeight)); 481 482 // FUTURE: maybe not destroy the whole mipmap? 483 _diffuseMap.size(5, width, height); 484 _depthMap.size(4, width, height); 485 _materialMap.size(0, width, height); 486 487 // Extends buffer 488 size_t sizeNeeded = byteStride(width) * height; 489 _renderedBuffer = cast(ubyte*) alignedRealloc(_renderedBuffer, sizeNeeded, 16); 490 491 ImageRef!RGBA wfb; 492 wfb.w = _askedWidth; 493 wfb.h = _askedHeight; 494 wfb.pitch = byteStride(_askedWidth); 495 wfb.pixels = cast(RGBA*)(_renderedBuffer); 496 return wfb; 497 } 498 499 /// Redraw UIElements 500 void renderElements() nothrow @nogc 501 { 502 // recompute draw list 503 _elemsToDraw.clearContents(); 504 getDrawList(_elemsToDraw); 505 506 // Sort by ascending z-order (high z-order gets drawn last) 507 // This sort must be stable to avoid messing with tree natural order. 508 _elemsToDrawScratch.resize(_elemsToDraw.length); 509 int compareZOrder(in UIElement a, in UIElement b) nothrow @nogc 510 { 511 return a.zOrder() - b.zOrder(); 512 } 513 mergeSort!UIElement(_elemsToDraw[], _elemsToDrawScratch[] , &compareZOrder); 514 515 enum bool parallelDraw = true; 516 517 auto diffuseRef = _diffuseMap.levels[0].toRef(); 518 auto depthRef = _depthMap.levels[0].toRef(); 519 auto materialRef = _materialMap.levels[0].toRef(); 520 521 static if (parallelDraw) 522 { 523 int drawn = 0; 524 int maxParallelElements = 32; 525 int N = cast(int)_elemsToDraw.length; 526 527 while(drawn < N) 528 { 529 int canBeDrawn = 1; // at least one can be drawn without collision 530 531 // Search max number of parallelizable draws until the end of the list or a collision is found 532 bool foundIntersection = false; 533 for ( ; (canBeDrawn < maxParallelElements) && (drawn + canBeDrawn < N); ++canBeDrawn) 534 { 535 box2i candidate = _elemsToDraw[drawn + canBeDrawn].position; 536 537 for (int j = 0; j < canBeDrawn; ++j) 538 { 539 if (_elemsToDraw[drawn + j].position.intersects(candidate)) 540 { 541 foundIntersection = true; 542 break; 543 } 544 } 545 if (foundIntersection) 546 break; 547 } 548 549 assert(canBeDrawn >= 1 && canBeDrawn <= maxParallelElements); 550 551 // Draw a number of UIElement in parallel 552 void drawOneItem(int i) nothrow @nogc 553 { 554 _elemsToDraw[drawn + i].render(diffuseRef, depthRef, materialRef, _areasToUpdateNonOverlapping[]); 555 } 556 //_threadPool.waitForCompletion(); 557 _threadPool.parallelFor/*Async*/(canBeDrawn, &drawOneItem); 558 559 drawn += canBeDrawn; 560 assert(drawn <= N); 561 } 562 assert(drawn == N); 563 } 564 else 565 { 566 // Render required areas in diffuse and depth maps, base level 567 foreach(elem; _elemsToDraw) 568 elem.render(diffuseRef, depthRef, _areasToUpdate[]); 569 } 570 } 571 572 /// Compose lighting effects from depth and diffuse into a result. 573 /// takes output image and non-overlapping areas as input 574 /// Useful multithreading code. 575 void compositeGUI(ImageRef!RGBA wfb, WindowPixelFormat pf) nothrow @nogc 576 { 577 // Was tuned for performance, maybe the tradeoff has changed now that we use LDC. 578 enum tileWidth = 64; 579 enum tileHeight = 32; 580 581 _areasToRenderNonOverlappingTiled.clearContents(); 582 tileAreas(_areasToRenderNonOverlapping[], tileWidth, tileHeight,_areasToRenderNonOverlappingTiled); 583 584 int numAreas = cast(int)_areasToRenderNonOverlappingTiled.length; 585 586 void compositeOneTile(int i) nothrow @nogc 587 { 588 compositor.compositeTile(wfb, pf, _areasToRenderNonOverlappingTiled[i], 589 _diffuseMap, _materialMap, _depthMap, context.skybox); 590 } 591 //_threadPool.waitForCompletion(); // wait for mipmaps to be generated 592 _threadPool.parallelFor/*Async*/(numAreas, &compositeOneTile); 593 } 594 595 /// Compose lighting effects from depth and diffuse into a result. 596 /// takes output image and non-overlapping areas as input 597 /// Useful multithreading code. 598 void regenerateMipmaps() nothrow @nogc 599 { 600 int numAreas = cast(int)_areasToUpdateNonOverlapping.length; 601 602 // Fill update rect buffer with the content of _areasToUpdateNonOverlapping 603 for (int i = 0; i < 2; ++i) 604 { 605 _updateRectScratch[i].clearContents(); 606 _updateRectScratch[i].pushBack(_areasToUpdateNonOverlapping[]); 607 } 608 609 // We can't use tiled parallelism for mipmapping here because there is overdraw beyond level 0 610 // So instead what we do is using up to 2 threads. 611 void processOneMipmap(int i) nothrow @nogc 612 { 613 if (i == 0) 614 { 615 // diffuse 616 Mipmap!RGBA mipmap = _diffuseMap; 617 int levelMax = min(mipmap.numLevels(), 5); 618 foreach(level; 1 .. mipmap.numLevels()) 619 { 620 Mipmap!RGBA.Quality quality; 621 if (level == 1) 622 quality = Mipmap!RGBA.Quality.boxAlphaCovIntoPremul; 623 else 624 quality = Mipmap!RGBA.Quality.cubic; 625 626 foreach(ref area; _updateRectScratch[i]) 627 { 628 area = mipmap.generateNextLevel(quality, area, level); 629 } 630 } 631 } 632 else 633 { 634 // depth 635 Mipmap!L16 mipmap = _depthMap; 636 foreach(level; 1 .. mipmap.numLevels()) 637 { 638 auto quality = level >= 3 ? Mipmap!L16.Quality.cubic : Mipmap!L16.Quality.box; 639 foreach(ref area; _updateRectScratch[i]) 640 { 641 area = mipmap.generateNextLevel(quality, area, level); 642 } 643 } 644 } 645 } 646 //_threadPool.waitForCompletion(); // wait for UI widgets to be drawn 647 _threadPool.parallelFor/*Async*/(2, &processOneMipmap); 648 } 649 } 650 651 652 enum scanLineAlignment = 4; // could be anything 653 654 // given a width, how long in bytes should scanlines be 655 int byteStride(int width) pure nothrow @nogc 656 { 657 int widthInBytes = width * 4; 658 return (widthInBytes + (scanLineAlignment - 1)) & ~(scanLineAlignment-1); 659 }