1 /** 2 * A GUIGraphics is the interface between a plugin client and a IWindow. 3 * 4 * Copyright: Copyright Auburn Sounds 2015 and later. 5 * License: $(LINK2 http://www.boost.org/LICENSE_1_0.txt, Boost License 1.0) 6 * Authors: Guillaume Piolat 7 */ 8 module dplug.gui.graphics; 9 10 import std.math; 11 import std.algorithm.comparison; 12 import std.algorithm.sorting; 13 import std.algorithm.mutation; 14 15 import dplug.core.math; 16 import dplug.core.thread; 17 18 import dplug.client.client; 19 import dplug.client.graphics; 20 import dplug.client.daw; 21 22 import dplug.window.window; 23 24 import dplug.graphics.mipmap; 25 26 import dplug.gui.boxlist; 27 import dplug.gui.context; 28 import dplug.gui.element; 29 import dplug.gui.compositor; 30 31 /// In the whole package: 32 /// The diffuse maps contains: 33 /// RGBA = red/green/blue/emissiveness 34 /// The depth maps contains depth. 35 /// The material map contains: 36 /// RGBA = roughness / metalness / specular / physical (allows to bypass PBR) 37 38 alias RMSP = RGBA; // reminder 39 40 // Uncomment to benchmark compositing and devise optimizations. 41 //version = BenchmarkCompositing; 42 43 // A GUIGraphics is the interface between a plugin client and a IWindow. 44 // It is also an UIElement and the root element of the plugin UI hierarchy. 45 // You have to derive it to have a GUI. 46 // It dispatches window events to the GUI hierarchy. 47 class GUIGraphics : UIElement, IGraphics 48 { 49 nothrow: 50 @nogc: 51 52 ICompositor compositor; 53 54 this(int initialWidth, int initialHeight) 55 { 56 _uiContext = mallocNew!UIContext(); 57 super(_uiContext); 58 59 // Don't like the default rendering? Make another compositor. 60 compositor = mallocNew!PBRCompositor(); 61 62 _windowListener = mallocNew!WindowListener(this); 63 64 _window = null; 65 _askedWidth = initialWidth; 66 _askedHeight = initialHeight; 67 68 _threadPool = mallocNew!ThreadPool(); 69 70 _areasToUpdateNonOverlapping = makeVec!box2i; 71 _areasToUpdateTemp = makeVec!box2i; 72 73 _updateRectScratch[0] = makeVec!box2i; 74 _updateRectScratch[1] = makeVec!box2i; 75 76 _areasToRender = makeVec!box2i; 77 _areasToRenderNonOverlapping = makeVec!box2i; 78 _areasToRenderNonOverlappingTiled = makeVec!box2i; 79 80 _elemsToDraw = makeVec!UIElement; 81 82 version(BenchmarkCompositing) 83 { 84 _compositingWatch = new StopWatch("Compositing = "); 85 _drawWatch = new StopWatch("Draw = "); 86 _mipmapWatch = new StopWatch("Mipmap = "); 87 } 88 89 _diffuseMap = mallocNew!(Mipmap!RGBA)(); 90 _materialMap = mallocNew!(Mipmap!RGBA)(); 91 _depthMap = mallocNew!(Mipmap!L16)(); 92 } 93 94 95 ~this() 96 { 97 closeUI(); 98 _uiContext.destroyFree(); 99 100 _threadPool.destroyFree(); 101 102 compositor.destroyFree(); 103 _diffuseMap.destroyFree(); 104 _materialMap.destroyFree(); 105 _depthMap.destroyFree(); 106 107 _windowListener.destroyFree(); 108 alignedFree(_renderedBuffer, 16); 109 } 110 111 // Graphics implementation 112 113 override void* openUI(void* parentInfo, void* controlInfo, DAW daw, GraphicsBackend backend) 114 { 115 WindowBackend wbackend = void; 116 final switch(backend) 117 { 118 case GraphicsBackend.autodetect: wbackend = WindowBackend.autodetect; break; 119 case GraphicsBackend.win32: wbackend = WindowBackend.win32; break; 120 case GraphicsBackend.cocoa: wbackend = WindowBackend.cocoa; break; 121 case GraphicsBackend.carbon: wbackend = WindowBackend.carbon; break; 122 case GraphicsBackend.x11: wbackend = WindowBackend.x11; break; 123 } 124 125 // We create this window each time. 126 _window = createWindow(WindowUsage.plugin, parentInfo, controlInfo, _windowListener, wbackend, _askedWidth, _askedHeight); 127 128 reflow(box2i(0, 0, _askedWidth, _askedHeight)); 129 130 // Sets the whole UI dirty 131 setDirtyWhole(); 132 133 return _window.systemHandle(); 134 } 135 136 override void closeUI() 137 { 138 // Destroy window. 139 if (_window !is null) 140 { 141 _window.destroyFree(); 142 _window = null; 143 } 144 } 145 146 override void getGUISize(int* width, int* height) 147 { 148 *width = _askedWidth; 149 *height = _askedHeight; 150 } 151 152 version(BenchmarkCompositing) 153 { 154 class StopWatch 155 { 156 this(string title) 157 { 158 _title = title; 159 } 160 161 void start() 162 { 163 _lastTime = _window.getTimeMs(); 164 } 165 166 enum WARMUP = 30; 167 168 void stop() 169 { 170 uint now = _window.getTimeMs(); 171 int timeDiff = cast(int)(now - _lastTime); 172 173 if (times >= WARMUP) 174 sum += timeDiff; // first samples are discarded 175 176 times++; 177 string msg = _title ~ to!string(timeDiff) ~ " ms"; 178 _window.debugOutput(msg); 179 } 180 181 void displayMean() 182 { 183 if (times > WARMUP) 184 { 185 string msg = _title ~ to!string(sum / (times - WARMUP)) ~ " ms mean"; 186 _window.debugOutput(msg); 187 } 188 } 189 190 string _title; 191 uint _lastTime; 192 double sum = 0; 193 int times = 0; 194 } 195 } 196 197 198 // This class is only here to avoid name conflicts between 199 // UIElement and IWindowListener methods :| 200 // Explicit outer to avoid emplace crashing 201 static class WindowListener : IWindowListener 202 { 203 nothrow: 204 @nogc: 205 GUIGraphics outer; 206 207 this(GUIGraphics outer) 208 { 209 this.outer = outer; 210 } 211 212 override bool onMouseClick(int x, int y, MouseButton mb, bool isDoubleClick, MouseState mstate) 213 { 214 return outer.mouseClick(x, y, mb, isDoubleClick, mstate); 215 } 216 217 override bool onMouseRelease(int x, int y, MouseButton mb, MouseState mstate) 218 { 219 outer.mouseRelease(x, y, mb, mstate); 220 return true; 221 } 222 223 override bool onMouseWheel(int x, int y, int wheelDeltaX, int wheelDeltaY, MouseState mstate) 224 { 225 return outer.mouseWheel(x, y, wheelDeltaX, wheelDeltaY, mstate); 226 } 227 228 override void onMouseMove(int x, int y, int dx, int dy, MouseState mstate) 229 { 230 outer.mouseMove(x, y, dx, dy, mstate); 231 } 232 233 override void recomputeDirtyAreas() 234 { 235 return outer.recomputeDirtyAreas(); 236 } 237 238 override bool onKeyDown(Key key) 239 { 240 // Sends the event to the last clicked element first 241 if (outer._uiContext.focused !is null) 242 if (outer._uiContext.focused.onKeyDown(key)) 243 return true; 244 245 // else to all Elements 246 return outer.keyDown(key); 247 } 248 249 override bool onKeyUp(Key key) 250 { 251 // Sends the event to the last clicked element first 252 if (outer._uiContext.focused !is null) 253 if (outer._uiContext.focused.onKeyUp(key)) 254 return true; 255 // else to all Elements 256 return outer.keyUp(key); 257 } 258 259 /// Returns areas affected by updates. 260 override box2i getDirtyRectangle() nothrow @nogc 261 { 262 return outer._areasToRender[].boundingBox(); 263 } 264 265 override ImageRef!RGBA onResized(int width, int height) 266 { 267 return outer.doResize(width, height); 268 } 269 270 // Redraw dirtied controls in depth and diffuse maps. 271 // Update composited cache. 272 override void onDraw(WindowPixelFormat pf) nothrow @nogc 273 { 274 return outer.doDraw(pf); 275 } 276 277 override void onMouseCaptureCancelled() 278 { 279 // Stop an eventual drag operation 280 outer._uiContext.stopDragging(); 281 } 282 283 override void onAnimate(double dt, double time) 284 { 285 outer.animate(dt, time); 286 } 287 } 288 289 /// Tune this to tune the trade-off between light quality and speed. 290 /// The default value was tuned by hand on very shiny light sources. 291 /// Too high and processing becomes very expensive. 292 /// Too little and the ligth decay doesn't feel natural. 293 void setUpdateMargin(int margin = 20) nothrow @nogc 294 { 295 _updateMargin = margin; 296 } 297 298 protected: 299 300 UIContext _uiContext; 301 302 WindowListener _windowListener; 303 304 // An interface to the underlying window 305 IWindow _window; 306 307 // Task pool for multi-threaded image work 308 ThreadPool _threadPool; 309 310 int _askedWidth = 0; 311 int _askedHeight = 0; 312 313 // Diffuse color values for the whole UI. 314 Mipmap!RGBA _diffuseMap; 315 316 // Depth values for the whole UI. 317 Mipmap!L16 _depthMap; 318 319 // Depth values for the whole UI. 320 Mipmap!RGBA _materialMap; 321 322 // The list of areas whose diffuse/depth data have been changed. 323 Vec!box2i _areasToUpdateNonOverlapping; 324 325 // Used to maintain the _areasToUpdate invariant of no overlap 326 Vec!box2i _areasToUpdateTemp; 327 328 // Same, but temporary variable for mipmap generation 329 Vec!box2i[2] _updateRectScratch; 330 331 // The list of areas that must be effectively updated in the composite buffer 332 // (sligthly larger than _areasToUpdate). 333 Vec!box2i _areasToRender; 334 335 // same list, but reorganized to avoid overlap 336 Vec!box2i _areasToRenderNonOverlapping; 337 338 // same list, but separated in smaller tiles 339 Vec!box2i _areasToRenderNonOverlappingTiled; 340 341 // The list of UIElement to draw 342 // Note: AlignedBuffer memory isn't scanned, 343 // but this doesn't matter since UIElement are the UI hierarchy anyway. 344 Vec!UIElement _elemsToDraw; 345 346 /// Amount of pixels dirty rectangles are extended with. 347 int _updateMargin = 20; 348 349 // The fully rendered framebuffer. 350 // This should point into comitted virtual memory for faster (maybe) upload to device 351 ubyte* _renderedBuffer = null; 352 353 version(BenchmarkCompositing) 354 { 355 StopWatch _compositingWatch; 356 StopWatch _mipmapWatch; 357 StopWatch _drawWatch; 358 } 359 360 void doDraw(WindowPixelFormat pf) nothrow @nogc 361 { 362 ImageRef!RGBA wfb; 363 wfb.w = _askedWidth; 364 wfb.h = _askedHeight; 365 wfb.pitch = byteStride(_askedWidth); 366 wfb.pixels = cast(RGBA*)_renderedBuffer; 367 368 // Composite GUI 369 // Most of the cost of rendering is here 370 version(BenchmarkCompositing) 371 _drawWatch.start(); 372 373 renderElements(); 374 375 version(BenchmarkCompositing) 376 { 377 _drawWatch.stop(); 378 _drawWatch.displayMean(); 379 } 380 381 version(BenchmarkCompositing) 382 _mipmapWatch.start(); 383 384 // Split boxes to avoid overlapped work 385 // Note: this is done separately for update areas and render areas 386 _areasToRenderNonOverlapping.clearContents(); 387 removeOverlappingAreas(_areasToRender, _areasToRenderNonOverlapping); 388 389 regenerateMipmaps(); 390 391 version(BenchmarkCompositing) 392 { 393 _mipmapWatch.stop(); 394 _mipmapWatch.displayMean(); 395 } 396 397 version(BenchmarkCompositing) 398 _compositingWatch.start(); 399 400 compositeGUI(wfb, pf); 401 402 // only then is the list of rectangles to update cleared 403 _areasToUpdateNonOverlapping.clearContents(); 404 405 // wait for compositing completion 406 //_threadPool.waitForCompletion(); 407 408 version(BenchmarkCompositing) 409 { 410 _compositingWatch.stop(); 411 _compositingWatch.displayMean(); 412 } 413 } 414 415 // Fills _areasToUpdate and _areasToRender 416 void recomputeDirtyAreas() nothrow @nogc 417 { 418 // Get areas to update 419 _areasToRender.clearContents(); 420 421 // First we pull dirty rectangles from the UI 422 context().dirtyList.pullAllRectangles(_areasToUpdateNonOverlapping); 423 424 // TECHNICAL DEBT HERE 425 // The problem here is that if the window isn't shown there may be duplicates in 426 // _areasToUpdate, so we have to maintain unicity again 427 // The code with dirty rects is a big mess, it needs a severe rewrite. 428 // 429 // SOLUTION 430 // The fundamental problem is that dirtyList should probably be merged with 431 // _areasToUpdateNonOverlapping. 432 // _areasToRender should also be purely derived from _areasToUpdateNonOverlapping 433 // Finally the interface of IWindowListener is poorly defined, this ties the window 434 // to the renderer in a bad way. 435 { 436 _areasToUpdateTemp.clearContents(); 437 removeOverlappingAreas(_areasToUpdateNonOverlapping, _areasToUpdateTemp); 438 _areasToUpdateNonOverlapping.clearContents(); 439 _areasToUpdateNonOverlapping.pushBack(_areasToUpdateTemp); 440 } 441 assert(haveNoOverlap(_areasToUpdateNonOverlapping[])); 442 443 foreach(dirtyRect; _areasToUpdateNonOverlapping) 444 { 445 assert(dirtyRect.isSorted); 446 assert(!dirtyRect.empty); 447 _areasToRender.pushBack( extendsDirtyRect(dirtyRect, _askedWidth, _askedHeight) ); 448 } 449 } 450 451 box2i extendsDirtyRect(box2i rect, int width, int height) nothrow @nogc 452 { 453 int xmin = rect.min.x - _updateMargin; 454 int ymin = rect.min.y - _updateMargin; 455 int xmax = rect.max.x + _updateMargin; 456 int ymax = rect.max.y + _updateMargin; 457 458 if (xmin < 0) xmin = 0; 459 if (ymin < 0) ymin = 0; 460 if (xmax > width) xmax = width; 461 if (ymax > height) ymax = height; 462 463 // This could also happen if an UIElement is moved quickly 464 if (xmax < 0) xmax = 0; 465 if (ymax < 0) ymax = 0; 466 if (xmin > width) xmin = width; 467 if (ymin > height) ymin = height; 468 469 box2i result = box2i(xmin, ymin, xmax, ymax); 470 assert(result.isSorted); 471 return result; 472 } 473 474 ImageRef!RGBA doResize(int width, int height) nothrow @nogc 475 { 476 _askedWidth = width; 477 _askedHeight = height; 478 479 reflow(box2i(0, 0, _askedWidth, _askedHeight)); 480 481 // FUTURE: maybe not destroy the whole mipmap? 482 _diffuseMap.size(5, width, height); 483 _depthMap.size(4, width, height); 484 _materialMap.size(0, width, height); 485 486 // Extends buffer 487 size_t sizeNeeded = byteStride(width) * height; 488 _renderedBuffer = cast(ubyte*) alignedRealloc(_renderedBuffer, sizeNeeded, 16); 489 490 ImageRef!RGBA wfb; 491 wfb.w = _askedWidth; 492 wfb.h = _askedHeight; 493 wfb.pitch = byteStride(_askedWidth); 494 wfb.pixels = cast(RGBA*)(_renderedBuffer); 495 return wfb; 496 } 497 498 /// Redraw UIElements 499 void renderElements() nothrow @nogc 500 { 501 // recompute draw list 502 _elemsToDraw.clearContents(); 503 getDrawList(_elemsToDraw); 504 505 // Sort by ascending z-order (high z-order gets drawn last) 506 // This sort must be stable to avoid messing with tree natural order. 507 int compareZOrder(in UIElement a, in UIElement b) nothrow @nogc 508 { 509 return a.zOrder() - b.zOrder(); 510 } 511 grailSort!UIElement(_elemsToDraw[], &compareZOrder); 512 513 enum bool parallelDraw = true; 514 515 auto diffuseRef = _diffuseMap.levels[0].toRef(); 516 auto depthRef = _depthMap.levels[0].toRef(); 517 auto materialRef = _materialMap.levels[0].toRef(); 518 519 static if (parallelDraw) 520 { 521 int drawn = 0; 522 int maxParallelElements = 32; 523 int N = cast(int)_elemsToDraw.length; 524 525 while(drawn < N) 526 { 527 int canBeDrawn = 1; // at least one can be drawn without collision 528 529 // Search max number of parallelizable draws until the end of the list or a collision is found 530 bool foundIntersection = false; 531 for ( ; (canBeDrawn < maxParallelElements) && (drawn + canBeDrawn < N); ++canBeDrawn) 532 { 533 box2i candidate = _elemsToDraw[drawn + canBeDrawn].position; 534 535 for (int j = 0; j < canBeDrawn; ++j) 536 { 537 if (_elemsToDraw[drawn + j].position.intersects(candidate)) 538 { 539 foundIntersection = true; 540 break; 541 } 542 } 543 if (foundIntersection) 544 break; 545 } 546 547 assert(canBeDrawn >= 1 && canBeDrawn <= maxParallelElements); 548 549 // Draw a number of UIElement in parallel 550 void drawOneItem(int i) nothrow @nogc 551 { 552 _elemsToDraw[drawn + i].render(diffuseRef, depthRef, materialRef, _areasToUpdateNonOverlapping[]); 553 } 554 //_threadPool.waitForCompletion(); 555 _threadPool.parallelFor/*Async*/(canBeDrawn, &drawOneItem); 556 557 drawn += canBeDrawn; 558 assert(drawn <= N); 559 } 560 assert(drawn == N); 561 } 562 else 563 { 564 // Render required areas in diffuse and depth maps, base level 565 foreach(elem; _elemsToDraw) 566 elem.render(diffuseRef, depthRef, _areasToUpdate[]); 567 } 568 } 569 570 /// Compose lighting effects from depth and diffuse into a result. 571 /// takes output image and non-overlapping areas as input 572 /// Useful multithreading code. 573 void compositeGUI(ImageRef!RGBA wfb, WindowPixelFormat pf) nothrow @nogc 574 { 575 // Was tuned for performance, maybe the tradeoff has changed now that we use LDC. 576 enum tileWidth = 64; 577 enum tileHeight = 32; 578 579 _areasToRenderNonOverlappingTiled.clearContents(); 580 tileAreas(_areasToRenderNonOverlapping[], tileWidth, tileHeight,_areasToRenderNonOverlappingTiled); 581 582 int numAreas = cast(int)_areasToRenderNonOverlappingTiled.length; 583 584 void compositeOneTile(int i) nothrow @nogc 585 { 586 compositor.compositeTile(wfb, pf, _areasToRenderNonOverlappingTiled[i], 587 _diffuseMap, _materialMap, _depthMap, context.skybox); 588 } 589 //_threadPool.waitForCompletion(); // wait for mipmaps to be generated 590 _threadPool.parallelFor/*Async*/(numAreas, &compositeOneTile); 591 } 592 593 /// Compose lighting effects from depth and diffuse into a result. 594 /// takes output image and non-overlapping areas as input 595 /// Useful multithreading code. 596 void regenerateMipmaps() nothrow @nogc 597 { 598 int numAreas = cast(int)_areasToUpdateNonOverlapping.length; 599 600 // Fill update rect buffer with the content of _areasToUpdateNonOverlapping 601 for (int i = 0; i < 2; ++i) 602 { 603 _updateRectScratch[i].clearContents(); 604 _updateRectScratch[i].pushBack(_areasToUpdateNonOverlapping[]); 605 } 606 607 // We can't use tiled parallelism for mipmapping here because there is overdraw beyond level 0 608 // So instead what we do is using up to 2 threads. 609 void processOneMipmap(int i) nothrow @nogc 610 { 611 if (i == 0) 612 { 613 // diffuse 614 Mipmap!RGBA mipmap = _diffuseMap; 615 int levelMax = min(mipmap.numLevels(), 5); 616 foreach(level; 1 .. mipmap.numLevels()) 617 { 618 Mipmap!RGBA.Quality quality; 619 if (level == 1) 620 quality = Mipmap!RGBA.Quality.boxAlphaCovIntoPremul; 621 else 622 quality = Mipmap!RGBA.Quality.cubic; 623 624 foreach(ref area; _updateRectScratch[i]) 625 { 626 area = mipmap.generateNextLevel(quality, area, level); 627 } 628 } 629 } 630 else 631 { 632 // depth 633 Mipmap!L16 mipmap = _depthMap; 634 foreach(level; 1 .. mipmap.numLevels()) 635 { 636 auto quality = level >= 3 ? Mipmap!L16.Quality.cubic : Mipmap!L16.Quality.box; 637 foreach(ref area; _updateRectScratch[i]) 638 { 639 area = mipmap.generateNextLevel(quality, area, level); 640 } 641 } 642 } 643 } 644 //_threadPool.waitForCompletion(); // wait for UI widgets to be drawn 645 _threadPool.parallelFor/*Async*/(2, &processOneMipmap); 646 } 647 } 648 649 650 enum scanLineAlignment = 4; // could be anything 651 652 // given a width, how long in bytes should scanlines be 653 int byteStride(int width) pure nothrow @nogc 654 { 655 int widthInBytes = width * 4; 656 return (widthInBytes + (scanLineAlignment - 1)) & ~(scanLineAlignment-1); 657 }