Add depth map export (closes #371)

docs/content.md

@@ -389,6 +389,21 @@ Using a **Video Editing Software** like **Adobe After Effects** or **Sony Vegas*
 
 > **Note:** For best results, you should **disable clouds** before rendering, as they are transparent.
 
+#### Depth Map [depth-map]
+![](img/depthmapsuzanne.jpg)
+"Suzanne" model added into a scene using Blender
+
+This setting causes depth information to be recorded and included in video formats which support it
+(currently only [OpenEXR](#replaying-render-settings-quality)).
+
+Using a video editor which supports it (e.g. [Blender](https://www.blender.org/)), you can then for example (among many
+other effects) add arbitrary geometry onto your rendered video in such a way that foreground objects in the
+video will still occlude background geometry as if it actually was part of the world.
+
+> **Note:** For best results, avoid semi-transparent blocks in front of your geometry, as they cannot be accurately represented in the depth map.
+
+> **Note:** Anti-aliasing is not supported when exporting depth. Instead simply render and edit your video at a higher resolution and scale it down at the very end.
+
 ### Command Line Settings [commandline]
 ![](img/rendersettings-commandline.jpg)
 The **Command Line Render Settings**

src/main/java/com/replaymod/extras/advancedscreenshots/ScreenshotWriter.java

@@ -5,6 +5,7 @@ import com.replaymod.core.utils.Utils;
 import com.replaymod.core.versions.MCVer;
 import com.replaymod.extras.ReplayModExtras;
 import com.replaymod.render.frame.BitmapFrame;
+import com.replaymod.render.rendering.Channel;
 import com.replaymod.render.rendering.FrameConsumer;
 import com.replaymod.replay.ReplayModReplay;
 import de.johni0702.minecraft.gui.utils.lwjgl.ReadableDimension;
@@ -13,6 +14,7 @@ import net.minecraft.util.crash.CrashReport;
 
 import java.io.File;
 import java.io.IOException;
+import java.util.Map;
 
 public class ScreenshotWriter implements FrameConsumer<BitmapFrame> {
 
@@ -23,7 +25,9 @@ public class ScreenshotWriter implements FrameConsumer<BitmapFrame> {
     }
 
     @Override
-    public void consume(BitmapFrame frame) {
+    public void consume(Map<Channel, BitmapFrame> channels) {
+        BitmapFrame frame = channels.get(Channel.BRGA);
+
         // skip the first frame, in which not all chunks are properly loaded
         if (frame.getFrameId() == 0) return;
 

src/main/java/com/replaymod/render/EXRWriter.java

@@ -2,7 +2,8 @@
 package com.replaymod.render;
 
 import com.replaymod.core.versions.MCVer;
-import com.replaymod.render.frame.EXRFrame;
+import com.replaymod.render.frame.BitmapFrame;
+import com.replaymod.render.rendering.Channel;
 import com.replaymod.render.rendering.FrameConsumer;
 import com.replaymod.render.utils.ByteBufferPool;
 import de.johni0702.minecraft.gui.utils.lwjgl.ReadableDimension;
@@ -18,12 +19,13 @@ import java.nio.FloatBuffer;
 import java.nio.IntBuffer;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.Map;
 
 import static org.lwjgl.system.MemoryStack.*;
 import static org.lwjgl.system.MemoryUtil.*;
 import static org.lwjgl.util.tinyexr.TinyEXR.*;
 
-public class EXRWriter implements FrameConsumer<EXRFrame> {
+public class EXRWriter implements FrameConsumer<BitmapFrame> {
 
     private final Path outputFolder;
 
@@ -34,13 +36,16 @@ public class EXRWriter implements FrameConsumer<EXRFrame> {
     }
 
     @Override
-    public void consume(EXRFrame frame) {
+    public void consume(Map<Channel, BitmapFrame> channels) {
-        Path path = outputFolder.resolve(frame.getFrameId() + ".exr");
+        BitmapFrame bgraFrame = channels.get(Channel.BRGA);
-        ReadableDimension size = frame.getSize();
+        BitmapFrame depthFrame = channels.get(Channel.DEPTH);
-        ByteBuffer bgra = frame.getBgraBuffer();
+
+        Path path = outputFolder.resolve(bgraFrame.getFrameId() + ".exr");
+        ReadableDimension size = bgraFrame.getSize();
+        ByteBuffer bgra = bgraFrame.getByteBuffer();
         int width = size.getWidth();
         int height = size.getHeight();
-        int numChannels = 3;
+        int numChannels = 4 + (depthFrame != null ? 1 : 0);
 
         stackPush();
         EXRHeader header = EXRHeader.mallocStack(); InitEXRHeader(header);
@@ -58,33 +63,45 @@ public class EXRWriter implements FrameConsumer<EXRFrame> {
             header.requested_pixel_types(requestedPixelTypes);
 
             // Some readers ignore this, so we use the most expected order
-            memASCII("B", true, channelInfos.get(0).name());
+            memASCII("A", true, channelInfos.get(0).name());
-            memASCII("G", true, channelInfos.get(1).name());
+            memASCII("B", true, channelInfos.get(1).name());
-            memASCII("R", true, channelInfos.get(2).name());
+            memASCII("G", true, channelInfos.get(2).name());
+            memASCII("R", true, channelInfos.get(3).name());
             for (int i = 0; i < numChannels; i++) {
                 pixelTypes.put(i, TINYEXR_PIXELTYPE_FLOAT);
                 requestedPixelTypes.put(i, TINYEXR_PIXELTYPE_HALF);
             }
+            if (depthFrame != null) {
+                memASCII("Z", true, channelInfos.get(4).name());
+                requestedPixelTypes.put(4, TINYEXR_PIXELTYPE_FLOAT);
+            }
 
             image.num_channels(numChannels);
             image.width(width);
             image.height(height);
             image.images(imagePointers);
-            FloatBuffer[] bgrChannels = new FloatBuffer[3];
+            FloatBuffer[] bgrChannels = new FloatBuffer[4];
+            FloatBuffer depthChannel = null;
             for (int i = 0; i < numChannels; i++) {
                 FloatBuffer channel = images.slice();
                 channel.position(width * height * i);
                 imagePointers.put(i, channel.slice());
-                bgrChannels[i] = channel;
+                if (i == 4) {
+                    depthChannel = channel;
+                } else {
+                    bgrChannels[(i + 3) % 4] = channel;
+                }
             }
             for (int y = 0; y < height; y++) {
                 for (int x = 0; x < width; x++) {
                     for (FloatBuffer channel : bgrChannels) {
                         channel.put(((int) bgra.get() & 0xff) / 255f);
                     }
-                    bgra.get(); // alpha
                 }
             }
+            if (depthFrame != null && depthChannel != null) {
+                depthChannel.put(depthFrame.getByteBuffer().asFloatBuffer());
+            }
 
             int ret = SaveEXRImageToFile(image, header, path.toString(), err);
             if (ret != TINYEXR_SUCCESS) {
@@ -97,7 +114,7 @@ public class EXRWriter implements FrameConsumer<EXRFrame> {
         } finally {
             memFree(images);
             stackPop();
-            ByteBufferPool.release(bgra);
+            channels.values().forEach(it -> ByteBufferPool.release(it.getByteBuffer()));
         }
     }
 

src/main/java/com/replaymod/render/FFmpegWriter.java

@@ -2,6 +2,7 @@ package com.replaymod.render;
 
 import com.replaymod.core.versions.MCVer;
 import com.replaymod.render.frame.BitmapFrame;
+import com.replaymod.render.rendering.Channel;
 import com.replaymod.render.rendering.FrameConsumer;
 import com.replaymod.render.rendering.VideoRenderer;
 import com.replaymod.render.utils.ByteBufferPool;
@@ -21,6 +22,7 @@ import java.io.IOException;
 import java.io.OutputStream;
 import java.nio.channels.Channels;
 import java.nio.channels.WritableByteChannel;
+import java.util.Map;
 import java.util.concurrent.TimeUnit;
 
 import static com.replaymod.render.ReplayModRender.LOGGER;
@@ -103,11 +105,11 @@ public class FFmpegWriter implements FrameConsumer<BitmapFrame> {
     }
 
     @Override
-    public void consume(BitmapFrame frame) {
+    public void consume(Map<Channel, BitmapFrame> channels) {
+        BitmapFrame frame = channels.get(Channel.BRGA);
         try {
             checkSize(frame.getSize());
             channel.write(frame.getByteBuffer());
-            ByteBufferPool.release(frame.getByteBuffer());
         } catch (Throwable t) {
             if (aborted) {
                 return;
@@ -127,6 +129,8 @@ public class FFmpegWriter implements FrameConsumer<BitmapFrame> {
             MCVer.addDetail(exportDetails, "Export command", settings::getExportCommand);
             MCVer.addDetail(exportDetails, "Export args", commandArgs::toString);
             MCVer.getMinecraft().setCrashReport(report);
+        } finally {
+            channels.values().forEach(it -> ByteBufferPool.release(it.getByteBuffer()));
         }
     }
 

src/main/java/com/replaymod/render/RenderSettings.java

@@ -165,6 +165,7 @@ public class RenderSettings {
     private final int sphericalFovX;
     private final int sphericalFovY;
     private final boolean injectSphericalMetadata;
+    private final boolean depthMap;
     private final AntiAliasing antiAliasing;
 
     private final String exportCommand;
@@ -195,6 +196,7 @@ public class RenderSettings {
             int sphericalFovX,
             int sphericalFovY,
             boolean injectSphericalMetadata,
+            boolean depthMap,
             AntiAliasing antiAliasing,
             String exportCommand,
             String exportArguments,
@@ -215,6 +217,7 @@ public class RenderSettings {
         this.sphericalFovX = sphericalFovX;
         this.sphericalFovY = sphericalFovY;
         this.injectSphericalMetadata = injectSphericalMetadata;
+        this.depthMap = depthMap;
         this.antiAliasing = antiAliasing;
         this.exportCommand = exportCommand;
         this.exportArguments = exportArguments;
@@ -238,6 +241,7 @@ public class RenderSettings {
                 sphericalFovX,
                 sphericalFovY,
                 injectSphericalMetadata,
+                depthMap,
                 antiAliasing,
                 exportCommand,
                 exportArguments,
@@ -412,6 +416,10 @@ public class RenderSettings {
         return injectSphericalMetadata;
     }
 
+    public boolean isDepthMap() {
+        return depthMap;
+    }
+
     public AntiAliasing getAntiAliasing() {
         return antiAliasing;
     }

src/main/java/com/replaymod/render/blend/BlendFrameCapturer.java

@@ -4,11 +4,14 @@ import com.replaymod.core.versions.MCVer;
 import com.replaymod.render.capturer.RenderInfo;
 import com.replaymod.render.capturer.WorldRenderer;
 import com.replaymod.render.frame.BitmapFrame;
+import com.replaymod.render.rendering.Channel;
 import com.replaymod.render.rendering.FrameCapturer;
 import com.replaymod.render.utils.ByteBufferPool;
 import de.johni0702.minecraft.gui.utils.lwjgl.Dimension;
 
 import java.io.IOException;
+import java.util.Collections;
+import java.util.Map;
 
 public class BlendFrameCapturer implements FrameCapturer<BitmapFrame> {
     protected final WorldRenderer worldRenderer;
@@ -26,7 +29,7 @@ public class BlendFrameCapturer implements FrameCapturer<BitmapFrame> {
     }
 
     @Override
-    public BitmapFrame process() {
+    public Map<Channel, BitmapFrame> process() {
         if (framesDone == 0) {
             BlendState.getState().setup();
         }
@@ -37,7 +40,8 @@ public class BlendFrameCapturer implements FrameCapturer<BitmapFrame> {
         worldRenderer.renderWorld(MCVer.getRenderPartialTicks(), null);
         BlendState.getState().postFrame(framesDone);
 
-        return new BitmapFrame(framesDone++, new Dimension(0, 0), 0, ByteBufferPool.allocate(0));
+        BitmapFrame frame = new BitmapFrame(framesDone++, new Dimension(0, 0), 0, ByteBufferPool.allocate(0));
+        return Collections.singletonMap(Channel.BRGA, frame);
     }
 
     @Override

src/main/java/com/replaymod/render/capturer/CubicOpenGlFrameCapturer.java

@@ -1,6 +1,10 @@
 package com.replaymod.render.capturer;
 
 import com.replaymod.render.frame.CubicOpenGlFrame;
+import com.replaymod.render.rendering.Channel;
+
+import java.util.Collections;
+import java.util.Map;
 
 public class CubicOpenGlFrameCapturer extends OpenGlFrameCapturer<CubicOpenGlFrame, CubicOpenGlFrameCapturer.Data> {
     public enum Data implements CaptureData {
@@ -25,14 +29,15 @@ public class CubicOpenGlFrameCapturer extends OpenGlFrameCapturer<CubicOpenGlFra
     }
 
     @Override
-    public CubicOpenGlFrame process() {
+    public Map<Channel, CubicOpenGlFrame> process() {
         float partialTicks = renderInfo.updateForNextFrame();
         int frameId = framesDone++;
-        return new CubicOpenGlFrame(renderFrame(frameId, partialTicks, Data.LEFT),
+        CubicOpenGlFrame frame =  new CubicOpenGlFrame(renderFrame(frameId, partialTicks, Data.LEFT),
                 renderFrame(frameId, partialTicks, Data.RIGHT),
                 renderFrame(frameId, partialTicks, Data.FRONT),
                 renderFrame(frameId, partialTicks, Data.BACK),
                 renderFrame(frameId, partialTicks, Data.TOP),
                 renderFrame(frameId, partialTicks, Data.BOTTOM));
+        return Collections.singletonMap(Channel.BRGA, frame);
     }
 }

src/main/java/com/replaymod/render/capturer/ODSFrameCapturer.java

@@ -1,5 +1,6 @@
 package com.replaymod.render.capturer;
 
+import com.replaymod.render.rendering.Channel;
 import de.johni0702.minecraft.gui.utils.EventRegistrations;
 import com.replaymod.render.RenderSettings;
 import com.replaymod.render.frame.CubicOpenGlFrame;
@@ -17,6 +18,8 @@ import net.minecraft.util.Identifier;
 import static com.mojang.blaze3d.platform.GlStateManager.*;
 
 import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
 
 import static org.lwjgl.opengl.GL11.GL_COLOR_BUFFER_BIT;
 import static org.lwjgl.opengl.GL11.GL_DEPTH_BUFFER_BIT;
@@ -125,16 +128,22 @@ public class ODSFrameCapturer implements FrameCapturer<ODSOpenGlFrame> {
     }
 
     @Override
-    public ODSOpenGlFrame process() {
+    public Map<Channel, ODSOpenGlFrame> process() {
         bindProgram();
         leftEyeVariable.set(true);
-        CubicOpenGlFrame leftFrame = left.process();
+        Map<Channel, CubicOpenGlFrame> leftChannels = left.process();
         leftEyeVariable.set(false);
-        CubicOpenGlFrame rightFrame = right.process();
+        Map<Channel, CubicOpenGlFrame> rightChannels = right.process();
         unbindProgram();
 
-        if (leftFrame != null && rightFrame != null) {
+        if (leftChannels != null && rightChannels != null) {
-            return new ODSOpenGlFrame(leftFrame, rightFrame);
+            Map<Channel, ODSOpenGlFrame> result = new HashMap<>();
+            for (Channel channel : Channel.values()) {
+                CubicOpenGlFrame leftFrame = leftChannels.get(channel);
+                CubicOpenGlFrame rightFrame = rightChannels.get(channel);
+                result.put(channel, new ODSOpenGlFrame(leftFrame, rightFrame));
+            }
+            return result;
         }
         return null;
     }

src/main/java/com/replaymod/render/capturer/PboOpenGlFrameCapturer.java

@@ -1,6 +1,8 @@
 package com.replaymod.render.capturer;
 
+import com.mojang.blaze3d.platform.GlStateManager;
 import com.replaymod.render.frame.OpenGlFrame;
+import com.replaymod.render.rendering.Channel;
 import com.replaymod.render.rendering.Frame;
 import com.replaymod.render.utils.ByteBufferPool;
 import com.replaymod.render.utils.PixelBufferObject;
@@ -9,17 +11,21 @@ import org.lwjgl.opengl.GL12;
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
+import java.util.HashMap;
+import java.util.Map;
 
 public abstract class PboOpenGlFrameCapturer<F extends Frame, D extends Enum<D> & CaptureData>
         extends OpenGlFrameCapturer<F, D> {
+    private final boolean withDepth;
     private final D[] data;
     private PixelBufferObject pbo, otherPBO;
 
     public PboOpenGlFrameCapturer(WorldRenderer worldRenderer, RenderInfo renderInfo, Class<D> type, int framePixels) {
         super(worldRenderer, renderInfo);
 
+        withDepth = renderInfo.getRenderSettings().isDepthMap();
         data = type.getEnumConstants();
-        int bufferSize = framePixels * 4 * data.length;
+        int bufferSize = framePixels * (4 /* bgra */ + (withDepth ? 4 /* float */ : 0)) * data.length;
         pbo = new PixelBufferObject(bufferSize, PixelBufferObject.Usage.READ);
         otherPBO = new PixelBufferObject(bufferSize, PixelBufferObject.Usage.READ);
     }
@@ -37,29 +43,36 @@ public abstract class PboOpenGlFrameCapturer<F extends Frame, D extends Enum<D>
         return framesDone >= renderInfo.getTotalFrames() + 2;
     }
 
+    private F readFromPbo(ByteBuffer pboBuffer, int bytesPerPixel) {
+        OpenGlFrame[] frames = new OpenGlFrame[data.length];
+        int frameBufferSize = getFrameWidth() * getFrameHeight() * bytesPerPixel;
+        for (int i = 0; i < frames.length; i++) {
+            ByteBuffer frameBuffer = ByteBufferPool.allocate(frameBufferSize);
+            pboBuffer.limit(pboBuffer.position() + frameBufferSize);
+            frameBuffer.put(pboBuffer);
+            frameBuffer.rewind();
+            frames[i] = new OpenGlFrame(framesDone - 2, frameSize, bytesPerPixel, frameBuffer);
+        }
+        return create(frames);
+    }
+
     @Override
-    public F process() {
+    public Map<Channel, F> process() {
-        F frame = null;
+        Map<Channel, F> channels = null;
 
         if (framesDone > 1) {
             // Read pbo to memory
             pbo.bind();
             ByteBuffer pboBuffer = pbo.mapReadOnly();
 
-            OpenGlFrame[] frames = new OpenGlFrame[data.length];
+            channels = new HashMap<>();
-            int frameBufferSize = getFrameWidth() * getFrameHeight() * 4;
+            channels.put(Channel.BRGA, readFromPbo(pboBuffer, 4));
-            for (int i = 0; i < frames.length; i++) {
+            if (withDepth) {
-                ByteBuffer frameBuffer = ByteBufferPool.allocate(frameBufferSize);
+                channels.put(Channel.DEPTH, readFromPbo(pboBuffer, 4));
-                pboBuffer.limit(pboBuffer.position() + frameBufferSize);
-                frameBuffer.put(pboBuffer);
-                frameBuffer.rewind();
-                frames[i] = new OpenGlFrame(framesDone - 2, frameSize, 4, frameBuffer);
             }
 
             pbo.unmap();
             pbo.unbind();
-
-            frame = create(frames);
         }
 
         if (framesDone < renderInfo.getTotalFrames()) {
@@ -72,7 +85,7 @@ public abstract class PboOpenGlFrameCapturer<F extends Frame, D extends Enum<D>
 
         framesDone++;
         swapPBOs();
-        return frame;
+        return channels;
     }
 
     @Override
@@ -82,6 +95,10 @@ public abstract class PboOpenGlFrameCapturer<F extends Frame, D extends Enum<D>
         int offset = captureData.ordinal() * getFrameWidth() * getFrameHeight() * 4;
         frameBuffer().beginWrite(true);
         GL11.glReadPixels(0, 0, getFrameWidth(), getFrameHeight(), GL12.GL_BGRA, GL11.GL_UNSIGNED_BYTE, offset);
+        if (withDepth) {
+            offset += data.length * getFrameWidth() * getFrameHeight() * 4;
+            GL11.glReadPixels(0, 0, getFrameWidth(), getFrameHeight(), GL11.GL_DEPTH_COMPONENT, GL11.GL_FLOAT, offset);
+        }
         frameBuffer().endWrite();
 
         pbo.unbind();

src/main/java/com/replaymod/render/capturer/SimpleOpenGlFrameCapturer.java

@@ -1,6 +1,10 @@
 package com.replaymod.render.capturer;
 
 import com.replaymod.render.frame.OpenGlFrame;
+import com.replaymod.render.rendering.Channel;
+
+import java.util.Collections;
+import java.util.Map;
 
 public class SimpleOpenGlFrameCapturer extends OpenGlFrameCapturer<OpenGlFrame, CaptureData> {
 
@@ -9,8 +13,9 @@ public class SimpleOpenGlFrameCapturer extends OpenGlFrameCapturer<OpenGlFrame,
     }
 
     @Override
-    public OpenGlFrame process() {
+    public Map<Channel, OpenGlFrame> process() {
         float partialTicks = renderInfo.updateForNextFrame();
-        return renderFrame(framesDone++, partialTicks);
+        OpenGlFrame frame = renderFrame(framesDone++, partialTicks);
+        return Collections.singletonMap(Channel.BRGA, frame);
     }
 }

src/main/java/com/replaymod/render/capturer/StereoscopicOpenGlFrameCapturer.java

@@ -2,6 +2,10 @@ package com.replaymod.render.capturer;
 
 import com.replaymod.render.frame.OpenGlFrame;
 import com.replaymod.render.frame.StereoscopicOpenGlFrame;
+import com.replaymod.render.rendering.Channel;
+
+import java.util.Collections;
+import java.util.Map;
 
 public class StereoscopicOpenGlFrameCapturer
         extends OpenGlFrameCapturer<StereoscopicOpenGlFrame, StereoscopicOpenGlFrameCapturer.Data> {
@@ -19,11 +23,12 @@ public class StereoscopicOpenGlFrameCapturer
     }
 
     @Override
-    public StereoscopicOpenGlFrame process() {
+    public Map<Channel, StereoscopicOpenGlFrame> process() {
         float partialTicks = renderInfo.updateForNextFrame();
         int frameId = framesDone++;
         OpenGlFrame left = renderFrame(frameId, partialTicks, Data.LEFT_EYE);
         OpenGlFrame right = renderFrame(frameId, partialTicks, Data.RIGHT_EYE);
-        return new StereoscopicOpenGlFrame(left, right);
+        StereoscopicOpenGlFrame frame = new StereoscopicOpenGlFrame(left, right);
+        return Collections.singletonMap(Channel.BRGA, frame);
     }
 }

src/main/java/com/replaymod/render/frame/EXRFrame.java

@@ -1,34 +0,0 @@
-package com.replaymod.render.frame;
-
-import com.replaymod.render.rendering.Frame;
-import de.johni0702.minecraft.gui.utils.lwjgl.ReadableDimension;
-import org.apache.commons.lang3.Validate;
-
-import java.nio.ByteBuffer;
-
-public class EXRFrame implements Frame {
-    private final int frameId;
-    private final ReadableDimension size;
-    private final ByteBuffer bgraBuffer;
-
-    public EXRFrame(int frameId, ReadableDimension size, ByteBuffer bgraBuffer) {
-        Validate.isTrue(size.getWidth() * size.getHeight() * 4 == bgraBuffer.remaining(),
-                "BGRA buffer size is %d (cap: %d) but should be %d",
-                bgraBuffer.remaining(), bgraBuffer.capacity(), size.getWidth() * size.getHeight() * 4);
-        this.frameId = frameId;
-        this.size = size;
-        this.bgraBuffer = bgraBuffer;
-    }
-
-    public int getFrameId() {
-        return this.frameId;
-    }
-
-    public ReadableDimension getSize() {
-        return this.size;
-    }
-
-    public ByteBuffer getBgraBuffer() {
-        return this.bgraBuffer;
-    }
-}

src/main/java/com/replaymod/render/gui/GuiExportFailed.java

@@ -98,6 +98,7 @@ public class GuiExportFailed extends GuiScreen {
                     oldSettings.getSphericalFovX(),
                     oldSettings.getSphericalFovY(),
                     oldSettings.isInjectSphericalMetadata(),
+                    oldSettings.isDepthMap(),
                     oldSettings.getAntiAliasing(),
                     oldSettings.getExportCommand(),
                     oldSettings.getEncodingPreset().getValue(),

src/main/java/com/replaymod/render/gui/GuiRenderSettings.java

@@ -196,6 +196,9 @@ public class GuiRenderSettings extends AbstractGuiPopup<GuiRenderSettings> {
     public final GuiCheckbox injectSphericalMetadata = new GuiCheckbox()
             .setI18nLabel("replaymod.gui.rendersettings.sphericalmetadata");
 
+    public final GuiCheckbox depthMap = new GuiCheckbox()
+            .setI18nLabel("replaymod.gui.rendersettings.depthmap");
+
     public final GuiDropdownMenu<RenderSettings.AntiAliasing> antiAliasingDropdown = new GuiDropdownMenu<RenderSettings.AntiAliasing>()
             .setSize(200, 20).setValues(RenderSettings.AntiAliasing.values()).setSelected(RenderSettings.AntiAliasing.NONE);
 
@@ -206,6 +209,7 @@ public class GuiRenderSettings extends AbstractGuiPopup<GuiRenderSettings> {
                             new GuiLabel().setI18nText("replaymod.gui.rendersettings.stabilizecamera"), stabilizePanel,
                             chromaKeyingCheckbox, chromaKeyingColor,
                             injectSphericalMetadata, sphericalFovSlider,
+                            depthMap, new GuiLabel(),
                             new GuiLabel().setI18nText("replaymod.gui.rendersettings.antialiasing"), antiAliasingDropdown));
 
     public final GuiTextField exportCommand = new GuiTextField().setI18nHint("replaymod.gui.rendersettings.command")
@@ -417,6 +421,13 @@ public class GuiRenderSettings extends AbstractGuiPopup<GuiRenderSettings> {
         exportArguments.setEnabled(isFFmpeg);
         antiAliasingDropdown.setEnabled(isFFmpeg);
 
+        if (isEXR) {
+            depthMap.setEnabled().setTooltip(null);
+        } else {
+            depthMap.setDisabled().setTooltip(new GuiTooltip().setColor(Colors.RED)
+                    .setI18nText("replaymod.gui.rendersettings.depthmap.onlyexr"));
+        }
+
         // Enable/Disable export args reset button
         boolean commandChanged = !exportCommand.getText().isEmpty();
         boolean argsChanged = !encodingPresetDropdown.getSelectedValue().getValue().equals(exportArguments.getText());
@@ -542,6 +553,7 @@ public class GuiRenderSettings extends AbstractGuiPopup<GuiRenderSettings> {
         }
         sphericalFovSlider.setValue((settings.getSphericalFovX() - MIN_SPHERICAL_FOV) / SPHERICAL_FOV_STEP_SIZE);
         injectSphericalMetadata.setChecked(settings.isInjectSphericalMetadata());
+        depthMap.setChecked(settings.isDepthMap());
         antiAliasingDropdown.setSelected(settings.getAntiAliasing());
         exportCommand.setText(settings.getExportCommand());
         String exportArguments = settings.getExportArguments();
@@ -572,6 +584,7 @@ public class GuiRenderSettings extends AbstractGuiPopup<GuiRenderSettings> {
                 chromaKeyingCheckbox.isChecked() ? chromaKeyingColor.getColor() : null,
                 sphericalFov, Math.min(180, sphericalFov),
                 injectSphericalMetadata.isChecked() && (serialize || injectSphericalMetadata.isEnabled()),
+                depthMap.isChecked() && (serialize || depthMap.isEnabled()),
                 serialize || antiAliasingDropdown.isEnabled() ? antiAliasingDropdown.getSelectedValue() : RenderSettings.AntiAliasing.NONE,
                 exportCommand.getText(),
                 exportArguments.getText(),
@@ -617,7 +630,7 @@ public class GuiRenderSettings extends AbstractGuiPopup<GuiRenderSettings> {
 
     private RenderSettings getDefaultRenderSettings() {
         return new RenderSettings(RenderSettings.RenderMethod.DEFAULT, RenderSettings.EncodingPreset.MP4_DEFAULT, 1920, 1080, 60, 10 << 20, null,
-                true, false, false, false, null, 360, 180, false, RenderSettings.AntiAliasing.NONE, "", RenderSettings.EncodingPreset.MP4_DEFAULT.getValue(), false);
+                true, false, false, false, null, 360, 180, false, false, RenderSettings.AntiAliasing.NONE, "", RenderSettings.EncodingPreset.MP4_DEFAULT.getValue(), false);
     }
 
     @Override

src/main/java/com/replaymod/render/mixin/Mixin_PreserveDepthDuringHandRendering.java

@@ -0,0 +1,24 @@
+package com.replaymod.render.mixin;
+
+import com.replaymod.render.hooks.EntityRendererHandler;
+import net.minecraft.client.render.GameRenderer;
+import org.lwjgl.opengl.GL11;
+import org.spongepowered.asm.mixin.Mixin;
+import org.spongepowered.asm.mixin.injection.At;
+import org.spongepowered.asm.mixin.injection.ModifyArg;
+
+@Mixin(GameRenderer.class)
+public abstract class Mixin_PreserveDepthDuringHandRendering {
+    @ModifyArg(
+            method = "renderWorld",
+            at = @At(value = "INVOKE", target = "Lcom/mojang/blaze3d/systems/RenderSystem;clear(IZ)V"),
+            index = 0
+    )
+    private int replayModRender_skipClearWhenRecordingDepth(int mask) {
+        EntityRendererHandler handler = ((EntityRendererHandler.IEntityRenderer) this).replayModRender_getHandler();
+        if (handler != null && handler.getSettings().isDepthMap()) {
+            mask = mask & ~GL11.GL_DEPTH_BUFFER_BIT;
+        }
+        return mask;
+    }
+}

src/main/java/com/replaymod/render/processor/GlToAbsoluteDepthProcessor.java

@@ -0,0 +1,33 @@
+package com.replaymod.render.processor;
+
+import com.replaymod.render.frame.BitmapFrame;
+
+import java.nio.FloatBuffer;
+
+public class GlToAbsoluteDepthProcessor extends AbstractFrameProcessor<BitmapFrame, BitmapFrame> {
+    // absolute depth is 2 * near * far / (far + near - (far - near) * (2 * z - 1))
+    // precomputed:     [      a       ]  [    b     ]  [    c     ]
+    private final float a;
+    private final float b;
+    private final float c;
+
+    public GlToAbsoluteDepthProcessor(float near, float far) {
+        a = 2 * near * far;
+        b = far + near;
+        c = far - near;
+    }
+
+    @Override
+    public BitmapFrame process(BitmapFrame frame) {
+
+        FloatBuffer buffer = frame.getByteBuffer().asFloatBuffer();
+        int len = buffer.remaining();
+        for (int i = 0; i < len; i++) {
+            float z = buffer.get(i);
+            z = a / (b - c * (2 * z - 1));
+            buffer.put(i, z);
+        }
+
+        return frame;
+    }
+}

src/main/java/com/replaymod/render/rendering/Channel.java

@@ -0,0 +1,6 @@
+package com.replaymod.render.rendering;
+
+public enum Channel {
+    BRGA,
+    DEPTH,
+}

src/main/java/com/replaymod/render/rendering/FrameCapturer.java

@@ -1,11 +1,12 @@
 package com.replaymod.render.rendering;
 
 import java.io.Closeable;
+import java.util.Map;
 
 public interface FrameCapturer<R extends Frame> extends Closeable {
 
     boolean isDone();
 
-    R process();
+    Map<Channel, R> process();
 
 }

src/main/java/com/replaymod/render/rendering/FrameConsumer.java

@@ -1,9 +1,10 @@
 package com.replaymod.render.rendering;
 
 import java.io.Closeable;
+import java.util.Map;
 
 public interface FrameConsumer<P extends Frame> extends Closeable {
 
-    void consume(P frame);
+    void consume(Map<Channel, P> channels);
 
 }

src/main/java/com/replaymod/render/rendering/Pipeline.java

@@ -3,6 +3,8 @@ package com.replaymod.render.rendering;
 import com.replaymod.core.mixin.MinecraftAccessor;
 import com.replaymod.core.versions.MCVer;
 import com.replaymod.render.capturer.WorldRenderer;
+import com.replaymod.render.frame.BitmapFrame;
+import com.replaymod.render.processor.GlToAbsoluteDepthProcessor;
 import net.minecraft.client.MinecraftClient;
 import net.minecraft.util.crash.CrashReport;
 import net.minecraft.util.crash.CrashException;
@@ -13,11 +15,14 @@ import org.lwjgl.glfw.GLFW;
 //$$ import org.lwjgl.opengl.Display;
 //#endif
 
+import java.util.HashMap;
+import java.util.Map;
 import java.util.concurrent.ArrayBlockingQueue;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 
+import static com.replaymod.core.versions.MCVer.getMinecraft;
 //#if MC>=11400
 import static com.replaymod.core.versions.MCVer.getWindow;
 //#endif
@@ -27,6 +32,7 @@ public class Pipeline<R extends Frame, P extends Frame> implements Runnable {
     private final WorldRenderer worldRenderer;
     private final FrameCapturer<R> capturer;
     private final FrameProcessor<R, P> processor;
+    private final GlToAbsoluteDepthProcessor depthProcessor;
     private int consumerNextFrame;
     private final Object consumerLock = new Object();
     private final FrameConsumer<P> consumer;
@@ -38,6 +44,10 @@ public class Pipeline<R extends Frame, P extends Frame> implements Runnable {
         this.capturer = capturer;
         this.processor = processor;
         this.consumer = consumer;
+
+        float near = 0.05f;
+        float far = getMinecraft().options.viewDistance * 16 * 4;
+        this.depthProcessor = new GlToAbsoluteDepthProcessor(near, far);
     }
 
     @Override
@@ -70,7 +80,7 @@ public class Pipeline<R extends Frame, P extends Frame> implements Runnable {
                 processService.shutdown();
                 return;
             }
-            R rawFrame = capturer.process();
+            Map<Channel, R> rawFrame = capturer.process();
             if (rawFrame != null) {
                 processService.submit(new ProcessTask(rawFrame));
             }
@@ -99,25 +109,37 @@ public class Pipeline<R extends Frame, P extends Frame> implements Runnable {
     }
 
     private class ProcessTask implements Runnable {
-        private final R rawFrame;
+        private final Map<Channel, R> rawChannels;
 
-        public ProcessTask(R rawFrame) {
+        public ProcessTask(Map<Channel, R> rawChannels) {
-            this.rawFrame = rawFrame;
+            this.rawChannels = rawChannels;
         }
 
         @Override
         public void run() {
             try {
-                P processedFrame = processor.process(rawFrame);
+                Integer frameId = null;
+                Map<Channel, P> processedChannels = new HashMap<>();
+                for (Map.Entry<Channel, R> entry : rawChannels.entrySet()) {
+                    P processedFrame = processor.process(entry.getValue());
+                    if (entry.getKey() == Channel.DEPTH && processedFrame instanceof BitmapFrame) {
+                        depthProcessor.process((BitmapFrame) processedFrame);
+                    }
+                    processedChannels.put(entry.getKey(), processedFrame);
+                    frameId = processedFrame.getFrameId();
+                }
+                if (frameId == null) {
+                    return;
+                }
                 synchronized (consumerLock) {
-                    while (consumerNextFrame != processedFrame.getFrameId()) {
+                    while (consumerNextFrame != frameId) {
                         try {
                             consumerLock.wait();
                         } catch (InterruptedException e) {
                             Thread.currentThread().interrupt();
                         }
                     }
-                    consumer.consume(processedFrame);
+                    consumer.consume(processedChannels);
                     consumerNextFrame++;
                     consumerLock.notifyAll();
                 }

src/main/java/com/replaymod/render/rendering/Pipelines.java

@@ -12,7 +12,6 @@ import com.replaymod.render.capturer.StereoscopicOpenGlFrameCapturer;
 import com.replaymod.render.capturer.StereoscopicPboOpenGlFrameCapturer;
 import com.replaymod.render.capturer.WorldRenderer;
 import com.replaymod.render.frame.CubicOpenGlFrame;
-import com.replaymod.render.frame.EXRFrame;
 import com.replaymod.render.frame.ODSOpenGlFrame;
 import com.replaymod.render.frame.OpenGlFrame;
 import com.replaymod.render.frame.BitmapFrame;
@@ -26,23 +25,9 @@ import com.replaymod.render.processor.OpenGlToBitmapProcessor;
 import com.replaymod.render.processor.StereoscopicToBitmapProcessor;
 import com.replaymod.render.utils.PixelBufferObject;
 
-import java.io.IOException;
+import java.util.Map;
 
 public class Pipelines {
-    public static Pipeline newEXRPipeline(RenderSettings.RenderMethod method, RenderInfo renderInfo, FrameConsumer<EXRFrame> consumer) {
-        return newPipeline(method, renderInfo, new FrameConsumer<BitmapFrame>() {
-            @Override
-            public void consume(BitmapFrame frame) {
-                consumer.consume(new EXRFrame(frame.getFrameId(), frame.getSize(), frame.getByteBuffer()));
-            }
-
-            @Override
-            public void close() throws IOException {
-                consumer.close();
-            }
-        });
-    }
-
     public static Pipeline newPipeline(RenderSettings.RenderMethod method, RenderInfo renderInfo, FrameConsumer<BitmapFrame> consumer) {
         switch (method) {
             case DEFAULT:
@@ -65,7 +50,7 @@ public class Pipelines {
         RenderSettings settings = renderInfo.getRenderSettings();
         WorldRenderer worldRenderer = new EntityRendererHandler(settings, renderInfo);
         FrameCapturer<OpenGlFrame> capturer;
-        if (PixelBufferObject.SUPPORTED) {
+        if (PixelBufferObject.SUPPORTED || settings.isDepthMap()) {
             capturer = new SimplePboOpenGlFrameCapturer(worldRenderer, renderInfo);
         } else {
             capturer = new SimpleOpenGlFrameCapturer(worldRenderer, renderInfo);
@@ -77,7 +62,7 @@ public class Pipelines {
         RenderSettings settings = renderInfo.getRenderSettings();
         WorldRenderer worldRenderer = new EntityRendererHandler(settings, renderInfo);
         FrameCapturer<StereoscopicOpenGlFrame> capturer;
-        if (PixelBufferObject.SUPPORTED) {
+        if (PixelBufferObject.SUPPORTED || settings.isDepthMap()) {
             capturer = new StereoscopicPboOpenGlFrameCapturer(worldRenderer, renderInfo);
         } else {
             capturer = new StereoscopicOpenGlFrameCapturer(worldRenderer, renderInfo);
@@ -89,7 +74,7 @@ public class Pipelines {
         RenderSettings settings = renderInfo.getRenderSettings();
         WorldRenderer worldRenderer = new EntityRendererHandler(settings, renderInfo);
         FrameCapturer<CubicOpenGlFrame> capturer;
-        if (PixelBufferObject.SUPPORTED) {
+        if (PixelBufferObject.SUPPORTED || settings.isDepthMap()) {
             capturer = new CubicPboOpenGlFrameCapturer(worldRenderer, renderInfo, settings.getVideoWidth() / 4);
         } else {
             capturer = new CubicOpenGlFrameCapturer(worldRenderer, renderInfo, settings.getVideoWidth() / 4);
@@ -105,7 +90,7 @@ public class Pipelines {
                 settings.getVideoHeight(), settings.getSphericalFovX());
 
         FrameCapturer<CubicOpenGlFrame> capturer;
-        if (PixelBufferObject.SUPPORTED) {
+        if (PixelBufferObject.SUPPORTED || settings.isDepthMap()) {
             capturer = new CubicPboOpenGlFrameCapturer(worldRenderer, renderInfo, processor.getFrameSize());
         } else {
             capturer = new CubicOpenGlFrameCapturer(worldRenderer, renderInfo, processor.getFrameSize());
@@ -131,7 +116,7 @@ public class Pipelines {
         FrameCapturer<BitmapFrame> capturer = new BlendFrameCapturer(worldRenderer, renderInfo);
         FrameConsumer<BitmapFrame> consumer = new FrameConsumer<BitmapFrame>() {
             @Override
-            public void consume(BitmapFrame frame) {
+            public void consume(Map<Channel, BitmapFrame> channels) {
             }
 
             @Override

src/main/java/com/replaymod/render/rendering/VideoRenderer.java

@@ -44,7 +44,6 @@ import org.lwjgl.opengl.GL11;
 
 //#if MC>=11400
 import com.replaymod.render.EXRWriter;
-import com.replaymod.render.frame.EXRFrame;
 import com.replaymod.render.mixin.MainWindowAccessor;
 import net.minecraft.client.gui.screen.Screen;
 import org.lwjgl.glfw.GLFW;
@@ -122,26 +121,34 @@ public class VideoRenderer implements RenderInfo {
 
             this.renderingPipeline = Pipelines.newBlendPipeline(this);
             this.ffmpegWriter = null;
-        //#if MC>=11400
-        } else if (settings.getEncodingPreset() == RenderSettings.EncodingPreset.EXR) {
-            this.renderingPipeline = Pipelines.newEXRPipeline(settings.getRenderMethod(), this, new EXRWriter(settings.getOutputFile().toPath()) {
-                @Override
-                public void consume(EXRFrame frame) {
-                    gui.updatePreview(frame.getBgraBuffer(), frame.getSize());
-                    super.consume(frame);
-                }
-            });
-            this.ffmpegWriter = null;
-        //#endif
         } else {
-            this.renderingPipeline = Pipelines.newPipeline(settings.getRenderMethod(), this,
+            FrameConsumer<BitmapFrame> frameConsumer;
-                    ffmpegWriter = new FFmpegWriter(this) {
+            if (settings.getEncodingPreset() == RenderSettings.EncodingPreset.EXR) {
-                        @Override
+                ffmpegWriter = null;
-                        public void consume(BitmapFrame frame) {
+                //#if MC>=11400
-                            gui.updatePreview(frame.getByteBuffer(), frame.getSize());
+                frameConsumer = new EXRWriter(settings.getOutputFile().toPath());
-                            super.consume(frame);
+                //#else
-                        }
+                //$$ throw new UnsupportedOperationException("EXR requires LWJGL3");
-                    });
+                //#endif
+            } else {
+                frameConsumer = ffmpegWriter = new FFmpegWriter(this);
+            }
+            FrameConsumer<BitmapFrame> previewingFrameConsumer = new FrameConsumer<BitmapFrame>() {
+                @Override
+                public void consume(Map<Channel, BitmapFrame> channels) {
+                    BitmapFrame bgra = channels.get(Channel.BRGA);
+                    if (bgra != null) {
+                        gui.updatePreview(bgra.getByteBuffer(), bgra.getSize());
+                    }
+                    frameConsumer.consume(channels);
+                }
+
+                @Override
+                public void close() throws IOException {
+                    frameConsumer.close();
+                }
+            };
+            this.renderingPipeline = Pipelines.newPipeline(settings.getRenderMethod(), this, previewingFrameConsumer);
         }
     }
 

src/main/resources/mixins.render.replaymod.json

@@ -18,6 +18,7 @@
     "Mixin_ForceChunkLoading",
     //#endif
     //#if MC>=11400
+    "Mixin_PreserveDepthDuringHandRendering",
     "MainWindowAccessor",
     //#endif
     "WorldRendererAccessor",