From 2b6d1b235944f8fbd4e7edc90f2b8b1393f92879 Mon Sep 17 00:00:00 2001 From: Jonas Herzig Date: Sat, 3 Oct 2020 19:38:13 +0200 Subject: [PATCH] Add OpenEXR export option In preparation for depth map export. --- docs/content.md | 6 + .../java/com/replaymod/render/EXRWriter.java | 108 ++++++++++++++++++ .../com/replaymod/render/RenderSettings.java | 9 ++ .../com/replaymod/render/frame/EXRFrame.java | 34 ++++++ .../render/gui/GuiRenderSettings.java | 13 ++- .../render/gui/GuiVideoRenderer.java | 4 +- .../replaymod/render/rendering/Pipelines.java | 17 +++ .../render/rendering/VideoRenderer.java | 15 ++- src/main/resources/assets/replaymod/lang | 2 +- versions/common.gradle | 12 +- 10 files changed, 209 insertions(+), 11 deletions(-) create mode 100644 src/main/java/com/replaymod/render/EXRWriter.java create mode 100644 src/main/java/com/replaymod/render/frame/EXRFrame.java diff --git a/docs/content.md b/docs/content.md index 14e7ef52..ea225be6 100755 --- a/docs/content.md +++ b/docs/content.md @@ -357,6 +357,12 @@ There are **7 Encoding Presets** you can choose from: - **PNG Sequence** Exports the sequence as individual frames in the **PNG Format**. **Warning:** This can create a huge amount of files, so make sure to save them in a separate folder. +- **OpenEXR Sequence** (Minecraft 1.14 and above) + Exports the sequence as individual frames in the **OpenEXR Format**. + The images are of perfect quality and may contain + additional image layers beyond the visible one (e.g. [depth](#replaying-render-advanced-depth-map)). + As such, this format is a good choice if you have plenty of disk space to spare and want to perform additional editing + on the video in a third-party video editor. ### Advanced Settings [advanced] ![](img/rendersettings-advanced.jpg) diff --git a/src/main/java/com/replaymod/render/EXRWriter.java b/src/main/java/com/replaymod/render/EXRWriter.java new file mode 100644 index 00000000..2a738ef6 --- /dev/null +++ b/src/main/java/com/replaymod/render/EXRWriter.java @@ -0,0 +1,108 @@ +//#if MC>=11400 +package com.replaymod.render; + +import com.replaymod.core.versions.MCVer; +import com.replaymod.render.frame.EXRFrame; +import com.replaymod.render.rendering.FrameConsumer; +import com.replaymod.render.utils.ByteBufferPool; +import de.johni0702.minecraft.gui.utils.lwjgl.ReadableDimension; +import net.minecraft.util.crash.CrashReport; +import org.lwjgl.PointerBuffer; +import org.lwjgl.util.tinyexr.EXRChannelInfo; +import org.lwjgl.util.tinyexr.EXRHeader; +import org.lwjgl.util.tinyexr.EXRImage; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.FloatBuffer; +import java.nio.IntBuffer; +import java.nio.file.Files; +import java.nio.file.Path; + +import static org.lwjgl.system.MemoryStack.*; +import static org.lwjgl.system.MemoryUtil.*; +import static org.lwjgl.util.tinyexr.TinyEXR.*; + +public class EXRWriter implements FrameConsumer { + + private final Path outputFolder; + + public EXRWriter(Path outputFolder) throws IOException { + this.outputFolder = outputFolder; + + Files.createDirectories(outputFolder); + } + + @Override + public void consume(EXRFrame frame) { + Path path = outputFolder.resolve(frame.getFrameId() + ".exr"); + ReadableDimension size = frame.getSize(); + ByteBuffer bgra = frame.getBgraBuffer(); + int width = size.getWidth(); + int height = size.getHeight(); + int numChannels = 3; + + stackPush(); + EXRHeader header = EXRHeader.mallocStack(); InitEXRHeader(header); + EXRChannelInfo.Buffer channelInfos = EXRChannelInfo.mallocStack(numChannels); + IntBuffer pixelTypes = stackMallocInt(numChannels); + IntBuffer requestedPixelTypes = stackMallocInt(numChannels); + EXRImage image = EXRImage.mallocStack(); InitEXRImage(image); + PointerBuffer imagePointers = stackMallocPointer(numChannels); + FloatBuffer images = memAllocFloat(width * height * numChannels); + PointerBuffer err = stackMallocPointer(1); + try { + header.num_channels(numChannels); + header.channels(channelInfos); + header.pixel_types(pixelTypes); + header.requested_pixel_types(requestedPixelTypes); + + // Some readers ignore this, so we use the most expected order + memASCII("B", true, channelInfos.get(0).name()); + memASCII("G", true, channelInfos.get(1).name()); + memASCII("R", true, channelInfos.get(2).name()); + for (int i = 0; i < numChannels; i++) { + pixelTypes.put(i, TINYEXR_PIXELTYPE_FLOAT); + requestedPixelTypes.put(i, TINYEXR_PIXELTYPE_HALF); + } + + image.num_channels(numChannels); + image.width(width); + image.height(height); + image.images(imagePointers); + FloatBuffer[] bgrChannels = new FloatBuffer[3]; + for (int i = 0; i < numChannels; i++) { + FloatBuffer channel = images.slice(); + channel.position(width * height * i); + imagePointers.put(i, channel.slice()); + bgrChannels[i] = channel; + } + for (int y = 0; y < height; y++) { + for (int x = 0; x < width; x++) { + for (FloatBuffer channel : bgrChannels) { + channel.put(((int) bgra.get() & 0xff) / 255f); + } + bgra.get(); // alpha + } + } + + int ret = SaveEXRImageToFile(image, header, path.toString(), err); + if (ret != TINYEXR_SUCCESS) { + String message = err.getStringASCII(0); + FreeEXRErrorMessage(err.getByteBuffer(0)); + throw new IOException(message); + } + } catch (Throwable t) { + MCVer.getMinecraft().setCrashReport(CrashReport.create(t, "Exporting EXR frame")); + } finally { + memFree(images); + stackPop(); + ByteBufferPool.release(bgra); + } + } + + @Override + public void close() { + } +} +//#endif diff --git a/src/main/java/com/replaymod/render/RenderSettings.java b/src/main/java/com/replaymod/render/RenderSettings.java index 3b057e91..1378cfb2 100644 --- a/src/main/java/com/replaymod/render/RenderSettings.java +++ b/src/main/java/com/replaymod/render/RenderSettings.java @@ -79,6 +79,8 @@ public class RenderSettings { BLEND(null, "blend"), + EXR(null, "exr"), + PNG("\"%FILENAME%-%06d.png\"", "png"); private final String preset; @@ -111,6 +113,13 @@ public class RenderSettings { public boolean isSupported() { if (this == BLEND) { return RenderMethod.BLEND.isSupported(); + } else if (this == EXR) { + // Need LJWGL 3 + //#if MC>=11400 + return true; + //#else + //$$ return false; + //#endif } else { return true; } diff --git a/src/main/java/com/replaymod/render/frame/EXRFrame.java b/src/main/java/com/replaymod/render/frame/EXRFrame.java new file mode 100644 index 00000000..a71be1d8 --- /dev/null +++ b/src/main/java/com/replaymod/render/frame/EXRFrame.java @@ -0,0 +1,34 @@ +package com.replaymod.render.frame; + +import com.replaymod.render.rendering.Frame; +import de.johni0702.minecraft.gui.utils.lwjgl.ReadableDimension; +import org.apache.commons.lang3.Validate; + +import java.nio.ByteBuffer; + +public class EXRFrame implements Frame { + private final int frameId; + private final ReadableDimension size; + private final ByteBuffer bgraBuffer; + + public EXRFrame(int frameId, ReadableDimension size, ByteBuffer bgraBuffer) { + Validate.isTrue(size.getWidth() * size.getHeight() * 4 == bgraBuffer.remaining(), + "BGRA buffer size is %d (cap: %d) but should be %d", + bgraBuffer.remaining(), bgraBuffer.capacity(), size.getWidth() * size.getHeight() * 4); + this.frameId = frameId; + this.size = size; + this.bgraBuffer = bgraBuffer; + } + + public int getFrameId() { + return this.frameId; + } + + public ReadableDimension getSize() { + return this.size; + } + + public ByteBuffer getBgraBuffer() { + return this.bgraBuffer; + } +} diff --git a/src/main/java/com/replaymod/render/gui/GuiRenderSettings.java b/src/main/java/com/replaymod/render/gui/GuiRenderSettings.java index 3e12ff6a..bc624e9b 100644 --- a/src/main/java/com/replaymod/render/gui/GuiRenderSettings.java +++ b/src/main/java/com/replaymod/render/gui/GuiRenderSettings.java @@ -405,15 +405,17 @@ public class GuiRenderSettings extends AbstractGuiPopup { } // Enable/Disable various options for blend export + boolean isEXR = encodingPresetDropdown.getSelectedValue() == RenderSettings.EncodingPreset.EXR; boolean isBlend = renderMethod == RenderSettings.RenderMethod.BLEND; + boolean isFFmpeg = !isBlend && !isEXR; if (isBlend) { videoWidth.setDisabled(); videoHeight.setDisabled(); } encodingPresetDropdown.setEnabled(!isBlend); - exportCommand.setEnabled(!isBlend); - exportArguments.setEnabled(!isBlend); - antiAliasingDropdown.setEnabled(!isBlend); + exportCommand.setEnabled(isFFmpeg); + exportArguments.setEnabled(isFFmpeg); + antiAliasingDropdown.setEnabled(isFFmpeg); // Enable/Disable export args reset button boolean commandChanged = !exportCommand.getText().isEmpty(); @@ -519,7 +521,8 @@ public class GuiRenderSettings extends AbstractGuiPopup { userDefinedOutputFileName = false; } else if (savedOutputFile.exists()) { String name = generateOutputFile(encodingPreset).getName(); - this.outputFile = new File(savedOutputFile.isDirectory() ? savedOutputFile : savedOutputFile.getParentFile(), name); + boolean isFolder = savedOutputFile.isDirectory() && !savedOutputFile.getName().endsWith(".exr"); + this.outputFile = new File(isFolder ? savedOutputFile : savedOutputFile.getParentFile(), name); userDefinedOutputFileName = false; } else { this.outputFile = conformExtension(savedOutputFile, encodingPreset); @@ -569,7 +572,7 @@ public class GuiRenderSettings extends AbstractGuiPopup { chromaKeyingCheckbox.isChecked() ? chromaKeyingColor.getColor() : null, sphericalFov, Math.min(180, sphericalFov), injectSphericalMetadata.isChecked() && (serialize || injectSphericalMetadata.isEnabled()), - antiAliasingDropdown.getSelectedValue(), + serialize || antiAliasingDropdown.isEnabled() ? antiAliasingDropdown.getSelectedValue() : RenderSettings.AntiAliasing.NONE, exportCommand.getText(), exportArguments.getText(), net.minecraft.client.gui.screen.Screen.hasControlDown() diff --git a/src/main/java/com/replaymod/render/gui/GuiVideoRenderer.java b/src/main/java/com/replaymod/render/gui/GuiVideoRenderer.java index c5ade8f5..bc69f061 100644 --- a/src/main/java/com/replaymod/render/gui/GuiVideoRenderer.java +++ b/src/main/java/com/replaymod/render/gui/GuiVideoRenderer.java @@ -265,9 +265,8 @@ public class GuiVideoRenderer extends GuiScreen implements Tickable { guiRenderer.drawTexturedRect(x, y, 0, 0, width, height, videoWidth, videoHeight, videoWidth, videoHeight); } - public void updatePreview(BitmapFrame frame) { + public void updatePreview(ByteBuffer buffer, ReadableDimension size) { if (previewCheckbox.isChecked() && previewTexture != null) { - ByteBuffer buffer = frame.getByteBuffer(); buffer.mark(); synchronized (this) { //#if MC>=11400 @@ -279,7 +278,6 @@ public class GuiVideoRenderer extends GuiScreen implements Tickable { // Note: Optifine changes the texture data array to be three times as long (for use by shaders), // we only want to initialize the first third and since we use our frame size, not the array size, // we're good to go. - ReadableDimension size = frame.getSize(); int width = size.getWidth(); for (int y = 0; y < size.getHeight(); y++) { for (int x = 0; x < width; x++) { diff --git a/src/main/java/com/replaymod/render/rendering/Pipelines.java b/src/main/java/com/replaymod/render/rendering/Pipelines.java index 5cd0de6f..6507b1ee 100644 --- a/src/main/java/com/replaymod/render/rendering/Pipelines.java +++ b/src/main/java/com/replaymod/render/rendering/Pipelines.java @@ -12,6 +12,7 @@ import com.replaymod.render.capturer.StereoscopicOpenGlFrameCapturer; import com.replaymod.render.capturer.StereoscopicPboOpenGlFrameCapturer; import com.replaymod.render.capturer.WorldRenderer; import com.replaymod.render.frame.CubicOpenGlFrame; +import com.replaymod.render.frame.EXRFrame; import com.replaymod.render.frame.ODSOpenGlFrame; import com.replaymod.render.frame.OpenGlFrame; import com.replaymod.render.frame.BitmapFrame; @@ -25,7 +26,23 @@ import com.replaymod.render.processor.OpenGlToBitmapProcessor; import com.replaymod.render.processor.StereoscopicToBitmapProcessor; import com.replaymod.render.utils.PixelBufferObject; +import java.io.IOException; + public class Pipelines { + public static Pipeline newEXRPipeline(RenderSettings.RenderMethod method, RenderInfo renderInfo, FrameConsumer consumer) { + return newPipeline(method, renderInfo, new FrameConsumer() { + @Override + public void consume(BitmapFrame frame) { + consumer.consume(new EXRFrame(frame.getFrameId(), frame.getSize(), frame.getByteBuffer())); + } + + @Override + public void close() throws IOException { + consumer.close(); + } + }); + } + public static Pipeline newPipeline(RenderSettings.RenderMethod method, RenderInfo renderInfo, FrameConsumer consumer) { switch (method) { case DEFAULT: diff --git a/src/main/java/com/replaymod/render/rendering/VideoRenderer.java b/src/main/java/com/replaymod/render/rendering/VideoRenderer.java index befaf946..8dd5da9f 100644 --- a/src/main/java/com/replaymod/render/rendering/VideoRenderer.java +++ b/src/main/java/com/replaymod/render/rendering/VideoRenderer.java @@ -43,6 +43,8 @@ import org.lwjgl.opengl.GL11; //#endif //#if MC>=11400 +import com.replaymod.render.EXRWriter; +import com.replaymod.render.frame.EXRFrame; import com.replaymod.render.mixin.MainWindowAccessor; import net.minecraft.client.gui.screen.Screen; import org.lwjgl.glfw.GLFW; @@ -120,12 +122,23 @@ public class VideoRenderer implements RenderInfo { this.renderingPipeline = Pipelines.newBlendPipeline(this); this.ffmpegWriter = null; + //#if MC>=11400 + } else if (settings.getEncodingPreset() == RenderSettings.EncodingPreset.EXR) { + this.renderingPipeline = Pipelines.newEXRPipeline(settings.getRenderMethod(), this, new EXRWriter(settings.getOutputFile().toPath()) { + @Override + public void consume(EXRFrame frame) { + gui.updatePreview(frame.getBgraBuffer(), frame.getSize()); + super.consume(frame); + } + }); + this.ffmpegWriter = null; + //#endif } else { this.renderingPipeline = Pipelines.newPipeline(settings.getRenderMethod(), this, ffmpegWriter = new FFmpegWriter(this) { @Override public void consume(BitmapFrame frame) { - gui.updatePreview(frame); + gui.updatePreview(frame.getByteBuffer(), frame.getSize()); super.consume(frame); } }); diff --git a/src/main/resources/assets/replaymod/lang b/src/main/resources/assets/replaymod/lang index 8c08d804..0cb71d34 160000 --- a/src/main/resources/assets/replaymod/lang +++ b/src/main/resources/assets/replaymod/lang @@ -1 +1 @@ -Subproject commit 8c08d8049ed164e763dacbcd4e1a3db59d45b30e +Subproject commit 0cb71d34236310e3c309833e205a1674c28f2208 diff --git a/versions/common.gradle b/versions/common.gradle index 8f1bb427..406036f5 100644 --- a/versions/common.gradle +++ b/versions/common.gradle @@ -286,7 +286,15 @@ dependencies { shadow 'com.google.api-client:google-api-client-gson:1.20.0', shadeExclusions shadow 'com.google.api-client:google-api-client-java6:1.20.0', shadeExclusions shadow 'com.google.oauth-client:google-oauth-client-jetty:1.20.0' - + + if (mcVersion >= 11400) { // need lwjgl 3 + for (suffix in ['', ':natives-linux', ':natives-windows', ':natives-macos']) { + shadow('org.lwjgl:lwjgl-tinyexr:3.2.2' + suffix) { + exclude group: 'org.lwjgl', module: 'lwjgl' // comes with MC + } + } + } + if (FABRIC) { shadow 'org.apache.maven:maven-artifact:3.6.1' } @@ -381,6 +389,8 @@ task configureRelocation() { } else if (!pkg.startsWith('org/apache/logging')) { pkgs << pkg.substring(0, pkg.indexOf('/', 'org/apache/'.length())) } + } else if (pkg.startsWith('org/lwjgl')) { + return // either bundled with MC or uses natives which we can't relocate } else if (!pkg.startsWith('org/spongepowered')) { pkgs << pkg.substring(0, pkg.indexOf('/', 4)) }