From d3230e1e38105c080805e744215cf4f7fa05cbdb Mon Sep 17 00:00:00 2001 From: Jonas Herzig Date: Thu, 5 Nov 2020 14:23:59 +0100 Subject: [PATCH] Add camera path export (closes #391) --- .../java/com/replaymod/core/utils/Utils.java | 5 + .../replaymod/render/CameraPathExporter.java | 227 ++++++++++++++++++ .../com/replaymod/render/RenderSettings.java | 8 + .../replaymod/render/gui/GuiExportFailed.java | 1 + .../render/gui/GuiRenderSettings.java | 8 +- .../render/rendering/VideoRenderer.java | 24 ++ src/main/resources/assets/replaymod/lang | 2 +- versions/common.gradle | 2 + 8 files changed, 275 insertions(+), 2 deletions(-) create mode 100644 src/main/java/com/replaymod/render/CameraPathExporter.java diff --git a/src/main/java/com/replaymod/core/utils/Utils.java b/src/main/java/com/replaymod/core/utils/Utils.java index f15ec91f..a318c793 100644 --- a/src/main/java/com/replaymod/core/utils/Utils.java +++ b/src/main/java/com/replaymod/core/utils/Utils.java @@ -367,4 +367,9 @@ public class Utils { open(); } } + + public static T configure(T instance, Consumer configure) { + configure.accept(instance); + return instance; + } } diff --git a/src/main/java/com/replaymod/render/CameraPathExporter.java b/src/main/java/com/replaymod/render/CameraPathExporter.java new file mode 100644 index 00000000..abb9e994 --- /dev/null +++ b/src/main/java/com/replaymod/render/CameraPathExporter.java @@ -0,0 +1,227 @@ +package com.replaymod.render; + +import com.replaymod.core.ReplayMod; +import com.replaymod.core.versions.MCVer; +import com.replaymod.replay.camera.CameraEntity; +import de.javagl.jgltf.impl.v2.Accessor; +import de.javagl.jgltf.impl.v2.Animation; +import de.javagl.jgltf.impl.v2.AnimationChannel; +import de.javagl.jgltf.impl.v2.AnimationChannelTarget; +import de.javagl.jgltf.impl.v2.AnimationSampler; +import de.javagl.jgltf.impl.v2.Asset; +import de.javagl.jgltf.impl.v2.Buffer; +import de.javagl.jgltf.impl.v2.BufferView; +import de.javagl.jgltf.impl.v2.Camera; +import de.javagl.jgltf.impl.v2.CameraPerspective; +import de.javagl.jgltf.impl.v2.GlTF; +import de.javagl.jgltf.impl.v2.Node; +import de.javagl.jgltf.model.io.v2.GltfAssetV2; +import de.javagl.jgltf.model.io.v2.GltfAssetWriterV2; +import de.johni0702.minecraft.gui.utils.lwjgl.vector.Quaternion; +import de.johni0702.minecraft.gui.utils.lwjgl.vector.Vector4f; +import net.minecraft.client.MinecraftClient; +import net.minecraft.entity.Entity; +import org.apache.commons.io.FilenameUtils; +import org.lwjgl.opengl.GL11; + +//#if MC>=11400 +import net.minecraft.util.math.Vec3d; +//#else +//#endif + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.file.Files; + +import static com.replaymod.core.utils.Utils.configure; + +public class CameraPathExporter { + + private final MinecraftClient mc = MCVer.getMinecraft(); + private final RenderSettings settings; + private int framesDone; + private ByteBuffer timeBuffer; + private ByteBuffer cameraTranslationBuffer; + private ByteBuffer cameraRotationBuffer; + + public CameraPathExporter(RenderSettings settings) { + this.settings = settings; + } + + public void setup(int totalFrames) { + timeBuffer = ByteBuffer.allocate(4 * totalFrames).order(ByteOrder.LITTLE_ENDIAN); + cameraTranslationBuffer = ByteBuffer.allocate(4 * totalFrames * 3).order(ByteOrder.LITTLE_ENDIAN); + cameraRotationBuffer = ByteBuffer.allocate(4 * totalFrames * 4).order(ByteOrder.LITTLE_ENDIAN); + } + + public void recordFrame(float tickDelta) { + //#if MC>=10800 + Entity entity = mc.getCameraEntity() == null ? mc.player : mc.getCameraEntity(); + //#else + //$$ Entity entity = mc.renderViewEntity == null ? mc.thePlayer : mc.renderViewEntity; + //#endif + + //#if MC>=11400 + net.minecraft.client.render.Camera camera = mc.gameRenderer.getCamera(); + Vec3d vec = camera.getPos(); + float x = (float) vec.getX(); + float y = (float) vec.getY(); + float z = (float) vec.getZ(); + float yaw = camera.getYaw() + 180; + float pitch = camera.getPitch(); + //#else + //#if MC>=10800 + //$$ float eyeHeight = entity.getEyeHeight(); + //#else + //$$ float eyeHeight = 1.62f - entity.yOffset; + //#endif + //$$ float x = (float) (entity.prevPosX + (entity.posX - entity.prevPosX) * tickDelta); + //$$ float y = (float) (entity.prevPosY + (entity.posY - entity.prevPosY) * tickDelta + eyeHeight); + //$$ float z = (float) (entity.prevPosZ + (entity.posZ - entity.prevPosZ) * tickDelta); + //$$ float yaw = entity.prevRotationYaw + (entity.rotationYaw - entity.prevRotationYaw) * tickDelta + 180; + //$$ float pitch = entity.prevRotationPitch + (entity.rotationPitch - entity.prevRotationPitch) * tickDelta; + //#endif + float roll = entity instanceof CameraEntity ? ((CameraEntity) entity).roll : 0; + + Quaternion quatYaw = new Quaternion(); + Quaternion quatPitch = new Quaternion(); + Quaternion quatRoll = new Quaternion(); + + quatYaw.setFromAxisAngle(new Vector4f(0, -1, 0, (float) Math.toRadians(yaw))); + quatPitch.setFromAxisAngle(new Vector4f(-1, 0, 0, (float) Math.toRadians(pitch))); + quatRoll.setFromAxisAngle(new Vector4f(0, 0, 1, (float) Math.toRadians(roll))); + + Quaternion quaternion = new Quaternion(0, 0, 0, 1); + Quaternion.mul(quaternion, quatYaw, quaternion); + Quaternion.mul(quaternion, quatPitch, quaternion); + Quaternion.mul(quaternion, quatRoll, quaternion); + quaternion.normalise(quaternion); + + float[] translation = new float[] { x, y, z }; + float[] rotation = new float[] { quaternion.getX(), quaternion.getY(), quaternion.getZ(), quaternion.getW() }; + + timeBuffer.putFloat(framesDone / (float) settings.getFramesPerSecond()); + for (float f : translation) { + cameraTranslationBuffer.putFloat(f); + } + for (float f : rotation) { + cameraRotationBuffer.putFloat(f); + } + + framesDone++; + } + + public void finish() throws IOException { + int timeBufferSize = timeBuffer.rewind().remaining(); + int cameraTranslationBufferSize = cameraTranslationBuffer.rewind().remaining(); + int cameraRotationBufferSize = cameraRotationBuffer.rewind().remaining(); + + int binaryDataSize = 0; + binaryDataSize += timeBufferSize; + binaryDataSize += cameraTranslationBufferSize; + binaryDataSize += cameraRotationBufferSize; + + ByteBuffer binaryData = ByteBuffer.allocate(binaryDataSize); + int timeBufferOffset = binaryData.position(); + binaryData.put(timeBuffer); + int cameraTranslationBufferOffset = binaryData.position(); + binaryData.put(cameraTranslationBuffer); + int cameraRotationBufferOffset = binaryData.position(); + binaryData.put(cameraRotationBuffer); + binaryData.rewind(); + + GlTF glTF = new GlTF(); + glTF.setAsset(configure(new Asset(), asset -> { + asset.setVersion("2.0"); + asset.setGenerator("ReplayMod v" + ReplayMod.instance.getVersion()); + })); + glTF.addAnimations(configure(new Animation(), animation -> { + animation.addChannels(configure(new AnimationChannel(), channel -> { + channel.setTarget(configure(new AnimationChannelTarget(), target -> { + target.setNode(0); + target.setPath("translation"); + })); + channel.setSampler(0); + })); + animation.addChannels(configure(new AnimationChannel(), channel -> { + channel.setTarget(configure(new AnimationChannelTarget(), target -> { + target.setNode(0); + target.setPath("rotation"); + })); + channel.setSampler(1); + })); + animation.addSamplers(configure(new AnimationSampler(), sampler -> { + sampler.setInput(0); + sampler.setOutput(1); + })); + animation.addSamplers(configure(new AnimationSampler(), sampler -> { + sampler.setInput(0); + sampler.setOutput(2); + })); + })); + glTF.addCameras(configure(new Camera(), camera -> { + camera.setType("perspective"); + camera.setPerspective(configure(new CameraPerspective(), perspective -> { + float aspectRatio = (float) settings.getVideoWidth() / (float) settings.getVideoHeight(); + perspective.setAspectRatio(aspectRatio); + perspective.setYfov((float) Math.toRadians(mc.options.fov)); + perspective.setZnear(0.05f); + perspective.setZfar((float) mc.options.viewDistance * 16 * 4); + })); + })); + glTF.addNodes(configure(new Node(), node -> node.setCamera(0))); + glTF.addBuffers(configure(new Buffer(), buffer -> buffer.setByteLength(binaryData.limit()))); + // Time + glTF.addBufferViews(configure(new BufferView(), bufferView -> { + bufferView.setBuffer(0); + bufferView.setByteOffset(timeBufferOffset); + bufferView.setByteLength(timeBufferSize); + })); + glTF.addAccessors(configure(new Accessor(), accessor -> { + accessor.setBufferView(0); + accessor.setType("SCALAR"); + accessor.setComponentType(GL11.GL_FLOAT); + accessor.setCount(framesDone); + })); + // Camera translation + glTF.addBufferViews(configure(new BufferView(), bufferView -> { + bufferView.setBuffer(0); + bufferView.setByteOffset(cameraTranslationBufferOffset); + bufferView.setByteLength(cameraTranslationBufferSize); + })); + glTF.addAccessors(configure(new Accessor(), accessor -> { + accessor.setBufferView(1); + accessor.setType("VEC3"); + accessor.setComponentType(GL11.GL_FLOAT); + accessor.setCount(framesDone); + })); + // Camera rotation + glTF.addBufferViews(configure(new BufferView(), bufferView -> { + bufferView.setBuffer(0); + bufferView.setByteOffset(cameraRotationBufferOffset); + bufferView.setByteLength(cameraRotationBufferSize); + })); + glTF.addAccessors(configure(new Accessor(), accessor -> { + accessor.setBufferView(2); + accessor.setType("VEC4"); + accessor.setComponentType(GL11.GL_FLOAT); + accessor.setCount(framesDone); + })); + + java.nio.file.Path videoPath = settings.getOutputFile().toPath(); + java.nio.file.Path glbBasePath = Files.isDirectory(videoPath) + ? videoPath.resolve("camera.glb") + : videoPath.resolveSibling(videoPath.getFileName() + ".glb"); + java.nio.file.Path glbPath = glbBasePath; + for (int i = 0; Files.exists(glbPath); i++) { + String baseName = FilenameUtils.getBaseName(glbBasePath.getFileName().toString()); + glbPath = glbBasePath.resolveSibling(baseName + "." + i + ".glb"); + } + try (OutputStream out = Files.newOutputStream(glbPath)) { + GltfAssetV2 asset = new GltfAssetV2(glTF, binaryData); + new GltfAssetWriterV2().writeBinary(asset, out); + } + } +} diff --git a/src/main/java/com/replaymod/render/RenderSettings.java b/src/main/java/com/replaymod/render/RenderSettings.java index f8ee6356..1043792d 100644 --- a/src/main/java/com/replaymod/render/RenderSettings.java +++ b/src/main/java/com/replaymod/render/RenderSettings.java @@ -166,6 +166,7 @@ public class RenderSettings { private final int sphericalFovY; private final boolean injectSphericalMetadata; private final boolean depthMap; + private final boolean cameraPathExport; private final AntiAliasing antiAliasing; private final String exportCommand; @@ -197,6 +198,7 @@ public class RenderSettings { int sphericalFovY, boolean injectSphericalMetadata, boolean depthMap, + boolean cameraPathExport, AntiAliasing antiAliasing, String exportCommand, String exportArguments, @@ -218,6 +220,7 @@ public class RenderSettings { this.sphericalFovY = sphericalFovY; this.injectSphericalMetadata = injectSphericalMetadata; this.depthMap = depthMap; + this.cameraPathExport = cameraPathExport; this.antiAliasing = antiAliasing; this.exportCommand = exportCommand; this.exportArguments = exportArguments; @@ -242,6 +245,7 @@ public class RenderSettings { sphericalFovY, injectSphericalMetadata, depthMap, + cameraPathExport, antiAliasing, exportCommand, exportArguments, @@ -420,6 +424,10 @@ public class RenderSettings { return depthMap; } + public boolean isCameraPathExport() { + return cameraPathExport; + } + public AntiAliasing getAntiAliasing() { return antiAliasing; } diff --git a/src/main/java/com/replaymod/render/gui/GuiExportFailed.java b/src/main/java/com/replaymod/render/gui/GuiExportFailed.java index 66a92a76..b3a8a702 100644 --- a/src/main/java/com/replaymod/render/gui/GuiExportFailed.java +++ b/src/main/java/com/replaymod/render/gui/GuiExportFailed.java @@ -99,6 +99,7 @@ public class GuiExportFailed extends GuiScreen { oldSettings.getSphericalFovY(), oldSettings.isInjectSphericalMetadata(), oldSettings.isDepthMap(), + oldSettings.isCameraPathExport(), oldSettings.getAntiAliasing(), oldSettings.getExportCommand(), oldSettings.getEncodingPreset().getValue(), diff --git a/src/main/java/com/replaymod/render/gui/GuiRenderSettings.java b/src/main/java/com/replaymod/render/gui/GuiRenderSettings.java index 24a29335..52103d3e 100644 --- a/src/main/java/com/replaymod/render/gui/GuiRenderSettings.java +++ b/src/main/java/com/replaymod/render/gui/GuiRenderSettings.java @@ -199,6 +199,9 @@ public class GuiRenderSettings extends AbstractGuiPopup { public final GuiCheckbox depthMap = new GuiCheckbox() .setI18nLabel("replaymod.gui.rendersettings.depthmap"); + public final GuiCheckbox cameraPathExport = new GuiCheckbox() + .setI18nLabel("replaymod.gui.rendersettings.camerapath"); + public final GuiDropdownMenu antiAliasingDropdown = new GuiDropdownMenu() .setSize(200, 20).setValues(RenderSettings.AntiAliasing.values()).setSelected(RenderSettings.AntiAliasing.NONE); @@ -210,6 +213,7 @@ public class GuiRenderSettings extends AbstractGuiPopup { chromaKeyingCheckbox, chromaKeyingColor, injectSphericalMetadata, sphericalFovSlider, depthMap, new GuiLabel(), + cameraPathExport, new GuiLabel(), new GuiLabel().setI18nText("replaymod.gui.rendersettings.antialiasing"), antiAliasingDropdown)); public final GuiTextField exportCommand = new GuiTextField().setI18nHint("replaymod.gui.rendersettings.command") @@ -554,6 +558,7 @@ public class GuiRenderSettings extends AbstractGuiPopup { sphericalFovSlider.setValue((settings.getSphericalFovX() - MIN_SPHERICAL_FOV) / SPHERICAL_FOV_STEP_SIZE); injectSphericalMetadata.setChecked(settings.isInjectSphericalMetadata()); depthMap.setChecked(settings.isDepthMap()); + cameraPathExport.setChecked(settings.isCameraPathExport()); antiAliasingDropdown.setSelected(settings.getAntiAliasing()); exportCommand.setText(settings.getExportCommand()); String exportArguments = settings.getExportArguments(); @@ -585,6 +590,7 @@ public class GuiRenderSettings extends AbstractGuiPopup { sphericalFov, Math.min(180, sphericalFov), injectSphericalMetadata.isChecked() && (serialize || injectSphericalMetadata.isEnabled()), depthMap.isChecked() && (serialize || depthMap.isEnabled()), + cameraPathExport.isChecked(), serialize || antiAliasingDropdown.isEnabled() ? antiAliasingDropdown.getSelectedValue() : RenderSettings.AntiAliasing.NONE, exportCommand.getText(), exportArguments.getText(), @@ -630,7 +636,7 @@ public class GuiRenderSettings extends AbstractGuiPopup { private RenderSettings getDefaultRenderSettings() { return new RenderSettings(RenderSettings.RenderMethod.DEFAULT, RenderSettings.EncodingPreset.MP4_DEFAULT, 1920, 1080, 60, 10 << 20, null, - true, false, false, false, null, 360, 180, false, false, RenderSettings.AntiAliasing.NONE, "", RenderSettings.EncodingPreset.MP4_DEFAULT.getValue(), false); + true, false, false, false, null, 360, 180, false, false, false, RenderSettings.AntiAliasing.NONE, "", RenderSettings.EncodingPreset.MP4_DEFAULT.getValue(), false); } @Override diff --git a/src/main/java/com/replaymod/render/rendering/VideoRenderer.java b/src/main/java/com/replaymod/render/rendering/VideoRenderer.java index 2a2e2ce5..3d47734b 100644 --- a/src/main/java/com/replaymod/render/rendering/VideoRenderer.java +++ b/src/main/java/com/replaymod/render/rendering/VideoRenderer.java @@ -7,6 +7,7 @@ import com.replaymod.core.versions.MCVer; import com.replaymod.pathing.player.AbstractTimelinePlayer; import com.replaymod.pathing.player.ReplayTimer; import com.replaymod.pathing.properties.TimestampProperty; +import com.replaymod.render.CameraPathExporter; import com.replaymod.render.RenderSettings; import com.replaymod.render.ReplayModRender; import com.replaymod.render.FFmpegWriter; @@ -85,6 +86,7 @@ public class VideoRenderer implements RenderInfo { private final Timeline timeline; private final Pipeline renderingPipeline; private final FFmpegWriter ffmpegWriter; + private final CameraPathExporter cameraPathExporter; private int fps; private boolean mouseWasGrabbed; @@ -150,6 +152,12 @@ public class VideoRenderer implements RenderInfo { }; this.renderingPipeline = Pipelines.newPipeline(settings.getRenderMethod(), this, previewingFrameConsumer); } + + if (settings.isCameraPathExport()) { + this.cameraPathExporter = new CameraPathExporter(settings); + } else { + this.cameraPathExporter = null; + } } /** @@ -287,6 +295,10 @@ public class VideoRenderer implements RenderInfo { //$$ mc.displayHeight = displayHeightBefore; //#endif + if (cameraPathExporter != null) { + cameraPathExporter.recordFrame(timer.tickDelta); + } + framesDone++; return timer.tickDelta; } @@ -348,6 +360,10 @@ public class VideoRenderer implements RenderInfo { totalFrames = (int) (duration*fps/1000); + if (cameraPathExporter != null) { + cameraPathExporter.setup(totalFrames); + } + updateDisplaySize(); //#if MC<=10809 @@ -398,6 +414,14 @@ public class VideoRenderer implements RenderInfo { } //#endif + if (!hasFailed() && cameraPathExporter != null) { + try { + cameraPathExporter.finish(); + } catch (IOException e) { + setFailure(e); + } + } + new SoundHandler().playRenderSuccessSound(); try { diff --git a/src/main/resources/assets/replaymod/lang b/src/main/resources/assets/replaymod/lang index 730ddf82..8dc8c18c 160000 --- a/src/main/resources/assets/replaymod/lang +++ b/src/main/resources/assets/replaymod/lang @@ -1 +1 @@ -Subproject commit 730ddf82a77b7c24d31de58d31688c9f0d4b53f4 +Subproject commit 8dc8c18c37bf0c8f5ca41f14c067db248fa0244d diff --git a/versions/common.gradle b/versions/common.gradle index 406036f5..18f1fccb 100644 --- a/versions/common.gradle +++ b/versions/common.gradle @@ -295,6 +295,8 @@ dependencies { } } + shadow 'com.github.javagl.JglTF:jgltf-model:3af6de4' + if (FABRIC) { shadow 'org.apache.maven:maven-artifact:3.6.1' }