From 8bd051eb278631f5d076547182da49e134bf31dd Mon Sep 17 00:00:00 2001 From: CrushedPixel Date: Sat, 12 Sep 2015 13:04:27 +0200 Subject: [PATCH] Add metadata injection for 360 degree videos (mp4 format only) Add com.googlecode.mp4parser:isoparser Library to build.gradle Create inject360Metadata boolean setting in RenderOptions and added respective checkbox to GuiRenderSettings Create MetadataInjector to inject 360 Degree Spherical Metadata into mp4 videos --- build.gradle | 7 +- .../replaymod/gui/GuiRenderSettings.java | 42 ++++++- .../replaymod/settings/RenderOptions.java | 14 +++ .../replaymod/video/VideoRenderer.java | 9 +- .../video/metadata/MetadataInjector.java | 106 ++++++++++++++++++ .../assets/replaymod/lang/en_US.lang | 2 + 6 files changed, 176 insertions(+), 4 deletions(-) create mode 100644 src/main/java/eu/crushedpixel/replaymod/video/metadata/MetadataInjector.java diff --git a/build.gradle b/build.gradle index 2a58bc2e..6182fa4e 100755 --- a/build.gradle +++ b/build.gradle @@ -52,8 +52,11 @@ dependencies { compile 'org.projectlombok:lombok:1.16.4' compile 'org.spongepowered:mixin:0.4.3' + compile 'com.googlecode.mp4parser:isoparser:1.1.7' compile libFiles + compile 'org.aspectj:aspectjrt:1.8.2' + // you may put jars on which you depend on in ./libs // or you may define them like so.. //compile "some.group:artifact:version:classifier" @@ -74,7 +77,9 @@ jar { from project.mixinRefMap - def shade = {libFiles + files(configurations.compile.find { it.name.startsWith("mixin-") })} + def shade = {libFiles + files(configurations.compile.findAll { c -> + ['mixin', 'isoparser', 'aspectjrt'].any {c.name.startsWith("$it-")} + })} def noticeDir = file("$buildDir/NOTICE") doFirst { diff --git a/src/main/java/eu/crushedpixel/replaymod/gui/GuiRenderSettings.java b/src/main/java/eu/crushedpixel/replaymod/gui/GuiRenderSettings.java index d74c9a05..5a4025ac 100644 --- a/src/main/java/eu/crushedpixel/replaymod/gui/GuiRenderSettings.java +++ b/src/main/java/eu/crushedpixel/replaymod/gui/GuiRenderSettings.java @@ -131,13 +131,21 @@ public class GuiRenderSettings extends GuiScreen implements GuiReplayOverlay.NoO }); } + private GuiAdvancedCheckBox injectMetadata = new GuiAdvancedCheckBox(0, 0, I18n.format("replaymod.gui.rendersettings.360metadata"), false); + private GuiVideoFramerateSlider framerateSlider = new GuiVideoFramerateSlider(0, 0, ReplayMod.replaySettings.getVideoFramerate(), I18n.format("replaymod.gui.rendersettings.framerate")); private GuiNumberInput bitrateInput = new GuiNumberInputWithText(mc.fontRendererObj, 0, 0, 50, 1D, null, 10000D, false, " kbps"); private GuiColorPicker colorPicker = new GuiColorPicker(GuiConstants.RENDER_SETTINGS_COLOR_PICKER, 0, 0, I18n.format("replaymod.gui.rendersettings.skycolor")+": ", 0, 0); private GuiAdvancedTextField commandInput = new GuiAdvancedTextField(mc.fontRendererObj, I18n.format("replaymod.gui.rendersettings.command"), 3000); - private GuiAdvancedTextField ffmpegArguments = new GuiAdvancedTextField(mc.fontRendererObj, I18n.format("replaymod.gui.rendersettings.ffmpeghint"), 3000); + private GuiAdvancedTextField ffmpegArguments = new GuiAdvancedTextField(mc.fontRendererObj, I18n.format("replaymod.gui.rendersettings.ffmpeghint"), 3000) { + @Override + public void moveCursorBy(int move) { + super.moveCursorBy(move); + validateInputs(); //the file extension may have changed, so we revalidate the inputs + } + }; private File defaultFile = null; { @@ -193,7 +201,7 @@ public class GuiRenderSettings extends GuiScreen implements GuiReplayOverlay.NoO xRes, yRes, framerateSlider, bitrateString, bitrateInput, fileChooserString, outputFileChooser); private ComposedElement advancedSettings = new ComposedElement(renderButton, cancelButton, toggleTabButton, - renderNameTags, stabilizeString, stablePitch, stableYaw, stableRoll, enableGreenscreen, colorPicker); + renderNameTags, injectMetadata, stabilizeString, stablePitch, stableYaw, stableRoll, enableGreenscreen, colorPicker); private ComposedElement commandLineSettings = new ComposedElement(renderButton, cancelButton, toggleTabButton, commandInput, ffmpegArguments); @@ -319,6 +327,9 @@ public class GuiRenderSettings extends GuiScreen implements GuiReplayOverlay.NoO colorPicker.pickerY = colorPicker.yPosition + 20; colorPicker.setElementEnabled(enableGreenscreen.isChecked()); + + injectMetadata.xPos(leftX); + injectMetadata.yPos(enableGreenscreen.yPos()+heightDiff); } private void initializeCommandLineTab() { @@ -369,6 +380,8 @@ public class GuiRenderSettings extends GuiScreen implements GuiReplayOverlay.NoO enableGreenscreen.setIsChecked(!initialRenderOptions.isDefaultSky()); renderNameTags.setIsChecked(!initialRenderOptions.isHideNameTags()); + injectMetadata.setIsChecked(initialRenderOptions.isInject360Metadata()); + framerateSlider.setFPS(initialRenderOptions.getFps()); String bitrateString = initialRenderOptions.getBitrate(); @@ -490,6 +503,10 @@ public class GuiRenderSettings extends GuiScreen implements GuiReplayOverlay.NoO options.setHideNameTags(!renderNameTags.isChecked()); + if(injectMetadata.enabled) { + options.setInject360Metadata(injectMetadata.isChecked()); + } + options.setWidth(getWidthSetting()); options.setHeight(getHeightSetting()); @@ -567,6 +584,8 @@ public class GuiRenderSettings extends GuiScreen implements GuiReplayOverlay.NoO String newName = FilenameUtils.getBaseName(selectedFile.getAbsolutePath())+"."+preset.getFileExtension(); outputFileChooser.setSelectedFile(new File(selectedFile.getParent(), newName)); } + + validateInputs(); } } @@ -579,6 +598,8 @@ public class GuiRenderSettings extends GuiScreen implements GuiReplayOverlay.NoO private void validateInputs() { boolean valid = true; + boolean isMp4 = false; + boolean isPreset = false; EncodingPreset curPreset = encodingPresetDropdown.getElement(encodingPresetDropdown.getSelectionIndex()); if(ffmpegArguments.getText().trim().equals(curPreset.getCommandLineArgs())) { @@ -592,9 +613,17 @@ public class GuiRenderSettings extends GuiScreen implements GuiReplayOverlay.NoO renderButton.hoverText = I18n.format("replaymod.gui.rendersettings.customresolution.warning.yuv420"); } } + + //injecting Metadata is only possible with mp4 containers. + //To assure the mp4 file extension (which we need to obtain the file path), + //we can't allow custom command line arguments + if("mp4".equals(curPreset.getFileExtension())) { + isMp4 = true; + } } boolean isPanoramic = false; + boolean isEquirectangular = false; switch (rendererDropdown.getElement(rendererDropdown.getSelectionIndex())) { case CUBIC: @@ -607,6 +636,7 @@ public class GuiRenderSettings extends GuiScreen implements GuiReplayOverlay.NoO break; case EQUIRECTANGULAR: isPanoramic = true; + isEquirectangular = true; if (getWidthSetting() / 2 != getHeightSetting() || getWidthSetting() % 2 != 0) { valid = false; @@ -620,6 +650,14 @@ public class GuiRenderSettings extends GuiScreen implements GuiReplayOverlay.NoO stablePitch.setElementEnabled(isPanoramic); stableRoll.setElementEnabled(isPanoramic); + injectMetadata.setElementEnabled(isEquirectangular && isMp4); + if(injectMetadata.enabled) { + injectMetadata.hoverText = null; + } else { + injectMetadata.hoverText = I18n.format("replaymod.gui.rendersettings.360metadata.error"); + injectMetadata.hoverTextColor = Color.RED; + } + if (valid) { renderButton.enabled = true; renderButton.hoverText = ""; diff --git a/src/main/java/eu/crushedpixel/replaymod/settings/RenderOptions.java b/src/main/java/eu/crushedpixel/replaymod/settings/RenderOptions.java index a86079c8..0207e9fd 100644 --- a/src/main/java/eu/crushedpixel/replaymod/settings/RenderOptions.java +++ b/src/main/java/eu/crushedpixel/replaymod/settings/RenderOptions.java @@ -31,6 +31,8 @@ public final class RenderOptions { private int width = Minecraft.getMinecraft().displayWidth; private int height = Minecraft.getMinecraft().displayHeight; + private boolean inject360Metadata = false; + // Highly advanced private boolean highPerformance; private String exportCommand = "ffmpeg"; @@ -65,6 +67,7 @@ public final class RenderOptions { copy.skyColor = this.skyColor; copy.width = this.width; copy.height = this.height; + copy.inject360Metadata = this.inject360Metadata; copy.highPerformance = this.highPerformance; copy.exportCommand = this.exportCommand; copy.exportCommandArgs = this.exportCommandArgs; @@ -85,6 +88,7 @@ public final class RenderOptions { private static final String LINEAR_ENTRY = "linearMovement"; private static final String HIDE_NAMETAGS_ENTRY = "hideNametags"; private static final String SKY_COLOR_ENTRY = "skyColor"; + private static final String INJECT_METADATA_ENTRY = "injectMetadata"; private static final String WIDTH_ENTRY = "xRes"; private static final String HEIGHT_ENTRY = "yRes"; private static final String HIGH_PERFORMANCE_ENTRY = "highPerformance"; @@ -104,6 +108,7 @@ public final class RenderOptions { loaded.isLinearMovement = getLinearFromConfig(config); loaded.hideNameTags = getHideNametagsFromConfig(config); loaded.skyColor = getSkyColorFromConfig(config); + loaded.inject360Metadata = getInject360MetadataFromConfig(config); loaded.width = getWidthFromConfig(config); loaded.height = getHeightFromConfig(config); loaded.highPerformance = getHighPerformanceFromConfig(config); @@ -126,6 +131,7 @@ public final class RenderOptions { getLinearFromConfig(config, isLinearMovement()); getHideNametagsFromConfig(config, isHideNameTags()); getSkyColorFromConfig(config, getSkyColor()); + getInject360MetadataFromConfig(config, isInject360Metadata()); getWidthFromConfig(config, getWidth()); getHeightFromConfig(config, getHeight()); getHighPerformanceFromConfig(config, isHighPerformance()); @@ -211,6 +217,14 @@ public final class RenderOptions { return config.get(CATEGORY_NAME, SKY_COLOR_ENTRY, def).getInt(); } + private static boolean getInject360MetadataFromConfig(Configuration config) { + return getInject360MetadataFromConfig(config, false); + } + + private static boolean getInject360MetadataFromConfig(Configuration config, boolean def) { + return config.get(CATEGORY_NAME, INJECT_METADATA_ENTRY, def).getBoolean(); + } + private static int getWidthFromConfig(Configuration config) { return getWidthFromConfig(config, Minecraft.getMinecraft().displayWidth); } diff --git a/src/main/java/eu/crushedpixel/replaymod/video/VideoRenderer.java b/src/main/java/eu/crushedpixel/replaymod/video/VideoRenderer.java index 973a0117..f5ae743b 100644 --- a/src/main/java/eu/crushedpixel/replaymod/video/VideoRenderer.java +++ b/src/main/java/eu/crushedpixel/replaymod/video/VideoRenderer.java @@ -6,7 +6,7 @@ import eu.crushedpixel.replaymod.holders.AdvancedPosition; import eu.crushedpixel.replaymod.holders.Keyframe; import eu.crushedpixel.replaymod.holders.SpectatorData; import eu.crushedpixel.replaymod.holders.TimestampValue; -import eu.crushedpixel.replaymod.interpolation.*; +import eu.crushedpixel.replaymod.interpolation.KeyframeList; import eu.crushedpixel.replaymod.renderer.ChunkLoadingRenderGlobal; import eu.crushedpixel.replaymod.replay.ReplayHandler; import eu.crushedpixel.replaymod.replay.ReplaySender; @@ -15,6 +15,7 @@ import eu.crushedpixel.replaymod.timer.EnchantmentTimer; import eu.crushedpixel.replaymod.timer.ReplayTimer; import eu.crushedpixel.replaymod.video.capturer.RenderInfo; import eu.crushedpixel.replaymod.video.frame.RGBFrame; +import eu.crushedpixel.replaymod.video.metadata.MetadataInjector; import eu.crushedpixel.replaymod.video.rendering.Pipeline; import eu.crushedpixel.replaymod.video.rendering.Pipelines; import net.minecraft.client.Minecraft; @@ -25,6 +26,7 @@ import org.lwjgl.opengl.Display; import org.lwjgl.util.Dimension; import org.lwjgl.util.ReadableDimension; +import java.io.File; import java.io.IOException; import java.util.concurrent.FutureTask; @@ -104,6 +106,11 @@ public class VideoRenderer implements RenderInfo { renderingPipeline.run(); + if(options.isInject360Metadata()) { + File outputFile = options.getOutputFile(); + MetadataInjector.inject360Metadata(new File(outputFile.getParentFile(), outputFile.getName() + ".mp4")); + } + finish(); return !cancelled; } diff --git a/src/main/java/eu/crushedpixel/replaymod/video/metadata/MetadataInjector.java b/src/main/java/eu/crushedpixel/replaymod/video/metadata/MetadataInjector.java new file mode 100644 index 00000000..bde64fd0 --- /dev/null +++ b/src/main/java/eu/crushedpixel/replaymod/video/metadata/MetadataInjector.java @@ -0,0 +1,106 @@ +package eu.crushedpixel.replaymod.video.metadata; + +import com.coremedia.iso.IsoFile; +import com.coremedia.iso.boxes.*; +import com.google.common.primitives.Bytes; +import com.googlecode.mp4parser.BasicContainer; +import net.minecraftforge.fml.common.FMLLog; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; + +public class MetadataInjector { + + private static final String STITCHING_SOFTWARE = "Minecraft ReplayMod"; + + /* + * The Spherical XML is a modified version of https://github.com/google/spatial-media/tree/master/360-Videos-Metadata + */ + private static final String SPHERICAL_XML_HEADER = + " " + + " "; + private static final String SPHERICAL_XML_CONTENTS = + "true " + + "true " + + ""+STITCHING_SOFTWARE+" " + + "equirectangular "; + private static final String SPHERICAL_XML_FOOTER = ""; + + private static final String XML_360_METADATA = SPHERICAL_XML_HEADER + SPHERICAL_XML_CONTENTS + SPHERICAL_XML_FOOTER; + + /** + * These bytes are taken from the variable 'spherical_uuid_id' + * in https://github.com/google/spatial-media/tree/master/360-Videos-Metadata + */ + private static final byte[] UUID_BYTES = new byte[] { + (byte)0xff, (byte)0xcc, (byte)0x82, (byte)0x63, + (byte)0xf8, (byte)0x55, (byte)0x4a, (byte)0x93, + (byte)0x88, (byte)0x14, (byte)0x58, (byte)0x7a, + (byte)0x02, (byte)0x52, (byte)0x1f, (byte)0xdd + }; + + private static final byte[] BYTES_360_METADATA = Bytes.concat(UUID_BYTES, XML_360_METADATA.getBytes()); + + public static void inject360Metadata(File videoFile) { + writeMetadata(videoFile, BYTES_360_METADATA); + } + + private static void writeMetadata(File videoFile, byte[] metadata) { + File tempFile = null; + FileOutputStream videoFileOutputStream = null; + IsoFile tempIsoFile = null; + + try { + tempFile = File.createTempFile("videoCopy", "mp4"); + FileUtils.copyFile(videoFile, tempFile); + + tempIsoFile = new IsoFile(tempFile.getAbsolutePath()); + + //first, get the "moov" box from the mp4 + MovieBox moovBox = (MovieBox)getBoxByName(tempIsoFile, "moov"); + if(moovBox == null) throw new IOException("Could not find moov box inside IsoFile"); + + //get the Movie Track to which we will add the Metadata + TrackBox trackBox = (TrackBox)getBoxByName(moovBox, "trak"); + if(trackBox == null) throw new IOException("Could not find trak box inside moov box"); + + //create a new UserBox, which actually contains the Metadata bytes + UserBox metadataBox = new UserBox(new byte[0]); + metadataBox.setData(metadata); + + //add the Metadata UserBox to the Movie Track + trackBox.addBox(metadataBox); + + //finally, reduce the Video's FreeBox, whose size we will need + //to reduce by the Metadata Box's size to preserve the Video's File size + FreeBox freeBox = (FreeBox)getBoxByName(tempIsoFile, "free"); + if(freeBox == null) throw new IOException("Could not find free box inside IsoFile"); + + int freeSize = Math.max(0, freeBox.getData().capacity() - (int)metadataBox.getSize()); + freeBox.setData(ByteBuffer.allocate(freeSize)); + + //save the ISO file to disk + videoFileOutputStream = new FileOutputStream(videoFile); + tempIsoFile.getBox(videoFileOutputStream.getChannel()); + } catch(Exception e) { + FMLLog.getLogger().error("360 Degree Metadata couldn't be injected", e); + } finally { + IOUtils.closeQuietly(tempIsoFile); + IOUtils.closeQuietly(videoFileOutputStream); + FileUtils.deleteQuietly(tempFile); + } + } + + private static Box getBoxByName(BasicContainer container, String boxName) { + for (Box box : container.getBoxes()) { + if(box.getType().equals(boxName)) return box; + } + return null; + } + +} \ No newline at end of file diff --git a/src/main/resources/assets/replaymod/lang/en_US.lang b/src/main/resources/assets/replaymod/lang/en_US.lang index ef9bb39e..42c3e46d 100644 --- a/src/main/resources/assets/replaymod/lang/en_US.lang +++ b/src/main/resources/assets/replaymod/lang/en_US.lang @@ -348,6 +348,8 @@ replaymod.gui.rendersettings.chromakey=Chroma Keying replaymod.gui.rendersettings.skycolor=Sky Color replaymod.gui.rendersettings.nametags=Render Name Tags replaymod.gui.rendersettings.outputfile=Output File +replaymod.gui.rendersettings.360metadata=Inject 360° Metadata +replaymod.gui.rendersettings.360metadata.error=360° Metadata can only be added to Equirectangular videos in MP4 format replaymod.gui.rendersettings.command=Command replaymod.gui.rendersettings.arguments=Command Line Arguments