Add metadata injection for 360 degree videos (mp4 format only)

Add com.googlecode.mp4parser:isoparser Library to build.gradle
Create inject360Metadata boolean setting in RenderOptions and added respective checkbox to GuiRenderSettings
Create MetadataInjector to inject 360 Degree Spherical Metadata into mp4 videos

build.gradle

@@ -52,8 +52,11 @@ dependencies {
     compile 'org.projectlombok:lombok:1.16.4'
 
     compile 'org.spongepowered:mixin:0.4.3'
+    compile 'com.googlecode.mp4parser:isoparser:1.1.7'
     compile libFiles
 
+    compile 'org.aspectj:aspectjrt:1.8.2'
+
     // you may put jars on which you depend on in ./libs
     // or you may define them like so..
     //compile "some.group:artifact:version:classifier"
@@ -74,7 +77,9 @@ jar {
 
     from project.mixinRefMap
 
-    def shade = {libFiles + files(configurations.compile.find { it.name.startsWith("mixin-") })}
+    def shade = {libFiles + files(configurations.compile.findAll { c ->
+        ['mixin', 'isoparser', 'aspectjrt'].any {c.name.startsWith("$it-")}
+    })}
 
     def noticeDir = file("$buildDir/NOTICE")
     doFirst {

src/main/java/eu/crushedpixel/replaymod/gui/GuiRenderSettings.java

@@ -131,13 +131,21 @@ public class GuiRenderSettings extends GuiScreen implements GuiReplayOverlay.NoO
         });
     }
 
+    private GuiAdvancedCheckBox injectMetadata = new GuiAdvancedCheckBox(0, 0, I18n.format("replaymod.gui.rendersettings.360metadata"), false);
+
     private GuiVideoFramerateSlider framerateSlider = new GuiVideoFramerateSlider(0, 0, ReplayMod.replaySettings.getVideoFramerate(),
             I18n.format("replaymod.gui.rendersettings.framerate"));
 
     private GuiNumberInput bitrateInput = new GuiNumberInputWithText(mc.fontRendererObj, 0, 0, 50, 1D, null, 10000D, false, " kbps");
     private GuiColorPicker colorPicker = new GuiColorPicker(GuiConstants.RENDER_SETTINGS_COLOR_PICKER, 0, 0, I18n.format("replaymod.gui.rendersettings.skycolor")+": ", 0, 0);
     private GuiAdvancedTextField commandInput = new GuiAdvancedTextField(mc.fontRendererObj, I18n.format("replaymod.gui.rendersettings.command"), 3000);
-    private GuiAdvancedTextField ffmpegArguments = new GuiAdvancedTextField(mc.fontRendererObj, I18n.format("replaymod.gui.rendersettings.ffmpeghint"), 3000);
+    private GuiAdvancedTextField ffmpegArguments = new GuiAdvancedTextField(mc.fontRendererObj, I18n.format("replaymod.gui.rendersettings.ffmpeghint"), 3000) {
+        @Override
+        public void moveCursorBy(int move) {
+            super.moveCursorBy(move);
+            validateInputs(); //the file extension may have changed, so we revalidate the inputs
+        }
+    };
 
     private File defaultFile = null;
     {
@@ -193,7 +201,7 @@ public class GuiRenderSettings extends GuiScreen implements GuiReplayOverlay.NoO
                 xRes, yRes, framerateSlider, bitrateString, bitrateInput, fileChooserString, outputFileChooser);
 
         private ComposedElement advancedSettings = new ComposedElement(renderButton, cancelButton, toggleTabButton,
-                renderNameTags, stabilizeString, stablePitch, stableYaw, stableRoll, enableGreenscreen, colorPicker);
+                renderNameTags, injectMetadata, stabilizeString, stablePitch, stableYaw, stableRoll, enableGreenscreen, colorPicker);
 
         private ComposedElement commandLineSettings = new ComposedElement(renderButton, cancelButton, toggleTabButton,
                 commandInput, ffmpegArguments);
@@ -319,6 +327,9 @@ public class GuiRenderSettings extends GuiScreen implements GuiReplayOverlay.NoO
         colorPicker.pickerY = colorPicker.yPosition + 20;
 
         colorPicker.setElementEnabled(enableGreenscreen.isChecked());
+
+        injectMetadata.xPos(leftX);
+        injectMetadata.yPos(enableGreenscreen.yPos()+heightDiff);
     }
 
     private void initializeCommandLineTab() {
@@ -369,6 +380,8 @@ public class GuiRenderSettings extends GuiScreen implements GuiReplayOverlay.NoO
         enableGreenscreen.setIsChecked(!initialRenderOptions.isDefaultSky());
         renderNameTags.setIsChecked(!initialRenderOptions.isHideNameTags());
 
+        injectMetadata.setIsChecked(initialRenderOptions.isInject360Metadata());
+
         framerateSlider.setFPS(initialRenderOptions.getFps());
 
         String bitrateString = initialRenderOptions.getBitrate();
@@ -490,6 +503,10 @@ public class GuiRenderSettings extends GuiScreen implements GuiReplayOverlay.NoO
 
         options.setHideNameTags(!renderNameTags.isChecked());
 
+        if(injectMetadata.enabled) {
+            options.setInject360Metadata(injectMetadata.isChecked());
+        }
+
         options.setWidth(getWidthSetting());
         options.setHeight(getHeightSetting());
 
@@ -567,6 +584,8 @@ public class GuiRenderSettings extends GuiScreen implements GuiReplayOverlay.NoO
                 String newName = FilenameUtils.getBaseName(selectedFile.getAbsolutePath())+"."+preset.getFileExtension();
                 outputFileChooser.setSelectedFile(new File(selectedFile.getParent(), newName));
             }
+
+            validateInputs();
         }
     }
 
@@ -579,6 +598,8 @@ public class GuiRenderSettings extends GuiScreen implements GuiReplayOverlay.NoO
     private void validateInputs() {
         boolean valid = true;
 
+        boolean isMp4 = false;
+
         boolean isPreset = false;
         EncodingPreset curPreset = encodingPresetDropdown.getElement(encodingPresetDropdown.getSelectionIndex());
         if(ffmpegArguments.getText().trim().equals(curPreset.getCommandLineArgs())) {
@@ -592,9 +613,17 @@ public class GuiRenderSettings extends GuiScreen implements GuiReplayOverlay.NoO
                     renderButton.hoverText = I18n.format("replaymod.gui.rendersettings.customresolution.warning.yuv420");
                 }
             }
+
+            //injecting Metadata is only possible with mp4 containers.
+            //To assure the mp4 file extension (which we need to obtain the file path),
+            //we can't allow custom command line arguments
+            if("mp4".equals(curPreset.getFileExtension())) {
+                isMp4 = true;
+            }
         }
 
         boolean isPanoramic = false;
+        boolean isEquirectangular = false;
 
         switch (rendererDropdown.getElement(rendererDropdown.getSelectionIndex())) {
             case CUBIC:
@@ -607,6 +636,7 @@ public class GuiRenderSettings extends GuiScreen implements GuiReplayOverlay.NoO
                 break;
             case EQUIRECTANGULAR:
                 isPanoramic = true;
+                isEquirectangular = true;
                 if (getWidthSetting() / 2 != getHeightSetting()
                         || getWidthSetting() % 2 != 0) {
                     valid = false;
@@ -620,6 +650,14 @@ public class GuiRenderSettings extends GuiScreen implements GuiReplayOverlay.NoO
         stablePitch.setElementEnabled(isPanoramic);
         stableRoll.setElementEnabled(isPanoramic);
 
+        injectMetadata.setElementEnabled(isEquirectangular && isMp4);
+        if(injectMetadata.enabled) {
+            injectMetadata.hoverText = null;
+        } else {
+            injectMetadata.hoverText = I18n.format("replaymod.gui.rendersettings.360metadata.error");
+            injectMetadata.hoverTextColor = Color.RED;
+        }
+
         if (valid) {
             renderButton.enabled = true;
             renderButton.hoverText = "";

src/main/java/eu/crushedpixel/replaymod/settings/RenderOptions.java

@@ -31,6 +31,8 @@ public final class RenderOptions {
     private int width = Minecraft.getMinecraft().displayWidth;
     private int height = Minecraft.getMinecraft().displayHeight;
 
+    private boolean inject360Metadata = false;
+
     // Highly advanced
     private boolean highPerformance;
     private String exportCommand = "ffmpeg";
@@ -65,6 +67,7 @@ public final class RenderOptions {
         copy.skyColor = this.skyColor;
         copy.width = this.width;
         copy.height = this.height;
+        copy.inject360Metadata = this.inject360Metadata;
         copy.highPerformance = this.highPerformance;
         copy.exportCommand = this.exportCommand;
         copy.exportCommandArgs = this.exportCommandArgs;
@@ -85,6 +88,7 @@ public final class RenderOptions {
     private static final String LINEAR_ENTRY = "linearMovement";
     private static final String HIDE_NAMETAGS_ENTRY = "hideNametags";
     private static final String SKY_COLOR_ENTRY = "skyColor";
+    private static final String INJECT_METADATA_ENTRY = "injectMetadata";
     private static final String WIDTH_ENTRY = "xRes";
     private static final String HEIGHT_ENTRY = "yRes";
     private static final String HIGH_PERFORMANCE_ENTRY = "highPerformance";
@@ -104,6 +108,7 @@ public final class RenderOptions {
         loaded.isLinearMovement = getLinearFromConfig(config);
         loaded.hideNameTags = getHideNametagsFromConfig(config);
         loaded.skyColor = getSkyColorFromConfig(config);
+        loaded.inject360Metadata = getInject360MetadataFromConfig(config);
         loaded.width = getWidthFromConfig(config);
         loaded.height = getHeightFromConfig(config);
         loaded.highPerformance = getHighPerformanceFromConfig(config);
@@ -126,6 +131,7 @@ public final class RenderOptions {
         getLinearFromConfig(config, isLinearMovement());
         getHideNametagsFromConfig(config, isHideNameTags());
         getSkyColorFromConfig(config, getSkyColor());
+        getInject360MetadataFromConfig(config, isInject360Metadata());
         getWidthFromConfig(config, getWidth());
         getHeightFromConfig(config, getHeight());
         getHighPerformanceFromConfig(config, isHighPerformance());
@@ -211,6 +217,14 @@ public final class RenderOptions {
         return config.get(CATEGORY_NAME, SKY_COLOR_ENTRY, def).getInt();
     }
 
+    private static boolean getInject360MetadataFromConfig(Configuration config) {
+        return getInject360MetadataFromConfig(config, false);
+    }
+
+    private static boolean getInject360MetadataFromConfig(Configuration config, boolean def) {
+        return config.get(CATEGORY_NAME, INJECT_METADATA_ENTRY, def).getBoolean();
+    }
+
     private static int getWidthFromConfig(Configuration config) {
         return getWidthFromConfig(config, Minecraft.getMinecraft().displayWidth);
     }

src/main/java/eu/crushedpixel/replaymod/video/VideoRenderer.java

@@ -6,7 +6,7 @@ import eu.crushedpixel.replaymod.holders.AdvancedPosition;
 import eu.crushedpixel.replaymod.holders.Keyframe;
 import eu.crushedpixel.replaymod.holders.SpectatorData;
 import eu.crushedpixel.replaymod.holders.TimestampValue;
-import eu.crushedpixel.replaymod.interpolation.*;
+import eu.crushedpixel.replaymod.interpolation.KeyframeList;
 import eu.crushedpixel.replaymod.renderer.ChunkLoadingRenderGlobal;
 import eu.crushedpixel.replaymod.replay.ReplayHandler;
 import eu.crushedpixel.replaymod.replay.ReplaySender;
@@ -15,6 +15,7 @@ import eu.crushedpixel.replaymod.timer.EnchantmentTimer;
 import eu.crushedpixel.replaymod.timer.ReplayTimer;
 import eu.crushedpixel.replaymod.video.capturer.RenderInfo;
 import eu.crushedpixel.replaymod.video.frame.RGBFrame;
+import eu.crushedpixel.replaymod.video.metadata.MetadataInjector;
 import eu.crushedpixel.replaymod.video.rendering.Pipeline;
 import eu.crushedpixel.replaymod.video.rendering.Pipelines;
 import net.minecraft.client.Minecraft;
@@ -25,6 +26,7 @@ import org.lwjgl.opengl.Display;
 import org.lwjgl.util.Dimension;
 import org.lwjgl.util.ReadableDimension;
 
+import java.io.File;
 import java.io.IOException;
 import java.util.concurrent.FutureTask;
 
@@ -104,6 +106,11 @@ public class VideoRenderer implements RenderInfo {
 
         renderingPipeline.run();
 
+        if(options.isInject360Metadata()) {
+            File outputFile = options.getOutputFile();
+            MetadataInjector.inject360Metadata(new File(outputFile.getParentFile(), outputFile.getName() + ".mp4"));
+        }
+
         finish();
         return !cancelled;
     }

src/main/java/eu/crushedpixel/replaymod/video/metadata/MetadataInjector.java

@@ -0,0 +1,106 @@
+package eu.crushedpixel.replaymod.video.metadata;
+
+import com.coremedia.iso.IsoFile;
+import com.coremedia.iso.boxes.*;
+import com.google.common.primitives.Bytes;
+import com.googlecode.mp4parser.BasicContainer;
+import net.minecraftforge.fml.common.FMLLog;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+public class MetadataInjector {
+
+    private static final String STITCHING_SOFTWARE = "Minecraft ReplayMod";
+
+    /*
+     * The Spherical XML is a modified version of https://github.com/google/spatial-media/tree/master/360-Videos-Metadata
+     */
+    private static final String SPHERICAL_XML_HEADER =
+            "<?xml version=\"1.0\"?> " +
+            "<rdf:SphericalVideo xmlns:rdf=" +
+            "\"http://www.w3.org/1999/02/22-rdf-syntax-ns#\" xmlns:GSpherical=\"http://ns.google.com/videos/1.0/spherical/\"> ";
+    private static final String SPHERICAL_XML_CONTENTS =
+            "<GSpherical:Spherical>true</GSpherical:Spherical> " +
+            "<GSpherical:Stitched>true</GSpherical:Stitched> " +
+            "<GSpherical:StitchingSoftware>"+STITCHING_SOFTWARE+"</GSpherical:StitchingSoftware> " +
+            "<GSpherical:ProjectionType>equirectangular</GSpherical:ProjectionType> ";
+    private static final String SPHERICAL_XML_FOOTER = "</rdf:SphericalVideo>";
+
+    private static final String XML_360_METADATA = SPHERICAL_XML_HEADER + SPHERICAL_XML_CONTENTS + SPHERICAL_XML_FOOTER;
+
+    /**
+     * These bytes are taken from the variable 'spherical_uuid_id'
+     * in https://github.com/google/spatial-media/tree/master/360-Videos-Metadata
+     */
+    private static final byte[] UUID_BYTES = new byte[] {
+            (byte)0xff, (byte)0xcc, (byte)0x82, (byte)0x63,
+            (byte)0xf8, (byte)0x55, (byte)0x4a, (byte)0x93,
+            (byte)0x88, (byte)0x14, (byte)0x58, (byte)0x7a,
+            (byte)0x02, (byte)0x52, (byte)0x1f, (byte)0xdd
+    };
+
+    private static final byte[] BYTES_360_METADATA = Bytes.concat(UUID_BYTES, XML_360_METADATA.getBytes());
+
+    public static void inject360Metadata(File videoFile) {
+        writeMetadata(videoFile, BYTES_360_METADATA);
+    }
+
+    private static void writeMetadata(File videoFile, byte[] metadata) {
+        File tempFile = null;
+        FileOutputStream videoFileOutputStream = null;
+        IsoFile tempIsoFile = null;
+
+        try {
+            tempFile = File.createTempFile("videoCopy", "mp4");
+            FileUtils.copyFile(videoFile, tempFile);
+
+            tempIsoFile = new IsoFile(tempFile.getAbsolutePath());
+
+            //first, get the "moov" box from the mp4
+            MovieBox moovBox = (MovieBox)getBoxByName(tempIsoFile, "moov");
+            if(moovBox == null) throw new IOException("Could not find moov box inside IsoFile");
+
+            //get the Movie Track to which we will add the Metadata
+            TrackBox trackBox = (TrackBox)getBoxByName(moovBox, "trak");
+            if(trackBox == null) throw new IOException("Could not find trak box inside moov box");
+
+            //create a new UserBox, which actually contains the Metadata bytes
+            UserBox metadataBox = new UserBox(new byte[0]);
+            metadataBox.setData(metadata);
+
+            //add the Metadata UserBox to the Movie Track
+            trackBox.addBox(metadataBox);
+
+            //finally, reduce the Video's FreeBox, whose size we will need
+            //to reduce by the Metadata Box's size to preserve the Video's File size
+            FreeBox freeBox = (FreeBox)getBoxByName(tempIsoFile, "free");
+            if(freeBox == null) throw new IOException("Could not find free box inside IsoFile");
+
+            int freeSize = Math.max(0, freeBox.getData().capacity() - (int)metadataBox.getSize());
+            freeBox.setData(ByteBuffer.allocate(freeSize));
+
+            //save the ISO file to disk
+            videoFileOutputStream = new FileOutputStream(videoFile);
+            tempIsoFile.getBox(videoFileOutputStream.getChannel());
+        } catch(Exception e) {
+            FMLLog.getLogger().error("360 Degree Metadata couldn't be injected", e);
+        } finally {
+            IOUtils.closeQuietly(tempIsoFile);
+            IOUtils.closeQuietly(videoFileOutputStream);
+            FileUtils.deleteQuietly(tempFile);
+        }
+    }
+
+    private static Box getBoxByName(BasicContainer container, String boxName) {
+        for (Box box : container.getBoxes()) {
+            if(box.getType().equals(boxName)) return box;
+        }
+        return null;
+    }
+
+}

src/main/resources/assets/replaymod/lang/en_US.lang

@@ -348,6 +348,8 @@ replaymod.gui.rendersettings.chromakey=Chroma Keying
 replaymod.gui.rendersettings.skycolor=Sky Color
 replaymod.gui.rendersettings.nametags=Render Name Tags
 replaymod.gui.rendersettings.outputfile=Output File
+replaymod.gui.rendersettings.360metadata=Inject 360° Metadata
+replaymod.gui.rendersettings.360metadata.error=360° Metadata can only be added to Equirectangular videos in MP4 format
 
 replaymod.gui.rendersettings.command=Command
 replaymod.gui.rendersettings.arguments=Command Line Arguments