buffered file systems (only save data to disk when the world is saving); the data stored to disk directly is really just a copy of the data, which is also stored in the world, since we use a virtual file system for that (same as for /tmp)

assets/opencomputers/lua/drivers/filesystem.lua

@@ -435,12 +435,10 @@ function file:seek(whence, offset)
   checkArg(2, offset, "number")
   assert(math.floor(offset) == offset, "bad argument #2 (not an integer)")
 
-  local result, reason
   if whence == "cur" then
-    result, reason = self.stream:seek(whence, offset - #self.buffer)
+    offset = offset - #self.buffer
-  else
-    result, reason = self.stream:seek(whence, offset)
   end
+  local result, reason = self.stream:seek(whence, offset)
   if result then
     self.buffer = ""
     return result

li/cil/oc/api/FileSystem.scala

@@ -259,13 +259,20 @@ object FileSystem extends FileSystemAPI {
    * <p/>
    * Usually the name will be the name of the node used to represent the
    * file system.
+   * <p/>
+   * Note that by default file systems are "buffered", meaning that any
+   * changes made to them are only saved to disk when the world is saved. This
+   * ensured that the file system contents do not go "out of sync" when the
+   * game crashes, but introduces additional memory overhead, since all files
+   * in the file system have to be kept in memory.
    *
    * @param root the name of the file system.
    * @param capacity the amount of space in bytes to allow being used.
+   * @param buffered whether data should only be written to disk when saving.
    * @return a file system wrapping the specified folder.
    */
-  def fromSaveDir(root: String, capacity: Long) =
+  def fromSaveDir(root: String, capacity: Long, buffered: Boolean = true) =
-    instance.fold(None: Option[FileSystem])(_.fromSaveDir(root, capacity))
+    instance.fold(None: Option[FileSystem])(_.fromSaveDir(root, capacity, buffered))
 
   /**
    * Creates a new *writable* file system that resides in memory.

li/cil/oc/api/detail/FileSystemAPI.scala

@@ -7,7 +7,7 @@ import li.cil.oc.api.network.Node
 trait FileSystemAPI {
   def fromClass(clazz: Class[_], domain: String, root: String): Option[FileSystem]
 
-  def fromSaveDir(root: String, capacity: Long): Option[FileSystem]
+  def fromSaveDir(root: String, capacity: Long, buffered: Boolean): Option[FileSystem]
 
   def fromRam(capacity: Long): Option[FileSystem]
 

li/cil/oc/server/component/Filesystem.scala

@@ -84,7 +84,7 @@ class FileSystem(val fileSystem: api.FileSystem) extends Node {
 
         case Array(handle: Double, n: Double) if message.name == "fs.read" && n > 0 =>
           fileSystem.file(handle.toInt) match {
-            case None => None
+            case None => throw new IOException("bad file descriptor")
             case Some(file) =>
               // Limit reading to chunks of 8KB to avoid crazy allocations.
               val buffer = new Array[Byte](n.toInt min (8 * 1024))
@@ -106,7 +106,7 @@ class FileSystem(val fileSystem: api.FileSystem) extends Node {
           }
         case Array(handle: Double, whence: Array[Byte], offset: Double) if message.name == "fs.seek" =>
           fileSystem.file(handle.toInt) match {
-            case None => None
+            case None => throw new IOException("bad file descriptor")
             case Some(file) =>
               new String(whence, "UTF-8") match {
                 case "cur" => file.seek(file.position + offset.toInt)
@@ -118,7 +118,7 @@ class FileSystem(val fileSystem: api.FileSystem) extends Node {
           }
         case Array(handle: Double, value: Array[Byte]) if message.name == "fs.write" =>
           fileSystem.file(handle.toInt) match {
-            case None => None
+            case None => throw new IOException("bad file descriptor")
             case Some(file) => file.write(value); result(true)
           }
         case _ => None

li/cil/oc/server/fs/Buffered.scala

@@ -0,0 +1,89 @@
+package li.cil.oc.server.fs
+
+import java.io
+import li.cil.oc.api.fs.Mode
+import net.minecraft.nbt.NBTTagCompound
+import scala.collection.mutable
+
+trait Buffered extends OutputStreamFileSystem {
+  protected def fileRoot: io.File
+
+  private val deletions = mutable.Set.empty[String]
+
+  // ----------------------------------------------------------------------- //
+
+  override def rename(from: String, to: String) = {
+    if (super.rename(from, to)) {
+      deletions += from
+      true
+    }
+    else false
+  }
+
+  override protected def delete(path: String) = {
+    if (super.delete(path)) {
+      deletions += path
+      true
+    }
+    else false
+  }
+
+  // ----------------------------------------------------------------------- //
+
+  override def load(nbt: NBTTagCompound) = {
+    super.load(nbt)
+    def recurse(path: String, directory: io.File) {
+      makeDirectories(path)
+      if (isDirectory(path)) for (child <- directory.listFiles()) {
+        val childPath = path + child.getName
+        val childFile = new io.File(directory, child.getName)
+        if (child.isDirectory) {
+          recurse(childPath + "/", childFile)
+        }
+        else if (!exists(childPath) || !isDirectory(childPath)) {
+          openOutputStream(childPath, Mode.Write) match {
+            case Some(stream) =>
+              val in = new io.FileInputStream(childFile).getChannel
+              val out = java.nio.channels.Channels.newChannel(stream)
+              in.transferTo(0, Long.MaxValue, out)
+              in.close()
+              out.close()
+            case _ => // File is open for writing.
+          }
+        }
+      }
+    }
+    recurse("", fileRoot)
+  }
+
+  override def save(nbt: NBTTagCompound) = {
+    super.save(nbt)
+
+    for (path <- deletions)
+      org.apache.commons.io.FileUtils.deleteQuietly(new io.File(fileRoot, path))
+    deletions.clear()
+
+    def recurse(path: String) {
+      val directory = new io.File(fileRoot, path)
+      if (directory.exists() && !directory.isDirectory)
+        org.apache.commons.io.FileUtils.deleteQuietly(directory)
+      directory.mkdirs()
+      for (child <- list(path).get) {
+        val childPath = path + child
+        if (isDirectory(childPath))
+          recurse(childPath)
+        else {
+          val childFile = new io.File(fileRoot, childPath)
+          org.apache.commons.io.FileUtils.deleteQuietly(childFile)
+          childFile.createNewFile()
+          val out = new io.FileOutputStream(childFile).getChannel
+          val in = java.nio.channels.Channels.newChannel(openInputStream(childPath).get)
+          out.transferFrom(in, 0, Long.MaxValue)
+          out.close()
+          in.close()
+        }
+      }
+    }
+    recurse("")
+  }
+}

li/cil/oc/server/fs/FileSystem.scala

@@ -38,11 +38,15 @@ object FileSystem extends api.detail.FileSystemAPI {
     }
   }
 
-  override def fromSaveDir(root: String, capacity: Long) = {
+  override def fromSaveDir(root: String, capacity: Long, buffered: Boolean) = {
     val path = new io.File(DimensionManager.getCurrentSaveRootDirectory, Config.savePath + root)
     path.mkdirs()
-    if (path.exists() && path.isDirectory)
+    if (path.exists() && path.isDirectory) {
-      Some(new ReadWriteFileSystem(path, capacity))
+      if (buffered)
+        Some(new BufferedFileSystem(path, capacity))
+      else
+        Some(new ReadWriteFileSystem(path, capacity))
+    }
     else None
   }
 
@@ -63,4 +67,8 @@ object FileSystem extends api.detail.FileSystemAPI {
     extends VirtualFileSystem
     with Capacity
 
+  private class BufferedFileSystem(protected val fileRoot: io.File, capacity: Long)
+    extends RamFileSystem(capacity)
+    with Buffered
+
 }

li/cil/oc/server/fs/VirtualFileSystem.scala

@@ -105,11 +105,15 @@ class VirtualFileSystem extends OutputStreamFileSystem {
     def get(path: Iterable[String]): Option[VirtualObject] = if (path.isEmpty) Some(this) else None
   }
 
+  // ----------------------------------------------------------------------- //
+
   private class VirtualFile extends VirtualObject {
-    var data = Array.empty[Byte]
+    val data = mutable.ArrayBuffer.empty[Byte]
 
     var stream: Option[VirtualFileOutputStream] = None
 
+    // ----------------------------------------------------------------------- //
+
     override def isDirectory(path: Iterable[String]) = false
 
     override def size(path: Iterable[String]) = data.length
@@ -122,29 +126,38 @@ class VirtualFileSystem extends OutputStreamFileSystem {
 
     override def canDelete = stream.isEmpty
 
+    // ----------------------------------------------------------------------- //
+
     override def openInputStream(path: Iterable[String]) =
       if (path.isEmpty) Some(new VirtualFileInputStream(this))
       else None
 
     override def openOutputStream(path: Iterable[String], mode: Mode.Value) =
       if (path.isEmpty) {
-        if (stream.isDefined) throw new io.FileNotFoundException()
+        if (stream.isDefined) None
-        if (mode == Mode.Write)
+        else {
-          data = Array.empty[Byte]
+          if (mode == Mode.Write)
-        stream = Some(new VirtualFileOutputStream(this))
+            data.clear()
-        stream
+          stream = Some(new VirtualFileOutputStream(this))
+          stream
+        }
       }
       else None
 
+    // ----------------------------------------------------------------------- //
+
     override def load(nbt: NBTTagCompound) {
-      data = nbt.getByteArray("data")
+      data.clear()
+      data ++= nbt.getByteArray("data")
     }
 
     override def save(nbt: NBTTagCompound) {
-      nbt.setByteArray("data", data)
+      nbt.setByteArray("data", data.toArray)
     }
   }
 
+  // ----------------------------------------------------------------------- //
+
   private class VirtualDirectory extends VirtualObject {
     val children = mutable.Map.empty[String, VirtualObject]
 
@@ -204,6 +217,8 @@ class VirtualFileSystem extends OutputStreamFileSystem {
 
     override def canDelete = children.isEmpty
 
+    // ----------------------------------------------------------------------- //
+
     override def openInputStream(path: Iterable[String]) =
       if (path.isEmpty) None
       else children.get(path.head) match {
@@ -223,6 +238,8 @@ class VirtualFileSystem extends OutputStreamFileSystem {
         case _ => None
       }
 
+    // ----------------------------------------------------------------------- //
+
     override def load(nbt: NBTTagCompound) {
       val childrenNbt = nbt.getTagList("children")
       (0 until childrenNbt.tagCount).map(childrenNbt.tagAt).map(_.asInstanceOf[NBTTagCompound]).foreach(childNbt => {
@@ -246,6 +263,8 @@ class VirtualFileSystem extends OutputStreamFileSystem {
       nbt.setTag("children", childrenNbt)
     }
 
+    // ----------------------------------------------------------------------- //
+
     override def get(path: Iterable[String]) =
       super.get(path) orElse {
         children.get(path.head) match {
@@ -283,7 +302,7 @@ class VirtualFileSystem extends OutputStreamFileSystem {
         if (available == 0) -1
         else {
           val n = len min available
-          Array.copy(file.data, position, b, off, n)
+          file.data.view(position, file.data.length).copyToArray(b, off, n)
           position += n
           n
         }
@@ -304,27 +323,20 @@ class VirtualFileSystem extends OutputStreamFileSystem {
       else throw new io.IOException("file is closed")
   }
 
-  private class VirtualFileOutputStream(val file: VirtualFile) extends io.ByteArrayOutputStream {
+  private class VirtualFileOutputStream(val file: VirtualFile) extends io.OutputStream {
     private var isClosed = false
 
     override def close() = if (!isClosed) {
-      flush()
       isClosed = true
       file.stream = None
     }
 
-    override def flush() =
-      if (!isClosed) {
-        file.data ++= toByteArray
-        reset()
-      } else throw new io.IOException("file is closed")
-
     override def write(b: Array[Byte], off: Int, len: Int) =
-      if (!isClosed) super.write(b, off, len)
+      if (!isClosed) file.data ++= b.view(off, off + len)
       else throw new io.IOException("file is closed")
 
     override def write(b: Int) =
-      if (!isClosed) super.write(b)
+      if (!isClosed) file.data += b.toByte
       else throw new io.IOException("file is closed")
   }