Rework Internet Card filtering system.

This commit is contained in:
Adrian Siekierka 2023-06-24 18:23:45 +02:00
parent de8f207f3b
commit c30f083072
9 changed files with 455 additions and 72 deletions

View File

@ -0,0 +1,27 @@
package com.typesafe.config.impl;
import com.typesafe.config.Config;
import com.typesafe.config.ConfigValue;
import org.luaj.vm2.ast.Str;
import java.util.List;
public final class OpenComputersConfigCommentManipulationHook {
private OpenComputersConfigCommentManipulationHook() {
}
public static Config setComments(Config config, String path, List<String> comments) {
return config.withValue(path, setComments(config.getValue(path), comments));
}
public static ConfigValue setComments(ConfigValue value, List<String> comments) {
if (value.origin() instanceof SimpleConfigOrigin && value instanceof AbstractConfigValue) {
return ((AbstractConfigValue) value).withOrigin(
((SimpleConfigOrigin) value.origin()).setComments(comments)
);
} else {
return value;
}
}
}

View File

@ -0,0 +1,64 @@
package li.cil.oc.util;
import com.google.common.net.InetAddresses;
import java.net.InetAddress;
// Originally by SquidDev
public final class InetAddressRange {
private final byte[] min;
private final byte[] max;
InetAddressRange(byte[] min, byte[] max) {
this.min = min;
this.max = max;
}
public boolean matches(InetAddress address) {
byte[] entry = address.getAddress();
if (entry.length != min.length) return false;
for (int i = 0; i < entry.length; i++) {
int value = 0xFF & entry[i];
if (value < (0xFF & min[i]) || value > (0xFF & max[i])) return false;
}
return true;
}
public static InetAddressRange parse(String addressStr, String prefixSizeStr) {
int prefixSize;
try {
prefixSize = Integer.parseInt(prefixSizeStr);
} catch (NumberFormatException e) {
throw new IllegalArgumentException(String.format("Malformed address range entry '%s': Cannot extract size of CIDR mask from '%s'.",
addressStr + '/' + prefixSizeStr, prefixSizeStr));
}
InetAddress address;
try {
address = InetAddresses.forString(addressStr);
} catch (IllegalArgumentException e) {
throw new IllegalArgumentException(String.format("Malformed address range entry '%s': Cannot extract IP address from '%s'.",
addressStr + '/' + prefixSizeStr, prefixSizeStr));
}
// Mask the bytes of the IP address.
byte[] minBytes = address.getAddress(), maxBytes = address.getAddress();
int size = prefixSize;
for (int i = 0; i < minBytes.length; i++) {
if (size <= 0) {
minBytes[i] = (byte) 0;
maxBytes[i] = (byte) 0xFF;
} else if (size < 8) {
minBytes[i] = (byte) (minBytes[i] & 0xFF << (8 - size));
maxBytes[i] = (byte) (maxBytes[i] | ~(0xFF << (8 - size)));
}
size -= 8;
}
return new InetAddressRange(minBytes, maxBytes);
}
}

View File

@ -963,37 +963,44 @@ opencomputers {
# the `connect` method on internet card components becomes available.
enableTcp: true
# This is a list of forbidden domain names. If an HTTP request is made
# or a socket connection is opened the target address will be compared
# to the addresses / address ranges in this list. It it is present in this
# list, the request will be denied.
# Entries are either domain names (www.example.com) or IP addresses in
# string format (10.0.0.3), optionally in CIDR notation to make it easier
# to define address ranges (1.0.0.0/8). Domains are resolved to their
# actual IP once on startup, future requests are resolved and compared
# to the resolved addresses.
# By default all local addresses are blocked. This is only meant as a
# This is a list of filtering rules. For any HTTP request or TCP socket
# connection, the target address will be processed by each rule, starting
# from first to last. The first matching rule will be applied; if no rule
# contains a match, the request will be denied.
# Two types of rules are currently supported: "allow", which allows an
# address to be accessed, and "deny", which forbids such access.
# Rules can be suffixed with additional filters to limit their scope:
# - all: apply to all addresses
# - default: apply built-in allow/deny rules; these may not be up to date,
# so one should primarily rely on them as a fallback
# - private: apply to all private addresses
# - bogon: apply to all known bogon addresses
# - ipv4: apply to all IPv4 addresses
# - ipv6: apply to all IPv6 addresses
# - ipv4-embedded-ipv6: apply to all IPv4 addresses embedded in IPv6
# addresses
# - ip:[address]: apply to this IP address in string format (10.0.0.3).
# CIDR notation is supported and allows defining address ranges
# (1.0.0.0/8).
# - domain:[domain]: apply to this domain. Domains are resolved to their
# actual IP only once (on startup), future requests are resolved and
# compared to the resolved addresses. Wildcards are not supported.
# The "removeme" rule does not have any use, but is instead present to
# detect whether to emit a warning on dedicated server configurations.
# Modpack authors are asked not to remove this rule; server administrators
# are free to remove it once the filtering rules have been adjusted.
# By default all private addresses are blocked. This is only meant as a
# thin layer of security, to avoid average users hosting a game on their
# local machine having players access services in their local network.
# Server hosters are expected to configure their network outside of the
# mod's context in an appropriate manner, e.g. using a system firewall.
blacklist: [
"127.0.0.0/8"
"0.0.0.0/8"
"10.0.0.0/8"
"192.168.0.0/16"
"172.16.0.0/12"
filteringRules: [
"removeme",
"deny private",
"deny bogon",
"allow default"
]
# This is a list of allowed domain names. Requests may only be made
# to addresses that are present in this list. If this list is empty,
# requests may be made to all addresses not forbidden. Note that the
# blacklist is always applied, so if an entry is present in both the
# whitelist and the blacklist, the blacklist will win.
# Entries are of the same format as in the blacklist. Examples:
# "gist.github.com", "www.pastebin.com"
whitelist: []
# The time in seconds to wait for a response to a request before timing
# out and returning an error message. If this is zero (the default) the
# request will never time out.

View File

@ -60,6 +60,20 @@ object OpenComputers {
def serverStart(e: FMLServerStartingEvent): Unit = {
CommandHandler.register(e)
ThreadPoolFactory.safePools.foreach(_.newThreadPool())
if (e.getServer.isDedicatedServer) {
if ((Settings.get.httpEnabled || Settings.get.tcpEnabled) && !Settings.get.internetFilteringRulesObserved) {
OpenComputers.log.warn("####################################################")
OpenComputers.log.warn("# #")
OpenComputers.log.warn("# It appears that you're running a dedicated #")
OpenComputers.log.warn("# server with OpenComputers installed! Make sure #")
OpenComputers.log.warn("# to review the Internet Card address filtering #")
OpenComputers.log.warn("# list to ensure it is appropriately configured. #")
OpenComputers.log.warn("# (config/OpenComputers.cfg => filteringRules) #")
OpenComputers.log.warn("# #")
OpenComputers.log.warn("####################################################")
}
}
}
@EventHandler

View File

@ -1,28 +1,28 @@
package li.cil.oc
import java.io._
import java.net.Inet4Address
import java.net.InetAddress
import java.nio.charset.StandardCharsets
import java.security.SecureRandom
import java.util.UUID
import com.google.common.net.InetAddresses
import com.mojang.authlib.GameProfile
import com.typesafe.config._
import com.typesafe.config.impl.OpenComputersConfigCommentManipulationHook
import cpw.mods.fml.common.Loader
import cpw.mods.fml.common.versioning.{DefaultArtifactVersion, VersionRange}
import li.cil.oc.Settings.DebugCardAccess
import li.cil.oc.common.Tier
import li.cil.oc.integration.Mods
import li.cil.oc.server.component.DebugCard
import li.cil.oc.server.component.DebugCard.AccessContext
import li.cil.oc.util.{InetAddressRange, InternetFilteringRule}
import org.apache.commons.codec.binary.Hex
import org.apache.commons.lang3.StringEscapeUtils
import java.io._
import java.net.{Inet4Address, Inet6Address, InetAddress}
import java.nio.charset.StandardCharsets
import java.security.SecureRandom
import java.util.UUID
import scala.collection.JavaConverters._
import scala.collection.convert.WrapAsScala._
import scala.collection.mutable
import scala.io.Codec
import scala.io.Source
import scala.io.{Codec, Source}
import scala.util.matching.Regex
class Settings(val config: Config) {
@ -299,8 +299,11 @@ class Settings(val config: Config) {
val httpEnabled = config.getBoolean("internet.enableHttp")
val httpHeadersEnabled = config.getBoolean("internet.enableHttpHeaders")
val tcpEnabled = config.getBoolean("internet.enableTcp")
val httpHostBlacklist = Array(config.getStringList("internet.blacklist").map(new Settings.AddressValidator(_)): _*)
val httpHostWhitelist = Array(config.getStringList("internet.whitelist").map(new Settings.AddressValidator(_)): _*)
val internetFilteringRules = Array(config.getStringList("internet.filteringRules")
.filter(p => !p.equals("removeme"))
.map(new InternetFilteringRule(_)): _*)
val internetFilteringRulesObserved = !config.getStringList("internet.filteringRules")
.contains("removeme")
val httpTimeout = (config.getInt("internet.requestTimeout") max 0) * 1000
val maxConnections = config.getInt("internet.maxTcpConnections") max 0
val internetThreads = config.getInt("internet.threads") max 1
@ -581,6 +584,7 @@ object Settings {
"computer.robot.limitFlightHeight"
)
)
private val fileringRulesPatchVersion = VersionRange.createFromVersionSpec("[0.0, 1.8.3)")
// Checks the config version (i.e. the version of the mod the config was
// created by) against the current version to see if some hard changes
@ -596,7 +600,7 @@ object Settings {
for ((version, paths) <- configPatches if version.containsVersion(configVersion)) {
for (path <- paths) {
val fullPath = prefix + path
OpenComputers.log.info(s"Updating setting '$fullPath'. ")
OpenComputers.log.info(s"=> Updating setting '$fullPath'. ")
if (defaults.hasPath(fullPath)) {
patched = patched.withValue(fullPath, defaults.getValue(fullPath))
}
@ -605,37 +609,55 @@ object Settings {
}
}
}
// Migrate filtering rules to 1.8.3+
if (fileringRulesPatchVersion.containsVersion(configVersion)) {
OpenComputers.log.info(s"=> Migrating Internet Card filtering rules. ")
val cidrPattern = """(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})(?:/(\d{1,2}))""".r
val httpHostWhitelist = patched.getStringList(prefix + "internet.whitelist")
val httpHostBlacklist = patched.getStringList(prefix + "internet.blacklist")
val internetFilteringRules = mutable.MutableList[String]()
for (blockedAddress <- httpHostBlacklist) {
if (cidrPattern.findFirstIn(blockedAddress).isDefined) {
internetFilteringRules += "deny ip:" + blockedAddress
} else {
internetFilteringRules += "deny domain:" + blockedAddress
}
}
for (allowedAddress <- httpHostWhitelist) {
if (cidrPattern.findFirstIn(allowedAddress).isDefined) {
internetFilteringRules += "allow ip:" + allowedAddress
} else {
internetFilteringRules += "allow domain:" + allowedAddress
}
}
if (!httpHostWhitelist.isEmpty) {
internetFilteringRules += "deny all"
}
for (defaultRule <- defaults.getStringList(prefix + "internet.filteringRules")) {
internetFilteringRules += defaultRule
}
var patchedRules: ConfigValue = ConfigValueFactory.fromIterable(internetFilteringRules.asJava)
// We need to use the private API here, unfortunately.
try {
patched = OpenComputersConfigCommentManipulationHook.setComments(
patched, prefix + "internet.whitelist", List("No longer used! See internet.filteringRules.").asJava
)
patched = OpenComputersConfigCommentManipulationHook.setComments(
patched, prefix + "internet.blacklist", List("No longer used! See internet.filteringRules.").asJava
)
patchedRules = OpenComputersConfigCommentManipulationHook.setComments(
patchedRules, defaults.getValue(prefix + "internet.filteringRules").origin().comments()
)
} catch {
case _: Throwable => /* pass */
}
patched = patched.withValue(prefix + "internet.filteringRules", patchedRules)
}
}
patched
}
val cidrPattern = """(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})(?:/(\d{1,2}))""".r
class AddressValidator(val value: String) {
val validator: (InetAddress, String) => Option[Boolean] = try cidrPattern.findFirstIn(value) match {
case Some(cidrPattern(address, prefix)) =>
val addr = InetAddresses.coerceToInteger(InetAddresses.forString(address))
val mask = 0xFFFFFFFF << (32 - prefix.toInt)
val min = addr & mask
val max = min | ~mask
(inetAddress: InetAddress, host: String) => Some(inetAddress match {
case v4: Inet4Address =>
val numeric = InetAddresses.coerceToInteger(v4)
min <= numeric && numeric <= max
case _ => true // Can't check IPv6 addresses so we pass them.
})
case _ =>
val address = InetAddress.getByName(value)
(inetAddress: InetAddress, host: String) => Some(host == value || inetAddress == address)
} catch {
case t: Throwable =>
OpenComputers.log.warn("Invalid entry in internet blacklist / whitelist: " + value, t)
(inetAddress: InetAddress, host: String) => None
}
def apply(inetAddress: InetAddress, host: String) = validator(inetAddress, host)
}
sealed trait DebugCardAccess {
def checkAccess(ctx: Option[DebugCard.AccessContext]): Option[String]
}

View File

@ -1,5 +1,7 @@
package li.cil.oc.server.component
import com.google.common.net.InetAddresses
import java.io.BufferedWriter
import java.io.FileNotFoundException
import java.io.IOException
@ -13,7 +15,6 @@ import java.nio.channels.SocketChannel
import java.util
import java.util.UUID
import java.util.concurrent._
import li.cil.oc.Constants
import li.cil.oc.OpenComputers
import li.cil.oc.Settings
@ -179,7 +180,11 @@ class InternetCard extends prefab.ManagedEnvironment with DeviceInfo {
}
object InternetCard {
private val threadPool = ThreadPoolFactory.create("Internet", Settings.get.internetThreads)
// For InternetFilteringRuleTest, where Settings.get is not provided.
private val threadPool = ThreadPoolFactory.create("Internet", Option(Settings.get) match {
case None => 1
case Some(settings) => settings.internetThreads
})
trait Closable {
def close(): Unit
@ -355,12 +360,36 @@ object InternetCard {
}
def checkLists(inetAddress: InetAddress, host: String) {
if (Settings.get.httpHostWhitelist.length > 0 && !Settings.get.httpHostWhitelist.exists(i => i.apply(inetAddress, host).getOrElse(false))) {
throw new FileNotFoundException("address is not whitelisted")
def isRequestAllowed(settings: Settings, inetAddress: InetAddress, host: String): Boolean = {
val rules = settings.internetFilteringRules
inetAddress match {
// IPv6 handling
case inet6Address: Inet6Address =>
// If the IP address is an IPv6 address with an embedded IPv4 address, and the IPv4 address is blocked,
// block this request.
if (InetAddresses.hasEmbeddedIPv4ClientAddress(inet6Address)) {
val inet4in6Address = InetAddresses.getEmbeddedIPv4ClientAddress(inet6Address)
if (!rules.map(r => r.apply(inet4in6Address, host)).collectFirst({ case Some(r) => r }).getOrElse(true)) {
return false
}
}
// Process address as an IPv6 address.
rules.map(r => r.apply(inet6Address, host)).collectFirst({ case Some(r) => r }).getOrElse(false)
// IPv4 handling
case inet4Address: Inet4Address =>
// Process address as an IPv4 address.
rules.map(r => r.apply(inet4Address, host)).collectFirst({ case Some(r) => r }).getOrElse(false)
case _ =>
// Unrecognized address type - block.
OpenComputers.log.warn("Internet Card blocked unrecognized address type: " + inetAddress.toString)
false
}
if (Settings.get.httpHostBlacklist.length > 0 && Settings.get.httpHostBlacklist.exists(i => i.apply(inetAddress, host).getOrElse(true))) {
throw new FileNotFoundException("address is blacklisted")
}
def checkLists(inetAddress: InetAddress, host: String): Unit = {
if (!isRequestAllowed(Settings.get, inetAddress, host)) {
throw new FileNotFoundException("address is not allowed")
}
}

View File

@ -0,0 +1,121 @@
package li.cil.oc.util
import com.google.common.net.InetAddresses
import li.cil.oc.OpenComputers
import java.net.{Inet4Address, Inet6Address, InetAddress}
import scala.collection.mutable
class InternetFilteringRule(val ruleString: String) {
val validator: (InetAddress, String) => Option[Boolean] = {
try {
val ruleParts = ruleString.split(' ')
ruleParts.head match {
case "allow" | "deny" =>
val value = ruleParts.head.equals("allow")
val predicates = mutable.MutableList[(InetAddress, String) => Boolean]()
ruleParts.tail.foreach(f => {
val filter = f.split(":", 2)
filter.head match {
case "default" =>
if (!value) {
predicates += ((_: InetAddress, _: String) => { false })
} else {
predicates += ((inetAddress: InetAddress, host: String) => {
InternetFilteringRule.defaultRules.map(r => r.apply(inetAddress, host)).collectFirst({ case Some(r) => r }).getOrElse(false)
})
}
case "private" =>
predicates += ((inetAddress: InetAddress, _: String) => {
inetAddress.isAnyLocalAddress || inetAddress.isLoopbackAddress || inetAddress.isLinkLocalAddress || inetAddress.isSiteLocalAddress
})
case "bogon" =>
predicates += ((inetAddress: InetAddress, _: String) => {
InternetFilteringRule.bogonMatchingRules.exists(rule => rule.matches(inetAddress))
})
case "ipv4" =>
predicates += ((inetAddress: InetAddress, _: String) => {
inetAddress.isInstanceOf[Inet4Address]
})
case "ipv6" =>
predicates += ((inetAddress: InetAddress, _: String) => {
inetAddress.isInstanceOf[Inet6Address]
})
case "ipv4-embedded-ipv6" =>
predicates += ((inetAddress: InetAddress, _: String) => {
inetAddress.isInstanceOf[Inet6Address] && InetAddresses.hasEmbeddedIPv4ClientAddress(inetAddress.asInstanceOf[Inet6Address])
})
case "domain" =>
val domain = filter(1)
val addresses = InetAddress.getAllByName(domain)
predicates += ((inetAddress: InetAddress, host: String) => {
host == domain || addresses.exists(a => a.equals(inetAddress))
})
case "ip" =>
val ipStringParts = f.split("/", 2)
if (ipStringParts.length == 2) {
val ipRange = InetAddressRange.parse(ipStringParts(0), ipStringParts(1))
predicates += ((inetAddress: InetAddress, _: String) => ipRange.matches(inetAddress))
} else {
val ipAddress = InetAddresses.forString(ipStringParts(0))
predicates += ((inetAddress: InetAddress, _: String) => ipAddress.equals(inetAddress))
}
predicates += ((inetAddress: InetAddress, _: String) => {
inetAddress.isAnyLocalAddress || inetAddress.isLoopbackAddress || inetAddress.isLinkLocalAddress || inetAddress.isSiteLocalAddress
})
case "all" =>
}
})
(inetAddress: InetAddress, host: String) => {
if (predicates.forall(p => p(inetAddress, host)))
Some(value)
else
None
}
case "removeme" =>
// Ignore this rule.
(_: InetAddress, _: String) => None
}
} catch {
case t: Throwable =>
OpenComputers.log.warn("Invalid internet filtering rule in configuration: " + ruleString, t)
(_: InetAddress, _: String) => None
}
}
def apply(inetAddress: InetAddress, host: String) = validator(inetAddress, host)
}
object InternetFilteringRule {
private val defaultRules = Array(
new InternetFilteringRule("deny private"),
new InternetFilteringRule("deny bogon"),
new InternetFilteringRule("allow all")
)
private val bogonMatchingRules = Array(
"0.0.0.0/8",
"10.0.0.0/8",
"100.64.0.0/10",
"127.0.0.0/8",
"169.254.0.0/16",
"172.16.0.0/12",
"192.0.0.0/24",
"192.0.2.0/24",
"192.168.0.0/16",
"198.18.0.0/15",
"198.51.100.0/24",
"203.0.113.0/24",
"224.0.0.0/3",
"::/128",
"::1/128",
"::ffff:0:0/96",
"::/96",
"100::/64",
"2001:10::/28",
"2001:db8::/32",
"fc00::/7",
"fe80::/10",
"fec0::/10",
"ff00::/8"
).map(s => s.split("/", 2)).map(s => InetAddressRange.parse(s(0), s(1)))
}

View File

@ -14,7 +14,11 @@ import scala.collection.mutable
object ThreadPoolFactory {
val priority = {
val custom = Settings.get.threadPriority
// For InternetFilteringRuleTest, where Settings.get is not provided.
val custom = Option(Settings.get) match {
case None => -1
case Some(settings) => settings.threadPriority
}
if (custom < 1) Thread.MIN_PRIORITY + (Thread.NORM_PRIORITY - Thread.MIN_PRIORITY) / 2
else custom max Thread.MIN_PRIORITY min Thread.MAX_PRIORITY
}

View File

@ -0,0 +1,95 @@
import com.typesafe.config.ConfigFactory
import li.cil.oc.Settings
import li.cil.oc.server.component.InternetCard
import org.junit.runner.RunWith
import org.scalatest.{FlatSpec, FunSpec, WordSpec}
import org.scalatest.Matchers.{be, convertToAnyShouldWrapper}
import org.scalatest.junit.JUnitRunner
import org.scalatest.mock.MockitoSugar
import java.net.InetAddress
import scala.compat.Platform.EOL
import scala.io.{Codec, Source}
@RunWith(classOf[JUnitRunner])
class InternetFilteringRuleTest extends FunSpec with MockitoSugar {
val config = autoClose(classOf[Settings].getResourceAsStream("/application.conf")) { in =>
val configStr = Source.fromInputStream(in)(Codec.UTF8).getLines().mkString("", EOL, EOL)
ConfigFactory.parseString(configStr)
}
val settings = new Settings(config.getConfig("opencomputers"))
describe("The default AddressValidators") {
// Many of these payloads are pulled from PayloadsAllTheThings
// https://github.com/swisskyrepo/PayloadsAllTheThings/blob/master/Server%20Side%20Request%20Forgery/README.md
it("should accept a valid external address") {
isUriBlacklisted("https://google.com") should be(false)
}
it("should reject localhost") {
isUriBlacklisted("http://localhost") should be(true)
}
it("should reject the local host in IPv4 format") {
isUriBlacklisted("http://127.0.0.1") should be(true)
isUriBlacklisted("http://127.0.1") should be(true)
isUriBlacklisted("http://127.1") should be(true)
isUriBlacklisted("http://0") should be (true)
}
it("should reject the local host in IPv6") {
isUriBlacklisted("http://[::1]") should be(true)
isUriBlacklisted("http://[::]") should be(true)
}
it("should reject IPv6/IPv4 Address Embedding") {
isUriBlacklisted("http://[0:0:0:0:0:ffff:127.0.0.1]") should be(true)
isUriBlacklisted("http://[::ffff:127.0.0.1]") should be(true)
}
it("should reject an attempt to bypass using a decimal IP location") {
isUriBlacklisted("http://2130706433") should be(true) // 127.0.0.1
isUriBlacklisted("http://3232235521") should be(true) // 192.168.0.1
isUriBlacklisted("http://3232235777") should be(true) // 192.168.1.1
}
it("should reject the IMDS address in IPv4 format") {
isUriBlacklisted("http://169.254.169.254") should be(true)
isUriBlacklisted("http://2852039166") should be(true) // 169.254.169.254
}
it("should reject the IMDS address in IPv6 format") {
isUriBlacklisted("http://[fd00:ec2::254]") should be(true)
}
it("should reject the IMDS in for Oracle Cloud") {
isUriBlacklisted("http://192.0.0.192") should be(true)
}
it("should reject the IMDS in for Alibaba Cloud") {
isUriBlacklisted("http://100.100.100.200") should be(true)
}
}
def isUriBlacklisted(uri: String): Boolean = {
val uriObj = new java.net.URI(uri)
val resolved = InetAddress.getByName(uriObj.getHost)
!InternetCard.isRequestAllowed(settings, resolved, uriObj.getHost)
}
def autoClose[A <: AutoCloseable, B](closeable: A)(fun: (A) B): B = {
var t: Throwable = null
try {
fun(closeable)
} catch {
case funT: Throwable
t = funT
throw t
} finally {
if (t != null) {
try {
closeable.close()
} catch {
case closeT: Throwable
t.addSuppressed(closeT)
throw t
}
} else {
closeable.close()
}
}
}
}