scripting API for IP bans

See discussion on #68.

conventional.yaml

             #     max-concurrent-connections: 128
             #     max-connections-per-window: 1024
 
+    # pluggable IP ban mechanism, via subprocess invocation
+    # this can be used to check new connections against a DNSBL, for example
+    # see the manual for details on how to write an IP ban checking script
+    ip-check-script:
+        enabled: false
+        command: "/usr/local/bin/check-ip-ban"
+        # constant list of args to pass to the command; the actual query
+        # and result are transmitted over stdin/stdout:
+        args: []
+        # timeout for process execution, after which we send a SIGTERM:
+        timeout: 9s
+        # how long after the SIGTERM before we follow up with a SIGKILL:
+        kill-timeout: 1s
+        # how many scripts are allowed to run at once? 0 for no limit:
+        max-concurrency: 64
+
     # IP cloaking hides users' IP addresses from other users and from channel admins
     # (but not from server admins), while still allowing channel admins to ban
     # offending IP addresses or networks. In place of hostnames derived from reverse
         timeout: 9s
         # how long after the SIGTERM before we follow up with a SIGKILL:
         kill-timeout: 1s
+        # how many scripts are allowed to run at once? 0 for no limit:
+        max-concurrency: 64
 
 # channel options
 channels:

default.yaml

             #     max-concurrent-connections: 128
             #     max-connections-per-window: 1024
 
+    # pluggable IP ban mechanism, via subprocess invocation
+    # this can be used to check new connections against a DNSBL, for example
+    # see the manual for details on how to write an IP ban checking script
+    ip-check-script:
+        enabled: false
+        command: "/usr/local/bin/check-ip-ban"
+        # constant list of args to pass to the command; the actual query
+        # and result are transmitted over stdin/stdout:
+        args: []
+        # timeout for process execution, after which we send a SIGTERM:
+        timeout: 9s
+        # how long after the SIGTERM before we follow up with a SIGKILL:
+        kill-timeout: 1s
+        # how many scripts are allowed to run at once? 0 for no limit:
+        max-concurrency: 64
+
     # IP cloaking hides users' IP addresses from other users and from channel admins
     # (but not from server admins), while still allowing channel admins to ban
     # offending IP addresses or networks. In place of hostnames derived from reverse
         timeout: 9s
         # how long after the SIGTERM before we follow up with a SIGKILL:
         kill-timeout: 1s
+        # how many scripts are allowed to run at once? 0 for no limit:
+        max-concurrency: 64
 
 # channel options
 channels:

irc/accounts.go

 	config := am.server.Config()
 	if config.Accounts.AuthScript.Enabled {
 		var output AuthScriptOutput
-		output, err = CheckAuthScript(config.Accounts.AuthScript,
+		output, err = CheckAuthScript(am.server.semaphores.AuthScript, config.Accounts.AuthScript.ScriptConfig,
 			AuthScriptInput{AccountName: accountName, Passphrase: passphrase, IP: client.IP().String()})
 		if err != nil {
 			am.server.logger.Error("internal", "failed shell auth invocation", err.Error())
 	config := am.server.Config()
 	if config.Accounts.AuthScript.Enabled {
 		var output AuthScriptOutput
-		output, err = CheckAuthScript(config.Accounts.AuthScript,
+		output, err = CheckAuthScript(am.server.semaphores.AuthScript, config.Accounts.AuthScript.ScriptConfig,
 			AuthScriptInput{Certfp: certfp, IP: client.IP().String()})
 		if err != nil {
 			am.server.logger.Error("internal", "failed shell auth invocation", err.Error())

irc/authscript.go

 package irc
 
 import (
-	"bufio"
 	"encoding/json"
 	"fmt"
-	"io"
-	"os/exec"
-	"syscall"
-	"time"
+	"net"
+
+	"github.com/oragono/oragono/irc/utils"
 )
 
 // JSON-serializable input and output types for the script
 	Error       string `json:"error"`
 }
 
-// internal tupling of output and error for passing over a channel
-type authScriptResponse struct {
-	output AuthScriptOutput
-	err    error
-}
+func CheckAuthScript(sem utils.Semaphore, config ScriptConfig, input AuthScriptInput) (output AuthScriptOutput, err error) {
+	if sem != nil {
+		sem.Acquire()
+		defer sem.Release()
+	}
 
-func CheckAuthScript(config AuthScriptConfig, input AuthScriptInput) (output AuthScriptOutput, err error) {
 	inputBytes, err := json.Marshal(input)
 	if err != nil {
 		return
 	}
-	cmd := exec.Command(config.Command, config.Args...)
-	stdin, err := cmd.StdinPipe()
+	outBytes, err := RunScript(config.Command, config.Args, inputBytes, config.Timeout, config.KillTimeout)
 	if err != nil {
 		return
 	}
-	stdout, err := cmd.StdoutPipe()
+	err = json.Unmarshal(outBytes, &output)
 	if err != nil {
 		return
 	}
 
-	channel := make(chan authScriptResponse, 1)
-	err = cmd.Start()
-	if err != nil {
-		return
-	}
-	stdin.Write(inputBytes)
-	stdin.Write([]byte{'\n'})
-
-	// lots of potential race conditions here. we want to ensure that Wait()
-	// will be called, and will return, on the other goroutine, no matter
-	// where it is blocked. If it's blocked on ReadBytes(), we will kill it
-	// (first with SIGTERM, then with SIGKILL) and ReadBytes will return
-	// with EOF. If it's blocked on Wait(), then one of the kill signals
-	// will succeed and unblock it.
-	go processAuthScriptOutput(cmd, stdout, channel)
-	outputTimer := time.NewTimer(config.Timeout)
-	select {
-	case response := <-channel:
-		return response.output, response.err
-	case <-outputTimer.C:
+	if output.Error != "" {
+		err = fmt.Errorf("Authentication process reported error: %s", output.Error)
 	}
+	return
+}
 
-	err = errTimedOut
-	cmd.Process.Signal(syscall.SIGTERM)
-	termTimer := time.NewTimer(config.Timeout)
-	select {
-	case <-channel:
-		return
-	case <-termTimer.C:
-	}
+type IPScriptResult uint
 
-	cmd.Process.Kill()
-	return
+const (
+	IPNotChecked  IPScriptResult = 0
+	IPAccepted    IPScriptResult = 1
+	IPBanned      IPScriptResult = 2
+	IPRequireSASL IPScriptResult = 3
+)
+
+type IPScriptInput struct {
+	IP string `json:"ip"`
 }
 
-func processAuthScriptOutput(cmd *exec.Cmd, stdout io.Reader, channel chan authScriptResponse) {
-	var response authScriptResponse
-	var out AuthScriptOutput
-
-	reader := bufio.NewReader(stdout)
-	outBytes, err := reader.ReadBytes('\n')
-	if err == nil {
-		err = json.Unmarshal(outBytes, &out)
-		if err == nil {
-			response.output = out
-			if out.Error != "" {
-				err = fmt.Errorf("Authentication process reported error: %s", out.Error)
-			}
-		}
+type IPScriptOutput struct {
+	Result     IPScriptResult `json:"result"`
+	BanMessage string         `json:"banMessage"`
+	// for caching: the network to which this result is applicable, and a TTL in seconds:
+	CacheNet     string `json:"cacheNet"`
+	CacheSeconds int    `json:"cacheSeconds"`
+	Error        string `json:"error"`
+}
+
+func CheckIPBan(sem utils.Semaphore, config ScriptConfig, addr net.IP) (output IPScriptOutput, err error) {
+	if sem != nil {
+		sem.Acquire()
+		defer sem.Release()
 	}
-	response.err = err
 
-	// always call Wait() to ensure resource cleanup
-	err = cmd.Wait()
+	inputBytes, err := json.Marshal(IPScriptInput{IP: addr.String()})
+	if err != nil {
+		return
+	}
+	outBytes, err := RunScript(config.Command, config.Args, inputBytes, config.Timeout, config.KillTimeout)
+	if err != nil {
+		return
+	}
+	err = json.Unmarshal(outBytes, &output)
 	if err != nil {
-		response.err = err
+		return
 	}
 
-	channel <- response
+	if output.Error != "" {
+		err = fmt.Errorf("IP ban process reported error: %s", output.Error)
+	} else if !(IPAccepted <= output.Result && output.Result <= IPRequireSASL) {
+		err = fmt.Errorf("Invalid result from IP checking script: %d", output.Result)
+	}
+
+	return
 }

irc/client.go

 	cloakedHostname    string
 	realname           string
 	realIP             net.IP
+	requireSASL        bool
 	registered         bool
 	registrationTimer  *time.Timer
 	resumeID           string
 
 // RunClient sets up a new client and runs its goroutine.
 func (server *Server) RunClient(conn IRCConn) {
+	config := server.Config()
 	wConn := conn.UnderlyingConn()
-	var isBanned bool
+	var isBanned, requireSASL bool
 	var banMsg string
 	realIP := utils.AddrToIP(wConn.RemoteAddr())
 	var proxiedIP net.IP
 			proxiedIP = wConn.ProxiedIP
 			ipToCheck = proxiedIP
 		}
-		isBanned, banMsg = server.checkBans(ipToCheck)
+		// XXX only run the check script now if the IP cannot be replaced by PROXY or WEBIRC,
+		// otherwise we'll do it in ApplyProxiedIP.
+		checkScripts := proxiedIP != nil || !utils.IPInNets(realIP, config.Server.proxyAllowedFromNets)
+		isBanned, requireSASL, banMsg = server.checkBans(config, ipToCheck, checkScripts)
 	}
 
 	if isBanned {
 	server.logger.Info("connect-ip", fmt.Sprintf("Client connecting: real IP %v, proxied IP %v", realIP, proxiedIP))
 
 	now := time.Now().UTC()
-	config := server.Config()
 	// give them 1k of grace over the limit:
 	socket := NewSocket(conn, config.Server.MaxSendQBytes)
 	client := &Client{
 		nickMaskString: "*", // * is used until actual nick is given
 		realIP:         realIP,
 		proxiedIP:      proxiedIP,
+		requireSASL:    requireSASL,
 	}
 	client.writerSemaphore.Initialize(1)
 	client.history.Initialize(config.History.ClientLength, time.Duration(config.History.AutoresizeWindow))
 	authFailSaslRequired
 )
 
-func (client *Client) isAuthorized(server *Server, config *Config, session *Session) AuthOutcome {
+func (client *Client) isAuthorized(server *Server, config *Config, session *Session, forceRequireSASL bool) AuthOutcome {
 	saslSent := client.account != ""
 	// PASS requirement
 	if (config.Server.passwordBytes != nil) && session.passStatus != serverPassSuccessful && !(config.Accounts.SkipServerPassword && saslSent) {
 		return authFailTorSaslRequired
 	}
 	// finally, enforce require-sasl
-	if !saslSent && (config.Accounts.RequireSasl.Enabled || server.Defcon() <= 2) &&
+	if !saslSent && (forceRequireSASL || config.Accounts.RequireSasl.Enabled || server.Defcon() <= 2) &&
 		!utils.IPInNets(session.IP(), config.Accounts.RequireSasl.exemptedNets) {
 		return authFailSaslRequired
 	}

irc/config.go

 	AuthScript  AuthScriptConfig `yaml:"auth-script"`
 }
 
+type ScriptConfig struct {
+	Enabled        bool
+	Command        string
+	Args           []string
+	Timeout        time.Duration
+	KillTimeout    time.Duration `yaml:"kill-timeout"`
+	MaxConcurrency uint          `yaml:"max-concurrency"`
+}
+
 type AuthScriptConfig struct {
-	Enabled     bool
-	Command     string
-	Args        []string
-	Autocreate  bool
-	Timeout     time.Duration
-	KillTimeout time.Duration `yaml:"kill-timeout"`
+	ScriptConfig `yaml:",inline"`
+	Autocreate   bool
 }
 
 // AccountRegistrationConfig controls account registration.
 		supportedCaps *caps.Set
 		capValues     caps.Values
 		Casemapping   Casemapping
-		EnforceUtf8   bool   `yaml:"enforce-utf8"`
-		OutputPath    string `yaml:"output-path"`
+		EnforceUtf8   bool         `yaml:"enforce-utf8"`
+		OutputPath    string       `yaml:"output-path"`
+		IPCheckScript ScriptConfig `yaml:"ip-check-script"`
 	}
 
 	Roleplay struct {

irc/gateways.go

 	}
 	proxiedIP = proxiedIP.To16()
 
-	isBanned, banMsg := client.server.checkBans(proxiedIP)
+	isBanned, requireSASL, banMsg := client.server.checkBans(client.server.Config(), proxiedIP, true)
 	if isBanned {
 		return errBanned, banMsg
 	}
+	client.requireSASL = requireSASL
 	// successfully added a limiter entry for the proxied IP;
 	// remove the entry for the real IP if applicable (#197)
 	client.server.connectionLimiter.RemoveClient(session.realIP)

irc/script.go

+// Copyright (c) 2020 Shivaram Lingamneni
+// released under the MIT license
+
+package irc
+
+import (
+	"bufio"
+	"io"
+	"os/exec"
+	"syscall"
+	"time"
+)
+
+// general-purpose scripting API for oragono "plugins"
+// invoke a command, send it a single newline-terminated string of bytes (typically JSON)
+// get back another newline-terminated string of bytes (or an error)
+
+// internal tupling of output and error for passing over a channel
+type scriptResponse struct {
+	output []byte
+	err    error
+}
+
+func RunScript(command string, args []string, input []byte, timeout, killTimeout time.Duration) (output []byte, err error) {
+	cmd := exec.Command(command, args...)
+	stdin, err := cmd.StdinPipe()
+	if err != nil {
+		return
+	}
+	stdout, err := cmd.StdoutPipe()
+	if err != nil {
+		return
+	}
+
+	channel := make(chan scriptResponse, 1)
+	err = cmd.Start()
+	if err != nil {
+		return
+	}
+	stdin.Write(input)
+	stdin.Write([]byte{'\n'})
+
+	// lots of potential race conditions here. we want to ensure that Wait()
+	// will be called, and will return, on the other goroutine, no matter
+	// where it is blocked. If it's blocked on ReadBytes(), we will kill it
+	// (first with SIGTERM, then with SIGKILL) and ReadBytes will return
+	// with EOF. If it's blocked on Wait(), then one of the kill signals
+	// will succeed and unblock it.
+	go processScriptOutput(cmd, stdout, channel)
+	outputTimer := time.NewTimer(timeout)
+	select {
+	case response := <-channel:
+		return response.output, response.err
+	case <-outputTimer.C:
+	}
+
+	err = errTimedOut
+	cmd.Process.Signal(syscall.SIGTERM)
+	termTimer := time.NewTimer(killTimeout)
+	select {
+	case <-channel:
+		return
+	case <-termTimer.C:
+	}
+
+	cmd.Process.Kill()
+	return
+}
+
+func processScriptOutput(cmd *exec.Cmd, stdout io.Reader, channel chan scriptResponse) {
+	var response scriptResponse
+
+	reader := bufio.NewReader(stdout)
+	response.output, response.err = reader.ReadBytes('\n')
+
+	// always call Wait() to ensure resource cleanup
+	err := cmd.Wait()
+	if err != nil {
+		response.err = err
+	}
+
+	channel <- response
+}

irc/semaphores.go

 	// each distinct operation MUST have its own semaphore;
 	// methods that acquire a semaphore MUST NOT call methods that acquire another
 	ClientDestroy utils.Semaphore
+	IPCheckScript utils.Semaphore
+	AuthScript    utils.Semaphore
 }
 
 // Initialize initializes a set of server semaphores.

irc/server.go

 	}
 }
 
-func (server *Server) checkBans(ipaddr net.IP) (banned bool, message string) {
+func (server *Server) checkBans(config *Config, ipaddr net.IP, checkScripts bool) (banned bool, requireSASL bool, message string) {
 	if server.Defcon() == 1 {
 		if !(ipaddr.IsLoopback() || utils.IPInNets(ipaddr, server.Config().Server.secureNets)) {
-			return true, "New connections to this server are temporarily restricted"
+			return true, false, "New connections to this server are temporarily restricted"
 		}
 	}
 
 	isBanned, info := server.dlines.CheckIP(ipaddr)
 	if isBanned {
 		server.logger.Info("connect-ip", fmt.Sprintf("Client from %v rejected by d-line", ipaddr))
-		return true, info.BanMessage("You are banned from this server (%s)")
+		return true, false, info.BanMessage("You are banned from this server (%s)")
 	}
 
 	// check connection limits
 	if err == connection_limits.ErrLimitExceeded {
 		// too many connections from one client, tell the client and close the connection
 		server.logger.Info("connect-ip", fmt.Sprintf("Client from %v rejected for connection limit", ipaddr))
-		return true, "Too many clients from your network"
+		return true, false, "Too many clients from your network"
 	} else if err == connection_limits.ErrThrottleExceeded {
-		duration := server.Config().Server.IPLimits.BanDuration
-		if duration == 0 {
-			return false, ""
+		duration := config.Server.IPLimits.BanDuration
+		if duration != 0 {
+			server.dlines.AddIP(ipaddr, duration, throttleMessage,
+				"Exceeded automated connection throttle", "auto.connection.throttler")
+			// they're DLINE'd for 15 minutes or whatever, so we can reset the connection throttle now,
+			// and once their temporary DLINE is finished they can fill up the throttler again
+			server.connectionLimiter.ResetThrottle(ipaddr)
 		}
-		server.dlines.AddIP(ipaddr, duration, throttleMessage, "Exceeded automated connection throttle", "auto.connection.throttler")
-		// they're DLINE'd for 15 minutes or whatever, so we can reset the connection throttle now,
-		// and once their temporary DLINE is finished they can fill up the throttler again
-		server.connectionLimiter.ResetThrottle(ipaddr)
-
-		// this might not show up properly on some clients, but our objective here is just to close it out before it has a load impact on us
 		server.logger.Info(
 			"connect-ip",
 			fmt.Sprintf("Client from %v exceeded connection throttle, d-lining for %v", ipaddr, duration))
-		return true, throttleMessage
+		return true, false, throttleMessage
 	} else if err != nil {
 		server.logger.Warning("internal", "unexpected ban result", err.Error())
 	}
 
-	return false, ""
+	if checkScripts && config.Server.IPCheckScript.Enabled {
+		output, err := CheckIPBan(server.semaphores.IPCheckScript, config.Server.IPCheckScript, ipaddr)
+		if err != nil {
+			server.logger.Error("internal", "couldn't check IP ban script", ipaddr.String(), err.Error())
+			return false, false, ""
+		}
+		// TODO: currently no way to cache results other than IPBanned
+		if output.Result == IPBanned && output.CacheSeconds != 0 {
+			network, err := utils.NormalizedNetFromString(output.CacheNet)
+			if err != nil {
+				server.logger.Error("internal", "invalid dline net from IP ban script", ipaddr.String(), output.CacheNet)
+			} else {
+				dlineDuration := time.Duration(output.CacheSeconds) * time.Second
+				err := server.dlines.AddNetwork(network, dlineDuration, output.BanMessage, "", "")
+				if err != nil {
+					server.logger.Error("internal", "couldn't set dline from IP ban script", ipaddr.String(), err.Error())
+				}
+			}
+		}
+		if output.Result == IPBanned {
+			// XXX roll back IP connection/throttling addition for the IP
+			server.connectionLimiter.RemoveClient(ipaddr)
+			server.logger.Info("connect-ip", "Rejected client due to ip-check-script", ipaddr.String())
+			return true, false, output.BanMessage
+		} else if output.Result == IPRequireSASL {
+			server.logger.Info("connect-ip", "Requiring SASL from client due to ip-check-script", ipaddr.String())
+			return false, true, ""
+		}
+	}
+
+	return false, false, ""
 }
 
 func (server *Server) checkTorLimits() (banned bool, message string) {
 		return // whether we succeeded or failed, either way `c` is not getting registered
 	}
 
+	// XXX PROXY or WEBIRC MUST be sent as the first line of the session;
+	// if we are here at all that means we have the final value of the IP
+	if session.rawHostname == "" {
+		session.client.lookupHostname(session, false)
+	}
+
 	// try to complete registration normally
 	// XXX(#1057) username can be filled in by an ident query without the client
 	// having sent USER: check for both username and realname to ensure they did
 	// client MUST send PASS if necessary, or authenticate with SASL if necessary,
 	// before completing the other registration commands
 	config := server.Config()
-	authOutcome := c.isAuthorized(server, config, session)
+	authOutcome := c.isAuthorized(server, config, session, c.requireSASL)
 	var quitMessage string
 	switch authOutcome {
 	case authFailPass:
 		return true
 	}
 
-	// we have the final value of the IP address: do the hostname lookup
-	// (nickmask will be set below once nickname assignment succeeds)
-	if session.rawHostname == "" {
-		session.client.lookupHostname(session, false)
-	}
-
 	rb := NewResponseBuffer(session)
 	nickError := performNickChange(server, c, c, session, c.preregNick, rb)
 	rb.Send(true)
 			return fmt.Errorf("Cannot enable or disable relaying after launching the server, rehash aborted")
 		} else if oldConfig.Server.Relaymsg.Separators != config.Server.Relaymsg.Separators {
 			return fmt.Errorf("Cannot change relaying separators after launching the server, rehash aborted")
+		} else if oldConfig.Server.IPCheckScript.MaxConcurrency != config.Server.IPCheckScript.MaxConcurrency ||
+			oldConfig.Accounts.AuthScript.MaxConcurrency != config.Accounts.AuthScript.MaxConcurrency {
+			return fmt.Errorf("Cannot change max-concurrency for scripts after launching the server, rehash aborted")
 		}
 	}
 
 	server.logger.Debug("server", "Regenerating HELP indexes for new languages")
 	server.helpIndexManager.GenerateIndices(config.languageManager)
 
+	if initial {
+		maxIPConc := int(config.Server.IPCheckScript.MaxConcurrency)
+		if maxIPConc != 0 {
+			server.semaphores.IPCheckScript.Initialize(maxIPConc)
+		}
+		maxAuthConc := int(config.Accounts.AuthScript.MaxConcurrency)
+		if maxAuthConc != 0 {
+			server.semaphores.AuthScript.Initialize(maxAuthConc)
+		}
+	}
+
 	if oldConfig != nil {
 		// if certain features were enabled by rehash, we need to load the corresponding data
 		// from the store