Optionally decode generic HTTP payloads

examples/irccat.json

@@ -8,7 +8,10 @@
     "tls_key": "",
     "tls_cert": "",
     "listeners": {
-      "generic": true,
+      "generic": {
+        "secret": "",
+        "strict": false
+      },
       "grafana": "#channel",
       "github": {
 	"secret": "my_secret",

httplistener/generic.go

@@ -2,17 +2,78 @@ package httplistener
 
 import (
 	"bytes"
+	"encoding/base64"
 	"fmt"
+	"io"
+	"net/http"
+	"strings"
+
 	"github.com/irccloud/irccat/dispatcher"
 	"github.com/spf13/viper"
-	"net/http"
 )
 
+func handleUrlEncodedPostForm(request *http.Request) string {
+	request.ParseForm()
+	parts := []string{}
+	for key, val := range request.PostForm {
+		if key != "" {
+			parts = append(parts, key)
+		}
+		for _, v := range val {
+			if v != "" {
+				parts = append(parts, v)
+			}
+		}
+	}
+	return strings.Join(parts, " ")
+}
+
+// handleMixed blindly concatenates all bodies in a mixed message.
+//
+// Headers are discarded. Binary data, including illegal IRC chars, are
+// retained as is. Quoted-printable and base64 encodings are recognized.
+func handleMixed(request *http.Request) (string, error) {
+	mr, err := request.MultipartReader()
+	if err != nil {
+		return "", err
+	}
+	parts := []string{}
+	for {
+		p, err := mr.NextPart()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return "", err
+		}
+		b, err := io.ReadAll(p)
+		if err != nil {
+			return "", err
+		}
+		if len(b) != 0 {
+			if p.Header.Get("content-transfer-encoding") == "base64" {
+				encoder := base64.StdEncoding
+				if decoded, err := encoder.DecodeString(string(b)); err != nil {
+					return "", err
+				} else if len(decoded) > 0 {
+					b = decoded
+				}
+			}
+			parts = append(parts, string(b))
+		}
+	}
+	return strings.Join(parts, " "), nil
+}
+
+var genericSender = dispatcher.Send
+
 // Examples of using curl to post to /send.
 //
 // echo "Hello, world" | curl -d @- http://irccat.example.com/send
 // echo "#test,@alice Hello, world" | curl -d @- http://irccat.example.com/send
 //
+// See httplistener/generic_tests.go for info on strict mode, which behaves
+// differently and is enabled by config option http.listeners.generic.strict
 func (hl *HTTPListener) genericHandler(w http.ResponseWriter, request *http.Request) {
 	if request.Method != "POST" {
 		http.NotFound(w, request)
@@ -30,14 +91,31 @@ func (hl *HTTPListener) genericHandler(w http.ResponseWriter, request *http.Requ
 			return
 		}
 	}
+	var message string
+	strict := viper.GetBool("http.listeners.generic.strict")
+	if strict {
+		contentType := request.Header.Get("Content-Type")
+		if strings.HasPrefix(contentType, "application/x-www-form-urlencoded") {
+			message = handleUrlEncodedPostForm(request)
+		} else if strings.HasPrefix(contentType, "multipart/") {
+			if msg, err := handleMixed(request); err == nil {
+				message = msg // otherwise message is "", which triggers 400
+			}
+		}
+	}
 
-	body := new(bytes.Buffer)
-	body.ReadFrom(request.Body)
-	message := body.String()
+	if message == "" {
+		body := new(bytes.Buffer)
+		body.ReadFrom(request.Body)
+		message = body.String()
+	}
 
 	if message == "" {
+		if strict {
+			http.Error(w, "Bad Request", http.StatusBadRequest)
+		}
 		log.Warningf("%s - No message body in POST request", request.RemoteAddr)
 		return
 	}
-	dispatcher.Send(hl.irc, message, log, request.RemoteAddr)
+	genericSender(hl.irc, message, log, request.RemoteAddr)
 }

httplistener/generic_test.go

@@ -0,0 +1,222 @@
+// Some clients refuse to send payloads that don't match the Content-Type. In
+// other words, trying to send "%BOLD hw" or "\x02 hw", as written, wouldn't
+// be allowed for Content-Type "application/x-www-form-urlencoded" without
+// first being encoded. But irccat by default doesn't do any decoding. Thus,
+// when faced with one of these problematic clients, specify config option
+// http.listeners.generic.strict to get the behavior shown here:
+//
+// mismatch
+//
+//  $ echo "%BOLDhw" | curl -d @- http://localhost/send
+//  400 Bad Request
+//
+// urlencoded
+//
+//  $ echo "%BOLDhw" | curl --data-urlencode  @- http://localhost/send
+//  200 OK
+//
+// urlencoded non-printable
+//
+//  $ printf "\x02hw" | curl --data-urlencode  @- http://localhost/send
+//  200 OK
+//
+// octetstream
+//
+//  $ echo "%BOLDhw" | curl --data-binary @- \
+//    -H 'Content-Type: application/octet-stream' http://localhost/send
+//  200 OK
+//
+// multipart quoted-printable
+//
+//  $ echo '%BOLDhw' | curl -F 'foo=@-;encoder=quoted-printable' \
+//    http://localhost/send
+//  200 OK
+//
+// multipart 8bit
+//
+//  $ echo '%BOLDhw' | curl -F 'foo=@-;encoder=8bit' http://localhost/send
+//  200 OK
+//
+// multipart base64
+//
+//  $ echo '%BOLDhw' | curl -F 'foo=@-;encoder=base64' http://localhost/send
+//  200 OK
+//
+// The gist is that when strict mode is active, popular encodings will work
+// while mismatches won't, even though they may still appear to at times.
+//
+package httplistener
+
+import (
+	"context"
+	"io"
+	"net"
+	"net/http"
+	"os"
+	"path"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/juju/loggo"
+	"github.com/spf13/viper"
+	irc "github.com/thoj/go-ircevent"
+)
+
+var genericTestListen = "localhost:18045"
+
+func genericTestStartHTTPServer(t *testing.T, endpoint string) {
+	hl := HTTPListener{
+		http: http.Server{Addr: genericTestListen},
+	}
+
+	http.HandleFunc(endpoint, hl.genericHandler)
+	go hl.http.ListenAndServe()
+	t.Cleanup(func() {hl.http.Shutdown(context.Background());})
+	time.Sleep(time.Millisecond)
+}
+
+func genericTestSendOutput(message []byte) ([]byte, error) {
+	conn, err := net.Dial("tcp", genericTestListen)
+	if err != nil {
+		return nil, err
+	}
+	_, err = conn.Write(message)
+	if err != nil {
+		return nil, err
+	}
+	b := make([]byte, 1024)
+	_ , err = io.ReadAtLeast(conn, b, 24)
+	if err != nil {
+		return nil, err
+	}
+	return b, nil
+}
+
+func runGeneric(t *testing.T, reqFileName string) (string, string) {
+	var message string
+	origSender := genericSender
+	genericSender = func(_ *irc.Connection, m string, _ loggo.Logger, _ string) {
+		message = m
+	}
+	t.Cleanup(func(){genericSender = origSender})
+
+	src, err := os.ReadFile(path.Join("testdata", reqFileName))
+	if err != nil {
+		t.Fatal(err)
+	}
+	resp, err := genericTestSendOutput(src)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return message, string(resp)
+}
+
+// Non-strict
+
+func testGenericBaseline(t *testing.T) {
+	message, resp := runGeneric(t, "mismatch")
+	if message != "%BOLDhw" {
+		t.Fatalf("Expected %q, got: %q", "%BOLDhw", message)
+	}
+	if !strings.HasPrefix(string(resp), "HTTP/1.1 200 OK") {
+		t.Fatalf("Unexpected message: %s", resp)
+	}
+}
+
+// Strict
+
+func testGenericStrict(t *testing.T) {
+	message, resp := runGeneric(t, "mismatch")
+	if message != "" {
+		t.Fatalf("Expected %q, got: %q", "", message)
+	}
+	if !strings.HasPrefix(string(resp), "HTTP/1.1 400 Bad Request") {
+		t.Fatalf("Unexpected message: %s", resp)
+	}
+}
+
+func testURLEncoded(t *testing.T) {
+	message, resp := runGeneric(t, "urlencoded")
+	if message != "%BOLDhw\n" { // Note the linefeed
+		t.Fatalf("Expected %q, got: %q", "%BOLDhw\n", message)
+	}
+	if !strings.HasPrefix(string(resp), "HTTP/1.1 200 OK") {
+		t.Fatalf("Unexpected message: %s", resp)
+	}
+}
+
+func testURLEncodedNonPrintable(t *testing.T) {
+	message, resp := runGeneric(t, "urlencoded_npc")
+	if message != "\x02hw" {
+		t.Fatalf("Expected %q, got: %q", "\x02hw", message)
+	}
+	if !strings.HasPrefix(string(resp), "HTTP/1.1 200 OK") {
+		t.Fatalf("Unexpected message: %s", resp)
+	}
+}
+
+func testOctetStream(t *testing.T) {
+	message, resp := runGeneric(t, "octetstream")
+	if message != "%BOLDhw\n" {
+		t.Fatalf("Expected %q, got: %q", "%BOLDhw\n", message)
+	}
+	if !strings.HasPrefix(string(resp), "HTTP/1.1 200 OK") {
+		t.Fatalf("Unexpected message: %s", resp)
+	}
+}
+
+func testMultipartQP(t *testing.T) {
+	message, resp := runGeneric(t, "multipart_qp")
+	if message != "%BOLDhw\n" {
+		t.Fatalf("Expected %q, got: %q", "%BOLDhw\n", message)
+	}
+	if !strings.HasPrefix(string(resp), "HTTP/1.1 200 OK") {
+		t.Fatalf("Unexpected message: %s", resp)
+	}
+}
+
+func testMultipart8bit(t *testing.T) {
+	message, resp := runGeneric(t, "multipart_8bit")
+	if message != "%BOLDhw\n" {
+		t.Fatalf("Expected %q, got: %q", "%BOLDhw\n", message)
+	}
+	if !strings.HasPrefix(string(resp), "HTTP/1.1 200 OK") {
+		t.Fatalf("Unexpected message: %s", resp)
+	}
+}
+
+func testMultipartBase64(t *testing.T) {
+	message, resp := runGeneric(t, "multipart_base64")
+	if message != "%BOLDhw\n" {
+		t.Fatalf("Expected %q, got: %q", "%BOLDhw\n", message)
+	}
+	if !strings.HasPrefix(string(resp), "HTTP/1.1 200 OK") {
+		t.Fatalf("Unexpected message: %s", resp)
+	}
+}
+
+func TestAll(t *testing.T) {
+	writer, err := loggo.RemoveWriter("default")
+	if err != nil {
+		t.Error(err)
+	}
+	t.Cleanup(func() {loggo.DefaultContext().AddWriter("default", writer)})
+	genericTestStartHTTPServer(t, "/send")
+
+	t.Run("Baseline", testGenericBaseline)
+
+	// Turn on strict for the rest of these
+	viper.Set("http.listeners.generic.strict", true)
+
+	t.Run("Strict Mismatch", testGenericStrict)
+	t.Run("Strict URL Encoded", testURLEncoded)
+	t.Run("Strict URL Encoded Non-printable", testURLEncodedNonPrintable)
+	t.Run("Strict Octet Stream", testOctetStream)
+	t.Run("Strict Multipart Quoted Printable", testMultipartQP)
+	t.Run("Strict Multipart 8bit", testMultipart8bit)
+	t.Run("Strict Multipart Base 64", testMultipartBase64)
+
+	// Restore to zero value
+	viper.Set("http.listeners.generic.strict", false)
+}

httplistener/testdata/mismatch

@@ -0,0 +1,8 @@
+POST /send HTTP/1.1
+Host: localhost:18045
+User-Agent: curl/7.76.1
+Accept: */*
+Content-Length: 7
+Content-Type: application/x-www-form-urlencoded
+
+%BOLDhw

httplistener/testdata/multipart_8bit

@@ -0,0 +1,14 @@
+POST /send HTTP/1.1
+Host: localhost:18045
+User-Agent: curl/7.76.1
+Accept: */*
+Content-Length: 193
+Content-Type: multipart/form-data; boundary=------------------------3e056db8b78cdf1d
+
+--------------------------3e056db8b78cdf1d
+Content-Disposition: form-data; name="foo"; filename="-"
+Content-Transfer-Encoding: 8bit
+
+%BOLDhw
+
+--------------------------3e056db8b78cdf1d--

httplistener/testdata/multipart_base64

@@ -0,0 +1,13 @@
+POST /send HTTP/1.1
+Host: localhost:18045
+User-Agent: curl/7.76.1
+Accept: */*
+Content-Length: 199
+Content-Type: multipart/form-data; boundary=------------------------53b3d5d529be2ef6
+
+--------------------------53b3d5d529be2ef6
+Content-Disposition: form-data; name="foo"; filename="-"
+Content-Transfer-Encoding: base64
+
+JUJPTERodwo=
+--------------------------53b3d5d529be2ef6--

httplistener/testdata/multipart_qp

@@ -0,0 +1,13 @@
+POST /send HTTP/1.1
+Host: localhost:18045
+User-Agent: curl/7.76.1
+Accept: */*
+Content-Length: 207
+Content-Type: multipart/form-data; boundary=------------------------5cbcc75e58188254
+
+--------------------------5cbcc75e58188254
+Content-Disposition: form-data; name="foo"; filename="-"
+Content-Transfer-Encoding: quoted-printable
+
+%BOLDhw=0A
+--------------------------5cbcc75e58188254--

httplistener/testdata/octetstream

@@ -0,0 +1,8 @@
+POST /send HTTP/1.1
+Host: localhost:18045
+User-Agent: curl/7.76.1
+Accept: */*
+Content-Type: application/octet-stream
+Content-Length: 8
+
+%BOLDhw

httplistener/testdata/urlencoded

@@ -0,0 +1,8 @@
+POST /send HTTP/1.1
+Host: localhost:18045
+User-Agent: curl/7.76.1
+Accept: */*
+Content-Length: 12
+Content-Type: application/x-www-form-urlencoded
+
+%25BOLDhw%0A

httplistener/testdata/urlencoded_npc

@@ -0,0 +1,8 @@
+POST /send HTTP/1.1
+Host: localhost:18045
+User-Agent: curl/7.76.1
+Accept: */*
+Content-Length: 5
+Content-Type: application/x-www-form-urlencoded
+
+%02hw