Add support for message tags from IRCv3 to IRCReader. Fixes #6

ReadLine now parses tags out of incoming lines (tags are not currently passed through the encoder)

IRCParser.tokeniseLine() now understands that tags may exist at the start of a line and tokenises accordingly.
(In reality, the only time it ever sees them is during testing)

Parser processLine now looks for timestamps from tags from the ReadLine, not by parsing token[0] itself.

Made IRCReader tests a lot less annoying to work with.

irc/src/com/dmdirc/parser/irc/IRCParser.java

@@ -59,6 +59,8 @@ import java.security.KeyManagementException;
 import java.security.NoSuchAlgorithmException;
 import java.security.SecureRandom;
 import java.security.cert.X509Certificate;
+import java.text.SimpleDateFormat;
+import java.text.ParseException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
@@ -915,9 +917,28 @@ public class IRCParser extends BaseSocketAwareParser implements SecureParser, En
             return new String[]{""};
         }
 
-        final int lastarg = line.indexOf(" :");
+        int lastarg = line.indexOf(" :");
         String[] tokens;
 
+        // Check for IRC Tags.
+        if (line.charAt(0) == '@') {
+            // We have tags.
+            tokens = line.split(" ", 2);
+            final int tsEnd = tokens[0].indexOf('@', 1);
+            boolean hasTSIRCDate = false;
+            if (tsEnd > -1) {
+                try {
+                    final long ts = Long.parseLong(tokens[0].substring(1, tsEnd));
+                    hasTSIRCDate = true;
+                } catch (final NumberFormatException nfe) { /* Not a timestamp. */ }
+            }
+
+            if (!hasTSIRCDate) {
+                // IRCv3 Tags, last arg is actually the second " :"
+                lastarg = line.indexOf(" :", lastarg+1);
+            }
+        }
+
         if (lastarg > -1) {
             final String[] temp = line.substring(0, lastarg).split(" ");
             tokens = new String[temp.length + 1];
@@ -1076,13 +1097,17 @@ public class IRCParser extends BaseSocketAwareParser implements SecureParser, En
         callDataIn(line.getLine());
         final String[] token = line.getTokens();
         Date lineTS = new Date();
-        if (!token[0].isEmpty() && token[0].charAt(0) == '@') {
+
+        if (line.getTags().containsKey("tsirc date")) {
             try {
-                final int tsEnd = token[0].indexOf('@', 1);
-                final long ts = Long.parseLong(token[0].substring(1, tsEnd));
-                token[0] = token[0].substring(tsEnd + 1);
+                final long ts = Long.parseLong(line.getTags().get("tsirc date"));
                 lineTS = new Date(ts - tsdiff);
             } catch (final NumberFormatException nfe) { /* Do nothing. */ }
+        } else if (line.getTags().containsKey("time")) {
+            final SimpleDateFormat servertime = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'");
+            try {
+                lineTS = servertime.parse(line.getTags().get("time"));
+            } catch (final ParseException pe) { /* Do nothing. */ }
         }
 
         setPingNeeded(false);

irc/src/com/dmdirc/parser/irc/IRCReader.java

@@ -32,6 +32,12 @@ import java.nio.charset.CharacterCodingException;
 import java.nio.charset.Charset;
 import java.nio.charset.CharsetDecoder;
 import java.nio.charset.CodingErrorAction;
+import java.text.SimpleDateFormat;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Map;
 
 /**
  * A {@link java.io.BufferedReader}-style reader that is aware of the IRC
@@ -42,7 +48,7 @@ import java.nio.charset.CodingErrorAction;
 public class IRCReader implements Closeable {
 
     /** Maximum length for an IRC line in bytes. */
-    private static final int LINE_LENGTH = 512;
+    private static final int LINE_LENGTH = 1024;
     /** The input stream to read input from. */
     private final InputStream stream;
     /** The encoder to use to encode lines. */
@@ -93,7 +99,12 @@ public class IRCReader implements Closeable {
         int paramOffset = -1;
         int chr = 0, lastChr = 0;
 
-        while (offset < 512 && (chr = stream.read()) > -1) {
+        boolean hasTags = false;
+        boolean endOfTags = false;
+        boolean hasV3Tags = false;
+        boolean foundFirstSpace = false;
+
+        while (offset < LINE_LENGTH && (chr = stream.read()) > -1) {
             if (chr == '\r') {
                 continue;
             } else if (chr == '\n') {
@@ -101,11 +112,32 @@ public class IRCReader implements Closeable {
                 break;
             }
 
+            if (hasTags && !endOfTags) {
+                // Tags end either at the first @ for non-v3 tags or space for v3
+                if (offset > 0 && ((chr == '@' && !hasV3Tags) || chr == ' ')) {
+                    endOfTags = true;
+                    hasV3Tags = (chr == ' ');
+                }
+                // If we are still possibly looking at tags, and we find a non-numeric
+                // character, then we probably have v3Tags
+                if (!endOfTags && (chr < '0' || chr > '9')) {
+                    hasV3Tags = true;
+                }
+            } else if (offset == 0 && chr == '@') {
+                hasTags = true;
+            } else if (offset == 0) {
+                endOfTags = true;
+            }
+
             line[offset++] = (byte) chr;
 
             if (lastChr == ' ' && chr == ':' && paramOffset == -1) {
                 // We've found the last param
-                paramOffset = offset;
+                if (!hasV3Tags || foundFirstSpace) {
+                    paramOffset = offset;
+                } else if (hasV3Tags) {
+                    foundFirstSpace = true;
+                }
             }
 
             lastChr = chr;
@@ -209,16 +241,47 @@ public class IRCReader implements Closeable {
         private final String line;
         /** The tokens found in the line, individually encoded as appropriate. */
         private final String[] tokens;
+        /** The tags (if any) found in the line, individually encoded as appropriate. */
+        private final Map<String,String> tags = new HashMap<>();
 
         /**
          * Creates a new instance of {@link ReadLine} with the specified line
          * and tokens.
          *
          * @param line A string representation of the line
-         * @param tokens The tokens which make up the line
+         * @param lineTokens The tokens which make up the line
          */
-        public ReadLine(final String line, final String[] tokens) {
+        public ReadLine(final String line, final String[] lineTokens) {
             this.line = line;
+
+            String[] tokens = lineTokens;
+            if (!tokens[0].isEmpty() && tokens[0].charAt(0) == '@') {
+                // Look for old-style TSIRC timestamp first.
+                final int tsEnd = tokens[0].indexOf('@', 1);
+                boolean hasTSIRCDate = false;
+                if (tsEnd > -1) {
+                    try {
+                        final long ts = Long.parseLong(tokens[0].substring(1, tsEnd));
+                        tags.put("tsirc date", tokens[0].substring(1, tsEnd));
+                        hasTSIRCDate = true;
+                        tokens[0] = tokens[0].substring(tsEnd + 1);
+                    } catch (final NumberFormatException nfe) { /* Not a timestamp. */ }
+                }
+
+                if (!hasTSIRCDate) {
+                    final String[] lineTags = tokens[0].substring(1).split(";");
+                    for (final String keyVal : lineTags) {
+                        if (!keyVal.isEmpty()) {
+                            final String[] keyValue = keyVal.split("=", 2);
+                            tags.put(keyValue[0], keyValue.length > 1 ? keyValue[1] : "");
+                        }
+                    }
+
+                    tokens = new String[lineTokens.length - 1];
+                    System.arraycopy(lineTokens, 1, tokens, 0, lineTokens.length - 1);
+                }
+            }
+
             this.tokens = tokens;
         }
 
@@ -243,5 +306,14 @@ public class IRCReader implements Closeable {
         public String[] getTokens() {
             return tokens;
         }
+
+        /**
+         * Retrieves a map of tags extracted from the specified line.
+         *
+         * @return The line's tags
+         */
+        public Map<String,String> getTags() {
+            return tags;
+        }
     }
 }

irc/test/com/dmdirc/parser/irc/IRCReaderTest.java

@@ -25,9 +25,13 @@ package com.dmdirc.parser.irc;
 import com.dmdirc.parser.interfaces.Encoder;
 import com.dmdirc.parser.irc.IRCReader.ReadLine;
 
+import java.io.ByteArrayInputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.charset.Charset;
+import java.text.SimpleDateFormat;
+import java.util.Arrays;
+import java.util.Date;
 
 import org.junit.Assert;
 import org.junit.Test;
@@ -40,12 +44,9 @@ public class IRCReaderTest {
     /** Reads and verifies a single line. */
     @Test
     public void testReadLine() throws IOException {
-        final InputStream stream = mock(InputStream.class);
+        final InputStream stream = new ByteArrayInputStream("test\r\n".getBytes());
         final Encoder encoder = mock(Encoder.class);
 
-        when(stream.read()).thenReturn((int) 't', (int) 'e', (int) 's',
-                (int) 't', (int) '\r', (int) '\n');
-
         final IRCReader reader = new IRCReader(stream, encoder);
         final ReadLine line = reader.readLine();
 
@@ -55,12 +56,9 @@ public class IRCReaderTest {
     /** Reads and verifies a single line with only a trailing LF. */
     @Test
     public void testReadLineWithOnlyLF() throws IOException {
-        final InputStream stream = mock(InputStream.class);
+        final InputStream stream = new ByteArrayInputStream("test\n".getBytes());
         final Encoder encoder = mock(Encoder.class);
 
-        when(stream.read()).thenReturn((int) 't', (int) 'e', (int) 's',
-                (int) 't', (int) '\n');
-
         final IRCReader reader = new IRCReader(stream, encoder);
         final ReadLine line = reader.readLine();
 
@@ -70,12 +68,9 @@ public class IRCReaderTest {
     /** Reads and verifies a single line with a trailing parameter. */
     @Test
     public void testReadLineWithParameter() throws IOException {
-        final InputStream stream = mock(InputStream.class);
+        final InputStream stream = new ByteArrayInputStream("te t :foo\r\n".getBytes());
         final Encoder encoder = mock(Encoder.class);
 
-        when(stream.read()).thenReturn((int) 't', (int) 'e', (int) ' ',
-                (int) 't', (int) ' ', (int) ':', (int) 'f', (int) 'o',
-                (int) 'o', (int) '\r', (int) '\n');
         when(encoder.encode((String) isNull(), (String) isNull(),
                 (byte[]) anyObject(), anyInt(), eq(3)))
                 .thenReturn("encoded");
@@ -90,15 +85,9 @@ public class IRCReaderTest {
     /** Reads and verifies a single line with a trailing parameter. */
     @Test
     public void testReadLineWithMultipleSpaces() throws IOException {
-        final InputStream stream = mock(InputStream.class);
+        final InputStream stream = new ByteArrayInputStream("foo bar  baz  :qux  baz\r\n".getBytes());
         final Encoder encoder = mock(Encoder.class);
 
-        when(stream.read()).thenReturn((int) 'f', (int) 'o', (int) 'o',
-                (int) ' ', (int) 'b', (int) 'a', (int) 'r',
-                (int) ' ', (int) ' ', (int) 'b', (int) 'a', (int) 'z',
-                (int) ' ', (int) ' ', (int) ':', (int) 'q', (int) 'u',
-                (int) 'x', (int) ' ', (int) ' ', (int) 'b', (int) 'a', (int) 'z',
-                (int) '\r', (int) '\n');
         when(encoder.encode((String) isNull(), (String) isNull(),
                 (byte[]) anyObject(), eq(15), eq(8)))
                 .thenReturn("qux  baz");
@@ -106,20 +95,15 @@ public class IRCReaderTest {
         final IRCReader reader = new IRCReader(stream, encoder);
         final ReadLine line = reader.readLine();
 
-        assertArrayEquals(new String[]{"foo", "bar", "baz", "qux  baz",},
-                line.getTokens());
+        assertArrayEquals(new String[]{"foo", "bar", "baz", "qux  baz",}, line.getTokens());
     }
 
     /** Verifies that a source is extracted properly. */
     @Test
     public void testGetSource() throws IOException {
-        final InputStream stream = mock(InputStream.class);
+        final InputStream stream = new ByteArrayInputStream(":src x :o\r\n".getBytes());
         final Encoder encoder = mock(Encoder.class);
 
-        when(stream.read()).thenReturn((int) ':', (int) 's', (int) 'r',
-                (int) 'c', (int) ' ', (int) 'x', (int) ' ', (int) ':',
-                (int) 'o', (int) '\r', (int) '\n');
-
         new IRCReader(stream, encoder).readLine();
 
         verify(encoder).encode(eq("src"), anyString(),
@@ -129,13 +113,9 @@ public class IRCReaderTest {
     /** Verifies that no source is passed if the source is empty. */
     @Test
     public void testGetEmptySource() throws IOException {
-        final InputStream stream = mock(InputStream.class);
+        final InputStream stream = new ByteArrayInputStream(": rc x :o\r\n".getBytes());
         final Encoder encoder = mock(Encoder.class);
 
-        when(stream.read()).thenReturn((int) ':', (int) ' ', (int) 'r',
-                (int) 'c', (int) ' ', (int) 'x', (int) ' ', (int) ':',
-                (int) 'o', (int) '\r', (int) '\n');
-
         new IRCReader(stream, encoder).readLine();
 
         verify(encoder).encode((String) isNull(), anyString(),
@@ -145,13 +125,9 @@ public class IRCReaderTest {
     /** Verifies that a destination is extracted properly. */
     @Test
     public void testGetDestination() throws IOException {
-        final InputStream stream = mock(InputStream.class);
+        final InputStream stream = new ByteArrayInputStream(":src x y :z\r\n".getBytes());
         final Encoder encoder = mock(Encoder.class);
 
-        when(stream.read()).thenReturn((int) ':', (int) 's', (int) 'r',
-                (int) 'c', (int) ' ', (int) 'x', (int) ' ', (int) 'y',
-                (int) ' ', (int) ':', (int) 'z', (int) '\r', (int) '\n');
-
         new IRCReader(stream, encoder).readLine();
 
         verify(encoder).encode(anyString(), eq("y"), (byte[]) anyObject(),
@@ -161,13 +137,9 @@ public class IRCReaderTest {
     /** Verifies that no destination is extracted if there's no source. */
     @Test
     public void testGetDestinationNoSource() throws IOException {
-        final InputStream stream = mock(InputStream.class);
+        final InputStream stream = new ByteArrayInputStream("_src x y :z\r\n".getBytes());
         final Encoder encoder = mock(Encoder.class);
 
-        when(stream.read()).thenReturn((int) '_', (int) 's', (int) 'r',
-                (int) 'c', (int) ' ', (int) 'x', (int) ' ', (int) 'y',
-                (int) ' ', (int) ':', (int) 'z', (int) '\r', (int) '\n');
-
         new IRCReader(stream, encoder).readLine();
 
         verify(encoder).encode(anyString(), (String) isNull(),
@@ -177,13 +149,9 @@ public class IRCReaderTest {
     /** Verifies that no destination is extracted if there are too few args. */
     @Test
     public void testGetDestinationTooShort() throws IOException {
-        final InputStream stream = mock(InputStream.class);
+        final InputStream stream = new ByteArrayInputStream(":src x :abz\r\n".getBytes());
         final Encoder encoder = mock(Encoder.class);
 
-        when(stream.read()).thenReturn((int) ':', (int) 's', (int) 'r',
-                (int) 'c', (int) ' ', (int) 'x', (int) ' ', (int) ':',
-                (int) 'a', (int) 'b', (int) 'z', (int) '\r', (int) '\n');
-
         new IRCReader(stream, encoder).readLine();
 
         verify(encoder).encode(anyString(), (String) isNull(),
@@ -193,18 +161,12 @@ public class IRCReaderTest {
     /** Verifies that a numeric's destination is extracted properly. */
     @Test
     public void testGetDestinationWithNumeric() throws IOException {
-        final InputStream stream = mock(InputStream.class);
+        final InputStream stream = new ByteArrayInputStream(":src 1 x y z :x\r\n".getBytes());
         final Encoder encoder = mock(Encoder.class);
 
-        when(stream.read()).thenReturn((int) ':', (int) 's', (int) 'r',
-                (int) 'c', (int) ' ', (int) '1', (int) ' ', (int) 'x',
-                (int) ' ', (int) 'y', (int) ' ', (int) 'z', (int) ' ',
-                (int) ':', (int) 'x', (int) '\r', (int) '\n');
-
         new IRCReader(stream, encoder).readLine();
 
-        verify(encoder).encode(anyString(), eq("y"),
-                (byte[]) anyObject(), anyInt(), anyInt());
+        verify(encoder).encode(anyString(), eq("y"), (byte[]) anyObject(), anyInt(), anyInt());
     }
 
     /** Verifies that the close call is proxied to the stream. */
@@ -224,20 +186,95 @@ public class IRCReaderTest {
         final InputStream stream = mock(InputStream.class);
         final Encoder encoder = mock(Encoder.class);
 
-        when(stream.read()).thenReturn((int) ':', (int) 's', (int) 'r',
-                (int) 'c', (int) ' ', (int) '1', (int) ' ', 0xF6,
-                (int) ' ', (int) 'y', (int) ' ', (int) 'z', (int) ' ',
+        when(stream.read()).thenReturn((int) ':', (int) 's', (int) 'r', (int) 'c', (int) ' ',
+                (int) '1', (int) ' ', 0xF6, (int) ' ', (int) 'y', (int) ' ', (int) 'z', (int) ' ',
                 (int) ':', (int) 'x', (int) '\r', (int) '\n');
 
         when(encoder.encode(anyString(), anyString(),
                 (byte[]) any(), anyInt(), anyInt())).thenReturn("x");
 
-        final ReadLine line = new IRCReader(stream, encoder,
-                Charset.forName("UTF-8")).readLine();
+        final ReadLine line = new IRCReader(stream, encoder, Charset.forName("UTF-8")).readLine();
+
+        Assert.assertArrayEquals(new String[]{":src", "1", "\uFFFD", "y", "z", "x"}, line.getTokens());
+    }
+
+    /** Verify tokeniser with TSIRC Time Stamp */
+    @Test
+    public void testReadLineTSIRC() throws IOException {
+        final ReadLine line = new ReadLine("", IRCParser.tokeniseLine("@123@:test ing"));
 
-        Assert.assertArrayEquals(new String[] {
-            ":src", "1", "\uFFFD", "y", "z", "x"
-        }, line.getTokens());
+        assertEquals(":test", line.getTokens()[0]);
+        assertEquals("ing", line.getTokens()[1]);
+        assertEquals("123", line.getTags().get("tsirc date"));
+        assertNull(line.getTags().get("time"));
     }
 
+    /** Verify tokeniser with IRCv3 tags */
+    @Test
+    public void testReadLineIRCv3TS() throws IOException {
+        final SimpleDateFormat servertime = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'");
+        final ReadLine line = new ReadLine("", IRCParser.tokeniseLine("@time="+servertime.format(new Date(123))+";something=else;foobar; :test ing"));
+
+        assertEquals(":test", line.getTokens()[0]);
+        assertEquals("ing", line.getTokens()[1]);
+        assertNull(line.getTags().get("tsirc date"));
+        assertEquals(servertime.format(new Date(123)), line.getTags().get("time"));
+    }
+
+
+    /** Verify tokeniser with a numeric tag timestamp */
+    @Test
+    public void testReadLineNumericTag() throws IOException {
+        final ReadLine line = new ReadLine("", IRCParser.tokeniseLine("@123 :test ing"));
+
+        assertEquals(":test", line.getTokens()[0]);
+        assertEquals("ing", line.getTokens()[1]);
+        assertTrue(line.getTags().containsKey("123"));
+    }
+
+
+    /** Verify line with TSIRC Time Stamp */
+    @Test
+    public void testReaderTSIRC() throws IOException {
+        final InputStream stream = new ByteArrayInputStream("@123@:test ing\r\n".getBytes());
+        final Encoder encoder = mock(Encoder.class);
+
+        final IRCReader reader = new IRCReader(stream, encoder);
+        final ReadLine line = reader.readLine();
+
+        assertEquals(":test", line.getTokens()[0]);
+        assertEquals("ing", line.getTokens()[1]);
+        assertEquals("123", line.getTags().get("tsirc date"));
+        assertNull(line.getTags().get("time"));
+    }
+
+    /** Verify line with IRCv3 timestamp */
+    @Test
+    public void testReaderIRCv3TS() throws IOException {
+        final SimpleDateFormat servertime = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'");
+        final InputStream stream = new ByteArrayInputStream(("@time="+servertime.format(new Date(123))+";something=else;foobar; :test ing\r\n").getBytes());
+        final Encoder encoder = mock(Encoder.class);
+
+        final IRCReader reader = new IRCReader(stream, encoder);
+        final ReadLine line = reader.readLine();
+
+        assertEquals(":test", line.getTokens()[0]);
+        assertEquals("ing", line.getTokens()[1]);
+        assertNull(line.getTags().get("tsirc date"));
+        assertEquals(servertime.format(new Date(123)), line.getTags().get("time"));
+    }
+
+    /** Verify line with a numeric tag timestamp */
+    @Test
+    public void testReaderNumericTag() throws IOException {
+        final InputStream stream = new ByteArrayInputStream("@123 :test ing\r\n".getBytes());
+        final Encoder encoder = mock(Encoder.class);
+
+        final IRCReader reader = new IRCReader(stream, encoder);
+        final ReadLine line = reader.readLine();
+
+        assertEquals(":test", line.getTokens()[0]);
+        assertEquals("ing", line.getTokens()[1]);
+        assertTrue(line.getTags().containsKey("123"));
+    }
 }