initial persistent history implementation

Makefile

 	cd irc/history && go test . && go vet .
 	cd irc/isupport && go test . && go vet .
 	cd irc/modes && go test . && go vet .
+	cd irc/mysql && go test . && go vet .
 	cd irc/passwd && go test . && go vet .
 	cd irc/utils && go test . && go vet .
 	./.check-gofmt.sh

gencapdefs.py

         url="https://github.com/ircv3/ircv3-specifications/pull/398",
         standard="proposed IRCv3",
     ),
+    CapDef(
+        identifier="Chathistory",
+        name="draft/chathistory",
+        url="https://github.com/ircv3/ircv3-specifications/pull/393",
+        standard="proposed IRCv3",
+    ),
 ]
 
 def validate_defs():

go.mod

 	code.cloudfoundry.org/bytefmt v0.0.0-20190819182555-854d396b647c
 	github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815
 	github.com/go-ldap/ldap/v3 v3.1.6
+	github.com/go-sql-driver/mysql v1.5.0
 	github.com/goshuirc/e-nfa v0.0.0-20160917075329-7071788e3940 // indirect
 	github.com/goshuirc/irc-go v0.0.0-20190713001546-05ecc95249a0
 	github.com/mattn/go-colorable v0.1.4
 	github.com/mattn/go-isatty v0.0.10 // indirect
 	github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b
+	github.com/onsi/ginkgo v1.12.0 // indirect
+	github.com/onsi/gomega v1.9.0 // indirect
 	github.com/oragono/confusables v0.0.0-20190624102032-fe1cf31a24b0
 	github.com/oragono/go-ident v0.0.0-20170110123031-337fed0fd21a
-	github.com/tidwall/btree v0.0.0-20191029221954-400434d76274 // indirect
+	github.com/stretchr/testify v1.4.0 // indirect
 	github.com/tidwall/buntdb v1.1.2
-	github.com/tidwall/gjson v1.3.4 // indirect
-	github.com/tidwall/grect v0.0.0-20161006141115-ba9a043346eb // indirect
-	github.com/tidwall/match v1.0.1 // indirect
-	github.com/tidwall/pretty v1.0.0 // indirect
-	github.com/tidwall/rtree v0.0.0-20180113144539-6cd427091e0e // indirect
-	github.com/tidwall/tinyqueue v0.0.0-20180302190814-1e39f5511563 // indirect
 	golang.org/x/crypto v0.0.0-20191112222119-e1110fd1c708
-	golang.org/x/sys v0.0.0-20191115151921-52ab43148777 // indirect
 	golang.org/x/text v0.3.2
 	gopkg.in/yaml.v2 v2.2.5
 )

irc/accounts.go

 	keyAccountSettings         = "account.settings %s"
 	keyAccountVHost            = "account.vhost %s"
 	keyCertToAccount           = "account.creds.certfp %s"
-	keyAccountChannels         = "account.channels %s"
+	keyAccountChannels         = "account.channels %s" // channels registered to the account
+	keyAccountJoinedChannels   = "account.joinedto %s" // channels a persistent client has joined
 
 	keyVHostQueueAcctToId = "vhostQueue %s"
 	vhostRequestIdx       = "vhostQueue"
 	config := server.Config()
 	am.buildNickToAccountIndex(config)
 	am.initVHostRequestQueue(config)
+	am.createAlwaysOnClients(config)
+}
+
+func (am *AccountManager) createAlwaysOnClients(config *Config) {
+	if config.Accounts.Bouncer.AlwaysOn == PersistentDisabled {
+		return
+	}
+
+	verifiedPrefix := fmt.Sprintf(keyAccountVerified, "")
+
+	am.serialCacheUpdateMutex.Lock()
+	defer am.serialCacheUpdateMutex.Unlock()
+
+	var accounts []string
+
+	am.server.store.View(func(tx *buntdb.Tx) error {
+		err := tx.AscendGreaterOrEqual("", verifiedPrefix, func(key, value string) bool {
+			if !strings.HasPrefix(key, verifiedPrefix) {
+				return false
+			}
+			account := strings.TrimPrefix(key, verifiedPrefix)
+			accounts = append(accounts, account)
+			return true
+		})
+		return err
+	})
+
+	for _, accountName := range accounts {
+		account, err := am.LoadAccount(accountName)
+		if err == nil && account.Verified &&
+			persistenceEnabled(config.Accounts.Bouncer.AlwaysOn, account.Settings.AlwaysOn) {
+			am.server.AddAlwaysOnClient(account, am.loadChannels(accountName))
+		}
+	}
 }
 
 func (am *AccountManager) buildNickToAccountIndex(config *Config) {
 	return err
 }
 
+func (am *AccountManager) saveChannels(account string, channels []string) {
+	channelsStr := strings.Join(channels, ",")
+	key := fmt.Sprintf(keyAccountJoinedChannels, account)
+	am.server.store.Update(func(tx *buntdb.Tx) error {
+		tx.Set(key, channelsStr, nil)
+		return nil
+	})
+}
+
+func (am *AccountManager) loadChannels(account string) (channels []string) {
+	key := fmt.Sprintf(keyAccountJoinedChannels, account)
+	var channelsStr string
+	am.server.store.View(func(tx *buntdb.Tx) error {
+		channelsStr, _ = tx.Get(key)
+		return nil
+	})
+	if channelsStr != "" {
+		return strings.Split(channelsStr, ",")
+	}
+	return
+}
+
 func (am *AccountManager) addRemoveCertfp(account, certfp string, add bool, hasPrivs bool) (err error) {
 	certfp, err = utils.NormalizeCertfp(certfp)
 	if err != nil {
 	}
 	am.server.logger.Info("accounts", "client", nick, "registered account", casefoldedAccount)
 	raw.Verified = true
-	clientAccount, err := am.deserializeRawAccount(raw)
+	clientAccount, err := am.deserializeRawAccount(raw, casefoldedAccount)
 	if err != nil {
 		return err
 	}
 		return
 	}
 
-	result, err = am.deserializeRawAccount(raw)
-	result.NameCasefolded = casefoldedAccount
+	result, err = am.deserializeRawAccount(raw, casefoldedAccount)
 	return
 }
 
-func (am *AccountManager) deserializeRawAccount(raw rawClientAccount) (result ClientAccount, err error) {
+func (am *AccountManager) deserializeRawAccount(raw rawClientAccount, cfName string) (result ClientAccount, err error) {
 	result.Name = raw.Name
+	result.NameCasefolded = cfName
 	regTimeInt, _ := strconv.ParseInt(raw.RegisteredAt, 10, 64)
 	result.RegisteredAt = time.Unix(regTimeInt, 0).UTC()
 	e := json.Unmarshal([]byte(raw.Credentials), &result.Credentials)
 	vhostKey := fmt.Sprintf(keyAccountVHost, casefoldedAccount)
 	vhostQueueKey := fmt.Sprintf(keyVHostQueueAcctToId, casefoldedAccount)
 	channelsKey := fmt.Sprintf(keyAccountChannels, casefoldedAccount)
+	joinedChannelsKey := fmt.Sprintf(keyAccountJoinedChannels, casefoldedAccount)
 
 	var clients []*Client
 
 		tx.Delete(vhostKey)
 		channelsStr, _ = tx.Get(channelsKey)
 		tx.Delete(channelsKey)
+		tx.Delete(joinedChannelsKey)
 
 		_, err := tx.Delete(vhostQueueKey)
 		am.decrementVHostQueueCount(casefoldedAccount, err)
 }
 
 func (am *AccountManager) Login(client *Client, account ClientAccount) {
-	changed := client.SetAccountName(account.Name)
-	if !changed {
-		return
-	}
+	client.Login(account)
 
 	client.nickTimer.Touch(nil)
 
 	am.Lock()
 	defer am.Unlock()
 	am.accountToClients[casefoldedAccount] = append(am.accountToClients[casefoldedAccount], client)
-	for _, client := range am.accountToClients[casefoldedAccount] {
-		client.SetAccountSettings(account.Settings)
-	}
 }
 
 func (am *AccountManager) Logout(client *Client) {
 }
 
 type AccountSettings struct {
-	AutoreplayLines *int
-	NickEnforcement NickEnforcementMethod
-	AllowBouncer    BouncerAllowedSetting
-	ReplayJoins     ReplayJoinsSetting
+	AutoreplayLines  *int
+	NickEnforcement  NickEnforcementMethod
+	AllowBouncer     BouncerAllowedSetting
+	ReplayJoins      ReplayJoinsSetting
+	AlwaysOn         PersistentStatus
+	AutoreplayMissed bool
+	DMHistory        HistoryStatus
 }
 
 // ClientAccount represents a user account.
 		return
 	}
 
-	client.SetAccountName("")
+	client.Logout()
 	go client.nickTimer.Touch(nil)
 
 	// dispatch account-notify

irc/caps/defs.go

 
 const (
 	// number of recognized capabilities:
-	numCapabs = 26
+	numCapabs = 27
 	// length of the uint64 array that represents the bitset:
 	bitsetLen = 1
 )
 	// https://ircv3.net/specs/extensions/chghost-3.2.html
 	ChgHost Capability = iota
 
+	// Chathistory is the proposed IRCv3 capability named "draft/chathistory":
+	// https://github.com/ircv3/ircv3-specifications/pull/393
+	Chathistory Capability = iota
+
 	// EventPlayback is the proposed IRCv3 capability named "draft/event-playback":
 	// https://github.com/ircv3/ircv3-specifications/pull/362
 	EventPlayback Capability = iota
 		"batch",
 		"cap-notify",
 		"chghost",
+		"draft/chathistory",
 		"draft/event-playback",
 		"draft/languages",
 		"draft/multiline",

irc/channel.go

 	histServMask = "HistServ!HistServ@localhost"
 )
 
+type ChannelSettings struct {
+	History HistoryStatus
+}
+
 // Channel represents a channel that clients can join.
 type Channel struct {
 	flags             modes.ModeSet
 	joinPartMutex     sync.Mutex      // tier 3
 	ensureLoaded      utils.Once      // manages loading stored registration info from the database
 	dirtyBits         uint
+	settings          ChannelSettings
 }
 
 // NewChannel creates a new channel from a `Server` and a `name`
 
 	channel.initializeLists()
 	channel.writerSemaphore.Initialize(1)
-	channel.history.Initialize(config.History.ChannelLength, config.History.AutoresizeWindow)
+	channel.history.Initialize(0, 0)
 
 	if !registered {
+		channel.resizeHistory(config)
 		for _, mode := range config.Channels.defaultModes {
 			channel.flags.SetMode(mode, true)
 		}
 	return channel.ensureLoaded.Done()
 }
 
+func (channel *Channel) resizeHistory(config *Config) {
+	_, ephemeral, _ := channel.historyStatus(config)
+	if ephemeral {
+		channel.history.Resize(config.History.ChannelLength, config.History.AutoresizeWindow)
+	} else {
+		channel.history.Resize(0, 0)
+	}
+}
+
 // read in channel state that was persisted in the DB
 func (channel *Channel) applyRegInfo(chanReg RegisteredChannel) {
+	defer channel.resizeHistory(channel.server.Config())
+
 	channel.stateMutex.Lock()
 	defer channel.stateMutex.Unlock()
 
 	channel.createdTime = chanReg.RegisteredAt
 	channel.key = chanReg.Key
 	channel.userLimit = chanReg.UserLimit
+	channel.settings = chanReg.Settings
 
 	for _, mode := range chanReg.Modes {
 		channel.flags.SetMode(mode, true)
 		}
 	}
 
+	if includeFlags&IncludeSettings != 0 {
+		info.Settings = channel.settings
+	}
+
 	return
 }
 
 			if modeSet == nil {
 				continue
 			}
-			if !isJoined && target.flags.HasMode(modes.Invisible) && !isOper {
+			if !isJoined && target.HasMode(modes.Invisible) && !isOper {
 				continue
 			}
 			prefix := modeSet.Prefixes(isMultiPrefix)
 	return len(channel.members) == 0
 }
 
+// figure out where history is being stored: persistent, ephemeral, or neither
+// target is only needed if we're doing persistent history
+func (channel *Channel) historyStatus(config *Config) (persistent, ephemeral bool, target string) {
+	if !config.History.Persistent.Enabled {
+		return false, config.History.Enabled, ""
+	}
+
+	channel.stateMutex.RLock()
+	target = channel.nameCasefolded
+	historyStatus := channel.settings.History
+	registered := channel.registeredFounder != ""
+	channel.stateMutex.RUnlock()
+
+	historyStatus = historyEnabled(config.History.Persistent.RegisteredChannels, historyStatus)
+
+	// ephemeral history: either the channel owner explicitly set the ephemeral preference,
+	// or persistent history is disabled for unregistered channels
+	if registered {
+		ephemeral = (historyStatus == HistoryEphemeral)
+		persistent = (historyStatus == HistoryPersistent)
+	} else {
+		ephemeral = config.History.Enabled && !config.History.Persistent.UnregisteredChannels
+		persistent = config.History.Persistent.UnregisteredChannels
+	}
+	return
+}
+
+func (channel *Channel) AddHistoryItem(item history.Item) (err error) {
+	if !item.IsStorable() {
+		return
+	}
+
+	persistent, ephemeral, target := channel.historyStatus(channel.server.Config())
+	if ephemeral {
+		channel.history.Add(item)
+	}
+	if persistent {
+		return channel.server.historyDB.AddChannelItem(target, item)
+	}
+	return nil
+}
+
 // Join joins the given client to this channel (if they can be joined).
 func (channel *Channel) Join(client *Client, key string, isSajoin bool, rb *ResponseBuffer) {
 	details := client.Details()
 
 		channel.regenerateMembersCache()
 
-		message = utils.MakeMessage("")
-		histItem := history.Item{
-			Type:        history.Join,
-			Nick:        details.nickMask,
-			AccountName: details.accountName,
-			Message:     message,
+		// no history item for fake persistent joins
+		if rb != nil {
+			message = utils.MakeMessage("")
+			histItem := history.Item{
+				Type:        history.Join,
+				Nick:        details.nickMask,
+				AccountName: details.accountName,
+				Message:     message,
+			}
+			histItem.Params[0] = details.realname
+			channel.AddHistoryItem(histItem)
 		}
-		histItem.Params[0] = details.realname
-		channel.history.Add(histItem)
 
 		return
 	}()
 
 	for _, member := range channel.Members() {
 		for _, session := range member.Sessions() {
-			if session == rb.session {
+			if rb != nil && session == rb.session {
 				continue
 			} else if client == session.client {
 				channel.playJoinForSession(session)
 		}
 	}
 
-	if rb.session.capabilities.Has(caps.ExtendedJoin) {
+	if rb != nil && rb.session.capabilities.Has(caps.ExtendedJoin) {
 		rb.AddFromClient(message.Time, message.Msgid, details.nickMask, details.accountName, nil, "JOIN", chname, details.accountName, details.realname)
 	} else {
 		rb.AddFromClient(message.Time, message.Msgid, details.nickMask, details.accountName, nil, "JOIN", chname)
 	}
 
-	if rb.session.client == client {
+	if rb != nil && rb.session.client == client {
 		// don't send topic and names for a SAJOIN of a different client
 		channel.SendTopic(client, rb, false)
 		channel.Names(client, rb)
 	// TODO #259 can be implemented as Flush(false) (i.e., nonblocking) while holding joinPartMutex
 	rb.Flush(true)
 
-	channel.autoReplayHistory(client, rb, message.Msgid)
+	if rb != nil {
+		channel.autoReplayHistory(client, rb, message.Msgid)
+	}
 }
 
 func (channel *Channel) autoReplayHistory(client *Client, rb *ResponseBuffer, skipMsgid string) {
 	// autoreplay any messages as necessary
-	config := channel.server.Config()
 	var items []history.Item
+
+	var after, before time.Time
 	if rb.session.zncPlaybackTimes != nil && (rb.session.zncPlaybackTimes.targets == nil || rb.session.zncPlaybackTimes.targets.Has(channel.NameCasefolded())) {
-		items, _ = channel.history.Between(rb.session.zncPlaybackTimes.after, rb.session.zncPlaybackTimes.before, false, config.History.ChathistoryMax)
+		after, before = rb.session.zncPlaybackTimes.after, rb.session.zncPlaybackTimes.before
+	} else if !rb.session.lastSignoff.IsZero() {
+		// we already checked for history caps in `playReattachMessages`
+		after = rb.session.lastSignoff
+	}
+
+	if !after.IsZero() || !before.IsZero() {
+		_, seq, _ := channel.server.GetHistorySequence(channel, client, "")
+		if seq != nil {
+			zncMax := channel.server.Config().History.ZNCMax
+			items, _, _ = seq.Between(history.Selector{Time: after}, history.Selector{Time: before}, zncMax)
+		}
 	} else if !rb.session.HasHistoryCaps() {
 		var replayLimit int
 		customReplayLimit := client.AccountSettings().AutoreplayLines
 			replayLimit = channel.server.Config().History.AutoreplayOnJoin
 		}
 		if 0 < replayLimit {
-			items = channel.history.Latest(replayLimit)
+			_, seq, _ := channel.server.GetHistorySequence(channel, client, "")
+			if seq != nil {
+				items, _, _ = seq.Between(history.Selector{}, history.Selector{}, replayLimit)
+			}
 		}
 	}
 	// remove the client's own JOIN line from the replay
 		}
 	}
 
-	channel.history.Add(history.Item{
+	channel.AddHistoryItem(history.Item{
 		Type:        history.Part,
 		Nick:        details.nickMask,
 		AccountName: details.accountName,
 // 2. Send JOIN and MODE lines to channel participants (including the new client)
 // 3. Replay missed message history to the client
 func (channel *Channel) Resume(session *Session, timestamp time.Time) {
-	now := time.Now().UTC()
 	channel.resumeAndAnnounce(session)
 	if !timestamp.IsZero() {
-		channel.replayHistoryForResume(session, timestamp, now)
+		channel.replayHistoryForResume(session, timestamp, time.Time{})
 	}
 }
 
 }
 
 func (channel *Channel) replayHistoryForResume(session *Session, after time.Time, before time.Time) {
-	items, complete := channel.history.Between(after, before, false, 0)
+	var items []history.Item
+	var complete bool
+	afterS, beforeS := history.Selector{Time: after}, history.Selector{Time: before}
+	_, seq, _ := channel.server.GetHistorySequence(channel, session.client, "")
+	if seq != nil {
+		items, complete, _ = seq.Between(afterS, beforeS, channel.server.Config().History.ZNCMax)
+	}
 	rb := NewResponseBuffer(session)
 	channel.replayHistoryItems(rb, items, false)
 	if !complete && !session.resumeDetails.HistoryIncomplete {
 		case history.Join:
 			if eventPlayback {
 				if extendedJoin {
-					rb.AddFromClient(item.Message.Time, item.Message.Msgid, item.Nick, item.AccountName, nil, "HEVENT", "JOIN", chname, item.AccountName, item.Params[0])
+					rb.AddFromClient(item.Message.Time, item.Message.Msgid, item.Nick, item.AccountName, nil, "JOIN", chname, item.AccountName, item.Params[0])
 				} else {
-					rb.AddFromClient(item.Message.Time, item.Message.Msgid, item.Nick, item.AccountName, nil, "HEVENT", "JOIN", chname)
+					rb.AddFromClient(item.Message.Time, item.Message.Msgid, item.Nick, item.AccountName, nil, "JOIN", chname)
 				}
 			} else {
 				if !playJoinsAsPrivmsg {
 			}
 		case history.Part:
 			if eventPlayback {
-				rb.AddFromClient(item.Message.Time, item.Message.Msgid, item.Nick, item.AccountName, nil, "HEVENT", "PART", chname, item.Message.Message)
+				rb.AddFromClient(item.Message.Time, item.Message.Msgid, item.Nick, item.AccountName, nil, "PART", chname, item.Message.Message)
 			} else {
 				if !playJoinsAsPrivmsg {
 					continue // #474
 			}
 		case history.Kick:
 			if eventPlayback {
-				rb.AddFromClient(item.Message.Time, item.Message.Msgid, item.Nick, item.AccountName, nil, "HEVENT", "KICK", chname, item.Params[0], item.Message.Message)
+				rb.AddFromClient(item.Message.Time, item.Message.Msgid, item.Nick, item.AccountName, nil, "KICK", chname, item.Params[0], item.Message.Message)
 			} else {
 				message := fmt.Sprintf(client.t("%[1]s kicked %[2]s (%[3]s)"), nick, item.Params[0], item.Message.Message)
 				rb.AddFromClient(item.Message.Time, utils.MungeSecretToken(item.Message.Msgid), histServMask, "*", nil, "PRIVMSG", chname, message)
 			}
 		case history.Quit:
 			if eventPlayback {
-				rb.AddFromClient(item.Message.Time, item.Message.Msgid, item.Nick, item.AccountName, nil, "HEVENT", "QUIT", item.Message.Message)
+				rb.AddFromClient(item.Message.Time, item.Message.Msgid, item.Nick, item.AccountName, nil, "QUIT", item.Message.Message)
 			} else {
 				if !playJoinsAsPrivmsg {
 					continue // #474
 			}
 		case history.Nick:
 			if eventPlayback {
-				rb.AddFromClient(item.Message.Time, item.Message.Msgid, item.Nick, item.AccountName, nil, "HEVENT", "NICK", item.Params[0])
+				rb.AddFromClient(item.Message.Time, item.Message.Msgid, item.Nick, item.AccountName, nil, "NICK", item.Params[0])
 			} else {
 				message := fmt.Sprintf(client.t("%[1]s changed nick to %[2]s"), nick, item.Params[0])
 				rb.AddFromClient(item.Message.Time, utils.MungeSecretToken(item.Message.Msgid), histServMask, "*", nil, "PRIVMSG", chname, message)
 			// STATUSMSG
 			continue
 		}
-		if isCTCP && member.isTor {
-			continue // #753
-		}
 
 		for _, session := range member.Sessions() {
+			if isCTCP && session.isTor {
+				continue // #753
+			}
+
 			var tagsToUse map[string]string
 			if session.capabilities.Has(caps.MessageTags) {
 				tagsToUse = clientOnlyTags
 		}
 	}
 
-	channel.history.Add(history.Item{
+	channel.AddHistoryItem(history.Item{
 		Type:        histType,
 		Message:     message,
 		Nick:        nickmask,
 		Message:     message,
 	}
 	histItem.Params[0] = targetNick
-	channel.history.Add(histItem)
+	channel.AddHistoryItem(histItem)
 
 	channel.Quit(target)
 }

irc/channelreg.go

 	keyChannelModes          = "channel.modes %s"
 	keyChannelAccountToUMode = "channel.accounttoumode %s"
 	keyChannelUserLimit      = "channel.userlimit %s"
+	keyChannelSettings       = "channel.settings %s"
 
 	keyChannelPurged = "channel.purged %s"
 )
 		keyChannelModes,
 		keyChannelAccountToUMode,
 		keyChannelUserLimit,
+		keyChannelSettings,
 	}
 )
 
 	IncludeTopic
 	IncludeModes
 	IncludeLists
+	IncludeSettings
 )
 
 // this is an OR of all possible flags
 	Excepts map[string]MaskInfo
 	// Invites represents the invite exceptions set on the channel.
 	Invites map[string]MaskInfo
+	// Settings are the chanserv-modifiable settings
+	Settings ChannelSettings
 }
 
 type ChannelPurgeRecord struct {
 		exceptlistString, _ := tx.Get(fmt.Sprintf(keyChannelExceptlist, channelKey))
 		invitelistString, _ := tx.Get(fmt.Sprintf(keyChannelInvitelist, channelKey))
 		accountToUModeString, _ := tx.Get(fmt.Sprintf(keyChannelAccountToUMode, channelKey))
+		settingsString, _ := tx.Get(fmt.Sprintf(keyChannelSettings, channelKey))
 
 		modeSlice := make([]modes.Mode, len(modeString))
 		for i, mode := range modeString {
 		accountToUMode := make(map[string]modes.Mode)
 		_ = json.Unmarshal([]byte(accountToUModeString), &accountToUMode)
 
+		var settings ChannelSettings
+		_ = json.Unmarshal([]byte(settingsString), &settings)
+
 		info = RegisteredChannel{
 			Name:           name,
 			RegisteredAt:   time.Unix(regTimeInt, 0).UTC(),
 			Invites:        invitelist,
 			AccountToUMode: accountToUMode,
 			UserLimit:      int(userLimit),
+			Settings:       settings,
 		}
 		return nil
 	})
 		accountToUModeString, _ := json.Marshal(channelInfo.AccountToUMode)
 		tx.Set(fmt.Sprintf(keyChannelAccountToUMode, channelKey), string(accountToUModeString), nil)
 	}
+
+	if includeFlags&IncludeSettings != 0 {
+		settingsString, _ := json.Marshal(channelInfo.Settings)
+		tx.Set(fmt.Sprintf(keyChannelSettings, channelKey), string(settingsString), nil)
+	}
 }
 
 // PurgeChannel records a channel purge.

irc/chanserv.go

 			enabled:   chanregEnabled,
 			minParams: 1,
 		},
+		"get": {
+			handler: csGetHandler,
+			help: `Syntax: $bGET #channel <setting>$b
+
+GET queries the current values of the channel settings. For more information
+on the settings and their possible values, see HELP SET.`,
+			helpShort: `$bGET$b queries the current values of a channel's settings`,
+			enabled:   chanregEnabled,
+			minParams: 2,
+		},
+		"set": {
+			handler:   csSetHandler,
+			helpShort: `$bSET$b modifies a channel's settings`,
+			// these are broken out as separate strings so they can be translated separately
+			helpStrings: []string{
+				`Syntax $bSET #channel <setting> <value>$b
+
+SET modifies a channel's settings. The following settings are available:`,
+
+				`$bHISTORY$b
+'history' lets you control how channel history is stored. Your options are:
+1. 'off'        [no history]
+2. 'ephemeral'  [a limited amount of temporary history, not stored on disk]
+3. 'on'         [history stored in a permanent database, if available]
+4. 'default'    [use the server default]`,
+			},
+			enabled:   chanregEnabled,
+			minParams: 3,
+		},
 	}
 )
 
 	return
 }
 
+func csPrivsCheck(channel RegisteredChannel, client *Client, rb *ResponseBuffer) (success bool) {
+	founder := channel.Founder
+	if founder == "" {
+		csNotice(rb, client.t("That channel is not registered"))
+		return false
+	}
+	if client.HasRoleCapabs("chanreg") {
+		return true
+	}
+	if founder != client.Account() {
+		csNotice(rb, client.t("Insufficient privileges"))
+		return false
+	}
+	return true
+}
+
 func csUnregisterHandler(server *Server, client *Client, command string, params []string, rb *ResponseBuffer) {
 	channelName := params[0]
 	var verificationCode string
 		verificationCode = params[1]
 	}
 
-	channelKey, err := CasefoldChannel(channelName)
-	if channelKey == "" || err != nil {
-		csNotice(rb, client.t("Channel name is not valid"))
-		return
-	}
-
-	channel := server.channels.Get(channelKey)
+	channel := server.channels.Get(channelName)
 	if channel == nil {
 		csNotice(rb, client.t("No such channel"))
 		return
 	}
 
-	founder := channel.Founder()
-	if founder == "" {
-		csNotice(rb, client.t("That channel is not registered"))
-		return
-	}
-
-	hasPrivs := client.HasRoleCapabs("chanreg") || founder == client.Account()
-	if !hasPrivs {
-		csNotice(rb, client.t("Insufficient privileges"))
+	info := channel.ExportRegistration(0)
+	channelKey := info.NameCasefolded
+	if !csPrivsCheck(info, client, rb) {
 		return
 	}
 
-	info := channel.ExportRegistration(0)
 	expectedCode := unregisterConfirmationCode(info.Name, info.RegisteredAt)
 	if expectedCode != verificationCode {
 		csNotice(rb, ircfmt.Unescape(client.t("$bWarning: unregistering this channel will remove all stored channel attributes.$b")))
 		return
 	}
 
-	server.channels.SetUnregistered(channelKey, founder)
+	server.channels.SetUnregistered(channelKey, info.Founder)
 	csNotice(rb, fmt.Sprintf(client.t("Channel %s is now unregistered"), channelKey))
 }
 
 		csNotice(rb, client.t("Channel does not exist"))
 		return
 	}
-	account := client.Account()
-	if !(client.HasRoleCapabs("chanreg") || (account != "" && account == channel.Founder())) {
-		csNotice(rb, client.t("Insufficient privileges"))
+	if !csPrivsCheck(channel.ExportRegistration(0), client, rb) {
 		return
 	}
 
 	csNotice(rb, fmt.Sprintf(client.t("Founder: %s"), chinfo.Founder))
 	csNotice(rb, fmt.Sprintf(client.t("Registered at: %s"), chinfo.RegisteredAt.Format(time.RFC1123)))
 }
+
+func displayChannelSetting(settingName string, settings ChannelSettings, client *Client, rb *ResponseBuffer) {
+	config := client.server.Config()
+
+	switch strings.ToLower(settingName) {
+	case "history":
+		effectiveValue := historyEnabled(config.History.Persistent.RegisteredChannels, settings.History)
+		csNotice(rb, fmt.Sprintf(client.t("The stored channel history setting is: %s"), historyStatusToString(settings.History)))
+		csNotice(rb, fmt.Sprintf(client.t("Given current server settings, the channel history setting is: %s"), historyStatusToString(effectiveValue)))
+	default:
+		csNotice(rb, client.t("Invalid params"))
+	}
+}
+
+func csGetHandler(server *Server, client *Client, command string, params []string, rb *ResponseBuffer) {
+	chname, setting := params[0], params[1]
+	channel := server.channels.Get(chname)
+	if channel == nil {
+		csNotice(rb, client.t("No such channel"))
+		return
+	}
+	info := channel.ExportRegistration(IncludeSettings)
+	if !csPrivsCheck(info, client, rb) {
+		return
+	}
+
+	displayChannelSetting(setting, info.Settings, client, rb)
+}
+
+func csSetHandler(server *Server, client *Client, command string, params []string, rb *ResponseBuffer) {
+	chname, setting, value := params[0], params[1], params[2]
+	channel := server.channels.Get(chname)
+	if channel == nil {
+		csNotice(rb, client.t("No such channel"))
+		return
+	}
+	info := channel.ExportRegistration(IncludeSettings)
+	settings := info.Settings
+	if !csPrivsCheck(info, client, rb) {
+		return
+	}
+
+	var err error
+	switch strings.ToLower(setting) {
+	case "history":
+		settings.History, err = historyStatusFromString(value)
+		if err != nil {
+			err = errInvalidParams
+			break
+		}
+		channel.SetSettings(settings)
+		channel.resizeHistory(server.Config())
+	}
+
+	switch err {
+	case nil:
+		csNotice(rb, client.t("Successfully changed the channel settings"))
+		displayChannelSetting(setting, settings, client, rb)
+	case errInvalidParams:
+		csNotice(rb, client.t("Invalid parameters"))
+	default:
+		server.logger.Error("internal", "CS SET error:", err.Error())
+		csNotice(rb, client.t("An error occurred"))
+	}
+}

irc/client.go

 const (
 	// IdentTimeoutSeconds is how many seconds before our ident (username) check times out.
 	IdentTimeoutSeconds  = 1.5
-	IRCv3TimestampFormat = "2006-01-02T15:04:05.000Z"
+	IRCv3TimestampFormat = utils.IRCv3TimestampFormat
 )
 
 // ResumeDetails is a place to stash data at various stages of
 type Client struct {
 	account            string
 	accountName        string // display name of the account: uncasefolded, '*' if not logged in
+	accountRegDate     time.Time
 	accountSettings    AccountSettings
 	atime              time.Time
 	away               bool
 	ctime              time.Time
 	destroyed          bool
 	exitedSnomaskSent  bool
-	flags              modes.ModeSet
+	modes              modes.ModeSet
 	hostname           string
 	invitedTo          map[string]bool
 	isSTSOnly          bool
-	isTor              bool
 	languages          []string
+	lastSignoff        time.Time // for always-on clients, the time their last session quit
 	loginThrottle      connection_limits.GenericThrottle
 	nick               string
 	nickCasefolded     string
 	skeleton           string
 	sessions           []*Session
 	stateMutex         sync.RWMutex // tier 1
+	alwaysOn           bool
 	username           string
 	vhost              string
 	history            history.Buffer
+	dirtyBits          uint
+	writerSemaphore    utils.Semaphore // tier 1.5
 }
 
 // Session is an individual client connection to the server (TCP connection
 	realIP      net.IP
 	proxiedIP   net.IP
 	rawHostname string
+	isTor       bool
 
 	idletimer IdleTimer
 	fakelag   Fakelag
 	resumeID         string
 	resumeDetails    *ResumeDetails
 	zncPlaybackTimes *zncPlaybackTimes
+	lastSignoff      time.Time
 
 	batch MultilineBatch
 }
 	}
 }
 
+func (s *Session) IP() net.IP {
+	if s.proxiedIP != nil {
+		return s.proxiedIP
+	}
+	return s.realIP
+}
+
 // returns whether the session was actively destroyed (for example, by ping
 // timeout or NS GHOST).
 // avoids a race condition between asynchronous idle-timing-out of sessions,
 // returns whether the client supports a smart history replay cap,
 // and therefore autoreplay-on-join and similar should be suppressed
 func (session *Session) HasHistoryCaps() bool {
-	// TODO the chathistory cap will go here as well
-	return session.capabilities.Has(caps.ZNCPlayback)
+	return session.capabilities.Has(caps.Chathistory) || session.capabilities.Has(caps.ZNCPlayback)
 }
 
 // generates a batch ID. the uniqueness requirements for this are fairly weak:
 		channels:  make(ChannelSet),
 		ctime:     now,
 		isSTSOnly: conn.Config.STSOnly,
-		isTor:     conn.Config.Tor,
 		languages: server.Languages().Default(),
 		loginThrottle: connection_limits.GenericThrottle{
 			Duration: config.Accounts.LoginThrottling.Duration,
 		ctime:      now,
 		atime:      now,
 		realIP:     realIP,
+		isTor:      conn.Config.Tor,
 	}
 	client.sessions = []*Session{session}
 
 		client.rawHostname = session.rawHostname
 	} else {
 		remoteAddr := conn.Conn.RemoteAddr()
-		if utils.AddrIsLocal(remoteAddr) {
+		if realIP.IsLoopback() || utils.IPInNets(realIP, config.Server.secureNets) {
 			// treat local connections as secure (may be overridden later by WEBIRC)
 			client.SetMode(modes.TLS, true)
 		}
 	client.run(session, proxyLine)
 }
 
+func (server *Server) AddAlwaysOnClient(account ClientAccount, chnames []string) {
+	now := time.Now().UTC()
+	config := server.Config()
+
+	client := &Client{
+		atime:     now,
+		channels:  make(ChannelSet),
+		ctime:     now,
+		languages: server.Languages().Default(),
+		server:    server,
+
+		// TODO figure out how to set these on reattach?
+		username:    "~user",
+		rawHostname: server.name,
+		realIP:      utils.IPv4LoopbackAddress,
+
+		alwaysOn: true,
+	}
+
+	client.SetMode(modes.TLS, true)
+	client.writerSemaphore.Initialize(1)
+	client.history.Initialize(0, 0)
+	client.brbTimer.Initialize(client)
+
+	server.accounts.Login(client, account)
+
+	client.resizeHistory(config)
+
+	_, err := server.clients.SetNick(client, nil, account.Name)
+	if err != nil {
+		server.logger.Error("internal", "could not establish always-on client", account.Name, err.Error())
+		return
+	} else {
+		server.logger.Debug("accounts", "established always-on client", account.Name)
+	}
+
+	// XXX set this last to avoid confusing SetNick:
+	client.registered = true
+
+	for _, chname := range chnames {
+		// XXX we're using isSajoin=true, to make these joins succeed even without channel key
+		// this is *probably* ok as long as the persisted memberships are accurate
+		server.channels.Join(client, chname, "", true, nil)
+	}
+}
+
+func (client *Client) resizeHistory(config *Config) {
+	_, ephemeral := client.historyStatus(config)
+	if ephemeral {
+		client.history.Resize(config.History.ClientLength, config.History.AutoresizeWindow)
+	} else {
+		client.history.Resize(0, 0)
+	}
+}
+
 // resolve an IP to an IRC-ready hostname, using reverse DNS, forward-confirming if necessary,
 // and sending appropriate notices to the client
 func (client *Client) lookupHostname(session *Session, overwrite bool) {
-	if client.isTor {
+	if session.isTor {
 		return
 	} // else: even if cloaking is enabled, look up the real hostname to show to operators
 
 	authFailSaslRequired
 )
 
-func (client *Client) isAuthorized(config *Config) AuthOutcome {
+func (client *Client) isAuthorized(config *Config, isTor bool) AuthOutcome {
 	saslSent := client.account != ""
 	// PASS requirement
 	if (config.Server.passwordBytes != nil) && !client.sentPassCommand && !(config.Accounts.SkipServerPassword && saslSent) {
 		return authFailPass
 	}
 	// Tor connections may be required to authenticate with SASL
-	if client.isTor && config.Server.TorListeners.RequireSasl && !saslSent {
+	if isTor && config.Server.TorListeners.RequireSasl && !saslSent {
 		return authFailTorSaslRequired
 	}
 	// finally, enforce require-sasl
 
 func (client *Client) playReattachMessages(session *Session) {
 	client.server.playRegistrationBurst(session)
+	hasHistoryCaps := session.HasHistoryCaps()
 	for _, channel := range session.client.Channels() {
 		channel.playJoinForSession(session)
-		// clients should receive autoreplay-on-join lines, if applicable;
+		// clients should receive autoreplay-on-join lines, if applicable:
+		if hasHistoryCaps {
+			continue
+		}
 		// if they negotiated znc.in/playback or chathistory, they will receive nothing,
 		// because those caps disable autoreplay-on-join and they haven't sent the relevant
 		// *playback PRIVMSG or CHATHISTORY command yet
 		channel.autoReplayHistory(client, rb, "")
 		rb.Send(true)
 	}
+	if !session.lastSignoff.IsZero() && !hasHistoryCaps {
+		rb := NewResponseBuffer(session)
+		zncPlayPrivmsgs(client, rb, session.lastSignoff, time.Time{})
+		rb.Send(true)
+	}
+	session.lastSignoff = time.Time{}
 }
 
 //
 		return
 	}
 
-	if oldClient.isTor != client.isTor {
-		session.Send(nil, server.name, "FAIL", "RESUME", "INSECURE_SESSION", client.t("Cannot resume connection from Tor to non-Tor or vice versa"))
-		return
-	}
-
 	err := server.clients.Resume(oldClient, session)
 	if err != nil {
 		session.Send(nil, server.name, "FAIL", "RESUME", "CANNOT_RESUME", client.t("Cannot resume connection"))
 func (session *Session) playResume() {
 	client := session.client
 	server := client.server
+	config := server.Config()
 
 	friends := make(ClientSet)
-	oldestLostMessage := time.Now().UTC()
+	var oldestLostMessage time.Time
 
 	// work out how much time, if any, is not covered by history buffers
+	// assume that a persistent buffer covers the whole resume period
 	for _, channel := range client.Channels() {
 		for _, member := range channel.Members() {
 			friends.Add(member)
+		}
+		_, ephemeral, _ := channel.historyStatus(config)
+		if ephemeral {
 			lastDiscarded := channel.history.LastDiscarded()
-			if lastDiscarded.Before(oldestLostMessage) {
+			if oldestLostMessage.Before(lastDiscarded) {
 				oldestLostMessage = lastDiscarded
 			}
 		}
 	}
-	privmsgMatcher := func(item history.Item) bool {
-		return item.Type == history.Privmsg || item.Type == history.Notice || item.Type == history.Tagmsg
-	}
-	privmsgHistory := client.history.Match(privmsgMatcher, false, 0)
-	lastDiscarded := client.history.LastDiscarded()
-	if lastDiscarded.Before(oldestLostMessage) {
-		oldestLostMessage = lastDiscarded
+	_, cEphemeral := client.historyStatus(config)
+	if cEphemeral {
+		lastDiscarded := client.history.LastDiscarded()
+		if oldestLostMessage.Before(lastDiscarded) {
+			oldestLostMessage = lastDiscarded
+		}
 	}
-	for _, item := range privmsgHistory {
-		sender := server.clients.Get(stripMaskFromNick(item.Nick))
-		if sender != nil {
-			friends.Add(sender)
+	_, privmsgSeq, _ := server.GetHistorySequence(nil, client, "*")
+	if privmsgSeq != nil {
+		privmsgs, _, _ := privmsgSeq.Between(history.Selector{}, history.Selector{}, config.History.ClientLength)
+		for _, item := range privmsgs {
+			sender := server.clients.Get(stripMaskFromNick(item.Nick))
+			if sender != nil {
+				friends.Add(sender)
+			}
 		}
 	}
 
 	timestamp := session.resumeDetails.Timestamp
-	gap := lastDiscarded.Sub(timestamp)
+	gap := oldestLostMessage.Sub(timestamp)
 	session.resumeDetails.HistoryIncomplete = gap > 0 || timestamp.IsZero()
 	gapSeconds := int(gap.Seconds()) + 1 // round up to avoid confusion
 
 		}
 	}
 
-	if session.resumeDetails.HistoryIncomplete && !timestamp.IsZero() {
-		session.Send(nil, client.server.name, "WARN", "RESUME", "HISTORY_LOST", fmt.Sprintf(client.t("Resume may have lost up to %d seconds of history"), gapSeconds))
-	} else {
-		session.Send(nil, client.server.name, "WARN", "RESUME", "HISTORY_LOST", client.t("Resume may have lost some message history"))
+	if session.resumeDetails.HistoryIncomplete {
+		if !timestamp.IsZero() {
+			session.Send(nil, client.server.name, "WARN", "RESUME", "HISTORY_LOST", fmt.Sprintf(client.t("Resume may have lost up to %d seconds of history"), gapSeconds))
+		} else {
+			session.Send(nil, client.server.name, "WARN", "RESUME", "HISTORY_LOST", client.t("Resume may have lost some message history"))
+		}
 	}
 
 	session.Send(nil, client.server.name, "RESUME", "SUCCESS", details.nick)
 	}
 
 	// replay direct PRIVSMG history
-	if !timestamp.IsZero() {
-		now := time.Now().UTC()
-		items, complete := client.history.Between(timestamp, now, false, 0)
-		rb := NewResponseBuffer(client.Sessions()[0])
-		client.replayPrivmsgHistory(rb, items, complete)
+	if !timestamp.IsZero() && privmsgSeq != nil {
+		after := history.Selector{Time: timestamp}
+		items, complete, _ := privmsgSeq.Between(after, history.Selector{}, config.History.ZNCMax)
+		rb := NewResponseBuffer(session)
+		client.replayPrivmsgHistory(rb, items, "", complete)
 		rb.Send(true)
 	}
 
 	session.resumeDetails = nil
 }
 
-func (client *Client) replayPrivmsgHistory(rb *ResponseBuffer, items []history.Item, complete bool) {
+func (client *Client) replayPrivmsgHistory(rb *ResponseBuffer, items []history.Item, target string, complete bool) {
 	var batchID string
 	details := client.Details()
 	nick := details.nick
 	if 0 < len(items) {
-		batchID = rb.StartNestedHistoryBatch(nick)
+		if target == "" {
+			target = nick
+		}
+		batchID = rb.StartNestedHistoryBatch(target)
 	}
 
 	allowTags := rb.session.capabilities.Has(caps.MessageTags)
 		if allowTags {
 			tags = item.Tags
 		}
-		if item.Params[0] == "" {
+		if item.Params[0] == "" || item.Params[0] == nick {
 			// this message was sent *to* the client from another nick
 			rb.AddSplitMessageFromClient(item.Nick, item.AccountName, tags, command, nick, item.Message)
 		} else {
 			// this message was sent *from* the client to another nick; the target is item.Params[0]
-			// substitute the client's current nickmask in case they changed nick
+			// substitute client's current nickmask in case client changed nick
 			rb.AddSplitMessageFromClient(details.nickMask, item.AccountName, tags, command, item.Params[0], item.Message)
 		}
 	}
 
 // ModeString returns the mode string for this client.
 func (client *Client) ModeString() (str string) {
-	return "+" + client.flags.String()
+	return "+" + client.modes.String()
 }
 
 // Friends refers to clients that share a channel with this client.
 // has no more sessions.
 func (client *Client) destroy(session *Session) {
 	var sessionsToDestroy []*Session
+	var lastSignoff time.Time
+	if session != nil {
+		lastSignoff = session.idletimer.LastTouch()
+	} else {
+		lastSignoff = time.Now().UTC()
+	}
 
 	client.stateMutex.Lock()
 	details := client.detailsNoMutex()
 	brbAt := client.brbTimer.brbAt
 	wasReattach := session != nil && session.client != client
 	sessionRemoved := false
+	registered := client.registered
+	alwaysOn := client.alwaysOn
 	var remainingSessions int
 	if session == nil {
 		sessionsToDestroy = client.sessions
 
 	// should we destroy the whole client this time?
 	// BRB is not respected if this is a destroy of the whole client (i.e., session == nil)
-	brbEligible := session != nil && (brbState == BrbEnabled || brbState == BrbSticky)
+	brbEligible := session != nil && (brbState == BrbEnabled || alwaysOn)
 	shouldDestroy := !client.destroyed && remainingSessions == 0 && !brbEligible
 	if shouldDestroy {
 		// if it's our job to destroy it, don't let anyone else try
 		client.destroyed = true
 	}
+	if alwaysOn && remainingSessions == 0 {
+		client.lastSignoff = lastSignoff
+	}
 	exitedSnomaskSent := client.exitedSnomaskSent
 	client.stateMutex.Unlock()
 
 
 		// remove from connection limits
 		var source string
-		if client.isTor {
+		if session.isTor {
 			client.server.torLimiter.RemoveClient()
 			source = "tor"
 		} else {
 		client.server.logger.Info("localconnect-ip", fmt.Sprintf("disconnecting session of %s from %s", details.nick, source))
 	}
 
+	// decrement stats if we have no more sessions, even if the client will not be destroyed
+	if shouldDestroy || remainingSessions == 0 {
+		invisible := client.HasMode(modes.Invisible)
+		operator := client.HasMode(modes.LocalOperator) || client.HasMode(modes.Operator)
+		client.server.stats.Remove(registered, invisible, operator)
+	}
+
 	// do not destroy the client if it has either remaining sessions, or is BRB'ed
 	if !shouldDestroy {
 		return
 	}
 
+	splitQuitMessage := utils.MakeMessage(quitMessage)
+	quitItem := history.Item{
+		Type:        history.Quit,
+		Nick:        details.nickMask,
+		AccountName: details.accountName,
+		Message:     splitQuitMessage,
+	}
+	var channels []*Channel
+	defer func() {
+		for _, channel := range channels {
+			// TODO it's dangerous to write to mysql while holding the destroy semaphore
+			channel.AddHistoryItem(quitItem)
+		}
+	}()
+
 	// see #235: deduplicating the list of PART recipients uses (comparatively speaking)
 	// a lot of RAM, so limit concurrency to avoid thrashing
 	client.server.semaphores.ClientDestroy.Acquire()
 		client.server.logger.Debug("quit", fmt.Sprintf("%s is no longer on the server", details.nick))
 	}
 
-	registered := client.Registered()
 	if registered {
 		client.server.whoWas.Append(client.WhoWas())
 	}
 	// clean up monitor state
 	client.server.monitorManager.RemoveAll(client)
 
-	splitQuitMessage := utils.MakeMessage(quitMessage)
 	// clean up channels
 	// (note that if this is a reattach, client has no channels and therefore no friends)
 	friends := make(ClientSet)
-	for _, channel := range client.Channels() {
+	channels = client.Channels()
+	for _, channel := range channels {
 		channel.Quit(client)
-		channel.history.Add(history.Item{
-			Type:        history.Quit,
-			Nick:        details.nickMask,
-			AccountName: details.accountName,
-			Message:     splitQuitMessage,
-		})
 		for _, member := range channel.Members() {
 			friends.Add(member)
 		}
 
 	client.server.accounts.Logout(client)
 
-	client.server.stats.Remove(registered, client.HasMode(modes.Invisible),
-		client.HasMode(modes.Operator) || client.HasMode(modes.LocalOperator))
-
 	// this happens under failure to return from BRB
 	if quitMessage == "" {
 		if brbState == BrbDead && !brbAt.IsZero() {
 // SendSplitMsgFromClient sends an IRC PRIVMSG/NOTICE coming from a specific client.
 // Adds account-tag to the line as well.
 func (session *Session) sendSplitMsgFromClientInternal(blocking bool, nickmask, accountName string, tags map[string]string, command, target string, message utils.SplitMessage) {
-	// TODO no maxline support
 	if message.Is512() {
 		session.sendFromClientInternal(blocking, message.Time, message.Msgid, nickmask, accountName, tags, command, target, message.Message)
 	} else {
-		if message.IsMultiline() && session.capabilities.Has(caps.Multiline) {
+		if session.capabilities.Has(caps.Multiline) {
 			for _, msg := range session.composeMultilineBatch(nickmask, accountName, tags, command, target, message) {
 				session.SendRawMessage(msg, blocking)
 			}
 func (client *Client) addChannel(channel *Channel) {
 	client.stateMutex.Lock()
 	client.channels[channel] = true
+	alwaysOn := client.alwaysOn
 	client.stateMutex.Unlock()
+
+	if alwaysOn {
+		client.markDirty(IncludeChannels)
+	}
 }
 
 func (client *Client) removeChannel(channel *Channel) {
 	client.stateMutex.Lock()
 	delete(client.channels, channel)
+	alwaysOn := client.alwaysOn
 	client.stateMutex.Unlock()
+
+	if alwaysOn {
+		client.markDirty(IncludeChannels)
+	}
 }
 
 // Records that the client has been invited to join an invite-only channel
 		}
 	}
 }
+
+func (client *Client) historyStatus(config *Config) (persistent, ephemeral bool) {
+	if !config.History.Enabled {
+		return false, false
+	} else if !config.History.Persistent.Enabled {
+		return false, true
+	}
+
+	client.stateMutex.RLock()
+	alwaysOn := client.alwaysOn
+	historyStatus := client.accountSettings.DMHistory
+	client.stateMutex.RUnlock()
+
+	if !alwaysOn {
+		return false, true
+	}
+
+	historyStatus = historyEnabled(config.History.Persistent.DirectMessages, historyStatus)
+	ephemeral = (historyStatus == HistoryEphemeral)
+	persistent = (historyStatus == HistoryPersistent)
+	return
+}
+
+// these are bit flags indicating what part of the client status is "dirty"
+// and needs to be read from memory and written to the db
+// TODO add a dirty flag for lastSignoff
+const (
+	IncludeChannels uint = 1 << iota
+)
+
+func (client *Client) markDirty(dirtyBits uint) {
+	client.stateMutex.Lock()
+	alwaysOn := client.alwaysOn
+	client.dirtyBits = client.dirtyBits | dirtyBits
+	client.stateMutex.Unlock()
+
+	if alwaysOn {
+		client.wakeWriter()
+	}
+}
+
+func (client *Client) wakeWriter() {
+	if client.writerSemaphore.TryAcquire() {
+		go client.writeLoop()
+	}
+}
+
+func (client *Client) writeLoop() {
+	for {
+		client.performWrite()
+		client.writerSemaphore.Release()
+
+		client.stateMutex.RLock()
+		isDirty := client.dirtyBits != 0
+		client.stateMutex.RUnlock()
+
+		if !isDirty || !client.writerSemaphore.TryAcquire() {
+			return
+		}
+	}
+}
+
+func (client *Client) performWrite() {
+	client.stateMutex.Lock()
+	// TODO actually read dirtyBits in the future
+	client.dirtyBits = 0
+	account := client.account
+	client.stateMutex.Unlock()
+
+	if account == "" {
+		client.server.logger.Error("internal", "attempting to persist logged-out client", client.Nick())
+		return
+	}
+
+	channels := client.Channels()
+	channelNames := make([]string, len(channels))
+	for i, channel := range channels {
+		channelNames[i] = channel.Name()
+	}
+	client.server.accounts.saveChannels(account, channelNames)
+}

irc/client_lookup_set.go

 		return errNickMissing
 	}
 
-	if !oldClient.AddSession(session) {
+	success, _, _ := oldClient.AddSession(session)
+	if !success {
 		return errNickMissing
 	}
 
 }
 
 // SetNick sets a client's nickname, validating it against nicknames in use
-func (clients *ClientManager) SetNick(client *Client, session *Session, newNick string) error {
-	if len(newNick) > client.server.Config().Limits.NickLen {
-		return errNicknameInvalid
-	}
+func (clients *ClientManager) SetNick(client *Client, session *Session, newNick string) (setNick string, err error) {
 	newcfnick, err := CasefoldName(newNick)
 	if err != nil {
-		return errNicknameInvalid
+		return "", errNicknameInvalid
+	}
+	if len(newcfnick) > client.server.Config().Limits.NickLen {
+		return "", errNicknameInvalid
 	}
 	newSkeleton, err := Skeleton(newNick)
 	if err != nil {
-		return errNicknameInvalid
+		return "", errNicknameInvalid
 	}
 
 	if restrictedCasefoldedNicks[newcfnick] || restrictedSkeletons[newSkeleton] {
-		return errNicknameInvalid
+		return "", errNicknameInvalid
 	}
 
 	reservedAccount, method := client.server.accounts.EnforcementStatus(newcfnick, newSkeleton)
-	account := client.Account()
 	config := client.server.Config()
+	client.stateMutex.RLock()
+	account := client.account
+	accountName := client.accountName
+	settings := client.accountSettings
+	registered := client.registered
+	realname := client.realname
+	client.stateMutex.RUnlock()
+
+	// recompute this (client.alwaysOn is not set for unregistered clients):
+	alwaysOn := account != "" && persistenceEnabled(config.Accounts.Bouncer.AlwaysOn, settings.AlwaysOn)
+
+	if alwaysOn && registered {
+		return "", errCantChangeNick
+	}
+
 	var bouncerAllowed bool
 	if config.Accounts.Bouncer.Enabled {
-		if session != nil && session.capabilities.Has(caps.Bouncer) {
+		if alwaysOn {
+			// ignore the pre-reg nick, force a reattach
+			newNick = accountName
+			newcfnick = account
+			bouncerAllowed = true
+		} else if session != nil && session.capabilities.Has(caps.Bouncer) {
 			bouncerAllowed = true
 		} else {
-			settings := client.AccountSettings()
 			if config.Accounts.Bouncer.AllowedByDefault && settings.AllowBouncer != BouncerDisallowedByUser {
 				bouncerAllowed = true
 			} else if settings.AllowBouncer == BouncerAllowedByUser {
 	// the client may just be changing case
 	if currentClient != nil && currentClient != client && session != nil {
 		// these conditions forbid reattaching to an existing session:
-		if client.Registered() || !bouncerAllowed || account == "" || account != currentClient.Account() || client.HasMode(modes.TLS) != currentClient.HasMode(modes.TLS) {
-			return errNicknameInUse
+		if registered || !bouncerAllowed || account == "" || account != currentClient.Account() || client.HasMode(modes.TLS) != currentClient.HasMode(modes.TLS) {
+			return "", errNicknameInUse
+		}
+		reattachSuccessful, numSessions, lastSignoff := currentClient.AddSession(session)
+		if !reattachSuccessful {
+			return "", errNicknameInUse
 		}
-		if !currentClient.AddSession(session) {
-			return errNicknameInUse
+		if numSessions == 1 {
+			invisible := client.HasMode(modes.Invisible)
+			operator := client.HasMode(modes.Operator) || client.HasMode(modes.LocalOperator)
+			client.server.stats.AddRegistered(invisible, operator)
 		}
+		session.lastSignoff = lastSignoff
+		// XXX SetNames only changes names if they are unset, so the realname change only
+		// takes effect on first attach to an always-on client (good), but the user/ident
+		// change is always a no-op (bad). we could make user/ident act the same way as
+		// realname, but then we'd have to send CHGHOST and i don't want to deal with that
+		// for performance reasons
+		currentClient.SetNames("user", realname, true)
 		// successful reattach!
-		return nil
+		return newNick, nil
 	}
 	// analogous checks for skeletons
 	skeletonHolder := clients.bySkeleton[newSkeleton]
 	if skeletonHolder != nil && skeletonHolder != client {
-		return errNicknameInUse
+		return "", errNicknameInUse
 	}
 	if method == NickEnforcementStrict && reservedAccount != "" && reservedAccount != account {
-		return errNicknameReserved
+		return "", errNicknameReserved
 	}
 	clients.removeInternal(client)
 	clients.byNick[newcfnick] = client
 	clients.bySkeleton[newSkeleton] = client
 	client.updateNick(newNick, newcfnick, newSkeleton)
-	return nil
+	return newNick, nil
 }
 
 func (clients *ClientManager) AllClients() (result []*Client) {

irc/commands.go

 		return cmd.handler(server, client, msg, rb)
 	}()
 
+	// most servers do this only for PING/PONG, but we'll do it for any command:
+	if client.registered {
+		// touch even if `exiting`, so we record the time of a QUIT accurately
+		session.idletimer.Touch()
+	}
+
 	if exiting {
 		return
 	}
 		exiting = server.tryRegister(client, session)
 	}
 
-	// most servers do this only for PING/PONG, but we'll do it for any command:
-	if client.registered {
-		session.idletimer.Touch()
-	}
-
 	if client.registered && !cmd.leaveClientIdle {
 		client.Active(session)
 	}
 		},
 		"CHATHISTORY": {
 			handler:   chathistoryHandler,
-			minParams: 3,
+			minParams: 4,
 		},
 		"DEBUG": {
 			handler:   debugHandler,

irc/config.go

 	ProxyBeforeTLS bool
 }
 
+type PersistentStatus uint
+
+const (
+	PersistentUnspecified PersistentStatus = iota
+	PersistentDisabled
+	PersistentOptIn
+	PersistentOptOut
+	PersistentMandatory
+)
+
+func persistentStatusToString(status PersistentStatus) string {
+	switch status {
+	case PersistentUnspecified:
+		return "default"
+	case PersistentDisabled:
+		return "disabled"
+	case PersistentOptIn:
+		return "opt-in"
+	case PersistentOptOut:
+		return "opt-out"
+	case PersistentMandatory:
+		return "mandatory"
+	default:
+		return ""
+	}
+}
+
+func persistentStatusFromString(status string) (PersistentStatus, error) {
+	switch strings.ToLower(status) {
+	case "default":
+		return PersistentUnspecified, nil
+	case "":
+		return PersistentDisabled, nil
+	case "opt-in":
+		return PersistentOptIn, nil
+	case "opt-out":
+		return PersistentOptOut, nil
+	case "mandatory":
+		return PersistentMandatory, nil
+	default:
+		b, err := utils.StringToBool(status)
+		if b {
+			return PersistentMandatory, err
+		} else {
+			return PersistentDisabled, err
+		}
+	}
+}
+
+func (ps *PersistentStatus) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	var orig string
+	var err error
+	if err = unmarshal(&orig); err != nil {
+		return err
+	}
+	result, err := persistentStatusFromString(orig)
+	if err == nil {
+		if result == PersistentUnspecified {
+			result = PersistentDisabled
+		}
+		*ps = result
+	}
+	return err
+}
+
+func persistenceEnabled(serverSetting, clientSetting PersistentStatus) (enabled bool) {
+	if serverSetting == PersistentDisabled {
+		return false
+	} else if serverSetting == PersistentMandatory {
+		return true
+	} else if clientSetting == PersistentDisabled {
+		return false
+	} else if clientSetting == PersistentMandatory {
+		return true
+	} else if serverSetting == PersistentOptOut {
+		return true
+	} else {
+		return false
+	}
+}
+
+type HistoryStatus uint
+
+const (
+	HistoryDefault HistoryStatus = iota
+	HistoryDisabled
+	HistoryEphemeral
+	HistoryPersistent
+)
+
+func historyStatusFromString(str string) (status HistoryStatus, err error) {
+	switch strings.ToLower(str) {
+	case "default":
+		return HistoryDefault, nil
+	case "ephemeral":
+		return HistoryEphemeral, nil
+	case "persistent":
+		return HistoryPersistent, nil
+	default:
+		b, err := utils.StringToBool(str)
+		if b {
+			return HistoryPersistent, err
+		} else {
+			return HistoryDisabled, err
+		}
+	}
+}
+
+func historyStatusToString(status HistoryStatus) string {
+	switch status {
+	case HistoryDefault:
+		return "default"
+	case HistoryDisabled:
+		return "disabled"
+	case HistoryEphemeral:
+		return "ephemeral"
+	case HistoryPersistent:
+		return "persistent"
+	default:
+		return ""
+	}
+}
+
+func historyEnabled(serverSetting PersistentStatus, localSetting HistoryStatus) (result HistoryStatus) {
+	if serverSetting == PersistentDisabled {
+		return HistoryDisabled
+	} else if serverSetting == PersistentMandatory {
+		return HistoryPersistent
+	} else if serverSetting == PersistentOptOut {
+		if localSetting == HistoryDefault {
+			return HistoryPersistent
+		} else {
+			return localSetting
+		}
+	} else if serverSetting == PersistentOptIn {
+		if localSetting >= HistoryEphemeral {
+			return localSetting
+		} else {
+			return HistoryDisabled
+		}
+	} else {
+		return HistoryDisabled
+	}
+}
+
 type AccountConfig struct {
 	Registration          AccountRegistrationConfig
 	AuthenticationEnabled bool `yaml:"authentication-enabled"`
 	NickReservation    NickReservationConfig `yaml:"nick-reservation"`
 	Bouncer            struct {
 		Enabled          bool
-		AllowedByDefault bool `yaml:"allowed-by-default"`
+		AllowedByDefault bool             `yaml:"allowed-by-default"`
+		AlwaysOn         PersistentStatus `yaml:"always-on"`
 	}
 	VHosts VHostConfig
 }
 		isupport      isupport.List
 		IPLimits      connection_limits.LimiterConfig `yaml:"ip-limits"`
 		Cloaks        cloaks.CloakConfig              `yaml:"ip-cloaking"`
+		SecureNetDefs []string                        `yaml:"secure-nets"`
+		secureNets    []net.IPNet
 		supportedCaps *caps.Set
 		capValues     caps.Values
 		Casemapping   Casemapping
 	Datastore struct {
 		Path        string
 		AutoUpgrade bool
+		MySQL       struct {
+			Enabled         bool
+			Host            string
+			Port            int
+			User            string
+			Password        string
+			HistoryDatabase string `yaml:"history-database"`
+		}
 	}
 
 	Accounts AccountConfig
 		AutoresizeWindow time.Duration `yaml:"autoresize-window"`
 		AutoreplayOnJoin int           `yaml:"autoreplay-on-join"`
 		ChathistoryMax   int           `yaml:"chathistory-maxmessages"`
+		ZNCMax           int           `yaml:"znc-maxmessages"`
+		Restrictions     struct {
+			ExpireTime              time.Duration `yaml:"expire-time"`
+			EnforceRegistrationDate bool          `yaml:"enforce-registration-date"`
+			GracePeriod             time.Duration `yaml:"grace-period"`
+		}
+		Persistent struct {
+			Enabled              bool
+			UnregisteredChannels bool             `yaml:"unregistered-channels"`
+			RegisteredChannels   PersistentStatus `yaml:"registered-channels"`
+			DirectMessages       PersistentStatus `yaml:"direct-messages"`
+		}
 	}
 
 	Filename string
 	}
 
 	if !config.Accounts.Bouncer.Enabled {
+		config.Accounts.Bouncer.AlwaysOn = PersistentDisabled
 		config.Server.supportedCaps.Disable(caps.Bouncer)
+	} else if config.Accounts.Bouncer.AlwaysOn >= PersistentOptOut {
+		config.Accounts.Bouncer.AllowedByDefault = true
 	}
 
 	var newLogConfigs []logger.LoggingConfig
 		return nil, fmt.Errorf("Could not parse proxy-allowed-from nets: %v", err.Error())
 	}
 
+	config.Server.secureNets, err = utils.ParseNetList(config.Server.SecureNetDefs)
+	if err != nil {
+		return nil, fmt.Errorf("Could not parse secure-nets: %v\n", err.Error())
+	}
+
 	rawRegexp := config.Accounts.VHosts.ValidRegexpRaw
 	if rawRegexp != "" {
 		regexp, err := regexp.Compile(rawRegexp)
 		config.History.ClientLength = 0
 	}
 
+	if !config.History.Enabled || !config.History.Persistent.Enabled {
+		config.History.Persistent.UnregisteredChannels = false
+		config.History.Persistent.RegisteredChannels = PersistentDisabled
+		config.History.Persistent.DirectMessages = PersistentDisabled
+	}
+
+	if config.History.ZNCMax == 0 {
+		config.History.ZNCMax = config.History.ChathistoryMax
+	}
+
 	config.Server.Cloaks.Initialize()
 	if config.Server.Cloaks.Enabled {
 		if config.Server.Cloaks.Secret == "" || config.Server.Cloaks.Secret == "siaELnk6Kaeo65K3RCrwJjlWaZ-Bt3WuZ2L8MXLbNb4" {

irc/database.go

 // maps an initial version to a schema change capable of upgrading it
 var schemaChanges map[string]SchemaChange
 
-type incompatibleSchemaError struct {
-	currentVersion  string
-	requiredVersion string
-}
-
-func IncompatibleSchemaError(currentVersion string) (result *incompatibleSchemaError) {
-	return &incompatibleSchemaError{
-		currentVersion:  currentVersion,
-		requiredVersion: latestDbSchema,
-	}
-}
-
-func (err *incompatibleSchemaError) Error() string {
-	return fmt.Sprintf("Database requires update. Expected schema v%s, got v%s", err.requiredVersion, err.currentVersion)
-}
-
 // InitDB creates the database, implementing the `oragono initdb` command.
 func InitDB(path string) {
 	_, err := os.Stat(path)
 		// successful autoupgrade, let's try this again:
 		return openDatabaseInternal(config, false)
 	} else {
-		err = IncompatibleSchemaError(version)
+		err = &utils.IncompatibleSchemaError{CurrentVersion: version, RequiredVersion: latestDbSchema}
 		return
 	}
 }
 					break
 				}
 				// unable to upgrade to the desired version, roll back
-				return IncompatibleSchemaError(version)
+				return &utils.IncompatibleSchemaError{CurrentVersion: version, RequiredVersion: latestDbSchema}
 			}
 			log.Println("attempting to update schema from version " + version)
 			err := change.Changer(config, tx)

irc/errors.go

 	errChannelNotOwnedByAccount       = errors.New("Channel not owned by the specified account")
 	errChannelTransferNotOffered      = errors.New(`You weren't offered ownership of that channel`)
 	errChannelAlreadyRegistered       = errors.New("Channel is already registered")
+	errChannelNotRegistered           = errors.New("Channel is not registered")
 	errChannelNameInUse               = errors.New(`Channel name in use`)
 	errInvalidChannelName             = errors.New(`Invalid channel name`)
 	errMonitorLimitExceeded           = errors.New("Monitor limit exceeded")
 	errNicknameInvalid                = errors.New("invalid nickname")
 	errNicknameInUse                  = errors.New("nickname in use")
 	errNicknameReserved               = errors.New("nickname is reserved")
+	errCantChangeNick                 = errors.New(`Always-on clients can't change nicknames`)
 	errNoExistingBan                  = errors.New("Ban does not exist")
 	errNoSuchChannel                  = errors.New(`No such channel`)
 	errChannelPurged                  = errors.New(`This channel was purged by the server operators and cannot be used`)

irc/gateways.go

 func (client *Client) ApplyProxiedIP(session *Session, proxiedIP string, tls bool) (err error, quitMsg string) {
 	// PROXY and WEBIRC are never accepted from a Tor listener, even if the address itself
 	// is whitelisted:
-	if client.isTor {
+	if session.isTor {
 		return errBadProxyLine, ""
 	}
 

irc/getters.go

 	return
 }
 
-func (client *Client) AddSession(session *Session) (success bool) {
+func (client *Client) AddSession(session *Session) (success bool, numSessions int, lastSignoff time.Time) {
 	client.stateMutex.Lock()
 	defer client.stateMutex.Unlock()
 
 	// client may be dying and ineligible to receive another session
 	if client.destroyed {
-		return false
+		return
 	}
 	// success, attach the new session to the client
 	session.client = client
 	newSessions := make([]*Session, len(client.sessions)+1)
 	copy(newSessions, client.sessions)
 	newSessions[len(newSessions)-1] = session
+	if len(client.sessions) == 0 && client.accountSettings.AutoreplayMissed {
+		// n.b. this is only possible if client is persistent and remained
+		// on the server with no sessions:
+		lastSignoff = client.lastSignoff
+		client.lastSignoff = time.Time{}
+	}
 	client.sessions = newSessions
-	return true
+	return true, len(client.sessions), lastSignoff
 }
 
 func (client *Client) removeSession(session *Session) (success bool, length int) {
 	client.stateMutex.Unlock()
 }
 
+func (client *Client) AlwaysOn() (alwaysOn bool) {
+	client.stateMutex.Lock()
+	alwaysOn = client.alwaysOn
+	client.stateMutex.Unlock()
+	return
+}
+
 // uniqueIdentifiers returns the strings for which the server enforces per-client
 // uniqueness/ownership; no two clients can have colliding casefolded nicks or
 // skeletons.
 	return client.accountName
 }
 
-func (client *Client) SetAccountName(account string) (changed bool) {
-	var casefoldedAccount string
-	var err error
-	if account != "" {
-		if casefoldedAccount, err = CasefoldName(account); err != nil {
-			return
-		}
+func (client *Client) Login(account ClientAccount) {
+	alwaysOn := persistenceEnabled(client.server.Config().Accounts.Bouncer.AlwaysOn, account.Settings.AlwaysOn)
+	client.stateMutex.Lock()
+	defer client.stateMutex.Unlock()
+	client.account = account.NameCasefolded
+	client.accountName = account.Name
+	client.accountSettings = account.Settings
+	// check `registered` to avoid incorrectly marking a temporary (pre-reattach),
+	// SASL'ing client as always-on
+	if client.registered {
+		client.alwaysOn = alwaysOn
 	}
+	client.accountRegDate = account.RegisteredAt
+	return
+}
 
+func (client *Client) historyCutoff() (cutoff time.Time) {
 	client.stateMutex.Lock()
-	defer client.stateMutex.Unlock()
-	changed = client.account != casefoldedAccount
-	client.account = casefoldedAccount
-	client.accountName = account
+	if client.account != "" {
+		cutoff = client.accountRegDate
+	} else {
+		cutoff = client.ctime
+	}
+	client.stateMutex.Unlock()
 	return
 }
 
+func (client *Client) Logout() {
+	client.stateMutex.Lock()
+	client.account = ""
+	client.accountName = ""
+	client.alwaysOn = false
+	client.stateMutex.Unlock()
+}
+
 func (client *Client) AccountSettings() (result AccountSettings) {
 	client.stateMutex.RLock()
 	result = client.accountSettings
 }
 
 func (client *Client) SetAccountSettings(settings AccountSettings) {
+	alwaysOn := persistenceEnabled(client.server.Config().Accounts.Bouncer.AlwaysOn, settings.AlwaysOn)
 	client.stateMutex.Lock()
 	client.accountSettings = settings
+	if client.registered {
+		client.alwaysOn = alwaysOn
+	}
 	client.stateMutex.Unlock()
 }
 
 
 func (client *Client) HasMode(mode modes.Mode) bool {
 	// client.flags has its own synch
-	return client.flags.HasMode(mode)
+	return client.modes.HasMode(mode)
 }
 
 func (client *Client) SetMode(mode modes.Mode, on bool) bool {
-	return client.flags.SetMode(mode, on)
+	return client.modes.SetMode(mode, on)
+}
+
+func (client *Client) SetRealname(realname string) {
+	client.stateMutex.Lock()
+	client.realname = realname
+	client.stateMutex.Unlock()
 }
 
 func (client *Client) Channels() (result []*Channel) {
 	channel.stateMutex.RUnlock()
 	return clientModes.HighestChannelUserMode()
 }
+
+func (channel *Channel) Settings() (result ChannelSettings) {
+	channel.stateMutex.RLock()
+	result = channel.settings
+	channel.stateMutex.RUnlock()
+	return result
+}
+
+func (channel *Channel) SetSettings(settings ChannelSettings) {
+	channel.stateMutex.Lock()
+	channel.settings = settings
+	channel.stateMutex.Unlock()
+	channel.MarkDirty(IncludeSettings)
+}

irc/handlers.go

 // CHATHISTORY <target> BETWEEN <query> <query> <direction> [<limit>]
 // e.g., CHATHISTORY #ircv3 BETWEEN timestamp=YYYY-MM-DDThh:mm:ss.sssZ timestamp=YYYY-MM-DDThh:mm:ss.sssZ + 100
 func chathistoryHandler(server *Server, client *Client, msg ircmsg.IrcMessage, rb *ResponseBuffer) (exiting bool) {
-	config := server.Config()
-
 	var items []history.Item
-	success := false
-	var hist *history.Buffer
+	unknown_command := false
+	var target string
 	var channel *Channel
+	var sequence history.Sequence
+	var err error
 	defer func() {
 		// successful responses are sent as a chathistory or history batch
-		if success && 0 < len(items) {
-			if channel == nil {
-				client.replayPrivmsgHistory(rb, items, true)
-			} else {
+		if err == nil && 0 < len(items) {
+			if channel != nil {
 				channel.replayHistoryItems(rb, items, false)
+			} else {
+				client.replayPrivmsgHistory(rb, items, target, true)
 			}
 			return
 		}
 
 		// errors are sent either without a batch, or in a draft/labeled-response batch as usual
-		// TODO: send `WARN CHATHISTORY MAX_MESSAGES_EXCEEDED` when appropriate
-		if hist == nil {
-			rb.Add(nil, server.name, "ERR", "CHATHISTORY", "NO_SUCH_CHANNEL")
-		} else if len(items) == 0 {
-			rb.Add(nil, server.name, "ERR", "CHATHISTORY", "NO_TEXT_TO_SEND")
-		} else if !success {
-			rb.Add(nil, server.name, "ERR", "CHATHISTORY", "NEED_MORE_PARAMS")
+		if unknown_command {
+			rb.Add(nil, server.name, "FAIL", "CHATHISTORY", "UNKNOWN_COMMAND", utils.SafeErrorParam(msg.Params[0]), client.t("Unknown command"))
+		} else if err == utils.ErrInvalidParams {
+			rb.Add(nil, server.name, "FAIL", "CHATHISTORY", "INVALID_PARAMETERS", msg.Params[0], client.t("Invalid parameters"))
+		} else if err != nil {
+			rb.Add(nil, server.name, "FAIL", "CHATHISTORY", "MESSAGE_ERROR", msg.Params[0], client.t("Messages could not be retrieved"))
+		} else if sequence == nil {
+			rb.Add(nil, server.name, "FAIL", "CHATHISTORY", "NO_SUCH_CHANNEL", utils.SafeErrorParam(msg.Params[1]), client.t("No such channel"))
 		}
 	}()
 
-	target := msg.Params[0]
-	channel = server.channels.Get(target)
-	if channel != nil && channel.hasClient(client) {
-		// "If [...] the user does not have permission to view the requested content, [...]
-		// NO_SUCH_CHANNEL SHOULD be returned"
-		hist = &channel.history
-	} else {
-		targetClient := server.clients.Get(target)
-		if targetClient != nil {
-			myAccount := client.Account()
-			targetAccount := targetClient.Account()
-			if myAccount != "" && targetAccount != "" && myAccount == targetAccount {
-				hist = &targetClient.history
-			}
-		}
-	}
-	if hist == nil {
+	config := server.Config()
+	maxChathistoryLimit := config.History.ChathistoryMax
+	if maxChathistoryLimit == 0 {
 		return
 	}
 
-	preposition := strings.ToLower(msg.Params[1])
-
 	parseQueryParam := func(param string) (msgid string, timestamp time.Time, err error) {
-		err = errInvalidParams
+		err = utils.ErrInvalidParams
 		pieces := strings.SplitN(param, "=", 2)
 		if len(pieces) < 2 {
 			return
 		}
 		identifier, value := strings.ToLower(pieces[0]), pieces[1]
-		if identifier == "id" {
+		if identifier == "msgid" {
 			msgid, err = value, nil
 			return
 		} else if identifier == "timestamp" {
 		return
 	}
 
-	maxChathistoryLimit := config.History.ChathistoryMax
-	if maxChathistoryLimit == 0 {
-		return
-	}
 	parseHistoryLimit := func(paramIndex int) (limit int) {
 		if len(msg.Params) < (paramIndex + 1) {
 			return maxChathistoryLimit
 		return
 	}
 
-	// TODO: as currently implemented, almost all of thes queries are worst-case O(n)
-	// in the number of stored history entries. Every one of them can be made O(1)
-	// if necessary, without too much difficulty. Some ideas:
-	// * Ensure that the ring buffer is sorted by time, enabling binary search for times
-	// * Maintain a map from msgid to position in the ring buffer
-
-	if preposition == "between" {
-		if len(msg.Params) >= 5 {
-			startMsgid, startTimestamp, startErr := parseQueryParam(msg.Params[2])
-			endMsgid, endTimestamp, endErr := parseQueryParam(msg.Params[3])
-			ascending := msg.Params[4] == "+"
-			limit := parseHistoryLimit(5)
-			if startErr != nil || endErr != nil {
-				success = false
-			} else if startMsgid != "" && endMsgid != "" {
-				inInterval := false
-				matches := func(item history.Item) (result bool) {
-					result = inInterval
-					if item.HasMsgid(startMsgid) {
-						if ascending {
-							inInterval = true
-						} else {
-							inInterval = false
-							return false // interval is exclusive
-						}
-					} else if item.HasMsgid(endMsgid) {
-						if ascending {
-							inInterval = false
-							return false
-						} else {
-							inInterval = true
-						}
-					}
-					return
-				}
-				items = hist.Match(matches, ascending, limit)
-				success = true
-			} else if !startTimestamp.IsZero() && !endTimestamp.IsZero() {
-				items, _ = hist.Between(startTimestamp, endTimestamp, ascending, limit)
-				if !ascending {
-					history.Reverse(items)
-				}
-				success = true
-			}
-			// else: mismatched params, success = false, fail
-		}
+	preposition := strings.ToLower(msg.Params[0])
+	target = msg.Params[1]
+	channel, sequence, err = server.GetHistorySequence(nil, client, target)
+	if err != nil || sequence == nil {
 		return
 	}
 
-	// before, after, latest, around
-	queryParam := msg.Params[2]
-	msgid, timestamp, err := parseQueryParam(queryParam)
-	limit := parseHistoryLimit(3)
-	before := false
+	roundUp := func(endpoint time.Time) (result time.Time) {
+		return endpoint.Truncate(time.Millisecond).Add(time.Millisecond)
+	}
+
+	var start, end history.Selector
+	var limit int
 	switch preposition {
-	case "before":
-		before = true
-		fallthrough
-	case "after":
-		var matches history.Predicate
+	case "between":
+		start.Msgid, start.Time, err = parseQueryParam(msg.Params[2])
 		if err != nil {
-			break
-		} else if msgid != "" {
-			inInterval := false
-			matches = func(item history.Item) (result bool) {
-				result = inInterval
-				if item.HasMsgid(msgid) {
-					inInterval = true
-				}
-				return
-			}
-		} else {
-			matches = func(item history.Item) bool {
-				return before == item.Message.Time.Before(timestamp)
-			}
+			return
 		}
-		items = hist.Match(matches, !before, limit)
-		success = true
-	case "latest":
-		if queryParam == "*" {
-			items = hist.Latest(limit)
-		} else if err != nil {
-			break
-		} else {
-			var matches history.Predicate
-			if msgid != "" {
-				shouldStop := false
-				matches = func(item history.Item) bool {
-					if shouldStop {
-						return false
-					}
-					shouldStop = item.HasMsgid(msgid)
-					return !shouldStop
-				}
+		end.Msgid, end.Time, err = parseQueryParam(msg.Params[3])
+		if err != nil {
+			return
+		}
+		// XXX preserve the ordering of the two parameters, since we might be going backwards,
+		// but round up the chronologically first one, whichever it is, to make it exclusive
+		if !start.Time.IsZero() && !end.Time.IsZero() {
+			if start.Time.Before(end.Time) {
+				start.Time = roundUp(start.Time)
 			} else {
-				matches = func(item history.Item) bool {
-					return item.Message.Time.After(timestamp)
-				}
+				end.Time = roundUp(end.Time)
 			}
-			items = hist.Match(matches, false, limit)
 		}
-		success = true
-	case "around":
+		limit = parseHistoryLimit(4)
+	case "before", "after", "around":
+		start.Msgid, start.Time, err = parseQueryParam(msg.Params[2])
 		if err != nil {
-			break
+			return
 		}
-		var initialMatcher history.Predicate
-		if msgid != "" {
-			inInterval := false
-			initialMatcher = func(item history.Item) (result bool) {
-				if inInterval {
-					return true
-				} else {
-					inInterval = item.HasMsgid(msgid)
-					return inInterval
-				}
+		if preposition == "after" && !start.Time.IsZero() {
+			start.Time = roundUp(start.Time)
+		}
+		if preposition == "before" {
+			end = start
+			start = history.Selector{}
+		}
+		limit = parseHistoryLimit(3)
+	case "latest":
+		if msg.Params[2] != "*" {
+			end.Msgid, end.Time, err = parseQueryParam(msg.Params[2])
+			if err != nil {
+				return
 			}
-		} else {
-			initialMatcher = func(item history.Item) (result bool) {
-				return item.Message.Time.Before(timestamp)
+			if !end.Time.IsZero() {
+				end.Time = roundUp(end.Time)
 			}
+			start.Time = time.Now().UTC()
 		}
-		var halfLimit int
-		halfLimit = (limit + 1) / 2
-		firstPass := hist.Match(initialMatcher, false, halfLimit)
-		if len(firstPass) > 0 {
-			timeWindowStart := firstPass[0].Message.Time
-			items = hist.Match(func(item history.Item) bool {
-				return item.Message.Time.Equal(timeWindowStart) || item.Message.Time.After(timeWindowStart)
-			}, true, limit)
-		}
-		success = true
+		limit = parseHistoryLimit(3)
+	default:
+		unknown_command = true
+		return
 	}
 
+	if preposition == "around" {
+		items, err = sequence.Around(start, limit)
+	} else {
+		items, _, err = sequence.Between(start, end, limit)
+	}
 	return
 }
 
 // HISTORY <target> [<limit>]
 // e.g., HISTORY #ubuntu 10
 // HISTORY me 15
+// HISTORY #darwin 1h
 func historyHandler(server *Server, client *Client, msg ircmsg.IrcMessage, rb *ResponseBuffer) bool {
 	config := server.Config()
 	if !config.History.Enabled {
 	}
 
 	target := msg.Params[0]
-	var hist *history.Buffer
-	channel := server.channels.Get(target)
-	if channel != nil && channel.hasClient(client) {
-		hist = &channel.history
-	} else {
-		if strings.ToLower(target) == "me" {
-			hist = &client.history
-		} else {
-			targetClient := server.clients.Get(target)
-			if targetClient != nil {
-				myAccount, targetAccount := client.Account(), targetClient.Account()
-				if myAccount != "" && targetAccount != "" && myAccount == targetAccount {
-					hist = &targetClient.history
-				}
-			}
-		}
+	if strings.ToLower(target) == "me" {
+		target = "*"
 	}
+	channel, sequence, err := server.GetHistorySequence(nil, client, target)
 
-	if hist == nil {
-		if channel == nil {
-			rb.Add(nil, server.name, ERR_NOSUCHCHANNEL, client.Nick(), utils.SafeErrorParam(target), client.t("No such channel"))
-		} else {
-			rb.Add(nil, server.name, ERR_NOTONCHANNEL, client.Nick(), target, client.t("You're not on that channel"))
-		}
+	if sequence == nil || err != nil {
+		// whatever
+		rb.Add(nil, server.name, ERR_NOSUCHCHANNEL, client.Nick(), utils.SafeErrorParam(target), client.t("No such channel"))
 		return false
 	}
 
-	limit := 10
+	var duration time.Duration
 	maxChathistoryLimit := config.History.ChathistoryMax
+	limit := 100
+	if maxChathistoryLimit < limit {
+		limit = maxChathistoryLimit
+	}
 	if len(msg.Params) > 1 {
 		providedLimit, err := strconv.Atoi(msg.Params[1])
-		if providedLimit > maxChathistoryLimit {
-			providedLimit = maxChathistoryLimit
-		}
 		if err == nil && providedLimit != 0 {
 			limit = providedLimit
+			if maxChathistoryLimit < limit {
+				limit = maxChathistoryLimit
+			}
+		} else if err != nil {
+			duration, err = time.ParseDuration(msg.Params[1])
+			if err == nil {
+				limit = maxChathistoryLimit
+			}
 		}
 	}
 
-	items := hist.Latest(limit)
-
-	if channel != nil {
-		channel.replayHistoryItems(rb, items, false)
+	var items []history.Item
+	if duration == 0 {
+		items, _, err = sequence.Between(history.Selector{}, history.Selector{}, limit)
 	} else {
-		client.replayPrivmsgHistory(rb, items, true)
+		now := time.Now().UTC()
+		start := history.Selector{Time: now}
+		end := history.Selector{Time: now.Add(-duration)}
+		items, _, err = sequence.Between(start, end, limit)
 	}
 
+	if err == nil {
+		if channel != nil {
+			channel.replayHistoryItems(rb, items, false)
+		} else {
+			client.replayPrivmsgHistory(rb, items, "", true)
+		}
+	}
 	return false
 }
 
 		return false
 	}
 
-	if client.isTor && utils.IsRestrictedCTCPMessage(message) {
+	if rb.session.isTor && utils.IsRestrictedCTCPMessage(message) {
 		// note that error replies are never sent for NOTICE
 		if histType != history.Notice {
 			rb.Notice(client.t("CTCP messages are disabled over Tor"))
 			}
 			return
 		}
-		tnick := user.Nick()
+		tDetails := user.Details()
+		tnick := tDetails.nick
 
-		nickMaskString := client.NickMaskString()
-		accountName := client.AccountName()
+		details := client.Details()
+		nickMaskString := details.nickMask
+		accountName := details.accountName
 		// restrict messages appropriately when +R is set
 		// intentionally make the sending user think the message went through fine
-		allowedPlusR := !user.HasMode(modes.RegisteredOnly) || client.LoggedIntoAccount()
-		allowedTor := !user.isTor || !message.IsRestrictedCTCPMessage()
-		if allowedPlusR && allowedTor {
+		allowedPlusR := !user.HasMode(modes.RegisteredOnly) || details.account != ""
+		if allowedPlusR {
 			for _, session := range user.Sessions() {
 				hasTagsCap := session.capabilities.Has(caps.MessageTags)
 				// don't send TAGMSG at all if they don't have the tags cap
 				if histType == history.Tagmsg && hasTagsCap {
 					session.sendFromClientInternal(false, message.Time, message.Msgid, nickMaskString, accountName, tags, command, tnick)
-				} else if histType != history.Tagmsg {
+				} else if histType != history.Tagmsg && !(session.isTor && message.IsRestrictedCTCPMessage()) {
 					tagsToSend := tags
 					if !hasTagsCap {
 						tagsToSend = nil
 			rb.Add(nil, server.name, RPL_AWAY, client.Nick(), tnick, user.AwayMessage())
 		}
 
+		config := server.Config()
+		if !config.History.Enabled {
+			return
+		}
 		item := history.Item{
 			Type:        histType,
 			Message:     message,
 			Nick:        nickMaskString,
 			AccountName: accountName,
+			Tags:        tags,
+		}
+		if !item.IsStorable() {
+			return
+		}
+		targetedItem := item
+		targetedItem.Params[0] = tnick
+		cPersistent, cEphemeral := client.historyStatus(config)
+		tPersistent, tEphemeral := user.historyStatus(config)
+		// add to ephemeral history
+		if cEphemeral {
+			targetedItem.CfCorrespondent = tDetails.nickCasefolded
+			client.history.Add(targetedItem)
+		}
+		if tEphemeral {
+			item.CfCorrespondent = details.nickCasefolded
+			user.history.Add(item)
+		}
+		if cPersistent || tPersistent {
+			item.CfCorrespondent = ""
+			server.historyDB.AddDirectMessage(details.nickCasefolded, user.NickCasefolded(), cPersistent, tPersistent, targetedItem)
 		}
-		// add to the target's history:
-		user.history.Add(item)
-		// add this to the client's history as well, recording the target:
-		item.Params[0] = tnick
-		client.history.Add(item)
 	}
 }
 
 // SETNAME <realname>
 func setnameHandler(server *Server, client *Client, msg ircmsg.IrcMessage, rb *ResponseBuffer) bool {
 	realname := msg.Params[0]
-
-	client.stateMutex.Lock()
-	client.realname = realname
-	client.stateMutex.Unlock()
-
+	client.SetRealname(realname)
 	details := client.Details()
 
 	// alert friends

irc/help.go

 
 Replay message history. <target> can be a channel name, "me" to replay direct
 message history, or a nickname to replay another client's direct message
-history (they must be logged into the same account as you). At most [limit]
-messages will be replayed.`,
+history (they must be logged into the same account as you). [limit] can be
+either an integer (the maximum number of messages to replay), or a time
+duration like 10m or 1h (the time window within which to replay messages).`,
 	},
 	"info": {
 		text: `INFO

irc/history/history.go

 import (
 	"github.com/oragono/oragono/irc/utils"
 	"sync"
-	"sync/atomic"
 	"time"
 )
 
 	// this is the uncasefolded account name, if there's no account it should be set to "*"
 	AccountName string
 	// for non-privmsg items, we may stuff some other data in here
-	Message utils.SplitMessage
-	Tags    map[string]string
-	Params  [1]string
+	Message         utils.SplitMessage
+	Tags            map[string]string
+	Params          [1]string
+	CfCorrespondent string
 }
 
 // HasMsgid tests whether a message has the message id `msgid`.
 	return item.Message.Msgid == msgid
 }
 
-func (item *Item) isStorable() bool {
-	if item.Type == Tagmsg {
+func (item *Item) IsStorable() bool {
+	switch item.Type {
+	case Tagmsg:
 		for name := range item.Tags {
 			if !transientTags[name] {
 				return true
 			}
 		}
 		return false // all tags were blacklisted
-	} else {
+	case Privmsg, Notice:
+		// don't store CTCP other than ACTION
+		return !item.Message.IsRestrictedCTCPMessage()
+	default:
 		return true
 	}
 }
 
-type Predicate func(item Item) (matches bool)
+type Predicate func(item *Item) (matches bool)
+
+func Reverse(results []Item) {
+	for i, j := 0, len(results)-1; i < j; i, j = i+1, j-1 {
+		results[i], results[j] = results[j], results[i]
+	}
+}
 
 // Buffer is a ring buffer holding message/event history for a channel or user
 type Buffer struct {
 
 	lastDiscarded time.Time
 
-	enabled uint32
-
 	nowFunc func() time.Time
 }
 
 	hist.window = window
 	hist.maximumSize = size
 	hist.nowFunc = time.Now
-
-	hist.setEnabled(size)
 }
 
 // compute the initial size for the buffer, taking into account autoresize
 	return
 }
 
-func (hist *Buffer) setEnabled(size int) {
-	var enabled uint32
-	if size != 0 {
-		enabled = 1
-	}
-	atomic.StoreUint32(&hist.enabled, enabled)
-}
-
-// Enabled returns whether the buffer is currently storing messages
-// (a disabled buffer blackholes everything it sees)
-func (list *Buffer) Enabled() bool {
-	return atomic.LoadUint32(&list.enabled) != 0
-}
-
 // Add adds a history item to the buffer
 func (list *Buffer) Add(item Item) {
-	// fast path without a lock acquisition for when we are not storing history
-	if !list.Enabled() {
-		return
-	}
-
-	if !item.isStorable() {
-		return
-	}
-
 	if item.Message.Time.IsZero() {
 		item.Message.Time = time.Now().UTC()
 	}
 	list.Lock()
 	defer list.Unlock()
 
+	if len(list.buffer) == 0 {
+		return
+	}
+
 	list.maybeExpand()
 
 	var pos int
 	list.buffer[pos] = item
 }
 
-// Reverse reverses an []Item, in-place.
-func Reverse(results []Item) {
-	for i, j := 0, len(results)-1; i < j; i, j = i+1, j-1 {
-		results[i], results[j] = results[j], results[i]
+func (list *Buffer) lookup(msgid string) (result Item, found bool) {
+	predicate := func(item *Item) bool {
+		return item.HasMsgid(msgid)
+	}
+	results := list.matchInternal(predicate, false, 1)
+	if len(results) != 0 {
+		return results[0], true
 	}
+	return
 }
 
 // Between returns all history items with a time `after` <= time <= `before`,
 // with an indication of whether the results are complete or are missing items
 // because some of that period was discarded. A zero value of `before` is considered
 // higher than all other times.
-func (list *Buffer) Between(after, before time.Time, ascending bool, limit int) (results []Item, complete bool) {
-	if !list.Enabled() {
-		return
-	}
+func (list *Buffer) betweenHelper(start, end Selector, cutoff time.Time, pred Predicate, limit int) (results []Item, complete bool, err error) {
+	var ascending bool
+
+	defer func() {
+		if !ascending {
+			Reverse(results)
+		}
+	}()
 
 	list.RLock()
 	defer list.RUnlock()
 
+	if len(list.buffer) == 0 {
+		return
+	}
+
+	after := start.Time
+	if start.Msgid != "" {
+		item, found := list.lookup(start.Msgid)
+		if !found {
+			return
+		}
+		after = item.Message.Time
+	}
+	before := end.Time
+	if end.Msgid != "" {
+		item, found := list.lookup(end.Msgid)
+		if !found {
+			return
+		}
+		before = item.Message.Time
+	}
+
+	after, before, ascending = MinMaxAsc(after, before, cutoff)
+
 	complete = after.Equal(list.lastDiscarded) || after.After(list.lastDiscarded)
 
-	satisfies := func(item Item) bool {
-		return (after.IsZero() || item.Message.Time.After(after)) && (before.IsZero() || item.Message.Time.Before(before))
+	satisfies := func(item *Item) bool {
+		return (after.IsZero() || item.Message.Time.After(after)) &&
+			(before.IsZero() || item.Message.Time.Before(before)) &&
+			(pred == nil || pred(item))
 	}
 
-	return list.matchInternal(satisfies, ascending, limit), complete
+	return list.matchInternal(satisfies, ascending, limit), complete, nil
 }
 
-// Match returns all history items such that `predicate` returns true for them.
-// Items are considered in reverse insertion order if `ascending` is false, or
-// in insertion order if `ascending` is true, up to a total of `limit` matches
-// if `limit` > 0 (unlimited otherwise).
-// `predicate` MAY be a closure that maintains its own state across invocations;
-// it MUST NOT acquire any locks or otherwise do anything weird.
-// Results are always returned in insertion order.
-func (list *Buffer) Match(predicate Predicate, ascending bool, limit int) (results []Item) {
-	if !list.Enabled() {
-		return
+// implements history.Sequence, emulating a single history buffer (for a channel,
+// a single user's DMs, or a DM conversation)
+type bufferSequence struct {
+	list   *Buffer
+	pred   Predicate
+	cutoff time.Time
+}
+
+func (list *Buffer) MakeSequence(correspondent string, cutoff time.Time) Sequence {
+	var pred Predicate
+	if correspondent != "" {
+		pred = func(item *Item) bool {
+			return item.CfCorrespondent == correspondent
+		}
 	}
+	return &bufferSequence{
+		list:   list,
+		pred:   pred,
+		cutoff: cutoff,
+	}
+}
 
-	list.RLock()
-	defer list.RUnlock()
+func (seq *bufferSequence) Between(start, end Selector, limit int) (results []Item, complete bool, err error) {
+	return seq.list.betweenHelper(start, end, seq.cutoff, seq.pred, limit)
+}
 
-	return list.matchInternal(predicate, ascending, limit)
+func (seq *bufferSequence) Around(start Selector, limit int) (results []Item, err error) {
+	return GenericAround(seq, start, limit)
 }
 
 // you must be holding the read lock to call this
 func (list *Buffer) matchInternal(predicate Predicate, ascending bool, limit int) (results []Item) {
-	if list.start == -1 {
+	if list.start == -1 || len(list.buffer) == 0 {
 		return
 	}
 
 	}
 
 	for {
-		if predicate(list.buffer[pos]) {
+		if predicate(&list.buffer[pos]) {
 			results = append(results, list.buffer[pos])
 		}
 		if pos == stop || (limit != 0 && len(results) == limit) {
 		}
 	}
 
-	// TODO sort by time instead?
-	if !ascending {
-		Reverse(results)
-	}
 	return
 }
 
-// Latest returns the items most recently added, up to `limit`. If `limit` is 0,
+// latest returns the items most recently added, up to `limit`. If `limit` is 0,
 // it returns all items.
-func (list *Buffer) Latest(limit int) (results []Item) {
-	matchAll := func(item Item) bool { return true }
-	return list.Match(matchAll, false, limit)
+func (list *Buffer) latest(limit int) (results []Item) {
+	results, _, _ = list.betweenHelper(Selector{}, Selector{}, time.Time{}, nil, limit)
+	return
 }
 
 // LastDiscarded returns the latest time of any entry that was evicted
 func (list *Buffer) resize(size int) {
 	newbuffer := make([]Item, size)
 
-	list.setEnabled(size)
-
 	if list.start == -1 {
 		// indices are already correct and nothing needs to be copied
 	} else if size == 0 {

irc/history/history_test.go

 	timeFormat = "2006-01-02 15:04:05Z"
 )
 
+func betweenTimestamps(buf *Buffer, start, end time.Time, limit int) (result []Item, complete bool) {
+	result, complete, _ = buf.betweenHelper(Selector{Time: start}, Selector{Time: end}, time.Time{}, nil, limit)
+	return
+}
+
 func TestEmptyBuffer(t *testing.T) {
 	pastTime := easyParse(timeFormat)
 
 	buf := NewHistoryBuffer(0, 0)
-	if buf.Enabled() {
-		t.Error("the buffer of size 0 must be considered disabled")
-	}
 
 	buf.Add(Item{
 		Nick: "testnick",
 	})
 
-	since, complete := buf.Between(pastTime, time.Now(), false, 0)
+	since, complete := betweenTimestamps(buf, pastTime, time.Now(), 0)
 	if len(since) != 0 {
 		t.Error("shouldn't be able to add to disabled buf")
 	}
 	}
 
 	buf.Resize(1, 0)
-	if !buf.Enabled() {
-		t.Error("the buffer of size 1 must be considered enabled")
-	}
-	since, complete = buf.Between(pastTime, time.Now(), false, 0)
+	since, complete = betweenTimestamps(buf, pastTime, time.Now(), 0)
 	assertEqual(complete, true, t)
 	assertEqual(len(since), 0, t)
 	buf.Add(Item{
 		Nick: "testnick",
 	})
-	since, complete = buf.Between(pastTime, time.Now(), false, 0)
+	since, complete = betweenTimestamps(buf, pastTime, time.Now(), 0)
 	if len(since) != 1 {
 		t.Error("should be able to store items in a nonempty buffer")
 	}
 	buf.Add(Item{
 		Nick: "testnick2",
 	})
-	since, complete = buf.Between(pastTime, time.Now(), false, 0)
+	since, complete = betweenTimestamps(buf, pastTime, time.Now(), 0)
 	if len(since) != 1 {
 		t.Error("expect exactly 1 item")
 	}
 	if since[0].Nick != "testnick2" {
 		t.Error("retrieved junk data")
 	}
-	matchAll := func(item Item) bool { return true }
-	assertEqual(toNicks(buf.Match(matchAll, false, 0)), []string{"testnick2"}, t)
+	assertEqual(toNicks(buf.latest(0)), []string{"testnick2"}, t)
 }
 
 func toNicks(items []Item) (result []string) {
 
 	buf.Add(easyItem("testnick2", "2006-01-03 15:04:05Z"))
 
-	since, complete := buf.Between(start, time.Now(), false, 0)
+	since, complete := betweenTimestamps(buf, start, time.Now(), 0)
 	assertEqual(complete, true, t)
 	assertEqual(toNicks(since), []string{"testnick0", "testnick1", "testnick2"}, t)
 
 	// add another item, evicting the first
 	buf.Add(easyItem("testnick3", "2006-01-04 15:04:05Z"))
 
-	since, complete = buf.Between(start, time.Now(), false, 0)
+	since, complete = betweenTimestamps(buf, start, time.Now(), 0)
 	assertEqual(complete, false, t)
 	assertEqual(toNicks(since), []string{"testnick1", "testnick2", "testnick3"}, t)
 	// now exclude the time of the discarded entry; results should be complete again
-	since, complete = buf.Between(easyParse("2006-01-02 00:00:00Z"), time.Now(), false, 0)
+	since, complete = betweenTimestamps(buf, easyParse("2006-01-02 00:00:00Z"), time.Now(), 0)
 	assertEqual(complete, true, t)
 	assertEqual(toNicks(since), []string{"testnick1", "testnick2", "testnick3"}, t)
-	since, complete = buf.Between(easyParse("2006-01-02 00:00:00Z"), easyParse("2006-01-03 00:00:00Z"), false, 0)
+	since, complete = betweenTimestamps(buf, easyParse("2006-01-02 00:00:00Z"), easyParse("2006-01-03 00:00:00Z"), 0)
 	assertEqual(complete, true, t)
 	assertEqual(toNicks(since), []string{"testnick1"}, t)
 
 	// shrink the buffer, cutting off testnick1
 	buf.Resize(2, 0)
-	since, complete = buf.Between(easyParse("2006-01-02 00:00:00Z"), time.Now(), false, 0)
+	since, complete = betweenTimestamps(buf, easyParse("2006-01-02 00:00:00Z"), time.Now(), 0)
 	assertEqual(complete, false, t)
 	assertEqual(toNicks(since), []string{"testnick2", "testnick3"}, t)
 
 	buf.Add(easyItem("testnick4", "2006-01-05 15:04:05Z"))
 	buf.Add(easyItem("testnick5", "2006-01-06 15:04:05Z"))
 	buf.Add(easyItem("testnick6", "2006-01-07 15:04:05Z"))
-	since, complete = buf.Between(easyParse("2006-01-03 00:00:00Z"), time.Now(), false, 0)
+	since, complete = betweenTimestamps(buf, easyParse("2006-01-03 00:00:00Z"), time.Now(), 0)
 	assertEqual(complete, true, t)
 	assertEqual(toNicks(since), []string{"testnick2", "testnick3", "testnick4", "testnick5", "testnick6"}, t)
 
 	// test ascending order
-	since, _ = buf.Between(easyParse("2006-01-03 00:00:00Z"), time.Now(), true, 2)
+	since, _ = betweenTimestamps(buf, easyParse("2006-01-03 00:00:00Z"), time.Time{}, 2)
 	assertEqual(toNicks(since), []string{"testnick2", "testnick3"}, t)
 }
 
 func autoItem(id int, t time.Time) (result Item) {
 	result.Message.Time = t
 	result.Nick = strconv.Itoa(id)
+	result.Message.Msgid = result.Nick
 	return
 }
 
 		now = now.Add(time.Minute * 10)
 		id += 1
 	}
-	items := buf.Latest(0)
+	items := buf.latest(0)
 	assertEqual(len(items), initialAutoSize, t)
 	assertEqual(atoi(items[0].Nick), 40, t)
 	assertEqual(atoi(items[len(items)-1].Nick), 71, t)
 	// ok, 5 items from the first batch are still in the 1-hour window;
 	// we should overwrite until only those 5 are left, then start expanding
 	// the buffer so that it retains those 5 and the 100 new items
-	items = buf.Latest(0)
+	items = buf.latest(0)
 	assertEqual(len(items), 105, t)
 	assertEqual(atoi(items[0].Nick), 67, t)
 	assertEqual(atoi(items[len(items)-1].Nick), 171, t)
 		id += 1
 	}
 	// should fill up to the maximum size of 128 and start overwriting
-	items = buf.Latest(0)
+	items = buf.latest(0)
 	assertEqual(len(items), 128, t)
 	assertEqual(atoi(items[0].Nick), 144, t)
 	assertEqual(atoi(items[len(items)-1].Nick), 271, t)
 	buf.Resize(128, time.Hour)
 	// add an item and test that it is stored and retrievable
 	buf.Add(autoItem(0, now))
-	items := buf.Latest(0)
+	items := buf.latest(0)
 	assertEqual(len(items), 1, t)
 	assertEqual(atoi(items[0].Nick), 0, t)
 }
 	// enabled autoresizing buffer
 	buf := NewHistoryBuffer(128, time.Hour)
 	buf.Add(autoItem(0, now))
-	items := buf.Latest(0)
+	items := buf.latest(0)
 	assertEqual(len(items), 1, t)
 	assertEqual(atoi(items[0].Nick), 0, t)
 
 	// disable as during a rehash, confirm that nothing can be retrieved
 	buf.Resize(0, time.Hour)
-	items = buf.Latest(0)
+	items = buf.latest(0)
 	assertEqual(len(items), 0, t)
 }
 
 	assertEqual(roundUpToPowerOfTwo(1025), 2048, t)
 	assertEqual(roundUpToPowerOfTwo(269435457), 536870912, t)
 }
+
+func BenchmarkInsert(b *testing.B) {
+	buf := NewHistoryBuffer(1024, 0)
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		buf.Add(Item{})
+	}
+}
+
+func BenchmarkMatch(b *testing.B) {
+	buf := NewHistoryBuffer(1024, 0)
+	var now time.Time
+	for i := 0; i < 1024; i += 1 {
+		buf.Add(autoItem(i, now))
+		now = now.Add(time.Second)
+	}
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		buf.lookup("512")
+	}
+}

irc/history/queries.go

+// Copyright (c) 2020 Shivaram Lingamneni <slingamn@cs.stanford.edu>
+// released under the MIT license
+
+package history
+
+import (
+	"time"
+)
+
+// Selector represents a parameter to a CHATHISTORY command;
+// at most one of Msgid or Time may be nonzero
+type Selector struct {
+	Msgid string
+	Time  time.Time
+}
+
+// Sequence is an abstract sequence of history entries that can be queried;
+// it encapsulates restrictions such as registration time cutoffs, or
+// only looking at a single "query buffer" (DMs with a particular correspondent)
+type Sequence interface {
+	Between(start, end Selector, limit int) (results []Item, complete bool, err error)
+	Around(start Selector, limit int) (results []Item, err error)
+}
+
+// This is a bad, slow implementation of CHATHISTORY AROUND using the BETWEEN semantics
+func GenericAround(seq Sequence, start Selector, limit int) (results []Item, err error) {
+	var halfLimit int
+	halfLimit = (limit + 1) / 2
+	initialResults, _, err := seq.Between(Selector{}, start, halfLimit)
+	if err != nil {
+		return
+	} else if len(initialResults) == 0 {
+		// TODO: this fails if we're doing an AROUND on the first message in the buffer
+		// would be nice to fix this but whatever
+		return
+	}
+	newStart := Selector{Time: initialResults[0].Message.Time}
+	results, _, err = seq.Between(newStart, Selector{}, limit)
+	return
+}
+
+// MinMaxAsc converts CHATHISTORY arguments into time intervals, handling the most
+// general case (BETWEEN going forwards or backwards) natively and the other ordering
+// queries (AFTER, BEFORE, LATEST) as special cases.
+func MinMaxAsc(after, before, cutoff time.Time) (min, max time.Time, ascending bool) {
+	startIsZero, endIsZero := after.IsZero(), before.IsZero()
+	if !startIsZero && endIsZero {
+		// AFTER
+		ascending = true
+	} else if startIsZero && !endIsZero {
+		// BEFORE
+		ascending = false
+	} else if !startIsZero && !endIsZero {
+		if before.Before(after) {
+			// BETWEEN going backwards
+			before, after = after, before
+			ascending = false
+		} else {
+			// BETWEEN going forwards
+			ascending = true
+		}
+	} else if startIsZero && endIsZero {
+		// LATEST
+		ascending = false
+	}
+	if after.IsZero() || after.Before(cutoff) {
+		// this may result in an impossible query, which is fine
+		after = cutoff
+	}
+	return after, before, ascending
+}

irc/idletimer.go

 	quitTimeout time.Duration
 	state       TimerState
 	timer       *time.Timer
+	lastTouch   time.Time
 }
 
 // Initialize sets up an IdleTimer and starts counting idle time;
 	it.registerTimeout = RegisterTimeout
 	it.idleTimeout, it.quitTimeout = it.recomputeDurations()
 	registered := session.client.Registered()
+	now := time.Now().UTC()
 
 	it.Lock()
 	defer it.Unlock()
+	it.lastTouch = now
 	if registered {
 		it.state = TimerActive
 	} else {
 	}
 
 	idleTimeout = DefaultIdleTimeout
-	if it.session.client.isTor {
+	if it.session.isTor {
 		idleTimeout = TorIdleTimeout
 	}
 
 
 func (it *IdleTimer) Touch() {
 	idleTimeout, quitTimeout := it.recomputeDurations()
+	now := time.Now().UTC()
 
 	it.Lock()
 	defer it.Unlock()
 	it.idleTimeout, it.quitTimeout = idleTimeout, quitTimeout
+	it.lastTouch = now
 	// a touch transitions TimerUnregistered or TimerIdle into TimerActive
 	if it.state != TimerDead {
 		it.state = TimerActive
 	}
 }
 
+func (it *IdleTimer) LastTouch() (result time.Time) {
+	it.Lock()
+	result = it.lastTouch
+	it.Unlock()
+	return
+}
+
 func (it *IdleTimer) processTimeout() {
 	idleTimeout, quitTimeout := it.recomputeDurations()
 
 	// BrbDead is the state of a client after its timeout has expired; it will be removed
 	// and therefore new sessions cannot be attached to it
 	BrbDead
-	// BrbSticky allows a client to remain online without sessions, with no timeout.
-	// This is not used yet.
-	BrbSticky
 )
 
 type BrbTimer struct {
 
 // attempts to enable BRB for a client, returns whether it succeeded
 func (bt *BrbTimer) Enable() (success bool, duration time.Duration) {
-	if !bt.client.Registered() || bt.client.ResumeID() == "" {
-		return
-	}
-
 	// TODO make this configurable
 	duration = ResumeableTotalTimeout
 
 	bt.client.stateMutex.Lock()
 	defer bt.client.stateMutex.Unlock()
 
+	if !bt.client.registered || bt.client.alwaysOn || bt.client.resumeID == "" {
+		return
+	}
+
 	switch bt.state {
 	case BrbDisabled, BrbEnabled:
 		bt.state = BrbEnabled
 			bt.brbAt = time.Now().UTC()
 		}
 		success = true
-	case BrbSticky:
-		success = true
 	default:
 		// BrbDead
 		success = false
 	bt.client.stateMutex.Lock()
 	defer bt.client.stateMutex.Unlock()
 
+	if bt.client.alwaysOn {
+		return
+	}
+
 	switch bt.state {
 	case BrbDisabled, BrbEnabled:
 		if len(bt.client.sessions) == 0 {
 	}
 	bt.resetTimeout()
 }
-
-// sets a client to be "sticky", i.e., indefinitely exempt from removal for
-// lack of sessions
-func (bt *BrbTimer) SetSticky() (success bool) {
-	bt.client.stateMutex.Lock()
-	defer bt.client.stateMutex.Unlock()
-	if bt.state != BrbDead {
-		success = true
-		bt.state = BrbSticky
-	}
-	bt.resetTimeout()
-	return
-}

irc/mysql/history.go

+package mysql
+
+import (
+	"bytes"
+	"database/sql"
+	"fmt"
+	"runtime/debug"
+	"strconv"
+	"sync"
+	"time"
+
+	_ "github.com/go-sql-driver/mysql"
+	"github.com/oragono/oragono/irc/history"
+	"github.com/oragono/oragono/irc/logger"
+	"github.com/oragono/oragono/irc/utils"
+)
+
+const (
+	// latest schema of the db
+	latestDbSchema   = "1"
+	keySchemaVersion = "db.version"
+	cleanupRowLimit  = 50
+	cleanupPauseTime = 10 * time.Minute
+)
+
+type MySQL struct {
+	db     *sql.DB
+	logger *logger.Manager
+
+	insertHistory      *sql.Stmt
+	insertSequence     *sql.Stmt
+	insertConversation *sql.Stmt
+
+	stateMutex sync.Mutex
+	expireTime time.Duration
+}
+
+func (mysql *MySQL) Initialize(logger *logger.Manager, expireTime time.Duration) {
+	mysql.logger = logger
+	mysql.expireTime = expireTime
+}
+
+func (mysql *MySQL) SetExpireTime(expireTime time.Duration) {
+	mysql.stateMutex.Lock()
+	mysql.expireTime = expireTime
+	mysql.stateMutex.Unlock()
+}
+
+func (mysql *MySQL) getExpireTime() (expireTime time.Duration) {
+	mysql.stateMutex.Lock()
+	expireTime = mysql.expireTime
+	mysql.stateMutex.Unlock()
+	return
+}
+
+func (mysql *MySQL) Open(username, password, host string, port int, database string) (err error) {
+	// TODO: timeouts!
+	var address string
+	if port != 0 {
+		address = fmt.Sprintf("tcp(%s:%d)", host, port)
+	}
+
+	mysql.db, err = sql.Open("mysql", fmt.Sprintf("%s:%s@%s/%s", username, password, address, database))
+	if err != nil {
+		return err
+	}
+
+	err = mysql.fixSchemas()
+	if err != nil {
+		return err
+	}
+
+	err = mysql.prepareStatements()
+	if err != nil {
+		return err
+	}
+
+	go mysql.cleanupLoop()
+
+	return nil
+}
+
+func (mysql *MySQL) fixSchemas() (err error) {
+	_, err = mysql.db.Exec(`CREATE TABLE IF NOT EXISTS metadata (
+		key_name VARCHAR(32) primary key,
+		value VARCHAR(32) NOT NULL
+	) CHARSET=ascii COLLATE=ascii_bin;`)
+	if err != nil {
+		return err
+	}
+
+	var schema string
+	err = mysql.db.QueryRow(`select value from metadata where key_name = ?;`, keySchemaVersion).Scan(&schema)
+	if err == sql.ErrNoRows {
+		err = mysql.createTables()
+		if err != nil {
+			return
+		}
+		_, err = mysql.db.Exec(`insert into metadata (key_name, value) values (?, ?);`, keySchemaVersion, latestDbSchema)
+		if err != nil {
+			return
+		}
+	} else if err == nil && schema != latestDbSchema {
+		// TODO figure out what to do about schema changes
+		return &utils.IncompatibleSchemaError{CurrentVersion: schema, RequiredVersion: latestDbSchema}
+	} else {
+		return err
+	}
+
+	return nil
+}
+
+func (mysql *MySQL) createTables() (err error) {
+	_, err = mysql.db.Exec(`CREATE TABLE history (
+		id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
+		data BLOB NOT NULL,
+		msgid BINARY(16) NOT NULL,
+		KEY (msgid(4))
+	) CHARSET=ascii COLLATE=ascii_bin;`)
+	if err != nil {
+		return err
+	}
+
+	_, err = mysql.db.Exec(`CREATE TABLE sequence (
+		id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
+		target VARBINARY(64) NOT NULL,
+		nanotime BIGINT UNSIGNED NOT NULL,
+		history_id BIGINT NOT NULL,
+		KEY (target, nanotime),
+		KEY (history_id)
+	) CHARSET=ascii COLLATE=ascii_bin;`)
+	if err != nil {
+		return err
+	}
+
+	_, err = mysql.db.Exec(`CREATE TABLE conversations (
+		id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
+		lower_target VARBINARY(64) NOT NULL,
+		upper_target VARBINARY(64) NOT NULL,
+		nanotime BIGINT UNSIGNED NOT NULL,
+		history_id BIGINT NOT NULL,
+		KEY (lower_target, upper_target, nanotime),
+		KEY (history_id)
+	) CHARSET=ascii COLLATE=ascii_bin;`)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (mysql *MySQL) cleanupLoop() {
+	defer func() {
+		if r := recover(); r != nil {
+			mysql.logger.Error("mysql",
+				fmt.Sprintf("Panic in cleanup routine: %v\n%s", r, debug.Stack()))
+			time.Sleep(cleanupPauseTime)
+			go mysql.cleanupLoop()
+		}
+	}()
+
+	for {
+		expireTime := mysql.getExpireTime()
+		if expireTime != 0 {
+			for {
+				startTime := time.Now()
+				rowsDeleted, err := mysql.doCleanup(expireTime)
+				elapsed := time.Now().Sub(startTime)
+				mysql.logError("error during row cleanup", err)
+				// keep going as long as we're accomplishing significant work
+				// (don't busy-wait on small numbers of rows expiring):
+				if rowsDeleted < (cleanupRowLimit / 10) {
+					break
+				}
+				// crude backpressure mechanism: if the database is slow,
+				// give it time to process other queries
+				time.Sleep(elapsed)
+			}
+		}
+		time.Sleep(cleanupPauseTime)
+	}
+}
+
+func (mysql *MySQL) doCleanup(age time.Duration) (count int, err error) {
+	ids, maxNanotime, err := mysql.selectCleanupIDs(age)
+	if len(ids) == 0 {
+		mysql.logger.Debug("mysql", "found no rows to clean up")
+		return
+	}
+
+	mysql.logger.Debug("mysql", fmt.Sprintf("deleting %d history rows, max age %s", len(ids), utils.NanoToTimestamp(maxNanotime)))
+
+	// can't use ? binding for a variable number of arguments, build the IN clause manually
+	var inBuf bytes.Buffer
+	inBuf.WriteByte('(')
+	for i, id := range ids {
+		if i != 0 {
+			inBuf.WriteRune(',')
+		}
+		inBuf.WriteString(strconv.FormatInt(int64(id), 10))
+	}
+	inBuf.WriteRune(')')
+
+	_, err = mysql.db.Exec(fmt.Sprintf(`DELETE FROM conversations WHERE history_id in %s;`, inBuf.Bytes()))
+	if err != nil {
+		return
+	}
+	_, err = mysql.db.Exec(fmt.Sprintf(`DELETE FROM sequence WHERE history_id in %s;`, inBuf.Bytes()))
+	if err != nil {
+		return
+	}
+	_, err = mysql.db.Exec(fmt.Sprintf(`DELETE FROM history WHERE id in %s;`, inBuf.Bytes()))
+	if err != nil {
+		return
+	}
+
+	count = len(ids)
+	return
+}
+
+func (mysql *MySQL) selectCleanupIDs(age time.Duration) (ids []uint64, maxNanotime int64, err error) {
+	rows, err := mysql.db.Query(`
+		SELECT history.id, sequence.nanotime
+		FROM history
+		LEFT JOIN sequence ON history.id = sequence.history_id
+		ORDER BY history.id LIMIT ?;`, cleanupRowLimit)
+	if err != nil {
+		return
+	}
+	defer rows.Close()
+
+	// a history ID may have 0-2 rows in sequence: 1 for a channel entry,
+	// 2 for a DM, 0 if the data is inconsistent. therefore, deduplicate
+	// and delete anything that doesn't have a sequence entry:
+	idset := make(map[uint64]struct{}, cleanupRowLimit)
+	threshold := time.Now().Add(-age).UnixNano()
+	for rows.Next() {
+		var id uint64
+		var nanotime sql.NullInt64
+		err = rows.Scan(&id, &nanotime)
+		if err != nil {
+			return
+		}
+		if !nanotime.Valid || nanotime.Int64 < threshold {
+			idset[id] = struct{}{}
+			if nanotime.Valid && nanotime.Int64 > maxNanotime {
+				maxNanotime = nanotime.Int64
+			}
+		}
+	}
+	ids = make([]uint64, len(idset))
+	i := 0
+	for id := range idset {
+		ids[i] = id
+		i++
+	}
+	return
+}
+
+func (mysql *MySQL) prepareStatements() (err error) {
+	mysql.insertHistory, err = mysql.db.Prepare(`INSERT INTO history
+		(data, msgid) VALUES (?, ?);`)
+	if err != nil {
+		return
+	}
+	mysql.insertSequence, err = mysql.db.Prepare(`INSERT INTO sequence
+		(target, nanotime, history_id) VALUES (?, ?, ?);`)
+	if err != nil {
+		return
+	}
+	mysql.insertConversation, err = mysql.db.Prepare(`INSERT INTO conversations
+		(lower_target, upper_target, nanotime, history_id) VALUES (?, ?, ?, ?);`)
+	if err != nil {
+		return
+	}
+
+	return
+}
+
+func (mysql *MySQL) logError(context string, err error) (quit bool) {
+	if err != nil {
+		mysql.logger.Error("mysql", context, err.Error())
+		return true
+	}
+	return false
+}
+
+func (mysql *MySQL) AddChannelItem(target string, item history.Item) (err error) {
+	if mysql.db == nil {
+		return
+	}
+
+	if target == "" {
+		return utils.ErrInvalidParams
+	}
+
+	id, err := mysql.insertBase(item)
+	if err != nil {
+		return
+	}
+
+	err = mysql.insertSequenceEntry(target, item.Message.Time, id)
+	return
+}
+
+func (mysql *MySQL) insertSequenceEntry(target string, messageTime time.Time, id int64) (err error) {
+	_, err = mysql.insertSequence.Exec(target, messageTime.UnixNano(), id)
+	mysql.logError("could not insert sequence entry", err)
+	return
+}
+
+func (mysql *MySQL) insertConversationEntry(sender, recipient string, messageTime time.Time, id int64) (err error) {
+	lower, higher := stringMinMax(sender, recipient)
+	_, err = mysql.insertConversation.Exec(lower, higher, messageTime.UnixNano(), id)
+	mysql.logError("could not insert conversations entry", err)
+	return
+}
+
+func (mysql *MySQL) insertBase(item history.Item) (id int64, err error) {
+	value, err := marshalItem(&item)
+	if mysql.logError("could not marshal item", err) {
+		return
+	}
+
+	msgidBytes, err := decodeMsgid(item.Message.Msgid)
+	if mysql.logError("could not decode msgid", err) {
+		return
+	}
+
+	result, err := mysql.insertHistory.Exec(value, msgidBytes)
+	if mysql.logError("could not insert item", err) {
+		return
+	}
+	id, err = result.LastInsertId()
+	if mysql.logError("could not insert item", err) {
+		return
+	}
+
+	return
+}
+
+func stringMinMax(first, second string) (min, max string) {
+	if first < second {
+		return first, second
+	} else {
+		return second, first
+	}
+}
+
+func (mysql *MySQL) AddDirectMessage(sender, recipient string, senderPersistent, recipientPersistent bool, item history.Item) (err error) {
+	if mysql.db == nil {
+		return
+	}
+
+	if !(senderPersistent || recipientPersistent) {
+		return
+	}
+
+	if sender == "" || recipient == "" {
+		return utils.ErrInvalidParams
+	}
+
+	id, err := mysql.insertBase(item)
+	if err != nil {
+		return
+	}
+
+	if senderPersistent {
+		mysql.insertSequenceEntry(sender, item.Message.Time, id)
+		if err != nil {
+			return
+		}
+	}
+
+	if recipientPersistent && sender != recipient {
+		err = mysql.insertSequenceEntry(recipient, item.Message.Time, id)
+		if err != nil {
+			return
+		}
+	}
+
+	err = mysql.insertConversationEntry(sender, recipient, item.Message.Time, id)
+
+	return
+}
+
+func (mysql *MySQL) msgidToTime(msgid string) (result time.Time, err error) {
+	// in theory, we could optimize out a roundtrip to the database by using a subquery instead:
+	// sequence.nanotime > (
+	//     SELECT sequence.nanotime FROM sequence, history
+	//     WHERE sequence.history_id = history.id AND history.msgid = ?
+	//     LIMIT 1)
+	// however, this doesn't handle the BETWEEN case with one or two msgids, where we
+	// don't initially know whether the interval is going forwards or backwards. to simplify
+	// the logic,  resolve msgids to timestamps "manually" in all cases, using a separate query.
+	decoded, err := decodeMsgid(msgid)
+	if err != nil {
+		return
+	}
+	row := mysql.db.QueryRow(`
+		SELECT sequence.nanotime FROM sequence
+		INNER JOIN history ON history.id = sequence.history_id
+		WHERE history.msgid = ? LIMIT 1;`, decoded)
+	var nanotime int64
+	err = row.Scan(&nanotime)
+	if mysql.logError("could not resolve msgid to time", err) {
+		return
+	}
+	result = time.Unix(0, nanotime)
+	return
+}
+
+func (mysql *MySQL) selectItems(query string, args ...interface{}) (results []history.Item, err error) {
+	rows, err := mysql.db.Query(query, args...)
+	if mysql.logError("could not select history items", err) {
+		return
+	}
+
+	defer rows.Close()
+
+	for rows.Next() {
+		var blob []byte
+		var item history.Item
+		err = rows.Scan(&blob)
+		if mysql.logError("could not scan history item", err) {
+			return
+		}
+		err = unmarshalItem(blob, &item)
+		if mysql.logError("could not unmarshal history item", err) {
+			return
+		}
+		results = append(results, item)
+	}
+	return
+}
+
+func (mysql *MySQL) BetweenTimestamps(sender, recipient string, after, before, cutoff time.Time, limit int) (results []history.Item, err error) {
+	useSequence := true
+	var lowerTarget, upperTarget string
+	if sender != "" {
+		lowerTarget, upperTarget = stringMinMax(sender, recipient)
+		useSequence = false
+	}
+
+	table := "sequence"
+	if !useSequence {
+		table = "conversations"
+	}
+
+	after, before, ascending := history.MinMaxAsc(after, before, cutoff)
+	direction := "ASC"
+	if !ascending {
+		direction = "DESC"
+	}
+
+	var queryBuf bytes.Buffer
+
+	args := make([]interface{}, 0, 6)
+	fmt.Fprintf(&queryBuf,
+		"SELECT history.data from history INNER JOIN %[1]s ON history.id = %[1]s.history_id WHERE", table)
+	if useSequence {
+		fmt.Fprintf(&queryBuf, " sequence.target = ?")
+		args = append(args, recipient)
+	} else {
+		fmt.Fprintf(&queryBuf, " conversations.lower_target = ? AND conversations.upper_target = ?")
+		args = append(args, lowerTarget)
+		args = append(args, upperTarget)
+	}
+	if !after.IsZero() {
+		fmt.Fprintf(&queryBuf, " AND %s.nanotime > ?", table)
+		args = append(args, after.UnixNano())
+	}
+	if !before.IsZero() {
+		fmt.Fprintf(&queryBuf, " AND %s.nanotime < ?", table)
+		args = append(args, before.UnixNano())
+	}
+	fmt.Fprintf(&queryBuf, " ORDER BY %[1]s.nanotime %[2]s LIMIT ?;", table, direction)
+	args = append(args, limit)
+
+	results, err = mysql.selectItems(queryBuf.String(), args...)
+	if err == nil && !ascending {
+		history.Reverse(results)
+	}
+	return
+}
+
+func (mysql *MySQL) Close() {
+	// closing the database will close our prepared statements as well
+	if mysql.db != nil {
+		mysql.db.Close()
+	}
+	mysql.db = nil
+}
+
+// implements history.Sequence, emulating a single history buffer (for a channel,
+// a single user's DMs, or a DM conversation)
+type mySQLHistorySequence struct {
+	mysql     *MySQL
+	sender    string
+	recipient string
+	cutoff    time.Time
+}
+
+func (s *mySQLHistorySequence) Between(start, end history.Selector, limit int) (results []history.Item, complete bool, err error) {
+	startTime := start.Time
+	if start.Msgid != "" {
+		startTime, err = s.mysql.msgidToTime(start.Msgid)
+		if err != nil {
+			return nil, false, err
+		}
+	}
+	endTime := end.Time
+	if end.Msgid != "" {
+		endTime, err = s.mysql.msgidToTime(end.Msgid)
+		if err != nil {
+			return nil, false, err
+		}
+	}
+
+	results, err = s.mysql.BetweenTimestamps(s.sender, s.recipient, startTime, endTime, s.cutoff, limit)
+	return results, (err == nil), err
+}
+
+func (s *mySQLHistorySequence) Around(start history.Selector, limit int) (results []history.Item, err error) {
+	return history.GenericAround(s, start, limit)
+}
+
+func (mysql *MySQL) MakeSequence(sender, recipient string, cutoff time.Time) history.Sequence {
+	return &mySQLHistorySequence{
+		sender:    sender,
+		recipient: recipient,
+		mysql:     mysql,
+		cutoff:    cutoff,
+	}
+}

irc/mysql/serialization.go

+package mysql
+
+import (
+	"encoding/json"
+
+	"github.com/oragono/oragono/irc/history"
+	"github.com/oragono/oragono/irc/utils"
+)
+
+// 123 / '{' is the magic number that means JSON;
+// if we want to do a binary encoding later, we just have to add different magic version numbers
+
+func marshalItem(item *history.Item) (result []byte, err error) {
+	return json.Marshal(item)
+}
+
+func unmarshalItem(data []byte, result *history.Item) (err error) {
+	return json.Unmarshal(data, result)
+}
+
+func decodeMsgid(msgid string) ([]byte, error) {
+	return utils.B32Encoder.DecodeString(msgid)
+}

irc/nickname.go

 	hadNick := target.HasNick()
 	origNickMask := target.NickMaskString()
 	details := target.Details()
-	err := client.server.clients.SetNick(target, session, nickname)
+	assignedNickname, err := client.server.clients.SetNick(target, session, nickname)
 	if err == errNicknameInUse {
 		rb.Add(nil, server.name, ERR_NICKNAMEINUSE, currentNick, nickname, client.t("Nickname is already in use"))
 	} else if err == errNicknameReserved {
 		rb.Add(nil, server.name, ERR_NICKNAMEINUSE, currentNick, nickname, client.t("Nickname is reserved by a different account"))
 	} else if err == errNicknameInvalid {
 		rb.Add(nil, server.name, ERR_ERRONEUSNICKNAME, currentNick, utils.SafeErrorParam(nickname), client.t("Erroneous nickname"))
+	} else if err == errCantChangeNick {
+		rb.Add(nil, server.name, ERR_NICKNAMEINUSE, currentNick, utils.SafeErrorParam(nickname), client.t(err.Error()))
 	} else if err != nil {
 		rb.Add(nil, server.name, ERR_UNKNOWNERROR, currentNick, "NICK", fmt.Sprintf(client.t("Could not set or change nickname: %s"), err.Error()))
 	}
 		AccountName: details.accountName,
 		Message:     message,
 	}
-	histItem.Params[0] = nickname
+	histItem.Params[0] = assignedNickname
 
-	client.server.logger.Debug("nick", fmt.Sprintf("%s changed nickname to %s [%s]", origNickMask, nickname, client.NickCasefolded()))
+	client.server.logger.Debug("nick", fmt.Sprintf("%s changed nickname to %s [%s]", origNickMask, assignedNickname, client.NickCasefolded()))
 	if hadNick {
 		if client == target {
-			target.server.snomasks.Send(sno.LocalNicks, fmt.Sprintf(ircfmt.Unescape("$%s$r changed nickname to %s"), details.nick, nickname))
+			target.server.snomasks.Send(sno.LocalNicks, fmt.Sprintf(ircfmt.Unescape("$%s$r changed nickname to %s"), details.nick, assignedNickname))
 		} else {
-			target.server.snomasks.Send(sno.LocalNicks, fmt.Sprintf(ircfmt.Unescape("Operator %s changed nickname of $%s$r to %s"), client.Nick(), details.nick, nickname))
+			target.server.snomasks.Send(sno.LocalNicks, fmt.Sprintf(ircfmt.Unescape("Operator %s changed nickname of $%s$r to %s"), client.Nick(), details.nick, assignedNickname))
 		}
 		target.server.whoWas.Append(details.WhoWas)
-		rb.AddFromClient(message.Time, message.Msgid, origNickMask, details.accountName, nil, "NICK", nickname)
+		rb.AddFromClient(message.Time, message.Msgid, origNickMask, details.accountName, nil, "NICK", assignedNickname)
 		for session := range target.Friends() {
 			if session != rb.session {
-				session.sendFromClientInternal(false, message.Time, message.Msgid, origNickMask, details.accountName, nil, "NICK", nickname)
+				session.sendFromClientInternal(false, message.Time, message.Msgid, origNickMask, details.accountName, nil, "NICK", assignedNickname)
 			}
 		}
 	}
 
 	for _, channel := range client.Channels() {
-		channel.history.Add(histItem)
+		channel.AddHistoryItem(histItem)
 	}
 
 	if target.Registered() {

irc/nickserv.go

 			helpStrings: []string{
 				`Syntax $bSET <setting> <value>$b
 
-Set modifies your account settings. The following settings are available:`,
+SET modifies your account settings. The following settings are available:`,
 
 				`$bENFORCE$b
 'enforce' lets you specify a custom enforcement mechanism for your registered
 messages, but may be spammy. Your options are 'always', 'never', and the default
 of 'commands-only' (the messages will be replayed in /HISTORY output, but not
 during autoreplay).`,
+				`$bALWAYS-ON$b
+'always-on' controls whether your nickname/identity will remain active
+even while you are disconnected from the server. Your options are 'true',
+'false', and 'default' (use the server default value).`,
+				`$bAUTOREPLAY-MISSED$b
+'autoreplay-missed' is only effective for always-on clients. If enabled,
+if you have at most one active session, the server will remember the time
+you disconnect and then replay missed messages to you when you reconnect.
+Your options are 'on' and 'off'.`,
+				`$bDM-HISTORY$b
+'dm-history' is only effective for always-on clients. It lets you control
+how the history of your direct messages is stored. Your options are:
+1. 'off'        [no history]
+2. 'ephemeral'  [a limited amount of temporary history, not stored on disk]
+3. 'on'         [history stored in a permanent database, if available]
+4. 'default'    [use the server default]`,
 			},
 			authRequired: true,
 			enabled:      servCmdRequiresAccreg,
 				nsNotice(rb, client.t("Bouncer functionality is currently enabled for your account"))
 			}
 		}
+	case "always-on":
+		stored := settings.AlwaysOn
+		actual := client.AlwaysOn()
+		nsNotice(rb, fmt.Sprintf(client.t("Your stored always-on setting is: %s"), persistentStatusToString(stored)))
+		if actual {
+			nsNotice(rb, client.t("Given current server settings, your client is always-on"))
+		} else {
+			nsNotice(rb, client.t("Given current server settings, your client is not always-on"))
+		}
+	case "autoreplay-missed":
+		stored := settings.AutoreplayMissed
+		if stored {
+			if client.AlwaysOn() {
+				nsNotice(rb, client.t("Autoreplay of missed messages is enabled"))
+			} else {
+				nsNotice(rb, client.t("You have enabled autoreplay of missed messages, but you can't receive them because your client isn't set to always-on"))
+			}
+		} else {
+			nsNotice(rb, client.t("Your account is not configured to receive autoreplayed missed messages"))
+		}
+	case "dm-history":
+		effectiveValue := historyEnabled(config.History.Persistent.DirectMessages, settings.DMHistory)
+		csNotice(rb, fmt.Sprintf(client.t("Your stored direct message history setting is: %s"), historyStatusToString(settings.DMHistory)))
+		csNotice(rb, fmt.Sprintf(client.t("Given current server settings, your direct message history setting is: %s"), historyStatusToString(effectiveValue)))
+
 	default:
 		nsNotice(rb, client.t("No such setting"))
 	}
 				return
 			}
 		}
+	case "always-on":
+		var newValue PersistentStatus
+		newValue, err = persistentStatusFromString(params[1])
+		// "opt-in" and "opt-out" don't make sense as user preferences
+		if err == nil && newValue != PersistentOptIn && newValue != PersistentOptOut {
+			munger = func(in AccountSettings) (out AccountSettings, err error) {
+				out = in
+				out.AlwaysOn = newValue
+				return
+			}
+		}
+	case "autoreplay-missed":
+		var newValue bool
+		newValue, err = utils.StringToBool(params[1])
+		if err == nil {
+			munger = func(in AccountSettings) (out AccountSettings, err error) {
+				out = in
+				out.AutoreplayMissed = newValue
+				return
+			}
+		}
+	case "dm-history":
+		var newValue HistoryStatus
+		newValue, err = historyStatusFromString(params[1])
+		if err == nil {
+			munger = func(in AccountSettings) (out AccountSettings, err error) {
+				out = in
+				out.DMHistory = newValue
+				return
+			}
+		}
 	default:
 		err = errInvalidParams
 	}
 	} else if ghost == client {
 		nsNotice(rb, client.t("You can't GHOST yourself (try /QUIT instead)"))
 		return
+	} else if ghost.AlwaysOn() {
+		nsNotice(rb, client.t("You can't GHOST an always-on client"))
+		return
 	}
 
 	authorized := false

irc/responsebuffer.go

 }
 
 func (rb *ResponseBuffer) AddMessage(msg ircmsg.IrcMessage) {
+	if rb == nil {
+		return
+	}
+
 	if rb.finalized {
 		rb.target.server.logger.Error("internal", "message added to finalized ResponseBuffer, undefined behavior")
 		debug.PrintStack()
 
 // Add adds a standard new message to our queue.
 func (rb *ResponseBuffer) Add(tags map[string]string, prefix string, command string, params ...string) {
+	if rb == nil {
+		return
+	}
+
 	rb.AddMessage(ircmsg.MakeMessage(tags, prefix, command, params...))
 }
 
 // Broadcast adds a standard new message to our queue, then sends an unlabeled copy
 // to all other sessions.
 func (rb *ResponseBuffer) Broadcast(tags map[string]string, prefix string, command string, params ...string) {
+	if rb == nil {
+		return
+	}
+
 	// can't reuse the IrcMessage object because of tag pollution :-\
 	rb.Add(tags, prefix, command, params...)
 	for _, session := range rb.session.client.Sessions() {
 
 // AddFromClient adds a new message from a specific client to our queue.
 func (rb *ResponseBuffer) AddFromClient(time time.Time, msgid string, fromNickMask string, fromAccount string, tags map[string]string, command string, params ...string) {
+	if rb == nil {
+		return
+	}
+
 	msg := ircmsg.MakeMessage(nil, fromNickMask, command, params...)
 	if rb.session.capabilities.Has(caps.MessageTags) {
 		msg.UpdateTags(tags)
 
 // AddSplitMessageFromClient adds a new split message from a specific client to our queue.
 func (rb *ResponseBuffer) AddSplitMessageFromClient(fromNickMask string, fromAccount string, tags map[string]string, command string, target string, message utils.SplitMessage) {
+	if rb == nil {
+		return
+	}
+
 	if message.Is512() {
 		rb.AddFromClient(message.Time, message.Msgid, fromNickMask, fromAccount, tags, command, target, message.Message)
 	} else {
-		if message.IsMultiline() && rb.session.capabilities.Has(caps.Multiline) {
+		if rb.session.capabilities.Has(caps.Multiline) {
 			batch := rb.session.composeMultilineBatch(fromNickMask, fromAccount, tags, command, target, message)
 			rb.setNestedBatchTag(&batch[0])
 			rb.setNestedBatchTag(&batch[len(batch)-1])
 // Starts a nested batch (see the ResponseBuffer struct definition for a description of
 // how this works)
 func (rb *ResponseBuffer) StartNestedBatch(batchType string, params ...string) (batchID string) {
+	if rb == nil {
+		return
+	}
+
 	batchID = rb.session.generateBatchID()
 	msgParams := make([]string, len(params)+2)
 	msgParams[0] = "+" + batchID
 // Convenience to start a nested batch for history lines, at the highest level
 // supported by the client (`history`, `chathistory`, or no batch, in descending order).
 func (rb *ResponseBuffer) StartNestedHistoryBatch(params ...string) (batchID string) {
+	if rb == nil {
+		return
+	}
+
 	var batchType string
 	if rb.session.capabilities.Has(caps.EventPlayback) {
 		batchType = "history"
 // Afterwards, the buffer is in an undefined state and MUST NOT be used further.
 // If `blocking` is true you MUST be sending to the client from its own goroutine.
 func (rb *ResponseBuffer) Send(blocking bool) error {
+	if rb == nil {
+		return nil
+	}
+
 	return rb.flushInternal(true, blocking)
 }
 
 // to ensure that the final `BATCH -` message is sent.
 // If `blocking` is true you MUST be sending to the client from its own goroutine.
 func (rb *ResponseBuffer) Flush(blocking bool) error {
+	if rb == nil {
+		return nil
+	}
+
 	return rb.flushInternal(false, blocking)
 }
 
 
 // Notice sends the client the given notice from the server.
 func (rb *ResponseBuffer) Notice(text string) {
-	rb.Add(nil, rb.target.server.name, "NOTICE", rb.target.nick, text)
+	if rb == nil {
+		return
+	}
+
+	rb.Add(nil, rb.target.server.name, "NOTICE", rb.target.Nick(), text)
 }

irc/server.go

 	"github.com/goshuirc/irc-go/ircfmt"
 	"github.com/oragono/oragono/irc/caps"
 	"github.com/oragono/oragono/irc/connection_limits"
+	"github.com/oragono/oragono/irc/history"
 	"github.com/oragono/oragono/irc/logger"
 	"github.com/oragono/oragono/irc/modes"
+	"github.com/oragono/oragono/irc/mysql"
 	"github.com/oragono/oragono/irc/sno"
 	"github.com/tidwall/buntdb"
 )
 	signals           chan os.Signal
 	snomasks          SnoManager
 	store             *buntdb.DB
+	historyDB         mysql.MySQL
 	torLimiter        connection_limits.TorLimiter
 	whoWas            WhoWasList
 	stats             Stats
 	server.monitorManager.Initialize()
 	server.snomasks.Initialize()
 
-	if err := server.applyConfig(config, true); err != nil {
+	if err := server.applyConfig(config); err != nil {
 		return nil, err
 	}
 
 	if err := server.store.Close(); err != nil {
 		server.logger.Error("shutdown", fmt.Sprintln("Could not close datastore:", err))
 	}
+
+	server.historyDB.Close()
 }
 
 // Run starts the server.
 
 	// client MUST send PASS if necessary, or authenticate with SASL if necessary,
 	// before completing the other registration commands
-	authOutcome := c.isAuthorized(server.Config())
+	authOutcome := c.isAuthorized(server.Config(), session.isTor)
 	var quitMessage string
 	switch authOutcome {
 	case authFailPass:
 	// continue registration
 	d := c.Details()
 	server.logger.Info("localconnect", fmt.Sprintf("Client connected [%s] [u:%s] [r:%s]", d.nick, d.username, d.realname))
-	server.snomasks.Send(sno.LocalConnects, fmt.Sprintf("Client connected [%s] [u:%s] [h:%s] [ip:%s] [r:%s]", d.nick, d.username, c.RawHostname(), c.IPString(), d.realname))
+	server.snomasks.Send(sno.LocalConnects, fmt.Sprintf("Client connected [%s] [u:%s] [h:%s] [ip:%s] [r:%s]", d.nick, d.username, session.rawHostname, session.IP().String(), d.realname))
 
 	// send welcome text
 	//NOTE(dan): we specifically use the NICK here instead of the nickmask
 		return fmt.Errorf("Error loading config file config: %s", err.Error())
 	}
 
-	err = server.applyConfig(config, false)
+	err = server.applyConfig(config)
 	if err != nil {
 		return fmt.Errorf("Error applying config changes: %s", err.Error())
 	}
 	return nil
 }
 
-func (server *Server) applyConfig(config *Config, initial bool) (err error) {
+func (server *Server) applyConfig(config *Config) (err error) {
+	oldConfig := server.Config()
+	initial := oldConfig == nil
+
 	if initial {
 		server.configFilename = config.Filename
 		server.name = config.Server.Name
 		// enforce configs that can't be changed after launch:
 		if server.name != config.Server.Name {
 			return fmt.Errorf("Server name cannot be changed after launching the server, rehash aborted")
-		} else if server.Config().Datastore.Path != config.Datastore.Path {
+		} else if oldConfig.Datastore.Path != config.Datastore.Path {
 			return fmt.Errorf("Datastore path cannot be changed after launching the server, rehash aborted")
 		} else if globalCasemappingSetting != config.Server.Casemapping {
 			return fmt.Errorf("Casemapping cannot be changed after launching the server, rehash aborted")
 	}
 
 	server.logger.Info("server", "Using config file", server.configFilename)
-	oldConfig := server.Config()
 
 	// first, reload config sections for functionality implemented in subpackages:
 	wasLoggingRawIO := !initial && server.logger.IsLoggingRawIO()
 		if !oldConfig.Channels.Registration.Enabled {
 			server.channels.loadRegisteredChannels(config)
 		}
-
 		// resize history buffers as needed
 		if oldConfig.History != config.History {
 			for _, channel := range server.channels.Channels() {
-				channel.history.Resize(config.History.ChannelLength, config.History.AutoresizeWindow)
+				channel.resizeHistory(config)
 			}
 			for _, client := range server.clients.AllClients() {
-				client.history.Resize(config.History.ClientLength, config.History.AutoresizeWindow)
+				client.resizeHistory(config)
 			}
 		}
 	}
 		if err := server.loadDatastore(config); err != nil {
 			return err
 		}
+	} else {
+		if config.Datastore.MySQL.Enabled {
+			server.historyDB.SetExpireTime(config.History.Restrictions.ExpireTime)
+		}
 	}
 
 	server.setupPprofListener(config)
 	server.channels.Initialize(server)
 	server.accounts.Initialize(server)
 
+	if config.Datastore.MySQL.Enabled {
+		server.historyDB.Initialize(server.logger, config.History.Restrictions.ExpireTime)
+		err = server.historyDB.Open(config.Datastore.MySQL.User, config.Datastore.MySQL.Password, config.Datastore.MySQL.Host, config.Datastore.MySQL.Port, config.Datastore.MySQL.HistoryDatabase)
+		if err != nil {
+			server.logger.Error("internal", "could not connect to mysql", err.Error())
+			return err
+		}
+	}
+
 	return nil
 }
 
 	return
 }
 
+// Gets the abstract sequence from which we're going to query history;
+// we may already know the channel we're querying, or we may have
+// to look it up via a string target. This function is responsible for
+// privilege checking.
+func (server *Server) GetHistorySequence(providedChannel *Channel, client *Client, target string) (channel *Channel, sequence history.Sequence, err error) {
+	config := server.Config()
+	var sender, recipient string
+	var hist *history.Buffer
+	if target == "*" {
+		if client.AlwaysOn() {
+			recipient = client.NickCasefolded()
+		} else {
+			hist = &client.history
+		}
+	} else {
+		channel = providedChannel
+		if channel == nil {
+			channel = server.channels.Get(target)
+		}
+		if channel != nil {
+			if !channel.hasClient(client) {
+				err = errInsufficientPrivs
+				return
+			}
+			persistent, ephemeral, cfTarget := channel.historyStatus(config)
+			if persistent {
+				recipient = cfTarget
+			} else if ephemeral {
+				hist = &channel.history
+			} else {
+				return
+			}
+		} else {
+			sender = client.NickCasefolded()
+			var cfTarget string
+			cfTarget, err = CasefoldName(target)
+			if err != nil {
+				return
+			}
+			recipient = cfTarget
+			if !client.AlwaysOn() {
+				hist = &client.history
+			}
+		}
+	}
+
+	var cutoff time.Time
+	if config.History.Restrictions.ExpireTime != 0 {
+		cutoff = time.Now().UTC().Add(-config.History.Restrictions.ExpireTime)
+	}
+	if config.History.Restrictions.EnforceRegistrationDate {
+		regCutoff := client.historyCutoff()
+		regCutoff.Add(-config.History.Restrictions.GracePeriod)
+		// take the earlier of the two cutoffs
+		if regCutoff.After(cutoff) {
+			cutoff = regCutoff
+		}
+	}
+	if hist != nil {
+		sequence = hist.MakeSequence(recipient, cutoff)
+	} else if recipient != "" {
+		sequence = server.historyDB.MakeSequence(sender, recipient, cutoff)
+	}
+	return
+}
+
 // elistMatcher takes and matches ELIST conditions
 type elistMatcher struct {
 	MinClientsActive bool

irc/stats.go

 	s.mutex.Unlock()
 }
 
+// Activates a registered client, e.g., for the initial attach to a persistent client
+func (s *Stats) AddRegistered(invisible, operator bool) {
+	s.mutex.Lock()
+	if invisible {
+		s.Invisible += 1
+	}
+	if operator {
+		s.Operators += 1
+	}
+	s.Total += 1
+	s.setMax()
+	s.mutex.Unlock()
+}
+
 // Transition a client from unregistered to registered
 func (s *Stats) Register() {
 	s.mutex.Lock()
 	s.Unknown -= 1
 	s.Total += 1
+	s.setMax()
+	s.mutex.Unlock()
+}
+
+func (s *Stats) setMax() {
 	if s.Max < s.Total {
 		s.Max = s.Total
 	}
-	s.mutex.Unlock()
 }
 
 // Modify the Invisible count

irc/utils/args.go

 
 import (
 	"errors"
+	"fmt"
 	"strings"
+	"time"
+)
+
+const (
+	IRCv3TimestampFormat = "2006-01-02T15:04:05.000Z"
 )
 
 var (
 
 func StringToBool(str string) (result bool, err error) {
 	switch strings.ToLower(str) {
-	case "on", "true", "t", "yes", "y":
+	case "on", "true", "t", "yes", "y", "disabled":
 		result = true
-	case "off", "false", "f", "no", "n":
+	case "off", "false", "f", "no", "n", "enabled":
 		result = false
 	default:
 		err = ErrInvalidParams
 	}
 	return param
 }
+
+type IncompatibleSchemaError struct {
+	CurrentVersion  string
+	RequiredVersion string
+}
+
+func (err *IncompatibleSchemaError) Error() string {
+	return fmt.Sprintf("Database requires update. Expected schema v%s, got v%s", err.RequiredVersion, err.CurrentVersion)
+}
+
+func NanoToTimestamp(nanotime int64) string {
+	return time.Unix(0, nanotime).Format(IRCv3TimestampFormat)
+}

irc/utils/net.go

 	validHostnameLabelRegexp = regexp.MustCompile(`^[0-9A-Za-z.\-]+$`)
 )
 
-// AddrIsLocal returns whether the address is from a trusted local connection (loopback or unix).
-func AddrIsLocal(addr net.Addr) bool {
-	if tcpaddr, ok := addr.(*net.TCPAddr); ok {
-		return tcpaddr.IP.IsLoopback()
-	}
-	return AddrIsUnix(addr)
-}
-
 // AddrToIP returns the IP address for a net.Addr; unix domain sockets are treated as IPv4 loopback
 func AddrToIP(addr net.Addr) net.IP {
 	if tcpaddr, ok := addr.(*net.TCPAddr); ok {

irc/utils/text.go

 // SplitMessage represents a message that's been split for sending.
 // Two possibilities:
 // (a) Standard message that can be relayed on a single 512-byte line
-//     (MessagePair contains the message, Wrapped == nil)
+//     (MessagePair contains the message, Split == nil)
 // (b) multiline message that was split on the client side
-//     (Message == "", Wrapped contains the split lines)
+//     (Message == "", Split contains the split lines)
 type SplitMessage struct {
 	Message string
 	Msgid   string
 func MakeMessage(original string) (result SplitMessage) {
 	result.Message = original
 	result.Msgid = GenerateSecretToken()
-	result.Time = time.Now().UTC()
+	result.SetTime()
 
 	return
 }
 }
 
 func (sm *SplitMessage) SetTime() {
-	sm.Time = time.Now().UTC()
+	// strip the monotonic time, it's a potential source of problems:
+	sm.Time = time.Now().UTC().Round(0)
 }
 
 func (sm *SplitMessage) LenLines() int {
 	return false
 }
 
-func (sm *SplitMessage) IsMultiline() bool {
-	return sm.Message == "" && len(sm.Split) != 0
-}
-
 func (sm *SplitMessage) Is512() bool {
 	return sm.Message != ""
 }

irc/znc.go

 	"strconv"
 	"strings"
 	"time"
+
+	"github.com/oragono/oragono/irc/history"
 )
 
 type zncCommandHandler func(client *Client, command string, params []string, rb *ResponseBuffer)
 	//     3.3  When the client sends a subsequent redundant JOIN line for those
 	//          channels; redundant JOIN is a complete no-op so we won't replay twice
 
-	config := client.server.Config()
 	if params[1] == "*" {
-		items, _ := client.history.Between(after, before, false, config.History.ChathistoryMax)
-		client.replayPrivmsgHistory(rb, items, true)
+		zncPlayPrivmsgs(client, rb, after, before)
 	} else {
 		targets = make(StringSet)
 		// TODO actually handle nickname targets
 		}
 	}
 }
+
+func zncPlayPrivmsgs(client *Client, rb *ResponseBuffer, after, before time.Time) {
+	_, sequence, _ := client.server.GetHistorySequence(nil, client, "*")
+	if sequence == nil {
+		return
+	}
+	zncMax := client.server.Config().History.ZNCMax
+	items, _, err := sequence.Between(history.Selector{Time: after}, history.Selector{Time: before}, zncMax)
+	if err == nil {
+		client.replayPrivmsgHistory(rb, items, "", true)
+	}
+}

oragono.yaml

         # all users will receive simply `netname` as their cloaked hostname.
         num-bits: 80
 
+    # secure-nets identifies IPs and CIDRs which are secure at layer 3,
+    # for example, because they are on a trusted internal LAN or a VPN.
+    # plaintext connections from these IPs and CIDRs will be considered
+    # secure (clients will receive the +Z mode and be allowed to resume
+    # or reattach to secure connections). note that loopback IPs are always
+    # considered secure:
+    secure-nets:
+        # - "10.0.0.0/8"
+
 
 # account options
 accounts:
         # via nickserv
         allowed-by-default: true
 
+        # whether to allow clients that remain on the server even
+        # when they have no active connections. The possible values are:
+        # "disabled", "opt-in", "opt-out", or "mandatory".
+        always-on: "disabled"
+
     # vhosts controls the assignment of vhosts (strings displayed in place of the user's
     # hostname/IP) by the HostServ service
     vhosts:
     # up, and if the upgrade fails, the original database will be restored.
     autoupgrade: true
 
+    # connection information for MySQL (currently only used for persistent history):
+    mysql:
+        enabled: false
+        host: "localhost"
+        # port is unnecessary for connections via unix domain socket:
+        #port: 3306
+        user: "oragono"
+        password: "KOHw8WSaRwaoo-avo0qVpQ"
+        history-database: "oragono_history"
+
 # languages config
 languages:
     # whether to load languages
 # message history tracking, for the RESUME extension and possibly other uses in future
 history:
     # should we store messages for later playback?
-    # the current implementation stores messages in RAM only; they do not persist
+    # by default, messages are stored in RAM only; they do not persist
     # across server restarts. however, you should not enable this unless you understand
     # how it interacts with the GDPR and/or any data privacy laws that apply
     # in your country and the countries of your users.
     # maximum number of CHATHISTORY messages that can be
     # requested at once (0 disables support for CHATHISTORY)
     chathistory-maxmessages: 100
+
+    # maximum number of messages that can be replayed at once during znc emulation
+    # (znc.in/playback, or automatic replay on initial reattach to a persistent client):
+    znc-maxmessages: 2048
+
+    # options to delete old messages, or prevent them from being retrieved
+    restrictions:
+        # if this is set, messages older than this cannot be retrieved by anyone
+        # (and will eventually be deleted from persistent storage, if that's enabled)
+        #expire-time: 168h # 7 days
+
+        # if this is set, logged-in users cannot retrieve messages older than their
+        # account registration date, and logged-out users cannot retrieve messages
+        # older than their sign-on time (modulo grace-period, see below):
+        enforce-registration-date: false
+
+        # but if this is set, you can retrieve messages that are up to `grace-period`
+        # older than the above cutoff time. this is recommended to allow logged-out
+        # users to do session resumption / query history after disconnections.
+        grace-period: 1h
+
+    # options to store history messages in a persistent database (currently only MySQL):
+    persistent:
+        enabled: false
+
+        # store unregistered channel messages in the persistent database?
+        unregistered-channels: false
+
+        # for a registered channel, the channel owner can potentially customize
+        # the history storage setting. as the server operator, your options are
+        # 'disabled' (no persistent storage, regardless of per-channel setting),
+        # 'opt-in', 'opt-out', and 'mandatory' (force persistent storage, ignoring
+        # per-channel setting):
+        registered-channels: "opt-out"
+
+        # direct messages are only stored in the database for persistent clients;
+        # you can control how they are stored here (same options as above)
+        direct-messages: "opt-out"