|
@@ -4,7 +4,6 @@
|
4
|
4
|
package mysql
|
5
|
5
|
|
6
|
6
|
import (
|
7
|
|
- "bytes"
|
8
|
7
|
"context"
|
9
|
8
|
"database/sql"
|
10
|
9
|
"encoding/json"
|
|
@@ -12,6 +11,7 @@ import (
|
12
|
11
|
"fmt"
|
13
|
12
|
"io"
|
14
|
13
|
"runtime/debug"
|
|
14
|
+ "strings"
|
15
|
15
|
"sync"
|
16
|
16
|
"sync/atomic"
|
17
|
17
|
"time"
|
|
@@ -36,7 +36,7 @@ const (
|
36
|
36
|
keySchemaVersion = "db.version"
|
37
|
37
|
// minor version indicates rollback-safe upgrades, i.e.,
|
38
|
38
|
// you can downgrade oragono and everything will work
|
39
|
|
- latestDbMinorVersion = "1"
|
|
39
|
+ latestDbMinorVersion = "2"
|
40
|
40
|
keySchemaMinorVersion = "db.minorversion"
|
41
|
41
|
cleanupRowLimit = 50
|
42
|
42
|
cleanupPauseTime = 10 * time.Minute
|
|
@@ -53,6 +53,7 @@ type MySQL struct {
|
53
|
53
|
insertHistory *sql.Stmt
|
54
|
54
|
insertSequence *sql.Stmt
|
55
|
55
|
insertConversation *sql.Stmt
|
|
56
|
+ insertCorrespondent *sql.Stmt
|
56
|
57
|
insertAccountMessage *sql.Stmt
|
57
|
58
|
|
58
|
59
|
stateMutex sync.Mutex
|
|
@@ -155,10 +156,24 @@ func (mysql *MySQL) fixSchemas() (err error) {
|
155
|
156
|
if err != nil {
|
156
|
157
|
return
|
157
|
158
|
}
|
|
159
|
+ err = mysql.createCorrespondentsTable()
|
|
160
|
+ if err != nil {
|
|
161
|
+ return
|
|
162
|
+ }
|
158
|
163
|
_, err = mysql.db.Exec(`insert into metadata (key_name, value) values (?, ?);`, keySchemaMinorVersion, latestDbMinorVersion)
|
159
|
164
|
if err != nil {
|
160
|
165
|
return
|
161
|
166
|
}
|
|
167
|
+ } else if err == nil && minorVersion == "1" {
|
|
168
|
+ // upgrade from 2.1 to 2.2: create the correspondents table
|
|
169
|
+ err = mysql.createCorrespondentsTable()
|
|
170
|
+ if err != nil {
|
|
171
|
+ return
|
|
172
|
+ }
|
|
173
|
+ _, err = mysql.db.Exec(`update metadata set value = ? where key_name = ?;`, latestDbMinorVersion, keySchemaMinorVersion)
|
|
174
|
+ if err != nil {
|
|
175
|
+ return
|
|
176
|
+ }
|
162
|
177
|
} else if err == nil && minorVersion != latestDbMinorVersion {
|
163
|
178
|
// TODO: if minorVersion < latestDbMinorVersion, upgrade,
|
164
|
179
|
// if latestDbMinorVersion < minorVersion, ignore because backwards compatible
|
|
@@ -202,6 +217,11 @@ func (mysql *MySQL) createTables() (err error) {
|
202
|
217
|
return err
|
203
|
218
|
}
|
204
|
219
|
|
|
220
|
+ err = mysql.createCorrespondentsTable()
|
|
221
|
+ if err != nil {
|
|
222
|
+ return err
|
|
223
|
+ }
|
|
224
|
+
|
205
|
225
|
err = mysql.createComplianceTables()
|
206
|
226
|
if err != nil {
|
207
|
227
|
return err
|
|
@@ -210,6 +230,19 @@ func (mysql *MySQL) createTables() (err error) {
|
210
|
230
|
return nil
|
211
|
231
|
}
|
212
|
232
|
|
|
233
|
+func (mysql *MySQL) createCorrespondentsTable() (err error) {
|
|
234
|
+ _, err = mysql.db.Exec(fmt.Sprintf(`CREATE TABLE correspondents (
|
|
235
|
+ id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
|
|
236
|
+ target VARBINARY(%[1]d) NOT NULL,
|
|
237
|
+ correspondent VARBINARY(%[1]d) NOT NULL,
|
|
238
|
+ nanotime BIGINT UNSIGNED NOT NULL,
|
|
239
|
+ UNIQUE KEY (target, correspondent),
|
|
240
|
+ KEY (target, nanotime),
|
|
241
|
+ KEY (nanotime)
|
|
242
|
+ ) CHARSET=ascii COLLATE=ascii_bin;`, MaxTargetLength))
|
|
243
|
+ return
|
|
244
|
+}
|
|
245
|
+
|
213
|
246
|
func (mysql *MySQL) createComplianceTables() (err error) {
|
214
|
247
|
_, err = mysql.db.Exec(fmt.Sprintf(`CREATE TABLE account_messages (
|
215
|
248
|
history_id BIGINT UNSIGNED NOT NULL PRIMARY KEY,
|
|
@@ -275,12 +308,16 @@ func (mysql *MySQL) doCleanup(age time.Duration) (count int, err error) {
|
275
|
308
|
|
276
|
309
|
mysql.logger.Debug("mysql", fmt.Sprintf("deleting %d history rows, max age %s", len(ids), utils.NanoToTimestamp(maxNanotime)))
|
277
|
310
|
|
|
311
|
+ if maxNanotime != 0 {
|
|
312
|
+ mysql.deleteCorrespondents(ctx, maxNanotime)
|
|
313
|
+ }
|
|
314
|
+
|
278
|
315
|
return len(ids), mysql.deleteHistoryIDs(ctx, ids)
|
279
|
316
|
}
|
280
|
317
|
|
281
|
318
|
func (mysql *MySQL) deleteHistoryIDs(ctx context.Context, ids []uint64) (err error) {
|
282
|
319
|
// can't use ? binding for a variable number of arguments, build the IN clause manually
|
283
|
|
- var inBuf bytes.Buffer
|
|
320
|
+ var inBuf strings.Builder
|
284
|
321
|
inBuf.WriteByte('(')
|
285
|
322
|
for i, id := range ids {
|
286
|
323
|
if i != 0 {
|
|
@@ -289,22 +326,23 @@ func (mysql *MySQL) deleteHistoryIDs(ctx context.Context, ids []uint64) (err err
|
289
|
326
|
fmt.Fprintf(&inBuf, "%d", id)
|
290
|
327
|
}
|
291
|
328
|
inBuf.WriteRune(')')
|
|
329
|
+ inClause := inBuf.String()
|
292
|
330
|
|
293
|
|
- _, err = mysql.db.ExecContext(ctx, fmt.Sprintf(`DELETE FROM conversations WHERE history_id in %s;`, inBuf.Bytes()))
|
|
331
|
+ _, err = mysql.db.ExecContext(ctx, fmt.Sprintf(`DELETE FROM conversations WHERE history_id in %s;`, inClause))
|
294
|
332
|
if err != nil {
|
295
|
333
|
return
|
296
|
334
|
}
|
297
|
|
- _, err = mysql.db.ExecContext(ctx, fmt.Sprintf(`DELETE FROM sequence WHERE history_id in %s;`, inBuf.Bytes()))
|
|
335
|
+ _, err = mysql.db.ExecContext(ctx, fmt.Sprintf(`DELETE FROM sequence WHERE history_id in %s;`, inClause))
|
298
|
336
|
if err != nil {
|
299
|
337
|
return
|
300
|
338
|
}
|
301
|
339
|
if mysql.isTrackingAccountMessages() {
|
302
|
|
- _, err = mysql.db.ExecContext(ctx, fmt.Sprintf(`DELETE FROM account_messages WHERE history_id in %s;`, inBuf.Bytes()))
|
|
340
|
+ _, err = mysql.db.ExecContext(ctx, fmt.Sprintf(`DELETE FROM account_messages WHERE history_id in %s;`, inClause))
|
303
|
341
|
if err != nil {
|
304
|
342
|
return
|
305
|
343
|
}
|
306
|
344
|
}
|
307
|
|
- _, err = mysql.db.ExecContext(ctx, fmt.Sprintf(`DELETE FROM history WHERE id in %s;`, inBuf.Bytes()))
|
|
345
|
+ _, err = mysql.db.ExecContext(ctx, fmt.Sprintf(`DELETE FROM history WHERE id in %s;`, inClause))
|
308
|
346
|
if err != nil {
|
309
|
347
|
return
|
310
|
348
|
}
|
|
@@ -351,6 +389,18 @@ func (mysql *MySQL) selectCleanupIDs(ctx context.Context, age time.Duration) (id
|
351
|
389
|
return
|
352
|
390
|
}
|
353
|
391
|
|
|
392
|
+func (mysql *MySQL) deleteCorrespondents(ctx context.Context, threshold int64) {
|
|
393
|
+ result, err := mysql.db.ExecContext(ctx, `DELETE FROM correspondents WHERE nanotime <= (?);`, threshold)
|
|
394
|
+ if err != nil {
|
|
395
|
+ mysql.logError("error deleting correspondents", err)
|
|
396
|
+ } else {
|
|
397
|
+ count, err := result.RowsAffected()
|
|
398
|
+ if err != nil {
|
|
399
|
+ mysql.logger.Debug(fmt.Sprintf("deleted %d correspondents entries", count))
|
|
400
|
+ }
|
|
401
|
+ }
|
|
402
|
+}
|
|
403
|
+
|
354
|
404
|
// wait for forget queue items and process them one by one
|
355
|
405
|
func (mysql *MySQL) forgetLoop() {
|
356
|
406
|
defer func() {
|
|
@@ -470,6 +520,12 @@ func (mysql *MySQL) prepareStatements() (err error) {
|
470
|
520
|
if err != nil {
|
471
|
521
|
return
|
472
|
522
|
}
|
|
523
|
+ mysql.insertCorrespondent, err = mysql.db.Prepare(`INSERT INTO correspondents
|
|
524
|
+ (target, correspondent, nanotime) VALUES (?, ?, ?)
|
|
525
|
+ ON DUPLICATE KEY UPDATE nanotime = GREATEST(nanotime, ?);`)
|
|
526
|
+ if err != nil {
|
|
527
|
+ return
|
|
528
|
+ }
|
473
|
529
|
mysql.insertAccountMessage, err = mysql.db.Prepare(`INSERT INTO account_messages
|
474
|
530
|
(history_id, account) VALUES (?, ?);`)
|
475
|
531
|
if err != nil {
|
|
@@ -557,6 +613,12 @@ func (mysql *MySQL) insertConversationEntry(ctx context.Context, target, corresp
|
557
|
613
|
return
|
558
|
614
|
}
|
559
|
615
|
|
|
616
|
+func (mysql *MySQL) insertCorrespondentsEntry(ctx context.Context, target, correspondent string, messageTime int64, historyId int64) (err error) {
|
|
617
|
+ _, err = mysql.insertCorrespondent.ExecContext(ctx, target, correspondent, messageTime, messageTime)
|
|
618
|
+ mysql.logError("could not insert conversations entry", err)
|
|
619
|
+ return
|
|
620
|
+}
|
|
621
|
+
|
560
|
622
|
func (mysql *MySQL) insertBase(ctx context.Context, item history.Item) (id int64, err error) {
|
561
|
623
|
value, err := marshalItem(&item)
|
562
|
624
|
if mysql.logError("could not marshal item", err) {
|
|
@@ -621,6 +683,10 @@ func (mysql *MySQL) AddDirectMessage(sender, senderAccount, recipient, recipient
|
621
|
683
|
if err != nil {
|
622
|
684
|
return
|
623
|
685
|
}
|
|
686
|
+ err = mysql.insertCorrespondentsEntry(ctx, senderAccount, recipient, nanotime, id)
|
|
687
|
+ if err != nil {
|
|
688
|
+ return
|
|
689
|
+ }
|
624
|
690
|
}
|
625
|
691
|
|
626
|
692
|
if recipientAccount != "" && sender != recipient {
|
|
@@ -632,6 +698,10 @@ func (mysql *MySQL) AddDirectMessage(sender, senderAccount, recipient, recipient
|
632
|
698
|
if err != nil {
|
633
|
699
|
return
|
634
|
700
|
}
|
|
701
|
+ err = mysql.insertCorrespondentsEntry(ctx, recipientAccount, sender, nanotime, id)
|
|
702
|
+ if err != nil {
|
|
703
|
+ return
|
|
704
|
+ }
|
635
|
705
|
}
|
636
|
706
|
|
637
|
707
|
err = mysql.insertAccountMessageEntry(ctx, id, senderAccount)
|
|
@@ -804,7 +874,7 @@ func (mysql *MySQL) betweenTimestamps(ctx context.Context, target, correspondent
|
804
|
874
|
direction = "DESC"
|
805
|
875
|
}
|
806
|
876
|
|
807
|
|
- var queryBuf bytes.Buffer
|
|
877
|
+ var queryBuf strings.Builder
|
808
|
878
|
|
809
|
879
|
args := make([]interface{}, 0, 6)
|
810
|
880
|
fmt.Fprintf(&queryBuf,
|
|
@@ -835,6 +905,55 @@ func (mysql *MySQL) betweenTimestamps(ctx context.Context, target, correspondent
|
835
|
905
|
return
|
836
|
906
|
}
|
837
|
907
|
|
|
908
|
+func (mysql *MySQL) listCorrespondentsInternal(ctx context.Context, target string, after, before, cutoff time.Time, limit int) (results []history.CorrespondentListing, err error) {
|
|
909
|
+ after, before, ascending := history.MinMaxAsc(after, before, cutoff)
|
|
910
|
+ direction := "ASC"
|
|
911
|
+ if !ascending {
|
|
912
|
+ direction = "DESC"
|
|
913
|
+ }
|
|
914
|
+
|
|
915
|
+ var queryBuf strings.Builder
|
|
916
|
+ args := make([]interface{}, 0, 4)
|
|
917
|
+ queryBuf.WriteString(`SELECT correspondents.correspondent, correspondents.nanotime from correspondents
|
|
918
|
+ WHERE target = ?`)
|
|
919
|
+ args = append(args, target)
|
|
920
|
+ if !after.IsZero() {
|
|
921
|
+ queryBuf.WriteString(" AND correspondents.nanotime > ?")
|
|
922
|
+ args = append(args, after.UnixNano())
|
|
923
|
+ }
|
|
924
|
+ if !before.IsZero() {
|
|
925
|
+ queryBuf.WriteString(" AND correspondents.nanotime < ?")
|
|
926
|
+ args = append(args, before.UnixNano())
|
|
927
|
+ }
|
|
928
|
+ fmt.Fprintf(&queryBuf, " ORDER BY correspondents.nanotime %s LIMIT ?;", direction)
|
|
929
|
+ args = append(args, limit)
|
|
930
|
+ query := queryBuf.String()
|
|
931
|
+
|
|
932
|
+ rows, err := mysql.db.QueryContext(ctx, query, args...)
|
|
933
|
+ if err != nil {
|
|
934
|
+ return
|
|
935
|
+ }
|
|
936
|
+ defer rows.Close()
|
|
937
|
+ var correspondent string
|
|
938
|
+ var nanotime int64
|
|
939
|
+ for rows.Next() {
|
|
940
|
+ err = rows.Scan(&correspondent, &nanotime)
|
|
941
|
+ if err != nil {
|
|
942
|
+ return
|
|
943
|
+ }
|
|
944
|
+ results = append(results, history.CorrespondentListing{
|
|
945
|
+ CfCorrespondent: correspondent,
|
|
946
|
+ Time: time.Unix(0, nanotime),
|
|
947
|
+ })
|
|
948
|
+ }
|
|
949
|
+
|
|
950
|
+ if !ascending {
|
|
951
|
+ history.ReverseCorrespondents(results)
|
|
952
|
+ }
|
|
953
|
+
|
|
954
|
+ return
|
|
955
|
+}
|
|
956
|
+
|
838
|
957
|
func (mysql *MySQL) Close() {
|
839
|
958
|
// closing the database will close our prepared statements as well
|
840
|
959
|
if mysql.db != nil {
|
|
@@ -852,7 +971,7 @@ type mySQLHistorySequence struct {
|
852
|
971
|
cutoff time.Time
|
853
|
972
|
}
|
854
|
973
|
|
855
|
|
-func (s *mySQLHistorySequence) Between(start, end history.Selector, limit int) (results []history.Item, complete bool, err error) {
|
|
974
|
+func (s *mySQLHistorySequence) Between(start, end history.Selector, limit int) (results []history.Item, err error) {
|
856
|
975
|
ctx, cancel := context.WithTimeout(context.Background(), s.mysql.getTimeout())
|
857
|
976
|
defer cancel()
|
858
|
977
|
|
|
@@ -860,25 +979,38 @@ func (s *mySQLHistorySequence) Between(start, end history.Selector, limit int) (
|
860
|
979
|
if start.Msgid != "" {
|
861
|
980
|
startTime, _, _, err = s.mysql.lookupMsgid(ctx, start.Msgid, false)
|
862
|
981
|
if err != nil {
|
863
|
|
- return nil, false, err
|
|
982
|
+ return nil, err
|
864
|
983
|
}
|
865
|
984
|
}
|
866
|
985
|
endTime := end.Time
|
867
|
986
|
if end.Msgid != "" {
|
868
|
987
|
endTime, _, _, err = s.mysql.lookupMsgid(ctx, end.Msgid, false)
|
869
|
988
|
if err != nil {
|
870
|
|
- return nil, false, err
|
|
989
|
+ return nil, err
|
871
|
990
|
}
|
872
|
991
|
}
|
873
|
992
|
|
874
|
993
|
results, err = s.mysql.betweenTimestamps(ctx, s.target, s.correspondent, startTime, endTime, s.cutoff, limit)
|
875
|
|
- return results, (err == nil), err
|
|
994
|
+ return results, err
|
876
|
995
|
}
|
877
|
996
|
|
878
|
997
|
func (s *mySQLHistorySequence) Around(start history.Selector, limit int) (results []history.Item, err error) {
|
879
|
998
|
return history.GenericAround(s, start, limit)
|
880
|
999
|
}
|
881
|
1000
|
|
|
1001
|
+func (seq *mySQLHistorySequence) ListCorrespondents(start, end history.Selector, limit int) (results []history.CorrespondentListing, err error) {
|
|
1002
|
+ ctx, cancel := context.WithTimeout(context.Background(), seq.mysql.getTimeout())
|
|
1003
|
+ defer cancel()
|
|
1004
|
+
|
|
1005
|
+ // TODO accept msgids here?
|
|
1006
|
+ startTime := start.Time
|
|
1007
|
+ endTime := end.Time
|
|
1008
|
+
|
|
1009
|
+ results, err = seq.mysql.listCorrespondentsInternal(ctx, seq.target, startTime, endTime, seq.cutoff, limit)
|
|
1010
|
+ seq.mysql.logError("could not read correspondents", err)
|
|
1011
|
+ return
|
|
1012
|
+}
|
|
1013
|
+
|
882
|
1014
|
func (mysql *MySQL) MakeSequence(target, correspondent string, cutoff time.Time) history.Sequence {
|
883
|
1015
|
return &mySQLHistorySequence{
|
884
|
1016
|
target: target,
|