msgstore: move ZNC log functions to separate package

This commit is contained in:
Simon Ser 2023-02-17 14:35:09 +01:00
parent e510f7a461
commit b6c0841291
4 changed files with 235 additions and 202 deletions

View File

@ -14,6 +14,7 @@ import (
"git.sr.ht/~emersion/soju/database" "git.sr.ht/~emersion/soju/database"
"git.sr.ht/~emersion/soju/msgstore" "git.sr.ht/~emersion/soju/msgstore"
"git.sr.ht/~emersion/soju/msgstore/znclog"
) )
const usage = `usage: migrate-logs <source logs> <destination database> const usage = `usage: migrate-logs <source logs> <destination database>
@ -88,7 +89,7 @@ func migrateNetwork(ctx context.Context, db database.Database, user *database.Us
} }
sc := bufio.NewScanner(entry) sc := bufio.NewScanner(entry)
for sc.Scan() { for sc.Scan() {
msg, _, err := msgstore.FSParseMessage(sc.Text(), user, network, target, ref, true) msg, _, err := znclog.UnmarshalLine(sc.Text(), user, network, target, ref, true)
if err != nil { if err != nil {
return fmt.Errorf("unable to parse entry: %s: %s", entryPath, sc.Text()) return fmt.Errorf("unable to parse entry: %s: %s", entryPath, sc.Text())
} else if msg == nil { } else if msg == nil {

View File

@ -15,6 +15,7 @@ import (
"gopkg.in/irc.v4" "gopkg.in/irc.v4"
"git.sr.ht/~emersion/soju/database" "git.sr.ht/~emersion/soju/database"
"git.sr.ht/~emersion/soju/msgstore/znclog"
"git.sr.ht/~emersion/soju/xirc" "git.sr.ht/~emersion/soju/xirc"
) )
@ -136,11 +137,6 @@ func (ms *fsMessageStore) LastMsgID(network *database.Network, entity string, t
} }
func (ms *fsMessageStore) Append(network *database.Network, entity string, msg *irc.Message) (string, error) { func (ms *fsMessageStore) Append(network *database.Network, entity string, msg *irc.Message) (string, error) {
s := formatMessage(msg)
if s == "" {
return "", nil
}
var t time.Time var t time.Time
if tag, ok := msg.Tags["time"]; ok { if tag, ok := msg.Tags["time"]; ok {
var err error var err error
@ -153,6 +149,11 @@ func (ms *fsMessageStore) Append(network *database.Network, entity string, msg *
t = time.Now() t = time.Now()
} }
s := znclog.MarshalLine(msg, t)
if s == "" {
return "", nil
}
f := ms.files[entity] f := ms.files[entity]
// TODO: handle non-monotonic clock behaviour // TODO: handle non-monotonic clock behaviour
@ -198,7 +199,7 @@ func (ms *fsMessageStore) Append(network *database.Network, entity string, msg *
return "", fmt.Errorf("failed to generate message ID: %v", err) return "", fmt.Errorf("failed to generate message ID: %v", err)
} }
_, err = fmt.Fprintf(f, "[%02d:%02d:%02d] %s\n", t.Hour(), t.Minute(), t.Second(), s) _, err = fmt.Fprintf(f, "%s\n", s)
if err != nil { if err != nil {
return "", fmt.Errorf("failed to log message to %q: %v", f.Name(), err) return "", fmt.Errorf("failed to log message to %q: %v", f.Name(), err)
} }
@ -216,202 +217,8 @@ func (ms *fsMessageStore) Close() error {
return closeErr return closeErr
} }
// formatMessage formats a message log line. It assumes a well-formed IRC
// message.
func formatMessage(msg *irc.Message) string {
switch strings.ToUpper(msg.Command) {
case "NICK":
return fmt.Sprintf("*** %s is now known as %s", msg.Prefix.Name, msg.Params[0])
case "JOIN":
return fmt.Sprintf("*** Joins: %s (%s@%s)", msg.Prefix.Name, msg.Prefix.User, msg.Prefix.Host)
case "PART":
var reason string
if len(msg.Params) > 1 {
reason = msg.Params[1]
}
return fmt.Sprintf("*** Parts: %s (%s@%s) (%s)", msg.Prefix.Name, msg.Prefix.User, msg.Prefix.Host, reason)
case "KICK":
nick := msg.Params[1]
var reason string
if len(msg.Params) > 2 {
reason = msg.Params[2]
}
return fmt.Sprintf("*** %s was kicked by %s (%s)", nick, msg.Prefix.Name, reason)
case "QUIT":
var reason string
if len(msg.Params) > 0 {
reason = msg.Params[0]
}
return fmt.Sprintf("*** Quits: %s (%s@%s) (%s)", msg.Prefix.Name, msg.Prefix.User, msg.Prefix.Host, reason)
case "TOPIC":
var topic string
if len(msg.Params) > 1 {
topic = msg.Params[1]
}
return fmt.Sprintf("*** %s changes topic to '%s'", msg.Prefix.Name, topic)
case "MODE":
return fmt.Sprintf("*** %s sets mode: %s", msg.Prefix.Name, strings.Join(msg.Params[1:], " "))
case "NOTICE":
return fmt.Sprintf("-%s- %s", msg.Prefix.Name, msg.Params[1])
case "PRIVMSG":
if cmd, params, ok := xirc.ParseCTCPMessage(msg); ok && cmd == "ACTION" {
return fmt.Sprintf("* %s %s", msg.Prefix.Name, params)
} else {
return fmt.Sprintf("<%s> %s", msg.Prefix.Name, msg.Params[1])
}
default:
return ""
}
}
func (ms *fsMessageStore) parseMessage(line string, network *database.Network, entity string, ref time.Time, events bool) (*irc.Message, time.Time, error) { func (ms *fsMessageStore) parseMessage(line string, network *database.Network, entity string, ref time.Time, events bool) (*irc.Message, time.Time, error) {
return FSParseMessage(line, ms.user, network, entity, ref, events) return znclog.UnmarshalLine(line, ms.user, network, entity, ref, events)
}
func FSParseMessage(line string, user *database.User, network *database.Network, entity string, ref time.Time, events bool) (*irc.Message, time.Time, error) {
var hour, minute, second int
_, err := fmt.Sscanf(line, "[%02d:%02d:%02d] ", &hour, &minute, &second)
if err != nil {
return nil, time.Time{}, fmt.Errorf("malformed timestamp prefix: %v", err)
}
line = line[11:]
var cmd string
var prefix *irc.Prefix
var params []string
if events && strings.HasPrefix(line, "*** ") {
parts := strings.SplitN(line[4:], " ", 2)
if len(parts) != 2 {
return nil, time.Time{}, nil
}
switch parts[0] {
case "Joins:", "Parts:", "Quits:":
args := strings.SplitN(parts[1], " ", 3)
if len(args) < 2 {
return nil, time.Time{}, nil
}
nick := args[0]
mask := strings.TrimSuffix(strings.TrimPrefix(args[1], "("), ")")
maskParts := strings.SplitN(mask, "@", 2)
if len(maskParts) != 2 {
return nil, time.Time{}, nil
}
prefix = &irc.Prefix{
Name: nick,
User: maskParts[0],
Host: maskParts[1],
}
var reason string
if len(args) > 2 {
reason = strings.TrimSuffix(strings.TrimPrefix(args[2], "("), ")")
}
switch parts[0] {
case "Joins:":
cmd = "JOIN"
params = []string{entity}
case "Parts:":
cmd = "PART"
if reason != "" {
params = []string{entity, reason}
} else {
params = []string{entity}
}
case "Quits:":
cmd = "QUIT"
if reason != "" {
params = []string{reason}
}
}
default:
nick := parts[0]
rem := parts[1]
if r := strings.TrimPrefix(rem, "is now known as "); r != rem {
cmd = "NICK"
prefix = &irc.Prefix{
Name: nick,
}
params = []string{r}
} else if r := strings.TrimPrefix(rem, "was kicked by "); r != rem {
args := strings.SplitN(r, " ", 2)
if len(args) != 2 {
return nil, time.Time{}, nil
}
cmd = "KICK"
prefix = &irc.Prefix{
Name: args[0],
}
reason := strings.TrimSuffix(strings.TrimPrefix(args[1], "("), ")")
params = []string{entity, nick}
if reason != "" {
params = append(params, reason)
}
} else if r := strings.TrimPrefix(rem, "changes topic to "); r != rem {
cmd = "TOPIC"
prefix = &irc.Prefix{
Name: nick,
}
topic := strings.TrimSuffix(strings.TrimPrefix(r, "'"), "'")
params = []string{entity, topic}
} else if r := strings.TrimPrefix(rem, "sets mode: "); r != rem {
cmd = "MODE"
prefix = &irc.Prefix{
Name: nick,
}
params = append([]string{entity}, strings.Split(r, " ")...)
} else {
return nil, time.Time{}, nil
}
}
} else {
var sender, text string
if strings.HasPrefix(line, "<") {
cmd = "PRIVMSG"
parts := strings.SplitN(line[1:], "> ", 2)
if len(parts) != 2 {
return nil, time.Time{}, nil
}
sender, text = parts[0], parts[1]
} else if strings.HasPrefix(line, "-") {
cmd = "NOTICE"
parts := strings.SplitN(line[1:], "- ", 2)
if len(parts) != 2 {
return nil, time.Time{}, nil
}
sender, text = parts[0], parts[1]
} else if strings.HasPrefix(line, "* ") {
cmd = "PRIVMSG"
parts := strings.SplitN(line[2:], " ", 2)
if len(parts) != 2 {
return nil, time.Time{}, nil
}
sender, text = parts[0], "\x01ACTION "+parts[1]+"\x01"
} else {
return nil, time.Time{}, nil
}
prefix = &irc.Prefix{Name: sender}
if entity == sender {
// This is a direct message from a user to us. We don't store own
// our nickname in the logs, so grab it from the network settings.
// Not very accurate since this may not match our nick at the time
// the message was received, but we can't do a lot better.
entity = database.GetNick(user, network)
}
params = []string{entity, text}
}
year, month, day := ref.Date()
t := time.Date(year, month, day, hour, minute, second, 0, time.Local)
msg := &irc.Message{
Tags: map[string]string{
"time": xirc.FormatServerTime(t),
},
Prefix: prefix,
Command: cmd,
Params: params,
}
return msg, t, nil
} }
func (ms *fsMessageStore) parseMessagesBefore(ref time.Time, end time.Time, options *LoadMessageOptions, afterOffset int64, selector func(m *irc.Message) bool) ([]*irc.Message, error) { func (ms *fsMessageStore) parseMessagesBefore(ref time.Time, end time.Time, options *LoadMessageOptions, afterOffset int64, selector func(m *irc.Message) bool) ([]*irc.Message, error) {

158
msgstore/znclog/reader.go Normal file
View File

@ -0,0 +1,158 @@
package znclog
import (
"fmt"
"strings"
"time"
"gopkg.in/irc.v4"
"git.sr.ht/~emersion/soju/database"
"git.sr.ht/~emersion/soju/xirc"
)
func UnmarshalLine(line string, user *database.User, network *database.Network, entity string, ref time.Time, events bool) (*irc.Message, time.Time, error) {
var hour, minute, second int
_, err := fmt.Sscanf(line, "[%02d:%02d:%02d] ", &hour, &minute, &second)
if err != nil {
return nil, time.Time{}, fmt.Errorf("malformed timestamp prefix: %v", err)
}
line = line[11:]
var cmd string
var prefix *irc.Prefix
var params []string
if events && strings.HasPrefix(line, "*** ") {
parts := strings.SplitN(line[4:], " ", 2)
if len(parts) != 2 {
return nil, time.Time{}, nil
}
switch parts[0] {
case "Joins:", "Parts:", "Quits:":
args := strings.SplitN(parts[1], " ", 3)
if len(args) < 2 {
return nil, time.Time{}, nil
}
nick := args[0]
mask := strings.TrimSuffix(strings.TrimPrefix(args[1], "("), ")")
maskParts := strings.SplitN(mask, "@", 2)
if len(maskParts) != 2 {
return nil, time.Time{}, nil
}
prefix = &irc.Prefix{
Name: nick,
User: maskParts[0],
Host: maskParts[1],
}
var reason string
if len(args) > 2 {
reason = strings.TrimSuffix(strings.TrimPrefix(args[2], "("), ")")
}
switch parts[0] {
case "Joins:":
cmd = "JOIN"
params = []string{entity}
case "Parts:":
cmd = "PART"
if reason != "" {
params = []string{entity, reason}
} else {
params = []string{entity}
}
case "Quits:":
cmd = "QUIT"
if reason != "" {
params = []string{reason}
}
}
default:
nick := parts[0]
rem := parts[1]
if r := strings.TrimPrefix(rem, "is now known as "); r != rem {
cmd = "NICK"
prefix = &irc.Prefix{
Name: nick,
}
params = []string{r}
} else if r := strings.TrimPrefix(rem, "was kicked by "); r != rem {
args := strings.SplitN(r, " ", 2)
if len(args) != 2 {
return nil, time.Time{}, nil
}
cmd = "KICK"
prefix = &irc.Prefix{
Name: args[0],
}
reason := strings.TrimSuffix(strings.TrimPrefix(args[1], "("), ")")
params = []string{entity, nick}
if reason != "" {
params = append(params, reason)
}
} else if r := strings.TrimPrefix(rem, "changes topic to "); r != rem {
cmd = "TOPIC"
prefix = &irc.Prefix{
Name: nick,
}
topic := strings.TrimSuffix(strings.TrimPrefix(r, "'"), "'")
params = []string{entity, topic}
} else if r := strings.TrimPrefix(rem, "sets mode: "); r != rem {
cmd = "MODE"
prefix = &irc.Prefix{
Name: nick,
}
params = append([]string{entity}, strings.Split(r, " ")...)
} else {
return nil, time.Time{}, nil
}
}
} else {
var sender, text string
if strings.HasPrefix(line, "<") {
cmd = "PRIVMSG"
parts := strings.SplitN(line[1:], "> ", 2)
if len(parts) != 2 {
return nil, time.Time{}, nil
}
sender, text = parts[0], parts[1]
} else if strings.HasPrefix(line, "-") {
cmd = "NOTICE"
parts := strings.SplitN(line[1:], "- ", 2)
if len(parts) != 2 {
return nil, time.Time{}, nil
}
sender, text = parts[0], parts[1]
} else if strings.HasPrefix(line, "* ") {
cmd = "PRIVMSG"
parts := strings.SplitN(line[2:], " ", 2)
if len(parts) != 2 {
return nil, time.Time{}, nil
}
sender, text = parts[0], "\x01ACTION "+parts[1]+"\x01"
} else {
return nil, time.Time{}, nil
}
prefix = &irc.Prefix{Name: sender}
if entity == sender {
// This is a direct message from a user to us. We don't store own
// our nickname in the logs, so grab it from the network settings.
// Not very accurate since this may not match our nick at the time
// the message was received, but we can't do a lot better.
entity = database.GetNick(user, network)
}
params = []string{entity, text}
}
year, month, day := ref.Date()
t := time.Date(year, month, day, hour, minute, second, 0, time.Local)
msg := &irc.Message{
Tags: map[string]string{
"time": xirc.FormatServerTime(t),
},
Prefix: prefix,
Command: cmd,
Params: params,
}
return msg, t, nil
}

67
msgstore/znclog/writer.go Normal file
View File

@ -0,0 +1,67 @@
package znclog
import (
"fmt"
"strings"
"time"
"gopkg.in/irc.v4"
"git.sr.ht/~emersion/soju/xirc"
)
func MarshalLine(msg *irc.Message, t time.Time) string {
s := formatMessage(msg)
if s == "" {
return ""
}
return fmt.Sprintf("[%02d:%02d:%02d] %s", t.Hour(), t.Minute(), t.Second(), s)
}
// formatMessage formats a message log line. It assumes a well-formed IRC
// message.
func formatMessage(msg *irc.Message) string {
switch strings.ToUpper(msg.Command) {
case "NICK":
return fmt.Sprintf("*** %s is now known as %s", msg.Prefix.Name, msg.Params[0])
case "JOIN":
return fmt.Sprintf("*** Joins: %s (%s@%s)", msg.Prefix.Name, msg.Prefix.User, msg.Prefix.Host)
case "PART":
var reason string
if len(msg.Params) > 1 {
reason = msg.Params[1]
}
return fmt.Sprintf("*** Parts: %s (%s@%s) (%s)", msg.Prefix.Name, msg.Prefix.User, msg.Prefix.Host, reason)
case "KICK":
nick := msg.Params[1]
var reason string
if len(msg.Params) > 2 {
reason = msg.Params[2]
}
return fmt.Sprintf("*** %s was kicked by %s (%s)", nick, msg.Prefix.Name, reason)
case "QUIT":
var reason string
if len(msg.Params) > 0 {
reason = msg.Params[0]
}
return fmt.Sprintf("*** Quits: %s (%s@%s) (%s)", msg.Prefix.Name, msg.Prefix.User, msg.Prefix.Host, reason)
case "TOPIC":
var topic string
if len(msg.Params) > 1 {
topic = msg.Params[1]
}
return fmt.Sprintf("*** %s changes topic to '%s'", msg.Prefix.Name, topic)
case "MODE":
return fmt.Sprintf("*** %s sets mode: %s", msg.Prefix.Name, strings.Join(msg.Params[1:], " "))
case "NOTICE":
return fmt.Sprintf("-%s- %s", msg.Prefix.Name, msg.Params[1])
case "PRIVMSG":
if cmd, params, ok := xirc.ParseCTCPMessage(msg); ok && cmd == "ACTION" {
return fmt.Sprintf("* %s %s", msg.Prefix.Name, params)
} else {
return fmt.Sprintf("<%s> %s", msg.Prefix.Name, msg.Params[1])
}
default:
return ""
}
}