diff --git a/irc.go b/irc.go
index 876bd90..9f417a0 100644
--- a/irc.go
+++ b/irc.go
@@ -848,3 +848,15 @@ func (cr *capRegistry) SetEnabled(name string, enabled bool) {
 		delete(cr.Enabled, name)
 	}
 }
+
+func isNumeric(cmd string) bool {
+	if len(cmd) != 3 {
+		return false
+	}
+	for i := 0; i < 3; i++ {
+		if cmd[i] < '0' || cmd[i] > '9' {
+			return false
+		}
+	}
+	return true
+}
diff --git a/upstream.go b/upstream.go
index 7961faf..56d604e 100644
--- a/upstream.go
+++ b/upstream.go
@@ -450,7 +450,7 @@ func (uc *upstreamConn) handleMessage(ctx context.Context, msg *irc.Message) err
 		msg.Prefix = uc.serverPrefix
 	}
 
-	if _, ok := msg.Tags["time"]; !ok {
+	if _, ok := msg.Tags["time"]; !ok && !isNumeric(msg.Command) {
 		msg.Tags["time"] = irc.TagValue(formatServerTime(time.Now()))
 	}