chatDB tests + message length identical with WA

.gitignore

@@ -11,3 +11,4 @@ auth_info2.json
 lib
 docs
 browser-token.json
+auth_info_messcat.json

src/Tests/Common.ts

@@ -1,4 +1,4 @@
-import { WAConnection, MessageLogLevel, MessageOptions, MessageType, unixTimestampSeconds } from '../WAConnection/WAConnection'
+import { WAConnection, MessageLogLevel, MessageOptions, MessageType, unixTimestampSeconds, toNumber } from '../WAConnection/WAConnection'
 import * as assert from 'assert'
 import {promises as fs} from 'fs'
 
@@ -13,6 +13,7 @@ export async function sendAndRetreiveMessage(conn: WAConnection, content, type:
 
     const chat = conn.chats.get(testJid)
 
+    assertChatDBIntegrity (conn)
     assert.ok (chat.messages.find(m => m.key.id === response.key.id))
     assert.ok (chat.t >= (unixTimestampSeconds()-5) )
     return message
@@ -32,4 +33,18 @@ export const WAConnectionTest = (name: string, func: (conn: WAConnection) => voi
         
         func(conn)
     })
-)
+)
+export const assertChatDBIntegrity = (conn: WAConnection) => {
+    conn.chats.all ().forEach (chat => (
+        assert.equal (
+            chat.messages.sort ((m1, m2) => toNumber(m1.messageTimestamp)-toNumber(m2.messageTimestamp)),
+            chat.messages
+        )
+    ))
+    conn.chats.all ().forEach (chat => (
+        assert.deepEqual (
+            chat.messages.filter (m => chat.messages.filter(m1 => m1.key.id === m.key.id).length > 1),
+            []
+        )
+    ))
+}

src/Tests/Tests.Connect.ts

@@ -2,6 +2,7 @@ import * as assert from 'assert'
 import {WAConnection} from '../WAConnection/WAConnection'
 import { AuthenticationCredentialsBase64, BaileysError, ReconnectMode, DisconnectReason } from '../WAConnection/Constants'
 import { delay } from '../WAConnection/Utils'
+import { assertChatDBIntegrity } from './Common'
 
 describe('QR Generation', () => {
     it('should generate QR', async () => {
@@ -37,6 +38,8 @@ describe('Test Connect', () => {
         assert.ok(conn.user?.phone)
         assert.ok (conn.user?.imgUrl || conn.user.imgUrl === '')
 
+        assertChatDBIntegrity (conn)
+
         conn.close()
         auth = conn.base64EncodedAuthInfo()
     })

src/WAConnection/4.Events.ts

@@ -1,7 +1,7 @@
 import * as QR from 'qrcode-terminal'
 import { WAConnection as Base } from './3.Connect'
 import { WAMessageStatusUpdate, WAMessage, WAContact, WAChat, WAMessageProto, WA_MESSAGE_STUB_TYPE, WA_MESSAGE_STATUS_TYPE, MessageLogLevel, PresenceUpdate, BaileysEvent, DisconnectReason, WANode } from './Constants'
-import { whatsappID, unixTimestampSeconds, isGroupID } from './Utils'
+import { whatsappID, unixTimestampSeconds, isGroupID, toNumber } from './Utils'
 
 export class WAConnection extends Base {
 
@@ -204,7 +204,6 @@ export class WAConnection extends Base {
         if (!message.key.fromMe && message.message) chat.count += 1
 
         const protocolMessage = message.message?.protocolMessage
-        
         // if it's a message to delete another message
         if (protocolMessage) {
             switch (protocolMessage.type) {
@@ -223,10 +222,19 @@ export class WAConnection extends Base {
                 default:
                     break
             }
-        } else if (!chat.messages.find(m => m.key.id === message.key.id)) {
-           // this.log ('adding new message from ' + chat.jid)
-            chat.messages.push(message)
-            chat.messages = chat.messages.slice (-this.maxCachedMessages) // only keep the last 5 messages
+        } else {
+            const messages = chat.messages
+            const messageTimestamp = toNumber (message.messageTimestamp)
+            const idx = messages.findIndex(m => toNumber(m.messageTimestamp) >= messageTimestamp)
+            // if the message is already there
+            if (messages[idx]?.key.id === message.key.id) return
+            //this.log (`adding message ID: ${messageTimestamp} to ${JSON.stringify(messages.map(m => toNumber(messageTimestamp)))}`, MessageLogLevel.info)
+
+            if (idx < 0) messages.push(message) // add to end
+            else if (toNumber(messages[idx].messageTimestamp) === toNumber(message.messageTimestamp)) messages.splice (idx+1, 0, message) // insert
+            else messages.splice (idx, 0, message) // insert
+            
+            messages.splice(0, messages.length-this.maxCachedMessages)
             
             // only update if it's an actual message
             if (message.message) this.chatUpdateTime (chat)

src/WAConnection/7.MessagesExtra.ts

@@ -122,7 +122,7 @@ export class WAConnection extends Base {
                 null,
             ]
             const response = await this.query({json, binaryTags: [WAMetric.queryMessages, WAFlag.ignore], expect200: false})
-            return response[2]?.map(item => item[2] as WAMessage) || []
+            return (response[2] as WANode[])?.map(item => item[2] as WAMessage) || []
         }
         const chat = this.chats.get (jid)
         

src/WAConnection/Utils.ts

@@ -28,6 +28,7 @@ function hashCode(s: string) {
         h = Math.imul(31, h) + s.charCodeAt(i) | 0;
     return h;
 }
+export const toNumber = (t: Long | number) => (t['low'] || t) as number
 export const waChatUniqueKey = (c: WAChat) => ((c.t*100000) + (hashCode(c.jid)%100000))*-1 // -1 to sort descending
 export const whatsappID = (jid: string) => jid?.replace ('@c.us', '@s.whatsapp.net')
 export const isGroupID = (jid: string) => jid?.includes ('@g.us')
@@ -147,9 +148,9 @@ export function generateMessageTag(epoch?: number) {
 export function generateClientID() {
     return randomBytes(16).toString('base64')
 }
-// generate a random 10 byte ID to attach to a message
+// generate a random 16 byte ID to attach to a message
 export function generateMessageID() {
-    return randomBytes(10).toString('hex').toUpperCase()
+    return randomBytes(16).toString('hex').toUpperCase()
 }
 export function decryptWA (message: string | Buffer, macKey: Buffer, encKey: Buffer, decoder: Decoder, fromMe: boolean=false): [string, Object, [number, number]?] {
     let commaIndex = message.indexOf(',') // all whatsapp messages have a tag and a comma, followed by the actual message
@@ -189,12 +190,13 @@ export function decryptWA (message: string | Buffer, macKey: Buffer, encKey: Buf
         const computedChecksum = hmacSign(data, macKey) // compute the sign of the message we recieved using our macKey
         
         if (!checksum.equals(computedChecksum)) {
-            throw new Error (`
+            console.error (`
                 Checksums don't match:
                 og: ${checksum.toString('hex')}
                 computed: ${computedChecksum.toString('hex')}
                 message: ${message.slice(0, 80).toString()}
             `)
+            return
         }
         // the checksum the server sent, must match the one we computed for the message to be valid
         const decrypted = aesDecrypt(data, encKey) // decrypt using AES