Updates

- removed timeout, use maxIdleTimeMs
- made messages a keyedDB to better utitlize message cache
- possible fix for group ID bug

Example/example.ts

@@ -16,15 +16,14 @@ async function example() {
     const conn = new WAConnection() // instantiate
     conn.autoReconnect = ReconnectMode.onConnectionLost // only automatically reconnect when the connection breaks
     conn.logLevel = MessageLogLevel.info // set to unhandled to see what kind of stuff you can implement
-
+    // if the gap between two messages is greater than 10s, fail the connection
+    conn.connectOptions.maxIdleTimeMs = 10*1000
+    conn.connectOptions.regenerateQRIntervalMs = 5000
+    // attempt to reconnect at most 10 times
+    conn.connectOptions.maxRetries = 10
 
     // loads the auth file credentials if present
     fs.existsSync('./auth_info.json') && conn.loadAuthInfo ('./auth_info.json')
-    
-    // connect or timeout in 60 seconds
-    conn.connectOptions.timeoutMs = 60*1000
-    // attempt to reconnect at most 10 times
-    conn.connectOptions.maxRetries = 10
     // uncomment the following line to proxy the connection; some random proxy I got off of: https://proxyscrape.com/free-proxy-list
     //conn.connectOptions.agent = ProxyAgent ('http://1.0.180.120:8080')
     await conn.connect()
@@ -32,7 +31,7 @@ async function example() {
     const unread = await conn.loadAllUnreadMessages ()
     
     console.log('oh hello ' + conn.user.name + ' (' + conn.user.jid + ')')
-    console.log('you have ' + conn.chats.all().length + ' chats & ' + Object.keys(conn.contacts).length + ' contacts')
+    console.log('you have ' + conn.chats.length + ' chats & ' + Object.keys(conn.contacts).length + ' contacts')
     console.log ('you have ' + unread.length + ' unread messages')
 
     const authInfo = conn.base64EncodedAuthInfo() // get all the auth info we need to restore this session

package.json

@@ -32,7 +32,7 @@
     "url": "git@github.com:adiwajshing/baileys.git"
   },
   "dependencies": {
-    "@adiwajshing/keyed-db": "^0.1.4",
+    "@adiwajshing/keyed-db": "^0.1.5",
     "curve25519-js": "^0.0.4",
     "futoin-hkdf": "^1.3.2",
     "https-proxy-agent": "^5.0.0",

src/Tests/Common.ts

@@ -1,4 +1,4 @@
-import { WAConnection, MessageLogLevel, MessageOptions, MessageType, unixTimestampSeconds, toNumber } from '../WAConnection/WAConnection'
+import { WAConnection, MessageLogLevel, MessageOptions, MessageType, unixTimestampSeconds, toNumber, GET_MESSAGE_ID, WA_MESSAGE_KEY } from '../WAConnection/WAConnection'
 import * as assert from 'assert'
 import {promises as fs} from 'fs'
 
@@ -13,8 +13,7 @@ export async function sendAndRetreiveMessage(conn: WAConnection, content, type:
 
     const chat = conn.chats.get(testJid)
 
-    assertChatDBIntegrity (conn)
-    assert.ok (chat.messages.find(m => m.key.id === response.key.id))
+    assert.ok (chat.messages.get(GET_MESSAGE_ID(message.key)))
     assert.ok (chat.t >= (unixTimestampSeconds()-5) )
     return message
 }
@@ -37,13 +36,13 @@ export const WAConnectionTest = (name: string, func: (conn: WAConnection) => voi
 export const assertChatDBIntegrity = (conn: WAConnection) => {
     conn.chats.all ().forEach (chat => (
         assert.deepEqual (
-            [...chat.messages].sort ((m1, m2) => toNumber(m1.messageTimestamp)-toNumber(m2.messageTimestamp)),
-            chat.messages
+            [...chat.messages.all()].sort ((m1, m2) => WA_MESSAGE_KEY(m1)-WA_MESSAGE_KEY(m2)),
+            chat.messages.all()
         )
     ))
     conn.chats.all ().forEach (chat => (
         assert.deepEqual (
-            chat.messages.filter (m => chat.messages.filter(m1 => m1.key.id === m.key.id).length > 1),
+            chat.messages.all().filter (m => chat.messages.all().filter(m1 => m1.key.id === m.key.id).length > 1),
             []
         )
     ))

src/Tests/Tests.Connect.ts

@@ -7,7 +7,7 @@ import { assertChatDBIntegrity } from './Common'
 describe('QR Generation', () => {
     it('should generate QR', async () => {
         const conn = new WAConnection()
-        conn.regenerateQRIntervalMs = 5000
+        conn.connectOptions.regenerateQRIntervalMs = 5000
 
         let calledQR = 0
         conn.removeAllListeners ('qr')
@@ -45,7 +45,6 @@ describe('Test Connect', () => {
     })
     it('should reconnect', async () => {
         const conn = new WAConnection()
-        conn.connectOptions.timeoutMs = 20*1000
         
         await conn.loadAuthInfo (auth).connect ()
         assert.ok(conn.user)
@@ -131,7 +130,6 @@ describe ('Reconnects', () => {
     it('should disrupt connect loop', async () => {
         const conn = new WAConnection()
         conn.autoReconnect = ReconnectMode.onAllErrors
-        conn.connectOptions.timeoutMs = 20000
         conn.loadAuthInfo ('./auth_info.json')
 
         let timeout = 1000
@@ -231,7 +229,6 @@ describe ('Reconnects', () => {
 describe ('Pending Requests', () => {
     it ('should correctly send updates', async () => {
         const conn = new WAConnection ()
-        conn.connectOptions.timeoutMs = 20*1000
         conn.pendingRequestTimeoutMs = null
 
         conn.loadAuthInfo('./auth_info.json')

src/Tests/Tests.Messages.ts

@@ -1,9 +1,12 @@
-import { MessageType, Mimetype, delay, promiseTimeout, WA_MESSAGE_STATUS_TYPE, WAMessageStatusUpdate } from '../WAConnection/WAConnection'
+import { MessageType, Mimetype, delay, promiseTimeout, WA_MESSAGE_STATUS_TYPE, WAMessageStatusUpdate, MessageOptions, toNumber } from '../WAConnection/WAConnection'
 import {promises as fs} from 'fs'
 import * as assert from 'assert'
-import { WAConnectionTest, testJid, sendAndRetreiveMessage } from './Common'
+import { WAConnectionTest, testJid, sendAndRetreiveMessage, assertChatDBIntegrity } from './Common'
 
 WAConnectionTest('Messages', conn => {
+
+    afterEach (() => assertChatDBIntegrity (conn))
+
     it('should send a text message', async () => {
         const message = await sendAndRetreiveMessage(conn, 'hello fren', MessageType.text)
         assert.strictEqual(message.message.conversation || message.message.extendedTextMessage?.text, 'hello fren')
@@ -160,13 +163,40 @@ WAConnectionTest('Messages', conn => {
             }
         })
     })
-    it('should not duplicate messages', async () => {
+    it('should maintain message integrity', async () => {
+        // loading twice does not alter the results
         const results = await Promise.all ([
             conn.loadMessages (testJid, 50),
             conn.loadMessages (testJid, 50)
         ])
-        assert.deepEqual (results[0].messages, results[1].messages)
+        assert.equal (results[0].messages.length, results[1].messages.length)
+        for (let i = 0; i < results[1].messages.length;i++) {
+            assert.deepEqual (results[0].messages[i], results[1].messages[i], `failed equal at ${i}`)
+        }
         assert.ok (results[0].messages.length <= 50)
+
+        // check if messages match server
+        let msgs = await conn.fetchMessagesFromWA (testJid, 50)
+        for (let i = 0; i < results[1].messages.length;i++) {
+            assert.deepEqual (results[0].messages[i].key, msgs[i].key, `failed equal at ${i}`)
+        }
+        // check with some arbitary cursors
+        let cursor = results[0].messages.slice(-1)[0].key
+        
+        msgs = await conn.fetchMessagesFromWA (testJid, 20, cursor)
+        let {messages} = await conn.loadMessages (testJid, 20, cursor)
+        for (let i = 0; i < messages.length;i++) {
+            assert.deepEqual (messages[i].key, msgs[i].key, `failed equal at ${i}`)
+        }
+
+        cursor = results[0].messages[2].key
+        
+        msgs = await conn.fetchMessagesFromWA (testJid, 20, cursor)
+        messages = (await conn.loadMessages (testJid, 20, cursor)).messages
+        for (let i = 0; i < messages.length;i++) {
+            assert.deepEqual (messages[i].key, msgs[i].key, `failed equal at ${i}`)
+        }
+
     })
     it('should deliver a message', async () => {
         const waitForUpdate = 

src/WAConnection/0.Base.ts

@@ -30,7 +30,7 @@ import { STATUS_CODES, Agent } from 'http'
 
 export class WAConnection extends EventEmitter {
     /** The version of WhatsApp Web we're telling the servers we are */
-    version: [number, number, number] = [2, 2037, 6]
+    version: [number, number, number] = [2, 2039, 6]
     /** The Browser we're telling the WhatsApp Web servers we are */
     browserDescription: [string, string, string] = Utils.Browsers.baileys ('Chrome')
     /** Metadata like WhatsApp id, name set on WhatsApp etc. */
@@ -41,11 +41,9 @@ export class WAConnection extends EventEmitter {
     pendingRequestTimeoutMs: number = null
     /** The connection state */
     state: WAConnectionState = 'close'
-    /** New QR generation interval, set to null if you don't want to regenerate */
-    regenerateQRIntervalMs = 30*1000
     connectOptions: WAConnectOptions = {
-        timeoutMs: 60*1000,
-        maxIdleTimeMs: 10*1000,
+        regenerateQRIntervalMs: 30_1000,
+        maxIdleTimeMs: 15_1000,
         waitOnlyForLastMessage: false,
         waitForChats: true,
         maxRetries: 5,
@@ -56,10 +54,12 @@ export class WAConnection extends EventEmitter {
     autoReconnect = ReconnectMode.onConnectionLost 
     /** Whether the phone is connected */
     phoneConnected: boolean = false
+    /** key to use to order chats */
+    chatOrderingKey = Utils.WA_CHAT_KEY
 
     maxCachedMessages = 50
 
-    chats: KeyedDB<WAChat> = new KeyedDB (Utils.waChatUniqueKey, value => value.jid)
+    chats: KeyedDB<WAChat> = new KeyedDB (Utils.WA_CHAT_KEY, value => value.jid)
     contacts: { [k: string]: WAContact } = {}
 
     /** Data structure of tokens & IDs used to establish one's identiy to WhatsApp Web */

src/WAConnection/1.Validation.ts

@@ -1,12 +1,12 @@
 import * as Curve from 'curve25519-js'
 import * as Utils from './Utils'
 import {WAConnection as Base} from './0.Base'
-import { MessageLogLevel, WAMetric, WAFlag, BaileysError, Presence, WAUser } from './Constants'
+import { MessageLogLevel, WAMetric, WAFlag, BaileysError, Presence, WAUser, DisconnectReason } from './Constants'
 
 export class WAConnection extends Base {
     
     /** Authenticate the connection */
-    protected async authenticate (onConnectionValidated: () => void, reconnect?: string) {
+    protected async authenticate (startDebouncedTimeout: () => void, stopDebouncedTimeout: () => void, reconnect?: string) {
         // if no auth info is present, that is, a new session has to be established
         // generate a client ID
         if (!this.authInfo?.clientID) {
@@ -15,6 +15,8 @@ export class WAConnection extends Base {
 
         const canLogin = this.authInfo?.encKey && this.authInfo?.macKey        
         this.referenceDate = new Date () // refresh reference date
+
+        startDebouncedTimeout ()
         
         const initQueries = [
             (async () => {
@@ -25,7 +27,9 @@ export class WAConnection extends Base {
                     longTag: true
                 })
                 if (!canLogin) {
+                    stopDebouncedTimeout () // stop the debounced timeout for QR gen
                     const result = await this.generateKeysForAuth (ref)
+                    startDebouncedTimeout () // restart debounced timeout
                     return result
                 }
             })()
@@ -58,7 +62,6 @@ export class WAConnection extends Base {
         const validationJSON = (await Promise.all (initQueries)).slice(-1)[0] // get the last result
         this.user = await this.validateNewConnection(validationJSON[1]) // validate the connection
         
-        onConnectionValidated ()
         this.log('validated connection successfully', MessageLogLevel.info)
 
         const response = await this.query({ json: ['query', 'ProfilePicThumb', this.user.jid], waitForOpen: false, expect200: false })
@@ -85,7 +88,13 @@ export class WAConnection extends Base {
      * @returns the new ref
      */
     async generateNewQRCodeRef() {
-        const response = await this.query({json: ['admin', 'Conn', 'reref'], expect200: true, waitForOpen: false, longTag: true})
+        const response = await this.query({
+            json: ['admin', 'Conn', 'reref'], 
+            expect200: true, 
+            waitForOpen: false, 
+            longTag: true,
+            timeoutMs: this.connectOptions.maxIdleTimeMs
+        })
         return response.ref as string
     }
     /**
@@ -177,12 +186,17 @@ export class WAConnection extends Base {
                 .then (newRef => ref = newRef)
                 .then (emitQR)
                 .then (regenQR)
-                .catch (err => this.log (`error in QR gen: ${err}`, MessageLogLevel.info))
-            }, this.regenerateQRIntervalMs)
+                .catch (err => {
+                    this.log (`error in QR gen: ${err}`, MessageLogLevel.info)
+                    if (err.status === 429) { // too many QR requests
+                        this.endConnection ()
+                    }
+                })
+            }, this.connectOptions.regenerateQRIntervalMs)
         }
 
         emitQR ()
-        if (this.regenerateQRIntervalMs) regenQR ()
+        if (this.connectOptions.regenerateQRIntervalMs) regenQR ()
 
         const json = await this.waitForMessage('s1', [])
         this.qrTimeout && clearTimeout (this.qrTimeout)

src/WAConnection/3.Connect.ts

@@ -64,22 +64,25 @@ export class WAConnection extends Base {
     protected async connectInternal (options: WAConnectOptions, delayMs?: number) {
         // actual connect
         const connect = () => {
-            const timeoutMs = options?.timeoutMs || 60*1000
-            
             let cancel: () => void
-            const task = Utils.promiseTimeout(timeoutMs, (resolve, reject) => {
+            const task = new Promise((resolve, reject) => {
                 cancel = () => reject (CancelledError())
-                const checkIdleTime = () => {
-                    this.debounceTimeout && clearTimeout (this.debounceTimeout)
-                    this.debounceTimeout = setTimeout (() => rejectSafe (TimedOutError()), this.connectOptions.maxIdleTimeMs)
-                }
-                const debouncedTimeout = () => this.connectOptions.maxIdleTimeMs && this.conn.addEventListener ('message', checkIdleTime)
                 // determine whether reconnect should be used or not
                 const shouldUseReconnect = this.lastDisconnectReason !== DisconnectReason.replaced && 
                                             this.lastDisconnectReason !== DisconnectReason.unknown &&
                                             this.lastDisconnectReason !== DisconnectReason.intentional && 
                                             this.user?.jid
                 
+                const checkIdleTime = () => {
+                    this.debounceTimeout && clearTimeout (this.debounceTimeout)
+                    this.debounceTimeout = setTimeout (() => rejectSafe (TimedOutError()), this.connectOptions.maxIdleTimeMs)
+                }
+                const startDebouncedTimeout = () => this.connectOptions.maxIdleTimeMs && this.conn.addEventListener ('message', checkIdleTime)
+                const stopDebouncedTimeout = () => {
+                    clearTimeout (this.debounceTimeout)
+                    this.conn.removeEventListener ('message', checkIdleTime)
+                }
+
                 const reconnectID = shouldUseReconnect ? this.user.jid.replace ('@s.whatsapp.net', '@c.us') : null
 
                 this.conn = new WS(WS_URL, null, { 
@@ -110,7 +113,7 @@ export class WAConnection extends Base {
                         }
                     }
                     try {
-                        await this.authenticate (debouncedTimeout, reconnectID)
+                        await this.authenticate (startDebouncedTimeout, stopDebouncedTimeout, reconnectID)
                         this.conn
                             .removeAllListeners ('error')
                             .removeAllListeners ('close')
@@ -124,6 +127,7 @@ export class WAConnection extends Base {
                 const rejectSafe = error => reject (error)
                 this.conn.on('error', rejectSafe)
                 this.conn.on('close', () => rejectSafe(new Error('close')))
+
             }) as Promise<void | { [k: string]: Partial<WAChat> }>
 
             return { promise: task, cancel: cancel }
@@ -163,7 +167,7 @@ export class WAConnection extends Base {
      * Must be called immediately after connect
      */
     protected receiveChatsAndContacts(waitOnlyForLast: boolean) {
-        const chats = new KeyedDB<WAChat>(Utils.waChatUniqueKey, c => c.jid)
+        const chats = new KeyedDB<WAChat>(this.chatOrderingKey, c => c.jid)
         const contacts = {}
 
         let receivedContacts = false
@@ -194,7 +198,14 @@ export class WAConnection extends Base {
                 messages.reverse().forEach (([,, message]: ['message', null, WAMessage]) => {
                     const jid = message.key.remoteJid
                     const chat = chats.get(jid)
-                    chat?.messages.unshift (message)
+                    if (chat) {
+                        const fm = chat.messages.all()[0]
+                        const prevEpoch = (fm && fm['epoch']) || 0
+                        
+                        message['epoch'] = prevEpoch-1
+                        chat.messages.insert (message)
+                    }
+                    
                 })
             }
             // if received contacts before messages
@@ -220,7 +231,7 @@ export class WAConnection extends Base {
                 chat.jid = Utils.whatsappID (chat.jid)
                 chat.t = +chat.t
                 chat.count = +chat.count
-                chat.messages = []
+                chat.messages = new KeyedDB (Utils.WA_MESSAGE_KEY, Utils.WA_MESSAGE_ID)
 
                 // chats data (log json to see what it looks like)
                 !chats.get (chat.jid) && chats.insert (chat) 

src/WAConnection/4.Events.ts

@@ -1,7 +1,8 @@
 import * as QR from 'qrcode-terminal'
 import { WAConnection as Base } from './3.Connect'
 import { WAMessageStatusUpdate, WAMessage, WAContact, WAChat, WAMessageProto, WA_MESSAGE_STUB_TYPE, WA_MESSAGE_STATUS_TYPE, MessageLogLevel, PresenceUpdate, BaileysEvent, DisconnectReason, WANode, WAOpenResult, Presence } from './Constants'
-import { whatsappID, unixTimestampSeconds, isGroupID, toNumber } from './Utils'
+import { whatsappID, unixTimestampSeconds, isGroupID, toNumber, GET_MESSAGE_ID, WA_MESSAGE_ID, WA_MESSAGE_KEY } from './Utils'
+import KeyedDB from '@adiwajshing/keyed-db'
 
 export class WAConnection extends Base {
 
@@ -35,16 +36,15 @@ export class WAConnection extends Base {
 
             this.emit('user-presence-update', update)
         })
-        // If a message has been updated (usually called when a video message gets its upload url)
+        // If a message has been updated (usually called when a video message gets its upload url, or live locations)
         this.registerCallback (['action', 'add:update', 'message'], json => {
             const message: WAMessage = json[2][0][2]
             const jid = whatsappID(message.key.remoteJid)
             const chat = this.chats.get(jid)
             if (!chat) return
+            // reinsert to update
+            if (chat.messages.delete (message)) chat.messages.insert (message)
 
-            const messageIndex = chat.messages.findIndex(m => m.key.id === message.key.id)
-            if (messageIndex >= 0) chat.messages[messageIndex] = message
-            
             this.emit ('message-update', message)
         })
         // If a user's contact has changed
@@ -183,7 +183,7 @@ export class WAConnection extends Base {
         const chat: WAChat = {
             jid: jid,
             t: unixTimestampSeconds(),
-            messages: [],
+            messages: new KeyedDB(WA_MESSAGE_KEY, WA_MESSAGE_ID),
             count: 0,
             modify_tag: '',
             spam: 'false',
@@ -217,14 +217,12 @@ export class WAConnection extends Base {
         if (protocolMessage) {
             switch (protocolMessage.type) {
                 case WAMessageProto.ProtocolMessage.PROTOCOL_MESSAGE_TYPE.REVOKE:
-                    const found = chat.messages.find(m => m.key.id === protocolMessage.key.id)
-                    if (found && found.message) {
-                        
+                    const found = chat.messages.get (GET_MESSAGE_ID(protocolMessage.key))
+                    if (found?.message) {
                         this.log ('deleting message: ' + protocolMessage.key.id + ' in chat: ' + protocolMessage.key.remoteJid, MessageLogLevel.info)
                         
                         found.messageStubType = WA_MESSAGE_STUB_TYPE.REVOKE
-                        found.message = null
-                        
+                        delete found.message
                         this.emit ('message-update', found)
                     }
                     break
@@ -233,20 +231,18 @@ export class WAConnection extends Base {
             }
         } else {
             const messages = chat.messages
-            const messageTimestamp = toNumber (message.messageTimestamp)
-            let idx = messages.length-1
-            for (idx; idx >= 0; idx--) {
-                if (toNumber(messages[idx].messageTimestamp) <= messageTimestamp) {
-                    break
-                }
-            }
             // if the message is already there
-            if (messages[idx]?.key.id === message.key.id) return
-            //this.log (`adding message ID: ${messageTimestamp} to ${JSON.stringify(messages.map(m => toNumber(messageTimestamp)))}`, MessageLogLevel.info)
-            
-            messages.splice (idx+1, 0, message) // insert
-            messages.splice(0, messages.length-this.maxCachedMessages)
+            if (messages.get(WA_MESSAGE_ID(message))) return
             
+            const last = messages.all().slice(-1)
+            const lastEpoch = ((last && last[0]) && last[0]['epoch']) || 0
+            message['epoch'] = lastEpoch+1
+
+            messages.insert (message)
+
+            while (messages.length > this.maxCachedMessages) {
+                messages.delete (messages.all()[0]) // delete oldest messages
+            }            
             // only update if it's an actual message
             if (message.message) this.chatUpdateTime (chat)
             
@@ -299,8 +295,9 @@ export class WAConnection extends Base {
         }
     }
     protected chatUpdatedMessage (messageIDs: string[], status: WA_MESSAGE_STATUS_TYPE, chat: WAChat) {
-        for (let msg of chat.messages) {
-            if (messageIDs.includes(msg.key.id) && msg.status < status) {
+        for (let id of messageIDs) {
+            let msg = chat.messages.get (GET_MESSAGE_ID({ id, fromMe: true })) || chat.messages.get (GET_MESSAGE_ID({ id, fromMe: false }))
+            if (msg && msg.status < status) {
                 if (status <= WA_MESSAGE_STATUS_TYPE.PENDING) msg.status = status
                 else if (isGroupID(chat.jid)) msg.status = status-1
                 else msg.status = status

src/WAConnection/5.User.ts

@@ -6,7 +6,7 @@ import {
     WAMetric,
     WAFlag,
 } from '../WAConnection/Constants'
-import { generateProfilePicture, waChatUniqueKey, whatsappID, unixTimestampSeconds } from './Utils'
+import { generateProfilePicture, WA_CHAT_KEY, whatsappID, unixTimestampSeconds } from './Utils'
 import { Mutex } from './Mutex'
 
 // All user related functions -- get profile picture, set status etc.
@@ -106,7 +106,7 @@ export class WAConnection extends Base {
                 chat.imgUrl === undefined && await this.setProfilePicture (chat)
             ))
         )
-        const cursor = (chats[chats.length-1] && chats.length >= count) ? waChatUniqueKey (chats[chats.length-1]) : null
+        const cursor = (chats[chats.length-1] && chats.length >= count) ? WA_CHAT_KEY (chats[chats.length-1]) : null
         return { chats, cursor }
     }
     /**

src/WAConnection/6.MessagesSend.ts

@@ -208,6 +208,8 @@ export class WAConnection extends Base {
         const json = ['action', {epoch: this.msgCount.toString(), type: 'relay'}, [['message', null, message]]]
         const flag = message.key.remoteJid === this.user.jid ? WAFlag.acknowledge : WAFlag.ignore // acknowledge when sending message to oneself
         await this.query({json, binaryTags: [WAMetric.message, flag], tag: message.key.id, expect200: true})
+        
+        message.status = WA_MESSAGE_STATUS_TYPE.SERVER_ACK
         await this.chatAddMessageAppropriate (message)
     }
     /**

src/WAConnection/7.MessagesExtra.ts

@@ -1,6 +1,6 @@
 import {WAConnection as Base} from './6.MessagesSend'
 import { MessageType, WAMessageKey, MessageInfo, WAMessageContent, WAMetric, WAFlag, WANode, WAMessage, WAMessageProto } from './Constants'
-import { whatsappID, delay, toNumber, unixTimestampSeconds } from './Utils'
+import { whatsappID, delay, toNumber, unixTimestampSeconds, GET_MESSAGE_ID, WA_MESSAGE_ID } from './Utils'
 import { Mutex } from './Mutex'
 
 export class WAConnection extends Base {
@@ -70,53 +70,71 @@ export class WAConnection extends Base {
         const read = await this.setQuery ([['read', attributes, null]])
         return read
     }
+    async fetchMessagesFromWA (jid: string, count: number, indexMessage?: { id?: string; fromMe?: boolean }, mostRecentFirst: boolean = true) {
+        const json = [
+            'query',
+            {
+                epoch: this.msgCount.toString(),
+                type: 'message',
+                jid: jid,
+                kind: mostRecentFirst ? 'before' : 'after',
+                count: count.toString(),
+                index: indexMessage?.id,
+                owner: indexMessage?.fromMe === false ? 'false' : 'true',
+            },
+            null,
+        ]
+        const response = await this.query({json, binaryTags: [WAMetric.queryMessages, WAFlag.ignore], expect200: false})
+        return (response[2] as WANode[])?.map(item => item[2] as WAMessage) || []
+    }
     /**
      * Load the conversation with a group or person
      * @param count the number of messages to load
-     * @param before the data for which message to offset the query by
+     * @param cursor the data for which message to offset the query by
      * @param mostRecentFirst retreive the most recent message first or retreive from the converation start
      */
     @Mutex ((jid, _, before, mostRecentFirst) => jid + (before?.id || '') + mostRecentFirst)
     async loadMessages (
         jid: string,
         count: number,
-        before?: { id?: string; fromMe?: boolean },
+        cursor?: { id?: string; fromMe?: boolean },
         mostRecentFirst: boolean = true
     ) {
         jid = whatsappID(jid)
 
-        const retreive = async (count: number, indexMessage: any) => {
-            const json = [
-                'query',
-                {
-                    epoch: this.msgCount.toString(),
-                    type: 'message',
-                    jid: jid,
-                    kind: mostRecentFirst ? 'before' : 'after',
-                    count: count.toString(),
-                    index: indexMessage?.id,
-                    owner: indexMessage?.fromMe === false ? 'false' : 'true',
-                },
-                null,
-            ]
-            const response = await this.query({json, binaryTags: [WAMetric.queryMessages, WAFlag.ignore], expect200: false})
-            return (response[2] as WANode[])?.map(item => item[2] as WAMessage) || []
-        }
+        const retreive = (count: number, indexMessage: any) => this.fetchMessagesFromWA (jid, count, indexMessage, mostRecentFirst)
+        
         const chat = this.chats.get (jid)
+        const hasCursor = cursor?.id && typeof cursor?.fromMe !== 'undefined'
+        const cursorValue = hasCursor && chat.messages.get (GET_MESSAGE_ID(cursor))
+        
         let messages: WAMessage[]
-        if (!before && chat && mostRecentFirst) {
-            messages = chat.messages
+        if (chat && mostRecentFirst && (!hasCursor || cursorValue)) {
+            messages = chat.messages.paginatedByValue (cursorValue, count, null, 'before')
+            
             const diff = count - messages.length
             if (diff < 0) {
-                messages = messages.slice(-count); // get the last X messages
+                messages = messages.slice(-count) // get the last X messages
             } else if (diff > 0) {
+                let fepoch = (messages[0] && messages[0]['epoch']) || 0
                 const extra = await retreive (diff, messages[0]?.key)
+                // add to DB
+                for (let i = extra.length-1;i >= 0; i--) {
+                    const m = extra[i]
+                    fepoch -= 1
+                    m['epoch'] = fepoch
+                    
+                    if(chat.messages.length < this.maxCachedMessages && !chat.messages.get (WA_MESSAGE_ID(m))) {
+                        chat.messages.insert(m)
+                    }
+                }
                 messages.unshift (...extra)
             }
-        } else messages = await retreive (count, before)
-
-        let cursor
+        } else messages = await retreive (count, cursor)
+        
         if (messages[0]) cursor = { id: messages[0].key.id, fromMe: messages[0].key.fromMe }
+        else cursor = null
+
         return {messages, cursor}
     }
     /**
@@ -160,7 +178,7 @@ export class WAConnection extends Base {
      */
     async findMessage (jid: string, chunkSize: number, onMessage: (m: WAMessage) => boolean) {
         const chat = this.chats.get (whatsappID(jid))
-        let count = chat?.messages?.length || chunkSize
+        let count = chat?.messages?.all().length || chunkSize
         let offsetID
         while (true) {
             const {messages, cursor} = await this.loadMessages(jid, count, offsetID, true)
@@ -196,13 +214,24 @@ export class WAConnection extends Base {
         return messages
     }
     /** Load a single message specified by the ID */
-    async loadMessage (jid: string, messageID: string) {
-        // load the message before the given message
-        let messages = (await this.loadMessages (jid, 1, {id: messageID, fromMe: true})).messages
-        if (!messages[0]) messages = (await this.loadMessages (jid, 1, {id: messageID, fromMe: false})).messages
-        // the message after the loaded message is the message required
-        const actual = await this.loadMessages (jid, 1, messages[0] && messages[0].key, false)
-        return actual.messages[0]
+    async loadMessage (jid: string, id: string) {
+        let message: WAMessage
+
+        jid = whatsappID (jid)
+        const chat = this.chats.get (jid)
+        if (chat) {
+            // see if message is present in cache
+            message = chat.messages.get (GET_MESSAGE_ID({ id, fromMe: true })) || chat.messages.get (GET_MESSAGE_ID({ id, fromMe: false }))
+        }
+        if (!message) {
+            // load the message before the given message
+            let messages = (await this.loadMessages (jid, 1, {id, fromMe: true})).messages
+            if (!messages[0]) messages = (await this.loadMessages (jid, 1, {id, fromMe: false})).messages
+            // the message after the loaded message is the message required
+            const actual = await this.loadMessages (jid, 1, messages[0] && messages[0].key, false)
+            message = actual.messages[0]
+        }
+        return message
     }
     /**
      * Search WhatsApp messages with a given text string
@@ -246,9 +275,9 @@ export class WAConnection extends Base {
 
         const chat = this.chats.get (whatsappID(messageKey.remoteJid))
         if (chat) {
-            chat.messages = chat.messages.filter (m => m.key.id !== messageKey.id)
+            const value = chat.messages.get (GET_MESSAGE_ID(messageKey))
+            value && chat.messages.delete (value)
         }
-
         return result
     }
     /**

src/WAConnection/8.Groups.ts

@@ -1,5 +1,5 @@
 import {WAConnection as Base} from './7.MessagesExtra'
-import { WAMetric, WAFlag, WANode, WAGroupMetadata, WAGroupCreateResponse, WAGroupModification } from '../WAConnection/Constants'
+import { WAMetric, WAFlag, WANode, WAGroupMetadata, WAGroupCreateResponse, WAGroupModification, MessageLogLevel } from '../WAConnection/Constants'
 import { GroupSettingChange } from './Constants'
 import { generateMessageID } from '../WAConnection/Utils'
 
@@ -48,6 +48,18 @@ export class WAConnection extends Base {
      */
     groupCreate = async (title: string, participants: string[]) => {
         const response = await this.groupQuery('create', null, title, participants) as WAGroupCreateResponse
+        const gid = response.gid
+        try {
+            await this.groupMetadata (gid)
+        } catch (error) {
+            this.log (`error in group creation: ${error}, switching gid & checking`, MessageLogLevel.info)
+            // if metadata is not available
+            const comps = gid.replace ('@g.us', '').split ('-')
+            response.gid = `${comps[0]}-${+comps[1] + 1}@g.us`
+
+            await this.groupMetadata (gid)
+            this.log (`group ID switched from ${gid} to ${response.gid}`, MessageLogLevel.info)
+        }
         await this.chatAdd (response.gid, title)
         return response
     }

src/WAConnection/Constants.ts

@@ -19,6 +19,7 @@ export type WAContextInfo = proto.IContextInfo
 export type WAGenericMediaMessage = proto.IVideoMessage | proto.IImageMessage | proto.IAudioMessage | proto.IDocumentMessage | proto.IStickerMessage
 export import WA_MESSAGE_STUB_TYPE = proto.WebMessageInfo.WEB_MESSAGE_INFO_STUBTYPE
 export import WA_MESSAGE_STATUS_TYPE = proto.WebMessageInfo.WEB_MESSAGE_INFO_STATUS
+import KeyedDB from '@adiwajshing/keyed-db'
 
 export interface WALocationMessage {
     degreesLatitude: number
@@ -65,8 +66,8 @@ export enum ReconnectMode {
     onAllErrors = 2
 }
 export type WAConnectOptions = {
-    /** timeout after which the connect attempt will fail, set to null for default timeout value */
-    timeoutMs?: number
+    /** New QR generation interval, set to null if you don't want to regenerate */
+    regenerateQRIntervalMs?: number
     /** fails the connection if no data is received for X seconds */
     maxIdleTimeMs?: number
     /** maximum attempts to connect */
@@ -202,7 +203,7 @@ export interface WAChat {
     name?: string
     
     // Baileys added properties
-    messages: WAMessage[]
+    messages: KeyedDB<WAMessage>
     imgUrl?: string
 }
 export enum WAMetric {

src/WAConnection/Utils.ts

@@ -9,7 +9,7 @@ import { URL } from 'url'
 import { Agent } from 'https'
 
 import Decoder from '../Binary/Decoder'
-import { MessageType, HKDFInfoKeys, MessageOptions, WAChat, WAMessageContent, BaileysError, WAMessageProto, TimedOutError, CancelledError, WAGenericMediaMessage } from './Constants'
+import { MessageType, HKDFInfoKeys, MessageOptions, WAChat, WAMessageContent, BaileysError, WAMessageProto, TimedOutError, CancelledError, WAGenericMediaMessage, WAMessage, WAMessageKey } from './Constants'
 
 const platformMap = {
     'aix': 'AIX',
@@ -30,9 +30,16 @@ function hashCode(s: string) {
     return h;
 }
 export const toNumber = (t: Long | number) => (t['low'] || t) as number
-export const waChatUniqueKey = (c: WAChat) => ((c.t*100000) + (hashCode(c.jid)%100000))*-1 // -1 to sort descending
+
+export const WA_CHAT_KEY = (c: WAChat) => ((c.t*100000) + (hashCode(c.jid)%100000))*-1 // -1 to sort descending
+export const WA_CHAT_KEY_PIN = (c: WAChat) => ((c.pin ? 1 : 0)*1000000 + (c.t*100000) + (hashCode(c.jid)%100000))*-1 // -1 to sort descending
+
+export const WA_MESSAGE_KEY = (m: WAMessage) => toNumber (m.messageTimestamp)*1000 + (m['epoch'] || 0)%1000
+export const WA_MESSAGE_ID = (m: WAMessage) => GET_MESSAGE_ID (m.key)
+export const GET_MESSAGE_ID = (key: WAMessageKey) => `${key.id}|${key.fromMe ? 1 : 0}`
+
 export const whatsappID = (jid: string) => jid?.replace ('@c.us', '@s.whatsapp.net')
-export const isGroupID = (jid: string) => jid?.includes ('@g.us')
+export const isGroupID = (jid: string) => jid?.endsWith ('@g.us')
 
 export function shallowChanges <T> (old: T, current: T): Partial<T> {
     let changes: Partial<T> = {}

yarn.lock

@@ -2,10 +2,10 @@
 # yarn lockfile v1
 
 
-"@adiwajshing/keyed-db@^0.1.4":
-  version "0.1.4"
-  resolved "https://registry.yarnpkg.com/@adiwajshing/keyed-db/-/keyed-db-0.1.4.tgz#ed82563d8738cf2877a3c7c53a8739aa52f5efbf"
-  integrity sha512-Sed2xppK0b1Ayfwy+ONl4GKTUahzIRmaxHULzraPbIiRx9QHOQ3PJon9ZT1qtkebGdSRX9uHvwruz2VU03/sCQ==
+"@adiwajshing/keyed-db@^0.1.5":
+  version "0.1.5"
+  resolved "https://registry.yarnpkg.com/@adiwajshing/keyed-db/-/keyed-db-0.1.5.tgz#f31c28d7e532bf0b9a1446c6b746fe836409ac64"
+  integrity sha512-YrCehVjVZSxl53iX0a0fvFvbZiF0y59DScCfAwjeQ3hAV8hAwNtQzD8bmcx2qieWSsLnxBWYajJEvUxD3M1hLg==
 
 "@babel/runtime@^7.7.2":
   version "7.11.2"