Cleanup + add remaining utils

src/Connection/auth.ts

@@ -3,8 +3,7 @@ import EventEmitter from "events"
 import * as Curve from 'curve25519-js'
 import { BaileysEventEmitter, BaileysEventMap, SocketConfig, CurveKeyPair, WAInitResponse, ConnectionState, DisconnectReason } from "../Types"
 import { makeSocket } from "./socket"
-import { generateClientID, promiseTimeout } from "../Utils/generics"
-import { normalizedAuthInfo, computeChallengeResponse, validateNewConnection } from "../Utils/validate-connection"
+import { generateClientID, promiseTimeout, normalizedAuthInfo, computeChallengeResponse, validateNewConnection } from "../Utils"
 import { randomBytes } from "crypto"
 import { AuthenticationCredentials } from "../Types"
 
@@ -106,7 +105,7 @@ const makeAuthSocket = (config: SocketConfig) => {
                 }
             )
             .finally(() => (
-				ev.off('state.update', listener)
+				ev.off('connection.update', listener)
 			))
         )
     }

src/Connection/index.ts

@@ -1,12 +1,15 @@
 import { SocketConfig } from '../Types'
 import { DEFAULT_CONNECTION_CONFIG } from '../Defaults'
 import { EventEmitter } from 'events'
-import * as Connection from './groups'
+import _makeConnection from './groups'
 // export the last socket layer
 const makeConnection = (config: Partial<SocketConfig>) => (
-	Connection.default({
+	_makeConnection({
 		...DEFAULT_CONNECTION_CONFIG,
 		...config
 	})
 )
+
+export type Connection = ReturnType<typeof makeConnection>
+
 export default makeConnection

src/Connection/messages.ts

@@ -1,12 +1,10 @@
 import BinaryNode from "../BinaryNode";
 import { Boom } from '@hapi/boom'
 import { EventEmitter } from 'events'
-import { Chat, Presence, SocketConfig, WAMessage, WAMessageKey, ParticipantAction, WAMessageProto, WAMessageStatus, WAMessageStubType, GroupMetadata, AnyMessageContent, MiscMessageGenerationOptions, WAFlag, WAMetric, WAUrlInfo, MediaConnInfo, MessageUpdateType, MessageInfo } from "../Types";
-import { isGroupID, toNumber, whatsappID } from "../Utils/generics";
+import { Chat, Presence, WAMessageCursor, SocketConfig, WAMessage, WAMessageKey, ParticipantAction, WAMessageProto, WAMessageStatus, WAMessageStubType, GroupMetadata, AnyMessageContent, MiscMessageGenerationOptions, WAFlag, WAMetric, WAUrlInfo, MediaConnInfo, MessageUpdateType, MessageInfo } from "../Types";
+import { isGroupID, toNumber, whatsappID, generateWAMessage, decryptMediaMessageBuffer } from "../Utils";
 import makeChatsSocket from "./chats";
 import { WA_DEFAULT_EPHEMERAL } from "../Defaults";
-import { generateWAMessage } from "../Utils/messages";
-import { decryptMediaMessageBuffer } from "../Utils/messages-media";
 
 const STATUS_MAP = {
 	read: WAMessageStatus.READ,
@@ -47,9 +45,12 @@ const makeMessagesSocket = (config: SocketConfig) => {
 	const fetchMessagesFromWA = async(
 		jid: string, 
 		count: number, 
-		indexMessage?: { id?: string; fromMe?: boolean }, 
-		mostRecentFirst: boolean = true
+		cursor?: WAMessageCursor
 	) => {
+		let key: WAMessageKey
+		if(cursor) {
+			key = 'before' in cursor ? cursor.before : cursor.after
+		}
         const { data }:BinaryNode = await query({
 			json: new BinaryNode(
 				'query',
@@ -57,10 +58,10 @@ const makeMessagesSocket = (config: SocketConfig) => {
 					epoch: currentEpoch().toString(),
 					type: 'message',
 					jid: jid,
-					kind: mostRecentFirst ? 'before' : 'after',
+					kind: !cursor || 'before' in cursor ? 'before' : 'after',
 					count: count.toString(),
-					index: indexMessage?.id,
-					owner: indexMessage?.fromMe === false ? 'false' : 'true',
+					index: key?.id,
+					owner: key?.fromMe === false ? 'false' : 'true',
 				}
 			), 
 			binaryTag: [WAMetric.queryMessages, WAFlag.ignore], 
@@ -91,7 +92,7 @@ const makeMessagesSocket = (config: SocketConfig) => {
 		})
 		Object.keys(response[1]).forEach (key => content[key] = response[1][key]) // update message
 
-		ev.emit('messages.upsert', { messages: [message], type: 'append' })
+		ev.emit('messages.update', [{ key: message.key, message: message.message }])
 
 		return response
 	}
@@ -365,6 +366,18 @@ const makeMessagesSocket = (config: SocketConfig) => {
 		},
 		updateMediaMessage,
 		fetchMessagesFromWA,
+		/** Load a single message specified by the ID */
+		loadMessageFromWA: async(jid: string, id: string) => {
+			let message: WAMessage
+	
+			// load the message before the given message
+			let messages = (await fetchMessagesFromWA(jid, 1, { before: {id, fromMe: true} }))
+			if(!messages[0]) messages = (await fetchMessagesFromWA(jid, 1, { before: {id, fromMe: false} }))
+			// the message after the loaded message is the message required
+			const [actual] = await fetchMessagesFromWA(jid, 1, { after: messages[0] && messages[0].key })
+			message = actual
+			return message
+		},
 		searchMessages: async(txt: string, inJid: string | null, count: number, page: number) => {
 			const {data, attributes}: BinaryNode = await query({
 				json: new BinaryNode(
@@ -419,10 +432,6 @@ const makeMessagesSocket = (config: SocketConfig) => {
 					{
 						...options,
 						userJid: userJid,
-						/*ephemeralOptions: chat?.ephemeral ? {
-							expiration: chat.ephemeral,
-							eph_setting_ts: chat.eph_setting_ts
-						} : undefined,*/
 						getUrlInfo: generateUrlInfo,
 						getMediaOptions: refreshMediaConn
 					}

src/Connection/socket.ts

@@ -5,8 +5,7 @@ import { promisify } from "util"
 import WebSocket from "ws"
 import BinaryNode from "../BinaryNode"
 import { DisconnectReason, SocketConfig, SocketQueryOptions, SocketSendMessageOptions } from "../Types"
-import { aesEncrypt, hmacSign, promiseTimeout, unixTimestampSeconds } from "../Utils/generics"
-import { decodeWAMessage } from "../Utils/decode-wa-message"
+import { aesEncrypt, hmacSign, promiseTimeout, unixTimestampSeconds, decodeWAMessage } from "../Utils"
 import { WAFlag, WAMetric, WATag } from "../Types"
 import { DEFAULT_ORIGIN, DEF_CALLBACK_PREFIX, DEF_TAG_PREFIX, PHONE_CONNECTION_CB } from "../Defaults"
 

src/Defaults/index.ts

@@ -4,6 +4,8 @@ import { Browsers } from "../Utils/generics"
 
 export const UNAUTHORIZED_CODES = [401, 403, 419]
 
+export const STORIES_JID = 'status@broadcast'
+
 export const DEFAULT_ORIGIN = 'https://web.whatsapp.com'
 export const DEF_CALLBACK_PREFIX = 'CB:'
 export const DEF_TAG_PREFIX = 'TAG:'

src/Store/in-memory-store.ts

@@ -1,39 +1,44 @@
 import KeyedDB from "@adiwajshing/keyed-db"
+import { Comparable } from "@adiwajshing/keyed-db/lib/Types"
 import { Logger } from "pino"
-import makeConnection from "../Connection"
-import { BaileysEventEmitter, Chat, ConnectionState, Contact, WAMessage, WAMessageKey } from "../Types"
+import type { Connection } from "../Connection"
+import type { BaileysEventEmitter, Chat, ConnectionState, Contact, WAMessage, WAMessageCursor } from "../Types"
+import { toNumber } from "../Utils"
+import makeOrderedDictionary from "./ordered-dictionary"
 
 export const waChatKey = (pin: boolean) => ({
     key: (c: Chat) => (pin ? (c.pin ? '1' : '0') : '') + (c.archive === 'true' ? '0' : '1') + c.t.toString(16).padStart(8, '0') + c.jid,
     compare: (k1: string, k2: string) => k2.localeCompare (k1)
 })
 
+export const waMessageID = (m: WAMessage) => m.key.id
+
 export type BaileysInMemoryStoreConfig = {
+	chatKey: Comparable<Chat, string>
 	logger: Logger
 }
 
+const makeMessagesDictionary = () => makeOrderedDictionary(waMessageID)
+
 export default(
-	{ logger }: BaileysInMemoryStoreConfig
+	{ logger, chatKey }: BaileysInMemoryStoreConfig
 ) => {
-	const chats = new KeyedDB<Chat, string>(waChatKey(true), c => c.jid)
-	const messages: { [_: string]: WAMessage[] } = {}
+	
+	const chats = new KeyedDB<Chat, string>(chatKey, c => c.jid)
+	const messages: { [_: string]: ReturnType<typeof makeMessagesDictionary> } = {}
 	const contacts: { [_: string]: Contact } = {}
 	const state: ConnectionState = {
 		connection: 'close',
 		phoneConnected: false
 	}
 
-	const messageIndex = (key: WAMessageKey) => {
-		const messageList = messages[key.remoteJid!]
-		if(messageList) {
-			const idx = messageList.findIndex(m => m.key.id === key.id)
-			if(idx >= 0) {
-				return { messageList, idx }
-			}
-		}
+	const assertMessageList = (jid: string) => {
+		if(!messages[jid]) messages[jid] = makeMessagesDictionary()
+		return messages[jid]
 	}
 
 	const listen = (ev: BaileysEventEmitter) => {
+		
 		ev.on('connection.update', update => {
 			Object.assign(state, update)
 		})
@@ -91,55 +96,54 @@ export default(
 				case 'notify':
 					for(const msg of newMessages) {
 						const jid = msg.key.remoteJid!
-						const result = messageIndex(newMessages[0].key)
-						if(!result) {
-							if(!messages[jid]) {
-								messages[jid] = []
-							}
-							messages[jid].push(msg)
-						} else {
-							result.messageList[result.idx] = msg
+						const list = assertMessageList(jid)
+						list.upsert(msg, 'append')
+
+						if(type === 'notify' && !chats.get(jid)) {
+							ev.emit('chats.upsert', { 
+								chats: [ { jid, t: toNumber(msg.messageTimestamp), count: 1 } ],
+								type: 'upsert'
+							})
 						}
 					}
 				break
 				case 'last':
 					for(const msg of newMessages) {
 						const jid = msg.key.remoteJid!
-						if(!messages[jid]) {
-							messages[jid] = []
-						}
-						const [lastItem] = messages[jid].slice(-1)
+						const list = assertMessageList(jid)
+						const [lastItem] = list.array.slice(-1)
 						// reset message list
 						if(lastItem && lastItem.key.id !== msg.key.id) {
-							messages[jid] = [msg]
+							list.clear()
+							list.upsert(msg, 'append')
 						}
 					}
 				break
 				case 'prepend':
-
+					for(const msg of newMessages) {
+						const jid = msg.key.remoteJid!
+						const list = assertMessageList(jid)
+						list.upsert(msg, 'prepend')
+					}
 				break
 			}
 		})
 		ev.on('messages.update', updates => {
 			for(const update of updates) {
-				const result = messageIndex(update.key!)
-				if(result) {
-					Object.assign(result.messageList[result.idx], update)
-				} else {
+				const list = assertMessageList(update.key.remoteJid)
+				const result = list.updateAssign(update)
+				if(!result) {
 					logger.debug({ update }, `got update for non-existant message`)
 				}
 			}
 		})
 		ev.on('messages.delete', item => {
+			const list = assertMessageList(item.jid)
 			if('all' in item) {
-				messages[item.jid] = []
+				list.clear()
 			} else {
 				const idSet = new Set(item.ids)
-				if(messages[item.jid]) {
-					messages[item.jid] = messages[item.jid].filter(
-						m => !idSet.has(m.key.id)
-					)
-				}
+				list.filter(m => !idSet.has(m.key.id))
 			}
 		})
 	}
@@ -148,14 +152,60 @@ export default(
 		chats,
 		contacts,
 		messages,
+		state,
 		listen,
-		fetchImageUrl: async(jid: string, sock: ReturnType<typeof makeConnection>) => {
+		loadMessages: async(jid: string, count: number, cursor: WAMessageCursor, sock: Connection | undefined) => {
+			const list = assertMessageList(jid)
+			const retrieve = async(count: number, cursor: WAMessageCursor) => {
+				const result = await sock?.fetchMessagesFromWA(jid, count, cursor)
+				return result || []
+			}
+			const mode = !cursor || 'before' in cursor ? 'before' : 'after'
+			const cursorKey = !!cursor ? ('before' in cursor ? cursor.before : cursor.after) : undefined
+			const cursorValue = cursorKey ? list.get(cursorKey.id) : undefined
+			
+			let messages: WAMessage[]
+			if(messages && mode ==='before' && (!cursorKey || cursorValue)) {
+				const msgIdx = messages.findIndex(m => m.key.id === cursorKey.id)
+				messages = list.array.slice(0, msgIdx)
+				
+				const diff = count - messages.length
+				if (diff < 0) {
+					messages = messages.slice(-count) // get the last X messages
+				} else if (diff > 0) {
+					const [fMessage] = messages
+					const extra = await retrieve (diff, { before: fMessage?.key || cursorKey })
+					// add to DB
+					for(let i = extra.length-1; i >= 0;i--) {
+						list.upsert(extra[i], 'prepend')
+					}
+				}
+			} else messages = await retrieve(count, cursor)
+
+			return messages
+		},
+		loadMessage: async(jid: string, id: string, sock: Connection | undefined) => {
+			let message = messages[jid]?.get(id)
+			if(!message) {
+				message = await sock?.loadMessageFromWA(jid, id)
+			}
+			return message
+		},
+		mostRecentMessage: async(jid: string, sock: Connection | undefined) => {
+			let message = messages[jid]?.array.slice(-1)[0]
+			if(!message) {
+				const [result] = await sock?.fetchMessagesFromWA(jid, 1, undefined)
+				message = result
+			}
+			return message
+		},
+		fetchImageUrl: async(jid: string, sock: Connection | undefined) => {
 			const contact = contacts[jid]
 			if(!contact) {
-				return sock.fetchImageUrl(jid)
+				return sock?.fetchImageUrl(jid)
 			}
 			if(!contact.imgUrl) {
-				contact.imgUrl = await sock.fetchImageUrl(jid)
+				contact.imgUrl = await sock?.fetchImageUrl(jid)
 			}
 			return contact.imgUrl
 		}

src/Store/index.ts

@@ -1,2 +1,2 @@
-import inMemoryStore from "./in-memory-store";
-export default inMemoryStore
+import makeInMemoryStore from './in-memory-store'
+export { makeInMemoryStore }

src/Store/ordered-dictionary.ts

@@ -0,0 +1,66 @@
+
+const makeOrderedDictionary = function<T>(idGetter: (item: T) => string) {
+	const array: T[] = []
+	const dict: { [_: string]: T } = { }
+
+	const get = (id: string) => dict[id]
+
+	const update = (item: T) => {
+		const id = idGetter(item)
+		const idx = array.findIndex(i => idGetter(i) === id)
+		if(idx >= 0) {
+			array[idx] = item
+			dict[id] = item
+		}
+		return false
+	}
+
+	const upsert = (item: T, mode: 'append' | 'prepend') => {
+		const id = idGetter(item)
+		if(get(id)) {
+			update(item)
+		} else {
+			if(mode === 'append') {
+				array.push(item)
+			} else {
+				array.splice(0, 0, item)
+			}
+			dict[id] = item
+		}
+	}
+	
+	const remove = (item: T) => {
+		const id = idGetter(item)
+		const idx = array.findIndex(i => idGetter(i) === id)
+		if(idx >= 0) {
+			array.splice(idx, 1)
+			delete dict[id]
+			return true
+		}
+		return false
+	}
+
+	return {
+		array,
+		get,
+		upsert,
+		update,
+		remove,
+		updateAssign: (update: Partial<T>) => {
+			const item = get(idGetter(update as any))
+			if(item) {
+				Object.assign(item, update)
+			}
+			return false
+		},
+		clear: () => {
+			array.splice(0, array.length)
+			Object.keys(dict).forEach(key => { delete dict[key] })
+		},
+		filter: (contain: (item: T) => boolean) => {
+			//const copy = 
+		}
+	}
+}
+export default makeOrderedDictionary
+//export type OrderedDictionary<T> = ReturnType<typeof makeOrderedDictionary>

src/Types/Chat.ts

@@ -54,4 +54,5 @@ export type ChatModification =
             messages: { id: string, fromMe?: boolean }[],
             star: boolean
         }
-    }
+    } | 
+    { delete: true }

src/Types/Message.ts

@@ -78,7 +78,6 @@ export type AnyMediaMessageContent = (
     { mimetype?: string }
 
 export type AnyRegularMessageContent = 
-    string | 
     ({
 	    text: string
     } 
@@ -109,14 +108,15 @@ export type MiscMessageGenerationOptions = {
 	timestamp?: Date
     /** the message you want to quote */
 	quoted?: WAMessage
-}
-export type MessageGenerationOptionsFromContent = MiscMessageGenerationOptions & {
-	userJid: string
+    /** disappearing messages settings */
     ephemeralOptions?: { 
         expiration: number | string
         eph_setting_ts: number | string
     }
 }
+export type MessageGenerationOptionsFromContent = MiscMessageGenerationOptions & {
+	userJid: string
+}
 export type MediaGenerationOptions = {
 	logger?: Logger
 	agent?: Agent
@@ -132,4 +132,20 @@ export type MessageUpdateType = 'prepend' | 'append' | 'notify' | 'last'
 export interface MessageInfo {
     reads: {jid: string, t: string}[]
     deliveries: {jid: string, t: string}[]
-}
+}
+
+
+export interface MessageStatusUpdate {
+    from: string
+    to: string
+    /** Which participant caused the update (only for groups) */
+    participant?: string
+    timestamp: Date
+    /** Message IDs read/delivered */
+    ids: string[]
+    /** Status of the Message IDs */
+    type: WAMessageStatus
+}
+
+
+export type WAMessageCursor = { before: WAMessageKey | undefined } | { after: WAMessageKey | undefined }

src/Types/index.ts

@@ -195,5 +195,7 @@ export type BaileysEventMap = {
 }
 export interface BaileysEventEmitter extends EventEmitter {
 	on<T extends keyof BaileysEventMap>(event: T, listener: (arg: BaileysEventMap[T]) => void): this
+    off<T extends keyof BaileysEventMap>(event: T, listener: (arg: BaileysEventMap[T]) => void): this
+    removeAllListeners<T extends keyof BaileysEventMap>(event: T): this
 	emit<T extends keyof BaileysEventMap>(event: T, arg: BaileysEventMap[T]): boolean
 }

src/Utils/index.ts

@@ -0,0 +1,5 @@
+export * from './decode-wa-message'
+export * from './generics'
+export * from './messages'
+export * from './messages-media'
+export * from './validate-connection'

src/Utils/messages.ts

@@ -220,9 +220,6 @@ export const generateWAMessageContent = async(
 	options: MessageContentGenerationOptions
 ) => {
 	let m: WAMessageContent = {}
-	if(typeof message === 'string') {
-		message = { text: message }
-	}
 	if('text' in message) {
 		const extContent = { ...message } as WATextMessage
 		if (!!options.getUrlInfo && message.text.match(URL_REGEX)) {

src/Utils/validate-connection.ts

@@ -1,7 +1,7 @@
 import {Boom} from '@hapi/boom'
 import * as Curve from 'curve25519-js'
 import type { Contact } from '../Types/Contact'
-import type { AnyAuthenticationCredentials, AuthenticationCredentials, CurveKeyPair } from "../Types"
+import type { AnyAuthenticationCredentials, AuthenticationCredentials, AuthenticationCredentialsBase64, CurveKeyPair } from "../Types"
 import { aesDecrypt, hkdf, hmacSign, whatsappID } from './generics'
 import { readFileSync } from 'fs'
 
@@ -32,6 +32,16 @@ export const normalizedAuthInfo = (authInfo: AnyAuthenticationCredentials | stri
     }   
     return authInfo as AuthenticationCredentials
 }
+
+export const base64EncodedAuthenticationCredentials = (creds: AnyAuthenticationCredentials) => {
+    const normalized = normalizedAuthInfo(creds)
+    return {
+        ...normalized,
+        encKey: normalized.encKey.toString('base64'),
+        macKey: normalized.macKey.toString('base64')
+    } as AuthenticationCredentialsBase64
+}
+
 /**
 * Once the QR code is scanned and we can validate our connection, or we resolved the challenge when logging back in
 * @private

src/index.ts

@@ -1,8 +1,9 @@
 import makeConnection from './Connection'
 
 export * from '../WAMessage/WAMessage'
-export * from './Utils/messages'
+export * from './Utils'
 export * from './Types'
 export * from './Store'
+export * from './Defaults'
 
 export default makeConnection