refactor!: cleaner message history sync

This is a breaking change,
1. three events (chats.set, contacts.set, messages.set) are now just one `messaging-history.set` event
2. no need to debounce for app state sync
3. added a new "conditional" chat update to allow for correct app state sync despite not having the chat available on hand

src/Utils/event-buffer.ts

@@ -1,11 +1,13 @@
 import EventEmitter from 'events'
 import { Logger } from 'pino'
 import { proto } from '../../WAProto'
-import { BaileysEvent, BaileysEventEmitter, BaileysEventMap, BufferedEventData, Chat, Contact, WAMessage, WAMessageStatus } from '../Types'
+import { BaileysEvent, BaileysEventEmitter, BaileysEventMap, BufferedEventData, Chat, ChatUpdate, Contact, WAMessage, WAMessageStatus } from '../Types'
+import { trimUndefineds } from './generics'
 import { updateMessageWithReaction, updateMessageWithReceipt } from './messages'
 import { isRealMessage, shouldIncrementChatUnread } from './process-message'
 
 const BUFFERABLE_EVENT = [
+	'messaging-history.set',
 	'chats.upsert',
 	'chats.update',
 	'chats.delete',
@@ -55,10 +57,10 @@ type BaileysBufferableEventEmitter = BaileysEventEmitter & {
  */
 export const makeEventBuffer = (logger: Logger): BaileysBufferableEventEmitter => {
 	const ev = new EventEmitter()
+	const historyCache = new Set<string>()
 
 	let data = makeBufferData()
 	let isBuffering = false
-
 	let preBufferTask: Promise<any> = Promise.resolve()
 
 	// take the generic event and fire it as a baileys event
@@ -88,14 +90,29 @@ export const makeEventBuffer = (logger: Logger): BaileysBufferableEventEmitter =
 
 		isBuffering = false
 
+		const newData = makeBufferData()
+		const chatUpdates = Object.values(data.chatUpdates)
+		// gather the remaining conditional events so we re-queue them
+		let conditionalChatUpdatesLeft = 0
+		for(const update of chatUpdates) {
+			if(update.conditional) {
+				conditionalChatUpdatesLeft += 1
+				newData.chatUpdates[update.id!] = update
+				delete data.chatUpdates[update.id!]
+			}
+		}
+
 		const consolidatedData = consolidateEvents(data)
 		if(Object.keys(consolidatedData).length) {
 			ev.emit('event', consolidatedData)
 		}
 
-		data = makeBufferData()
+		data = newData
 
-		logger.trace('released buffered events')
+		logger.trace(
+			{ conditionalChatUpdatesLeft },
+			'released buffered events'
+		)
 	}
 
 	return {
@@ -111,7 +128,7 @@ export const makeEventBuffer = (logger: Logger): BaileysBufferableEventEmitter =
 		},
 		emit<T extends BaileysEvent>(event: BaileysEvent, evData: BaileysEventMap[T]) {
 			if(isBuffering && BUFFERABLE_EVENT_SET.has(event)) {
-				append(data, event as any, evData, logger)
+				append(data, historyCache, event as any, evData, logger)
 				return true
 			}
 
@@ -145,6 +162,13 @@ export const makeEventBuffer = (logger: Logger): BaileysBufferableEventEmitter =
 
 const makeBufferData = (): BufferedEventData => {
 	return {
+		historySets: {
+			chats: { },
+			messages: { },
+			contacts: { },
+			isLatest: false,
+			empty: true
+		},
 		chatUpserts: { },
 		chatUpdates: { },
 		chatDeletes: new Set(),
@@ -161,41 +185,100 @@ const makeBufferData = (): BufferedEventData => {
 
 function append<E extends BufferableEvent>(
 	data: BufferedEventData,
+	historyCache: Set<string>,
 	event: E,
 	eventData: any,
 	logger: Logger
 ) {
 	switch (event) {
+	case 'messaging-history.set':
+		for(const chat of eventData.chats as Chat[]) {
+			const existingChat = data.historySets.chats[chat.id]
+			if(existingChat) {
+				existingChat.endOfHistoryTransferType = chat.endOfHistoryTransferType
+			}
+
+			if(!existingChat && !historyCache.has(chat.id)) {
+				data.historySets.chats[chat.id] = chat
+				historyCache.add(chat.id)
+
+				absorbingChatUpdate(chat)
+			}
+		}
+
+		for(const contact of eventData.contacts as Contact[]) {
+			const existingContact = data.historySets.contacts[contact.id]
+			if(existingContact) {
+				Object.assign(existingContact, trimUndefineds(contact))
+			} else {
+				const historyContactId = `c:${contact.id}`
+				const hasAnyName = contact.notify || contact.name || contact.verifiedName
+				if(!historyCache.has(historyContactId) || hasAnyName) {
+					data.historySets.contacts[contact.id] = contact
+					historyCache.add(historyContactId)
+				}
+			}
+		}
+
+		for(const message of eventData.messages as WAMessage[]) {
+			const key = stringifyMessageKey(message.key)
+			const existingMsg = data.historySets.messages[key]
+			if(!existingMsg && !historyCache.has(key)) {
+				data.historySets.messages[key] = message
+				historyCache.add(key)
+			}
+		}
+
+		data.historySets.empty = false
+		data.historySets.isLatest = eventData.isLatest || data.historySets.isLatest
+
+		break
 	case 'chats.upsert':
 		for(const chat of eventData as Chat[]) {
-			let upsert = data.chatUpserts[chat.id] || { } as Chat
-			upsert = concatChats(upsert, chat)
-			if(data.chatUpdates[chat.id]) {
-				logger.debug({ chatId: chat.id }, 'absorbed chat update in chat upsert')
-				upsert = concatChats(data.chatUpdates[chat.id] as Chat, upsert)
-				delete data.chatUpdates[chat.id]
+			let upsert = data.chatUpserts[chat.id]
+			if(!upsert) {
+				upsert = data.historySets[chat.id]
+				if(upsert) {
+					logger.debug({ chatId: chat.id }, 'absorbed chat upsert in chat set')
+				}
 			}
 
+			if(upsert) {
+				upsert = concatChats(upsert, chat)
+			} else {
+				upsert = chat
+				data.chatUpserts[chat.id] = upsert
+			}
+
+			absorbingChatUpdate(upsert)
+
 			if(data.chatDeletes.has(chat.id)) {
 				data.chatDeletes.delete(chat.id)
 			}
-
-			data.chatUpserts[chat.id] = upsert
 		}
 
 		break
 	case 'chats.update':
-		for(const update of eventData as Partial<Chat>[]) {
+		for(const update of eventData as ChatUpdate[]) {
 			const chatId = update.id!
-			// if there is an existing upsert, merge the update into it
-			const upsert = data.chatUpserts[chatId]
-			if(upsert) {
-				concatChats(upsert, update)
-			} else {
-				// merge the update into the existing update
-				const chatUpdate = data.chatUpdates[chatId] || { }
-				data.chatUpdates[chatId] = concatChats(chatUpdate, update)
+			const conditionMatches = update.conditional ? update.conditional(data) : true
+			if(conditionMatches) {
+				delete update.conditional
+
+				// if there is an existing upsert, merge the update into it
+				const upsert = data.historySets.chats[chatId] || data.chatUpserts[chatId]
+				if(upsert) {
+					concatChats(upsert, update)
+				} else {
+					// merge the update into the existing update
+					const chatUpdate = data.chatUpdates[chatId] || { }
+					data.chatUpdates[chatId] = concatChats(chatUpdate, update)
+				}
+			} else if(conditionMatches === undefined) {
+				// condition yet to be fulfilled
+				data.chatUpdates[chatId] = update
 			}
+			// otherwise -- condition not met, update is invalid
 
 			// if the chat has been updated
 			// ignore any existing chat delete
@@ -207,7 +290,10 @@ function append<E extends BufferableEvent>(
 		break
 	case 'chats.delete':
 		for(const chatId of eventData as string[]) {
-			data.chatDeletes.add(chatId)
+			if(!data.chatDeletes.has(chatId)) {
+				data.chatDeletes.add(chatId)
+			}
+
 			// remove any prior updates & upserts
 			if(data.chatUpdates[chatId]) {
 				delete data.chatUpdates[chatId]
@@ -215,20 +301,36 @@ function append<E extends BufferableEvent>(
 
 			if(data.chatUpserts[chatId]) {
 				delete data.chatUpserts[chatId]
+
+			}
+
+			if(data.historySets.chats[chatId]) {
+				delete data.historySets.chats[chatId]
 			}
 		}
 
 		break
 	case 'contacts.upsert':
 		for(const contact of eventData as Contact[]) {
-			let upsert = data.contactUpserts[contact.id] || { } as Contact
-			upsert = Object.assign(upsert, contact)
-			if(data.contactUpdates[contact.id]) {
-				upsert = Object.assign(data.contactUpdates[contact.id], upsert)
-				delete data.contactUpdates[contact.id]
+			let upsert = data.contactUpserts[contact.id]
+			if(!upsert) {
+				upsert = data.historySets.contacts[contact.id]
+				if(upsert) {
+					logger.debug({ contactId: contact.id }, 'absorbed contact upsert in contact set')
+				}
 			}
 
-			data.contactUpserts[contact.id] = upsert
+			if(upsert) {
+				upsert = Object.assign(upsert, trimUndefineds(contact))
+			} else {
+				upsert = contact
+				data.contactUpserts[contact.id] = upsert
+			}
+
+			if(data.contactUpdates[contact.id]) {
+				upsert = Object.assign(data.contactUpdates[contact.id], trimUndefineds(contact))
+				delete data.contactUpdates[contact.id]
+			}
 		}
 
 		break
@@ -237,7 +339,7 @@ function append<E extends BufferableEvent>(
 		for(const update of contactUpdates) {
 			const id = update.id!
 			// merge into prior upsert
-			const upsert = data.contactUpserts[update.id!]
+			const upsert = data.historySets.contacts[id] || data.contactUpserts[id]
 			if(upsert) {
 				Object.assign(upsert, update)
 			} else {
@@ -252,9 +354,16 @@ function append<E extends BufferableEvent>(
 		const { messages, type } = eventData as BaileysEventMap['messages.upsert']
 		for(const message of messages) {
 			const key = stringifyMessageKey(message.key)
-			const existing = data.messageUpserts[key]
+			let existing = data.messageUpserts[key]?.message
+			if(!existing) {
+				existing = data.historySets.messages[key]
+				if(existing) {
+					logger.debug({ messageId: key }, 'absorbed message upsert in message set')
+				}
+			}
+
 			if(existing) {
-				message.messageTimestamp = existing.message.messageTimestamp
+				message.messageTimestamp = existing.messageTimestamp
 			}
 
 			if(data.messageUpdates[key]) {
@@ -263,11 +372,15 @@ function append<E extends BufferableEvent>(
 				delete data.messageUpdates[key]
 			}
 
-			data.messageUpserts[key] = {
-				message,
-				type: type === 'notify' || existing?.type === 'notify'
-					? 'notify'
-					: type
+			if(data.historySets.messages[key]) {
+				data.historySets.messages[key] = message
+			} else {
+				data.messageUpserts[key] = {
+					message,
+					type: type === 'notify' || data.messageUpserts[key]?.type === 'notify'
+						? 'notify'
+						: type
+				}
 			}
 		}
 
@@ -276,14 +389,14 @@ function append<E extends BufferableEvent>(
 		const msgUpdates = eventData as BaileysEventMap['messages.update']
 		for(const { key, update } of msgUpdates) {
 			const keyStr = stringifyMessageKey(key)
-			const existing = data.messageUpserts[keyStr]
+			const existing = data.historySets.messages[keyStr] || data.messageUpserts[keyStr]?.message
 			if(existing) {
-				Object.assign(existing.message, update)
+				Object.assign(existing, update)
 				// if the message was received & read by us
 				// the chat counter must have been incremented
 				// so we need to decrement it
 				if(update.status === WAMessageStatus.READ && !key.fromMe) {
-					decrementChatReadCounterIfMsgDidUnread(existing.message)
+					decrementChatReadCounterIfMsgDidUnread(existing)
 				}
 			} else {
 				const msgUpdate = data.messageUpdates[keyStr] || { key, update: { } }
@@ -299,7 +412,10 @@ function append<E extends BufferableEvent>(
 			const { keys } = deleteData
 			for(const key of keys) {
 				const keyStr = stringifyMessageKey(key)
-				data.messageDeletes[keyStr] = key
+				if(!data.messageDeletes[keyStr]) {
+					data.messageDeletes[keyStr] = key
+
+				}
 
 				if(data.messageUpserts[keyStr]) {
 					delete data.messageUpserts[keyStr]
@@ -349,7 +465,10 @@ function append<E extends BufferableEvent>(
 		for(const update of groupUpdates) {
 			const id = update.id!
 			const groupUpdate = data.groupUpdates[id] || { }
-			data.groupUpdates[id] = Object.assign(groupUpdate, update)
+			if(!data.groupUpdates[id]) {
+				data.groupUpdates[id] = Object.assign(groupUpdate, update)
+
+			}
 		}
 
 		break
@@ -357,6 +476,23 @@ function append<E extends BufferableEvent>(
 		throw new Error(`"${event}" cannot be buffered`)
 	}
 
+	function absorbingChatUpdate(existing: Chat) {
+		const chatId = existing.id
+		const update = data.chatUpdates[chatId]
+		if(update) {
+			const conditionMatches = update.conditional ? update.conditional(data) : true
+			if(conditionMatches) {
+				delete update.conditional
+				logger.debug({ chatId }, 'absorbed chat update in existing chat')
+				Object.assign(existing, concatChats(update as Chat, existing))
+				delete data.chatUpdates[chatId]
+			} else if(conditionMatches === false) {
+				logger.debug({ chatId }, 'chat update condition fail, removing')
+				delete data.chatUpdates[chatId]
+			}
+		}
+	}
+
 	function decrementChatReadCounterIfMsgDidUnread(message: WAMessage) {
 		// decrement chat unread counter
 		// if the message has already been marked read by us
@@ -380,6 +516,15 @@ function append<E extends BufferableEvent>(
 function consolidateEvents(data: BufferedEventData) {
 	const map: BaileysEventData = { }
 
+	if(!data.historySets.empty) {
+		map['messaging-history.set'] = {
+			chats: Object.values(data.historySets.chats),
+			messages: Object.values(data.historySets.messages),
+			contacts: Object.values(data.historySets.contacts),
+			isLatest: data.historySets.isLatest
+		}
+	}
+
 	const chatUpsertList = Object.values(data.chatUpserts)
 	if(chatUpsertList.length) {
 		map['chats.upsert'] = chatUpsertList
@@ -446,7 +591,7 @@ function consolidateEvents(data: BufferedEventData) {
 	return map
 }
 
-function concatChats<C extends Partial<Chat>>(a: C, b: C) {
+function concatChats<C extends Partial<Chat>>(a: C, b: Partial<Chat>) {
 	if(b.unreadCount === null) {
 		// neutralize unread counter
 		if(a.unreadCount! < 0) {