From 71e34fc5f8e4dc9ac72037026b21442b10dfeb3c Mon Sep 17 00:00:00 2001 From: Adhiraj Singh Date: Mon, 12 Jul 2021 23:50:08 +0530 Subject: [PATCH] Cleanup + add remaining utils --- src/Connection/auth.ts | 5 +- src/Connection/index.ts | 7 +- src/Connection/messages.ts | 37 +++++---- src/Connection/socket.ts | 3 +- src/Defaults/index.ts | 2 + src/Store/in-memory-store.ts | 130 +++++++++++++++++++++---------- src/Store/index.ts | 4 +- src/Store/ordered-dictionary.ts | 66 ++++++++++++++++ src/Types/Chat.ts | 3 +- src/Types/Message.ts | 26 +++++-- src/Types/index.ts | 2 + src/Utils/index.ts | 5 ++ src/Utils/messages.ts | 3 - src/Utils/validate-connection.ts | 12 ++- src/index.ts | 3 +- 15 files changed, 234 insertions(+), 74 deletions(-) create mode 100644 src/Store/ordered-dictionary.ts create mode 100644 src/Utils/index.ts diff --git a/src/Connection/auth.ts b/src/Connection/auth.ts index 87f42c2..8834191 100644 --- a/src/Connection/auth.ts +++ b/src/Connection/auth.ts @@ -3,8 +3,7 @@ import EventEmitter from "events" import * as Curve from 'curve25519-js' import { BaileysEventEmitter, BaileysEventMap, SocketConfig, CurveKeyPair, WAInitResponse, ConnectionState, DisconnectReason } from "../Types" import { makeSocket } from "./socket" -import { generateClientID, promiseTimeout } from "../Utils/generics" -import { normalizedAuthInfo, computeChallengeResponse, validateNewConnection } from "../Utils/validate-connection" +import { generateClientID, promiseTimeout, normalizedAuthInfo, computeChallengeResponse, validateNewConnection } from "../Utils" import { randomBytes } from "crypto" import { AuthenticationCredentials } from "../Types" @@ -106,7 +105,7 @@ const makeAuthSocket = (config: SocketConfig) => { } ) .finally(() => ( - ev.off('state.update', listener) + ev.off('connection.update', listener) )) ) } diff --git a/src/Connection/index.ts b/src/Connection/index.ts index 4fe93a9..6c3eedd 100644 --- a/src/Connection/index.ts +++ b/src/Connection/index.ts @@ -1,12 +1,15 @@ import { SocketConfig } from '../Types' import { DEFAULT_CONNECTION_CONFIG } from '../Defaults' import { EventEmitter } from 'events' -import * as Connection from './groups' +import _makeConnection from './groups' // export the last socket layer const makeConnection = (config: Partial) => ( - Connection.default({ + _makeConnection({ ...DEFAULT_CONNECTION_CONFIG, ...config }) ) + +export type Connection = ReturnType + export default makeConnection \ No newline at end of file diff --git a/src/Connection/messages.ts b/src/Connection/messages.ts index e6481e0..1b83d8b 100644 --- a/src/Connection/messages.ts +++ b/src/Connection/messages.ts @@ -1,12 +1,10 @@ import BinaryNode from "../BinaryNode"; import { Boom } from '@hapi/boom' import { EventEmitter } from 'events' -import { Chat, Presence, SocketConfig, WAMessage, WAMessageKey, ParticipantAction, WAMessageProto, WAMessageStatus, WAMessageStubType, GroupMetadata, AnyMessageContent, MiscMessageGenerationOptions, WAFlag, WAMetric, WAUrlInfo, MediaConnInfo, MessageUpdateType, MessageInfo } from "../Types"; -import { isGroupID, toNumber, whatsappID } from "../Utils/generics"; +import { Chat, Presence, WAMessageCursor, SocketConfig, WAMessage, WAMessageKey, ParticipantAction, WAMessageProto, WAMessageStatus, WAMessageStubType, GroupMetadata, AnyMessageContent, MiscMessageGenerationOptions, WAFlag, WAMetric, WAUrlInfo, MediaConnInfo, MessageUpdateType, MessageInfo } from "../Types"; +import { isGroupID, toNumber, whatsappID, generateWAMessage, decryptMediaMessageBuffer } from "../Utils"; import makeChatsSocket from "./chats"; import { WA_DEFAULT_EPHEMERAL } from "../Defaults"; -import { generateWAMessage } from "../Utils/messages"; -import { decryptMediaMessageBuffer } from "../Utils/messages-media"; const STATUS_MAP = { read: WAMessageStatus.READ, @@ -47,9 +45,12 @@ const makeMessagesSocket = (config: SocketConfig) => { const fetchMessagesFromWA = async( jid: string, count: number, - indexMessage?: { id?: string; fromMe?: boolean }, - mostRecentFirst: boolean = true + cursor?: WAMessageCursor ) => { + let key: WAMessageKey + if(cursor) { + key = 'before' in cursor ? cursor.before : cursor.after + } const { data }:BinaryNode = await query({ json: new BinaryNode( 'query', @@ -57,10 +58,10 @@ const makeMessagesSocket = (config: SocketConfig) => { epoch: currentEpoch().toString(), type: 'message', jid: jid, - kind: mostRecentFirst ? 'before' : 'after', + kind: !cursor || 'before' in cursor ? 'before' : 'after', count: count.toString(), - index: indexMessage?.id, - owner: indexMessage?.fromMe === false ? 'false' : 'true', + index: key?.id, + owner: key?.fromMe === false ? 'false' : 'true', } ), binaryTag: [WAMetric.queryMessages, WAFlag.ignore], @@ -91,7 +92,7 @@ const makeMessagesSocket = (config: SocketConfig) => { }) Object.keys(response[1]).forEach (key => content[key] = response[1][key]) // update message - ev.emit('messages.upsert', { messages: [message], type: 'append' }) + ev.emit('messages.update', [{ key: message.key, message: message.message }]) return response } @@ -365,6 +366,18 @@ const makeMessagesSocket = (config: SocketConfig) => { }, updateMediaMessage, fetchMessagesFromWA, + /** Load a single message specified by the ID */ + loadMessageFromWA: async(jid: string, id: string) => { + let message: WAMessage + + // load the message before the given message + let messages = (await fetchMessagesFromWA(jid, 1, { before: {id, fromMe: true} })) + if(!messages[0]) messages = (await fetchMessagesFromWA(jid, 1, { before: {id, fromMe: false} })) + // the message after the loaded message is the message required + const [actual] = await fetchMessagesFromWA(jid, 1, { after: messages[0] && messages[0].key }) + message = actual + return message + }, searchMessages: async(txt: string, inJid: string | null, count: number, page: number) => { const {data, attributes}: BinaryNode = await query({ json: new BinaryNode( @@ -419,10 +432,6 @@ const makeMessagesSocket = (config: SocketConfig) => { { ...options, userJid: userJid, - /*ephemeralOptions: chat?.ephemeral ? { - expiration: chat.ephemeral, - eph_setting_ts: chat.eph_setting_ts - } : undefined,*/ getUrlInfo: generateUrlInfo, getMediaOptions: refreshMediaConn } diff --git a/src/Connection/socket.ts b/src/Connection/socket.ts index d86c072..d6ae622 100644 --- a/src/Connection/socket.ts +++ b/src/Connection/socket.ts @@ -5,8 +5,7 @@ import { promisify } from "util" import WebSocket from "ws" import BinaryNode from "../BinaryNode" import { DisconnectReason, SocketConfig, SocketQueryOptions, SocketSendMessageOptions } from "../Types" -import { aesEncrypt, hmacSign, promiseTimeout, unixTimestampSeconds } from "../Utils/generics" -import { decodeWAMessage } from "../Utils/decode-wa-message" +import { aesEncrypt, hmacSign, promiseTimeout, unixTimestampSeconds, decodeWAMessage } from "../Utils" import { WAFlag, WAMetric, WATag } from "../Types" import { DEFAULT_ORIGIN, DEF_CALLBACK_PREFIX, DEF_TAG_PREFIX, PHONE_CONNECTION_CB } from "../Defaults" diff --git a/src/Defaults/index.ts b/src/Defaults/index.ts index 669ad7f..caca01f 100644 --- a/src/Defaults/index.ts +++ b/src/Defaults/index.ts @@ -4,6 +4,8 @@ import { Browsers } from "../Utils/generics" export const UNAUTHORIZED_CODES = [401, 403, 419] +export const STORIES_JID = 'status@broadcast' + export const DEFAULT_ORIGIN = 'https://web.whatsapp.com' export const DEF_CALLBACK_PREFIX = 'CB:' export const DEF_TAG_PREFIX = 'TAG:' diff --git a/src/Store/in-memory-store.ts b/src/Store/in-memory-store.ts index 33043e7..5b092dc 100644 --- a/src/Store/in-memory-store.ts +++ b/src/Store/in-memory-store.ts @@ -1,39 +1,44 @@ import KeyedDB from "@adiwajshing/keyed-db" +import { Comparable } from "@adiwajshing/keyed-db/lib/Types" import { Logger } from "pino" -import makeConnection from "../Connection" -import { BaileysEventEmitter, Chat, ConnectionState, Contact, WAMessage, WAMessageKey } from "../Types" +import type { Connection } from "../Connection" +import type { BaileysEventEmitter, Chat, ConnectionState, Contact, WAMessage, WAMessageCursor } from "../Types" +import { toNumber } from "../Utils" +import makeOrderedDictionary from "./ordered-dictionary" export const waChatKey = (pin: boolean) => ({ key: (c: Chat) => (pin ? (c.pin ? '1' : '0') : '') + (c.archive === 'true' ? '0' : '1') + c.t.toString(16).padStart(8, '0') + c.jid, compare: (k1: string, k2: string) => k2.localeCompare (k1) }) +export const waMessageID = (m: WAMessage) => m.key.id + export type BaileysInMemoryStoreConfig = { + chatKey: Comparable logger: Logger } +const makeMessagesDictionary = () => makeOrderedDictionary(waMessageID) + export default( - { logger }: BaileysInMemoryStoreConfig + { logger, chatKey }: BaileysInMemoryStoreConfig ) => { - const chats = new KeyedDB(waChatKey(true), c => c.jid) - const messages: { [_: string]: WAMessage[] } = {} + + const chats = new KeyedDB(chatKey, c => c.jid) + const messages: { [_: string]: ReturnType } = {} const contacts: { [_: string]: Contact } = {} const state: ConnectionState = { connection: 'close', phoneConnected: false } - const messageIndex = (key: WAMessageKey) => { - const messageList = messages[key.remoteJid!] - if(messageList) { - const idx = messageList.findIndex(m => m.key.id === key.id) - if(idx >= 0) { - return { messageList, idx } - } - } + const assertMessageList = (jid: string) => { + if(!messages[jid]) messages[jid] = makeMessagesDictionary() + return messages[jid] } const listen = (ev: BaileysEventEmitter) => { + ev.on('connection.update', update => { Object.assign(state, update) }) @@ -91,55 +96,54 @@ export default( case 'notify': for(const msg of newMessages) { const jid = msg.key.remoteJid! - const result = messageIndex(newMessages[0].key) - if(!result) { - if(!messages[jid]) { - messages[jid] = [] - } - messages[jid].push(msg) - } else { - result.messageList[result.idx] = msg + const list = assertMessageList(jid) + list.upsert(msg, 'append') + + if(type === 'notify' && !chats.get(jid)) { + ev.emit('chats.upsert', { + chats: [ { jid, t: toNumber(msg.messageTimestamp), count: 1 } ], + type: 'upsert' + }) } } break case 'last': for(const msg of newMessages) { const jid = msg.key.remoteJid! - if(!messages[jid]) { - messages[jid] = [] - } - const [lastItem] = messages[jid].slice(-1) + const list = assertMessageList(jid) + const [lastItem] = list.array.slice(-1) // reset message list if(lastItem && lastItem.key.id !== msg.key.id) { - messages[jid] = [msg] + list.clear() + list.upsert(msg, 'append') } } break case 'prepend': - + for(const msg of newMessages) { + const jid = msg.key.remoteJid! + const list = assertMessageList(jid) + list.upsert(msg, 'prepend') + } break } }) ev.on('messages.update', updates => { for(const update of updates) { - const result = messageIndex(update.key!) - if(result) { - Object.assign(result.messageList[result.idx], update) - } else { + const list = assertMessageList(update.key.remoteJid) + const result = list.updateAssign(update) + if(!result) { logger.debug({ update }, `got update for non-existant message`) } } }) ev.on('messages.delete', item => { + const list = assertMessageList(item.jid) if('all' in item) { - messages[item.jid] = [] + list.clear() } else { const idSet = new Set(item.ids) - if(messages[item.jid]) { - messages[item.jid] = messages[item.jid].filter( - m => !idSet.has(m.key.id) - ) - } + list.filter(m => !idSet.has(m.key.id)) } }) } @@ -148,14 +152,60 @@ export default( chats, contacts, messages, + state, listen, - fetchImageUrl: async(jid: string, sock: ReturnType) => { + loadMessages: async(jid: string, count: number, cursor: WAMessageCursor, sock: Connection | undefined) => { + const list = assertMessageList(jid) + const retrieve = async(count: number, cursor: WAMessageCursor) => { + const result = await sock?.fetchMessagesFromWA(jid, count, cursor) + return result || [] + } + const mode = !cursor || 'before' in cursor ? 'before' : 'after' + const cursorKey = !!cursor ? ('before' in cursor ? cursor.before : cursor.after) : undefined + const cursorValue = cursorKey ? list.get(cursorKey.id) : undefined + + let messages: WAMessage[] + if(messages && mode ==='before' && (!cursorKey || cursorValue)) { + const msgIdx = messages.findIndex(m => m.key.id === cursorKey.id) + messages = list.array.slice(0, msgIdx) + + const diff = count - messages.length + if (diff < 0) { + messages = messages.slice(-count) // get the last X messages + } else if (diff > 0) { + const [fMessage] = messages + const extra = await retrieve (diff, { before: fMessage?.key || cursorKey }) + // add to DB + for(let i = extra.length-1; i >= 0;i--) { + list.upsert(extra[i], 'prepend') + } + } + } else messages = await retrieve(count, cursor) + + return messages + }, + loadMessage: async(jid: string, id: string, sock: Connection | undefined) => { + let message = messages[jid]?.get(id) + if(!message) { + message = await sock?.loadMessageFromWA(jid, id) + } + return message + }, + mostRecentMessage: async(jid: string, sock: Connection | undefined) => { + let message = messages[jid]?.array.slice(-1)[0] + if(!message) { + const [result] = await sock?.fetchMessagesFromWA(jid, 1, undefined) + message = result + } + return message + }, + fetchImageUrl: async(jid: string, sock: Connection | undefined) => { const contact = contacts[jid] if(!contact) { - return sock.fetchImageUrl(jid) + return sock?.fetchImageUrl(jid) } if(!contact.imgUrl) { - contact.imgUrl = await sock.fetchImageUrl(jid) + contact.imgUrl = await sock?.fetchImageUrl(jid) } return contact.imgUrl } diff --git a/src/Store/index.ts b/src/Store/index.ts index 7e6ed66..1bfc30c 100644 --- a/src/Store/index.ts +++ b/src/Store/index.ts @@ -1,2 +1,2 @@ -import inMemoryStore from "./in-memory-store"; -export default inMemoryStore \ No newline at end of file +import makeInMemoryStore from './in-memory-store' +export { makeInMemoryStore } \ No newline at end of file diff --git a/src/Store/ordered-dictionary.ts b/src/Store/ordered-dictionary.ts new file mode 100644 index 0000000..50272a0 --- /dev/null +++ b/src/Store/ordered-dictionary.ts @@ -0,0 +1,66 @@ + +const makeOrderedDictionary = function(idGetter: (item: T) => string) { + const array: T[] = [] + const dict: { [_: string]: T } = { } + + const get = (id: string) => dict[id] + + const update = (item: T) => { + const id = idGetter(item) + const idx = array.findIndex(i => idGetter(i) === id) + if(idx >= 0) { + array[idx] = item + dict[id] = item + } + return false + } + + const upsert = (item: T, mode: 'append' | 'prepend') => { + const id = idGetter(item) + if(get(id)) { + update(item) + } else { + if(mode === 'append') { + array.push(item) + } else { + array.splice(0, 0, item) + } + dict[id] = item + } + } + + const remove = (item: T) => { + const id = idGetter(item) + const idx = array.findIndex(i => idGetter(i) === id) + if(idx >= 0) { + array.splice(idx, 1) + delete dict[id] + return true + } + return false + } + + return { + array, + get, + upsert, + update, + remove, + updateAssign: (update: Partial) => { + const item = get(idGetter(update as any)) + if(item) { + Object.assign(item, update) + } + return false + }, + clear: () => { + array.splice(0, array.length) + Object.keys(dict).forEach(key => { delete dict[key] }) + }, + filter: (contain: (item: T) => boolean) => { + //const copy = + } + } +} +export default makeOrderedDictionary +//export type OrderedDictionary = ReturnType \ No newline at end of file diff --git a/src/Types/Chat.ts b/src/Types/Chat.ts index 8f285cb..d35b929 100644 --- a/src/Types/Chat.ts +++ b/src/Types/Chat.ts @@ -54,4 +54,5 @@ export type ChatModification = messages: { id: string, fromMe?: boolean }[], star: boolean } - } \ No newline at end of file + } | + { delete: true } \ No newline at end of file diff --git a/src/Types/Message.ts b/src/Types/Message.ts index 476181b..0c7e52d 100644 --- a/src/Types/Message.ts +++ b/src/Types/Message.ts @@ -78,7 +78,6 @@ export type AnyMediaMessageContent = ( { mimetype?: string } export type AnyRegularMessageContent = - string | ({ text: string } @@ -109,14 +108,15 @@ export type MiscMessageGenerationOptions = { timestamp?: Date /** the message you want to quote */ quoted?: WAMessage -} -export type MessageGenerationOptionsFromContent = MiscMessageGenerationOptions & { - userJid: string + /** disappearing messages settings */ ephemeralOptions?: { expiration: number | string eph_setting_ts: number | string } } +export type MessageGenerationOptionsFromContent = MiscMessageGenerationOptions & { + userJid: string +} export type MediaGenerationOptions = { logger?: Logger agent?: Agent @@ -132,4 +132,20 @@ export type MessageUpdateType = 'prepend' | 'append' | 'notify' | 'last' export interface MessageInfo { reads: {jid: string, t: string}[] deliveries: {jid: string, t: string}[] -} \ No newline at end of file +} + + +export interface MessageStatusUpdate { + from: string + to: string + /** Which participant caused the update (only for groups) */ + participant?: string + timestamp: Date + /** Message IDs read/delivered */ + ids: string[] + /** Status of the Message IDs */ + type: WAMessageStatus +} + + +export type WAMessageCursor = { before: WAMessageKey | undefined } | { after: WAMessageKey | undefined } \ No newline at end of file diff --git a/src/Types/index.ts b/src/Types/index.ts index efdd6c8..6da2733 100644 --- a/src/Types/index.ts +++ b/src/Types/index.ts @@ -195,5 +195,7 @@ export type BaileysEventMap = { } export interface BaileysEventEmitter extends EventEmitter { on(event: T, listener: (arg: BaileysEventMap[T]) => void): this + off(event: T, listener: (arg: BaileysEventMap[T]) => void): this + removeAllListeners(event: T): this emit(event: T, arg: BaileysEventMap[T]): boolean } \ No newline at end of file diff --git a/src/Utils/index.ts b/src/Utils/index.ts new file mode 100644 index 0000000..15973e8 --- /dev/null +++ b/src/Utils/index.ts @@ -0,0 +1,5 @@ +export * from './decode-wa-message' +export * from './generics' +export * from './messages' +export * from './messages-media' +export * from './validate-connection' \ No newline at end of file diff --git a/src/Utils/messages.ts b/src/Utils/messages.ts index 5cf2202..0a7c39f 100644 --- a/src/Utils/messages.ts +++ b/src/Utils/messages.ts @@ -220,9 +220,6 @@ export const generateWAMessageContent = async( options: MessageContentGenerationOptions ) => { let m: WAMessageContent = {} - if(typeof message === 'string') { - message = { text: message } - } if('text' in message) { const extContent = { ...message } as WATextMessage if (!!options.getUrlInfo && message.text.match(URL_REGEX)) { diff --git a/src/Utils/validate-connection.ts b/src/Utils/validate-connection.ts index c3c542e..c506bd7 100644 --- a/src/Utils/validate-connection.ts +++ b/src/Utils/validate-connection.ts @@ -1,7 +1,7 @@ import {Boom} from '@hapi/boom' import * as Curve from 'curve25519-js' import type { Contact } from '../Types/Contact' -import type { AnyAuthenticationCredentials, AuthenticationCredentials, CurveKeyPair } from "../Types" +import type { AnyAuthenticationCredentials, AuthenticationCredentials, AuthenticationCredentialsBase64, CurveKeyPair } from "../Types" import { aesDecrypt, hkdf, hmacSign, whatsappID } from './generics' import { readFileSync } from 'fs' @@ -32,6 +32,16 @@ export const normalizedAuthInfo = (authInfo: AnyAuthenticationCredentials | stri } return authInfo as AuthenticationCredentials } + +export const base64EncodedAuthenticationCredentials = (creds: AnyAuthenticationCredentials) => { + const normalized = normalizedAuthInfo(creds) + return { + ...normalized, + encKey: normalized.encKey.toString('base64'), + macKey: normalized.macKey.toString('base64') + } as AuthenticationCredentialsBase64 +} + /** * Once the QR code is scanned and we can validate our connection, or we resolved the challenge when logging back in * @private diff --git a/src/index.ts b/src/index.ts index 9e38e6c..331f02b 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,8 +1,9 @@ import makeConnection from './Connection' export * from '../WAMessage/WAMessage' -export * from './Utils/messages' +export * from './Utils' export * from './Types' export * from './Store' +export * from './Defaults' export default makeConnection \ No newline at end of file