Merge branch 'master' into pr/2472

src/Utils/auth-utils.ts

@@ -1,7 +1,8 @@
 import { randomBytes } from 'crypto'
 import NodeCache from 'node-cache'
 import type { Logger } from 'pino'
-import type { AuthenticationCreds, SignalDataSet, SignalDataTypeMap, SignalKeyStore, SignalKeyStoreWithTransaction, TransactionCapabilityOptions } from '../Types'
+import { DEFAULT_CACHE_TTLS } from '../Defaults'
+import type { AuthenticationCreds, CacheStore, SignalDataSet, SignalDataTypeMap, SignalKeyStore, SignalKeyStoreWithTransaction, TransactionCapabilityOptions } from '../Types'
 import { Curve, signedKeyPair } from './crypto'
 import { delay, generateRegistrationId } from './generics'
 
@@ -9,16 +10,17 @@ import { delay, generateRegistrationId } from './generics'
  * Adds caching capability to a SignalKeyStore
  * @param store the store to add caching to
  * @param logger to log trace events
- * @param opts NodeCache options
+ * @param _cache cache store to use
  */
 export function makeCacheableSignalKeyStore(
 	store: SignalKeyStore,
 	logger: Logger,
-	opts?: NodeCache.Options
+	_cache?: CacheStore
 ): SignalKeyStore {
-	const cache = new NodeCache({
-		...opts || { },
+	const cache = _cache || new NodeCache({
+		stdTTL: DEFAULT_CACHE_TTLS.SIGNAL_STORE, // 5 minutes
 		useClones: false,
+		deleteOnExpire: true,
 	})
 
 	function getUniqueId(type: string, id: string) {

src/Utils/decode-wa-message.ts

@@ -9,7 +9,11 @@ const NO_MESSAGE_FOUND_ERROR_TEXT = 'Message absent from node'
 
 type MessageType = 'chat' | 'peer_broadcast' | 'other_broadcast' | 'group' | 'direct_peer_status' | 'other_status'
 
-export const decodeMessageStanza = (stanza: BinaryNode, auth: AuthenticationState) => {
+/**
+ * Decode the received node as a message.
+ * @note this will only parse the message, not decrypt it
+ */
+export function decodeMessageNode(stanza: BinaryNode, meId: string) {
 	let msgType: MessageType
 	let chatId: string
 	let author: string
@@ -19,7 +23,7 @@ export const decodeMessageStanza = (stanza: BinaryNode, auth: AuthenticationStat
 	const participant: string | undefined = stanza.attrs.participant
 	const recipient: string | undefined = stanza.attrs.recipient
 
-	const isMe = (jid: string) => areJidsSameUser(jid, auth.creds.me!.id)
+	const isMe = (jid: string) => areJidsSameUser(jid, meId)
 
 	if(isJidUser(from)) {
 		if(recipient) {
@@ -60,8 +64,6 @@ export const decodeMessageStanza = (stanza: BinaryNode, auth: AuthenticationStat
 		throw new Boom('Unknown message type', { data: stanza })
 	}
 
-	const sender = msgType === 'chat' ? author : chatId
-
 	const fromMe = isMe(stanza.attrs.participant || stanza.attrs.from)
 	const pushname = stanza.attrs.notify
 
@@ -75,13 +77,23 @@ export const decodeMessageStanza = (stanza: BinaryNode, auth: AuthenticationStat
 	const fullMessage: proto.IWebMessageInfo = {
 		key,
 		messageTimestamp: +stanza.attrs.t,
-		pushName: pushname
+		pushName: pushname,
+		broadcast: isJidBroadcast(from)
 	}
 
 	if(key.fromMe) {
 		fullMessage.status = proto.WebMessageInfo.Status.SERVER_ACK
 	}
 
+	return {
+		fullMessage,
+		author,
+		sender: msgType === 'chat' ? author : chatId
+	}
+}
+
+export const decryptMessageNode = (stanza: BinaryNode, auth: AuthenticationState) => {
+	const { fullMessage, author, sender } = decodeMessageNode(stanza, auth.creds.me!.id)
 	return {
 		fullMessage,
 		category: stanza.attrs.category,

src/Utils/generics.ts

@@ -138,13 +138,13 @@ export const delayCancellable = (ms: number) => {
 
 export async function promiseTimeout<T>(ms: number | undefined, promise: (resolve: (v?: T) => void, reject: (error) => void) => void) {
 	if(!ms) {
-		return new Promise (promise)
+		return new Promise(promise)
 	}
 
 	const stack = new Error().stack
 	// Create a promise that rejects in <ms> milliseconds
 	const { delay, cancel } = delayCancellable (ms)
-	const p = new Promise ((resolve, reject) => {
+	const p = new Promise((resolve, reject) => {
 		delay
 			.then(() => reject(
 				new Boom('Timed Out', {
@@ -353,7 +353,10 @@ export const getCodeFromWSError = (error: Error) => {
 		if(!Number.isNaN(code) && code >= 400) {
 			statusCode = code
 		}
-	} else if((error as any).code?.startsWith('E')) { // handle ETIMEOUT, ENOTFOUND etc
+	} else if(
+		(error as any).code?.startsWith('E')
+		|| error?.message?.includes('timed out')
+	) { // handle ETIMEOUT, ENOTFOUND etc
 		statusCode = 408
 	}
 

src/Utils/link-preview.ts

@@ -1,3 +1,4 @@
+import { AxiosRequestConfig } from 'axios'
 import { Logger } from 'pino'
 import { WAMediaUploadFunction, WAUrlInfo } from '../Types'
 import { prepareWAMessageMedia } from './messages'
@@ -21,7 +22,7 @@ export type URLGenerationOptions = {
 		/** Timeout in ms */
 		timeout: number
 		proxyUrl?: string
-		headers?: { [key: string]: string }
+		headers?: AxiosRequestConfig<{}>['headers']
 	}
 	uploadImage?: WAMediaUploadFunction
 	logger?: Logger
@@ -47,7 +48,10 @@ export const getUrlInfo = async(
 			previewLink = 'https://' + previewLink
 		}
 
-		const info = await getLinkPreview(previewLink, opts.fetchOpts)
+		const info = await getLinkPreview(previewLink, {
+			...opts.fetchOpts,
+			headers: opts.fetchOpts as {}
+		})
 		if(info && 'title' in info && info.title) {
 			const [image] = info.images
 
@@ -62,7 +66,11 @@ export const getUrlInfo = async(
 			if(opts.uploadImage) {
 				const { imageMessage } = await prepareWAMessageMedia(
 					{ image: { url: image } },
-					{ upload: opts.uploadImage, mediaTypeOverride: 'thumbnail-link' }
+					{
+						upload: opts.uploadImage,
+						mediaTypeOverride: 'thumbnail-link',
+						options: opts.fetchOpts
+					}
 				)
 				urlInfo.jpegThumbnail = imageMessage?.jpegThumbnail
 					? Buffer.from(imageMessage.jpegThumbnail)

src/Utils/messages-media.ts

@@ -192,8 +192,11 @@ export async function getAudioDuration(buffer: Buffer | string | Readable) {
 		metadata = await musicMetadata.parseBuffer(buffer, undefined, { duration: true })
 	} else if(typeof buffer === 'string') {
 		const rStream = createReadStream(buffer)
-		metadata = await musicMetadata.parseStream(rStream, undefined, { duration: true })
-		rStream.close()
+		try {
+			metadata = await musicMetadata.parseStream(rStream, undefined, { duration: true })
+		} finally {
+			rStream.destroy()
+		}
 	} else {
 		metadata = await musicMetadata.parseStream(buffer, undefined, { duration: true })
 	}
@@ -209,29 +212,29 @@ export const toReadable = (buffer: Buffer) => {
 }
 
 export const toBuffer = async(stream: Readable) => {
-	let buff = Buffer.alloc(0)
+	const chunks: Buffer[] = []
 	for await (const chunk of stream) {
-		buff = Buffer.concat([ buff, chunk ])
+		chunks.push(chunk)
 	}
 
 	stream.destroy()
-	return buff
+	return Buffer.concat(chunks)
 }
 
-export const getStream = async(item: WAMediaUpload) => {
+export const getStream = async(item: WAMediaUpload, opts?: AxiosRequestConfig) => {
 	if(Buffer.isBuffer(item)) {
-		return { stream: toReadable(item), type: 'buffer' }
+		return { stream: toReadable(item), type: 'buffer' } as const
 	}
 
 	if('stream' in item) {
-		return { stream: item.stream, type: 'readable' }
+		return { stream: item.stream, type: 'readable' } as const
 	}
 
 	if(item.url.toString().startsWith('http://') || item.url.toString().startsWith('https://')) {
-		return { stream: await getHttpStream(item.url), type: 'remote' }
+		return { stream: await getHttpStream(item.url, opts), type: 'remote' } as const
 	}
 
-	return { stream: createReadStream(item.url), type: 'file' }
+	return { stream: createReadStream(item.url), type: 'file' } as const
 }
 
 /** generates a thumbnail for a given media, if required */
@@ -278,21 +281,23 @@ export const getHttpStream = async(url: string | URL, options: AxiosRequestConfi
 	return fetched.data as Readable
 }
 
+type EncryptedStreamOptions = {
+	saveOriginalFileIfRequired?: boolean
+	logger?: Logger
+	opts?: AxiosRequestConfig
+}
+
 export const encryptedStream = async(
 	media: WAMediaUpload,
 	mediaType: MediaType,
-	saveOriginalFileIfRequired = true,
-	logger?: Logger
+	{ logger, saveOriginalFileIfRequired, opts }: EncryptedStreamOptions = {}
 ) => {
-	const { stream, type } = await getStream(media)
+	const { stream, type } = await getStream(media, opts)
 
 	logger?.debug('fetched media stream')
 
 	const mediaKey = Crypto.randomBytes(32)
 	const { cipherKey, iv, macKey } = getMediaKeys(mediaKey, mediaType)
-	// random name
-	//const encBodyPath = join(getTmpFilesDirectory(), mediaType + generateMessageID() + '.enc')
-	// const encWriteStream = createWriteStream(encBodyPath)
 	const encWriteStream = new Readable({ read: () => {} })
 
 	let bodyPath: string | undefined
@@ -312,15 +317,23 @@ export const encryptedStream = async(
 	let sha256Plain = Crypto.createHash('sha256')
 	let sha256Enc = Crypto.createHash('sha256')
 
-	const onChunk = (buff: Buffer) => {
-		sha256Enc = sha256Enc.update(buff)
-		hmac = hmac.update(buff)
-		encWriteStream.push(buff)
-	}
-
 	try {
 		for await (const data of stream) {
 			fileLength += data.length
+
+			if(
+				type === 'remote'
+				&& opts?.maxContentLength
+				&& fileLength + data.length > opts.maxContentLength
+			) {
+				throw new Boom(
+					`content length exceeded when encrypting "${type}"`,
+					{
+						data: { media, type }
+					}
+				)
+			}
+
 			sha256Plain = sha256Plain.update(data)
 			if(writeStream) {
 				if(!writeStream.write(data)) {
@@ -342,7 +355,7 @@ export const encryptedStream = async(
 		encWriteStream.push(mac)
 		encWriteStream.push(null)
 
-		writeStream && writeStream.end()
+		writeStream?.end()
 		stream.destroy()
 
 		logger?.debug('encrypted data successfully')
@@ -366,8 +379,22 @@ export const encryptedStream = async(
 		sha256Enc.destroy(error)
 		stream.destroy(error)
 
+		if(didSaveToTmpPath) {
+			try {
+				await fs.unlink(bodyPath!)
+			} catch(err) {
+				logger?.error({ err }, 'failed to save to tmp path')
+			}
+		}
+
 		throw error
 	}
+
+	function onChunk(buff: Buffer) {
+		sha256Enc = sha256Enc.update(buff)
+		hmac = hmac.update(buff)
+		encWriteStream.push(buff)
+	}
 }
 
 const DEF_HOST = 'mmg.whatsapp.net'
@@ -421,14 +448,14 @@ export const downloadEncryptedContent = async(
 
 	const endChunk = endByte ? toSmallestChunkSize(endByte || 0) + AES_CHUNK_SIZE : undefined
 
-	const headers: { [_: string]: string } = {
+	const headers: AxiosRequestConfig['headers'] = {
 		...options?.headers || { },
 		Origin: DEFAULT_ORIGIN,
 	}
 	if(startChunk || endChunk) {
-		headers.Range = `bytes=${startChunk}-`
+		headers!.Range = `bytes=${startChunk}-`
 		if(endChunk) {
-			headers.Range += endChunk
+			headers!.Range += endChunk
 		}
 	}
 
@@ -644,7 +671,7 @@ export const encryptMediaRetryRequest = (
 				tag: 'rmr',
 				attrs: {
 					jid: key.remoteJid!,
-					from_me: (!!key.fromMe).toString(),
+					'from_me': (!!key.fromMe).toString(),
 					// @ts-ignore
 					participant: key.participant || undefined
 				}

src/Utils/messages.ts

@@ -151,7 +151,11 @@ export const prepareWAMessageMedia = async(
 	} = await encryptedStream(
 		uploadData.media,
 		options.mediaTypeOverride || mediaType,
-		requiresOriginalForSomeProcessing
+		{
+			logger,
+			saveOriginalFileIfRequired: requiresOriginalForSomeProcessing,
+			opts: options.options
+		}
 	)
 	 // url safe Base64 encode the SHA256 hash of the body
 	const fileEncSha256B64 = fileEncSha256.toString('base64')

src/Utils/process-message.ts

@@ -3,7 +3,7 @@ import type { Logger } from 'pino'
 import { proto } from '../../WAProto'
 import { AuthenticationCreds, BaileysEventEmitter, Chat, GroupMetadata, ParticipantAction, SignalKeyStoreWithTransaction, WAMessageStubType } from '../Types'
 import { downloadAndProcessHistorySyncNotification, getContentType, normalizeMessageContent, toNumber } from '../Utils'
-import { areJidsSameUser, jidNormalizedUser } from '../WABinary'
+import { areJidsSameUser, isJidBroadcast, isJidStatusBroadcast, jidNormalizedUser } from '../WABinary'
 
 type ProcessMessageContext = {
 	shouldProcessHistoryMsg: boolean
@@ -72,6 +72,22 @@ export const shouldIncrementChatUnread = (message: proto.IWebMessageInfo) => (
 	!message.key.fromMe && !message.messageStubType
 )
 
+/**
+ * Get the ID of the chat from the given key.
+ * Typically -- that'll be the remoteJid, but for broadcasts, it'll be the participant
+ */
+export const getChatId = ({ remoteJid, participant, fromMe }: proto.IMessageKey) => {
+	if(
+		isJidBroadcast(remoteJid!)
+		&& !isJidStatusBroadcast(remoteJid!)
+		&& !fromMe
+	) {
+		return participant!
+	}
+
+	return remoteJid!
+}
+
 const processMessage = async(
 	message: proto.IWebMessageInfo,
 	{
@@ -86,7 +102,7 @@ const processMessage = async(
 	const meId = creds.me!.id
 	const { accountSettings } = creds
 
-	const chat: Partial<Chat> = { id: jidNormalizedUser(message.key.remoteJid!) }
+	const chat: Partial<Chat> = { id: jidNormalizedUser(getChatId(message.key)) }
 	const isRealMsg = isRealMessage(message, meId)
 
 	if(isRealMsg) {