From 5d6eb4f965dfb598deead944fc1b55cfa1264743 Mon Sep 17 00:00:00 2001 From: siepra Date: Wed, 15 Nov 2023 16:30:00 +0100 Subject: [PATCH] chore: move certificates to its own store --- .../certificates/certificates.store.ts | 105 +++++++---- .../src/nest/storage/storage.service.ts | 177 ++++-------------- .../backend/src/nest/storage/storage.types.ts | 1 + 3 files changed, 105 insertions(+), 178 deletions(-) diff --git a/packages/backend/src/nest/storage/certificates/certificates.store.ts b/packages/backend/src/nest/storage/certificates/certificates.store.ts index 0d126226c0..acb9bc05c4 100644 --- a/packages/backend/src/nest/storage/certificates/certificates.store.ts +++ b/packages/backend/src/nest/storage/certificates/certificates.store.ts @@ -4,9 +4,9 @@ import { StorageEvents } from '../storage.types' import EventStore from 'orbit-db-eventstore' import OrbitDB from 'orbit-db' -import { loadCertificate, keyFromCertificate } from '@quiet/identity' +import { keyFromCertificate, CertFieldsTypes, parseCertificate, getReqFieldValue, getCertFieldValue } from '@quiet/identity' -import { ConnectionProcessInfo, SocketActionTypes } from '@quiet/types' +import { ConnectionProcessInfo, SocketActionTypes, UserData } from '@quiet/types' import createLogger from '../../common/logger' @@ -16,8 +16,12 @@ export class CertificatesStore { public orbitDb: OrbitDB public store: EventStore + private filteredCertificatesMapping: Map> + private usernameMapping: Map + constructor(orbitDb: OrbitDB) { this.orbitDb = orbitDb + this.filteredCertificatesMapping = new Map() } public async init(emitter: EventEmitter) { @@ -30,38 +34,45 @@ export class CertificatesStore { }, }) - this.store.events.on('write', async (_address, entry) => { - logger('Saved certificate locally') - - emitter.emit(StorageEvents.LOAD_CERTIFICATES, { - certificates: await this.getCertificates(), - }) - - // await this.updatePeersList() - }) - this.store.events.on('ready', async () => { logger('Loaded certificates to memory') - emitter.emit(SocketActionTypes.CONNECTION_PROCESS_INFO, ConnectionProcessInfo.LOADED_CERTIFICATES) + }) - emitter.emit(StorageEvents.LOAD_CERTIFICATES, { - certificates: await this.getCertificates(), - }) + this.store.events.on('write', async () => { + logger('Saved certificate locally') + await loadedCertificates() }) this.store.events.on('replicated', async () => { logger('REPLICATED: Certificates') - emitter.emit(SocketActionTypes.CONNECTION_PROCESS_INFO, ConnectionProcessInfo.CERTIFICATES_REPLICATED) + await loadedCertificates() + }) - emitter.emit(StorageEvents.LOAD_CERTIFICATES, { - certificates: await this.getCertificates(), + const loadedCertificates = async () => { + emitter.emit(StorageEvents.LOADED_CERTIFICATES, { + certificates: await this.getCertificates(), }) + } + } - // await this.updatePeersList() - }) + public async close() { + await this.store?.close() + } + public getAddress() { + return this.store?.address + } + + public async addCertificate(certificate: string) { + logger('Adding user certificate') + await this.store.add(certificate) + return true + } + + public async loadAllCertificates() { + return this.getCertificates() } /* @@ -79,30 +90,44 @@ export class CertificatesStore { * https://github.com/TryQuiet/quiet/issues/1899 */ protected async getCertificates() { - const filteredCertificatesMap: Map = new Map() - + const allCertificates = this.store .iterator({ limit: -1 }) .collect() .map(e => e.payload.value) - await Promise.all( - allCertificates - .filter(async certificate => { - const validation = await this.validateCertificate(certificate) - return Boolean(validation) - }).map(async certificate => { - const parsedCertificate = loadCertificate(certificate) - const pubKey = keyFromCertificate(parsedCertificate) - - if (filteredCertificatesMap.has(pubKey)) { - filteredCertificatesMap.delete(pubKey) - } - - filteredCertificatesMap.set(pubKey, certificate) - }) - ) - return [...filteredCertificatesMap.values()] + const validCertificates = allCertificates.map(async (certificate) => { + if (this.filteredCertificatesMapping.has(certificate)) { + return certificate // Only validate certificates + } + + const validation = await this.validateCertificate(certificate) + if (validation) { + + const parsedCertificate = parseCertificate(certificate) + const pubkey = keyFromCertificate(parsedCertificate) + + const username = getCertFieldValue(parsedCertificate, CertFieldsTypes.nickName) + + // @ts-expect-error + this.usernameMapping.set(pubkey, username) + + const data: Partial = { + // @ts-expect-error + username: username + } + + this.filteredCertificatesMapping.set(certificate, data) + + return certificate + } + }) + + return validCertificates.filter(i => Boolean(i)) // Filter out undefineds + } + + public getCertificateUsername(pubkey: string) { + return this.usernameMapping.get(pubkey) } } diff --git a/packages/backend/src/nest/storage/storage.service.ts b/packages/backend/src/nest/storage/storage.service.ts index 83b8916ba0..379b334613 100644 --- a/packages/backend/src/nest/storage/storage.service.ts +++ b/packages/backend/src/nest/storage/storage.service.ts @@ -6,9 +6,7 @@ import { keyObjectFromString, parseCertificate, verifySignature, - verifyUserCert, parseCertificationRequest, - getReqFieldValue, loadCSR, } from '@quiet/identity' import type { IPFS } from 'ipfs-core' @@ -18,7 +16,7 @@ import KeyValueStore from 'orbit-db-kvstore' import path from 'path' import { EventEmitter } from 'events' import PeerId from 'peer-id' -import { CertificationRequest, getCrypto } from 'pkijs' +import { getCrypto } from 'pkijs' import { stringToArrayBuffer } from 'pvutils' import validate from '../validation/validators' import { CID } from 'multiformats/cid' @@ -51,7 +49,7 @@ import Logger from '../common/logger' import { DirectMessagesRepo, PublicChannelsRepo } from '../common/types' import { removeFiles, removeDirs, createPaths, getUsersAddresses } from '../common/utils' import { StorageEvents } from './storage.types' -import { RegistrationEvents } from '../registration/registration.types' +import { CertificatesStore } from './certificates/certificates.store' interface DBOptions { replicate: boolean @@ -65,12 +63,11 @@ interface CsrReplicatedPromiseValues { @Injectable() export class StorageService extends EventEmitter { public channels: KeyValueStore - private certificatesRequests: EventStore public publicChannelsRepos: Map = new Map() public directMessagesRepos: Map = new Map() private publicKeysMap: Map = new Map() - private userNamesMap: Map = new Map() private communityMetadata: KeyValueStore + private certificatesStore: CertificatesStore private ipfs: IPFS private orbitDb: OrbitDB private filesManager: IpfsFileManagerService @@ -166,8 +163,8 @@ export class StorageService extends EventEmitter { if (this.channels?.address) { dbs.push(this.channels.address) } - if (this.certificatesRequests?.address) { - dbs.push(this.certificatesRequests.address) + if (this.certificatesStore.getAddress()) { + dbs.push(this.certificatesStore.getAddress()) } if (this.communityMetadata?.address) { dbs.push(this.communityMetadata.address) @@ -228,12 +225,14 @@ export class StorageService extends EventEmitter { this.logger('1/5') await this.createDbForChannels() this.logger('2/5') - // await this.attachCertificatesStoreListeners() + await this.attachCertificatesStoreListeners() this.logger('3/5') await this.createDbForCertificatesRequests() this.logger('4/5') await this.createDbForCommunityMetadata() this.logger('5/5') + this.certificatesStore = new CertificatesStore(this.orbitDb) + await this.certificatesStore.init(this) await this.initAllChannels() this.logger('Initialized DBs') this.emit(SocketActionTypes.CONNECTION_PROCESS_INFO, ConnectionProcessInfo.INITIALIZED_DBS) @@ -309,11 +308,11 @@ export class StorageService extends EventEmitter { this.logger.error('Error closing channels db', e) } - // try { - // await this.certificates?.close() - // } catch (e) { - // this.logger.error('Error closing certificates db', e) - // } + try { + await this.certificatesStore?.close() + } catch (e) { + this.logger.error('Error closing certificates db', e) + } try { await this.certificatesRequests?.close() @@ -331,66 +330,21 @@ export class StorageService extends EventEmitter { await this.__stopIPFS() } - // public async loadAllCertificates() { - // this.logger('Getting all certificates') - // this.emit(StorageEvents.LOAD_CERTIFICATES, { - // certificates: this.getAllEventLogEntries(this.certificates), - // }) - // } - - // public async createDbForCertificates() { - // this.logger('createDbForCertificates init') - // this.certificates = await this.orbitDb.log('certificates', { - // replicate: false, - // accessController: { - // write: ['*'], - // }, - // }) - // this.certificates.events.on('replicate.progress', async (_address, _hash, entry, _progress, _total) => { - // const certificate = entry.payload.value - - // const parsedCertificate = parseCertificate(certificate) - // const key = keyFromCertificate(parsedCertificate) - - // const username = getCertFieldValue(parsedCertificate, CertFieldsTypes.nickName) - // if (!username) { - // this.logger.error( - // `Certificates replicate.progress: could not parse certificate for field type ${CertFieldsTypes.nickName}` - // ) - // return - // } - - // this.userNamesMap.set(key, username) - // }) - // this.certificates.events.on('replicated', async () => { - // this.logger('REPLICATED: Certificates') - // this.emit(SocketActionTypes.CONNECTION_PROCESS_INFO, ConnectionProcessInfo.CERTIFICATES_REPLICATED) - // this.emit(StorageEvents.LOAD_CERTIFICATES, { - // certificates: this.getAllEventLogEntries(this.certificates), - // }) - // await this.updatePeersList() - // }) - // this.certificates.events.on('write', async (_address, entry) => { - // this.logger('Saved certificate locally') - // this.emit(StorageEvents.LOAD_CERTIFICATES, { - // certificates: this.getAllEventLogEntries(this.certificates), - // }) - // await this.updatePeersList() - // }) - // this.certificates.events.on('ready', () => { - // this.logger('Loaded certificates to memory') - // this.emit(SocketActionTypes.CONNECTION_PROCESS_INFO, ConnectionProcessInfo.LOADED_CERTIFICATES) - // this.emit(StorageEvents.LOAD_CERTIFICATES, { - // certificates: this.getAllEventLogEntries(this.certificates), - // }) - // }) - - // // @ts-expect-error - OrbitDB's type declaration of `load` lacks 'options' - // await this.certificates.load({ fetchEntryTimeout: 15000 }) - // const allCertificates = this.getAllEventLogEntries(this.certificates) - // this.logger('ALL Certificates COUNT:', allCertificates.length) - // this.logger('STORAGE: Finished createDbForCertificates') - // } + public async updatePeersList() {} + + public async loadAllCertificates() { + this.logger('Loading all certificates') + this.emit(StorageEvents.LOAD_CERTIFICATES, { + certificates: await this.certificatesStore.loadAllCertificates(), + }) + } + + public async attachCertificatesStoreListeners() { + this.on(StorageEvents.LOADED_CERTIFICATES, async (payload) => { + this.emit(StorageEvents.LOADED_CERTIFICATES, payload) + await this.updatePeersList() + }) + } private createCsrReplicatedPromise(id: number) { let resolveFunction @@ -635,7 +589,7 @@ export class StorageService extends EventEmitter { // @ts-ignore if (parseInt(message.createdAt) < parseInt(process.env.CONNECTION_TIME || '')) return - const username = this.getUserNameFromCert(message.pubKey) + const username = this.certificatesStore.getCertificateUsername(message.pubKey) if (!username) { this.logger.error(`Can't send push notification, no username found for public key '${message.pubKey}'`) return @@ -855,16 +809,16 @@ export class StorageService extends EventEmitter { this.filesManager.emit(IpfsFilesManagerEvents.CANCEL_DOWNLOAD, mid) } - // public async saveCertificate(payload: SaveCertificatePayload): Promise { - // this.logger('About to save certificate...') - // if (!payload.certificate) { - // this.logger('Certificate is either null or undefined, not saving to db') - // return false - // } - // this.logger('Saving certificate...') - // await this.certificates.add(payload.certificate) - // return true - // } + public async saveCertificate(payload: SaveCertificatePayload): Promise { + this.logger('About to save certificate...') + if (!payload.certificate) { + this.logger('Certificate is either null or undefined, not saving to db') + return false + } + this.logger('Saving certificate...') + const result = await this.certificatesStore.addCertificate(payload.certificate) + return result + } public async saveCSR(payload: SaveCSRPayload): Promise { this.logger('About to save csr...') @@ -891,58 +845,6 @@ export class StorageService extends EventEmitter { return true } - public getAllRegisteredUsers(): UserData[] { - const certs = this.getAllEventLogEntries(this.certificates) - const allUsers: UserData[] = [] - for (const cert of certs) { - const parsedCert = parseCertificate(cert) - const onionAddress = getCertFieldValue(parsedCert, CertFieldsTypes.commonName) - const peerId = getCertFieldValue(parsedCert, CertFieldsTypes.peerId) - const username = getCertFieldValue(parsedCert, CertFieldsTypes.nickName) - const dmPublicKey = getCertFieldValue(parsedCert, CertFieldsTypes.dmPublicKey) - if (!onionAddress || !peerId || !username || !dmPublicKey) continue - allUsers.push({ onionAddress, peerId, username, dmPublicKey }) - } - return allUsers - } - - public usernameCert(username: string): string | null { - /** - * Check if given username is already in use - */ - const certificates = this.getAllEventLogEntries(this.certificates) - for (const cert of certificates) { - const parsedCert = parseCertificate(cert) - const certUsername = getCertFieldValue(parsedCert, CertFieldsTypes.nickName) - if (certUsername?.localeCompare(username, 'en', { sensitivity: 'base' }) === 0) { - return cert - } - } - return null - } - - public getUserNameFromCert(publicKey: string): string | undefined { - if (!this.userNamesMap.get(publicKey)) { - const certificates = this.getAllEventLogEntries(this.certificates) - - for (const cert of certificates) { - const parsedCertificate = parseCertificate(cert) - const key = keyFromCertificate(parsedCertificate) - - const value = getCertFieldValue(parsedCertificate, CertFieldsTypes.nickName) - if (!value) { - this.logger.error( - `Get user name from cert: Could not parse certificate for field type ${CertFieldsTypes.nickName}` - ) - continue - } - this.userNamesMap.set(key, value) - } - } - - return this.userNamesMap.get(publicKey) - } - public async deleteFilesFromChannel(payload: DeleteFilesFromChannelSocketPayload) { const { messages } = payload Object.keys(messages).map(async key => { @@ -987,7 +889,6 @@ export class StorageService extends EventEmitter { this.publicChannelsRepos = new Map() this.directMessagesRepos = new Map() this.publicKeysMap = new Map() - this.userNamesMap = new Map() // @ts-ignore this.ipfs = null // @ts-ignore diff --git a/packages/backend/src/nest/storage/storage.types.ts b/packages/backend/src/nest/storage/storage.types.ts index 84b3286b5d..a3a6b0a93e 100644 --- a/packages/backend/src/nest/storage/storage.types.ts +++ b/packages/backend/src/nest/storage/storage.types.ts @@ -4,6 +4,7 @@ export enum StorageEvents { // Peers UPDATE_PEERS_LIST = 'updatePeersList', LOAD_CERTIFICATES = 'loadCertificates', + LOADED_CERTIFICATES = 'loadedCertificates', REPLICATED_CSR = 'replicatedCsr', // Public Channels LOAD_PUBLIC_CHANNELS = 'loadPublicChannels',