From 9c123bc53375dbe7b7f33eefa6126fc3dfb79064 Mon Sep 17 00:00:00 2001 From: siepra Date: Wed, 15 Nov 2023 13:35:40 +0100 Subject: [PATCH] chore: start moving certificates to its own store --- .../registration/registration.validators.ts | 1 + .../certificates/certificates.store.spec.ts | 0 .../certificates/certificates.store.ts | 138 +++++++++++++ .../src/nest/storage/storage.service.ts | 185 ++++++++---------- 4 files changed, 219 insertions(+), 105 deletions(-) create mode 100644 packages/backend/src/nest/storage/certificates/certificates.store.spec.ts create mode 100644 packages/backend/src/nest/storage/certificates/certificates.store.ts diff --git a/packages/backend/src/nest/registration/registration.validators.ts b/packages/backend/src/nest/registration/registration.validators.ts index d821bfccf4..f699f4688a 100644 --- a/packages/backend/src/nest/registration/registration.validators.ts +++ b/packages/backend/src/nest/registration/registration.validators.ts @@ -3,6 +3,7 @@ import { registerDecorator, ValidationArguments, ValidationOptions } from 'class import Logger from '../common/logger' const logger = Logger('registration.validators') + export function IsCsr(validationOptions?: ValidationOptions) { return function (object: object, propertyName: string) { registerDecorator({ diff --git a/packages/backend/src/nest/storage/certificates/certificates.store.spec.ts b/packages/backend/src/nest/storage/certificates/certificates.store.spec.ts new file mode 100644 index 0000000000..e69de29bb2 diff --git a/packages/backend/src/nest/storage/certificates/certificates.store.ts b/packages/backend/src/nest/storage/certificates/certificates.store.ts new file mode 100644 index 0000000000..4ce24bec9b --- /dev/null +++ b/packages/backend/src/nest/storage/certificates/certificates.store.ts @@ -0,0 +1,138 @@ +import { getCrypto } from 'pkijs' + +import { EventEmitter } from 'events' +import { StorageEvents } from '../storage.types' + +import EventStore from 'orbit-db-eventstore' +import OrbitDB from 'orbit-db' + +import { loadCertificate, keyFromCertificate } from '@quiet/identity' + +import { ConnectionProcessInfo, NoCryptoEngineError, SocketActionTypes } from '@quiet/types' + +import { IsNotEmpty, IsBase64, validate } from 'class-validator' +import { ValidationError } from '@nestjs/common' + +import createLogger from '../../common/logger' + +const logger = createLogger('CertificatesStore') + +class UserCertificateData { + @IsNotEmpty() + @IsBase64() + certificate: string +} + +export class CertificatesStore { + public orbitDb: OrbitDB + public store: EventStore + + constructor(orbitDb: OrbitDB) { + this.orbitDb = orbitDb + } + + public async init(emitter: EventEmitter) { + logger('Initializing certificates log store') + + this.store = await this.orbitDb.log('certificates', { + replicate: false, + accessController: { + write: ['*'], + }, + }) + + this.store.events.on('write', async (_address, entry) => { + logger('Saved certificate locally') + + emitter.emit(StorageEvents.LOAD_CERTIFICATES, { + certificates: await this.getCertificates(), + }) + + // await this.updatePeersList() + }) + + this.store.events.on('ready', async () => { + logger('Loaded certificates to memory') + + emitter.emit(SocketActionTypes.CONNECTION_PROCESS_INFO, ConnectionProcessInfo.LOADED_CERTIFICATES) + + emitter.emit(StorageEvents.LOAD_CERTIFICATES, { + certificates: await this.getCertificates(), + }) + }) + + this.store.events.on('replicated', async () => { + logger('REPLICATED: Certificates') + + emitter.emit(SocketActionTypes.CONNECTION_PROCESS_INFO, ConnectionProcessInfo.CERTIFICATES_REPLICATED) + + emitter.emit(StorageEvents.LOAD_CERTIFICATES, { + certificates: await this.getCertificates(), + }) + + // await this.updatePeersList() + }) + + } + + private async validateCertificate(certificate: string): Promise { + logger('Validating certificate') + try { + const crypto = getCrypto() + + if (!crypto) { + throw new NoCryptoEngineError() + } + + const parsedCertificate = loadCertificate(certificate) + await parsedCertificate.verify() + + await this.validateCertificateFormat(certificate) + + // Validate + + } catch (err) { + logger.error('Failed to validate user certificate:', certificate, err?.message) + return false + } + + return true + } + + private async validateCertificateFormat(certificate: string): Promise { + const data = new UserCertificateData() + data.certificate = certificate + + const validationErrors = await validate(data) + + return validationErrors + } + + protected async getCertificates() { + const filteredCertificatesMap: Map = new Map() + + const allCertificates = this.store + .iterator({ limit: -1 }) + .collect() + .map(e => e.payload.value) + + await Promise.all( + allCertificates + .filter(async certificate => { + const validation = await this.validateCertificate(certificate) + return Boolean(validation) + }).map(async certificate => { + const parsedCertificate = loadCertificate(certificate) + const pubKey = keyFromCertificate(parsedCertificate) + + if (filteredCertificatesMap.has(pubKey)) { + filteredCertificatesMap.delete(pubKey) + } + + filteredCertificatesMap.set(pubKey, certificate) + }) + ) + return [...filteredCertificatesMap.values()] + } + +} diff --git a/packages/backend/src/nest/storage/storage.service.ts b/packages/backend/src/nest/storage/storage.service.ts index fdef474a25..83b8916ba0 100644 --- a/packages/backend/src/nest/storage/storage.service.ts +++ b/packages/backend/src/nest/storage/storage.service.ts @@ -65,7 +65,6 @@ interface CsrReplicatedPromiseValues { @Injectable() export class StorageService extends EventEmitter { public channels: KeyValueStore - private certificates: EventStore private certificatesRequests: EventStore public publicChannelsRepos: Map = new Map() public directMessagesRepos: Map = new Map() @@ -167,9 +166,6 @@ export class StorageService extends EventEmitter { if (this.channels?.address) { dbs.push(this.channels.address) } - if (this.certificates?.address) { - dbs.push(this.certificates.address) - } if (this.certificatesRequests?.address) { dbs.push(this.certificatesRequests.address) } @@ -232,7 +228,7 @@ export class StorageService extends EventEmitter { this.logger('1/5') await this.createDbForChannels() this.logger('2/5') - await this.createDbForCertificates() + // await this.attachCertificatesStoreListeners() this.logger('3/5') await this.createDbForCertificatesRequests() this.logger('4/5') @@ -313,94 +309,88 @@ export class StorageService extends EventEmitter { this.logger.error('Error closing channels db', e) } - try { - await this.certificates?.close() - } catch (e) { - this.logger.error('Error closing certificates db', e) - } + // try { + // await this.certificates?.close() + // } catch (e) { + // this.logger.error('Error closing certificates db', e) + // } + try { await this.certificatesRequests?.close() } catch (e) { this.logger.error('Error closing certificates db', e) } + try { await this.communityMetadata?.close() } catch (e) { this.logger.error('Error closing community metadata db', e) } + await this.__stopOrbitDb() await this.__stopIPFS() } - public async updatePeersList() { - const allUsers = this.getAllUsers() - const registeredUsers = this.getAllRegisteredUsers() - const peers = [...new Set(await getUsersAddresses(allUsers.concat(registeredUsers)))] - console.log('updatePeersList, peers count:', peers.length) - const community = await this.localDbService.get(LocalDBKeys.COMMUNITY) - this.emit(StorageEvents.UPDATE_PEERS_LIST, { communityId: community.id, peerList: peers }) - } - - public async loadAllCertificates() { - this.logger('Getting all certificates') - this.emit(StorageEvents.LOAD_CERTIFICATES, { - certificates: this.getAllEventLogEntries(this.certificates), - }) - } - - public async createDbForCertificates() { - this.logger('createDbForCertificates init') - this.certificates = await this.orbitDb.log('certificates', { - replicate: false, - accessController: { - write: ['*'], - }, - }) - this.certificates.events.on('replicate.progress', async (_address, _hash, entry, _progress, _total) => { - const certificate = entry.payload.value - - const parsedCertificate = parseCertificate(certificate) - const key = keyFromCertificate(parsedCertificate) - - const username = getCertFieldValue(parsedCertificate, CertFieldsTypes.nickName) - if (!username) { - this.logger.error( - `Certificates replicate.progress: could not parse certificate for field type ${CertFieldsTypes.nickName}` - ) - return - } - - this.userNamesMap.set(key, username) - }) - this.certificates.events.on('replicated', async () => { - this.logger('REPLICATED: Certificates') - this.emit(SocketActionTypes.CONNECTION_PROCESS_INFO, ConnectionProcessInfo.CERTIFICATES_REPLICATED) - this.emit(StorageEvents.LOAD_CERTIFICATES, { - certificates: this.getAllEventLogEntries(this.certificates), - }) - await this.updatePeersList() - }) - this.certificates.events.on('write', async (_address, entry) => { - this.logger('Saved certificate locally') - this.emit(StorageEvents.LOAD_CERTIFICATES, { - certificates: this.getAllEventLogEntries(this.certificates), - }) - await this.updatePeersList() - }) - this.certificates.events.on('ready', () => { - this.logger('Loaded certificates to memory') - this.emit(SocketActionTypes.CONNECTION_PROCESS_INFO, ConnectionProcessInfo.LOADED_CERTIFICATES) - this.emit(StorageEvents.LOAD_CERTIFICATES, { - certificates: this.getAllEventLogEntries(this.certificates), - }) - }) - - // @ts-expect-error - OrbitDB's type declaration of `load` lacks 'options' - await this.certificates.load({ fetchEntryTimeout: 15000 }) - const allCertificates = this.getAllEventLogEntries(this.certificates) - this.logger('ALL Certificates COUNT:', allCertificates.length) - this.logger('STORAGE: Finished createDbForCertificates') - } + // public async loadAllCertificates() { + // this.logger('Getting all certificates') + // this.emit(StorageEvents.LOAD_CERTIFICATES, { + // certificates: this.getAllEventLogEntries(this.certificates), + // }) + // } + + // public async createDbForCertificates() { + // this.logger('createDbForCertificates init') + // this.certificates = await this.orbitDb.log('certificates', { + // replicate: false, + // accessController: { + // write: ['*'], + // }, + // }) + // this.certificates.events.on('replicate.progress', async (_address, _hash, entry, _progress, _total) => { + // const certificate = entry.payload.value + + // const parsedCertificate = parseCertificate(certificate) + // const key = keyFromCertificate(parsedCertificate) + + // const username = getCertFieldValue(parsedCertificate, CertFieldsTypes.nickName) + // if (!username) { + // this.logger.error( + // `Certificates replicate.progress: could not parse certificate for field type ${CertFieldsTypes.nickName}` + // ) + // return + // } + + // this.userNamesMap.set(key, username) + // }) + // this.certificates.events.on('replicated', async () => { + // this.logger('REPLICATED: Certificates') + // this.emit(SocketActionTypes.CONNECTION_PROCESS_INFO, ConnectionProcessInfo.CERTIFICATES_REPLICATED) + // this.emit(StorageEvents.LOAD_CERTIFICATES, { + // certificates: this.getAllEventLogEntries(this.certificates), + // }) + // await this.updatePeersList() + // }) + // this.certificates.events.on('write', async (_address, entry) => { + // this.logger('Saved certificate locally') + // this.emit(StorageEvents.LOAD_CERTIFICATES, { + // certificates: this.getAllEventLogEntries(this.certificates), + // }) + // await this.updatePeersList() + // }) + // this.certificates.events.on('ready', () => { + // this.logger('Loaded certificates to memory') + // this.emit(SocketActionTypes.CONNECTION_PROCESS_INFO, ConnectionProcessInfo.LOADED_CERTIFICATES) + // this.emit(StorageEvents.LOAD_CERTIFICATES, { + // certificates: this.getAllEventLogEntries(this.certificates), + // }) + // }) + + // // @ts-expect-error - OrbitDB's type declaration of `load` lacks 'options' + // await this.certificates.load({ fetchEntryTimeout: 15000 }) + // const allCertificates = this.getAllEventLogEntries(this.certificates) + // this.logger('ALL Certificates COUNT:', allCertificates.length) + // this.logger('STORAGE: Finished createDbForCertificates') + // } private createCsrReplicatedPromise(id: number) { let resolveFunction @@ -459,6 +449,7 @@ export class StorageService extends EventEmitter { await this.updatePeersList() }) + this.certificatesRequests.events.on('write', async (_address, entry) => { const csr: string = entry.payload.value this.logger('Saved CSR locally') @@ -864,16 +855,16 @@ export class StorageService extends EventEmitter { this.filesManager.emit(IpfsFilesManagerEvents.CANCEL_DOWNLOAD, mid) } - public async saveCertificate(payload: SaveCertificatePayload): Promise { - this.logger('About to save certificate...') - if (!payload.certificate) { - this.logger('Certificate is either null or undefined, not saving to db') - return false - } - this.logger('Saving certificate...') - await this.certificates.add(payload.certificate) - return true - } + // public async saveCertificate(payload: SaveCertificatePayload): Promise { + // this.logger('About to save certificate...') + // if (!payload.certificate) { + // this.logger('Certificate is either null or undefined, not saving to db') + // return false + // } + // this.logger('Saving certificate...') + // await this.certificates.add(payload.certificate) + // return true + // } public async saveCSR(payload: SaveCSRPayload): Promise { this.logger('About to save csr...') @@ -915,22 +906,6 @@ export class StorageService extends EventEmitter { return allUsers } - public getAllUsers(): UserData[] { - const csrs = this.getAllEventLogEntries(this.certificatesRequests) - this.logger('CSRs count:', csrs.length) - const allUsers: UserData[] = [] - for (const csr of csrs) { - const parsedCert = parseCertificationRequest(csr) - const onionAddress = getReqFieldValue(parsedCert, CertFieldsTypes.commonName) - const peerId = getReqFieldValue(parsedCert, CertFieldsTypes.peerId) - const username = getReqFieldValue(parsedCert, CertFieldsTypes.nickName) - const dmPublicKey = getReqFieldValue(parsedCert, CertFieldsTypes.dmPublicKey) - if (!onionAddress || !peerId || !username || !dmPublicKey) continue - allUsers.push({ onionAddress, peerId, username, dmPublicKey }) - } - return allUsers - } - public usernameCert(username: string): string | null { /** * Check if given username is already in use