From f3b50c4fea98f91a3f9eccfdba9f58c967f9363b Mon Sep 17 00:00:00 2001 From: cgeek Date: Fri, 8 May 2020 11:03:00 +0200 Subject: [PATCH] [mod] e-mails plus explicites (titre + motif erreur) --- src/lib/mail.ts | 27 +++------ src/lib/types/state.ts | 27 ++++++--- src/lib/watcherLoop.ts | 8 +-- src/lib/watchers/abstract/url-watcher.ts | 56 +++++++++++++++---- src/lib/watchers/bma/bma-watcher.ts | 10 +++- src/lib/watchers/bma/head-watcher.ts | 21 ++++--- .../dprobe/dprobe-heartbeat-watcher.ts | 10 +++- src/lib/watchers/webdiff/webdiff-watcher.ts | 6 +- src/lib/watchers/wotwizard/json-watcher.ts | 12 ++-- src/lib/watchers/ws2p/ws2p-watcher.ts | 2 - 10 files changed, 110 insertions(+), 69 deletions(-) diff --git a/src/lib/mail.ts b/src/lib/mail.ts index 22c2fd2..db24e5d 100644 --- a/src/lib/mail.ts +++ b/src/lib/mail.ts @@ -6,6 +6,8 @@ import * as os from 'os' export async function sendMail(conf: ConfMail, subject: string, html: string, cc?: string) { + console.log(`[mail] Subject: ${subject}`) + console.log(`[mail] Body: ${html}`) if (conf.enabled) { let transporter = nodemailer.createTransport({ host: conf.host, @@ -38,23 +40,22 @@ export const mail = {

`) => { return async (cc?: string) => { - console.log(`${message}`) - await sendMail(conf.mail, `[dwatcher] [${os.hostname}] ${message}`, getHtml(), cc) + await sendMail(conf.mail, `[dw] [${os.hostname}] ${message}`, getHtml(), cc) } }, - onDisconnect: (conf: Conf, target: string, message = `Connection closed for ${target}`, getHtml: (waitingDelay: number) => string = (waitingDelay: number) => ` + onDisconnect: (conf: Conf, target: string, message = `Connection closed for ${target}`, getErrorMessage: () => string = () => '', getHtml: (waitingDelay: number) => string = (waitingDelay: number) => `

Connection from [${os.hostname}] to ${target} was lost on ${moment().format('dd-MM-YYYY HH:mm:ss')}.

Waiting ${(waitingDelay / 1000).toFixed(0)} seconds before trying to reconnect.

+ ${getErrorMessage()} `) => { return async (waitingDelay: number, cc?: string) => { - console.log(`${message}`) console.log('Waiting %s seconds...', (waitingDelay / 1000).toFixed(0)) - await sendMail(conf.mail, `[dwatcher] [${os.hostname}] ${message}`, getHtml(waitingDelay), cc) + await sendMail(conf.mail, `[dw] [${os.hostname}] ${message}`, getHtml(waitingDelay), cc) } }, @@ -65,21 +66,7 @@ export const mail = { `) => { return async (cc?: string) => { console.log(`${message}`) - await sendMail(conf.mail, `[dwatcher] [${os.hostname}] ${message}`, getHtml(), cc) + await sendMail(conf.mail, `[dw] [${os.hostname}] ${message}`, getHtml(), cc) } }, - - onError: (conf: Conf, target: string) => { - return async (e: Error) => { - console.error(e.message || e) - // await sendMail(conf.mail, `[dwatcher] Connection error`, ` - //

- // Connection from [${os.hostname}] error with ${target} on ${moment().format('dd-MM-YYYY HH:mm:ss')}: - //

- //

- // ${e.message} - //

- // `) - } - } } diff --git a/src/lib/types/state.ts b/src/lib/types/state.ts index 98b7762..94f4104 100644 --- a/src/lib/types/state.ts +++ b/src/lib/types/state.ts @@ -1,21 +1,32 @@ export class Watcher { - private state: 'INIT'|'OK'|'FAILURE'|'RECOVERED' - private failureMessage?: string + private _state: 'INIT'|'OK'|'FAILURE'|'RECOVERED' + private _error?: any constructor(public readonly name: string) { - this.state = "INIT" + this._state = "INIT" } public stateOK() { - this.state = "OK" + this._state = "OK" } public stateRecovered() { - this.state = "RECOVERED" + this._state = "RECOVERED" } - public stateFailure(error: string) { - this.state = "FAILURE" - this.failureMessage = error + public stateFailure(error: any) { + this._state = "FAILURE" + this._error = error + } + + public get error() { + return this._error + } + + public get failureMessage(): string|undefined { + if (!this._error) { + return undefined + } + return this._error.message || this._error } } diff --git a/src/lib/watcherLoop.ts b/src/lib/watcherLoop.ts index a2858aa..4e1c83a 100644 --- a/src/lib/watcherLoop.ts +++ b/src/lib/watcherLoop.ts @@ -6,10 +6,9 @@ export function watcherLoop( onConnectionClosed: () => Promise, reconnectionDelays: number[], onStart: () => Promise, - onDisconnection: (waitingDelay: number) => Promise, + onDisconnection: (waitingDelay: number, error: any) => Promise, onRestart: () => Promise, onRestartSuccess: () => Promise, - onError: (e: Error) => Promise, ): Watcher { let watcher: Watcher = new Watcher(name) @@ -44,12 +43,11 @@ export function watcherLoop( await onConnectionClosed() } catch (e) { - await onError(e) - watcher.stateFailure(e && e.message || e) + watcher.stateFailure(e) } // Wait before reconnecting const waitingDelay = reconnectionDelays[Math.min(reconnectionDelays.length - 1, i)] - await onDisconnection(waitingDelay) + await onDisconnection(waitingDelay, watcher.error) await new Promise(resolve => setTimeout(resolve, waitingDelay)) i++ } diff --git a/src/lib/watchers/abstract/url-watcher.ts b/src/lib/watchers/abstract/url-watcher.ts index c55cb70..0f449ce 100644 --- a/src/lib/watchers/abstract/url-watcher.ts +++ b/src/lib/watchers/abstract/url-watcher.ts @@ -4,23 +4,33 @@ import Axios from "axios"; import {mail} from "../../mail"; import {Watcher} from "../../types/state"; -export function urlWatcher(conf: Conf, checkValidity: (data: any) => Promise) { +export function urlWatcher(conf: Conf, checkValidity: (data: any) => Promise) { - return async (urlConf: ConfURL): Promise => { + return async (urlConf: ConfURL, + getOkTitle: () => string, + getKoTitle: () => string, + getRecoveredTitle: () => string): Promise => { let nodeDownRes: () => void let nodeDownPromise: Promise = new Promise(res => nodeDownRes = res) + async function checkResult(data: any) { + const validity = await checkValidity(data) + if (validity.error) { + throw new UrlWatcherError('Validity error') + } + } + return watcherLoop( urlConf.name, async () => { let interval: NodeJS.Timer; const res = await Axios.get(urlConf.address) - await checkValidity(res.data) + await checkResult(res.data) interval = setInterval(async () => { try { const res = await Axios.get(urlConf.address) - await checkValidity(res.data) + await checkResult(res.data) } catch (e) { if (interval) { clearInterval(interval) @@ -36,21 +46,45 @@ export function urlWatcher(conf: Conf, checkValidity: (data: any) => Promise { + let koTitle: string|undefined + let koMessage: (() => string)|undefined + if (error && error instanceof UrlWatcherError) { + koTitle = getKoTitle() + koMessage = () => `

${error.errorMessage}

` + } + return mail.onDisconnect(conf, urlConf.address, koTitle, koMessage)(waitingDelay) + }, async () => { console.log('Trying to connect to %s', urlConf.address) }, - mail.onRestartSuccess(conf, urlConf.address), - - async (e) => { - console.error(e.message || e) - } + mail.onRestartSuccess(conf, urlConf.address, getRecoveredTitle()), ) } } + +export class UrlWatcherError { + + constructor (public errorMessage: string) { + } + +} + +export class UrlWatcherResult { + error?: string + + public static ok() { + return {} + } + + public static ko(errorMessage: string) { + return { error: errorMessage } + } + +} \ No newline at end of file diff --git a/src/lib/watchers/bma/bma-watcher.ts b/src/lib/watchers/bma/bma-watcher.ts index 3bbf9af..c5bfa5d 100644 --- a/src/lib/watchers/bma/bma-watcher.ts +++ b/src/lib/watchers/bma/bma-watcher.ts @@ -1,5 +1,5 @@ import {Conf, ConfBMA} from "../../types/conf"; -import {urlWatcher} from '../abstract/url-watcher' +import {urlWatcher, UrlWatcherResult} from '../abstract/url-watcher' import {moment} from 'duniter/app/lib/common-libs/moment' export function bmaWatcher(conf: Conf) { @@ -11,12 +11,16 @@ export function bmaWatcher(conf: Conf) { return urlWatcher(conf, async (data) => { const block = data as { medianTime: number } if (bmaServer.maxLate && moment().unix() - block.medianTime > bmaServer.maxLate) { - throw 'Server is late' + return UrlWatcherResult.ko('Server is late') } + return UrlWatcherResult.ok() })({ name: `BMA ${bmaServer.address}`, address: bmaServer.address + URL_PATH, frequency: bmaServer.frequency - }) + }, + () => `[OK] BMA of ${bmaServer.address}`, + () => `[FAILURE] BMA of ${bmaServer.address}`, + () => `[RECOVERED] BMA of ${bmaServer.address}`) } } diff --git a/src/lib/watchers/bma/head-watcher.ts b/src/lib/watchers/bma/head-watcher.ts index 28bb774..5350354 100644 --- a/src/lib/watchers/bma/head-watcher.ts +++ b/src/lib/watchers/bma/head-watcher.ts @@ -1,27 +1,29 @@ import {Conf, ConfHead} from "../../types/conf"; -import {urlWatcher} from '../abstract/url-watcher' +import {urlWatcher, UrlWatcherResult} from '../abstract/url-watcher' import {WS2PHead} from "duniter/app/modules/ws2p/lib/WS2PCluster"; -function handleLateness(confHead: ConfHead, mainHeads: HeadMetric[], observedHead?: TrameWS2P) { +function handleLateness(confHead: ConfHead, mainHeads: HeadMetric[], observedHead?: TrameWS2P): UrlWatcherResult { if (!mainHeads.length) { - throw 'Observed pubkey ${confHead.observedPubkey}: no consensus found' + return UrlWatcherResult.ko('Observed pubkey ${confHead.observedPubkey}: no consensus found') } if (!observedHead) { - throw `Observed pubkey ${confHead.observedPubkey} not found in heads` + return UrlWatcherResult.ko(`Observed pubkey ${confHead.observedPubkey} not found in heads`) } const matchingHeads = mainHeads.filter(h => h.blockstamp === observedHead.blockstamp) if (!matchingHeads.length) { // Check how much late is the node const farAwayHeads = mainHeads.filter(h => h.blockNumber - observedHead.blockNumber >= confHead.maxLateBlocks) if (farAwayHeads.length) { - throw `Observed pubkey ${confHead.observedPubkey} is too late for ${farAwayHeads.length} consensus by at least ${confHead.maxLateBlocks}` + return UrlWatcherResult.ko(`Observed pubkey ${confHead.observedPubkey} is too late for ${farAwayHeads.length} consensus by at least ${confHead.maxLateBlocks}`) } } + return UrlWatcherResult.ok() } export function headWatcher(conf: Conf) { const URL_PATH = '/network/ws2p/heads' + const KEY_LENGTH = 10 return async (confHead: ConfHead) => { @@ -29,12 +31,15 @@ export function headWatcher(conf: Conf) { const heads = data as { heads: WS2PHead[] } const mainHeads = getMain(heads) const observedHead = getObserved(heads, confHead.observedPubkey) - handleLateness(confHead, mainHeads, observedHead) + return handleLateness(confHead, mainHeads, observedHead) })({ - name: `head watcher ${confHead.address} => ${confHead.observedPubkey.substr(0, 10)}`, + name: `head watcher ${confHead.address} => ${confHead.observedPubkey.substr(0, KEY_LENGTH)}`, address: confHead.address + URL_PATH, frequency: confHead.frequency - }) + }, + () => `[OK] HEADS of ${confHead.observedPubkey.substr(0, KEY_LENGTH)}`, + () => `[FAILURE] HEADS of ${confHead.observedPubkey.substr(0, KEY_LENGTH)}`, + () => `[RECOVERED] HEADS of ${confHead.observedPubkey.substr(0, KEY_LENGTH)}`) } } diff --git a/src/lib/watchers/dprobe/dprobe-heartbeat-watcher.ts b/src/lib/watchers/dprobe/dprobe-heartbeat-watcher.ts index 77f48a1..1dc6ee6 100644 --- a/src/lib/watchers/dprobe/dprobe-heartbeat-watcher.ts +++ b/src/lib/watchers/dprobe/dprobe-heartbeat-watcher.ts @@ -1,5 +1,5 @@ import {Conf, ConfDprobeHeartbeat} from "../../types/conf"; -import {urlWatcher} from '../abstract/url-watcher' +import {urlWatcher, UrlWatcherResult} from '../abstract/url-watcher' import {moment} from 'duniter/app/lib/common-libs/moment' export function dprobeHeartbeat(conf: Conf) { @@ -10,8 +10,12 @@ export function dprobeHeartbeat(conf: Conf) { const last = moment(data, 'YYYY-MM-DD HH:mm:ss\n') const past = moment().diff(last) if (past > dconf.lastBeat) { - throw 'Delay is over' + return UrlWatcherResult.ko('Delay is over') } - })(dconf) + return UrlWatcherResult.ok() + })(dconf, + () => `[OK] hearbeat is up-to-date`, + () => `[FAILURE] hearbeat`, + () => `[RECOVERED] hearbeat`) } } diff --git a/src/lib/watchers/webdiff/webdiff-watcher.ts b/src/lib/watchers/webdiff/webdiff-watcher.ts index 1c14bce..fea9383 100644 --- a/src/lib/watchers/webdiff/webdiff-watcher.ts +++ b/src/lib/watchers/webdiff/webdiff-watcher.ts @@ -66,7 +66,7 @@ export function webDiffWatcher(conf: Conf) { () => mail.onEstablished(conf, target, 'webdiff successfully started')(webDiffConf.cc), // When a disconnection is detected - (waitingDelay: number) => mail.onDisconnect(conf, target, 'Diff detected', (waitingDelay: number) => ` + (waitingDelay: number) => mail.onDisconnect(conf, target, 'Diff detected', undefined, (waitingDelay: number) => ` ${htmlDiff}

Waiting ${(waitingDelay / 1000).toFixed(0)} seconds before trying to reconnect. @@ -78,10 +78,6 @@ export function webDiffWatcher(conf: Conf) { }, () => mail.onRestartSuccess(conf, target)(webDiffConf.cc), - - async (e) => { - console.error(e.message) - } ) } diff --git a/src/lib/watchers/wotwizard/json-watcher.ts b/src/lib/watchers/wotwizard/json-watcher.ts index 83c6d63..b777289 100644 --- a/src/lib/watchers/wotwizard/json-watcher.ts +++ b/src/lib/watchers/wotwizard/json-watcher.ts @@ -1,11 +1,12 @@ import {Conf, ConfWWMeta} from "../../types/conf"; -import {urlWatcher} from '../abstract/url-watcher' +import {urlWatcher, UrlWatcherResult} from '../abstract/url-watcher' function handleLateness(confHead: ConfWWMeta, data: WWMetaJson) { const diff = Math.round(Date.now()/1000 - data.now) if (diff >= confHead.maxLate) { - throw `WWMeta.json is late by ${diff}s (>= ${confHead.maxLate})` + return UrlWatcherResult.ko(`WWMeta.json is late by ${diff}s (>= ${confHead.maxLate})`) } + return UrlWatcherResult.ok() } export function jsonWatcher(conf: Conf) { @@ -15,12 +16,15 @@ export function jsonWatcher(conf: Conf) { return async (confWWMeta: ConfWWMeta) => { return urlWatcher(conf, async (data) => { - handleLateness(confWWMeta, data) + return handleLateness(confWWMeta, data) })({ name: `WWMeta.json watcher ${confWWMeta.address}`, address: confWWMeta.address + URL_PATH, frequency: confWWMeta.frequency - }) + }, + () => `[OK] WWMeta.json is up-to-date`, + () => `[FAILURE] WWMeta.json`, + () => `[RECOVERED] WWMeta.json`) } } diff --git a/src/lib/watchers/ws2p/ws2p-watcher.ts b/src/lib/watchers/ws2p/ws2p-watcher.ts index a73e9dc..9908ebf 100644 --- a/src/lib/watchers/ws2p/ws2p-watcher.ts +++ b/src/lib/watchers/ws2p/ws2p-watcher.ts @@ -54,8 +54,6 @@ export function ws2pWatcher(conf: Conf) { }, mail.onRestartSuccess(conf, target), - - mail.onError(conf, target) ) }