cgeek/dwatcher
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[mod] e-mails plus explicites (titre + motif erreur)

Browse Source
This commit is contained in:
cgeek
2020-05-08 11:03:00 +02:00
parent 2be3471067
commit f3b50c4fea
10 changed files with 110 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
src/lib/mail.ts
View File

@@ -6,6 +6,8 @@ import * as os from 'os'
export async function sendMail(conf: ConfMail, subject: string, html: string, cc?: string) {
console.log(`[mail] Subject: ${subject}`)
console.log(`[mail] Body: ${html}`)
if (conf.enabled) {
let transporter = nodemailer.createTransport({
host: conf.host,
@@ -38,23 +40,22 @@ export const mail = {
</p>
`) => {
return async (cc?: string) => {
console.log(`${message}`)
await sendMail(conf.mail, `[dwatcher] [${os.hostname}] ${message}`, getHtml(), cc)
await sendMail(conf.mail, `[dw] [${os.hostname}] ${message}`, getHtml(), cc)
}
},
onDisconnect: (conf: Conf, target: string, message = `Connection closed for ${target}`, getHtml: (waitingDelay: number) => string = (waitingDelay: number) => `
onDisconnect: (conf: Conf, target: string, message = `Connection closed for ${target}`, getErrorMessage: () => string = () => '', getHtml: (waitingDelay: number) => string = (waitingDelay: number) => `
<p>
Connection from [${os.hostname}] to ${target} was lost on ${moment().format('dd-MM-YYYY HH:mm:ss')}.
</p>
<p>
Waiting ${(waitingDelay / 1000).toFixed(0)} seconds before trying to reconnect.
</p>
${getErrorMessage()}
`) => {
return async (waitingDelay: number, cc?: string) => {
console.log(`${message}`)
console.log('Waiting %s seconds...', (waitingDelay / 1000).toFixed(0))
await sendMail(conf.mail, `[dwatcher] [${os.hostname}] ${message}`, getHtml(waitingDelay), cc)
await sendMail(conf.mail, `[dw] [${os.hostname}] ${message}`, getHtml(waitingDelay), cc)
}
},
@@ -65,21 +66,7 @@ export const mail = {
`) => {
return async (cc?: string) => {
console.log(`${message}`)
await sendMail(conf.mail, `[dwatcher] [${os.hostname}] ${message}`, getHtml(), cc)
await sendMail(conf.mail, `[dw] [${os.hostname}] ${message}`, getHtml(), cc)
}
},
onError: (conf: Conf, target: string) => {
return async (e: Error) => {
console.error(e.message || e)
// await sendMail(conf.mail, `[dwatcher] Connection error`, `
// <p>
// Connection from [${os.hostname}] error with ${target} on ${moment().format('dd-MM-YYYY HH:mm:ss')}:
// </p>
// <p>
// ${e.message}
// </p>
// `)
}
}
}

27
src/lib/types/state.ts
View File

@@ -1,21 +1,32 @@
export class Watcher {
private state: 'INIT'|'OK'|'FAILURE'|'RECOVERED'
private failureMessage?: string
private _state: 'INIT'|'OK'|'FAILURE'|'RECOVERED'
private _error?: any
constructor(public readonly name: string) {
this.state = "INIT"
this._state = "INIT"
}
public stateOK() {
this.state = "OK"
this._state = "OK"
}
public stateRecovered() {
this.state = "RECOVERED"
this._state = "RECOVERED"
}
public stateFailure(error: string) {
this.state = "FAILURE"
this.failureMessage = error
public stateFailure(error: any) {
this._state = "FAILURE"
this._error = error
}
public get error() {
return this._error
}
public get failureMessage(): string|undefined {
if (!this._error) {
return undefined
}
return this._error.message || this._error
}
}

8
src/lib/watcherLoop.ts
View File

@@ -6,10 +6,9 @@ export function watcherLoop(
onConnectionClosed: () => Promise<void>,
reconnectionDelays: number[],
onStart: () => Promise<void>,
onDisconnection: (waitingDelay: number) => Promise<void>,
onDisconnection: (waitingDelay: number, error: any) => Promise<void>,
onRestart: () => Promise<void>,
onRestartSuccess: () => Promise<void>,
onError: (e: Error) => Promise<void>,
): Watcher {
let watcher: Watcher = new Watcher(name)
@@ -44,12 +43,11 @@ export function watcherLoop(
await onConnectionClosed()
} catch (e) {
await onError(e)
watcher.stateFailure(e && e.message || e)
watcher.stateFailure(e)
}
// Wait before reconnecting
const waitingDelay = reconnectionDelays[Math.min(reconnectionDelays.length - 1, i)]
await onDisconnection(waitingDelay)
await onDisconnection(waitingDelay, watcher.error)
await new Promise(resolve => setTimeout(resolve, waitingDelay))
i++
}

56
src/lib/watchers/abstract/url-watcher.ts
View File

@@ -4,23 +4,33 @@ import Axios from "axios";
import {mail} from "../../mail";
import {Watcher} from "../../types/state";
export function urlWatcher(conf: Conf, checkValidity: (data: any) => Promise<void>) {
export function urlWatcher(conf: Conf, checkValidity: (data: any) => Promise<UrlWatcherResult>) {
return async (urlConf: ConfURL): Promise<Watcher> => {
return async (urlConf: ConfURL,
getOkTitle: () => string,
getKoTitle: () => string,
getRecoveredTitle: () => string): Promise<Watcher> => {
let nodeDownRes: () => void
let nodeDownPromise: Promise<void> = new Promise(res => nodeDownRes = res)
async function checkResult(data: any) {
const validity = await checkValidity(data)
if (validity.error) {
throw new UrlWatcherError('Validity error')
}
}
return watcherLoop(
urlConf.name,
async () => {
let interval: NodeJS.Timer;
const res = await Axios.get(urlConf.address)
await checkValidity(res.data)
await checkResult(res.data)
interval = setInterval(async () => {
try {
const res = await Axios.get(urlConf.address)
await checkValidity(res.data)
await checkResult(res.data)
} catch (e) {
if (interval) {
clearInterval(interval)
@@ -36,21 +46,45 @@ export function urlWatcher(conf: Conf, checkValidity: (data: any) => Promise<voi
conf.reconnectionDelays,
mail.onEstablished(conf, urlConf.address),
mail.onEstablished(conf, urlConf.address, getOkTitle()),
// When a disconnection is detected
mail.onDisconnect(conf, urlConf.address),
(waitingDelay: number, error?: any) => {
let koTitle: string|undefined
let koMessage: (() => string)|undefined
if (error && error instanceof UrlWatcherError) {
koTitle = getKoTitle()
koMessage = () => `<p>${error.errorMessage}</p>`
}
return mail.onDisconnect(conf, urlConf.address, koTitle, koMessage)(waitingDelay)
},
async () => {
console.log('Trying to connect to %s', urlConf.address)
},
mail.onRestartSuccess(conf, urlConf.address),
async (e) => {
console.error(e.message || e)
}
mail.onRestartSuccess(conf, urlConf.address, getRecoveredTitle()),
)
}
}
export class UrlWatcherError {
constructor (public errorMessage: string) {
}
}
export class UrlWatcherResult {
error?: string
public static ok() {
return {}
}
public static ko(errorMessage: string) {
return { error: errorMessage }
}
}

10
src/lib/watchers/bma/bma-watcher.ts
View File

@@ -1,5 +1,5 @@
import {Conf, ConfBMA} from "../../types/conf";
import {urlWatcher} from '../abstract/url-watcher'
import {urlWatcher, UrlWatcherResult} from '../abstract/url-watcher'
import {moment} from 'duniter/app/lib/common-libs/moment'
export function bmaWatcher(conf: Conf) {
@@ -11,12 +11,16 @@ export function bmaWatcher(conf: Conf) {
return urlWatcher(conf, async (data) => {
const block = data as { medianTime: number }
if (bmaServer.maxLate && moment().unix() - block.medianTime > bmaServer.maxLate) {
throw 'Server is late'
return UrlWatcherResult.ko('Server is late')
}
return UrlWatcherResult.ok()
})({
name: `BMA ${bmaServer.address}`,
address: bmaServer.address + URL_PATH,
frequency: bmaServer.frequency
})
},
() => `[OK] BMA of ${bmaServer.address}`,
() => `[FAILURE] BMA of ${bmaServer.address}`,
() => `[RECOVERED] BMA of ${bmaServer.address}`)
}
}

21
src/lib/watchers/bma/head-watcher.ts
View File

@@ -1,27 +1,29 @@
import {Conf, ConfHead} from "../../types/conf";
import {urlWatcher} from '../abstract/url-watcher'
import {urlWatcher, UrlWatcherResult} from '../abstract/url-watcher'
import {WS2PHead} from "duniter/app/modules/ws2p/lib/WS2PCluster";
function handleLateness(confHead: ConfHead, mainHeads: HeadMetric[], observedHead?: TrameWS2P) {
function handleLateness(confHead: ConfHead, mainHeads: HeadMetric[], observedHead?: TrameWS2P): UrlWatcherResult {
if (!mainHeads.length) {
throw 'Observed pubkey ${confHead.observedPubkey}: no consensus found'
return UrlWatcherResult.ko('Observed pubkey ${confHead.observedPubkey}: no consensus found')
}
if (!observedHead) {
throw `Observed pubkey ${confHead.observedPubkey} not found in heads`
return UrlWatcherResult.ko(`Observed pubkey ${confHead.observedPubkey} not found in heads`)
}
const matchingHeads = mainHeads.filter(h => h.blockstamp === observedHead.blockstamp)
if (!matchingHeads.length) {
// Check how much late is the node
const farAwayHeads = mainHeads.filter(h => h.blockNumber - observedHead.blockNumber >= confHead.maxLateBlocks)
if (farAwayHeads.length) {
throw `Observed pubkey ${confHead.observedPubkey} is too late for ${farAwayHeads.length} consensus by at least ${confHead.maxLateBlocks}`
return UrlWatcherResult.ko(`Observed pubkey ${confHead.observedPubkey} is too late for ${farAwayHeads.length} consensus by at least ${confHead.maxLateBlocks}`)
}
}
return UrlWatcherResult.ok()
}
export function headWatcher(conf: Conf) {
const URL_PATH = '/network/ws2p/heads'
const KEY_LENGTH = 10
return async (confHead: ConfHead) => {
@@ -29,12 +31,15 @@ export function headWatcher(conf: Conf) {
const heads = data as { heads: WS2PHead[] }
const mainHeads = getMain(heads)
const observedHead = getObserved(heads, confHead.observedPubkey)
handleLateness(confHead, mainHeads, observedHead)
return handleLateness(confHead, mainHeads, observedHead)
})({
name: `head watcher ${confHead.address} => ${confHead.observedPubkey.substr(0, 10)}`,
name: `head watcher ${confHead.address} => ${confHead.observedPubkey.substr(0, KEY_LENGTH)}`,
address: confHead.address + URL_PATH,
frequency: confHead.frequency
})
},
() => `[OK] HEADS of ${confHead.observedPubkey.substr(0, KEY_LENGTH)}`,
() => `[FAILURE] HEADS of ${confHead.observedPubkey.substr(0, KEY_LENGTH)}`,
() => `[RECOVERED] HEADS of ${confHead.observedPubkey.substr(0, KEY_LENGTH)}`)
}
}

10
src/lib/watchers/dprobe/dprobe-heartbeat-watcher.ts
View File

@@ -1,5 +1,5 @@
import {Conf, ConfDprobeHeartbeat} from "../../types/conf";
import {urlWatcher} from '../abstract/url-watcher'
import {urlWatcher, UrlWatcherResult} from '../abstract/url-watcher'
import {moment} from 'duniter/app/lib/common-libs/moment'
export function dprobeHeartbeat(conf: Conf) {
@@ -10,8 +10,12 @@ export function dprobeHeartbeat(conf: Conf) {
const last = moment(data, 'YYYY-MM-DD HH:mm:ss\n')
const past = moment().diff(last)
if (past > dconf.lastBeat) {
throw 'Delay is over'
return UrlWatcherResult.ko('Delay is over')
}
})(dconf)
return UrlWatcherResult.ok()
})(dconf,
() => `[OK] hearbeat is up-to-date`,
() => `[FAILURE] hearbeat`,
() => `[RECOVERED] hearbeat`)
}
}

6
src/lib/watchers/webdiff/webdiff-watcher.ts
View File

@@ -66,7 +66,7 @@ export function webDiffWatcher(conf: Conf) {
() => mail.onEstablished(conf, target, 'webdiff successfully started')(webDiffConf.cc),
// When a disconnection is detected
(waitingDelay: number) => mail.onDisconnect(conf, target, 'Diff detected', (waitingDelay: number) => `
(waitingDelay: number) => mail.onDisconnect(conf, target, 'Diff detected', undefined, (waitingDelay: number) => `
${htmlDiff}
<p>
Waiting ${(waitingDelay / 1000).toFixed(0)} seconds before trying to reconnect.
@@ -78,10 +78,6 @@ export function webDiffWatcher(conf: Conf) {
},
() => mail.onRestartSuccess(conf, target)(webDiffConf.cc),
async (e) => {
console.error(e.message)
}
)
}

12
src/lib/watchers/wotwizard/json-watcher.ts
View File

@@ -1,11 +1,12 @@
import {Conf, ConfWWMeta} from "../../types/conf";
import {urlWatcher} from '../abstract/url-watcher'
import {urlWatcher, UrlWatcherResult} from '../abstract/url-watcher'
function handleLateness(confHead: ConfWWMeta, data: WWMetaJson) {
const diff = Math.round(Date.now()/1000 - data.now)
if (diff >= confHead.maxLate) {
throw `WWMeta.json is late by ${diff}s (>= ${confHead.maxLate})`
return UrlWatcherResult.ko(`WWMeta.json is late by ${diff}s (>= ${confHead.maxLate})`)
}
return UrlWatcherResult.ok()
}
export function jsonWatcher(conf: Conf) {
@@ -15,12 +16,15 @@ export function jsonWatcher(conf: Conf) {
return async (confWWMeta: ConfWWMeta) => {
return urlWatcher(conf, async (data) => {
handleLateness(confWWMeta, data)
return handleLateness(confWWMeta, data)
})({
name: `WWMeta.json watcher ${confWWMeta.address}`,
address: confWWMeta.address + URL_PATH,
frequency: confWWMeta.frequency
})
},
() => `[OK] WWMeta.json is up-to-date`,
() => `[FAILURE] WWMeta.json`,
() => `[RECOVERED] WWMeta.json`)
}
}

2
src/lib/watchers/ws2p/ws2p-watcher.ts
View File

@@ -54,8 +54,6 @@ export function ws2pWatcher(conf: Conf) {
},
mail.onRestartSuccess(conf, target),
mail.onError(conf, target)
)
}