[mod] e-mails plus explicites (titre + motif erreur)

src/lib/mail.ts

@@ -6,6 +6,8 @@ import * as os from 'os'
 
 export async function sendMail(conf: ConfMail, subject: string, html: string, cc?: string) {
 
+  console.log(`[mail] Subject: ${subject}`)
+  console.log(`[mail] Body: ${html}`)
   if (conf.enabled) {
     let transporter = nodemailer.createTransport({
       host: conf.host,
@@ -38,23 +40,22 @@ export const mail = {
       </p>
     `) => {
     return async (cc?: string) => {
-      console.log(`${message}`)
+      await sendMail(conf.mail, `[dw] [${os.hostname}] ${message}`, getHtml(), cc)
-      await sendMail(conf.mail, `[dwatcher] [${os.hostname}] ${message}`, getHtml(), cc)
     }
   },
 
-  onDisconnect: (conf: Conf, target: string, message = `Connection closed for ${target}`, getHtml: (waitingDelay: number) => string = (waitingDelay: number) => `
+  onDisconnect: (conf: Conf, target: string, message = `Connection closed for ${target}`, getErrorMessage: () => string = () => '', getHtml: (waitingDelay: number) => string = (waitingDelay: number) => `
       <p>
         Connection from [${os.hostname}] to ${target} was lost on ${moment().format('dd-MM-YYYY HH:mm:ss')}.
       </p>
       <p>
         Waiting ${(waitingDelay / 1000).toFixed(0)} seconds before trying to reconnect.
       </p>
+      ${getErrorMessage()}
   `) => {
     return async (waitingDelay: number, cc?: string) => {
-      console.log(`${message}`)
       console.log('Waiting %s seconds...', (waitingDelay / 1000).toFixed(0))
-      await sendMail(conf.mail, `[dwatcher] [${os.hostname}] ${message}`, getHtml(waitingDelay), cc)
+      await sendMail(conf.mail, `[dw] [${os.hostname}] ${message}`, getHtml(waitingDelay), cc)
     }
   },
 
@@ -65,21 +66,7 @@ export const mail = {
     `) => {
     return async (cc?: string) => {
       console.log(`${message}`)
-      await sendMail(conf.mail, `[dwatcher] [${os.hostname}] ${message}`, getHtml(), cc)
+      await sendMail(conf.mail, `[dw] [${os.hostname}] ${message}`, getHtml(), cc)
     }
   },
-
-  onError: (conf: Conf, target: string) => {
-    return async (e: Error) => {
-      console.error(e.message || e)
-    //   await sendMail(conf.mail, `[dwatcher] Connection error`, `
-    //   <p>
-    //     Connection from [${os.hostname}] error with ${target} on ${moment().format('dd-MM-YYYY HH:mm:ss')}:
-    //   </p>
-    //   <p>
-    //     ${e.message}
-    //   </p>
-    // `)
-    }
-  }
 }

src/lib/types/state.ts

@@ -1,21 +1,32 @@
 export class Watcher {
-    private state: 'INIT'|'OK'|'FAILURE'|'RECOVERED'
+    private _state: 'INIT'|'OK'|'FAILURE'|'RECOVERED'
-    private failureMessage?: string
+    private _error?: any
 
     constructor(public readonly name: string) {
-        this.state = "INIT"
+        this._state = "INIT"
     }
 
     public stateOK() {
-        this.state = "OK"
+        this._state = "OK"
     }
 
     public stateRecovered() {
-        this.state = "RECOVERED"
+        this._state = "RECOVERED"
     }
 
-    public stateFailure(error: string) {
+    public stateFailure(error: any) {
-        this.state = "FAILURE"
+        this._state = "FAILURE"
-        this.failureMessage = error
+        this._error = error
+    }
+
+    public get error() {
+        return this._error
+    }
+
+    public get failureMessage(): string|undefined {
+        if (!this._error) {
+            return undefined
+        }
+        return this._error.message || this._error
     }
 }

src/lib/watcherLoop.ts

@@ -6,10 +6,9 @@ export function watcherLoop(
   onConnectionClosed: () => Promise<void>,
   reconnectionDelays: number[],
   onStart: () => Promise<void>,
-  onDisconnection: (waitingDelay: number) => Promise<void>,
+  onDisconnection: (waitingDelay: number, error: any) => Promise<void>,
   onRestart: () => Promise<void>,
   onRestartSuccess: () => Promise<void>,
-  onError: (e: Error) => Promise<void>,
 ): Watcher {
 
   let watcher: Watcher = new Watcher(name)
@@ -44,12 +43,11 @@ export function watcherLoop(
         await onConnectionClosed()
 
       } catch (e) {
-        await onError(e)
+        watcher.stateFailure(e)
-        watcher.stateFailure(e && e.message || e)
       }
       // Wait before reconnecting
       const waitingDelay = reconnectionDelays[Math.min(reconnectionDelays.length - 1, i)]
-      await onDisconnection(waitingDelay)
+      await onDisconnection(waitingDelay, watcher.error)
       await new Promise(resolve => setTimeout(resolve, waitingDelay))
       i++
     }

src/lib/watchers/abstract/url-watcher.ts

@@ -4,23 +4,33 @@ import Axios from "axios";
 import {mail} from "../../mail";
 import {Watcher} from "../../types/state";
 
-export function urlWatcher(conf: Conf, checkValidity: (data: any) => Promise<void>) {
+export function urlWatcher(conf: Conf, checkValidity: (data: any) => Promise<UrlWatcherResult>) {
 
-  return async (urlConf: ConfURL): Promise<Watcher> => {
+  return async (urlConf: ConfURL,
+                getOkTitle: () => string,
+                getKoTitle: () => string,
+                getRecoveredTitle: () => string): Promise<Watcher> => {
 
     let nodeDownRes: () => void
     let nodeDownPromise: Promise<void> = new Promise(res => nodeDownRes = res)
 
+    async function checkResult(data: any) {
+      const validity = await checkValidity(data)
+      if (validity.error) {
+        throw new UrlWatcherError('Validity error')
+      }
+    }
+
     return watcherLoop(
       urlConf.name,
       async () => {
         let interval: NodeJS.Timer;
         const res = await Axios.get(urlConf.address)
-        await checkValidity(res.data)
+        await checkResult(res.data)
         interval = setInterval(async () => {
           try {
             const res = await Axios.get(urlConf.address)
-            await checkValidity(res.data)
+            await checkResult(res.data)
           } catch (e) {
             if (interval) {
               clearInterval(interval)
@@ -36,21 +46,45 @@ export function urlWatcher(conf: Conf, checkValidity: (data: any) => Promise<voi
 
       conf.reconnectionDelays,
 
-      mail.onEstablished(conf, urlConf.address),
+      mail.onEstablished(conf, urlConf.address, getOkTitle()),
 
       // When a disconnection is detected
-      mail.onDisconnect(conf, urlConf.address),
+        (waitingDelay: number, error?: any) => {
+        let koTitle: string|undefined
+        let koMessage: (() => string)|undefined
+        if (error && error instanceof UrlWatcherError) {
+          koTitle = getKoTitle()
+          koMessage = () => `<p>${error.errorMessage}</p>`
+        }
+        return mail.onDisconnect(conf, urlConf.address, koTitle, koMessage)(waitingDelay)
+      },
 
       async () => {
         console.log('Trying to connect to %s', urlConf.address)
       },
 
-      mail.onRestartSuccess(conf, urlConf.address),
+      mail.onRestartSuccess(conf, urlConf.address, getRecoveredTitle()),
-
-      async (e) => {
-        console.error(e.message || e)
-      }
     )
 
   }
 }
+
+export class UrlWatcherError {
+
+  constructor (public errorMessage: string) {
+  }
+
+}
+
+export class UrlWatcherResult {
+  error?: string
+
+  public static ok() {
+    return {}
+  }
+
+  public static ko(errorMessage: string) {
+    return { error: errorMessage }
+  }
+
+}

src/lib/watchers/bma/bma-watcher.ts

@@ -1,5 +1,5 @@
 import {Conf, ConfBMA} from "../../types/conf";
-import {urlWatcher} from '../abstract/url-watcher'
+import {urlWatcher, UrlWatcherResult} from '../abstract/url-watcher'
 import {moment} from 'duniter/app/lib/common-libs/moment'
 
 export function bmaWatcher(conf: Conf) {
@@ -11,12 +11,16 @@ export function bmaWatcher(conf: Conf) {
     return urlWatcher(conf, async (data) => {
       const block = data as { medianTime: number }
       if (bmaServer.maxLate && moment().unix() - block.medianTime > bmaServer.maxLate) {
-        throw 'Server is late'
+        return UrlWatcherResult.ko('Server is late')
       }
+      return UrlWatcherResult.ok()
     })({
       name: `BMA ${bmaServer.address}`,
       address: bmaServer.address + URL_PATH,
       frequency: bmaServer.frequency
-    })
+    },
+        () => `[OK] BMA of ${bmaServer.address}`,
+        () => `[FAILURE] BMA of ${bmaServer.address}`,
+        () => `[RECOVERED] BMA of ${bmaServer.address}`)
   }
 }

src/lib/watchers/bma/head-watcher.ts

@@ -1,27 +1,29 @@
 import {Conf, ConfHead} from "../../types/conf";
-import {urlWatcher} from '../abstract/url-watcher'
+import {urlWatcher, UrlWatcherResult} from '../abstract/url-watcher'
 import {WS2PHead} from "duniter/app/modules/ws2p/lib/WS2PCluster";
 
-function handleLateness(confHead: ConfHead, mainHeads: HeadMetric[], observedHead?: TrameWS2P) {
+function handleLateness(confHead: ConfHead, mainHeads: HeadMetric[], observedHead?: TrameWS2P): UrlWatcherResult {
   if (!mainHeads.length) {
-    throw 'Observed pubkey ${confHead.observedPubkey}: no consensus found'
+    return UrlWatcherResult.ko('Observed pubkey ${confHead.observedPubkey}: no consensus found')
   }
   if (!observedHead) {
-    throw `Observed pubkey ${confHead.observedPubkey} not found in heads`
+    return UrlWatcherResult.ko(`Observed pubkey ${confHead.observedPubkey} not found in heads`)
   }
   const matchingHeads = mainHeads.filter(h => h.blockstamp === observedHead.blockstamp)
   if (!matchingHeads.length) {
     // Check how much late is the node
     const farAwayHeads = mainHeads.filter(h => h.blockNumber - observedHead.blockNumber >= confHead.maxLateBlocks)
     if (farAwayHeads.length) {
-      throw `Observed pubkey ${confHead.observedPubkey} is too late for ${farAwayHeads.length} consensus by at least ${confHead.maxLateBlocks}`
+      return UrlWatcherResult.ko(`Observed pubkey ${confHead.observedPubkey} is too late for ${farAwayHeads.length} consensus by at least ${confHead.maxLateBlocks}`)
     }
   }
+  return UrlWatcherResult.ok()
 }
 
 export function headWatcher(conf: Conf) {
 
   const URL_PATH = '/network/ws2p/heads'
+  const KEY_LENGTH = 10
 
   return async (confHead: ConfHead) => {
 
@@ -29,12 +31,15 @@ export function headWatcher(conf: Conf) {
       const heads = data as { heads: WS2PHead[] }
       const mainHeads = getMain(heads)
       const observedHead = getObserved(heads, confHead.observedPubkey)
-      handleLateness(confHead, mainHeads, observedHead)
+      return handleLateness(confHead, mainHeads, observedHead)
     })({
-      name: `head watcher ${confHead.address} => ${confHead.observedPubkey.substr(0, 10)}`,
+      name: `head watcher ${confHead.address} => ${confHead.observedPubkey.substr(0, KEY_LENGTH)}`,
       address: confHead.address + URL_PATH,
       frequency: confHead.frequency
-    })
+    },
+        () => `[OK] HEADS of ${confHead.observedPubkey.substr(0, KEY_LENGTH)}`,
+        () => `[FAILURE] HEADS of ${confHead.observedPubkey.substr(0, KEY_LENGTH)}`,
+        () => `[RECOVERED] HEADS of ${confHead.observedPubkey.substr(0, KEY_LENGTH)}`)
   }
 }
 

src/lib/watchers/dprobe/dprobe-heartbeat-watcher.ts

@@ -1,5 +1,5 @@
 import {Conf, ConfDprobeHeartbeat} from "../../types/conf";
-import {urlWatcher} from '../abstract/url-watcher'
+import {urlWatcher, UrlWatcherResult} from '../abstract/url-watcher'
 import {moment} from 'duniter/app/lib/common-libs/moment'
 
 export function dprobeHeartbeat(conf: Conf) {
@@ -10,8 +10,12 @@ export function dprobeHeartbeat(conf: Conf) {
       const last = moment(data, 'YYYY-MM-DD HH:mm:ss\n')
       const past = moment().diff(last)
       if (past > dconf.lastBeat) {
-        throw 'Delay is over'
+        return UrlWatcherResult.ko('Delay is over')
       }
-    })(dconf)
+      return UrlWatcherResult.ok()
+    })(dconf,
+        () => `[OK] hearbeat is up-to-date`,
+        () => `[FAILURE] hearbeat`,
+        () => `[RECOVERED] hearbeat`)
   }
 }

src/lib/watchers/webdiff/webdiff-watcher.ts

@@ -66,7 +66,7 @@ export function webDiffWatcher(conf: Conf) {
       () => mail.onEstablished(conf, target, 'webdiff successfully started')(webDiffConf.cc),
 
       // When a disconnection is detected
-      (waitingDelay: number) => mail.onDisconnect(conf, target, 'Diff detected', (waitingDelay: number) => `
+      (waitingDelay: number) => mail.onDisconnect(conf, target, 'Diff detected', undefined, (waitingDelay: number) => `
         ${htmlDiff}
         <p>
           Waiting ${(waitingDelay / 1000).toFixed(0)} seconds before trying to reconnect.
@@ -78,10 +78,6 @@ export function webDiffWatcher(conf: Conf) {
       },
 
       () => mail.onRestartSuccess(conf, target)(webDiffConf.cc),
-
-      async (e) => {
-        console.error(e.message)
-      }
     )
 
   }

src/lib/watchers/wotwizard/json-watcher.ts

@@ -1,11 +1,12 @@
 import {Conf, ConfWWMeta} from "../../types/conf";
-import {urlWatcher} from '../abstract/url-watcher'
+import {urlWatcher, UrlWatcherResult} from '../abstract/url-watcher'
 
 function handleLateness(confHead: ConfWWMeta, data: WWMetaJson) {
   const diff = Math.round(Date.now()/1000 - data.now)
   if (diff >= confHead.maxLate) {
-    throw `WWMeta.json is late by ${diff}s (>= ${confHead.maxLate})`
+    return UrlWatcherResult.ko(`WWMeta.json is late by ${diff}s (>= ${confHead.maxLate})`)
   }
+  return UrlWatcherResult.ok()
 }
 
 export function jsonWatcher(conf: Conf) {
@@ -15,12 +16,15 @@ export function jsonWatcher(conf: Conf) {
   return async (confWWMeta: ConfWWMeta) => {
 
     return urlWatcher(conf, async (data) => {
-      handleLateness(confWWMeta, data)
+      return handleLateness(confWWMeta, data)
     })({
       name: `WWMeta.json watcher ${confWWMeta.address}`,
       address: confWWMeta.address + URL_PATH,
       frequency: confWWMeta.frequency
-    })
+    },
+        () => `[OK] WWMeta.json is up-to-date`,
+        () => `[FAILURE] WWMeta.json`,
+        () => `[RECOVERED] WWMeta.json`)
   }
 }
 

src/lib/watchers/ws2p/ws2p-watcher.ts

@@ -54,8 +54,6 @@ export function ws2pWatcher(conf: Conf) {
       },
 
       mail.onRestartSuccess(conf, target),
-
-      mail.onError(conf, target)
     )
 
   }