From 4bed33d66f6a942a869cba6139e23ccbac85aacb Mon Sep 17 00:00:00 2001 From: cgeek Date: Fri, 8 May 2020 11:40:25 +0200 Subject: [PATCH] [mod] recall of FAILURE state periodically --- app.yml | 3 +- src/lib/mail.ts | 8 ++--- src/lib/types/conf.ts | 1 + src/lib/types/state.ts | 35 +++++++++++++++------ src/lib/watcherLoop.ts | 7 +++-- src/lib/watchers/abstract/url-watcher.ts | 5 +-- src/lib/watchers/webdiff/webdiff-watcher.ts | 5 +-- src/lib/watchers/ws2p/ws2p-watcher.ts | 1 + 8 files changed, 43 insertions(+), 22 deletions(-) diff --git a/app.yml b/app.yml index 7ba7b3a..f3ac6fa 100644 --- a/app.yml +++ b/app.yml @@ -1,5 +1,6 @@ connectionTimeout: 10000 # 10" -waitingDelay: 60000 # 1' +waitingDelay: 5000 # 5" +recallDelay: 60000 ws2pServers: - address: ws://g1-test.cgeek.fr:22001 expectedPubkey: 3dnbnYY9i2bHMQUGyFp5GVvJ2wBkVpus31cDJA5cfRpj diff --git a/src/lib/mail.ts b/src/lib/mail.ts index db24e5d..fb78075 100644 --- a/src/lib/mail.ts +++ b/src/lib/mail.ts @@ -44,18 +44,18 @@ export const mail = { } }, - onDisconnect: (conf: Conf, target: string, message = `Connection closed for ${target}`, getErrorMessage: () => string = () => '', getHtml: (waitingDelay: number) => string = (waitingDelay: number) => ` + onDisconnect: (conf: Conf, target: string, message = `Connection closed for ${target}`, getErrorMessage: () => string = () => '', getHtml: (waitingDelay: number, recallDelay: number) => string = (waitingDelay: number, recallDelay: number) => `

Connection from [${os.hostname}] to ${target} was lost on ${moment().format('dd-MM-YYYY HH:mm:ss')}.

- Waiting ${(waitingDelay / 1000).toFixed(0)} seconds before trying to reconnect. + Waiting ${(waitingDelay / 1000).toFixed(0)} seconds before trying to reconnect (${(recallDelay / 1000).toFixed(0)} seconds for recall).

${getErrorMessage()} `) => { - return async (waitingDelay: number, cc?: string) => { + return async (waitingDelay: number, recallDelay: number, cc?: string) => { console.log('Waiting %s seconds...', (waitingDelay / 1000).toFixed(0)) - await sendMail(conf.mail, `[dw] [${os.hostname}] ${message}`, getHtml(waitingDelay), cc) + await sendMail(conf.mail, `[dw] [${os.hostname}] ${message}`, getHtml(waitingDelay, recallDelay), cc) } }, diff --git a/src/lib/types/conf.ts b/src/lib/types/conf.ts index 5384048..99b8a80 100644 --- a/src/lib/types/conf.ts +++ b/src/lib/types/conf.ts @@ -1,6 +1,7 @@ export interface Conf { connectionTimeout: number waitingDelay: number + recallDelay: number ws2pServers: ConfWS2P[] bmaServers: ConfBMA[] dprobeHeartbeats: ConfDprobeHeartbeat[] diff --git a/src/lib/types/state.ts b/src/lib/types/state.ts index dc6ccce..49d684e 100644 --- a/src/lib/types/state.ts +++ b/src/lib/types/state.ts @@ -1,26 +1,26 @@ export class Watcher { - private _state: 'INIT'|'OK'|'FAILURE'|'RECOVERED' + private _state: WatcherState private _stateChanged: boolean + private _stateChangedLastTime: number private _error?: any - constructor(public readonly name: string) { - this._stateChanged = false - this._state = "INIT" + constructor( + public readonly name: string, + private recallDelay: number + ) { + this.changeToState("INIT") } public stateOK() { - this._stateChanged = this._state !== "OK" - this._state = "OK" + this.changeToState("OK") } public stateRecovered() { - this._stateChanged = this._state !== "RECOVERED" - this._state = "RECOVERED" + this.changeToState("RECOVERED") } public stateFailure(error: any) { - this._stateChanged = this._state !== "FAILURE" - this._state = "FAILURE" + this.changeToState("FAILURE") this._error = error } @@ -42,4 +42,19 @@ export class Watcher { } return this._error.message || this._error } + + private changeToState(newState: WatcherState) { + this._stateChanged = this._state !== newState + this._state = newState + if (this._stateChanged) { + this._stateChangedLastTime = Date.now() + } + else if (Date.now() - this._stateChangedLastTime > this.recallDelay) { + // Same state for long enough time + this._stateChanged = true + this._stateChangedLastTime = Date.now() + } + } } + +type WatcherState = 'INIT'|'OK'|'FAILURE'|'RECOVERED' \ No newline at end of file diff --git a/src/lib/watcherLoop.ts b/src/lib/watcherLoop.ts index d5c8998..61ab203 100644 --- a/src/lib/watcherLoop.ts +++ b/src/lib/watcherLoop.ts @@ -5,13 +5,14 @@ export function watcherLoop( connect: () => Promise, onConnectionClosed: () => Promise, waitingDelay: number, + recallDelay: number, onStart: () => Promise, - onDisconnection: (waitingDelay: number, error: any) => Promise, + onDisconnection: (waitingDelay: number, recallDelay: number, error: any) => Promise, onRestart: () => Promise, onRestartSuccess: () => Promise, ): Watcher { - let watcher: Watcher = new Watcher(name) + let watcher: Watcher = new Watcher(name, recallDelay) let hasStarted = false ;(async () => { @@ -48,7 +49,7 @@ export function watcherLoop( // Wait before reconnecting if (watcher.stateChanged) { // Notify only if state changed since - await onDisconnection(waitingDelay, watcher.error) + await onDisconnection(waitingDelay, recallDelay, watcher.error) } await new Promise(resolve => setTimeout(resolve, waitingDelay)) i++ diff --git a/src/lib/watchers/abstract/url-watcher.ts b/src/lib/watchers/abstract/url-watcher.ts index c88b929..d956783 100644 --- a/src/lib/watchers/abstract/url-watcher.ts +++ b/src/lib/watchers/abstract/url-watcher.ts @@ -45,18 +45,19 @@ export function urlWatcher(conf: Conf, checkValidity: (data: any) => Promise nodeDownPromise, conf.waitingDelay, + conf.recallDelay, mail.onEstablished(conf, urlConf.address, getOkTitle()), // When a disconnection is detected - (waitingDelay: number, error?: any) => { + (waitingDelay: number, recallDelay, error?: any) => { let koTitle: string|undefined let koMessage: (() => string)|undefined if (error && error instanceof UrlWatcherError) { koTitle = getKoTitle() koMessage = () => `

${error.errorMessage}

` } - return mail.onDisconnect(conf, urlConf.address, koTitle, koMessage)(waitingDelay) + return mail.onDisconnect(conf, urlConf.address, koTitle, koMessage)(waitingDelay, recallDelay) }, async () => { diff --git a/src/lib/watchers/webdiff/webdiff-watcher.ts b/src/lib/watchers/webdiff/webdiff-watcher.ts index 3dec296..fe7da3a 100644 --- a/src/lib/watchers/webdiff/webdiff-watcher.ts +++ b/src/lib/watchers/webdiff/webdiff-watcher.ts @@ -62,16 +62,17 @@ export function webDiffWatcher(conf: Conf) { () => nodeDownPromise, conf.waitingDelay, + conf.recallDelay, () => mail.onEstablished(conf, target, 'webdiff successfully started')(webDiffConf.cc), // When a disconnection is detected - (waitingDelay: number) => mail.onDisconnect(conf, target, 'Diff detected', undefined, (waitingDelay: number) => ` + (waitingDelay: number, recallDelay: number) => mail.onDisconnect(conf, target, 'Diff detected', undefined, (waitingDelay: number) => ` ${htmlDiff}

Waiting ${(waitingDelay / 1000).toFixed(0)} seconds before trying to reconnect.

- `)(waitingDelay, webDiffConf.cc), + `)(waitingDelay, recallDelay, webDiffConf.cc), async () => { console.log('Trying to connect to %s', target) diff --git a/src/lib/watchers/ws2p/ws2p-watcher.ts b/src/lib/watchers/ws2p/ws2p-watcher.ts index 1c4d561..a78f23f 100644 --- a/src/lib/watchers/ws2p/ws2p-watcher.ts +++ b/src/lib/watchers/ws2p/ws2p-watcher.ts @@ -43,6 +43,7 @@ export function ws2pWatcher(conf: Conf) { () => c.closed, conf.waitingDelay, + conf.recallDelay, mail.onEstablished(conf, target),