[mod] recall of FAILURE state periodically

app.yml

@@ -1,5 +1,6 @@
 connectionTimeout: 10000 # 10"
-waitingDelay: 60000 # 1'
+waitingDelay: 5000 # 5"
+recallDelay: 60000
 ws2pServers:
   - address: ws://g1-test.cgeek.fr:22001
     expectedPubkey: 3dnbnYY9i2bHMQUGyFp5GVvJ2wBkVpus31cDJA5cfRpj

src/lib/mail.ts

@@ -44,18 +44,18 @@ export const mail = {
     }
   },
 
-  onDisconnect: (conf: Conf, target: string, message = `Connection closed for ${target}`, getErrorMessage: () => string = () => '', getHtml: (waitingDelay: number) => string = (waitingDelay: number) => `
+  onDisconnect: (conf: Conf, target: string, message = `Connection closed for ${target}`, getErrorMessage: () => string = () => '', getHtml: (waitingDelay: number, recallDelay: number) => string = (waitingDelay: number, recallDelay: number) => `
       <p>
         Connection from [${os.hostname}] to ${target} was lost on ${moment().format('dd-MM-YYYY HH:mm:ss')}.
       </p>
       <p>
-        Waiting ${(waitingDelay / 1000).toFixed(0)} seconds before trying to reconnect.
+        Waiting ${(waitingDelay / 1000).toFixed(0)} seconds before trying to reconnect (${(recallDelay / 1000).toFixed(0)} seconds for recall).
       </p>
       ${getErrorMessage()}
   `) => {
-    return async (waitingDelay: number, cc?: string) => {
+    return async (waitingDelay: number, recallDelay: number, cc?: string) => {
       console.log('Waiting %s seconds...', (waitingDelay / 1000).toFixed(0))
-      await sendMail(conf.mail, `[dw] [${os.hostname}] ${message}`, getHtml(waitingDelay), cc)
+      await sendMail(conf.mail, `[dw] [${os.hostname}] ${message}`, getHtml(waitingDelay, recallDelay), cc)
     }
   },
 

src/lib/types/conf.ts

@@ -1,6 +1,7 @@
 export interface Conf {
   connectionTimeout: number
   waitingDelay: number
+  recallDelay: number
   ws2pServers: ConfWS2P[]
   bmaServers: ConfBMA[]
   dprobeHeartbeats: ConfDprobeHeartbeat[]

src/lib/types/state.ts

@@ -1,26 +1,26 @@
 export class Watcher {
-    private _state: 'INIT'|'OK'|'FAILURE'|'RECOVERED'
+    private _state: WatcherState
     private _stateChanged: boolean
+    private _stateChangedLastTime: number
     private _error?: any
 
-    constructor(public readonly name: string) {
-        this._stateChanged = false
-        this._state = "INIT"
+    constructor(
+        public readonly name: string,
+        private recallDelay: number
+    ) {
+        this.changeToState("INIT")
     }
 
     public stateOK() {
-        this._stateChanged = this._state !== "OK"
-        this._state = "OK"
+        this.changeToState("OK")
     }
 
     public stateRecovered() {
-        this._stateChanged = this._state !== "RECOVERED"
-        this._state = "RECOVERED"
+        this.changeToState("RECOVERED")
     }
 
     public stateFailure(error: any) {
-        this._stateChanged = this._state !== "FAILURE"
-        this._state = "FAILURE"
+        this.changeToState("FAILURE")
         this._error = error
     }
 
@@ -42,4 +42,19 @@ export class Watcher {
         }
         return this._error.message || this._error
     }
+
+    private changeToState(newState: WatcherState) {
+        this._stateChanged = this._state !== newState
+        this._state = newState
+        if (this._stateChanged) {
+            this._stateChangedLastTime = Date.now()
+        }
+        else if (Date.now() - this._stateChangedLastTime > this.recallDelay) {
+            // Same state for long enough time
+            this._stateChanged = true
+            this._stateChangedLastTime = Date.now()
+        }
+    }
 }
+
+type WatcherState = 'INIT'|'OK'|'FAILURE'|'RECOVERED'

src/lib/watcherLoop.ts

@@ -5,13 +5,14 @@ export function watcherLoop(
   connect: () => Promise<void>,
   onConnectionClosed: () => Promise<void>,
   waitingDelay: number,
+  recallDelay: number,
   onStart: () => Promise<void>,
-  onDisconnection: (waitingDelay: number, error: any) => Promise<void>,
+  onDisconnection: (waitingDelay: number, recallDelay: number, error: any) => Promise<void>,
   onRestart: () => Promise<void>,
   onRestartSuccess: () => Promise<void>,
 ): Watcher {
 
-  let watcher: Watcher = new Watcher(name)
+  let watcher: Watcher = new Watcher(name, recallDelay)
   let hasStarted = false
 
   ;(async () => {
@@ -48,7 +49,7 @@ export function watcherLoop(
       // Wait before reconnecting
       if (watcher.stateChanged) {
         // Notify only if state changed since
-        await onDisconnection(waitingDelay, watcher.error)
+        await onDisconnection(waitingDelay, recallDelay, watcher.error)
       }
       await new Promise(resolve => setTimeout(resolve, waitingDelay))
       i++

src/lib/watchers/abstract/url-watcher.ts

@@ -45,18 +45,19 @@ export function urlWatcher(conf: Conf, checkValidity: (data: any) => Promise<Url
       () => nodeDownPromise,
 
       conf.waitingDelay,
+      conf.recallDelay,
 
       mail.onEstablished(conf, urlConf.address, getOkTitle()),
 
       // When a disconnection is detected
-        (waitingDelay: number, error?: any) => {
+        (waitingDelay: number, recallDelay, error?: any) => {
         let koTitle: string|undefined
         let koMessage: (() => string)|undefined
         if (error && error instanceof UrlWatcherError) {
           koTitle = getKoTitle()
           koMessage = () => `<p>${error.errorMessage}</p>`
         }
-        return mail.onDisconnect(conf, urlConf.address, koTitle, koMessage)(waitingDelay)
+        return mail.onDisconnect(conf, urlConf.address, koTitle, koMessage)(waitingDelay, recallDelay)
       },
 
       async () => {

src/lib/watchers/webdiff/webdiff-watcher.ts

@@ -62,16 +62,17 @@ export function webDiffWatcher(conf: Conf) {
       () => nodeDownPromise,
 
       conf.waitingDelay,
+      conf.recallDelay,
 
       () => mail.onEstablished(conf, target, 'webdiff successfully started')(webDiffConf.cc),
 
       // When a disconnection is detected
-      (waitingDelay: number) => mail.onDisconnect(conf, target, 'Diff detected', undefined, (waitingDelay: number) => `
+      (waitingDelay: number, recallDelay: number) => mail.onDisconnect(conf, target, 'Diff detected', undefined, (waitingDelay: number) => `
         ${htmlDiff}
         <p>
           Waiting ${(waitingDelay / 1000).toFixed(0)} seconds before trying to reconnect.
         </p>
-    `)(waitingDelay, webDiffConf.cc),
+    `)(waitingDelay, recallDelay, webDiffConf.cc),
 
       async () => {
         console.log('Trying to connect to %s', target)

src/lib/watchers/ws2p/ws2p-watcher.ts

@@ -43,6 +43,7 @@ export function ws2pWatcher(conf: Conf) {
       () => c.closed,
 
       conf.waitingDelay,
+      conf.recallDelay,
 
       mail.onEstablished(conf, target),