Core synchronous queue with an example worker

This commit is contained in:
2020-11-02 10:51:08 +00:00
parent d6b5b033f2
commit 0a30f2d466
16 changed files with 1103 additions and 1 deletions

91
app.js Normal file
View File

@@ -0,0 +1,91 @@
const dotenv = require('dotenv')
const compression = require('compression')
const express = require('express')
const morgan = require('morgan')
const bodyParser = require('body-parser')
const path = require('path')
const rfs = require('rotating-file-stream')
const cors = require('cors')
const errorhandler = require('errorhandler')
const statusCodes = require('http-status-codes').StatusCodes
const createError = require('http-errors')
const apiRouter = require('./routes')
// load .env config file
dotenv.config()
// main app file
const app = express()
// create a rotating write stream
const accessLogStream = rfs.createStream('access.log', {
interval: '1d', // rotate daily
path: path.join(__dirname, 'log')
})
// setup the file logger
app.use(morgan('common', { stream: accessLogStream }))
// setup the console logger
if (process.env.NODE_ENV === 'development') {
// [debug]
app.use(morgan('dev'))
} else {
// [production]
app.use(morgan('dev', {
skip: function (req, res) { return res.statusCode < 400 }
}))
}
// be proxy aware
app.enable('trust proxy')
// cross-origin resource sharing
app.use(cors())
// json middleware
app.use(bodyParser.json())
// healthcheck (can be used for heart-beat)
app.get('/status', (req, res) => {
res.status(200).end()
});
app.head('/status', (req, res) => {
res.status(200).end()
});
// routes
app.use('/api', apiRouter)
// 404 middleware
app.use((req, res, next) => {
next(createError(statusCodes.NOT_FOUND))
})
// error handling
app.use((err, req, res, next) => {
/* Handle 401 */
if (err.status === statusCodes.UNAUTHORIZED) {
return res
.status(err.status)
.send({ message: err.message })
.end()
}
return next(err)
})
if (process.env.NODE_ENV !== 'development') {
app.use((err, req, res, next) => {
res.status(err.status || 500)
res.json({
errors: {
message: err.message,
},
})
})
}
// start the server
app.listen(process.env.PORT, process.env.HOST)
console.log('service started on port ' process.env.HOST + ':' + process.env.PORT)

16
example.env Normal file
View File

@@ -0,0 +1,16 @@
# Choices for NODE_ENV:
# development: for extra debug logging including all requests displayed in console
# production: reduced logging, only errors in console, all requests logged to file
NODE_ENV=development
# API key clients should use when talking to this server
# TODO: Add capability for multiple API keys for different clients
API_KEY=
# Host (ip address) the server should listen on.
# Change to 0.0.0.0 to allow access from network. Ideally use a local front-end server
# (for example nginx or apache) for https handling.
HOST=127.0.0.1
# Port the server should listen on
PORT=3000

552
package-lock.json generated Normal file
View File

@@ -0,0 +1,552 @@
{
"name": "Calc-Scheduler",
"version": "0.1.0",
"lockfileVersion": 1,
"requires": true,
"dependencies": {
"accepts": {
"version": "1.3.7",
"resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.7.tgz",
"integrity": "sha512-Il80Qs2WjYlJIBNzNkK6KYqlVMTbZLXgHx2oT0pU/fjRHyEp+PEfEPY0R3WCwAGVOtauxh1hOxNgIf5bv7dQpA==",
"requires": {
"mime-types": "~2.1.24",
"negotiator": "0.6.2"
}
},
"array-flatten": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz",
"integrity": "sha1-ml9pkFGx5wczKPKgCJaLZOopVdI="
},
"basic-auth": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/basic-auth/-/basic-auth-2.0.1.tgz",
"integrity": "sha512-NF+epuEdnUYVlGuhaxbbq+dvJttwLnGY+YixlXlME5KpQ5W3CnXA5cVTneY3SPbPDRkcjMbifrwmFYcClgOZeg==",
"requires": {
"safe-buffer": "5.1.2"
}
},
"body-parser": {
"version": "1.19.0",
"resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.19.0.tgz",
"integrity": "sha512-dhEPs72UPbDnAQJ9ZKMNTP6ptJaionhP5cBb541nXPlW60Jepo9RV/a4fX4XWW9CuFNK22krhrj1+rgzifNCsw==",
"requires": {
"bytes": "3.1.0",
"content-type": "~1.0.4",
"debug": "2.6.9",
"depd": "~1.1.2",
"http-errors": "1.7.2",
"iconv-lite": "0.4.24",
"on-finished": "~2.3.0",
"qs": "6.7.0",
"raw-body": "2.4.0",
"type-is": "~1.6.17"
},
"dependencies": {
"http-errors": {
"version": "1.7.2",
"resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.7.2.tgz",
"integrity": "sha512-uUQBt3H/cSIVfch6i1EuPNy/YsRSOUBXTVfZ+yR7Zjez3qjBz6i9+i4zjNaoqcoFVI4lQJ5plg63TvGfRSDCRg==",
"requires": {
"depd": "~1.1.2",
"inherits": "2.0.3",
"setprototypeof": "1.1.1",
"statuses": ">= 1.5.0 < 2",
"toidentifier": "1.0.0"
}
}
}
},
"bytes": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.0.tgz",
"integrity": "sha512-zauLjrfCG+xvoyaqLoV8bLVXXNGC4JqlxFCutSDWA6fJrTo2ZuvLYTqZ7aHBLZSMOopbzwv8f+wZcVzfVTI2Dg=="
},
"compressible": {
"version": "2.0.18",
"resolved": "https://registry.npmjs.org/compressible/-/compressible-2.0.18.tgz",
"integrity": "sha512-AF3r7P5dWxL8MxyITRMlORQNaOA2IkAFaTr4k7BUumjPtRpGDTZpl0Pb1XCO6JeDCBdp126Cgs9sMxqSjgYyRg==",
"requires": {
"mime-db": ">= 1.43.0 < 2"
}
},
"compression": {
"version": "1.7.4",
"resolved": "https://registry.npmjs.org/compression/-/compression-1.7.4.tgz",
"integrity": "sha512-jaSIDzP9pZVS4ZfQ+TzvtiWhdpFhE2RDHz8QJkpX9SIpLq88VueF5jJw6t+6CUQcAoA6t+x89MLrWAqpfDE8iQ==",
"requires": {
"accepts": "~1.3.5",
"bytes": "3.0.0",
"compressible": "~2.0.16",
"debug": "2.6.9",
"on-headers": "~1.0.2",
"safe-buffer": "5.1.2",
"vary": "~1.1.2"
},
"dependencies": {
"bytes": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/bytes/-/bytes-3.0.0.tgz",
"integrity": "sha1-0ygVQE1olpn4Wk6k+odV3ROpYEg="
}
}
},
"content-disposition": {
"version": "0.5.3",
"resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.3.tgz",
"integrity": "sha512-ExO0774ikEObIAEV9kDo50o+79VCUdEB6n6lzKgGwupcVeRlhrj3qGAfwq8G6uBJjkqLrhT0qEYFcWng8z1z0g==",
"requires": {
"safe-buffer": "5.1.2"
}
},
"content-type": {
"version": "1.0.4",
"resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz",
"integrity": "sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA=="
},
"cookie": {
"version": "0.4.0",
"resolved": "https://registry.npmjs.org/cookie/-/cookie-0.4.0.tgz",
"integrity": "sha512-+Hp8fLp57wnUSt0tY0tHEXh4voZRDnoIrZPqlo3DPiI4y9lwg/jqx+1Om94/W6ZaPDOUbnjOt/99w66zk+l1Xg=="
},
"cookie-signature": {
"version": "1.0.6",
"resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz",
"integrity": "sha1-4wOogrNCzD7oylE6eZmXNNqzriw="
},
"cors": {
"version": "2.8.5",
"resolved": "https://registry.npmjs.org/cors/-/cors-2.8.5.tgz",
"integrity": "sha512-KIHbLJqu73RGr/hnbrO9uBeixNGuvSQjul/jdFvS/KFSIH1hWVd1ng7zOHx+YrEfInLG7q4n6GHQ9cDtxv/P6g==",
"requires": {
"object-assign": "^4",
"vary": "^1"
}
},
"debug": {
"version": "2.6.9",
"resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
"integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
"requires": {
"ms": "2.0.0"
}
},
"depd": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz",
"integrity": "sha1-m81S4UwJd2PnSbJ0xDRu0uVgtak="
},
"destroy": {
"version": "1.0.4",
"resolved": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz",
"integrity": "sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA="
},
"dotenv": {
"version": "8.2.0",
"resolved": "https://registry.npmjs.org/dotenv/-/dotenv-8.2.0.tgz",
"integrity": "sha512-8sJ78ElpbDJBHNeBzUbUVLsqKdccaa/BXF1uPTw3GrvQTBgrQrtObr2mUrE38vzYd8cEv+m/JBfDLioYcfXoaw=="
},
"ee-first": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz",
"integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0="
},
"encodeurl": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz",
"integrity": "sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k="
},
"errorhandler": {
"version": "1.5.1",
"resolved": "https://registry.npmjs.org/errorhandler/-/errorhandler-1.5.1.tgz",
"integrity": "sha512-rcOwbfvP1WTViVoUjcfZicVzjhjTuhSMntHh6mW3IrEiyE6mJyXvsToJUJGlGlw/2xU9P5whlWNGlIDVeCiT4A==",
"requires": {
"accepts": "~1.3.7",
"escape-html": "~1.0.3"
}
},
"escape-html": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz",
"integrity": "sha1-Aljq5NPQwJdN4cFpGI7wBR0dGYg="
},
"etag": {
"version": "1.8.1",
"resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz",
"integrity": "sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc="
},
"express": {
"version": "4.17.1",
"resolved": "https://registry.npmjs.org/express/-/express-4.17.1.tgz",
"integrity": "sha512-mHJ9O79RqluphRrcw2X/GTh3k9tVv8YcoyY4Kkh4WDMUYKRZUq0h1o0w2rrrxBqM7VoeUVqgb27xlEMXTnYt4g==",
"requires": {
"accepts": "~1.3.7",
"array-flatten": "1.1.1",
"body-parser": "1.19.0",
"content-disposition": "0.5.3",
"content-type": "~1.0.4",
"cookie": "0.4.0",
"cookie-signature": "1.0.6",
"debug": "2.6.9",
"depd": "~1.1.2",
"encodeurl": "~1.0.2",
"escape-html": "~1.0.3",
"etag": "~1.8.1",
"finalhandler": "~1.1.2",
"fresh": "0.5.2",
"merge-descriptors": "1.0.1",
"methods": "~1.1.2",
"on-finished": "~2.3.0",
"parseurl": "~1.3.3",
"path-to-regexp": "0.1.7",
"proxy-addr": "~2.0.5",
"qs": "6.7.0",
"range-parser": "~1.2.1",
"safe-buffer": "5.1.2",
"send": "0.17.1",
"serve-static": "1.14.1",
"setprototypeof": "1.1.1",
"statuses": "~1.5.0",
"type-is": "~1.6.18",
"utils-merge": "1.0.1",
"vary": "~1.1.2"
}
},
"finalhandler": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.1.2.tgz",
"integrity": "sha512-aAWcW57uxVNrQZqFXjITpW3sIUQmHGG3qSb9mUah9MgMC4NeWhNOlNjXEYq3HjRAvL6arUviZGGJsBg6z0zsWA==",
"requires": {
"debug": "2.6.9",
"encodeurl": "~1.0.2",
"escape-html": "~1.0.3",
"on-finished": "~2.3.0",
"parseurl": "~1.3.3",
"statuses": "~1.5.0",
"unpipe": "~1.0.0"
}
},
"forwarded": {
"version": "0.1.2",
"resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.1.2.tgz",
"integrity": "sha1-mMI9qxF1ZXuMBXPozszZGw/xjIQ="
},
"fresh": {
"version": "0.5.2",
"resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz",
"integrity": "sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac="
},
"http-errors": {
"version": "1.8.0",
"resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.8.0.tgz",
"integrity": "sha512-4I8r0C5JDhT5VkvI47QktDW75rNlGVsUf/8hzjCC/wkWI/jdTRmBb9aI7erSG82r1bjKY3F6k28WnsVxB1C73A==",
"requires": {
"depd": "~1.1.2",
"inherits": "2.0.4",
"setprototypeof": "1.2.0",
"statuses": ">= 1.5.0 < 2",
"toidentifier": "1.0.0"
},
"dependencies": {
"inherits": {
"version": "2.0.4",
"resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
"integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="
},
"setprototypeof": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz",
"integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw=="
}
}
},
"http-status-codes": {
"version": "2.1.4",
"resolved": "https://registry.npmjs.org/http-status-codes/-/http-status-codes-2.1.4.tgz",
"integrity": "sha512-MZVIsLKGVOVE1KEnldppe6Ij+vmemMuApDfjhVSLzyYP+td0bREEYyAoIw9yFePoBXManCuBqmiNP5FqJS5Xkg=="
},
"iconv-lite": {
"version": "0.4.24",
"resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz",
"integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==",
"requires": {
"safer-buffer": ">= 2.1.2 < 3"
}
},
"inherits": {
"version": "2.0.3",
"resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz",
"integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4="
},
"ipaddr.js": {
"version": "1.9.1",
"resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz",
"integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g=="
},
"media-typer": {
"version": "0.3.0",
"resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz",
"integrity": "sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g="
},
"merge-descriptors": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz",
"integrity": "sha1-sAqqVW3YtEVoFQ7J0blT8/kMu2E="
},
"methods": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz",
"integrity": "sha1-VSmk1nZUE07cxSZmVoNbD4Ua/O4="
},
"mime": {
"version": "1.6.0",
"resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz",
"integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg=="
},
"mime-db": {
"version": "1.44.0",
"resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.44.0.tgz",
"integrity": "sha512-/NOTfLrsPBVeH7YtFPgsVWveuL+4SjjYxaQ1xtM1KMFj7HdxlBlxeyNLzhyJVx7r4rZGJAZ/6lkKCitSc/Nmpg=="
},
"mime-types": {
"version": "2.1.27",
"resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.27.tgz",
"integrity": "sha512-JIhqnCasI9yD+SsmkquHBxTSEuZdQX5BuQnS2Vc7puQQQ+8yiP5AY5uWhpdv4YL4VM5c6iliiYWPgJ/nJQLp7w==",
"requires": {
"mime-db": "1.44.0"
}
},
"morgan": {
"version": "1.10.0",
"resolved": "https://registry.npmjs.org/morgan/-/morgan-1.10.0.tgz",
"integrity": "sha512-AbegBVI4sh6El+1gNwvD5YIck7nSA36weD7xvIxG4in80j/UoK8AEGaWnnz8v1GxonMCltmlNs5ZKbGvl9b1XQ==",
"requires": {
"basic-auth": "~2.0.1",
"debug": "2.6.9",
"depd": "~2.0.0",
"on-finished": "~2.3.0",
"on-headers": "~1.0.2"
},
"dependencies": {
"depd": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz",
"integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw=="
}
}
},
"ms": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
"integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g="
},
"negotiator": {
"version": "0.6.2",
"resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.2.tgz",
"integrity": "sha512-hZXc7K2e+PgeI1eDBe/10Ard4ekbfrrqG8Ep+8Jmf4JID2bNg7NvCPOZN+kfF574pFQI7mum2AUqDidoKqcTOw=="
},
"object-assign": {
"version": "4.1.1",
"resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz",
"integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM="
},
"on-finished": {
"version": "2.3.0",
"resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz",
"integrity": "sha1-IPEzZIGwg811M3mSoWlxqi2QaUc=",
"requires": {
"ee-first": "1.1.1"
}
},
"on-headers": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/on-headers/-/on-headers-1.0.2.tgz",
"integrity": "sha512-pZAE+FJLoyITytdqK0U5s+FIpjN0JP3OzFi/u8Rx+EV5/W+JTWGXG8xFzevE7AjBfDqHv/8vL8qQsIhHnqRkrA=="
},
"parseurl": {
"version": "1.3.3",
"resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz",
"integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ=="
},
"path": {
"version": "0.12.7",
"resolved": "https://registry.npmjs.org/path/-/path-0.12.7.tgz",
"integrity": "sha1-1NwqUGxM4hl+tIHr/NWzbAFAsQ8=",
"requires": {
"process": "^0.11.1",
"util": "^0.10.3"
}
},
"path-to-regexp": {
"version": "0.1.7",
"resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz",
"integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w="
},
"process": {
"version": "0.11.10",
"resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz",
"integrity": "sha1-czIwDoQBYb2j5podHZGn1LwW8YI="
},
"proxy-addr": {
"version": "2.0.6",
"resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.6.tgz",
"integrity": "sha512-dh/frvCBVmSsDYzw6n926jv974gddhkFPfiN8hPOi30Wax25QZyZEGveluCgliBnqmuM+UJmBErbAUFIoDbjOw==",
"requires": {
"forwarded": "~0.1.2",
"ipaddr.js": "1.9.1"
}
},
"qs": {
"version": "6.7.0",
"resolved": "https://registry.npmjs.org/qs/-/qs-6.7.0.tgz",
"integrity": "sha512-VCdBRNFTX1fyE7Nb6FYoURo/SPe62QCaAyzJvUjwRaIsc+NePBEniHlvxFmmX56+HZphIGtV0XeCirBtpDrTyQ=="
},
"range-parser": {
"version": "1.2.1",
"resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz",
"integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg=="
},
"raw-body": {
"version": "2.4.0",
"resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.4.0.tgz",
"integrity": "sha512-4Oz8DUIwdvoa5qMJelxipzi/iJIi40O5cGV1wNYp5hvZP8ZN0T+jiNkL0QepXs+EsQ9XJ8ipEDoiH70ySUJP3Q==",
"requires": {
"bytes": "3.1.0",
"http-errors": "1.7.2",
"iconv-lite": "0.4.24",
"unpipe": "1.0.0"
},
"dependencies": {
"http-errors": {
"version": "1.7.2",
"resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.7.2.tgz",
"integrity": "sha512-uUQBt3H/cSIVfch6i1EuPNy/YsRSOUBXTVfZ+yR7Zjez3qjBz6i9+i4zjNaoqcoFVI4lQJ5plg63TvGfRSDCRg==",
"requires": {
"depd": "~1.1.2",
"inherits": "2.0.3",
"setprototypeof": "1.1.1",
"statuses": ">= 1.5.0 < 2",
"toidentifier": "1.0.0"
}
}
}
},
"rotating-file-stream": {
"version": "2.1.3",
"resolved": "https://registry.npmjs.org/rotating-file-stream/-/rotating-file-stream-2.1.3.tgz",
"integrity": "sha512-zZ4Tkngxispo7DgiTqX0s4ChLtM3qET6iYsDA9tmgDEqJ3BFgRq/ZotsKEDAYQt9pAn9JwwqT27CSwQt3CTxNg=="
},
"safe-buffer": {
"version": "5.1.2",
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
"integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g=="
},
"safer-buffer": {
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz",
"integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="
},
"send": {
"version": "0.17.1",
"resolved": "https://registry.npmjs.org/send/-/send-0.17.1.tgz",
"integrity": "sha512-BsVKsiGcQMFwT8UxypobUKyv7irCNRHk1T0G680vk88yf6LBByGcZJOTJCrTP2xVN6yI+XjPJcNuE3V4fT9sAg==",
"requires": {
"debug": "2.6.9",
"depd": "~1.1.2",
"destroy": "~1.0.4",
"encodeurl": "~1.0.2",
"escape-html": "~1.0.3",
"etag": "~1.8.1",
"fresh": "0.5.2",
"http-errors": "~1.7.2",
"mime": "1.6.0",
"ms": "2.1.1",
"on-finished": "~2.3.0",
"range-parser": "~1.2.1",
"statuses": "~1.5.0"
},
"dependencies": {
"http-errors": {
"version": "1.7.3",
"resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.7.3.tgz",
"integrity": "sha512-ZTTX0MWrsQ2ZAhA1cejAwDLycFsd7I7nVtnkT3Ol0aqodaKW+0CTZDQ1uBv5whptCnc8e8HeRRJxRs0kmm/Qfw==",
"requires": {
"depd": "~1.1.2",
"inherits": "2.0.4",
"setprototypeof": "1.1.1",
"statuses": ">= 1.5.0 < 2",
"toidentifier": "1.0.0"
}
},
"inherits": {
"version": "2.0.4",
"resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
"integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="
},
"ms": {
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz",
"integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg=="
}
}
},
"serve-static": {
"version": "1.14.1",
"resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.14.1.tgz",
"integrity": "sha512-JMrvUwE54emCYWlTI+hGrGv5I8dEwmco/00EvkzIIsR7MqrHonbD9pO2MOfFnpFntl7ecpZs+3mW+XbQZu9QCg==",
"requires": {
"encodeurl": "~1.0.2",
"escape-html": "~1.0.3",
"parseurl": "~1.3.3",
"send": "0.17.1"
}
},
"setprototypeof": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.1.tgz",
"integrity": "sha512-JvdAWfbXeIGaZ9cILp38HntZSFSo3mWg6xGcJJsd+d4aRMOqauag1C63dJfDw7OaMYwEbHMOxEZ1lqVRYP2OAw=="
},
"statuses": {
"version": "1.5.0",
"resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz",
"integrity": "sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow="
},
"toidentifier": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.0.tgz",
"integrity": "sha512-yaOH/Pk/VEhBWWTlhI+qXxDFXlejDGcQipMlyxda9nthulaxLZUNcUqFxokp0vcYnvteJln5FNQDRrxj3YcbVw=="
},
"type-is": {
"version": "1.6.18",
"resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz",
"integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==",
"requires": {
"media-typer": "0.3.0",
"mime-types": "~2.1.24"
}
},
"unpipe": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz",
"integrity": "sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw="
},
"util": {
"version": "0.10.4",
"resolved": "https://registry.npmjs.org/util/-/util-0.10.4.tgz",
"integrity": "sha512-0Pm9hTQ3se5ll1XihRic3FDIku70C+iHUdT/W926rSgHV5QgXsYbKZN8MSC3tJtSkhuROzvsQjAaFENRXr+19A==",
"requires": {
"inherits": "2.0.3"
}
},
"utils-merge": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz",
"integrity": "sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM="
},
"vary": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz",
"integrity": "sha1-IpnwLG3tMNSllhsLn3RSShj2NPw="
}
}
}

View File

@@ -2,7 +2,7 @@
"name": "Calc-Scheduler",
"version": "0.1.0",
"description": "A schedular for the calculation of FIDGITS/MIBAT scenarios",
"main": "index.js",
"main": "app.js",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1"
},
@@ -12,5 +12,18 @@
"repository": {
"type": "git",
"url": "https://git.avsdev.uk/AVSDev/calc-scheduler"
},
"dependencies": {
"body-parser": "^1.19.0",
"compression": "^1.7.4",
"cors": "^2.8.5",
"dotenv": "^8.2.0",
"errorhandler": "^1.5.1",
"express": "^4.17.1",
"http-errors": "^1.8.0",
"http-status-codes": "^2.1.4",
"morgan": "^1.10.0",
"path": "^0.12.7",
"rotating-file-stream": "^2.1.3"
}
}

89
routes/calc-queue.js Normal file
View File

@@ -0,0 +1,89 @@
const express = require('express')
const statusCodes = require('http-status-codes').StatusCodes
const calcQueueSvc = require('../src/services/calc-queue')
const CalcQueueExceptions = require('../src/services/calc-queue/exceptions.js')
const router = express.Router()
// Return busy/not busy flag
router.get('/busy', function(req, res) {
res.send({ busy: calcQueueSvc.isBusy() })
})
// Get the current queue of tasks
router.get('/queue', function(req, res) {
res.send({ 'queue': calcQueueSvc.getQueue() })
})
// Add a calc task to the queue
router.put('/queue/:id/:calcFn', function(req, res) {
try {
calcQueueSvc.enqueue(req.params.id, req.params.calcFn)
res.send({ success: true })
} catch(ex) {
if (ex instanceof CalcQueueExceptions.AlreadyQueued
|| ex instanceof CalcQueueExceptions.AlreadyInProgress
|| ex instanceof CalcQueueExceptions.CalculationUnknown) {
res.status(
statusCodes.BAD_REQUEST
).send(
{ success: false, error: ex.message }
)
} else {
throw ex
}
}
})
// Remove a task or collection of tasks from the queue
router.delete('/queue/:id', function(req, res) {
calcQueueSvc.dequeue(req.params.id, null)
res.send({ success: true })
})
// Remove a task from the queue
router.delete('/queue/:id/:calcFn', function(req, res) {
calcQueueSvc.dequeue(req.params.id, req.params.calcFn)
res.send({ success: true })
})
// Get the status of a task as a text string
const processStatus = function(id, calcFn) {
if (calcQueueSvc.isInProgress(id, calcFn)) {
return 'in_progress'
} else if (calcQueueSvc.isQueued(id, calcFn)) {
return 'queued'
} else if (calcQueueSvc.isFinished(id, calcFn)) {
return 'finished'
} else {
return 'not_queued'
}
}
// Get the status of the queue and all tasks
router.get('/status', function(req, res) {
res.status(statusCodes.OK).send({
'queued': calcQueueSvc.getQueue(),
'in_progress': calcQueueSvc.getInProgress(),
'finished': calcQueueSvc.getFinished()
})
})
// Get the status of all tasks with by id
router.get('/status/:id', function(req, res) {
res.status(statusCodes.OK).send({
'id': req.params.id,
'status': processStatus(req.params.id, null)
})
})
// Get the status of an individual task
router.get('/status/:id/:calcFn', function(req, res) {
res.status(statusCodes.OK).send({
'id': req.params.id,
'calcFn': req.params.calcFn,
'status': processStatus(req.params.id, req.params.calcFn)
})
})
module.exports = router

37
routes/index.js Normal file
View File

@@ -0,0 +1,37 @@
const dotenv = require('dotenv')
const express = require('express')
const statusCodes = require('http-status-codes').StatusCodes
const createError = require('http-errors')
dotenv.config()
const router = express.Router()
// TODO: Add a mechanism for multiple clients/API keys
const apiKeys = [
process.env.API_KEY
]
// Middleware to verify API access is granted
router.use('/', function(req, res, next){
var key = req.query['api_key'];
// key isn't present
if (!key) {
return next(createError(statusCodes.BAD_REQUEST, 'api key required'))
}
// key is invalid
if (!~apiKeys.indexOf(key)) {
return next(createError(statusCodes.UNAUTHORIZED, 'invalid api key'))
}
// all good, store req.key for route access
req.key = key
next()
})
// Add API routes to default router
router.use('/', require('./calc-queue.js'))
module.exports = router

4
src/index.js Normal file
View File

@@ -0,0 +1,4 @@
module.exports = {
'services': require('./services')
}

View File

@@ -0,0 +1,191 @@
const EventEmitter = require('events')
const threads = require('worker_threads')
const path = require('path')
const fs = require('fs')
const Exceptions = require('./exceptions.js')
const CalcTask = require('./CalcTask.js')
// Basic utility functions to help the queue manager
const CalcQueueUtils = {
getCalcWorker: function(calcFn) {
return path.resolve(path.join('src/workers', calcFn + '.js'))
},
isWorkerValid: function(workerPath) {
return fs.existsSync(workerPath)
},
// Start the next task if there are any queued
processNext: function() {
if (this.queue.length == 0) {
return false
}
/* TODO: work out if the process can be run in parallel with outer
* processes or must be done sequentially
*/
const task = this.queue.shift()
task._worker = new threads.Worker(task.workerPath())
task._worker.on('message', (msg) => {
task._messages.push(msg)
})
task._worker.on('error', (err) => {
task._error = err
task._messages.push('Error: ' + err.message)
})
task._worker.on('exit', (exitCode) => {
task._result = exitCode
task._worker = null
const idx = this.inProgress.findIndex((t) => {
return t.id() == task.id() && t.calcFn() == task.calcFn()
})
this.inProgress.splice(idx, 1)
this.finished.push(task)
console.log(task)
})
this.inProgress.push(task)
}
}
// A calculation task. Only one of these per app
/* Allows calculations to be queued as tasks. Currently tasks are ryun
* synchronously.
* TODO: Add async tasks.
*/
const CalcQueue = function() {
if (!(this instanceof CalcQueue)) {
return new CalcQueue()
}
this.queue = []
this.inProgress = []
this.finished = []
return this
}
CalcQueue.prototype = {
isBusy: function() {
return this.inProgress.length > 0;
},
// Report if a task or collection of tasks by id are finished
isFinished: function(id, calcFn = null) {
if (calcFn == null) {
const fin = this.finished.filter((el) => {
return el.id() == id
}).reduce((agg, el) => {
return agg && el.result() != null
}, true)
return fin
} else {
const idx = this.finished.findIndex((el) => {
return el.id() == id && (calcFn == null || el.calcFn() == calcFn)
})
return idx != -1
}
},
// Get the list of finished tasks
getFinished: function() {
return this.finished.map((q) => {
return {
'id': q.id(),
'calcFn': q.calcFn(),
'result': q.result(),
'messages': q.messages(),
'error': (q.error() == null ? null : q.error().message)
}
})
},
// Report if a task or collection of tasks by id are in progress
isInProgress: function(id, calcFn = null) {
const idx = this.inProgress.findIndex((el) => {
return el.id() == id && (calcFn == null || el.calcFn() == calcFn)
})
return idx != -1
},
// Get the list of in-progress tasks
getInProgress: function() {
return this.inProgress.map((q) => {
return { 'id': q.id(), 'calcFn': q.calcFn() }
})
},
// Report if a task or collection of tasks by id are queued
isQueued: function(id, calcFn = null) {
return (this.queuePosition(id, calcFn) != -1)
},
// Report the position of a task in the queue
queuePosition: function(id, calcFn = null) {
return this.queue.findIndex((el) => {
return el.id() == id && (calcFn == null || el.calcFn() == calcFn)
})
},
// Get the list of queued tasks
getQueue: function() {
return this.queue.map((q) => {
return { 'id': q.id(), 'calcFn': q.calcFn() }
})
},
// Remove all tasks from the queue, does not affect running tasks
clearQueue: function() {
var okay = true
while (this.queue.length > 0) {
okay |= this.dequeue(this.queue[0].id)
this.queue.splice(0, 1)
}
return okay
},
// Add a new task to the queue if it does not already exist
enqueue: function(id, calcFn) {
if (this.isQueued(id, calcFn)) {
throw new Exceptions.AlreadyQueued()
}
if (this.isInProgress(id, calcFn)) {
throw new Exceptions.AlreadyInProgress()
}
const workerPath = CalcQueueUtils.getCalcWorker(calcFn)
if (!CalcQueueUtils.isWorkerValid(workerPath)) {
throw new Exceptions.CalculationUnknown()
}
this.queue.push(new CalcTask(id, calcFn, workerPath))
if (!this.isBusy()) {
CalcQueueUtils.processNext.call(this)
}
return true
},
// Remove a task from the queue if it is queued
dequeue: function(id, calcFn) {
var idx = this.queuePosition(id, calcFn)
if (idx == -1) {
return false
}
// TODO: if item is in process, wait for it to finish
while (idx != -1) {
this.queue.splice(idx, 1)
idx = this.queuePosition(id, calcFn)
}
return true
}
// TODO: find, store & list worker scripts on start up instead of on request?
}
module.exports = new CalcQueue()

View File

@@ -0,0 +1,50 @@
// A calculation task.
/* Contains the queued calculation and the results post calculation as well as
* any output messages from the task
*/
const CalcTask = function(id, calcFn, workerPath) {
if (!(this instanceof CalcTask)) {
return new CalcTask(id, calcFn, workerPath);
}
this._id = id
this._calcFn = calcFn
this._workerPath = workerPath
this._result = null
this._messages = []
this._error = null
this._worker = null
return this
}
CalcTask.prototype = {
id: function() {
return this._id
},
calcFn: function() {
return this._calcFn
},
workerPath: function() {
return this._workerPath
},
isInProgress: function() {
return this._worker != null
},
isFinished: function() {
return this._result != null
},
messages: function() {
return this._messages
},
error: function() {
return this._error
},
result: function() {
return this._result
}
}
module.exports = CalcTask

View File

@@ -0,0 +1,4 @@
module.exports.AlreadyQueued = require('./exceptions/AlreadyQueued.js')
module.exports.AlreadyInProgress = require('./exceptions/AlreadyInProgress.js')
module.exports.CalculationUnknown = require('./exceptions/CalculationUnknown.js')

View File

@@ -0,0 +1,11 @@
module.exports = function AlreadyInProgress(extra) {
Error.captureStackTrace(this, this.constructor)
this.name = this.constructor.name
this.message = 'calculation request is already in progress'
if (typeof extra !== 'undefined') {
this.extra = extra
}
}
require('util').inherits(module.exports, Error)

View File

@@ -0,0 +1,11 @@
module.exports = function AlreadyQueued(extra) {
Error.captureStackTrace(this, this.constructor)
this.name = this.constructor.name
this.message = 'calculation request is already queued'
if (typeof extra !== 'undefined') {
this.extra = extra
}
};
require('util').inherits(module.exports, Error)

View File

@@ -0,0 +1,11 @@
module.exports = function CalculationUnknown(extra) {
Error.captureStackTrace(this, this.constructor)
this.name = this.constructor.name
this.message = 'calculation requested is unknown'
if (typeof extra !== 'undefined') {
this.extra = extra
}
};
require('util').inherits(module.exports, Error)

View File

@@ -0,0 +1,3 @@
module.exports = require('./CalcQueue.js')
module.exports.CalcQueueExceptions = require('./exceptions.js')

4
src/services/index.js Normal file
View File

@@ -0,0 +1,4 @@
module.exports = {
'calcQueue': require('./calc-queue')
}

15
src/workers/example.js Normal file
View File

@@ -0,0 +1,15 @@
console.log('This is an example worker')
new Promise((resolve, reject) => {
setTimeout((() => {
console.log('I did something (nothing) for 5 seconds!');
resolve()
}).bind(this), 5000)
})
if (!module.parent) {
console.log('not a module')
} else {
console.log('loaded as a module')
}