From ecbf93c068e706bae20d1f86e359ed5c8db8b5b9 Mon Sep 17 00:00:00 2001 From: Brian Broll Date: Sun, 11 Nov 2018 15:45:22 -0800 Subject: [PATCH 01/45] WIP #1186 rough refactor of the job execution backend --- src/common/api/JobLogsClient.js | 1 + src/common/execution/backends/GME.js | 74 +++++++ src/common/execution/backends/Local.js | 5 + src/common/execution/index.js | 9 + src/plugins/ExecuteJob/ExecuteJob.js | 293 ++++++++++++++----------- 5 files changed, 249 insertions(+), 133 deletions(-) create mode 100644 src/common/execution/backends/GME.js create mode 100644 src/common/execution/backends/Local.js create mode 100644 src/common/execution/index.js diff --git a/src/common/api/JobLogsClient.js b/src/common/api/JobLogsClient.js index 69a56b2e2..359e475ae 100644 --- a/src/common/api/JobLogsClient.js +++ b/src/common/api/JobLogsClient.js @@ -91,6 +91,7 @@ define([ this._modifiedJobs.push(jobId); this.logger.info(`Appending logs to ${jobId}`); + metadata.lineCount = metadata.lineCount || 0; // TODO FIXME: TMP HACK if (metadata && !hasRequiredFields(metadata)) { throw Error(`Required metadata fields: ${METADATA_FIELDS.join(', ')}`); } diff --git a/src/common/execution/backends/GME.js b/src/common/execution/backends/GME.js new file mode 100644 index 000000000..35a8d4c80 --- /dev/null +++ b/src/common/execution/backends/GME.js @@ -0,0 +1,74 @@ +define([ + 'executor/ExecutorClient' +], function( + ExecutorClient +) { + // TODO + const GMEExecutor = function(logger, gmeConfig) { + const isHttps = typeof window === 'undefined' ? false : + window.location.protocol !== 'http:'; + + this.logger = logger.fork('GME'); + this.pollInterval = 1500; + this.executor = new ExecutorClient({ + logger: this.logger, + serverPort: gmeConfig.server.port, + httpsecure: isHttps + }); + + this._events = {}; // FIXME: there must be a better way... + }; + + GMEExecutor.prototype.cancelJob = function(job) { + return this.executor.cancelJob(job.hash, job.secret); + }; + + // TODO: Standardize this + GMEExecutor.prototype.getInfo = function(job) { + return this.executor.getInfo(job.hash); + }; + + GMEExecutor.prototype.createJob = async function(hash) { + const result = await this.executor.createJob({hash}); + + this.startPolling(hash); + // When to stop polling? + // TODO + + return result; + }; + + GMEExecutor.prototype.on = function(ev, cb) { + this._events[ev] = this._events[ev] || []; + this._events[ev].push(cb); + }; + + GMEExecutor.prototype.emit = function(ev) { + const args = Array.prototype.slice.call(arguments, 1); + const handlers = this._events[ev] || []; + handlers.forEach(fn => fn.apply(this, args)); + }; + + GMEExecutor.prototype.startPolling = async function(id) { + const info = await this.executor.getInfo(id); + + // Check for new stdout. Emit 'data' with the content + // TODO + + if (info.status === 'CREATED' || info.status === 'RUNNING') { + setTimeout(() => this.startPolling(id), this.pollInterval); + } else { + this.emit('end', id, info); + } + }; + + // What is the API for the executor? + // It should "push" the data to the client + // - createJob + // - cancelJob + // - getInfo + // - getOutput + // TODO + + return GMEExecutor; +}); diff --git a/src/common/execution/backends/Local.js b/src/common/execution/backends/Local.js new file mode 100644 index 000000000..2bebd88c1 --- /dev/null +++ b/src/common/execution/backends/Local.js @@ -0,0 +1,5 @@ +define([ +], function( +) { + // TODO +}); diff --git a/src/common/execution/index.js b/src/common/execution/index.js new file mode 100644 index 000000000..4a66189e4 --- /dev/null +++ b/src/common/execution/index.js @@ -0,0 +1,9 @@ +define([ + './backends/GME' +], function( + GME +) { + // FIXME: Add more intelligent interface here... + // - fetch a given backend and configure + return GME; +}); diff --git a/src/plugins/ExecuteJob/ExecuteJob.js b/src/plugins/ExecuteJob/ExecuteJob.js index 7df57f2ea..d93edcf40 100644 --- a/src/plugins/ExecuteJob/ExecuteJob.js +++ b/src/plugins/ExecuteJob/ExecuteJob.js @@ -4,7 +4,7 @@ define([ 'common/util/assert', 'text!./metadata.json', - 'executor/ExecutorClient', + 'deepforge/execution/index', 'plugin/PluginBase', 'deepforge/ExecutionEnv', 'deepforge/plugin/LocalExecutor', @@ -95,19 +95,30 @@ define([ port: this.gmeConfig.server.port, branchName: this.branchName, projectId: this.projectId - }, - isHttps = typeof window === 'undefined' ? false : - window.location.protocol !== 'http:'; + }; this.logManager = new JobLogsClient(params); this.originManager = new JobOriginClient(params); this.pulseClient = new ExecPulseClient(params); + this._execHashToJobNode = {}; + + // TODO: load a custom executor + this.executor = new ExecutorClient(this.logger, this.gmeConfig); + // Look up the job from the id... + // TODO + this.executor.on('data', + (id, data) => this.onConsoleOutput(job, data) + ); + this.executor.on('end', + (id, info) => { + try { + this.onOperationEnd(id, info); + } catch (err) { + this.logger.error(`Error when processing operation end: ${err}`); + } + } + ); - this.executor = new ExecutorClient({ - logger: this.logger, - serverPort: this.gmeConfig.server.port, - httpsecure: isHttps - }); return result; }; @@ -213,6 +224,7 @@ define([ }; ExecuteJob.prototype.resumeJob = function (job) { + console.log('resuming job...'); var hash = this.getAttribute(job, 'jobId'), name = this.getAttribute(job, 'name'), id = this.core.getPath(job), @@ -234,7 +246,7 @@ define([ }) .then(count => { // update line count (to inform logClient appendTo) this.outputLineCount[id] = count; - return this.executor.getOutput(hash, 0, count); + return this.executor.getOutput(hash, 0, count); // TODO FIXME }) .then(output => { // parse the stdout to update the job metadata var stdout = output.map(o => o.output).join(''), @@ -248,8 +260,8 @@ define([ promise = this.save(msg); } return promise.then(() => this.getOperation(job)); - }) - .then(opNode => this.watchOperation(hash, opNode, job)); + }); + //.then(opNode => this.watchOperation(hash, opNode, job)); }; ExecuteJob.prototype.updateForkName = function (basename) { @@ -317,7 +329,7 @@ define([ this.setAttribute(job, 'status', 'canceled'); this.resultMsg(msg); - this.onComplete(op, null); + return this.onComplete(op, null); }; ExecuteJob.prototype.onOperationFail = @@ -372,7 +384,7 @@ define([ } this.createMessage(null, msg); - promise + return promise .then(() => this.save(msg)) .then(() => { this.result.setSuccess(!err); @@ -440,7 +452,7 @@ define([ }); }; - ExecuteJob.prototype.executeDistOperation = function (job, opNode, hash) { + ExecuteJob.prototype.executeDistOperation = async function (job, opNode, hash) { var name = this.getAttribute(opNode, 'name'), jobId = this.core.getPath(job); @@ -453,26 +465,45 @@ define([ this.logManager.deleteLog(jobId); this.logger.info(`Setting ${jobId} status to "queued" (${this.currentHash})`); this.logger.debug(`Making a commit from ${this.currentHash}`); - this.save(`Queued "${name}" operation in ${this.pipelineName}`) - .then(() => this.executor.createJob({hash})) - .then(info => { - this.setAttribute(job, 'jobId', info.hash); - if (info.secret) { // o.w. it is a cached job! - this.setAttribute(job, 'secret', info.secret); - } - if (!this.currentRunId) { - this.currentRunId = info.hash; - if (this._beating === null) { - this.startExecHeartBeat(); - } - } - return this.recordJobOrigin(hash, job); - }) - .then(() => this.watchOperation(hash, opNode, job)) - .catch(err => this.logger.error(`Could not execute "${name}": ${err}`)); + + try { + await this.save(`Queued "${name}" operation in ${this.pipelineName}`); + await this.createJob(job, opNode, hash); + } catch (err) { + this.logger.error(`Could not execute "${name}": ${err}`); + } }; + ExecuteJob.prototype.createJob = async function (job, opNode, hash) { + // Record the job info for the given hash + this._execHashToJobNode[hash] = [job, opNode]; + const info = this.executor.createJob(hash); + this.setAttribute(job, 'jobId', info.hash); + // Store the entire info object? This could be used for canceling! + // TODO + this.setAttribute(job, 'jobInfo', JSON.stringify(info)); + if (!this.currentRunId) { + this.currentRunId = info.hash; + if (this._beating === null) { + this.startExecHeartBeat(); + } + } + + // Add event handlers to the executor + // TODO + // Should the metadata handlers be here or somewhere else? + // Probably somewhere else (in the executor client/base class) + // TODO + + // Don't set these callbacks up for each operation... + return await this.recordJobOrigin(hash, job); + }; + + ExecuteJob.prototype.getNodesForJobHash = function (hash) { + return this._execHashToJobNode[hash]; + }; + ExecuteJob.prototype.recordJobOrigin = function (hash, job) { var execNode = this.core.getParent(job), info; @@ -487,6 +518,17 @@ define([ return this.originManager.record(hash, info); }; + ExecuteJob.prototype.cleanJobHashInfo = function (hash) { + const i = this.runningJobHashes.indexOf(hash); + if (i !== -1) { + this.runningJobHashes.splice(i, 1); + } else { + this.logger.warn(`Could not find running job hash ${hash}`); + } + + delete this._execHashToJobNode[hash]; + }; + ExecuteJob.prototype.notifyStdoutUpdate = function (nodeId) { this.sendNotification({ @@ -528,72 +570,60 @@ define([ }); }; + ExecuteJob.prototype.onConsoleOutput = function (job, newOutput) { + var stdout = this.getAttribute(job, 'stdout'), + last = stdout.lastIndexOf('\n'), + result, + lastLine, + next = Q(), + msg; + + // parse deepforge commands + if (last !== -1) { + stdout = stdout.substring(0, last+1); + lastLine = stdout.substring(last+1); + output = lastLine + output; + } + result = this.processStdout(job, output, true); + output = result.stdout; + + if (output) { + // Send notification to all clients watching the branch + next = next + .then(() => this.logManager.appendTo(jobId, output)) + .then(() => this.notifyStdoutUpdate(jobId)); + } + + if (result.hasMetadata) { + msg = `Updated graph/image output for ${name}`; + next = next.then(() => this.save(msg)); + } + return next; + }; + ExecuteJob.prototype.watchOperation = function (hash, op, job) { var jobId = this.core.getPath(job), opId = this.core.getPath(op), info, - secret, + jobInfo = this.getAttribute(job, 'jobInfo'), name = this.getAttribute(job, 'name'); // If canceled, stop the operation if (this.canceled || this.isExecutionCanceled()) { - secret = this.getAttribute(job, 'secret'); - if (secret) { - this.executor.cancelJob(hash, secret); - this.delAttribute(job, 'secret'); + if (jobInfo) { + // Will all cancelJob signatures look like this? + // Should they be passing a custom JSON object? + // TODO + this.executor.cancelJob(jobInfo); // TODO FIXME + this.delAttribute(job, 'jobInfo'); this.canceled = true; return this.onOperationCanceled(op); } } - return this.executor.getInfo(hash) - .then(_info => { // Update the job's stdout - var actualLine, // on executing job - currentLine = this.outputLineCount[jobId], - prep = Q(); - - info = _info; - actualLine = info.outputNumber; - if (actualLine !== null && actualLine >= currentLine) { - this.outputLineCount[jobId] = actualLine + 1; - return prep - .then(() => this.executor.getOutput(hash, currentLine, actualLine+1)) - .then(outputLines => { - var stdout = this.getAttribute(job, 'stdout'), - output = outputLines.map(o => o.output).join(''), - last = stdout.lastIndexOf('\n'), - result, - lastLine, - next = Q(), - msg; - - // parse deepforge commands - if (last !== -1) { - stdout = stdout.substring(0, last+1); - lastLine = stdout.substring(last+1); - output = lastLine + output; - } - result = this.processStdout(job, output, true); - output = result.stdout; - - if (output) { - // Send notification to all clients watching the branch - var metadata = { - lineCount: this.outputLineCount[jobId] - }; - next = next - .then(() => this.logManager.appendTo(jobId, output, metadata)) - .then(() => this.notifyStdoutUpdate(jobId)); - } - if (result.hasMetadata) { - msg = `Updated graph/image output for ${name}`; - next = next.then(() => this.save(msg)); - } - return next; - }); - } - }) - .then(() => { + return this.executor.getInfo(jobInfo) // TODO FIXME + .then(info => { // Update the job's stdout + if (info.status === 'CREATED' || info.status === 'RUNNING') { var time = Date.now(), next = Q(); @@ -619,58 +649,55 @@ define([ }); } - // Record that the job hash is no longer running - this.logger.info(`Job "${name}" has finished (${info.status})`); - var i = this.runningJobHashes.indexOf(hash); - if (i !== -1) { - this.runningJobHashes.splice(i, 1); - } else { - this.logger.warn(`Could not find running job hash ${hash}`); - } - - if (info.status === 'CANCELED') { - // If it was cancelled, the pipeline has been stopped - this.logger.debug(`"${name}" has been CANCELED!`); - this.canceled = true; - return this.logManager.getLog(jobId) - .then(stdout => { - this.setAttribute(job, 'stdout', stdout); - return this.onOperationCanceled(op); - }); - } - - if (info.status === 'SUCCESS' || info.status === 'FAILED_TO_EXECUTE') { - this.setAttribute(job, 'execFiles', info.resultHashes[name + '-all-files']); - return this.blobClient.getArtifact(info.resultHashes.stdout) - .then(artifact => { - var stdoutHash = artifact.descriptor.content[STDOUT_FILE].content; - return this.blobClient.getObjectAsString(stdoutHash); - }) - .then(stdout => { - // Parse the remaining code - var result = this.processStdout(job, stdout); - this.setAttribute(job, 'stdout', result.stdout); - this.logManager.deleteLog(jobId); - if (info.status !== 'SUCCESS') { - // Download all files - this.result.addArtifact(info.resultHashes[name + '-all-files']); - // Set the job to failed! Store the error - this.onOperationFail(op, `Operation "${opId}" failed! ${JSON.stringify(info)}`); - } else { - this.onDistOperationComplete(op, info); - } - }); - } else { // something bad happened... - var err = `Failed to execute operation "${opId}": ${info.status}`, - consoleErr = `Failed to execute operation: ${info.status}`; - this.setAttribute(job, 'stdout', consoleErr); - this.logger.error(err); - this.onOperationFail(op, err); - } }) .catch(err => this.logger.error(`Could not get op info for ${opId}: ${err}`)); }; + ExecuteJob.prototype.onOperationEnd = async function (hash, info) { + // Record that the job hash is no longer running + const [job, op] = this.getNodesForJobHash(hash); + const name = this.getAttribute(job, 'name'); + const jobId = this.core.getPath(job); + + this.logger.info(`Job "${name}" has finished (${info.status})`); + this.cleanJobHashInfo(hash); + + if (info.status === 'CANCELED') { + // If it was cancelled, the pipeline has been stopped + this.logger.debug(`"${name}" has been CANCELED!`); + this.canceled = true; + const stdout = this.logManager.getLog(jobId) + this.setAttribute(job, 'stdout', stdout); + return this.onOperationCanceled(op); + } + + if (info.status === 'SUCCESS' || info.status === 'FAILED_TO_EXECUTE') { + // TODO: + this.setAttribute(job, 'execFiles', info.resultHashes[name + '-all-files']); + const artifact = await this.blobClient.getArtifact(info.resultHashes.stdout) + const stdoutHash = artifact.descriptor.content[STDOUT_FILE].content; + const stdout = await this.blobClient.getObjectAsString(stdoutHash); + // Parse the remaining code + const result = this.processStdout(job, stdout); + this.setAttribute(job, 'stdout', result.stdout); + this.logManager.deleteLog(jobId); + if (info.status !== 'SUCCESS') { + // Download all files + this.result.addArtifact(info.resultHashes[name + '-all-files']); + // Set the job to failed! Store the error + this.onOperationFail(op, `Operation "${opId}" failed! ${JSON.stringify(info)}`); + } else { + this.onDistOperationComplete(op, info); + } + } else { // something bad happened... + var err = `Failed to execute operation "${opId}": ${info.status}`, + consoleErr = `Failed to execute operation: ${info.status}`; + this.setAttribute(job, 'stdout', consoleErr); + this.logger.error(err); + return this.onOperationFail(op, err); + } + }; + ExecuteJob.prototype.onDistOperationComplete = function (node, result) { let nodeId = this.core.getPath(node), outputMap = {}, From 66b39b99b796077b57bb2470dec7e1e2028d9f6e Mon Sep 17 00:00:00 2001 From: Brian Broll Date: Fri, 23 Nov 2018 09:44:09 -0800 Subject: [PATCH 02/45] WIP #1186 added more support for local execution --- src/common/execution/backends/Local.js | 130 ++++++++++++++++++++++++- src/common/execution/index.js | 8 +- src/plugins/ExecuteJob/ExecuteJob.js | 35 +++---- 3 files changed, 152 insertions(+), 21 deletions(-) diff --git a/src/common/execution/backends/Local.js b/src/common/execution/backends/Local.js index 2bebd88c1..09c686810 100644 --- a/src/common/execution/backends/Local.js +++ b/src/common/execution/backends/Local.js @@ -1,5 +1,133 @@ define([ + './BaseExecutor', + 'blob/BlobClient', + 'child_process', + 'rimraf', + 'fs', + 'os', + 'path', ], function( + BaseExecutor, + BlobClient, + childProcess, + rimraf, + fs, + os, + path, ) { - // TODO + // TODO: Show an error if not running on the server... + + const spawn = childProcess.spawn; + const {promisify} = require.nodeRequire('util'); + const mkdir = promisify(fs.mkdir); + const rm_rf = promisify(rimraf); + const writeFile = promisify(fs.writeFile); + const readFile = promisify(fs.readFile); + const execFile = promisify(childProcess.execFile); + + // UNZIP must be available on the machine, first ensure that it exists... + ensureHasUnzip(); + const UNZIP_EXE = '/usr/bin/unzip'; // FIXME: more platform support + const UNZIP_ARGS = ['-o']; // FIXME: more platform support + + const LocalExecutor = function(logger, gmeConfig) { + BaseExecutor.apply(this, arguments); + // FIXME: set this meaningfully! + this.blobClient = new BlobClient({ + server: '127.0.0.1', + serverPort: gmeConfig.server.port, + httpsecure: false, + logger: this.logger.fork('BlobClient') + }); + }; + + LocalExecutor.prototype = Object.create(BaseExecutor.prototype); + + //LocalExecutor.prototype.cancelJob = function(job) { + //return this.executor.cancelJob(job.hash, job.secret); + //}; + + //LocalExecutor.prototype.getInfo = function(job) { + //return this.executor.getInfo(job.hash); + //}; + + // TODO: Add cancel support! TODO + LocalExecutor.prototype.createJob = async function(hash) { + + // TODO: Set up a directory to work in... + // Create tmp directory + const tmpdir = path.join(os.tmpdir(), `deepforge-local-exec-${hash}`); + try { + await mkdir(tmpdir); + } catch (err) { + if (err.code === 'EEXIST') { + await rm_rf(tmpdir); + await mkdir(tmpdir); + } else { + throw err; + } + } + console.log('created working directory at', tmpdir); + + // Fetch the required files from deepforge + try { + await this.prepareWorkspace(hash, tmpdir); + } catch (err) { + console.log(`Error: ${err}`); + } + + // Spin up a subprocess + const config = JSON.parse(await readFile(tmpdir.replace(path.sep, '/') + '/executor_config.json', 'utf8')); + console.log('config:', config); + + const env = {cwd: tmpdir}; + execJob = spawn(config.cmd, config.args, env); + execJob.stdout.on('data', data => this.emit('data', hash, data)); // TODO: should this be stdout? + execJob.stderr.on('data', data => this.emit('data', hash, data)); + execJob.on('close', code => { + const jobInfo = { + resultHashes: [], // TODO: upload data and add result hashes + status: 'SUCCESS' + } + + if (code === 0) { // Success + // TODO: upload data and record hashes.. + } else { + jobInfo.status = 'FAILED_TO_EXECUTE'; + } + this.emit('end', hash, jobInfo); + }); + + // upload the resultArtifacts + // TODO + + //return result; + }; + + LocalExecutor.prototype.prepareWorkspace = async function(hash, dirname) { + this.logger.info(`about to fetch job data`); + const content = new Buffer(new Uint8Array(await this.blobClient.getObject(hash))); // TODO: Handle errors... + const zipPath = path.join(dirname, `${hash}.zip`); + await writeFile(zipPath, content); + this.logger.info(`Fetched job data: ${zipPath}`); + + this.logger.info(`unzipping ${zipPath} in ${dirname}`); + await unzip(zipPath, dirname); + }; + + async function unzip(filename, dirname) { + const args = UNZIP_ARGS.concat(path.basename(filename)); + console.log('running:', UNZIP_EXE, args.join(' ')); + await execFile(UNZIP_EXE, args, {cwd: dirname}); + + await rm_rf(filename); + } + + function ensureHasUnzip() { + // FIXME: check for unzip here! + } + // - [ ] emit updates on stdout... + + return LocalExecutor; + }); diff --git a/src/common/execution/index.js b/src/common/execution/index.js index 4a66189e4..fb2477342 100644 --- a/src/common/execution/index.js +++ b/src/common/execution/index.js @@ -1,9 +1,11 @@ define([ - './backends/GME' + './backends/GME', + './backends/Local' ], function( - GME + GME, + Local ) { // FIXME: Add more intelligent interface here... // - fetch a given backend and configure - return GME; + return Local; }); diff --git a/src/plugins/ExecuteJob/ExecuteJob.js b/src/plugins/ExecuteJob/ExecuteJob.js index d93edcf40..5aa168fdf 100644 --- a/src/plugins/ExecuteJob/ExecuteJob.js +++ b/src/plugins/ExecuteJob/ExecuteJob.js @@ -106,8 +106,12 @@ define([ this.executor = new ExecutorClient(this.logger, this.gmeConfig); // Look up the job from the id... // TODO - this.executor.on('data', - (id, data) => this.onConsoleOutput(job, data) + this.executor.on( + 'data', + (id, data) => { + const [job] = this.getNodesForJobHash(id); + this.onConsoleOutput(job, data.toString()); + } ); this.executor.on('end', (id, info) => { @@ -570,35 +574,32 @@ define([ }); }; - ExecuteJob.prototype.onConsoleOutput = function (job, newOutput) { + ExecuteJob.prototype.onConsoleOutput = async function (job, output) { + const jobId = this.core.getPath(job); var stdout = this.getAttribute(job, 'stdout'), last = stdout.lastIndexOf('\n'), result, lastLine, - next = Q(), msg; // parse deepforge commands - if (last !== -1) { - stdout = stdout.substring(0, last+1); - lastLine = stdout.substring(last+1); - output = lastLine + output; - } + // FIXME: This needs to be added back! + //if (last !== -1) { + //stdout = stdout.substring(0, last+1); + //lastLine = stdout.substring(last+1); + //output = lastLine + output; + //} result = this.processStdout(job, output, true); output = result.stdout; - if (output) { - // Send notification to all clients watching the branch - next = next - .then(() => this.logManager.appendTo(jobId, output)) - .then(() => this.notifyStdoutUpdate(jobId)); - } + await this.logManager.appendTo(jobId, output); + // Send notification to all clients watching the branch + await this.notifyStdoutUpdate(jobId); if (result.hasMetadata) { msg = `Updated graph/image output for ${name}`; - next = next.then(() => this.save(msg)); + await this.save(msg); } - return next; }; ExecuteJob.prototype.watchOperation = function (hash, op, job) { From 1441cd502be07807d3802ad0059f52a54e94c5c4 Mon Sep 17 00:00:00 2001 From: Brian Broll Date: Sat, 12 Jan 2019 08:54:15 -0800 Subject: [PATCH 03/45] WIP extra comments and tmp workarounds --- src/common/api/JobLogsClient.js | 4 +++- src/common/execution/backends/Local.js | 12 ++++++++++++ src/routers/JobLogsAPI/JobLogsAPI.js | 2 ++ 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/src/common/api/JobLogsClient.js b/src/common/api/JobLogsClient.js index 359e475ae..622abf0c9 100644 --- a/src/common/api/JobLogsClient.js +++ b/src/common/api/JobLogsClient.js @@ -91,7 +91,9 @@ define([ this._modifiedJobs.push(jobId); this.logger.info(`Appending logs to ${jobId}`); - metadata.lineCount = metadata.lineCount || 0; // TODO FIXME: TMP HACK + if (metadata) { + metadata.lineCount = metadata.lineCount || 0; // TODO FIXME: TMP HACK + } if (metadata && !hasRequiredFields(metadata)) { throw Error(`Required metadata fields: ${METADATA_FIELDS.join(', ')}`); } diff --git a/src/common/execution/backends/Local.js b/src/common/execution/backends/Local.js index 09c686810..825d7a939 100644 --- a/src/common/execution/backends/Local.js +++ b/src/common/execution/backends/Local.js @@ -92,6 +92,7 @@ define([ if (code === 0) { // Success // TODO: upload data and record hashes.. + jobInfo.resultHashes = await this._uploadResults(tmpdir, config); } else { jobInfo.status = 'FAILED_TO_EXECUTE'; } @@ -104,6 +105,17 @@ define([ //return result; }; + LocalExecutor.prototype._uploadResults = async function(workdir, config) { + // Get all the matching result artifacts + // TODO + + // Upload all the artifacts + // TODO + + // Return the hashes + // TODO + }; + LocalExecutor.prototype.prepareWorkspace = async function(hash, dirname) { this.logger.info(`about to fetch job data`); const content = new Buffer(new Uint8Array(await this.blobClient.getObject(hash))); // TODO: Handle errors... diff --git a/src/routers/JobLogsAPI/JobLogsAPI.js b/src/routers/JobLogsAPI/JobLogsAPI.js index d816166b6..a11869b4d 100644 --- a/src/routers/JobLogsAPI/JobLogsAPI.js +++ b/src/routers/JobLogsAPI/JobLogsAPI.js @@ -76,6 +76,8 @@ function initialize(middlewareOpts) { project: req.params.project, branch: req.params.branch, job: req.params.job, + // Do I need the lineCount??? + // TODO FIXME lineCount: req.body.lineCount || -1, createdIds: req.body.createdIds || [], cmdCount: req.body.cmdCount || 0 From 856fffe94caeda918dc4c7fc1ee327e05e3669c0 Mon Sep 17 00:00:00 2001 From: Brian Broll Date: Thu, 28 Feb 2019 21:48:38 -0800 Subject: [PATCH 04/45] WIP added symlink to node_modules in local worker --- package-lock.json | 3921 +----------------------- package.json | 2 +- src/common/ExecutionEnv.js | 1 + src/common/execution/backends/Local.js | 40 +- src/plugins/ExecuteJob/ExecuteJob.js | 4 + 5 files changed, 154 insertions(+), 3814 deletions(-) diff --git a/package-lock.json b/package-lock.json index 6080d1bfe..e9c68a053 100644 --- a/package-lock.json +++ b/package-lock.json @@ -144,24 +144,6 @@ "resolved": "https://registry.npmjs.org/after/-/after-0.8.2.tgz", "integrity": "sha1-/ts5T58OAqqXaOcCvaI7UF+ufh8=" }, - "agent-base": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-2.1.1.tgz", - "integrity": "sha1-1t4Q1a9hMtW9aSQn1G/FOFOQlMc=", - "dev": true, - "requires": { - "extend": "3.0.1", - "semver": "5.0.3" - }, - "dependencies": { - "semver": { - "version": "5.0.3", - "resolved": "https://registry.npmjs.org/semver/-/semver-5.0.3.tgz", - "integrity": "sha1-d0Zt5YnNXTyV8TiqeLxWmjy10no=", - "dev": true - } - } - }, "agentkeepalive": { "version": "3.3.0", "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-3.3.0.tgz", @@ -170,24 +152,6 @@ "humanize-ms": "1.2.1" } }, - "ajv": { - "version": "5.5.2", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-5.5.2.tgz", - "integrity": "sha1-c7Xuyj+rZT49P5Qis0GtQiBdyWU=", - "dev": true, - "requires": { - "co": "4.6.0", - "fast-deep-equal": "1.0.0", - "fast-json-stable-stringify": "2.0.0", - "json-schema-traverse": "0.3.1" - } - }, - "amdefine": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/amdefine/-/amdefine-1.0.1.tgz", - "integrity": "sha1-SlKCrBZHKek2Gbz9OtFR+BfOkfU=", - "dev": true - }, "ansi-align": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ansi-align/-/ansi-align-2.0.0.tgz", @@ -230,12 +194,6 @@ } } }, - "ansi-escapes": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-3.1.0.tgz", - "integrity": "sha512-UgAb8H9D41AQnu/PbWlCofQVcnV4Gs2bBJi9eZPxfU/hgglFh3SMDMENRIqdr7H6XFnXdoknctFByVsCOotTVw==", - "dev": true - }, "ansi-regex": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", @@ -345,15 +303,6 @@ "resolved": "https://registry.npmjs.org/array-reduce/-/array-reduce-0.0.0.tgz", "integrity": "sha1-FziZ0//Rx9k4PkR5Ul2+J4yrXys=" }, - "array-union": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/array-union/-/array-union-1.0.2.tgz", - "integrity": "sha1-mjRBDk9OPaI96jdb5b5w8kd47Dk=", - "dev": true, - "requires": { - "array-uniq": "1.0.3" - } - }, "array-uniq": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/array-uniq/-/array-uniq-1.0.3.tgz", @@ -374,12 +323,6 @@ "resolved": "https://registry.npmjs.org/asap/-/asap-2.0.6.tgz", "integrity": "sha1-5QNHYR1+aQlDIIu9r+vLwvuGbUY=" }, - "asn1": { - "version": "0.2.3", - "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.3.tgz", - "integrity": "sha1-2sh4dxPJlmhJ/IGAd36+nB3fO4Y=", - "dev": true - }, "asn1.js": { "version": "4.10.1", "resolved": "https://registry.npmjs.org/asn1.js/-/asn1.js-4.10.1.tgz", @@ -398,12 +341,6 @@ "util": "0.10.3" } }, - "assert-plus": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", - "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=", - "dev": true - }, "assertion-error": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-1.0.2.tgz", @@ -439,12 +376,6 @@ "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=" }, - "atob": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/atob/-/atob-1.1.3.tgz", - "integrity": "sha1-lfE2KbEsOlGl0hWr3OKqnzL4B3M=", - "dev": true - }, "aws-sdk": { "version": "2.142.0", "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.142.0.tgz", @@ -469,801 +400,6 @@ } } }, - "aws-sign2": { - "version": "0.7.0", - "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz", - "integrity": "sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg=", - "dev": true - }, - "aws4": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.6.0.tgz", - "integrity": "sha1-g+9cqGCysy5KDe7e6MdxudtXRx4=", - "dev": true - }, - "babel-code-frame": { - "version": "6.26.0", - "resolved": "https://registry.npmjs.org/babel-code-frame/-/babel-code-frame-6.26.0.tgz", - "integrity": "sha1-Y/1D99weO7fONZR9uP42mj9Yx0s=", - "dev": true, - "requires": { - "chalk": "1.1.3", - "esutils": "2.0.2", - "js-tokens": "3.0.2" - } - }, - "babel-core": { - "version": "6.26.0", - "resolved": "https://registry.npmjs.org/babel-core/-/babel-core-6.26.0.tgz", - "integrity": "sha1-rzL3izGm/O8RnIew/Y2XU/A6C7g=", - "dev": true, - "requires": { - "babel-code-frame": "6.26.0", - "babel-generator": "6.26.1", - "babel-helpers": "6.24.1", - "babel-messages": "6.23.0", - "babel-register": "6.26.0", - "babel-runtime": "6.26.0", - "babel-template": "6.26.0", - "babel-traverse": "6.26.0", - "babel-types": "6.26.0", - "babylon": "6.18.0", - "convert-source-map": "1.5.1", - "debug": "2.6.9", - "json5": "0.5.1", - "lodash": "4.17.4", - "minimatch": "3.0.4", - "path-is-absolute": "1.0.1", - "private": "0.1.8", - "slash": "1.0.0", - "source-map": "0.5.7" - }, - "dependencies": { - "babylon": { - "version": "6.18.0", - "resolved": "https://registry.npmjs.org/babylon/-/babylon-6.18.0.tgz", - "integrity": "sha512-q/UEjfGJ2Cm3oKV71DJz9d25TPnq5rhBVL2Q4fA5wcC3jcrdn7+SssEybFIxwAvvP+YCsCYNKughoF33GxgycQ==", - "dev": true - }, - "convert-source-map": { - "version": "1.5.1", - "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-1.5.1.tgz", - "integrity": "sha1-uCeAl7m8IpNl3lxiz1/K7YtVmeU=", - "dev": true - } - } - }, - "babel-generator": { - "version": "6.26.1", - "resolved": "https://registry.npmjs.org/babel-generator/-/babel-generator-6.26.1.tgz", - "integrity": "sha512-HyfwY6ApZj7BYTcJURpM5tznulaBvyio7/0d4zFOeMPUmfxkCjHocCuoLa2SAGzBI8AREcH3eP3758F672DppA==", - "dev": true, - "requires": { - "babel-messages": "6.23.0", - "babel-runtime": "6.26.0", - "babel-types": "6.26.0", - "detect-indent": "4.0.0", - "jsesc": "1.3.0", - "lodash": "4.17.4", - "source-map": "0.5.7", - "trim-right": "1.0.1" - } - }, - "babel-helper-bindify-decorators": { - "version": "6.24.1", - "resolved": "https://registry.npmjs.org/babel-helper-bindify-decorators/-/babel-helper-bindify-decorators-6.24.1.tgz", - "integrity": "sha1-FMGeXxQte0fxmlJDHlKxzLxAozA=", - "dev": true, - "requires": { - "babel-runtime": "6.26.0", - "babel-traverse": "6.26.0", - "babel-types": "6.26.0" - } - }, - "babel-helper-builder-binary-assignment-operator-visitor": { - "version": "6.24.1", - "resolved": "https://registry.npmjs.org/babel-helper-builder-binary-assignment-operator-visitor/-/babel-helper-builder-binary-assignment-operator-visitor-6.24.1.tgz", - "integrity": "sha1-zORReto1b0IgvK6KAsKzRvmlZmQ=", - "dev": true, - "requires": { - "babel-helper-explode-assignable-expression": "6.24.1", - "babel-runtime": "6.26.0", - "babel-types": "6.26.0" - } - }, - "babel-helper-call-delegate": { - "version": "6.24.1", - "resolved": "https://registry.npmjs.org/babel-helper-call-delegate/-/babel-helper-call-delegate-6.24.1.tgz", - "integrity": "sha1-7Oaqzdx25Bw0YfiL/Fdb0Nqi340=", - "dev": true, - "requires": { - "babel-helper-hoist-variables": "6.24.1", - "babel-runtime": "6.26.0", - "babel-traverse": "6.26.0", - "babel-types": "6.26.0" - } - }, - "babel-helper-define-map": { - "version": "6.26.0", - "resolved": "https://registry.npmjs.org/babel-helper-define-map/-/babel-helper-define-map-6.26.0.tgz", - "integrity": "sha1-pfVtq0GiX5fstJjH66ypgZ+Vvl8=", - "dev": true, - "requires": { - "babel-helper-function-name": "6.24.1", - "babel-runtime": "6.26.0", - "babel-types": "6.26.0", - "lodash": "4.17.4" - } - }, - "babel-helper-explode-assignable-expression": { - "version": "6.24.1", - "resolved": "https://registry.npmjs.org/babel-helper-explode-assignable-expression/-/babel-helper-explode-assignable-expression-6.24.1.tgz", - "integrity": "sha1-8luCz33BBDPFX3BZLVdGQArCLKo=", - "dev": true, - "requires": { - "babel-runtime": "6.26.0", - "babel-traverse": "6.26.0", - "babel-types": "6.26.0" - } - }, - "babel-helper-explode-class": { - "version": "6.24.1", - "resolved": "https://registry.npmjs.org/babel-helper-explode-class/-/babel-helper-explode-class-6.24.1.tgz", - "integrity": "sha1-fcKjkQ3uAHBW4eMdZAztPVTqqes=", - "dev": true, - "requires": { - "babel-helper-bindify-decorators": "6.24.1", - "babel-runtime": "6.26.0", - "babel-traverse": "6.26.0", - "babel-types": "6.26.0" - } - }, - "babel-helper-function-name": { - "version": "6.24.1", - "resolved": "https://registry.npmjs.org/babel-helper-function-name/-/babel-helper-function-name-6.24.1.tgz", - "integrity": "sha1-00dbjAPtmCQqJbSDUasYOZ01gKk=", - "dev": true, - "requires": { - "babel-helper-get-function-arity": "6.24.1", - "babel-runtime": "6.26.0", - "babel-template": "6.26.0", - "babel-traverse": "6.26.0", - "babel-types": "6.26.0" - } - }, - "babel-helper-get-function-arity": { - "version": "6.24.1", - "resolved": "https://registry.npmjs.org/babel-helper-get-function-arity/-/babel-helper-get-function-arity-6.24.1.tgz", - "integrity": "sha1-j3eCqpNAfEHTqlCQj4mwMbG2hT0=", - "dev": true, - "requires": { - "babel-runtime": "6.26.0", - "babel-types": "6.26.0" - } - }, - "babel-helper-hoist-variables": { - "version": "6.24.1", - "resolved": "https://registry.npmjs.org/babel-helper-hoist-variables/-/babel-helper-hoist-variables-6.24.1.tgz", - "integrity": "sha1-HssnaJydJVE+rbyZFKc/VAi+enY=", - "dev": true, - "requires": { - "babel-runtime": "6.26.0", - "babel-types": "6.26.0" - } - }, - "babel-helper-optimise-call-expression": { - "version": "6.24.1", - "resolved": "https://registry.npmjs.org/babel-helper-optimise-call-expression/-/babel-helper-optimise-call-expression-6.24.1.tgz", - "integrity": "sha1-96E0J7qfc/j0+pk8VKl4gtEkQlc=", - "dev": true, - "requires": { - "babel-runtime": "6.26.0", - "babel-types": "6.26.0" - } - }, - "babel-helper-regex": { - "version": "6.26.0", - "resolved": "https://registry.npmjs.org/babel-helper-regex/-/babel-helper-regex-6.26.0.tgz", - "integrity": "sha1-MlxZ+QL4LyS3T6zu0DY5VPZJXnI=", - "dev": true, - "requires": { - "babel-runtime": "6.26.0", - "babel-types": "6.26.0", - "lodash": "4.17.4" - } - }, - "babel-helper-remap-async-to-generator": { - "version": "6.24.1", - "resolved": "https://registry.npmjs.org/babel-helper-remap-async-to-generator/-/babel-helper-remap-async-to-generator-6.24.1.tgz", - "integrity": "sha1-XsWBgnrXI/7N04HxySg5BnbkVRs=", - "dev": true, - "requires": { - "babel-helper-function-name": "6.24.1", - "babel-runtime": "6.26.0", - "babel-template": "6.26.0", - "babel-traverse": "6.26.0", - "babel-types": "6.26.0" - } - }, - "babel-helper-replace-supers": { - "version": "6.24.1", - "resolved": "https://registry.npmjs.org/babel-helper-replace-supers/-/babel-helper-replace-supers-6.24.1.tgz", - "integrity": "sha1-v22/5Dk40XNpohPKiov3S2qQqxo=", - "dev": true, - "requires": { - "babel-helper-optimise-call-expression": "6.24.1", - "babel-messages": "6.23.0", - "babel-runtime": "6.26.0", - "babel-template": "6.26.0", - "babel-traverse": "6.26.0", - "babel-types": "6.26.0" - } - }, - "babel-helpers": { - "version": "6.24.1", - "resolved": "https://registry.npmjs.org/babel-helpers/-/babel-helpers-6.24.1.tgz", - "integrity": "sha1-NHHenK7DiOXIUOWX5Yom3fN2ArI=", - "dev": true, - "requires": { - "babel-runtime": "6.26.0", - "babel-template": "6.26.0" - } - }, - "babel-messages": { - "version": "6.23.0", - "resolved": "https://registry.npmjs.org/babel-messages/-/babel-messages-6.23.0.tgz", - "integrity": "sha1-8830cDhYA1sqKVHG7F7fbGLyYw4=", - "dev": true, - "requires": { - "babel-runtime": "6.26.0" - } - }, - "babel-plugin-check-es2015-constants": { - "version": "6.22.0", - "resolved": "https://registry.npmjs.org/babel-plugin-check-es2015-constants/-/babel-plugin-check-es2015-constants-6.22.0.tgz", - "integrity": "sha1-NRV7EBQm/S/9PaP3XH0ekYNbv4o=", - "dev": true, - "requires": { - "babel-runtime": "6.26.0" - } - }, - "babel-plugin-syntax-async-functions": { - "version": "6.13.0", - "resolved": "https://registry.npmjs.org/babel-plugin-syntax-async-functions/-/babel-plugin-syntax-async-functions-6.13.0.tgz", - "integrity": "sha1-ytnK0RkbWtY0vzCuCHI5HgZHvpU=", - "dev": true - }, - "babel-plugin-syntax-async-generators": { - "version": "6.13.0", - "resolved": "https://registry.npmjs.org/babel-plugin-syntax-async-generators/-/babel-plugin-syntax-async-generators-6.13.0.tgz", - "integrity": "sha1-a8lj67FuzLrmuStZbrfzXDQqi5o=", - "dev": true - }, - "babel-plugin-syntax-class-properties": { - "version": "6.13.0", - "resolved": "https://registry.npmjs.org/babel-plugin-syntax-class-properties/-/babel-plugin-syntax-class-properties-6.13.0.tgz", - "integrity": "sha1-1+sjt5oxf4VDlixQW4J8fWysJ94=", - "dev": true - }, - "babel-plugin-syntax-decorators": { - "version": "6.13.0", - "resolved": "https://registry.npmjs.org/babel-plugin-syntax-decorators/-/babel-plugin-syntax-decorators-6.13.0.tgz", - "integrity": "sha1-MSVjtNvePMgGzuPkFszurd0RrAs=", - "dev": true - }, - "babel-plugin-syntax-dynamic-import": { - "version": "6.18.0", - "resolved": "https://registry.npmjs.org/babel-plugin-syntax-dynamic-import/-/babel-plugin-syntax-dynamic-import-6.18.0.tgz", - "integrity": "sha1-jWomIpyDdFqZgqRBBRVyyqF5sdo=", - "dev": true - }, - "babel-plugin-syntax-exponentiation-operator": { - "version": "6.13.0", - "resolved": "https://registry.npmjs.org/babel-plugin-syntax-exponentiation-operator/-/babel-plugin-syntax-exponentiation-operator-6.13.0.tgz", - "integrity": "sha1-nufoM3KQ2pUoggGmpX9BcDF4MN4=", - "dev": true - }, - "babel-plugin-syntax-object-rest-spread": { - "version": "6.13.0", - "resolved": "https://registry.npmjs.org/babel-plugin-syntax-object-rest-spread/-/babel-plugin-syntax-object-rest-spread-6.13.0.tgz", - "integrity": "sha1-/WU28rzhODb/o6VFjEkDpZe7O/U=", - "dev": true - }, - "babel-plugin-syntax-trailing-function-commas": { - "version": "6.22.0", - "resolved": "https://registry.npmjs.org/babel-plugin-syntax-trailing-function-commas/-/babel-plugin-syntax-trailing-function-commas-6.22.0.tgz", - "integrity": "sha1-ugNgk3+NBuQBgKQ/4NVhb/9TLPM=", - "dev": true - }, - "babel-plugin-transform-async-generator-functions": { - "version": "6.24.1", - "resolved": "https://registry.npmjs.org/babel-plugin-transform-async-generator-functions/-/babel-plugin-transform-async-generator-functions-6.24.1.tgz", - "integrity": "sha1-8FiQAUX9PpkHpt3yjaWfIVJYpds=", - "dev": true, - "requires": { - "babel-helper-remap-async-to-generator": "6.24.1", - "babel-plugin-syntax-async-generators": "6.13.0", - "babel-runtime": "6.26.0" - } - }, - "babel-plugin-transform-async-to-generator": { - "version": "6.24.1", - "resolved": "https://registry.npmjs.org/babel-plugin-transform-async-to-generator/-/babel-plugin-transform-async-to-generator-6.24.1.tgz", - "integrity": "sha1-ZTbjeK/2yx1VF6wOQOs+n8jQh2E=", - "dev": true, - "requires": { - "babel-helper-remap-async-to-generator": "6.24.1", - "babel-plugin-syntax-async-functions": "6.13.0", - "babel-runtime": "6.26.0" - } - }, - "babel-plugin-transform-class-properties": { - "version": "6.24.1", - "resolved": "https://registry.npmjs.org/babel-plugin-transform-class-properties/-/babel-plugin-transform-class-properties-6.24.1.tgz", - "integrity": "sha1-anl2PqYdM9NvN7YRqp3vgagbRqw=", - "dev": true, - "requires": { - "babel-helper-function-name": "6.24.1", - "babel-plugin-syntax-class-properties": "6.13.0", - "babel-runtime": "6.26.0", - "babel-template": "6.26.0" - } - }, - "babel-plugin-transform-decorators": { - "version": "6.24.1", - "resolved": "https://registry.npmjs.org/babel-plugin-transform-decorators/-/babel-plugin-transform-decorators-6.24.1.tgz", - "integrity": "sha1-eIAT2PjGtSIr33s0Q5Df13Vp4k0=", - "dev": true, - "requires": { - "babel-helper-explode-class": "6.24.1", - "babel-plugin-syntax-decorators": "6.13.0", - "babel-runtime": "6.26.0", - "babel-template": "6.26.0", - "babel-types": "6.26.0" - } - }, - "babel-plugin-transform-es2015-arrow-functions": { - "version": "6.22.0", - "resolved": "https://registry.npmjs.org/babel-plugin-transform-es2015-arrow-functions/-/babel-plugin-transform-es2015-arrow-functions-6.22.0.tgz", - "integrity": "sha1-RSaSy3EdX3ncf4XkQM5BufJE0iE=", - "dev": true, - "requires": { - "babel-runtime": "6.26.0" - } - }, - "babel-plugin-transform-es2015-block-scoped-functions": { - "version": "6.22.0", - "resolved": "https://registry.npmjs.org/babel-plugin-transform-es2015-block-scoped-functions/-/babel-plugin-transform-es2015-block-scoped-functions-6.22.0.tgz", - "integrity": "sha1-u8UbSflk1wy42OC5ToICRs46YUE=", - "dev": true, - "requires": { - "babel-runtime": "6.26.0" - } - }, - "babel-plugin-transform-es2015-block-scoping": { - "version": "6.26.0", - "resolved": "https://registry.npmjs.org/babel-plugin-transform-es2015-block-scoping/-/babel-plugin-transform-es2015-block-scoping-6.26.0.tgz", - "integrity": "sha1-1w9SmcEwjQXBL0Y4E7CgnnOxiV8=", - "dev": true, - "requires": { - "babel-runtime": "6.26.0", - "babel-template": "6.26.0", - "babel-traverse": "6.26.0", - "babel-types": "6.26.0", - "lodash": "4.17.4" - } - }, - "babel-plugin-transform-es2015-classes": { - "version": "6.24.1", - "resolved": "https://registry.npmjs.org/babel-plugin-transform-es2015-classes/-/babel-plugin-transform-es2015-classes-6.24.1.tgz", - "integrity": "sha1-WkxYpQyclGHlZLSyo7+ryXolhNs=", - "dev": true, - "requires": { - "babel-helper-define-map": "6.26.0", - "babel-helper-function-name": "6.24.1", - "babel-helper-optimise-call-expression": "6.24.1", - "babel-helper-replace-supers": "6.24.1", - "babel-messages": "6.23.0", - "babel-runtime": "6.26.0", - "babel-template": "6.26.0", - "babel-traverse": "6.26.0", - "babel-types": "6.26.0" - } - }, - "babel-plugin-transform-es2015-computed-properties": { - "version": "6.24.1", - "resolved": "https://registry.npmjs.org/babel-plugin-transform-es2015-computed-properties/-/babel-plugin-transform-es2015-computed-properties-6.24.1.tgz", - "integrity": "sha1-b+Ko0WiV1WNPTNmZttNICjCBWbM=", - "dev": true, - "requires": { - "babel-runtime": "6.26.0", - "babel-template": "6.26.0" - } - }, - "babel-plugin-transform-es2015-destructuring": { - "version": "6.23.0", - "resolved": "https://registry.npmjs.org/babel-plugin-transform-es2015-destructuring/-/babel-plugin-transform-es2015-destructuring-6.23.0.tgz", - "integrity": "sha1-mXux8auWf2gtKwh2/jWNYOdlxW0=", - "dev": true, - "requires": { - "babel-runtime": "6.26.0" - } - }, - "babel-plugin-transform-es2015-duplicate-keys": { - "version": "6.24.1", - "resolved": "https://registry.npmjs.org/babel-plugin-transform-es2015-duplicate-keys/-/babel-plugin-transform-es2015-duplicate-keys-6.24.1.tgz", - "integrity": "sha1-c+s9MQypaePvnskcU3QabxV2Qj4=", - "dev": true, - "requires": { - "babel-runtime": "6.26.0", - "babel-types": "6.26.0" - } - }, - "babel-plugin-transform-es2015-for-of": { - "version": "6.23.0", - "resolved": "https://registry.npmjs.org/babel-plugin-transform-es2015-for-of/-/babel-plugin-transform-es2015-for-of-6.23.0.tgz", - "integrity": "sha1-9HyVsrYT3x0+zC/bdXNiPHUkhpE=", - "dev": true, - "requires": { - "babel-runtime": "6.26.0" - } - }, - "babel-plugin-transform-es2015-function-name": { - "version": "6.24.1", - "resolved": "https://registry.npmjs.org/babel-plugin-transform-es2015-function-name/-/babel-plugin-transform-es2015-function-name-6.24.1.tgz", - "integrity": "sha1-g0yJhTvDaxrw86TF26qU/Y6sqos=", - "dev": true, - "requires": { - "babel-helper-function-name": "6.24.1", - "babel-runtime": "6.26.0", - "babel-types": "6.26.0" - } - }, - "babel-plugin-transform-es2015-literals": { - "version": "6.22.0", - "resolved": "https://registry.npmjs.org/babel-plugin-transform-es2015-literals/-/babel-plugin-transform-es2015-literals-6.22.0.tgz", - "integrity": "sha1-T1SgLWzWbPkVKAAZox0xklN3yi4=", - "dev": true, - "requires": { - "babel-runtime": "6.26.0" - } - }, - "babel-plugin-transform-es2015-modules-amd": { - "version": "6.24.1", - "resolved": "https://registry.npmjs.org/babel-plugin-transform-es2015-modules-amd/-/babel-plugin-transform-es2015-modules-amd-6.24.1.tgz", - "integrity": "sha1-Oz5UAXI5hC1tGcMBHEvS8AoA0VQ=", - "dev": true, - "requires": { - "babel-plugin-transform-es2015-modules-commonjs": "6.26.0", - "babel-runtime": "6.26.0", - "babel-template": "6.26.0" - } - }, - "babel-plugin-transform-es2015-modules-commonjs": { - "version": "6.26.0", - "resolved": "https://registry.npmjs.org/babel-plugin-transform-es2015-modules-commonjs/-/babel-plugin-transform-es2015-modules-commonjs-6.26.0.tgz", - "integrity": "sha1-DYOUApt9xqvhqX7xgeAHWN0uXYo=", - "dev": true, - "requires": { - "babel-plugin-transform-strict-mode": "6.24.1", - "babel-runtime": "6.26.0", - "babel-template": "6.26.0", - "babel-types": "6.26.0" - } - }, - "babel-plugin-transform-es2015-modules-systemjs": { - "version": "6.24.1", - "resolved": "https://registry.npmjs.org/babel-plugin-transform-es2015-modules-systemjs/-/babel-plugin-transform-es2015-modules-systemjs-6.24.1.tgz", - "integrity": "sha1-/4mhQrkRmpBhlfXxBuzzBdlAfSM=", - "dev": true, - "requires": { - "babel-helper-hoist-variables": "6.24.1", - "babel-runtime": "6.26.0", - "babel-template": "6.26.0" - } - }, - "babel-plugin-transform-es2015-modules-umd": { - "version": "6.24.1", - "resolved": "https://registry.npmjs.org/babel-plugin-transform-es2015-modules-umd/-/babel-plugin-transform-es2015-modules-umd-6.24.1.tgz", - "integrity": "sha1-rJl+YoXNGO1hdq22B9YCNErThGg=", - "dev": true, - "requires": { - "babel-plugin-transform-es2015-modules-amd": "6.24.1", - "babel-runtime": "6.26.0", - "babel-template": "6.26.0" - } - }, - "babel-plugin-transform-es2015-object-super": { - "version": "6.24.1", - "resolved": "https://registry.npmjs.org/babel-plugin-transform-es2015-object-super/-/babel-plugin-transform-es2015-object-super-6.24.1.tgz", - "integrity": "sha1-JM72muIcuDp/hgPa0CH1cusnj40=", - "dev": true, - "requires": { - "babel-helper-replace-supers": "6.24.1", - "babel-runtime": "6.26.0" - } - }, - "babel-plugin-transform-es2015-parameters": { - "version": "6.24.1", - "resolved": "https://registry.npmjs.org/babel-plugin-transform-es2015-parameters/-/babel-plugin-transform-es2015-parameters-6.24.1.tgz", - "integrity": "sha1-V6w1GrScrxSpfNE7CfZv3wpiXys=", - "dev": true, - "requires": { - "babel-helper-call-delegate": "6.24.1", - "babel-helper-get-function-arity": "6.24.1", - "babel-runtime": "6.26.0", - "babel-template": "6.26.0", - "babel-traverse": "6.26.0", - "babel-types": "6.26.0" - } - }, - "babel-plugin-transform-es2015-shorthand-properties": { - "version": "6.24.1", - "resolved": "https://registry.npmjs.org/babel-plugin-transform-es2015-shorthand-properties/-/babel-plugin-transform-es2015-shorthand-properties-6.24.1.tgz", - "integrity": "sha1-JPh11nIch2YbvZmkYi5R8U3jiqA=", - "dev": true, - "requires": { - "babel-runtime": "6.26.0", - "babel-types": "6.26.0" - } - }, - "babel-plugin-transform-es2015-spread": { - "version": "6.22.0", - "resolved": "https://registry.npmjs.org/babel-plugin-transform-es2015-spread/-/babel-plugin-transform-es2015-spread-6.22.0.tgz", - "integrity": "sha1-1taKmfia7cRTbIGlQujdnxdG+NE=", - "dev": true, - "requires": { - "babel-runtime": "6.26.0" - } - }, - "babel-plugin-transform-es2015-sticky-regex": { - "version": "6.24.1", - "resolved": "https://registry.npmjs.org/babel-plugin-transform-es2015-sticky-regex/-/babel-plugin-transform-es2015-sticky-regex-6.24.1.tgz", - "integrity": "sha1-AMHNsaynERLN8M9hJsLta0V8zbw=", - "dev": true, - "requires": { - "babel-helper-regex": "6.26.0", - "babel-runtime": "6.26.0", - "babel-types": "6.26.0" - } - }, - "babel-plugin-transform-es2015-template-literals": { - "version": "6.22.0", - "resolved": "https://registry.npmjs.org/babel-plugin-transform-es2015-template-literals/-/babel-plugin-transform-es2015-template-literals-6.22.0.tgz", - "integrity": "sha1-qEs0UPfp+PH2g51taH2oS7EjbY0=", - "dev": true, - "requires": { - "babel-runtime": "6.26.0" - } - }, - "babel-plugin-transform-es2015-typeof-symbol": { - "version": "6.23.0", - "resolved": "https://registry.npmjs.org/babel-plugin-transform-es2015-typeof-symbol/-/babel-plugin-transform-es2015-typeof-symbol-6.23.0.tgz", - "integrity": "sha1-3sCfHN3/lLUqxz1QXITfWdzOs3I=", - "dev": true, - "requires": { - "babel-runtime": "6.26.0" - } - }, - "babel-plugin-transform-es2015-unicode-regex": { - "version": "6.24.1", - "resolved": "https://registry.npmjs.org/babel-plugin-transform-es2015-unicode-regex/-/babel-plugin-transform-es2015-unicode-regex-6.24.1.tgz", - "integrity": "sha1-04sS9C6nMj9yk4fxinxa4frrNek=", - "dev": true, - "requires": { - "babel-helper-regex": "6.26.0", - "babel-runtime": "6.26.0", - "regexpu-core": "2.0.0" - } - }, - "babel-plugin-transform-exponentiation-operator": { - "version": "6.24.1", - "resolved": "https://registry.npmjs.org/babel-plugin-transform-exponentiation-operator/-/babel-plugin-transform-exponentiation-operator-6.24.1.tgz", - "integrity": "sha1-KrDJx/MJj6SJB3cruBP+QejeOg4=", - "dev": true, - "requires": { - "babel-helper-builder-binary-assignment-operator-visitor": "6.24.1", - "babel-plugin-syntax-exponentiation-operator": "6.13.0", - "babel-runtime": "6.26.0" - } - }, - "babel-plugin-transform-object-rest-spread": { - "version": "6.26.0", - "resolved": "https://registry.npmjs.org/babel-plugin-transform-object-rest-spread/-/babel-plugin-transform-object-rest-spread-6.26.0.tgz", - "integrity": "sha1-DzZpLVD+9rfi1LOsFHgTepY7ewY=", - "dev": true, - "requires": { - "babel-plugin-syntax-object-rest-spread": "6.13.0", - "babel-runtime": "6.26.0" - } - }, - "babel-plugin-transform-regenerator": { - "version": "6.26.0", - "resolved": "https://registry.npmjs.org/babel-plugin-transform-regenerator/-/babel-plugin-transform-regenerator-6.26.0.tgz", - "integrity": "sha1-4HA2lvveJ/Cj78rPi03KL3s6jy8=", - "dev": true, - "requires": { - "regenerator-transform": "0.10.1" - } - }, - "babel-plugin-transform-runtime": { - "version": "6.23.0", - "resolved": "https://registry.npmjs.org/babel-plugin-transform-runtime/-/babel-plugin-transform-runtime-6.23.0.tgz", - "integrity": "sha1-iEkNRGUC6puOfvsP4J7E2ZR5se4=", - "dev": true, - "requires": { - "babel-runtime": "6.26.0" - } - }, - "babel-plugin-transform-strict-mode": { - "version": "6.24.1", - "resolved": "https://registry.npmjs.org/babel-plugin-transform-strict-mode/-/babel-plugin-transform-strict-mode-6.24.1.tgz", - "integrity": "sha1-1fr3qleKZbvlkc9e2uBKDGcCB1g=", - "dev": true, - "requires": { - "babel-runtime": "6.26.0", - "babel-types": "6.26.0" - } - }, - "babel-polyfill": { - "version": "6.26.0", - "resolved": "https://registry.npmjs.org/babel-polyfill/-/babel-polyfill-6.26.0.tgz", - "integrity": "sha1-N5k3q8Z9eJWXCtxiHyhM2WbPIVM=", - "dev": true, - "requires": { - "babel-runtime": "6.26.0", - "core-js": "2.5.4", - "regenerator-runtime": "0.10.5" - }, - "dependencies": { - "regenerator-runtime": { - "version": "0.10.5", - "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.10.5.tgz", - "integrity": "sha1-M2w+/BIgrc7dosn6tntaeVWjNlg=", - "dev": true - } - } - }, - "babel-preset-es2015": { - "version": "6.24.1", - "resolved": "https://registry.npmjs.org/babel-preset-es2015/-/babel-preset-es2015-6.24.1.tgz", - "integrity": "sha1-1EBQ1rwsn+6nAqrzjXJ6AhBTiTk=", - "dev": true, - "requires": { - "babel-plugin-check-es2015-constants": "6.22.0", - "babel-plugin-transform-es2015-arrow-functions": "6.22.0", - "babel-plugin-transform-es2015-block-scoped-functions": "6.22.0", - "babel-plugin-transform-es2015-block-scoping": "6.26.0", - "babel-plugin-transform-es2015-classes": "6.24.1", - "babel-plugin-transform-es2015-computed-properties": "6.24.1", - "babel-plugin-transform-es2015-destructuring": "6.23.0", - "babel-plugin-transform-es2015-duplicate-keys": "6.24.1", - "babel-plugin-transform-es2015-for-of": "6.23.0", - "babel-plugin-transform-es2015-function-name": "6.24.1", - "babel-plugin-transform-es2015-literals": "6.22.0", - "babel-plugin-transform-es2015-modules-amd": "6.24.1", - "babel-plugin-transform-es2015-modules-commonjs": "6.26.0", - "babel-plugin-transform-es2015-modules-systemjs": "6.24.1", - "babel-plugin-transform-es2015-modules-umd": "6.24.1", - "babel-plugin-transform-es2015-object-super": "6.24.1", - "babel-plugin-transform-es2015-parameters": "6.24.1", - "babel-plugin-transform-es2015-shorthand-properties": "6.24.1", - "babel-plugin-transform-es2015-spread": "6.22.0", - "babel-plugin-transform-es2015-sticky-regex": "6.24.1", - "babel-plugin-transform-es2015-template-literals": "6.22.0", - "babel-plugin-transform-es2015-typeof-symbol": "6.23.0", - "babel-plugin-transform-es2015-unicode-regex": "6.24.1", - "babel-plugin-transform-regenerator": "6.26.0" - } - }, - "babel-preset-stage-2": { - "version": "6.24.1", - "resolved": "https://registry.npmjs.org/babel-preset-stage-2/-/babel-preset-stage-2-6.24.1.tgz", - "integrity": "sha1-2eKWD7PXEYfw5k7sYrwHdnIZvcE=", - "dev": true, - "requires": { - "babel-plugin-syntax-dynamic-import": "6.18.0", - "babel-plugin-transform-class-properties": "6.24.1", - "babel-plugin-transform-decorators": "6.24.1", - "babel-preset-stage-3": "6.24.1" - } - }, - "babel-preset-stage-3": { - "version": "6.24.1", - "resolved": "https://registry.npmjs.org/babel-preset-stage-3/-/babel-preset-stage-3-6.24.1.tgz", - "integrity": "sha1-g2raCp56f6N8sTj7kyb4eTSkg5U=", - "dev": true, - "requires": { - "babel-plugin-syntax-trailing-function-commas": "6.22.0", - "babel-plugin-transform-async-generator-functions": "6.24.1", - "babel-plugin-transform-async-to-generator": "6.24.1", - "babel-plugin-transform-exponentiation-operator": "6.24.1", - "babel-plugin-transform-object-rest-spread": "6.26.0" - } - }, - "babel-register": { - "version": "6.26.0", - "resolved": "https://registry.npmjs.org/babel-register/-/babel-register-6.26.0.tgz", - "integrity": "sha1-btAhFz4vy0htestFxgCahW9kcHE=", - "dev": true, - "requires": { - "babel-core": "6.26.0", - "babel-runtime": "6.26.0", - "core-js": "2.5.4", - "home-or-tmp": "2.0.0", - "lodash": "4.17.4", - "mkdirp": "0.5.1", - "source-map-support": "0.4.18" - } - }, - "babel-runtime": { - "version": "6.26.0", - "resolved": "https://registry.npmjs.org/babel-runtime/-/babel-runtime-6.26.0.tgz", - "integrity": "sha1-llxwWGaOgrVde/4E/yM3vItWR/4=", - "dev": true, - "requires": { - "core-js": "2.5.4", - "regenerator-runtime": "0.11.1" - } - }, - "babel-template": { - "version": "6.26.0", - "resolved": "https://registry.npmjs.org/babel-template/-/babel-template-6.26.0.tgz", - "integrity": "sha1-3gPi0WOWsGn0bdn/+FIfsaDjXgI=", - "dev": true, - "requires": { - "babel-runtime": "6.26.0", - "babel-traverse": "6.26.0", - "babel-types": "6.26.0", - "babylon": "6.18.0", - "lodash": "4.17.4" - }, - "dependencies": { - "babylon": { - "version": "6.18.0", - "resolved": "https://registry.npmjs.org/babylon/-/babylon-6.18.0.tgz", - "integrity": "sha512-q/UEjfGJ2Cm3oKV71DJz9d25TPnq5rhBVL2Q4fA5wcC3jcrdn7+SssEybFIxwAvvP+YCsCYNKughoF33GxgycQ==", - "dev": true - } - } - }, - "babel-traverse": { - "version": "6.26.0", - "resolved": "https://registry.npmjs.org/babel-traverse/-/babel-traverse-6.26.0.tgz", - "integrity": "sha1-RqnL1+3MYsjlwGTi0tjQ9ANXZu4=", - "dev": true, - "requires": { - "babel-code-frame": "6.26.0", - "babel-messages": "6.23.0", - "babel-runtime": "6.26.0", - "babel-types": "6.26.0", - "babylon": "6.18.0", - "debug": "2.6.9", - "globals": "9.18.0", - "invariant": "2.2.4", - "lodash": "4.17.4" - }, - "dependencies": { - "babylon": { - "version": "6.18.0", - "resolved": "https://registry.npmjs.org/babylon/-/babylon-6.18.0.tgz", - "integrity": "sha512-q/UEjfGJ2Cm3oKV71DJz9d25TPnq5rhBVL2Q4fA5wcC3jcrdn7+SssEybFIxwAvvP+YCsCYNKughoF33GxgycQ==", - "dev": true - } - } - }, - "babel-types": { - "version": "6.26.0", - "resolved": "https://registry.npmjs.org/babel-types/-/babel-types-6.26.0.tgz", - "integrity": "sha1-o7Bz+Uq0nrb6Vc1lInozQ4BjJJc=", - "dev": true, - "requires": { - "babel-runtime": "6.26.0", - "esutils": "2.0.2", - "lodash": "4.17.4", - "to-fast-properties": "1.0.3" - } - }, "babylon": { "version": "7.0.0-beta.19", "resolved": "https://registry.npmjs.org/babylon/-/babylon-7.0.0-beta.19.tgz", @@ -1294,16 +430,6 @@ "resolved": "https://registry.npmjs.org/base64id/-/base64id-1.0.0.tgz", "integrity": "sha1-R2iMuZu2gE8OBtPnY7HDLlfY5rY=" }, - "bcrypt-pbkdf": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.1.tgz", - "integrity": "sha1-Y7xdy2EzG5K8Bf1SiVPDNGKgb40=", - "dev": true, - "optional": true, - "requires": { - "tweetnacl": "0.14.5" - } - }, "bcryptjs": { "version": "2.4.3", "resolved": "https://registry.npmjs.org/bcryptjs/-/bcryptjs-2.4.3.tgz", @@ -1445,15 +571,6 @@ } } }, - "boom": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/boom/-/boom-4.3.1.tgz", - "integrity": "sha1-T4owBctKfjiJ90kDD9JbluAdLjE=", - "dev": true, - "requires": { - "hoek": "4.2.1" - } - }, "bops": { "version": "0.0.7", "resolved": "https://registry.npmjs.org/bops/-/bops-0.0.7.tgz", @@ -1637,12 +754,6 @@ } } }, - "browser-stdout": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/browser-stdout/-/browser-stdout-1.3.0.tgz", - "integrity": "sha1-81HTKWnTL6XXpVZxVCY9korjvR8=", - "dev": true - }, "browserify": { "version": "14.5.0", "resolved": "https://registry.npmjs.org/browserify/-/browserify-14.5.0.tgz", @@ -1884,16 +995,6 @@ "resolved": "https://registry.npmjs.org/callsites/-/callsites-0.2.0.tgz", "integrity": "sha1-r6uWJikQp/M8GaV3WCXGnzTjUMo=" }, - "camel-case": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/camel-case/-/camel-case-3.0.0.tgz", - "integrity": "sha1-yjw2iKTpzzpM2nd9xNy8cTJJz3M=", - "dev": true, - "requires": { - "no-case": "2.3.2", - "upper-case": "1.1.3" - } - }, "camelcase": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-2.1.1.tgz", @@ -1905,12 +1006,6 @@ "integrity": "sha1-Sm+gc5nCa7pH8LJJa00PtAjFVQ0=", "dev": true }, - "caseless": { - "version": "0.12.0", - "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz", - "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw=", - "dev": true - }, "catharsis": { "version": "0.8.9", "resolved": "https://registry.npmjs.org/catharsis/-/catharsis-0.8.9.tgz", @@ -1930,28 +1025,6 @@ "type-detect": "1.0.0" } }, - "chai-as-promised": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/chai-as-promised/-/chai-as-promised-6.0.0.tgz", - "integrity": "sha1-GgKkM6byTa+sY7nJb6FoTbGqjaY=", - "dev": true, - "requires": { - "check-error": "1.0.2" - } - }, - "chalk": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", - "integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=", - "dev": true, - "requires": { - "ansi-styles": "2.2.1", - "escape-string-regexp": "1.0.5", - "has-ansi": "2.0.0", - "strip-ansi": "3.0.1", - "supports-color": "2.0.0" - } - }, "chance": { "version": "1.0.12", "resolved": "https://registry.npmjs.org/chance/-/chance-1.0.12.tgz", @@ -1965,194 +1038,28 @@ "camel-case": "1.2.2", "constant-case": "1.1.2", "dot-case": "1.1.2", - "is-lower-case": "1.1.3", - "is-upper-case": "1.1.2", - "lower-case": "1.1.4", - "lower-case-first": "1.0.2", - "param-case": "1.1.2", - "pascal-case": "1.1.2", - "path-case": "1.1.2", - "sentence-case": "1.1.3", - "snake-case": "1.1.2", - "swap-case": "1.1.2", - "title-case": "1.1.2", - "upper-case": "1.1.3", - "upper-case-first": "1.1.2" - }, - "dependencies": { - "camel-case": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/camel-case/-/camel-case-1.2.2.tgz", - "integrity": "sha1-Gsp8TRlTWaLOmVV5NDPG5VQlEfI=", - "requires": { - "sentence-case": "1.1.3", - "upper-case": "1.1.3" - } - } - } - }, - "chardet": { - "version": "0.4.2", - "resolved": "https://registry.npmjs.org/chardet/-/chardet-0.4.2.tgz", - "integrity": "sha1-tUc7M9yXxCTl2Y3IfVXU2KKci/I=", - "dev": true - }, - "check-error": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/check-error/-/check-error-1.0.2.tgz", - "integrity": "sha1-V00xLt2Iu13YkS6Sht1sCu1KrII=", - "dev": true - }, - "child-process-debug": { - "version": "0.0.7", - "resolved": "https://registry.npmjs.org/child-process-debug/-/child-process-debug-0.0.7.tgz", - "integrity": "sha1-VOEfuBw7b5Spa2MfrKk+0a9/itA=", - "dev": true - }, - "chimp": { - "version": "0.50.2", - "resolved": "https://registry.npmjs.org/chimp/-/chimp-0.50.2.tgz", - "integrity": "sha1-XI2JnVbdOls1TlH0MoDjkVXnpZI=", - "dev": true, - "requires": { - "async": "0.9.2", - "babel-core": "6.26.0", - "babel-plugin-transform-runtime": "6.23.0", - "babel-polyfill": "6.26.0", - "babel-preset-es2015": "6.24.1", - "babel-preset-stage-2": "6.24.1", - "babel-register": "6.26.0", - "babel-runtime": "6.26.0", - "bluebird": "3.5.1", - "chai": "3.5.0", - "chai-as-promised": "6.0.0", - "child-process-debug": "0.0.7", - "chokidar": "1.6.1", - "chromedriver": "2.37.0", - "colors": "1.1.2", - "commander": "2.12.2", - "cucumber": "git://github.com/xolvio/cucumber-js.git#cf953cb5b5de30dbcc279f59e4ebff3aa040071c", - "deep-extend": "0.4.2", - "exit": "0.1.2", - "fibers": "1.0.15", - "freeport": "1.0.5", - "fs-extra": "1.0.0", - "glob": "git://github.com/lucetius/node-glob.git#51c7ca6e69bfbd17db5f1ea710e3f2a7a457d9ce", - "hapi": "8.8.0", - "jasmine": "2.99.0", - "loglevel": "1.4.1", - "minimist": "1.2.0", - "mocha": "3.5.3", - "phantomjs-prebuilt": "2.1.13", - "progress": "1.1.8", - "request": "2.85.0", - "requestretry": "1.5.0", - "saucelabs": "1.4.0", - "selenium-standalone": "6.13.0", - "underscore": "1.8.3", - "xolvio-ddp": "0.12.3", - "xolvio-jasmine-expect": "1.1.0", - "xolvio-sync-webdriverio": "9.0.1" + "is-lower-case": "1.1.3", + "is-upper-case": "1.1.2", + "lower-case": "1.1.4", + "lower-case-first": "1.0.2", + "param-case": "1.1.2", + "pascal-case": "1.1.2", + "path-case": "1.1.2", + "sentence-case": "1.1.3", + "snake-case": "1.1.2", + "swap-case": "1.1.2", + "title-case": "1.1.2", + "upper-case": "1.1.3", + "upper-case-first": "1.1.2" }, "dependencies": { - "colors": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/colors/-/colors-1.1.2.tgz", - "integrity": "sha1-FopHAXVran9RoSzgyXv6KMCE7WM=", - "dev": true - }, - "cucumber": { - "version": "git://github.com/xolvio/cucumber-js.git#cf953cb5b5de30dbcc279f59e4ebff3aa040071c", - "dev": true, - "requires": { - "camel-case": "3.0.0", - "cli-table": "0.3.1", - "co": "4.6.0", - "colors": "1.1.2", - "commander": "2.12.2", - "duration": "0.2.0", - "fibers": "1.0.15", - "figures": "1.7.0", - "gherkin": "4.0.0", - "glob": "git://github.com/lucetius/node-glob.git#51c7ca6e69bfbd17db5f1ea710e3f2a7a457d9ce", - "is-generator": "1.0.3", - "lodash": "4.17.4", - "meteor-promise": "0.8.6", - "stack-chain": "1.3.7", - "stacktrace-js": "1.3.1" - } - }, - "debug": { - "version": "2.6.8", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.8.tgz", - "integrity": "sha1-5zFTHKLt4n0YgiJCfaF4IdaP9Pw=", - "dev": true, - "requires": { - "ms": "2.0.0" - } - }, - "diff": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/diff/-/diff-3.2.0.tgz", - "integrity": "sha1-yc45Okt8vQsFinJck98pkCeGj/k=", - "dev": true - }, - "glob": { - "version": "git://github.com/lucetius/node-glob.git#51c7ca6e69bfbd17db5f1ea710e3f2a7a457d9ce", - "dev": true, - "requires": { - "fs.realpath": "1.0.0", - "inflight": "1.0.6", - "inherits": "2.0.3", - "minimatch": "3.0.4", - "once": "1.3.0", - "path-is-absolute": "1.0.1" - } - }, - "minimist": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz", - "integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ=", - "dev": true - }, - "mocha": { - "version": "3.5.3", - "resolved": "https://registry.npmjs.org/mocha/-/mocha-3.5.3.tgz", - "integrity": "sha512-/6na001MJWEtYxHOV1WLfsmR4YIynkUEhBwzsb+fk2qmQ3iqsi258l/Q2MWHJMImAcNpZ8DEdYAK72NHoIQ9Eg==", - "dev": true, - "requires": { - "browser-stdout": "1.3.0", - "commander": "2.9.0", - "debug": "2.6.8", - "diff": "3.2.0", - "escape-string-regexp": "1.0.5", - "glob": "git://github.com/lucetius/node-glob.git#51c7ca6e69bfbd17db5f1ea710e3f2a7a457d9ce", - "growl": "1.9.2", - "he": "1.1.1", - "json3": "3.3.2", - "lodash.create": "3.1.1", - "mkdirp": "0.5.1", - "supports-color": "3.1.2" - }, - "dependencies": { - "commander": { - "version": "2.9.0", - "resolved": "https://registry.npmjs.org/commander/-/commander-2.9.0.tgz", - "integrity": "sha1-nJkJQXbhIkDLItbFFGCYQA/g99Q=", - "dev": true, - "requires": { - "graceful-readlink": "1.0.1" - } - } - } - }, - "supports-color": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-3.1.2.tgz", - "integrity": "sha1-cqJiiU2dQIuVbKBf83su2KbiotU=", - "dev": true, + "camel-case": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/camel-case/-/camel-case-1.2.2.tgz", + "integrity": "sha1-Gsp8TRlTWaLOmVV5NDPG5VQlEfI=", "requires": { - "has-flag": "1.0.0" + "sentence-case": "1.1.3", + "upper-case": "1.1.3" } } } @@ -2173,19 +1080,6 @@ "readdirp": "2.1.0" } }, - "chromedriver": { - "version": "2.37.0", - "resolved": "https://registry.npmjs.org/chromedriver/-/chromedriver-2.37.0.tgz", - "integrity": "sha512-Dz3ktXp+9T0ygMIEZX3SNL3grXywi2kC1swiD9cjISlLcoenzhOpsj/R/Gr2hJvrC49aGE2BhSpuUevdGq6J4w==", - "dev": true, - "requires": { - "del": "3.0.0", - "extract-zip": "1.6.6", - "kew": "0.7.0", - "mkdirp": "0.5.1", - "request": "2.85.0" - } - }, "cipher-base": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/cipher-base/-/cipher-base-1.0.4.tgz", @@ -2246,15 +1140,6 @@ "timers-ext": "0.1.5" } }, - "cli-cursor": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-2.1.0.tgz", - "integrity": "sha1-s12sN2R5+sw+lHR9QdDQ9SOP/LU=", - "dev": true, - "requires": { - "restore-cursor": "2.0.0" - } - }, "cli-table": { "version": "0.3.1", "resolved": "https://registry.npmjs.org/cli-table/-/cli-table-0.3.1.tgz", @@ -2264,12 +1149,6 @@ "colors": "1.0.3" } }, - "cli-width": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/cli-width/-/cli-width-2.2.0.tgz", - "integrity": "sha1-/xnt6Kml5XkyQUewwR8PvLq+1jk=", - "dev": true - }, "cliui": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/cliui/-/cliui-3.2.0.tgz", @@ -2280,12 +1159,6 @@ "wrap-ansi": "2.1.0" } }, - "co": { - "version": "4.6.0", - "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz", - "integrity": "sha1-bqa989hTrlTMuOR7+gvz+QMfsYQ=", - "dev": true - }, "code-point-at": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz", @@ -2536,12 +1409,6 @@ "resolved": "https://registry.npmjs.org/copy-to/-/copy-to-2.0.1.tgz", "integrity": "sha1-JoD7uAaKSNCGVrYJgJK9r8kG9KU=" }, - "core-js": { - "version": "2.5.4", - "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.5.4.tgz", - "integrity": "sha1-8si/GB8qgLkvNgEhQpzmOi8K6uA=", - "dev": true - }, "core-util-is": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", @@ -2627,26 +1494,6 @@ } } }, - "cryptiles": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/cryptiles/-/cryptiles-3.1.2.tgz", - "integrity": "sha1-qJ+7Ig9c4l7FboxKqKT9e1sNKf4=", - "dev": true, - "requires": { - "boom": "5.2.0" - }, - "dependencies": { - "boom": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/boom/-/boom-5.2.0.tgz", - "integrity": "sha512-Z5BTk6ZRe4tXXQlkqftmsAUANpXmuwlsF5Oov8ThoMbQRzdGTA1ngYRW160GexgOgjsFOKJz0LYhoNi+2AMBUw==", - "dev": true, - "requires": { - "hoek": "4.2.1" - } - } - } - }, "crypto-browserify": { "version": "1.0.9", "resolved": "https://registry.npmjs.org/crypto-browserify/-/crypto-browserify-1.0.9.tgz", @@ -2658,50 +1505,6 @@ "integrity": "sha1-ojD2T1aDEOFJgAmUB5DsmVRbyn4=", "dev": true }, - "css": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/css/-/css-2.2.1.tgz", - "integrity": "sha1-c6TIHehdtmTU7mdPfUcIXjstVdw=", - "dev": true, - "requires": { - "inherits": "2.0.3", - "source-map": "0.1.43", - "source-map-resolve": "0.3.1", - "urix": "0.1.0" - }, - "dependencies": { - "source-map": { - "version": "0.1.43", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.1.43.tgz", - "integrity": "sha1-wkvBRspRfBRx9drL4lcbK3+eM0Y=", - "dev": true, - "requires": { - "amdefine": "1.0.1" - } - } - } - }, - "css-parse": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/css-parse/-/css-parse-2.0.0.tgz", - "integrity": "sha1-pGjuZnwW2BzPBcWMONKpfHgNv9Q=", - "dev": true, - "requires": { - "css": "2.2.1" - } - }, - "css-value": { - "version": "0.0.1", - "resolved": "https://registry.npmjs.org/css-value/-/css-value-0.0.1.tgz", - "integrity": "sha1-Xv1sLupeof1rasV+wEJ7GEUkJOo=", - "dev": true - }, - "ctype": { - "version": "0.5.3", - "resolved": "https://registry.npmjs.org/ctype/-/ctype-0.5.3.tgz", - "integrity": "sha1-gsGMJGH3QRTvFsE1IkrQuRRMoS8=", - "dev": true - }, "cycle": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/cycle/-/cycle-1.0.3.tgz", @@ -2715,35 +1518,11 @@ "es5-ext": "0.10.39" } }, - "dashdash": { - "version": "1.14.1", - "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz", - "integrity": "sha1-hTz6D3y+L+1d4gMmuN1YEDX24vA=", - "dev": true, - "requires": { - "assert-plus": "1.0.0" - } - }, "date-now": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/date-now/-/date-now-0.1.4.tgz", "integrity": "sha1-6vQ5/U1ISK105cx9vvIAZyueNFs=" }, - "ddp-ejson": { - "version": "0.8.1-3", - "resolved": "https://registry.npmjs.org/ddp-ejson/-/ddp-ejson-0.8.1-3.tgz", - "integrity": "sha1-6dZ0Zqt4m9dOfZcHSjbiQGkO7sI=", - "dev": true, - "requires": { - "ddp-underscore-patched": "0.8.1-2" - } - }, - "ddp-underscore-patched": { - "version": "0.8.1-2", - "resolved": "https://registry.npmjs.org/ddp-underscore-patched/-/ddp-underscore-patched-0.8.1-2.tgz", - "integrity": "sha1-ZaQU6fIuxagqoWOG40NmtI/Ozx0=", - "dev": true - }, "debug": { "version": "2.6.9", "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", @@ -2780,37 +1559,25 @@ "integrity": "sha1-9dJgKStmDghO/0zbyfCK0yR0SLU=", "dev": true }, - "deep-extend": { - "version": "0.4.2", - "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.4.2.tgz", - "integrity": "sha1-SLaZwn4zS/ifEIkr5DL25MfTSn8=", - "dev": true - }, "deep-readdir": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/deep-readdir/-/deep-readdir-0.2.0.tgz", "integrity": "sha1-xNib//LX8ilN8DFP2wvxHCeUQxQ=" }, "deepforge-keras": { - "version": "git://github.com/deepforge-dev/deepforge-keras.git#b33783b2072a2d53089344926b48a7a151a6e64a", + "version": "github:deepforge-dev/deepforge-keras#ded3641a1dd33ff1f6a09c58b3d9c67cdf7ede9f", "requires": { "rimraf": "2.6.2", "webgme-autoviz": "2.2.2", - "webgme-easydag": "git://github.com/dfst/webgme-easydag.git#cb461f2687c8a2aa00adc827ea3688b7f0e24ada", - "webgme-simple-nodes": "git://github.com/brollb/webgme-simple-nodes.git#2a9fc79c93efd55067ef7c2e1559f9b31b0f97e5" + "webgme-easydag": "github:dfst/webgme-easydag#fd56486ae9b1df3c0749141c8dabdb6a90818fed", + "webgme-simple-nodes": "github:brollb/webgme-simple-nodes#2a9fc79c93efd55067ef7c2e1559f9b31b0f97e5" }, "dependencies": { "webgme-simple-nodes": { - "version": "git://github.com/brollb/webgme-simple-nodes.git#2a9fc79c93efd55067ef7c2e1559f9b31b0f97e5" + "version": "github:brollb/webgme-simple-nodes#2a9fc79c93efd55067ef7c2e1559f9b31b0f97e5" } } }, - "deepmerge": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-2.0.1.tgz", - "integrity": "sha512-VIPwiMJqJ13ZQfaCsIFnp5Me9tnjURiaIFxfz7EH0Ci0dTSQpZtSLrqOicXqEd/z2r+z+Klk9GzmnRsgpgbOsQ==", - "dev": true - }, "default-user-agent": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/default-user-agent/-/default-user-agent-1.0.0.tgz", @@ -2819,35 +1586,11 @@ "os-name": "1.0.3" } }, - "define-properties": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.2.tgz", - "integrity": "sha1-g6c/L+pWmJj7c3GTyPhzyvbUXJQ=", - "dev": true, - "requires": { - "foreach": "2.0.5", - "object-keys": "1.0.11" - } - }, "defined": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/defined/-/defined-1.0.0.tgz", "integrity": "sha1-yY2bzvdWdBiOEQlpFRGZ45sfppM=" }, - "del": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/del/-/del-3.0.0.tgz", - "integrity": "sha1-U+z2mf/LyzljdpGrE7rxYIGXZuU=", - "dev": true, - "requires": { - "globby": "6.1.0", - "is-path-cwd": "1.0.0", - "is-path-in-cwd": "1.0.1", - "p-map": "1.2.0", - "pify": "3.0.0", - "rimraf": "2.6.2" - } - }, "delayed-stream": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", @@ -2883,15 +1626,6 @@ "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz", "integrity": "sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA=" }, - "detect-indent": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/detect-indent/-/detect-indent-4.0.0.tgz", - "integrity": "sha1-920GQ1LN9Docts5hnE7jqUdd4gg=", - "dev": true, - "requires": { - "repeating": "2.0.1" - } - }, "detective": { "version": "4.7.1", "resolved": "https://registry.npmjs.org/detective/-/detective-4.7.1.tgz", @@ -3056,26 +1790,6 @@ "stream-shift": "1.0.0" } }, - "duration": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/duration/-/duration-0.2.0.tgz", - "integrity": "sha1-X5xN+q//ZV3phhEu/iXFl43YUUY=", - "dev": true, - "requires": { - "d": "0.1.1", - "es5-ext": "0.10.39" - } - }, - "ecc-jsbn": { - "version": "0.1.1", - "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.1.tgz", - "integrity": "sha1-D8c6ntXw1Tw4GTOYUj735UN3dQU=", - "dev": true, - "optional": true, - "requires": { - "jsbn": "0.1.1" - } - }, "ecdsa-sig-formatter": { "version": "1.0.10", "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.10.tgz", @@ -3218,15 +1932,6 @@ "resolved": "https://registry.npmjs.org/env-variable/-/env-variable-0.0.4.tgz", "integrity": "sha512-+jpGxSWG4vr6gVxUHOc4p+ilPnql7NzZxOZBxNldsKGjCF+97df3CbuX7XMaDa5oAVkKQj4rKp38rYdC4VcpDg==" }, - "error-stack-parser": { - "version": "1.3.6", - "resolved": "https://registry.npmjs.org/error-stack-parser/-/error-stack-parser-1.3.6.tgz", - "integrity": "sha1-4Oc7k+QXE40c18C3RrGkoUhUwpI=", - "dev": true, - "requires": { - "stackframe": "0.3.1" - } - }, "es5-ext": { "version": "0.10.39", "resolved": "https://registry.npmjs.org/es5-ext/-/es5-ext-0.10.39.tgz", @@ -3333,12 +2038,6 @@ "integrity": "sha1-r2fy3JIlgkFZUJJgkaQAXSnJu0Q=", "dev": true }, - "esutils": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.2.tgz", - "integrity": "sha1-Cr9PHKpbyx96nYrMbepPqqBLrJs=", - "dev": true - }, "etag": { "version": "1.8.1", "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", @@ -3745,17 +2444,6 @@ "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.1.tgz", "integrity": "sha1-p1Xqe8Gt/MWjHOfnYtuq3F5jZEQ=" }, - "external-editor": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/external-editor/-/external-editor-2.1.0.tgz", - "integrity": "sha512-E44iT5QVOUJBKij4IIV3uvxuNlbKS38Tw1HiupxEIHPv9qtC2PrDYohbXV5U+1jnfIXttny8gUhj+oZvflFlzA==", - "dev": true, - "requires": { - "chardet": "0.4.2", - "iconv-lite": "0.4.19", - "tmp": "0.0.33" - } - }, "extglob": { "version": "0.3.2", "resolved": "https://registry.npmjs.org/extglob/-/extglob-0.3.2.tgz", @@ -3764,46 +2452,6 @@ "is-extglob": "1.0.0" } }, - "extract-zip": { - "version": "1.6.6", - "resolved": "https://registry.npmjs.org/extract-zip/-/extract-zip-1.6.6.tgz", - "integrity": "sha1-EpDt6NINCHK0Kf0/NRyhKOxe+Fw=", - "dev": true, - "requires": { - "concat-stream": "1.6.0", - "debug": "2.6.9", - "mkdirp": "0.5.0", - "yauzl": "2.4.1" - }, - "dependencies": { - "concat-stream": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.0.tgz", - "integrity": "sha1-CqxmL9Ur54lk1VMvaUeE5wEQrPc=", - "dev": true, - "requires": { - "inherits": "2.0.3", - "readable-stream": "2.3.4", - "typedarray": "0.0.6" - } - }, - "mkdirp": { - "version": "0.5.0", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.0.tgz", - "integrity": "sha1-HXMHam35hs2TROFecfzAWkyavxI=", - "dev": true, - "requires": { - "minimist": "0.0.8" - } - } - } - }, - "extsprintf": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz", - "integrity": "sha1-lpGEQOMEGnpBT4xS48V06zw+HgU=", - "dev": true - }, "eyes": { "version": "0.1.8", "resolved": "https://registry.npmjs.org/eyes/-/eyes-0.1.8.tgz", @@ -3819,15 +2467,6 @@ "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.0.0.tgz", "integrity": "sha1-1RQsDK7msRifh9OnYREGT4bIu/I=" }, - "faye-websocket": { - "version": "0.9.4", - "resolved": "https://registry.npmjs.org/faye-websocket/-/faye-websocket-0.9.4.tgz", - "integrity": "sha1-iFk0x57/sECVSeDAo4Ae0XpAza0=", - "dev": true, - "requires": { - "websocket-driver": "0.7.0" - } - }, "fd-slicer": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.0.1.tgz", @@ -3836,40 +2475,6 @@ "pend": "1.2.0" } }, - "fg-lodash": { - "version": "0.0.2", - "resolved": "https://registry.npmjs.org/fg-lodash/-/fg-lodash-0.0.2.tgz", - "integrity": "sha1-mINSU39CfaavIiEpu2OsyknmL6M=", - "dev": true, - "requires": { - "lodash": "2.4.2", - "underscore.string": "2.3.3" - }, - "dependencies": { - "lodash": { - "version": "2.4.2", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-2.4.2.tgz", - "integrity": "sha1-+t2DS5aDBz2hebPq5tnA0VBT9z4=", - "dev": true - } - } - }, - "fibers": { - "version": "1.0.15", - "resolved": "https://registry.npmjs.org/fibers/-/fibers-1.0.15.tgz", - "integrity": "sha1-IvA5yPGLhWGQ+75N7PBWFUwerpw=", - "dev": true - }, - "figures": { - "version": "1.7.0", - "resolved": "https://registry.npmjs.org/figures/-/figures-1.7.0.tgz", - "integrity": "sha1-y+Hjr/zxzUS4DK3+0o3Hk6lwHS4=", - "dev": true, - "requires": { - "escape-string-regexp": "1.0.5", - "object-assign": "4.1.1" - } - }, "filename-regex": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/filename-regex/-/filename-regex-2.0.1.tgz", @@ -3921,18 +2526,6 @@ "for-in": "1.0.2" } }, - "foreach": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/foreach/-/foreach-2.0.5.tgz", - "integrity": "sha1-C+4AUBiusmDQo6865ljdATbsG5k=", - "dev": true - }, - "forever-agent": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz", - "integrity": "sha1-+8cfDEGt6zf5bFd60e1C2P2sypE=", - "dev": true - }, "form-data": { "version": "2.3.2", "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.2.tgz", @@ -3958,38 +2551,10 @@ "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.1.2.tgz", "integrity": "sha1-mMI9qxF1ZXuMBXPozszZGw/xjIQ=" }, - "freeport": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/freeport/-/freeport-1.0.5.tgz", - "integrity": "sha1-JV6KuEFwwzuoXZkOghrl9KGpvF0=", - "dev": true - }, - "fresh": { - "version": "0.5.2", - "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", - "integrity": "sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac=" - }, - "fs-extra": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-1.0.0.tgz", - "integrity": "sha1-zTzl9+fLYUWIP8rjGR6Yd/hYeVA=", - "dev": true, - "requires": { - "graceful-fs": "4.1.11", - "jsonfile": "2.4.0", - "klaw": "1.3.1" - }, - "dependencies": { - "klaw": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/klaw/-/klaw-1.3.1.tgz", - "integrity": "sha1-QIhDO0azsbolnXh4XY6W9zugJDk=", - "dev": true, - "requires": { - "graceful-fs": "4.1.11" - } - } - } + "fresh": { + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", + "integrity": "sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac=" }, "fs-write-stream-atomic": { "version": "1.0.10", @@ -4036,8 +2601,8 @@ "bundled": true, "optional": true, "requires": { - "delegates": "1.0.0", - "readable-stream": "2.3.6" + "delegates": "^1.0.0", + "readable-stream": "^2.0.6" } }, "balanced-match": { @@ -4048,7 +2613,7 @@ "version": "1.1.11", "bundled": true, "requires": { - "balanced-match": "1.0.0", + "balanced-match": "^1.0.0", "concat-map": "0.0.1" } }, @@ -4102,7 +2667,7 @@ "bundled": true, "optional": true, "requires": { - "minipass": "2.2.4" + "minipass": "^2.2.1" } }, "fs.realpath": { @@ -4115,14 +2680,14 @@ "bundled": true, "optional": true, "requires": { - "aproba": "1.2.0", - "console-control-strings": "1.1.0", - "has-unicode": "2.0.1", - "object-assign": "4.1.1", - "signal-exit": "3.0.2", - "string-width": "1.0.2", - "strip-ansi": "3.0.1", - "wide-align": "1.1.2" + "aproba": "^1.0.3", + "console-control-strings": "^1.0.0", + "has-unicode": "^2.0.0", + "object-assign": "^4.1.0", + "signal-exit": "^3.0.0", + "string-width": "^1.0.1", + "strip-ansi": "^3.0.1", + "wide-align": "^1.1.0" } }, "glob": { @@ -4130,12 +2695,12 @@ "bundled": true, "optional": true, "requires": { - "fs.realpath": "1.0.0", - "inflight": "1.0.6", - "inherits": "2.0.3", - "minimatch": "3.0.4", - "once": "1.4.0", - "path-is-absolute": "1.0.1" + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.0.4", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" } }, "has-unicode": { @@ -4148,7 +2713,7 @@ "bundled": true, "optional": true, "requires": { - "safer-buffer": "2.1.2" + "safer-buffer": "^2.1.0" } }, "ignore-walk": { @@ -4156,7 +2721,7 @@ "bundled": true, "optional": true, "requires": { - "minimatch": "3.0.4" + "minimatch": "^3.0.4" } }, "inflight": { @@ -4164,8 +2729,8 @@ "bundled": true, "optional": true, "requires": { - "once": "1.4.0", - "wrappy": "1.0.2" + "once": "^1.3.0", + "wrappy": "1" } }, "inherits": { @@ -4181,7 +2746,7 @@ "version": "1.0.0", "bundled": true, "requires": { - "number-is-nan": "1.0.1" + "number-is-nan": "^1.0.0" } }, "isarray": { @@ -4193,7 +2758,7 @@ "version": "3.0.4", "bundled": true, "requires": { - "brace-expansion": "1.1.11" + "brace-expansion": "^1.1.7" } }, "minimist": { @@ -4204,8 +2769,8 @@ "version": "2.2.4", "bundled": true, "requires": { - "safe-buffer": "5.1.1", - "yallist": "3.0.2" + "safe-buffer": "^5.1.1", + "yallist": "^3.0.0" } }, "minizlib": { @@ -4213,7 +2778,7 @@ "bundled": true, "optional": true, "requires": { - "minipass": "2.2.4" + "minipass": "^2.2.1" } }, "mkdirp": { @@ -4233,9 +2798,9 @@ "bundled": true, "optional": true, "requires": { - "debug": "2.6.9", - "iconv-lite": "0.4.21", - "sax": "1.2.4" + "debug": "^2.1.2", + "iconv-lite": "^0.4.4", + "sax": "^1.2.4" } }, "node-pre-gyp": { @@ -4243,16 +2808,16 @@ "bundled": true, "optional": true, "requires": { - "detect-libc": "1.0.3", - "mkdirp": "0.5.1", - "needle": "2.2.0", - "nopt": "4.0.1", - "npm-packlist": "1.1.10", - "npmlog": "4.1.2", - "rc": "1.2.7", - "rimraf": "2.6.2", - "semver": "5.5.0", - "tar": "4.4.1" + "detect-libc": "^1.0.2", + "mkdirp": "^0.5.1", + "needle": "^2.2.0", + "nopt": "^4.0.1", + "npm-packlist": "^1.1.6", + "npmlog": "^4.0.2", + "rc": "^1.1.7", + "rimraf": "^2.6.1", + "semver": "^5.3.0", + "tar": "^4" } }, "nopt": { @@ -4260,8 +2825,8 @@ "bundled": true, "optional": true, "requires": { - "abbrev": "1.1.1", - "osenv": "0.1.5" + "abbrev": "1", + "osenv": "^0.1.4" } }, "npm-bundled": { @@ -4274,8 +2839,8 @@ "bundled": true, "optional": true, "requires": { - "ignore-walk": "3.0.1", - "npm-bundled": "1.0.3" + "ignore-walk": "^3.0.1", + "npm-bundled": "^1.0.1" } }, "npmlog": { @@ -4283,10 +2848,10 @@ "bundled": true, "optional": true, "requires": { - "are-we-there-yet": "1.1.4", - "console-control-strings": "1.1.0", - "gauge": "2.7.4", - "set-blocking": "2.0.0" + "are-we-there-yet": "~1.1.2", + "console-control-strings": "~1.1.0", + "gauge": "~2.7.3", + "set-blocking": "~2.0.0" } }, "number-is-nan": { @@ -4302,7 +2867,7 @@ "version": "1.4.0", "bundled": true, "requires": { - "wrappy": "1.0.2" + "wrappy": "1" } }, "os-homedir": { @@ -4320,8 +2885,8 @@ "bundled": true, "optional": true, "requires": { - "os-homedir": "1.0.2", - "os-tmpdir": "1.0.2" + "os-homedir": "^1.0.0", + "os-tmpdir": "^1.0.0" } }, "path-is-absolute": { @@ -4339,10 +2904,10 @@ "bundled": true, "optional": true, "requires": { - "deep-extend": "0.5.1", - "ini": "1.3.5", - "minimist": "1.2.0", - "strip-json-comments": "2.0.1" + "deep-extend": "^0.5.1", + "ini": "~1.3.0", + "minimist": "^1.2.0", + "strip-json-comments": "~2.0.1" }, "dependencies": { "minimist": { @@ -4357,13 +2922,13 @@ "bundled": true, "optional": true, "requires": { - "core-util-is": "1.0.2", - "inherits": "2.0.3", - "isarray": "1.0.0", - "process-nextick-args": "2.0.0", - "safe-buffer": "5.1.1", - "string_decoder": "1.1.1", - "util-deprecate": "1.0.2" + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" } }, "rimraf": { @@ -4371,7 +2936,7 @@ "bundled": true, "optional": true, "requires": { - "glob": "7.1.2" + "glob": "^7.0.5" } }, "safe-buffer": { @@ -4407,9 +2972,9 @@ "version": "1.0.2", "bundled": true, "requires": { - "code-point-at": "1.1.0", - "is-fullwidth-code-point": "1.0.0", - "strip-ansi": "3.0.1" + "code-point-at": "^1.0.0", + "is-fullwidth-code-point": "^1.0.0", + "strip-ansi": "^3.0.0" } }, "string_decoder": { @@ -4417,14 +2982,14 @@ "bundled": true, "optional": true, "requires": { - "safe-buffer": "5.1.1" + "safe-buffer": "~5.1.0" } }, "strip-ansi": { "version": "3.0.1", "bundled": true, "requires": { - "ansi-regex": "2.1.1" + "ansi-regex": "^2.0.0" } }, "strip-json-comments": { @@ -4437,13 +3002,13 @@ "bundled": true, "optional": true, "requires": { - "chownr": "1.0.1", - "fs-minipass": "1.2.5", - "minipass": "2.2.4", - "minizlib": "1.1.0", - "mkdirp": "0.5.1", - "safe-buffer": "5.1.1", - "yallist": "3.0.2" + "chownr": "^1.0.1", + "fs-minipass": "^1.2.5", + "minipass": "^2.2.4", + "minizlib": "^1.1.0", + "mkdirp": "^0.5.0", + "safe-buffer": "^5.1.1", + "yallist": "^3.0.2" } }, "util-deprecate": { @@ -4456,7 +3021,7 @@ "bundled": true, "optional": true, "requires": { - "string-width": "1.0.2" + "string-width": "^1.0.2" } }, "wrappy": { @@ -4474,30 +3039,6 @@ "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==" }, - "gaze": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/gaze/-/gaze-1.1.2.tgz", - "integrity": "sha1-hHIkZ3rbiHDWeSV+0ziP22HkAQU=", - "dev": true, - "requires": { - "globule": "1.2.0" - } - }, - "generate-function": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/generate-function/-/generate-function-2.0.0.tgz", - "integrity": "sha1-aFj+fAlpt9TpCTM3ZHrHn2DfvnQ=", - "dev": true - }, - "generate-object-property": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/generate-object-property/-/generate-object-property-1.2.0.tgz", - "integrity": "sha1-nA4cQDCM6AT0eDYYuTf6iPmdUNA=", - "dev": true, - "requires": { - "is-property": "1.0.2" - } - }, "get-stdin": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/get-stdin/-/get-stdin-4.0.1.tgz", @@ -4510,21 +3051,6 @@ "integrity": "sha1-jpQ9E1jcN1VQVOy+LtsFqhdO3hQ=", "dev": true }, - "getpass": { - "version": "0.1.7", - "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz", - "integrity": "sha1-Xv+OPmhNVprkyysSgmBOi6YhSfo=", - "dev": true, - "requires": { - "assert-plus": "1.0.0" - } - }, - "gherkin": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/gherkin/-/gherkin-4.0.0.tgz", - "integrity": "sha1-edzgTRIj6kO0hip2vlzo+JwSwyw=", - "dev": true - }, "glob": { "version": "github:lucetius/node-glob#51c7ca6e69bfbd17db5f1ea710e3f2a7a457d9ce", "requires": { @@ -4570,44 +3096,6 @@ } } }, - "globals": { - "version": "9.18.0", - "resolved": "https://registry.npmjs.org/globals/-/globals-9.18.0.tgz", - "integrity": "sha512-S0nG3CLEQiY/ILxqtztTWH/3iRRdyBLw6KMDxnKMchrtbj2OFmehVh0WUCfW3DUrIgx/qFrJPICrq4Z4sTR9UQ==", - "dev": true - }, - "globby": { - "version": "6.1.0", - "resolved": "https://registry.npmjs.org/globby/-/globby-6.1.0.tgz", - "integrity": "sha1-9abXDoOV4hyFj7BInWTfAkJNUGw=", - "dev": true, - "requires": { - "array-union": "1.0.2", - "glob": "github:lucetius/node-glob#51c7ca6e69bfbd17db5f1ea710e3f2a7a457d9ce", - "object-assign": "4.1.1", - "pify": "2.3.0", - "pinkie-promise": "2.0.1" - }, - "dependencies": { - "pify": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", - "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw=", - "dev": true - } - } - }, - "globule": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/globule/-/globule-1.2.0.tgz", - "integrity": "sha1-HcScaCLdnoovoAuiopUAboZkvQk=", - "dev": true, - "requires": { - "glob": "github:lucetius/node-glob#51c7ca6e69bfbd17db5f1ea710e3f2a7a457d9ce", - "lodash": "4.17.4", - "minimatch": "3.0.4" - } - }, "got": { "version": "2.4.0", "resolved": "https://registry.npmjs.org/got/-/got-2.4.0.tgz", @@ -4645,382 +3133,7 @@ "version": "1.9.2", "resolved": "https://registry.npmjs.org/growl/-/growl-1.9.2.tgz", "integrity": "sha1-Dqd0NxXbjY3ixe3hd14bRayFwC8=", - "dev": true - }, - "hapi": { - "version": "8.8.0", - "resolved": "https://registry.npmjs.org/hapi/-/hapi-8.8.0.tgz", - "integrity": "sha1-h+N6Bum0meiXkOLcERqpZotuYX8=", - "dev": true, - "requires": { - "accept": "1.0.0", - "ammo": "1.0.0", - "boom": "2.7.2", - "call": "2.0.1", - "catbox": "4.3.0", - "catbox-memory": "1.1.1", - "cryptiles": "2.0.4", - "h2o2": "4.0.1", - "heavy": "3.0.0", - "hoek": "2.14.0", - "inert": "2.1.5", - "iron": "2.1.2", - "items": "1.1.0", - "joi": "6.4.1", - "kilt": "1.1.1", - "mimos": "2.0.2", - "peekaboo": "1.0.0", - "qs": "4.0.0", - "shot": "1.5.3", - "statehood": "2.1.1", - "subtext": "1.1.1", - "topo": "1.0.2", - "vision": "2.0.1" - }, - "dependencies": { - "accept": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/accept/-/accept-1.0.0.tgz", - "integrity": "sha1-g++IOWi4WkDFARYEKCoiD/AeYq0=", - "dev": true, - "requires": { - "boom": "2.7.2", - "hoek": "2.14.0" - } - }, - "ammo": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/ammo/-/ammo-1.0.0.tgz", - "integrity": "sha1-4FlIG/aAhzj66G1GT3L6DBLWeoU=", - "dev": true, - "requires": { - "boom": "2.7.2", - "hoek": "2.14.0" - } - }, - "boom": { - "version": "2.7.2", - "resolved": "https://registry.npmjs.org/boom/-/boom-2.7.2.tgz", - "integrity": "sha1-2tYo2Jf3/S4yzIIZfxMweXHPg1Q=", - "dev": true, - "requires": { - "hoek": "2.14.0" - } - }, - "call": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/call/-/call-2.0.1.tgz", - "integrity": "sha1-SbQnCZQ96JoyJYqpEbWHUeI3eg4=", - "dev": true, - "requires": { - "boom": "2.7.2", - "hoek": "2.14.0" - } - }, - "catbox": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/catbox/-/catbox-4.3.0.tgz", - "integrity": "sha1-IiN3vWfxKRrA4l0AAC0GWp3385o=", - "dev": true, - "requires": { - "boom": "2.7.2", - "hoek": "2.14.0", - "joi": "6.4.1" - } - }, - "catbox-memory": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/catbox-memory/-/catbox-memory-1.1.1.tgz", - "integrity": "sha1-QqUvgLye+nJmAeltQBYDNhJIGig=", - "dev": true, - "requires": { - "hoek": "2.14.0" - } - }, - "cryptiles": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/cryptiles/-/cryptiles-2.0.4.tgz", - "integrity": "sha1-CeoXdbnhx95+YKmdQqtvCM4aEoU=", - "dev": true, - "requires": { - "boom": "2.7.2" - } - }, - "h2o2": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/h2o2/-/h2o2-4.0.1.tgz", - "integrity": "sha1-eg4rztHZcXjsVs48ykjgxW3un40=", - "dev": true, - "requires": { - "boom": "2.7.2", - "hoek": "2.14.0", - "joi": "6.4.1", - "wreck": "6.0.0" - } - }, - "heavy": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/heavy/-/heavy-3.0.0.tgz", - "integrity": "sha1-/QEIdiExYy+IVIontVQSws9SKwA=", - "dev": true, - "requires": { - "boom": "2.7.2", - "hoek": "2.14.0", - "joi": "6.4.1" - } - }, - "hoek": { - "version": "2.14.0", - "resolved": "https://registry.npmjs.org/hoek/-/hoek-2.14.0.tgz", - "integrity": "sha1-gSEWkfUqWoNa5J7b8eickANHaqQ=", - "dev": true - }, - "inert": { - "version": "2.1.5", - "resolved": "https://registry.npmjs.org/inert/-/inert-2.1.5.tgz", - "integrity": "sha1-eybZTEHGLAPsHU726LRe1WuDSFk=", - "dev": true, - "requires": { - "ammo": "1.0.0", - "boom": "2.7.2", - "hoek": "2.14.0", - "items": "1.1.0", - "joi": "6.4.1", - "lru-cache": "2.6.4" - }, - "dependencies": { - "lru-cache": { - "version": "2.6.4", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-2.6.4.tgz", - "integrity": "sha1-JnUZDM0bBwHsL2UqTQ09QA12wN0=", - "dev": true - } - } - }, - "iron": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/iron/-/iron-2.1.2.tgz", - "integrity": "sha1-WR2RiiVAdTxEbY5DfNzwz6gBEU8=", - "dev": true, - "requires": { - "boom": "2.7.2", - "cryptiles": "2.0.4", - "hoek": "2.14.0" - } - }, - "items": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/items/-/items-1.1.0.tgz", - "integrity": "sha1-rZ1VhAsimGDLPRYLMidMLUvZ4mI=", - "dev": true - }, - "joi": { - "version": "6.4.1", - "resolved": "https://registry.npmjs.org/joi/-/joi-6.4.1.tgz", - "integrity": "sha1-9Q9CRTVgBo5jg9oVrC0w3Xzra24=", - "dev": true, - "requires": { - "hoek": "2.14.0", - "isemail": "1.1.1", - "moment": "2.10.3", - "topo": "1.0.2" - }, - "dependencies": { - "isemail": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/isemail/-/isemail-1.1.1.tgz", - "integrity": "sha1-4Mj23D9HCX53dzlcaJYnGqJWw7U=", - "dev": true - }, - "moment": { - "version": "2.10.3", - "resolved": "https://registry.npmjs.org/moment/-/moment-2.10.3.tgz", - "integrity": "sha1-CruZ8wf2UhgwjGk17+KcV7Ggon8=", - "dev": true - } - } - }, - "kilt": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/kilt/-/kilt-1.1.1.tgz", - "integrity": "sha1-d7SmFjyn+lshN6iMFzNCFuwj1ds=", - "dev": true, - "requires": { - "hoek": "2.14.0" - } - }, - "mimos": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/mimos/-/mimos-2.0.2.tgz", - "integrity": "sha1-wyQXF+dblZkr54esfdbbGptTmx4=", - "dev": true, - "requires": { - "hoek": "2.14.0", - "mime-db": "1.14.0" - }, - "dependencies": { - "mime-db": { - "version": "1.14.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.14.0.tgz", - "integrity": "sha1-1WHxC27mbbUflK5leilRp0IX7YM=", - "dev": true - } - } - }, - "peekaboo": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/peekaboo/-/peekaboo-1.0.0.tgz", - "integrity": "sha1-wNspJq1lTSygH3ymUKtFkadk/EI=", - "dev": true - }, - "qs": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/qs/-/qs-4.0.0.tgz", - "integrity": "sha1-wx2bdOwn33XlQ6hseHKO2NRiNgc=", - "dev": true - }, - "shot": { - "version": "1.5.3", - "resolved": "https://registry.npmjs.org/shot/-/shot-1.5.3.tgz", - "integrity": "sha1-SGEHREO8VHLCNRthpGtOrsAH9Xo=", - "dev": true, - "requires": { - "hoek": "2.14.0" - } - }, - "statehood": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/statehood/-/statehood-2.1.1.tgz", - "integrity": "sha1-AfFwtmxeklqvZ5qdMiulkYb8AAk=", - "dev": true, - "requires": { - "boom": "2.7.2", - "cryptiles": "2.0.4", - "hoek": "2.14.0", - "iron": "2.1.2", - "items": "1.1.0", - "joi": "6.4.1" - } - }, - "subtext": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/subtext/-/subtext-1.1.1.tgz", - "integrity": "sha1-DJGCWuZdUXhVWT2DHjPvdaKEFWs=", - "dev": true, - "requires": { - "boom": "2.7.2", - "content": "1.0.1", - "hoek": "2.14.0", - "pez": "1.0.0", - "qs": "4.0.0", - "wreck": "6.0.0" - }, - "dependencies": { - "content": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/content/-/content-1.0.1.tgz", - "integrity": "sha1-gD60s7eJVGD9jGnGhMd1RmmvG6E=", - "dev": true, - "requires": { - "boom": "2.7.2", - "hoek": "2.14.0" - } - }, - "pez": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/pez/-/pez-1.0.0.tgz", - "integrity": "sha1-hEMYpc5wku7d/6KV4YB5rHefoBg=", - "dev": true, - "requires": { - "b64": "2.0.0", - "boom": "2.7.2", - "content": "1.0.1", - "hoek": "2.14.0", - "nigel": "1.0.1" - }, - "dependencies": { - "b64": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/b64/-/b64-2.0.0.tgz", - "integrity": "sha1-tZlbJPR+v9nxMQF6bntdZHVvtvM=", - "dev": true, - "requires": { - "hoek": "2.14.0" - } - }, - "nigel": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/nigel/-/nigel-1.0.1.tgz", - "integrity": "sha1-RjmJr4gSePuqHTzJOCPb0XtDYKE=", - "dev": true, - "requires": { - "hoek": "2.14.0", - "vise": "1.0.0" - }, - "dependencies": { - "vise": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/vise/-/vise-1.0.0.tgz", - "integrity": "sha1-KDRb5N5aNB4V/SgW/Z6j5zA+jfM=", - "dev": true, - "requires": { - "hoek": "2.14.0" - } - } - } - } - } - } - } - }, - "topo": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/topo/-/topo-1.0.2.tgz", - "integrity": "sha1-QhV8N8HeTTeIPM3R1skChHqGDbk=", - "dev": true, - "requires": { - "hoek": "2.14.0" - } - }, - "vision": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/vision/-/vision-2.0.1.tgz", - "integrity": "sha1-0BIlW6buQm0GlqNOHfMy/sVeZzw=", - "dev": true, - "requires": { - "boom": "2.7.2", - "hoek": "2.14.0", - "items": "1.1.0", - "joi": "6.4.1" - } - }, - "wreck": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/wreck/-/wreck-6.0.0.tgz", - "integrity": "sha1-T0CGaWHl14rOBPMqa38x8/PFFjg=", - "dev": true, - "requires": { - "boom": "2.7.2", - "hoek": "2.14.0" - } - } - } - }, - "har-schema": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz", - "integrity": "sha1-qUwiJOvKwEeCoNkDVSHyRzW37JI=", - "dev": true - }, - "har-validator": { - "version": "5.0.3", - "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.0.3.tgz", - "integrity": "sha1-ukAsJmGU8VlW7xXg/PJCmT9qff0=", - "dev": true, - "requires": { - "ajv": "5.5.2", - "har-schema": "2.0.0" - } + "dev": true }, "has": { "version": "1.0.1", @@ -5030,15 +3143,6 @@ "function-bind": "1.1.1" } }, - "has-ansi": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/has-ansi/-/has-ansi-2.0.0.tgz", - "integrity": "sha1-NPUEnOHs3ysGSa8+8k5F7TVBbZE=", - "dev": true, - "requires": { - "ansi-regex": "2.1.1" - } - }, "has-binary2": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/has-binary2/-/has-binary2-1.0.3.tgz", @@ -5059,18 +3163,6 @@ "resolved": "https://registry.npmjs.org/has-cors/-/has-cors-1.1.0.tgz", "integrity": "sha1-XkdHk/fqmEPRu5nCPu9J/xJv/zk=" }, - "has-flag": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-1.0.0.tgz", - "integrity": "sha1-nZ55MWXOAXoA8AQYxD+UKnsdEfo=", - "dev": true - }, - "has-symbols": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.0.tgz", - "integrity": "sha1-uhqPGvKg/DllD1yFA2dwQSIGO0Q=", - "dev": true - }, "hash-base": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/hash-base/-/hash-base-3.0.4.tgz", @@ -5089,34 +3181,6 @@ "minimalistic-assert": "1.0.1" } }, - "hasha": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/hasha/-/hasha-2.2.0.tgz", - "integrity": "sha1-eNfL/B5tZjA/55g3NlmEUXsvbuE=", - "dev": true, - "requires": { - "is-stream": "1.1.0", - "pinkie-promise": "2.0.1" - } - }, - "hawk": { - "version": "6.0.2", - "resolved": "https://registry.npmjs.org/hawk/-/hawk-6.0.2.tgz", - "integrity": "sha512-miowhl2+U7Qle4vdLqDdPt9m09K6yZhkLDTWGoUiUzrQCn+mHHSmfJgAyGaLRZbPmTqfFFjRV1QWCW0VWUJBbQ==", - "dev": true, - "requires": { - "boom": "4.3.1", - "cryptiles": "3.1.2", - "hoek": "4.2.1", - "sntp": "2.1.0" - } - }, - "he": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/he/-/he-1.1.1.tgz", - "integrity": "sha1-k0EP0hsAlzUVH4howvJx80J+I/0=", - "dev": true - }, "hmac-drbg": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/hmac-drbg/-/hmac-drbg-1.0.1.tgz", @@ -5127,22 +3191,6 @@ "minimalistic-crypto-utils": "1.0.1" } }, - "hoek": { - "version": "4.2.1", - "resolved": "https://registry.npmjs.org/hoek/-/hoek-4.2.1.tgz", - "integrity": "sha512-QLg82fGkfnJ/4iy1xZ81/9SIJiq1NGFUMGs6ParyjBZr6jW2Ufj/snDqTHixNlHdPNwN2RLVD0Pi3igeK9+JfA==", - "dev": true - }, - "home-or-tmp": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/home-or-tmp/-/home-or-tmp-2.0.0.tgz", - "integrity": "sha1-42w/LSyufXRqhX440Y1fMqeILbg=", - "dev": true, - "requires": { - "os-homedir": "1.0.2", - "os-tmpdir": "1.0.2" - } - }, "hosted-git-info": { "version": "2.5.0", "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-2.5.0.tgz", @@ -5177,39 +3225,11 @@ "statuses": "1.5.0" } }, - "http-parser-js": { - "version": "0.4.11", - "resolved": "https://registry.npmjs.org/http-parser-js/-/http-parser-js-0.4.11.tgz", - "integrity": "sha512-QCR5O2AjjMW8Mo4HyI1ctFcv+O99j/0g367V3YoVnrNw5hkDvAWZD0lWGcc+F4yN3V55USPCVix4efb75HxFfA==", - "dev": true - }, - "http-signature": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz", - "integrity": "sha1-muzZJRFHcvPZW2WmCruPfBj7rOE=", - "dev": true, - "requires": { - "assert-plus": "1.0.0", - "jsprim": "1.4.1", - "sshpk": "1.14.1" - } - }, "https-browserify": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/https-browserify/-/https-browserify-1.0.0.tgz", "integrity": "sha1-7AbBDgo0wPL68Zn3/X/Hj//QPHM=" }, - "https-proxy-agent": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-1.0.0.tgz", - "integrity": "sha1-NffabEjOTdv6JkiRrFk+5f+GceY=", - "dev": true, - "requires": { - "agent-base": "2.1.1", - "debug": "2.6.9", - "extend": "3.0.1" - } - }, "humanize-ms": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz", @@ -5291,105 +3311,6 @@ "source-map": "0.5.7" } }, - "inquirer": { - "version": "3.3.0", - "resolved": "https://registry.npmjs.org/inquirer/-/inquirer-3.3.0.tgz", - "integrity": "sha512-h+xtnyk4EwKvFWHrUYsWErEVR+igKtLdchu+o0Z1RL7VU/jVMFbYir2bp6bAj8efFNxWqHX0dIss6fJQ+/+qeQ==", - "dev": true, - "requires": { - "ansi-escapes": "3.1.0", - "chalk": "2.3.2", - "cli-cursor": "2.1.0", - "cli-width": "2.2.0", - "external-editor": "2.1.0", - "figures": "2.0.0", - "lodash": "4.17.4", - "mute-stream": "0.0.7", - "run-async": "2.3.0", - "rx-lite": "4.0.8", - "rx-lite-aggregates": "4.0.8", - "string-width": "2.1.1", - "strip-ansi": "4.0.0", - "through": "2.3.8" - }, - "dependencies": { - "ansi-regex": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.0.tgz", - "integrity": "sha1-7QMXwyIGT3lGbAKWa922Bas32Zg=", - "dev": true - }, - "ansi-styles": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", - "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", - "dev": true, - "requires": { - "color-convert": "1.9.1" - } - }, - "chalk": { - "version": "2.3.2", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.3.2.tgz", - "integrity": "sha512-ZM4j2/ld/YZDc3Ma8PgN7gyAk+kHMMMyzLNryCPGhWrsfAuDVeuid5bpRFTDgMH9JBK2lA4dyyAkkZYF/WcqDQ==", - "dev": true, - "requires": { - "ansi-styles": "3.2.1", - "escape-string-regexp": "1.0.5", - "supports-color": "5.3.0" - } - }, - "figures": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/figures/-/figures-2.0.0.tgz", - "integrity": "sha1-OrGi0qYsi/tDGgyUy3l6L84nyWI=", - "dev": true, - "requires": { - "escape-string-regexp": "1.0.5" - } - }, - "has-flag": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", - "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=", - "dev": true - }, - "is-fullwidth-code-point": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", - "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=", - "dev": true - }, - "string-width": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz", - "integrity": "sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw==", - "dev": true, - "requires": { - "is-fullwidth-code-point": "2.0.0", - "strip-ansi": "4.0.0" - } - }, - "strip-ansi": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz", - "integrity": "sha1-qEeQIusaw2iocTibY1JixQXuNo8=", - "dev": true, - "requires": { - "ansi-regex": "3.0.0" - } - }, - "supports-color": { - "version": "5.3.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.3.0.tgz", - "integrity": "sha512-0aP01LLIskjKs3lq52EC0aGBAJhLq7B2Rd8HC/DR/PtNNpcLilNmHC12O+hu0usQpo7wtHNRqtrhBwtDb0+dNg==", - "dev": true, - "requires": { - "has-flag": "3.0.0" - } - } - } - }, "insert-module-globals": { "version": "7.1.0", "resolved": "https://registry.npmjs.org/insert-module-globals/-/insert-module-globals-7.1.0.tgz", @@ -5419,15 +3340,6 @@ } } }, - "invariant": { - "version": "2.2.4", - "resolved": "https://registry.npmjs.org/invariant/-/invariant-2.2.4.tgz", - "integrity": "sha512-phJfQVBuaJM5raOpJjSfkiD6BpbCE4Ns//LaXl6wGYtUBY83nWS6Rf9tXm2e8VaK60JEjYldbPif/A2B1C2gNA==", - "dev": true, - "requires": { - "loose-envify": "1.3.1" - } - }, "invert-kv": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/invert-kv/-/invert-kv-1.0.0.tgz", @@ -5474,15 +3386,6 @@ "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-1.0.0.tgz", "integrity": "sha1-rEaBd8SUNAWgkvyPKXYMb/xiBsA=" }, - "is-finite": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/is-finite/-/is-finite-1.0.2.tgz", - "integrity": "sha1-zGZ3aVYCvlUO8R6LSqYwU0K20Ko=", - "dev": true, - "requires": { - "number-is-nan": "1.0.1" - } - }, "is-fullwidth-code-point": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz", @@ -5491,12 +3394,6 @@ "number-is-nan": "1.0.1" } }, - "is-generator": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/is-generator/-/is-generator-1.0.3.tgz", - "integrity": "sha1-wUwhBX7TbjKNuANHlmxpP4hjifM=", - "dev": true - }, "is-glob": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-2.0.1.tgz", @@ -5523,25 +3420,6 @@ "lower-case": "1.1.4" } }, - "is-my-ip-valid": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/is-my-ip-valid/-/is-my-ip-valid-1.0.0.tgz", - "integrity": "sha512-gmh/eWXROncUzRnIa1Ubrt5b8ep/MGSnfAUI3aRp+sqTCs1tv1Isl8d8F6JmkN3dXKc3ehZMrtiPN9eL03NuaQ==", - "dev": true - }, - "is-my-json-valid": { - "version": "2.17.2", - "resolved": "https://registry.npmjs.org/is-my-json-valid/-/is-my-json-valid-2.17.2.tgz", - "integrity": "sha512-IBhBslgngMQN8DDSppmgDv7RNrlFotuuDsKcrCP3+HbFaVivIBU7u9oiiErw8sH4ynx3+gOGQ3q2otkgiSi6kg==", - "dev": true, - "requires": { - "generate-function": "2.0.0", - "generate-object-property": "1.2.0", - "is-my-ip-valid": "1.0.0", - "jsonpointer": "4.0.1", - "xtend": "4.0.1" - } - }, "is-number": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/is-number/-/is-number-2.1.0.tgz", @@ -5556,21 +3434,6 @@ "integrity": "sha1-PkcprB9f3gJc19g6iW2rn09n2w8=", "dev": true }, - "is-path-cwd": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/is-path-cwd/-/is-path-cwd-1.0.0.tgz", - "integrity": "sha1-0iXsIxMuie3Tj9p2dHLmLmXxEG0=", - "dev": true - }, - "is-path-in-cwd": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/is-path-in-cwd/-/is-path-in-cwd-1.0.1.tgz", - "integrity": "sha512-FjV1RTW48E7CWM7eE/J2NJvAEEVektecDBVBE5Hh3nM1Jd0kvhHtX68Pr3xsDf857xt3Y4AkwVULK1Vku62aaQ==", - "dev": true, - "requires": { - "is-path-inside": "1.0.1" - } - }, "is-path-inside": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/is-path-inside/-/is-path-inside-1.0.1.tgz", @@ -5590,18 +3453,6 @@ "resolved": "https://registry.npmjs.org/is-primitive/-/is-primitive-2.0.0.tgz", "integrity": "sha1-IHurkWOEmcB7Kt8kCkGochADRXU=" }, - "is-promise": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-2.1.0.tgz", - "integrity": "sha1-eaKp7OfwlugPNtKy87wWwf9L8/o=", - "dev": true - }, - "is-property": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/is-property/-/is-property-1.0.2.tgz", - "integrity": "sha1-V/4cTkhHTt1lsJkR8msc1Ald2oQ=", - "dev": true - }, "is-retry-allowed": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/is-retry-allowed/-/is-retry-allowed-1.1.0.tgz", @@ -5613,12 +3464,6 @@ "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-1.1.0.tgz", "integrity": "sha1-EtSj3U5o4Lec6428hBc66A2RykQ=" }, - "is-typedarray": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz", - "integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=", - "dev": true - }, "is-upper-case": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/is-upper-case/-/is-upper-case-1.1.2.tgz", @@ -5675,23 +3520,6 @@ } } }, - "jasmine": { - "version": "2.99.0", - "resolved": "https://registry.npmjs.org/jasmine/-/jasmine-2.99.0.tgz", - "integrity": "sha1-jKctEC5jm4Z8ZImFbg4YqceqQrc=", - "dev": true, - "requires": { - "exit": "0.1.2", - "glob": "github:lucetius/node-glob#51c7ca6e69bfbd17db5f1ea710e3f2a7a457d9ce", - "jasmine-core": "2.99.1" - } - }, - "jasmine-core": { - "version": "2.99.1", - "resolved": "https://registry.npmjs.org/jasmine-core/-/jasmine-core-2.99.1.tgz", - "integrity": "sha1-5kAN8ea1bhMLYcS80JPap/boyhU=", - "dev": true - }, "jju": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/jju/-/jju-1.2.1.tgz", @@ -5702,12 +3530,6 @@ "resolved": "https://registry.npmjs.org/jmespath/-/jmespath-0.15.0.tgz", "integrity": "sha1-o/Iiqarp+Wb10nx5ZRDigJF2Qhc=" }, - "js-tokens": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-3.0.2.tgz", - "integrity": "sha1-mGbfOVECEw449/mWvOtlRDIJwls=", - "dev": true - }, "js-yaml": { "version": "3.11.0", "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.11.0.tgz", @@ -5725,13 +3547,6 @@ "xmlcreate": "1.0.2" } }, - "jsbn": { - "version": "0.1.1", - "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz", - "integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM=", - "dev": true, - "optional": true - }, "jscs": { "version": "1.12.0", "resolved": "https://registry.npmjs.org/jscs/-/jscs-1.12.0.tgz", @@ -5881,12 +3696,6 @@ "underscore": "1.8.3" } }, - "jsesc": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-1.3.0.tgz", - "integrity": "sha1-RsP+yMGJKxKwgz25vHYiF226s0s=", - "dev": true - }, "jshint": { "version": "2.5.11", "resolved": "https://registry.npmjs.org/jshint/-/jshint-2.5.11.tgz", @@ -5989,12 +3798,6 @@ } } }, - "json-schema": { - "version": "0.2.3", - "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.3.tgz", - "integrity": "sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM=", - "dev": true - }, "json-schema-ref-parser": { "version": "3.3.1", "resolved": "https://registry.npmjs.org/json-schema-ref-parser/-/json-schema-ref-parser-3.3.1.tgz", @@ -6031,24 +3834,6 @@ "jsonify": "0.0.0" } }, - "json-stringify-safe": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", - "integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus=", - "dev": true - }, - "json3": { - "version": "3.3.2", - "resolved": "https://registry.npmjs.org/json3/-/json3-3.3.2.tgz", - "integrity": "sha1-PAQ0dD35Pi9cQq7nsZvLSDV19OE=", - "dev": true - }, - "json5": { - "version": "0.5.1", - "resolved": "https://registry.npmjs.org/json5/-/json5-0.5.1.tgz", - "integrity": "sha1-Hq3nrMASA0rYTiOWdn6tn6VJWCE=", - "dev": true - }, "jsonfile": { "version": "2.4.0", "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-2.4.0.tgz", @@ -6067,12 +3852,6 @@ "resolved": "https://registry.npmjs.org/jsonparse/-/jsonparse-1.3.1.tgz", "integrity": "sha1-P02uSpH6wxX3EGL4UhzCOfE2YoA=" }, - "jsonpointer": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/jsonpointer/-/jsonpointer-4.0.1.tgz", - "integrity": "sha1-T9kss04OnbPInIYi7PUfm5eMbLk=", - "dev": true - }, "jsonwebtoken": { "version": "8.1.0", "resolved": "https://registry.npmjs.org/jsonwebtoken/-/jsonwebtoken-8.1.0.tgz", @@ -6090,18 +3869,6 @@ "xtend": "4.0.1" } }, - "jsprim": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.1.tgz", - "integrity": "sha1-MT5mvB5cwG5Di8G3SZwuXFastqI=", - "dev": true, - "requires": { - "assert-plus": "1.0.0", - "extsprintf": "1.3.0", - "json-schema": "0.2.3", - "verror": "1.10.0" - } - }, "jszip": { "version": "2.6.1", "resolved": "https://registry.npmjs.org/jszip/-/jszip-2.6.1.tgz", @@ -6138,12 +3905,6 @@ "safe-buffer": "5.1.1" } }, - "kew": { - "version": "0.7.0", - "resolved": "https://registry.npmjs.org/kew/-/kew-0.7.0.tgz", - "integrity": "sha1-edk9LTM2PW/dKXCzNdkUGtWR15s=", - "dev": true - }, "kind-of": { "version": "3.2.2", "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", @@ -6230,12 +3991,6 @@ "integrity": "sha1-jaDmqHbPNEwK2KVIghEd08XHyjY=", "dev": true }, - "lodash._basecreate": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/lodash._basecreate/-/lodash._basecreate-3.0.3.tgz", - "integrity": "sha1-G8ZhYU2qf8MRt9A78WgGoCE8+CE=", - "dev": true - }, "lodash._bindcallback": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/lodash._bindcallback/-/lodash._bindcallback-3.0.1.tgz", @@ -6280,17 +4035,6 @@ "resolved": "https://registry.npmjs.org/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz", "integrity": "sha1-4j8/nE+Pvd6HJSnBBxhXoIblzO8=" }, - "lodash.create": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/lodash.create/-/lodash.create-3.1.1.tgz", - "integrity": "sha1-1/KEnw29p+BGgruM1yqwIkYd6+c=", - "dev": true, - "requires": { - "lodash._baseassign": "3.2.0", - "lodash._basecreate": "3.0.3", - "lodash._isiterateecall": "3.0.9" - } - }, "lodash.difference": { "version": "4.5.0", "resolved": "https://registry.npmjs.org/lodash.difference/-/lodash.difference-4.5.0.tgz", @@ -6414,21 +4158,6 @@ } } }, - "loglevel": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/loglevel/-/loglevel-1.4.1.tgz", - "integrity": "sha1-lbOD+Ro8J1b9SrCTZn5DCRYfK80=", - "dev": true - }, - "loose-envify": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.3.1.tgz", - "integrity": "sha1-0aitM/qc4OcT1l/dCsi3SNR4yEg=", - "dev": true, - "requires": { - "js-tokens": "3.0.2" - } - }, "lower-case": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/lower-case/-/lower-case-1.1.4.tgz", @@ -6508,12 +4237,6 @@ "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz", "integrity": "sha1-sAqqVW3YtEVoFQ7J0blT8/kMu2E=" }, - "meteor-promise": { - "version": "0.8.6", - "resolved": "https://registry.npmjs.org/meteor-promise/-/meteor-promise-0.8.6.tgz", - "integrity": "sha512-HP6tOr67z/9XU2Dr0F2SSr8WRTuE23AG9Dj578DCJPEYHs67OLKBviU8A8rwvbwMD7Lu2+Of+yAMz2Wd8r4yxg==", - "dev": true - }, "method-override": { "version": "2.3.10", "resolved": "https://registry.npmjs.org/method-override/-/method-override-2.3.10.tgz", @@ -6577,12 +4300,6 @@ "mime-db": "1.33.0" } }, - "mimic-fn": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-1.2.0.tgz", - "integrity": "sha512-jf84uxzwiuiIVKiOLpfYk7N46TSy8ubTonmneY9vrpHNAnp0QBt2BxWV9dO3/j+BoVAb+a5G6YDPW3M5HOdMWQ==", - "dev": true - }, "minimalistic-assert": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/minimalistic-assert/-/minimalistic-assert-1.0.1.tgz", @@ -6926,21 +4643,6 @@ "resolved": "https://registry.npmjs.org/next-tick/-/next-tick-0.2.2.tgz", "integrity": "sha1-ddpKkn7liH45BliABltzNkE7MQ0=" }, - "nice-try": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/nice-try/-/nice-try-1.0.4.tgz", - "integrity": "sha512-2NpiFHqC87y/zFke0fC0spBXL3bBsoh/p5H1EFhshxjCR5+0g2d6BiXbUFz9v1sAcxsk2htp2eQnNIci2dIYcA==", - "dev": true - }, - "no-case": { - "version": "2.3.2", - "resolved": "https://registry.npmjs.org/no-case/-/no-case-2.3.2.tgz", - "integrity": "sha512-rmTZ9kz+f3rCvK2TD1Ue/oZlns7OGoIWP4fc3llxxRXlOkHKoWPPWJOfFYpITabSow43QJbRIoHQXtt10VldyQ==", - "dev": true, - "requires": { - "lower-case": "1.1.4" - } - }, "nodemon": { "version": "1.13.3", "resolved": "https://registry.npmjs.org/nodemon/-/nodemon-1.13.3.tgz", @@ -10242,12 +7944,6 @@ "semver": "5.5.0" } }, - "npm-install-package": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/npm-install-package/-/npm-install-package-2.1.0.tgz", - "integrity": "sha1-1+/jz816sAYUuJbqUxGdyaslkSU=", - "dev": true - }, "npm-package-arg": { "version": "4.2.1", "resolved": "https://registry.npmjs.org/npm-package-arg/-/npm-package-arg-4.2.1.tgz", @@ -10286,12 +7982,6 @@ "resolved": "https://registry.npmjs.org/nunjucks-markdown/-/nunjucks-markdown-2.0.1.tgz", "integrity": "sha1-1V51Qzo1hQ4sNFZR/j+THtmxVqI=" }, - "oauth-sign": { - "version": "0.8.2", - "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.8.2.tgz", - "integrity": "sha1-Rqarfwrq2N6unsBWV4C31O/rnUM=", - "dev": true - }, "object-assign": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", @@ -10302,24 +7992,6 @@ "resolved": "https://registry.npmjs.org/object-component/-/object-component-0.0.3.tgz", "integrity": "sha1-8MaapQ78lbhmwYb0AKM3acsvEpE=" }, - "object-keys": { - "version": "1.0.11", - "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.0.11.tgz", - "integrity": "sha1-xUYBd4rVYPEULODgG8yotW0TQm0=", - "dev": true - }, - "object.assign": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/object.assign/-/object.assign-4.1.0.tgz", - "integrity": "sha512-exHJeq6kBKj58mqGyTQ9DFvrZC/eR6OwxzoM9YRoGBqrXYonaFyGiFMuc9VZrXf7DarreEwMpurG3dd+CNyW5w==", - "dev": true, - "requires": { - "define-properties": "1.1.2", - "function-bind": "1.1.1", - "has-symbols": "1.0.0", - "object-keys": "1.0.11" - } - }, "object.omit": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/object.omit/-/object.omit-2.0.1.tgz", @@ -10347,15 +8019,6 @@ "resolved": "https://registry.npmjs.org/once/-/once-1.3.0.tgz", "integrity": "sha1-FRr4a/wfCMS58H0GqyUP/L61ZYE=" }, - "onetime": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/onetime/-/onetime-2.0.1.tgz", - "integrity": "sha1-BnQoIw/WdEOyeUsiu6UotoZ5YtQ=", - "dev": true, - "requires": { - "mimic-fn": "1.2.0" - } - }, "ono": { "version": "4.0.5", "resolved": "https://registry.npmjs.org/ono/-/ono-4.0.5.tgz", @@ -10364,26 +8027,10 @@ "format-util": "1.0.3" } }, - "optimist": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/optimist/-/optimist-0.6.1.tgz", - "integrity": "sha1-2j6nRob6IaGaERwybpDrFaAZZoY=", - "dev": true, - "requires": { - "minimist": "0.0.8", - "wordwrap": "0.0.3" - } - }, "os-browserify": { "version": "0.3.0", - "resolved": "https://registry.npmjs.org/os-browserify/-/os-browserify-0.3.0.tgz", - "integrity": "sha1-hUNzx/XCMVkU/Jv8a9gjj92h7Cc=" - }, - "os-homedir": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/os-homedir/-/os-homedir-1.0.2.tgz", - "integrity": "sha1-/7xJiDNuDoM94MFox+8VISGqf7M=", - "dev": true + "resolved": "https://registry.npmjs.org/os-browserify/-/os-browserify-0.3.0.tgz", + "integrity": "sha1-hUNzx/XCMVkU/Jv8a9gjj92h7Cc=" }, "os-locale": { "version": "1.4.0", @@ -10402,12 +8049,6 @@ "win-release": "1.1.1" } }, - "os-tmpdir": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz", - "integrity": "sha1-u+Z0BseaqFxc/sdm/lc0VV36EnQ=", - "dev": true - }, "osx-release": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/osx-release/-/osx-release-1.1.0.tgz", @@ -10429,12 +8070,6 @@ "integrity": "sha1-P7z7FbiZpEEjs0ttzBi3JDNqLK4=", "dev": true }, - "p-map": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/p-map/-/p-map-1.2.0.tgz", - "integrity": "sha512-r6zKACMNhjPJMTl8KcFH4li//gkrXWfbD6feV8l6doRHlzljFWGJ2AP6iKaCJXyZmAUMOPtvbW7EXkbWO/pLEA==", - "dev": true - }, "pako": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.6.tgz", @@ -10582,297 +8217,12 @@ "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", "integrity": "sha1-elfrVQpng/kRUzH89GY9XI4AelA=" }, - "performance-now": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", - "integrity": "sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=", - "dev": true - }, - "phantomjs-prebuilt": { - "version": "2.1.13", - "resolved": "https://registry.npmjs.org/phantomjs-prebuilt/-/phantomjs-prebuilt-2.1.13.tgz", - "integrity": "sha1-ZlVq2ell2JPKWn3J52PffoaX920=", - "dev": true, - "requires": { - "es6-promise": "4.0.5", - "extract-zip": "1.5.0", - "fs-extra": "0.30.0", - "hasha": "2.2.0", - "kew": "0.7.0", - "progress": "1.1.8", - "request": "2.74.0", - "request-progress": "2.0.1", - "which": "1.2.14" - }, - "dependencies": { - "assert-plus": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-0.2.0.tgz", - "integrity": "sha1-104bh+ev/A24qttwIfP+SBAasjQ=", - "dev": true - }, - "async": { - "version": "2.6.0", - "resolved": "https://registry.npmjs.org/async/-/async-2.6.0.tgz", - "integrity": "sha512-xAfGg1/NTLBBKlHFmnd7PlmUW9KhVQIUuSrYem9xzFUZy13ScvtyGGejaae9iAVRiRq9+Cx7DPFaAAhCpyxyPw==", - "dev": true, - "requires": { - "lodash": "4.17.4" - } - }, - "aws-sign2": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.6.0.tgz", - "integrity": "sha1-FDQt0428yU0OW4fXY81jYSwOeU8=", - "dev": true - }, - "boom": { - "version": "2.10.1", - "resolved": "https://registry.npmjs.org/boom/-/boom-2.10.1.tgz", - "integrity": "sha1-OciRjO/1eZ+D+UkqhI9iWt0Mdm8=", - "dev": true, - "requires": { - "hoek": "2.16.3" - } - }, - "caseless": { - "version": "0.11.0", - "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.11.0.tgz", - "integrity": "sha1-cVuW6phBWTzDMGeSP17GDr2k99c=", - "dev": true - }, - "concat-stream": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.5.0.tgz", - "integrity": "sha1-U/fUPFHF5D+ByP3QMyHGMb5o1hE=", - "dev": true, - "requires": { - "inherits": "2.0.3", - "readable-stream": "2.0.6", - "typedarray": "0.0.6" - } - }, - "cryptiles": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/cryptiles/-/cryptiles-2.0.5.tgz", - "integrity": "sha1-O9/s3GCBR8HGcgL6KR59ylnqo7g=", - "dev": true, - "requires": { - "boom": "2.10.1" - } - }, - "debug": { - "version": "0.7.4", - "resolved": "https://registry.npmjs.org/debug/-/debug-0.7.4.tgz", - "integrity": "sha1-BuHqgILCyxTjmAbiLi9vdX+Srzk=", - "dev": true - }, - "es6-promise": { - "version": "4.0.5", - "resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-4.0.5.tgz", - "integrity": "sha1-eILzCt3lskDM+n99eMVIMwlRrkI=", - "dev": true - }, - "extract-zip": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/extract-zip/-/extract-zip-1.5.0.tgz", - "integrity": "sha1-ksz22B73Cp+kwXRxFMzvbYaIpsQ=", - "dev": true, - "requires": { - "concat-stream": "1.5.0", - "debug": "0.7.4", - "mkdirp": "0.5.0", - "yauzl": "2.4.1" - } - }, - "form-data": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-1.0.1.tgz", - "integrity": "sha1-rjFduaSQf6BlUCMEpm13M0de43w=", - "dev": true, - "requires": { - "async": "2.6.0", - "combined-stream": "1.0.6", - "mime-types": "2.1.18" - } - }, - "fs-extra": { - "version": "0.30.0", - "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-0.30.0.tgz", - "integrity": "sha1-8jP/zAjU2n1DLapEl3aYnbHfk/A=", - "dev": true, - "requires": { - "graceful-fs": "4.1.11", - "jsonfile": "2.4.0", - "klaw": "1.3.1", - "path-is-absolute": "1.0.1", - "rimraf": "2.6.2" - } - }, - "har-validator": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-2.0.6.tgz", - "integrity": "sha1-zcvAgYgmWtEZtqWnyKtw7s+10n0=", - "dev": true, - "requires": { - "chalk": "1.1.3", - "commander": "2.12.2", - "is-my-json-valid": "2.17.2", - "pinkie-promise": "2.0.1" - } - }, - "hawk": { - "version": "3.1.3", - "resolved": "https://registry.npmjs.org/hawk/-/hawk-3.1.3.tgz", - "integrity": "sha1-B4REvXwWQLD+VA0sm3PVlnjo4cQ=", - "dev": true, - "requires": { - "boom": "2.10.1", - "cryptiles": "2.0.5", - "hoek": "2.16.3", - "sntp": "1.0.9" - } - }, - "hoek": { - "version": "2.16.3", - "resolved": "https://registry.npmjs.org/hoek/-/hoek-2.16.3.tgz", - "integrity": "sha1-ILt0A9POo5jpHcRxCo/xuCdKJe0=", - "dev": true - }, - "http-signature": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.1.1.tgz", - "integrity": "sha1-33LiZwZs0Kxn+3at+OE0qPvPkb8=", - "dev": true, - "requires": { - "assert-plus": "0.2.0", - "jsprim": "1.4.1", - "sshpk": "1.14.1" - } - }, - "klaw": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/klaw/-/klaw-1.3.1.tgz", - "integrity": "sha1-QIhDO0azsbolnXh4XY6W9zugJDk=", - "dev": true, - "requires": { - "graceful-fs": "4.1.11" - } - }, - "mkdirp": { - "version": "0.5.0", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.0.tgz", - "integrity": "sha1-HXMHam35hs2TROFecfzAWkyavxI=", - "dev": true, - "requires": { - "minimist": "0.0.8" - } - }, - "node-uuid": { - "version": "1.4.8", - "resolved": "https://registry.npmjs.org/node-uuid/-/node-uuid-1.4.8.tgz", - "integrity": "sha1-sEDrCSOWivq/jTL7HxfxFn/auQc=", - "dev": true - }, - "process-nextick-args": { - "version": "1.0.7", - "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-1.0.7.tgz", - "integrity": "sha1-FQ4gt1ZZCtP5EJPyWk8q2L/zC6M=", - "dev": true - }, - "qs": { - "version": "6.2.3", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.2.3.tgz", - "integrity": "sha1-HPyyXBCpsrSDBT/zn138kjOQjP4=", - "dev": true - }, - "readable-stream": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.0.6.tgz", - "integrity": "sha1-j5A0HmilPMySh4jaz80Rs265t44=", - "dev": true, - "requires": { - "core-util-is": "1.0.2", - "inherits": "2.0.3", - "isarray": "1.0.0", - "process-nextick-args": "1.0.7", - "string_decoder": "0.10.31", - "util-deprecate": "1.0.2" - } - }, - "request": { - "version": "2.74.0", - "resolved": "https://registry.npmjs.org/request/-/request-2.74.0.tgz", - "integrity": "sha1-dpPKdou7DqXIzgjAhKRe+gW4kqs=", - "dev": true, - "requires": { - "aws-sign2": "0.6.0", - "aws4": "1.6.0", - "bl": "1.1.2", - "caseless": "0.11.0", - "combined-stream": "1.0.6", - "extend": "3.0.1", - "forever-agent": "0.6.1", - "form-data": "1.0.1", - "har-validator": "2.0.6", - "hawk": "3.1.3", - "http-signature": "1.1.1", - "is-typedarray": "1.0.0", - "isstream": "0.1.2", - "json-stringify-safe": "5.0.1", - "mime-types": "2.1.18", - "node-uuid": "1.4.8", - "oauth-sign": "0.8.2", - "qs": "6.2.3", - "stringstream": "0.0.5", - "tough-cookie": "2.3.4", - "tunnel-agent": "0.4.3" - } - }, - "sntp": { - "version": "1.0.9", - "resolved": "https://registry.npmjs.org/sntp/-/sntp-1.0.9.tgz", - "integrity": "sha1-ZUEYTMkK7qbG57NeJlkIJEPGYZg=", - "dev": true, - "requires": { - "hoek": "2.16.3" - } - }, - "string_decoder": { - "version": "0.10.31", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", - "integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=", - "dev": true - }, - "tunnel-agent": { - "version": "0.4.3", - "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.4.3.tgz", - "integrity": "sha1-Y3PbdpCf5XDgjXNYM2Xtgop07us=", - "dev": true - } - } - }, "pify": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/pify/-/pify-3.0.0.tgz", "integrity": "sha1-5aSs0sEB/fPZpNB/DbxNtJ3SgXY=", "dev": true }, - "pinkie": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/pinkie/-/pinkie-2.0.4.tgz", - "integrity": "sha1-clVrgM+g1IqXToDnckjoDtT3+HA=", - "dev": true - }, - "pinkie-promise": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/pinkie-promise/-/pinkie-promise-2.0.1.tgz", - "integrity": "sha1-ITXW36ejWMBprJsXh3YogihFD/o=", - "dev": true, - "requires": { - "pinkie": "2.0.4" - } - }, "pkginfo": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/pkginfo/-/pkginfo-0.4.1.tgz", @@ -10947,12 +8297,6 @@ "resolved": "https://registry.npmjs.org/preserve/-/preserve-0.2.0.tgz", "integrity": "sha1-gV7R9uvGWSb4ZbMQwHE7yzMVzks=" }, - "private": { - "version": "0.1.8", - "resolved": "https://registry.npmjs.org/private/-/private-0.1.8.tgz", - "integrity": "sha512-VvivMrbvd2nKkiG38qjULzlc+4Vx4wm/whI9pQD35YrARNnhxeiRktSOhSukRLFNlzg6Br/cJPet5J/u19r/mg==", - "dev": true - }, "process": { "version": "0.11.10", "resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz", @@ -10963,12 +8307,6 @@ "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.0.tgz", "integrity": "sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw==" }, - "progress": { - "version": "1.1.8", - "resolved": "https://registry.npmjs.org/progress/-/progress-1.1.8.tgz", - "integrity": "sha1-4mDHj2Fhzdmw5WzD4Khd4Xx6V74=", - "dev": true - }, "prompt": { "version": "0.2.14", "resolved": "https://registry.npmjs.org/prompt/-/prompt-0.2.14.tgz", @@ -11318,29 +8656,6 @@ "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-2.6.0.tgz", "integrity": "sha1-Uu0J2srBCPGmMcB+m2mUHnoZUEs=" }, - "regenerate": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/regenerate/-/regenerate-1.3.3.tgz", - "integrity": "sha512-jVpo1GadrDAK59t/0jRx5VxYWQEDkkEKi6+HjE3joFVLfDOh9Xrdh0dF1eSq+BI/SwvTQ44gSscJ8N5zYL61sg==", - "dev": true - }, - "regenerator-runtime": { - "version": "0.11.1", - "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.11.1.tgz", - "integrity": "sha512-MguG95oij0fC3QV3URf4V2SDYGJhJnJGqvIIgdECeODCT98wSWDAJ94SSuVpYQUoTcGUIL6L4yNB7j1DFFHSBg==", - "dev": true - }, - "regenerator-transform": { - "version": "0.10.1", - "resolved": "https://registry.npmjs.org/regenerator-transform/-/regenerator-transform-0.10.1.tgz", - "integrity": "sha512-PJepbvDbuK1xgIgnau7Y90cwaAmO/LCLMI2mPvaXq2heGMR3aWW5/BQvYrhJ8jgmQjXewXvBjzfqKcVOmhjZ6Q==", - "dev": true, - "requires": { - "babel-runtime": "6.26.0", - "babel-types": "6.26.0", - "private": "0.1.8" - } - }, "regex-cache": { "version": "0.4.4", "resolved": "https://registry.npmjs.org/regex-cache/-/regex-cache-0.4.4.tgz", @@ -11349,17 +8664,6 @@ "is-equal-shallow": "0.1.3" } }, - "regexpu-core": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/regexpu-core/-/regexpu-core-2.0.0.tgz", - "integrity": "sha1-SdA4g3uNz4v6W5pCE5k45uoq4kA=", - "dev": true, - "requires": { - "regenerate": "1.3.3", - "regjsgen": "0.2.0", - "regjsparser": "0.1.5" - } - }, "registry-auth-token": { "version": "3.3.1", "resolved": "https://registry.npmjs.org/registry-auth-token/-/registry-auth-token-3.3.1.tgz", @@ -11414,29 +8718,6 @@ } } }, - "regjsgen": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/regjsgen/-/regjsgen-0.2.0.tgz", - "integrity": "sha1-bAFq3qxVT3WCP+N6wFuS1aTtsfc=", - "dev": true - }, - "regjsparser": { - "version": "0.1.5", - "resolved": "https://registry.npmjs.org/regjsparser/-/regjsparser-0.1.5.tgz", - "integrity": "sha1-fuj4Tcb6eS0/0K4ijSS9lJ6tIFw=", - "dev": true, - "requires": { - "jsesc": "0.5.0" - }, - "dependencies": { - "jsesc": { - "version": "0.5.0", - "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-0.5.0.tgz", - "integrity": "sha1-597mbjXW/Bb3EP6R1c9p9w8IkR0=", - "dev": true - } - } - }, "remove-trailing-separator": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/remove-trailing-separator/-/remove-trailing-separator-1.1.0.tgz", @@ -11452,64 +8733,6 @@ "resolved": "https://registry.npmjs.org/repeat-string/-/repeat-string-1.6.1.tgz", "integrity": "sha1-jcrkcOHIirwtYA//Sndihtp15jc=" }, - "repeating": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/repeating/-/repeating-2.0.1.tgz", - "integrity": "sha1-UhTFOpJtNVJwdSf7q0FdvAjQbdo=", - "dev": true, - "requires": { - "is-finite": "1.0.2" - } - }, - "request": { - "version": "2.85.0", - "resolved": "https://registry.npmjs.org/request/-/request-2.85.0.tgz", - "integrity": "sha512-8H7Ehijd4js+s6wuVPLjwORxD4zeuyjYugprdOXlPSqaApmL/QOy+EB/beICHVCHkGMKNh5rvihb5ov+IDw4mg==", - "dev": true, - "requires": { - "aws-sign2": "0.7.0", - "aws4": "1.6.0", - "caseless": "0.12.0", - "combined-stream": "1.0.6", - "extend": "3.0.1", - "forever-agent": "0.6.1", - "form-data": "2.3.2", - "har-validator": "5.0.3", - "hawk": "6.0.2", - "http-signature": "1.2.0", - "is-typedarray": "1.0.0", - "isstream": "0.1.2", - "json-stringify-safe": "5.0.1", - "mime-types": "2.1.18", - "oauth-sign": "0.8.2", - "performance-now": "2.1.0", - "qs": "6.5.1", - "safe-buffer": "5.1.1", - "stringstream": "0.0.5", - "tough-cookie": "2.3.4", - "tunnel-agent": "0.6.0", - "uuid": "3.2.1" - } - }, - "request-progress": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/request-progress/-/request-progress-2.0.1.tgz", - "integrity": "sha1-XTa7V5YcZzqlt4jbyBQf3yO0Tgg=", - "dev": true, - "requires": { - "throttleit": "1.0.0" - } - }, - "requestretry": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/requestretry/-/requestretry-1.5.0.tgz", - "integrity": "sha1-7RV7ulNSbt6z7DKo5wSkmYvs5ic=", - "dev": true, - "requires": { - "fg-lodash": "0.0.2", - "request": "2.85.0" - } - }, "require-uncached": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/require-uncached/-/require-uncached-1.0.3.tgz", @@ -11569,25 +8792,9 @@ } }, "resolve-from": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-1.0.1.tgz", - "integrity": "sha1-Jsv+k10a7uq7Kbw/5a6wHpPUQiY=" - }, - "resolve-url": { - "version": "0.2.1", - "resolved": "https://registry.npmjs.org/resolve-url/-/resolve-url-0.2.1.tgz", - "integrity": "sha1-LGN/53yJOv0qZj/iGqkIAGjiBSo=", - "dev": true - }, - "restore-cursor": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-2.0.0.tgz", - "integrity": "sha1-n37ih/gv0ybU/RYpI9YhKe7g368=", - "dev": true, - "requires": { - "onetime": "2.0.1", - "signal-exit": "3.0.2" - } + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-1.0.1.tgz", + "integrity": "sha1-Jsv+k10a7uq7Kbw/5a6wHpPUQiY=" }, "revalidator": { "version": "0.1.8", @@ -11595,12 +8802,6 @@ "integrity": "sha1-/s5hv6DBtSoga9axgZgYS91SOjs=", "dev": true }, - "rgb2hex": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/rgb2hex/-/rgb2hex-0.1.0.tgz", - "integrity": "sha1-zNVfhgrgxcTqN1BLlY5ELY0SMls=", - "dev": true - }, "rimraf": { "version": "2.6.2", "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.6.2.tgz", @@ -11697,30 +8898,6 @@ "inherits": "2.0.3" } }, - "run-async": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/run-async/-/run-async-2.3.0.tgz", - "integrity": "sha1-A3GrSuC91yDUFm19/aZP96RFpsA=", - "dev": true, - "requires": { - "is-promise": "2.1.0" - } - }, - "rx-lite": { - "version": "4.0.8", - "resolved": "https://registry.npmjs.org/rx-lite/-/rx-lite-4.0.8.tgz", - "integrity": "sha1-Cx4Rr4vESDbwSmQH6S2kJGe3lEQ=", - "dev": true - }, - "rx-lite-aggregates": { - "version": "4.0.8", - "resolved": "https://registry.npmjs.org/rx-lite-aggregates/-/rx-lite-aggregates-4.0.8.tgz", - "integrity": "sha1-dTuHqJoRyVRnxKwWJsTvxOBcZ74=", - "dev": true, - "requires": { - "rx-lite": "4.0.8" - } - }, "safe-buffer": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.1.tgz", @@ -11781,233 +8958,11 @@ } } }, - "saucelabs": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/saucelabs/-/saucelabs-1.4.0.tgz", - "integrity": "sha1-uTSpr52ih0s/QKrh/N5QpEZvXzg=", - "dev": true, - "requires": { - "https-proxy-agent": "1.0.0" - } - }, "sax": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.1.tgz", "integrity": "sha1-e45lYZCyKOgaZq6nSEgNgozS03o=" }, - "selenium-standalone": { - "version": "6.13.0", - "resolved": "https://registry.npmjs.org/selenium-standalone/-/selenium-standalone-6.13.0.tgz", - "integrity": "sha512-JV1+AySJ9RccsAIZlnDb8QUx/SDeVGK8j/zCJcNlbz/LCRiketU3Su0YE5Uc1raaBGpT/venof0+9Ry0oGuvYw==", - "dev": true, - "requires": { - "async": "2.6.0", - "commander": "2.12.2", - "cross-spawn": "6.0.5", - "debug": "3.1.0", - "lodash": "4.17.4", - "minimist": "1.2.0", - "mkdirp": "0.5.1", - "progress": "2.0.0", - "request": "2.79.0", - "tar-stream": "1.5.2", - "urijs": "1.19.1", - "which": "1.2.14", - "yauzl": "2.9.1" - }, - "dependencies": { - "assert-plus": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-0.2.0.tgz", - "integrity": "sha1-104bh+ev/A24qttwIfP+SBAasjQ=", - "dev": true - }, - "async": { - "version": "2.6.0", - "resolved": "https://registry.npmjs.org/async/-/async-2.6.0.tgz", - "integrity": "sha512-xAfGg1/NTLBBKlHFmnd7PlmUW9KhVQIUuSrYem9xzFUZy13ScvtyGGejaae9iAVRiRq9+Cx7DPFaAAhCpyxyPw==", - "dev": true, - "requires": { - "lodash": "4.17.4" - } - }, - "aws-sign2": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.6.0.tgz", - "integrity": "sha1-FDQt0428yU0OW4fXY81jYSwOeU8=", - "dev": true - }, - "boom": { - "version": "2.10.1", - "resolved": "https://registry.npmjs.org/boom/-/boom-2.10.1.tgz", - "integrity": "sha1-OciRjO/1eZ+D+UkqhI9iWt0Mdm8=", - "dev": true, - "requires": { - "hoek": "2.16.3" - } - }, - "caseless": { - "version": "0.11.0", - "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.11.0.tgz", - "integrity": "sha1-cVuW6phBWTzDMGeSP17GDr2k99c=", - "dev": true - }, - "cross-spawn": { - "version": "6.0.5", - "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-6.0.5.tgz", - "integrity": "sha512-eTVLrBSt7fjbDygz805pMnstIs2VTBNkRm0qxZd+M7A5XDdxVRWO5MxGBXZhjY4cqLYLdtrGqRf8mBPmzwSpWQ==", - "dev": true, - "requires": { - "nice-try": "1.0.4", - "path-key": "2.0.1", - "semver": "5.5.0", - "shebang-command": "1.2.0", - "which": "1.2.14" - } - }, - "cryptiles": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/cryptiles/-/cryptiles-2.0.5.tgz", - "integrity": "sha1-O9/s3GCBR8HGcgL6KR59ylnqo7g=", - "dev": true, - "requires": { - "boom": "2.10.1" - } - }, - "debug": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz", - "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==", - "dev": true, - "requires": { - "ms": "2.0.0" - } - }, - "form-data": { - "version": "2.1.4", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.1.4.tgz", - "integrity": "sha1-M8GDrPGTJ27KqYFDpp6Uv+4XUNE=", - "dev": true, - "requires": { - "asynckit": "0.4.0", - "combined-stream": "1.0.6", - "mime-types": "2.1.18" - } - }, - "har-validator": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-2.0.6.tgz", - "integrity": "sha1-zcvAgYgmWtEZtqWnyKtw7s+10n0=", - "dev": true, - "requires": { - "chalk": "1.1.3", - "commander": "2.12.2", - "is-my-json-valid": "2.17.2", - "pinkie-promise": "2.0.1" - } - }, - "hawk": { - "version": "3.1.3", - "resolved": "https://registry.npmjs.org/hawk/-/hawk-3.1.3.tgz", - "integrity": "sha1-B4REvXwWQLD+VA0sm3PVlnjo4cQ=", - "dev": true, - "requires": { - "boom": "2.10.1", - "cryptiles": "2.0.5", - "hoek": "2.16.3", - "sntp": "1.0.9" - } - }, - "hoek": { - "version": "2.16.3", - "resolved": "https://registry.npmjs.org/hoek/-/hoek-2.16.3.tgz", - "integrity": "sha1-ILt0A9POo5jpHcRxCo/xuCdKJe0=", - "dev": true - }, - "http-signature": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.1.1.tgz", - "integrity": "sha1-33LiZwZs0Kxn+3at+OE0qPvPkb8=", - "dev": true, - "requires": { - "assert-plus": "0.2.0", - "jsprim": "1.4.1", - "sshpk": "1.14.1" - } - }, - "minimist": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz", - "integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ=", - "dev": true - }, - "progress": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.0.tgz", - "integrity": "sha1-ihvjZr+Pwj2yvSPxDG/pILQ4nR8=", - "dev": true - }, - "qs": { - "version": "6.3.2", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.3.2.tgz", - "integrity": "sha1-51vV9uJoEioqDgvaYwslUMFmUCw=", - "dev": true - }, - "request": { - "version": "2.79.0", - "resolved": "https://registry.npmjs.org/request/-/request-2.79.0.tgz", - "integrity": "sha1-Tf5b9r6LjNw3/Pk+BLZVd3InEN4=", - "dev": true, - "requires": { - "aws-sign2": "0.6.0", - "aws4": "1.6.0", - "caseless": "0.11.0", - "combined-stream": "1.0.6", - "extend": "3.0.1", - "forever-agent": "0.6.1", - "form-data": "2.1.4", - "har-validator": "2.0.6", - "hawk": "3.1.3", - "http-signature": "1.1.1", - "is-typedarray": "1.0.0", - "isstream": "0.1.2", - "json-stringify-safe": "5.0.1", - "mime-types": "2.1.18", - "oauth-sign": "0.8.2", - "qs": "6.3.2", - "stringstream": "0.0.5", - "tough-cookie": "2.3.4", - "tunnel-agent": "0.4.3", - "uuid": "3.2.1" - } - }, - "sntp": { - "version": "1.0.9", - "resolved": "https://registry.npmjs.org/sntp/-/sntp-1.0.9.tgz", - "integrity": "sha1-ZUEYTMkK7qbG57NeJlkIJEPGYZg=", - "dev": true, - "requires": { - "hoek": "2.16.3" - } - }, - "tunnel-agent": { - "version": "0.4.3", - "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.4.3.tgz", - "integrity": "sha1-Y3PbdpCf5XDgjXNYM2Xtgop07us=", - "dev": true - }, - "yauzl": { - "version": "2.9.1", - "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-2.9.1.tgz", - "integrity": "sha1-qBmB6nCleUYTOIPwKcWCGok1mn8=", - "dev": true, - "requires": { - "buffer-crc32": "0.2.13", - "fd-slicer": "1.0.1" - } - } - } - }, "semver": { "version": "5.5.0", "resolved": "https://registry.npmjs.org/semver/-/semver-5.5.0.tgz", @@ -12192,12 +9147,6 @@ } } }, - "slash": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/slash/-/slash-1.0.0.tgz", - "integrity": "sha1-xB8vbDn8FtHNF61LXYlhFK5HDVU=", - "dev": true - }, "snake-case": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/snake-case/-/snake-case-1.1.2.tgz", @@ -12206,15 +9155,6 @@ "sentence-case": "1.1.3" } }, - "sntp": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/sntp/-/sntp-2.1.0.tgz", - "integrity": "sha512-FL1b58BDrqS3A11lJ0zEdnJ3UOKqVxawAkF3k7F0CVN7VQ34aZrV+G8BZ1WC9ZL7NyrwsW0oviwsWDgRuVYtJg==", - "dev": true, - "requires": { - "hoek": "4.2.1" - } - }, "socket.io": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/socket.io/-/socket.io-2.0.4.tgz", @@ -12295,33 +9235,6 @@ "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.5.7.tgz", "integrity": "sha1-igOdLRAh0i0eoUyA2OpGi6LvP8w=" }, - "source-map-resolve": { - "version": "0.3.1", - "resolved": "https://registry.npmjs.org/source-map-resolve/-/source-map-resolve-0.3.1.tgz", - "integrity": "sha1-YQ9hIqRFuN1RU1oqcbeD38Ekh2E=", - "dev": true, - "requires": { - "atob": "1.1.3", - "resolve-url": "0.2.1", - "source-map-url": "0.3.0", - "urix": "0.1.0" - } - }, - "source-map-support": { - "version": "0.4.18", - "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.4.18.tgz", - "integrity": "sha512-try0/JqxPLF9nOjvSta7tVondkP5dwgyLDjVoyMDlmjugT2lRZ1OfsrYTkCd2hkDnJTKRbO/Rl3orm8vlsUzbA==", - "dev": true, - "requires": { - "source-map": "0.5.7" - } - }, - "source-map-url": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/source-map-url/-/source-map-url-0.3.0.tgz", - "integrity": "sha1-fsrxO1e80J2opAxdJp2zN5nUqvk=", - "dev": true - }, "spdx-license-ids": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/spdx-license-ids/-/spdx-license-ids-1.2.2.tgz", @@ -12341,85 +9254,11 @@ "number-is-nan": "1.0.1" } }, - "sshpk": { - "version": "1.14.1", - "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.14.1.tgz", - "integrity": "sha1-Ew9Zde3a2WPx1W+SuaxsUfqfg+s=", - "dev": true, - "requires": { - "asn1": "0.2.3", - "assert-plus": "1.0.0", - "bcrypt-pbkdf": "1.0.1", - "dashdash": "1.14.1", - "ecc-jsbn": "0.1.1", - "getpass": "0.1.7", - "jsbn": "0.1.1", - "tweetnacl": "0.14.5" - } - }, - "stack-chain": { - "version": "1.3.7", - "resolved": "https://registry.npmjs.org/stack-chain/-/stack-chain-1.3.7.tgz", - "integrity": "sha1-0ZLJ/06moiyUxN1FkXHj8AzqEoU=", - "dev": true - }, - "stack-generator": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/stack-generator/-/stack-generator-1.1.0.tgz", - "integrity": "sha1-NvapIHUabBD0maE8Msu19RoLiyU=", - "dev": true, - "requires": { - "stackframe": "1.0.4" - }, - "dependencies": { - "stackframe": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/stackframe/-/stackframe-1.0.4.tgz", - "integrity": "sha512-to7oADIniaYwS3MhtCa/sQhrxidCCQiF/qp4/m5iN3ipf0Y7Xlri0f6eG29r08aL7JYl8n32AF3Q5GYBZ7K8vw==", - "dev": true - } - } - }, "stack-trace": { "version": "0.0.10", "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.10.tgz", "integrity": "sha1-VHxws0fo0ytOEI6hoqFZ5f3eGcA=" }, - "stackframe": { - "version": "0.3.1", - "resolved": "https://registry.npmjs.org/stackframe/-/stackframe-0.3.1.tgz", - "integrity": "sha1-M6qE8Rd6VUjIk1Uzy/6zQgl19aQ=", - "dev": true - }, - "stacktrace-gps": { - "version": "2.4.4", - "resolved": "https://registry.npmjs.org/stacktrace-gps/-/stacktrace-gps-2.4.4.tgz", - "integrity": "sha1-acgn6dbW9Bz0ONfxleLjy/zyjEQ=", - "dev": true, - "requires": { - "source-map": "0.5.6", - "stackframe": "0.3.1" - }, - "dependencies": { - "source-map": { - "version": "0.5.6", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.5.6.tgz", - "integrity": "sha1-dc449SvwczxafwwRjYEzSiu19BI=", - "dev": true - } - } - }, - "stacktrace-js": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/stacktrace-js/-/stacktrace-js-1.3.1.tgz", - "integrity": "sha1-Z8qyWJr1xBe5Yvc2mUAne7O2oYs=", - "dev": true, - "requires": { - "error-stack-parser": "1.3.6", - "stack-generator": "1.1.0", - "stacktrace-gps": "2.4.4" - } - }, "statuses": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", @@ -12511,12 +9350,6 @@ "safe-buffer": "5.1.1" } }, - "stringstream": { - "version": "0.0.5", - "resolved": "https://registry.npmjs.org/stringstream/-/stringstream-0.0.5.tgz", - "integrity": "sha1-TkhM1N5aC7vuGORjB3EKioFiGHg=", - "dev": true - }, "strip-ansi": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", @@ -12578,12 +9411,6 @@ } } }, - "supports-color": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz", - "integrity": "sha1-U10EXOa2Nj+kARcIRimZXp3zJMc=", - "dev": true - }, "swap-case": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/swap-case/-/swap-case-1.1.2.tgz", @@ -12669,12 +9496,6 @@ "resolved": "https://registry.npmjs.org/text-hex/-/text-hex-0.0.0.tgz", "integrity": "sha1-V4+8haapJjbkLdF7QdAhjM6esrM=" }, - "throttleit": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/throttleit/-/throttleit-1.0.0.tgz", - "integrity": "sha1-nnhYNtr0Z0MUWlmEtiaNgoUorGw=", - "dev": true - }, "through": { "version": "2.3.8", "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", @@ -12727,15 +9548,6 @@ "upper-case": "1.1.3" } }, - "tmp": { - "version": "0.0.33", - "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.33.tgz", - "integrity": "sha512-jRCJlojKnZ3addtTOjdIqoRuPEKBvNXcGYqzO6zWZX8KfKEpnGY5jfggJQ3EjKuu8D4bJRr0y+cYJFmYbImXGw==", - "dev": true, - "requires": { - "os-tmpdir": "1.0.2" - } - }, "to-array": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/to-array/-/to-array-0.1.4.tgz", @@ -12746,12 +9558,6 @@ "resolved": "https://registry.npmjs.org/to-arraybuffer/-/to-arraybuffer-1.0.1.tgz", "integrity": "sha1-fSKbH8xjfkZsoIEYCDanqr/4P0M=" }, - "to-fast-properties": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-1.0.3.tgz", - "integrity": "sha1-uDVx+k2MJbguIxsG46MFXeTKGkc=", - "dev": true - }, "to-iso-string": { "version": "0.0.2", "resolved": "https://registry.npmjs.org/to-iso-string/-/to-iso-string-0.0.2.tgz", @@ -12763,42 +9569,11 @@ "resolved": "https://registry.npmjs.org/to-utf8/-/to-utf8-0.0.1.tgz", "integrity": "sha1-0Xrqcv8vujm55DYBvns/9y4ImFI=" }, - "tough-cookie": { - "version": "2.3.4", - "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.3.4.tgz", - "integrity": "sha512-TZ6TTfI5NtZnuyy/Kecv+CnoROnyXn2DN97LontgQpCwsX2XyLYCC0ENhYkehSOwAp8rTQKc/NUIF7BkQ5rKLA==", - "dev": true, - "requires": { - "punycode": "1.4.1" - } - }, - "trim-right": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/trim-right/-/trim-right-1.0.1.tgz", - "integrity": "sha1-yy4SAwZ+DI3h9hQJS5/kVwTqYAM=", - "dev": true - }, "tty-browserify": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/tty-browserify/-/tty-browserify-0.0.1.tgz", "integrity": "sha512-C3TaO7K81YvjCgQH9Q1S3R3P3BtN3RIM8n+OvX4il1K1zgE8ZhI0op7kClgkxtutIE8hQrcrHBXvIheqKUUCxw==" }, - "tunnel-agent": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", - "integrity": "sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0=", - "dev": true, - "requires": { - "safe-buffer": "5.1.1" - } - }, - "tweetnacl": { - "version": "0.14.5", - "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz", - "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=", - "dev": true, - "optional": true - }, "type-detect": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-1.0.0.tgz", @@ -12854,12 +9629,6 @@ } } }, - "underscore.string": { - "version": "2.3.3", - "resolved": "https://registry.npmjs.org/underscore.string/-/underscore.string-2.3.3.tgz", - "integrity": "sha1-ccCL9rQosRM/N+ePo6Icgvcymw0=", - "dev": true - }, "unique-slug": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/unique-slug/-/unique-slug-2.0.0.tgz", @@ -12901,23 +9670,11 @@ "upper-case": "1.1.3" } }, - "urijs": { - "version": "1.19.1", - "resolved": "https://registry.npmjs.org/urijs/-/urijs-1.19.1.tgz", - "integrity": "sha512-xVrGVi94ueCJNrBSTjWqjvtgvl3cyOTThp2zaMaFNGp3F542TR6sM3f2o8RqZl+AwteClSVmoCyt0ka4RjQOQg==", - "dev": true - }, "uritemplate": { "version": "0.3.4", "resolved": "https://registry.npmjs.org/uritemplate/-/uritemplate-0.3.4.tgz", "integrity": "sha1-BdCoU/+8iw9Jqj1NKtd3sNHuBww=" }, - "urix": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/urix/-/urix-0.1.0.tgz", - "integrity": "sha1-2pN/emLiH+wf0Y1Js1wpNQZ6bHI=", - "dev": true - }, "url": { "version": "0.10.3", "resolved": "https://registry.npmjs.org/url/-/url-0.10.3.tgz", @@ -13037,12 +9794,6 @@ "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", "integrity": "sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM=" }, - "uuid": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.2.1.tgz", - "integrity": "sha512-jZnMwlb9Iku/O3smGWvZhauCf6cvvpKi4BKRiliS3cxnI+Gz9j5MEpTz2UFuXiKPJocb7gnsLHwiS05ige5BEA==", - "dev": true - }, "uws": { "version": "9.14.0", "resolved": "https://registry.npmjs.org/uws/-/uws-9.14.0.tgz", @@ -13055,20 +9806,9 @@ "integrity": "sha512-gz/uknWtNfZTj1BLUzYHDxOoiQ7A4wZ6xPuuE6RpxswR4cNyT4I5kN9jmU0AQr7IBEap9vfYChI2TpssTN6Itg==" }, "vary": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", - "integrity": "sha1-IpnwLG3tMNSllhsLn3RSShj2NPw=" - }, - "verror": { - "version": "1.10.0", - "resolved": "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz", - "integrity": "sha1-OhBcoXBTr1XW4nDB+CiGguGNpAA=", - "dev": true, - "requires": { - "assert-plus": "1.0.0", - "core-util-is": "1.0.2", - "extsprintf": "1.3.0" - } + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", + "integrity": "sha1-IpnwLG3tMNSllhsLn3RSShj2NPw=" }, "vm-browserify": { "version": "0.0.4", @@ -13113,139 +9853,6 @@ "vow": "0.4.17" } }, - "wdio-dot-reporter": { - "version": "0.0.9", - "resolved": "https://registry.npmjs.org/wdio-dot-reporter/-/wdio-dot-reporter-0.0.9.tgz", - "integrity": "sha1-kpsq2v1J1rBTT9oGjocxm0fjj+U=", - "dev": true - }, - "wdio-sync": { - "version": "0.6.14", - "resolved": "https://registry.npmjs.org/wdio-sync/-/wdio-sync-0.6.14.tgz", - "integrity": "sha1-odzVkHuh0EFUquYXbGItkQw8qbM=", - "dev": true, - "requires": { - "babel-runtime": "6.23.0", - "fibers": "1.0.15", - "object.assign": "4.1.0" - }, - "dependencies": { - "babel-runtime": { - "version": "6.23.0", - "resolved": "https://registry.npmjs.org/babel-runtime/-/babel-runtime-6.23.0.tgz", - "integrity": "sha1-CpSJ8UTecO+zzkMArM2zKeL8VDs=", - "dev": true, - "requires": { - "core-js": "2.5.4", - "regenerator-runtime": "0.10.5" - } - }, - "regenerator-runtime": { - "version": "0.10.5", - "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.10.5.tgz", - "integrity": "sha1-M2w+/BIgrc7dosn6tntaeVWjNlg=", - "dev": true - } - } - }, - "webdriverio": { - "version": "4.12.0", - "resolved": "https://registry.npmjs.org/webdriverio/-/webdriverio-4.12.0.tgz", - "integrity": "sha1-40De8nIYPIFopN0LOCMi+de+4Q0=", - "dev": true, - "requires": { - "archiver": "2.1.1", - "babel-runtime": "6.26.0", - "css-parse": "2.0.0", - "css-value": "0.0.1", - "deepmerge": "2.0.1", - "ejs": "2.5.7", - "gaze": "1.1.2", - "glob": "github:lucetius/node-glob#51c7ca6e69bfbd17db5f1ea710e3f2a7a457d9ce", - "inquirer": "3.3.0", - "json-stringify-safe": "5.0.1", - "mkdirp": "0.5.1", - "npm-install-package": "2.1.0", - "optimist": "0.6.1", - "q": "1.5.1", - "request": "2.83.0", - "rgb2hex": "0.1.0", - "safe-buffer": "5.1.1", - "supports-color": "5.0.1", - "url": "0.11.0", - "wdio-dot-reporter": "0.0.9", - "wgxpath": "1.0.0" - }, - "dependencies": { - "has-flag": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-2.0.0.tgz", - "integrity": "sha1-6CB68cx7MNRGzHC3NLXovhj4jVE=", - "dev": true - }, - "punycode": { - "version": "1.3.2", - "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", - "integrity": "sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0=", - "dev": true - }, - "q": { - "version": "1.5.1", - "resolved": "https://registry.npmjs.org/q/-/q-1.5.1.tgz", - "integrity": "sha1-fjL3W0E4EpHQRhHxvxQQmsAGUdc=", - "dev": true - }, - "request": { - "version": "2.83.0", - "resolved": "https://registry.npmjs.org/request/-/request-2.83.0.tgz", - "integrity": "sha512-lR3gD69osqm6EYLk9wB/G1W/laGWjzH90t1vEa2xuxHD5KUrSzp9pUSfTm+YC5Nxt2T8nMPEvKlhbQayU7bgFw==", - "dev": true, - "requires": { - "aws-sign2": "0.7.0", - "aws4": "1.6.0", - "caseless": "0.12.0", - "combined-stream": "1.0.6", - "extend": "3.0.1", - "forever-agent": "0.6.1", - "form-data": "2.3.2", - "har-validator": "5.0.3", - "hawk": "6.0.2", - "http-signature": "1.2.0", - "is-typedarray": "1.0.0", - "isstream": "0.1.2", - "json-stringify-safe": "5.0.1", - "mime-types": "2.1.18", - "oauth-sign": "0.8.2", - "performance-now": "2.1.0", - "qs": "6.5.1", - "safe-buffer": "5.1.1", - "stringstream": "0.0.5", - "tough-cookie": "2.3.4", - "tunnel-agent": "0.6.0", - "uuid": "3.2.1" - } - }, - "supports-color": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.0.1.tgz", - "integrity": "sha512-7FQGOlSQ+AQxBNXJpVDj8efTA/FtyB5wcNE1omXXJ0cq6jm1jjDwuROlYDbnzHqdNPqliWFhcioCWSyav+xBnA==", - "dev": true, - "requires": { - "has-flag": "2.0.0" - } - }, - "url": { - "version": "0.11.0", - "resolved": "https://registry.npmjs.org/url/-/url-0.11.0.tgz", - "integrity": "sha1-ODjpfPxgUh63PFJajlW/3Z4uKPE=", - "dev": true, - "requires": { - "punycode": "1.3.2", - "querystring": "0.2.0" - } - } - } - }, "webgme": { "version": "2.29.0", "resolved": "https://registry.npmjs.org/webgme/-/webgme-2.29.0.tgz", @@ -13996,7 +10603,7 @@ "webgme-autoviz": { "version": "2.2.2", "resolved": "https://registry.npmjs.org/webgme-autoviz/-/webgme-autoviz-2.2.2.tgz", - "integrity": "sha1-dr7GVNekoITovBNvui2zMrLaqX8=" + "integrity": "sha512-QR1vNPj117zXZhFp67j8mteNNVjTLaoVQ5a7sTm2YfAqiUDZafPtvy/O4OabrgvCMZZrmkVz8TlwV1coGW0v+w==" }, "webgme-breadcrumbheader": { "version": "2.1.1", @@ -16105,7 +12712,7 @@ } }, "webgme-easydag": { - "version": "git://github.com/dfst/webgme-easydag.git#cb461f2687c8a2aa00adc827ea3688b7f0e24ada" + "version": "github:dfst/webgme-easydag#fd56486ae9b1df3c0749141c8dabdb6a90818fed" }, "webgme-engine": { "version": "2.19.8", @@ -16999,28 +13606,6 @@ } } }, - "websocket-driver": { - "version": "0.7.0", - "resolved": "https://registry.npmjs.org/websocket-driver/-/websocket-driver-0.7.0.tgz", - "integrity": "sha1-DK+dLXVdk67gSdS90NP+LMoqJOs=", - "dev": true, - "requires": { - "http-parser-js": "0.4.11", - "websocket-extensions": "0.1.3" - } - }, - "websocket-extensions": { - "version": "0.1.3", - "resolved": "https://registry.npmjs.org/websocket-extensions/-/websocket-extensions-0.1.3.tgz", - "integrity": "sha512-nqHUnMXmBzT0w570r2JpJxfiSD1IzoI+HGVdd3aZ0yNi3ngvQ4jv1dtHt5VGxfI2yj5yqImPhOK4vmIh2xMbGg==", - "dev": true - }, - "wgxpath": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/wgxpath/-/wgxpath-1.0.0.tgz", - "integrity": "sha1-7vikudVYzEla06mit1FZfs2a9pA=", - "dev": true - }, "which": { "version": "1.2.14", "resolved": "https://registry.npmjs.org/which/-/which-1.2.14.tgz", @@ -17105,12 +13690,6 @@ } } }, - "wordwrap": { - "version": "0.0.3", - "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-0.0.3.tgz", - "integrity": "sha1-o9XabNXAvAAI03I0u68b7WMFkQc=", - "dev": true - }, "wrap-ansi": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-2.1.0.tgz", @@ -17168,269 +13747,6 @@ "resolved": "https://registry.npmjs.org/xmlhttprequest-ssl/-/xmlhttprequest-ssl-1.5.5.tgz", "integrity": "sha1-wodrBhaKrcQOV9l+gRkayPQ5iz4=" }, - "xolvio-ddp": { - "version": "0.12.3", - "resolved": "https://registry.npmjs.org/xolvio-ddp/-/xolvio-ddp-0.12.3.tgz", - "integrity": "sha1-NqarlhKyQLWg0cCoNJCK8XwLjwI=", - "dev": true, - "requires": { - "bluebird": "2.11.0", - "ddp-ejson": "0.8.1-3", - "ddp-underscore-patched": "0.8.1-2", - "faye-websocket": "0.9.4", - "request": "2.53.0" - }, - "dependencies": { - "asn1": { - "version": "0.1.11", - "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.1.11.tgz", - "integrity": "sha1-VZvhg3bQik7E2+gId9J4GGObLfc=", - "dev": true - }, - "assert-plus": { - "version": "0.1.5", - "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-0.1.5.tgz", - "integrity": "sha1-7nQAlBMALYTOxyGcasgRgS5yMWA=", - "dev": true - }, - "aws-sign2": { - "version": "0.5.0", - "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.5.0.tgz", - "integrity": "sha1-xXED96F/wDfwLXwuZLYC6iI/fWM=", - "dev": true - }, - "bl": { - "version": "0.9.5", - "resolved": "https://registry.npmjs.org/bl/-/bl-0.9.5.tgz", - "integrity": "sha1-wGt5evCF6gC8Unr8jvzxHeIjIFQ=", - "dev": true, - "requires": { - "readable-stream": "1.0.34" - } - }, - "bluebird": { - "version": "2.11.0", - "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-2.11.0.tgz", - "integrity": "sha1-U0uQM8AiyVecVro7Plpcqvu2UOE=", - "dev": true - }, - "boom": { - "version": "2.10.1", - "resolved": "https://registry.npmjs.org/boom/-/boom-2.10.1.tgz", - "integrity": "sha1-OciRjO/1eZ+D+UkqhI9iWt0Mdm8=", - "dev": true, - "requires": { - "hoek": "2.16.3" - } - }, - "caseless": { - "version": "0.9.0", - "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.9.0.tgz", - "integrity": "sha1-t7Zc5r8UE4hlOc/VM/CzDv+pz4g=", - "dev": true - }, - "combined-stream": { - "version": "0.0.7", - "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-0.0.7.tgz", - "integrity": "sha1-ATfmV7qlp1QcV6w3rF/AfXO03B8=", - "dev": true, - "requires": { - "delayed-stream": "0.0.5" - } - }, - "cryptiles": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/cryptiles/-/cryptiles-2.0.5.tgz", - "integrity": "sha1-O9/s3GCBR8HGcgL6KR59ylnqo7g=", - "dev": true, - "requires": { - "boom": "2.10.1" - } - }, - "delayed-stream": { - "version": "0.0.5", - "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-0.0.5.tgz", - "integrity": "sha1-1LH0OpPoKW3+AmlPRoC8N6MTxz8=", - "dev": true - }, - "forever-agent": { - "version": "0.5.2", - "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.5.2.tgz", - "integrity": "sha1-bQ4JxJIflKJ/Y9O0nF/v8epMUTA=", - "dev": true - }, - "form-data": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-0.2.0.tgz", - "integrity": "sha1-Jvi8JtpkQOKZy9z7aQNcT3em5GY=", - "dev": true, - "requires": { - "async": "0.9.2", - "combined-stream": "0.0.7", - "mime-types": "2.0.14" - } - }, - "hawk": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/hawk/-/hawk-2.3.1.tgz", - "integrity": "sha1-HnMc45RH+h0PbXB/e87r7A/R7B8=", - "dev": true, - "requires": { - "boom": "2.10.1", - "cryptiles": "2.0.5", - "hoek": "2.16.3", - "sntp": "1.0.9" - } - }, - "hoek": { - "version": "2.16.3", - "resolved": "https://registry.npmjs.org/hoek/-/hoek-2.16.3.tgz", - "integrity": "sha1-ILt0A9POo5jpHcRxCo/xuCdKJe0=", - "dev": true - }, - "http-signature": { - "version": "0.10.1", - "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-0.10.1.tgz", - "integrity": "sha1-T72sEyVZqoMjEh5UB3nAoBKyfmY=", - "dev": true, - "requires": { - "asn1": "0.1.11", - "assert-plus": "0.1.5", - "ctype": "0.5.3" - } - }, - "isarray": { - "version": "0.0.1", - "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", - "integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8=", - "dev": true - }, - "mime-db": { - "version": "1.12.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.12.0.tgz", - "integrity": "sha1-PQxjGA9FjrENMlqqN9fFiuMS6dc=", - "dev": true - }, - "mime-types": { - "version": "2.0.14", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.0.14.tgz", - "integrity": "sha1-MQ4VnbI+B3+Lsit0jav6SVcUCqY=", - "dev": true, - "requires": { - "mime-db": "1.12.0" - } - }, - "node-uuid": { - "version": "1.4.8", - "resolved": "https://registry.npmjs.org/node-uuid/-/node-uuid-1.4.8.tgz", - "integrity": "sha1-sEDrCSOWivq/jTL7HxfxFn/auQc=", - "dev": true - }, - "oauth-sign": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.6.0.tgz", - "integrity": "sha1-fb6uRPbKRU4fFoRR1jB0ZzWBPOM=", - "dev": true - }, - "qs": { - "version": "2.3.3", - "resolved": "https://registry.npmjs.org/qs/-/qs-2.3.3.tgz", - "integrity": "sha1-6eha2+ddoLvkyOBHaghikPhjtAQ=", - "dev": true - }, - "readable-stream": { - "version": "1.0.34", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.0.34.tgz", - "integrity": "sha1-Elgg40vIQtLyqq+v5MKRbuMsFXw=", - "dev": true, - "requires": { - "core-util-is": "1.0.2", - "inherits": "2.0.3", - "isarray": "0.0.1", - "string_decoder": "0.10.31" - } - }, - "request": { - "version": "2.53.0", - "resolved": "https://registry.npmjs.org/request/-/request-2.53.0.tgz", - "integrity": "sha1-GAo66St7Y5gC5PlUXdj83rcddgw=", - "dev": true, - "requires": { - "aws-sign2": "0.5.0", - "bl": "0.9.5", - "caseless": "0.9.0", - "combined-stream": "0.0.7", - "forever-agent": "0.5.2", - "form-data": "0.2.0", - "hawk": "2.3.1", - "http-signature": "0.10.1", - "isstream": "0.1.2", - "json-stringify-safe": "5.0.1", - "mime-types": "2.0.14", - "node-uuid": "1.4.8", - "oauth-sign": "0.6.0", - "qs": "2.3.3", - "stringstream": "0.0.5", - "tough-cookie": "2.3.4", - "tunnel-agent": "0.4.3" - } - }, - "sntp": { - "version": "1.0.9", - "resolved": "https://registry.npmjs.org/sntp/-/sntp-1.0.9.tgz", - "integrity": "sha1-ZUEYTMkK7qbG57NeJlkIJEPGYZg=", - "dev": true, - "requires": { - "hoek": "2.16.3" - } - }, - "string_decoder": { - "version": "0.10.31", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", - "integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=", - "dev": true - }, - "tunnel-agent": { - "version": "0.4.3", - "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.4.3.tgz", - "integrity": "sha1-Y3PbdpCf5XDgjXNYM2Xtgop07us=", - "dev": true - } - } - }, - "xolvio-fiber-utils": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/xolvio-fiber-utils/-/xolvio-fiber-utils-2.0.3.tgz", - "integrity": "sha1-vsjXDHQGGjFjFbun0w0lyz6C3FA=", - "dev": true, - "requires": { - "fibers": "1.0.15", - "underscore": "1.8.3" - } - }, - "xolvio-jasmine-expect": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/xolvio-jasmine-expect/-/xolvio-jasmine-expect-1.1.0.tgz", - "integrity": "sha1-vCud1ghCMR8EV59agtzqaisxnH0=", - "dev": true, - "requires": { - "jasmine-core": "2.99.1" - } - }, - "xolvio-sync-webdriverio": { - "version": "9.0.1", - "resolved": "https://registry.npmjs.org/xolvio-sync-webdriverio/-/xolvio-sync-webdriverio-9.0.1.tgz", - "integrity": "sha1-WRri2MiqynQiZJWfzM+QtPndUWA=", - "dev": true, - "requires": { - "fibers": "1.0.15", - "meteor-promise": "0.8.6", - "underscore": "1.8.3", - "wdio-sync": "0.6.14", - "webdriverio": "4.12.0", - "xolvio-fiber-utils": "2.0.3" - } - }, "xtend": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.1.tgz", @@ -17461,15 +13777,6 @@ "y18n": "3.2.1" } }, - "yauzl": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-2.4.1.tgz", - "integrity": "sha1-lSj0QtqxsihOWLQ3m7GU4i4MQAU=", - "dev": true, - "requires": { - "fd-slicer": "1.0.1" - } - }, "yeast": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/yeast/-/yeast-0.1.2.tgz", diff --git a/package.json b/package.json index d8badc0a6..d3a4e073b 100644 --- a/package.json +++ b/package.json @@ -29,6 +29,7 @@ "lodash.difference": "^4.1.2", "lodash.merge": "^4.5.1", "lodash.template": "^4.4.0", + "minimatch": "^3.0.4", "mongodb": "^2.2.10", "npm": "^4.0.5", "q": "1.4.1", @@ -45,7 +46,6 @@ "webgme-simple-nodes": "^2.1.3" }, "devDependencies": { - "chimp": "^0.50.2", "nodemon": "^1.9.2", "chai": "^3.0.0", "jszip": "^2.5.0", diff --git a/src/common/ExecutionEnv.js b/src/common/ExecutionEnv.js index e1e3da6a8..e4f006f8f 100644 --- a/src/common/ExecutionEnv.js +++ b/src/common/ExecutionEnv.js @@ -39,6 +39,7 @@ define([ }; ExecutionEnv.getWorkers = function() { + return Q(1); // TODO return this.get(WORKER_ENDPOINT) .then(workerDict => values(workerDict)); }; diff --git a/src/common/execution/backends/Local.js b/src/common/execution/backends/Local.js index 825d7a939..701c7a94d 100644 --- a/src/common/execution/backends/Local.js +++ b/src/common/execution/backends/Local.js @@ -2,6 +2,8 @@ define([ './BaseExecutor', 'blob/BlobClient', 'child_process', + 'minimatch', + 'module', 'rimraf', 'fs', 'os', @@ -10,6 +12,8 @@ define([ BaseExecutor, BlobClient, childProcess, + minimatch, + module, rimraf, fs, os, @@ -20,6 +24,7 @@ define([ const spawn = childProcess.spawn; const {promisify} = require.nodeRequire('util'); const mkdir = promisify(fs.mkdir); + const readdir = promisify(fs.readdir); const rm_rf = promisify(rimraf); const writeFile = promisify(fs.writeFile); const readFile = promisify(fs.readFile); @@ -29,6 +34,11 @@ define([ ensureHasUnzip(); const UNZIP_EXE = '/usr/bin/unzip'; // FIXME: more platform support const UNZIP_ARGS = ['-o']; // FIXME: more platform support + const PROJECT_ROOT = path.join(path.dirname(module.uri), '..', '..', '..', '..'); + const NODE_MODULES = path.join(PROJECT_ROOT, 'node_modules'); // TODO + console.log('module', module.uri); + console.log('NODE_MODULES', NODE_MODULES); + const symlink = promisify(fs.symlink); const LocalExecutor = function(logger, gmeConfig) { BaseExecutor.apply(this, arguments); @@ -78,13 +88,13 @@ define([ // Spin up a subprocess const config = JSON.parse(await readFile(tmpdir.replace(path.sep, '/') + '/executor_config.json', 'utf8')); - console.log('config:', config); const env = {cwd: tmpdir}; execJob = spawn(config.cmd, config.args, env); execJob.stdout.on('data', data => this.emit('data', hash, data)); // TODO: should this be stdout? execJob.stderr.on('data', data => this.emit('data', hash, data)); - execJob.on('close', code => { + execJob.on('close', async code => { + console.log('code', code); const jobInfo = { resultHashes: [], // TODO: upload data and add result hashes status: 'SUCCESS' @@ -98,22 +108,37 @@ define([ } this.emit('end', hash, jobInfo); }); + }; - // upload the resultArtifacts - // TODO + LocalExecutor.prototype._getAllFiles = async function(workdir) { + const dirs = (await readdir(workdir)) + .filter(n => !n.includes('node_modules')); + const files = []; + + // Read each directory + while (dirs.length) { + const isDirectory = (await fs.stat(dirs[0])).isDirectory(); + if (isDirectory) { + dirs.push.apply(dirs, await readdir(workdir)); + } else { + files.push(dirs.shift()); + } + } - //return result; + return files; }; LocalExecutor.prototype._uploadResults = async function(workdir, config) { // Get all the matching result artifacts - // TODO + const allFiles = await this._getAllFiles(workdir); + const // Upload all the artifacts // TODO // Return the hashes // TODO + return {}; }; LocalExecutor.prototype.prepareWorkspace = async function(hash, dirname) { @@ -125,6 +150,9 @@ define([ this.logger.info(`unzipping ${zipPath} in ${dirname}`); await unzip(zipPath, dirname); + + // Set up a symbolic link to the node_modules + await symlink(NODE_MODULES, path.join(dirname, 'node_modules')); }; async function unzip(filename, dirname) { diff --git a/src/plugins/ExecuteJob/ExecuteJob.js b/src/plugins/ExecuteJob/ExecuteJob.js index 224ec34c7..5f5c910bd 100644 --- a/src/plugins/ExecuteJob/ExecuteJob.js +++ b/src/plugins/ExecuteJob/ExecuteJob.js @@ -706,6 +706,7 @@ define([ this.setAttribute(job, 'execFiles', info.resultHashes[name + '-all-files']); return this.blobClient.getArtifact(info.resultHashes.stdout) .then(artifact => { + // FIXME: Improve the error handling here var stdoutHash = artifact.descriptor.content[STDOUT_FILE].content; return this.blobClient.getObjectAsString(stdoutHash); }) @@ -762,6 +763,7 @@ define([ // TODO: this.setAttribute(job, 'execFiles', info.resultHashes[name + '-all-files']); const artifact = await this.blobClient.getArtifact(info.resultHashes.stdout) + // FIXME: Improve the error handling here const stdoutHash = artifact.descriptor.content[STDOUT_FILE].content; const stdout = await this.blobClient.getObjectAsString(stdoutHash); // Parse the remaining code @@ -818,6 +820,7 @@ define([ // Create new metadata for each artifacts.forEach((artifact, i) => { var name = outputs[i][0], + // FIXME: Uniform the error handling here outputData = artifact.descriptor.content[`outputs/${name}`], hash = outputData && outputData.content, dataType = resultTypes[name]; @@ -844,6 +847,7 @@ define([ const hash = result.resultHashes['result-types']; return this.blobClient.getArtifact(hash) .then(data => { + // FIXME: Improve the error handling here const contents = data.descriptor.content; const contentHash = contents['result-types.json'].content; return this.blobClient.getObjectAsJSON(contentHash); From 55dfec21064ea61d98af0eb3c31f7f04e981df98 Mon Sep 17 00:00:00 2001 From: Brian Broll Date: Sat, 9 Mar 2019 11:35:06 -0800 Subject: [PATCH 05/45] WIP fixed file collection --- src/common/execution/backends/Local.js | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/common/execution/backends/Local.js b/src/common/execution/backends/Local.js index 701c7a94d..1e10bf97f 100644 --- a/src/common/execution/backends/Local.js +++ b/src/common/execution/backends/Local.js @@ -25,6 +25,7 @@ define([ const {promisify} = require.nodeRequire('util'); const mkdir = promisify(fs.mkdir); const readdir = promisify(fs.readdir); + const statFile = promisify(fs.stat); const rm_rf = promisify(rimraf); const writeFile = promisify(fs.writeFile); const readFile = promisify(fs.readFile); @@ -112,27 +113,32 @@ define([ LocalExecutor.prototype._getAllFiles = async function(workdir) { const dirs = (await readdir(workdir)) - .filter(n => !n.includes('node_modules')); + .filter(n => !n.includes('node_modules')) + .map(name => path.join(workdir, name)); const files = []; // Read each directory while (dirs.length) { - const isDirectory = (await fs.stat(dirs[0])).isDirectory(); + const abspath = dirs.shift(); + const isDirectory = (await statFile(abspath)).isDirectory(); if (isDirectory) { - dirs.push.apply(dirs, await readdir(workdir)); + const childpaths = (await readdir(abspath)) + .map(name => path.join(abspath, name)); + dirs.push.apply(dirs, childpaths); } else { - files.push(dirs.shift()); + files.push(abspath); } } - return files; + return files.map(file => path.relative(workdir, file)); // TODO: get the relative paths }; LocalExecutor.prototype._uploadResults = async function(workdir, config) { // Get all the matching result artifacts const allFiles = await this._getAllFiles(workdir); - const + console.log(allFiles) + // TODO: Get the relative pt // Upload all the artifacts // TODO From ff3294fc67cffb666d05268cade79e1cfb82446c Mon Sep 17 00:00:00 2001 From: Brian Broll Date: Sun, 17 Mar 2019 08:57:02 -0700 Subject: [PATCH 06/45] WIP #1186 updated local executor to upload results --- src/common/execution/backends/Local.js | 196 ++++++++++++++++++++++--- 1 file changed, 174 insertions(+), 22 deletions(-) diff --git a/src/common/execution/backends/Local.js b/src/common/execution/backends/Local.js index 1e10bf97f..a0a01ffbd 100644 --- a/src/common/execution/backends/Local.js +++ b/src/common/execution/backends/Local.js @@ -25,6 +25,7 @@ define([ const {promisify} = require.nodeRequire('util'); const mkdir = promisify(fs.mkdir); const readdir = promisify(fs.readdir); + const appendFile = promisify(fs.appendFile); const statFile = promisify(fs.stat); const rm_rf = promisify(rimraf); const writeFile = promisify(fs.writeFile); @@ -65,7 +66,6 @@ define([ // TODO: Add cancel support! TODO LocalExecutor.prototype.createJob = async function(hash) { - // TODO: Set up a directory to work in... // Create tmp directory const tmpdir = path.join(os.tmpdir(), `deepforge-local-exec-${hash}`); try { @@ -92,25 +92,26 @@ define([ const env = {cwd: tmpdir}; execJob = spawn(config.cmd, config.args, env); - execJob.stdout.on('data', data => this.emit('data', hash, data)); // TODO: should this be stdout? - execJob.stderr.on('data', data => this.emit('data', hash, data)); + execJob.stdout.on('data', data => this.onConsoleOutput(tmpdir, hash, data)); + execJob.stderr.on('data', data => this.onConsoleOutput(tmpdir, hash, data)); + execJob.on('close', async code => { - console.log('code', code); const jobInfo = { resultHashes: [], // TODO: upload data and add result hashes - status: 'SUCCESS' + status: code !== 0 ? 'FAILED_TO_EXECUTE' : 'SUCCESS' } - if (code === 0) { // Success - // TODO: upload data and record hashes.. - jobInfo.resultHashes = await this._uploadResults(tmpdir, config); - } else { - jobInfo.status = 'FAILED_TO_EXECUTE'; - } + await this._uploadResults(jobInfo, tmpdir, config); this.emit('end', hash, jobInfo); }); }; + LocalExecutor.prototype.onConsoleOutput = async function(workdir, hash, data) { + const filename = path.join(workdir, 'job_stdout.txt'); + appendFile(filename, data); + this.emit('data', hash, data); + }; + LocalExecutor.prototype._getAllFiles = async function(workdir) { const dirs = (await readdir(workdir)) .filter(n => !n.includes('node_modules')) @@ -130,23 +131,174 @@ define([ } } - return files.map(file => path.relative(workdir, file)); // TODO: get the relative paths + return files; }; - LocalExecutor.prototype._uploadResults = async function(workdir, config) { - // Get all the matching result artifacts - const allFiles = await this._getAllFiles(workdir); - console.log(allFiles) + LocalExecutor.prototype._uploadResults = async function(jobInfo, directory, executorConfig) { + var self = this, + i, + jointArtifact = self.blobClient.createArtifact('jobInfo_resultSuperSetHash'), + resultsArtifacts = [], + afterWalk, + archiveFile, + afterAllFilesArchived, + addObjectHashesAndSaveArtifact; + + jobInfo.resultHashes = {}; + + for (i = 0; i < executorConfig.resultArtifacts.length; i += 1) { + resultsArtifacts.push( + { + name: executorConfig.resultArtifacts[i].name, + artifact: self.blobClient.createArtifact(executorConfig.resultArtifacts[i].name), + patterns: executorConfig.resultArtifacts[i].resultPatterns instanceof Array ? + executorConfig.resultArtifacts[i].resultPatterns : [], + files: {} + } + ); + } + + afterWalk = function (filesToArchive) { + if (filesToArchive.length === 0) { + self.logger.info(jobInfo.hash + ' There were no files to archive..'); + } + + return Promise.all(filesToArchive.map(f => archiveFile(f.filename, f.filePath))); + }; + + archiveFile = promisify(function (filename, filePath, callback) { + var archiveData = function (err, data) { + jointArtifact.addFileAsSoftLink(filename, data, function (err, hash) { + var j; + if (err) { + self.logger.error(jobInfo.hash + ' Failed to archive as "' + filename + '" from "' + + filePath + '", err: ' + err); + self.logger.error(err); + callback(new Error('FAILED_TO_ARCHIVE_FILE')); + } else { + // Add the file-hash to the results artifacts containing the filename. + for (j = 0; j < resultsArtifacts.length; j += 1) { + if (resultsArtifacts[j].files[filename] === true) { + resultsArtifacts[j].files[filename] = hash; + } + } + callback(null); + } + }); + }; + + if (typeof File === 'undefined') { // nodejs doesn't have File + fs.readFile(filePath, function (err, data) { + if (err) { + self.logger.error(jobInfo.hash + ' Failed to archive as "' + filename + '" from "' + filePath + + '", err: ' + err); + return callback(new Error('FAILED_TO_ARCHIVE_FILE')); + } + archiveData(null, data); + }); + } else { + archiveData(null, new File(filePath, filename)); + } + }); + + afterAllFilesArchived = async function () { + let resultHash = null; + try { + resultHash = await jointArtifact.save(); + } catch (err) { + this.logger.warn(`Failed to save joint artifact: ${err} (${jobInfo.hash})`); + throw new Error('FAILED_TO_SAVE_JOINT_ARTIFACT'); + } + + try { + await rm_rf(directory); + } catch (err) { + self.logger.error('Could not delete executor-temp file, err: ' + err); + } + jobInfo.resultSuperSetHash = resultHash; + return Promise.all(resultsArtifacts.map(r => addObjectHashesAndSaveArtifact(r))) + }; + + addObjectHashesAndSaveArtifact = promisify(function (resultArtifact, callback) { + resultArtifact.artifact.addMetadataHashes(resultArtifact.files, function (err/*, hashes*/) { + if (err) { + self.logger.error(jobInfo.hash + ' ' + err); + return callback('FAILED_TO_ADD_OBJECT_HASHES'); + } + resultArtifact.artifact.save(function (err, resultHash) { + if (err) { + self.logger.error(jobInfo.hash + ' ' + err); + return callback('FAILED_TO_SAVE_ARTIFACT'); + } + jobInfo.resultHashes[resultArtifact.name] = resultHash; + callback(null); + }); + }); + }); - // TODO: Get the relative pt - // Upload all the artifacts - // TODO + const allFiles = await this._getAllFiles(directory); - // Return the hashes - // TODO - return {}; + console.log('final', allFiles); + const filesToArchive = []; + let archive, + filename, + matched; + + for (let i = 0; i < allFiles.length; i += 1) { + filename = path.relative(directory, allFiles[i]).replace(/\\/g, '/'); + archive = false; + for (let a = 0; a < resultsArtifacts.length; a += 1) { + if (resultsArtifacts[a].patterns.length === 0) { + resultsArtifacts[a].files[filename] = true; + archive = true; + } else { + for (let j = 0; j < resultsArtifacts[a].patterns.length; j += 1) { + matched = minimatch(filename, resultsArtifacts[a].patterns[j]); + console.log(filename, 'matches',resultsArtifacts[a].patterns[j], '?', matched); + if (matched) { + resultsArtifacts[a].files[filename] = true; + archive = true; + break; + } + } + } + } + if (archive) { + filesToArchive.push({filename: filename, filePath: allFiles[i]}); + } + } + + try { + await afterWalk(filesToArchive); + return await afterAllFilesArchived(); + } catch (err) { + jobInfo.status = err.message; + } }; + //LocalExecutor.prototype._uploadResults = async function(workdir, config) { + //// Get all the matching result artifacts + //const allFiles = await this._getAllFiles(workdir); + //console.log(allFiles) + + //// Upload all the artifacts + //const artifacts = config.resultArtifacts + //.map(info => { + //return { + //name: info.name, + //artifact: self.blobClient.createArtifact(info.name), + //patterns: info.resultPatterns instanceof Array ? + //info.resultPatterns : [], + //files: {} + //} + //}) + //// TODO + + //// Return the hashes + //// TODO + //return {}; + //}; + LocalExecutor.prototype.prepareWorkspace = async function(hash, dirname) { this.logger.info(`about to fetch job data`); const content = new Buffer(new Uint8Array(await this.blobClient.getObject(hash))); // TODO: Handle errors... From 8811089e73ab573c3b33b6e43b0256b722654b37 Mon Sep 17 00:00:00 2001 From: Brian Broll Date: Sun, 17 Mar 2019 08:58:40 -0700 Subject: [PATCH 07/45] WIP #1186 removed stderr (debug) output --- src/common/execution/backends/Local.js | 1 - 1 file changed, 1 deletion(-) diff --git a/src/common/execution/backends/Local.js b/src/common/execution/backends/Local.js index a0a01ffbd..aadc4e0d9 100644 --- a/src/common/execution/backends/Local.js +++ b/src/common/execution/backends/Local.js @@ -93,7 +93,6 @@ define([ const env = {cwd: tmpdir}; execJob = spawn(config.cmd, config.args, env); execJob.stdout.on('data', data => this.onConsoleOutput(tmpdir, hash, data)); - execJob.stderr.on('data', data => this.onConsoleOutput(tmpdir, hash, data)); execJob.on('close', async code => { const jobInfo = { From f749d46490c58e56ec76591ed2a638dfc1d441b5 Mon Sep 17 00:00:00 2001 From: Brian Broll Date: Thu, 21 Mar 2019 17:23:31 -0700 Subject: [PATCH 08/45] WIP Added notes for next steps and stdout logging --- src/common/execution/backends/Local.js | 17 +++++++++++++---- src/plugins/ExecuteJob/ExecuteJob.js | 7 +++++-- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/src/common/execution/backends/Local.js b/src/common/execution/backends/Local.js index aadc4e0d9..dd83622b8 100644 --- a/src/common/execution/backends/Local.js +++ b/src/common/execution/backends/Local.js @@ -55,9 +55,9 @@ define([ LocalExecutor.prototype = Object.create(BaseExecutor.prototype); - //LocalExecutor.prototype.cancelJob = function(job) { - //return this.executor.cancelJob(job.hash, job.secret); - //}; + LocalExecutor.prototype.cancelJob = function(job) { + return this.executor.cancelJob(job.hash, job.secret); + }; //LocalExecutor.prototype.getInfo = function(job) { //return this.executor.getInfo(job.hash); @@ -65,6 +65,11 @@ define([ // TODO: Add cancel support! TODO LocalExecutor.prototype.createJob = async function(hash) { + // TODO: Queue the job if a job is already running... + return this._createJob(hash); + }; + + LocalExecutor.prototype._createJob = async function(hash) { // Create tmp directory const tmpdir = path.join(os.tmpdir(), `deepforge-local-exec-${hash}`); @@ -91,23 +96,27 @@ define([ const config = JSON.parse(await readFile(tmpdir.replace(path.sep, '/') + '/executor_config.json', 'utf8')); const env = {cwd: tmpdir}; + this.logger.info(`Running ${config.cmd} ${config.args.join(' ')}`); execJob = spawn(config.cmd, config.args, env); execJob.stdout.on('data', data => this.onConsoleOutput(tmpdir, hash, data)); execJob.on('close', async code => { const jobInfo = { - resultHashes: [], // TODO: upload data and add result hashes + resultHashes: [], status: code !== 0 ? 'FAILED_TO_EXECUTE' : 'SUCCESS' } await this._uploadResults(jobInfo, tmpdir, config); this.emit('end', hash, jobInfo); }); + + return {}; }; LocalExecutor.prototype.onConsoleOutput = async function(workdir, hash, data) { const filename = path.join(workdir, 'job_stdout.txt'); appendFile(filename, data); + this.logger.info('stdout:', data); this.emit('data', hash, data); }; diff --git a/src/plugins/ExecuteJob/ExecuteJob.js b/src/plugins/ExecuteJob/ExecuteJob.js index 4bd23df8e..4e48a9136 100644 --- a/src/plugins/ExecuteJob/ExecuteJob.js +++ b/src/plugins/ExecuteJob/ExecuteJob.js @@ -759,10 +759,13 @@ define([ if (info.status === 'SUCCESS' || info.status === 'FAILED_TO_EXECUTE') { // TODO: this.setAttribute(job, 'execFiles', info.resultHashes[name + '-all-files']); + // TODO: Add getStdout to executor? const artifact = await this.blobClient.getArtifact(info.resultHashes.stdout) // FIXME: Improve the error handling here + console.log(artifact.descriptor); const stdoutHash = artifact.descriptor.content[STDOUT_FILE].content; const stdout = await this.blobClient.getObjectAsString(stdoutHash); + // Parse the remaining code const result = this.processStdout(job, stdout); this.setAttribute(job, 'stdout', result.stdout); @@ -771,12 +774,12 @@ define([ // Download all files this.result.addArtifact(info.resultHashes[name + '-all-files']); // Set the job to failed! Store the error - this.onOperationFail(op, `Operation "${opId}" failed! ${JSON.stringify(info)}`); + this.onOperationFail(op, `Operation "${jobId}" failed! ${JSON.stringify(info)}`); } else { this.onDistOperationComplete(op, info); } } else { // something bad happened... - var err = `Failed to execute operation "${opId}": ${info.status}`, + var err = `Failed to execute operation "${jobId}": ${info.status}`, consoleErr = `Failed to execute operation: ${info.status}`; this.setAttribute(job, 'stdout', consoleErr); this.logger.error(err); From 1b656c98e77b256d5b75151fbccf6d64d7aa3d01 Mon Sep 17 00:00:00 2001 From: Brian Broll Date: Thu, 5 Sep 2019 14:51:12 -0500 Subject: [PATCH 09/45] WIP #1186 upload, job queueing for local executor --- src/common/execution/backends/BaseExecutor.js | 57 ++++++++++++++++++ src/common/execution/backends/Local.js | 58 ++++++++++++------- src/plugins/ExecuteJob/ExecuteJob.js | 56 ++++++++---------- 3 files changed, 120 insertions(+), 51 deletions(-) create mode 100644 src/common/execution/backends/BaseExecutor.js diff --git a/src/common/execution/backends/BaseExecutor.js b/src/common/execution/backends/BaseExecutor.js new file mode 100644 index 000000000..aa5ad85e6 --- /dev/null +++ b/src/common/execution/backends/BaseExecutor.js @@ -0,0 +1,57 @@ +define([ +], function( +) { + const BaseExecutor = function(logger, gmeConfig) { + const isHttps = typeof window === 'undefined' ? false : + window.location.protocol !== 'http:'; + + this.logger = logger.fork('executor'); + this._events = {}; // FIXME: there must be a better way... + }; + + BaseExecutor.prototype.cancelJob = function(job) { + const msg = `cancelJob is not implemented for current executor backend!`; + this.logger.warn(msg); + return Promise.reject(new Error(msg)) + }; + + BaseExecutor.prototype.getInfo = function(job) { + const msg = `getInfo is not implemented for current executor backend!`; + this.logger.warn(msg); + return Promise.reject(new Error(msg)) + }; + + BaseExecutor.prototype.createJob = async function(hash) { + const msg = `createJob is not implemented for current executor backend!`; + this.logger.warn(msg); + return Promise.reject(new Error(msg)) + }; + + BaseExecutor.prototype.getOutput = async function(hash) { + const msg = `getOutput is not implemented for current executor backend!`; + this.logger.warn(msg); + return Promise.reject(new Error(msg)) + }; + + // Some functions for event support + BaseExecutor.prototype.on = function(ev, cb) { + this._events[ev] = this._events[ev] || []; + this._events[ev].push(cb); + }; + + BaseExecutor.prototype.emit = function(ev) { + const args = Array.prototype.slice.call(arguments, 1); + console.log('emitting'); + args.forEach(a => { + if (a instanceof Buffer) { + console.log(a.toString()); + } else { + console.log(a); + } + }); + const handlers = this._events[ev] || []; + handlers.forEach(fn => fn.apply(this, args)); + }; + + return BaseExecutor; +}); diff --git a/src/common/execution/backends/Local.js b/src/common/execution/backends/Local.js index dd83622b8..114bcc955 100644 --- a/src/common/execution/backends/Local.js +++ b/src/common/execution/backends/Local.js @@ -31,6 +31,8 @@ define([ const writeFile = promisify(fs.writeFile); const readFile = promisify(fs.readFile); const execFile = promisify(childProcess.execFile); + const openFile = promisify(fs.open); + const closeFile = promisify(fs.close); // UNZIP must be available on the machine, first ensure that it exists... ensureHasUnzip(); @@ -38,12 +40,13 @@ define([ const UNZIP_ARGS = ['-o']; // FIXME: more platform support const PROJECT_ROOT = path.join(path.dirname(module.uri), '..', '..', '..', '..'); const NODE_MODULES = path.join(PROJECT_ROOT, 'node_modules'); // TODO - console.log('module', module.uri); - console.log('NODE_MODULES', NODE_MODULES); const symlink = promisify(fs.symlink); + const touch = async name => await closeFile(await openFile(name, 'w')); const LocalExecutor = function(logger, gmeConfig) { BaseExecutor.apply(this, arguments); + this.jobQueue = []; + this.currentJob = null; // FIXME: set this meaningfully! this.blobClient = new BlobClient({ server: '127.0.0.1', @@ -59,20 +62,40 @@ define([ return this.executor.cancelJob(job.hash, job.secret); }; - //LocalExecutor.prototype.getInfo = function(job) { - //return this.executor.getInfo(job.hash); - //}; + LocalExecutor.prototype.getOutput = async function(hash) { + const filename = path.join(this._getWorkingDir(hash), 'job_stdout.txt'); + return await readFile(filename, 'utf8'); + }; // TODO: Add cancel support! TODO LocalExecutor.prototype.createJob = async function(hash) { - // TODO: Queue the job if a job is already running... - return this._createJob(hash); + this.jobQueue.push(hash); + this._processNextJob(); + + return {hash}; }; - LocalExecutor.prototype._createJob = async function(hash) { + LocalExecutor.prototype._onJobCompleted = function() { + this.currentJob = null; + this._processNextJob(); + }; + + LocalExecutor.prototype._processNextJob = function() { + if (this.currentJob) return; + + this.currentJob = this.jobQueue.shift(); + if (this.currentJob) { + return this._createJob(this.currentJob); + } + }; + LocalExecutor.prototype._getWorkingDir = function(hash) { + return path.join(os.tmpdir(), `deepforge-local-exec-${hash}`); + }; + + LocalExecutor.prototype._createJob = async function(hash) { // Create tmp directory - const tmpdir = path.join(os.tmpdir(), `deepforge-local-exec-${hash}`); + const tmpdir = this._getWorkingDir(hash); try { await mkdir(tmpdir); } catch (err) { @@ -83,14 +106,10 @@ define([ throw err; } } - console.log('created working directory at', tmpdir); + this.logger.info('created working directory at', tmpdir); // Fetch the required files from deepforge - try { - await this.prepareWorkspace(hash, tmpdir); - } catch (err) { - console.log(`Error: ${err}`); - } + await this.prepareWorkspace(hash, tmpdir); // Spin up a subprocess const config = JSON.parse(await readFile(tmpdir.replace(path.sep, '/') + '/executor_config.json', 'utf8')); @@ -107,10 +126,9 @@ define([ } await this._uploadResults(jobInfo, tmpdir, config); + this._onJobCompleted(); this.emit('end', hash, jobInfo); }); - - return {}; }; LocalExecutor.prototype.onConsoleOutput = async function(workdir, hash, data) { @@ -246,7 +264,6 @@ define([ const allFiles = await this._getAllFiles(directory); - console.log('final', allFiles); const filesToArchive = []; let archive, filename, @@ -262,7 +279,6 @@ define([ } else { for (let j = 0; j < resultsArtifacts[a].patterns.length; j += 1) { matched = minimatch(filename, resultsArtifacts[a].patterns[j]); - console.log(filename, 'matches',resultsArtifacts[a].patterns[j], '?', matched); if (matched) { resultsArtifacts[a].files[filename] = true; archive = true; @@ -319,11 +335,13 @@ define([ // Set up a symbolic link to the node_modules await symlink(NODE_MODULES, path.join(dirname, 'node_modules')); + + // Prepare for the stdout + await touch(path.join(dirname, 'job_stdout.txt')); }; async function unzip(filename, dirname) { const args = UNZIP_ARGS.concat(path.basename(filename)); - console.log('running:', UNZIP_EXE, args.join(' ')); await execFile(UNZIP_EXE, args, {cwd: dirname}); await rm_rf(filename); diff --git a/src/plugins/ExecuteJob/ExecuteJob.js b/src/plugins/ExecuteJob/ExecuteJob.js index 4e48a9136..60819d380 100644 --- a/src/plugins/ExecuteJob/ExecuteJob.js +++ b/src/plugins/ExecuteJob/ExecuteJob.js @@ -227,43 +227,37 @@ define([ return !!this.getAttribute(job, 'jobId'); }; - ExecuteJob.prototype.resumeJob = function (job) { - console.log('resuming job...'); + ExecuteJob.prototype.resumeJob = async function (job) { var hash = this.getAttribute(job, 'jobId'), name = this.getAttribute(job, 'name'), - id = this.core.getPath(job), - msg; + id = this.core.getPath(job); this.logger.info(`Resuming job ${name} (${id})`); - return this.logManager.getMetadata(id) - .then(metadata => { - var count = metadata.lineCount; + const metadata = await this.logManager.getMetadata(id); + let count = metadata.lineCount; - if (count === -1) { - this.logger.warn(`No line count found for ${id}. Setting count to 0`); - count = 0; - return this.logManager.deleteLog(id) - .then(() => count); - } - return count; - }) - .then(count => { // update line count (to inform logClient appendTo) - this.outputLineCount[id] = count; - return this.executor.getOutput(hash, 0, count); // TODO FIXME - }) - .then(async output => { // parse the stdout to update the job metadata - var stdout = output.map(o => o.output).join(''), - result = this.processStdout(job, stdout), - name = this.getAttribute(job, 'name'); - - if (result.hasMetadata) { - msg = `Updated graph/image output for ${name}`; - await this.save(msg); - } - return this.getOperation(job); - }) - //.then(opNode => this.watchOperation(hash, opNode, job)); + if (count === -1) { + this.logger.warn(`No line count found for ${id}. Setting count to 0`); + count = 0; + await this.logManager.deleteLog(id); + } + + this.outputLineCount[id] = count; + + // TODO: Need to be able to poll the stdout... + const output = await this.executor.getOutput(hash); + const stdout = output.map(o => o.output).join(''); + + const result = this.processStdout(job, stdout); + + if (result.hasMetadata) { + const name = this.getAttribute(job, 'name'); + const msg = `Updated graph/image output for ${name}`; + await this.save(msg); + } + + return this.getOperation(job); }; ExecuteJob.prototype.updateForkName = function (basename) { From f70dc371b8081c0eaa08e4f290c68c8971ffc9df Mon Sep 17 00:00:00 2001 From: Brian Broll Date: Mon, 9 Sep 2019 15:06:18 -0500 Subject: [PATCH 10/45] WIP #1186 adding cancel support --- src/common/execution/backends/Local.js | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/src/common/execution/backends/Local.js b/src/common/execution/backends/Local.js index 114bcc955..0ef4da911 100644 --- a/src/common/execution/backends/Local.js +++ b/src/common/execution/backends/Local.js @@ -47,6 +47,7 @@ define([ BaseExecutor.apply(this, arguments); this.jobQueue = []; this.currentJob = null; + this.subprocess = null; // FIXME: set this meaningfully! this.blobClient = new BlobClient({ server: '127.0.0.1', @@ -58,8 +59,16 @@ define([ LocalExecutor.prototype = Object.create(BaseExecutor.prototype); - LocalExecutor.prototype.cancelJob = function(job) { - return this.executor.cancelJob(job.hash, job.secret); + LocalExecutor.prototype.cancelJob = function(jobInfo) { + const {hash} = jobInfo; + + console.log('>>> CANCELING job!!'); + if (this.currentJob === hash) { + this.subprocess.kill(); + } else if (this.jobQueue.includes(hash)) { + const i = this.jobQueue.indexOf(hash); + this.jobQueue.splice(i, 1); + } }; LocalExecutor.prototype.getOutput = async function(hash) { @@ -67,11 +76,11 @@ define([ return await readFile(filename, 'utf8'); }; - // TODO: Add cancel support! TODO LocalExecutor.prototype.createJob = async function(hash) { this.jobQueue.push(hash); this._processNextJob(); + console.log('>>> CREATING job!!'); return {hash}; }; @@ -116,10 +125,10 @@ define([ const env = {cwd: tmpdir}; this.logger.info(`Running ${config.cmd} ${config.args.join(' ')}`); - execJob = spawn(config.cmd, config.args, env); - execJob.stdout.on('data', data => this.onConsoleOutput(tmpdir, hash, data)); + this.subprocess = spawn(config.cmd, config.args, env); + this.subprocess.stdout.on('data', data => this.onConsoleOutput(tmpdir, hash, data)); - execJob.on('close', async code => { + this.subprocess.on('close', async code => { const jobInfo = { resultHashes: [], status: code !== 0 ? 'FAILED_TO_EXECUTE' : 'SUCCESS' From efe4ee9860191c6de50395171c3707731bc3bb3a Mon Sep 17 00:00:00 2001 From: Brian Broll Date: Tue, 10 Sep 2019 12:07:13 -0500 Subject: [PATCH 11/45] WIP #1186 changed from polling to events --- src/common/execution/backends/BaseExecutor.js | 36 +- src/common/execution/backends/GME.js | 26 + src/common/execution/backends/Local.js | 92 +-- src/common/plugin/LocalExecutor.js | 5 +- src/plugins/ExecuteJob/ExecuteJob.SafeSave.js | 116 ++-- src/plugins/ExecuteJob/ExecuteJob.js | 572 ++++++++---------- .../ExecutePipeline/ExecutePipeline.js | 68 +-- 7 files changed, 435 insertions(+), 480 deletions(-) diff --git a/src/common/execution/backends/BaseExecutor.js b/src/common/execution/backends/BaseExecutor.js index aa5ad85e6..6fe8154fd 100644 --- a/src/common/execution/backends/BaseExecutor.js +++ b/src/common/execution/backends/BaseExecutor.js @@ -6,31 +6,44 @@ define([ window.location.protocol !== 'http:'; this.logger = logger.fork('executor'); - this._events = {}; // FIXME: there must be a better way... + this._events = {}; }; BaseExecutor.prototype.cancelJob = function(job) { const msg = `cancelJob is not implemented for current executor backend!`; this.logger.warn(msg); - return Promise.reject(new Error(msg)) + throw new Error(msg); }; BaseExecutor.prototype.getInfo = function(job) { const msg = `getInfo is not implemented for current executor backend!`; this.logger.warn(msg); - return Promise.reject(new Error(msg)) + throw new Error(msg); }; BaseExecutor.prototype.createJob = async function(hash) { const msg = `createJob is not implemented for current executor backend!`; this.logger.warn(msg); - return Promise.reject(new Error(msg)) + throw new Error(msg); }; - BaseExecutor.prototype.getOutput = async function(hash) { - const msg = `getOutput is not implemented for current executor backend!`; + BaseExecutor.prototype.getStatus = async function(jobInfo) { + const msg = `getStatus is not implemented for current executor backend!`; this.logger.warn(msg); - return Promise.reject(new Error(msg)) + throw new Error(msg); + }; + + BaseExecutor.prototype.getOutputHashes = async function(jobInfo) { + const msg = `getOutputHashes is not implemented for current executor backend!`; + this.logger.warn(msg); + throw new Error(msg); + }; + + // TODO: Should I remove this? + BaseExecutor.prototype.getConsoleOutput = async function(hash) { + const msg = `getConsoleOutput is not implemented for current executor backend!`; + this.logger.warn(msg); + throw new Error(msg); }; // Some functions for event support @@ -53,5 +66,14 @@ define([ handlers.forEach(fn => fn.apply(this, args)); }; + // TODO: Make these match the values in the model (`status` enum on pipeline.Job) + BaseExecutor.prototype.QUEUED = 'queued'; + BaseExecutor.prototype.PENDING = 'pending'; + BaseExecutor.prototype.RUNNING = 'running'; + BaseExecutor.prototype.SUCCESS = 'success'; + BaseExecutor.prototype.FAILED = 'failed'; + BaseExecutor.prototype.CANCELED = 'canceled'; + BaseExecutor.prototype.NOT_FOUND = 'NOT_FOUND'; + return BaseExecutor; }); diff --git a/src/common/execution/backends/GME.js b/src/common/execution/backends/GME.js index 35a8d4c80..97db8444b 100644 --- a/src/common/execution/backends/GME.js +++ b/src/common/execution/backends/GME.js @@ -1,6 +1,8 @@ define([ + 'deepforge/ExecutionEnv', 'executor/ExecutorClient' ], function( + ExecutionEnv, ExecutorClient ) { // TODO @@ -19,16 +21,40 @@ define([ this._events = {}; // FIXME: there must be a better way... }; + GMEExecutor.prototype.getConsoleOutput = async function(hash) { + return (await this.executor.getOutput(hash)) + .map(o => o.output).join(''); + }; + GMEExecutor.prototype.cancelJob = function(job) { return this.executor.cancelJob(job.hash, job.secret); }; + GMEExecutor.prototype.getOutputHashes = async function(job) { + return (await this.executor.getInfo(job)).resultHashes; + }; + // TODO: Standardize this + GMEExecutor.prototype.getStatus = async function(job) { + // TODO: Convert the status to the appropriate code + }; + GMEExecutor.prototype.getInfo = function(job) { return this.executor.getInfo(job.hash); }; + GMEExecutor.prototype.checkExecutionEnv = async function () { + this.logger.info(`Checking execution environment`); + const workers = await ExecutionEnv.getWorkers(); + if (workers.length === 0) { + this.logger.info(`Cannot execute job(s): No connected workers`); + throw new Error('No connected workers'); + } + }; + GMEExecutor.prototype.createJob = async function(hash) { + await this.checkExecutionEnv(); + const result = await this.executor.createJob({hash}); this.startPolling(hash); diff --git a/src/common/execution/backends/Local.js b/src/common/execution/backends/Local.js index 0ef4da911..7875dd7bb 100644 --- a/src/common/execution/backends/Local.js +++ b/src/common/execution/backends/Local.js @@ -45,9 +45,11 @@ define([ const LocalExecutor = function(logger, gmeConfig) { BaseExecutor.apply(this, arguments); + this.completedJobs = {}; this.jobQueue = []; this.currentJob = null; this.subprocess = null; + this.canceled = false; // FIXME: set this meaningfully! this.blobClient = new BlobClient({ server: '127.0.0.1', @@ -64,28 +66,59 @@ define([ console.log('>>> CANCELING job!!'); if (this.currentJob === hash) { + this.canceled = true; this.subprocess.kill(); } else if (this.jobQueue.includes(hash)) { const i = this.jobQueue.indexOf(hash); this.jobQueue.splice(i, 1); + this._onJobCompleted(hash, new JobResults(this.CANCELED)); } }; - LocalExecutor.prototype.getOutput = async function(hash) { + LocalExecutor.prototype.getStatus = async function(jobInfo) { + const {hash} = jobInfo; + if (hash === this.currentJob) { + return this.RUNNING; + } else if (this.jobQueue.includes(hash)) { + return this.QUEUED; + } else if (this.completedJobs[hash]) { + return this.completedJobs[hash].status; + } else { + console.log(`Could not find ${hash} in`, this.completedJobs); + throw new Error('Job Not Found'); + } + }; + + LocalExecutor.prototype.getConsoleOutput = async function(hash) { const filename = path.join(this._getWorkingDir(hash), 'job_stdout.txt'); return await readFile(filename, 'utf8'); }; + LocalExecutor.prototype.getOutputHashes = async function(jobInfo) { + const {hash} = jobInfo; + + if (this.completedJobs[hash]) { + return this.completedJobs[hash].resultHashes; + } else { + throw new Error(`Job Not Found: ${hash}`); + } + }; + LocalExecutor.prototype.createJob = async function(hash) { this.jobQueue.push(hash); this._processNextJob(); - console.log('>>> CREATING job!!'); return {hash}; }; - LocalExecutor.prototype._onJobCompleted = function() { - this.currentJob = null; + LocalExecutor.prototype._onJobCompleted = function(hash, jobResults) { + if (hash === this.currentJob) { + this.currentJob = null; + } + + this.completedJobs[hash] = jobResults; + this.emit('update', {hash}, jobResults.status); + this.emit('end', hash, jobResults); this._processNextJob(); }; @@ -104,6 +137,8 @@ define([ LocalExecutor.prototype._createJob = async function(hash) { // Create tmp directory + const jobInfo = {hash}; + this.emit('update', jobInfo, this.PENDING); const tmpdir = this._getWorkingDir(hash); try { await mkdir(tmpdir); @@ -126,17 +161,17 @@ define([ const env = {cwd: tmpdir}; this.logger.info(`Running ${config.cmd} ${config.args.join(' ')}`); this.subprocess = spawn(config.cmd, config.args, env); + this.emit('update', jobInfo, this.RUNNING); this.subprocess.stdout.on('data', data => this.onConsoleOutput(tmpdir, hash, data)); this.subprocess.on('close', async code => { - const jobInfo = { - resultHashes: [], - status: code !== 0 ? 'FAILED_TO_EXECUTE' : 'SUCCESS' - } + const status = this.canceled ? this.CANCELED : + (code !== 0 ? this.FAILED : this.SUCCESS); + const jobResults = new JobResults(status); + this.canceled = false; - await this._uploadResults(jobInfo, tmpdir, config); - this._onJobCompleted(); - this.emit('end', hash, jobInfo); + await this._uploadResults(jobResults, tmpdir, config); + this._onJobCompleted(hash, jobResults); }); }; @@ -251,7 +286,7 @@ define([ self.logger.error('Could not delete executor-temp file, err: ' + err); } jobInfo.resultSuperSetHash = resultHash; - return Promise.all(resultsArtifacts.map(r => addObjectHashesAndSaveArtifact(r))) + return Promise.all(resultsArtifacts.map(r => addObjectHashesAndSaveArtifact(r))); }; addObjectHashesAndSaveArtifact = promisify(function (resultArtifact, callback) { @@ -309,31 +344,8 @@ define([ } }; - //LocalExecutor.prototype._uploadResults = async function(workdir, config) { - //// Get all the matching result artifacts - //const allFiles = await this._getAllFiles(workdir); - //console.log(allFiles) - - //// Upload all the artifacts - //const artifacts = config.resultArtifacts - //.map(info => { - //return { - //name: info.name, - //artifact: self.blobClient.createArtifact(info.name), - //patterns: info.resultPatterns instanceof Array ? - //info.resultPatterns : [], - //files: {} - //} - //}) - //// TODO - - //// Return the hashes - //// TODO - //return {}; - //}; - LocalExecutor.prototype.prepareWorkspace = async function(hash, dirname) { - this.logger.info(`about to fetch job data`); + this.logger.info('about to fetch job data'); const content = new Buffer(new Uint8Array(await this.blobClient.getObject(hash))); // TODO: Handle errors... const zipPath = path.join(dirname, `${hash}.zip`); await writeFile(zipPath, content); @@ -361,6 +373,12 @@ define([ } // - [ ] emit updates on stdout... - return LocalExecutor; + class JobResults { + constructor(status) { + this.status = status || LocalExecutor.prototype.CREATED; + this.resultHashes = []; + } + } + return LocalExecutor; }); diff --git a/src/common/plugin/LocalExecutor.js b/src/common/plugin/LocalExecutor.js index d857a5ee3..2ddb25827 100644 --- a/src/common/plugin/LocalExecutor.js +++ b/src/common/plugin/LocalExecutor.js @@ -174,9 +174,8 @@ define([ // Helper methods LocalExecutor.prototype.getLocalOperationType = function(node) { - var type; - for (var i = LocalExecutor.OPERATIONS.length; i--;) { - type = LocalExecutor.OPERATIONS[i]; + for (let i = LocalExecutor.OPERATIONS.length; i--;) { + const type = LocalExecutor.OPERATIONS[i]; if (!this.META[type]) { this.logger.warn(`Missing local operation: ${type}`); continue; diff --git a/src/plugins/ExecuteJob/ExecuteJob.SafeSave.js b/src/plugins/ExecuteJob/ExecuteJob.SafeSave.js index e6155405c..675b2e0a0 100644 --- a/src/plugins/ExecuteJob/ExecuteJob.SafeSave.js +++ b/src/plugins/ExecuteJob/ExecuteJob.SafeSave.js @@ -50,33 +50,13 @@ define([ }; ExecuteJob.prototype.setAttribute = function (node, attr, value) { - const changes = this.getChangesForNode(node); - changes.attr[attr] = value; - }; - - ExecuteJob.prototype.getChangesForNode = function (node) { - var nodeId; - - if (this.isCreateId(node)) { - nodeId = node; - } else { - nodeId = this.core.getPath(node); - assert(typeof nodeId === 'string', `Cannot set attribute of ${nodeId}`); - } - - if (!this.changes[nodeId]) { - this.changes[nodeId] = { - attr: {}, - ptr: {}, - }; + if (value === undefined) { + throw new Error(`Cannot set attributes to undefined (${attr})`); } - return this.changes[nodeId]; - }; - - ExecuteJob.prototype.setPointer = function (node, name, target) { + this.logger.warn(`setting ${attr} to ${value}`); const changes = this.getChangesForNode(node); - changes.ptr[name] = target; + changes.attr[attr] = value; }; ExecuteJob.prototype.getAttribute = function (node, attr) { @@ -108,6 +88,31 @@ define([ return this.core.getAttribute(node, attr); }; + ExecuteJob.prototype.getChangesForNode = function (node) { + var nodeId; + + if (this.isCreateId(node)) { + nodeId = node; + } else { + nodeId = this.core.getPath(node); + assert(typeof nodeId === 'string', `Cannot set attribute of ${nodeId}`); + } + + if (!this.changes[nodeId]) { + this.changes[nodeId] = { + attr: {}, + ptr: {}, + }; + } + + return this.changes[nodeId]; + }; + + ExecuteJob.prototype.setPointer = function (node, name, target) { + const changes = this.getChangesForNode(node); + changes.ptr[name] = target; + }; + ExecuteJob.prototype._getValueFrom = function (nodeId, attr, node, changes) { var base; if (changes[nodeId] && changes[nodeId].attr[attr] !== undefined) { @@ -374,45 +379,38 @@ define([ this.sendNotification(msg); }; - ExecuteJob.prototype.updateNodes = function (hash) { - var activeId = this.core.getPath(this.activeNode); + //ExecuteJob.prototype.registerExistingNodeDict = function (hash) { + ExecuteJob.prototype.updateNodes = async function (hash) { + const activeId = this.core.getPath(this.activeNode); hash = hash || this.currentHash; - return Q.ninvoke(this.project, 'loadObject', hash) - .then(commitObject => { - return this.core.loadRoot(commitObject.root); - }) - .then(rootObject => { - this.rootNode = rootObject; - return this.core.loadByPath(rootObject,activeId); - }) - .then(activeObject => this.activeNode = activeObject) - .then(() => { - var metaNames = Object.keys(this.META); - - return Q.all(metaNames.map(name => this.updateMetaNode(name))); - }) - .then(() => { - var mdNodes, - mdIds; - - mdIds = Object.keys(this._metadata) - .filter(id => !this.isCreateId(this._metadata[id])); - - mdNodes = mdIds.map(id => this.core.getPath(this._metadata[id])) - .map(nodeId => this.core.loadByPath(this.rootNode, nodeId)); - - return Q.all(mdNodes).then(nodes => { - for (var i = nodes.length; i--;) { - this._metadata[mdIds[i]] = nodes[i]; - } - }); - }); + const commitObject = await Q.ninvoke(this.project, 'loadObject', hash); + this.rootNode = await this.core.loadRoot(commitObject.root); + this.activeNode = await this.core.loadByPath(this.rootNode, activeId); + + // TODO: Can I refactor this? + await this.updateExistingNodeDict(this.META); + await this.updateExistingNodeDict(this._execHashToJobNode); + + const existingIds = Object.keys(this._metadata) + .filter(id => !this.isCreateId(this._metadata[id])); + + await this.updateExistingNodeDict(this._metadata, existingIds); }; - ExecuteJob.prototype.updateMetaNode = function (name) { - var id = this.core.getPath(this.META[name]); - return this.core.loadByPath(this.rootNode, id).then(node => this.META[name] = node); + /** + * Update a dictionary of *existing* nodes to the node instances in the + * current commit. + */ + ExecuteJob.prototype.updateExistingNodeDict = function (dict, keys) { + keys = keys || Object.keys(dict); + + return Q.all(keys.map(key => { + const oldNode = dict[key]; + const nodePath = this.core.getPath(oldNode); + return this.core.loadByPath(this.rootNode, nodePath) + .then(newNode => dict[key] = newNode); + })); }; return ExecuteJob; diff --git a/src/plugins/ExecuteJob/ExecuteJob.js b/src/plugins/ExecuteJob/ExecuteJob.js index 60819d380..c8fd70184 100644 --- a/src/plugins/ExecuteJob/ExecuteJob.js +++ b/src/plugins/ExecuteJob/ExecuteJob.js @@ -6,7 +6,6 @@ define([ 'text!./metadata.json', 'deepforge/execution/index', 'plugin/PluginBase', - 'deepforge/ExecutionEnv', 'deepforge/plugin/LocalExecutor', 'deepforge/plugin/PtrCodeGen', 'deepforge/plugin/Operation', @@ -25,7 +24,6 @@ define([ pluginMetadata, ExecutorClient, PluginBase, - ExecutionEnv, LocalExecutor, // DeepForge operation primitives PtrCodeGen, OperationPlugin, @@ -61,7 +59,7 @@ define([ ExecuteJobSafeSave.call(this); ExecuteJobMetadata.call(this); this.pluginMetadata = pluginMetadata; - this._beating = null; + this._running = null; // Metadata updating this.lastAppliedCmd = {}; @@ -75,13 +73,8 @@ define([ this.logManager = null; }; - /** - * Metadata associated with the plugin. Contains id, name, version, description, icon, configStructue etc. - * This is also available at the instance at this.pluginMetadata. - * @type {object} - */ + // TODO: Update plugin metadata for the executor options ExecuteJob.metadata = pluginMetadata; - ExecuteJob.UPDATE_INTERVAL = 1500; ExecuteJob.HEARTBEAT_INTERVAL = 2500; // Prototypical inheritance from PluginBase. @@ -104,8 +97,6 @@ define([ // TODO: load a custom executor this.executor = new ExecutorClient(this.logger, this.gmeConfig); - // Look up the job from the id... - // TODO this.executor.on( 'data', (id, data) => { @@ -113,10 +104,19 @@ define([ this.onConsoleOutput(job, data.toString()); } ); + + this.executor.on('update', (jobInfo, status) => { + try { + this.onUpdate(jobInfo, status); + } catch (err) { + this.logger.error(`Error when processing operation update: ${err}`); + } + }); + this.executor.on('end', (id, info) => { try { - this.onOperationEnd(id, info); + this.onOperationEnd(id); } catch (err) { this.logger.error(`Error when processing operation end: ${err}`); } @@ -135,92 +135,67 @@ define([ * * @param {function(string, plugin.PluginResult)} callback - the result callback */ - ExecuteJob.prototype.main = function (callback) { + ExecuteJob.prototype.main = async function (callback) { // Check the activeNode to make sure it is a valid node var type = this.core.getMetaType(this.activeNode), typeName = type && this.getAttribute(type, 'name'), - execNode, - status; + execNode; if (typeName !== 'Job') { - return callback(`Cannot execute ${typeName} (expected Job)`, this.result); + return callback(new Error(`Cannot execute ${typeName} (expected Job)`), this.result); } // Set the parent execution to 'running' execNode = this.core.getParent(this.activeNode); - status = this.getAttribute(execNode, 'status'); - if (status !== 'running') { - this.setAttribute(execNode, 'status', 'running'); - } + this.setAttribute(execNode, 'status', 'running'); this._callback = callback; this.currentForkName = null; this.forkNameBase = this.getAttribute(this.activeNode, 'name'); - this.checkExecutionEnv() - .then(() => this.isResuming(this.activeNode)) - .then(resuming => { - this._resumed = resuming; - return this.prepare(resuming); - }) - .then(() => { - if (this._resumed) { - this.currentRunId = this.getAttribute(this.activeNode, 'jobId'); - this.startExecHeartBeat(); - if (this.canResumeJob(this.activeNode)) { - return this.resumeJob(this.activeNode); - } else { - var name = this.getAttribute(this.activeNode, 'name'), - id = this.core.getPath(this.activeNode), - msg = `Cannot resume ${name} (${id}). Missing jobId.`; - this.logger.error(msg); - return callback(msg); - } - } else { - this.currentRunId = null; // will be set after exec files created - return this.executeJob(this.activeNode); - } - }) - .catch(err => this._callback(err, this.result)); - }; + const isResuming = await this.isResuming(this.activeNode); + this.watchForCancel(); + await this.prepare(isResuming); - ExecuteJob.prototype.checkExecutionEnv = function () { - // Throw an exception if no resources - this.logger.info(`Checking execution environment`); - return ExecutionEnv.getWorkers() - .then(workers => { - if (workers.length === 0) { - this.logger.info(`Cannot execute job(s): No connected workers`); - throw new Error('No connected workers'); - } - }); + if (isResuming) { + this.currentRunId = this.getAttribute(this.activeNode, 'jobId'); + this.startExecHeartBeat(); + if (this.canResumeJob(this.activeNode)) { + return this.resumeJob(this.activeNode); + } else { + var name = this.getAttribute(this.activeNode, 'name'), + id = this.core.getPath(this.activeNode), + msg = `Cannot resume ${name} (${id}). Missing jobId.`; + + this.logger.error(msg); + return callback(msg); + } + } else { + this.currentRunId = null; // will be set after exec files created + return this.executeJob(this.activeNode); + } + //.catch(err => this._callback(err, this.result)); // TODO: Is this needed? }; - ExecuteJob.prototype.isResuming = function (job) { + ExecuteJob.prototype.isResuming = async function (job) { job = job || this.activeNode; - var deferred = Q.defer(), - status = this.getAttribute(job, 'status'), + var status = this.getAttribute(job, 'status'), jobId; if (status === 'running') { jobId = this.getAttribute(job, 'jobId'); // Check if on the origin branch - this.originManager.getOrigin(jobId) - .then(origin => { - if (this.branchName === origin.branch) { - // Check if plugin is no longer running - return this.pulseClient.check(jobId) - .then(alive => { - deferred.resolve(alive !== CONSTANTS.PULSE.ALIVE); - }); - } else { - deferred.resolve(false); - } - }); - } else { - deferred.resolve(false); + const origin = await this.originManager.getOrigin(jobId); + if (this.branchName === origin.branch) { + // Check if plugin is no longer running + const alive = await this.pulseClient.check(jobId); + return alive !== CONSTANTS.PULSE.ALIVE; + } else { + return false; + } } - return deferred.promise; + + return false; }; ExecuteJob.prototype.canResumeJob = function (job) { @@ -246,8 +221,7 @@ define([ this.outputLineCount[id] = count; // TODO: Need to be able to poll the stdout... - const output = await this.executor.getOutput(hash); - const stdout = output.map(o => o.output).join(''); + const stdout = await this.executor.getConsoleOutput(hash); const result = this.processStdout(job, stdout); @@ -289,39 +263,35 @@ define([ return conns; }; - ExecuteJob.prototype.prepare = function (isResuming) { - var dstPortId, - srcPortId, - conns, - executionNode = this.core.getParent(this.activeNode); + ExecuteJob.prototype.prepare = async function (isResuming) { + const executionNode = this.core.getParent(this.activeNode); + const nodes = await this.core.loadSubTree(executionNode); this.pipelineName = this.getAttribute(executionNode, 'name'); - return this.core.loadSubTree(executionNode) - .then(nodes => { - this.inputPortsFor = {}; - this.outputLineCount = {}; + this.inputPortsFor = {}; + this.outputLineCount = {}; - conns = this.getConnections(nodes); + const conns = this.getConnections(nodes); - // Create inputPortsFor for the given input ports - for (var i = conns.length; i--;) { - dstPortId = this.core.getPointerPath(conns[i], 'dst'); - srcPortId = this.core.getPointerPath(conns[i], 'src'); + // Create inputPortsFor for the given input ports + for (var i = conns.length; i--;) { + const dstPortId = this.core.getPointerPath(conns[i], 'dst'); + const srcPortId = this.core.getPointerPath(conns[i], 'src'); - if (!this.inputPortsFor[dstPortId]) { - this.inputPortsFor[dstPortId] = [srcPortId]; - } else { - this.inputPortsFor[dstPortId].push(srcPortId); - } - } - }) - .then(() => this.recordOldMetadata(this.activeNode, isResuming)); + if (!this.inputPortsFor[dstPortId]) { + this.inputPortsFor[dstPortId] = [srcPortId]; + } else { + this.inputPortsFor[dstPortId].push(srcPortId); + } + } + + return await this.recordOldMetadata(this.activeNode, isResuming); }; ExecuteJob.prototype.onOperationCanceled = function(op) { - var job = this.core.getParent(op), - name = this.getAttribute(op, 'name'), - msg = `"${name}" canceled!`; + const job = this.core.getParent(op); + const name = this.getAttribute(op, 'name'); + const msg = `"${name}" canceled!`; this.setAttribute(job, 'status', 'canceled'); this.resultMsg(msg); @@ -331,12 +301,12 @@ define([ ExecuteJob.prototype.onOperationFail = ExecuteJob.prototype.onOperationComplete = ExecuteJob.prototype.onComplete = async function (opNode, err) { - var job = this.core.getParent(opNode), - exec = this.core.getParent(job), - name = this.getAttribute(job, 'name'), - jobId = this.core.getPath(job), - status = err ? 'fail' : (this.canceled ? 'canceled' : 'success'), - msg = err ? `${name} execution failed!` : + const job = this.core.getParent(opNode); + const exec = this.core.getParent(job); + const name = this.getAttribute(job, 'name'); + const jobId = this.core.getPath(job); + const status = err ? 'fail' : (this.canceled ? 'canceled' : 'success'); + const msg = err ? `${name} execution failed!` : `${name} executed successfully!`; this.setAttribute(job, 'status', status); @@ -385,9 +355,40 @@ define([ } }; - ExecuteJob.prototype.getOperation = function (job) { - return this.core.loadChildren(job).then(children => - children.find(child => this.isMetaTypeOf(child, this.META.Operation))); + ExecuteJob.prototype.getOperation = async function (job) { + const children = await this.core.loadChildren(job); + return children.find(child => this.isMetaTypeOf(child, this.META.Operation)); + }; + + ExecuteJob.prototype.executeJob = async function (job) { + const node = await this.getOperation(job); + const name = this.getAttribute(node, 'name'); + + // Execute any special operation types here - not on an executor + this.logger.debug(`Executing operation "${name}"`); + if (this.isLocalOperation(node)) { + return this.executeLocalOperation(node); + } else { + // Generate all execution files + let hash; + try { + hash = await this.getPtrCodeHash(this.core.getPath(node)); + } catch (err) { + this.logger.error(`Could not generate files: ${err}`); + if (err.message.indexOf('BLOB_FETCH_FAILED') > -1) { + this.onBlobRetrievalFail(node, err.message.split(':')[1]); + } + throw err; + } + + this.logger.info(`Saved execution files`); + this.result.addArtifact(hash); + try { + this.executeDistOperation(job, node, hash); + } catch (err) { + this.onOperationFail(node, `Distributed operation "${name}" failed ${err}`); + } + } }; // Handle the blob retrieval failed error @@ -410,37 +411,6 @@ define([ this.onOperationFail(node, `Blob retrieval failed for "${name}": ${e}`); }; - ExecuteJob.prototype.executeJob = function (job) { - return this.getOperation(job).then(node => { - var name = this.getAttribute(node, 'name'), - localTypeId = this.getLocalOperationType(node); - - // Execute any special operation types here - not on an executor - this.logger.debug(`Executing operation "${name}"`); - if (localTypeId !== null) { - return this.executeLocalOperation(localTypeId, node); - } else { - // Generate all execution files - return this.getPtrCodeHash(this.core.getPath(node)) - .fail(err => { - this.logger.error(`Could not generate files: ${err}`); - if (err.message.indexOf('BLOB_FETCH_FAILED') > -1) { - this.onBlobRetrievalFail(node, err.message.split(':')[1]); - } - throw err; - }) - .then(hash => { - this.logger.info(`Saved execution files`); - this.result.addArtifact(hash); // Probably only need this for debugging... - this.executeDistOperation(job, node, hash); - }) - .fail(e => { - this.onOperationFail(node, `Distributed operation "${name}" failed ${e}`); - }); - } - }); - }; - ExecuteJob.prototype.executeDistOperation = async function (job, opNode, hash) { var name = this.getAttribute(opNode, 'name'), jobId = this.core.getPath(job); @@ -467,25 +437,18 @@ define([ ExecuteJob.prototype.createJob = async function (job, opNode, hash) { // Record the job info for the given hash this._execHashToJobNode[hash] = [job, opNode]; - const info = this.executor.createJob(hash); - this.setAttribute(job, 'jobId', info.hash); - // Store the entire info object? This could be used for canceling! - // TODO - this.setAttribute(job, 'jobInfo', JSON.stringify(info)); + const jobInfo = await this.executor.createJob(hash); + this.setAttribute(job, 'jobId', jobInfo.hash); + this.setAttribute(job, 'jobInfo', JSON.stringify(jobInfo)); if (!this.currentRunId) { - this.currentRunId = info.hash; - if (this._beating === null) { - this.startExecHeartBeat(); - } + this.currentRunId = jobInfo.hash; } - // Add event handlers to the executor - // TODO - // Should the metadata handlers be here or somewhere else? - // Probably somewhere else (in the executor client/base class) - // TODO + if (this._running === null) { + this.startExecHeartBeat(); + this.watchForCancel(); + } - // Don't set these callbacks up for each operation... return await this.recordJobOrigin(hash, job); }; @@ -494,10 +457,9 @@ define([ }; ExecuteJob.prototype.recordJobOrigin = function (hash, job) { - var execNode = this.core.getParent(job), - info; + const execNode = this.core.getParent(job); - info = { + const info = { hash: hash, nodeId: this.core.getPath(job), job: this.getAttribute(job, 'name'), @@ -527,23 +489,23 @@ define([ }; ExecuteJob.prototype.isExecutionCanceled = function () { - var execNode = this.core.getParent(this.activeNode); + const execNode = this.core.getParent(this.activeNode); return this.getAttribute(execNode, 'status') === 'canceled'; }; ExecuteJob.prototype.startExecHeartBeat = function () { - this._beating = true; + this._running = true; this.updateExecHeartBeat(); }; ExecuteJob.prototype.stopExecHeartBeat = function () { - this._beating = false; + this._running = false; }; ExecuteJob.prototype.updateExecHeartBeat = function () { var time = Date.now(), next = () => { - if (this._beating) { + if (this._running) { setTimeout(this.updateExecHeartBeat.bind(this), ExecuteJob.HEARTBEAT_INTERVAL - (Date.now() - time)); } @@ -559,13 +521,59 @@ define([ }); }; + ExecuteJob.prototype.onUpdate = async function (jobInfo, status) { + const [job] = this.getNodesForJobHash(jobInfo.hash); + const name = this.getAttribute(job, 'name'); + + console.log(`setting status to`, status); + this.setAttribute(job, 'status', status); + await this.save(`"${name}" operation in ${this.pipelineName} is now "${status}"`); + }; + ExecuteJob.prototype.onConsoleOutput = async function (job, output) { const jobId = this.core.getPath(job); - var stdout = this.getAttribute(job, 'stdout'), - last = stdout.lastIndexOf('\n'), - result, - lastLine, - msg; + let stdout = this.getAttribute(job, 'stdout'); + let last = stdout.lastIndexOf('\n'); + let lastLine; + + //const currentLine = this.outputLineCount[jobId]; + //const actualLine = info.outputNumber; // TODO: Keep this?? + + //// TODO: Move this to the onOutput handler + //if (actualLine !== null && actualLine >= currentLine) { + //this.outputLineCount[jobId] = actualLine + 1; + //// TODO: Keep this?? + //let output = await this.executor.getConsoleOutput(hash, currentLine, actualLine+1)) + + //var stdout = this.getAttribute(job, 'stdout'), + //last = stdout.lastIndexOf('\n'), + //result, + //lastLine, + //next = Q(), + //msg; + + // parse deepforge commands + if (last !== -1) { + stdout = stdout.substring(0, last+1); + lastLine = stdout.substring(last+1); + output = lastLine + output; + } + //result = this.processStdout(job, output, true); + //output = result.stdout; + + //if (output) { + //// Send notification to all clients watching the branch + //const metadata = { + //lineCount: this.outputLineCount[jobId] + //}; + //await this.logManager.appendTo(jobId, output, metadata); + //await this.notifyStdoutUpdate(jobId); + //} + //if (result.hasMetadata) { + //msg = `Updated graph/image output for ${name}`; + //await this.save(msg); + //} + //} // parse deepforge commands // FIXME: This needs to be added back! @@ -574,7 +582,7 @@ define([ //lastLine = stdout.substring(last+1); //output = lastLine + output; //} - result = this.processStdout(job, output, true); + const result = this.processStdout(job, output, true); output = result.stdout; await this.logManager.appendTo(jobId, output); @@ -582,206 +590,100 @@ define([ await this.notifyStdoutUpdate(jobId); if (result.hasMetadata) { - msg = `Updated graph/image output for ${name}`; + const msg = `Updated graph/image output for ${name}`; await this.save(msg); } }; - ExecuteJob.prototype.watchOperation = function (hash, op, job) { - var jobId = this.core.getPath(job), - opId = this.core.getPath(op), - info, - jobInfo = this.getAttribute(job, 'jobInfo'), - name = this.getAttribute(job, 'name'); - - // If canceled, stop the operation + // TODO: Watch if the execution is ever canceled (on the origin branch). + // If so, cancel the executing jobs + /* if (this.canceled || this.isExecutionCanceled()) { if (jobInfo) { - // Will all cancelJob signatures look like this? - // Should they be passing a custom JSON object? - // TODO - this.executor.cancelJob(jobInfo); // TODO FIXME + this.executor.cancelJob(jobInfo); this.delAttribute(job, 'jobInfo'); this.canceled = true; return this.onOperationCanceled(op); } } - - return this.executor.getInfo(jobInfo) - .then(_info => { // Update the job's stdout - var actualLine, // on executing job - currentLine = this.outputLineCount[jobId], - prep = Q(); - - info = _info; - actualLine = info.outputNumber; - if (actualLine !== null && actualLine >= currentLine) { - this.outputLineCount[jobId] = actualLine + 1; - return prep - .then(() => this.executor.getOutput(hash, currentLine, actualLine+1)) - .then(async outputLines => { - var stdout = this.getAttribute(job, 'stdout'), - output = outputLines.map(o => o.output).join(''), - last = stdout.lastIndexOf('\n'), - result, - lastLine, - next = Q(), - msg; - - // parse deepforge commands - if (last !== -1) { - stdout = stdout.substring(0, last+1); - lastLine = stdout.substring(last+1); - output = lastLine + output; - } - result = this.processStdout(job, output, true); - output = result.stdout; - - if (output) { - // Send notification to all clients watching the branch - var metadata = { - lineCount: this.outputLineCount[jobId] - }; - await this.logManager.appendTo(jobId, output, metadata); - await this.notifyStdoutUpdate(jobId); - } - if (result.hasMetadata) { - msg = `Updated graph/image output for ${name}`; - await this.save(msg); - } - }); - } - }) - .then(async () => { - if (info.status === 'CREATED' || info.status === 'RUNNING') { - var time = Date.now(), - next = Q(); - - if (info.status === 'RUNNING' && - this.getAttribute(job, 'status') !== 'running') { - - this.setAttribute(job, 'status', 'running'); - await this.save(`Started "${name}" operation in ${this.pipelineName}`); - } - - const delta = Date.now() - time; - - if (delta > ExecuteJob.UPDATE_INTERVAL) { - return this.watchOperation(hash, op, job); - } - - return setTimeout( - this.watchOperation.bind(this, hash, op, job), - ExecuteJob.UPDATE_INTERVAL - delta - ); - } - - // Record that the job hash is no longer running - this.logger.info(`Job "${name}" has finished (${info.status})`); - var i = this.runningJobHashes.indexOf(hash); - if (i !== -1) { - this.runningJobHashes.splice(i, 1); - } else { - this.logger.warn(`Could not find running job hash ${hash}`); - } - - if (info.status === 'CANCELED') { - // If it was cancelled, the pipeline has been stopped - this.logger.debug(`"${name}" has been CANCELED!`); - this.canceled = true; - return this.logManager.getLog(jobId) - .then(stdout => { - this.setAttribute(job, 'stdout', stdout); - return this.onOperationCanceled(op); - }); - } - - if (info.status === 'SUCCESS' || info.status === 'FAILED_TO_EXECUTE') { - this.setAttribute(job, 'execFiles', info.resultHashes[name + '-all-files']); - const opName = this.getAttribute(op, 'name'); - return this.getContentHashSafe(info.resultHashes.stdout, STDOUT_FILE, ERROR.NO_STDOUT_FILE) - .then(stdoutHash => this.blobClient.getObjectAsString(stdoutHash)) - .then(stdout => { - // Parse the remaining code - var result = this.processStdout(job, stdout); - this.setAttribute(job, 'stdout', result.stdout); - this.logManager.deleteLog(jobId); - if (info.status !== 'SUCCESS') { - // Download all files - this.result.addArtifact(info.resultHashes[name + '-all-files']); - // Parse the most precise error and present it in the toast... - const lastline = result.stdout.split('\n').filter(l => !!l).pop() || ''; - if (lastline.includes('Error')) { - this.onOperationFail(op, lastline); - } else { - this.onOperationFail(op, `Operation "${opName}" failed!`); - } - } else { - this.onDistOperationComplete(op, info); - } - }) - .catch(err => this.onOperationFail(op, `Operation "${opName}" failed: ${err}`)); - } else { // something bad happened... - var err = `Failed to execute operation "${opId}": ${info.status}`, - consoleErr = `Failed to execute operation: ${info.status}`; - this.setAttribute(job, 'stdout', consoleErr); - this.logger.error(err); - this.onOperationFail(op, err); - } - }) - .catch(err => this.logger.error(`Could not get op info for ${JSON.stringify(opId)}: ${err}`)); + */ + + ExecuteJob.prototype.onUserCancelDetected = function () { + console.log('>>> onUserCancelDetected'); + this.runningJobHashes + .map(hash => this.getNodesForJobHash(hash)[0]) + .map(node => this.getAttribute(node, 'jobInfo')) + .forEach(jobInfo => this.executor.cancelJob(jobInfo)); + }; + + ExecuteJob.prototype.watchForCancel = function () { + if (this.isExecutionCanceled()) { + this.onUserCancelDetected(); + } else if (this._running) { + return setTimeout( + this.watchForCancel.bind(this), + 100 + ); + } }; - ExecuteJob.prototype.onOperationEnd = async function (hash, info) { + ExecuteJob.prototype.onOperationEnd = async function (hash) { // Record that the job hash is no longer running const [job, op] = this.getNodesForJobHash(hash); const name = this.getAttribute(job, 'name'); const jobId = this.core.getPath(job); + const jobInfo = JSON.parse(this.getAttribute(job, 'jobInfo')); - this.logger.info(`Job "${name}" has finished (${info.status})`); + const status = await this.executor.getStatus(jobInfo); + this.logger.info(`Job "${name}" has finished (${status})`); this.cleanJobHashInfo(hash); - if (info.status === 'CANCELED') { - // If it was cancelled, the pipeline has been stopped + if (status === this.executor.CANCELED) { + // If it was canceled, the pipeline has been stopped this.logger.debug(`"${name}" has been CANCELED!`); this.canceled = true; - const stdout = this.logManager.getLog(jobId) + const stdout = await this.logManager.getLog(jobId); this.setAttribute(job, 'stdout', stdout); return this.onOperationCanceled(op); } - if (info.status === 'SUCCESS' || info.status === 'FAILED_TO_EXECUTE') { - // TODO: - this.setAttribute(job, 'execFiles', info.resultHashes[name + '-all-files']); - // TODO: Add getStdout to executor? - const artifact = await this.blobClient.getArtifact(info.resultHashes.stdout) - // FIXME: Improve the error handling here - console.log(artifact.descriptor); - const stdoutHash = artifact.descriptor.content[STDOUT_FILE].content; + if (status === this.executor.SUCCESS || status === this.executor.FAILED) { + const fileHashes = await this.executor.getOutputHashes(jobInfo); + const execFilesHash = fileHashes[name + '-all-files']; + this.setAttribute(job, 'execFiles', execFilesHash); + + const opName = this.getAttribute(op, 'name'); + const stdoutHash = await this.getContentHashSafe(fileHashes.stdout, STDOUT_FILE, ERROR.NO_STDOUT_FILE); const stdout = await this.blobClient.getObjectAsString(stdoutHash); + const result = this.processStdout(job, stdout); // Parse the remaining code - const result = this.processStdout(job, stdout); this.setAttribute(job, 'stdout', result.stdout); this.logManager.deleteLog(jobId); - if (info.status !== 'SUCCESS') { - // Download all files - this.result.addArtifact(info.resultHashes[name + '-all-files']); - // Set the job to failed! Store the error - this.onOperationFail(op, `Operation "${jobId}" failed! ${JSON.stringify(info)}`); + if (status === this.executor.SUCCESS) { + this.onDistOperationComplete(op, fileHashes); } else { - this.onDistOperationComplete(op, info); + // Download all files + this.result.addArtifact(execFilesHash); + // Parse the most precise error and present it in the toast... + const lastline = result.stdout.split('\n').filter(l => !!l).pop() || ''; + if (lastline.includes('Error')) { + this.onOperationFail(op, lastline); + } else { + this.onOperationFail(op, `Operation "${opName}" failed!`); + } } } else { // something bad happened... - var err = `Failed to execute operation "${jobId}": ${info.status}`, - consoleErr = `Failed to execute operation: ${info.status}`; + const err = `Failed to execute operation "${jobId}": ${status}`; + const consoleErr = `Failed to execute operation: ${status}`; + this.setAttribute(job, 'stdout', consoleErr); this.logger.error(err); return this.onOperationFail(op, err); } }; - ExecuteJob.prototype.onDistOperationComplete = function (node, result) { + ExecuteJob.prototype.onDistOperationComplete = function (node, fileHashes) { const opName = this.getAttribute(node, 'name'); let nodeId = this.core.getPath(node), outputMap = {}, @@ -792,7 +694,7 @@ define([ // Create an array of [name, node] // For now, just match by type. Later we may use ports for input/outputs // Store the results in the outgoing ports - return this.getResultTypes(result) + return this.getResultTypes(fileHashes) .then(types => { resultTypes = types; return this.getOutputs(node); @@ -804,7 +706,7 @@ define([ // this should not be in directories -> flatten the data! const hashes = outputs.map(tuple => { // [ name, node ] let [name] = tuple; - let artifactHash = result.resultHashes[name]; + let artifactHash = fileHashes[name]; return this.getContentHash(artifactHash, `outputs/${name}`); }); @@ -834,8 +736,8 @@ define([ .catch(e => this.onOperationFail(node, `"${opName}" failed: ${e}`)); }; - ExecuteJob.prototype.getResultTypes = async function (result) { - const artifactHash = result.resultHashes['result-types']; + ExecuteJob.prototype.getResultTypes = async function (fileHashes) { + const artifactHash = fileHashes['result-types']; return await this.getContentHashSafe(artifactHash, 'result-types.json', ERROR.NO_TYPES_FILE); }; @@ -855,7 +757,9 @@ define([ }; //////////////////////////// Special Operations //////////////////////////// - ExecuteJob.prototype.executeLocalOperation = function (type, node) { + ExecuteJob.prototype.executeLocalOperation = function (node) { + const type = this.getLocalOperationType(node); + // Retrieve the given LOCAL_OP type if (!this[type]) { this.logger.error(`No local operation handler for ${type}`); diff --git a/src/plugins/ExecutePipeline/ExecutePipeline.js b/src/plugins/ExecutePipeline/ExecutePipeline.js index 5bf3c4d36..85a89447a 100644 --- a/src/plugins/ExecutePipeline/ExecutePipeline.js +++ b/src/plugins/ExecutePipeline/ExecutePipeline.js @@ -100,22 +100,18 @@ define([ * * @param {function(string, plugin.PluginResult)} callback - the result callback */ - ExecutePipeline.prototype.main = function (callback) { - var startPromise = this.checkExecutionEnv(), - runId; + ExecutePipeline.prototype.main = async function (callback) { this.initRun(); if (!this.META.Pipeline) { return callback(new Error('Incorrect namespace. Expected to be executed in the "pipeline" namespace')); } + if (this.core.isTypeOf(this.activeNode, this.META.Pipeline)) { // If starting with a pipeline, we will create an Execution first - startPromise = startPromise - .then(() => this.createExecution(this.activeNode)) - .then(execNode => { - this.logger.debug(`Finished creating execution "${this.getAttribute(execNode, 'name')}"`); - this.activeNode = execNode; - }); + const execNode = await this.createExecution(this.activeNode); + this.logger.debug(`Finished creating execution "${this.getAttribute(execNode, 'name')}"`); + this.activeNode = execNode; } else if (this.core.isTypeOf(this.activeNode, this.META.Execution)) { this.logger.debug('Restarting execution'); } else { @@ -125,39 +121,31 @@ define([ this._callback = callback; this.currentForkName = null; - startPromise - .then(() => this.core.loadSubTree(this.activeNode)) - .then(subtree => { - var children; - - children = subtree - .filter(n => this.core.getParent(n) === this.activeNode); - - this.pipelineName = this.getAttribute(this.activeNode, 'name'); - this.forkNameBase = this.pipelineName; - this.logger.debug(`Loaded subtree of ${this.pipelineName}. About to build cache`); - this.buildCache(subtree); - this.logger.debug('Parsing execution for job inter-dependencies'); - this.parsePipeline(children); // record deps, etc - - // Detect if resuming execution - runId = this.getAttribute(this.activeNode, 'runId'); - return this.isResuming().then(resuming => { - if (resuming) { - this.currentRunId = runId; - this.startExecHeartBeat(); - return this.resumePipeline(); - } - return this.startPipeline(); - }); + const subtree = await this.core.loadSubTree(this.activeNode); + const children = subtree + .filter(n => this.core.getParent(n) === this.activeNode); + + this.pipelineName = this.getAttribute(this.activeNode, 'name'); + this.forkNameBase = this.pipelineName; + this.logger.debug(`Loaded subtree of ${this.pipelineName}. About to build cache`); + this.buildCache(subtree); + this.logger.debug('Parsing execution for job inter-dependencies'); + this.parsePipeline(children); // record deps, etc + + // Detect if resuming execution + const runId = this.getAttribute(this.activeNode, 'runId'); + const isResuming = await this.isResuming(); + if (isResuming) { + this.currentRunId = runId; + this.startExecHeartBeat(); + return this.resumePipeline(); + } - }) - .fail(err => { - this.logger.error(err); - callback(err, this.result); - }); + return this.startPipeline(); }; + //this.(); + //ExecutePipeline.prototype.isResuming = function () { ExecutePipeline.prototype.isResuming = function () { var currentlyRunning = this.getAttribute(this.activeNode, 'status') === 'running', runId = this.getAttribute(this.activeNode, 'runId'); @@ -230,7 +218,7 @@ define([ .fail(err => this._callback(err)); }; - ExecutePipeline.prototype.startPipeline = function () { + ExecutePipeline.prototype.startPipeline = async function () { var rand = Math.floor(Math.random()*10000), commit = this.commitHash.replace('#', ''); From d109c4a485b1b3119517bf4ea0817b6f6f0c17c4 Mon Sep 17 00:00:00 2001 From: Brian Broll Date: Wed, 11 Sep 2019 12:28:14 -0500 Subject: [PATCH 12/45] WIP Update stop execution to communicate with backend --- src/common/viz/Execute.js | 43 +++++++++------------------------------ 1 file changed, 10 insertions(+), 33 deletions(-) diff --git a/src/common/viz/Execute.js b/src/common/viz/Execute.js index 11e75af0d..4e6f0dc76 100644 --- a/src/common/viz/Execute.js +++ b/src/common/viz/Execute.js @@ -2,7 +2,7 @@ // Mixin for executing jobs and pipelines define([ 'q', - 'executor/ExecutorClient', + 'deepforge/execution/index', 'deepforge/api/ExecPulseClient', 'deepforge/api/JobOriginClient', 'deepforge/Constants', @@ -22,11 +22,7 @@ define([ this.pulseClient = new ExecPulseClient({ logger: this.logger }); - this._executor = new ExecutorClient({ - logger: this.logger.fork('ExecutorClient'), - serverPort: WebGMEGlobal.gmeConfig.server.port, - httpsecure: window.location.protocol === 'https:' - }); + this._executor = new ExecutorClient(this.logger, WebGMEGlobal.gmeConfig); this.originManager = new JobOriginClient({logger: this.logger}); }; @@ -96,19 +92,18 @@ define([ }; Execute.prototype.silentStopJob = function(job) { - var jobHash, - secret; + var jobInfo; job = job || this.client.getNode(this._currentNodeId); - jobHash = job.getAttribute('jobId'); - secret = job.getAttribute('secret'); - if (!jobHash || !secret) { - this.logger.error('Cannot stop job. Missing jobHash or secret'); + try { + jobInfo = JSON.parse(job.getAttribute('jobInfo')); + } catch (err) { + this.logger.error('Cannot stop job. Missing jobInfo.'); return; } - return this._executor.cancelJob(jobHash, secret) - .then(() => this.logger.info(`${jobHash} has been cancelled!`)) + return this._executor.cancelJob(jobInfo) + .then(() => this.logger.info(`${jobInfo.hash} has been cancelled!`)) .fail(err => this.logger.error(`Job cancel failed: ${err}`)); }; @@ -167,7 +162,7 @@ define([ var execNode = this.client.getNode(id || this._currentNodeId); return this.loadChildren(id) - .then(() => this._stopExecution(execNode, inTransaction)); + .then(() => this._silentStopExecution(execNode, inTransaction)); }; Execute.prototype.silentStopExecution = function(id) { @@ -178,24 +173,6 @@ define([ .then(() => this._silentStopExecution(execNode)); }; - Execute.prototype._stopExecution = function(execNode, inTransaction) { - var msg = `Canceling ${execNode.getAttribute('name')} execution`, - jobIds; - - if (!inTransaction) { - this.client.startTransaction(msg); - } - - jobIds = this._silentStopExecution(execNode); - - this.client.setAttribute(execNode.getId(), 'status', 'canceled'); - jobIds.forEach(jobId => this._setJobStopped(jobId, true)); - - if (!inTransaction) { - this.client.completeTransaction(); - } - }; - Execute.prototype._silentStopExecution = function(execNode) { var runningJobIds = execNode.getChildrenIds() .map(id => this.client.getNode(id)) From d829f5b1a9398cdb6e2401ededdd18c8f0a8a368 Mon Sep 17 00:00:00 2001 From: Brian Broll Date: Wed, 11 Sep 2019 12:55:45 -0500 Subject: [PATCH 13/45] WIP rename silentStopExecution --- src/common/viz/Execute.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/common/viz/Execute.js b/src/common/viz/Execute.js index 4e6f0dc76..84a5b78f6 100644 --- a/src/common/viz/Execute.js +++ b/src/common/viz/Execute.js @@ -162,7 +162,7 @@ define([ var execNode = this.client.getNode(id || this._currentNodeId); return this.loadChildren(id) - .then(() => this._silentStopExecution(execNode, inTransaction)); + .then(() => this._stopExecution(execNode, inTransaction)); }; Execute.prototype.silentStopExecution = function(id) { @@ -170,10 +170,10 @@ define([ // Stop the execution w/o setting any attributes return this.loadChildren(id) - .then(() => this._silentStopExecution(execNode)); + .then(() => this._stopExecution(execNode)); }; - Execute.prototype._silentStopExecution = function(execNode) { + Execute.prototype._stopExecution = function(execNode) { var runningJobIds = execNode.getChildrenIds() .map(id => this.client.getNode(id)) .filter(job => this.isRunning(job)); // get running jobs From 315c30efaa324ccec5611a2c38069b73d5ed6103 Mon Sep 17 00:00:00 2001 From: Brian Broll Date: Mon, 16 Sep 2019 11:41:33 -0500 Subject: [PATCH 14/45] WIP #1186 convert dict values to nodes so it will be updated on save --- src/plugins/ExecuteJob/ExecuteJob.js | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/plugins/ExecuteJob/ExecuteJob.js b/src/plugins/ExecuteJob/ExecuteJob.js index dbbc19f13..9598ca9bb 100644 --- a/src/plugins/ExecuteJob/ExecuteJob.js +++ b/src/plugins/ExecuteJob/ExecuteJob.js @@ -100,7 +100,7 @@ define([ this.executor.on( 'data', (id, data) => { - const [job] = this.getNodesForJobHash(id); + const job = this.getNodeForJobId(id); this.onConsoleOutput(job, data.toString()); } ); @@ -187,7 +187,7 @@ define([ ExecuteJob.prototype.onUserCancelDetected = function () { this.logger.info('>>>>> Received Abort. Canceling jobs.'); this.runningJobHashes - .map(hash => this.getNodesForJobHash(hash)[0]) + .map(hash => this.getNodeForJobId(hash)) .map(node => JSON.parse(this.getAttribute(node, 'jobInfo'))) .forEach(jobInfo => this.executor.cancelJob(jobInfo)); }; @@ -451,7 +451,7 @@ define([ ExecuteJob.prototype.createJob = async function (job, opNode, hash) { // Record the job info for the given hash - this._execHashToJobNode[hash] = [job, opNode]; + this._execHashToJobNode[hash] = job; const jobInfo = await this.executor.createJob(hash); this.setAttribute(job, 'jobInfo', JSON.stringify(jobInfo)); if (!this.currentRunId) { @@ -462,10 +462,10 @@ define([ this.startExecHeartBeat(); } - return await this.recordJobOrigin(hash, job); + return await this.recordJobOrigin(jobInfo.hash, job); }; - ExecuteJob.prototype.getNodesForJobHash = function (hash) { + ExecuteJob.prototype.getNodeForJobId = function (hash) { return this._execHashToJobNode[hash]; }; @@ -535,10 +535,9 @@ define([ }; ExecuteJob.prototype.onUpdate = async function (jobInfo, status) { - const [job] = this.getNodesForJobHash(jobInfo.hash); + const job = this.getNodeForJobId(jobInfo.hash); const name = this.getAttribute(job, 'name'); - console.log(`setting status to`, status); this.setAttribute(job, 'status', status); await this.save(`"${name}" operation in ${this.pipelineName} is now "${status}"`); }; @@ -623,7 +622,8 @@ define([ ExecuteJob.prototype.onOperationEnd = async function (hash) { // Record that the job hash is no longer running - const [job, op] = this.getNodesForJobHash(hash); + const job = this.getNodeForJobId(hash); + const op = await this.getOperation(job); const name = this.getAttribute(job, 'name'); const jobId = this.core.getPath(job); const jobInfo = JSON.parse(this.getAttribute(job, 'jobInfo')); From 4e948999c69e0bdcfd8c5cfbc3a1128d9ce0a8c0 Mon Sep 17 00:00:00 2001 From: Brian Broll Date: Mon, 16 Sep 2019 11:44:17 -0500 Subject: [PATCH 15/45] Update cached nodes on SYNCED. Fixes #1226 --- src/plugins/ExecuteJob/ExecuteJob.SafeSave.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/plugins/ExecuteJob/ExecuteJob.SafeSave.js b/src/plugins/ExecuteJob/ExecuteJob.SafeSave.js index 675b2e0a0..3fcdf9525 100644 --- a/src/plugins/ExecuteJob/ExecuteJob.SafeSave.js +++ b/src/plugins/ExecuteJob/ExecuteJob.SafeSave.js @@ -360,8 +360,9 @@ define([ this.logger.info(`Save finished w/ status: ${result.status}`); if (result.status === STORAGE_CONSTANTS.FORKED) { return this.onSaveForked(result.forkName); - } else if (result.status === STORAGE_CONSTANTS.MERGED) { - this.logger.debug('Merged changes. About to update plugin nodes'); + } else if (result.status === STORAGE_CONSTANTS.MERGED || + result.status === STORAGE_CONSTANTS.SYNCED) { + this.logger.debug('Applied changes successfully. About to update plugin nodes'); return this.updateNodes(); } }); From 437e87c8c506efad60438691450ea6837f052bc8 Mon Sep 17 00:00:00 2001 From: Brian Broll Date: Tue, 17 Sep 2019 12:00:50 -0500 Subject: [PATCH 16/45] WIP #1186 expose execution backend as component setting --- config/components.json | 3 +++ src/plugins/ExecuteJob/ExecuteJob.js | 33 +++++++++++++++++++++++----- 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/config/components.json b/config/components.json index 888bff9ac..aa92e027a 100644 --- a/config/components.json +++ b/config/components.json @@ -129,5 +129,8 @@ "description": "Artifacts from pipeline executions are stored here." } ] + }, + "ExecuteJob": { + "executionBackend": "GME" } } diff --git a/src/plugins/ExecuteJob/ExecuteJob.js b/src/plugins/ExecuteJob/ExecuteJob.js index 9598ca9bb..6be55cb26 100644 --- a/src/plugins/ExecuteJob/ExecuteJob.js +++ b/src/plugins/ExecuteJob/ExecuteJob.js @@ -1,4 +1,4 @@ -/*globals define*/ +/*globals define, requirejs*/ /*jshint node:true, browser:true*/ define([ @@ -18,11 +18,12 @@ define([ 'deepforge/utils', 'q', 'superagent', - 'underscore' + 'underscore', + 'module' ], function ( assert, pluginMetadata, - ExecutorClient, + Execution, PluginBase, LocalExecutor, // DeepForge operation primitives PtrCodeGen, @@ -38,10 +39,12 @@ define([ utils, Q, superagent, - _ + _, + module ) { 'use strict'; + const DEFAULT_SETTINGS = {executionBackend: 'GME'}; pluginMetadata = JSON.parse(pluginMetadata); var STDOUT_FILE = 'job_stdout.txt'; @@ -95,8 +98,22 @@ define([ this.pulseClient = new ExecPulseClient(params); this._execHashToJobNode = {}; - // TODO: load a custom executor - this.executor = new ExecutorClient(this.logger, this.gmeConfig); + this.settings = _.extend({}, DEFAULT_SETTINGS); + if (require.isBrowser) { + const ComponentSettings = requirejs('js/Utils/ComponentSettings'); + ComponentSettings.resolveWithWebGMEGlobal( + this.settings, + this.getComponentId() + ); + } else { // Running in NodeJS + const path = require('path'); + const dirname = path.dirname(module.uri); + const deploymentSettings = JSON.parse(requirejs('text!' + dirname + '/../../../config/components.json')); + _.extend(this.settings, deploymentSettings[this.getComponentId()]); + } + + const backend = Execution.getBackend(this.settings.executionBackend); + this.executor = backend.getClient(this.logger); this.executor.on( 'data', (id, data) => { @@ -126,6 +143,10 @@ define([ return result; }; + ExecuteJob.prototype.getComponentId = function () { + return 'ExecuteJob'; + }; + /** * Main function for the plugin to execute. This will perform the execution. * Notes: From 039669bec9f903314041204aba8bb76a19d28054 Mon Sep 17 00:00:00 2001 From: Brian Broll Date: Tue, 17 Sep 2019 14:02:45 -0500 Subject: [PATCH 17/45] WIP #1186 Fixed indentation --- src/plugins/ExecuteJob/ExecuteJob.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/plugins/ExecuteJob/ExecuteJob.js b/src/plugins/ExecuteJob/ExecuteJob.js index 6be55cb26..613046830 100644 --- a/src/plugins/ExecuteJob/ExecuteJob.js +++ b/src/plugins/ExecuteJob/ExecuteJob.js @@ -123,11 +123,11 @@ define([ ); this.executor.on('update', (jobInfo, status) => { - try { - this.onUpdate(jobInfo, status); - } catch (err) { - this.logger.error(`Error when processing operation update: ${err}`); - } + try { + this.onUpdate(jobInfo, status); + } catch (err) { + this.logger.error(`Error when processing operation update: ${err}`); + } }); this.executor.on('end', From 8b7c468bd9f4b28d3e481302215b9f9ccd5163aa Mon Sep 17 00:00:00 2001 From: Brian Broll Date: Tue, 17 Sep 2019 14:03:05 -0500 Subject: [PATCH 18/45] WIP rename "View workers" -> "View compute" --- .../panels/WorkerHeader/ProjectNavigatorController.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/visualizers/panels/WorkerHeader/ProjectNavigatorController.js b/src/visualizers/panels/WorkerHeader/ProjectNavigatorController.js index 1150050ed..9413a5272 100644 --- a/src/visualizers/panels/WorkerHeader/ProjectNavigatorController.js +++ b/src/visualizers/panels/WorkerHeader/ProjectNavigatorController.js @@ -77,7 +77,7 @@ define([ }, { id: 'manageWorkers', - label: 'View workers ...', + label: 'View compute ...', iconClass: 'glyphicon glyphicon-cloud', action: manageWorkers } From ea1c7e5d9128142fb04b090b5de5e17997b3ff67 Mon Sep 17 00:00:00 2001 From: Brian Broll Date: Tue, 17 Sep 2019 14:17:32 -0500 Subject: [PATCH 19/45] WIP #1186 Moved backend whitelisting to execution --- config/components.json | 4 +-- src/common/execution/index.js | 51 ++++++++++++++++++++++++---- src/plugins/ExecuteJob/ExecuteJob.js | 17 ++-------- 3 files changed, 48 insertions(+), 24 deletions(-) diff --git a/config/components.json b/config/components.json index aa92e027a..7eb461c07 100644 --- a/config/components.json +++ b/config/components.json @@ -130,7 +130,7 @@ } ] }, - "ExecuteJob": { - "executionBackend": "GME" + "Compute": { + "backends": ["local", "gme"] } } diff --git a/src/common/execution/index.js b/src/common/execution/index.js index fb2477342..9bf417d94 100644 --- a/src/common/execution/index.js +++ b/src/common/execution/index.js @@ -1,11 +1,48 @@ +/*globals define, requirejs */ define([ - './backends/GME', - './backends/Local' + 'underscore', + 'module', ], function( - GME, - Local + _, + module ) { - // FIXME: Add more intelligent interface here... - // - fetch a given backend and configure - return Local; + const Execution = {}; + const BACKENDS = ['gme', 'local']; + + Execution.getBackend = function(name) { + name = name.toLowerCase(); + if (!BACKENDS.includes(name)) { + throw new Error(`Execution backend not found: ${name}`); + } + + const relativePath = `backends/${name}/index`; + const Backend = requirejs(`deepforge/execution/${relativePath}`); + return new Backend(); + }; + + Execution.getAvailableBackends = function() { + const settings = {backends: ['local', 'gme']}; + if (require.isBrowser) { + const ComponentSettings = requirejs('js/Utils/ComponentSettings'); + ComponentSettings.resolveWithWebGMEGlobal( + settings, + this.getComponentId() + ); + } else { // Running in NodeJS + const path = require('path'); + const dirname = path.dirname(module.uri); + const deploymentSettings = JSON.parse(requirejs('text!' + dirname + '/../../../config/components.json')); + _.extend(settings, deploymentSettings[this.getComponentId()]); + } + + return settings.backends; + }; + + Execution.getComponentId = function() { + return 'Compute'; + }; + + //Execution.getProjectRoot = function() { + + return Execution; }); diff --git a/src/plugins/ExecuteJob/ExecuteJob.js b/src/plugins/ExecuteJob/ExecuteJob.js index 613046830..1f4aad652 100644 --- a/src/plugins/ExecuteJob/ExecuteJob.js +++ b/src/plugins/ExecuteJob/ExecuteJob.js @@ -98,21 +98,8 @@ define([ this.pulseClient = new ExecPulseClient(params); this._execHashToJobNode = {}; - this.settings = _.extend({}, DEFAULT_SETTINGS); - if (require.isBrowser) { - const ComponentSettings = requirejs('js/Utils/ComponentSettings'); - ComponentSettings.resolveWithWebGMEGlobal( - this.settings, - this.getComponentId() - ); - } else { // Running in NodeJS - const path = require('path'); - const dirname = path.dirname(module.uri); - const deploymentSettings = JSON.parse(requirejs('text!' + dirname + '/../../../config/components.json')); - _.extend(this.settings, deploymentSettings[this.getComponentId()]); - } - - const backend = Execution.getBackend(this.settings.executionBackend); + const name = Execution.getAvailableBackends()[0]; // FIXME: enable the user to select one + const backend = Execution.getBackend(name); this.executor = backend.getClient(this.logger); this.executor.on( 'data', From 4ad7824eb9ec25c600e3bb6576a96a1f86e93925 Mon Sep 17 00:00:00 2001 From: Brian Broll Date: Tue, 17 Sep 2019 14:18:54 -0500 Subject: [PATCH 20/45] WIP minor cleanup --- src/common/execution/index.js | 2 -- src/plugins/ExecuteJob/ExecuteJob.SafeSave.js | 1 - src/plugins/ExecuteJob/ExecuteJob.js | 3 --- 3 files changed, 6 deletions(-) diff --git a/src/common/execution/index.js b/src/common/execution/index.js index 9bf417d94..b68f2319f 100644 --- a/src/common/execution/index.js +++ b/src/common/execution/index.js @@ -42,7 +42,5 @@ define([ return 'Compute'; }; - //Execution.getProjectRoot = function() { - return Execution; }); diff --git a/src/plugins/ExecuteJob/ExecuteJob.SafeSave.js b/src/plugins/ExecuteJob/ExecuteJob.SafeSave.js index 3fcdf9525..324ed34f0 100644 --- a/src/plugins/ExecuteJob/ExecuteJob.SafeSave.js +++ b/src/plugins/ExecuteJob/ExecuteJob.SafeSave.js @@ -389,7 +389,6 @@ define([ this.rootNode = await this.core.loadRoot(commitObject.root); this.activeNode = await this.core.loadByPath(this.rootNode, activeId); - // TODO: Can I refactor this? await this.updateExistingNodeDict(this.META); await this.updateExistingNodeDict(this._execHashToJobNode); diff --git a/src/plugins/ExecuteJob/ExecuteJob.js b/src/plugins/ExecuteJob/ExecuteJob.js index 1f4aad652..2732b6c4f 100644 --- a/src/plugins/ExecuteJob/ExecuteJob.js +++ b/src/plugins/ExecuteJob/ExecuteJob.js @@ -19,7 +19,6 @@ define([ 'q', 'superagent', 'underscore', - 'module' ], function ( assert, pluginMetadata, @@ -40,11 +39,9 @@ define([ Q, superagent, _, - module ) { 'use strict'; - const DEFAULT_SETTINGS = {executionBackend: 'GME'}; pluginMetadata = JSON.parse(pluginMetadata); var STDOUT_FILE = 'job_stdout.txt'; From e6ec47187e3cbe617a5f2d944a8ed5dfb2814661 Mon Sep 17 00:00:00 2001 From: Brian Broll Date: Tue, 17 Sep 2019 14:19:14 -0500 Subject: [PATCH 21/45] WIP #1186 Refactored backends for client and status info --- src/common/execution/backends/BaseBackend.js | 29 ++++ src/common/execution/backends/BaseExecutor.js | 132 +++++++++--------- .../backends/{GME.js => gme/Client.js} | 0 src/common/execution/backends/gme/index.js | 15 ++ .../backends/{Local.js => local/Client.js} | 16 ++- src/common/execution/backends/local/index.js | 15 ++ 6 files changed, 132 insertions(+), 75 deletions(-) create mode 100644 src/common/execution/backends/BaseBackend.js rename src/common/execution/backends/{GME.js => gme/Client.js} (100%) create mode 100644 src/common/execution/backends/gme/index.js rename src/common/execution/backends/{Local.js => local/Client.js} (97%) create mode 100644 src/common/execution/backends/local/index.js diff --git a/src/common/execution/backends/BaseBackend.js b/src/common/execution/backends/BaseBackend.js new file mode 100644 index 000000000..e01e01ed5 --- /dev/null +++ b/src/common/execution/backends/BaseBackend.js @@ -0,0 +1,29 @@ +/* globals define, requirejs */ +define([ + 'module' +], function( + module +) { + + const BaseBackend = function(name) { + console.log(`Creating backend: ${name}`); + this.name = name; + }; + + BaseBackend.prototype.getClient = function(logger) { + const path = require('path'); + const dirname = path.dirname(module.uri); + const Client = requirejs(`${dirname}/${this.name.toLowerCase()}/Client.js`); + return new Client(logger); + }; + + BaseBackend.prototype.getOptions = function() { + return []; + }; + + BaseBackend.prototype.getDashboard = function() { + return null; + }; + + return BaseBackend; +}); diff --git a/src/common/execution/backends/BaseExecutor.js b/src/common/execution/backends/BaseExecutor.js index 6fe8154fd..1da69e46e 100644 --- a/src/common/execution/backends/BaseExecutor.js +++ b/src/common/execution/backends/BaseExecutor.js @@ -1,79 +1,75 @@ -define([ -], function( -) { - const BaseExecutor = function(logger, gmeConfig) { - const isHttps = typeof window === 'undefined' ? false : - window.location.protocol !== 'http:'; +const BaseExecutor = function(logger, gmeConfig) { + const isHttps = typeof window === 'undefined' ? false : + window.location.protocol !== 'http:'; - this.logger = logger.fork('executor'); - this._events = {}; - }; + this.logger = logger.fork('executor'); + this._events = {}; +}; - BaseExecutor.prototype.cancelJob = function(job) { - const msg = `cancelJob is not implemented for current executor backend!`; - this.logger.warn(msg); - throw new Error(msg); - }; +BaseExecutor.prototype.cancelJob = function(job) { + const msg = `cancelJob is not implemented for current executor backend!`; + this.logger.warn(msg); + throw new Error(msg); +}; - BaseExecutor.prototype.getInfo = function(job) { - const msg = `getInfo is not implemented for current executor backend!`; - this.logger.warn(msg); - throw new Error(msg); - }; +BaseExecutor.prototype.getInfo = function(job) { + const msg = `getInfo is not implemented for current executor backend!`; + this.logger.warn(msg); + throw new Error(msg); +}; - BaseExecutor.prototype.createJob = async function(hash) { - const msg = `createJob is not implemented for current executor backend!`; - this.logger.warn(msg); - throw new Error(msg); - }; +BaseExecutor.prototype.createJob = async function(hash) { + const msg = `createJob is not implemented for current executor backend!`; + this.logger.warn(msg); + throw new Error(msg); +}; - BaseExecutor.prototype.getStatus = async function(jobInfo) { - const msg = `getStatus is not implemented for current executor backend!`; - this.logger.warn(msg); - throw new Error(msg); - }; +BaseExecutor.prototype.getStatus = async function(jobInfo) { + const msg = `getStatus is not implemented for current executor backend!`; + this.logger.warn(msg); + throw new Error(msg); +}; - BaseExecutor.prototype.getOutputHashes = async function(jobInfo) { - const msg = `getOutputHashes is not implemented for current executor backend!`; - this.logger.warn(msg); - throw new Error(msg); - }; +BaseExecutor.prototype.getOutputHashes = async function(jobInfo) { + const msg = `getOutputHashes is not implemented for current executor backend!`; + this.logger.warn(msg); + throw new Error(msg); +}; - // TODO: Should I remove this? - BaseExecutor.prototype.getConsoleOutput = async function(hash) { - const msg = `getConsoleOutput is not implemented for current executor backend!`; - this.logger.warn(msg); - throw new Error(msg); - }; +// TODO: Should I remove this? +BaseExecutor.prototype.getConsoleOutput = async function(hash) { + const msg = `getConsoleOutput is not implemented for current executor backend!`; + this.logger.warn(msg); + throw new Error(msg); +}; - // Some functions for event support - BaseExecutor.prototype.on = function(ev, cb) { - this._events[ev] = this._events[ev] || []; - this._events[ev].push(cb); - }; +// Some functions for event support +BaseExecutor.prototype.on = function(ev, cb) { + this._events[ev] = this._events[ev] || []; + this._events[ev].push(cb); +}; - BaseExecutor.prototype.emit = function(ev) { - const args = Array.prototype.slice.call(arguments, 1); - console.log('emitting'); - args.forEach(a => { - if (a instanceof Buffer) { - console.log(a.toString()); - } else { - console.log(a); - } - }); - const handlers = this._events[ev] || []; - handlers.forEach(fn => fn.apply(this, args)); - }; +BaseExecutor.prototype.emit = function(ev) { + const args = Array.prototype.slice.call(arguments, 1); + console.log('emitting'); + args.forEach(a => { + if (a instanceof Buffer) { + console.log(a.toString()); + } else { + console.log(a); + } + }); + const handlers = this._events[ev] || []; + handlers.forEach(fn => fn.apply(this, args)); +}; - // TODO: Make these match the values in the model (`status` enum on pipeline.Job) - BaseExecutor.prototype.QUEUED = 'queued'; - BaseExecutor.prototype.PENDING = 'pending'; - BaseExecutor.prototype.RUNNING = 'running'; - BaseExecutor.prototype.SUCCESS = 'success'; - BaseExecutor.prototype.FAILED = 'failed'; - BaseExecutor.prototype.CANCELED = 'canceled'; - BaseExecutor.prototype.NOT_FOUND = 'NOT_FOUND'; +// TODO: Make these match the values in the model (`status` enum on pipeline.Job) +BaseExecutor.prototype.QUEUED = 'queued'; +BaseExecutor.prototype.PENDING = 'pending'; +BaseExecutor.prototype.RUNNING = 'running'; +BaseExecutor.prototype.SUCCESS = 'success'; +BaseExecutor.prototype.FAILED = 'failed'; +BaseExecutor.prototype.CANCELED = 'canceled'; +BaseExecutor.prototype.NOT_FOUND = 'NOT_FOUND'; - return BaseExecutor; -}); +module.exports = BaseExecutor; diff --git a/src/common/execution/backends/GME.js b/src/common/execution/backends/gme/Client.js similarity index 100% rename from src/common/execution/backends/GME.js rename to src/common/execution/backends/gme/Client.js diff --git a/src/common/execution/backends/gme/index.js b/src/common/execution/backends/gme/index.js new file mode 100644 index 000000000..fed872790 --- /dev/null +++ b/src/common/execution/backends/gme/index.js @@ -0,0 +1,15 @@ +/* globals define */ +define([ + '../BaseBackend' +], function( + BaseBackend +) { + + const GMEBackend = function() { + BaseBackend.call(this, 'GME'); + }; + + GMEBackend.prototype = Object.create(BaseBackend.prototype); + + return GMEBackend; +}); diff --git a/src/common/execution/backends/Local.js b/src/common/execution/backends/local/Client.js similarity index 97% rename from src/common/execution/backends/Local.js rename to src/common/execution/backends/local/Client.js index 7875dd7bb..a5e939706 100644 --- a/src/common/execution/backends/Local.js +++ b/src/common/execution/backends/local/Client.js @@ -1,5 +1,7 @@ +/*globals define*/ +// TODO: Show an error if not running on the server... define([ - './BaseExecutor', + '../BaseExecutor', 'blob/BlobClient', 'child_process', 'minimatch', @@ -19,8 +21,6 @@ define([ os, path, ) { - // TODO: Show an error if not running on the server... - const spawn = childProcess.spawn; const {promisify} = require.nodeRequire('util'); const mkdir = promisify(fs.mkdir); @@ -38,19 +38,21 @@ define([ ensureHasUnzip(); const UNZIP_EXE = '/usr/bin/unzip'; // FIXME: more platform support const UNZIP_ARGS = ['-o']; // FIXME: more platform support - const PROJECT_ROOT = path.join(path.dirname(module.uri), '..', '..', '..', '..'); + const PROJECT_ROOT = path.join(path.dirname(module.uri), '..', '..', '..', '..', '..'); const NODE_MODULES = path.join(PROJECT_ROOT, 'node_modules'); // TODO const symlink = promisify(fs.symlink); const touch = async name => await closeFile(await openFile(name, 'w')); - const LocalExecutor = function(logger, gmeConfig) { + const LocalExecutor = function(/*logger*/) { BaseExecutor.apply(this, arguments); + + const configPath = path.join(PROJECT_ROOT, 'config'); + const gmeConfig = require.nodeRequire(configPath); this.completedJobs = {}; this.jobQueue = []; this.currentJob = null; this.subprocess = null; this.canceled = false; - // FIXME: set this meaningfully! this.blobClient = new BlobClient({ server: '127.0.0.1', serverPort: gmeConfig.server.port, @@ -64,7 +66,7 @@ define([ LocalExecutor.prototype.cancelJob = function(jobInfo) { const {hash} = jobInfo; - console.log('>>> CANCELING job!!'); + console.log('>>> CANCELING job!!', hash, this.currentJob, this.jobQueue); if (this.currentJob === hash) { this.canceled = true; this.subprocess.kill(); diff --git a/src/common/execution/backends/local/index.js b/src/common/execution/backends/local/index.js new file mode 100644 index 000000000..64f8c74ac --- /dev/null +++ b/src/common/execution/backends/local/index.js @@ -0,0 +1,15 @@ +/* globals define */ +define([ + '../BaseBackend' +], function( + BaseBackend +) { + + const LocalBackend = function() { + BaseBackend.call(this, 'Local'); + }; + + LocalBackend.prototype = Object.create(BaseBackend.prototype); + + return LocalBackend; +}); From e17beb8948f5344b52c1563836511833713b9879 Mon Sep 17 00:00:00 2001 From: Brian Broll Date: Tue, 17 Sep 2019 15:25:07 -0500 Subject: [PATCH 22/45] WIP #1186 refactored JobResults --- src/common/execution/backends/JobResults.js | 10 ++++++++++ src/common/execution/backends/local/Client.js | 9 ++------- 2 files changed, 12 insertions(+), 7 deletions(-) create mode 100644 src/common/execution/backends/JobResults.js diff --git a/src/common/execution/backends/JobResults.js b/src/common/execution/backends/JobResults.js new file mode 100644 index 000000000..187c6a95a --- /dev/null +++ b/src/common/execution/backends/JobResults.js @@ -0,0 +1,10 @@ +const BaseExecutor = require('./BaseExecutor'); + +class JobResults { + constructor(status=BaseExecutor.prototype.CREATED) { + this.status = status; + this.resultHashes = []; + } +} + +module.exports = JobResults; diff --git a/src/common/execution/backends/local/Client.js b/src/common/execution/backends/local/Client.js index a5e939706..758b23aae 100644 --- a/src/common/execution/backends/local/Client.js +++ b/src/common/execution/backends/local/Client.js @@ -2,6 +2,7 @@ // TODO: Show an error if not running on the server... define([ '../BaseExecutor', + '../JobResults', 'blob/BlobClient', 'child_process', 'minimatch', @@ -12,6 +13,7 @@ define([ 'path', ], function( BaseExecutor, + JobResults, BlobClient, childProcess, minimatch, @@ -375,12 +377,5 @@ define([ } // - [ ] emit updates on stdout... - class JobResults { - constructor(status) { - this.status = status || LocalExecutor.prototype.CREATED; - this.resultHashes = []; - } - } - return LocalExecutor; }); From 6b107e37996297a975fdaf392aa14cf380f062b8 Mon Sep 17 00:00:00 2001 From: Brian Broll Date: Tue, 17 Sep 2019 15:54:11 -0500 Subject: [PATCH 23/45] WIP Updated 'update' event to accept jobId (not jobInfo) --- src/common/execution/backends/local/Client.js | 6 +++--- src/plugins/ExecuteJob/ExecuteJob.js | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/common/execution/backends/local/Client.js b/src/common/execution/backends/local/Client.js index 758b23aae..6cf6b2b88 100644 --- a/src/common/execution/backends/local/Client.js +++ b/src/common/execution/backends/local/Client.js @@ -121,7 +121,7 @@ define([ } this.completedJobs[hash] = jobResults; - this.emit('update', {hash}, jobResults.status); + this.emit('update', hash, jobResults.status); this.emit('end', hash, jobResults); this._processNextJob(); }; @@ -142,7 +142,7 @@ define([ LocalExecutor.prototype._createJob = async function(hash) { // Create tmp directory const jobInfo = {hash}; - this.emit('update', jobInfo, this.PENDING); + this.emit('update', jobInfo.hash, this.PENDING); const tmpdir = this._getWorkingDir(hash); try { await mkdir(tmpdir); @@ -165,7 +165,7 @@ define([ const env = {cwd: tmpdir}; this.logger.info(`Running ${config.cmd} ${config.args.join(' ')}`); this.subprocess = spawn(config.cmd, config.args, env); - this.emit('update', jobInfo, this.RUNNING); + this.emit('update', jobInfo.hash, this.RUNNING); this.subprocess.stdout.on('data', data => this.onConsoleOutput(tmpdir, hash, data)); this.subprocess.on('close', async code => { diff --git a/src/plugins/ExecuteJob/ExecuteJob.js b/src/plugins/ExecuteJob/ExecuteJob.js index 2732b6c4f..7b22108c5 100644 --- a/src/plugins/ExecuteJob/ExecuteJob.js +++ b/src/plugins/ExecuteJob/ExecuteJob.js @@ -106,9 +106,9 @@ define([ } ); - this.executor.on('update', (jobInfo, status) => { + this.executor.on('update', (jobId, status) => { try { - this.onUpdate(jobInfo, status); + this.onUpdate(jobId, status); } catch (err) { this.logger.error(`Error when processing operation update: ${err}`); } @@ -539,8 +539,8 @@ define([ }); }; - ExecuteJob.prototype.onUpdate = async function (jobInfo, status) { - const job = this.getNodeForJobId(jobInfo.hash); + ExecuteJob.prototype.onUpdate = async function (jobId, status) { + const job = this.getNodeForJobId(jobId); const name = this.getAttribute(job, 'name'); this.setAttribute(job, 'status', status); From 475b3c86f5fc66447823b4fea84521aa4534b7d1 Mon Sep 17 00:00:00 2001 From: Brian Broll Date: Tue, 17 Sep 2019 15:54:45 -0500 Subject: [PATCH 24/45] WIP #1186 update gme compute backend compute --- src/common/execution/backends/gme/Client.js | 112 +++++++++++--------- 1 file changed, 63 insertions(+), 49 deletions(-) diff --git a/src/common/execution/backends/gme/Client.js b/src/common/execution/backends/gme/Client.js index 97db8444b..5b41eec1e 100644 --- a/src/common/execution/backends/gme/Client.js +++ b/src/common/execution/backends/gme/Client.js @@ -1,25 +1,33 @@ +/* globals define */ define([ + '../BaseExecutor', + '../JobResults', 'deepforge/ExecutionEnv', - 'executor/ExecutorClient' + 'executor/ExecutorClient', + 'path', + 'module', ], function( + BaseExecutor, + JobResults, ExecutionEnv, - ExecutorClient + ExecutorClient, + path, + module, ) { - // TODO - const GMEExecutor = function(logger, gmeConfig) { - const isHttps = typeof window === 'undefined' ? false : - window.location.protocol !== 'http:'; - - this.logger = logger.fork('GME'); + const PROJECT_ROOT = path.join(path.dirname(module.uri), '..', '..', '..', '..', '..'); + const GMEExecutor = function(/*logger*/) { + BaseExecutor.apply(this, arguments); + const configPath = path.join(PROJECT_ROOT, 'config'); + const gmeConfig = require.nodeRequire(configPath); this.pollInterval = 1500; + this.previousGMEInfo = {}; this.executor = new ExecutorClient({ logger: this.logger, serverPort: gmeConfig.server.port, - httpsecure: isHttps + httpsecure: false }); - - this._events = {}; // FIXME: there must be a better way... }; + GMEExecutor.prototype = Object.create(BaseExecutor.prototype); GMEExecutor.prototype.getConsoleOutput = async function(hash) { return (await this.executor.getOutput(hash)) @@ -31,25 +39,28 @@ define([ }; GMEExecutor.prototype.getOutputHashes = async function(job) { - return (await this.executor.getInfo(job)).resultHashes; + return (await this.executor.getInfo(job.hash)).resultHashes; }; - // TODO: Standardize this GMEExecutor.prototype.getStatus = async function(job) { - // TODO: Convert the status to the appropriate code + const info = await this.executor.getInfo(job.hash); + return this.getJobResultsFrom(info).status; }; - GMEExecutor.prototype.getInfo = function(job) { - return this.executor.getInfo(job.hash); + GMEExecutor.prototype.getJobResultsFrom = function(gmeInfo) { + const gmeStatus = gmeInfo.status; + const gmeStatusToStatus = { + 'CREATED': this.QUEUED, + 'SUCCESS': this.SUCCESS, + 'CANCELED': this.CANCELED, + 'FAILED_TO_EXECUTE': this.FAILED, + 'RUNNING': this.RUNNING, + }; + return new JobResults(gmeStatusToStatus[gmeStatus] || gmeStatus); }; - GMEExecutor.prototype.checkExecutionEnv = async function () { - this.logger.info(`Checking execution environment`); - const workers = await ExecutionEnv.getWorkers(); - if (workers.length === 0) { - this.logger.info(`Cannot execute job(s): No connected workers`); - throw new Error('No connected workers'); - } + GMEExecutor.prototype.getInfo = function(job) { + return this.executor.getInfo(job.hash); }; GMEExecutor.prototype.createJob = async function(hash) { @@ -57,44 +68,47 @@ define([ const result = await this.executor.createJob({hash}); - this.startPolling(hash); - // When to stop polling? - // TODO + this.poll(hash); return result; }; - GMEExecutor.prototype.on = function(ev, cb) { - this._events[ev] = this._events[ev] || []; - this._events[ev].push(cb); - }; - - GMEExecutor.prototype.emit = function(ev) { - const args = Array.prototype.slice.call(arguments, 1); - const handlers = this._events[ev] || []; - handlers.forEach(fn => fn.apply(this, args)); + GMEExecutor.prototype.checkExecutionEnv = async function () { + this.logger.info(`Checking execution environment`); + const workers = await ExecutionEnv.getWorkers(); + if (workers.length === 0) { + this.logger.info(`Cannot execute job(s): No connected workers`); + throw new Error('No connected workers'); + } }; - GMEExecutor.prototype.startPolling = async function(id) { - const info = await this.executor.getInfo(id); + GMEExecutor.prototype.poll = async function(id) { + const gmeInfo = await this.executor.getInfo(id); // Check for new stdout. Emit 'data' with the content - // TODO + const prevInfo = this.previousGMEInfo[id] || {}; + const currentLine = prevInfo.outputNumber + 1; + const actualLine = gmeInfo.outputNumber; + if (actualLine !== null && actualLine >= currentLine) { + const stdout = (await this.executor.getOutput(id, currentLine, actualLine + 1)) + .map(o => o.output).join(''); + this.emit('data', id, stdout); + } - if (info.status === 'CREATED' || info.status === 'RUNNING') { - setTimeout(() => this.startPolling(id), this.pollInterval); + if (gmeInfo.status !== prevInfo.status) { + const results = this.getJobResultsFrom(gmeInfo); + this.emit('update', id, results.status); + } + + this.previousGMEInfo[id] = gmeInfo; + if (gmeInfo.status === 'CREATED' || gmeInfo.status === 'RUNNING') { + setTimeout(() => this.poll(id), this.pollInterval); } else { - this.emit('end', id, info); + const results = this.getJobResultsFrom(gmeInfo); + this.emit('end', id, results); + delete this.previousGMEInfo[id]; } }; - // What is the API for the executor? - // It should "push" the data to the client - // - createJob - // - cancelJob - // - getInfo - // - getOutput - // TODO - return GMEExecutor; }); From 9f4ec7cc4fd397813895d126b90336b26bafca6c Mon Sep 17 00:00:00 2001 From: Brian Broll Date: Tue, 17 Sep 2019 15:55:55 -0500 Subject: [PATCH 25/45] WIP #1186 misc code clean --- src/common/execution/backends/BaseExecutor.js | 8 -------- src/common/execution/backends/local/Client.js | 2 -- src/plugins/ExecutePipeline/ExecutePipeline.js | 2 -- 3 files changed, 12 deletions(-) diff --git a/src/common/execution/backends/BaseExecutor.js b/src/common/execution/backends/BaseExecutor.js index 1da69e46e..73ebac6b9 100644 --- a/src/common/execution/backends/BaseExecutor.js +++ b/src/common/execution/backends/BaseExecutor.js @@ -51,14 +51,6 @@ BaseExecutor.prototype.on = function(ev, cb) { BaseExecutor.prototype.emit = function(ev) { const args = Array.prototype.slice.call(arguments, 1); - console.log('emitting'); - args.forEach(a => { - if (a instanceof Buffer) { - console.log(a.toString()); - } else { - console.log(a); - } - }); const handlers = this._events[ev] || []; handlers.forEach(fn => fn.apply(this, args)); }; diff --git a/src/common/execution/backends/local/Client.js b/src/common/execution/backends/local/Client.js index 6cf6b2b88..3e0decb14 100644 --- a/src/common/execution/backends/local/Client.js +++ b/src/common/execution/backends/local/Client.js @@ -68,7 +68,6 @@ define([ LocalExecutor.prototype.cancelJob = function(jobInfo) { const {hash} = jobInfo; - console.log('>>> CANCELING job!!', hash, this.currentJob, this.jobQueue); if (this.currentJob === hash) { this.canceled = true; this.subprocess.kill(); @@ -88,7 +87,6 @@ define([ } else if (this.completedJobs[hash]) { return this.completedJobs[hash].status; } else { - console.log(`Could not find ${hash} in`, this.completedJobs); throw new Error('Job Not Found'); } }; diff --git a/src/plugins/ExecutePipeline/ExecutePipeline.js b/src/plugins/ExecutePipeline/ExecutePipeline.js index 85a89447a..abaf83209 100644 --- a/src/plugins/ExecutePipeline/ExecutePipeline.js +++ b/src/plugins/ExecutePipeline/ExecutePipeline.js @@ -144,8 +144,6 @@ define([ return this.startPipeline(); }; - //this.(); - //ExecutePipeline.prototype.isResuming = function () { ExecutePipeline.prototype.isResuming = function () { var currentlyRunning = this.getAttribute(this.activeNode, 'status') === 'running', runId = this.getAttribute(this.activeNode, 'runId'); From f4de4e582a559398ae0048ab889285f9e678e8a3 Mon Sep 17 00:00:00 2001 From: Brian Broll Date: Wed, 18 Sep 2019 10:56:37 -0500 Subject: [PATCH 26/45] WIP #1186 ensure compute backends can be loaded synchronously --- src/common/execution/index.js | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/common/execution/index.js b/src/common/execution/index.js index b68f2319f..3bf463401 100644 --- a/src/common/execution/index.js +++ b/src/common/execution/index.js @@ -1,17 +1,19 @@ /*globals define, requirejs */ +const COMPUTE_BACKENDS = ['gme', 'local']; + define([ 'underscore', - 'module', -], function( + 'module' +].concat(COMPUTE_BACKENDS.map(name => `deepforge/execution/backends/${name}/index`)), +function( _, module ) { const Execution = {}; - const BACKENDS = ['gme', 'local']; Execution.getBackend = function(name) { name = name.toLowerCase(); - if (!BACKENDS.includes(name)) { + if (!COMPUTE_BACKENDS.includes(name)) { throw new Error(`Execution backend not found: ${name}`); } From 4a39da1de7884c66d1e97621eb2d41c83030ce73 Mon Sep 17 00:00:00 2001 From: Brian Broll Date: Wed, 18 Sep 2019 10:57:05 -0500 Subject: [PATCH 27/45] WIP Created basic ComputeDialog for compute dashboards --- .../panels/WorkerHeader/ComputeDialog.js | 44 +++++++++++++++++++ .../panels/WorkerHeader/ComputeModal.css | 17 +++++++ .../panels/WorkerHeader/ComputeModal.html | 13 ++++++ .../panels/WorkerHeader/EmptyDashboard.css | 6 +++ .../panels/WorkerHeader/EmptyDashboard.js | 19 ++++++++ .../ProjectNavigatorController.js | 17 ++++--- 6 files changed, 107 insertions(+), 9 deletions(-) create mode 100644 src/visualizers/panels/WorkerHeader/ComputeDialog.js create mode 100644 src/visualizers/panels/WorkerHeader/ComputeModal.css create mode 100644 src/visualizers/panels/WorkerHeader/ComputeModal.html create mode 100644 src/visualizers/panels/WorkerHeader/EmptyDashboard.css create mode 100644 src/visualizers/panels/WorkerHeader/EmptyDashboard.js diff --git a/src/visualizers/panels/WorkerHeader/ComputeDialog.js b/src/visualizers/panels/WorkerHeader/ComputeDialog.js new file mode 100644 index 000000000..1cdb188f0 --- /dev/null +++ b/src/visualizers/panels/WorkerHeader/ComputeDialog.js @@ -0,0 +1,44 @@ +/* globals define, $ */ +define([ + 'deepforge/execution/index', + 'q', + 'deepforge/viz/Utils', + './EmptyDashboard', + 'text!./ComputeModal.html', + 'css!./ComputeModal.css' +], function( + Execution, + Q, + utils, + EmptyDashboard, + ComputeHtml, +) { + 'use strict'; + + const ComputeDialog = function(logger) { + this.active = false; + this.logger = logger.fork('ComputeDialog'); + this.$el = $(ComputeHtml); + // TODO: Handle this differently if there are multiple dashboards? + this.$content = this.$el.find('.dashboard-content'); + this.dashboards = Execution.getAvailableBackends().slice(0, 1) // FIXME + .map(name => Execution.getBackend(name).getDashboard() || EmptyDashboard.bind(null, name)) + .map(ctor => new ctor(this.$content)); + }; + + ComputeDialog.prototype.initialize = function() { + this.$el.modal('show'); + this.$el.on('hidden.bs.modal', () => this.onHide()); + }; + + ComputeDialog.prototype.show = function() { + this.initialize(); + this.dashboards.forEach(dashboard => dashboard.onShow()); + }; + + ComputeDialog.prototype.onHide = function() { + this.dashboards.forEach(dashboard => dashboard.onHide()); + }; + + return ComputeDialog; +}); diff --git a/src/visualizers/panels/WorkerHeader/ComputeModal.css b/src/visualizers/panels/WorkerHeader/ComputeModal.css new file mode 100644 index 000000000..7b75373ff --- /dev/null +++ b/src/visualizers/panels/WorkerHeader/ComputeModal.css @@ -0,0 +1,17 @@ +.compute-modal .modal-content .modal-header span { + font-size: 28px; + vertical-align: middle; +} + +.compute-modal .modal-content .modal-header .header-icon { + height: 28px; + width: 28px; + margin-right: 1ex; + display: inline-block; + vertical-align: middle; + background-size: 28px 28px; +} + +.compute-modal .modal-content .modal-body th { + text-align: left; +} diff --git a/src/visualizers/panels/WorkerHeader/ComputeModal.html b/src/visualizers/panels/WorkerHeader/ComputeModal.html new file mode 100644 index 000000000..5c2d52e1f --- /dev/null +++ b/src/visualizers/panels/WorkerHeader/ComputeModal.html @@ -0,0 +1,13 @@ + diff --git a/src/visualizers/panels/WorkerHeader/EmptyDashboard.css b/src/visualizers/panels/WorkerHeader/EmptyDashboard.css new file mode 100644 index 000000000..068d964b0 --- /dev/null +++ b/src/visualizers/panels/WorkerHeader/EmptyDashboard.css @@ -0,0 +1,6 @@ +.compute-modal .empty-dashboard { + font-style: italic; + font-size: 1.3em; + color: #777; + text-align: center; +} diff --git a/src/visualizers/panels/WorkerHeader/EmptyDashboard.js b/src/visualizers/panels/WorkerHeader/EmptyDashboard.js new file mode 100644 index 000000000..c26262b8f --- /dev/null +++ b/src/visualizers/panels/WorkerHeader/EmptyDashboard.js @@ -0,0 +1,19 @@ +/* globals define, $ */ +define([ + 'css!./EmptyDashboard.css' +], function( +) { + + const EmptyDashboard = function(name, $container) { + this.$el = $('
', {class: 'empty-dashboard'}); + this.$el.text(`No dashboard available for ${name} backend`); + $container.append(this.$el); + }; + + EmptyDashboard.prototype.onShow = + EmptyDashboard.prototype.onHide = + EmptyDashboard.prototype.onActivate = + EmptyDashboard.prototype.onDeactivate = () => {}; + + return EmptyDashboard; +}); diff --git a/src/visualizers/panels/WorkerHeader/ProjectNavigatorController.js b/src/visualizers/panels/WorkerHeader/ProjectNavigatorController.js index 9413a5272..7720d9090 100644 --- a/src/visualizers/panels/WorkerHeader/ProjectNavigatorController.js +++ b/src/visualizers/panels/WorkerHeader/ProjectNavigatorController.js @@ -1,11 +1,11 @@ /* globals define, WebGMEGlobal */ define([ 'js/Dialogs/Projects/ProjectsDialog', - './WorkerDialog', + './ComputeDialog', 'js/Panels/Header/ProjectNavigatorController' ], function( ProjectsDialog, - WorkerDialog, + ComputeDialog, GMEProjectNavigatorController ) { 'use strict'; @@ -18,8 +18,7 @@ define([ ProjectNavigatorController.prototype.initialize = function () { var self = this, newProject, - manageProjects, - manageWorkers; + manageProjects; // initialize model structure for view @@ -39,10 +38,10 @@ define([ }; self.userId = WebGMEGlobal.userInfo._id; - manageWorkers = function() { + const viewCompute = function() { // Create the worker dialog - var pd = new WorkerDialog(self.logger); - pd.show(); + const dialog = new ComputeDialog(self.logger); + dialog.show(); }; // initialize root menu @@ -76,10 +75,10 @@ define([ actionData: {newType: 'import'} }, { - id: 'manageWorkers', + id: 'viewCompute', label: 'View compute ...', iconClass: 'glyphicon glyphicon-cloud', - action: manageWorkers + action: viewCompute } ] }, From 7d14781161bd688e7a95dd64fd16c9a4fc9e5543 Mon Sep 17 00:00:00 2001 From: Brian Broll Date: Wed, 18 Sep 2019 13:31:31 -0500 Subject: [PATCH 28/45] WIP pass logger to dashboards --- src/visualizers/panels/WorkerHeader/ComputeDialog.js | 2 +- src/visualizers/panels/WorkerHeader/EmptyDashboard.js | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/visualizers/panels/WorkerHeader/ComputeDialog.js b/src/visualizers/panels/WorkerHeader/ComputeDialog.js index 1cdb188f0..2c8689cd1 100644 --- a/src/visualizers/panels/WorkerHeader/ComputeDialog.js +++ b/src/visualizers/panels/WorkerHeader/ComputeDialog.js @@ -23,7 +23,7 @@ define([ this.$content = this.$el.find('.dashboard-content'); this.dashboards = Execution.getAvailableBackends().slice(0, 1) // FIXME .map(name => Execution.getBackend(name).getDashboard() || EmptyDashboard.bind(null, name)) - .map(ctor => new ctor(this.$content)); + .map(ctor => new ctor(this.logger, this.$content)); }; ComputeDialog.prototype.initialize = function() { diff --git a/src/visualizers/panels/WorkerHeader/EmptyDashboard.js b/src/visualizers/panels/WorkerHeader/EmptyDashboard.js index c26262b8f..17fc80882 100644 --- a/src/visualizers/panels/WorkerHeader/EmptyDashboard.js +++ b/src/visualizers/panels/WorkerHeader/EmptyDashboard.js @@ -4,9 +4,10 @@ define([ ], function( ) { - const EmptyDashboard = function(name, $container) { + const EmptyDashboard = function(name, logger, $container) { this.$el = $('
', {class: 'empty-dashboard'}); this.$el.text(`No dashboard available for ${name} backend`); + this.logger = logger.fork(name); $container.append(this.$el); }; From fcc2887c29dd61d48fff08c53ba98418ecdba642 Mon Sep 17 00:00:00 2001 From: Brian Broll Date: Wed, 18 Sep 2019 13:32:21 -0500 Subject: [PATCH 29/45] WIP #1186 Added gme dashboard --- .../backends/gme/dashboard/WorkerJobItem.html | 7 + .../backends/gme/dashboard/WorkerModal.css | 47 ++++ .../backends/gme/dashboard/WorkerModal.html | 37 +++ .../gme/dashboard/WorkerTemplate.html.ejs | 5 + .../execution/backends/gme/dashboard/index.js | 224 ++++++++++++++++++ src/common/execution/backends/gme/index.js | 6 + 6 files changed, 326 insertions(+) create mode 100644 src/common/execution/backends/gme/dashboard/WorkerJobItem.html create mode 100644 src/common/execution/backends/gme/dashboard/WorkerModal.css create mode 100644 src/common/execution/backends/gme/dashboard/WorkerModal.html create mode 100644 src/common/execution/backends/gme/dashboard/WorkerTemplate.html.ejs create mode 100644 src/common/execution/backends/gme/dashboard/index.js diff --git a/src/common/execution/backends/gme/dashboard/WorkerJobItem.html b/src/common/execution/backends/gme/dashboard/WorkerJobItem.html new file mode 100644 index 000000000..3fa3b920a --- /dev/null +++ b/src/common/execution/backends/gme/dashboard/WorkerJobItem.html @@ -0,0 +1,7 @@ + + + unknown + unknown + unknown + unknown + diff --git a/src/common/execution/backends/gme/dashboard/WorkerModal.css b/src/common/execution/backends/gme/dashboard/WorkerModal.css new file mode 100644 index 000000000..b5c874f78 --- /dev/null +++ b/src/common/execution/backends/gme/dashboard/WorkerModal.css @@ -0,0 +1,47 @@ +.gme-dashboard .modal-content .modal-header span { + font-size: 28px; + vertical-align: middle; +} + +.gme-dashboard .modal-content .modal-header .header-icon { + height: 28px; + width: 28px; + margin-right: 1ex; + display: inline-block; + vertical-align: middle; + background-size: 28px 28px; +} + +.gme-dashboard .modal-content .modal-body th { + text-align: left; +} + +.gme-dashboard .job-tag { + margin-right: 5px; +} + +.gme-dashboard .queue-title { + float: left; + margin-right: 5px; + font-size: 1.4em; + color: #555555; + font-weight: bold; + width: 100%; + text-align: center; +} + +.gme-dashboard .no-jobs-msg { + font-style: italic; + font-size: 1.3em; + color: #777; +} + +.gme-dashboard .no-workers-msg { + font-style: italic; + font-size: 1.3em; + color: #777; +} + +.gme-dashboard .job-queue { + padding-top: 1em; +} diff --git a/src/common/execution/backends/gme/dashboard/WorkerModal.html b/src/common/execution/backends/gme/dashboard/WorkerModal.html new file mode 100644 index 000000000..2a705548d --- /dev/null +++ b/src/common/execution/backends/gme/dashboard/WorkerModal.html @@ -0,0 +1,37 @@ +
+
+
Connected Workers
+ + + + + + + + + + + +
Worker Id + Last Seen + Status +
No Connected Workers...
+
+
+
Job Queue
+ + + + + + + + + + + + + +
JobExecutionProjectCreation DateStatus
No Running Jobs...
+
+
diff --git a/src/common/execution/backends/gme/dashboard/WorkerTemplate.html.ejs b/src/common/execution/backends/gme/dashboard/WorkerTemplate.html.ejs new file mode 100644 index 000000000..5a79ce84d --- /dev/null +++ b/src/common/execution/backends/gme/dashboard/WorkerTemplate.html.ejs @@ -0,0 +1,5 @@ + + + unknown + unknown + diff --git a/src/common/execution/backends/gme/dashboard/index.js b/src/common/execution/backends/gme/dashboard/index.js new file mode 100644 index 000000000..8699b4784 --- /dev/null +++ b/src/common/execution/backends/gme/dashboard/index.js @@ -0,0 +1,224 @@ +/* globals define, $ */ +define([ + 'deepforge/ExecutionEnv', + 'q', + 'deepforge/viz/Utils', + 'deepforge/api/JobOriginClient', + 'text!./WorkerModal.html', + 'text!./WorkerTemplate.html.ejs', + 'text!./WorkerJobItem.html', + 'css!./WorkerModal.css' +], function( + ExecutionEnv, + Q, + utils, + JobOriginClient, + WorkerHtml, + WorkerTemplate, + WorkerJobItem +) { + 'use strict'; + + var WorkerDialog = function(logger, $container) { + this.logger = logger.fork('GME'); + this.workerDict = {}; + this.workers = {}; + this.runningWorkers = []; + this.jobsDict = {}; + this.jobs = {}; + this.active = false; + this.$el = $(WorkerHtml); + $container.append(this.$el); + this.originManager = new JobOriginClient({ + logger: this.logger + }); + }; + + WorkerDialog.prototype.initialize = function() { + this.$table = this._dialog.find('.worker-list'); + this.$noJobs = this._dialog.find('.no-jobs-msg'); + this.$noWorkers = this._dialog.find('.no-workers-msg'); + this._isShowingJobs = false; + this._isShowingWorkers = true; + this.$queue = this._dialog.find('.job-queue-list'); + }; + + WorkerDialog.prototype.onDeactivate = + WorkerDialog.prototype.onActivate = function() {}; + WorkerDialog.prototype.onShow = function() { + this.active = true; + this.update(); + }; + + WorkerDialog.prototype.onHide = function() { + this.active = false; + }; + + WorkerDialog.prototype.update = async function() { + const workers = await ExecutionEnv.getWorkers(); + const jobs = await ExecutionEnv.getJobs(); + + await Q.all([this.updateWorkers(workers), this.updateJobs(jobs)]); + + if (this.active) { + setTimeout(this.update.bind(this), 1000); + } + }; + + WorkerDialog.prototype.updateWorkers = function(workerDict) { + var ids = Object.keys(workerDict), + oldWorkerIds, + visibleWorkers = false, + i; + + this.runningWorkers = []; + for (i = ids.length; i--;) { + this.updateWorker(workerDict[ids[i]]); + visibleWorkers = true; + delete this.workerDict[ids[i]]; + } + this.toggleNoWorkersMsg(!visibleWorkers); + + // Clear old workers + oldWorkerIds = Object.keys(this.workerDict); + for (i = oldWorkerIds.length; i--;) { + this.removeWorker(oldWorkerIds[i]); + } + + this.workerDict = workerDict; + }; + + WorkerDialog.prototype.updateWorker = function(worker) { + var row = this.workers[worker.clientId] || $(WorkerTemplate), + clazz; + + worker.lastSeen = utils.getDisplayTime(worker.lastSeen*1000); + worker.status = worker.jobs.length ? 'RUNNING' : 'READY'; + + clazz = worker.status === 'RUNNING' ? 'warning' : 'success'; + row[0].className = clazz; + + row.find('.lastSeen').text(worker.lastSeen); + row.find('.clientId').text(worker.clientId); + row.find('.status').text(worker.status); + if (!this.workers[worker.clientId]) { + this.$table.append(row); + this.workers[worker.clientId] = row; + } + + if (worker.status === 'RUNNING') { + this.runningWorkers.push(worker); + } + }; + + WorkerDialog.prototype.removeWorker = function(workerId) { + this.workers[workerId].remove(); + delete this.workers[workerId]; + }; + + WorkerDialog.prototype.updateJobs = function(jobsDict) { + var allJobIds = Object.keys(jobsDict), + hasJobs = false, + id; + + this.jobsDict = jobsDict; + for (var i = allJobIds.length; i--;) { + id = allJobIds[i]; + if (this.jobs[id] || !this.isFinished(id)) { + hasJobs = this.updateJobItem(id) || hasJobs; + } + } + this.setNoJobsMessage(!hasJobs); // hide if no queue + }; + + WorkerDialog.prototype.setNoJobsMessage = function(visible) { + var visibility = visible ? 'inherit' : 'none', + wasVisible = !this._isShowingJobs; + + if (visible !== wasVisible) { + this.$noJobs.css('display', visibility); + this._isShowingJobs = !visible; + } + }; + + WorkerDialog.prototype.toggleNoWorkersMsg = function(visible) { + var visibility = visible ? 'inherit' : 'none'; + + if (visible !== this._isShowingWorkers) { + this.$noWorkers.css('display', visibility); + this._isShowingWorkers = visible; + } + }; + + WorkerDialog.prototype.isFinished = function(jobId) { + return this.jobsDict[jobId].status === 'FAILED_TO_EXECUTE' || + this.jobsDict[jobId].status === 'SUCCESS' || + this.jobsDict[jobId].status === 'CANCELED'; + }; + + WorkerDialog.prototype.updateJobItemName = function(jobId) { + return this.originManager.getOrigin(jobId) + .then(info => { + var job = this.jobs[jobId], + project = info.project.replace(/^guest\+/, ''); + + if (job && this.active) { + if (info.branch !== 'master') { + project += ' (' + info.branch + ')'; + } + job.find('.job-id').text(info.job); + job.find('.execution').text(info.execution); + job.find('.project').text(project); + } + }); + }; + + WorkerDialog.prototype.getWorkerWithJob = function(jobId) { + var jobs; + + for (var i = this.runningWorkers.length; i--;) { + jobs = this.runningWorkers[i].jobs; + for (var j = jobs.length; j--;) { + if (jobs[j].hash === jobId) { + return this.runningWorkers[i].clientId; + } + } + } + + return 'unknown'; + }; + + WorkerDialog.prototype.updateJobItem = function(jobId) { + var job = this.jobs[jobId] || $(WorkerJobItem), + info = this.jobsDict[jobId], + createdTime = new Date(info.createTime).getTime(), + clazz = utils.ClassForJobStatus[info.status.toLowerCase()], + status = info.status; + + job[0].className = `job-tag ${clazz}`; + + // Add the worker id if running + if (info.status.toLowerCase() === 'running') { + var workerId = this.getWorkerWithJob(jobId); + status += ' (' + workerId + ')'; + } + job.find('.status').text(status); + + if (!this.jobs[jobId]) { + job.find('.job-id').text('Loading'); + job.find('.createdAt').text(utils.getDisplayTime(createdTime)); + this.updateJobItemName(jobId); + this.$queue.append(job); + this.jobs[jobId] = job; + } + + if (this.isFinished(jobId)) { + job.remove(); + delete this.jobs[jobId]; + return false; + } + return true; + }; + + return WorkerDialog; +}); diff --git a/src/common/execution/backends/gme/index.js b/src/common/execution/backends/gme/index.js index fed872790..54a13b8b5 100644 --- a/src/common/execution/backends/gme/index.js +++ b/src/common/execution/backends/gme/index.js @@ -1,7 +1,9 @@ /* globals define */ define([ + './dashboard/index', '../BaseBackend' ], function( + Dashboard, BaseBackend ) { @@ -11,5 +13,9 @@ define([ GMEBackend.prototype = Object.create(BaseBackend.prototype); + GMEBackend.prototype.getDashboard = function() { + return Dashboard; + }; + return GMEBackend; }); From 3e8a30cbd9cb7c1ad27bcb4934cc1e51639abb15 Mon Sep 17 00:00:00 2001 From: Brian Broll Date: Wed, 18 Sep 2019 14:06:06 -0500 Subject: [PATCH 30/45] WIP Fixed gme compute dashboad --- .../execution/backends/gme/dashboard/index.js | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/common/execution/backends/gme/dashboard/index.js b/src/common/execution/backends/gme/dashboard/index.js index 8699b4784..2b3af1620 100644 --- a/src/common/execution/backends/gme/dashboard/index.js +++ b/src/common/execution/backends/gme/dashboard/index.js @@ -27,20 +27,21 @@ define([ this.jobsDict = {}; this.jobs = {}; this.active = false; - this.$el = $(WorkerHtml); - $container.append(this.$el); this.originManager = new JobOriginClient({ logger: this.logger }); - }; - WorkerDialog.prototype.initialize = function() { - this.$table = this._dialog.find('.worker-list'); - this.$noJobs = this._dialog.find('.no-jobs-msg'); - this.$noWorkers = this._dialog.find('.no-workers-msg'); + this.$el = $(WorkerHtml); + this.$table = this.$el.find('.worker-list'); + this.$noJobs = this.$el.find('.no-jobs-msg'); + this.$noWorkers = this.$el.find('.no-workers-msg'); this._isShowingJobs = false; this._isShowingWorkers = true; - this.$queue = this._dialog.find('.job-queue-list'); + this.$queue = this.$el.find('.job-queue-list'); + $container.append(this.$el); + }; + + WorkerDialog.prototype.initialize = function() { }; WorkerDialog.prototype.onDeactivate = From d6d513999688df973b5ead88b69c77086a1568a6 Mon Sep 17 00:00:00 2001 From: Brian Broll Date: Wed, 18 Sep 2019 14:07:26 -0500 Subject: [PATCH 31/45] WIP Add tabs and support for multiple backends --- .../panels/WorkerHeader/ComputeDialog.js | 22 +- .../panels/WorkerHeader/ComputeModal.css | 6 +- .../panels/WorkerHeader/ComputeModal.html | 13 - .../panels/WorkerHeader/ComputeModal.html.ejs | 31 +++ .../panels/WorkerHeader/WorkerDialog.js | 222 ------------------ .../panels/WorkerHeader/WorkerJobItem.html | 7 - .../panels/WorkerHeader/WorkerModal.css | 47 ---- .../panels/WorkerHeader/WorkerModal.html | 60 ----- .../WorkerHeader/WorkerTemplate.html.ejs | 5 - 9 files changed, 51 insertions(+), 362 deletions(-) delete mode 100644 src/visualizers/panels/WorkerHeader/ComputeModal.html create mode 100644 src/visualizers/panels/WorkerHeader/ComputeModal.html.ejs delete mode 100644 src/visualizers/panels/WorkerHeader/WorkerDialog.js delete mode 100644 src/visualizers/panels/WorkerHeader/WorkerJobItem.html delete mode 100644 src/visualizers/panels/WorkerHeader/WorkerModal.css delete mode 100644 src/visualizers/panels/WorkerHeader/WorkerModal.html delete mode 100644 src/visualizers/panels/WorkerHeader/WorkerTemplate.html.ejs diff --git a/src/visualizers/panels/WorkerHeader/ComputeDialog.js b/src/visualizers/panels/WorkerHeader/ComputeDialog.js index 2c8689cd1..003325be0 100644 --- a/src/visualizers/panels/WorkerHeader/ComputeDialog.js +++ b/src/visualizers/panels/WorkerHeader/ComputeDialog.js @@ -4,26 +4,34 @@ define([ 'q', 'deepforge/viz/Utils', './EmptyDashboard', - 'text!./ComputeModal.html', + 'underscore', + 'text!./ComputeModal.html.ejs', 'css!./ComputeModal.css' ], function( Execution, Q, utils, EmptyDashboard, + _, ComputeHtml, ) { 'use strict'; + const ComputeHtmlTpl = _.template(ComputeHtml); const ComputeDialog = function(logger) { this.active = false; this.logger = logger.fork('ComputeDialog'); - this.$el = $(ComputeHtml); - // TODO: Handle this differently if there are multiple dashboards? - this.$content = this.$el.find('.dashboard-content'); - this.dashboards = Execution.getAvailableBackends().slice(0, 1) // FIXME - .map(name => Execution.getBackend(name).getDashboard() || EmptyDashboard.bind(null, name)) - .map(ctor => new ctor(this.logger, this.$content)); + + const backendNames = Execution.getAvailableBackends(); + this.$el = $(ComputeHtmlTpl({tabs: backendNames})); + this.dashboards = backendNames + .map(name => { + const backend = Execution.getBackend(name); + const Dashboard = backend.getDashboard() || EmptyDashboard.bind(null, name); + const $container = this.$el.find(`#${name}-dashboard-container`); + + return new Dashboard(this.logger, $container); + }); }; ComputeDialog.prototype.initialize = function() { diff --git a/src/visualizers/panels/WorkerHeader/ComputeModal.css b/src/visualizers/panels/WorkerHeader/ComputeModal.css index 7b75373ff..8360253c0 100644 --- a/src/visualizers/panels/WorkerHeader/ComputeModal.css +++ b/src/visualizers/panels/WorkerHeader/ComputeModal.css @@ -12,6 +12,10 @@ background-size: 28px 28px; } -.compute-modal .modal-content .modal-body th { +.compute-modal .modal-content .modal-body { text-align: left; } + +.compute-modal .tab-pane { + padding: 15px; +} diff --git a/src/visualizers/panels/WorkerHeader/ComputeModal.html b/src/visualizers/panels/WorkerHeader/ComputeModal.html deleted file mode 100644 index 5c2d52e1f..000000000 --- a/src/visualizers/panels/WorkerHeader/ComputeModal.html +++ /dev/null @@ -1,13 +0,0 @@ - diff --git a/src/visualizers/panels/WorkerHeader/ComputeModal.html.ejs b/src/visualizers/panels/WorkerHeader/ComputeModal.html.ejs new file mode 100644 index 000000000..a8e1458d5 --- /dev/null +++ b/src/visualizers/panels/WorkerHeader/ComputeModal.html.ejs @@ -0,0 +1,31 @@ + diff --git a/src/visualizers/panels/WorkerHeader/WorkerDialog.js b/src/visualizers/panels/WorkerHeader/WorkerDialog.js deleted file mode 100644 index 1db0b93b7..000000000 --- a/src/visualizers/panels/WorkerHeader/WorkerDialog.js +++ /dev/null @@ -1,222 +0,0 @@ -/* globals define, $ */ -define([ - 'deepforge/ExecutionEnv', - 'q', - 'deepforge/viz/Utils', - 'deepforge/api/JobOriginClient', - 'text!./WorkerModal.html', - 'text!./WorkerTemplate.html.ejs', - 'text!./WorkerJobItem.html', - 'css!./WorkerModal.css' -], function( - ExecutionEnv, - Q, - utils, - JobOriginClient, - WorkerHtml, - WorkerTemplate, - WorkerJobItem -) { - 'use strict'; - - var WorkerDialog = function(logger) { - this.workerDict = {}; - this.workers = {}; - this.runningWorkers = []; - this.jobsDict = {}; - this.jobs = {}; - this.active = false; - this.logger = logger.fork('WorkerDialog'); - this.originManager = new JobOriginClient({ - logger: this.logger - }); - }; - - WorkerDialog.prototype.initialize = function() { - this._dialog = $(WorkerHtml); - this._table = this._dialog.find('.worker-list'); - this.$noJobs = this._dialog.find('.no-jobs-msg'); - this.$noWorkers = this._dialog.find('.no-workers-msg'); - this._isShowingJobs = false; - this._isShowingWorkers = true; - this._queue = this._dialog.find('.job-queue-list'); - this._dialog.modal('show'); - this._dialog.on('hidden.bs.modal', () => this.active = false); - }; - - WorkerDialog.prototype.show = function() { - this.active = true; - this.update(); - this.initialize(); - }; - - WorkerDialog.prototype.update = function() { - // Poll the workers - return Q.all([ - ExecutionEnv.getWorkers().then(workers => this.updateWorkers(workers)), - ExecutionEnv.getJobs().then(jobs => this.updateJobs(jobs)) - ]).then(() => { - if (this.active) { - setTimeout(this.update.bind(this), 1000); - } - }) - .catch(err => this.logger.error('Update failed:', err)); - }; - - WorkerDialog.prototype.updateWorkers = function(workerDict) { - var ids = Object.keys(workerDict), - oldWorkerIds, - visibleWorkers = false, - i; - - this.runningWorkers = []; - for (i = ids.length; i--;) { - this.updateWorker(workerDict[ids[i]]); - visibleWorkers = true; - delete this.workerDict[ids[i]]; - } - this.toggleNoWorkersMsg(!visibleWorkers); - - // Clear old workers - oldWorkerIds = Object.keys(this.workerDict); - for (i = oldWorkerIds.length; i--;) { - this.removeWorker(oldWorkerIds[i]); - } - - this.workerDict = workerDict; - }; - - WorkerDialog.prototype.updateWorker = function(worker) { - var row = this.workers[worker.clientId] || $(WorkerTemplate), - clazz; - - worker.lastSeen = utils.getDisplayTime(worker.lastSeen*1000); - worker.status = worker.jobs.length ? 'RUNNING' : 'READY'; - - clazz = worker.status === 'RUNNING' ? 'warning' : 'success'; - row[0].className = clazz; - - row.find('.lastSeen').text(worker.lastSeen); - row.find('.clientId').text(worker.clientId); - row.find('.status').text(worker.status); - if (!this.workers[worker.clientId]) { - this._table.append(row); - this.workers[worker.clientId] = row; - } - - if (worker.status === 'RUNNING') { - this.runningWorkers.push(worker); - } - }; - - WorkerDialog.prototype.removeWorker = function(workerId) { - this.workers[workerId].remove(); - delete this.workers[workerId]; - }; - - WorkerDialog.prototype.updateJobs = function(jobsDict) { - var allJobIds = Object.keys(jobsDict), - hasJobs = false, - id; - - this.jobsDict = jobsDict; - for (var i = allJobIds.length; i--;) { - id = allJobIds[i]; - if (this.jobs[id] || !this.isFinished(id)) { - hasJobs = this.updateJobItem(id) || hasJobs; - } - } - this.setNoJobsMessage(!hasJobs); // hide if no queue - }; - - WorkerDialog.prototype.setNoJobsMessage = function(visible) { - var visibility = visible ? 'inherit' : 'none', - wasVisible = !this._isShowingJobs; - - if (visible !== wasVisible) { - this.$noJobs.css('display', visibility); - this._isShowingJobs = !visible; - } - }; - - WorkerDialog.prototype.toggleNoWorkersMsg = function(visible) { - var visibility = visible ? 'inherit' : 'none'; - - if (visible !== this._isShowingWorkers) { - this.$noWorkers.css('display', visibility); - this._isShowingWorkers = visible; - } - }; - - WorkerDialog.prototype.isFinished = function(jobId) { - return this.jobsDict[jobId].status === 'FAILED_TO_EXECUTE' || - this.jobsDict[jobId].status === 'SUCCESS' || - this.jobsDict[jobId].status === 'CANCELED'; - }; - - WorkerDialog.prototype.updateJobItemName = function(jobId) { - return this.originManager.getOrigin(jobId) - .then(info => { - var job = this.jobs[jobId], - project = info.project.replace(/^guest\+/, ''); - - if (job && this.active) { - if (info.branch !== 'master') { - project += ' (' + info.branch + ')'; - } - job.find('.job-id').text(info.job); - job.find('.execution').text(info.execution); - job.find('.project').text(project); - } - }); - }; - - WorkerDialog.prototype.getWorkerWithJob = function(jobId) { - var jobs; - - for (var i = this.runningWorkers.length; i--;) { - jobs = this.runningWorkers[i].jobs; - for (var j = jobs.length; j--;) { - if (jobs[j].hash === jobId) { - return this.runningWorkers[i].clientId; - } - } - } - - return 'unknown'; - }; - - WorkerDialog.prototype.updateJobItem = function(jobId) { - var job = this.jobs[jobId] || $(WorkerJobItem), - info = this.jobsDict[jobId], - createdTime = new Date(info.createTime).getTime(), - clazz = utils.ClassForJobStatus[info.status.toLowerCase()], - status = info.status; - - job[0].className = `job-tag ${clazz}`; - - // Add the worker id if running - if (info.status.toLowerCase() === 'running') { - var workerId = this.getWorkerWithJob(jobId); - status += ' (' + workerId + ')'; - } - job.find('.status').text(status); - - if (!this.jobs[jobId]) { - job.find('.job-id').text('Loading'); - job.find('.createdAt').text(utils.getDisplayTime(createdTime)); - this.updateJobItemName(jobId); - this._queue.append(job); - this.jobs[jobId] = job; - } - - if (this.isFinished(jobId)) { - job.remove(); - delete this.jobs[jobId]; - return false; - } - return true; - }; - - return WorkerDialog; -}); diff --git a/src/visualizers/panels/WorkerHeader/WorkerJobItem.html b/src/visualizers/panels/WorkerHeader/WorkerJobItem.html deleted file mode 100644 index 3fa3b920a..000000000 --- a/src/visualizers/panels/WorkerHeader/WorkerJobItem.html +++ /dev/null @@ -1,7 +0,0 @@ - - - unknown - unknown - unknown - unknown - diff --git a/src/visualizers/panels/WorkerHeader/WorkerModal.css b/src/visualizers/panels/WorkerHeader/WorkerModal.css deleted file mode 100644 index 7ecaef5be..000000000 --- a/src/visualizers/panels/WorkerHeader/WorkerModal.css +++ /dev/null @@ -1,47 +0,0 @@ -.worker-modal .modal-content .modal-header span { - font-size: 28px; - vertical-align: middle; -} - -.worker-modal .modal-content .modal-header .header-icon { - height: 28px; - width: 28px; - margin-right: 1ex; - display: inline-block; - vertical-align: middle; - background-size: 28px 28px; -} - -.worker-modal .modal-content .modal-body th { - text-align: left; -} - -.worker-modal .job-tag { - margin-right: 5px; -} - -.worker-modal .queue-title { - float: left; - margin-right: 5px; - font-size: 1.4em; - color: #555555; - font-weight: bold; - width: 100%; - text-align: center; -} - -.worker-modal .no-jobs-msg { - font-style: italic; - font-size: 1.3em; - color: #777; -} - -.worker-modal .no-workers-msg { - font-style: italic; - font-size: 1.3em; - color: #777; -} - -.worker-modal .job-queue { - padding-top: 1em; -} diff --git a/src/visualizers/panels/WorkerHeader/WorkerModal.html b/src/visualizers/panels/WorkerHeader/WorkerModal.html deleted file mode 100644 index 0c52ed467..000000000 --- a/src/visualizers/panels/WorkerHeader/WorkerModal.html +++ /dev/null @@ -1,60 +0,0 @@ - diff --git a/src/visualizers/panels/WorkerHeader/WorkerTemplate.html.ejs b/src/visualizers/panels/WorkerHeader/WorkerTemplate.html.ejs deleted file mode 100644 index 5a79ce84d..000000000 --- a/src/visualizers/panels/WorkerHeader/WorkerTemplate.html.ejs +++ /dev/null @@ -1,5 +0,0 @@ - - - unknown - unknown - From aeb87705b6cdcb95c8ff446fe6cf5dc1a1e5b1a4 Mon Sep 17 00:00:00 2001 From: Brian Broll Date: Wed, 18 Sep 2019 14:10:51 -0500 Subject: [PATCH 32/45] WIP Add msg when no compute backends --- src/visualizers/panels/WorkerHeader/ComputeModal.html.ejs | 2 +- src/visualizers/panels/WorkerHeader/EmptyDashboard.css | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/visualizers/panels/WorkerHeader/ComputeModal.html.ejs b/src/visualizers/panels/WorkerHeader/ComputeModal.html.ejs index a8e1458d5..8a8e047ea 100644 --- a/src/visualizers/panels/WorkerHeader/ComputeModal.html.ejs +++ b/src/visualizers/panels/WorkerHeader/ComputeModal.html.ejs @@ -23,7 +23,7 @@ <% } else if (tabs.length === 1) { %>
<% } else { %> - +
No compute backends enabled.
<% } %>
diff --git a/src/visualizers/panels/WorkerHeader/EmptyDashboard.css b/src/visualizers/panels/WorkerHeader/EmptyDashboard.css index 068d964b0..ab73eec44 100644 --- a/src/visualizers/panels/WorkerHeader/EmptyDashboard.css +++ b/src/visualizers/panels/WorkerHeader/EmptyDashboard.css @@ -4,3 +4,10 @@ color: #777; text-align: center; } + +.compute-modal .no-dashboards { + font-style: italic; + font-size: 1.3em; + color: #777; + text-align: center; +} From 0a6d31090fd4ba2c9e9568742f289b7a48661932 Mon Sep 17 00:00:00 2001 From: Brian Broll Date: Wed, 18 Sep 2019 16:16:38 -0500 Subject: [PATCH 33/45] WIP renamed execution->compute --- src/common/compute/backends/ComputeBackend.js | 32 +++++++++++++++++ .../backends/ComputeClient.js} | 36 +++++++++---------- .../compute/backends/ComputeDashboard.js | 15 ++++++++ .../backends/JobResults.js | 4 +-- .../backends/gme/Client.js | 8 ++--- .../backends/gme/dashboard/WorkerJobItem.html | 0 .../backends/gme/dashboard/WorkerModal.css | 0 .../backends/gme/dashboard/WorkerModal.html | 0 .../gme/dashboard/WorkerTemplate.html.ejs | 0 .../backends/gme/dashboard/index.js | 7 ++-- src/common/compute/backends/gme/index.js | 20 +++++++++++ .../backends/local/Client.js | 8 ++--- src/common/compute/backends/local/index.js | 15 ++++++++ src/common/{execution => compute}/index.js | 23 ++++++------ src/common/execution/backends/BaseBackend.js | 29 --------------- src/common/execution/backends/gme/index.js | 21 ----------- src/common/execution/backends/local/index.js | 15 -------- src/common/utils.js | 1 - 18 files changed, 124 insertions(+), 110 deletions(-) create mode 100644 src/common/compute/backends/ComputeBackend.js rename src/common/{execution/backends/BaseExecutor.js => compute/backends/ComputeClient.js} (57%) create mode 100644 src/common/compute/backends/ComputeDashboard.js rename src/common/{execution => compute}/backends/JobResults.js (52%) rename src/common/{execution => compute}/backends/gme/Client.js (95%) rename src/common/{execution => compute}/backends/gme/dashboard/WorkerJobItem.html (100%) rename src/common/{execution => compute}/backends/gme/dashboard/WorkerModal.css (100%) rename src/common/{execution => compute}/backends/gme/dashboard/WorkerModal.html (100%) rename src/common/{execution => compute}/backends/gme/dashboard/WorkerTemplate.html.ejs (100%) rename src/common/{execution => compute}/backends/gme/dashboard/index.js (98%) create mode 100644 src/common/compute/backends/gme/index.js rename src/common/{execution => compute}/backends/local/Client.js (98%) create mode 100644 src/common/compute/backends/local/index.js rename src/common/{execution => compute}/index.js (64%) delete mode 100644 src/common/execution/backends/BaseBackend.js delete mode 100644 src/common/execution/backends/gme/index.js delete mode 100644 src/common/execution/backends/local/index.js diff --git a/src/common/compute/backends/ComputeBackend.js b/src/common/compute/backends/ComputeBackend.js new file mode 100644 index 000000000..1a4d7e894 --- /dev/null +++ b/src/common/compute/backends/ComputeBackend.js @@ -0,0 +1,32 @@ +/* globals define, requirejs */ +define([ + 'module', + 'q', +], function( + module, + Q, +) { + + const ComputeBackend = function(name) { + this.name = name; + }; + + ComputeBackend.prototype.getClient = function(logger) { + const path = require('path'); + const dirname = path.dirname(module.uri); + const Client = requirejs(`${dirname}/${this.name.toLowerCase()}/Client.js`); + return new Client(logger); + }; + + ComputeBackend.prototype.getDashboard = async function() { + return null; + }; + + ComputeBackend.prototype.require = function(path) { // helper for loading async + const deferred = Q.defer(); + require([path], deferred.resolve, deferred.reject); + return deferred.promise; + }; + + return ComputeBackend; +}); diff --git a/src/common/execution/backends/BaseExecutor.js b/src/common/compute/backends/ComputeClient.js similarity index 57% rename from src/common/execution/backends/BaseExecutor.js rename to src/common/compute/backends/ComputeClient.js index 73ebac6b9..1cd97981d 100644 --- a/src/common/execution/backends/BaseExecutor.js +++ b/src/common/compute/backends/ComputeClient.js @@ -1,4 +1,4 @@ -const BaseExecutor = function(logger, gmeConfig) { +const ComputeClient = function(logger, gmeConfig) { const isHttps = typeof window === 'undefined' ? false : window.location.protocol !== 'http:'; @@ -6,62 +6,60 @@ const BaseExecutor = function(logger, gmeConfig) { this._events = {}; }; -BaseExecutor.prototype.cancelJob = function(job) { +ComputeClient.prototype.cancelJob = function(job) { const msg = `cancelJob is not implemented for current executor backend!`; this.logger.warn(msg); throw new Error(msg); }; -BaseExecutor.prototype.getInfo = function(job) { +ComputeClient.prototype.getInfo = function(job) { const msg = `getInfo is not implemented for current executor backend!`; this.logger.warn(msg); throw new Error(msg); }; -BaseExecutor.prototype.createJob = async function(hash) { +ComputeClient.prototype.createJob = async function(hash) { const msg = `createJob is not implemented for current executor backend!`; this.logger.warn(msg); throw new Error(msg); }; -BaseExecutor.prototype.getStatus = async function(jobInfo) { +ComputeClient.prototype.getStatus = async function(jobInfo) { const msg = `getStatus is not implemented for current executor backend!`; this.logger.warn(msg); throw new Error(msg); }; -BaseExecutor.prototype.getOutputHashes = async function(jobInfo) { +ComputeClient.prototype.getOutputHashes = async function(jobInfo) { const msg = `getOutputHashes is not implemented for current executor backend!`; this.logger.warn(msg); throw new Error(msg); }; -// TODO: Should I remove this? -BaseExecutor.prototype.getConsoleOutput = async function(hash) { +ComputeClient.prototype.getConsoleOutput = async function(hash) { const msg = `getConsoleOutput is not implemented for current executor backend!`; this.logger.warn(msg); throw new Error(msg); }; // Some functions for event support -BaseExecutor.prototype.on = function(ev, cb) { +ComputeClient.prototype.on = function(ev, cb) { this._events[ev] = this._events[ev] || []; this._events[ev].push(cb); }; -BaseExecutor.prototype.emit = function(ev) { +ComputeClient.prototype.emit = function(ev) { const args = Array.prototype.slice.call(arguments, 1); const handlers = this._events[ev] || []; handlers.forEach(fn => fn.apply(this, args)); }; -// TODO: Make these match the values in the model (`status` enum on pipeline.Job) -BaseExecutor.prototype.QUEUED = 'queued'; -BaseExecutor.prototype.PENDING = 'pending'; -BaseExecutor.prototype.RUNNING = 'running'; -BaseExecutor.prototype.SUCCESS = 'success'; -BaseExecutor.prototype.FAILED = 'failed'; -BaseExecutor.prototype.CANCELED = 'canceled'; -BaseExecutor.prototype.NOT_FOUND = 'NOT_FOUND'; +ComputeClient.prototype.QUEUED = 'queued'; +ComputeClient.prototype.PENDING = 'pending'; +ComputeClient.prototype.RUNNING = 'running'; +ComputeClient.prototype.SUCCESS = 'success'; +ComputeClient.prototype.FAILED = 'failed'; +ComputeClient.prototype.CANCELED = 'canceled'; +ComputeClient.prototype.NOT_FOUND = 'NOT_FOUND'; -module.exports = BaseExecutor; +module.exports = ComputeClient; diff --git a/src/common/compute/backends/ComputeDashboard.js b/src/common/compute/backends/ComputeDashboard.js new file mode 100644 index 000000000..0c9a91f88 --- /dev/null +++ b/src/common/compute/backends/ComputeDashboard.js @@ -0,0 +1,15 @@ +/* globals define */ +define([ +], function( +) { + + const ComputeDashboard = function(/*logger, $container*/) { + }; + + ComputeDashboard.prototype.onShow = + ComputeDashboard.prototype.onHide = + ComputeDashboard.prototype.onActivate = + ComputeDashboard.prototype.onDeactivate = () => {}; + + return ComputeDashboard; +}); diff --git a/src/common/execution/backends/JobResults.js b/src/common/compute/backends/JobResults.js similarity index 52% rename from src/common/execution/backends/JobResults.js rename to src/common/compute/backends/JobResults.js index 187c6a95a..48c5655db 100644 --- a/src/common/execution/backends/JobResults.js +++ b/src/common/compute/backends/JobResults.js @@ -1,7 +1,7 @@ -const BaseExecutor = require('./BaseExecutor'); +const ComputeClient = require('./ComputeClient'); class JobResults { - constructor(status=BaseExecutor.prototype.CREATED) { + constructor(status=ComputeClient.prototype.CREATED) { this.status = status; this.resultHashes = []; } diff --git a/src/common/execution/backends/gme/Client.js b/src/common/compute/backends/gme/Client.js similarity index 95% rename from src/common/execution/backends/gme/Client.js rename to src/common/compute/backends/gme/Client.js index 5b41eec1e..430e32631 100644 --- a/src/common/execution/backends/gme/Client.js +++ b/src/common/compute/backends/gme/Client.js @@ -1,13 +1,13 @@ /* globals define */ define([ - '../BaseExecutor', + '../ComputeClient', '../JobResults', 'deepforge/ExecutionEnv', 'executor/ExecutorClient', 'path', 'module', ], function( - BaseExecutor, + ComputeClient, JobResults, ExecutionEnv, ExecutorClient, @@ -16,7 +16,7 @@ define([ ) { const PROJECT_ROOT = path.join(path.dirname(module.uri), '..', '..', '..', '..', '..'); const GMEExecutor = function(/*logger*/) { - BaseExecutor.apply(this, arguments); + ComputeClient.apply(this, arguments); const configPath = path.join(PROJECT_ROOT, 'config'); const gmeConfig = require.nodeRequire(configPath); this.pollInterval = 1500; @@ -27,7 +27,7 @@ define([ httpsecure: false }); }; - GMEExecutor.prototype = Object.create(BaseExecutor.prototype); + GMEExecutor.prototype = Object.create(ComputeClient.prototype); GMEExecutor.prototype.getConsoleOutput = async function(hash) { return (await this.executor.getOutput(hash)) diff --git a/src/common/execution/backends/gme/dashboard/WorkerJobItem.html b/src/common/compute/backends/gme/dashboard/WorkerJobItem.html similarity index 100% rename from src/common/execution/backends/gme/dashboard/WorkerJobItem.html rename to src/common/compute/backends/gme/dashboard/WorkerJobItem.html diff --git a/src/common/execution/backends/gme/dashboard/WorkerModal.css b/src/common/compute/backends/gme/dashboard/WorkerModal.css similarity index 100% rename from src/common/execution/backends/gme/dashboard/WorkerModal.css rename to src/common/compute/backends/gme/dashboard/WorkerModal.css diff --git a/src/common/execution/backends/gme/dashboard/WorkerModal.html b/src/common/compute/backends/gme/dashboard/WorkerModal.html similarity index 100% rename from src/common/execution/backends/gme/dashboard/WorkerModal.html rename to src/common/compute/backends/gme/dashboard/WorkerModal.html diff --git a/src/common/execution/backends/gme/dashboard/WorkerTemplate.html.ejs b/src/common/compute/backends/gme/dashboard/WorkerTemplate.html.ejs similarity index 100% rename from src/common/execution/backends/gme/dashboard/WorkerTemplate.html.ejs rename to src/common/compute/backends/gme/dashboard/WorkerTemplate.html.ejs diff --git a/src/common/execution/backends/gme/dashboard/index.js b/src/common/compute/backends/gme/dashboard/index.js similarity index 98% rename from src/common/execution/backends/gme/dashboard/index.js rename to src/common/compute/backends/gme/dashboard/index.js index 2b3af1620..d4e668b86 100644 --- a/src/common/execution/backends/gme/dashboard/index.js +++ b/src/common/compute/backends/gme/dashboard/index.js @@ -1,5 +1,6 @@ /* globals define, $ */ define([ + '../../ComputeDashboard', 'deepforge/ExecutionEnv', 'q', 'deepforge/viz/Utils', @@ -9,6 +10,7 @@ define([ 'text!./WorkerJobItem.html', 'css!./WorkerModal.css' ], function( + ComputeDashboard, ExecutionEnv, Q, utils, @@ -41,16 +43,15 @@ define([ $container.append(this.$el); }; - WorkerDialog.prototype.initialize = function() { - }; + WorkerDialog.prototype.initialize = Object.create(ComputeDashboard.prototype); - WorkerDialog.prototype.onDeactivate = WorkerDialog.prototype.onActivate = function() {}; WorkerDialog.prototype.onShow = function() { this.active = true; this.update(); }; + WorkerDialog.prototype.onDeactivate = WorkerDialog.prototype.onHide = function() { this.active = false; }; diff --git a/src/common/compute/backends/gme/index.js b/src/common/compute/backends/gme/index.js new file mode 100644 index 000000000..4f1ab9f16 --- /dev/null +++ b/src/common/compute/backends/gme/index.js @@ -0,0 +1,20 @@ +/* globals define */ +define([ + '../ComputeBackend', +], function( + ComputeBackend, +) { + + const GMEBackend = function() { + ComputeBackend.call(this, 'GME'); + }; + + GMEBackend.prototype = Object.create(ComputeBackend.prototype); + + GMEBackend.prototype.getDashboard = async function() { + console.log(this, this.require); + return await this.require('deepforge/compute/backends/gme/dashboard/index'); + }; + + return GMEBackend; +}); diff --git a/src/common/execution/backends/local/Client.js b/src/common/compute/backends/local/Client.js similarity index 98% rename from src/common/execution/backends/local/Client.js rename to src/common/compute/backends/local/Client.js index 3e0decb14..5de8b02e3 100644 --- a/src/common/execution/backends/local/Client.js +++ b/src/common/compute/backends/local/Client.js @@ -1,7 +1,7 @@ /*globals define*/ // TODO: Show an error if not running on the server... define([ - '../BaseExecutor', + '../ComputeClient', '../JobResults', 'blob/BlobClient', 'child_process', @@ -12,7 +12,7 @@ define([ 'os', 'path', ], function( - BaseExecutor, + ComputeClient, JobResults, BlobClient, childProcess, @@ -46,7 +46,7 @@ define([ const touch = async name => await closeFile(await openFile(name, 'w')); const LocalExecutor = function(/*logger*/) { - BaseExecutor.apply(this, arguments); + ComputeClient.apply(this, arguments); const configPath = path.join(PROJECT_ROOT, 'config'); const gmeConfig = require.nodeRequire(configPath); @@ -63,7 +63,7 @@ define([ }); }; - LocalExecutor.prototype = Object.create(BaseExecutor.prototype); + LocalExecutor.prototype = Object.create(ComputeClient.prototype); LocalExecutor.prototype.cancelJob = function(jobInfo) { const {hash} = jobInfo; diff --git a/src/common/compute/backends/local/index.js b/src/common/compute/backends/local/index.js new file mode 100644 index 000000000..ed546dcd8 --- /dev/null +++ b/src/common/compute/backends/local/index.js @@ -0,0 +1,15 @@ +/* globals define */ +define([ + '../ComputeBackend' +], function( + ComputeBackend +) { + + const LocalBackend = function() { + ComputeBackend.call(this, 'Local'); + }; + + LocalBackend.prototype = Object.create(ComputeBackend.prototype); + + return LocalBackend; +}); diff --git a/src/common/execution/index.js b/src/common/compute/index.js similarity index 64% rename from src/common/execution/index.js rename to src/common/compute/index.js index 3bf463401..ce09c8dc5 100644 --- a/src/common/execution/index.js +++ b/src/common/compute/index.js @@ -1,28 +1,27 @@ /*globals define, requirejs */ const COMPUTE_BACKENDS = ['gme', 'local']; - define([ - 'underscore', + 'q', 'module' -].concat(COMPUTE_BACKENDS.map(name => `deepforge/execution/backends/${name}/index`)), +].concat(COMPUTE_BACKENDS.map(name => `deepforge/compute/backends/${name}/index`)), function( - _, + Q, module ) { - const Execution = {}; + const Compute = {}; - Execution.getBackend = function(name) { + Compute.getBackend = function(name) { name = name.toLowerCase(); if (!COMPUTE_BACKENDS.includes(name)) { - throw new Error(`Execution backend not found: ${name}`); + throw new Error(`Compute backend not found: ${name}`); } const relativePath = `backends/${name}/index`; - const Backend = requirejs(`deepforge/execution/${relativePath}`); + const Backend = requirejs(`deepforge/compute/${relativePath}`); return new Backend(); }; - Execution.getAvailableBackends = function() { + Compute.getAvailableBackends = function() { const settings = {backends: ['local', 'gme']}; if (require.isBrowser) { const ComponentSettings = requirejs('js/Utils/ComponentSettings'); @@ -34,15 +33,15 @@ function( const path = require('path'); const dirname = path.dirname(module.uri); const deploymentSettings = JSON.parse(requirejs('text!' + dirname + '/../../../config/components.json')); - _.extend(settings, deploymentSettings[this.getComponentId()]); + Object.assign(settings, deploymentSettings[this.getComponentId()]); } return settings.backends; }; - Execution.getComponentId = function() { + Compute.getComponentId = function() { return 'Compute'; }; - return Execution; + return Compute; }); diff --git a/src/common/execution/backends/BaseBackend.js b/src/common/execution/backends/BaseBackend.js deleted file mode 100644 index e01e01ed5..000000000 --- a/src/common/execution/backends/BaseBackend.js +++ /dev/null @@ -1,29 +0,0 @@ -/* globals define, requirejs */ -define([ - 'module' -], function( - module -) { - - const BaseBackend = function(name) { - console.log(`Creating backend: ${name}`); - this.name = name; - }; - - BaseBackend.prototype.getClient = function(logger) { - const path = require('path'); - const dirname = path.dirname(module.uri); - const Client = requirejs(`${dirname}/${this.name.toLowerCase()}/Client.js`); - return new Client(logger); - }; - - BaseBackend.prototype.getOptions = function() { - return []; - }; - - BaseBackend.prototype.getDashboard = function() { - return null; - }; - - return BaseBackend; -}); diff --git a/src/common/execution/backends/gme/index.js b/src/common/execution/backends/gme/index.js deleted file mode 100644 index 54a13b8b5..000000000 --- a/src/common/execution/backends/gme/index.js +++ /dev/null @@ -1,21 +0,0 @@ -/* globals define */ -define([ - './dashboard/index', - '../BaseBackend' -], function( - Dashboard, - BaseBackend -) { - - const GMEBackend = function() { - BaseBackend.call(this, 'GME'); - }; - - GMEBackend.prototype = Object.create(BaseBackend.prototype); - - GMEBackend.prototype.getDashboard = function() { - return Dashboard; - }; - - return GMEBackend; -}); diff --git a/src/common/execution/backends/local/index.js b/src/common/execution/backends/local/index.js deleted file mode 100644 index 64f8c74ac..000000000 --- a/src/common/execution/backends/local/index.js +++ /dev/null @@ -1,15 +0,0 @@ -/* globals define */ -define([ - '../BaseBackend' -], function( - BaseBackend -) { - - const LocalBackend = function() { - BaseBackend.call(this, 'Local'); - }; - - LocalBackend.prototype = Object.create(BaseBackend.prototype); - - return LocalBackend; -}); diff --git a/src/common/utils.js b/src/common/utils.js index 985acb56a..6cd4163de 100644 --- a/src/common/utils.js +++ b/src/common/utils.js @@ -84,7 +84,6 @@ return lines; }; - return { getSetterSchema: getSetterSchema, resolveCarriageReturns: resolveCarriageReturns, From 09e87d1209aae8085fd03f4b628e4ee5b92c6362 Mon Sep 17 00:00:00 2001 From: Brian Broll Date: Wed, 18 Sep 2019 16:21:18 -0500 Subject: [PATCH 34/45] WIP (more) rename execution-compute --- src/plugins/ExecuteJob/ExecuteJob.js | 95 ++++--------------- .../panels/WorkerHeader/ComputeDialog.js | 25 +++-- .../panels/WorkerHeader/EmptyDashboard.js | 9 +- 3 files changed, 41 insertions(+), 88 deletions(-) diff --git a/src/plugins/ExecuteJob/ExecuteJob.js b/src/plugins/ExecuteJob/ExecuteJob.js index 7b22108c5..ebaddbd15 100644 --- a/src/plugins/ExecuteJob/ExecuteJob.js +++ b/src/plugins/ExecuteJob/ExecuteJob.js @@ -1,10 +1,10 @@ -/*globals define, requirejs*/ +/*globals define */ /*jshint node:true, browser:true*/ define([ 'common/util/assert', 'text!./metadata.json', - 'deepforge/execution/index', + 'deepforge/compute/index', 'plugin/PluginBase', 'deepforge/plugin/LocalExecutor', 'deepforge/plugin/PtrCodeGen', @@ -22,7 +22,7 @@ define([ ], function ( assert, pluginMetadata, - Execution, + Compute, PluginBase, LocalExecutor, // DeepForge operation primitives PtrCodeGen, @@ -95,10 +95,10 @@ define([ this.pulseClient = new ExecPulseClient(params); this._execHashToJobNode = {}; - const name = Execution.getAvailableBackends()[0]; // FIXME: enable the user to select one - const backend = Execution.getBackend(name); - this.executor = backend.getClient(this.logger); - this.executor.on( + const name = Compute.getAvailableBackends()[0]; // FIXME: enable the user to select one + const backend = Compute.getBackend(name); + this.compute = backend.getClient(this.logger); + this.compute.on( 'data', (id, data) => { const job = this.getNodeForJobId(id); @@ -106,7 +106,7 @@ define([ } ); - this.executor.on('update', (jobId, status) => { + this.compute.on('update', (jobId, status) => { try { this.onUpdate(jobId, status); } catch (err) { @@ -114,7 +114,7 @@ define([ } }); - this.executor.on('end', + this.compute.on('end', (id, info) => { try { this.onOperationEnd(id); @@ -190,11 +190,11 @@ define([ ExecuteJob.prototype.onAbort = ExecuteJob.prototype.onUserCancelDetected = function () { - this.logger.info('>>>>> Received Abort. Canceling jobs.'); + this.logger.info('Received Abort. Canceling jobs.'); this.runningJobHashes .map(hash => this.getNodeForJobId(hash)) .map(node => JSON.parse(this.getAttribute(node, 'jobInfo'))) - .forEach(jobInfo => this.executor.cancelJob(jobInfo)); + .forEach(jobInfo => this.compute.cancelJob(jobInfo)); }; ExecuteJob.prototype.isResuming = async function (job) { @@ -240,9 +240,7 @@ define([ this.outputLineCount[id] = count; - // TODO: Need to be able to poll the stdout... - const stdout = await this.executor.getConsoleOutput(hash); - + const stdout = await this.compute.getConsoleOutput(hash); const result = this.processStdout(job, stdout); if (result.hasMetadata) { @@ -384,7 +382,7 @@ define([ const node = await this.getOperation(job); const name = this.getAttribute(node, 'name'); - // Execute any special operation types here - not on an executor + // Execute any special operation types here - not on an compute this.logger.debug(`Executing operation "${name}"`); if (this.isLocalOperation(node)) { return this.executeLocalOperation(node); @@ -457,7 +455,7 @@ define([ ExecuteJob.prototype.createJob = async function (job, opNode, hash) { // Record the job info for the given hash this._execHashToJobNode[hash] = job; - const jobInfo = await this.executor.createJob(hash); + const jobInfo = await this.compute.createJob(hash); this.setAttribute(job, 'jobInfo', JSON.stringify(jobInfo)); if (!this.currentRunId) { this.currentRunId = jobInfo.hash; @@ -553,52 +551,12 @@ define([ let last = stdout.lastIndexOf('\n'); let lastLine; - //const currentLine = this.outputLineCount[jobId]; - //const actualLine = info.outputNumber; // TODO: Keep this?? - - //// TODO: Move this to the onOutput handler - //if (actualLine !== null && actualLine >= currentLine) { - //this.outputLineCount[jobId] = actualLine + 1; - //// TODO: Keep this?? - //let output = await this.executor.getConsoleOutput(hash, currentLine, actualLine+1)) - - //var stdout = this.getAttribute(job, 'stdout'), - //last = stdout.lastIndexOf('\n'), - //result, - //lastLine, - //next = Q(), - //msg; - - // parse deepforge commands if (last !== -1) { stdout = stdout.substring(0, last+1); lastLine = stdout.substring(last+1); output = lastLine + output; } - //result = this.processStdout(job, output, true); - //output = result.stdout; - - //if (output) { - //// Send notification to all clients watching the branch - //const metadata = { - //lineCount: this.outputLineCount[jobId] - //}; - //await this.logManager.appendTo(jobId, output, metadata); - //await this.notifyStdoutUpdate(jobId); - //} - //if (result.hasMetadata) { - //msg = `Updated graph/image output for ${name}`; - //await this.save(msg); - //} - //} - - // parse deepforge commands - // FIXME: This needs to be added back! - //if (last !== -1) { - //stdout = stdout.substring(0, last+1); - //lastLine = stdout.substring(last+1); - //output = lastLine + output; - //} + const result = this.processStdout(job, output, true); output = result.stdout; @@ -612,19 +570,6 @@ define([ } }; - // TODO: Watch if the execution is ever canceled (on the origin branch). - // If so, cancel the executing jobs - /* - if (this.canceled || this.isExecutionCanceled()) { - if (jobInfo) { - this.executor.cancelJob(jobInfo); - this.delAttribute(job, 'jobInfo'); - this.canceled = true; - return this.onOperationCanceled(op); - } - } - */ - ExecuteJob.prototype.onOperationEnd = async function (hash) { // Record that the job hash is no longer running const job = this.getNodeForJobId(hash); @@ -633,11 +578,11 @@ define([ const jobId = this.core.getPath(job); const jobInfo = JSON.parse(this.getAttribute(job, 'jobInfo')); - const status = await this.executor.getStatus(jobInfo); + const status = await this.compute.getStatus(jobInfo); this.logger.info(`Job "${name}" has finished (${status})`); this.cleanJobHashInfo(hash); - if (status === this.executor.CANCELED) { + if (status === this.compute.CANCELED) { // If it was canceled, the pipeline has been stopped this.logger.debug(`"${name}" has been CANCELED!`); this.canceled = true; @@ -646,8 +591,8 @@ define([ return this.onOperationCanceled(op); } - if (status === this.executor.SUCCESS || status === this.executor.FAILED) { - const fileHashes = await this.executor.getOutputHashes(jobInfo); + if (status === this.compute.SUCCESS || status === this.compute.FAILED) { + const fileHashes = await this.compute.getOutputHashes(jobInfo); const execFilesHash = fileHashes[name + '-all-files']; this.setAttribute(job, 'execFiles', execFilesHash); @@ -659,7 +604,7 @@ define([ // Parse the remaining code this.setAttribute(job, 'stdout', result.stdout); this.logManager.deleteLog(jobId); - if (status === this.executor.SUCCESS) { + if (status === this.compute.SUCCESS) { this.onDistOperationComplete(op, fileHashes); } else { // Download all files diff --git a/src/visualizers/panels/WorkerHeader/ComputeDialog.js b/src/visualizers/panels/WorkerHeader/ComputeDialog.js index 003325be0..2f2a65231 100644 --- a/src/visualizers/panels/WorkerHeader/ComputeDialog.js +++ b/src/visualizers/panels/WorkerHeader/ComputeDialog.js @@ -1,6 +1,6 @@ /* globals define, $ */ define([ - 'deepforge/execution/index', + 'deepforge/compute/index', 'q', 'deepforge/viz/Utils', './EmptyDashboard', @@ -8,7 +8,7 @@ define([ 'text!./ComputeModal.html.ejs', 'css!./ComputeModal.css' ], function( - Execution, + Compute, Q, utils, EmptyDashboard, @@ -22,16 +22,24 @@ define([ this.active = false; this.logger = logger.fork('ComputeDialog'); - const backendNames = Execution.getAvailableBackends(); + const backendNames = Compute.getAvailableBackends(); this.$el = $(ComputeHtmlTpl({tabs: backendNames})); - this.dashboards = backendNames - .map(name => { - const backend = Execution.getBackend(name); - const Dashboard = backend.getDashboard() || EmptyDashboard.bind(null, name); + this.backends = backendNames; + this.dashboards = null; + }; + + ComputeDialog.prototype.loadDashboards = async function() { + const fetchDashboards = this.backends + .map(async name => { + const backend = Compute.getBackend(name); + console.log('loading dashboard for', name); + const Dashboard = await backend.getDashboard() || EmptyDashboard.bind(null, name); const $container = this.$el.find(`#${name}-dashboard-container`); return new Dashboard(this.logger, $container); }); + + this.dashboards = await Promise.all(fetchDashboards); }; ComputeDialog.prototype.initialize = function() { @@ -39,8 +47,9 @@ define([ this.$el.on('hidden.bs.modal', () => this.onHide()); }; - ComputeDialog.prototype.show = function() { + ComputeDialog.prototype.show = async function() { this.initialize(); + await this.loadDashboards(); this.dashboards.forEach(dashboard => dashboard.onShow()); }; diff --git a/src/visualizers/panels/WorkerHeader/EmptyDashboard.js b/src/visualizers/panels/WorkerHeader/EmptyDashboard.js index 17fc80882..c86b6caf8 100644 --- a/src/visualizers/panels/WorkerHeader/EmptyDashboard.js +++ b/src/visualizers/panels/WorkerHeader/EmptyDashboard.js @@ -1,7 +1,9 @@ /* globals define, $ */ define([ - 'css!./EmptyDashboard.css' + 'deepforge/compute/backends/ComputeDashboard', + 'css!./EmptyDashboard.css', ], function( + ComputeDashboard ) { const EmptyDashboard = function(name, logger, $container) { @@ -11,10 +13,7 @@ define([ $container.append(this.$el); }; - EmptyDashboard.prototype.onShow = - EmptyDashboard.prototype.onHide = - EmptyDashboard.prototype.onActivate = - EmptyDashboard.prototype.onDeactivate = () => {}; + EmptyDashboard.prototype = Object.create(ComputeDashboard.prototype); return EmptyDashboard; }); From 4169b1fa4e53860f9a967e837d237d8698cb4aee Mon Sep 17 00:00:00 2001 From: Brian Broll Date: Wed, 18 Sep 2019 17:33:09 -0500 Subject: [PATCH 35/45] WIP Changed from index.js to metadata.json files --- src/common/compute/backends/ComputeBackend.js | 21 +++- src/common/compute/backends/ComputeClient.js | 110 +++++++++--------- src/common/compute/backends/JobResults.js | 20 ++-- src/common/compute/backends/gme/index.js | 20 ---- src/common/compute/backends/gme/metadata.json | 5 + src/common/compute/backends/local/index.js | 15 --- .../compute/backends/local/metadata.json | 5 + src/common/compute/index.js | 24 ++-- 8 files changed, 108 insertions(+), 112 deletions(-) delete mode 100644 src/common/compute/backends/gme/index.js create mode 100644 src/common/compute/backends/gme/metadata.json delete mode 100644 src/common/compute/backends/local/index.js create mode 100644 src/common/compute/backends/local/metadata.json diff --git a/src/common/compute/backends/ComputeBackend.js b/src/common/compute/backends/ComputeBackend.js index 1a4d7e894..99b16e69e 100644 --- a/src/common/compute/backends/ComputeBackend.js +++ b/src/common/compute/backends/ComputeBackend.js @@ -7,19 +7,30 @@ define([ Q, ) { - const ComputeBackend = function(name) { + const ComputeBackend = function(id, metadata) { + const {name, dashboard, client} = metadata; + this.id = id; this.name = name; + this.dashboardPath = dashboard; + this.clientPath = client || './Client'; }; ComputeBackend.prototype.getClient = function(logger) { - const path = require('path'); - const dirname = path.dirname(module.uri); - const Client = requirejs(`${dirname}/${this.name.toLowerCase()}/Client.js`); + if (require.isBrowser) { + throw new Error('Compute clients cannot be loaded in the browser.'); + } + + const Client = requirejs(`deepforge/compute/backends/${this.id}/${this.clientPath}`); return new Client(logger); }; ComputeBackend.prototype.getDashboard = async function() { - return null; + if (this.dashboardPath) { + const absPath = `deepforge/compute/backends/${this.id}/${this.dashboardPath}`; + return await this.require(absPath); + } else { + return null; + } }; ComputeBackend.prototype.require = function(path) { // helper for loading async diff --git a/src/common/compute/backends/ComputeClient.js b/src/common/compute/backends/ComputeClient.js index 1cd97981d..7d09f533b 100644 --- a/src/common/compute/backends/ComputeClient.js +++ b/src/common/compute/backends/ComputeClient.js @@ -1,65 +1,67 @@ -const ComputeClient = function(logger, gmeConfig) { - const isHttps = typeof window === 'undefined' ? false : - window.location.protocol !== 'http:'; +define([], function() { + const ComputeClient = function(logger, gmeConfig) { + const isHttps = typeof window === 'undefined' ? false : + window.location.protocol !== 'http:'; - this.logger = logger.fork('executor'); - this._events = {}; -}; + this.logger = logger.fork('executor'); + this._events = {}; + }; -ComputeClient.prototype.cancelJob = function(job) { - const msg = `cancelJob is not implemented for current executor backend!`; - this.logger.warn(msg); - throw new Error(msg); -}; + ComputeClient.prototype.cancelJob = function(job) { + const msg = `cancelJob is not implemented for current executor backend!`; + this.logger.warn(msg); + throw new Error(msg); + }; -ComputeClient.prototype.getInfo = function(job) { - const msg = `getInfo is not implemented for current executor backend!`; - this.logger.warn(msg); - throw new Error(msg); -}; + ComputeClient.prototype.getInfo = function(job) { + const msg = `getInfo is not implemented for current executor backend!`; + this.logger.warn(msg); + throw new Error(msg); + }; -ComputeClient.prototype.createJob = async function(hash) { - const msg = `createJob is not implemented for current executor backend!`; - this.logger.warn(msg); - throw new Error(msg); -}; + ComputeClient.prototype.createJob = async function(hash) { + const msg = `createJob is not implemented for current executor backend!`; + this.logger.warn(msg); + throw new Error(msg); + }; -ComputeClient.prototype.getStatus = async function(jobInfo) { - const msg = `getStatus is not implemented for current executor backend!`; - this.logger.warn(msg); - throw new Error(msg); -}; + ComputeClient.prototype.getStatus = async function(jobInfo) { + const msg = `getStatus is not implemented for current executor backend!`; + this.logger.warn(msg); + throw new Error(msg); + }; -ComputeClient.prototype.getOutputHashes = async function(jobInfo) { - const msg = `getOutputHashes is not implemented for current executor backend!`; - this.logger.warn(msg); - throw new Error(msg); -}; + ComputeClient.prototype.getOutputHashes = async function(jobInfo) { + const msg = `getOutputHashes is not implemented for current executor backend!`; + this.logger.warn(msg); + throw new Error(msg); + }; -ComputeClient.prototype.getConsoleOutput = async function(hash) { - const msg = `getConsoleOutput is not implemented for current executor backend!`; - this.logger.warn(msg); - throw new Error(msg); -}; + ComputeClient.prototype.getConsoleOutput = async function(hash) { + const msg = `getConsoleOutput is not implemented for current executor backend!`; + this.logger.warn(msg); + throw new Error(msg); + }; -// Some functions for event support -ComputeClient.prototype.on = function(ev, cb) { - this._events[ev] = this._events[ev] || []; - this._events[ev].push(cb); -}; + // Some functions for event support + ComputeClient.prototype.on = function(ev, cb) { + this._events[ev] = this._events[ev] || []; + this._events[ev].push(cb); + }; -ComputeClient.prototype.emit = function(ev) { - const args = Array.prototype.slice.call(arguments, 1); - const handlers = this._events[ev] || []; - handlers.forEach(fn => fn.apply(this, args)); -}; + ComputeClient.prototype.emit = function(ev) { + const args = Array.prototype.slice.call(arguments, 1); + const handlers = this._events[ev] || []; + handlers.forEach(fn => fn.apply(this, args)); + }; -ComputeClient.prototype.QUEUED = 'queued'; -ComputeClient.prototype.PENDING = 'pending'; -ComputeClient.prototype.RUNNING = 'running'; -ComputeClient.prototype.SUCCESS = 'success'; -ComputeClient.prototype.FAILED = 'failed'; -ComputeClient.prototype.CANCELED = 'canceled'; -ComputeClient.prototype.NOT_FOUND = 'NOT_FOUND'; + ComputeClient.prototype.QUEUED = 'queued'; + ComputeClient.prototype.PENDING = 'pending'; + ComputeClient.prototype.RUNNING = 'running'; + ComputeClient.prototype.SUCCESS = 'success'; + ComputeClient.prototype.FAILED = 'failed'; + ComputeClient.prototype.CANCELED = 'canceled'; + ComputeClient.prototype.NOT_FOUND = 'NOT_FOUND'; -module.exports = ComputeClient; + return ComputeClient; +}); diff --git a/src/common/compute/backends/JobResults.js b/src/common/compute/backends/JobResults.js index 48c5655db..b0fe4d806 100644 --- a/src/common/compute/backends/JobResults.js +++ b/src/common/compute/backends/JobResults.js @@ -1,10 +1,16 @@ -const ComputeClient = require('./ComputeClient'); +/* globals define */ +define([ + './ComputeClient', +], function( + ComputeClient, +) { -class JobResults { - constructor(status=ComputeClient.prototype.CREATED) { - this.status = status; - this.resultHashes = []; + class JobResults { + constructor(status=ComputeClient.prototype.CREATED) { + this.status = status; + this.resultHashes = []; + } } -} -module.exports = JobResults; + return JobResults; +}); diff --git a/src/common/compute/backends/gme/index.js b/src/common/compute/backends/gme/index.js deleted file mode 100644 index 4f1ab9f16..000000000 --- a/src/common/compute/backends/gme/index.js +++ /dev/null @@ -1,20 +0,0 @@ -/* globals define */ -define([ - '../ComputeBackend', -], function( - ComputeBackend, -) { - - const GMEBackend = function() { - ComputeBackend.call(this, 'GME'); - }; - - GMEBackend.prototype = Object.create(ComputeBackend.prototype); - - GMEBackend.prototype.getDashboard = async function() { - console.log(this, this.require); - return await this.require('deepforge/compute/backends/gme/dashboard/index'); - }; - - return GMEBackend; -}); diff --git a/src/common/compute/backends/gme/metadata.json b/src/common/compute/backends/gme/metadata.json new file mode 100644 index 000000000..04cffa6e7 --- /dev/null +++ b/src/common/compute/backends/gme/metadata.json @@ -0,0 +1,5 @@ +{ + "name": "WebGME Executor Framework", + "dashboard": "./dashboard/index", + "client": "./Client" +} diff --git a/src/common/compute/backends/local/index.js b/src/common/compute/backends/local/index.js deleted file mode 100644 index ed546dcd8..000000000 --- a/src/common/compute/backends/local/index.js +++ /dev/null @@ -1,15 +0,0 @@ -/* globals define */ -define([ - '../ComputeBackend' -], function( - ComputeBackend -) { - - const LocalBackend = function() { - ComputeBackend.call(this, 'Local'); - }; - - LocalBackend.prototype = Object.create(ComputeBackend.prototype); - - return LocalBackend; -}); diff --git a/src/common/compute/backends/local/metadata.json b/src/common/compute/backends/local/metadata.json new file mode 100644 index 000000000..b5e0f9383 --- /dev/null +++ b/src/common/compute/backends/local/metadata.json @@ -0,0 +1,5 @@ +{ + "name": "Server Execution", + "dashboard": null, + "client": "./Client" +} diff --git a/src/common/compute/index.js b/src/common/compute/index.js index ce09c8dc5..1cb74b77e 100644 --- a/src/common/compute/index.js +++ b/src/common/compute/index.js @@ -2,27 +2,29 @@ const COMPUTE_BACKENDS = ['gme', 'local']; define([ 'q', - 'module' -].concat(COMPUTE_BACKENDS.map(name => `deepforge/compute/backends/${name}/index`)), + 'module', + 'deepforge/compute/backends/ComputeBackend', +].concat(COMPUTE_BACKENDS.map(name => `text!deepforge/compute/backends/${name}/metadata.json`)), function( Q, - module + module, + ComputeBackend, ) { const Compute = {}; - Compute.getBackend = function(name) { - name = name.toLowerCase(); - if (!COMPUTE_BACKENDS.includes(name)) { - throw new Error(`Compute backend not found: ${name}`); + Compute.getBackend = function(id) { + id = id.toLowerCase(); + if (!COMPUTE_BACKENDS.includes(id)) { + throw new Error(`Compute backend not found: ${id}`); } - const relativePath = `backends/${name}/index`; - const Backend = requirejs(`deepforge/compute/${relativePath}`); - return new Backend(); + const relativePath = `backends/${id}/metadata.json`; + const metadata = JSON.parse(requirejs(`text!deepforge/compute/${relativePath}`)); + return new ComputeBackend(id, metadata); }; Compute.getAvailableBackends = function() { - const settings = {backends: ['local', 'gme']}; + const settings = {backends: COMPUTE_BACKENDS}; // all by default if (require.isBrowser) { const ComponentSettings = requirejs('js/Utils/ComponentSettings'); ComponentSettings.resolveWithWebGMEGlobal( From 5b7c3da43f05b38e680550ee5931b214a6aafabe Mon Sep 17 00:00:00 2001 From: Brian Broll Date: Wed, 18 Sep 2019 17:33:53 -0500 Subject: [PATCH 36/45] WIP updated dashboard to use actual name --- .../panels/WorkerHeader/ComputeDialog.js | 27 ++++++++++--------- .../panels/WorkerHeader/ComputeModal.html.ejs | 4 +-- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/src/visualizers/panels/WorkerHeader/ComputeDialog.js b/src/visualizers/panels/WorkerHeader/ComputeDialog.js index 2f2a65231..443918be3 100644 --- a/src/visualizers/panels/WorkerHeader/ComputeDialog.js +++ b/src/visualizers/panels/WorkerHeader/ComputeDialog.js @@ -22,34 +22,35 @@ define([ this.active = false; this.logger = logger.fork('ComputeDialog'); - const backendNames = Compute.getAvailableBackends(); - this.$el = $(ComputeHtmlTpl({tabs: backendNames})); - this.backends = backendNames; + this.$el = null; + this.backends = null; this.dashboards = null; }; ComputeDialog.prototype.loadDashboards = async function() { - const fetchDashboards = this.backends - .map(async name => { - const backend = Compute.getBackend(name); - console.log('loading dashboard for', name); - const Dashboard = await backend.getDashboard() || EmptyDashboard.bind(null, name); - const $container = this.$el.find(`#${name}-dashboard-container`); + }; + + ComputeDialog.prototype.initialize = async function() { + const backends = Compute.getAvailableBackends() + .map(name => Compute.getBackend(name)); + + this.$el = $(ComputeHtmlTpl({tabs: backends})); + const fetchDashboards = backends + .map(async backend => { + const Dashboard = await backend.getDashboard() || EmptyDashboard.bind(null, backend.name); + const $container = this.$el.find(`#${backend.id}-dashboard-container`); return new Dashboard(this.logger, $container); }); this.dashboards = await Promise.all(fetchDashboards); - }; - ComputeDialog.prototype.initialize = function() { this.$el.modal('show'); this.$el.on('hidden.bs.modal', () => this.onHide()); }; ComputeDialog.prototype.show = async function() { - this.initialize(); - await this.loadDashboards(); + await this.initialize(); this.dashboards.forEach(dashboard => dashboard.onShow()); }; diff --git a/src/visualizers/panels/WorkerHeader/ComputeModal.html.ejs b/src/visualizers/panels/WorkerHeader/ComputeModal.html.ejs index 8a8e047ea..60fbc5f9c 100644 --- a/src/visualizers/panels/WorkerHeader/ComputeModal.html.ejs +++ b/src/visualizers/panels/WorkerHeader/ComputeModal.html.ejs @@ -11,12 +11,12 @@
<% tabs.forEach((tab, i) => { %> -
+
<% }); %>
From 385b2a60a89301d6367d484be16ad806aab5a056 Mon Sep 17 00:00:00 2001 From: Brian Broll Date: Wed, 18 Sep 2019 17:51:43 -0500 Subject: [PATCH 37/45] WIP renamed ExecutionEnv->ExecutorHelper --- src/common/compute/backends/gme/Client.js | 6 +++--- .../backends/gme/ExecutorHelper.js} | 15 +++++++-------- .../compute/backends/gme/dashboard/index.js | 8 ++++---- 3 files changed, 14 insertions(+), 15 deletions(-) rename src/common/{ExecutionEnv.js => compute/backends/gme/ExecutorHelper.js} (80%) diff --git a/src/common/compute/backends/gme/Client.js b/src/common/compute/backends/gme/Client.js index 430e32631..8f51b5318 100644 --- a/src/common/compute/backends/gme/Client.js +++ b/src/common/compute/backends/gme/Client.js @@ -2,14 +2,14 @@ define([ '../ComputeClient', '../JobResults', - 'deepforge/ExecutionEnv', + './ExecutorHelper', 'executor/ExecutorClient', 'path', 'module', ], function( ComputeClient, JobResults, - ExecutionEnv, + ExecutorHelper, ExecutorClient, path, module, @@ -75,7 +75,7 @@ define([ GMEExecutor.prototype.checkExecutionEnv = async function () { this.logger.info(`Checking execution environment`); - const workers = await ExecutionEnv.getWorkers(); + const workers = await ExecutorHelper.getWorkers(); if (workers.length === 0) { this.logger.info(`Cannot execute job(s): No connected workers`); throw new Error('No connected workers'); diff --git a/src/common/ExecutionEnv.js b/src/common/compute/backends/gme/ExecutorHelper.js similarity index 80% rename from src/common/ExecutionEnv.js rename to src/common/compute/backends/gme/ExecutorHelper.js index e4f006f8f..dcd2c5dd7 100644 --- a/src/common/ExecutionEnv.js +++ b/src/common/compute/backends/gme/ExecutorHelper.js @@ -11,19 +11,18 @@ define([ const WORKER_ENDPOINT = '/rest/executor/worker'; const JOBS_ENDPOINT = '/rest/executor'; const values = dict => Object.keys(dict).map(k => dict[k]); + const ExecutorHelper = {}; - const ExecutionEnv = {}; - - ExecutionEnv.url = function(urlPath) { + ExecutorHelper.url = function(urlPath) { if (typeof window === 'undefined') { - const configPath = module.uri.replace('src/common/ExecutionEnv.js', 'config/index.js'); + const configPath = module.uri.replace('src/common/ExecutorHelper.js', 'config/index.js'); const gmeConfig = require.nodeRequire(configPath); return `http://127.0.0.1:${gmeConfig.server.port}${urlPath}`; } return urlPath; }; - ExecutionEnv.get = function(urlPath) { + ExecutorHelper.get = function(urlPath) { const deferred = Q.defer(); const url = this.url(urlPath); @@ -38,15 +37,15 @@ define([ return deferred.promise; }; - ExecutionEnv.getWorkers = function() { + ExecutorHelper.getWorkers = function() { return Q(1); // TODO return this.get(WORKER_ENDPOINT) .then(workerDict => values(workerDict)); }; - ExecutionEnv.getJobs = function() { + ExecutorHelper.getJobs = function() { return this.get(JOBS_ENDPOINT); }; - return ExecutionEnv; + return ExecutorHelper; }); diff --git a/src/common/compute/backends/gme/dashboard/index.js b/src/common/compute/backends/gme/dashboard/index.js index d4e668b86..cc482082e 100644 --- a/src/common/compute/backends/gme/dashboard/index.js +++ b/src/common/compute/backends/gme/dashboard/index.js @@ -1,7 +1,7 @@ /* globals define, $ */ define([ '../../ComputeDashboard', - 'deepforge/ExecutionEnv', + '../ExecutorHelper', 'q', 'deepforge/viz/Utils', 'deepforge/api/JobOriginClient', @@ -11,7 +11,7 @@ define([ 'css!./WorkerModal.css' ], function( ComputeDashboard, - ExecutionEnv, + ExecutorHelper, Q, utils, JobOriginClient, @@ -57,8 +57,8 @@ define([ }; WorkerDialog.prototype.update = async function() { - const workers = await ExecutionEnv.getWorkers(); - const jobs = await ExecutionEnv.getJobs(); + const workers = await ExecutorHelper.getWorkers(); + const jobs = await ExecutorHelper.getJobs(); await Q.all([this.updateWorkers(workers), this.updateJobs(jobs)]); From a631a3ea28fa639e887f10e532f342352c1dd6ce Mon Sep 17 00:00:00 2001 From: Brian Broll Date: Thu, 19 Sep 2019 08:22:02 -0500 Subject: [PATCH 38/45] WIP #1186 delete executionId attribute on completion --- src/plugins/ExecuteJob/ExecuteJob.js | 3 ++- src/plugins/ExecutePipeline/ExecutePipeline.js | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/plugins/ExecuteJob/ExecuteJob.js b/src/plugins/ExecuteJob/ExecuteJob.js index ebaddbd15..c2c26a793 100644 --- a/src/plugins/ExecuteJob/ExecuteJob.js +++ b/src/plugins/ExecuteJob/ExecuteJob.js @@ -325,9 +325,10 @@ define([ const jobId = this.core.getPath(job); const status = err ? 'fail' : (this.canceled ? 'canceled' : 'success'); const msg = err ? `${name} execution failed!` : - `${name} executed successfully!`; + `${name} executed successfully!`; this.setAttribute(job, 'status', status); + this.delAttribute(job, 'executionId'); this.logger.info(`Setting ${name} (${jobId}) status to ${status}`); this.clearOldMetadata(job); diff --git a/src/plugins/ExecutePipeline/ExecutePipeline.js b/src/plugins/ExecutePipeline/ExecutePipeline.js index abaf83209..08bc4f171 100644 --- a/src/plugins/ExecutePipeline/ExecutePipeline.js +++ b/src/plugins/ExecutePipeline/ExecutePipeline.js @@ -440,6 +440,7 @@ define([ (this.canceled ? 'canceled' : 'success') ) ); + this.delAttribute(this.activeNode, 'executionId'); this._finished = true; this.resultMsg(msg); From 931980786d7a65fa001a95e7989b5167e3a69365 Mon Sep 17 00:00:00 2001 From: Brian Broll Date: Thu, 19 Sep 2019 09:07:21 -0500 Subject: [PATCH 39/45] Ensure gme backend is used by default (don't change default behavior) --- config/components.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/components.json b/config/components.json index 7eb461c07..939cf1317 100644 --- a/config/components.json +++ b/config/components.json @@ -131,6 +131,6 @@ ] }, "Compute": { - "backends": ["local", "gme"] + "backends": ["gme", "local"] } } From 8da2adfa44f360492304f5d94f1ebe6b4f79cbe7 Mon Sep 17 00:00:00 2001 From: Brian Broll Date: Thu, 19 Sep 2019 09:08:28 -0500 Subject: [PATCH 40/45] WIP fixed bug in ExecuteJob --- src/plugins/ExecuteJob/ExecuteJob.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/plugins/ExecuteJob/ExecuteJob.js b/src/plugins/ExecuteJob/ExecuteJob.js index c2c26a793..9f3879fd8 100644 --- a/src/plugins/ExecuteJob/ExecuteJob.js +++ b/src/plugins/ExecuteJob/ExecuteJob.js @@ -44,7 +44,7 @@ define([ pluginMetadata = JSON.parse(pluginMetadata); - var STDOUT_FILE = 'job_stdout.txt'; + const STDOUT_FILE = 'job_stdout.txt'; /** * Initializes a new instance of ExecuteJob. @@ -177,7 +177,6 @@ define([ this.currentRunId = null; // will be set after exec files created return this.executeJob(this.activeNode); } - //.catch(err => this._callback(err, this.result)); // TODO: Is this needed? }; ExecuteJob.prototype.getJobId = function (node) { @@ -566,6 +565,7 @@ define([ await this.notifyStdoutUpdate(jobId); if (result.hasMetadata) { + const name = this.getAttribute(job, 'name'); const msg = `Updated graph/image output for ${name}`; await this.save(msg); } From e95587bbbd05d96aca62159143e0dd1c63e98113 Mon Sep 17 00:00:00 2001 From: Brian Broll Date: Thu, 19 Sep 2019 09:09:01 -0500 Subject: [PATCH 41/45] WIP update terminology --- src/plugins/ExecuteJob/ExecuteJob.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/ExecuteJob/ExecuteJob.js b/src/plugins/ExecuteJob/ExecuteJob.js index 9f3879fd8..72acf2473 100644 --- a/src/plugins/ExecuteJob/ExecuteJob.js +++ b/src/plugins/ExecuteJob/ExecuteJob.js @@ -73,7 +73,7 @@ define([ this.logManager = null; }; - // TODO: Update plugin metadata for the executor options + // TODO: Update plugin metadata for the compute options ExecuteJob.metadata = pluginMetadata; ExecuteJob.HEARTBEAT_INTERVAL = 2500; From f807c9667d9f04e06dff84023ea7b9442c046b86 Mon Sep 17 00:00:00 2001 From: Brian Broll Date: Thu, 19 Sep 2019 09:20:52 -0500 Subject: [PATCH 42/45] WIP Fixed linting issues --- src/common/compute/backends/ComputeClient.js | 32 +++++++++---------- .../compute/backends/gme/ExecutorHelper.js | 1 - .../backends/gme/dashboard/WorkerModal.css | 20 ++++++------ 3 files changed, 25 insertions(+), 28 deletions(-) diff --git a/src/common/compute/backends/ComputeClient.js b/src/common/compute/backends/ComputeClient.js index 7d09f533b..373e8ae3d 100644 --- a/src/common/compute/backends/ComputeClient.js +++ b/src/common/compute/backends/ComputeClient.js @@ -1,44 +1,42 @@ +/* globals define */ define([], function() { - const ComputeClient = function(logger, gmeConfig) { - const isHttps = typeof window === 'undefined' ? false : - window.location.protocol !== 'http:'; - - this.logger = logger.fork('executor'); + const ComputeClient = function(logger) { + this.logger = logger.fork('compute'); this._events = {}; }; - ComputeClient.prototype.cancelJob = function(job) { - const msg = `cancelJob is not implemented for current executor backend!`; + ComputeClient.prototype.cancelJob = function(/*job*/) { + const msg = `cancelJob is not implemented for current compute backend!`; this.logger.warn(msg); throw new Error(msg); }; - ComputeClient.prototype.getInfo = function(job) { - const msg = `getInfo is not implemented for current executor backend!`; + ComputeClient.prototype.getInfo = function(/*job*/) { + const msg = `getInfo is not implemented for current compute backend!`; this.logger.warn(msg); throw new Error(msg); }; - ComputeClient.prototype.createJob = async function(hash) { - const msg = `createJob is not implemented for current executor backend!`; + ComputeClient.prototype.createJob = async function(/*hash*/) { + const msg = `createJob is not implemented for current compute backend!`; this.logger.warn(msg); throw new Error(msg); }; - ComputeClient.prototype.getStatus = async function(jobInfo) { - const msg = `getStatus is not implemented for current executor backend!`; + ComputeClient.prototype.getStatus = async function(/*jobInfo*/) { + const msg = `getStatus is not implemented for current compute backend!`; this.logger.warn(msg); throw new Error(msg); }; - ComputeClient.prototype.getOutputHashes = async function(jobInfo) { - const msg = `getOutputHashes is not implemented for current executor backend!`; + ComputeClient.prototype.getOutputHashes = async function(/*jobInfo*/) { + const msg = `getOutputHashes is not implemented for current compute backend!`; this.logger.warn(msg); throw new Error(msg); }; - ComputeClient.prototype.getConsoleOutput = async function(hash) { - const msg = `getConsoleOutput is not implemented for current executor backend!`; + ComputeClient.prototype.getConsoleOutput = async function(/*hash*/) { + const msg = `getConsoleOutput is not implemented for current compute backend!`; this.logger.warn(msg); throw new Error(msg); }; diff --git a/src/common/compute/backends/gme/ExecutorHelper.js b/src/common/compute/backends/gme/ExecutorHelper.js index dcd2c5dd7..132215f26 100644 --- a/src/common/compute/backends/gme/ExecutorHelper.js +++ b/src/common/compute/backends/gme/ExecutorHelper.js @@ -38,7 +38,6 @@ define([ }; ExecutorHelper.getWorkers = function() { - return Q(1); // TODO return this.get(WORKER_ENDPOINT) .then(workerDict => values(workerDict)); }; diff --git a/src/common/compute/backends/gme/dashboard/WorkerModal.css b/src/common/compute/backends/gme/dashboard/WorkerModal.css index b5c874f78..6340d83c3 100644 --- a/src/common/compute/backends/gme/dashboard/WorkerModal.css +++ b/src/common/compute/backends/gme/dashboard/WorkerModal.css @@ -4,12 +4,12 @@ } .gme-dashboard .modal-content .modal-header .header-icon { + background-size: 28px 28px; + display: inline-block; height: 28px; - width: 28px; margin-right: 1ex; - display: inline-block; vertical-align: middle; - background-size: 28px 28px; + width: 28px; } .gme-dashboard .modal-content .modal-body th { @@ -21,25 +21,25 @@ } .gme-dashboard .queue-title { + color: #555555; float: left; - margin-right: 5px; font-size: 1.4em; - color: #555555; font-weight: bold; - width: 100%; + margin-right: 5px; text-align: center; + width: 100%; } .gme-dashboard .no-jobs-msg { - font-style: italic; - font-size: 1.3em; color: #777; + font-size: 1.3em; + font-style: italic; } .gme-dashboard .no-workers-msg { - font-style: italic; - font-size: 1.3em; color: #777; + font-size: 1.3em; + font-style: italic; } .gme-dashboard .job-queue { From 4036195951e226ae2db71640985ae5238c7e7e98 Mon Sep 17 00:00:00 2001 From: Brian Broll Date: Thu, 19 Sep 2019 10:00:45 -0500 Subject: [PATCH 43/45] WIP Updated tests --- src/plugins/ExecuteJob/ExecuteJob.js | 6 +- .../plugins/ExecuteJob/ExecuteJob.spec.js | 103 +++++++----------- 2 files changed, 42 insertions(+), 67 deletions(-) diff --git a/src/plugins/ExecuteJob/ExecuteJob.js b/src/plugins/ExecuteJob/ExecuteJob.js index 72acf2473..961afb164 100644 --- a/src/plugins/ExecuteJob/ExecuteJob.js +++ b/src/plugins/ExecuteJob/ExecuteJob.js @@ -161,9 +161,9 @@ define([ await this.prepare(isResuming); if (isResuming) { - this.currentRunId = this.getJobId(this.activeNode); this.startExecHeartBeat(); if (this.canResumeJob(this.activeNode)) { + this.currentRunId = this.getJobId(this.activeNode); return this.resumeJob(this.activeNode); } else { var name = this.getAttribute(this.activeNode, 'name'), @@ -445,14 +445,14 @@ define([ try { await this.save(`Queued "${name}" operation in ${this.pipelineName}`); - await this.createJob(job, opNode, hash); + await this.createJob(job, hash); } catch (err) { this.logger.error(`Could not execute "${name}": ${err}`); } }; - ExecuteJob.prototype.createJob = async function (job, opNode, hash) { + ExecuteJob.prototype.createJob = async function (job, hash) { // Record the job info for the given hash this._execHashToJobNode[hash] = job; const jobInfo = await this.compute.createJob(hash); diff --git a/test/unit/plugins/ExecuteJob/ExecuteJob.spec.js b/test/unit/plugins/ExecuteJob/ExecuteJob.spec.js index efe9397bb..9d7dc3177 100644 --- a/test/unit/plugins/ExecuteJob/ExecuteJob.spec.js +++ b/test/unit/plugins/ExecuteJob/ExecuteJob.spec.js @@ -21,40 +21,28 @@ describe('ExecuteJob', function () { return Q(); }; - before(function (done) { + before(async function () { this.timeout(10000); - testFixture.clearDBAndGetGMEAuth(gmeConfig, projectName) - .then(function (gmeAuth_) { - gmeAuth = gmeAuth_; - // This uses in memory storage. Use testFixture.getMongoStorage to persist test to database. - storage = testFixture.getMemoryStorage(logger, gmeConfig, gmeAuth); - return storage.openDatabase(); - }) - .then(function () { - var importParam = { - projectSeed: testFixture.path.join(testFixture.DF_SEED_DIR, 'devProject', 'devProject.webgmex'), - projectName: projectName, - branchName: 'master', - logger: logger, - gmeConfig: gmeConfig - }; - - return testFixture.importProject(storage, importParam); - }) - .then(function (importResult) { - project = importResult.project; - commitHash = importResult.commitHash; - return project.createBranch('test', commitHash); - }) - .nodeify(done); + gmeAuth = await testFixture.clearDBAndGetGMEAuth(gmeConfig, projectName); + storage = testFixture.getMemoryStorage(logger, gmeConfig, gmeAuth); + await storage.openDatabase(); + const importParam = { + projectSeed: testFixture.path.join(testFixture.DF_SEED_DIR, 'devProject', 'devProject.webgmex'), + projectName: projectName, + branchName: 'master', + logger: logger, + gmeConfig: gmeConfig + }; + + const importResult = await testFixture.importProject(storage, importParam); + project = importResult.project; + commitHash = importResult.commitHash; + await project.createBranch('test', commitHash); }); - after(function (done) { - storage.closeDatabase() - .then(function () { - return gmeAuth.unload(); - }) - .nodeify(done); + after(async function () { + await storage.closeDatabase(); + await gmeAuth.unload(); }); it('should verify activeNode is "Job"', function (done) { @@ -89,7 +77,6 @@ describe('ExecuteJob', function () { return manager.initializePlugin(pluginName) .then(plugin_ => { plugin = plugin_; - plugin.checkExecutionEnv = () => Q(); return manager.configurePlugin(plugin, {}, context); }) .then(() => node = plugin.activeNode) @@ -185,7 +172,7 @@ describe('ExecuteJob', function () { plugin.createIdToMetadataId[graphTmp] = id; // Check that the value is correct before applying node changes - var applyModelChanges = plugin.applyModelChanges; + const applyModelChanges = plugin.applyModelChanges; plugin.applyModelChanges = function() { return applyModelChanges.apply(this, arguments) .then(() => { @@ -196,17 +183,14 @@ describe('ExecuteJob', function () { plugin.save().nodeify(done); }); - it('should update _metadata in updateNodes', function(done) { - var id = 'testId'; + it('should update _metadata in updateNodes', async function() { + const id = 'testId'; plugin._metadata[id] = node; node.old = true; - plugin.updateNodes() - .then(() => { - var graph = plugin._metadata[id]; - expect(graph.old).to.not.equal(true); - }) - .nodeify(done); + await plugin.updateNodes(); + const graph = plugin._metadata[id]; + expect(graph.old).to.not.equal(true); }); // Check that it gets the correct value from a newly created node after @@ -260,30 +244,21 @@ describe('ExecuteJob', function () { describe('cancel', function() { beforeEach(preparePlugin); - it('should stop the job if the execution is canceled', function(done) { - var job = node, - hash = 'abc123'; - - plugin.setAttribute(node, 'jobInfo', JSON.stringify({secret:'abc'})); - plugin.isExecutionCanceled = () => true; - plugin.onOperationCanceled = () => done(); - plugin.executor = { - cancelJob: jobHash => expect(jobHash).equal(hash) + // TODO: Update this so that they can be canceled synchronously + it('should cancel running jobs on plugin abort', function(done) { + const jobInfo = {hash: 'abc123', secret: 'abc'}; + const mockCompute = {}; + mockCompute.cancelJob = job => { + if (job.hash !== jobInfo.hash) { + done(new Error('Invalid jobInfo')); + } + done(); }; - plugin.watchOperation(hash, job, job); - }); + mockCompute.createJob = async () => jobInfo; + plugin.compute = mockCompute; - it('should stop the job if a job is canceled', function(done) { - var job = node, - hash = 'abc123'; - - plugin.setAttribute(node, 'jobInfo', JSON.stringify({secret:'abc'})); - plugin.canceled = true; - plugin.onOperationCanceled = () => done(); - plugin.executor = { - cancelJob: jobHash => expect(jobHash).equal(hash) - }; - plugin.watchOperation(hash, job, job); + return plugin.createJob(node, jobInfo.hash) + .then(() => plugin.onAbort()); }); it('should set exec to running', function(done) { @@ -392,7 +367,7 @@ describe('ExecuteJob', function () { it('should handle error if missing jobId', function(done) { // Remove jobId - plugin.delAttribute(plugin.activeNode, 'runId'); + plugin.delAttribute(plugin.activeNode, 'jobInfo'); plugin.startExecHeartBeat = () => {}; plugin.isResuming = () => Q(true); plugin.main(function(err) { From c6fe66d1eb2ff77b0f036b350a2bd821c41c7148 Mon Sep 17 00:00:00 2001 From: Brian Broll Date: Thu, 19 Sep 2019 10:03:29 -0500 Subject: [PATCH 44/45] WIP removed unreachable code --- src/plugins/ExecuteJob/ExecuteJob.js | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/plugins/ExecuteJob/ExecuteJob.js b/src/plugins/ExecuteJob/ExecuteJob.js index 961afb164..7d381c8eb 100644 --- a/src/plugins/ExecuteJob/ExecuteJob.js +++ b/src/plugins/ExecuteJob/ExecuteJob.js @@ -183,10 +183,6 @@ define([ return JSON.parse(this.getAttribute(node, 'jobInfo')).hash; }; - ExecuteJob.prototype.onAbort = function () { - this.canceled = true; - }; - ExecuteJob.prototype.onAbort = ExecuteJob.prototype.onUserCancelDetected = function () { this.logger.info('Received Abort. Canceling jobs.'); From faf7c50d88c2369df551ca93bd8ace620121ca84 Mon Sep 17 00:00:00 2001 From: Brian Broll Date: Thu, 19 Sep 2019 12:59:08 -0500 Subject: [PATCH 45/45] WIP fixed failing test --- test/unit/plugins/ExecuteJob/ExecuteJob.spec.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/unit/plugins/ExecuteJob/ExecuteJob.spec.js b/test/unit/plugins/ExecuteJob/ExecuteJob.spec.js index 9d7dc3177..07efd66b3 100644 --- a/test/unit/plugins/ExecuteJob/ExecuteJob.spec.js +++ b/test/unit/plugins/ExecuteJob/ExecuteJob.spec.js @@ -257,8 +257,8 @@ describe('ExecuteJob', function () { mockCompute.createJob = async () => jobInfo; plugin.compute = mockCompute; - return plugin.createJob(node, jobInfo.hash) - .then(() => plugin.onAbort()); + return Q(plugin.createJob(node, jobInfo.hash)) + .finally(() => plugin.onAbort()); }); it('should set exec to running', function(done) {