function getOutputPath (options, page, parse) {
  var pagePart = urlm.parse(page),
      // if outputPath was normalized with an object, let the key passthru
      outputPath = options.outputPath(page, true);

  // check for bad output path
  if (!outputPath) {
    return false;
  }

  // if the outputPath is really still a url, fix it to path+hash
  if (common.isUrl(outputPath)) {
    outputPath = pagePart.path + (pagePart.hash ? pagePart.hash : "");
  }

  // if the caller wants the url parse output, return it
  if (parse) {
    parse.url = pagePart;
  }

  return outputPath;
}

Get an output path for a page.

Parse a given page as a url and return its corresponding outputPath.
If parse is an object, parse.url will be a reference to the url parse.
Returns false if no output path found.

@param {Object} options - output path options.
@param {Function} options.outputPath - Derives the outputPath from page.
@param {String} page - The url to the page resource.
@param {Object} [parse] - Supply to get page (url) parse output in prop `url`.
@returns {String} The 'path' part of the file output path.


function mapOutputFile (options, page, parse) {
  if (!_.isFunction(options.outputPath)) {
    options.outputPath = normalize(options.outputPath);
  }

  var outputPath = getOutputPath(options, page, parse);
  var outputDir = options.outputDir;
  var fileName = "index.html";

  if (options.sitemapOutputDir) {
    outputDir = path.join(options.outputDir, options.sitemapOutputDir);
    fileName = "";
  }

  return ( outputPath && path.join(outputDir, outputPath, fileName) ) || false;
}

function mapOutputFile ( options , page , parse ) { if ( ! _ . isFunction ( options . outputPath ) ) { options . outputPath = normalize ( options . outputPath ) ; } var outputPath = getOutputPath ( options , page , parse ) ; var outputDir = options . outputDir ; var fileName = "index.html" ; if ( options . sitemapOutputDir ) { outputDir = path . join ( options . outputDir , options . sitemapOutputDir ) ; fileName = "" ; } return ( outputPath && path . join ( outputDir , outputPath , fileName ) ) || false ; }

Using the options, map the given page to an output path.

@param {Object} options - The input generator options.
@param {String} options.outputDir - The output directory root path.
@param {String} [options.outputPath] - optional function or object to map pages to outputPaths.
@param {String} [options.sitemapOutputDir] - optional root path for sitemaps within outputDir.
@returns {String|Boolean} The full path to the output file or false on failure.

Using the options map the given page to an output path .


function (options, generator, listener) {
    options = options || {};
    prepOptions(options, listener);
    return generator(options);
  }

function ( options , generator , listener ) { options = options || { } ; prepOptions ( options , listener ) ; return generator ( options ) ; }

Run the input generator.

@param {Object} options - Input generator options.
@param {Function} generator - The input generator to run.
@param {Function} listener - The callback that receives each input generated.
@returns {Promise} The promise returned by the input generator.


function (options, page) {
    var parse = {};
    var outputFile = mapOutputFile(options, page, parse);

    if (outputFile) {

      options._inputEmitter.emit("input", {
        outputFile: outputFile,
        // make the url
        url: urlm.format({
                protocol: options.protocol,
                auth: options.auth,
                hostname: options.hostname,
                port: options.port,
                pathname: parse.url.pathname,
                search: parse.url.search,
                hash: parse.url.hash
              }),
        // map the input page to a selector
        selector: options.selector(page),
        // map the input page to a timeout
        timeout: options.timeout(page),
        checkInterval: options.checkInterval,
        // map the input page to a useJQuery flag
        useJQuery: options.useJQuery(page),
        // map the input page to a verbose flag
        verbose: options.verbose(page),
        // map the input page to phantomJS options
        phantomjsOptions: options.phantomjsOptions(page),
        // useful for testing, debugging
        __page: page
      });

    }

    return outputFile;
  }

Generate the input
Emit the event that contains the input hash

@param {Object} options - input generator options.
@param {String} options.protocol - https?
@param {String} options.auth - user:pass for old style urls.
@param {String} options.hostname - The hostname part of the url.
@param {Number} options.port - The port part of the url.
@param {Function} options.selector - Produces a selector given a page url.
@param {Function} options.timeout - Produces a timeout given a page url.
@param {Number} options.checkInterval - The millisecond resolute poll interval.
@param {Function} options.userJQuery - Produces a jquery bool given a page url.
@param {Function} options.verbose - Produces a verbose bool given a page url.
@param {Function} options.phantomjsOptions - Produces phantomjs options given a page url.
@param {Function} options._inputEmitter - The EventEmitter to generate input.
@param {String} page - The url to the page resource.
@returns {String} The full output file path.


function pathExists (path, options) {
  options = options || {
    returnFile: false
  };

  // Defaults to F_OK
  return nodeCall(fs.access, path)
    .then(function () {
      return options.returnFile ? path : true;
    })
    .catch(function () {
      if (fs.existsSync(path)) {
        return options.returnFile ? path : true;
      }
      return false;
    });
}

function
pathExists
(
path
,
options
)
{
options
=
options
||
{
returnFile
:
false
}
;
// Defaults to F_OK
return
nodeCall
(
fs
.
access
,
path
)
.
then
(
function
(
)
{
return
options
.
returnFile
path
:
true
;
}
)
.
catch
(
function
(
)
{
if
(
fs
.
existsSync
(
path
)
)
{
return
options
.
returnFile
path
:
true
;
}
return
false
;
}
)
;
}

Check if a path exists.

@param {String} path - The full path to the file to check.
@param {Object} [options] - fileExists options.
@param {Boolean} [options.returnFile] - If true, resolve to input filename on
success. Otherwise, resolve to boolean. Defaults to false (boolean).
@returns {Promise} Resolves to true (or file) if exists, false otherwise.


function () {
  // If the path we're given by phantomjs is to a .cmd, it is pointing to a global copy.
  // Using the cmd as the process to execute causes problems cleaning up the processes
  // so we walk from the cmd to the phantomjs.exe and use that instead.

  var phantomSource = require("phantomjs-prebuilt").path;

  if (path.extname(phantomSource).toLowerCase() === ".cmd") {
    return path.join(path.dirname( phantomSource ), "//node_modules//phantomjs-prebuilt//lib//phantom//bin//phantomjs.exe");
  }

  return phantomSource;
}

Determine the default phantomJS module path. This is overridden by the
phatomjs option.

This function, (technique and concerns) originated from karma-phantomjs-launcher:

The MIT License
Copyright (C) 2011-2013 Google, Inc.

Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
of the Software, and to permit persons to whom the Software is furnished to do
so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

Determine the default phantomJS module path . This is overridden by the phatomjs option .


function worker (input, options, notifier, qcb) {
  var cp,
      customModule,
      snapshotScript = options.snapshotScript,
      phantomjsOptions = Array.isArray(input.phantomjsOptions) ? input.phantomjsOptions : [input.phantomjsOptions];

  // If the outputFile has NOT already been seen by the notifier, process.
  if (!notifier.known(input.outputFile)) {

    // map snapshotScript object script to a real path
    if (_.isObject(options.snapshotScript)) {
      snapshotScript = path.join(__dirname, phantomDir, options.snapshotScript.script) + ".js";
      customModule = options.snapshotScript.module;
    }

    cp = spawn(
      options.phantomjs,
      phantomjsOptions.concat([
        snapshotScript,
        input.outputFile,
        input.url,
        input.selector,
        input.timeout,
        input.checkInterval,
        input.useJQuery,
        input.verbose,
        customModule
      ]), { cwd: process.cwd(), stdio: "inherit", detached: true }
    );

    cp.on("error", function (e) {
      notifier.remove(input.outputFile);
      notifier.setError(e);
      console.error(e);
      qcb(e);
    });

    cp.on("exit", function (code) {
      qcb(code);
    });

    // start counting
    notifier.add(input.outputFile, input.timeout);
  }
  else {
    // The input.outputFile is being or has been processed this run.
    qcb(0);
  }
}

The worker task that launches phantomjs.

@param {Object} input - An input object from an input generator.
@param {Array} input.phantomjsOptions - Actual Phantomjs options.
@param {String} input.outputFile - The output file that the html is written to.
@param {String} input.url - The url to get the html from.
@param {String} input.selector - The selector to wait for.
@param {String} input.timeout - The phantomjs timeout.
@param {String} input.checkInterval - The interval to poll for results.
@param {Boolean} input.useJQuery - True if selector is jquery selector.
@param {Boolean} input.verbose - True if verbose output is desired.
@param {Object} options - Input options.
@param {Object} options.snapshotScript - Snapshot script options.
@param {String} options.snapshotScript.script - Snapshot script name.
@param {String} options.snapshotScript.module - Snapshot script module.
@param {String} options.phantomjs - Full path to the Phantomjs exe.
@param {Object} notifier - The async Notifier instance for this run.
@param {Function} notifier.remove - Remove an outputFile from notifier.
@param {Function} notifier.setError - Set an error on the notifier.
@param {Function} notifier.add - Add a file, timeout to the notifier.
@param {Function} notifier.known - Check if a file is or has been processed.
@param {Function} qcb - async.queue callback function.

The worker task that launches phantomjs .


function prepOptions (options) {
  // ensure this module's defaults are represented in the options.
  common.ensure(options, defaults);

  // if array data source, ensure input type is "array".
  if (Array.isArray(options.source)) {
    options.input = "array";
  }
}

Prepare html snapshots options.

@param {Object} options - html snapshots options.
@param {Array|String} options.source - html snapshots data source.


function (options, listener) {
    var inputGenerator, notifier, started, result, q, emitter, completion;

    options = options || {};
    prepOptions(options);

    // create the inputGenerator, default to robots
    inputGenerator = inputFactory.create(options.input);

    // clean the snapshot output directory
    if (options.outputDirClean) {
      rimraf(options.outputDir);
    }

    // start async completion notification.
    notifier = new Notifier();
    emitter = new EventEmitter();
    started = notifier.start(options.pollInterval, inputGenerator,
      function (err, completed) {
        emitter.emit("complete", err, completed);
      });

    if (started) {
      // create the completion Promise.
      completion = new Promise(function (resolve, reject) {
        function completionResolver (err, completed) {
          try {
            _.isFunction(listener) && listener(err, completed);
          } catch (e) {
            console.error("User supplied listener exception", e);
          }
          if (err) {
            err.notCompleted = notifier.filesNotDone;
            err.completed = completed;
            reject(err);
          } else {
            resolve(completed);
          }
        }
        emitter.addListener("complete", completionResolver);
      });

      // create a worker queue with a parallel process limit.
      q = asyncLib.queue(function (task, callback) {
        task(_.once(callback));
      }, options.processLimit);

      // have the queue call notifier.empty when last item
      //  from the queue is given to a worker.
      q.empty = notifier.qEmpty.bind(notifier);

      // expose abort callback to input generators via options.
      options._abort = function (err) {
        notifier.abort(q, err);
      };

      // generate input for the snapshots.
      result = inputGenerator.run(options, function (input) {
        // give the worker the input and place into the queue
        q.push(_.partial(worker, input, options, notifier));
      })
        // after input generation, resolve on browser completion.
        .then(function () {
          return completion;
        });
    } else {
      result = Promise.reject("failed to start async notifier");
    }

    return result;
  }

//  from the queue is given to a worker.

Run all the snapshots using the requested inputGenerator

@param {Object} options - ALL user supplied html snapshots options.
@param {String} options.outputDir - Directory to write the html files to.
@param {String} [options.input] - Input source type "robots", "sitemap", "array";
Defaults to "robots".
@param {Boolean} [options.outputDirClean] - True if output dir should be rm -rf;
Defaults to false.
@param {Number} [options.pollInterval] -
@param {Function} [listener] - User supplied optional callback.
@returns {Promise} Resolves on completion.


function createLockFactory () {
  // Create the per instance async lock.
  var lock = new AsyncLock();

  // Make a random id
  var rid = crypto.randomBytes(16).toString("hex");

  /**
   * Force a serial execution context.
   *
   * @param {Function} fn - The function to guard.
   * @param {Number} timeout - The max time to wait for the lock.
   */
  return function lockFactory (fn, timeout) {
    return function protectedContext () {
      lock.acquire("cs-guard-" + rid, function (done) {
        fn(function () {
          done(null, 0);
        });
      }, NOOP, {
        timeout: timeout
      });
    };
  };
}


function Notifier () {

  // Create the serial execution context mechanism.
  this.csFactory = createLockFactory();

  // The private files collection
  // Contains a file and timer: "filename": {timer: timerid}
  // Used for tracking work left to do. When empty, work is done.
  this.files = {};

  // Contains files successfully processed
  this.filesDone = [];

  // Contains files unsuccessfully processed
  this.filesNotDone = [];

  // true if a timeout occurred, or set by abort
  this.errors = [];

  // the holder of the current failure timeout padding
  this.padTimeout = TIMEOUT_PAD_FLOOR;

  // our reference to the listener
  this.callback = null;

  // our reference to the watcher (an interval id)
  // initial value undefined is important
  // this.watcher;

  // the working pollInterval for this run
  // this.interval

  // flag set by qEmpty callback
  // when the last item from the queue is given to a worker
  this.qempty = false;
}

Notifier Constructor
Polls the outputDir and when all the files exist,
calls the listener indicating the snapshots are done.

Notifier Constructor Polls the outputDir and when all the files exist calls the listener indicating the snapshots are done .


function start (pollInterval, input, listener) {
    var result = (
      pollInterval > 0 &&
      typeof listener === "function" &&
      (!!input)
      );

    if (result) {
      if (this.isStarted()) {
        throw new Error("Notifier already started");
      }

      this.callback = listener;
      this.interval = parseInt(pollInterval, 10);

      // Poll the filesystem for the files to exist
      // Checks the child process expected output to determine success or failure
      // if the file exists, then it succeeded.
      this.watcher = setInterval(this.csFactory(function (done) {
        var self = this;
        var eoi = typeof input.EOI === "function" && input.EOI();

        if (eoi) {
          Promise.all(Object.keys(self.files).map(function (file) {
            return pathExists(file, {
              returnFile: true
            });
          }))
            .then(function (files) {
              var callback = self.callback;

              try {
                files.forEach(function (file) {
                  file && self._remove(file, true);
                });

                if (self._isDone()) {
                  self._closeWatcher();
                  if (self.callback) {
                    self.callback = null;
                    setImmediate(function () {
                      callback(self.getError(), self.filesDone);
                    });
                  }
                }
              } catch (e) {
                console.error(e);
              }
              done();
            });
        } else {
          done();
        }
      }.bind(this), L_WAIT), this.interval);

    } else {
      console.error("Bad poll interval, async listener, or input generator supplied");
    }

    return result;
  }

Start a watch interval, when a file exists, remove it from our files array.
If the files array is empty, call the listener and stop the watch interval.

@param {Number} pollInterval - The millisecond poll interval to check for
file completion.
@param {Object} input - The inputGenerator.
@param {Function} listener - The callback to notify when complete.
@param {Function} input.EOI - The inputGenerator function that signals end
of input.
@returns {Boolean} true if started successfully, false otherwise.

Start a watch interval when a file exists remove it from our files array . If the files array is empty call the listener and stop the watch interval .


function add (outputFile, timeout) {
    var failTimeout = timeout;
    var timer;

    if (!this.isStarted()) {
      throw new Error("MUST call `start` before `add`");
    }

    if (!this._exists(outputFile)) {
      // make sure we evaluate after the child process
      failTimeout = parseInt(timeout, 10) + parseInt(this.padTimeout, 10);
      // Stagger and grow the failure timeout padding, add 1s every 10 processes
      this.padTimeout += 100;

      // setup a timeout handler to detect failure
      timer = setTimeout(this.csFactory(function (done) {
        var self = this;

        // if the output file has not already been removed
        if (self._exists(outputFile)) {
          pathExists(outputFile)
            .then(function (fsExists) {
              var callback = self.callback;

              try {
                if (!fsExists) {
                  self._setError(new Error(
                    "'" + outputFile + "' did not get a snapshot before timeout"
                  ));
                }
                self._remove(outputFile, fsExists);

                if (self._isDone()) {
                  self._closeWatcher();
                  if (self.callback) {
                    self.callback = null;
                    setImmediate(function () {
                      callback(self.getError(), self.filesDone);
                    });
                  }
                }
              } catch (e) {
                console.error(e);
              }
              done();
            });
        } else {
          done();
        }
      }.bind(this), L_WAIT), parseInt(failTimeout, 10));

      // add the file tracking object
      this.files[outputFile] = {
        timer: timer
      };
    }
  }

Add a file to the files array if it's not there.

@param {String} outputFile - The full path to the output file to add.
@param {Number} timeout - timeout milliseconds to wait (max).

Add a file to the files array if it s not there .


function known (outputFile) {
    var result = false;
    this.csFactory(function (done) {
      result =
        this._exists(outputFile) || this.filesDone.indexOf(outputFile) > -1;
      done();
    }.bind(this), L_WAIT)();
    return result;
  }

function known ( outputFile ) { var result = false ; this . csFactory ( function ( done ) { result = this . _exists ( outputFile ) || this . filesDone . indexOf ( outputFile ) > - 1 ; done ( ) ; } . bind ( this ) , L_WAIT ) ( ) ; return result ; }

Check if a file is being processed or has already been processed.

@param {String} outputFile - The outputFile to check.
@returns {Boolean} true if the fi

Check if a file is being processed or has already been processed .


function _remove (outputFile, done) {
    if (this._exists(outputFile)) {
      if (done) {
        this.filesDone.push(outputFile);
      } else {
        this.filesNotDone.push(outputFile);
      }
      clearTimeout(this.files[outputFile].timer);
      delete this.files[outputFile];
    }
  }

function _remove ( outputFile , done ) { if ( this . _exists ( outputFile ) ) { if ( done ) { this . filesDone . push ( outputFile ) ; } else { this . filesNotDone . push ( outputFile ) ; } clearTimeout ( this . files [ outputFile ] . timer ) ; delete this . files [ outputFile ] ; } }

Remove a file from the files array if it's there.
Unprotected version.

@param {String} outputFile - The outputFile to remove.
@param {Boolean} done - If true, add to filesDone collection.

Remove a file from the files array if it s there . Unprotected version .


function remove (outputFile, done) {
    this.csFactory(function (_done) {
      this._remove(outputFile, done);
      _done();
    }.bind(this), L_WAIT)();
  }

function remove ( outputFile , done ) { this . csFactory ( function ( _done ) { this . _remove ( outputFile , done ) ; _done ( ) ; } . bind ( this ) , L_WAIT ) ( ) ; }

Remove a file from the files array if it's there.
Protected version.

@param {String} outputFile - The outputFile to remove.
@param {Boolean} done - If true, add to filesDone collection.

Remove a file from the files array if it s there . Protected version .


function (time) {
          fs.write(options.outputFile, filter(page.content), "w");
          globals.exit(0, "snapshot for "+options.url+" finished in "+time+" ms\n  written to "+options.outputFile);
        }

function
(
time
)
{
fs
.
write
(
options
.
outputFile
,
filter
(
page
.
content
)
,
"w"
)
;
globals
.
exit
(
0
,
"snapshot for "
+
options
.
url
+
" finished in "
+
time
+
" ms\n  written to "
+
options
.
outputFile
)
;
}


function oneline (line, options) {
  var key = "Allow: ",
      index = line.indexOf(key);

  if (index !== -1) {
    var page = line.substr(index + key.length).replace(/^\s+|\s+$/g, "");
    return page.indexOf("*") === -1 && base.input(options, page);
  }

  return true;
}

function
oneline
(
line
,
options
)
{
var
key
=
"Allow: "
,
index
=
line
.
indexOf
(
key
)
;
if
(
index
!==
1
)
{
var
page
=
line
.
substr
(
index
+
key
.
length
)
.
replace
(
/
^\s+|\s+$
/
g
,
""
)
;
return
page
.
indexOf
(
"*"
)
===
1
&&
base
.
input
(
options
,
page
)
;
}
return
true
;
}

Generate input for one line of a simple robots.txt file
Does not support wildcards.

Generate input for one line of a simple robots . txt file Does not support wildcards .


function getRobotsUrl (options, callback) {
  request({
    url: options.source,
    timeout: options.timeout()
  }, function (err, res, body) {
    var error = err || common.checkResponse(res, "text/plain");

    if (error) {
      callback(common.prependMsgToErr(error, options.source, true));
    } else {
      body.toString().split('\n').every(function(line) {
        // Process the line input, but break if base.input returns false.
        // For now, this can only happen if no outputDir is defined,
        //   which is a fatal bad option problem and will happen immediately.
        if (!oneline(line, options)) {
          error = common.prependMsgToErr(base.generatorError(), line, true);
          return false;
        }
        return true;
      });

      callback(error);
    }
  });
}

Retrieves robots.txt from url and parses it.

@param {Object} options - Robots.txt options
@param {String} options.source - The URL to a robots.txt
@param {Function} options.timeout - Returns the robots.txt request timeout.
@param {Function} callback - A completion callback.

Retrieves robots . txt from url and parses it .


function getRobotsFile (options, callback) {
  fs.readFile(options.source, function (err, data) {
    if (!err) {
      data.toString().split('\n').every(function (line) {
        // Process the line input, but break if base.input returns false.
        // For now, this can only happen if no outputDir is defined,
        //   which is a fatal bad option problem and will happen immediately.
        if (!oneline(line, options)) {
          err = common.prependMsgToErr(base.generatorError(), line, true);
          return false;
        }
        return true;
      });
    }

    callback(err);
  });
}

Reads the robots.txt file and parses it.

Reads the robots . txt file and parses it .


function bubble(values) {

  return values.map(d => {

    if (d.key && d.values) {
      if (d.values[0].key === "undefined") return d.values[0].values[0];
      else d.values = bubble(d.values);
    }

    return d;

  });

}

function bubble ( values ) { return values . map ( d => { if ( d . key && d . values ) { if ( d . values [ 0 ] . key === "undefined" ) return d . values [ 0 ] . values [ 0 ] ; else d . values = bubble ( d . values ) ; } return d ; } ) ; }

Bubbles up values that do not nest to the furthest key.
@param {Array} *values* The "values" of a nest object.
@private

Bubbles up values that do not nest to the furthest key .


function exclude(a, b, v) {
  const aStart = a.start({type: "bigInteger"});
  const bStart = b.start({type: "bigInteger"});
  const aEnd = a.end({type: "bigInteger"});
  const bEnd = b.end({type: "bigInteger"});
  const parts = [];

  // compareTo returns negative if left is less than right

  //       aaa
  //   bbb
  //   aaa
  //       bbb
  if (aStart.compareTo(bEnd) > 0 || aEnd.compareTo(bStart) < 0) {
    return [a.cidr];
  }

  //   aaa
  //   bbb
  if (aStart.compareTo(bStart) === 0 && aEnd.compareTo(bEnd) === 0) {
    return [];
  }

  //   aa
  //  bbbb
  if (aStart.compareTo(bStart) > 0 && aEnd.compareTo(bEnd) < 0) {
    return [];
  }

  // aaaa
  //   bbbb
  // aaaa
  //   bb
  if (aStart.compareTo(bStart) < 0 && aEnd.compareTo(bEnd) <= 0) {
    parts.push({
      start: aStart,
      end: bStart.subtract(one),
    });
  }

  //    aaa
  //   bbb
  //   aaaa
  //   bbb
  if (aStart.compareTo(bStart) >= 0 && aEnd.compareTo(bEnd) > 0) {
    parts.push({
      start: bEnd.add(one),
      end: aEnd,
    });
  }

  //  aaaa
  //   bb
  if (aStart.compareTo(bStart) < 0 && aEnd.compareTo(bEnd) > 0) {
    parts.push({
      start: aStart,
      end: bStart.subtract(one),
    });
    parts.push({
      start: bEnd.add(one),
      end: aEnd,
    });
  }

  const remaining = [];
  for (const part of parts) {
    for (const subpart of subparts(part, v)) {
      remaining.push(formatPart(subpart, v));
    }
  }

  return cidrTools.merge(remaining);
}

exclude b from a and return remainder cidrs


function getMsTimestamp(){
        var ts = new Date().getTime();
        if(lastMsTs >= ts){
            lastMsTs++;
        }
        else{
            lastMsTs = ts;
        }
        return lastMsTs;
    }

function getMsTimestamp ( ) { var ts = new Date ( ) . getTime ( ) ; if ( lastMsTs >= ts ) { lastMsTs ++ ; } else { lastMsTs = ts ; } return lastMsTs ; }


function parseUrl(url) {
        var serverOptions = {
            host: "localhost",
            port: 80
        };
        if(url.indexOf("https") === 0){
            serverOptions.port = 443;
        }
        var host = url.split("://").pop();
        serverOptions.host = host;
        var lastPos = host.indexOf(":");
        if (lastPos > -1) {
            serverOptions.host = host.slice(0,lastPos);
            serverOptions.port = Number(host.slice(lastPos+1,host.length));
        }
        return serverOptions;
    }

function parseUrl ( url ) { var serverOptions = { host : "localhost" , port : 80 } ; if ( url . indexOf ( "https" ) === 0 ) { serverOptions . port = 443 ; } var host = url . split ( "://" ) . pop ( ) ; serverOptions . host = host ; var lastPos = host . indexOf ( ":" ) ; if ( lastPos > - 1 ) { serverOptions . host = host . slice ( 0 , lastPos ) ; serverOptions . port = Number ( host . slice ( lastPos + 1 , host . length ) ) ; } return serverOptions ; }

parsing host and port information from url


function prepareParams(params){
        var str = [];
        for(var i in params){
            str.push(i+"="+encodeURIComponent(params[i]));
        }
        return str.join("&");
    }

function prepareParams ( params ) { var str = [ ] ; for ( var i in params ) { str . push ( i + "=" + encodeURIComponent ( params [ i ] ) ) ; } return str . join ( "&" ) ; }


function stripTrailingSlash(str) {
        if(str.substr(str.length - 1) === "/") {
            return str.substr(0, str.length - 1);
        }
        return str;
    }

function stripTrailingSlash ( str ) { if ( str . substr ( str . length - 1 ) === "/" ) { return str . substr ( 0 , str . length - 1 ) ; } return str ; }


function getProperties(orig, props){
        var ob = {};
        var prop;
        for(var i = 0; i < props.length; i++){
            prop = props[i];
            if(typeof orig[prop] !== "undefined"){
                ob[prop] = orig[prop];
            }
        }
        return ob;
    }

function getProperties ( orig , props ) { var ob = { } ; var prop ; for ( var i = 0 ; i < props . length ; i ++ ) { prop = props [ i ] ; if ( typeof orig [ prop ] !== "undefined" ) { ob [ prop ] = orig [ prop ] ; } } return ob ; }

retrieve only specific properties from object


function add_cly_events(event){
        if(!event.key){
            log("Event must have key property");
            return;
        }
        if(cluster.isMaster){
            if(!event.count){
                event.count = 1;
            }

            var props = ["key", "count", "sum", "dur", "segmentation"];
            var e = getProperties(event, props);
            e.timestamp = getMsTimestamp();
            var date = new Date();
            e.hour = date.getHours();
            e.dow = date.getDay();
            log("Adding event: ", event);
            eventQueue.push(e);
            storeSet("cly_event", eventQueue);
        }
        else{
            process.send({ cly: {event: event} });
        }
    }

function
add_cly_events
(
event
)
{
if
(
!
event
.
key
)
{
log
(
"Event must have key property"
)
;
return
;
}
if
(
cluster
.
isMaster
)
{
if
(
!
event
.
count
)
{
event
.
count
=
1
;
}
var
props
=
[
"key"
,
"count"
,
"sum"
,
"dur"
,
"segmentation"
]
;
var
e
=
getProperties
(
event
,
props
)
;
e
.
timestamp
=
getMsTimestamp
(
)
;
var
date
=
new
Date
(
)
;
e
.
hour
=
date
.
getHours
(
)
;
e
.
dow
=
date
.
getDay
(
)
;
log
(
"Adding event: "
,
event
)
;
eventQueue
.
push
(
e
)
;
storeSet
(
"cly_event"
,
eventQueue
)
;
}
else
{
process
.
send
(
{
cly
:
{
event
:
event
}
}
)
;
}
}

internal method for events, in case there is no consent for custom events, but internal events has consents

internal method for events in case there is no consent for custom events but internal events has consents


function prepareRequest(request) {
        request.app_key = Countly.app_key;
        request.device_id = Countly.device_id;
        request.sdk_name = SDK_NAME;
        request.sdk_version = SDK_VERSION;
        if(Countly.check_consent("location")){
            if(Countly.country_code){
                request.country_code = Countly.country_code;
            }
            if(Countly.city){
                request.city = Countly.city;
            }
            if(Countly.ip_address !== null){
                request.ip_address = Countly.ip_address;
            }
        }
        else{
            request.location = "";
        }

        request.timestamp = getMsTimestamp();
        var date = new Date();
        request.hour = date.getHours();
        request.dow = date.getDay();
    }

function prepareRequest ( request ) { request . app_key = Countly . app_key ; request . device_id = Countly . device_id ; request . sdk_name = SDK_NAME ; request . sdk_version = SDK_VERSION ; if ( Countly . check_consent ( "location" ) ) { if ( Countly . country_code ) { request . country_code = Countly . country_code ; } if ( Countly . city ) { request . city = Countly . city ; } if ( Countly . ip_address !== null ) { request . ip_address = Countly . ip_address ; } } else { request . location = "" ; } request . timestamp = getMsTimestamp ( ) ; var date = new Date ( ) ; request . hour = date . getHours ( ) ; request . dow = date . getDay ( ) ; }

prepare request by adding basic info to it


function toRequestQueue(request){
        if(cluster.isMaster){
            if(!Countly.app_key || !Countly.device_id){
                log("app_key or device_id is missing");
                return;
            }
            prepareRequest(request);

            if(requestQueue.length > queueSize){
                requestQueue.shift();
            }

            requestQueue.push(request);
            storeSet("cly_queue", requestQueue);
        }
        else{
            process.send({ cly: {cly_queue: request} });
        }
    }

function
toRequestQueue
(
request
)
{
if
(
cluster
.
isMaster
)
{
if
(
!
Countly
.
app_key
||
!
Countly
.
device_id
)
{
log
(
"app_key or device_id is missing"
)
;
return
;
}
prepareRequest
(
request
)
;
if
(
requestQueue
.
length
>
queueSize
)
{
requestQueue
.
shift
(
)
;
}
requestQueue
.
push
(
request
)
;
storeSet
(
"cly_queue"
,
requestQueue
)
;
}
else
{
process
.
send
(
{
cly
:
{
cly_queue
:
request
}
}
)
;
}
}


function getMetrics(){
        var m = JSON.parse(JSON.stringify(metrics));

        //getting app version
        m._app_version = m._app_version || Countly.app_version;

        m._os = m._os || os.type();
        m._os_version = m._os_version || os.release();
        platform = os.type();

        log("Got metrics", m);
        return m;
    }

function
getMetrics
(
)
{
var
m
=
JSON
.
parse
(
JSON
.
stringify
(
metrics
)
)
;
//getting app version
m
.
_app_version
=
m
.
_app_version
||
Countly
.
app_version
;
m
.
_os
=
m
.
_os
||
os
.
type
(
)
;
m
.
_os_version
=
m
.
_os_version
||
os
.
release
(
)
;
platform
=
os
.
type
(
)
;
log
(
"Got metrics"
,
m
)
;
return
m
;
}


function makeRequest(url, path, params, callback) {
        try {
            log("Sending HTTP request");
            var serverOptions = parseUrl(url);
            var data = prepareParams(params);
            var method = "GET";
            var options = {
                host: serverOptions.host,
                port: serverOptions.port,
                path: path+"?"+data,
                method: "GET"
            };

            if(data.length >= 2000){
                method = "POST";
            }
            else if(Countly.force_post){
                method = "POST";
            }

            if(method === "POST"){
                options.method = "POST";
                options.path = path;
                options.headers = {
                    "Content-Type": "application/x-www-form-urlencoded",
                    "Content-Length": Buffer.byteLength(data)
                };
            }
            var protocol = http;
            if(url.indexOf("https") === 0){
                protocol = https;
            }
            var req = protocol.request(options, function(res) {
                var str = "";
                res.on("data", function (chunk) {
                    str += chunk;
                });

                res.on("end", function () {
                    if(res.statusCode >= 200 && res.statusCode < 300){
                        callback(false, params, str);
                    }
                    else{
                        callback(true, params);
                    }
                });
            });
            if(method === "POST"){
                // write data to request body
                req.write(data);
            }

            req.on("error", function(err){
                log("Connection failed.", err);
                if (typeof callback === "function") {
                    callback(true, params);
                }
            });

            req.end();
        } catch (e) {
            // fallback
            log("Failed HTTP request", e);
            if (typeof callback === "function") { callback(true, params); }
        }
    }

function
makeRequest
(
url
,
path
,
params
,
callback
)
{
try
{
log
(
"Sending HTTP request"
)
;
var
serverOptions
=
parseUrl
(
url
)
;
var
data
=
prepareParams
(
params
)
;
var
method
=
"GET"
;
var
options
=
{
host
:
serverOptions
.
host
,
port
:
serverOptions
.
port
,
path
:
path
+
"?"
+
data
,
method
:
"GET"
}
;
if
(
data
.
length
>=
2000
)
{
method
=
"POST"
;
}
else
if
(
Countly
.
force_post
)
{
method
=
"POST"
;
}
if
(
method
===
"POST"
)
{
options
.
method
=
"POST"
;
options
.
path
=
path
;
options
.
headers
=
{
"Content-Type"
:
"application/x-www-form-urlencoded"
,
"Content-Length"
:
Buffer
.
byteLength
(
data
)
}
;
}
var
protocol
=
http
;
if
(
url
.
indexOf
(
"https"
)
===
0
)
{
protocol
=
https
;
}
var
req
=
protocol
.
request
(
options
,
function
(
res
)
{
var
str
=
""
;
res
.
on
(
"data"
,
function
(
chunk
)
{
str
+=
chunk
;
}
)
;
res
.
on
(
"end"
,
function
(
)
{
if
(
res
.
statusCode
>=
200
&&
res
.
statusCode
<
300
)
{
callback
(
false
,
params
,
str
)
;
}
else
{
callback
(
true
,
params
)
;
}
}
)
;
}
)
;
if
(
method
===
"POST"
)
{
// write data to request body
req
.
write
(
data
)
;
}
req
.
on
(
"error"
,
function
(
err
)
{
log
(
"Connection failed."
,
err
)
;
if
(
typeof
callback
===
"function"
)
{
callback
(
true
,
params
)
;
}
}
)
;
req
.
end
(
)
;
}
catch
(
e
)
{
// fallback
log
(
"Failed HTTP request"
,
e
)
;
if
(
typeof
callback
===
"function"
)
{
callback
(
true
,
params
)
;
}
}
}


function allSettled(promises) {
    "use strict";
    const wrappedPromises = promises.map((curPromise) => curPromise.reflect());
    return Promise.all(wrappedPromises);
}

function
allSettled
(
promises
)
{
"use strict"
;
const
wrappedPromises
=
promises
.
map
(
(
curPromise
)
=>
curPromise
.
reflect
(
)
)
;
return
Promise
.
all
(
wrappedPromises
)
;
}

Creates a promise that is resolved when all input promises have been
settled.  The returned Promise is resolved with an array of
Promise.Inspection objects.

This is the commonly accepted way of implementing allSettled() in Bluebird.
See:  http://bluebirdjs.com/docs/api/reflect.html

@param promises - The array of input promises.
@returns {Promise<Promise.Inspection[]>} A promise that will be resolved once
all input Promises have settled. The returned Promise will be resolved with a
corresponding array of Promise.Inspection objects.

Creates a promise that is resolved when all input promises have been settled . The returned Promise is resolved with an array of Promise . Inspection objects .


function after(parent, index) {
  var siblings = parent.children
  var sibling = siblings[++index]
  var other

  if (is('WhiteSpaceNode', sibling)) {
    sibling = siblings[++index]

    if (is('PunctuationNode', sibling) && punctuation.test(toString(sibling))) {
      sibling = siblings[++index]
    }

    if (is('WordNode', sibling)) {
      other = sibling
    }
  }

  return other
}

function after ( parent , index ) { var siblings = parent . children var sibling = siblings [ ++ index ] var other if ( is ( 'WhiteSpaceNode' , sibling ) ) { sibling = siblings [ ++ index ] if ( is ( 'PunctuationNode' , sibling ) && punctuation . test ( toString ( sibling ) ) ) { sibling = siblings [ ++ index ] } if ( is ( 'WordNode' , sibling ) ) { other = sibling } } return other }


function classify(value) {
  var type = null
  var normal

  value = value.replace(digits, toWords).split(split, 1)[0]
  normal = lower(value)

  if (requiresA(value)) {
    type = 'a'
  }

  if (requiresAn(value)) {
    type = type === 'a' ? 'a-or-an' : 'an'
  }

  if (!type && normal === value) {
    type = vowel.test(normal.charAt(0)) ? 'an' : 'a'
  }

  return type
}

function classify ( value ) { var type = null var normal value = value . replace ( digits , toWords ) . split ( split , 1 ) [ 0 ] normal = lower ( value ) if ( requiresA ( value ) ) { type = 'a' } if ( requiresAn ( value ) ) { type = type === 'a' ? 'a-or-an' : 'an' } if ( ! type && normal === value ) { type = vowel . test ( normal . charAt ( 0 ) ) ? 'an' : 'a' } return type }


function factory(list) {
  var expressions = []
  var sensitive = []
  var insensitive = []

  construct()

  return test

  function construct() {
    var length = list.length
    var index = -1
    var value
    var normal

    while (++index < length) {
      value = list[index]
      normal = value === lower(value)

      if (value.charAt(value.length - 1) === '*') {
        // Regexes are insensitive now, once we need them this should check for
        // `normal` as well.
        expressions.push(new RegExp('^' + value.slice(0, -1), 'i'))
      } else if (normal) {
        insensitive.push(value)
      } else {
        sensitive.push(value)
      }
    }
  }

  function test(value) {
    var normal = lower(value)
    var length
    var index

    if (sensitive.indexOf(value) !== -1 || insensitive.indexOf(normal) !== -1) {
      return true
    }

    length = expressions.length
    index = -1

    while (++index < length) {
      if (expressions[index].test(value)) {
        return true
      }
    }

    return false
  }
}

Create a test based on a list of phrases.

Create a test based on a list of phrases .


function(rcb) {
                    return function() {
                        if (frc || !db[cname]) {
                            bindColCtls(db);
                        }
                        if (rcb) {
                            rcb.apply(this, arguments);
                        }
                    }
                }

function ( rcb ) { return function ( ) { if ( frc || ! db [ cname ] ) { bindColCtls ( db ) ; } if ( rcb ) { rcb . apply ( this , arguments ) ; } } }

creating callback with collection rebuilding


function() {
    Object.defineProperty(this, "_impl", {
        value : new EJDBImpl(),
        configurable : false,
        enumerable : false,
        writable : false
    });
    return this;
}

function ( ) { Object . defineProperty ( this , "_impl" , { value : new EJDBImpl ( ) , configurable : false , enumerable : false , writable : false } ) ; return this ; }

The nodejs database wrapper.
@class
@memberOf ejdb


function dummyText ( opts ) {
        var corpus = opts.corpus || 'lorem',
            i = opts.start,
            isRandom = typeof(i) === 'undefined',
            mustReset = typeof(origin) === 'undefined',
            skip = opts.skip || 1,
            sentences = opts.sentences || 1,
            words = opts.words,
            text = texts[corpus] || texts.lorem,
            len = text.length,
            output = [],
            s;

        if ( isRandom ) {
            i = Math.floor( Math.random() * len );
        }

        if ( mustReset ) {
            origin = i;
        }

        if ( isRandom ) {
            // possible modulo of a negative number, so take care here.
            i = ((i + len - origin) % len + len) % len;
        }

        while( sentences-- ) {
            s = text[i];
            if ( words ) {
                s = s.split(' ').slice(0,words).join(' ');
            }
            output.push( s );
            i = (i + skip) % len;
        }

        return output.join(' ');
    }


function Back(options) {
  if (!(this instanceof Back)) {
    return new Back(options);
  }

  this.settings = extend(options);
  this.reconnect = null;
}

function Back ( options ) { if ( ! ( this instanceof Back ) ) { return new Back ( options ) ; } this . settings = extend ( options ) ; this . reconnect = null ; }

Takes a set of reconnect options defined in README


function css(files, output, options) {
  options = Object.assign({
    banner: false
  }, options);

  return () => {
    var build = gulp.src(files)

    if (options.banner)
      build = build.pipe($.header(banner, {pkg}));

    build = build
      .pipe($.rename('d3.compose.css'))
      .pipe(gulp.dest(output));

    return build;
  };
}

function css ( files , output , options ) { options = Object . assign ( { banner : false } , options ) ; return ( ) => { var build = gulp . src ( files ) if ( options . banner ) build = build . pipe ( $ . header ( banner , { pkg } ) ) ; build = build . pipe ( $ . rename ( 'd3.compose.css' ) ) . pipe ( gulp . dest ( output ) ) ; return build ; } ; }

Create css build function

@method css
@param {String|Array} files glob
@param {String} output folder
@param {Object} [options]
@param {Boolean} [options.banner = false] Include banner with build
@return {Function}


function series() {
  const tasks = Array.prototype.slice.call(arguments);
  var fn = cb => cb();

  if (typeof tasks[tasks.length - 1] === 'function')
    fn = tasks.pop();

  return (cb) => {
    const tasks_with_cb = tasks.concat([(err) => {
      if (err) return cb(err);
      fn(cb);
    }]);

    runSequence.apply(this, tasks_with_cb);
  }
}

function series ( ) { const tasks = Array . prototype . slice . call ( arguments ) ; var fn = cb => cb ( ) ; if ( typeof tasks [ tasks . length - 1 ] === 'function' ) fn = tasks . pop ( ) ; return ( cb ) => { const tasks_with_cb = tasks . concat ( [ ( err ) => { if ( err ) return cb ( err ) ; fn ( cb ) ; } ] ) ; runSequence . apply ( this , tasks_with_cb ) ; } }

Approximate gulp 4.0 series

@param {...String} ...tasks
@param {Function} [fn] Function to call at end of series
@return {Function}


function simpleTypeFilter(doc, oldDoc, candidateDocType) {
    if (oldDoc) {
      if (doc._deleted) {
        return oldDoc.type === candidateDocType;
      } else {
        return doc.type === oldDoc.type && oldDoc.type === candidateDocType;
      }
    } else {
      return doc.type === candidateDocType;
    }
  }

function simpleTypeFilter ( doc , oldDoc , candidateDocType ) { if ( oldDoc ) { if ( doc . _deleted ) { return oldDoc . type === candidateDocType ; } else { return doc . type === oldDoc . type && oldDoc . type === candidateDocType ; } } else { return doc . type === candidateDocType ; } }

A type filter that matches on the document's type property

A type filter that matches on the document s type property


function padRight(value, desiredLength, padding) {
    while (value.length < desiredLength) {
      value += padding;
    }

    return value;
  }

function padRight ( value , desiredLength , padding ) { while ( value . length < desiredLength ) { value += padding ; } return value ; }

Add the specified padding to the right of the given string value until its length matches the desired length


function resolveCollectionDefinition(doc, oldDoc, collectionDefinition, itemPrefix) {
    if (utils.isValueNullOrUndefined(collectionDefinition)) {
      return [ ];
    } else {
      if (typeof collectionDefinition === 'function') {
        var fnResults = collectionDefinition(doc, oldDoc);

        return resolveCollectionItems(fnResults, itemPrefix);
      } else {
        return resolveCollectionItems(collectionDefinition, itemPrefix);
      }
    }
  }

function resolveCollectionDefinition ( doc , oldDoc , collectionDefinition , itemPrefix ) { if ( utils . isValueNullOrUndefined ( collectionDefinition ) ) { return [ ] ; } else { if ( typeof collectionDefinition === 'function' ) { var fnResults = collectionDefinition ( doc , oldDoc ) ; return resolveCollectionItems ( fnResults , itemPrefix ) ; } else { return resolveCollectionItems ( collectionDefinition , itemPrefix ) ; } } }

Transforms the given collection definition, which may have been defined as a single item, a list of items or a function that returns a list of items into a simple list, where each item has the specified prefix, if any

Transforms the given collection definition which may have been defined as a single item a list of items or a function that returns a list of items into a simple list where each item has the specified prefix if any


function assignRolesToUsers(doc, oldDoc, accessAssignmentDefinition) {
    var users = resolveCollectionDefinition(doc, oldDoc, accessAssignmentDefinition.users);
    var roles = resolveRoleCollectionDefinition(doc, oldDoc, accessAssignmentDefinition.roles);

    role(users, roles);

    return {
      type: 'role',
      users: users,
      roles: roles
    };
  }

function assignRolesToUsers ( doc , oldDoc , accessAssignmentDefinition ) { var users = resolveCollectionDefinition ( doc , oldDoc , accessAssignmentDefinition . users ) ; var roles = resolveRoleCollectionDefinition ( doc , oldDoc , accessAssignmentDefinition . roles ) ; role ( users , roles ) ; return { type : 'role' , users : users , roles : roles } ; }


function getAllDocChannels(docDefinition) {
    var docChannelMap = utils.resolveDocumentConstraint(docDefinition.channels);

    var allChannels = [ ];
    if (docChannelMap) {
      appendToAuthorizationList(allChannels, docChannelMap.view);
      appendToAuthorizationList(allChannels, docChannelMap.write);
      appendToAuthorizationList(allChannels, docChannelMap.add);
      appendToAuthorizationList(allChannels, docChannelMap.replace);
      appendToAuthorizationList(allChannels, docChannelMap.remove);
    }

    return allChannels;
  }

function getAllDocChannels ( docDefinition ) { var docChannelMap = utils . resolveDocumentConstraint ( docDefinition . channels ) ; var allChannels = [ ] ; if ( docChannelMap ) { appendToAuthorizationList ( allChannels , docChannelMap . view ) ; appendToAuthorizationList ( allChannels , docChannelMap . write ) ; appendToAuthorizationList ( allChannels , docChannelMap . add ) ; appendToAuthorizationList ( allChannels , docChannelMap . replace ) ; appendToAuthorizationList ( allChannels , docChannelMap . remove ) ; } return allChannels ; }

Retrieves a list of channels the document belongs to based on its specified type


function outputHelpIfNecessary(cmd, options) {
  options = options || [];
  for (var i = 0; i < options.length; i++) {
    if (options[i] === '--help' || options[i] === '-h') {
      cmd.outputHelp();
      process.exit(0);
    }
  }
}

function outputHelpIfNecessary ( cmd , options ) { options = options || [ ] ; for ( var i = 0 ; i < options . length ; i ++ ) { if ( options [ i ] === '--help' || options [ i ] === '-h' ) { cmd . outputHelp ( ) ; process . exit ( 0 ) ; } } }

Output help information if necessary

@param {Command} command to output help for
@param {Array} array of options to search for -h or --help
@api private


function humanReadableArgName(arg) {
  var nameOutput = arg.name + (arg.variadic === true ? '...' : '');

  return arg.required
    ? '<' + nameOutput + '>'
    : '[' + nameOutput + ']';
}

function humanReadableArgName ( arg ) { var nameOutput = arg . name + ( arg . variadic === true ? '...' : '' ) ; return arg . required ? '<' + nameOutput + '>' : '[' + nameOutput + ']' ; }

Takes an argument an returns its human readable equivalent for help usage.

@param {Object} arg
@return {String}
@api private

Takes an argument an returns its human readable equivalent for help usage .


function validateObjectProperties(propertyValidators, allowUnknownProperties, ignoreInternalProperties) {
        var currentItemEntry = itemStack[itemStack.length - 1];
        var objectValue = currentItemEntry.itemValue;
        var oldObjectValue = currentItemEntry.oldItemValue;

        var supportedProperties = [ ];
        for (var propertyValidatorName in propertyValidators) {
          var validator = propertyValidators[propertyValidatorName];
          if (utils.isValueNullOrUndefined(validator) || utils.isValueNullOrUndefined(resolveItemConstraint(validator.type))) {
            // Skip over non-validator fields/properties
            continue;
          }

          var propertyValue = objectValue[propertyValidatorName];

          var oldPropertyValue;
          if (!utils.isValueNullOrUndefined(oldObjectValue)) {
            oldPropertyValue = oldObjectValue[propertyValidatorName];
          }

          supportedProperties.push(propertyValidatorName);

          itemStack.push({
            itemValue: propertyValue,
            oldItemValue: oldPropertyValue,
            itemName: propertyValidatorName
          });

          validateItemValue(validator);

          itemStack.pop();
        }

        // Verify there are no unsupported properties in the object
        if (!allowUnknownProperties) {
          for (var propertyName in objectValue) {
            if (ignoreInternalProperties && propertyName.indexOf('_') === 0) {
              // These properties are special cases that should always be allowed - generally only applied at the root
              // level of the document
              continue;
            }

            if (supportedProperties.indexOf(propertyName) < 0) {
              var objectPath = buildItemPath(itemStack);
              var fullPropertyPath = objectPath ? objectPath + '.' + propertyName : propertyName;
              validationErrors.push('property "' + fullPropertyPath + '" is not supported');
            }
          }
        }
      }

The following functions are nested within this function so they can share access to the doc, oldDoc and validationErrors params and the attachmentReferenceValidators and itemStack variables

The following functions are nested within this function so they can share access to the doc oldDoc and validationErrors params and the attachmentReferenceValidators and itemStack variables


function buildItemPath(itemStack) {
    var nameComponents = [ ];
    for (var i = 0; i < itemStack.length; i++) {
      var itemName = itemStack[i].itemName;

      if (!itemName) {
        // Skip null or empty names (e.g. the first element is typically the root of the document, which has no name)
        continue;
      } else if (nameComponents.length < 1 || itemName.indexOf('[') === 0) {
        nameComponents.push(itemName);
      } else {
        nameComponents.push('.' + itemName);
      }
    }

    return nameComponents.join('');
  }

Constructs the fully qualified path of the item at the top of the given stack


function getBusinessId(doc, oldDoc) {
    var regex = /^biz\.([A-Za-z0-9_-]+)(?:\..+)?$/;
    var matchGroups = regex.exec(doc._id);
    if (matchGroups) {
      return matchGroups[1];
    } else if (oldDoc && oldDoc.businessId) {
      // The document ID doesn't contain a business ID, so use the property from the old document
      return oldDoc.businessId || null;
    } else {
      // Neither the document ID nor the old document's contents contain a business ID, so use the property from the new document
      return doc.businessId || null;
    }
  }

Retrieves the ID of the business to which the document belongs


function toDefaultSyncChannels(doc, oldDoc, basePrivilegeName) {
    var businessId = getBusinessId(doc, oldDoc);

    return function(doc, oldDoc) {
      return {
        view: [ toSyncChannel(businessId, 'VIEW_' + basePrivilegeName) ],
        add: [ toSyncChannel(businessId, 'ADD_' + basePrivilegeName) ],
        replace: [ toSyncChannel(businessId, 'CHANGE_' + basePrivilegeName) ],
        remove: [ toSyncChannel(businessId, 'REMOVE_' + basePrivilegeName) ]
      };
    };
  }

function toDefaultSyncChannels ( doc , oldDoc , basePrivilegeName ) { var businessId = getBusinessId ( doc , oldDoc ) ; return function ( doc , oldDoc ) { return { view : [ toSyncChannel ( businessId , 'VIEW_' + basePrivilegeName ) ] , add : [ toSyncChannel ( businessId , 'ADD_' + basePrivilegeName ) ] , replace : [ toSyncChannel ( businessId , 'CHANGE_' + basePrivilegeName ) ] , remove : [ toSyncChannel ( businessId , 'REMOVE_' + basePrivilegeName ) ] } ; } ; }

Builds a function that returns the view, add, replace, remove channels extrapolated from the specified base privilege, name which is formatted according to the de facto Books convention of "VIEW_FOOBAR", "ADD_FOOBAR", "CHANGE_FOOBAR" and "REMOVE_FOOBAR" assuming the base privilege name is "FOOBAR"

Builds a function that returns the view add replace remove channels extrapolated from the specified base privilege name which is formatted according to the de facto Books convention of VIEW_FOOBAR ADD_FOOBAR CHANGE_FOOBAR and REMOVE_FOOBAR assuming the base privilege name is FOOBAR


function isIso8601DateTimeString(value) {
    var dateAndTimePieces = splitDateAndTime(value);
    var date = extractDateStructureFromDateAndTime(dateAndTimePieces);
    if (date) {
      var timeAndTimezone = extractTimeStructuresFromDateAndTime(dateAndTimePieces);
      var time = timeAndTimezone.time;
      var timezone = timeAndTimezone.timezone;

      return isValidDateStructure(date) &&
        isValidTimeStructure(time) &&
        (timezone === null || isValidTimeZoneStructure(timezone));
    } else {
      return false;
    }
  }

function isIso8601DateTimeString ( value ) { var dateAndTimePieces = splitDateAndTime ( value ) ; var date = extractDateStructureFromDateAndTime ( dateAndTimePieces ) ; if ( date ) { var timeAndTimezone = extractTimeStructuresFromDateAndTime ( dateAndTimePieces ) ; var time = timeAndTimezone . time ; var timezone = timeAndTimezone . timezone ; return isValidDateStructure ( date ) && isValidTimeStructure ( time ) && ( timezone === null || isValidTimeZoneStructure ( timezone ) ) ; } else { return false ; } }

Check that a given value is a valid ISO 8601 format date string with optional time and time zone components


function normalizeIso8601Time(time, timezoneOffsetMinutes) {
    var msPerSecond = 1000;
    var msPerMinute = 60000;
    var msPerHour = 3600000;
    var effectiveTimezoneOffset = timezoneOffsetMinutes || 0;

    var rawTimeMs =
      (time.hour * msPerHour) + (time.minute * msPerMinute) + (time.second * msPerSecond) + time.millisecond;

    return rawTimeMs - (effectiveTimezoneOffset * msPerMinute);
  }

function normalizeIso8601Time ( time , timezoneOffsetMinutes ) { var msPerSecond = 1000 ; var msPerMinute = 60000 ; var msPerHour = 3600000 ; var effectiveTimezoneOffset = timezoneOffsetMinutes || 0 ; var rawTimeMs = ( time . hour * msPerHour ) + ( time . minute * msPerMinute ) + ( time . second * msPerSecond ) + time . millisecond ; return rawTimeMs - ( effectiveTimezoneOffset * msPerMinute ) ; }

Converts the given time to the number of milliseconds since hour 0


function compareTimes(a, b) {
    if (typeof a !== 'string' || typeof b !== 'string') {
      return NaN;
    }

    return normalizeIso8601Time(parseIso8601Time(a)) - normalizeIso8601Time(parseIso8601Time(b));
  }

function compareTimes ( a , b ) { if ( typeof a !== 'string' || typeof b !== 'string' ) { return NaN ; } return normalizeIso8601Time ( parseIso8601Time ( a ) ) - normalizeIso8601Time ( parseIso8601Time ( b ) ) ; }

Compares the given time strings. Returns a negative number if a is less than b, a positive number if a is greater than b, or zero if a and b are equal.

Compares the given time strings . Returns a negative number if a is less than b a positive number if a is greater than b or zero if a and b are equal .


function compareDates(a, b) {
    var aPieces = extractDatePieces(a);
    var bPieces = extractDatePieces(b);

    if (aPieces === null || bPieces === null) {
      return NaN;
    }

    for (var pieceIndex = 0; pieceIndex < aPieces.length; pieceIndex++) {
      if (aPieces[pieceIndex] < bPieces[pieceIndex]) {
        return -1;
      } else if (aPieces[pieceIndex] > bPieces[pieceIndex]) {
        return 1;
      }
    }

    // If we got here, the two parameters represent the same date/point in time
    return 0;
  }

Compares the given date representations. Returns a negative number if a is less than b, a positive number if a is greater than b, or zero if a and b are equal.

Compares the given date representations . Returns a negative number if a is less than b a positive number if a is greater than b or zero if a and b are equal .


function normalizeIso8601TimeZone(value) {
    return value ? value.multiplicationFactor * ((value.hour * 60) + value.minute) : -(new Date().getTimezoneOffset());
  }

function normalizeIso8601TimeZone ( value ) { return value ? value . multiplicationFactor * ( ( value . hour * 60 ) + value . minute ) : - ( new Date ( ) . getTimezoneOffset ( ) ) ; }

Converts an ISO 8601 time zone into the number of minutes offset from UTC


async function start() {
					log.i('--Nexus/Start');
					//build the setup promise array
					let startArray = [];

					for (let pid in Start) {
						startArray.push(new Promise((resolve, _reject) => {
							let com = {};
							com.Cmd = Start[pid];
							com.Passport = {};
							com.Passport.To = pid;
							com.Passport.Pid = genPid();
							sendMessage(com, resolve);
						}));
					}

					await Promise.all(startArray);
					log.v('--Nexus: All Starts Complete');
				}

async
function
start
(
)
{
log
.
i
(
'--Nexus/Start'
)
;
//build the setup promise array
let
startArray
=
[
]
;
for
(
let
pid
in
Start
)
{
startArray
.
push
(
new
Promise
(
(
resolve
,
_reject
)
=>
{
let
com
=
{
}
;
com
.
Cmd
=
Start
[
pid
]
;
com
.
Passport
=
{
}
;
com
.
Passport
.
To
=
pid
;
com
.
Passport
.
Pid
=
genPid
(
)
;
sendMessage
(
com
,
resolve
)
;
}
)
)
;
}
await
Promise
.
all
(
startArray
)
;
log
.
v
(
'--Nexus: All Starts Complete'
)
;
}

Call Start on the required Module Apexes


async function exit(code = 0) {
				log.i('--Nexus/Stop');
				//build the Stop promise array
				let stopTasks = [];

				log.i('Nexus unloading node modules');
				log.v(Object.keys(require.cache).join('\n'));

				for (let pid in Stop) {
					stopTasks.push(new Promise((resolve, _reject) => {
						let com = {};
						com.Cmd = Stop[pid];
						com.Passport = {};
						com.Passport.To = pid;
						com.Passport.Pid = genPid();
						sendMessage(com, resolve);
					}));
				}
				await Promise.all(stopTasks);
				log.v('--Nexus: All Stops Complete');

				dispatchEvent('exit', { exitCode: code });
			}

async
function
exit
(
code
=
0
)
{
log
.
i
(
'--Nexus/Stop'
)
;
//build the Stop promise array
let
stopTasks
=
[
]
;
log
.
i
(
'Nexus unloading node modules'
)
;
log
.
v
(
Object
.
keys
(
require
.
cache
)
.
join
(
'\n'
)
)
;
for
(
let
pid
in
Stop
)
{
stopTasks
.
push
(
new
Promise
(
(
resolve
,
_reject
)
=>
{
let
com
=
{
}
;
com
.
Cmd
=
Stop
[
pid
]
;
com
.
Passport
=
{
}
;
com
.
Passport
.
To
=
pid
;
com
.
Passport
.
Pid
=
genPid
(
)
;
sendMessage
(
com
,
resolve
)
;
}
)
)
;
}
await
Promise
.
all
(
stopTasks
)
;
log
.
v
(
'--Nexus: All Stops Complete'
)
;
dispatchEvent
(
'exit'
,
{
exitCode
:
code
}
)
;
}

Helper Functions as well as Entity definition
@description stop the system by calling all stop methods and
releasing all entities from memory
exit code 0 will gracefully exit,
@param {number} [code=0] the code to exit with


function sendMessage(com, fun = _ => _) {
				if (!('Passport' in com)) {
					log.w('Message has no Passport, ignored');
					log.w('    ' + JSON.stringify(com));
					fun('No Passport');
					return;
				}
				if (!('To' in com.Passport) || !com.Passport.To) {
					log.w('Message has no destination entity, ignored');
					log.w('    ' + JSON.stringify(com));
					fun('No recipient in message', com);
					return;
				}
				if (!('Pid' in com.Passport)) {
					log.w('Message has no message id, ignored');
					log.w('    ' + JSON.stringify(com));
					fun('No message id', com);
					return;
				}

				let pid = com.Passport.To;
				let apx = com.Passport.Apex || pid;
				if (pid in EntCache) {
					done(null, EntCache[pid]);
					return;
				} else {
					getEntityContext(pid, done);
				}

				async function done(err, entContextVolatile) {
					let entApex = await new Promise(res =>
						entContextVolatile.lock((val) => {
							res(val.Apex);
							return val;
						})
					);

					if (err) {
						log.w(err);
						log.w(JSON.stringify(com, null, 2));
						fun(err, com);
						return;
					}

					//TODO pid instanceOf ApexEntity
					if ((EntCache[pid].Apex == EntCache[pid].Pid) || (entApex == apx)) {
						let entContext = await new Promise(res => entContextVolatile.lock((context) => {
							res(context); return context;
						}));
						entContext.instance.dispatch(com, reply);
					} else {
						let err = 'Trying to send a message to a non-Apex'
							+ 'entity outside of the sending module';
						log.w(err);
						log.w(JSON.stringify(com, null, 2));
						fun(err, com);
					}
				}
				function reply(err, q) {
					// log.i('NEXUS MESSAGE:', com)
					fun(err, q);
				}
			}

Send a message from an entity to an Apex entity.
If a callback is provided, return when finished
@param {object} com 			the message object
@param {string} com.Cmd 		the command of the message
@param {object} com.Passport	the information about the message
@param {string} com.Passport.To the Pid of the recipient module
@param {string} com.Passport.Pid the ID of the message
@param {string=} com.Passport.From the Pid of the sending module
@callback fun 				the callback function to return to when finished

Send a message from an entity to an Apex entity . If a callback is provided return when finished


function deleteEntity(pid, fun = (err, _pid) => { if (err) log.e(err); }) {
				cacheInterface.deleteEntity(pid, (err, removedPidArray) => {

					//remove ent from EntCache (in RAM)
					for (let i = 0; i < removedPidArray.length; i++) {
						let entPid = removedPidArray[i];
						if (entPid in EntCache) {
							delete EntCache[entPid];
						}
					}
					log.v(`Removed ${(removedPidArray.length == 1) ? 'Entity' : 'Entities'
					} ${removedPidArray.join(' ')}`);
					fun(err, pid);
				});
			}

function
deleteEntity
(
pid
,
fun
=
(
err
,
_pid
)
=>
{
if
(
err
)
log
.
e
(
err
)
;
}
)
{
cacheInterface
.
deleteEntity
(
pid
,
(
err
,
removedPidArray
)
=>
{
//remove ent from EntCache (in RAM)
for
(
let
i
=
0
;
i
<
removedPidArray
.
length
;
i
++
)
{
let
entPid
=
removedPidArray
[
i
]
;
if
(
entPid
in
EntCache
)
{
delete
EntCache
[
entPid
]
;
}
}
log
.
v
(
`
${
(
removedPidArray
.
length
==
1
)
'Entity'
:
'Entities'
}
${
removedPidArray
.
join
(
' '
)
}
`
)
;
fun
(
err
,
pid
)
;
}
)
;
}

Delete an entity from the module's memory.  If the entity is an Apex of a Module,
then delete all the entities found in that module as well.
@param {string} pid 		the pid of the entity
@callback fun  			the callback to return the pid of the generated entity to

Delete an entity from the module s memory . If the entity is an Apex of a Module then delete all the entities found in that module as well .


async function saveEntity(par, fun = (err, _pid) => { if (err) log.e(err); }) {
				let saveEntity = (async (par) => {
					await new Promise((res, rej) => {
						cacheInterface.saveEntityPar(par, (err, pid) => {
							if (err){
								log.e(err, 'saving ', pid);
								rej(err);
							}
							log.v(`Saved entity ${par.Pid}`);
							res();
						});
					});
				});

				//check if the entity is the modules Apex
				if (par.Pid != par.Apex) {
					//check if the Apex exists in the cache
					cacheInterface.getEntityPar(par.Apex, async (err) => {
						if (err) {

							//get the Apex's par from the EntCache
							let apexPar = await new Promise((res, _rej) => {
								EntCache[par.Apex].lock((entityContext) => {
									res(entityContext.Par);
									return entityContext;
								});
							});

							log.v('Must first save the Apex -- Saving...');
							await saveEntity(apexPar);
							await saveEntity(par);
							fun(null, par.Pid);
						} else {
							//this entity is not the apex and the apex is alread in the cache
							await saveEntity(par);
							fun(null, par.Pid);
						}
					});
				} else {
					await saveEntity(par);
					fun(null, par.Pid);
				}
			}

Save an entity file. Make sure that all nested files exist in the
cache prior to saving said file
@param {object} par 		the par of the entity
@callback fun  			the callback to return the pid of the generated entity to

Save an entity file . Make sure that all nested files exist in the cache prior to saving said file


function getFile(module, filename, fun = _ => _) {
				let mod = ModCache[module];
				if (filename in mod.files) {
					mod.file(filename).async('string').then((dat) => {
						fun(null, dat);
					});
					return;
				}
				let err = `Error: File ${filename} does not exist in module ${module}`;
				log.e(err);
				fun(err);
			}

function getFile ( module , filename , fun = _ => _ ) { let mod = ModCache [ module ] ; if ( filename in mod . files ) { mod . file ( filename ) . async ( 'string' ) . then ( ( dat ) => { fun ( null , dat ) ; } ) ; return ; } let err = ` ${ filename } ${ module } ` ; log . e ( err ) ; fun ( err ) ; }

Access a file that exists in the module.json
@param {string} module 		the module to look for the file in
@param {string} filename 	the name of the file we're looking for
@callback fun				the callback to return the pid of the generated entity to

Access a file that exists in the module . json


async function getEntityContext(pid, fun = _ => _) {
				EntCache[pid] = new Volatile({});
				await EntCache[pid].lock((_entityContext) => {
					return new Promise((res, _rej) => {
						cacheInterface.getEntityPar(pid, (err, data) => {
							let par = JSON.parse(data.toString());
							if (err) {
								log.e(`Error retrieving a ${data.moduleType} from cache. Pid: ${pid}`);
								log.e(err);
								fun('Unavailable');
								return;
							}
							let impkey = par.Module + '/' + par.Entity;
							if (impkey in ImpCache) {
								BuildEnt();
								return;
							}

							GetModule(par.Module, async function (err, mod) {
								if (err) {
									log.e('Module <' + par.Module + '> not available');
									fun('Module not available');
									return;
								}

								if (!(par.Entity in mod.files)) {
									log.e('<' + par.Entity + '> not in module <' + par.Module + '>');
									fun('Null entity');
									return;
								}

								let entString = await new Promise(async (res, _rej) => {
									mod.file(par.Entity).async('string').then((string) => res(string));
								});

								log.v(`Spinning up entity ${par.Module}-${par.Entity.split('.')[0]}`);
								ImpCache[impkey] = indirectEvalImp(impkey, entString, log,
									createRequireFromModuleType(par.Module));
								BuildEnt();
							});

							function BuildEnt() {
								// TODO: rethink the whole process of having to call out a setup and start
								res(new Entity(Nxs, ImpCache[impkey], par, log));

							}
						});
					});
				});
				fun(null, EntCache[pid]);
			}

Spin up an entity from cache into memory and retrievd its context
otherwise just return it's context from memory
@param {string} apx 		the pid of the entities apex
@param {string} pid 		the pid of the entity
@callback fun  			the callback to return the pid of the generated entity to

Spin up an entity from cache into memory and retrievd its context otherwise just return it s context from memory


function GetModule(ModName, fun = _ => _) {
				ModName = ModName.replace(/:\//g, '.');
				if (ModName in ModCache) return fun(null, ModCache[ModName]);
				else cacheInterface.getModule(ModName, (err, moduleZip) => {
					if (err) return fun(err);
					ModCache[ModName] = moduleZip;
					return fun(null, ModCache[ModName]);
				});
			}

function GetModule ( ModName , fun = _ => _ ) { ModName = ModName . replace ( / :\/ / g , '.' ) ; if ( ModName in ModCache ) return fun ( null , ModCache [ ModName ] ) ; else cacheInterface . getModule ( ModName , ( err , moduleZip ) => { if ( err ) return fun ( err ) ; ModCache [ ModName ] = moduleZip ; return fun ( null , ModCache [ ModName ] ) ; } ) ; }

For retrieving modules
Modules come from the cache directory on the harddrive or the
ModCache if its already been read to RAM.
@param {Object} modRequest
@param {String} modRequest.Module
@param {String=} modRequest.Source
@param {Function} fun
@returns mod

For retrieving modules Modules come from the cache directory on the harddrive or the ModCache if its already been read to RAM .


function processSources(cfg) {

					if (typeof cfg['Sources'] === 'undefined') {
						log.e('You must defined a Sources object.\n');
						rejectSetup('You must defined a Sources object.');
						return;
					}
					let val, sources, subval;
					for (let key in cfg) {
						val = cfg[key];
						if (key == 'Sources') {
							Config.Sources = {};
							sources = cfg['Sources'];
							for (let subkey in sources) {
								subval = sources[subkey];
								switch (typeof subval) {
									case 'string': {
										Config.Sources[subkey] = Macro(subval);
										break;
									}
									case 'object': {
										Config.Sources[subkey] = {};
										for (let id in subval) {
											Config.Sources[subkey][id.toLowerCase()] =
												(typeof subval[id] == 'string') ?
													Macro(subval[id]) : subval[id];
										}
										if (!('port' in Config.Sources[subkey])) {
											Config.Sources[subkey]['port'] = 27000;
										}
										break;
									}
									default: {
										log.e(`Invalid Source ${subkey} of type ${typeof subval}.` +
											'Must be of type string or object');
									}
								}
							}
						} else {
							Config[key] = val;
						}
					}
				}

function
processSources
(
cfg
)
{
if
(
typeof
cfg
[
'Sources'
]
===
'undefined'
)
{
log
.
e
(
'You must defined a Sources object.\n'
)
;
rejectSetup
(
'You must defined a Sources object.'
)
;
return
;
}
let
val
,
sources
,
subval
;
for
(
let
key
in
cfg
)
{
val
=
cfg
[
key
]
;
if
(
key
==
'Sources'
)
{
Config
.
Sources
=
{
}
;
sources
=
cfg
[
'Sources'
]
;
for
(
let
subkey
in
sources
)
{
subval
=
sources
[
subkey
]
;
switch
(
typeof
subval
)
{
case
'string'
:
{
Config
.
Sources
[
subkey
]
=
Macro
(
subval
)
;
break
;
}
case
'object'
:
{
Config
.
Sources
[
subkey
]
=
{
}
;
for
(
let
id
in
subval
)
{
Config
.
Sources
[
subkey
]
[
id
.
toLowerCase
(
)
]
=
(
typeof
subval
[
id
]
==
'string'
)
Macro
(
subval
[
id
]
)
:
subval
[
id
]
;
}
if
(
!
(
'port'
in
Config
.
Sources
[
subkey
]
)
)
{
Config
.
Sources
[
subkey
]
[
'port'
]
=
27000
;
}
break
;
}
default
:
{
log
.
e
(
`
${
subkey
}
${
typeof
subval
}
`
+
'Must be of type string or object'
)
;
}
}
}
}
else
{
Config
[
key
]
=
val
;
}
}
}

Reads in the given config and fills in the Sources Macros


function generateModuleCatalog() {
					// Create new cache and install high level
					// module subdirectories. Each of these also
					// has a link to the source of that module (Module.json).
					let keys = Object.keys(Config.Modules);
					for (let i = 0; i < keys.length; i++) {
						let key = keys[i];
						if (key == 'Deferred') {
							let arr = Config.Modules['Deferred'];
							for (let idx = 0; idx < arr.length; idx++) {
								let mod = arr[idx];
								log.v(`Deferring ${mod.Module || mod}`);
								if (typeof mod == 'string') {
									log.w('Adding Module names directly to Deferred is deprecated');
									log.w(`Deferring { Module: '${mod}' } instead`);
									mod = { Module: mod };
								}
								if (!('Module' in mod)) {
									log.e('Malformed Deferred Module listing', mod);
									rejectSetup('Malformed Deferred Module listing');
									return;
								}
								logModule(key, mod);
							}
						} else {
							if (typeof Config.Modules[key].Module != 'string') {
								log.e('Malformed Module Definition');
								log.e(JSON.stringify(Config.Modules[key], null, 2));
							}
							logModule(key, Config.Modules[key]);
						}
					}

					/**
					 * Add the module to the Modules object if unique
					 * @param {object} mod 		The module object
					 * @param {string} mod.Module	The name of the module
					 * @param {object, string} mod.Source The Module broker or path reference
					 */
					function logModule(key, mod) {
						let folder = mod.Module.replace(/[/:]/g, '.');

						if (!('Source' in mod)) {
							log.e(`No Source Declared in module: ${key}: ${mod.Module}`);
							rejectSetup(`No Source Declared in module: ${key}`);
							return;
						}

						let source = {
							Source: mod.Source,
							Version: mod.Version
						};

						if (!(folder in Modules)) {
							Modules[folder] = source;
						} else {
							if (Modules[folder].Source != source.Source
								|| (Modules[folder].Version != source.Version)) {
								log.e(`Broker Mismatch Exception: ${key}\n`
									+ `${JSON.stringify(Modules[folder], null, 2)} - `
									+ `\n${JSON.stringify(source, null, 2)}`);
								rejectSetup('Broker Mismatch Exception');
								return;
							}
						}
					}
				}

Create a list of all required modules and their brokers


function logModule(key, mod) {
						let folder = mod.Module.replace(/[/:]/g, '.');

						if (!('Source' in mod)) {
							log.e(`No Source Declared in module: ${key}: ${mod.Module}`);
							rejectSetup(`No Source Declared in module: ${key}`);
							return;
						}

						let source = {
							Source: mod.Source,
							Version: mod.Version
						};

						if (!(folder in Modules)) {
							Modules[folder] = source;
						} else {
							if (Modules[folder].Source != source.Source
								|| (Modules[folder].Version != source.Version)) {
								log.e(`Broker Mismatch Exception: ${key}\n`
									+ `${JSON.stringify(Modules[folder], null, 2)} - `
									+ `\n${JSON.stringify(source, null, 2)}`);
								rejectSetup('Broker Mismatch Exception');
								return;
							}
						}
					}

function
logModule
(
key
,
mod
)
{
let
folder
=
mod
.
Module
.
replace
(
/
[/:]
/
g
,
'.'
)
;
if
(
!
(
'Source'
in
mod
)
)
{
log
.
e
(
`
${
key
}
${
mod
.
Module
}
`
)
;
rejectSetup
(
`
${
key
}
`
)
;
return
;
}
let
source
=
{
Source
:
mod
.
Source
,
Version
:
mod
.
Version
}
;
if
(
!
(
folder
in
Modules
)
)
{
Modules
[
folder
]
=
source
;
}
else
{
if
(
Modules
[
folder
]
.
Source
!=
source
.
Source
||
(
Modules
[
folder
]
.
Version
!=
source
.
Version
)
)
{
log
.
e
(
`
${
key
}
\n
`
+
`
${
JSON
.
stringify
(
Modules
[
folder
]
,
,
2
)
}
`
+
`
\n
${
JSON
.
stringify
(
source
,
,
2
)
}
`
)
;
rejectSetup
(
'Broker Mismatch Exception'
)
;
return
;
}
}
}

Add the module to the Modules object if unique
@param {object} mod 		The module object
@param {string} mod.Module	The name of the module
@param {object, string} mod.Source The Module broker or path reference


async function buildApexInstances(processPidReferences) {
					if (processPidReferences) {
						// Assign pids to all instance in Config.Modules
						for (let instname in Config.Modules) {
							if (instname == 'Deferred')
								continue;
							Apex[instname] = genPid();
						}
						log.v('Apex List', JSON.stringify(Apex, null, 2));
					}

					// Now populate all of the modules from config.json
					for (let instname in Config.Modules) {
						if (instname === 'Deferred')
							continue;
						await processApexPar(Apex[instname], Config.Modules[instname], processPidReferences);
					}
				}

Write the modules and all instances to the cache


async function buildDir(path) {
					let dirObj = {};
					if (fs.existsSync(path)) {
						let files = fs.readdirSync(path);
						let itemPromises = [];
						for (let file of files) {
							itemPromises.push(new Promise(async (resolve) => {
								let curPath = path + '/' + file;
								if (fs.lstatSync(curPath).isDirectory()) {
									// recurse
									dirObj[file] = await buildDir(curPath);
									resolve();
								} else {
									fs.readFile(curPath, function (err, data) {
										// log.v(curPath.length > 80 ? curPath.substr(0, 35)
										// + ' ... ' + curPath.substr(-40, 40) : curPath);
										dirObj[file] = data.toString();
										resolve();
									});
									// dirObj[file] = fs.readFileSync(curPath).toString(encoding);
								}
							}));
						}
						await Promise.all(itemPromises);
						return dirObj;
					}
				}

Build an object to represent a directory
@param {String} path


function genPid() {
					if (!Uuid) {
						// module.paths = [Path.join(Path.resolve(CacheDir), 'node_modules')];
						Uuid = require('uuid/v4');
					}
					let str = Uuid();
					let pid = str.replace(/-/g, '').toUpperCase();
					return pid;
				}


async function genesis(system) {
			log.i(' [Save Cache]'.padStart(80, '='));
			log.i('Genesis Compile Start:');

			let cacheState = null;
			if (fs.existsSync(CacheDir)) cacheState = 'exists';

			cacheInterface = new CacheInterface({
				path: CacheDir, log
			});

			cleanCache();

			log.i('Saving modules and updating dependencies ...');
			await cacheModules(system.ModCache);

			if (!(__options.state == 'updateOnly')) {
				log.i('Saving entities ...');
				await cacheApexes(system.Apex, system.Config.Modules);
			}

			Stop();

			/////////////////////////////////////////////////////////////
			//
			//	Only helper functions beyond this point of this scope
			//

			/**
			*  Remove the cache if it currently exists in the given directory
			*/
			function cleanCache() {
				// Remove the provided cache directory
				if (__options.state == 'development' && cacheState) {
					__options.state = 'updateOnly';
					return;
				}
				log.v('Removing the old cache.');
				cacheInterface.clean();
			}

			/**
			 * Write the modules to the cache
			 * @param {Object} ModCache 	//the set of module zips required for this system
			 */
			async function cacheModules(ModCache) {
				let timer = log.time('cacheModules');
				let ModulePromiseArray = [];
				for (let folder in ModCache) {
					ModulePromiseArray.push(new Promise(async (res) => {
						await cacheInterface.addModule(folder, ModCache[folder]);
						log.v(`Finished installing dependencies for ${folder}`);
						res();
					}));

				}
				await Promise.all(ModulePromiseArray);

				log.timeEnd(timer);
			}

			/**
			 * Write the module Apexes to the cache
			 * @param {Object} Apexes 						//The id:Pid of each apex
			 * @param {Object} ModuleDefinitions 	//the id:ModuleDefinition from Config
			 */
			async function cacheApexes(Apexes, ModuleDefinitions) {
				let ModulePromiseArray = [];
				for (let moduleId in Apexes) {
					ModulePromiseArray.push(
						await cacheInterface.createInstance(ModuleDefinitions[moduleId], Apexes[moduleId])
					);
				}
				await Promise.all(ModulePromiseArray);
			}

			/**
			 * Resolves the main promise created during genesis call
			 */
			async function Stop() {
				log.i(`Genesis Compile Stop: ${new Date().toString()}`);
				log.i(' [Finished]'.padStart(80, '='));
				for(const xgrl in BrokerCache) {
					const broker = BrokerCache[xgrl];
					broker.cleanup();
				}
				log.timeEnd(compileTimer);
				resolveMain();
			}
		}


async function cacheModules(ModCache) {
				let timer = log.time('cacheModules');
				let ModulePromiseArray = [];
				for (let folder in ModCache) {
					ModulePromiseArray.push(new Promise(async (res) => {
						await cacheInterface.addModule(folder, ModCache[folder]);
						log.v(`Finished installing dependencies for ${folder}`);
						res();
					}));

				}
				await Promise.all(ModulePromiseArray);

				log.timeEnd(timer);
			}

async function cacheModules ( ModCache ) { let timer = log . time ( 'cacheModules' ) ; let ModulePromiseArray = [ ] ; for ( let folder in ModCache ) { ModulePromiseArray . push ( new Promise ( async ( res ) => { await cacheInterface . addModule ( folder , ModCache [ folder ] ) ; log . v ( ` ${ folder } ` ) ; res ( ) ; } ) ) ; } await Promise . all ( ModulePromiseArray ) ; log . timeEnd ( timer ) ; }

Write the modules to the cache
@param {Object} ModCache 	//the set of module zips required for this system


async function cacheApexes(Apexes, ModuleDefinitions) {
				let ModulePromiseArray = [];
				for (let moduleId in Apexes) {
					ModulePromiseArray.push(
						await cacheInterface.createInstance(ModuleDefinitions[moduleId], Apexes[moduleId])
					);
				}
				await Promise.all(ModulePromiseArray);
			}

async function cacheApexes ( Apexes , ModuleDefinitions ) { let ModulePromiseArray = [ ] ; for ( let moduleId in Apexes ) { ModulePromiseArray . push ( await cacheInterface . createInstance ( ModuleDefinitions [ moduleId ] , Apexes [ moduleId ] ) ) ; } await Promise . all ( ModulePromiseArray ) ; }

Write the module Apexes to the cache
@param {Object} Apexes 						//The id:Pid of each apex
@param {Object} ModuleDefinitions 	//the id:ModuleDefinition from Config


async function Stop() {
				log.i(`Genesis Compile Stop: ${new Date().toString()}`);
				log.i(' [Finished]'.padStart(80, '='));
				for(const xgrl in BrokerCache) {
					const broker = BrokerCache[xgrl];
					broker.cleanup();
				}
				log.timeEnd(compileTimer);
				resolveMain();
			}

async
function
Stop
(
)
{
log
.
i
(
`
${
new
Date
(
)
.
toString
(
)
}
`
)
;
log
.
i
(
' [Finished]'
.
padStart
(
80
,
'='
)
)
;
for
(
const
xgrl
in
BrokerCache
)
{
const
broker
=
BrokerCache
[
xgrl
]
;
broker
.
cleanup
(
)
;
}
log
.
timeEnd
(
compileTimer
)
;
resolveMain
(
)
;
}

Resolves the main promise created during genesis call


function getProtocolModule(protocol) {
	return new Promise(function (resolve, reject) {
		let cacheFilepath = path.join(appdata, protocol);
		if (fs.existsSync(cacheFilepath)) {
			return resolve(JSON.parse(fs.readFileSync(cacheFilepath).toString()));
		}

		let options = {
			host: 'protocols.xgraphdev.com',
			port: 443,
			path: '/' + protocol,
			method: 'GET',
			rejectUnauthorized: false,
		};

		let req = https.request(options, function (res) {
			res.setEncoding('utf8');
			let response = '';
			res.on('data', function (chunk) {
				response += chunk;
			});
			res.on('end', _ => {
				try {
					resolve(JSON.parse(response));
					try { fs.writeFileSync(cacheFilepath, response); } catch (e) {
						reject({
							code: 1,
							text: `fail to save protocol at ${cacheFilepath}` +
								'\n delete file and try again'
						});
					}
				} catch (e) {
					reject({ code: 0, text: 'try and retrieve locally' });
				}
			});
		});

		req.on('error', function (e) {
			log.e('problem with request: ' + e.message);
			reject({ code: 1, text: 'problem with request: ' + e.message });
		});

		// write data to request body
		req.end();
	});
}

function
getProtocolModule
(
protocol
)
{
return
new
Promise
(
function
(
resolve
,
reject
)
{
let
cacheFilepath
=
path
.
join
(
appdata
,
protocol
)
;
if
(
fs
.
existsSync
(
cacheFilepath
)
)
{
return
resolve
(
JSON
.
parse
(
fs
.
readFileSync
(
cacheFilepath
)
.
toString
(
)
)
)
;
}
let
options
=
{
host
:
'protocols.xgraphdev.com'
,
port
:
443
,
path
:
'/'
+
protocol
,
method
:
'GET'
,
rejectUnauthorized
:
false
,
}
;
let
req
=
https
.
request
(
options
,
function
(
res
)
{
res
.
setEncoding
(
'utf8'
)
;
let
response
=
''
;
res
.
on
(
'data'
,
function
(
chunk
)
{
response
+=
chunk
;
}
)
;
res
.
on
(
'end'
,
_
=>
{
try
{
resolve
(
JSON
.
parse
(
response
)
)
;
try
{
fs
.
writeFileSync
(
cacheFilepath
,
response
)
;
}
catch
(
e
)
{
reject
(
{
code
:
1
,
text
:
`
${
cacheFilepath
}
`
+
'\n delete file and try again'
}
)
;
}
}
catch
(
e
)
{
reject
(
{
code
:
0
,
text
:
'try and retrieve locally'
}
)
;
}
}
)
;
}
)
;
req
.
on
(
'error'
,
function
(
e
)
{
log
.
e
(
'problem with request: '
+
e
.
message
)
;
reject
(
{
code
:
1
,
text
:
'problem with request: '
+
e
.
message
}
)
;
}
)
;
// write data to request body
req
.
end
(
)
;
}
)
;
}


function remDir(path) {
			return (new Promise(async (resolve, _reject) => {
				if (fs.existsSync(path)) {
					let files = fs.readdirSync(path);
					let promiseArray = [];

					for (let fileIndex = 0; fileIndex < files.length; fileIndex++) {
						promiseArray.push(new Promise(async (resolve2, _reject2) => {
							let curPath = path + '/' + files[fileIndex];
							if (fs.lstatSync(curPath).isDirectory()) {
								// recurse
								await remDir(curPath);
								resolve2();
							} else {
								// delete file
								log.v('Removing Entity ', files[fileIndex].split('.')[0]);
								fs.unlinkSync(curPath);
								resolve2();
							}
						}));
					}
					//make sure all the sub files and directories have been removed;
					await Promise.all(promiseArray);
					log.v('Removing Module Directory ', path);
					fs.rmdirSync(path);
					resolve();
				} else {
					log.v('trying to remove nonexistant path ', path);
					resolve();
				}
			}));
		}

Recursive directory deletion
@param {string} path the directory to be recursively removed


function getMousePosition(e) {
                var mouseObj = void 0,
                    originalEvent = e.originalEvent ? e.originalEvent : e;
                mouseObj = 'changedTouches' in originalEvent && originalEvent.changedTouches ? originalEvent.changedTouches[0] : originalEvent;
                // clientX, Y 쓰면 스크롤에서 문제 발생
                return {
                    clientX: mouseObj.pageX,
                    clientY: mouseObj.pageY
                };
            }

function
getMousePosition
(
e
)
{
var
mouseObj
=
void
0
,
originalEvent
=
e
.
originalEvent
e
.
originalEvent
:
e
;
mouseObj
=
'changedTouches'
in
originalEvent
&&
originalEvent
.
changedTouches
originalEvent
.
changedTouches
[
0
]
:
originalEvent
;
// clientX, Y 쓰면 스크롤에서 문제 발생
return
{
clientX
:
mouseObj
.
pageX
,
clientY
:
mouseObj
.
pageY
}
;
}

get mouse position
@param e
@returns {{clientX, clientY}}


function proxyRequest(req, res, rule) {
  var router,
      target,
      path;

  injectProxyHeaders(req, rule);

  // rewrite the base path of the requested URL
  path = req.url.replace(rule.regexp, rule.target.path);

  if (useGateway) {
    // for HTTP LAN proxies, rewrite the request URL from a relative URL to an absolute URL
    // also add a header that can be inspected by the unit tests
    req.url = url.parse(util.format('%s//%s:%s%s', rule.target.protocol, rule.target.host, rule.target.port, path)).href;
    req.headers['X-Forwarded-Url'] = req.url;

    // the proxy target is really the HTTP LAN proxy
    target = config.gateway;
    logger.info('proxy: %s %s --> %s:%s --> %s//%s:%s%s', req.method, req.url, config.gateway.host, config.gateway.port, rule.target.protocol, rule.target.host, rule.target.port, path);
  } else {
    target = rule.target;
    logger.info('proxy: %s %s --> %s//%s:%s%s', req.method, req.url, rule.target.protocol, rule.target.host, rule.target.port, path);
    req.url = path;
  }

  var errorCallback = function errorCallback(err, proxyRequest, proxyResponse) {
    var status = 500;
    if (proxyResponse !== undefined && proxyResponse !== null && proxyResponse.statusCode >= 400) {
      status = proxyResponse.statusCode;
    }

    logger.error('proxy: error - %s %s - %s', proxyRequest.method, proxyRequest.url, err.message);
    if( res.status && typeof res.status === 'function'  ){
        res.status(status).json({ error: status, message: err.message });
    }
  };

  // get a ProxyServer from the cache
  router = createRouter(target);

  // proxy the request
  router.web(req, res, errorCallback);
}

injects any LAN proxy servers into the request


function injectProxyHeaders(req, rule){
  // the HTTP host header is often needed by the target webserver config
  req.headers['host'] = rule.target.host + (rule.target.originalPort ? util.format(':%d', rule.target.originalPort) : '');
  // document that this request was proxied
  req.headers['via'] =  util.format('http://%s:%s', req.connection.address().address, req.connection.address().port);

  // inject any custom headers as configured
  config.headers.forEach(function(header) {
    var value = header.value,
        name = header.name;

    if(typeof(value) === 'function') {
      value = value.call(undefined, req);
    }

    if(typeof(value) !== 'string') {
      value = '';
    }

    if (typeof(name) === 'string') {
      req.headers[name.toLowerCase()] = value;
    }
  });

  injectAuthHeader(req);
}

inject any custom header values into a proxy request along with the x-forwarded-for, x-forwarded-port, and via headers

inject any custom header values into a proxy request along with the x - forwarded - for x - forwarded - port and via headers


function parseFile(filepath, config) {
  var contents;

  filepath = filepath || path.join(process.cwd(), '/json-proxy.json');

  // if we were passed a config file, read and parse it
  if (fs.existsSync(filepath)) {
    try {
      var data = fs.readFileSync(filepath);
      contents = JSON.parse(data.toString());
      config = parseConfig(contents, config);

      // replace the token $config_dir in the webroot arg
      if (config.server.webroot && config.server.webroot.length > 0) {
        config.server.webroot = config.server.webroot.replace("$config_dir", path.dirname(filepath));
      }
    } catch (ex) {
      throw new Error('Cannot parse the config file "' + filepath + '": ' + ex);
    }

  }

  return config;
}

reads a config file from either the config file specified on the command line, or fallback to a file name json-proxy.config in the working directory return true if the file can be read, otherwise return false

reads a config file from either the config file specified on the command line or fallback to a file name json - proxy . config in the working directory return true if the file can be read otherwise return false


function parseConfig(contents, config) {
  contents.server = contents.server || {};
  contents.proxy = contents.proxy || {};

  if (contents.proxy.gateway && typeof(contents.proxy.gateway) === "string" && contents.proxy.gateway.length > 0) {
    contents.proxy.gateway = parseGateway(contents.proxy.gateway);
  }

  contents.proxy.forward = parseConfigMap(contents.proxy.forward, parseForwardRule);
  contents.proxy.headers = parseConfigMap(contents.proxy.headers, parseHeaderRule);

  // override any values in the config object with values specified in the file;
  config.server.port = contents.server.port || config.server.port;
  config.server.webroot = contents.server.webroot || config.server.webroot;
  config.server.html5mode = contents.server.html5mode || config.server.html5mode;
  config.proxy.gateway = contents.proxy.gateway || config.proxy.gateway;
  config.proxy.forward = contents.proxy.forward || config.proxy.forward;
  config.proxy.headers = contents.proxy.headers || config.proxy.headers;

  return config;
}

parse a config structure, overriding any values in config

parse a config structure overriding any values in config


function parseConfigMap(map, callback) {
  var result = [];

  if (!(map instanceof Object)) {
    return map;
  }

  for(var property in map) {
    if (map.hasOwnProperty(property)) {
      result.push(callback(property, map[property]));
    }
  }

  return result;
}

function parseConfigMap ( map , callback ) { var result = [ ] ; if ( ! ( map instanceof Object ) ) { return map ; } for ( var property in map ) { if ( map . hasOwnProperty ( property ) ) { result . push ( callback ( property , map [ property ] ) ) ; } } return result ; }

transform a config hash object into an array


function parseCommandLine(argv, config) {
  if (argv) {
    // read the command line arguments if no config file was given
    parseCommandLineArgument(argv.port, function(item){
      config.server.port = item;
    });

    parseCommandLineArgument(argv.html5mode, function(item){
      config.server.html5mode = item;
    });

    parseCommandLineArgument(argv._, function(item){
      config.server.webroot = path.normalize(item);
    });

    parseCommandLineArgument(argv.gateway, function(item){
      config.proxy.gateway = parseGateway(item);
    });

    parseCommandLineArgument(argv.forward, function(item){
      var rule = parseForwardRule(item);
      var match = false;

      config.proxy.forward.forEach(function(item) {
        if (item.regexp.source === rule.regexp.source) {
          item.target = rule.target;
          match = true;
        }
      });

      if (!match) {
        config.proxy.forward.push(rule);
      }
    });

    parseCommandLineArgument(argv.header, function(item){
      var rule = parseHeaderRule(item);
      var match = false;

      config.proxy.headers.forEach(function(item) {
        if (item.name === rule.name) {
          item.value = rule.value;
          match = true;
        }
      });

      if (!match) {
        config.proxy.headers.push(rule);
      }
    });
  }

  return config;
}


function parseCommandLineArgument(arg, fn) {
  if (typeof(fn) !== 'function')
    return;

  if (Array.isArray(arg)) {
    arg.forEach(function(item) {
      fn.call(null, item);
    });
  } else {
    if (arg !== null && arg !== undefined) {
      fn.call(null, arg);
    }
  }
}

function parseCommandLineArgument ( arg , fn ) { if ( typeof ( fn ) !== 'function' ) return ; if ( Array . isArray ( arg ) ) { arg . forEach ( function ( item ) { fn . call ( null , item ) ; } ) ; } else { if ( arg !== null && arg !== undefined ) { fn . call ( null , arg ) ; } } }

argv.X will be an array if multiple -X options are provided otherwise argv.X will just be a scalar value

argv . X will be an array if multiple - X options are provided otherwise argv . X will just be a scalar value


function parseForwardRule() {
  var token,
      rule;

  if (arguments[0] === undefined || arguments[0] === null) {
    return;
  }

  if (typeof(arguments[0]) === "object") {
    return arguments[0];
  }

  try {
    token = tokenize.apply(null, arguments);
    rule = { regexp: new RegExp('^' + token.name, 'i'), target: parseTargetServer(token.value) };
  } catch (e) {
    throw new Error('cannot parse the forwarding rule ' + arguments[0] + ' - ' + e);
  }

  return rule;
}

function
parseForwardRule
(
)
{
var
token
,
rule
;
if
(
arguments
[
0
]
===
undefined
||
arguments
[
0
]
===
)
{
return
;
}
if
(
typeof
(
arguments
[
0
]
)
===
"object"
)
{
return
arguments
[
0
]
;
}
try
{
token
=
tokenize
.
apply
(
,
arguments
)
;
rule
=
{
regexp
:
new
RegExp
(
'^'
+
token
.
name
,
'i'
)
,
target
:
parseTargetServer
(
token
.
value
)
}
;
}
catch
(
e
)
{
throw
new
Error
(
'cannot parse the forwarding rule '
+
arguments
[
0
]
+
' - '
+
e
)
;
}
return
rule
;
}

parses rule syntax to create forwarding rules


function withCode(code, msg) {
    const err = new Error(msg);
    err.code = code;
    return err;
}

function withCode ( code , msg ) { const err = new Error ( msg ) ; err . code = code ; return err ; }