function updateWorkingState(repoState, branch, newWorkingState) {
    let workingStates = repoState.getWorkingStates();

    const key = branch.getFullName();
    if (newWorkingState === null) {
        // Delete
        workingStates = workingStates.delete(key);
    } else {
        // Update the entry in the map
        workingStates = workingStates.set(key, newWorkingState);
    }

    return repoState.set('workingStates', workingStates);
}

function
updateWorkingState
(
repoState
,
branch
,
newWorkingState
)
{
let
workingStates
=
repoState
.
getWorkingStates
(
)
;
const
key
=
branch
.
getFullName
(
)
;
if
(
newWorkingState
===
)
{
// Delete
workingStates
=
workingStates
.
delete
(
key
)
;
}
else
{
// Update the entry in the map
workingStates
=
workingStates
.
set
(
key
,
newWorkingState
)
;
}
return
repoState
.
set
(
'workingStates'
,
workingStates
)
;
}

Change workinState for a specific branch
@param {RepositoryState} repoState
@param {Branch} branch
@param {WorkingState | Null} newWorkingState Pass null to delete
@return {RepositoryState}


function fetchBranches(repoState, driver) {
    const oldBranches = repoState.getBranches();
    return driver.fetchBranches()
    .then((branches) => {
        return repoState.set('branches', branches);
    })
    .then(function refreshWorkingStates(repoState) {
        // Remove outdated WorkingStates
        return oldBranches.reduce((repoState, oldBranch) => {
            const fullName = oldBranch.getFullName();
            const newBranch = repoState.getBranch(fullName);
            if (newBranch === null || newBranch.getSha() !== oldBranch.getSha()) {
                // Was removed OR updated
                return updateWorkingState(repoState, oldBranch, null);
            } else {
                // Unchanged
                return repoState;
            }
        }, repoState);
    });
}

function
fetchBranches
(
repoState
,
driver
)
{
const
oldBranches
=
repoState
.
getBranches
(
)
;
return
driver
.
fetchBranches
(
)
.
then
(
(
branches
)
=>
{
return
repoState
.
set
(
'branches'
,
branches
)
;
}
)
.
then
(
function
refreshWorkingStates
(
repoState
)
{
// Remove outdated WorkingStates
return
oldBranches
.
reduce
(
(
repoState
,
oldBranch
)
=>
{
const
fullName
=
oldBranch
.
getFullName
(
)
;
const
newBranch
=
repoState
.
getBranch
(
fullName
)
;
if
(
newBranch
===
||
newBranch
.
getSha
(
)
!==
oldBranch
.
getSha
(
)
)
{
// Was removed OR updated
return
updateWorkingState
(
repoState
,
oldBranch
,
)
;
}
else
{
// Unchanged
return
repoState
;
}
}
,
repoState
)
;
}
)
;
}

Fetch the list of branches in the repository and update them all. Will clear the
WorkingStates of all branches that have updated.
@param {RepositoryState} repoState
@param {Driver} driver
@return {Promise<RepositoryState>} with list of branches fetched

Fetch the list of branches in the repository and update them all . Will clear the WorkingStates of all branches that have updated .


function initialize(driver) {
    const repoState = RepositoryState.createEmpty();
    return fetchBranches(repoState, driver)
    .then((repoState) => {
        const branches = repoState.getBranches();
        const master = branches.find(function isMaster(branch) {
            return branch.getFullName() === 'master';
        });
        const branch = master || branches.first();

        return fetchTree(repoState, driver, branch)
        .then((repoState) => {
            return checkout(repoState, branch);
        });
    });
}

function initialize ( driver ) { const repoState = RepositoryState . createEmpty ( ) ; return fetchBranches ( repoState , driver ) . then ( ( repoState ) => { const branches = repoState . getBranches ( ) ; const master = branches . find ( function isMaster ( branch ) { return branch . getFullName ( ) === 'master' ; } ) ; const branch = master || branches . first ( ) ; return fetchTree ( repoState , driver , branch ) . then ( ( repoState ) => { return checkout ( repoState , branch ) ; } ) ; } ) ; }

Initialize a new RepositoryState from the repo of a Driver. Fetch
the branches, and checkout master or the first available branch.
@param {Driver} driver
@return {Promise<RepositoryState>}

Initialize a new RepositoryState from the repo of a Driver . Fetch the branches and checkout master or the first available branch .


function enforceArrayBuffer(b, encoding) {
    if (isArrayBuffer(b)) return b;
    else if (isBuffer(b)) return fromBuffer(b);
    else return fromString(b, encoding);
}

function enforceArrayBuffer ( b , encoding ) { if ( isArrayBuffer ( b ) ) return b ; else if ( isBuffer ( b ) ) return fromBuffer ( b ) ; else return fromString ( b , encoding ) ; }


function enforceString(b, encoding) {
    if (is.string(b)) return b;
    if (isArrayBuffer(b)) b = toBuffer(b);

    return b.toString(encoding);
}

function enforceString ( b , encoding ) { if ( is . string ( b ) ) return b ; if ( isArrayBuffer ( b ) ) b = toBuffer ( b ) ; return b . toString ( encoding ) ; }

Force conversion to string with specific encoding


function equals(buf1, buf2) {
    if (buf1.byteLength != buf2.byteLength) return false;
    const dv1 = new Int8Array(buf1);
    const dv2 = new Int8Array(buf2);
    for (let i = 0 ; i != buf1.byteLength ; i++) {
        if (dv1[i] != dv2[i]) return false;
    }
    return true;
}

function equals ( buf1 , buf2 ) { if ( buf1 . byteLength != buf2 . byteLength ) return false ; const dv1 = new Int8Array ( buf1 ) ; const dv2 = new Int8Array ( buf2 ) ; for ( let i = 0 ; i != buf1 . byteLength ; i ++ ) { if ( dv1 [ i ] != dv2 [ i ] ) return false ; } return true ; }

Tests equality of two ArrayBuffer
@param {ArrayBuffer} buf1
@param {ArrayBuffer} buf2
@return {Boolean}


function getMergedFileSet(workingState) {
    return Immutable.Set.fromKeys(
        getMergedTreeEntries(workingState).filter(
            treeEntry => treeEntry.getType() === TreeEntry.TYPES.BLOB
        )
    );
}

function getMergedFileSet ( workingState ) { return Immutable . Set . fromKeys ( getMergedTreeEntries ( workingState ) . filter ( treeEntry => treeEntry . getType ( ) === TreeEntry . TYPES . BLOB ) ) ; }

Returns a Seq of tree mixing changes and the fetched tree
@param {WorkingState}
@return {Set<Path>}


function getMergedTreeEntries(workingState) {
    const removedOrModified = workingState.getChanges().groupBy((change, path) => {
        if (change.getType() === CHANGES.REMOVE) {
            return 'remove';
        } else {
            // Must be UDPATE or CREATE
            return 'modified';
        }
    });

    const setToRemove = Immutable.Set.fromKeys(removedOrModified.get('remove', []));

    const withoutRemoved = workingState.getTreeEntries().filter((treeEntry, path) => {
        return !setToRemove.contains(path);
    });

    const addedTreeEntries = removedOrModified.get('modified', []).map(
        function toTreeEntry(change) {
            return new TreeEntry({
                sha: change.hasSha() ? change.getSha() : null,
                mode: '100644'
            });
        }
    );

    return withoutRemoved.concat(addedTreeEntries);
}

function
getMergedTreeEntries
(
workingState
)
{
const
removedOrModified
=
workingState
.
getChanges
(
)
.
groupBy
(
(
change
,
path
)
=>
{
if
(
change
.
getType
(
)
===
CHANGES
.
REMOVE
)
{
return
'remove'
;
}
else
{
// Must be UDPATE or CREATE
return
'modified'
;
}
}
)
;
const
setToRemove
=
Immutable
.
Set
.
fromKeys
(
removedOrModified
.
get
(
'remove'
,
[
]
)
)
;
const
withoutRemoved
=
workingState
.
getTreeEntries
(
)
.
filter
(
(
treeEntry
,
path
)
=>
{
return
!
setToRemove
.
contains
(
path
)
;
}
)
;
const
addedTreeEntries
=
removedOrModified
.
get
(
'modified'
,
[
]
)
.
map
(
function
toTreeEntry
(
change
)
{
return
new
TreeEntry
(
{
sha
:
change
.
hasSha
(
)
change
.
getSha
(
)
:
,
mode
:
'100644'
}
)
;
}
)
;
return
withoutRemoved
.
concat
(
addedTreeEntries
)
;
}

Returns a Map of TreeEntry, with sha null when the content is not available as sha.
@param {WorkingState}
@return {Map<TreeEntries>}

Returns a Map of TreeEntry with sha null when the content is not available as sha .


function findSha(workingState, filepath) {
    // Lookup potential changes
    const change = workingState.getChanges().get(filepath);
    // Else lookup tree
    const treeEntry = workingState.getTreeEntries().get(filepath);

    if (change) {
        if (change.getType() == CHANGES.REMOVE) {
            throw error.fileNotFound(filepath);
        } else {
            return change.getSha();
        }
    } else if (treeEntry) {
        return treeEntry.getSha();
    } else {
        throw error.fileNotFound(filepath);
    }
}

function
findSha
(
workingState
,
filepath
)
{
// Lookup potential changes
const
change
=
workingState
.
getChanges
(
)
.
get
(
filepath
)
;
// Else lookup tree
const
treeEntry
=
workingState
.
getTreeEntries
(
)
.
get
(
filepath
)
;
if
(
change
)
{
if
(
change
.
getType
(
)
==
CHANGES
.
REMOVE
)
{
throw
error
.
fileNotFound
(
filepath
)
;
}
else
{
return
change
.
getSha
(
)
;
}
}
else
if
(
treeEntry
)
{
return
treeEntry
.
getSha
(
)
;
}
else
{
throw
error
.
fileNotFound
(
filepath
)
;
}
}

Attempts to find a SHA if available for the  given file
@param {Path}
@return {Sha | Null} null if Sha cannot be retrieved (because of pending change)
@throws NOT_FOUND if the file does not exist or was removed


function prepare(repoState, opts) {
    const workingState = repoState.getCurrentState();
    const changes = workingState.getChanges();

    // Is this an empty commit ?
    opts.empty = workingState.isClean();

    // Parent SHA
    opts.parents = new Immutable.List([
        workingState.getHead()
    ]);

    // Get merged tree (with applied changes)
    opts.treeEntries = WorkingUtils.getMergedTreeEntries(workingState);

    // Create map of blobs that needs to be created
    opts.blobs = changes.filter((change) => {
        return !change.hasSha();
    }).map((change) => {
        return change.getContent();
    });

    return CommitBuilder.create(opts);
}

Create a commit builder from the changes on current branch
@param {RepositoryState}
@param {Author} opts.author
@param {String} [opts.message]
@return {CommitBuilder}


function flush(repoState, driver, commitBuilder, options = {}) {
    options = Object.assign({
        branch: repoState.getCurrentBranch(),
        ignoreEmpty: true
    }, options);

    if (options.ignoreEmpty
        && commitBuilder.isEmpty()
        && commitBuilder.getParents().count() < 2) {
        return Q(repoState);
    }

    // Create new commit
    return driver.flushCommit(commitBuilder)
    // Forward the branch
    .then((commit) => {
        return driver.forwardBranch(options.branch, commit.getSha())
        // Fetch new workingState and replace old one
        .then(function updateBranch() {
            const updated = options.branch.merge({ commit });
            return repoState.updateBranch(options.branch, updated);

        }, function nonFF(err) {
            if (err.code === ERRORS.NOT_FAST_FORWARD) {
                // Provide the created commit to allow merging it back.
                err.commit = commit;
            }
            throw err;
        });
    })
    .then(function updateWorkingState(forwardedRepoState) {
        const forwardedBranch = forwardedRepoState.getBranch(options.branch.getFullName());
        return RepoUtils.fetchTree(forwardedRepoState, driver, forwardedBranch);
    });
}

Flush a commit from the current branch using a driver
Then update the reference, and pull new workingState
@param {RepositoryState} repoState
@param {Driver} driver
@param {CommitBuilder} commitBuilder
@param {Branch} [options.branch] Optional branch to use instead of
current branch
@param {Boolean} [options.ignoreEmpty=true] Empty commits are
ignored, unless they merge several branches.
@return {Promise<RepositoryState, ERRORS.NOT_FAST_FORWARD>} If the
branch cannot be fast forwarded to the created commit, fails with
NOT_FAST_FORWARD. The error will contains the created Commit.

Flush a commit from the current branch using a driver Then update the reference and pull new workingState


function format(color, messages) {
  var length = messages.length;

  if (length === 0 || typeof(color) !== 'string') {
    return;
  }

  return (util.format.apply(null, messages)[color]);
}

function format ( color , messages ) { var length = messages . length ; if ( length === 0 || typeof ( color ) !== 'string' ) { return ; } return ( util . format . apply ( null , messages ) [ color ] ) ; }

colorize strings and send to console.log

colorize strings and send to console . log


function push(repoState, driver, opts = {}) {
    opts = Object.assign({
        branch: repoState.getCurrentBranch(),
        force: false,
        remote: {
            name: 'origin'
        }
    }, opts);

    return driver.push(opts) // Can fail with NOT_FAST_FORWARD
    // TODO update remote branch in repoState list of branches
    .thenResolve(repoState);
}

Push a local branch to a remote repository
@param {RepositoryState} repoState
@param {Driver} driver
@param {Branch} [opts.branch] Branch to push. Default to current
@param {String} [opts.remote.name=origin] Name of the remote
@param {String} [opts.remote.url] URL if the remote needs to be created
@param {Boolean} [opts.force=false] Ignore non fast forward
@param {String} [opts.auth.username] Authentication username
@param {String} [opts.auth.password] Authentication password
@return {Promise<RepositoryState>}
@throws {Promise<ERROR.NOT_FAST_FORWARD>}
@throws {Promise<ERROR.AUTHENTICATION_FAILED>}
@throws {Promise<ERROR.UNKNOWN_REMOTE>}


function pull(repoState, driver, opts = {}) {
    opts = Object.assign({
        branch: repoState.getCurrentBranch(),
        force: false,
        remote: {
            name: 'origin'
        }
    }, opts);

    return driver.pull(opts)
    // Update branch SHA
    .then(() => {
        return driver.fetchBranches();
    })
    .then((branches) => {
        const updatedBranch = branches.find((br) => {
            return br.name === opts.branch.name;
        });
        repoState = repoState.updateBranch(opts.branch, updatedBranch);

        return RepoUtils.fetchTree(repoState, driver, updatedBranch);
    });
}

function
pull
(
repoState
,
driver
,
opts
=
{
}
)
{
opts
=
Object
.
assign
(
{
branch
:
repoState
.
getCurrentBranch
(
)
,
force
:
false
,
remote
:
{
name
:
'origin'
}
}
,
opts
)
;
return
driver
.
pull
(
opts
)
// Update branch SHA
.
then
(
(
)
=>
{
return
driver
.
fetchBranches
(
)
;
}
)
.
then
(
(
branches
)
=>
{
const
updatedBranch
=
branches
.
find
(
(
br
)
=>
{
return
br
.
name
===
opts
.
branch
.
name
;
}
)
;
repoState
=
repoState
.
updateBranch
(
opts
.
branch
,
updatedBranch
)
;
return
RepoUtils
.
fetchTree
(
repoState
,
driver
,
updatedBranch
)
;
}
)
;
}

Pulls changes for local branch, from remote repository. Loses any
pending changes on it.
@param {RepositoryState} repoState
@param {Driver} driver
@param {Branch} [opts.branch] Branch to pull. Default to current
@param {String} [opts.remote.name=origin] Name of the remote
@param {String} [opts.remote.url] URL if the remote needs to be created
@param {Boolean} [opts.force=false] Ignore non fast forward
@param {String} [opts.auth.username] Authentication username
@param {String} [opts.auth.password] Authentication password
@return {Promise<RepositoryState>}
@throws {Promise<ERROR.NOT_FAST_FORWARD>}
@throws {Promise<ERROR.AUTHENTICATION_FAILED>}
@throws {Promise<ERROR.UNKNOWN_REMOTE>}

Pulls changes for local branch from remote repository . Loses any pending changes on it .


function compareRefs(driver, base, head) {
    const baseRef = base instanceof Branch ? base.getFullName() : base;
    const headRef = head instanceof Branch ? head.getFullName() : head;

    return driver.findParentCommit(baseRef, headRef)
    .then((parentCommit) => {
        // There can be no parent commit
        return Q.all([
            parentCommit ? parentCommit.getSha() : null,
            baseRef,
            headRef
        ].map((ref) => {
            return ref ? driver.fetchWorkingState(ref) : WorkingState.createEmpty();
        }));
    })
    .spread((parent, base, head) => {
        const conflicts = _compareTrees(parent.getTreeEntries(),
                                     base.getTreeEntries(),
                                     head.getTreeEntries());

        return new TreeConflict({
            base,
            head,
            parent,
            conflicts
        });
    });
}

function
compareRefs
(
driver
,
base
,
head
)
{
const
baseRef
=
base
instanceof
Branch
base
.
getFullName
(
)
:
base
;
const
headRef
=
head
instanceof
Branch
head
.
getFullName
(
)
:
head
;
return
driver
.
findParentCommit
(
baseRef
,
headRef
)
.
then
(
(
parentCommit
)
=>
{
// There can be no parent commit
return
Q
.
all
(
[
parentCommit
parentCommit
.
getSha
(
)
:
,
baseRef
,
headRef
]
.
map
(
(
ref
)
=>
{
return
ref
driver
.
fetchWorkingState
(
ref
)
:
WorkingState
.
createEmpty
(
)
;
}
)
)
;
}
)
.
spread
(
(
parent
,
base
,
head
)
=>
{
const
conflicts
=
_compareTrees
(
parent
.
getTreeEntries
(
)
,
base
.
getTreeEntries
(
)
,
head
.
getTreeEntries
(
)
)
;
return
new
TreeConflict
(
{
base
,
head
,
parent
,
conflicts
}
)
;
}
)
;
}

Computes a TreeConflict between to tree references. Fetches the
trees from the repo. The list of conflicts is the minimal set of
conflicts.
@param {Driver} driver
@param {Branch | String} base A branch, branch name, or SHA
@param {Branch | String} head A branch, branch name, or SHA
@return {Promise<TreeConflict>}

Computes a TreeConflict between to tree references . Fetches the trees from the repo . The list of conflicts is the minimal set of conflicts .


function solveTree(treeConflict, solved) {
    solved = treeConflict.getConflicts()
    .merge(solved)
    // Solve unresolved conflicts
    .map(function defaultSolve(conflict) {
        if (!conflict.isSolved()) {
            return conflict.keepBase();
        } else {
            return conflict;
        }
    });
    return treeConflict.set('conflicts', solved);
}

function
solveTree
(
treeConflict
,
solved
)
{
solved
=
treeConflict
.
getConflicts
(
)
.
merge
(
solved
)
// Solve unresolved conflicts
.
map
(
function
defaultSolve
(
conflict
)
{
if
(
!
conflict
.
isSolved
(
)
)
{
return
conflict
.
keepBase
(
)
;
}
else
{
return
conflict
;
}
}
)
;
return
treeConflict
.
set
(
'conflicts'
,
solved
)
;
}

Merge solved Conflicts back into a TreeConflict. Unsolved conflicts
default to keep base.
@param {TreeConflict} treeConflict
@param {Map<Path, Conflict>} solved
@return {TreeConflict}

Merge solved Conflicts back into a TreeConflict . Unsolved conflicts default to keep base .


function mergeCommit(treeConflict, parents, options) {
    options = options || {};

    const opts = {};

    // Assume the commit is not empty
    opts.empty = false;

    // Parent SHAs
    opts.parents = new Immutable.List(parents);

    opts.author = options.author;
    opts.message = options.message || 'Merged commit';

    // Get the solved tree entries
    const solvedEntries = _getSolvedEntries(treeConflict);
    opts.treeEntries = solvedEntries;

    // Create map of blobs that needs to be created
    const solvedConflicts = treeConflict.getConflicts();
    opts.blobs = solvedEntries.filter((treeEntry) => {
        return !treeEntry.hasSha();
    }).map((treeEntry, path) => {
        return solvedConflicts.get(path).getSolvedContent();
    });

    return CommitBuilder.create(opts);
}

Create a merge commit builder
@param {TreeConflict} treeConflict The solved TreeConflict
@param {Array<SHA>} parents Parent commits
@param {Author} options.author
@param {String} [options.message='Merge commit']
@return {CommitBuilder}


function _getSolvedEntries(treeConflict) {
    const parentEntries = treeConflict.getParent().getTreeEntries();
    const baseEntries = treeConflict.getBase().getTreeEntries();
    const headEntries = treeConflict.getHead().getTreeEntries();

    const baseDiff = _diffEntries(parentEntries, baseEntries);
    const headDiff = _diffEntries(parentEntries, headEntries);

    const resolvedEntries = treeConflict.getConflicts().map((solvedConflict) => {
        // Convert to TreeEntries (or null for deletion)
        if (solvedConflict.isDeleted()) {
            return null;
        } else {
            return new TreeEntry({
                sha: solvedConflict.getSolvedSha() || null
            });
        }
    });

    return parentEntries.merge(baseDiff, headDiff, resolvedEntries)
    // Remove deleted entries
    .filter(function nonNull(entry) {
        return entry !== null;
    });
}

Returns the final TreeEntries for a solved TreeConflict.
@param {TreeConflict} treeConflict
@return {Map<Path, TreeEntry>} Some TreeEntries have a null SHA
because of new solved content.

Returns the final TreeEntries for a solved TreeConflict .


function addBlob(cache, sha, blob) {
    const blobs = cache.getBlobs();
    const newBlobs = blobs.set(sha, blob);

    const newCache = cache.set('blobs', newBlobs);
    return newCache;
}

function addBlob ( cache , sha , blob ) { const blobs = cache . getBlobs ( ) ; const newBlobs = blobs . set ( sha , blob ) ; const newCache = cache . set ( 'blobs' , newBlobs ) ; return newCache ; }

Add a new blob to a cache instance
@param {Cache} cache
@param {String} sha Used as key
@param {Blob} blob
@return {Cache}


function get(repoState, dirPath) {
    // Remove trailing '/' etc.
    const normDirPath = Path.join(dirPath, '.');
    const filepaths = DirUtils.readFilenamesRecursive(repoState, normDirPath);

    const tree = {
        value: File.createDir(normDirPath),
        children: {}
    };

    for (let i = 0; i < filepaths.length; i++) {
        const relativePath = Path.relative(normDirPath, filepaths[i]);
        const parts = relativePath.split('/');
        let node = tree;
        let prefix = normDirPath;
        for (let j = 0; j < parts.length; j++) {
            const head = parts[j];
            const isLeaf = (j === parts.length - 1);
            prefix = Path.join(prefix, head);

            // Create node if doesn't exist
            if (!node.children[head]) {
                if (isLeaf) {
                    node.children[head] = {
                        value: FileUtils.stat(repoState, filepaths[i])
                    };
                } else {
                    node.children[head] = {
                        value: File.createDir(prefix),
                        children: {}
                    };
                }
            }
            node = node.children[head];
        }
    }

    return TreeNode.fromJS(tree);
}

function
get
(
repoState
,
dirPath
)
{
// Remove trailing '/' etc.
const
normDirPath
=
Path
.
join
(
dirPath
,
'.'
)
;
const
filepaths
=
DirUtils
.
readFilenamesRecursive
(
repoState
,
normDirPath
)
;
const
tree
=
{
value
:
File
.
createDir
(
normDirPath
)
,
children
:
{
}
}
;
for
(
let
i
=
0
;
i
<
filepaths
.
length
;
i
++
)
{
const
relativePath
=
Path
.
relative
(
normDirPath
,
filepaths
[
i
]
)
;
const
parts
=
relativePath
.
split
(
'/'
)
;
let
node
=
tree
;
let
prefix
=
normDirPath
;
for
(
let
j
=
0
;
j
<
parts
.
length
;
j
++
)
{
const
head
=
parts
[
j
]
;
const
isLeaf
=
(
j
===
parts
.
length
1
)
;
prefix
=
Path
.
join
(
prefix
,
head
)
;
// Create node if doesn't exist
if
(
!
node
.
children
[
head
]
)
{
if
(
isLeaf
)
{
node
.
children
[
head
]
=
{
value
:
FileUtils
.
stat
(
repoState
,
filepaths
[
i
]
)
}
;
}
else
{
node
.
children
[
head
]
=
{
value
:
File
.
createDir
(
prefix
)
,
children
:
{
}
}
;
}
}
node
=
node
.
children
[
head
]
;
}
}
return
TreeNode
.
fromJS
(
tree
)
;
}

Generate a files tree from the current branch, taking pending changes into account.
@param {RepositoryState} repoState
@param {Path} [dir='.'] The directory to get the subtree from, default to root
@return {TreeNode<File>} The directory TreeNode with all its children

Generate a files tree from the current branch taking pending changes into account .


function normCreatedCommit(ghCommit) {
    const commit = Commit.create({
        sha: ghCommit.sha,
        message: ghCommit.message,
        author: getSimpleAuthor(ghCommit.author),
        date: ghCommit.author.date,
        parents: ghCommit.parents.map(function getSha(o) { return o.sha; })
    });

    return commit;
}

function normCreatedCommit ( ghCommit ) { const commit = Commit . create ( { sha : ghCommit . sha , message : ghCommit . message , author : getSimpleAuthor ( ghCommit . author ) , date : ghCommit . author . date , parents : ghCommit . parents . map ( function getSha ( o ) { return o . sha ; } ) } ) ; return commit ; }

Normalize a commit coming from the GitHub commit creation API
@param {JSON} ghCommit
@return {Commit}


function normListedCommit(ghCommit) {
    const commit = Commit.create({
        sha: ghCommit.sha,
        message: ghCommit.commit.message,
        author: getCompleteAuthor(ghCommit),
        date: ghCommit.commit.author.date,
        files: ghCommit.files,
        parents: ghCommit.parents.map(c => c.sha)
    });

    return commit;
}

function normListedCommit ( ghCommit ) { const commit = Commit . create ( { sha : ghCommit . sha , message : ghCommit . commit . message , author : getCompleteAuthor ( ghCommit ) , date : ghCommit . commit . author . date , files : ghCommit . files , parents : ghCommit . parents . map ( c => c . sha ) } ) ; return commit ; }

Normalize a commit coming from the GitHub commit listing API
@param {JSON} ghCommit
@return {Commit}


function stat(repoState, filepath) {
    const workingState = repoState.getCurrentState();

    // Lookup potential changes
    const change = workingState.getChanges().get(filepath);
    // Lookup file entry
    const treeEntry = workingState.getTreeEntries().get(filepath);

    // Determine SHA of the blob
    let blobSHA;
    if (change) {
        blobSHA = change.getSha();
    } else {
        blobSHA = treeEntry.getSha();
    }

    // Get the blob from change or cache
    let blob;
    if (blobSHA) {
        // Get content from cache
        blob = repoState.getCache().getBlob(blobSHA);
    } else {
        // No sha, so it must be in changes
        blob = change.getContent();
    }

    let fileSize;
    if (blob) {
        fileSize = blob.getByteLength();
    } else {
        // It might have been moved (but not fetched)
        const originalEntry = workingState.getTreeEntries().find((entry) => {
            return entry.getSha() === blobSHA;
        });
        fileSize = originalEntry.getBlobSize();
    }

    return new File({
        type: FILETYPE.FILE,
        fileSize,
        path: filepath,
        content: blob
    });
}

Stat details about a file.
@param {RepositoryState} repoState
@param {Path} filepath
@return {File}


function readAsString(repoState, filepath, encoding) {
    const blob = read(repoState, filepath);
    return blob.getAsString(encoding);
}

function readAsString ( repoState , filepath , encoding ) { const blob = read ( repoState , filepath ) ; return blob . getAsString ( encoding ) ; }

Read content of a file, returns a String
@return {String}


function exists(repoState, filepath) {
    const workingState = repoState.getCurrentState();
    const mergedFileSet = WorkingUtils.getMergedTreeEntries(workingState);

    return mergedFileSet.has(filepath);
}

function exists ( repoState , filepath ) { const workingState = repoState . getCurrentState ( ) ; const mergedFileSet = WorkingUtils . getMergedTreeEntries ( workingState ) ; return mergedFileSet . has ( filepath ) ; }

Return true if file exists in working tree, false otherwise

Return true if file exists in working tree false otherwise


function remove(repoState, filepath) {
    if (!exists(repoState, filepath)) {
        throw error.fileNotFound(filepath);
    }

    const change = Change.createRemove();
    return ChangeUtils.setChange(repoState, filepath, change);
}

function remove ( repoState , filepath ) { if ( ! exists ( repoState , filepath ) ) { throw error . fileNotFound ( filepath ) ; } const change = Change . createRemove ( ) ; return ChangeUtils . setChange ( repoState , filepath , change ) ; }


function move(repoState, filepath, newFilepath) {
    if (filepath === newFilepath) {
        return repoState;
    }

    const initialWorkingState = repoState.getCurrentState();

    // Create new file, with Sha if possible
    const sha = WorkingUtils.findSha(initialWorkingState, filepath);
    let changeNewFile;
    if (sha) {
        changeNewFile = Change.createCreateFromSha(sha);
    } else {
        // Content not available as blob
        const blob = read(repoState, filepath);
        const contentBuffer = blob.getAsBuffer();
        changeNewFile = Change.createCreate(contentBuffer);
    }

    // Remove old file
    const removedRepoState = remove(repoState, filepath);
    // Add new file
    return ChangeUtils.setChange(removedRepoState, newFilepath, changeNewFile);
}


function hasChanged(previousState, newState, filepath) {
    const previouslyExists = exists(previousState, filepath);
    const newExists = exists(newState, filepath);
    if (!previouslyExists && !newExists) {
        // Still non existing
        return false;
    } else if (exists(previousState, filepath) !== exists(newState, filepath)) {
        // The file is absent from one
        return true;
    } else {
        // Both files exist
        const prevWorking = previousState.getCurrentState();
        const newWorking = newState.getCurrentState();

        const prevSha = WorkingUtils.findSha(prevWorking, filepath);
        const newSha = WorkingUtils.findSha(newWorking, filepath);
        if (prevSha === null && newSha === null) {
            // Both have are in pending changes. We can compare their contents
            return read(previousState, filepath).getAsString() !==
                read(newState, filepath).getAsString();
        } else {
            // Content changed if Shas are different, or one of them is null
            return prevSha !== newSha;
        }
    }
}

Returns true if the given file has the same content in both
RepositoryState's current working state, or is absent from both.
@param {RepositoryState} previousState
@param {RepositoryState} newState
@param {Path} filepath
@return {Boolean}

Returns true if the given file has the same content in both RepositoryState s current working state or is absent from both .


function setup(connection, done) {
  var config = createDefaultConfig(),
      options = {
        proxy: false,
        headers: {}
      };

  // reset the global variable with handles to port numbers
  handles = {};

  if (connection !== 'direct') {
    options.proxy = true;

    config.proxy.gateway = {
      protocol: 'http:',
      host: 'localhost',
      port: 0,
      auth: 'proxyuser:C0mp13x_!d0rd$$@P!'
    };

    // optionally test a non-RFC proxy that expects explicit values
    // for the  Via and/or Host request headers
    if (connection === 'non-rfc-proxy') {
      config.proxy.headers['Via'] = 'http://jedi.example.com';
      config.proxy.headers['Host'] = 'force.example.com';
    }

    // the config map will be mutated by the json-proxy library, so clone it
    options.headers = require('util')._extend(config.proxy.headers);

    configureLanProxy(options, config, function() {
      configureNock(options, config);
      configureExpress(config, done);
    })
  } else {
    configureNock(options, config);
    configureExpress(config, done);
  }
}

Initializes the testing infrastructure needed for
verifying the behavior of the core proxy library -
nock, express, and httpServer.

@param  {String}   connection one of 'direct', 'proxy', or 'non-rfc-proxy'
@param  {Function} done    the async callback

Initializes the testing infrastructure needed for verifying the behavior of the core proxy library - nock express and httpServer .


function configureNock(options, config) {
  var result = {};
  // deny all real net connections except for localhost
  nock.disableNetConnect();
  nock.enableNetConnect('localhost');

  function createNock(url) {
    var instance = nock(url),
        expectedViaHeader = options.headers['Via'] ||
                            'http://localhost:' + config.proxy.gateway.port,
        expectedHostHeader = options.headers['Host'] || /.*/;

    if (options.proxy === true) {
      // verify that the request was actually proxied
      // optionally support downstream proxies with non-RFC expectations on
      // the Host and Via request headers
      instance.matchHeader('via', expectedViaHeader);
      instance.matchHeader('host', expectedHostHeader);
    }

    // verify the injected header
    instance.matchHeader('x-test-header', 'John Doe');

    return instance;
  }

  rules = [
    createNock('http://api.example.com')
    .get('/api/hello')
    .reply(200, '{ "hello": "world" }')
    .get('/account?id=1&email=2&sort=asc')
    .reply(200, '{ "email": "john.doe@example.com" }')
    .get('/api/notfound')
    .reply(404),

    createNock('http://rewrite.example.com')
    .get('/hello')
    .reply(200, '{ "hello": "world" }')
    .get('/foo/bar')
    .reply(200, '{ "foo": "bar" }'),

    createNock('http://www.example.com')
    .get('/foo/12345/bar')
    .reply(200, '{ "foo": "bar" }')
    .get('/subapp/junction/customer/1')
    .reply(200, '{ "id": 1 }'),

    createNock('https://api.example.biz')
    .get('/issue/8')
    .reply(200, '{ "reporter": "@heygrady" }'),

    createNock('https://secure.example.com')
    .get('/secure/api/hello')
    .reply(200, '{ "hello": "world" }')
    .get('/secure/api/notfound')
    .reply(404),

    createNock('https://authorization.example.com')
    .matchHeader('X-Test-Header-Function', 'Bearer 0123456789abcdef')
    .get('/token')
    .reply(200, '{ "author": "ehtb" }')
  ];
}

configures nock globally for a test run

@param  {Object.<string, string>} options  the options for configuring this
test suite
@param  {Object.<string, string>} config   the config of the json-proxy for
this test suite
@returns {Array}  an array of configured nock instances


function configureExpress(config, done) {
  var portfinder = require('portfinder');

  tmp.dir(function(err, filepath){
    handles.filepath = filepath;

    portfinder.getPort(function (err, port) {
      if (err) throw(err);

      handles.port = port;

      fs.writeFileSync(path.join(handles.filepath, 'index.txt'), 'hello, world');

      app.use(proxy.initialize(config));
      app.use(express.static(handles.filepath));

      handles.server = require('http').createServer(app);
      handles.server.listen(handles.port, function() {
        done(null, handles.port);
      });
    });
  });
}

function
configureExpress
(
config
,
done
)
{
var
portfinder
=
require
(
'portfinder'
)
;
tmp
.
dir
(
function
(
err
,
filepath
)
{
handles
.
filepath
=
filepath
;
portfinder
.
getPort
(
function
(
err
,
port
)
{
if
(
err
)
throw
(
err
)
;
handles
.
port
=
port
;
fs
.
writeFileSync
(
path
.
join
(
handles
.
filepath
,
'index.txt'
)
,
'hello, world'
)
;
app
.
use
(
proxy
.
initialize
(
config
)
)
;
app
.
use
(
express
.
static
(
handles
.
filepath
)
)
;
handles
.
server
=
require
(
'http'
)
.
createServer
(
app
)
;
handles
.
server
.
listen
(
handles
.
port
,
function
(
)
{
done
(
,
handles
.
port
)
;
}
)
;
}
)
;
}
)
;
}

Configures an express instance on a dynamically assigned port
for serving static files and proxying requests based on the config.

@param  {Object.<string, string>} config   the config of the json-proxy for
this test suite
@param  {Function} done callback

Configures an express instance on a dynamically assigned port for serving static files and proxying requests based on the config .


function configureLanProxy(options, config, done) {
  var portfinder = require('portfinder'),
      request = require('request'),
      credentials = config.proxy.gateway.auth,
      gatewayPort,
      expectedAuthorizationHeader,
      requestViaHeader,
      responseViaHeader;

  handles = handles || {};

  handles.gatewayServer = require('http').createServer(function (req, res) {
    expectedAuthorizationHeader = 'Basic ' + new Buffer(credentials).toString('base64');

    // HACK: node 0.12.x appears to inject a slash at the front
    //       of absolute URLs
    //       ex., GET http://www.example.com --> GET /http://www.exampel.com
    if (req.url.charAt(0) === '/') {
      req.url = req.url.substr(1);
    }

    // validate the proxy target
    if (req.url !== req.headers['x-forwarded-url']) {
        res.writeHead(500);
        res.end('{ "error": 500, "message": "invalid proxy request, expected X-Forwarded-Url header ' + req.headers['x-forwarded-url'] + '" }');
        return;
    }

    // validate the proxy credentials
    if (req.headers['authorization'] !== expectedAuthorizationHeader) {
      res.writeHead(401);
      res.end('{ "error": 401, "message": "invalid proxy credentials, expected ' + expectedAuthorizationHeader + '" }');
      return;
    }

    // determine if we are using a proxy that is not RFC compliant
    requestViaHeader = options.headers['Via'] ||
                        '127.0.0.1:' + handles.port;

    responseHostHeader = options.headers['Host'] ||
                        req.headers['host'];

    responseViaHeader = options.headers['Via'] ||
                        'http://localhost:' + gatewayPort;

    // validate the via header was injected and points to 127.0.0.1 in either ipv4 or ipv6 format
    if (req.headers['via'] === undefined || req.headers['via'] === null || req.headers['via'].indexOf(requestViaHeader) === -1) {
      res.writeHead(400);
      res.end('{ "error": 400, "message": "invalid via header, expected ' + requestViaHeader + '" }');
      return;
    }

    // strip the proxy credentials header
    req.headers['authorization'] = null;
    // simulate the behavior of x-forwarded-for with multiple proxies
    req.headers['x-forwarded-for'] = [req.headers['x-forwarded-for'], req.headers['via']].join(', ');
    // change the via header to this server
    req.headers['via'] = responseViaHeader;
    req.headers['host'] = responseHostHeader;

    var errorCallback = function errorCallback(err, repsonse, body) {
      if (err) {
        res.writeHead(500);
        res.end(JSON.stringify({ "error": 500, "message": err.message  }));
        return;
      }
    }

    request(req, errorCallback).pipe(res);
  });

  portfinder.getPort(function (err, port) {
    if (err) done(err);

    config.proxy.gateway.port = port;
    gatewayPort = port;

    handles.gatewayServer.listen(port, function() {
      done(null);
    });
  });
}

Creates a simple LAN proxy using a vanilla HTTP server
that verifies the state of the proxy credentials and the x-forwarded-url
are correct.

@param  {Object.<string, string>} options  the options for configuring this
test suite
@param  {Object.<string, string>} config   the config of the json-proxy for
this test suite
@param  {Function} done

Creates a simple LAN proxy using a vanilla HTTP server that verifies the state of the proxy credentials and the x - forwarded - url are correct .


function cleanup(done) {
  config = null;

  rules.forEach(function(rule){
    rule.done();
  });
  nock.cleanAll();

  handles.server.close();
  if (handles.gatewayServer !== undefined && handles.gatewayServer !== null) {
    handles.gatewayServer.close();
  }

  fs.unlinkSync(path.join(handles.filepath, '/index.txt'));

  handles = null;

  done();
}

function cleanup ( done ) { config = null ; rules . forEach ( function ( rule ) { rule . done ( ) ; } ) ; nock . cleanAll ( ) ; handles . server . close ( ) ; if ( handles . gatewayServer !== undefined && handles . gatewayServer !== null ) { handles . gatewayServer . close ( ) ; } fs . unlinkSync ( path . join ( handles . filepath , '/index.txt' ) ) ; handles = null ; done ( ) ; }

Teardown logic for the reusable test suites
@param  {Function} done the async callback


function setChange(repoState, filepath, change) {
    let workingState = repoState.getCurrentState();
    let changes = workingState.getChanges();
    const type = change.getType();

    // Simplify change when possible
    if (type === CHANGE_TYPE.REMOVE
        && !workingState.getTreeEntries().has(filepath)) {
        // Removing a file that did not exist before
        changes = changes.delete(filepath);

    } else if (type === CHANGE_TYPE.CREATE
               && workingState.getTreeEntries().has(filepath)) {
        // Adding back a file that existed already
        changes = changes.set(filepath, change.set('type', CHANGE_TYPE.UPDATE));

    } else {
        // Push changes to list
        changes = changes.set(filepath, change);
    }

    // Update workingState and repoState
    workingState = workingState.set('changes', changes);
    return RepoUtils.updateCurrentWorkingState(repoState, workingState);
}

Set a new change to the current WorkingState.
Attempt to resolve some cases like removing a file that was added
in the first place.
@param {RepositoryState}
@param {String}
@param {Change}

Set a new change to the current WorkingState . Attempt to resolve some cases like removing a file that was added in the first place .


function revertAll(repoState) {
    let workingState = repoState.getCurrentState();

    // Create empty list of changes
    const changes = new Immutable.OrderedMap();

    // Update workingState and repoState
    workingState = workingState.set('changes', changes);
    return RepoUtils.updateCurrentWorkingState(repoState, workingState);
}

function
revertAll
(
repoState
)
{
let
workingState
=
repoState
.
getCurrentState
(
)
;
// Create empty list of changes
const
changes
=
new
Immutable
.
OrderedMap
(
)
;
// Update workingState and repoState
workingState
=
workingState
.
set
(
'changes'
,
changes
)
;
return
RepoUtils
.
updateCurrentWorkingState
(
repoState
,
workingState
)
;
}

Revert all changes
@param {RepositoryState}
@return {RepositoryState}


function revertForFile(repoState, filePath) {
    let workingState = repoState.getCurrentState();

    // Remove file from changes map
    const changes = workingState.getChanges().delete(filePath);

    // Update workingState and repoState
    workingState = workingState.set('changes', changes);
    return RepoUtils.updateCurrentWorkingState(repoState, workingState);
}

function
revertForFile
(
repoState
,
filePath
)
{
let
workingState
=
repoState
.
getCurrentState
(
)
;
// Remove file from changes map
const
changes
=
workingState
.
getChanges
(
)
.
delete
(
filePath
)
;
// Update workingState and repoState
workingState
=
workingState
.
set
(
'changes'
,
changes
)
;
return
RepoUtils
.
updateCurrentWorkingState
(
repoState
,
workingState
)
;
}

Revert change for a specific file
@param {RepositoryState}
@param {Path}
@return {RepositoryState}


function revertForDir(repoState, dirPath) {
    let workingState = repoState.getCurrentState();
    let changes = workingState.getChanges();

    // Remove all changes that are in the directory
    changes = changes.filter((change, filePath) => {
        return !PathUtils.contains(dirPath, filePath);
    });

    // Update workingState and repoState
    workingState = workingState.set('changes', changes);
    return RepoUtils.updateCurrentWorkingState(repoState, workingState);
}

Revert changes for a specific directory
@param {RepositoryState}
@param {Path}
@return {RepositoryState}


function revertAllRemoved(repoState) {
    let workingState = repoState.getCurrentState();
    const changes = workingState.getChanges().filter(
        // Remove all changes that are in the directory
        (change) => {
            return change.getType() === CHANGE_TYPE.REMOVE;
        }
    );

    // Update workingState and repoState
    workingState = workingState.set('changes', changes);
    return RepoUtils.updateCurrentWorkingState(repoState, workingState);
}

Revert all removed files
@param {RepositoryState}
@return {RepositoryState}


function normPath(p) {
    p = path.normalize(p);
    if (p[0] == '/') p = p.slice(1);
    if (p[p.length - 1] == '/') p = p.slice(0, -1);
    if (p == '.') p = '';
    return p;
}

function normPath ( p ) { p = path . normalize ( p ) ; if ( p [ 0 ] == '/' ) p = p . slice ( 1 ) ; if ( p [ p . length - 1 ] == '/' ) p = p . slice ( 0 , - 1 ) ; if ( p == '.' ) p = '' ; return p ; }


function pathContains(dir, path) {
    dir = dir ? normPath(dir) + '/' : dir;
    path = normPath(path);

    return path.indexOf(dir) === 0;
}

function pathContains ( dir , path ) { dir = dir ? normPath ( dir ) + '/' : dir ; path = normPath ( path ) ; return path . indexOf ( dir ) === 0 ; }


function readFilenamesRecursive(repoState, dirName) {
    dirName = PathUtils.norm(dirName);

    const workingState = repoState.getCurrentState();
    const fileSet = WorkingUtils.getMergedFileSet(workingState);

    return fileSet.filter((path) => {
        return PathUtils.contains(dirName, path);
    }).toArray();
}

function readFilenamesRecursive ( repoState , dirName ) { dirName = PathUtils . norm ( dirName ) ; const workingState = repoState . getCurrentState ( ) ; const fileSet = WorkingUtils . getMergedFileSet ( workingState ) ; return fileSet . filter ( ( path ) => { return PathUtils . contains ( dirName , path ) ; } ) . toArray ( ) ; }

List files recursively in a directory
@param {RepositoryState} repoState
@param {Path} dirName
@return {Array<Path>}


function move(repoState, dirName, newDirName) {
    // List entries to move
    const filesToMove = readFilenamesRecursive(repoState, dirName);

    // Push change to remove all entries
    return filesToMove.reduce((repoState, oldPath) => {
        const newPath = Path.join(
            newDirName,
            Path.relative(dirName, oldPath)
        );

        return FileUtils.move(repoState, oldPath, newPath);
    }, repoState);
}

function
move
(
repoState
,
dirName
,
newDirName
)
{
// List entries to move
const
filesToMove
=
readFilenamesRecursive
(
repoState
,
dirName
)
;
// Push change to remove all entries
return
filesToMove
.
reduce
(
(
repoState
,
oldPath
)
=>
{
const
newPath
=
Path
.
join
(
newDirName
,
Path
.
relative
(
dirName
,
oldPath
)
)
;
return
FileUtils
.
move
(
repoState
,
oldPath
,
newPath
)
;
}
,
repoState
)
;
}


function create(repositoryState, driver, name, opts = {}) {
    const {
        // Base branch for the new branch
        base = repositoryState.getCurrentBranch(),
        // Fetch the working state and switch to it ?
        checkout = true,
        // Drop changes from base branch the new working state ?
        clean = true,
        // Drop changes from the base branch ?
        cleanBase = false
    } = opts;

    let createdBranch;

    return driver.createBranch(base, name)
    // Update list of branches
    .then((branch) => {
        createdBranch = branch;
        let branches = repositoryState.getBranches();
        branches = branches.push(createdBranch);
        return repositoryState.set('branches', branches);
    })

    // Update working state or fetch it if needed
    .then((repoState) => {
        let baseWk = repoState.getWorkingStateForBranch(base);

        if (!baseWk) {
            return checkout ? RepoUtils.fetchTree(repoState, driver, createdBranch) : repoState;
        }

        // Reuse base WorkingState clean
        const headWk = clean ? baseWk.asClean() : baseWk;
        repoState = RepoUtils.updateWorkingState(repoState, createdBranch, headWk);

        // Clean base WorkingState
        baseWk = cleanBase ? baseWk.asClean() : baseWk;
        repoState = RepoUtils.updateWorkingState(repoState, base, baseWk);

        return repoState;
    })

    // Checkout the branch
    .then((repoState) => {
        if (!checkout) {
            return repoState;
        }

        return RepoUtils.checkout(repoState, createdBranch);
    });
}

Create a new branch with the given name.
@param {RepositoryState} repoState
@param {Driver} driver
@param {String} name
@param {Branch} [opts.base] Base branch, default to current branch
@param {Boolean} [opts.checkout=false] Directly fetch and checkout the branch
@return {Promise<RepositoryState>}

Create a new branch with the given name .


function update(repoState, driver, branchName) {
    branchName = Normalize.branchName(branchName || repoState.getCurrentBranch());

    return driver.fetchBranches()
    .then((branches) => {
        const newBranch = branches.find((branch) => {
            return branch.getFullName() === branchName;
        });

        if (!newBranch) {
            return repoState;
        } else {
            return RepoUtils.fetchTree(repoState, driver, newBranch);
        }
    });
}

function update ( repoState , driver , branchName ) { branchName = Normalize . branchName ( branchName || repoState . getCurrentBranch ( ) ) ; return driver . fetchBranches ( ) . then ( ( branches ) => { const newBranch = branches . find ( ( branch ) => { return branch . getFullName ( ) === branchName ; } ) ; if ( ! newBranch ) { return repoState ; } else { return RepoUtils . fetchTree ( repoState , driver , newBranch ) ; } } ) ; }

Fetch the list of branches, and update the given branch only. Will update
the WorkingState of the branch (and discard previous
@param {RepositoryState} repoState
@param {Driver} driver
@param {Branch | String} branchName The branch to update
@return {Promise<RepositoryState>} with the branch updated

Fetch the list of branches and update the given branch only . Will update the WorkingState of the branch ( and discard previous


function remove(repoState, driver, branch) {
    return driver.deleteBranch(branch)
    .then(() => {
        return repoState.updateBranch(branch, null);
    });
}

function remove ( repoState , driver , branch ) { return driver . deleteBranch ( branch ) . then ( ( ) => { return repoState . updateBranch ( branch , null ) ; } ) ; }

Remove the given branch from the repository.
@param {RepositoryState} repoState
@param {Driver} driver
@param {Branch} branch to remove
@return {Promise<RepositoryState>}

Remove the given branch from the repository .


function fetch(repoState, driver, sha) {
    if (isFetched(repoState, sha)) {
        // No op if already fetched
        return Q(repoState);
    }

    const cache = repoState.getCache();
    // Fetch the blob
    return driver.fetchBlob(sha)
    // Then store it in the cache
    .then((blob) => {
        const newCache = CacheUtils.addBlob(cache, sha, blob);
        return repoState.set('cache', newCache);
    });
}

function
fetch
(
repoState
,
driver
,
sha
)
{
if
(
isFetched
(
repoState
,
sha
)
)
{
// No op if already fetched
return
Q
(
repoState
)
;
}
const
cache
=
repoState
.
getCache
(
)
;
// Fetch the blob
return
driver
.
fetchBlob
(
sha
)
// Then store it in the cache
.
then
(
(
blob
)
=>
{
const
newCache
=
CacheUtils
.
addBlob
(
cache
,
sha
,
blob
)
;
return
repoState
.
set
(
'cache'
,
newCache
)
;
}
)
;
}

Fetch a blob from SHA.
@param {RepositoryState} repoState
@param {Driver} driver
@param {SHA} sha
@return {Promise<RepositoryState>}


function (context, options, callback) {
    // add the current request to the queue
    context.retryQueue.push([options, callback]);
    // bail if the token is currently being refreshed
    if (context.refreshActive) {
        return false;
    }
    // ready to refresh
    context.refreshActive = true;
    return request({
        uri: baseUrl + '/oauth2/token',
        method: 'POST',
        headers: {
            'Authorization': 'Basic ' + new Buffer(context.clientId + ':' + context.clientSecret).toString('base64'),
            'User-Agent': userAgent
        },
        form: {
            grant_type: 'client_credentials'
        }
    }, function (err, res, body) {
        context.refreshActive = false;
        // if anything but a 200 is returned from the token refresh call, we return the error to the
        // caller and blow out the retry queue
        if (res.statusCode != 200) {
            context.retryQueue = [];
            return callback && callback(res.body, res);
        }
        // set the access token on the connection
        var token = JSON.parse(body);
        context.accessToken = token.access_token;
        // process the queue of requests for the current connection
        while (0 < context.retryQueue.length) {
            var reqArgs = context.retryQueue.pop();
            context.apiRequest(reqArgs[0], reqArgs[1]);
        }
    });
}

a private function to automatically refresh the access token when receiving a 401. Adds rejected requests to a queue to be processed

a private function to automatically refresh the access token when receiving a 401 . Adds rejected requests to a queue to be processed


function PokitDok(clientId, clientSecret, version) {
    this.clientId = clientId;
    this.clientSecret = clientSecret;
    this.version = version || 'v4';
    this.refreshActive = false;
    this.retryQueue = [];
    this.accessToken = null;
}

function PokitDok ( clientId , clientSecret , version ) { this . clientId = clientId ; this . clientSecret = clientSecret ; this . version = version || 'v4' ; this . refreshActive = false ; this . retryQueue = [ ] ; this . accessToken = null ; }

Create a connection to the pokitdok API. The version defaults to v4. You must enter your client ID and client secret
or all requests made with your connection will return errors.
@name PokitDok
@param {string} clientId - The client id of your PokitDok App
@param {string} clientSecret - The client secret of your PokitDok App
@param {string} version - the version of the API the connection should use
@constructor
@example
```js
// get a connection to the PokitDok Platform for the most recent version
var PokitDok = require('pokitdok-nodejs');
var pokitdok = new PokitDok(process.env.POKITDOK_CLIENT_ID, process.env.POKITDOK_CLIENT_SECRET);
```
@example
```js
// get a connection to the PokitDok Platform for version 3
var PokitDok = require('pokitdok-nodejs');
var pokitdokV3 = new PokitDok(process.env.POKITDOK_CLIENT_ID, process.env.POKITDOK_CLIENT_SECRET, 'v3');
```

Create a connection to the pokitdok API . The version defaults to v4 . You must enter your client ID and client secret or all requests made with your connection will return errors .


function featureArrayToFeatureString(features, bias, firstFeatureNumber) {
	if (!Array.isArray(features))
		throw new Error("Expected an array, but got "+JSON.stringify(features))
	var line = (bias? " "+firstFeatureNumber+":"+bias: "");
	for (var feature=0; feature<features.length; ++feature) {
		var value = features[feature];
		if (value)
			line += (" "+(feature+firstFeatureNumber+(bias?1:0))+":"+value.toPrecision(5));
	}
	return line;
}

function
featureArrayToFeatureString
(
features
,
bias
,
firstFeatureNumber
)
{
if
(
!
Array
.
isArray
(
features
)
)
throw
new
Error
(
"Expected an array, but got "
+
JSON
.
stringify
(
features
)
)
var
line
=
(
bias
" "
+
firstFeatureNumber
+
":"
+
bias
:
""
)
;
for
(
var
feature
=
0
;
feature
<
features
.
length
;
++
feature
)
{
var
value
=
features
[
feature
]
;
if
(
value
)
line
+=
(
" "
+
(
feature
+
firstFeatureNumber
+
(
bias
1
:
0
)
)
+
":"
+
value
.
toPrecision
(
5
)
)
;
}
return
line
;
}

convert an array of features to a single line in SVM-light format. The line starts with a space.

convert an array of features to a single line in SVM - light format . The line starts with a space .


function(feature) {
		if (!(feature in this.featureNameToFeatureIndex)) {
			var newIndex = this.featureIndexToFeatureName.length;
			this.featureIndexToFeatureName.push(feature);
			this.featureNameToFeatureIndex[feature] = newIndex;
		}
	}

function ( feature ) { if ( ! ( feature in this . featureNameToFeatureIndex ) ) { var newIndex = this . featureIndexToFeatureName . length ; this . featureIndexToFeatureName . push ( feature ) ; this . featureNameToFeatureIndex [ feature ] = newIndex ; } }

add a single feature, if it does not exist

add a single feature if it does not exist


function(hash) {
		if (hash instanceof Array) {
			for (var index in hash)
				this.addFeature(hash[index]);
		} else if (hash instanceof Object) {
			for (var feature in hash)
				this.addFeature(feature);
		}
		else throw new Error("FeatureLookupTable.addFeatures expects a hash or an array, but got: "+JSON.stringify(hash));
	}

add all features in the given hash or array


function(hash) {
		this.addFeatures(hash);
		var array = [];
		for (var featureIndex=0; featureIndex<this.featureIndexToFeatureName.length; ++featureIndex)
			array[featureIndex]=0;
		if (hash instanceof Array) {
			for (var i in hash)
				array[this.featureNameToFeatureIndex[hash[i]]] = true;
		} else if (hash instanceof Object) {
			for (var feature in hash)
				array[this.featureNameToFeatureIndex[feature]] = hash[feature];
		}
		else throw new Error("Unsupported type: "+JSON.stringify(hash));
		return array;
	}

function
(
hash
)
{
this
.
addFeatures
(
hash
)
;
var
array
=
[
]
;
for
(
var
featureIndex
=
0
;
featureIndex
<
this
.
featureIndexToFeatureName
.
length
;
++
featureIndex
)
array
[
featureIndex
]
=
0
;
if
(
hash
instanceof
Array
)
{
for
(
var
i
in
hash
)
array
[
this
.
featureNameToFeatureIndex
[
hash
[
i
]
]
]
=
true
;
}
else
if
(
hash
instanceof
Object
)
{
for
(
var
feature
in
hash
)
array
[
this
.
featureNameToFeatureIndex
[
feature
]
]
=
hash
[
feature
]
;
}
else
throw
new
Error
(
"Unsupported type: "
+
JSON
.
stringify
(
hash
)
)
;
return
array
;
}

Convert the given hash of features to a numeric array, using 0 for padding.
If some features in the hash do not exist - they will be added.
@param hash any hash, for example, {a: 111, b: 222, c: 333}
@return a matching array, based on the current feature table. For example: [0, 111, 222, 0, 333]
@note some code borrowed from Heather Arthur: https://github.com/harthur/brain/blob/master/lib/lookup.js

Convert the given hash of features to a numeric array using 0 for padding . If some features in the hash do not exist - they will be added .


function(hashes) {
		this.addFeaturess(hashes);

		var arrays = [];
		for (var i=0; i<hashes.length; ++i) {
			arrays[i] = [];
			for (var feature in this.featureNameToFeatureIndex)
				arrays[i][this.featureNameToFeatureIndex[feature]] = hashes[i][feature] || 0;
		}
		return arrays;
	}

function ( hashes ) { this . addFeaturess ( hashes ) ; var arrays = [ ] ; for ( var i = 0 ; i < hashes . length ; ++ i ) { arrays [ i ] = [ ] ; for ( var feature in this . featureNameToFeatureIndex ) arrays [ i ] [ this . featureNameToFeatureIndex [ feature ] ] = hashes [ i ] [ feature ] || 0 ; } return arrays ; }

Convert all the given hashes of features to numeric arrays, using 0 for padding.
If some features in some of the hashes do not exist - they will be added.
@param hashes an array of hashes, for example, [{a: 111, b: 222}, {a: 11, c: 33}, ...]
@return an array of matching arrays, based on the current feature table. For example: [[111, 222], [11, 0, 33]]

Convert all the given hashes of features to numeric arrays using 0 for padding . If some features in some of the hashes do not exist - they will be added .


function(array) {
		var hash = {};
		for (var feature in this.featureNameToFeatureIndex) {
			if (array[this.featureNameToFeatureIndex[feature]])
				hash[feature] = array[this.featureNameToFeatureIndex[feature]];
		}
		return hash;
	}

function ( array ) { var hash = { } ; for ( var feature in this . featureNameToFeatureIndex ) { if ( array [ this . featureNameToFeatureIndex [ feature ] ] ) hash [ feature ] = array [ this . featureNameToFeatureIndex [ feature ] ] ; } return hash ; }

Convert the given numeric array to a hash of features, ignoring zero values.
@note some code borrowed from Heather Arthur: https://github.com/harthur/brain/blob/master/lib/lookup.js

Convert the given numeric array to a hash of features ignoring zero values .


function(arrays) {
		var hashes = [];
		for (var i=0; i<arrays.length; ++i)
			hashes[i] = this.arrayToHash(arrays[i]);
		return hashes;
	}

function ( arrays ) { var hashes = [ ] ; for ( var i = 0 ; i < arrays . length ; ++ i ) hashes [ i ] = this . arrayToHash ( arrays [ i ] ) ; return hashes ; }

Convert the given numeric arrays to array of hashes of features, ignoring zero values.

Convert the given numeric arrays to array of hashes of features ignoring zero values .


function(sample, splitLabels, treeNode) {
		var superlabels = {}; // the first parts of each of the splitLabels
		var mapSuperlabelToRest = {};   // each value is a list of continuations of the key.
		for (var i in splitLabels) {
			var splitLabel = splitLabels[i];
			var superlabel = splitLabel[0];
			superlabels[superlabel] = true;
			if (splitLabel.length>1) {
				if (!mapSuperlabelToRest[superlabel])
					mapSuperlabelToRest[superlabel] = [];
				mapSuperlabelToRest[superlabel].push(splitLabel.slice(1));
			}
		}

		treeNode.superlabelClassifier.trainOnline(sample, Object.keys(superlabels));
		for (var superlabel in mapSuperlabelToRest) {
			if (!(superlabel in treeNode.mapSuperlabelToBranch)) {
				treeNode.mapSuperlabelToBranch[superlabel] = {
					superlabelClassifier: this.newMultilabelClassifier(),
					mapSuperlabelToBranch: {}
				}
			}
			this.trainOnlineRecursive(sample, mapSuperlabelToRest[superlabel], treeNode.mapSuperlabelToBranch[superlabel]);
		}
	}

Recursive internal subroutine of trainOnline.
@param splitLabels an array of arrays: each internal array represents the parts of a single label.

Recursive internal subroutine of trainOnline .


function(dataset, treeNode) {
		var superlabelsDataset = [];
		var mapSuperlabelToRestDataset = {};
		dataset.forEach(function(datum) {
			var splitLabels = datum.output;	// [ [ 'Offer', 'Leased Car', 'Without leased car' ], [ 'Offer', 'Working Hours', '9 hours' ] ]
			var superlabels = {};           // the first parts of each of the splitLabels
			var mapSuperlabelToRest = {};   // each value is a list of continuations of the key.
			for (var i in splitLabels) {
				var splitLabel = splitLabels[i];//[ 'Offer', 'Leased Car', 'Without leased car' ]
				var superlabel = splitLabel[0];
				superlabels[superlabel] = true; //superlabels['Offer'] = true
				if (splitLabel.length>1) { 		// if it have more than one label (superlabel)
					if (!mapSuperlabelToRest[superlabel])
						mapSuperlabelToRest[superlabel] = [];
					mapSuperlabelToRest[superlabel].push(splitLabel.slice(1));//['Leased Car', 'Without leased car']
				}
			}

/*			Sample of mapSuperlabelToRest
			{ Offer:
			[ [ 'Leased Car', 'Without leased car' ],
   			  [ 'Working Hours', '9 hours' ] ] }

			Sample of superlabelsDataset, initial dataset with superlabel instead of entire output
			'. [end]': 0.965080896043587 },
			output: [ 'Offer' ] } ]
*/
			superlabelsDataset.push({
				input: datum.input,
				output: Object.keys(superlabels)
			});

			for (var superlabel in mapSuperlabelToRest) {
				if (!(superlabel in mapSuperlabelToRestDataset))
					mapSuperlabelToRestDataset[superlabel] = [];
				mapSuperlabelToRestDataset[superlabel].push({
					input: datum.input,
					output: mapSuperlabelToRest[superlabel]
				});
			}
		}, this);

/*		Sample of mapSuperlabelToRestDataset
		{ Offer: [ { input: [Object], output: [["Leased Car","Without leased car"],["Working Hours","9 hours"]] } ] }
*/

		// train the classifier only on superlabels
		treeNode.superlabelClassifier.trainBatch(superlabelsDataset);

		for (var superlabel in mapSuperlabelToRestDataset) {
			if (!(superlabel in treeNode.mapSuperlabelToBranch)) {
				treeNode.mapSuperlabelToBranch[superlabel] = {
					superlabelClassifier: this.newMultilabelClassifier(),
					mapSuperlabelToBranch: {}
				}
			}
/*			train the next level classifier for a give superlabel classifier superlabel (from loop)
			with the dataset from new structure mapSuperlabelToRestDataset (see above)
*/			this.trainBatchRecursive(mapSuperlabelToRestDataset[superlabel], treeNode.mapSuperlabelToBranch[superlabel]);
		}
	}

Recursive internal subroutine of trainBatch.

Recursive internal subroutine of trainBatch .


function(sample, explain, treeNode, depth) {
		if (!depth) depth = 1;
		// classify the superlabel
		var superlabelsWithExplain = treeNode.superlabelClassifier.classify(sample, explain);
		var superlabels = (explain>0? superlabelsWithExplain.classes: superlabelsWithExplain);

		var splitLabels = [];
		if (explain>0) {
			var explanations = ["depth="+depth+": "+superlabels, superlabelsWithExplain.explanation];
		}

		// for all superlabels that were classified, may be there are more than one that were classified with it
		for (var i in superlabels) {
			var superlabel = superlabels[i];
			var splitLabel = [superlabel];

			// classifier of [Offer] types / second level / classifies Offer's parameters
			var branch = treeNode.mapSuperlabelToBranch[superlabel];

			if (branch) {

				// [ [ 'Without leased car' ] ]
				var branchLabelsWithExplain = this.classifyRecursive(sample, explain, branch, depth+1);

				var branchLabels = (explain>0? branchLabelsWithExplain.classes: branchLabelsWithExplain);

				for (var j in branchLabels)
					splitLabels.push(splitLabel.concat(branchLabels[j]));
				if (explain>0)
					explanations = explanations.concat(branchLabelsWithExplain.explanation);
			} else {
				splitLabels.push(splitLabel);
			}
		}
		return (explain>0?
				{classes: splitLabels, explanation: explanations}:
				splitLabels);
	}

Recursive internal subroutine of classify.
@return an array of arrays, where each internal array represents a split label.

Recursive internal subroutine of classify .


function(opts) {
	if (!('binaryClassifierType' in opts)) {
		console.dir(opts);
		throw new Error("opts must contain binaryClassifierType");
	}
	if (!opts.binaryClassifierType) {
		console.dir(opts);
		throw new Error("opts.binaryClassifierType is null");
	}

	this.binaryClassifierType = opts.binaryClassifierType;
	this.classifier = new this.binaryClassifierType();

	switch (opts.segmentSplitStrategy) {
	case 'shortestSegment': this.segmentSplitStrategy = this.shortestSegmentSplitStrategy; break;
	case 'longestSegment':  this.segmentSplitStrategy = this.longestSegmentSplitStrategy;  break;
	case 'cheapestSegment':  this.segmentSplitStrategy = this.cheapestSegmentSplitStrategy;  break;
	default: this.segmentSplitStrategy = null;
	}

	this.mapClassnameToClassifier = {};
}

BinarySegmentation - Multi-label text classifier, based on a segmentation scheme using base binary classifiers.

Inspired by:

Morbini Fabrizio, Sagae Kenji. Joint Identification and Segmentation of Domain-Specific Dialogue Acts for Conversational Dialogue Systems. ACL-HLT 2011
http://www.citeulike.org/user/erelsegal-halevi/article/10259046

@author Erel Segal-haLevi

@param opts
binaryClassifierType (mandatory) - the type of the base binary classifier.
featureExtractor (optional) - a single feature-extractor (see the "features" folder), or an array of extractors, for extracting features from the text segments during classification.

BinarySegmentation - Multi - label text classifier based on a segmentation scheme using base binary classifiers .


function(opts) {
	this.retrain_count = opts.retrain_count || 10;
	this.Constant = opts.Constant || 5.0;
	this.weights = {
		//DUMMY_CLASS:{}
	};
	this.weights_sum = {
		//DUMMY_CLASS:{}
	};
	this.seenFeatures = {};
	this.num_iterations = 0
}

function ( opts ) { this . retrain_count = opts . retrain_count || 10 ; this . Constant = opts . Constant || 5.0 ; this . weights = { //DUMMY_CLASS:{} } ; this . weights_sum = { //DUMMY_CLASS:{} } ; this . seenFeatures = { } ; this . num_iterations = 0 }

Multilabel online classifier based on Perceptron and Passive-Aggressive.

features and categories are strings; samples are hashes.

Based on Python code Copyright (c) 2013 Joseph Keshet.

Conversion to Node.js started by Erel Segal-haLevi, but not finished yet.

@since 2013-08

@param opts
Constant (optional) - tradeoff constant (default=5.0)
retrain_count (optional) - number of epoches to run in trainBatch mode (default=10)

Multilabel online classifier based on Perceptron and Passive - Aggressive .


function(classes) {
		classes = hash.normalized(classes);
		for (var aClass in classes) {
			if (!(aClass in this.weights)) {
				this.weights[aClass]={};
				this.weights_sum[aClass]={};
			}
		}
	}

function ( classes ) { classes = hash . normalized ( classes ) ; for ( var aClass in classes ) { if ( ! ( aClass in this . weights ) ) { this . weights [ aClass ] = { } ; this . weights_sum [ aClass ] = { } ; } } }

Tell the classifier that the given classes will be used for the following
samples, so that it will know to add negative samples to classes that do
not appear.

@param classes an object whose KEYS are classes, or an array whose VALUES are classes.

Tell the classifier that the given classes will be used for the following samples so that it will know to add negative samples to classes that do not appear .


function(opts) {

	opts = opts || {};
	if (!opts.multilabelClassifierType) {
		console.dir(opts);
		throw new Error("opts.multilabelClassifierType is null");
	}

	if (!opts.numberofclassifiers) {
		console.dir(opts);
		throw new Error("opts.numberofclassifiers is null");
	}

	// this.splitLabel = opts.splitLabel || function(label)      {return label.split(/@/);}
	this.classifier = this.intializeClassifiers(opts.numberofclassifiers, opts.multilabelClassifierType)
}

PartialClassification is a test classifier that learns and classifies the components
of the labels separately according to the splitLabel routine. One of the examples could be
classifying intent, attribute, value separately by three different classifiers.
When performing test by trainAndTest module, there is a check for toFormat routine, if it exists
then pretest format converting occurs.

@author Vasily Konovalov
@since March 2014

PartialClassification is a test classifier that learns and classifies the components of the labels separately according to the splitLabel routine . One of the examples could be classifying intent attribute value separately by three different classifiers . When performing test by trainAndTest module there is a check for toFormat routine if it exists then pretest format converting occurs .


function(expected, actual) {
		this.count++;
		if (expected && actual) this.TP++;
		if (!expected && actual) this.FP++;
		if (expected && !actual) this.FN++;
		if (!expected && !actual) this.TN++;
		if (expected==actual) this.TRUE++;
	}

function ( expected , actual ) { this . count ++ ; if ( expected && actual ) this . TP ++ ; if ( ! expected && actual ) this . FP ++ ; if ( expected && ! actual ) this . FN ++ ; if ( ! expected && ! actual ) this . TN ++ ; if ( expected == actual ) this . TRUE ++ ; }

Record the result of a new binary experiment.

@param expected - the expected result (true/false).
@param actual   - the actual   result (true/false).

Record the result of a new binary experiment .


function (expectedClasses, actualClasses ) {
		var explanations = [];

		actualClasses = hash.normalized(actualClasses);
		expectedClasses = hash.normalized(expectedClasses);

		var allTrue = true;

		if (!(Object.keys(expectedClasses)[0] in this.confusion))
			this.confusion[Object.keys(expectedClasses)[0]] = {}

		if (!(Object.keys(actualClasses)[0] in this.confusion[Object.keys(expectedClasses)[0]]))
			this.confusion[Object.keys(expectedClasses)[0]][Object.keys(actualClasses)[0]] = 0

		this.confusion[Object.keys(expectedClasses)[0]][Object.keys(actualClasses)[0]] += 1

		for (var actualClass in actualClasses) {

			if (!(actualClass in this.confusion))
				this.confusion[actualClass]={}

			if (!(actualClass in this.labels)) {
				this.labels[actualClass]={}
				this.labels[actualClass]['TP']=0
				this.labels[actualClass]['FP']=0
				this.labels[actualClass]['FN']=0
				}

			if (actualClass in expectedClasses) {
				this.labels[actualClass]['TP'] += 1

			} else {
				this.labels[actualClass]['FP'] += 1
			}
		}
		for (var expectedClass in expectedClasses) {

			if (!(expectedClass in this.labels)) {
				this.labels[expectedClass]={}
				this.labels[expectedClass]['TP']=0
				this.labels[expectedClass]['FP']=0
				this.labels[expectedClass]['FN']=0
				}

			if (!(expectedClass in actualClasses)) {
				this.labels[expectedClass]['FN'] += 1
			}
		}
	}

function ( expectedClasses , actualClasses ) { var explanations = [ ] ; actualClasses = hash . normalized ( actualClasses ) ; expectedClasses = hash . normalized ( expectedClasses ) ; var allTrue = true ; if ( ! ( Object . keys ( expectedClasses ) [ 0 ] in this . confusion ) ) this . confusion [ Object . keys ( expectedClasses ) [ 0 ] ] = { } if ( ! ( Object . keys ( actualClasses ) [ 0 ] in this . confusion [ Object . keys ( expectedClasses ) [ 0 ] ] ) ) this . confusion [ Object . keys ( expectedClasses ) [ 0 ] ] [ Object . keys ( actualClasses ) [ 0 ] ] = 0 this . confusion [ Object . keys ( expectedClasses ) [ 0 ] ] [ Object . keys ( actualClasses ) [ 0 ] ] += 1 for ( var actualClass in actualClasses ) { if ( ! ( actualClass in this . confusion ) ) this . confusion [ actualClass ] = { } if ( ! ( actualClass in this . labels ) ) { this . labels [ actualClass ] = { } this . labels [ actualClass ] [ 'TP' ] = 0 this . labels [ actualClass ] [ 'FP' ] = 0 this . labels [ actualClass ] [ 'FN' ] = 0 } if ( actualClass in expectedClasses ) { this . labels [ actualClass ] [ 'TP' ] += 1 } else { this . labels [ actualClass ] [ 'FP' ] += 1 } } for ( var expectedClass in expectedClasses ) { if ( ! ( expectedClass in this . labels ) ) { this . labels [ expectedClass ] = { } this . labels [ expectedClass ] [ 'TP' ] = 0 this . labels [ expectedClass ] [ 'FP' ] = 0 this . labels [ expectedClass ] [ 'FN' ] = 0 } if ( ! ( expectedClass in actualClasses ) ) { this . labels [ expectedClass ] [ 'FN' ] += 1 } } }

Record the result of a new classes experiment per labels.

@param expectedClasses - the expected set of classes (as an array or a hash).
@param actualClasses   - the actual   set of classes (as an array or a hash).
@return an array of explanations "FALSE POSITIVE", "FALSE NEGATIVE", and maybe also "TRUE POSITIVE"

Record the result of a new classes experiment per labels .


function (expectedClasses, actualClasses, logTruePositives) {
		var explanations = [];
		actualClasses = hash.normalized(actualClasses);
		expectedClasses = hash.normalized(expectedClasses);

		var allTrue = true;
		for (var actualClass in actualClasses) {
			if (actualClass in expectedClasses) {
				if (logTruePositives) explanations.push("\t\t+++ TRUE POSITIVE: "+actualClass);
				this.TP++;
			} else {
				explanations.push("\t\t--- FALSE POSITIVE: "+actualClass);
				this.FP++;
				allTrue = false;
			}
		}
		for (var expectedClass in expectedClasses) {
			if (!(expectedClass in actualClasses)) {
				explanations.push("\t\t--- FALSE NEGATIVE: "+expectedClass);
				this.FN++;
				allTrue = false;
			}
		}
		if (allTrue) {
			if (logTruePositives) explanations.push("\t\t*** ALL TRUE!");
			this.TRUE++;
		}
		this.count++;
		return explanations;
	}

function
(
expectedClasses
,
actualClasses
,
logTruePositives
)
{
var
explanations
=
[
]
;
actualClasses
=
hash
.
normalized
(
actualClasses
)
;
expectedClasses
=
hash
.
normalized
(
expectedClasses
)
;
var
allTrue
=
true
;
for
(
var
actualClass
in
actualClasses
)
{
if
(
actualClass
in
expectedClasses
)
{
if
(
logTruePositives
)
explanations
.
push
(
"\t\t+++ TRUE POSITIVE: "
+
actualClass
)
;
this
.
TP
++
;
}
else
{
explanations
.
push
(
"\t\t--- FALSE POSITIVE: "
+
actualClass
)
;
this
.
FP
++
;
allTrue
=
false
;
}
}
for
(
var
expectedClass
in
expectedClasses
)
{
if
(
!
(
expectedClass
in
actualClasses
)
)
{
explanations
.
push
(
"\t\t--- FALSE NEGATIVE: "
+
expectedClass
)
;
this
.
FN
++
;
allTrue
=
false
;
}
}
if
(
allTrue
)
{
if
(
logTruePositives
)
explanations
.
push
(
"\t\t*** ALL TRUE!"
)
;
this
.
TRUE
++
;
}
this
.
count
++
;
return
explanations
;
}

Record the result of a new classes experiment.

@param expectedClasses - the expected set of classes (as an array or a hash).
@param actualClasses   - the actual   set of classes (as an array or a hash).
@param logTruePositives- if true, log the true positives.
@return an array of explanations "FALSE POSITIVE", "FALSE NEGATIVE", and maybe also "TRUE POSITIVE"

Record the result of a new classes experiment .


function (expectedClasses, actualClasses, logTruePositives ) {
		var explanations = {};
		explanations['TP'] = []; explanations['FP'] = []; explanations['FN'] = [];

		actualClasses = hash.normalized(actualClasses);
		expectedClasses = hash.normalized(expectedClasses);

		var allTrue = true;
		for (var actualClass in actualClasses) {
			if (actualClass in expectedClasses) {
				if (logTruePositives) explanations['TP'].push(actualClass);
				this.TP++;
			} else {
				explanations['FP'].push(actualClass);
				this.FP++;
				allTrue = false;
			}
		}
		for (var expectedClass in expectedClasses) {
			if (!(expectedClass in actualClasses)) {
				explanations['FN'].push(expectedClass);
				this.FN++;
				allTrue = false;
			}
		}
		if (allTrue) {
			// if ((logTruePositives)&& (!only_false_cases)) explanations.push("\t\t*** ALL TRUE!");
			this.TRUE++;
		}
		this.count++;

		_.each(explanations, function(value, key, list){
			// explanations[key] = _.sortBy(explanations[key], function(num){ num });
			explanations[key].sort()
		}, this)

		if (explanations['FP'].length == 0)
			delete explanations['FP']

		if (explanations['FN'].length == 0)
			delete explanations['FN']

		return explanations;
	}

Record the result of a new classes experiment in a hash manner.
Doesn't allowed to do a inner output, all stats are put in hash
@param expectedClasses - the expected set of classes (as an array or a hash).
@param actualClasses   - the actual   set of classes (as an array or a hash).
@param logTruePositives- if true, log the true positives.
@return an array of explanations "FALSE POSITIVE", "FALSE NEGATIVE", and maybe also "TRUE POSITIVE"
@author Vasily Konovalov
 micro - average

Record the result of a new classes experiment in a hash manner . Doesn t allowed to do a inner output all stats are put in hash


function(dataset) {
			if (this.debug) console.log("trainBatch start");

			var timestamp = new Date().getTime()+"_"+process.pid
			var learnFile = svmcommon.writeDatasetToFile(dataset, this.bias, /*binarize=*/true, this.model_file_prefix+"_"+timestamp, "SvmPerf", FIRST_FEATURE_NUMBER);
			var modelFile = learnFile.replace(/[.]learn/,".model");
			var command = "svm_perf_learn "+this.learn_args+" "+learnFile + " "+modelFile;
			if (this.debug) console.log("running "+command);
			console.log(command)

			var result = execSync(command);
			if (result.code>0) {
				console.dir(result);
				console.log(fs.readFileSync(learnFile, 'utf-8'));
				throw new Error("Failed to execute: "+command);
			}

			this.setModel(fs.readFileSync(modelFile, "utf-8"));
			if (this.debug) console.log("trainBatch end");
		}

function
(
dataset
)
{
if
(
this
.
debug
)
console
.
log
(
"trainBatch start"
)
;
var
timestamp
=
new
Date
(
)
.
getTime
(
)
+
"_"
+
process
.
pid
var
learnFile
=
svmcommon
.
writeDatasetToFile
(
dataset
,
this
.
bias
,
/*binarize=*/
true
,
this
.
model_file_prefix
+
"_"
+
timestamp
,
"SvmPerf"
,
FIRST_FEATURE_NUMBER
)
;
var
modelFile
=
learnFile
.
replace
(
/
[.]learn
/
,
".model"
)
;
var
command
=
"svm_perf_learn "
+
this
.
learn_args
+
" "
+
learnFile
+
" "
+
modelFile
;
if
(
this
.
debug
)
console
.
log
(
"running "
+
command
)
;
console
.
log
(
command
)
var
result
=
execSync
(
command
)
;
if
(
result
.
code
>
0
)
{
console
.
dir
(
result
)
;
console
.
log
(
fs
.
readFileSync
(
learnFile
,
'utf-8'
)
)
;
throw
new
Error
(
"Failed to execute: "
+
command
)
;
}
this
.
setModel
(
fs
.
readFileSync
(
modelFile
,
"utf-8"
)
)
;
if
(
this
.
debug
)
console
.
log
(
"trainBatch end"
)
;
}

Send the given dataset to svm_perf_learn.

@param dataset an array of samples of the form {input: [value1, value2, ...] , output: 0/1}

Send the given dataset to svm_perf_learn .


function modelStringToModelMap(modelString) {
	var matches = SVM_PERF_MODEL_PATTERN.exec(modelString);
	if (!matches) {
		console.log(modelString);
		throw new Error("Model does not match SVM-perf format");
	};
	//var threshold = parseFloat(matches[1]);  // not needed - we use our own bias
	var featuresAndWeights = matches[2].split(" ");
	var mapFeatureToWeight = {};
	//mapFeatureToWeight.threshold = threshold; // not needed - we use our own bias

	//String alphaTimesY = featuresAndWeights[0]; // always 1 in svmperf
	for (var i=1; i<featuresAndWeights.length; ++i) {
		var featureAndWeight = featuresAndWeights[i];
		var featureWeight = featureAndWeight.split(":");
		if (featureWeight.length!=2)
			throw new Error("Model featureAndWeight doesn't match svm-perf pattern: featureAndWeight="+featureAndWeight);
		var feature = parseInt(featureWeight[0]);
		if (feature<=0)
			throw new IllegalArgumentException("Non-positive feature id: featureAndWeight="+featureAndWeight);
		var weight = parseFloat(featureWeight[1]);
		if (Math.abs(weight)>=MIN_WEIGHT)
			mapFeatureToWeight[feature-FIRST_FEATURE_NUMBER]=weight;   // start feature values from 0.
			// Note: if there is bias, then mapFeatureToWeight[0] is its weight.
	}
	return mapFeatureToWeight;
}

weights smaller than this are ignored, to save space
A utility that converts a model in the SVMPerf format to a map of feature weights.
@param modelString a string.
@returns a map.

weights smaller than this are ignored to save space A utility that converts a model in the SVMPerf format to a map of feature weights .


function(dataset, relationName, featureLookupTable) {
	var arff = "% Automatically generated by Node.js\n";
	arff += "@relation "+relationName+"\n";

	featureLookupTable.featureIndexToFeatureName.forEach(function(featureName) {
		if (_.isUndefined(featureName))
			arff += "@attribute undefined {0,1}"+"\n";
		else if (!_.isString(featureName))
			throw new Error("Expected featureName to be a string, but found "+JSON.stringify(featureName));
		else arff += "@attribute "+featureName.replace(/[^a-zA-Z0-9]/g, "_")+" "+"{0,1}"+"\n";
	});

	arff += "\n@data\n";

	dataset.forEach(function(datum) {
		var datumArff = _.clone(datum.input, {});
		for (var i=0; i<datum.output.length; ++i)
			datumArff[datum.output[i]]=1;
		//console.dir(datumArff);
		var array = featureLookupTable.hashToArray(datumArff);
		arff += array + "\n";
	});

	return arff;
}

convert a single dataset to Weka ARFF string.
@param dataset an array of samples in the format {input: {feature1: xxx, feature2: yyy, ...}, output: [1,2,3]}
@param relationName string for the @relation on the top of the file.
@param featureLookupTable maps features to indices
@return an ARFF string.

convert a single dataset to Weka ARFF string .


function SvmLinear(opts) {
	this.learn_args = opts.learn_args || "";
	this.model_file_prefix = opts.model_file_prefix || null;
	this.bias = opts.bias || 1.0;
	this.multiclass = opts.multiclass || false;
	this.debug = opts.debug||false;
	this.train_command = opts.train_command || 'liblinear_train';
	this.test_command = opts.test_command || 'liblinear_test';
	this.timestamp = ""

	if (!SvmLinear.isInstalled()) {
                var msg = "Cannot find the executable 'liblinear_train'. Please download it from the LibLinear website, and put a link to it in your path.";
                console.error(msg)
                throw new Error(msg);
  }
}

in lib linear, feature numbers start with 1

in lib linear feature numbers start with 1


function(dataset) {
			this.timestamp = new Date().getTime()+"_"+process.pid

			// check for multilabel
			_.each(dataset, function(datum, key, list){
				if (_.isArray(datum.output))
					if (datum.output.length > 1)
					{
						console.log("Multi-label is not allowed")
						console.log(JSON.stringify(darum.output, null, 4))
						process.exit(0)
					}
            }, this)

            //  convert all arraay-like outputs to just values
			dataset = _.map(dataset, function(datum){
				if (_.isArray(datum.output))
					datum.output = datum.output[0]
				return datum });

			this.allLabels = _(dataset).map(function(datum){return datum.output});
			this.allLabels = _.uniq(_.flatten(this.allLabels))

			// dataset = _.map(dataset, function(datum){
			// 	datum.output = this.allLabels.indexOf(datum.output)
			// 	return datum });

			if (this.allLabels.length==1) // a single label
				return;
			//console.log(util.inspect(dataset,{depth:1}));
			if (this.debug) console.log("trainBatch start");
			var learnFile = svmcommon.writeDatasetToFile(
					dataset, this.bias, /*binarize=*/false, this.model_file_prefix+"_"+this.timestamp, "SvmLinear", FIRST_FEATURE_NUMBER);
			var modelFile = learnFile.replace(/[.]learn/,".model");

			var command = this.train_command+" "+this.learn_args+" "+learnFile + " "+modelFile;
			console.log("running "+command);

			var result = child_process.execSync(command);
			if (result.code>0) {
				console.dir(result);
				console.log(fs.readFileSync(learnFile, 'utf-8'));
				throw new Error("Failed to execute: "+command);
			}

			this.modelFileString = modelFile;

			if (this.debug) console.log("trainBatch end");
		}

Send the given dataset to liblinear_train.

@param dataset an array of samples of the form {input: [value1, value2, ...] , output: 0/1}

Send the given dataset to liblinear_train .


function modelStringToModelMap(modelString) {
	var matches = LIB_LINEAR_MODEL_PATTERN.exec(modelString);
	if (!matches) {
		console.log(modelString);
		throw new Error("Model does not match SVM-Linear format");
	};
	var labels = matches[1].split(/\s+/);
	var mapLabelToMapFeatureToWeight = {};
	for (var iLabel in labels) {
		var label = labels[iLabel];
		mapLabelToMapFeatureToWeight[label]={};
	}

	var weightsMatrix = matches[3];
	// each line represents a feature; each column represents a label:

	var weightsLines = weightsMatrix.split(NEWLINE);
	for (var feature in weightsLines) {
		var weights = weightsLines[feature].split(/\s+/);
		weights.pop(); // ignore lal]st weight, which is empty (-space)
		if (weights.length==0)
			continue; // ignore empty lines
//		if (isNaN(parseFloat(weights[weights.length-1])))
//			weights.pop();
		if (weights.length==1 && labels.length==2)
			weights[1] = -weights[0];
		if (weights.length!=labels.length)
			throw new Error("Model does not match SVM-Linear format: there are "+labels.length+" labels ("+labels+") and "+weights.length+" weights ("+weights+")");
		for (var iLabel in labels) {
			var label = labels[iLabel];
			var weight = parseFloat(weights[iLabel]);
			if (Math.abs(weight)>=MIN_WEIGHT)
				mapLabelToMapFeatureToWeight[label][feature]=weight;
		}
	}

	return mapLabelToMapFeatureToWeight;
}

weights smaller than this are ignored, to save space
A utility that converts a model in the SvmLinear format to a matrix of feature weights per label.
@param modelString a string.
@returns mapLabelToMapFeatureToWeight.

weights smaller than this are ignored to save space A utility that converts a model in the SvmLinear format to a matrix of feature weights per label .


function(sample, labels) {
		labels = multilabelutils.normalizeOutputLabels(labels);
		for (var l in labels) {
			var positiveLabel = labels[l];
			this.makeSureClassifierExists(positiveLabel);
			this.mapClassnameToClassifier[positiveLabel].trainOnline(sample, 1);
		}
		for (var negativeLabel in this.mapClassnameToClassifier) {
			if (labels.indexOf(negativeLabel)<0)
				this.mapClassnameToClassifier[negativeLabel].trainOnline(sample, 0);
		}
	}

function ( sample , labels ) { labels = multilabelutils . normalizeOutputLabels ( labels ) ; for ( var l in labels ) { var positiveLabel = labels [ l ] ; this . makeSureClassifierExists ( positiveLabel ) ; this . mapClassnameToClassifier [ positiveLabel ] . trainOnline ( sample , 1 ) ; } for ( var negativeLabel in this . mapClassnameToClassifier ) { if ( labels . indexOf ( negativeLabel ) < 0 ) this . mapClassnameToClassifier [ negativeLabel ] . trainOnline ( sample , 0 ) ; } }

Tell the classifier that the given sample belongs to the given labels.

@param sample
a document.
@param labels
an object whose KEYS are labels, or an array whose VALUES are labels.

Tell the classifier that the given sample belongs to the given labels .


function(opts) {
	if (!opts.multiclassClassifierType) {
		console.dir(opts);
		throw new Error("opts.multiclassClassifierType not found");
	}
	this.multiclassClassifierType = opts.multiclassClassifierType;
	this.featureExtractor = FeaturesUnit.normalize(opts.featureExtractor);

	this.multiclassClassifier = new this.multiclassClassifierType();
}

MulticlassSegmentation - Multi-label text classifier, based on a segmentation scheme using a base multi-class classifier.

Inspired by:

Morbini Fabrizio, Sagae Kenji. Joint Identification and Segmentation of Domain-Specific Dialogue Acts for Conversational Dialogue Systems. ACL-HLT 2011
http://www.citeulike.org/user/erelsegal-halevi/article/10259046

@author Erel Segal-haLevi

@param opts
binaryClassifierType (mandatory) - the type of the base binary classifier.
featureExtractor (optional) - a single feature-extractor (see the "features" folder), or an array of extractors, for extracting features from the text segments during classification.

MulticlassSegmentation - Multi - label text classifier based on a segmentation scheme using a base multi - class classifier .


function concatOptionDataArrays(options, data, prop) {
    if (!_.has(options, prop) && !_.has(data, prop)) {
        return;
    }

    var combined = [];
    if (_.isArray(options[prop])) {
        combined = combined.concat(options[prop]);
    }
    if (_.isArray(data[prop])) {
        combined = combined.concat(data[prop]);
    }
    options[prop] = combined;
}

function concatOptionDataArrays ( options , data , prop ) { if ( ! _ . has ( options , prop ) && ! _ . has ( data , prop ) ) { return ; } var combined = [ ] ; if ( _ . isArray ( options [ prop ] ) ) { combined = combined . concat ( options [ prop ] ) ; } if ( _ . isArray ( data [ prop ] ) ) { combined = combined . concat ( data [ prop ] ) ; } options [ prop ] = combined ; }

Sets the `prop` property on options to the concatenation of that property from both options and data, if both exist.  Otherwise, if either exist exclusively, that array will be set to the options.  When neither exist, nothing is done to the options object.

Sets the prop property on options to the concatenation of that property from both options and data if both exist . Otherwise if either exist exclusively that array will be set to the options . When neither exist nothing is done to the options object .


function preorder(node, nodeIndex, parent) {
    var children
    var length
    var index
    var position
    var child

    if (is(test, node, nodeIndex, parent)) {
      return null
    }

    children = node.children

    if (!children || children.length === 0) {
      return node
    }

    // Move all living children to the beginning of the children array.
    position = 0
    length = children.length
    index = -1

    while (++index < length) {
      child = preorder(children[index], index, node)

      if (child) {
        children[position++] = child
      }
    }

    // Cascade delete.
    if (cascade && position === 0) {
      return null
    }

    // Drop other nodes.
    children.length = position

    return node
  }

Check and remove nodes recursively in preorder. For each composite node, modify its children array in-place.

Check and remove nodes recursively in preorder . For each composite node modify its children array in - place .


function filterRelations(relation) {
    var mappedData = includedData.find(function (inc) {
      return inc.id === relation.id;
    });
    var RelationModel = getModel(relation.type);
    var modeledData = new RelationModel(mappedData);

    return checkForRelations(modeledData, modeledData.data);
  }

function filterRelations ( relation ) { var mappedData = includedData . find ( function ( inc ) { return inc . id === relation . id ; } ) ; var RelationModel = getModel ( relation . type ) ; var modeledData = new RelationModel ( mappedData ) ; return checkForRelations ( modeledData , modeledData . data ) ; }

Filters models so they are formatted correctly.

Filters models so they are formatted correctly .


function (bundleName, filter) {
        var bundle,
            files = [];
        bundle = this._bundles[bundleName];
        if (!bundle) {
            throw new Error('Unknown bundle "' + bundleName + '"');
        }
        Object.keys(bundle.files).forEach(function (fullpath) {
            var res = {
                    ext: libpath.extname(fullpath).substr(1)
                };
            if (this._filterResource(res, filter)) {
                files.push(fullpath);
            }
        }, this);
        return files;
    }

function
(
bundleName
,
filter
)
{
var
bundle
,
files
=
[
]
;
bundle
=
this
.
_bundles
[
bundleName
]
;
if
(
!
bundle
)
{
throw
new
Error
(
'Unknown bundle "'
+
bundleName
+
'"'
)
;
}
Object
.
keys
(
bundle
.
files
)
.
forEach
(
function
(
fullpath
)
{
var
res
=
{
ext
:
libpath
.
extname
(
fullpath
)
.
substr
(
1
)
}
;
if
(
this
.
_filterResource
(
res
,
filter
)
)
{
files
.
push
(
fullpath
)
;
}
}
,
this
)
;
return
files
;
}

Utility method for listing all files in a bundle.
@method getBundleFiles
@param {string} bundleName The name of the bundle.
@param {object} filter Filter for deciding which files to return.
@param {string|[string]} [filter.extensions] The filesystem extensions (NOT including dot) for which files to return.
@return {[string]} An array of filesystem paths.

Utility method for listing all files in a bundle .


function (bundleName, filter) {
        var bundle = this._bundles[bundleName];
        if (!bundle) {
            throw new Error('Unknown bundle "' + bundleName + '"');
        }
        return this._walkBundleResources(bundle, filter);
    }

function
(
bundleName
,
filter
)
{
var
bundle
=
this
.
_bundles
[
bundleName
]
;
if
(
!
bundle
)
{
throw
new
Error
(
'Unknown bundle "'
+
bundleName
+
'"'
)
;
}
return
this
.
_walkBundleResources
(
bundle
,
filter
)
;
}

Utility method for listing all resources in a bundle.
@method getBundleResources
@param {string} bundleName The name of the bundle.
@param {object} filter Filter for deciding which resources to return.
@param {string|[string]} [filter.extensions] The filesystem extensions (NOT including dot) for which resources to return.
@param {string|[string]} [filter.types] The resources types for which resources to return.
@return {[string]} An array of filesystem paths.

Utility method for listing all resources in a bundle .


function (filter) {
        var self = this,
            ress = [];
        Object.keys(this._bundles).forEach(function (bundleName) {
            var bundle = self._bundles[bundleName];
            self._walkBundleResources(bundle, filter).forEach(function (res) {
                ress.push(res);
            });
        });
        return ress;
    }

function ( filter ) { var self = this , ress = [ ] ; Object . keys ( this . _bundles ) . forEach ( function ( bundleName ) { var bundle = self . _bundles [ bundleName ] ; self . _walkBundleResources ( bundle , filter ) . forEach ( function ( res ) { ress . push ( res ) ; } ) ; } ) ; return ress ; }

Returns a list of resources in all the bundles.
@method getAllResources
@param {object} filter Filter for deciding which resources to list.
@param {string|[string]} [filter.extensions] The filesystem extensions (NOT including dot) to return.
@param {string|[string]} [filter.types] The resources types to return.
@return {[LocatorResource]} An array of resources.

Returns a list of resources in all the bundles .


function (filter) {
        var bundleName,
            bundles = this._bundles,
            bundleNames = [];
        if ('function' !== typeof filter) {
            return Object.keys(this._bundles);
        }
        for (bundleName in bundles) {
            if (bundles.hasOwnProperty(bundleName)) {
                if (filter(bundles[bundleName])) {
                    bundleNames.push(bundleName);
                }
            }
        }
        return bundleNames;
    }

function ( filter ) { var bundleName , bundles = this . _bundles , bundleNames = [ ] ; if ( 'function' !== typeof filter ) { return Object . keys ( this . _bundles ) ; } for ( bundleName in bundles ) { if ( bundles . hasOwnProperty ( bundleName ) ) { if ( filter ( bundles [ bundleName ] ) ) { bundleNames . push ( bundleName ) ; } } } return bundleNames ; }

Returns a list of all located bundle names.
The names are not ordered.
@method getBundleNames
@param {Function} [filter] function to execute on each bundle.
If no `filter` is supplied, all bundle names will be returned. The
function will receive the following argument:
@param {Object} filter.bundle the current bundle being iterated on
@param {boolean} filter.return Return true to indicate that the
bundle name should be returned in the list. Otherise the bundle
name will be skipped.
@return {Array} list of bundles

Returns a list of all located bundle names . The names are not ordered .


function (findPath) {
        // FUTURE OPTIMIZATION:  use a more complicated datastructure for faster lookups
        var found = {}, // length: path
            longest;
        // expands path in case of symlinks
        findPath = libfs.realpathSync(findPath);
        // searchs based on expanded path
        Object.keys(this._bundlePaths).forEach(function (bundlePath) {
            if (0 === findPath.indexOf(bundlePath) &&
                    (findPath.length === bundlePath.length ||
                    libpath.sep === findPath.charAt(bundlePath.length))) {
                found[bundlePath.length] = bundlePath;
            }
        });
        longest = Math.max.apply(Math, Object.keys(found));
        return this._bundlePaths[found[longest]];
    }

Returns the name of the bundle to which the path belongs.
@private
@method _getBundleNameByPath
@param {string} findPath The path to consider.
@return {string} The name the bundle to which the findPath most closely matches.

Returns the name of the bundle to which the path belongs .