add support for markdown mode rendering

package.json

@@ -10,18 +10,19 @@
     "serve": "node src/server/babel.index.js",
     "webpack-stats": "webpack --profile --json > stats.json",
     "test": "mocha --watch --compilers js:babel-register"
-
   },
   "author": "",
   "license": "BSD-2-Clause",
   "dependencies": {
     "babel-preset-es2015-mod": "^6.6.0",
     "babel-preset-es3": "^1.0.1",
+    "babel-preset-stage-2": "^6.18.0",
     "body-parser": "^1.15.2",
     "diff": "^3.0.1",
     "express": "^4.14.0",
     "jsonfile": "^2.4.0",
     "markdown-it": "^5.1.0",
+    "markdown-to-jsx": "^4.0.3",
     "react": "^0.14.5",
     "react-dom": "^0.14.5",
     "react-redux": "^4.4.6",
@@ -43,6 +44,7 @@
     "babel-register": "^6.18.0",
     "chai": "^3.5.0",
     "copyfiles": "^0.2.2",
+    "json-loader": "^0.5.4",
     "mocha": "^3.2.0",
     "piping": "^1.0.0-rc.4",
     "webpack": "^2.1.0-beta.27"

src/client/index.js

@@ -11,6 +11,8 @@ import  {Router, Route, IndexRoute, Redirect } from 'react-router'
 import * as localStore from '../common/localStore'
 import * as reducers from '../common/reducers'
 import routes from '../common/routes'
+import * as Actions from '../common/actions'
+
 
 
 //the localStore implementation is naive
@@ -29,8 +31,11 @@ const store = Redux.createStore(
 )
 
 const localInput = localStore.get('dubdiff')
+console.log(localInput)
 if (localInput.input) {
   //dispatch localStore data to store
+  store.dispatch(Actions.updateOriginalInput(localInput.input.original))
+  store.dispatch(Actions.updateFinalInput(localInput.input.final))
   //should this be done after the first render?
 }
 

src/common/components/Compare.js

@@ -11,6 +11,7 @@ import Footer from './Footer'
 import CompareControls from './CompareControls'
 
 import ShowPlaintext from './ShowPlaintext'
+import ShowMarkdown from './ShowMarkdown'
 
 const mapStateToProps = (state) => ({
   isMarkdownFormat: Selectors.isMarkdownFormat(state),
@@ -29,6 +30,7 @@ const mapDispatchToProps = dispatch => ({
 class Compare extends React.Component {
 
   render() {
+    console.log({isMarkdownFormat: this.props.isMarkdownFormat, isShowDifference: this.props.isShowDifference})
     return (
       <div>
         <Header/>
@@ -40,14 +42,20 @@ class Compare extends React.Component {
             </Grid.Column>
             <Grid.Column width="13">
               <Segment>
-                { this.props.isShowDifference ?
+                {
-
+                  (!this.props.isMarkdownFormat && this.props.isShowDifference) ? 
-                    <ShowPlaintext diff={this.props.diff} isMarkdownFormat={this.props.isMarkdownFormat}>{this.props.diff}</ShowPlaintext>: 
+                      <ShowPlaintext diff={this.props.diff}>{this.props.diff}</ShowPlaintext>: 
-
+                  (this.props.isMarkdownFormat && this.props.isShowDifference) ?
+                      <ShowMarkdown diff={this.props.diff}>{this.props.diff}</ShowMarkdown>: 
+                  (!this.props.isMarkdownFormat && !this.props.isShowDifference) ?
                       <ShowPlaintext 
                         text={this.props.isShowOriginal? this.props.compare.original: this.props.compare.final} 
-                      isMarkdownFormat={this.props.isMarkdownFormat}
+                      /> :
-                    />
+                  (this.props.isMarkdownFormat && !this.props.isShowDifference) ?
+                      <ShowMarkdown 
+                        text={this.props.isShowOriginal? this.props.compare.original: this.props.compare.final} 
+                      /> :
+                  null
                 }
               </Segment>
             </Grid.Column>

src/common/components/ShowMarkdown.js

@@ -1,34 +1,23 @@
 import React from 'react'
 
+import markdownCompiler  from 'markdown-to-jsx'
 
-//use markdown-it to render markdown
+import {diffToString, diffToHtml} from '../util/dubdiff'
-//alternately use markdown to jsx
 
 const ShowMarkdown = (props) => {
+  if (props.diff)
+    console.log(diffToString(props.diff))
+
   return <div>
-    <pre style={{whiteSpace:'pre-wrap'}}>
+      {
-      {props.text ?
+        props.text ?
-        props.text: 
+          markdownCompiler(props.text) :
         props.diff ?
-          diffToPre(props.diff) :
+          markdownCompiler(diffToHtml(props.diff)) :
         null
       }
-    </pre>
   </div>
 }
 
 export default ShowMarkdown
 
-function diffToPre(diff) {
-  return diff.map(part => (
-      part.added ? <span><ins>{part.value}</ins>{ifNotNewlineSpace(part.value)}</span> :
-      part.removed ? <span><del>{part.value}</del>{ifNotNewlineSpace(part.value)}</span> :
-      <span>{part.value}{ifNotNewlineSpace(part.value)}</span>
-    ))
-}
-
-
-
-const ifNotNewlineSpace = str => {
-  return !str.endsWith('\n') ? ' ' : ''
-}

src/common/components/ShowPlaintext.js

@@ -18,14 +18,8 @@ export default ShowPlaintext
 
 function diffToPre(diff) {
   return diff.map((part, index) => (
-      part.added ? <span key={index}><ins>{part.value}</ins>{ifNotNewlineSpace(part.value)}</span> :
+      part.added ? <ins key={index}>{part.value}</ins> :
-      part.removed ? <span key={index}><del>{part.value}</del>{ifNotNewlineSpace(part.value)}</span> :
+      part.removed ? <del key={index}>{part.value}</del> :
-      <span key={index}>{part.value}{ifNotNewlineSpace(part.value)}</span>
+      <span key={index}>{part.value}</span>
     ))
 }
-
-
-
-const ifNotNewlineSpace = str => {
-  return !str.endsWith('\n') ? ' ' : ''
-}

src/common/util/EditorsDiff.js

@@ -1,16 +1,30 @@
 
 import {Diff} from 'diff'
 
-const EditorsDiff = new Diff()
+// EditorsDiff is a custom Diff implementation from the jsdiff library
+// It allows diffing by phrases. Whitespace is ignored for the purpose of comparison,
+// but is preserved and included in the output.
 
-EditorsDiff.equals = function(left, right) {
+const TOKEN_BOUNDARYS = /([\s,.:])/
+    
+class EditorsDiff extends Diff {
+  constructor (tokenBoundaries=TOKEN_BOUNDARYS) {
+    super()
+    this.tokenBoundaries = tokenBoundaries
+  }
+
+  equals (left, right) {
     return (
       left.string == right.string 
     )
       
   }
-EditorsDiff.tokenize = function(value) {
+
-  let tokens = value.split(/([ ]+)|(\n)/)
+
+  //splits the input string into a series of word and punctuation tokens
+    //each token is associated with an optional trailing array of spaces
+  tokenize (value) {
+    let tokens = value.split(this.tokenBoundaries)
     let annotatedTokens = [] 
     tokens.forEach( token => {
       if (isSpace(token)) {
@@ -24,10 +38,12 @@ EditorsDiff.tokenize = function(value) {
         annotatedTokens.push({string:token, whitespace:[]})
       }
     })
-  console.log(annotatedTokens)
+
+    //this final empty token is necessary for the jsdiff diffing engine to work properly
+    annotatedTokens.push({string:'', whitespace:[]})
     return annotatedTokens
   }
-EditorsDiff.join = function (annotatedTokens) {
+  join(annotatedTokens) {
     let tokens = []
     annotatedTokens.forEach(annotatedToken => {
       tokens.push(annotatedToken.string)
@@ -35,10 +51,12 @@ EditorsDiff.join = function (annotatedTokens) {
         tokens.push(item)
       })
     })
-
-  console.log(tokens.join(''))
     return tokens.join('')
   }
+} 
+
+
+
 
 export default EditorsDiff
 

src/common/util/dubdiff.js

@@ -1,32 +1,18 @@
 import * as JsDiff from 'diff'
 import EditorsDiff from './EditorsDiff'
 
+let plaintextDiffer = new EditorsDiff()
+let markdownDiffer = new EditorsDiff(/([\s,.:]|[*\[\]\(\)])/)
 
-//!!! this deal with adding and removing spaces could be done more elegantly by 
+//returns a comparison of the texts as plaintext
-// diffing on an array of simple data structures that contain the text and the adjacent space
-// the diff would use a custom compare function that would disregard the spaces
-// alternately, the text could be split with the spaces included in the array and then compared with a 
-// custom diff function that would treat the space elements as null/ignored
-
-//the current mechanism for adding and removing spaces is fragile and broken
 export function plaintextDiff(original, final) {
-    //let arrOriginal = plaintextSplit(original)
+    let diff = plaintextDiffer.diff(original, final)
-    //let arrFinal = plaintextSplit(final)
-
-    let diff = EditorsDiff.diff(original, final)
-    //diff = plaintextRestoreSpaces(diff)
-
     return diff
 }
 
+//returns a comparison of the texts as markdown
 export function markdownDiff(original, final) {
-//    let arrOriginal = plaintextSplit(original)
+    let diff = markdownDiffer.diff(original, final)
-//    let arrFinal = plaintextSplit(final)
-
-//    let diff = JsDiff.diffArrays(arrOriginal, arrFinal)
-//    diff = plaintextRestoreSpaces(diff)
-
-    let diff = EditorsDiff.diff(original, final)
     diff = rewriteMarkdownDiff(diff)
 
     return diff  
@@ -34,10 +20,12 @@ export function markdownDiff(original, final) {
 
 // returns a string version of the diff, with "{+ ... +}" and "[- ... -]"
 // representing ins and del blocks
-export function diffToString(diff) {
+export function diffToString(diff, tags={added:{start:'{+', end:'+}'}, removed:{start:'[-', end:'-]'}, same:{start:'', end:''}}) {
+
   return diff.map(({added, removed, value}) => {
-      let start = added ? '{+' : removed ? '[-' : ''
+
-      let end = added ? '+}' : removed ? '-]' : ''
+      let {start,end} = added ? tags.added : (removed ? tags.removed : tags.same)
+
       let string = value
       if (Array.isArray(value))
         string = value.join('')
@@ -46,16 +34,8 @@ export function diffToString(diff) {
   }).join('')
 }
 
-let plaintextSplit = text =>text.split(/[ ]|(\n)/)
+export function diffToHtml(diff) {
-
+  return diffToString(diff, {added:{start:'<ins>', end:'</ins>'}, removed:{start:'<del>', end:'</del>'}, same:{start:'', end:''}})  
-function plaintextRestoreSpaces (diff) {
-  return diff.map(({added, removed, value}) => ({
-    added, 
-    removed, 
-    value:value.map((str, idx, arr) => (
-      (str!='\n' && (idx<arr.length-1)) ? str+" " : str)
-    )
-  }))
 }
 
 
@@ -80,7 +60,7 @@ function rewriteMarkdownDiff(diff) {
   //apply transformation rules
   let transformedDiff = diff
   transformedDiff= applyTransformationRule1(transformedDiff)
-  //transformedDiff= applyTransformationRule2(transformedDiff)
+  transformedDiff= applyTransformationRule2(transformedDiff)
   return transformedDiff
 }
 
@@ -109,7 +89,6 @@ function applyTransformationRule1(diff) {
     // the previous block was a del and had multiple lines
     // the current block is an ins
     if (previousBlockType == B_REMOVED && currentBlockType == B_ADDED && previousBlockWasMultiline) {
-      console.log('trigger rule 1')
 
       //split the first line from the current block
       let currentBlockSplit = splitMultilineDiffBlock(currentBlock)
@@ -120,7 +99,6 @@ function applyTransformationRule1(diff) {
       //split the first line from the previous block
       let previousBlockSplit = splitMultilineDiffBlock(previousBlock)
 
-      console.log({currentBlock, currentBlockSplit, previousBlock, previousBlockSplit})
 
       //now add the blocks back, interleaving del and ins blocks
       for (let i=0; i<Math.max(previousBlockSplit.length, currentBlockSplit.length); i++) {
@@ -139,9 +117,8 @@ function applyTransformationRule1(diff) {
   return transformedDiff
 }
 
-function applyTransformationRule2(diff) {
+
-  // we need to find markdown prefixes that should be pulled out of added/removed blocks to the start of the line
+// matches markdown prefixes that affect the formatting of the whole subsequent line
-  // prefixes are matched as follows:
   // ^                  - start of line
   // ([ \t]*\>)*       - blockquotes (possibly nested)
   // (
@@ -150,86 +127,70 @@ function applyTransformationRule2(diff) {
   //  |([ \t]+[0-9]+\.)  - numeric lists
   // )?
   // [ \t]*             - trailing whitespace
-  const PREFIX =  /^([ \t]*\>)*(([ \t]*#*)|([ \t]*[\*\+-])|([ \t]*[\d]+\.))?[ \t]*/
+const MARKDOWN_PREFIX =  /^([ \t]*\>)*(([ \t]*#*)|([ \t]*[\*\+-])|([ \t]*[\d]+\.))?[ \t]*/
 
+//matches strings that end with a newline followed by some whitespace
+const NEWLINE_SUFFIX = /\n\s*$/
+
+// transformation rule 2:
+//    after a newline, if an ins or del block begins with a markdown line formatting prefix (eg. for a title or list)
+//    then that prefix should be moved out of the block
+//    also, if an ins block begins with a formatting prefix and follows immediately after a del block that follows a newline,
+//    the prefix should be moved out of the block _and_ an extra newline character should be added to the beginning of it
+function applyTransformationRule2(diff) {
   let transformedDiff = []
-  return transformedDiff
 
+  let isNewline = true
+  let newlineString = '\n'
 
-  /// ...
+  //iterate the input tokens to create the intermediate representation
-  /*
+  diff.forEach((currentBlock) => {
-  transform.forEach(function(item) {
-    //newlines are undecorated
-    if (item.string == '\n') {
-      output += '\n';
 
-      //flag the new line
+    if (isNewline && (currentBlock.added || currentBlock.removed) ) {
-      newline = true;
+      let match = currentBlock.value.match(MARKDOWN_PREFIX)
-      //and record the offset in the output string
+      if (match) {
-      newlineIndex = output.length;
+        let preBlock = {value:match[0]}
-      return
+        let postBlock = {added:currentBlock.added, removed:currentBlock.removed, value:currentBlock.value.substring(match[0].length)}
+        
+        if (currentBlock.added) {
+          let newlineBlock = {value: newlineString}
+          transformedDiff.push(newlineBlock)
         }
-
+        transformedDiff.push(preBlock)
-    //wrap del strings with tags
+        transformedDiff.push(postBlock)
-    if (item.state == SDEL) {
-      output += '<del>' + item.string + '</del>';
-      //del doesn't reset the newline state
       }
-
-    //ins strings have to be handled a little differently:
-    //if this is an ins just after a newline, or after a del after a newline, 
-    //we need to peel off any markdown formatting prefixes and insert them at the beginning of the line outside the del/ins tags
-    else if (item.state == SINS && newline) {
-      var prestring, poststring;
-      var match = item.string.match(PREFIX);
-      if (match == null)
-        prestring ="";
-      else
-        prestring = match[0];
-
-      poststring = item.string.substring(prestring.length);
-
-      output = output.substring(0, newlineIndex) + prestring + output.substring(newlineIndex);
-      output += '<ins>' + poststring + '</ins>';
-      newline = false;
-      newlineIndex = -1;
-
-    }
-
-    else if (item.state == SINS) {
-      output += '<ins>' + item.string + '</ins>';
-    }
-
-    //and just output other strings
       else {
-      output += item.string;
+        transformedDiff.push(currentBlock)
-      //this resets the newline state
-      newline = false;
-      newlineIndex = -1;
       }
+    }
+    else {
+      transformedDiff.push(currentBlock)
+      isNewline = NEWLINE_SUFFIX.test(currentBlock.value)
+      if (isNewline)
+        newlineString = currentBlock.value.match(NEWLINE_SUFFIX)[0]
+    }
+  })
 
-  });
+  return transformedDiff
-*/
-
 }
 
 
 
 //returns true if the given diff block contains a newline element
 function isMultilineDiffBlock({value}) {
-  return value.find(word => word == '\n')
+  return value.indexOf('\n') != -1
 }
 
 
 //returns an array of diff blocks that have the same added, removed fields as the given one
-//but with the array of words split by newlines
+//but with the string split by newlines
 //if the diff block has no newlines, an array containing only that diff will be returned
 //if the diff block has newlines, the resulting array will have a series of blocks, 
 // each of which subsequent to the first block will begin with a newline
 //if the diff block begins with a newline, the returned array will begin with an empty diff
 function splitMultilineDiffBlock({added, removed, value}) {
   //find the indices of the diff block that coorespond to newlines
-  const splits = findIndicesOf(value, str => str=='\n')
+  const splits = indicesOf(value, c => (c=='\n') )
 
   splits.push(value.length)
 
@@ -249,16 +210,16 @@ function splitMultilineDiffBlock({added, removed, value}) {
   //map the ranges into blocks
   const blocks = ranges.map(
     //each block is the same as the given original block, but with the values split at newlines
-    ({start, end}) => ({added, removed, value:value.slice(start, end)})
+    ({start, end}) => ({added, removed, value:value.substring(start, end)})
   )
 
-  console.log({value, splits, ranges, blocks})
+  //console.log({value, splits, ranges, blocks})
 
   return blocks
 }
 
-//collect all the indices of the given array that satisfy the test function
+//collect all the indices of the given string that satisfy the test function
-const findIndicesOf = (array, test) => array.reduce( 
+const indicesOf = (string, test) => string.split('').reduce( 
   //add indexes that satisfy the test function to the array
   (acc, x, i) => (test(x) ? acc.concat([i]) : acc ), 
   //start with the empty array

test/dubdiffMarkdown.js

@@ -45,4 +45,30 @@ other`,
 `# Subtitle`
     )).to.equal('# [-Title-]{+Subtitle+}[-\nother-]')
   })
+
+  it('pulls prefixes out of ins or del blocks after newline',  () => {
+    expect(diff(
+      '# Title\n > hello',
+      '# Title\n - goodbye'
+    )).to.equal('# Title\n > [-hello-]\n - {+goodbye+}')
+  })
+
+  it('respects bold and italic boundaries',  () => {
+    expect(diff(
+      'This *word* **isn\'t** changed.',
+      'This *other one* **is** changed.'
+    )).to.equal('This *[-word-]{+other one+}* **[-isn\'t-]{+is+}** changed.')
+  })
+  it('respects link boundaries in link text',  () => {
+    expect(diff(
+      'This [link](https://somewhere.com) is the same.',
+      'This [target](https://somewhere.com) changed.'
+    )).to.equal('This [[-link-]{+target+}](https://somewhere.com) [-is the same-]{+changed+}.')
+  })
+  it('respects link boundaries in link target',  () => {
+    expect(diff(
+      'This [link](https://somewhere.com) is the same.',
+      'This [link](https://somewhere.org) changed.'
+    )).to.equal('This [link](https://somewhere.[-com-]{+org+}) [-is the same-]{+changed+}.')
+  })
 })

test/dubdiffPlaintext.js

@@ -60,4 +60,10 @@ describe('dubdiff', () => {
       'there\n'
     )).to.equal('there\n')
   })
+  it('treats punctuation separately', () => {
+    expect(diff(
+      'Hello world.',
+      'Hello, world.'
+    )).to.equal('Hello{+, +}world.')
+  })
 })

webpack.config.js

@@ -17,6 +17,7 @@ module.exports = {
           compact: "true"
         }
       },
+      { test: /\.json$/, loader: "json-loader" },
     ]
   },
 };