vmario
/
dubdiff
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

add support for markdown mode rendering

This commit is contained in:
Adam Brown 2016-12-14 01:30:09 -05:00
parent 9c10e06c15
commit 414c1d570e
11 changed files with 7455 additions and 1907 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8976
dist/browser-bundle.js vendored
View File

File diff suppressed because one or more lines are too long

4
package.json
View File

@ -10,18 +10,19 @@
"serve": "node src/server/babel.index.js",
"webpack-stats": "webpack --profile --json > stats.json",
"test": "mocha --watch --compilers js:babel-register"
},
"author": "",
"license": "BSD-2-Clause",
"dependencies": {
"babel-preset-es2015-mod": "^6.6.0",
"babel-preset-es3": "^1.0.1",
"babel-preset-stage-2": "^6.18.0",
"body-parser": "^1.15.2",
"diff": "^3.0.1",
"express": "^4.14.0",
"jsonfile": "^2.4.0",
"markdown-it": "^5.1.0",
"markdown-to-jsx": "^4.0.3",
"react": "^0.14.5",
"react-dom": "^0.14.5",
"react-redux": "^4.4.6",
@ -43,6 +44,7 @@
"babel-register": "^6.18.0",
"chai": "^3.5.0",
"copyfiles": "^0.2.2",
"json-loader": "^0.5.4",
"mocha": "^3.2.0",
"piping": "^1.0.0-rc.4",
"webpack": "^2.1.0-beta.27"

5
src/client/index.js
View File

@ -11,6 +11,8 @@ import {Router, Route, IndexRoute, Redirect } from 'react-router'
import * as localStore from '../common/localStore'
import * as reducers from '../common/reducers'
import routes from '../common/routes'
import * as Actions from '../common/actions'
//the localStore implementation is naive
@ -29,8 +31,11 @@ const store = Redux.createStore(
)
const localInput = localStore.get('dubdiff')
console.log(localInput)
if (localInput.input) {
//dispatch localStore data to store
store.dispatch(Actions.updateOriginalInput(localInput.input.original))
store.dispatch(Actions.updateFinalInput(localInput.input.final))
//should this be done after the first render?
}

30
src/common/components/Compare.js
View File

@ -11,24 +11,26 @@ import Footer from './Footer'
import CompareControls from './CompareControls'
import ShowPlaintext from './ShowPlaintext'
import ShowMarkdown from './ShowMarkdown'
const mapStateToProps = (state) => ({
isMarkdownFormat: Selectors.isMarkdownFormat(state),
isShowOriginal: Selectors.isShowOriginal(state),
isShowOriginal: Selectors.isShowOriginal(state),
isShowFinal: Selectors.isShowFinal(state),
isShowDifference: Selectors.isShowDifference(state),
compare: state.compare,
compare: state.compare,
diff: Selectors.diff(state)
})
const mapDispatchToProps = dispatch => ({
})
class Compare extends React.Component {
render() {
console.log({isMarkdownFormat: this.props.isMarkdownFormat, isShowDifference: this.props.isShowDifference})
return (
<div>
<Header/>
@ -40,14 +42,20 @@ class Compare extends React.Component {
</Grid.Column>
<Grid.Column width="13">
<Segment>
{ this.props.isShowDifference ?
<ShowPlaintext diff={this.props.diff} isMarkdownFormat={this.props.isMarkdownFormat}>{this.props.diff}</ShowPlaintext>:
<ShowPlaintext
text={this.props.isShowOriginal? this.props.compare.original: this.props.compare.final}
isMarkdownFormat={this.props.isMarkdownFormat}
/>
{
(!this.props.isMarkdownFormat && this.props.isShowDifference) ?
<ShowPlaintext diff={this.props.diff}>{this.props.diff}</ShowPlaintext>:
(this.props.isMarkdownFormat && this.props.isShowDifference) ?
<ShowMarkdown diff={this.props.diff}>{this.props.diff}</ShowMarkdown>:
(!this.props.isMarkdownFormat && !this.props.isShowDifference) ?
<ShowPlaintext
text={this.props.isShowOriginal? this.props.compare.original: this.props.compare.final}
/> :
(this.props.isMarkdownFormat && !this.props.isShowDifference) ?
<ShowMarkdown
text={this.props.isShowOriginal? this.props.compare.original: this.props.compare.final}
/> :
null
}
</Segment>
</Grid.Column>

31
src/common/components/ShowMarkdown.js
View File

@ -1,34 +1,23 @@
import React from 'react'
import markdownCompiler from 'markdown-to-jsx'
//use markdown-it to render markdown
//alternately use markdown to jsx
import {diffToString, diffToHtml} from '../util/dubdiff'
const ShowMarkdown = (props) => {
if (props.diff)
console.log(diffToString(props.diff))
return <div>
<pre style={{whiteSpace:'pre-wrap'}}>
{props.text ?
props.text:
{
props.text ?
markdownCompiler(props.text) :
props.diff ?
diffToPre(props.diff) :
null
markdownCompiler(diffToHtml(props.diff)) :
null
}
</pre>
</div>
}
export default ShowMarkdown
function diffToPre(diff) {
return diff.map(part => (
part.added ? <span><ins>{part.value}</ins>{ifNotNewlineSpace(part.value)}</span> :
part.removed ? <span><del>{part.value}</del>{ifNotNewlineSpace(part.value)}</span> :
<span>{part.value}{ifNotNewlineSpace(part.value)}</span>
))
}
const ifNotNewlineSpace = str => {
return !str.endsWith('\n') ? ' ' : ''
}

14
src/common/components/ShowPlaintext.js
View File

@ -8,7 +8,7 @@ const ShowPlaintext = (props) => {
props.text:
props.diff ?
diffToPre(props.diff) :
null
null
}
</pre>
</div>
@ -18,14 +18,8 @@ export default ShowPlaintext
function diffToPre(diff) {
return diff.map((part, index) => (
part.added ? <span key={index}><ins>{part.value}</ins>{ifNotNewlineSpace(part.value)}</span> :
part.removed ? <span key={index}><del>{part.value}</del>{ifNotNewlineSpace(part.value)}</span> :
<span key={index}>{part.value}{ifNotNewlineSpace(part.value)}</span>
part.added ? <ins key={index}>{part.value}</ins> :
part.removed ? <del key={index}>{part.value}</del> :
<span key={index}>{part.value}</span>
))
}
const ifNotNewlineSpace = str => {
return !str.endsWith('\n') ? ' ' : ''
}

84
src/common/util/EditorsDiff.js
View File

@ -1,44 +1,62 @@
import {Diff} from 'diff'
const EditorsDiff = new Diff()
// EditorsDiff is a custom Diff implementation from the jsdiff library
// It allows diffing by phrases. Whitespace is ignored for the purpose of comparison,
// but is preserved and included in the output.
EditorsDiff.equals = function(left, right) {
return (
left.string == right.string
)
const TOKEN_BOUNDARYS = /([\s,.:])/
}
EditorsDiff.tokenize = function(value) {
let tokens = value.split(/([ ]+)|(\n)/)
let annotatedTokens = []
tokens.forEach( token => {
if (isSpace(token)) {
if (annotatedTokens.length == 0)
annotatedTokens.push({string:'', whitespace:[]})
class EditorsDiff extends Diff {
constructor (tokenBoundaries=TOKEN_BOUNDARYS) {
super()
this.tokenBoundaries = tokenBoundaries
}
let last = annotatedTokens[annotatedTokens.length-1]
last.whitespace.push(token)
}
else {
annotatedTokens.push({string:token, whitespace:[]})
}
})
console.log(annotatedTokens)
return annotatedTokens
}
EditorsDiff.join = function (annotatedTokens) {
let tokens = []
annotatedTokens.forEach(annotatedToken => {
tokens.push(annotatedToken.string)
annotatedToken.whitespace.forEach(item => {
tokens.push(item)
equals (left, right) {
return (
left.string == right.string
)
}
//splits the input string into a series of word and punctuation tokens
//each token is associated with an optional trailing array of spaces
tokenize (value) {
let tokens = value.split(this.tokenBoundaries)
let annotatedTokens = []
tokens.forEach( token => {
if (isSpace(token)) {
if (annotatedTokens.length == 0)
annotatedTokens.push({string:'', whitespace:[]})
let last = annotatedTokens[annotatedTokens.length-1]
last.whitespace.push(token)
}
else {
annotatedTokens.push({string:token, whitespace:[]})
}
})
})
console.log(tokens.join(''))
return tokens.join('')
}
//this final empty token is necessary for the jsdiff diffing engine to work properly
annotatedTokens.push({string:'', whitespace:[]})
return annotatedTokens
}
join(annotatedTokens) {
let tokens = []
annotatedTokens.forEach(annotatedToken => {
tokens.push(annotatedToken.string)
annotatedToken.whitespace.forEach(item => {
tokens.push(item)
})
})
return tokens.join('')
}
}
export default EditorsDiff

175
src/common/util/dubdiff.js
View File

@ -1,32 +1,18 @@
import * as JsDiff from 'diff'
import EditorsDiff from './EditorsDiff'
let plaintextDiffer = new EditorsDiff()
let markdownDiffer = new EditorsDiff(/([\s,.:]|[*\[\]\(\)])/)
//!!! this deal with adding and removing spaces could be done more elegantly by
// diffing on an array of simple data structures that contain the text and the adjacent space
// the diff would use a custom compare function that would disregard the spaces
// alternately, the text could be split with the spaces included in the array and then compared with a
// custom diff function that would treat the space elements as null/ignored
//the current mechanism for adding and removing spaces is fragile and broken
//returns a comparison of the texts as plaintext
export function plaintextDiff(original, final) {
//let arrOriginal = plaintextSplit(original)
//let arrFinal = plaintextSplit(final)
let diff = EditorsDiff.diff(original, final)
//diff = plaintextRestoreSpaces(diff)
let diff = plaintextDiffer.diff(original, final)
return diff
}
//returns a comparison of the texts as markdown
export function markdownDiff(original, final) {
// let arrOriginal = plaintextSplit(original)
// let arrFinal = plaintextSplit(final)
// let diff = JsDiff.diffArrays(arrOriginal, arrFinal)
// diff = plaintextRestoreSpaces(diff)
let diff = EditorsDiff.diff(original, final)
let diff = markdownDiffer.diff(original, final)
diff = rewriteMarkdownDiff(diff)
return diff
@ -34,10 +20,12 @@ export function markdownDiff(original, final) {
// returns a string version of the diff, with "{+ ... +}" and "[- ... -]"
// representing ins and del blocks
export function diffToString(diff) {
export function diffToString(diff, tags={added:{start:'{+', end:'+}'}, removed:{start:'[-', end:'-]'}, same:{start:'', end:''}}) {
return diff.map(({added, removed, value}) => {
let start = added ? '{+' : removed ? '[-' : ''
let end = added ? '+}' : removed ? '-]' : ''
let {start,end} = added ? tags.added : (removed ? tags.removed : tags.same)
let string = value
if (Array.isArray(value))
string = value.join('')
@ -46,16 +34,8 @@ export function diffToString(diff) {
}).join('')
}
let plaintextSplit = text =>text.split(/[ ]|(\n)/)
function plaintextRestoreSpaces (diff) {
return diff.map(({added, removed, value}) => ({
added,
removed,
value:value.map((str, idx, arr) => (
(str!='\n' && (idx<arr.length-1)) ? str+" " : str)
)
}))
export function diffToHtml(diff) {
return diffToString(diff, {added:{start:'<ins>', end:'</ins>'}, removed:{start:'<del>', end:'</del>'}, same:{start:'', end:''}})
}
@ -80,7 +60,7 @@ function rewriteMarkdownDiff(diff) {
//apply transformation rules
let transformedDiff = diff
transformedDiff= applyTransformationRule1(transformedDiff)
//transformedDiff= applyTransformationRule2(transformedDiff)
transformedDiff= applyTransformationRule2(transformedDiff)
return transformedDiff
}
@ -109,7 +89,6 @@ function applyTransformationRule1(diff) {
// the previous block was a del and had multiple lines
// the current block is an ins
if (previousBlockType == B_REMOVED && currentBlockType == B_ADDED && previousBlockWasMultiline) {
console.log('trigger rule 1')
//split the first line from the current block
let currentBlockSplit = splitMultilineDiffBlock(currentBlock)
@ -120,7 +99,6 @@ function applyTransformationRule1(diff) {
//split the first line from the previous block
let previousBlockSplit = splitMultilineDiffBlock(previousBlock)
console.log({currentBlock, currentBlockSplit, previousBlock, previousBlockSplit})
//now add the blocks back, interleaving del and ins blocks
for (let i=0; i<Math.max(previousBlockSplit.length, currentBlockSplit.length); i++) {
@ -139,102 +117,85 @@ function applyTransformationRule1(diff) {
return transformedDiff
}
// matches markdown prefixes that affect the formatting of the whole subsequent line
// ^ - start of line
// ([ \t]*\>)* - blockquotes (possibly nested)
// (
// ([ \t]*#*) - headers
// |([ \t]+[\*\+-]) - unordered lists
// |([ \t]+[0-9]+\.) - numeric lists
// )?
// [ \t]* - trailing whitespace
const MARKDOWN_PREFIX = /^([ \t]*\>)*(([ \t]*#*)|([ \t]*[\*\+-])|([ \t]*[\d]+\.))?[ \t]*/
//matches strings that end with a newline followed by some whitespace
const NEWLINE_SUFFIX = /\n\s*$/
// transformation rule 2:
// after a newline, if an ins or del block begins with a markdown line formatting prefix (eg. for a title or list)
// then that prefix should be moved out of the block
// also, if an ins block begins with a formatting prefix and follows immediately after a del block that follows a newline,
// the prefix should be moved out of the block _and_ an extra newline character should be added to the beginning of it
function applyTransformationRule2(diff) {
// we need to find markdown prefixes that should be pulled out of added/removed blocks to the start of the line
// prefixes are matched as follows:
// ^ - start of line
// ([ \t]*\>)* - blockquotes (possibly nested)
// (
// ([ \t]*#*) - headers
// |([ \t]+[\*\+-]) - unordered lists
// |([ \t]+[0-9]+\.) - numeric lists
// )?
// [ \t]* - trailing whitespace
const PREFIX = /^([ \t]*\>)*(([ \t]*#*)|([ \t]*[\*\+-])|([ \t]*[\d]+\.))?[ \t]*/
let transformedDiff = []
return transformedDiff
let isNewline = true
let newlineString = '\n'
/// ...
/*
transform.forEach(function(item) {
//newlines are undecorated
if (item.string == '\n') {
output += '\n';
//iterate the input tokens to create the intermediate representation
diff.forEach((currentBlock) => {
//flag the new line
newline = true;
//and record the offset in the output string
newlineIndex = output.length;
return
if (isNewline && (currentBlock.added || currentBlock.removed) ) {
let match = currentBlock.value.match(MARKDOWN_PREFIX)
if (match) {
let preBlock = {value:match[0]}
let postBlock = {added:currentBlock.added, removed:currentBlock.removed, value:currentBlock.value.substring(match[0].length)}
if (currentBlock.added) {
let newlineBlock = {value: newlineString}
transformedDiff.push(newlineBlock)
}
transformedDiff.push(preBlock)
transformedDiff.push(postBlock)
}
else {
transformedDiff.push(currentBlock)
}
}
//wrap del strings with tags
if (item.state == SDEL) {
output += '<del>' + item.string + '</del>';
//del doesn't reset the newline state
}
//ins strings have to be handled a little differently:
//if this is an ins just after a newline, or after a del after a newline,
//we need to peel off any markdown formatting prefixes and insert them at the beginning of the line outside the del/ins tags
else if (item.state == SINS && newline) {
var prestring, poststring;
var match = item.string.match(PREFIX);
if (match == null)
prestring ="";
else
prestring = match[0];
poststring = item.string.substring(prestring.length);
output = output.substring(0, newlineIndex) + prestring + output.substring(newlineIndex);
output += '<ins>' + poststring + '</ins>';
newline = false;
newlineIndex = -1;
}
else if (item.state == SINS) {
output += '<ins>' + item.string + '</ins>';
}
//and just output other strings
else {
output += item.string;
//this resets the newline state
newline = false;
newlineIndex = -1;
transformedDiff.push(currentBlock)
isNewline = NEWLINE_SUFFIX.test(currentBlock.value)
if (isNewline)
newlineString = currentBlock.value.match(NEWLINE_SUFFIX)[0]
}
})
});
*/
return transformedDiff
}
//returns true if the given diff block contains a newline element
function isMultilineDiffBlock({value}) {
return value.find(word => word == '\n')
return value.indexOf('\n') != -1
}
//returns an array of diff blocks that have the same added, removed fields as the given one
//but with the array of words split by newlines
//but with the string split by newlines
//if the diff block has no newlines, an array containing only that diff will be returned
//if the diff block has newlines, the resulting array will have a series of blocks,
// each of which subsequent to the first block will begin with a newline
//if the diff block begins with a newline, the returned array will begin with an empty diff
function splitMultilineDiffBlock({added, removed, value}) {
//find the indices of the diff block that coorespond to newlines
const splits = findIndicesOf(value, str => str=='\n')
const splits = indicesOf(value, c => (c=='\n') )
splits.push(value.length)
//create a range from each index
const ranges = splits.reduce(
const ranges = splits.reduce(
//the accumulator is a structure with the last index and the list of ranges
//the ranges are a {start, end} structure
({last, ranges}, i) => {
@ -249,16 +210,16 @@ function splitMultilineDiffBlock({added, removed, value}) {
//map the ranges into blocks
const blocks = ranges.map(
//each block is the same as the given original block, but with the values split at newlines
({start, end}) => ({added, removed, value:value.slice(start, end)})
({start, end}) => ({added, removed, value:value.substring(start, end)})
)
console.log({value, splits, ranges, blocks})
//console.log({value, splits, ranges, blocks})
return blocks
}
//collect all the indices of the given array that satisfy the test function
const findIndicesOf = (array, test) => array.reduce(
//collect all the indices of the given string that satisfy the test function
const indicesOf = (string, test) => string.split('').reduce(
//add indexes that satisfy the test function to the array
(acc, x, i) => (test(x) ? acc.concat([i]) : acc ),
//start with the empty array

34
test/dubdiffMarkdown.js
View File

@ -21,21 +21,21 @@ describe('dubdiff', () => {
expect(diff(
'This is a smlb sentnce with no errors.',
'This is a simple sentence with no errors.'
)).to.equal('This is a [-smlb sentnce-] {+simple sentence+} with no errors.')
)).to.equal('This is a [-smlb sentnce -]{+simple sentence +}with no errors.')
})
it('plaintext diffs with word deletion', ()=>{
expect(diff(
'Gonna delete a word.',
'Gonna delete word.'
)).to.equal('Gonna delete [-a-] word.')
)).to.equal('Gonna delete [-a -]word.')
})
it('plaintext diffs with word insertion', ()=>{
expect(diff(
'Gonna delete word.',
'Gonna delete a word.'
)).to.equal('Gonna delete {+a+} word.')
)).to.equal('Gonna delete {+a +}word.')
})
it('reorganizes insertions after multiline deletions', ()=>{
@ -43,6 +43,32 @@ describe('dubdiff', () => {
`# Title
other`,
`# Subtitle`
)).to.equal('# [-Title-] {+Subtitle+}[-\nother-]')
)).to.equal('# [-Title-]{+Subtitle+}[-\nother-]')
})
it('pulls prefixes out of ins or del blocks after newline', () => {
expect(diff(
'# Title\n > hello',
'# Title\n - goodbye'
)).to.equal('# Title\n > [-hello-]\n - {+goodbye+}')
})
it('respects bold and italic boundaries', () => {
expect(diff(
'This *word* **isn\'t** changed.',
'This *other one* **is** changed.'
)).to.equal('This *[-word-]{+other one+}* **[-isn\'t-]{+is+}** changed.')
})
it('respects link boundaries in link text', () => {
expect(diff(
'This [link](https://somewhere.com) is the same.',
'This [target](https://somewhere.com) changed.'
)).to.equal('This [[-link-]{+target+}](https://somewhere.com) [-is the same-]{+changed+}.')
})
it('respects link boundaries in link target', () => {
expect(diff(
'This [link](https://somewhere.com) is the same.',
'This [link](https://somewhere.org) changed.'
)).to.equal('This [link](https://somewhere.[-com-]{+org+}) [-is the same-]{+changed+}.')
})
})

8
test/dubdiffPlaintext.js
View File

@ -27,7 +27,7 @@ describe('dubdiff', () => {
expect(diff(
'This is a smlb sentnce with no errors.',
'This is a simple sentence with no errors.'
)).to.equal('This is a [-smlb sentnce-]{+simple sentence+} with no errors.')
)).to.equal('This is a [-smlb sentnce -]{+simple sentence +}with no errors.')
})
it('diffs with word deletion', ()=>{
@ -60,4 +60,10 @@ describe('dubdiff', () => {
'there\n'
)).to.equal('there\n')
})
it('treats punctuation separately', () => {
expect(diff(
'Hello world.',
'Hello, world.'
)).to.equal('Hello{+, +}world.')
})
})

1
webpack.config.js
View File

@ -17,6 +17,7 @@ module.exports = {
compact: "true"
}
},
{ test: /\.json$/, loader: "json-loader" },
]
},
};