-
Notifications
You must be signed in to change notification settings - Fork 57
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Initialize Markdown project with marked parser ported in Java. See
- Loading branch information
1 parent
698a4e0
commit 8b5eecb
Showing
37 changed files
with
2,637 additions
and
76 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
14 changes: 14 additions & 0 deletions
14
org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/model/ITokenizationSupport.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 1,14 @@ | ||
| package org.eclipse.tm4e.core.model; | ||
|
|
||
| public interface ITokenizationSupport { | ||
|
|
||
| TMState getInitialState(); | ||
|
|
||
| LineTokens tokenize(String line, TMState state); | ||
|
|
||
| // add offsetDelta to each of the returned indices | ||
| // stop tokenizing at absolute value stopAtOffset (i.e. stream.pos() | ||
| // offsetDelta > stopAtOffset) | ||
| LineTokens tokenize(String line, TMState state, Integer offsetDelta, Integer stopAtOffset); | ||
|
|
||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
102 changes: 102 additions & 0 deletions
102
org.eclipse.tm4e.core/src/main/java/org/eclipse/tm4e/core/model/Tokenizer.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 1,102 @@ | ||
| package org.eclipse.tm4e.core.model; | ||
|
|
||
| import java.util.ArrayList; | ||
| import java.util.LinkedHashMap; | ||
| import java.util.List; | ||
| import java.util.Map; | ||
|
|
||
| import org.eclipse.tm4e.core.grammar.IGrammar; | ||
| import org.eclipse.tm4e.core.grammar.IToken; | ||
| import org.eclipse.tm4e.core.grammar.ITokenizeLineResult; | ||
|
|
||
| public class Tokenizer implements ITokenizationSupport { | ||
|
|
||
| private final IGrammar grammar; | ||
| private final DecodeMap decodeMap; | ||
|
|
||
| public Tokenizer(IGrammar grammar) { | ||
| this.grammar = grammar; | ||
| this.decodeMap = new DecodeMap(); | ||
| } | ||
|
|
||
| @Override | ||
| public TMState getInitialState() { | ||
| return new TMState(null, null); | ||
| } | ||
|
|
||
| @Override | ||
| public LineTokens tokenize(String line, TMState state) { | ||
| return tokenize(line, state, null, null); | ||
| } | ||
|
|
||
| @Override | ||
| public LineTokens tokenize(String line, TMState state, Integer offsetDelta, Integer stopAtOffset) { | ||
| if (offsetDelta == null) { | ||
| offsetDelta = 0; | ||
| } | ||
| // Do not attempt to tokenize if a line has over 20k | ||
| // or if the rule stack contains more than 100 rules (indicator of | ||
| // broken grammar that forgets to pop rules) | ||
| // if (line.length >= 20000 || depth(state.ruleStack) > 100) { | ||
| // return new RawLineTokens( | ||
| // [new Token(offsetDelta, '')], | ||
| // [new ModeTransition(offsetDelta, this._modeId)], | ||
| // offsetDelta, | ||
| // state | ||
| // ); | ||
| // } | ||
| TMState freshState = state.clone(); | ||
| ITokenizeLineResult textMateResult = grammar.tokenizeLine(line, freshState.getRuleStack()); | ||
| freshState.setRuleStack(textMateResult.getRuleStack()); | ||
|
|
||
| // Create the result early and fill in the tokens later | ||
| List<TMToken> tokens = new ArrayList<>(); | ||
| String lastTokenType = null; | ||
| for (int tokenIndex = 0, len = textMateResult.getTokens().length; tokenIndex < len; tokenIndex ) { | ||
| IToken token = textMateResult.getTokens()[tokenIndex]; | ||
| int tokenStartIndex = token.getStartIndex(); | ||
| String tokenType = decodeTextMateToken(this.decodeMap, token.getScopes().toArray(new String[0])); | ||
|
|
||
| // do not push a new token if the type is exactly the same (also | ||
| // helps with ligatures) | ||
| if (!tokenType.equals(lastTokenType)) { | ||
| tokens.add(new TMToken(tokenStartIndex offsetDelta, tokenType)); | ||
| lastTokenType = tokenType; | ||
| } | ||
| } | ||
| return new LineTokens(tokens, offsetDelta line.length(), freshState); | ||
|
|
||
| } | ||
|
|
||
| private String decodeTextMateToken(DecodeMap decodeMap, String[] scopes) { | ||
| String[] prevTokenScopes = decodeMap.prevToken.scopes; | ||
| int prevTokenScopesLength = prevTokenScopes.length; | ||
| Map<Integer, Map<Integer, Boolean>> prevTokenScopeTokensMaps = decodeMap.prevToken.scopeTokensMaps; | ||
|
|
||
| Map<Integer, Map<Integer, Boolean>> scopeTokensMaps = new LinkedHashMap<>(); | ||
| Map<Integer, Boolean> prevScopeTokensMaps = new LinkedHashMap<>(); | ||
| boolean sameAsPrev = true; | ||
| for (int level = 1/* deliberately skip scope 0 */; level < scopes.length; level ) { | ||
| String scope = scopes[level]; | ||
|
|
||
| if (sameAsPrev) { | ||
| if (level < prevTokenScopesLength && prevTokenScopes[level].equals(scope)) { | ||
| prevScopeTokensMaps = prevTokenScopeTokensMaps.get(level); | ||
| scopeTokensMaps.put(level, prevScopeTokensMaps); | ||
| continue; | ||
| } | ||
| sameAsPrev = false; | ||
| } | ||
|
|
||
| int[] tokens = decodeMap.getTokenIds(scope); | ||
| prevScopeTokensMaps = new LinkedHashMap<>(prevScopeTokensMaps); | ||
| for (int i = 0; i < tokens.length; i ) { | ||
| prevScopeTokensMaps.put(tokens[i], true); | ||
| } | ||
| scopeTokensMaps.put(level, prevScopeTokensMaps); | ||
| } | ||
|
|
||
| decodeMap.prevToken = new TMTokenDecodeData(scopes, scopeTokensMaps); | ||
| return decodeMap.getToken(prevScopeTokensMaps); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 1,12 @@ | ||
| <?xml version="1.0" encoding="UTF-8"?> | ||
| <classpath> | ||
| <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.8"> | ||
| <attributes> | ||
| <attribute name="maven.pomderived" value="true"/> | ||
| </attributes> | ||
| </classpathentry> | ||
| <classpathentry kind="con" path="org.eclipse.pde.core.requiredPlugins"/> | ||
| <classpathentry kind="src" path="src/main/java"/> | ||
| <classpathentry kind="src" path="src/main/resources"/> | ||
| <classpathentry kind="output" path="target/classes"/> | ||
| </classpath> |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 1,2 @@ | ||
| /bin | ||
| /target |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 1,34 @@ | ||
| <?xml version="1.0" encoding="UTF-8"?> | ||
| <projectDescription> | ||
| <name>org.eclipse.tm4e.markdown</name> | ||
| <comment></comment> | ||
| <projects> | ||
| </projects> | ||
| <buildSpec> | ||
| <buildCommand> | ||
| <name>org.eclipse.jdt.core.javabuilder</name> | ||
| <arguments> | ||
| </arguments> | ||
| </buildCommand> | ||
| <buildCommand> | ||
| <name>org.eclipse.pde.ManifestBuilder</name> | ||
| <arguments> | ||
| </arguments> | ||
| </buildCommand> | ||
| <buildCommand> | ||
| <name>org.eclipse.pde.SchemaBuilder</name> | ||
| <arguments> | ||
| </arguments> | ||
| </buildCommand> | ||
| <buildCommand> | ||
| <name>org.eclipse.m2e.core.maven2Builder</name> | ||
| <arguments> | ||
| </arguments> | ||
| </buildCommand> | ||
| </buildSpec> | ||
| <natures> | ||
| <nature>org.eclipse.jdt.core.javanature</nature> | ||
| <nature>org.eclipse.m2e.core.maven2Nature</nature> | ||
| <nature>org.eclipse.pde.PluginNature</nature> | ||
| </natures> | ||
| </projectDescription> |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 1,14 @@ | ||
| Manifest-Version: 1.0 | ||
| Bundle-ManifestVersion: 2 | ||
| Bundle-Name: %pluginName | ||
| Bundle-Vendor: %providerName | ||
| Bundle-Localization: plugin | ||
| Bundle-SymbolicName: org.eclipse.tm4e.markdown;singleton:=true | ||
| Bundle-Version: 1.0.0.qualifier | ||
| Require-Bundle: org.eclipse.core.runtime, | ||
| org.eclipse.tm4e.core, | ||
| org.eclipse.tm4e.ui | ||
| Bundle-RequiredExecutionEnvironment: JavaSE-1.8 | ||
| Export-Package: org.eclipse.tm4e.markdown, | ||
| org.eclipse.tm4e.markdown.marked | ||
| Bundle-ActivationPolicy: lazy |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 1,5 @@ | ||
| source.. = src/main/java/,\ | ||
| src/main/resources/ | ||
| bin.includes = META-INF/,\ | ||
| .,\ | ||
| plugin.properties |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 1,48 @@ | ||
| <html> | ||
|
|
||
| <head> | ||
|
|
||
| <script src="marked.js" > </script> | ||
|
|
||
| <script type="text/javascript"> | ||
|
|
||
| function f() { | ||
|
|
||
| var html = marked(document.getElementById('a').value) | ||
| document.getElementById('result').value = html; | ||
|
|
||
| } | ||
| </script> | ||
|
|
||
| </head> | ||
| <body onload="f()"> | ||
| <textarea id="a" onkeyup="f()" cols="100" rows="20" >Every `cf push` deploys applications to one particular Cloud Foundry instance. Every Cloud Foundry instance may have a shared domain set by an admin. Unless you specify a domain, Cloud Foundry incorporates that shared domain in the route to your application. | ||
|
|
||
| You can use the `domain` attribute when you want your application to be served from a domain other than the default shared domain. | ||
|
|
||
| ``` | ||
| --- | ||
| ... | ||
| domain: unique-example.com | ||
| ``` | ||
|
|
||
| The command line option that overrides this attribute is `-d`. | ||
|
|
||
| ### The domains attribute | ||
|
|
||
| Use the `domains` attribute to provide multiple domains. If you define both `domain` and `domains` attributes, Cloud Foundry creates routes for domains defined in both of these fields. | ||
|
|
||
| ``` | ||
| --- | ||
| ... | ||
| domains: | ||
| - domain-example1.com | ||
| - domain-example2.org | ||
| ``` | ||
|
|
||
| The command line option that overrides this attribute is `-d`.</textarea> | ||
| <textarea id="result" cols="100" rows="20" > | ||
| </textarea> | ||
| </body> | ||
|
|
||
| </html> |
Oops, something went wrong.