Skip to content

Commit

Permalink
add support for single quoted strings
Browse files Browse the repository at this point in the history 
see #9
  • Loading branch information
@laktak
laktak committed Jul 17, 2017
1 parent 1879d82 commit dfd7189
Show file tree
Hide file tree
Showing 20 changed files with 149 additions and 44 deletions.
22 changes: 14 additions & 8 deletions hjson/decoder.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,9 28,9 @@ def _floatconstants():
PUNCTUATOR = '{}[],:'

NUMBER_RE = re.compile(r'[\t ]*(-?(?:0|[1-9]\d*))(\.\d )?([eE][- ]?\d )?[\t ]*')
STRINGCHUNK = re.compile(r'(.*?)(["\\\x00-\x1f])', FLAGS)
STRINGCHUNK = re.compile(r'(.*?)([\'"\\\x00-\x1f])', FLAGS)
BACKSLASH = {
'"': u('"'), '\\': u('\u005c'), '/': u('/'),
'"': u('"'), '\'': u('\''), '\\': u('\u005c'), '/': u('/'),
'b': u('\b'), 'f': u('\f'), 'n': u('\n'), 'r': u('\r'), 't': u('\t'),
}

Expand Down Expand Up @@ -97,6 97,8 @@ def scanstring(s, end, encoding=None, strict=True,
chunks = []
_append = chunks.append
begin = end - 1
# callers make sure that string starts with " or '
exitCh = s[begin]
while 1:
chunk = _m(s, end)
if chunk is None:
Expand All @@ -111,8 113,11 @@ def scanstring(s, end, encoding=None, strict=True,
_append(content)
# Terminator is the end of string, a literal control character,
# or a backslash denoting that an escape sequence follows
if terminator == '"':
if terminator == exitCh:
break
elif terminator == '"' or terminator == '\'':
_append(terminator)
continue
elif terminator != '\\':
if strict:
msg = "Invalid control character %r at"
Expand Down Expand Up @@ -263,7 268,7 @@ def scanKeyName(s, end, encoding=None, strict=True):

ch, end = getNext(s, end)

if ch == '"':
if ch == '"' or ch == '\'':
return scanstring(s, end 1, encoding, strict)

begin = end
Expand Down Expand Up @@ -305,15 310,16 @@ def _scan_once(string, idx):
except IndexError:
raise HjsonDecodeError('Expecting value', string, idx)

if ch == '"':
return parse_string(string, idx 1, encoding, strict)
if ch == '"' or ch == '\'':
if string[idx:idx 3] == '\'\'\'':
return parse_mlstring(string, idx)
else:
return parse_string(string, idx 1, encoding, strict)
elif ch == '{':
return parse_object((string, idx 1), encoding, strict,
_scan_once, object_hook, object_pairs_hook, memo)
elif ch == '[':
return parse_array((string, idx 1), _scan_once)
elif ch == '\'' and string[idx:idx 3] == '\'\'\'':
return parse_mlstring(string, idx)

return parse_tfnns(context, string, idx)

Expand Down
6 changes: 3 additions & 3 deletions hjson/encoderH.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,14 33,14 @@
# NEEDSESCAPE tests if the string can be written without escapes
NEEDSESCAPE = re.compile(u'[\\\"\x00-\x1f' COMMONRANGE ']')
# NEEDSQUOTES tests if the string can be written as a quoteless string (like needsEscape but without \\ and \")
NEEDSQUOTES = re.compile(u'^\s|^"|^\'\'\'|^#|^\/\*|^\/\/|^\{|^\}|^\[|^\]|^:|^,|\s$|[\x00-\x1f' COMMONRANGE u']')
NEEDSQUOTES = re.compile(u'^\\s|^"|^\'|^#|^\\/\\*|^\\/\\/|^\\{|^\\}|^\\[|^\\]|^:|^,|\\s$|[\x00-\x1f' COMMONRANGE u']')
# NEEDSESCAPEML tests if the string can be written as a multiline string (like needsEscape but without \n, \r, \\, \", \t)
NEEDSESCAPEML = re.compile(u'\'\'\'|^[\s] $|[\x00-\x08\x0b\x0c\x0e-\x1f' COMMONRANGE u']')
NEEDSESCAPEML = re.compile(u'\'\'\'|^[\\s] $|[\x00-\x08\x0b\x0c\x0e-\x1f' COMMONRANGE u']')

WHITESPACE = ' \t\n\r'
STARTSWITHNUMBER = re.compile(r'^[\t ]*(-?(?:0|[1-9]\d*))(\.\d )?([eE][- ]?\d )?\s*((,|\]|\}|#|\/\/|\/\*).*)?$');
STARTSWITHKEYWORD = re.compile(r'^(true|false|null)\s*((,|\]|\}|#|\/\/|\/\*).*)?$');
NEEDSESCAPENAME = re.compile(r'[,\{\[\}\]\s:#"]|\/\/|\/\*|' "'''")
NEEDSESCAPENAME = re.compile(r'[,\{\[\}\]\s:#"\']|\/\/|\/\*|' "'''")

FLOAT_REPR = repr

Expand Down
14 changes: 8 additions & 6 deletions hjson/tests/assets/charset_test.hjson
View file
Original file line number Diff line number Diff line change
@@ -1,6 1,8 @@
ql-ascii: ! "#$%&'()* ,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~
js-ascii: "! \"#$%&'()* ,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~"
ml-ascii:
'''
! "#$%&'()* ,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~
'''
{
ql-ascii: ! "#$%&'()* ,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~
js-ascii: "! \"#$%&'()* ,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~"
ml-ascii:
'''
! "#$%&'()* ,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~
'''
}
0 hjson/tests/assets/root_result.hjson → hjson/tests/assets/extra/root_result.hjson
View file
File renamed without changes.
0 hjson/tests/assets/root_result.json → hjson/tests/assets/extra/root_result.json
View file
File renamed without changes.
0 hjson/tests/assets/root_test.hjson → hjson/tests/assets/extra/root_test.hjson
View file
File renamed without changes.
5 changes: 5 additions & 0 deletions hjson/tests/assets/extra/root_testmeta.hjson
View file
Original file line number Diff line number Diff line change
@@ -0,0 1,5 @@
{
options: {
legacyRoot: true
}
}
1 change: 0 additions & 1 deletion hjson/tests/assets/failJSON24_test.json
View file

This file was deleted.

4 changes: 4 additions & 0 deletions hjson/tests/assets/failKey5_test.hjson
View file
Original file line number Diff line number Diff line change
@@ -0,0 1,4 @@
{
# invalid name
'''foo''': 0
}
4 changes: 4 additions & 0 deletions hjson/tests/assets/failStr8a_test.hjson
View file
Original file line number Diff line number Diff line change
@@ -0,0 1,4 @@
{
# invalid ml-string
foo : ""'text'''
}
3 changes: 3 additions & 0 deletions hjson/tests/assets/keys_result.hjson
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 18,9 @@
"foo\"bar": test
"'''": test
"foo'''bar": test
"'": test
"'foo": test
"foo'bar": test
":": test
"foo:bar": test
"{": test
Expand Down
3 changes: 3 additions & 0 deletions hjson/tests/assets/keys_result.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 18,9 @@
"foo\"bar": "test",
"'''": "test",
"foo'''bar": "test",
"'": "test",
"'foo": "test",
"foo'bar": "test",
":": "test",
"foo:bar": "test",
"{": "test",
Expand Down
3 changes: 3 additions & 0 deletions hjson/tests/assets/keys_test.hjson
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 22,9 @@
"foo\"bar": test
"'''": test
"foo'''bar": test
"'": test
"'foo": test
"foo'bar": test
# control char in key name
":": test
"foo:bar": test
Expand Down
29 changes: 29 additions & 0 deletions hjson/tests/assets/strings2_result.hjson
View file
Original file line number Diff line number Diff line change
@@ -0,0 1,29 @@
{
key1: a key in single quotes
"key 2": a key in single quotes
"key \"": a key in single quotes
text:
[
single quoted string
'''You need quotes for escapes'''
" untrimmed "
"untrimmed "
containing " double quotes
containing " double quotes
containing " double quotes
'''"containing more " double quotes"'''
containing ' single quotes
containing ' single quotes
containing ' single quotes
"'containing more ' single quotes'"
"'containing more ' single quotes'"
"\n"
" \n"
"\n \n \n \n"
"\t\n"
]
foo3a: asdf'''
foo3b: "'''asdf"
foo4a: "asdf'''\nasdf"
foo4b: "asdf\n'''asdf"
}
28 changes: 28 additions & 0 deletions hjson/tests/assets/strings2_result.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 1,28 @@
{
"key1": "a key in single quotes",
"key 2": "a key in single quotes",
"key \"": "a key in single quotes",
"text": [
"single quoted string",
"You need quotes\tfor escapes",
" untrimmed ",
"untrimmed ",
"containing \" double quotes",
"containing \" double quotes",
"containing \" double quotes",
"\"containing more \" double quotes\"",
"containing ' single quotes",
"containing ' single quotes",
"containing ' single quotes",
"'containing more ' single quotes'",
"'containing more ' single quotes'",
"\n",
" \n",
"\n \n \n \n",
"\t\n"
],
"foo3a": "asdf'''",
"foo3b": "'''asdf",
"foo4a": "asdf'''\nasdf",
"foo4b": "asdf\n'''asdf"
}
36 changes: 36 additions & 0 deletions hjson/tests/assets/strings2_test.hjson
View file
Original file line number Diff line number Diff line change
@@ -0,0 1,36 @@
{
# Hjson 3 allows the use of single quotes

'key1': a key in single quotes
'key 2': a key in single quotes
'key "': a key in single quotes

text: [
'single quoted string'
'You need quotes\tfor escapes'
' untrimmed '
'untrimmed '
'containing " double quotes'
'containing \" double quotes'
"containing \" double quotes"
'"containing more " double quotes"'
'containing \' single quotes'
"containing ' single quotes"
"containing \' single quotes"
"'containing more ' single quotes'"
"\'containing more \' single quotes\'"

'\n'
' \n'
'\n \n \n \n'
'\t\n'
]

# escapes/no escape

foo3a: 'asdf\'\'\''
foo3b: '\'\'\'asdf'

foo4a: 'asdf\'\'\'\nasdf'
foo4b: 'asdf\n\'\'\'asdf'
}
6 changes: 4 additions & 2 deletions hjson/tests/assets/testlist.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 20,6 @@ failJSON20_test.json
failJSON21_test.json
failJSON22_test.json
failJSON23_test.json
failJSON24_test.json
failJSON26_test.json
failJSON28_test.json
failJSON29_test.json
Expand All @@ -33,6 32,7 @@ failKey1_test.hjson
failKey2_test.hjson
failKey3_test.hjson
failKey4_test.hjson
failKey5_test.hjson
failMLStr1_test.hjson
failObj1_test.hjson
failObj2_test.hjson
Expand Down Expand Up @@ -62,6 62,7 @@ failStr6b_test.hjson
failStr6c_test.hjson
failStr6d_test.hjson
failStr7a_test.hjson
failStr8a_test.hjson
kan_test.hjson
keys_test.hjson
mltabs_test.json
Expand All @@ -71,8 72,8 @@ pass2_test.json
pass3_test.json
pass4_test.json
passSingle_test.hjson
root_test.hjson
stringify1_test.hjson
strings2_test.hjson
strings_test.hjson
trail_test.hjson
stringify/quotes_all_test.hjson
Expand All @@ -81,4 82,5 @@ stringify/quotes_keys_test.hjson
stringify/quotes_strings_ml_test.json
stringify/quotes_strings_test.hjson
extra/notabs_test.json
extra/root_test.hjson
extra/separator_test.json
6 changes: 4 additions & 2 deletions hjson/tests/assets/trail_test.hjson
View file
Original file line number Diff line number Diff line change
@@ -1,2 1,4 @@
// the following line contains trailing whitespace:
foo: 0 -- this string starts at 0 and ends at 1, preceding and trailing whitespace is ignored -- 1
{
// the following line contains trailing whitespace:
foo: 0 -- this string starts at 0 and ends at 1, preceding and trailing whitespace is ignored -- 1
}
2 changes: 1 addition & 1 deletion hjson/tests/test_fail.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 52,7 @@
# http://json.org/JSON_checker/test/fail23.json
'["Bad value", truth]',
# http://json.org/JSON_checker/test/fail24.json
"['single quote']",
#"['single quote']",
# http://json.org/JSON_checker/test/fail25.json
'["\ttab\tcharacter\tin\tstring\t"]',
# http://json.org/JSON_checker/test/fail26.json
Expand Down
21 changes: 0 additions & 21 deletions hjson/tests/test_scanstring.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -111,33 111,12 @@ def _test_scanstring(self, scanstring):
ValueError,
scanstring, c '"', 0, None, True)

self.assertRaises(ValueError, scanstring, '', 0, None, True)
self.assertRaises(ValueError, scanstring, 'a', 0, None, True)
self.assertRaises(ValueError, scanstring, '\\', 0, None, True)
self.assertRaises(ValueError, scanstring, '\\u', 0, None, True)
self.assertRaises(ValueError, scanstring, '\\u0', 0, None, True)
self.assertRaises(ValueError, scanstring, '\\u01', 0, None, True)
self.assertRaises(ValueError, scanstring, '\\u012', 0, None, True)
self.assertRaises(ValueError, scanstring, '\\u0123', 0, None, True)
if sys.maxunicode > 65535:
self.assertRaises(ValueError,
scanstring, '\\ud834\\u"', 0, None, True)
self.assertRaises(ValueError,
scanstring, '\\ud834\\x0123"', 0, None, True)

def test_issue3623(self):
self.assertRaises(ValueError, json.decoder.scanstring, "xxx", 1,
"xxx")
self.assertRaises(UnicodeDecodeError,
json.encoder.encode_basestring_ascii, b("xx\xff"))

def test_overflow(self):
# Python 2.5 does not have maxsize, Python 3 does not have maxint
maxsize = getattr(sys, 'maxsize', getattr(sys, 'maxint', None))
assert maxsize is not None
self.assertRaises(OverflowError, json.decoder.scanstring, "xxx",
maxsize 1)

def test_surrogates(self):
scanstring = json.decoder.scanstring

Expand Down

0 comments on commit dfd7189

Please sign in to comment.