feat(#871-strictMode): improved validation, strict mode for quotes implementation

src/main/java/org/json/JSONArray.java

@@ -109,8 +109,12 @@ public class JSONArray implements Iterable<Object> {
                     this.myArrayList.add(JSONObject.NULL);
                 } else {
                     x.back();
+                    if (jsonParserConfiguration.isStrictMode()) {
+                        this.myArrayList.add(x.nextValue(true));
+                    } else {
                         this.myArrayList.add(x.nextValue());
                     }
+                }
                 switch (x.nextClean()) {
                     case 0:
                         // array is unclosed. No ']' found, instead EOF
@@ -1693,7 +1697,8 @@ public class JSONArray implements Iterable<Object> {
      * @throws JSONException        If not an array or if an array value is non-finite number.
      * @throws NullPointerException Thrown if the array parameter is null.
      */
-    private void addAll(Object array, boolean wrap, int recursionDepth, JSONParserConfiguration jsonParserConfiguration)
+    private void addAll(Object array, boolean wrap, int recursionDepth, JSONParserConfiguration
+        jsonParserConfiguration)
         throws JSONException {
         if (array.getClass().isArray()) {
             int length = Array.getLength(array);

src/main/java/org/json/JSONTokener.java

@@ -8,28 +8,45 @@ Public Domain.
  */
 
 /**
- * A JSONTokener takes a source string and extracts characters and tokens from
+ * A JSONTokener takes a source string and extracts characters and tokens from it. It is used by the JSONObject and
- * it. It is used by the JSONObject and JSONArray constructors to parse
+ * JSONArray constructors to parse JSON source strings.
- * JSON source strings.
+ *
  * @author JSON.org
  * @version 2014-05-03
  */
 public class JSONTokener {
-    /** current read character position on the current line. */
+
+    /**
+     * current read character position on the current line.
+     */
     private long character;
-    /** flag to indicate if the end of the input has been found. */
+    /**
+     * flag to indicate if the end of the input has been found.
+     */
     private boolean eof;
-    /** current read index of the input. */
+    /**
+     * current read index of the input.
+     */
     private long index;
-    /** current line of the input. */
+    /**
+     * current line of the input.
+     */
     private long line;
-    /** previous character read from the input. */
+    /**
+     * previous character read from the input.
+     */
     private char previous;
-    /** Reader for the input. */
+    /**
+     * Reader for the input.
+     */
     private final Reader reader;
-    /** flag to indicate that a previous character was requested. */
+    /**
+     * flag to indicate that a previous character was requested.
+     */
     private boolean usePrevious;
-    /** the number of characters read in the previous line. */
+    /**
+     * the number of characters read in the previous line.
+     */
     private long characterPreviousLine;
 
 
@@ -54,6 +71,7 @@ public class JSONTokener {
 
     /**
      * Construct a JSONTokener from an InputStream. The caller must close the input stream.
+     *
      * @param inputStream The source.
      */
     public JSONTokener(InputStream inputStream) {
@@ -72,11 +90,10 @@ public class JSONTokener {
 
 
     /**
-     * Back up one character. This provides a sort of lookahead capability,
+     * Back up one character. This provides a sort of lookahead capability, so that you can test for a digit or letter
-     * so that you can test for a digit or letter before attempting to parse
+     * before attempting to parse the next number or identifier.
-     * the next number or identifier.
+     *
-     * @throws JSONException Thrown if trying to step back more than 1 step
+     * @throws JSONException Thrown if trying to step back more than 1 step or if already at the start of the string
-     *  or if already at the start of the string
      */
     public void back() throws JSONException {
         if (this.usePrevious || this.index <= 0) {
@@ -102,8 +119,8 @@ public class JSONTokener {
 
     /**
      * Get the hex value of a character (base16).
-     * @param c A character between '0' and '9' or between 'A' and 'F' or
+     *
-     * between 'a' and 'f'.
+     * @param c A character between '0' and '9' or between 'A' and 'F' or between 'a' and 'f'.
      * @return An int between 0 and 15, or -1 if c was not a hex digit.
      */
     public static int dehexchar(char c) {
@@ -130,11 +147,10 @@ public class JSONTokener {
 
 
     /**
-     * Determine if the source string still contains characters that next()
+     * Determine if the source string still contains characters that next() can consume.
-     * can consume.
+     *
      * @return true if not yet at the end of the source.
-     * @throws JSONException thrown if there is an error stepping forward
+     * @throws JSONException thrown if there is an error stepping forward or backward while checking for more data.
-     *  or backward while checking for more data.
      */
     public boolean more() throws JSONException {
         if (this.usePrevious) {
@@ -188,13 +204,17 @@ public class JSONTokener {
 
     /**
      * Get the last character read from the input or '\0' if nothing has been read yet.
+     *
      * @return the last character read from the input.
      */
-    protected char getPrevious() { return this.previous;}
+    protected char getPrevious() {
+        return this.previous;
+    }
 
     /**
-     * Increments the internal indexes according to the previous character
+     * Increments the internal indexes according to the previous character read and the character passed as the current
-     * read and the character passed as the current character.
+     * character.
+     *
      * @param c the current character read.
      */
     private void incrementIndexes(int c) {
@@ -217,8 +237,8 @@ public class JSONTokener {
     }
 
     /**
-     * Consume the next character, and check that it matches a specified
+     * Consume the next character, and check that it matches a specified character.
-     * character.
+     *
      * @param c The character to match.
      * @return The character.
      * @throws JSONException if the character does not match.
@@ -241,9 +261,7 @@ public class JSONTokener {
      *
      * @param n The number of characters to take.
      * @return A string of n characters.
-     * @throws JSONException
+     * @throws JSONException Substring bounds error if there are not n characters remaining in the source string.
-     *   Substring bounds error if there are not
-     *   n characters remaining in the source string.
      */
     public String next(int n) throws JSONException {
         if (n == 0) {
@@ -266,8 +284,9 @@ public class JSONTokener {
 
     /**
      * Get the next char in the string, skipping whitespace.
-     * @throws JSONException Thrown if there is an error reading the source string.
+     *
      * @return A character, or 0 if there are no more characters.
+     * @throws JSONException Thrown if there is an error reading the source string.
      */
     public char nextClean() throws JSONException {
         for (; ; ) {
@@ -280,10 +299,9 @@ public class JSONTokener {
 
 
     /**
-     * Return the characters up to the next close quote character.
+     * Return the characters up to the next close quote character. Backslash processing is done. The formal JSON format
-     * Backslash processing is done. The formal JSON format does not
+     * does not allow strings in single quotes, but an implementation is allowed to accept them.
-     * allow strings in single quotes, but an implementation is allowed to
+     *
-     * accept them.
      * @param quote The quoting character, either
      *              <code>"</code>&nbsp;<small>(double quote)</small> or
      *              <code>'</code>&nbsp;<small>(single quote)</small>.
@@ -346,12 +364,11 @@ public class JSONTokener {
 
 
     /**
-     * Get the text up but not including the specified character or the
+     * Get the text up but not including the specified character or the end of line, whichever comes first.
-     * end of line, whichever comes first.
+     *
      * @param delimiter A delimiter character.
      * @return A string.
-     * @throws JSONException Thrown if there is an error while searching
+     * @throws JSONException Thrown if there is an error while searching for the delimiter
-     *  for the delimiter
      */
     public String nextTo(char delimiter) throws JSONException {
         StringBuilder sb = new StringBuilder();
@@ -369,12 +386,12 @@ public class JSONTokener {
 
 
     /**
-     * Get the text up but not including one of the specified delimiter
+     * Get the text up but not including one of the specified delimiter characters or the end of line, whichever comes
-     * characters or the end of line, whichever comes first.
+     * first.
+     *
      * @param delimiters A set of delimiter characters.
      * @return A string, trimmed.
-     * @throws JSONException Thrown if there is an error while searching
+     * @throws JSONException Thrown if there is an error while searching for the delimiter
-     *  for the delimiter
      */
     public String nextTo(String delimiters) throws JSONException {
         char c;
@@ -394,51 +411,149 @@ public class JSONTokener {
 
 
     /**
-     * Get the next value. The value can be a Boolean, Double, Integer,
+     * Get the next value. The value can be a Boolean, Double, Integer, JSONArray, JSONObject, Long, or String, or the
-     * JSONArray, JSONObject, Long, or String, or the JSONObject.NULL object.
+     * JSONObject.NULL object.
-     * @throws JSONException If syntax error.
      *
      * @return An object.
+     * @throws JSONException If syntax error.
      */
     public Object nextValue() throws JSONException {
+        return nextValue(false);
+    }
+
+    /**
+     * Get the next value. The value can be a Boolean, Double, Integer, JSONArray, JSONObject, Long, or String, or the
+     * JSONObject.NULL object. The strictMode parameter controls the behavior of the method when parsing the value.
+     *
+     * @param strictMode If true, the method will strictly adhere to the JSON syntax, throwing a JSONException for any
+     *                   deviations.
+     * @return An object.
+     * @throws JSONException If syntax error.
+     */
+    public Object nextValue(boolean strictMode) throws JSONException {
         char c = this.nextClean();
         switch (c) {
             case '{':
                 this.back();
+                return getJsonObject(strictMode);
+            case '[':
+                this.back();
+                return getJsonArray();
+            default:
+                return getValue(c, strictMode);
+        }
+    }
+
+    /**
+     * This method is used to get the next value.
+     *
+     * @param c          The next character in the JSONTokener.
+     * @param strictMode If true, the method will strictly adhere to the JSON syntax, throwing a JSONException if the
+     *                   value is not surrounded by quotes.
+     * @return An object which is the next value in the JSONTokener.
+     * @throws JSONException If the value is not surrounded by quotes when strictMode is true.
+     */
+    private Object getValue(char c, boolean strictMode) {
+        if (strictMode) {
+            Object valueToValidate = nextSimpleValue(c, true);
+
+            boolean isNumeric = valueToValidate.toString().chars().allMatch( Character::isDigit );
+
+            if(isNumeric){
+                return valueToValidate;
+            }
+
+            boolean hasQuotes = valueIsWrappedByQuotes(valueToValidate);
+
+            if (!hasQuotes) {
+                throw new JSONException("Value is not surrounded by quotes: " + valueToValidate);
+            }
+
+            return valueToValidate;
+        }
+
+        return nextSimpleValue(c);
+    }
+
+    /**
+     * This method is used to get a JSONObject from the JSONTokener. The strictMode parameter controls the behavior of
+     * the method when parsing the JSONObject.
+     *
+     * @param strictMode If true, the method will strictly adhere to the JSON syntax, throwing a JSONException for any
+     *                   deviations.
+     * @return A JSONObject which is the next value in the JSONTokener.
+     * @throws JSONException If the JSONObject or JSONArray depth is too large to process.
+     */
+    private JSONObject getJsonObject(boolean strictMode) {
         try {
+            if (strictMode) {
+                return new JSONObject(this, new JSONParserConfiguration().withStrictMode(true));
+            }
+
             return new JSONObject(this);
         } catch (StackOverflowError e) {
             throw new JSONException("JSON Array or Object depth too large to process.", e);
         }
-        case '[':
+    }
-            this.back();
+
+    /**
+     * This method is used to get a JSONArray from the JSONTokener.
+     *
+     * @return A JSONArray which is the next value in the JSONTokener.
+     * @throws JSONException If the JSONArray depth is too large to process.
+     */
+    private JSONArray getJsonArray() {
         try {
             return new JSONArray(this);
         } catch (StackOverflowError e) {
             throw new JSONException("JSON Array or Object depth too large to process.", e);
         }
     }
-        return nextSimpleValue(c);
+
+    /**
+     * This method checks if the provided value is wrapped by quotes.
+     *
+     * @param valueToValidate The value to be checked. It is converted to a string before checking.
+     * @return A boolean indicating whether the value is wrapped by quotes. It returns true if the value is wrapped by
+     * either single or double quotes.
+     */
+    private boolean valueIsWrappedByQuotes(Object valueToValidate) {
+        String stringToValidate = valueToValidate.toString();
+        boolean isWrappedByDoubleQuotes = isWrappedByQuotes(stringToValidate, "\"");
+        boolean isWrappedBySingleQuotes = isWrappedByQuotes(stringToValidate, "'");
+        return isWrappedByDoubleQuotes || isWrappedBySingleQuotes;
+    }
+
+    private boolean isWrappedByQuotes(String valueToValidate, String quoteType) {
+        return valueToValidate.startsWith(quoteType) && valueToValidate.endsWith(quoteType);
     }
 
     Object nextSimpleValue(char c) {
-        String string;
+        return nextSimpleValue(c, false);
-
-        switch (c) {
-        case '"':
-        case '\'':
-            return this.nextString(c);
     }
 
-        /*
+    Object nextSimpleValue(char c, boolean strictMode) {
-         * Handle unquoted text. This could be the values true, false, or
+        if (c == '"' || c == '\'') {
-         * null, or it can be a number. An implementation (such as this one)
+            String str = this.nextString(c);
-         * is allowed to also accept non-standard forms.
+            if (strictMode) {
-         *
+                return String.format("\"%s\"", str);
-         * Accumulate characters until we reach the end of the text or a
+            }
-         * formatting character.
+            return str;
-         */
+        }
 
+        return parsedUnquotedText(c);
+    }
+
+    /**
+     * Parses unquoted text from the JSON input. This could be the values true, false, or null, or it can be a number.
+     * Non-standard forms are also accepted. Characters are accumulated until the end of the text or a formatting
+     * character is reached.
+     *
+     * @param c The starting character.
+     * @return The parsed object.
+     * @throws JSONException If the parsed string is empty.
+     */
+    private Object parsedUnquotedText(char c) {
         StringBuilder sb = new StringBuilder();
         while (c >= ' ' && ",:]}/\\\"[{;=#".indexOf(c) < 0) {
             sb.append(c);
@@ -448,8 +563,8 @@ public class JSONTokener {
             this.back();
         }
 
-        string = sb.toString().trim();
+        String string = sb.toString().trim();
-        if ("".equals(string)) {
+        if (string.isEmpty()) {
             throw this.syntaxError("Missing value");
         }
         return JSONObject.stringToValue(string);
@@ -457,13 +572,12 @@ public class JSONTokener {
 
 
     /**
-     * Skip characters until the next character is the requested character.
+     * Skip characters until the next character is the requested character. If the requested character is not found, no
-     * If the requested character is not found, no characters are skipped.
+     * characters are skipped.
+     *
      * @param to A character to skip to.
-     * @return The requested character, or zero if the requested character
+     * @return The requested character, or zero if the requested character is not found.
-     * is not found.
+     * @throws JSONException Thrown if there is an error while searching for the to character
-     * @throws JSONException Thrown if there is an error while searching
-     *  for the to character
      */
     public char skipTo(char to) throws JSONException {
         char c;

src/test/java/org/json/junit/JSONParserConfigurationTest.java

@@ -2,6 +2,7 @@ package org.json.junit;
 
 import java.util.Arrays;
 import java.util.List;
+import java.util.stream.Collectors;
 import org.json.JSONArray;
 import org.json.JSONException;
 import org.json.JSONObject;
@@ -31,25 +32,23 @@ public class JSONParserConfigurationTest {
 
     @Test
     public void givenInvalidInputArrays_testStrictModeTrue_shouldThrowJsonException() {
-        List<String> strictModeInputTestCases = Arrays.asList("[1,2];[3,4]", "", "[1, 2,3]:[4,5]", "[{test: implied}]");
+        List<String> strictModeInputTestCases = getNonCompliantJSONList();
         JSONParserConfiguration jsonParserConfiguration = new JSONParserConfiguration()
             .withStrictMode(true);
 
         strictModeInputTestCases.forEach(testCase -> {
-            System.out.println("Test case: " + testCase);
             assertThrows("expected non-compliant array but got instead: " + testCase, JSONException.class,
                 () -> new JSONArray(testCase, jsonParserConfiguration));
-            System.out.println("Passed");
         });
     }
 
     @Test
     public void givenInvalidInputArrays_testStrictModeFalse_shouldNotThrowAnyException() {
-        List<String> strictModeInputTestCases = Arrays.asList("[1,2];[3,4]", "[1, 2,3]:[4,5]", "[{test: implied}]");
+        List<String> strictModeInputTestCases = getNonCompliantJSONList();
         JSONParserConfiguration jsonParserConfiguration = new JSONParserConfiguration()
             .withStrictMode(false);
 
-        strictModeInputTestCases.stream().peek(System.out::println).forEach(testCase -> new JSONArray(testCase, jsonParserConfiguration));
+        strictModeInputTestCases.forEach(testCase -> new JSONArray(testCase, jsonParserConfiguration));
     }
 
     @Test
@@ -71,4 +70,14 @@ public class JSONParserConfigurationTest {
         assertTrue(jsonParserConfiguration.isOverwriteDuplicateKey());
         assertEquals(42, jsonParserConfiguration.getMaxNestingDepth());
     }
+
+    private List<String> getNonCompliantJSONList() {
+        return Arrays.asList(
+            "[1,2];[3,4]",
+            "[1, 2,3]:[4,5]",
+            "[{test: implied}]",
+            "[{\"test\": implied}]",
+            "[{\"number\":\"7990154836330\",\"color\":'c'},{\"number\":8784148854580,\"color\":RosyBrown},{\"number\":\"5875770107113\",\"color\":\"DarkSeaGreen\"}]",
+            "[{test: \"implied\"}]");
+    }
 }