Merge pull request #210 from emqx/scan-invalid-utf8-error

zhongwencool · web-flow · commit 900ce562184b · 2022-11-28T20:29:32.000+08:00
chore: make scan invalid utf8 error more clear
diff --git a/src/hocon_token.erl b/src/hocon_token.erl
@@ -58,7 +58,12 @@ read(Filename) ->
 
 -spec scan(binary() | string(), hocon:ctx()) -> list().
 scan(Input, Ctx) when is_binary(Input) ->
-    scan(unicode_list(Input), Ctx);
+    case unicode_list(Input) of
+        {error, _Ok, Invalid} ->
+            throw({scan_invalid_utf8, Invalid, Ctx});
+        InputList ->
+            scan(InputList, Ctx)
+    end;
 scan(Input, Ctx) when is_list(Input) ->
     case hocon_scanner:string(Input) of
         {ok, Tokens, _EndLine} ->
diff --git a/test/data/invalid-utf8.conf b/test/data/invalid-utf8.conf
@@ -0,0 +1,28 @@
+bridges {
+  webhook {
+    test {
+      body = "<!-- Edited by XMLSpy� --> \n<note> \n<to>Tove</to> \n<from>Jani</from> \n<heading>Reminder</heading> \n<body>Don't forget me this weekend!</body> \n</note>"
+      connect_timeout = "11s"
+      direction = "egress"
+      enable = true
+      enable_pipelining = 100
+      headers {"content-type" = "application/json"}
+      max_retries = 3
+      method = "post"
+      pool_size = 9
+      pool_type = "random"
+      request_timeout = "5s"
+      ssl {
+        ciphers = ""
+        depth = 10
+        enable = false
+        reuse_sessions = true
+        secure_renegotiate = true
+        user_lookup_fun = "emqx_tls_psk:lookup"
+        verify = "verify_peer"
+        versions = ["tlsv1.3", "tlsv1.2", "tlsv1.1", "tlsv1"]
+      }
+      url = "http://127.0.0.1:18083"
+    }
+  }
+}
diff --git a/test/data/unicode-utf8.conf b/test/data/unicode-utf8.conf
@@ -0,0 +1,6 @@
+# unicode:characters_to_list(<<"®">>, utf8) return {error, _, _}
+# but unicode:characters_to_list(<<"®"/utf8>>, utf8) is ok.
+test {
+  body = "<!-- Edited by XML-XXX® --><note>\n</note>"
+  text = "你我他"
+}
diff --git a/test/hocon_tests.erl b/test/hocon_tests.erl
@@ -881,3 +881,22 @@ unescape_test() ->
         },
         Conf
     ).
+
+unicode_utf8_test() ->
+    {ok, Conf} = hocon:load("./test/data/unicode-utf8.conf"),
+    ?assertEqual(
+        #{
+            <<"test">> =>
+                #{
+                    <<"body">> => <<"<!-- Edited by XML-XXX® --><note>\n</note>"/utf8>>,
+                    <<"text">> => <<"你我他"/utf8>>
+                }
+        },
+        Conf
+    ).
+
+invalid_utf8_test() ->
+    ?assertMatch(
+        {error, {scan_invalid_utf8, _, _}},
+        hocon:load("./test/data/invalid-utf8.conf")
+    ).
