refactor lexer

Author: Schultzer

lib/bnf.ex

@@ -43,7 +43,7 @@ defmodule SQL.BNF do
         cond do
           String.ends_with?(r, "word>") == true ->
             e = if is_map_key(opts, r), do: e ++ opts[r], else: e
-            {[{r, (for v <- e, v not in ["|", "AS"], do: {atom(v), quoted_match(v), quoted_guard(v)})} | keywords], operators, letters, digits, terminals}
+            {[{r, (for v <- e, v not in ["|", "AS"], do: {atom(v), quoted_inline_match(v)})} | keywords], operators, letters, digits, terminals}
           String.ends_with?(r, "letter>") == true -> {keywords, operators, [{r, Enum.reject(e, &(&1 == "|"))}|letters], digits, terminals}
           String.ends_with?(r, "digit>") == true -> {keywords, operators, letters, [{r, Enum.reject(e, &(&1 == "|"))}|digits], terminals}
           String.ends_with?(r, "operator>") == true -> {keywords, [rule | operators], letters, digits, terminals}
@@ -201,7 +201,7 @@ defmodule SQL.BNF do
   @doc false
   def cast(<<?<, b, _::binary>> = expr) when b in ?a..?z or b in ?A..?Z, do: expr
   def cast(expr) when expr in ["|", "{", "}", "[", "]", :ignore, :self, "\u0020", "\u0009", "\u000D", "\u00A0", "\u00A0", "\u1680", "\u2000", "\u2001", "\u2002", "\u2003", "\u2004", "\u2005", "\u2006", "\u2007", "\u2008", "\u2009", "\u200A", "\u202F", "\u205F", "\u3000", "\u180E", "\u200B", "\u200C", "\u200D", "\u2060", "\uFEFF", "\u000A", "\u000B", "\u000C", "\u000D", "\u0085", "\u2028", "\u2029"], do: expr
-  def cast(expr) when is_binary(expr), do: {atom(expr), quoted_match(expr), quoted_guard(expr)}
+  def cast(expr) when is_binary(expr), do: {atom(expr), quoted_inline_match(expr)}
   def cast(expr) when is_tuple(expr) or is_list(expr) or is_atom(expr), do: expr
 
   @doc false
@@ -213,34 +213,18 @@ defmodule SQL.BNF do
       if String.match?(e, ~r/[^A-Za-z]/) do
         {String.to_atom(e), quoted_inline_match(e)}
       else
-        {atom(e), quoted_match(e), quoted_guard(e)}
+        {atom(e), quoted_inline_match(e)}
       end
     end
   end
 
-  @doc false
-  def quoted_match(value), do: Enum.reduce(1..byte_size(value), [], fn n, acc -> [{:"b#{n}", [], Elixir}|acc] end)
-
   @doc false
   def quoted_inline_match(value) do
-    for <<k <- value>>, reduce: [] do
+    for <<k <- String.downcase(value)>>, reduce: [] do
       acc -> [k|acc]
     end
   end
 
-  @doc false
-  def quoted_guard(value, acc \\ []) do
-    {value, _n} = for <<k <- String.downcase(value)>>, reduce: {acc, 1} do
-      {[], n}  -> {__guard__(k, n), n+1}
-      {acc, n} -> {{:and, [context: Elixir, imports: [{2, Kernel}]], [acc,__guard__(k, n)]}, n+1}
-    end
-    value
-  end
-
-  @doc false
-  def __guard__(k, n) do
-    {:in, [context: Elixir, imports: [{2, Kernel}]],[{:"b#{n}", [], Elixir},{:sigil_c, [delimiter: "\"", context: Elixir, imports: [{2, Kernel}]],[{:<<>>, [], ["#{<<k>>}#{String.upcase(<<k>>)}"]}, []]}]}
-  end
 
   @doc false
   def get_rules() do

lib/format.ex

@@ -11,8 +11,9 @@ defmodule SQL.Format do
   @doc since: "0.4.0"
   def to_iodata(tokens, context, indent \\ 0), do: newline(to_iodata(tokens, context.binding, context.case, context.errors, indent, []), indent)
 
-  defp indention(acc, [{:preset, {_,0}},_,{:offset, {_,0,_,_}}|_], 0), do: acc
+  defp indention(acc, [{:preset, {_,0}},_,{:offset, {_,0,_,_}}|_], _), do: acc
   defp indention(acc, [_,{:offset, {_,0}}|_], 0), do: acc
+  defp indention(["\e[33m"|_]=acc, [_,{:offset, {_,0}}|_], _), do: acc
   defp indention(acc, _, 0), do: [?\s|acc]
   defp indention(acc, _, 1), do: [?\s,?\s|acc]
   defp indention(acc, _, 2), do: [?\s,?\s,?\s,?\s|acc]

lib/formatter.ex

@@ -13,8 +13,7 @@ defmodule SQL.MixFormatter do
     opts = Map.new(Keyword.merge([validate: nil], opts))
     {:ok, context, tokens} = SQL.Lexer.lex(source)
     {:ok, context, tokens} = SQL.Parser.parse(tokens, Map.merge(context, opts))
-    iodata = SQL.Format.to_iodata(tokens, context)
-    if context.errors != [], do: IO.warn([?\n, SQL.format_error(context.errors), "  \n  ", iodata, ?\n])
-    IO.iodata_to_binary(iodata)
+    if context.errors != [], do: IO.warn([?\n,SQL.format_error(context.errors),"  \n  ",SQL.Format.to_iodata(tokens, context, 1),?\n])
+    IO.iodata_to_binary(SQL.Format.to_iodata(tokens, context))
   end
 end

lib/lexer.ex

@@ -215,12 +215,12 @@ defmodule SQL do
 
   @doc false
   def __inspect__(tokens, context, stack) do
-    inspect = IO.iodata_to_binary(["\e[0m", "  ~SQL\"\"\""|[SQL.Format.to_iodata(tokens, context, 1)|~c"\n  \"\"\""]])
+    inspect = IO.iodata_to_binary(["\e[0m", "~SQL\"\"\""|[SQL.Format.to_iodata(tokens, context)|~c"\n\"\"\""]])
     case context.errors do
       [] -> inspect
       errors ->
         {:current_stacktrace, [_|t]} = Process.info(self(), :current_stacktrace)
-        IO.warn([?\n,format_error(errors), inspect], [stack|t])
+        IO.warn([?\n,format_error(errors), IO.iodata_to_binary(["\e[0m", "  ~SQL\"\"\""|[SQL.Format.to_iodata(tokens, context, 1)|~c"\n  \"\"\""]])], [stack|t])
         inspect
     end
   end

lib/sql.ex

@@ -90,6 +90,9 @@ defmodule SQL.Token do
       defp __to_iodata__({tag, [_,_,{:type, type}|_]=m, [left, right]}, format, case, acc) when type == :operator or tag in ~w[between cursor for to union except intersect]a do
         to_iodata(left, format, case, indention(to_iodata(tag, format, case, to_iodata(right, format, case, acc)), format, m))
       end
+      defp __to_iodata__({:ident, m, value}=node, format, :upper, acc) do
+        indention([:string.uppercase(value)|acc], format, m)
+      end
       defp __to_iodata__({tag, m, value}=node, format, _case, acc) when tag in ~w[ident numeric special]a do
         indention([value|acc], format, m)
       end

lib/token.ex

@@ -181,7 +181,7 @@ defmodule SQL.Adapters.MySQLTest do
       assert "select \"db.users.id\"" == to_string(~SQL[select "db.users.id"])
       assert "select 'db.users'" == to_string(~SQL[select 'db.users'])
       assert "select \"db.users.id\", 'db.users'" == to_string(~SQL[select "db.users.id", 'db.users'])
-      assert "select X'616263'" == to_string(~SQL[select X'616263'])
+      assert "select x'616263'" == to_string(~SQL[select X'616263'])
     end
   end
 

test/adapters/mysql_test.exs

@@ -9,10 +9,10 @@ defmodule SQL.LexerTest do
       assert {:ok, _, [{:colon, [{:span, {1, 9, 1, 9}}, {:offset, {0, 0}}|_], _}|_]} = SQL.Lexer.lex("SELECT 1;\nSELECT 2;")
     end
     test "U+000D cr" do
-      assert {:ok, _, [{:colon, [{:span, {0, 19, 0, 19}}, {:offset, {0, 0}}|_], _}|_]} = SQL.Lexer.lex("SELECT 1;\rSELECT 2;")
+      assert {:ok, _, [{:colon, [{:span, {1, 9, 1, 9}}, {:offset, {0, 0}}|_], _}|_]} = SQL.Lexer.lex("SELECT 1;\rSELECT 2;")
     end
     test "U+000D+000A crlf" do
-      assert {:ok, _, [{:colon, [{:span, {1, 9, 1, 9}}, {:offset, {0, 0}}|_], _}|_]} = SQL.Lexer.lex("SELECT 1;\r\nSELECT 2;")
+      assert {:ok, _, [{:colon, [{:span, {2, 9, 2, 9}}, {:offset, {0, 0}}|_], _}|_]} = SQL.Lexer.lex("SELECT 1;\r\nSELECT 2;")
     end
     test "U+0085 nel" do
       assert {:ok, _, [{:colon, [{:span, {1, 9, 1, 9}}, {:offset, {0, 0}}|_], _}|_]} = SQL.Lexer.lex("SELECT 1;\u0085SELECT 2;")

test/lexer_test.exs

@@ -47,7 +47,7 @@ defmodule SQLTest do
 
 
   test "inspect/1" do
-    assert ~s(\e[0m  ~SQL"""\n  \e[35mselect\e[0m\n    \e[33m+1000\e[0m\n  """) == inspect(~SQL[select +1000])
+    assert ~s(\e[0m~SQL\"\"\"\n\e[35mselect\e[0m\n  \e[35m+\e[0m\e[33m1000\e[0m\n\"\"\") == inspect(~SQL[select +1000])
   end
 
   test "to_sql/1" do