diff --git a/CHANGELOG.md b/CHANGELOG.md index aa928c0c..95d1f92c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,19 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a ## [Unreleased] +## [2.8.0] - 2022-06-21 + +### Added + +- [#95](https://github.com/ruby-syntax-tree/syntax_tree/pull/95) - The `HeredocEnd` node has been added which effectively results in the ability to determine the location of the ending of a heredoc from source. +- [#99](https://github.com/ruby-syntax-tree/syntax_tree/pull/99) - The LSP now allows you to pass the same configuration options as the other CLI commands which allows formatting to be modified in the VSCode extension. +- [#100](https://github.com/ruby-syntax-tree/syntax_tree/pull/100) - The LSP now explicitly responds to the shutdown request so that VSCode never deadlocks. + +### Changed + +- [#96](https://github.com/ruby-syntax-tree/syntax_tree/pull/96) - The CLI now runs in parallel by default. There is a worker created for each processor on the running machine (as determined by `Etc.nprocessors`). +- [#97](https://github.com/ruby-syntax-tree/syntax_tree/pull/97) - Syntax Tree now handles the case where `DidYouMean` is not available for whatever reason, as well as handles the newer `detailed_message` API for errors. + ## [2.7.1] - 2022-05-25 ### Added @@ -259,7 +272,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a - 🎉 Initial release! 🎉 -[unreleased]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v2.7.1...HEAD +[unreleased]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v2.8.0...HEAD +[2.8.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v2.7.1...v2.8.0 [2.7.1]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v2.7.0...v2.7.1 [2.7.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v2.6.0...v2.7.0 [2.6.0]: https://github.com/ruby-syntax-tree/syntax_tree/compare/v2.5.0...v2.6.0 diff --git a/Gemfile.lock b/Gemfile.lock index d707afd5..92ad8d38 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - syntax_tree (2.7.1) + syntax_tree (2.8.0) prettier_print GEM @@ -9,25 +9,25 @@ GEM specs: ast (2.4.2) docile (1.4.0) - minitest (5.15.0) + minitest (5.16.0) parallel (1.22.1) parser (3.1.2.0) ast (~> 2.4.1) prettier_print (0.1.0) rainbow (3.1.1) rake (13.0.6) - regexp_parser (2.4.0) + regexp_parser (2.5.0) rexml (3.2.5) - rubocop (1.29.1) + rubocop (1.30.1) parallel (~> 1.10) parser (>= 3.1.0.0) rainbow (>= 2.2.2, < 4.0) regexp_parser (>= 1.8, < 3.0) rexml (>= 3.2.5, < 4.0) - rubocop-ast (>= 1.17.0, < 2.0) + rubocop-ast (>= 1.18.0, < 2.0) ruby-progressbar (~> 1.7) unicode-display_width (>= 1.4.0, < 3.0) - rubocop-ast (1.17.0) + rubocop-ast (1.18.0) parser (>= 3.1.1.0) ruby-progressbar (1.11.0) simplecov (0.21.2) diff --git a/lib/syntax_tree.rb b/lib/syntax_tree.rb index 60979d04..1dbd3ac8 100644 --- a/lib/syntax_tree.rb +++ b/lib/syntax_tree.rb @@ -1,5 +1,6 @@ # frozen_string_literal: true +require "etc" require "json" require "pp" require "prettier_print" diff --git a/lib/syntax_tree/basic_visitor.rb b/lib/syntax_tree/basic_visitor.rb index 1ad6a80f..9e6a84c1 100644 --- a/lib/syntax_tree/basic_visitor.rb +++ b/lib/syntax_tree/basic_visitor.rb @@ -33,7 +33,11 @@ def corrections ).correct(visit_method) end - DidYouMean.correct_error(VisitMethodError, self) + # In some setups with Ruby you can turn off DidYouMean, so we're going to + # respect that setting here. + if defined?(DidYouMean) && DidYouMean.method_defined?(:correct_error) + DidYouMean.correct_error(VisitMethodError, self) + end end class << self diff --git a/lib/syntax_tree/cli.rb b/lib/syntax_tree/cli.rb index 64848ca6..a7c6a684 100644 --- a/lib/syntax_tree/cli.rb +++ b/lib/syntax_tree/cli.rb @@ -34,9 +34,41 @@ def self.yellow(value) end end + # An item of work that corresponds to a file to be processed. + class FileItem + attr_reader :filepath + + def initialize(filepath) + @filepath = filepath + end + + def handler + HANDLERS[File.extname(filepath)] + end + + def source + handler.read(filepath) + end + end + + # An item of work that corresponds to the stdin content. + class STDINItem + def handler + HANDLERS[".rb"] + end + + def filepath + :stdin + end + + def source + $stdin.read + end + end + # The parent action class for the CLI that implements the basics. class Action - def run(handler, filepath, source) + def run(item) end def success @@ -48,8 +80,8 @@ def failure # An action of the CLI that prints out the AST for the given source. class AST < Action - def run(handler, _filepath, source) - pp handler.parse(source) + def run(item) + pp item.handler.parse(item.source) end end @@ -59,10 +91,11 @@ class Check < Action class UnformattedError < StandardError end - def run(handler, filepath, source) - raise UnformattedError if source != handler.format(source) + def run(item) + source = item.source + raise UnformattedError if source != item.handler.format(source) rescue StandardError - warn("[#{Color.yellow("warn")}] #{filepath}") + warn("[#{Color.yellow("warn")}] #{item.filepath}") raise end @@ -81,9 +114,11 @@ class Debug < Action class NonIdempotentFormatError < StandardError end - def run(handler, filepath, source) - warning = "[#{Color.yellow("warn")}] #{filepath}" - formatted = handler.format(source) + def run(item) + handler = item.handler + + warning = "[#{Color.yellow("warn")}] #{item.filepath}" + formatted = handler.format(item.source) raise NonIdempotentFormatError if formatted != handler.format(formatted) rescue StandardError @@ -102,25 +137,27 @@ def failure # An action of the CLI that prints out the doc tree IR for the given source. class Doc < Action - def run(handler, _filepath, source) + def run(item) + source = item.source + formatter = Formatter.new(source, []) - handler.parse(source).format(formatter) + item.handler.parse(source).format(formatter) pp formatter.groups.first end end # An action of the CLI that formats the input source and prints it out. class Format < Action - def run(handler, _filepath, source) - puts handler.format(source) + def run(item) + puts item.handler.format(item.source) end end # An action of the CLI that converts the source into its equivalent JSON # representation. class Json < Action - def run(handler, _filepath, source) - object = Visitor::JSONVisitor.new.visit(handler.parse(source)) + def run(item) + object = Visitor::JSONVisitor.new.visit(item.handler.parse(item.source)) puts JSON.pretty_generate(object) end end @@ -128,27 +165,28 @@ def run(handler, _filepath, source) # An action of the CLI that outputs a pattern-matching Ruby expression that # would match the input given. class Match < Action - def run(handler, _filepath, source) - puts handler.parse(source).construct_keys + def run(item) + puts item.handler.parse(item.source).construct_keys end end # An action of the CLI that formats the input source and writes the # formatted output back to the file. class Write < Action - def run(handler, filepath, source) - print filepath + def run(item) + filepath = item.filepath start = Time.now - formatted = handler.format(source) + source = item.source + formatted = item.handler.format(source) File.write(filepath, formatted) if filepath != :stdin color = source == formatted ? Color.gray(filepath) : filepath delta = ((Time.now - start) * 1000).round - puts "\r#{color} #{delta}ms" + puts "#{color} #{delta}ms" rescue StandardError - puts "\r#{filepath}" + puts filepath raise end end @@ -180,7 +218,7 @@ def run(handler, filepath, source) #{Color.bold("stree help")} Display this help message - #{Color.bold("stree lsp")} + #{Color.bold("stree lsp [OPTIONS]")} Run syntax tree in language server mode #{Color.bold("stree version")} @@ -201,6 +239,20 @@ class << self def run(argv) name, *arguments = argv + # If there are any plugins specified on the command line, then load them + # by requiring them here. We do this by transforming something like + # + # stree format --plugins=haml template.haml + # + # into + # + # require "syntax_tree/haml" + # + if arguments.first&.start_with?("--plugins=") + plugins = arguments.shift[/^--plugins=(.*)$/, 1] + plugins.split(",").each { |plugin| require "syntax_tree/#{plugin}" } + end + case name when "help" puts HELP @@ -244,38 +296,41 @@ def run(argv) return 1 end - # If there are any plugins specified on the command line, then load them - # by requiring them here. We do this by transforming something like - # - # stree format --plugins=haml template.haml - # - # into - # - # require "syntax_tree/haml" - # - if arguments.first&.start_with?("--plugins=") - plugins = arguments.shift[/^--plugins=(.*)$/, 1] - plugins.split(",").each { |plugin| require "syntax_tree/#{plugin}" } - end + # We're going to build up a queue of items to process. + queue = Queue.new - # Track whether or not there are any errors from any of the files that - # we take action on so that we can properly clean up and exit. - errored = false - - each_file(arguments) do |handler, filepath, source| - action.run(handler, filepath, source) - rescue Parser::ParseError => error - warn("Error: #{error.message}") - highlight_error(error, source) - errored = true - rescue Check::UnformattedError, Debug::NonIdempotentFormatError - errored = true - rescue StandardError => error - warn(error.message) - warn(error.backtrace) - errored = true + # If we're reading from stdin, then we'll just add the stdin object to + # the queue. Otherwise, we'll add each of the filepaths to the queue. + if $stdin.tty? || arguments.any? + arguments.each do |pattern| + Dir + .glob(pattern) + .each do |filepath| + queue << FileItem.new(filepath) if File.file?(filepath) + end + end + else + queue << STDINItem.new end + # At the end, we're going to return whether or not this worker ever + # encountered an error. + errored = + with_workers(queue) do |item| + action.run(item) + false + rescue Parser::ParseError => error + warn("Error: #{error.message}") + highlight_error(error, item.source) + true + rescue Check::UnformattedError, Debug::NonIdempotentFormatError + true + rescue StandardError => error + warn(error.message) + warn(error.backtrace) + true + end + if errored action.failure 1 @@ -287,22 +342,33 @@ def run(argv) private - def each_file(arguments) - if $stdin.tty? || arguments.any? - arguments.each do |pattern| - Dir - .glob(pattern) - .each do |filepath| - next unless File.file?(filepath) - - handler = HANDLERS[File.extname(filepath)] - source = handler.read(filepath) - yield handler, filepath, source - end + def with_workers(queue) + # If the queue is just 1 item, then we're not going to bother going + # through the whole ceremony of parallelizing the work. + return yield queue.shift if queue.size == 1 + + workers = + Etc.nprocessors.times.map do + Thread.new do + # Propagate errors in the worker threads up to the parent thread. + Thread.current.abort_on_exception = true + + # Track whether or not there are any errors from any of the files + # that we take action on so that we can properly clean up and + # exit. + errored = false + + # While there is still work left to do, shift off the queue and + # process the item. + (errored ||= yield queue.shift) until queue.empty? + + # At the end, we're going to return whether or not this worker + # ever encountered an error. + errored + end end - else - yield HANDLERS[".rb"], :stdin, $stdin.read - end + + workers.inject(false) { |accum, thread| accum || thread.value } end # Highlights a snippet from a source and parse error. diff --git a/lib/syntax_tree/language_server.rb b/lib/syntax_tree/language_server.rb index 3853ee18..31c91f9c 100644 --- a/lib/syntax_tree/language_server.rb +++ b/lib/syntax_tree/language_server.rb @@ -36,8 +36,9 @@ def run write(id: id, result: { capabilities: capabilities }) in method: "initialized" # ignored - in method: "shutdown" + in method: "shutdown" # tolerate missing ID to be a good citizen store.clear + write(id: request[:id], result: {}) return in { method: "textDocument/didChange", diff --git a/lib/syntax_tree/node.rb b/lib/syntax_tree/node.rb index 85956c8c..58335b00 100644 --- a/lib/syntax_tree/node.rb +++ b/lib/syntax_tree/node.rb @@ -4813,7 +4813,7 @@ class Heredoc < Node # [HeredocBeg] the opening of the heredoc attr_reader :beginning - # [String] the ending of the heredoc + # [HeredocEnd] the ending of the heredoc attr_reader :ending # [Integer] how far to dedent the heredoc @@ -4847,7 +4847,7 @@ def accept(visitor) end def child_nodes - [beginning, *parts] + [beginning, *parts, ending] end alias deconstruct child_nodes @@ -4883,7 +4883,7 @@ def format(q) end end - q.text(ending) + q.format(ending) end end end @@ -4929,6 +4929,45 @@ def format(q) end end + # HeredocEnd represents the closing declaration of a heredoc. + # + # <<~DOC + # contents + # DOC + # + # In the example above the HeredocEnd node represents the closing DOC. + class HeredocEnd < Node + # [String] the closing declaration of the heredoc + attr_reader :value + + # [Array[ Comment | EmbDoc ]] the comments attached to this node + attr_reader :comments + + def initialize(value:, location:, comments: []) + @value = value + @location = location + @comments = comments + end + + def accept(visitor) + visitor.visit_heredoc_end(self) + end + + def child_nodes + [] + end + + alias deconstruct child_nodes + + def deconstruct_keys(_keys) + { value: value, location: location, comments: comments } + end + + def format(q) + q.text(value) + end + end + # HshPtn represents matching against a hash pattern using the Ruby 2.7+ # pattern matching syntax. # diff --git a/lib/syntax_tree/parser.rb b/lib/syntax_tree/parser.rb index fdffbeb9..0f8332b1 100644 --- a/lib/syntax_tree/parser.rb +++ b/lib/syntax_tree/parser.rb @@ -1640,9 +1640,19 @@ def on_heredoc_dedent(string, width) def on_heredoc_end(value) heredoc = @heredocs[-1] + location = + Location.token( + line: lineno, + char: char_pos, + column: current_column, + size: value.size + 1 + ) + + heredoc_end = HeredocEnd.new(value: value.chomp, location: location) + @heredocs[-1] = Heredoc.new( beginning: heredoc.beginning, - ending: value.chomp, + ending: heredoc_end, dedent: heredoc.dedent, parts: heredoc.parts, location: diff --git a/lib/syntax_tree/version.rb b/lib/syntax_tree/version.rb index 7754cf7a..881c65aa 100644 --- a/lib/syntax_tree/version.rb +++ b/lib/syntax_tree/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module SyntaxTree - VERSION = "2.7.1" + VERSION = "2.8.0" end diff --git a/lib/syntax_tree/visitor.rb b/lib/syntax_tree/visitor.rb index 348a05a2..e3b52077 100644 --- a/lib/syntax_tree/visitor.rb +++ b/lib/syntax_tree/visitor.rb @@ -194,6 +194,9 @@ class Visitor < BasicVisitor # Visit a HeredocBeg node. alias visit_heredoc_beg visit_child_nodes + # Visit a HeredocEnd node. + alias visit_heredoc_end visit_child_nodes + # Visit a HshPtn node. alias visit_hshptn visit_child_nodes diff --git a/lib/syntax_tree/visitor/field_visitor.rb b/lib/syntax_tree/visitor/field_visitor.rb index 1cc74f3d..6c5c6139 100644 --- a/lib/syntax_tree/visitor/field_visitor.rb +++ b/lib/syntax_tree/visitor/field_visitor.rb @@ -497,6 +497,10 @@ def visit_heredoc_beg(node) visit_token(node, "heredoc_beg") end + def visit_heredoc_end(node) + visit_token(node, "heredoc_end") + end + def visit_hshptn(node) node(node, "hshptn") do field("constant", node.constant) if node.constant diff --git a/test/language_server_test.rb b/test/language_server_test.rb index f8a61003..519bada3 100644 --- a/test/language_server_test.rb +++ b/test/language_server_test.rb @@ -11,9 +11,9 @@ def to_hash end end - class Shutdown + class Shutdown < Struct.new(:id) def to_hash - { method: "shutdown" } + { method: "shutdown", id: id } end end @@ -107,13 +107,14 @@ def test_formatting TextDocumentDidChange.new("file:///path/to/file.rb", "class Bar; end"), TextDocumentFormatting.new(2, "file:///path/to/file.rb"), TextDocumentDidClose.new("file:///path/to/file.rb"), - Shutdown.new + Shutdown.new(3) ] case run_server(messages) in [ { id: 1, result: { capabilities: Hash } }, - { id: 2, result: [{ newText: new_text }] } + { id: 2, result: [{ newText: new_text }] }, + { id: 3, result: {} } ] assert_equal("class Bar\nend\n", new_text) end @@ -129,13 +130,14 @@ def test_inlay_hints end RUBY TextDocumentInlayHints.new(2, "file:///path/to/file.rb"), - Shutdown.new + Shutdown.new(3) ] case run_server(messages) in [ { id: 1, result: { capabilities: Hash } }, - { id: 2, result: { before:, after: } } + { id: 2, result: { before:, after: } }, + { id: 3, result: {} } ] assert_equal(1, before.length) assert_equal(2, after.length) @@ -147,11 +149,15 @@ def test_visualizing Initialize.new(1), TextDocumentDidOpen.new("file:///path/to/file.rb", "1 + 2"), SyntaxTreeVisualizing.new(2, "file:///path/to/file.rb"), - Shutdown.new + Shutdown.new(3) ] case run_server(messages) - in [{ id: 1, result: { capabilities: Hash } }, { id: 2, result: }] + in [ + { id: 1, result: { capabilities: Hash } }, + { id: 2, result: }, + { id: 3, result: {} } + ] assert_equal( "(program (statements ((binary (int \"1\") + (int \"2\")))))\n", result @@ -167,13 +173,14 @@ def test_reading_file messages = [ Initialize.new(1), TextDocumentFormatting.new(2, "file://#{file.path}"), - Shutdown.new + Shutdown.new(3) ] case run_server(messages) in [ { id: 1, result: { capabilities: Hash } }, - { id: 2, result: [{ newText: new_text }] } + { id: 2, result: [{ newText: new_text }] }, + { id: 3, result: {} } ] assert_equal("class Foo\nend\n", new_text) end @@ -186,6 +193,15 @@ def test_bogus_request end end + def test_clean_shutdown + messages = [Initialize.new(1), Shutdown.new(2)] + + case run_server(messages) + in [{ id: 1, result: { capabilities: Hash } }, { id: 2, result: {} }] + assert_equal(true, true) + end + end + private def write(content) diff --git a/test/node_test.rb b/test/node_test.rb index ffd00fa5..30776f9d 100644 --- a/test/node_test.rb +++ b/test/node_test.rb @@ -546,6 +546,17 @@ def test_heredoc_beg assert_node(HeredocBeg, source, at: at, &:beginning) end + def test_heredoc_end + source = <<~SOURCE + <<~HEREDOC + contents + HEREDOC + SOURCE + + at = location(lines: 3..3, chars: 22..31, columns: 0..9) + assert_node(HeredocEnd, source, at: at, &:ending) + end + def test_hshptn source = <<~SOURCE case value diff --git a/test/visitor_test.rb b/test/visitor_test.rb index 5e4f134d..27bad364 100644 --- a/test/visitor_test.rb +++ b/test/visitor_test.rb @@ -40,9 +40,18 @@ def initialize end end - def test_visit_method_correction - error = assert_raises { Visitor.visit_method(:visit_binar) } - assert_match(/visit_binary/, error.message) + if defined?(DidYouMean) && DidYouMean.method_defined?(:correct_error) + def test_visit_method_correction + error = assert_raises { Visitor.visit_method(:visit_binar) } + message = + if Exception.method_defined?(:detailed_message) + error.detailed_message + else + error.message + end + + assert_match(/visit_binary/, message) + end end end end