Skip to content

Commit

Permalink
Make @meta[:files] a hash again.
Browse files Browse the repository at this point in the history
This is more convenient, and we no longer run into weird symbol/string issues
now that we aren't forcing all hash-keys to symbols when loading the JSON.

Also more strictly validate the DB in some tests.
  • Loading branch information
eapache committed Apr 13, 2014
1 parent 8570cfc commit 1dfffb2
Show file tree
Hide file tree
Showing 2 changed files with 38 additions and 33 deletions.
43 changes: 23 additions & 20 deletions lib/starscope/db.rb
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ class UnknownDBFormatError < StandardError; end

def initialize(progress, verbose)
@output = StarScope::Output.new(progress, verbose)
@meta = {:paths => [], :files => [], :excludes => []}
@meta = {:paths => [], :files => {}, :excludes => []}
@tables = {}
end

Expand Down Expand Up @@ -73,9 +73,9 @@ def add_excludes(paths)
paths = paths.map {|p| normalize_fnmatch(p)}
@meta[:excludes] += paths
@meta[:excludes].uniq!
@meta[:files].delete_if do |f|
if matches_exclude(paths, f[:name])
remove_file(f)
@meta[:files].delete_if do |name, record|
if matches_exclude(paths, name)
remove_file(name)
true
else
false
Expand All @@ -97,13 +97,13 @@ def add_paths(paths)
end

def update
new_files = (Dir.glob(@meta[:paths]).select {|f| File.file? f}) - @meta[:files].map {|f| f[:name]}
new_files = (Dir.glob(@meta[:paths]).select {|f| File.file? f}) - @meta[:files].keys
new_files.delete_if {|f| matches_exclude(@meta[:excludes], f)}
@output.new_pbar("Updating", new_files.length + @meta[:files].length)
changed = false
@meta[:files].delete_if do |f|
@output.log("Updating `#{f[:name]}`")
ret = update_file(f)
@meta[:files].delete_if do |name, record|
@output.log("Updating `#{name}`")
ret = update_file(name)
@output.inc_pbar
changed = true if ret == :update
ret == :delete
Expand Down Expand Up @@ -133,7 +133,11 @@ def dump_meta(key)
end
raise NoTableError if not @meta[key]
puts "== Metadata: #{key} =="
@meta[key].each {|x| puts x}
if @meta[key].is_a? Array
@meta[key].sort.each {|x| puts x}
else
@meta[key].sort.each {|k,v| puts "#{k}: #{v}"}
end
end

def dump_all
Expand Down Expand Up @@ -233,9 +237,8 @@ def export_cscope(filename)
def add_new_files(files)
files.each do |file|
@output.log("Adding `#{file}`")
record = {:name => file}
parse_file(record)
@meta[:files] << record
@meta[:files][file] = {}
parse_file(file)
@output.inc_pbar
end
end
Expand Down Expand Up @@ -282,29 +285,29 @@ def matches_exclude(patterns, file)
end

def parse_file(file)
file[:last_updated] = File.mtime(file[:name]).to_i
@meta[:files][file][:last_updated] = File.mtime(file).to_i

This comment has been minimized.

Copy link
@guilherme

guilherme Oct 18, 2015

i'm taking a look on the way starscope load files. i found a detail that might have something: it is really needed to store meta data for files that aren't used? for e.g: if a file don't match the LANGS file matchers (in this commit, but the current name is EXTRACTORS), it won't be loaded within the database, so, still worth it to save the last updated?

This comment has been minimized.

Copy link
@guilherme

guilherme Oct 18, 2015

if don't this meta could be added after the matching of the files, below the line:

next if not lang.match_file file

This comment has been minimized.

Copy link
@eapache

eapache Oct 19, 2015

Author Owner

Hmm, that's possible. This code is somewhat different in the latest version, but the same general comment might still apply.

I guess the interesting case is if you have a lot of files that don't match, since if they're not in the database they'll run through all the matchers again. So is it faster to re-match them every time, or to load that extra data from the database?

Benchmarking would be welcome here :)

This comment has been minimized.

Copy link
@eapache

eapache Oct 19, 2015

Author Owner

Related ideas I've had include #95 and #142


LANGS.each do |lang|
next if not lang.match_file file[:name]
lang.extract file[:name] do |tbl, name, args|
next if not lang.match_file file
lang.extract file do |tbl, name, args|
@tables[tbl] ||= []
@tables[tbl] << StarScope::Record.build(file[:name], name, args)
@tables[tbl] << StarScope::Record.build(file, name, args)
end
file[:lang] = lang.name.split('::').last.to_sym
@meta[:files][file][:lang] = lang.name.split('::').last.to_sym
end
end

def remove_file(file)
@tables.each do |name, tbl|
tbl.delete_if {|val| val[:file] == file[:name]}
tbl.delete_if {|val| val[:file] == file}
end
end

def update_file(file)
if not File.exists?(file[:name]) or not File.file?(file[:name])
if not File.exists?(file) or not File.file?(file)
remove_file(file)
:delete
elsif file[:last_updated] < File.mtime(file[:name]).to_i
elsif @meta[:files][file][:last_updated] < File.mtime(file).to_i
remove_file(file)
parse_file(file)
:update
Expand Down
28 changes: 15 additions & 13 deletions test/lib/test_db.rb
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,14 @@

describe StarScope::DB do

def validate_db
files = @db.instance_eval('@meta[:files]')
files.keys.must_include GOLANG_SAMPLE
files.keys.must_include RUBY_SAMPLE
files[GOLANG_SAMPLE][:last_updated].must_equal File.mtime(GOLANG_SAMPLE).to_i
files[RUBY_SAMPLE][:last_updated].must_equal File.mtime(RUBY_SAMPLE).to_i
end

before do
@db = StarScope::DB.new(false, false)
end
Expand All @@ -15,33 +23,27 @@
paths = [GOLANG_SAMPLE, 'test/files/**/*']
@db.add_paths(paths)
@db.instance_eval('@meta[:paths]').must_equal paths
files = @db.instance_eval('@meta[:files]').map{|x|x[:name]}
files.must_include GOLANG_SAMPLE
files.must_include RUBY_SAMPLE
validate_db
end

it "must correctly pick up new files in old paths" do
@db.instance_eval('@meta[:paths] = ["test/files/**/*"]')
@db.update
files = @db.instance_eval('@meta[:files]').map{|x|x[:name]}
files.must_include GOLANG_SAMPLE
files.must_include RUBY_SAMPLE
validate_db
end

it "must correctly remove old files in existing paths" do
@db.instance_eval('@meta[:paths] = ["test/files"]')
@db.instance_eval('@meta[:files] = [{:name=>"test/files/foo", :last_update=>1}]')
@db.instance_eval('@meta[:files]').map{|x|x[:name]}.must_include 'test/files/foo'
@db.instance_eval('@meta[:files] = {"test/files/foo" => {:last_update=>1}}')
@db.instance_eval('@meta[:files]').keys.must_include 'test/files/foo'
@db.update
@db.instance_eval('@meta[:files]').map{|x|x[:name]}.wont_include 'test/files/foo'
@db.instance_eval('@meta[:files]').keys.wont_include 'test/files/foo'
end

it "must correctly load an old DB file" do
@db.load('test/files/db_old.json.gz')
@db.instance_eval('@meta[:paths]').must_equal ['test/files/**/*']
files = @db.instance_eval('@meta[:files]').map{|x|x[:name]}
files.must_include GOLANG_SAMPLE
files.must_include RUBY_SAMPLE
validate_db
end

it "must correctly round-trip a database" do
Expand All @@ -60,7 +62,7 @@
tbls = tmp.instance_eval('@tables')

meta[:paths].must_equal ['test/files/**/*']
files = meta[:files].map {|x| x[:name]}
files = meta[:files].keys
files.must_include GOLANG_SAMPLE
files.must_include RUBY_SAMPLE

Expand Down

0 comments on commit 1dfffb2

Please sign in to comment.