Updated importer, schema, classes - warvox - VoIP based wardialing tool, forked from rapid7/warvox.
DIR Log
DIR Files
DIR Refs
DIR README
---
DIR commit 7bcac597dbf94b510242581bb21f8d9ddae4a514
DIR parent 74be216401da43b3f2234fdcf38c2a07da7c9108
HTML Author: HD Moore <hd_moore@rapid7.com>
Date: Mon, 31 Dec 2012 13:54:05 -0600
Updated importer, schema, classes
Diffstat:
D app/models/dial_result.rb | 36 -------------------------------
D app/models/dial_result_medium.rb | 3 ---
M app/models/job.rb | 8 ++++++++
M app/models/project.rb | 4 +++-
M bin/import_audio.rb | 122 ++++++++++++++++++++-----------
M db/migrate/20121228171549_initial_… | 109 +++++++++++++++++++++----------
M db/schema.rb | 112 ++++++++++++++++++++-----------
7 files changed, 239 insertions(+), 155 deletions(-)
---
DIR diff --git a/app/models/dial_result.rb b/app/models/dial_result.rb
@@ -1,36 +0,0 @@
-class DialResult < ActiveRecord::Base
- belongs_to :provider
- belongs_to :dial_job
- has_one :dial_result_medium, :dependent => :delete
-
- has_many :matches, :class_name => 'DialResult', :finder_sql => proc {
- 'SELECT dial_results.*, ' +
- " (( icount(\'{#{fprint.map{|x| x.to_s}.join(",")}}\'::int[] & dial_results.fprint::int[]) / icount(\'{#{fprint.map{|x| x.to_s}.join(",")}}'::int[])::float ) * 100.0 ) AS matchscore " +
- 'FROM dial_results ' +
- 'WHERE icount(dial_results.fprint) > 0 AND ' +
- "dial_results.dial_job_id = \'#{dial_job_id}\' AND " +
- "dial_results.id != \'#{id}\' " +
-# "AND (( icount(\'{#{fprint.map{|x| x.to_s}.join(",")}}\'::int[] & dial_results.fprint::int[]) / icount(\'{#{fprint.map{|x| x.to_s}.join(",")}}'::int[])::float ) * 100.0 ) > 10.0 " +
- 'ORDER BY matchscore DESC'
- }
-
- has_many :matches_all_jobs, :class_name => 'DialResult', :finder_sql => proc {
- 'SELECT dial_results.*, ' +
- " (( icount(\'{#{fprint.map{|x| x.to_s}.join(",")}}\'::int[] & dial_results.fprint::int[]) / icount(\'{#{fprint.map{|x| x.to_s}.join(",")}}\'::int[])::float ) * 100.0 ) AS matchscore " +
- 'FROM dial_results ' +
- 'WHERE icount(dial_results.fprint) > 0 AND ' +
- "dial_results.id != \'#{id}\' " +
-# "AND (( icount(\'{#{fprint.map{|x| x.to_s}.join(",")}}\'::int[] & dial_results.fprint::int[]) / icount(\'{#{fprint.map{|x| x.to_s}.join(",")}}'::int[])::float ) * 100.0 ) > 10.0 " +
- 'ORDER BY matchscore DESC'
- }
-
-
- def media
- DialResultMedium.find_or_create_by_dial_result_id(self[:id])
- end
-
- def media_fields
- DialResultMedium.columns_hash.keys.reject{|x| x =~ /^id|_id$/}
- end
-
-end
DIR diff --git a/app/models/dial_result_medium.rb b/app/models/dial_result_medium.rb
@@ -1,3 +0,0 @@
-class DialResultMedium < ActiveRecord::Base
- belongs_to :dial_result
-end
DIR diff --git a/app/models/job.rb b/app/models/job.rb
@@ -1,4 +1,12 @@
class Job < ActiveRecord::Base
has_many :calls
belongs_to :project
+
+ def update_progress(pct)
+ if pct >= 100
+ self.class.update_all({ :progress => pct, :completed_at => Time.now.utc }, { :id => self.id })
+ else
+ self.class.update_all({ :progress => pct }, { :id => self.id })
+ end
+ end
end
DIR diff --git a/app/models/project.rb b/app/models/project.rb
@@ -3,5 +3,7 @@ class Project < ActiveRecord::Base
attr_accessible :name, :description, :included, :excluded
- has_many :lines, :calls, :jobs
+ has_many :lines
+ has_many :calls
+ has_many :jobs
end
DIR diff --git a/bin/import_audio.rb b/bin/import_audio.rb
@@ -21,7 +21,7 @@ require 'config/boot'
require 'config/environment'
def usage
- $stderr.puts "Usage: #{$0} [Input Dir] <Job ID>"
+ $stderr.puts "Usage: #{$0} [Input Directory] <Project ID> <Provider ID>"
exit(1)
end
@@ -29,59 +29,89 @@ end
# Script
#
-dir = ARGV.shift
-jid = ARGV.shift
-
+dir = ARGV.shift() || usage()
if (dir and dir =="-h") or (! dir)
usage()
end
+
+project_id = ARGV.shift
+provider_id = ARGV.shift
+
+todo = Dir["#{dir}/**/*.raw"].to_a
+
+if todo.empty?
+ $stderr.puts "Error: No raw audio files found within #{dir}"
+ exit(1)
+end
+
+project = nil
+provider = nil
+
+if project_id
+ project = Project.find(pid)
+ unless project
+ $stderr.puts "Error: Specified Project ID not found"
+ exit(1)
+ end
+end
+
+if provider_id
+ provider = Provider.find(pid)
+ unless provider
+ $stderr.puts "Error: Specified Provider ID not found"
+ exit(1)
+ end
+end
+
+unless project
+ project = Project.create(
+ :name => "Import from #{dir}",
+ :created_by => "importer"
+ )
+end
+
provider = Provider.first
unless provider
provider = Provider.create(
- :name => 'Import Provider',
- :host => 'localhost',
- :port => 4369,
- :user => "null",
- :pass => "null",
- :lines => 1,
+ :name => 'Import Provider',
+ :host => 'localhost',
+ :port => 4369,
+ :user => "null",
+ :pass => "null",
+ :lines => 1,
:enabled => false
)
end
-job = nil
-if jid
- job = DialJob.find(jid.to_i)
- unless job
- $stderr.puts "Error: Specified Job ID not found"
- exit(1)
- end
-else
- job = DialJob.new
- job.range = "IMPORTED"
- job.seconds = 60
- job.lines = 1
- job.cid_mask = "XXXXX"
- job.status = "completed"
- job.progress = 100
- job.started_at = Time.now
- job.completed_at = Time.now
- job.processed = false
- job.save
-end
-Dir["#{dir}/**/*.raw"].each do |rfile|
+job = Job.new
+job.project_id = project.id
+job.name = "Import Audio Job"
+job.locked_by = "importer"
+job.locked_at = Time.now.utc
+job.started_at = Time.now.utc
+job.created_by = "importer"
+job.task = "import"
+job.args = Marshal.dump({ :directory => dir, :project_id => project.id, :provider_id => provider.id })
+job.status = "running"
+job.save
+
+pct = 0
+cnt = 0
+
+todo.each do |rfile|
num, ext = File.basename(rfile).split(".", 2)
- dr = DialResult.new
- dr.dial_job_id = job[:id]
- dr.number = num
- dr.provider_id = provider[:id]
- dr.completed = true
- dr.busy = false
- dr.seconds = File.size(rfile) / 16000.0
- dr.ringtime = 0
- dr.processed = false
- dr.cid = num
+ dr = Call.new
+ dr.number = num
+ dr.job_id = job.id
+ dr.project_id = project.id
+ dr.provider_id = provider.id
+ dr.answered = true
+ dr.busy = false
+ dr.audio_length = File.size(rfile) / 16000.0
+ dr.ring_length = 0
+ dr.caller_id = num
dr.save
mr = dr.media
@@ -90,5 +120,13 @@ Dir["#{dir}/**/*.raw"].each do |rfile|
mr.save
end
- $stdout.puts "[*] Imported #{num}"
+ cnt += 1
+ pct = (cnt / todo.length.to_f) * 100.0
+ if cnt % 10 == 0
+ job.update_progress(pct)
+ end
+
+ $stdout.puts "[ %#{"%.3d" % pct.to_i} ] Imported #{num} into project '#{project.name}' ##{project.id}"
end
+
+job.update_progress(100)
DIR diff --git a/db/migrate/20121228171549_initial_schema.rb b/db/migrate/20121228171549_initial_schema.rb
@@ -4,7 +4,6 @@ class InitialSchema < ActiveRecord::Migration
# Require the intarray extension
execute("CREATE EXTENSION IF NOT EXISTS intarray")
-
create_table :settings do |t|
t.string :var, :null => false
t.text :value, :null => true
@@ -40,49 +39,70 @@ class InitialSchema < ActiveRecord::Migration
create_table 'projects' do |t|
t.timestamps
- t.text "name"
+ t.text "name", :null => false
t.text "description"
t.text "included"
t.text "excluded"
t.string "created_by"
end
- create_table "dial_jobs" do |t|
+ create_table "jobs" do |t|
+ t.timestamps
+ t.integer "project_id", :null => false
+ t.string "name", :null => false
+ t.string "locked_by"
+ t.timestamp "locked_at"
+ t.timestamp "started_at"
+ t.timestamp "completed_at"
+ t.string "created_by"
+ t.string "task", :null => false
+ t.binary "args"
+ t.string "status"
+ t.text "error"
+ t.integer "progress", :default => 0
+ end
+
+ create_table "lines" do |t|
+ t.timestamps
+ t.text "number", :null => false
+ t.integer "project_id", :null => false
+ t.text "type"
+ t.text "notes"
+ end
+
+ create_table "line_attributes" do |t|
t.timestamps
- t.text "range"
- t.integer "seconds"
- t.integer "lines"
- t.text "status"
- t.integer "progress"
- t.datetime "started_at"
- t.datetime "completed_at"
- t.boolean "processed"
- t.text "cid_mask"
+ t.integer "line_id", :null => false
+ t.text "name", :null => false
+ t.binary "value", :null => false
+ t.string "content_type", :default => "text"
end
- create_table "dial_results" do |t|
+ create_table "calls" do |t|
+ # Created by the dialer job
t.timestamps
- t.text "number"
- t.integer "dial_job_id"
- t.integer "provider_id"
- t.boolean "completed"
+ t.text "number", :null => false
+ t.integer "project_id", :null => false
+ t.integer "job_id", :null => false
+ t.integer "provider_id", :null => false
+ t.boolean "answered"
t.boolean "busy"
- t.integer "seconds"
- t.integer "ringtime"
- t.boolean "processed"
- t.datetime "processed_at"
- t.text "cid"
+ t.integer "audio_length"
+ t.integer "ring_length"
+ t.text "caller_id"
+
+ # Generated by the analysis job
+ t.integer "analysis_job_id"
+ t.boolean "analysis_started_at"
+ t.boolean "analysis_completed_at"
t.float "peak_freq"
t.text "peak_freq_data"
- t.text "sig_data"
t.text "line_type"
- t.text "notes"
- t.text "signatures"
t.integer "fprint", :array => true
end
- create_table "dial_result_media" do |t|
- t.integer "dial_result_id"
+ create_table "call_media" do |t|
+ t.integer "call_id", :null => false
t.binary "audio"
t.binary "mp3"
t.binary "png_big"
@@ -92,24 +112,43 @@ class InitialSchema < ActiveRecord::Migration
t.binary "png_sig_freq"
end
+ create_table "signatures" do |t|
+ t.timestamps
+ t.text "name", :null => false
+ t.string "source"
+ t.text "description"
+ t.string "category"
+ t.string "line_type"
+ t.integer "risk"
+ end
+
+ create_table "signature_fp" do |t|
+ t.integer "signature_id", :null => false
+ t.integer "fprint", :array => true
+ end
+
create_table "providers" do |t|
t.timestamps
- t.text "name"
- t.text "host"
- t.integer "port"
+ t.text "name", :null => false
+ t.text "host", :null => false
+ t.integer "port", :null => false
t.text "user"
t.text "pass"
- t.integer "lines"
- t.boolean "enabled"
+ t.integer "lines", :null => false, :default => 1
+ t.boolean "enabled", :default => true
end
end
def down
drop_table "providers"
- drop_table "dial_result_media"
- drop_table "dial_results"
- drop_table "dial_jobs"
+ drop_table "signature_fp"
+ drop_table "signatures"
+ drop_table "call_media"
+ drop_table "calls"
+ drop_table "line_attributes"
+ drop_table "lines"
+ drop_table "jobs"
drop_table "projects"
drop_table "users"
drop_table "settings"
DIR diff --git a/db/schema.rb b/db/schema.rb
@@ -15,22 +15,8 @@ ActiveRecord::Schema.define(:version => 20121228171549) do
add_extension "intarray"
- create_table "dial_jobs", :force => true do |t|
- t.datetime "created_at", :null => false
- t.datetime "updated_at", :null => false
- t.text "range"
- t.integer "seconds"
- t.integer "lines"
- t.text "status"
- t.integer "progress"
- t.datetime "started_at"
- t.datetime "completed_at"
- t.boolean "processed"
- t.text "cid_mask"
- end
-
- create_table "dial_result_media", :force => true do |t|
- t.integer "dial_result_id"
+ create_table "call_media", :force => true do |t|
+ t.integer "call_id", :null => false
t.binary "audio"
t.binary "mp3"
t.binary "png_big"
@@ -40,26 +26,60 @@ ActiveRecord::Schema.define(:version => 20121228171549) do
t.binary "png_sig_freq"
end
- create_table "dial_results", :force => true do |t|
- t.datetime "created_at", :null => false
- t.datetime "updated_at", :null => false
- t.text "number"
- t.integer "dial_job_id"
- t.integer "provider_id"
- t.boolean "completed"
+ create_table "calls", :force => true do |t|
+ t.datetime "created_at", :null => false
+ t.datetime "updated_at", :null => false
+ t.text "number", :null => false
+ t.integer "project_id", :null => false
+ t.integer "job_id", :null => false
+ t.integer "provider_id", :null => false
+ t.boolean "answered"
t.boolean "busy"
- t.integer "seconds"
- t.integer "ringtime"
- t.boolean "processed"
- t.datetime "processed_at"
- t.text "cid"
+ t.integer "audio_length"
+ t.integer "ring_length"
+ t.text "caller_id"
+ t.integer "analysis_job_id"
+ t.boolean "analysis_started_at"
+ t.boolean "analysis_completed_at"
t.float "peak_freq"
t.text "peak_freq_data"
- t.text "sig_data"
t.text "line_type"
+ t.integer "fprint", :array => true
+ end
+
+ create_table "jobs", :force => true do |t|
+ t.datetime "created_at", :null => false
+ t.datetime "updated_at", :null => false
+ t.integer "project_id", :null => false
+ t.string "name", :null => false
+ t.string "locked_by"
+ t.datetime "locked_at"
+ t.datetime "started_at"
+ t.datetime "completed_at"
+ t.string "created_by"
+ t.string "task", :null => false
+ t.binary "args"
+ t.string "status"
+ t.text "error"
+ t.integer "progress", :default => 0
+ end
+
+ create_table "line_attributes", :force => true do |t|
+ t.datetime "created_at", :null => false
+ t.datetime "updated_at", :null => false
+ t.integer "line_id", :null => false
+ t.text "name", :null => false
+ t.binary "value", :null => false
+ t.string "content_type", :default => "text"
+ end
+
+ create_table "lines", :force => true do |t|
+ t.datetime "created_at", :null => false
+ t.datetime "updated_at", :null => false
+ t.text "number", :null => false
+ t.integer "project_id", :null => false
+ t.text "type"
t.text "notes"
- t.text "signatures"
- t.integer "fprint", :array => true
end
create_table "projects", :force => true do |t|
@@ -73,15 +93,15 @@ ActiveRecord::Schema.define(:version => 20121228171549) do
end
create_table "providers", :force => true do |t|
- t.datetime "created_at", :null => false
- t.datetime "updated_at", :null => false
- t.text "name"
- t.text "host"
- t.integer "port"
+ t.datetime "created_at", :null => false
+ t.datetime "updated_at", :null => false
+ t.text "name", :null => false
+ t.text "host", :null => false
+ t.integer "port", :null => false
t.text "user"
t.text "pass"
- t.integer "lines"
- t.boolean "enabled"
+ t.integer "lines", :default => 1, :null => false
+ t.boolean "enabled", :default => true
end
create_table "settings", :force => true do |t|
@@ -95,6 +115,22 @@ ActiveRecord::Schema.define(:version => 20121228171549) do
add_index "settings", ["thing_type", "thing_id", "var"], :name => "index_settings_on_thing_type_and_thing_id_and_var", :unique => true
+ create_table "signature_fp", :force => true do |t|
+ t.integer "signature_id", :null => false
+ t.integer "fprint", :array => true
+ end
+
+ create_table "signatures", :force => true do |t|
+ t.datetime "created_at", :null => false
+ t.datetime "updated_at", :null => false
+ t.text "name", :null => false
+ t.string "source"
+ t.text "description"
+ t.string "category"
+ t.string "line_type"
+ t.integer "risk"
+ end
+
create_table "users", :force => true do |t|
t.string "login", :null => false
t.string "email"