We Ain't Seen Nothin' Yet.

idf = Math.log10(all_words_in_all_page / (wc ? wc.count + count : count))

idf = Math.log( 全文書数 / 単語tの出現する文書数 )

require 'connect'

ActiveRecord::Schema.define do |s|
  s.verbose = false
  create_table 'documents', :force => true do |t|
    t.column :title, :string
    t.column :uri, :string, :limit => 1024
    t.column :body, :text
  end
  add_index :documents, :uri

  create_table 'words', :force => true do |t|
    t.column :name, :string
    t.column :df, :integer, :default => 0
  end
  add_index :words, :name

  create_table 'showups', :force => true do |t|
    t.column :document_id, :integer
    t.column :word_id, :integer
    t.column :tf, :integer, :default => 0
  end
  add_index :showups, :document_id
  add_index :showups, :word_id
end

require 'rubygems'
require 'active_record'

ActiveRecord::Base.establish_connection(
  :adapter => 'sqlite3',
  :database => 'tfidf.db',
  :timeout => 5000
  )

class Document < ActiveRecord::Base
  has_many :showups, :dependent => :destroy
  has_many :words, :through => :showups
end

class Word < ActiveRecord::Base
  has_many :showups, :dependent => :destroy
  has_many :documents, :through => :showups
end

class Showup < ActiveRecord::Base
  belongs_to :document
  belongs_to :word, :counter_cache => 'df'
end

$KCODE = 'u'
require 'connect'
load 'tables.rb'

require 'ma'
ma = POS::Tagger.new

Dir.glob('docs/**/*').each do |filename|
  next unless FileTest.file?(filename)
  print "importing '#{filename}' ... "
  $stdout.flush
  if Document.find_by_uri(filename)
    puts "skip."
    next
  end
  fr = File.open(filename) do |f|
    doc = ma.prepare(f)
    document = Document.create(
      :uri => filename,
      :title => doc[:title],
      :body => doc[:body]
      )
    if doc[:body].blank?
      puts "skip."
      break
    end
    begin
      words = ma.parse(doc[:body])
    rescue Timeout::Error
      puts "Timeout::Error."
      break
    end
    words.each do |word|
      w = Word.find_or_create_by_name(word.surface)
      begin
        Showup.create(
          :document => document,
          :word => w,
          :tf => word.count
          )
      rescue
        next
      end
    end
  end
  puts "done." if fr
end

require 'net/http'
Net::HTTP.version_1_2
require 'kconv'
require 'hpricot'
require 'net/yjws'

class Net::YJWS::MAService
  def execute
    uri = BASE_URI.dup
    uri.query = to_query
    uniq_result = nil
    Net::HTTP.start(uri.host) do |http|
      response = http.post(uri.path, uri.query)
      src = response.body
      xml = REXML::Document.new(src)
      REXML::XPath.match(xml, "/ResultSet/uniq_result").each {|r|
        uniq_result = Result.new
        uniq_result.total_count = REXML::XPath.first(r, "total_count").text.to_i
        uniq_result.word_list = []
        REXML::XPath.match(r, "word_list/word").each {|w|
          word = Word.new
          if surface = REXML::XPath.first(w, "surface")
            word.surface = surface.text
          end
          if count = REXML::XPath.first(w, "count")
            word.count = count.text.to_i
          end
          uniq_result.word_list << word
        }
      } # uniq_result
    end
    { :uniq_result=> uniq_result }
  end
end

module POS
  class Tagger
    def initialize
      @yjws = Net::YJWS::MAService.new
      @yjws.appid = 'morphological_analysis'
      @yjws.results = 'uniq'
      @yjws.uniq_response = 'surface'
      @yjws.filter = 9
    end

    def prepare(file)
      doc = Hpricot(file.read.toutf8)
      title = doc.at(:title).inner_text rescue "untitled"
      begin
        body = doc.at(:body)
        (body/:script).remove
        (body/:style).remove
        text = doc.inner_text
        text = text.strip.tr_s(" \t\n\r\f\v", ' ')
      rescue
        text = ""
      end
      {:title => title, :body => text}
    end

    def parse(text)
      @yjws.sentence = text
      @yjws.execute[:uniq_result].word_list
    end
  end
end

$ ruby tfidf.rb http://blog.fulltext-search.biz/articles/2007/09/08/tf-idf-by-ruby
end:    75.6907302844764
IDF:    64.1173309637195
rb:     43.6308134484654
単語:   55.7947666917104
require:        48.355464466149
text:   37.8133716553367
エントリ:       11.716287108625
計算:   52.0953314080221
body:   36.3180471866489
count:  52.9535795200876
出現:   48.0879982227897
文書:   44.0806650375572
Ruby:   3.96415272588246
do:     27.0805020110221
uri:    19.2789164355263
TF:     36.0659986670922
（以下省略）

require 'connect'
load 'tables.rb'

require 'ma'
ma = POS::Tagger.new

require 'open-uri'

n = Document.count

open(ARGV.shift) do |io|
  result = ma.prepare(io)
  ma.parse(result[:body]).each do |word|
    next unless w = Word.find_by_name(word.surface)
    idf = Math.log(n / w.df.to_f)
    puts "#{word.surface}:\t#{word.count * idf}"
  end
end