1require 'net/http'
 2
 3module Proxies
 4  Fiddler = Net::HTTP::Proxy('localhost', 8888)
 5end
 6
 7class WebClient
 8  def initialize(proxy = nil, user_agent = 'WebSpider')
 9    @http = if proxy then proxy else Net::HTTP end
10    @user_agent = user_agent
11  end
12
13  def visit(url, params = nil, rest = nil)
14      uri = URI.parse(URI.encode(url))
15      uri.query = URI.encode_www_form(params) if params
16      puts "Get #{uri}"
17      @http.start(uri.host, uri.port) do |http|
18        request = Net::HTTP::Get.new uri.request_uri
19        request["User-Agent"] = @user_agent
20        response = http.request request
21
22        puts "#{response.code}#{response.message}"
23        puts "wait #{sleep rest}" if rest
24        response
25    end
26  end
27end

 1module MongoDoc
 2  module Document
 3    def to_hash
 4      hash = {}
 5      instance_variables.each do |var|
 6        key = var.to_s.delete('@')
 7        hash[key] = instance_variable_get(var)
 8      end
 9      hash
10    end
11  end
12end

 1require_relative 'mongodoc'
 2
 3class Website
 4  attr_accessor :domain, :encoding
 5  def initialize(domain, encoding)
 6    @domain = domain
 7    @encoding = encoding
 8  end
 9end
10
11class Webpage
12  include MongoDoc::Document
13  attr_accessor :url, :data
14  def initialize(url, data)
15    @url = url
16    @data = data
17  end
18end

1require 'mongo'
2
3DbServer = "mongodb://localhost:27017"
4WebsitesDb = Mongo::Connection.from_uri(DbServer).db("Websites")

 1require 'nokogiri'
 2require 'date'
 3require_relative 'webclient'
 4require_relative 'website'
 5require_relative 'database'
 6
 7module Websites
 8  Cnbeta = Website.new('cnbeta.com', Encoding::GBK)
 9  def Cnbeta.page_list(client)
10    latest = get_latest(client).to_i
11    recent = get_recent().to_i
12    if recent == 0 then recent = latest - 200 end
13    for id in recent.upto(latest)
14      yield "http://www.cnbeta.com/articles/#{id}.htm"
15    end
16  end
17
18  def Cnbeta.get_latest(client)
19    response = client.visit("http://www.cnbeta.com/")
20    if response.is_a? Net::HTTPOK
21      text = response.body.force_encoding(encoding)
22      doc = Nokogiri::HTML(text)
23      url = doc.css("div.newslist>dl>dt>a").first["href"]
24      id = url.match(/\/(?<id>\d+)\.htm/)["id"]
25    end
26  end
27
28  def Cnbeta.get_recent()
29    website = WebsitesDb.collection(domain)
30    if webpage = website.find(pubDate:{"$gt" => DateTime.now.prev_day.to_time}).sort(pubDate:"asc").next
31      id = webpage["url"].match(/\/(?<id>\d+)\.htm/)["id"]
32    end
33  end
34
35  def Cnbeta.process_page(client, webpage)
36    puts "process #{webpage["url"]}"
37    id = webpage["url"].match(/\/(?<id>\d+)\.htm/)["id"]
38    text = webpage["data"].to_s.force_encoding(encoding)
39    
40    begin
41    if text =~ /^<meta http-equiv="refresh"/
42      return nil
43    end
44    rescue ArgumentError => error
45      puts error.message
46    end
47
48    doc = Nokogiri::HTML(text)
49    title = doc.css("#news_title").inner_html.encode("utf-8")
50    time = doc.css("#news_author > span").inner_html.encode("utf-8").match(/\u53D1\u5E03\u4E8E (?<time>.*)\|/)["time"]
51
52    time = DateTime.parse(time + " +0800")
53    if time < DateTime.now.prev_day
54      puts "skip"
55      return nil
56    end
57    
58    if g_content = doc.css("#g_content").first
59      comments = client.visit("http://www.cnbeta.com/comment/g_content/#{id}.html", nil, 0.2).body.force_encoding("utf-8")
60      g_content.inner_html = comments.encode(encoding)
61    end
62
63    if normal = doc.css("#normal").first
64      comments = client.visit("http://www.cnbeta.com/comment/normal/#{id}.html", nil, 0.2).body.force_encoding("utf-8")
65      normal.inner_html = comments.encode(encoding)
66    end
67
68    webpage["title"] = title
69    webpage["pubDate"] = time.to_time
70    if g_content and normal
71      webpage["data"] = BSON::Binary.new(doc.to_html)
72    else
73      puts "Comment on #{id} skipped"
74    end
75    puts "done"
76    return webpage
77  end
78end

 1require_relative 'websites'
 2
 3class WebSpider
 4  def initialize(proxy = nil)
 5    @client = WebClient.new proxy
 6  end
 7
 8  def collect(site)
 9    website = WebsitesDb.collection(site.domain)
10    site.page_list(@client) do |pageUrl|
11      if webpage = website.find_one(url:pageUrl)
12        if processed = site.process_page(@client, webpage)
13          website.save(processed)
14        end
15      elsif webpage = collect_page(pageUrl)
16        webpage.data = BSON::Binary.new(webpage.data)
17        webpage = webpage.to_hash
18        if processed = site.process_page(@client, webpage)
19          website.insert(processed)
20        else
21          website.insert(webpage)
22        end
23      end
24    end
25  end
26
27  def collect_page(pageUrl)
28    response = @client.visit(pageUrl, nil, 0.5)
29    case response
30    when Net::HTTPOK
31      webpage = Webpage.new pageUrl, response.body
32    else
33      return
34    end
35  end
36end
37
38# spider = WebSpider.new Proxies::Fiddler
39spider = WebSpider.new 
40spider.collect Websites::Cnbeta

 1# encoding: utf-8
 2require "webrick"
 3require "webrick/httpproxy"
 4require_relative 'database'
 5
 6class OfflineProxyServer < WEBrick::HTTPProxyServer
 7  def do_GET(req, res)
 8      uri = req.request_uri.to_s
 9      WebsitesDb.collections.each do |site|
10        if page = site.find_one(url:uri)
11          @logger.info("Found #{uri}")
12          res['connection'] = "close"
13          res.status = 200
14          res.body = page["data"].to_s
15          return
16        end
17      end
18      super
19  end
20end
21
22def search_db(keyword)
23  keyword.force_encoding("utf-8")
24  site = WebsitesDb.collection("cnbeta.com")
25  site.find(title:/#{keyword}/).sort(pubDate:"desc").limit(100).to_a
26end
27
28search = WEBrick::HTTPServlet::ProcHandler.new ->(req, resp) do
29  keyword = req.query["keyword"].to_s
30  result = if keyword.empty? then [] else search_db(keyword) end
31  resp['Content-Type'] = "text/html"
32  resp.body = %{
33    <html>
34    <head><meta charset="utf-8"><title>离线搜索</title><style>form{margin-left:30px}li{margin:10px}</style></head>
35    <body>
36    <form>
37    <em>cnBeta</em>
38    <input type="text" name="keyword" value="#{keyword}"><input type="submit" value="搜索">
39    </form>
40    <ul>
41#{
42      result.map{|page|"<li><a href=#{page['url']} target=_blank>#{page['title']}</a> #{page['pubDate']}</li>"}.join
43    }
44    </ul>
45    </body></html>
46}
47end
48
49server = OfflineProxyServer.new(Port: 9999)
50server.mount("/search", search)
51Signal.trap(:INT){ server.shutdown }
52server.start