清华大佬耗费三个月吐血整理的几百G的资源,免费分享!....>>>
#coding utf-8
#!/usr/bin/env ruby -w
require "open-uri"
require "nokogiri"
require "thread"
require 'erlectricity'
class Spider
def initialize(dir, url)
@url = url
@dir = dir
end
def create_dir
`rm -rf #{@dir}` if File.directory? @dir
Dir.mkdir @dir
end
def get_imgs(doc)
links = doc.css('img.BDE_Image').collect { |img| img['src'] }
end
def run(pn, f)
create_dir()
threads = []
(1..pn).each do |i|
threads << Thread.new do
puts "going to #@url in page #{i}"
doc = Nokogiri::HTML(open("#@url?pn=#{i}"))
links = get_imgs(doc)
f.send!([:links, links])
end
end
threads.each &:join
end
end
receive do |f|
_pn = 1
f.when([:pn, Fixnum]) do |pn|
_pn = pn
f.send!([:pn, _pn])
f.receive_loop
end
f.when([:url, String]) do |url|
if url =~ /.*\/p\/(\d+)\/?/
spider = Spider.new($1, url)
spider.run(_pn, f)
f.send!(:exit)
else
f.send!([:error, "error."])
f.receive_loop
end
end
end