From 3e4e0726068b390fe63c3dbf7fc33f5992176ab2 Mon Sep 17 00:00:00 2001 From: lev Date: Wed, 31 Oct 2012 13:42:44 +0400 Subject: [PATCH 1/4] =?UTF-8?q?=D0=9F=D0=BE=D0=BF=D1=80=D0=B0=D0=B2=D0=B8?= =?UTF-8?q?=D0=BB=20=D1=81=D0=BE=D0=BE=D0=B1=D1=89=D0=B5=D0=BD=D0=B8=D0=B5?= =?UTF-8?q?=20=D0=BE=D0=B1=20=D0=BE=D1=88=D0=B8=D0=B1=D0=BA=D0=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/tasks/ipgeobase_local.rake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/tasks/ipgeobase_local.rake b/lib/tasks/ipgeobase_local.rake index 5246a35..9fb59bb 100644 --- a/lib/tasks/ipgeobase_local.rake +++ b/lib/tasks/ipgeobase_local.rake @@ -6,7 +6,7 @@ namespace :ipgeobase_local do require 'net/http' dir = "#{IpgeobaseLocal.base_directory}" if dir.blank? - raise "Not set base directory" + raise 'The destination directory is not defined, please assign IpgeobaseLocal.base_directory' end uri = URI('http://ipgeobase.ru/files/db/Main/geo_files.tar.gz') file_name = uri.path.split('/').last From ea1712ffe530a50f444da821c31a4a2a8582900c Mon Sep 17 00:00:00 2001 From: lev Date: Wed, 31 Oct 2012 14:06:17 +0400 Subject: [PATCH 2/4] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D0=B8=D0=BB?= =?UTF-8?q?=20=D0=BF=D1=80=D0=BE=D0=B3=D1=80=D0=B5=D1=81=D1=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/tasks/ipgeobase_local.rake | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/lib/tasks/ipgeobase_local.rake b/lib/tasks/ipgeobase_local.rake index 9fb59bb..4362244 100644 --- a/lib/tasks/ipgeobase_local.rake +++ b/lib/tasks/ipgeobase_local.rake @@ -4,16 +4,17 @@ namespace :ipgeobase_local do desc "Update base file form ipgeobase.ru" task :update => :environment do require 'net/http' - dir = "#{IpgeobaseLocal.base_directory}" - if dir.blank? + base_dir = IpgeobaseLocal.base_directory + if base_dir.blank? raise 'The destination directory is not defined, please assign IpgeobaseLocal.base_directory' end uri = URI('http://ipgeobase.ru/files/db/Main/geo_files.tar.gz') file_name = uri.path.split('/').last - file_name = [dir, file_name].join + file_name = [base_dir, file_name].join('/') - FileUtils.mkdir_p "#{dir}" + FileUtils.mkdir_p base_dir + puts "Downloading geo data from #{uri}..." Net::HTTP.start(uri.host, uri.port) do |http| request = Net::HTTP::Get.new uri.request_uri http.request request do |response| @@ -24,8 +25,11 @@ namespace :ipgeobase_local do end end end + puts "Done." - system("tar -xzf #{file_name} -C #{dir}") + puts "Uncompressing geo data into #{base_dir}..." + system("tar -xzf #{file_name} -C #{base_dir}") FileUtils.rm file_name + puts "Done." end end From 998cc203abdc67049b2418aeaccf5306fa665758 Mon Sep 17 00:00:00 2001 From: lev Date: Thu, 1 Nov 2012 01:12:47 +0400 Subject: [PATCH 3/4] =?UTF-8?q?=D0=A1=D0=B4=D0=B5=D0=BB=D0=B0=D0=BB=20?= =?UTF-8?q?=D0=B8=D0=BC=D0=BF=D0=BE=D1=80=D1=82=20=D0=B8=D0=B7=20ipgeobase?= =?UTF-8?q?=20=D0=B2=20sql?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 8 ++- lib/ipgeobase_local/ip_meta_data.rb | 10 +-- lib/ipgeobase_local/memory_base.rb | 21 ++++-- lib/tasks/ipgeobase_local.rake | 104 ++++++++++++++++++++++++++++ 4 files changed, 129 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index bcb1fe9..e5f5a1c 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,8 @@ # IpgeobaseLocal -География российских и украинских IP-адресов. Поиск местонахождения (города) IP-адреса, локальный поиск +География российских и украинских IP-адресов. Поиск местонахождения (города) IP-адреса, локальный поиск. +Импорт данных о городах из ipgeobase в SQL. +Работает только если подключать как gem в Rails 3.2+ ## Installation @@ -18,7 +20,7 @@ Or install it yourself as: ## Usage - rake ipgeobase_local:update# => загрузка базы c ipgeobase.ru + rake ipgeobase_local:update # => загрузка/обновление базы c ipgeobase.ru IpgeobaseLocal.load#предварительная загрузка базы в пямять @@ -27,12 +29,14 @@ Or install it yourself as: ip_meta.city # => Москва ip_meta.country # => Россия + ## Usage with Rails config/initializers/ipgeo.rb IpgeobaseLocal.base_directory = "db/geo_files/" IpgeobaseLocal.load + ## Contributing 1. Fork it diff --git a/lib/ipgeobase_local/ip_meta_data.rb b/lib/ipgeobase_local/ip_meta_data.rb index 3bedd61..63be8f6 100644 --- a/lib/ipgeobase_local/ip_meta_data.rb +++ b/lib/ipgeobase_local/ip_meta_data.rb @@ -5,13 +5,13 @@ class IpMetaData def initialize(country='', region='', city='') @country = country - @region = region - @city = city + @region = region + @city = city end - def city;@city;end - def country;@country;end - def region;@region;end + def city; @city; end + def country; @country; end + def region; @region; end end end diff --git a/lib/ipgeobase_local/memory_base.rb b/lib/ipgeobase_local/memory_base.rb index 68306ba..e833d2f 100644 --- a/lib/ipgeobase_local/memory_base.rb +++ b/lib/ipgeobase_local/memory_base.rb @@ -11,11 +11,11 @@ def self.parse(ip) end def self.load + instance.load_base #unless defined?(Rake) instance end def initialize() - load_base unless defined?(Rake) end def find(ip) @@ -44,17 +44,21 @@ def binary_find(ip) false end - private + def cidr_base; @cidr_base; end def load_base - city_base = {} + + city_base = {} @cidr_base = [] - cidr_file = "#{IpgeobaseLocal.base_directory}cidr_optim.txt" - cities_file = "#{IpgeobaseLocal.base_directory}cities.txt" + cidr_file = [IpgeobaseLocal.base_directory, 'cidr_optim.txt'].join('/') + cities_file = [IpgeobaseLocal.base_directory, 'cities.txt'].join('/') + unless File.exists?(cidr_file) && File.exists?(cities_file) - raise "Not exist ipgeobase files, pls run 'rake ipgeobase_local:update'" + raise "ipgeobase data files (#{cidr_file} and/or #{cities_file}) do not exist, please run 'rake ipgeobase_local:update'" end + + puts "Loading data from #{cidr_file} and #{cities_file}..." CSV.foreach(cities_file, col_sep: "\t", quote_char:"'", encoding: "windows-1251:utf-8") do |line| city_base[line[0]] = line[1..5] end @@ -72,9 +76,12 @@ def load_base end @cidr_base << {start: line[0].to_i, end: line[1].to_i, meta_data: IpMetaData.new(country, region, city)} end - + puts "Done." + @cidr_base end + private + def convert_ip(ip) if ::IPAddr.new(ip).ipv4? part = ip.split(".").map{|i| i.to_i} diff --git a/lib/tasks/ipgeobase_local.rake b/lib/tasks/ipgeobase_local.rake index 4362244..160d3a2 100644 --- a/lib/tasks/ipgeobase_local.rake +++ b/lib/tasks/ipgeobase_local.rake @@ -32,4 +32,108 @@ namespace :ipgeobase_local do FileUtils.rm file_name puts "Done." end + + +# Предполагается наличие следующих моделей: + +# class Country < ActiveRecord::Base +# attr_accessible :abbr2 +# end + +# class Region < ActiveRecord::Base +# belongs_to :country +# attr_accessible :country +# attr_accessible :name +# end + +# class City < ActiveRecord::Base +# belongs_to :country +# belongs_to :region +# attr_accessible :country, :region +# attr_accessible :name +# end + + +# Предполагается наличие следующих таблиц/миграций: + +# class CreateCountries < ActiveRecord::Migration +# def up +# execute " +# create table countries ( +# id serial primary key, +# created_at timestamp not null, +# updated_at timestamp not null, +# deleted_at timestamp not null DEFAULT '1970-01-01 00:00:00', +# abbr2 varchar not null unique +# )" +# end +# +# def down +# execute "drop table countries" +# end +# end + +# class CreateRegions < ActiveRecord::Migration +# def up +# execute " +# create table regions ( +# id serial primary key, +# created_at timestamp not null, +# updated_at timestamp not null, +# deleted_at timestamp not null DEFAULT '1970-01-01 00:00:00', +# country_id int_id not null, +# name varchar not null, +# unique(country_id,name) +# )" +# end +# +# def down +# execute "drop table regions" +# end +# end + +# class CreateCities < ActiveRecord::Migration +# def up +# execute " +# create table cities ( +# id serial primary key, +# created_at timestamp not null, +# updated_at timestamp not null, +# deleted_at timestamp not null DEFAULT '1970-01-01 00:00:00', +# country_id int_id not null, +# region_id int_id not null, +# name varchar not null, +# unique(country_id,region_id,name) +# )" +# end +# +# def down +# execute "drop table cities" +# end +# end + + desc "Импорт country/regions/cities в базу данных" + task :import_localities => :environment do + Country.connection.execute "truncate table countries; alter sequence countries_id_seq restart with 1;" + Region.connection.execute "truncate table regions; alter sequence regions_id_seq restart with 1;" + City.connection.execute "truncate table cities; alter sequence cities_id_seq restart with 1;" + + data = {} + IpgeobaseLocal::MemoryBase.load.cidr_base.each do |d| + d = d[:meta_data] + country = d.country + region = d.region + city = d.city + if country.present? + c = data[country] ||= { obj: Country.create({abbr2: country}) } + if region.present? + r = c[region] ||= { obj: Region.create({country: c[:obj], name: region}) } + if city.present? + cc = r[city] ||= { obj: City.create({country: c[:obj], region: r[:obj], name: city} ) } + end + end + end + end + end end + From 805a6dbc776ee47c8a4bd1160e9d747b06881a74 Mon Sep 17 00:00:00 2001 From: lev Date: Thu, 1 Nov 2012 01:42:46 +0400 Subject: [PATCH 4/4] =?UTF-8?q?=D0=A3=D0=BB=D1=83=D1=87=D1=88=D0=B8=D0=BB?= =?UTF-8?q?=20=D0=BF=D1=80=D0=BE=D0=B3=D1=80=D0=B5=D1=81=D1=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/tasks/ipgeobase_local.rake | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/tasks/ipgeobase_local.rake b/lib/tasks/ipgeobase_local.rake index 160d3a2..55ac102 100644 --- a/lib/tasks/ipgeobase_local.rake +++ b/lib/tasks/ipgeobase_local.rake @@ -114,12 +114,16 @@ namespace :ipgeobase_local do desc "Импорт country/regions/cities в базу данных" task :import_localities => :environment do + puts "Resetting the database..." Country.connection.execute "truncate table countries; alter sequence countries_id_seq restart with 1;" Region.connection.execute "truncate table regions; alter sequence regions_id_seq restart with 1;" City.connection.execute "truncate table cities; alter sequence cities_id_seq restart with 1;" + puts "Done." data = {} - IpgeobaseLocal::MemoryBase.load.cidr_base.each do |d| + cidr_base = IpgeobaseLocal::MemoryBase.load.cidr_base + puts "Exporting to SQL DB..." + cidr_base.each do |d| d = d[:meta_data] country = d.country region = d.region @@ -134,6 +138,7 @@ namespace :ipgeobase_local do end end end + puts "Done." end end