diff --git a/autoupdater.rb b/autoupdater.rb index 233f4ceac9d..0b09b640786 100755 --- a/autoupdater.rb +++ b/autoupdater.rb @@ -32,7 +32,7 @@ if v.check_domain_verbose(domain, enable_cdnlist: false, show_green: true) Filelock '.git.lock' do - puts `./updater.py -a #{domain}` + puts `./updater.rb -a #{domain}` puts `git commit -S -am "accelerated-domains: add #{domain}"` if $?.success? puts `./update-local` if $?.success? end diff --git a/find_redundant.py b/find_redundant.py deleted file mode 100755 index c61c490577a..00000000000 --- a/find_redundant.py +++ /dev/null @@ -1,77 +0,0 @@ -#!/usr/bin/env python3 - -''' Find redundant items in accelerated-domains.china.conf. - e.g. 'bar.foo.com' is redundant for 'foo.com'. -''' - - -from collections.abc import Iterable - - -def load(conf_file): - ''' Parse conf file & Prepare data structure - Returns: [ ['abc', 'com'], - ['bar', 'foo', 'com'], - ... ] - ''' - - results = [] - if isinstance(conf_file, str): - lines = open(conf_file, 'r').readlines() - elif isinstance(conf_file, Iterable): - lines = iter(conf_file) - for line in lines: - line = line.strip() - if line == '' or line.startswith('#'): - continue - # A domain name is case-insensitive and - # consists of several labels, separated by a full stop - domain_name = line.split('/')[1] - domain_name = domain_name.lower() - domain_labels = domain_name.split('.') - results.append(domain_labels) - - # Sort results by domain labels' length - results.sort(key=len) - return results - - -def find(labelses): - ''' Find redundant items by a tree of top-level domain label to sub-level. - `tree` is like { 'com': { 'foo: { 'bar': LEAF }, - 'abc': LEAF }, - 'org': ... } - ''' - redundant = [] - tree = {} - LEAF = 1 - for labels in labelses: - domain = '.'.join(labels) - # Init root node as current node - node = tree - while len(labels) > 0: - label = labels.pop() - if label in node: - # If child node is a LEAF node, - # current domain must be an existed domain or a subdomain of an existed. - if node[label] == LEAF: - print(f"Redundant found: {domain} at {'.'.join(labels)}") - redundant.append(domain) - break - else: - # Create a leaf node if current label is last one - if len(labels) == 0: - node[label] = LEAF - # Create a branch node - else: - node[label] = {} - # Iterate to child node - node = node[label] - return redundant - - -def find_redundant(conf_file): - return find(load(conf_file)) - -if __name__ == '__main__': - find_redundant('accelerated-domains.china.conf') diff --git a/find_redundant.rb b/find_redundant.rb new file mode 100755 index 00000000000..4d16f38fb1d --- /dev/null +++ b/find_redundant.rb @@ -0,0 +1,81 @@ +#!/usr/bin/ruby + +""" Find redundant items in accelerated-domains.china.conf. + e.g. 'bar.foo.com' is redundant for 'foo.com'. +""" + +def load(conf_file) + """ Parse conf file & Prepare data structure + Returns: [ ['abc', 'com'], + ['bar', 'foo', 'com'], + ... ] + """ + + results = [] + if conf_file.is_a? String + lines = File.readlines(conf_file) + elsif conf_file.is_a? Array + lines = conf_file + end + lines.map do |line| + line = line.chomp + next if line.empty? or line.start_with?('#') + # A domain name is case-insensitive and + # consists of several labels, separated by a full stop + domain_name = line.split('/')[1] + domain_name = domain_name.downcase + domain_labels = domain_name.split('.') + results << domain_labels + end + + # Sort results by domain labels' length + results.sort_by(&:length) +end + + +LEAF = 1 +def find(labelses) + """ Find redundant items by a tree of top-level domain label to sub-level. + `tree` is like { 'com': { 'foo: { 'bar': LEAF }, + 'abc': LEAF }, + 'org': ... } + """ + redundant = [] + tree = {} + labelses.each do |labels| + domain = labels.join('.') + # Init root node as current node + node = tree + until labels.empty? + label = labels.pop + if node.include? label + # If child node is a LEAF node, + # current domain must be an existed domain or a subdomain of an existed. + if node[label] == LEAF + puts "Redundant found: #{domain} at #{labels.join('.')}" + redundant << domain + break + end + else + # Create a leaf node if current label is last one + if labels.empty? + node[label] = LEAF + # Create a branch node + else + node[label] = {} + end + end + # Iterate to child node + node = node[label] + end + end + redundant +end + +def find_redundant(conf_file) + return find(load(conf_file)) +end + +if __FILE__ == $0 + find_redundant('accelerated-domains.china.conf') +end diff --git a/updater.py b/updater.py deleted file mode 100755 index 68e6d75194a..00000000000 --- a/updater.py +++ /dev/null @@ -1,83 +0,0 @@ -#!/usr/bin/env python -from __future__ import unicode_literals -from argparse import ArgumentParser -import idna -import sys -import find_redundant - -if __name__ == "__main__": - parser = ArgumentParser(description="dnsmasq-china-list updater") - parser.add_argument( - '-a', '--add', - metavar="DOMAIN", - nargs="+", - help='Add one or more new domain(s) (implies -s)', - ) - parser.add_argument( - '-d', '--delete', - metavar="DOMAIN", - nargs="+", - default=[], - help='Remove one or more old domain(s) (implies -s)', - ) - parser.add_argument( - '-s', '--sort', - action='store_true', - default=True, - help='Sort the list (default action)', - ) - parser.add_argument( - '-f', '--file', - nargs=1, - default=["accelerated-domains.china.conf"], - help="Specify the file to update (accelerated-domains.china.conf by default)", - ) - - options = parser.parse_args() - - with open(options.file[0]) as f: - lines = list(f) - - changed = False - - if options.add: - options.sort = True - - for domain in options.add: - encoded_domain = idna.encode(domain).decode() - new_line = f"server=/{encoded_domain}/114.114.114.114\n" - disabled_line = f"#server=/{encoded_domain}/114.114.114.114" - if new_line in lines: - print(f"Domain already exists: {domain}") - else: - for line in lines: - if line.startswith(disabled_line): - print(f"Domain already disabled: {domain}") - break - else: - print(f"New domain added: {domain}") - lines.append(new_line) - changed = True - - options.delete += find_redundant.find_redundant(lines) - - if options.delete: - options.sort = True - - for domain in options.delete: - target_line = f"server=/{idna.encode(domain).decode()}/114.114.114.114\n" - if target_line not in lines: - print(f"Failed to remove domain {domain}: not found.") - else: - print(f"Domain removed: {domain}") - lines.remove(target_line) - changed = True - - if (options.add or options.delete) and not changed: - sys.exit(1) - - if options.sort: - lines.sort(key=lambda x: x.lstrip("#")) - - with open(options.file[0], "w") as f: - f.write(''.join(filter(lambda line: line.strip(), lines))) diff --git a/updater.rb b/updater.rb new file mode 100755 index 00000000000..77b8e965d52 --- /dev/null +++ b/updater.rb @@ -0,0 +1,87 @@ +#!/usr/bin/ruby +require 'domain_name' +require 'optparse' +require 'ostruct' + +options = OpenStruct.new +options.sort = true +options.file = "accelerated-domains.china.conf" +options.add = [] +options.delete = [] +OptionParser.new do |opts| + opts.banner = "dnsmasq-china-list updater" + + opts.on("-s", "--[no-]sort", "Sort the list (default action)") do |s| + options.sort = s + end + + opts.on("-f", "--file FILE", "Specify the file to update (accelerated-domains.china.conf by default)") do |f| + options.file = f + end + + opts.on("-a", "--add domain1,domain2", Array, "Add domain(s) to the list (implies -s)") do |a| + options.add = a + options.sort = true + end + + opts.on("-d", "--delete domain1,domain2", Array, "Remove domain(s) from the list (implies -s)") do |d| + options.delete = d + options.sort = true + end +end.parse! + +lines = File.readlines(options.file).filter { |line| !line.empty? } +disabled_lines = lines.filter { |line| line.start_with?("#") } + +changed = false + +options.add.each do |domain| + domain = DomainName.normalize(domain) + new_line = "server=/#{domain}/114.114.114.114\n" + disabled_line = "#server=/#{domain}/114.114.114.114" + if lines.include? new_line + puts "Domain already exists: #{domain}" + else + if disabled_lines.any? { |line| line.start_with? disabled_line } + puts "Domain already disabled: #{domain}" + else + # Check for duplicates + test_domain = domain + while test_domain.include? '.' + test_domain = test_domain.partition('.').last + _new_line = "server=/#{test_domain}/114.114.114.114\n" + _disabled_line = "#server=/#{test_domain}/114.114.114.114" + if lines.include? _new_line + puts "Redundant domain already exists: #{test_domain}" + break + elsif disabled_lines.any? { |line| line.start_with? _disabled_line } + puts "Redundant domain already disabled: #{test_domain}" + break + end + end + next if test_domain.include? '.' + + puts "New domain added: #{domain}" + lines << new_line + changed = true + end + end +end + +options.delete.each do |domain| + domain = DomainName.normalize(domain) + target_line = "server=/#{domain}/114.114.114.114\n" + unless lines.include? target_line + puts "Failed to remove domain #{domain}: not found." + else + puts "Domain removed: #{domain}" + lines.delete(target_line) + changed = true + end +end + +fail "No changes made." if (options.add.length || options.delete.length) && !changed + +lines.sort_by! { |x| x.delete_prefix("#") } if options.sort + +File.write(options.file, lines.join)