diff --git a/CHANGELOG.txt b/CHANGELOG.txt index c4d7da6c6c..d0409f10db 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,6 +1,15 @@ Changelog ========= +0.3.24 +------ + +*Date: 2013-10-30* + +* Experimental: Sogou proxy server +* Fix issues for: + - Vimeo + 0.3.23 ------ diff --git a/README.md b/README.md index e8b0b77537..d8a89bae03 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,8 @@ See the project homepage for further documentat Fork me on GitHub: +__中文说明__已移至[wiki](https://github.com/soimort/you-get/wiki/%E4%B8%AD%E6%96%87%E8%AF%B4%E6%98%8E)。 + ## Features ### Supported Sites (As of Now) @@ -65,23 +67,15 @@ Fork me on GitHub: ## Installation -### 1. Install via [Pip](http://www.pip-installer.org/): - - $ pip install you-get - - Check if the installation was successful: - - $ you-get -V - -### 2. Install via [EasyInstall](http://pypi.python.org/pypi/setuptools): +### 1. Install via Pip: - $ easy_install you-get + $ [sudo] pip install you-get Check if the installation was successful: $ you-get -V -### 3. Install from Git: +### 2. Install from Git: $ git clone git://github.com/soimort/you-get.git @@ -98,7 +92,7 @@ Fork me on GitHub: $ you-get -V -### 4. Direct download (from ): +### 3. Direct download (from ): $ wget -O you-get.zip https://github.com/soimort/you-get/zipball/master $ unzip you-get.zip @@ -116,27 +110,19 @@ Fork me on GitHub: $ you-get -V -### 5. Install from [AUR (Arch User Repository)](http://aur.archlinux.org/): - - Click [here](https://aur.archlinux.org/packages.php\?ID=62576). +### 4. Install from your distro's repo: -### Upgrading: - -Using Pip: +* __AUR (Arch)__: - $ pip install --upgrade you-get +* __Overlay (Gentoo)__: -### FAQ (For Windows Users): +## Upgrading -* Q: I don't know how to install it on Windows. - -* A: Then don't do it. Just put your `you-get` folder into system `%PATH%`. - -* Q: I got something like `UnicodeDecodeError: 'gbk' codec can't decode byte 0xb0 in position 1012: illegal multibyte sequence`. +Using Pip: -* A: Run `set PYTHONIOENCODING=utf-8`. + $ [sudo] pip install --upgrade you-get -## Examples (For End-Users) +## Examples Display the information of the video without downloading: @@ -173,31 +159,23 @@ By default, Python will apply the system proxy settings (i.e. environment variab For a complete list of all available options, see: $ you-get --help - -## Examples (For Developers) - -In Python 3 (interactive): - - >>> from you_get.downloader import * - >>> youtube.download("http://www.youtube.com/watch?v=8bQlxQJEzLk", info_only = True) - Video Site: YouTube.com - Title: If you're good at something, never do it for free! - Type: WebM video (video/webm) - Size: 0.13 MB (133176 Bytes) - - >>> import you_get - >>> you_get.any_download("http://www.youtube.com/watch?v=sGwy8DsUJ4M") - Video Site: YouTube.com - Title: Mort from Madagascar LIKES - Type: WebM video (video/webm) - Size: 1.78 MB (1867072 Bytes) - - Downloading Mort from Madagascar LIKES.webm ... - 100.0% ( 1.8/1.8 MB) [========================================] 1/1 - -## API Reference - -See source code. + Usage: you-get [OPTION]... [URL]... + + Startup options: + -V | --version Display the version and exit. + -h | --help Print this help and exit. + + Download options (use with URLs): + -f | --force Force overwriting existed files. + -i | --info Display the information of videos without downloading. + -u | --url Display the real URLs of videos without downloading. + -n | --no-merge Don't merge video parts. + -o | --output-dir Set the output directory for downloaded videos. + -x | --http-proxy Use specific HTTP proxy for downloading. + --no-proxy Don't use any proxy. (ignore $http_proxy) + -S | --sogou Use a Sogou proxy server for downloading. + --sogou-proxy Run a standalone Sogou proxy server. + --debug Show traceback on KeyboardInterrupt. ## License @@ -206,228 +184,3 @@ You-Get is licensed under the [MIT license](https://raw.github.com/soimort/you-g ## Contributing Please see [CONTRIBUTING.md](https://github.com/soimort/you-get/blob/master/CONTRIBUTING.md). - - - -*** - - - -# You-Get - 中文说明 - -[You-Get](https://github.com/soimort/you-get)是一个基于Python 3的视频下载工具。之所以写它的主要原因是，我找不到一个现成的下载工具能够同时支持[YouTube](http://www.youtube.com/)和[优酷](http://www.youku.com/)；而且，几乎所有以前的视频下载程序都是基于Python 2的。 - -项目主页： - -GitHub地址： - -## 特点 - -### 说明 - -You-Get基于优酷下载脚本[iambus/youku-lixian](https://github.com/iambus/youku-lixian)用Python 3改写而成，增加了以下功能： - -* 支持YouTube、Vimeo等国外视频网站 -* 支持断点续传 -* 可设置HTTP代理 - -### 支持的站点（截至目前） - -已实现对以下站点的支持，以后会陆续增加（・∀・） - -* YouTube -* Vimeo -* Coursera -* Blip -* Dailymotion -* eHow -* Facebook -* Google+ -* Google Drive -* Khan Academy -* TED -* Tumblr -* Vine -* Instagram -* SoundCloud -* Mixcloud -* Freesound -* JPopsuki -* VID48 -* NICONICO动画 -* 优酷 -* 土豆 -* 音悦台 -* AcFun -* bilibili -* CNTV -* 豆瓣 -* 凤凰视频 -* 爱奇艺 -* 激动网 -* 酷6网 -* MioMio -* 网易视频 -* PPTV -* 腾讯视频 -* 新浪视频 -* 搜狐视频 -* 56网 -* 虾米 -* 5sing -* 百度音乐 -* 百度网盘 -* SongTaste -* Alive.in.th - -## 依赖 - -* [Python 3](http://www.python.org/download/releases/) -* __（可选）__ [FFmpeg](http://ffmpeg.org) - * 用于转换与合并视频文件。 - -## 安装说明 - -（以下命令格式均以Linux shell为例） - -### 1. 通过[Pip](http://www.pip-installer.org/)安装： - - $ pip install you-get - - 检查安装是否成功： - - $ you-get -V - -### 2. 通过[EasyInstall](http://pypi.python.org/pypi/setuptools)安装： - - $ easy_install you-get - - 检查安装是否成功： - - $ you-get -V - -### 3. 从Git安装： - - $ git clone git://github.com/soimort/you-get.git - - 在不安装的情况下直接使用脚本： - - $ cd you-get/ - $ ./you-get -V - - 若要将Python package安装到系统默认路径，执行： - - $ make install - - 检查安装是否成功： - - $ you-get -V - -### 4. 直接下载（从）： - - $ wget -O you-get.zip https://github.com/soimort/you-get/zipball/master - $ unzip you-get.zip - - 在不安装的情况下直接使用脚本： - - $ cd soimort-you-get-*/ - $ ./you-get -V - - 若要将Python package安装到系统默认路径，执行： - - $ make install - - 检查安装是否成功： - - $ you-get -V - -### 5. 从[AUR (Arch User Repository)](http://aur.archlinux.org/)安装： - - 点击[这里](https://aur.archlinux.org/packages.php\?ID=62576)。 - -### 升级： - -使用Pip： - - $ pip install --upgrade you-get - -### FAQ（针对Windows用户)： - -* Q：我不知道该如何在Windows下安装。 - -* A：不需要安装。直接把`you-get`目录放到系统`%PATH%`中。 - -* Q：出现错误提示`UnicodeDecodeError: 'gbk' codec can't decode byte 0xb0 in position 1012: illegal multibyte sequence`。 - -* A：执行`set PYTHONIOENCODING=utf-8`。 - -## 使用方法示例 - -### 如何下载视频 - -显示视频信息，但不进行下载（`-i`或`--info`选项）： - - $ you-get -i http://www.yinyuetai.com/video/463772 - -下载视频： - - $ you-get http://www.yinyuetai.com/video/463772 - -下载多个视频： - - $ you-get http://www.yinyuetai.com/video/463772 http://www.yinyuetai.com/video/471500 - -若当前目录下已有与视频标题同名的文件，下载时会自动跳过。若有同名的`.download`临时文件，程序会从上次中断处开始下载。 -如要强制重新下载该视频，可使用`-f`（`--force`）选项： - - $ you-get -f http://www.yinyuetai.com/video/463772 - -`-l`（`--playlist`）选项用于下载播放列表（只对某些网站适用）： - - $ you-get -l http://www.youku.com/playlist_show/id_5344313.html - -__注：从0.1.3以后的版本起，`-l`选项不再必须。You-Get可以自动识别并处理播放列表的下载。__ - -指定视频文件的下载目录： - - $ you-get -o ~/Downloads http://www.yinyuetai.com/video/463772 - -显示详细帮助： - - $ you-get -h - -### 如何设置代理 - -默认情况下，Python自动使用系统的代理配置。可以通过环境变量`http_proxy`来设置系统的HTTP代理。 - -`-x`（`--http-proxy`）选项用于手动指定You-Get所使用的HTTP代理。例如：GoAgent的代理服务器是`http://127.0.0.1:8087`，则通过该代理下载某YouTube视频的命令是： - - $ you-get -x 127.0.0.1:8087 http://www.youtube.com/watch?v=KbtO_Ayjw0M - -Windows下的自由门等翻墙软件会自动设置系统全局代理，因此无需指定HTTP代理即可下载YouTube视频： - - $ you-get http://www.youtube.com/watch?v=KbtO_Ayjw0M - -如果不希望程序在下载过程中使用任何代理（包括系统的代理配置），可以显式地指定`--no-proxy`选项： - - $ you-get --no-proxy http://v.youku.com/v_show/id_XMjI0ODc1NTc2.html - -### 断点续传 - -下载未完成时被中止（因为`Ctrl+C`终止程序或者网络中断等原因），在目标路径中会有一个扩展名为`.download`的临时文件。 - -下次运行只要在目标路径中找到相应的`.download`临时文件，程序会自动从中断处继续下载。（除非指定了`-f`选项） - -## 使用Python 2？ - -优酷等国内视频网站的下载，请移步：[iambus/youku-lixian](https://github.com/iambus/youku-lixian) - -YouTube等国外视频网站的下载，请移步：[rg3/youtube-dl](https://github.com/rg3/youtube-dl) - -## 许可证 - -You-Get在[MIT License](https://raw.github.com/soimort/you-get/master/LICENSE.txt)下发布。 - -## 如何参与贡献 / 报告issue - -请阅读 [CONTRIBUTING.md](https://github.com/soimort/you-get/blob/master/CONTRIBUTING.md)。 diff --git a/README.txt b/README.txt index c4b68af855..02a9408e21 100644 --- a/README.txt +++ b/README.txt @@ -72,17 +72,9 @@ Dependencies Installation ------------ -#) Install via `Pip `_:: +#) Install via Pip:: - $ pip install you-get - - Check if the installation was successful:: - - $ you-get -V - -#) Install via `EasyInstall `_:: - - $ easy_install you-get + $ [sudo] pip install you-get Check if the installation was successful:: @@ -123,12 +115,21 @@ Installation $ you-get -V -#) Install from `AUR (Arch User Repository) `_: +#) Install from your distro's repo: + +* `AUR (Arch) `_ + +* `Overlay (Gentoo) `_ + +Upgrading +--------- - Click `here `_. +Using Pip:: -Examples (For End-Users) ------------------------- + $ [sudo] pip install --upgrade you-get + +Examples +-------- Display the information of the video without downloading:: @@ -166,33 +167,23 @@ Command-Line Options For a complete list of all available options, see:: $ you-get --help - -Examples (For Developers) -------------------------- - -In Python 3 (interactive):: - - >>> from you_get.downloader import * - >>> youtube.download("http://www.youtube.com/watch?v=8bQlxQJEzLk", info_only = True) - Video Site: YouTube.com - Title: If you're good at something, never do it for free! - Type: WebM video (video/webm) - Size: 0.13 MB (133176 Bytes) - - >>> import you_get - >>> you_get.any_download("http://www.youtube.com/watch?v=sGwy8DsUJ4M") - Video Site: YouTube.com - Title: Mort from Madagascar LIKES - Type: WebM video (video/webm) - Size: 1.78 MB (1867072 Bytes) - - Downloading Mort from Madagascar LIKES.webm ... - 100.0% ( 1.8/1.8 MB) [========================================] 1/1 - -API Reference -------------- - -See source code. + Usage: you-get [OPTION]... [URL]... + + Startup options: + -V | --version Display the version and exit. + -h | --help Print this help and exit. + + Download options (use with URLs): + -f | --force Force overwriting existed files. + -i | --info Display the information of videos without downloading. + -u | --url Display the real URLs of videos without downloading. + -n | --no-merge Don't merge video parts. + -o | --output-dir Set the output directory for downloaded videos. + -x | --http-proxy Use specific HTTP proxy for downloading. + --no-proxy Don't use any proxy. (ignore $http_proxy) + -S | --sogou Use a Sogou proxy server for downloading. + --sogou-proxy Run a standalone Sogou proxy server. + --debug Show traceback on KeyboardInterrupt. License ------- diff --git a/src/you_get/common.py b/src/you_get/common.py index 2c99976f0b..25df109060 100644 --- a/src/you_get/common.py +++ b/src/you_get/common.py @@ -8,12 +8,15 @@ import sys from urllib import request, parse import platform +import threading from .version import __version__ -from .util import log, legitimize +from .util import log, legitimize, sogou_proxy_server dry_run = False force = False +sogou_proxy = None +sogou_env = None fake_headers = { 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', @@ -712,9 +715,38 @@ def print_info(site_info, title, type, size): print("Video Site:", site_info) print("Title: ", tr(title)) print("Type: ", type_info) - print("Size: ", round(size / 1048576, 2), "MB (" + str(size) + " Bytes)") + print("Size: ", round(size / 1048576, 2), "MiB (" + str(size) + " Bytes)") print() +def parse_host(host): + """Parses host name and port number from a string. + """ + if re.match(r'^(\d+)$', host) is not None: + return ("0.0.0.0", int(host)) + if re.match(r'^(\w+)://', host) is None: + host = "//" + host + o = parse.urlparse(host) + hostname = o.hostname or "0.0.0.0" + port = o.port or 0 + return (hostname, port) + +def get_sogou_proxy(): + return sogou_proxy + +def set_proxy(proxy): + proxy_handler = request.ProxyHandler({ + 'http': '%s:%s' % proxy, + 'https': '%s:%s' % proxy, + }) + opener = request.build_opener(proxy_handler) + request.install_opener(opener) + +def unset_proxy(): + proxy_handler = request.ProxyHandler({}) + opener = request.build_opener(proxy_handler) + request.install_opener(opener) + +# DEPRECATED in favor of set_proxy() and unset_proxy() def set_http_proxy(proxy): if proxy == None: # Use system default setting proxy_support = request.ProxyHandler() @@ -760,13 +792,15 @@ def script_main(script_name, download, download_playlist = None): -u | --url Display the real URLs of videos without downloading. -n | --no-merge Don't merge video parts. -o | --output-dir Set the output directory for downloaded videos. - -x | --http-proxy Use specific HTTP proxy for downloading. + -x | --http-proxy Use specific HTTP proxy for downloading. --no-proxy Don't use any proxy. (ignore $http_proxy) + -S | --sogou Use a Sogou proxy server for downloading. + --sogou-proxy Run a standalone Sogou proxy server. --debug Show traceback on KeyboardInterrupt. ''' - short_opts = 'Vhfiuno:x:' - opts = ['version', 'help', 'force', 'info', 'url', 'no-merge', 'no-proxy', 'debug', 'output-dir=', 'http-proxy='] + short_opts = 'VhfiunSo:x:' + opts = ['version', 'help', 'force', 'info', 'url', 'no-merge', 'no-proxy', 'debug', 'sogou', 'output-dir=', 'http-proxy=', 'sogou-proxy=', 'sogou-env='] if download_playlist: short_opts = 'l' + short_opts opts = ['playlist'] + opts @@ -778,6 +812,11 @@ def script_main(script_name, download, download_playlist = None): log.e("try 'you-get --help' for more options") sys.exit(2) + global force + global dry_run + global sogou_proxy + global sogou_env + info_only = False playlist = False merge = True @@ -793,12 +832,10 @@ def script_main(script_name, download, download_playlist = None): print(help) sys.exit() elif o in ('-f', '--force'): - global force force = True elif o in ('-i', '--info'): info_only = True elif o in ('-u', '--url'): - global dry_run dry_run = True elif o in ('-l', '--playlist'): playlist = True @@ -812,19 +849,38 @@ def script_main(script_name, download, download_playlist = None): output_dir = a elif o in ('-x', '--http-proxy'): proxy = a + elif o in ('-S', '--sogou'): + sogou_proxy = ("0.0.0.0", 0) + elif o in ('--sogou-proxy'): + sogou_proxy = parse_host(a) + elif o in ('--sogou-env'): + sogou_env = a else: log.e("try 'you-get --help' for more options") sys.exit(2) if not args: - print(help) - sys.exit() + if sogou_proxy is not None: + try: + if sogou_env is not None: + server = sogou_proxy_server(sogou_proxy, network_env=sogou_env) + else: + server = sogou_proxy_server(sogou_proxy) + server.serve_forever() + except KeyboardInterrupt: + if traceback: + raise + else: + sys.exit() + else: + print(help) + sys.exit() set_http_proxy(proxy) - - if traceback: + + try: download_main(download, download_playlist, args, playlist, output_dir, merge, info_only) - else: - try: - download_main(download, download_playlist, args, playlist, output_dir, merge, info_only) - except KeyboardInterrupt: + except KeyboardInterrupt: + if traceback: + raise + else: sys.exit(1) diff --git a/src/you_get/extractor/sohu.py b/src/you_get/extractor/sohu.py index c364917f5d..a084f116a0 100644 --- a/src/you_get/extractor/sohu.py +++ b/src/you_get/extractor/sohu.py @@ -17,6 +17,14 @@ def sohu_download(url, output_dir = '.', merge = True, info_only = False): if not vid: vid = r1('vid\s*:\s*"(\d+)"', html) + # Open Sogou proxy if required + if get_sogou_proxy() is not None: + server = sogou_proxy_server(get_sogou_proxy(), ostream=open(os.devnull, 'w')) + server_thread = threading.Thread(target=server.serve_forever) + server_thread.daemon = True + server_thread.start() + set_proxy(server.server_address) + if vid: data = json.loads(get_decoded_html('http://hot.vrs.sohu.com/vrs_flash.action?vid=%s' % vid)) for qtyp in ["oriVid","superVid","highVid" ,"norVid","relativeId"]: @@ -52,6 +60,11 @@ def sohu_download(url, output_dir = '.', merge = True, info_only = False): urls.append(real_url(host, prot, file, new)) assert data['clipsURL'][0].endswith('.mp4') + # Close Sogou proxy if required + if get_sogou_proxy() is not None: + server.shutdown() + unset_proxy() + print_info(site_info, title, 'mp4', size) if not info_only: download_urls(urls, title, 'mp4', size, output_dir, refer = url, merge = merge) diff --git a/src/you_get/extractor/youku.py b/src/you_get/extractor/youku.py index 20c79c4daf..4abedc9783 100644 --- a/src/you_get/extractor/youku.py +++ b/src/you_get/extractor/youku.py @@ -121,7 +121,21 @@ def file_type_of_url(url): return str(re.search(r'/st/([^/]+)/', url).group(1)) def youku_download_by_id(id, title, output_dir = '.', stream_type = None, merge = True, info_only = False): + # Open Sogou proxy if required + if get_sogou_proxy() is not None: + server = sogou_proxy_server(get_sogou_proxy(), ostream=open(os.devnull, 'w')) + server_thread = threading.Thread(target=server.serve_forever) + server_thread.daemon = True + server_thread.start() + set_proxy(server.server_address) + info = get_info(id) + + # Close Sogou proxy if required + if get_sogou_proxy() is not None: + server.shutdown() + unset_proxy() + urls, sizes = zip(*find_video(info, stream_type)) ext = file_type_of_url(urls[0]) total_size = sum(sizes) diff --git a/src/you_get/util/__init__.py b/src/you_get/util/__init__.py index b097d246de..4c43c5fad5 100644 --- a/src/you_get/util/__init__.py +++ b/src/you_get/util/__init__.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python from .fs import * from .log import * +from .sogou_proxy import * diff --git a/src/you_get/util/log.py b/src/you_get/util/log.py index 6f02c3a1df..c28fd4e731 100644 --- a/src/you_get/util/log.py +++ b/src/you_get/util/log.py @@ -10,6 +10,7 @@ 'vt100', 'linux', 'eterm-color', + 'screen', ): has_colors = True else: @@ -68,10 +69,15 @@ 'bold-white': '\033[97;1m', } +def underlined(text): + """Returns an underlined text. + """ + return "\33[4m%s\33[24m" % text if has_colors else text + def println(text, color=None, ostream=sys.stdout): """Prints a text line to stream. """ - if color in colors: + if has_colors and color in colors: ostream.write("{0}{1}{2}\n".format(colors[color], text, colors['reset'])) else: ostream.write("{0}\n".format(text)) @@ -79,7 +85,7 @@ def println(text, color=None, ostream=sys.stdout): def printlog(message, color=None, ostream=sys.stderr): """Prints a log message to stream. """ - if color in colors: + if has_colors and color in colors: ostream.write("{0}{1}: {2}{3}\n".format(colors[color], __name__, message, colors['reset'])) else: ostream.write("{0}: {1}\n".format(__name__, message)) @@ -88,7 +94,7 @@ def i(message, ostream=sys.stderr): """Sends an info log message. """ printlog(message, - 'white' if has_colors else None, + None, ostream=ostream) def d(message, ostream=sys.stderr): diff --git a/src/you_get/util/sogou_proxy.py b/src/you_get/util/sogou_proxy.py new file mode 100644 index 0000000000..01ffb5720c --- /dev/null +++ b/src/you_get/util/sogou_proxy.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python + +# Original code from: +# http://xiaoxia.org/2011/03/26/using-python-to-write-a-local-sogou-proxy-server-procedures/ + +from . import log + +from http.client import HTTPResponse +from http.server import BaseHTTPRequestHandler, HTTPServer +from socketserver import ThreadingMixIn +from threading import Thread +import random, socket, struct, sys, time + +def sogou_proxy_server( + host=("0.0.0.0", 0), + network_env='CERNET', + ostream=sys.stderr): + """Returns a Sogou proxy server object. + """ + + x_sogou_auth = '9CD285F1E7ADB0BD403C22AD1D545F40/30/853edc6d49ba4e27' + proxy_host = 'h0.cnc.bj.ie.sogou.com' + proxy_port = 80 + + def sogou_hash(t, host): + s = (t + host + 'SogouExplorerProxy').encode('ascii') + code = len(s) + dwords = int(len(s) / 4) + rest = len(s) % 4 + v = struct.unpack(str(dwords) + 'i' + str(rest) + 's', s) + for vv in v: + if type(vv) != bytes: + a = (vv & 0xFFFF) + b = (vv >> 16) + code += a + code = code ^ (((code << 5) ^ b) << 0xb) + # To avoid overflows + code &= 0xffffffff + code += code >> 0xb + if rest == 3: + code += s[len(s) - 2] * 256 + s[len(s) - 3] + code = code ^ ((code ^ (s[len(s) - 1]) * 4) << 0x10) + code &= 0xffffffff + code += code >> 0xb + elif rest == 2: + code += (s[len(s) - 1]) * 256 + (s[len(s) - 2]) + code ^= code << 0xb + code &= 0xffffffff + code += code >> 0x11 + elif rest == 1: + code += s[len(s) - 1] + code ^= code << 0xa + code &= 0xffffffff + code += code >> 0x1 + code ^= code * 8 + code &= 0xffffffff + code += code >> 5 + code ^= code << 4 + code = code & 0xffffffff + code += code >> 0x11 + code ^= code << 0x19 + code = code & 0xffffffff + code += code >> 6 + code = code & 0xffffffff + return hex(code)[2:].rstrip('L').zfill(8) + + class Handler(BaseHTTPRequestHandler): + _socket = None + def do_proxy(self): + try: + if self._socket is None: + self._socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self._socket.connect((proxy_host, proxy_port)) + self._socket.send(self.requestline.encode('ascii') + b'\r\n') + log.d(self.requestline, ostream) + + # Add Sogou Verification Tags + self.headers['X-Sogou-Auth'] = x_sogou_auth + t = hex(int(time.time()))[2:].rstrip('L').zfill(8) + self.headers['X-Sogou-Tag'] = sogou_hash(t, self.headers['Host']) + self.headers['X-Sogou-Timestamp'] = t + self._socket.send(str(self.headers).encode('ascii') + b'\r\n') + + # Send POST data + if self.command == 'POST': + self._socket.send(self.rfile.read(int(self.headers['Content-Length']))) + response = HTTPResponse(self._socket, method=self.command) + response.begin() + + # Response + status = 'HTTP/1.1 %s %s' % (response.status, response.reason) + self.wfile.write(status.encode('ascii') + b'\r\n') + h = '' + for hh, vv in response.getheaders(): + if hh.upper() != 'TRANSFER-ENCODING': + h += hh + ': ' + vv + '\r\n' + self.wfile.write(h.encode('ascii') + b'\r\n') + while True: + response_data = response.read(8192) + if len(response_data) == 0: + break + self.wfile.write(response_data) + + except socket.error: + log.e('Socket error for ' + self.requestline, ostream) + + def do_POST(self): + self.do_proxy() + + def do_GET(self): + self.do_proxy() + + class ThreadingHTTPServer(ThreadingMixIn, HTTPServer): + pass + + # Server starts + log.printlog('Sogou Proxy Mini-Server', color='bold-green', ostream=ostream) + + try: + server = ThreadingHTTPServer(host, Handler) + except Exception as ex: + log.wtf("Socket error: %s" % ex, ostream) + exit(1) + host = server.server_address + + if network_env.upper() == 'CERNET': + proxy_host = 'h%s.edu.bj.ie.sogou.com' % random.randint(0, 10) + elif network_env.upper() == 'CTCNET': + proxy_host = 'h%s.ctc.bj.ie.sogou.com' % random.randint(0, 3) + elif network_env.upper() == 'CNCNET': + proxy_host = 'h%s.cnc.bj.ie.sogou.com' % random.randint(0, 3) + elif network_env.upper() == 'DXT': + proxy_host = 'h%s.dxt.bj.ie.sogou.com' % random.randint(0, 10) + else: + proxy_host = 'h%s.edu.bj.ie.sogou.com' % random.randint(0, 10) + + log.i('Remote host: %s' % log.underlined(proxy_host), ostream) + log.i('Proxy server running on %s' % + log.underlined("%s:%s" % host), ostream) + + return server diff --git a/src/you_get/version.py b/src/you_get/version.py index 7c09b706a8..4e98358315 100644 --- a/src/you_get/version.py +++ b/src/you_get/version.py @@ -2,5 +2,5 @@ __all__ = ['__version__', '__date__'] __name__ = 'you-get' -__version__ = '0.3.23' -__date__ = '2013-10-23' +__version__ = '0.3.24' +__date__ = '2013-10-30'