diff --git a/README.md b/README.md index d98481f..eb5bc57 100644 --- a/README.md +++ b/README.md @@ -1,141 +1,106 @@ -# whois -* A Python package for retrieving WHOIS information of domains. -* This package will not support querying ip CIDR ranges or AS information -* It requires the whois cli component of your os to be installed: e.g. `/usr/bin/whois` on Linux - -## NOTE -* 2023-04-25: mboot - * when DannyCork returns he can decide on the future of this repo. - * in his absence future development will take place in: https://github.com/mboot-github/WhoisDomain - * and new pypi releases will come from: https://pypi.org/project/whoisdomain/ - * efforts will be made to keep the v1.x.y version of whoisdomain compatible with this repo - * changes will be verified and back copied also here for the time being - * starting 2024-02, this repo will be abandon-ware - -## Support - * Python 3.x is supported for x >= 9 - * Python 2.x IS NOT supported. +# whoisdomain + * A Python package for retrieving WHOIS information of DOMAIN'S ONLY. + * Python 2.x IS NOT supported. + * Currently no additional python packages need to be installed. -## Features - * Python wrapper for the "whois" cli command of your operating system. - * Simple interface to access parsed WHOIS data for a given domain. - * Able to extract data for all the popular TLDs (com, org, net, biz, info, pl, jp, uk, nz, ...). - * Query a WHOIS server directly instead of going through an intermediate web service like many others do. - * Works with Python >= 3.9 - * All dates as datetime objects. - * Possibility to cache results. - * Verbose output on stderr during debugging to see how the internal functions are doing their work - * raise a exception on Quota ecceeded type responses - * raise a exception on PrivateRegistry tld's where we know the tld and know we don't know anything - * allow for optional cleaning the whois response before extracting information - * optionally allow IDN's to be translated to Punycode - * optional specify the whois command on query(...,cmd="whois") as in: https://github.com/gen1us2k/python-whois/ - -## Dependencies - * please install also the command line "whois" of your distribution - * this library parses the output of the "whois" cli command of your operating system +--- -## Docker - * docker pull mbootgithub/whoisdomain:latest +## Notes -## Help Wanted -Your contributions are welcome, look for the Help wanted tag https://github.com/DannyCork/python-whois/labels/help%20wanted + * This package will not support querying ip CIDR ranges or AS information + * This was a copy of the original DanyCork 'whois'. + * Significantly refactored in 2023. + * The output is still compatible with DanyCork 'whois' -## Usage example +## Versioning -Install the cli `whois` of your operating system if it is not present already + * I will start versioning at 1.x.x + * the second item will be YYYYMMDD, + * the third item will start from 1 and be only used if more than one update will have to be done in one day. -Install `whois` package from your distribution (e.g apt install whois) +Versions `1.x.x` will keep the output compatible with Danny Cork until 2024-02-03 (February 2024) +## Releases - $pip install whois + * Releases are avalable at: [Pypi](https://pypi.org/project/whoisdomain/) - >>> import whois - >>> domain = whois.query('google.com') +Pypi releases can be installed with: - >>> print(domain.__dict__) - { - 'expiration_date': datetime.datetime(2020, 9, 14, 0, 0), - 'last_updated': datetime.datetime(2011, 7, 20, 0, 0), - 'registrar': 'MARKMONITOR INC.', - 'name': 'google.com', - 'creation_date': datetime.datetime(1997, 9, 15, 0, 0) - } + * `pip install whoisdomain` - >>> print(domain.name) - google.com +## Features + * See: [Features](docs/Features.md) - >>> print(domain.expiration_date) - 2020-09-14 00:00:00 +## Dependencies + * please install also the command line "whois" of your distribution as this library parses the output of the "whois" cli command of your operating system -## ccTLD & TLD support -see the file: ./whois/tld_regexpr.py -or call whois.validTlds() +### Notes for Mac users + * it has been observed that the default cli whois on Mac is showing each forward step in its output, this makes parsing the result very unreliable. + * using a brew install whois will give in general better results. -## Issues - * Raise an issue https://github.com/DannyCork/python-whois/issues/new +## Docker release + * See [Docker](docs/Docker.md) -## Changes: 2022-06-09: maarten_boot: - * the returned list of name_servers is now a sorted unique list and not a set - * the help function whois.validTlds() now outputs the true tld with dots +## Usage example + * See [Usage](docs/Usage.md) -## 2022-09-27: maarten_boot - * add test2.py to replace test.py - * ./test2.py -h will show the possible usage - * all tests from the original program are now files in the ./tests directory - * test can be done on all supported tld's with -a or --all and limitest by regex with -r or --reg= +## whoisdomain + * the cli `whoisdomain` is documented in [whoisdomain-cli](docs/whoisdomain-cli.md) -## 2022-11-04: maarten_boot - * add support for Iana example.com, example.net +## ccTLD & TLD support -## 2022-11-07: maarten_boot - * add testing against static known data in dir: ./testdata//output - * test.sh will test all domains in testdata without actually calling whois, the input data is instead read from testdata//input +Most `tld's` are now autodetected via IANA root db, see the Analizer directory +and `make suggest`. -## 2022-11-11: maarten_boot + * see the file: [tld_regexpr](./whoisdomain/tldDb/tld_regexpr.py) + * for python use: `whoisdomain.validTlds()` + * for cli use `whoisdomain -S` - * add support for returning the raw data from the whois command: flag include_raw_whois_text - * add support for handling unsupported domains via whois raw text only: flag return_raw_text_for_unsupported_tld +--- -## 2023-01-18: sorrowless +## Support + * Python 3.x is supported for x >= 9 + * Python 2.x IS NOT supported. - * add an opportunity to specify maximum cache age +## Author's + * See: [Authors](docs/Authors.md) -## 2023-01-25: maarten_boot +--- - * convert the tld file to a Dict, we now no longer need a mappper for python keywords or second level domains. - * utf8 level domains also need no mapper anymore an can be added as is with a translation to xn-- - * added xn-- tlds for all known utf-8 domains we currently have - * we can now add new domains on the fly or change them: whois.mergeExternalDictWithRegex(aDictToOverride) see example exampleExtend.py +## Updates + * see [Updates](docs/Updates.md) for a full history of changes. + * Only the latest update is mentioned here -## 2023-01-27: maarten_boot +### 1.20230906.1 + * introduce parsing based on functions + * allow contextual search in splitted data and plain data + * allow contextual search based on earlier result + * fix a few tld to return the proper registrant string (not nic handle) - * add autodetect via iana tld file (this has only tld's) - * add a central collection of all compiled regexes and reuse them: REG_COLLECTION_BY_KEY in _0_init_tld.py - * refresh testdata now that tld has dot instead of _ if more then one level - * add additional strings meaning domain does not exist +### 1.20230913.1 + * if you have installed `tld` (pip install tld) you can enable withPublicSuffix=True to process untill you reach the pseudo tld. + * the public_suffix info is added if available (and if requested) + * example case is: ./test2.py -d www.dublin.airport.aero --withPublicSuffix -## 2023-02-02: maarten_boot +### 1.20230913.3 + * fix re.NOFLAGS, it is not compatible with 3.9, it appears in 3.11 - * whois.QuotaStringsAdd(str) to add additional strings for over quota detection. whois.QuotaStrings() lists the current configured strings - * whois.NoneStringsAdd(str) to add additional string for NoSuchDomainExists detection (whois.query() retuning None). whois.NoneStrings() lsts the current configured strings - * suppress messages to stderr if not verbose=True +## 1.20230917.1 + * prepare work on pylint + * switch to logging: all verbose is currently log.debug(); to show set LOGLEVEL=DEBUG before calling, see Makefile: make test + * experimental: add extractServers: bool default False; when true we will try to extract the "redirect info chain" on rcf1036/whois and jwhois for linux/darwin + * add missing option to query(), test in production environment done -## 2023-07-20: maarten_boot +## 1.20231102.1 + * fix from kazet for .pl tld. - * sync with https://github.com/mboot-github/WhoisDomain; 1.20230720.1; (gov.tr), (com.ru, msk.ru, spb.ru), (option to preserve partial output after timeout) - * sync with https://github.com/mboot-github/WhoisDomain; 1.20230720.2; add t_test hint support; fix some server hints +## 1.20231115.1 + New tld's and removal of a few tlds no longer supported at iana -## 2023-08-21: mboot-github (maarten_boot) + * abb, bw, bn, crown, crs, fj (does not work), gp (does not work), weir, realtor, post, mw, pf (a strange one), iq (gives timout), mm, int, hm (does not work) - * abandon any python below 3.9 (mypy compatibilities) - * major refactor into more object based approch and parameterContext - * allow custom caching backends (e.g. redis, dbm, ...) +--- -## 2023-09-22 see new paramaters in whois/context/parameterContext.oy +## in progress - * Sync with latest whoisdomain - * Allow cleaning up the http(s) info in the status response. - * Allow correlation with tld (pip install tld) public_suffix. - * Allow display of what whois-servers were used until we reach the final item. diff --git a/docs/Authors.md b/docs/Authors.md new file mode 100644 index 0000000..7aac7c0 --- /dev/null +++ b/docs/Authors.md @@ -0,0 +1,7 @@ +# Author's + + * this is a rename copy of original work done in: https://github.com/DannyCork/python-whois + * the project is also related to the project: https://github.com/gen1us2k/python-whois + * both seem derived from a older google.code site: https://code.google.com/archive/p/python-whois + * aside from the original authors, many others already contributed to these repositories + * if authors/contributors prefer to be named explicitly, they can add a line in [Historical.txt](Historical.txt) diff --git a/docs/Docker.md b/docs/Docker.md new file mode 100644 index 0000000..dae9666 --- /dev/null +++ b/docs/Docker.md @@ -0,0 +1,11 @@ +# Docker + +[Docker](https://hub.docker.com/r/mbootgithub/whoisdomain) + + * docker pull mbootgithub/whoisdomain:latest + * docker run mbootgithub/whoisdomain -V # show version + * docker run mbootgithub/whoisdomain -d google.com # run one domain + * docker run mbootgithub/whoisdomain -a # run all tld + * docker run mbootgithub/whoisdomain -d google.com -j | jq -r . # run one domains , output in json and reformat with jq + * docker run mbootgithub/whoisdomain -d google.com -j | jq -r '.expiration_date' # output only expire date + * docker run mbootgithub/whoisdomain -d google.com -j | jq -r '[ .expiration_date, .creation_date ] diff --git a/docs/Features.md b/docs/Features.md new file mode 100644 index 0000000..a7551f2 --- /dev/null +++ b/docs/Features.md @@ -0,0 +1,20 @@ +# Features + * Python wrapper for the "whois" cli command of your operating system. + * Simple interface to access parsed WHOIS data for a given domain. + * Able to extract data for all the popular TLDs (com, org, net, biz, info, pl, jp, uk, nz, ...). + * Query a WHOIS server directly instead of going through an intermediate web service like many others do. + * Works with Python >= 3.9 + * All dates as datetime objects. + * Possibility to cache results. + * Verbose output on stderr during debugging to see how the internal functions are doing their work + * raise a exception on Quota ecceeded type responses + * raise a exception on PrivateRegistry tld's where we know the tld and know we don't know anything + * allow for optional cleaning the response before extracting information + * optionally allow IDN's to be translated to Punycode + * optional specify the whois command on query(...,cmd="whois") as in: https://github.com/gen1us2k/python-whois/ + * the module is now 'mypy --strict' clean + * the module now also exports a cli command domainwhois + * both the module and the cli now support showing the version with lib:whois.getVersion() or cli:whoisdomain -V + * the whoisdomain can now output json data (one per domain: e.g 'whoisdomain -d google.com -j' ) + * withRedacted: bool = False has been added to query(), if set to True any redacted fields will now be shown also (also supported in the cli whoisdomain as --withRedacted) + * a analizer directory is presend in the github repo that will be used to look for new IANA tls's currently unsupported but maching known whois servers diff --git a/docs/Updates.md b/docs/Updates.md new file mode 100644 index 0000000..1c7a042 --- /dev/null +++ b/docs/Updates.md @@ -0,0 +1,106 @@ +# Updates +## 1.20230627.2 + * add Kenia proper whois server and known second level domains +## 1.20230627.3 + * add rw tld proper whois server and second level ; restore mistakenly deleted .toml file +## 1.20230627.3 + * additional kenia second level domains +## 1.20230712.2 + * tld .edu now can have up to 10 nameservers; remove action on pull request +## 1.20230717.1 + * add tld: com.ru, msk.ru, spb.ru (all have a test documented), also update the tld: ru, the newlines are not needed. +## 1.20230717.2 + * add option to parse partial result after timout has occurred (parse_partial_response:bool default False); this will need `stdbuf` installed otherwise it will fail +## 1.20230718.3 + * fix typo in whois server hint for tld: ru +## 1.20230720.1 + * add gov.tr; switch off status:available and status:free as None response, we should not interprete the result by default (we can add a option later) +## 1.20230720.2 + * fix server hints for derived second level "xxx.tr", add processing "_test" hints during 'test2.py -a' + * add external caching framework that can be overridden for use of your own caching implementation + * renaming various vars to mak them more verbose + * preparing for capturing all parameters in one object and parring that object around instead of many arguments in methods/functions + * switch to json so we dont need a additional dependency in ParamContext + * finish rework args to ParameterContext, split of domain as file +## 1.20230803.1 + * frenzy refactor-release +## 1.20230804.1 + * testing +## 1.20230804.2 + * testing after remove of leading dot in rw second level domains +## 1.20230804.3 + * simplefy cache implementation after feedback from `baderdean` + * "more lembas bread", refactor parse and query + * remove option to typecheck CACHE_STUB, use try/catch/exit instead, does not work when timout happens, removed ;-( + * refactor doQuery create processWhoisDomainRequest, split of lastWhois +## 1.20230806.1 + * testing done, prep new release: "more lembas bread" + * bug found with the default timeout: if no timeout is specified the program fails: all pypi releases before 2023-07-17 yanked +## 1.20230807.1 + * fix default timeout + * add DummyCache, DBMCache, RedisCache with simple test in testCache.py, testing custom cache options +## 1.20230811.1 + * replace type hint | with Union for py3.9 compat; switch off experimental redis tools + * switch off 3.[6-8] minimal is 3.9 we test against + * start working on dataContext; + * add more \_test items; reorder parts of tld_regexpr; + * propagate all meta domains servers as they are not inherited, testing , some domains have been retracted mboot; 2023-08-23; + * add suggestion from baderdean to parse fr domains with more focus on ORGANISATION + * 2023-08-24: mboot: more \_test added to tld + * verify all \_test on whois.nic. \_test: nic. fix where needed; remove some abandoned tld's +## 1.20230824.1 + * mboot; to combine all new tests and changes, "the galloping Chutzpah release" +## 1.20230824.5 + * mboot; fix missing module in whl + * restore python 3.6 test as i still use it on one remaining app with python 3.6 (make testP36) + * finalize verification of all tld's in iana, add test where this can be auto generated from whois.nic. 2023-08-28; mboot +## 1.20230829.1 + * mboot; all \_test now work, using analizer tool to verify that iana tld db web site and tl-regexpr match + * add DEBUG to all verbose strings + * remove tldString and dList and domain , all go via dc (dataContect) now + * run tests and add new TODO + * moving all TLD_RE activities to tldInfo.py, and all exported helper funcs to helpers.py + * thinking about adding more complicated nested regex extractors to target contact info + * start with dependency inject: parser is passed as arg + * add cli interface to dependency inject, rightsize after test + * finish dependency inject move Domain create outside + * prep for other types or regex; all simple regex strings in tld_regexpr.py now need R() around them + * use currying to make all regex strings into function cal in whoisParser.py; all regexes in tld_regexpr.py are now converted on import to function calls via R() + * update tld: sk to use contextual extract, test with google.sk + * add findFromToAndLookForWithFindFirst contextual search based on a previous findFirst, used in "fr" tld, example google.fr, {} is used to add to fromStr + +## 1.20230904.1 + * only on pypi-test + +--- + +## 1.20230906.1 + * introduce parsing based on functions + * allow contextual search in splitted data and plain data + * allow contextual search based on earlier result + * fix a few tld to return the proper registrant string (not nic handle) + * introduce parsing based on functions, allow contextual search in splitted data and plain data, allow contextual search based on earlier result; fix a few tld to return the proper registrant string (not nic handle) + +### 1.20230906.1 + * introduce parsing based on functions + * allow contextual search in splitted data and plain data + * allow contextual search based on earlier result + * fix a few tld to return the proper registrant string (not nic handle) + +### 1.20230913.1 + * if you have installed `tld` (pip install tld) you can enable withPublicSuffix=True to process untill you reach the pseudo tld. + * the public_suffix info is added if available (and if requested) + * example case is: ./test2.py -d www.dublin.airport.aero --withPublicSuffix + +### 1.20230913.3 + * fix re.NOFLAGS, it is not compatible with 3.9, it appears in 3.11 + +--- + +## in progress + + * prepare work on pylint + * switch to logging: all verbose is currently log.debug(); to show set LOGLEVEL=DEBUG before calling, see Makefile: make test + + + diff --git a/docs/Usage.md b/docs/Usage.md new file mode 100644 index 0000000..d3b2f61 --- /dev/null +++ b/docs/Usage.md @@ -0,0 +1,34 @@ +# Usage + +## Requirements + + * Install the cli `whois` of your operating system if it is not present already, + * Debian / Ubuntu: + * `sudo apt install whois` + * Fedora/Centos/Rocky: + * `sudo yum install whois` + +## whois used in python (compatible with the Danny Cork version) + +### example for fedora 37 + + + sudo yum install whois + pip install whoisdomain + + python + # to make it compatible with Danny_Cork whois + >>> import whoisdomain as whois + >>> d = whois.query('google.com') + >>> print(d.__dict__) + + {'name': 'google.com', 'tld': 'com', 'registrar': 'MarkMonitor, Inc.', 'registrant_country': 'US', 'creation_date': datetime.datetime(1997, 9, 15, 9, 0), 'expiration_date': datetime.datetime(2028, 9, 13, 9, 0), 'last_updated': datetime.datetime(2019, 9, 9, 17, 39, 4), 'status': 'clientUpdateProhibited (https://www.icann.org/epp#clientUpdateProhibited)', 'statuses': ['clientDeleteProhibited (https://www.icann.org/epp#clientDeleteProhibited)', 'clientTransferProhibited (https://www.icann.org/epp#clientTransferProhibited)', 'clientUpdateProhibited (https://www.icann.org/epp#clientUpdateProhibited)', 'serverDeleteProhibited (https://www.icann.org/epp#serverDeleteProhibited)', 'serverTransferProhibited (https://www.icann.org/epp#serverTransferProhibited)', 'serverUpdateProhibited (https://www.icann.org/epp#serverUpdateProhibited)'], 'dnssec': False, 'name_servers': ['ns1.google.com', 'ns2.google.com', 'ns3.google.com', 'ns4.google.com'], 'registrant': 'Google LLC', 'emails': ['abusecomplaints@markmonitor.com', 'whoisrequest@markmonitor.com']} + + >>> print (d.expiration_date) + 2028-09-13 09:00:00 + + >>> print(d.name) + google.com + + >>> print (d.creation_date) + 1997-09-15 09:00:00 diff --git a/docs/_config.yml b/docs/_config.yml new file mode 100644 index 0000000..8bfdc9c --- /dev/null +++ b/docs/_config.yml @@ -0,0 +1,10 @@ +lsi: false +safe: true +source: [your repo's top level directory] +incremental: false +highlighter: rouge +gist: + noscript: false +kramdown: + math_engine: mathjax + syntax_highlighter: rouge diff --git a/docs/index.html b/docs/index.html new file mode 100644 index 0000000..621ef4a --- /dev/null +++ b/docs/index.html @@ -0,0 +1,107 @@ +

EntryPoint

+ +The main entry point into the library module: + +
+def query(
+    domain: str,
+    force: bool = False,
+    cache_file: Optional[str] = None,
+    cache_age: int = 60 * 60 * 48,
+    slow_down: int = 0,
+    ignore_returncode: bool = False,
+    server: Optional[str] = None,
+    verbose: bool = False,
+    with_cleanup_results: bool = False,
+    internationalized: bool = False,
+    include_raw_whois_text: bool = False,
+    return_raw_text_for_unsupported_tld: bool = False,
+    timeout: Optional[float] = None,
+    cmd: str = "whois",
+) -> Optional[Domain]:
+
+ +

domain

+ +

force

+ +

cache_file

+ +

cache_age

+ +

slow_down

+ +

ignore_returncode

+ +

server

+ +

verbose

+ +

with_cleanup_results

+ +

internationalized

+ +

include_raw_whois_text

+ +

return_raw_text_for_unsupported_tld

+ +

timeout

+ +

cmd

+ +

The command line interface to the module

+
+whoisdomain
+    [ -v | --verbose ]
+        # set verbose to True, this will be forwarded to whois.query
+
+    [ -I | --IgnoreReturncode ]
+        # sets the IgnoreReturncode to True, this will be forwarded to whois.query
+
+    [ -a | --all]
+        # test all existing tld currently supported,
+
+    [ -d  | --domain =  " ]
+        # only analyze the given domains
+        # the option can be repeated to specify more then one domain
+
+    [ -f  | --file =  " ]
+        # use the named file to test all domains (one domain per line)
+        # lines starting with # or empty lines are skipped, anything after the domain is ignored
+        # the option can be repeated to specify more then one file
+
+    [ -D  | --Directory =  " ]
+        # use the named directory, ald use all files ending in .txt as files containing domains
+        # files are processed as in the -f option so comments and empty lines are skipped
+        # the option can be repeated to specify more then one directory
+
+    [ -p | --print ]
+    also print text containing the raw output of whois
+
+    [ -R | --Ruleset ]
+    dump the ruleset for the tld and exit
+
+    [ -S | --SupportedTld ]
+    print all supported top level domains we know and exit
+
+    [ -C  | --Cleanup  ]
+    read the input file specified and run the same cleanup as in whois.query , then exit
+
+    # options are exclusive and without any options the whoisdomain program does nothing
+
+    # test one specific file with verbose and IgnoreReturncode
+    example: whoisdomain -v -I -f tests/ok-domains.txt 2>2 >out
+
+    # test one specific directory with verbose and IgnoreReturncode
+    example: whoisdomain -v -I -D tests 2>2 >out
+
+    # test two domains with verbose and IgnoreReturncode
+    example: whoisdomain -v -I -d meta.org -d meta.com 2>2 >out
+
+    # test all supported tld's with verbose and IgnoreReturncode
+    example: whoisdomain -v -I -a 2>2 >out
+
+    # test nothing
+    example: whoisdomain -v -I 2>2 >out
+
+
diff --git a/docs/whoisdomain-cli.md b/docs/whoisdomain-cli.md new file mode 100644 index 0000000..3f058a2 --- /dev/null +++ b/docs/whoisdomain-cli.md @@ -0,0 +1,119 @@ +# WhoisDomain as CLI command + +## example for `fedora 37` + + sudo yum install whois + pip3 install whoisdomain + whoisdomain -d google.com + + test domain: <<<<<<<<<< google.com >>>>>>>>>>>>>>>>>>>> + name 'google.com' + tld 'com' + registrar 'MarkMonitor, Inc.' + registrant_country 'US' + creation_date 1997-09-15 09:00:00 + expiration_date 2028-09-13 09:00:00 + last_updated 2019-09-09 17:39:04 + status 'clientUpdateProhibited (https://www.icann.org/epp#clientUpdateProhibited)' + statuses ['clientDeleteProhibited (https://www.icann.org/epp#clientDeleteProhibited)', 'clientTransferProhibited (https://www.icann.org/epp#clientTransferProhibited)', 'clientUpdateProhibited (https://www.icann.org/epp#clientUpdateProhibited)', 'serverDeleteProhibited (https://www.icann.org/epp#serverDeleteProhibited)', 'serverTransferProhibited (https://www.icann.org/epp#serverTransferProhibited)', 'serverUpdateProhibited (https://www.icann.org/epp#serverUpdateProhibited)'] + dnssec False + name_servers ['ns1.google.com', 'ns2.google.com', 'ns3.google.com', 'ns4.google.com'] + registrant 'Google LLC' + emails ['abusecomplaints@markmonitor.com', 'whoisrequest@markmonitor.com'] + + +## A short intro into the cli whoisdomain command + + whoisdomain + [ -h | --usage ] + print this text and exit + + [ -V | --Version ] + print the build version string + and exit + + [ -S | --SupportedTld ] + print all known top level domains + and exit + + [ -a | --all] + test all existing tld currently supported + and exit + + [ -f | --file = " ] + use the named file to test all domains (one domain per line) + lines starting with # or empty lines are skipped, anything after the domain is ignored + the option can be repeated to specify more then one file + exits after processing all the files + + [ -D | --Directory = " ] + use the named directory, ald use all files ending in .txt as files containing domains + files are processed as in the -f option so comments and empty lines are skipped + the option can be repeated to specify more then one directory + exits after processing all the dirs + + [ -d | --domain = " ] + only analyze the given domains + the option can be repeated to specify more domain's + + [ -v | --verbose ] + set verbose to True, + verbose output will be printed on stderr only + + [ -j | --json ] + print each result as json + + [ -I | --IgnoreReturncode ] + sets the IgnoreReturncode to True, + + [ -p | --print ] + also print text containing the raw output of the cli whois + + [ -R | --Ruleset ] + dump the ruleset for the requested tld and exit + should be combined with -d to specify tld's + + [ -C | --Cleanup ] + read the input file specified and run the same cleanup as in whois.query, + then exit + + # test two domains with verbose and IgnoreReturncode + example: whoisdomain -v -I -d meta.org -d meta.com + + # test all supported tld's with verbose and IgnoreReturncode + example: whoisdomain -v -I -a + + # test one specific file with verbose and IgnoreReturncode + example: whoisdomain -v -I -f tests/ok-domains.txt + + # test one specific directory with verbose and IgnoreReturncode + example: whoisdomain -v -I -D tests + + + +## example for Json output with the cli `whoisdomain` + + whoisdomain -j -d hello.xyz | jq -r . + + { + "name": "hello.xyz", + "tld": "xyz", + "registrar": "Namecheap", + "registrant_country": "IS", + "creation_date": "2014-03-20 15:01:22", + "expiration_date": "2024-03-20 23:59:59", + "last_updated": "2023-03-14 09:24:32", + "status": "clientTransferProhibited https://icann.org/epp#clientTransferProhibited", + "statuses": [ + "clientTransferProhibited https://icann.org/epp#clientTransferProhibited" + ], + "dnssec": false, + "name_servers": [ + "dns1.registrar-servers.com", + "dns2.registrar-servers.com" + ], + "registrant": "Privacy service provided by Withheld for Privacy ehf", + "emails": [ + "abuse@namecheap.com" + ] + } diff --git a/reformat-code.sh b/reformat-code.sh index 2fed941..bf5d28c 100755 --- a/reformat-code.sh +++ b/reformat-code.sh @@ -1,9 +1,11 @@ #! /bin/bash +LL=160 + doIt() { - black --line-length 160 whois - pylama --max-line-length 160 whois/ + black --line-length $LL whois + pylama --max-line-length $LL whois/ } main() diff --git a/whois/cache/redisCache.py b/whois/cache/redisCache.py index ee6d31a..f74293d 100644 --- a/whois/cache/redisCache.py +++ b/whois/cache/redisCache.py @@ -39,7 +39,6 @@ def get( self, keyString: str, ) -> Optional[str]: - data = self.redis.get(keyString) if data: sdata: str = data.decode("utf-8") @@ -51,7 +50,6 @@ def put( keyString: str, data: str, ) -> str: - self.redis.set( keyString, bytes(data, "utf-8"), diff --git a/whois/cache/simpleCacheBase.py b/whois/cache/simpleCacheBase.py index 9db2d4d..e503418 100755 --- a/whois/cache/simpleCacheBase.py +++ b/whois/cache/simpleCacheBase.py @@ -31,7 +31,6 @@ def put( keyString: str, data: str, ) -> str: - # store the currentTime and data tuple (time, data) self.memCache[keyString] = ( int(time.time()), @@ -43,7 +42,6 @@ def get( self, keyString: str, ) -> Optional[str]: - cData = self.memCache.get(keyString) if cData is None: return None diff --git a/whois/main.py b/whois/main.py index 0a5fddd..6b075ca 100755 --- a/whois/main.py +++ b/whois/main.py @@ -17,8 +17,6 @@ import whois -# import whoisdomain as whois # to be compatible with dannycork - log = logging.getLogger(__name__) logging.basicConfig(level=os.environ.get("LOGLEVEL", "INFO")) diff --git a/whois/tldDb/finders.py b/whois/tldDb/finders.py index 07a0381..9c7b2ea 100644 --- a/whois/tldDb/finders.py +++ b/whois/tldDb/finders.py @@ -133,7 +133,6 @@ def findFromToAndLookForWithFindFirst( ignoreCase: bool = True, verbose: bool = False, ) -> Callable[[str, List[str], bool], List[str]]: - # look for a particular string like R() with find first # then build a from ,to context using the result from findFirst (google.fr is a example) # but limit the context we look in @@ -198,7 +197,6 @@ def findInSplitedLookForHavingFindFirst( ignoreCase: bool = True, verbose: bool = False, ) -> Callable[[str, List[str], bool], List[str]]: - # requires splitted data def xfindInSplitedLookForHavingFindFirst( textStr: str, diff --git a/whois/tldDb/tld_regexpr.py b/whois/tldDb/tld_regexpr.py index dd367f4..839cbb0 100644 --- a/whois/tldDb/tld_regexpr.py +++ b/whois/tldDb/tld_regexpr.py @@ -735,6 +735,7 @@ def xStr(what: str, times: int = 1, firstMandatory: bool = True) -> str: "registrar": R(r"\nREGISTRAR:\s*(.+)\n"), "creation_date": R(r"\ncreated:\s*(.+)\n"), "updated_date": R(r"\nlast modified:\s*(.+)\n"), + # "expiration_date": R(r"\noption expiration date:\s*(.+)\n"), # If no "option expiration date:" is present, use "renewal date:", but only # if it's not not followed by "option expiration date:". google.pl is the test case here "expiration_date": R(r"(?:\noption expiration date:|renewal date:(?!(?:.|\n)*\noption expiration date:))\s*(.+)\n"), @@ -1377,6 +1378,26 @@ def xStr(what: str, times: int = 1, firstMandatory: bool = True) -> str: "registrant": r"Registrant:\s*([^\n]*)\n", } +# unknown tld pf, pf, pf, pf, whois.registry.pf, +ZZ["pf"] = { + "_server": "whois.registry.pf", + "_test": "registry.pf", + "status": R(r"Status\s*:(\S*)\n"), + "creation_date": R(r"Created[^:]*:(\S*)\n"), + "expiration_date": R(r"Expire[^:]*:(\S*)\n"), + "updated_date": R(r"Last renewed[^:]*:(\S*)\n"), + "name_servers": R(r"Name Server[^:]*:(\S*)\n"), + "domain_name": R(r"Informations about\s*'([^']*)'\s*:"), + "registrar": None, + "registrant_country": None, +} +# Informations about 'registry.pf' : +# Status : active +# Created (JJ/MM/AAAA) : 04/02/2013 +# Last renewed (JJ/MM/AAAA) : 02/02/2023 +# Expire (JJ/MM/AAAA) : 02/02/2024 +# Name server 1 : ns1.mana.pf +# Name server 2 : ns2.mana.pf # ====================================== # ====================================== # ====================================== @@ -1552,7 +1573,7 @@ def xStr(what: str, times: int = 1, firstMandatory: bool = True) -> str: ZZ["catholic"] = {"_server": "whois.nic.catholic", "extend": "com", "_test": "nic.catholic"} ZZ["ca.ug"] = {"extend": "ug"} ZZ["cba"] = {"_server": "whois.nic.cba", "extend": "com", "_test": "nic.cba"} -ZZ["cbs"] = {"_server": "whois.afilias-srs.net", "extend": "com"} +# ZZ["cbs"] = {"_server": "whois.afilias-srs.net", "extend": "com"} ZZ["cd"] = {"extend": "ac", "_server": "whois.nic.cd", "registrant_country": R(r"Registrant\s+Country:\s+(.+)"), "_test": "nic.cd"} ZZ["center"] = {"extend": "_donuts", "_server": "whois.donuts.co"} ZZ["ceo"] = {"extend": "_centralnic", "_server": "whois.centralnic.com"} @@ -1572,7 +1593,7 @@ def xStr(what: str, times: int = 1, firstMandatory: bool = True) -> str: ZZ["cipriani"] = {"_server": "whois.afilias-srs.net", "extend": "com"} ZZ["circle"] = {"_server": "whois.nic.circle", "extend": "com", "_test": "nic.circle"} ZZ["ci"] = {"_server": "whois.nic.ci", "extend": "com", "_test": "nic.ci"} -ZZ["cityeats"] = {"_server": "whois.nic.cityeats", "extend": "com", "_test": "nic.cityeats"} +# ZZ["cityeats"] = {"_server": "whois.nic.cityeats", "extend": "com", "_test": "nic.cityeats"} ZZ["city"] = {"extend": "_donuts", "_server": "whois.donuts.co"} ZZ["claims"] = {"extend": "_donuts", "_server": "whois.donuts.co"} ZZ["cleaning"] = {"extend": "_donuts", "_server": "whois.donuts.co"} @@ -1778,7 +1799,7 @@ def xStr(what: str, times: int = 1, firstMandatory: bool = True) -> str: ZZ["fresenius"] = {"_server": "whois.nic.fresenius", "extend": "com", "_test": "nic.fresenius"} ZZ["frl"] = {"extend": "_centralnic", "_server": "whois.centralnic.com"} ZZ["frogans"] = {"_server": "whois.nic.frogans", "extend": "com", "_test": "nic.frogans"} -ZZ["frontdoor"] = {"_server": "whois.nic.frontdoor", "extend": "com", "_test": "nic.frontdoor"} +# ZZ["frontdoor"] = {"_server": "whois.nic.frontdoor", "extend": "com", "_test": "nic.frontdoor"} ZZ["fujitsu"] = {"_server": "whois.nic.gmo", "extend": "com"} ZZ["fund"] = {"extend": "_donuts", "_server": "whois.donuts.co"} ZZ["fun"] = {"extend": "_centralnic", "_server": "whois.centralnic.com"} @@ -2315,7 +2336,7 @@ def xStr(what: str, times: int = 1, firstMandatory: bool = True) -> str: ZZ["shopping"] = {"extend": "_donuts", "_server": "whois.donuts.co"} ZZ["shouji"] = {"extend": "_teleinfo", "_server": "whois.teleinfo.cn"} ZZ["show"] = {"extend": "_donuts", "_server": "whois.donuts.co"} -ZZ["showtime"] = {"_server": "whois.afilias-srs.net", "extend": "com"} +# ZZ["showtime"] = {"_server": "whois.afilias-srs.net", "extend": "com"} ZZ["silk"] = {"_server": "whois.nic.silk", "extend": "com", "_test": "nic.silk"} ZZ["sina"] = {"_server": "whois.nic.sina", "extend": "com", "_test": "nic.sina"} ZZ["singles"] = {"extend": "_donuts", "_server": "whois.donuts.co"} @@ -2666,7 +2687,7 @@ def xStr(what: str, times: int = 1, firstMandatory: bool = True) -> str: ZZ["नट"] = {"_server": "whois.nic.xn--c2br7g", "extend": "xn--c2br7g"} ZZ["भरत"] = {"_server": "whois.registry.in", "extend": "in"} ZZ["भरत"] = {"_server": "whois.registry.in", "extend": "in"} -ZZ["भरतम"] = {"_server": "whois.registry.in", "extend": "in"} +# ZZ["भरतम"] = {"_server": "whois.registry.in", "extend": "in"} ZZ["सगठन"] = {"_server": "whois.nic.xn--i1b6b1a6a2e", "extend": "xn--i1b6b1a6a2e"} ZZ["ভরত"] = {"_server": "whois.registry.in", "extend": "in"} ZZ["ভৰত"] = {"_server": "whois.registry.in", "extend": "in"} @@ -2818,7 +2839,7 @@ def xStr(what: str, times: int = 1, firstMandatory: bool = True) -> str: ZZ["jpmorgan"] = {"_privateRegistry": True} # no whois server found in iana ZZ["jprs"] = {"_privateRegistry": True} # no whois server found in iana ZZ["kh"] = {"_privateRegistry": True} # no whois server found in iana -ZZ["kinder"] = {"_privateRegistry": True} # no whois server found in iana +# ZZ["kinder"] = {"_privateRegistry": True} # no whois server found in iana ZZ["km"] = {"_privateRegistry": True} # no whois server found in iana ZZ["kp"] = {"_privateRegistry": True} # no whois server found in iana ZZ["kpmg"] = {"_privateRegistry": True} # no whois server found in iana @@ -2865,7 +2886,7 @@ def xStr(what: str, times: int = 1, firstMandatory: bool = True) -> str: ZZ["praxi"] = {"_privateRegistry": True} # no whois server found in iana ZZ["pru"] = {"_privateRegistry": True} # no whois server found in iana ZZ["prudential"] = {"_privateRegistry": True} # no whois server found in iana -ZZ["rocher"] = {"_privateRegistry": True} # no whois server found in iana +# ZZ["rocher"] = {"_privateRegistry": True} # no whois server found in iana ZZ["sakura"] = {"_privateRegistry": True} # no whois server found in iana ZZ["sas"] = {"_privateRegistry": True} # no whois server found in iana ZZ["sener"] = {"_privateRegistry": True} # no whois server found in iana @@ -2922,41 +2943,37 @@ def xStr(what: str, times: int = 1, firstMandatory: bool = True) -> str: ZZ["onion"] = {"_privateRegistry": True} # this is a special case https://tools.ietf.org/html/rfc7686 +# 2023-11-14 # unknown tld abb, abb, abb, abb, whois.nic.abb, -# unknown tld arpa, arpa, arpa, arpa, whois.iana.org, +ZZ["abb"] = {"_server": "whois.nic.abb", "_test": "nic.abb", "extend": "com"} # unknown tld bn, bn, bn, bn, whois.bnnic.bn, +ZZ["bn"] = {"_server": "whois.bnnic.bn", "_test": "bnnic.bn", "extend": "com"} +ZZ["gov.bn"] = {"_server": "whois.bnnic.bn", "_test": "egc.gov.bn", "extend": "com"} # unknown tld bw, bw, bw, bw, whois.nic.net.bw, +ZZ["bw"] = {"_server": "whois.nic.net.bw", "_test": "net.bw", "extend": "com"} # unknown tld crown, crown, crown, crown, whois.nic.crown, +ZZ["crown"] = {"_server": "whois.nic.crown", "_test": "nic.crown", "extend": "com"} # unknown tld crs, crs, crs, crs, whois.nic.crs, +ZZ["crs"] = {"_server": "whois.nic.crs", "_test": "nic.crs", "extend": "com"} # unknown tld fj, fj, fj, fj, www.whois.fj, +ZZ["fj"] = {"_server": "whois.nic.fj", "_test": "nic.fj", "extend": "com"} # actually not working but server exists (dns) # unknown tld gp, gp, gp, gp, whois.nic.gp, +ZZ["gp"] = {"_server": "whois.nic.gp", "_test": "nic.gp", "extend": "com"} # actually not working but server exists (dns) # unknown tld hm, hm, hm, hm, whois.registry.hm, +ZZ["hm"] = {"_server": "whois.registry.hm", "_test": "registry.hm", "extend": "com"} # actually not working but server exists (dns) # unknown tld il, il, il, il, whois.isoc.org.il, # unknown tld int, int, int, int, whois.iana.org, -# unknown tld iq, iq, iq, iq, whois.cmc.iq, -# unknown tld mm, mm, mm, mm, whois.registry.gov.mm, +ZZ["int"] = {"_server": "whois.iana.org", "_test": "eu.int", "extend": "cz"} + +# unknown tld iq, iq, iq, iq, whois.cmc.iq +ZZ["iq"] = {"_server": "whois.cmc.iq", "_test": "cmc.iq", "extend": "com"} +# unknown tld mm, mm, mm, mm, whois.registry.gov.mm +ZZ["mm"] = {"_server": "whois.registry.gov.mm", "_test": "registry.gov.mm", "extend": "com"} # unknown tld mw, mw, mw, mw, whois.nic.mw, -# unknown tld pf, pf, pf, pf, whois.registry.pf, +ZZ["mw"] = {"_server": "whois.nic.mw", "_test": "nic.mw", "extend": "fr"} # unknown tld post, post, post, post, whois.dotpostregistry.net, +ZZ["post"] = {"_server": "whois.dotpostregistry.net", "_test": "us.post", "extend": "com"} # unknown tld realtor, realtor, realtor, realtor, whois.nic.realtor, +ZZ["realtor"] = {"_server": "whois.nic.realtor", "_test": "nic.realtor", "extend": "com"} # unknown tld weir, weir, weir, weir, whois.nic.weir, -# unknown tld xn--4dbrk0ce, ישראל, ישראל, ישראל, whois.isoc.org.il, -# unknown tld xn--55qw42g, 公益, 公益, 公益, whois.conac.cn, -# unknown tld xn--80ao21a, қаз, қаз, қаз, whois.nic.kz, -# unknown tld xn--90a3ac, срб, срб, срб, whois.rnids.rs, -# unknown tld xn--90ae, бг, бг, бг, whois.imena.bg, -# unknown tld xn--90ais, бел, бел, бел, whois.cctld.by, -# unknown tld xn--d1acj3b, дети, дети, дети, whois.nic.xn--d1acj3b, -# unknown tld xn--j1amh, укр, укр, укр, whois.dotukr.com, -# unknown tld xn--lgbbat1ad8j, الجزائر, الجزائر, الجزائر, whois.nic.dz, -# unknown tld xn--mgba3a4f16a, ایران, ایران, ایران, whois.nic.ir, -# unknown tld xn--mgbaam7a8h, امارات, امارات, امارات, whois.aeda.net.ae, -# unknown tld xn--mgberp4a5d4ar, السعودية, السعودية, السعودية, whois.nic.net.sa, -# unknown tld xn--mgbtx2b, عراق, عراق, عراق, whois.cmc.iq, -# unknown tld xn--mgbx4cd0ab, مليسيا, مليسيا, مليسيا, whois.mynic.my, -# unknown tld xn--node, გე, გე, გე, whois.itdc.ge, -# unknown tld xn--pgbs0dh, تونس, تونس, تونس, whois.ati.tn, -# unknown tld xn--q7ce6a, ລາວ, ລາວ, ລາວ, whois.nic.la, -# unknown tld xn--y9a3aq, հայ, հայ, հայ, whois.amnic.net, -# unknown tld xn--ygbi2ammx, فلسطين, فلسطين, فلسطين, whois.pnina.ps, -# unknown tld xn--zfr164b, 政务, 政务, 政务, whois.conac.cn, +ZZ["weir"] = {"_server": "whois.nic.weir", "_test": "nic.weir", "extend": "com"} diff --git a/whois/version.py b/whois/version.py old mode 100755 new mode 100644 index 5edf3ef..c2479ab --- a/whois/version.py +++ b/whois/version.py @@ -1,2 +1,2 @@ """This module only makes the version available for dynamic versioning""" -VERSION = "1.20230917.1" +VERSION = "1.20231115.1" diff --git a/whois/whoisCliInterface.py b/whois/whoisCliInterface.py index a174ead..b5f5425 100755 --- a/whois/whoisCliInterface.py +++ b/whois/whoisCliInterface.py @@ -131,12 +131,13 @@ def _runWhoisCliOnThisOs(self) -> str: stderr=subprocess.STDOUT, env={"LANG": "en"} if self.domain.endswith(".jp") else None, ) as self.processHandle: - msg = f"timout: {self.pc.timeout}" log.debug(msg) try: - self.rawWhoisResultString = self.processHandle.communicate(timeout=self.pc.timeout,)[ + self.rawWhoisResultString = self.processHandle.communicate( + timeout=self.pc.timeout, + )[ 0 ].decode(errors="ignore") except subprocess.TimeoutExpired as ex: