Skip to content

Commit 1b5a11b

Browse files
damies13damies13
committed
Not working yet
Exception: a bytes-like object is required, not 'str' Issue #8
1 parent a166e63 commit 1b5a11b

File tree

2 files changed

+185
-73
lines changed

2 files changed

+185
-73
lines changed

‎MyNoSQL/MyNoSQL.py

+184-72
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import uuid
22
import json
33
import lzma
4+
import gzip
45
import os
56
import datetime
67
import time
@@ -36,56 +37,73 @@ def do_HEAD(self):
3637

3738
def do_POST(self):
3839

39-
print("")
40-
print("--------------- <do_POST> ---------------")
40+
debuglvl = 5
41+
42+
db.debugmsg(debuglvl, "")
43+
db.debugmsg(debuglvl, "--------------- <do_POST> ---------------")
4144
threadstart = time.time()
4245
# default result
4346
httpcode = 404
4447
try:
4548
parsed_path = urllib.parse.urlparse(self.path)
46-
db.debugmsg(5, "parsed_path:", parsed_path)
49+
db.debugmsg(debuglvl, "parsed_path:", parsed_path)
4750
patharr = parsed_path.path.split("/")
48-
db.debugmsg(5, "patharr:", patharr)
51+
db.debugmsg(debuglvl, "patharr:", patharr)
4952

5053
message = "Unrecognised request: '{}'".format(parsed_path)
5154

5255
if (patharr[1].lower() in ["peer", "doc"]):
5356
httpcode = 200
5457
message = "OK"
5558

56-
db.debugmsg(5, "patharr[1].lower():", patharr[1].lower())
59+
db.debugmsg(debuglvl, "patharr[1].lower():", patharr[1].lower())
60+
db.debugmsg(debuglvl, "self.headers():", self.headers())
61+
62+
content_len = int(self.headers.get('Content-Length'))
63+
post_body = self.rfile.read(content_len)
64+
# content-encoding: gzip
65+
if self.headers.index("Content-Encoding")>0:
66+
contentencoding = self.headers.get("Content-Encoding")
67+
db.debugmsg(debuglvl, "contentencoding:", contentencoding)
68+
if contentencoding == "lzma":
69+
post_body = lzma.decompress(post_body)
70+
if contentencoding == "gzip":
71+
post_body = gzip.decompress(post_body)
5772

5873
if patharr[1].lower() == "peer":
59-
content_len = int(self.headers.get('Content-Length'))
60-
post_body = self.rfile.read(content_len)
74+
# content_len = int(self.headers.get('Content-Length'))
75+
# post_body = self.rfile.read(content_len)
76+
# lzma.compress(data_in) lzma.decompress(data_in)
77+
# if contentencoding.lower() == "lzma":
78+
# message = lzma.compress(message)
6179
doc = json.loads(post_body)
6280

63-
db.debugmsg(5, "doc:", doc)
81+
db.debugmsg(debuglvl, "doc:", doc)
6482

65-
db.debugmsg(5, "db:", db)
66-
db.debugmsg(5, "db.doc_id:", db.doc_id)
83+
db.debugmsg(debuglvl, "db:", db)
84+
db.debugmsg(debuglvl, "db.doc_id:", db.doc_id)
6785

6886
if "id" in doc:
6987
db._registerpeer(doc["id"])
7088

7189
saved = db._saveremotedoc(doc)
72-
db.debugmsg(5, "saved:", saved)
90+
db.debugmsg(debuglvl, "saved:", saved)
7391

7492
if patharr[1].lower() == "doc":
7593

76-
content_len = int(self.headers.get('Content-Length'))
77-
post_body = self.rfile.read(content_len)
94+
# content_len = int(self.headers.get('Content-Length'))
95+
# post_body = self.rfile.read(content_len)
7896
doc = json.loads(post_body)
7997

80-
db.debugmsg(5, "doc:", doc)
98+
db.debugmsg(debuglvl, "doc:", doc)
8199

82-
db.debugmsg(5, "db:", db)
83-
db.debugmsg(5, "db.doc_id:", db.doc_id)
100+
db.debugmsg(debuglvl, "db:", db)
101+
db.debugmsg(debuglvl, "db.doc_id:", db.doc_id)
84102

85103
if "id" in doc:
86104

87105
saved = db._saveremotedoc(doc)
88-
db.debugmsg(5, "saved:", saved)
106+
db.debugmsg(debuglvl, "saved:", saved)
89107

90108

91109

@@ -94,36 +112,61 @@ def do_POST(self):
94112
httpcode = 500
95113
message = str(e)
96114

97-
db.debugmsg(5, "httpcode:", httpcode, " message:", message)
115+
db.debugmsg(debuglvl, "httpcode:", httpcode, " message:", message)
116+
98117
self.send_response(httpcode)
118+
# content-encoding: gzip
119+
if db.contentencoding != None:
120+
self.send_header("Content-Encoding", db.contentencoding)
121+
# lzma.compress(data_in) lzma.decompress(data_in)
122+
if db.contentencoding.lower() == "lzma":
123+
smessage = bytearray(json.dumps(message).encode("utf8"))
124+
message = lzma.compress(smessage)
125+
if db.contentencoding.lower() == "gzip":
126+
smessage = bytearray(json.dumps(message).encode("utf8"))
127+
message = gzip.compress(smessage, compresslevel=9)
128+
129+
self.send_header("Content-Length", len(message))
130+
131+
99132
self.end_headers()
100-
self.wfile.write(bytes(message,"utf-8"))
133+
if db.contentencoding != None:
134+
db.debugmsg(debuglvl, "message:", message)
135+
self.wfile.write(message)
136+
else:
137+
self.wfile.write(bytes(message,"utf-8"))
101138
threadend = time.time()
102139
# base.debugmsg(5, parsed_path.path, " threadstart:", "%.3f" % threadstart, "threadend:", "%.3f" % threadend, "Time Taken:", "%.3f" % (threadend-threadstart))
103-
db.debugmsg(5, "%.3f" % (threadend-threadstart), "seconds for ", parsed_path.path)
104-
print("--------------- </do_POST> ---------------")
105-
print("")
140+
db.debugmsg(debuglvl, "%.3f" % (threadend-threadstart), "seconds for ", parsed_path.path)
141+
db.debugmsg(debuglvl, "--------------- </do_POST> ---------------")
142+
db.debugmsg(debuglvl, "")
106143
return
107144

108145
def do_GET(self):
109146

110-
print("")
111-
print("--------------- <do_GET> ---------------")
147+
debuglvl = 5
148+
149+
150+
db.debugmsg(debuglvl, "")
151+
db.debugmsg(debuglvl, "--------------- <do_GET> ---------------")
112152
threadstart = time.time()
113153
httpcode = 404
114154
message = "Not Found"
115155
try:
116156
parsed_path = urllib.parse.urlparse(self.path)
117-
db.debugmsg(5, "parsed_path:", parsed_path)
157+
db.debugmsg(debuglvl, "parsed_path:", parsed_path)
118158
patharr = parsed_path.path.split("/")
119-
db.debugmsg(5, "patharr:", patharr)
159+
db.debugmsg(debuglvl, "patharr:", patharr)
120160
if (patharr[1].lower() in ["peer", "index", "doc"]):
161+
162+
db.debugmsg(debuglvl, "self.headers():", self.headers)
163+
121164
httpcode = 200
122165
message = "OK"
123166
if patharr[1].lower() == "peer":
124-
db.debugmsg(5, "db:", db)
167+
db.debugmsg(debuglvl, "db:", db)
125168
doc = db.getselfdoc()
126-
db.debugmsg(5, "doc:", doc)
169+
db.debugmsg(debuglvl, "doc:", doc)
127170
# message = json.dumps(doc).encode("utf8")
128171
message = json.dumps(doc)
129172

@@ -157,14 +200,34 @@ def do_GET(self):
157200
db.debugmsg(5, "e:", e)
158201
httpcode = 500
159202
message = str(e)
203+
160204
self.send_response(httpcode)
205+
# content-encoding: gzip
206+
if db.contentencoding != None:
207+
self.send_header("Content-Encoding", db.contentencoding)
208+
# lzma.compress(data_in) lzma.decompress(data_in)
209+
if db.contentencoding.lower() == "lzma":
210+
smessage = bytearray(json.dumps(message).encode("utf8"))
211+
message = lzma.compress(smessage)
212+
if db.contentencoding.lower() == "gzip":
213+
smessage = bytearray(json.dumps(message).encode("utf8"))
214+
message = gzip.compress(smessage, compresslevel=9)
215+
216+
self.send_header("Content-Length", len(message))
217+
218+
219+
161220
self.end_headers()
162-
self.wfile.write(bytes(message,"utf-8"))
221+
if db.contentencoding != None:
222+
db.debugmsg(debuglvl, "message:", message)
223+
self.wfile.write(message)
224+
else:
225+
self.wfile.write(bytes(message,"utf-8"))
163226
threadend = time.time()
164227
# base.debugmsg(5, parsed_path.path, " threadstart:", "%.3f" % threadstart, "threadend:", "%.3f" % threadend, "Time Taken:", "%.3f" % (threadend-threadstart))
165-
db.debugmsg(5, "%.3f" % (threadend-threadstart), "seconds for ", parsed_path.path)
166-
print("--------------- </do_GET> ---------------")
167-
print("")
228+
db.debugmsg(debuglvl, "%.3f" % (threadend-threadstart), "seconds for ", parsed_path.path)
229+
db.debugmsg(debuglvl, "--------------- </do_GET> ---------------")
230+
db.debugmsg(debuglvl, "")
168231
return
169232

170233
def handle_http(self):
@@ -183,10 +246,16 @@ def setdb(self, db):
183246

184247
class MyNoSQL:
185248
version = "0.0.5"
186-
debuglvl = 7
249+
debuglvl = 5
187250
timeout=600
188251
defaultspeed=999999999
189252

253+
# content-encoding: gzip
254+
# contentencoding = "lzma"
255+
# contentencoding = "gzip"
256+
contentencoding = None
257+
258+
190259
# dbopen = False
191260

192261
def __init__(self):
@@ -405,39 +474,49 @@ def _peerupdates(self):
405474
self._getpeerupdates(peer['id'], selfdoc["dbmode"])
406475

407476
def _getpeerupdates(self, doc_id, mode):
408-
self.debugmsg(8, "doc_id:", doc_id)
409-
peerdoc = self.readdoc(doc_id)
410-
if "dbserver" in peerdoc:
411-
peerindexes = self._getremote(peerdoc["dbserver"] + "/Index")
412-
if peerindexes is not None:
413-
self.debugmsg(8, "peerindexes:", peerindexes)
414-
for index in peerindexes:
415-
self._updatepeerindex(peerdoc["dbserver"], peerdoc["dbmode"], index)
416-
# if mode == "Mirror":
417-
if mode != "Peer":
418-
peerrevs = self._getremote(peerdoc["dbserver"] + "/Index/rev")
419-
self.debugmsg(8, "peerrevs:", peerrevs)
420-
if peerrevs is not None:
421-
t = datetime.datetime.now()
422-
for peerrev in peerrevs:
423-
rdet = self._revdetail(peerrevs[peerrev])
424-
self.debugmsg(8, "rdet:", rdet)
425-
getremote = False
426-
if rdet["epoch"] > (t.timestamp() - ONE_YEAR):
427-
islocal = self._islocal(peerrev)
428-
if islocal is None:
429-
getremote = True
430-
else:
431-
ldet = self._revdetail(islocal)
432-
if rdet["number"] > ldet["number"]:
477+
doupdate = True
478+
debuglvl = 8
479+
self.debugmsg(debuglvl, "doc_id:", doc_id, " self.doc_id:", self.doc_id)
480+
if doc_id != self.doc_id:
481+
peerdoc = self.readdoc(doc_id)
482+
self.debugmsg(debuglvl, "peerdoc[dbserver]:", peerdoc["dbserver"], " self.selfurl:", self.selfurl)
483+
if peerdoc["dbserver"] == self.selfurl:
484+
doupdate = False
485+
else:
486+
doupdate = False
487+
488+
if doupdate:
489+
if "dbserver" in peerdoc:
490+
peerindexes = self._getremote(peerdoc["dbserver"] + "/Index")
491+
if peerindexes is not None:
492+
self.debugmsg(debuglvl, "peerindexes:", peerindexes)
493+
for index in peerindexes:
494+
self._updatepeerindex(peerdoc["dbserver"], peerdoc["dbmode"], index)
495+
# if mode == "Mirror":
496+
if mode != "Peer":
497+
peerrevs = self._getremote(peerdoc["dbserver"] + "/Index/rev")
498+
self.debugmsg(debuglvl, "peerrevs:", peerrevs)
499+
if peerrevs is not None:
500+
t = datetime.datetime.now()
501+
for peerrev in peerrevs:
502+
rdet = self._revdetail(peerrevs[peerrev])
503+
self.debugmsg(debuglvl, "rdet:", rdet)
504+
getremote = False
505+
if rdet["epoch"] > (t.timestamp() - ONE_YEAR):
506+
islocal = self._islocal(peerrev)
507+
if islocal is None:
433508
getremote = True
509+
else:
510+
ldet = self._revdetail(islocal)
511+
if rdet["number"] > ldet["number"]:
512+
getremote = True
434513

435-
self.debugmsg(8, "getremote:", getremote)
436-
if getremote:
437-
self.debugmsg(8, "_getremote url:", peerdoc["dbserver"] + "/Doc/" + peerrev)
438-
rdoc = self._getremote(peerdoc["dbserver"] + "/Doc/" + peerrev)
439-
self.debugmsg(8, "rdoc:", rdoc)
440-
self._saveremotedoc(rdoc)
514+
self.debugmsg(debuglvl, "getremote:", getremote)
515+
if getremote:
516+
self.debugmsg(debuglvl, "_getremote url:", peerdoc["dbserver"] + "/Doc/" + peerrev)
517+
rdoc = self._getremote(peerdoc["dbserver"] + "/Doc/" + peerrev)
518+
self.debugmsg(debuglvl, "rdoc:", rdoc)
519+
self._saveremotedoc(rdoc)
441520

442521

443522
def _updatepeerindex(self, peerurl, mode, index):
@@ -483,22 +562,55 @@ def _updatepeerindex(self, peerurl, mode, index):
483562

484563

485564
def _getremote(self, uri):
565+
debuglvl = 4
486566
try:
487-
r = requests.get(uri, timeout=self.timeout)
488-
self.debugmsg(9, "resp: ", r.status_code, "r.text:", r.text)
567+
self.debugmsg(debuglvl, "uri: ", uri)
568+
r = requests.get(uri, timeout=self.timeout, stream=True)
569+
# r = requests.get(uri, timeout=self.timeout)
570+
self.debugmsg(debuglvl, "r: ", r)
571+
self.debugmsg(debuglvl, "resp: ", r.status_code, "r.text:", r.text)
572+
self.debugmsg(debuglvl, "r.headers: ", r.headers)
573+
489574
if (r.status_code != requests.codes.ok):
490-
self.debugmsg(9, "r.status_code:", r.status_code, "!=", requests.codes.ok)
575+
self.debugmsg(debuglvl, "r.status_code:", r.status_code, "!=", requests.codes.ok)
491576
return None
492577
else:
493-
if "{" in r.text or "[" in r.text:
494-
jsonresp = json.loads(r.text)
495-
self.debugmsg(9, "jsonresp: ", jsonresp)
578+
self.debugmsg(debuglvl, "r.headers: ", r.headers)
579+
bodytxt = r.text
580+
# self.debugmsg(debuglvl, "r.text: ", r.text)
581+
# content-encoding: gzip
582+
# contentencoding = "lzma"
583+
cl = 0
584+
# content-length: 1016
585+
if "Content-Length" in r.headers:
586+
cl = r.headers["Content-Length"]
587+
# self.debugmsg(debuglvl, "cl: ", cl)
588+
if "Content-Encoding" in r.headers:
589+
590+
# r.raw.read(10)
591+
self.debugmsg(debuglvl, "cl: ", cl)
592+
self.debugmsg(debuglvl, "r.raw: ", r.raw)
593+
# bodyraw = r.raw.read(cl)
594+
# bodyraw = r.raw.read(10)
595+
bodyraw = r.content
596+
self.debugmsg(debuglvl, "bodyraw: ", bodyraw)
597+
598+
if r.headers["Content-Encoding"].lower() == "lzma":
599+
# bbodytxt = bytes(bodytxt, "utf-8")
600+
# bbodytxt = bodytxt.encode()
601+
bodytxt = lzma.decompress(bodyraw)
602+
# if r.headers["Content-Encoding"].lower() == "gzip":
603+
# bodytxt = gzip.decompress(bodytxt)
604+
605+
if "{" in bodytxt or "[" in bodytxt:
606+
jsonresp = json.loads(bodytxt)
607+
self.debugmsg(debuglvl, "jsonresp: ", jsonresp)
496608
return jsonresp
497609
else:
498-
return r.text
610+
return bodytxt
499611

500612
except Exception as e:
501-
self.debugmsg(8, "Exception:", e)
613+
self.debugmsg(debuglvl, "Exception:", e)
502614
return None
503615

504616
def _sendremote(self, uri, payload):

‎testing/ftest7.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111
db = MyNoSQL.MyNoSQL()
1212
db.opendb("mydb7")
1313
# print("db.db: ", db.db)
14-
db.debuglvl = 7
14+
db.debuglvl = 5
1515

1616
print("")
1717
dbself = db.getselfdoc()

0 commit comments

Comments
 (0)