From 44c94a13a940c934c0198eaa8e4a7ced68b4353e Mon Sep 17 00:00:00 2001 From: Ivan Voras Date: Mon, 17 Dec 2018 21:10:22 +0100 Subject: [PATCH 1/4] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 310f490..86a7913 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,7 @@ Talks / presentations I gave about Daisy: ## ToDo * Add a [DHT](https://github.com/nictuku/dht) implementation for node discovery -* Make creating new blockchains from scratch possible without changing the code +* ✔ Make creating new blockchains from scratch possible without changing the code * Implement nicer error handling when replying to messages * Refactor db..., action... and blockchain... funcs into struct methods * Implement the "URL" encoding for block transfers: so the data in the JSON messages isn't block data, but an URL to the block data. From ea3576ed82d077711226185228b22a16eaea4c77 Mon Sep 17 00:00:00 2001 From: ivoras Date: Fri, 1 Feb 2019 21:07:13 +0100 Subject: [PATCH 2/4] Added the "Description" field to the genesis block & chainparams --- blockchain.go | 2 +- chainparams.example.json | 5 +++-- chainparams.go | 3 +++ cliactions.go | 4 ++++ 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/blockchain.go b/blockchain.go index f4c45b5..7633a6f 100644 --- a/blockchain.go +++ b/blockchain.go @@ -47,7 +47,7 @@ var blockchainSubdirectory string * CreatorPublicKey|1:a3c07ef6cbee246f231a61ff36bbcd8e8563723e3703eb345ecdd933d7709ae2 * Version|1 * - * Of these, only the Creator field is optional. By default, for new block, it is taken + * Of these, only the Creator field is optional. By default, for new blocks, it is taken * from the "BlockCreator" field in the pubkey metadata (if it exists). */ diff --git a/chainparams.example.json b/chainparams.example.json index 7ebb507..ea840e8 100644 --- a/chainparams.example.json +++ b/chainparams.example.json @@ -1,5 +1,6 @@ { "creator": "Ivan Voras ", "genesis_block_timestamp": "2018-08-16T12:49:32+02:00", - "bootstrap_peers": [ "cosmos.ivoras.net:2017" ] -} \ No newline at end of file + "bootstrap_peers": [ "cosmos.ivoras.net:2017" ], + "description": "The Mighty Blockchain" +} diff --git a/chainparams.go b/chainparams.go index 0199ba0..a5d5dd4 100644 --- a/chainparams.go +++ b/chainparams.go @@ -16,4 +16,7 @@ type ChainParams struct { // List of host:port string specifying default peers for this blockchain. If empty, the defaults are used. BootstrapPeers []string `json:"bootstrap_peers"` + + // Description of the blockchain (e.g. its purpose) + Description string `json:"description"` } diff --git a/cliactions.go b/cliactions.go index b405693..3fa1bca 100644 --- a/cliactions.go +++ b/cliactions.go @@ -282,6 +282,10 @@ func actionNewChain(jsonFilename string) { if err != nil { log.Fatal(err) } + err = dbSetMetaString(db, "Description", ncp.Description) + if err != nil { + log.Fatal(err) + } if len(ncp.BootstrapPeers) > 0 { // bootstrapPeers is required to be filled in before dbInit() From 7265c5ec8e6f538e3629882c11e93a6d8b2d5b5d Mon Sep 17 00:00:00 2001 From: ivoras Date: Fri, 1 Feb 2019 23:11:30 +0100 Subject: [PATCH 3/4] Implemented block file serving over HTTP --- blockchain.go | 5 +++++ blockwebserver.go | 47 +++++++++++++++++++++++++++++++++++++++++++++++ config.go | 5 +++++ main.go | 1 + p2p.go | 2 +- 5 files changed, 59 insertions(+), 1 deletion(-) create mode 100644 blockwebserver.go diff --git a/blockchain.go b/blockchain.go index 7633a6f..5dcc5fd 100644 --- a/blockchain.go +++ b/blockchain.go @@ -198,6 +198,11 @@ func blockchainInit(createDefault bool) { // Verifies the entire blockchain to see if there are errors. // TODO: Dynamic adding and revoking of key is not yet checked func blockchainVerifyEverything() error { + if cfg.faster { + log.Println("Skipping blockchain consistency checks") + return nil + } + log.Println("Verifying all the blocks...") maxHeight := dbGetBlockchainHeight() for height := 0; height <= maxHeight; height++ { if height > 0 && height%1000 == 0 { diff --git a/blockwebserver.go b/blockwebserver.go new file mode 100644 index 0000000..2378dc6 --- /dev/null +++ b/blockwebserver.go @@ -0,0 +1,47 @@ +package main + +import ( + "fmt" + "log" + "net/http" + "os" + "strconv" + + "github.com/gorilla/mux" +) + +func blockWebSendBlock(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + + blockHeight, err := strconv.Atoi(vars["height"]) + if err != nil { + log.Println(vars) + w.WriteHeader(http.StatusBadRequest) + return + } + + blockFilename := blockchainGetFilename(blockHeight) + if _, err := os.Stat(blockFilename); os.IsNotExist(err) { + w.WriteHeader(http.StatusNotFound) + log.Println("Block file not found:", blockFilename) + return + } + + log.Println("Serving block", blockHeight) + w.Header().Set("Content-Type", "application/x-sqlite3") + w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%08x.db\"", blockHeight)) + http.ServeFile(w, r, blockFilename) +} + +func blockWebServer() { + r := mux.NewRouter() + r.HandleFunc("/block/{height}", blockWebSendBlock) + + serverAddress := fmt.Sprintf(":%d", DefaultBlockWebServerPort) + + log.Println("HTTP listening on", serverAddress) + err := http.ListenAndServe(serverAddress, r) + if err != nil { + panic(err) + } +} diff --git a/config.go b/config.go index 9810c69..1bb5bcf 100644 --- a/config.go +++ b/config.go @@ -13,6 +13,9 @@ import ( // DefaultP2PPort is the default TCP port for p2p connections const DefaultP2PPort = 2017 +// DefaultBlockWebServerPort is the default TCP port for the HTTP server +const DefaultBlockWebServerPort = 2018 + // DefaultConfigFile is the default configuration filename const DefaultConfigFile = "/etc/daisy/config.json" @@ -24,6 +27,7 @@ var cfg struct { P2pPort int `json:"p2p_port"` DataDir string `json:"data_dir"` showHelp bool + faster bool } // Initialises defaults, parses command line @@ -54,6 +58,7 @@ func configInit() { flag.IntVar(&cfg.P2pPort, "port", cfg.P2pPort, "P2P port") flag.StringVar(&cfg.DataDir, "dir", cfg.DataDir, "Data directory") flag.BoolVar(&cfg.showHelp, "help", false, "Shows CLI usage information") + flag.BoolVar(&cfg.faster, "faster", false, "Be faster when starting up") flag.Parse() if cfg.showHelp { diff --git a/main.go b/main.go index eb3d9e9..d5f1650 100644 --- a/main.go +++ b/main.go @@ -40,6 +40,7 @@ func main() { go p2pCoordinator.Run() go p2pServer() go p2pClient() + go blockWebServer() for { select { diff --git a/p2p.go b/p2p.go index 3d3db79..40d4a34 100644 --- a/p2p.go +++ b/p2p.go @@ -239,7 +239,7 @@ func p2pServer() { log.Fatalf("p2pServer l.Close: %v", err) } }() - log.Println("Listening on", serverAddress) + log.Println("P2P listening on", serverAddress) for { conn, err := l.Accept() if err != nil { From d78af6e99b761b476bf32fd4f4516203f0fef3b1 Mon Sep 17 00:00:00 2001 From: ivoras Date: Fri, 1 Feb 2019 23:34:12 +0100 Subject: [PATCH 4/4] Implemented sending and receiving blocks over http instead of inline --- blockwebserver.go | 2 +- config.go | 15 +++-- p2p.go | 164 ++++++++++++++++++++++++++++------------------ 3 files changed, 113 insertions(+), 68 deletions(-) diff --git a/blockwebserver.go b/blockwebserver.go index 2378dc6..1f3af5f 100644 --- a/blockwebserver.go +++ b/blockwebserver.go @@ -37,7 +37,7 @@ func blockWebServer() { r := mux.NewRouter() r.HandleFunc("/block/{height}", blockWebSendBlock) - serverAddress := fmt.Sprintf(":%d", DefaultBlockWebServerPort) + serverAddress := fmt.Sprintf(":%d", cfg.httpPort) log.Println("HTTP listening on", serverAddress) err := http.ListenAndServe(serverAddress, r) diff --git a/config.go b/config.go index 1bb5bcf..f9b3ef3 100644 --- a/config.go +++ b/config.go @@ -23,11 +23,13 @@ const DefaultConfigFile = "/etc/daisy/config.json" const DefaultDataDir = ".daisy" var cfg struct { - configFile string - P2pPort int `json:"p2p_port"` - DataDir string `json:"data_dir"` - showHelp bool - faster bool + configFile string + P2pPort int `json:"p2p_port"` + DataDir string `json:"data_dir"` + httpPort int + showHelp bool + faster bool + p2pBlockInline bool } // Initialises defaults, parses command line @@ -40,6 +42,7 @@ func configInit() { // Init defaults cfg.P2pPort = DefaultP2PPort + cfg.httpPort = DefaultBlockWebServerPort // Config file is parsed first for i, arg := range os.Args { @@ -56,9 +59,11 @@ func configInit() { // Then override the configuration with command-line flags flag.IntVar(&cfg.P2pPort, "port", cfg.P2pPort, "P2P port") + flag.IntVar(&cfg.httpPort, "http-port", cfg.httpPort, "HTTP port") flag.StringVar(&cfg.DataDir, "dir", cfg.DataDir, "Data directory") flag.BoolVar(&cfg.showHelp, "help", false, "Shows CLI usage information") flag.BoolVar(&cfg.faster, "faster", false, "Be faster when starting up") + flag.BoolVar(&cfg.p2pBlockInline, "p2pblockinline", false, "Send blocks to peers inline instead of over HTTP") flag.Parse() if cfg.showHelp { diff --git a/p2p.go b/p2p.go index 40d4a34..e144dae 100644 --- a/p2p.go +++ b/p2p.go @@ -13,6 +13,7 @@ import ( "io/ioutil" "log" "net" + "net/http" "os" "sort" "strconv" @@ -541,33 +542,43 @@ func (p2pc *p2pConnection) handleGetBlock(msg StrIfMap) { return } fileSize := st.Size() - f, err := os.Open(fileName) - if err != nil { - log.Println(err) - return - } - defer func() { - err = f.Close() + + var msgBlockEncoding, msgBlockData string + + if cfg.p2pBlockInline { + f, err := os.Open(fileName) if err != nil { - log.Printf("handleGetBlock f.Close: %v", err) + log.Println(err) + return } - }() - var zbuf bytes.Buffer - w := zlib.NewWriter(&zbuf) - written, err := io.Copy(w, f) - if err != nil { - log.Println(err) - return - } - if written != fileSize { - log.Println("Something broke when working with zlib:", written, "vs", fileSize) - return - } - err = w.Close() - if err != nil { - log.Panic(err) + defer func() { + err = f.Close() + if err != nil { + log.Printf("handleGetBlock f.Close: %v", err) + } + }() + var zbuf bytes.Buffer + w := zlib.NewWriter(&zbuf) + written, err := io.Copy(w, f) + if err != nil { + log.Println(err) + return + } + if written != fileSize { + log.Println("Something broke when working with zlib:", written, "vs", fileSize) + return + } + err = w.Close() + if err != nil { + log.Panic(err) + } + msgBlockEncoding = "zlib-base64" + msgBlockData = base64.StdEncoding.EncodeToString(zbuf.Bytes()) + } else { + msgBlockEncoding = "http" + msgBlockData = fmt.Sprintf("http://%s:%d/block/%d", getLocalAddresses()[0], cfg.httpPort, dbb.Height) } - b64block := base64.StdEncoding.EncodeToString(zbuf.Bytes()) + respMsg := p2pMsgBlockStruct{ p2pMsgHeader: p2pMsgHeader{ P2pID: p2pEphemeralID, @@ -576,8 +587,8 @@ func (p2pc *p2pConnection) handleGetBlock(msg StrIfMap) { }, Hash: hash, HashSignature: hex.EncodeToString(dbb.HashSignature), - Encoding: "zlib-base64", - Data: b64block, + Encoding: msgBlockEncoding, + Data: msgBlockData, Size: fileSize, } p2pc.chanToPeer <- respMsg @@ -615,50 +626,79 @@ func (p2pc *p2pConnection) handleBlock(msg StrIfMap) { log.Printf("encoding: %v", err) return } - if encoding != "zlib-base64" { - log.Println("Unsupported encoding:", encoding) - return - } - zlibData, err := base64.StdEncoding.DecodeString(dataString) - if err != nil { - log.Println(err) - return - } - blockFile, err = ioutil.TempFile("", "daisy") - if err != nil { - log.Println(err) - return - } - defer func() { - err = blockFile.Close() + if encoding == "zlib-base64" { + zlibData, err := base64.StdEncoding.DecodeString(dataString) + if err != nil { + log.Println(err) + return + } + blockFile, err = ioutil.TempFile("", "daisy") if err != nil { - log.Printf("handleBlock blockFile.Close: %v", err) + log.Println(err) + return } - err = os.Remove(blockFile.Name()) + defer func() { + err = blockFile.Close() + if err != nil { + log.Printf("handleBlock blockFile.Close: %v", err) + } + err = os.Remove(blockFile.Name()) + if err != nil { + log.Printf("remove: %v", err) + } + }() + r, err := zlib.NewReader(bytes.NewReader(zlibData)) if err != nil { - log.Printf("remove: %v", err) + log.Println(err) + return } - }() - r, err := zlib.NewReader(bytes.NewReader(zlibData)) - if err != nil { - log.Println(err) - return - } - defer func() { - err = r.Close() + defer func() { + err = r.Close() + if err != nil { + log.Printf("handleBlock r.Close: %v", err) + } + }() + written, err := io.Copy(blockFile, r) if err != nil { - log.Printf("handleBlock r.Close: %v", err) + log.Println(err) + return } - }() - written, err := io.Copy(blockFile, r) - if err != nil { - log.Println(err) - return - } - if written != fileSize { - log.Println("Error decoding block: sizes don't match:", written, "vs", fileSize) + if written != fileSize { + log.Println("Error decoding block: sizes don't match:", written, "vs", fileSize) + return + } + } else if encoding == "http" { + blockFile, err = ioutil.TempFile("", "daisy") + if err != nil { + log.Println(err) + return + } + defer func() { + err = blockFile.Close() + if err != nil { + log.Printf("handleBlock blockFile.Close: %v", err) + } + err = os.Remove(blockFile.Name()) + if err != nil { + log.Printf("remove: %v", err) + } + }() + resp, err := http.Get(dataString) + if err != nil { + log.Println("Error receiving block at", dataString, err) + return + } + defer resp.Body.Close() + written, err := io.Copy(blockFile, resp.Body) + if written != fileSize { + log.Println("Error decoding block: sizes don't match:", written, "vs", fileSize) + return + } + } else { + log.Println("Unknown block encoding:", encoding) return } + blk, err := OpenBlockFile(blockFile.Name()) if err != nil { log.Println(p2pc.conn, err)