Merge branch 'master' of https://github.com/ivoras/daisy

README.md

@@ -56,7 +56,7 @@ Talks / presentations I gave about Daisy:
 ## ToDo
 
 * Add a [DHT](https://github.com/nictuku/dht) implementation for node discovery
-* Make creating new blockchains from scratch possible without changing the code
+* ✔ Make creating new blockchains from scratch possible without changing the code
 * Implement nicer error handling when replying to messages
 * Refactor db..., action... and blockchain... funcs into struct methods
 * Implement the "URL" encoding for block transfers: so the data in the JSON messages isn't block data, but an URL to the block data.

blockchain.go

@@ -47,7 +47,7 @@ var blockchainSubdirectory string
  * CreatorPublicKey|1:a3c07ef6cbee246f231a61ff36bbcd8e8563723e3703eb345ecdd933d7709ae2
  * Version|1
  *
- * Of these, only the Creator field is optional. By default, for new block, it is taken
+ * Of these, only the Creator field is optional. By default, for new blocks, it is taken
  * from the "BlockCreator" field in the pubkey metadata (if it exists).
  */
 
@@ -198,6 +198,11 @@ func blockchainInit(createDefault bool) {
 // Verifies the entire blockchain to see if there are errors.
 // TODO: Dynamic adding and revoking of key is not yet checked
 func blockchainVerifyEverything() error {
+	if cfg.faster {
+		log.Println("Skipping blockchain consistency checks")
+		return nil
+	}
+	log.Println("Verifying all the blocks...")
 	maxHeight := dbGetBlockchainHeight()
 	for height := 0; height <= maxHeight; height++ {
 		if height > 0 && height%1000 == 0 {

blockwebserver.go

@@ -0,0 +1,47 @@
+package main
+
+import (
+	"fmt"
+	"log"
+	"net/http"
+	"os"
+	"strconv"
+
+	"github.com/gorilla/mux"
+)
+
+func blockWebSendBlock(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+
+	blockHeight, err := strconv.Atoi(vars["height"])
+	if err != nil {
+		log.Println(vars)
+		w.WriteHeader(http.StatusBadRequest)
+		return
+	}
+
+	blockFilename := blockchainGetFilename(blockHeight)
+	if _, err := os.Stat(blockFilename); os.IsNotExist(err) {
+		w.WriteHeader(http.StatusNotFound)
+		log.Println("Block file not found:", blockFilename)
+		return
+	}
+
+	log.Println("Serving block", blockHeight)
+	w.Header().Set("Content-Type", "application/x-sqlite3")
+	w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%08x.db\"", blockHeight))
+	http.ServeFile(w, r, blockFilename)
+}
+
+func blockWebServer() {
+	r := mux.NewRouter()
+	r.HandleFunc("/block/{height}", blockWebSendBlock)
+
+	serverAddress := fmt.Sprintf(":%d", cfg.httpPort)
+
+	log.Println("HTTP listening on", serverAddress)
+	err := http.ListenAndServe(serverAddress, r)
+	if err != nil {
+		panic(err)
+	}
+}

chainparams.example.json

@@ -2,5 +2,6 @@
     "consensus_type": "PoA",
     "creator": "Ivan Voras <ivoras@gmail.com>",
     "genesis_block_timestamp": "2018-08-16T12:49:32+02:00",
-    "bootstrap_peers": [ "cosmos.ivoras.net:2017" ]
-}
+    "bootstrap_peers": [ "cosmos.ivoras.net:2017" ],
+    "description": "The Mighty Blockchain"
+}

chainparams.go

@@ -25,4 +25,7 @@ type ChainParams struct {
 	// Consensus algorithm used: "PoA", "PoW"
 	ConsensusTypeString string `json:"consensus_type"`
 	ConsensusType       int
+
+	// Description of the blockchain (e.g. its purpose)
+	Description string `json:"description"`
 }

cliactions.go

@@ -282,6 +282,10 @@ func actionNewChain(jsonFilename string) {
 	if err != nil {
 		log.Fatal(err)
 	}
+	err = dbSetMetaString(db, "Description", ncp.Description)
+	if err != nil {
+		log.Fatal(err)
+	}
 
 	if len(ncp.BootstrapPeers) > 0 {
 		// bootstrapPeers is required to be filled in before dbInit()

config.go

@@ -13,17 +13,23 @@ import (
 // DefaultP2PPort is the default TCP port for p2p connections
 const DefaultP2PPort = 2017
 
+// DefaultBlockWebServerPort is the default TCP port for the HTTP server
+const DefaultBlockWebServerPort = 2018
+
 // DefaultConfigFile is the default configuration filename
 const DefaultConfigFile = "/etc/daisy/config.json"
 
 // DefaultDataDir is the default data directory
 const DefaultDataDir = ".daisy"
 
 var cfg struct {
-	configFile string
-	P2pPort    int    `json:"p2p_port"`
-	DataDir    string `json:"data_dir"`
-	showHelp   bool
+	configFile     string
+	P2pPort        int    `json:"p2p_port"`
+	DataDir        string `json:"data_dir"`
+	httpPort       int
+	showHelp       bool
+	faster         bool
+	p2pBlockInline bool
 }
 
 // Initialises defaults, parses command line
@@ -36,6 +42,7 @@ func configInit() {
 
 	// Init defaults
 	cfg.P2pPort = DefaultP2PPort
+	cfg.httpPort = DefaultBlockWebServerPort
 
 	// Config file is parsed first
 	for i, arg := range os.Args {
@@ -52,8 +59,11 @@ func configInit() {
 
 	// Then override the configuration with command-line flags
 	flag.IntVar(&cfg.P2pPort, "port", cfg.P2pPort, "P2P port")
+	flag.IntVar(&cfg.httpPort, "http-port", cfg.httpPort, "HTTP port")
 	flag.StringVar(&cfg.DataDir, "dir", cfg.DataDir, "Data directory")
 	flag.BoolVar(&cfg.showHelp, "help", false, "Shows CLI usage information")
+	flag.BoolVar(&cfg.faster, "faster", false, "Be faster when starting up")
+	flag.BoolVar(&cfg.p2pBlockInline, "p2pblockinline", false, "Send blocks to peers inline instead of over HTTP")
 	flag.Parse()
 
 	if cfg.showHelp {

main.go

@@ -40,6 +40,7 @@ func main() {
 	go p2pCoordinator.Run()
 	go p2pServer()
 	go p2pClient()
+	go blockWebServer()
 
 	for {
 		select {

p2p.go

@@ -13,6 +13,7 @@ import (
 	"io/ioutil"
 	"log"
 	"net"
+	"net/http"
 	"os"
 	"sort"
 	"strconv"
@@ -239,7 +240,7 @@ func p2pServer() {
 			log.Fatalf("p2pServer l.Close: %v", err)
 		}
 	}()
-	log.Println("Listening on", serverAddress)
+	log.Println("P2P listening on", serverAddress)
 	for {
 		conn, err := l.Accept()
 		if err != nil {
@@ -541,33 +542,43 @@ func (p2pc *p2pConnection) handleGetBlock(msg StrIfMap) {
 		return
 	}
 	fileSize := st.Size()
-	f, err := os.Open(fileName)
-	if err != nil {
-		log.Println(err)
-		return
-	}
-	defer func() {
-		err = f.Close()
+
+	var msgBlockEncoding, msgBlockData string
+
+	if cfg.p2pBlockInline {
+		f, err := os.Open(fileName)
 		if err != nil {
-			log.Printf("handleGetBlock f.Close: %v", err)
+			log.Println(err)
+			return
 		}
-	}()
-	var zbuf bytes.Buffer
-	w := zlib.NewWriter(&zbuf)
-	written, err := io.Copy(w, f)
-	if err != nil {
-		log.Println(err)
-		return
-	}
-	if written != fileSize {
-		log.Println("Something broke when working with zlib:", written, "vs", fileSize)
-		return
-	}
-	err = w.Close()
-	if err != nil {
-		log.Panic(err)
+		defer func() {
+			err = f.Close()
+			if err != nil {
+				log.Printf("handleGetBlock f.Close: %v", err)
+			}
+		}()
+		var zbuf bytes.Buffer
+		w := zlib.NewWriter(&zbuf)
+		written, err := io.Copy(w, f)
+		if err != nil {
+			log.Println(err)
+			return
+		}
+		if written != fileSize {
+			log.Println("Something broke when working with zlib:", written, "vs", fileSize)
+			return
+		}
+		err = w.Close()
+		if err != nil {
+			log.Panic(err)
+		}
+		msgBlockEncoding = "zlib-base64"
+		msgBlockData = base64.StdEncoding.EncodeToString(zbuf.Bytes())
+	} else {
+		msgBlockEncoding = "http"
+		msgBlockData = fmt.Sprintf("http://%s:%d/block/%d", getLocalAddresses()[0], cfg.httpPort, dbb.Height)
 	}
-	b64block := base64.StdEncoding.EncodeToString(zbuf.Bytes())
+
 	respMsg := p2pMsgBlockStruct{
 		p2pMsgHeader: p2pMsgHeader{
 			P2pID: p2pEphemeralID,
@@ -576,8 +587,8 @@ func (p2pc *p2pConnection) handleGetBlock(msg StrIfMap) {
 		},
 		Hash:          hash,
 		HashSignature: hex.EncodeToString(dbb.HashSignature),
-		Encoding:      "zlib-base64",
-		Data:          b64block,
+		Encoding:      msgBlockEncoding,
+		Data:          msgBlockData,
 		Size:          fileSize,
 	}
 	p2pc.chanToPeer <- respMsg
@@ -615,50 +626,79 @@ func (p2pc *p2pConnection) handleBlock(msg StrIfMap) {
 		log.Printf("encoding: %v", err)
 		return
 	}
-	if encoding != "zlib-base64" {
-		log.Println("Unsupported encoding:", encoding)
-		return
-	}
-	zlibData, err := base64.StdEncoding.DecodeString(dataString)
-	if err != nil {
-		log.Println(err)
-		return
-	}
-	blockFile, err = ioutil.TempFile("", "daisy")
-	if err != nil {
-		log.Println(err)
-		return
-	}
-	defer func() {
-		err = blockFile.Close()
+	if encoding == "zlib-base64" {
+		zlibData, err := base64.StdEncoding.DecodeString(dataString)
+		if err != nil {
+			log.Println(err)
+			return
+		}
+		blockFile, err = ioutil.TempFile("", "daisy")
 		if err != nil {
-			log.Printf("handleBlock blockFile.Close: %v", err)
+			log.Println(err)
+			return
 		}
-		err = os.Remove(blockFile.Name())
+		defer func() {
+			err = blockFile.Close()
+			if err != nil {
+				log.Printf("handleBlock blockFile.Close: %v", err)
+			}
+			err = os.Remove(blockFile.Name())
+			if err != nil {
+				log.Printf("remove: %v", err)
+			}
+		}()
+		r, err := zlib.NewReader(bytes.NewReader(zlibData))
 		if err != nil {
-			log.Printf("remove: %v", err)
+			log.Println(err)
+			return
 		}
-	}()
-	r, err := zlib.NewReader(bytes.NewReader(zlibData))
-	if err != nil {
-		log.Println(err)
-		return
-	}
-	defer func() {
-		err = r.Close()
+		defer func() {
+			err = r.Close()
+			if err != nil {
+				log.Printf("handleBlock r.Close: %v", err)
+			}
+		}()
+		written, err := io.Copy(blockFile, r)
 		if err != nil {
-			log.Printf("handleBlock r.Close: %v", err)
+			log.Println(err)
+			return
 		}
-	}()
-	written, err := io.Copy(blockFile, r)
-	if err != nil {
-		log.Println(err)
-		return
-	}
-	if written != fileSize {
-		log.Println("Error decoding block: sizes don't match:", written, "vs", fileSize)
+		if written != fileSize {
+			log.Println("Error decoding block: sizes don't match:", written, "vs", fileSize)
+			return
+		}
+	} else if encoding == "http" {
+		blockFile, err = ioutil.TempFile("", "daisy")
+		if err != nil {
+			log.Println(err)
+			return
+		}
+		defer func() {
+			err = blockFile.Close()
+			if err != nil {
+				log.Printf("handleBlock blockFile.Close: %v", err)
+			}
+			err = os.Remove(blockFile.Name())
+			if err != nil {
+				log.Printf("remove: %v", err)
+			}
+		}()
+		resp, err := http.Get(dataString)
+		if err != nil {
+			log.Println("Error receiving block at", dataString, err)
+			return
+		}
+		defer resp.Body.Close()
+		written, err := io.Copy(blockFile, resp.Body)
+		if written != fileSize {
+			log.Println("Error decoding block: sizes don't match:", written, "vs", fileSize)
+			return
+		}
+	} else {
+		log.Println("Unknown block encoding:", encoding)
 		return
 	}
+
 	blk, err := OpenBlockFile(blockFile.Name())
 	if err != nil {
 		log.Println(p2pc.conn, err)