Merge pull request #224 from root-gg/1.2.1

1.2.1
root-gg · Oct 19, 2017 · 9ff6861 · 9ff6861
2 parents d44bb58 + b412713
commit 9ff6861
Show file tree

Hide file tree

Showing 604 changed files with 14,056 additions and 14,086 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -1,9 +1,13 @@
 language: go
 
+sudo: required
+
 go:
-  - 1.6.2
+  - 1.9.1
 
 before_install:
+  # Update node_js version : https://github.com/travis-ci/travis-ci/issues/7108
+  - rm -rf ~/.nvm && git clone https://github.com/creationix/nvm.git ~/.nvm && (cd ~/.nvm && git checkout `git describe --abbrev=0 --tags`) && source ~/.nvm/nvm.sh && nvm install stable
   - npm install -g bower
 
 before_script:

diff --git a/Makefile b/Makefile
@@ -24,7 +24,7 @@
 # THE SOFTWARE.
 ###
 
-RELEASE_VERSION="1.2"
+RELEASE_VERSION="1.2.1"
 RELEASE_DIR="release/plik-$(RELEASE_VERSION)"
 RELEASE_TARGETS=darwin-386 darwin-amd64 freebsd-386 \
 freebsd-amd64 linux-386 linux-amd64 linux-arm openbsd-386 \
@@ -36,6 +36,14 @@ GOHOSTARCH=`go env GOHOSTARCH`
 DEBROOT_SERVER=debs/server
 DEBROOT_CLIENT=debs/client
 
+race_detector = GORACE="halt_on_error=1" go build -race
+ifdef ENABLE_RACE_DETECTOR
+	build = $(race_detector)
+else
+	build = go build
+endif
+test: build = $(race_detector)
+
 all: clean clean-frontend frontend clients server
 
 ###
@@ -52,7 +60,7 @@ frontend:
 ###
 server:
 	@server/gen_build_info.sh $(RELEASE_VERSION)
-	@cd server && go build -o plikd ./
+	@cd server && $(build) -o plikd ./
 
 ###
 # Build plik server for all architectures
@@ -68,7 +76,7 @@ servers: frontend
 		if [ $$GOOS = "windows" ] ; then SERVER_PATH=$$SERVER_DIR/plikd.exe ; fi ; \
 		if [ -e $$SERVER_PATH ] ; then continue ; fi ; \
 		echo "Compiling plik server for $$target to $$SERVER_PATH"; \
-		go build -o $$SERVER_PATH ;	\
+		$(build) -o $$SERVER_PATH ;	\
 	done
 
 
@@ -86,7 +94,7 @@ utils: servers
             if [ $$GOOS = "windows" ] ; then UTIL_PATH=$$UTIL_DIR/$$UTIL_BASE.exe ; fi ; \
             if [ -e $$UTIL_PATH ] ; then continue ; fi ; \
             echo "Compiling plik util file2bolt for $$target to $$UTIL_PATH"; \
-            go build -o $$UTIL_PATH $$util ; \
+            $(build) -o $$UTIL_PATH $$util ; \
         done ; \
 	done
 
@@ -96,7 +104,7 @@ utils: servers
 ###
 client:
 	@server/gen_build_info.sh $(RELEASE_VERSION)
-	@cd client && go build -o plik ./
+	@cd client && $(build) -o plik ./
 
 ###
 # Build plik client for all architectures
@@ -113,7 +121,7 @@ clients:
 		if [ $$GOOS = "windows" ] ; then CLIENT_PATH=$$CLIENT_DIR/plik.exe ; fi ; \
 		if [ -e $$CLIENT_PATH ] ; then continue ; fi ; \
 		echo "Compiling plik client for $$target to $$CLIENT_PATH"; \
-		go build -o $$CLIENT_PATH ; \
+		$(build) -o $$CLIENT_PATH ; \
 		md5sum $$CLIENT_PATH | awk '{print $$1}' > $$CLIENT_MD5; \
 	done
 	@mkdir -p clients/bash && cp client/plik.sh clients/bash
@@ -184,6 +192,7 @@ release-template: clean frontend clients
 	@mkdir -p $(RELEASE_DIR)/server/utils
 
 	@cp -R clients $(RELEASE_DIR)
+	@cp -R changelog $(RELEASE_DIR)
 	@cp -R server/plikd.cfg $(RELEASE_DIR)/server
 	@cp -R server/public/css $(RELEASE_DIR)/server/public
 	@cp -R server/public/fonts $(RELEASE_DIR)/server/public
@@ -239,38 +248,37 @@ releases: release-template servers utils
 # Run tests and sanity checks
 ###
 test:
-
+	@if curl -s 127.0.0.1:8080 > /dev/null ; then echo "Plik server probably already running" && exit 1 ; fi
 	@server/gen_build_info.sh $(RELEASE_VERSION)
 	@ERR="" ; for directory in server client ; do \
 		cd $$directory; \
 		echo -n "go test $$directory : "; \
-		TEST=`go test ./... 2>&1`; \
+		TEST=`go test -race ./... 2>&1`; \
 		if [ $$? = 0 ] ; then echo "OK" ; else echo "$$TEST" | grep -v "no test files" | grep -v "^\[" && ERR="1"; fi ; \
 		echo "go fmt $$directory : "; \
-		for file in $$(find -name "*.go" | grep -v Godeps ); do \
+		for file in $$(find -name "*.go" | grep -v vendor ); do \
 			echo -n " - file $$file : " ; \
 			FMT=`gofmt -l $$file` ; \
 			if [ "$$FMT" = "" ] ; then echo "OK" ; else echo "FAIL" && ERR="1" ; fi ; \
 		done; \
 		echo -n "go vet $$directory : "; \
-		VET=`go vet ./... 2>&1`; \
-		if [ $$? = 0 ] ; then echo "OK" ; else echo "FAIL" && echo "$$VET" && ERR="1" ; fi ; \
-		echo -n "go lint $$directory : "; \
-		LINT=`golint ./...`; \
-		if [ "$$LINT" = "" ] ; then echo "OK" ; else echo "FAIL" && echo "$$LINT" && ERR="1" ; fi ; \
+		for file in $$(find -name "*.go" | grep -v vendor ); do \
+			echo -n " - file $$file : " ; \
+			FMT=`go vet $$file` ; \
+			if [ "$$FMT" = "" ] ; then echo "OK" ; else echo "FAIL" && ERR="1" ; fi ; \
+		done; \
+		echo -n "golint $$directory : "; \
+		for file in $$(find -name "*.go" | grep -v vendor ); do \
+			echo -n " - file $$file : " ; \
+			FMT=`golint $$file` ; \
+			if [ "$$FMT" = "" ] ; then echo "OK" ; else echo "FAIL" && ERR="1" ; fi ; \
+		done; \
 		cd - 2>&1 > /dev/null; \
 	done ; if [ "$$ERR" = "1" ] ; then exit 1 ; fi
 	@echo "cli client integration tests :\n" && cd client && ./test.sh
 
 ###
-# Remove frontend build files
-###
-clean-frontend:
-	@rm -rf server/public/bower_components
-	@rm -rf server/public/public
-
-###
-# Remove all build files
+# Remove server build files
 ###
 clean:
 	@rm -rf server/common/version.go
@@ -282,10 +290,17 @@ clean:
 	@rm -rf release
 	@rm -rf releases
 
+###
+# Remove frontend build files
+###
+clean-frontend:
+	@rm -rf server/public/bower_components
+	@rm -rf server/public/public
+
 ###
 # Remove all build files and node modules
 ###
-clean-all: clean
+clean-all: clean clean-frontend
 	@rm -rf server/public/node_modules
 
 ###

diff --git a/README.md b/README.md
@@ -9,7 +9,6 @@ Plik is a scalable & friendly temporary file upload system ( wetransfer like ) i
 ### Main features
    - Multiple data backends : File, OpenStack Swift, WeedFS
    - Multiple metadata backends : File, MongoDB, Bolt
-   - Shorten backends : Shorten upload urls (is.gd && w000t.me available)
    - OneShot : Files are destructed after the first download
    - Stream : Files are streamed from the uploader to the downloader (nothing stored server side)  
    - Removable : Give the ability to the uploader to remove files at any time
@@ -21,34 +20,34 @@ Plik is a scalable & friendly temporary file upload system ( wetransfer like ) i
    - Upload restriction : Source IP / Token
 
 ### Version
-1.2
+1.2.1
 
 ### Installation
 
 ##### From release
 To run plik, it's very simple :
 ```sh
-$ wget https://github.com/root-gg/plik/releases/download/1.2/plik-1.2-linux-64bits.tar.gz
-$ tar xzvf plik-1.2-linux-64bits.tar.gz
-$ cd plik-1.2/server
+$ wget https://github.com/root-gg/plik/releases/download/1.2.1/plik-1.2.1-linux-64bits.tar.gz
+$ tar xzvf plik-1.2.1-linux-64bits.tar.gz
+$ cd plik-1.2.1/server
 $ ./plikd
 ```
 Et voilà ! You now have a fully functional instance of plik running on http://127.0.0.1:8080.  
 You can edit server/plikd.cfg to adapt the configuration to your needs (ports, ssl, ttl, backends params,...)
 
-##### From root.gg Debian repository
+##### From root.gg Debian repository
 
 Configure root.gg repository and install server and/or client
 ```
-    wget -O - http://mir.root.gg/gg.key | apt-key add -
-    echo "deb http://mir.root.gg/ $(lsb_release --codename --short) main" > /etc/apt/sources.list.d/root.gg.list
-    apt-get update
-    apt-get install plikd plik
+wget -O - http://mir.root.gg/gg.key | apt-key add -
+echo "deb http://mir.root.gg/ $(lsb_release --codename --short) main" > /etc/apt/sources.list.d/root.gg.list
+apt-get update
+apt-get install plikd plik
 ```
 
 Edit server configuration at /etc/plikd.cfg and start the server 
 ```
-    service plikd start
+service plikd start
 ```
 
 ##### From sources
@@ -114,7 +113,7 @@ Options:
 
 For example to create directory tar.gz archive and encrypt it with openssl :
 ```
-$plik -a -s mydirectory/
+$ plik -a -s mydirectory/
 Passphrase : 30ICoKdFeoKaKNdnFf36n0kMH
 Upload successfully created : 
     https://127.0.0.1:8080/#/?id=0KfNj6eMb93ilCrl
@@ -125,9 +124,9 @@ Commands :
 curl -s 'https://127.0.0.1:8080/file/0KfNj6eMb93ilCrl/q73tEBEqM04b22GP/mydirectory.tar.gz' | openssl aes-256-cbc -d -pass pass:30ICoKdFeoKaKNdnFf36n0kMH | tar xvf - --gzip
 ```
 
-Client configuration and preferences are stored at ~/.plikrc ( overridable with PLIKRC environement variable )
+Client configuration and preferences are stored at ~/.plikrc or /etc/plik/plikrc ( overridable with PLIKRC environement variable )
 
-### Available data backends
+###  Available data backends
 
 Plik is shipped with multiple data backend for uploaded files and metadata backend for the upload metadata.
 
@@ -143,7 +142,7 @@ Openstack Swift is a highly available, distributed, eventually consistent object
 
 SeaweedFS is a simple and highly scalable distributed file system.
 
-### Available metadata backends
+### Available metadata backends
 
  - File metadata backend : (DEPRECATED)
 
@@ -174,15 +173,17 @@ Suitable for distributed / High Availability deployment.
 ### Authentication
 
 Plik can authenticate users using Google and/or OVH API. 
-Once authenticated the only call Plik will ever make to those API is get the user ID, name and email. 
+Once authenticated the only call Plik will ever make to those API is to get the user ID, name and email. 
 Plik will never forward any upload data or metadata to any third party.   
 If source IP address restriction is enabled, user accounts can only be created from trusted IPs. But then 
-authenticated users can upload files without source IP restriction.   
+authenticated users can upload files without source IP restriction.
+It is also possible to deny unauthenticated uploads totally.
 
    - **Google** :
       - You'll need to create a new application in the [Google Developper Console](https://console.developers.google.com)
       - You'll be handed a Google API ClientID and a Google API ClientSecret that you'll need to put in the plikd.cfg file.
       - Do not forget to whitelist valid origin and redirect url ( https://yourdomain/auth/google/callback ) for your domain.
+      - It is possible to whitelist only one or more email domains.
 
    - **OVH** :
       - You'll need to create a new application in the OVH API : https://eu.api.ovh.com/createApp/
@@ -211,6 +212,10 @@ Plik comes with a simple Dockerfile that allows you to run it in a container :
 
 See the [Plik Docker reference](documentation/docker.md)
 
+Plik also comes with some useful scripts to test backends in standalone docker instances :
+
+See the [Plik Docker backend testing](testing)
+
 ### FAQ
 
 * Why is stream mode broken in multiple instance deployement ?
@@ -275,11 +280,11 @@ You might want to install the "nginx-extras" Debian package with built-in HttpCh
 And add in your server configuration :
 
 ```sh
-        chunkin on;
-        error_page 411 = @my_411_error;
-        location @my_411_error {
-                chunkin_resume;
-        }
+chunkin on;
+error_page 411 = @my_411_error;
+location @my_411_error {
+        chunkin_resume;
+}
 ```
 
 * How to disable nginx buffering ?
@@ -288,19 +293,28 @@ By default nginx buffers large HTTP requests and reponses to a temporary file. T
 
 Detailed documentation : http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffering
 ```
-   proxy_buffering off;
-   proxy_request_buffering off;
-   proxy_http_version 1.1;
-   proxy_buffer_size 1M;
-   proxy_buffers 8 1M;
-   client_body_buffer_size 1M;
+proxy_buffering off;
+proxy_request_buffering off;
+proxy_http_version 1.1;
+proxy_buffer_size 1M;
+proxy_buffers 8 1M;
+client_body_buffer_size 1M;
 ```
 
-* Why authentication don't work with HTTP connections ?
+* Why authentication does not work with HTTP connections ?
 
 Plik session cookies have the "secure" flag set, so they can only be transmitted over secure HTTPS connections.
 
-*  How to take and upload screenshots like a boss ?
+* Build failure "/usr/bin/env: ‘node’: No such file or directory"
+
+Debian users might need to install the nodejs-legacy package.
+
+```
+This package contains a symlink for legacy Node.js code requiring
+binary to be /usr/bin/node (not /usr/bin/nodejs as provided in Debian).
+```
+
+* How to take and upload screenshots like a boss ?
 
 ```
 alias pshot="scrot -s -e 'plik -q \$f | xclip ; xclip -o ; rm \$f'"
@@ -313,7 +327,6 @@ The screenshot is then removed of your home directory to avoid garbage.
 
 * How to contribute to the project ?
 
-Contributions are welcome, feel free to open issues and/or submit pull requests.  
-Please make your pull requests against the current development (RC) branch, not against master.  
+Contributions are welcome, feel free to open issues and/or submit pull requests.
 Please run/update the test suite using the makefile test target.
 
diff --git a/changelog/1.0 b/changelog/1.0
@@ -0,0 +1,22 @@
+Hi !
+
+We're very happy to release Plik 1.0 today !
+Here is the changelog since the last RC version : 
+
+    - Webapp : Fix streaming checkbox label
+
+    - Client : Cleaner output
+    - Client : Compare pgp recipients in lowercase
+    - Client : Fix file name when archive mode enabled
+    - Client : Fix updateing choice case (Y/y)
+    - Client : Add quotes in commands (avoid issues on filenames with spaces)
+
+    - Server : Listen port override with --port param (usefull for multi-instance with same config file)
+    - Server : Using crypto/rand instead of math/rand (thanks to @jedisct1)
+    - Server : Clean API paths
+
+
+We are already working on exciting new features for our next major release !
+
+Stay tuned !
+The plik development team.
diff --git a/changelog/1.0-RC1 b/changelog/1.0-RC1
@@ -0,0 +1,20 @@
+Hi !
+
+We are releasing plik 1.0 RC1, our simple, powerful, multi-backend uploading system. Yay !
+
+Here are the main features : 
+- OneShot uploads (optional) : upload is deleted after one download
+- Password (optional) : upload is protected by user/password (auth basic)
+- Yubikey (optional) : upload is protected by Yubikey OTP
+- Removable (optional) : upload files can be removed after uploads
+- Time To Live (optional) : set upload expiration
+- Comments (optional) : set comments on the upload
+- Metadata backends : file, mongodb
+- Data backends : file, openstack swift, weedfs
+- Shorten backends : shorten upload url 
+- Ssl, Cli client, ...
+
+Et voilà !
+
+Feel free to report bugs, create issues && contribute :)
+
diff --git a/changelog/1.0-RC2 b/changelog/1.0-RC2
@@ -0,0 +1,8 @@
+New Features : 
+  - is.gd shorten backend
+  - cli client pgp secure method
+  - cli client quiet mode
+  - cli client download command
+
+Server logs refactor, code clean, comments
+Various bugfixs ( see commit logs )