From eb7074b25f5d807eeb4bdd0f4ab3f142068474d0 Mon Sep 17 00:00:00 2001 From: Greg Hecquet Date: Tue, 19 Jun 2018 10:06:12 +0200 Subject: [PATCH 1/8] Build arg --- tools/docker/cells/dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/docker/cells/dockerfile b/tools/docker/cells/dockerfile index 5b4d09c7d5..3ab9b20ea7 100755 --- a/tools/docker/cells/dockerfile +++ b/tools/docker/cells/dockerfile @@ -1,6 +1,7 @@ FROM busybox:glibc -ENV CELLS_VERSION 1.0.0 +ARG version +ENV CELLS_VERSION ${version} WORKDIR /cells From ba785211460c643b618a51e981754cdcc2193b2b Mon Sep 17 00:00:00 2001 From: Greg Hecquet Date: Tue, 19 Jun 2018 10:56:40 +0200 Subject: [PATCH 2/8] Updating docker-compose --- tools/docker/docker-compose.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/docker/docker-compose.yaml b/tools/docker/docker-compose.yaml index b01c2e32ea..ddb9c33fc4 100644 --- a/tools/docker/docker-compose.yaml +++ b/tools/docker/docker-compose.yaml @@ -3,7 +3,7 @@ services: # Cells image with two named volumes for the static and for the data cells: - image: cells:latest + image: pydio/cells:latest restart: always volumes: ["static:/root/.config/pydio/cells/static/pydio", "data:/root/.config/pydio/cells/data"] ports: ["8080:8080"] @@ -25,7 +25,7 @@ services: # PHP FPM image with the static named volume from the cells container php: - image: cells-php:latest + image: pydio/cells-php-fpm:latest restart: always volumes: ["static:/root/.config/pydio/cells/static/pydio"] From 353eb6362825db23cc426c6527d2b7c93e8cc53d Mon Sep 17 00:00:00 2001 From: cdujeu Date: Tue, 19 Jun 2018 16:04:14 +0200 Subject: [PATCH 3/8] Update commands descriptions and generate docs for pydio website. --- cmd/config-del.go | 8 +- cmd/config-set.go | 8 +- cmd/config-ssl-mode.go | 12 ++- cmd/config-ssl.go | 2 +- cmd/config-versions.go | 8 +- cmd/ctl/cmd/doc.go | 35 ++---- cmd/ctl/cmd/root.go | 2 +- cmd/doc.go | 26 ++--- cmd/root.go | 16 +-- cmd/services-list.go | 20 ++-- cmd/services-start.go | 8 +- common/utils/md_docs.go | 229 ++++++++++++++++++++++++++++++++++++++++ 12 files changed, 297 insertions(+), 77 deletions(-) create mode 100644 common/utils/md_docs.go diff --git a/cmd/config-del.go b/cmd/config-del.go index b14b94bc32..63aa1aae44 100644 --- a/cmd/config-del.go +++ b/cmd/config-del.go @@ -38,14 +38,14 @@ var delCmd = &cobra.Command{ Short: "Delete a configuration item", Long: `Deletes a configuration item. It will be removed both from the pydio.json file and from the database. -SYNTAX -====== +### Syntax + Configurations are represented by two parameters that you must pass as arguments : - serviceName: name of the corresponding service - configName: name of the parameter you want to delete -EXAMPLES -======== +### Examples + Delete the port entry for the micro.web service (rest api) $ ` + os.Args[0] + ` config delete micro.web port diff --git a/cmd/config-set.go b/cmd/config-set.go index b49a5e25d3..e150f18024 100644 --- a/cmd/config-set.go +++ b/cmd/config-set.go @@ -39,15 +39,15 @@ var updateCmd = &cobra.Command{ Short: "Store a configuration", Long: `Stores a configuration. Will be stored in both your pydio.json file and in the database. -SYNTAX -====== +### Syntax + Configurations are represented by three parameters that you must pass as arguments : - serviceName: name of the corresponding service - configName: name of the parameter - configValue: json-encoded value of the parameter you want to set/change -EXAMPLES -======== +### Examples + Change the port of micro.web service (rest api) $ ` + os.Args[0] + ` config set micro.web port 8083 diff --git a/cmd/config-ssl-mode.go b/cmd/config-ssl-mode.go index 0ffc2f45b2..587f664d8f 100644 --- a/cmd/config-ssl-mode.go +++ b/cmd/config-ssl-mode.go @@ -34,8 +34,16 @@ import ( // enableCmd represents the enable command var SslModeCmd = &cobra.Command{ Use: "mode", - Short: "Enable HTTPS on proxy", - Long: `Setup SSL on application main access point`, + Short: "Manage HTTPS support on proxy", + Long: ` +This command lets you enable/disabled SSL on application main access point. + +Three modes are currently supported : +- TLS mode : provide the paths to certificate and key (as you would on an apache server) +- Self-Signed : a self-signed certificate will be generated at each application start +- Disabled : application will be served on HTTP + +`, Run: func(cmd *cobra.Command, args []string) { enabled, e := promptSslMode() diff --git a/cmd/config-ssl.go b/cmd/config-ssl.go index 42601968f3..8151575106 100644 --- a/cmd/config-ssl.go +++ b/cmd/config-ssl.go @@ -27,7 +27,7 @@ import ( // SslCmd represents the ssl command var SslCmd = &cobra.Command{ Use: "ssl", - Short: "Enable/Disable SSL support", + Short: "Manage SSL configuration", Run: func(cmd *cobra.Command, args []string) { cmd.Help() }, diff --git a/cmd/config-versions.go b/cmd/config-versions.go index 0620a5c61f..9cd56c1691 100644 --- a/cmd/config-versions.go +++ b/cmd/config-versions.go @@ -48,7 +48,13 @@ var ( var versionsListCmd = &cobra.Command{ Use: "versions", Short: "List all configurations versions", - Long: `Display the various versions of configuration`, + Long: ` +This command allows to manage configurations changes history and eventually +revert to a given version. + +A version is created at each call to config.Save() inside the application, along with a log message +and the user originating this call. +`, Run: func(cmd *cobra.Command, args []string) { var store file.VersionsStore diff --git a/cmd/ctl/cmd/doc.go b/cmd/ctl/cmd/doc.go index c30b2c4273..2bc72a3992 100644 --- a/cmd/ctl/cmd/doc.go +++ b/cmd/ctl/cmd/doc.go @@ -21,8 +21,10 @@ package cmd import ( + "log" + + "github.com/pydio/cells/common/utils" "github.com/spf13/cobra" - // "github.com/spf13/cobra/doc" ) var docPath string @@ -39,29 +41,14 @@ Note: this command is currently broken due to a dependency issue introduced by github.com/spf13/cobra/doc and the vendor/github.com/cpuguy83/go-md2man/md2man/roff.go `, Run: func(cmd *cobra.Command, args []string) { - cmd.Print("This command is currently unavailable.") - - // if docPath == "" { - // log.Fatal("Cannot get path flag") - // } else { - - // // log.Fatal("Broken command, should be fixed in a next future") - - // // Sphinx cross-referencing format - // linkHandler := func(name, ref string) string { - // return fmt.Sprintf(":ref:`%s <%s>`", name, ref) - // } - - // filePrepender := func(filename string) string { - // return "" - // } - - // err := doc.GenReSTTreeCustom(RootCmd, docPath, filePrepender, linkHandler) - // if err != nil { - // log.Fatal(err) - // } - // } - + if docPath == "" { + log.Fatal("Cannot get path flag") + } else { + err := utils.GenMarkdownTree(RootCmd, docPath) + if err != nil { + log.Fatal(err) + } + } }, } diff --git a/cmd/ctl/cmd/root.go b/cmd/ctl/cmd/root.go index 4fd2fbd275..33e51faa66 100644 --- a/cmd/ctl/cmd/root.go +++ b/cmd/ctl/cmd/root.go @@ -41,7 +41,7 @@ var ( // RootCmd represents the base command when called without any subcommands. var RootCmd = &cobra.Command{ - Use: "pydioctl", + Use: "cells-ctl", Short: "Pydio Cells Client application", Long: ` Pydio Cells client allows you to interact with the micro services directly. diff --git a/cmd/doc.go b/cmd/doc.go index 1007c32449..bbfe69eb13 100644 --- a/cmd/doc.go +++ b/cmd/doc.go @@ -21,11 +21,11 @@ package cmd import ( - // "fmt" "log" "github.com/spf13/cobra" - // "github.com/spf13/cobra/doc" + + "github.com/pydio/cells/common/utils" ) var docPath string @@ -33,33 +33,21 @@ var docPath string // versionCmd represents the versioning command var docCmd = &cobra.Command{ Use: "doc", - Short: "Generate ReST documentation for this command", - Long: `Generate ReStructuredText documentation for this command. + Short: "Generate MD documentation for this command", + Long: `Generate Markdown documentation for this command. Provide a target folder where to put the generated files. +This command also generates yaml files for pydio.com documentation format. `, Run: func(cmd *cobra.Command, args []string) { - log.Fatal("Currently broken because of govendor issues - TODO") - if docPath == "" { - log.Fatal("Cannot get path flag") + log.Fatal("Please provide a path to store output files") } else { - // Sphinx cross-referencing format - /** - linkHandler := func(name, ref string) string { - return fmt.Sprintf(":ref:`%s <%s>`", name, ref) - } - - filePrepender := func(filename string) string { - return "" - } - - err := doc.GenReSTTreeCustom(RootCmd, docPath, filePrepender, linkHandler) + err := utils.GenMarkdownTree(RootCmd, docPath) if err != nil { log.Fatal(err) } - */ } }, diff --git a/cmd/root.go b/cmd/root.go index 0144474c19..db761dce94 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -59,24 +59,24 @@ var RootCmd = &cobra.Command{ Long: `Thank you for using Pydio Cells. Comprehensive sync & share solution for your collaborators. Open-source software deployed on-premise or in a private cloud. -INSTALL -======= +### Installation + For the very first run, use '` + os.Args[0] + ` install' to load browser-based or command-line based installation wizard. Services will start at the end of the installation. -RUN -=== +### Run + Run '` + os.Args[0] + ` start' to load all services. -LOGS LEVEL -========== +### Logs level + By default, logs are outputted in console format at the Info level. You can set the --log flag or set the PYDIO_LOGS_LEVEL environment variable to one of the following values: - debug, info, error : logs are written in console format with the according level - production : logs are written in json format, for usage with a log aggregator tool. -SERVICES DISCOVERY -================== +### Services Discovery + Micro services need a registry mechanism to discover each other. By default, Pydio Cells ships with Nats.io and Consul.io implementations. You don't need to install any dependency. By default, Cells uses the NATS implementation. You can switch to consul by using the flag --registry=consul. diff --git a/cmd/services-list.go b/cmd/services-list.go index 69e9ad42e7..b1b9d33194 100644 --- a/cmd/services-list.go +++ b/cmd/services-list.go @@ -76,18 +76,20 @@ Use this command to list all running services on this machine. Services fall into main categories (GENERIC, GRPC, REST, API) and are then organized by tags (broker, data, idm, etc.) -EXAMPLE -======= +### Example + Use the --tags/-t flag to limit display to one specific tag, use lowercase for tags. $ pydio list -t=broker - pydio.grpc.activity [X] - pydio.grpc.chat [X] - pydio.grpc.mailer [X] - pydio.api.websocket [X] - pydio.rest.activity [X] - pydio.rest.frontlogs [X] - pydio.rest.mailer [X] + +- pydio.grpc.activity [X] +- pydio.grpc.chat [X] +- pydio.grpc.mailer [X] +- pydio.api.websocket [X] +- pydio.rest.activity [X] +- pydio.rest.frontlogs [X] +- pydio.rest.mailer [X] + `, PreRun: func(cmd *cobra.Command, args []string) { // If we have an error (registry not running) the running list simply is empty diff --git a/cmd/services-start.go b/cmd/services-start.go index e1aa98a89e..fe0c3f1556 100644 --- a/cmd/services-start.go +++ b/cmd/services-start.go @@ -51,8 +51,8 @@ var StartCmd = &cobra.Command{ Short: "Start Cells services", Long: `Start one or more services on this machine -SYNTAX -====== +### Syntax + $ ` + os.Args[0] + ` start [flags] args... Additional arguments are regexp that can match any of the service names available (see 'list' command). @@ -60,8 +60,8 @@ The -t/--tags flag may limit to only a certain category of services, use lowerca The -x/--exclude flag may exclude one or more services Both flags may be used in conjunction with the regexp arguments. -EXAMPLES -======== +### Examples + Start only services starting with grpc $ ` + os.Args[0] + ` start pydio.grpc diff --git a/common/utils/md_docs.go b/common/utils/md_docs.go new file mode 100644 index 0000000000..dad5066ef7 --- /dev/null +++ b/common/utils/md_docs.go @@ -0,0 +1,229 @@ +//Copyright 2015 Red Hat Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package utils + +import ( + "bytes" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strings" + "time" + + "github.com/spf13/cobra" +) + +var ( + yamlPosition = 0 +) + +func printOptions(buf *bytes.Buffer, cmd *cobra.Command, name string) error { + flags := cmd.NonInheritedFlags() + flags.SetOutput(buf) + if flags.HasFlags() { + buf.WriteString("### Options\n\n```\n") + flags.PrintDefaults() + buf.WriteString("```\n\n") + } + + parentFlags := cmd.InheritedFlags() + parentFlags.SetOutput(buf) + if parentFlags.HasFlags() { + buf.WriteString("### Options inherited from parent commands\n\n```\n") + parentFlags.PrintDefaults() + buf.WriteString("```\n\n") + } + return nil +} + +// GenMarkdown creates markdown output. +func GenMarkdown(cmd *cobra.Command, w io.Writer) error { + return GenMarkdownCustom(cmd, w, func(s string) string { return s }) +} + +// GenPydioYaml generates yaml file for pydio docs +// Sample output +// title: "Command Name" +// language: und +// menu: "Command Name" +// weight: 1 +// menu_name: menu-administration-guide +func GenPydioYaml(cmd *cobra.Command, position int, w io.Writer) error { + + name := filepath.Base(cmd.CommandPath()) + buf := new(bytes.Buffer) + buf.WriteString("title: \"" + name + "\"\n") + buf.WriteString("menu: \"" + name + "\"\n") + buf.WriteString("language: und\n") + buf.WriteString("menu_name: menu-administration-guide\n") + buf.WriteString(fmt.Sprintf("weight: %d\n", position)) + + _, err := buf.WriteTo(w) + return err +} + +// GenMarkdownCustom creates custom markdown output. +func GenMarkdownCustom(cmd *cobra.Command, w io.Writer, linkHandler func(string) string) error { + cmd.InitDefaultHelpCmd() + cmd.InitDefaultHelpFlag() + + buf := new(bytes.Buffer) + name := cmd.CommandPath() + + short := cmd.Short + long := cmd.Long + if len(long) == 0 { + long = short + } + + buf.WriteString("## " + name + "\n\n") + buf.WriteString(short + "\n\n") + buf.WriteString("### Synopsis\n\n") + buf.WriteString(long + "\n\n") + + if cmd.Runnable() { + buf.WriteString(fmt.Sprintf("```\n%s\n```\n\n", cmd.UseLine())) + } + + if len(cmd.Example) > 0 { + buf.WriteString("### Examples\n\n") + buf.WriteString(fmt.Sprintf("```\n%s\n```\n\n", cmd.Example)) + } + + if err := printOptions(buf, cmd, name); err != nil { + return err + } + if cmdHasSeeAlso(cmd) { + buf.WriteString("### SEE ALSO\n\n") + if cmd.HasParent() { + parent := cmd.Parent() + pname := parent.CommandPath() + link := pname + ".md" + link = strings.Replace(link, " ", "_", -1) + buf.WriteString(fmt.Sprintf("* [%s](%s)\t - %s\n", pname, linkHandler(link), parent.Short)) + cmd.VisitParents(func(c *cobra.Command) { + if c.DisableAutoGenTag { + cmd.DisableAutoGenTag = c.DisableAutoGenTag + } + }) + } + + children := cmd.Commands() + sort.Sort(cmdByName(children)) + + for _, child := range children { + if !child.IsAvailableCommand() || child.IsAdditionalHelpTopicCommand() { + continue + } + cname := name + " " + child.Name() + link := cname + ".md" + link = strings.Replace(link, " ", "_", -1) + buf.WriteString(fmt.Sprintf("* [%s](%s)\t - %s\n", cname, linkHandler(link), child.Short)) + } + buf.WriteString("\n") + } + if !cmd.DisableAutoGenTag { + buf.WriteString("###### Auto generated by spf13/cobra on " + time.Now().Format("2-Jan-2006") + "\n") + } + _, err := buf.WriteTo(w) + return err +} + +// GenMarkdownTree will generate a markdown page for this command and all +// descendants in the directory given. The header may be nil. +// This function may not work correctly if your command names have `-` in them. +// If you have `cmd` with two subcmds, `sub` and `sub-third`, +// and `sub` has a subcommand called `third`, it is undefined which +// help output will be in the file `cmd-sub-third.1`. +func GenMarkdownTree(cmd *cobra.Command, dir string) error { + linkHandler := func(s string) string { + s = strings.Replace(s, "_", "-", -1) + s = strings.TrimSuffix(s, ".md") + return s + } + emptyStr := func(s string) string { return "" } + + return GenMarkdownTreeCustom(cmd, dir, emptyStr, linkHandler) +} + +// GenMarkdownTreeCustom is the the same as GenMarkdownTree, but +// with custom filePrepender and linkHandler. +func GenMarkdownTreeCustom(cmd *cobra.Command, dir string, filePrepender, linkHandler func(string) string) error { + yamlPosition++ + + basename := strings.Replace(cmd.CommandPath(), " ", "_", -1) + ".md" + filename := filepath.Join(dir, basename) + f, err := os.Create(filename) + if err != nil { + return err + } + defer f.Close() + + if _, err := io.WriteString(f, filePrepender(filename)); err != nil { + return err + } + if err := GenMarkdownCustom(cmd, f, linkHandler); err != nil { + return err + } + + yamlname := strings.Replace(cmd.CommandPath(), " ", "_", -1) + ".yaml" + yamlfile := filepath.Join(dir, yamlname) + f1, err := os.Create(yamlfile) + if err != nil { + return err + } + defer f1.Close() + + if _, err := io.WriteString(f1, filePrepender(yamlfile)); err != nil { + return err + } + if err := GenPydioYaml(cmd, yamlPosition, f1); err != nil { + return err + } + + for _, c := range cmd.Commands() { + if !c.IsAvailableCommand() || c.IsAdditionalHelpTopicCommand() { + continue + } + if err := GenMarkdownTreeCustom(c, dir, filePrepender, linkHandler); err != nil { + return err + } + } + + return nil +} + +// Test to see if we have a reason to print See Also information in docs +// Basically this is a test for a parent commend or a subcommand which is +// both not deprecated and not the autogenerated help command. +func cmdHasSeeAlso(cmd *cobra.Command) bool { + if cmd.HasParent() { + return true + } + for _, c := range cmd.Commands() { + if !c.IsAvailableCommand() || c.IsAdditionalHelpTopicCommand() { + continue + } + return true + } + return false +} + +type cmdByName []*cobra.Command + +func (s cmdByName) Len() int { return len(s) } +func (s cmdByName) Swap(i, j int) { s[i], s[j] = s[j], s[i] } +func (s cmdByName) Less(i, j int) bool { return s[i].Name() < s[j].Name() } From 8a5a63ff09b97bd6c2863b8c872efc57eec50b88 Mon Sep 17 00:00:00 2001 From: Greg Hecquet Date: Fri, 22 Jun 2018 02:09:04 +0200 Subject: [PATCH 4/8] Data changes split --- common/service/proto/common.pb.go | 165 +++++++++++++++++------- common/service/proto/common.proto | 9 ++ data/changes/dao.go | 2 + data/changes/grpc/handler.go | 45 +++++++ data/changes/grpc/handler_test.go | 36 ++++++ data/changes/grpc/plugins.go | 2 + data/changes/migrations/mysql/0.2.sql | 11 ++ data/changes/migrations/sqlite3/0.2.sql | 11 ++ data/changes/sql.go | 103 ++++++++++++++- main.go | 5 +- scheduler/actions/changes/archive.go | 94 ++++++++++++++ scheduler/actions/changes/init.go | 35 +++++ scheduler/jobs/grpc/defaults.go | 19 +++ tools/docker/docker-compose.yaml | 1 + 14 files changed, 488 insertions(+), 50 deletions(-) create mode 100644 data/changes/migrations/mysql/0.2.sql create mode 100644 data/changes/migrations/sqlite3/0.2.sql create mode 100644 scheduler/actions/changes/archive.go create mode 100644 scheduler/actions/changes/init.go diff --git a/common/service/proto/common.pb.go b/common/service/proto/common.pb.go index 02d6a4a66d..cb725f9a14 100644 --- a/common/service/proto/common.pb.go +++ b/common/service/proto/common.pb.go @@ -15,6 +15,7 @@ It has these top-level messages: SourceSingleQuery StopEvent StatusResponse + ChangesArchiveQuery */ package service @@ -397,6 +398,23 @@ func (m *StatusResponse) GetOK() bool { return false } +// TODO - move from there +type ChangesArchiveQuery struct { + RemainingRows uint64 `protobuf:"varint,1,opt,name=RemainingRows" json:"RemainingRows,omitempty"` +} + +func (m *ChangesArchiveQuery) Reset() { *m = ChangesArchiveQuery{} } +func (m *ChangesArchiveQuery) String() string { return proto.CompactTextString(m) } +func (*ChangesArchiveQuery) ProtoMessage() {} +func (*ChangesArchiveQuery) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{7} } + +func (m *ChangesArchiveQuery) GetRemainingRows() uint64 { + if m != nil { + return m.RemainingRows + } + return 0 +} + func init() { proto.RegisterType((*Query)(nil), "service.Query") proto.RegisterType((*ResourcePolicyQuery)(nil), "service.ResourcePolicyQuery") @@ -405,6 +423,7 @@ func init() { proto.RegisterType((*SourceSingleQuery)(nil), "service.SourceSingleQuery") proto.RegisterType((*StopEvent)(nil), "service.StopEvent") proto.RegisterType((*StatusResponse)(nil), "service.StatusResponse") + proto.RegisterType((*ChangesArchiveQuery)(nil), "service.ChangesArchiveQuery") proto.RegisterEnum("service.OperationType", OperationType_name, OperationType_value) proto.RegisterEnum("service.ResourcePolicyAction", ResourcePolicyAction_name, ResourcePolicyAction_value) proto.RegisterEnum("service.ResourcePolicy_PolicyEffect", ResourcePolicy_PolicyEffect_name, ResourcePolicy_PolicyEffect_value) @@ -519,52 +538,108 @@ func (h *Service) Status(ctx context.Context, in *google_protobuf1.Empty, out *S return h.ServiceHandler.Status(ctx, in, out) } +// Client API for Archiver service + +type ArchiverClient interface { + Archive(ctx context.Context, in *Query, opts ...client.CallOption) (*StatusResponse, error) +} + +type archiverClient struct { + c client.Client + serviceName string +} + +func NewArchiverClient(serviceName string, c client.Client) ArchiverClient { + if c == nil { + c = client.NewClient() + } + if len(serviceName) == 0 { + serviceName = "service" + } + return &archiverClient{ + c: c, + serviceName: serviceName, + } +} + +func (c *archiverClient) Archive(ctx context.Context, in *Query, opts ...client.CallOption) (*StatusResponse, error) { + req := c.c.NewRequest(c.serviceName, "Archiver.Archive", in) + out := new(StatusResponse) + err := c.c.Call(ctx, req, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// Server API for Archiver service + +type ArchiverHandler interface { + Archive(context.Context, *Query, *StatusResponse) error +} + +func RegisterArchiverHandler(s server.Server, hdlr ArchiverHandler, opts ...server.HandlerOption) { + s.Handle(s.NewHandler(&Archiver{hdlr}, opts...)) +} + +type Archiver struct { + ArchiverHandler +} + +func (h *Archiver) Archive(ctx context.Context, in *Query, out *StatusResponse) error { + return h.ArchiverHandler.Archive(ctx, in, out) +} + func init() { proto.RegisterFile("common.proto", fileDescriptor0) } var fileDescriptor0 = []byte{ - // 696 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x74, 0x54, 0xdd, 0x6e, 0xda, 0x48, - 0x14, 0x8e, 0x4d, 0x0c, 0xf8, 0x90, 0x65, 0x9d, 0xd9, 0x28, 0x6b, 0xa1, 0xac, 0x64, 0xb1, 0x55, - 0x85, 0xa2, 0x96, 0x0b, 0x9a, 0xaa, 0xaa, 0xd4, 0x1b, 0x12, 0x5c, 0x89, 0x26, 0xc5, 0xe9, 0x38, - 0x51, 0x94, 0xab, 0xca, 0x98, 0x01, 0xb9, 0x01, 0x8f, 0x35, 0x1e, 0xa7, 0xf2, 0x45, 0x9f, 0xa0, - 0x2f, 0xd0, 0xb7, 0xeb, 0xab, 0x54, 0xf3, 0x03, 0x21, 0x09, 0x5c, 0x79, 0xbe, 0x33, 0xdf, 0x9c, - 0xf3, 0x9d, 0xef, 0xcc, 0x18, 0xf6, 0x62, 0xba, 0x58, 0xd0, 0xb4, 0x9b, 0x31, 0xca, 0x29, 0xaa, - 0xe5, 0x84, 0xdd, 0x27, 0x31, 0x69, 0xd9, 0x51, 0x5a, 0xaa, 0x58, 0xab, 0x41, 0x16, 0x19, 0xd7, - 0xa0, 0xfd, 0xd3, 0x04, 0xeb, 0x4b, 0x41, 0x58, 0x89, 0x4e, 0x00, 0xc2, 0x62, 0x2c, 0xd6, 0x09, - 0xc9, 0x5d, 0xc3, 0xab, 0x74, 0x1a, 0xbd, 0x83, 0xee, 0x8c, 0xd2, 0xd9, 0x9c, 0x28, 0xf2, 0xb8, - 0x98, 0x76, 0xfb, 0x69, 0x89, 0xd7, 0x78, 0xe8, 0x04, 0xec, 0x20, 0x23, 0x2c, 0xe2, 0x09, 0x4d, - 0x5d, 0xd3, 0x33, 0x3a, 0xcd, 0xde, 0x61, 0x57, 0x17, 0xed, 0xae, 0x76, 0xae, 0xca, 0x8c, 0xe0, - 0x07, 0x22, 0x1a, 0xc1, 0x3f, 0x98, 0xe4, 0xb4, 0x60, 0x31, 0xb9, 0xa4, 0xf3, 0x24, 0x2e, 0xa5, - 0x04, 0xb7, 0xe2, 0x19, 0x9d, 0x46, 0xef, 0x68, 0x75, 0x7e, 0x03, 0x07, 0x6f, 0x3a, 0x88, 0x0e, - 0xa1, 0x1a, 0x4c, 0xa7, 0x39, 0xe1, 0xee, 0xae, 0x67, 0x74, 0x2a, 0x58, 0x23, 0x74, 0x00, 0xd6, - 0x45, 0xb2, 0x48, 0xb8, 0x6b, 0xc9, 0xb0, 0x02, 0xc8, 0x85, 0xda, 0x8c, 0xd1, 0x22, 0x3b, 0x2d, - 0xdd, 0xaa, 0x67, 0x74, 0x2c, 0xbc, 0x84, 0xed, 0xdb, 0x8d, 0xba, 0x50, 0x0b, 0xea, 0x61, 0x31, - 0xfe, 0x46, 0x62, 0xae, 0x8c, 0xb1, 0xf1, 0x0a, 0x8b, 0x12, 0xbe, 0xf0, 0x53, 0x36, 0x5f, 0xc7, - 0x0a, 0x20, 0x07, 0x2a, 0xfd, 0x54, 0x35, 0x54, 0xc7, 0x62, 0xd9, 0xfe, 0x65, 0x42, 0xf3, 0x71, - 0x6e, 0xd4, 0x04, 0x33, 0x99, 0xb8, 0x86, 0x94, 0x66, 0x26, 0x13, 0x51, 0x66, 0xc9, 0x90, 0xd9, - 0x6c, 0xbc, 0xc2, 0xe8, 0x2d, 0x54, 0xfb, 0xb1, 0x34, 0xb9, 0x22, 0x4d, 0xfe, 0x6f, 0x8b, 0x49, - 0x8a, 0x84, 0x35, 0x59, 0xb4, 0xaa, 0x95, 0x4a, 0x67, 0x6c, 0xbc, 0x84, 0xe8, 0x03, 0x54, 0xfd, - 0xe9, 0x54, 0x6c, 0x58, 0x32, 0xe1, 0x8b, 0x2d, 0x09, 0xbb, 0xea, 0xa3, 0xb8, 0x58, 0x9f, 0x41, - 0x2f, 0xa1, 0xf9, 0x29, 0xa7, 0xe9, 0x19, 0x4d, 0x27, 0x89, 0x28, 0x94, 0x4b, 0x27, 0x6d, 0xfc, - 0x24, 0xda, 0xfe, 0x1f, 0xf6, 0xd6, 0xcf, 0xa3, 0x3a, 0xec, 0x4e, 0x48, 0x5a, 0x3a, 0x3b, 0xc8, - 0x06, 0x2b, 0x9a, 0xcf, 0xe9, 0x77, 0xc7, 0x68, 0xff, 0x36, 0x60, 0x5f, 0xe9, 0x0d, 0x0a, 0x9e, - 0x15, 0x5c, 0x99, 0x2e, 0xa5, 0xc7, 0x31, 0xc9, 0x73, 0x69, 0x51, 0x1d, 0x2f, 0xa1, 0x98, 0xf6, - 0xc7, 0x28, 0x99, 0x93, 0x89, 0xf6, 0x5c, 0x23, 0xf4, 0x0a, 0xf6, 0x43, 0xce, 0x92, 0x74, 0x76, - 0x4a, 0x27, 0xe5, 0x19, 0x5d, 0x64, 0x11, 0x23, 0xd2, 0x2e, 0x1b, 0x3f, 0xdf, 0x40, 0x1d, 0xf8, - 0x5b, 0x88, 0x5d, 0xe7, 0x2a, 0x8b, 0x9e, 0x86, 0x51, 0x17, 0x90, 0xcf, 0x18, 0x65, 0x2a, 0xc7, - 0x92, 0x6c, 0x49, 0xf2, 0x86, 0x1d, 0x31, 0xfc, 0x11, 0xe5, 0xd2, 0x91, 0x3a, 0x16, 0xcb, 0xf6, - 0x0f, 0xd8, 0x0f, 0xa5, 0xa7, 0x61, 0x92, 0xce, 0xe6, 0x64, 0x75, 0x69, 0x87, 0x97, 0x9f, 0xa3, - 0xfc, 0x4e, 0x0f, 0x5b, 0x23, 0x74, 0x04, 0xf6, 0x75, 0x4e, 0x58, 0x7f, 0x46, 0x52, 0xae, 0xe5, - 0x3f, 0x04, 0x90, 0x07, 0x8d, 0x1b, 0xca, 0xee, 0xf2, 0x2c, 0x8a, 0xc9, 0x70, 0xa2, 0x25, 0xaf, - 0x87, 0x96, 0xe5, 0xad, 0x87, 0xf2, 0xaf, 0xc1, 0x0e, 0x39, 0xcd, 0xfc, 0x7b, 0x9d, 0x20, 0x54, - 0x93, 0x1e, 0x45, 0x0b, 0x22, 0xbd, 0xb5, 0xf1, 0x7a, 0xa8, 0xed, 0x41, 0x33, 0xe4, 0x11, 0x2f, - 0x72, 0x4c, 0xf2, 0x8c, 0xa6, 0x39, 0x11, 0x37, 0x35, 0x38, 0xd7, 0x63, 0x30, 0x83, 0xf3, 0x63, - 0x0f, 0xfe, 0x7a, 0xf4, 0xb6, 0x51, 0x15, 0xcc, 0x00, 0x3b, 0x3b, 0xa8, 0x06, 0x95, 0xfe, 0x68, - 0xe0, 0x18, 0xc7, 0x01, 0x1c, 0x6c, 0xba, 0x98, 0x8a, 0x70, 0xab, 0xe6, 0x1f, 0xdc, 0x8c, 0x7c, - 0xec, 0x18, 0xe2, 0x52, 0x60, 0xbf, 0x3f, 0x70, 0x4c, 0x11, 0xbc, 0xc1, 0xc3, 0x2b, 0xdf, 0xa9, - 0xa0, 0x26, 0x80, 0x3f, 0x18, 0x5e, 0x7d, 0xc5, 0xd7, 0x17, 0x7e, 0xe8, 0xec, 0xf6, 0x4e, 0xa1, - 0x16, 0xf2, 0x88, 0x71, 0xc2, 0xd0, 0x3b, 0xb0, 0xe4, 0x12, 0x1d, 0x3e, 0xfb, 0x3d, 0xc9, 0xd7, - 0xd7, 0xda, 0x12, 0xef, 0x0d, 0xa0, 0xa6, 0xfb, 0x44, 0xef, 0xa1, 0xaa, 0x7a, 0xdc, 0x9a, 0xe4, - 0xdf, 0xd5, 0x83, 0x78, 0x6c, 0xc6, 0xb8, 0x2a, 0x89, 0x6f, 0xfe, 0x04, 0x00, 0x00, 0xff, 0xff, - 0xdc, 0x42, 0x7a, 0x56, 0x6a, 0x05, 0x00, 0x00, + // 760 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x74, 0x94, 0xdd, 0x6e, 0xe3, 0x44, + 0x14, 0xc7, 0xd7, 0x4e, 0x9c, 0xc4, 0xa7, 0xbb, 0xc1, 0x9d, 0xad, 0x8a, 0x09, 0x8b, 0x64, 0x99, + 0x15, 0x8a, 0x56, 0xe0, 0x95, 0x42, 0x11, 0x42, 0x20, 0xa4, 0xb4, 0x31, 0x52, 0x68, 0x89, 0xcb, + 0xb8, 0x55, 0xd5, 0x2b, 0xe4, 0x38, 0x13, 0xd7, 0x34, 0xf1, 0x58, 0xe3, 0x71, 0x2b, 0x5f, 0xf0, + 0x04, 0xbc, 0x00, 0x6f, 0xc7, 0xab, 0xac, 0x3c, 0x33, 0x49, 0x93, 0x36, 0xb9, 0xca, 0x9c, 0x73, + 0xfe, 0xe7, 0x63, 0x7e, 0x67, 0x62, 0x78, 0x1d, 0xd3, 0xe5, 0x92, 0x66, 0x5e, 0xce, 0x28, 0xa7, + 0xa8, 0x5d, 0x10, 0xf6, 0x90, 0xc6, 0xa4, 0xf7, 0x45, 0x42, 0x69, 0xb2, 0x20, 0x1f, 0x85, 0x7b, + 0x5a, 0xce, 0x3f, 0x46, 0x59, 0x25, 0x35, 0xbd, 0x2f, 0x9f, 0x87, 0xc8, 0x32, 0xe7, 0x2a, 0xe8, + 0xfe, 0xab, 0x83, 0xf1, 0x67, 0x49, 0x58, 0x85, 0x4e, 0x00, 0xc2, 0x72, 0x5a, 0x9f, 0x53, 0x52, + 0xd8, 0x9a, 0xd3, 0xe8, 0x1f, 0x0c, 0x8e, 0x3c, 0x99, 0xeb, 0xad, 0x72, 0xbd, 0x61, 0x56, 0xe1, + 0x0d, 0x1d, 0x3a, 0x01, 0x33, 0xc8, 0x09, 0x8b, 0x78, 0x4a, 0x33, 0x5b, 0x77, 0xb4, 0x7e, 0x77, + 0x70, 0xec, 0xa9, 0xa1, 0xbc, 0x75, 0xe4, 0xaa, 0xca, 0x09, 0x7e, 0x12, 0xa2, 0x09, 0xbc, 0xc5, + 0xa4, 0xa0, 0x25, 0x8b, 0xc9, 0x25, 0x5d, 0xa4, 0x71, 0x25, 0x46, 0xb0, 0x1b, 0x8e, 0xd6, 0x3f, + 0x18, 0xbc, 0x5b, 0xe7, 0xef, 0xd0, 0xe0, 0x5d, 0x89, 0xe8, 0x18, 0x5a, 0xc1, 0x7c, 0x5e, 0x10, + 0x6e, 0x37, 0x1d, 0xad, 0xdf, 0xc0, 0xca, 0x42, 0x47, 0x60, 0x5c, 0xa4, 0xcb, 0x94, 0xdb, 0x86, + 0x70, 0x4b, 0x03, 0xd9, 0xd0, 0x4e, 0x18, 0x2d, 0xf3, 0xd3, 0xca, 0x6e, 0x39, 0x5a, 0xdf, 0xc0, + 0x2b, 0xd3, 0xbd, 0xdd, 0x39, 0x17, 0xea, 0x41, 0x27, 0x2c, 0xa7, 0x7f, 0x93, 0x98, 0x4b, 0x30, + 0x26, 0x5e, 0xdb, 0x75, 0x0b, 0xbf, 0xe6, 0x29, 0x2e, 0xdf, 0xc1, 0xd2, 0x40, 0x16, 0x34, 0x86, + 0x99, 0xbc, 0x50, 0x07, 0xd7, 0x47, 0xf7, 0x3f, 0x1d, 0xba, 0xdb, 0xb5, 0x51, 0x17, 0xf4, 0x74, + 0x66, 0x6b, 0x62, 0x34, 0x3d, 0x9d, 0xd5, 0x6d, 0x56, 0x0a, 0x51, 0xcd, 0xc4, 0x6b, 0x1b, 0xfd, + 0x00, 0xad, 0x61, 0x2c, 0x20, 0x37, 0x04, 0xe4, 0xaf, 0xf6, 0x40, 0x92, 0x22, 0xac, 0xc4, 0xf5, + 0x55, 0xd5, 0xa4, 0x82, 0x8c, 0x89, 0x57, 0x26, 0xfa, 0x05, 0x5a, 0xfe, 0x7c, 0x5e, 0x07, 0x0c, + 0x51, 0xf0, 0xfd, 0x9e, 0x82, 0x9e, 0xfc, 0x91, 0x5a, 0xac, 0x72, 0xd0, 0x37, 0xd0, 0xfd, 0xbd, + 0xa0, 0xd9, 0x19, 0xcd, 0x66, 0x69, 0xdd, 0xa8, 0x10, 0x24, 0x4d, 0xfc, 0xcc, 0xeb, 0x7e, 0x0d, + 0xaf, 0x37, 0xf3, 0x51, 0x07, 0x9a, 0x33, 0x92, 0x55, 0xd6, 0x2b, 0x64, 0x82, 0x11, 0x2d, 0x16, + 0xf4, 0xd1, 0xd2, 0xdc, 0xff, 0x35, 0x38, 0x94, 0xf3, 0x06, 0x25, 0xcf, 0x4b, 0x2e, 0xa1, 0x8b, + 0xd1, 0xe3, 0x98, 0x14, 0x85, 0x40, 0xd4, 0xc1, 0x2b, 0xb3, 0xde, 0xf6, 0x6f, 0x51, 0xba, 0x20, + 0x33, 0xc5, 0x5c, 0x59, 0xe8, 0x5b, 0x38, 0x0c, 0x39, 0x4b, 0xb3, 0xe4, 0x94, 0xce, 0xaa, 0x33, + 0xba, 0xcc, 0x23, 0x46, 0x04, 0x2e, 0x13, 0xbf, 0x0c, 0xa0, 0x3e, 0x7c, 0x56, 0x0f, 0xbb, 0xa9, + 0x95, 0x88, 0x9e, 0xbb, 0x91, 0x07, 0xc8, 0x67, 0x8c, 0x32, 0x59, 0x63, 0x25, 0x36, 0x84, 0x78, + 0x47, 0xa4, 0x5e, 0xfe, 0x84, 0x72, 0x41, 0xa4, 0x83, 0xeb, 0xa3, 0xfb, 0x0f, 0x1c, 0x86, 0x82, + 0x69, 0x98, 0x66, 0xc9, 0x82, 0xac, 0x1f, 0xed, 0xf8, 0xf2, 0x8f, 0xa8, 0xb8, 0x57, 0xcb, 0x56, + 0x16, 0x7a, 0x07, 0xe6, 0x75, 0x41, 0xd8, 0x30, 0x21, 0x19, 0x57, 0xe3, 0x3f, 0x39, 0x90, 0x03, + 0x07, 0x37, 0x94, 0xdd, 0x17, 0x79, 0x14, 0x93, 0xf1, 0x4c, 0x8d, 0xbc, 0xe9, 0x5a, 0xb5, 0x37, + 0x9e, 0xda, 0x7f, 0x07, 0x66, 0xc8, 0x69, 0xee, 0x3f, 0xa8, 0x02, 0xa1, 0xdc, 0xf4, 0x24, 0x5a, + 0x12, 0xc1, 0xd6, 0xc4, 0x9b, 0x2e, 0xd7, 0x81, 0x6e, 0xc8, 0x23, 0x5e, 0x16, 0x98, 0x14, 0x39, + 0xcd, 0x0a, 0x52, 0xbf, 0xd4, 0xe0, 0x5c, 0xad, 0x41, 0x0f, 0xce, 0xdd, 0x9f, 0xe1, 0xed, 0xd9, + 0x5d, 0x94, 0x25, 0xa4, 0x18, 0xb2, 0xf8, 0x2e, 0x7d, 0x50, 0x37, 0x7a, 0x0f, 0x6f, 0x30, 0x59, + 0x46, 0x69, 0x96, 0x66, 0x09, 0xa6, 0x8f, 0x72, 0x71, 0x4d, 0xbc, 0xed, 0xfc, 0xe0, 0xc0, 0x9b, + 0xad, 0x0f, 0x03, 0x6a, 0x81, 0x1e, 0x60, 0xeb, 0x15, 0x6a, 0x43, 0x63, 0x38, 0x19, 0x59, 0xda, + 0x87, 0x00, 0x8e, 0x76, 0xbd, 0x6a, 0x29, 0xb8, 0x95, 0x8f, 0x27, 0xb8, 0x99, 0xf8, 0xd8, 0xd2, + 0xea, 0x17, 0x85, 0xfd, 0xe1, 0xc8, 0xd2, 0x6b, 0xe7, 0x0d, 0x1e, 0x5f, 0xf9, 0x56, 0x03, 0x75, + 0x01, 0xfc, 0xd1, 0xf8, 0xea, 0x2f, 0x7c, 0x7d, 0xe1, 0x87, 0x56, 0x73, 0x70, 0x0a, 0xed, 0x90, + 0x47, 0x8c, 0x13, 0x86, 0x7e, 0x04, 0x43, 0x1c, 0xd1, 0xf1, 0x8b, 0x6f, 0x9b, 0xf8, 0xeb, 0xf6, + 0xf6, 0xf8, 0x07, 0x23, 0x68, 0x2b, 0x48, 0xe8, 0x27, 0x68, 0x49, 0x40, 0x7b, 0x8b, 0x7c, 0xbe, + 0xfe, 0x37, 0x6d, 0x93, 0x1c, 0xfc, 0x0a, 0x1d, 0x85, 0x8c, 0xa1, 0x01, 0xb4, 0xd5, 0x19, 0x75, + 0xd7, 0x7a, 0x41, 0x72, 0x6f, 0xfe, 0xb4, 0x25, 0x1a, 0x7d, 0xff, 0x29, 0x00, 0x00, 0xff, 0xff, + 0xbc, 0x85, 0xa9, 0x82, 0x07, 0x06, 0x00, 0x00, } diff --git a/common/service/proto/common.proto b/common/service/proto/common.proto index 38760f50dc..d66097ca89 100644 --- a/common/service/proto/common.proto +++ b/common/service/proto/common.proto @@ -88,3 +88,12 @@ service Service { message StatusResponse { bool OK = 1; } + +service Archiver { + rpc Archive(Query) returns (StatusResponse); +} + +// TODO - move from there +message ChangesArchiveQuery { + uint64 RemainingRows = 1; +} diff --git a/data/changes/dao.go b/data/changes/dao.go index 7214bc6ab8..54a8d1b0a7 100644 --- a/data/changes/dao.go +++ b/data/changes/dao.go @@ -34,8 +34,10 @@ type DAO interface { Put(*tree.SyncChange) error BulkPut([]*tree.SyncChange) error Get(uint64, string) (chan *tree.SyncChange, error) + FirstSeq() (uint64, error) LastSeq() (uint64, error) HasNodeById(id string) (bool, error) + Archive(uint64) error } func NewDAO(o dao.DAO) dao.DAO { diff --git a/data/changes/grpc/handler.go b/data/changes/grpc/handler.go index 61833369b9..ade9557f56 100644 --- a/data/changes/grpc/handler.go +++ b/data/changes/grpc/handler.go @@ -29,6 +29,7 @@ import ( "go.uber.org/zap" + "github.com/micro/protobuf/ptypes" "github.com/pydio/cells/common" "github.com/pydio/cells/common/log" "github.com/pydio/cells/common/proto/jobs" @@ -36,6 +37,7 @@ import ( "github.com/pydio/cells/common/proto/tree" "github.com/pydio/cells/common/service/context" "github.com/pydio/cells/common/service/defaults" + service "github.com/pydio/cells/common/service/proto" "github.com/pydio/cells/data/changes" ) @@ -250,3 +252,46 @@ func (h Handler) Search(ctx context.Context, req *tree.SearchSyncChangeRequest, } return changes.NewOptimizer(ctx, changes.ChangeChan(res)).Output(ctx, stream) } + +// Archive change in the service storage +func (h Handler) Archive(ctx context.Context, req *service.Query, resp *service.StatusResponse) error { + + log.Logger(ctx).Debug("Archive") + + if servicecontext.GetDAO(ctx) == nil { + return fmt.Errorf("no DAO found, wrong initialization") + } + dao := servicecontext.GetDAO(ctx).(changes.DAO) + + subqueries := req.GetSubQueries() + if len(subqueries) != 1 { + return fmt.Errorf("Can only treate one query with a seq id") + } + + query := &service.ChangesArchiveQuery{} + + if err := ptypes.UnmarshalAny(subqueries[0], query); err != nil { + return err + } + + first, err := dao.FirstSeq() + if err != nil { + return err + } + + last, err := dao.LastSeq() + if err != nil { + return err + } + log.Logger(ctx).Debug(fmt.Sprintf("Archiving from seq %d > %d", last-query.RemainingRows, first)) + + if seq := last - query.RemainingRows; seq >= first { + if err := dao.Archive(seq); err != nil { + return err + } + } + + resp.OK = true + + return nil +} diff --git a/data/changes/grpc/handler_test.go b/data/changes/grpc/handler_test.go index 3740961df3..78f8efa191 100644 --- a/data/changes/grpc/handler_test.go +++ b/data/changes/grpc/handler_test.go @@ -26,12 +26,15 @@ import ( "testing" // SQLite Driver + "github.com/golang/protobuf/ptypes" + "github.com/golang/protobuf/ptypes/any" _ "github.com/mattn/go-sqlite3" . "github.com/smartystreets/goconvey/convey" "github.com/pydio/cells/common/config" "github.com/pydio/cells/common/proto/tree" "github.com/pydio/cells/common/service/context" + service "github.com/pydio/cells/common/service/proto" commonsql "github.com/pydio/cells/common/sql" changessql "github.com/pydio/cells/data/changes" ) @@ -103,6 +106,39 @@ func TestBasicEvents(t *testing.T) { So(streamMock.counter, ShouldEqual, 1) }) + Convey("Archive changes with an emty query", t, func() { + + req := &service.Query{} + resp := &service.StatusResponse{} + + err := mockHandler.Archive(ctx, req, resp) + So(err, ShouldNotBeNil) + }) + + Convey("Archive changes", t, func() { + + query, err := ptypes.MarshalAny(&service.ChangesArchiveQuery{RemainingRows: 0}) + So(err, ShouldBeNil) + + req := &service.Query{ + SubQueries: []*any.Any{query}, + } + resp := &service.StatusResponse{} + + err = mockHandler.Archive(ctx, req, resp) + So(err, ShouldBeNil) + }) + + Convey("Test search for a change", t, func() { + + streamMock := newSearchStreamMock() + req := NewSearchRequest(0, "/putchange", false, false) + + err := mockHandler.Search(ctx, req, streamMock) + So(err, ShouldBeNil) + So(streamMock.counter, ShouldEqual, 1) + }) + // // Creates 3 events that should be flattened in a single one // Convey("Test flattened changes", t, func() { // ev := NewSyncChange("testId-flattened-events", "/flattentest/flatten-test", "", tree.SyncChange_create) diff --git a/data/changes/grpc/plugins.go b/data/changes/grpc/plugins.go index fc2b767803..74417edf66 100644 --- a/data/changes/grpc/plugins.go +++ b/data/changes/grpc/plugins.go @@ -33,6 +33,7 @@ import ( "github.com/pydio/cells/common/proto/tree" "github.com/pydio/cells/common/registry" "github.com/pydio/cells/common/service" + serviceproto "github.com/pydio/cells/common/service/proto" "github.com/pydio/cells/data/changes" ) @@ -50,6 +51,7 @@ func init() { service.WithStorage(changes.NewDAO, "data_changes"), service.WithMicro(func(m micro.Service) error { h := NewHandler(m.Options().Context) + serviceproto.RegisterArchiverHandler(m.Options().Server, h) tree.RegisterSyncChangesHandler(m.Options().Server, h) sync.RegisterSyncEndpointHandler(m.Options().Server, h) diff --git a/data/changes/migrations/mysql/0.2.sql b/data/changes/migrations/mysql/0.2.sql new file mode 100644 index 0000000000..a15555585f --- /dev/null +++ b/data/changes/migrations/mysql/0.2.sql @@ -0,0 +1,11 @@ +-- +migrate Up +CREATE TABLE IF NOT EXISTS data_changes_archive ( + seq int(20) primary key, + node_id varchar(255), + type enum('create', 'delete', 'path', 'content'), + source text, + target text +); + +-- +migrate Down +DROP TABLE data_changes_archive; diff --git a/data/changes/migrations/sqlite3/0.2.sql b/data/changes/migrations/sqlite3/0.2.sql new file mode 100644 index 0000000000..14758faf87 --- /dev/null +++ b/data/changes/migrations/sqlite3/0.2.sql @@ -0,0 +1,11 @@ +-- +migrate Up +CREATE TABLE IF NOT EXISTS data_changes_archive ( + seq integer primary key, + node_id varchar(255), + type varchar(255), + source text, + target text +); + +-- +migrate Down +DROP TABLE data_changes_archive; diff --git a/data/changes/sql.go b/data/changes/sql.go index fb1cdd1048..09fc24484c 100644 --- a/data/changes/sql.go +++ b/data/changes/sql.go @@ -51,9 +51,13 @@ var ( return str }, // Order by (node_id,seq) so that events are grouped by node id - "select": `SELECT seq, node_id, type, source, target FROM data_changes WHERE seq > ? and (source LIKE ? or target LIKE ?) ORDER BY node_id,seq`, - "lastSeq": `SELECT MAX(seq) FROM data_changes`, - "nodeById": `SELECT node_id FROM data_changes WHERE node_id = ? LIMIT 0,1`, + "select": `SELECT seq, node_id, type, source, target FROM data_changes WHERE seq > ? and (source LIKE ? or target LIKE ?) ORDER BY node_id,seq`, + "selectFromArchive": `SELECT seq, node_id, type, source, target FROM data_changes_archive WHERE seq > ? and (source LIKE ? or target LIKE ?) ORDER BY node_id,seq`, + "firstSeq": `SELECT MIN(seq) FROM data_changes`, + "lastSeq": `SELECT MAX(seq) FROM data_changes`, + "nodeById": `SELECT node_id FROM data_changes WHERE node_id = ? LIMIT 0,1`, + "delete": `DELETE FROM data_changes where seq <= ?`, + "archive": `INSERT INTO data_changes_archive (seq, node_id, type, source, target) SELECT seq, node_id, type, source, target from data_changes where seq <= ?`, } ) @@ -137,6 +141,43 @@ func (h *sqlimpl) Get(seq uint64, prefix string) (chan *tree.SyncChange, error) defer close(res) p := fmt.Sprintf("%s%%", prefix) + + // Checking if we need to retrieve from archive + if first, err := h.FirstSeq(); err != nil || first > seq { + + rarch, err := h.GetStmt("selectFromArchive").Query(seq, p, p) + if err != nil { + return + } + + defer rarch.Close() + for rarch.Next() { + + row := new(tree.SyncChange) + var stringType string + rarch.Scan( + &row.Seq, + &row.NodeId, + &stringType, + &row.Source, + &row.Target, + ) + + if row.Source == "" { + row.Source = "NULL" + } + if row.Target == "" { + row.Target = "NULL" + } + row.Type = tree.SyncChange_Type(tree.SyncChange_Type_value[stringType]) + log.Logger(context.Background()).Debug("[Grpc Changes] Reading Row", + zap.String("type", stringType), + zap.Any("r", row), + zap.Any("intType", tree.SyncChange_Type_value[stringType])) + res <- row + } + } + r, err := h.GetStmt("select").Query(seq, p, p) if err != nil { return @@ -153,6 +194,7 @@ func (h *sqlimpl) Get(seq uint64, prefix string) (chan *tree.SyncChange, error) &row.Source, &row.Target, ) + if row.Source == "" { row.Source = "NULL" } @@ -171,9 +213,64 @@ func (h *sqlimpl) Get(seq uint64, prefix string) (chan *tree.SyncChange, error) return res, nil } +// FirstSeq returns the first sequence id in the data changes table (without archive) +func (h *sqlimpl) FirstSeq() (uint64, error) { + var last uint64 + row := h.GetStmt("firstSeq").QueryRow() + err := row.Scan(&last) + return last, err +} + +// LastSeq returns the last sequence id in the data changes table func (h *sqlimpl) LastSeq() (uint64, error) { var last uint64 row := h.GetStmt("lastSeq").QueryRow() err := row.Scan(&last) return last, err } + +// Archive places all rows before the sequence id in an archive table and delete them from the main table +func (h *sqlimpl) Archive(seq uint64) error { + + db := h.DB() + + // Starting a transaction + tx, err := db.BeginTx(context.Background(), nil) + if err != nil { + return err + } + + // Checking transaction went fine + defer func() { + if err != nil { + tx.Rollback() + } else { + tx.Commit() + } + }() + + archive := h.GetStmt("archive") + delete := h.GetStmt("delete") + + if stmt := tx.Stmt(archive); stmt != nil { + defer stmt.Close() + + if _, err = stmt.Exec(seq); err != nil { + return err + } + } else { + return fmt.Errorf("Empty statement") + } + + if stmt := tx.Stmt(delete); stmt != nil { + defer stmt.Close() + + if _, err = stmt.Exec(seq); err != nil { + return err + } + } else { + return fmt.Errorf("Empty statement") + } + + return nil +} diff --git a/main.go b/main.go index c35c8503f9..7aa7179272 100644 --- a/main.go +++ b/main.go @@ -41,6 +41,8 @@ import ( _ "github.com/pydio/cells/frontend/front-srv/rest" _ "github.com/pydio/cells/frontend/front-srv/web" + _ "github.com/pydio/cells/data/changes/grpc" + _ "github.com/pydio/cells/data/changes/rest" _ "github.com/pydio/cells/data/docstore/grpc" _ "github.com/pydio/cells/data/docstore/rest" _ "github.com/pydio/cells/data/key/grpc" @@ -52,8 +54,6 @@ import ( _ "github.com/pydio/cells/data/tree/grpc" _ "github.com/pydio/cells/data/tree/rest" _ "github.com/pydio/cells/data/versions/grpc" - _ "github.com/pydio/cells/data/changes/grpc" - _ "github.com/pydio/cells/data/changes/rest" _ "github.com/pydio/cells/discovery/config/grpc" _ "github.com/pydio/cells/discovery/config/rest" @@ -93,6 +93,7 @@ import ( // All Actions for scheduler _ "github.com/pydio/cells/broker/activity/actions" _ "github.com/pydio/cells/scheduler/actions/archive" + _ "github.com/pydio/cells/scheduler/actions/changes" _ "github.com/pydio/cells/scheduler/actions/cmd" _ "github.com/pydio/cells/scheduler/actions/images" _ "github.com/pydio/cells/scheduler/actions/scheduler" diff --git a/scheduler/actions/changes/archive.go b/scheduler/actions/changes/archive.go new file mode 100644 index 0000000000..a3c5ec76ad --- /dev/null +++ b/scheduler/actions/changes/archive.go @@ -0,0 +1,94 @@ +/* + * Copyright (c) 2018. Abstrium SAS + * This file is part of Pydio Cells. + * + * Pydio Cells is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Pydio Cells is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with Pydio Cells. If not, see . + * + * The latest code can be found at . + */ + +package archive + +import ( + "context" + "strconv" + + "github.com/golang/protobuf/ptypes" + "github.com/golang/protobuf/ptypes/any" + "github.com/micro/go-micro/client" + + "github.com/pydio/cells/common" + "github.com/pydio/cells/common/proto/jobs" + "github.com/pydio/cells/common/registry" + service "github.com/pydio/cells/common/service/proto" + "github.com/pydio/cells/scheduler/actions" +) + +var ( + archiveActionName = "actions.changes.archive" +) + +// ArchiveAction implements archiving. +type ArchiveAction struct { + RemainingRows uint64 +} + +// GetName returns this action unique identifier +func (a *ArchiveAction) GetName() string { + return archiveActionName +} + +// Init passes parameters to the action +func (a *ArchiveAction) Init(job *jobs.Job, cl client.Client, action *jobs.Action) error { + if remainingRows, ok := action.Parameters["remainingRows"]; ok { + u, err := strconv.ParseUint(remainingRows, 10, 64) + if err != nil { + return err + } + + a.RemainingRows = u + } else { + a.RemainingRows = 1 + } + return nil +} + +// Run the actual action code +func (a *ArchiveAction) Run(ctx context.Context, channels *actions.RunnableChannels, input jobs.ActionMessage) (jobs.ActionMessage, error) { + + c := service.NewArchiverClient(registry.GetClient(common.SERVICE_CHANGES)) + + query, err := ptypes.MarshalAny(&service.ChangesArchiveQuery{RemainingRows: a.RemainingRows}) + if err != nil { + return input.WithError(err), err + } + + req := &service.Query{ + SubQueries: []*any.Any{query}, + } + + resp, err := c.Archive(ctx, req) + if err != nil { + return input.WithError(err), err + } + + if resp.GetOK() { + input.AppendOutput(&jobs.ActionOutput{ + Success: true, + StringBody: "Archived latest changes", + }) + } + + return input, nil +} diff --git a/scheduler/actions/changes/init.go b/scheduler/actions/changes/init.go new file mode 100644 index 0000000000..5c14f94256 --- /dev/null +++ b/scheduler/actions/changes/init.go @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2018. Abstrium SAS + * This file is part of Pydio Cells. + * + * Pydio Cells is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Pydio Cells is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with Pydio Cells. If not, see . + * + * The latest code can be found at . + */ + +// Package archive provides implementation of actions to work with archive files. +package archive + +import ( + "github.com/pydio/cells/scheduler/actions" +) + +func init() { + + manager := actions.GetActionsManager() + + manager.Register(archiveActionName, func() actions.ConcreteAction { + return &ArchiveAction{} + }) +} diff --git a/scheduler/jobs/grpc/defaults.go b/scheduler/jobs/grpc/defaults.go index 126fe2d846..4949cdf985 100644 --- a/scheduler/jobs/grpc/defaults.go +++ b/scheduler/jobs/grpc/defaults.go @@ -108,6 +108,24 @@ func getDefaultJobs() []*jobs.Job { }, } + archiveChangesJob := &jobs.Job{ + ID: "archive-changes-job", + Owner: common.PYDIO_SYSTEM_USERNAME, + Label: "Jobs.Default.ArchiveJobs", + MaxConcurrency: 1, + Schedule: &jobs.Schedule{ + Iso8601Schedule: "R/2012-06-04T19:25:16.828696-07:03/PT10M", + }, + Actions: []*jobs.Action{ + { + ID: "actions.changes.archive", + Parameters: map[string]string{ + "remainingRows": "1000", + }, + }, + }, + } + fakeLongJob := &jobs.Job{ ID: "fake-long-job", Owner: common.PYDIO_SYSTEM_USERNAME, @@ -163,6 +181,7 @@ func getDefaultJobs() []*jobs.Job { thumbnailsJob, cleanThumbsJob, stuckTasksJob, + archiveChangesJob, // Testing Jobs fakeLongJob, fakeRPCJob, diff --git a/tools/docker/docker-compose.yaml b/tools/docker/docker-compose.yaml index ddb9c33fc4..2a53450b7b 100644 --- a/tools/docker/docker-compose.yaml +++ b/tools/docker/docker-compose.yaml @@ -10,6 +10,7 @@ services: environment: - CELLS_BIND=localhost:8080 - CELLS_EXTERNAL=localhost:8080 + - CELLS_NO_SSL=1 # MySQL image with a default database cells and a dedicated user pydio mysql: From dde4ce4cbdfbf7713a1b43c491b98282f099dc7d Mon Sep 17 00:00:00 2001 From: cdujeu Date: Fri, 22 Jun 2018 15:29:40 +0200 Subject: [PATCH 5/8] Never store pydio:* metadata namespaces. (cherry picked from commit 91c2f93) --- data/meta/grpc/handler.go | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/data/meta/grpc/handler.go b/data/meta/grpc/handler.go index a094632101..f0b75bddf5 100644 --- a/data/meta/grpc/handler.go +++ b/data/meta/grpc/handler.go @@ -193,7 +193,7 @@ func (s *MetaServer) CreateNode(ctx context.Context, req *tree.CreateNodeRequest author = claims.Name } - if err := dao.SetMetadata(req.Node.Uuid, author, req.Node.MetaStore); err != nil { + if err := dao.SetMetadata(req.Node.Uuid, author, s.filterMetaToStore(ctx, req.Node.MetaStore)); err != nil { resp.Success = false } @@ -221,7 +221,7 @@ func (s *MetaServer) UpdateNode(ctx context.Context, req *tree.UpdateNodeRequest author = claims.Name } - if err := dao.SetMetadata(req.To.Uuid, author, req.To.MetaStore); err != nil { + if err := dao.SetMetadata(req.To.Uuid, author, s.filterMetaToStore(ctx, req.To.MetaStore)); err != nil { log.Logger(ctx).Error("failed to update meta node", zap.Any("error", err)) resp.Success = false return err @@ -279,3 +279,17 @@ func (s *MetaServer) Search(ctx context.Context, request *tree.SearchRequest, re result.Close() return nil } + +func (s *MetaServer) filterMetaToStore(ctx context.Context, metaStore map[string]string) map[string]string { + + filtered := make(map[string]string) + for k, v := range metaStore { + if k == common.META_NAMESPACE_DATASOURCE_NAME || k == common.META_NAMESPACE_DATASOURCE_PATH { + continue + } + filtered[k] = v + } + + return filtered + +} From 1d17de49139001745048db09b4039420af28c936 Mon Sep 17 00:00:00 2001 From: Bruno Sinou Date: Tue, 26 Jun 2018 10:36:31 +0200 Subject: [PATCH 6/8] Add missing result count --- idm/workspace/rest/rest.go | 1 + 1 file changed, 1 insertion(+) diff --git a/idm/workspace/rest/rest.go b/idm/workspace/rest/rest.go index 7672323c82..26596a757a 100644 --- a/idm/workspace/rest/rest.go +++ b/idm/workspace/rest/rest.go @@ -200,6 +200,7 @@ func (h *WorkspaceHandler) SearchWorkspaces(req *restful.Request, rsp *restful.R } resp.Workspace.PoliciesContextEditable = h.IsContextEditable(ctx, resp.Workspace.UUID, resp.Workspace.Policies) collection.Workspaces = append(collection.Workspaces, resp.Workspace) + collection.Total++ } rsp.WriteEntity(collection) From f3b35c4b0dded9c9b043f7be4200db56db6ce210 Mon Sep 17 00:00:00 2001 From: Greg Hecquet Date: Wed, 27 Jun 2018 15:55:19 +0200 Subject: [PATCH 7/8] Dockerfile --- tools/docker/cells/dockerfile | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tools/docker/cells/dockerfile b/tools/docker/cells/dockerfile index 5b4d09c7d5..fc0d204850 100755 --- a/tools/docker/cells/dockerfile +++ b/tools/docker/cells/dockerfile @@ -5,14 +5,20 @@ ENV CELLS_VERSION 1.0.0 WORKDIR /cells RUN wget "https://download.pydio.com/pub/cells/release/${CELLS_VERSION}/linux-amd64/cells" +RUN wget "https://download.pydio.com/pub/cells/release/${CELLS_VERSION}/linux-amd64/cells-ctl" +RUN wget "https://download.pydio.com/pub/cells-sdk-go/nightly/latest/linux-amd64/cells-sdk-go" COPY docker-entrypoint.sh /cells/docker-entrypoint.sh COPY libdl.so.2 /cells/libdl.so.2 RUN chmod +x /cells/cells +RUN chmod +x /cells/cells-ctl +RUN chmod +x /cells/cells-sdk-go RUN chmod +x /cells/docker-entrypoint.sh RUN ln -s /cells/cells /bin/cells +RUN ln -s /cells/cells-ctl /bin/cells-ctl +RUN ln -s /cells/cells-sdk-go /bin/cells-sdk-go RUN ln -s /cells/libdl.so.2 /lib64/libdl.so.2 RUN ln -s /cells/docker-entrypoint.sh /bin/docker-entrypoint.sh From 2dcac28e5f47f28e07223080f4421ab7d250c93d Mon Sep 17 00:00:00 2001 From: cdujeu Date: Tue, 3 Jul 2018 09:17:28 +0200 Subject: [PATCH 8/8] Migrate data/meta : remove pydio:meta-source-% stored metadata (should never be stored). --- data/meta/migrations/mysql/0.2.sql | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 data/meta/migrations/mysql/0.2.sql diff --git a/data/meta/migrations/mysql/0.2.sql b/data/meta/migrations/mysql/0.2.sql new file mode 100644 index 0000000000..6bbbd42d99 --- /dev/null +++ b/data/meta/migrations/mysql/0.2.sql @@ -0,0 +1,2 @@ +-- +migrate Up +DELETE FROM data_meta WHERE namespace LIKE 'pydio:meta-data-source-%'; \ No newline at end of file